Hash Ciphers and check with Signature in C - c

Question

I have three binary files: cipher01.bin, cipher02.bin and cipher03.bin.
Additionally I have a sign.bin and a pubkey.pem file. The task is to hash all three ciphers and compare it to the signature. Therefore I used RSA to decrypt the sign.bin with the public key from pubkey.pem.
The result looks good but none of the cipher-hashes belongs to the signature. But I know, that there is at least one cipher which belongs to the signature because it is a task from our university. Maybe I forgot something but I can't figure out what.
Here is my code so far:
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <openssl/sha.h>
#include <openssl/err.h>
#include <openssl/evp.h>
#include <openssl/pem.h>
#include <openssl/ssl.h>
#include <openssl/rsa.h>
#include <openssl/bio.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
/** converts unsigned character to readble string*/
char *pt(unsigned char *md) {
int i;
char *buf = (char*)malloc(sizeof(char)*80);
for(int i = 0; i < SHA_DIGEST_LENGTH;i++) {
sprintf(&(buf[i*2]),"%02x",md[i]);
}
return (buf);
}
/** returns error */
void err_exit(void) {
printf("%s\n",ERR_error_string(ERR_get_error(),NULL));
ERR_free_strings();
exit(EXIT_FAILURE);
}
/** reads a file */
char * readFile(char * filename,long int * filesize) {
FILE *fin;
char *buf;
if((fin=fopen(filename,"r"))==NULL) {
printf("Error opening %s.\n",filename);
exit(EXIT_FAILURE);
}
fseek(fin,0L,SEEK_END);
*filesize = ftell(fin);
rewind(fin);
if(!(buf=malloc(*filesize))) {
printf("Memory exhausted. Stop.\n");
exit(EXIT_FAILURE);
}
fread(buf,*filesize,1,fin);
fclose(fin);
return buf;
}
/** hash a file with sha1 */
char * hashBinaryFile(char * filename) {
long int filesize = 0;
EVP_MD_CTX c;
unsigned char md[SHA_DIGEST_LENGTH];
ERR_load_crypto_strings();
EVP_MD_CTX_init(&c);
/** reads files into buf */
char * buf = readFile(filename,&filesize);
if((EVP_DigestInit(&c,EVP_sha1()))==0) {
err_exit();
}
if((EVP_DigestUpdate(&c,buf,filesize))==0) {
err_exit();
}
if((EVP_DigestFinal(&c,md,NULL))==0) {
err_exit();
}
//printf("%s\n",pt(md));
EVP_MD_CTX_cleanup(&c);
free(buf);
ERR_free_strings();
return pt(md);
}
int padding = RSA_PKCS1_PADDING;
/** loads public key and creates rsa */
RSA * createRSAWithFilename(char * filename,int public) {
FILE * fp = fopen(filename,"rb");
if(fp == NULL) {
printf("Unable to open file %s \n",filename);
return NULL;
}
RSA *rsa= RSA_new() ;
if(public) {
rsa = PEM_read_RSA_PUBKEY(fp, &rsa,NULL, NULL);
} else {
rsa = PEM_read_RSAPrivateKey(fp, &rsa,NULL, NULL);
}
return rsa;
}
/** decrypt signature */
char * public_decrypt(unsigned char * enc_data,int data_len, unsigned char *decrypted) {
RSA * rsa = createRSAWithFilename("archieve/pubkey.pem",1);
int result = RSA_public_decrypt(data_len,enc_data,decrypted,rsa,padding);
return pt(decrypted);
}
int main(int argc,char *argv[]) {
/** decrypt signature */
long int encrypted_length;
long int decrypted_length;
unsigned char decrypted[4098]={};
char * encrypted = readFile("archieve/s72897-sig.bin",&encrypted_length);
char * sign = public_decrypt(encrypted,encrypted_length, decrypted);
char * cipher01 = hashBinaryFile("archieve/s72897-cipher01.bin");
char * cipher02 = hashBinaryFile("archieve/s72897-cipher02.bin");
char * cipher03 = hashBinaryFile("archieve/s72897-cipher03.bin");
if(strcmp(sign,cipher01)==0) {
printf("cipher01\n");
} else if(strcmp(sign,cipher02)==0) {
printf("cipher02\n");
} else if(strcmp(sign,cipher03)==0) {
printf("cipher03\n");
} else {
printf("No cipher matches the signature\n");
}
return 0;
}
Thanks for any kind of help.
Edit: fixed some code
Edit2: link to the *.zip https://ufile.io/tqwoh

Answer 1

You transform your files in human readable format twice:
char * public_decrypt(...)
{
return pt(decrypted);
// ^
}
int main(int argc,char *argv[])
{
char * sign = pt(public_decrypt(encrypted,encrypted_length, decrypted));
// ^
Additionally, you actually have some memory leaks: You do not free the rsa instance in public_decrypt neither do you free the strings returned (encrypted, sign, cypher0x)...
Further recommendation in for pt:
char *buf = (char*)malloc(sizeof(char) * 2 * SHA_DIGEST_LENGTH);
If you have an appropriate constant already, use it... sizeof(char) always is 1 by definition, so you could drop it...