program crashes when encrypting system files - c

Question

I wrote this code to encrypt and decrypt the contents of folder, both function works well when encrypting normal files but when im changing the folder to system folder, program crashes and when i check the latest file before crashing, im not able to open some of them (File is Open in Another Program), in some of them i just can't make changes.
i'm handling all errors i think, but it still keep crashing when it reachs to opened file by another program, how to solve this problem to ignore these types of files and keep continue instead of crashing?
and the structure is not important to post i think.
char ListFiles(const wchar_t* folder, CIPHER* conf)
{
wchar_t wildcard[MAX_PATH + 1];
swprintf(wildcard, sizeof(wildcard) / sizeof(*wildcard), L"%s\\*", folder);
WIN32_FIND_DATAW fd;
HANDLE handle = FindFirstFileW(wildcard, &fd);
if (handle == INVALID_HANDLE_VALUE) return 1;
do
{
if (wcscmp(fd.cFileName, L".") == 0 || wcscmp(fd.cFileName, L"..") == 0)
continue;
wchar_t path[MAX_PATH + 1];
swprintf(path, sizeof(path) / sizeof(*path), L"%s\\%s", folder, fd.cFileName);
if ((fd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) && !(fd.dwFileAttributes & FILE_ATTRIBUTE_SYSTEM))
ListFiles(path, &conf);
if (fd.dwFileAttributes & FILE_ATTRIBUTE_ARCHIVE && !(fd.dwFileAttributes & FILE_ATTRIBUTE_SYSTEM))
{
wprintf(L"%s\n", path);
FILE* f_dec;
FILE* f_input = _wfopen(path, L"rb");
FILE* f_enc = _wfopen(wcscat(path, L".encrypted"), L"wb");
if (!f_input || !f_enc) {
fprintf(stderr, "fopen error: %s\n", strerror(errno));
continue;
}
conf->encrypt = 1; // encryption
if (AES_L(conf, f_input, f_enc) != 0)
continue;
f_enc = _wfopen(path, L"rb");
f_dec = _wfopen(wcscat(path, L".decrypted"), L"wb");
if (!f_dec || !f_enc) {
fprintf(stderr, "ERROR: fopen error: %s\n", strerror(errno));
continue;
}
conf->encrypt = 0; // decryption
if (AES_L(conf, f_enc, f_dec) != 0)
continue;
puts("\n\n");
}
} while (FindNextFileW(handle, &fd));
FindClose(handle);
return 0;
}
char AES_L(CIPHER* params, FILE* ifp, FILE* ofp)
{
unsigned int inlen, outlen;
unsigned char* inbuf = (unsigned char*)malloc(params->bufsize);
unsigned char* outbuf = (unsigned char*)malloc(params->bufsize + EVP_MAX_BLOCK_LENGTH);
if (inbuf == NULL || outbuf == NULL)
{
printf("memory cannot be allocated\n");
cleanup(ifp, ofp, inbuf, outbuf);
return 1;
}
EVP_CIPHER_CTX* ctx;
ctx = EVP_CIPHER_CTX_new();
if (ctx == NULL) {
fprintf(stderr, "ERROR: EVP_CIPHER_CTX_new failed. OpenSSL error: %s\n",
ERR_error_string(ERR_get_error(), NULL));
cleanup(ifp, ofp, inbuf, outbuf);
return 1;
}
if (!EVP_CipherInit_ex(ctx, params->cipher_type, NULL, params->key, params->iv, params->encrypt)) {
fprintf(stderr, "ERROR: EVP_CipherInit_ex failed. OpenSSL error: %s\n",
ERR_error_string(ERR_get_error(), NULL));
EVP_CIPHER_CTX_cleanup(ctx);
cleanup(ifp, ofp, inbuf, outbuf);
return 1;
}
while (1) {
// Read in data in blocks until EOF. Update the ciphering with each read.
inlen = fread(inbuf, sizeof(*inbuf), params->bufsize, ifp);
if (ferror(ifp)) {
fprintf(stderr, "ERROR: fread error: %s\n", strerror(errno));
EVP_CIPHER_CTX_cleanup(ctx);
cleanup(ifp, ofp, inbuf, outbuf, errno);
return 1;
}
if (!EVP_CipherUpdate(ctx, outbuf, &outlen, inbuf, inlen)) {
fprintf(stderr, "ERROR: EVP_CipherUpdate failed. OpenSSL error: %s\n",
ERR_error_string(ERR_get_error(), NULL));
EVP_CIPHER_CTX_cleanup(ctx);
cleanup(ifp, ofp, inbuf, outbuf);
return 1;
}
fwrite(outbuf, sizeof(*outbuf), outlen, ofp);
if (ferror(ofp)) {
fprintf(stderr, "ERROR: fwrite error: %s\n", strerror(errno));
EVP_CIPHER_CTX_cleanup(ctx);
cleanup(ifp, ofp, inbuf, outbuf, errno);
return 1;
}
if (inlen < params->bufsize) /* Reached End of file */
break;
}
/* Now cipher the final block and write it out to file */
if (!EVP_CipherFinal_ex(ctx, outbuf, &outlen)) {
fprintf(stderr, "ERROR: EVP_CipherFinal_ex failed. OpenSSL error: %s\n",
ERR_error_string(ERR_get_error(), NULL));
EVP_CIPHER_CTX_cleanup(ctx);
cleanup(ifp, ofp, inbuf, outbuf);
return 1;
}
fwrite(outbuf, sizeof(*outbuf), outlen, ofp);
if (ferror(ofp)) {
fprintf(stderr, "ERROR: fwrite error: %s\n", strerror(errno));
EVP_CIPHER_CTX_cleanup(ctx);
cleanup(ifp, ofp, inbuf, outbuf);
return 1;
}
EVP_CIPHER_CTX_cleanup(ctx);
cleanup(ifp, ofp, inbuf, outbuf);
return 0;
}
update:
void cleanup(FILE* ifp, FILE* ofp, unsigned char* inputBuf, unsigned char* outputBuf)
{
free(inputBuf);
free(outputBuf);
fclose(ifp);
fclose(ofp);
}
typedef struct {
unsigned int key_size;
unsigned int block_size;
unsigned int bufsize;
unsigned char* key;
unsigned char* iv;
unsigned int encrypt;
const EVP_CIPHER* cipher_type;
} CIPHER;

Answer 1

I see some problems with your code, although it's hard to know if they can lead to a crash without the rest of the code (for instance, we can't see the cleanup method), or how you create and initialize params.
The first problem is that you may be leaking file handles. When you open the files in ListFiles you open them in pairs, then check if any of them is NULL and if one is, you go on with the loop.
FILE* f_input = _wfopen(path, L"rb");
FILE* f_enc = _wfopen(wcscat(path, L".encrypted"), L"wb");
if (!f_input || !f_enc) {
fprintf(stderr, "fopen error: %s\n", strerror(errno));
continue;
}
What if f_input is opened correctly but f_enc fails? The source file would remain open until the program ends. You should check each of them separately.
A similar problem arises when you allocate memory in AES_L:
unsigned char* inbuf = (unsigned char*)malloc(params->bufsize);
unsigned char* outbuf = (unsigned char*)malloc(params->bufsize + EVP_MAX_BLOCK_LENGTH);
if (inbuf == NULL || outbuf == NULL)
{
printf("memory cannot be allocated\n");
return 1;
}
If one of the buffers (probably inbuf) is allocated correctly but outbuf fails, you would be leaking memory because you don't free the buffer that was allocated correctly (although, to be honest, I don't think this is directly the problem with the crash because this would happen in situations when you are already very low on memory).
Another problem may appear when you create the path for the encrypted and decrypted files. The path buffer is of size MAX_PATH+1, which is enough for the original file name, but then you perform a couple of wcscat operations that lead additional data to be added to the path. What if the original file name was already almost in the MAX_PATH limit? When you performed the wcscat you would be overflowing a buffer in the stack, which may also lead to a crash.
And finally, ListFiles is recursive, so if there are many nested calls you may be running out of stack, which would also lead to a crash (in fact, from the problems I mention I think it's the main suspect). I would make it iterative.
Anyway, it's very difficult to know if the crash is due to these problems, and your best option is running it in the debugger. The error message from the crash will tell you a lot of information to identify the reason.