OpenSSL RSA: Unable to encrypt/decrypt messages longer than 16 bytes - c

Question

I'm working on a simple program that uses OpenSSL to do basic RSA encryption and decryption. It is working fine for small messages (<16 bytes), but fails for anything over that. I understand that a limitation of public key cryptography is that you cannot encrypt anything longer than the key size. In my case, I'm using a 1024bit key, so I should have 128bytes to work with (maybe slightly less due to padding), correct? If so, that's not what I'm experiencing.
Here's the output from my program with 15 bytes:
Generating RSA keypair...done.
Message to encrypt: 0123456789ABCDE
16 bytes encrypted
Decrypted message: 0123456789ABCDE
And with 16 bytes:
Generating RSA keypair...done.
Message to encrypt: 0123456789ABCDEF
16 bytes encrypted
140153837057696:error:06065064:digital envelope routines:EVP_DecryptFinal_ex:bad decrypt:evp_enc.c:467:
Decrypted message: (null)
It seems that no matter what, only a total of 16 bytes are encrypted.
My encryption function (updated with fix):
unsigned char* rsa_seal(EVP_PKEY *pub_key, unsigned char *msg, size_t **enc_msg_len, unsigned char **sym_key, int *sym_key_len, unsigned char **iv) {
size_t msg_len = strlen((char*)msg);
unsigned char *encrypt = malloc(EVP_PKEY_size(pub_key));
EVP_CIPHER_CTX *ctx = malloc(sizeof(EVP_CIPHER_CTX));
EVP_CIPHER_CTX_init(ctx);
*sym_key = malloc(EVP_PKEY_size(pub_key));
*iv = malloc(EVP_MAX_IV_LENGTH);
**enc_msg_len = 0;
if(!EVP_SealInit(ctx, EVP_aes_128_cbc(), sym_key, sym_key_len, *iv, &pub_key, 1)) {
ERR_print_errors_fp(stderr);
encrypt = NULL;
goto return_free;
}
if(!EVP_SealUpdate(ctx, encrypt, (int*)*enc_msg_len, msg, (int)msg_len)) {
ERR_print_errors_fp(stderr);
encrypt = NULL;
goto return_free;
}
if(!EVP_SealFinal(ctx, encrypt, (int*)*enc_msg_len)) {
ERR_print_errors_fp(stderr);
encrypt = NULL;
goto return_free;
}
return_free:
EVP_CIPHER_CTX_cleanup(ctx);
free(ctx);
ctx = NULL;
return encrypt;
}
The corresponding decryption function (updated with fix):
char* rsa_open(EVP_PKEY *pri_key, unsigned char *enc_msg, size_t *enc_msg_len, unsigned char *sym_key, int sym_key_len, unsigned char *iv) {
size_t dec_len = 0;
unsigned char *decrypt = malloc((*enc_msg_len) + EVP_MAX_IV_LENGTH);
if(decrypt == NULL) return NULL;
EVP_CIPHER_CTX *ctx = malloc(sizeof(EVP_CIPHER_CTX));
EVP_CIPHER_CTX_init(ctx);
if(!EVP_OpenInit(ctx, EVP_aes_128_cbc(), sym_key, sym_key_len, iv, pri_key)) {
ERR_print_errors_fp(stderr);
decrypt = NULL;
goto return_free;
}
if(!EVP_OpenUpdate(ctx, decrypt, (int*)&dec_len, enc_msg, (int)*enc_msg_len)) {
ERR_print_errors_fp(stderr);
decrypt = NULL;
goto return_free;
}
if(!EVP_OpenFinal(ctx, decrypt, (int*)&dec_len)) {
ERR_print_errors_fp(stderr);
decrypt = NULL;
goto return_free;
}
decrypt[dec_len] = '\0';
return_free:
EVP_CIPHER_CTX_cleanup(ctx);
free(ctx);
ctx = NULL;
return (char*)decrypt;
}
The key generation function:
int rsa_init(EVP_PKEY **rsa_keypair) {
EVP_PKEY_CTX *ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_RSA, NULL);
if(!EVP_PKEY_keygen_init(ctx)) {
ERR_print_errors_fp(stderr);
return -1;
}
if(!EVP_PKEY_CTX_set_rsa_keygen_bits(ctx, KEY_LENGTH)) {
ERR_print_errors_fp(stderr);
return -1;
}
if(!EVP_PKEY_keygen(ctx, rsa_keypair)) {
ERR_print_errors_fp(stderr);
return -1;
}
EVP_PKEY_CTX_free(ctx);
return 0;
}
And finally, my main:
int main() {
EVP_PKEY *rsa_keypair = NULL; // RSA keypair
char msg[BUFFER]; // Message to encrypt
unsigned char *encrypt = NULL; // Encrypted message
char *decrypt = NULL; // Decrypted message
// Generate key pair
printf("Generating RSA keypair...");
if(rsa_init(&rsa_keypair) == -1) {
fprintf(stderr, "\nError generating RSA keypair.\n");
exit(1);
}
printf("done.\n");
// Get the message to encrypt
printf("Message to encrypt: ");
fgets(msg, BUFFER-1, stdin);
msg[strlen(msg)-1] = '\0';
// Load error strings in anticipation of error
ERR_load_crypto_strings();
// Encrypt the message
size_t *encrypt_len = malloc(sizeof(size_t));
unsigned char *sym_key = NULL;
unsigned char *iv = NULL;
int sym_key_len;
encrypt = rsa_seal(rsa_keypair, (unsigned char*)msg, &encrypt_len, &sym_key, &sym_key_len, &iv);
printf("%d bytes encrypted\n", (int)*encrypt_len);
// Decrypt it
decrypt = rsa_open(rsa_keypair, (unsigned char*)encrypt, (size_t*)encrypt_len, sym_key, sym_key_len, iv);
printf("Decrypted message: %s\n", decrypt);
free(encrypt);
free(decrypt);
free(encrypt_len);
free(sym_key);
free(iv);
EVP_PKEY_free(rsa_keypair);
return 0;
}
Any help is greatly appreciated! Thank you.
EDIT: As pointed out by math below, it seems that the answer to my mistake was hiding in the OpenSSL here: https://www.openssl.org/docs/crypto/EVP_EncryptInit.html#

Answer 1

This is because you do not properly handle out and outl parameters in EVP_SealUpdate(), EVP_SealFinal(), EVP_OpenUpdate() and EVP_OpenFinal().
Each EVP_XxxxUpdate() and EVP_XxxxFinal() call will contribute to the output buffer. So, you are required to keep track of the seal/open process by summing each outl returned and providing the expected buffer each time (start of buffer + already handled bytes).
unsigned char* rsa_seal(...)
{
...
**enc_msg_len = 0;
EVP_SealUpdate(ctx, encrypt + **enc_msg_len, &outl, msg, (int)msg_len);
**enc_msg_len += outl;
EVP_SealFinal(ctx, encrypt + **enc_msg_len, &outl);
**enc_msg_len += outl;
...
}
char* rsa_open(...)
{
...
dec_len = 0;
EVP_OpenUpdate(ctx, decrypt + dec_len, &outl, enc_msg, (int)*enc_msg_len);
dec_len += outl;
EVP_OpenFinal(ctx, decrypt + dec_len, &outl);
dec_len += outl;
...
}
The program was working with 15-bytes buffer because in that case, the EVP_XxxxUpdate() call is returning 0 in outl (not enough data to seal/open a block), hiding the problem in your code logic.
Note: The data is not directly encrypted using the RSA key but using a generated symetric key (AES-128 in your case). This is why the block size is 16 bytes.