Encryption and decryption error 0x0407106B using OpenSSL - c

Question

I'm writing a routine in C that reads a base64 string with the public key and proceeds to encrypt a string. I also test the same string's decryption but I'm getting error 0x0407106B when trying to do the decoding:
$ openssl errstr 0x0407106B
error:0407106B:rsa routines:RSA_padding_check_PKCS1_type_2:block type is not 02
Here's the code
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <ctype.h>
#include <openssl/rsa.h>
#include <openssl/engine.h>
//#define PADDING RSA_PKCS1_OAEP_PADDING
#define PADDING RSA_PKCS1_PADDING
//#define PADDING RSA_NO_PADDING
main() {
// public key
char *b64_pKey = "-----BEGIN PUBLIC KEY-----\nMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCp2w+8HUdECo8V5yuKYrWJmUbL\ntD6nSyVifN543axXvNSFzQfWNOGVkMsCo6W4hpl5eHv1p9Hqdcf/ZYQDWCK726u6\nhsZA81AblAOOXKaUaxvFC+ZKRJf+MtUGnv0v7CrGoblm1mMC/OQI1JfSsYi68Epn\naOLepTZw+GLTnusQgwIDAQAB\n-----END PUBLIC KEY-----\n";
// private key
char *b64priv_key = "-----BEGIN RSA PRIVATE KEY-----\nMIICXAIBAAKBgQCp2w+8HUdECo8V5yuKYrWJmUbLtD6nSyVifN543axXvNSFzQfW\nNOGVkMsCo6W4hpl5eHv1p9Hqdcf/ZYQDWCK726u6hsZA81AblAOOXKaUaxvFC+ZK\nRJf+MtUGnv0v7CrGoblm1mMC/OQI1JfSsYi68EpnaOLepTZw+GLTnusQgwIDAQAB\nAoGBAKDuq3PikblH/9YS11AgwjwC++7ZcltzeZJdGTSPY1El2n6Dip9ML0hUjeSM\nROIWtac/nsNcJCnvOnUjK/c3NIAaGJcfRPiH/S0Ga6ROiDfFj2UXAmk/v4wRRUzr\n5lsA0jgEt5qcq2Xr/JPQVGB4wUgL/yQK0dDhW0EdrJ707e3BAkEA1aIHbmcVfCP8\nY/uWuK0lvWxrIWfR5MlHhI8tD9lvkot2kyXiV+jB6/gktwk1QaFsy7dCXn7w03+k\nxrjEGGN+kQJBAMuKf55lDtU9K2Js3YSStTZAXP+Hz7XpoLxmbWFyGvBx806WjgAD\n624irwS+0tBxkERbRcisfb2cXmAx8earT9MCQDZuVCpjBWxd1t66qYpgQ29iAmG+\njBIY3qn9uOOC6RSTiCCx1FvFqDMxRFmGdRVFxeyZwsVE3qNksF0Zko0MPKECQCEe\noDV97DP2iCCz5je0R5hUUM2jo8DOC0GcyR+aGZgWcqjPBrwp5x08t43mHxeb4wW8\ndFZ6+trnntO4TMxkA9ECQB+yCPgO1zisJWYuD46KISoesYhwHe5C1BQElQgi9bio\nU39fFo88w1pok23a2CZBEXguSvCvexeB68OggdDXvy0=\n-----END RSA PRIVATE KEY-----\n";
// String to encrypt
char *str = "1234";
ERR_load_crypto_strings();
BIO *bpo = BIO_new_mem_buf(b64_pKey, -1);
RSA *pubKey = PEM_read_bio_RSA_PUBKEY(bpo, NULL, NULL, NULL);
if ( !pubKey ) {
printf("%s\n", ERR_error_string(ERR_get_error(), NULL));
return;
}
int rsa_length = RSA_size(pubKey);
BIO *b64 = NULL;
BIO *bmem = NULL;
BUF_MEM *bptr = NULL;
unsigned char encrypted[2560] = { 0 };
unsigned char retencrypted[2560] = { 0 };
int resultEncrypt = RSA_public_encrypt(PADDING, str, encrypted, pubKey, PADDING);
if ( resultEncrypt == -1 ) {
printf("%s\n", ERR_error_string(ERR_get_error(), NULL));
return;
}
/*
* Show base 64 encrypted string
*/
b64 = BIO_new((BIO_METHOD *)BIO_f_base64());
BIO_set_flags(b64,BIO_FLAGS_BASE64_NO_NL);
bmem = BIO_new(BIO_s_mem());
b64 = BIO_push(b64, bmem);
BIO_write(b64, encrypted, resultEncrypt);
BIO_flush(b64);
BIO_get_mem_ptr(b64, &bptr);
memcpy(retencrypted, bptr->data, bptr->length);
BIO_free(b64);
BIO_free(bpo);
RSA_free(pubKey);
printf("Encrypted string:%s\n",retencrypted);
/*
* Now decrypt this very string with the private key
*/
BIO *bpop = BIO_new_mem_buf(b64priv_key, -1);
RSA *privKey = PEM_read_bio_RSAPrivateKey(bpop, NULL, NULL, NULL);
if ( !privKey ) {
printf("%s\n", ERR_error_string(ERR_get_error(), NULL));
return;
}
rsa_length = RSA_size(privKey);
unsigned char decrypted[2560] = { 0 };
int resultDecrypt = RSA_private_decrypt( RSA_size(privKey), retencrypted, decrypted, privKey, PADDING);
if ( resultDecrypt == -1 ) {
printf("%s\n", ERR_error_string(ERR_get_error(), NULL));
return;
}
printf("resultDecrypt=%d\ndecrypted string: %s\n",resultDecrypt,decrypted);
BIO_free(bpop);
RSA_free(privKey);
ERR_free_strings();
}
Note: I exported the private key using
openssl rsa -in rsa_privatekey.pem -check
and the public key:
openssl rsa -in rsa_privatekey.pem -pubout
Why am I getting the error?

Answer 1

The problem is that you're trying to decrypt the base64 encoded result.
You should try to decrypt the result of the encryption.
That is, instead of:
int resultDecrypt = RSA_private_decrypt( RSA_size(privKey), retencrypted, decrypted, privKey, PADDING);
You should call:
int resultDecrypt = RSA_private_decrypt( RSA_size(privKey), encrypted, decrypted, privKey, PADDING);
Also, there is a problem in the encryption call:
int resultEncrypt = RSA_public_encrypt(PADDING, str, encrypted, pubKey, PADDING);
Why are you passing PADDING as flen? This should be the length of the string (i.e. 4 or 5, depending on whether you want to encrypt the null character).
If you want to write the encrypted string as ASCII (encoded using base64), that's fine. But you have to decode it back before you decrypt it.

Answer 2

The error you're getting is block type is not 02.
Although Omri is correct that you're passing the wrong data, and you are only going to encrypt 1 byte, the error is because the sizeof( encrypted ) is way too large (2560). In other words your data receiver for RSA_public_encrypt must be a regular unsigned char* pointer, not an unsigned char[2560].
Where you have
unsigned char encrypted[2560] = { 0 }; //X 2560?? RSA_public_encrypt fails.
You should be using
unsigned char *encrypted = (unsigned char*)malloc( rsa_length ) ;
RSA_public_encrypt( DATALEN, (const unsigned char*)str, encrypted, pubKey, PADDING ) ;
Notice the error Omri pointed out, that you used PADDING as the first arg to RSA_public_encrypt, while it should be the DATALEN data length.
If you fix that you'll get a similar error later with the private key decrypt. Fix it and you're on your way.