RSA Encryption C code - c

Question

I am having trouble doing my homework.
I am doing an RSA application which can encrypt and decrypt.
The problem is that after I Input things to encrypt the results are weird and I can't decrypt anything. This is because when I copied the results of encryption which are symbols, I got more weird stuffs.
I'm guessing it has something to do with my formula getting negative ASCIIs as results.
Below is what I tried, and, in order to understand what I meant by weird, just compile and try it out(I have some unused stuffs which I haven't removed yet):
Output:
Code:
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <ctype.h>
#include <math.h>
#define boolean int
#define true 1
#define false 0
//===================================================//
int p = 0;
int q = 0;
int n = 0;
int m = 0;
int divider = 2;
int tempdivider = 2;
int initial = 0;
int x = 0;
int y = 0;
char msg[100];
char alphabet[27];
//===================================================//
void cls();
void menu();
void init();
void reinit();
void inputencrypt();
//int encrypt(int num);
void encrypt();
char decrypt(char text[]);
int fpb(int num);
int d(int num);
int primecheck(int a);
boolean checkdigit(char text[]);
//===================================================//
int main() {
frontpage();
init();
menu();
getchar();
return 0;
}
//===================================================//
void cls() {
for (int i = 0;i < 25;i++) {
printf("\n");
}
}
//===================================================//
boolean checkdigit(char text[]) {
int len = strlen(text);
for (int i = 0;i < len;++i) {
if (text[i]<'0' || text[i]>'9') {
return false;
}
}
return true;
}
int primecheck(int a) {
if (a == 1) {
return false;
}
for (int i = 2;i < a;i++) {
if (a%i == 0) {
return false;
}
}
return true;
}
//===================================================//
void reinit() {
for (int i = 1;i < 27;i++) {
alphabet[i] = 'a' + i - 1;
}
p = 0;
q = 0;
n = 0;
m = 0;
divider = 2;
tempdivider = 2;
initial = 120;
x = 0;
y = 0;
}
void init() {
reinit();
do {
printf("p = ");
scanf("%d", &p);fflush(stdin);
if (!primecheck(p)) {
printf("must be prime number! \n");
}
} while (!primecheck(p));
do {
printf("q = ");
scanf("%d", &q);fflush(stdin);
if (!primecheck(q)) {
printf("must be prime number! \n");
}
} while (!primecheck(q));
n = p*q;
m = (p - 1)*(q - 1);
initial = m;
x = fpb(m);
y = d(m);
printf("n = %d\n", n);
printf("m = %d\n", m);
printf("e = %d\n", x);
printf("d = %d\n", y);
system("pause");
}
//===================================================//
void menu() {
char input[2];
int input1 = 0;
do {
do {
cls();
printf("main menu\n");
printf("================\n");
printf("1. encrypt\n");
printf("2. decrypt\n");
printf("3. exit\n");
printf(">> ");
scanf("%s", input);fflush(stdin);
if (checkdigit(input)) {
input1 = atoi(input);
}
} while (!checkdigit(input));
switch (input1) {
case 1:
int c;
char encrypted[100];
char word[100];
printf("input word to encrypt : ");
scanf(" %[^\n]", word);fflush(stdin);
for (int i = 0;i < strlen(word);i++) {
if (word[i] == ' ') {
encrypted[i] = ' ';
//i++;
}
else {
for (int j = 1;j < 27;j++) {
if (word[i] == alphabet[j]) {
c = 0;
c = pow(j, x);
c = c%n;
encrypted[i] = c;
break;
}
}
}
}
printf("\n\nWord ASCII [ ");
for (int i = 0;i < strlen(word);i++) {
//printf("%d", c);
printf("%d ", word[i]);
}
printf(" ]\n");
printf("\n\nEncrypted ASCII [ ");
for (int i = 0;i < strlen(word);i++) {
//printf("%d", c);
printf("%d ", encrypted[i]);
}
printf(" ]\n");
printf("\n\nEncrypted [ ");
for (int i = 0;i < strlen(word);i++) {
//printf("%d", c);
printf("%c", encrypted[i]);
}
printf(" ]");
printf("\n\n\n");
system("pause");
break;
case 2:
int temp[100];
char decrypted[100];
char wordx[100];
int h;
printf("input word to decrypt : ");
scanf(" %[^\n]", wordx);fflush(stdin);
for (int i = 0;i < strlen(wordx);i++) {
temp[i] = wordx[i];
//temp[i] -= 97;
//printf("%d ::: %c\n", temp[i], temp[i]);
}
for (int i = 0;i < strlen(wordx);i++) {
if (wordx[i] == ' ') {
decrypted[i] = ' ';
}
else {
h = 0;
h = pow(temp[i], y);
h = h%n;
decrypted[i] = h;
for (int j = 1;j < 27;j++) {
if (decrypted[i] == j) {
decrypted[i] = alphabet[j];
}
}
}
}
printf("\n\nWord ASCII [ ");
for (int i = 0;i < strlen(wordx);i++) {
//printf("%d", c);
printf("%d ", wordx[i]);
}
printf(" ]\n");
printf("\n\nDecrypted ASCII [ ");
for (int i = 0;i < strlen(wordx);i++) {
//printf("%d", c);
printf("%d ", decrypted[i]);
}
printf(" ]\n");
printf("\n\nDecrypted [ ");
for (int i = 0;i < strlen(wordx);i++) {
//printf("%d", decrypted[i]);
printf("%c", decrypted[i]);
}
printf(" ]");
printf("\n\n\n");
system("pause");
break;
}
} while (input1 != 3);
}
//===================================================//
int fpb(int num) {
if (!primecheck(num)) {
if (num%divider == 0) {
num = num / divider;
divider = 2;
}
else {
divider++;
}
fpb(num);
}
else if (primecheck(num)) {
if (!primecheck(num + divider)) {
tempdivider++;
divider = tempdivider;
num = initial;
fpb(num);
}
else {
return num + divider;
}
}
}
int d(int num) {
for (int i = 1;i < num;i++) {
if ((x*i) % num == 1) {
return i;
}
}
}

Answer 1

You have a general comprehension problem. Your console is only able to represent 96 characters correctly (known as printable 7bit-ASCII characters, 0x20 to 0x7F), but a byte can hold 255 different values. Your encryption algorithm does not care about this limited range, it will encrypt any value in the range [0..255] into another value in the range [0..255]. So your ASCII input characters will most likely be encrypted into values that cannot be represented by your console correctly. Copy&Past will not work correctly for non-printable characters (like 0x0B, which is a tab).
But now you will wonder: Why does that work for e.g. E-Mails? Simply: Because those characters are converted into an ASCII representation. Please google a bit for Base64 encoding.
As an alternative, you can always stream your encrypted characters into a file and later read back from that. This way you will bypass the limitations of your console.
Btw: Please have a look at the documentation of printf() and you will know, why you get those negative values.

Answer 2

The encrypt option has these three statements consecutively
c = 0;
c = pow(j, x);
c = c%n;
and the last of those will leave c in the range 0..(n-1).
Apart from that, there is no else clause and int c; can remain uninitialised.
So all in all it is inevitable that when you print c values as characters you will get "weird" results.
As for negative values, char encrypted[100]; is probably signed char so any integer value in the range 128..255 assigned to that, although undefined behaviour, may possibly show up as a negative number because the signed char is promoted back to int when passed as format %d to printf.