Stack smashing detected - from C to assembly - c

I'm trying to translate from C to assembly two functions, one that sort an array of index and one that find the minimum number starting from an index. When I run the program it works , the new array is sorted but at the end it gives me this error
1 2 3 3 4 6 7
*** stack smashing detected ***: ./invidia terminated
Annullato (core dump creato)
this is my .c file
#include <stdio.h>
#include <stdlib.h>
int* sort(int* v, int dim);
int min(int v[], int dim, int t);
int main(){
int v[] = {1,3,2,4,6,7,3};
int dim = 7,i;
int *c = sort(v,dim);
for (i = 0; i<dim; i++)
printf("%d ", c[i]);
return 0;
}
/*int* sort(int* v, int dim){
int i,j=0,t;
int mino,temp;
for (t = 0; t<dim; t++){
for(i=t;i<dim;i++){
mino = min (v,dim,t); <- min number from index t
if (v[i]==mino){
temp = v[j];
v[j] = v[i];
v[i] = temp;
j++;
break;
}
}
}
return v;
}
int min(int v[], int dim, int t){
int i, min = 1000;
for (i=t; i<dim;i++)
if (v[i]<min)
min = v[i];
return min;
}
*/
and my .s file
.globl sort
sort: #v->ebx dim-> esi i->edi, j->ecx t->edx, mino->eax, temp ->ebp
pushl %ebx
pushl %esi
pushl %edi
pushl %ebp
subl $32, %esp
movl 52(%esp), %ebx
movl 56(%esp), %esi
movl $0, %edi
movl $0, %ecx
movl $0, %eax
movl $0, %edx
movl $0, %ebp
FOR1:
cmpl %esi, %edx
jge EXIT
movl %edx, %edi
FOR2:
cmpl %esi, %edi
jge EXIT2
movl %ebx, (%esp)
movl %esi, 4(%esp)
movl %edx, 8(%esp)
movl %ebx, 12(%esp)
movl %eax, 16(%esp)
movl %ecx, 20(%esp)
movl %edx, 24(%esp)
call min
movl 12(%esp), %ebx
movl 20(%esp), %ecx
movl 24(%esp), %edx
cmpl (%ebx, %edi, 4), %eax
jne PSEUDOEND
movl (%ebx, %ecx, 4), %ebp
movl %esi, 12(%esp)
movl (%ebx, %edi, 4), %esi
movl %esi, (%ebx, %ecx, 4)
movl %ebp, (%ebx, %edi, 4)
movl 12(%esp), %esi
incl %ecx
incl %edx
jmp FOR1
PSEUDOEND:
incl %edi
jmp FOR2
EXIT:
movl %ebx, %eax
popl %ebp
popl %edi
popl %esi
popl %ebx
addl $32, %esp
ret
EXIT2:
incl %edx
jmp FOR1
.globl min
min:
movl 4(%esp), %ecx
movl 8(%esp), %edx
movl 12(%esp), %ebx
movl $1000, %eax
FOR:
cmpl %edx, %ebx
jge END
cmpl %eax, (%ecx, %ebx, 4)
cmovl (%ecx, %ebx, 4), %eax
incl %ebx
jmp FOR
END:
ret
to compile I use
gcc -m32 filename.c filename.s -o filename
I don't think that the problem is in the assembly translation because it prints correctly the new array. What could be the problem?

Related

Segmentation fault when pushing to stack in assembly

I am writing a problem to calculate smallest common multiple in assembly. GDB shows sigsegv exception at the very first line pushl %ebp. Can you tell me how to fix this?
test.c
#include <stdio.h>
extern int nww(int a, int b); // Funkcja z asm
int main()
{
int a = 10, b = 3;
int nw= nww(a,b);
printf("NWW %d i %d = %d ",a,b,nw);
return 0;
}
nww.s
.text
.global nww
.type nww, #function
nww:
pushl %ebp
movl %esp, %ebp
movl 8(%ebp), %eax
movl 12(%ebp), %ebx
nwd:
movl $0, %edx # miejsce na reszte
divl %ebx # podziel a przez b, reszta w d
movl %ebx, %eax
movl %edx, %ebx # przygotowanie nastepnego dzielenia
cmp $0, %edx
jnz nww
movl %eax, %edx # NWD do dx
movl 8(%ebp), %eax # nowe A i B
movl 12(%ebp), %ebx
imull %ebx, %eax
divl %edx
exit:
movl %ebp, %esp
popl %ebp
ret

undefined reference to 'nCr'

I'm trying to compile my program to run on linux but it doesn't work because of this error.
I compile with gcc -ansi -pedantic -Wall -m64 -o formula formula.c nCr.s and the error i get is:
/tmp/ccnNz7Jr.o: In function `main':
formula.c:(.text+0xc1): undefined reference to `nCr'
collect2: ld returned 1 exit status
make: *** [build] Error 1
Here's the code from the file formula.c
#include <stdio.h>
#include <stdlib.h>
#include "nCr.h"
#include <time.h>
#include <sys/time.h>
int main(int argc, const char * argv[])
{
int x = 0;
int y;
int z;
struct timeval start, end;
if (argv[1][0] == '-' && argv[1][1] == 'h') {
printf("Usage: formula <positive integer>");
} else {
y = atoi(argv[1]);
gettimeofday(&start, NULL);
printf("(1 + x)^%i = 1+", y);
if (y == 0)
printf("0");
if (y > 12) {
printf("%s\n","Please enter a number 12 or below. Anything higher results in overflow, the answer you want is not the answer you will get.");
}
for (; x <= y; x++) {
z = nCr(y, x);
if (z == -1) {
printf("Multiplication overflow. \n");
return 1;
} else {
if (x != 0)
printf("%i x^%i ",z , x);
if (x != y && x != 0)
printf("+ ");
}
}
gettimeofday(&end, NULL);
}
printf("\n%ld microseconds\n", ((end.tv_sec * 1000000 + end.tv_usec)
- (start.tv_sec * 1000000 + start.tv_usec)));
return 0;
}
ncr.h
#ifndef _NCR_H_
#define _NCR_H_
extern int Factorial(int n);
extern int nCr(int n, int r);
#endif /* _NCR_H_ */
and there's an assembly file called nCr.s
.globl _factorial
_factorial:
Leh_func_begin2:
pushq %rbp
Ltmp3:
movq %rsp, %rbp
Ltmp4:
movl %edi, -4(%rbp)
movl $1, -16(%rbp)
movl $1, -20(%rbp)
jmp LBB2_2
LBB2_1:
movl -16(%rbp), %eax
movl -20(%rbp), %ecx
imull %ecx, %eax
movl %eax, -16(%rbp)
movl -20(%rbp), %eax
addl $1, %eax
movl %eax, -20(%rbp)
LBB2_2:
movl -20(%rbp), %eax
movl -4(%rbp), %ecx
cmpl %ecx, %eax
jle LBB2_1
movl -16(%rbp), %eax
movl %eax, -12(%rbp)
movl -12(%rbp), %eax
movl %eax, -8(%rbp)
movl -8(%rbp), %eax
popq %rbp
ret
.globl _nCr
_nCr:
Leh_func_begin1:
pushq %rbp
Ltmp0:
movq %rsp, %rbp
Ltmp1:
subq $32, %rsp
Ltmp2:
movl %edi, -4(%rbp)
movl %esi, -8(%rbp)
movl -4(%rbp), %eax
xorb %cl, %cl
movl %eax, %edi
movb %cl, %al
callq _factorial
movl %eax, %ecx
movl %ecx, -20(%rbp)
movl -4(%rbp), %ecx
movl -8(%rbp), %edx
subl %edx, %ecx
movl %ecx, -24(%rbp)
movl -8(%rbp), %ecx
xorb %dl, %dl
movl %ecx, %edi
movb %dl, %al
callq _factorial
movl %eax, %ecx
movl -24(%rbp), %edx
xorb %sil, %sil
movl %edx, %edi
movb %sil, %al
movl %ecx, -32(%rbp)
callq _factorial
movl %eax, %ecx
movl -32(%rbp), %esi
imull %ecx, %esi
movl %esi, -28(%rbp)
movl -20(%rbp), %ecx
movl -28(%rbp), %esi
movl %ecx, %eax
cltd
idivl %esi
movl %eax, %ecx
movl %ecx, -16(%rbp)
movl %ecx, -12(%rbp)
movl -12(%rbp), %eax
addq $32, %rsp
popq %rbp
ret
Any help is greatly appreciated, can't seem to figure out why it's not compiling. It compiles just fine on mac but I need to get it run on linux.
GCC is not able to find extern int Factorial(int n); and extern int nCr(int n, int r); definitions because they are called _factorial: and _nCr: in your asm file. Change those labels to match the decl and your code will compile.
I noticed that you have only one call of nCr in the main function, so I think it is better to change decls in your .h. So your nCr.h will be:
#ifndef _NCR_H_
#define _NCR_H_
extern int _factorial(int n);
extern int _nCr(int n, int r);
#endif /* _NCR_H_ */
And you have to call _nCr(...) at main.

Trouble with y86 bubble sort program

So I am trying to convert a bubble sort program from assembly to Y86. I started with this C code and then converted it to assembly:
#include <stdio.h>
void bubble(int *, int);
int main(){
int count = 5;
int data[5]= {3, 2, 6, 1, 9,};
bubble(data, count);
return 0;
}
void bubble(int *data, int count){
int i, last;
for(last = count-1; last > 0; last--){
for (i=0; i < last; i++)
if (data[i+1] < data[i]){
/* Swap adjacent elements */
int t = data[i+1];
data[i+1] = data[i];
data[i] = t;
}
}
}
and this is my Y86 code:
init: irmovl Stack, %esp # Set up stack pointer
irmovl Stack, %ebp # Set up base pointer
call main # call main program
halt # Terminate program
main:
pushl %ebp
rrmovl %esp, %ebp
irmovl -16, %ecx
andl %ecx, %esp
irmovl 48, %ecx
subl %ecx, %esp
irmovl 5, %esi
rmmovl %esi, 44(%esp)
irmovl 3, %esi
rmmovl %esi, 24(%esp)
irmovl 2, %esi
rmmovl %esi, 28(%esp)
irmovl 6, %esi
rmmovl %esi, 32(%esp)
irmovl 1, %esi
rmmovl %esi, 36(%esp)
irmovl 9, %esi
rmmovl %esi, 40(%esp)
mrmovl 44(%esp), %eax
rmmovl %eax, 4(%esp)
mrmovl 24(%esp), %ecx
addl %ecx, %eax
rmmovl %eax, (%esp)
call bubble
irmovl 0, %eax
rrmovl %ebp, %esp
popl %ebp
ret
bubble:
pushl %ebp
rrmovl %esp, %ebp
irmovl 16, %esi
subl %esi, %esp
mrmovl 12(%ebp), %eax
irmovl 1, %esi
subl %esi, %eax
rmmovl %eax, -8(%ebp)
jmp L4
L8:
irmovl 0, %ebx
rmmovl %ebx, -12(%ebp)
jmp L5
L7:
mrmovl -12(%ebp), %eax
irmovl 1, %ecx
addl %ecx, %eax
addl %eax, %eax
addl %eax, %eax
addl %eax, %eax
mrmovl 8(%ebp), %esi
addl %esi, %eax
mrmovl (%eax), %edx
mrmovl -12(%ebp), %eax
addl %eax, %eax
addl %eax, %eax
addl %eax, %eax
mrmovl 8(%ebp), %esi
addl %esi, %eax
mrmovl (%eax), %eax
subl %eax, %edx
jge L6
mrmovl -12(%ebp), %eax
irmovl 1, %esi
addl %esi, %eax
addl %eax, %eax
addl %eax, %eax
addl %eax, %eax
mrmovl 8(%ebp), %esi
addl %esi, %eax
mrmovl (%eax), %eax
rmmovl %eax, -4(%ebp)
mrmovl -12(%ebp), %eax
irmovl 1, %esi
addl %esi, %eax
addl %eax, %eax
addl %eax, %eax
addl %eax, %eax
mrmovl 8(%ebp), %esi
addl %esi, %eax
mrmovl -12(%ebp), %edx
addl %edx, %edx
addl %edx, %edx
addl %edx, %edx
mrmovl 8(%ebp), %esi
addl %esi, %eax
mrmovl (%edx), %edx
rmmovl %edx, (%eax)
mrmovl -12(%ebp), %eax
addl %eax, %eax
addl %eax, %eax
addl %eax, %eax
mrmovl 8(%ebp), %esi
addl %esi, %eax
mrmovl -4(%ebp), %edx
rmmovl %edx, (%eax)
L6:
irmovl 1, %esi
mrmovl -12(%ebp), %ebx
addl %esi, %ebx
L5:
mrmovl -12(%ebp), %eax
mrmovl -8(%ebp), %ebx
subl %ebx, %eax
jl L7
irmovl 1, %esi
mrmovl -8(%ebp), %ebx
subl %esi, %ebx
L4:
irmovl 0, %esi #i = 0
mrmovl -8(%ebp), %ebx
subl %esi, %ebx
jg L8
rrmovl %ebp, %esp
popl %ebp
ret
#stack starts here
.pos 0x200
Stack: .long 0
It seems as though the sign flag is never being raised, and I am not quite sure why this is. When I run the simulator, the elements of the array are never sorted, they just remain in the same position. Any tips on how to fix this would be much appreciated!

Why "movl $1, %edx" instead of "movl $0, %edx" in a do while

Recent times I am having a look at assembly IA32, I did a simple toy example:
#include <stdio.h>
int array[10];
int i = 0;
int sum = 0;
int main(void)
{
for (i = 0; i < 10; i++)
{
array[i] = i;
sum += array[i];
}
printf("SUM = %d\n",sum);
return 0;
}
Yes, I know it is not very recommended to use global variables. I compiled the above code without optimizations and using the flag -s, I got this assembly :
main:
...
movl $0, %eax
subl %eax, %esp
movl $0, i
.L2:
cmpl $9, i
jle .L5
jmp .L3
.L5:
movl i, %edx
movl i, %eax
movl %eax, array(,%edx,4)
movl i, %eax
movl array(,%eax,4), %eax
addl %eax, sum
incl i
jmp .L2
Nothing too fancy and easy to understand, it is a normal while loop. Then I compiled the same code with -O2 and got the following assembly :
main:
...
xorl %eax, %eax
movl $0, i
movl $1, %edx
.p2align 2,,3
.L6:
movl sum, %ecx
addl %eax, %ecx
movl %eax, array-4(,%edx,4)
movl %edx, %eax
incl %edx
cmpl $9, %eax
movl %ecx, sum
movl %eax, i
jle .L6
subl $8, %esp
pushl %ecx
pushl $.LC0
call printf
xorl %eax, %eax
leave
ret
In this case it transformed in a do while type of loop. From the above assembly what I am not understanding is why "movl $1, %edx" and then "movl %eax, array-4(,%edx,4)".
The %edx starts with 1 instead of 0 and then when accessing the array it does -4 from the initial position (4 bytes = integer) . Why not simply ?
movl $0, %edx
...
array (,%edx,4)
instead of starting with 1 if you need to do -4 all the time.
I am using "GCC: (GNU) 3.2.3 20030502 (Red Hat Linux 3.2.3-24)", for educational reasons to generate easily understandable assembly.
I think I finally get the point, I test with:
...
int main(void)
{
for (i = 0; i < 10; i+=2)
{
...
}
}
and got:
movl $2, %edx
and with for (i = 0; i < 10; i +=3) and got :
movl $3, %edx
and finally with (i = 1; i < 10; i +=3) and got:
movl $4, %edx
Therefore, the compiler is initializing %edx = i (initial value of i) + incrementStep;

assembly, two functions in 1 file, getting Error: junk `(%ebp)' after expression

The assembly functions with commented c version:
/*
int f (int x)
{
return x+2;
}
void map2 (int* one, int * another, int n)
{
int i;
for (i=0; i<n; i++)
*(another+i) = f(*(one+i));
}
*/
.text
.globl f
f:
/********************************** prologue *************************************/
pushl %ebp
movl %esp, %ebp
/********************************************************************************/
movl 8(%ebp), %eax
addl $2, %eax
/************************************* end **************************************/
movl %ebp, %esp
popl %ebp
ret
/*********************************************************************************/
.text
.globl map2
map2:
/********************************** prologue *************************************/
pushl %ebp
movl %esp, %ebp
pushl %ebx
pushl %esi
pushl %edi
/********************************************************************************/
movl $0, %ebx
L1: cmpl $16(%ebp), %ebx // error
jge out
movl $12(%ebp), %esi // error
addl %ebx, %esi
movl $8(%ebp), %edi // error
addl %ebx, %edi
/********************************** calls f *************************************/
pushl %eax
pushl %ecx
pushl %edx
pushl (%edi)
call f
movl %eax, (%esi) /* *(another+i) = f(*(one+i)); */
popl %edx
popl %ecx
popl %eax
/********************************************************************************/
/************************************* end **************************************/
popl %edi
popl %esi
popl %ebx
movl %ebp, %esp
popl %ebp
ret
/********************************************************************************/
I'm getting
fmap2.s:44: Error: junk `(%ebp)' after expression
fmap2.s:47: Error: junk `(%ebp)' after expression
fmap2.s:50: Error: junk `(%ebp)' after expression
Why? There is something to do with having 2 prologues and endings in the same file?
The base-dislocation numbers of those error lines are correct.
According to gcc documentation, you need to remove dollar signs in front of the numeric constants used for displacement:
One point to bear in mind is that, when a constant is used for disp/scale, ’$’ shouldn’t be prefixed.
movl $0, %ebx
L1: cmpl 16(%ebp), %ebx
jge out
movl 12(%ebp), %esi
addl %ebx, %esi
movl 8(%ebp), %edi
addl %ebx, %edi

Resources