I am having trouble this piece of code in assembly language.
Essentially I have to input 2 numbers that matches 2 numbers the code is comparing with.
On line 0x08048c47 in phase_4, it compares the first input with 2, so I know the first input has to be 2. It then moves 4 spaces from the first input to next input, which then gets 2 subtracted from it. Now the (input-2) is compared with 2. It will continue the instruction if the inputs are below than or equal to 2. I've tested this with numbers 2,3,4 which pass the comparison. Other numbers greater than 4 and less than 2 do not pass the comparison and will cause the bomb to explode.
I'm stuck on this part because the value being returned from func4 is not the same was the value represented at 0x08048c6e in phase_4, which is 8(%esp). On my computer when I debug it, it shows that it is 8, and the answers to my inputs 2,3,4 are 40, 60, 80 respectively.
disas func4
0x08048bda <+0>: push %edi
0x08048bdb <+1>: push %esi
0x08048bdc <+2>: push %ebx
0x08048bdd <+3>: mov 0x10(%esp),%ebx
0x08048be1 <+7>: mov 0x14(%esp),%edi
0x08048be5 <+11>: test %ebx,%ebx
0x08048be7 <+13>: jle 0x8048c14 <func4+58>
0x08048be9 <+15>: mov %edi,%eax
0x08048beb <+17>: cmp $0x1,%ebx
0x08048bee <+20>: je 0x8048c19 <func4+63>
0x08048bf0 <+22>: sub $0x8,%esp
0x08048bf3 <+25>: push %edi
0x08048bf4 <+26>: lea -0x1(%ebx),%eax
0x08048bf7 <+29>: push %eax
0x08048bf8 <+30>: call 0x8048bda <func4>
0x08048bfd <+35>: add $0x8,%esp
0x08048c00 <+38>: lea (%edi,%eax,1),%esi
0x08048c03 <+41>: push %edi
0x08048c04 <+42>: sub $0x2,%ebx
0x08048c07 <+45>: push %ebx
0x08048c08 <+46>: call 0x8048bda <func4>
0x08048c0d <+51>: add $0x10,%esp
0x08048c10 <+54>: add %esi,%eax
0x08048c12 <+56>: jmp 0x8048c19 <func4+63>
0x08048c14 <+58>: mov $0x0,%eax
0x08048c19 <+63>: pop %ebx
0x08048c1a <+64>: pop %esi
0x08048c1b <+65>: pop %edi
0x08048c1c <+66>: ret
disas phase_4
0x08048c1d <+0>: sub $0x1c,%esp
0x08048c20 <+3>: mov %gs:0x14,%eax
0x08048c26 <+9>: mov %eax,0xc(%esp)
0x08048c2a <+13>: xor %eax,%eax
0x08048c2c <+15>: lea 0x4(%esp),%eax
0x08048c30 <+19>: push %eax
0x08048c31 <+20>: lea 0xc(%esp),%eax
0x08048c35 <+24>: push %eax
0x08048c36 <+25>: push $0x804a25f
0x08048c3b <+30>: pushl 0x2c(%esp)
0x08048c3f <+34>: call 0x8048810 <__isoc99_sscanf#plt>
0x08048c44 <+39>: add $0x10,%esp
0x08048c47 <+42>: cmp $0x2,%eax
0x08048c4a <+45>: jne 0x8048c58 <phase_4+59>
0x08048c4c <+47>: mov 0x4(%esp),%eax
0x08048c50 <+51>: sub $0x2,%eax
0x08048c53 <+54>: cmp $0x2,%eax
0x08048c56 <+57>: jbe 0x8048c5d <phase_4+64>
0x08048c58 <+59>: call 0x8049123 <explode_bomb>
0x08048c5d <+64>: sub $0x8,%esp
0x08048c60 <+67>: pushl 0xc(%esp)
0x08048c64 <+71>: push $0x6
0x08048c66 <+73>: call 0x8048bda <func4>
0x08048c6b <+78>: add $0x10,%esp
0x08048c6e <+81>: cmp 0x8(%esp),%eax
0x08048c72 <+85>: je 0x8048c79 <phase_4+92>
0x08048c74 <+87>: call 0x8049123 <explode_bomb>
0x08048c79 <+92>: mov 0xc(%esp),%eax
0x08048c7d <+96>: xor %gs:0x14,%eax
0x08048c84 <+103>: je 0x8048c8b <phase_4+110>
0x08048c86 <+105>: call 0x8048790 <__stack_chk_fail#plt>
0x08048c8b <+110>: add $0x1c,%esp
0x08048c8e <+113>: ret
8(%esp) is the first number, under the framework of x86.
enter 40 2 or 60 3 or 80 4 should work.
Equivalent to the following logic
#include <stdio.h>
#include <stdlib.h>
void explode_bomb()
{
printf("explode bomb.\n");
exit(1);
}
unsigned func4(int val, unsigned num)
{
int ret;
if (val <= 0)
return 0;
if (num == 1)
return 1;
ret = func4(val - 1, num);
ret += num;
val -= 2;
ret += func4(val, num);
return ret;
}
void phase_4(const char *input)
{
unsigned num1, num2;
if (sscanf(input, "%u %u", &num1, &num2) != 2)
explode_bomb();
if (num2 - 2 > 2)
explode_bomb();
if (func4(6, num2) != num1)
explode_bomb();
}
int main()
{
phase_4("40 2");
phase_4("60 3");
phase_4("80 4");
printf("success.\n");
return 0;
}
Related
Hi i'm currently doing a binary bomb and am wondering if I am understanding some stuff correctly. I have this;
0x00000000004011d4 <+0>: sub $0x8,%rsp
0x00000000004011d8 <+4>: cmpb $0x59,(%rdi)
0x00000000004011db <+7>: jne 0x4011fd <phase_1+41>
0x00000000004011dd <+9>: cmpb $0x46,0x2(%rdi)
0x00000000004011e1 <+13>: jne 0x4011fd <phase_1+41>
0x00000000004011e3 <+15>: cmpb $0x68,0x1(%rdi)
0x00000000004011e7 <+19>: je 0x40120b <phase_1+55>
0x00000000004011e9 <+21>: movsbl 0x10(%rdi),%ecx
0x00000000004011ed <+25>: movsbl 0x5(%rdi),%edx
0x00000000004011f1 <+29>: add $0xb,%edx
0x00000000004011f4 <+32>: mov $0x1,%eax
0x00000000004011f9 <+37>: cmp %edx,%ecx
0x00000000004011fb <+39>: je 0x401210 <phase_1+60>
0x00000000004011fd <+41>: callq 0x401b20 <bomb_activation>
0x0000000000401202 <+46>: mov $0xffffffffffffffff,%rax
0x0000000000401209 <+53>: jmp 0x401210 <phase_1+60>
0x000000000040120b <+55>: mov $0x0,%eax
0x0000000000401210 <+60>: add $0x8,%rsp
0x0000000000401214 <+64>: retq
and so far I have translated it to this;
if(arr[0] != 'Y'){
bomb_activation();
}
if(arr[2] != 'F'){
bomb_activation();
}
if(arr[1] == 'h'){
bomb_activation();
}
int a = arr[10];
int b = arr[5];
b += 11;
status = 1;
if(t1 != t2){
bomb_activation();
}
return status;
}
As you can probably tell i'm really confused on how exactly to read these lines, I see it as moving the 10th element of the array into the ecx registry and filling the rest of the registry with 0s and the same logic to edx, however i'm not too sure how to determine what the value of arr[5] or arr[10] is just from this.
0x00000000004011e9 <+21>: movsbl 0x10(%rdi),%ecx
0x00000000004011ed <+25>: movsbl 0x5(%rdi),%edx
0x00000000004011f1 <+29>: add $0xb,%edx
0x00000000004011f4 <+32>: mov $0x1,%eax
0x00000000004011f9 <+37>: cmp %edx,%ecx
and more specifically how I am meant to determine the size of the array, maybe I am not understanding it at all though, any help would be great thanks.
I have to override a function's return address using buffer oveflow.
The function itself looks like this(I am passing the name argument):
void vuln(char *name)
{
int n = 20;
char buf[1024];
int f[n];
int i;
for (i=0; i<n; i++) {
f[i] = fib(i);
}
strcpy(buf, name);
...
}
I am disassembling it using gdb which gives me the following
0x080485ae <+0>: push %ebp
0x080485af <+1>: mov %esp,%ebp
0x080485b1 <+3>: push %ebx
0x080485b2 <+4>: sub $0x414,%esp
0x080485b8 <+10>: mov %esp,%eax
0x080485ba <+12>: mov %eax,%ebx
0x080485bc <+14>: movl $0x14,-0x10(%ebp)
0x080485c3 <+21>: mov -0x10(%ebp),%eax
0x080485c6 <+24>: lea -0x1(%eax),%edx
0x080485c9 <+27>: mov %edx,-0x14(%ebp)
0x080485cc <+30>: shl $0x2,%eax
0x080485cf <+33>: lea 0x3(%eax),%edx
0x080485d2 <+36>: mov $0x10,%eax
0x080485d7 <+41>: sub $0x1,%eax
0x080485da <+44>: add %edx,%eax
0x080485dc <+46>: mov $0x10,%ecx
0x080485e1 <+51>: mov $0x0,%edx
0x080485e6 <+56>: div %ecx
0x080485e8 <+58>: imul $0x10,%eax,%eax
0x080485eb <+61>: sub %eax,%esp
0x080485ed <+63>: mov %esp,%eax
0x080485ef <+65>: add $0x3,%eax
0x080485f2 <+68>: shr $0x2,%eax
0x080485f5 <+71>: shl $0x2,%eax
0x080485f8 <+74>: mov %eax,-0x18(%ebp)
0x080485fb <+77>: movl $0x0,-0xc(%ebp)
0x08048602 <+84>: jmp 0x8048621 <vuln+115>
0x08048604 <+86>: sub $0xc,%esp
0x08048607 <+89>: pushl -0xc(%ebp)
0x0804860a <+92>: call 0x8048560 <fib>
0x0804860f <+97>: add $0x10,%esp
0x08048612 <+100>: mov %eax,%ecx
0x08048614 <+102>: mov -0x18(%ebp),%eax
0x08048617 <+105>: mov -0xc(%ebp),%edx
0x0804861a <+108>: mov %ecx,(%eax,%edx,4)
0x0804861d <+111>: addl $0x1,-0xc(%ebp)
0x08048621 <+115>: mov -0xc(%ebp),%eax
0x08048624 <+118>: cmp -0x10(%ebp),%eax
0x08048627 <+121>: jl 0x8048604 <vuln+86>
0x08048629 <+123>: sub $0x8,%esp
0x0804862c <+126>: pushl 0x8(%ebp)
0x0804862f <+129>: lea -0x418(%ebp),%eax
0x08048635 <+135>: push %eax
0x08048636 <+136>: call 0x80483c0 <strcpy#plt>
0x0804863b <+141>: add $0x10,%esp
0x0804863e <+144>: sub $0x8,%esp
0x08048641 <+147>: lea -0x418(%ebp),%eax
0x08048647 <+153>: push %eax
0x08048648 <+154>: push $0x80487b7
0x0804864d <+159>: call 0x80483a0 <printf#plt>
0x08048652 <+164>: add $0x10,%esp
0x08048655 <+167>: movl $0x0,-0xc(%ebp)
0x0804865c <+174>: jmp 0x804867f <vuln+209>
0x0804865e <+176>: mov -0x18(%ebp),%eax
0x08048661 <+179>: mov -0xc(%ebp),%edx
=> 0x08048664 <+182>: mov (%eax,%edx,4),%eax
0x08048667 <+185>: sub $0x4,%esp
0x0804866a <+188>: push %eax
0x0804866b <+189>: pushl -0xc(%ebp)
0x0804866e <+192>: push $0x80487c4
0x08048673 <+197>: call 0x80483a0 <printf#plt>
0x08048678 <+202>: add $0x10,%esp
0x0804867b <+205>: addl $0x1,-0xc(%ebp)
0x0804867f <+209>: cmpl $0x13,-0xc(%ebp)
0x08048683 <+213>: jle 0x804865e <vuln+176>
0x08048685 <+215>: mov %ebx,%esp
0x08048687 <+217>: nop
0x08048688 <+218>: mov -0x4(%ebp),%ebx
0x0804868b <+221>: leave
0x0804868c <+222>: ret
The address of the function which should be called with the return of vuln() is 0x804850b.
How am I supposed to know the amount of fillers until I reach the return address to be overwritten?
I guess the name argument should be in the form "a"*n + "\x0b\x85\x04\x08", where n is some number I am trying to guess. I suppose this should be basic stuff but I am still a beginner so please don't judge me...
How am I supposed to know ...
Your code is:
0x080485ae <+0>: push %ebp
0x080485af <+1>: mov %esp,%ebp
...
0x0804862f <+129>: lea -0x418(%ebp),%eax
0x08048635 <+135>: push %eax
0x08048636 <+136>: call 0x80483c0 <strcpy#plt>
Before you enter the function, the return address is at offset 0(%esp).
After the first push, it's at 4(%esp). Since %esp is next copied to %ebp, it's also at 4(%ebp).
Next you see that the location you start copying into is at -0x418(%ebp).
Conclusion: the delta between &buf[0] and &return_address is 0x418 + 4 == 0x41C.
Alternative solution: fill name input with invalid addresses: 0x01010101, 0x01010102, ... 0x010102FF. Execute the code and observe on which address it crashed.
If my calculations are correct, it would crash when vuln tries to return to "slot" 0x41C / 4 == 0x107, which should contain 0x01010208.
I have binary Phase that is not returning required result i.e 12. Any suggestions?
Phase 4
Dump of assembler code for function phase_4:
0x000000000040100b <+0>: sub $0x18,%rsp
0x000000000040100f <+4>: lea 0x8(%rsp),%rcx
0x0000000000401014 <+9>: lea 0xc(%rsp),%rdx
0x0000000000401019 <+14>: mov $0x40278d,%esi
0x000000000040101e <+19>: mov $0x0,%eax
0x0000000000401023 <+24>: callq 0x400b90 <__isoc99_sscanf#plt>
0x0000000000401028 <+29>: cmp $0x2,%eax
=> 0x000000000040102b <+32>: je 0x401054 <phase_4+73>
0x000000000040102d <+34>: callq 0x401538 <explode_bomb>
0x0000000000401032 <+39>: mov $0xe,%edx
0x0000000000401037 <+44>: mov $0x0,%esi
0x000000000040103c <+49>: mov 0xc(%rsp),%edi
0x0000000000401040 <+53>: callq 0x400fd7 <func4>
0x0000000000401045 <+58>: cmp $0x12,%eax
0x0000000000401048 <+61>: je 0x40105d <phase_4+82>
0x000000000040104a <+63>: callq 0x401538 <explode_bomb>
0x000000000040104f <+68>: add $0x18,%rsp
0x0000000000401053 <+72>: retq
0x0000000000401054 <+73>: cmpl $0xe,0xc(%rsp)
0x0000000000401059 <+78>: jbe 0x401032 <phase_4+39>
0x000000000040105b <+80>: jmp 0x40102d <phase_4+34>
0x000000000040105d <+82>: cmpl $0x12,0x8(%rsp)
0x0000000000401062 <+87>: jne 0x40104a <phase_4+63>
0x0000000000401064 <+89>: jmp 0x40104f <phase_4+68>
End of assembler dump.
func4 is as follows:
Dump of assembler code for function func4:
=> 0x0000000000400fd7 <+0>: push %rbx
0x0000000000400fd8 <+1>: mov %edx,%eax
0x0000000000400fda <+3>: sub %esi,%eax
0x0000000000400fdc <+5>: mov %eax,%ebx
0x0000000000400fde <+7>: shr $0x1f,%ebx
0x0000000000400fe1 <+10>: add %eax,%ebx
0x0000000000400fe3 <+12>: sar %ebx
0x0000000000400fe5 <+14>: add %esi,%ebx
0x0000000000400fe7 <+16>: cmp %edi,%ebx
0x0000000000400fe9 <+18>: jg 0x400ff3 <func4+28>
0x0000000000400feb <+20>: cmp %edi,%ebx
0x0000000000400fed <+22>: jl 0x400fff <func4+40>
0x0000000000400fef <+24>: mov %ebx,%eax
0x0000000000400ff1 <+26>: pop %rbx
0x0000000000400ff2 <+27>: retq
0x0000000000400ff3 <+28>: lea -0x1(%rbx),%edx
0x0000000000400ff6 <+31>: callq 0x400fd7 <func4>
0x0000000000400ffb <+36>: add %eax,%ebx
0x0000000000400ffd <+38>: jmp 0x400fef <func4+24>
0x0000000000400fff <+40>: lea 0x1(%rbx),%esi
0x0000000000401002 <+43>: callq 0x400fd7 <func4>
0x0000000000401007 <+48>: add %eax,%ebx
0x0000000000401009 <+50>: jmp 0x400fef <func4+24>
End of assembler dump.
I have written a test C program that I believe equivalent to above assembly code for func4.
#include <stdio.h>
int main()
{
int i=0;
for(int i=0;i<15;i++)
{
int z=func4(i,0,14);
printf("in main program: For input %d -> %d\n",i,z);
}
return 0;
}
int func4(int x, int low, int high) {
int mid = (low + high) / 2;
if (x == mid) {
return (mid);
} else if (x < mid) {
int w=mid+func4(x, low, mid - 1);
return w;
} else {
int p=mid+func4(x, mid + 1, high);
return p;
}
}
This program returns OUTPUT as follows:
in main program: For input 0 -> 11
in main program: For input 1 -> 11
in main program: For input 2 -> 13
in main program: For input 3 -> 10
in main program: For input 4 -> 19
in main program: For input 5 -> 15
in main program: For input 6 -> 21
in main program: For input 7 -> 7
in main program: For input 8 -> 35
in main program: For input 9 -> 27
in main program: For input 10 -> 37
in main program: For input 11 -> 18
in main program: For input 12 -> 43
in main program: For input 13 -> 31
in main program: For input 14 -> 45
I figured out that function will take two arguments and the second argument should be 12. But I am not to get value 12 returned from func4.
Any suggestions??
The assembly code is actually expecting a value of 0x12 (18 rather than 12) from func4:
0x0000000000401040 <+53>: callq 0x400fd7 <func4>
0x0000000000401045 <+58>: cmp $0x12,%eax <- Compare result to 0x12
0x0000000000401048 <+61>: je 0x40105d <phase_4+82>
0x000000000040104a <+63>: callq 0x401538 <explode_bomb>
I haven't looked at your C translation of func4 for correctness, but assuming it's correct it looks like func4(11, 0, 14) gives you what you want.
In the program binary, how to determine the instructions related to parameter passing of variable argument function "printf"? For example:
#include <stdio.h>
#include <string.h>
int fun(int a, int b){
return a+b;
}
void main (int argc, char* argv[]){
int a = 0;
int b = 1;
int c = 2;
int d = 3;
printf("a:fun(b,c):d: %d:%d:%d\n", a, fun(b,c), d);
}
is assembled as follows:
(gdb) disas main
Dump of assembler code for function main:
0x080483f1 <+0>: push %ebp
0x080483f2 <+1>: mov %esp,%ebp
0x080483f4 <+3>: and $0xfffffff0,%esp
0x080483f7 <+6>: sub $0x20,%esp
0x080483fa <+9>: movl $0x0,0x10(%esp)
0x08048402 <+17>: movl $0x1,0x14(%esp)
0x0804840a <+25>: movl $0x2,0x18(%esp)
0x08048412 <+33>: movl $0x3,0x1c(%esp)
0x0804841a <+41>: mov 0x18(%esp),%eax
0x0804841e <+45>: mov %eax,0x4(%esp)
0x08048422 <+49>: mov 0x14(%esp),%eax
0x08048426 <+53>: mov %eax,(%esp)
0x08048429 <+56>: call 0x80483e4 <fun>
=> 0x0804842e <+61>: mov $0x8048530,%edx
0x08048433 <+66>: mov 0x1c(%esp),%ecx
0x08048437 <+70>: mov %ecx,0xc(%esp)
0x0804843b <+74>: mov %eax,0x8(%esp)
0x0804843f <+78>: mov 0x10(%esp),%eax
0x08048443 <+82>: mov %eax,0x4(%esp)
=> 0x08048447 <+86>: mov %edx,(%esp)
0x0804844a <+89>: call 0x8048300 <printf#plt>
0x0804844f <+94>: leave
0x08048450 <+95>: ret
Whether the instructions related to parameter passing of variable argument function "print" are instructions between two instructions "0x0804842e <+61>: mov $0x8048530,%edx" and " 0x08048447 <+86>: mov %edx,(%esp)".
I have tested many cases of function "printf". In all my tested cases, the instructions related to parameter passing are between these two instructions.
You can find this out by knowing the C calling convention. That is arguments are pushed onto the stack in reverse order.
`0x0804842e <+61>: mov $0x8048530,%edx //Probably the string literal
0x08048433 <+66>: mov 0x1c(%esp),%ecx //Moving 3 literal into %ecx
0x08048437 <+70>: mov %ecx,0xc(%esp) // moving 3 onto top of the arguments on the stack (%esp is the stack pointer)
0x0804843b <+74>: mov %eax,0x8(%esp) //Moving return value from fun onto next slot in the stack, %eax store the return value from a function.
0x0804843f <+78>: mov 0x10(%esp),%eax //Moving 0 literal into %eax
0x08048443 <+82>: mov %eax,0x4(%esp) //Moving %eax into next slot in the stack
0x08048447 <+86>: mov %edx,(%esp) //moving string literal onto the stack
0x0804844a <+89>: call 0x8048300 `//calling printf
I'm having trouble with the password for phase_4 of my binary bomb.
So far, I understand...
the inputs must be 2 integers (line 29)
and the second integer should <= than or equal to 2 when 2 is subtracted from it (lines 38-44). Which means it should be a number 2-4
Then, it calls func4, and compares the output of func4 to rsp (which I've done some testing and rsp is always 0) so i need to find a number that returns 0 when inputed into func4.
Whats confusing me is: if the result of func4 needs to be 0, that means it's input should be 0. Taking into account line 38 in phase_4 where it subtracts 2 from my input, that should mean my second # should be 2...? However I have tried that and it makes it explode in line 46 as it doesn't satisfy the first comparison.
I have tried numbers like (0 2) (2 0) (0 1) (0 3) and they all failed. Any suggestions to point me in the right direction? Thanks!
phase_4:
0x000000000040101e <+0>: sub $0x18,%rsp
0x0000000000401022 <+4>: lea 0xc(%rsp),%rcx
0x0000000000401027 <+9>: lea 0x8(%rsp),%rdx
0x000000000040102c <+14>: mov $0x4027cd,%esi
0x0000000000401031 <+19>: mov $0x0,%eax
0x0000000000401036 <+24>: callq 0x400c30 <__isoc99_sscanf#plt>
0x000000000040103b <+29>: cmp $0x2,%eax //check if 2 inputs
0x000000000040103e <+32>: jne 0x40104c <phase_4+46>
0x0000000000401040 <+34>: mov 0xc(%rsp),%eax
=> 0x0000000000401044 <+38>: sub $0x2,%eax
0x0000000000401047 <+41>: cmp $0x2,%eax
0x000000000040104a <+44>: jbe 0x401051 <phase_4+51>//if unsigned eax <= 2
0x000000000040104c <+46>: callq 0x401554 <explode_bomb>
0x0000000000401051 <+51>: mov 0xc(%rsp),%esi
0x0000000000401055 <+55>: mov $0x7,%edi
0x000000000040105a <+60>: callq 0x400fe6 <func4>
0x000000000040105f <+65>: cmp 0x8(%rsp),%eax //comparing eax to 0
0x0000000000401063 <+69>: je 0x40106a <phase_4+76>
0x0000000000401065 <+71>: callq 0x401554 <explode_bomb> //explode if output != 0
0x000000000040106a <+76>: add $0x18,%rsp
0x000000000040106e <+80>: retq
Func4
0x0000000000400fe6 <+0>: push %r12
0x0000000000400fe8 <+2>: push %rbp
0x0000000000400fe9 <+3>: push %rbx
0x0000000000400fea <+4>: mov %edi,%ebx
0x0000000000400fec <+6>: test %edi,%edi
0x0000000000400fee <+8>: jle 0x401014 <func4+46> //if input <= 0
0x0000000000400ff0 <+10>: mov %esi,%ebp
0x0000000000400ff2 <+12>: mov %esi,%eax
0x0000000000400ff4 <+14>: cmp $0x1,%edi
0x0000000000400ff7 <+17>: je 0x401019 <func4+51>
0x0000000000400ff9 <+19>: lea -0x1(%rdi),%edi
0x0000000000400ffc <+22>: callq 0x400fe6 <func4>
0x0000000000401001 <+27>: lea (%rax,%rbp,1),%r12d
0x0000000000401005 <+31>: lea -0x2(%rbx),%edi
0x0000000000401008 <+34>: mov %ebp,%esi
0x000000000040100a <+36>: callq 0x400fe6 <func4>
0x000000000040100f <+41>: add %r12d,%eax
0x0000000000401012 <+44>: jmp 0x401019 <func4+51>
0x0000000000401014 <+46>: mov $0x0,%eax //make return val 0
0x0000000000401019 <+51>: pop %rbx
0x000000000040101a <+52>: pop %rbp
0x000000000040101b <+53>: pop %r12
0x000000000040101d <+55>: retq
Then, it calls func4, and compares the output of func4 to rsp (which
I've done some testing and rsp is always 0) so i need to find a number
that returns 0 when inputed into func4.
This is incorrect. The output of func4 is compared with [rsp + 8], in which the first number was stored.
If we write the desired input as (a, b), then we have a = func4 (7, b) and 2 <= b <= 4.
To understand what func4 (x, y) does I recommend that you convert it to C. See my answer to this question for an illustration.