How Calling Functions with lots of args in Assembly Works - c

Question

Here is an example set of functions, the first with 20 args the second with 2:
int a(int n1, int n2, int n3, int n4, int n5, int n6, int n7, int n8, int n9, int n10, int n11, int n12, int n13, int n14, int n15, int n16, int n17, int n18, int n19, int n20) {
return n1 * n2 * n3 * n4 * n5 * n6 * n7 * n8 * n9 * n10 * n11 * n12 * n13 * n14 * n15 * n16 * n17 * n18 * n19 * n20;
}
int b(int n1, int n2) {
return a(n1, n2, n1, n2, n1, n1, n2, n1, n2, n1, n1, n2, n1, n2, n1, n1, n2, n1, n2, n1)
+ a(n1, n2, n1, n2, n1, n1, n2, n1, n2, n1, n1, n2, n1, n2, n1, n1, n2, n1, n2, n1)
+ a(n1, n2, n1, n2, n1, n1, n2, n1, n2, n1, n1, n2, n1, n2, n1, n1, n2, n1, n2, n1);
}
It gets compiled to this assembly:
a(int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int):
push rbp
mov rbp, rsp
mov DWORD PTR [rbp-4], edi
mov DWORD PTR [rbp-8], esi
mov DWORD PTR [rbp-12], edx
mov DWORD PTR [rbp-16], ecx
mov DWORD PTR [rbp-20], r8d
mov DWORD PTR [rbp-24], r9d
mov eax, DWORD PTR [rbp-4]
imul eax, DWORD PTR [rbp-8]
imul eax, DWORD PTR [rbp-12]
imul eax, DWORD PTR [rbp-16]
imul eax, DWORD PTR [rbp-20]
imul eax, DWORD PTR [rbp-24]
imul eax, DWORD PTR [rbp+16]
imul eax, DWORD PTR [rbp+24]
imul eax, DWORD PTR [rbp+32]
imul eax, DWORD PTR [rbp+40]
imul eax, DWORD PTR [rbp+48]
imul eax, DWORD PTR [rbp+56]
imul eax, DWORD PTR [rbp+64]
imul eax, DWORD PTR [rbp+72]
imul eax, DWORD PTR [rbp+80]
imul eax, DWORD PTR [rbp+88]
imul eax, DWORD PTR [rbp+96]
imul eax, DWORD PTR [rbp+104]
imul eax, DWORD PTR [rbp+112]
imul eax, DWORD PTR [rbp+120]
pop rbp
ret
b(int, int):
push rbp
mov rbp, rsp
push rbx
sub rsp, 8
mov DWORD PTR [rbp-12], edi
mov DWORD PTR [rbp-16], esi
mov r9d, DWORD PTR [rbp-12]
mov r8d, DWORD PTR [rbp-12]
mov ecx, DWORD PTR [rbp-16]
mov edx, DWORD PTR [rbp-12]
mov esi, DWORD PTR [rbp-16]
mov eax, DWORD PTR [rbp-12]
mov edi, DWORD PTR [rbp-12]
push rdi
mov edi, DWORD PTR [rbp-16]
push rdi
mov edi, DWORD PTR [rbp-12]
push rdi
mov edi, DWORD PTR [rbp-16]
push rdi
mov edi, DWORD PTR [rbp-12]
push rdi
mov edi, DWORD PTR [rbp-12]
push rdi
mov edi, DWORD PTR [rbp-16]
push rdi
mov edi, DWORD PTR [rbp-12]
push rdi
mov edi, DWORD PTR [rbp-16]
push rdi
mov edi, DWORD PTR [rbp-12]
push rdi
mov edi, DWORD PTR [rbp-12]
push rdi
mov edi, DWORD PTR [rbp-16]
push rdi
mov edi, DWORD PTR [rbp-12]
push rdi
mov edi, DWORD PTR [rbp-16]
push rdi
mov edi, eax
call a(int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int)
add rsp, 112
mov ebx, eax
mov r9d, DWORD PTR [rbp-12]
mov r8d, DWORD PTR [rbp-12]
mov ecx, DWORD PTR [rbp-16]
mov edx, DWORD PTR [rbp-12]
mov esi, DWORD PTR [rbp-16]
mov eax, DWORD PTR [rbp-12]
mov edi, DWORD PTR [rbp-12]
push rdi
mov edi, DWORD PTR [rbp-16]
push rdi
mov edi, DWORD PTR [rbp-12]
push rdi
mov edi, DWORD PTR [rbp-16]
push rdi
mov edi, DWORD PTR [rbp-12]
push rdi
mov edi, DWORD PTR [rbp-12]
push rdi
mov edi, DWORD PTR [rbp-16]
push rdi
mov edi, DWORD PTR [rbp-12]
push rdi
mov edi, DWORD PTR [rbp-16]
push rdi
mov edi, DWORD PTR [rbp-12]
push rdi
mov edi, DWORD PTR [rbp-12]
push rdi
mov edi, DWORD PTR [rbp-16]
push rdi
mov edi, DWORD PTR [rbp-12]
push rdi
mov edi, DWORD PTR [rbp-16]
push rdi
mov edi, eax
call a(int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int)
add rsp, 112
add ebx, eax
mov r9d, DWORD PTR [rbp-12]
mov r8d, DWORD PTR [rbp-12]
mov ecx, DWORD PTR [rbp-16]
mov edx, DWORD PTR [rbp-12]
mov esi, DWORD PTR [rbp-16]
mov eax, DWORD PTR [rbp-12]
mov edi, DWORD PTR [rbp-12]
push rdi
mov edi, DWORD PTR [rbp-16]
push rdi
mov edi, DWORD PTR [rbp-12]
push rdi
mov edi, DWORD PTR [rbp-16]
push rdi
mov edi, DWORD PTR [rbp-12]
push rdi
mov edi, DWORD PTR [rbp-12]
push rdi
mov edi, DWORD PTR [rbp-16]
push rdi
mov edi, DWORD PTR [rbp-12]
push rdi
mov edi, DWORD PTR [rbp-16]
push rdi
mov edi, DWORD PTR [rbp-12]
push rdi
mov edi, DWORD PTR [rbp-12]
push rdi
mov edi, DWORD PTR [rbp-16]
push rdi
mov edi, DWORD PTR [rbp-12]
push rdi
mov edi, DWORD PTR [rbp-16]
push rdi
mov edi, eax
call a(int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int)
add rsp, 112
add eax, ebx
mov rbx, QWORD PTR [rbp-8]
leave
ret
I have a few questions about this. First, I noticed that it sort of seems to switch how it handles the args as the number increases:
push rbp
mov rbp, rsp
mov DWORD PTR [rbp-4], edi
...
imul eax, DWORD PTR [rbp-8]
...
Wondering what is going on there, why it is doing that. It seems to treat the first arg with push, then the next 8 or so with mov, then the remaining with imul only relative to the eax register. Wonder if there is a limit to how many args you can have.
The second thing I'm wondering is the following. Say instead of a(), the function b() called out to some "system function" or some other external library call. Wondering how does it know how to unpack the args. Well, I guess, nevermind, I assume the C compiler will compile into assembly/machine code all linked external libraries. So yeah nevermind there I guess.
The last thing is if the syscalls all have a set number of arguments, similar to how in x86 the max operands is 3. Or perhaps the syscalls can have any number of arguments. It seems they would want to limit it for performance, so they only use those earlier stage instructions like push and mov, instead of imul.
Thanks for the help, just looking for clarification on how arguments are handled when calling functions in assembly, especially when there are a large number of arguments.

Answer 1

You need to learn more about stack frame and application binary interface (ABI, or calling convention). ABI defines the way parameters are passed by the caller to the callee, which registers are volatile, as well as how the stack is cleaned up.
Many ABIs exist, because anyone can design their own ABI as long as the caller and the callee agree. However, only a few ABIs are widely used. On Windows most 32-bit programs use stdcall, cdecl, Microsoft's fastcall or Borland's fastcall, whereas 64-bit programs mostly use Microsoft x64 calling convention. On Unix 64-bit programs always use System V AMD64 ABI, which is also the one used by your compiler.
Let's look at your code, with comments:
push rbp ; save the old stack frame
mov rbp, rsp ; establish new stack frame
mov DWORD PTR [rbp-4], edi ; save the first six arguments
mov DWORD PTR [rbp-8], esi
mov DWORD PTR [rbp-12], edx
mov DWORD PTR [rbp-16], ecx
mov DWORD PTR [rbp-20], r8d
mov DWORD PTR [rbp-24], r9d
mov eax, DWORD PTR [rbp-4] ; load n1
imul eax, DWORD PTR [rbp-8] ; eax = eax * n2
imul eax, DWORD PTR [rbp-12] ; eax = eax * n3
imul eax, DWORD PTR [rbp-16]
imul eax, DWORD PTR [rbp-20]
imul eax, DWORD PTR [rbp-24]
imul eax, DWORD PTR [rbp+16] ; eax = eax * n7
imul eax, DWORD PTR [rbp+24] ; eax = eax * n8
imul eax, DWORD PTR [rbp+32]
imul eax, DWORD PTR [rbp+40]
imul eax, DWORD PTR [rbp+48]
imul eax, DWORD PTR [rbp+56]
imul eax, DWORD PTR [rbp+64]
imul eax, DWORD PTR [rbp+72]
imul eax, DWORD PTR [rbp+80]
imul eax, DWORD PTR [rbp+88]
imul eax, DWORD PTR [rbp+96]
imul eax, DWORD PTR [rbp+104]
imul eax, DWORD PTR [rbp+112]
imul eax, DWORD PTR [rbp+120]
pop rbp ; restore old stack frame
ret ; exit
Note: the first two line has nothing to do with the arguments; they are to create a stack frame, so that you can easily access local variables and arguments. Without the stack frame you can still access them with [rsp+*], but the offset need be adjusted according to any PUSH and POP you used.
Following that are instructions to store the arguments to local variables. Registers are frequently changed and arguments passed in registers need to be stored, in case you need to use them later. In this case, however, is not necessary. So the optimized code can be
push rbp ; save the old stack frame
mov rbp, rsp ; establish new stack frame
mov eax, edi ; eax = n1
imul eax, esi ; eax = eax * n2
imul eax, edx ; eax = eax * n3
imul eax, ecx ; eax = eax * n4
imul eax, e8d ; eax = eax * n5
imul eax, e9d ; eax = eax * n6
imul eax, DWORD PTR [rbp+16] ; eax = eax * n7
imul eax, DWORD PTR [rbp+24] ; eax = eax * n8
imul eax, DWORD PTR [rbp+32] ; eax = eax * n9
imul eax, DWORD PTR [rbp+40]
imul eax, DWORD PTR [rbp+48]
imul eax, DWORD PTR [rbp+56]
imul eax, DWORD PTR [rbp+64]
imul eax, DWORD PTR [rbp+72]
imul eax, DWORD PTR [rbp+80]
imul eax, DWORD PTR [rbp+88]
imul eax, DWORD PTR [rbp+96]
imul eax, DWORD PTR [rbp+104]
imul eax, DWORD PTR [rbp+112]
imul eax, DWORD PTR [rbp+120]
pop rbp ; restore old stack frame
ret ; exit
From the above example, you should be able to guess that the first argument is passed in edi (or rdi, di, dil depends on the size), the second one is in esi, and then edx, ecx, r8d and r9d (Integer only, floats are passed in vector registers). When you have more than 6 arguments, the other are pushed on to the stack and can be accessed using [rbp+16], [rbp+24], .... ([rbp+8] is old rbp; [rbp] is returning address).
For the caller
mov r9d, DWORD PTR [rbp-12] ; r9d = n6
mov r8d, DWORD PTR [rbp-12] ; r8d = n5
mov ecx, DWORD PTR [rbp-16] ; ecx = n4
mov edx, DWORD PTR [rbp-12] ; edx = n3
mov esi, DWORD PTR [rbp-16] ; esi = n2
mov eax, DWORD PTR [rbp-12] ; eax = n1 ; will assign to edi
mov edi, DWORD PTR [rbp-12] ; push n20
push rdi
mov edi, DWORD PTR [rbp-16] ; push n19
push rdi
mov edi, DWORD PTR [rbp-12]
push rdi
mov edi, DWORD PTR [rbp-16]
push rdi
mov edi, DWORD PTR [rbp-12]
push rdi
mov edi, DWORD PTR [rbp-12]
push rdi
mov edi, DWORD PTR [rbp-16]
push rdi
mov edi, DWORD PTR [rbp-12]
push rdi
mov edi, DWORD PTR [rbp-16]
push rdi
mov edi, DWORD PTR [rbp-12]
push rdi
mov edi, DWORD PTR [rbp-12]
push rdi
mov edi, DWORD PTR [rbp-16]
push rdi
mov edi, DWORD PTR [rbp-12]
push rdi
mov edi, DWORD PTR [rbp-16] ; push n7
push rdi
mov edi, eax ; edi = n1
call a() ; call the function
add rsp, 112 ; clean up the stack, 14 * 8 = 112 bytes
mov ebx, eax ; result is in eax
a more straightforward version is
mov r9d, DWORD PTR [rbp-12] ; r9d = n6
mov r8d, DWORD PTR [rbp-12] ; r8d = n5
mov ecx, DWORD PTR [rbp-16] ; ecx = n4
mov edx, DWORD PTR [rbp-12] ; edx = n3
mov esi, DWORD PTR [rbp-16] ; esi = n2
mov edi, DWORD PTR [rbp-12] ; edi = n1
push [rbp-12] ; push n20
push [rbp-16] ; push n19
push [rbp-12]
push [rbp-16]
push [rbp-12]
push [rbp-16]
push [rbp-12]
push [rbp-16]
push [rbp-12]
push [rbp-16]
push [rbp-12]
push [rbp-16]
push [rbp-12] ; push n8
push [rbp-16] ; push n7
call a() ; call the function
add rsp, 112 ; clean up the stack, 14 * 8 = 112 bytes
mov ebx, eax ; result is in eax
Note that the arguments are pushed in reversed order.
Since you can push any number to the stack (before it overflows) before you call the function, there is no limitation on the number of arguments.