Calling Convention

Abhijit A M
The need for calling convention

An essential task of the compiler

Generates object code (file) for given source code (file)
Processors provide simple features
Registers, machine instructions (add, mov, jmp, call, etc.), imp registers like stack-pointer, etc; ability to do byte/word
sized operations
No notion of data types, functions, variables, etc.

But languages like C provide high level features

Data types, variables, functions, recursion, etc

Compiler needs to map the features of C into processor’s features, and then generate machine
code
In reality, the language designers design a language feature only after answering the question of
conversion into machine code
The need for calling convention

Examples of some of the challenges before the compiler

“call” + “ret” does not make a C-function call!
A “call” instruction in processor simply does this
Pushes IP(that is PC) on stack + Jumps to given address
This is not like calling a C-function !
Unsolved problem: How to handle parameters, return value?

Processor does not understand data types!

Although it has instructions for byte, word sized data and can differentiate between
integers and reals (mov, movw, addl, addf, etc. )
Compiler and Machine code generation

Example, code inside a function

int a, b, c;
c = a + b;
What kind of code is generated by compiler for this?
sub 12, <esp> #normally local variables are located on stack, make space
mov <location of a in memory>, r1 #location is on stack, e,g. -4(esp)
mov <location of b in memory>, r2
add r1, r2 # result in r1
mov r1, <location of c in memory>
Compiler and Machine code generation
Across function calls
g() may be called from f() or
int f(int m, n) {
int x = m, y = n;
from x()
return g(x, y);
Sequence of function calls can
}
int x(int a) {
NOT be predicted by compiler
return g (a, a+ 1);
Compiler has to generate
}
int g(int p, int q) {
machine code for each function
p = p * q + p;
assuming nothing about the
return p caller
}
Compiler and Machine code generation

Machine code generation for functions

Mapping C language features to existing machine code instructions.
Typical examples
a =100 ; ==> mov instruction
a = b+ c; ==> mov, add instructions
while(a < 5) { j++; } ==> mov, cmp, jlt, add, etc. Instruction

Where are the local variables in memory?

The only way to store them is on a stack.
Why?
Function calls

LIFO
Last in First Out
Must need a “stack” like feature to implement them
Processor Stack
Processors provide a stack pointer
%esp on x86
Instructions like push and pop are provided by hardware
They automatically increment/decrement the stack pointer. On x86 stack grows downwards (substract from esp!)
Unlike a “stack data type” data structure, this “stack” is simply implemented with only the esp (as the “top”). The
entire memory can be treated as the “array”.
Function calls

System stack, compilers, Languages

Compilers generate machine code with the ‘esp’. The pointer is initialized to a proper value at the time of
fork-exec by the OS for each process.
Langauges like C which provide for function calls, and recursion also assume that they will run on
processors with a stack support in hardware
Convention needed
How to use the stack for effective implementation of function calls ?
What goes on stack?
Local variables
Function Parameters
Return address of instruction which called a function !
Activation Record

Local Vars + parameters + return address

When functions call each other
One activation record is built on stack for each function call
On function return, the record is destroyed
On x86
ebp and esp pointers are used to denote the activation record.
How? We will see soon. You may start exploring with “gcc -S” output assembly
code.
X86 instructions
leave call x
Equivalent to Equivalent to
mov %ebp, %esp # esp = ebp push %eip
pop %ebp jmp x
ret
Equivalent to
pop %ecx
Jmp %ecx
X86 instructions

endbr64
Normally a NOP
Let’s see some examples now

Let’s compile using

gcc -S
See code and understand
 simple.c and simple.s
main: f:
int f(int x) { endbr64 endbr64
int y; pushq %rbp pushq %rbp
y = x + 3; movq %rsp, %rbp movq %rsp, %rbp
return y; subq $16, %rsp movl %edi, -20(%rbp)
} movl $20, -8(%rbp) movl -20(%rbp), %eax
int main() { movl $30, -4(%rbp) addl $3, %eax
int a = 20, b = 30; movl -8(%rbp), %eax movl %eax, -4(%rbp)
b = f(a); movl %eax, %edi movl -4(%rbp), %eax
return b; call f popq %rbp
} movl %eax, -4(%rbp) ret
movl -4(%rbp), %eax
leave
ret
 main:
X rbp
endbr64
pushq %rbp
Y rsp movq %rsp, %rbp
subq $16, %rsp
movl $20, -8(%rbp)
movl $30, -4(%rbp)
movl -8(%rbp), %eax
movl %eax, %edi
call f
movl %eax, -4(%rbp)
movl -4(%rbp), %eax
leave
ret

int main() {
int a = 20, b = 30;
b = f(a);
return b;
}
 main:
X ebp
rbp
endbr64
pushq %rbp
Y movq %rsp, %rbp
X
subq $16, %rsp
movl $20, -8(%rbp)
rsp movl $30, -4(%rbp)
movl -8(%rbp), %eax
movl %eax, %edi
call f
movl %eax, -4(%rbp)
movl -4(%rbp), %eax
leave
ret

int main() {
int a = 20, b = 30;
b = f(a);
return b;
}
 main:
X ebp
rbp
endbr64
pushq %rbp
Y movq %rsp, %rbp
X
subq $16, %rsp
movl $20, -8(%rbp)
rsp movl $30, -4(%rbp)
movl -8(%rbp), %eax
movl %eax, %edi
call f
movl %eax, -4(%rbp)
movl -4(%rbp), %eax
leave
ret

int main() {
int a = 20, b = 30;
b = f(a);
return b;
}
 main:
X ebp
rbp
endbr64
pushq %rbp
Y movq %rsp, %rbp
X
subq $16, %rsp
movl $20, -8(%rbp)
rsp movl $30, -4(%rbp)
movl -8(%rbp), %eax
movl %eax, %edi
call f
movl %eax, -4(%rbp)
movl -4(%rbp), %eax
leave
ret

int main() {
int a = 20, b = 30;
b = f(a);
return b;
}
 main:
X ebp
rbp
endbr64
pushq %rbp
Y movq %rsp, %rbp
X
subq $16, %rsp
b 30 movl $20, -8(%rbp)
a 20 rsp movl $30, -4(%rbp)
movl -8(%rbp), %eax
movl %eax, %edi
call f
movl %eax, -4(%rbp)
movl -4(%rbp), %eax
leave
ret

int main() {
int a = 20, b = 30;
b = f(a);
return b;
}
 main:
X ebp
rbp
endbr64
pushq %rbp
Y movq %rsp, %rbp
X
subq $16, %rsp
b 30 movl $20, -8(%rbp)
a 20 rsp movl $30, -4(%rbp)
movl -8(%rbp), %eax
movl %eax, %edi
call f
movl %eax, -4(%rbp)
edi = 20 movl -4(%rbp), %eax
leave
ret

int main() {
int a = 20, b = 30;
b = f(a);
return b;
}
 main: f:
X ebp
rbp
endbr64 endbr64
pushq %rbp pushq %rbp
Y movq %rsp, %rbp movq %rsp, %rbp
X
subq $16, %rsp movl %edi, -20(%rbp)
b 30 movl $20, -8(%rbp) movl -20(%rbp), %eax
a 20 rsp movl $30, -4(%rbp) addl $3, %eax
movl -8(%rbp), %eax movl %eax, -4(%rbp)
RA movl %eax, %edi movl -4(%rbp), %eax
call f popq %rbp
RA: ret
movl %eax, -4(%rbp)
movl -4(%rbp), %eax
Leave edi = 20
ret
int f(int x) { int main() {
int y; int a = 20, b = 30;
y = x + 3; b = f(a);
return y; return b;
} }
 main: f:
X ebp
rbp
endbr64 endbr64
pushq %rbp pushq %rbp
Y movq %rsp, %rbp movq %rsp, %rbp
X
subq $16, %rsp movl %edi, -20(%rbp)
b 30 movl $20, -8(%rbp) movl -20(%rbp), %eax
a 20 rsp movl $30, -4(%rbp) addl $3, %eax
movl -8(%rbp), %eax movl %eax, -4(%rbp)
RA movl %eax, %edi movl -4(%rbp), %eax
Y-4 call f popq %rbp
RA: ret
movl %eax, -4(%rbp)
movl -4(%rbp), %eax
Leave edi = 20
ret
int f(int x) { int main() {
int y; int a = 20, b = 30;
y = x + 3; b = f(a);
return y; return b;
} }
 main: f:
X ebp
rbp
endbr64 endbr64
pushq %rbp pushq %rbp
Y movq %rsp, %rbp movq %rsp, %rbp
X
subq $16, %rsp movl %edi, -20(%rbp)
b 30 movl $20, -8(%rbp) movl -20(%rbp), %eax
a 20 rsp movl $30, -4(%rbp) addl $3, %eax
movl -8(%rbp), %eax movl %eax, -4(%rbp)
RA movl %eax, %edi movl -4(%rbp), %eax
Y-4 call f popq %rbp
RA: ret
movl %eax, -4(%rbp)
movl -4(%rbp), %eax
Leave edi = 20
ret
int f(int x) { int main() {
int y; int a = 20, b = 30;
y = x + 3; b = f(a);
return y; return b;
} }
 main: f:
X ebp
rbp
endbr64 endbr64
pushq %rbp pushq %rbp
Y movq %rsp, %rbp movq %rsp, %rbp
X
subq $16, %rsp movl %edi, -20(%rbp)
b 30 movl $20, -8(%rbp) movl -20(%rbp), %eax
a 20 rsp movl $30, -4(%rbp) addl $3, %eax
movl -8(%rbp), %eax movl %eax, -4(%rbp)
RA movl %eax, %edi movl -4(%rbp), %eax
Y-4 call f popq %rbp
RA: ret
movl %eax, -4(%rbp)
movl -4(%rbp), %eax
x 20 Leave edi = 20
ret
int f(int x) { int main() {
int y; int a = 20, b = 30;
y = x + 3; b = f(a);
return y; return b;
} }
 main: f:
X ebp
rbp
endbr64 endbr64
pushq %rbp pushq %rbp
Y movq %rsp, %rbp movq %rsp, %rbp
X
subq $16, %rsp movl %edi, -20(%rbp)
b 30 movl $20, -8(%rbp) movl -20(%rbp), %eax
a 20 rsp movl $30, -4(%rbp) addl $3, %eax
movl -8(%rbp), %eax movl %eax, -4(%rbp)
RA movl %eax, %edi movl -4(%rbp), %eax
Y-4 call f popq %rbp
RA: ret
movl %eax, -4(%rbp)
movl -4(%rbp), %eax
x 20 Leave edi = 20
ret
int main() { eax = 20
int f(int x) {
int y; int a = 20, b = 30;
y = x + 3; b = f(a);
return y; return b;
} }
 main: f:
X ebp
rbp
endbr64 endbr64
pushq %rbp pushq %rbp
Y movq %rsp, %rbp movq %rsp, %rbp
X
subq $16, %rsp movl %edi, -20(%rbp)
b 30 movl $20, -8(%rbp) movl -20(%rbp), %eax
a 20 rsp movl $30, -4(%rbp) addl $3, %eax
movl -8(%rbp), %eax movl %eax, -4(%rbp)
RA movl %eax, %edi movl -4(%rbp), %eax
Y-4 call f popq %rbp
RA: ret
movl %eax, -4(%rbp)
movl -4(%rbp), %eax
x 20 Leave edi = 20
ret
int main() { eax = 23
int f(int x) {
int y; int a = 20, b = 30;
y = x + 3; b = f(a);
return y; return b;
} }
 main: f:
X ebp
rbp
endbr64 endbr64
pushq %rbp pushq %rbp
Y movq %rsp, %rbp movq %rsp, %rbp
X
subq $16, %rsp movl %edi, -20(%rbp)
b 30 movl $20, -8(%rbp) movl -20(%rbp), %eax
a 20 rsp movl $30, -4(%rbp) addl $3, %eax
movl -8(%rbp), %eax movl %eax, -4(%rbp)
RA movl %eax, %edi movl -4(%rbp), %eax
Y-4 call f popq %rbp
y 23 RA: ret
movl %eax, -4(%rbp)
movl -4(%rbp), %eax
x 20 Leave edi = 20
ret
int main() { eax = 23
int f(int x) {
int y; int a = 20, b = 30;
y = x + 3; b = f(a);
return y; return b;
} }
 main: f:
X ebp
rbp
endbr64 endbr64
pushq %rbp pushq %rbp
Y movq %rsp, %rbp movq %rsp, %rbp
X
subq $16, %rsp movl %edi, -20(%rbp)
b 30 movl $20, -8(%rbp) movl -20(%rbp), %eax
a 20 rsp movl $30, -4(%rbp) addl $3, %eax
movl -8(%rbp), %eax movl %eax, -4(%rbp)
RA movl %eax, %edi movl -4(%rbp), %eax
Y-4 call f popq %rbp
y 23 RA: ret
movl %eax, -4(%rbp)
movl -4(%rbp), %eax edi = 20
x 20 Leave
ret Eax = 23
int f(int x) { int main() {
int y; int a = 20, b = 30;
y = x + 3; b = f(a);
return y; return b;
} }
 main: f:
X ebp
rbp
endbr64 endbr64
pushq %rbp pushq %rbp
Y movq %rsp, %rbp movq %rsp, %rbp
X
subq $16, %rsp movl %edi, -20(%rbp)
b 30 movl $20, -8(%rbp) movl -20(%rbp), %eax
a 20 rsp movl $30, -4(%rbp) addl $3, %eax
movl -8(%rbp), %eax movl %eax, -4(%rbp)
RA movl %eax, %edi movl -4(%rbp), %eax
Y-4 call f popq %rbp
y 23 RA: ret
movl %eax, -4(%rbp)
movl -4(%rbp), %eax edi = 20
x 20 Leave
ret eax = 23
int f(int x) { int main() {
int y; int a = 20, b = 30;
y = x + 3; b = f(a);
return y; return b; eip = RA
} }
 main: f:
X ebp
rbp
endbr64 endbr64
pushq %rbp pushq %rbp
Y movq %rsp, %rbp movq %rsp, %rbp
X
subq $16, %rsp movl %edi, -20(%rbp)
b 23 movl $20, -8(%rbp) movl -20(%rbp), %eax
a 20 rsp movl $30, -4(%rbp) addl $3, %eax
movl -8(%rbp), %eax movl %eax, -4(%rbp)
RA movl %eax, %edi movl -4(%rbp), %eax
Y-4 call f popq %rbp
y 23 RA: ret
movl %eax, -4(%rbp)
movl -4(%rbp), %eax edi = 20
x 20 leave
ret eax = 23
int f(int x) { int main() {
int y; int a = 20, b = 30;
y = x + 3; b = f(a);
return y; return b; eip = RA
} }
 main:
endbr64 f:
X ebp
rbp
pushq %rbp endbr64
movq %rsp, %rbp pushq %rbp
Y subq $16, %rsp movq %rsp, %rbp
X
movl $20, -8(%rbp) movl %edi, -20(%rbp)
b 23 movl $30, -4(%rbp) movl -20(%rbp), %eax
a 20 rsp movl -8(%rbp), %eax addl $3, %eax
movl %eax, %edi movl %eax, -4(%rbp)
RA call f movl -4(%rbp), %eax
Y-4 RA: popq %rbp
y 23 movl %eax, -4(%rbp) ret
movl -4(%rbp), %eax
leave edi = 20
x 20 # mov rbp rsp; pop rbp
ret eax = 23
int f(int x) { int main() {
int y; int a = 20, b = 30;
y = x + 3; b = f(a);
return y; return b; eip = RA
} }
 main:
endbr64 f:
X ebp
rbp
pushq %rbp endbr64
movq %rsp, %rbp pushq %rbp
Y subq $16, %rsp movq %rsp, %rbp
X
movl $20, -8(%rbp) movl %edi, -20(%rbp)
b 23 movl $30, -4(%rbp) movl -20(%rbp), %eax
a 20 rsp movl -8(%rbp), %eax addl $3, %eax
movl %eax, %edi movl %eax, -4(%rbp)
RA call f movl -4(%rbp), %eax
Y-4 RA: popq %rbp
y 23 movl %eax, -4(%rbp) ret
movl -4(%rbp), %eax
leave edi = 20
x 20 # mov ebp esp; pop ebp
ret eax = 23
int f(int x) { int main() {
int y; int a = 20, b = 30;
y = x + 3; b = f(a);
return y; return b; eip = RA
} }
Further on calling convention

This was a simple program

The parameter was passed in a register!
What if there were many parameters?
CPUs have different numbers of registers.
More parameters, more functions demand a more sophisticated
convention
May be slightely different on different processors, or 32-bit, 64-bit
variants also.
Caller save and Callee save registers

Local variables
Are visible only within the function
Recursion: different copies of variables
Stored on “stack”
Registers
Are only one copy
Are within the CPU
Local Variables & Registers conflict
Compiler’s dillemma: While generating code for a function, which registers to use?
The register might have been in use in earlier function call
Caller save and Callee save registers

Caller Save registers

Which registers need to be saved by caller function . They can be used by the callee
function!
The caller function will push them (if already in use, otherwise no need) on the stack
Callee save registers
Will be pushed on to the stack by called (callee) function
How to return values?
On the stack itself – then caller will have to pop
In a register, e.g. eax
X86 convention – caller, callee saved
32 bit
The caller-saved registers are EAX, ECX, EDX.
The callee-saved registers are EBX, EDI, and ESI
Activation record looks like this

Higher Addresses
F() called g()
[ebp+16] Parameters-i refers to
parameter 3

parameter 2
[ebp+12] parameters passed by f() to g()
[ebp+8]
parameter 1
Local variable is a variable in
g()
(return address)
ebp
(saved value of ebp)
[ebp-4]
local variable 1
Return address is the location
saved value of edi

saved value of esi

esp
in f() where call should go back
Lower Addresses
X86 caller and callee rules(32 bit)

Caller rules on call

Push caller saved registers on stack
Push parameters on the stack – in reverse order. Why?
Substract esp, copy data

call f() // push + jmp

Caller rules on return
return value is in eax
remove parameters from stack : Add to esp.
Restore caller saved registers (if any)
X86 caller and callee rules

Callee rules on call

1) push ebp
mov ebp, esp
ebp(+-offset) normally used to locate local vars and parameters on stack
ebp holds a copy of esp
Ebp is pushed so that it can be later popped while returnig
2) Allocate local variables
3) Save callee-saved registers
https://www.cs.virginia.edu/~evans/cs216/guides/x86.html
X86 caller and callee rules

Callee rules on return

1) Leave return value in eax
2) Restore callee saved registers
3) Deallocate local variables
4) restore the ebp
5) return

https://www.cs.virginia.edu/~evans/cs216/guides/x86.html
32 bit vs 64 bit calling convention

Registers are used for passing parameters in 64 bit , to a large

extent
Upto 6 parameters
More parameters pushed on stack
See
https://aaronbloomfield.github.io/pdr/book/x86-64bit-ccc-chapter.pdf
Beware

When you read assembly code generated using

gcc -S
You will find
More complex instructions
But they will essentially follow the convention mentioned
Comparison

From the textbook by Misruda

 simple3.c and simple3.s
int f(int x1, int x2, int x3, int x4, int x5, int x6, int x7, int x8, int x9, int x10) {
int h;
h = x1 + x2 + x3 + x4 + x5 + x6 + x7 + x8 + x9 + x10 + 3;
return h;
}
int main() {
int a1 = 10, a2 = 20, a3 = 30, a4 = 40, a5 = 50, a6 = 60, a7 = 70, a8 = 80, a9 = 90, a10 = 100;
int b;
b = f(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10);
return b;
}
 simple3.c and simple3.s
main:
endbr64 movl -24(%rbp), %r9d
pushq %rbp movl -28(%rbp), %r8d
movq %rsp, %rbp movl -32(%rbp), %ecx movl %eax, %edi
subq $48, %rsp movl -36(%rbp), %edx call f
movl $10, -44(%rbp) movl -40(%rbp), %esi addq $32, %rsp
movl $20, -40(%rbp) movl -44(%rbp), %eax movl %eax, -4(%rbp)
movl $30, -36(%rbp) movl -8(%rbp), %edi movl -4(%rbp), %eax
movl $40, -32(%rbp) pushq %rdi leave
movl $50, -28(%rbp) movl -12(%rbp), %edi ret
movl $60, -24(%rbp) pushq %rdi
movl $70, -20(%rbp) movl -16(%rbp), %edi
movl $80, -16(%rbp) pushq %rdi
movl $90, -12(%rbp) movl -20(%rbp), %edi
movl $100, -8(%rbp) pushq %rdi
 simple3.c and simple3.s
f:
endbr64
pushq %rbp addl %eax, %edx
movq %rsp, %rbp movl -36(%rbp), %eax movl 40(%rbp), %eax
movl %edi, -20(%rbp) addl %eax, %edx addl %edx, %eax
movl %esi, -24(%rbp) movl -40(%rbp), %eax addl $3, %eax
movl %edx, -28(%rbp) addl %eax, %edx movl %eax, -4(%rbp)
movl %ecx, -32(%rbp) movl 16(%rbp), %eax movl -4(%rbp), %eax
movl %r8d, -36(%rbp) addl %eax, %edx popq %rbp
movl %r9d, -40(%rbp) movl 24(%rbp), %eax ret
movl -20(%rbp), %edx addl %eax, %edx
movl -24(%rbp), %eax movl 32(%rbp), %eax
addl %eax, %edx addl %eax, %edx
movl -28(%rbp), %eax
addl %eax, %edx
movl -32(%rbp), %eax