From: jbeulich@novell.com Subject: fix unwind annotations Patch-mainline: tbd References: bnc#472783, bnc#588458 --- arch/x86/kernel/entry_64.S | 131 +++++++++++++++++++++++---------------------- arch/x86/kernel/head_64.S | 13 ++++ lib/rwsem_64.S | 56 ++++++++++++++--------- 3 files changed, 114 insertions(+), 84 deletions(-) --- a/arch/x86/kernel/entry_64.S +++ b/arch/x86/kernel/entry_64.S @@ -234,21 +234,21 @@ ENDPROC(native_usergs_sysret64) /* * initial frame state for interrupts (and exceptions without error code) */ - .macro EMPTY_FRAME start=1 offset=0 - .if \start + .macro EMPTY_FRAME offset=0 CFI_STARTPROC simple CFI_SIGNAL_FRAME - CFI_DEF_CFA rsp,8+\offset - .else - CFI_DEF_CFA_OFFSET 8+\offset - .endif + CFI_DEF_CFA rsp,\offset .endm /* * initial frame state for interrupts (and exceptions without error code) */ .macro INTR_FRAME start=1 offset=0 - EMPTY_FRAME \start, SS+8+\offset-RIP + .if \start + EMPTY_FRAME SS+8+\offset-RIP + .else + CFI_DEF_CFA_OFFSET SS+8+\offset-RIP + .endif /*CFI_REL_OFFSET ss, SS+\offset-RIP*/ CFI_REL_OFFSET rsp, RSP+\offset-RIP /*CFI_REL_OFFSET rflags, EFLAGS+\offset-RIP*/ @@ -262,14 +262,15 @@ ENDPROC(native_usergs_sysret64) */ .macro XCPT_FRAME start=1 offset=0 INTR_FRAME \start, RIP+\offset-ORIG_RAX - /*CFI_REL_OFFSET orig_rax, ORIG_RAX-ORIG_RAX*/ .endm /* * frame that enables calling into C. */ .macro PARTIAL_FRAME start=1 offset=0 + .if \start >= 0 XCPT_FRAME \start, ORIG_RAX+\offset-ARGOFFSET + .endif CFI_REL_OFFSET rdi, RDI+\offset-ARGOFFSET CFI_REL_OFFSET rsi, RSI+\offset-ARGOFFSET CFI_REL_OFFSET rdx, RDX+\offset-ARGOFFSET @@ -285,7 +286,9 @@ ENDPROC(native_usergs_sysret64) * frame that enables passing a complete pt_regs to a C function. */ .macro DEFAULT_FRAME start=1 offset=0 + .if \start >= -1 PARTIAL_FRAME \start, R11+\offset-R15 + .endif CFI_REL_OFFSET rbx, RBX+\offset CFI_REL_OFFSET rbp, RBP+\offset CFI_REL_OFFSET r12, R12+\offset @@ -297,25 +300,27 @@ ENDPROC(native_usergs_sysret64) /* save partial stack frame */ .pushsection .kprobes.text, "ax" ENTRY(save_args) - XCPT_FRAME + XCPT_FRAME offset=ORIG_RAX-RBP+8 cld /* * start from rbp in pt_regs and jump over * return address. */ movq_cfi rdi, RDI+8-RBP - movq_cfi rsi, RSI+8-RBP - movq_cfi rdx, RDX+8-RBP - movq_cfi rcx, RCX+8-RBP + movq %rsi, RSI+8-RBP(%rsp) + movq %rdx, RDX+8-RBP(%rsp) + movq %rcx, RCX+8-RBP(%rsp) movq_cfi rax, RAX+8-RBP - movq_cfi r8, R8+8-RBP - movq_cfi r9, R9+8-RBP - movq_cfi r10, R10+8-RBP - movq_cfi r11, R11+8-RBP + movq %r8, R8+8-RBP(%rsp) + movq %r9, R9+8-RBP(%rsp) + movq %r10, R10+8-RBP(%rsp) + movq %r11, R11+8-RBP(%rsp) leaq -RBP+8(%rsp),%rdi /* arg1 for handler */ movq_cfi rbp, 8 /* push %rbp */ leaq 8(%rsp), %rbp /* mov %rsp, %ebp */ + CFI_DEF_CFA_REGISTER rbp + CFI_ADJUST_CFA_OFFSET -8 testl $3, CS(%rdi) je 1f SWAPGS @@ -327,11 +332,10 @@ ENTRY(save_args) */ 1: incl PER_CPU_VAR(irq_count) jne 2f - popq_cfi %rax /* move return address... */ + popq %rax /* move return address... */ mov PER_CPU_VAR(irq_stack_ptr),%rsp - EMPTY_FRAME 0 - pushq_cfi %rbp /* backlink for unwinder */ - pushq_cfi %rax /* ... to the new stack */ + pushq %rbp /* backlink for unwinder */ + pushq %rax /* ... to the new stack */ /* * We entered an interrupt context - irqs are off: */ @@ -342,14 +346,14 @@ END(save_args) .popsection ENTRY(save_rest) - PARTIAL_FRAME 1 REST_SKIP+8 + CFI_STARTPROC movq 5*8+16(%rsp), %r11 /* save return address */ - movq_cfi rbx, RBX+16 - movq_cfi rbp, RBP+16 - movq_cfi r12, R12+16 - movq_cfi r13, R13+16 - movq_cfi r14, R14+16 - movq_cfi r15, R15+16 + movq %rbx, RBX+16(%rsp) + movq %rbp, RBP+16(%rsp) + movq %r12, R12+16(%rsp) + movq %r13, R13+16(%rsp) + movq %r14, R14+16(%rsp) + movq %r15, R15+16(%rsp) movq %r11, 8(%rsp) /* return address */ FIXUP_TOP_OF_STACK %r11, 16 ret @@ -359,23 +363,23 @@ END(save_rest) /* save complete stack frame */ .pushsection .kprobes.text, "ax" ENTRY(save_paranoid) - XCPT_FRAME 1 RDI+8 + XCPT_FRAME offset=ORIG_RAX-R15+8 cld - movq_cfi rdi, RDI+8 - movq_cfi rsi, RSI+8 + movq %rdi, RDI+8(%rsp) + movq %rsi, RSI+8(%rsp) movq_cfi rdx, RDX+8 movq_cfi rcx, RCX+8 movq_cfi rax, RAX+8 - movq_cfi r8, R8+8 - movq_cfi r9, R9+8 - movq_cfi r10, R10+8 - movq_cfi r11, R11+8 + movq %r8, R8+8(%rsp) + movq %r9, R9+8(%rsp) + movq %r10, R10+8(%rsp) + movq %r11, R11+8(%rsp) movq_cfi rbx, RBX+8 - movq_cfi rbp, RBP+8 - movq_cfi r12, R12+8 - movq_cfi r13, R13+8 - movq_cfi r14, R14+8 - movq_cfi r15, R15+8 + movq %rbp, RBP+8(%rsp) + movq %r12, R12+8(%rsp) + movq %r13, R13+8(%rsp) + movq %r14, R14+8(%rsp) + movq %r15, R15+8(%rsp) movl $1,%ebx movl $MSR_GS_BASE,%ecx rdmsr @@ -677,7 +681,7 @@ ENTRY(\label) subq $REST_SKIP, %rsp CFI_ADJUST_CFA_OFFSET REST_SKIP call save_rest - DEFAULT_FRAME 0 8 /* offset 8: return address */ + DEFAULT_FRAME -2 8 /* offset 8: return address */ leaq 8(%rsp), \arg /* pt_regs pointer */ call \func jmp ptregscall_common @@ -794,7 +798,9 @@ END(interrupt) subq $ORIG_RAX-RBP, %rsp CFI_ADJUST_CFA_OFFSET ORIG_RAX-RBP call save_args - PARTIAL_FRAME 0 + PARTIAL_FRAME -1 ARGOFFSET-RBP + CFI_REL_OFFSET rbp, 0 + CFI_DEF_CFA_REGISTER rbp call \func .endm @@ -813,7 +819,6 @@ ret_from_intr: TRACE_IRQS_OFF decl PER_CPU_VAR(irq_count) leaveq - CFI_RESTORE rbp CFI_DEF_CFA_REGISTER rsp CFI_ADJUST_CFA_OFFSET -8 @@ -1021,7 +1026,7 @@ ENTRY(\sym) subq $ORIG_RAX-R15, %rsp CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15 call error_entry - DEFAULT_FRAME 0 + DEFAULT_FRAME -1 movq %rsp,%rdi /* pt_regs pointer */ xorl %esi,%esi /* no error code */ call \do_sym @@ -1038,6 +1043,7 @@ ENTRY(\sym) subq $ORIG_RAX-R15, %rsp CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15 call save_paranoid + DEFAULT_FRAME -1 TRACE_IRQS_OFF movq %rsp,%rdi /* pt_regs pointer */ xorl %esi,%esi /* no error code */ @@ -1056,6 +1062,7 @@ ENTRY(\sym) subq $ORIG_RAX-R15, %rsp CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15 call save_paranoid + DEFAULT_FRAME -1 TRACE_IRQS_OFF movq %rsp,%rdi /* pt_regs pointer */ xorl %esi,%esi /* no error code */ @@ -1074,7 +1081,7 @@ ENTRY(\sym) subq $ORIG_RAX-R15, %rsp CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15 call error_entry - DEFAULT_FRAME 0 + DEFAULT_FRAME -1 movq %rsp,%rdi /* pt_regs pointer */ movq ORIG_RAX(%rsp),%rsi /* get error code */ movq $-1,ORIG_RAX(%rsp) /* no syscall to restart */ @@ -1092,7 +1099,7 @@ ENTRY(\sym) subq $ORIG_RAX-R15, %rsp CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15 call save_paranoid - DEFAULT_FRAME 0 + DEFAULT_FRAME -1 TRACE_IRQS_OFF movq %rsp,%rdi /* pt_regs pointer */ movq ORIG_RAX(%rsp),%rsi /* get error code */ @@ -1435,25 +1442,24 @@ END(paranoid_exit) * returns in "no swapgs flag" in %ebx. */ ENTRY(error_entry) - XCPT_FRAME - CFI_ADJUST_CFA_OFFSET 15*8 + XCPT_FRAME offset=ORIG_RAX-R15+8 /* oldrax contains error code */ cld - movq_cfi rdi, RDI+8 - movq_cfi rsi, RSI+8 - movq_cfi rdx, RDX+8 - movq_cfi rcx, RCX+8 - movq_cfi rax, RAX+8 - movq_cfi r8, R8+8 - movq_cfi r9, R9+8 - movq_cfi r10, R10+8 - movq_cfi r11, R11+8 + movq %rdi, RDI+8(%rsp) + movq %rsi, RSI+8(%rsp) + movq %rdx, RDX+8(%rsp) + movq %rcx, RCX+8(%rsp) + movq %rax, RAX+8(%rsp) + movq %r8, R8+8(%rsp) + movq %r9, R9+8(%rsp) + movq %r10, R10+8(%rsp) + movq %r11, R11+8(%rsp) movq_cfi rbx, RBX+8 - movq_cfi rbp, RBP+8 - movq_cfi r12, R12+8 - movq_cfi r13, R13+8 - movq_cfi r14, R14+8 - movq_cfi r15, R15+8 + movq %rbp, RBP+8(%rsp) + movq %r12, R12+8(%rsp) + movq %r13, R13+8(%rsp) + movq %r14, R14+8(%rsp) + movq %r15, R15+8(%rsp) xorl %ebx,%ebx testl $3,CS+8(%rsp) je error_kernelspace @@ -1471,6 +1477,7 @@ error_sti: * compat mode. Check for these here too. */ error_kernelspace: + CFI_REL_OFFSET rcx, RCX+8 incl %ebx leaq irq_return(%rip),%rcx cmpq %rcx,RIP+8(%rsp) @@ -1518,7 +1523,7 @@ ENTRY(nmi) subq $ORIG_RAX-R15, %rsp CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15 call save_paranoid - DEFAULT_FRAME 0 + DEFAULT_FRAME -1 /* paranoidentry do_nmi, 0; without TRACE_IRQS_OFF */ movq %rsp,%rdi movq $-1,%rsi --- a/arch/x86/kernel/head_64.S +++ b/arch/x86/kernel/head_64.S @@ -284,6 +284,8 @@ early_idt_handlers: ENTRY(early_idt_handler) #ifdef CONFIG_EARLY_PRINTK +#include +#include cmpl $2,early_recursion_flag(%rip) jz 1f incl early_recursion_flag(%rip) @@ -299,6 +301,16 @@ ENTRY(early_idt_handler) testl $0x27d00,%eax je 0f popq %r8 # get error code + + CFI_STARTPROC simple + CFI_SIGNAL_FRAME + CFI_DEF_CFA rsp, SS+8-RIP +# CFI_REL_OFFSET ss, SS-RIP + CFI_REL_OFFSET rsp, RSP-RIP +# CFI_REL_OFFSET rflags, EFLAGS-RIP +# CFI_REL_OFFSET cs, CS-RIP + CFI_REL_OFFSET rip, RIP-RIP + 0: movq 0(%rsp),%rcx # get ip movq 8(%rsp),%rdx # get cs xorl %eax,%eax @@ -312,6 +324,7 @@ ENTRY(early_idt_handler) movq 0(%rsp),%rsi # get rip again call __print_symbol #endif + CFI_ENDPROC #endif /* EARLY_PRINTK */ 1: hlt jmp 1b --- a/arch/x86/lib/rwsem_64.S +++ b/arch/x86/lib/rwsem_64.S @@ -23,43 +23,50 @@ #include #define save_common_regs \ - pushq %rdi; \ - pushq %rsi; \ - pushq %rcx; \ - pushq %r8; \ - pushq %r9; \ - pushq %r10; \ - pushq %r11 + pushq_cfi %rdi; CFI_REL_OFFSET rdi, 0; \ + pushq_cfi %rsi; CFI_REL_OFFSET rsi, 0; \ + pushq_cfi %rcx; CFI_REL_OFFSET rcx, 0; \ + pushq_cfi %r8; CFI_REL_OFFSET r8, 0; \ + pushq_cfi %r9; CFI_REL_OFFSET r9, 0; \ + pushq_cfi %r10; CFI_REL_OFFSET r10, 0; \ + pushq_cfi %r11; CFI_REL_OFFSET r11, 0 #define restore_common_regs \ - popq %r11; \ - popq %r10; \ - popq %r9; \ - popq %r8; \ - popq %rcx; \ - popq %rsi; \ - popq %rdi + popq_cfi %r11; CFI_RESTORE r11; \ + popq_cfi %r10; CFI_RESTORE r10; \ + popq_cfi %r9; CFI_RESTORE r9; \ + popq_cfi %r8; CFI_RESTORE r8; \ + popq_cfi %rcx; CFI_RESTORE rcx; \ + popq_cfi %rsi; CFI_RESTORE rsi; \ + popq_cfi %rdi; CFI_RESTORE rdi /* Fix up special calling conventions */ ENTRY(call_rwsem_down_read_failed) + CFI_STARTPROC save_common_regs - pushq %rdx + pushq_cfi %rdx + CFI_REL_OFFSET rdx, 0 movq %rax,%rdi call rwsem_down_read_failed - popq %rdx + popq_cfi %rdx + CFI_RESTORE rdx restore_common_regs ret - ENDPROC(call_rwsem_down_read_failed) + CFI_ENDPROC +ENDPROC(call_rwsem_down_read_failed) ENTRY(call_rwsem_down_write_failed) + CFI_STARTPROC save_common_regs movq %rax,%rdi call rwsem_down_write_failed restore_common_regs ret - ENDPROC(call_rwsem_down_write_failed) + CFI_ENDPROC +ENDPROC(call_rwsem_down_write_failed) ENTRY(call_rwsem_wake) + CFI_STARTPROC decl %edx /* do nothing if still outstanding active readers */ jnz 1f save_common_regs @@ -67,15 +74,20 @@ ENTRY(call_rwsem_wake) call rwsem_wake restore_common_regs 1: ret - ENDPROC(call_rwsem_wake) + CFI_ENDPROC +ENDPROC(call_rwsem_wake) /* Fix up special calling conventions */ ENTRY(call_rwsem_downgrade_wake) + CFI_STARTPROC save_common_regs - pushq %rdx + pushq_cfi %rdx + CFI_REL_OFFSET rdx, 0 movq %rax,%rdi call rwsem_downgrade_wake - popq %rdx + popq_cfi %rdx + CFI_RESTORE rdx restore_common_regs ret - ENDPROC(call_rwsem_downgrade_wake) + CFI_ENDPROC +ENDPROC(call_rwsem_downgrade_wake)