You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
440 lines
11 KiB
440 lines
11 KiB
From: jbeulich@novell.com
|
|
Subject: fix unwind annotations
|
|
Patch-mainline: tbd
|
|
References: bnc#472783, bnc#588458
|
|
|
|
---
|
|
arch/x86/kernel/entry_64.S | 131 +++++++++++++++++++++++----------------------
|
|
arch/x86/kernel/head_64.S | 13 ++++
|
|
lib/rwsem_64.S | 56 ++++++++++++++---------
|
|
3 files changed, 114 insertions(+), 84 deletions(-)
|
|
|
|
--- a/arch/x86/kernel/entry_64.S
|
|
+++ b/arch/x86/kernel/entry_64.S
|
|
@@ -234,21 +234,21 @@ ENDPROC(native_usergs_sysret64)
|
|
/*
|
|
* initial frame state for interrupts (and exceptions without error code)
|
|
*/
|
|
- .macro EMPTY_FRAME start=1 offset=0
|
|
- .if \start
|
|
+ .macro EMPTY_FRAME offset=0
|
|
CFI_STARTPROC simple
|
|
CFI_SIGNAL_FRAME
|
|
- CFI_DEF_CFA rsp,8+\offset
|
|
- .else
|
|
- CFI_DEF_CFA_OFFSET 8+\offset
|
|
- .endif
|
|
+ CFI_DEF_CFA rsp,\offset
|
|
.endm
|
|
|
|
/*
|
|
* initial frame state for interrupts (and exceptions without error code)
|
|
*/
|
|
.macro INTR_FRAME start=1 offset=0
|
|
- EMPTY_FRAME \start, SS+8+\offset-RIP
|
|
+ .if \start
|
|
+ EMPTY_FRAME SS+8+\offset-RIP
|
|
+ .else
|
|
+ CFI_DEF_CFA_OFFSET SS+8+\offset-RIP
|
|
+ .endif
|
|
/*CFI_REL_OFFSET ss, SS+\offset-RIP*/
|
|
CFI_REL_OFFSET rsp, RSP+\offset-RIP
|
|
/*CFI_REL_OFFSET rflags, EFLAGS+\offset-RIP*/
|
|
@@ -262,14 +262,15 @@ ENDPROC(native_usergs_sysret64)
|
|
*/
|
|
.macro XCPT_FRAME start=1 offset=0
|
|
INTR_FRAME \start, RIP+\offset-ORIG_RAX
|
|
- /*CFI_REL_OFFSET orig_rax, ORIG_RAX-ORIG_RAX*/
|
|
.endm
|
|
|
|
/*
|
|
* frame that enables calling into C.
|
|
*/
|
|
.macro PARTIAL_FRAME start=1 offset=0
|
|
+ .if \start >= 0
|
|
XCPT_FRAME \start, ORIG_RAX+\offset-ARGOFFSET
|
|
+ .endif
|
|
CFI_REL_OFFSET rdi, RDI+\offset-ARGOFFSET
|
|
CFI_REL_OFFSET rsi, RSI+\offset-ARGOFFSET
|
|
CFI_REL_OFFSET rdx, RDX+\offset-ARGOFFSET
|
|
@@ -285,7 +286,9 @@ ENDPROC(native_usergs_sysret64)
|
|
* frame that enables passing a complete pt_regs to a C function.
|
|
*/
|
|
.macro DEFAULT_FRAME start=1 offset=0
|
|
+ .if \start >= -1
|
|
PARTIAL_FRAME \start, R11+\offset-R15
|
|
+ .endif
|
|
CFI_REL_OFFSET rbx, RBX+\offset
|
|
CFI_REL_OFFSET rbp, RBP+\offset
|
|
CFI_REL_OFFSET r12, R12+\offset
|
|
@@ -297,25 +300,27 @@ ENDPROC(native_usergs_sysret64)
|
|
/* save partial stack frame */
|
|
.pushsection .kprobes.text, "ax"
|
|
ENTRY(save_args)
|
|
- XCPT_FRAME
|
|
+ XCPT_FRAME offset=ORIG_RAX-RBP+8
|
|
cld
|
|
/*
|
|
* start from rbp in pt_regs and jump over
|
|
* return address.
|
|
*/
|
|
movq_cfi rdi, RDI+8-RBP
|
|
- movq_cfi rsi, RSI+8-RBP
|
|
- movq_cfi rdx, RDX+8-RBP
|
|
- movq_cfi rcx, RCX+8-RBP
|
|
+ movq %rsi, RSI+8-RBP(%rsp)
|
|
+ movq %rdx, RDX+8-RBP(%rsp)
|
|
+ movq %rcx, RCX+8-RBP(%rsp)
|
|
movq_cfi rax, RAX+8-RBP
|
|
- movq_cfi r8, R8+8-RBP
|
|
- movq_cfi r9, R9+8-RBP
|
|
- movq_cfi r10, R10+8-RBP
|
|
- movq_cfi r11, R11+8-RBP
|
|
+ movq %r8, R8+8-RBP(%rsp)
|
|
+ movq %r9, R9+8-RBP(%rsp)
|
|
+ movq %r10, R10+8-RBP(%rsp)
|
|
+ movq %r11, R11+8-RBP(%rsp)
|
|
|
|
leaq -RBP+8(%rsp),%rdi /* arg1 for handler */
|
|
movq_cfi rbp, 8 /* push %rbp */
|
|
leaq 8(%rsp), %rbp /* mov %rsp, %ebp */
|
|
+ CFI_DEF_CFA_REGISTER rbp
|
|
+ CFI_ADJUST_CFA_OFFSET -8
|
|
testl $3, CS(%rdi)
|
|
je 1f
|
|
SWAPGS
|
|
@@ -327,11 +332,10 @@ ENTRY(save_args)
|
|
*/
|
|
1: incl PER_CPU_VAR(irq_count)
|
|
jne 2f
|
|
- popq_cfi %rax /* move return address... */
|
|
+ popq %rax /* move return address... */
|
|
mov PER_CPU_VAR(irq_stack_ptr),%rsp
|
|
- EMPTY_FRAME 0
|
|
- pushq_cfi %rbp /* backlink for unwinder */
|
|
- pushq_cfi %rax /* ... to the new stack */
|
|
+ pushq %rbp /* backlink for unwinder */
|
|
+ pushq %rax /* ... to the new stack */
|
|
/*
|
|
* We entered an interrupt context - irqs are off:
|
|
*/
|
|
@@ -342,14 +346,14 @@ END(save_args)
|
|
.popsection
|
|
|
|
ENTRY(save_rest)
|
|
- PARTIAL_FRAME 1 REST_SKIP+8
|
|
+ CFI_STARTPROC
|
|
movq 5*8+16(%rsp), %r11 /* save return address */
|
|
- movq_cfi rbx, RBX+16
|
|
- movq_cfi rbp, RBP+16
|
|
- movq_cfi r12, R12+16
|
|
- movq_cfi r13, R13+16
|
|
- movq_cfi r14, R14+16
|
|
- movq_cfi r15, R15+16
|
|
+ movq %rbx, RBX+16(%rsp)
|
|
+ movq %rbp, RBP+16(%rsp)
|
|
+ movq %r12, R12+16(%rsp)
|
|
+ movq %r13, R13+16(%rsp)
|
|
+ movq %r14, R14+16(%rsp)
|
|
+ movq %r15, R15+16(%rsp)
|
|
movq %r11, 8(%rsp) /* return address */
|
|
FIXUP_TOP_OF_STACK %r11, 16
|
|
ret
|
|
@@ -359,23 +363,23 @@ END(save_rest)
|
|
/* save complete stack frame */
|
|
.pushsection .kprobes.text, "ax"
|
|
ENTRY(save_paranoid)
|
|
- XCPT_FRAME 1 RDI+8
|
|
+ XCPT_FRAME offset=ORIG_RAX-R15+8
|
|
cld
|
|
- movq_cfi rdi, RDI+8
|
|
- movq_cfi rsi, RSI+8
|
|
+ movq %rdi, RDI+8(%rsp)
|
|
+ movq %rsi, RSI+8(%rsp)
|
|
movq_cfi rdx, RDX+8
|
|
movq_cfi rcx, RCX+8
|
|
movq_cfi rax, RAX+8
|
|
- movq_cfi r8, R8+8
|
|
- movq_cfi r9, R9+8
|
|
- movq_cfi r10, R10+8
|
|
- movq_cfi r11, R11+8
|
|
+ movq %r8, R8+8(%rsp)
|
|
+ movq %r9, R9+8(%rsp)
|
|
+ movq %r10, R10+8(%rsp)
|
|
+ movq %r11, R11+8(%rsp)
|
|
movq_cfi rbx, RBX+8
|
|
- movq_cfi rbp, RBP+8
|
|
- movq_cfi r12, R12+8
|
|
- movq_cfi r13, R13+8
|
|
- movq_cfi r14, R14+8
|
|
- movq_cfi r15, R15+8
|
|
+ movq %rbp, RBP+8(%rsp)
|
|
+ movq %r12, R12+8(%rsp)
|
|
+ movq %r13, R13+8(%rsp)
|
|
+ movq %r14, R14+8(%rsp)
|
|
+ movq %r15, R15+8(%rsp)
|
|
movl $1,%ebx
|
|
movl $MSR_GS_BASE,%ecx
|
|
rdmsr
|
|
@@ -677,7 +681,7 @@ ENTRY(\label)
|
|
subq $REST_SKIP, %rsp
|
|
CFI_ADJUST_CFA_OFFSET REST_SKIP
|
|
call save_rest
|
|
- DEFAULT_FRAME 0 8 /* offset 8: return address */
|
|
+ DEFAULT_FRAME -2 8 /* offset 8: return address */
|
|
leaq 8(%rsp), \arg /* pt_regs pointer */
|
|
call \func
|
|
jmp ptregscall_common
|
|
@@ -794,7 +798,9 @@ END(interrupt)
|
|
subq $ORIG_RAX-RBP, %rsp
|
|
CFI_ADJUST_CFA_OFFSET ORIG_RAX-RBP
|
|
call save_args
|
|
- PARTIAL_FRAME 0
|
|
+ PARTIAL_FRAME -1 ARGOFFSET-RBP
|
|
+ CFI_REL_OFFSET rbp, 0
|
|
+ CFI_DEF_CFA_REGISTER rbp
|
|
call \func
|
|
.endm
|
|
|
|
@@ -813,7 +819,6 @@ ret_from_intr:
|
|
TRACE_IRQS_OFF
|
|
decl PER_CPU_VAR(irq_count)
|
|
leaveq
|
|
-
|
|
CFI_RESTORE rbp
|
|
CFI_DEF_CFA_REGISTER rsp
|
|
CFI_ADJUST_CFA_OFFSET -8
|
|
@@ -1021,7 +1026,7 @@ ENTRY(\sym)
|
|
subq $ORIG_RAX-R15, %rsp
|
|
CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15
|
|
call error_entry
|
|
- DEFAULT_FRAME 0
|
|
+ DEFAULT_FRAME -1
|
|
movq %rsp,%rdi /* pt_regs pointer */
|
|
xorl %esi,%esi /* no error code */
|
|
call \do_sym
|
|
@@ -1038,6 +1043,7 @@ ENTRY(\sym)
|
|
subq $ORIG_RAX-R15, %rsp
|
|
CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15
|
|
call save_paranoid
|
|
+ DEFAULT_FRAME -1
|
|
TRACE_IRQS_OFF
|
|
movq %rsp,%rdi /* pt_regs pointer */
|
|
xorl %esi,%esi /* no error code */
|
|
@@ -1056,6 +1062,7 @@ ENTRY(\sym)
|
|
subq $ORIG_RAX-R15, %rsp
|
|
CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15
|
|
call save_paranoid
|
|
+ DEFAULT_FRAME -1
|
|
TRACE_IRQS_OFF
|
|
movq %rsp,%rdi /* pt_regs pointer */
|
|
xorl %esi,%esi /* no error code */
|
|
@@ -1074,7 +1081,7 @@ ENTRY(\sym)
|
|
subq $ORIG_RAX-R15, %rsp
|
|
CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15
|
|
call error_entry
|
|
- DEFAULT_FRAME 0
|
|
+ DEFAULT_FRAME -1
|
|
movq %rsp,%rdi /* pt_regs pointer */
|
|
movq ORIG_RAX(%rsp),%rsi /* get error code */
|
|
movq $-1,ORIG_RAX(%rsp) /* no syscall to restart */
|
|
@@ -1092,7 +1099,7 @@ ENTRY(\sym)
|
|
subq $ORIG_RAX-R15, %rsp
|
|
CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15
|
|
call save_paranoid
|
|
- DEFAULT_FRAME 0
|
|
+ DEFAULT_FRAME -1
|
|
TRACE_IRQS_OFF
|
|
movq %rsp,%rdi /* pt_regs pointer */
|
|
movq ORIG_RAX(%rsp),%rsi /* get error code */
|
|
@@ -1435,25 +1442,24 @@ END(paranoid_exit)
|
|
* returns in "no swapgs flag" in %ebx.
|
|
*/
|
|
ENTRY(error_entry)
|
|
- XCPT_FRAME
|
|
- CFI_ADJUST_CFA_OFFSET 15*8
|
|
+ XCPT_FRAME offset=ORIG_RAX-R15+8
|
|
/* oldrax contains error code */
|
|
cld
|
|
- movq_cfi rdi, RDI+8
|
|
- movq_cfi rsi, RSI+8
|
|
- movq_cfi rdx, RDX+8
|
|
- movq_cfi rcx, RCX+8
|
|
- movq_cfi rax, RAX+8
|
|
- movq_cfi r8, R8+8
|
|
- movq_cfi r9, R9+8
|
|
- movq_cfi r10, R10+8
|
|
- movq_cfi r11, R11+8
|
|
+ movq %rdi, RDI+8(%rsp)
|
|
+ movq %rsi, RSI+8(%rsp)
|
|
+ movq %rdx, RDX+8(%rsp)
|
|
+ movq %rcx, RCX+8(%rsp)
|
|
+ movq %rax, RAX+8(%rsp)
|
|
+ movq %r8, R8+8(%rsp)
|
|
+ movq %r9, R9+8(%rsp)
|
|
+ movq %r10, R10+8(%rsp)
|
|
+ movq %r11, R11+8(%rsp)
|
|
movq_cfi rbx, RBX+8
|
|
- movq_cfi rbp, RBP+8
|
|
- movq_cfi r12, R12+8
|
|
- movq_cfi r13, R13+8
|
|
- movq_cfi r14, R14+8
|
|
- movq_cfi r15, R15+8
|
|
+ movq %rbp, RBP+8(%rsp)
|
|
+ movq %r12, R12+8(%rsp)
|
|
+ movq %r13, R13+8(%rsp)
|
|
+ movq %r14, R14+8(%rsp)
|
|
+ movq %r15, R15+8(%rsp)
|
|
xorl %ebx,%ebx
|
|
testl $3,CS+8(%rsp)
|
|
je error_kernelspace
|
|
@@ -1471,6 +1477,7 @@ error_sti:
|
|
* compat mode. Check for these here too.
|
|
*/
|
|
error_kernelspace:
|
|
+ CFI_REL_OFFSET rcx, RCX+8
|
|
incl %ebx
|
|
leaq irq_return(%rip),%rcx
|
|
cmpq %rcx,RIP+8(%rsp)
|
|
@@ -1518,7 +1523,7 @@ ENTRY(nmi)
|
|
subq $ORIG_RAX-R15, %rsp
|
|
CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15
|
|
call save_paranoid
|
|
- DEFAULT_FRAME 0
|
|
+ DEFAULT_FRAME -1
|
|
/* paranoidentry do_nmi, 0; without TRACE_IRQS_OFF */
|
|
movq %rsp,%rdi
|
|
movq $-1,%rsi
|
|
--- a/arch/x86/kernel/head_64.S
|
|
+++ b/arch/x86/kernel/head_64.S
|
|
@@ -284,6 +284,8 @@ early_idt_handlers:
|
|
|
|
ENTRY(early_idt_handler)
|
|
#ifdef CONFIG_EARLY_PRINTK
|
|
+#include <asm/calling.h>
|
|
+#include <asm/dwarf2.h>
|
|
cmpl $2,early_recursion_flag(%rip)
|
|
jz 1f
|
|
incl early_recursion_flag(%rip)
|
|
@@ -299,6 +301,16 @@ ENTRY(early_idt_handler)
|
|
testl $0x27d00,%eax
|
|
je 0f
|
|
popq %r8 # get error code
|
|
+
|
|
+ CFI_STARTPROC simple
|
|
+ CFI_SIGNAL_FRAME
|
|
+ CFI_DEF_CFA rsp, SS+8-RIP
|
|
+# CFI_REL_OFFSET ss, SS-RIP
|
|
+ CFI_REL_OFFSET rsp, RSP-RIP
|
|
+# CFI_REL_OFFSET rflags, EFLAGS-RIP
|
|
+# CFI_REL_OFFSET cs, CS-RIP
|
|
+ CFI_REL_OFFSET rip, RIP-RIP
|
|
+
|
|
0: movq 0(%rsp),%rcx # get ip
|
|
movq 8(%rsp),%rdx # get cs
|
|
xorl %eax,%eax
|
|
@@ -312,6 +324,7 @@ ENTRY(early_idt_handler)
|
|
movq 0(%rsp),%rsi # get rip again
|
|
call __print_symbol
|
|
#endif
|
|
+ CFI_ENDPROC
|
|
#endif /* EARLY_PRINTK */
|
|
1: hlt
|
|
jmp 1b
|
|
--- a/arch/x86/lib/rwsem_64.S
|
|
+++ b/arch/x86/lib/rwsem_64.S
|
|
@@ -23,43 +23,50 @@
|
|
#include <asm/dwarf2.h>
|
|
|
|
#define save_common_regs \
|
|
- pushq %rdi; \
|
|
- pushq %rsi; \
|
|
- pushq %rcx; \
|
|
- pushq %r8; \
|
|
- pushq %r9; \
|
|
- pushq %r10; \
|
|
- pushq %r11
|
|
+ pushq_cfi %rdi; CFI_REL_OFFSET rdi, 0; \
|
|
+ pushq_cfi %rsi; CFI_REL_OFFSET rsi, 0; \
|
|
+ pushq_cfi %rcx; CFI_REL_OFFSET rcx, 0; \
|
|
+ pushq_cfi %r8; CFI_REL_OFFSET r8, 0; \
|
|
+ pushq_cfi %r9; CFI_REL_OFFSET r9, 0; \
|
|
+ pushq_cfi %r10; CFI_REL_OFFSET r10, 0; \
|
|
+ pushq_cfi %r11; CFI_REL_OFFSET r11, 0
|
|
|
|
#define restore_common_regs \
|
|
- popq %r11; \
|
|
- popq %r10; \
|
|
- popq %r9; \
|
|
- popq %r8; \
|
|
- popq %rcx; \
|
|
- popq %rsi; \
|
|
- popq %rdi
|
|
+ popq_cfi %r11; CFI_RESTORE r11; \
|
|
+ popq_cfi %r10; CFI_RESTORE r10; \
|
|
+ popq_cfi %r9; CFI_RESTORE r9; \
|
|
+ popq_cfi %r8; CFI_RESTORE r8; \
|
|
+ popq_cfi %rcx; CFI_RESTORE rcx; \
|
|
+ popq_cfi %rsi; CFI_RESTORE rsi; \
|
|
+ popq_cfi %rdi; CFI_RESTORE rdi
|
|
|
|
/* Fix up special calling conventions */
|
|
ENTRY(call_rwsem_down_read_failed)
|
|
+ CFI_STARTPROC
|
|
save_common_regs
|
|
- pushq %rdx
|
|
+ pushq_cfi %rdx
|
|
+ CFI_REL_OFFSET rdx, 0
|
|
movq %rax,%rdi
|
|
call rwsem_down_read_failed
|
|
- popq %rdx
|
|
+ popq_cfi %rdx
|
|
+ CFI_RESTORE rdx
|
|
restore_common_regs
|
|
ret
|
|
- ENDPROC(call_rwsem_down_read_failed)
|
|
+ CFI_ENDPROC
|
|
+ENDPROC(call_rwsem_down_read_failed)
|
|
|
|
ENTRY(call_rwsem_down_write_failed)
|
|
+ CFI_STARTPROC
|
|
save_common_regs
|
|
movq %rax,%rdi
|
|
call rwsem_down_write_failed
|
|
restore_common_regs
|
|
ret
|
|
- ENDPROC(call_rwsem_down_write_failed)
|
|
+ CFI_ENDPROC
|
|
+ENDPROC(call_rwsem_down_write_failed)
|
|
|
|
ENTRY(call_rwsem_wake)
|
|
+ CFI_STARTPROC
|
|
decl %edx /* do nothing if still outstanding active readers */
|
|
jnz 1f
|
|
save_common_regs
|
|
@@ -67,15 +74,20 @@ ENTRY(call_rwsem_wake)
|
|
call rwsem_wake
|
|
restore_common_regs
|
|
1: ret
|
|
- ENDPROC(call_rwsem_wake)
|
|
+ CFI_ENDPROC
|
|
+ENDPROC(call_rwsem_wake)
|
|
|
|
/* Fix up special calling conventions */
|
|
ENTRY(call_rwsem_downgrade_wake)
|
|
+ CFI_STARTPROC
|
|
save_common_regs
|
|
- pushq %rdx
|
|
+ pushq_cfi %rdx
|
|
+ CFI_REL_OFFSET rdx, 0
|
|
movq %rax,%rdi
|
|
call rwsem_downgrade_wake
|
|
- popq %rdx
|
|
+ popq_cfi %rdx
|
|
+ CFI_RESTORE rdx
|
|
restore_common_regs
|
|
ret
|
|
- ENDPROC(call_rwsem_downgrade_wake)
|
|
+ CFI_ENDPROC
|
|
+ENDPROC(call_rwsem_downgrade_wake)
|