x86/debug: Remove perpetually broken, unmaintainable dwarf annotations
So the dwarf2 annotations in low level assembly code have become an increasing hindrance: unreadable, messy macros mixed into some of the most security sensitive code paths of the Linux kernel. These debug info annotations don't even buy the upstream kernel anything: dwarf driven stack unwinding has caused problems in the past so it's out of tree, and the upstream kernel only uses the much more robust framepointers based stack unwinding method. In addition to that there's a steady, slow bitrot going on with these annotations, requiring frequent fixups. There's no tooling and no functionality upstream that keeps it correct. So burn down the sick forest, allowing new, healthier growth: 27 files changed, 350 insertions(+), 1101 deletions(-) Someone who has the willingness and time to do this properly can attempt to reintroduce dwarf debuginfo in x86 assembly code plus dwarf unwinding from first principles, with the following conditions: - it should be maximally readable, and maximally low-key to 'ordinary' code reading and maintenance. - find a build time method to insert dwarf annotations automatically in the most common cases, for pop/push instructions that manipulate the stack pointer. This could be done for example via a preprocessing step that just looks for common patterns - plus special annotations for the few cases where we want to depart from the default. We have hundreds of CFI annotations, so automating most of that makes sense. - it should come with build tooling checks that ensure that CFI annotations are sensible. We've seen such efforts from the framepointer side, and there's no reason it couldn't be done on the dwarf side. Cc: Andy Lutomirski <luto@amacapital.net> Cc: Borislav Petkov <bp@alien8.de> Cc: Brian Gerst <brgerst@gmail.com> Cc: Denys Vlasenko <dvlasenk@redhat.com> Cc: Frédéric Weisbecker <fweisbec@gmail.com Cc: H. Peter Anvin <hpa@zytor.com> Cc: Jan Beulich <JBeulich@suse.com> Cc: Josh Poimboeuf <jpoimboe@redhat.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: linux-kernel@vger.kernel.org Signed-off-by: Ingo Molnar <mingo@kernel.org>
This commit is contained in:
@@ -50,7 +50,6 @@
|
||||
#include <asm/smp.h>
|
||||
#include <asm/page_types.h>
|
||||
#include <asm/percpu.h>
|
||||
#include <asm/dwarf2.h>
|
||||
#include <asm/processor-flags.h>
|
||||
#include <asm/ftrace.h>
|
||||
#include <asm/irq_vectors.h>
|
||||
@@ -113,11 +112,10 @@
|
||||
|
||||
/* unfortunately push/pop can't be no-op */
|
||||
.macro PUSH_GS
|
||||
pushl_cfi $0
|
||||
pushl $0
|
||||
.endm
|
||||
.macro POP_GS pop=0
|
||||
addl $(4 + \pop), %esp
|
||||
CFI_ADJUST_CFA_OFFSET -(4 + \pop)
|
||||
.endm
|
||||
.macro POP_GS_EX
|
||||
.endm
|
||||
@@ -137,16 +135,13 @@
|
||||
#else /* CONFIG_X86_32_LAZY_GS */
|
||||
|
||||
.macro PUSH_GS
|
||||
pushl_cfi %gs
|
||||
/*CFI_REL_OFFSET gs, 0*/
|
||||
pushl %gs
|
||||
.endm
|
||||
|
||||
.macro POP_GS pop=0
|
||||
98: popl_cfi %gs
|
||||
/*CFI_RESTORE gs*/
|
||||
98: popl %gs
|
||||
.if \pop <> 0
|
||||
add $\pop, %esp
|
||||
CFI_ADJUST_CFA_OFFSET -\pop
|
||||
.endif
|
||||
.endm
|
||||
.macro POP_GS_EX
|
||||
@@ -170,11 +165,9 @@
|
||||
|
||||
.macro GS_TO_REG reg
|
||||
movl %gs, \reg
|
||||
/*CFI_REGISTER gs, \reg*/
|
||||
.endm
|
||||
.macro REG_TO_PTGS reg
|
||||
movl \reg, PT_GS(%esp)
|
||||
/*CFI_REL_OFFSET gs, PT_GS*/
|
||||
.endm
|
||||
.macro SET_KERNEL_GS reg
|
||||
movl $(__KERNEL_STACK_CANARY), \reg
|
||||
@@ -186,26 +179,16 @@
|
||||
.macro SAVE_ALL
|
||||
cld
|
||||
PUSH_GS
|
||||
pushl_cfi %fs
|
||||
/*CFI_REL_OFFSET fs, 0;*/
|
||||
pushl_cfi %es
|
||||
/*CFI_REL_OFFSET es, 0;*/
|
||||
pushl_cfi %ds
|
||||
/*CFI_REL_OFFSET ds, 0;*/
|
||||
pushl_cfi %eax
|
||||
CFI_REL_OFFSET eax, 0
|
||||
pushl_cfi %ebp
|
||||
CFI_REL_OFFSET ebp, 0
|
||||
pushl_cfi %edi
|
||||
CFI_REL_OFFSET edi, 0
|
||||
pushl_cfi %esi
|
||||
CFI_REL_OFFSET esi, 0
|
||||
pushl_cfi %edx
|
||||
CFI_REL_OFFSET edx, 0
|
||||
pushl_cfi %ecx
|
||||
CFI_REL_OFFSET ecx, 0
|
||||
pushl_cfi %ebx
|
||||
CFI_REL_OFFSET ebx, 0
|
||||
pushl %fs
|
||||
pushl %es
|
||||
pushl %ds
|
||||
pushl %eax
|
||||
pushl %ebp
|
||||
pushl %edi
|
||||
pushl %esi
|
||||
pushl %edx
|
||||
pushl %ecx
|
||||
pushl %ebx
|
||||
movl $(__USER_DS), %edx
|
||||
movl %edx, %ds
|
||||
movl %edx, %es
|
||||
@@ -215,30 +198,20 @@
|
||||
.endm
|
||||
|
||||
.macro RESTORE_INT_REGS
|
||||
popl_cfi %ebx
|
||||
CFI_RESTORE ebx
|
||||
popl_cfi %ecx
|
||||
CFI_RESTORE ecx
|
||||
popl_cfi %edx
|
||||
CFI_RESTORE edx
|
||||
popl_cfi %esi
|
||||
CFI_RESTORE esi
|
||||
popl_cfi %edi
|
||||
CFI_RESTORE edi
|
||||
popl_cfi %ebp
|
||||
CFI_RESTORE ebp
|
||||
popl_cfi %eax
|
||||
CFI_RESTORE eax
|
||||
popl %ebx
|
||||
popl %ecx
|
||||
popl %edx
|
||||
popl %esi
|
||||
popl %edi
|
||||
popl %ebp
|
||||
popl %eax
|
||||
.endm
|
||||
|
||||
.macro RESTORE_REGS pop=0
|
||||
RESTORE_INT_REGS
|
||||
1: popl_cfi %ds
|
||||
/*CFI_RESTORE ds;*/
|
||||
2: popl_cfi %es
|
||||
/*CFI_RESTORE es;*/
|
||||
3: popl_cfi %fs
|
||||
/*CFI_RESTORE fs;*/
|
||||
1: popl %ds
|
||||
2: popl %es
|
||||
3: popl %fs
|
||||
POP_GS \pop
|
||||
.pushsection .fixup, "ax"
|
||||
4: movl $0, (%esp)
|
||||
@@ -254,64 +227,27 @@
|
||||
POP_GS_EX
|
||||
.endm
|
||||
|
||||
.macro RING0_INT_FRAME
|
||||
CFI_STARTPROC simple
|
||||
CFI_SIGNAL_FRAME
|
||||
CFI_DEF_CFA esp, 3*4
|
||||
/*CFI_OFFSET cs, -2*4;*/
|
||||
CFI_OFFSET eip, -3*4
|
||||
.endm
|
||||
|
||||
.macro RING0_EC_FRAME
|
||||
CFI_STARTPROC simple
|
||||
CFI_SIGNAL_FRAME
|
||||
CFI_DEF_CFA esp, 4*4
|
||||
/*CFI_OFFSET cs, -2*4;*/
|
||||
CFI_OFFSET eip, -3*4
|
||||
.endm
|
||||
|
||||
.macro RING0_PTREGS_FRAME
|
||||
CFI_STARTPROC simple
|
||||
CFI_SIGNAL_FRAME
|
||||
CFI_DEF_CFA esp, PT_OLDESP-PT_EBX
|
||||
/*CFI_OFFSET cs, PT_CS-PT_OLDESP;*/
|
||||
CFI_OFFSET eip, PT_EIP-PT_OLDESP
|
||||
/*CFI_OFFSET es, PT_ES-PT_OLDESP;*/
|
||||
/*CFI_OFFSET ds, PT_DS-PT_OLDESP;*/
|
||||
CFI_OFFSET eax, PT_EAX-PT_OLDESP
|
||||
CFI_OFFSET ebp, PT_EBP-PT_OLDESP
|
||||
CFI_OFFSET edi, PT_EDI-PT_OLDESP
|
||||
CFI_OFFSET esi, PT_ESI-PT_OLDESP
|
||||
CFI_OFFSET edx, PT_EDX-PT_OLDESP
|
||||
CFI_OFFSET ecx, PT_ECX-PT_OLDESP
|
||||
CFI_OFFSET ebx, PT_EBX-PT_OLDESP
|
||||
.endm
|
||||
|
||||
ENTRY(ret_from_fork)
|
||||
CFI_STARTPROC
|
||||
pushl_cfi %eax
|
||||
pushl %eax
|
||||
call schedule_tail
|
||||
GET_THREAD_INFO(%ebp)
|
||||
popl_cfi %eax
|
||||
pushl_cfi $0x0202 # Reset kernel eflags
|
||||
popfl_cfi
|
||||
popl %eax
|
||||
pushl $0x0202 # Reset kernel eflags
|
||||
popfl
|
||||
jmp syscall_exit
|
||||
CFI_ENDPROC
|
||||
END(ret_from_fork)
|
||||
|
||||
ENTRY(ret_from_kernel_thread)
|
||||
CFI_STARTPROC
|
||||
pushl_cfi %eax
|
||||
pushl %eax
|
||||
call schedule_tail
|
||||
GET_THREAD_INFO(%ebp)
|
||||
popl_cfi %eax
|
||||
pushl_cfi $0x0202 # Reset kernel eflags
|
||||
popfl_cfi
|
||||
popl %eax
|
||||
pushl $0x0202 # Reset kernel eflags
|
||||
popfl
|
||||
movl PT_EBP(%esp),%eax
|
||||
call *PT_EBX(%esp)
|
||||
movl $0,PT_EAX(%esp)
|
||||
jmp syscall_exit
|
||||
CFI_ENDPROC
|
||||
ENDPROC(ret_from_kernel_thread)
|
||||
|
||||
/*
|
||||
@@ -323,7 +259,6 @@ ENDPROC(ret_from_kernel_thread)
|
||||
|
||||
# userspace resumption stub bypassing syscall exit tracing
|
||||
ALIGN
|
||||
RING0_PTREGS_FRAME
|
||||
ret_from_exception:
|
||||
preempt_stop(CLBR_ANY)
|
||||
ret_from_intr:
|
||||
@@ -367,17 +302,12 @@ need_resched:
|
||||
jmp need_resched
|
||||
END(resume_kernel)
|
||||
#endif
|
||||
CFI_ENDPROC
|
||||
|
||||
/* SYSENTER_RETURN points to after the "sysenter" instruction in
|
||||
the vsyscall page. See vsyscall-sysentry.S, which defines the symbol. */
|
||||
|
||||
# sysenter call handler stub
|
||||
ENTRY(ia32_sysenter_target)
|
||||
CFI_STARTPROC simple
|
||||
CFI_SIGNAL_FRAME
|
||||
CFI_DEF_CFA esp, 0
|
||||
CFI_REGISTER esp, ebp
|
||||
movl TSS_sysenter_sp0(%esp),%esp
|
||||
sysenter_past_esp:
|
||||
/*
|
||||
@@ -385,14 +315,11 @@ sysenter_past_esp:
|
||||
* enough kernel state to call TRACE_IRQS_OFF can be called - but
|
||||
* we immediately enable interrupts at that point anyway.
|
||||
*/
|
||||
pushl_cfi $__USER_DS
|
||||
/*CFI_REL_OFFSET ss, 0*/
|
||||
pushl_cfi %ebp
|
||||
CFI_REL_OFFSET esp, 0
|
||||
pushfl_cfi
|
||||
pushl $__USER_DS
|
||||
pushl %ebp
|
||||
pushfl
|
||||
orl $X86_EFLAGS_IF, (%esp)
|
||||
pushl_cfi $__USER_CS
|
||||
/*CFI_REL_OFFSET cs, 0*/
|
||||
pushl $__USER_CS
|
||||
/*
|
||||
* Push current_thread_info()->sysenter_return to the stack.
|
||||
* A tiny bit of offset fixup is necessary: TI_sysenter_return
|
||||
@@ -401,10 +328,9 @@ sysenter_past_esp:
|
||||
* TOP_OF_KERNEL_STACK_PADDING takes us to the top of the stack;
|
||||
* and THREAD_SIZE takes us to the bottom.
|
||||
*/
|
||||
pushl_cfi ((TI_sysenter_return) - THREAD_SIZE + TOP_OF_KERNEL_STACK_PADDING + 4*4)(%esp)
|
||||
CFI_REL_OFFSET eip, 0
|
||||
pushl ((TI_sysenter_return) - THREAD_SIZE + TOP_OF_KERNEL_STACK_PADDING + 4*4)(%esp)
|
||||
|
||||
pushl_cfi %eax
|
||||
pushl %eax
|
||||
SAVE_ALL
|
||||
ENABLE_INTERRUPTS(CLBR_NONE)
|
||||
|
||||
@@ -453,11 +379,11 @@ sysenter_audit:
|
||||
/* movl PT_EAX(%esp), %eax already set, syscall number: 1st arg to audit */
|
||||
movl PT_EBX(%esp), %edx /* ebx/a0: 2nd arg to audit */
|
||||
/* movl PT_ECX(%esp), %ecx already set, a1: 3nd arg to audit */
|
||||
pushl_cfi PT_ESI(%esp) /* a3: 5th arg */
|
||||
pushl_cfi PT_EDX+4(%esp) /* a2: 4th arg */
|
||||
pushl PT_ESI(%esp) /* a3: 5th arg */
|
||||
pushl PT_EDX+4(%esp) /* a2: 4th arg */
|
||||
call __audit_syscall_entry
|
||||
popl_cfi %ecx /* get that remapped edx off the stack */
|
||||
popl_cfi %ecx /* get that remapped esi off the stack */
|
||||
popl %ecx /* get that remapped edx off the stack */
|
||||
popl %ecx /* get that remapped esi off the stack */
|
||||
movl PT_EAX(%esp),%eax /* reload syscall number */
|
||||
jmp sysenter_do_call
|
||||
|
||||
@@ -480,7 +406,6 @@ sysexit_audit:
|
||||
jmp sysenter_exit
|
||||
#endif
|
||||
|
||||
CFI_ENDPROC
|
||||
.pushsection .fixup,"ax"
|
||||
2: movl $0,PT_FS(%esp)
|
||||
jmp 1b
|
||||
@@ -491,9 +416,8 @@ ENDPROC(ia32_sysenter_target)
|
||||
|
||||
# system call handler stub
|
||||
ENTRY(system_call)
|
||||
RING0_INT_FRAME # can't unwind into user space anyway
|
||||
ASM_CLAC
|
||||
pushl_cfi %eax # save orig_eax
|
||||
pushl %eax # save orig_eax
|
||||
SAVE_ALL
|
||||
GET_THREAD_INFO(%ebp)
|
||||
# system call tracing in operation / emulation
|
||||
@@ -527,7 +451,6 @@ restore_all_notrace:
|
||||
movb PT_CS(%esp), %al
|
||||
andl $(X86_EFLAGS_VM | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax
|
||||
cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax
|
||||
CFI_REMEMBER_STATE
|
||||
je ldt_ss # returning to user-space with LDT SS
|
||||
#endif
|
||||
restore_nocheck:
|
||||
@@ -543,7 +466,6 @@ ENTRY(iret_exc)
|
||||
_ASM_EXTABLE(irq_return,iret_exc)
|
||||
|
||||
#ifdef CONFIG_X86_ESPFIX32
|
||||
CFI_RESTORE_STATE
|
||||
ldt_ss:
|
||||
#ifdef CONFIG_PARAVIRT
|
||||
/*
|
||||
@@ -577,22 +499,19 @@ ldt_ss:
|
||||
shr $16, %edx
|
||||
mov %dl, GDT_ESPFIX_SS + 4 /* bits 16..23 */
|
||||
mov %dh, GDT_ESPFIX_SS + 7 /* bits 24..31 */
|
||||
pushl_cfi $__ESPFIX_SS
|
||||
pushl_cfi %eax /* new kernel esp */
|
||||
pushl $__ESPFIX_SS
|
||||
pushl %eax /* new kernel esp */
|
||||
/* Disable interrupts, but do not irqtrace this section: we
|
||||
* will soon execute iret and the tracer was already set to
|
||||
* the irqstate after the iret */
|
||||
DISABLE_INTERRUPTS(CLBR_EAX)
|
||||
lss (%esp), %esp /* switch to espfix segment */
|
||||
CFI_ADJUST_CFA_OFFSET -8
|
||||
jmp restore_nocheck
|
||||
#endif
|
||||
CFI_ENDPROC
|
||||
ENDPROC(system_call)
|
||||
|
||||
# perform work that needs to be done immediately before resumption
|
||||
ALIGN
|
||||
RING0_PTREGS_FRAME # can't unwind into user space anyway
|
||||
work_pending:
|
||||
testb $_TIF_NEED_RESCHED, %cl
|
||||
jz work_notifysig
|
||||
@@ -634,9 +553,9 @@ work_notifysig: # deal with pending signals and
|
||||
#ifdef CONFIG_VM86
|
||||
ALIGN
|
||||
work_notifysig_v86:
|
||||
pushl_cfi %ecx # save ti_flags for do_notify_resume
|
||||
pushl %ecx # save ti_flags for do_notify_resume
|
||||
call save_v86_state # %eax contains pt_regs pointer
|
||||
popl_cfi %ecx
|
||||
popl %ecx
|
||||
movl %eax, %esp
|
||||
jmp 1b
|
||||
#endif
|
||||
@@ -666,9 +585,7 @@ syscall_exit_work:
|
||||
call syscall_trace_leave
|
||||
jmp resume_userspace
|
||||
END(syscall_exit_work)
|
||||
CFI_ENDPROC
|
||||
|
||||
RING0_INT_FRAME # can't unwind into user space anyway
|
||||
syscall_fault:
|
||||
ASM_CLAC
|
||||
GET_THREAD_INFO(%ebp)
|
||||
@@ -685,7 +602,6 @@ sysenter_badsys:
|
||||
movl $-ENOSYS,%eax
|
||||
jmp sysenter_after_call
|
||||
END(sysenter_badsys)
|
||||
CFI_ENDPROC
|
||||
|
||||
.macro FIXUP_ESPFIX_STACK
|
||||
/*
|
||||
@@ -701,10 +617,9 @@ END(sysenter_badsys)
|
||||
mov GDT_ESPFIX_SS + 7, %ah /* bits 24..31 */
|
||||
shl $16, %eax
|
||||
addl %esp, %eax /* the adjusted stack pointer */
|
||||
pushl_cfi $__KERNEL_DS
|
||||
pushl_cfi %eax
|
||||
pushl $__KERNEL_DS
|
||||
pushl %eax
|
||||
lss (%esp), %esp /* switch to the normal stack segment */
|
||||
CFI_ADJUST_CFA_OFFSET -8
|
||||
#endif
|
||||
.endm
|
||||
.macro UNWIND_ESPFIX_STACK
|
||||
@@ -728,13 +643,11 @@ END(sysenter_badsys)
|
||||
*/
|
||||
.align 8
|
||||
ENTRY(irq_entries_start)
|
||||
RING0_INT_FRAME
|
||||
vector=FIRST_EXTERNAL_VECTOR
|
||||
.rept (FIRST_SYSTEM_VECTOR - FIRST_EXTERNAL_VECTOR)
|
||||
pushl_cfi $(~vector+0x80) /* Note: always in signed byte range */
|
||||
pushl $(~vector+0x80) /* Note: always in signed byte range */
|
||||
vector=vector+1
|
||||
jmp common_interrupt
|
||||
CFI_ADJUST_CFA_OFFSET -4
|
||||
.align 8
|
||||
.endr
|
||||
END(irq_entries_start)
|
||||
@@ -753,19 +666,16 @@ common_interrupt:
|
||||
call do_IRQ
|
||||
jmp ret_from_intr
|
||||
ENDPROC(common_interrupt)
|
||||
CFI_ENDPROC
|
||||
|
||||
#define BUILD_INTERRUPT3(name, nr, fn) \
|
||||
ENTRY(name) \
|
||||
RING0_INT_FRAME; \
|
||||
ASM_CLAC; \
|
||||
pushl_cfi $~(nr); \
|
||||
pushl $~(nr); \
|
||||
SAVE_ALL; \
|
||||
TRACE_IRQS_OFF \
|
||||
movl %esp,%eax; \
|
||||
call fn; \
|
||||
jmp ret_from_intr; \
|
||||
CFI_ENDPROC; \
|
||||
ENDPROC(name)
|
||||
|
||||
|
||||
@@ -784,37 +694,31 @@ ENDPROC(name)
|
||||
#include <asm/entry_arch.h>
|
||||
|
||||
ENTRY(coprocessor_error)
|
||||
RING0_INT_FRAME
|
||||
ASM_CLAC
|
||||
pushl_cfi $0
|
||||
pushl_cfi $do_coprocessor_error
|
||||
pushl $0
|
||||
pushl $do_coprocessor_error
|
||||
jmp error_code
|
||||
CFI_ENDPROC
|
||||
END(coprocessor_error)
|
||||
|
||||
ENTRY(simd_coprocessor_error)
|
||||
RING0_INT_FRAME
|
||||
ASM_CLAC
|
||||
pushl_cfi $0
|
||||
pushl $0
|
||||
#ifdef CONFIG_X86_INVD_BUG
|
||||
/* AMD 486 bug: invd from userspace calls exception 19 instead of #GP */
|
||||
ALTERNATIVE "pushl_cfi $do_general_protection", \
|
||||
ALTERNATIVE "pushl $do_general_protection", \
|
||||
"pushl $do_simd_coprocessor_error", \
|
||||
X86_FEATURE_XMM
|
||||
#else
|
||||
pushl_cfi $do_simd_coprocessor_error
|
||||
pushl $do_simd_coprocessor_error
|
||||
#endif
|
||||
jmp error_code
|
||||
CFI_ENDPROC
|
||||
END(simd_coprocessor_error)
|
||||
|
||||
ENTRY(device_not_available)
|
||||
RING0_INT_FRAME
|
||||
ASM_CLAC
|
||||
pushl_cfi $-1 # mark this as an int
|
||||
pushl_cfi $do_device_not_available
|
||||
pushl $-1 # mark this as an int
|
||||
pushl $do_device_not_available
|
||||
jmp error_code
|
||||
CFI_ENDPROC
|
||||
END(device_not_available)
|
||||
|
||||
#ifdef CONFIG_PARAVIRT
|
||||
@@ -830,115 +734,89 @@ END(native_irq_enable_sysexit)
|
||||
#endif
|
||||
|
||||
ENTRY(overflow)
|
||||
RING0_INT_FRAME
|
||||
ASM_CLAC
|
||||
pushl_cfi $0
|
||||
pushl_cfi $do_overflow
|
||||
pushl $0
|
||||
pushl $do_overflow
|
||||
jmp error_code
|
||||
CFI_ENDPROC
|
||||
END(overflow)
|
||||
|
||||
ENTRY(bounds)
|
||||
RING0_INT_FRAME
|
||||
ASM_CLAC
|
||||
pushl_cfi $0
|
||||
pushl_cfi $do_bounds
|
||||
pushl $0
|
||||
pushl $do_bounds
|
||||
jmp error_code
|
||||
CFI_ENDPROC
|
||||
END(bounds)
|
||||
|
||||
ENTRY(invalid_op)
|
||||
RING0_INT_FRAME
|
||||
ASM_CLAC
|
||||
pushl_cfi $0
|
||||
pushl_cfi $do_invalid_op
|
||||
pushl $0
|
||||
pushl $do_invalid_op
|
||||
jmp error_code
|
||||
CFI_ENDPROC
|
||||
END(invalid_op)
|
||||
|
||||
ENTRY(coprocessor_segment_overrun)
|
||||
RING0_INT_FRAME
|
||||
ASM_CLAC
|
||||
pushl_cfi $0
|
||||
pushl_cfi $do_coprocessor_segment_overrun
|
||||
pushl $0
|
||||
pushl $do_coprocessor_segment_overrun
|
||||
jmp error_code
|
||||
CFI_ENDPROC
|
||||
END(coprocessor_segment_overrun)
|
||||
|
||||
ENTRY(invalid_TSS)
|
||||
RING0_EC_FRAME
|
||||
ASM_CLAC
|
||||
pushl_cfi $do_invalid_TSS
|
||||
pushl $do_invalid_TSS
|
||||
jmp error_code
|
||||
CFI_ENDPROC
|
||||
END(invalid_TSS)
|
||||
|
||||
ENTRY(segment_not_present)
|
||||
RING0_EC_FRAME
|
||||
ASM_CLAC
|
||||
pushl_cfi $do_segment_not_present
|
||||
pushl $do_segment_not_present
|
||||
jmp error_code
|
||||
CFI_ENDPROC
|
||||
END(segment_not_present)
|
||||
|
||||
ENTRY(stack_segment)
|
||||
RING0_EC_FRAME
|
||||
ASM_CLAC
|
||||
pushl_cfi $do_stack_segment
|
||||
pushl $do_stack_segment
|
||||
jmp error_code
|
||||
CFI_ENDPROC
|
||||
END(stack_segment)
|
||||
|
||||
ENTRY(alignment_check)
|
||||
RING0_EC_FRAME
|
||||
ASM_CLAC
|
||||
pushl_cfi $do_alignment_check
|
||||
pushl $do_alignment_check
|
||||
jmp error_code
|
||||
CFI_ENDPROC
|
||||
END(alignment_check)
|
||||
|
||||
ENTRY(divide_error)
|
||||
RING0_INT_FRAME
|
||||
ASM_CLAC
|
||||
pushl_cfi $0 # no error code
|
||||
pushl_cfi $do_divide_error
|
||||
pushl $0 # no error code
|
||||
pushl $do_divide_error
|
||||
jmp error_code
|
||||
CFI_ENDPROC
|
||||
END(divide_error)
|
||||
|
||||
#ifdef CONFIG_X86_MCE
|
||||
ENTRY(machine_check)
|
||||
RING0_INT_FRAME
|
||||
ASM_CLAC
|
||||
pushl_cfi $0
|
||||
pushl_cfi machine_check_vector
|
||||
pushl $0
|
||||
pushl machine_check_vector
|
||||
jmp error_code
|
||||
CFI_ENDPROC
|
||||
END(machine_check)
|
||||
#endif
|
||||
|
||||
ENTRY(spurious_interrupt_bug)
|
||||
RING0_INT_FRAME
|
||||
ASM_CLAC
|
||||
pushl_cfi $0
|
||||
pushl_cfi $do_spurious_interrupt_bug
|
||||
pushl $0
|
||||
pushl $do_spurious_interrupt_bug
|
||||
jmp error_code
|
||||
CFI_ENDPROC
|
||||
END(spurious_interrupt_bug)
|
||||
|
||||
#ifdef CONFIG_XEN
|
||||
/* Xen doesn't set %esp to be precisely what the normal sysenter
|
||||
entrypoint expects, so fix it up before using the normal path. */
|
||||
ENTRY(xen_sysenter_target)
|
||||
RING0_INT_FRAME
|
||||
addl $5*4, %esp /* remove xen-provided frame */
|
||||
CFI_ADJUST_CFA_OFFSET -5*4
|
||||
jmp sysenter_past_esp
|
||||
CFI_ENDPROC
|
||||
|
||||
ENTRY(xen_hypervisor_callback)
|
||||
CFI_STARTPROC
|
||||
pushl_cfi $-1 /* orig_ax = -1 => not a system call */
|
||||
pushl $-1 /* orig_ax = -1 => not a system call */
|
||||
SAVE_ALL
|
||||
TRACE_IRQS_OFF
|
||||
|
||||
@@ -962,7 +840,6 @@ ENTRY(xen_do_upcall)
|
||||
call xen_maybe_preempt_hcall
|
||||
#endif
|
||||
jmp ret_from_intr
|
||||
CFI_ENDPROC
|
||||
ENDPROC(xen_hypervisor_callback)
|
||||
|
||||
# Hypervisor uses this for application faults while it executes.
|
||||
@@ -976,8 +853,7 @@ ENDPROC(xen_hypervisor_callback)
|
||||
# to pop the stack frame we end up in an infinite loop of failsafe callbacks.
|
||||
# We distinguish between categories by maintaining a status value in EAX.
|
||||
ENTRY(xen_failsafe_callback)
|
||||
CFI_STARTPROC
|
||||
pushl_cfi %eax
|
||||
pushl %eax
|
||||
movl $1,%eax
|
||||
1: mov 4(%esp),%ds
|
||||
2: mov 8(%esp),%es
|
||||
@@ -986,15 +862,13 @@ ENTRY(xen_failsafe_callback)
|
||||
/* EAX == 0 => Category 1 (Bad segment)
|
||||
EAX != 0 => Category 2 (Bad IRET) */
|
||||
testl %eax,%eax
|
||||
popl_cfi %eax
|
||||
popl %eax
|
||||
lea 16(%esp),%esp
|
||||
CFI_ADJUST_CFA_OFFSET -16
|
||||
jz 5f
|
||||
jmp iret_exc
|
||||
5: pushl_cfi $-1 /* orig_ax = -1 => not a system call */
|
||||
5: pushl $-1 /* orig_ax = -1 => not a system call */
|
||||
SAVE_ALL
|
||||
jmp ret_from_exception
|
||||
CFI_ENDPROC
|
||||
|
||||
.section .fixup,"ax"
|
||||
6: xorl %eax,%eax
|
||||
@@ -1195,34 +1069,28 @@ return_to_handler:
|
||||
|
||||
#ifdef CONFIG_TRACING
|
||||
ENTRY(trace_page_fault)
|
||||
RING0_EC_FRAME
|
||||
ASM_CLAC
|
||||
pushl_cfi $trace_do_page_fault
|
||||
pushl $trace_do_page_fault
|
||||
jmp error_code
|
||||
CFI_ENDPROC
|
||||
END(trace_page_fault)
|
||||
#endif
|
||||
|
||||
ENTRY(page_fault)
|
||||
RING0_EC_FRAME
|
||||
ASM_CLAC
|
||||
pushl_cfi $do_page_fault
|
||||
pushl $do_page_fault
|
||||
ALIGN
|
||||
error_code:
|
||||
/* the function address is in %gs's slot on the stack */
|
||||
pushl_cfi %fs
|
||||
/*CFI_REL_OFFSET fs, 0*/
|
||||
pushl_cfi %es
|
||||
/*CFI_REL_OFFSET es, 0*/
|
||||
pushl_cfi %ds
|
||||
/*CFI_REL_OFFSET ds, 0*/
|
||||
pushl_cfi_reg eax
|
||||
pushl_cfi_reg ebp
|
||||
pushl_cfi_reg edi
|
||||
pushl_cfi_reg esi
|
||||
pushl_cfi_reg edx
|
||||
pushl_cfi_reg ecx
|
||||
pushl_cfi_reg ebx
|
||||
pushl %fs
|
||||
pushl %es
|
||||
pushl %ds
|
||||
pushl %eax
|
||||
pushl %ebp
|
||||
pushl %edi
|
||||
pushl %esi
|
||||
pushl %edx
|
||||
pushl %ecx
|
||||
pushl %ebx
|
||||
cld
|
||||
movl $(__KERNEL_PERCPU), %ecx
|
||||
movl %ecx, %fs
|
||||
@@ -1240,7 +1108,6 @@ error_code:
|
||||
movl %esp,%eax # pt_regs pointer
|
||||
call *%edi
|
||||
jmp ret_from_exception
|
||||
CFI_ENDPROC
|
||||
END(page_fault)
|
||||
|
||||
/*
|
||||
@@ -1261,29 +1128,24 @@ END(page_fault)
|
||||
jne \ok
|
||||
\label:
|
||||
movl TSS_sysenter_sp0 + \offset(%esp), %esp
|
||||
CFI_DEF_CFA esp, 0
|
||||
CFI_UNDEFINED eip
|
||||
pushfl_cfi
|
||||
pushl_cfi $__KERNEL_CS
|
||||
pushl_cfi $sysenter_past_esp
|
||||
CFI_REL_OFFSET eip, 0
|
||||
pushfl
|
||||
pushl $__KERNEL_CS
|
||||
pushl $sysenter_past_esp
|
||||
.endm
|
||||
|
||||
ENTRY(debug)
|
||||
RING0_INT_FRAME
|
||||
ASM_CLAC
|
||||
cmpl $ia32_sysenter_target,(%esp)
|
||||
jne debug_stack_correct
|
||||
FIX_STACK 12, debug_stack_correct, debug_esp_fix_insn
|
||||
debug_stack_correct:
|
||||
pushl_cfi $-1 # mark this as an int
|
||||
pushl $-1 # mark this as an int
|
||||
SAVE_ALL
|
||||
TRACE_IRQS_OFF
|
||||
xorl %edx,%edx # error code 0
|
||||
movl %esp,%eax # pt_regs pointer
|
||||
call do_debug
|
||||
jmp ret_from_exception
|
||||
CFI_ENDPROC
|
||||
END(debug)
|
||||
|
||||
/*
|
||||
@@ -1295,45 +1157,40 @@ END(debug)
|
||||
* fault happened on the sysenter path.
|
||||
*/
|
||||
ENTRY(nmi)
|
||||
RING0_INT_FRAME
|
||||
ASM_CLAC
|
||||
#ifdef CONFIG_X86_ESPFIX32
|
||||
pushl_cfi %eax
|
||||
pushl %eax
|
||||
movl %ss, %eax
|
||||
cmpw $__ESPFIX_SS, %ax
|
||||
popl_cfi %eax
|
||||
popl %eax
|
||||
je nmi_espfix_stack
|
||||
#endif
|
||||
cmpl $ia32_sysenter_target,(%esp)
|
||||
je nmi_stack_fixup
|
||||
pushl_cfi %eax
|
||||
pushl %eax
|
||||
movl %esp,%eax
|
||||
/* Do not access memory above the end of our stack page,
|
||||
* it might not exist.
|
||||
*/
|
||||
andl $(THREAD_SIZE-1),%eax
|
||||
cmpl $(THREAD_SIZE-20),%eax
|
||||
popl_cfi %eax
|
||||
popl %eax
|
||||
jae nmi_stack_correct
|
||||
cmpl $ia32_sysenter_target,12(%esp)
|
||||
je nmi_debug_stack_check
|
||||
nmi_stack_correct:
|
||||
/* We have a RING0_INT_FRAME here */
|
||||
pushl_cfi %eax
|
||||
pushl %eax
|
||||
SAVE_ALL
|
||||
xorl %edx,%edx # zero error code
|
||||
movl %esp,%eax # pt_regs pointer
|
||||
call do_nmi
|
||||
jmp restore_all_notrace
|
||||
CFI_ENDPROC
|
||||
|
||||
nmi_stack_fixup:
|
||||
RING0_INT_FRAME
|
||||
FIX_STACK 12, nmi_stack_correct, 1
|
||||
jmp nmi_stack_correct
|
||||
|
||||
nmi_debug_stack_check:
|
||||
/* We have a RING0_INT_FRAME here */
|
||||
cmpw $__KERNEL_CS,16(%esp)
|
||||
jne nmi_stack_correct
|
||||
cmpl $debug,(%esp)
|
||||
@@ -1345,57 +1202,48 @@ nmi_debug_stack_check:
|
||||
|
||||
#ifdef CONFIG_X86_ESPFIX32
|
||||
nmi_espfix_stack:
|
||||
/* We have a RING0_INT_FRAME here.
|
||||
*
|
||||
/*
|
||||
* create the pointer to lss back
|
||||
*/
|
||||
pushl_cfi %ss
|
||||
pushl_cfi %esp
|
||||
pushl %ss
|
||||
pushl %esp
|
||||
addl $4, (%esp)
|
||||
/* copy the iret frame of 12 bytes */
|
||||
.rept 3
|
||||
pushl_cfi 16(%esp)
|
||||
pushl 16(%esp)
|
||||
.endr
|
||||
pushl_cfi %eax
|
||||
pushl %eax
|
||||
SAVE_ALL
|
||||
FIXUP_ESPFIX_STACK # %eax == %esp
|
||||
xorl %edx,%edx # zero error code
|
||||
call do_nmi
|
||||
RESTORE_REGS
|
||||
lss 12+4(%esp), %esp # back to espfix stack
|
||||
CFI_ADJUST_CFA_OFFSET -24
|
||||
jmp irq_return
|
||||
#endif
|
||||
CFI_ENDPROC
|
||||
END(nmi)
|
||||
|
||||
ENTRY(int3)
|
||||
RING0_INT_FRAME
|
||||
ASM_CLAC
|
||||
pushl_cfi $-1 # mark this as an int
|
||||
pushl $-1 # mark this as an int
|
||||
SAVE_ALL
|
||||
TRACE_IRQS_OFF
|
||||
xorl %edx,%edx # zero error code
|
||||
movl %esp,%eax # pt_regs pointer
|
||||
call do_int3
|
||||
jmp ret_from_exception
|
||||
CFI_ENDPROC
|
||||
END(int3)
|
||||
|
||||
ENTRY(general_protection)
|
||||
RING0_EC_FRAME
|
||||
pushl_cfi $do_general_protection
|
||||
pushl $do_general_protection
|
||||
jmp error_code
|
||||
CFI_ENDPROC
|
||||
END(general_protection)
|
||||
|
||||
#ifdef CONFIG_KVM_GUEST
|
||||
ENTRY(async_page_fault)
|
||||
RING0_EC_FRAME
|
||||
ASM_CLAC
|
||||
pushl_cfi $do_async_page_fault
|
||||
pushl $do_async_page_fault
|
||||
jmp error_code
|
||||
CFI_ENDPROC
|
||||
END(async_page_fault)
|
||||
#endif
|
||||
|
||||
|
@@ -19,8 +19,6 @@
|
||||
* at the top of the kernel process stack.
|
||||
*
|
||||
* Some macro usage:
|
||||
* - CFI macros are used to generate dwarf2 unwind information for better
|
||||
* backtraces. They don't change any code.
|
||||
* - ENTRY/END Define functions in the symbol table.
|
||||
* - TRACE_IRQ_* - Trace hard interrupt state for lock debugging.
|
||||
* - idtentry - Define exception entry points.
|
||||
@@ -30,7 +28,6 @@
|
||||
#include <asm/segment.h>
|
||||
#include <asm/cache.h>
|
||||
#include <asm/errno.h>
|
||||
#include <asm/dwarf2.h>
|
||||
#include <asm/calling.h>
|
||||
#include <asm/asm-offsets.h>
|
||||
#include <asm/msr.h>
|
||||
@@ -112,61 +109,6 @@ ENDPROC(native_usergs_sysret64)
|
||||
# define TRACE_IRQS_IRETQ_DEBUG TRACE_IRQS_IRETQ
|
||||
#endif
|
||||
|
||||
/*
|
||||
* empty frame
|
||||
*/
|
||||
.macro EMPTY_FRAME start=1 offset=0
|
||||
.if \start
|
||||
CFI_STARTPROC simple
|
||||
CFI_SIGNAL_FRAME
|
||||
CFI_DEF_CFA rsp,8+\offset
|
||||
.else
|
||||
CFI_DEF_CFA_OFFSET 8+\offset
|
||||
.endif
|
||||
.endm
|
||||
|
||||
/*
|
||||
* initial frame state for interrupts (and exceptions without error code)
|
||||
*/
|
||||
.macro INTR_FRAME start=1 offset=0
|
||||
EMPTY_FRAME \start, 5*8+\offset
|
||||
/*CFI_REL_OFFSET ss, 4*8+\offset*/
|
||||
CFI_REL_OFFSET rsp, 3*8+\offset
|
||||
/*CFI_REL_OFFSET rflags, 2*8+\offset*/
|
||||
/*CFI_REL_OFFSET cs, 1*8+\offset*/
|
||||
CFI_REL_OFFSET rip, 0*8+\offset
|
||||
.endm
|
||||
|
||||
/*
|
||||
* initial frame state for exceptions with error code (and interrupts
|
||||
* with vector already pushed)
|
||||
*/
|
||||
.macro XCPT_FRAME start=1 offset=0
|
||||
INTR_FRAME \start, 1*8+\offset
|
||||
.endm
|
||||
|
||||
/*
|
||||
* frame that enables passing a complete pt_regs to a C function.
|
||||
*/
|
||||
.macro DEFAULT_FRAME start=1 offset=0
|
||||
XCPT_FRAME \start, ORIG_RAX+\offset
|
||||
CFI_REL_OFFSET rdi, RDI+\offset
|
||||
CFI_REL_OFFSET rsi, RSI+\offset
|
||||
CFI_REL_OFFSET rdx, RDX+\offset
|
||||
CFI_REL_OFFSET rcx, RCX+\offset
|
||||
CFI_REL_OFFSET rax, RAX+\offset
|
||||
CFI_REL_OFFSET r8, R8+\offset
|
||||
CFI_REL_OFFSET r9, R9+\offset
|
||||
CFI_REL_OFFSET r10, R10+\offset
|
||||
CFI_REL_OFFSET r11, R11+\offset
|
||||
CFI_REL_OFFSET rbx, RBX+\offset
|
||||
CFI_REL_OFFSET rbp, RBP+\offset
|
||||
CFI_REL_OFFSET r12, R12+\offset
|
||||
CFI_REL_OFFSET r13, R13+\offset
|
||||
CFI_REL_OFFSET r14, R14+\offset
|
||||
CFI_REL_OFFSET r15, R15+\offset
|
||||
.endm
|
||||
|
||||
/*
|
||||
* 64bit SYSCALL instruction entry. Up to 6 arguments in registers.
|
||||
*
|
||||
@@ -196,12 +138,6 @@ ENDPROC(native_usergs_sysret64)
|
||||
*/
|
||||
|
||||
ENTRY(system_call)
|
||||
CFI_STARTPROC simple
|
||||
CFI_SIGNAL_FRAME
|
||||
CFI_DEF_CFA rsp,0
|
||||
CFI_REGISTER rip,rcx
|
||||
/*CFI_REGISTER rflags,r11*/
|
||||
|
||||
/*
|
||||
* Interrupts are off on entry.
|
||||
* We do not frame this tiny irq-off block with TRACE_IRQS_OFF/ON,
|
||||
@@ -219,8 +155,8 @@ GLOBAL(system_call_after_swapgs)
|
||||
movq PER_CPU_VAR(cpu_current_top_of_stack),%rsp
|
||||
|
||||
/* Construct struct pt_regs on stack */
|
||||
pushq_cfi $__USER_DS /* pt_regs->ss */
|
||||
pushq_cfi PER_CPU_VAR(rsp_scratch) /* pt_regs->sp */
|
||||
pushq $__USER_DS /* pt_regs->ss */
|
||||
pushq PER_CPU_VAR(rsp_scratch) /* pt_regs->sp */
|
||||
/*
|
||||
* Re-enable interrupts.
|
||||
* We use 'rsp_scratch' as a scratch space, hence irq-off block above
|
||||
@@ -229,22 +165,20 @@ GLOBAL(system_call_after_swapgs)
|
||||
* with using rsp_scratch:
|
||||
*/
|
||||
ENABLE_INTERRUPTS(CLBR_NONE)
|
||||
pushq_cfi %r11 /* pt_regs->flags */
|
||||
pushq_cfi $__USER_CS /* pt_regs->cs */
|
||||
pushq_cfi %rcx /* pt_regs->ip */
|
||||
CFI_REL_OFFSET rip,0
|
||||
pushq_cfi_reg rax /* pt_regs->orig_ax */
|
||||
pushq_cfi_reg rdi /* pt_regs->di */
|
||||
pushq_cfi_reg rsi /* pt_regs->si */
|
||||
pushq_cfi_reg rdx /* pt_regs->dx */
|
||||
pushq_cfi_reg rcx /* pt_regs->cx */
|
||||
pushq_cfi $-ENOSYS /* pt_regs->ax */
|
||||
pushq_cfi_reg r8 /* pt_regs->r8 */
|
||||
pushq_cfi_reg r9 /* pt_regs->r9 */
|
||||
pushq_cfi_reg r10 /* pt_regs->r10 */
|
||||
pushq_cfi_reg r11 /* pt_regs->r11 */
|
||||
pushq %r11 /* pt_regs->flags */
|
||||
pushq $__USER_CS /* pt_regs->cs */
|
||||
pushq %rcx /* pt_regs->ip */
|
||||
pushq %rax /* pt_regs->orig_ax */
|
||||
pushq %rdi /* pt_regs->di */
|
||||
pushq %rsi /* pt_regs->si */
|
||||
pushq %rdx /* pt_regs->dx */
|
||||
pushq %rcx /* pt_regs->cx */
|
||||
pushq $-ENOSYS /* pt_regs->ax */
|
||||
pushq %r8 /* pt_regs->r8 */
|
||||
pushq %r9 /* pt_regs->r9 */
|
||||
pushq %r10 /* pt_regs->r10 */
|
||||
pushq %r11 /* pt_regs->r11 */
|
||||
sub $(6*8),%rsp /* pt_regs->bp,bx,r12-15 not saved */
|
||||
CFI_ADJUST_CFA_OFFSET 6*8
|
||||
|
||||
testl $_TIF_WORK_SYSCALL_ENTRY, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
|
||||
jnz tracesys
|
||||
@@ -282,13 +216,9 @@ system_call_fastpath:
|
||||
testl $_TIF_ALLWORK_MASK, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
|
||||
jnz int_ret_from_sys_call_irqs_off /* Go to the slow path */
|
||||
|
||||
CFI_REMEMBER_STATE
|
||||
|
||||
RESTORE_C_REGS_EXCEPT_RCX_R11
|
||||
movq RIP(%rsp),%rcx
|
||||
CFI_REGISTER rip,rcx
|
||||
movq EFLAGS(%rsp),%r11
|
||||
/*CFI_REGISTER rflags,r11*/
|
||||
movq RSP(%rsp),%rsp
|
||||
/*
|
||||
* 64bit SYSRET restores rip from rcx,
|
||||
@@ -307,8 +237,6 @@ system_call_fastpath:
|
||||
*/
|
||||
USERGS_SYSRET64
|
||||
|
||||
CFI_RESTORE_STATE
|
||||
|
||||
/* Do syscall entry tracing */
|
||||
tracesys:
|
||||
movq %rsp, %rdi
|
||||
@@ -374,9 +302,9 @@ int_careful:
|
||||
jnc int_very_careful
|
||||
TRACE_IRQS_ON
|
||||
ENABLE_INTERRUPTS(CLBR_NONE)
|
||||
pushq_cfi %rdi
|
||||
pushq %rdi
|
||||
SCHEDULE_USER
|
||||
popq_cfi %rdi
|
||||
popq %rdi
|
||||
DISABLE_INTERRUPTS(CLBR_NONE)
|
||||
TRACE_IRQS_OFF
|
||||
jmp int_with_check
|
||||
@@ -389,10 +317,10 @@ int_very_careful:
|
||||
/* Check for syscall exit trace */
|
||||
testl $_TIF_WORK_SYSCALL_EXIT,%edx
|
||||
jz int_signal
|
||||
pushq_cfi %rdi
|
||||
pushq %rdi
|
||||
leaq 8(%rsp),%rdi # &ptregs -> arg1
|
||||
call syscall_trace_leave
|
||||
popq_cfi %rdi
|
||||
popq %rdi
|
||||
andl $~(_TIF_WORK_SYSCALL_EXIT|_TIF_SYSCALL_EMU),%edi
|
||||
jmp int_restore_rest
|
||||
|
||||
@@ -475,27 +403,21 @@ syscall_return:
|
||||
* perf profiles. Nothing jumps here.
|
||||
*/
|
||||
syscall_return_via_sysret:
|
||||
CFI_REMEMBER_STATE
|
||||
/* rcx and r11 are already restored (see code above) */
|
||||
RESTORE_C_REGS_EXCEPT_RCX_R11
|
||||
movq RSP(%rsp),%rsp
|
||||
USERGS_SYSRET64
|
||||
CFI_RESTORE_STATE
|
||||
|
||||
opportunistic_sysret_failed:
|
||||
SWAPGS
|
||||
jmp restore_c_regs_and_iret
|
||||
CFI_ENDPROC
|
||||
END(system_call)
|
||||
|
||||
|
||||
.macro FORK_LIKE func
|
||||
ENTRY(stub_\func)
|
||||
CFI_STARTPROC
|
||||
DEFAULT_FRAME 0, 8 /* offset 8: return address */
|
||||
SAVE_EXTRA_REGS 8
|
||||
jmp sys_\func
|
||||
CFI_ENDPROC
|
||||
END(stub_\func)
|
||||
.endm
|
||||
|
||||
@@ -504,8 +426,6 @@ END(stub_\func)
|
||||
FORK_LIKE vfork
|
||||
|
||||
ENTRY(stub_execve)
|
||||
CFI_STARTPROC
|
||||
DEFAULT_FRAME 0, 8
|
||||
call sys_execve
|
||||
return_from_execve:
|
||||
testl %eax, %eax
|
||||
@@ -515,11 +435,9 @@ return_from_execve:
|
||||
1:
|
||||
/* must use IRET code path (pt_regs->cs may have changed) */
|
||||
addq $8, %rsp
|
||||
CFI_ADJUST_CFA_OFFSET -8
|
||||
ZERO_EXTRA_REGS
|
||||
movq %rax,RAX(%rsp)
|
||||
jmp int_ret_from_sys_call
|
||||
CFI_ENDPROC
|
||||
END(stub_execve)
|
||||
/*
|
||||
* Remaining execve stubs are only 7 bytes long.
|
||||
@@ -527,32 +445,23 @@ END(stub_execve)
|
||||
*/
|
||||
.align 8
|
||||
GLOBAL(stub_execveat)
|
||||
CFI_STARTPROC
|
||||
DEFAULT_FRAME 0, 8
|
||||
call sys_execveat
|
||||
jmp return_from_execve
|
||||
CFI_ENDPROC
|
||||
END(stub_execveat)
|
||||
|
||||
#if defined(CONFIG_X86_X32_ABI) || defined(CONFIG_IA32_EMULATION)
|
||||
.align 8
|
||||
GLOBAL(stub_x32_execve)
|
||||
GLOBAL(stub32_execve)
|
||||
CFI_STARTPROC
|
||||
DEFAULT_FRAME 0, 8
|
||||
call compat_sys_execve
|
||||
jmp return_from_execve
|
||||
CFI_ENDPROC
|
||||
END(stub32_execve)
|
||||
END(stub_x32_execve)
|
||||
.align 8
|
||||
GLOBAL(stub_x32_execveat)
|
||||
GLOBAL(stub32_execveat)
|
||||
CFI_STARTPROC
|
||||
DEFAULT_FRAME 0, 8
|
||||
call compat_sys_execveat
|
||||
jmp return_from_execve
|
||||
CFI_ENDPROC
|
||||
END(stub32_execveat)
|
||||
END(stub_x32_execveat)
|
||||
#endif
|
||||
@@ -562,8 +471,6 @@ END(stub_x32_execveat)
|
||||
* This cannot be done with SYSRET, so use the IRET return path instead.
|
||||
*/
|
||||
ENTRY(stub_rt_sigreturn)
|
||||
CFI_STARTPROC
|
||||
DEFAULT_FRAME 0, 8
|
||||
/*
|
||||
* SAVE_EXTRA_REGS result is not normally needed:
|
||||
* sigreturn overwrites all pt_regs->GPREGS.
|
||||
@@ -575,21 +482,16 @@ ENTRY(stub_rt_sigreturn)
|
||||
call sys_rt_sigreturn
|
||||
return_from_stub:
|
||||
addq $8, %rsp
|
||||
CFI_ADJUST_CFA_OFFSET -8
|
||||
RESTORE_EXTRA_REGS
|
||||
movq %rax,RAX(%rsp)
|
||||
jmp int_ret_from_sys_call
|
||||
CFI_ENDPROC
|
||||
END(stub_rt_sigreturn)
|
||||
|
||||
#ifdef CONFIG_X86_X32_ABI
|
||||
ENTRY(stub_x32_rt_sigreturn)
|
||||
CFI_STARTPROC
|
||||
DEFAULT_FRAME 0, 8
|
||||
SAVE_EXTRA_REGS 8
|
||||
call sys32_x32_rt_sigreturn
|
||||
jmp return_from_stub
|
||||
CFI_ENDPROC
|
||||
END(stub_x32_rt_sigreturn)
|
||||
#endif
|
||||
|
||||
@@ -599,12 +501,11 @@ END(stub_x32_rt_sigreturn)
|
||||
* rdi: prev task we switched from
|
||||
*/
|
||||
ENTRY(ret_from_fork)
|
||||
DEFAULT_FRAME
|
||||
|
||||
LOCK ; btr $TIF_FORK,TI_flags(%r8)
|
||||
|
||||
pushq_cfi $0x0002
|
||||
popfq_cfi # reset kernel eflags
|
||||
pushq $0x0002
|
||||
popfq # reset kernel eflags
|
||||
|
||||
call schedule_tail # rdi: 'prev' task parameter
|
||||
|
||||
@@ -628,7 +529,6 @@ ENTRY(ret_from_fork)
|
||||
movl $0, RAX(%rsp)
|
||||
RESTORE_EXTRA_REGS
|
||||
jmp int_ret_from_sys_call
|
||||
CFI_ENDPROC
|
||||
END(ret_from_fork)
|
||||
|
||||
/*
|
||||
@@ -637,16 +537,13 @@ END(ret_from_fork)
|
||||
*/
|
||||
.align 8
|
||||
ENTRY(irq_entries_start)
|
||||
INTR_FRAME
|
||||
vector=FIRST_EXTERNAL_VECTOR
|
||||
.rept (FIRST_SYSTEM_VECTOR - FIRST_EXTERNAL_VECTOR)
|
||||
pushq_cfi $(~vector+0x80) /* Note: always in signed byte range */
|
||||
pushq $(~vector+0x80) /* Note: always in signed byte range */
|
||||
vector=vector+1
|
||||
jmp common_interrupt
|
||||
CFI_ADJUST_CFA_OFFSET -8
|
||||
.align 8
|
||||
.endr
|
||||
CFI_ENDPROC
|
||||
END(irq_entries_start)
|
||||
|
||||
/*
|
||||
@@ -688,17 +585,7 @@ END(irq_entries_start)
|
||||
movq %rsp, %rsi
|
||||
incl PER_CPU_VAR(irq_count)
|
||||
cmovzq PER_CPU_VAR(irq_stack_ptr),%rsp
|
||||
CFI_DEF_CFA_REGISTER rsi
|
||||
pushq %rsi
|
||||
/*
|
||||
* For debugger:
|
||||
* "CFA (Current Frame Address) is the value on stack + offset"
|
||||
*/
|
||||
CFI_ESCAPE 0x0f /* DW_CFA_def_cfa_expression */, 6, \
|
||||
0x77 /* DW_OP_breg7 (rsp) */, 0, \
|
||||
0x06 /* DW_OP_deref */, \
|
||||
0x08 /* DW_OP_const1u */, SIZEOF_PTREGS-RBP, \
|
||||
0x22 /* DW_OP_plus */
|
||||
/* We entered an interrupt context - irqs are off: */
|
||||
TRACE_IRQS_OFF
|
||||
|
||||
@@ -711,7 +598,6 @@ END(irq_entries_start)
|
||||
*/
|
||||
.p2align CONFIG_X86_L1_CACHE_SHIFT
|
||||
common_interrupt:
|
||||
XCPT_FRAME
|
||||
ASM_CLAC
|
||||
addq $-0x80,(%rsp) /* Adjust vector to [-256,-1] range */
|
||||
interrupt do_IRQ
|
||||
@@ -723,11 +609,8 @@ ret_from_intr:
|
||||
|
||||
/* Restore saved previous stack */
|
||||
popq %rsi
|
||||
CFI_DEF_CFA rsi,SIZEOF_PTREGS-RBP /* reg/off reset after def_cfa_expr */
|
||||
/* return code expects complete pt_regs - adjust rsp accordingly: */
|
||||
leaq -RBP(%rsi),%rsp
|
||||
CFI_DEF_CFA_REGISTER rsp
|
||||
CFI_ADJUST_CFA_OFFSET RBP
|
||||
|
||||
testb $3, CS(%rsp)
|
||||
jz retint_kernel
|
||||
@@ -743,7 +626,6 @@ retint_check:
|
||||
LOCKDEP_SYS_EXIT_IRQ
|
||||
movl TI_flags(%rcx),%edx
|
||||
andl %edi,%edx
|
||||
CFI_REMEMBER_STATE
|
||||
jnz retint_careful
|
||||
|
||||
retint_swapgs: /* return to user-space */
|
||||
@@ -807,8 +689,8 @@ native_irq_return_iret:
|
||||
|
||||
#ifdef CONFIG_X86_ESPFIX64
|
||||
native_irq_return_ldt:
|
||||
pushq_cfi %rax
|
||||
pushq_cfi %rdi
|
||||
pushq %rax
|
||||
pushq %rdi
|
||||
SWAPGS
|
||||
movq PER_CPU_VAR(espfix_waddr),%rdi
|
||||
movq %rax,(0*8)(%rdi) /* RAX */
|
||||
@@ -823,24 +705,23 @@ native_irq_return_ldt:
|
||||
movq (5*8)(%rsp),%rax /* RSP */
|
||||
movq %rax,(4*8)(%rdi)
|
||||
andl $0xffff0000,%eax
|
||||
popq_cfi %rdi
|
||||
popq %rdi
|
||||
orq PER_CPU_VAR(espfix_stack),%rax
|
||||
SWAPGS
|
||||
movq %rax,%rsp
|
||||
popq_cfi %rax
|
||||
popq %rax
|
||||
jmp native_irq_return_iret
|
||||
#endif
|
||||
|
||||
/* edi: workmask, edx: work */
|
||||
retint_careful:
|
||||
CFI_RESTORE_STATE
|
||||
bt $TIF_NEED_RESCHED,%edx
|
||||
jnc retint_signal
|
||||
TRACE_IRQS_ON
|
||||
ENABLE_INTERRUPTS(CLBR_NONE)
|
||||
pushq_cfi %rdi
|
||||
pushq %rdi
|
||||
SCHEDULE_USER
|
||||
popq_cfi %rdi
|
||||
popq %rdi
|
||||
GET_THREAD_INFO(%rcx)
|
||||
DISABLE_INTERRUPTS(CLBR_NONE)
|
||||
TRACE_IRQS_OFF
|
||||
@@ -862,7 +743,6 @@ retint_signal:
|
||||
GET_THREAD_INFO(%rcx)
|
||||
jmp retint_with_reschedule
|
||||
|
||||
CFI_ENDPROC
|
||||
END(common_interrupt)
|
||||
|
||||
/*
|
||||
@@ -870,13 +750,11 @@ END(common_interrupt)
|
||||
*/
|
||||
.macro apicinterrupt3 num sym do_sym
|
||||
ENTRY(\sym)
|
||||
INTR_FRAME
|
||||
ASM_CLAC
|
||||
pushq_cfi $~(\num)
|
||||
pushq $~(\num)
|
||||
.Lcommon_\sym:
|
||||
interrupt \do_sym
|
||||
jmp ret_from_intr
|
||||
CFI_ENDPROC
|
||||
END(\sym)
|
||||
.endm
|
||||
|
||||
@@ -959,24 +837,17 @@ ENTRY(\sym)
|
||||
.error "using shift_ist requires paranoid=1"
|
||||
.endif
|
||||
|
||||
.if \has_error_code
|
||||
XCPT_FRAME
|
||||
.else
|
||||
INTR_FRAME
|
||||
.endif
|
||||
|
||||
ASM_CLAC
|
||||
PARAVIRT_ADJUST_EXCEPTION_FRAME
|
||||
|
||||
.ifeq \has_error_code
|
||||
pushq_cfi $-1 /* ORIG_RAX: no syscall to restart */
|
||||
pushq $-1 /* ORIG_RAX: no syscall to restart */
|
||||
.endif
|
||||
|
||||
ALLOC_PT_GPREGS_ON_STACK
|
||||
|
||||
.if \paranoid
|
||||
.if \paranoid == 1
|
||||
CFI_REMEMBER_STATE
|
||||
testb $3, CS(%rsp) /* If coming from userspace, switch */
|
||||
jnz 1f /* stacks. */
|
||||
.endif
|
||||
@@ -986,8 +857,6 @@ ENTRY(\sym)
|
||||
.endif
|
||||
/* returned flag: ebx=0: need swapgs on exit, ebx=1: don't need it */
|
||||
|
||||
DEFAULT_FRAME 0
|
||||
|
||||
.if \paranoid
|
||||
.if \shift_ist != -1
|
||||
TRACE_IRQS_OFF_DEBUG /* reload IDT in case of recursion */
|
||||
@@ -1023,7 +892,6 @@ ENTRY(\sym)
|
||||
.endif
|
||||
|
||||
.if \paranoid == 1
|
||||
CFI_RESTORE_STATE
|
||||
/*
|
||||
* Paranoid entry from userspace. Switch stacks and treat it
|
||||
* as a normal entry. This means that paranoid handlers
|
||||
@@ -1032,7 +900,6 @@ ENTRY(\sym)
|
||||
1:
|
||||
call error_entry
|
||||
|
||||
DEFAULT_FRAME 0
|
||||
|
||||
movq %rsp,%rdi /* pt_regs pointer */
|
||||
call sync_regs
|
||||
@@ -1051,8 +918,6 @@ ENTRY(\sym)
|
||||
|
||||
jmp error_exit /* %ebx: no swapgs flag */
|
||||
.endif
|
||||
|
||||
CFI_ENDPROC
|
||||
END(\sym)
|
||||
.endm
|
||||
|
||||
@@ -1085,17 +950,15 @@ idtentry simd_coprocessor_error do_simd_coprocessor_error has_error_code=0
|
||||
/* Reload gs selector with exception handling */
|
||||
/* edi: new selector */
|
||||
ENTRY(native_load_gs_index)
|
||||
CFI_STARTPROC
|
||||
pushfq_cfi
|
||||
pushfq
|
||||
DISABLE_INTERRUPTS(CLBR_ANY & ~CLBR_RDI)
|
||||
SWAPGS
|
||||
gs_change:
|
||||
movl %edi,%gs
|
||||
2: mfence /* workaround */
|
||||
SWAPGS
|
||||
popfq_cfi
|
||||
popfq
|
||||
ret
|
||||
CFI_ENDPROC
|
||||
END(native_load_gs_index)
|
||||
|
||||
_ASM_EXTABLE(gs_change,bad_gs)
|
||||
@@ -1110,22 +973,15 @@ bad_gs:
|
||||
|
||||
/* Call softirq on interrupt stack. Interrupts are off. */
|
||||
ENTRY(do_softirq_own_stack)
|
||||
CFI_STARTPROC
|
||||
pushq_cfi %rbp
|
||||
CFI_REL_OFFSET rbp,0
|
||||
pushq %rbp
|
||||
mov %rsp,%rbp
|
||||
CFI_DEF_CFA_REGISTER rbp
|
||||
incl PER_CPU_VAR(irq_count)
|
||||
cmove PER_CPU_VAR(irq_stack_ptr),%rsp
|
||||
push %rbp # backlink for old unwinder
|
||||
call __do_softirq
|
||||
leaveq
|
||||
CFI_RESTORE rbp
|
||||
CFI_DEF_CFA_REGISTER rsp
|
||||
CFI_ADJUST_CFA_OFFSET -8
|
||||
decl PER_CPU_VAR(irq_count)
|
||||
ret
|
||||
CFI_ENDPROC
|
||||
END(do_softirq_own_stack)
|
||||
|
||||
#ifdef CONFIG_XEN
|
||||
@@ -1145,28 +1001,22 @@ idtentry xen_hypervisor_callback xen_do_hypervisor_callback has_error_code=0
|
||||
* activation and restart the handler using the previous one.
|
||||
*/
|
||||
ENTRY(xen_do_hypervisor_callback) # do_hypervisor_callback(struct *pt_regs)
|
||||
CFI_STARTPROC
|
||||
/*
|
||||
* Since we don't modify %rdi, evtchn_do_upall(struct *pt_regs) will
|
||||
* see the correct pointer to the pt_regs
|
||||
*/
|
||||
movq %rdi, %rsp # we don't return, adjust the stack frame
|
||||
CFI_ENDPROC
|
||||
DEFAULT_FRAME
|
||||
11: incl PER_CPU_VAR(irq_count)
|
||||
movq %rsp,%rbp
|
||||
CFI_DEF_CFA_REGISTER rbp
|
||||
cmovzq PER_CPU_VAR(irq_stack_ptr),%rsp
|
||||
pushq %rbp # backlink for old unwinder
|
||||
call xen_evtchn_do_upcall
|
||||
popq %rsp
|
||||
CFI_DEF_CFA_REGISTER rsp
|
||||
decl PER_CPU_VAR(irq_count)
|
||||
#ifndef CONFIG_PREEMPT
|
||||
call xen_maybe_preempt_hcall
|
||||
#endif
|
||||
jmp error_exit
|
||||
CFI_ENDPROC
|
||||
END(xen_do_hypervisor_callback)
|
||||
|
||||
/*
|
||||
@@ -1183,16 +1033,8 @@ END(xen_do_hypervisor_callback)
|
||||
* with its current contents: any discrepancy means we in category 1.
|
||||
*/
|
||||
ENTRY(xen_failsafe_callback)
|
||||
INTR_FRAME 1 (6*8)
|
||||
/*CFI_REL_OFFSET gs,GS*/
|
||||
/*CFI_REL_OFFSET fs,FS*/
|
||||
/*CFI_REL_OFFSET es,ES*/
|
||||
/*CFI_REL_OFFSET ds,DS*/
|
||||
CFI_REL_OFFSET r11,8
|
||||
CFI_REL_OFFSET rcx,0
|
||||
movl %ds,%ecx
|
||||
cmpw %cx,0x10(%rsp)
|
||||
CFI_REMEMBER_STATE
|
||||
jne 1f
|
||||
movl %es,%ecx
|
||||
cmpw %cx,0x18(%rsp)
|
||||
@@ -1205,29 +1047,21 @@ ENTRY(xen_failsafe_callback)
|
||||
jne 1f
|
||||
/* All segments match their saved values => Category 2 (Bad IRET). */
|
||||
movq (%rsp),%rcx
|
||||
CFI_RESTORE rcx
|
||||
movq 8(%rsp),%r11
|
||||
CFI_RESTORE r11
|
||||
addq $0x30,%rsp
|
||||
CFI_ADJUST_CFA_OFFSET -0x30
|
||||
pushq_cfi $0 /* RIP */
|
||||
pushq_cfi %r11
|
||||
pushq_cfi %rcx
|
||||
pushq $0 /* RIP */
|
||||
pushq %r11
|
||||
pushq %rcx
|
||||
jmp general_protection
|
||||
CFI_RESTORE_STATE
|
||||
1: /* Segment mismatch => Category 1 (Bad segment). Retry the IRET. */
|
||||
movq (%rsp),%rcx
|
||||
CFI_RESTORE rcx
|
||||
movq 8(%rsp),%r11
|
||||
CFI_RESTORE r11
|
||||
addq $0x30,%rsp
|
||||
CFI_ADJUST_CFA_OFFSET -0x30
|
||||
pushq_cfi $-1 /* orig_ax = -1 => not a system call */
|
||||
pushq $-1 /* orig_ax = -1 => not a system call */
|
||||
ALLOC_PT_GPREGS_ON_STACK
|
||||
SAVE_C_REGS
|
||||
SAVE_EXTRA_REGS
|
||||
jmp error_exit
|
||||
CFI_ENDPROC
|
||||
END(xen_failsafe_callback)
|
||||
|
||||
apicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \
|
||||
@@ -1263,7 +1097,6 @@ idtentry machine_check has_error_code=0 paranoid=1 do_sym=*machine_check_vector(
|
||||
* Return: ebx=0: need swapgs on exit, ebx=1: otherwise
|
||||
*/
|
||||
ENTRY(paranoid_entry)
|
||||
XCPT_FRAME 1 15*8
|
||||
cld
|
||||
SAVE_C_REGS 8
|
||||
SAVE_EXTRA_REGS 8
|
||||
@@ -1275,7 +1108,6 @@ ENTRY(paranoid_entry)
|
||||
SWAPGS
|
||||
xorl %ebx,%ebx
|
||||
1: ret
|
||||
CFI_ENDPROC
|
||||
END(paranoid_entry)
|
||||
|
||||
/*
|
||||
@@ -1290,7 +1122,6 @@ END(paranoid_entry)
|
||||
*/
|
||||
/* On entry, ebx is "no swapgs" flag (1: don't need swapgs, 0: need it) */
|
||||
ENTRY(paranoid_exit)
|
||||
DEFAULT_FRAME
|
||||
DISABLE_INTERRUPTS(CLBR_NONE)
|
||||
TRACE_IRQS_OFF_DEBUG
|
||||
testl %ebx,%ebx /* swapgs needed? */
|
||||
@@ -1305,7 +1136,6 @@ paranoid_exit_restore:
|
||||
RESTORE_C_REGS
|
||||
REMOVE_PT_GPREGS_FROM_STACK 8
|
||||
INTERRUPT_RETURN
|
||||
CFI_ENDPROC
|
||||
END(paranoid_exit)
|
||||
|
||||
/*
|
||||
@@ -1313,7 +1143,6 @@ END(paranoid_exit)
|
||||
* Return: ebx=0: need swapgs on exit, ebx=1: otherwise
|
||||
*/
|
||||
ENTRY(error_entry)
|
||||
XCPT_FRAME 1 15*8
|
||||
cld
|
||||
SAVE_C_REGS 8
|
||||
SAVE_EXTRA_REGS 8
|
||||
@@ -1333,7 +1162,6 @@ error_sti:
|
||||
* for these here too.
|
||||
*/
|
||||
error_kernelspace:
|
||||
CFI_REL_OFFSET rcx, RCX+8
|
||||
incl %ebx
|
||||
leaq native_irq_return_iret(%rip),%rcx
|
||||
cmpq %rcx,RIP+8(%rsp)
|
||||
@@ -1357,13 +1185,11 @@ error_bad_iret:
|
||||
mov %rax,%rsp
|
||||
decl %ebx /* Return to usergs */
|
||||
jmp error_sti
|
||||
CFI_ENDPROC
|
||||
END(error_entry)
|
||||
|
||||
|
||||
/* On entry, ebx is "no swapgs" flag (1: don't need swapgs, 0: need it) */
|
||||
ENTRY(error_exit)
|
||||
DEFAULT_FRAME
|
||||
movl %ebx,%eax
|
||||
RESTORE_EXTRA_REGS
|
||||
DISABLE_INTERRUPTS(CLBR_NONE)
|
||||
@@ -1377,12 +1203,10 @@ ENTRY(error_exit)
|
||||
andl %edi,%edx
|
||||
jnz retint_careful
|
||||
jmp retint_swapgs
|
||||
CFI_ENDPROC
|
||||
END(error_exit)
|
||||
|
||||
/* Runs on exception stack */
|
||||
ENTRY(nmi)
|
||||
INTR_FRAME
|
||||
PARAVIRT_ADJUST_EXCEPTION_FRAME
|
||||
/*
|
||||
* We allow breakpoints in NMIs. If a breakpoint occurs, then
|
||||
@@ -1417,8 +1241,7 @@ ENTRY(nmi)
|
||||
*/
|
||||
|
||||
/* Use %rdx as our temp variable throughout */
|
||||
pushq_cfi %rdx
|
||||
CFI_REL_OFFSET rdx, 0
|
||||
pushq %rdx
|
||||
|
||||
/*
|
||||
* If %cs was not the kernel segment, then the NMI triggered in user
|
||||
@@ -1452,8 +1275,6 @@ ENTRY(nmi)
|
||||
jb first_nmi
|
||||
/* Ah, it is within the NMI stack, treat it as nested */
|
||||
|
||||
CFI_REMEMBER_STATE
|
||||
|
||||
nested_nmi:
|
||||
/*
|
||||
* Do nothing if we interrupted the fixup in repeat_nmi.
|
||||
@@ -1471,26 +1292,22 @@ nested_nmi:
|
||||
/* Set up the interrupted NMIs stack to jump to repeat_nmi */
|
||||
leaq -1*8(%rsp), %rdx
|
||||
movq %rdx, %rsp
|
||||
CFI_ADJUST_CFA_OFFSET 1*8
|
||||
leaq -10*8(%rsp), %rdx
|
||||
pushq_cfi $__KERNEL_DS
|
||||
pushq_cfi %rdx
|
||||
pushfq_cfi
|
||||
pushq_cfi $__KERNEL_CS
|
||||
pushq_cfi $repeat_nmi
|
||||
pushq $__KERNEL_DS
|
||||
pushq %rdx
|
||||
pushfq
|
||||
pushq $__KERNEL_CS
|
||||
pushq $repeat_nmi
|
||||
|
||||
/* Put stack back */
|
||||
addq $(6*8), %rsp
|
||||
CFI_ADJUST_CFA_OFFSET -6*8
|
||||
|
||||
nested_nmi_out:
|
||||
popq_cfi %rdx
|
||||
CFI_RESTORE rdx
|
||||
popq %rdx
|
||||
|
||||
/* No need to check faults here */
|
||||
INTERRUPT_RETURN
|
||||
|
||||
CFI_RESTORE_STATE
|
||||
first_nmi:
|
||||
/*
|
||||
* Because nested NMIs will use the pushed location that we
|
||||
@@ -1529,22 +1346,19 @@ first_nmi:
|
||||
*/
|
||||
/* Do not pop rdx, nested NMIs will corrupt that part of the stack */
|
||||
movq (%rsp), %rdx
|
||||
CFI_RESTORE rdx
|
||||
|
||||
/* Set the NMI executing variable on the stack. */
|
||||
pushq_cfi $1
|
||||
pushq $1
|
||||
|
||||
/*
|
||||
* Leave room for the "copied" frame
|
||||
*/
|
||||
subq $(5*8), %rsp
|
||||
CFI_ADJUST_CFA_OFFSET 5*8
|
||||
|
||||
/* Copy the stack frame to the Saved frame */
|
||||
.rept 5
|
||||
pushq_cfi 11*8(%rsp)
|
||||
pushq 11*8(%rsp)
|
||||
.endr
|
||||
CFI_DEF_CFA_OFFSET 5*8
|
||||
|
||||
/* Everything up to here is safe from nested NMIs */
|
||||
|
||||
@@ -1567,12 +1381,10 @@ repeat_nmi:
|
||||
|
||||
/* Make another copy, this one may be modified by nested NMIs */
|
||||
addq $(10*8), %rsp
|
||||
CFI_ADJUST_CFA_OFFSET -10*8
|
||||
.rept 5
|
||||
pushq_cfi -6*8(%rsp)
|
||||
pushq -6*8(%rsp)
|
||||
.endr
|
||||
subq $(5*8), %rsp
|
||||
CFI_DEF_CFA_OFFSET 5*8
|
||||
end_repeat_nmi:
|
||||
|
||||
/*
|
||||
@@ -1580,7 +1392,7 @@ end_repeat_nmi:
|
||||
* NMI if the first NMI took an exception and reset our iret stack
|
||||
* so that we repeat another NMI.
|
||||
*/
|
||||
pushq_cfi $-1 /* ORIG_RAX: no syscall to restart */
|
||||
pushq $-1 /* ORIG_RAX: no syscall to restart */
|
||||
ALLOC_PT_GPREGS_ON_STACK
|
||||
|
||||
/*
|
||||
@@ -1591,7 +1403,6 @@ end_repeat_nmi:
|
||||
* exceptions might do.
|
||||
*/
|
||||
call paranoid_entry
|
||||
DEFAULT_FRAME 0
|
||||
|
||||
/*
|
||||
* Save off the CR2 register. If we take a page fault in the NMI then
|
||||
@@ -1628,13 +1439,10 @@ nmi_restore:
|
||||
/* Clear the NMI executing stack variable */
|
||||
movq $0, 5*8(%rsp)
|
||||
jmp irq_return
|
||||
CFI_ENDPROC
|
||||
END(nmi)
|
||||
|
||||
ENTRY(ignore_sysret)
|
||||
CFI_STARTPROC
|
||||
mov $-ENOSYS,%eax
|
||||
sysret
|
||||
CFI_ENDPROC
|
||||
END(ignore_sysret)
|
||||
|
||||
|
Reference in New Issue
Block a user