entry_64.S 43 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. /*
  3. * linux/arch/x86_64/entry.S
  4. *
  5. * Copyright (C) 1991, 1992 Linus Torvalds
  6. * Copyright (C) 2000, 2001, 2002 Andi Kleen SuSE Labs
  7. * Copyright (C) 2000 Pavel Machek <[email protected]>
  8. *
  9. * entry.S contains the system-call and fault low-level handling routines.
  10. *
  11. * Some of this is documented in Documentation/x86/entry_64.rst
  12. *
  13. * A note on terminology:
  14. * - iret frame: Architecture defined interrupt frame from SS to RIP
  15. * at the top of the kernel process stack.
  16. *
  17. * Some macro usage:
  18. * - SYM_FUNC_START/END:Define functions in the symbol table.
  19. * - idtentry: Define exception entry points.
  20. */
  21. #include <linux/linkage.h>
  22. #include <asm/segment.h>
  23. #include <asm/cache.h>
  24. #include <asm/errno.h>
  25. #include <asm/asm-offsets.h>
  26. #include <asm/msr.h>
  27. #include <asm/unistd.h>
  28. #include <asm/thread_info.h>
  29. #include <asm/hw_irq.h>
  30. #include <asm/page_types.h>
  31. #include <asm/irqflags.h>
  32. #include <asm/paravirt.h>
  33. #include <asm/percpu.h>
  34. #include <asm/asm.h>
  35. #include <asm/smap.h>
  36. #include <asm/pgtable_types.h>
  37. #include <asm/export.h>
  38. #include <asm/frame.h>
  39. #include <asm/trapnr.h>
  40. #include <asm/nospec-branch.h>
  41. #include <asm/fsgsbase.h>
  42. #include <linux/err.h>
  43. #include "calling.h"
  44. .code64
  45. .section .entry.text, "ax"
  46. /*
  47. * 64-bit SYSCALL instruction entry. Up to 6 arguments in registers.
  48. *
  49. * This is the only entry point used for 64-bit system calls. The
  50. * hardware interface is reasonably well designed and the register to
  51. * argument mapping Linux uses fits well with the registers that are
  52. * available when SYSCALL is used.
  53. *
  54. * SYSCALL instructions can be found inlined in libc implementations as
  55. * well as some other programs and libraries. There are also a handful
  56. * of SYSCALL instructions in the vDSO used, for example, as a
  57. * clock_gettimeofday fallback.
  58. *
  59. * 64-bit SYSCALL saves rip to rcx, clears rflags.RF, then saves rflags to r11,
  60. * then loads new ss, cs, and rip from previously programmed MSRs.
  61. * rflags gets masked by a value from another MSR (so CLD and CLAC
  62. * are not needed). SYSCALL does not save anything on the stack
  63. * and does not change rsp.
  64. *
  65. * Registers on entry:
  66. * rax system call number
  67. * rcx return address
  68. * r11 saved rflags (note: r11 is callee-clobbered register in C ABI)
  69. * rdi arg0
  70. * rsi arg1
  71. * rdx arg2
  72. * r10 arg3 (needs to be moved to rcx to conform to C ABI)
  73. * r8 arg4
  74. * r9 arg5
  75. * (note: r12-r15, rbp, rbx are callee-preserved in C ABI)
  76. *
  77. * Only called from user space.
  78. *
  79. * When user can change pt_regs->foo always force IRET. That is because
  80. * it deals with uncanonical addresses better. SYSRET has trouble
  81. * with them due to bugs in both AMD and Intel CPUs.
  82. */
  83. SYM_CODE_START(entry_SYSCALL_64)
  84. UNWIND_HINT_ENTRY
  85. ENDBR
  86. swapgs
  87. /* tss.sp2 is scratch space. */
  88. movq %rsp, PER_CPU_VAR(cpu_tss_rw + TSS_sp2)
  89. SWITCH_TO_KERNEL_CR3 scratch_reg=%rsp
  90. movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp
  91. SYM_INNER_LABEL(entry_SYSCALL_64_safe_stack, SYM_L_GLOBAL)
  92. ANNOTATE_NOENDBR
  93. /* Construct struct pt_regs on stack */
  94. pushq $__USER_DS /* pt_regs->ss */
  95. pushq PER_CPU_VAR(cpu_tss_rw + TSS_sp2) /* pt_regs->sp */
  96. pushq %r11 /* pt_regs->flags */
  97. pushq $__USER_CS /* pt_regs->cs */
  98. pushq %rcx /* pt_regs->ip */
  99. SYM_INNER_LABEL(entry_SYSCALL_64_after_hwframe, SYM_L_GLOBAL)
  100. pushq %rax /* pt_regs->orig_ax */
  101. PUSH_AND_CLEAR_REGS rax=$-ENOSYS
  102. /* IRQs are off. */
  103. movq %rsp, %rdi
  104. /* Sign extend the lower 32bit as syscall numbers are treated as int */
  105. movslq %eax, %rsi
  106. /* clobbers %rax, make sure it is after saving the syscall nr */
  107. IBRS_ENTER
  108. UNTRAIN_RET
  109. call do_syscall_64 /* returns with IRQs disabled */
  110. /*
  111. * Try to use SYSRET instead of IRET if we're returning to
  112. * a completely clean 64-bit userspace context. If we're not,
  113. * go to the slow exit path.
  114. * In the Xen PV case we must use iret anyway.
  115. */
  116. ALTERNATIVE "", "jmp swapgs_restore_regs_and_return_to_usermode", \
  117. X86_FEATURE_XENPV
  118. movq RCX(%rsp), %rcx
  119. movq RIP(%rsp), %r11
  120. cmpq %rcx, %r11 /* SYSRET requires RCX == RIP */
  121. jne swapgs_restore_regs_and_return_to_usermode
  122. /*
  123. * On Intel CPUs, SYSRET with non-canonical RCX/RIP will #GP
  124. * in kernel space. This essentially lets the user take over
  125. * the kernel, since userspace controls RSP.
  126. *
  127. * If width of "canonical tail" ever becomes variable, this will need
  128. * to be updated to remain correct on both old and new CPUs.
  129. *
  130. * Change top bits to match most significant bit (47th or 56th bit
  131. * depending on paging mode) in the address.
  132. */
  133. #ifdef CONFIG_X86_5LEVEL
  134. ALTERNATIVE "shl $(64 - 48), %rcx; sar $(64 - 48), %rcx", \
  135. "shl $(64 - 57), %rcx; sar $(64 - 57), %rcx", X86_FEATURE_LA57
  136. #else
  137. shl $(64 - (__VIRTUAL_MASK_SHIFT+1)), %rcx
  138. sar $(64 - (__VIRTUAL_MASK_SHIFT+1)), %rcx
  139. #endif
  140. /* If this changed %rcx, it was not canonical */
  141. cmpq %rcx, %r11
  142. jne swapgs_restore_regs_and_return_to_usermode
  143. cmpq $__USER_CS, CS(%rsp) /* CS must match SYSRET */
  144. jne swapgs_restore_regs_and_return_to_usermode
  145. movq R11(%rsp), %r11
  146. cmpq %r11, EFLAGS(%rsp) /* R11 == RFLAGS */
  147. jne swapgs_restore_regs_and_return_to_usermode
  148. /*
  149. * SYSCALL clears RF when it saves RFLAGS in R11 and SYSRET cannot
  150. * restore RF properly. If the slowpath sets it for whatever reason, we
  151. * need to restore it correctly.
  152. *
  153. * SYSRET can restore TF, but unlike IRET, restoring TF results in a
  154. * trap from userspace immediately after SYSRET. This would cause an
  155. * infinite loop whenever #DB happens with register state that satisfies
  156. * the opportunistic SYSRET conditions. For example, single-stepping
  157. * this user code:
  158. *
  159. * movq $stuck_here, %rcx
  160. * pushfq
  161. * popq %r11
  162. * stuck_here:
  163. *
  164. * would never get past 'stuck_here'.
  165. */
  166. testq $(X86_EFLAGS_RF|X86_EFLAGS_TF), %r11
  167. jnz swapgs_restore_regs_and_return_to_usermode
  168. /* nothing to check for RSP */
  169. cmpq $__USER_DS, SS(%rsp) /* SS must match SYSRET */
  170. jne swapgs_restore_regs_and_return_to_usermode
  171. /*
  172. * We win! This label is here just for ease of understanding
  173. * perf profiles. Nothing jumps here.
  174. */
  175. syscall_return_via_sysret:
  176. IBRS_EXIT
  177. POP_REGS pop_rdi=0
  178. /*
  179. * Now all regs are restored except RSP and RDI.
  180. * Save old stack pointer and switch to trampoline stack.
  181. */
  182. movq %rsp, %rdi
  183. movq PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %rsp
  184. UNWIND_HINT_EMPTY
  185. pushq RSP-RDI(%rdi) /* RSP */
  186. pushq (%rdi) /* RDI */
  187. /*
  188. * We are on the trampoline stack. All regs except RDI are live.
  189. * We can do future final exit work right here.
  190. */
  191. STACKLEAK_ERASE_NOCLOBBER
  192. SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi
  193. popq %rdi
  194. popq %rsp
  195. SYM_INNER_LABEL(entry_SYSRETQ_unsafe_stack, SYM_L_GLOBAL)
  196. ANNOTATE_NOENDBR
  197. swapgs
  198. sysretq
  199. SYM_INNER_LABEL(entry_SYSRETQ_end, SYM_L_GLOBAL)
  200. ANNOTATE_NOENDBR
  201. int3
  202. SYM_CODE_END(entry_SYSCALL_64)
  203. /*
  204. * %rdi: prev task
  205. * %rsi: next task
  206. */
  207. .pushsection .text, "ax"
  208. SYM_FUNC_START(__switch_to_asm)
  209. /*
  210. * Save callee-saved registers
  211. * This must match the order in inactive_task_frame
  212. */
  213. pushq %rbp
  214. pushq %rbx
  215. pushq %r12
  216. pushq %r13
  217. pushq %r14
  218. pushq %r15
  219. /* switch stack */
  220. movq %rsp, TASK_threadsp(%rdi)
  221. movq TASK_threadsp(%rsi), %rsp
  222. #ifdef CONFIG_STACKPROTECTOR
  223. movq TASK_stack_canary(%rsi), %rbx
  224. movq %rbx, PER_CPU_VAR(fixed_percpu_data) + stack_canary_offset
  225. #endif
  226. /*
  227. * When switching from a shallower to a deeper call stack
  228. * the RSB may either underflow or use entries populated
  229. * with userspace addresses. On CPUs where those concerns
  230. * exist, overwrite the RSB with entries which capture
  231. * speculative execution to prevent attack.
  232. */
  233. FILL_RETURN_BUFFER %r12, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_CTXSW
  234. /* restore callee-saved registers */
  235. popq %r15
  236. popq %r14
  237. popq %r13
  238. popq %r12
  239. popq %rbx
  240. popq %rbp
  241. jmp __switch_to
  242. SYM_FUNC_END(__switch_to_asm)
  243. .popsection
  244. /*
  245. * A newly forked process directly context switches into this address.
  246. *
  247. * rax: prev task we switched from
  248. * rbx: kernel thread func (NULL for user thread)
  249. * r12: kernel thread arg
  250. */
  251. .pushsection .text, "ax"
  252. SYM_CODE_START(ret_from_fork)
  253. UNWIND_HINT_EMPTY
  254. ANNOTATE_NOENDBR // copy_thread
  255. movq %rax, %rdi
  256. call schedule_tail /* rdi: 'prev' task parameter */
  257. testq %rbx, %rbx /* from kernel_thread? */
  258. jnz 1f /* kernel threads are uncommon */
  259. 2:
  260. UNWIND_HINT_REGS
  261. movq %rsp, %rdi
  262. call syscall_exit_to_user_mode /* returns with IRQs disabled */
  263. jmp swapgs_restore_regs_and_return_to_usermode
  264. 1:
  265. /* kernel thread */
  266. UNWIND_HINT_EMPTY
  267. movq %r12, %rdi
  268. CALL_NOSPEC rbx
  269. /*
  270. * A kernel thread is allowed to return here after successfully
  271. * calling kernel_execve(). Exit to userspace to complete the execve()
  272. * syscall.
  273. */
  274. movq $0, RAX(%rsp)
  275. jmp 2b
  276. SYM_CODE_END(ret_from_fork)
  277. .popsection
  278. .macro DEBUG_ENTRY_ASSERT_IRQS_OFF
  279. #ifdef CONFIG_DEBUG_ENTRY
  280. pushq %rax
  281. SAVE_FLAGS
  282. testl $X86_EFLAGS_IF, %eax
  283. jz .Lokay_\@
  284. ud2
  285. .Lokay_\@:
  286. popq %rax
  287. #endif
  288. .endm
  289. SYM_CODE_START_LOCAL(xen_error_entry)
  290. UNWIND_HINT_FUNC
  291. PUSH_AND_CLEAR_REGS save_ret=1
  292. ENCODE_FRAME_POINTER 8
  293. UNTRAIN_RET
  294. RET
  295. SYM_CODE_END(xen_error_entry)
  296. /**
  297. * idtentry_body - Macro to emit code calling the C function
  298. * @cfunc: C function to be called
  299. * @has_error_code: Hardware pushed error code on stack
  300. */
  301. .macro idtentry_body cfunc has_error_code:req
  302. /*
  303. * Call error_entry() and switch to the task stack if from userspace.
  304. *
  305. * When in XENPV, it is already in the task stack, and it can't fault
  306. * for native_iret() nor native_load_gs_index() since XENPV uses its
  307. * own pvops for IRET and load_gs_index(). And it doesn't need to
  308. * switch the CR3. So it can skip invoking error_entry().
  309. */
  310. ALTERNATIVE "call error_entry; movq %rax, %rsp", \
  311. "call xen_error_entry", X86_FEATURE_XENPV
  312. ENCODE_FRAME_POINTER
  313. UNWIND_HINT_REGS
  314. movq %rsp, %rdi /* pt_regs pointer into 1st argument*/
  315. .if \has_error_code == 1
  316. movq ORIG_RAX(%rsp), %rsi /* get error code into 2nd argument*/
  317. movq $-1, ORIG_RAX(%rsp) /* no syscall to restart */
  318. .endif
  319. call \cfunc
  320. /* For some configurations \cfunc ends up being a noreturn. */
  321. REACHABLE
  322. jmp error_return
  323. .endm
  324. /**
  325. * idtentry - Macro to generate entry stubs for simple IDT entries
  326. * @vector: Vector number
  327. * @asmsym: ASM symbol for the entry point
  328. * @cfunc: C function to be called
  329. * @has_error_code: Hardware pushed error code on stack
  330. *
  331. * The macro emits code to set up the kernel context for straight forward
  332. * and simple IDT entries. No IST stack, no paranoid entry checks.
  333. */
  334. .macro idtentry vector asmsym cfunc has_error_code:req
  335. SYM_CODE_START(\asmsym)
  336. UNWIND_HINT_IRET_REGS offset=\has_error_code*8
  337. ENDBR
  338. ASM_CLAC
  339. cld
  340. .if \has_error_code == 0
  341. pushq $-1 /* ORIG_RAX: no syscall to restart */
  342. .endif
  343. .if \vector == X86_TRAP_BP
  344. /*
  345. * If coming from kernel space, create a 6-word gap to allow the
  346. * int3 handler to emulate a call instruction.
  347. */
  348. testb $3, CS-ORIG_RAX(%rsp)
  349. jnz .Lfrom_usermode_no_gap_\@
  350. .rept 6
  351. pushq 5*8(%rsp)
  352. .endr
  353. UNWIND_HINT_IRET_REGS offset=8
  354. .Lfrom_usermode_no_gap_\@:
  355. .endif
  356. idtentry_body \cfunc \has_error_code
  357. _ASM_NOKPROBE(\asmsym)
  358. SYM_CODE_END(\asmsym)
  359. .endm
  360. /*
  361. * Interrupt entry/exit.
  362. *
  363. + The interrupt stubs push (vector) onto the stack, which is the error_code
  364. * position of idtentry exceptions, and jump to one of the two idtentry points
  365. * (common/spurious).
  366. *
  367. * common_interrupt is a hotpath, align it to a cache line
  368. */
  369. .macro idtentry_irq vector cfunc
  370. .p2align CONFIG_X86_L1_CACHE_SHIFT
  371. idtentry \vector asm_\cfunc \cfunc has_error_code=1
  372. .endm
  373. /*
  374. * System vectors which invoke their handlers directly and are not
  375. * going through the regular common device interrupt handling code.
  376. */
  377. .macro idtentry_sysvec vector cfunc
  378. idtentry \vector asm_\cfunc \cfunc has_error_code=0
  379. .endm
  380. /**
  381. * idtentry_mce_db - Macro to generate entry stubs for #MC and #DB
  382. * @vector: Vector number
  383. * @asmsym: ASM symbol for the entry point
  384. * @cfunc: C function to be called
  385. *
  386. * The macro emits code to set up the kernel context for #MC and #DB
  387. *
  388. * If the entry comes from user space it uses the normal entry path
  389. * including the return to user space work and preemption checks on
  390. * exit.
  391. *
  392. * If hits in kernel mode then it needs to go through the paranoid
  393. * entry as the exception can hit any random state. No preemption
  394. * check on exit to keep the paranoid path simple.
  395. */
  396. .macro idtentry_mce_db vector asmsym cfunc
  397. SYM_CODE_START(\asmsym)
  398. UNWIND_HINT_IRET_REGS
  399. ENDBR
  400. ASM_CLAC
  401. cld
  402. pushq $-1 /* ORIG_RAX: no syscall to restart */
  403. /*
  404. * If the entry is from userspace, switch stacks and treat it as
  405. * a normal entry.
  406. */
  407. testb $3, CS-ORIG_RAX(%rsp)
  408. jnz .Lfrom_usermode_switch_stack_\@
  409. /* paranoid_entry returns GS information for paranoid_exit in EBX. */
  410. call paranoid_entry
  411. UNWIND_HINT_REGS
  412. movq %rsp, %rdi /* pt_regs pointer */
  413. call \cfunc
  414. jmp paranoid_exit
  415. /* Switch to the regular task stack and use the noist entry point */
  416. .Lfrom_usermode_switch_stack_\@:
  417. idtentry_body noist_\cfunc, has_error_code=0
  418. _ASM_NOKPROBE(\asmsym)
  419. SYM_CODE_END(\asmsym)
  420. .endm
  421. #ifdef CONFIG_AMD_MEM_ENCRYPT
  422. /**
  423. * idtentry_vc - Macro to generate entry stub for #VC
  424. * @vector: Vector number
  425. * @asmsym: ASM symbol for the entry point
  426. * @cfunc: C function to be called
  427. *
  428. * The macro emits code to set up the kernel context for #VC. The #VC handler
  429. * runs on an IST stack and needs to be able to cause nested #VC exceptions.
  430. *
  431. * To make this work the #VC entry code tries its best to pretend it doesn't use
  432. * an IST stack by switching to the task stack if coming from user-space (which
  433. * includes early SYSCALL entry path) or back to the stack in the IRET frame if
  434. * entered from kernel-mode.
  435. *
  436. * If entered from kernel-mode the return stack is validated first, and if it is
  437. * not safe to use (e.g. because it points to the entry stack) the #VC handler
  438. * will switch to a fall-back stack (VC2) and call a special handler function.
  439. *
  440. * The macro is only used for one vector, but it is planned to be extended in
  441. * the future for the #HV exception.
  442. */
  443. .macro idtentry_vc vector asmsym cfunc
  444. SYM_CODE_START(\asmsym)
  445. UNWIND_HINT_IRET_REGS
  446. ENDBR
  447. ASM_CLAC
  448. cld
  449. /*
  450. * If the entry is from userspace, switch stacks and treat it as
  451. * a normal entry.
  452. */
  453. testb $3, CS-ORIG_RAX(%rsp)
  454. jnz .Lfrom_usermode_switch_stack_\@
  455. /*
  456. * paranoid_entry returns SWAPGS flag for paranoid_exit in EBX.
  457. * EBX == 0 -> SWAPGS, EBX == 1 -> no SWAPGS
  458. */
  459. call paranoid_entry
  460. UNWIND_HINT_REGS
  461. /*
  462. * Switch off the IST stack to make it free for nested exceptions. The
  463. * vc_switch_off_ist() function will switch back to the interrupted
  464. * stack if it is safe to do so. If not it switches to the VC fall-back
  465. * stack.
  466. */
  467. movq %rsp, %rdi /* pt_regs pointer */
  468. call vc_switch_off_ist
  469. movq %rax, %rsp /* Switch to new stack */
  470. ENCODE_FRAME_POINTER
  471. UNWIND_HINT_REGS
  472. /* Update pt_regs */
  473. movq ORIG_RAX(%rsp), %rsi /* get error code into 2nd argument*/
  474. movq $-1, ORIG_RAX(%rsp) /* no syscall to restart */
  475. movq %rsp, %rdi /* pt_regs pointer */
  476. call kernel_\cfunc
  477. /*
  478. * No need to switch back to the IST stack. The current stack is either
  479. * identical to the stack in the IRET frame or the VC fall-back stack,
  480. * so it is definitely mapped even with PTI enabled.
  481. */
  482. jmp paranoid_exit
  483. /* Switch to the regular task stack */
  484. .Lfrom_usermode_switch_stack_\@:
  485. idtentry_body user_\cfunc, has_error_code=1
  486. _ASM_NOKPROBE(\asmsym)
  487. SYM_CODE_END(\asmsym)
  488. .endm
  489. #endif
  490. /*
  491. * Double fault entry. Straight paranoid. No checks from which context
  492. * this comes because for the espfix induced #DF this would do the wrong
  493. * thing.
  494. */
  495. .macro idtentry_df vector asmsym cfunc
  496. SYM_CODE_START(\asmsym)
  497. UNWIND_HINT_IRET_REGS offset=8
  498. ENDBR
  499. ASM_CLAC
  500. cld
  501. /* paranoid_entry returns GS information for paranoid_exit in EBX. */
  502. call paranoid_entry
  503. UNWIND_HINT_REGS
  504. movq %rsp, %rdi /* pt_regs pointer into first argument */
  505. movq ORIG_RAX(%rsp), %rsi /* get error code into 2nd argument*/
  506. movq $-1, ORIG_RAX(%rsp) /* no syscall to restart */
  507. call \cfunc
  508. /* For some configurations \cfunc ends up being a noreturn. */
  509. REACHABLE
  510. jmp paranoid_exit
  511. _ASM_NOKPROBE(\asmsym)
  512. SYM_CODE_END(\asmsym)
  513. .endm
  514. /*
  515. * Include the defines which emit the idt entries which are shared
  516. * shared between 32 and 64 bit and emit the __irqentry_text_* markers
  517. * so the stacktrace boundary checks work.
  518. */
  519. .align 16
  520. .globl __irqentry_text_start
  521. __irqentry_text_start:
  522. #include <asm/idtentry.h>
  523. .align 16
  524. .globl __irqentry_text_end
  525. __irqentry_text_end:
  526. ANNOTATE_NOENDBR
  527. SYM_CODE_START_LOCAL(common_interrupt_return)
  528. SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL)
  529. IBRS_EXIT
  530. #ifdef CONFIG_DEBUG_ENTRY
  531. /* Assert that pt_regs indicates user mode. */
  532. testb $3, CS(%rsp)
  533. jnz 1f
  534. ud2
  535. 1:
  536. #endif
  537. #ifdef CONFIG_XEN_PV
  538. ALTERNATIVE "", "jmp xenpv_restore_regs_and_return_to_usermode", X86_FEATURE_XENPV
  539. #endif
  540. POP_REGS pop_rdi=0
  541. /*
  542. * The stack is now user RDI, orig_ax, RIP, CS, EFLAGS, RSP, SS.
  543. * Save old stack pointer and switch to trampoline stack.
  544. */
  545. movq %rsp, %rdi
  546. movq PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %rsp
  547. UNWIND_HINT_EMPTY
  548. /* Copy the IRET frame to the trampoline stack. */
  549. pushq 6*8(%rdi) /* SS */
  550. pushq 5*8(%rdi) /* RSP */
  551. pushq 4*8(%rdi) /* EFLAGS */
  552. pushq 3*8(%rdi) /* CS */
  553. pushq 2*8(%rdi) /* RIP */
  554. /* Push user RDI on the trampoline stack. */
  555. pushq (%rdi)
  556. /*
  557. * We are on the trampoline stack. All regs except RDI are live.
  558. * We can do future final exit work right here.
  559. */
  560. STACKLEAK_ERASE_NOCLOBBER
  561. SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi
  562. /* Restore RDI. */
  563. popq %rdi
  564. swapgs
  565. jmp .Lnative_iret
  566. SYM_INNER_LABEL(restore_regs_and_return_to_kernel, SYM_L_GLOBAL)
  567. #ifdef CONFIG_DEBUG_ENTRY
  568. /* Assert that pt_regs indicates kernel mode. */
  569. testb $3, CS(%rsp)
  570. jz 1f
  571. ud2
  572. 1:
  573. #endif
  574. POP_REGS
  575. addq $8, %rsp /* skip regs->orig_ax */
  576. /*
  577. * ARCH_HAS_MEMBARRIER_SYNC_CORE rely on IRET core serialization
  578. * when returning from IPI handler.
  579. */
  580. #ifdef CONFIG_XEN_PV
  581. SYM_INNER_LABEL(early_xen_iret_patch, SYM_L_GLOBAL)
  582. ANNOTATE_NOENDBR
  583. .byte 0xe9
  584. .long .Lnative_iret - (. + 4)
  585. #endif
  586. .Lnative_iret:
  587. UNWIND_HINT_IRET_REGS
  588. /*
  589. * Are we returning to a stack segment from the LDT? Note: in
  590. * 64-bit mode SS:RSP on the exception stack is always valid.
  591. */
  592. #ifdef CONFIG_X86_ESPFIX64
  593. testb $4, (SS-RIP)(%rsp)
  594. jnz native_irq_return_ldt
  595. #endif
  596. SYM_INNER_LABEL(native_irq_return_iret, SYM_L_GLOBAL)
  597. ANNOTATE_NOENDBR // exc_double_fault
  598. /*
  599. * This may fault. Non-paranoid faults on return to userspace are
  600. * handled by fixup_bad_iret. These include #SS, #GP, and #NP.
  601. * Double-faults due to espfix64 are handled in exc_double_fault.
  602. * Other faults here are fatal.
  603. */
  604. iretq
  605. #ifdef CONFIG_X86_ESPFIX64
  606. native_irq_return_ldt:
  607. /*
  608. * We are running with user GSBASE. All GPRs contain their user
  609. * values. We have a percpu ESPFIX stack that is eight slots
  610. * long (see ESPFIX_STACK_SIZE). espfix_waddr points to the bottom
  611. * of the ESPFIX stack.
  612. *
  613. * We clobber RAX and RDI in this code. We stash RDI on the
  614. * normal stack and RAX on the ESPFIX stack.
  615. *
  616. * The ESPFIX stack layout we set up looks like this:
  617. *
  618. * --- top of ESPFIX stack ---
  619. * SS
  620. * RSP
  621. * RFLAGS
  622. * CS
  623. * RIP <-- RSP points here when we're done
  624. * RAX <-- espfix_waddr points here
  625. * --- bottom of ESPFIX stack ---
  626. */
  627. pushq %rdi /* Stash user RDI */
  628. swapgs /* to kernel GS */
  629. SWITCH_TO_KERNEL_CR3 scratch_reg=%rdi /* to kernel CR3 */
  630. movq PER_CPU_VAR(espfix_waddr), %rdi
  631. movq %rax, (0*8)(%rdi) /* user RAX */
  632. movq (1*8)(%rsp), %rax /* user RIP */
  633. movq %rax, (1*8)(%rdi)
  634. movq (2*8)(%rsp), %rax /* user CS */
  635. movq %rax, (2*8)(%rdi)
  636. movq (3*8)(%rsp), %rax /* user RFLAGS */
  637. movq %rax, (3*8)(%rdi)
  638. movq (5*8)(%rsp), %rax /* user SS */
  639. movq %rax, (5*8)(%rdi)
  640. movq (4*8)(%rsp), %rax /* user RSP */
  641. movq %rax, (4*8)(%rdi)
  642. /* Now RAX == RSP. */
  643. andl $0xffff0000, %eax /* RAX = (RSP & 0xffff0000) */
  644. /*
  645. * espfix_stack[31:16] == 0. The page tables are set up such that
  646. * (espfix_stack | (X & 0xffff0000)) points to a read-only alias of
  647. * espfix_waddr for any X. That is, there are 65536 RO aliases of
  648. * the same page. Set up RSP so that RSP[31:16] contains the
  649. * respective 16 bits of the /userspace/ RSP and RSP nonetheless
  650. * still points to an RO alias of the ESPFIX stack.
  651. */
  652. orq PER_CPU_VAR(espfix_stack), %rax
  653. SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi
  654. swapgs /* to user GS */
  655. popq %rdi /* Restore user RDI */
  656. movq %rax, %rsp
  657. UNWIND_HINT_IRET_REGS offset=8
  658. /*
  659. * At this point, we cannot write to the stack any more, but we can
  660. * still read.
  661. */
  662. popq %rax /* Restore user RAX */
  663. /*
  664. * RSP now points to an ordinary IRET frame, except that the page
  665. * is read-only and RSP[31:16] are preloaded with the userspace
  666. * values. We can now IRET back to userspace.
  667. */
  668. jmp native_irq_return_iret
  669. #endif
  670. SYM_CODE_END(common_interrupt_return)
  671. _ASM_NOKPROBE(common_interrupt_return)
  672. /*
  673. * Reload gs selector with exception handling
  674. * edi: new selector
  675. *
  676. * Is in entry.text as it shouldn't be instrumented.
  677. */
  678. SYM_FUNC_START(asm_load_gs_index)
  679. FRAME_BEGIN
  680. swapgs
  681. .Lgs_change:
  682. ANNOTATE_NOENDBR // error_entry
  683. movl %edi, %gs
  684. 2: ALTERNATIVE "", "mfence", X86_BUG_SWAPGS_FENCE
  685. swapgs
  686. FRAME_END
  687. RET
  688. /* running with kernelgs */
  689. .Lbad_gs:
  690. swapgs /* switch back to user gs */
  691. .macro ZAP_GS
  692. /* This can't be a string because the preprocessor needs to see it. */
  693. movl $__USER_DS, %eax
  694. movl %eax, %gs
  695. .endm
  696. ALTERNATIVE "", "ZAP_GS", X86_BUG_NULL_SEG
  697. xorl %eax, %eax
  698. movl %eax, %gs
  699. jmp 2b
  700. _ASM_EXTABLE(.Lgs_change, .Lbad_gs)
  701. SYM_FUNC_END(asm_load_gs_index)
  702. EXPORT_SYMBOL(asm_load_gs_index)
  703. #ifdef CONFIG_XEN_PV
  704. /*
  705. * A note on the "critical region" in our callback handler.
  706. * We want to avoid stacking callback handlers due to events occurring
  707. * during handling of the last event. To do this, we keep events disabled
  708. * until we've done all processing. HOWEVER, we must enable events before
  709. * popping the stack frame (can't be done atomically) and so it would still
  710. * be possible to get enough handler activations to overflow the stack.
  711. * Although unlikely, bugs of that kind are hard to track down, so we'd
  712. * like to avoid the possibility.
  713. * So, on entry to the handler we detect whether we interrupted an
  714. * existing activation in its critical region -- if so, we pop the current
  715. * activation and restart the handler using the previous one.
  716. *
  717. * C calling convention: exc_xen_hypervisor_callback(struct *pt_regs)
  718. */
  719. SYM_CODE_START_LOCAL(exc_xen_hypervisor_callback)
  720. /*
  721. * Since we don't modify %rdi, evtchn_do_upall(struct *pt_regs) will
  722. * see the correct pointer to the pt_regs
  723. */
  724. UNWIND_HINT_FUNC
  725. movq %rdi, %rsp /* we don't return, adjust the stack frame */
  726. UNWIND_HINT_REGS
  727. call xen_pv_evtchn_do_upcall
  728. jmp error_return
  729. SYM_CODE_END(exc_xen_hypervisor_callback)
  730. /*
  731. * Hypervisor uses this for application faults while it executes.
  732. * We get here for two reasons:
  733. * 1. Fault while reloading DS, ES, FS or GS
  734. * 2. Fault while executing IRET
  735. * Category 1 we do not need to fix up as Xen has already reloaded all segment
  736. * registers that could be reloaded and zeroed the others.
  737. * Category 2 we fix up by killing the current process. We cannot use the
  738. * normal Linux return path in this case because if we use the IRET hypercall
  739. * to pop the stack frame we end up in an infinite loop of failsafe callbacks.
  740. * We distinguish between categories by comparing each saved segment register
  741. * with its current contents: any discrepancy means we in category 1.
  742. */
  743. SYM_CODE_START(xen_failsafe_callback)
  744. UNWIND_HINT_EMPTY
  745. ENDBR
  746. movl %ds, %ecx
  747. cmpw %cx, 0x10(%rsp)
  748. jne 1f
  749. movl %es, %ecx
  750. cmpw %cx, 0x18(%rsp)
  751. jne 1f
  752. movl %fs, %ecx
  753. cmpw %cx, 0x20(%rsp)
  754. jne 1f
  755. movl %gs, %ecx
  756. cmpw %cx, 0x28(%rsp)
  757. jne 1f
  758. /* All segments match their saved values => Category 2 (Bad IRET). */
  759. movq (%rsp), %rcx
  760. movq 8(%rsp), %r11
  761. addq $0x30, %rsp
  762. pushq $0 /* RIP */
  763. UNWIND_HINT_IRET_REGS offset=8
  764. jmp asm_exc_general_protection
  765. 1: /* Segment mismatch => Category 1 (Bad segment). Retry the IRET. */
  766. movq (%rsp), %rcx
  767. movq 8(%rsp), %r11
  768. addq $0x30, %rsp
  769. UNWIND_HINT_IRET_REGS
  770. pushq $-1 /* orig_ax = -1 => not a system call */
  771. PUSH_AND_CLEAR_REGS
  772. ENCODE_FRAME_POINTER
  773. jmp error_return
  774. SYM_CODE_END(xen_failsafe_callback)
  775. #endif /* CONFIG_XEN_PV */
  776. /*
  777. * Save all registers in pt_regs. Return GSBASE related information
  778. * in EBX depending on the availability of the FSGSBASE instructions:
  779. *
  780. * FSGSBASE R/EBX
  781. * N 0 -> SWAPGS on exit
  782. * 1 -> no SWAPGS on exit
  783. *
  784. * Y GSBASE value at entry, must be restored in paranoid_exit
  785. *
  786. * R14 - old CR3
  787. * R15 - old SPEC_CTRL
  788. */
  789. SYM_CODE_START_LOCAL(paranoid_entry)
  790. UNWIND_HINT_FUNC
  791. PUSH_AND_CLEAR_REGS save_ret=1
  792. ENCODE_FRAME_POINTER 8
  793. /*
  794. * Always stash CR3 in %r14. This value will be restored,
  795. * verbatim, at exit. Needed if paranoid_entry interrupted
  796. * another entry that already switched to the user CR3 value
  797. * but has not yet returned to userspace.
  798. *
  799. * This is also why CS (stashed in the "iret frame" by the
  800. * hardware at entry) can not be used: this may be a return
  801. * to kernel code, but with a user CR3 value.
  802. *
  803. * Switching CR3 does not depend on kernel GSBASE so it can
  804. * be done before switching to the kernel GSBASE. This is
  805. * required for FSGSBASE because the kernel GSBASE has to
  806. * be retrieved from a kernel internal table.
  807. */
  808. SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg=%rax save_reg=%r14
  809. /*
  810. * Handling GSBASE depends on the availability of FSGSBASE.
  811. *
  812. * Without FSGSBASE the kernel enforces that negative GSBASE
  813. * values indicate kernel GSBASE. With FSGSBASE no assumptions
  814. * can be made about the GSBASE value when entering from user
  815. * space.
  816. */
  817. ALTERNATIVE "jmp .Lparanoid_entry_checkgs", "", X86_FEATURE_FSGSBASE
  818. /*
  819. * Read the current GSBASE and store it in %rbx unconditionally,
  820. * retrieve and set the current CPUs kernel GSBASE. The stored value
  821. * has to be restored in paranoid_exit unconditionally.
  822. *
  823. * The unconditional write to GS base below ensures that no subsequent
  824. * loads based on a mispredicted GS base can happen, therefore no LFENCE
  825. * is needed here.
  826. */
  827. SAVE_AND_SET_GSBASE scratch_reg=%rax save_reg=%rbx
  828. jmp .Lparanoid_gsbase_done
  829. .Lparanoid_entry_checkgs:
  830. /* EBX = 1 -> kernel GSBASE active, no restore required */
  831. movl $1, %ebx
  832. /*
  833. * The kernel-enforced convention is a negative GSBASE indicates
  834. * a kernel value. No SWAPGS needed on entry and exit.
  835. */
  836. movl $MSR_GS_BASE, %ecx
  837. rdmsr
  838. testl %edx, %edx
  839. js .Lparanoid_kernel_gsbase
  840. /* EBX = 0 -> SWAPGS required on exit */
  841. xorl %ebx, %ebx
  842. swapgs
  843. .Lparanoid_kernel_gsbase:
  844. FENCE_SWAPGS_KERNEL_ENTRY
  845. .Lparanoid_gsbase_done:
  846. /*
  847. * Once we have CR3 and %GS setup save and set SPEC_CTRL. Just like
  848. * CR3 above, keep the old value in a callee saved register.
  849. */
  850. IBRS_ENTER save_reg=%r15
  851. UNTRAIN_RET
  852. RET
  853. SYM_CODE_END(paranoid_entry)
  854. /*
  855. * "Paranoid" exit path from exception stack. This is invoked
  856. * only on return from non-NMI IST interrupts that came
  857. * from kernel space.
  858. *
  859. * We may be returning to very strange contexts (e.g. very early
  860. * in syscall entry), so checking for preemption here would
  861. * be complicated. Fortunately, there's no good reason to try
  862. * to handle preemption here.
  863. *
  864. * R/EBX contains the GSBASE related information depending on the
  865. * availability of the FSGSBASE instructions:
  866. *
  867. * FSGSBASE R/EBX
  868. * N 0 -> SWAPGS on exit
  869. * 1 -> no SWAPGS on exit
  870. *
  871. * Y User space GSBASE, must be restored unconditionally
  872. *
  873. * R14 - old CR3
  874. * R15 - old SPEC_CTRL
  875. */
  876. SYM_CODE_START_LOCAL(paranoid_exit)
  877. UNWIND_HINT_REGS
  878. /*
  879. * Must restore IBRS state before both CR3 and %GS since we need access
  880. * to the per-CPU x86_spec_ctrl_shadow variable.
  881. */
  882. IBRS_EXIT save_reg=%r15
  883. /*
  884. * The order of operations is important. RESTORE_CR3 requires
  885. * kernel GSBASE.
  886. *
  887. * NB to anyone to try to optimize this code: this code does
  888. * not execute at all for exceptions from user mode. Those
  889. * exceptions go through error_exit instead.
  890. */
  891. RESTORE_CR3 scratch_reg=%rax save_reg=%r14
  892. /* Handle the three GSBASE cases */
  893. ALTERNATIVE "jmp .Lparanoid_exit_checkgs", "", X86_FEATURE_FSGSBASE
  894. /* With FSGSBASE enabled, unconditionally restore GSBASE */
  895. wrgsbase %rbx
  896. jmp restore_regs_and_return_to_kernel
  897. .Lparanoid_exit_checkgs:
  898. /* On non-FSGSBASE systems, conditionally do SWAPGS */
  899. testl %ebx, %ebx
  900. jnz restore_regs_and_return_to_kernel
  901. /* We are returning to a context with user GSBASE */
  902. swapgs
  903. jmp restore_regs_and_return_to_kernel
  904. SYM_CODE_END(paranoid_exit)
  905. /*
  906. * Switch GS and CR3 if needed.
  907. */
  908. SYM_CODE_START_LOCAL(error_entry)
  909. UNWIND_HINT_FUNC
  910. PUSH_AND_CLEAR_REGS save_ret=1
  911. ENCODE_FRAME_POINTER 8
  912. testb $3, CS+8(%rsp)
  913. jz .Lerror_kernelspace
  914. /*
  915. * We entered from user mode or we're pretending to have entered
  916. * from user mode due to an IRET fault.
  917. */
  918. swapgs
  919. FENCE_SWAPGS_USER_ENTRY
  920. /* We have user CR3. Change to kernel CR3. */
  921. SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
  922. IBRS_ENTER
  923. UNTRAIN_RET
  924. leaq 8(%rsp), %rdi /* arg0 = pt_regs pointer */
  925. .Lerror_entry_from_usermode_after_swapgs:
  926. /* Put us onto the real thread stack. */
  927. call sync_regs
  928. RET
  929. /*
  930. * There are two places in the kernel that can potentially fault with
  931. * usergs. Handle them here. B stepping K8s sometimes report a
  932. * truncated RIP for IRET exceptions returning to compat mode. Check
  933. * for these here too.
  934. */
  935. .Lerror_kernelspace:
  936. leaq native_irq_return_iret(%rip), %rcx
  937. cmpq %rcx, RIP+8(%rsp)
  938. je .Lerror_bad_iret
  939. movl %ecx, %eax /* zero extend */
  940. cmpq %rax, RIP+8(%rsp)
  941. je .Lbstep_iret
  942. cmpq $.Lgs_change, RIP+8(%rsp)
  943. jne .Lerror_entry_done_lfence
  944. /*
  945. * hack: .Lgs_change can fail with user gsbase. If this happens, fix up
  946. * gsbase and proceed. We'll fix up the exception and land in
  947. * .Lgs_change's error handler with kernel gsbase.
  948. */
  949. swapgs
  950. /*
  951. * Issue an LFENCE to prevent GS speculation, regardless of whether it is a
  952. * kernel or user gsbase.
  953. */
  954. .Lerror_entry_done_lfence:
  955. FENCE_SWAPGS_KERNEL_ENTRY
  956. leaq 8(%rsp), %rax /* return pt_regs pointer */
  957. ANNOTATE_UNRET_END
  958. RET
  959. .Lbstep_iret:
  960. /* Fix truncated RIP */
  961. movq %rcx, RIP+8(%rsp)
  962. /* fall through */
  963. .Lerror_bad_iret:
  964. /*
  965. * We came from an IRET to user mode, so we have user
  966. * gsbase and CR3. Switch to kernel gsbase and CR3:
  967. */
  968. swapgs
  969. FENCE_SWAPGS_USER_ENTRY
  970. SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
  971. IBRS_ENTER
  972. UNTRAIN_RET
  973. /*
  974. * Pretend that the exception came from user mode: set up pt_regs
  975. * as if we faulted immediately after IRET.
  976. */
  977. leaq 8(%rsp), %rdi /* arg0 = pt_regs pointer */
  978. call fixup_bad_iret
  979. mov %rax, %rdi
  980. jmp .Lerror_entry_from_usermode_after_swapgs
  981. SYM_CODE_END(error_entry)
  982. SYM_CODE_START_LOCAL(error_return)
  983. UNWIND_HINT_REGS
  984. DEBUG_ENTRY_ASSERT_IRQS_OFF
  985. testb $3, CS(%rsp)
  986. jz restore_regs_and_return_to_kernel
  987. jmp swapgs_restore_regs_and_return_to_usermode
  988. SYM_CODE_END(error_return)
  989. /*
  990. * Runs on exception stack. Xen PV does not go through this path at all,
  991. * so we can use real assembly here.
  992. *
  993. * Registers:
  994. * %r14: Used to save/restore the CR3 of the interrupted context
  995. * when PAGE_TABLE_ISOLATION is in use. Do not clobber.
  996. */
  997. SYM_CODE_START(asm_exc_nmi)
  998. UNWIND_HINT_IRET_REGS
  999. ENDBR
  1000. /*
  1001. * We allow breakpoints in NMIs. If a breakpoint occurs, then
  1002. * the iretq it performs will take us out of NMI context.
  1003. * This means that we can have nested NMIs where the next
  1004. * NMI is using the top of the stack of the previous NMI. We
  1005. * can't let it execute because the nested NMI will corrupt the
  1006. * stack of the previous NMI. NMI handlers are not re-entrant
  1007. * anyway.
  1008. *
  1009. * To handle this case we do the following:
  1010. * Check the a special location on the stack that contains
  1011. * a variable that is set when NMIs are executing.
  1012. * The interrupted task's stack is also checked to see if it
  1013. * is an NMI stack.
  1014. * If the variable is not set and the stack is not the NMI
  1015. * stack then:
  1016. * o Set the special variable on the stack
  1017. * o Copy the interrupt frame into an "outermost" location on the
  1018. * stack
  1019. * o Copy the interrupt frame into an "iret" location on the stack
  1020. * o Continue processing the NMI
  1021. * If the variable is set or the previous stack is the NMI stack:
  1022. * o Modify the "iret" location to jump to the repeat_nmi
  1023. * o return back to the first NMI
  1024. *
  1025. * Now on exit of the first NMI, we first clear the stack variable
  1026. * The NMI stack will tell any nested NMIs at that point that it is
  1027. * nested. Then we pop the stack normally with iret, and if there was
  1028. * a nested NMI that updated the copy interrupt stack frame, a
  1029. * jump will be made to the repeat_nmi code that will handle the second
  1030. * NMI.
  1031. *
  1032. * However, espfix prevents us from directly returning to userspace
  1033. * with a single IRET instruction. Similarly, IRET to user mode
  1034. * can fault. We therefore handle NMIs from user space like
  1035. * other IST entries.
  1036. */
  1037. ASM_CLAC
  1038. cld
  1039. /* Use %rdx as our temp variable throughout */
  1040. pushq %rdx
  1041. testb $3, CS-RIP+8(%rsp)
  1042. jz .Lnmi_from_kernel
  1043. /*
  1044. * NMI from user mode. We need to run on the thread stack, but we
  1045. * can't go through the normal entry paths: NMIs are masked, and
  1046. * we don't want to enable interrupts, because then we'll end
  1047. * up in an awkward situation in which IRQs are on but NMIs
  1048. * are off.
  1049. *
  1050. * We also must not push anything to the stack before switching
  1051. * stacks lest we corrupt the "NMI executing" variable.
  1052. */
  1053. swapgs
  1054. FENCE_SWAPGS_USER_ENTRY
  1055. SWITCH_TO_KERNEL_CR3 scratch_reg=%rdx
  1056. movq %rsp, %rdx
  1057. movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp
  1058. UNWIND_HINT_IRET_REGS base=%rdx offset=8
  1059. pushq 5*8(%rdx) /* pt_regs->ss */
  1060. pushq 4*8(%rdx) /* pt_regs->rsp */
  1061. pushq 3*8(%rdx) /* pt_regs->flags */
  1062. pushq 2*8(%rdx) /* pt_regs->cs */
  1063. pushq 1*8(%rdx) /* pt_regs->rip */
  1064. UNWIND_HINT_IRET_REGS
  1065. pushq $-1 /* pt_regs->orig_ax */
  1066. PUSH_AND_CLEAR_REGS rdx=(%rdx)
  1067. ENCODE_FRAME_POINTER
  1068. IBRS_ENTER
  1069. UNTRAIN_RET
  1070. /*
  1071. * At this point we no longer need to worry about stack damage
  1072. * due to nesting -- we're on the normal thread stack and we're
  1073. * done with the NMI stack.
  1074. */
  1075. movq %rsp, %rdi
  1076. movq $-1, %rsi
  1077. call exc_nmi
  1078. /*
  1079. * Return back to user mode. We must *not* do the normal exit
  1080. * work, because we don't want to enable interrupts.
  1081. */
  1082. jmp swapgs_restore_regs_and_return_to_usermode
  1083. .Lnmi_from_kernel:
  1084. /*
  1085. * Here's what our stack frame will look like:
  1086. * +---------------------------------------------------------+
  1087. * | original SS |
  1088. * | original Return RSP |
  1089. * | original RFLAGS |
  1090. * | original CS |
  1091. * | original RIP |
  1092. * +---------------------------------------------------------+
  1093. * | temp storage for rdx |
  1094. * +---------------------------------------------------------+
  1095. * | "NMI executing" variable |
  1096. * +---------------------------------------------------------+
  1097. * | iret SS } Copied from "outermost" frame |
  1098. * | iret Return RSP } on each loop iteration; overwritten |
  1099. * | iret RFLAGS } by a nested NMI to force another |
  1100. * | iret CS } iteration if needed. |
  1101. * | iret RIP } |
  1102. * +---------------------------------------------------------+
  1103. * | outermost SS } initialized in first_nmi; |
  1104. * | outermost Return RSP } will not be changed before |
  1105. * | outermost RFLAGS } NMI processing is done. |
  1106. * | outermost CS } Copied to "iret" frame on each |
  1107. * | outermost RIP } iteration. |
  1108. * +---------------------------------------------------------+
  1109. * | pt_regs |
  1110. * +---------------------------------------------------------+
  1111. *
  1112. * The "original" frame is used by hardware. Before re-enabling
  1113. * NMIs, we need to be done with it, and we need to leave enough
  1114. * space for the asm code here.
  1115. *
  1116. * We return by executing IRET while RSP points to the "iret" frame.
  1117. * That will either return for real or it will loop back into NMI
  1118. * processing.
  1119. *
  1120. * The "outermost" frame is copied to the "iret" frame on each
  1121. * iteration of the loop, so each iteration starts with the "iret"
  1122. * frame pointing to the final return target.
  1123. */
  1124. /*
  1125. * Determine whether we're a nested NMI.
  1126. *
  1127. * If we interrupted kernel code between repeat_nmi and
  1128. * end_repeat_nmi, then we are a nested NMI. We must not
  1129. * modify the "iret" frame because it's being written by
  1130. * the outer NMI. That's okay; the outer NMI handler is
  1131. * about to about to call exc_nmi() anyway, so we can just
  1132. * resume the outer NMI.
  1133. */
  1134. movq $repeat_nmi, %rdx
  1135. cmpq 8(%rsp), %rdx
  1136. ja 1f
  1137. movq $end_repeat_nmi, %rdx
  1138. cmpq 8(%rsp), %rdx
  1139. ja nested_nmi_out
  1140. 1:
  1141. /*
  1142. * Now check "NMI executing". If it's set, then we're nested.
  1143. * This will not detect if we interrupted an outer NMI just
  1144. * before IRET.
  1145. */
  1146. cmpl $1, -8(%rsp)
  1147. je nested_nmi
  1148. /*
  1149. * Now test if the previous stack was an NMI stack. This covers
  1150. * the case where we interrupt an outer NMI after it clears
  1151. * "NMI executing" but before IRET. We need to be careful, though:
  1152. * there is one case in which RSP could point to the NMI stack
  1153. * despite there being no NMI active: naughty userspace controls
  1154. * RSP at the very beginning of the SYSCALL targets. We can
  1155. * pull a fast one on naughty userspace, though: we program
  1156. * SYSCALL to mask DF, so userspace cannot cause DF to be set
  1157. * if it controls the kernel's RSP. We set DF before we clear
  1158. * "NMI executing".
  1159. */
  1160. lea 6*8(%rsp), %rdx
  1161. /* Compare the NMI stack (rdx) with the stack we came from (4*8(%rsp)) */
  1162. cmpq %rdx, 4*8(%rsp)
  1163. /* If the stack pointer is above the NMI stack, this is a normal NMI */
  1164. ja first_nmi
  1165. subq $EXCEPTION_STKSZ, %rdx
  1166. cmpq %rdx, 4*8(%rsp)
  1167. /* If it is below the NMI stack, it is a normal NMI */
  1168. jb first_nmi
  1169. /* Ah, it is within the NMI stack. */
  1170. testb $(X86_EFLAGS_DF >> 8), (3*8 + 1)(%rsp)
  1171. jz first_nmi /* RSP was user controlled. */
  1172. /* This is a nested NMI. */
  1173. nested_nmi:
  1174. /*
  1175. * Modify the "iret" frame to point to repeat_nmi, forcing another
  1176. * iteration of NMI handling.
  1177. */
  1178. subq $8, %rsp
  1179. leaq -10*8(%rsp), %rdx
  1180. pushq $__KERNEL_DS
  1181. pushq %rdx
  1182. pushfq
  1183. pushq $__KERNEL_CS
  1184. pushq $repeat_nmi
  1185. /* Put stack back */
  1186. addq $(6*8), %rsp
  1187. nested_nmi_out:
  1188. popq %rdx
  1189. /* We are returning to kernel mode, so this cannot result in a fault. */
  1190. iretq
  1191. first_nmi:
  1192. /* Restore rdx. */
  1193. movq (%rsp), %rdx
  1194. /* Make room for "NMI executing". */
  1195. pushq $0
  1196. /* Leave room for the "iret" frame */
  1197. subq $(5*8), %rsp
  1198. /* Copy the "original" frame to the "outermost" frame */
  1199. .rept 5
  1200. pushq 11*8(%rsp)
  1201. .endr
  1202. UNWIND_HINT_IRET_REGS
  1203. /* Everything up to here is safe from nested NMIs */
  1204. #ifdef CONFIG_DEBUG_ENTRY
  1205. /*
  1206. * For ease of testing, unmask NMIs right away. Disabled by
  1207. * default because IRET is very expensive.
  1208. */
  1209. pushq $0 /* SS */
  1210. pushq %rsp /* RSP (minus 8 because of the previous push) */
  1211. addq $8, (%rsp) /* Fix up RSP */
  1212. pushfq /* RFLAGS */
  1213. pushq $__KERNEL_CS /* CS */
  1214. pushq $1f /* RIP */
  1215. iretq /* continues at repeat_nmi below */
  1216. UNWIND_HINT_IRET_REGS
  1217. 1:
  1218. #endif
  1219. repeat_nmi:
  1220. ANNOTATE_NOENDBR // this code
  1221. /*
  1222. * If there was a nested NMI, the first NMI's iret will return
  1223. * here. But NMIs are still enabled and we can take another
  1224. * nested NMI. The nested NMI checks the interrupted RIP to see
  1225. * if it is between repeat_nmi and end_repeat_nmi, and if so
  1226. * it will just return, as we are about to repeat an NMI anyway.
  1227. * This makes it safe to copy to the stack frame that a nested
  1228. * NMI will update.
  1229. *
  1230. * RSP is pointing to "outermost RIP". gsbase is unknown, but, if
  1231. * we're repeating an NMI, gsbase has the same value that it had on
  1232. * the first iteration. paranoid_entry will load the kernel
  1233. * gsbase if needed before we call exc_nmi(). "NMI executing"
  1234. * is zero.
  1235. */
  1236. movq $1, 10*8(%rsp) /* Set "NMI executing". */
  1237. /*
  1238. * Copy the "outermost" frame to the "iret" frame. NMIs that nest
  1239. * here must not modify the "iret" frame while we're writing to
  1240. * it or it will end up containing garbage.
  1241. */
  1242. addq $(10*8), %rsp
  1243. .rept 5
  1244. pushq -6*8(%rsp)
  1245. .endr
  1246. subq $(5*8), %rsp
  1247. end_repeat_nmi:
  1248. ANNOTATE_NOENDBR // this code
  1249. /*
  1250. * Everything below this point can be preempted by a nested NMI.
  1251. * If this happens, then the inner NMI will change the "iret"
  1252. * frame to point back to repeat_nmi.
  1253. */
  1254. pushq $-1 /* ORIG_RAX: no syscall to restart */
  1255. /*
  1256. * Use paranoid_entry to handle SWAPGS, but no need to use paranoid_exit
  1257. * as we should not be calling schedule in NMI context.
  1258. * Even with normal interrupts enabled. An NMI should not be
  1259. * setting NEED_RESCHED or anything that normal interrupts and
  1260. * exceptions might do.
  1261. */
  1262. call paranoid_entry
  1263. UNWIND_HINT_REGS
  1264. movq %rsp, %rdi
  1265. movq $-1, %rsi
  1266. call exc_nmi
  1267. /* Always restore stashed SPEC_CTRL value (see paranoid_entry) */
  1268. IBRS_EXIT save_reg=%r15
  1269. /* Always restore stashed CR3 value (see paranoid_entry) */
  1270. RESTORE_CR3 scratch_reg=%r15 save_reg=%r14
  1271. /*
  1272. * The above invocation of paranoid_entry stored the GSBASE
  1273. * related information in R/EBX depending on the availability
  1274. * of FSGSBASE.
  1275. *
  1276. * If FSGSBASE is enabled, restore the saved GSBASE value
  1277. * unconditionally, otherwise take the conditional SWAPGS path.
  1278. */
  1279. ALTERNATIVE "jmp nmi_no_fsgsbase", "", X86_FEATURE_FSGSBASE
  1280. wrgsbase %rbx
  1281. jmp nmi_restore
  1282. nmi_no_fsgsbase:
  1283. /* EBX == 0 -> invoke SWAPGS */
  1284. testl %ebx, %ebx
  1285. jnz nmi_restore
  1286. nmi_swapgs:
  1287. swapgs
  1288. nmi_restore:
  1289. POP_REGS
  1290. /*
  1291. * Skip orig_ax and the "outermost" frame to point RSP at the "iret"
  1292. * at the "iret" frame.
  1293. */
  1294. addq $6*8, %rsp
  1295. /*
  1296. * Clear "NMI executing". Set DF first so that we can easily
  1297. * distinguish the remaining code between here and IRET from
  1298. * the SYSCALL entry and exit paths.
  1299. *
  1300. * We arguably should just inspect RIP instead, but I (Andy) wrote
  1301. * this code when I had the misapprehension that Xen PV supported
  1302. * NMIs, and Xen PV would break that approach.
  1303. */
  1304. std
  1305. movq $0, 5*8(%rsp) /* clear "NMI executing" */
  1306. /*
  1307. * iretq reads the "iret" frame and exits the NMI stack in a
  1308. * single instruction. We are returning to kernel mode, so this
  1309. * cannot result in a fault. Similarly, we don't need to worry
  1310. * about espfix64 on the way back to kernel mode.
  1311. */
  1312. iretq
  1313. SYM_CODE_END(asm_exc_nmi)
  1314. #ifndef CONFIG_IA32_EMULATION
  1315. /*
  1316. * This handles SYSCALL from 32-bit code. There is no way to program
  1317. * MSRs to fully disable 32-bit SYSCALL.
  1318. */
  1319. SYM_CODE_START(ignore_sysret)
  1320. UNWIND_HINT_EMPTY
  1321. ENDBR
  1322. mov $-ENOSYS, %eax
  1323. sysretl
  1324. SYM_CODE_END(ignore_sysret)
  1325. #endif
  1326. .pushsection .text, "ax"
  1327. SYM_CODE_START(rewind_stack_and_make_dead)
  1328. UNWIND_HINT_FUNC
  1329. /* Prevent any naive code from trying to unwind to our caller. */
  1330. xorl %ebp, %ebp
  1331. movq PER_CPU_VAR(cpu_current_top_of_stack), %rax
  1332. leaq -PTREGS_SIZE(%rax), %rsp
  1333. UNWIND_HINT_REGS
  1334. call make_task_dead
  1335. SYM_CODE_END(rewind_stack_and_make_dead)
  1336. .popsection