entry_32.S 33 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. /*
  3. * Copyright (C) 1991,1992 Linus Torvalds
  4. *
  5. * entry_32.S contains the system-call and low-level fault and trap handling routines.
  6. *
  7. * Stack layout while running C code:
  8. * ptrace needs to have all registers on the stack.
  9. * If the order here is changed, it needs to be
  10. * updated in fork.c:copy_process(), signal.c:do_signal(),
  11. * ptrace.c and ptrace.h
  12. *
  13. * 0(%esp) - %ebx
  14. * 4(%esp) - %ecx
  15. * 8(%esp) - %edx
  16. * C(%esp) - %esi
  17. * 10(%esp) - %edi
  18. * 14(%esp) - %ebp
  19. * 18(%esp) - %eax
  20. * 1C(%esp) - %ds
  21. * 20(%esp) - %es
  22. * 24(%esp) - %fs
  23. * 28(%esp) - unused -- was %gs on old stackprotector kernels
  24. * 2C(%esp) - orig_eax
  25. * 30(%esp) - %eip
  26. * 34(%esp) - %cs
  27. * 38(%esp) - %eflags
  28. * 3C(%esp) - %oldesp
  29. * 40(%esp) - %oldss
  30. */
  31. #include <linux/linkage.h>
  32. #include <linux/err.h>
  33. #include <asm/thread_info.h>
  34. #include <asm/irqflags.h>
  35. #include <asm/errno.h>
  36. #include <asm/segment.h>
  37. #include <asm/smp.h>
  38. #include <asm/percpu.h>
  39. #include <asm/processor-flags.h>
  40. #include <asm/irq_vectors.h>
  41. #include <asm/cpufeatures.h>
  42. #include <asm/alternative.h>
  43. #include <asm/asm.h>
  44. #include <asm/smap.h>
  45. #include <asm/frame.h>
  46. #include <asm/trapnr.h>
  47. #include <asm/nospec-branch.h>
  48. #include "calling.h"
  49. .section .entry.text, "ax"
  50. #define PTI_SWITCH_MASK (1 << PAGE_SHIFT)
  51. /* Unconditionally switch to user cr3 */
  52. .macro SWITCH_TO_USER_CR3 scratch_reg:req
  53. ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI
  54. movl %cr3, \scratch_reg
  55. orl $PTI_SWITCH_MASK, \scratch_reg
  56. movl \scratch_reg, %cr3
  57. .Lend_\@:
  58. .endm
  59. .macro BUG_IF_WRONG_CR3 no_user_check=0
  60. #ifdef CONFIG_DEBUG_ENTRY
  61. ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI
  62. .if \no_user_check == 0
  63. /* coming from usermode? */
  64. testl $USER_SEGMENT_RPL_MASK, PT_CS(%esp)
  65. jz .Lend_\@
  66. .endif
  67. /* On user-cr3? */
  68. movl %cr3, %eax
  69. testl $PTI_SWITCH_MASK, %eax
  70. jnz .Lend_\@
  71. /* From userspace with kernel cr3 - BUG */
  72. ud2
  73. .Lend_\@:
  74. #endif
  75. .endm
  76. /*
  77. * Switch to kernel cr3 if not already loaded and return current cr3 in
  78. * \scratch_reg
  79. */
  80. .macro SWITCH_TO_KERNEL_CR3 scratch_reg:req
  81. ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI
  82. movl %cr3, \scratch_reg
  83. /* Test if we are already on kernel CR3 */
  84. testl $PTI_SWITCH_MASK, \scratch_reg
  85. jz .Lend_\@
  86. andl $(~PTI_SWITCH_MASK), \scratch_reg
  87. movl \scratch_reg, %cr3
  88. /* Return original CR3 in \scratch_reg */
  89. orl $PTI_SWITCH_MASK, \scratch_reg
  90. .Lend_\@:
  91. .endm
  92. #define CS_FROM_ENTRY_STACK (1 << 31)
  93. #define CS_FROM_USER_CR3 (1 << 30)
  94. #define CS_FROM_KERNEL (1 << 29)
  95. #define CS_FROM_ESPFIX (1 << 28)
  96. .macro FIXUP_FRAME
  97. /*
  98. * The high bits of the CS dword (__csh) are used for CS_FROM_*.
  99. * Clear them in case hardware didn't do this for us.
  100. */
  101. andl $0x0000ffff, 4*4(%esp)
  102. #ifdef CONFIG_VM86
  103. testl $X86_EFLAGS_VM, 5*4(%esp)
  104. jnz .Lfrom_usermode_no_fixup_\@
  105. #endif
  106. testl $USER_SEGMENT_RPL_MASK, 4*4(%esp)
  107. jnz .Lfrom_usermode_no_fixup_\@
  108. orl $CS_FROM_KERNEL, 4*4(%esp)
  109. /*
  110. * When we're here from kernel mode; the (exception) stack looks like:
  111. *
  112. * 6*4(%esp) - <previous context>
  113. * 5*4(%esp) - flags
  114. * 4*4(%esp) - cs
  115. * 3*4(%esp) - ip
  116. * 2*4(%esp) - orig_eax
  117. * 1*4(%esp) - gs / function
  118. * 0*4(%esp) - fs
  119. *
  120. * Lets build a 5 entry IRET frame after that, such that struct pt_regs
  121. * is complete and in particular regs->sp is correct. This gives us
  122. * the original 6 entries as gap:
  123. *
  124. * 14*4(%esp) - <previous context>
  125. * 13*4(%esp) - gap / flags
  126. * 12*4(%esp) - gap / cs
  127. * 11*4(%esp) - gap / ip
  128. * 10*4(%esp) - gap / orig_eax
  129. * 9*4(%esp) - gap / gs / function
  130. * 8*4(%esp) - gap / fs
  131. * 7*4(%esp) - ss
  132. * 6*4(%esp) - sp
  133. * 5*4(%esp) - flags
  134. * 4*4(%esp) - cs
  135. * 3*4(%esp) - ip
  136. * 2*4(%esp) - orig_eax
  137. * 1*4(%esp) - gs / function
  138. * 0*4(%esp) - fs
  139. */
  140. pushl %ss # ss
  141. pushl %esp # sp (points at ss)
  142. addl $7*4, (%esp) # point sp back at the previous context
  143. pushl 7*4(%esp) # flags
  144. pushl 7*4(%esp) # cs
  145. pushl 7*4(%esp) # ip
  146. pushl 7*4(%esp) # orig_eax
  147. pushl 7*4(%esp) # gs / function
  148. pushl 7*4(%esp) # fs
  149. .Lfrom_usermode_no_fixup_\@:
  150. .endm
  151. .macro IRET_FRAME
  152. /*
  153. * We're called with %ds, %es, %fs, and %gs from the interrupted
  154. * frame, so we shouldn't use them. Also, we may be in ESPFIX
  155. * mode and therefore have a nonzero SS base and an offset ESP,
  156. * so any attempt to access the stack needs to use SS. (except for
  157. * accesses through %esp, which automatically use SS.)
  158. */
  159. testl $CS_FROM_KERNEL, 1*4(%esp)
  160. jz .Lfinished_frame_\@
  161. /*
  162. * Reconstruct the 3 entry IRET frame right after the (modified)
  163. * regs->sp without lowering %esp in between, such that an NMI in the
  164. * middle doesn't scribble our stack.
  165. */
  166. pushl %eax
  167. pushl %ecx
  168. movl 5*4(%esp), %eax # (modified) regs->sp
  169. movl 4*4(%esp), %ecx # flags
  170. movl %ecx, %ss:-1*4(%eax)
  171. movl 3*4(%esp), %ecx # cs
  172. andl $0x0000ffff, %ecx
  173. movl %ecx, %ss:-2*4(%eax)
  174. movl 2*4(%esp), %ecx # ip
  175. movl %ecx, %ss:-3*4(%eax)
  176. movl 1*4(%esp), %ecx # eax
  177. movl %ecx, %ss:-4*4(%eax)
  178. popl %ecx
  179. lea -4*4(%eax), %esp
  180. popl %eax
  181. .Lfinished_frame_\@:
  182. .endm
  183. .macro SAVE_ALL pt_regs_ax=%eax switch_stacks=0 skip_gs=0 unwind_espfix=0
  184. cld
  185. .if \skip_gs == 0
  186. pushl $0
  187. .endif
  188. pushl %fs
  189. pushl %eax
  190. movl $(__KERNEL_PERCPU), %eax
  191. movl %eax, %fs
  192. .if \unwind_espfix > 0
  193. UNWIND_ESPFIX_STACK
  194. .endif
  195. popl %eax
  196. FIXUP_FRAME
  197. pushl %es
  198. pushl %ds
  199. pushl \pt_regs_ax
  200. pushl %ebp
  201. pushl %edi
  202. pushl %esi
  203. pushl %edx
  204. pushl %ecx
  205. pushl %ebx
  206. movl $(__USER_DS), %edx
  207. movl %edx, %ds
  208. movl %edx, %es
  209. /* Switch to kernel stack if necessary */
  210. .if \switch_stacks > 0
  211. SWITCH_TO_KERNEL_STACK
  212. .endif
  213. .endm
  214. .macro SAVE_ALL_NMI cr3_reg:req unwind_espfix=0
  215. SAVE_ALL unwind_espfix=\unwind_espfix
  216. BUG_IF_WRONG_CR3
  217. /*
  218. * Now switch the CR3 when PTI is enabled.
  219. *
  220. * We can enter with either user or kernel cr3, the code will
  221. * store the old cr3 in \cr3_reg and switches to the kernel cr3
  222. * if necessary.
  223. */
  224. SWITCH_TO_KERNEL_CR3 scratch_reg=\cr3_reg
  225. .Lend_\@:
  226. .endm
  227. .macro RESTORE_INT_REGS
  228. popl %ebx
  229. popl %ecx
  230. popl %edx
  231. popl %esi
  232. popl %edi
  233. popl %ebp
  234. popl %eax
  235. .endm
  236. .macro RESTORE_REGS pop=0
  237. RESTORE_INT_REGS
  238. 1: popl %ds
  239. 2: popl %es
  240. 3: popl %fs
  241. 4: addl $(4 + \pop), %esp /* pop the unused "gs" slot */
  242. IRET_FRAME
  243. /*
  244. * There is no _ASM_EXTABLE_TYPE_REG() for ASM, however since this is
  245. * ASM the registers are known and we can trivially hard-code them.
  246. */
  247. _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_POP_ZERO|EX_REG_DS)
  248. _ASM_EXTABLE_TYPE(2b, 3b, EX_TYPE_POP_ZERO|EX_REG_ES)
  249. _ASM_EXTABLE_TYPE(3b, 4b, EX_TYPE_POP_ZERO|EX_REG_FS)
  250. .endm
  251. .macro RESTORE_ALL_NMI cr3_reg:req pop=0
  252. /*
  253. * Now switch the CR3 when PTI is enabled.
  254. *
  255. * We enter with kernel cr3 and switch the cr3 to the value
  256. * stored on \cr3_reg, which is either a user or a kernel cr3.
  257. */
  258. ALTERNATIVE "jmp .Lswitched_\@", "", X86_FEATURE_PTI
  259. testl $PTI_SWITCH_MASK, \cr3_reg
  260. jz .Lswitched_\@
  261. /* User cr3 in \cr3_reg - write it to hardware cr3 */
  262. movl \cr3_reg, %cr3
  263. .Lswitched_\@:
  264. BUG_IF_WRONG_CR3
  265. RESTORE_REGS pop=\pop
  266. .endm
  267. .macro CHECK_AND_APPLY_ESPFIX
  268. #ifdef CONFIG_X86_ESPFIX32
  269. #define GDT_ESPFIX_OFFSET (GDT_ENTRY_ESPFIX_SS * 8)
  270. #define GDT_ESPFIX_SS PER_CPU_VAR(gdt_page) + GDT_ESPFIX_OFFSET
  271. ALTERNATIVE "jmp .Lend_\@", "", X86_BUG_ESPFIX
  272. movl PT_EFLAGS(%esp), %eax # mix EFLAGS, SS and CS
  273. /*
  274. * Warning: PT_OLDSS(%esp) contains the wrong/random values if we
  275. * are returning to the kernel.
  276. * See comments in process.c:copy_thread() for details.
  277. */
  278. movb PT_OLDSS(%esp), %ah
  279. movb PT_CS(%esp), %al
  280. andl $(X86_EFLAGS_VM | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax
  281. cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax
  282. jne .Lend_\@ # returning to user-space with LDT SS
  283. /*
  284. * Setup and switch to ESPFIX stack
  285. *
  286. * We're returning to userspace with a 16 bit stack. The CPU will not
  287. * restore the high word of ESP for us on executing iret... This is an
  288. * "official" bug of all the x86-compatible CPUs, which we can work
  289. * around to make dosemu and wine happy. We do this by preloading the
  290. * high word of ESP with the high word of the userspace ESP while
  291. * compensating for the offset by changing to the ESPFIX segment with
  292. * a base address that matches for the difference.
  293. */
  294. mov %esp, %edx /* load kernel esp */
  295. mov PT_OLDESP(%esp), %eax /* load userspace esp */
  296. mov %dx, %ax /* eax: new kernel esp */
  297. sub %eax, %edx /* offset (low word is 0) */
  298. shr $16, %edx
  299. mov %dl, GDT_ESPFIX_SS + 4 /* bits 16..23 */
  300. mov %dh, GDT_ESPFIX_SS + 7 /* bits 24..31 */
  301. pushl $__ESPFIX_SS
  302. pushl %eax /* new kernel esp */
  303. /*
  304. * Disable interrupts, but do not irqtrace this section: we
  305. * will soon execute iret and the tracer was already set to
  306. * the irqstate after the IRET:
  307. */
  308. cli
  309. lss (%esp), %esp /* switch to espfix segment */
  310. .Lend_\@:
  311. #endif /* CONFIG_X86_ESPFIX32 */
  312. .endm
  313. /*
  314. * Called with pt_regs fully populated and kernel segments loaded,
  315. * so we can access PER_CPU and use the integer registers.
  316. *
  317. * We need to be very careful here with the %esp switch, because an NMI
  318. * can happen everywhere. If the NMI handler finds itself on the
  319. * entry-stack, it will overwrite the task-stack and everything we
  320. * copied there. So allocate the stack-frame on the task-stack and
  321. * switch to it before we do any copying.
  322. */
  323. .macro SWITCH_TO_KERNEL_STACK
  324. BUG_IF_WRONG_CR3
  325. SWITCH_TO_KERNEL_CR3 scratch_reg=%eax
  326. /*
  327. * %eax now contains the entry cr3 and we carry it forward in
  328. * that register for the time this macro runs
  329. */
  330. /* Are we on the entry stack? Bail out if not! */
  331. movl PER_CPU_VAR(cpu_entry_area), %ecx
  332. addl $CPU_ENTRY_AREA_entry_stack + SIZEOF_entry_stack, %ecx
  333. subl %esp, %ecx /* ecx = (end of entry_stack) - esp */
  334. cmpl $SIZEOF_entry_stack, %ecx
  335. jae .Lend_\@
  336. /* Load stack pointer into %esi and %edi */
  337. movl %esp, %esi
  338. movl %esi, %edi
  339. /* Move %edi to the top of the entry stack */
  340. andl $(MASK_entry_stack), %edi
  341. addl $(SIZEOF_entry_stack), %edi
  342. /* Load top of task-stack into %edi */
  343. movl TSS_entry2task_stack(%edi), %edi
  344. /* Special case - entry from kernel mode via entry stack */
  345. #ifdef CONFIG_VM86
  346. movl PT_EFLAGS(%esp), %ecx # mix EFLAGS and CS
  347. movb PT_CS(%esp), %cl
  348. andl $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %ecx
  349. #else
  350. movl PT_CS(%esp), %ecx
  351. andl $SEGMENT_RPL_MASK, %ecx
  352. #endif
  353. cmpl $USER_RPL, %ecx
  354. jb .Lentry_from_kernel_\@
  355. /* Bytes to copy */
  356. movl $PTREGS_SIZE, %ecx
  357. #ifdef CONFIG_VM86
  358. testl $X86_EFLAGS_VM, PT_EFLAGS(%esi)
  359. jz .Lcopy_pt_regs_\@
  360. /*
  361. * Stack-frame contains 4 additional segment registers when
  362. * coming from VM86 mode
  363. */
  364. addl $(4 * 4), %ecx
  365. #endif
  366. .Lcopy_pt_regs_\@:
  367. /* Allocate frame on task-stack */
  368. subl %ecx, %edi
  369. /* Switch to task-stack */
  370. movl %edi, %esp
  371. /*
  372. * We are now on the task-stack and can safely copy over the
  373. * stack-frame
  374. */
  375. shrl $2, %ecx
  376. cld
  377. rep movsl
  378. jmp .Lend_\@
  379. .Lentry_from_kernel_\@:
  380. /*
  381. * This handles the case when we enter the kernel from
  382. * kernel-mode and %esp points to the entry-stack. When this
  383. * happens we need to switch to the task-stack to run C code,
  384. * but switch back to the entry-stack again when we approach
  385. * iret and return to the interrupted code-path. This usually
  386. * happens when we hit an exception while restoring user-space
  387. * segment registers on the way back to user-space or when the
  388. * sysenter handler runs with eflags.tf set.
  389. *
  390. * When we switch to the task-stack here, we can't trust the
  391. * contents of the entry-stack anymore, as the exception handler
  392. * might be scheduled out or moved to another CPU. Therefore we
  393. * copy the complete entry-stack to the task-stack and set a
  394. * marker in the iret-frame (bit 31 of the CS dword) to detect
  395. * what we've done on the iret path.
  396. *
  397. * On the iret path we copy everything back and switch to the
  398. * entry-stack, so that the interrupted kernel code-path
  399. * continues on the same stack it was interrupted with.
  400. *
  401. * Be aware that an NMI can happen anytime in this code.
  402. *
  403. * %esi: Entry-Stack pointer (same as %esp)
  404. * %edi: Top of the task stack
  405. * %eax: CR3 on kernel entry
  406. */
  407. /* Calculate number of bytes on the entry stack in %ecx */
  408. movl %esi, %ecx
  409. /* %ecx to the top of entry-stack */
  410. andl $(MASK_entry_stack), %ecx
  411. addl $(SIZEOF_entry_stack), %ecx
  412. /* Number of bytes on the entry stack to %ecx */
  413. sub %esi, %ecx
  414. /* Mark stackframe as coming from entry stack */
  415. orl $CS_FROM_ENTRY_STACK, PT_CS(%esp)
  416. /*
  417. * Test the cr3 used to enter the kernel and add a marker
  418. * so that we can switch back to it before iret.
  419. */
  420. testl $PTI_SWITCH_MASK, %eax
  421. jz .Lcopy_pt_regs_\@
  422. orl $CS_FROM_USER_CR3, PT_CS(%esp)
  423. /*
  424. * %esi and %edi are unchanged, %ecx contains the number of
  425. * bytes to copy. The code at .Lcopy_pt_regs_\@ will allocate
  426. * the stack-frame on task-stack and copy everything over
  427. */
  428. jmp .Lcopy_pt_regs_\@
  429. .Lend_\@:
  430. .endm
  431. /*
  432. * Switch back from the kernel stack to the entry stack.
  433. *
  434. * The %esp register must point to pt_regs on the task stack. It will
  435. * first calculate the size of the stack-frame to copy, depending on
  436. * whether we return to VM86 mode or not. With that it uses 'rep movsl'
  437. * to copy the contents of the stack over to the entry stack.
  438. *
  439. * We must be very careful here, as we can't trust the contents of the
  440. * task-stack once we switched to the entry-stack. When an NMI happens
  441. * while on the entry-stack, the NMI handler will switch back to the top
  442. * of the task stack, overwriting our stack-frame we are about to copy.
  443. * Therefore we switch the stack only after everything is copied over.
  444. */
  445. .macro SWITCH_TO_ENTRY_STACK
  446. /* Bytes to copy */
  447. movl $PTREGS_SIZE, %ecx
  448. #ifdef CONFIG_VM86
  449. testl $(X86_EFLAGS_VM), PT_EFLAGS(%esp)
  450. jz .Lcopy_pt_regs_\@
  451. /* Additional 4 registers to copy when returning to VM86 mode */
  452. addl $(4 * 4), %ecx
  453. .Lcopy_pt_regs_\@:
  454. #endif
  455. /* Initialize source and destination for movsl */
  456. movl PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %edi
  457. subl %ecx, %edi
  458. movl %esp, %esi
  459. /* Save future stack pointer in %ebx */
  460. movl %edi, %ebx
  461. /* Copy over the stack-frame */
  462. shrl $2, %ecx
  463. cld
  464. rep movsl
  465. /*
  466. * Switch to entry-stack - needs to happen after everything is
  467. * copied because the NMI handler will overwrite the task-stack
  468. * when on entry-stack
  469. */
  470. movl %ebx, %esp
  471. .Lend_\@:
  472. .endm
  473. /*
  474. * This macro handles the case when we return to kernel-mode on the iret
  475. * path and have to switch back to the entry stack and/or user-cr3
  476. *
  477. * See the comments below the .Lentry_from_kernel_\@ label in the
  478. * SWITCH_TO_KERNEL_STACK macro for more details.
  479. */
  480. .macro PARANOID_EXIT_TO_KERNEL_MODE
  481. /*
  482. * Test if we entered the kernel with the entry-stack. Most
  483. * likely we did not, because this code only runs on the
  484. * return-to-kernel path.
  485. */
  486. testl $CS_FROM_ENTRY_STACK, PT_CS(%esp)
  487. jz .Lend_\@
  488. /* Unlikely slow-path */
  489. /* Clear marker from stack-frame */
  490. andl $(~CS_FROM_ENTRY_STACK), PT_CS(%esp)
  491. /* Copy the remaining task-stack contents to entry-stack */
  492. movl %esp, %esi
  493. movl PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %edi
  494. /* Bytes on the task-stack to ecx */
  495. movl PER_CPU_VAR(cpu_tss_rw + TSS_sp1), %ecx
  496. subl %esi, %ecx
  497. /* Allocate stack-frame on entry-stack */
  498. subl %ecx, %edi
  499. /*
  500. * Save future stack-pointer, we must not switch until the
  501. * copy is done, otherwise the NMI handler could destroy the
  502. * contents of the task-stack we are about to copy.
  503. */
  504. movl %edi, %ebx
  505. /* Do the copy */
  506. shrl $2, %ecx
  507. cld
  508. rep movsl
  509. /* Safe to switch to entry-stack now */
  510. movl %ebx, %esp
  511. /*
  512. * We came from entry-stack and need to check if we also need to
  513. * switch back to user cr3.
  514. */
  515. testl $CS_FROM_USER_CR3, PT_CS(%esp)
  516. jz .Lend_\@
  517. /* Clear marker from stack-frame */
  518. andl $(~CS_FROM_USER_CR3), PT_CS(%esp)
  519. SWITCH_TO_USER_CR3 scratch_reg=%eax
  520. .Lend_\@:
  521. .endm
  522. /**
  523. * idtentry - Macro to generate entry stubs for simple IDT entries
  524. * @vector: Vector number
  525. * @asmsym: ASM symbol for the entry point
  526. * @cfunc: C function to be called
  527. * @has_error_code: Hardware pushed error code on stack
  528. */
  529. .macro idtentry vector asmsym cfunc has_error_code:req
  530. SYM_CODE_START(\asmsym)
  531. ASM_CLAC
  532. cld
  533. .if \has_error_code == 0
  534. pushl $0 /* Clear the error code */
  535. .endif
  536. /* Push the C-function address into the GS slot */
  537. pushl $\cfunc
  538. /* Invoke the common exception entry */
  539. jmp handle_exception
  540. SYM_CODE_END(\asmsym)
  541. .endm
  542. .macro idtentry_irq vector cfunc
  543. .p2align CONFIG_X86_L1_CACHE_SHIFT
  544. SYM_CODE_START_LOCAL(asm_\cfunc)
  545. ASM_CLAC
  546. SAVE_ALL switch_stacks=1
  547. ENCODE_FRAME_POINTER
  548. movl %esp, %eax
  549. movl PT_ORIG_EAX(%esp), %edx /* get the vector from stack */
  550. movl $-1, PT_ORIG_EAX(%esp) /* no syscall to restart */
  551. call \cfunc
  552. jmp handle_exception_return
  553. SYM_CODE_END(asm_\cfunc)
  554. .endm
  555. .macro idtentry_sysvec vector cfunc
  556. idtentry \vector asm_\cfunc \cfunc has_error_code=0
  557. .endm
  558. /*
  559. * Include the defines which emit the idt entries which are shared
  560. * shared between 32 and 64 bit and emit the __irqentry_text_* markers
  561. * so the stacktrace boundary checks work.
  562. */
  563. .align 16
  564. .globl __irqentry_text_start
  565. __irqentry_text_start:
  566. #include <asm/idtentry.h>
  567. .align 16
  568. .globl __irqentry_text_end
  569. __irqentry_text_end:
  570. /*
  571. * %eax: prev task
  572. * %edx: next task
  573. */
  574. .pushsection .text, "ax"
  575. SYM_CODE_START(__switch_to_asm)
  576. /*
  577. * Save callee-saved registers
  578. * This must match the order in struct inactive_task_frame
  579. */
  580. pushl %ebp
  581. pushl %ebx
  582. pushl %edi
  583. pushl %esi
  584. /*
  585. * Flags are saved to prevent AC leakage. This could go
  586. * away if objtool would have 32bit support to verify
  587. * the STAC/CLAC correctness.
  588. */
  589. pushfl
  590. /* switch stack */
  591. movl %esp, TASK_threadsp(%eax)
  592. movl TASK_threadsp(%edx), %esp
  593. #ifdef CONFIG_STACKPROTECTOR
  594. movl TASK_stack_canary(%edx), %ebx
  595. movl %ebx, PER_CPU_VAR(__stack_chk_guard)
  596. #endif
  597. /*
  598. * When switching from a shallower to a deeper call stack
  599. * the RSB may either underflow or use entries populated
  600. * with userspace addresses. On CPUs where those concerns
  601. * exist, overwrite the RSB with entries which capture
  602. * speculative execution to prevent attack.
  603. */
  604. FILL_RETURN_BUFFER %ebx, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_CTXSW
  605. /* Restore flags or the incoming task to restore AC state. */
  606. popfl
  607. /* restore callee-saved registers */
  608. popl %esi
  609. popl %edi
  610. popl %ebx
  611. popl %ebp
  612. jmp __switch_to
  613. SYM_CODE_END(__switch_to_asm)
  614. .popsection
  615. /*
  616. * The unwinder expects the last frame on the stack to always be at the same
  617. * offset from the end of the page, which allows it to validate the stack.
  618. * Calling schedule_tail() directly would break that convention because its an
  619. * asmlinkage function so its argument has to be pushed on the stack. This
  620. * wrapper creates a proper "end of stack" frame header before the call.
  621. */
  622. .pushsection .text, "ax"
  623. SYM_FUNC_START(schedule_tail_wrapper)
  624. FRAME_BEGIN
  625. pushl %eax
  626. call schedule_tail
  627. popl %eax
  628. FRAME_END
  629. RET
  630. SYM_FUNC_END(schedule_tail_wrapper)
  631. .popsection
  632. /*
  633. * A newly forked process directly context switches into this address.
  634. *
  635. * eax: prev task we switched from
  636. * ebx: kernel thread func (NULL for user thread)
  637. * edi: kernel thread arg
  638. */
  639. .pushsection .text, "ax"
  640. SYM_CODE_START(ret_from_fork)
  641. call schedule_tail_wrapper
  642. testl %ebx, %ebx
  643. jnz 1f /* kernel threads are uncommon */
  644. 2:
  645. /* When we fork, we trace the syscall return in the child, too. */
  646. movl %esp, %eax
  647. call syscall_exit_to_user_mode
  648. jmp .Lsyscall_32_done
  649. /* kernel thread */
  650. 1: movl %edi, %eax
  651. CALL_NOSPEC ebx
  652. /*
  653. * A kernel thread is allowed to return here after successfully
  654. * calling kernel_execve(). Exit to userspace to complete the execve()
  655. * syscall.
  656. */
  657. movl $0, PT_EAX(%esp)
  658. jmp 2b
  659. SYM_CODE_END(ret_from_fork)
  660. .popsection
  661. SYM_ENTRY(__begin_SYSENTER_singlestep_region, SYM_L_GLOBAL, SYM_A_NONE)
  662. /*
  663. * All code from here through __end_SYSENTER_singlestep_region is subject
  664. * to being single-stepped if a user program sets TF and executes SYSENTER.
  665. * There is absolutely nothing that we can do to prevent this from happening
  666. * (thanks Intel!). To keep our handling of this situation as simple as
  667. * possible, we handle TF just like AC and NT, except that our #DB handler
  668. * will ignore all of the single-step traps generated in this range.
  669. */
  670. /*
  671. * 32-bit SYSENTER entry.
  672. *
  673. * 32-bit system calls through the vDSO's __kernel_vsyscall enter here
  674. * if X86_FEATURE_SEP is available. This is the preferred system call
  675. * entry on 32-bit systems.
  676. *
  677. * The SYSENTER instruction, in principle, should *only* occur in the
  678. * vDSO. In practice, a small number of Android devices were shipped
  679. * with a copy of Bionic that inlined a SYSENTER instruction. This
  680. * never happened in any of Google's Bionic versions -- it only happened
  681. * in a narrow range of Intel-provided versions.
  682. *
  683. * SYSENTER loads SS, ESP, CS, and EIP from previously programmed MSRs.
  684. * IF and VM in RFLAGS are cleared (IOW: interrupts are off).
  685. * SYSENTER does not save anything on the stack,
  686. * and does not save old EIP (!!!), ESP, or EFLAGS.
  687. *
  688. * To avoid losing track of EFLAGS.VM (and thus potentially corrupting
  689. * user and/or vm86 state), we explicitly disable the SYSENTER
  690. * instruction in vm86 mode by reprogramming the MSRs.
  691. *
  692. * Arguments:
  693. * eax system call number
  694. * ebx arg1
  695. * ecx arg2
  696. * edx arg3
  697. * esi arg4
  698. * edi arg5
  699. * ebp user stack
  700. * 0(%ebp) arg6
  701. */
  702. SYM_FUNC_START(entry_SYSENTER_32)
  703. /*
  704. * On entry-stack with all userspace-regs live - save and
  705. * restore eflags and %eax to use it as scratch-reg for the cr3
  706. * switch.
  707. */
  708. pushfl
  709. pushl %eax
  710. BUG_IF_WRONG_CR3 no_user_check=1
  711. SWITCH_TO_KERNEL_CR3 scratch_reg=%eax
  712. popl %eax
  713. popfl
  714. /* Stack empty again, switch to task stack */
  715. movl TSS_entry2task_stack(%esp), %esp
  716. .Lsysenter_past_esp:
  717. pushl $__USER_DS /* pt_regs->ss */
  718. pushl $0 /* pt_regs->sp (placeholder) */
  719. pushfl /* pt_regs->flags (except IF = 0) */
  720. pushl $__USER_CS /* pt_regs->cs */
  721. pushl $0 /* pt_regs->ip = 0 (placeholder) */
  722. pushl %eax /* pt_regs->orig_ax */
  723. SAVE_ALL pt_regs_ax=$-ENOSYS /* save rest, stack already switched */
  724. /*
  725. * SYSENTER doesn't filter flags, so we need to clear NT, AC
  726. * and TF ourselves. To save a few cycles, we can check whether
  727. * either was set instead of doing an unconditional popfq.
  728. * This needs to happen before enabling interrupts so that
  729. * we don't get preempted with NT set.
  730. *
  731. * If TF is set, we will single-step all the way to here -- do_debug
  732. * will ignore all the traps. (Yes, this is slow, but so is
  733. * single-stepping in general. This allows us to avoid having
  734. * a more complicated code to handle the case where a user program
  735. * forces us to single-step through the SYSENTER entry code.)
  736. *
  737. * NB.: .Lsysenter_fix_flags is a label with the code under it moved
  738. * out-of-line as an optimization: NT is unlikely to be set in the
  739. * majority of the cases and instead of polluting the I$ unnecessarily,
  740. * we're keeping that code behind a branch which will predict as
  741. * not-taken and therefore its instructions won't be fetched.
  742. */
  743. testl $X86_EFLAGS_NT|X86_EFLAGS_AC|X86_EFLAGS_TF, PT_EFLAGS(%esp)
  744. jnz .Lsysenter_fix_flags
  745. .Lsysenter_flags_fixed:
  746. movl %esp, %eax
  747. call do_SYSENTER_32
  748. testl %eax, %eax
  749. jz .Lsyscall_32_done
  750. STACKLEAK_ERASE
  751. /* Opportunistic SYSEXIT */
  752. /*
  753. * Setup entry stack - we keep the pointer in %eax and do the
  754. * switch after almost all user-state is restored.
  755. */
  756. /* Load entry stack pointer and allocate frame for eflags/eax */
  757. movl PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %eax
  758. subl $(2*4), %eax
  759. /* Copy eflags and eax to entry stack */
  760. movl PT_EFLAGS(%esp), %edi
  761. movl PT_EAX(%esp), %esi
  762. movl %edi, (%eax)
  763. movl %esi, 4(%eax)
  764. /* Restore user registers and segments */
  765. movl PT_EIP(%esp), %edx /* pt_regs->ip */
  766. movl PT_OLDESP(%esp), %ecx /* pt_regs->sp */
  767. 1: mov PT_FS(%esp), %fs
  768. popl %ebx /* pt_regs->bx */
  769. addl $2*4, %esp /* skip pt_regs->cx and pt_regs->dx */
  770. popl %esi /* pt_regs->si */
  771. popl %edi /* pt_regs->di */
  772. popl %ebp /* pt_regs->bp */
  773. /* Switch to entry stack */
  774. movl %eax, %esp
  775. /* Now ready to switch the cr3 */
  776. SWITCH_TO_USER_CR3 scratch_reg=%eax
  777. /*
  778. * Restore all flags except IF. (We restore IF separately because
  779. * STI gives a one-instruction window in which we won't be interrupted,
  780. * whereas POPF does not.)
  781. */
  782. btrl $X86_EFLAGS_IF_BIT, (%esp)
  783. BUG_IF_WRONG_CR3 no_user_check=1
  784. popfl
  785. popl %eax
  786. /*
  787. * Return back to the vDSO, which will pop ecx and edx.
  788. * Don't bother with DS and ES (they already contain __USER_DS).
  789. */
  790. sti
  791. sysexit
  792. 2: movl $0, PT_FS(%esp)
  793. jmp 1b
  794. _ASM_EXTABLE(1b, 2b)
  795. .Lsysenter_fix_flags:
  796. pushl $X86_EFLAGS_FIXED
  797. popfl
  798. jmp .Lsysenter_flags_fixed
  799. SYM_ENTRY(__end_SYSENTER_singlestep_region, SYM_L_GLOBAL, SYM_A_NONE)
  800. SYM_FUNC_END(entry_SYSENTER_32)
  801. /*
  802. * 32-bit legacy system call entry.
  803. *
  804. * 32-bit x86 Linux system calls traditionally used the INT $0x80
  805. * instruction. INT $0x80 lands here.
  806. *
  807. * This entry point can be used by any 32-bit perform system calls.
  808. * Instances of INT $0x80 can be found inline in various programs and
  809. * libraries. It is also used by the vDSO's __kernel_vsyscall
  810. * fallback for hardware that doesn't support a faster entry method.
  811. * Restarted 32-bit system calls also fall back to INT $0x80
  812. * regardless of what instruction was originally used to do the system
  813. * call. (64-bit programs can use INT $0x80 as well, but they can
  814. * only run on 64-bit kernels and therefore land in
  815. * entry_INT80_compat.)
  816. *
  817. * This is considered a slow path. It is not used by most libc
  818. * implementations on modern hardware except during process startup.
  819. *
  820. * Arguments:
  821. * eax system call number
  822. * ebx arg1
  823. * ecx arg2
  824. * edx arg3
  825. * esi arg4
  826. * edi arg5
  827. * ebp arg6
  828. */
  829. SYM_FUNC_START(entry_INT80_32)
  830. ASM_CLAC
  831. pushl %eax /* pt_regs->orig_ax */
  832. SAVE_ALL pt_regs_ax=$-ENOSYS switch_stacks=1 /* save rest */
  833. movl %esp, %eax
  834. call do_int80_syscall_32
  835. .Lsyscall_32_done:
  836. STACKLEAK_ERASE
  837. restore_all_switch_stack:
  838. SWITCH_TO_ENTRY_STACK
  839. CHECK_AND_APPLY_ESPFIX
  840. /* Switch back to user CR3 */
  841. SWITCH_TO_USER_CR3 scratch_reg=%eax
  842. BUG_IF_WRONG_CR3
  843. /* Restore user state */
  844. RESTORE_REGS pop=4 # skip orig_eax/error_code
  845. .Lirq_return:
  846. /*
  847. * ARCH_HAS_MEMBARRIER_SYNC_CORE rely on IRET core serialization
  848. * when returning from IPI handler and when returning from
  849. * scheduler to user-space.
  850. */
  851. iret
  852. .Lasm_iret_error:
  853. pushl $0 # no error code
  854. pushl $iret_error
  855. #ifdef CONFIG_DEBUG_ENTRY
  856. /*
  857. * The stack-frame here is the one that iret faulted on, so its a
  858. * return-to-user frame. We are on kernel-cr3 because we come here from
  859. * the fixup code. This confuses the CR3 checker, so switch to user-cr3
  860. * as the checker expects it.
  861. */
  862. pushl %eax
  863. SWITCH_TO_USER_CR3 scratch_reg=%eax
  864. popl %eax
  865. #endif
  866. jmp handle_exception
  867. _ASM_EXTABLE(.Lirq_return, .Lasm_iret_error)
  868. SYM_FUNC_END(entry_INT80_32)
  869. .macro FIXUP_ESPFIX_STACK
  870. /*
  871. * Switch back for ESPFIX stack to the normal zerobased stack
  872. *
  873. * We can't call C functions using the ESPFIX stack. This code reads
  874. * the high word of the segment base from the GDT and swiches to the
  875. * normal stack and adjusts ESP with the matching offset.
  876. *
  877. * We might be on user CR3 here, so percpu data is not mapped and we can't
  878. * access the GDT through the percpu segment. Instead, use SGDT to find
  879. * the cpu_entry_area alias of the GDT.
  880. */
  881. #ifdef CONFIG_X86_ESPFIX32
  882. /* fixup the stack */
  883. pushl %ecx
  884. subl $2*4, %esp
  885. sgdt (%esp)
  886. movl 2(%esp), %ecx /* GDT address */
  887. /*
  888. * Careful: ECX is a linear pointer, so we need to force base
  889. * zero. %cs is the only known-linear segment we have right now.
  890. */
  891. mov %cs:GDT_ESPFIX_OFFSET + 4(%ecx), %al /* bits 16..23 */
  892. mov %cs:GDT_ESPFIX_OFFSET + 7(%ecx), %ah /* bits 24..31 */
  893. shl $16, %eax
  894. addl $2*4, %esp
  895. popl %ecx
  896. addl %esp, %eax /* the adjusted stack pointer */
  897. pushl $__KERNEL_DS
  898. pushl %eax
  899. lss (%esp), %esp /* switch to the normal stack segment */
  900. #endif
  901. .endm
  902. .macro UNWIND_ESPFIX_STACK
  903. /* It's safe to clobber %eax, all other regs need to be preserved */
  904. #ifdef CONFIG_X86_ESPFIX32
  905. movl %ss, %eax
  906. /* see if on espfix stack */
  907. cmpw $__ESPFIX_SS, %ax
  908. jne .Lno_fixup_\@
  909. /* switch to normal stack */
  910. FIXUP_ESPFIX_STACK
  911. .Lno_fixup_\@:
  912. #endif
  913. .endm
  914. SYM_CODE_START_LOCAL_NOALIGN(handle_exception)
  915. /* the function address is in %gs's slot on the stack */
  916. SAVE_ALL switch_stacks=1 skip_gs=1 unwind_espfix=1
  917. ENCODE_FRAME_POINTER
  918. movl PT_GS(%esp), %edi # get the function address
  919. /* fixup orig %eax */
  920. movl PT_ORIG_EAX(%esp), %edx # get the error code
  921. movl $-1, PT_ORIG_EAX(%esp) # no syscall to restart
  922. movl %esp, %eax # pt_regs pointer
  923. CALL_NOSPEC edi
  924. handle_exception_return:
  925. #ifdef CONFIG_VM86
  926. movl PT_EFLAGS(%esp), %eax # mix EFLAGS and CS
  927. movb PT_CS(%esp), %al
  928. andl $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %eax
  929. #else
  930. /*
  931. * We can be coming here from child spawned by kernel_thread().
  932. */
  933. movl PT_CS(%esp), %eax
  934. andl $SEGMENT_RPL_MASK, %eax
  935. #endif
  936. cmpl $USER_RPL, %eax # returning to v8086 or userspace ?
  937. jnb ret_to_user
  938. PARANOID_EXIT_TO_KERNEL_MODE
  939. BUG_IF_WRONG_CR3
  940. RESTORE_REGS 4
  941. jmp .Lirq_return
  942. ret_to_user:
  943. movl %esp, %eax
  944. jmp restore_all_switch_stack
  945. SYM_CODE_END(handle_exception)
  946. SYM_CODE_START(asm_exc_double_fault)
  947. 1:
  948. /*
  949. * This is a task gate handler, not an interrupt gate handler.
  950. * The error code is on the stack, but the stack is otherwise
  951. * empty. Interrupts are off. Our state is sane with the following
  952. * exceptions:
  953. *
  954. * - CR0.TS is set. "TS" literally means "task switched".
  955. * - EFLAGS.NT is set because we're a "nested task".
  956. * - The doublefault TSS has back_link set and has been marked busy.
  957. * - TR points to the doublefault TSS and the normal TSS is busy.
  958. * - CR3 is the normal kernel PGD. This would be delightful, except
  959. * that the CPU didn't bother to save the old CR3 anywhere. This
  960. * would make it very awkward to return back to the context we came
  961. * from.
  962. *
  963. * The rest of EFLAGS is sanitized for us, so we don't need to
  964. * worry about AC or DF.
  965. *
  966. * Don't even bother popping the error code. It's always zero,
  967. * and ignoring it makes us a bit more robust against buggy
  968. * hypervisor task gate implementations.
  969. *
  970. * We will manually undo the task switch instead of doing a
  971. * task-switching IRET.
  972. */
  973. clts /* clear CR0.TS */
  974. pushl $X86_EFLAGS_FIXED
  975. popfl /* clear EFLAGS.NT */
  976. call doublefault_shim
  977. /* We don't support returning, so we have no IRET here. */
  978. 1:
  979. hlt
  980. jmp 1b
  981. SYM_CODE_END(asm_exc_double_fault)
  982. /*
  983. * NMI is doubly nasty. It can happen on the first instruction of
  984. * entry_SYSENTER_32 (just like #DB), but it can also interrupt the beginning
  985. * of the #DB handler even if that #DB in turn hit before entry_SYSENTER_32
  986. * switched stacks. We handle both conditions by simply checking whether we
  987. * interrupted kernel code running on the SYSENTER stack.
  988. */
  989. SYM_CODE_START(asm_exc_nmi)
  990. ASM_CLAC
  991. #ifdef CONFIG_X86_ESPFIX32
  992. /*
  993. * ESPFIX_SS is only ever set on the return to user path
  994. * after we've switched to the entry stack.
  995. */
  996. pushl %eax
  997. movl %ss, %eax
  998. cmpw $__ESPFIX_SS, %ax
  999. popl %eax
  1000. je .Lnmi_espfix_stack
  1001. #endif
  1002. pushl %eax # pt_regs->orig_ax
  1003. SAVE_ALL_NMI cr3_reg=%edi
  1004. ENCODE_FRAME_POINTER
  1005. xorl %edx, %edx # zero error code
  1006. movl %esp, %eax # pt_regs pointer
  1007. /* Are we currently on the SYSENTER stack? */
  1008. movl PER_CPU_VAR(cpu_entry_area), %ecx
  1009. addl $CPU_ENTRY_AREA_entry_stack + SIZEOF_entry_stack, %ecx
  1010. subl %eax, %ecx /* ecx = (end of entry_stack) - esp */
  1011. cmpl $SIZEOF_entry_stack, %ecx
  1012. jb .Lnmi_from_sysenter_stack
  1013. /* Not on SYSENTER stack. */
  1014. call exc_nmi
  1015. jmp .Lnmi_return
  1016. .Lnmi_from_sysenter_stack:
  1017. /*
  1018. * We're on the SYSENTER stack. Switch off. No one (not even debug)
  1019. * is using the thread stack right now, so it's safe for us to use it.
  1020. */
  1021. movl %esp, %ebx
  1022. movl PER_CPU_VAR(cpu_current_top_of_stack), %esp
  1023. call exc_nmi
  1024. movl %ebx, %esp
  1025. .Lnmi_return:
  1026. #ifdef CONFIG_X86_ESPFIX32
  1027. testl $CS_FROM_ESPFIX, PT_CS(%esp)
  1028. jnz .Lnmi_from_espfix
  1029. #endif
  1030. CHECK_AND_APPLY_ESPFIX
  1031. RESTORE_ALL_NMI cr3_reg=%edi pop=4
  1032. jmp .Lirq_return
  1033. #ifdef CONFIG_X86_ESPFIX32
  1034. .Lnmi_espfix_stack:
  1035. /*
  1036. * Create the pointer to LSS back
  1037. */
  1038. pushl %ss
  1039. pushl %esp
  1040. addl $4, (%esp)
  1041. /* Copy the (short) IRET frame */
  1042. pushl 4*4(%esp) # flags
  1043. pushl 4*4(%esp) # cs
  1044. pushl 4*4(%esp) # ip
  1045. pushl %eax # orig_ax
  1046. SAVE_ALL_NMI cr3_reg=%edi unwind_espfix=1
  1047. ENCODE_FRAME_POINTER
  1048. /* clear CS_FROM_KERNEL, set CS_FROM_ESPFIX */
  1049. xorl $(CS_FROM_ESPFIX | CS_FROM_KERNEL), PT_CS(%esp)
  1050. xorl %edx, %edx # zero error code
  1051. movl %esp, %eax # pt_regs pointer
  1052. jmp .Lnmi_from_sysenter_stack
  1053. .Lnmi_from_espfix:
  1054. RESTORE_ALL_NMI cr3_reg=%edi
  1055. /*
  1056. * Because we cleared CS_FROM_KERNEL, IRET_FRAME 'forgot' to
  1057. * fix up the gap and long frame:
  1058. *
  1059. * 3 - original frame (exception)
  1060. * 2 - ESPFIX block (above)
  1061. * 6 - gap (FIXUP_FRAME)
  1062. * 5 - long frame (FIXUP_FRAME)
  1063. * 1 - orig_ax
  1064. */
  1065. lss (1+5+6)*4(%esp), %esp # back to espfix stack
  1066. jmp .Lirq_return
  1067. #endif
  1068. SYM_CODE_END(asm_exc_nmi)
  1069. .pushsection .text, "ax"
  1070. SYM_CODE_START(rewind_stack_and_make_dead)
  1071. /* Prevent any naive code from trying to unwind to our caller. */
  1072. xorl %ebp, %ebp
  1073. movl PER_CPU_VAR(cpu_current_top_of_stack), %esi
  1074. leal -TOP_OF_KERNEL_STACK_PADDING-PTREGS_SIZE(%esi), %esp
  1075. call make_task_dead
  1076. 1: jmp 1b
  1077. SYM_CODE_END(rewind_stack_and_make_dead)
  1078. .popsection