vmenter.S 8.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. #include <linux/linkage.h>
  3. #include <asm/asm.h>
  4. #include <asm/bitsperlong.h>
  5. #include <asm/kvm_vcpu_regs.h>
  6. #include <asm/nospec-branch.h>
  7. #include <asm/percpu.h>
  8. #include <asm/segment.h>
  9. #include "kvm-asm-offsets.h"
  10. #include "run_flags.h"
  11. #define WORD_SIZE (BITS_PER_LONG / 8)
  12. #define VCPU_RAX __VCPU_REGS_RAX * WORD_SIZE
  13. #define VCPU_RCX __VCPU_REGS_RCX * WORD_SIZE
  14. #define VCPU_RDX __VCPU_REGS_RDX * WORD_SIZE
  15. #define VCPU_RBX __VCPU_REGS_RBX * WORD_SIZE
  16. /* Intentionally omit RSP as it's context switched by hardware */
  17. #define VCPU_RBP __VCPU_REGS_RBP * WORD_SIZE
  18. #define VCPU_RSI __VCPU_REGS_RSI * WORD_SIZE
  19. #define VCPU_RDI __VCPU_REGS_RDI * WORD_SIZE
  20. #ifdef CONFIG_X86_64
  21. #define VCPU_R8 __VCPU_REGS_R8 * WORD_SIZE
  22. #define VCPU_R9 __VCPU_REGS_R9 * WORD_SIZE
  23. #define VCPU_R10 __VCPU_REGS_R10 * WORD_SIZE
  24. #define VCPU_R11 __VCPU_REGS_R11 * WORD_SIZE
  25. #define VCPU_R12 __VCPU_REGS_R12 * WORD_SIZE
  26. #define VCPU_R13 __VCPU_REGS_R13 * WORD_SIZE
  27. #define VCPU_R14 __VCPU_REGS_R14 * WORD_SIZE
  28. #define VCPU_R15 __VCPU_REGS_R15 * WORD_SIZE
  29. #endif
  30. .section .noinstr.text, "ax"
  31. /**
  32. * __vmx_vcpu_run - Run a vCPU via a transition to VMX guest mode
  33. * @vmx: struct vcpu_vmx *
  34. * @regs: unsigned long * (to guest registers)
  35. * @flags: VMX_RUN_VMRESUME: use VMRESUME instead of VMLAUNCH
  36. * VMX_RUN_SAVE_SPEC_CTRL: save guest SPEC_CTRL into vmx->spec_ctrl
  37. *
  38. * Returns:
  39. * 0 on VM-Exit, 1 on VM-Fail
  40. */
  41. SYM_FUNC_START(__vmx_vcpu_run)
  42. push %_ASM_BP
  43. mov %_ASM_SP, %_ASM_BP
  44. #ifdef CONFIG_X86_64
  45. push %r15
  46. push %r14
  47. push %r13
  48. push %r12
  49. #else
  50. push %edi
  51. push %esi
  52. #endif
  53. push %_ASM_BX
  54. /* Save @vmx for SPEC_CTRL handling */
  55. push %_ASM_ARG1
  56. /* Save @flags for SPEC_CTRL handling */
  57. push %_ASM_ARG3
  58. /*
  59. * Save @regs, _ASM_ARG2 may be modified by vmx_update_host_rsp() and
  60. * @regs is needed after VM-Exit to save the guest's register values.
  61. */
  62. push %_ASM_ARG2
  63. /* Copy @flags to BL, _ASM_ARG3 is volatile. */
  64. mov %_ASM_ARG3B, %bl
  65. lea (%_ASM_SP), %_ASM_ARG2
  66. call vmx_update_host_rsp
  67. ALTERNATIVE "jmp .Lspec_ctrl_done", "", X86_FEATURE_MSR_SPEC_CTRL
  68. /*
  69. * SPEC_CTRL handling: if the guest's SPEC_CTRL value differs from the
  70. * host's, write the MSR.
  71. *
  72. * IMPORTANT: To avoid RSB underflow attacks and any other nastiness,
  73. * there must not be any returns or indirect branches between this code
  74. * and vmentry.
  75. */
  76. mov 2*WORD_SIZE(%_ASM_SP), %_ASM_DI
  77. movl VMX_spec_ctrl(%_ASM_DI), %edi
  78. movl PER_CPU_VAR(x86_spec_ctrl_current), %esi
  79. cmp %edi, %esi
  80. je .Lspec_ctrl_done
  81. mov $MSR_IA32_SPEC_CTRL, %ecx
  82. xor %edx, %edx
  83. mov %edi, %eax
  84. wrmsr
  85. .Lspec_ctrl_done:
  86. /*
  87. * Since vmentry is serializing on affected CPUs, there's no need for
  88. * an LFENCE to stop speculation from skipping the wrmsr.
  89. */
  90. /* Load @regs to RAX. */
  91. mov (%_ASM_SP), %_ASM_AX
  92. /* Check if vmlaunch or vmresume is needed */
  93. testb $VMX_RUN_VMRESUME, %bl
  94. /* Load guest registers. Don't clobber flags. */
  95. mov VCPU_RCX(%_ASM_AX), %_ASM_CX
  96. mov VCPU_RDX(%_ASM_AX), %_ASM_DX
  97. mov VCPU_RBX(%_ASM_AX), %_ASM_BX
  98. mov VCPU_RBP(%_ASM_AX), %_ASM_BP
  99. mov VCPU_RSI(%_ASM_AX), %_ASM_SI
  100. mov VCPU_RDI(%_ASM_AX), %_ASM_DI
  101. #ifdef CONFIG_X86_64
  102. mov VCPU_R8 (%_ASM_AX), %r8
  103. mov VCPU_R9 (%_ASM_AX), %r9
  104. mov VCPU_R10(%_ASM_AX), %r10
  105. mov VCPU_R11(%_ASM_AX), %r11
  106. mov VCPU_R12(%_ASM_AX), %r12
  107. mov VCPU_R13(%_ASM_AX), %r13
  108. mov VCPU_R14(%_ASM_AX), %r14
  109. mov VCPU_R15(%_ASM_AX), %r15
  110. #endif
  111. /* Load guest RAX. This kills the @regs pointer! */
  112. mov VCPU_RAX(%_ASM_AX), %_ASM_AX
  113. /* Check EFLAGS.ZF from 'testb' above */
  114. jz .Lvmlaunch
  115. /*
  116. * After a successful VMRESUME/VMLAUNCH, control flow "magically"
  117. * resumes below at 'vmx_vmexit' due to the VMCS HOST_RIP setting.
  118. * So this isn't a typical function and objtool needs to be told to
  119. * save the unwind state here and restore it below.
  120. */
  121. UNWIND_HINT_SAVE
  122. /*
  123. * If VMRESUME/VMLAUNCH and corresponding vmexit succeed, execution resumes at
  124. * the 'vmx_vmexit' label below.
  125. */
  126. .Lvmresume:
  127. vmresume
  128. jmp .Lvmfail
  129. .Lvmlaunch:
  130. vmlaunch
  131. jmp .Lvmfail
  132. _ASM_EXTABLE(.Lvmresume, .Lfixup)
  133. _ASM_EXTABLE(.Lvmlaunch, .Lfixup)
  134. SYM_INNER_LABEL(vmx_vmexit, SYM_L_GLOBAL)
  135. /* Restore unwind state from before the VMRESUME/VMLAUNCH. */
  136. UNWIND_HINT_RESTORE
  137. ENDBR
  138. /* Temporarily save guest's RAX. */
  139. push %_ASM_AX
  140. /* Reload @regs to RAX. */
  141. mov WORD_SIZE(%_ASM_SP), %_ASM_AX
  142. /* Save all guest registers, including RAX from the stack */
  143. pop VCPU_RAX(%_ASM_AX)
  144. mov %_ASM_CX, VCPU_RCX(%_ASM_AX)
  145. mov %_ASM_DX, VCPU_RDX(%_ASM_AX)
  146. mov %_ASM_BX, VCPU_RBX(%_ASM_AX)
  147. mov %_ASM_BP, VCPU_RBP(%_ASM_AX)
  148. mov %_ASM_SI, VCPU_RSI(%_ASM_AX)
  149. mov %_ASM_DI, VCPU_RDI(%_ASM_AX)
  150. #ifdef CONFIG_X86_64
  151. mov %r8, VCPU_R8 (%_ASM_AX)
  152. mov %r9, VCPU_R9 (%_ASM_AX)
  153. mov %r10, VCPU_R10(%_ASM_AX)
  154. mov %r11, VCPU_R11(%_ASM_AX)
  155. mov %r12, VCPU_R12(%_ASM_AX)
  156. mov %r13, VCPU_R13(%_ASM_AX)
  157. mov %r14, VCPU_R14(%_ASM_AX)
  158. mov %r15, VCPU_R15(%_ASM_AX)
  159. #endif
  160. /* Clear return value to indicate VM-Exit (as opposed to VM-Fail). */
  161. xor %ebx, %ebx
  162. .Lclear_regs:
  163. /* Discard @regs. The register is irrelevant, it just can't be RBX. */
  164. pop %_ASM_AX
  165. /*
  166. * Clear all general purpose registers except RSP and RBX to prevent
  167. * speculative use of the guest's values, even those that are reloaded
  168. * via the stack. In theory, an L1 cache miss when restoring registers
  169. * could lead to speculative execution with the guest's values.
  170. * Zeroing XORs are dirt cheap, i.e. the extra paranoia is essentially
  171. * free. RSP and RBX are exempt as RSP is restored by hardware during
  172. * VM-Exit and RBX is explicitly loaded with 0 or 1 to hold the return
  173. * value.
  174. */
  175. xor %eax, %eax
  176. xor %ecx, %ecx
  177. xor %edx, %edx
  178. xor %ebp, %ebp
  179. xor %esi, %esi
  180. xor %edi, %edi
  181. #ifdef CONFIG_X86_64
  182. xor %r8d, %r8d
  183. xor %r9d, %r9d
  184. xor %r10d, %r10d
  185. xor %r11d, %r11d
  186. xor %r12d, %r12d
  187. xor %r13d, %r13d
  188. xor %r14d, %r14d
  189. xor %r15d, %r15d
  190. #endif
  191. /*
  192. * IMPORTANT: RSB filling and SPEC_CTRL handling must be done before
  193. * the first unbalanced RET after vmexit!
  194. *
  195. * For retpoline or IBRS, RSB filling is needed to prevent poisoned RSB
  196. * entries and (in some cases) RSB underflow.
  197. *
  198. * eIBRS has its own protection against poisoned RSB, so it doesn't
  199. * need the RSB filling sequence. But it does need to be enabled, and a
  200. * single call to retire, before the first unbalanced RET.
  201. */
  202. FILL_RETURN_BUFFER %_ASM_CX, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_VMEXIT,\
  203. X86_FEATURE_RSB_VMEXIT_LITE
  204. pop %_ASM_ARG2 /* @flags */
  205. pop %_ASM_ARG1 /* @vmx */
  206. call vmx_spec_ctrl_restore_host
  207. /* Put return value in AX */
  208. mov %_ASM_BX, %_ASM_AX
  209. pop %_ASM_BX
  210. #ifdef CONFIG_X86_64
  211. pop %r12
  212. pop %r13
  213. pop %r14
  214. pop %r15
  215. #else
  216. pop %esi
  217. pop %edi
  218. #endif
  219. pop %_ASM_BP
  220. RET
  221. .Lfixup:
  222. cmpb $0, kvm_rebooting
  223. jne .Lvmfail
  224. ud2
  225. .Lvmfail:
  226. /* VM-Fail: set return value to 1 */
  227. mov $1, %_ASM_BX
  228. jmp .Lclear_regs
  229. SYM_FUNC_END(__vmx_vcpu_run)
  230. .section .text, "ax"
  231. /**
  232. * vmread_error_trampoline - Trampoline from inline asm to vmread_error()
  233. * @field: VMCS field encoding that failed
  234. * @fault: %true if the VMREAD faulted, %false if it failed
  235. * Save and restore volatile registers across a call to vmread_error(). Note,
  236. * all parameters are passed on the stack.
  237. */
  238. SYM_FUNC_START(vmread_error_trampoline)
  239. push %_ASM_BP
  240. mov %_ASM_SP, %_ASM_BP
  241. push %_ASM_AX
  242. push %_ASM_CX
  243. push %_ASM_DX
  244. #ifdef CONFIG_X86_64
  245. push %rdi
  246. push %rsi
  247. push %r8
  248. push %r9
  249. push %r10
  250. push %r11
  251. #endif
  252. /* Load @field and @fault to arg1 and arg2 respectively. */
  253. mov 3*WORD_SIZE(%_ASM_BP), %_ASM_ARG2
  254. mov 2*WORD_SIZE(%_ASM_BP), %_ASM_ARG1
  255. call vmread_error
  256. /* Zero out @fault, which will be popped into the result register. */
  257. _ASM_MOV $0, 3*WORD_SIZE(%_ASM_BP)
  258. #ifdef CONFIG_X86_64
  259. pop %r11
  260. pop %r10
  261. pop %r9
  262. pop %r8
  263. pop %rsi
  264. pop %rdi
  265. #endif
  266. pop %_ASM_DX
  267. pop %_ASM_CX
  268. pop %_ASM_AX
  269. pop %_ASM_BP
  270. RET
  271. SYM_FUNC_END(vmread_error_trampoline)
  272. SYM_FUNC_START(vmx_do_interrupt_nmi_irqoff)
  273. /*
  274. * Unconditionally create a stack frame, getting the correct RSP on the
  275. * stack (for x86-64) would take two instructions anyways, and RBP can
  276. * be used to restore RSP to make objtool happy (see below).
  277. */
  278. push %_ASM_BP
  279. mov %_ASM_SP, %_ASM_BP
  280. #ifdef CONFIG_X86_64
  281. /*
  282. * Align RSP to a 16-byte boundary (to emulate CPU behavior) before
  283. * creating the synthetic interrupt stack frame for the IRQ/NMI.
  284. */
  285. and $-16, %rsp
  286. push $__KERNEL_DS
  287. push %rbp
  288. #endif
  289. pushf
  290. push $__KERNEL_CS
  291. CALL_NOSPEC _ASM_ARG1
  292. /*
  293. * "Restore" RSP from RBP, even though IRET has already unwound RSP to
  294. * the correct value. objtool doesn't know the callee will IRET and,
  295. * without the explicit restore, thinks the stack is getting walloped.
  296. * Using an unwind hint is problematic due to x86-64's dynamic alignment.
  297. */
  298. mov %_ASM_BP, %_ASM_SP
  299. pop %_ASM_BP
  300. RET
  301. SYM_FUNC_END(vmx_do_interrupt_nmi_irqoff)