62ebfeb0dc
commit ba5ca5e5e6a1d55923e88b4a83da452166f5560e upstream.
Use LEA instead of ADD when adjusting %rsp in srso_safe_ret{,_alias}()
so as to avoid clobbering flags. Drop one of the INT3 instructions to
account for the LEA consuming one more byte than the ADD.
KVM's emulator makes indirect calls into a jump table of sorts, where
the destination of each call is a small blob of code that performs fast
emulation by executing the target instruction with fixed operands.
E.g. to emulate ADC, fastop() invokes adcb_al_dl():
adcb_al_dl:
<+0>: adc %dl,%al
<+2>: jmp <__x86_return_thunk>
A major motivation for doing fast emulation is to leverage the CPU to
handle consumption and manipulation of arithmetic flags, i.e. RFLAGS is
both an input and output to the target of the call. fastop() collects
the RFLAGS result by pushing RFLAGS onto the stack and popping them back
into a variable (held in %rdi in this case):
asm("push %[flags]; popf; " CALL_NOSPEC " ; pushf; pop %[flags]\n"
<+71>: mov 0xc0(%r8),%rdx
<+78>: mov 0x100(%r8),%rcx
<+85>: push %rdi
<+86>: popf
<+87>: call *%rsi
<+89>: nop
<+90>: nop
<+91>: nop
<+92>: pushf
<+93>: pop %rdi
and then propagating the arithmetic flags into the vCPU's emulator state:
ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
<+64>: and $0xfffffffffffff72a,%r9
<+94>: and $0x8d5,%edi
<+109>: or %rdi,%r9
<+122>: mov %r9,0x10(%r8)
The failures can be most easily reproduced by running the "emulator"
test in KVM-Unit-Tests.
If you're feeling a bit of deja vu, see commit b63f20a778
("x86/retpoline: Don't clobber RFLAGS during CALL_NOSPEC on i386").
In addition, this breaks booting of clang-compiled guest on
a gcc-compiled host where the host contains the %rsp-modifying SRSO
mitigations.
[ bp: Massage commit message, extend, remove addresses. ]
Fixes: fb3bd914b3ec ("x86/srso: Add a Speculative RAS Overflow mitigation")
Closes: https://lore.kernel.org/all/de474347-122d-54cd-eabf-9dcc95ab9eae@amd.com
Reported-by: Srikanth Aithal <sraithal@amd.com>
Reported-by: Nathan Chancellor <nathan@kernel.org>
Signed-off-by: Sean Christopherson <seanjc@google.com>
Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
Tested-by: Nathan Chancellor <nathan@kernel.org>
Cc: stable@vger.kernel.org
Link: https://lore.kernel.org/20230810013334.GA5354@dev-arch.thelio-3990X/
Link: https://lore.kernel.org/r/20230811155255.250835-1-seanjc@google.com
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
268 lines
7.6 KiB
ArmAsm
268 lines
7.6 KiB
ArmAsm
/* SPDX-License-Identifier: GPL-2.0 */
|
|
|
|
#include <linux/stringify.h>
|
|
#include <linux/linkage.h>
|
|
#include <asm/dwarf2.h>
|
|
#include <asm/cpufeatures.h>
|
|
#include <asm/alternative.h>
|
|
#include <asm/export.h>
|
|
#include <asm/nospec-branch.h>
|
|
#include <asm/unwind_hints.h>
|
|
#include <asm/frame.h>
|
|
#include <asm/nops.h>
|
|
|
|
.section .text.__x86.indirect_thunk
|
|
|
|
.macro RETPOLINE reg
|
|
ANNOTATE_INTRA_FUNCTION_CALL
|
|
call .Ldo_rop_\@
|
|
.Lspec_trap_\@:
|
|
UNWIND_HINT_EMPTY
|
|
pause
|
|
lfence
|
|
jmp .Lspec_trap_\@
|
|
.Ldo_rop_\@:
|
|
mov %\reg, (%_ASM_SP)
|
|
UNWIND_HINT_FUNC
|
|
RET
|
|
.endm
|
|
|
|
.macro THUNK reg
|
|
|
|
.align RETPOLINE_THUNK_SIZE
|
|
SYM_INNER_LABEL(__x86_indirect_thunk_\reg, SYM_L_GLOBAL)
|
|
UNWIND_HINT_EMPTY
|
|
|
|
ALTERNATIVE_2 __stringify(RETPOLINE \reg), \
|
|
__stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *%\reg; int3), X86_FEATURE_RETPOLINE_LFENCE, \
|
|
__stringify(ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), ALT_NOT(X86_FEATURE_RETPOLINE)
|
|
|
|
.endm
|
|
|
|
/*
|
|
* Despite being an assembler file we can't just use .irp here
|
|
* because __KSYM_DEPS__ only uses the C preprocessor and would
|
|
* only see one instance of "__x86_indirect_thunk_\reg" rather
|
|
* than one per register with the correct names. So we do it
|
|
* the simple and nasty way...
|
|
*
|
|
* Worse, you can only have a single EXPORT_SYMBOL per line,
|
|
* and CPP can't insert newlines, so we have to repeat everything
|
|
* at least twice.
|
|
*/
|
|
|
|
#define __EXPORT_THUNK(sym) _ASM_NOKPROBE(sym); EXPORT_SYMBOL(sym)
|
|
#define EXPORT_THUNK(reg) __EXPORT_THUNK(__x86_indirect_thunk_ ## reg)
|
|
|
|
.align RETPOLINE_THUNK_SIZE
|
|
SYM_CODE_START(__x86_indirect_thunk_array)
|
|
|
|
#define GEN(reg) THUNK reg
|
|
#include <asm/GEN-for-each-reg.h>
|
|
#undef GEN
|
|
|
|
.align RETPOLINE_THUNK_SIZE
|
|
SYM_CODE_END(__x86_indirect_thunk_array)
|
|
|
|
#define GEN(reg) EXPORT_THUNK(reg)
|
|
#include <asm/GEN-for-each-reg.h>
|
|
#undef GEN
|
|
|
|
/*
|
|
* This function name is magical and is used by -mfunction-return=thunk-extern
|
|
* for the compiler to generate JMPs to it.
|
|
*/
|
|
#ifdef CONFIG_RETHUNK
|
|
|
|
/*
|
|
* srso_alias_untrain_ret() and srso_alias_safe_ret() are placed at
|
|
* special addresses:
|
|
*
|
|
* - srso_alias_untrain_ret() is 2M aligned
|
|
* - srso_alias_safe_ret() is also in the same 2M page but bits 2, 8, 14
|
|
* and 20 in its virtual address are set (while those bits in the
|
|
* srso_alias_untrain_ret() function are cleared).
|
|
*
|
|
* This guarantees that those two addresses will alias in the branch
|
|
* target buffer of Zen3/4 generations, leading to any potential
|
|
* poisoned entries at that BTB slot to get evicted.
|
|
*
|
|
* As a result, srso_alias_safe_ret() becomes a safe return.
|
|
*/
|
|
#ifdef CONFIG_CPU_SRSO
|
|
.section .text.__x86.rethunk_untrain
|
|
|
|
SYM_START(srso_alias_untrain_ret, SYM_L_GLOBAL, SYM_A_NONE)
|
|
UNWIND_HINT_FUNC
|
|
ASM_NOP2
|
|
lfence
|
|
jmp srso_alias_return_thunk
|
|
SYM_FUNC_END(srso_alias_untrain_ret)
|
|
__EXPORT_THUNK(srso_alias_untrain_ret)
|
|
|
|
.section .text.__x86.rethunk_safe
|
|
#else
|
|
/* dummy definition for alternatives */
|
|
SYM_START(srso_alias_untrain_ret, SYM_L_GLOBAL, SYM_A_NONE)
|
|
ANNOTATE_UNRET_SAFE
|
|
ret
|
|
int3
|
|
SYM_FUNC_END(srso_alias_untrain_ret)
|
|
#endif
|
|
|
|
SYM_START(srso_alias_safe_ret, SYM_L_GLOBAL, SYM_A_NONE)
|
|
lea 8(%_ASM_SP), %_ASM_SP
|
|
UNWIND_HINT_FUNC
|
|
ANNOTATE_UNRET_SAFE
|
|
ret
|
|
int3
|
|
SYM_FUNC_END(srso_alias_safe_ret)
|
|
|
|
.section .text.__x86.return_thunk
|
|
|
|
SYM_CODE_START(srso_alias_return_thunk)
|
|
UNWIND_HINT_FUNC
|
|
ANNOTATE_NOENDBR
|
|
call srso_alias_safe_ret
|
|
ud2
|
|
SYM_CODE_END(srso_alias_return_thunk)
|
|
|
|
/*
|
|
* Some generic notes on the untraining sequences:
|
|
*
|
|
* They are interchangeable when it comes to flushing potentially wrong
|
|
* RET predictions from the BTB.
|
|
*
|
|
* The SRSO Zen1/2 (MOVABS) untraining sequence is longer than the
|
|
* Retbleed sequence because the return sequence done there
|
|
* (srso_safe_ret()) is longer and the return sequence must fully nest
|
|
* (end before) the untraining sequence. Therefore, the untraining
|
|
* sequence must fully overlap the return sequence.
|
|
*
|
|
* Regarding alignment - the instructions which need to be untrained,
|
|
* must all start at a cacheline boundary for Zen1/2 generations. That
|
|
* is, instruction sequences starting at srso_safe_ret() and
|
|
* the respective instruction sequences at retbleed_return_thunk()
|
|
* must start at a cacheline boundary.
|
|
*/
|
|
|
|
/*
|
|
* Safety details here pertain to the AMD Zen{1,2} microarchitecture:
|
|
* 1) The RET at retbleed_return_thunk must be on a 64 byte boundary, for
|
|
* alignment within the BTB.
|
|
* 2) The instruction at retbleed_untrain_ret must contain, and not
|
|
* end with, the 0xc3 byte of the RET.
|
|
* 3) STIBP must be enabled, or SMT disabled, to prevent the sibling thread
|
|
* from re-poisioning the BTB prediction.
|
|
*/
|
|
.align 64
|
|
.skip 64 - (retbleed_return_thunk - retbleed_untrain_ret), 0xcc
|
|
SYM_FUNC_START_NOALIGN(retbleed_untrain_ret);
|
|
|
|
/*
|
|
* As executed from retbleed_untrain_ret, this is:
|
|
*
|
|
* TEST $0xcc, %bl
|
|
* LFENCE
|
|
* JMP retbleed_return_thunk
|
|
*
|
|
* Executing the TEST instruction has a side effect of evicting any BTB
|
|
* prediction (potentially attacker controlled) attached to the RET, as
|
|
* retbleed_return_thunk + 1 isn't an instruction boundary at the moment.
|
|
*/
|
|
.byte 0xf6
|
|
|
|
/*
|
|
* As executed from retbleed_return_thunk, this is a plain RET.
|
|
*
|
|
* As part of the TEST above, RET is the ModRM byte, and INT3 the imm8.
|
|
*
|
|
* We subsequently jump backwards and architecturally execute the RET.
|
|
* This creates a correct BTB prediction (type=ret), but in the
|
|
* meantime we suffer Straight Line Speculation (because the type was
|
|
* no branch) which is halted by the INT3.
|
|
*
|
|
* With SMT enabled and STIBP active, a sibling thread cannot poison
|
|
* RET's prediction to a type of its choice, but can evict the
|
|
* prediction due to competitive sharing. If the prediction is
|
|
* evicted, retbleed_return_thunk will suffer Straight Line Speculation
|
|
* which will be contained safely by the INT3.
|
|
*/
|
|
SYM_INNER_LABEL(retbleed_return_thunk, SYM_L_GLOBAL)
|
|
ret
|
|
int3
|
|
SYM_CODE_END(retbleed_return_thunk)
|
|
|
|
/*
|
|
* Ensure the TEST decoding / BTB invalidation is complete.
|
|
*/
|
|
lfence
|
|
|
|
/*
|
|
* Jump back and execute the RET in the middle of the TEST instruction.
|
|
* INT3 is for SLS protection.
|
|
*/
|
|
jmp retbleed_return_thunk
|
|
int3
|
|
SYM_FUNC_END(retbleed_untrain_ret)
|
|
__EXPORT_THUNK(retbleed_untrain_ret)
|
|
|
|
/*
|
|
* SRSO untraining sequence for Zen1/2, similar to retbleed_untrain_ret()
|
|
* above. On kernel entry, srso_untrain_ret() is executed which is a
|
|
*
|
|
* movabs $0xccccc30824648d48,%rax
|
|
*
|
|
* and when the return thunk executes the inner label srso_safe_ret()
|
|
* later, it is a stack manipulation and a RET which is mispredicted and
|
|
* thus a "safe" one to use.
|
|
*/
|
|
.align 64
|
|
.skip 64 - (srso_safe_ret - srso_untrain_ret), 0xcc
|
|
SYM_START(srso_untrain_ret, SYM_L_GLOBAL, SYM_A_NONE)
|
|
.byte 0x48, 0xb8
|
|
|
|
/*
|
|
* This forces the function return instruction to speculate into a trap
|
|
* (UD2 in srso_return_thunk() below). This RET will then mispredict
|
|
* and execution will continue at the return site read from the top of
|
|
* the stack.
|
|
*/
|
|
SYM_INNER_LABEL(srso_safe_ret, SYM_L_GLOBAL)
|
|
lea 8(%_ASM_SP), %_ASM_SP
|
|
ret
|
|
int3
|
|
int3
|
|
/* end of movabs */
|
|
lfence
|
|
call srso_safe_ret
|
|
ud2
|
|
SYM_CODE_END(srso_safe_ret)
|
|
SYM_FUNC_END(srso_untrain_ret)
|
|
__EXPORT_THUNK(srso_untrain_ret)
|
|
|
|
SYM_CODE_START(srso_return_thunk)
|
|
UNWIND_HINT_FUNC
|
|
ANNOTATE_NOENDBR
|
|
call srso_safe_ret
|
|
ud2
|
|
SYM_CODE_END(srso_return_thunk)
|
|
|
|
SYM_FUNC_START(entry_untrain_ret)
|
|
ALTERNATIVE_2 "jmp retbleed_untrain_ret", \
|
|
"jmp srso_untrain_ret", X86_FEATURE_SRSO, \
|
|
"jmp srso_alias_untrain_ret", X86_FEATURE_SRSO_ALIAS
|
|
SYM_FUNC_END(entry_untrain_ret)
|
|
__EXPORT_THUNK(entry_untrain_ret)
|
|
|
|
SYM_CODE_START(__x86_return_thunk)
|
|
UNWIND_HINT_FUNC
|
|
ANNOTATE_NOENDBR
|
|
ANNOTATE_UNRET_SAFE
|
|
ret
|
|
int3
|
|
SYM_CODE_END(__x86_return_thunk)
|
|
EXPORT_SYMBOL(__x86_return_thunk)
|
|
|
|
#endif /* CONFIG_RETHUNK */
|