x86: Prepare asm files for straight-line-speculation
commit f94909ceb1ed4bfdb2ada72f93236305e6d6951f upstream. Replace all ret/retq instructions with RET in preparation of making RET a macro. Since AS is case insensitive it's a big no-op without RET defined. find arch/x86/ -name \*.S | while read file do sed -i 's/\<ret[q]*\>/RET/' $file done Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Signed-off-by: Borislav Petkov <bp@suse.de> Link: https://lore.kernel.org/r/20211204134907.905503893@infradead.org [bwh: Backported to 5.10: ran the above command] Signed-off-by: Ben Hutchings <ben@decadent.org.uk> Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
This commit is contained in:

committed by
Greg Kroah-Hartman

parent
a512fcd881
commit
3c91e22576
@@ -89,7 +89,7 @@ SYM_FUNC_START(__efi64_thunk)
|
|||||||
|
|
||||||
pop %rbx
|
pop %rbx
|
||||||
pop %rbp
|
pop %rbp
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(__efi64_thunk)
|
SYM_FUNC_END(__efi64_thunk)
|
||||||
|
|
||||||
.code32
|
.code32
|
||||||
|
@@ -786,7 +786,7 @@ SYM_FUNC_START(efi32_pe_entry)
|
|||||||
2: popl %edi // restore callee-save registers
|
2: popl %edi // restore callee-save registers
|
||||||
popl %ebx
|
popl %ebx
|
||||||
leave
|
leave
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(efi32_pe_entry)
|
SYM_FUNC_END(efi32_pe_entry)
|
||||||
|
|
||||||
.section ".rodata"
|
.section ".rodata"
|
||||||
@@ -868,7 +868,7 @@ SYM_FUNC_START(startup32_check_sev_cbit)
|
|||||||
popl %ebx
|
popl %ebx
|
||||||
popl %eax
|
popl %eax
|
||||||
#endif
|
#endif
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(startup32_check_sev_cbit)
|
SYM_FUNC_END(startup32_check_sev_cbit)
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
@@ -58,7 +58,7 @@ SYM_FUNC_START(get_sev_encryption_bit)
|
|||||||
|
|
||||||
#endif /* CONFIG_AMD_MEM_ENCRYPT */
|
#endif /* CONFIG_AMD_MEM_ENCRYPT */
|
||||||
|
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(get_sev_encryption_bit)
|
SYM_FUNC_END(get_sev_encryption_bit)
|
||||||
|
|
||||||
.code64
|
.code64
|
||||||
@@ -99,7 +99,7 @@ SYM_FUNC_START(set_sev_encryption_mask)
|
|||||||
#endif
|
#endif
|
||||||
|
|
||||||
xor %rax, %rax
|
xor %rax, %rax
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(set_sev_encryption_mask)
|
SYM_FUNC_END(set_sev_encryption_mask)
|
||||||
|
|
||||||
.data
|
.data
|
||||||
|
@@ -122,7 +122,7 @@ SYM_FUNC_START_LOCAL(__load_partial)
|
|||||||
pxor T0, MSG
|
pxor T0, MSG
|
||||||
|
|
||||||
.Lld_partial_8:
|
.Lld_partial_8:
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(__load_partial)
|
SYM_FUNC_END(__load_partial)
|
||||||
|
|
||||||
/*
|
/*
|
||||||
@@ -180,7 +180,7 @@ SYM_FUNC_START_LOCAL(__store_partial)
|
|||||||
mov %r10b, (%r9)
|
mov %r10b, (%r9)
|
||||||
|
|
||||||
.Lst_partial_1:
|
.Lst_partial_1:
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(__store_partial)
|
SYM_FUNC_END(__store_partial)
|
||||||
|
|
||||||
/*
|
/*
|
||||||
@@ -225,7 +225,7 @@ SYM_FUNC_START(crypto_aegis128_aesni_init)
|
|||||||
movdqu STATE4, 0x40(STATEP)
|
movdqu STATE4, 0x40(STATEP)
|
||||||
|
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(crypto_aegis128_aesni_init)
|
SYM_FUNC_END(crypto_aegis128_aesni_init)
|
||||||
|
|
||||||
/*
|
/*
|
||||||
@@ -337,7 +337,7 @@ SYM_FUNC_START(crypto_aegis128_aesni_ad)
|
|||||||
movdqu STATE3, 0x30(STATEP)
|
movdqu STATE3, 0x30(STATEP)
|
||||||
movdqu STATE4, 0x40(STATEP)
|
movdqu STATE4, 0x40(STATEP)
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret
|
RET
|
||||||
|
|
||||||
.Lad_out_1:
|
.Lad_out_1:
|
||||||
movdqu STATE4, 0x00(STATEP)
|
movdqu STATE4, 0x00(STATEP)
|
||||||
@@ -346,7 +346,7 @@ SYM_FUNC_START(crypto_aegis128_aesni_ad)
|
|||||||
movdqu STATE2, 0x30(STATEP)
|
movdqu STATE2, 0x30(STATEP)
|
||||||
movdqu STATE3, 0x40(STATEP)
|
movdqu STATE3, 0x40(STATEP)
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret
|
RET
|
||||||
|
|
||||||
.Lad_out_2:
|
.Lad_out_2:
|
||||||
movdqu STATE3, 0x00(STATEP)
|
movdqu STATE3, 0x00(STATEP)
|
||||||
@@ -355,7 +355,7 @@ SYM_FUNC_START(crypto_aegis128_aesni_ad)
|
|||||||
movdqu STATE1, 0x30(STATEP)
|
movdqu STATE1, 0x30(STATEP)
|
||||||
movdqu STATE2, 0x40(STATEP)
|
movdqu STATE2, 0x40(STATEP)
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret
|
RET
|
||||||
|
|
||||||
.Lad_out_3:
|
.Lad_out_3:
|
||||||
movdqu STATE2, 0x00(STATEP)
|
movdqu STATE2, 0x00(STATEP)
|
||||||
@@ -364,7 +364,7 @@ SYM_FUNC_START(crypto_aegis128_aesni_ad)
|
|||||||
movdqu STATE0, 0x30(STATEP)
|
movdqu STATE0, 0x30(STATEP)
|
||||||
movdqu STATE1, 0x40(STATEP)
|
movdqu STATE1, 0x40(STATEP)
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret
|
RET
|
||||||
|
|
||||||
.Lad_out_4:
|
.Lad_out_4:
|
||||||
movdqu STATE1, 0x00(STATEP)
|
movdqu STATE1, 0x00(STATEP)
|
||||||
@@ -373,11 +373,11 @@ SYM_FUNC_START(crypto_aegis128_aesni_ad)
|
|||||||
movdqu STATE4, 0x30(STATEP)
|
movdqu STATE4, 0x30(STATEP)
|
||||||
movdqu STATE0, 0x40(STATEP)
|
movdqu STATE0, 0x40(STATEP)
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret
|
RET
|
||||||
|
|
||||||
.Lad_out:
|
.Lad_out:
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(crypto_aegis128_aesni_ad)
|
SYM_FUNC_END(crypto_aegis128_aesni_ad)
|
||||||
|
|
||||||
.macro encrypt_block a s0 s1 s2 s3 s4 i
|
.macro encrypt_block a s0 s1 s2 s3 s4 i
|
||||||
@@ -452,7 +452,7 @@ SYM_FUNC_START(crypto_aegis128_aesni_enc)
|
|||||||
movdqu STATE2, 0x30(STATEP)
|
movdqu STATE2, 0x30(STATEP)
|
||||||
movdqu STATE3, 0x40(STATEP)
|
movdqu STATE3, 0x40(STATEP)
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret
|
RET
|
||||||
|
|
||||||
.Lenc_out_1:
|
.Lenc_out_1:
|
||||||
movdqu STATE3, 0x00(STATEP)
|
movdqu STATE3, 0x00(STATEP)
|
||||||
@@ -461,7 +461,7 @@ SYM_FUNC_START(crypto_aegis128_aesni_enc)
|
|||||||
movdqu STATE1, 0x30(STATEP)
|
movdqu STATE1, 0x30(STATEP)
|
||||||
movdqu STATE2, 0x40(STATEP)
|
movdqu STATE2, 0x40(STATEP)
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret
|
RET
|
||||||
|
|
||||||
.Lenc_out_2:
|
.Lenc_out_2:
|
||||||
movdqu STATE2, 0x00(STATEP)
|
movdqu STATE2, 0x00(STATEP)
|
||||||
@@ -470,7 +470,7 @@ SYM_FUNC_START(crypto_aegis128_aesni_enc)
|
|||||||
movdqu STATE0, 0x30(STATEP)
|
movdqu STATE0, 0x30(STATEP)
|
||||||
movdqu STATE1, 0x40(STATEP)
|
movdqu STATE1, 0x40(STATEP)
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret
|
RET
|
||||||
|
|
||||||
.Lenc_out_3:
|
.Lenc_out_3:
|
||||||
movdqu STATE1, 0x00(STATEP)
|
movdqu STATE1, 0x00(STATEP)
|
||||||
@@ -479,7 +479,7 @@ SYM_FUNC_START(crypto_aegis128_aesni_enc)
|
|||||||
movdqu STATE4, 0x30(STATEP)
|
movdqu STATE4, 0x30(STATEP)
|
||||||
movdqu STATE0, 0x40(STATEP)
|
movdqu STATE0, 0x40(STATEP)
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret
|
RET
|
||||||
|
|
||||||
.Lenc_out_4:
|
.Lenc_out_4:
|
||||||
movdqu STATE0, 0x00(STATEP)
|
movdqu STATE0, 0x00(STATEP)
|
||||||
@@ -488,11 +488,11 @@ SYM_FUNC_START(crypto_aegis128_aesni_enc)
|
|||||||
movdqu STATE3, 0x30(STATEP)
|
movdqu STATE3, 0x30(STATEP)
|
||||||
movdqu STATE4, 0x40(STATEP)
|
movdqu STATE4, 0x40(STATEP)
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret
|
RET
|
||||||
|
|
||||||
.Lenc_out:
|
.Lenc_out:
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(crypto_aegis128_aesni_enc)
|
SYM_FUNC_END(crypto_aegis128_aesni_enc)
|
||||||
|
|
||||||
/*
|
/*
|
||||||
@@ -532,7 +532,7 @@ SYM_FUNC_START(crypto_aegis128_aesni_enc_tail)
|
|||||||
movdqu STATE3, 0x40(STATEP)
|
movdqu STATE3, 0x40(STATEP)
|
||||||
|
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(crypto_aegis128_aesni_enc_tail)
|
SYM_FUNC_END(crypto_aegis128_aesni_enc_tail)
|
||||||
|
|
||||||
.macro decrypt_block a s0 s1 s2 s3 s4 i
|
.macro decrypt_block a s0 s1 s2 s3 s4 i
|
||||||
@@ -606,7 +606,7 @@ SYM_FUNC_START(crypto_aegis128_aesni_dec)
|
|||||||
movdqu STATE2, 0x30(STATEP)
|
movdqu STATE2, 0x30(STATEP)
|
||||||
movdqu STATE3, 0x40(STATEP)
|
movdqu STATE3, 0x40(STATEP)
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret
|
RET
|
||||||
|
|
||||||
.Ldec_out_1:
|
.Ldec_out_1:
|
||||||
movdqu STATE3, 0x00(STATEP)
|
movdqu STATE3, 0x00(STATEP)
|
||||||
@@ -615,7 +615,7 @@ SYM_FUNC_START(crypto_aegis128_aesni_dec)
|
|||||||
movdqu STATE1, 0x30(STATEP)
|
movdqu STATE1, 0x30(STATEP)
|
||||||
movdqu STATE2, 0x40(STATEP)
|
movdqu STATE2, 0x40(STATEP)
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret
|
RET
|
||||||
|
|
||||||
.Ldec_out_2:
|
.Ldec_out_2:
|
||||||
movdqu STATE2, 0x00(STATEP)
|
movdqu STATE2, 0x00(STATEP)
|
||||||
@@ -624,7 +624,7 @@ SYM_FUNC_START(crypto_aegis128_aesni_dec)
|
|||||||
movdqu STATE0, 0x30(STATEP)
|
movdqu STATE0, 0x30(STATEP)
|
||||||
movdqu STATE1, 0x40(STATEP)
|
movdqu STATE1, 0x40(STATEP)
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret
|
RET
|
||||||
|
|
||||||
.Ldec_out_3:
|
.Ldec_out_3:
|
||||||
movdqu STATE1, 0x00(STATEP)
|
movdqu STATE1, 0x00(STATEP)
|
||||||
@@ -633,7 +633,7 @@ SYM_FUNC_START(crypto_aegis128_aesni_dec)
|
|||||||
movdqu STATE4, 0x30(STATEP)
|
movdqu STATE4, 0x30(STATEP)
|
||||||
movdqu STATE0, 0x40(STATEP)
|
movdqu STATE0, 0x40(STATEP)
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret
|
RET
|
||||||
|
|
||||||
.Ldec_out_4:
|
.Ldec_out_4:
|
||||||
movdqu STATE0, 0x00(STATEP)
|
movdqu STATE0, 0x00(STATEP)
|
||||||
@@ -642,11 +642,11 @@ SYM_FUNC_START(crypto_aegis128_aesni_dec)
|
|||||||
movdqu STATE3, 0x30(STATEP)
|
movdqu STATE3, 0x30(STATEP)
|
||||||
movdqu STATE4, 0x40(STATEP)
|
movdqu STATE4, 0x40(STATEP)
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret
|
RET
|
||||||
|
|
||||||
.Ldec_out:
|
.Ldec_out:
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(crypto_aegis128_aesni_dec)
|
SYM_FUNC_END(crypto_aegis128_aesni_dec)
|
||||||
|
|
||||||
/*
|
/*
|
||||||
@@ -696,7 +696,7 @@ SYM_FUNC_START(crypto_aegis128_aesni_dec_tail)
|
|||||||
movdqu STATE3, 0x40(STATEP)
|
movdqu STATE3, 0x40(STATEP)
|
||||||
|
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(crypto_aegis128_aesni_dec_tail)
|
SYM_FUNC_END(crypto_aegis128_aesni_dec_tail)
|
||||||
|
|
||||||
/*
|
/*
|
||||||
@@ -743,5 +743,5 @@ SYM_FUNC_START(crypto_aegis128_aesni_final)
|
|||||||
movdqu MSG, (%rsi)
|
movdqu MSG, (%rsi)
|
||||||
|
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(crypto_aegis128_aesni_final)
|
SYM_FUNC_END(crypto_aegis128_aesni_final)
|
||||||
|
@@ -525,7 +525,7 @@ ddq_add_8:
|
|||||||
/* return updated IV */
|
/* return updated IV */
|
||||||
vpshufb xbyteswap, xcounter, xcounter
|
vpshufb xbyteswap, xcounter, xcounter
|
||||||
vmovdqu xcounter, (p_iv)
|
vmovdqu xcounter, (p_iv)
|
||||||
ret
|
RET
|
||||||
.endm
|
.endm
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
@@ -1598,7 +1598,7 @@ SYM_FUNC_START(aesni_gcm_dec)
|
|||||||
GCM_ENC_DEC dec
|
GCM_ENC_DEC dec
|
||||||
GCM_COMPLETE arg10, arg11
|
GCM_COMPLETE arg10, arg11
|
||||||
FUNC_RESTORE
|
FUNC_RESTORE
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(aesni_gcm_dec)
|
SYM_FUNC_END(aesni_gcm_dec)
|
||||||
|
|
||||||
|
|
||||||
@@ -1687,7 +1687,7 @@ SYM_FUNC_START(aesni_gcm_enc)
|
|||||||
|
|
||||||
GCM_COMPLETE arg10, arg11
|
GCM_COMPLETE arg10, arg11
|
||||||
FUNC_RESTORE
|
FUNC_RESTORE
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(aesni_gcm_enc)
|
SYM_FUNC_END(aesni_gcm_enc)
|
||||||
|
|
||||||
/*****************************************************************************
|
/*****************************************************************************
|
||||||
@@ -1705,7 +1705,7 @@ SYM_FUNC_START(aesni_gcm_init)
|
|||||||
FUNC_SAVE
|
FUNC_SAVE
|
||||||
GCM_INIT %arg3, %arg4,%arg5, %arg6
|
GCM_INIT %arg3, %arg4,%arg5, %arg6
|
||||||
FUNC_RESTORE
|
FUNC_RESTORE
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(aesni_gcm_init)
|
SYM_FUNC_END(aesni_gcm_init)
|
||||||
|
|
||||||
/*****************************************************************************
|
/*****************************************************************************
|
||||||
@@ -1720,7 +1720,7 @@ SYM_FUNC_START(aesni_gcm_enc_update)
|
|||||||
FUNC_SAVE
|
FUNC_SAVE
|
||||||
GCM_ENC_DEC enc
|
GCM_ENC_DEC enc
|
||||||
FUNC_RESTORE
|
FUNC_RESTORE
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(aesni_gcm_enc_update)
|
SYM_FUNC_END(aesni_gcm_enc_update)
|
||||||
|
|
||||||
/*****************************************************************************
|
/*****************************************************************************
|
||||||
@@ -1735,7 +1735,7 @@ SYM_FUNC_START(aesni_gcm_dec_update)
|
|||||||
FUNC_SAVE
|
FUNC_SAVE
|
||||||
GCM_ENC_DEC dec
|
GCM_ENC_DEC dec
|
||||||
FUNC_RESTORE
|
FUNC_RESTORE
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(aesni_gcm_dec_update)
|
SYM_FUNC_END(aesni_gcm_dec_update)
|
||||||
|
|
||||||
/*****************************************************************************
|
/*****************************************************************************
|
||||||
@@ -1750,7 +1750,7 @@ SYM_FUNC_START(aesni_gcm_finalize)
|
|||||||
FUNC_SAVE
|
FUNC_SAVE
|
||||||
GCM_COMPLETE %arg3 %arg4
|
GCM_COMPLETE %arg3 %arg4
|
||||||
FUNC_RESTORE
|
FUNC_RESTORE
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(aesni_gcm_finalize)
|
SYM_FUNC_END(aesni_gcm_finalize)
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
@@ -1766,7 +1766,7 @@ SYM_FUNC_START_LOCAL(_key_expansion_256a)
|
|||||||
pxor %xmm1, %xmm0
|
pxor %xmm1, %xmm0
|
||||||
movaps %xmm0, (TKEYP)
|
movaps %xmm0, (TKEYP)
|
||||||
add $0x10, TKEYP
|
add $0x10, TKEYP
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(_key_expansion_256a)
|
SYM_FUNC_END(_key_expansion_256a)
|
||||||
SYM_FUNC_END_ALIAS(_key_expansion_128)
|
SYM_FUNC_END_ALIAS(_key_expansion_128)
|
||||||
|
|
||||||
@@ -1791,7 +1791,7 @@ SYM_FUNC_START_LOCAL(_key_expansion_192a)
|
|||||||
shufps $0b01001110, %xmm2, %xmm1
|
shufps $0b01001110, %xmm2, %xmm1
|
||||||
movaps %xmm1, 0x10(TKEYP)
|
movaps %xmm1, 0x10(TKEYP)
|
||||||
add $0x20, TKEYP
|
add $0x20, TKEYP
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(_key_expansion_192a)
|
SYM_FUNC_END(_key_expansion_192a)
|
||||||
|
|
||||||
SYM_FUNC_START_LOCAL(_key_expansion_192b)
|
SYM_FUNC_START_LOCAL(_key_expansion_192b)
|
||||||
@@ -1810,7 +1810,7 @@ SYM_FUNC_START_LOCAL(_key_expansion_192b)
|
|||||||
|
|
||||||
movaps %xmm0, (TKEYP)
|
movaps %xmm0, (TKEYP)
|
||||||
add $0x10, TKEYP
|
add $0x10, TKEYP
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(_key_expansion_192b)
|
SYM_FUNC_END(_key_expansion_192b)
|
||||||
|
|
||||||
SYM_FUNC_START_LOCAL(_key_expansion_256b)
|
SYM_FUNC_START_LOCAL(_key_expansion_256b)
|
||||||
@@ -1822,7 +1822,7 @@ SYM_FUNC_START_LOCAL(_key_expansion_256b)
|
|||||||
pxor %xmm1, %xmm2
|
pxor %xmm1, %xmm2
|
||||||
movaps %xmm2, (TKEYP)
|
movaps %xmm2, (TKEYP)
|
||||||
add $0x10, TKEYP
|
add $0x10, TKEYP
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(_key_expansion_256b)
|
SYM_FUNC_END(_key_expansion_256b)
|
||||||
|
|
||||||
/*
|
/*
|
||||||
@@ -1937,7 +1937,7 @@ SYM_FUNC_START(aesni_set_key)
|
|||||||
popl KEYP
|
popl KEYP
|
||||||
#endif
|
#endif
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(aesni_set_key)
|
SYM_FUNC_END(aesni_set_key)
|
||||||
|
|
||||||
/*
|
/*
|
||||||
@@ -1961,7 +1961,7 @@ SYM_FUNC_START(aesni_enc)
|
|||||||
popl KEYP
|
popl KEYP
|
||||||
#endif
|
#endif
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(aesni_enc)
|
SYM_FUNC_END(aesni_enc)
|
||||||
|
|
||||||
/*
|
/*
|
||||||
@@ -2018,7 +2018,7 @@ SYM_FUNC_START_LOCAL(_aesni_enc1)
|
|||||||
aesenc KEY, STATE
|
aesenc KEY, STATE
|
||||||
movaps 0x70(TKEYP), KEY
|
movaps 0x70(TKEYP), KEY
|
||||||
aesenclast KEY, STATE
|
aesenclast KEY, STATE
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(_aesni_enc1)
|
SYM_FUNC_END(_aesni_enc1)
|
||||||
|
|
||||||
/*
|
/*
|
||||||
@@ -2126,7 +2126,7 @@ SYM_FUNC_START_LOCAL(_aesni_enc4)
|
|||||||
aesenclast KEY, STATE2
|
aesenclast KEY, STATE2
|
||||||
aesenclast KEY, STATE3
|
aesenclast KEY, STATE3
|
||||||
aesenclast KEY, STATE4
|
aesenclast KEY, STATE4
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(_aesni_enc4)
|
SYM_FUNC_END(_aesni_enc4)
|
||||||
|
|
||||||
/*
|
/*
|
||||||
@@ -2151,7 +2151,7 @@ SYM_FUNC_START(aesni_dec)
|
|||||||
popl KEYP
|
popl KEYP
|
||||||
#endif
|
#endif
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(aesni_dec)
|
SYM_FUNC_END(aesni_dec)
|
||||||
|
|
||||||
/*
|
/*
|
||||||
@@ -2208,7 +2208,7 @@ SYM_FUNC_START_LOCAL(_aesni_dec1)
|
|||||||
aesdec KEY, STATE
|
aesdec KEY, STATE
|
||||||
movaps 0x70(TKEYP), KEY
|
movaps 0x70(TKEYP), KEY
|
||||||
aesdeclast KEY, STATE
|
aesdeclast KEY, STATE
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(_aesni_dec1)
|
SYM_FUNC_END(_aesni_dec1)
|
||||||
|
|
||||||
/*
|
/*
|
||||||
@@ -2316,7 +2316,7 @@ SYM_FUNC_START_LOCAL(_aesni_dec4)
|
|||||||
aesdeclast KEY, STATE2
|
aesdeclast KEY, STATE2
|
||||||
aesdeclast KEY, STATE3
|
aesdeclast KEY, STATE3
|
||||||
aesdeclast KEY, STATE4
|
aesdeclast KEY, STATE4
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(_aesni_dec4)
|
SYM_FUNC_END(_aesni_dec4)
|
||||||
|
|
||||||
/*
|
/*
|
||||||
@@ -2376,7 +2376,7 @@ SYM_FUNC_START(aesni_ecb_enc)
|
|||||||
popl LEN
|
popl LEN
|
||||||
#endif
|
#endif
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(aesni_ecb_enc)
|
SYM_FUNC_END(aesni_ecb_enc)
|
||||||
|
|
||||||
/*
|
/*
|
||||||
@@ -2437,7 +2437,7 @@ SYM_FUNC_START(aesni_ecb_dec)
|
|||||||
popl LEN
|
popl LEN
|
||||||
#endif
|
#endif
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(aesni_ecb_dec)
|
SYM_FUNC_END(aesni_ecb_dec)
|
||||||
|
|
||||||
/*
|
/*
|
||||||
@@ -2481,7 +2481,7 @@ SYM_FUNC_START(aesni_cbc_enc)
|
|||||||
popl IVP
|
popl IVP
|
||||||
#endif
|
#endif
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(aesni_cbc_enc)
|
SYM_FUNC_END(aesni_cbc_enc)
|
||||||
|
|
||||||
/*
|
/*
|
||||||
@@ -2574,7 +2574,7 @@ SYM_FUNC_START(aesni_cbc_dec)
|
|||||||
popl IVP
|
popl IVP
|
||||||
#endif
|
#endif
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(aesni_cbc_dec)
|
SYM_FUNC_END(aesni_cbc_dec)
|
||||||
|
|
||||||
#ifdef __x86_64__
|
#ifdef __x86_64__
|
||||||
@@ -2602,7 +2602,7 @@ SYM_FUNC_START_LOCAL(_aesni_inc_init)
|
|||||||
mov $1, TCTR_LOW
|
mov $1, TCTR_LOW
|
||||||
movq TCTR_LOW, INC
|
movq TCTR_LOW, INC
|
||||||
movq CTR, TCTR_LOW
|
movq CTR, TCTR_LOW
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(_aesni_inc_init)
|
SYM_FUNC_END(_aesni_inc_init)
|
||||||
|
|
||||||
/*
|
/*
|
||||||
@@ -2630,7 +2630,7 @@ SYM_FUNC_START_LOCAL(_aesni_inc)
|
|||||||
.Linc_low:
|
.Linc_low:
|
||||||
movaps CTR, IV
|
movaps CTR, IV
|
||||||
pshufb BSWAP_MASK, IV
|
pshufb BSWAP_MASK, IV
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(_aesni_inc)
|
SYM_FUNC_END(_aesni_inc)
|
||||||
|
|
||||||
/*
|
/*
|
||||||
@@ -2693,7 +2693,7 @@ SYM_FUNC_START(aesni_ctr_enc)
|
|||||||
movups IV, (IVP)
|
movups IV, (IVP)
|
||||||
.Lctr_enc_just_ret:
|
.Lctr_enc_just_ret:
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(aesni_ctr_enc)
|
SYM_FUNC_END(aesni_ctr_enc)
|
||||||
|
|
||||||
/*
|
/*
|
||||||
@@ -2778,7 +2778,7 @@ SYM_FUNC_START(aesni_xts_encrypt)
|
|||||||
movups IV, (IVP)
|
movups IV, (IVP)
|
||||||
|
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(aesni_xts_encrypt)
|
SYM_FUNC_END(aesni_xts_encrypt)
|
||||||
|
|
||||||
/*
|
/*
|
||||||
@@ -2846,7 +2846,7 @@ SYM_FUNC_START(aesni_xts_decrypt)
|
|||||||
movups IV, (IVP)
|
movups IV, (IVP)
|
||||||
|
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(aesni_xts_decrypt)
|
SYM_FUNC_END(aesni_xts_decrypt)
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
@@ -1777,7 +1777,7 @@ SYM_FUNC_START(aesni_gcm_init_avx_gen2)
|
|||||||
FUNC_SAVE
|
FUNC_SAVE
|
||||||
INIT GHASH_MUL_AVX, PRECOMPUTE_AVX
|
INIT GHASH_MUL_AVX, PRECOMPUTE_AVX
|
||||||
FUNC_RESTORE
|
FUNC_RESTORE
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(aesni_gcm_init_avx_gen2)
|
SYM_FUNC_END(aesni_gcm_init_avx_gen2)
|
||||||
|
|
||||||
###############################################################################
|
###############################################################################
|
||||||
@@ -1798,15 +1798,15 @@ SYM_FUNC_START(aesni_gcm_enc_update_avx_gen2)
|
|||||||
# must be 192
|
# must be 192
|
||||||
GCM_ENC_DEC INITIAL_BLOCKS_AVX, GHASH_8_ENCRYPT_8_PARALLEL_AVX, GHASH_LAST_8_AVX, GHASH_MUL_AVX, ENC, 11
|
GCM_ENC_DEC INITIAL_BLOCKS_AVX, GHASH_8_ENCRYPT_8_PARALLEL_AVX, GHASH_LAST_8_AVX, GHASH_MUL_AVX, ENC, 11
|
||||||
FUNC_RESTORE
|
FUNC_RESTORE
|
||||||
ret
|
RET
|
||||||
key_128_enc_update:
|
key_128_enc_update:
|
||||||
GCM_ENC_DEC INITIAL_BLOCKS_AVX, GHASH_8_ENCRYPT_8_PARALLEL_AVX, GHASH_LAST_8_AVX, GHASH_MUL_AVX, ENC, 9
|
GCM_ENC_DEC INITIAL_BLOCKS_AVX, GHASH_8_ENCRYPT_8_PARALLEL_AVX, GHASH_LAST_8_AVX, GHASH_MUL_AVX, ENC, 9
|
||||||
FUNC_RESTORE
|
FUNC_RESTORE
|
||||||
ret
|
RET
|
||||||
key_256_enc_update:
|
key_256_enc_update:
|
||||||
GCM_ENC_DEC INITIAL_BLOCKS_AVX, GHASH_8_ENCRYPT_8_PARALLEL_AVX, GHASH_LAST_8_AVX, GHASH_MUL_AVX, ENC, 13
|
GCM_ENC_DEC INITIAL_BLOCKS_AVX, GHASH_8_ENCRYPT_8_PARALLEL_AVX, GHASH_LAST_8_AVX, GHASH_MUL_AVX, ENC, 13
|
||||||
FUNC_RESTORE
|
FUNC_RESTORE
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(aesni_gcm_enc_update_avx_gen2)
|
SYM_FUNC_END(aesni_gcm_enc_update_avx_gen2)
|
||||||
|
|
||||||
###############################################################################
|
###############################################################################
|
||||||
@@ -1827,15 +1827,15 @@ SYM_FUNC_START(aesni_gcm_dec_update_avx_gen2)
|
|||||||
# must be 192
|
# must be 192
|
||||||
GCM_ENC_DEC INITIAL_BLOCKS_AVX, GHASH_8_ENCRYPT_8_PARALLEL_AVX, GHASH_LAST_8_AVX, GHASH_MUL_AVX, DEC, 11
|
GCM_ENC_DEC INITIAL_BLOCKS_AVX, GHASH_8_ENCRYPT_8_PARALLEL_AVX, GHASH_LAST_8_AVX, GHASH_MUL_AVX, DEC, 11
|
||||||
FUNC_RESTORE
|
FUNC_RESTORE
|
||||||
ret
|
RET
|
||||||
key_128_dec_update:
|
key_128_dec_update:
|
||||||
GCM_ENC_DEC INITIAL_BLOCKS_AVX, GHASH_8_ENCRYPT_8_PARALLEL_AVX, GHASH_LAST_8_AVX, GHASH_MUL_AVX, DEC, 9
|
GCM_ENC_DEC INITIAL_BLOCKS_AVX, GHASH_8_ENCRYPT_8_PARALLEL_AVX, GHASH_LAST_8_AVX, GHASH_MUL_AVX, DEC, 9
|
||||||
FUNC_RESTORE
|
FUNC_RESTORE
|
||||||
ret
|
RET
|
||||||
key_256_dec_update:
|
key_256_dec_update:
|
||||||
GCM_ENC_DEC INITIAL_BLOCKS_AVX, GHASH_8_ENCRYPT_8_PARALLEL_AVX, GHASH_LAST_8_AVX, GHASH_MUL_AVX, DEC, 13
|
GCM_ENC_DEC INITIAL_BLOCKS_AVX, GHASH_8_ENCRYPT_8_PARALLEL_AVX, GHASH_LAST_8_AVX, GHASH_MUL_AVX, DEC, 13
|
||||||
FUNC_RESTORE
|
FUNC_RESTORE
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(aesni_gcm_dec_update_avx_gen2)
|
SYM_FUNC_END(aesni_gcm_dec_update_avx_gen2)
|
||||||
|
|
||||||
###############################################################################
|
###############################################################################
|
||||||
@@ -1856,15 +1856,15 @@ SYM_FUNC_START(aesni_gcm_finalize_avx_gen2)
|
|||||||
# must be 192
|
# must be 192
|
||||||
GCM_COMPLETE GHASH_MUL_AVX, 11, arg3, arg4
|
GCM_COMPLETE GHASH_MUL_AVX, 11, arg3, arg4
|
||||||
FUNC_RESTORE
|
FUNC_RESTORE
|
||||||
ret
|
RET
|
||||||
key_128_finalize:
|
key_128_finalize:
|
||||||
GCM_COMPLETE GHASH_MUL_AVX, 9, arg3, arg4
|
GCM_COMPLETE GHASH_MUL_AVX, 9, arg3, arg4
|
||||||
FUNC_RESTORE
|
FUNC_RESTORE
|
||||||
ret
|
RET
|
||||||
key_256_finalize:
|
key_256_finalize:
|
||||||
GCM_COMPLETE GHASH_MUL_AVX, 13, arg3, arg4
|
GCM_COMPLETE GHASH_MUL_AVX, 13, arg3, arg4
|
||||||
FUNC_RESTORE
|
FUNC_RESTORE
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(aesni_gcm_finalize_avx_gen2)
|
SYM_FUNC_END(aesni_gcm_finalize_avx_gen2)
|
||||||
|
|
||||||
###############################################################################
|
###############################################################################
|
||||||
@@ -2745,7 +2745,7 @@ SYM_FUNC_START(aesni_gcm_init_avx_gen4)
|
|||||||
FUNC_SAVE
|
FUNC_SAVE
|
||||||
INIT GHASH_MUL_AVX2, PRECOMPUTE_AVX2
|
INIT GHASH_MUL_AVX2, PRECOMPUTE_AVX2
|
||||||
FUNC_RESTORE
|
FUNC_RESTORE
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(aesni_gcm_init_avx_gen4)
|
SYM_FUNC_END(aesni_gcm_init_avx_gen4)
|
||||||
|
|
||||||
###############################################################################
|
###############################################################################
|
||||||
@@ -2766,15 +2766,15 @@ SYM_FUNC_START(aesni_gcm_enc_update_avx_gen4)
|
|||||||
# must be 192
|
# must be 192
|
||||||
GCM_ENC_DEC INITIAL_BLOCKS_AVX2, GHASH_8_ENCRYPT_8_PARALLEL_AVX2, GHASH_LAST_8_AVX2, GHASH_MUL_AVX2, ENC, 11
|
GCM_ENC_DEC INITIAL_BLOCKS_AVX2, GHASH_8_ENCRYPT_8_PARALLEL_AVX2, GHASH_LAST_8_AVX2, GHASH_MUL_AVX2, ENC, 11
|
||||||
FUNC_RESTORE
|
FUNC_RESTORE
|
||||||
ret
|
RET
|
||||||
key_128_enc_update4:
|
key_128_enc_update4:
|
||||||
GCM_ENC_DEC INITIAL_BLOCKS_AVX2, GHASH_8_ENCRYPT_8_PARALLEL_AVX2, GHASH_LAST_8_AVX2, GHASH_MUL_AVX2, ENC, 9
|
GCM_ENC_DEC INITIAL_BLOCKS_AVX2, GHASH_8_ENCRYPT_8_PARALLEL_AVX2, GHASH_LAST_8_AVX2, GHASH_MUL_AVX2, ENC, 9
|
||||||
FUNC_RESTORE
|
FUNC_RESTORE
|
||||||
ret
|
RET
|
||||||
key_256_enc_update4:
|
key_256_enc_update4:
|
||||||
GCM_ENC_DEC INITIAL_BLOCKS_AVX2, GHASH_8_ENCRYPT_8_PARALLEL_AVX2, GHASH_LAST_8_AVX2, GHASH_MUL_AVX2, ENC, 13
|
GCM_ENC_DEC INITIAL_BLOCKS_AVX2, GHASH_8_ENCRYPT_8_PARALLEL_AVX2, GHASH_LAST_8_AVX2, GHASH_MUL_AVX2, ENC, 13
|
||||||
FUNC_RESTORE
|
FUNC_RESTORE
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(aesni_gcm_enc_update_avx_gen4)
|
SYM_FUNC_END(aesni_gcm_enc_update_avx_gen4)
|
||||||
|
|
||||||
###############################################################################
|
###############################################################################
|
||||||
@@ -2795,15 +2795,15 @@ SYM_FUNC_START(aesni_gcm_dec_update_avx_gen4)
|
|||||||
# must be 192
|
# must be 192
|
||||||
GCM_ENC_DEC INITIAL_BLOCKS_AVX2, GHASH_8_ENCRYPT_8_PARALLEL_AVX2, GHASH_LAST_8_AVX2, GHASH_MUL_AVX2, DEC, 11
|
GCM_ENC_DEC INITIAL_BLOCKS_AVX2, GHASH_8_ENCRYPT_8_PARALLEL_AVX2, GHASH_LAST_8_AVX2, GHASH_MUL_AVX2, DEC, 11
|
||||||
FUNC_RESTORE
|
FUNC_RESTORE
|
||||||
ret
|
RET
|
||||||
key_128_dec_update4:
|
key_128_dec_update4:
|
||||||
GCM_ENC_DEC INITIAL_BLOCKS_AVX2, GHASH_8_ENCRYPT_8_PARALLEL_AVX2, GHASH_LAST_8_AVX2, GHASH_MUL_AVX2, DEC, 9
|
GCM_ENC_DEC INITIAL_BLOCKS_AVX2, GHASH_8_ENCRYPT_8_PARALLEL_AVX2, GHASH_LAST_8_AVX2, GHASH_MUL_AVX2, DEC, 9
|
||||||
FUNC_RESTORE
|
FUNC_RESTORE
|
||||||
ret
|
RET
|
||||||
key_256_dec_update4:
|
key_256_dec_update4:
|
||||||
GCM_ENC_DEC INITIAL_BLOCKS_AVX2, GHASH_8_ENCRYPT_8_PARALLEL_AVX2, GHASH_LAST_8_AVX2, GHASH_MUL_AVX2, DEC, 13
|
GCM_ENC_DEC INITIAL_BLOCKS_AVX2, GHASH_8_ENCRYPT_8_PARALLEL_AVX2, GHASH_LAST_8_AVX2, GHASH_MUL_AVX2, DEC, 13
|
||||||
FUNC_RESTORE
|
FUNC_RESTORE
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(aesni_gcm_dec_update_avx_gen4)
|
SYM_FUNC_END(aesni_gcm_dec_update_avx_gen4)
|
||||||
|
|
||||||
###############################################################################
|
###############################################################################
|
||||||
@@ -2824,13 +2824,13 @@ SYM_FUNC_START(aesni_gcm_finalize_avx_gen4)
|
|||||||
# must be 192
|
# must be 192
|
||||||
GCM_COMPLETE GHASH_MUL_AVX2, 11, arg3, arg4
|
GCM_COMPLETE GHASH_MUL_AVX2, 11, arg3, arg4
|
||||||
FUNC_RESTORE
|
FUNC_RESTORE
|
||||||
ret
|
RET
|
||||||
key_128_finalize4:
|
key_128_finalize4:
|
||||||
GCM_COMPLETE GHASH_MUL_AVX2, 9, arg3, arg4
|
GCM_COMPLETE GHASH_MUL_AVX2, 9, arg3, arg4
|
||||||
FUNC_RESTORE
|
FUNC_RESTORE
|
||||||
ret
|
RET
|
||||||
key_256_finalize4:
|
key_256_finalize4:
|
||||||
GCM_COMPLETE GHASH_MUL_AVX2, 13, arg3, arg4
|
GCM_COMPLETE GHASH_MUL_AVX2, 13, arg3, arg4
|
||||||
FUNC_RESTORE
|
FUNC_RESTORE
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(aesni_gcm_finalize_avx_gen4)
|
SYM_FUNC_END(aesni_gcm_finalize_avx_gen4)
|
||||||
|
@@ -171,7 +171,7 @@ SYM_FUNC_START(blake2s_compress_ssse3)
|
|||||||
movdqu %xmm1,0x10(%rdi)
|
movdqu %xmm1,0x10(%rdi)
|
||||||
movdqu %xmm14,0x20(%rdi)
|
movdqu %xmm14,0x20(%rdi)
|
||||||
.Lendofloop:
|
.Lendofloop:
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(blake2s_compress_ssse3)
|
SYM_FUNC_END(blake2s_compress_ssse3)
|
||||||
|
|
||||||
#ifdef CONFIG_AS_AVX512
|
#ifdef CONFIG_AS_AVX512
|
||||||
@@ -251,6 +251,6 @@ SYM_FUNC_START(blake2s_compress_avx512)
|
|||||||
vmovdqu %xmm1,0x10(%rdi)
|
vmovdqu %xmm1,0x10(%rdi)
|
||||||
vmovdqu %xmm4,0x20(%rdi)
|
vmovdqu %xmm4,0x20(%rdi)
|
||||||
vzeroupper
|
vzeroupper
|
||||||
retq
|
RET
|
||||||
SYM_FUNC_END(blake2s_compress_avx512)
|
SYM_FUNC_END(blake2s_compress_avx512)
|
||||||
#endif /* CONFIG_AS_AVX512 */
|
#endif /* CONFIG_AS_AVX512 */
|
||||||
|
@@ -135,10 +135,10 @@ SYM_FUNC_START(__blowfish_enc_blk)
|
|||||||
jnz .L__enc_xor;
|
jnz .L__enc_xor;
|
||||||
|
|
||||||
write_block();
|
write_block();
|
||||||
ret;
|
RET;
|
||||||
.L__enc_xor:
|
.L__enc_xor:
|
||||||
xor_block();
|
xor_block();
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(__blowfish_enc_blk)
|
SYM_FUNC_END(__blowfish_enc_blk)
|
||||||
|
|
||||||
SYM_FUNC_START(blowfish_dec_blk)
|
SYM_FUNC_START(blowfish_dec_blk)
|
||||||
@@ -170,7 +170,7 @@ SYM_FUNC_START(blowfish_dec_blk)
|
|||||||
|
|
||||||
movq %r11, %r12;
|
movq %r11, %r12;
|
||||||
|
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(blowfish_dec_blk)
|
SYM_FUNC_END(blowfish_dec_blk)
|
||||||
|
|
||||||
/**********************************************************************
|
/**********************************************************************
|
||||||
@@ -322,14 +322,14 @@ SYM_FUNC_START(__blowfish_enc_blk_4way)
|
|||||||
|
|
||||||
popq %rbx;
|
popq %rbx;
|
||||||
popq %r12;
|
popq %r12;
|
||||||
ret;
|
RET;
|
||||||
|
|
||||||
.L__enc_xor4:
|
.L__enc_xor4:
|
||||||
xor_block4();
|
xor_block4();
|
||||||
|
|
||||||
popq %rbx;
|
popq %rbx;
|
||||||
popq %r12;
|
popq %r12;
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(__blowfish_enc_blk_4way)
|
SYM_FUNC_END(__blowfish_enc_blk_4way)
|
||||||
|
|
||||||
SYM_FUNC_START(blowfish_dec_blk_4way)
|
SYM_FUNC_START(blowfish_dec_blk_4way)
|
||||||
@@ -364,5 +364,5 @@ SYM_FUNC_START(blowfish_dec_blk_4way)
|
|||||||
popq %rbx;
|
popq %rbx;
|
||||||
popq %r12;
|
popq %r12;
|
||||||
|
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(blowfish_dec_blk_4way)
|
SYM_FUNC_END(blowfish_dec_blk_4way)
|
||||||
|
@@ -193,7 +193,7 @@ SYM_FUNC_START_LOCAL(roundsm16_x0_x1_x2_x3_x4_x5_x6_x7_y0_y1_y2_y3_y4_y5_y6_y7_c
|
|||||||
roundsm16(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
|
roundsm16(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
|
||||||
%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, %xmm15,
|
%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, %xmm15,
|
||||||
%rcx, (%r9));
|
%rcx, (%r9));
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(roundsm16_x0_x1_x2_x3_x4_x5_x6_x7_y0_y1_y2_y3_y4_y5_y6_y7_cd)
|
SYM_FUNC_END(roundsm16_x0_x1_x2_x3_x4_x5_x6_x7_y0_y1_y2_y3_y4_y5_y6_y7_cd)
|
||||||
|
|
||||||
.align 8
|
.align 8
|
||||||
@@ -201,7 +201,7 @@ SYM_FUNC_START_LOCAL(roundsm16_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_a
|
|||||||
roundsm16(%xmm4, %xmm5, %xmm6, %xmm7, %xmm0, %xmm1, %xmm2, %xmm3,
|
roundsm16(%xmm4, %xmm5, %xmm6, %xmm7, %xmm0, %xmm1, %xmm2, %xmm3,
|
||||||
%xmm12, %xmm13, %xmm14, %xmm15, %xmm8, %xmm9, %xmm10, %xmm11,
|
%xmm12, %xmm13, %xmm14, %xmm15, %xmm8, %xmm9, %xmm10, %xmm11,
|
||||||
%rax, (%r9));
|
%rax, (%r9));
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(roundsm16_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab)
|
SYM_FUNC_END(roundsm16_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab)
|
||||||
|
|
||||||
/*
|
/*
|
||||||
@@ -787,7 +787,7 @@ SYM_FUNC_START_LOCAL(__camellia_enc_blk16)
|
|||||||
%xmm15, (key_table)(CTX, %r8, 8), (%rax), 1 * 16(%rax));
|
%xmm15, (key_table)(CTX, %r8, 8), (%rax), 1 * 16(%rax));
|
||||||
|
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret;
|
RET;
|
||||||
|
|
||||||
.align 8
|
.align 8
|
||||||
.Lenc_max32:
|
.Lenc_max32:
|
||||||
@@ -874,7 +874,7 @@ SYM_FUNC_START_LOCAL(__camellia_dec_blk16)
|
|||||||
%xmm15, (key_table)(CTX), (%rax), 1 * 16(%rax));
|
%xmm15, (key_table)(CTX), (%rax), 1 * 16(%rax));
|
||||||
|
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret;
|
RET;
|
||||||
|
|
||||||
.align 8
|
.align 8
|
||||||
.Ldec_max32:
|
.Ldec_max32:
|
||||||
@@ -915,7 +915,7 @@ SYM_FUNC_START(camellia_ecb_enc_16way)
|
|||||||
%xmm8, %rsi);
|
%xmm8, %rsi);
|
||||||
|
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(camellia_ecb_enc_16way)
|
SYM_FUNC_END(camellia_ecb_enc_16way)
|
||||||
|
|
||||||
SYM_FUNC_START(camellia_ecb_dec_16way)
|
SYM_FUNC_START(camellia_ecb_dec_16way)
|
||||||
@@ -945,7 +945,7 @@ SYM_FUNC_START(camellia_ecb_dec_16way)
|
|||||||
%xmm8, %rsi);
|
%xmm8, %rsi);
|
||||||
|
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(camellia_ecb_dec_16way)
|
SYM_FUNC_END(camellia_ecb_dec_16way)
|
||||||
|
|
||||||
SYM_FUNC_START(camellia_cbc_dec_16way)
|
SYM_FUNC_START(camellia_cbc_dec_16way)
|
||||||
@@ -996,7 +996,7 @@ SYM_FUNC_START(camellia_cbc_dec_16way)
|
|||||||
%xmm8, %rsi);
|
%xmm8, %rsi);
|
||||||
|
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(camellia_cbc_dec_16way)
|
SYM_FUNC_END(camellia_cbc_dec_16way)
|
||||||
|
|
||||||
#define inc_le128(x, minus_one, tmp) \
|
#define inc_le128(x, minus_one, tmp) \
|
||||||
@@ -1109,7 +1109,7 @@ SYM_FUNC_START(camellia_ctr_16way)
|
|||||||
%xmm8, %rsi);
|
%xmm8, %rsi);
|
||||||
|
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(camellia_ctr_16way)
|
SYM_FUNC_END(camellia_ctr_16way)
|
||||||
|
|
||||||
#define gf128mul_x_ble(iv, mask, tmp) \
|
#define gf128mul_x_ble(iv, mask, tmp) \
|
||||||
@@ -1253,7 +1253,7 @@ SYM_FUNC_START_LOCAL(camellia_xts_crypt_16way)
|
|||||||
%xmm8, %rsi);
|
%xmm8, %rsi);
|
||||||
|
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(camellia_xts_crypt_16way)
|
SYM_FUNC_END(camellia_xts_crypt_16way)
|
||||||
|
|
||||||
SYM_FUNC_START(camellia_xts_enc_16way)
|
SYM_FUNC_START(camellia_xts_enc_16way)
|
||||||
|
@@ -227,7 +227,7 @@ SYM_FUNC_START_LOCAL(roundsm32_x0_x1_x2_x3_x4_x5_x6_x7_y0_y1_y2_y3_y4_y5_y6_y7_c
|
|||||||
roundsm32(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7,
|
roundsm32(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7,
|
||||||
%ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, %ymm15,
|
%ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, %ymm15,
|
||||||
%rcx, (%r9));
|
%rcx, (%r9));
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(roundsm32_x0_x1_x2_x3_x4_x5_x6_x7_y0_y1_y2_y3_y4_y5_y6_y7_cd)
|
SYM_FUNC_END(roundsm32_x0_x1_x2_x3_x4_x5_x6_x7_y0_y1_y2_y3_y4_y5_y6_y7_cd)
|
||||||
|
|
||||||
.align 8
|
.align 8
|
||||||
@@ -235,7 +235,7 @@ SYM_FUNC_START_LOCAL(roundsm32_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_a
|
|||||||
roundsm32(%ymm4, %ymm5, %ymm6, %ymm7, %ymm0, %ymm1, %ymm2, %ymm3,
|
roundsm32(%ymm4, %ymm5, %ymm6, %ymm7, %ymm0, %ymm1, %ymm2, %ymm3,
|
||||||
%ymm12, %ymm13, %ymm14, %ymm15, %ymm8, %ymm9, %ymm10, %ymm11,
|
%ymm12, %ymm13, %ymm14, %ymm15, %ymm8, %ymm9, %ymm10, %ymm11,
|
||||||
%rax, (%r9));
|
%rax, (%r9));
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(roundsm32_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab)
|
SYM_FUNC_END(roundsm32_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab)
|
||||||
|
|
||||||
/*
|
/*
|
||||||
@@ -825,7 +825,7 @@ SYM_FUNC_START_LOCAL(__camellia_enc_blk32)
|
|||||||
%ymm15, (key_table)(CTX, %r8, 8), (%rax), 1 * 32(%rax));
|
%ymm15, (key_table)(CTX, %r8, 8), (%rax), 1 * 32(%rax));
|
||||||
|
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret;
|
RET;
|
||||||
|
|
||||||
.align 8
|
.align 8
|
||||||
.Lenc_max32:
|
.Lenc_max32:
|
||||||
@@ -912,7 +912,7 @@ SYM_FUNC_START_LOCAL(__camellia_dec_blk32)
|
|||||||
%ymm15, (key_table)(CTX), (%rax), 1 * 32(%rax));
|
%ymm15, (key_table)(CTX), (%rax), 1 * 32(%rax));
|
||||||
|
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret;
|
RET;
|
||||||
|
|
||||||
.align 8
|
.align 8
|
||||||
.Ldec_max32:
|
.Ldec_max32:
|
||||||
@@ -957,7 +957,7 @@ SYM_FUNC_START(camellia_ecb_enc_32way)
|
|||||||
vzeroupper;
|
vzeroupper;
|
||||||
|
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(camellia_ecb_enc_32way)
|
SYM_FUNC_END(camellia_ecb_enc_32way)
|
||||||
|
|
||||||
SYM_FUNC_START(camellia_ecb_dec_32way)
|
SYM_FUNC_START(camellia_ecb_dec_32way)
|
||||||
@@ -991,7 +991,7 @@ SYM_FUNC_START(camellia_ecb_dec_32way)
|
|||||||
vzeroupper;
|
vzeroupper;
|
||||||
|
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(camellia_ecb_dec_32way)
|
SYM_FUNC_END(camellia_ecb_dec_32way)
|
||||||
|
|
||||||
SYM_FUNC_START(camellia_cbc_dec_32way)
|
SYM_FUNC_START(camellia_cbc_dec_32way)
|
||||||
@@ -1059,7 +1059,7 @@ SYM_FUNC_START(camellia_cbc_dec_32way)
|
|||||||
vzeroupper;
|
vzeroupper;
|
||||||
|
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(camellia_cbc_dec_32way)
|
SYM_FUNC_END(camellia_cbc_dec_32way)
|
||||||
|
|
||||||
#define inc_le128(x, minus_one, tmp) \
|
#define inc_le128(x, minus_one, tmp) \
|
||||||
@@ -1199,7 +1199,7 @@ SYM_FUNC_START(camellia_ctr_32way)
|
|||||||
vzeroupper;
|
vzeroupper;
|
||||||
|
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(camellia_ctr_32way)
|
SYM_FUNC_END(camellia_ctr_32way)
|
||||||
|
|
||||||
#define gf128mul_x_ble(iv, mask, tmp) \
|
#define gf128mul_x_ble(iv, mask, tmp) \
|
||||||
@@ -1366,7 +1366,7 @@ SYM_FUNC_START_LOCAL(camellia_xts_crypt_32way)
|
|||||||
vzeroupper;
|
vzeroupper;
|
||||||
|
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(camellia_xts_crypt_32way)
|
SYM_FUNC_END(camellia_xts_crypt_32way)
|
||||||
|
|
||||||
SYM_FUNC_START(camellia_xts_enc_32way)
|
SYM_FUNC_START(camellia_xts_enc_32way)
|
||||||
|
@@ -213,13 +213,13 @@ SYM_FUNC_START(__camellia_enc_blk)
|
|||||||
enc_outunpack(mov, RT1);
|
enc_outunpack(mov, RT1);
|
||||||
|
|
||||||
movq RR12, %r12;
|
movq RR12, %r12;
|
||||||
ret;
|
RET;
|
||||||
|
|
||||||
.L__enc_xor:
|
.L__enc_xor:
|
||||||
enc_outunpack(xor, RT1);
|
enc_outunpack(xor, RT1);
|
||||||
|
|
||||||
movq RR12, %r12;
|
movq RR12, %r12;
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(__camellia_enc_blk)
|
SYM_FUNC_END(__camellia_enc_blk)
|
||||||
|
|
||||||
SYM_FUNC_START(camellia_dec_blk)
|
SYM_FUNC_START(camellia_dec_blk)
|
||||||
@@ -257,7 +257,7 @@ SYM_FUNC_START(camellia_dec_blk)
|
|||||||
dec_outunpack();
|
dec_outunpack();
|
||||||
|
|
||||||
movq RR12, %r12;
|
movq RR12, %r12;
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(camellia_dec_blk)
|
SYM_FUNC_END(camellia_dec_blk)
|
||||||
|
|
||||||
/**********************************************************************
|
/**********************************************************************
|
||||||
@@ -448,14 +448,14 @@ SYM_FUNC_START(__camellia_enc_blk_2way)
|
|||||||
|
|
||||||
movq RR12, %r12;
|
movq RR12, %r12;
|
||||||
popq %rbx;
|
popq %rbx;
|
||||||
ret;
|
RET;
|
||||||
|
|
||||||
.L__enc2_xor:
|
.L__enc2_xor:
|
||||||
enc_outunpack2(xor, RT2);
|
enc_outunpack2(xor, RT2);
|
||||||
|
|
||||||
movq RR12, %r12;
|
movq RR12, %r12;
|
||||||
popq %rbx;
|
popq %rbx;
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(__camellia_enc_blk_2way)
|
SYM_FUNC_END(__camellia_enc_blk_2way)
|
||||||
|
|
||||||
SYM_FUNC_START(camellia_dec_blk_2way)
|
SYM_FUNC_START(camellia_dec_blk_2way)
|
||||||
@@ -495,5 +495,5 @@ SYM_FUNC_START(camellia_dec_blk_2way)
|
|||||||
|
|
||||||
movq RR12, %r12;
|
movq RR12, %r12;
|
||||||
movq RXOR, %rbx;
|
movq RXOR, %rbx;
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(camellia_dec_blk_2way)
|
SYM_FUNC_END(camellia_dec_blk_2way)
|
||||||
|
@@ -279,7 +279,7 @@ SYM_FUNC_START_LOCAL(__cast5_enc_blk16)
|
|||||||
outunpack_blocks(RR3, RL3, RTMP, RX, RKM);
|
outunpack_blocks(RR3, RL3, RTMP, RX, RKM);
|
||||||
outunpack_blocks(RR4, RL4, RTMP, RX, RKM);
|
outunpack_blocks(RR4, RL4, RTMP, RX, RKM);
|
||||||
|
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(__cast5_enc_blk16)
|
SYM_FUNC_END(__cast5_enc_blk16)
|
||||||
|
|
||||||
.align 16
|
.align 16
|
||||||
@@ -352,7 +352,7 @@ SYM_FUNC_START_LOCAL(__cast5_dec_blk16)
|
|||||||
outunpack_blocks(RR3, RL3, RTMP, RX, RKM);
|
outunpack_blocks(RR3, RL3, RTMP, RX, RKM);
|
||||||
outunpack_blocks(RR4, RL4, RTMP, RX, RKM);
|
outunpack_blocks(RR4, RL4, RTMP, RX, RKM);
|
||||||
|
|
||||||
ret;
|
RET;
|
||||||
|
|
||||||
.L__skip_dec:
|
.L__skip_dec:
|
||||||
vpsrldq $4, RKR, RKR;
|
vpsrldq $4, RKR, RKR;
|
||||||
@@ -393,7 +393,7 @@ SYM_FUNC_START(cast5_ecb_enc_16way)
|
|||||||
|
|
||||||
popq %r15;
|
popq %r15;
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(cast5_ecb_enc_16way)
|
SYM_FUNC_END(cast5_ecb_enc_16way)
|
||||||
|
|
||||||
SYM_FUNC_START(cast5_ecb_dec_16way)
|
SYM_FUNC_START(cast5_ecb_dec_16way)
|
||||||
@@ -431,7 +431,7 @@ SYM_FUNC_START(cast5_ecb_dec_16way)
|
|||||||
|
|
||||||
popq %r15;
|
popq %r15;
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(cast5_ecb_dec_16way)
|
SYM_FUNC_END(cast5_ecb_dec_16way)
|
||||||
|
|
||||||
SYM_FUNC_START(cast5_cbc_dec_16way)
|
SYM_FUNC_START(cast5_cbc_dec_16way)
|
||||||
@@ -483,7 +483,7 @@ SYM_FUNC_START(cast5_cbc_dec_16way)
|
|||||||
popq %r15;
|
popq %r15;
|
||||||
popq %r12;
|
popq %r12;
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(cast5_cbc_dec_16way)
|
SYM_FUNC_END(cast5_cbc_dec_16way)
|
||||||
|
|
||||||
SYM_FUNC_START(cast5_ctr_16way)
|
SYM_FUNC_START(cast5_ctr_16way)
|
||||||
@@ -559,5 +559,5 @@ SYM_FUNC_START(cast5_ctr_16way)
|
|||||||
popq %r15;
|
popq %r15;
|
||||||
popq %r12;
|
popq %r12;
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(cast5_ctr_16way)
|
SYM_FUNC_END(cast5_ctr_16way)
|
||||||
|
@@ -291,7 +291,7 @@ SYM_FUNC_START_LOCAL(__cast6_enc_blk8)
|
|||||||
outunpack_blocks(RA1, RB1, RC1, RD1, RTMP, RX, RKRF, RKM);
|
outunpack_blocks(RA1, RB1, RC1, RD1, RTMP, RX, RKRF, RKM);
|
||||||
outunpack_blocks(RA2, RB2, RC2, RD2, RTMP, RX, RKRF, RKM);
|
outunpack_blocks(RA2, RB2, RC2, RD2, RTMP, RX, RKRF, RKM);
|
||||||
|
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(__cast6_enc_blk8)
|
SYM_FUNC_END(__cast6_enc_blk8)
|
||||||
|
|
||||||
.align 8
|
.align 8
|
||||||
@@ -338,7 +338,7 @@ SYM_FUNC_START_LOCAL(__cast6_dec_blk8)
|
|||||||
outunpack_blocks(RA1, RB1, RC1, RD1, RTMP, RX, RKRF, RKM);
|
outunpack_blocks(RA1, RB1, RC1, RD1, RTMP, RX, RKRF, RKM);
|
||||||
outunpack_blocks(RA2, RB2, RC2, RD2, RTMP, RX, RKRF, RKM);
|
outunpack_blocks(RA2, RB2, RC2, RD2, RTMP, RX, RKRF, RKM);
|
||||||
|
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(__cast6_dec_blk8)
|
SYM_FUNC_END(__cast6_dec_blk8)
|
||||||
|
|
||||||
SYM_FUNC_START(cast6_ecb_enc_8way)
|
SYM_FUNC_START(cast6_ecb_enc_8way)
|
||||||
@@ -361,7 +361,7 @@ SYM_FUNC_START(cast6_ecb_enc_8way)
|
|||||||
|
|
||||||
popq %r15;
|
popq %r15;
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(cast6_ecb_enc_8way)
|
SYM_FUNC_END(cast6_ecb_enc_8way)
|
||||||
|
|
||||||
SYM_FUNC_START(cast6_ecb_dec_8way)
|
SYM_FUNC_START(cast6_ecb_dec_8way)
|
||||||
@@ -384,7 +384,7 @@ SYM_FUNC_START(cast6_ecb_dec_8way)
|
|||||||
|
|
||||||
popq %r15;
|
popq %r15;
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(cast6_ecb_dec_8way)
|
SYM_FUNC_END(cast6_ecb_dec_8way)
|
||||||
|
|
||||||
SYM_FUNC_START(cast6_cbc_dec_8way)
|
SYM_FUNC_START(cast6_cbc_dec_8way)
|
||||||
@@ -410,7 +410,7 @@ SYM_FUNC_START(cast6_cbc_dec_8way)
|
|||||||
popq %r15;
|
popq %r15;
|
||||||
popq %r12;
|
popq %r12;
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(cast6_cbc_dec_8way)
|
SYM_FUNC_END(cast6_cbc_dec_8way)
|
||||||
|
|
||||||
SYM_FUNC_START(cast6_ctr_8way)
|
SYM_FUNC_START(cast6_ctr_8way)
|
||||||
@@ -438,7 +438,7 @@ SYM_FUNC_START(cast6_ctr_8way)
|
|||||||
popq %r15;
|
popq %r15;
|
||||||
popq %r12;
|
popq %r12;
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(cast6_ctr_8way)
|
SYM_FUNC_END(cast6_ctr_8way)
|
||||||
|
|
||||||
SYM_FUNC_START(cast6_xts_enc_8way)
|
SYM_FUNC_START(cast6_xts_enc_8way)
|
||||||
@@ -465,7 +465,7 @@ SYM_FUNC_START(cast6_xts_enc_8way)
|
|||||||
|
|
||||||
popq %r15;
|
popq %r15;
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(cast6_xts_enc_8way)
|
SYM_FUNC_END(cast6_xts_enc_8way)
|
||||||
|
|
||||||
SYM_FUNC_START(cast6_xts_dec_8way)
|
SYM_FUNC_START(cast6_xts_dec_8way)
|
||||||
@@ -492,5 +492,5 @@ SYM_FUNC_START(cast6_xts_dec_8way)
|
|||||||
|
|
||||||
popq %r15;
|
popq %r15;
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(cast6_xts_dec_8way)
|
SYM_FUNC_END(cast6_xts_dec_8way)
|
||||||
|
@@ -193,7 +193,7 @@ SYM_FUNC_START(chacha_2block_xor_avx2)
|
|||||||
|
|
||||||
.Ldone2:
|
.Ldone2:
|
||||||
vzeroupper
|
vzeroupper
|
||||||
ret
|
RET
|
||||||
|
|
||||||
.Lxorpart2:
|
.Lxorpart2:
|
||||||
# xor remaining bytes from partial register into output
|
# xor remaining bytes from partial register into output
|
||||||
@@ -498,7 +498,7 @@ SYM_FUNC_START(chacha_4block_xor_avx2)
|
|||||||
|
|
||||||
.Ldone4:
|
.Ldone4:
|
||||||
vzeroupper
|
vzeroupper
|
||||||
ret
|
RET
|
||||||
|
|
||||||
.Lxorpart4:
|
.Lxorpart4:
|
||||||
# xor remaining bytes from partial register into output
|
# xor remaining bytes from partial register into output
|
||||||
@@ -992,7 +992,7 @@ SYM_FUNC_START(chacha_8block_xor_avx2)
|
|||||||
.Ldone8:
|
.Ldone8:
|
||||||
vzeroupper
|
vzeroupper
|
||||||
lea -8(%r10),%rsp
|
lea -8(%r10),%rsp
|
||||||
ret
|
RET
|
||||||
|
|
||||||
.Lxorpart8:
|
.Lxorpart8:
|
||||||
# xor remaining bytes from partial register into output
|
# xor remaining bytes from partial register into output
|
||||||
|
@@ -166,7 +166,7 @@ SYM_FUNC_START(chacha_2block_xor_avx512vl)
|
|||||||
|
|
||||||
.Ldone2:
|
.Ldone2:
|
||||||
vzeroupper
|
vzeroupper
|
||||||
ret
|
RET
|
||||||
|
|
||||||
.Lxorpart2:
|
.Lxorpart2:
|
||||||
# xor remaining bytes from partial register into output
|
# xor remaining bytes from partial register into output
|
||||||
@@ -432,7 +432,7 @@ SYM_FUNC_START(chacha_4block_xor_avx512vl)
|
|||||||
|
|
||||||
.Ldone4:
|
.Ldone4:
|
||||||
vzeroupper
|
vzeroupper
|
||||||
ret
|
RET
|
||||||
|
|
||||||
.Lxorpart4:
|
.Lxorpart4:
|
||||||
# xor remaining bytes from partial register into output
|
# xor remaining bytes from partial register into output
|
||||||
@@ -812,7 +812,7 @@ SYM_FUNC_START(chacha_8block_xor_avx512vl)
|
|||||||
|
|
||||||
.Ldone8:
|
.Ldone8:
|
||||||
vzeroupper
|
vzeroupper
|
||||||
ret
|
RET
|
||||||
|
|
||||||
.Lxorpart8:
|
.Lxorpart8:
|
||||||
# xor remaining bytes from partial register into output
|
# xor remaining bytes from partial register into output
|
||||||
|
@@ -108,7 +108,7 @@ SYM_FUNC_START_LOCAL(chacha_permute)
|
|||||||
sub $2,%r8d
|
sub $2,%r8d
|
||||||
jnz .Ldoubleround
|
jnz .Ldoubleround
|
||||||
|
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(chacha_permute)
|
SYM_FUNC_END(chacha_permute)
|
||||||
|
|
||||||
SYM_FUNC_START(chacha_block_xor_ssse3)
|
SYM_FUNC_START(chacha_block_xor_ssse3)
|
||||||
@@ -166,7 +166,7 @@ SYM_FUNC_START(chacha_block_xor_ssse3)
|
|||||||
|
|
||||||
.Ldone:
|
.Ldone:
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret
|
RET
|
||||||
|
|
||||||
.Lxorpart:
|
.Lxorpart:
|
||||||
# xor remaining bytes from partial register into output
|
# xor remaining bytes from partial register into output
|
||||||
@@ -217,7 +217,7 @@ SYM_FUNC_START(hchacha_block_ssse3)
|
|||||||
movdqu %xmm3,0x10(%rsi)
|
movdqu %xmm3,0x10(%rsi)
|
||||||
|
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(hchacha_block_ssse3)
|
SYM_FUNC_END(hchacha_block_ssse3)
|
||||||
|
|
||||||
SYM_FUNC_START(chacha_4block_xor_ssse3)
|
SYM_FUNC_START(chacha_4block_xor_ssse3)
|
||||||
@@ -762,7 +762,7 @@ SYM_FUNC_START(chacha_4block_xor_ssse3)
|
|||||||
|
|
||||||
.Ldone4:
|
.Ldone4:
|
||||||
lea -8(%r10),%rsp
|
lea -8(%r10),%rsp
|
||||||
ret
|
RET
|
||||||
|
|
||||||
.Lxorpart4:
|
.Lxorpart4:
|
||||||
# xor remaining bytes from partial register into output
|
# xor remaining bytes from partial register into output
|
||||||
|
@@ -236,5 +236,5 @@ fold_64:
|
|||||||
pxor %xmm2, %xmm1
|
pxor %xmm2, %xmm1
|
||||||
pextrd $0x01, %xmm1, %eax
|
pextrd $0x01, %xmm1, %eax
|
||||||
|
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(crc32_pclmul_le_16)
|
SYM_FUNC_END(crc32_pclmul_le_16)
|
||||||
|
@@ -309,7 +309,7 @@ do_return:
|
|||||||
popq %rsi
|
popq %rsi
|
||||||
popq %rdi
|
popq %rdi
|
||||||
popq %rbx
|
popq %rbx
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(crc_pcl)
|
SYM_FUNC_END(crc_pcl)
|
||||||
|
|
||||||
.section .rodata, "a", @progbits
|
.section .rodata, "a", @progbits
|
||||||
|
@@ -257,7 +257,7 @@ SYM_FUNC_START(crc_t10dif_pcl)
|
|||||||
# Final CRC value (x^16 * M(x)) mod G(x) is in low 16 bits of xmm0.
|
# Final CRC value (x^16 * M(x)) mod G(x) is in low 16 bits of xmm0.
|
||||||
|
|
||||||
pextrw $0, %xmm0, %eax
|
pextrw $0, %xmm0, %eax
|
||||||
ret
|
RET
|
||||||
|
|
||||||
.align 16
|
.align 16
|
||||||
.Lless_than_256_bytes:
|
.Lless_than_256_bytes:
|
||||||
|
@@ -243,7 +243,7 @@ SYM_FUNC_START(des3_ede_x86_64_crypt_blk)
|
|||||||
popq %r12;
|
popq %r12;
|
||||||
popq %rbx;
|
popq %rbx;
|
||||||
|
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(des3_ede_x86_64_crypt_blk)
|
SYM_FUNC_END(des3_ede_x86_64_crypt_blk)
|
||||||
|
|
||||||
/***********************************************************************
|
/***********************************************************************
|
||||||
@@ -528,7 +528,7 @@ SYM_FUNC_START(des3_ede_x86_64_crypt_blk_3way)
|
|||||||
popq %r12;
|
popq %r12;
|
||||||
popq %rbx;
|
popq %rbx;
|
||||||
|
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(des3_ede_x86_64_crypt_blk_3way)
|
SYM_FUNC_END(des3_ede_x86_64_crypt_blk_3way)
|
||||||
|
|
||||||
.section .rodata, "a", @progbits
|
.section .rodata, "a", @progbits
|
||||||
|
@@ -85,7 +85,7 @@ SYM_FUNC_START_LOCAL(__clmul_gf128mul_ble)
|
|||||||
psrlq $1, T2
|
psrlq $1, T2
|
||||||
pxor T2, T1
|
pxor T2, T1
|
||||||
pxor T1, DATA
|
pxor T1, DATA
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(__clmul_gf128mul_ble)
|
SYM_FUNC_END(__clmul_gf128mul_ble)
|
||||||
|
|
||||||
/* void clmul_ghash_mul(char *dst, const u128 *shash) */
|
/* void clmul_ghash_mul(char *dst, const u128 *shash) */
|
||||||
@@ -99,7 +99,7 @@ SYM_FUNC_START(clmul_ghash_mul)
|
|||||||
pshufb BSWAP, DATA
|
pshufb BSWAP, DATA
|
||||||
movups DATA, (%rdi)
|
movups DATA, (%rdi)
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(clmul_ghash_mul)
|
SYM_FUNC_END(clmul_ghash_mul)
|
||||||
|
|
||||||
/*
|
/*
|
||||||
@@ -128,5 +128,5 @@ SYM_FUNC_START(clmul_ghash_update)
|
|||||||
movups DATA, (%rdi)
|
movups DATA, (%rdi)
|
||||||
.Lupdate_just_ret:
|
.Lupdate_just_ret:
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(clmul_ghash_update)
|
SYM_FUNC_END(clmul_ghash_update)
|
||||||
|
@@ -153,5 +153,5 @@ SYM_FUNC_START(nh_avx2)
|
|||||||
vpaddq T1, T0, T0
|
vpaddq T1, T0, T0
|
||||||
vpaddq T4, T0, T0
|
vpaddq T4, T0, T0
|
||||||
vmovdqu T0, (HASH)
|
vmovdqu T0, (HASH)
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(nh_avx2)
|
SYM_FUNC_END(nh_avx2)
|
||||||
|
@@ -119,5 +119,5 @@ SYM_FUNC_START(nh_sse2)
|
|||||||
paddq PASS2_SUMS, T1
|
paddq PASS2_SUMS, T1
|
||||||
movdqu T0, 0x00(HASH)
|
movdqu T0, 0x00(HASH)
|
||||||
movdqu T1, 0x10(HASH)
|
movdqu T1, 0x10(HASH)
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(nh_sse2)
|
SYM_FUNC_END(nh_sse2)
|
||||||
|
@@ -605,7 +605,7 @@ SYM_FUNC_START_LOCAL(__serpent_enc_blk8_avx)
|
|||||||
write_blocks(RA1, RB1, RC1, RD1, RK0, RK1, RK2);
|
write_blocks(RA1, RB1, RC1, RD1, RK0, RK1, RK2);
|
||||||
write_blocks(RA2, RB2, RC2, RD2, RK0, RK1, RK2);
|
write_blocks(RA2, RB2, RC2, RD2, RK0, RK1, RK2);
|
||||||
|
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(__serpent_enc_blk8_avx)
|
SYM_FUNC_END(__serpent_enc_blk8_avx)
|
||||||
|
|
||||||
.align 8
|
.align 8
|
||||||
@@ -659,7 +659,7 @@ SYM_FUNC_START_LOCAL(__serpent_dec_blk8_avx)
|
|||||||
write_blocks(RC1, RD1, RB1, RE1, RK0, RK1, RK2);
|
write_blocks(RC1, RD1, RB1, RE1, RK0, RK1, RK2);
|
||||||
write_blocks(RC2, RD2, RB2, RE2, RK0, RK1, RK2);
|
write_blocks(RC2, RD2, RB2, RE2, RK0, RK1, RK2);
|
||||||
|
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(__serpent_dec_blk8_avx)
|
SYM_FUNC_END(__serpent_dec_blk8_avx)
|
||||||
|
|
||||||
SYM_FUNC_START(serpent_ecb_enc_8way_avx)
|
SYM_FUNC_START(serpent_ecb_enc_8way_avx)
|
||||||
@@ -677,7 +677,7 @@ SYM_FUNC_START(serpent_ecb_enc_8way_avx)
|
|||||||
store_8way(%rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
|
store_8way(%rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
|
||||||
|
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(serpent_ecb_enc_8way_avx)
|
SYM_FUNC_END(serpent_ecb_enc_8way_avx)
|
||||||
|
|
||||||
SYM_FUNC_START(serpent_ecb_dec_8way_avx)
|
SYM_FUNC_START(serpent_ecb_dec_8way_avx)
|
||||||
@@ -695,7 +695,7 @@ SYM_FUNC_START(serpent_ecb_dec_8way_avx)
|
|||||||
store_8way(%rsi, RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2);
|
store_8way(%rsi, RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2);
|
||||||
|
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(serpent_ecb_dec_8way_avx)
|
SYM_FUNC_END(serpent_ecb_dec_8way_avx)
|
||||||
|
|
||||||
SYM_FUNC_START(serpent_cbc_dec_8way_avx)
|
SYM_FUNC_START(serpent_cbc_dec_8way_avx)
|
||||||
@@ -713,7 +713,7 @@ SYM_FUNC_START(serpent_cbc_dec_8way_avx)
|
|||||||
store_cbc_8way(%rdx, %rsi, RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2);
|
store_cbc_8way(%rdx, %rsi, RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2);
|
||||||
|
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(serpent_cbc_dec_8way_avx)
|
SYM_FUNC_END(serpent_cbc_dec_8way_avx)
|
||||||
|
|
||||||
SYM_FUNC_START(serpent_ctr_8way_avx)
|
SYM_FUNC_START(serpent_ctr_8way_avx)
|
||||||
@@ -733,7 +733,7 @@ SYM_FUNC_START(serpent_ctr_8way_avx)
|
|||||||
store_ctr_8way(%rdx, %rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
|
store_ctr_8way(%rdx, %rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
|
||||||
|
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(serpent_ctr_8way_avx)
|
SYM_FUNC_END(serpent_ctr_8way_avx)
|
||||||
|
|
||||||
SYM_FUNC_START(serpent_xts_enc_8way_avx)
|
SYM_FUNC_START(serpent_xts_enc_8way_avx)
|
||||||
@@ -755,7 +755,7 @@ SYM_FUNC_START(serpent_xts_enc_8way_avx)
|
|||||||
store_xts_8way(%rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
|
store_xts_8way(%rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
|
||||||
|
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(serpent_xts_enc_8way_avx)
|
SYM_FUNC_END(serpent_xts_enc_8way_avx)
|
||||||
|
|
||||||
SYM_FUNC_START(serpent_xts_dec_8way_avx)
|
SYM_FUNC_START(serpent_xts_dec_8way_avx)
|
||||||
@@ -777,5 +777,5 @@ SYM_FUNC_START(serpent_xts_dec_8way_avx)
|
|||||||
store_xts_8way(%rsi, RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2);
|
store_xts_8way(%rsi, RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2);
|
||||||
|
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(serpent_xts_dec_8way_avx)
|
SYM_FUNC_END(serpent_xts_dec_8way_avx)
|
||||||
|
@@ -611,7 +611,7 @@ SYM_FUNC_START_LOCAL(__serpent_enc_blk16)
|
|||||||
write_blocks(RA1, RB1, RC1, RD1, RK0, RK1, RK2);
|
write_blocks(RA1, RB1, RC1, RD1, RK0, RK1, RK2);
|
||||||
write_blocks(RA2, RB2, RC2, RD2, RK0, RK1, RK2);
|
write_blocks(RA2, RB2, RC2, RD2, RK0, RK1, RK2);
|
||||||
|
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(__serpent_enc_blk16)
|
SYM_FUNC_END(__serpent_enc_blk16)
|
||||||
|
|
||||||
.align 8
|
.align 8
|
||||||
@@ -665,7 +665,7 @@ SYM_FUNC_START_LOCAL(__serpent_dec_blk16)
|
|||||||
write_blocks(RC1, RD1, RB1, RE1, RK0, RK1, RK2);
|
write_blocks(RC1, RD1, RB1, RE1, RK0, RK1, RK2);
|
||||||
write_blocks(RC2, RD2, RB2, RE2, RK0, RK1, RK2);
|
write_blocks(RC2, RD2, RB2, RE2, RK0, RK1, RK2);
|
||||||
|
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(__serpent_dec_blk16)
|
SYM_FUNC_END(__serpent_dec_blk16)
|
||||||
|
|
||||||
SYM_FUNC_START(serpent_ecb_enc_16way)
|
SYM_FUNC_START(serpent_ecb_enc_16way)
|
||||||
@@ -687,7 +687,7 @@ SYM_FUNC_START(serpent_ecb_enc_16way)
|
|||||||
vzeroupper;
|
vzeroupper;
|
||||||
|
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(serpent_ecb_enc_16way)
|
SYM_FUNC_END(serpent_ecb_enc_16way)
|
||||||
|
|
||||||
SYM_FUNC_START(serpent_ecb_dec_16way)
|
SYM_FUNC_START(serpent_ecb_dec_16way)
|
||||||
@@ -709,7 +709,7 @@ SYM_FUNC_START(serpent_ecb_dec_16way)
|
|||||||
vzeroupper;
|
vzeroupper;
|
||||||
|
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(serpent_ecb_dec_16way)
|
SYM_FUNC_END(serpent_ecb_dec_16way)
|
||||||
|
|
||||||
SYM_FUNC_START(serpent_cbc_dec_16way)
|
SYM_FUNC_START(serpent_cbc_dec_16way)
|
||||||
@@ -732,7 +732,7 @@ SYM_FUNC_START(serpent_cbc_dec_16way)
|
|||||||
vzeroupper;
|
vzeroupper;
|
||||||
|
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(serpent_cbc_dec_16way)
|
SYM_FUNC_END(serpent_cbc_dec_16way)
|
||||||
|
|
||||||
SYM_FUNC_START(serpent_ctr_16way)
|
SYM_FUNC_START(serpent_ctr_16way)
|
||||||
@@ -757,7 +757,7 @@ SYM_FUNC_START(serpent_ctr_16way)
|
|||||||
vzeroupper;
|
vzeroupper;
|
||||||
|
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(serpent_ctr_16way)
|
SYM_FUNC_END(serpent_ctr_16way)
|
||||||
|
|
||||||
SYM_FUNC_START(serpent_xts_enc_16way)
|
SYM_FUNC_START(serpent_xts_enc_16way)
|
||||||
@@ -783,7 +783,7 @@ SYM_FUNC_START(serpent_xts_enc_16way)
|
|||||||
vzeroupper;
|
vzeroupper;
|
||||||
|
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(serpent_xts_enc_16way)
|
SYM_FUNC_END(serpent_xts_enc_16way)
|
||||||
|
|
||||||
SYM_FUNC_START(serpent_xts_dec_16way)
|
SYM_FUNC_START(serpent_xts_dec_16way)
|
||||||
@@ -809,5 +809,5 @@ SYM_FUNC_START(serpent_xts_dec_16way)
|
|||||||
vzeroupper;
|
vzeroupper;
|
||||||
|
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(serpent_xts_dec_16way)
|
SYM_FUNC_END(serpent_xts_dec_16way)
|
||||||
|
@@ -553,12 +553,12 @@ SYM_FUNC_START(__serpent_enc_blk_4way)
|
|||||||
|
|
||||||
write_blocks(%eax, RA, RB, RC, RD, RT0, RT1, RE);
|
write_blocks(%eax, RA, RB, RC, RD, RT0, RT1, RE);
|
||||||
|
|
||||||
ret;
|
RET;
|
||||||
|
|
||||||
.L__enc_xor4:
|
.L__enc_xor4:
|
||||||
xor_blocks(%eax, RA, RB, RC, RD, RT0, RT1, RE);
|
xor_blocks(%eax, RA, RB, RC, RD, RT0, RT1, RE);
|
||||||
|
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(__serpent_enc_blk_4way)
|
SYM_FUNC_END(__serpent_enc_blk_4way)
|
||||||
|
|
||||||
SYM_FUNC_START(serpent_dec_blk_4way)
|
SYM_FUNC_START(serpent_dec_blk_4way)
|
||||||
@@ -612,5 +612,5 @@ SYM_FUNC_START(serpent_dec_blk_4way)
|
|||||||
movl arg_dst(%esp), %eax;
|
movl arg_dst(%esp), %eax;
|
||||||
write_blocks(%eax, RC, RD, RB, RE, RT0, RT1, RA);
|
write_blocks(%eax, RC, RD, RB, RE, RT0, RT1, RA);
|
||||||
|
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(serpent_dec_blk_4way)
|
SYM_FUNC_END(serpent_dec_blk_4way)
|
||||||
|
@@ -675,13 +675,13 @@ SYM_FUNC_START(__serpent_enc_blk_8way)
|
|||||||
write_blocks(%rsi, RA1, RB1, RC1, RD1, RK0, RK1, RK2);
|
write_blocks(%rsi, RA1, RB1, RC1, RD1, RK0, RK1, RK2);
|
||||||
write_blocks(%rax, RA2, RB2, RC2, RD2, RK0, RK1, RK2);
|
write_blocks(%rax, RA2, RB2, RC2, RD2, RK0, RK1, RK2);
|
||||||
|
|
||||||
ret;
|
RET;
|
||||||
|
|
||||||
.L__enc_xor8:
|
.L__enc_xor8:
|
||||||
xor_blocks(%rsi, RA1, RB1, RC1, RD1, RK0, RK1, RK2);
|
xor_blocks(%rsi, RA1, RB1, RC1, RD1, RK0, RK1, RK2);
|
||||||
xor_blocks(%rax, RA2, RB2, RC2, RD2, RK0, RK1, RK2);
|
xor_blocks(%rax, RA2, RB2, RC2, RD2, RK0, RK1, RK2);
|
||||||
|
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(__serpent_enc_blk_8way)
|
SYM_FUNC_END(__serpent_enc_blk_8way)
|
||||||
|
|
||||||
SYM_FUNC_START(serpent_dec_blk_8way)
|
SYM_FUNC_START(serpent_dec_blk_8way)
|
||||||
@@ -735,5 +735,5 @@ SYM_FUNC_START(serpent_dec_blk_8way)
|
|||||||
write_blocks(%rsi, RC1, RD1, RB1, RE1, RK0, RK1, RK2);
|
write_blocks(%rsi, RC1, RD1, RB1, RE1, RK0, RK1, RK2);
|
||||||
write_blocks(%rax, RC2, RD2, RB2, RE2, RK0, RK1, RK2);
|
write_blocks(%rax, RC2, RD2, RB2, RE2, RK0, RK1, RK2);
|
||||||
|
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(serpent_dec_blk_8way)
|
SYM_FUNC_END(serpent_dec_blk_8way)
|
||||||
|
@@ -674,7 +674,7 @@ _loop3:
|
|||||||
pop %r12
|
pop %r12
|
||||||
pop %rbx
|
pop %rbx
|
||||||
|
|
||||||
ret
|
RET
|
||||||
|
|
||||||
SYM_FUNC_END(\name)
|
SYM_FUNC_END(\name)
|
||||||
.endm
|
.endm
|
||||||
|
@@ -290,7 +290,7 @@ SYM_FUNC_START(sha1_ni_transform)
|
|||||||
.Ldone_hash:
|
.Ldone_hash:
|
||||||
mov RSPSAVE, %rsp
|
mov RSPSAVE, %rsp
|
||||||
|
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(sha1_ni_transform)
|
SYM_FUNC_END(sha1_ni_transform)
|
||||||
|
|
||||||
.section .rodata.cst16.PSHUFFLE_BYTE_FLIP_MASK, "aM", @progbits, 16
|
.section .rodata.cst16.PSHUFFLE_BYTE_FLIP_MASK, "aM", @progbits, 16
|
||||||
|
@@ -99,7 +99,7 @@
|
|||||||
pop %rbp
|
pop %rbp
|
||||||
pop %r12
|
pop %r12
|
||||||
pop %rbx
|
pop %rbx
|
||||||
ret
|
RET
|
||||||
|
|
||||||
SYM_FUNC_END(\name)
|
SYM_FUNC_END(\name)
|
||||||
.endm
|
.endm
|
||||||
|
@@ -458,7 +458,7 @@ done_hash:
|
|||||||
popq %r13
|
popq %r13
|
||||||
popq %r12
|
popq %r12
|
||||||
popq %rbx
|
popq %rbx
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(sha256_transform_avx)
|
SYM_FUNC_END(sha256_transform_avx)
|
||||||
|
|
||||||
.section .rodata.cst256.K256, "aM", @progbits, 256
|
.section .rodata.cst256.K256, "aM", @progbits, 256
|
||||||
|
@@ -711,7 +711,7 @@ done_hash:
|
|||||||
popq %r13
|
popq %r13
|
||||||
popq %r12
|
popq %r12
|
||||||
popq %rbx
|
popq %rbx
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(sha256_transform_rorx)
|
SYM_FUNC_END(sha256_transform_rorx)
|
||||||
|
|
||||||
.section .rodata.cst512.K256, "aM", @progbits, 512
|
.section .rodata.cst512.K256, "aM", @progbits, 512
|
||||||
|
@@ -472,7 +472,7 @@ done_hash:
|
|||||||
popq %r12
|
popq %r12
|
||||||
popq %rbx
|
popq %rbx
|
||||||
|
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(sha256_transform_ssse3)
|
SYM_FUNC_END(sha256_transform_ssse3)
|
||||||
|
|
||||||
.section .rodata.cst256.K256, "aM", @progbits, 256
|
.section .rodata.cst256.K256, "aM", @progbits, 256
|
||||||
|
@@ -326,7 +326,7 @@ SYM_FUNC_START(sha256_ni_transform)
|
|||||||
|
|
||||||
.Ldone_hash:
|
.Ldone_hash:
|
||||||
|
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(sha256_ni_transform)
|
SYM_FUNC_END(sha256_ni_transform)
|
||||||
|
|
||||||
.section .rodata.cst256.K256, "aM", @progbits, 256
|
.section .rodata.cst256.K256, "aM", @progbits, 256
|
||||||
|
@@ -364,7 +364,7 @@ updateblock:
|
|||||||
mov frame_RSPSAVE(%rsp), %rsp
|
mov frame_RSPSAVE(%rsp), %rsp
|
||||||
|
|
||||||
nowork:
|
nowork:
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(sha512_transform_avx)
|
SYM_FUNC_END(sha512_transform_avx)
|
||||||
|
|
||||||
########################################################################
|
########################################################################
|
||||||
|
@@ -681,7 +681,7 @@ done_hash:
|
|||||||
|
|
||||||
# Restore Stack Pointer
|
# Restore Stack Pointer
|
||||||
mov frame_RSPSAVE(%rsp), %rsp
|
mov frame_RSPSAVE(%rsp), %rsp
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(sha512_transform_rorx)
|
SYM_FUNC_END(sha512_transform_rorx)
|
||||||
|
|
||||||
########################################################################
|
########################################################################
|
||||||
|
@@ -366,7 +366,7 @@ updateblock:
|
|||||||
mov frame_RSPSAVE(%rsp), %rsp
|
mov frame_RSPSAVE(%rsp), %rsp
|
||||||
|
|
||||||
nowork:
|
nowork:
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(sha512_transform_ssse3)
|
SYM_FUNC_END(sha512_transform_ssse3)
|
||||||
|
|
||||||
########################################################################
|
########################################################################
|
||||||
|
@@ -272,7 +272,7 @@ SYM_FUNC_START_LOCAL(__twofish_enc_blk8)
|
|||||||
outunpack_blocks(RC1, RD1, RA1, RB1, RK1, RX0, RY0, RK2);
|
outunpack_blocks(RC1, RD1, RA1, RB1, RK1, RX0, RY0, RK2);
|
||||||
outunpack_blocks(RC2, RD2, RA2, RB2, RK1, RX0, RY0, RK2);
|
outunpack_blocks(RC2, RD2, RA2, RB2, RK1, RX0, RY0, RK2);
|
||||||
|
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(__twofish_enc_blk8)
|
SYM_FUNC_END(__twofish_enc_blk8)
|
||||||
|
|
||||||
.align 8
|
.align 8
|
||||||
@@ -312,7 +312,7 @@ SYM_FUNC_START_LOCAL(__twofish_dec_blk8)
|
|||||||
outunpack_blocks(RA1, RB1, RC1, RD1, RK1, RX0, RY0, RK2);
|
outunpack_blocks(RA1, RB1, RC1, RD1, RK1, RX0, RY0, RK2);
|
||||||
outunpack_blocks(RA2, RB2, RC2, RD2, RK1, RX0, RY0, RK2);
|
outunpack_blocks(RA2, RB2, RC2, RD2, RK1, RX0, RY0, RK2);
|
||||||
|
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(__twofish_dec_blk8)
|
SYM_FUNC_END(__twofish_dec_blk8)
|
||||||
|
|
||||||
SYM_FUNC_START(twofish_ecb_enc_8way)
|
SYM_FUNC_START(twofish_ecb_enc_8way)
|
||||||
@@ -332,7 +332,7 @@ SYM_FUNC_START(twofish_ecb_enc_8way)
|
|||||||
store_8way(%r11, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2);
|
store_8way(%r11, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2);
|
||||||
|
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(twofish_ecb_enc_8way)
|
SYM_FUNC_END(twofish_ecb_enc_8way)
|
||||||
|
|
||||||
SYM_FUNC_START(twofish_ecb_dec_8way)
|
SYM_FUNC_START(twofish_ecb_dec_8way)
|
||||||
@@ -352,7 +352,7 @@ SYM_FUNC_START(twofish_ecb_dec_8way)
|
|||||||
store_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
|
store_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
|
||||||
|
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(twofish_ecb_dec_8way)
|
SYM_FUNC_END(twofish_ecb_dec_8way)
|
||||||
|
|
||||||
SYM_FUNC_START(twofish_cbc_dec_8way)
|
SYM_FUNC_START(twofish_cbc_dec_8way)
|
||||||
@@ -377,7 +377,7 @@ SYM_FUNC_START(twofish_cbc_dec_8way)
|
|||||||
popq %r12;
|
popq %r12;
|
||||||
|
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(twofish_cbc_dec_8way)
|
SYM_FUNC_END(twofish_cbc_dec_8way)
|
||||||
|
|
||||||
SYM_FUNC_START(twofish_ctr_8way)
|
SYM_FUNC_START(twofish_ctr_8way)
|
||||||
@@ -404,7 +404,7 @@ SYM_FUNC_START(twofish_ctr_8way)
|
|||||||
popq %r12;
|
popq %r12;
|
||||||
|
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(twofish_ctr_8way)
|
SYM_FUNC_END(twofish_ctr_8way)
|
||||||
|
|
||||||
SYM_FUNC_START(twofish_xts_enc_8way)
|
SYM_FUNC_START(twofish_xts_enc_8way)
|
||||||
@@ -428,7 +428,7 @@ SYM_FUNC_START(twofish_xts_enc_8way)
|
|||||||
store_xts_8way(%r11, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2);
|
store_xts_8way(%r11, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2);
|
||||||
|
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(twofish_xts_enc_8way)
|
SYM_FUNC_END(twofish_xts_enc_8way)
|
||||||
|
|
||||||
SYM_FUNC_START(twofish_xts_dec_8way)
|
SYM_FUNC_START(twofish_xts_dec_8way)
|
||||||
@@ -452,5 +452,5 @@ SYM_FUNC_START(twofish_xts_dec_8way)
|
|||||||
store_xts_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
|
store_xts_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
|
||||||
|
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(twofish_xts_dec_8way)
|
SYM_FUNC_END(twofish_xts_dec_8way)
|
||||||
|
@@ -260,7 +260,7 @@ SYM_FUNC_START(twofish_enc_blk)
|
|||||||
pop %ebx
|
pop %ebx
|
||||||
pop %ebp
|
pop %ebp
|
||||||
mov $1, %eax
|
mov $1, %eax
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(twofish_enc_blk)
|
SYM_FUNC_END(twofish_enc_blk)
|
||||||
|
|
||||||
SYM_FUNC_START(twofish_dec_blk)
|
SYM_FUNC_START(twofish_dec_blk)
|
||||||
@@ -317,5 +317,5 @@ SYM_FUNC_START(twofish_dec_blk)
|
|||||||
pop %ebx
|
pop %ebx
|
||||||
pop %ebp
|
pop %ebp
|
||||||
mov $1, %eax
|
mov $1, %eax
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(twofish_dec_blk)
|
SYM_FUNC_END(twofish_dec_blk)
|
||||||
|
@@ -258,7 +258,7 @@ SYM_FUNC_START(__twofish_enc_blk_3way)
|
|||||||
popq %rbx;
|
popq %rbx;
|
||||||
popq %r12;
|
popq %r12;
|
||||||
popq %r13;
|
popq %r13;
|
||||||
ret;
|
RET;
|
||||||
|
|
||||||
.L__enc_xor3:
|
.L__enc_xor3:
|
||||||
outunpack_enc3(xor);
|
outunpack_enc3(xor);
|
||||||
@@ -266,7 +266,7 @@ SYM_FUNC_START(__twofish_enc_blk_3way)
|
|||||||
popq %rbx;
|
popq %rbx;
|
||||||
popq %r12;
|
popq %r12;
|
||||||
popq %r13;
|
popq %r13;
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(__twofish_enc_blk_3way)
|
SYM_FUNC_END(__twofish_enc_blk_3way)
|
||||||
|
|
||||||
SYM_FUNC_START(twofish_dec_blk_3way)
|
SYM_FUNC_START(twofish_dec_blk_3way)
|
||||||
@@ -301,5 +301,5 @@ SYM_FUNC_START(twofish_dec_blk_3way)
|
|||||||
popq %rbx;
|
popq %rbx;
|
||||||
popq %r12;
|
popq %r12;
|
||||||
popq %r13;
|
popq %r13;
|
||||||
ret;
|
RET;
|
||||||
SYM_FUNC_END(twofish_dec_blk_3way)
|
SYM_FUNC_END(twofish_dec_blk_3way)
|
||||||
|
@@ -252,7 +252,7 @@ SYM_FUNC_START(twofish_enc_blk)
|
|||||||
|
|
||||||
popq R1
|
popq R1
|
||||||
movl $1,%eax
|
movl $1,%eax
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(twofish_enc_blk)
|
SYM_FUNC_END(twofish_enc_blk)
|
||||||
|
|
||||||
SYM_FUNC_START(twofish_dec_blk)
|
SYM_FUNC_START(twofish_dec_blk)
|
||||||
@@ -304,5 +304,5 @@ SYM_FUNC_START(twofish_dec_blk)
|
|||||||
|
|
||||||
popq R1
|
popq R1
|
||||||
movl $1,%eax
|
movl $1,%eax
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(twofish_dec_blk)
|
SYM_FUNC_END(twofish_dec_blk)
|
||||||
|
@@ -821,7 +821,7 @@ SYM_FUNC_START(schedule_tail_wrapper)
|
|||||||
popl %eax
|
popl %eax
|
||||||
|
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(schedule_tail_wrapper)
|
SYM_FUNC_END(schedule_tail_wrapper)
|
||||||
.popsection
|
.popsection
|
||||||
|
|
||||||
|
@@ -740,7 +740,7 @@ SYM_FUNC_START(asm_load_gs_index)
|
|||||||
2: ALTERNATIVE "", "mfence", X86_BUG_SWAPGS_FENCE
|
2: ALTERNATIVE "", "mfence", X86_BUG_SWAPGS_FENCE
|
||||||
swapgs
|
swapgs
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(asm_load_gs_index)
|
SYM_FUNC_END(asm_load_gs_index)
|
||||||
EXPORT_SYMBOL(asm_load_gs_index)
|
EXPORT_SYMBOL(asm_load_gs_index)
|
||||||
|
|
||||||
@@ -799,7 +799,7 @@ SYM_INNER_LABEL(asm_call_irq_on_stack, SYM_L_GLOBAL)
|
|||||||
|
|
||||||
/* Restore the previous stack pointer from RBP. */
|
/* Restore the previous stack pointer from RBP. */
|
||||||
leaveq
|
leaveq
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(asm_call_on_stack)
|
SYM_FUNC_END(asm_call_on_stack)
|
||||||
|
|
||||||
#ifdef CONFIG_XEN_PV
|
#ifdef CONFIG_XEN_PV
|
||||||
@@ -932,7 +932,7 @@ SYM_CODE_START_LOCAL(paranoid_entry)
|
|||||||
* is needed here.
|
* is needed here.
|
||||||
*/
|
*/
|
||||||
SAVE_AND_SET_GSBASE scratch_reg=%rax save_reg=%rbx
|
SAVE_AND_SET_GSBASE scratch_reg=%rax save_reg=%rbx
|
||||||
ret
|
RET
|
||||||
|
|
||||||
.Lparanoid_entry_checkgs:
|
.Lparanoid_entry_checkgs:
|
||||||
/* EBX = 1 -> kernel GSBASE active, no restore required */
|
/* EBX = 1 -> kernel GSBASE active, no restore required */
|
||||||
@@ -953,7 +953,7 @@ SYM_CODE_START_LOCAL(paranoid_entry)
|
|||||||
.Lparanoid_kernel_gsbase:
|
.Lparanoid_kernel_gsbase:
|
||||||
|
|
||||||
FENCE_SWAPGS_KERNEL_ENTRY
|
FENCE_SWAPGS_KERNEL_ENTRY
|
||||||
ret
|
RET
|
||||||
SYM_CODE_END(paranoid_entry)
|
SYM_CODE_END(paranoid_entry)
|
||||||
|
|
||||||
/*
|
/*
|
||||||
@@ -1032,7 +1032,7 @@ SYM_CODE_START_LOCAL(error_entry)
|
|||||||
movq %rax, %rsp /* switch stack */
|
movq %rax, %rsp /* switch stack */
|
||||||
ENCODE_FRAME_POINTER
|
ENCODE_FRAME_POINTER
|
||||||
pushq %r12
|
pushq %r12
|
||||||
ret
|
RET
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* There are two places in the kernel that can potentially fault with
|
* There are two places in the kernel that can potentially fault with
|
||||||
@@ -1063,7 +1063,7 @@ SYM_CODE_START_LOCAL(error_entry)
|
|||||||
*/
|
*/
|
||||||
.Lerror_entry_done_lfence:
|
.Lerror_entry_done_lfence:
|
||||||
FENCE_SWAPGS_KERNEL_ENTRY
|
FENCE_SWAPGS_KERNEL_ENTRY
|
||||||
ret
|
RET
|
||||||
|
|
||||||
.Lbstep_iret:
|
.Lbstep_iret:
|
||||||
/* Fix truncated RIP */
|
/* Fix truncated RIP */
|
||||||
|
@@ -24,7 +24,7 @@ SYM_CODE_START_NOALIGN(\name)
|
|||||||
popl %edx
|
popl %edx
|
||||||
popl %ecx
|
popl %ecx
|
||||||
popl %eax
|
popl %eax
|
||||||
ret
|
RET
|
||||||
_ASM_NOKPROBE(\name)
|
_ASM_NOKPROBE(\name)
|
||||||
SYM_CODE_END(\name)
|
SYM_CODE_END(\name)
|
||||||
.endm
|
.endm
|
||||||
|
@@ -55,7 +55,7 @@ SYM_CODE_START_LOCAL_NOALIGN(__thunk_restore)
|
|||||||
popq %rsi
|
popq %rsi
|
||||||
popq %rdi
|
popq %rdi
|
||||||
popq %rbp
|
popq %rbp
|
||||||
ret
|
RET
|
||||||
_ASM_NOKPROBE(__thunk_restore)
|
_ASM_NOKPROBE(__thunk_restore)
|
||||||
SYM_CODE_END(__thunk_restore)
|
SYM_CODE_END(__thunk_restore)
|
||||||
#endif
|
#endif
|
||||||
|
@@ -78,7 +78,7 @@ SYM_INNER_LABEL(int80_landing_pad, SYM_L_GLOBAL)
|
|||||||
popl %ecx
|
popl %ecx
|
||||||
CFI_RESTORE ecx
|
CFI_RESTORE ecx
|
||||||
CFI_ADJUST_CFA_OFFSET -4
|
CFI_ADJUST_CFA_OFFSET -4
|
||||||
ret
|
RET
|
||||||
CFI_ENDPROC
|
CFI_ENDPROC
|
||||||
|
|
||||||
.size __kernel_vsyscall,.-__kernel_vsyscall
|
.size __kernel_vsyscall,.-__kernel_vsyscall
|
||||||
|
@@ -19,17 +19,17 @@ __vsyscall_page:
|
|||||||
|
|
||||||
mov $__NR_gettimeofday, %rax
|
mov $__NR_gettimeofday, %rax
|
||||||
syscall
|
syscall
|
||||||
ret
|
RET
|
||||||
|
|
||||||
.balign 1024, 0xcc
|
.balign 1024, 0xcc
|
||||||
mov $__NR_time, %rax
|
mov $__NR_time, %rax
|
||||||
syscall
|
syscall
|
||||||
ret
|
RET
|
||||||
|
|
||||||
.balign 1024, 0xcc
|
.balign 1024, 0xcc
|
||||||
mov $__NR_getcpu, %rax
|
mov $__NR_getcpu, %rax
|
||||||
syscall
|
syscall
|
||||||
ret
|
RET
|
||||||
|
|
||||||
.balign 4096, 0xcc
|
.balign 4096, 0xcc
|
||||||
|
|
||||||
|
@@ -60,7 +60,7 @@ save_registers:
|
|||||||
popl saved_context_eflags
|
popl saved_context_eflags
|
||||||
|
|
||||||
movl $ret_point, saved_eip
|
movl $ret_point, saved_eip
|
||||||
ret
|
RET
|
||||||
|
|
||||||
|
|
||||||
restore_registers:
|
restore_registers:
|
||||||
@@ -70,7 +70,7 @@ restore_registers:
|
|||||||
movl saved_context_edi, %edi
|
movl saved_context_edi, %edi
|
||||||
pushl saved_context_eflags
|
pushl saved_context_eflags
|
||||||
popfl
|
popfl
|
||||||
ret
|
RET
|
||||||
|
|
||||||
SYM_CODE_START(do_suspend_lowlevel)
|
SYM_CODE_START(do_suspend_lowlevel)
|
||||||
call save_processor_state
|
call save_processor_state
|
||||||
@@ -86,7 +86,7 @@ SYM_CODE_START(do_suspend_lowlevel)
|
|||||||
ret_point:
|
ret_point:
|
||||||
call restore_registers
|
call restore_registers
|
||||||
call restore_processor_state
|
call restore_processor_state
|
||||||
ret
|
RET
|
||||||
SYM_CODE_END(do_suspend_lowlevel)
|
SYM_CODE_END(do_suspend_lowlevel)
|
||||||
|
|
||||||
.data
|
.data
|
||||||
|
@@ -19,7 +19,7 @@
|
|||||||
#endif
|
#endif
|
||||||
|
|
||||||
SYM_FUNC_START(__fentry__)
|
SYM_FUNC_START(__fentry__)
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(__fentry__)
|
SYM_FUNC_END(__fentry__)
|
||||||
EXPORT_SYMBOL(__fentry__)
|
EXPORT_SYMBOL(__fentry__)
|
||||||
|
|
||||||
@@ -84,7 +84,7 @@ ftrace_graph_call:
|
|||||||
|
|
||||||
/* This is weak to keep gas from relaxing the jumps */
|
/* This is weak to keep gas from relaxing the jumps */
|
||||||
SYM_INNER_LABEL_ALIGN(ftrace_stub, SYM_L_WEAK)
|
SYM_INNER_LABEL_ALIGN(ftrace_stub, SYM_L_WEAK)
|
||||||
ret
|
RET
|
||||||
SYM_CODE_END(ftrace_caller)
|
SYM_CODE_END(ftrace_caller)
|
||||||
|
|
||||||
SYM_CODE_START(ftrace_regs_caller)
|
SYM_CODE_START(ftrace_regs_caller)
|
||||||
@@ -177,7 +177,7 @@ SYM_CODE_START(ftrace_graph_caller)
|
|||||||
popl %edx
|
popl %edx
|
||||||
popl %ecx
|
popl %ecx
|
||||||
popl %eax
|
popl %eax
|
||||||
ret
|
RET
|
||||||
SYM_CODE_END(ftrace_graph_caller)
|
SYM_CODE_END(ftrace_graph_caller)
|
||||||
|
|
||||||
.globl return_to_handler
|
.globl return_to_handler
|
||||||
|
@@ -132,7 +132,7 @@
|
|||||||
#ifdef CONFIG_DYNAMIC_FTRACE
|
#ifdef CONFIG_DYNAMIC_FTRACE
|
||||||
|
|
||||||
SYM_FUNC_START(__fentry__)
|
SYM_FUNC_START(__fentry__)
|
||||||
retq
|
RET
|
||||||
SYM_FUNC_END(__fentry__)
|
SYM_FUNC_END(__fentry__)
|
||||||
EXPORT_SYMBOL(__fentry__)
|
EXPORT_SYMBOL(__fentry__)
|
||||||
|
|
||||||
@@ -170,10 +170,10 @@ SYM_INNER_LABEL(ftrace_graph_call, SYM_L_GLOBAL)
|
|||||||
|
|
||||||
/*
|
/*
|
||||||
* This is weak to keep gas from relaxing the jumps.
|
* This is weak to keep gas from relaxing the jumps.
|
||||||
* It is also used to copy the retq for trampolines.
|
* It is also used to copy the RET for trampolines.
|
||||||
*/
|
*/
|
||||||
SYM_INNER_LABEL_ALIGN(ftrace_stub, SYM_L_WEAK)
|
SYM_INNER_LABEL_ALIGN(ftrace_stub, SYM_L_WEAK)
|
||||||
retq
|
RET
|
||||||
SYM_FUNC_END(ftrace_epilogue)
|
SYM_FUNC_END(ftrace_epilogue)
|
||||||
|
|
||||||
SYM_FUNC_START(ftrace_regs_caller)
|
SYM_FUNC_START(ftrace_regs_caller)
|
||||||
@@ -287,7 +287,7 @@ fgraph_trace:
|
|||||||
#endif
|
#endif
|
||||||
|
|
||||||
SYM_INNER_LABEL(ftrace_stub, SYM_L_GLOBAL)
|
SYM_INNER_LABEL(ftrace_stub, SYM_L_GLOBAL)
|
||||||
retq
|
RET
|
||||||
|
|
||||||
trace:
|
trace:
|
||||||
/* save_mcount_regs fills in first two parameters */
|
/* save_mcount_regs fills in first two parameters */
|
||||||
@@ -319,7 +319,7 @@ SYM_FUNC_START(ftrace_graph_caller)
|
|||||||
|
|
||||||
restore_mcount_regs
|
restore_mcount_regs
|
||||||
|
|
||||||
retq
|
RET
|
||||||
SYM_FUNC_END(ftrace_graph_caller)
|
SYM_FUNC_END(ftrace_graph_caller)
|
||||||
|
|
||||||
SYM_CODE_START(return_to_handler)
|
SYM_CODE_START(return_to_handler)
|
||||||
|
@@ -354,7 +354,7 @@ setup_once:
|
|||||||
#endif
|
#endif
|
||||||
|
|
||||||
andl $0,setup_once_ref /* Once is enough, thanks */
|
andl $0,setup_once_ref /* Once is enough, thanks */
|
||||||
ret
|
RET
|
||||||
|
|
||||||
SYM_FUNC_START(early_idt_handler_array)
|
SYM_FUNC_START(early_idt_handler_array)
|
||||||
# 36(%esp) %eflags
|
# 36(%esp) %eflags
|
||||||
|
@@ -10,7 +10,7 @@
|
|||||||
SYM_FUNC_START(native_save_fl)
|
SYM_FUNC_START(native_save_fl)
|
||||||
pushf
|
pushf
|
||||||
pop %_ASM_AX
|
pop %_ASM_AX
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(native_save_fl)
|
SYM_FUNC_END(native_save_fl)
|
||||||
EXPORT_SYMBOL(native_save_fl)
|
EXPORT_SYMBOL(native_save_fl)
|
||||||
|
|
||||||
@@ -21,6 +21,6 @@ EXPORT_SYMBOL(native_save_fl)
|
|||||||
SYM_FUNC_START(native_restore_fl)
|
SYM_FUNC_START(native_restore_fl)
|
||||||
push %_ASM_ARG1
|
push %_ASM_ARG1
|
||||||
popf
|
popf
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(native_restore_fl)
|
SYM_FUNC_END(native_restore_fl)
|
||||||
EXPORT_SYMBOL(native_restore_fl)
|
EXPORT_SYMBOL(native_restore_fl)
|
||||||
|
@@ -91,7 +91,7 @@ SYM_CODE_START_NOALIGN(relocate_kernel)
|
|||||||
movl %edi, %eax
|
movl %edi, %eax
|
||||||
addl $(identity_mapped - relocate_kernel), %eax
|
addl $(identity_mapped - relocate_kernel), %eax
|
||||||
pushl %eax
|
pushl %eax
|
||||||
ret
|
RET
|
||||||
SYM_CODE_END(relocate_kernel)
|
SYM_CODE_END(relocate_kernel)
|
||||||
|
|
||||||
SYM_CODE_START_LOCAL_NOALIGN(identity_mapped)
|
SYM_CODE_START_LOCAL_NOALIGN(identity_mapped)
|
||||||
@@ -159,7 +159,7 @@ SYM_CODE_START_LOCAL_NOALIGN(identity_mapped)
|
|||||||
xorl %edx, %edx
|
xorl %edx, %edx
|
||||||
xorl %esi, %esi
|
xorl %esi, %esi
|
||||||
xorl %ebp, %ebp
|
xorl %ebp, %ebp
|
||||||
ret
|
RET
|
||||||
1:
|
1:
|
||||||
popl %edx
|
popl %edx
|
||||||
movl CP_PA_SWAP_PAGE(%edi), %esp
|
movl CP_PA_SWAP_PAGE(%edi), %esp
|
||||||
@@ -190,7 +190,7 @@ SYM_CODE_START_LOCAL_NOALIGN(identity_mapped)
|
|||||||
movl %edi, %eax
|
movl %edi, %eax
|
||||||
addl $(virtual_mapped - relocate_kernel), %eax
|
addl $(virtual_mapped - relocate_kernel), %eax
|
||||||
pushl %eax
|
pushl %eax
|
||||||
ret
|
RET
|
||||||
SYM_CODE_END(identity_mapped)
|
SYM_CODE_END(identity_mapped)
|
||||||
|
|
||||||
SYM_CODE_START_LOCAL_NOALIGN(virtual_mapped)
|
SYM_CODE_START_LOCAL_NOALIGN(virtual_mapped)
|
||||||
@@ -208,7 +208,7 @@ SYM_CODE_START_LOCAL_NOALIGN(virtual_mapped)
|
|||||||
popl %edi
|
popl %edi
|
||||||
popl %esi
|
popl %esi
|
||||||
popl %ebx
|
popl %ebx
|
||||||
ret
|
RET
|
||||||
SYM_CODE_END(virtual_mapped)
|
SYM_CODE_END(virtual_mapped)
|
||||||
|
|
||||||
/* Do the copies */
|
/* Do the copies */
|
||||||
@@ -271,7 +271,7 @@ SYM_CODE_START_LOCAL_NOALIGN(swap_pages)
|
|||||||
popl %edi
|
popl %edi
|
||||||
popl %ebx
|
popl %ebx
|
||||||
popl %ebp
|
popl %ebp
|
||||||
ret
|
RET
|
||||||
SYM_CODE_END(swap_pages)
|
SYM_CODE_END(swap_pages)
|
||||||
|
|
||||||
.globl kexec_control_code_size
|
.globl kexec_control_code_size
|
||||||
|
@@ -104,7 +104,7 @@ SYM_CODE_START_NOALIGN(relocate_kernel)
|
|||||||
/* jump to identity mapped page */
|
/* jump to identity mapped page */
|
||||||
addq $(identity_mapped - relocate_kernel), %r8
|
addq $(identity_mapped - relocate_kernel), %r8
|
||||||
pushq %r8
|
pushq %r8
|
||||||
ret
|
RET
|
||||||
SYM_CODE_END(relocate_kernel)
|
SYM_CODE_END(relocate_kernel)
|
||||||
|
|
||||||
SYM_CODE_START_LOCAL_NOALIGN(identity_mapped)
|
SYM_CODE_START_LOCAL_NOALIGN(identity_mapped)
|
||||||
@@ -191,7 +191,7 @@ SYM_CODE_START_LOCAL_NOALIGN(identity_mapped)
|
|||||||
xorl %r14d, %r14d
|
xorl %r14d, %r14d
|
||||||
xorl %r15d, %r15d
|
xorl %r15d, %r15d
|
||||||
|
|
||||||
ret
|
RET
|
||||||
|
|
||||||
1:
|
1:
|
||||||
popq %rdx
|
popq %rdx
|
||||||
@@ -210,7 +210,7 @@ SYM_CODE_START_LOCAL_NOALIGN(identity_mapped)
|
|||||||
call swap_pages
|
call swap_pages
|
||||||
movq $virtual_mapped, %rax
|
movq $virtual_mapped, %rax
|
||||||
pushq %rax
|
pushq %rax
|
||||||
ret
|
RET
|
||||||
SYM_CODE_END(identity_mapped)
|
SYM_CODE_END(identity_mapped)
|
||||||
|
|
||||||
SYM_CODE_START_LOCAL_NOALIGN(virtual_mapped)
|
SYM_CODE_START_LOCAL_NOALIGN(virtual_mapped)
|
||||||
@@ -231,7 +231,7 @@ SYM_CODE_START_LOCAL_NOALIGN(virtual_mapped)
|
|||||||
popq %r12
|
popq %r12
|
||||||
popq %rbp
|
popq %rbp
|
||||||
popq %rbx
|
popq %rbx
|
||||||
ret
|
RET
|
||||||
SYM_CODE_END(virtual_mapped)
|
SYM_CODE_END(virtual_mapped)
|
||||||
|
|
||||||
/* Do the copies */
|
/* Do the copies */
|
||||||
@@ -288,7 +288,7 @@ SYM_CODE_START_LOCAL_NOALIGN(swap_pages)
|
|||||||
lea PAGE_SIZE(%rax), %rsi
|
lea PAGE_SIZE(%rax), %rsi
|
||||||
jmp 0b
|
jmp 0b
|
||||||
3:
|
3:
|
||||||
ret
|
RET
|
||||||
SYM_CODE_END(swap_pages)
|
SYM_CODE_END(swap_pages)
|
||||||
|
|
||||||
.globl kexec_control_code_size
|
.globl kexec_control_code_size
|
||||||
|
@@ -85,5 +85,5 @@ SYM_FUNC_START(sev_verify_cbit)
|
|||||||
#endif
|
#endif
|
||||||
/* Return page-table pointer */
|
/* Return page-table pointer */
|
||||||
movq %rdi, %rax
|
movq %rdi, %rax
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(sev_verify_cbit)
|
SYM_FUNC_END(sev_verify_cbit)
|
||||||
|
@@ -132,9 +132,9 @@ SYM_FUNC_START_LOCAL(verify_cpu)
|
|||||||
.Lverify_cpu_no_longmode:
|
.Lverify_cpu_no_longmode:
|
||||||
popf # Restore caller passed flags
|
popf # Restore caller passed flags
|
||||||
movl $1,%eax
|
movl $1,%eax
|
||||||
ret
|
RET
|
||||||
.Lverify_cpu_sse_ok:
|
.Lverify_cpu_sse_ok:
|
||||||
popf # Restore caller passed flags
|
popf # Restore caller passed flags
|
||||||
xorl %eax, %eax
|
xorl %eax, %eax
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(verify_cpu)
|
SYM_FUNC_END(verify_cpu)
|
||||||
|
@@ -166,5 +166,5 @@ SYM_FUNC_START(__svm_vcpu_run)
|
|||||||
pop %edi
|
pop %edi
|
||||||
#endif
|
#endif
|
||||||
pop %_ASM_BP
|
pop %_ASM_BP
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(__svm_vcpu_run)
|
SYM_FUNC_END(__svm_vcpu_run)
|
||||||
|
@@ -49,14 +49,14 @@ SYM_FUNC_START_LOCAL(vmx_vmenter)
|
|||||||
je 2f
|
je 2f
|
||||||
|
|
||||||
1: vmresume
|
1: vmresume
|
||||||
ret
|
RET
|
||||||
|
|
||||||
2: vmlaunch
|
2: vmlaunch
|
||||||
ret
|
RET
|
||||||
|
|
||||||
3: cmpb $0, kvm_rebooting
|
3: cmpb $0, kvm_rebooting
|
||||||
je 4f
|
je 4f
|
||||||
ret
|
RET
|
||||||
4: ud2
|
4: ud2
|
||||||
|
|
||||||
_ASM_EXTABLE(1b, 3b)
|
_ASM_EXTABLE(1b, 3b)
|
||||||
@@ -89,7 +89,7 @@ SYM_FUNC_START(vmx_vmexit)
|
|||||||
pop %_ASM_AX
|
pop %_ASM_AX
|
||||||
.Lvmexit_skip_rsb:
|
.Lvmexit_skip_rsb:
|
||||||
#endif
|
#endif
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(vmx_vmexit)
|
SYM_FUNC_END(vmx_vmexit)
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -228,7 +228,7 @@ SYM_FUNC_START(__vmx_vcpu_run)
|
|||||||
pop %edi
|
pop %edi
|
||||||
#endif
|
#endif
|
||||||
pop %_ASM_BP
|
pop %_ASM_BP
|
||||||
ret
|
RET
|
||||||
|
|
||||||
/* VM-Fail. Out-of-line to avoid a taken Jcc after VM-Exit. */
|
/* VM-Fail. Out-of-line to avoid a taken Jcc after VM-Exit. */
|
||||||
2: mov $1, %eax
|
2: mov $1, %eax
|
||||||
@@ -293,7 +293,7 @@ SYM_FUNC_START(vmread_error_trampoline)
|
|||||||
pop %_ASM_AX
|
pop %_ASM_AX
|
||||||
pop %_ASM_BP
|
pop %_ASM_BP
|
||||||
|
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(vmread_error_trampoline)
|
SYM_FUNC_END(vmread_error_trampoline)
|
||||||
|
|
||||||
SYM_FUNC_START(vmx_do_interrupt_nmi_irqoff)
|
SYM_FUNC_START(vmx_do_interrupt_nmi_irqoff)
|
||||||
@@ -326,5 +326,5 @@ SYM_FUNC_START(vmx_do_interrupt_nmi_irqoff)
|
|||||||
*/
|
*/
|
||||||
mov %_ASM_BP, %_ASM_SP
|
mov %_ASM_BP, %_ASM_SP
|
||||||
pop %_ASM_BP
|
pop %_ASM_BP
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(vmx_do_interrupt_nmi_irqoff)
|
SYM_FUNC_END(vmx_do_interrupt_nmi_irqoff)
|
||||||
|
@@ -30,7 +30,7 @@ SYM_FUNC_START(atomic64_##op##_386); \
|
|||||||
|
|
||||||
#define RET_IRQ_RESTORE \
|
#define RET_IRQ_RESTORE \
|
||||||
IRQ_RESTORE v; \
|
IRQ_RESTORE v; \
|
||||||
ret
|
RET
|
||||||
|
|
||||||
#define v %ecx
|
#define v %ecx
|
||||||
BEGIN_IRQ_SAVE(read)
|
BEGIN_IRQ_SAVE(read)
|
||||||
|
@@ -18,7 +18,7 @@
|
|||||||
|
|
||||||
SYM_FUNC_START(atomic64_read_cx8)
|
SYM_FUNC_START(atomic64_read_cx8)
|
||||||
read64 %ecx
|
read64 %ecx
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(atomic64_read_cx8)
|
SYM_FUNC_END(atomic64_read_cx8)
|
||||||
|
|
||||||
SYM_FUNC_START(atomic64_set_cx8)
|
SYM_FUNC_START(atomic64_set_cx8)
|
||||||
@@ -28,7 +28,7 @@ SYM_FUNC_START(atomic64_set_cx8)
|
|||||||
cmpxchg8b (%esi)
|
cmpxchg8b (%esi)
|
||||||
jne 1b
|
jne 1b
|
||||||
|
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(atomic64_set_cx8)
|
SYM_FUNC_END(atomic64_set_cx8)
|
||||||
|
|
||||||
SYM_FUNC_START(atomic64_xchg_cx8)
|
SYM_FUNC_START(atomic64_xchg_cx8)
|
||||||
@@ -37,7 +37,7 @@ SYM_FUNC_START(atomic64_xchg_cx8)
|
|||||||
cmpxchg8b (%esi)
|
cmpxchg8b (%esi)
|
||||||
jne 1b
|
jne 1b
|
||||||
|
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(atomic64_xchg_cx8)
|
SYM_FUNC_END(atomic64_xchg_cx8)
|
||||||
|
|
||||||
.macro addsub_return func ins insc
|
.macro addsub_return func ins insc
|
||||||
@@ -68,7 +68,7 @@ SYM_FUNC_START(atomic64_\func\()_return_cx8)
|
|||||||
popl %esi
|
popl %esi
|
||||||
popl %ebx
|
popl %ebx
|
||||||
popl %ebp
|
popl %ebp
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(atomic64_\func\()_return_cx8)
|
SYM_FUNC_END(atomic64_\func\()_return_cx8)
|
||||||
.endm
|
.endm
|
||||||
|
|
||||||
@@ -93,7 +93,7 @@ SYM_FUNC_START(atomic64_\func\()_return_cx8)
|
|||||||
movl %ebx, %eax
|
movl %ebx, %eax
|
||||||
movl %ecx, %edx
|
movl %ecx, %edx
|
||||||
popl %ebx
|
popl %ebx
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(atomic64_\func\()_return_cx8)
|
SYM_FUNC_END(atomic64_\func\()_return_cx8)
|
||||||
.endm
|
.endm
|
||||||
|
|
||||||
@@ -118,7 +118,7 @@ SYM_FUNC_START(atomic64_dec_if_positive_cx8)
|
|||||||
movl %ebx, %eax
|
movl %ebx, %eax
|
||||||
movl %ecx, %edx
|
movl %ecx, %edx
|
||||||
popl %ebx
|
popl %ebx
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(atomic64_dec_if_positive_cx8)
|
SYM_FUNC_END(atomic64_dec_if_positive_cx8)
|
||||||
|
|
||||||
SYM_FUNC_START(atomic64_add_unless_cx8)
|
SYM_FUNC_START(atomic64_add_unless_cx8)
|
||||||
@@ -149,7 +149,7 @@ SYM_FUNC_START(atomic64_add_unless_cx8)
|
|||||||
addl $8, %esp
|
addl $8, %esp
|
||||||
popl %ebx
|
popl %ebx
|
||||||
popl %ebp
|
popl %ebp
|
||||||
ret
|
RET
|
||||||
4:
|
4:
|
||||||
cmpl %edx, 4(%esp)
|
cmpl %edx, 4(%esp)
|
||||||
jne 2b
|
jne 2b
|
||||||
@@ -176,5 +176,5 @@ SYM_FUNC_START(atomic64_inc_not_zero_cx8)
|
|||||||
movl $1, %eax
|
movl $1, %eax
|
||||||
3:
|
3:
|
||||||
popl %ebx
|
popl %ebx
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(atomic64_inc_not_zero_cx8)
|
SYM_FUNC_END(atomic64_inc_not_zero_cx8)
|
||||||
|
@@ -127,7 +127,7 @@ SYM_FUNC_START(csum_partial)
|
|||||||
8:
|
8:
|
||||||
popl %ebx
|
popl %ebx
|
||||||
popl %esi
|
popl %esi
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(csum_partial)
|
SYM_FUNC_END(csum_partial)
|
||||||
|
|
||||||
#else
|
#else
|
||||||
@@ -245,7 +245,7 @@ SYM_FUNC_START(csum_partial)
|
|||||||
90:
|
90:
|
||||||
popl %ebx
|
popl %ebx
|
||||||
popl %esi
|
popl %esi
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(csum_partial)
|
SYM_FUNC_END(csum_partial)
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
@@ -371,7 +371,7 @@ EXC( movb %cl, (%edi) )
|
|||||||
popl %esi
|
popl %esi
|
||||||
popl %edi
|
popl %edi
|
||||||
popl %ecx # equivalent to addl $4,%esp
|
popl %ecx # equivalent to addl $4,%esp
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(csum_partial_copy_generic)
|
SYM_FUNC_END(csum_partial_copy_generic)
|
||||||
|
|
||||||
#else
|
#else
|
||||||
@@ -447,7 +447,7 @@ EXC( movb %dl, (%edi) )
|
|||||||
popl %esi
|
popl %esi
|
||||||
popl %edi
|
popl %edi
|
||||||
popl %ebx
|
popl %ebx
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(csum_partial_copy_generic)
|
SYM_FUNC_END(csum_partial_copy_generic)
|
||||||
|
|
||||||
#undef ROUND
|
#undef ROUND
|
||||||
|
@@ -17,7 +17,7 @@ SYM_FUNC_START(clear_page_rep)
|
|||||||
movl $4096/8,%ecx
|
movl $4096/8,%ecx
|
||||||
xorl %eax,%eax
|
xorl %eax,%eax
|
||||||
rep stosq
|
rep stosq
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(clear_page_rep)
|
SYM_FUNC_END(clear_page_rep)
|
||||||
EXPORT_SYMBOL_GPL(clear_page_rep)
|
EXPORT_SYMBOL_GPL(clear_page_rep)
|
||||||
|
|
||||||
@@ -39,7 +39,7 @@ SYM_FUNC_START(clear_page_orig)
|
|||||||
leaq 64(%rdi),%rdi
|
leaq 64(%rdi),%rdi
|
||||||
jnz .Lloop
|
jnz .Lloop
|
||||||
nop
|
nop
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(clear_page_orig)
|
SYM_FUNC_END(clear_page_orig)
|
||||||
EXPORT_SYMBOL_GPL(clear_page_orig)
|
EXPORT_SYMBOL_GPL(clear_page_orig)
|
||||||
|
|
||||||
@@ -47,6 +47,6 @@ SYM_FUNC_START(clear_page_erms)
|
|||||||
movl $4096,%ecx
|
movl $4096,%ecx
|
||||||
xorl %eax,%eax
|
xorl %eax,%eax
|
||||||
rep stosb
|
rep stosb
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(clear_page_erms)
|
SYM_FUNC_END(clear_page_erms)
|
||||||
EXPORT_SYMBOL_GPL(clear_page_erms)
|
EXPORT_SYMBOL_GPL(clear_page_erms)
|
||||||
|
@@ -37,11 +37,11 @@ SYM_FUNC_START(this_cpu_cmpxchg16b_emu)
|
|||||||
|
|
||||||
popfq
|
popfq
|
||||||
mov $1, %al
|
mov $1, %al
|
||||||
ret
|
RET
|
||||||
|
|
||||||
.Lnot_same:
|
.Lnot_same:
|
||||||
popfq
|
popfq
|
||||||
xor %al,%al
|
xor %al,%al
|
||||||
ret
|
RET
|
||||||
|
|
||||||
SYM_FUNC_END(this_cpu_cmpxchg16b_emu)
|
SYM_FUNC_END(this_cpu_cmpxchg16b_emu)
|
||||||
|
@@ -32,7 +32,7 @@ SYM_FUNC_START(cmpxchg8b_emu)
|
|||||||
movl %ecx, 4(%esi)
|
movl %ecx, 4(%esi)
|
||||||
|
|
||||||
popfl
|
popfl
|
||||||
ret
|
RET
|
||||||
|
|
||||||
.Lnot_same:
|
.Lnot_same:
|
||||||
movl (%esi), %eax
|
movl (%esi), %eax
|
||||||
@@ -40,7 +40,7 @@ SYM_FUNC_START(cmpxchg8b_emu)
|
|||||||
movl 4(%esi), %edx
|
movl 4(%esi), %edx
|
||||||
|
|
||||||
popfl
|
popfl
|
||||||
ret
|
RET
|
||||||
|
|
||||||
SYM_FUNC_END(cmpxchg8b_emu)
|
SYM_FUNC_END(cmpxchg8b_emu)
|
||||||
EXPORT_SYMBOL(cmpxchg8b_emu)
|
EXPORT_SYMBOL(cmpxchg8b_emu)
|
||||||
|
@@ -86,7 +86,7 @@ SYM_FUNC_START(copy_mc_fragile)
|
|||||||
.L_done_memcpy_trap:
|
.L_done_memcpy_trap:
|
||||||
xorl %eax, %eax
|
xorl %eax, %eax
|
||||||
.L_done:
|
.L_done:
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(copy_mc_fragile)
|
SYM_FUNC_END(copy_mc_fragile)
|
||||||
EXPORT_SYMBOL_GPL(copy_mc_fragile)
|
EXPORT_SYMBOL_GPL(copy_mc_fragile)
|
||||||
|
|
||||||
@@ -142,7 +142,7 @@ SYM_FUNC_START(copy_mc_enhanced_fast_string)
|
|||||||
rep movsb
|
rep movsb
|
||||||
/* Copy successful. Return zero */
|
/* Copy successful. Return zero */
|
||||||
xorl %eax, %eax
|
xorl %eax, %eax
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(copy_mc_enhanced_fast_string)
|
SYM_FUNC_END(copy_mc_enhanced_fast_string)
|
||||||
|
|
||||||
.section .fixup, "ax"
|
.section .fixup, "ax"
|
||||||
@@ -155,7 +155,7 @@ SYM_FUNC_END(copy_mc_enhanced_fast_string)
|
|||||||
* user-copy routines.
|
* user-copy routines.
|
||||||
*/
|
*/
|
||||||
movq %rcx, %rax
|
movq %rcx, %rax
|
||||||
ret
|
RET
|
||||||
|
|
||||||
.previous
|
.previous
|
||||||
|
|
||||||
|
@@ -17,7 +17,7 @@ SYM_FUNC_START(copy_page)
|
|||||||
ALTERNATIVE "jmp copy_page_regs", "", X86_FEATURE_REP_GOOD
|
ALTERNATIVE "jmp copy_page_regs", "", X86_FEATURE_REP_GOOD
|
||||||
movl $4096/8, %ecx
|
movl $4096/8, %ecx
|
||||||
rep movsq
|
rep movsq
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(copy_page)
|
SYM_FUNC_END(copy_page)
|
||||||
EXPORT_SYMBOL(copy_page)
|
EXPORT_SYMBOL(copy_page)
|
||||||
|
|
||||||
@@ -85,5 +85,5 @@ SYM_FUNC_START_LOCAL(copy_page_regs)
|
|||||||
movq (%rsp), %rbx
|
movq (%rsp), %rbx
|
||||||
movq 1*8(%rsp), %r12
|
movq 1*8(%rsp), %r12
|
||||||
addq $2*8, %rsp
|
addq $2*8, %rsp
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(copy_page_regs)
|
SYM_FUNC_END(copy_page_regs)
|
||||||
|
@@ -105,7 +105,7 @@ SYM_FUNC_START(copy_user_generic_unrolled)
|
|||||||
jnz 21b
|
jnz 21b
|
||||||
23: xor %eax,%eax
|
23: xor %eax,%eax
|
||||||
ASM_CLAC
|
ASM_CLAC
|
||||||
ret
|
RET
|
||||||
|
|
||||||
.section .fixup,"ax"
|
.section .fixup,"ax"
|
||||||
30: shll $6,%ecx
|
30: shll $6,%ecx
|
||||||
@@ -173,7 +173,7 @@ SYM_FUNC_START(copy_user_generic_string)
|
|||||||
movsb
|
movsb
|
||||||
xorl %eax,%eax
|
xorl %eax,%eax
|
||||||
ASM_CLAC
|
ASM_CLAC
|
||||||
ret
|
RET
|
||||||
|
|
||||||
.section .fixup,"ax"
|
.section .fixup,"ax"
|
||||||
11: leal (%rdx,%rcx,8),%ecx
|
11: leal (%rdx,%rcx,8),%ecx
|
||||||
@@ -207,7 +207,7 @@ SYM_FUNC_START(copy_user_enhanced_fast_string)
|
|||||||
movsb
|
movsb
|
||||||
xorl %eax,%eax
|
xorl %eax,%eax
|
||||||
ASM_CLAC
|
ASM_CLAC
|
||||||
ret
|
RET
|
||||||
|
|
||||||
.section .fixup,"ax"
|
.section .fixup,"ax"
|
||||||
12: movl %ecx,%edx /* ecx is zerorest also */
|
12: movl %ecx,%edx /* ecx is zerorest also */
|
||||||
@@ -239,7 +239,7 @@ SYM_CODE_START_LOCAL(.Lcopy_user_handle_tail)
|
|||||||
1: rep movsb
|
1: rep movsb
|
||||||
2: mov %ecx,%eax
|
2: mov %ecx,%eax
|
||||||
ASM_CLAC
|
ASM_CLAC
|
||||||
ret
|
RET
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Return zero to pretend that this copy succeeded. This
|
* Return zero to pretend that this copy succeeded. This
|
||||||
@@ -250,7 +250,7 @@ SYM_CODE_START_LOCAL(.Lcopy_user_handle_tail)
|
|||||||
*/
|
*/
|
||||||
3: xorl %eax,%eax
|
3: xorl %eax,%eax
|
||||||
ASM_CLAC
|
ASM_CLAC
|
||||||
ret
|
RET
|
||||||
|
|
||||||
_ASM_EXTABLE_CPY(1b, 2b)
|
_ASM_EXTABLE_CPY(1b, 2b)
|
||||||
SYM_CODE_END(.Lcopy_user_handle_tail)
|
SYM_CODE_END(.Lcopy_user_handle_tail)
|
||||||
@@ -361,7 +361,7 @@ SYM_FUNC_START(__copy_user_nocache)
|
|||||||
xorl %eax,%eax
|
xorl %eax,%eax
|
||||||
ASM_CLAC
|
ASM_CLAC
|
||||||
sfence
|
sfence
|
||||||
ret
|
RET
|
||||||
|
|
||||||
.section .fixup,"ax"
|
.section .fixup,"ax"
|
||||||
.L_fixup_4x8b_copy:
|
.L_fixup_4x8b_copy:
|
||||||
|
@@ -201,7 +201,7 @@ SYM_FUNC_START(csum_partial_copy_generic)
|
|||||||
movq 3*8(%rsp), %r13
|
movq 3*8(%rsp), %r13
|
||||||
movq 4*8(%rsp), %r15
|
movq 4*8(%rsp), %r15
|
||||||
addq $5*8, %rsp
|
addq $5*8, %rsp
|
||||||
ret
|
RET
|
||||||
.Lshort:
|
.Lshort:
|
||||||
movl %ecx, %r10d
|
movl %ecx, %r10d
|
||||||
jmp .L1
|
jmp .L1
|
||||||
|
@@ -57,7 +57,7 @@ SYM_FUNC_START(__get_user_1)
|
|||||||
1: movzbl (%_ASM_AX),%edx
|
1: movzbl (%_ASM_AX),%edx
|
||||||
xor %eax,%eax
|
xor %eax,%eax
|
||||||
ASM_CLAC
|
ASM_CLAC
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(__get_user_1)
|
SYM_FUNC_END(__get_user_1)
|
||||||
EXPORT_SYMBOL(__get_user_1)
|
EXPORT_SYMBOL(__get_user_1)
|
||||||
|
|
||||||
@@ -71,7 +71,7 @@ SYM_FUNC_START(__get_user_2)
|
|||||||
2: movzwl (%_ASM_AX),%edx
|
2: movzwl (%_ASM_AX),%edx
|
||||||
xor %eax,%eax
|
xor %eax,%eax
|
||||||
ASM_CLAC
|
ASM_CLAC
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(__get_user_2)
|
SYM_FUNC_END(__get_user_2)
|
||||||
EXPORT_SYMBOL(__get_user_2)
|
EXPORT_SYMBOL(__get_user_2)
|
||||||
|
|
||||||
@@ -85,7 +85,7 @@ SYM_FUNC_START(__get_user_4)
|
|||||||
3: movl (%_ASM_AX),%edx
|
3: movl (%_ASM_AX),%edx
|
||||||
xor %eax,%eax
|
xor %eax,%eax
|
||||||
ASM_CLAC
|
ASM_CLAC
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(__get_user_4)
|
SYM_FUNC_END(__get_user_4)
|
||||||
EXPORT_SYMBOL(__get_user_4)
|
EXPORT_SYMBOL(__get_user_4)
|
||||||
|
|
||||||
@@ -100,7 +100,7 @@ SYM_FUNC_START(__get_user_8)
|
|||||||
4: movq (%_ASM_AX),%rdx
|
4: movq (%_ASM_AX),%rdx
|
||||||
xor %eax,%eax
|
xor %eax,%eax
|
||||||
ASM_CLAC
|
ASM_CLAC
|
||||||
ret
|
RET
|
||||||
#else
|
#else
|
||||||
LOAD_TASK_SIZE_MINUS_N(7)
|
LOAD_TASK_SIZE_MINUS_N(7)
|
||||||
cmp %_ASM_DX,%_ASM_AX
|
cmp %_ASM_DX,%_ASM_AX
|
||||||
@@ -112,7 +112,7 @@ SYM_FUNC_START(__get_user_8)
|
|||||||
5: movl 4(%_ASM_AX),%ecx
|
5: movl 4(%_ASM_AX),%ecx
|
||||||
xor %eax,%eax
|
xor %eax,%eax
|
||||||
ASM_CLAC
|
ASM_CLAC
|
||||||
ret
|
RET
|
||||||
#endif
|
#endif
|
||||||
SYM_FUNC_END(__get_user_8)
|
SYM_FUNC_END(__get_user_8)
|
||||||
EXPORT_SYMBOL(__get_user_8)
|
EXPORT_SYMBOL(__get_user_8)
|
||||||
@@ -124,7 +124,7 @@ SYM_FUNC_START(__get_user_nocheck_1)
|
|||||||
6: movzbl (%_ASM_AX),%edx
|
6: movzbl (%_ASM_AX),%edx
|
||||||
xor %eax,%eax
|
xor %eax,%eax
|
||||||
ASM_CLAC
|
ASM_CLAC
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(__get_user_nocheck_1)
|
SYM_FUNC_END(__get_user_nocheck_1)
|
||||||
EXPORT_SYMBOL(__get_user_nocheck_1)
|
EXPORT_SYMBOL(__get_user_nocheck_1)
|
||||||
|
|
||||||
@@ -134,7 +134,7 @@ SYM_FUNC_START(__get_user_nocheck_2)
|
|||||||
7: movzwl (%_ASM_AX),%edx
|
7: movzwl (%_ASM_AX),%edx
|
||||||
xor %eax,%eax
|
xor %eax,%eax
|
||||||
ASM_CLAC
|
ASM_CLAC
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(__get_user_nocheck_2)
|
SYM_FUNC_END(__get_user_nocheck_2)
|
||||||
EXPORT_SYMBOL(__get_user_nocheck_2)
|
EXPORT_SYMBOL(__get_user_nocheck_2)
|
||||||
|
|
||||||
@@ -144,7 +144,7 @@ SYM_FUNC_START(__get_user_nocheck_4)
|
|||||||
8: movl (%_ASM_AX),%edx
|
8: movl (%_ASM_AX),%edx
|
||||||
xor %eax,%eax
|
xor %eax,%eax
|
||||||
ASM_CLAC
|
ASM_CLAC
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(__get_user_nocheck_4)
|
SYM_FUNC_END(__get_user_nocheck_4)
|
||||||
EXPORT_SYMBOL(__get_user_nocheck_4)
|
EXPORT_SYMBOL(__get_user_nocheck_4)
|
||||||
|
|
||||||
@@ -159,7 +159,7 @@ SYM_FUNC_START(__get_user_nocheck_8)
|
|||||||
#endif
|
#endif
|
||||||
xor %eax,%eax
|
xor %eax,%eax
|
||||||
ASM_CLAC
|
ASM_CLAC
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(__get_user_nocheck_8)
|
SYM_FUNC_END(__get_user_nocheck_8)
|
||||||
EXPORT_SYMBOL(__get_user_nocheck_8)
|
EXPORT_SYMBOL(__get_user_nocheck_8)
|
||||||
|
|
||||||
@@ -169,7 +169,7 @@ SYM_CODE_START_LOCAL(.Lbad_get_user_clac)
|
|||||||
bad_get_user:
|
bad_get_user:
|
||||||
xor %edx,%edx
|
xor %edx,%edx
|
||||||
mov $(-EFAULT),%_ASM_AX
|
mov $(-EFAULT),%_ASM_AX
|
||||||
ret
|
RET
|
||||||
SYM_CODE_END(.Lbad_get_user_clac)
|
SYM_CODE_END(.Lbad_get_user_clac)
|
||||||
|
|
||||||
#ifdef CONFIG_X86_32
|
#ifdef CONFIG_X86_32
|
||||||
@@ -179,7 +179,7 @@ bad_get_user_8:
|
|||||||
xor %edx,%edx
|
xor %edx,%edx
|
||||||
xor %ecx,%ecx
|
xor %ecx,%ecx
|
||||||
mov $(-EFAULT),%_ASM_AX
|
mov $(-EFAULT),%_ASM_AX
|
||||||
ret
|
RET
|
||||||
SYM_CODE_END(.Lbad_get_user_8_clac)
|
SYM_CODE_END(.Lbad_get_user_8_clac)
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
@@ -32,7 +32,7 @@ SYM_FUNC_START(__sw_hweight32)
|
|||||||
imull $0x01010101, %eax, %eax # w_tmp *= 0x01010101
|
imull $0x01010101, %eax, %eax # w_tmp *= 0x01010101
|
||||||
shrl $24, %eax # w = w_tmp >> 24
|
shrl $24, %eax # w = w_tmp >> 24
|
||||||
__ASM_SIZE(pop,) %__ASM_REG(dx)
|
__ASM_SIZE(pop,) %__ASM_REG(dx)
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(__sw_hweight32)
|
SYM_FUNC_END(__sw_hweight32)
|
||||||
EXPORT_SYMBOL(__sw_hweight32)
|
EXPORT_SYMBOL(__sw_hweight32)
|
||||||
|
|
||||||
@@ -65,7 +65,7 @@ SYM_FUNC_START(__sw_hweight64)
|
|||||||
|
|
||||||
popq %rdx
|
popq %rdx
|
||||||
popq %rdi
|
popq %rdi
|
||||||
ret
|
RET
|
||||||
#else /* CONFIG_X86_32 */
|
#else /* CONFIG_X86_32 */
|
||||||
/* We're getting an u64 arg in (%eax,%edx): unsigned long hweight64(__u64 w) */
|
/* We're getting an u64 arg in (%eax,%edx): unsigned long hweight64(__u64 w) */
|
||||||
pushl %ecx
|
pushl %ecx
|
||||||
@@ -77,7 +77,7 @@ SYM_FUNC_START(__sw_hweight64)
|
|||||||
addl %ecx, %eax # result
|
addl %ecx, %eax # result
|
||||||
|
|
||||||
popl %ecx
|
popl %ecx
|
||||||
ret
|
RET
|
||||||
#endif
|
#endif
|
||||||
SYM_FUNC_END(__sw_hweight64)
|
SYM_FUNC_END(__sw_hweight64)
|
||||||
EXPORT_SYMBOL(__sw_hweight64)
|
EXPORT_SYMBOL(__sw_hweight64)
|
||||||
|
@@ -11,5 +11,5 @@
|
|||||||
SYM_FUNC_START(__iowrite32_copy)
|
SYM_FUNC_START(__iowrite32_copy)
|
||||||
movl %edx,%ecx
|
movl %edx,%ecx
|
||||||
rep movsd
|
rep movsd
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(__iowrite32_copy)
|
SYM_FUNC_END(__iowrite32_copy)
|
||||||
|
@@ -39,7 +39,7 @@ SYM_FUNC_START_WEAK(memcpy)
|
|||||||
rep movsq
|
rep movsq
|
||||||
movl %edx, %ecx
|
movl %edx, %ecx
|
||||||
rep movsb
|
rep movsb
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(memcpy)
|
SYM_FUNC_END(memcpy)
|
||||||
SYM_FUNC_END_ALIAS(__memcpy)
|
SYM_FUNC_END_ALIAS(__memcpy)
|
||||||
EXPORT_SYMBOL(memcpy)
|
EXPORT_SYMBOL(memcpy)
|
||||||
@@ -53,7 +53,7 @@ SYM_FUNC_START_LOCAL(memcpy_erms)
|
|||||||
movq %rdi, %rax
|
movq %rdi, %rax
|
||||||
movq %rdx, %rcx
|
movq %rdx, %rcx
|
||||||
rep movsb
|
rep movsb
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(memcpy_erms)
|
SYM_FUNC_END(memcpy_erms)
|
||||||
|
|
||||||
SYM_FUNC_START_LOCAL(memcpy_orig)
|
SYM_FUNC_START_LOCAL(memcpy_orig)
|
||||||
@@ -137,7 +137,7 @@ SYM_FUNC_START_LOCAL(memcpy_orig)
|
|||||||
movq %r9, 1*8(%rdi)
|
movq %r9, 1*8(%rdi)
|
||||||
movq %r10, -2*8(%rdi, %rdx)
|
movq %r10, -2*8(%rdi, %rdx)
|
||||||
movq %r11, -1*8(%rdi, %rdx)
|
movq %r11, -1*8(%rdi, %rdx)
|
||||||
retq
|
RET
|
||||||
.p2align 4
|
.p2align 4
|
||||||
.Lless_16bytes:
|
.Lless_16bytes:
|
||||||
cmpl $8, %edx
|
cmpl $8, %edx
|
||||||
@@ -149,7 +149,7 @@ SYM_FUNC_START_LOCAL(memcpy_orig)
|
|||||||
movq -1*8(%rsi, %rdx), %r9
|
movq -1*8(%rsi, %rdx), %r9
|
||||||
movq %r8, 0*8(%rdi)
|
movq %r8, 0*8(%rdi)
|
||||||
movq %r9, -1*8(%rdi, %rdx)
|
movq %r9, -1*8(%rdi, %rdx)
|
||||||
retq
|
RET
|
||||||
.p2align 4
|
.p2align 4
|
||||||
.Lless_8bytes:
|
.Lless_8bytes:
|
||||||
cmpl $4, %edx
|
cmpl $4, %edx
|
||||||
@@ -162,7 +162,7 @@ SYM_FUNC_START_LOCAL(memcpy_orig)
|
|||||||
movl -4(%rsi, %rdx), %r8d
|
movl -4(%rsi, %rdx), %r8d
|
||||||
movl %ecx, (%rdi)
|
movl %ecx, (%rdi)
|
||||||
movl %r8d, -4(%rdi, %rdx)
|
movl %r8d, -4(%rdi, %rdx)
|
||||||
retq
|
RET
|
||||||
.p2align 4
|
.p2align 4
|
||||||
.Lless_3bytes:
|
.Lless_3bytes:
|
||||||
subl $1, %edx
|
subl $1, %edx
|
||||||
@@ -180,7 +180,7 @@ SYM_FUNC_START_LOCAL(memcpy_orig)
|
|||||||
movb %cl, (%rdi)
|
movb %cl, (%rdi)
|
||||||
|
|
||||||
.Lend:
|
.Lend:
|
||||||
retq
|
RET
|
||||||
SYM_FUNC_END(memcpy_orig)
|
SYM_FUNC_END(memcpy_orig)
|
||||||
|
|
||||||
.popsection
|
.popsection
|
||||||
|
@@ -40,7 +40,7 @@ SYM_FUNC_START(__memmove)
|
|||||||
/* FSRM implies ERMS => no length checks, do the copy directly */
|
/* FSRM implies ERMS => no length checks, do the copy directly */
|
||||||
.Lmemmove_begin_forward:
|
.Lmemmove_begin_forward:
|
||||||
ALTERNATIVE "cmp $0x20, %rdx; jb 1f", "", X86_FEATURE_FSRM
|
ALTERNATIVE "cmp $0x20, %rdx; jb 1f", "", X86_FEATURE_FSRM
|
||||||
ALTERNATIVE "", "movq %rdx, %rcx; rep movsb; retq", X86_FEATURE_ERMS
|
ALTERNATIVE "", "movq %rdx, %rcx; rep movsb; RET", X86_FEATURE_ERMS
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* movsq instruction have many startup latency
|
* movsq instruction have many startup latency
|
||||||
@@ -205,7 +205,7 @@ SYM_FUNC_START(__memmove)
|
|||||||
movb (%rsi), %r11b
|
movb (%rsi), %r11b
|
||||||
movb %r11b, (%rdi)
|
movb %r11b, (%rdi)
|
||||||
13:
|
13:
|
||||||
retq
|
RET
|
||||||
SYM_FUNC_END(__memmove)
|
SYM_FUNC_END(__memmove)
|
||||||
SYM_FUNC_END_ALIAS(memmove)
|
SYM_FUNC_END_ALIAS(memmove)
|
||||||
EXPORT_SYMBOL(__memmove)
|
EXPORT_SYMBOL(__memmove)
|
||||||
|
@@ -40,7 +40,7 @@ SYM_FUNC_START(__memset)
|
|||||||
movl %edx,%ecx
|
movl %edx,%ecx
|
||||||
rep stosb
|
rep stosb
|
||||||
movq %r9,%rax
|
movq %r9,%rax
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(__memset)
|
SYM_FUNC_END(__memset)
|
||||||
SYM_FUNC_END_ALIAS(memset)
|
SYM_FUNC_END_ALIAS(memset)
|
||||||
EXPORT_SYMBOL(memset)
|
EXPORT_SYMBOL(memset)
|
||||||
@@ -63,7 +63,7 @@ SYM_FUNC_START_LOCAL(memset_erms)
|
|||||||
movq %rdx,%rcx
|
movq %rdx,%rcx
|
||||||
rep stosb
|
rep stosb
|
||||||
movq %r9,%rax
|
movq %r9,%rax
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(memset_erms)
|
SYM_FUNC_END(memset_erms)
|
||||||
|
|
||||||
SYM_FUNC_START_LOCAL(memset_orig)
|
SYM_FUNC_START_LOCAL(memset_orig)
|
||||||
@@ -125,7 +125,7 @@ SYM_FUNC_START_LOCAL(memset_orig)
|
|||||||
|
|
||||||
.Lende:
|
.Lende:
|
||||||
movq %r10,%rax
|
movq %r10,%rax
|
||||||
ret
|
RET
|
||||||
|
|
||||||
.Lbad_alignment:
|
.Lbad_alignment:
|
||||||
cmpq $7,%rdx
|
cmpq $7,%rdx
|
||||||
|
@@ -35,7 +35,7 @@ SYM_FUNC_START(\op\()_safe_regs)
|
|||||||
movl %edi, 28(%r10)
|
movl %edi, 28(%r10)
|
||||||
popq %r12
|
popq %r12
|
||||||
popq %rbx
|
popq %rbx
|
||||||
ret
|
RET
|
||||||
3:
|
3:
|
||||||
movl $-EIO, %r11d
|
movl $-EIO, %r11d
|
||||||
jmp 2b
|
jmp 2b
|
||||||
@@ -77,7 +77,7 @@ SYM_FUNC_START(\op\()_safe_regs)
|
|||||||
popl %esi
|
popl %esi
|
||||||
popl %ebp
|
popl %ebp
|
||||||
popl %ebx
|
popl %ebx
|
||||||
ret
|
RET
|
||||||
3:
|
3:
|
||||||
movl $-EIO, 4(%esp)
|
movl $-EIO, 4(%esp)
|
||||||
jmp 2b
|
jmp 2b
|
||||||
|
@@ -52,7 +52,7 @@ SYM_INNER_LABEL(__put_user_nocheck_1, SYM_L_GLOBAL)
|
|||||||
1: movb %al,(%_ASM_CX)
|
1: movb %al,(%_ASM_CX)
|
||||||
xor %ecx,%ecx
|
xor %ecx,%ecx
|
||||||
ASM_CLAC
|
ASM_CLAC
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(__put_user_1)
|
SYM_FUNC_END(__put_user_1)
|
||||||
EXPORT_SYMBOL(__put_user_1)
|
EXPORT_SYMBOL(__put_user_1)
|
||||||
EXPORT_SYMBOL(__put_user_nocheck_1)
|
EXPORT_SYMBOL(__put_user_nocheck_1)
|
||||||
@@ -66,7 +66,7 @@ SYM_INNER_LABEL(__put_user_nocheck_2, SYM_L_GLOBAL)
|
|||||||
2: movw %ax,(%_ASM_CX)
|
2: movw %ax,(%_ASM_CX)
|
||||||
xor %ecx,%ecx
|
xor %ecx,%ecx
|
||||||
ASM_CLAC
|
ASM_CLAC
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(__put_user_2)
|
SYM_FUNC_END(__put_user_2)
|
||||||
EXPORT_SYMBOL(__put_user_2)
|
EXPORT_SYMBOL(__put_user_2)
|
||||||
EXPORT_SYMBOL(__put_user_nocheck_2)
|
EXPORT_SYMBOL(__put_user_nocheck_2)
|
||||||
@@ -80,7 +80,7 @@ SYM_INNER_LABEL(__put_user_nocheck_4, SYM_L_GLOBAL)
|
|||||||
3: movl %eax,(%_ASM_CX)
|
3: movl %eax,(%_ASM_CX)
|
||||||
xor %ecx,%ecx
|
xor %ecx,%ecx
|
||||||
ASM_CLAC
|
ASM_CLAC
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(__put_user_4)
|
SYM_FUNC_END(__put_user_4)
|
||||||
EXPORT_SYMBOL(__put_user_4)
|
EXPORT_SYMBOL(__put_user_4)
|
||||||
EXPORT_SYMBOL(__put_user_nocheck_4)
|
EXPORT_SYMBOL(__put_user_nocheck_4)
|
||||||
|
@@ -23,7 +23,7 @@
|
|||||||
.Ldo_rop_\@:
|
.Ldo_rop_\@:
|
||||||
mov %\reg, (%_ASM_SP)
|
mov %\reg, (%_ASM_SP)
|
||||||
UNWIND_HINT_FUNC
|
UNWIND_HINT_FUNC
|
||||||
ret
|
RET
|
||||||
.endm
|
.endm
|
||||||
|
|
||||||
.macro THUNK reg
|
.macro THUNK reg
|
||||||
|
@@ -341,7 +341,7 @@ L_exit:
|
|||||||
popl %esi
|
popl %esi
|
||||||
|
|
||||||
leave
|
leave
|
||||||
ret
|
RET
|
||||||
|
|
||||||
|
|
||||||
#ifdef PARANOID
|
#ifdef PARANOID
|
||||||
|
@@ -44,5 +44,5 @@ SYM_FUNC_START(FPU_div_small)
|
|||||||
popl %esi
|
popl %esi
|
||||||
|
|
||||||
leave
|
leave
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(FPU_div_small)
|
SYM_FUNC_END(FPU_div_small)
|
||||||
|
@@ -62,7 +62,7 @@ SYM_FUNC_START(mul32_Xsig)
|
|||||||
|
|
||||||
popl %esi
|
popl %esi
|
||||||
leave
|
leave
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(mul32_Xsig)
|
SYM_FUNC_END(mul32_Xsig)
|
||||||
|
|
||||||
|
|
||||||
@@ -115,7 +115,7 @@ SYM_FUNC_START(mul64_Xsig)
|
|||||||
|
|
||||||
popl %esi
|
popl %esi
|
||||||
leave
|
leave
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(mul64_Xsig)
|
SYM_FUNC_END(mul64_Xsig)
|
||||||
|
|
||||||
|
|
||||||
@@ -175,5 +175,5 @@ SYM_FUNC_START(mul_Xsig_Xsig)
|
|||||||
|
|
||||||
popl %esi
|
popl %esi
|
||||||
leave
|
leave
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(mul_Xsig_Xsig)
|
SYM_FUNC_END(mul_Xsig_Xsig)
|
||||||
|
@@ -133,5 +133,5 @@ L_accum_done:
|
|||||||
popl %edi
|
popl %edi
|
||||||
popl %esi
|
popl %esi
|
||||||
leave
|
leave
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(polynomial_Xsig)
|
SYM_FUNC_END(polynomial_Xsig)
|
||||||
|
@@ -72,7 +72,7 @@ L_exit_valid:
|
|||||||
L_exit:
|
L_exit:
|
||||||
popl %ebx
|
popl %ebx
|
||||||
leave
|
leave
|
||||||
ret
|
RET
|
||||||
|
|
||||||
|
|
||||||
L_zero:
|
L_zero:
|
||||||
@@ -138,7 +138,7 @@ L_exit_nuo_valid:
|
|||||||
|
|
||||||
popl %ebx
|
popl %ebx
|
||||||
leave
|
leave
|
||||||
ret
|
RET
|
||||||
|
|
||||||
L_exit_nuo_zero:
|
L_exit_nuo_zero:
|
||||||
movl TAG_Zero,%eax
|
movl TAG_Zero,%eax
|
||||||
@@ -146,5 +146,5 @@ L_exit_nuo_zero:
|
|||||||
|
|
||||||
popl %ebx
|
popl %ebx
|
||||||
leave
|
leave
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(FPU_normalize_nuo)
|
SYM_FUNC_END(FPU_normalize_nuo)
|
||||||
|
@@ -437,7 +437,7 @@ fpu_Arith_exit:
|
|||||||
popl %edi
|
popl %edi
|
||||||
popl %esi
|
popl %esi
|
||||||
leave
|
leave
|
||||||
ret
|
RET
|
||||||
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
@@ -164,6 +164,6 @@ L_exit:
|
|||||||
popl %edi
|
popl %edi
|
||||||
popl %esi
|
popl %esi
|
||||||
leave
|
leave
|
||||||
ret
|
RET
|
||||||
#endif /* PARANOID */
|
#endif /* PARANOID */
|
||||||
SYM_FUNC_END(FPU_u_add)
|
SYM_FUNC_END(FPU_u_add)
|
||||||
|
@@ -468,7 +468,7 @@ L_exit:
|
|||||||
popl %esi
|
popl %esi
|
||||||
|
|
||||||
leave
|
leave
|
||||||
ret
|
RET
|
||||||
#endif /* PARANOID */
|
#endif /* PARANOID */
|
||||||
|
|
||||||
SYM_FUNC_END(FPU_u_div)
|
SYM_FUNC_END(FPU_u_div)
|
||||||
|
@@ -144,7 +144,7 @@ L_exit:
|
|||||||
popl %edi
|
popl %edi
|
||||||
popl %esi
|
popl %esi
|
||||||
leave
|
leave
|
||||||
ret
|
RET
|
||||||
#endif /* PARANOID */
|
#endif /* PARANOID */
|
||||||
|
|
||||||
SYM_FUNC_END(FPU_u_mul)
|
SYM_FUNC_END(FPU_u_mul)
|
||||||
|
@@ -270,5 +270,5 @@ L_exit:
|
|||||||
popl %edi
|
popl %edi
|
||||||
popl %esi
|
popl %esi
|
||||||
leave
|
leave
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(FPU_u_sub)
|
SYM_FUNC_END(FPU_u_sub)
|
||||||
|
@@ -78,7 +78,7 @@ L_exit:
|
|||||||
popl %esi
|
popl %esi
|
||||||
popl %ebx
|
popl %ebx
|
||||||
leave
|
leave
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(round_Xsig)
|
SYM_FUNC_END(round_Xsig)
|
||||||
|
|
||||||
|
|
||||||
@@ -138,5 +138,5 @@ L_n_exit:
|
|||||||
popl %esi
|
popl %esi
|
||||||
popl %ebx
|
popl %ebx
|
||||||
leave
|
leave
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(norm_Xsig)
|
SYM_FUNC_END(norm_Xsig)
|
||||||
|
@@ -45,7 +45,7 @@ SYM_FUNC_START(shr_Xsig)
|
|||||||
popl %ebx
|
popl %ebx
|
||||||
popl %esi
|
popl %esi
|
||||||
leave
|
leave
|
||||||
ret
|
RET
|
||||||
|
|
||||||
L_more_than_31:
|
L_more_than_31:
|
||||||
cmpl $64,%ecx
|
cmpl $64,%ecx
|
||||||
@@ -61,7 +61,7 @@ L_more_than_31:
|
|||||||
movl $0,8(%esi)
|
movl $0,8(%esi)
|
||||||
popl %esi
|
popl %esi
|
||||||
leave
|
leave
|
||||||
ret
|
RET
|
||||||
|
|
||||||
L_more_than_63:
|
L_more_than_63:
|
||||||
cmpl $96,%ecx
|
cmpl $96,%ecx
|
||||||
@@ -76,7 +76,7 @@ L_more_than_63:
|
|||||||
movl %edx,8(%esi)
|
movl %edx,8(%esi)
|
||||||
popl %esi
|
popl %esi
|
||||||
leave
|
leave
|
||||||
ret
|
RET
|
||||||
|
|
||||||
L_more_than_95:
|
L_more_than_95:
|
||||||
xorl %eax,%eax
|
xorl %eax,%eax
|
||||||
@@ -85,5 +85,5 @@ L_more_than_95:
|
|||||||
movl %eax,8(%esi)
|
movl %eax,8(%esi)
|
||||||
popl %esi
|
popl %esi
|
||||||
leave
|
leave
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(shr_Xsig)
|
SYM_FUNC_END(shr_Xsig)
|
||||||
|
@@ -55,7 +55,7 @@ SYM_FUNC_START(FPU_shrx)
|
|||||||
popl %ebx
|
popl %ebx
|
||||||
popl %esi
|
popl %esi
|
||||||
leave
|
leave
|
||||||
ret
|
RET
|
||||||
|
|
||||||
L_more_than_31:
|
L_more_than_31:
|
||||||
cmpl $64,%ecx
|
cmpl $64,%ecx
|
||||||
@@ -70,7 +70,7 @@ L_more_than_31:
|
|||||||
movl $0,4(%esi)
|
movl $0,4(%esi)
|
||||||
popl %esi
|
popl %esi
|
||||||
leave
|
leave
|
||||||
ret
|
RET
|
||||||
|
|
||||||
L_more_than_63:
|
L_more_than_63:
|
||||||
cmpl $96,%ecx
|
cmpl $96,%ecx
|
||||||
@@ -84,7 +84,7 @@ L_more_than_63:
|
|||||||
movl %edx,4(%esi)
|
movl %edx,4(%esi)
|
||||||
popl %esi
|
popl %esi
|
||||||
leave
|
leave
|
||||||
ret
|
RET
|
||||||
|
|
||||||
L_more_than_95:
|
L_more_than_95:
|
||||||
xorl %eax,%eax
|
xorl %eax,%eax
|
||||||
@@ -92,7 +92,7 @@ L_more_than_95:
|
|||||||
movl %eax,4(%esi)
|
movl %eax,4(%esi)
|
||||||
popl %esi
|
popl %esi
|
||||||
leave
|
leave
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(FPU_shrx)
|
SYM_FUNC_END(FPU_shrx)
|
||||||
|
|
||||||
|
|
||||||
@@ -146,7 +146,7 @@ SYM_FUNC_START(FPU_shrxs)
|
|||||||
popl %ebx
|
popl %ebx
|
||||||
popl %esi
|
popl %esi
|
||||||
leave
|
leave
|
||||||
ret
|
RET
|
||||||
|
|
||||||
/* Shift by [0..31] bits */
|
/* Shift by [0..31] bits */
|
||||||
Ls_less_than_32:
|
Ls_less_than_32:
|
||||||
@@ -163,7 +163,7 @@ Ls_less_than_32:
|
|||||||
popl %ebx
|
popl %ebx
|
||||||
popl %esi
|
popl %esi
|
||||||
leave
|
leave
|
||||||
ret
|
RET
|
||||||
|
|
||||||
/* Shift by [64..95] bits */
|
/* Shift by [64..95] bits */
|
||||||
Ls_more_than_63:
|
Ls_more_than_63:
|
||||||
@@ -189,7 +189,7 @@ Ls_more_than_63:
|
|||||||
popl %ebx
|
popl %ebx
|
||||||
popl %esi
|
popl %esi
|
||||||
leave
|
leave
|
||||||
ret
|
RET
|
||||||
|
|
||||||
Ls_more_than_95:
|
Ls_more_than_95:
|
||||||
/* Shift by [96..inf) bits */
|
/* Shift by [96..inf) bits */
|
||||||
@@ -203,5 +203,5 @@ Ls_more_than_95:
|
|||||||
popl %ebx
|
popl %ebx
|
||||||
popl %esi
|
popl %esi
|
||||||
leave
|
leave
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(FPU_shrxs)
|
SYM_FUNC_END(FPU_shrxs)
|
||||||
|
@@ -65,7 +65,7 @@ SYM_FUNC_START(sme_encrypt_execute)
|
|||||||
movq %rbp, %rsp /* Restore original stack pointer */
|
movq %rbp, %rsp /* Restore original stack pointer */
|
||||||
pop %rbp
|
pop %rbp
|
||||||
|
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(sme_encrypt_execute)
|
SYM_FUNC_END(sme_encrypt_execute)
|
||||||
|
|
||||||
SYM_FUNC_START(__enc_copy)
|
SYM_FUNC_START(__enc_copy)
|
||||||
@@ -151,6 +151,6 @@ SYM_FUNC_START(__enc_copy)
|
|||||||
pop %r12
|
pop %r12
|
||||||
pop %r15
|
pop %r15
|
||||||
|
|
||||||
ret
|
RET
|
||||||
.L__enc_copy_end:
|
.L__enc_copy_end:
|
||||||
SYM_FUNC_END(__enc_copy)
|
SYM_FUNC_END(__enc_copy)
|
||||||
|
@@ -56,5 +56,5 @@ SYM_FUNC_START(efi_call_svam)
|
|||||||
|
|
||||||
movl 16(%esp), %ebx
|
movl 16(%esp), %ebx
|
||||||
leave
|
leave
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(efi_call_svam)
|
SYM_FUNC_END(efi_call_svam)
|
||||||
|
@@ -23,5 +23,5 @@ SYM_FUNC_START(__efi_call)
|
|||||||
mov %rsi, %rcx
|
mov %rsi, %rcx
|
||||||
CALL_NOSPEC rdi
|
CALL_NOSPEC rdi
|
||||||
leave
|
leave
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(__efi_call)
|
SYM_FUNC_END(__efi_call)
|
||||||
|
@@ -63,7 +63,7 @@ SYM_CODE_START(__efi64_thunk)
|
|||||||
1: movq 24(%rsp), %rsp
|
1: movq 24(%rsp), %rsp
|
||||||
pop %rbx
|
pop %rbx
|
||||||
pop %rbp
|
pop %rbp
|
||||||
retq
|
RET
|
||||||
|
|
||||||
.code32
|
.code32
|
||||||
2: pushl $__KERNEL_CS
|
2: pushl $__KERNEL_CS
|
||||||
|
@@ -77,7 +77,7 @@ save_registers:
|
|||||||
pushfl
|
pushfl
|
||||||
popl saved_context_eflags
|
popl saved_context_eflags
|
||||||
|
|
||||||
ret
|
RET
|
||||||
|
|
||||||
restore_registers:
|
restore_registers:
|
||||||
movl saved_context_ebp, %ebp
|
movl saved_context_ebp, %ebp
|
||||||
@@ -88,7 +88,7 @@ restore_registers:
|
|||||||
pushl saved_context_eflags
|
pushl saved_context_eflags
|
||||||
popfl
|
popfl
|
||||||
|
|
||||||
ret
|
RET
|
||||||
|
|
||||||
SYM_CODE_START(do_olpc_suspend_lowlevel)
|
SYM_CODE_START(do_olpc_suspend_lowlevel)
|
||||||
call save_processor_state
|
call save_processor_state
|
||||||
@@ -109,7 +109,7 @@ ret_point:
|
|||||||
|
|
||||||
call restore_registers
|
call restore_registers
|
||||||
call restore_processor_state
|
call restore_processor_state
|
||||||
ret
|
RET
|
||||||
SYM_CODE_END(do_olpc_suspend_lowlevel)
|
SYM_CODE_END(do_olpc_suspend_lowlevel)
|
||||||
|
|
||||||
.data
|
.data
|
||||||
|
@@ -32,7 +32,7 @@ SYM_FUNC_START(swsusp_arch_suspend)
|
|||||||
FRAME_BEGIN
|
FRAME_BEGIN
|
||||||
call swsusp_save
|
call swsusp_save
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(swsusp_arch_suspend)
|
SYM_FUNC_END(swsusp_arch_suspend)
|
||||||
|
|
||||||
SYM_CODE_START(restore_image)
|
SYM_CODE_START(restore_image)
|
||||||
@@ -108,5 +108,5 @@ SYM_FUNC_START(restore_registers)
|
|||||||
/* tell the hibernation core that we've just restored the memory */
|
/* tell the hibernation core that we've just restored the memory */
|
||||||
movl %eax, in_suspend
|
movl %eax, in_suspend
|
||||||
|
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(restore_registers)
|
SYM_FUNC_END(restore_registers)
|
||||||
|
@@ -49,7 +49,7 @@ SYM_FUNC_START(swsusp_arch_suspend)
|
|||||||
FRAME_BEGIN
|
FRAME_BEGIN
|
||||||
call swsusp_save
|
call swsusp_save
|
||||||
FRAME_END
|
FRAME_END
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(swsusp_arch_suspend)
|
SYM_FUNC_END(swsusp_arch_suspend)
|
||||||
|
|
||||||
SYM_CODE_START(restore_image)
|
SYM_CODE_START(restore_image)
|
||||||
@@ -143,5 +143,5 @@ SYM_FUNC_START(restore_registers)
|
|||||||
/* tell the hibernation core that we've just restored the memory */
|
/* tell the hibernation core that we've just restored the memory */
|
||||||
movq %rax, in_suspend(%rip)
|
movq %rax, in_suspend(%rip)
|
||||||
|
|
||||||
ret
|
RET
|
||||||
SYM_FUNC_END(restore_registers)
|
SYM_FUNC_END(restore_registers)
|
||||||
|
@@ -110,7 +110,7 @@ csum_partial:
|
|||||||
7:
|
7:
|
||||||
popl %ebx
|
popl %ebx
|
||||||
popl %esi
|
popl %esi
|
||||||
ret
|
RET
|
||||||
|
|
||||||
#else
|
#else
|
||||||
|
|
||||||
@@ -208,7 +208,7 @@ csum_partial:
|
|||||||
80:
|
80:
|
||||||
popl %ebx
|
popl %ebx
|
||||||
popl %esi
|
popl %esi
|
||||||
ret
|
RET
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
EXPORT_SYMBOL(csum_partial)
|
EXPORT_SYMBOL(csum_partial)
|
||||||
|
@@ -34,7 +34,7 @@ kernel_setjmp:
|
|||||||
movl %esi,12(%edx)
|
movl %esi,12(%edx)
|
||||||
movl %edi,16(%edx)
|
movl %edi,16(%edx)
|
||||||
movl %ecx,20(%edx) # Return address
|
movl %ecx,20(%edx) # Return address
|
||||||
ret
|
RET
|
||||||
|
|
||||||
.size kernel_setjmp,.-kernel_setjmp
|
.size kernel_setjmp,.-kernel_setjmp
|
||||||
|
|
||||||
|
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user