| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160 |
- /* SPDX-License-Identifier: GPL-2.0-only */
- /*
- * Accelerated CRC32(C) using AArch64 CRC instructions
- *
- * Copyright (C) 2016 - 2018 Linaro Ltd <[email protected]>
- */
- #include <linux/linkage.h>
- #include <asm/alternative.h>
- #include <asm/assembler.h>
- .arch armv8-a+crc
- .macro byteorder, reg, be
- .if \be
- CPU_LE( rev \reg, \reg )
- .else
- CPU_BE( rev \reg, \reg )
- .endif
- .endm
- .macro byteorder16, reg, be
- .if \be
- CPU_LE( rev16 \reg, \reg )
- .else
- CPU_BE( rev16 \reg, \reg )
- .endif
- .endm
- .macro bitorder, reg, be
- .if \be
- rbit \reg, \reg
- .endif
- .endm
- .macro bitorder16, reg, be
- .if \be
- rbit \reg, \reg
- lsr \reg, \reg, #16
- .endif
- .endm
- .macro bitorder8, reg, be
- .if \be
- rbit \reg, \reg
- lsr \reg, \reg, #24
- .endif
- .endm
- .macro __crc32, c, be=0
- bitorder w0, \be
- cmp x2, #16
- b.lt 8f // less than 16 bytes
- and x7, x2, #0x1f
- and x2, x2, #~0x1f
- cbz x7, 32f // multiple of 32 bytes
- and x8, x7, #0xf
- ldp x3, x4, [x1]
- add x8, x8, x1
- add x1, x1, x7
- ldp x5, x6, [x8]
- byteorder x3, \be
- byteorder x4, \be
- byteorder x5, \be
- byteorder x6, \be
- bitorder x3, \be
- bitorder x4, \be
- bitorder x5, \be
- bitorder x6, \be
- tst x7, #8
- crc32\c\()x w8, w0, x3
- csel x3, x3, x4, eq
- csel w0, w0, w8, eq
- tst x7, #4
- lsr x4, x3, #32
- crc32\c\()w w8, w0, w3
- csel x3, x3, x4, eq
- csel w0, w0, w8, eq
- tst x7, #2
- lsr w4, w3, #16
- crc32\c\()h w8, w0, w3
- csel w3, w3, w4, eq
- csel w0, w0, w8, eq
- tst x7, #1
- crc32\c\()b w8, w0, w3
- csel w0, w0, w8, eq
- tst x7, #16
- crc32\c\()x w8, w0, x5
- crc32\c\()x w8, w8, x6
- csel w0, w0, w8, eq
- cbz x2, 0f
- 32: ldp x3, x4, [x1], #32
- sub x2, x2, #32
- ldp x5, x6, [x1, #-16]
- byteorder x3, \be
- byteorder x4, \be
- byteorder x5, \be
- byteorder x6, \be
- bitorder x3, \be
- bitorder x4, \be
- bitorder x5, \be
- bitorder x6, \be
- crc32\c\()x w0, w0, x3
- crc32\c\()x w0, w0, x4
- crc32\c\()x w0, w0, x5
- crc32\c\()x w0, w0, x6
- cbnz x2, 32b
- 0: bitorder w0, \be
- ret
- 8: tbz x2, #3, 4f
- ldr x3, [x1], #8
- byteorder x3, \be
- bitorder x3, \be
- crc32\c\()x w0, w0, x3
- 4: tbz x2, #2, 2f
- ldr w3, [x1], #4
- byteorder w3, \be
- bitorder w3, \be
- crc32\c\()w w0, w0, w3
- 2: tbz x2, #1, 1f
- ldrh w3, [x1], #2
- byteorder16 w3, \be
- bitorder16 w3, \be
- crc32\c\()h w0, w0, w3
- 1: tbz x2, #0, 0f
- ldrb w3, [x1]
- bitorder8 w3, \be
- crc32\c\()b w0, w0, w3
- 0: bitorder w0, \be
- ret
- .endm
- .align 5
- SYM_FUNC_START(crc32_le)
- alternative_if_not ARM64_HAS_CRC32
- b crc32_le_base
- alternative_else_nop_endif
- __crc32
- SYM_FUNC_END(crc32_le)
- .align 5
- SYM_FUNC_START(__crc32c_le)
- alternative_if_not ARM64_HAS_CRC32
- b __crc32c_le_base
- alternative_else_nop_endif
- __crc32 c
- SYM_FUNC_END(__crc32c_le)
- .align 5
- SYM_FUNC_START(crc32_be)
- alternative_if_not ARM64_HAS_CRC32
- b crc32_be_base
- alternative_else_nop_endif
- __crc32 be=1
- SYM_FUNC_END(crc32_be)
|
