123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444 |
- /* SPDX-License-Identifier: GPL-2.0-or-later */
- /*
- * INET An implementation of the TCP/IP protocol suite for the LINUX
- * operating system. INET is implemented using the BSD Socket
- * interface as the means of communication with the user level.
- *
- * IP/TCP/UDP checksumming routines
- *
- * Authors: Jorge Cwik, <[email protected]>
- * Arnt Gulbrandsen, <[email protected]>
- * Tom May, <[email protected]>
- * Pentium Pro/II routines:
- * Alexander Kjeldaas <[email protected]>
- * Finn Arne Gangstad <[email protected]>
- * Lots of code moved from tcp.c and ip.c; see those files
- * for more names.
- *
- * Changes: Ingo Molnar, converted csum_partial_copy() to 2.1 exception
- * handling.
- * Andi Kleen, add zeroing on error
- * converted to pure assembler
- */
- #include <linux/linkage.h>
- #include <asm/errno.h>
- #include <asm/asm.h>
- #include <asm/export.h>
- #include <asm/nospec-branch.h>
- /*
- * computes a partial checksum, e.g. for TCP/UDP fragments
- */
- /*
- unsigned int csum_partial(const unsigned char * buff, int len, unsigned int sum)
- */
-
- .text
-
- #ifndef CONFIG_X86_USE_PPRO_CHECKSUM
- /*
- * Experiments with Ethernet and SLIP connections show that buff
- * is aligned on either a 2-byte or 4-byte boundary. We get at
- * least a twofold speedup on 486 and Pentium if it is 4-byte aligned.
- * Fortunately, it is easy to convert 2-byte alignment to 4-byte
- * alignment for the unrolled loop.
- */
- SYM_FUNC_START(csum_partial)
- pushl %esi
- pushl %ebx
- movl 20(%esp),%eax # Function arg: unsigned int sum
- movl 16(%esp),%ecx # Function arg: int len
- movl 12(%esp),%esi # Function arg: unsigned char *buff
- testl $3, %esi # Check alignment.
- jz 2f # Jump if alignment is ok.
- testl $1, %esi # Check alignment.
- jz 10f # Jump if alignment is boundary of 2 bytes.
- # buf is odd
- dec %ecx
- jl 8f
- movzbl (%esi), %ebx
- adcl %ebx, %eax
- roll $8, %eax
- inc %esi
- testl $2, %esi
- jz 2f
- 10:
- subl $2, %ecx # Alignment uses up two bytes.
- jae 1f # Jump if we had at least two bytes.
- addl $2, %ecx # ecx was < 2. Deal with it.
- jmp 4f
- 1: movw (%esi), %bx
- addl $2, %esi
- addw %bx, %ax
- adcl $0, %eax
- 2:
- movl %ecx, %edx
- shrl $5, %ecx
- jz 2f
- testl %esi, %esi
- 1: movl (%esi), %ebx
- adcl %ebx, %eax
- movl 4(%esi), %ebx
- adcl %ebx, %eax
- movl 8(%esi), %ebx
- adcl %ebx, %eax
- movl 12(%esi), %ebx
- adcl %ebx, %eax
- movl 16(%esi), %ebx
- adcl %ebx, %eax
- movl 20(%esi), %ebx
- adcl %ebx, %eax
- movl 24(%esi), %ebx
- adcl %ebx, %eax
- movl 28(%esi), %ebx
- adcl %ebx, %eax
- lea 32(%esi), %esi
- dec %ecx
- jne 1b
- adcl $0, %eax
- 2: movl %edx, %ecx
- andl $0x1c, %edx
- je 4f
- shrl $2, %edx # This clears CF
- 3: adcl (%esi), %eax
- lea 4(%esi), %esi
- dec %edx
- jne 3b
- adcl $0, %eax
- 4: andl $3, %ecx
- jz 7f
- cmpl $2, %ecx
- jb 5f
- movw (%esi),%cx
- leal 2(%esi),%esi
- je 6f
- shll $16,%ecx
- 5: movb (%esi),%cl
- 6: addl %ecx,%eax
- adcl $0, %eax
- 7:
- testb $1, 12(%esp)
- jz 8f
- roll $8, %eax
- 8:
- popl %ebx
- popl %esi
- RET
- SYM_FUNC_END(csum_partial)
- #else
- /* Version for PentiumII/PPro */
- SYM_FUNC_START(csum_partial)
- pushl %esi
- pushl %ebx
- movl 20(%esp),%eax # Function arg: unsigned int sum
- movl 16(%esp),%ecx # Function arg: int len
- movl 12(%esp),%esi # Function arg: const unsigned char *buf
- testl $3, %esi
- jnz 25f
- 10:
- movl %ecx, %edx
- movl %ecx, %ebx
- andl $0x7c, %ebx
- shrl $7, %ecx
- addl %ebx,%esi
- shrl $2, %ebx
- negl %ebx
- lea 45f(%ebx,%ebx,2), %ebx
- testl %esi, %esi
- JMP_NOSPEC ebx
- # Handle 2-byte-aligned regions
- 20: addw (%esi), %ax
- lea 2(%esi), %esi
- adcl $0, %eax
- jmp 10b
- 25:
- testl $1, %esi
- jz 30f
- # buf is odd
- dec %ecx
- jl 90f
- movzbl (%esi), %ebx
- addl %ebx, %eax
- adcl $0, %eax
- roll $8, %eax
- inc %esi
- testl $2, %esi
- jz 10b
- 30: subl $2, %ecx
- ja 20b
- je 32f
- addl $2, %ecx
- jz 80f
- movzbl (%esi),%ebx # csumming 1 byte, 2-aligned
- addl %ebx, %eax
- adcl $0, %eax
- jmp 80f
- 32:
- addw (%esi), %ax # csumming 2 bytes, 2-aligned
- adcl $0, %eax
- jmp 80f
- 40:
- addl -128(%esi), %eax
- adcl -124(%esi), %eax
- adcl -120(%esi), %eax
- adcl -116(%esi), %eax
- adcl -112(%esi), %eax
- adcl -108(%esi), %eax
- adcl -104(%esi), %eax
- adcl -100(%esi), %eax
- adcl -96(%esi), %eax
- adcl -92(%esi), %eax
- adcl -88(%esi), %eax
- adcl -84(%esi), %eax
- adcl -80(%esi), %eax
- adcl -76(%esi), %eax
- adcl -72(%esi), %eax
- adcl -68(%esi), %eax
- adcl -64(%esi), %eax
- adcl -60(%esi), %eax
- adcl -56(%esi), %eax
- adcl -52(%esi), %eax
- adcl -48(%esi), %eax
- adcl -44(%esi), %eax
- adcl -40(%esi), %eax
- adcl -36(%esi), %eax
- adcl -32(%esi), %eax
- adcl -28(%esi), %eax
- adcl -24(%esi), %eax
- adcl -20(%esi), %eax
- adcl -16(%esi), %eax
- adcl -12(%esi), %eax
- adcl -8(%esi), %eax
- adcl -4(%esi), %eax
- 45:
- lea 128(%esi), %esi
- adcl $0, %eax
- dec %ecx
- jge 40b
- movl %edx, %ecx
- 50: andl $3, %ecx
- jz 80f
- # Handle the last 1-3 bytes without jumping
- notl %ecx # 1->2, 2->1, 3->0, higher bits are masked
- movl $0xffffff,%ebx # by the shll and shrl instructions
- shll $3,%ecx
- shrl %cl,%ebx
- andl -128(%esi),%ebx # esi is 4-aligned so should be ok
- addl %ebx,%eax
- adcl $0,%eax
- 80:
- testb $1, 12(%esp)
- jz 90f
- roll $8, %eax
- 90:
- popl %ebx
- popl %esi
- RET
- SYM_FUNC_END(csum_partial)
-
- #endif
- EXPORT_SYMBOL(csum_partial)
- /*
- unsigned int csum_partial_copy_generic (const char *src, char *dst,
- int len)
- */
- /*
- * Copy from ds while checksumming, otherwise like csum_partial
- */
- #define EXC(y...) \
- 9999: y; \
- _ASM_EXTABLE_TYPE(9999b, 7f, EX_TYPE_UACCESS | EX_FLAG_CLEAR_AX)
- #ifndef CONFIG_X86_USE_PPRO_CHECKSUM
- #define ARGBASE 16
- #define FP 12
-
- SYM_FUNC_START(csum_partial_copy_generic)
- subl $4,%esp
- pushl %edi
- pushl %esi
- pushl %ebx
- movl ARGBASE+12(%esp),%ecx # len
- movl ARGBASE+4(%esp),%esi # src
- movl ARGBASE+8(%esp),%edi # dst
- movl $-1, %eax # sum
- testl $2, %edi # Check alignment.
- jz 2f # Jump if alignment is ok.
- subl $2, %ecx # Alignment uses up two bytes.
- jae 1f # Jump if we had at least two bytes.
- addl $2, %ecx # ecx was < 2. Deal with it.
- jmp 4f
- EXC(1: movw (%esi), %bx )
- addl $2, %esi
- EXC( movw %bx, (%edi) )
- addl $2, %edi
- addw %bx, %ax
- adcl $0, %eax
- 2:
- movl %ecx, FP(%esp)
- shrl $5, %ecx
- jz 2f
- testl %esi, %esi # what's wrong with clc?
- EXC(1: movl (%esi), %ebx )
- EXC( movl 4(%esi), %edx )
- adcl %ebx, %eax
- EXC( movl %ebx, (%edi) )
- adcl %edx, %eax
- EXC( movl %edx, 4(%edi) )
- EXC( movl 8(%esi), %ebx )
- EXC( movl 12(%esi), %edx )
- adcl %ebx, %eax
- EXC( movl %ebx, 8(%edi) )
- adcl %edx, %eax
- EXC( movl %edx, 12(%edi) )
- EXC( movl 16(%esi), %ebx )
- EXC( movl 20(%esi), %edx )
- adcl %ebx, %eax
- EXC( movl %ebx, 16(%edi) )
- adcl %edx, %eax
- EXC( movl %edx, 20(%edi) )
- EXC( movl 24(%esi), %ebx )
- EXC( movl 28(%esi), %edx )
- adcl %ebx, %eax
- EXC( movl %ebx, 24(%edi) )
- adcl %edx, %eax
- EXC( movl %edx, 28(%edi) )
- lea 32(%esi), %esi
- lea 32(%edi), %edi
- dec %ecx
- jne 1b
- adcl $0, %eax
- 2: movl FP(%esp), %edx
- movl %edx, %ecx
- andl $0x1c, %edx
- je 4f
- shrl $2, %edx # This clears CF
- EXC(3: movl (%esi), %ebx )
- adcl %ebx, %eax
- EXC( movl %ebx, (%edi) )
- lea 4(%esi), %esi
- lea 4(%edi), %edi
- dec %edx
- jne 3b
- adcl $0, %eax
- 4: andl $3, %ecx
- jz 7f
- cmpl $2, %ecx
- jb 5f
- EXC( movw (%esi), %cx )
- leal 2(%esi), %esi
- EXC( movw %cx, (%edi) )
- leal 2(%edi), %edi
- je 6f
- shll $16,%ecx
- EXC(5: movb (%esi), %cl )
- EXC( movb %cl, (%edi) )
- 6: addl %ecx, %eax
- adcl $0, %eax
- 7:
- popl %ebx
- popl %esi
- popl %edi
- popl %ecx # equivalent to addl $4,%esp
- RET
- SYM_FUNC_END(csum_partial_copy_generic)
- #else
- /* Version for PentiumII/PPro */
- #define ROUND1(x) \
- EXC(movl x(%esi), %ebx ) ; \
- addl %ebx, %eax ; \
- EXC(movl %ebx, x(%edi) ) ;
- #define ROUND(x) \
- EXC(movl x(%esi), %ebx ) ; \
- adcl %ebx, %eax ; \
- EXC(movl %ebx, x(%edi) ) ;
- #define ARGBASE 12
-
- SYM_FUNC_START(csum_partial_copy_generic)
- pushl %ebx
- pushl %edi
- pushl %esi
- movl ARGBASE+4(%esp),%esi #src
- movl ARGBASE+8(%esp),%edi #dst
- movl ARGBASE+12(%esp),%ecx #len
- movl $-1, %eax #sum
- # movl %ecx, %edx
- movl %ecx, %ebx
- movl %esi, %edx
- shrl $6, %ecx
- andl $0x3c, %ebx
- negl %ebx
- subl %ebx, %esi
- subl %ebx, %edi
- lea -1(%esi),%edx
- andl $-32,%edx
- lea 3f(%ebx,%ebx), %ebx
- testl %esi, %esi
- JMP_NOSPEC ebx
- 1: addl $64,%esi
- addl $64,%edi
- EXC(movb -32(%edx),%bl) ; EXC(movb (%edx),%bl)
- ROUND1(-64) ROUND(-60) ROUND(-56) ROUND(-52)
- ROUND (-48) ROUND(-44) ROUND(-40) ROUND(-36)
- ROUND (-32) ROUND(-28) ROUND(-24) ROUND(-20)
- ROUND (-16) ROUND(-12) ROUND(-8) ROUND(-4)
- 3: adcl $0,%eax
- addl $64, %edx
- dec %ecx
- jge 1b
- 4: movl ARGBASE+12(%esp),%edx #len
- andl $3, %edx
- jz 7f
- cmpl $2, %edx
- jb 5f
- EXC( movw (%esi), %dx )
- leal 2(%esi), %esi
- EXC( movw %dx, (%edi) )
- leal 2(%edi), %edi
- je 6f
- shll $16,%edx
- 5:
- EXC( movb (%esi), %dl )
- EXC( movb %dl, (%edi) )
- 6: addl %edx, %eax
- adcl $0, %eax
- 7:
- popl %esi
- popl %edi
- popl %ebx
- RET
- SYM_FUNC_END(csum_partial_copy_generic)
-
- #undef ROUND
- #undef ROUND1
-
- #endif
- EXPORT_SYMBOL(csum_partial_copy_generic)
|