memset_64.S 2.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. /* Copyright 2002 Andi Kleen, SuSE Labs */
  3. #include <linux/linkage.h>
  4. #include <asm/cpufeatures.h>
  5. #include <asm/alternative.h>
  6. #include <asm/export.h>
  7. /*
  8. * ISO C memset - set a memory block to a byte value. This function uses fast
  9. * string to get better performance than the original function. The code is
  10. * simpler and shorter than the original function as well.
  11. *
  12. * rdi destination
  13. * rsi value (char)
  14. * rdx count (bytes)
  15. *
  16. * rax original destination
  17. */
  18. SYM_FUNC_START(__memset)
  19. /*
  20. * Some CPUs support enhanced REP MOVSB/STOSB feature. It is recommended
  21. * to use it when possible. If not available, use fast string instructions.
  22. *
  23. * Otherwise, use original memset function.
  24. */
  25. ALTERNATIVE_2 "jmp memset_orig", "", X86_FEATURE_REP_GOOD, \
  26. "jmp memset_erms", X86_FEATURE_ERMS
  27. movq %rdi,%r9
  28. movq %rdx,%rcx
  29. andl $7,%edx
  30. shrq $3,%rcx
  31. /* expand byte value */
  32. movzbl %sil,%esi
  33. movabs $0x0101010101010101,%rax
  34. imulq %rsi,%rax
  35. rep stosq
  36. movl %edx,%ecx
  37. rep stosb
  38. movq %r9,%rax
  39. RET
  40. SYM_FUNC_END(__memset)
  41. EXPORT_SYMBOL(__memset)
  42. SYM_FUNC_ALIAS_WEAK(memset, __memset)
  43. EXPORT_SYMBOL(memset)
  44. /*
  45. * ISO C memset - set a memory block to a byte value. This function uses
  46. * enhanced rep stosb to override the fast string function.
  47. * The code is simpler and shorter than the fast string function as well.
  48. *
  49. * rdi destination
  50. * rsi value (char)
  51. * rdx count (bytes)
  52. *
  53. * rax original destination
  54. */
  55. SYM_FUNC_START_LOCAL(memset_erms)
  56. movq %rdi,%r9
  57. movb %sil,%al
  58. movq %rdx,%rcx
  59. rep stosb
  60. movq %r9,%rax
  61. RET
  62. SYM_FUNC_END(memset_erms)
  63. SYM_FUNC_START_LOCAL(memset_orig)
  64. movq %rdi,%r10
  65. /* expand byte value */
  66. movzbl %sil,%ecx
  67. movabs $0x0101010101010101,%rax
  68. imulq %rcx,%rax
  69. /* align dst */
  70. movl %edi,%r9d
  71. andl $7,%r9d
  72. jnz .Lbad_alignment
  73. .Lafter_bad_alignment:
  74. movq %rdx,%rcx
  75. shrq $6,%rcx
  76. jz .Lhandle_tail
  77. .p2align 4
  78. .Lloop_64:
  79. decq %rcx
  80. movq %rax,(%rdi)
  81. movq %rax,8(%rdi)
  82. movq %rax,16(%rdi)
  83. movq %rax,24(%rdi)
  84. movq %rax,32(%rdi)
  85. movq %rax,40(%rdi)
  86. movq %rax,48(%rdi)
  87. movq %rax,56(%rdi)
  88. leaq 64(%rdi),%rdi
  89. jnz .Lloop_64
  90. /* Handle tail in loops. The loops should be faster than hard
  91. to predict jump tables. */
  92. .p2align 4
  93. .Lhandle_tail:
  94. movl %edx,%ecx
  95. andl $63&(~7),%ecx
  96. jz .Lhandle_7
  97. shrl $3,%ecx
  98. .p2align 4
  99. .Lloop_8:
  100. decl %ecx
  101. movq %rax,(%rdi)
  102. leaq 8(%rdi),%rdi
  103. jnz .Lloop_8
  104. .Lhandle_7:
  105. andl $7,%edx
  106. jz .Lende
  107. .p2align 4
  108. .Lloop_1:
  109. decl %edx
  110. movb %al,(%rdi)
  111. leaq 1(%rdi),%rdi
  112. jnz .Lloop_1
  113. .Lende:
  114. movq %r10,%rax
  115. RET
  116. .Lbad_alignment:
  117. cmpq $7,%rdx
  118. jbe .Lhandle_7
  119. movq %rax,(%rdi) /* unaligned store */
  120. movq $8,%r8
  121. subq %r9,%r8
  122. addq %r8,%rdi
  123. subq %r8,%rdx
  124. jmp .Lafter_bad_alignment
  125. .Lfinal:
  126. SYM_FUNC_END(memset_orig)