memmove_64.S 3.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. /*
  3. * Normally compiler builtins are used, but sometimes the compiler calls out
  4. * of line code. Based on asm-i386/string.h.
  5. *
  6. * This assembly file is re-written from memmove_64.c file.
  7. * - Copyright 2011 Fenghua Yu <[email protected]>
  8. */
  9. #include <linux/linkage.h>
  10. #include <asm/cpufeatures.h>
  11. #include <asm/alternative.h>
  12. #include <asm/export.h>
  13. #undef memmove
  14. /*
  15. * Implement memmove(). This can handle overlap between src and dst.
  16. *
  17. * Input:
  18. * rdi: dest
  19. * rsi: src
  20. * rdx: count
  21. *
  22. * Output:
  23. * rax: dest
  24. */
  25. SYM_FUNC_START(__memmove)
  26. mov %rdi, %rax
  27. /* Decide forward/backward copy mode */
  28. cmp %rdi, %rsi
  29. jge .Lmemmove_begin_forward
  30. mov %rsi, %r8
  31. add %rdx, %r8
  32. cmp %rdi, %r8
  33. jg 2f
  34. /* FSRM implies ERMS => no length checks, do the copy directly */
  35. .Lmemmove_begin_forward:
  36. ALTERNATIVE "cmp $0x20, %rdx; jb 1f", "", X86_FEATURE_FSRM
  37. ALTERNATIVE "", "jmp .Lmemmove_erms", X86_FEATURE_ERMS
  38. /*
  39. * movsq instruction have many startup latency
  40. * so we handle small size by general register.
  41. */
  42. cmp $680, %rdx
  43. jb 3f
  44. /*
  45. * movsq instruction is only good for aligned case.
  46. */
  47. cmpb %dil, %sil
  48. je 4f
  49. 3:
  50. sub $0x20, %rdx
  51. /*
  52. * We gobble 32 bytes forward in each loop.
  53. */
  54. 5:
  55. sub $0x20, %rdx
  56. movq 0*8(%rsi), %r11
  57. movq 1*8(%rsi), %r10
  58. movq 2*8(%rsi), %r9
  59. movq 3*8(%rsi), %r8
  60. leaq 4*8(%rsi), %rsi
  61. movq %r11, 0*8(%rdi)
  62. movq %r10, 1*8(%rdi)
  63. movq %r9, 2*8(%rdi)
  64. movq %r8, 3*8(%rdi)
  65. leaq 4*8(%rdi), %rdi
  66. jae 5b
  67. addq $0x20, %rdx
  68. jmp 1f
  69. /*
  70. * Handle data forward by movsq.
  71. */
  72. .p2align 4
  73. 4:
  74. movq %rdx, %rcx
  75. movq -8(%rsi, %rdx), %r11
  76. lea -8(%rdi, %rdx), %r10
  77. shrq $3, %rcx
  78. rep movsq
  79. movq %r11, (%r10)
  80. jmp 13f
  81. .Lmemmove_end_forward:
  82. /*
  83. * Handle data backward by movsq.
  84. */
  85. .p2align 4
  86. 7:
  87. movq %rdx, %rcx
  88. movq (%rsi), %r11
  89. movq %rdi, %r10
  90. leaq -8(%rsi, %rdx), %rsi
  91. leaq -8(%rdi, %rdx), %rdi
  92. shrq $3, %rcx
  93. std
  94. rep movsq
  95. cld
  96. movq %r11, (%r10)
  97. jmp 13f
  98. /*
  99. * Start to prepare for backward copy.
  100. */
  101. .p2align 4
  102. 2:
  103. cmp $0x20, %rdx
  104. jb 1f
  105. cmp $680, %rdx
  106. jb 6f
  107. cmp %dil, %sil
  108. je 7b
  109. 6:
  110. /*
  111. * Calculate copy position to tail.
  112. */
  113. addq %rdx, %rsi
  114. addq %rdx, %rdi
  115. subq $0x20, %rdx
  116. /*
  117. * We gobble 32 bytes backward in each loop.
  118. */
  119. 8:
  120. subq $0x20, %rdx
  121. movq -1*8(%rsi), %r11
  122. movq -2*8(%rsi), %r10
  123. movq -3*8(%rsi), %r9
  124. movq -4*8(%rsi), %r8
  125. leaq -4*8(%rsi), %rsi
  126. movq %r11, -1*8(%rdi)
  127. movq %r10, -2*8(%rdi)
  128. movq %r9, -3*8(%rdi)
  129. movq %r8, -4*8(%rdi)
  130. leaq -4*8(%rdi), %rdi
  131. jae 8b
  132. /*
  133. * Calculate copy position to head.
  134. */
  135. addq $0x20, %rdx
  136. subq %rdx, %rsi
  137. subq %rdx, %rdi
  138. 1:
  139. cmpq $16, %rdx
  140. jb 9f
  141. /*
  142. * Move data from 16 bytes to 31 bytes.
  143. */
  144. movq 0*8(%rsi), %r11
  145. movq 1*8(%rsi), %r10
  146. movq -2*8(%rsi, %rdx), %r9
  147. movq -1*8(%rsi, %rdx), %r8
  148. movq %r11, 0*8(%rdi)
  149. movq %r10, 1*8(%rdi)
  150. movq %r9, -2*8(%rdi, %rdx)
  151. movq %r8, -1*8(%rdi, %rdx)
  152. jmp 13f
  153. .p2align 4
  154. 9:
  155. cmpq $8, %rdx
  156. jb 10f
  157. /*
  158. * Move data from 8 bytes to 15 bytes.
  159. */
  160. movq 0*8(%rsi), %r11
  161. movq -1*8(%rsi, %rdx), %r10
  162. movq %r11, 0*8(%rdi)
  163. movq %r10, -1*8(%rdi, %rdx)
  164. jmp 13f
  165. 10:
  166. cmpq $4, %rdx
  167. jb 11f
  168. /*
  169. * Move data from 4 bytes to 7 bytes.
  170. */
  171. movl (%rsi), %r11d
  172. movl -4(%rsi, %rdx), %r10d
  173. movl %r11d, (%rdi)
  174. movl %r10d, -4(%rdi, %rdx)
  175. jmp 13f
  176. 11:
  177. cmp $2, %rdx
  178. jb 12f
  179. /*
  180. * Move data from 2 bytes to 3 bytes.
  181. */
  182. movw (%rsi), %r11w
  183. movw -2(%rsi, %rdx), %r10w
  184. movw %r11w, (%rdi)
  185. movw %r10w, -2(%rdi, %rdx)
  186. jmp 13f
  187. 12:
  188. cmp $1, %rdx
  189. jb 13f
  190. /*
  191. * Move data for 1 byte.
  192. */
  193. movb (%rsi), %r11b
  194. movb %r11b, (%rdi)
  195. 13:
  196. RET
  197. .Lmemmove_erms:
  198. movq %rdx, %rcx
  199. rep movsb
  200. RET
  201. SYM_FUNC_END(__memmove)
  202. EXPORT_SYMBOL(__memmove)
  203. SYM_FUNC_ALIAS_WEAK(memmove, __memmove)
  204. EXPORT_SYMBOL(memmove)