copy_32.S 9.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515
  1. /* SPDX-License-Identifier: GPL-2.0-or-later */
  2. /*
  3. * Memory copy functions for 32-bit PowerPC.
  4. *
  5. * Copyright (C) 1996-2005 Paul Mackerras.
  6. */
  7. #include <asm/processor.h>
  8. #include <asm/cache.h>
  9. #include <asm/errno.h>
  10. #include <asm/ppc_asm.h>
  11. #include <asm/export.h>
  12. #include <asm/code-patching-asm.h>
  13. #include <asm/kasan.h>
  14. #define COPY_16_BYTES \
  15. lwz r7,4(r4); \
  16. lwz r8,8(r4); \
  17. lwz r9,12(r4); \
  18. lwzu r10,16(r4); \
  19. stw r7,4(r6); \
  20. stw r8,8(r6); \
  21. stw r9,12(r6); \
  22. stwu r10,16(r6)
  23. #define COPY_16_BYTES_WITHEX(n) \
  24. 8 ## n ## 0: \
  25. lwz r7,4(r4); \
  26. 8 ## n ## 1: \
  27. lwz r8,8(r4); \
  28. 8 ## n ## 2: \
  29. lwz r9,12(r4); \
  30. 8 ## n ## 3: \
  31. lwzu r10,16(r4); \
  32. 8 ## n ## 4: \
  33. stw r7,4(r6); \
  34. 8 ## n ## 5: \
  35. stw r8,8(r6); \
  36. 8 ## n ## 6: \
  37. stw r9,12(r6); \
  38. 8 ## n ## 7: \
  39. stwu r10,16(r6)
  40. #define COPY_16_BYTES_EXCODE(n) \
  41. 9 ## n ## 0: \
  42. addi r5,r5,-(16 * n); \
  43. b 104f; \
  44. 9 ## n ## 1: \
  45. addi r5,r5,-(16 * n); \
  46. b 105f; \
  47. EX_TABLE(8 ## n ## 0b,9 ## n ## 0b); \
  48. EX_TABLE(8 ## n ## 1b,9 ## n ## 0b); \
  49. EX_TABLE(8 ## n ## 2b,9 ## n ## 0b); \
  50. EX_TABLE(8 ## n ## 3b,9 ## n ## 0b); \
  51. EX_TABLE(8 ## n ## 4b,9 ## n ## 1b); \
  52. EX_TABLE(8 ## n ## 5b,9 ## n ## 1b); \
  53. EX_TABLE(8 ## n ## 6b,9 ## n ## 1b); \
  54. EX_TABLE(8 ## n ## 7b,9 ## n ## 1b)
  55. .text
  56. CACHELINE_BYTES = L1_CACHE_BYTES
  57. LG_CACHELINE_BYTES = L1_CACHE_SHIFT
  58. CACHELINE_MASK = (L1_CACHE_BYTES-1)
  59. #ifndef CONFIG_KASAN
  60. _GLOBAL(memset16)
  61. rlwinm. r0 ,r5, 31, 1, 31
  62. addi r6, r3, -4
  63. beq- 2f
  64. rlwimi r4 ,r4 ,16 ,0 ,15
  65. mtctr r0
  66. 1: stwu r4, 4(r6)
  67. bdnz 1b
  68. 2: andi. r0, r5, 1
  69. beqlr
  70. sth r4, 4(r6)
  71. blr
  72. EXPORT_SYMBOL(memset16)
  73. #endif
  74. /*
  75. * Use dcbz on the complete cache lines in the destination
  76. * to set them to zero. This requires that the destination
  77. * area is cacheable. -- paulus
  78. *
  79. * During early init, cache might not be active yet, so dcbz cannot be used.
  80. * We therefore skip the optimised bloc that uses dcbz. This jump is
  81. * replaced by a nop once cache is active. This is done in machine_init()
  82. */
  83. _GLOBAL_KASAN(memset)
  84. cmplwi 0,r5,4
  85. blt 7f
  86. rlwimi r4,r4,8,16,23
  87. rlwimi r4,r4,16,0,15
  88. stw r4,0(r3)
  89. beqlr
  90. andi. r0,r3,3
  91. add r5,r0,r5
  92. subf r6,r0,r3
  93. cmplwi 0,r4,0
  94. /*
  95. * Skip optimised bloc until cache is enabled. Will be replaced
  96. * by 'bne' during boot to use normal procedure if r4 is not zero
  97. */
  98. 5: b 2f
  99. patch_site 5b, patch__memset_nocache
  100. clrlwi r7,r6,32-LG_CACHELINE_BYTES
  101. add r8,r7,r5
  102. srwi r9,r8,LG_CACHELINE_BYTES
  103. addic. r9,r9,-1 /* total number of complete cachelines */
  104. ble 2f
  105. xori r0,r7,CACHELINE_MASK & ~3
  106. srwi. r0,r0,2
  107. beq 3f
  108. mtctr r0
  109. 4: stwu r4,4(r6)
  110. bdnz 4b
  111. 3: mtctr r9
  112. li r7,4
  113. 10: dcbz r7,r6
  114. addi r6,r6,CACHELINE_BYTES
  115. bdnz 10b
  116. clrlwi r5,r8,32-LG_CACHELINE_BYTES
  117. addi r5,r5,4
  118. 2: srwi r0,r5,2
  119. mtctr r0
  120. bdz 6f
  121. 1: stwu r4,4(r6)
  122. bdnz 1b
  123. 6: andi. r5,r5,3
  124. beqlr
  125. mtctr r5
  126. addi r6,r6,3
  127. 8: stbu r4,1(r6)
  128. bdnz 8b
  129. blr
  130. 7: cmpwi 0,r5,0
  131. beqlr
  132. mtctr r5
  133. addi r6,r3,-1
  134. 9: stbu r4,1(r6)
  135. bdnz 9b
  136. blr
  137. EXPORT_SYMBOL(memset)
  138. EXPORT_SYMBOL_KASAN(memset)
  139. /*
  140. * This version uses dcbz on the complete cache lines in the
  141. * destination area to reduce memory traffic. This requires that
  142. * the destination area is cacheable.
  143. * We only use this version if the source and dest don't overlap.
  144. * -- paulus.
  145. *
  146. * During early init, cache might not be active yet, so dcbz cannot be used.
  147. * We therefore jump to generic_memcpy which doesn't use dcbz. This jump is
  148. * replaced by a nop once cache is active. This is done in machine_init()
  149. */
  150. _GLOBAL_KASAN(memmove)
  151. cmplw 0,r3,r4
  152. bgt backwards_memcpy
  153. /* fall through */
  154. _GLOBAL_KASAN(memcpy)
  155. 1: b generic_memcpy
  156. patch_site 1b, patch__memcpy_nocache
  157. add r7,r3,r5 /* test if the src & dst overlap */
  158. add r8,r4,r5
  159. cmplw 0,r4,r7
  160. cmplw 1,r3,r8
  161. crand 0,0,4 /* cr0.lt &= cr1.lt */
  162. blt generic_memcpy /* if regions overlap */
  163. addi r4,r4,-4
  164. addi r6,r3,-4
  165. neg r0,r3
  166. andi. r0,r0,CACHELINE_MASK /* # bytes to start of cache line */
  167. beq 58f
  168. cmplw 0,r5,r0 /* is this more than total to do? */
  169. blt 63f /* if not much to do */
  170. andi. r8,r0,3 /* get it word-aligned first */
  171. subf r5,r0,r5
  172. mtctr r8
  173. beq+ 61f
  174. 70: lbz r9,4(r4) /* do some bytes */
  175. addi r4,r4,1
  176. addi r6,r6,1
  177. stb r9,3(r6)
  178. bdnz 70b
  179. 61: srwi. r0,r0,2
  180. mtctr r0
  181. beq 58f
  182. 72: lwzu r9,4(r4) /* do some words */
  183. stwu r9,4(r6)
  184. bdnz 72b
  185. 58: srwi. r0,r5,LG_CACHELINE_BYTES /* # complete cachelines */
  186. clrlwi r5,r5,32-LG_CACHELINE_BYTES
  187. li r11,4
  188. mtctr r0
  189. beq 63f
  190. 53:
  191. dcbz r11,r6
  192. COPY_16_BYTES
  193. #if L1_CACHE_BYTES >= 32
  194. COPY_16_BYTES
  195. #if L1_CACHE_BYTES >= 64
  196. COPY_16_BYTES
  197. COPY_16_BYTES
  198. #if L1_CACHE_BYTES >= 128
  199. COPY_16_BYTES
  200. COPY_16_BYTES
  201. COPY_16_BYTES
  202. COPY_16_BYTES
  203. #endif
  204. #endif
  205. #endif
  206. bdnz 53b
  207. 63: srwi. r0,r5,2
  208. mtctr r0
  209. beq 64f
  210. 30: lwzu r0,4(r4)
  211. stwu r0,4(r6)
  212. bdnz 30b
  213. 64: andi. r0,r5,3
  214. mtctr r0
  215. beq+ 65f
  216. addi r4,r4,3
  217. addi r6,r6,3
  218. 40: lbzu r0,1(r4)
  219. stbu r0,1(r6)
  220. bdnz 40b
  221. 65: blr
  222. EXPORT_SYMBOL(memcpy)
  223. EXPORT_SYMBOL(memmove)
  224. EXPORT_SYMBOL_KASAN(memcpy)
  225. EXPORT_SYMBOL_KASAN(memmove)
  226. generic_memcpy:
  227. srwi. r7,r5,3
  228. addi r6,r3,-4
  229. addi r4,r4,-4
  230. beq 2f /* if less than 8 bytes to do */
  231. andi. r0,r6,3 /* get dest word aligned */
  232. mtctr r7
  233. bne 5f
  234. 1: lwz r7,4(r4)
  235. lwzu r8,8(r4)
  236. stw r7,4(r6)
  237. stwu r8,8(r6)
  238. bdnz 1b
  239. andi. r5,r5,7
  240. 2: cmplwi 0,r5,4
  241. blt 3f
  242. lwzu r0,4(r4)
  243. addi r5,r5,-4
  244. stwu r0,4(r6)
  245. 3: cmpwi 0,r5,0
  246. beqlr
  247. mtctr r5
  248. addi r4,r4,3
  249. addi r6,r6,3
  250. 4: lbzu r0,1(r4)
  251. stbu r0,1(r6)
  252. bdnz 4b
  253. blr
  254. 5: subfic r0,r0,4
  255. mtctr r0
  256. 6: lbz r7,4(r4)
  257. addi r4,r4,1
  258. stb r7,4(r6)
  259. addi r6,r6,1
  260. bdnz 6b
  261. subf r5,r0,r5
  262. rlwinm. r7,r5,32-3,3,31
  263. beq 2b
  264. mtctr r7
  265. b 1b
  266. _GLOBAL(backwards_memcpy)
  267. rlwinm. r7,r5,32-3,3,31 /* r0 = r5 >> 3 */
  268. add r6,r3,r5
  269. add r4,r4,r5
  270. beq 2f
  271. andi. r0,r6,3
  272. mtctr r7
  273. bne 5f
  274. 1: lwz r7,-4(r4)
  275. lwzu r8,-8(r4)
  276. stw r7,-4(r6)
  277. stwu r8,-8(r6)
  278. bdnz 1b
  279. andi. r5,r5,7
  280. 2: cmplwi 0,r5,4
  281. blt 3f
  282. lwzu r0,-4(r4)
  283. subi r5,r5,4
  284. stwu r0,-4(r6)
  285. 3: cmpwi 0,r5,0
  286. beqlr
  287. mtctr r5
  288. 4: lbzu r0,-1(r4)
  289. stbu r0,-1(r6)
  290. bdnz 4b
  291. blr
  292. 5: mtctr r0
  293. 6: lbzu r7,-1(r4)
  294. stbu r7,-1(r6)
  295. bdnz 6b
  296. subf r5,r0,r5
  297. rlwinm. r7,r5,32-3,3,31
  298. beq 2b
  299. mtctr r7
  300. b 1b
  301. _GLOBAL(__copy_tofrom_user)
  302. addi r4,r4,-4
  303. addi r6,r3,-4
  304. neg r0,r3
  305. andi. r0,r0,CACHELINE_MASK /* # bytes to start of cache line */
  306. beq 58f
  307. cmplw 0,r5,r0 /* is this more than total to do? */
  308. blt 63f /* if not much to do */
  309. andi. r8,r0,3 /* get it word-aligned first */
  310. mtctr r8
  311. beq+ 61f
  312. 70: lbz r9,4(r4) /* do some bytes */
  313. 71: stb r9,4(r6)
  314. addi r4,r4,1
  315. addi r6,r6,1
  316. bdnz 70b
  317. 61: subf r5,r0,r5
  318. srwi. r0,r0,2
  319. mtctr r0
  320. beq 58f
  321. 72: lwzu r9,4(r4) /* do some words */
  322. 73: stwu r9,4(r6)
  323. bdnz 72b
  324. EX_TABLE(70b,100f)
  325. EX_TABLE(71b,101f)
  326. EX_TABLE(72b,102f)
  327. EX_TABLE(73b,103f)
  328. 58: srwi. r0,r5,LG_CACHELINE_BYTES /* # complete cachelines */
  329. clrlwi r5,r5,32-LG_CACHELINE_BYTES
  330. li r11,4
  331. beq 63f
  332. /* Here we decide how far ahead to prefetch the source */
  333. li r3,4
  334. cmpwi r0,1
  335. li r7,0
  336. ble 114f
  337. li r7,1
  338. #if MAX_COPY_PREFETCH > 1
  339. /* Heuristically, for large transfers we prefetch
  340. MAX_COPY_PREFETCH cachelines ahead. For small transfers
  341. we prefetch 1 cacheline ahead. */
  342. cmpwi r0,MAX_COPY_PREFETCH
  343. ble 112f
  344. li r7,MAX_COPY_PREFETCH
  345. 112: mtctr r7
  346. 111: dcbt r3,r4
  347. addi r3,r3,CACHELINE_BYTES
  348. bdnz 111b
  349. #else
  350. dcbt r3,r4
  351. addi r3,r3,CACHELINE_BYTES
  352. #endif /* MAX_COPY_PREFETCH > 1 */
  353. 114: subf r8,r7,r0
  354. mr r0,r7
  355. mtctr r8
  356. 53: dcbt r3,r4
  357. 54: dcbz r11,r6
  358. EX_TABLE(54b,105f)
  359. /* the main body of the cacheline loop */
  360. COPY_16_BYTES_WITHEX(0)
  361. #if L1_CACHE_BYTES >= 32
  362. COPY_16_BYTES_WITHEX(1)
  363. #if L1_CACHE_BYTES >= 64
  364. COPY_16_BYTES_WITHEX(2)
  365. COPY_16_BYTES_WITHEX(3)
  366. #if L1_CACHE_BYTES >= 128
  367. COPY_16_BYTES_WITHEX(4)
  368. COPY_16_BYTES_WITHEX(5)
  369. COPY_16_BYTES_WITHEX(6)
  370. COPY_16_BYTES_WITHEX(7)
  371. #endif
  372. #endif
  373. #endif
  374. bdnz 53b
  375. cmpwi r0,0
  376. li r3,4
  377. li r7,0
  378. bne 114b
  379. 63: srwi. r0,r5,2
  380. mtctr r0
  381. beq 64f
  382. 30: lwzu r0,4(r4)
  383. 31: stwu r0,4(r6)
  384. bdnz 30b
  385. 64: andi. r0,r5,3
  386. mtctr r0
  387. beq+ 65f
  388. 40: lbz r0,4(r4)
  389. 41: stb r0,4(r6)
  390. addi r4,r4,1
  391. addi r6,r6,1
  392. bdnz 40b
  393. 65: li r3,0
  394. blr
  395. /* read fault, initial single-byte copy */
  396. 100: li r9,0
  397. b 90f
  398. /* write fault, initial single-byte copy */
  399. 101: li r9,1
  400. 90: subf r5,r8,r5
  401. li r3,0
  402. b 99f
  403. /* read fault, initial word copy */
  404. 102: li r9,0
  405. b 91f
  406. /* write fault, initial word copy */
  407. 103: li r9,1
  408. 91: li r3,2
  409. b 99f
  410. /*
  411. * this stuff handles faults in the cacheline loop and branches to either
  412. * 104f (if in read part) or 105f (if in write part), after updating r5
  413. */
  414. COPY_16_BYTES_EXCODE(0)
  415. #if L1_CACHE_BYTES >= 32
  416. COPY_16_BYTES_EXCODE(1)
  417. #if L1_CACHE_BYTES >= 64
  418. COPY_16_BYTES_EXCODE(2)
  419. COPY_16_BYTES_EXCODE(3)
  420. #if L1_CACHE_BYTES >= 128
  421. COPY_16_BYTES_EXCODE(4)
  422. COPY_16_BYTES_EXCODE(5)
  423. COPY_16_BYTES_EXCODE(6)
  424. COPY_16_BYTES_EXCODE(7)
  425. #endif
  426. #endif
  427. #endif
  428. /* read fault in cacheline loop */
  429. 104: li r9,0
  430. b 92f
  431. /* fault on dcbz (effectively a write fault) */
  432. /* or write fault in cacheline loop */
  433. 105: li r9,1
  434. 92: li r3,LG_CACHELINE_BYTES
  435. mfctr r8
  436. add r0,r0,r8
  437. b 106f
  438. /* read fault in final word loop */
  439. 108: li r9,0
  440. b 93f
  441. /* write fault in final word loop */
  442. 109: li r9,1
  443. 93: andi. r5,r5,3
  444. li r3,2
  445. b 99f
  446. /* read fault in final byte loop */
  447. 110: li r9,0
  448. b 94f
  449. /* write fault in final byte loop */
  450. 111: li r9,1
  451. 94: li r5,0
  452. li r3,0
  453. /*
  454. * At this stage the number of bytes not copied is
  455. * r5 + (ctr << r3), and r9 is 0 for read or 1 for write.
  456. */
  457. 99: mfctr r0
  458. 106: slw r3,r0,r3
  459. add. r3,r3,r5
  460. beq 120f /* shouldn't happen */
  461. cmpwi 0,r9,0
  462. bne 120f
  463. /* for a read fault, first try to continue the copy one byte at a time */
  464. mtctr r3
  465. 130: lbz r0,4(r4)
  466. 131: stb r0,4(r6)
  467. addi r4,r4,1
  468. addi r6,r6,1
  469. bdnz 130b
  470. /* then clear out the destination: r3 bytes starting at 4(r6) */
  471. 132: mfctr r3
  472. 120: blr
  473. EX_TABLE(30b,108b)
  474. EX_TABLE(31b,109b)
  475. EX_TABLE(40b,110b)
  476. EX_TABLE(41b,111b)
  477. EX_TABLE(130b,132b)
  478. EX_TABLE(131b,120b)
  479. EXPORT_SYMBOL(__copy_tofrom_user)