memcpy-sh4.S 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. /*
  3. * "memcpy" implementation of SuperH
  4. *
  5. * Copyright (C) 1999 Niibe Yutaka
  6. * Copyright (c) 2002 STMicroelectronics Ltd
  7. * Modified from memcpy.S and micro-optimised for SH4
  8. * Stuart Menefy ([email protected])
  9. *
  10. */
  11. #include <linux/linkage.h>
  12. /*
  13. * void *memcpy(void *dst, const void *src, size_t n);
  14. *
  15. * It is assumed that there is no overlap between src and dst.
  16. * If there is an overlap, then the results are undefined.
  17. */
  18. !
  19. ! GHIJ KLMN OPQR --> ...G HIJK LMNO PQR.
  20. !
  21. ! Size is 16 or greater, and may have trailing bytes
  22. .balign 32
  23. .Lcase1:
  24. ! Read a long word and write a long word at once
  25. ! At the start of each iteration, r7 contains last long load
  26. add #-1,r5 ! 79 EX
  27. mov r4,r2 ! 5 MT (0 cycles latency)
  28. mov.l @(r0,r5),r7 ! 21 LS (2 cycles latency)
  29. add #-4,r5 ! 50 EX
  30. add #7,r2 ! 79 EX
  31. !
  32. #ifdef CONFIG_CPU_LITTLE_ENDIAN
  33. ! 6 cycles, 4 bytes per iteration
  34. 3: mov.l @(r0,r5),r1 ! 21 LS (latency=2) ! NMLK
  35. mov r7, r3 ! 5 MT (latency=0) ! RQPO
  36. cmp/hi r2,r0 ! 57 MT
  37. shll16 r3 ! 103 EX
  38. mov r1,r6 ! 5 MT (latency=0)
  39. shll8 r3 ! 102 EX ! Oxxx
  40. shlr8 r6 ! 106 EX ! xNML
  41. mov r1, r7 ! 5 MT (latency=0)
  42. or r6,r3 ! 82 EX ! ONML
  43. bt/s 3b ! 109 BR
  44. mov.l r3,@-r0 ! 30 LS
  45. #else
  46. 3: mov.l @(r0,r5),r1 ! 21 LS (latency=2) ! KLMN
  47. mov r7,r3 ! 5 MT (latency=0) ! OPQR
  48. cmp/hi r2,r0 ! 57 MT
  49. shlr16 r3 ! 107 EX
  50. shlr8 r3 ! 106 EX ! xxxO
  51. mov r1,r6 ! 5 MT (latency=0)
  52. shll8 r6 ! 102 EX ! LMNx
  53. mov r1,r7 ! 5 MT (latency=0)
  54. or r6,r3 ! 82 EX ! LMNO
  55. bt/s 3b ! 109 BR
  56. mov.l r3,@-r0 ! 30 LS
  57. #endif
  58. ! Finally, copy a byte at once, if necessary
  59. add #4,r5 ! 50 EX
  60. cmp/eq r4,r0 ! 54 MT
  61. add #-6,r2 ! 50 EX
  62. bt 9f ! 109 BR
  63. 8: cmp/hi r2,r0 ! 57 MT
  64. mov.b @(r0,r5),r1 ! 20 LS (latency=2)
  65. bt/s 8b ! 109 BR
  66. mov.b r1,@-r0 ! 29 LS
  67. 9: rts
  68. nop
  69. !
  70. ! GHIJ KLMN OPQR --> .GHI JKLM NOPQ R...
  71. !
  72. ! Size is 16 or greater, and may have trailing bytes
  73. .balign 32
  74. .Lcase3:
  75. ! Read a long word and write a long word at once
  76. ! At the start of each iteration, r7 contains last long load
  77. add #-3,r5 ! 79 EX
  78. mov r4,r2 ! 5 MT (0 cycles latency)
  79. mov.l @(r0,r5),r7 ! 21 LS (2 cycles latency)
  80. add #-4,r5 ! 50 EX
  81. add #7,r2 ! 79 EX
  82. !
  83. #ifdef CONFIG_CPU_LITTLE_ENDIAN
  84. ! 6 cycles, 4 bytes per iteration
  85. 3: mov.l @(r0,r5),r1 ! 21 LS (latency=2) ! NMLK
  86. mov r7, r3 ! 5 MT (latency=0) ! RQPO
  87. cmp/hi r2,r0 ! 57 MT
  88. shll8 r3 ! 102 EX ! QPOx
  89. mov r1,r6 ! 5 MT (latency=0)
  90. shlr16 r6 ! 107 EX
  91. shlr8 r6 ! 106 EX ! xxxN
  92. mov r1, r7 ! 5 MT (latency=0)
  93. or r6,r3 ! 82 EX ! QPON
  94. bt/s 3b ! 109 BR
  95. mov.l r3,@-r0 ! 30 LS
  96. #else
  97. 3: mov r7,r3 ! OPQR
  98. shlr8 r3 ! xOPQ
  99. mov.l @(r0,r5),r7 ! KLMN
  100. mov r7,r6
  101. shll16 r6
  102. shll8 r6 ! Nxxx
  103. or r6,r3 ! NOPQ
  104. cmp/hi r2,r0
  105. bt/s 3b
  106. mov.l r3,@-r0
  107. #endif
  108. ! Finally, copy a byte at once, if necessary
  109. add #6,r5 ! 50 EX
  110. cmp/eq r4,r0 ! 54 MT
  111. add #-6,r2 ! 50 EX
  112. bt 9f ! 109 BR
  113. 8: cmp/hi r2,r0 ! 57 MT
  114. mov.b @(r0,r5),r1 ! 20 LS (latency=2)
  115. bt/s 8b ! 109 BR
  116. mov.b r1,@-r0 ! 29 LS
  117. 9: rts
  118. nop
  119. ENTRY(memcpy)
  120. ! Calculate the invariants which will be used in the remainder
  121. ! of the code:
  122. !
  123. ! r4 --> [ ... ] DST [ ... ] SRC
  124. ! [ ... ] [ ... ]
  125. ! : :
  126. ! r0 --> [ ... ] r0+r5 --> [ ... ]
  127. !
  128. !
  129. ! Short circuit the common case of src, dst and len being 32 bit aligned
  130. ! and test for zero length move
  131. mov r6, r0 ! 5 MT (0 cycle latency)
  132. or r4, r0 ! 82 EX
  133. or r5, r0 ! 82 EX
  134. tst r6, r6 ! 86 MT
  135. bt/s 99f ! 111 BR (zero len)
  136. tst #3, r0 ! 87 MT
  137. mov r4, r0 ! 5 MT (0 cycle latency)
  138. add r6, r0 ! 49 EX
  139. mov #16, r1 ! 6 EX
  140. bt/s .Lcase00 ! 111 BR (aligned)
  141. sub r4, r5 ! 75 EX
  142. ! Arguments are not nicely long word aligned or zero len.
  143. ! Check for small copies, and if so do a simple byte at a time copy.
  144. !
  145. ! Deciding on an exact value of 'small' is not easy, as the point at which
  146. ! using the optimised routines become worthwhile varies (these are the
  147. ! cycle counts for differnet sizes using byte-at-a-time vs. optimised):
  148. ! size byte-at-time long word byte
  149. ! 16 42 39-40 46-50 50-55
  150. ! 24 58 43-44 54-58 62-67
  151. ! 36 82 49-50 66-70 80-85
  152. ! However the penalty for getting it 'wrong' is much higher for long word
  153. ! aligned data (and this is more common), so use a value of 16.
  154. cmp/gt r6,r1 ! 56 MT
  155. add #-1,r5 ! 50 EX
  156. bf/s 6f ! 108 BR (not small)
  157. mov r5, r3 ! 5 MT (latency=0)
  158. shlr r6 ! 104 EX
  159. mov.b @(r0,r5),r1 ! 20 LS (latency=2)
  160. bf/s 4f ! 111 BR
  161. add #-1,r3 ! 50 EX
  162. tst r6, r6 ! 86 MT
  163. bt/s 98f ! 110 BR
  164. mov.b r1,@-r0 ! 29 LS
  165. ! 4 cycles, 2 bytes per iteration
  166. 3: mov.b @(r0,r5),r1 ! 20 LS (latency=2)
  167. 4: mov.b @(r0,r3),r2 ! 20 LS (latency=2)
  168. dt r6 ! 67 EX
  169. mov.b r1,@-r0 ! 29 LS
  170. bf/s 3b ! 111 BR
  171. mov.b r2,@-r0 ! 29 LS
  172. 98:
  173. rts
  174. nop
  175. 99: rts
  176. mov r4, r0
  177. ! Size is not small, so its worthwhile looking for optimisations.
  178. ! First align destination to a long word boundary.
  179. !
  180. ! r5 = normal value -1
  181. 6: tst #3, r0 ! 87 MT
  182. mov #3, r3 ! 6 EX
  183. bt/s 2f ! 111 BR
  184. and r0,r3 ! 78 EX
  185. ! 3 cycles, 1 byte per iteration
  186. 1: dt r3 ! 67 EX
  187. mov.b @(r0,r5),r1 ! 19 LS (latency=2)
  188. add #-1, r6 ! 79 EX
  189. bf/s 1b ! 109 BR
  190. mov.b r1,@-r0 ! 28 LS
  191. 2: add #1, r5 ! 79 EX
  192. ! Now select the appropriate bulk transfer code based on relative
  193. ! alignment of src and dst.
  194. mov r0, r3 ! 5 MT (latency=0)
  195. mov r5, r0 ! 5 MT (latency=0)
  196. tst #1, r0 ! 87 MT
  197. bf/s 1f ! 111 BR
  198. mov #64, r7 ! 6 EX
  199. ! bit 0 clear
  200. cmp/ge r7, r6 ! 55 MT
  201. bt/s 2f ! 111 BR
  202. tst #2, r0 ! 87 MT
  203. ! small
  204. bt/s .Lcase0
  205. mov r3, r0
  206. bra .Lcase2
  207. nop
  208. ! big
  209. 2: bt/s .Lcase0b
  210. mov r3, r0
  211. bra .Lcase2b
  212. nop
  213. ! bit 0 set
  214. 1: tst #2, r0 ! 87 MT
  215. bt/s .Lcase1
  216. mov r3, r0
  217. bra .Lcase3
  218. nop
  219. !
  220. ! GHIJ KLMN OPQR --> GHIJ KLMN OPQR
  221. !
  222. ! src, dst and size are all long word aligned
  223. ! size is non-zero
  224. .balign 32
  225. .Lcase00:
  226. mov #64, r1 ! 6 EX
  227. mov r5, r3 ! 5 MT (latency=0)
  228. cmp/gt r6, r1 ! 56 MT
  229. add #-4, r5 ! 50 EX
  230. bf .Lcase00b ! 108 BR (big loop)
  231. shlr2 r6 ! 105 EX
  232. shlr r6 ! 104 EX
  233. mov.l @(r0, r5), r1 ! 21 LS (latency=2)
  234. bf/s 4f ! 111 BR
  235. add #-8, r3 ! 50 EX
  236. tst r6, r6 ! 86 MT
  237. bt/s 5f ! 110 BR
  238. mov.l r1,@-r0 ! 30 LS
  239. ! 4 cycles, 2 long words per iteration
  240. 3: mov.l @(r0, r5), r1 ! 21 LS (latency=2)
  241. 4: mov.l @(r0, r3), r2 ! 21 LS (latency=2)
  242. dt r6 ! 67 EX
  243. mov.l r1, @-r0 ! 30 LS
  244. bf/s 3b ! 109 BR
  245. mov.l r2, @-r0 ! 30 LS
  246. 5: rts
  247. nop
  248. ! Size is 16 or greater and less than 64, but may have trailing bytes
  249. .balign 32
  250. .Lcase0:
  251. add #-4, r5 ! 50 EX
  252. mov r4, r7 ! 5 MT (latency=0)
  253. mov.l @(r0, r5), r1 ! 21 LS (latency=2)
  254. mov #4, r2 ! 6 EX
  255. add #11, r7 ! 50 EX
  256. tst r2, r6 ! 86 MT
  257. mov r5, r3 ! 5 MT (latency=0)
  258. bt/s 4f ! 111 BR
  259. add #-4, r3 ! 50 EX
  260. mov.l r1,@-r0 ! 30 LS
  261. ! 4 cycles, 2 long words per iteration
  262. 3: mov.l @(r0, r5), r1 ! 21 LS (latency=2)
  263. 4: mov.l @(r0, r3), r2 ! 21 LS (latency=2)
  264. cmp/hi r7, r0
  265. mov.l r1, @-r0 ! 30 LS
  266. bt/s 3b ! 109 BR
  267. mov.l r2, @-r0 ! 30 LS
  268. ! Copy the final 0-3 bytes
  269. add #3,r5 ! 50 EX
  270. cmp/eq r0, r4 ! 54 MT
  271. add #-10, r7 ! 50 EX
  272. bt 9f ! 110 BR
  273. ! 3 cycles, 1 byte per iteration
  274. 1: mov.b @(r0,r5),r1 ! 19 LS
  275. cmp/hi r7,r0 ! 57 MT
  276. bt/s 1b ! 111 BR
  277. mov.b r1,@-r0 ! 28 LS
  278. 9: rts
  279. nop
  280. ! Size is at least 64 bytes, so will be going round the big loop at least once.
  281. !
  282. ! r2 = rounded up r4
  283. ! r3 = rounded down r0
  284. .balign 32
  285. .Lcase0b:
  286. add #-4, r5 ! 50 EX
  287. .Lcase00b:
  288. mov r0, r3 ! 5 MT (latency=0)
  289. mov #(~0x1f), r1 ! 6 EX
  290. and r1, r3 ! 78 EX
  291. mov r4, r2 ! 5 MT (latency=0)
  292. cmp/eq r3, r0 ! 54 MT
  293. add #0x1f, r2 ! 50 EX
  294. bt/s 1f ! 110 BR
  295. and r1, r2 ! 78 EX
  296. ! copy initial words until cache line aligned
  297. mov.l @(r0, r5), r1 ! 21 LS (latency=2)
  298. tst #4, r0 ! 87 MT
  299. mov r5, r6 ! 5 MT (latency=0)
  300. add #-4, r6 ! 50 EX
  301. bt/s 4f ! 111 BR
  302. add #8, r3 ! 50 EX
  303. tst #0x18, r0 ! 87 MT
  304. bt/s 1f ! 109 BR
  305. mov.l r1,@-r0 ! 30 LS
  306. ! 4 cycles, 2 long words per iteration
  307. 3: mov.l @(r0, r5), r1 ! 21 LS (latency=2)
  308. 4: mov.l @(r0, r6), r7 ! 21 LS (latency=2)
  309. cmp/eq r3, r0 ! 54 MT
  310. mov.l r1, @-r0 ! 30 LS
  311. bf/s 3b ! 109 BR
  312. mov.l r7, @-r0 ! 30 LS
  313. ! Copy the cache line aligned blocks
  314. !
  315. ! In use: r0, r2, r4, r5
  316. ! Scratch: r1, r3, r6, r7
  317. !
  318. ! We could do this with the four scratch registers, but if src
  319. ! and dest hit the same cache line, this will thrash, so make
  320. ! use of additional registers.
  321. !
  322. ! We also need r0 as a temporary (for movca), so 'undo' the invariant:
  323. ! r5: src (was r0+r5)
  324. ! r1: dest (was r0)
  325. ! this can be reversed at the end, so we don't need to save any extra
  326. ! state.
  327. !
  328. 1: mov.l r8, @-r15 ! 30 LS
  329. add r0, r5 ! 49 EX
  330. mov.l r9, @-r15 ! 30 LS
  331. mov r0, r1 ! 5 MT (latency=0)
  332. mov.l r10, @-r15 ! 30 LS
  333. add #-0x1c, r5 ! 50 EX
  334. mov.l r11, @-r15 ! 30 LS
  335. ! 16 cycles, 32 bytes per iteration
  336. 2: mov.l @(0x00,r5),r0 ! 18 LS (latency=2)
  337. add #-0x20, r1 ! 50 EX
  338. mov.l @(0x04,r5),r3 ! 18 LS (latency=2)
  339. mov.l @(0x08,r5),r6 ! 18 LS (latency=2)
  340. mov.l @(0x0c,r5),r7 ! 18 LS (latency=2)
  341. mov.l @(0x10,r5),r8 ! 18 LS (latency=2)
  342. mov.l @(0x14,r5),r9 ! 18 LS (latency=2)
  343. mov.l @(0x18,r5),r10 ! 18 LS (latency=2)
  344. mov.l @(0x1c,r5),r11 ! 18 LS (latency=2)
  345. movca.l r0,@r1 ! 40 LS (latency=3-7)
  346. mov.l r3,@(0x04,r1) ! 33 LS
  347. mov.l r6,@(0x08,r1) ! 33 LS
  348. mov.l r7,@(0x0c,r1) ! 33 LS
  349. mov.l r8,@(0x10,r1) ! 33 LS
  350. add #-0x20, r5 ! 50 EX
  351. mov.l r9,@(0x14,r1) ! 33 LS
  352. cmp/eq r2,r1 ! 54 MT
  353. mov.l r10,@(0x18,r1) ! 33 LS
  354. bf/s 2b ! 109 BR
  355. mov.l r11,@(0x1c,r1) ! 33 LS
  356. mov r1, r0 ! 5 MT (latency=0)
  357. mov.l @r15+, r11 ! 15 LS
  358. sub r1, r5 ! 75 EX
  359. mov.l @r15+, r10 ! 15 LS
  360. cmp/eq r4, r0 ! 54 MT
  361. bf/s 1f ! 109 BR
  362. mov.l @r15+, r9 ! 15 LS
  363. rts
  364. 1: mov.l @r15+, r8 ! 15 LS
  365. sub r4, r1 ! 75 EX (len remaining)
  366. ! number of trailing bytes is non-zero
  367. !
  368. ! invariants restored (r5 already decremented by 4)
  369. ! also r1=num bytes remaining
  370. mov #4, r2 ! 6 EX
  371. mov r4, r7 ! 5 MT (latency=0)
  372. add #0x1c, r5 ! 50 EX (back to -4)
  373. cmp/hs r2, r1 ! 58 MT
  374. bf/s 5f ! 108 BR
  375. add #11, r7 ! 50 EX
  376. mov.l @(r0, r5), r6 ! 21 LS (latency=2)
  377. tst r2, r1 ! 86 MT
  378. mov r5, r3 ! 5 MT (latency=0)
  379. bt/s 4f ! 111 BR
  380. add #-4, r3 ! 50 EX
  381. cmp/hs r2, r1 ! 58 MT
  382. bt/s 5f ! 111 BR
  383. mov.l r6,@-r0 ! 30 LS
  384. ! 4 cycles, 2 long words per iteration
  385. 3: mov.l @(r0, r5), r6 ! 21 LS (latency=2)
  386. 4: mov.l @(r0, r3), r2 ! 21 LS (latency=2)
  387. cmp/hi r7, r0
  388. mov.l r6, @-r0 ! 30 LS
  389. bt/s 3b ! 109 BR
  390. mov.l r2, @-r0 ! 30 LS
  391. ! Copy the final 0-3 bytes
  392. 5: cmp/eq r0, r4 ! 54 MT
  393. add #-10, r7 ! 50 EX
  394. bt 9f ! 110 BR
  395. add #3,r5 ! 50 EX
  396. ! 3 cycles, 1 byte per iteration
  397. 1: mov.b @(r0,r5),r1 ! 19 LS
  398. cmp/hi r7,r0 ! 57 MT
  399. bt/s 1b ! 111 BR
  400. mov.b r1,@-r0 ! 28 LS
  401. 9: rts
  402. nop
  403. !
  404. ! GHIJ KLMN OPQR --> ..GH IJKL MNOP QR..
  405. !
  406. .balign 32
  407. .Lcase2:
  408. ! Size is 16 or greater and less then 64, but may have trailing bytes
  409. 2: mov r5, r6 ! 5 MT (latency=0)
  410. add #-2,r5 ! 50 EX
  411. mov r4,r2 ! 5 MT (latency=0)
  412. add #-4,r6 ! 50 EX
  413. add #7,r2 ! 50 EX
  414. 3: mov.w @(r0,r5),r1 ! 20 LS (latency=2)
  415. mov.w @(r0,r6),r3 ! 20 LS (latency=2)
  416. cmp/hi r2,r0 ! 57 MT
  417. mov.w r1,@-r0 ! 29 LS
  418. bt/s 3b ! 111 BR
  419. mov.w r3,@-r0 ! 29 LS
  420. bra 10f
  421. nop
  422. .balign 32
  423. .Lcase2b:
  424. ! Size is at least 64 bytes, so will be going round the big loop at least once.
  425. !
  426. ! r2 = rounded up r4
  427. ! r3 = rounded down r0
  428. mov r0, r3 ! 5 MT (latency=0)
  429. mov #(~0x1f), r1 ! 6 EX
  430. and r1, r3 ! 78 EX
  431. mov r4, r2 ! 5 MT (latency=0)
  432. cmp/eq r3, r0 ! 54 MT
  433. add #0x1f, r2 ! 50 EX
  434. add #-2, r5 ! 50 EX
  435. bt/s 1f ! 110 BR
  436. and r1, r2 ! 78 EX
  437. ! Copy a short word one at a time until we are cache line aligned
  438. ! Normal values: r0, r2, r3, r4
  439. ! Unused: r1, r6, r7
  440. ! Mod: r5 (=r5-2)
  441. !
  442. add #2, r3 ! 50 EX
  443. 2: mov.w @(r0,r5),r1 ! 20 LS (latency=2)
  444. cmp/eq r3,r0 ! 54 MT
  445. bf/s 2b ! 111 BR
  446. mov.w r1,@-r0 ! 29 LS
  447. ! Copy the cache line aligned blocks
  448. !
  449. ! In use: r0, r2, r4, r5 (=r5-2)
  450. ! Scratch: r1, r3, r6, r7
  451. !
  452. ! We could do this with the four scratch registers, but if src
  453. ! and dest hit the same cache line, this will thrash, so make
  454. ! use of additional registers.
  455. !
  456. ! We also need r0 as a temporary (for movca), so 'undo' the invariant:
  457. ! r5: src (was r0+r5)
  458. ! r1: dest (was r0)
  459. ! this can be reversed at the end, so we don't need to save any extra
  460. ! state.
  461. !
  462. 1: mov.l r8, @-r15 ! 30 LS
  463. add r0, r5 ! 49 EX
  464. mov.l r9, @-r15 ! 30 LS
  465. mov r0, r1 ! 5 MT (latency=0)
  466. mov.l r10, @-r15 ! 30 LS
  467. add #-0x1e, r5 ! 50 EX
  468. mov.l r11, @-r15 ! 30 LS
  469. mov.l r12, @-r15 ! 30 LS
  470. ! 17 cycles, 32 bytes per iteration
  471. #ifdef CONFIG_CPU_LITTLE_ENDIAN
  472. 2: mov.w @r5+, r0 ! 14 LS (latency=2) ..JI
  473. add #-0x20, r1 ! 50 EX
  474. mov.l @r5+, r3 ! 15 LS (latency=2) NMLK
  475. mov.l @r5+, r6 ! 15 LS (latency=2) RQPO
  476. shll16 r0 ! 103 EX JI..
  477. mov.l @r5+, r7 ! 15 LS (latency=2)
  478. xtrct r3, r0 ! 48 EX LKJI
  479. mov.l @r5+, r8 ! 15 LS (latency=2)
  480. xtrct r6, r3 ! 48 EX PONM
  481. mov.l @r5+, r9 ! 15 LS (latency=2)
  482. xtrct r7, r6 ! 48 EX
  483. mov.l @r5+, r10 ! 15 LS (latency=2)
  484. xtrct r8, r7 ! 48 EX
  485. mov.l @r5+, r11 ! 15 LS (latency=2)
  486. xtrct r9, r8 ! 48 EX
  487. mov.w @r5+, r12 ! 15 LS (latency=2)
  488. xtrct r10, r9 ! 48 EX
  489. movca.l r0,@r1 ! 40 LS (latency=3-7)
  490. xtrct r11, r10 ! 48 EX
  491. mov.l r3, @(0x04,r1) ! 33 LS
  492. xtrct r12, r11 ! 48 EX
  493. mov.l r6, @(0x08,r1) ! 33 LS
  494. mov.l r7, @(0x0c,r1) ! 33 LS
  495. mov.l r8, @(0x10,r1) ! 33 LS
  496. add #-0x40, r5 ! 50 EX
  497. mov.l r9, @(0x14,r1) ! 33 LS
  498. cmp/eq r2,r1 ! 54 MT
  499. mov.l r10, @(0x18,r1) ! 33 LS
  500. bf/s 2b ! 109 BR
  501. mov.l r11, @(0x1c,r1) ! 33 LS
  502. #else
  503. 2: mov.w @(0x1e,r5), r0 ! 17 LS (latency=2)
  504. add #-2, r5 ! 50 EX
  505. mov.l @(0x1c,r5), r3 ! 18 LS (latency=2)
  506. add #-4, r1 ! 50 EX
  507. mov.l @(0x18,r5), r6 ! 18 LS (latency=2)
  508. shll16 r0 ! 103 EX
  509. mov.l @(0x14,r5), r7 ! 18 LS (latency=2)
  510. xtrct r3, r0 ! 48 EX
  511. mov.l @(0x10,r5), r8 ! 18 LS (latency=2)
  512. xtrct r6, r3 ! 48 EX
  513. mov.l @(0x0c,r5), r9 ! 18 LS (latency=2)
  514. xtrct r7, r6 ! 48 EX
  515. mov.l @(0x08,r5), r10 ! 18 LS (latency=2)
  516. xtrct r8, r7 ! 48 EX
  517. mov.l @(0x04,r5), r11 ! 18 LS (latency=2)
  518. xtrct r9, r8 ! 48 EX
  519. mov.l @(0x00,r5), r12 ! 18 LS (latency=2)
  520. xtrct r10, r9 ! 48 EX
  521. movca.l r0,@r1 ! 40 LS (latency=3-7)
  522. add #-0x1c, r1 ! 50 EX
  523. mov.l r3, @(0x18,r1) ! 33 LS
  524. xtrct r11, r10 ! 48 EX
  525. mov.l r6, @(0x14,r1) ! 33 LS
  526. xtrct r12, r11 ! 48 EX
  527. mov.l r7, @(0x10,r1) ! 33 LS
  528. mov.l r8, @(0x0c,r1) ! 33 LS
  529. add #-0x1e, r5 ! 50 EX
  530. mov.l r9, @(0x08,r1) ! 33 LS
  531. cmp/eq r2,r1 ! 54 MT
  532. mov.l r10, @(0x04,r1) ! 33 LS
  533. bf/s 2b ! 109 BR
  534. mov.l r11, @(0x00,r1) ! 33 LS
  535. #endif
  536. mov.l @r15+, r12
  537. mov r1, r0 ! 5 MT (latency=0)
  538. mov.l @r15+, r11 ! 15 LS
  539. sub r1, r5 ! 75 EX
  540. mov.l @r15+, r10 ! 15 LS
  541. cmp/eq r4, r0 ! 54 MT
  542. bf/s 1f ! 109 BR
  543. mov.l @r15+, r9 ! 15 LS
  544. rts
  545. 1: mov.l @r15+, r8 ! 15 LS
  546. add #0x1e, r5 ! 50 EX
  547. ! Finish off a short word at a time
  548. ! r5 must be invariant - 2
  549. 10: mov r4,r2 ! 5 MT (latency=0)
  550. add #1,r2 ! 50 EX
  551. cmp/hi r2, r0 ! 57 MT
  552. bf/s 1f ! 109 BR
  553. add #2, r2 ! 50 EX
  554. 3: mov.w @(r0,r5),r1 ! 20 LS
  555. cmp/hi r2,r0 ! 57 MT
  556. bt/s 3b ! 109 BR
  557. mov.w r1,@-r0 ! 29 LS
  558. 1:
  559. !
  560. ! Finally, copy the last byte if necessary
  561. cmp/eq r4,r0 ! 54 MT
  562. bt/s 9b
  563. add #1,r5
  564. mov.b @(r0,r5),r1
  565. rts
  566. mov.b r1,@-r0