usercopy_64.c 3.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * User address space access functions.
  4. *
  5. * Copyright 1997 Andi Kleen <[email protected]>
  6. * Copyright 1997 Linus Torvalds
  7. * Copyright 2002 Andi Kleen <[email protected]>
  8. */
  9. #include <linux/export.h>
  10. #include <linux/uaccess.h>
  11. #include <linux/highmem.h>
  12. /*
  13. * Zero Userspace
  14. */
  15. #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
  16. /**
  17. * clean_cache_range - write back a cache range with CLWB
  18. * @vaddr: virtual start address
  19. * @size: number of bytes to write back
  20. *
  21. * Write back a cache range using the CLWB (cache line write back)
  22. * instruction. Note that @size is internally rounded up to be cache
  23. * line size aligned.
  24. */
  25. static void clean_cache_range(void *addr, size_t size)
  26. {
  27. u16 x86_clflush_size = boot_cpu_data.x86_clflush_size;
  28. unsigned long clflush_mask = x86_clflush_size - 1;
  29. void *vend = addr + size;
  30. void *p;
  31. for (p = (void *)((unsigned long)addr & ~clflush_mask);
  32. p < vend; p += x86_clflush_size)
  33. clwb(p);
  34. }
  35. void arch_wb_cache_pmem(void *addr, size_t size)
  36. {
  37. clean_cache_range(addr, size);
  38. }
  39. EXPORT_SYMBOL_GPL(arch_wb_cache_pmem);
  40. long __copy_user_flushcache(void *dst, const void __user *src, unsigned size)
  41. {
  42. unsigned long flushed, dest = (unsigned long) dst;
  43. long rc = __copy_user_nocache(dst, src, size, 0);
  44. /*
  45. * __copy_user_nocache() uses non-temporal stores for the bulk
  46. * of the transfer, but we need to manually flush if the
  47. * transfer is unaligned. A cached memory copy is used when
  48. * destination or size is not naturally aligned. That is:
  49. * - Require 8-byte alignment when size is 8 bytes or larger.
  50. * - Require 4-byte alignment when size is 4 bytes.
  51. */
  52. if (size < 8) {
  53. if (!IS_ALIGNED(dest, 4) || size != 4)
  54. clean_cache_range(dst, size);
  55. } else {
  56. if (!IS_ALIGNED(dest, 8)) {
  57. dest = ALIGN(dest, boot_cpu_data.x86_clflush_size);
  58. clean_cache_range(dst, 1);
  59. }
  60. flushed = dest - (unsigned long) dst;
  61. if (size > flushed && !IS_ALIGNED(size - flushed, 8))
  62. clean_cache_range(dst + size - 1, 1);
  63. }
  64. return rc;
  65. }
  66. void __memcpy_flushcache(void *_dst, const void *_src, size_t size)
  67. {
  68. unsigned long dest = (unsigned long) _dst;
  69. unsigned long source = (unsigned long) _src;
  70. /* cache copy and flush to align dest */
  71. if (!IS_ALIGNED(dest, 8)) {
  72. size_t len = min_t(size_t, size, ALIGN(dest, 8) - dest);
  73. memcpy((void *) dest, (void *) source, len);
  74. clean_cache_range((void *) dest, len);
  75. dest += len;
  76. source += len;
  77. size -= len;
  78. if (!size)
  79. return;
  80. }
  81. /* 4x8 movnti loop */
  82. while (size >= 32) {
  83. asm("movq (%0), %%r8\n"
  84. "movq 8(%0), %%r9\n"
  85. "movq 16(%0), %%r10\n"
  86. "movq 24(%0), %%r11\n"
  87. "movnti %%r8, (%1)\n"
  88. "movnti %%r9, 8(%1)\n"
  89. "movnti %%r10, 16(%1)\n"
  90. "movnti %%r11, 24(%1)\n"
  91. :: "r" (source), "r" (dest)
  92. : "memory", "r8", "r9", "r10", "r11");
  93. dest += 32;
  94. source += 32;
  95. size -= 32;
  96. }
  97. /* 1x8 movnti loop */
  98. while (size >= 8) {
  99. asm("movq (%0), %%r8\n"
  100. "movnti %%r8, (%1)\n"
  101. :: "r" (source), "r" (dest)
  102. : "memory", "r8");
  103. dest += 8;
  104. source += 8;
  105. size -= 8;
  106. }
  107. /* 1x4 movnti loop */
  108. while (size >= 4) {
  109. asm("movl (%0), %%r8d\n"
  110. "movnti %%r8d, (%1)\n"
  111. :: "r" (source), "r" (dest)
  112. : "memory", "r8");
  113. dest += 4;
  114. source += 4;
  115. size -= 4;
  116. }
  117. /* cache copy for remaining bytes */
  118. if (size) {
  119. memcpy((void *) dest, (void *) source, size);
  120. clean_cache_range((void *) dest, size);
  121. }
  122. }
  123. EXPORT_SYMBOL_GPL(__memcpy_flushcache);
  124. void memcpy_page_flushcache(char *to, struct page *page, size_t offset,
  125. size_t len)
  126. {
  127. char *from = kmap_atomic(page);
  128. memcpy_flushcache(to, from + offset, len);
  129. kunmap_atomic(from);
  130. }
  131. #endif