gup_test.c 5.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250
  1. #include <linux/kernel.h>
  2. #include <linux/mm.h>
  3. #include <linux/slab.h>
  4. #include <linux/uaccess.h>
  5. #include <linux/ktime.h>
  6. #include <linux/debugfs.h>
  7. #include "gup_test.h"
  8. static void put_back_pages(unsigned int cmd, struct page **pages,
  9. unsigned long nr_pages, unsigned int gup_test_flags)
  10. {
  11. unsigned long i;
  12. switch (cmd) {
  13. case GUP_FAST_BENCHMARK:
  14. case GUP_BASIC_TEST:
  15. for (i = 0; i < nr_pages; i++)
  16. put_page(pages[i]);
  17. break;
  18. case PIN_FAST_BENCHMARK:
  19. case PIN_BASIC_TEST:
  20. case PIN_LONGTERM_BENCHMARK:
  21. unpin_user_pages(pages, nr_pages);
  22. break;
  23. case DUMP_USER_PAGES_TEST:
  24. if (gup_test_flags & GUP_TEST_FLAG_DUMP_PAGES_USE_PIN) {
  25. unpin_user_pages(pages, nr_pages);
  26. } else {
  27. for (i = 0; i < nr_pages; i++)
  28. put_page(pages[i]);
  29. }
  30. break;
  31. }
  32. }
  33. static void verify_dma_pinned(unsigned int cmd, struct page **pages,
  34. unsigned long nr_pages)
  35. {
  36. unsigned long i;
  37. struct page *page;
  38. switch (cmd) {
  39. case PIN_FAST_BENCHMARK:
  40. case PIN_BASIC_TEST:
  41. case PIN_LONGTERM_BENCHMARK:
  42. for (i = 0; i < nr_pages; i++) {
  43. page = pages[i];
  44. if (WARN(!page_maybe_dma_pinned(page),
  45. "pages[%lu] is NOT dma-pinned\n", i)) {
  46. dump_page(page, "gup_test failure");
  47. break;
  48. } else if (cmd == PIN_LONGTERM_BENCHMARK &&
  49. WARN(!is_longterm_pinnable_page(page),
  50. "pages[%lu] is NOT pinnable but pinned\n",
  51. i)) {
  52. dump_page(page, "gup_test failure");
  53. break;
  54. }
  55. }
  56. break;
  57. }
  58. }
  59. static void dump_pages_test(struct gup_test *gup, struct page **pages,
  60. unsigned long nr_pages)
  61. {
  62. unsigned int index_to_dump;
  63. unsigned int i;
  64. /*
  65. * Zero out any user-supplied page index that is out of range. Remember:
  66. * .which_pages[] contains a 1-based set of page indices.
  67. */
  68. for (i = 0; i < GUP_TEST_MAX_PAGES_TO_DUMP; i++) {
  69. if (gup->which_pages[i] > nr_pages) {
  70. pr_warn("ZEROING due to out of range: .which_pages[%u]: %u\n",
  71. i, gup->which_pages[i]);
  72. gup->which_pages[i] = 0;
  73. }
  74. }
  75. for (i = 0; i < GUP_TEST_MAX_PAGES_TO_DUMP; i++) {
  76. index_to_dump = gup->which_pages[i];
  77. if (index_to_dump) {
  78. index_to_dump--; // Decode from 1-based, to 0-based
  79. pr_info("---- page #%u, starting from user virt addr: 0x%llx\n",
  80. index_to_dump, gup->addr);
  81. dump_page(pages[index_to_dump],
  82. "gup_test: dump_pages() test");
  83. }
  84. }
  85. }
  86. static int __gup_test_ioctl(unsigned int cmd,
  87. struct gup_test *gup)
  88. {
  89. ktime_t start_time, end_time;
  90. unsigned long i, nr_pages, addr, next;
  91. long nr;
  92. struct page **pages;
  93. int ret = 0;
  94. bool needs_mmap_lock =
  95. cmd != GUP_FAST_BENCHMARK && cmd != PIN_FAST_BENCHMARK;
  96. if (gup->size > ULONG_MAX)
  97. return -EINVAL;
  98. nr_pages = gup->size / PAGE_SIZE;
  99. pages = kvcalloc(nr_pages, sizeof(void *), GFP_KERNEL);
  100. if (!pages)
  101. return -ENOMEM;
  102. if (needs_mmap_lock && mmap_read_lock_killable(current->mm)) {
  103. ret = -EINTR;
  104. goto free_pages;
  105. }
  106. i = 0;
  107. nr = gup->nr_pages_per_call;
  108. start_time = ktime_get();
  109. for (addr = gup->addr; addr < gup->addr + gup->size; addr = next) {
  110. if (nr != gup->nr_pages_per_call)
  111. break;
  112. next = addr + nr * PAGE_SIZE;
  113. if (next > gup->addr + gup->size) {
  114. next = gup->addr + gup->size;
  115. nr = (next - addr) / PAGE_SIZE;
  116. }
  117. switch (cmd) {
  118. case GUP_FAST_BENCHMARK:
  119. nr = get_user_pages_fast(addr, nr, gup->gup_flags,
  120. pages + i);
  121. break;
  122. case GUP_BASIC_TEST:
  123. nr = get_user_pages(addr, nr, gup->gup_flags, pages + i,
  124. NULL);
  125. break;
  126. case PIN_FAST_BENCHMARK:
  127. nr = pin_user_pages_fast(addr, nr, gup->gup_flags,
  128. pages + i);
  129. break;
  130. case PIN_BASIC_TEST:
  131. nr = pin_user_pages(addr, nr, gup->gup_flags, pages + i,
  132. NULL);
  133. break;
  134. case PIN_LONGTERM_BENCHMARK:
  135. nr = pin_user_pages(addr, nr,
  136. gup->gup_flags | FOLL_LONGTERM,
  137. pages + i, NULL);
  138. break;
  139. case DUMP_USER_PAGES_TEST:
  140. if (gup->test_flags & GUP_TEST_FLAG_DUMP_PAGES_USE_PIN)
  141. nr = pin_user_pages(addr, nr, gup->gup_flags,
  142. pages + i, NULL);
  143. else
  144. nr = get_user_pages(addr, nr, gup->gup_flags,
  145. pages + i, NULL);
  146. break;
  147. default:
  148. ret = -EINVAL;
  149. goto unlock;
  150. }
  151. if (nr <= 0)
  152. break;
  153. i += nr;
  154. }
  155. end_time = ktime_get();
  156. /* Shifting the meaning of nr_pages: now it is actual number pinned: */
  157. nr_pages = i;
  158. gup->get_delta_usec = ktime_us_delta(end_time, start_time);
  159. gup->size = addr - gup->addr;
  160. /*
  161. * Take an un-benchmark-timed moment to verify DMA pinned
  162. * state: print a warning if any non-dma-pinned pages are found:
  163. */
  164. verify_dma_pinned(cmd, pages, nr_pages);
  165. if (cmd == DUMP_USER_PAGES_TEST)
  166. dump_pages_test(gup, pages, nr_pages);
  167. start_time = ktime_get();
  168. put_back_pages(cmd, pages, nr_pages, gup->test_flags);
  169. end_time = ktime_get();
  170. gup->put_delta_usec = ktime_us_delta(end_time, start_time);
  171. unlock:
  172. if (needs_mmap_lock)
  173. mmap_read_unlock(current->mm);
  174. free_pages:
  175. kvfree(pages);
  176. return ret;
  177. }
  178. static long gup_test_ioctl(struct file *filep, unsigned int cmd,
  179. unsigned long arg)
  180. {
  181. struct gup_test gup;
  182. int ret;
  183. switch (cmd) {
  184. case GUP_FAST_BENCHMARK:
  185. case PIN_FAST_BENCHMARK:
  186. case PIN_LONGTERM_BENCHMARK:
  187. case GUP_BASIC_TEST:
  188. case PIN_BASIC_TEST:
  189. case DUMP_USER_PAGES_TEST:
  190. break;
  191. default:
  192. return -EINVAL;
  193. }
  194. if (copy_from_user(&gup, (void __user *)arg, sizeof(gup)))
  195. return -EFAULT;
  196. ret = __gup_test_ioctl(cmd, &gup);
  197. if (ret)
  198. return ret;
  199. if (copy_to_user((void __user *)arg, &gup, sizeof(gup)))
  200. return -EFAULT;
  201. return 0;
  202. }
  203. static const struct file_operations gup_test_fops = {
  204. .open = nonseekable_open,
  205. .unlocked_ioctl = gup_test_ioctl,
  206. };
  207. static int __init gup_test_init(void)
  208. {
  209. debugfs_create_file_unsafe("gup_test", 0600, NULL, NULL,
  210. &gup_test_fops);
  211. return 0;
  212. }
  213. late_initcall(gup_test_init);