mmu_gather.c 9.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367
  1. #include <linux/gfp.h>
  2. #include <linux/highmem.h>
  3. #include <linux/kernel.h>
  4. #include <linux/kmsan-checks.h>
  5. #include <linux/mmdebug.h>
  6. #include <linux/mm_types.h>
  7. #include <linux/mm_inline.h>
  8. #include <linux/pagemap.h>
  9. #include <linux/rcupdate.h>
  10. #include <linux/smp.h>
  11. #include <linux/swap.h>
  12. #include <asm/pgalloc.h>
  13. #include <asm/tlb.h>
  14. #ifndef CONFIG_MMU_GATHER_NO_GATHER
  15. static bool tlb_next_batch(struct mmu_gather *tlb)
  16. {
  17. struct mmu_gather_batch *batch;
  18. batch = tlb->active;
  19. if (batch->next) {
  20. tlb->active = batch->next;
  21. return true;
  22. }
  23. if (tlb->batch_count == MAX_GATHER_BATCH_COUNT)
  24. return false;
  25. batch = (void *)__get_free_pages(GFP_NOWAIT | __GFP_NOWARN, 0);
  26. if (!batch)
  27. return false;
  28. tlb->batch_count++;
  29. batch->next = NULL;
  30. batch->nr = 0;
  31. batch->max = MAX_GATHER_BATCH;
  32. tlb->active->next = batch;
  33. tlb->active = batch;
  34. return true;
  35. }
  36. static void tlb_batch_pages_flush(struct mmu_gather *tlb)
  37. {
  38. struct mmu_gather_batch *batch;
  39. for (batch = &tlb->local; batch && batch->nr; batch = batch->next) {
  40. struct page **pages = batch->pages;
  41. do {
  42. /*
  43. * limit free batch count when PAGE_SIZE > 4K
  44. */
  45. unsigned int nr = min(512U, batch->nr);
  46. free_pages_and_swap_cache(pages, nr);
  47. pages += nr;
  48. batch->nr -= nr;
  49. cond_resched();
  50. } while (batch->nr);
  51. }
  52. tlb->active = &tlb->local;
  53. }
  54. static void tlb_batch_list_free(struct mmu_gather *tlb)
  55. {
  56. struct mmu_gather_batch *batch, *next;
  57. for (batch = tlb->local.next; batch; batch = next) {
  58. next = batch->next;
  59. free_pages((unsigned long)batch, 0);
  60. }
  61. tlb->local.next = NULL;
  62. }
  63. bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, int page_size)
  64. {
  65. struct mmu_gather_batch *batch;
  66. VM_BUG_ON(!tlb->end);
  67. #ifdef CONFIG_MMU_GATHER_PAGE_SIZE
  68. VM_WARN_ON(tlb->page_size != page_size);
  69. #endif
  70. batch = tlb->active;
  71. /*
  72. * Add the page and check if we are full. If so
  73. * force a flush.
  74. */
  75. batch->pages[batch->nr++] = page;
  76. if (batch->nr == batch->max) {
  77. if (!tlb_next_batch(tlb))
  78. return true;
  79. batch = tlb->active;
  80. }
  81. VM_BUG_ON_PAGE(batch->nr > batch->max, page);
  82. return false;
  83. }
  84. #endif /* MMU_GATHER_NO_GATHER */
  85. #ifdef CONFIG_MMU_GATHER_TABLE_FREE
  86. static void __tlb_remove_table_free(struct mmu_table_batch *batch)
  87. {
  88. int i;
  89. for (i = 0; i < batch->nr; i++)
  90. __tlb_remove_table(batch->tables[i]);
  91. free_page((unsigned long)batch);
  92. }
  93. #ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE
  94. /*
  95. * Semi RCU freeing of the page directories.
  96. *
  97. * This is needed by some architectures to implement software pagetable walkers.
  98. *
  99. * gup_fast() and other software pagetable walkers do a lockless page-table
  100. * walk and therefore needs some synchronization with the freeing of the page
  101. * directories. The chosen means to accomplish that is by disabling IRQs over
  102. * the walk.
  103. *
  104. * Architectures that use IPIs to flush TLBs will then automagically DTRT,
  105. * since we unlink the page, flush TLBs, free the page. Since the disabling of
  106. * IRQs delays the completion of the TLB flush we can never observe an already
  107. * freed page.
  108. *
  109. * Architectures that do not have this (PPC) need to delay the freeing by some
  110. * other means, this is that means.
  111. *
  112. * What we do is batch the freed directory pages (tables) and RCU free them.
  113. * We use the sched RCU variant, as that guarantees that IRQ/preempt disabling
  114. * holds off grace periods.
  115. *
  116. * However, in order to batch these pages we need to allocate storage, this
  117. * allocation is deep inside the MM code and can thus easily fail on memory
  118. * pressure. To guarantee progress we fall back to single table freeing, see
  119. * the implementation of tlb_remove_table_one().
  120. *
  121. */
  122. static void tlb_remove_table_smp_sync(void *arg)
  123. {
  124. /* Simply deliver the interrupt */
  125. }
  126. void tlb_remove_table_sync_one(void)
  127. {
  128. /*
  129. * This isn't an RCU grace period and hence the page-tables cannot be
  130. * assumed to be actually RCU-freed.
  131. *
  132. * It is however sufficient for software page-table walkers that rely on
  133. * IRQ disabling.
  134. */
  135. smp_call_function(tlb_remove_table_smp_sync, NULL, 1);
  136. }
  137. static void tlb_remove_table_rcu(struct rcu_head *head)
  138. {
  139. __tlb_remove_table_free(container_of(head, struct mmu_table_batch, rcu));
  140. }
  141. static void tlb_remove_table_free(struct mmu_table_batch *batch)
  142. {
  143. call_rcu(&batch->rcu, tlb_remove_table_rcu);
  144. }
  145. #else /* !CONFIG_MMU_GATHER_RCU_TABLE_FREE */
  146. static void tlb_remove_table_free(struct mmu_table_batch *batch)
  147. {
  148. __tlb_remove_table_free(batch);
  149. }
  150. #endif /* CONFIG_MMU_GATHER_RCU_TABLE_FREE */
  151. /*
  152. * If we want tlb_remove_table() to imply TLB invalidates.
  153. */
  154. static inline void tlb_table_invalidate(struct mmu_gather *tlb)
  155. {
  156. if (tlb_needs_table_invalidate()) {
  157. /*
  158. * Invalidate page-table caches used by hardware walkers. Then
  159. * we still need to RCU-sched wait while freeing the pages
  160. * because software walkers can still be in-flight.
  161. */
  162. tlb_flush_mmu_tlbonly(tlb);
  163. }
  164. }
  165. static void tlb_remove_table_one(void *table)
  166. {
  167. tlb_remove_table_sync_one();
  168. __tlb_remove_table(table);
  169. }
  170. static void tlb_table_flush(struct mmu_gather *tlb)
  171. {
  172. struct mmu_table_batch **batch = &tlb->batch;
  173. if (*batch) {
  174. tlb_table_invalidate(tlb);
  175. tlb_remove_table_free(*batch);
  176. *batch = NULL;
  177. }
  178. }
  179. void tlb_remove_table(struct mmu_gather *tlb, void *table)
  180. {
  181. struct mmu_table_batch **batch = &tlb->batch;
  182. if (*batch == NULL) {
  183. *batch = (struct mmu_table_batch *)__get_free_page(GFP_NOWAIT | __GFP_NOWARN);
  184. if (*batch == NULL) {
  185. tlb_table_invalidate(tlb);
  186. tlb_remove_table_one(table);
  187. return;
  188. }
  189. (*batch)->nr = 0;
  190. }
  191. (*batch)->tables[(*batch)->nr++] = table;
  192. if ((*batch)->nr == MAX_TABLE_BATCH)
  193. tlb_table_flush(tlb);
  194. }
  195. static inline void tlb_table_init(struct mmu_gather *tlb)
  196. {
  197. tlb->batch = NULL;
  198. }
  199. #else /* !CONFIG_MMU_GATHER_TABLE_FREE */
  200. static inline void tlb_table_flush(struct mmu_gather *tlb) { }
  201. static inline void tlb_table_init(struct mmu_gather *tlb) { }
  202. #endif /* CONFIG_MMU_GATHER_TABLE_FREE */
  203. static void tlb_flush_mmu_free(struct mmu_gather *tlb)
  204. {
  205. tlb_table_flush(tlb);
  206. #ifndef CONFIG_MMU_GATHER_NO_GATHER
  207. tlb_batch_pages_flush(tlb);
  208. #endif
  209. }
  210. void tlb_flush_mmu(struct mmu_gather *tlb)
  211. {
  212. tlb_flush_mmu_tlbonly(tlb);
  213. tlb_flush_mmu_free(tlb);
  214. }
  215. static void __tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm,
  216. bool fullmm)
  217. {
  218. /*
  219. * struct mmu_gather contains 7 1-bit fields packed into a 32-bit
  220. * unsigned int value. The remaining 25 bits remain uninitialized
  221. * and are never used, but KMSAN updates the origin for them in
  222. * zap_pXX_range() in mm/memory.c, thus creating very long origin
  223. * chains. This is technically correct, but consumes too much memory.
  224. * Unpoisoning the whole structure will prevent creating such chains.
  225. */
  226. kmsan_unpoison_memory(tlb, sizeof(*tlb));
  227. tlb->mm = mm;
  228. tlb->fullmm = fullmm;
  229. #ifndef CONFIG_MMU_GATHER_NO_GATHER
  230. tlb->need_flush_all = 0;
  231. tlb->local.next = NULL;
  232. tlb->local.nr = 0;
  233. tlb->local.max = ARRAY_SIZE(tlb->__pages);
  234. tlb->active = &tlb->local;
  235. tlb->batch_count = 0;
  236. #endif
  237. tlb_table_init(tlb);
  238. #ifdef CONFIG_MMU_GATHER_PAGE_SIZE
  239. tlb->page_size = 0;
  240. #endif
  241. __tlb_reset_range(tlb);
  242. inc_tlb_flush_pending(tlb->mm);
  243. }
  244. /**
  245. * tlb_gather_mmu - initialize an mmu_gather structure for page-table tear-down
  246. * @tlb: the mmu_gather structure to initialize
  247. * @mm: the mm_struct of the target address space
  248. *
  249. * Called to initialize an (on-stack) mmu_gather structure for page-table
  250. * tear-down from @mm.
  251. */
  252. void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm)
  253. {
  254. __tlb_gather_mmu(tlb, mm, false);
  255. }
  256. /**
  257. * tlb_gather_mmu_fullmm - initialize an mmu_gather structure for page-table tear-down
  258. * @tlb: the mmu_gather structure to initialize
  259. * @mm: the mm_struct of the target address space
  260. *
  261. * In this case, @mm is without users and we're going to destroy the
  262. * full address space (exit/execve).
  263. *
  264. * Called to initialize an (on-stack) mmu_gather structure for page-table
  265. * tear-down from @mm.
  266. */
  267. void tlb_gather_mmu_fullmm(struct mmu_gather *tlb, struct mm_struct *mm)
  268. {
  269. __tlb_gather_mmu(tlb, mm, true);
  270. }
  271. /**
  272. * tlb_finish_mmu - finish an mmu_gather structure
  273. * @tlb: the mmu_gather structure to finish
  274. *
  275. * Called at the end of the shootdown operation to free up any resources that
  276. * were required.
  277. */
  278. void tlb_finish_mmu(struct mmu_gather *tlb)
  279. {
  280. /*
  281. * If there are parallel threads are doing PTE changes on same range
  282. * under non-exclusive lock (e.g., mmap_lock read-side) but defer TLB
  283. * flush by batching, one thread may end up seeing inconsistent PTEs
  284. * and result in having stale TLB entries. So flush TLB forcefully
  285. * if we detect parallel PTE batching threads.
  286. *
  287. * However, some syscalls, e.g. munmap(), may free page tables, this
  288. * needs force flush everything in the given range. Otherwise this
  289. * may result in having stale TLB entries for some architectures,
  290. * e.g. aarch64, that could specify flush what level TLB.
  291. */
  292. if (mm_tlb_flush_nested(tlb->mm)) {
  293. /*
  294. * The aarch64 yields better performance with fullmm by
  295. * avoiding multiple CPUs spamming TLBI messages at the
  296. * same time.
  297. *
  298. * On x86 non-fullmm doesn't yield significant difference
  299. * against fullmm.
  300. */
  301. tlb->fullmm = 1;
  302. __tlb_reset_range(tlb);
  303. tlb->freed_tables = 1;
  304. }
  305. tlb_flush_mmu(tlb);
  306. #ifndef CONFIG_MMU_GATHER_NO_GATHER
  307. tlb_batch_list_free(tlb);
  308. #endif
  309. dec_tlb_flush_pending(tlb->mm);
  310. }