page_owner.c 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747
  1. // SPDX-License-Identifier: GPL-2.0
  2. #include <linux/debugfs.h>
  3. #include <linux/mm.h>
  4. #include <linux/slab.h>
  5. #include <linux/uaccess.h>
  6. #include <linux/memblock.h>
  7. #include <linux/stacktrace.h>
  8. #include <linux/page_owner.h>
  9. #include <linux/jump_label.h>
  10. #include <linux/migrate.h>
  11. #include <linux/stackdepot.h>
  12. #include <linux/seq_file.h>
  13. #include <linux/memcontrol.h>
  14. #include <linux/sched/clock.h>
  15. #include "internal.h"
  16. /*
  17. * TODO: teach PAGE_OWNER_STACK_DEPTH (__dump_page_owner and save_stack)
  18. * to use off stack temporal storage
  19. */
  20. #define PAGE_OWNER_STACK_DEPTH (16)
  21. struct page_owner {
  22. unsigned short order;
  23. short last_migrate_reason;
  24. gfp_t gfp_mask;
  25. depot_stack_handle_t handle;
  26. depot_stack_handle_t free_handle;
  27. u64 ts_nsec;
  28. u64 free_ts_nsec;
  29. char comm[TASK_COMM_LEN];
  30. pid_t pid;
  31. pid_t tgid;
  32. };
  33. static bool page_owner_enabled __initdata;
  34. DEFINE_STATIC_KEY_FALSE(page_owner_inited);
  35. EXPORT_SYMBOL_GPL(page_owner_inited);
  36. static depot_stack_handle_t dummy_handle;
  37. static depot_stack_handle_t failure_handle;
  38. static depot_stack_handle_t early_handle;
  39. static void init_early_allocated_pages(void);
  40. static int __init early_page_owner_param(char *buf)
  41. {
  42. int ret = kstrtobool(buf, &page_owner_enabled);
  43. if (page_owner_enabled)
  44. stack_depot_want_early_init();
  45. return ret;
  46. }
  47. early_param("page_owner", early_page_owner_param);
  48. static __init bool need_page_owner(void)
  49. {
  50. return page_owner_enabled;
  51. }
  52. static __always_inline depot_stack_handle_t create_dummy_stack(void)
  53. {
  54. unsigned long entries[4];
  55. unsigned int nr_entries;
  56. nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 0);
  57. return stack_depot_save(entries, nr_entries, GFP_KERNEL);
  58. }
  59. static noinline void register_dummy_stack(void)
  60. {
  61. dummy_handle = create_dummy_stack();
  62. }
  63. static noinline void register_failure_stack(void)
  64. {
  65. failure_handle = create_dummy_stack();
  66. }
  67. static noinline void register_early_stack(void)
  68. {
  69. early_handle = create_dummy_stack();
  70. }
  71. static __init void init_page_owner(void)
  72. {
  73. if (!page_owner_enabled)
  74. return;
  75. register_dummy_stack();
  76. register_failure_stack();
  77. register_early_stack();
  78. static_branch_enable(&page_owner_inited);
  79. init_early_allocated_pages();
  80. }
  81. struct page_ext_operations page_owner_ops = {
  82. .size = sizeof(struct page_owner),
  83. .need = need_page_owner,
  84. .init = init_page_owner,
  85. };
  86. static inline struct page_owner *get_page_owner(struct page_ext *page_ext)
  87. {
  88. return (void *)page_ext + page_owner_ops.offset;
  89. }
  90. depot_stack_handle_t get_page_owner_handle(struct page_ext *page_ext, unsigned long pfn)
  91. {
  92. struct page_owner *page_owner;
  93. depot_stack_handle_t handle;
  94. if (!static_branch_unlikely(&page_owner_inited))
  95. return 0;
  96. page_owner = get_page_owner(page_ext);
  97. /* skip handle for tail pages of higher order allocations */
  98. if (!IS_ALIGNED(pfn, 1 << page_owner->order))
  99. return 0;
  100. handle = READ_ONCE(page_owner->handle);
  101. return handle;
  102. }
  103. EXPORT_SYMBOL_NS_GPL(get_page_owner_handle, MINIDUMP);
  104. static noinline depot_stack_handle_t save_stack(gfp_t flags)
  105. {
  106. unsigned long entries[PAGE_OWNER_STACK_DEPTH];
  107. depot_stack_handle_t handle;
  108. unsigned int nr_entries;
  109. /*
  110. * Avoid recursion.
  111. *
  112. * Sometimes page metadata allocation tracking requires more
  113. * memory to be allocated:
  114. * - when new stack trace is saved to stack depot
  115. * - when backtrace itself is calculated (ia64)
  116. */
  117. if (current->in_page_owner)
  118. return dummy_handle;
  119. current->in_page_owner = 1;
  120. nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 2);
  121. handle = stack_depot_save(entries, nr_entries, flags);
  122. if (!handle)
  123. handle = failure_handle;
  124. current->in_page_owner = 0;
  125. return handle;
  126. }
  127. void __reset_page_owner(struct page *page, unsigned short order)
  128. {
  129. int i;
  130. struct page_ext *page_ext;
  131. depot_stack_handle_t handle;
  132. struct page_owner *page_owner;
  133. u64 free_ts_nsec = local_clock();
  134. page_ext = page_ext_get(page);
  135. if (unlikely(!page_ext))
  136. return;
  137. handle = save_stack(GFP_NOWAIT | __GFP_NOWARN);
  138. for (i = 0; i < (1 << order); i++) {
  139. __clear_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags);
  140. page_owner = get_page_owner(page_ext);
  141. page_owner->free_handle = handle;
  142. page_owner->free_ts_nsec = free_ts_nsec;
  143. page_ext = page_ext_next(page_ext);
  144. }
  145. page_ext_put(page_ext);
  146. }
  147. static inline void __set_page_owner_handle(struct page_ext *page_ext,
  148. depot_stack_handle_t handle,
  149. unsigned short order, gfp_t gfp_mask)
  150. {
  151. struct page_owner *page_owner;
  152. int i;
  153. for (i = 0; i < (1 << order); i++) {
  154. page_owner = get_page_owner(page_ext);
  155. page_owner->handle = handle;
  156. page_owner->order = order;
  157. page_owner->gfp_mask = gfp_mask;
  158. page_owner->last_migrate_reason = -1;
  159. page_owner->pid = current->pid;
  160. page_owner->tgid = current->tgid;
  161. page_owner->ts_nsec = local_clock();
  162. strscpy(page_owner->comm, current->comm,
  163. sizeof(page_owner->comm));
  164. __set_bit(PAGE_EXT_OWNER, &page_ext->flags);
  165. __set_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags);
  166. page_ext = page_ext_next(page_ext);
  167. }
  168. }
  169. noinline void __set_page_owner(struct page *page, unsigned short order,
  170. gfp_t gfp_mask)
  171. {
  172. struct page_ext *page_ext;
  173. depot_stack_handle_t handle;
  174. handle = save_stack(gfp_mask);
  175. page_ext = page_ext_get(page);
  176. if (unlikely(!page_ext))
  177. return;
  178. __set_page_owner_handle(page_ext, handle, order, gfp_mask);
  179. page_ext_put(page_ext);
  180. }
  181. EXPORT_SYMBOL_GPL(__set_page_owner);
  182. void __set_page_owner_migrate_reason(struct page *page, int reason)
  183. {
  184. struct page_ext *page_ext = page_ext_get(page);
  185. struct page_owner *page_owner;
  186. if (unlikely(!page_ext))
  187. return;
  188. page_owner = get_page_owner(page_ext);
  189. page_owner->last_migrate_reason = reason;
  190. page_ext_put(page_ext);
  191. }
  192. void __split_page_owner(struct page *page, unsigned int nr)
  193. {
  194. int i;
  195. struct page_ext *page_ext = page_ext_get(page);
  196. struct page_owner *page_owner;
  197. if (unlikely(!page_ext))
  198. return;
  199. for (i = 0; i < nr; i++) {
  200. page_owner = get_page_owner(page_ext);
  201. page_owner->order = 0;
  202. page_ext = page_ext_next(page_ext);
  203. }
  204. page_ext_put(page_ext);
  205. }
  206. void __folio_copy_owner(struct folio *newfolio, struct folio *old)
  207. {
  208. struct page_ext *old_ext;
  209. struct page_ext *new_ext;
  210. struct page_owner *old_page_owner, *new_page_owner;
  211. old_ext = page_ext_get(&old->page);
  212. if (unlikely(!old_ext))
  213. return;
  214. new_ext = page_ext_get(&newfolio->page);
  215. if (unlikely(!new_ext)) {
  216. page_ext_put(old_ext);
  217. return;
  218. }
  219. old_page_owner = get_page_owner(old_ext);
  220. new_page_owner = get_page_owner(new_ext);
  221. new_page_owner->order = old_page_owner->order;
  222. new_page_owner->gfp_mask = old_page_owner->gfp_mask;
  223. new_page_owner->last_migrate_reason =
  224. old_page_owner->last_migrate_reason;
  225. new_page_owner->handle = old_page_owner->handle;
  226. new_page_owner->pid = old_page_owner->pid;
  227. new_page_owner->tgid = old_page_owner->tgid;
  228. new_page_owner->ts_nsec = old_page_owner->ts_nsec;
  229. new_page_owner->free_ts_nsec = old_page_owner->ts_nsec;
  230. strcpy(new_page_owner->comm, old_page_owner->comm);
  231. /*
  232. * We don't clear the bit on the old folio as it's going to be freed
  233. * after migration. Until then, the info can be useful in case of
  234. * a bug, and the overall stats will be off a bit only temporarily.
  235. * Also, migrate_misplaced_transhuge_page() can still fail the
  236. * migration and then we want the old folio to retain the info. But
  237. * in that case we also don't need to explicitly clear the info from
  238. * the new page, which will be freed.
  239. */
  240. __set_bit(PAGE_EXT_OWNER, &new_ext->flags);
  241. __set_bit(PAGE_EXT_OWNER_ALLOCATED, &new_ext->flags);
  242. page_ext_put(new_ext);
  243. page_ext_put(old_ext);
  244. }
  245. void pagetypeinfo_showmixedcount_print(struct seq_file *m,
  246. pg_data_t *pgdat, struct zone *zone)
  247. {
  248. struct page *page;
  249. struct page_ext *page_ext;
  250. struct page_owner *page_owner;
  251. unsigned long pfn, block_end_pfn;
  252. unsigned long end_pfn = zone_end_pfn(zone);
  253. unsigned long count[MIGRATE_TYPES] = { 0, };
  254. int pageblock_mt, page_mt;
  255. int i;
  256. /* Scan block by block. First and last block may be incomplete */
  257. pfn = zone->zone_start_pfn;
  258. /*
  259. * Walk the zone in pageblock_nr_pages steps. If a page block spans
  260. * a zone boundary, it will be double counted between zones. This does
  261. * not matter as the mixed block count will still be correct
  262. */
  263. for (; pfn < end_pfn; ) {
  264. page = pfn_to_online_page(pfn);
  265. if (!page) {
  266. pfn = ALIGN(pfn + 1, MAX_ORDER_NR_PAGES);
  267. continue;
  268. }
  269. block_end_pfn = pageblock_end_pfn(pfn);
  270. block_end_pfn = min(block_end_pfn, end_pfn);
  271. pageblock_mt = get_pageblock_migratetype(page);
  272. for (; pfn < block_end_pfn; pfn++) {
  273. /* The pageblock is online, no need to recheck. */
  274. page = pfn_to_page(pfn);
  275. if (page_zone(page) != zone)
  276. continue;
  277. if (PageBuddy(page)) {
  278. unsigned long freepage_order;
  279. freepage_order = buddy_order_unsafe(page);
  280. if (freepage_order < MAX_ORDER)
  281. pfn += (1UL << freepage_order) - 1;
  282. continue;
  283. }
  284. if (PageReserved(page))
  285. continue;
  286. page_ext = page_ext_get(page);
  287. if (unlikely(!page_ext))
  288. continue;
  289. if (!test_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags))
  290. goto ext_put_continue;
  291. page_owner = get_page_owner(page_ext);
  292. page_mt = gfp_migratetype(page_owner->gfp_mask);
  293. if (pageblock_mt != page_mt) {
  294. if (is_migrate_cma(pageblock_mt))
  295. count[MIGRATE_MOVABLE]++;
  296. else
  297. count[pageblock_mt]++;
  298. pfn = block_end_pfn;
  299. page_ext_put(page_ext);
  300. break;
  301. }
  302. pfn += (1UL << page_owner->order) - 1;
  303. ext_put_continue:
  304. page_ext_put(page_ext);
  305. }
  306. }
  307. /* Print counts */
  308. seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name);
  309. for (i = 0; i < MIGRATE_TYPES; i++)
  310. seq_printf(m, "%12lu ", count[i]);
  311. seq_putc(m, '\n');
  312. }
  313. /*
  314. * Looking for memcg information and print it out
  315. */
  316. static inline int print_page_owner_memcg(char *kbuf, size_t count, int ret,
  317. struct page *page)
  318. {
  319. #ifdef CONFIG_MEMCG
  320. unsigned long memcg_data;
  321. struct mem_cgroup *memcg;
  322. bool online;
  323. char name[80];
  324. rcu_read_lock();
  325. memcg_data = READ_ONCE(page->memcg_data);
  326. if (!memcg_data)
  327. goto out_unlock;
  328. if (memcg_data & MEMCG_DATA_OBJCGS)
  329. ret += scnprintf(kbuf + ret, count - ret,
  330. "Slab cache page\n");
  331. memcg = page_memcg_check(page);
  332. if (!memcg)
  333. goto out_unlock;
  334. online = (memcg->css.flags & CSS_ONLINE);
  335. cgroup_name(memcg->css.cgroup, name, sizeof(name));
  336. ret += scnprintf(kbuf + ret, count - ret,
  337. "Charged %sto %smemcg %s\n",
  338. PageMemcgKmem(page) ? "(via objcg) " : "",
  339. online ? "" : "offline ",
  340. name);
  341. out_unlock:
  342. rcu_read_unlock();
  343. #endif /* CONFIG_MEMCG */
  344. return ret;
  345. }
  346. static ssize_t
  347. print_page_owner(char __user *buf, size_t count, unsigned long pfn,
  348. struct page *page, struct page_owner *page_owner,
  349. depot_stack_handle_t handle)
  350. {
  351. int ret, pageblock_mt, page_mt;
  352. char *kbuf;
  353. count = min_t(size_t, count, PAGE_SIZE);
  354. kbuf = kmalloc(count, GFP_KERNEL);
  355. if (!kbuf)
  356. return -ENOMEM;
  357. ret = scnprintf(kbuf, count,
  358. "Page allocated via order %u, mask %#x(%pGg), pid %d, tgid %d (%s), ts %llu ns, free_ts %llu ns\n",
  359. page_owner->order, page_owner->gfp_mask,
  360. &page_owner->gfp_mask, page_owner->pid,
  361. page_owner->tgid, page_owner->comm,
  362. page_owner->ts_nsec, page_owner->free_ts_nsec);
  363. /* Print information relevant to grouping pages by mobility */
  364. pageblock_mt = get_pageblock_migratetype(page);
  365. page_mt = gfp_migratetype(page_owner->gfp_mask);
  366. ret += scnprintf(kbuf + ret, count - ret,
  367. "PFN %lu type %s Block %lu type %s Flags %pGp\n",
  368. pfn,
  369. migratetype_names[page_mt],
  370. pfn >> pageblock_order,
  371. migratetype_names[pageblock_mt],
  372. &page->flags);
  373. ret += stack_depot_snprint(handle, kbuf + ret, count - ret, 0);
  374. if (ret >= count)
  375. goto err;
  376. if (page_owner->last_migrate_reason != -1) {
  377. ret += scnprintf(kbuf + ret, count - ret,
  378. "Page has been migrated, last migrate reason: %s\n",
  379. migrate_reason_names[page_owner->last_migrate_reason]);
  380. }
  381. ret = print_page_owner_memcg(kbuf, count, ret, page);
  382. ret += snprintf(kbuf + ret, count - ret, "\n");
  383. if (ret >= count)
  384. goto err;
  385. if (copy_to_user(buf, kbuf, ret))
  386. ret = -EFAULT;
  387. kfree(kbuf);
  388. return ret;
  389. err:
  390. kfree(kbuf);
  391. return -ENOMEM;
  392. }
  393. void __dump_page_owner(const struct page *page)
  394. {
  395. struct page_ext *page_ext = page_ext_get((void *)page);
  396. struct page_owner *page_owner;
  397. depot_stack_handle_t handle;
  398. gfp_t gfp_mask;
  399. int mt;
  400. if (unlikely(!page_ext)) {
  401. pr_alert("There is not page extension available.\n");
  402. return;
  403. }
  404. page_owner = get_page_owner(page_ext);
  405. gfp_mask = page_owner->gfp_mask;
  406. mt = gfp_migratetype(gfp_mask);
  407. if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags)) {
  408. pr_alert("page_owner info is not present (never set?)\n");
  409. page_ext_put(page_ext);
  410. return;
  411. }
  412. if (test_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags))
  413. pr_alert("page_owner tracks the page as allocated\n");
  414. else
  415. pr_alert("page_owner tracks the page as freed\n");
  416. pr_alert("page last allocated via order %u, migratetype %s, gfp_mask %#x(%pGg), pid %d, tgid %d (%s), ts %llu, free_ts %llu\n",
  417. page_owner->order, migratetype_names[mt], gfp_mask, &gfp_mask,
  418. page_owner->pid, page_owner->tgid, page_owner->comm,
  419. page_owner->ts_nsec, page_owner->free_ts_nsec);
  420. handle = READ_ONCE(page_owner->handle);
  421. if (!handle)
  422. pr_alert("page_owner allocation stack trace missing\n");
  423. else
  424. stack_depot_print(handle);
  425. handle = READ_ONCE(page_owner->free_handle);
  426. if (!handle) {
  427. pr_alert("page_owner free stack trace missing\n");
  428. } else {
  429. pr_alert("page last free stack trace:\n");
  430. stack_depot_print(handle);
  431. }
  432. if (page_owner->last_migrate_reason != -1)
  433. pr_alert("page has been migrated, last migrate reason: %s\n",
  434. migrate_reason_names[page_owner->last_migrate_reason]);
  435. page_ext_put(page_ext);
  436. }
  437. static ssize_t
  438. read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos)
  439. {
  440. unsigned long pfn;
  441. struct page *page;
  442. struct page_ext *page_ext;
  443. struct page_owner *page_owner;
  444. depot_stack_handle_t handle;
  445. if (!static_branch_unlikely(&page_owner_inited))
  446. return -EINVAL;
  447. page = NULL;
  448. if (*ppos == 0)
  449. pfn = min_low_pfn;
  450. else
  451. pfn = *ppos;
  452. /* Find a valid PFN or the start of a MAX_ORDER_NR_PAGES area */
  453. while (!pfn_valid(pfn) && (pfn & (MAX_ORDER_NR_PAGES - 1)) != 0)
  454. pfn++;
  455. /* Find an allocated page */
  456. for (; pfn < max_pfn; pfn++) {
  457. /*
  458. * This temporary page_owner is required so
  459. * that we can avoid the context switches while holding
  460. * the rcu lock and copying the page owner information to
  461. * user through copy_to_user() or GFP_KERNEL allocations.
  462. */
  463. struct page_owner page_owner_tmp;
  464. /*
  465. * If the new page is in a new MAX_ORDER_NR_PAGES area,
  466. * validate the area as existing, skip it if not
  467. */
  468. if ((pfn & (MAX_ORDER_NR_PAGES - 1)) == 0 && !pfn_valid(pfn)) {
  469. pfn += MAX_ORDER_NR_PAGES - 1;
  470. continue;
  471. }
  472. page = pfn_to_page(pfn);
  473. if (PageBuddy(page)) {
  474. unsigned long freepage_order = buddy_order_unsafe(page);
  475. if (freepage_order < MAX_ORDER)
  476. pfn += (1UL << freepage_order) - 1;
  477. continue;
  478. }
  479. page_ext = page_ext_get(page);
  480. if (unlikely(!page_ext))
  481. continue;
  482. /*
  483. * Some pages could be missed by concurrent allocation or free,
  484. * because we don't hold the zone lock.
  485. */
  486. if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags))
  487. goto ext_put_continue;
  488. /*
  489. * Although we do have the info about past allocation of free
  490. * pages, it's not relevant for current memory usage.
  491. */
  492. if (!test_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags))
  493. goto ext_put_continue;
  494. page_owner = get_page_owner(page_ext);
  495. /*
  496. * Don't print "tail" pages of high-order allocations as that
  497. * would inflate the stats.
  498. */
  499. if (!IS_ALIGNED(pfn, 1 << page_owner->order))
  500. goto ext_put_continue;
  501. /*
  502. * Access to page_ext->handle isn't synchronous so we should
  503. * be careful to access it.
  504. */
  505. handle = READ_ONCE(page_owner->handle);
  506. if (!handle)
  507. goto ext_put_continue;
  508. /* Record the next PFN to read in the file offset */
  509. *ppos = pfn + 1;
  510. page_owner_tmp = *page_owner;
  511. page_ext_put(page_ext);
  512. return print_page_owner(buf, count, pfn, page,
  513. &page_owner_tmp, handle);
  514. ext_put_continue:
  515. page_ext_put(page_ext);
  516. }
  517. return 0;
  518. }
  519. static loff_t lseek_page_owner(struct file *file, loff_t offset, int orig)
  520. {
  521. switch (orig) {
  522. case SEEK_SET:
  523. file->f_pos = offset;
  524. break;
  525. case SEEK_CUR:
  526. file->f_pos += offset;
  527. break;
  528. default:
  529. return -EINVAL;
  530. }
  531. return file->f_pos;
  532. }
  533. static void init_pages_in_zone(pg_data_t *pgdat, struct zone *zone)
  534. {
  535. unsigned long pfn = zone->zone_start_pfn;
  536. unsigned long end_pfn = zone_end_pfn(zone);
  537. unsigned long count = 0;
  538. /*
  539. * Walk the zone in pageblock_nr_pages steps. If a page block spans
  540. * a zone boundary, it will be double counted between zones. This does
  541. * not matter as the mixed block count will still be correct
  542. */
  543. for (; pfn < end_pfn; ) {
  544. unsigned long block_end_pfn;
  545. if (!pfn_valid(pfn)) {
  546. pfn = ALIGN(pfn + 1, MAX_ORDER_NR_PAGES);
  547. continue;
  548. }
  549. block_end_pfn = pageblock_end_pfn(pfn);
  550. block_end_pfn = min(block_end_pfn, end_pfn);
  551. for (; pfn < block_end_pfn; pfn++) {
  552. struct page *page = pfn_to_page(pfn);
  553. struct page_ext *page_ext;
  554. if (page_zone(page) != zone)
  555. continue;
  556. /*
  557. * To avoid having to grab zone->lock, be a little
  558. * careful when reading buddy page order. The only
  559. * danger is that we skip too much and potentially miss
  560. * some early allocated pages, which is better than
  561. * heavy lock contention.
  562. */
  563. if (PageBuddy(page)) {
  564. unsigned long order = buddy_order_unsafe(page);
  565. if (order > 0 && order < MAX_ORDER)
  566. pfn += (1UL << order) - 1;
  567. continue;
  568. }
  569. if (PageReserved(page))
  570. continue;
  571. page_ext = page_ext_get(page);
  572. if (unlikely(!page_ext))
  573. continue;
  574. /* Maybe overlapping zone */
  575. if (test_bit(PAGE_EXT_OWNER, &page_ext->flags))
  576. goto ext_put_continue;
  577. /* Found early allocated page */
  578. __set_page_owner_handle(page_ext, early_handle,
  579. 0, 0);
  580. count++;
  581. ext_put_continue:
  582. page_ext_put(page_ext);
  583. }
  584. cond_resched();
  585. }
  586. pr_info("Node %d, zone %8s: page owner found early allocated %lu pages\n",
  587. pgdat->node_id, zone->name, count);
  588. }
  589. static void init_zones_in_node(pg_data_t *pgdat)
  590. {
  591. struct zone *zone;
  592. struct zone *node_zones = pgdat->node_zones;
  593. for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) {
  594. if (!populated_zone(zone))
  595. continue;
  596. init_pages_in_zone(pgdat, zone);
  597. }
  598. }
  599. static void init_early_allocated_pages(void)
  600. {
  601. pg_data_t *pgdat;
  602. for_each_online_pgdat(pgdat)
  603. init_zones_in_node(pgdat);
  604. }
  605. static const struct file_operations proc_page_owner_operations = {
  606. .read = read_page_owner,
  607. .llseek = lseek_page_owner,
  608. };
  609. static int __init pageowner_init(void)
  610. {
  611. if (!static_branch_unlikely(&page_owner_inited)) {
  612. pr_info("page_owner is disabled\n");
  613. return 0;
  614. }
  615. debugfs_create_file("page_owner", 0400, NULL, NULL,
  616. &proc_page_owner_operations);
  617. return 0;
  618. }
  619. late_initcall(pageowner_init)