builtin-kmem.c 47 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054
  1. // SPDX-License-Identifier: GPL-2.0
  2. #include "builtin.h"
  3. #include "perf.h"
  4. #include "util/dso.h"
  5. #include "util/evlist.h"
  6. #include "util/evsel.h"
  7. #include "util/config.h"
  8. #include "util/map.h"
  9. #include "util/symbol.h"
  10. #include "util/thread.h"
  11. #include "util/header.h"
  12. #include "util/session.h"
  13. #include "util/tool.h"
  14. #include "util/callchain.h"
  15. #include "util/time-utils.h"
  16. #include <linux/err.h>
  17. #include <subcmd/pager.h>
  18. #include <subcmd/parse-options.h>
  19. #include "util/trace-event.h"
  20. #include "util/data.h"
  21. #include "util/cpumap.h"
  22. #include "util/debug.h"
  23. #include "util/string2.h"
  24. #include <linux/kernel.h>
  25. #include <linux/numa.h>
  26. #include <linux/rbtree.h>
  27. #include <linux/string.h>
  28. #include <linux/zalloc.h>
  29. #include <errno.h>
  30. #include <inttypes.h>
  31. #include <locale.h>
  32. #include <regex.h>
  33. #include <linux/ctype.h>
  34. static int kmem_slab;
  35. static int kmem_page;
  36. static long kmem_page_size;
  37. static enum {
  38. KMEM_SLAB,
  39. KMEM_PAGE,
  40. } kmem_default = KMEM_SLAB; /* for backward compatibility */
  41. struct alloc_stat;
  42. typedef int (*sort_fn_t)(void *, void *);
  43. static int alloc_flag;
  44. static int caller_flag;
  45. static int alloc_lines = -1;
  46. static int caller_lines = -1;
  47. static bool raw_ip;
  48. struct alloc_stat {
  49. u64 call_site;
  50. u64 ptr;
  51. u64 bytes_req;
  52. u64 bytes_alloc;
  53. u64 last_alloc;
  54. u32 hit;
  55. u32 pingpong;
  56. short alloc_cpu;
  57. struct rb_node node;
  58. };
  59. static struct rb_root root_alloc_stat;
  60. static struct rb_root root_alloc_sorted;
  61. static struct rb_root root_caller_stat;
  62. static struct rb_root root_caller_sorted;
  63. static unsigned long total_requested, total_allocated, total_freed;
  64. static unsigned long nr_allocs, nr_cross_allocs;
  65. /* filters for controlling start and stop of time of analysis */
  66. static struct perf_time_interval ptime;
  67. const char *time_str;
  68. static int insert_alloc_stat(unsigned long call_site, unsigned long ptr,
  69. int bytes_req, int bytes_alloc, int cpu)
  70. {
  71. struct rb_node **node = &root_alloc_stat.rb_node;
  72. struct rb_node *parent = NULL;
  73. struct alloc_stat *data = NULL;
  74. while (*node) {
  75. parent = *node;
  76. data = rb_entry(*node, struct alloc_stat, node);
  77. if (ptr > data->ptr)
  78. node = &(*node)->rb_right;
  79. else if (ptr < data->ptr)
  80. node = &(*node)->rb_left;
  81. else
  82. break;
  83. }
  84. if (data && data->ptr == ptr) {
  85. data->hit++;
  86. data->bytes_req += bytes_req;
  87. data->bytes_alloc += bytes_alloc;
  88. } else {
  89. data = malloc(sizeof(*data));
  90. if (!data) {
  91. pr_err("%s: malloc failed\n", __func__);
  92. return -1;
  93. }
  94. data->ptr = ptr;
  95. data->pingpong = 0;
  96. data->hit = 1;
  97. data->bytes_req = bytes_req;
  98. data->bytes_alloc = bytes_alloc;
  99. rb_link_node(&data->node, parent, node);
  100. rb_insert_color(&data->node, &root_alloc_stat);
  101. }
  102. data->call_site = call_site;
  103. data->alloc_cpu = cpu;
  104. data->last_alloc = bytes_alloc;
  105. return 0;
  106. }
  107. static int insert_caller_stat(unsigned long call_site,
  108. int bytes_req, int bytes_alloc)
  109. {
  110. struct rb_node **node = &root_caller_stat.rb_node;
  111. struct rb_node *parent = NULL;
  112. struct alloc_stat *data = NULL;
  113. while (*node) {
  114. parent = *node;
  115. data = rb_entry(*node, struct alloc_stat, node);
  116. if (call_site > data->call_site)
  117. node = &(*node)->rb_right;
  118. else if (call_site < data->call_site)
  119. node = &(*node)->rb_left;
  120. else
  121. break;
  122. }
  123. if (data && data->call_site == call_site) {
  124. data->hit++;
  125. data->bytes_req += bytes_req;
  126. data->bytes_alloc += bytes_alloc;
  127. } else {
  128. data = malloc(sizeof(*data));
  129. if (!data) {
  130. pr_err("%s: malloc failed\n", __func__);
  131. return -1;
  132. }
  133. data->call_site = call_site;
  134. data->pingpong = 0;
  135. data->hit = 1;
  136. data->bytes_req = bytes_req;
  137. data->bytes_alloc = bytes_alloc;
  138. rb_link_node(&data->node, parent, node);
  139. rb_insert_color(&data->node, &root_caller_stat);
  140. }
  141. return 0;
  142. }
  143. static int evsel__process_alloc_event(struct evsel *evsel, struct perf_sample *sample)
  144. {
  145. unsigned long ptr = evsel__intval(evsel, sample, "ptr"),
  146. call_site = evsel__intval(evsel, sample, "call_site");
  147. int bytes_req = evsel__intval(evsel, sample, "bytes_req"),
  148. bytes_alloc = evsel__intval(evsel, sample, "bytes_alloc");
  149. if (insert_alloc_stat(call_site, ptr, bytes_req, bytes_alloc, sample->cpu) ||
  150. insert_caller_stat(call_site, bytes_req, bytes_alloc))
  151. return -1;
  152. total_requested += bytes_req;
  153. total_allocated += bytes_alloc;
  154. nr_allocs++;
  155. /*
  156. * Commit 11e9734bcb6a ("mm/slab_common: unify NUMA and UMA
  157. * version of tracepoints") adds the field "node" into the
  158. * tracepoints 'kmalloc' and 'kmem_cache_alloc'.
  159. *
  160. * The legacy tracepoints 'kmalloc_node' and 'kmem_cache_alloc_node'
  161. * also contain the field "node".
  162. *
  163. * If the tracepoint contains the field "node" the tool stats the
  164. * cross allocation.
  165. */
  166. if (evsel__field(evsel, "node")) {
  167. int node1, node2;
  168. node1 = cpu__get_node((struct perf_cpu){.cpu = sample->cpu});
  169. node2 = evsel__intval(evsel, sample, "node");
  170. /*
  171. * If the field "node" is NUMA_NO_NODE (-1), we don't take it
  172. * as a cross allocation.
  173. */
  174. if ((node2 != NUMA_NO_NODE) && (node1 != node2))
  175. nr_cross_allocs++;
  176. }
  177. return 0;
  178. }
  179. static int ptr_cmp(void *, void *);
  180. static int slab_callsite_cmp(void *, void *);
  181. static struct alloc_stat *search_alloc_stat(unsigned long ptr,
  182. unsigned long call_site,
  183. struct rb_root *root,
  184. sort_fn_t sort_fn)
  185. {
  186. struct rb_node *node = root->rb_node;
  187. struct alloc_stat key = { .ptr = ptr, .call_site = call_site };
  188. while (node) {
  189. struct alloc_stat *data;
  190. int cmp;
  191. data = rb_entry(node, struct alloc_stat, node);
  192. cmp = sort_fn(&key, data);
  193. if (cmp < 0)
  194. node = node->rb_left;
  195. else if (cmp > 0)
  196. node = node->rb_right;
  197. else
  198. return data;
  199. }
  200. return NULL;
  201. }
  202. static int evsel__process_free_event(struct evsel *evsel, struct perf_sample *sample)
  203. {
  204. unsigned long ptr = evsel__intval(evsel, sample, "ptr");
  205. struct alloc_stat *s_alloc, *s_caller;
  206. s_alloc = search_alloc_stat(ptr, 0, &root_alloc_stat, ptr_cmp);
  207. if (!s_alloc)
  208. return 0;
  209. total_freed += s_alloc->last_alloc;
  210. if ((short)sample->cpu != s_alloc->alloc_cpu) {
  211. s_alloc->pingpong++;
  212. s_caller = search_alloc_stat(0, s_alloc->call_site,
  213. &root_caller_stat,
  214. slab_callsite_cmp);
  215. if (!s_caller)
  216. return -1;
  217. s_caller->pingpong++;
  218. }
  219. s_alloc->alloc_cpu = -1;
  220. return 0;
  221. }
  222. static u64 total_page_alloc_bytes;
  223. static u64 total_page_free_bytes;
  224. static u64 total_page_nomatch_bytes;
  225. static u64 total_page_fail_bytes;
  226. static unsigned long nr_page_allocs;
  227. static unsigned long nr_page_frees;
  228. static unsigned long nr_page_fails;
  229. static unsigned long nr_page_nomatch;
  230. static bool use_pfn;
  231. static bool live_page;
  232. static struct perf_session *kmem_session;
  233. #define MAX_MIGRATE_TYPES 6
  234. #define MAX_PAGE_ORDER 11
  235. static int order_stats[MAX_PAGE_ORDER][MAX_MIGRATE_TYPES];
  236. struct page_stat {
  237. struct rb_node node;
  238. u64 page;
  239. u64 callsite;
  240. int order;
  241. unsigned gfp_flags;
  242. unsigned migrate_type;
  243. u64 alloc_bytes;
  244. u64 free_bytes;
  245. int nr_alloc;
  246. int nr_free;
  247. };
  248. static struct rb_root page_live_tree;
  249. static struct rb_root page_alloc_tree;
  250. static struct rb_root page_alloc_sorted;
  251. static struct rb_root page_caller_tree;
  252. static struct rb_root page_caller_sorted;
  253. struct alloc_func {
  254. u64 start;
  255. u64 end;
  256. char *name;
  257. };
  258. static int nr_alloc_funcs;
  259. static struct alloc_func *alloc_func_list;
  260. static int funcmp(const void *a, const void *b)
  261. {
  262. const struct alloc_func *fa = a;
  263. const struct alloc_func *fb = b;
  264. if (fa->start > fb->start)
  265. return 1;
  266. else
  267. return -1;
  268. }
  269. static int callcmp(const void *a, const void *b)
  270. {
  271. const struct alloc_func *fa = a;
  272. const struct alloc_func *fb = b;
  273. if (fb->start <= fa->start && fa->end < fb->end)
  274. return 0;
  275. if (fa->start > fb->start)
  276. return 1;
  277. else
  278. return -1;
  279. }
  280. static int build_alloc_func_list(void)
  281. {
  282. int ret;
  283. struct map *kernel_map;
  284. struct symbol *sym;
  285. struct rb_node *node;
  286. struct alloc_func *func;
  287. struct machine *machine = &kmem_session->machines.host;
  288. regex_t alloc_func_regex;
  289. static const char pattern[] = "^_?_?(alloc|get_free|get_zeroed)_pages?";
  290. ret = regcomp(&alloc_func_regex, pattern, REG_EXTENDED);
  291. if (ret) {
  292. char err[BUFSIZ];
  293. regerror(ret, &alloc_func_regex, err, sizeof(err));
  294. pr_err("Invalid regex: %s\n%s", pattern, err);
  295. return -EINVAL;
  296. }
  297. kernel_map = machine__kernel_map(machine);
  298. if (map__load(kernel_map) < 0) {
  299. pr_err("cannot load kernel map\n");
  300. return -ENOENT;
  301. }
  302. map__for_each_symbol(kernel_map, sym, node) {
  303. if (regexec(&alloc_func_regex, sym->name, 0, NULL, 0))
  304. continue;
  305. func = realloc(alloc_func_list,
  306. (nr_alloc_funcs + 1) * sizeof(*func));
  307. if (func == NULL)
  308. return -ENOMEM;
  309. pr_debug("alloc func: %s\n", sym->name);
  310. func[nr_alloc_funcs].start = sym->start;
  311. func[nr_alloc_funcs].end = sym->end;
  312. func[nr_alloc_funcs].name = sym->name;
  313. alloc_func_list = func;
  314. nr_alloc_funcs++;
  315. }
  316. qsort(alloc_func_list, nr_alloc_funcs, sizeof(*func), funcmp);
  317. regfree(&alloc_func_regex);
  318. return 0;
  319. }
  320. /*
  321. * Find first non-memory allocation function from callchain.
  322. * The allocation functions are in the 'alloc_func_list'.
  323. */
  324. static u64 find_callsite(struct evsel *evsel, struct perf_sample *sample)
  325. {
  326. struct addr_location al;
  327. struct machine *machine = &kmem_session->machines.host;
  328. struct callchain_cursor_node *node;
  329. if (alloc_func_list == NULL) {
  330. if (build_alloc_func_list() < 0)
  331. goto out;
  332. }
  333. al.thread = machine__findnew_thread(machine, sample->pid, sample->tid);
  334. sample__resolve_callchain(sample, &callchain_cursor, NULL, evsel, &al, 16);
  335. callchain_cursor_commit(&callchain_cursor);
  336. while (true) {
  337. struct alloc_func key, *caller;
  338. u64 addr;
  339. node = callchain_cursor_current(&callchain_cursor);
  340. if (node == NULL)
  341. break;
  342. key.start = key.end = node->ip;
  343. caller = bsearch(&key, alloc_func_list, nr_alloc_funcs,
  344. sizeof(key), callcmp);
  345. if (!caller) {
  346. /* found */
  347. if (node->ms.map)
  348. addr = map__unmap_ip(node->ms.map, node->ip);
  349. else
  350. addr = node->ip;
  351. return addr;
  352. } else
  353. pr_debug3("skipping alloc function: %s\n", caller->name);
  354. callchain_cursor_advance(&callchain_cursor);
  355. }
  356. out:
  357. pr_debug2("unknown callsite: %"PRIx64 "\n", sample->ip);
  358. return sample->ip;
  359. }
  360. struct sort_dimension {
  361. const char name[20];
  362. sort_fn_t cmp;
  363. struct list_head list;
  364. };
  365. static LIST_HEAD(page_alloc_sort_input);
  366. static LIST_HEAD(page_caller_sort_input);
  367. static struct page_stat *
  368. __page_stat__findnew_page(struct page_stat *pstat, bool create)
  369. {
  370. struct rb_node **node = &page_live_tree.rb_node;
  371. struct rb_node *parent = NULL;
  372. struct page_stat *data;
  373. while (*node) {
  374. s64 cmp;
  375. parent = *node;
  376. data = rb_entry(*node, struct page_stat, node);
  377. cmp = data->page - pstat->page;
  378. if (cmp < 0)
  379. node = &parent->rb_left;
  380. else if (cmp > 0)
  381. node = &parent->rb_right;
  382. else
  383. return data;
  384. }
  385. if (!create)
  386. return NULL;
  387. data = zalloc(sizeof(*data));
  388. if (data != NULL) {
  389. data->page = pstat->page;
  390. data->order = pstat->order;
  391. data->gfp_flags = pstat->gfp_flags;
  392. data->migrate_type = pstat->migrate_type;
  393. rb_link_node(&data->node, parent, node);
  394. rb_insert_color(&data->node, &page_live_tree);
  395. }
  396. return data;
  397. }
  398. static struct page_stat *page_stat__find_page(struct page_stat *pstat)
  399. {
  400. return __page_stat__findnew_page(pstat, false);
  401. }
  402. static struct page_stat *page_stat__findnew_page(struct page_stat *pstat)
  403. {
  404. return __page_stat__findnew_page(pstat, true);
  405. }
  406. static struct page_stat *
  407. __page_stat__findnew_alloc(struct page_stat *pstat, bool create)
  408. {
  409. struct rb_node **node = &page_alloc_tree.rb_node;
  410. struct rb_node *parent = NULL;
  411. struct page_stat *data;
  412. struct sort_dimension *sort;
  413. while (*node) {
  414. int cmp = 0;
  415. parent = *node;
  416. data = rb_entry(*node, struct page_stat, node);
  417. list_for_each_entry(sort, &page_alloc_sort_input, list) {
  418. cmp = sort->cmp(pstat, data);
  419. if (cmp)
  420. break;
  421. }
  422. if (cmp < 0)
  423. node = &parent->rb_left;
  424. else if (cmp > 0)
  425. node = &parent->rb_right;
  426. else
  427. return data;
  428. }
  429. if (!create)
  430. return NULL;
  431. data = zalloc(sizeof(*data));
  432. if (data != NULL) {
  433. data->page = pstat->page;
  434. data->order = pstat->order;
  435. data->gfp_flags = pstat->gfp_flags;
  436. data->migrate_type = pstat->migrate_type;
  437. rb_link_node(&data->node, parent, node);
  438. rb_insert_color(&data->node, &page_alloc_tree);
  439. }
  440. return data;
  441. }
  442. static struct page_stat *page_stat__find_alloc(struct page_stat *pstat)
  443. {
  444. return __page_stat__findnew_alloc(pstat, false);
  445. }
  446. static struct page_stat *page_stat__findnew_alloc(struct page_stat *pstat)
  447. {
  448. return __page_stat__findnew_alloc(pstat, true);
  449. }
  450. static struct page_stat *
  451. __page_stat__findnew_caller(struct page_stat *pstat, bool create)
  452. {
  453. struct rb_node **node = &page_caller_tree.rb_node;
  454. struct rb_node *parent = NULL;
  455. struct page_stat *data;
  456. struct sort_dimension *sort;
  457. while (*node) {
  458. int cmp = 0;
  459. parent = *node;
  460. data = rb_entry(*node, struct page_stat, node);
  461. list_for_each_entry(sort, &page_caller_sort_input, list) {
  462. cmp = sort->cmp(pstat, data);
  463. if (cmp)
  464. break;
  465. }
  466. if (cmp < 0)
  467. node = &parent->rb_left;
  468. else if (cmp > 0)
  469. node = &parent->rb_right;
  470. else
  471. return data;
  472. }
  473. if (!create)
  474. return NULL;
  475. data = zalloc(sizeof(*data));
  476. if (data != NULL) {
  477. data->callsite = pstat->callsite;
  478. data->order = pstat->order;
  479. data->gfp_flags = pstat->gfp_flags;
  480. data->migrate_type = pstat->migrate_type;
  481. rb_link_node(&data->node, parent, node);
  482. rb_insert_color(&data->node, &page_caller_tree);
  483. }
  484. return data;
  485. }
  486. static struct page_stat *page_stat__find_caller(struct page_stat *pstat)
  487. {
  488. return __page_stat__findnew_caller(pstat, false);
  489. }
  490. static struct page_stat *page_stat__findnew_caller(struct page_stat *pstat)
  491. {
  492. return __page_stat__findnew_caller(pstat, true);
  493. }
  494. static bool valid_page(u64 pfn_or_page)
  495. {
  496. if (use_pfn && pfn_or_page == -1UL)
  497. return false;
  498. if (!use_pfn && pfn_or_page == 0)
  499. return false;
  500. return true;
  501. }
  502. struct gfp_flag {
  503. unsigned int flags;
  504. char *compact_str;
  505. char *human_readable;
  506. };
  507. static struct gfp_flag *gfps;
  508. static int nr_gfps;
  509. static int gfpcmp(const void *a, const void *b)
  510. {
  511. const struct gfp_flag *fa = a;
  512. const struct gfp_flag *fb = b;
  513. return fa->flags - fb->flags;
  514. }
  515. /* see include/trace/events/mmflags.h */
  516. static const struct {
  517. const char *original;
  518. const char *compact;
  519. } gfp_compact_table[] = {
  520. { "GFP_TRANSHUGE", "THP" },
  521. { "GFP_TRANSHUGE_LIGHT", "THL" },
  522. { "GFP_HIGHUSER_MOVABLE", "HUM" },
  523. { "GFP_HIGHUSER", "HU" },
  524. { "GFP_USER", "U" },
  525. { "GFP_KERNEL_ACCOUNT", "KAC" },
  526. { "GFP_KERNEL", "K" },
  527. { "GFP_NOFS", "NF" },
  528. { "GFP_ATOMIC", "A" },
  529. { "GFP_NOIO", "NI" },
  530. { "GFP_NOWAIT", "NW" },
  531. { "GFP_DMA", "D" },
  532. { "__GFP_HIGHMEM", "HM" },
  533. { "GFP_DMA32", "D32" },
  534. { "__GFP_HIGH", "H" },
  535. { "__GFP_ATOMIC", "_A" },
  536. { "__GFP_IO", "I" },
  537. { "__GFP_FS", "F" },
  538. { "__GFP_NOWARN", "NWR" },
  539. { "__GFP_RETRY_MAYFAIL", "R" },
  540. { "__GFP_NOFAIL", "NF" },
  541. { "__GFP_NORETRY", "NR" },
  542. { "__GFP_COMP", "C" },
  543. { "__GFP_ZERO", "Z" },
  544. { "__GFP_NOMEMALLOC", "NMA" },
  545. { "__GFP_MEMALLOC", "MA" },
  546. { "__GFP_HARDWALL", "HW" },
  547. { "__GFP_THISNODE", "TN" },
  548. { "__GFP_RECLAIMABLE", "RC" },
  549. { "__GFP_MOVABLE", "M" },
  550. { "__GFP_ACCOUNT", "AC" },
  551. { "__GFP_WRITE", "WR" },
  552. { "__GFP_RECLAIM", "R" },
  553. { "__GFP_DIRECT_RECLAIM", "DR" },
  554. { "__GFP_KSWAPD_RECLAIM", "KR" },
  555. };
  556. static size_t max_gfp_len;
  557. static char *compact_gfp_flags(char *gfp_flags)
  558. {
  559. char *orig_flags = strdup(gfp_flags);
  560. char *new_flags = NULL;
  561. char *str, *pos = NULL;
  562. size_t len = 0;
  563. if (orig_flags == NULL)
  564. return NULL;
  565. str = strtok_r(orig_flags, "|", &pos);
  566. while (str) {
  567. size_t i;
  568. char *new;
  569. const char *cpt;
  570. for (i = 0; i < ARRAY_SIZE(gfp_compact_table); i++) {
  571. if (strcmp(gfp_compact_table[i].original, str))
  572. continue;
  573. cpt = gfp_compact_table[i].compact;
  574. new = realloc(new_flags, len + strlen(cpt) + 2);
  575. if (new == NULL) {
  576. free(new_flags);
  577. free(orig_flags);
  578. return NULL;
  579. }
  580. new_flags = new;
  581. if (!len) {
  582. strcpy(new_flags, cpt);
  583. } else {
  584. strcat(new_flags, "|");
  585. strcat(new_flags, cpt);
  586. len++;
  587. }
  588. len += strlen(cpt);
  589. }
  590. str = strtok_r(NULL, "|", &pos);
  591. }
  592. if (max_gfp_len < len)
  593. max_gfp_len = len;
  594. free(orig_flags);
  595. return new_flags;
  596. }
  597. static char *compact_gfp_string(unsigned long gfp_flags)
  598. {
  599. struct gfp_flag key = {
  600. .flags = gfp_flags,
  601. };
  602. struct gfp_flag *gfp;
  603. gfp = bsearch(&key, gfps, nr_gfps, sizeof(*gfps), gfpcmp);
  604. if (gfp)
  605. return gfp->compact_str;
  606. return NULL;
  607. }
  608. static int parse_gfp_flags(struct evsel *evsel, struct perf_sample *sample,
  609. unsigned int gfp_flags)
  610. {
  611. struct tep_record record = {
  612. .cpu = sample->cpu,
  613. .data = sample->raw_data,
  614. .size = sample->raw_size,
  615. };
  616. struct trace_seq seq;
  617. char *str, *pos = NULL;
  618. if (nr_gfps) {
  619. struct gfp_flag key = {
  620. .flags = gfp_flags,
  621. };
  622. if (bsearch(&key, gfps, nr_gfps, sizeof(*gfps), gfpcmp))
  623. return 0;
  624. }
  625. trace_seq_init(&seq);
  626. tep_print_event(evsel->tp_format->tep,
  627. &seq, &record, "%s", TEP_PRINT_INFO);
  628. str = strtok_r(seq.buffer, " ", &pos);
  629. while (str) {
  630. if (!strncmp(str, "gfp_flags=", 10)) {
  631. struct gfp_flag *new;
  632. new = realloc(gfps, (nr_gfps + 1) * sizeof(*gfps));
  633. if (new == NULL)
  634. return -ENOMEM;
  635. gfps = new;
  636. new += nr_gfps++;
  637. new->flags = gfp_flags;
  638. new->human_readable = strdup(str + 10);
  639. new->compact_str = compact_gfp_flags(str + 10);
  640. if (!new->human_readable || !new->compact_str)
  641. return -ENOMEM;
  642. qsort(gfps, nr_gfps, sizeof(*gfps), gfpcmp);
  643. }
  644. str = strtok_r(NULL, " ", &pos);
  645. }
  646. trace_seq_destroy(&seq);
  647. return 0;
  648. }
  649. static int evsel__process_page_alloc_event(struct evsel *evsel, struct perf_sample *sample)
  650. {
  651. u64 page;
  652. unsigned int order = evsel__intval(evsel, sample, "order");
  653. unsigned int gfp_flags = evsel__intval(evsel, sample, "gfp_flags");
  654. unsigned int migrate_type = evsel__intval(evsel, sample,
  655. "migratetype");
  656. u64 bytes = kmem_page_size << order;
  657. u64 callsite;
  658. struct page_stat *pstat;
  659. struct page_stat this = {
  660. .order = order,
  661. .gfp_flags = gfp_flags,
  662. .migrate_type = migrate_type,
  663. };
  664. if (use_pfn)
  665. page = evsel__intval(evsel, sample, "pfn");
  666. else
  667. page = evsel__intval(evsel, sample, "page");
  668. nr_page_allocs++;
  669. total_page_alloc_bytes += bytes;
  670. if (!valid_page(page)) {
  671. nr_page_fails++;
  672. total_page_fail_bytes += bytes;
  673. return 0;
  674. }
  675. if (parse_gfp_flags(evsel, sample, gfp_flags) < 0)
  676. return -1;
  677. callsite = find_callsite(evsel, sample);
  678. /*
  679. * This is to find the current page (with correct gfp flags and
  680. * migrate type) at free event.
  681. */
  682. this.page = page;
  683. pstat = page_stat__findnew_page(&this);
  684. if (pstat == NULL)
  685. return -ENOMEM;
  686. pstat->nr_alloc++;
  687. pstat->alloc_bytes += bytes;
  688. pstat->callsite = callsite;
  689. if (!live_page) {
  690. pstat = page_stat__findnew_alloc(&this);
  691. if (pstat == NULL)
  692. return -ENOMEM;
  693. pstat->nr_alloc++;
  694. pstat->alloc_bytes += bytes;
  695. pstat->callsite = callsite;
  696. }
  697. this.callsite = callsite;
  698. pstat = page_stat__findnew_caller(&this);
  699. if (pstat == NULL)
  700. return -ENOMEM;
  701. pstat->nr_alloc++;
  702. pstat->alloc_bytes += bytes;
  703. order_stats[order][migrate_type]++;
  704. return 0;
  705. }
  706. static int evsel__process_page_free_event(struct evsel *evsel, struct perf_sample *sample)
  707. {
  708. u64 page;
  709. unsigned int order = evsel__intval(evsel, sample, "order");
  710. u64 bytes = kmem_page_size << order;
  711. struct page_stat *pstat;
  712. struct page_stat this = {
  713. .order = order,
  714. };
  715. if (use_pfn)
  716. page = evsel__intval(evsel, sample, "pfn");
  717. else
  718. page = evsel__intval(evsel, sample, "page");
  719. nr_page_frees++;
  720. total_page_free_bytes += bytes;
  721. this.page = page;
  722. pstat = page_stat__find_page(&this);
  723. if (pstat == NULL) {
  724. pr_debug2("missing free at page %"PRIx64" (order: %d)\n",
  725. page, order);
  726. nr_page_nomatch++;
  727. total_page_nomatch_bytes += bytes;
  728. return 0;
  729. }
  730. this.gfp_flags = pstat->gfp_flags;
  731. this.migrate_type = pstat->migrate_type;
  732. this.callsite = pstat->callsite;
  733. rb_erase(&pstat->node, &page_live_tree);
  734. free(pstat);
  735. if (live_page) {
  736. order_stats[this.order][this.migrate_type]--;
  737. } else {
  738. pstat = page_stat__find_alloc(&this);
  739. if (pstat == NULL)
  740. return -ENOMEM;
  741. pstat->nr_free++;
  742. pstat->free_bytes += bytes;
  743. }
  744. pstat = page_stat__find_caller(&this);
  745. if (pstat == NULL)
  746. return -ENOENT;
  747. pstat->nr_free++;
  748. pstat->free_bytes += bytes;
  749. if (live_page) {
  750. pstat->nr_alloc--;
  751. pstat->alloc_bytes -= bytes;
  752. if (pstat->nr_alloc == 0) {
  753. rb_erase(&pstat->node, &page_caller_tree);
  754. free(pstat);
  755. }
  756. }
  757. return 0;
  758. }
  759. static bool perf_kmem__skip_sample(struct perf_sample *sample)
  760. {
  761. /* skip sample based on time? */
  762. if (perf_time__skip_sample(&ptime, sample->time))
  763. return true;
  764. return false;
  765. }
  766. typedef int (*tracepoint_handler)(struct evsel *evsel,
  767. struct perf_sample *sample);
  768. static int process_sample_event(struct perf_tool *tool __maybe_unused,
  769. union perf_event *event,
  770. struct perf_sample *sample,
  771. struct evsel *evsel,
  772. struct machine *machine)
  773. {
  774. int err = 0;
  775. struct thread *thread = machine__findnew_thread(machine, sample->pid,
  776. sample->tid);
  777. if (thread == NULL) {
  778. pr_debug("problem processing %d event, skipping it.\n",
  779. event->header.type);
  780. return -1;
  781. }
  782. if (perf_kmem__skip_sample(sample))
  783. return 0;
  784. dump_printf(" ... thread: %s:%d\n", thread__comm_str(thread), thread->tid);
  785. if (evsel->handler != NULL) {
  786. tracepoint_handler f = evsel->handler;
  787. err = f(evsel, sample);
  788. }
  789. thread__put(thread);
  790. return err;
  791. }
  792. static struct perf_tool perf_kmem = {
  793. .sample = process_sample_event,
  794. .comm = perf_event__process_comm,
  795. .mmap = perf_event__process_mmap,
  796. .mmap2 = perf_event__process_mmap2,
  797. .namespaces = perf_event__process_namespaces,
  798. .ordered_events = true,
  799. };
  800. static double fragmentation(unsigned long n_req, unsigned long n_alloc)
  801. {
  802. if (n_alloc == 0)
  803. return 0.0;
  804. else
  805. return 100.0 - (100.0 * n_req / n_alloc);
  806. }
  807. static void __print_slab_result(struct rb_root *root,
  808. struct perf_session *session,
  809. int n_lines, int is_caller)
  810. {
  811. struct rb_node *next;
  812. struct machine *machine = &session->machines.host;
  813. printf("%.105s\n", graph_dotted_line);
  814. printf(" %-34s |", is_caller ? "Callsite": "Alloc Ptr");
  815. printf(" Total_alloc/Per | Total_req/Per | Hit | Ping-pong | Frag\n");
  816. printf("%.105s\n", graph_dotted_line);
  817. next = rb_first(root);
  818. while (next && n_lines--) {
  819. struct alloc_stat *data = rb_entry(next, struct alloc_stat,
  820. node);
  821. struct symbol *sym = NULL;
  822. struct map *map;
  823. char buf[BUFSIZ];
  824. u64 addr;
  825. if (is_caller) {
  826. addr = data->call_site;
  827. if (!raw_ip)
  828. sym = machine__find_kernel_symbol(machine, addr, &map);
  829. } else
  830. addr = data->ptr;
  831. if (sym != NULL)
  832. snprintf(buf, sizeof(buf), "%s+%" PRIx64 "", sym->name,
  833. addr - map->unmap_ip(map, sym->start));
  834. else
  835. snprintf(buf, sizeof(buf), "%#" PRIx64 "", addr);
  836. printf(" %-34s |", buf);
  837. printf(" %9llu/%-5lu | %9llu/%-5lu | %8lu | %9lu | %6.3f%%\n",
  838. (unsigned long long)data->bytes_alloc,
  839. (unsigned long)data->bytes_alloc / data->hit,
  840. (unsigned long long)data->bytes_req,
  841. (unsigned long)data->bytes_req / data->hit,
  842. (unsigned long)data->hit,
  843. (unsigned long)data->pingpong,
  844. fragmentation(data->bytes_req, data->bytes_alloc));
  845. next = rb_next(next);
  846. }
  847. if (n_lines == -1)
  848. printf(" ... | ... | ... | ... | ... | ... \n");
  849. printf("%.105s\n", graph_dotted_line);
  850. }
  851. static const char * const migrate_type_str[] = {
  852. "UNMOVABL",
  853. "RECLAIM",
  854. "MOVABLE",
  855. "RESERVED",
  856. "CMA/ISLT",
  857. "UNKNOWN",
  858. };
  859. static void __print_page_alloc_result(struct perf_session *session, int n_lines)
  860. {
  861. struct rb_node *next = rb_first(&page_alloc_sorted);
  862. struct machine *machine = &session->machines.host;
  863. const char *format;
  864. int gfp_len = max(strlen("GFP flags"), max_gfp_len);
  865. printf("\n%.105s\n", graph_dotted_line);
  866. printf(" %-16s | %5s alloc (KB) | Hits | Order | Mig.type | %-*s | Callsite\n",
  867. use_pfn ? "PFN" : "Page", live_page ? "Live" : "Total",
  868. gfp_len, "GFP flags");
  869. printf("%.105s\n", graph_dotted_line);
  870. if (use_pfn)
  871. format = " %16llu | %'16llu | %'9d | %5d | %8s | %-*s | %s\n";
  872. else
  873. format = " %016llx | %'16llu | %'9d | %5d | %8s | %-*s | %s\n";
  874. while (next && n_lines--) {
  875. struct page_stat *data;
  876. struct symbol *sym;
  877. struct map *map;
  878. char buf[32];
  879. char *caller = buf;
  880. data = rb_entry(next, struct page_stat, node);
  881. sym = machine__find_kernel_symbol(machine, data->callsite, &map);
  882. if (sym)
  883. caller = sym->name;
  884. else
  885. scnprintf(buf, sizeof(buf), "%"PRIx64, data->callsite);
  886. printf(format, (unsigned long long)data->page,
  887. (unsigned long long)data->alloc_bytes / 1024,
  888. data->nr_alloc, data->order,
  889. migrate_type_str[data->migrate_type],
  890. gfp_len, compact_gfp_string(data->gfp_flags), caller);
  891. next = rb_next(next);
  892. }
  893. if (n_lines == -1) {
  894. printf(" ... | ... | ... | ... | ... | %-*s | ...\n",
  895. gfp_len, "...");
  896. }
  897. printf("%.105s\n", graph_dotted_line);
  898. }
  899. static void __print_page_caller_result(struct perf_session *session, int n_lines)
  900. {
  901. struct rb_node *next = rb_first(&page_caller_sorted);
  902. struct machine *machine = &session->machines.host;
  903. int gfp_len = max(strlen("GFP flags"), max_gfp_len);
  904. printf("\n%.105s\n", graph_dotted_line);
  905. printf(" %5s alloc (KB) | Hits | Order | Mig.type | %-*s | Callsite\n",
  906. live_page ? "Live" : "Total", gfp_len, "GFP flags");
  907. printf("%.105s\n", graph_dotted_line);
  908. while (next && n_lines--) {
  909. struct page_stat *data;
  910. struct symbol *sym;
  911. struct map *map;
  912. char buf[32];
  913. char *caller = buf;
  914. data = rb_entry(next, struct page_stat, node);
  915. sym = machine__find_kernel_symbol(machine, data->callsite, &map);
  916. if (sym)
  917. caller = sym->name;
  918. else
  919. scnprintf(buf, sizeof(buf), "%"PRIx64, data->callsite);
  920. printf(" %'16llu | %'9d | %5d | %8s | %-*s | %s\n",
  921. (unsigned long long)data->alloc_bytes / 1024,
  922. data->nr_alloc, data->order,
  923. migrate_type_str[data->migrate_type],
  924. gfp_len, compact_gfp_string(data->gfp_flags), caller);
  925. next = rb_next(next);
  926. }
  927. if (n_lines == -1) {
  928. printf(" ... | ... | ... | ... | %-*s | ...\n",
  929. gfp_len, "...");
  930. }
  931. printf("%.105s\n", graph_dotted_line);
  932. }
  933. static void print_gfp_flags(void)
  934. {
  935. int i;
  936. printf("#\n");
  937. printf("# GFP flags\n");
  938. printf("# ---------\n");
  939. for (i = 0; i < nr_gfps; i++) {
  940. printf("# %08x: %*s: %s\n", gfps[i].flags,
  941. (int) max_gfp_len, gfps[i].compact_str,
  942. gfps[i].human_readable);
  943. }
  944. }
  945. static void print_slab_summary(void)
  946. {
  947. printf("\nSUMMARY (SLAB allocator)");
  948. printf("\n========================\n");
  949. printf("Total bytes requested: %'lu\n", total_requested);
  950. printf("Total bytes allocated: %'lu\n", total_allocated);
  951. printf("Total bytes freed: %'lu\n", total_freed);
  952. if (total_allocated > total_freed) {
  953. printf("Net total bytes allocated: %'lu\n",
  954. total_allocated - total_freed);
  955. }
  956. printf("Total bytes wasted on internal fragmentation: %'lu\n",
  957. total_allocated - total_requested);
  958. printf("Internal fragmentation: %f%%\n",
  959. fragmentation(total_requested, total_allocated));
  960. printf("Cross CPU allocations: %'lu/%'lu\n", nr_cross_allocs, nr_allocs);
  961. }
  962. static void print_page_summary(void)
  963. {
  964. int o, m;
  965. u64 nr_alloc_freed = nr_page_frees - nr_page_nomatch;
  966. u64 total_alloc_freed_bytes = total_page_free_bytes - total_page_nomatch_bytes;
  967. printf("\nSUMMARY (page allocator)");
  968. printf("\n========================\n");
  969. printf("%-30s: %'16lu [ %'16"PRIu64" KB ]\n", "Total allocation requests",
  970. nr_page_allocs, total_page_alloc_bytes / 1024);
  971. printf("%-30s: %'16lu [ %'16"PRIu64" KB ]\n", "Total free requests",
  972. nr_page_frees, total_page_free_bytes / 1024);
  973. printf("\n");
  974. printf("%-30s: %'16"PRIu64" [ %'16"PRIu64" KB ]\n", "Total alloc+freed requests",
  975. nr_alloc_freed, (total_alloc_freed_bytes) / 1024);
  976. printf("%-30s: %'16"PRIu64" [ %'16"PRIu64" KB ]\n", "Total alloc-only requests",
  977. nr_page_allocs - nr_alloc_freed,
  978. (total_page_alloc_bytes - total_alloc_freed_bytes) / 1024);
  979. printf("%-30s: %'16lu [ %'16"PRIu64" KB ]\n", "Total free-only requests",
  980. nr_page_nomatch, total_page_nomatch_bytes / 1024);
  981. printf("\n");
  982. printf("%-30s: %'16lu [ %'16"PRIu64" KB ]\n", "Total allocation failures",
  983. nr_page_fails, total_page_fail_bytes / 1024);
  984. printf("\n");
  985. printf("%5s %12s %12s %12s %12s %12s\n", "Order", "Unmovable",
  986. "Reclaimable", "Movable", "Reserved", "CMA/Isolated");
  987. printf("%.5s %.12s %.12s %.12s %.12s %.12s\n", graph_dotted_line,
  988. graph_dotted_line, graph_dotted_line, graph_dotted_line,
  989. graph_dotted_line, graph_dotted_line);
  990. for (o = 0; o < MAX_PAGE_ORDER; o++) {
  991. printf("%5d", o);
  992. for (m = 0; m < MAX_MIGRATE_TYPES - 1; m++) {
  993. if (order_stats[o][m])
  994. printf(" %'12d", order_stats[o][m]);
  995. else
  996. printf(" %12c", '.');
  997. }
  998. printf("\n");
  999. }
  1000. }
  1001. static void print_slab_result(struct perf_session *session)
  1002. {
  1003. if (caller_flag)
  1004. __print_slab_result(&root_caller_sorted, session, caller_lines, 1);
  1005. if (alloc_flag)
  1006. __print_slab_result(&root_alloc_sorted, session, alloc_lines, 0);
  1007. print_slab_summary();
  1008. }
  1009. static void print_page_result(struct perf_session *session)
  1010. {
  1011. if (caller_flag || alloc_flag)
  1012. print_gfp_flags();
  1013. if (caller_flag)
  1014. __print_page_caller_result(session, caller_lines);
  1015. if (alloc_flag)
  1016. __print_page_alloc_result(session, alloc_lines);
  1017. print_page_summary();
  1018. }
  1019. static void print_result(struct perf_session *session)
  1020. {
  1021. if (kmem_slab)
  1022. print_slab_result(session);
  1023. if (kmem_page)
  1024. print_page_result(session);
  1025. }
  1026. static LIST_HEAD(slab_caller_sort);
  1027. static LIST_HEAD(slab_alloc_sort);
  1028. static LIST_HEAD(page_caller_sort);
  1029. static LIST_HEAD(page_alloc_sort);
  1030. static void sort_slab_insert(struct rb_root *root, struct alloc_stat *data,
  1031. struct list_head *sort_list)
  1032. {
  1033. struct rb_node **new = &(root->rb_node);
  1034. struct rb_node *parent = NULL;
  1035. struct sort_dimension *sort;
  1036. while (*new) {
  1037. struct alloc_stat *this;
  1038. int cmp = 0;
  1039. this = rb_entry(*new, struct alloc_stat, node);
  1040. parent = *new;
  1041. list_for_each_entry(sort, sort_list, list) {
  1042. cmp = sort->cmp(data, this);
  1043. if (cmp)
  1044. break;
  1045. }
  1046. if (cmp > 0)
  1047. new = &((*new)->rb_left);
  1048. else
  1049. new = &((*new)->rb_right);
  1050. }
  1051. rb_link_node(&data->node, parent, new);
  1052. rb_insert_color(&data->node, root);
  1053. }
  1054. static void __sort_slab_result(struct rb_root *root, struct rb_root *root_sorted,
  1055. struct list_head *sort_list)
  1056. {
  1057. struct rb_node *node;
  1058. struct alloc_stat *data;
  1059. for (;;) {
  1060. node = rb_first(root);
  1061. if (!node)
  1062. break;
  1063. rb_erase(node, root);
  1064. data = rb_entry(node, struct alloc_stat, node);
  1065. sort_slab_insert(root_sorted, data, sort_list);
  1066. }
  1067. }
  1068. static void sort_page_insert(struct rb_root *root, struct page_stat *data,
  1069. struct list_head *sort_list)
  1070. {
  1071. struct rb_node **new = &root->rb_node;
  1072. struct rb_node *parent = NULL;
  1073. struct sort_dimension *sort;
  1074. while (*new) {
  1075. struct page_stat *this;
  1076. int cmp = 0;
  1077. this = rb_entry(*new, struct page_stat, node);
  1078. parent = *new;
  1079. list_for_each_entry(sort, sort_list, list) {
  1080. cmp = sort->cmp(data, this);
  1081. if (cmp)
  1082. break;
  1083. }
  1084. if (cmp > 0)
  1085. new = &parent->rb_left;
  1086. else
  1087. new = &parent->rb_right;
  1088. }
  1089. rb_link_node(&data->node, parent, new);
  1090. rb_insert_color(&data->node, root);
  1091. }
  1092. static void __sort_page_result(struct rb_root *root, struct rb_root *root_sorted,
  1093. struct list_head *sort_list)
  1094. {
  1095. struct rb_node *node;
  1096. struct page_stat *data;
  1097. for (;;) {
  1098. node = rb_first(root);
  1099. if (!node)
  1100. break;
  1101. rb_erase(node, root);
  1102. data = rb_entry(node, struct page_stat, node);
  1103. sort_page_insert(root_sorted, data, sort_list);
  1104. }
  1105. }
  1106. static void sort_result(void)
  1107. {
  1108. if (kmem_slab) {
  1109. __sort_slab_result(&root_alloc_stat, &root_alloc_sorted,
  1110. &slab_alloc_sort);
  1111. __sort_slab_result(&root_caller_stat, &root_caller_sorted,
  1112. &slab_caller_sort);
  1113. }
  1114. if (kmem_page) {
  1115. if (live_page)
  1116. __sort_page_result(&page_live_tree, &page_alloc_sorted,
  1117. &page_alloc_sort);
  1118. else
  1119. __sort_page_result(&page_alloc_tree, &page_alloc_sorted,
  1120. &page_alloc_sort);
  1121. __sort_page_result(&page_caller_tree, &page_caller_sorted,
  1122. &page_caller_sort);
  1123. }
  1124. }
  1125. static int __cmd_kmem(struct perf_session *session)
  1126. {
  1127. int err = -EINVAL;
  1128. struct evsel *evsel;
  1129. const struct evsel_str_handler kmem_tracepoints[] = {
  1130. /* slab allocator */
  1131. { "kmem:kmalloc", evsel__process_alloc_event, },
  1132. { "kmem:kmem_cache_alloc", evsel__process_alloc_event, },
  1133. { "kmem:kmalloc_node", evsel__process_alloc_event, },
  1134. { "kmem:kmem_cache_alloc_node", evsel__process_alloc_event, },
  1135. { "kmem:kfree", evsel__process_free_event, },
  1136. { "kmem:kmem_cache_free", evsel__process_free_event, },
  1137. /* page allocator */
  1138. { "kmem:mm_page_alloc", evsel__process_page_alloc_event, },
  1139. { "kmem:mm_page_free", evsel__process_page_free_event, },
  1140. };
  1141. if (!perf_session__has_traces(session, "kmem record"))
  1142. goto out;
  1143. if (perf_session__set_tracepoints_handlers(session, kmem_tracepoints)) {
  1144. pr_err("Initializing perf session tracepoint handlers failed\n");
  1145. goto out;
  1146. }
  1147. evlist__for_each_entry(session->evlist, evsel) {
  1148. if (!strcmp(evsel__name(evsel), "kmem:mm_page_alloc") &&
  1149. evsel__field(evsel, "pfn")) {
  1150. use_pfn = true;
  1151. break;
  1152. }
  1153. }
  1154. setup_pager();
  1155. err = perf_session__process_events(session);
  1156. if (err != 0) {
  1157. pr_err("error during process events: %d\n", err);
  1158. goto out;
  1159. }
  1160. sort_result();
  1161. print_result(session);
  1162. out:
  1163. return err;
  1164. }
  1165. /* slab sort keys */
  1166. static int ptr_cmp(void *a, void *b)
  1167. {
  1168. struct alloc_stat *l = a;
  1169. struct alloc_stat *r = b;
  1170. if (l->ptr < r->ptr)
  1171. return -1;
  1172. else if (l->ptr > r->ptr)
  1173. return 1;
  1174. return 0;
  1175. }
  1176. static struct sort_dimension ptr_sort_dimension = {
  1177. .name = "ptr",
  1178. .cmp = ptr_cmp,
  1179. };
  1180. static int slab_callsite_cmp(void *a, void *b)
  1181. {
  1182. struct alloc_stat *l = a;
  1183. struct alloc_stat *r = b;
  1184. if (l->call_site < r->call_site)
  1185. return -1;
  1186. else if (l->call_site > r->call_site)
  1187. return 1;
  1188. return 0;
  1189. }
  1190. static struct sort_dimension callsite_sort_dimension = {
  1191. .name = "callsite",
  1192. .cmp = slab_callsite_cmp,
  1193. };
  1194. static int hit_cmp(void *a, void *b)
  1195. {
  1196. struct alloc_stat *l = a;
  1197. struct alloc_stat *r = b;
  1198. if (l->hit < r->hit)
  1199. return -1;
  1200. else if (l->hit > r->hit)
  1201. return 1;
  1202. return 0;
  1203. }
  1204. static struct sort_dimension hit_sort_dimension = {
  1205. .name = "hit",
  1206. .cmp = hit_cmp,
  1207. };
  1208. static int bytes_cmp(void *a, void *b)
  1209. {
  1210. struct alloc_stat *l = a;
  1211. struct alloc_stat *r = b;
  1212. if (l->bytes_alloc < r->bytes_alloc)
  1213. return -1;
  1214. else if (l->bytes_alloc > r->bytes_alloc)
  1215. return 1;
  1216. return 0;
  1217. }
  1218. static struct sort_dimension bytes_sort_dimension = {
  1219. .name = "bytes",
  1220. .cmp = bytes_cmp,
  1221. };
  1222. static int frag_cmp(void *a, void *b)
  1223. {
  1224. double x, y;
  1225. struct alloc_stat *l = a;
  1226. struct alloc_stat *r = b;
  1227. x = fragmentation(l->bytes_req, l->bytes_alloc);
  1228. y = fragmentation(r->bytes_req, r->bytes_alloc);
  1229. if (x < y)
  1230. return -1;
  1231. else if (x > y)
  1232. return 1;
  1233. return 0;
  1234. }
  1235. static struct sort_dimension frag_sort_dimension = {
  1236. .name = "frag",
  1237. .cmp = frag_cmp,
  1238. };
  1239. static int pingpong_cmp(void *a, void *b)
  1240. {
  1241. struct alloc_stat *l = a;
  1242. struct alloc_stat *r = b;
  1243. if (l->pingpong < r->pingpong)
  1244. return -1;
  1245. else if (l->pingpong > r->pingpong)
  1246. return 1;
  1247. return 0;
  1248. }
  1249. static struct sort_dimension pingpong_sort_dimension = {
  1250. .name = "pingpong",
  1251. .cmp = pingpong_cmp,
  1252. };
  1253. /* page sort keys */
  1254. static int page_cmp(void *a, void *b)
  1255. {
  1256. struct page_stat *l = a;
  1257. struct page_stat *r = b;
  1258. if (l->page < r->page)
  1259. return -1;
  1260. else if (l->page > r->page)
  1261. return 1;
  1262. return 0;
  1263. }
  1264. static struct sort_dimension page_sort_dimension = {
  1265. .name = "page",
  1266. .cmp = page_cmp,
  1267. };
  1268. static int page_callsite_cmp(void *a, void *b)
  1269. {
  1270. struct page_stat *l = a;
  1271. struct page_stat *r = b;
  1272. if (l->callsite < r->callsite)
  1273. return -1;
  1274. else if (l->callsite > r->callsite)
  1275. return 1;
  1276. return 0;
  1277. }
  1278. static struct sort_dimension page_callsite_sort_dimension = {
  1279. .name = "callsite",
  1280. .cmp = page_callsite_cmp,
  1281. };
  1282. static int page_hit_cmp(void *a, void *b)
  1283. {
  1284. struct page_stat *l = a;
  1285. struct page_stat *r = b;
  1286. if (l->nr_alloc < r->nr_alloc)
  1287. return -1;
  1288. else if (l->nr_alloc > r->nr_alloc)
  1289. return 1;
  1290. return 0;
  1291. }
  1292. static struct sort_dimension page_hit_sort_dimension = {
  1293. .name = "hit",
  1294. .cmp = page_hit_cmp,
  1295. };
  1296. static int page_bytes_cmp(void *a, void *b)
  1297. {
  1298. struct page_stat *l = a;
  1299. struct page_stat *r = b;
  1300. if (l->alloc_bytes < r->alloc_bytes)
  1301. return -1;
  1302. else if (l->alloc_bytes > r->alloc_bytes)
  1303. return 1;
  1304. return 0;
  1305. }
  1306. static struct sort_dimension page_bytes_sort_dimension = {
  1307. .name = "bytes",
  1308. .cmp = page_bytes_cmp,
  1309. };
  1310. static int page_order_cmp(void *a, void *b)
  1311. {
  1312. struct page_stat *l = a;
  1313. struct page_stat *r = b;
  1314. if (l->order < r->order)
  1315. return -1;
  1316. else if (l->order > r->order)
  1317. return 1;
  1318. return 0;
  1319. }
  1320. static struct sort_dimension page_order_sort_dimension = {
  1321. .name = "order",
  1322. .cmp = page_order_cmp,
  1323. };
  1324. static int migrate_type_cmp(void *a, void *b)
  1325. {
  1326. struct page_stat *l = a;
  1327. struct page_stat *r = b;
  1328. /* for internal use to find free'd page */
  1329. if (l->migrate_type == -1U)
  1330. return 0;
  1331. if (l->migrate_type < r->migrate_type)
  1332. return -1;
  1333. else if (l->migrate_type > r->migrate_type)
  1334. return 1;
  1335. return 0;
  1336. }
  1337. static struct sort_dimension migrate_type_sort_dimension = {
  1338. .name = "migtype",
  1339. .cmp = migrate_type_cmp,
  1340. };
  1341. static int gfp_flags_cmp(void *a, void *b)
  1342. {
  1343. struct page_stat *l = a;
  1344. struct page_stat *r = b;
  1345. /* for internal use to find free'd page */
  1346. if (l->gfp_flags == -1U)
  1347. return 0;
  1348. if (l->gfp_flags < r->gfp_flags)
  1349. return -1;
  1350. else if (l->gfp_flags > r->gfp_flags)
  1351. return 1;
  1352. return 0;
  1353. }
  1354. static struct sort_dimension gfp_flags_sort_dimension = {
  1355. .name = "gfp",
  1356. .cmp = gfp_flags_cmp,
  1357. };
  1358. static struct sort_dimension *slab_sorts[] = {
  1359. &ptr_sort_dimension,
  1360. &callsite_sort_dimension,
  1361. &hit_sort_dimension,
  1362. &bytes_sort_dimension,
  1363. &frag_sort_dimension,
  1364. &pingpong_sort_dimension,
  1365. };
  1366. static struct sort_dimension *page_sorts[] = {
  1367. &page_sort_dimension,
  1368. &page_callsite_sort_dimension,
  1369. &page_hit_sort_dimension,
  1370. &page_bytes_sort_dimension,
  1371. &page_order_sort_dimension,
  1372. &migrate_type_sort_dimension,
  1373. &gfp_flags_sort_dimension,
  1374. };
  1375. static int slab_sort_dimension__add(const char *tok, struct list_head *list)
  1376. {
  1377. struct sort_dimension *sort;
  1378. int i;
  1379. for (i = 0; i < (int)ARRAY_SIZE(slab_sorts); i++) {
  1380. if (!strcmp(slab_sorts[i]->name, tok)) {
  1381. sort = memdup(slab_sorts[i], sizeof(*slab_sorts[i]));
  1382. if (!sort) {
  1383. pr_err("%s: memdup failed\n", __func__);
  1384. return -1;
  1385. }
  1386. list_add_tail(&sort->list, list);
  1387. return 0;
  1388. }
  1389. }
  1390. return -1;
  1391. }
  1392. static int page_sort_dimension__add(const char *tok, struct list_head *list)
  1393. {
  1394. struct sort_dimension *sort;
  1395. int i;
  1396. for (i = 0; i < (int)ARRAY_SIZE(page_sorts); i++) {
  1397. if (!strcmp(page_sorts[i]->name, tok)) {
  1398. sort = memdup(page_sorts[i], sizeof(*page_sorts[i]));
  1399. if (!sort) {
  1400. pr_err("%s: memdup failed\n", __func__);
  1401. return -1;
  1402. }
  1403. list_add_tail(&sort->list, list);
  1404. return 0;
  1405. }
  1406. }
  1407. return -1;
  1408. }
  1409. static int setup_slab_sorting(struct list_head *sort_list, const char *arg)
  1410. {
  1411. char *tok;
  1412. char *str = strdup(arg);
  1413. char *pos = str;
  1414. if (!str) {
  1415. pr_err("%s: strdup failed\n", __func__);
  1416. return -1;
  1417. }
  1418. while (true) {
  1419. tok = strsep(&pos, ",");
  1420. if (!tok)
  1421. break;
  1422. if (slab_sort_dimension__add(tok, sort_list) < 0) {
  1423. pr_err("Unknown slab --sort key: '%s'", tok);
  1424. free(str);
  1425. return -1;
  1426. }
  1427. }
  1428. free(str);
  1429. return 0;
  1430. }
  1431. static int setup_page_sorting(struct list_head *sort_list, const char *arg)
  1432. {
  1433. char *tok;
  1434. char *str = strdup(arg);
  1435. char *pos = str;
  1436. if (!str) {
  1437. pr_err("%s: strdup failed\n", __func__);
  1438. return -1;
  1439. }
  1440. while (true) {
  1441. tok = strsep(&pos, ",");
  1442. if (!tok)
  1443. break;
  1444. if (page_sort_dimension__add(tok, sort_list) < 0) {
  1445. pr_err("Unknown page --sort key: '%s'", tok);
  1446. free(str);
  1447. return -1;
  1448. }
  1449. }
  1450. free(str);
  1451. return 0;
  1452. }
  1453. static int parse_sort_opt(const struct option *opt __maybe_unused,
  1454. const char *arg, int unset __maybe_unused)
  1455. {
  1456. if (!arg)
  1457. return -1;
  1458. if (kmem_page > kmem_slab ||
  1459. (kmem_page == 0 && kmem_slab == 0 && kmem_default == KMEM_PAGE)) {
  1460. if (caller_flag > alloc_flag)
  1461. return setup_page_sorting(&page_caller_sort, arg);
  1462. else
  1463. return setup_page_sorting(&page_alloc_sort, arg);
  1464. } else {
  1465. if (caller_flag > alloc_flag)
  1466. return setup_slab_sorting(&slab_caller_sort, arg);
  1467. else
  1468. return setup_slab_sorting(&slab_alloc_sort, arg);
  1469. }
  1470. return 0;
  1471. }
  1472. static int parse_caller_opt(const struct option *opt __maybe_unused,
  1473. const char *arg __maybe_unused,
  1474. int unset __maybe_unused)
  1475. {
  1476. caller_flag = (alloc_flag + 1);
  1477. return 0;
  1478. }
  1479. static int parse_alloc_opt(const struct option *opt __maybe_unused,
  1480. const char *arg __maybe_unused,
  1481. int unset __maybe_unused)
  1482. {
  1483. alloc_flag = (caller_flag + 1);
  1484. return 0;
  1485. }
  1486. static int parse_slab_opt(const struct option *opt __maybe_unused,
  1487. const char *arg __maybe_unused,
  1488. int unset __maybe_unused)
  1489. {
  1490. kmem_slab = (kmem_page + 1);
  1491. return 0;
  1492. }
  1493. static int parse_page_opt(const struct option *opt __maybe_unused,
  1494. const char *arg __maybe_unused,
  1495. int unset __maybe_unused)
  1496. {
  1497. kmem_page = (kmem_slab + 1);
  1498. return 0;
  1499. }
  1500. static int parse_line_opt(const struct option *opt __maybe_unused,
  1501. const char *arg, int unset __maybe_unused)
  1502. {
  1503. int lines;
  1504. if (!arg)
  1505. return -1;
  1506. lines = strtoul(arg, NULL, 10);
  1507. if (caller_flag > alloc_flag)
  1508. caller_lines = lines;
  1509. else
  1510. alloc_lines = lines;
  1511. return 0;
  1512. }
  1513. static bool slab_legacy_tp_is_exposed(void)
  1514. {
  1515. /*
  1516. * The tracepoints "kmem:kmalloc_node" and
  1517. * "kmem:kmem_cache_alloc_node" have been removed on the latest
  1518. * kernel, if the tracepoint "kmem:kmalloc_node" is existed it
  1519. * means the tool is running on an old kernel, we need to
  1520. * rollback to support these legacy tracepoints.
  1521. */
  1522. return IS_ERR(trace_event__tp_format("kmem", "kmalloc_node")) ?
  1523. false : true;
  1524. }
  1525. static int __cmd_record(int argc, const char **argv)
  1526. {
  1527. const char * const record_args[] = {
  1528. "record", "-a", "-R", "-c", "1",
  1529. };
  1530. const char * const slab_events[] = {
  1531. "-e", "kmem:kmalloc",
  1532. "-e", "kmem:kfree",
  1533. "-e", "kmem:kmem_cache_alloc",
  1534. "-e", "kmem:kmem_cache_free",
  1535. };
  1536. const char * const slab_legacy_events[] = {
  1537. "-e", "kmem:kmalloc_node",
  1538. "-e", "kmem:kmem_cache_alloc_node",
  1539. };
  1540. const char * const page_events[] = {
  1541. "-e", "kmem:mm_page_alloc",
  1542. "-e", "kmem:mm_page_free",
  1543. };
  1544. unsigned int rec_argc, i, j;
  1545. const char **rec_argv;
  1546. unsigned int slab_legacy_tp_exposed = slab_legacy_tp_is_exposed();
  1547. rec_argc = ARRAY_SIZE(record_args) + argc - 1;
  1548. if (kmem_slab) {
  1549. rec_argc += ARRAY_SIZE(slab_events);
  1550. if (slab_legacy_tp_exposed)
  1551. rec_argc += ARRAY_SIZE(slab_legacy_events);
  1552. }
  1553. if (kmem_page)
  1554. rec_argc += ARRAY_SIZE(page_events) + 1; /* for -g */
  1555. rec_argv = calloc(rec_argc + 1, sizeof(char *));
  1556. if (rec_argv == NULL)
  1557. return -ENOMEM;
  1558. for (i = 0; i < ARRAY_SIZE(record_args); i++)
  1559. rec_argv[i] = strdup(record_args[i]);
  1560. if (kmem_slab) {
  1561. for (j = 0; j < ARRAY_SIZE(slab_events); j++, i++)
  1562. rec_argv[i] = strdup(slab_events[j]);
  1563. if (slab_legacy_tp_exposed) {
  1564. for (j = 0; j < ARRAY_SIZE(slab_legacy_events); j++, i++)
  1565. rec_argv[i] = strdup(slab_legacy_events[j]);
  1566. }
  1567. }
  1568. if (kmem_page) {
  1569. rec_argv[i++] = strdup("-g");
  1570. for (j = 0; j < ARRAY_SIZE(page_events); j++, i++)
  1571. rec_argv[i] = strdup(page_events[j]);
  1572. }
  1573. for (j = 1; j < (unsigned int)argc; j++, i++)
  1574. rec_argv[i] = argv[j];
  1575. return cmd_record(i, rec_argv);
  1576. }
  1577. static int kmem_config(const char *var, const char *value, void *cb __maybe_unused)
  1578. {
  1579. if (!strcmp(var, "kmem.default")) {
  1580. if (!strcmp(value, "slab"))
  1581. kmem_default = KMEM_SLAB;
  1582. else if (!strcmp(value, "page"))
  1583. kmem_default = KMEM_PAGE;
  1584. else
  1585. pr_err("invalid default value ('slab' or 'page' required): %s\n",
  1586. value);
  1587. return 0;
  1588. }
  1589. return 0;
  1590. }
  1591. int cmd_kmem(int argc, const char **argv)
  1592. {
  1593. const char * const default_slab_sort = "frag,hit,bytes";
  1594. const char * const default_page_sort = "bytes,hit";
  1595. struct perf_data data = {
  1596. .mode = PERF_DATA_MODE_READ,
  1597. };
  1598. const struct option kmem_options[] = {
  1599. OPT_STRING('i', "input", &input_name, "file", "input file name"),
  1600. OPT_INCR('v', "verbose", &verbose,
  1601. "be more verbose (show symbol address, etc)"),
  1602. OPT_CALLBACK_NOOPT(0, "caller", NULL, NULL,
  1603. "show per-callsite statistics", parse_caller_opt),
  1604. OPT_CALLBACK_NOOPT(0, "alloc", NULL, NULL,
  1605. "show per-allocation statistics", parse_alloc_opt),
  1606. OPT_CALLBACK('s', "sort", NULL, "key[,key2...]",
  1607. "sort by keys: ptr, callsite, bytes, hit, pingpong, frag, "
  1608. "page, order, migtype, gfp", parse_sort_opt),
  1609. OPT_CALLBACK('l', "line", NULL, "num", "show n lines", parse_line_opt),
  1610. OPT_BOOLEAN(0, "raw-ip", &raw_ip, "show raw ip instead of symbol"),
  1611. OPT_BOOLEAN('f', "force", &data.force, "don't complain, do it"),
  1612. OPT_CALLBACK_NOOPT(0, "slab", NULL, NULL, "Analyze slab allocator",
  1613. parse_slab_opt),
  1614. OPT_CALLBACK_NOOPT(0, "page", NULL, NULL, "Analyze page allocator",
  1615. parse_page_opt),
  1616. OPT_BOOLEAN(0, "live", &live_page, "Show live page stat"),
  1617. OPT_STRING(0, "time", &time_str, "str",
  1618. "Time span of interest (start,stop)"),
  1619. OPT_END()
  1620. };
  1621. const char *const kmem_subcommands[] = { "record", "stat", NULL };
  1622. const char *kmem_usage[] = {
  1623. NULL,
  1624. NULL
  1625. };
  1626. struct perf_session *session;
  1627. static const char errmsg[] = "No %s allocation events found. Have you run 'perf kmem record --%s'?\n";
  1628. int ret = perf_config(kmem_config, NULL);
  1629. if (ret)
  1630. return ret;
  1631. argc = parse_options_subcommand(argc, argv, kmem_options,
  1632. kmem_subcommands, kmem_usage,
  1633. PARSE_OPT_STOP_AT_NON_OPTION);
  1634. if (!argc)
  1635. usage_with_options(kmem_usage, kmem_options);
  1636. if (kmem_slab == 0 && kmem_page == 0) {
  1637. if (kmem_default == KMEM_SLAB)
  1638. kmem_slab = 1;
  1639. else
  1640. kmem_page = 1;
  1641. }
  1642. if (strlen(argv[0]) > 2 && strstarts("record", argv[0])) {
  1643. symbol__init(NULL);
  1644. return __cmd_record(argc, argv);
  1645. }
  1646. data.path = input_name;
  1647. kmem_session = session = perf_session__new(&data, &perf_kmem);
  1648. if (IS_ERR(session))
  1649. return PTR_ERR(session);
  1650. ret = -1;
  1651. if (kmem_slab) {
  1652. if (!evlist__find_tracepoint_by_name(session->evlist, "kmem:kmalloc")) {
  1653. pr_err(errmsg, "slab", "slab");
  1654. goto out_delete;
  1655. }
  1656. }
  1657. if (kmem_page) {
  1658. struct evsel *evsel = evlist__find_tracepoint_by_name(session->evlist, "kmem:mm_page_alloc");
  1659. if (evsel == NULL) {
  1660. pr_err(errmsg, "page", "page");
  1661. goto out_delete;
  1662. }
  1663. kmem_page_size = tep_get_page_size(evsel->tp_format->tep);
  1664. symbol_conf.use_callchain = true;
  1665. }
  1666. symbol__init(&session->header.env);
  1667. if (perf_time__parse_str(&ptime, time_str) != 0) {
  1668. pr_err("Invalid time string\n");
  1669. ret = -EINVAL;
  1670. goto out_delete;
  1671. }
  1672. if (!strcmp(argv[0], "stat")) {
  1673. setlocale(LC_ALL, "");
  1674. if (cpu__setup_cpunode_map())
  1675. goto out_delete;
  1676. if (list_empty(&slab_caller_sort))
  1677. setup_slab_sorting(&slab_caller_sort, default_slab_sort);
  1678. if (list_empty(&slab_alloc_sort))
  1679. setup_slab_sorting(&slab_alloc_sort, default_slab_sort);
  1680. if (list_empty(&page_caller_sort))
  1681. setup_page_sorting(&page_caller_sort, default_page_sort);
  1682. if (list_empty(&page_alloc_sort))
  1683. setup_page_sorting(&page_alloc_sort, default_page_sort);
  1684. if (kmem_page) {
  1685. setup_page_sorting(&page_alloc_sort_input,
  1686. "page,order,migtype,gfp");
  1687. setup_page_sorting(&page_caller_sort_input,
  1688. "callsite,order,migtype,gfp");
  1689. }
  1690. ret = __cmd_kmem(session);
  1691. } else
  1692. usage_with_options(kmem_usage, kmem_options);
  1693. out_delete:
  1694. perf_session__delete(session);
  1695. return ret;
  1696. }