stackmap.c 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /* Copyright (c) 2016 Facebook
  3. */
  4. #include <linux/bpf.h>
  5. #include <linux/jhash.h>
  6. #include <linux/filter.h>
  7. #include <linux/kernel.h>
  8. #include <linux/stacktrace.h>
  9. #include <linux/perf_event.h>
  10. #include <linux/btf_ids.h>
  11. #include <linux/buildid.h>
  12. #include "percpu_freelist.h"
  13. #include "mmap_unlock_work.h"
  14. #define STACK_CREATE_FLAG_MASK \
  15. (BPF_F_NUMA_NODE | BPF_F_RDONLY | BPF_F_WRONLY | \
  16. BPF_F_STACK_BUILD_ID)
  17. struct stack_map_bucket {
  18. struct pcpu_freelist_node fnode;
  19. u32 hash;
  20. u32 nr;
  21. u64 data[];
  22. };
  23. struct bpf_stack_map {
  24. struct bpf_map map;
  25. void *elems;
  26. struct pcpu_freelist freelist;
  27. u32 n_buckets;
  28. struct stack_map_bucket *buckets[];
  29. };
  30. static inline bool stack_map_use_build_id(struct bpf_map *map)
  31. {
  32. return (map->map_flags & BPF_F_STACK_BUILD_ID);
  33. }
  34. static inline int stack_map_data_size(struct bpf_map *map)
  35. {
  36. return stack_map_use_build_id(map) ?
  37. sizeof(struct bpf_stack_build_id) : sizeof(u64);
  38. }
  39. static int prealloc_elems_and_freelist(struct bpf_stack_map *smap)
  40. {
  41. u64 elem_size = sizeof(struct stack_map_bucket) +
  42. (u64)smap->map.value_size;
  43. int err;
  44. smap->elems = bpf_map_area_alloc(elem_size * smap->map.max_entries,
  45. smap->map.numa_node);
  46. if (!smap->elems)
  47. return -ENOMEM;
  48. err = pcpu_freelist_init(&smap->freelist);
  49. if (err)
  50. goto free_elems;
  51. pcpu_freelist_populate(&smap->freelist, smap->elems, elem_size,
  52. smap->map.max_entries);
  53. return 0;
  54. free_elems:
  55. bpf_map_area_free(smap->elems);
  56. return err;
  57. }
  58. /* Called from syscall */
  59. static struct bpf_map *stack_map_alloc(union bpf_attr *attr)
  60. {
  61. u32 value_size = attr->value_size;
  62. struct bpf_stack_map *smap;
  63. u64 cost, n_buckets;
  64. int err;
  65. if (!bpf_capable())
  66. return ERR_PTR(-EPERM);
  67. if (attr->map_flags & ~STACK_CREATE_FLAG_MASK)
  68. return ERR_PTR(-EINVAL);
  69. /* check sanity of attributes */
  70. if (attr->max_entries == 0 || attr->key_size != 4 ||
  71. value_size < 8 || value_size % 8)
  72. return ERR_PTR(-EINVAL);
  73. BUILD_BUG_ON(sizeof(struct bpf_stack_build_id) % sizeof(u64));
  74. if (attr->map_flags & BPF_F_STACK_BUILD_ID) {
  75. if (value_size % sizeof(struct bpf_stack_build_id) ||
  76. value_size / sizeof(struct bpf_stack_build_id)
  77. > sysctl_perf_event_max_stack)
  78. return ERR_PTR(-EINVAL);
  79. } else if (value_size / 8 > sysctl_perf_event_max_stack)
  80. return ERR_PTR(-EINVAL);
  81. /* hash table size must be power of 2 */
  82. n_buckets = roundup_pow_of_two(attr->max_entries);
  83. if (!n_buckets)
  84. return ERR_PTR(-E2BIG);
  85. cost = n_buckets * sizeof(struct stack_map_bucket *) + sizeof(*smap);
  86. smap = bpf_map_area_alloc(cost, bpf_map_attr_numa_node(attr));
  87. if (!smap)
  88. return ERR_PTR(-ENOMEM);
  89. bpf_map_init_from_attr(&smap->map, attr);
  90. smap->n_buckets = n_buckets;
  91. err = get_callchain_buffers(sysctl_perf_event_max_stack);
  92. if (err)
  93. goto free_smap;
  94. err = prealloc_elems_and_freelist(smap);
  95. if (err)
  96. goto put_buffers;
  97. return &smap->map;
  98. put_buffers:
  99. put_callchain_buffers();
  100. free_smap:
  101. bpf_map_area_free(smap);
  102. return ERR_PTR(err);
  103. }
  104. static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs,
  105. u64 *ips, u32 trace_nr, bool user)
  106. {
  107. int i;
  108. struct mmap_unlock_irq_work *work = NULL;
  109. bool irq_work_busy = bpf_mmap_unlock_get_irq_work(&work);
  110. struct vm_area_struct *vma, *prev_vma = NULL;
  111. const char *prev_build_id;
  112. /* If the irq_work is in use, fall back to report ips. Same
  113. * fallback is used for kernel stack (!user) on a stackmap with
  114. * build_id.
  115. */
  116. if (!user || !current || !current->mm || irq_work_busy ||
  117. !mmap_read_trylock(current->mm)) {
  118. /* cannot access current->mm, fall back to ips */
  119. for (i = 0; i < trace_nr; i++) {
  120. id_offs[i].status = BPF_STACK_BUILD_ID_IP;
  121. id_offs[i].ip = ips[i];
  122. memset(id_offs[i].build_id, 0, BUILD_ID_SIZE_MAX);
  123. }
  124. return;
  125. }
  126. for (i = 0; i < trace_nr; i++) {
  127. if (range_in_vma(prev_vma, ips[i], ips[i])) {
  128. vma = prev_vma;
  129. memcpy(id_offs[i].build_id, prev_build_id,
  130. BUILD_ID_SIZE_MAX);
  131. goto build_id_valid;
  132. }
  133. vma = find_vma(current->mm, ips[i]);
  134. if (!vma || build_id_parse(vma, id_offs[i].build_id, NULL)) {
  135. /* per entry fall back to ips */
  136. id_offs[i].status = BPF_STACK_BUILD_ID_IP;
  137. id_offs[i].ip = ips[i];
  138. memset(id_offs[i].build_id, 0, BUILD_ID_SIZE_MAX);
  139. continue;
  140. }
  141. build_id_valid:
  142. id_offs[i].offset = (vma->vm_pgoff << PAGE_SHIFT) + ips[i]
  143. - vma->vm_start;
  144. id_offs[i].status = BPF_STACK_BUILD_ID_VALID;
  145. prev_vma = vma;
  146. prev_build_id = id_offs[i].build_id;
  147. }
  148. bpf_mmap_unlock_mm(work, current->mm);
  149. }
  150. static struct perf_callchain_entry *
  151. get_callchain_entry_for_task(struct task_struct *task, u32 max_depth)
  152. {
  153. #ifdef CONFIG_STACKTRACE
  154. struct perf_callchain_entry *entry;
  155. int rctx;
  156. entry = get_callchain_entry(&rctx);
  157. if (!entry)
  158. return NULL;
  159. entry->nr = stack_trace_save_tsk(task, (unsigned long *)entry->ip,
  160. max_depth, 0);
  161. /* stack_trace_save_tsk() works on unsigned long array, while
  162. * perf_callchain_entry uses u64 array. For 32-bit systems, it is
  163. * necessary to fix this mismatch.
  164. */
  165. if (__BITS_PER_LONG != 64) {
  166. unsigned long *from = (unsigned long *) entry->ip;
  167. u64 *to = entry->ip;
  168. int i;
  169. /* copy data from the end to avoid using extra buffer */
  170. for (i = entry->nr - 1; i >= 0; i--)
  171. to[i] = (u64)(from[i]);
  172. }
  173. put_callchain_entry(rctx);
  174. return entry;
  175. #else /* CONFIG_STACKTRACE */
  176. return NULL;
  177. #endif
  178. }
  179. static long __bpf_get_stackid(struct bpf_map *map,
  180. struct perf_callchain_entry *trace, u64 flags)
  181. {
  182. struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
  183. struct stack_map_bucket *bucket, *new_bucket, *old_bucket;
  184. u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
  185. u32 hash, id, trace_nr, trace_len;
  186. bool user = flags & BPF_F_USER_STACK;
  187. u64 *ips;
  188. bool hash_matches;
  189. if (trace->nr <= skip)
  190. /* skipping more than usable stack trace */
  191. return -EFAULT;
  192. trace_nr = trace->nr - skip;
  193. trace_len = trace_nr * sizeof(u64);
  194. ips = trace->ip + skip;
  195. hash = jhash2((u32 *)ips, trace_len / sizeof(u32), 0);
  196. id = hash & (smap->n_buckets - 1);
  197. bucket = READ_ONCE(smap->buckets[id]);
  198. hash_matches = bucket && bucket->hash == hash;
  199. /* fast cmp */
  200. if (hash_matches && flags & BPF_F_FAST_STACK_CMP)
  201. return id;
  202. if (stack_map_use_build_id(map)) {
  203. /* for build_id+offset, pop a bucket before slow cmp */
  204. new_bucket = (struct stack_map_bucket *)
  205. pcpu_freelist_pop(&smap->freelist);
  206. if (unlikely(!new_bucket))
  207. return -ENOMEM;
  208. new_bucket->nr = trace_nr;
  209. stack_map_get_build_id_offset(
  210. (struct bpf_stack_build_id *)new_bucket->data,
  211. ips, trace_nr, user);
  212. trace_len = trace_nr * sizeof(struct bpf_stack_build_id);
  213. if (hash_matches && bucket->nr == trace_nr &&
  214. memcmp(bucket->data, new_bucket->data, trace_len) == 0) {
  215. pcpu_freelist_push(&smap->freelist, &new_bucket->fnode);
  216. return id;
  217. }
  218. if (bucket && !(flags & BPF_F_REUSE_STACKID)) {
  219. pcpu_freelist_push(&smap->freelist, &new_bucket->fnode);
  220. return -EEXIST;
  221. }
  222. } else {
  223. if (hash_matches && bucket->nr == trace_nr &&
  224. memcmp(bucket->data, ips, trace_len) == 0)
  225. return id;
  226. if (bucket && !(flags & BPF_F_REUSE_STACKID))
  227. return -EEXIST;
  228. new_bucket = (struct stack_map_bucket *)
  229. pcpu_freelist_pop(&smap->freelist);
  230. if (unlikely(!new_bucket))
  231. return -ENOMEM;
  232. memcpy(new_bucket->data, ips, trace_len);
  233. }
  234. new_bucket->hash = hash;
  235. new_bucket->nr = trace_nr;
  236. old_bucket = xchg(&smap->buckets[id], new_bucket);
  237. if (old_bucket)
  238. pcpu_freelist_push(&smap->freelist, &old_bucket->fnode);
  239. return id;
  240. }
  241. BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
  242. u64, flags)
  243. {
  244. u32 max_depth = map->value_size / stack_map_data_size(map);
  245. u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
  246. bool user = flags & BPF_F_USER_STACK;
  247. struct perf_callchain_entry *trace;
  248. bool kernel = !user;
  249. if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
  250. BPF_F_FAST_STACK_CMP | BPF_F_REUSE_STACKID)))
  251. return -EINVAL;
  252. max_depth += skip;
  253. if (max_depth > sysctl_perf_event_max_stack)
  254. max_depth = sysctl_perf_event_max_stack;
  255. trace = get_perf_callchain(regs, 0, kernel, user, max_depth,
  256. false, false);
  257. if (unlikely(!trace))
  258. /* couldn't fetch the stack trace */
  259. return -EFAULT;
  260. return __bpf_get_stackid(map, trace, flags);
  261. }
  262. const struct bpf_func_proto bpf_get_stackid_proto = {
  263. .func = bpf_get_stackid,
  264. .gpl_only = true,
  265. .ret_type = RET_INTEGER,
  266. .arg1_type = ARG_PTR_TO_CTX,
  267. .arg2_type = ARG_CONST_MAP_PTR,
  268. .arg3_type = ARG_ANYTHING,
  269. };
  270. static __u64 count_kernel_ip(struct perf_callchain_entry *trace)
  271. {
  272. __u64 nr_kernel = 0;
  273. while (nr_kernel < trace->nr) {
  274. if (trace->ip[nr_kernel] == PERF_CONTEXT_USER)
  275. break;
  276. nr_kernel++;
  277. }
  278. return nr_kernel;
  279. }
  280. BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx,
  281. struct bpf_map *, map, u64, flags)
  282. {
  283. struct perf_event *event = ctx->event;
  284. struct perf_callchain_entry *trace;
  285. bool kernel, user;
  286. __u64 nr_kernel;
  287. int ret;
  288. /* perf_sample_data doesn't have callchain, use bpf_get_stackid */
  289. if (!(event->attr.sample_type & PERF_SAMPLE_CALLCHAIN))
  290. return bpf_get_stackid((unsigned long)(ctx->regs),
  291. (unsigned long) map, flags, 0, 0);
  292. if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
  293. BPF_F_FAST_STACK_CMP | BPF_F_REUSE_STACKID)))
  294. return -EINVAL;
  295. user = flags & BPF_F_USER_STACK;
  296. kernel = !user;
  297. trace = ctx->data->callchain;
  298. if (unlikely(!trace))
  299. return -EFAULT;
  300. nr_kernel = count_kernel_ip(trace);
  301. if (kernel) {
  302. __u64 nr = trace->nr;
  303. trace->nr = nr_kernel;
  304. ret = __bpf_get_stackid(map, trace, flags);
  305. /* restore nr */
  306. trace->nr = nr;
  307. } else { /* user */
  308. u64 skip = flags & BPF_F_SKIP_FIELD_MASK;
  309. skip += nr_kernel;
  310. if (skip > BPF_F_SKIP_FIELD_MASK)
  311. return -EFAULT;
  312. flags = (flags & ~BPF_F_SKIP_FIELD_MASK) | skip;
  313. ret = __bpf_get_stackid(map, trace, flags);
  314. }
  315. return ret;
  316. }
  317. const struct bpf_func_proto bpf_get_stackid_proto_pe = {
  318. .func = bpf_get_stackid_pe,
  319. .gpl_only = false,
  320. .ret_type = RET_INTEGER,
  321. .arg1_type = ARG_PTR_TO_CTX,
  322. .arg2_type = ARG_CONST_MAP_PTR,
  323. .arg3_type = ARG_ANYTHING,
  324. };
  325. static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
  326. struct perf_callchain_entry *trace_in,
  327. void *buf, u32 size, u64 flags)
  328. {
  329. u32 trace_nr, copy_len, elem_size, num_elem, max_depth;
  330. bool user_build_id = flags & BPF_F_USER_BUILD_ID;
  331. u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
  332. bool user = flags & BPF_F_USER_STACK;
  333. struct perf_callchain_entry *trace;
  334. bool kernel = !user;
  335. int err = -EINVAL;
  336. u64 *ips;
  337. if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
  338. BPF_F_USER_BUILD_ID)))
  339. goto clear;
  340. if (kernel && user_build_id)
  341. goto clear;
  342. elem_size = (user && user_build_id) ? sizeof(struct bpf_stack_build_id)
  343. : sizeof(u64);
  344. if (unlikely(size % elem_size))
  345. goto clear;
  346. /* cannot get valid user stack for task without user_mode regs */
  347. if (task && user && !user_mode(regs))
  348. goto err_fault;
  349. num_elem = size / elem_size;
  350. max_depth = num_elem + skip;
  351. if (sysctl_perf_event_max_stack < max_depth)
  352. max_depth = sysctl_perf_event_max_stack;
  353. if (trace_in)
  354. trace = trace_in;
  355. else if (kernel && task)
  356. trace = get_callchain_entry_for_task(task, max_depth);
  357. else
  358. trace = get_perf_callchain(regs, 0, kernel, user, max_depth,
  359. false, false);
  360. if (unlikely(!trace))
  361. goto err_fault;
  362. if (trace->nr < skip)
  363. goto err_fault;
  364. trace_nr = trace->nr - skip;
  365. trace_nr = (trace_nr <= num_elem) ? trace_nr : num_elem;
  366. copy_len = trace_nr * elem_size;
  367. ips = trace->ip + skip;
  368. if (user && user_build_id)
  369. stack_map_get_build_id_offset(buf, ips, trace_nr, user);
  370. else
  371. memcpy(buf, ips, copy_len);
  372. if (size > copy_len)
  373. memset(buf + copy_len, 0, size - copy_len);
  374. return copy_len;
  375. err_fault:
  376. err = -EFAULT;
  377. clear:
  378. memset(buf, 0, size);
  379. return err;
  380. }
  381. BPF_CALL_4(bpf_get_stack, struct pt_regs *, regs, void *, buf, u32, size,
  382. u64, flags)
  383. {
  384. return __bpf_get_stack(regs, NULL, NULL, buf, size, flags);
  385. }
  386. const struct bpf_func_proto bpf_get_stack_proto = {
  387. .func = bpf_get_stack,
  388. .gpl_only = true,
  389. .ret_type = RET_INTEGER,
  390. .arg1_type = ARG_PTR_TO_CTX,
  391. .arg2_type = ARG_PTR_TO_UNINIT_MEM,
  392. .arg3_type = ARG_CONST_SIZE_OR_ZERO,
  393. .arg4_type = ARG_ANYTHING,
  394. };
  395. BPF_CALL_4(bpf_get_task_stack, struct task_struct *, task, void *, buf,
  396. u32, size, u64, flags)
  397. {
  398. struct pt_regs *regs;
  399. long res = -EINVAL;
  400. if (!try_get_task_stack(task))
  401. return -EFAULT;
  402. regs = task_pt_regs(task);
  403. if (regs)
  404. res = __bpf_get_stack(regs, task, NULL, buf, size, flags);
  405. put_task_stack(task);
  406. return res;
  407. }
  408. const struct bpf_func_proto bpf_get_task_stack_proto = {
  409. .func = bpf_get_task_stack,
  410. .gpl_only = false,
  411. .ret_type = RET_INTEGER,
  412. .arg1_type = ARG_PTR_TO_BTF_ID,
  413. .arg1_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK],
  414. .arg2_type = ARG_PTR_TO_UNINIT_MEM,
  415. .arg3_type = ARG_CONST_SIZE_OR_ZERO,
  416. .arg4_type = ARG_ANYTHING,
  417. };
  418. BPF_CALL_4(bpf_get_stack_pe, struct bpf_perf_event_data_kern *, ctx,
  419. void *, buf, u32, size, u64, flags)
  420. {
  421. struct pt_regs *regs = (struct pt_regs *)(ctx->regs);
  422. struct perf_event *event = ctx->event;
  423. struct perf_callchain_entry *trace;
  424. bool kernel, user;
  425. int err = -EINVAL;
  426. __u64 nr_kernel;
  427. if (!(event->attr.sample_type & PERF_SAMPLE_CALLCHAIN))
  428. return __bpf_get_stack(regs, NULL, NULL, buf, size, flags);
  429. if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
  430. BPF_F_USER_BUILD_ID)))
  431. goto clear;
  432. user = flags & BPF_F_USER_STACK;
  433. kernel = !user;
  434. err = -EFAULT;
  435. trace = ctx->data->callchain;
  436. if (unlikely(!trace))
  437. goto clear;
  438. nr_kernel = count_kernel_ip(trace);
  439. if (kernel) {
  440. __u64 nr = trace->nr;
  441. trace->nr = nr_kernel;
  442. err = __bpf_get_stack(regs, NULL, trace, buf, size, flags);
  443. /* restore nr */
  444. trace->nr = nr;
  445. } else { /* user */
  446. u64 skip = flags & BPF_F_SKIP_FIELD_MASK;
  447. skip += nr_kernel;
  448. if (skip > BPF_F_SKIP_FIELD_MASK)
  449. goto clear;
  450. flags = (flags & ~BPF_F_SKIP_FIELD_MASK) | skip;
  451. err = __bpf_get_stack(regs, NULL, trace, buf, size, flags);
  452. }
  453. return err;
  454. clear:
  455. memset(buf, 0, size);
  456. return err;
  457. }
  458. const struct bpf_func_proto bpf_get_stack_proto_pe = {
  459. .func = bpf_get_stack_pe,
  460. .gpl_only = true,
  461. .ret_type = RET_INTEGER,
  462. .arg1_type = ARG_PTR_TO_CTX,
  463. .arg2_type = ARG_PTR_TO_UNINIT_MEM,
  464. .arg3_type = ARG_CONST_SIZE_OR_ZERO,
  465. .arg4_type = ARG_ANYTHING,
  466. };
  467. /* Called from eBPF program */
  468. static void *stack_map_lookup_elem(struct bpf_map *map, void *key)
  469. {
  470. return ERR_PTR(-EOPNOTSUPP);
  471. }
  472. /* Called from syscall */
  473. int bpf_stackmap_copy(struct bpf_map *map, void *key, void *value)
  474. {
  475. struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
  476. struct stack_map_bucket *bucket, *old_bucket;
  477. u32 id = *(u32 *)key, trace_len;
  478. if (unlikely(id >= smap->n_buckets))
  479. return -ENOENT;
  480. bucket = xchg(&smap->buckets[id], NULL);
  481. if (!bucket)
  482. return -ENOENT;
  483. trace_len = bucket->nr * stack_map_data_size(map);
  484. memcpy(value, bucket->data, trace_len);
  485. memset(value + trace_len, 0, map->value_size - trace_len);
  486. old_bucket = xchg(&smap->buckets[id], bucket);
  487. if (old_bucket)
  488. pcpu_freelist_push(&smap->freelist, &old_bucket->fnode);
  489. return 0;
  490. }
  491. static int stack_map_get_next_key(struct bpf_map *map, void *key,
  492. void *next_key)
  493. {
  494. struct bpf_stack_map *smap = container_of(map,
  495. struct bpf_stack_map, map);
  496. u32 id;
  497. WARN_ON_ONCE(!rcu_read_lock_held());
  498. if (!key) {
  499. id = 0;
  500. } else {
  501. id = *(u32 *)key;
  502. if (id >= smap->n_buckets || !smap->buckets[id])
  503. id = 0;
  504. else
  505. id++;
  506. }
  507. while (id < smap->n_buckets && !smap->buckets[id])
  508. id++;
  509. if (id >= smap->n_buckets)
  510. return -ENOENT;
  511. *(u32 *)next_key = id;
  512. return 0;
  513. }
  514. static int stack_map_update_elem(struct bpf_map *map, void *key, void *value,
  515. u64 map_flags)
  516. {
  517. return -EINVAL;
  518. }
  519. /* Called from syscall or from eBPF program */
  520. static int stack_map_delete_elem(struct bpf_map *map, void *key)
  521. {
  522. struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
  523. struct stack_map_bucket *old_bucket;
  524. u32 id = *(u32 *)key;
  525. if (unlikely(id >= smap->n_buckets))
  526. return -E2BIG;
  527. old_bucket = xchg(&smap->buckets[id], NULL);
  528. if (old_bucket) {
  529. pcpu_freelist_push(&smap->freelist, &old_bucket->fnode);
  530. return 0;
  531. } else {
  532. return -ENOENT;
  533. }
  534. }
  535. /* Called when map->refcnt goes to zero, either from workqueue or from syscall */
  536. static void stack_map_free(struct bpf_map *map)
  537. {
  538. struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
  539. bpf_map_area_free(smap->elems);
  540. pcpu_freelist_destroy(&smap->freelist);
  541. bpf_map_area_free(smap);
  542. put_callchain_buffers();
  543. }
  544. BTF_ID_LIST_SINGLE(stack_trace_map_btf_ids, struct, bpf_stack_map)
  545. const struct bpf_map_ops stack_trace_map_ops = {
  546. .map_meta_equal = bpf_map_meta_equal,
  547. .map_alloc = stack_map_alloc,
  548. .map_free = stack_map_free,
  549. .map_get_next_key = stack_map_get_next_key,
  550. .map_lookup_elem = stack_map_lookup_elem,
  551. .map_update_elem = stack_map_update_elem,
  552. .map_delete_elem = stack_map_delete_elem,
  553. .map_check_btf = map_check_no_btf,
  554. .map_btf_id = &stack_trace_map_btf_ids[0],
  555. };