trace_syscalls.c 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808
  1. // SPDX-License-Identifier: GPL-2.0
  2. #include <trace/syscall.h>
  3. #include <trace/events/syscalls.h>
  4. #include <linux/syscalls.h>
  5. #include <linux/slab.h>
  6. #include <linux/kernel.h>
  7. #include <linux/module.h> /* for MODULE_NAME_LEN via KSYM_SYMBOL_LEN */
  8. #include <linux/ftrace.h>
  9. #include <linux/perf_event.h>
  10. #include <linux/xarray.h>
  11. #include <asm/syscall.h>
  12. #include "trace_output.h"
  13. #include "trace.h"
  14. static DEFINE_MUTEX(syscall_trace_lock);
  15. static int syscall_enter_register(struct trace_event_call *event,
  16. enum trace_reg type, void *data);
  17. static int syscall_exit_register(struct trace_event_call *event,
  18. enum trace_reg type, void *data);
  19. static struct list_head *
  20. syscall_get_enter_fields(struct trace_event_call *call)
  21. {
  22. struct syscall_metadata *entry = call->data;
  23. return &entry->enter_fields;
  24. }
  25. extern struct syscall_metadata *__start_syscalls_metadata[];
  26. extern struct syscall_metadata *__stop_syscalls_metadata[];
  27. static DEFINE_XARRAY(syscalls_metadata_sparse);
  28. static struct syscall_metadata **syscalls_metadata;
  29. #ifndef ARCH_HAS_SYSCALL_MATCH_SYM_NAME
  30. static inline bool arch_syscall_match_sym_name(const char *sym, const char *name)
  31. {
  32. /*
  33. * Only compare after the "sys" prefix. Archs that use
  34. * syscall wrappers may have syscalls symbols aliases prefixed
  35. * with ".SyS" or ".sys" instead of "sys", leading to an unwanted
  36. * mismatch.
  37. */
  38. return !strcmp(sym + 3, name + 3);
  39. }
  40. #endif
  41. #ifdef ARCH_TRACE_IGNORE_COMPAT_SYSCALLS
  42. /*
  43. * Some architectures that allow for 32bit applications
  44. * to run on a 64bit kernel, do not map the syscalls for
  45. * the 32bit tasks the same as they do for 64bit tasks.
  46. *
  47. * *cough*x86*cough*
  48. *
  49. * In such a case, instead of reporting the wrong syscalls,
  50. * simply ignore them.
  51. *
  52. * For an arch to ignore the compat syscalls it needs to
  53. * define ARCH_TRACE_IGNORE_COMPAT_SYSCALLS as well as
  54. * define the function arch_trace_is_compat_syscall() to let
  55. * the tracing system know that it should ignore it.
  56. */
  57. static int
  58. trace_get_syscall_nr(struct task_struct *task, struct pt_regs *regs)
  59. {
  60. if (unlikely(arch_trace_is_compat_syscall(regs)))
  61. return -1;
  62. return syscall_get_nr(task, regs);
  63. }
  64. #else
  65. static inline int
  66. trace_get_syscall_nr(struct task_struct *task, struct pt_regs *regs)
  67. {
  68. return syscall_get_nr(task, regs);
  69. }
  70. #endif /* ARCH_TRACE_IGNORE_COMPAT_SYSCALLS */
  71. static __init struct syscall_metadata *
  72. find_syscall_meta(unsigned long syscall)
  73. {
  74. struct syscall_metadata **start;
  75. struct syscall_metadata **stop;
  76. char str[KSYM_SYMBOL_LEN];
  77. start = __start_syscalls_metadata;
  78. stop = __stop_syscalls_metadata;
  79. kallsyms_lookup(syscall, NULL, NULL, NULL, str);
  80. if (arch_syscall_match_sym_name(str, "sys_ni_syscall"))
  81. return NULL;
  82. for ( ; start < stop; start++) {
  83. if ((*start)->name && arch_syscall_match_sym_name(str, (*start)->name))
  84. return *start;
  85. }
  86. return NULL;
  87. }
  88. static struct syscall_metadata *syscall_nr_to_meta(int nr)
  89. {
  90. if (IS_ENABLED(CONFIG_HAVE_SPARSE_SYSCALL_NR))
  91. return xa_load(&syscalls_metadata_sparse, (unsigned long)nr);
  92. if (!syscalls_metadata || nr >= NR_syscalls || nr < 0)
  93. return NULL;
  94. return syscalls_metadata[nr];
  95. }
  96. const char *get_syscall_name(int syscall)
  97. {
  98. struct syscall_metadata *entry;
  99. entry = syscall_nr_to_meta(syscall);
  100. if (!entry)
  101. return NULL;
  102. return entry->name;
  103. }
  104. static enum print_line_t
  105. print_syscall_enter(struct trace_iterator *iter, int flags,
  106. struct trace_event *event)
  107. {
  108. struct trace_array *tr = iter->tr;
  109. struct trace_seq *s = &iter->seq;
  110. struct trace_entry *ent = iter->ent;
  111. struct syscall_trace_enter *trace;
  112. struct syscall_metadata *entry;
  113. int i, syscall;
  114. trace = (typeof(trace))ent;
  115. syscall = trace->nr;
  116. entry = syscall_nr_to_meta(syscall);
  117. if (!entry)
  118. goto end;
  119. if (entry->enter_event->event.type != ent->type) {
  120. WARN_ON_ONCE(1);
  121. goto end;
  122. }
  123. trace_seq_printf(s, "%s(", entry->name);
  124. for (i = 0; i < entry->nb_args; i++) {
  125. if (trace_seq_has_overflowed(s))
  126. goto end;
  127. /* parameter types */
  128. if (tr && tr->trace_flags & TRACE_ITER_VERBOSE)
  129. trace_seq_printf(s, "%s ", entry->types[i]);
  130. /* parameter values */
  131. trace_seq_printf(s, "%s: %lx%s", entry->args[i],
  132. trace->args[i],
  133. i == entry->nb_args - 1 ? "" : ", ");
  134. }
  135. trace_seq_putc(s, ')');
  136. end:
  137. trace_seq_putc(s, '\n');
  138. return trace_handle_return(s);
  139. }
  140. static enum print_line_t
  141. print_syscall_exit(struct trace_iterator *iter, int flags,
  142. struct trace_event *event)
  143. {
  144. struct trace_seq *s = &iter->seq;
  145. struct trace_entry *ent = iter->ent;
  146. struct syscall_trace_exit *trace;
  147. int syscall;
  148. struct syscall_metadata *entry;
  149. trace = (typeof(trace))ent;
  150. syscall = trace->nr;
  151. entry = syscall_nr_to_meta(syscall);
  152. if (!entry) {
  153. trace_seq_putc(s, '\n');
  154. goto out;
  155. }
  156. if (entry->exit_event->event.type != ent->type) {
  157. WARN_ON_ONCE(1);
  158. return TRACE_TYPE_UNHANDLED;
  159. }
  160. trace_seq_printf(s, "%s -> 0x%lx\n", entry->name,
  161. trace->ret);
  162. out:
  163. return trace_handle_return(s);
  164. }
  165. #define SYSCALL_FIELD(_type, _name) { \
  166. .type = #_type, .name = #_name, \
  167. .size = sizeof(_type), .align = __alignof__(_type), \
  168. .is_signed = is_signed_type(_type), .filter_type = FILTER_OTHER }
  169. static int __init
  170. __set_enter_print_fmt(struct syscall_metadata *entry, char *buf, int len)
  171. {
  172. int i;
  173. int pos = 0;
  174. /* When len=0, we just calculate the needed length */
  175. #define LEN_OR_ZERO (len ? len - pos : 0)
  176. pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
  177. for (i = 0; i < entry->nb_args; i++) {
  178. pos += snprintf(buf + pos, LEN_OR_ZERO, "%s: 0x%%0%zulx%s",
  179. entry->args[i], sizeof(unsigned long),
  180. i == entry->nb_args - 1 ? "" : ", ");
  181. }
  182. pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
  183. for (i = 0; i < entry->nb_args; i++) {
  184. pos += snprintf(buf + pos, LEN_OR_ZERO,
  185. ", ((unsigned long)(REC->%s))", entry->args[i]);
  186. }
  187. #undef LEN_OR_ZERO
  188. /* return the length of print_fmt */
  189. return pos;
  190. }
  191. static int __init set_syscall_print_fmt(struct trace_event_call *call)
  192. {
  193. char *print_fmt;
  194. int len;
  195. struct syscall_metadata *entry = call->data;
  196. if (entry->enter_event != call) {
  197. call->print_fmt = "\"0x%lx\", REC->ret";
  198. return 0;
  199. }
  200. /* First: called with 0 length to calculate the needed length */
  201. len = __set_enter_print_fmt(entry, NULL, 0);
  202. print_fmt = kmalloc(len + 1, GFP_KERNEL);
  203. if (!print_fmt)
  204. return -ENOMEM;
  205. /* Second: actually write the @print_fmt */
  206. __set_enter_print_fmt(entry, print_fmt, len + 1);
  207. call->print_fmt = print_fmt;
  208. return 0;
  209. }
  210. static void __init free_syscall_print_fmt(struct trace_event_call *call)
  211. {
  212. struct syscall_metadata *entry = call->data;
  213. if (entry->enter_event == call)
  214. kfree(call->print_fmt);
  215. }
  216. static int __init syscall_enter_define_fields(struct trace_event_call *call)
  217. {
  218. struct syscall_trace_enter trace;
  219. struct syscall_metadata *meta = call->data;
  220. int offset = offsetof(typeof(trace), args);
  221. int ret = 0;
  222. int i;
  223. for (i = 0; i < meta->nb_args; i++) {
  224. ret = trace_define_field(call, meta->types[i],
  225. meta->args[i], offset,
  226. sizeof(unsigned long), 0,
  227. FILTER_OTHER);
  228. if (ret)
  229. break;
  230. offset += sizeof(unsigned long);
  231. }
  232. return ret;
  233. }
  234. static void ftrace_syscall_enter(void *data, struct pt_regs *regs, long id)
  235. {
  236. struct trace_array *tr = data;
  237. struct trace_event_file *trace_file;
  238. struct syscall_trace_enter *entry;
  239. struct syscall_metadata *sys_data;
  240. struct trace_event_buffer fbuffer;
  241. unsigned long args[6];
  242. int syscall_nr;
  243. int size;
  244. syscall_nr = trace_get_syscall_nr(current, regs);
  245. if (syscall_nr < 0 || syscall_nr >= NR_syscalls)
  246. return;
  247. /* Here we're inside tp handler's rcu_read_lock_sched (__DO_TRACE) */
  248. trace_file = rcu_dereference_sched(tr->enter_syscall_files[syscall_nr]);
  249. if (!trace_file)
  250. return;
  251. if (trace_trigger_soft_disabled(trace_file))
  252. return;
  253. sys_data = syscall_nr_to_meta(syscall_nr);
  254. if (!sys_data)
  255. return;
  256. size = sizeof(*entry) + sizeof(unsigned long) * sys_data->nb_args;
  257. entry = trace_event_buffer_reserve(&fbuffer, trace_file, size);
  258. if (!entry)
  259. return;
  260. entry = ring_buffer_event_data(fbuffer.event);
  261. entry->nr = syscall_nr;
  262. syscall_get_arguments(current, regs, args);
  263. memcpy(entry->args, args, sizeof(unsigned long) * sys_data->nb_args);
  264. trace_event_buffer_commit(&fbuffer);
  265. }
  266. static void ftrace_syscall_exit(void *data, struct pt_regs *regs, long ret)
  267. {
  268. struct trace_array *tr = data;
  269. struct trace_event_file *trace_file;
  270. struct syscall_trace_exit *entry;
  271. struct syscall_metadata *sys_data;
  272. struct trace_event_buffer fbuffer;
  273. int syscall_nr;
  274. syscall_nr = trace_get_syscall_nr(current, regs);
  275. if (syscall_nr < 0 || syscall_nr >= NR_syscalls)
  276. return;
  277. /* Here we're inside tp handler's rcu_read_lock_sched (__DO_TRACE()) */
  278. trace_file = rcu_dereference_sched(tr->exit_syscall_files[syscall_nr]);
  279. if (!trace_file)
  280. return;
  281. if (trace_trigger_soft_disabled(trace_file))
  282. return;
  283. sys_data = syscall_nr_to_meta(syscall_nr);
  284. if (!sys_data)
  285. return;
  286. entry = trace_event_buffer_reserve(&fbuffer, trace_file, sizeof(*entry));
  287. if (!entry)
  288. return;
  289. entry = ring_buffer_event_data(fbuffer.event);
  290. entry->nr = syscall_nr;
  291. entry->ret = syscall_get_return_value(current, regs);
  292. trace_event_buffer_commit(&fbuffer);
  293. }
  294. static int reg_event_syscall_enter(struct trace_event_file *file,
  295. struct trace_event_call *call)
  296. {
  297. struct trace_array *tr = file->tr;
  298. int ret = 0;
  299. int num;
  300. num = ((struct syscall_metadata *)call->data)->syscall_nr;
  301. if (WARN_ON_ONCE(num < 0 || num >= NR_syscalls))
  302. return -ENOSYS;
  303. mutex_lock(&syscall_trace_lock);
  304. if (!tr->sys_refcount_enter)
  305. ret = register_trace_sys_enter(ftrace_syscall_enter, tr);
  306. if (!ret) {
  307. rcu_assign_pointer(tr->enter_syscall_files[num], file);
  308. tr->sys_refcount_enter++;
  309. }
  310. mutex_unlock(&syscall_trace_lock);
  311. return ret;
  312. }
  313. static void unreg_event_syscall_enter(struct trace_event_file *file,
  314. struct trace_event_call *call)
  315. {
  316. struct trace_array *tr = file->tr;
  317. int num;
  318. num = ((struct syscall_metadata *)call->data)->syscall_nr;
  319. if (WARN_ON_ONCE(num < 0 || num >= NR_syscalls))
  320. return;
  321. mutex_lock(&syscall_trace_lock);
  322. tr->sys_refcount_enter--;
  323. RCU_INIT_POINTER(tr->enter_syscall_files[num], NULL);
  324. if (!tr->sys_refcount_enter)
  325. unregister_trace_sys_enter(ftrace_syscall_enter, tr);
  326. mutex_unlock(&syscall_trace_lock);
  327. }
  328. static int reg_event_syscall_exit(struct trace_event_file *file,
  329. struct trace_event_call *call)
  330. {
  331. struct trace_array *tr = file->tr;
  332. int ret = 0;
  333. int num;
  334. num = ((struct syscall_metadata *)call->data)->syscall_nr;
  335. if (WARN_ON_ONCE(num < 0 || num >= NR_syscalls))
  336. return -ENOSYS;
  337. mutex_lock(&syscall_trace_lock);
  338. if (!tr->sys_refcount_exit)
  339. ret = register_trace_sys_exit(ftrace_syscall_exit, tr);
  340. if (!ret) {
  341. rcu_assign_pointer(tr->exit_syscall_files[num], file);
  342. tr->sys_refcount_exit++;
  343. }
  344. mutex_unlock(&syscall_trace_lock);
  345. return ret;
  346. }
  347. static void unreg_event_syscall_exit(struct trace_event_file *file,
  348. struct trace_event_call *call)
  349. {
  350. struct trace_array *tr = file->tr;
  351. int num;
  352. num = ((struct syscall_metadata *)call->data)->syscall_nr;
  353. if (WARN_ON_ONCE(num < 0 || num >= NR_syscalls))
  354. return;
  355. mutex_lock(&syscall_trace_lock);
  356. tr->sys_refcount_exit--;
  357. RCU_INIT_POINTER(tr->exit_syscall_files[num], NULL);
  358. if (!tr->sys_refcount_exit)
  359. unregister_trace_sys_exit(ftrace_syscall_exit, tr);
  360. mutex_unlock(&syscall_trace_lock);
  361. }
  362. static int __init init_syscall_trace(struct trace_event_call *call)
  363. {
  364. int id;
  365. int num;
  366. num = ((struct syscall_metadata *)call->data)->syscall_nr;
  367. if (num < 0 || num >= NR_syscalls) {
  368. pr_debug("syscall %s metadata not mapped, disabling ftrace event\n",
  369. ((struct syscall_metadata *)call->data)->name);
  370. return -ENOSYS;
  371. }
  372. if (set_syscall_print_fmt(call) < 0)
  373. return -ENOMEM;
  374. id = trace_event_raw_init(call);
  375. if (id < 0) {
  376. free_syscall_print_fmt(call);
  377. return id;
  378. }
  379. return id;
  380. }
  381. static struct trace_event_fields __refdata syscall_enter_fields_array[] = {
  382. SYSCALL_FIELD(int, __syscall_nr),
  383. { .type = TRACE_FUNCTION_TYPE,
  384. .define_fields = syscall_enter_define_fields },
  385. {}
  386. };
  387. struct trace_event_functions enter_syscall_print_funcs = {
  388. .trace = print_syscall_enter,
  389. };
  390. struct trace_event_functions exit_syscall_print_funcs = {
  391. .trace = print_syscall_exit,
  392. };
  393. struct trace_event_class __refdata event_class_syscall_enter = {
  394. .system = "syscalls",
  395. .reg = syscall_enter_register,
  396. .fields_array = syscall_enter_fields_array,
  397. .get_fields = syscall_get_enter_fields,
  398. .raw_init = init_syscall_trace,
  399. };
  400. struct trace_event_class __refdata event_class_syscall_exit = {
  401. .system = "syscalls",
  402. .reg = syscall_exit_register,
  403. .fields_array = (struct trace_event_fields[]){
  404. SYSCALL_FIELD(int, __syscall_nr),
  405. SYSCALL_FIELD(long, ret),
  406. {}
  407. },
  408. .fields = LIST_HEAD_INIT(event_class_syscall_exit.fields),
  409. .raw_init = init_syscall_trace,
  410. };
  411. unsigned long __init __weak arch_syscall_addr(int nr)
  412. {
  413. return (unsigned long)sys_call_table[nr];
  414. }
  415. void __init init_ftrace_syscalls(void)
  416. {
  417. struct syscall_metadata *meta;
  418. unsigned long addr;
  419. int i;
  420. void *ret;
  421. if (!IS_ENABLED(CONFIG_HAVE_SPARSE_SYSCALL_NR)) {
  422. syscalls_metadata = kcalloc(NR_syscalls,
  423. sizeof(*syscalls_metadata),
  424. GFP_KERNEL);
  425. if (!syscalls_metadata) {
  426. WARN_ON(1);
  427. return;
  428. }
  429. }
  430. for (i = 0; i < NR_syscalls; i++) {
  431. addr = arch_syscall_addr(i);
  432. meta = find_syscall_meta(addr);
  433. if (!meta)
  434. continue;
  435. meta->syscall_nr = i;
  436. if (!IS_ENABLED(CONFIG_HAVE_SPARSE_SYSCALL_NR)) {
  437. syscalls_metadata[i] = meta;
  438. } else {
  439. ret = xa_store(&syscalls_metadata_sparse, i, meta,
  440. GFP_KERNEL);
  441. WARN(xa_is_err(ret),
  442. "Syscall memory allocation failed\n");
  443. }
  444. }
  445. }
  446. #ifdef CONFIG_PERF_EVENTS
  447. static DECLARE_BITMAP(enabled_perf_enter_syscalls, NR_syscalls);
  448. static DECLARE_BITMAP(enabled_perf_exit_syscalls, NR_syscalls);
  449. static int sys_perf_refcount_enter;
  450. static int sys_perf_refcount_exit;
  451. static int perf_call_bpf_enter(struct trace_event_call *call, struct pt_regs *regs,
  452. struct syscall_metadata *sys_data,
  453. struct syscall_trace_enter *rec)
  454. {
  455. struct syscall_tp_t {
  456. unsigned long long regs;
  457. unsigned long syscall_nr;
  458. unsigned long args[SYSCALL_DEFINE_MAXARGS];
  459. } param;
  460. int i;
  461. *(struct pt_regs **)&param = regs;
  462. param.syscall_nr = rec->nr;
  463. for (i = 0; i < sys_data->nb_args; i++)
  464. param.args[i] = rec->args[i];
  465. return trace_call_bpf(call, &param);
  466. }
  467. static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id)
  468. {
  469. struct syscall_metadata *sys_data;
  470. struct syscall_trace_enter *rec;
  471. struct hlist_head *head;
  472. unsigned long args[6];
  473. bool valid_prog_array;
  474. int syscall_nr;
  475. int rctx;
  476. int size;
  477. syscall_nr = trace_get_syscall_nr(current, regs);
  478. if (syscall_nr < 0 || syscall_nr >= NR_syscalls)
  479. return;
  480. if (!test_bit(syscall_nr, enabled_perf_enter_syscalls))
  481. return;
  482. sys_data = syscall_nr_to_meta(syscall_nr);
  483. if (!sys_data)
  484. return;
  485. head = this_cpu_ptr(sys_data->enter_event->perf_events);
  486. valid_prog_array = bpf_prog_array_valid(sys_data->enter_event);
  487. if (!valid_prog_array && hlist_empty(head))
  488. return;
  489. /* get the size after alignment with the u32 buffer size field */
  490. size = sizeof(unsigned long) * sys_data->nb_args + sizeof(*rec);
  491. size = ALIGN(size + sizeof(u32), sizeof(u64));
  492. size -= sizeof(u32);
  493. rec = perf_trace_buf_alloc(size, NULL, &rctx);
  494. if (!rec)
  495. return;
  496. rec->nr = syscall_nr;
  497. syscall_get_arguments(current, regs, args);
  498. memcpy(&rec->args, args, sizeof(unsigned long) * sys_data->nb_args);
  499. if ((valid_prog_array &&
  500. !perf_call_bpf_enter(sys_data->enter_event, regs, sys_data, rec)) ||
  501. hlist_empty(head)) {
  502. perf_swevent_put_recursion_context(rctx);
  503. return;
  504. }
  505. perf_trace_buf_submit(rec, size, rctx,
  506. sys_data->enter_event->event.type, 1, regs,
  507. head, NULL);
  508. }
  509. static int perf_sysenter_enable(struct trace_event_call *call)
  510. {
  511. int ret = 0;
  512. int num;
  513. num = ((struct syscall_metadata *)call->data)->syscall_nr;
  514. mutex_lock(&syscall_trace_lock);
  515. if (!sys_perf_refcount_enter)
  516. ret = register_trace_sys_enter(perf_syscall_enter, NULL);
  517. if (ret) {
  518. pr_info("event trace: Could not activate syscall entry trace point");
  519. } else {
  520. set_bit(num, enabled_perf_enter_syscalls);
  521. sys_perf_refcount_enter++;
  522. }
  523. mutex_unlock(&syscall_trace_lock);
  524. return ret;
  525. }
  526. static void perf_sysenter_disable(struct trace_event_call *call)
  527. {
  528. int num;
  529. num = ((struct syscall_metadata *)call->data)->syscall_nr;
  530. mutex_lock(&syscall_trace_lock);
  531. sys_perf_refcount_enter--;
  532. clear_bit(num, enabled_perf_enter_syscalls);
  533. if (!sys_perf_refcount_enter)
  534. unregister_trace_sys_enter(perf_syscall_enter, NULL);
  535. mutex_unlock(&syscall_trace_lock);
  536. }
  537. static int perf_call_bpf_exit(struct trace_event_call *call, struct pt_regs *regs,
  538. struct syscall_trace_exit *rec)
  539. {
  540. struct syscall_tp_t {
  541. unsigned long long regs;
  542. unsigned long syscall_nr;
  543. unsigned long ret;
  544. } param;
  545. *(struct pt_regs **)&param = regs;
  546. param.syscall_nr = rec->nr;
  547. param.ret = rec->ret;
  548. return trace_call_bpf(call, &param);
  549. }
  550. static void perf_syscall_exit(void *ignore, struct pt_regs *regs, long ret)
  551. {
  552. struct syscall_metadata *sys_data;
  553. struct syscall_trace_exit *rec;
  554. struct hlist_head *head;
  555. bool valid_prog_array;
  556. int syscall_nr;
  557. int rctx;
  558. int size;
  559. syscall_nr = trace_get_syscall_nr(current, regs);
  560. if (syscall_nr < 0 || syscall_nr >= NR_syscalls)
  561. return;
  562. if (!test_bit(syscall_nr, enabled_perf_exit_syscalls))
  563. return;
  564. sys_data = syscall_nr_to_meta(syscall_nr);
  565. if (!sys_data)
  566. return;
  567. head = this_cpu_ptr(sys_data->exit_event->perf_events);
  568. valid_prog_array = bpf_prog_array_valid(sys_data->exit_event);
  569. if (!valid_prog_array && hlist_empty(head))
  570. return;
  571. /* We can probably do that at build time */
  572. size = ALIGN(sizeof(*rec) + sizeof(u32), sizeof(u64));
  573. size -= sizeof(u32);
  574. rec = perf_trace_buf_alloc(size, NULL, &rctx);
  575. if (!rec)
  576. return;
  577. rec->nr = syscall_nr;
  578. rec->ret = syscall_get_return_value(current, regs);
  579. if ((valid_prog_array &&
  580. !perf_call_bpf_exit(sys_data->exit_event, regs, rec)) ||
  581. hlist_empty(head)) {
  582. perf_swevent_put_recursion_context(rctx);
  583. return;
  584. }
  585. perf_trace_buf_submit(rec, size, rctx, sys_data->exit_event->event.type,
  586. 1, regs, head, NULL);
  587. }
  588. static int perf_sysexit_enable(struct trace_event_call *call)
  589. {
  590. int ret = 0;
  591. int num;
  592. num = ((struct syscall_metadata *)call->data)->syscall_nr;
  593. mutex_lock(&syscall_trace_lock);
  594. if (!sys_perf_refcount_exit)
  595. ret = register_trace_sys_exit(perf_syscall_exit, NULL);
  596. if (ret) {
  597. pr_info("event trace: Could not activate syscall exit trace point");
  598. } else {
  599. set_bit(num, enabled_perf_exit_syscalls);
  600. sys_perf_refcount_exit++;
  601. }
  602. mutex_unlock(&syscall_trace_lock);
  603. return ret;
  604. }
  605. static void perf_sysexit_disable(struct trace_event_call *call)
  606. {
  607. int num;
  608. num = ((struct syscall_metadata *)call->data)->syscall_nr;
  609. mutex_lock(&syscall_trace_lock);
  610. sys_perf_refcount_exit--;
  611. clear_bit(num, enabled_perf_exit_syscalls);
  612. if (!sys_perf_refcount_exit)
  613. unregister_trace_sys_exit(perf_syscall_exit, NULL);
  614. mutex_unlock(&syscall_trace_lock);
  615. }
  616. #endif /* CONFIG_PERF_EVENTS */
  617. static int syscall_enter_register(struct trace_event_call *event,
  618. enum trace_reg type, void *data)
  619. {
  620. struct trace_event_file *file = data;
  621. switch (type) {
  622. case TRACE_REG_REGISTER:
  623. return reg_event_syscall_enter(file, event);
  624. case TRACE_REG_UNREGISTER:
  625. unreg_event_syscall_enter(file, event);
  626. return 0;
  627. #ifdef CONFIG_PERF_EVENTS
  628. case TRACE_REG_PERF_REGISTER:
  629. return perf_sysenter_enable(event);
  630. case TRACE_REG_PERF_UNREGISTER:
  631. perf_sysenter_disable(event);
  632. return 0;
  633. case TRACE_REG_PERF_OPEN:
  634. case TRACE_REG_PERF_CLOSE:
  635. case TRACE_REG_PERF_ADD:
  636. case TRACE_REG_PERF_DEL:
  637. return 0;
  638. #endif
  639. }
  640. return 0;
  641. }
  642. static int syscall_exit_register(struct trace_event_call *event,
  643. enum trace_reg type, void *data)
  644. {
  645. struct trace_event_file *file = data;
  646. switch (type) {
  647. case TRACE_REG_REGISTER:
  648. return reg_event_syscall_exit(file, event);
  649. case TRACE_REG_UNREGISTER:
  650. unreg_event_syscall_exit(file, event);
  651. return 0;
  652. #ifdef CONFIG_PERF_EVENTS
  653. case TRACE_REG_PERF_REGISTER:
  654. return perf_sysexit_enable(event);
  655. case TRACE_REG_PERF_UNREGISTER:
  656. perf_sysexit_disable(event);
  657. return 0;
  658. case TRACE_REG_PERF_OPEN:
  659. case TRACE_REG_PERF_CLOSE:
  660. case TRACE_REG_PERF_ADD:
  661. case TRACE_REG_PERF_DEL:
  662. return 0;
  663. #endif
  664. }
  665. return 0;
  666. }