usdt.bpf.h 8.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247
  1. /* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
  2. /* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */
  3. #ifndef __USDT_BPF_H__
  4. #define __USDT_BPF_H__
  5. #include <linux/errno.h>
  6. #include <bpf/bpf_helpers.h>
  7. #include <bpf/bpf_tracing.h>
  8. /* Below types and maps are internal implementation details of libbpf's USDT
  9. * support and are subjects to change. Also, bpf_usdt_xxx() API helpers should
  10. * be considered an unstable API as well and might be adjusted based on user
  11. * feedback from using libbpf's USDT support in production.
  12. */
  13. /* User can override BPF_USDT_MAX_SPEC_CNT to change default size of internal
  14. * map that keeps track of USDT argument specifications. This might be
  15. * necessary if there are a lot of USDT attachments.
  16. */
  17. #ifndef BPF_USDT_MAX_SPEC_CNT
  18. #define BPF_USDT_MAX_SPEC_CNT 256
  19. #endif
  20. /* User can override BPF_USDT_MAX_IP_CNT to change default size of internal
  21. * map that keeps track of IP (memory address) mapping to USDT argument
  22. * specification.
  23. * Note, if kernel supports BPF cookies, this map is not used and could be
  24. * resized all the way to 1 to save a bit of memory.
  25. */
  26. #ifndef BPF_USDT_MAX_IP_CNT
  27. #define BPF_USDT_MAX_IP_CNT (4 * BPF_USDT_MAX_SPEC_CNT)
  28. #endif
  29. enum __bpf_usdt_arg_type {
  30. BPF_USDT_ARG_CONST,
  31. BPF_USDT_ARG_REG,
  32. BPF_USDT_ARG_REG_DEREF,
  33. };
  34. struct __bpf_usdt_arg_spec {
  35. /* u64 scalar interpreted depending on arg_type, see below */
  36. __u64 val_off;
  37. /* arg location case, see bpf_udst_arg() for details */
  38. enum __bpf_usdt_arg_type arg_type;
  39. /* offset of referenced register within struct pt_regs */
  40. short reg_off;
  41. /* whether arg should be interpreted as signed value */
  42. bool arg_signed;
  43. /* number of bits that need to be cleared and, optionally,
  44. * sign-extended to cast arguments that are 1, 2, or 4 bytes
  45. * long into final 8-byte u64/s64 value returned to user
  46. */
  47. char arg_bitshift;
  48. };
  49. /* should match USDT_MAX_ARG_CNT in usdt.c exactly */
  50. #define BPF_USDT_MAX_ARG_CNT 12
  51. struct __bpf_usdt_spec {
  52. struct __bpf_usdt_arg_spec args[BPF_USDT_MAX_ARG_CNT];
  53. __u64 usdt_cookie;
  54. short arg_cnt;
  55. };
  56. struct {
  57. __uint(type, BPF_MAP_TYPE_ARRAY);
  58. __uint(max_entries, BPF_USDT_MAX_SPEC_CNT);
  59. __type(key, int);
  60. __type(value, struct __bpf_usdt_spec);
  61. } __bpf_usdt_specs SEC(".maps") __weak;
  62. struct {
  63. __uint(type, BPF_MAP_TYPE_HASH);
  64. __uint(max_entries, BPF_USDT_MAX_IP_CNT);
  65. __type(key, long);
  66. __type(value, __u32);
  67. } __bpf_usdt_ip_to_spec_id SEC(".maps") __weak;
  68. extern const _Bool LINUX_HAS_BPF_COOKIE __kconfig;
  69. static __always_inline
  70. int __bpf_usdt_spec_id(struct pt_regs *ctx)
  71. {
  72. if (!LINUX_HAS_BPF_COOKIE) {
  73. long ip = PT_REGS_IP(ctx);
  74. int *spec_id_ptr;
  75. spec_id_ptr = bpf_map_lookup_elem(&__bpf_usdt_ip_to_spec_id, &ip);
  76. return spec_id_ptr ? *spec_id_ptr : -ESRCH;
  77. }
  78. return bpf_get_attach_cookie(ctx);
  79. }
  80. /* Return number of USDT arguments defined for currently traced USDT. */
  81. __weak __hidden
  82. int bpf_usdt_arg_cnt(struct pt_regs *ctx)
  83. {
  84. struct __bpf_usdt_spec *spec;
  85. int spec_id;
  86. spec_id = __bpf_usdt_spec_id(ctx);
  87. if (spec_id < 0)
  88. return -ESRCH;
  89. spec = bpf_map_lookup_elem(&__bpf_usdt_specs, &spec_id);
  90. if (!spec)
  91. return -ESRCH;
  92. return spec->arg_cnt;
  93. }
  94. /* Fetch USDT argument #*arg_num* (zero-indexed) and put its value into *res.
  95. * Returns 0 on success; negative error, otherwise.
  96. * On error *res is guaranteed to be set to zero.
  97. */
  98. __weak __hidden
  99. int bpf_usdt_arg(struct pt_regs *ctx, __u64 arg_num, long *res)
  100. {
  101. struct __bpf_usdt_spec *spec;
  102. struct __bpf_usdt_arg_spec *arg_spec;
  103. unsigned long val;
  104. int err, spec_id;
  105. *res = 0;
  106. spec_id = __bpf_usdt_spec_id(ctx);
  107. if (spec_id < 0)
  108. return -ESRCH;
  109. spec = bpf_map_lookup_elem(&__bpf_usdt_specs, &spec_id);
  110. if (!spec)
  111. return -ESRCH;
  112. if (arg_num >= BPF_USDT_MAX_ARG_CNT || arg_num >= spec->arg_cnt)
  113. return -ENOENT;
  114. arg_spec = &spec->args[arg_num];
  115. switch (arg_spec->arg_type) {
  116. case BPF_USDT_ARG_CONST:
  117. /* Arg is just a constant ("-4@$-9" in USDT arg spec).
  118. * value is recorded in arg_spec->val_off directly.
  119. */
  120. val = arg_spec->val_off;
  121. break;
  122. case BPF_USDT_ARG_REG:
  123. /* Arg is in a register (e.g, "8@%rax" in USDT arg spec),
  124. * so we read the contents of that register directly from
  125. * struct pt_regs. To keep things simple user-space parts
  126. * record offsetof(struct pt_regs, <regname>) in arg_spec->reg_off.
  127. */
  128. err = bpf_probe_read_kernel(&val, sizeof(val), (void *)ctx + arg_spec->reg_off);
  129. if (err)
  130. return err;
  131. break;
  132. case BPF_USDT_ARG_REG_DEREF:
  133. /* Arg is in memory addressed by register, plus some offset
  134. * (e.g., "-4@-1204(%rbp)" in USDT arg spec). Register is
  135. * identified like with BPF_USDT_ARG_REG case, and the offset
  136. * is in arg_spec->val_off. We first fetch register contents
  137. * from pt_regs, then do another user-space probe read to
  138. * fetch argument value itself.
  139. */
  140. err = bpf_probe_read_kernel(&val, sizeof(val), (void *)ctx + arg_spec->reg_off);
  141. if (err)
  142. return err;
  143. err = bpf_probe_read_user(&val, sizeof(val), (void *)val + arg_spec->val_off);
  144. if (err)
  145. return err;
  146. #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
  147. val >>= arg_spec->arg_bitshift;
  148. #endif
  149. break;
  150. default:
  151. return -EINVAL;
  152. }
  153. /* cast arg from 1, 2, or 4 bytes to final 8 byte size clearing
  154. * necessary upper arg_bitshift bits, with sign extension if argument
  155. * is signed
  156. */
  157. val <<= arg_spec->arg_bitshift;
  158. if (arg_spec->arg_signed)
  159. val = ((long)val) >> arg_spec->arg_bitshift;
  160. else
  161. val = val >> arg_spec->arg_bitshift;
  162. *res = val;
  163. return 0;
  164. }
  165. /* Retrieve user-specified cookie value provided during attach as
  166. * bpf_usdt_opts.usdt_cookie. This serves the same purpose as BPF cookie
  167. * returned by bpf_get_attach_cookie(). Libbpf's support for USDT is itself
  168. * utilizing BPF cookies internally, so user can't use BPF cookie directly
  169. * for USDT programs and has to use bpf_usdt_cookie() API instead.
  170. */
  171. __weak __hidden
  172. long bpf_usdt_cookie(struct pt_regs *ctx)
  173. {
  174. struct __bpf_usdt_spec *spec;
  175. int spec_id;
  176. spec_id = __bpf_usdt_spec_id(ctx);
  177. if (spec_id < 0)
  178. return 0;
  179. spec = bpf_map_lookup_elem(&__bpf_usdt_specs, &spec_id);
  180. if (!spec)
  181. return 0;
  182. return spec->usdt_cookie;
  183. }
  184. /* we rely on ___bpf_apply() and ___bpf_narg() macros already defined in bpf_tracing.h */
  185. #define ___bpf_usdt_args0() ctx
  186. #define ___bpf_usdt_args1(x) ___bpf_usdt_args0(), ({ long _x; bpf_usdt_arg(ctx, 0, &_x); (void *)_x; })
  187. #define ___bpf_usdt_args2(x, args...) ___bpf_usdt_args1(args), ({ long _x; bpf_usdt_arg(ctx, 1, &_x); (void *)_x; })
  188. #define ___bpf_usdt_args3(x, args...) ___bpf_usdt_args2(args), ({ long _x; bpf_usdt_arg(ctx, 2, &_x); (void *)_x; })
  189. #define ___bpf_usdt_args4(x, args...) ___bpf_usdt_args3(args), ({ long _x; bpf_usdt_arg(ctx, 3, &_x); (void *)_x; })
  190. #define ___bpf_usdt_args5(x, args...) ___bpf_usdt_args4(args), ({ long _x; bpf_usdt_arg(ctx, 4, &_x); (void *)_x; })
  191. #define ___bpf_usdt_args6(x, args...) ___bpf_usdt_args5(args), ({ long _x; bpf_usdt_arg(ctx, 5, &_x); (void *)_x; })
  192. #define ___bpf_usdt_args7(x, args...) ___bpf_usdt_args6(args), ({ long _x; bpf_usdt_arg(ctx, 6, &_x); (void *)_x; })
  193. #define ___bpf_usdt_args8(x, args...) ___bpf_usdt_args7(args), ({ long _x; bpf_usdt_arg(ctx, 7, &_x); (void *)_x; })
  194. #define ___bpf_usdt_args9(x, args...) ___bpf_usdt_args8(args), ({ long _x; bpf_usdt_arg(ctx, 8, &_x); (void *)_x; })
  195. #define ___bpf_usdt_args10(x, args...) ___bpf_usdt_args9(args), ({ long _x; bpf_usdt_arg(ctx, 9, &_x); (void *)_x; })
  196. #define ___bpf_usdt_args11(x, args...) ___bpf_usdt_args10(args), ({ long _x; bpf_usdt_arg(ctx, 10, &_x); (void *)_x; })
  197. #define ___bpf_usdt_args12(x, args...) ___bpf_usdt_args11(args), ({ long _x; bpf_usdt_arg(ctx, 11, &_x); (void *)_x; })
  198. #define ___bpf_usdt_args(args...) ___bpf_apply(___bpf_usdt_args, ___bpf_narg(args))(args)
  199. /*
  200. * BPF_USDT serves the same purpose for USDT handlers as BPF_PROG for
  201. * tp_btf/fentry/fexit BPF programs and BPF_KPROBE for kprobes.
  202. * Original struct pt_regs * context is preserved as 'ctx' argument.
  203. */
  204. #define BPF_USDT(name, args...) \
  205. name(struct pt_regs *ctx); \
  206. static __always_inline typeof(name(0)) \
  207. ____##name(struct pt_regs *ctx, ##args); \
  208. typeof(name(0)) name(struct pt_regs *ctx) \
  209. { \
  210. _Pragma("GCC diagnostic push") \
  211. _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \
  212. return ____##name(___bpf_usdt_args(args)); \
  213. _Pragma("GCC diagnostic pop") \
  214. } \
  215. static __always_inline typeof(name(0)) \
  216. ____##name(struct pt_regs *ctx, ##args)
  217. #endif /* __USDT_BPF_H__ */