1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123 |
- // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
- /* Copyright (c) 2021 Facebook */
- #include <stdio.h>
- #include <stdlib.h>
- #include <string.h>
- #include <errno.h>
- #include <linux/filter.h>
- #include <sys/param.h>
- #include "btf.h"
- #include "bpf.h"
- #include "libbpf.h"
- #include "libbpf_internal.h"
- #include "hashmap.h"
- #include "bpf_gen_internal.h"
- #include "skel_internal.h"
- #include <asm/byteorder.h>
- #define MAX_USED_MAPS 64
- #define MAX_USED_PROGS 32
- #define MAX_KFUNC_DESCS 256
- #define MAX_FD_ARRAY_SZ (MAX_USED_MAPS + MAX_KFUNC_DESCS)
- /* The following structure describes the stack layout of the loader program.
- * In addition R6 contains the pointer to context.
- * R7 contains the result of the last sys_bpf command (typically error or FD).
- * R9 contains the result of the last sys_close command.
- *
- * Naming convention:
- * ctx - bpf program context
- * stack - bpf program stack
- * blob - bpf_attr-s, strings, insns, map data.
- * All the bytes that loader prog will use for read/write.
- */
- struct loader_stack {
- __u32 btf_fd;
- __u32 inner_map_fd;
- __u32 prog_fd[MAX_USED_PROGS];
- };
- #define stack_off(field) \
- (__s16)(-sizeof(struct loader_stack) + offsetof(struct loader_stack, field))
- #define attr_field(attr, field) (attr + offsetof(union bpf_attr, field))
- static int blob_fd_array_off(struct bpf_gen *gen, int index)
- {
- return gen->fd_array + index * sizeof(int);
- }
- static int realloc_insn_buf(struct bpf_gen *gen, __u32 size)
- {
- size_t off = gen->insn_cur - gen->insn_start;
- void *insn_start;
- if (gen->error)
- return gen->error;
- if (size > INT32_MAX || off + size > INT32_MAX) {
- gen->error = -ERANGE;
- return -ERANGE;
- }
- insn_start = realloc(gen->insn_start, off + size);
- if (!insn_start) {
- gen->error = -ENOMEM;
- free(gen->insn_start);
- gen->insn_start = NULL;
- return -ENOMEM;
- }
- gen->insn_start = insn_start;
- gen->insn_cur = insn_start + off;
- return 0;
- }
- static int realloc_data_buf(struct bpf_gen *gen, __u32 size)
- {
- size_t off = gen->data_cur - gen->data_start;
- void *data_start;
- if (gen->error)
- return gen->error;
- if (size > INT32_MAX || off + size > INT32_MAX) {
- gen->error = -ERANGE;
- return -ERANGE;
- }
- data_start = realloc(gen->data_start, off + size);
- if (!data_start) {
- gen->error = -ENOMEM;
- free(gen->data_start);
- gen->data_start = NULL;
- return -ENOMEM;
- }
- gen->data_start = data_start;
- gen->data_cur = data_start + off;
- return 0;
- }
- static void emit(struct bpf_gen *gen, struct bpf_insn insn)
- {
- if (realloc_insn_buf(gen, sizeof(insn)))
- return;
- memcpy(gen->insn_cur, &insn, sizeof(insn));
- gen->insn_cur += sizeof(insn);
- }
- static void emit2(struct bpf_gen *gen, struct bpf_insn insn1, struct bpf_insn insn2)
- {
- emit(gen, insn1);
- emit(gen, insn2);
- }
- static int add_data(struct bpf_gen *gen, const void *data, __u32 size);
- static void emit_sys_close_blob(struct bpf_gen *gen, int blob_off);
- void bpf_gen__init(struct bpf_gen *gen, int log_level, int nr_progs, int nr_maps)
- {
- size_t stack_sz = sizeof(struct loader_stack), nr_progs_sz;
- int i;
- gen->fd_array = add_data(gen, NULL, MAX_FD_ARRAY_SZ * sizeof(int));
- gen->log_level = log_level;
- /* save ctx pointer into R6 */
- emit(gen, BPF_MOV64_REG(BPF_REG_6, BPF_REG_1));
- /* bzero stack */
- emit(gen, BPF_MOV64_REG(BPF_REG_1, BPF_REG_10));
- emit(gen, BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -stack_sz));
- emit(gen, BPF_MOV64_IMM(BPF_REG_2, stack_sz));
- emit(gen, BPF_MOV64_IMM(BPF_REG_3, 0));
- emit(gen, BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel));
- /* amount of stack actually used, only used to calculate iterations, not stack offset */
- nr_progs_sz = offsetof(struct loader_stack, prog_fd[nr_progs]);
- /* jump over cleanup code */
- emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0,
- /* size of cleanup code below (including map fd cleanup) */
- (nr_progs_sz / 4) * 3 + 2 +
- /* 6 insns for emit_sys_close_blob,
- * 6 insns for debug_regs in emit_sys_close_blob
- */
- nr_maps * (6 + (gen->log_level ? 6 : 0))));
- /* remember the label where all error branches will jump to */
- gen->cleanup_label = gen->insn_cur - gen->insn_start;
- /* emit cleanup code: close all temp FDs */
- for (i = 0; i < nr_progs_sz; i += 4) {
- emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_10, -stack_sz + i));
- emit(gen, BPF_JMP_IMM(BPF_JSLE, BPF_REG_1, 0, 1));
- emit(gen, BPF_EMIT_CALL(BPF_FUNC_sys_close));
- }
- for (i = 0; i < nr_maps; i++)
- emit_sys_close_blob(gen, blob_fd_array_off(gen, i));
- /* R7 contains the error code from sys_bpf. Copy it into R0 and exit. */
- emit(gen, BPF_MOV64_REG(BPF_REG_0, BPF_REG_7));
- emit(gen, BPF_EXIT_INSN());
- }
- static int add_data(struct bpf_gen *gen, const void *data, __u32 size)
- {
- __u32 size8 = roundup(size, 8);
- __u64 zero = 0;
- void *prev;
- if (realloc_data_buf(gen, size8))
- return 0;
- prev = gen->data_cur;
- if (data) {
- memcpy(gen->data_cur, data, size);
- memcpy(gen->data_cur + size, &zero, size8 - size);
- } else {
- memset(gen->data_cur, 0, size8);
- }
- gen->data_cur += size8;
- return prev - gen->data_start;
- }
- /* Get index for map_fd/btf_fd slot in reserved fd_array, or in data relative
- * to start of fd_array. Caller can decide if it is usable or not.
- */
- static int add_map_fd(struct bpf_gen *gen)
- {
- if (gen->nr_maps == MAX_USED_MAPS) {
- pr_warn("Total maps exceeds %d\n", MAX_USED_MAPS);
- gen->error = -E2BIG;
- return 0;
- }
- return gen->nr_maps++;
- }
- static int add_kfunc_btf_fd(struct bpf_gen *gen)
- {
- int cur;
- if (gen->nr_fd_array == MAX_KFUNC_DESCS) {
- cur = add_data(gen, NULL, sizeof(int));
- return (cur - gen->fd_array) / sizeof(int);
- }
- return MAX_USED_MAPS + gen->nr_fd_array++;
- }
- static int insn_bytes_to_bpf_size(__u32 sz)
- {
- switch (sz) {
- case 8: return BPF_DW;
- case 4: return BPF_W;
- case 2: return BPF_H;
- case 1: return BPF_B;
- default: return -1;
- }
- }
- /* *(u64 *)(blob + off) = (u64)(void *)(blob + data) */
- static void emit_rel_store(struct bpf_gen *gen, int off, int data)
- {
- emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE,
- 0, 0, 0, data));
- emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
- 0, 0, 0, off));
- emit(gen, BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0, 0));
- }
- static void move_blob2blob(struct bpf_gen *gen, int off, int size, int blob_off)
- {
- emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_2, BPF_PSEUDO_MAP_IDX_VALUE,
- 0, 0, 0, blob_off));
- emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_2, 0));
- emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
- 0, 0, 0, off));
- emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_1, BPF_REG_0, 0));
- }
- static void move_blob2ctx(struct bpf_gen *gen, int ctx_off, int size, int blob_off)
- {
- emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
- 0, 0, 0, blob_off));
- emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_1, 0));
- emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_6, BPF_REG_0, ctx_off));
- }
- static void move_ctx2blob(struct bpf_gen *gen, int off, int size, int ctx_off,
- bool check_non_zero)
- {
- emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_6, ctx_off));
- if (check_non_zero)
- /* If value in ctx is zero don't update the blob.
- * For example: when ctx->map.max_entries == 0, keep default max_entries from bpf.c
- */
- emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 3));
- emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
- 0, 0, 0, off));
- emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_1, BPF_REG_0, 0));
- }
- static void move_stack2blob(struct bpf_gen *gen, int off, int size, int stack_off)
- {
- emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_10, stack_off));
- emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
- 0, 0, 0, off));
- emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_1, BPF_REG_0, 0));
- }
- static void move_stack2ctx(struct bpf_gen *gen, int ctx_off, int size, int stack_off)
- {
- emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_10, stack_off));
- emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_6, BPF_REG_0, ctx_off));
- }
- static void emit_sys_bpf(struct bpf_gen *gen, int cmd, int attr, int attr_size)
- {
- emit(gen, BPF_MOV64_IMM(BPF_REG_1, cmd));
- emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_2, BPF_PSEUDO_MAP_IDX_VALUE,
- 0, 0, 0, attr));
- emit(gen, BPF_MOV64_IMM(BPF_REG_3, attr_size));
- emit(gen, BPF_EMIT_CALL(BPF_FUNC_sys_bpf));
- /* remember the result in R7 */
- emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0));
- }
- static bool is_simm16(__s64 value)
- {
- return value == (__s64)(__s16)value;
- }
- static void emit_check_err(struct bpf_gen *gen)
- {
- __s64 off = -(gen->insn_cur - gen->insn_start - gen->cleanup_label) / 8 - 1;
- /* R7 contains result of last sys_bpf command.
- * if (R7 < 0) goto cleanup;
- */
- if (is_simm16(off)) {
- emit(gen, BPF_JMP_IMM(BPF_JSLT, BPF_REG_7, 0, off));
- } else {
- gen->error = -ERANGE;
- emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, -1));
- }
- }
- /* reg1 and reg2 should not be R1 - R5. They can be R0, R6 - R10 */
- static void emit_debug(struct bpf_gen *gen, int reg1, int reg2,
- const char *fmt, va_list args)
- {
- char buf[1024];
- int addr, len, ret;
- if (!gen->log_level)
- return;
- ret = vsnprintf(buf, sizeof(buf), fmt, args);
- if (ret < 1024 - 7 && reg1 >= 0 && reg2 < 0)
- /* The special case to accommodate common debug_ret():
- * to avoid specifying BPF_REG_7 and adding " r=%%d" to
- * prints explicitly.
- */
- strcat(buf, " r=%d");
- len = strlen(buf) + 1;
- addr = add_data(gen, buf, len);
- emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
- 0, 0, 0, addr));
- emit(gen, BPF_MOV64_IMM(BPF_REG_2, len));
- if (reg1 >= 0)
- emit(gen, BPF_MOV64_REG(BPF_REG_3, reg1));
- if (reg2 >= 0)
- emit(gen, BPF_MOV64_REG(BPF_REG_4, reg2));
- emit(gen, BPF_EMIT_CALL(BPF_FUNC_trace_printk));
- }
- static void debug_regs(struct bpf_gen *gen, int reg1, int reg2, const char *fmt, ...)
- {
- va_list args;
- va_start(args, fmt);
- emit_debug(gen, reg1, reg2, fmt, args);
- va_end(args);
- }
- static void debug_ret(struct bpf_gen *gen, const char *fmt, ...)
- {
- va_list args;
- va_start(args, fmt);
- emit_debug(gen, BPF_REG_7, -1, fmt, args);
- va_end(args);
- }
- static void __emit_sys_close(struct bpf_gen *gen)
- {
- emit(gen, BPF_JMP_IMM(BPF_JSLE, BPF_REG_1, 0,
- /* 2 is the number of the following insns
- * * 6 is additional insns in debug_regs
- */
- 2 + (gen->log_level ? 6 : 0)));
- emit(gen, BPF_MOV64_REG(BPF_REG_9, BPF_REG_1));
- emit(gen, BPF_EMIT_CALL(BPF_FUNC_sys_close));
- debug_regs(gen, BPF_REG_9, BPF_REG_0, "close(%%d) = %%d");
- }
- static void emit_sys_close_stack(struct bpf_gen *gen, int stack_off)
- {
- emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_10, stack_off));
- __emit_sys_close(gen);
- }
- static void emit_sys_close_blob(struct bpf_gen *gen, int blob_off)
- {
- emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE,
- 0, 0, 0, blob_off));
- emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_0, 0));
- __emit_sys_close(gen);
- }
- int bpf_gen__finish(struct bpf_gen *gen, int nr_progs, int nr_maps)
- {
- int i;
- if (nr_progs < gen->nr_progs || nr_maps != gen->nr_maps) {
- pr_warn("nr_progs %d/%d nr_maps %d/%d mismatch\n",
- nr_progs, gen->nr_progs, nr_maps, gen->nr_maps);
- gen->error = -EFAULT;
- return gen->error;
- }
- emit_sys_close_stack(gen, stack_off(btf_fd));
- for (i = 0; i < gen->nr_progs; i++)
- move_stack2ctx(gen,
- sizeof(struct bpf_loader_ctx) +
- sizeof(struct bpf_map_desc) * gen->nr_maps +
- sizeof(struct bpf_prog_desc) * i +
- offsetof(struct bpf_prog_desc, prog_fd), 4,
- stack_off(prog_fd[i]));
- for (i = 0; i < gen->nr_maps; i++)
- move_blob2ctx(gen,
- sizeof(struct bpf_loader_ctx) +
- sizeof(struct bpf_map_desc) * i +
- offsetof(struct bpf_map_desc, map_fd), 4,
- blob_fd_array_off(gen, i));
- emit(gen, BPF_MOV64_IMM(BPF_REG_0, 0));
- emit(gen, BPF_EXIT_INSN());
- pr_debug("gen: finish %d\n", gen->error);
- if (!gen->error) {
- struct gen_loader_opts *opts = gen->opts;
- opts->insns = gen->insn_start;
- opts->insns_sz = gen->insn_cur - gen->insn_start;
- opts->data = gen->data_start;
- opts->data_sz = gen->data_cur - gen->data_start;
- }
- return gen->error;
- }
- void bpf_gen__free(struct bpf_gen *gen)
- {
- if (!gen)
- return;
- free(gen->data_start);
- free(gen->insn_start);
- free(gen);
- }
- void bpf_gen__load_btf(struct bpf_gen *gen, const void *btf_raw_data,
- __u32 btf_raw_size)
- {
- int attr_size = offsetofend(union bpf_attr, btf_log_level);
- int btf_data, btf_load_attr;
- union bpf_attr attr;
- memset(&attr, 0, attr_size);
- pr_debug("gen: load_btf: size %d\n", btf_raw_size);
- btf_data = add_data(gen, btf_raw_data, btf_raw_size);
- attr.btf_size = btf_raw_size;
- btf_load_attr = add_data(gen, &attr, attr_size);
- /* populate union bpf_attr with user provided log details */
- move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_level), 4,
- offsetof(struct bpf_loader_ctx, log_level), false);
- move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_size), 4,
- offsetof(struct bpf_loader_ctx, log_size), false);
- move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_buf), 8,
- offsetof(struct bpf_loader_ctx, log_buf), false);
- /* populate union bpf_attr with a pointer to the BTF data */
- emit_rel_store(gen, attr_field(btf_load_attr, btf), btf_data);
- /* emit BTF_LOAD command */
- emit_sys_bpf(gen, BPF_BTF_LOAD, btf_load_attr, attr_size);
- debug_ret(gen, "btf_load size %d", btf_raw_size);
- emit_check_err(gen);
- /* remember btf_fd in the stack, if successful */
- emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7, stack_off(btf_fd)));
- }
- void bpf_gen__map_create(struct bpf_gen *gen,
- enum bpf_map_type map_type,
- const char *map_name,
- __u32 key_size, __u32 value_size, __u32 max_entries,
- struct bpf_map_create_opts *map_attr, int map_idx)
- {
- int attr_size = offsetofend(union bpf_attr, map_extra);
- bool close_inner_map_fd = false;
- int map_create_attr, idx;
- union bpf_attr attr;
- memset(&attr, 0, attr_size);
- attr.map_type = map_type;
- attr.key_size = key_size;
- attr.value_size = value_size;
- attr.map_flags = map_attr->map_flags;
- attr.map_extra = map_attr->map_extra;
- if (map_name)
- libbpf_strlcpy(attr.map_name, map_name, sizeof(attr.map_name));
- attr.numa_node = map_attr->numa_node;
- attr.map_ifindex = map_attr->map_ifindex;
- attr.max_entries = max_entries;
- attr.btf_key_type_id = map_attr->btf_key_type_id;
- attr.btf_value_type_id = map_attr->btf_value_type_id;
- pr_debug("gen: map_create: %s idx %d type %d value_type_id %d\n",
- attr.map_name, map_idx, map_type, attr.btf_value_type_id);
- map_create_attr = add_data(gen, &attr, attr_size);
- if (attr.btf_value_type_id)
- /* populate union bpf_attr with btf_fd saved in the stack earlier */
- move_stack2blob(gen, attr_field(map_create_attr, btf_fd), 4,
- stack_off(btf_fd));
- switch (attr.map_type) {
- case BPF_MAP_TYPE_ARRAY_OF_MAPS:
- case BPF_MAP_TYPE_HASH_OF_MAPS:
- move_stack2blob(gen, attr_field(map_create_attr, inner_map_fd), 4,
- stack_off(inner_map_fd));
- close_inner_map_fd = true;
- break;
- default:
- break;
- }
- /* conditionally update max_entries */
- if (map_idx >= 0)
- move_ctx2blob(gen, attr_field(map_create_attr, max_entries), 4,
- sizeof(struct bpf_loader_ctx) +
- sizeof(struct bpf_map_desc) * map_idx +
- offsetof(struct bpf_map_desc, max_entries),
- true /* check that max_entries != 0 */);
- /* emit MAP_CREATE command */
- emit_sys_bpf(gen, BPF_MAP_CREATE, map_create_attr, attr_size);
- debug_ret(gen, "map_create %s idx %d type %d value_size %d value_btf_id %d",
- attr.map_name, map_idx, map_type, value_size,
- attr.btf_value_type_id);
- emit_check_err(gen);
- /* remember map_fd in the stack, if successful */
- if (map_idx < 0) {
- /* This bpf_gen__map_create() function is called with map_idx >= 0
- * for all maps that libbpf loading logic tracks.
- * It's called with -1 to create an inner map.
- */
- emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7,
- stack_off(inner_map_fd)));
- } else if (map_idx != gen->nr_maps) {
- gen->error = -EDOM; /* internal bug */
- return;
- } else {
- /* add_map_fd does gen->nr_maps++ */
- idx = add_map_fd(gen);
- emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
- 0, 0, 0, blob_fd_array_off(gen, idx)));
- emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_7, 0));
- }
- if (close_inner_map_fd)
- emit_sys_close_stack(gen, stack_off(inner_map_fd));
- }
- void bpf_gen__record_attach_target(struct bpf_gen *gen, const char *attach_name,
- enum bpf_attach_type type)
- {
- const char *prefix;
- int kind, ret;
- btf_get_kernel_prefix_kind(type, &prefix, &kind);
- gen->attach_kind = kind;
- ret = snprintf(gen->attach_target, sizeof(gen->attach_target), "%s%s",
- prefix, attach_name);
- if (ret >= sizeof(gen->attach_target))
- gen->error = -ENOSPC;
- }
- static void emit_find_attach_target(struct bpf_gen *gen)
- {
- int name, len = strlen(gen->attach_target) + 1;
- pr_debug("gen: find_attach_tgt %s %d\n", gen->attach_target, gen->attach_kind);
- name = add_data(gen, gen->attach_target, len);
- emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
- 0, 0, 0, name));
- emit(gen, BPF_MOV64_IMM(BPF_REG_2, len));
- emit(gen, BPF_MOV64_IMM(BPF_REG_3, gen->attach_kind));
- emit(gen, BPF_MOV64_IMM(BPF_REG_4, 0));
- emit(gen, BPF_EMIT_CALL(BPF_FUNC_btf_find_by_name_kind));
- emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0));
- debug_ret(gen, "find_by_name_kind(%s,%d)",
- gen->attach_target, gen->attach_kind);
- emit_check_err(gen);
- /* if successful, btf_id is in lower 32-bit of R7 and
- * btf_obj_fd is in upper 32-bit
- */
- }
- void bpf_gen__record_extern(struct bpf_gen *gen, const char *name, bool is_weak,
- bool is_typeless, int kind, int insn_idx)
- {
- struct ksym_relo_desc *relo;
- relo = libbpf_reallocarray(gen->relos, gen->relo_cnt + 1, sizeof(*relo));
- if (!relo) {
- gen->error = -ENOMEM;
- return;
- }
- gen->relos = relo;
- relo += gen->relo_cnt;
- relo->name = name;
- relo->is_weak = is_weak;
- relo->is_typeless = is_typeless;
- relo->kind = kind;
- relo->insn_idx = insn_idx;
- gen->relo_cnt++;
- }
- /* returns existing ksym_desc with ref incremented, or inserts a new one */
- static struct ksym_desc *get_ksym_desc(struct bpf_gen *gen, struct ksym_relo_desc *relo)
- {
- struct ksym_desc *kdesc;
- int i;
- for (i = 0; i < gen->nr_ksyms; i++) {
- if (!strcmp(gen->ksyms[i].name, relo->name)) {
- gen->ksyms[i].ref++;
- return &gen->ksyms[i];
- }
- }
- kdesc = libbpf_reallocarray(gen->ksyms, gen->nr_ksyms + 1, sizeof(*kdesc));
- if (!kdesc) {
- gen->error = -ENOMEM;
- return NULL;
- }
- gen->ksyms = kdesc;
- kdesc = &gen->ksyms[gen->nr_ksyms++];
- kdesc->name = relo->name;
- kdesc->kind = relo->kind;
- kdesc->ref = 1;
- kdesc->off = 0;
- kdesc->insn = 0;
- return kdesc;
- }
- /* Overwrites BPF_REG_{0, 1, 2, 3, 4, 7}
- * Returns result in BPF_REG_7
- */
- static void emit_bpf_find_by_name_kind(struct bpf_gen *gen, struct ksym_relo_desc *relo)
- {
- int name_off, len = strlen(relo->name) + 1;
- name_off = add_data(gen, relo->name, len);
- emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
- 0, 0, 0, name_off));
- emit(gen, BPF_MOV64_IMM(BPF_REG_2, len));
- emit(gen, BPF_MOV64_IMM(BPF_REG_3, relo->kind));
- emit(gen, BPF_MOV64_IMM(BPF_REG_4, 0));
- emit(gen, BPF_EMIT_CALL(BPF_FUNC_btf_find_by_name_kind));
- emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0));
- debug_ret(gen, "find_by_name_kind(%s,%d)", relo->name, relo->kind);
- }
- /* Overwrites BPF_REG_{0, 1, 2, 3, 4, 7}
- * Returns result in BPF_REG_7
- * Returns u64 symbol addr in BPF_REG_9
- */
- static void emit_bpf_kallsyms_lookup_name(struct bpf_gen *gen, struct ksym_relo_desc *relo)
- {
- int name_off, len = strlen(relo->name) + 1, res_off;
- name_off = add_data(gen, relo->name, len);
- res_off = add_data(gen, NULL, 8); /* res is u64 */
- emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
- 0, 0, 0, name_off));
- emit(gen, BPF_MOV64_IMM(BPF_REG_2, len));
- emit(gen, BPF_MOV64_IMM(BPF_REG_3, 0));
- emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_4, BPF_PSEUDO_MAP_IDX_VALUE,
- 0, 0, 0, res_off));
- emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_4));
- emit(gen, BPF_EMIT_CALL(BPF_FUNC_kallsyms_lookup_name));
- emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_9, BPF_REG_7, 0));
- emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0));
- debug_ret(gen, "kallsyms_lookup_name(%s,%d)", relo->name, relo->kind);
- }
- /* Expects:
- * BPF_REG_8 - pointer to instruction
- *
- * We need to reuse BTF fd for same symbol otherwise each relocation takes a new
- * index, while kernel limits total kfunc BTFs to 256. For duplicate symbols,
- * this would mean a new BTF fd index for each entry. By pairing symbol name
- * with index, we get the insn->imm, insn->off pairing that kernel uses for
- * kfunc_tab, which becomes the effective limit even though all of them may
- * share same index in fd_array (such that kfunc_btf_tab has 1 element).
- */
- static void emit_relo_kfunc_btf(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insn)
- {
- struct ksym_desc *kdesc;
- int btf_fd_idx;
- kdesc = get_ksym_desc(gen, relo);
- if (!kdesc)
- return;
- /* try to copy from existing bpf_insn */
- if (kdesc->ref > 1) {
- move_blob2blob(gen, insn + offsetof(struct bpf_insn, imm), 4,
- kdesc->insn + offsetof(struct bpf_insn, imm));
- move_blob2blob(gen, insn + offsetof(struct bpf_insn, off), 2,
- kdesc->insn + offsetof(struct bpf_insn, off));
- goto log;
- }
- /* remember insn offset, so we can copy BTF ID and FD later */
- kdesc->insn = insn;
- emit_bpf_find_by_name_kind(gen, relo);
- if (!relo->is_weak)
- emit_check_err(gen);
- /* get index in fd_array to store BTF FD at */
- btf_fd_idx = add_kfunc_btf_fd(gen);
- if (btf_fd_idx > INT16_MAX) {
- pr_warn("BTF fd off %d for kfunc %s exceeds INT16_MAX, cannot process relocation\n",
- btf_fd_idx, relo->name);
- gen->error = -E2BIG;
- return;
- }
- kdesc->off = btf_fd_idx;
- /* jump to success case */
- emit(gen, BPF_JMP_IMM(BPF_JSGE, BPF_REG_7, 0, 3));
- /* set value for imm, off as 0 */
- emit(gen, BPF_ST_MEM(BPF_W, BPF_REG_8, offsetof(struct bpf_insn, imm), 0));
- emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), 0));
- /* skip success case for ret < 0 */
- emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 10));
- /* store btf_id into insn[insn_idx].imm */
- emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7, offsetof(struct bpf_insn, imm)));
- /* obtain fd in BPF_REG_9 */
- emit(gen, BPF_MOV64_REG(BPF_REG_9, BPF_REG_7));
- emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_9, 32));
- /* jump to fd_array store if fd denotes module BTF */
- emit(gen, BPF_JMP_IMM(BPF_JNE, BPF_REG_9, 0, 2));
- /* set the default value for off */
- emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), 0));
- /* skip BTF fd store for vmlinux BTF */
- emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 4));
- /* load fd_array slot pointer */
- emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE,
- 0, 0, 0, blob_fd_array_off(gen, btf_fd_idx)));
- /* store BTF fd in slot */
- emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_9, 0));
- /* store index into insn[insn_idx].off */
- emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), btf_fd_idx));
- log:
- if (!gen->log_level)
- return;
- emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_8,
- offsetof(struct bpf_insn, imm)));
- emit(gen, BPF_LDX_MEM(BPF_H, BPF_REG_9, BPF_REG_8,
- offsetof(struct bpf_insn, off)));
- debug_regs(gen, BPF_REG_7, BPF_REG_9, " func (%s:count=%d): imm: %%d, off: %%d",
- relo->name, kdesc->ref);
- emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE,
- 0, 0, 0, blob_fd_array_off(gen, kdesc->off)));
- emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_9, BPF_REG_0, 0));
- debug_regs(gen, BPF_REG_9, -1, " func (%s:count=%d): btf_fd",
- relo->name, kdesc->ref);
- }
- static void emit_ksym_relo_log(struct bpf_gen *gen, struct ksym_relo_desc *relo,
- int ref)
- {
- if (!gen->log_level)
- return;
- emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_8,
- offsetof(struct bpf_insn, imm)));
- emit(gen, BPF_LDX_MEM(BPF_H, BPF_REG_9, BPF_REG_8, sizeof(struct bpf_insn) +
- offsetof(struct bpf_insn, imm)));
- debug_regs(gen, BPF_REG_7, BPF_REG_9, " var t=%d w=%d (%s:count=%d): imm[0]: %%d, imm[1]: %%d",
- relo->is_typeless, relo->is_weak, relo->name, ref);
- emit(gen, BPF_LDX_MEM(BPF_B, BPF_REG_9, BPF_REG_8, offsetofend(struct bpf_insn, code)));
- debug_regs(gen, BPF_REG_9, -1, " var t=%d w=%d (%s:count=%d): insn.reg",
- relo->is_typeless, relo->is_weak, relo->name, ref);
- }
- /* Expects:
- * BPF_REG_8 - pointer to instruction
- */
- static void emit_relo_ksym_typeless(struct bpf_gen *gen,
- struct ksym_relo_desc *relo, int insn)
- {
- struct ksym_desc *kdesc;
- kdesc = get_ksym_desc(gen, relo);
- if (!kdesc)
- return;
- /* try to copy from existing ldimm64 insn */
- if (kdesc->ref > 1) {
- move_blob2blob(gen, insn + offsetof(struct bpf_insn, imm), 4,
- kdesc->insn + offsetof(struct bpf_insn, imm));
- move_blob2blob(gen, insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm), 4,
- kdesc->insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm));
- goto log;
- }
- /* remember insn offset, so we can copy ksym addr later */
- kdesc->insn = insn;
- /* skip typeless ksym_desc in fd closing loop in cleanup_relos */
- kdesc->typeless = true;
- emit_bpf_kallsyms_lookup_name(gen, relo);
- emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_7, -ENOENT, 1));
- emit_check_err(gen);
- /* store lower half of addr into insn[insn_idx].imm */
- emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_9, offsetof(struct bpf_insn, imm)));
- /* store upper half of addr into insn[insn_idx + 1].imm */
- emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_9, 32));
- emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_9,
- sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm)));
- log:
- emit_ksym_relo_log(gen, relo, kdesc->ref);
- }
- static __u32 src_reg_mask(void)
- {
- #if defined(__LITTLE_ENDIAN_BITFIELD)
- return 0x0f; /* src_reg,dst_reg,... */
- #elif defined(__BIG_ENDIAN_BITFIELD)
- return 0xf0; /* dst_reg,src_reg,... */
- #else
- #error "Unsupported bit endianness, cannot proceed"
- #endif
- }
- /* Expects:
- * BPF_REG_8 - pointer to instruction
- */
- static void emit_relo_ksym_btf(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insn)
- {
- struct ksym_desc *kdesc;
- __u32 reg_mask;
- kdesc = get_ksym_desc(gen, relo);
- if (!kdesc)
- return;
- /* try to copy from existing ldimm64 insn */
- if (kdesc->ref > 1) {
- move_blob2blob(gen, insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm), 4,
- kdesc->insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm));
- move_blob2blob(gen, insn + offsetof(struct bpf_insn, imm), 4,
- kdesc->insn + offsetof(struct bpf_insn, imm));
- /* jump over src_reg adjustment if imm (btf_id) is not 0, reuse BPF_REG_0 from move_blob2blob
- * If btf_id is zero, clear BPF_PSEUDO_BTF_ID flag in src_reg of ld_imm64 insn
- */
- emit(gen, BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 3));
- goto clear_src_reg;
- }
- /* remember insn offset, so we can copy BTF ID and FD later */
- kdesc->insn = insn;
- emit_bpf_find_by_name_kind(gen, relo);
- if (!relo->is_weak)
- emit_check_err(gen);
- /* jump to success case */
- emit(gen, BPF_JMP_IMM(BPF_JSGE, BPF_REG_7, 0, 3));
- /* set values for insn[insn_idx].imm, insn[insn_idx + 1].imm as 0 */
- emit(gen, BPF_ST_MEM(BPF_W, BPF_REG_8, offsetof(struct bpf_insn, imm), 0));
- emit(gen, BPF_ST_MEM(BPF_W, BPF_REG_8, sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm), 0));
- /* skip success case for ret < 0 */
- emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 4));
- /* store btf_id into insn[insn_idx].imm */
- emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7, offsetof(struct bpf_insn, imm)));
- /* store btf_obj_fd into insn[insn_idx + 1].imm */
- emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_7, 32));
- emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7,
- sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm)));
- /* skip src_reg adjustment */
- emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 3));
- clear_src_reg:
- /* clear bpf_object__relocate_data's src_reg assignment, otherwise we get a verifier failure */
- reg_mask = src_reg_mask();
- emit(gen, BPF_LDX_MEM(BPF_B, BPF_REG_9, BPF_REG_8, offsetofend(struct bpf_insn, code)));
- emit(gen, BPF_ALU32_IMM(BPF_AND, BPF_REG_9, reg_mask));
- emit(gen, BPF_STX_MEM(BPF_B, BPF_REG_8, BPF_REG_9, offsetofend(struct bpf_insn, code)));
- emit_ksym_relo_log(gen, relo, kdesc->ref);
- }
- void bpf_gen__record_relo_core(struct bpf_gen *gen,
- const struct bpf_core_relo *core_relo)
- {
- struct bpf_core_relo *relos;
- relos = libbpf_reallocarray(gen->core_relos, gen->core_relo_cnt + 1, sizeof(*relos));
- if (!relos) {
- gen->error = -ENOMEM;
- return;
- }
- gen->core_relos = relos;
- relos += gen->core_relo_cnt;
- memcpy(relos, core_relo, sizeof(*relos));
- gen->core_relo_cnt++;
- }
- static void emit_relo(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insns)
- {
- int insn;
- pr_debug("gen: emit_relo (%d): %s at %d\n", relo->kind, relo->name, relo->insn_idx);
- insn = insns + sizeof(struct bpf_insn) * relo->insn_idx;
- emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_8, BPF_PSEUDO_MAP_IDX_VALUE, 0, 0, 0, insn));
- switch (relo->kind) {
- case BTF_KIND_VAR:
- if (relo->is_typeless)
- emit_relo_ksym_typeless(gen, relo, insn);
- else
- emit_relo_ksym_btf(gen, relo, insn);
- break;
- case BTF_KIND_FUNC:
- emit_relo_kfunc_btf(gen, relo, insn);
- break;
- default:
- pr_warn("Unknown relocation kind '%d'\n", relo->kind);
- gen->error = -EDOM;
- return;
- }
- }
- static void emit_relos(struct bpf_gen *gen, int insns)
- {
- int i;
- for (i = 0; i < gen->relo_cnt; i++)
- emit_relo(gen, gen->relos + i, insns);
- }
- static void cleanup_core_relo(struct bpf_gen *gen)
- {
- if (!gen->core_relo_cnt)
- return;
- free(gen->core_relos);
- gen->core_relo_cnt = 0;
- gen->core_relos = NULL;
- }
- static void cleanup_relos(struct bpf_gen *gen, int insns)
- {
- int i, insn;
- for (i = 0; i < gen->nr_ksyms; i++) {
- /* only close fds for typed ksyms and kfuncs */
- if (gen->ksyms[i].kind == BTF_KIND_VAR && !gen->ksyms[i].typeless) {
- /* close fd recorded in insn[insn_idx + 1].imm */
- insn = gen->ksyms[i].insn;
- insn += sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm);
- emit_sys_close_blob(gen, insn);
- } else if (gen->ksyms[i].kind == BTF_KIND_FUNC) {
- emit_sys_close_blob(gen, blob_fd_array_off(gen, gen->ksyms[i].off));
- if (gen->ksyms[i].off < MAX_FD_ARRAY_SZ)
- gen->nr_fd_array--;
- }
- }
- if (gen->nr_ksyms) {
- free(gen->ksyms);
- gen->nr_ksyms = 0;
- gen->ksyms = NULL;
- }
- if (gen->relo_cnt) {
- free(gen->relos);
- gen->relo_cnt = 0;
- gen->relos = NULL;
- }
- cleanup_core_relo(gen);
- }
- void bpf_gen__prog_load(struct bpf_gen *gen,
- enum bpf_prog_type prog_type, const char *prog_name,
- const char *license, struct bpf_insn *insns, size_t insn_cnt,
- struct bpf_prog_load_opts *load_attr, int prog_idx)
- {
- int prog_load_attr, license_off, insns_off, func_info, line_info, core_relos;
- int attr_size = offsetofend(union bpf_attr, core_relo_rec_size);
- union bpf_attr attr;
- memset(&attr, 0, attr_size);
- pr_debug("gen: prog_load: type %d insns_cnt %zd progi_idx %d\n",
- prog_type, insn_cnt, prog_idx);
- /* add license string to blob of bytes */
- license_off = add_data(gen, license, strlen(license) + 1);
- /* add insns to blob of bytes */
- insns_off = add_data(gen, insns, insn_cnt * sizeof(struct bpf_insn));
- attr.prog_type = prog_type;
- attr.expected_attach_type = load_attr->expected_attach_type;
- attr.attach_btf_id = load_attr->attach_btf_id;
- attr.prog_ifindex = load_attr->prog_ifindex;
- attr.kern_version = 0;
- attr.insn_cnt = (__u32)insn_cnt;
- attr.prog_flags = load_attr->prog_flags;
- attr.func_info_rec_size = load_attr->func_info_rec_size;
- attr.func_info_cnt = load_attr->func_info_cnt;
- func_info = add_data(gen, load_attr->func_info,
- attr.func_info_cnt * attr.func_info_rec_size);
- attr.line_info_rec_size = load_attr->line_info_rec_size;
- attr.line_info_cnt = load_attr->line_info_cnt;
- line_info = add_data(gen, load_attr->line_info,
- attr.line_info_cnt * attr.line_info_rec_size);
- attr.core_relo_rec_size = sizeof(struct bpf_core_relo);
- attr.core_relo_cnt = gen->core_relo_cnt;
- core_relos = add_data(gen, gen->core_relos,
- attr.core_relo_cnt * attr.core_relo_rec_size);
- libbpf_strlcpy(attr.prog_name, prog_name, sizeof(attr.prog_name));
- prog_load_attr = add_data(gen, &attr, attr_size);
- /* populate union bpf_attr with a pointer to license */
- emit_rel_store(gen, attr_field(prog_load_attr, license), license_off);
- /* populate union bpf_attr with a pointer to instructions */
- emit_rel_store(gen, attr_field(prog_load_attr, insns), insns_off);
- /* populate union bpf_attr with a pointer to func_info */
- emit_rel_store(gen, attr_field(prog_load_attr, func_info), func_info);
- /* populate union bpf_attr with a pointer to line_info */
- emit_rel_store(gen, attr_field(prog_load_attr, line_info), line_info);
- /* populate union bpf_attr with a pointer to core_relos */
- emit_rel_store(gen, attr_field(prog_load_attr, core_relos), core_relos);
- /* populate union bpf_attr fd_array with a pointer to data where map_fds are saved */
- emit_rel_store(gen, attr_field(prog_load_attr, fd_array), gen->fd_array);
- /* populate union bpf_attr with user provided log details */
- move_ctx2blob(gen, attr_field(prog_load_attr, log_level), 4,
- offsetof(struct bpf_loader_ctx, log_level), false);
- move_ctx2blob(gen, attr_field(prog_load_attr, log_size), 4,
- offsetof(struct bpf_loader_ctx, log_size), false);
- move_ctx2blob(gen, attr_field(prog_load_attr, log_buf), 8,
- offsetof(struct bpf_loader_ctx, log_buf), false);
- /* populate union bpf_attr with btf_fd saved in the stack earlier */
- move_stack2blob(gen, attr_field(prog_load_attr, prog_btf_fd), 4,
- stack_off(btf_fd));
- if (gen->attach_kind) {
- emit_find_attach_target(gen);
- /* populate union bpf_attr with btf_id and btf_obj_fd found by helper */
- emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE,
- 0, 0, 0, prog_load_attr));
- emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_7,
- offsetof(union bpf_attr, attach_btf_id)));
- emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_7, 32));
- emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_7,
- offsetof(union bpf_attr, attach_btf_obj_fd)));
- }
- emit_relos(gen, insns_off);
- /* emit PROG_LOAD command */
- emit_sys_bpf(gen, BPF_PROG_LOAD, prog_load_attr, attr_size);
- debug_ret(gen, "prog_load %s insn_cnt %d", attr.prog_name, attr.insn_cnt);
- /* successful or not, close btf module FDs used in extern ksyms and attach_btf_obj_fd */
- cleanup_relos(gen, insns_off);
- if (gen->attach_kind) {
- emit_sys_close_blob(gen,
- attr_field(prog_load_attr, attach_btf_obj_fd));
- gen->attach_kind = 0;
- }
- emit_check_err(gen);
- /* remember prog_fd in the stack, if successful */
- emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7,
- stack_off(prog_fd[gen->nr_progs])));
- gen->nr_progs++;
- }
- void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue,
- __u32 value_size)
- {
- int attr_size = offsetofend(union bpf_attr, flags);
- int map_update_attr, value, key;
- union bpf_attr attr;
- int zero = 0;
- memset(&attr, 0, attr_size);
- pr_debug("gen: map_update_elem: idx %d\n", map_idx);
- value = add_data(gen, pvalue, value_size);
- key = add_data(gen, &zero, sizeof(zero));
- /* if (map_desc[map_idx].initial_value) {
- * if (ctx->flags & BPF_SKEL_KERNEL)
- * bpf_probe_read_kernel(value, value_size, initial_value);
- * else
- * bpf_copy_from_user(value, value_size, initial_value);
- * }
- */
- emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_6,
- sizeof(struct bpf_loader_ctx) +
- sizeof(struct bpf_map_desc) * map_idx +
- offsetof(struct bpf_map_desc, initial_value)));
- emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_3, 0, 8));
- emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
- 0, 0, 0, value));
- emit(gen, BPF_MOV64_IMM(BPF_REG_2, value_size));
- emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_6,
- offsetof(struct bpf_loader_ctx, flags)));
- emit(gen, BPF_JMP_IMM(BPF_JSET, BPF_REG_0, BPF_SKEL_KERNEL, 2));
- emit(gen, BPF_EMIT_CALL(BPF_FUNC_copy_from_user));
- emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 1));
- emit(gen, BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel));
- map_update_attr = add_data(gen, &attr, attr_size);
- move_blob2blob(gen, attr_field(map_update_attr, map_fd), 4,
- blob_fd_array_off(gen, map_idx));
- emit_rel_store(gen, attr_field(map_update_attr, key), key);
- emit_rel_store(gen, attr_field(map_update_attr, value), value);
- /* emit MAP_UPDATE_ELEM command */
- emit_sys_bpf(gen, BPF_MAP_UPDATE_ELEM, map_update_attr, attr_size);
- debug_ret(gen, "update_elem idx %d value_size %d", map_idx, value_size);
- emit_check_err(gen);
- }
- void bpf_gen__populate_outer_map(struct bpf_gen *gen, int outer_map_idx, int slot,
- int inner_map_idx)
- {
- int attr_size = offsetofend(union bpf_attr, flags);
- int map_update_attr, key;
- union bpf_attr attr;
- memset(&attr, 0, attr_size);
- pr_debug("gen: populate_outer_map: outer %d key %d inner %d\n",
- outer_map_idx, slot, inner_map_idx);
- key = add_data(gen, &slot, sizeof(slot));
- map_update_attr = add_data(gen, &attr, attr_size);
- move_blob2blob(gen, attr_field(map_update_attr, map_fd), 4,
- blob_fd_array_off(gen, outer_map_idx));
- emit_rel_store(gen, attr_field(map_update_attr, key), key);
- emit_rel_store(gen, attr_field(map_update_attr, value),
- blob_fd_array_off(gen, inner_map_idx));
- /* emit MAP_UPDATE_ELEM command */
- emit_sys_bpf(gen, BPF_MAP_UPDATE_ELEM, map_update_attr, attr_size);
- debug_ret(gen, "populate_outer_map outer %d key %d inner %d",
- outer_map_idx, slot, inner_map_idx);
- emit_check_err(gen);
- }
- void bpf_gen__map_freeze(struct bpf_gen *gen, int map_idx)
- {
- int attr_size = offsetofend(union bpf_attr, map_fd);
- int map_freeze_attr;
- union bpf_attr attr;
- memset(&attr, 0, attr_size);
- pr_debug("gen: map_freeze: idx %d\n", map_idx);
- map_freeze_attr = add_data(gen, &attr, attr_size);
- move_blob2blob(gen, attr_field(map_freeze_attr, map_fd), 4,
- blob_fd_array_off(gen, map_idx));
- /* emit MAP_FREEZE command */
- emit_sys_bpf(gen, BPF_MAP_FREEZE, map_freeze_attr, attr_size);
- debug_ret(gen, "map_freeze");
- emit_check_err(gen);
- }
|