dispatcher.c 4.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /* Copyright(c) 2019 Intel Corporation. */
  3. #include <linux/hash.h>
  4. #include <linux/bpf.h>
  5. #include <linux/filter.h>
  6. #include <linux/static_call.h>
  7. /* The BPF dispatcher is a multiway branch code generator. The
  8. * dispatcher is a mechanism to avoid the performance penalty of an
  9. * indirect call, which is expensive when retpolines are enabled. A
  10. * dispatch client registers a BPF program into the dispatcher, and if
  11. * there is available room in the dispatcher a direct call to the BPF
  12. * program will be generated. All calls to the BPF programs called via
  13. * the dispatcher will then be a direct call, instead of an
  14. * indirect. The dispatcher hijacks a trampoline function it via the
  15. * __fentry__ of the trampoline. The trampoline function has the
  16. * following signature:
  17. *
  18. * unsigned int trampoline(const void *ctx, const struct bpf_insn *insnsi,
  19. * unsigned int (*bpf_func)(const void *,
  20. * const struct bpf_insn *));
  21. */
  22. static struct bpf_dispatcher_prog *bpf_dispatcher_find_prog(
  23. struct bpf_dispatcher *d, struct bpf_prog *prog)
  24. {
  25. int i;
  26. for (i = 0; i < BPF_DISPATCHER_MAX; i++) {
  27. if (prog == d->progs[i].prog)
  28. return &d->progs[i];
  29. }
  30. return NULL;
  31. }
  32. static struct bpf_dispatcher_prog *bpf_dispatcher_find_free(
  33. struct bpf_dispatcher *d)
  34. {
  35. return bpf_dispatcher_find_prog(d, NULL);
  36. }
  37. static bool bpf_dispatcher_add_prog(struct bpf_dispatcher *d,
  38. struct bpf_prog *prog)
  39. {
  40. struct bpf_dispatcher_prog *entry;
  41. if (!prog)
  42. return false;
  43. entry = bpf_dispatcher_find_prog(d, prog);
  44. if (entry) {
  45. refcount_inc(&entry->users);
  46. return false;
  47. }
  48. entry = bpf_dispatcher_find_free(d);
  49. if (!entry)
  50. return false;
  51. bpf_prog_inc(prog);
  52. entry->prog = prog;
  53. refcount_set(&entry->users, 1);
  54. d->num_progs++;
  55. return true;
  56. }
  57. static bool bpf_dispatcher_remove_prog(struct bpf_dispatcher *d,
  58. struct bpf_prog *prog)
  59. {
  60. struct bpf_dispatcher_prog *entry;
  61. if (!prog)
  62. return false;
  63. entry = bpf_dispatcher_find_prog(d, prog);
  64. if (!entry)
  65. return false;
  66. if (refcount_dec_and_test(&entry->users)) {
  67. entry->prog = NULL;
  68. bpf_prog_put(prog);
  69. d->num_progs--;
  70. return true;
  71. }
  72. return false;
  73. }
  74. int __weak arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_funcs)
  75. {
  76. return -ENOTSUPP;
  77. }
  78. static int bpf_dispatcher_prepare(struct bpf_dispatcher *d, void *image, void *buf)
  79. {
  80. s64 ips[BPF_DISPATCHER_MAX] = {}, *ipsp = &ips[0];
  81. int i;
  82. for (i = 0; i < BPF_DISPATCHER_MAX; i++) {
  83. if (d->progs[i].prog)
  84. *ipsp++ = (s64)(uintptr_t)d->progs[i].prog->bpf_func;
  85. }
  86. return arch_prepare_bpf_dispatcher(image, buf, &ips[0], d->num_progs);
  87. }
  88. static void bpf_dispatcher_update(struct bpf_dispatcher *d, int prev_num_progs)
  89. {
  90. void *new, *tmp;
  91. u32 noff = 0;
  92. if (prev_num_progs)
  93. noff = d->image_off ^ (PAGE_SIZE / 2);
  94. new = d->num_progs ? d->image + noff : NULL;
  95. tmp = d->num_progs ? d->rw_image + noff : NULL;
  96. if (new) {
  97. /* Prepare the dispatcher in d->rw_image. Then use
  98. * bpf_arch_text_copy to update d->image, which is RO+X.
  99. */
  100. if (bpf_dispatcher_prepare(d, new, tmp))
  101. return;
  102. if (IS_ERR(bpf_arch_text_copy(new, tmp, PAGE_SIZE / 2)))
  103. return;
  104. }
  105. __BPF_DISPATCHER_UPDATE(d, new ?: (void *)&bpf_dispatcher_nop_func);
  106. if (new)
  107. d->image_off = noff;
  108. }
  109. void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from,
  110. struct bpf_prog *to)
  111. {
  112. bool changed = false;
  113. int prev_num_progs;
  114. if (from == to)
  115. return;
  116. mutex_lock(&d->mutex);
  117. if (!d->image) {
  118. d->image = bpf_prog_pack_alloc(PAGE_SIZE, bpf_jit_fill_hole_with_zero);
  119. if (!d->image)
  120. goto out;
  121. d->rw_image = bpf_jit_alloc_exec(PAGE_SIZE);
  122. if (!d->rw_image) {
  123. u32 size = PAGE_SIZE;
  124. bpf_arch_text_copy(d->image, &size, sizeof(size));
  125. bpf_prog_pack_free((struct bpf_binary_header *)d->image);
  126. d->image = NULL;
  127. goto out;
  128. }
  129. bpf_image_ksym_add(d->image, &d->ksym);
  130. }
  131. prev_num_progs = d->num_progs;
  132. changed |= bpf_dispatcher_remove_prog(d, from);
  133. changed |= bpf_dispatcher_add_prog(d, to);
  134. if (!changed)
  135. goto out;
  136. bpf_dispatcher_update(d, prev_num_progs);
  137. out:
  138. mutex_unlock(&d->mutex);
  139. }