1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright(c) 2019 Intel Corporation. */
3
4 #include <linux/hash.h>
5 #include <linux/bpf.h>
6 #include <linux/filter.h>
7 #include <linux/static_call.h>
8
9 /* The BPF dispatcher is a multiway branch code generator. The
10 * dispatcher is a mechanism to avoid the performance penalty of an
11 * indirect call, which is expensive when retpolines are enabled. A
12 * dispatch client registers a BPF program into the dispatcher, and if
13 * there is available room in the dispatcher a direct call to the BPF
14 * program will be generated. All calls to the BPF programs called via
15 * the dispatcher will then be a direct call, instead of an
16 * indirect. The dispatcher hijacks a trampoline function it via the
17 * __fentry__ of the trampoline. The trampoline function has the
18 * following signature:
19 *
20 * unsigned int trampoline(const void *ctx, const struct bpf_insn *insnsi,
21 * unsigned int (*bpf_func)(const void *,
22 * const struct bpf_insn *));
23 */
24
25 static struct bpf_dispatcher_prog *bpf_dispatcher_find_prog(
26 struct bpf_dispatcher *d, struct bpf_prog *prog)
27 {
28 int i;
29
30 for (i = 0; i < BPF_DISPATCHER_MAX; i++) {
31 if (prog == d->progs[i].prog)
32 return &d->progs[i];
33 }
34 return NULL;
35 }
36
37 static struct bpf_dispatcher_prog *bpf_dispatcher_find_free(
38 struct bpf_dispatcher *d)
39 {
40 return bpf_dispatcher_find_prog(d, NULL);
41 }
42
43 static bool bpf_dispatcher_add_prog(struct bpf_dispatcher *d,
44 struct bpf_prog *prog)
45 {
46 struct bpf_dispatcher_prog *entry;
47
48 if (!prog)
49 return false;
50
51 entry = bpf_dispatcher_find_prog(d, prog);
52 if (entry) {
53 refcount_inc(&entry->users);
54 return false;
55 }
56
57 entry = bpf_dispatcher_find_free(d);
58 if (!entry)
59 return false;
60
61 bpf_prog_inc(prog);
62 entry->prog = prog;
63 refcount_set(&entry->users, 1);
64 d->num_progs++;
65 return true;
66 }
67
68 static bool bpf_dispatcher_remove_prog(struct bpf_dispatcher *d,
69 struct bpf_prog *prog)
70 {
71 struct bpf_dispatcher_prog *entry;
72
73 if (!prog)
74 return false;
75
76 entry = bpf_dispatcher_find_prog(d, prog);
77 if (!entry)
78 return false;
79
80 if (refcount_dec_and_test(&entry->users)) {
81 entry->prog = NULL;
82 bpf_prog_put(prog);
83 d->num_progs--;
84 return true;
85 }
86 return false;
87 }
88
89 int __weak arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_funcs)
90 {
91 return -ENOTSUPP;
92 }
93
94 static int bpf_dispatcher_prepare(struct bpf_dispatcher *d, void *image, void *buf)
95 {
96 s64 ips[BPF_DISPATCHER_MAX] = {}, *ipsp = &ips[0];
97 int i;
98
99 for (i = 0; i < BPF_DISPATCHER_MAX; i++) {
100 if (d->progs[i].prog)
101 *ipsp++ = (s64)(uintptr_t)d->progs[i].prog->bpf_func;
102 }
103 return arch_prepare_bpf_dispatcher(image, buf, &ips[0], d->num_progs);
104 }
105
106 static void bpf_dispatcher_update(struct bpf_dispatcher *d, int prev_num_progs)
107 {
108 void *new, *tmp;
109 u32 noff = 0;
110
111 if (prev_num_progs)
112 noff = d->image_off ^ (PAGE_SIZE / 2);
113
114 new = d->num_progs ? d->image + noff : NULL;
115 tmp = d->num_progs ? d->rw_image + noff : NULL;
116 if (new) {
117 /* Prepare the dispatcher in d->rw_image. Then use
118 * bpf_arch_text_copy to update d->image, which is RO+X.
119 */
120 if (bpf_dispatcher_prepare(d, new, tmp))
121 return;
122 if (IS_ERR(bpf_arch_text_copy(new, tmp, PAGE_SIZE / 2)))
123 return;
124 }
125
126 __BPF_DISPATCHER_UPDATE(d, new ?: (void *)&bpf_dispatcher_nop_func);
127
128 /* Make sure all the callers executing the previous/old half of the
129 * image leave it, so following update call can modify it safely.
130 */
131 synchronize_rcu();
132
133 if (new)
134 d->image_off = noff;
135 }
136
137 void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from,
138 struct bpf_prog *to)
139 {
140 bool changed = false;
141 int prev_num_progs;
142
143 if (from == to)
144 return;
145
146 mutex_lock(&d->mutex);
147 if (!d->image) {
148 d->image = bpf_prog_pack_alloc(PAGE_SIZE, bpf_jit_fill_hole_with_zero);
149 if (!d->image)
150 goto out;
151 d->rw_image = bpf_jit_alloc_exec(PAGE_SIZE);
152 if (!d->rw_image) {
153 bpf_prog_pack_free(d->image, PAGE_SIZE);
154 d->image = NULL;
155 goto out;
156 }
157 bpf_image_ksym_add(d->image, PAGE_SIZE, &d->ksym);
158 }
159
160 prev_num_progs = d->num_progs;
161 changed |= bpf_dispatcher_remove_prog(d, from);
162 changed |= bpf_dispatcher_add_prog(d, to);
163
164 if (!changed)
165 goto out;
166
167 bpf_dispatcher_update(d, prev_num_progs);
168 out:
169 mutex_unlock(&d->mutex);
170 }
171
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.