1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Common functionality for RV32 and RV64 BPF JIT compilers
4 *
5 * Copyright (c) 2019 Björn Töpel <bjorn.topel@gmail.com>
6 *
7 */
8
9 #include <linux/bpf.h>
10 #include <linux/filter.h>
11 #include <linux/memory.h>
12 #include <asm/patch.h>
13 #include <asm/cfi.h>
14 #include "bpf_jit.h"
15
16 /* Number of iterations to try until offsets converge. */
17 #define NR_JIT_ITERATIONS 32
18
19 static int build_body(struct rv_jit_context *ctx, bool extra_pass, int *offset)
20 {
21 const struct bpf_prog *prog = ctx->prog;
22 int i;
23
24 for (i = 0; i < prog->len; i++) {
25 const struct bpf_insn *insn = &prog->insnsi[i];
26 int ret;
27
28 ret = bpf_jit_emit_insn(insn, ctx, extra_pass);
29 /* BPF_LD | BPF_IMM | BPF_DW: skip the next instruction. */
30 if (ret > 0)
31 i++;
32 if (offset)
33 offset[i] = ctx->ninsns;
34 if (ret < 0)
35 return ret;
36 }
37 return 0;
38 }
39
40 bool bpf_jit_needs_zext(void)
41 {
42 return true;
43 }
44
45 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
46 {
47 unsigned int prog_size = 0, extable_size = 0;
48 bool tmp_blinded = false, extra_pass = false;
49 struct bpf_prog *tmp, *orig_prog = prog;
50 int pass = 0, prev_ninsns = 0, i;
51 struct rv_jit_data *jit_data;
52 struct rv_jit_context *ctx;
53
54 if (!prog->jit_requested)
55 return orig_prog;
56
57 tmp = bpf_jit_blind_constants(prog);
58 if (IS_ERR(tmp))
59 return orig_prog;
60 if (tmp != prog) {
61 tmp_blinded = true;
62 prog = tmp;
63 }
64
65 jit_data = prog->aux->jit_data;
66 if (!jit_data) {
67 jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
68 if (!jit_data) {
69 prog = orig_prog;
70 goto out;
71 }
72 prog->aux->jit_data = jit_data;
73 }
74
75 ctx = &jit_data->ctx;
76
77 if (ctx->offset) {
78 extra_pass = true;
79 prog_size = sizeof(*ctx->insns) * ctx->ninsns;
80 goto skip_init_ctx;
81 }
82
83 ctx->arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena);
84 ctx->user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena);
85 ctx->prog = prog;
86 ctx->offset = kcalloc(prog->len, sizeof(int), GFP_KERNEL);
87 if (!ctx->offset) {
88 prog = orig_prog;
89 goto out_offset;
90 }
91
92 if (build_body(ctx, extra_pass, NULL)) {
93 prog = orig_prog;
94 goto out_offset;
95 }
96
97 for (i = 0; i < prog->len; i++) {
98 prev_ninsns += 32;
99 ctx->offset[i] = prev_ninsns;
100 }
101
102 for (i = 0; i < NR_JIT_ITERATIONS; i++) {
103 pass++;
104 ctx->ninsns = 0;
105
106 bpf_jit_build_prologue(ctx, bpf_is_subprog(prog));
107 ctx->prologue_len = ctx->ninsns;
108
109 if (build_body(ctx, extra_pass, ctx->offset)) {
110 prog = orig_prog;
111 goto out_offset;
112 }
113
114 ctx->epilogue_offset = ctx->ninsns;
115 bpf_jit_build_epilogue(ctx);
116
117 if (ctx->ninsns == prev_ninsns) {
118 if (jit_data->header)
119 break;
120 /* obtain the actual image size */
121 extable_size = prog->aux->num_exentries *
122 sizeof(struct exception_table_entry);
123 prog_size = sizeof(*ctx->insns) * ctx->ninsns;
124
125 jit_data->ro_header =
126 bpf_jit_binary_pack_alloc(prog_size + extable_size,
127 &jit_data->ro_image, sizeof(u32),
128 &jit_data->header, &jit_data->image,
129 bpf_fill_ill_insns);
130 if (!jit_data->ro_header) {
131 prog = orig_prog;
132 goto out_offset;
133 }
134
135 /*
136 * Use the image(RW) for writing the JITed instructions. But also save
137 * the ro_image(RX) for calculating the offsets in the image. The RW
138 * image will be later copied to the RX image from where the program
139 * will run. The bpf_jit_binary_pack_finalize() will do this copy in the
140 * final step.
141 */
142 ctx->ro_insns = (u16 *)jit_data->ro_image;
143 ctx->insns = (u16 *)jit_data->image;
144 /*
145 * Now, when the image is allocated, the image can
146 * potentially shrink more (auipc/jalr -> jal).
147 */
148 }
149 prev_ninsns = ctx->ninsns;
150 }
151
152 if (i == NR_JIT_ITERATIONS) {
153 pr_err("bpf-jit: image did not converge in <%d passes!\n", i);
154 prog = orig_prog;
155 goto out_free_hdr;
156 }
157
158 if (extable_size)
159 prog->aux->extable = (void *)ctx->ro_insns + prog_size;
160
161 skip_init_ctx:
162 pass++;
163 ctx->ninsns = 0;
164 ctx->nexentries = 0;
165
166 bpf_jit_build_prologue(ctx, bpf_is_subprog(prog));
167 if (build_body(ctx, extra_pass, NULL)) {
168 prog = orig_prog;
169 goto out_free_hdr;
170 }
171 bpf_jit_build_epilogue(ctx);
172
173 if (bpf_jit_enable > 1)
174 bpf_jit_dump(prog->len, prog_size, pass, ctx->insns);
175
176 prog->bpf_func = (void *)ctx->ro_insns + cfi_get_offset();
177 prog->jited = 1;
178 prog->jited_len = prog_size - cfi_get_offset();
179
180 if (!prog->is_func || extra_pass) {
181 if (WARN_ON(bpf_jit_binary_pack_finalize(jit_data->ro_header, jit_data->header))) {
182 /* ro_header has been freed */
183 jit_data->ro_header = NULL;
184 prog = orig_prog;
185 goto out_offset;
186 }
187 /*
188 * The instructions have now been copied to the ROX region from
189 * where they will execute.
190 * Write any modified data cache blocks out to memory and
191 * invalidate the corresponding blocks in the instruction cache.
192 */
193 bpf_flush_icache(jit_data->ro_header, ctx->ro_insns + ctx->ninsns);
194 for (i = 0; i < prog->len; i++)
195 ctx->offset[i] = ninsns_rvoff(ctx->offset[i]);
196 bpf_prog_fill_jited_linfo(prog, ctx->offset);
197 out_offset:
198 kfree(ctx->offset);
199 kfree(jit_data);
200 prog->aux->jit_data = NULL;
201 }
202 out:
203
204 if (tmp_blinded)
205 bpf_jit_prog_release_other(prog, prog == orig_prog ?
206 tmp : orig_prog);
207 return prog;
208
209 out_free_hdr:
210 if (jit_data->header) {
211 bpf_arch_text_copy(&jit_data->ro_header->size, &jit_data->header->size,
212 sizeof(jit_data->header->size));
213 bpf_jit_binary_pack_free(jit_data->ro_header, jit_data->header);
214 }
215 goto out_offset;
216 }
217
218 u64 bpf_jit_alloc_exec_limit(void)
219 {
220 return BPF_JIT_REGION_SIZE;
221 }
222
223 void *bpf_arch_text_copy(void *dst, void *src, size_t len)
224 {
225 int ret;
226
227 mutex_lock(&text_mutex);
228 ret = patch_text_nosync(dst, src, len);
229 mutex_unlock(&text_mutex);
230
231 if (ret)
232 return ERR_PTR(-EINVAL);
233
234 return dst;
235 }
236
237 int bpf_arch_text_invalidate(void *dst, size_t len)
238 {
239 int ret;
240
241 mutex_lock(&text_mutex);
242 ret = patch_text_set_nosync(dst, 0, len);
243 mutex_unlock(&text_mutex);
244
245 return ret;
246 }
247
248 void bpf_jit_free(struct bpf_prog *prog)
249 {
250 if (prog->jited) {
251 struct rv_jit_data *jit_data = prog->aux->jit_data;
252 struct bpf_binary_header *hdr;
253
254 /*
255 * If we fail the final pass of JIT (from jit_subprogs),
256 * the program may not be finalized yet. Call finalize here
257 * before freeing it.
258 */
259 if (jit_data) {
260 bpf_jit_binary_pack_finalize(jit_data->ro_header, jit_data->header);
261 kfree(jit_data);
262 }
263 hdr = bpf_jit_binary_pack_hdr(prog);
264 bpf_jit_binary_pack_free(hdr, NULL);
265 WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(prog));
266 }
267
268 bpf_prog_unlock_free(prog);
269 }
270
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.