1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
2
3 /*
4 * Common eBPF ELF object loading operations.
5 *
6 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org>
7 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com>
8 * Copyright (C) 2015 Huawei Inc.
9 * Copyright (C) 2017 Nicira, Inc.
10 * Copyright (C) 2019 Isovalent, Inc.
11 */
12
13 #ifndef _GNU_SOURCE
14 #define _GNU_SOURCE
15 #endif
16 #include <stdlib.h>
17 #include <stdio.h>
18 #include <stdarg.h>
19 #include <libgen.h>
20 #include <inttypes.h>
21 #include <limits.h>
22 #include <string.h>
23 #include <unistd.h>
24 #include <endian.h>
25 #include <fcntl.h>
26 #include <errno.h>
27 #include <ctype.h>
28 #include <asm/unistd.h>
29 #include <linux/err.h>
30 #include <linux/kernel.h>
31 #include <linux/bpf.h>
32 #include <linux/btf.h>
33 #include <linux/filter.h>
34 #include <linux/limits.h>
35 #include <linux/perf_event.h>
36 #include <linux/bpf_perf_event.h>
37 #include <linux/ring_buffer.h>
38 #include <sys/epoll.h>
39 #include <sys/ioctl.h>
40 #include <sys/mman.h>
41 #include <sys/stat.h>
42 #include <sys/types.h>
43 #include <sys/vfs.h>
44 #include <sys/utsname.h>
45 #include <sys/resource.h>
46 #include <libelf.h>
47 #include <gelf.h>
48 #include <zlib.h>
49
50 #include "libbpf.h"
51 #include "bpf.h"
52 #include "btf.h"
53 #include "str_error.h"
54 #include "libbpf_internal.h"
55 #include "hashmap.h"
56 #include "bpf_gen_internal.h"
57 #include "zip.h"
58
59 #ifndef BPF_FS_MAGIC
60 #define BPF_FS_MAGIC 0xcafe4a11
61 #endif
62
63 #define BPF_FS_DEFAULT_PATH "/sys/fs/bpf"
64
65 #define BPF_INSN_SZ (sizeof(struct bpf_insn))
66
67 /* vsprintf() in __base_pr() uses nonliteral format string. It may break
68 * compilation if user enables corresponding warning. Disable it explicitly.
69 */
70 #pragma GCC diagnostic ignored "-Wformat-nonliteral"
71
72 #define __printf(a, b) __attribute__((format(printf, a, b)))
73
74 static struct bpf_map *bpf_object__add_map(struct bpf_object *obj);
75 static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog);
76 static int map_set_def_max_entries(struct bpf_map *map);
77
78 static const char * const attach_type_name[] = {
79 [BPF_CGROUP_INET_INGRESS] = "cgroup_inet_ingress",
80 [BPF_CGROUP_INET_EGRESS] = "cgroup_inet_egress",
81 [BPF_CGROUP_INET_SOCK_CREATE] = "cgroup_inet_sock_create",
82 [BPF_CGROUP_INET_SOCK_RELEASE] = "cgroup_inet_sock_release",
83 [BPF_CGROUP_SOCK_OPS] = "cgroup_sock_ops",
84 [BPF_CGROUP_DEVICE] = "cgroup_device",
85 [BPF_CGROUP_INET4_BIND] = "cgroup_inet4_bind",
86 [BPF_CGROUP_INET6_BIND] = "cgroup_inet6_bind",
87 [BPF_CGROUP_INET4_CONNECT] = "cgroup_inet4_connect",
88 [BPF_CGROUP_INET6_CONNECT] = "cgroup_inet6_connect",
89 [BPF_CGROUP_UNIX_CONNECT] = "cgroup_unix_connect",
90 [BPF_CGROUP_INET4_POST_BIND] = "cgroup_inet4_post_bind",
91 [BPF_CGROUP_INET6_POST_BIND] = "cgroup_inet6_post_bind",
92 [BPF_CGROUP_INET4_GETPEERNAME] = "cgroup_inet4_getpeername",
93 [BPF_CGROUP_INET6_GETPEERNAME] = "cgroup_inet6_getpeername",
94 [BPF_CGROUP_UNIX_GETPEERNAME] = "cgroup_unix_getpeername",
95 [BPF_CGROUP_INET4_GETSOCKNAME] = "cgroup_inet4_getsockname",
96 [BPF_CGROUP_INET6_GETSOCKNAME] = "cgroup_inet6_getsockname",
97 [BPF_CGROUP_UNIX_GETSOCKNAME] = "cgroup_unix_getsockname",
98 [BPF_CGROUP_UDP4_SENDMSG] = "cgroup_udp4_sendmsg",
99 [BPF_CGROUP_UDP6_SENDMSG] = "cgroup_udp6_sendmsg",
100 [BPF_CGROUP_UNIX_SENDMSG] = "cgroup_unix_sendmsg",
101 [BPF_CGROUP_SYSCTL] = "cgroup_sysctl",
102 [BPF_CGROUP_UDP4_RECVMSG] = "cgroup_udp4_recvmsg",
103 [BPF_CGROUP_UDP6_RECVMSG] = "cgroup_udp6_recvmsg",
104 [BPF_CGROUP_UNIX_RECVMSG] = "cgroup_unix_recvmsg",
105 [BPF_CGROUP_GETSOCKOPT] = "cgroup_getsockopt",
106 [BPF_CGROUP_SETSOCKOPT] = "cgroup_setsockopt",
107 [BPF_SK_SKB_STREAM_PARSER] = "sk_skb_stream_parser",
108 [BPF_SK_SKB_STREAM_VERDICT] = "sk_skb_stream_verdict",
109 [BPF_SK_SKB_VERDICT] = "sk_skb_verdict",
110 [BPF_SK_MSG_VERDICT] = "sk_msg_verdict",
111 [BPF_LIRC_MODE2] = "lirc_mode2",
112 [BPF_FLOW_DISSECTOR] = "flow_dissector",
113 [BPF_TRACE_RAW_TP] = "trace_raw_tp",
114 [BPF_TRACE_FENTRY] = "trace_fentry",
115 [BPF_TRACE_FEXIT] = "trace_fexit",
116 [BPF_MODIFY_RETURN] = "modify_return",
117 [BPF_LSM_MAC] = "lsm_mac",
118 [BPF_LSM_CGROUP] = "lsm_cgroup",
119 [BPF_SK_LOOKUP] = "sk_lookup",
120 [BPF_TRACE_ITER] = "trace_iter",
121 [BPF_XDP_DEVMAP] = "xdp_devmap",
122 [BPF_XDP_CPUMAP] = "xdp_cpumap",
123 [BPF_XDP] = "xdp",
124 [BPF_SK_REUSEPORT_SELECT] = "sk_reuseport_select",
125 [BPF_SK_REUSEPORT_SELECT_OR_MIGRATE] = "sk_reuseport_select_or_migrate",
126 [BPF_PERF_EVENT] = "perf_event",
127 [BPF_TRACE_KPROBE_MULTI] = "trace_kprobe_multi",
128 [BPF_STRUCT_OPS] = "struct_ops",
129 [BPF_NETFILTER] = "netfilter",
130 [BPF_TCX_INGRESS] = "tcx_ingress",
131 [BPF_TCX_EGRESS] = "tcx_egress",
132 [BPF_TRACE_UPROBE_MULTI] = "trace_uprobe_multi",
133 [BPF_NETKIT_PRIMARY] = "netkit_primary",
134 [BPF_NETKIT_PEER] = "netkit_peer",
135 [BPF_TRACE_KPROBE_SESSION] = "trace_kprobe_session",
136 };
137
138 static const char * const link_type_name[] = {
139 [BPF_LINK_TYPE_UNSPEC] = "unspec",
140 [BPF_LINK_TYPE_RAW_TRACEPOINT] = "raw_tracepoint",
141 [BPF_LINK_TYPE_TRACING] = "tracing",
142 [BPF_LINK_TYPE_CGROUP] = "cgroup",
143 [BPF_LINK_TYPE_ITER] = "iter",
144 [BPF_LINK_TYPE_NETNS] = "netns",
145 [BPF_LINK_TYPE_XDP] = "xdp",
146 [BPF_LINK_TYPE_PERF_EVENT] = "perf_event",
147 [BPF_LINK_TYPE_KPROBE_MULTI] = "kprobe_multi",
148 [BPF_LINK_TYPE_STRUCT_OPS] = "struct_ops",
149 [BPF_LINK_TYPE_NETFILTER] = "netfilter",
150 [BPF_LINK_TYPE_TCX] = "tcx",
151 [BPF_LINK_TYPE_UPROBE_MULTI] = "uprobe_multi",
152 [BPF_LINK_TYPE_NETKIT] = "netkit",
153 [BPF_LINK_TYPE_SOCKMAP] = "sockmap",
154 };
155
156 static const char * const map_type_name[] = {
157 [BPF_MAP_TYPE_UNSPEC] = "unspec",
158 [BPF_MAP_TYPE_HASH] = "hash",
159 [BPF_MAP_TYPE_ARRAY] = "array",
160 [BPF_MAP_TYPE_PROG_ARRAY] = "prog_array",
161 [BPF_MAP_TYPE_PERF_EVENT_ARRAY] = "perf_event_array",
162 [BPF_MAP_TYPE_PERCPU_HASH] = "percpu_hash",
163 [BPF_MAP_TYPE_PERCPU_ARRAY] = "percpu_array",
164 [BPF_MAP_TYPE_STACK_TRACE] = "stack_trace",
165 [BPF_MAP_TYPE_CGROUP_ARRAY] = "cgroup_array",
166 [BPF_MAP_TYPE_LRU_HASH] = "lru_hash",
167 [BPF_MAP_TYPE_LRU_PERCPU_HASH] = "lru_percpu_hash",
168 [BPF_MAP_TYPE_LPM_TRIE] = "lpm_trie",
169 [BPF_MAP_TYPE_ARRAY_OF_MAPS] = "array_of_maps",
170 [BPF_MAP_TYPE_HASH_OF_MAPS] = "hash_of_maps",
171 [BPF_MAP_TYPE_DEVMAP] = "devmap",
172 [BPF_MAP_TYPE_DEVMAP_HASH] = "devmap_hash",
173 [BPF_MAP_TYPE_SOCKMAP] = "sockmap",
174 [BPF_MAP_TYPE_CPUMAP] = "cpumap",
175 [BPF_MAP_TYPE_XSKMAP] = "xskmap",
176 [BPF_MAP_TYPE_SOCKHASH] = "sockhash",
177 [BPF_MAP_TYPE_CGROUP_STORAGE] = "cgroup_storage",
178 [BPF_MAP_TYPE_REUSEPORT_SOCKARRAY] = "reuseport_sockarray",
179 [BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE] = "percpu_cgroup_storage",
180 [BPF_MAP_TYPE_QUEUE] = "queue",
181 [BPF_MAP_TYPE_STACK] = "stack",
182 [BPF_MAP_TYPE_SK_STORAGE] = "sk_storage",
183 [BPF_MAP_TYPE_STRUCT_OPS] = "struct_ops",
184 [BPF_MAP_TYPE_RINGBUF] = "ringbuf",
185 [BPF_MAP_TYPE_INODE_STORAGE] = "inode_storage",
186 [BPF_MAP_TYPE_TASK_STORAGE] = "task_storage",
187 [BPF_MAP_TYPE_BLOOM_FILTER] = "bloom_filter",
188 [BPF_MAP_TYPE_USER_RINGBUF] = "user_ringbuf",
189 [BPF_MAP_TYPE_CGRP_STORAGE] = "cgrp_storage",
190 [BPF_MAP_TYPE_ARENA] = "arena",
191 };
192
193 static const char * const prog_type_name[] = {
194 [BPF_PROG_TYPE_UNSPEC] = "unspec",
195 [BPF_PROG_TYPE_SOCKET_FILTER] = "socket_filter",
196 [BPF_PROG_TYPE_KPROBE] = "kprobe",
197 [BPF_PROG_TYPE_SCHED_CLS] = "sched_cls",
198 [BPF_PROG_TYPE_SCHED_ACT] = "sched_act",
199 [BPF_PROG_TYPE_TRACEPOINT] = "tracepoint",
200 [BPF_PROG_TYPE_XDP] = "xdp",
201 [BPF_PROG_TYPE_PERF_EVENT] = "perf_event",
202 [BPF_PROG_TYPE_CGROUP_SKB] = "cgroup_skb",
203 [BPF_PROG_TYPE_CGROUP_SOCK] = "cgroup_sock",
204 [BPF_PROG_TYPE_LWT_IN] = "lwt_in",
205 [BPF_PROG_TYPE_LWT_OUT] = "lwt_out",
206 [BPF_PROG_TYPE_LWT_XMIT] = "lwt_xmit",
207 [BPF_PROG_TYPE_SOCK_OPS] = "sock_ops",
208 [BPF_PROG_TYPE_SK_SKB] = "sk_skb",
209 [BPF_PROG_TYPE_CGROUP_DEVICE] = "cgroup_device",
210 [BPF_PROG_TYPE_SK_MSG] = "sk_msg",
211 [BPF_PROG_TYPE_RAW_TRACEPOINT] = "raw_tracepoint",
212 [BPF_PROG_TYPE_CGROUP_SOCK_ADDR] = "cgroup_sock_addr",
213 [BPF_PROG_TYPE_LWT_SEG6LOCAL] = "lwt_seg6local",
214 [BPF_PROG_TYPE_LIRC_MODE2] = "lirc_mode2",
215 [BPF_PROG_TYPE_SK_REUSEPORT] = "sk_reuseport",
216 [BPF_PROG_TYPE_FLOW_DISSECTOR] = "flow_dissector",
217 [BPF_PROG_TYPE_CGROUP_SYSCTL] = "cgroup_sysctl",
218 [BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE] = "raw_tracepoint_writable",
219 [BPF_PROG_TYPE_CGROUP_SOCKOPT] = "cgroup_sockopt",
220 [BPF_PROG_TYPE_TRACING] = "tracing",
221 [BPF_PROG_TYPE_STRUCT_OPS] = "struct_ops",
222 [BPF_PROG_TYPE_EXT] = "ext",
223 [BPF_PROG_TYPE_LSM] = "lsm",
224 [BPF_PROG_TYPE_SK_LOOKUP] = "sk_lookup",
225 [BPF_PROG_TYPE_SYSCALL] = "syscall",
226 [BPF_PROG_TYPE_NETFILTER] = "netfilter",
227 };
228
229 static int __base_pr(enum libbpf_print_level level, const char *format,
230 va_list args)
231 {
232 const char *env_var = "LIBBPF_LOG_LEVEL";
233 static enum libbpf_print_level min_level = LIBBPF_INFO;
234 static bool initialized;
235
236 if (!initialized) {
237 char *verbosity;
238
239 initialized = true;
240 verbosity = getenv(env_var);
241 if (verbosity) {
242 if (strcasecmp(verbosity, "warn") == 0)
243 min_level = LIBBPF_WARN;
244 else if (strcasecmp(verbosity, "debug") == 0)
245 min_level = LIBBPF_DEBUG;
246 else if (strcasecmp(verbosity, "info") == 0)
247 min_level = LIBBPF_INFO;
248 else
249 fprintf(stderr, "libbpf: unrecognized '%s' envvar value: '%s', should be one of 'warn', 'debug', or 'info'.\n",
250 env_var, verbosity);
251 }
252 }
253
254 /* if too verbose, skip logging */
255 if (level > min_level)
256 return 0;
257
258 return vfprintf(stderr, format, args);
259 }
260
261 static libbpf_print_fn_t __libbpf_pr = __base_pr;
262
263 libbpf_print_fn_t libbpf_set_print(libbpf_print_fn_t fn)
264 {
265 libbpf_print_fn_t old_print_fn;
266
267 old_print_fn = __atomic_exchange_n(&__libbpf_pr, fn, __ATOMIC_RELAXED);
268
269 return old_print_fn;
270 }
271
272 __printf(2, 3)
273 void libbpf_print(enum libbpf_print_level level, const char *format, ...)
274 {
275 va_list args;
276 int old_errno;
277 libbpf_print_fn_t print_fn;
278
279 print_fn = __atomic_load_n(&__libbpf_pr, __ATOMIC_RELAXED);
280 if (!print_fn)
281 return;
282
283 old_errno = errno;
284
285 va_start(args, format);
286 __libbpf_pr(level, format, args);
287 va_end(args);
288
289 errno = old_errno;
290 }
291
292 static void pr_perm_msg(int err)
293 {
294 struct rlimit limit;
295 char buf[100];
296
297 if (err != -EPERM || geteuid() != 0)
298 return;
299
300 err = getrlimit(RLIMIT_MEMLOCK, &limit);
301 if (err)
302 return;
303
304 if (limit.rlim_cur == RLIM_INFINITY)
305 return;
306
307 if (limit.rlim_cur < 1024)
308 snprintf(buf, sizeof(buf), "%zu bytes", (size_t)limit.rlim_cur);
309 else if (limit.rlim_cur < 1024*1024)
310 snprintf(buf, sizeof(buf), "%.1f KiB", (double)limit.rlim_cur / 1024);
311 else
312 snprintf(buf, sizeof(buf), "%.1f MiB", (double)limit.rlim_cur / (1024*1024));
313
314 pr_warn("permission error while running as root; try raising 'ulimit -l'? current value: %s\n",
315 buf);
316 }
317
318 #define STRERR_BUFSIZE 128
319
320 /* Copied from tools/perf/util/util.h */
321 #ifndef zfree
322 # define zfree(ptr) ({ free(*ptr); *ptr = NULL; })
323 #endif
324
325 #ifndef zclose
326 # define zclose(fd) ({ \
327 int ___err = 0; \
328 if ((fd) >= 0) \
329 ___err = close((fd)); \
330 fd = -1; \
331 ___err; })
332 #endif
333
334 static inline __u64 ptr_to_u64(const void *ptr)
335 {
336 return (__u64) (unsigned long) ptr;
337 }
338
339 int libbpf_set_strict_mode(enum libbpf_strict_mode mode)
340 {
341 /* as of v1.0 libbpf_set_strict_mode() is a no-op */
342 return 0;
343 }
344
345 __u32 libbpf_major_version(void)
346 {
347 return LIBBPF_MAJOR_VERSION;
348 }
349
350 __u32 libbpf_minor_version(void)
351 {
352 return LIBBPF_MINOR_VERSION;
353 }
354
355 const char *libbpf_version_string(void)
356 {
357 #define __S(X) #X
358 #define _S(X) __S(X)
359 return "v" _S(LIBBPF_MAJOR_VERSION) "." _S(LIBBPF_MINOR_VERSION);
360 #undef _S
361 #undef __S
362 }
363
364 enum reloc_type {
365 RELO_LD64,
366 RELO_CALL,
367 RELO_DATA,
368 RELO_EXTERN_LD64,
369 RELO_EXTERN_CALL,
370 RELO_SUBPROG_ADDR,
371 RELO_CORE,
372 };
373
374 struct reloc_desc {
375 enum reloc_type type;
376 int insn_idx;
377 union {
378 const struct bpf_core_relo *core_relo; /* used when type == RELO_CORE */
379 struct {
380 int map_idx;
381 int sym_off;
382 int ext_idx;
383 };
384 };
385 };
386
387 /* stored as sec_def->cookie for all libbpf-supported SEC()s */
388 enum sec_def_flags {
389 SEC_NONE = 0,
390 /* expected_attach_type is optional, if kernel doesn't support that */
391 SEC_EXP_ATTACH_OPT = 1,
392 /* legacy, only used by libbpf_get_type_names() and
393 * libbpf_attach_type_by_name(), not used by libbpf itself at all.
394 * This used to be associated with cgroup (and few other) BPF programs
395 * that were attachable through BPF_PROG_ATTACH command. Pretty
396 * meaningless nowadays, though.
397 */
398 SEC_ATTACHABLE = 2,
399 SEC_ATTACHABLE_OPT = SEC_ATTACHABLE | SEC_EXP_ATTACH_OPT,
400 /* attachment target is specified through BTF ID in either kernel or
401 * other BPF program's BTF object
402 */
403 SEC_ATTACH_BTF = 4,
404 /* BPF program type allows sleeping/blocking in kernel */
405 SEC_SLEEPABLE = 8,
406 /* BPF program support non-linear XDP buffer */
407 SEC_XDP_FRAGS = 16,
408 /* Setup proper attach type for usdt probes. */
409 SEC_USDT = 32,
410 };
411
412 struct bpf_sec_def {
413 char *sec;
414 enum bpf_prog_type prog_type;
415 enum bpf_attach_type expected_attach_type;
416 long cookie;
417 int handler_id;
418
419 libbpf_prog_setup_fn_t prog_setup_fn;
420 libbpf_prog_prepare_load_fn_t prog_prepare_load_fn;
421 libbpf_prog_attach_fn_t prog_attach_fn;
422 };
423
424 /*
425 * bpf_prog should be a better name but it has been used in
426 * linux/filter.h.
427 */
428 struct bpf_program {
429 char *name;
430 char *sec_name;
431 size_t sec_idx;
432 const struct bpf_sec_def *sec_def;
433 /* this program's instruction offset (in number of instructions)
434 * within its containing ELF section
435 */
436 size_t sec_insn_off;
437 /* number of original instructions in ELF section belonging to this
438 * program, not taking into account subprogram instructions possible
439 * appended later during relocation
440 */
441 size_t sec_insn_cnt;
442 /* Offset (in number of instructions) of the start of instruction
443 * belonging to this BPF program within its containing main BPF
444 * program. For the entry-point (main) BPF program, this is always
445 * zero. For a sub-program, this gets reset before each of main BPF
446 * programs are processed and relocated and is used to determined
447 * whether sub-program was already appended to the main program, and
448 * if yes, at which instruction offset.
449 */
450 size_t sub_insn_off;
451
452 /* instructions that belong to BPF program; insns[0] is located at
453 * sec_insn_off instruction within its ELF section in ELF file, so
454 * when mapping ELF file instruction index to the local instruction,
455 * one needs to subtract sec_insn_off; and vice versa.
456 */
457 struct bpf_insn *insns;
458 /* actual number of instruction in this BPF program's image; for
459 * entry-point BPF programs this includes the size of main program
460 * itself plus all the used sub-programs, appended at the end
461 */
462 size_t insns_cnt;
463
464 struct reloc_desc *reloc_desc;
465 int nr_reloc;
466
467 /* BPF verifier log settings */
468 char *log_buf;
469 size_t log_size;
470 __u32 log_level;
471
472 struct bpf_object *obj;
473
474 int fd;
475 bool autoload;
476 bool autoattach;
477 bool sym_global;
478 bool mark_btf_static;
479 enum bpf_prog_type type;
480 enum bpf_attach_type expected_attach_type;
481 int exception_cb_idx;
482
483 int prog_ifindex;
484 __u32 attach_btf_obj_fd;
485 __u32 attach_btf_id;
486 __u32 attach_prog_fd;
487
488 void *func_info;
489 __u32 func_info_rec_size;
490 __u32 func_info_cnt;
491
492 void *line_info;
493 __u32 line_info_rec_size;
494 __u32 line_info_cnt;
495 __u32 prog_flags;
496 };
497
498 struct bpf_struct_ops {
499 struct bpf_program **progs;
500 __u32 *kern_func_off;
501 /* e.g. struct tcp_congestion_ops in bpf_prog's btf format */
502 void *data;
503 /* e.g. struct bpf_struct_ops_tcp_congestion_ops in
504 * btf_vmlinux's format.
505 * struct bpf_struct_ops_tcp_congestion_ops {
506 * [... some other kernel fields ...]
507 * struct tcp_congestion_ops data;
508 * }
509 * kern_vdata-size == sizeof(struct bpf_struct_ops_tcp_congestion_ops)
510 * bpf_map__init_kern_struct_ops() will populate the "kern_vdata"
511 * from "data".
512 */
513 void *kern_vdata;
514 __u32 type_id;
515 };
516
517 #define DATA_SEC ".data"
518 #define BSS_SEC ".bss"
519 #define RODATA_SEC ".rodata"
520 #define KCONFIG_SEC ".kconfig"
521 #define KSYMS_SEC ".ksyms"
522 #define STRUCT_OPS_SEC ".struct_ops"
523 #define STRUCT_OPS_LINK_SEC ".struct_ops.link"
524 #define ARENA_SEC ".addr_space.1"
525
526 enum libbpf_map_type {
527 LIBBPF_MAP_UNSPEC,
528 LIBBPF_MAP_DATA,
529 LIBBPF_MAP_BSS,
530 LIBBPF_MAP_RODATA,
531 LIBBPF_MAP_KCONFIG,
532 };
533
534 struct bpf_map_def {
535 unsigned int type;
536 unsigned int key_size;
537 unsigned int value_size;
538 unsigned int max_entries;
539 unsigned int map_flags;
540 };
541
542 struct bpf_map {
543 struct bpf_object *obj;
544 char *name;
545 /* real_name is defined for special internal maps (.rodata*,
546 * .data*, .bss, .kconfig) and preserves their original ELF section
547 * name. This is important to be able to find corresponding BTF
548 * DATASEC information.
549 */
550 char *real_name;
551 int fd;
552 int sec_idx;
553 size_t sec_offset;
554 int map_ifindex;
555 int inner_map_fd;
556 struct bpf_map_def def;
557 __u32 numa_node;
558 __u32 btf_var_idx;
559 int mod_btf_fd;
560 __u32 btf_key_type_id;
561 __u32 btf_value_type_id;
562 __u32 btf_vmlinux_value_type_id;
563 enum libbpf_map_type libbpf_type;
564 void *mmaped;
565 struct bpf_struct_ops *st_ops;
566 struct bpf_map *inner_map;
567 void **init_slots;
568 int init_slots_sz;
569 char *pin_path;
570 bool pinned;
571 bool reused;
572 bool autocreate;
573 bool autoattach;
574 __u64 map_extra;
575 };
576
577 enum extern_type {
578 EXT_UNKNOWN,
579 EXT_KCFG,
580 EXT_KSYM,
581 };
582
583 enum kcfg_type {
584 KCFG_UNKNOWN,
585 KCFG_CHAR,
586 KCFG_BOOL,
587 KCFG_INT,
588 KCFG_TRISTATE,
589 KCFG_CHAR_ARR,
590 };
591
592 struct extern_desc {
593 enum extern_type type;
594 int sym_idx;
595 int btf_id;
596 int sec_btf_id;
597 const char *name;
598 char *essent_name;
599 bool is_set;
600 bool is_weak;
601 union {
602 struct {
603 enum kcfg_type type;
604 int sz;
605 int align;
606 int data_off;
607 bool is_signed;
608 } kcfg;
609 struct {
610 unsigned long long addr;
611
612 /* target btf_id of the corresponding kernel var. */
613 int kernel_btf_obj_fd;
614 int kernel_btf_id;
615
616 /* local btf_id of the ksym extern's type. */
617 __u32 type_id;
618 /* BTF fd index to be patched in for insn->off, this is
619 * 0 for vmlinux BTF, index in obj->fd_array for module
620 * BTF
621 */
622 __s16 btf_fd_idx;
623 } ksym;
624 };
625 };
626
627 struct module_btf {
628 struct btf *btf;
629 char *name;
630 __u32 id;
631 int fd;
632 int fd_array_idx;
633 };
634
635 enum sec_type {
636 SEC_UNUSED = 0,
637 SEC_RELO,
638 SEC_BSS,
639 SEC_DATA,
640 SEC_RODATA,
641 SEC_ST_OPS,
642 };
643
644 struct elf_sec_desc {
645 enum sec_type sec_type;
646 Elf64_Shdr *shdr;
647 Elf_Data *data;
648 };
649
650 struct elf_state {
651 int fd;
652 const void *obj_buf;
653 size_t obj_buf_sz;
654 Elf *elf;
655 Elf64_Ehdr *ehdr;
656 Elf_Data *symbols;
657 Elf_Data *arena_data;
658 size_t shstrndx; /* section index for section name strings */
659 size_t strtabidx;
660 struct elf_sec_desc *secs;
661 size_t sec_cnt;
662 int btf_maps_shndx;
663 __u32 btf_maps_sec_btf_id;
664 int text_shndx;
665 int symbols_shndx;
666 bool has_st_ops;
667 int arena_data_shndx;
668 };
669
670 struct usdt_manager;
671
672 struct bpf_object {
673 char name[BPF_OBJ_NAME_LEN];
674 char license[64];
675 __u32 kern_version;
676
677 struct bpf_program *programs;
678 size_t nr_programs;
679 struct bpf_map *maps;
680 size_t nr_maps;
681 size_t maps_cap;
682
683 char *kconfig;
684 struct extern_desc *externs;
685 int nr_extern;
686 int kconfig_map_idx;
687
688 bool loaded;
689 bool has_subcalls;
690 bool has_rodata;
691
692 struct bpf_gen *gen_loader;
693
694 /* Information when doing ELF related work. Only valid if efile.elf is not NULL */
695 struct elf_state efile;
696
697 struct btf *btf;
698 struct btf_ext *btf_ext;
699
700 /* Parse and load BTF vmlinux if any of the programs in the object need
701 * it at load time.
702 */
703 struct btf *btf_vmlinux;
704 /* Path to the custom BTF to be used for BPF CO-RE relocations as an
705 * override for vmlinux BTF.
706 */
707 char *btf_custom_path;
708 /* vmlinux BTF override for CO-RE relocations */
709 struct btf *btf_vmlinux_override;
710 /* Lazily initialized kernel module BTFs */
711 struct module_btf *btf_modules;
712 bool btf_modules_loaded;
713 size_t btf_module_cnt;
714 size_t btf_module_cap;
715
716 /* optional log settings passed to BPF_BTF_LOAD and BPF_PROG_LOAD commands */
717 char *log_buf;
718 size_t log_size;
719 __u32 log_level;
720
721 int *fd_array;
722 size_t fd_array_cap;
723 size_t fd_array_cnt;
724
725 struct usdt_manager *usdt_man;
726
727 struct bpf_map *arena_map;
728 void *arena_data;
729 size_t arena_data_sz;
730
731 struct kern_feature_cache *feat_cache;
732 char *token_path;
733 int token_fd;
734
735 char path[];
736 };
737
738 static const char *elf_sym_str(const struct bpf_object *obj, size_t off);
739 static const char *elf_sec_str(const struct bpf_object *obj, size_t off);
740 static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx);
741 static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name);
742 static Elf64_Shdr *elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn);
743 static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn);
744 static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn);
745 static Elf64_Sym *elf_sym_by_idx(const struct bpf_object *obj, size_t idx);
746 static Elf64_Rel *elf_rel_by_idx(Elf_Data *data, size_t idx);
747
748 void bpf_program__unload(struct bpf_program *prog)
749 {
750 if (!prog)
751 return;
752
753 zclose(prog->fd);
754
755 zfree(&prog->func_info);
756 zfree(&prog->line_info);
757 }
758
759 static void bpf_program__exit(struct bpf_program *prog)
760 {
761 if (!prog)
762 return;
763
764 bpf_program__unload(prog);
765 zfree(&prog->name);
766 zfree(&prog->sec_name);
767 zfree(&prog->insns);
768 zfree(&prog->reloc_desc);
769
770 prog->nr_reloc = 0;
771 prog->insns_cnt = 0;
772 prog->sec_idx = -1;
773 }
774
775 static bool insn_is_subprog_call(const struct bpf_insn *insn)
776 {
777 return BPF_CLASS(insn->code) == BPF_JMP &&
778 BPF_OP(insn->code) == BPF_CALL &&
779 BPF_SRC(insn->code) == BPF_K &&
780 insn->src_reg == BPF_PSEUDO_CALL &&
781 insn->dst_reg == 0 &&
782 insn->off == 0;
783 }
784
785 static bool is_call_insn(const struct bpf_insn *insn)
786 {
787 return insn->code == (BPF_JMP | BPF_CALL);
788 }
789
790 static bool insn_is_pseudo_func(struct bpf_insn *insn)
791 {
792 return is_ldimm64_insn(insn) && insn->src_reg == BPF_PSEUDO_FUNC;
793 }
794
795 static int
796 bpf_object__init_prog(struct bpf_object *obj, struct bpf_program *prog,
797 const char *name, size_t sec_idx, const char *sec_name,
798 size_t sec_off, void *insn_data, size_t insn_data_sz)
799 {
800 if (insn_data_sz == 0 || insn_data_sz % BPF_INSN_SZ || sec_off % BPF_INSN_SZ) {
801 pr_warn("sec '%s': corrupted program '%s', offset %zu, size %zu\n",
802 sec_name, name, sec_off, insn_data_sz);
803 return -EINVAL;
804 }
805
806 memset(prog, 0, sizeof(*prog));
807 prog->obj = obj;
808
809 prog->sec_idx = sec_idx;
810 prog->sec_insn_off = sec_off / BPF_INSN_SZ;
811 prog->sec_insn_cnt = insn_data_sz / BPF_INSN_SZ;
812 /* insns_cnt can later be increased by appending used subprograms */
813 prog->insns_cnt = prog->sec_insn_cnt;
814
815 prog->type = BPF_PROG_TYPE_UNSPEC;
816 prog->fd = -1;
817 prog->exception_cb_idx = -1;
818
819 /* libbpf's convention for SEC("?abc...") is that it's just like
820 * SEC("abc...") but the corresponding bpf_program starts out with
821 * autoload set to false.
822 */
823 if (sec_name[0] == '?') {
824 prog->autoload = false;
825 /* from now on forget there was ? in section name */
826 sec_name++;
827 } else {
828 prog->autoload = true;
829 }
830
831 prog->autoattach = true;
832
833 /* inherit object's log_level */
834 prog->log_level = obj->log_level;
835
836 prog->sec_name = strdup(sec_name);
837 if (!prog->sec_name)
838 goto errout;
839
840 prog->name = strdup(name);
841 if (!prog->name)
842 goto errout;
843
844 prog->insns = malloc(insn_data_sz);
845 if (!prog->insns)
846 goto errout;
847 memcpy(prog->insns, insn_data, insn_data_sz);
848
849 return 0;
850 errout:
851 pr_warn("sec '%s': failed to allocate memory for prog '%s'\n", sec_name, name);
852 bpf_program__exit(prog);
853 return -ENOMEM;
854 }
855
856 static int
857 bpf_object__add_programs(struct bpf_object *obj, Elf_Data *sec_data,
858 const char *sec_name, int sec_idx)
859 {
860 Elf_Data *symbols = obj->efile.symbols;
861 struct bpf_program *prog, *progs;
862 void *data = sec_data->d_buf;
863 size_t sec_sz = sec_data->d_size, sec_off, prog_sz, nr_syms;
864 int nr_progs, err, i;
865 const char *name;
866 Elf64_Sym *sym;
867
868 progs = obj->programs;
869 nr_progs = obj->nr_programs;
870 nr_syms = symbols->d_size / sizeof(Elf64_Sym);
871
872 for (i = 0; i < nr_syms; i++) {
873 sym = elf_sym_by_idx(obj, i);
874
875 if (sym->st_shndx != sec_idx)
876 continue;
877 if (ELF64_ST_TYPE(sym->st_info) != STT_FUNC)
878 continue;
879
880 prog_sz = sym->st_size;
881 sec_off = sym->st_value;
882
883 name = elf_sym_str(obj, sym->st_name);
884 if (!name) {
885 pr_warn("sec '%s': failed to get symbol name for offset %zu\n",
886 sec_name, sec_off);
887 return -LIBBPF_ERRNO__FORMAT;
888 }
889
890 if (sec_off + prog_sz > sec_sz) {
891 pr_warn("sec '%s': program at offset %zu crosses section boundary\n",
892 sec_name, sec_off);
893 return -LIBBPF_ERRNO__FORMAT;
894 }
895
896 if (sec_idx != obj->efile.text_shndx && ELF64_ST_BIND(sym->st_info) == STB_LOCAL) {
897 pr_warn("sec '%s': program '%s' is static and not supported\n", sec_name, name);
898 return -ENOTSUP;
899 }
900
901 pr_debug("sec '%s': found program '%s' at insn offset %zu (%zu bytes), code size %zu insns (%zu bytes)\n",
902 sec_name, name, sec_off / BPF_INSN_SZ, sec_off, prog_sz / BPF_INSN_SZ, prog_sz);
903
904 progs = libbpf_reallocarray(progs, nr_progs + 1, sizeof(*progs));
905 if (!progs) {
906 /*
907 * In this case the original obj->programs
908 * is still valid, so don't need special treat for
909 * bpf_close_object().
910 */
911 pr_warn("sec '%s': failed to alloc memory for new program '%s'\n",
912 sec_name, name);
913 return -ENOMEM;
914 }
915 obj->programs = progs;
916
917 prog = &progs[nr_progs];
918
919 err = bpf_object__init_prog(obj, prog, name, sec_idx, sec_name,
920 sec_off, data + sec_off, prog_sz);
921 if (err)
922 return err;
923
924 if (ELF64_ST_BIND(sym->st_info) != STB_LOCAL)
925 prog->sym_global = true;
926
927 /* if function is a global/weak symbol, but has restricted
928 * (STV_HIDDEN or STV_INTERNAL) visibility, mark its BTF FUNC
929 * as static to enable more permissive BPF verification mode
930 * with more outside context available to BPF verifier
931 */
932 if (prog->sym_global && (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN
933 || ELF64_ST_VISIBILITY(sym->st_other) == STV_INTERNAL))
934 prog->mark_btf_static = true;
935
936 nr_progs++;
937 obj->nr_programs = nr_progs;
938 }
939
940 return 0;
941 }
942
943 static const struct btf_member *
944 find_member_by_offset(const struct btf_type *t, __u32 bit_offset)
945 {
946 struct btf_member *m;
947 int i;
948
949 for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) {
950 if (btf_member_bit_offset(t, i) == bit_offset)
951 return m;
952 }
953
954 return NULL;
955 }
956
957 static const struct btf_member *
958 find_member_by_name(const struct btf *btf, const struct btf_type *t,
959 const char *name)
960 {
961 struct btf_member *m;
962 int i;
963
964 for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) {
965 if (!strcmp(btf__name_by_offset(btf, m->name_off), name))
966 return m;
967 }
968
969 return NULL;
970 }
971
972 static int find_ksym_btf_id(struct bpf_object *obj, const char *ksym_name,
973 __u16 kind, struct btf **res_btf,
974 struct module_btf **res_mod_btf);
975
976 #define STRUCT_OPS_VALUE_PREFIX "bpf_struct_ops_"
977 static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix,
978 const char *name, __u32 kind);
979
980 static int
981 find_struct_ops_kern_types(struct bpf_object *obj, const char *tname_raw,
982 struct module_btf **mod_btf,
983 const struct btf_type **type, __u32 *type_id,
984 const struct btf_type **vtype, __u32 *vtype_id,
985 const struct btf_member **data_member)
986 {
987 const struct btf_type *kern_type, *kern_vtype;
988 const struct btf_member *kern_data_member;
989 struct btf *btf;
990 __s32 kern_vtype_id, kern_type_id;
991 char tname[256];
992 __u32 i;
993
994 snprintf(tname, sizeof(tname), "%.*s",
995 (int)bpf_core_essential_name_len(tname_raw), tname_raw);
996
997 kern_type_id = find_ksym_btf_id(obj, tname, BTF_KIND_STRUCT,
998 &btf, mod_btf);
999 if (kern_type_id < 0) {
1000 pr_warn("struct_ops init_kern: struct %s is not found in kernel BTF\n",
1001 tname);
1002 return kern_type_id;
1003 }
1004 kern_type = btf__type_by_id(btf, kern_type_id);
1005
1006 /* Find the corresponding "map_value" type that will be used
1007 * in map_update(BPF_MAP_TYPE_STRUCT_OPS). For example,
1008 * find "struct bpf_struct_ops_tcp_congestion_ops" from the
1009 * btf_vmlinux.
1010 */
1011 kern_vtype_id = find_btf_by_prefix_kind(btf, STRUCT_OPS_VALUE_PREFIX,
1012 tname, BTF_KIND_STRUCT);
1013 if (kern_vtype_id < 0) {
1014 pr_warn("struct_ops init_kern: struct %s%s is not found in kernel BTF\n",
1015 STRUCT_OPS_VALUE_PREFIX, tname);
1016 return kern_vtype_id;
1017 }
1018 kern_vtype = btf__type_by_id(btf, kern_vtype_id);
1019
1020 /* Find "struct tcp_congestion_ops" from
1021 * struct bpf_struct_ops_tcp_congestion_ops {
1022 * [ ... ]
1023 * struct tcp_congestion_ops data;
1024 * }
1025 */
1026 kern_data_member = btf_members(kern_vtype);
1027 for (i = 0; i < btf_vlen(kern_vtype); i++, kern_data_member++) {
1028 if (kern_data_member->type == kern_type_id)
1029 break;
1030 }
1031 if (i == btf_vlen(kern_vtype)) {
1032 pr_warn("struct_ops init_kern: struct %s data is not found in struct %s%s\n",
1033 tname, STRUCT_OPS_VALUE_PREFIX, tname);
1034 return -EINVAL;
1035 }
1036
1037 *type = kern_type;
1038 *type_id = kern_type_id;
1039 *vtype = kern_vtype;
1040 *vtype_id = kern_vtype_id;
1041 *data_member = kern_data_member;
1042
1043 return 0;
1044 }
1045
1046 static bool bpf_map__is_struct_ops(const struct bpf_map *map)
1047 {
1048 return map->def.type == BPF_MAP_TYPE_STRUCT_OPS;
1049 }
1050
1051 static bool is_valid_st_ops_program(struct bpf_object *obj,
1052 const struct bpf_program *prog)
1053 {
1054 int i;
1055
1056 for (i = 0; i < obj->nr_programs; i++) {
1057 if (&obj->programs[i] == prog)
1058 return prog->type == BPF_PROG_TYPE_STRUCT_OPS;
1059 }
1060
1061 return false;
1062 }
1063
1064 /* For each struct_ops program P, referenced from some struct_ops map M,
1065 * enable P.autoload if there are Ms for which M.autocreate is true,
1066 * disable P.autoload if for all Ms M.autocreate is false.
1067 * Don't change P.autoload for programs that are not referenced from any maps.
1068 */
1069 static int bpf_object_adjust_struct_ops_autoload(struct bpf_object *obj)
1070 {
1071 struct bpf_program *prog, *slot_prog;
1072 struct bpf_map *map;
1073 int i, j, k, vlen;
1074
1075 for (i = 0; i < obj->nr_programs; ++i) {
1076 int should_load = false;
1077 int use_cnt = 0;
1078
1079 prog = &obj->programs[i];
1080 if (prog->type != BPF_PROG_TYPE_STRUCT_OPS)
1081 continue;
1082
1083 for (j = 0; j < obj->nr_maps; ++j) {
1084 const struct btf_type *type;
1085
1086 map = &obj->maps[j];
1087 if (!bpf_map__is_struct_ops(map))
1088 continue;
1089
1090 type = btf__type_by_id(obj->btf, map->st_ops->type_id);
1091 vlen = btf_vlen(type);
1092 for (k = 0; k < vlen; ++k) {
1093 slot_prog = map->st_ops->progs[k];
1094 if (prog != slot_prog)
1095 continue;
1096
1097 use_cnt++;
1098 if (map->autocreate)
1099 should_load = true;
1100 }
1101 }
1102 if (use_cnt)
1103 prog->autoload = should_load;
1104 }
1105
1106 return 0;
1107 }
1108
1109 /* Init the map's fields that depend on kern_btf */
1110 static int bpf_map__init_kern_struct_ops(struct bpf_map *map)
1111 {
1112 const struct btf_member *member, *kern_member, *kern_data_member;
1113 const struct btf_type *type, *kern_type, *kern_vtype;
1114 __u32 i, kern_type_id, kern_vtype_id, kern_data_off;
1115 struct bpf_object *obj = map->obj;
1116 const struct btf *btf = obj->btf;
1117 struct bpf_struct_ops *st_ops;
1118 const struct btf *kern_btf;
1119 struct module_btf *mod_btf;
1120 void *data, *kern_data;
1121 const char *tname;
1122 int err;
1123
1124 st_ops = map->st_ops;
1125 type = btf__type_by_id(btf, st_ops->type_id);
1126 tname = btf__name_by_offset(btf, type->name_off);
1127 err = find_struct_ops_kern_types(obj, tname, &mod_btf,
1128 &kern_type, &kern_type_id,
1129 &kern_vtype, &kern_vtype_id,
1130 &kern_data_member);
1131 if (err)
1132 return err;
1133
1134 kern_btf = mod_btf ? mod_btf->btf : obj->btf_vmlinux;
1135
1136 pr_debug("struct_ops init_kern %s: type_id:%u kern_type_id:%u kern_vtype_id:%u\n",
1137 map->name, st_ops->type_id, kern_type_id, kern_vtype_id);
1138
1139 map->mod_btf_fd = mod_btf ? mod_btf->fd : -1;
1140 map->def.value_size = kern_vtype->size;
1141 map->btf_vmlinux_value_type_id = kern_vtype_id;
1142
1143 st_ops->kern_vdata = calloc(1, kern_vtype->size);
1144 if (!st_ops->kern_vdata)
1145 return -ENOMEM;
1146
1147 data = st_ops->data;
1148 kern_data_off = kern_data_member->offset / 8;
1149 kern_data = st_ops->kern_vdata + kern_data_off;
1150
1151 member = btf_members(type);
1152 for (i = 0; i < btf_vlen(type); i++, member++) {
1153 const struct btf_type *mtype, *kern_mtype;
1154 __u32 mtype_id, kern_mtype_id;
1155 void *mdata, *kern_mdata;
1156 struct bpf_program *prog;
1157 __s64 msize, kern_msize;
1158 __u32 moff, kern_moff;
1159 __u32 kern_member_idx;
1160 const char *mname;
1161
1162 mname = btf__name_by_offset(btf, member->name_off);
1163 moff = member->offset / 8;
1164 mdata = data + moff;
1165 msize = btf__resolve_size(btf, member->type);
1166 if (msize < 0) {
1167 pr_warn("struct_ops init_kern %s: failed to resolve the size of member %s\n",
1168 map->name, mname);
1169 return msize;
1170 }
1171
1172 kern_member = find_member_by_name(kern_btf, kern_type, mname);
1173 if (!kern_member) {
1174 if (!libbpf_is_mem_zeroed(mdata, msize)) {
1175 pr_warn("struct_ops init_kern %s: Cannot find member %s in kernel BTF\n",
1176 map->name, mname);
1177 return -ENOTSUP;
1178 }
1179
1180 if (st_ops->progs[i]) {
1181 /* If we had declaratively set struct_ops callback, we need to
1182 * force its autoload to false, because it doesn't have
1183 * a chance of succeeding from POV of the current struct_ops map.
1184 * If this program is still referenced somewhere else, though,
1185 * then bpf_object_adjust_struct_ops_autoload() will update its
1186 * autoload accordingly.
1187 */
1188 st_ops->progs[i]->autoload = false;
1189 st_ops->progs[i] = NULL;
1190 }
1191
1192 /* Skip all-zero/NULL fields if they are not present in the kernel BTF */
1193 pr_info("struct_ops %s: member %s not found in kernel, skipping it as it's set to zero\n",
1194 map->name, mname);
1195 continue;
1196 }
1197
1198 kern_member_idx = kern_member - btf_members(kern_type);
1199 if (btf_member_bitfield_size(type, i) ||
1200 btf_member_bitfield_size(kern_type, kern_member_idx)) {
1201 pr_warn("struct_ops init_kern %s: bitfield %s is not supported\n",
1202 map->name, mname);
1203 return -ENOTSUP;
1204 }
1205
1206 kern_moff = kern_member->offset / 8;
1207 kern_mdata = kern_data + kern_moff;
1208
1209 mtype = skip_mods_and_typedefs(btf, member->type, &mtype_id);
1210 kern_mtype = skip_mods_and_typedefs(kern_btf, kern_member->type,
1211 &kern_mtype_id);
1212 if (BTF_INFO_KIND(mtype->info) !=
1213 BTF_INFO_KIND(kern_mtype->info)) {
1214 pr_warn("struct_ops init_kern %s: Unmatched member type %s %u != %u(kernel)\n",
1215 map->name, mname, BTF_INFO_KIND(mtype->info),
1216 BTF_INFO_KIND(kern_mtype->info));
1217 return -ENOTSUP;
1218 }
1219
1220 if (btf_is_ptr(mtype)) {
1221 prog = *(void **)mdata;
1222 /* just like for !kern_member case above, reset declaratively
1223 * set (at compile time) program's autload to false,
1224 * if user replaced it with another program or NULL
1225 */
1226 if (st_ops->progs[i] && st_ops->progs[i] != prog)
1227 st_ops->progs[i]->autoload = false;
1228
1229 /* Update the value from the shadow type */
1230 st_ops->progs[i] = prog;
1231 if (!prog)
1232 continue;
1233
1234 if (!is_valid_st_ops_program(obj, prog)) {
1235 pr_warn("struct_ops init_kern %s: member %s is not a struct_ops program\n",
1236 map->name, mname);
1237 return -ENOTSUP;
1238 }
1239
1240 kern_mtype = skip_mods_and_typedefs(kern_btf,
1241 kern_mtype->type,
1242 &kern_mtype_id);
1243
1244 /* mtype->type must be a func_proto which was
1245 * guaranteed in bpf_object__collect_st_ops_relos(),
1246 * so only check kern_mtype for func_proto here.
1247 */
1248 if (!btf_is_func_proto(kern_mtype)) {
1249 pr_warn("struct_ops init_kern %s: kernel member %s is not a func ptr\n",
1250 map->name, mname);
1251 return -ENOTSUP;
1252 }
1253
1254 if (mod_btf)
1255 prog->attach_btf_obj_fd = mod_btf->fd;
1256
1257 /* if we haven't yet processed this BPF program, record proper
1258 * attach_btf_id and member_idx
1259 */
1260 if (!prog->attach_btf_id) {
1261 prog->attach_btf_id = kern_type_id;
1262 prog->expected_attach_type = kern_member_idx;
1263 }
1264
1265 /* struct_ops BPF prog can be re-used between multiple
1266 * .struct_ops & .struct_ops.link as long as it's the
1267 * same struct_ops struct definition and the same
1268 * function pointer field
1269 */
1270 if (prog->attach_btf_id != kern_type_id) {
1271 pr_warn("struct_ops init_kern %s func ptr %s: invalid reuse of prog %s in sec %s with type %u: attach_btf_id %u != kern_type_id %u\n",
1272 map->name, mname, prog->name, prog->sec_name, prog->type,
1273 prog->attach_btf_id, kern_type_id);
1274 return -EINVAL;
1275 }
1276 if (prog->expected_attach_type != kern_member_idx) {
1277 pr_warn("struct_ops init_kern %s func ptr %s: invalid reuse of prog %s in sec %s with type %u: expected_attach_type %u != kern_member_idx %u\n",
1278 map->name, mname, prog->name, prog->sec_name, prog->type,
1279 prog->expected_attach_type, kern_member_idx);
1280 return -EINVAL;
1281 }
1282
1283 st_ops->kern_func_off[i] = kern_data_off + kern_moff;
1284
1285 pr_debug("struct_ops init_kern %s: func ptr %s is set to prog %s from data(+%u) to kern_data(+%u)\n",
1286 map->name, mname, prog->name, moff,
1287 kern_moff);
1288
1289 continue;
1290 }
1291
1292 kern_msize = btf__resolve_size(kern_btf, kern_mtype_id);
1293 if (kern_msize < 0 || msize != kern_msize) {
1294 pr_warn("struct_ops init_kern %s: Error in size of member %s: %zd != %zd(kernel)\n",
1295 map->name, mname, (ssize_t)msize,
1296 (ssize_t)kern_msize);
1297 return -ENOTSUP;
1298 }
1299
1300 pr_debug("struct_ops init_kern %s: copy %s %u bytes from data(+%u) to kern_data(+%u)\n",
1301 map->name, mname, (unsigned int)msize,
1302 moff, kern_moff);
1303 memcpy(kern_mdata, mdata, msize);
1304 }
1305
1306 return 0;
1307 }
1308
1309 static int bpf_object__init_kern_struct_ops_maps(struct bpf_object *obj)
1310 {
1311 struct bpf_map *map;
1312 size_t i;
1313 int err;
1314
1315 for (i = 0; i < obj->nr_maps; i++) {
1316 map = &obj->maps[i];
1317
1318 if (!bpf_map__is_struct_ops(map))
1319 continue;
1320
1321 if (!map->autocreate)
1322 continue;
1323
1324 err = bpf_map__init_kern_struct_ops(map);
1325 if (err)
1326 return err;
1327 }
1328
1329 return 0;
1330 }
1331
1332 static int init_struct_ops_maps(struct bpf_object *obj, const char *sec_name,
1333 int shndx, Elf_Data *data)
1334 {
1335 const struct btf_type *type, *datasec;
1336 const struct btf_var_secinfo *vsi;
1337 struct bpf_struct_ops *st_ops;
1338 const char *tname, *var_name;
1339 __s32 type_id, datasec_id;
1340 const struct btf *btf;
1341 struct bpf_map *map;
1342 __u32 i;
1343
1344 if (shndx == -1)
1345 return 0;
1346
1347 btf = obj->btf;
1348 datasec_id = btf__find_by_name_kind(btf, sec_name,
1349 BTF_KIND_DATASEC);
1350 if (datasec_id < 0) {
1351 pr_warn("struct_ops init: DATASEC %s not found\n",
1352 sec_name);
1353 return -EINVAL;
1354 }
1355
1356 datasec = btf__type_by_id(btf, datasec_id);
1357 vsi = btf_var_secinfos(datasec);
1358 for (i = 0; i < btf_vlen(datasec); i++, vsi++) {
1359 type = btf__type_by_id(obj->btf, vsi->type);
1360 var_name = btf__name_by_offset(obj->btf, type->name_off);
1361
1362 type_id = btf__resolve_type(obj->btf, vsi->type);
1363 if (type_id < 0) {
1364 pr_warn("struct_ops init: Cannot resolve var type_id %u in DATASEC %s\n",
1365 vsi->type, sec_name);
1366 return -EINVAL;
1367 }
1368
1369 type = btf__type_by_id(obj->btf, type_id);
1370 tname = btf__name_by_offset(obj->btf, type->name_off);
1371 if (!tname[0]) {
1372 pr_warn("struct_ops init: anonymous type is not supported\n");
1373 return -ENOTSUP;
1374 }
1375 if (!btf_is_struct(type)) {
1376 pr_warn("struct_ops init: %s is not a struct\n", tname);
1377 return -EINVAL;
1378 }
1379
1380 map = bpf_object__add_map(obj);
1381 if (IS_ERR(map))
1382 return PTR_ERR(map);
1383
1384 map->sec_idx = shndx;
1385 map->sec_offset = vsi->offset;
1386 map->name = strdup(var_name);
1387 if (!map->name)
1388 return -ENOMEM;
1389 map->btf_value_type_id = type_id;
1390
1391 /* Follow same convention as for programs autoload:
1392 * SEC("?.struct_ops") means map is not created by default.
1393 */
1394 if (sec_name[0] == '?') {
1395 map->autocreate = false;
1396 /* from now on forget there was ? in section name */
1397 sec_name++;
1398 }
1399
1400 map->def.type = BPF_MAP_TYPE_STRUCT_OPS;
1401 map->def.key_size = sizeof(int);
1402 map->def.value_size = type->size;
1403 map->def.max_entries = 1;
1404 map->def.map_flags = strcmp(sec_name, STRUCT_OPS_LINK_SEC) == 0 ? BPF_F_LINK : 0;
1405 map->autoattach = true;
1406
1407 map->st_ops = calloc(1, sizeof(*map->st_ops));
1408 if (!map->st_ops)
1409 return -ENOMEM;
1410 st_ops = map->st_ops;
1411 st_ops->data = malloc(type->size);
1412 st_ops->progs = calloc(btf_vlen(type), sizeof(*st_ops->progs));
1413 st_ops->kern_func_off = malloc(btf_vlen(type) *
1414 sizeof(*st_ops->kern_func_off));
1415 if (!st_ops->data || !st_ops->progs || !st_ops->kern_func_off)
1416 return -ENOMEM;
1417
1418 if (vsi->offset + type->size > data->d_size) {
1419 pr_warn("struct_ops init: var %s is beyond the end of DATASEC %s\n",
1420 var_name, sec_name);
1421 return -EINVAL;
1422 }
1423
1424 memcpy(st_ops->data,
1425 data->d_buf + vsi->offset,
1426 type->size);
1427 st_ops->type_id = type_id;
1428
1429 pr_debug("struct_ops init: struct %s(type_id=%u) %s found at offset %u\n",
1430 tname, type_id, var_name, vsi->offset);
1431 }
1432
1433 return 0;
1434 }
1435
1436 static int bpf_object_init_struct_ops(struct bpf_object *obj)
1437 {
1438 const char *sec_name;
1439 int sec_idx, err;
1440
1441 for (sec_idx = 0; sec_idx < obj->efile.sec_cnt; ++sec_idx) {
1442 struct elf_sec_desc *desc = &obj->efile.secs[sec_idx];
1443
1444 if (desc->sec_type != SEC_ST_OPS)
1445 continue;
1446
1447 sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx));
1448 if (!sec_name)
1449 return -LIBBPF_ERRNO__FORMAT;
1450
1451 err = init_struct_ops_maps(obj, sec_name, sec_idx, desc->data);
1452 if (err)
1453 return err;
1454 }
1455
1456 return 0;
1457 }
1458
1459 static struct bpf_object *bpf_object__new(const char *path,
1460 const void *obj_buf,
1461 size_t obj_buf_sz,
1462 const char *obj_name)
1463 {
1464 struct bpf_object *obj;
1465 char *end;
1466
1467 obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1);
1468 if (!obj) {
1469 pr_warn("alloc memory failed for %s\n", path);
1470 return ERR_PTR(-ENOMEM);
1471 }
1472
1473 strcpy(obj->path, path);
1474 if (obj_name) {
1475 libbpf_strlcpy(obj->name, obj_name, sizeof(obj->name));
1476 } else {
1477 /* Using basename() GNU version which doesn't modify arg. */
1478 libbpf_strlcpy(obj->name, basename((void *)path), sizeof(obj->name));
1479 end = strchr(obj->name, '.');
1480 if (end)
1481 *end = 0;
1482 }
1483
1484 obj->efile.fd = -1;
1485 /*
1486 * Caller of this function should also call
1487 * bpf_object__elf_finish() after data collection to return
1488 * obj_buf to user. If not, we should duplicate the buffer to
1489 * avoid user freeing them before elf finish.
1490 */
1491 obj->efile.obj_buf = obj_buf;
1492 obj->efile.obj_buf_sz = obj_buf_sz;
1493 obj->efile.btf_maps_shndx = -1;
1494 obj->kconfig_map_idx = -1;
1495
1496 obj->kern_version = get_kernel_version();
1497 obj->loaded = false;
1498
1499 return obj;
1500 }
1501
1502 static void bpf_object__elf_finish(struct bpf_object *obj)
1503 {
1504 if (!obj->efile.elf)
1505 return;
1506
1507 elf_end(obj->efile.elf);
1508 obj->efile.elf = NULL;
1509 obj->efile.symbols = NULL;
1510 obj->efile.arena_data = NULL;
1511
1512 zfree(&obj->efile.secs);
1513 obj->efile.sec_cnt = 0;
1514 zclose(obj->efile.fd);
1515 obj->efile.obj_buf = NULL;
1516 obj->efile.obj_buf_sz = 0;
1517 }
1518
1519 static int bpf_object__elf_init(struct bpf_object *obj)
1520 {
1521 Elf64_Ehdr *ehdr;
1522 int err = 0;
1523 Elf *elf;
1524
1525 if (obj->efile.elf) {
1526 pr_warn("elf: init internal error\n");
1527 return -LIBBPF_ERRNO__LIBELF;
1528 }
1529
1530 if (obj->efile.obj_buf_sz > 0) {
1531 /* obj_buf should have been validated by bpf_object__open_mem(). */
1532 elf = elf_memory((char *)obj->efile.obj_buf, obj->efile.obj_buf_sz);
1533 } else {
1534 obj->efile.fd = open(obj->path, O_RDONLY | O_CLOEXEC);
1535 if (obj->efile.fd < 0) {
1536 char errmsg[STRERR_BUFSIZE], *cp;
1537
1538 err = -errno;
1539 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
1540 pr_warn("elf: failed to open %s: %s\n", obj->path, cp);
1541 return err;
1542 }
1543
1544 elf = elf_begin(obj->efile.fd, ELF_C_READ_MMAP, NULL);
1545 }
1546
1547 if (!elf) {
1548 pr_warn("elf: failed to open %s as ELF file: %s\n", obj->path, elf_errmsg(-1));
1549 err = -LIBBPF_ERRNO__LIBELF;
1550 goto errout;
1551 }
1552
1553 obj->efile.elf = elf;
1554
1555 if (elf_kind(elf) != ELF_K_ELF) {
1556 err = -LIBBPF_ERRNO__FORMAT;
1557 pr_warn("elf: '%s' is not a proper ELF object\n", obj->path);
1558 goto errout;
1559 }
1560
1561 if (gelf_getclass(elf) != ELFCLASS64) {
1562 err = -LIBBPF_ERRNO__FORMAT;
1563 pr_warn("elf: '%s' is not a 64-bit ELF object\n", obj->path);
1564 goto errout;
1565 }
1566
1567 obj->efile.ehdr = ehdr = elf64_getehdr(elf);
1568 if (!obj->efile.ehdr) {
1569 pr_warn("elf: failed to get ELF header from %s: %s\n", obj->path, elf_errmsg(-1));
1570 err = -LIBBPF_ERRNO__FORMAT;
1571 goto errout;
1572 }
1573
1574 if (elf_getshdrstrndx(elf, &obj->efile.shstrndx)) {
1575 pr_warn("elf: failed to get section names section index for %s: %s\n",
1576 obj->path, elf_errmsg(-1));
1577 err = -LIBBPF_ERRNO__FORMAT;
1578 goto errout;
1579 }
1580
1581 /* ELF is corrupted/truncated, avoid calling elf_strptr. */
1582 if (!elf_rawdata(elf_getscn(elf, obj->efile.shstrndx), NULL)) {
1583 pr_warn("elf: failed to get section names strings from %s: %s\n",
1584 obj->path, elf_errmsg(-1));
1585 err = -LIBBPF_ERRNO__FORMAT;
1586 goto errout;
1587 }
1588
1589 /* Old LLVM set e_machine to EM_NONE */
1590 if (ehdr->e_type != ET_REL || (ehdr->e_machine && ehdr->e_machine != EM_BPF)) {
1591 pr_warn("elf: %s is not a valid eBPF object file\n", obj->path);
1592 err = -LIBBPF_ERRNO__FORMAT;
1593 goto errout;
1594 }
1595
1596 return 0;
1597 errout:
1598 bpf_object__elf_finish(obj);
1599 return err;
1600 }
1601
1602 static int bpf_object__check_endianness(struct bpf_object *obj)
1603 {
1604 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
1605 if (obj->efile.ehdr->e_ident[EI_DATA] == ELFDATA2LSB)
1606 return 0;
1607 #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
1608 if (obj->efile.ehdr->e_ident[EI_DATA] == ELFDATA2MSB)
1609 return 0;
1610 #else
1611 # error "Unrecognized __BYTE_ORDER__"
1612 #endif
1613 pr_warn("elf: endianness mismatch in %s.\n", obj->path);
1614 return -LIBBPF_ERRNO__ENDIAN;
1615 }
1616
1617 static int
1618 bpf_object__init_license(struct bpf_object *obj, void *data, size_t size)
1619 {
1620 if (!data) {
1621 pr_warn("invalid license section in %s\n", obj->path);
1622 return -LIBBPF_ERRNO__FORMAT;
1623 }
1624 /* libbpf_strlcpy() only copies first N - 1 bytes, so size + 1 won't
1625 * go over allowed ELF data section buffer
1626 */
1627 libbpf_strlcpy(obj->license, data, min(size + 1, sizeof(obj->license)));
1628 pr_debug("license of %s is %s\n", obj->path, obj->license);
1629 return 0;
1630 }
1631
1632 static int
1633 bpf_object__init_kversion(struct bpf_object *obj, void *data, size_t size)
1634 {
1635 __u32 kver;
1636
1637 if (!data || size != sizeof(kver)) {
1638 pr_warn("invalid kver section in %s\n", obj->path);
1639 return -LIBBPF_ERRNO__FORMAT;
1640 }
1641 memcpy(&kver, data, sizeof(kver));
1642 obj->kern_version = kver;
1643 pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version);
1644 return 0;
1645 }
1646
1647 static bool bpf_map_type__is_map_in_map(enum bpf_map_type type)
1648 {
1649 if (type == BPF_MAP_TYPE_ARRAY_OF_MAPS ||
1650 type == BPF_MAP_TYPE_HASH_OF_MAPS)
1651 return true;
1652 return false;
1653 }
1654
1655 static int find_elf_sec_sz(const struct bpf_object *obj, const char *name, __u32 *size)
1656 {
1657 Elf_Data *data;
1658 Elf_Scn *scn;
1659
1660 if (!name)
1661 return -EINVAL;
1662
1663 scn = elf_sec_by_name(obj, name);
1664 data = elf_sec_data(obj, scn);
1665 if (data) {
1666 *size = data->d_size;
1667 return 0; /* found it */
1668 }
1669
1670 return -ENOENT;
1671 }
1672
1673 static Elf64_Sym *find_elf_var_sym(const struct bpf_object *obj, const char *name)
1674 {
1675 Elf_Data *symbols = obj->efile.symbols;
1676 const char *sname;
1677 size_t si;
1678
1679 for (si = 0; si < symbols->d_size / sizeof(Elf64_Sym); si++) {
1680 Elf64_Sym *sym = elf_sym_by_idx(obj, si);
1681
1682 if (ELF64_ST_TYPE(sym->st_info) != STT_OBJECT)
1683 continue;
1684
1685 if (ELF64_ST_BIND(sym->st_info) != STB_GLOBAL &&
1686 ELF64_ST_BIND(sym->st_info) != STB_WEAK)
1687 continue;
1688
1689 sname = elf_sym_str(obj, sym->st_name);
1690 if (!sname) {
1691 pr_warn("failed to get sym name string for var %s\n", name);
1692 return ERR_PTR(-EIO);
1693 }
1694 if (strcmp(name, sname) == 0)
1695 return sym;
1696 }
1697
1698 return ERR_PTR(-ENOENT);
1699 }
1700
1701 /* Some versions of Android don't provide memfd_create() in their libc
1702 * implementation, so avoid complications and just go straight to Linux
1703 * syscall.
1704 */
1705 static int sys_memfd_create(const char *name, unsigned flags)
1706 {
1707 return syscall(__NR_memfd_create, name, flags);
1708 }
1709
1710 #ifndef MFD_CLOEXEC
1711 #define MFD_CLOEXEC 0x0001U
1712 #endif
1713
1714 static int create_placeholder_fd(void)
1715 {
1716 int fd;
1717
1718 fd = ensure_good_fd(sys_memfd_create("libbpf-placeholder-fd", MFD_CLOEXEC));
1719 if (fd < 0)
1720 return -errno;
1721 return fd;
1722 }
1723
1724 static struct bpf_map *bpf_object__add_map(struct bpf_object *obj)
1725 {
1726 struct bpf_map *map;
1727 int err;
1728
1729 err = libbpf_ensure_mem((void **)&obj->maps, &obj->maps_cap,
1730 sizeof(*obj->maps), obj->nr_maps + 1);
1731 if (err)
1732 return ERR_PTR(err);
1733
1734 map = &obj->maps[obj->nr_maps++];
1735 map->obj = obj;
1736 /* Preallocate map FD without actually creating BPF map just yet.
1737 * These map FD "placeholders" will be reused later without changing
1738 * FD value when map is actually created in the kernel.
1739 *
1740 * This is useful to be able to perform BPF program relocations
1741 * without having to create BPF maps before that step. This allows us
1742 * to finalize and load BTF very late in BPF object's loading phase,
1743 * right before BPF maps have to be created and BPF programs have to
1744 * be loaded. By having these map FD placeholders we can perform all
1745 * the sanitizations, relocations, and any other adjustments before we
1746 * start creating actual BPF kernel objects (BTF, maps, progs).
1747 */
1748 map->fd = create_placeholder_fd();
1749 if (map->fd < 0)
1750 return ERR_PTR(map->fd);
1751 map->inner_map_fd = -1;
1752 map->autocreate = true;
1753
1754 return map;
1755 }
1756
1757 static size_t array_map_mmap_sz(unsigned int value_sz, unsigned int max_entries)
1758 {
1759 const long page_sz = sysconf(_SC_PAGE_SIZE);
1760 size_t map_sz;
1761
1762 map_sz = (size_t)roundup(value_sz, 8) * max_entries;
1763 map_sz = roundup(map_sz, page_sz);
1764 return map_sz;
1765 }
1766
1767 static size_t bpf_map_mmap_sz(const struct bpf_map *map)
1768 {
1769 const long page_sz = sysconf(_SC_PAGE_SIZE);
1770
1771 switch (map->def.type) {
1772 case BPF_MAP_TYPE_ARRAY:
1773 return array_map_mmap_sz(map->def.value_size, map->def.max_entries);
1774 case BPF_MAP_TYPE_ARENA:
1775 return page_sz * map->def.max_entries;
1776 default:
1777 return 0; /* not supported */
1778 }
1779 }
1780
1781 static int bpf_map_mmap_resize(struct bpf_map *map, size_t old_sz, size_t new_sz)
1782 {
1783 void *mmaped;
1784
1785 if (!map->mmaped)
1786 return -EINVAL;
1787
1788 if (old_sz == new_sz)
1789 return 0;
1790
1791 mmaped = mmap(NULL, new_sz, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0);
1792 if (mmaped == MAP_FAILED)
1793 return -errno;
1794
1795 memcpy(mmaped, map->mmaped, min(old_sz, new_sz));
1796 munmap(map->mmaped, old_sz);
1797 map->mmaped = mmaped;
1798 return 0;
1799 }
1800
1801 static char *internal_map_name(struct bpf_object *obj, const char *real_name)
1802 {
1803 char map_name[BPF_OBJ_NAME_LEN], *p;
1804 int pfx_len, sfx_len = max((size_t)7, strlen(real_name));
1805
1806 /* This is one of the more confusing parts of libbpf for various
1807 * reasons, some of which are historical. The original idea for naming
1808 * internal names was to include as much of BPF object name prefix as
1809 * possible, so that it can be distinguished from similar internal
1810 * maps of a different BPF object.
1811 * As an example, let's say we have bpf_object named 'my_object_name'
1812 * and internal map corresponding to '.rodata' ELF section. The final
1813 * map name advertised to user and to the kernel will be
1814 * 'my_objec.rodata', taking first 8 characters of object name and
1815 * entire 7 characters of '.rodata'.
1816 * Somewhat confusingly, if internal map ELF section name is shorter
1817 * than 7 characters, e.g., '.bss', we still reserve 7 characters
1818 * for the suffix, even though we only have 4 actual characters, and
1819 * resulting map will be called 'my_objec.bss', not even using all 15
1820 * characters allowed by the kernel. Oh well, at least the truncated
1821 * object name is somewhat consistent in this case. But if the map
1822 * name is '.kconfig', we'll still have entirety of '.kconfig' added
1823 * (8 chars) and thus will be left with only first 7 characters of the
1824 * object name ('my_obje'). Happy guessing, user, that the final map
1825 * name will be "my_obje.kconfig".
1826 * Now, with libbpf starting to support arbitrarily named .rodata.*
1827 * and .data.* data sections, it's possible that ELF section name is
1828 * longer than allowed 15 chars, so we now need to be careful to take
1829 * only up to 15 first characters of ELF name, taking no BPF object
1830 * name characters at all. So '.rodata.abracadabra' will result in
1831 * '.rodata.abracad' kernel and user-visible name.
1832 * We need to keep this convoluted logic intact for .data, .bss and
1833 * .rodata maps, but for new custom .data.custom and .rodata.custom
1834 * maps we use their ELF names as is, not prepending bpf_object name
1835 * in front. We still need to truncate them to 15 characters for the
1836 * kernel. Full name can be recovered for such maps by using DATASEC
1837 * BTF type associated with such map's value type, though.
1838 */
1839 if (sfx_len >= BPF_OBJ_NAME_LEN)
1840 sfx_len = BPF_OBJ_NAME_LEN - 1;
1841
1842 /* if there are two or more dots in map name, it's a custom dot map */
1843 if (strchr(real_name + 1, '.') != NULL)
1844 pfx_len = 0;
1845 else
1846 pfx_len = min((size_t)BPF_OBJ_NAME_LEN - sfx_len - 1, strlen(obj->name));
1847
1848 snprintf(map_name, sizeof(map_name), "%.*s%.*s", pfx_len, obj->name,
1849 sfx_len, real_name);
1850
1851 /* sanitise map name to characters allowed by kernel */
1852 for (p = map_name; *p && p < map_name + sizeof(map_name); p++)
1853 if (!isalnum(*p) && *p != '_' && *p != '.')
1854 *p = '_';
1855
1856 return strdup(map_name);
1857 }
1858
1859 static int
1860 map_fill_btf_type_info(struct bpf_object *obj, struct bpf_map *map);
1861
1862 /* Internal BPF map is mmap()'able only if at least one of corresponding
1863 * DATASEC's VARs are to be exposed through BPF skeleton. I.e., it's a GLOBAL
1864 * variable and it's not marked as __hidden (which turns it into, effectively,
1865 * a STATIC variable).
1866 */
1867 static bool map_is_mmapable(struct bpf_object *obj, struct bpf_map *map)
1868 {
1869 const struct btf_type *t, *vt;
1870 struct btf_var_secinfo *vsi;
1871 int i, n;
1872
1873 if (!map->btf_value_type_id)
1874 return false;
1875
1876 t = btf__type_by_id(obj->btf, map->btf_value_type_id);
1877 if (!btf_is_datasec(t))
1878 return false;
1879
1880 vsi = btf_var_secinfos(t);
1881 for (i = 0, n = btf_vlen(t); i < n; i++, vsi++) {
1882 vt = btf__type_by_id(obj->btf, vsi->type);
1883 if (!btf_is_var(vt))
1884 continue;
1885
1886 if (btf_var(vt)->linkage != BTF_VAR_STATIC)
1887 return true;
1888 }
1889
1890 return false;
1891 }
1892
1893 static int
1894 bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type,
1895 const char *real_name, int sec_idx, void *data, size_t data_sz)
1896 {
1897 struct bpf_map_def *def;
1898 struct bpf_map *map;
1899 size_t mmap_sz;
1900 int err;
1901
1902 map = bpf_object__add_map(obj);
1903 if (IS_ERR(map))
1904 return PTR_ERR(map);
1905
1906 map->libbpf_type = type;
1907 map->sec_idx = sec_idx;
1908 map->sec_offset = 0;
1909 map->real_name = strdup(real_name);
1910 map->name = internal_map_name(obj, real_name);
1911 if (!map->real_name || !map->name) {
1912 zfree(&map->real_name);
1913 zfree(&map->name);
1914 return -ENOMEM;
1915 }
1916
1917 def = &map->def;
1918 def->type = BPF_MAP_TYPE_ARRAY;
1919 def->key_size = sizeof(int);
1920 def->value_size = data_sz;
1921 def->max_entries = 1;
1922 def->map_flags = type == LIBBPF_MAP_RODATA || type == LIBBPF_MAP_KCONFIG
1923 ? BPF_F_RDONLY_PROG : 0;
1924
1925 /* failures are fine because of maps like .rodata.str1.1 */
1926 (void) map_fill_btf_type_info(obj, map);
1927
1928 if (map_is_mmapable(obj, map))
1929 def->map_flags |= BPF_F_MMAPABLE;
1930
1931 pr_debug("map '%s' (global data): at sec_idx %d, offset %zu, flags %x.\n",
1932 map->name, map->sec_idx, map->sec_offset, def->map_flags);
1933
1934 mmap_sz = bpf_map_mmap_sz(map);
1935 map->mmaped = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE,
1936 MAP_SHARED | MAP_ANONYMOUS, -1, 0);
1937 if (map->mmaped == MAP_FAILED) {
1938 err = -errno;
1939 map->mmaped = NULL;
1940 pr_warn("failed to alloc map '%s' content buffer: %d\n",
1941 map->name, err);
1942 zfree(&map->real_name);
1943 zfree(&map->name);
1944 return err;
1945 }
1946
1947 if (data)
1948 memcpy(map->mmaped, data, data_sz);
1949
1950 pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name);
1951 return 0;
1952 }
1953
1954 static int bpf_object__init_global_data_maps(struct bpf_object *obj)
1955 {
1956 struct elf_sec_desc *sec_desc;
1957 const char *sec_name;
1958 int err = 0, sec_idx;
1959
1960 /*
1961 * Populate obj->maps with libbpf internal maps.
1962 */
1963 for (sec_idx = 1; sec_idx < obj->efile.sec_cnt; sec_idx++) {
1964 sec_desc = &obj->efile.secs[sec_idx];
1965
1966 /* Skip recognized sections with size 0. */
1967 if (!sec_desc->data || sec_desc->data->d_size == 0)
1968 continue;
1969
1970 switch (sec_desc->sec_type) {
1971 case SEC_DATA:
1972 sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx));
1973 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_DATA,
1974 sec_name, sec_idx,
1975 sec_desc->data->d_buf,
1976 sec_desc->data->d_size);
1977 break;
1978 case SEC_RODATA:
1979 obj->has_rodata = true;
1980 sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx));
1981 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_RODATA,
1982 sec_name, sec_idx,
1983 sec_desc->data->d_buf,
1984 sec_desc->data->d_size);
1985 break;
1986 case SEC_BSS:
1987 sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx));
1988 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_BSS,
1989 sec_name, sec_idx,
1990 NULL,
1991 sec_desc->data->d_size);
1992 break;
1993 default:
1994 /* skip */
1995 break;
1996 }
1997 if (err)
1998 return err;
1999 }
2000 return 0;
2001 }
2002
2003
2004 static struct extern_desc *find_extern_by_name(const struct bpf_object *obj,
2005 const void *name)
2006 {
2007 int i;
2008
2009 for (i = 0; i < obj->nr_extern; i++) {
2010 if (strcmp(obj->externs[i].name, name) == 0)
2011 return &obj->externs[i];
2012 }
2013 return NULL;
2014 }
2015
2016 static struct extern_desc *find_extern_by_name_with_len(const struct bpf_object *obj,
2017 const void *name, int len)
2018 {
2019 const char *ext_name;
2020 int i;
2021
2022 for (i = 0; i < obj->nr_extern; i++) {
2023 ext_name = obj->externs[i].name;
2024 if (strlen(ext_name) == len && strncmp(ext_name, name, len) == 0)
2025 return &obj->externs[i];
2026 }
2027 return NULL;
2028 }
2029
2030 static int set_kcfg_value_tri(struct extern_desc *ext, void *ext_val,
2031 char value)
2032 {
2033 switch (ext->kcfg.type) {
2034 case KCFG_BOOL:
2035 if (value == 'm') {
2036 pr_warn("extern (kcfg) '%s': value '%c' implies tristate or char type\n",
2037 ext->name, value);
2038 return -EINVAL;
2039 }
2040 *(bool *)ext_val = value == 'y' ? true : false;
2041 break;
2042 case KCFG_TRISTATE:
2043 if (value == 'y')
2044 *(enum libbpf_tristate *)ext_val = TRI_YES;
2045 else if (value == 'm')
2046 *(enum libbpf_tristate *)ext_val = TRI_MODULE;
2047 else /* value == 'n' */
2048 *(enum libbpf_tristate *)ext_val = TRI_NO;
2049 break;
2050 case KCFG_CHAR:
2051 *(char *)ext_val = value;
2052 break;
2053 case KCFG_UNKNOWN:
2054 case KCFG_INT:
2055 case KCFG_CHAR_ARR:
2056 default:
2057 pr_warn("extern (kcfg) '%s': value '%c' implies bool, tristate, or char type\n",
2058 ext->name, value);
2059 return -EINVAL;
2060 }
2061 ext->is_set = true;
2062 return 0;
2063 }
2064
2065 static int set_kcfg_value_str(struct extern_desc *ext, char *ext_val,
2066 const char *value)
2067 {
2068 size_t len;
2069
2070 if (ext->kcfg.type != KCFG_CHAR_ARR) {
2071 pr_warn("extern (kcfg) '%s': value '%s' implies char array type\n",
2072 ext->name, value);
2073 return -EINVAL;
2074 }
2075
2076 len = strlen(value);
2077 if (value[len - 1] != '"') {
2078 pr_warn("extern (kcfg) '%s': invalid string config '%s'\n",
2079 ext->name, value);
2080 return -EINVAL;
2081 }
2082
2083 /* strip quotes */
2084 len -= 2;
2085 if (len >= ext->kcfg.sz) {
2086 pr_warn("extern (kcfg) '%s': long string '%s' of (%zu bytes) truncated to %d bytes\n",
2087 ext->name, value, len, ext->kcfg.sz - 1);
2088 len = ext->kcfg.sz - 1;
2089 }
2090 memcpy(ext_val, value + 1, len);
2091 ext_val[len] = '\0';
2092 ext->is_set = true;
2093 return 0;
2094 }
2095
2096 static int parse_u64(const char *value, __u64 *res)
2097 {
2098 char *value_end;
2099 int err;
2100
2101 errno = 0;
2102 *res = strtoull(value, &value_end, 0);
2103 if (errno) {
2104 err = -errno;
2105 pr_warn("failed to parse '%s' as integer: %d\n", value, err);
2106 return err;
2107 }
2108 if (*value_end) {
2109 pr_warn("failed to parse '%s' as integer completely\n", value);
2110 return -EINVAL;
2111 }
2112 return 0;
2113 }
2114
2115 static bool is_kcfg_value_in_range(const struct extern_desc *ext, __u64 v)
2116 {
2117 int bit_sz = ext->kcfg.sz * 8;
2118
2119 if (ext->kcfg.sz == 8)
2120 return true;
2121
2122 /* Validate that value stored in u64 fits in integer of `ext->sz`
2123 * bytes size without any loss of information. If the target integer
2124 * is signed, we rely on the following limits of integer type of
2125 * Y bits and subsequent transformation:
2126 *
2127 * -2^(Y-1) <= X <= 2^(Y-1) - 1
2128 * 0 <= X + 2^(Y-1) <= 2^Y - 1
2129 * 0 <= X + 2^(Y-1) < 2^Y
2130 *
2131 * For unsigned target integer, check that all the (64 - Y) bits are
2132 * zero.
2133 */
2134 if (ext->kcfg.is_signed)
2135 return v + (1ULL << (bit_sz - 1)) < (1ULL << bit_sz);
2136 else
2137 return (v >> bit_sz) == 0;
2138 }
2139
2140 static int set_kcfg_value_num(struct extern_desc *ext, void *ext_val,
2141 __u64 value)
2142 {
2143 if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR &&
2144 ext->kcfg.type != KCFG_BOOL) {
2145 pr_warn("extern (kcfg) '%s': value '%llu' implies integer, char, or boolean type\n",
2146 ext->name, (unsigned long long)value);
2147 return -EINVAL;
2148 }
2149 if (ext->kcfg.type == KCFG_BOOL && value > 1) {
2150 pr_warn("extern (kcfg) '%s': value '%llu' isn't boolean compatible\n",
2151 ext->name, (unsigned long long)value);
2152 return -EINVAL;
2153
2154 }
2155 if (!is_kcfg_value_in_range(ext, value)) {
2156 pr_warn("extern (kcfg) '%s': value '%llu' doesn't fit in %d bytes\n",
2157 ext->name, (unsigned long long)value, ext->kcfg.sz);
2158 return -ERANGE;
2159 }
2160 switch (ext->kcfg.sz) {
2161 case 1:
2162 *(__u8 *)ext_val = value;
2163 break;
2164 case 2:
2165 *(__u16 *)ext_val = value;
2166 break;
2167 case 4:
2168 *(__u32 *)ext_val = value;
2169 break;
2170 case 8:
2171 *(__u64 *)ext_val = value;
2172 break;
2173 default:
2174 return -EINVAL;
2175 }
2176 ext->is_set = true;
2177 return 0;
2178 }
2179
2180 static int bpf_object__process_kconfig_line(struct bpf_object *obj,
2181 char *buf, void *data)
2182 {
2183 struct extern_desc *ext;
2184 char *sep, *value;
2185 int len, err = 0;
2186 void *ext_val;
2187 __u64 num;
2188
2189 if (!str_has_pfx(buf, "CONFIG_"))
2190 return 0;
2191
2192 sep = strchr(buf, '=');
2193 if (!sep) {
2194 pr_warn("failed to parse '%s': no separator\n", buf);
2195 return -EINVAL;
2196 }
2197
2198 /* Trim ending '\n' */
2199 len = strlen(buf);
2200 if (buf[len - 1] == '\n')
2201 buf[len - 1] = '\0';
2202 /* Split on '=' and ensure that a value is present. */
2203 *sep = '\0';
2204 if (!sep[1]) {
2205 *sep = '=';
2206 pr_warn("failed to parse '%s': no value\n", buf);
2207 return -EINVAL;
2208 }
2209
2210 ext = find_extern_by_name(obj, buf);
2211 if (!ext || ext->is_set)
2212 return 0;
2213
2214 ext_val = data + ext->kcfg.data_off;
2215 value = sep + 1;
2216
2217 switch (*value) {
2218 case 'y': case 'n': case 'm':
2219 err = set_kcfg_value_tri(ext, ext_val, *value);
2220 break;
2221 case '"':
2222 err = set_kcfg_value_str(ext, ext_val, value);
2223 break;
2224 default:
2225 /* assume integer */
2226 err = parse_u64(value, &num);
2227 if (err) {
2228 pr_warn("extern (kcfg) '%s': value '%s' isn't a valid integer\n", ext->name, value);
2229 return err;
2230 }
2231 if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR) {
2232 pr_warn("extern (kcfg) '%s': value '%s' implies integer type\n", ext->name, value);
2233 return -EINVAL;
2234 }
2235 err = set_kcfg_value_num(ext, ext_val, num);
2236 break;
2237 }
2238 if (err)
2239 return err;
2240 pr_debug("extern (kcfg) '%s': set to %s\n", ext->name, value);
2241 return 0;
2242 }
2243
2244 static int bpf_object__read_kconfig_file(struct bpf_object *obj, void *data)
2245 {
2246 char buf[PATH_MAX];
2247 struct utsname uts;
2248 int len, err = 0;
2249 gzFile file;
2250
2251 uname(&uts);
2252 len = snprintf(buf, PATH_MAX, "/boot/config-%s", uts.release);
2253 if (len < 0)
2254 return -EINVAL;
2255 else if (len >= PATH_MAX)
2256 return -ENAMETOOLONG;
2257
2258 /* gzopen also accepts uncompressed files. */
2259 file = gzopen(buf, "re");
2260 if (!file)
2261 file = gzopen("/proc/config.gz", "re");
2262
2263 if (!file) {
2264 pr_warn("failed to open system Kconfig\n");
2265 return -ENOENT;
2266 }
2267
2268 while (gzgets(file, buf, sizeof(buf))) {
2269 err = bpf_object__process_kconfig_line(obj, buf, data);
2270 if (err) {
2271 pr_warn("error parsing system Kconfig line '%s': %d\n",
2272 buf, err);
2273 goto out;
2274 }
2275 }
2276
2277 out:
2278 gzclose(file);
2279 return err;
2280 }
2281
2282 static int bpf_object__read_kconfig_mem(struct bpf_object *obj,
2283 const char *config, void *data)
2284 {
2285 char buf[PATH_MAX];
2286 int err = 0;
2287 FILE *file;
2288
2289 file = fmemopen((void *)config, strlen(config), "r");
2290 if (!file) {
2291 err = -errno;
2292 pr_warn("failed to open in-memory Kconfig: %d\n", err);
2293 return err;
2294 }
2295
2296 while (fgets(buf, sizeof(buf), file)) {
2297 err = bpf_object__process_kconfig_line(obj, buf, data);
2298 if (err) {
2299 pr_warn("error parsing in-memory Kconfig line '%s': %d\n",
2300 buf, err);
2301 break;
2302 }
2303 }
2304
2305 fclose(file);
2306 return err;
2307 }
2308
2309 static int bpf_object__init_kconfig_map(struct bpf_object *obj)
2310 {
2311 struct extern_desc *last_ext = NULL, *ext;
2312 size_t map_sz;
2313 int i, err;
2314
2315 for (i = 0; i < obj->nr_extern; i++) {
2316 ext = &obj->externs[i];
2317 if (ext->type == EXT_KCFG)
2318 last_ext = ext;
2319 }
2320
2321 if (!last_ext)
2322 return 0;
2323
2324 map_sz = last_ext->kcfg.data_off + last_ext->kcfg.sz;
2325 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_KCONFIG,
2326 ".kconfig", obj->efile.symbols_shndx,
2327 NULL, map_sz);
2328 if (err)
2329 return err;
2330
2331 obj->kconfig_map_idx = obj->nr_maps - 1;
2332
2333 return 0;
2334 }
2335
2336 const struct btf_type *
2337 skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id)
2338 {
2339 const struct btf_type *t = btf__type_by_id(btf, id);
2340
2341 if (res_id)
2342 *res_id = id;
2343
2344 while (btf_is_mod(t) || btf_is_typedef(t)) {
2345 if (res_id)
2346 *res_id = t->type;
2347 t = btf__type_by_id(btf, t->type);
2348 }
2349
2350 return t;
2351 }
2352
2353 static const struct btf_type *
2354 resolve_func_ptr(const struct btf *btf, __u32 id, __u32 *res_id)
2355 {
2356 const struct btf_type *t;
2357
2358 t = skip_mods_and_typedefs(btf, id, NULL);
2359 if (!btf_is_ptr(t))
2360 return NULL;
2361
2362 t = skip_mods_and_typedefs(btf, t->type, res_id);
2363
2364 return btf_is_func_proto(t) ? t : NULL;
2365 }
2366
2367 static const char *__btf_kind_str(__u16 kind)
2368 {
2369 switch (kind) {
2370 case BTF_KIND_UNKN: return "void";
2371 case BTF_KIND_INT: return "int";
2372 case BTF_KIND_PTR: return "ptr";
2373 case BTF_KIND_ARRAY: return "array";
2374 case BTF_KIND_STRUCT: return "struct";
2375 case BTF_KIND_UNION: return "union";
2376 case BTF_KIND_ENUM: return "enum";
2377 case BTF_KIND_FWD: return "fwd";
2378 case BTF_KIND_TYPEDEF: return "typedef";
2379 case BTF_KIND_VOLATILE: return "volatile";
2380 case BTF_KIND_CONST: return "const";
2381 case BTF_KIND_RESTRICT: return "restrict";
2382 case BTF_KIND_FUNC: return "func";
2383 case BTF_KIND_FUNC_PROTO: return "func_proto";
2384 case BTF_KIND_VAR: return "var";
2385 case BTF_KIND_DATASEC: return "datasec";
2386 case BTF_KIND_FLOAT: return "float";
2387 case BTF_KIND_DECL_TAG: return "decl_tag";
2388 case BTF_KIND_TYPE_TAG: return "type_tag";
2389 case BTF_KIND_ENUM64: return "enum64";
2390 default: return "unknown";
2391 }
2392 }
2393
2394 const char *btf_kind_str(const struct btf_type *t)
2395 {
2396 return __btf_kind_str(btf_kind(t));
2397 }
2398
2399 /*
2400 * Fetch integer attribute of BTF map definition. Such attributes are
2401 * represented using a pointer to an array, in which dimensionality of array
2402 * encodes specified integer value. E.g., int (*type)[BPF_MAP_TYPE_ARRAY];
2403 * encodes `type => BPF_MAP_TYPE_ARRAY` key/value pair completely using BTF
2404 * type definition, while using only sizeof(void *) space in ELF data section.
2405 */
2406 static bool get_map_field_int(const char *map_name, const struct btf *btf,
2407 const struct btf_member *m, __u32 *res)
2408 {
2409 const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL);
2410 const char *name = btf__name_by_offset(btf, m->name_off);
2411 const struct btf_array *arr_info;
2412 const struct btf_type *arr_t;
2413
2414 if (!btf_is_ptr(t)) {
2415 pr_warn("map '%s': attr '%s': expected PTR, got %s.\n",
2416 map_name, name, btf_kind_str(t));
2417 return false;
2418 }
2419
2420 arr_t = btf__type_by_id(btf, t->type);
2421 if (!arr_t) {
2422 pr_warn("map '%s': attr '%s': type [%u] not found.\n",
2423 map_name, name, t->type);
2424 return false;
2425 }
2426 if (!btf_is_array(arr_t)) {
2427 pr_warn("map '%s': attr '%s': expected ARRAY, got %s.\n",
2428 map_name, name, btf_kind_str(arr_t));
2429 return false;
2430 }
2431 arr_info = btf_array(arr_t);
2432 *res = arr_info->nelems;
2433 return true;
2434 }
2435
2436 static bool get_map_field_long(const char *map_name, const struct btf *btf,
2437 const struct btf_member *m, __u64 *res)
2438 {
2439 const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL);
2440 const char *name = btf__name_by_offset(btf, m->name_off);
2441
2442 if (btf_is_ptr(t)) {
2443 __u32 res32;
2444 bool ret;
2445
2446 ret = get_map_field_int(map_name, btf, m, &res32);
2447 if (ret)
2448 *res = (__u64)res32;
2449 return ret;
2450 }
2451
2452 if (!btf_is_enum(t) && !btf_is_enum64(t)) {
2453 pr_warn("map '%s': attr '%s': expected ENUM or ENUM64, got %s.\n",
2454 map_name, name, btf_kind_str(t));
2455 return false;
2456 }
2457
2458 if (btf_vlen(t) != 1) {
2459 pr_warn("map '%s': attr '%s': invalid __ulong\n",
2460 map_name, name);
2461 return false;
2462 }
2463
2464 if (btf_is_enum(t)) {
2465 const struct btf_enum *e = btf_enum(t);
2466
2467 *res = e->val;
2468 } else {
2469 const struct btf_enum64 *e = btf_enum64(t);
2470
2471 *res = btf_enum64_value(e);
2472 }
2473 return true;
2474 }
2475
2476 static int pathname_concat(char *buf, size_t buf_sz, const char *path, const char *name)
2477 {
2478 int len;
2479
2480 len = snprintf(buf, buf_sz, "%s/%s", path, name);
2481 if (len < 0)
2482 return -EINVAL;
2483 if (len >= buf_sz)
2484 return -ENAMETOOLONG;
2485
2486 return 0;
2487 }
2488
2489 static int build_map_pin_path(struct bpf_map *map, const char *path)
2490 {
2491 char buf[PATH_MAX];
2492 int err;
2493
2494 if (!path)
2495 path = BPF_FS_DEFAULT_PATH;
2496
2497 err = pathname_concat(buf, sizeof(buf), path, bpf_map__name(map));
2498 if (err)
2499 return err;
2500
2501 return bpf_map__set_pin_path(map, buf);
2502 }
2503
2504 /* should match definition in bpf_helpers.h */
2505 enum libbpf_pin_type {
2506 LIBBPF_PIN_NONE,
2507 /* PIN_BY_NAME: pin maps by name (in /sys/fs/bpf by default) */
2508 LIBBPF_PIN_BY_NAME,
2509 };
2510
2511 int parse_btf_map_def(const char *map_name, struct btf *btf,
2512 const struct btf_type *def_t, bool strict,
2513 struct btf_map_def *map_def, struct btf_map_def *inner_def)
2514 {
2515 const struct btf_type *t;
2516 const struct btf_member *m;
2517 bool is_inner = inner_def == NULL;
2518 int vlen, i;
2519
2520 vlen = btf_vlen(def_t);
2521 m = btf_members(def_t);
2522 for (i = 0; i < vlen; i++, m++) {
2523 const char *name = btf__name_by_offset(btf, m->name_off);
2524
2525 if (!name) {
2526 pr_warn("map '%s': invalid field #%d.\n", map_name, i);
2527 return -EINVAL;
2528 }
2529 if (strcmp(name, "type") == 0) {
2530 if (!get_map_field_int(map_name, btf, m, &map_def->map_type))
2531 return -EINVAL;
2532 map_def->parts |= MAP_DEF_MAP_TYPE;
2533 } else if (strcmp(name, "max_entries") == 0) {
2534 if (!get_map_field_int(map_name, btf, m, &map_def->max_entries))
2535 return -EINVAL;
2536 map_def->parts |= MAP_DEF_MAX_ENTRIES;
2537 } else if (strcmp(name, "map_flags") == 0) {
2538 if (!get_map_field_int(map_name, btf, m, &map_def->map_flags))
2539 return -EINVAL;
2540 map_def->parts |= MAP_DEF_MAP_FLAGS;
2541 } else if (strcmp(name, "numa_node") == 0) {
2542 if (!get_map_field_int(map_name, btf, m, &map_def->numa_node))
2543 return -EINVAL;
2544 map_def->parts |= MAP_DEF_NUMA_NODE;
2545 } else if (strcmp(name, "key_size") == 0) {
2546 __u32 sz;
2547
2548 if (!get_map_field_int(map_name, btf, m, &sz))
2549 return -EINVAL;
2550 if (map_def->key_size && map_def->key_size != sz) {
2551 pr_warn("map '%s': conflicting key size %u != %u.\n",
2552 map_name, map_def->key_size, sz);
2553 return -EINVAL;
2554 }
2555 map_def->key_size = sz;
2556 map_def->parts |= MAP_DEF_KEY_SIZE;
2557 } else if (strcmp(name, "key") == 0) {
2558 __s64 sz;
2559
2560 t = btf__type_by_id(btf, m->type);
2561 if (!t) {
2562 pr_warn("map '%s': key type [%d] not found.\n",
2563 map_name, m->type);
2564 return -EINVAL;
2565 }
2566 if (!btf_is_ptr(t)) {
2567 pr_warn("map '%s': key spec is not PTR: %s.\n",
2568 map_name, btf_kind_str(t));
2569 return -EINVAL;
2570 }
2571 sz = btf__resolve_size(btf, t->type);
2572 if (sz < 0) {
2573 pr_warn("map '%s': can't determine key size for type [%u]: %zd.\n",
2574 map_name, t->type, (ssize_t)sz);
2575 return sz;
2576 }
2577 if (map_def->key_size && map_def->key_size != sz) {
2578 pr_warn("map '%s': conflicting key size %u != %zd.\n",
2579 map_name, map_def->key_size, (ssize_t)sz);
2580 return -EINVAL;
2581 }
2582 map_def->key_size = sz;
2583 map_def->key_type_id = t->type;
2584 map_def->parts |= MAP_DEF_KEY_SIZE | MAP_DEF_KEY_TYPE;
2585 } else if (strcmp(name, "value_size") == 0) {
2586 __u32 sz;
2587
2588 if (!get_map_field_int(map_name, btf, m, &sz))
2589 return -EINVAL;
2590 if (map_def->value_size && map_def->value_size != sz) {
2591 pr_warn("map '%s': conflicting value size %u != %u.\n",
2592 map_name, map_def->value_size, sz);
2593 return -EINVAL;
2594 }
2595 map_def->value_size = sz;
2596 map_def->parts |= MAP_DEF_VALUE_SIZE;
2597 } else if (strcmp(name, "value") == 0) {
2598 __s64 sz;
2599
2600 t = btf__type_by_id(btf, m->type);
2601 if (!t) {
2602 pr_warn("map '%s': value type [%d] not found.\n",
2603 map_name, m->type);
2604 return -EINVAL;
2605 }
2606 if (!btf_is_ptr(t)) {
2607 pr_warn("map '%s': value spec is not PTR: %s.\n",
2608 map_name, btf_kind_str(t));
2609 return -EINVAL;
2610 }
2611 sz = btf__resolve_size(btf, t->type);
2612 if (sz < 0) {
2613 pr_warn("map '%s': can't determine value size for type [%u]: %zd.\n",
2614 map_name, t->type, (ssize_t)sz);
2615 return sz;
2616 }
2617 if (map_def->value_size && map_def->value_size != sz) {
2618 pr_warn("map '%s': conflicting value size %u != %zd.\n",
2619 map_name, map_def->value_size, (ssize_t)sz);
2620 return -EINVAL;
2621 }
2622 map_def->value_size = sz;
2623 map_def->value_type_id = t->type;
2624 map_def->parts |= MAP_DEF_VALUE_SIZE | MAP_DEF_VALUE_TYPE;
2625 }
2626 else if (strcmp(name, "values") == 0) {
2627 bool is_map_in_map = bpf_map_type__is_map_in_map(map_def->map_type);
2628 bool is_prog_array = map_def->map_type == BPF_MAP_TYPE_PROG_ARRAY;
2629 const char *desc = is_map_in_map ? "map-in-map inner" : "prog-array value";
2630 char inner_map_name[128];
2631 int err;
2632
2633 if (is_inner) {
2634 pr_warn("map '%s': multi-level inner maps not supported.\n",
2635 map_name);
2636 return -ENOTSUP;
2637 }
2638 if (i != vlen - 1) {
2639 pr_warn("map '%s': '%s' member should be last.\n",
2640 map_name, name);
2641 return -EINVAL;
2642 }
2643 if (!is_map_in_map && !is_prog_array) {
2644 pr_warn("map '%s': should be map-in-map or prog-array.\n",
2645 map_name);
2646 return -ENOTSUP;
2647 }
2648 if (map_def->value_size && map_def->value_size != 4) {
2649 pr_warn("map '%s': conflicting value size %u != 4.\n",
2650 map_name, map_def->value_size);
2651 return -EINVAL;
2652 }
2653 map_def->value_size = 4;
2654 t = btf__type_by_id(btf, m->type);
2655 if (!t) {
2656 pr_warn("map '%s': %s type [%d] not found.\n",
2657 map_name, desc, m->type);
2658 return -EINVAL;
2659 }
2660 if (!btf_is_array(t) || btf_array(t)->nelems) {
2661 pr_warn("map '%s': %s spec is not a zero-sized array.\n",
2662 map_name, desc);
2663 return -EINVAL;
2664 }
2665 t = skip_mods_and_typedefs(btf, btf_array(t)->type, NULL);
2666 if (!btf_is_ptr(t)) {
2667 pr_warn("map '%s': %s def is of unexpected kind %s.\n",
2668 map_name, desc, btf_kind_str(t));
2669 return -EINVAL;
2670 }
2671 t = skip_mods_and_typedefs(btf, t->type, NULL);
2672 if (is_prog_array) {
2673 if (!btf_is_func_proto(t)) {
2674 pr_warn("map '%s': prog-array value def is of unexpected kind %s.\n",
2675 map_name, btf_kind_str(t));
2676 return -EINVAL;
2677 }
2678 continue;
2679 }
2680 if (!btf_is_struct(t)) {
2681 pr_warn("map '%s': map-in-map inner def is of unexpected kind %s.\n",
2682 map_name, btf_kind_str(t));
2683 return -EINVAL;
2684 }
2685
2686 snprintf(inner_map_name, sizeof(inner_map_name), "%s.inner", map_name);
2687 err = parse_btf_map_def(inner_map_name, btf, t, strict, inner_def, NULL);
2688 if (err)
2689 return err;
2690
2691 map_def->parts |= MAP_DEF_INNER_MAP;
2692 } else if (strcmp(name, "pinning") == 0) {
2693 __u32 val;
2694
2695 if (is_inner) {
2696 pr_warn("map '%s': inner def can't be pinned.\n", map_name);
2697 return -EINVAL;
2698 }
2699 if (!get_map_field_int(map_name, btf, m, &val))
2700 return -EINVAL;
2701 if (val != LIBBPF_PIN_NONE && val != LIBBPF_PIN_BY_NAME) {
2702 pr_warn("map '%s': invalid pinning value %u.\n",
2703 map_name, val);
2704 return -EINVAL;
2705 }
2706 map_def->pinning = val;
2707 map_def->parts |= MAP_DEF_PINNING;
2708 } else if (strcmp(name, "map_extra") == 0) {
2709 __u64 map_extra;
2710
2711 if (!get_map_field_long(map_name, btf, m, &map_extra))
2712 return -EINVAL;
2713 map_def->map_extra = map_extra;
2714 map_def->parts |= MAP_DEF_MAP_EXTRA;
2715 } else {
2716 if (strict) {
2717 pr_warn("map '%s': unknown field '%s'.\n", map_name, name);
2718 return -ENOTSUP;
2719 }
2720 pr_debug("map '%s': ignoring unknown field '%s'.\n", map_name, name);
2721 }
2722 }
2723
2724 if (map_def->map_type == BPF_MAP_TYPE_UNSPEC) {
2725 pr_warn("map '%s': map type isn't specified.\n", map_name);
2726 return -EINVAL;
2727 }
2728
2729 return 0;
2730 }
2731
2732 static size_t adjust_ringbuf_sz(size_t sz)
2733 {
2734 __u32 page_sz = sysconf(_SC_PAGE_SIZE);
2735 __u32 mul;
2736
2737 /* if user forgot to set any size, make sure they see error */
2738 if (sz == 0)
2739 return 0;
2740 /* Kernel expects BPF_MAP_TYPE_RINGBUF's max_entries to be
2741 * a power-of-2 multiple of kernel's page size. If user diligently
2742 * satisified these conditions, pass the size through.
2743 */
2744 if ((sz % page_sz) == 0 && is_pow_of_2(sz / page_sz))
2745 return sz;
2746
2747 /* Otherwise find closest (page_sz * power_of_2) product bigger than
2748 * user-set size to satisfy both user size request and kernel
2749 * requirements and substitute correct max_entries for map creation.
2750 */
2751 for (mul = 1; mul <= UINT_MAX / page_sz; mul <<= 1) {
2752 if (mul * page_sz > sz)
2753 return mul * page_sz;
2754 }
2755
2756 /* if it's impossible to satisfy the conditions (i.e., user size is
2757 * very close to UINT_MAX but is not a power-of-2 multiple of
2758 * page_size) then just return original size and let kernel reject it
2759 */
2760 return sz;
2761 }
2762
2763 static bool map_is_ringbuf(const struct bpf_map *map)
2764 {
2765 return map->def.type == BPF_MAP_TYPE_RINGBUF ||
2766 map->def.type == BPF_MAP_TYPE_USER_RINGBUF;
2767 }
2768
2769 static void fill_map_from_def(struct bpf_map *map, const struct btf_map_def *def)
2770 {
2771 map->def.type = def->map_type;
2772 map->def.key_size = def->key_size;
2773 map->def.value_size = def->value_size;
2774 map->def.max_entries = def->max_entries;
2775 map->def.map_flags = def->map_flags;
2776 map->map_extra = def->map_extra;
2777
2778 map->numa_node = def->numa_node;
2779 map->btf_key_type_id = def->key_type_id;
2780 map->btf_value_type_id = def->value_type_id;
2781
2782 /* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */
2783 if (map_is_ringbuf(map))
2784 map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries);
2785
2786 if (def->parts & MAP_DEF_MAP_TYPE)
2787 pr_debug("map '%s': found type = %u.\n", map->name, def->map_type);
2788
2789 if (def->parts & MAP_DEF_KEY_TYPE)
2790 pr_debug("map '%s': found key [%u], sz = %u.\n",
2791 map->name, def->key_type_id, def->key_size);
2792 else if (def->parts & MAP_DEF_KEY_SIZE)
2793 pr_debug("map '%s': found key_size = %u.\n", map->name, def->key_size);
2794
2795 if (def->parts & MAP_DEF_VALUE_TYPE)
2796 pr_debug("map '%s': found value [%u], sz = %u.\n",
2797 map->name, def->value_type_id, def->value_size);
2798 else if (def->parts & MAP_DEF_VALUE_SIZE)
2799 pr_debug("map '%s': found value_size = %u.\n", map->name, def->value_size);
2800
2801 if (def->parts & MAP_DEF_MAX_ENTRIES)
2802 pr_debug("map '%s': found max_entries = %u.\n", map->name, def->max_entries);
2803 if (def->parts & MAP_DEF_MAP_FLAGS)
2804 pr_debug("map '%s': found map_flags = 0x%x.\n", map->name, def->map_flags);
2805 if (def->parts & MAP_DEF_MAP_EXTRA)
2806 pr_debug("map '%s': found map_extra = 0x%llx.\n", map->name,
2807 (unsigned long long)def->map_extra);
2808 if (def->parts & MAP_DEF_PINNING)
2809 pr_debug("map '%s': found pinning = %u.\n", map->name, def->pinning);
2810 if (def->parts & MAP_DEF_NUMA_NODE)
2811 pr_debug("map '%s': found numa_node = %u.\n", map->name, def->numa_node);
2812
2813 if (def->parts & MAP_DEF_INNER_MAP)
2814 pr_debug("map '%s': found inner map definition.\n", map->name);
2815 }
2816
2817 static const char *btf_var_linkage_str(__u32 linkage)
2818 {
2819 switch (linkage) {
2820 case BTF_VAR_STATIC: return "static";
2821 case BTF_VAR_GLOBAL_ALLOCATED: return "global";
2822 case BTF_VAR_GLOBAL_EXTERN: return "extern";
2823 default: return "unknown";
2824 }
2825 }
2826
2827 static int bpf_object__init_user_btf_map(struct bpf_object *obj,
2828 const struct btf_type *sec,
2829 int var_idx, int sec_idx,
2830 const Elf_Data *data, bool strict,
2831 const char *pin_root_path)
2832 {
2833 struct btf_map_def map_def = {}, inner_def = {};
2834 const struct btf_type *var, *def;
2835 const struct btf_var_secinfo *vi;
2836 const struct btf_var *var_extra;
2837 const char *map_name;
2838 struct bpf_map *map;
2839 int err;
2840
2841 vi = btf_var_secinfos(sec) + var_idx;
2842 var = btf__type_by_id(obj->btf, vi->type);
2843 var_extra = btf_var(var);
2844 map_name = btf__name_by_offset(obj->btf, var->name_off);
2845
2846 if (map_name == NULL || map_name[0] == '\0') {
2847 pr_warn("map #%d: empty name.\n", var_idx);
2848 return -EINVAL;
2849 }
2850 if ((__u64)vi->offset + vi->size > data->d_size) {
2851 pr_warn("map '%s' BTF data is corrupted.\n", map_name);
2852 return -EINVAL;
2853 }
2854 if (!btf_is_var(var)) {
2855 pr_warn("map '%s': unexpected var kind %s.\n",
2856 map_name, btf_kind_str(var));
2857 return -EINVAL;
2858 }
2859 if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED) {
2860 pr_warn("map '%s': unsupported map linkage %s.\n",
2861 map_name, btf_var_linkage_str(var_extra->linkage));
2862 return -EOPNOTSUPP;
2863 }
2864
2865 def = skip_mods_and_typedefs(obj->btf, var->type, NULL);
2866 if (!btf_is_struct(def)) {
2867 pr_warn("map '%s': unexpected def kind %s.\n",
2868 map_name, btf_kind_str(var));
2869 return -EINVAL;
2870 }
2871 if (def->size > vi->size) {
2872 pr_warn("map '%s': invalid def size.\n", map_name);
2873 return -EINVAL;
2874 }
2875
2876 map = bpf_object__add_map(obj);
2877 if (IS_ERR(map))
2878 return PTR_ERR(map);
2879 map->name = strdup(map_name);
2880 if (!map->name) {
2881 pr_warn("map '%s': failed to alloc map name.\n", map_name);
2882 return -ENOMEM;
2883 }
2884 map->libbpf_type = LIBBPF_MAP_UNSPEC;
2885 map->def.type = BPF_MAP_TYPE_UNSPEC;
2886 map->sec_idx = sec_idx;
2887 map->sec_offset = vi->offset;
2888 map->btf_var_idx = var_idx;
2889 pr_debug("map '%s': at sec_idx %d, offset %zu.\n",
2890 map_name, map->sec_idx, map->sec_offset);
2891
2892 err = parse_btf_map_def(map->name, obj->btf, def, strict, &map_def, &inner_def);
2893 if (err)
2894 return err;
2895
2896 fill_map_from_def(map, &map_def);
2897
2898 if (map_def.pinning == LIBBPF_PIN_BY_NAME) {
2899 err = build_map_pin_path(map, pin_root_path);
2900 if (err) {
2901 pr_warn("map '%s': couldn't build pin path.\n", map->name);
2902 return err;
2903 }
2904 }
2905
2906 if (map_def.parts & MAP_DEF_INNER_MAP) {
2907 map->inner_map = calloc(1, sizeof(*map->inner_map));
2908 if (!map->inner_map)
2909 return -ENOMEM;
2910 map->inner_map->fd = create_placeholder_fd();
2911 if (map->inner_map->fd < 0)
2912 return map->inner_map->fd;
2913 map->inner_map->sec_idx = sec_idx;
2914 map->inner_map->name = malloc(strlen(map_name) + sizeof(".inner") + 1);
2915 if (!map->inner_map->name)
2916 return -ENOMEM;
2917 sprintf(map->inner_map->name, "%s.inner", map_name);
2918
2919 fill_map_from_def(map->inner_map, &inner_def);
2920 }
2921
2922 err = map_fill_btf_type_info(obj, map);
2923 if (err)
2924 return err;
2925
2926 return 0;
2927 }
2928
2929 static int init_arena_map_data(struct bpf_object *obj, struct bpf_map *map,
2930 const char *sec_name, int sec_idx,
2931 void *data, size_t data_sz)
2932 {
2933 const long page_sz = sysconf(_SC_PAGE_SIZE);
2934 size_t mmap_sz;
2935
2936 mmap_sz = bpf_map_mmap_sz(obj->arena_map);
2937 if (roundup(data_sz, page_sz) > mmap_sz) {
2938 pr_warn("elf: sec '%s': declared ARENA map size (%zu) is too small to hold global __arena variables of size %zu\n",
2939 sec_name, mmap_sz, data_sz);
2940 return -E2BIG;
2941 }
2942
2943 obj->arena_data = malloc(data_sz);
2944 if (!obj->arena_data)
2945 return -ENOMEM;
2946 memcpy(obj->arena_data, data, data_sz);
2947 obj->arena_data_sz = data_sz;
2948
2949 /* make bpf_map__init_value() work for ARENA maps */
2950 map->mmaped = obj->arena_data;
2951
2952 return 0;
2953 }
2954
2955 static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict,
2956 const char *pin_root_path)
2957 {
2958 const struct btf_type *sec = NULL;
2959 int nr_types, i, vlen, err;
2960 const struct btf_type *t;
2961 const char *name;
2962 Elf_Data *data;
2963 Elf_Scn *scn;
2964
2965 if (obj->efile.btf_maps_shndx < 0)
2966 return 0;
2967
2968 scn = elf_sec_by_idx(obj, obj->efile.btf_maps_shndx);
2969 data = elf_sec_data(obj, scn);
2970 if (!scn || !data) {
2971 pr_warn("elf: failed to get %s map definitions for %s\n",
2972 MAPS_ELF_SEC, obj->path);
2973 return -EINVAL;
2974 }
2975
2976 nr_types = btf__type_cnt(obj->btf);
2977 for (i = 1; i < nr_types; i++) {
2978 t = btf__type_by_id(obj->btf, i);
2979 if (!btf_is_datasec(t))
2980 continue;
2981 name = btf__name_by_offset(obj->btf, t->name_off);
2982 if (strcmp(name, MAPS_ELF_SEC) == 0) {
2983 sec = t;
2984 obj->efile.btf_maps_sec_btf_id = i;
2985 break;
2986 }
2987 }
2988
2989 if (!sec) {
2990 pr_warn("DATASEC '%s' not found.\n", MAPS_ELF_SEC);
2991 return -ENOENT;
2992 }
2993
2994 vlen = btf_vlen(sec);
2995 for (i = 0; i < vlen; i++) {
2996 err = bpf_object__init_user_btf_map(obj, sec, i,
2997 obj->efile.btf_maps_shndx,
2998 data, strict,
2999 pin_root_path);
3000 if (err)
3001 return err;
3002 }
3003
3004 for (i = 0; i < obj->nr_maps; i++) {
3005 struct bpf_map *map = &obj->maps[i];
3006
3007 if (map->def.type != BPF_MAP_TYPE_ARENA)
3008 continue;
3009
3010 if (obj->arena_map) {
3011 pr_warn("map '%s': only single ARENA map is supported (map '%s' is also ARENA)\n",
3012 map->name, obj->arena_map->name);
3013 return -EINVAL;
3014 }
3015 obj->arena_map = map;
3016
3017 if (obj->efile.arena_data) {
3018 err = init_arena_map_data(obj, map, ARENA_SEC, obj->efile.arena_data_shndx,
3019 obj->efile.arena_data->d_buf,
3020 obj->efile.arena_data->d_size);
3021 if (err)
3022 return err;
3023 }
3024 }
3025 if (obj->efile.arena_data && !obj->arena_map) {
3026 pr_warn("elf: sec '%s': to use global __arena variables the ARENA map should be explicitly declared in SEC(\".maps\")\n",
3027 ARENA_SEC);
3028 return -ENOENT;
3029 }
3030
3031 return 0;
3032 }
3033
3034 static int bpf_object__init_maps(struct bpf_object *obj,
3035 const struct bpf_object_open_opts *opts)
3036 {
3037 const char *pin_root_path;
3038 bool strict;
3039 int err = 0;
3040
3041 strict = !OPTS_GET(opts, relaxed_maps, false);
3042 pin_root_path = OPTS_GET(opts, pin_root_path, NULL);
3043
3044 err = bpf_object__init_user_btf_maps(obj, strict, pin_root_path);
3045 err = err ?: bpf_object__init_global_data_maps(obj);
3046 err = err ?: bpf_object__init_kconfig_map(obj);
3047 err = err ?: bpf_object_init_struct_ops(obj);
3048
3049 return err;
3050 }
3051
3052 static bool section_have_execinstr(struct bpf_object *obj, int idx)
3053 {
3054 Elf64_Shdr *sh;
3055
3056 sh = elf_sec_hdr(obj, elf_sec_by_idx(obj, idx));
3057 if (!sh)
3058 return false;
3059
3060 return sh->sh_flags & SHF_EXECINSTR;
3061 }
3062
3063 static bool starts_with_qmark(const char *s)
3064 {
3065 return s && s[0] == '?';
3066 }
3067
3068 static bool btf_needs_sanitization(struct bpf_object *obj)
3069 {
3070 bool has_func_global = kernel_supports(obj, FEAT_BTF_GLOBAL_FUNC);
3071 bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC);
3072 bool has_float = kernel_supports(obj, FEAT_BTF_FLOAT);
3073 bool has_func = kernel_supports(obj, FEAT_BTF_FUNC);
3074 bool has_decl_tag = kernel_supports(obj, FEAT_BTF_DECL_TAG);
3075 bool has_type_tag = kernel_supports(obj, FEAT_BTF_TYPE_TAG);
3076 bool has_enum64 = kernel_supports(obj, FEAT_BTF_ENUM64);
3077 bool has_qmark_datasec = kernel_supports(obj, FEAT_BTF_QMARK_DATASEC);
3078
3079 return !has_func || !has_datasec || !has_func_global || !has_float ||
3080 !has_decl_tag || !has_type_tag || !has_enum64 || !has_qmark_datasec;
3081 }
3082
3083 static int bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf)
3084 {
3085 bool has_func_global = kernel_supports(obj, FEAT_BTF_GLOBAL_FUNC);
3086 bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC);
3087 bool has_float = kernel_supports(obj, FEAT_BTF_FLOAT);
3088 bool has_func = kernel_supports(obj, FEAT_BTF_FUNC);
3089 bool has_decl_tag = kernel_supports(obj, FEAT_BTF_DECL_TAG);
3090 bool has_type_tag = kernel_supports(obj, FEAT_BTF_TYPE_TAG);
3091 bool has_enum64 = kernel_supports(obj, FEAT_BTF_ENUM64);
3092 bool has_qmark_datasec = kernel_supports(obj, FEAT_BTF_QMARK_DATASEC);
3093 int enum64_placeholder_id = 0;
3094 struct btf_type *t;
3095 int i, j, vlen;
3096
3097 for (i = 1; i < btf__type_cnt(btf); i++) {
3098 t = (struct btf_type *)btf__type_by_id(btf, i);
3099
3100 if ((!has_datasec && btf_is_var(t)) || (!has_decl_tag && btf_is_decl_tag(t))) {
3101 /* replace VAR/DECL_TAG with INT */
3102 t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0);
3103 /*
3104 * using size = 1 is the safest choice, 4 will be too
3105 * big and cause kernel BTF validation failure if
3106 * original variable took less than 4 bytes
3107 */
3108 t->size = 1;
3109 *(int *)(t + 1) = BTF_INT_ENC(0, 0, 8);
3110 } else if (!has_datasec && btf_is_datasec(t)) {
3111 /* replace DATASEC with STRUCT */
3112 const struct btf_var_secinfo *v = btf_var_secinfos(t);
3113 struct btf_member *m = btf_members(t);
3114 struct btf_type *vt;
3115 char *name;
3116
3117 name = (char *)btf__name_by_offset(btf, t->name_off);
3118 while (*name) {
3119 if (*name == '.' || *name == '?')
3120 *name = '_';
3121 name++;
3122 }
3123
3124 vlen = btf_vlen(t);
3125 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen);
3126 for (j = 0; j < vlen; j++, v++, m++) {
3127 /* order of field assignments is important */
3128 m->offset = v->offset * 8;
3129 m->type = v->type;
3130 /* preserve variable name as member name */
3131 vt = (void *)btf__type_by_id(btf, v->type);
3132 m->name_off = vt->name_off;
3133 }
3134 } else if (!has_qmark_datasec && btf_is_datasec(t) &&
3135 starts_with_qmark(btf__name_by_offset(btf, t->name_off))) {
3136 /* replace '?' prefix with '_' for DATASEC names */
3137 char *name;
3138
3139 name = (char *)btf__name_by_offset(btf, t->name_off);
3140 if (name[0] == '?')
3141 name[0] = '_';
3142 } else if (!has_func && btf_is_func_proto(t)) {
3143 /* replace FUNC_PROTO with ENUM */
3144 vlen = btf_vlen(t);
3145 t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen);
3146 t->size = sizeof(__u32); /* kernel enforced */
3147 } else if (!has_func && btf_is_func(t)) {
3148 /* replace FUNC with TYPEDEF */
3149 t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0);
3150 } else if (!has_func_global && btf_is_func(t)) {
3151 /* replace BTF_FUNC_GLOBAL with BTF_FUNC_STATIC */
3152 t->info = BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0);
3153 } else if (!has_float && btf_is_float(t)) {
3154 /* replace FLOAT with an equally-sized empty STRUCT;
3155 * since C compilers do not accept e.g. "float" as a
3156 * valid struct name, make it anonymous
3157 */
3158 t->name_off = 0;
3159 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 0);
3160 } else if (!has_type_tag && btf_is_type_tag(t)) {
3161 /* replace TYPE_TAG with a CONST */
3162 t->name_off = 0;
3163 t->info = BTF_INFO_ENC(BTF_KIND_CONST, 0, 0);
3164 } else if (!has_enum64 && btf_is_enum(t)) {
3165 /* clear the kflag */
3166 t->info = btf_type_info(btf_kind(t), btf_vlen(t), false);
3167 } else if (!has_enum64 && btf_is_enum64(t)) {
3168 /* replace ENUM64 with a union */
3169 struct btf_member *m;
3170
3171 if (enum64_placeholder_id == 0) {
3172 enum64_placeholder_id = btf__add_int(btf, "enum64_placeholder", 1, 0);
3173 if (enum64_placeholder_id < 0)
3174 return enum64_placeholder_id;
3175
3176 t = (struct btf_type *)btf__type_by_id(btf, i);
3177 }
3178
3179 m = btf_members(t);
3180 vlen = btf_vlen(t);
3181 t->info = BTF_INFO_ENC(BTF_KIND_UNION, 0, vlen);
3182 for (j = 0; j < vlen; j++, m++) {
3183 m->type = enum64_placeholder_id;
3184 m->offset = 0;
3185 }
3186 }
3187 }
3188
3189 return 0;
3190 }
3191
3192 static bool libbpf_needs_btf(const struct bpf_object *obj)
3193 {
3194 return obj->efile.btf_maps_shndx >= 0 ||
3195 obj->efile.has_st_ops ||
3196 obj->nr_extern > 0;
3197 }
3198
3199 static bool kernel_needs_btf(const struct bpf_object *obj)
3200 {
3201 return obj->efile.has_st_ops;
3202 }
3203
3204 static int bpf_object__init_btf(struct bpf_object *obj,
3205 Elf_Data *btf_data,
3206 Elf_Data *btf_ext_data)
3207 {
3208 int err = -ENOENT;
3209
3210 if (btf_data) {
3211 obj->btf = btf__new(btf_data->d_buf, btf_data->d_size);
3212 err = libbpf_get_error(obj->btf);
3213 if (err) {
3214 obj->btf = NULL;
3215 pr_warn("Error loading ELF section %s: %d.\n", BTF_ELF_SEC, err);
3216 goto out;
3217 }
3218 /* enforce 8-byte pointers for BPF-targeted BTFs */
3219 btf__set_pointer_size(obj->btf, 8);
3220 }
3221 if (btf_ext_data) {
3222 struct btf_ext_info *ext_segs[3];
3223 int seg_num, sec_num;
3224
3225 if (!obj->btf) {
3226 pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n",
3227 BTF_EXT_ELF_SEC, BTF_ELF_SEC);
3228 goto out;
3229 }
3230 obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, btf_ext_data->d_size);
3231 err = libbpf_get_error(obj->btf_ext);
3232 if (err) {
3233 pr_warn("Error loading ELF section %s: %d. Ignored and continue.\n",
3234 BTF_EXT_ELF_SEC, err);
3235 obj->btf_ext = NULL;
3236 goto out;
3237 }
3238
3239 /* setup .BTF.ext to ELF section mapping */
3240 ext_segs[0] = &obj->btf_ext->func_info;
3241 ext_segs[1] = &obj->btf_ext->line_info;
3242 ext_segs[2] = &obj->btf_ext->core_relo_info;
3243 for (seg_num = 0; seg_num < ARRAY_SIZE(ext_segs); seg_num++) {
3244 struct btf_ext_info *seg = ext_segs[seg_num];
3245 const struct btf_ext_info_sec *sec;
3246 const char *sec_name;
3247 Elf_Scn *scn;
3248
3249 if (seg->sec_cnt == 0)
3250 continue;
3251
3252 seg->sec_idxs = calloc(seg->sec_cnt, sizeof(*seg->sec_idxs));
3253 if (!seg->sec_idxs) {
3254 err = -ENOMEM;
3255 goto out;
3256 }
3257
3258 sec_num = 0;
3259 for_each_btf_ext_sec(seg, sec) {
3260 /* preventively increment index to avoid doing
3261 * this before every continue below
3262 */
3263 sec_num++;
3264
3265 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);
3266 if (str_is_empty(sec_name))
3267 continue;
3268 scn = elf_sec_by_name(obj, sec_name);
3269 if (!scn)
3270 continue;
3271
3272 seg->sec_idxs[sec_num - 1] = elf_ndxscn(scn);
3273 }
3274 }
3275 }
3276 out:
3277 if (err && libbpf_needs_btf(obj)) {
3278 pr_warn("BTF is required, but is missing or corrupted.\n");
3279 return err;
3280 }
3281 return 0;
3282 }
3283
3284 static int compare_vsi_off(const void *_a, const void *_b)
3285 {
3286 const struct btf_var_secinfo *a = _a;
3287 const struct btf_var_secinfo *b = _b;
3288
3289 return a->offset - b->offset;
3290 }
3291
3292 static int btf_fixup_datasec(struct bpf_object *obj, struct btf *btf,
3293 struct btf_type *t)
3294 {
3295 __u32 size = 0, i, vars = btf_vlen(t);
3296 const char *sec_name = btf__name_by_offset(btf, t->name_off);
3297 struct btf_var_secinfo *vsi;
3298 bool fixup_offsets = false;
3299 int err;
3300
3301 if (!sec_name) {
3302 pr_debug("No name found in string section for DATASEC kind.\n");
3303 return -ENOENT;
3304 }
3305
3306 /* Extern-backing datasecs (.ksyms, .kconfig) have their size and
3307 * variable offsets set at the previous step. Further, not every
3308 * extern BTF VAR has corresponding ELF symbol preserved, so we skip
3309 * all fixups altogether for such sections and go straight to sorting
3310 * VARs within their DATASEC.
3311 */
3312 if (strcmp(sec_name, KCONFIG_SEC) == 0 || strcmp(sec_name, KSYMS_SEC) == 0)
3313 goto sort_vars;
3314
3315 /* Clang leaves DATASEC size and VAR offsets as zeroes, so we need to
3316 * fix this up. But BPF static linker already fixes this up and fills
3317 * all the sizes and offsets during static linking. So this step has
3318 * to be optional. But the STV_HIDDEN handling is non-optional for any
3319 * non-extern DATASEC, so the variable fixup loop below handles both
3320 * functions at the same time, paying the cost of BTF VAR <-> ELF
3321 * symbol matching just once.
3322 */
3323 if (t->size == 0) {
3324 err = find_elf_sec_sz(obj, sec_name, &size);
3325 if (err || !size) {
3326 pr_debug("sec '%s': failed to determine size from ELF: size %u, err %d\n",
3327 sec_name, size, err);
3328 return -ENOENT;
3329 }
3330
3331 t->size = size;
3332 fixup_offsets = true;
3333 }
3334
3335 for (i = 0, vsi = btf_var_secinfos(t); i < vars; i++, vsi++) {
3336 const struct btf_type *t_var;
3337 struct btf_var *var;
3338 const char *var_name;
3339 Elf64_Sym *sym;
3340
3341 t_var = btf__type_by_id(btf, vsi->type);
3342 if (!t_var || !btf_is_var(t_var)) {
3343 pr_debug("sec '%s': unexpected non-VAR type found\n", sec_name);
3344 return -EINVAL;
3345 }
3346
3347 var = btf_var(t_var);
3348 if (var->linkage == BTF_VAR_STATIC || var->linkage == BTF_VAR_GLOBAL_EXTERN)
3349 continue;
3350
3351 var_name = btf__name_by_offset(btf, t_var->name_off);
3352 if (!var_name) {
3353 pr_debug("sec '%s': failed to find name of DATASEC's member #%d\n",
3354 sec_name, i);
3355 return -ENOENT;
3356 }
3357
3358 sym = find_elf_var_sym(obj, var_name);
3359 if (IS_ERR(sym)) {
3360 pr_debug("sec '%s': failed to find ELF symbol for VAR '%s'\n",
3361 sec_name, var_name);
3362 return -ENOENT;
3363 }
3364
3365 if (fixup_offsets)
3366 vsi->offset = sym->st_value;
3367
3368 /* if variable is a global/weak symbol, but has restricted
3369 * (STV_HIDDEN or STV_INTERNAL) visibility, mark its BTF VAR
3370 * as static. This follows similar logic for functions (BPF
3371 * subprogs) and influences libbpf's further decisions about
3372 * whether to make global data BPF array maps as
3373 * BPF_F_MMAPABLE.
3374 */
3375 if (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN
3376 || ELF64_ST_VISIBILITY(sym->st_other) == STV_INTERNAL)
3377 var->linkage = BTF_VAR_STATIC;
3378 }
3379
3380 sort_vars:
3381 qsort(btf_var_secinfos(t), vars, sizeof(*vsi), compare_vsi_off);
3382 return 0;
3383 }
3384
3385 static int bpf_object_fixup_btf(struct bpf_object *obj)
3386 {
3387 int i, n, err = 0;
3388
3389 if (!obj->btf)
3390 return 0;
3391
3392 n = btf__type_cnt(obj->btf);
3393 for (i = 1; i < n; i++) {
3394 struct btf_type *t = btf_type_by_id(obj->btf, i);
3395
3396 /* Loader needs to fix up some of the things compiler
3397 * couldn't get its hands on while emitting BTF. This
3398 * is section size and global variable offset. We use
3399 * the info from the ELF itself for this purpose.
3400 */
3401 if (btf_is_datasec(t)) {
3402 err = btf_fixup_datasec(obj, obj->btf, t);
3403 if (err)
3404 return err;
3405 }
3406 }
3407
3408 return 0;
3409 }
3410
3411 static bool prog_needs_vmlinux_btf(struct bpf_program *prog)
3412 {
3413 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS ||
3414 prog->type == BPF_PROG_TYPE_LSM)
3415 return true;
3416
3417 /* BPF_PROG_TYPE_TRACING programs which do not attach to other programs
3418 * also need vmlinux BTF
3419 */
3420 if (prog->type == BPF_PROG_TYPE_TRACING && !prog->attach_prog_fd)
3421 return true;
3422
3423 return false;
3424 }
3425
3426 static bool map_needs_vmlinux_btf(struct bpf_map *map)
3427 {
3428 return bpf_map__is_struct_ops(map);
3429 }
3430
3431 static bool obj_needs_vmlinux_btf(const struct bpf_object *obj)
3432 {
3433 struct bpf_program *prog;
3434 struct bpf_map *map;
3435 int i;
3436
3437 /* CO-RE relocations need kernel BTF, only when btf_custom_path
3438 * is not specified
3439 */
3440 if (obj->btf_ext && obj->btf_ext->core_relo_info.len && !obj->btf_custom_path)
3441 return true;
3442
3443 /* Support for typed ksyms needs kernel BTF */
3444 for (i = 0; i < obj->nr_extern; i++) {
3445 const struct extern_desc *ext;
3446
3447 ext = &obj->externs[i];
3448 if (ext->type == EXT_KSYM && ext->ksym.type_id)
3449 return true;
3450 }
3451
3452 bpf_object__for_each_program(prog, obj) {
3453 if (!prog->autoload)
3454 continue;
3455 if (prog_needs_vmlinux_btf(prog))
3456 return true;
3457 }
3458
3459 bpf_object__for_each_map(map, obj) {
3460 if (map_needs_vmlinux_btf(map))
3461 return true;
3462 }
3463
3464 return false;
3465 }
3466
3467 static int bpf_object__load_vmlinux_btf(struct bpf_object *obj, bool force)
3468 {
3469 int err;
3470
3471 /* btf_vmlinux could be loaded earlier */
3472 if (obj->btf_vmlinux || obj->gen_loader)
3473 return 0;
3474
3475 if (!force && !obj_needs_vmlinux_btf(obj))
3476 return 0;
3477
3478 obj->btf_vmlinux = btf__load_vmlinux_btf();
3479 err = libbpf_get_error(obj->btf_vmlinux);
3480 if (err) {
3481 pr_warn("Error loading vmlinux BTF: %d\n", err);
3482 obj->btf_vmlinux = NULL;
3483 return err;
3484 }
3485 return 0;
3486 }
3487
3488 static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj)
3489 {
3490 struct btf *kern_btf = obj->btf;
3491 bool btf_mandatory, sanitize;
3492 int i, err = 0;
3493
3494 if (!obj->btf)
3495 return 0;
3496
3497 if (!kernel_supports(obj, FEAT_BTF)) {
3498 if (kernel_needs_btf(obj)) {
3499 err = -EOPNOTSUPP;
3500 goto report;
3501 }
3502 pr_debug("Kernel doesn't support BTF, skipping uploading it.\n");
3503 return 0;
3504 }
3505
3506 /* Even though some subprogs are global/weak, user might prefer more
3507 * permissive BPF verification process that BPF verifier performs for
3508 * static functions, taking into account more context from the caller
3509 * functions. In such case, they need to mark such subprogs with
3510 * __attribute__((visibility("hidden"))) and libbpf will adjust
3511 * corresponding FUNC BTF type to be marked as static and trigger more
3512 * involved BPF verification process.
3513 */
3514 for (i = 0; i < obj->nr_programs; i++) {
3515 struct bpf_program *prog = &obj->programs[i];
3516 struct btf_type *t;
3517 const char *name;
3518 int j, n;
3519
3520 if (!prog->mark_btf_static || !prog_is_subprog(obj, prog))
3521 continue;
3522
3523 n = btf__type_cnt(obj->btf);
3524 for (j = 1; j < n; j++) {
3525 t = btf_type_by_id(obj->btf, j);
3526 if (!btf_is_func(t) || btf_func_linkage(t) != BTF_FUNC_GLOBAL)
3527 continue;
3528
3529 name = btf__str_by_offset(obj->btf, t->name_off);
3530 if (strcmp(name, prog->name) != 0)
3531 continue;
3532
3533 t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_STATIC, 0);
3534 break;
3535 }
3536 }
3537
3538 sanitize = btf_needs_sanitization(obj);
3539 if (sanitize) {
3540 const void *raw_data;
3541 __u32 sz;
3542
3543 /* clone BTF to sanitize a copy and leave the original intact */
3544 raw_data = btf__raw_data(obj->btf, &sz);
3545 kern_btf = btf__new(raw_data, sz);
3546 err = libbpf_get_error(kern_btf);
3547 if (err)
3548 return err;
3549
3550 /* enforce 8-byte pointers for BPF-targeted BTFs */
3551 btf__set_pointer_size(obj->btf, 8);
3552 err = bpf_object__sanitize_btf(obj, kern_btf);
3553 if (err)
3554 return err;
3555 }
3556
3557 if (obj->gen_loader) {
3558 __u32 raw_size = 0;
3559 const void *raw_data = btf__raw_data(kern_btf, &raw_size);
3560
3561 if (!raw_data)
3562 return -ENOMEM;
3563 bpf_gen__load_btf(obj->gen_loader, raw_data, raw_size);
3564 /* Pretend to have valid FD to pass various fd >= 0 checks.
3565 * This fd == 0 will not be used with any syscall and will be reset to -1 eventually.
3566 */
3567 btf__set_fd(kern_btf, 0);
3568 } else {
3569 /* currently BPF_BTF_LOAD only supports log_level 1 */
3570 err = btf_load_into_kernel(kern_btf, obj->log_buf, obj->log_size,
3571 obj->log_level ? 1 : 0, obj->token_fd);
3572 }
3573 if (sanitize) {
3574 if (!err) {
3575 /* move fd to libbpf's BTF */
3576 btf__set_fd(obj->btf, btf__fd(kern_btf));
3577 btf__set_fd(kern_btf, -1);
3578 }
3579 btf__free(kern_btf);
3580 }
3581 report:
3582 if (err) {
3583 btf_mandatory = kernel_needs_btf(obj);
3584 pr_warn("Error loading .BTF into kernel: %d. %s\n", err,
3585 btf_mandatory ? "BTF is mandatory, can't proceed."
3586 : "BTF is optional, ignoring.");
3587 if (!btf_mandatory)
3588 err = 0;
3589 }
3590 return err;
3591 }
3592
3593 static const char *elf_sym_str(const struct bpf_object *obj, size_t off)
3594 {
3595 const char *name;
3596
3597 name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, off);
3598 if (!name) {
3599 pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n",
3600 off, obj->path, elf_errmsg(-1));
3601 return NULL;
3602 }
3603
3604 return name;
3605 }
3606
3607 static const char *elf_sec_str(const struct bpf_object *obj, size_t off)
3608 {
3609 const char *name;
3610
3611 name = elf_strptr(obj->efile.elf, obj->efile.shstrndx, off);
3612 if (!name) {
3613 pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n",
3614 off, obj->path, elf_errmsg(-1));
3615 return NULL;
3616 }
3617
3618 return name;
3619 }
3620
3621 static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx)
3622 {
3623 Elf_Scn *scn;
3624
3625 scn = elf_getscn(obj->efile.elf, idx);
3626 if (!scn) {
3627 pr_warn("elf: failed to get section(%zu) from %s: %s\n",
3628 idx, obj->path, elf_errmsg(-1));
3629 return NULL;
3630 }
3631 return scn;
3632 }
3633
3634 static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name)
3635 {
3636 Elf_Scn *scn = NULL;
3637 Elf *elf = obj->efile.elf;
3638 const char *sec_name;
3639
3640 while ((scn = elf_nextscn(elf, scn)) != NULL) {
3641 sec_name = elf_sec_name(obj, scn);
3642 if (!sec_name)
3643 return NULL;
3644
3645 if (strcmp(sec_name, name) != 0)
3646 continue;
3647
3648 return scn;
3649 }
3650 return NULL;
3651 }
3652
3653 static Elf64_Shdr *elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn)
3654 {
3655 Elf64_Shdr *shdr;
3656
3657 if (!scn)
3658 return NULL;
3659
3660 shdr = elf64_getshdr(scn);
3661 if (!shdr) {
3662 pr_warn("elf: failed to get section(%zu) header from %s: %s\n",
3663 elf_ndxscn(scn), obj->path, elf_errmsg(-1));
3664 return NULL;
3665 }
3666
3667 return shdr;
3668 }
3669
3670 static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn)
3671 {
3672 const char *name;
3673 Elf64_Shdr *sh;
3674
3675 if (!scn)
3676 return NULL;
3677
3678 sh = elf_sec_hdr(obj, scn);
3679 if (!sh)
3680 return NULL;
3681
3682 name = elf_sec_str(obj, sh->sh_name);
3683 if (!name) {
3684 pr_warn("elf: failed to get section(%zu) name from %s: %s\n",
3685 elf_ndxscn(scn), obj->path, elf_errmsg(-1));
3686 return NULL;
3687 }
3688
3689 return name;
3690 }
3691
3692 static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn)
3693 {
3694 Elf_Data *data;
3695
3696 if (!scn)
3697 return NULL;
3698
3699 data = elf_getdata(scn, 0);
3700 if (!data) {
3701 pr_warn("elf: failed to get section(%zu) %s data from %s: %s\n",
3702 elf_ndxscn(scn), elf_sec_name(obj, scn) ?: "<?>",
3703 obj->path, elf_errmsg(-1));
3704 return NULL;
3705 }
3706
3707 return data;
3708 }
3709
3710 static Elf64_Sym *elf_sym_by_idx(const struct bpf_object *obj, size_t idx)
3711 {
3712 if (idx >= obj->efile.symbols->d_size / sizeof(Elf64_Sym))
3713 return NULL;
3714
3715 return (Elf64_Sym *)obj->efile.symbols->d_buf + idx;
3716 }
3717
3718 static Elf64_Rel *elf_rel_by_idx(Elf_Data *data, size_t idx)
3719 {
3720 if (idx >= data->d_size / sizeof(Elf64_Rel))
3721 return NULL;
3722
3723 return (Elf64_Rel *)data->d_buf + idx;
3724 }
3725
3726 static bool is_sec_name_dwarf(const char *name)
3727 {
3728 /* approximation, but the actual list is too long */
3729 return str_has_pfx(name, ".debug_");
3730 }
3731
3732 static bool ignore_elf_section(Elf64_Shdr *hdr, const char *name)
3733 {
3734 /* no special handling of .strtab */
3735 if (hdr->sh_type == SHT_STRTAB)
3736 return true;
3737
3738 /* ignore .llvm_addrsig section as well */
3739 if (hdr->sh_type == SHT_LLVM_ADDRSIG)
3740 return true;
3741
3742 /* no subprograms will lead to an empty .text section, ignore it */
3743 if (hdr->sh_type == SHT_PROGBITS && hdr->sh_size == 0 &&
3744 strcmp(name, ".text") == 0)
3745 return true;
3746
3747 /* DWARF sections */
3748 if (is_sec_name_dwarf(name))
3749 return true;
3750
3751 if (str_has_pfx(name, ".rel")) {
3752 name += sizeof(".rel") - 1;
3753 /* DWARF section relocations */
3754 if (is_sec_name_dwarf(name))
3755 return true;
3756
3757 /* .BTF and .BTF.ext don't need relocations */
3758 if (strcmp(name, BTF_ELF_SEC) == 0 ||
3759 strcmp(name, BTF_EXT_ELF_SEC) == 0)
3760 return true;
3761 }
3762
3763 return false;
3764 }
3765
3766 static int cmp_progs(const void *_a, const void *_b)
3767 {
3768 const struct bpf_program *a = _a;
3769 const struct bpf_program *b = _b;
3770
3771 if (a->sec_idx != b->sec_idx)
3772 return a->sec_idx < b->sec_idx ? -1 : 1;
3773
3774 /* sec_insn_off can't be the same within the section */
3775 return a->sec_insn_off < b->sec_insn_off ? -1 : 1;
3776 }
3777
3778 static int bpf_object__elf_collect(struct bpf_object *obj)
3779 {
3780 struct elf_sec_desc *sec_desc;
3781 Elf *elf = obj->efile.elf;
3782 Elf_Data *btf_ext_data = NULL;
3783 Elf_Data *btf_data = NULL;
3784 int idx = 0, err = 0;
3785 const char *name;
3786 Elf_Data *data;
3787 Elf_Scn *scn;
3788 Elf64_Shdr *sh;
3789
3790 /* ELF section indices are 0-based, but sec #0 is special "invalid"
3791 * section. Since section count retrieved by elf_getshdrnum() does
3792 * include sec #0, it is already the necessary size of an array to keep
3793 * all the sections.
3794 */
3795 if (elf_getshdrnum(obj->efile.elf, &obj->efile.sec_cnt)) {
3796 pr_warn("elf: failed to get the number of sections for %s: %s\n",
3797 obj->path, elf_errmsg(-1));
3798 return -LIBBPF_ERRNO__FORMAT;
3799 }
3800 obj->efile.secs = calloc(obj->efile.sec_cnt, sizeof(*obj->efile.secs));
3801 if (!obj->efile.secs)
3802 return -ENOMEM;
3803
3804 /* a bunch of ELF parsing functionality depends on processing symbols,
3805 * so do the first pass and find the symbol table
3806 */
3807 scn = NULL;
3808 while ((scn = elf_nextscn(elf, scn)) != NULL) {
3809 sh = elf_sec_hdr(obj, scn);
3810 if (!sh)
3811 return -LIBBPF_ERRNO__FORMAT;
3812
3813 if (sh->sh_type == SHT_SYMTAB) {
3814 if (obj->efile.symbols) {
3815 pr_warn("elf: multiple symbol tables in %s\n", obj->path);
3816 return -LIBBPF_ERRNO__FORMAT;
3817 }
3818
3819 data = elf_sec_data(obj, scn);
3820 if (!data)
3821 return -LIBBPF_ERRNO__FORMAT;
3822
3823 idx = elf_ndxscn(scn);
3824
3825 obj->efile.symbols = data;
3826 obj->efile.symbols_shndx = idx;
3827 obj->efile.strtabidx = sh->sh_link;
3828 }
3829 }
3830
3831 if (!obj->efile.symbols) {
3832 pr_warn("elf: couldn't find symbol table in %s, stripped object file?\n",
3833 obj->path);
3834 return -ENOENT;
3835 }
3836
3837 scn = NULL;
3838 while ((scn = elf_nextscn(elf, scn)) != NULL) {
3839 idx = elf_ndxscn(scn);
3840 sec_desc = &obj->efile.secs[idx];
3841
3842 sh = elf_sec_hdr(obj, scn);
3843 if (!sh)
3844 return -LIBBPF_ERRNO__FORMAT;
3845
3846 name = elf_sec_str(obj, sh->sh_name);
3847 if (!name)
3848 return -LIBBPF_ERRNO__FORMAT;
3849
3850 if (ignore_elf_section(sh, name))
3851 continue;
3852
3853 data = elf_sec_data(obj, scn);
3854 if (!data)
3855 return -LIBBPF_ERRNO__FORMAT;
3856
3857 pr_debug("elf: section(%d) %s, size %ld, link %d, flags %lx, type=%d\n",
3858 idx, name, (unsigned long)data->d_size,
3859 (int)sh->sh_link, (unsigned long)sh->sh_flags,
3860 (int)sh->sh_type);
3861
3862 if (strcmp(name, "license") == 0) {
3863 err = bpf_object__init_license(obj, data->d_buf, data->d_size);
3864 if (err)
3865 return err;
3866 } else if (strcmp(name, "version") == 0) {
3867 err = bpf_object__init_kversion(obj, data->d_buf, data->d_size);
3868 if (err)
3869 return err;
3870 } else if (strcmp(name, "maps") == 0) {
3871 pr_warn("elf: legacy map definitions in 'maps' section are not supported by libbpf v1.0+\n");
3872 return -ENOTSUP;
3873 } else if (strcmp(name, MAPS_ELF_SEC) == 0) {
3874 obj->efile.btf_maps_shndx = idx;
3875 } else if (strcmp(name, BTF_ELF_SEC) == 0) {
3876 if (sh->sh_type != SHT_PROGBITS)
3877 return -LIBBPF_ERRNO__FORMAT;
3878 btf_data = data;
3879 } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) {
3880 if (sh->sh_type != SHT_PROGBITS)
3881 return -LIBBPF_ERRNO__FORMAT;
3882 btf_ext_data = data;
3883 } else if (sh->sh_type == SHT_SYMTAB) {
3884 /* already processed during the first pass above */
3885 } else if (sh->sh_type == SHT_PROGBITS && data->d_size > 0) {
3886 if (sh->sh_flags & SHF_EXECINSTR) {
3887 if (strcmp(name, ".text") == 0)
3888 obj->efile.text_shndx = idx;
3889 err = bpf_object__add_programs(obj, data, name, idx);
3890 if (err)
3891 return err;
3892 } else if (strcmp(name, DATA_SEC) == 0 ||
3893 str_has_pfx(name, DATA_SEC ".")) {
3894 sec_desc->sec_type = SEC_DATA;
3895 sec_desc->shdr = sh;
3896 sec_desc->data = data;
3897 } else if (strcmp(name, RODATA_SEC) == 0 ||
3898 str_has_pfx(name, RODATA_SEC ".")) {
3899 sec_desc->sec_type = SEC_RODATA;
3900 sec_desc->shdr = sh;
3901 sec_desc->data = data;
3902 } else if (strcmp(name, STRUCT_OPS_SEC) == 0 ||
3903 strcmp(name, STRUCT_OPS_LINK_SEC) == 0 ||
3904 strcmp(name, "?" STRUCT_OPS_SEC) == 0 ||
3905 strcmp(name, "?" STRUCT_OPS_LINK_SEC) == 0) {
3906 sec_desc->sec_type = SEC_ST_OPS;
3907 sec_desc->shdr = sh;
3908 sec_desc->data = data;
3909 obj->efile.has_st_ops = true;
3910 } else if (strcmp(name, ARENA_SEC) == 0) {
3911 obj->efile.arena_data = data;
3912 obj->efile.arena_data_shndx = idx;
3913 } else {
3914 pr_info("elf: skipping unrecognized data section(%d) %s\n",
3915 idx, name);
3916 }
3917 } else if (sh->sh_type == SHT_REL) {
3918 int targ_sec_idx = sh->sh_info; /* points to other section */
3919
3920 if (sh->sh_entsize != sizeof(Elf64_Rel) ||
3921 targ_sec_idx >= obj->efile.sec_cnt)
3922 return -LIBBPF_ERRNO__FORMAT;
3923
3924 /* Only do relo for section with exec instructions */
3925 if (!section_have_execinstr(obj, targ_sec_idx) &&
3926 strcmp(name, ".rel" STRUCT_OPS_SEC) &&
3927 strcmp(name, ".rel" STRUCT_OPS_LINK_SEC) &&
3928 strcmp(name, ".rel?" STRUCT_OPS_SEC) &&
3929 strcmp(name, ".rel?" STRUCT_OPS_LINK_SEC) &&
3930 strcmp(name, ".rel" MAPS_ELF_SEC)) {
3931 pr_info("elf: skipping relo section(%d) %s for section(%d) %s\n",
3932 idx, name, targ_sec_idx,
3933 elf_sec_name(obj, elf_sec_by_idx(obj, targ_sec_idx)) ?: "<?>");
3934 continue;
3935 }
3936
3937 sec_desc->sec_type = SEC_RELO;
3938 sec_desc->shdr = sh;
3939 sec_desc->data = data;
3940 } else if (sh->sh_type == SHT_NOBITS && (strcmp(name, BSS_SEC) == 0 ||
3941 str_has_pfx(name, BSS_SEC "."))) {
3942 sec_desc->sec_type = SEC_BSS;
3943 sec_desc->shdr = sh;
3944 sec_desc->data = data;
3945 } else {
3946 pr_info("elf: skipping section(%d) %s (size %zu)\n", idx, name,
3947 (size_t)sh->sh_size);
3948 }
3949 }
3950
3951 if (!obj->efile.strtabidx || obj->efile.strtabidx > idx) {
3952 pr_warn("elf: symbol strings section missing or invalid in %s\n", obj->path);
3953 return -LIBBPF_ERRNO__FORMAT;
3954 }
3955
3956 /* sort BPF programs by section name and in-section instruction offset
3957 * for faster search
3958 */
3959 if (obj->nr_programs)
3960 qsort(obj->programs, obj->nr_programs, sizeof(*obj->programs), cmp_progs);
3961
3962 return bpf_object__init_btf(obj, btf_data, btf_ext_data);
3963 }
3964
3965 static bool sym_is_extern(const Elf64_Sym *sym)
3966 {
3967 int bind = ELF64_ST_BIND(sym->st_info);
3968 /* externs are symbols w/ type=NOTYPE, bind=GLOBAL|WEAK, section=UND */
3969 return sym->st_shndx == SHN_UNDEF &&
3970 (bind == STB_GLOBAL || bind == STB_WEAK) &&
3971 ELF64_ST_TYPE(sym->st_info) == STT_NOTYPE;
3972 }
3973
3974 static bool sym_is_subprog(const Elf64_Sym *sym, int text_shndx)
3975 {
3976 int bind = ELF64_ST_BIND(sym->st_info);
3977 int type = ELF64_ST_TYPE(sym->st_info);
3978
3979 /* in .text section */
3980 if (sym->st_shndx != text_shndx)
3981 return false;
3982
3983 /* local function */
3984 if (bind == STB_LOCAL && type == STT_SECTION)
3985 return true;
3986
3987 /* global function */
3988 return bind == STB_GLOBAL && type == STT_FUNC;
3989 }
3990
3991 static int find_extern_btf_id(const struct btf *btf, const char *ext_name)
3992 {
3993 const struct btf_type *t;
3994 const char *tname;
3995 int i, n;
3996
3997 if (!btf)
3998 return -ESRCH;
3999
4000 n = btf__type_cnt(btf);
4001 for (i = 1; i < n; i++) {
4002 t = btf__type_by_id(btf, i);
4003
4004 if (!btf_is_var(t) && !btf_is_func(t))
4005 continue;
4006
4007 tname = btf__name_by_offset(btf, t->name_off);
4008 if (strcmp(tname, ext_name))
4009 continue;
4010
4011 if (btf_is_var(t) &&
4012 btf_var(t)->linkage != BTF_VAR_GLOBAL_EXTERN)
4013 return -EINVAL;
4014
4015 if (btf_is_func(t) && btf_func_linkage(t) != BTF_FUNC_EXTERN)
4016 return -EINVAL;
4017
4018 return i;
4019 }
4020
4021 return -ENOENT;
4022 }
4023
4024 static int find_extern_sec_btf_id(struct btf *btf, int ext_btf_id) {
4025 const struct btf_var_secinfo *vs;
4026 const struct btf_type *t;
4027 int i, j, n;
4028
4029 if (!btf)
4030 return -ESRCH;
4031
4032 n = btf__type_cnt(btf);
4033 for (i = 1; i < n; i++) {
4034 t = btf__type_by_id(btf, i);
4035
4036 if (!btf_is_datasec(t))
4037 continue;
4038
4039 vs = btf_var_secinfos(t);
4040 for (j = 0; j < btf_vlen(t); j++, vs++) {
4041 if (vs->type == ext_btf_id)
4042 return i;
4043 }
4044 }
4045
4046 return -ENOENT;
4047 }
4048
4049 static enum kcfg_type find_kcfg_type(const struct btf *btf, int id,
4050 bool *is_signed)
4051 {
4052 const struct btf_type *t;
4053 const char *name;
4054
4055 t = skip_mods_and_typedefs(btf, id, NULL);
4056 name = btf__name_by_offset(btf, t->name_off);
4057
4058 if (is_signed)
4059 *is_signed = false;
4060 switch (btf_kind(t)) {
4061 case BTF_KIND_INT: {
4062 int enc = btf_int_encoding(t);
4063
4064 if (enc & BTF_INT_BOOL)
4065 return t->size == 1 ? KCFG_BOOL : KCFG_UNKNOWN;
4066 if (is_signed)
4067 *is_signed = enc & BTF_INT_SIGNED;
4068 if (t->size == 1)
4069 return KCFG_CHAR;
4070 if (t->size < 1 || t->size > 8 || (t->size & (t->size - 1)))
4071 return KCFG_UNKNOWN;
4072 return KCFG_INT;
4073 }
4074 case BTF_KIND_ENUM:
4075 if (t->size != 4)
4076 return KCFG_UNKNOWN;
4077 if (strcmp(name, "libbpf_tristate"))
4078 return KCFG_UNKNOWN;
4079 return KCFG_TRISTATE;
4080 case BTF_KIND_ENUM64:
4081 if (strcmp(name, "libbpf_tristate"))
4082 return KCFG_UNKNOWN;
4083 return KCFG_TRISTATE;
4084 case BTF_KIND_ARRAY:
4085 if (btf_array(t)->nelems == 0)
4086 return KCFG_UNKNOWN;
4087 if (find_kcfg_type(btf, btf_array(t)->type, NULL) != KCFG_CHAR)
4088 return KCFG_UNKNOWN;
4089 return KCFG_CHAR_ARR;
4090 default:
4091 return KCFG_UNKNOWN;
4092 }
4093 }
4094
4095 static int cmp_externs(const void *_a, const void *_b)
4096 {
4097 const struct extern_desc *a = _a;
4098 const struct extern_desc *b = _b;
4099
4100 if (a->type != b->type)
4101 return a->type < b->type ? -1 : 1;
4102
4103 if (a->type == EXT_KCFG) {
4104 /* descending order by alignment requirements */
4105 if (a->kcfg.align != b->kcfg.align)
4106 return a->kcfg.align > b->kcfg.align ? -1 : 1;
4107 /* ascending order by size, within same alignment class */
4108 if (a->kcfg.sz != b->kcfg.sz)
4109 return a->kcfg.sz < b->kcfg.sz ? -1 : 1;
4110 }
4111
4112 /* resolve ties by name */
4113 return strcmp(a->name, b->name);
4114 }
4115
4116 static int find_int_btf_id(const struct btf *btf)
4117 {
4118 const struct btf_type *t;
4119 int i, n;
4120
4121 n = btf__type_cnt(btf);
4122 for (i = 1; i < n; i++) {
4123 t = btf__type_by_id(btf, i);
4124
4125 if (btf_is_int(t) && btf_int_bits(t) == 32)
4126 return i;
4127 }
4128
4129 return 0;
4130 }
4131
4132 static int add_dummy_ksym_var(struct btf *btf)
4133 {
4134 int i, int_btf_id, sec_btf_id, dummy_var_btf_id;
4135 const struct btf_var_secinfo *vs;
4136 const struct btf_type *sec;
4137
4138 if (!btf)
4139 return 0;
4140
4141 sec_btf_id = btf__find_by_name_kind(btf, KSYMS_SEC,
4142 BTF_KIND_DATASEC);
4143 if (sec_btf_id < 0)
4144 return 0;
4145
4146 sec = btf__type_by_id(btf, sec_btf_id);
4147 vs = btf_var_secinfos(sec);
4148 for (i = 0; i < btf_vlen(sec); i++, vs++) {
4149 const struct btf_type *vt;
4150
4151 vt = btf__type_by_id(btf, vs->type);
4152 if (btf_is_func(vt))
4153 break;
4154 }
4155
4156 /* No func in ksyms sec. No need to add dummy var. */
4157 if (i == btf_vlen(sec))
4158 return 0;
4159
4160 int_btf_id = find_int_btf_id(btf);
4161 dummy_var_btf_id = btf__add_var(btf,
4162 "dummy_ksym",
4163 BTF_VAR_GLOBAL_ALLOCATED,
4164 int_btf_id);
4165 if (dummy_var_btf_id < 0)
4166 pr_warn("cannot create a dummy_ksym var\n");
4167
4168 return dummy_var_btf_id;
4169 }
4170
4171 static int bpf_object__collect_externs(struct bpf_object *obj)
4172 {
4173 struct btf_type *sec, *kcfg_sec = NULL, *ksym_sec = NULL;
4174 const struct btf_type *t;
4175 struct extern_desc *ext;
4176 int i, n, off, dummy_var_btf_id;
4177 const char *ext_name, *sec_name;
4178 size_t ext_essent_len;
4179 Elf_Scn *scn;
4180 Elf64_Shdr *sh;
4181
4182 if (!obj->efile.symbols)
4183 return 0;
4184
4185 scn = elf_sec_by_idx(obj, obj->efile.symbols_shndx);
4186 sh = elf_sec_hdr(obj, scn);
4187 if (!sh || sh->sh_entsize != sizeof(Elf64_Sym))
4188 return -LIBBPF_ERRNO__FORMAT;
4189
4190 dummy_var_btf_id = add_dummy_ksym_var(obj->btf);
4191 if (dummy_var_btf_id < 0)
4192 return dummy_var_btf_id;
4193
4194 n = sh->sh_size / sh->sh_entsize;
4195 pr_debug("looking for externs among %d symbols...\n", n);
4196
4197 for (i = 0; i < n; i++) {
4198 Elf64_Sym *sym = elf_sym_by_idx(obj, i);
4199
4200 if (!sym)
4201 return -LIBBPF_ERRNO__FORMAT;
4202 if (!sym_is_extern(sym))
4203 continue;
4204 ext_name = elf_sym_str(obj, sym->st_name);
4205 if (!ext_name || !ext_name[0])
4206 continue;
4207
4208 ext = obj->externs;
4209 ext = libbpf_reallocarray(ext, obj->nr_extern + 1, sizeof(*ext));
4210 if (!ext)
4211 return -ENOMEM;
4212 obj->externs = ext;
4213 ext = &ext[obj->nr_extern];
4214 memset(ext, 0, sizeof(*ext));
4215 obj->nr_extern++;
4216
4217 ext->btf_id = find_extern_btf_id(obj->btf, ext_name);
4218 if (ext->btf_id <= 0) {
4219 pr_warn("failed to find BTF for extern '%s': %d\n",
4220 ext_name, ext->btf_id);
4221 return ext->btf_id;
4222 }
4223 t = btf__type_by_id(obj->btf, ext->btf_id);
4224 ext->name = btf__name_by_offset(obj->btf, t->name_off);
4225 ext->sym_idx = i;
4226 ext->is_weak = ELF64_ST_BIND(sym->st_info) == STB_WEAK;
4227
4228 ext_essent_len = bpf_core_essential_name_len(ext->name);
4229 ext->essent_name = NULL;
4230 if (ext_essent_len != strlen(ext->name)) {
4231 ext->essent_name = strndup(ext->name, ext_essent_len);
4232 if (!ext->essent_name)
4233 return -ENOMEM;
4234 }
4235
4236 ext->sec_btf_id = find_extern_sec_btf_id(obj->btf, ext->btf_id);
4237 if (ext->sec_btf_id <= 0) {
4238 pr_warn("failed to find BTF for extern '%s' [%d] section: %d\n",
4239 ext_name, ext->btf_id, ext->sec_btf_id);
4240 return ext->sec_btf_id;
4241 }
4242 sec = (void *)btf__type_by_id(obj->btf, ext->sec_btf_id);
4243 sec_name = btf__name_by_offset(obj->btf, sec->name_off);
4244
4245 if (strcmp(sec_name, KCONFIG_SEC) == 0) {
4246 if (btf_is_func(t)) {
4247 pr_warn("extern function %s is unsupported under %s section\n",
4248 ext->name, KCONFIG_SEC);
4249 return -ENOTSUP;
4250 }
4251 kcfg_sec = sec;
4252 ext->type = EXT_KCFG;
4253 ext->kcfg.sz = btf__resolve_size(obj->btf, t->type);
4254 if (ext->kcfg.sz <= 0) {
4255 pr_warn("failed to resolve size of extern (kcfg) '%s': %d\n",
4256 ext_name, ext->kcfg.sz);
4257 return ext->kcfg.sz;
4258 }
4259 ext->kcfg.align = btf__align_of(obj->btf, t->type);
4260 if (ext->kcfg.align <= 0) {
4261 pr_warn("failed to determine alignment of extern (kcfg) '%s': %d\n",
4262 ext_name, ext->kcfg.align);
4263 return -EINVAL;
4264 }
4265 ext->kcfg.type = find_kcfg_type(obj->btf, t->type,
4266 &ext->kcfg.is_signed);
4267 if (ext->kcfg.type == KCFG_UNKNOWN) {
4268 pr_warn("extern (kcfg) '%s': type is unsupported\n", ext_name);
4269 return -ENOTSUP;
4270 }
4271 } else if (strcmp(sec_name, KSYMS_SEC) == 0) {
4272 ksym_sec = sec;
4273 ext->type = EXT_KSYM;
4274 skip_mods_and_typedefs(obj->btf, t->type,
4275 &ext->ksym.type_id);
4276 } else {
4277 pr_warn("unrecognized extern section '%s'\n", sec_name);
4278 return -ENOTSUP;
4279 }
4280 }
4281 pr_debug("collected %d externs total\n", obj->nr_extern);
4282
4283 if (!obj->nr_extern)
4284 return 0;
4285
4286 /* sort externs by type, for kcfg ones also by (align, size, name) */
4287 qsort(obj->externs, obj->nr_extern, sizeof(*ext), cmp_externs);
4288
4289 /* for .ksyms section, we need to turn all externs into allocated
4290 * variables in BTF to pass kernel verification; we do this by
4291 * pretending that each extern is a 8-byte variable
4292 */
4293 if (ksym_sec) {
4294 /* find existing 4-byte integer type in BTF to use for fake
4295 * extern variables in DATASEC
4296 */
4297 int int_btf_id = find_int_btf_id(obj->btf);
4298 /* For extern function, a dummy_var added earlier
4299 * will be used to replace the vs->type and
4300 * its name string will be used to refill
4301 * the missing param's name.
4302 */
4303 const struct btf_type *dummy_var;
4304
4305 dummy_var = btf__type_by_id(obj->btf, dummy_var_btf_id);
4306 for (i = 0; i < obj->nr_extern; i++) {
4307 ext = &obj->externs[i];
4308 if (ext->type != EXT_KSYM)
4309 continue;
4310 pr_debug("extern (ksym) #%d: symbol %d, name %s\n",
4311 i, ext->sym_idx, ext->name);
4312 }
4313
4314 sec = ksym_sec;
4315 n = btf_vlen(sec);
4316 for (i = 0, off = 0; i < n; i++, off += sizeof(int)) {
4317 struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i;
4318 struct btf_type *vt;
4319
4320 vt = (void *)btf__type_by_id(obj->btf, vs->type);
4321 ext_name = btf__name_by_offset(obj->btf, vt->name_off);
4322 ext = find_extern_by_name(obj, ext_name);
4323 if (!ext) {
4324 pr_warn("failed to find extern definition for BTF %s '%s'\n",
4325 btf_kind_str(vt), ext_name);
4326 return -ESRCH;
4327 }
4328 if (btf_is_func(vt)) {
4329 const struct btf_type *func_proto;
4330 struct btf_param *param;
4331 int j;
4332
4333 func_proto = btf__type_by_id(obj->btf,
4334 vt->type);
4335 param = btf_params(func_proto);
4336 /* Reuse the dummy_var string if the
4337 * func proto does not have param name.
4338 */
4339 for (j = 0; j < btf_vlen(func_proto); j++)
4340 if (param[j].type && !param[j].name_off)
4341 param[j].name_off =
4342 dummy_var->name_off;
4343 vs->type = dummy_var_btf_id;
4344 vt->info &= ~0xffff;
4345 vt->info |= BTF_FUNC_GLOBAL;
4346 } else {
4347 btf_var(vt)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
4348 vt->type = int_btf_id;
4349 }
4350 vs->offset = off;
4351 vs->size = sizeof(int);
4352 }
4353 sec->size = off;
4354 }
4355
4356 if (kcfg_sec) {
4357 sec = kcfg_sec;
4358 /* for kcfg externs calculate their offsets within a .kconfig map */
4359 off = 0;
4360 for (i = 0; i < obj->nr_extern; i++) {
4361 ext = &obj->externs[i];
4362 if (ext->type != EXT_KCFG)
4363 continue;
4364
4365 ext->kcfg.data_off = roundup(off, ext->kcfg.align);
4366 off = ext->kcfg.data_off + ext->kcfg.sz;
4367 pr_debug("extern (kcfg) #%d: symbol %d, off %u, name %s\n",
4368 i, ext->sym_idx, ext->kcfg.data_off, ext->name);
4369 }
4370 sec->size = off;
4371 n = btf_vlen(sec);
4372 for (i = 0; i < n; i++) {
4373 struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i;
4374
4375 t = btf__type_by_id(obj->btf, vs->type);
4376 ext_name = btf__name_by_offset(obj->btf, t->name_off);
4377 ext = find_extern_by_name(obj, ext_name);
4378 if (!ext) {
4379 pr_warn("failed to find extern definition for BTF var '%s'\n",
4380 ext_name);
4381 return -ESRCH;
4382 }
4383 btf_var(t)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
4384 vs->offset = ext->kcfg.data_off;
4385 }
4386 }
4387 return 0;
4388 }
4389
4390 static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog)
4391 {
4392 return prog->sec_idx == obj->efile.text_shndx && obj->nr_programs > 1;
4393 }
4394
4395 struct bpf_program *
4396 bpf_object__find_program_by_name(const struct bpf_object *obj,
4397 const char *name)
4398 {
4399 struct bpf_program *prog;
4400
4401 bpf_object__for_each_program(prog, obj) {
4402 if (prog_is_subprog(obj, prog))
4403 continue;
4404 if (!strcmp(prog->name, name))
4405 return prog;
4406 }
4407 return errno = ENOENT, NULL;
4408 }
4409
4410 static bool bpf_object__shndx_is_data(const struct bpf_object *obj,
4411 int shndx)
4412 {
4413 switch (obj->efile.secs[shndx].sec_type) {
4414 case SEC_BSS:
4415 case SEC_DATA:
4416 case SEC_RODATA:
4417 return true;
4418 default:
4419 return false;
4420 }
4421 }
4422
4423 static bool bpf_object__shndx_is_maps(const struct bpf_object *obj,
4424 int shndx)
4425 {
4426 return shndx == obj->efile.btf_maps_shndx;
4427 }
4428
4429 static enum libbpf_map_type
4430 bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx)
4431 {
4432 if (shndx == obj->efile.symbols_shndx)
4433 return LIBBPF_MAP_KCONFIG;
4434
4435 switch (obj->efile.secs[shndx].sec_type) {
4436 case SEC_BSS:
4437 return LIBBPF_MAP_BSS;
4438 case SEC_DATA:
4439 return LIBBPF_MAP_DATA;
4440 case SEC_RODATA:
4441 return LIBBPF_MAP_RODATA;
4442 default:
4443 return LIBBPF_MAP_UNSPEC;
4444 }
4445 }
4446
4447 static int bpf_program__record_reloc(struct bpf_program *prog,
4448 struct reloc_desc *reloc_desc,
4449 __u32 insn_idx, const char *sym_name,
4450 const Elf64_Sym *sym, const Elf64_Rel *rel)
4451 {
4452 struct bpf_insn *insn = &prog->insns[insn_idx];
4453 size_t map_idx, nr_maps = prog->obj->nr_maps;
4454 struct bpf_object *obj = prog->obj;
4455 __u32 shdr_idx = sym->st_shndx;
4456 enum libbpf_map_type type;
4457 const char *sym_sec_name;
4458 struct bpf_map *map;
4459
4460 if (!is_call_insn(insn) && !is_ldimm64_insn(insn)) {
4461 pr_warn("prog '%s': invalid relo against '%s' for insns[%d].code 0x%x\n",
4462 prog->name, sym_name, insn_idx, insn->code);
4463 return -LIBBPF_ERRNO__RELOC;
4464 }
4465
4466 if (sym_is_extern(sym)) {
4467 int sym_idx = ELF64_R_SYM(rel->r_info);
4468 int i, n = obj->nr_extern;
4469 struct extern_desc *ext;
4470
4471 for (i = 0; i < n; i++) {
4472 ext = &obj->externs[i];
4473 if (ext->sym_idx == sym_idx)
4474 break;
4475 }
4476 if (i >= n) {
4477 pr_warn("prog '%s': extern relo failed to find extern for '%s' (%d)\n",
4478 prog->name, sym_name, sym_idx);
4479 return -LIBBPF_ERRNO__RELOC;
4480 }
4481 pr_debug("prog '%s': found extern #%d '%s' (sym %d) for insn #%u\n",
4482 prog->name, i, ext->name, ext->sym_idx, insn_idx);
4483 if (insn->code == (BPF_JMP | BPF_CALL))
4484 reloc_desc->type = RELO_EXTERN_CALL;
4485 else
4486 reloc_desc->type = RELO_EXTERN_LD64;
4487 reloc_desc->insn_idx = insn_idx;
4488 reloc_desc->ext_idx = i;
4489 return 0;
4490 }
4491
4492 /* sub-program call relocation */
4493 if (is_call_insn(insn)) {
4494 if (insn->src_reg != BPF_PSEUDO_CALL) {
4495 pr_warn("prog '%s': incorrect bpf_call opcode\n", prog->name);
4496 return -LIBBPF_ERRNO__RELOC;
4497 }
4498 /* text_shndx can be 0, if no default "main" program exists */
4499 if (!shdr_idx || shdr_idx != obj->efile.text_shndx) {
4500 sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx));
4501 pr_warn("prog '%s': bad call relo against '%s' in section '%s'\n",
4502 prog->name, sym_name, sym_sec_name);
4503 return -LIBBPF_ERRNO__RELOC;
4504 }
4505 if (sym->st_value % BPF_INSN_SZ) {
4506 pr_warn("prog '%s': bad call relo against '%s' at offset %zu\n",
4507 prog->name, sym_name, (size_t)sym->st_value);
4508 return -LIBBPF_ERRNO__RELOC;
4509 }
4510 reloc_desc->type = RELO_CALL;
4511 reloc_desc->insn_idx = insn_idx;
4512 reloc_desc->sym_off = sym->st_value;
4513 return 0;
4514 }
4515
4516 if (!shdr_idx || shdr_idx >= SHN_LORESERVE) {
4517 pr_warn("prog '%s': invalid relo against '%s' in special section 0x%x; forgot to initialize global var?..\n",
4518 prog->name, sym_name, shdr_idx);
4519 return -LIBBPF_ERRNO__RELOC;
4520 }
4521
4522 /* loading subprog addresses */
4523 if (sym_is_subprog(sym, obj->efile.text_shndx)) {
4524 /* global_func: sym->st_value = offset in the section, insn->imm = 0.
4525 * local_func: sym->st_value = 0, insn->imm = offset in the section.
4526 */
4527 if ((sym->st_value % BPF_INSN_SZ) || (insn->imm % BPF_INSN_SZ)) {
4528 pr_warn("prog '%s': bad subprog addr relo against '%s' at offset %zu+%d\n",
4529 prog->name, sym_name, (size_t)sym->st_value, insn->imm);
4530 return -LIBBPF_ERRNO__RELOC;
4531 }
4532
4533 reloc_desc->type = RELO_SUBPROG_ADDR;
4534 reloc_desc->insn_idx = insn_idx;
4535 reloc_desc->sym_off = sym->st_value;
4536 return 0;
4537 }
4538
4539 type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx);
4540 sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx));
4541
4542 /* arena data relocation */
4543 if (shdr_idx == obj->efile.arena_data_shndx) {
4544 reloc_desc->type = RELO_DATA;
4545 reloc_desc->insn_idx = insn_idx;
4546 reloc_desc->map_idx = obj->arena_map - obj->maps;
4547 reloc_desc->sym_off = sym->st_value;
4548 return 0;
4549 }
4550
4551 /* generic map reference relocation */
4552 if (type == LIBBPF_MAP_UNSPEC) {
4553 if (!bpf_object__shndx_is_maps(obj, shdr_idx)) {
4554 pr_warn("prog '%s': bad map relo against '%s' in section '%s'\n",
4555 prog->name, sym_name, sym_sec_name);
4556 return -LIBBPF_ERRNO__RELOC;
4557 }
4558 for (map_idx = 0; map_idx < nr_maps; map_idx++) {
4559 map = &obj->maps[map_idx];
4560 if (map->libbpf_type != type ||
4561 map->sec_idx != sym->st_shndx ||
4562 map->sec_offset != sym->st_value)
4563 continue;
4564 pr_debug("prog '%s': found map %zd (%s, sec %d, off %zu) for insn #%u\n",
4565 prog->name, map_idx, map->name, map->sec_idx,
4566 map->sec_offset, insn_idx);
4567 break;
4568 }
4569 if (map_idx >= nr_maps) {
4570 pr_warn("prog '%s': map relo failed to find map for section '%s', off %zu\n",
4571 prog->name, sym_sec_name, (size_t)sym->st_value);
4572 return -LIBBPF_ERRNO__RELOC;
4573 }
4574 reloc_desc->type = RELO_LD64;
4575 reloc_desc->insn_idx = insn_idx;
4576 reloc_desc->map_idx = map_idx;
4577 reloc_desc->sym_off = 0; /* sym->st_value determines map_idx */
4578 return 0;
4579 }
4580
4581 /* global data map relocation */
4582 if (!bpf_object__shndx_is_data(obj, shdr_idx)) {
4583 pr_warn("prog '%s': bad data relo against section '%s'\n",
4584 prog->name, sym_sec_name);
4585 return -LIBBPF_ERRNO__RELOC;
4586 }
4587 for (map_idx = 0; map_idx < nr_maps; map_idx++) {
4588 map = &obj->maps[map_idx];
4589 if (map->libbpf_type != type || map->sec_idx != sym->st_shndx)
4590 continue;
4591 pr_debug("prog '%s': found data map %zd (%s, sec %d, off %zu) for insn %u\n",
4592 prog->name, map_idx, map->name, map->sec_idx,
4593 map->sec_offset, insn_idx);
4594 break;
4595 }
4596 if (map_idx >= nr_maps) {
4597 pr_warn("prog '%s': data relo failed to find map for section '%s'\n",
4598 prog->name, sym_sec_name);
4599 return -LIBBPF_ERRNO__RELOC;
4600 }
4601
4602 reloc_desc->type = RELO_DATA;
4603 reloc_desc->insn_idx = insn_idx;
4604 reloc_desc->map_idx = map_idx;
4605 reloc_desc->sym_off = sym->st_value;
4606 return 0;
4607 }
4608
4609 static bool prog_contains_insn(const struct bpf_program *prog, size_t insn_idx)
4610 {
4611 return insn_idx >= prog->sec_insn_off &&
4612 insn_idx < prog->sec_insn_off + prog->sec_insn_cnt;
4613 }
4614
4615 static struct bpf_program *find_prog_by_sec_insn(const struct bpf_object *obj,
4616 size_t sec_idx, size_t insn_idx)
4617 {
4618 int l = 0, r = obj->nr_programs - 1, m;
4619 struct bpf_program *prog;
4620
4621 if (!obj->nr_programs)
4622 return NULL;
4623
4624 while (l < r) {
4625 m = l + (r - l + 1) / 2;
4626 prog = &obj->programs[m];
4627
4628 if (prog->sec_idx < sec_idx ||
4629 (prog->sec_idx == sec_idx && prog->sec_insn_off <= insn_idx))
4630 l = m;
4631 else
4632 r = m - 1;
4633 }
4634 /* matching program could be at index l, but it still might be the
4635 * wrong one, so we need to double check conditions for the last time
4636 */
4637 prog = &obj->programs[l];
4638 if (prog->sec_idx == sec_idx && prog_contains_insn(prog, insn_idx))
4639 return prog;
4640 return NULL;
4641 }
4642
4643 static int
4644 bpf_object__collect_prog_relos(struct bpf_object *obj, Elf64_Shdr *shdr, Elf_Data *data)
4645 {
4646 const char *relo_sec_name, *sec_name;
4647 size_t sec_idx = shdr->sh_info, sym_idx;
4648 struct bpf_program *prog;
4649 struct reloc_desc *relos;
4650 int err, i, nrels;
4651 const char *sym_name;
4652 __u32 insn_idx;
4653 Elf_Scn *scn;
4654 Elf_Data *scn_data;
4655 Elf64_Sym *sym;
4656 Elf64_Rel *rel;
4657
4658 if (sec_idx >= obj->efile.sec_cnt)
4659 return -EINVAL;
4660
4661 scn = elf_sec_by_idx(obj, sec_idx);
4662 scn_data = elf_sec_data(obj, scn);
4663 if (!scn_data)
4664 return -LIBBPF_ERRNO__FORMAT;
4665
4666 relo_sec_name = elf_sec_str(obj, shdr->sh_name);
4667 sec_name = elf_sec_name(obj, scn);
4668 if (!relo_sec_name || !sec_name)
4669 return -EINVAL;
4670
4671 pr_debug("sec '%s': collecting relocation for section(%zu) '%s'\n",
4672 relo_sec_name, sec_idx, sec_name);
4673 nrels = shdr->sh_size / shdr->sh_entsize;
4674
4675 for (i = 0; i < nrels; i++) {
4676 rel = elf_rel_by_idx(data, i);
4677 if (!rel) {
4678 pr_warn("sec '%s': failed to get relo #%d\n", relo_sec_name, i);
4679 return -LIBBPF_ERRNO__FORMAT;
4680 }
4681
4682 sym_idx = ELF64_R_SYM(rel->r_info);
4683 sym = elf_sym_by_idx(obj, sym_idx);
4684 if (!sym) {
4685 pr_warn("sec '%s': symbol #%zu not found for relo #%d\n",
4686 relo_sec_name, sym_idx, i);
4687 return -LIBBPF_ERRNO__FORMAT;
4688 }
4689
4690 if (sym->st_shndx >= obj->efile.sec_cnt) {
4691 pr_warn("sec '%s': corrupted symbol #%zu pointing to invalid section #%zu for relo #%d\n",
4692 relo_sec_name, sym_idx, (size_t)sym->st_shndx, i);
4693 return -LIBBPF_ERRNO__FORMAT;
4694 }
4695
4696 if (rel->r_offset % BPF_INSN_SZ || rel->r_offset >= scn_data->d_size) {
4697 pr_warn("sec '%s': invalid offset 0x%zx for relo #%d\n",
4698 relo_sec_name, (size_t)rel->r_offset, i);
4699 return -LIBBPF_ERRNO__FORMAT;
4700 }
4701
4702 insn_idx = rel->r_offset / BPF_INSN_SZ;
4703 /* relocations against static functions are recorded as
4704 * relocations against the section that contains a function;
4705 * in such case, symbol will be STT_SECTION and sym.st_name
4706 * will point to empty string (0), so fetch section name
4707 * instead
4708 */
4709 if (ELF64_ST_TYPE(sym->st_info) == STT_SECTION && sym->st_name == 0)
4710 sym_name = elf_sec_name(obj, elf_sec_by_idx(obj, sym->st_shndx));
4711 else
4712 sym_name = elf_sym_str(obj, sym->st_name);
4713 sym_name = sym_name ?: "<?";
4714
4715 pr_debug("sec '%s': relo #%d: insn #%u against '%s'\n",
4716 relo_sec_name, i, insn_idx, sym_name);
4717
4718 prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx);
4719 if (!prog) {
4720 pr_debug("sec '%s': relo #%d: couldn't find program in section '%s' for insn #%u, probably overridden weak function, skipping...\n",
4721 relo_sec_name, i, sec_name, insn_idx);
4722 continue;
4723 }
4724
4725 relos = libbpf_reallocarray(prog->reloc_desc,
4726 prog->nr_reloc + 1, sizeof(*relos));
4727 if (!relos)
4728 return -ENOMEM;
4729 prog->reloc_desc = relos;
4730
4731 /* adjust insn_idx to local BPF program frame of reference */
4732 insn_idx -= prog->sec_insn_off;
4733 err = bpf_program__record_reloc(prog, &relos[prog->nr_reloc],
4734 insn_idx, sym_name, sym, rel);
4735 if (err)
4736 return err;
4737
4738 prog->nr_reloc++;
4739 }
4740 return 0;
4741 }
4742
4743 static int map_fill_btf_type_info(struct bpf_object *obj, struct bpf_map *map)
4744 {
4745 int id;
4746
4747 if (!obj->btf)
4748 return -ENOENT;
4749
4750 /* if it's BTF-defined map, we don't need to search for type IDs.
4751 * For struct_ops map, it does not need btf_key_type_id and
4752 * btf_value_type_id.
4753 */
4754 if (map->sec_idx == obj->efile.btf_maps_shndx || bpf_map__is_struct_ops(map))
4755 return 0;
4756
4757 /*
4758 * LLVM annotates global data differently in BTF, that is,
4759 * only as '.data', '.bss' or '.rodata'.
4760 */
4761 if (!bpf_map__is_internal(map))
4762 return -ENOENT;
4763
4764 id = btf__find_by_name(obj->btf, map->real_name);
4765 if (id < 0)
4766 return id;
4767
4768 map->btf_key_type_id = 0;
4769 map->btf_value_type_id = id;
4770 return 0;
4771 }
4772
4773 static int bpf_get_map_info_from_fdinfo(int fd, struct bpf_map_info *info)
4774 {
4775 char file[PATH_MAX], buff[4096];
4776 FILE *fp;
4777 __u32 val;
4778 int err;
4779
4780 snprintf(file, sizeof(file), "/proc/%d/fdinfo/%d", getpid(), fd);
4781 memset(info, 0, sizeof(*info));
4782
4783 fp = fopen(file, "re");
4784 if (!fp) {
4785 err = -errno;
4786 pr_warn("failed to open %s: %d. No procfs support?\n", file,
4787 err);
4788 return err;
4789 }
4790
4791 while (fgets(buff, sizeof(buff), fp)) {
4792 if (sscanf(buff, "map_type:\t%u", &val) == 1)
4793 info->type = val;
4794 else if (sscanf(buff, "key_size:\t%u", &val) == 1)
4795 info->key_size = val;
4796 else if (sscanf(buff, "value_size:\t%u", &val) == 1)
4797 info->value_size = val;
4798 else if (sscanf(buff, "max_entries:\t%u", &val) == 1)
4799 info->max_entries = val;
4800 else if (sscanf(buff, "map_flags:\t%i", &val) == 1)
4801 info->map_flags = val;
4802 }
4803
4804 fclose(fp);
4805
4806 return 0;
4807 }
4808
4809 bool bpf_map__autocreate(const struct bpf_map *map)
4810 {
4811 return map->autocreate;
4812 }
4813
4814 int bpf_map__set_autocreate(struct bpf_map *map, bool autocreate)
4815 {
4816 if (map->obj->loaded)
4817 return libbpf_err(-EBUSY);
4818
4819 map->autocreate = autocreate;
4820 return 0;
4821 }
4822
4823 int bpf_map__set_autoattach(struct bpf_map *map, bool autoattach)
4824 {
4825 if (!bpf_map__is_struct_ops(map))
4826 return libbpf_err(-EINVAL);
4827
4828 map->autoattach = autoattach;
4829 return 0;
4830 }
4831
4832 bool bpf_map__autoattach(const struct bpf_map *map)
4833 {
4834 return map->autoattach;
4835 }
4836
4837 int bpf_map__reuse_fd(struct bpf_map *map, int fd)
4838 {
4839 struct bpf_map_info info;
4840 __u32 len = sizeof(info), name_len;
4841 int new_fd, err;
4842 char *new_name;
4843
4844 memset(&info, 0, len);
4845 err = bpf_map_get_info_by_fd(fd, &info, &len);
4846 if (err && errno == EINVAL)
4847 err = bpf_get_map_info_from_fdinfo(fd, &info);
4848 if (err)
4849 return libbpf_err(err);
4850
4851 name_len = strlen(info.name);
4852 if (name_len == BPF_OBJ_NAME_LEN - 1 && strncmp(map->name, info.name, name_len) == 0)
4853 new_name = strdup(map->name);
4854 else
4855 new_name = strdup(info.name);
4856
4857 if (!new_name)
4858 return libbpf_err(-errno);
4859
4860 /*
4861 * Like dup(), but make sure new FD is >= 3 and has O_CLOEXEC set.
4862 * This is similar to what we do in ensure_good_fd(), but without
4863 * closing original FD.
4864 */
4865 new_fd = fcntl(fd, F_DUPFD_CLOEXEC, 3);
4866 if (new_fd < 0) {
4867 err = -errno;
4868 goto err_free_new_name;
4869 }
4870
4871 err = reuse_fd(map->fd, new_fd);
4872 if (err)
4873 goto err_free_new_name;
4874
4875 free(map->name);
4876
4877 map->name = new_name;
4878 map->def.type = info.type;
4879 map->def.key_size = info.key_size;
4880 map->def.value_size = info.value_size;
4881 map->def.max_entries = info.max_entries;
4882 map->def.map_flags = info.map_flags;
4883 map->btf_key_type_id = info.btf_key_type_id;
4884 map->btf_value_type_id = info.btf_value_type_id;
4885 map->reused = true;
4886 map->map_extra = info.map_extra;
4887
4888 return 0;
4889
4890 err_free_new_name:
4891 free(new_name);
4892 return libbpf_err(err);
4893 }
4894
4895 __u32 bpf_map__max_entries(const struct bpf_map *map)
4896 {
4897 return map->def.max_entries;
4898 }
4899
4900 struct bpf_map *bpf_map__inner_map(struct bpf_map *map)
4901 {
4902 if (!bpf_map_type__is_map_in_map(map->def.type))
4903 return errno = EINVAL, NULL;
4904
4905 return map->inner_map;
4906 }
4907
4908 int bpf_map__set_max_entries(struct bpf_map *map, __u32 max_entries)
4909 {
4910 if (map->obj->loaded)
4911 return libbpf_err(-EBUSY);
4912
4913 map->def.max_entries = max_entries;
4914
4915 /* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */
4916 if (map_is_ringbuf(map))
4917 map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries);
4918
4919 return 0;
4920 }
4921
4922 static int bpf_object_prepare_token(struct bpf_object *obj)
4923 {
4924 const char *bpffs_path;
4925 int bpffs_fd = -1, token_fd, err;
4926 bool mandatory;
4927 enum libbpf_print_level level;
4928
4929 /* token is explicitly prevented */
4930 if (obj->token_path && obj->token_path[0] == '\0') {
4931 pr_debug("object '%s': token is prevented, skipping...\n", obj->name);
4932 return 0;
4933 }
4934
4935 mandatory = obj->token_path != NULL;
4936 level = mandatory ? LIBBPF_WARN : LIBBPF_DEBUG;
4937
4938 bpffs_path = obj->token_path ?: BPF_FS_DEFAULT_PATH;
4939 bpffs_fd = open(bpffs_path, O_DIRECTORY, O_RDWR);
4940 if (bpffs_fd < 0) {
4941 err = -errno;
4942 __pr(level, "object '%s': failed (%d) to open BPF FS mount at '%s'%s\n",
4943 obj->name, err, bpffs_path,
4944 mandatory ? "" : ", skipping optional step...");
4945 return mandatory ? err : 0;
4946 }
4947
4948 token_fd = bpf_token_create(bpffs_fd, 0);
4949 close(bpffs_fd);
4950 if (token_fd < 0) {
4951 if (!mandatory && token_fd == -ENOENT) {
4952 pr_debug("object '%s': BPF FS at '%s' doesn't have BPF token delegation set up, skipping...\n",
4953 obj->name, bpffs_path);
4954 return 0;
4955 }
4956 __pr(level, "object '%s': failed (%d) to create BPF token from '%s'%s\n",
4957 obj->name, token_fd, bpffs_path,
4958 mandatory ? "" : ", skipping optional step...");
4959 return mandatory ? token_fd : 0;
4960 }
4961
4962 obj->feat_cache = calloc(1, sizeof(*obj->feat_cache));
4963 if (!obj->feat_cache) {
4964 close(token_fd);
4965 return -ENOMEM;
4966 }
4967
4968 obj->token_fd = token_fd;
4969 obj->feat_cache->token_fd = token_fd;
4970
4971 return 0;
4972 }
4973
4974 static int
4975 bpf_object__probe_loading(struct bpf_object *obj)
4976 {
4977 char *cp, errmsg[STRERR_BUFSIZE];
4978 struct bpf_insn insns[] = {
4979 BPF_MOV64_IMM(BPF_REG_0, 0),
4980 BPF_EXIT_INSN(),
4981 };
4982 int ret, insn_cnt = ARRAY_SIZE(insns);
4983 LIBBPF_OPTS(bpf_prog_load_opts, opts,
4984 .token_fd = obj->token_fd,
4985 .prog_flags = obj->token_fd ? BPF_F_TOKEN_FD : 0,
4986 );
4987
4988 if (obj->gen_loader)
4989 return 0;
4990
4991 ret = bump_rlimit_memlock();
4992 if (ret)
4993 pr_warn("Failed to bump RLIMIT_MEMLOCK (err = %d), you might need to do it explicitly!\n", ret);
4994
4995 /* make sure basic loading works */
4996 ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, &opts);
4997 if (ret < 0)
4998 ret = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, &opts);
4999 if (ret < 0) {
5000 ret = errno;
5001 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg));
5002 pr_warn("Error in %s():%s(%d). Couldn't load trivial BPF "
5003 "program. Make sure your kernel supports BPF "
5004 "(CONFIG_BPF_SYSCALL=y) and/or that RLIMIT_MEMLOCK is "
5005 "set to big enough value.\n", __func__, cp, ret);
5006 return -ret;
5007 }
5008 close(ret);
5009
5010 return 0;
5011 }
5012
5013 bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id)
5014 {
5015 if (obj->gen_loader)
5016 /* To generate loader program assume the latest kernel
5017 * to avoid doing extra prog_load, map_create syscalls.
5018 */
5019 return true;
5020
5021 if (obj->token_fd)
5022 return feat_supported(obj->feat_cache, feat_id);
5023
5024 return feat_supported(NULL, feat_id);
5025 }
5026
5027 static bool map_is_reuse_compat(const struct bpf_map *map, int map_fd)
5028 {
5029 struct bpf_map_info map_info;
5030 char msg[STRERR_BUFSIZE];
5031 __u32 map_info_len = sizeof(map_info);
5032 int err;
5033
5034 memset(&map_info, 0, map_info_len);
5035 err = bpf_map_get_info_by_fd(map_fd, &map_info, &map_info_len);
5036 if (err && errno == EINVAL)
5037 err = bpf_get_map_info_from_fdinfo(map_fd, &map_info);
5038 if (err) {
5039 pr_warn("failed to get map info for map FD %d: %s\n", map_fd,
5040 libbpf_strerror_r(errno, msg, sizeof(msg)));
5041 return false;
5042 }
5043
5044 return (map_info.type == map->def.type &&
5045 map_info.key_size == map->def.key_size &&
5046 map_info.value_size == map->def.value_size &&
5047 map_info.max_entries == map->def.max_entries &&
5048 map_info.map_flags == map->def.map_flags &&
5049 map_info.map_extra == map->map_extra);
5050 }
5051
5052 static int
5053 bpf_object__reuse_map(struct bpf_map *map)
5054 {
5055 char *cp, errmsg[STRERR_BUFSIZE];
5056 int err, pin_fd;
5057
5058 pin_fd = bpf_obj_get(map->pin_path);
5059 if (pin_fd < 0) {
5060 err = -errno;
5061 if (err == -ENOENT) {
5062 pr_debug("found no pinned map to reuse at '%s'\n",
5063 map->pin_path);
5064 return 0;
5065 }
5066
5067 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
5068 pr_warn("couldn't retrieve pinned map '%s': %s\n",
5069 map->pin_path, cp);
5070 return err;
5071 }
5072
5073 if (!map_is_reuse_compat(map, pin_fd)) {
5074 pr_warn("couldn't reuse pinned map at '%s': parameter mismatch\n",
5075 map->pin_path);
5076 close(pin_fd);
5077 return -EINVAL;
5078 }
5079
5080 err = bpf_map__reuse_fd(map, pin_fd);
5081 close(pin_fd);
5082 if (err)
5083 return err;
5084
5085 map->pinned = true;
5086 pr_debug("reused pinned map at '%s'\n", map->pin_path);
5087
5088 return 0;
5089 }
5090
5091 static int
5092 bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map)
5093 {
5094 enum libbpf_map_type map_type = map->libbpf_type;
5095 char *cp, errmsg[STRERR_BUFSIZE];
5096 int err, zero = 0;
5097
5098 if (obj->gen_loader) {
5099 bpf_gen__map_update_elem(obj->gen_loader, map - obj->maps,
5100 map->mmaped, map->def.value_size);
5101 if (map_type == LIBBPF_MAP_RODATA || map_type == LIBBPF_MAP_KCONFIG)
5102 bpf_gen__map_freeze(obj->gen_loader, map - obj->maps);
5103 return 0;
5104 }
5105
5106 err = bpf_map_update_elem(map->fd, &zero, map->mmaped, 0);
5107 if (err) {
5108 err = -errno;
5109 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
5110 pr_warn("Error setting initial map(%s) contents: %s\n",
5111 map->name, cp);
5112 return err;
5113 }
5114
5115 /* Freeze .rodata and .kconfig map as read-only from syscall side. */
5116 if (map_type == LIBBPF_MAP_RODATA || map_type == LIBBPF_MAP_KCONFIG) {
5117 err = bpf_map_freeze(map->fd);
5118 if (err) {
5119 err = -errno;
5120 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
5121 pr_warn("Error freezing map(%s) as read-only: %s\n",
5122 map->name, cp);
5123 return err;
5124 }
5125 }
5126 return 0;
5127 }
5128
5129 static void bpf_map__destroy(struct bpf_map *map);
5130
5131 static bool map_is_created(const struct bpf_map *map)
5132 {
5133 return map->obj->loaded || map->reused;
5134 }
5135
5136 static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map, bool is_inner)
5137 {
5138 LIBBPF_OPTS(bpf_map_create_opts, create_attr);
5139 struct bpf_map_def *def = &map->def;
5140 const char *map_name = NULL;
5141 int err = 0, map_fd;
5142
5143 if (kernel_supports(obj, FEAT_PROG_NAME))
5144 map_name = map->name;
5145 create_attr.map_ifindex = map->map_ifindex;
5146 create_attr.map_flags = def->map_flags;
5147 create_attr.numa_node = map->numa_node;
5148 create_attr.map_extra = map->map_extra;
5149 create_attr.token_fd = obj->token_fd;
5150 if (obj->token_fd)
5151 create_attr.map_flags |= BPF_F_TOKEN_FD;
5152
5153 if (bpf_map__is_struct_ops(map)) {
5154 create_attr.btf_vmlinux_value_type_id = map->btf_vmlinux_value_type_id;
5155 if (map->mod_btf_fd >= 0) {
5156 create_attr.value_type_btf_obj_fd = map->mod_btf_fd;
5157 create_attr.map_flags |= BPF_F_VTYPE_BTF_OBJ_FD;
5158 }
5159 }
5160
5161 if (obj->btf && btf__fd(obj->btf) >= 0) {
5162 create_attr.btf_fd = btf__fd(obj->btf);
5163 create_attr.btf_key_type_id = map->btf_key_type_id;
5164 create_attr.btf_value_type_id = map->btf_value_type_id;
5165 }
5166
5167 if (bpf_map_type__is_map_in_map(def->type)) {
5168 if (map->inner_map) {
5169 err = map_set_def_max_entries(map->inner_map);
5170 if (err)
5171 return err;
5172 err = bpf_object__create_map(obj, map->inner_map, true);
5173 if (err) {
5174 pr_warn("map '%s': failed to create inner map: %d\n",
5175 map->name, err);
5176 return err;
5177 }
5178 map->inner_map_fd = map->inner_map->fd;
5179 }
5180 if (map->inner_map_fd >= 0)
5181 create_attr.inner_map_fd = map->inner_map_fd;
5182 }
5183
5184 switch (def->type) {
5185 case BPF_MAP_TYPE_PERF_EVENT_ARRAY:
5186 case BPF_MAP_TYPE_CGROUP_ARRAY:
5187 case BPF_MAP_TYPE_STACK_TRACE:
5188 case BPF_MAP_TYPE_ARRAY_OF_MAPS:
5189 case BPF_MAP_TYPE_HASH_OF_MAPS:
5190 case BPF_MAP_TYPE_DEVMAP:
5191 case BPF_MAP_TYPE_DEVMAP_HASH:
5192 case BPF_MAP_TYPE_CPUMAP:
5193 case BPF_MAP_TYPE_XSKMAP:
5194 case BPF_MAP_TYPE_SOCKMAP:
5195 case BPF_MAP_TYPE_SOCKHASH:
5196 case BPF_MAP_TYPE_QUEUE:
5197 case BPF_MAP_TYPE_STACK:
5198 case BPF_MAP_TYPE_ARENA:
5199 create_attr.btf_fd = 0;
5200 create_attr.btf_key_type_id = 0;
5201 create_attr.btf_value_type_id = 0;
5202 map->btf_key_type_id = 0;
5203 map->btf_value_type_id = 0;
5204 break;
5205 case BPF_MAP_TYPE_STRUCT_OPS:
5206 create_attr.btf_value_type_id = 0;
5207 break;
5208 default:
5209 break;
5210 }
5211
5212 if (obj->gen_loader) {
5213 bpf_gen__map_create(obj->gen_loader, def->type, map_name,
5214 def->key_size, def->value_size, def->max_entries,
5215 &create_attr, is_inner ? -1 : map - obj->maps);
5216 /* We keep pretenting we have valid FD to pass various fd >= 0
5217 * checks by just keeping original placeholder FDs in place.
5218 * See bpf_object__add_map() comment.
5219 * This placeholder fd will not be used with any syscall and
5220 * will be reset to -1 eventually.
5221 */
5222 map_fd = map->fd;
5223 } else {
5224 map_fd = bpf_map_create(def->type, map_name,
5225 def->key_size, def->value_size,
5226 def->max_entries, &create_attr);
5227 }
5228 if (map_fd < 0 && (create_attr.btf_key_type_id || create_attr.btf_value_type_id)) {
5229 char *cp, errmsg[STRERR_BUFSIZE];
5230
5231 err = -errno;
5232 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
5233 pr_warn("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n",
5234 map->name, cp, err);
5235 create_attr.btf_fd = 0;
5236 create_attr.btf_key_type_id = 0;
5237 create_attr.btf_value_type_id = 0;
5238 map->btf_key_type_id = 0;
5239 map->btf_value_type_id = 0;
5240 map_fd = bpf_map_create(def->type, map_name,
5241 def->key_size, def->value_size,
5242 def->max_entries, &create_attr);
5243 }
5244
5245 if (bpf_map_type__is_map_in_map(def->type) && map->inner_map) {
5246 if (obj->gen_loader)
5247 map->inner_map->fd = -1;
5248 bpf_map__destroy(map->inner_map);
5249 zfree(&map->inner_map);
5250 }
5251
5252 if (map_fd < 0)
5253 return map_fd;
5254
5255 /* obj->gen_loader case, prevent reuse_fd() from closing map_fd */
5256 if (map->fd == map_fd)
5257 return 0;
5258
5259 /* Keep placeholder FD value but now point it to the BPF map object.
5260 * This way everything that relied on this map's FD (e.g., relocated
5261 * ldimm64 instructions) will stay valid and won't need adjustments.
5262 * map->fd stays valid but now point to what map_fd points to.
5263 */
5264 return reuse_fd(map->fd, map_fd);
5265 }
5266
5267 static int init_map_in_map_slots(struct bpf_object *obj, struct bpf_map *map)
5268 {
5269 const struct bpf_map *targ_map;
5270 unsigned int i;
5271 int fd, err = 0;
5272
5273 for (i = 0; i < map->init_slots_sz; i++) {
5274 if (!map->init_slots[i])
5275 continue;
5276
5277 targ_map = map->init_slots[i];
5278 fd = targ_map->fd;
5279
5280 if (obj->gen_loader) {
5281 bpf_gen__populate_outer_map(obj->gen_loader,
5282 map - obj->maps, i,
5283 targ_map - obj->maps);
5284 } else {
5285 err = bpf_map_update_elem(map->fd, &i, &fd, 0);
5286 }
5287 if (err) {
5288 err = -errno;
5289 pr_warn("map '%s': failed to initialize slot [%d] to map '%s' fd=%d: %d\n",
5290 map->name, i, targ_map->name, fd, err);
5291 return err;
5292 }
5293 pr_debug("map '%s': slot [%d] set to map '%s' fd=%d\n",
5294 map->name, i, targ_map->name, fd);
5295 }
5296
5297 zfree(&map->init_slots);
5298 map->init_slots_sz = 0;
5299
5300 return 0;
5301 }
5302
5303 static int init_prog_array_slots(struct bpf_object *obj, struct bpf_map *map)
5304 {
5305 const struct bpf_program *targ_prog;
5306 unsigned int i;
5307 int fd, err;
5308
5309 if (obj->gen_loader)
5310 return -ENOTSUP;
5311
5312 for (i = 0; i < map->init_slots_sz; i++) {
5313 if (!map->init_slots[i])
5314 continue;
5315
5316 targ_prog = map->init_slots[i];
5317 fd = bpf_program__fd(targ_prog);
5318
5319 err = bpf_map_update_elem(map->fd, &i, &fd, 0);
5320 if (err) {
5321 err = -errno;
5322 pr_warn("map '%s': failed to initialize slot [%d] to prog '%s' fd=%d: %d\n",
5323 map->name, i, targ_prog->name, fd, err);
5324 return err;
5325 }
5326 pr_debug("map '%s': slot [%d] set to prog '%s' fd=%d\n",
5327 map->name, i, targ_prog->name, fd);
5328 }
5329
5330 zfree(&map->init_slots);
5331 map->init_slots_sz = 0;
5332
5333 return 0;
5334 }
5335
5336 static int bpf_object_init_prog_arrays(struct bpf_object *obj)
5337 {
5338 struct bpf_map *map;
5339 int i, err;
5340
5341 for (i = 0; i < obj->nr_maps; i++) {
5342 map = &obj->maps[i];
5343
5344 if (!map->init_slots_sz || map->def.type != BPF_MAP_TYPE_PROG_ARRAY)
5345 continue;
5346
5347 err = init_prog_array_slots(obj, map);
5348 if (err < 0)
5349 return err;
5350 }
5351 return 0;
5352 }
5353
5354 static int map_set_def_max_entries(struct bpf_map *map)
5355 {
5356 if (map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY && !map->def.max_entries) {
5357 int nr_cpus;
5358
5359 nr_cpus = libbpf_num_possible_cpus();
5360 if (nr_cpus < 0) {
5361 pr_warn("map '%s': failed to determine number of system CPUs: %d\n",
5362 map->name, nr_cpus);
5363 return nr_cpus;
5364 }
5365 pr_debug("map '%s': setting size to %d\n", map->name, nr_cpus);
5366 map->def.max_entries = nr_cpus;
5367 }
5368
5369 return 0;
5370 }
5371
5372 static int
5373 bpf_object__create_maps(struct bpf_object *obj)
5374 {
5375 struct bpf_map *map;
5376 char *cp, errmsg[STRERR_BUFSIZE];
5377 unsigned int i, j;
5378 int err;
5379 bool retried;
5380
5381 for (i = 0; i < obj->nr_maps; i++) {
5382 map = &obj->maps[i];
5383
5384 /* To support old kernels, we skip creating global data maps
5385 * (.rodata, .data, .kconfig, etc); later on, during program
5386 * loading, if we detect that at least one of the to-be-loaded
5387 * programs is referencing any global data map, we'll error
5388 * out with program name and relocation index logged.
5389 * This approach allows to accommodate Clang emitting
5390 * unnecessary .rodata.str1.1 sections for string literals,
5391 * but also it allows to have CO-RE applications that use
5392 * global variables in some of BPF programs, but not others.
5393 * If those global variable-using programs are not loaded at
5394 * runtime due to bpf_program__set_autoload(prog, false),
5395 * bpf_object loading will succeed just fine even on old
5396 * kernels.
5397 */
5398 if (bpf_map__is_internal(map) && !kernel_supports(obj, FEAT_GLOBAL_DATA))
5399 map->autocreate = false;
5400
5401 if (!map->autocreate) {
5402 pr_debug("map '%s': skipped auto-creating...\n", map->name);
5403 continue;
5404 }
5405
5406 err = map_set_def_max_entries(map);
5407 if (err)
5408 goto err_out;
5409
5410 retried = false;
5411 retry:
5412 if (map->pin_path) {
5413 err = bpf_object__reuse_map(map);
5414 if (err) {
5415 pr_warn("map '%s': error reusing pinned map\n",
5416 map->name);
5417 goto err_out;
5418 }
5419 if (retried && map->fd < 0) {
5420 pr_warn("map '%s': cannot find pinned map\n",
5421 map->name);
5422 err = -ENOENT;
5423 goto err_out;
5424 }
5425 }
5426
5427 if (map->reused) {
5428 pr_debug("map '%s': skipping creation (preset fd=%d)\n",
5429 map->name, map->fd);
5430 } else {
5431 err = bpf_object__create_map(obj, map, false);
5432 if (err)
5433 goto err_out;
5434
5435 pr_debug("map '%s': created successfully, fd=%d\n",
5436 map->name, map->fd);
5437
5438 if (bpf_map__is_internal(map)) {
5439 err = bpf_object__populate_internal_map(obj, map);
5440 if (err < 0)
5441 goto err_out;
5442 }
5443 if (map->def.type == BPF_MAP_TYPE_ARENA) {
5444 map->mmaped = mmap((void *)(long)map->map_extra,
5445 bpf_map_mmap_sz(map), PROT_READ | PROT_WRITE,
5446 map->map_extra ? MAP_SHARED | MAP_FIXED : MAP_SHARED,
5447 map->fd, 0);
5448 if (map->mmaped == MAP_FAILED) {
5449 err = -errno;
5450 map->mmaped = NULL;
5451 pr_warn("map '%s': failed to mmap arena: %d\n",
5452 map->name, err);
5453 return err;
5454 }
5455 if (obj->arena_data) {
5456 memcpy(map->mmaped, obj->arena_data, obj->arena_data_sz);
5457 zfree(&obj->arena_data);
5458 }
5459 }
5460 if (map->init_slots_sz && map->def.type != BPF_MAP_TYPE_PROG_ARRAY) {
5461 err = init_map_in_map_slots(obj, map);
5462 if (err < 0)
5463 goto err_out;
5464 }
5465 }
5466
5467 if (map->pin_path && !map->pinned) {
5468 err = bpf_map__pin(map, NULL);
5469 if (err) {
5470 if (!retried && err == -EEXIST) {
5471 retried = true;
5472 goto retry;
5473 }
5474 pr_warn("map '%s': failed to auto-pin at '%s': %d\n",
5475 map->name, map->pin_path, err);
5476 goto err_out;
5477 }
5478 }
5479 }
5480
5481 return 0;
5482
5483 err_out:
5484 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
5485 pr_warn("map '%s': failed to create: %s(%d)\n", map->name, cp, err);
5486 pr_perm_msg(err);
5487 for (j = 0; j < i; j++)
5488 zclose(obj->maps[j].fd);
5489 return err;
5490 }
5491
5492 static bool bpf_core_is_flavor_sep(const char *s)
5493 {
5494 /* check X___Y name pattern, where X and Y are not underscores */
5495 return s[0] != '_' && /* X */
5496 s[1] == '_' && s[2] == '_' && s[3] == '_' && /* ___ */
5497 s[4] != '_'; /* Y */
5498 }
5499
5500 /* Given 'some_struct_name___with_flavor' return the length of a name prefix
5501 * before last triple underscore. Struct name part after last triple
5502 * underscore is ignored by BPF CO-RE relocation during relocation matching.
5503 */
5504 size_t bpf_core_essential_name_len(const char *name)
5505 {
5506 size_t n = strlen(name);
5507 int i;
5508
5509 for (i = n - 5; i >= 0; i--) {
5510 if (bpf_core_is_flavor_sep(name + i))
5511 return i + 1;
5512 }
5513 return n;
5514 }
5515
5516 void bpf_core_free_cands(struct bpf_core_cand_list *cands)
5517 {
5518 if (!cands)
5519 return;
5520
5521 free(cands->cands);
5522 free(cands);
5523 }
5524
5525 int bpf_core_add_cands(struct bpf_core_cand *local_cand,
5526 size_t local_essent_len,
5527 const struct btf *targ_btf,
5528 const char *targ_btf_name,
5529 int targ_start_id,
5530 struct bpf_core_cand_list *cands)
5531 {
5532 struct bpf_core_cand *new_cands, *cand;
5533 const struct btf_type *t, *local_t;
5534 const char *targ_name, *local_name;
5535 size_t targ_essent_len;
5536 int n, i;
5537
5538 local_t = btf__type_by_id(local_cand->btf, local_cand->id);
5539 local_name = btf__str_by_offset(local_cand->btf, local_t->name_off);
5540
5541 n = btf__type_cnt(targ_btf);
5542 for (i = targ_start_id; i < n; i++) {
5543 t = btf__type_by_id(targ_btf, i);
5544 if (!btf_kind_core_compat(t, local_t))
5545 continue;
5546
5547 targ_name = btf__name_by_offset(targ_btf, t->name_off);
5548 if (str_is_empty(targ_name))
5549 continue;
5550
5551 targ_essent_len = bpf_core_essential_name_len(targ_name);
5552 if (targ_essent_len != local_essent_len)
5553 continue;
5554
5555 if (strncmp(local_name, targ_name, local_essent_len) != 0)
5556 continue;
5557
5558 pr_debug("CO-RE relocating [%d] %s %s: found target candidate [%d] %s %s in [%s]\n",
5559 local_cand->id, btf_kind_str(local_t),
5560 local_name, i, btf_kind_str(t), targ_name,
5561 targ_btf_name);
5562 new_cands = libbpf_reallocarray(cands->cands, cands->len + 1,
5563 sizeof(*cands->cands));
5564 if (!new_cands)
5565 return -ENOMEM;
5566
5567 cand = &new_cands[cands->len];
5568 cand->btf = targ_btf;
5569 cand->id = i;
5570
5571 cands->cands = new_cands;
5572 cands->len++;
5573 }
5574 return 0;
5575 }
5576
5577 static int load_module_btfs(struct bpf_object *obj)
5578 {
5579 struct bpf_btf_info info;
5580 struct module_btf *mod_btf;
5581 struct btf *btf;
5582 char name[64];
5583 __u32 id = 0, len;
5584 int err, fd;
5585
5586 if (obj->btf_modules_loaded)
5587 return 0;
5588
5589 if (obj->gen_loader)
5590 return 0;
5591
5592 /* don't do this again, even if we find no module BTFs */
5593 obj->btf_modules_loaded = true;
5594
5595 /* kernel too old to support module BTFs */
5596 if (!kernel_supports(obj, FEAT_MODULE_BTF))
5597 return 0;
5598
5599 while (true) {
5600 err = bpf_btf_get_next_id(id, &id);
5601 if (err && errno == ENOENT)
5602 return 0;
5603 if (err && errno == EPERM) {
5604 pr_debug("skipping module BTFs loading, missing privileges\n");
5605 return 0;
5606 }
5607 if (err) {
5608 err = -errno;
5609 pr_warn("failed to iterate BTF objects: %d\n", err);
5610 return err;
5611 }
5612
5613 fd = bpf_btf_get_fd_by_id(id);
5614 if (fd < 0) {
5615 if (errno == ENOENT)
5616 continue; /* expected race: BTF was unloaded */
5617 err = -errno;
5618 pr_warn("failed to get BTF object #%d FD: %d\n", id, err);
5619 return err;
5620 }
5621
5622 len = sizeof(info);
5623 memset(&info, 0, sizeof(info));
5624 info.name = ptr_to_u64(name);
5625 info.name_len = sizeof(name);
5626
5627 err = bpf_btf_get_info_by_fd(fd, &info, &len);
5628 if (err) {
5629 err = -errno;
5630 pr_warn("failed to get BTF object #%d info: %d\n", id, err);
5631 goto err_out;
5632 }
5633
5634 /* ignore non-module BTFs */
5635 if (!info.kernel_btf || strcmp(name, "vmlinux") == 0) {
5636 close(fd);
5637 continue;
5638 }
5639
5640 btf = btf_get_from_fd(fd, obj->btf_vmlinux);
5641 err = libbpf_get_error(btf);
5642 if (err) {
5643 pr_warn("failed to load module [%s]'s BTF object #%d: %d\n",
5644 name, id, err);
5645 goto err_out;
5646 }
5647
5648 err = libbpf_ensure_mem((void **)&obj->btf_modules, &obj->btf_module_cap,
5649 sizeof(*obj->btf_modules), obj->btf_module_cnt + 1);
5650 if (err)
5651 goto err_out;
5652
5653 mod_btf = &obj->btf_modules[obj->btf_module_cnt++];
5654
5655 mod_btf->btf = btf;
5656 mod_btf->id = id;
5657 mod_btf->fd = fd;
5658 mod_btf->name = strdup(name);
5659 if (!mod_btf->name) {
5660 err = -ENOMEM;
5661 goto err_out;
5662 }
5663 continue;
5664
5665 err_out:
5666 close(fd);
5667 return err;
5668 }
5669
5670 return 0;
5671 }
5672
5673 static struct bpf_core_cand_list *
5674 bpf_core_find_cands(struct bpf_object *obj, const struct btf *local_btf, __u32 local_type_id)
5675 {
5676 struct bpf_core_cand local_cand = {};
5677 struct bpf_core_cand_list *cands;
5678 const struct btf *main_btf;
5679 const struct btf_type *local_t;
5680 const char *local_name;
5681 size_t local_essent_len;
5682 int err, i;
5683
5684 local_cand.btf = local_btf;
5685 local_cand.id = local_type_id;
5686 local_t = btf__type_by_id(local_btf, local_type_id);
5687 if (!local_t)
5688 return ERR_PTR(-EINVAL);
5689
5690 local_name = btf__name_by_offset(local_btf, local_t->name_off);
5691 if (str_is_empty(local_name))
5692 return ERR_PTR(-EINVAL);
5693 local_essent_len = bpf_core_essential_name_len(local_name);
5694
5695 cands = calloc(1, sizeof(*cands));
5696 if (!cands)
5697 return ERR_PTR(-ENOMEM);
5698
5699 /* Attempt to find target candidates in vmlinux BTF first */
5700 main_btf = obj->btf_vmlinux_override ?: obj->btf_vmlinux;
5701 err = bpf_core_add_cands(&local_cand, local_essent_len, main_btf, "vmlinux", 1, cands);
5702 if (err)
5703 goto err_out;
5704
5705 /* if vmlinux BTF has any candidate, don't got for module BTFs */
5706 if (cands->len)
5707 return cands;
5708
5709 /* if vmlinux BTF was overridden, don't attempt to load module BTFs */
5710 if (obj->btf_vmlinux_override)
5711 return cands;
5712
5713 /* now look through module BTFs, trying to still find candidates */
5714 err = load_module_btfs(obj);
5715 if (err)
5716 goto err_out;
5717
5718 for (i = 0; i < obj->btf_module_cnt; i++) {
5719 err = bpf_core_add_cands(&local_cand, local_essent_len,
5720 obj->btf_modules[i].btf,
5721 obj->btf_modules[i].name,
5722 btf__type_cnt(obj->btf_vmlinux),
5723 cands);
5724 if (err)
5725 goto err_out;
5726 }
5727
5728 return cands;
5729 err_out:
5730 bpf_core_free_cands(cands);
5731 return ERR_PTR(err);
5732 }
5733
5734 /* Check local and target types for compatibility. This check is used for
5735 * type-based CO-RE relocations and follow slightly different rules than
5736 * field-based relocations. This function assumes that root types were already
5737 * checked for name match. Beyond that initial root-level name check, names
5738 * are completely ignored. Compatibility rules are as follows:
5739 * - any two STRUCTs/UNIONs/FWDs/ENUMs/INTs are considered compatible, but
5740 * kind should match for local and target types (i.e., STRUCT is not
5741 * compatible with UNION);
5742 * - for ENUMs, the size is ignored;
5743 * - for INT, size and signedness are ignored;
5744 * - for ARRAY, dimensionality is ignored, element types are checked for
5745 * compatibility recursively;
5746 * - CONST/VOLATILE/RESTRICT modifiers are ignored;
5747 * - TYPEDEFs/PTRs are compatible if types they pointing to are compatible;
5748 * - FUNC_PROTOs are compatible if they have compatible signature: same
5749 * number of input args and compatible return and argument types.
5750 * These rules are not set in stone and probably will be adjusted as we get
5751 * more experience with using BPF CO-RE relocations.
5752 */
5753 int bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id,
5754 const struct btf *targ_btf, __u32 targ_id)
5755 {
5756 return __bpf_core_types_are_compat(local_btf, local_id, targ_btf, targ_id, 32);
5757 }
5758
5759 int bpf_core_types_match(const struct btf *local_btf, __u32 local_id,
5760 const struct btf *targ_btf, __u32 targ_id)
5761 {
5762 return __bpf_core_types_match(local_btf, local_id, targ_btf, targ_id, false, 32);
5763 }
5764
5765 static size_t bpf_core_hash_fn(const long key, void *ctx)
5766 {
5767 return key;
5768 }
5769
5770 static bool bpf_core_equal_fn(const long k1, const long k2, void *ctx)
5771 {
5772 return k1 == k2;
5773 }
5774
5775 static int record_relo_core(struct bpf_program *prog,
5776 const struct bpf_core_relo *core_relo, int insn_idx)
5777 {
5778 struct reloc_desc *relos, *relo;
5779
5780 relos = libbpf_reallocarray(prog->reloc_desc,
5781 prog->nr_reloc + 1, sizeof(*relos));
5782 if (!relos)
5783 return -ENOMEM;
5784 relo = &relos[prog->nr_reloc];
5785 relo->type = RELO_CORE;
5786 relo->insn_idx = insn_idx;
5787 relo->core_relo = core_relo;
5788 prog->reloc_desc = relos;
5789 prog->nr_reloc++;
5790 return 0;
5791 }
5792
5793 static const struct bpf_core_relo *find_relo_core(struct bpf_program *prog, int insn_idx)
5794 {
5795 struct reloc_desc *relo;
5796 int i;
5797
5798 for (i = 0; i < prog->nr_reloc; i++) {
5799 relo = &prog->reloc_desc[i];
5800 if (relo->type != RELO_CORE || relo->insn_idx != insn_idx)
5801 continue;
5802
5803 return relo->core_relo;
5804 }
5805
5806 return NULL;
5807 }
5808
5809 static int bpf_core_resolve_relo(struct bpf_program *prog,
5810 const struct bpf_core_relo *relo,
5811 int relo_idx,
5812 const struct btf *local_btf,
5813 struct hashmap *cand_cache,
5814 struct bpf_core_relo_res *targ_res)
5815 {
5816 struct bpf_core_spec specs_scratch[3] = {};
5817 struct bpf_core_cand_list *cands = NULL;
5818 const char *prog_name = prog->name;
5819 const struct btf_type *local_type;
5820 const char *local_name;
5821 __u32 local_id = relo->type_id;
5822 int err;
5823
5824 local_type = btf__type_by_id(local_btf, local_id);
5825 if (!local_type)
5826 return -EINVAL;
5827
5828 local_name = btf__name_by_offset(local_btf, local_type->name_off);
5829 if (!local_name)
5830 return -EINVAL;
5831
5832 if (relo->kind != BPF_CORE_TYPE_ID_LOCAL &&
5833 !hashmap__find(cand_cache, local_id, &cands)) {
5834 cands = bpf_core_find_cands(prog->obj, local_btf, local_id);
5835 if (IS_ERR(cands)) {
5836 pr_warn("prog '%s': relo #%d: target candidate search failed for [%d] %s %s: %ld\n",
5837 prog_name, relo_idx, local_id, btf_kind_str(local_type),
5838 local_name, PTR_ERR(cands));
5839 return PTR_ERR(cands);
5840 }
5841 err = hashmap__set(cand_cache, local_id, cands, NULL, NULL);
5842 if (err) {
5843 bpf_core_free_cands(cands);
5844 return err;
5845 }
5846 }
5847
5848 return bpf_core_calc_relo_insn(prog_name, relo, relo_idx, local_btf, cands, specs_scratch,
5849 targ_res);
5850 }
5851
5852 static int
5853 bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path)
5854 {
5855 const struct btf_ext_info_sec *sec;
5856 struct bpf_core_relo_res targ_res;
5857 const struct bpf_core_relo *rec;
5858 const struct btf_ext_info *seg;
5859 struct hashmap_entry *entry;
5860 struct hashmap *cand_cache = NULL;
5861 struct bpf_program *prog;
5862 struct bpf_insn *insn;
5863 const char *sec_name;
5864 int i, err = 0, insn_idx, sec_idx, sec_num;
5865
5866 if (obj->btf_ext->core_relo_info.len == 0)
5867 return 0;
5868
5869 if (targ_btf_path) {
5870 obj->btf_vmlinux_override = btf__parse(targ_btf_path, NULL);
5871 err = libbpf_get_error(obj->btf_vmlinux_override);
5872 if (err) {
5873 pr_warn("failed to parse target BTF: %d\n", err);
5874 return err;
5875 }
5876 }
5877
5878 cand_cache = hashmap__new(bpf_core_hash_fn, bpf_core_equal_fn, NULL);
5879 if (IS_ERR(cand_cache)) {
5880 err = PTR_ERR(cand_cache);
5881 goto out;
5882 }
5883
5884 seg = &obj->btf_ext->core_relo_info;
5885 sec_num = 0;
5886 for_each_btf_ext_sec(seg, sec) {
5887 sec_idx = seg->sec_idxs[sec_num];
5888 sec_num++;
5889
5890 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);
5891 if (str_is_empty(sec_name)) {
5892 err = -EINVAL;
5893 goto out;
5894 }
5895
5896 pr_debug("sec '%s': found %d CO-RE relocations\n", sec_name, sec->num_info);
5897
5898 for_each_btf_ext_rec(seg, sec, i, rec) {
5899 if (rec->insn_off % BPF_INSN_SZ)
5900 return -EINVAL;
5901 insn_idx = rec->insn_off / BPF_INSN_SZ;
5902 prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx);
5903 if (!prog) {
5904 /* When __weak subprog is "overridden" by another instance
5905 * of the subprog from a different object file, linker still
5906 * appends all the .BTF.ext info that used to belong to that
5907 * eliminated subprogram.
5908 * This is similar to what x86-64 linker does for relocations.
5909 * So just ignore such relocations just like we ignore
5910 * subprog instructions when discovering subprograms.
5911 */
5912 pr_debug("sec '%s': skipping CO-RE relocation #%d for insn #%d belonging to eliminated weak subprogram\n",
5913 sec_name, i, insn_idx);
5914 continue;
5915 }
5916 /* no need to apply CO-RE relocation if the program is
5917 * not going to be loaded
5918 */
5919 if (!prog->autoload)
5920 continue;
5921
5922 /* adjust insn_idx from section frame of reference to the local
5923 * program's frame of reference; (sub-)program code is not yet
5924 * relocated, so it's enough to just subtract in-section offset
5925 */
5926 insn_idx = insn_idx - prog->sec_insn_off;
5927 if (insn_idx >= prog->insns_cnt)
5928 return -EINVAL;
5929 insn = &prog->insns[insn_idx];
5930
5931 err = record_relo_core(prog, rec, insn_idx);
5932 if (err) {
5933 pr_warn("prog '%s': relo #%d: failed to record relocation: %d\n",
5934 prog->name, i, err);
5935 goto out;
5936 }
5937
5938 if (prog->obj->gen_loader)
5939 continue;
5940
5941 err = bpf_core_resolve_relo(prog, rec, i, obj->btf, cand_cache, &targ_res);
5942 if (err) {
5943 pr_warn("prog '%s': relo #%d: failed to relocate: %d\n",
5944 prog->name, i, err);
5945 goto out;
5946 }
5947
5948 err = bpf_core_patch_insn(prog->name, insn, insn_idx, rec, i, &targ_res);
5949 if (err) {
5950 pr_warn("prog '%s': relo #%d: failed to patch insn #%u: %d\n",
5951 prog->name, i, insn_idx, err);
5952 goto out;
5953 }
5954 }
5955 }
5956
5957 out:
5958 /* obj->btf_vmlinux and module BTFs are freed after object load */
5959 btf__free(obj->btf_vmlinux_override);
5960 obj->btf_vmlinux_override = NULL;
5961
5962 if (!IS_ERR_OR_NULL(cand_cache)) {
5963 hashmap__for_each_entry(cand_cache, entry, i) {
5964 bpf_core_free_cands(entry->pvalue);
5965 }
5966 hashmap__free(cand_cache);
5967 }
5968 return err;
5969 }
5970
5971 /* base map load ldimm64 special constant, used also for log fixup logic */
5972 #define POISON_LDIMM64_MAP_BASE 2001000000
5973 #define POISON_LDIMM64_MAP_PFX "200100"
5974
5975 static void poison_map_ldimm64(struct bpf_program *prog, int relo_idx,
5976 int insn_idx, struct bpf_insn *insn,
5977 int map_idx, const struct bpf_map *map)
5978 {
5979 int i;
5980
5981 pr_debug("prog '%s': relo #%d: poisoning insn #%d that loads map #%d '%s'\n",
5982 prog->name, relo_idx, insn_idx, map_idx, map->name);
5983
5984 /* we turn single ldimm64 into two identical invalid calls */
5985 for (i = 0; i < 2; i++) {
5986 insn->code = BPF_JMP | BPF_CALL;
5987 insn->dst_reg = 0;
5988 insn->src_reg = 0;
5989 insn->off = 0;
5990 /* if this instruction is reachable (not a dead code),
5991 * verifier will complain with something like:
5992 * invalid func unknown#2001000123
5993 * where lower 123 is map index into obj->maps[] array
5994 */
5995 insn->imm = POISON_LDIMM64_MAP_BASE + map_idx;
5996
5997 insn++;
5998 }
5999 }
6000
6001 /* unresolved kfunc call special constant, used also for log fixup logic */
6002 #define POISON_CALL_KFUNC_BASE 2002000000
6003 #define POISON_CALL_KFUNC_PFX "2002"
6004
6005 static void poison_kfunc_call(struct bpf_program *prog, int relo_idx,
6006 int insn_idx, struct bpf_insn *insn,
6007 int ext_idx, const struct extern_desc *ext)
6008 {
6009 pr_debug("prog '%s': relo #%d: poisoning insn #%d that calls kfunc '%s'\n",
6010 prog->name, relo_idx, insn_idx, ext->name);
6011
6012 /* we turn kfunc call into invalid helper call with identifiable constant */
6013 insn->code = BPF_JMP | BPF_CALL;
6014 insn->dst_reg = 0;
6015 insn->src_reg = 0;
6016 insn->off = 0;
6017 /* if this instruction is reachable (not a dead code),
6018 * verifier will complain with something like:
6019 * invalid func unknown#2001000123
6020 * where lower 123 is extern index into obj->externs[] array
6021 */
6022 insn->imm = POISON_CALL_KFUNC_BASE + ext_idx;
6023 }
6024
6025 /* Relocate data references within program code:
6026 * - map references;
6027 * - global variable references;
6028 * - extern references.
6029 */
6030 static int
6031 bpf_object__relocate_data(struct bpf_object *obj, struct bpf_program *prog)
6032 {
6033 int i;
6034
6035 for (i = 0; i < prog->nr_reloc; i++) {
6036 struct reloc_desc *relo = &prog->reloc_desc[i];
6037 struct bpf_insn *insn = &prog->insns[relo->insn_idx];
6038 const struct bpf_map *map;
6039 struct extern_desc *ext;
6040
6041 switch (relo->type) {
6042 case RELO_LD64:
6043 map = &obj->maps[relo->map_idx];
6044 if (obj->gen_loader) {
6045 insn[0].src_reg = BPF_PSEUDO_MAP_IDX;
6046 insn[0].imm = relo->map_idx;
6047 } else if (map->autocreate) {
6048 insn[0].src_reg = BPF_PSEUDO_MAP_FD;
6049 insn[0].imm = map->fd;
6050 } else {
6051 poison_map_ldimm64(prog, i, relo->insn_idx, insn,
6052 relo->map_idx, map);
6053 }
6054 break;
6055 case RELO_DATA:
6056 map = &obj->maps[relo->map_idx];
6057 insn[1].imm = insn[0].imm + relo->sym_off;
6058 if (obj->gen_loader) {
6059 insn[0].src_reg = BPF_PSEUDO_MAP_IDX_VALUE;
6060 insn[0].imm = relo->map_idx;
6061 } else if (map->autocreate) {
6062 insn[0].src_reg = BPF_PSEUDO_MAP_VALUE;
6063 insn[0].imm = map->fd;
6064 } else {
6065 poison_map_ldimm64(prog, i, relo->insn_idx, insn,
6066 relo->map_idx, map);
6067 }
6068 break;
6069 case RELO_EXTERN_LD64:
6070 ext = &obj->externs[relo->ext_idx];
6071 if (ext->type == EXT_KCFG) {
6072 if (obj->gen_loader) {
6073 insn[0].src_reg = BPF_PSEUDO_MAP_IDX_VALUE;
6074 insn[0].imm = obj->kconfig_map_idx;
6075 } else {
6076 insn[0].src_reg = BPF_PSEUDO_MAP_VALUE;
6077 insn[0].imm = obj->maps[obj->kconfig_map_idx].fd;
6078 }
6079 insn[1].imm = ext->kcfg.data_off;
6080 } else /* EXT_KSYM */ {
6081 if (ext->ksym.type_id && ext->is_set) { /* typed ksyms */
6082 insn[0].src_reg = BPF_PSEUDO_BTF_ID;
6083 insn[0].imm = ext->ksym.kernel_btf_id;
6084 insn[1].imm = ext->ksym.kernel_btf_obj_fd;
6085 } else { /* typeless ksyms or unresolved typed ksyms */
6086 insn[0].imm = (__u32)ext->ksym.addr;
6087 insn[1].imm = ext->ksym.addr >> 32;
6088 }
6089 }
6090 break;
6091 case RELO_EXTERN_CALL:
6092 ext = &obj->externs[relo->ext_idx];
6093 insn[0].src_reg = BPF_PSEUDO_KFUNC_CALL;
6094 if (ext->is_set) {
6095 insn[0].imm = ext->ksym.kernel_btf_id;
6096 insn[0].off = ext->ksym.btf_fd_idx;
6097 } else { /* unresolved weak kfunc call */
6098 poison_kfunc_call(prog, i, relo->insn_idx, insn,
6099 relo->ext_idx, ext);
6100 }
6101 break;
6102 case RELO_SUBPROG_ADDR:
6103 if (insn[0].src_reg != BPF_PSEUDO_FUNC) {
6104 pr_warn("prog '%s': relo #%d: bad insn\n",
6105 prog->name, i);
6106 return -EINVAL;
6107 }
6108 /* handled already */
6109 break;
6110 case RELO_CALL:
6111 /* handled already */
6112 break;
6113 case RELO_CORE:
6114 /* will be handled by bpf_program_record_relos() */
6115 break;
6116 default:
6117 pr_warn("prog '%s': relo #%d: bad relo type %d\n",
6118 prog->name, i, relo->type);
6119 return -EINVAL;
6120 }
6121 }
6122
6123 return 0;
6124 }
6125
6126 static int adjust_prog_btf_ext_info(const struct bpf_object *obj,
6127 const struct bpf_program *prog,
6128 const struct btf_ext_info *ext_info,
6129 void **prog_info, __u32 *prog_rec_cnt,
6130 __u32 *prog_rec_sz)
6131 {
6132 void *copy_start = NULL, *copy_end = NULL;
6133 void *rec, *rec_end, *new_prog_info;
6134 const struct btf_ext_info_sec *sec;
6135 size_t old_sz, new_sz;
6136 int i, sec_num, sec_idx, off_adj;
6137
6138 sec_num = 0;
6139 for_each_btf_ext_sec(ext_info, sec) {
6140 sec_idx = ext_info->sec_idxs[sec_num];
6141 sec_num++;
6142 if (prog->sec_idx != sec_idx)
6143 continue;
6144
6145 for_each_btf_ext_rec(ext_info, sec, i, rec) {
6146 __u32 insn_off = *(__u32 *)rec / BPF_INSN_SZ;
6147
6148 if (insn_off < prog->sec_insn_off)
6149 continue;
6150 if (insn_off >= prog->sec_insn_off + prog->sec_insn_cnt)
6151 break;
6152
6153 if (!copy_start)
6154 copy_start = rec;
6155 copy_end = rec + ext_info->rec_size;
6156 }
6157
6158 if (!copy_start)
6159 return -ENOENT;
6160
6161 /* append func/line info of a given (sub-)program to the main
6162 * program func/line info
6163 */
6164 old_sz = (size_t)(*prog_rec_cnt) * ext_info->rec_size;
6165 new_sz = old_sz + (copy_end - copy_start);
6166 new_prog_info = realloc(*prog_info, new_sz);
6167 if (!new_prog_info)
6168 return -ENOMEM;
6169 *prog_info = new_prog_info;
6170 *prog_rec_cnt = new_sz / ext_info->rec_size;
6171 memcpy(new_prog_info + old_sz, copy_start, copy_end - copy_start);
6172
6173 /* Kernel instruction offsets are in units of 8-byte
6174 * instructions, while .BTF.ext instruction offsets generated
6175 * by Clang are in units of bytes. So convert Clang offsets
6176 * into kernel offsets and adjust offset according to program
6177 * relocated position.
6178 */
6179 off_adj = prog->sub_insn_off - prog->sec_insn_off;
6180 rec = new_prog_info + old_sz;
6181 rec_end = new_prog_info + new_sz;
6182 for (; rec < rec_end; rec += ext_info->rec_size) {
6183 __u32 *insn_off = rec;
6184
6185 *insn_off = *insn_off / BPF_INSN_SZ + off_adj;
6186 }
6187 *prog_rec_sz = ext_info->rec_size;
6188 return 0;
6189 }
6190
6191 return -ENOENT;
6192 }
6193
6194 static int
6195 reloc_prog_func_and_line_info(const struct bpf_object *obj,
6196 struct bpf_program *main_prog,
6197 const struct bpf_program *prog)
6198 {
6199 int err;
6200
6201 /* no .BTF.ext relocation if .BTF.ext is missing or kernel doesn't
6202 * support func/line info
6203 */
6204 if (!obj->btf_ext || !kernel_supports(obj, FEAT_BTF_FUNC))
6205 return 0;
6206
6207 /* only attempt func info relocation if main program's func_info
6208 * relocation was successful
6209 */
6210 if (main_prog != prog && !main_prog->func_info)
6211 goto line_info;
6212
6213 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->func_info,
6214 &main_prog->func_info,
6215 &main_prog->func_info_cnt,
6216 &main_prog->func_info_rec_size);
6217 if (err) {
6218 if (err != -ENOENT) {
6219 pr_warn("prog '%s': error relocating .BTF.ext function info: %d\n",
6220 prog->name, err);
6221 return err;
6222 }
6223 if (main_prog->func_info) {
6224 /*
6225 * Some info has already been found but has problem
6226 * in the last btf_ext reloc. Must have to error out.
6227 */
6228 pr_warn("prog '%s': missing .BTF.ext function info.\n", prog->name);
6229 return err;
6230 }
6231 /* Have problem loading the very first info. Ignore the rest. */
6232 pr_warn("prog '%s': missing .BTF.ext function info for the main program, skipping all of .BTF.ext func info.\n",
6233 prog->name);
6234 }
6235
6236 line_info:
6237 /* don't relocate line info if main program's relocation failed */
6238 if (main_prog != prog && !main_prog->line_info)
6239 return 0;
6240
6241 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->line_info,
6242 &main_prog->line_info,
6243 &main_prog->line_info_cnt,
6244 &main_prog->line_info_rec_size);
6245 if (err) {
6246 if (err != -ENOENT) {
6247 pr_warn("prog '%s': error relocating .BTF.ext line info: %d\n",
6248 prog->name, err);
6249 return err;
6250 }
6251 if (main_prog->line_info) {
6252 /*
6253 * Some info has already been found but has problem
6254 * in the last btf_ext reloc. Must have to error out.
6255 */
6256 pr_warn("prog '%s': missing .BTF.ext line info.\n", prog->name);
6257 return err;
6258 }
6259 /* Have problem loading the very first info. Ignore the rest. */
6260 pr_warn("prog '%s': missing .BTF.ext line info for the main program, skipping all of .BTF.ext line info.\n",
6261 prog->name);
6262 }
6263 return 0;
6264 }
6265
6266 static int cmp_relo_by_insn_idx(const void *key, const void *elem)
6267 {
6268 size_t insn_idx = *(const size_t *)key;
6269 const struct reloc_desc *relo = elem;
6270
6271 if (insn_idx == relo->insn_idx)
6272 return 0;
6273 return insn_idx < relo->insn_idx ? -1 : 1;
6274 }
6275
6276 static struct reloc_desc *find_prog_insn_relo(const struct bpf_program *prog, size_t insn_idx)
6277 {
6278 if (!prog->nr_reloc)
6279 return NULL;
6280 return bsearch(&insn_idx, prog->reloc_desc, prog->nr_reloc,
6281 sizeof(*prog->reloc_desc), cmp_relo_by_insn_idx);
6282 }
6283
6284 static int append_subprog_relos(struct bpf_program *main_prog, struct bpf_program *subprog)
6285 {
6286 int new_cnt = main_prog->nr_reloc + subprog->nr_reloc;
6287 struct reloc_desc *relos;
6288 int i;
6289
6290 if (main_prog == subprog)
6291 return 0;
6292 relos = libbpf_reallocarray(main_prog->reloc_desc, new_cnt, sizeof(*relos));
6293 /* if new count is zero, reallocarray can return a valid NULL result;
6294 * in this case the previous pointer will be freed, so we *have to*
6295 * reassign old pointer to the new value (even if it's NULL)
6296 */
6297 if (!relos && new_cnt)
6298 return -ENOMEM;
6299 if (subprog->nr_reloc)
6300 memcpy(relos + main_prog->nr_reloc, subprog->reloc_desc,
6301 sizeof(*relos) * subprog->nr_reloc);
6302
6303 for (i = main_prog->nr_reloc; i < new_cnt; i++)
6304 relos[i].insn_idx += subprog->sub_insn_off;
6305 /* After insn_idx adjustment the 'relos' array is still sorted
6306 * by insn_idx and doesn't break bsearch.
6307 */
6308 main_prog->reloc_desc = relos;
6309 main_prog->nr_reloc = new_cnt;
6310 return 0;
6311 }
6312
6313 static int
6314 bpf_object__append_subprog_code(struct bpf_object *obj, struct bpf_program *main_prog,
6315 struct bpf_program *subprog)
6316 {
6317 struct bpf_insn *insns;
6318 size_t new_cnt;
6319 int err;
6320
6321 subprog->sub_insn_off = main_prog->insns_cnt;
6322
6323 new_cnt = main_prog->insns_cnt + subprog->insns_cnt;
6324 insns = libbpf_reallocarray(main_prog->insns, new_cnt, sizeof(*insns));
6325 if (!insns) {
6326 pr_warn("prog '%s': failed to realloc prog code\n", main_prog->name);
6327 return -ENOMEM;
6328 }
6329 main_prog->insns = insns;
6330 main_prog->insns_cnt = new_cnt;
6331
6332 memcpy(main_prog->insns + subprog->sub_insn_off, subprog->insns,
6333 subprog->insns_cnt * sizeof(*insns));
6334
6335 pr_debug("prog '%s': added %zu insns from sub-prog '%s'\n",
6336 main_prog->name, subprog->insns_cnt, subprog->name);
6337
6338 /* The subprog insns are now appended. Append its relos too. */
6339 err = append_subprog_relos(main_prog, subprog);
6340 if (err)
6341 return err;
6342 return 0;
6343 }
6344
6345 static int
6346 bpf_object__reloc_code(struct bpf_object *obj, struct bpf_program *main_prog,
6347 struct bpf_program *prog)
6348 {
6349 size_t sub_insn_idx, insn_idx;
6350 struct bpf_program *subprog;
6351 struct reloc_desc *relo;
6352 struct bpf_insn *insn;
6353 int err;
6354
6355 err = reloc_prog_func_and_line_info(obj, main_prog, prog);
6356 if (err)
6357 return err;
6358
6359 for (insn_idx = 0; insn_idx < prog->sec_insn_cnt; insn_idx++) {
6360 insn = &main_prog->insns[prog->sub_insn_off + insn_idx];
6361 if (!insn_is_subprog_call(insn) && !insn_is_pseudo_func(insn))
6362 continue;
6363
6364 relo = find_prog_insn_relo(prog, insn_idx);
6365 if (relo && relo->type == RELO_EXTERN_CALL)
6366 /* kfunc relocations will be handled later
6367 * in bpf_object__relocate_data()
6368 */
6369 continue;
6370 if (relo && relo->type != RELO_CALL && relo->type != RELO_SUBPROG_ADDR) {
6371 pr_warn("prog '%s': unexpected relo for insn #%zu, type %d\n",
6372 prog->name, insn_idx, relo->type);
6373 return -LIBBPF_ERRNO__RELOC;
6374 }
6375 if (relo) {
6376 /* sub-program instruction index is a combination of
6377 * an offset of a symbol pointed to by relocation and
6378 * call instruction's imm field; for global functions,
6379 * call always has imm = -1, but for static functions
6380 * relocation is against STT_SECTION and insn->imm
6381 * points to a start of a static function
6382 *
6383 * for subprog addr relocation, the relo->sym_off + insn->imm is
6384 * the byte offset in the corresponding section.
6385 */
6386 if (relo->type == RELO_CALL)
6387 sub_insn_idx = relo->sym_off / BPF_INSN_SZ + insn->imm + 1;
6388 else
6389 sub_insn_idx = (relo->sym_off + insn->imm) / BPF_INSN_SZ;
6390 } else if (insn_is_pseudo_func(insn)) {
6391 /*
6392 * RELO_SUBPROG_ADDR relo is always emitted even if both
6393 * functions are in the same section, so it shouldn't reach here.
6394 */
6395 pr_warn("prog '%s': missing subprog addr relo for insn #%zu\n",
6396 prog->name, insn_idx);
6397 return -LIBBPF_ERRNO__RELOC;
6398 } else {
6399 /* if subprogram call is to a static function within
6400 * the same ELF section, there won't be any relocation
6401 * emitted, but it also means there is no additional
6402 * offset necessary, insns->imm is relative to
6403 * instruction's original position within the section
6404 */
6405 sub_insn_idx = prog->sec_insn_off + insn_idx + insn->imm + 1;
6406 }
6407
6408 /* we enforce that sub-programs should be in .text section */
6409 subprog = find_prog_by_sec_insn(obj, obj->efile.text_shndx, sub_insn_idx);
6410 if (!subprog) {
6411 pr_warn("prog '%s': no .text section found yet sub-program call exists\n",
6412 prog->name);
6413 return -LIBBPF_ERRNO__RELOC;
6414 }
6415
6416 /* if it's the first call instruction calling into this
6417 * subprogram (meaning this subprog hasn't been processed
6418 * yet) within the context of current main program:
6419 * - append it at the end of main program's instructions blog;
6420 * - process is recursively, while current program is put on hold;
6421 * - if that subprogram calls some other not yet processes
6422 * subprogram, same thing will happen recursively until
6423 * there are no more unprocesses subprograms left to append
6424 * and relocate.
6425 */
6426 if (subprog->sub_insn_off == 0) {
6427 err = bpf_object__append_subprog_code(obj, main_prog, subprog);
6428 if (err)
6429 return err;
6430 err = bpf_object__reloc_code(obj, main_prog, subprog);
6431 if (err)
6432 return err;
6433 }
6434
6435 /* main_prog->insns memory could have been re-allocated, so
6436 * calculate pointer again
6437 */
6438 insn = &main_prog->insns[prog->sub_insn_off + insn_idx];
6439 /* calculate correct instruction position within current main
6440 * prog; each main prog can have a different set of
6441 * subprograms appended (potentially in different order as
6442 * well), so position of any subprog can be different for
6443 * different main programs
6444 */
6445 insn->imm = subprog->sub_insn_off - (prog->sub_insn_off + insn_idx) - 1;
6446
6447 pr_debug("prog '%s': insn #%zu relocated, imm %d points to subprog '%s' (now at %zu offset)\n",
6448 prog->name, insn_idx, insn->imm, subprog->name, subprog->sub_insn_off);
6449 }
6450
6451 return 0;
6452 }
6453
6454 /*
6455 * Relocate sub-program calls.
6456 *
6457 * Algorithm operates as follows. Each entry-point BPF program (referred to as
6458 * main prog) is processed separately. For each subprog (non-entry functions,
6459 * that can be called from either entry progs or other subprogs) gets their
6460 * sub_insn_off reset to zero. This serves as indicator that this subprogram
6461 * hasn't been yet appended and relocated within current main prog. Once its
6462 * relocated, sub_insn_off will point at the position within current main prog
6463 * where given subprog was appended. This will further be used to relocate all
6464 * the call instructions jumping into this subprog.
6465 *
6466 * We start with main program and process all call instructions. If the call
6467 * is into a subprog that hasn't been processed (i.e., subprog->sub_insn_off
6468 * is zero), subprog instructions are appended at the end of main program's
6469 * instruction array. Then main program is "put on hold" while we recursively
6470 * process newly appended subprogram. If that subprogram calls into another
6471 * subprogram that hasn't been appended, new subprogram is appended again to
6472 * the *main* prog's instructions (subprog's instructions are always left
6473 * untouched, as they need to be in unmodified state for subsequent main progs
6474 * and subprog instructions are always sent only as part of a main prog) and
6475 * the process continues recursively. Once all the subprogs called from a main
6476 * prog or any of its subprogs are appended (and relocated), all their
6477 * positions within finalized instructions array are known, so it's easy to
6478 * rewrite call instructions with correct relative offsets, corresponding to
6479 * desired target subprog.
6480 *
6481 * Its important to realize that some subprogs might not be called from some
6482 * main prog and any of its called/used subprogs. Those will keep their
6483 * subprog->sub_insn_off as zero at all times and won't be appended to current
6484 * main prog and won't be relocated within the context of current main prog.
6485 * They might still be used from other main progs later.
6486 *
6487 * Visually this process can be shown as below. Suppose we have two main
6488 * programs mainA and mainB and BPF object contains three subprogs: subA,
6489 * subB, and subC. mainA calls only subA, mainB calls only subC, but subA and
6490 * subC both call subB:
6491 *
6492 * +--------+ +-------+
6493 * | v v |
6494 * +--+---+ +--+-+-+ +---+--+
6495 * | subA | | subB | | subC |
6496 * +--+---+ +------+ +---+--+
6497 * ^ ^
6498 * | |
6499 * +---+-------+ +------+----+
6500 * | mainA | | mainB |
6501 * +-----------+ +-----------+
6502 *
6503 * We'll start relocating mainA, will find subA, append it and start
6504 * processing sub A recursively:
6505 *
6506 * +-----------+------+
6507 * | mainA | subA |
6508 * +-----------+------+
6509 *
6510 * At this point we notice that subB is used from subA, so we append it and
6511 * relocate (there are no further subcalls from subB):
6512 *
6513 * +-----------+------+------+
6514 * | mainA | subA | subB |
6515 * +-----------+------+------+
6516 *
6517 * At this point, we relocate subA calls, then go one level up and finish with
6518 * relocatin mainA calls. mainA is done.
6519 *
6520 * For mainB process is similar but results in different order. We start with
6521 * mainB and skip subA and subB, as mainB never calls them (at least
6522 * directly), but we see subC is needed, so we append and start processing it:
6523 *
6524 * +-----------+------+
6525 * | mainB | subC |
6526 * +-----------+------+
6527 * Now we see subC needs subB, so we go back to it, append and relocate it:
6528 *
6529 * +-----------+------+------+
6530 * | mainB | subC | subB |
6531 * +-----------+------+------+
6532 *
6533 * At this point we unwind recursion, relocate calls in subC, then in mainB.
6534 */
6535 static int
6536 bpf_object__relocate_calls(struct bpf_object *obj, struct bpf_program *prog)
6537 {
6538 struct bpf_program *subprog;
6539 int i, err;
6540
6541 /* mark all subprogs as not relocated (yet) within the context of
6542 * current main program
6543 */
6544 for (i = 0; i < obj->nr_programs; i++) {
6545 subprog = &obj->programs[i];
6546 if (!prog_is_subprog(obj, subprog))
6547 continue;
6548
6549 subprog->sub_insn_off = 0;
6550 }
6551
6552 err = bpf_object__reloc_code(obj, prog, prog);
6553 if (err)
6554 return err;
6555
6556 return 0;
6557 }
6558
6559 static void
6560 bpf_object__free_relocs(struct bpf_object *obj)
6561 {
6562 struct bpf_program *prog;
6563 int i;
6564
6565 /* free up relocation descriptors */
6566 for (i = 0; i < obj->nr_programs; i++) {
6567 prog = &obj->programs[i];
6568 zfree(&prog->reloc_desc);
6569 prog->nr_reloc = 0;
6570 }
6571 }
6572
6573 static int cmp_relocs(const void *_a, const void *_b)
6574 {
6575 const struct reloc_desc *a = _a;
6576 const struct reloc_desc *b = _b;
6577
6578 if (a->insn_idx != b->insn_idx)
6579 return a->insn_idx < b->insn_idx ? -1 : 1;
6580
6581 /* no two relocations should have the same insn_idx, but ... */
6582 if (a->type != b->type)
6583 return a->type < b->type ? -1 : 1;
6584
6585 return 0;
6586 }
6587
6588 static void bpf_object__sort_relos(struct bpf_object *obj)
6589 {
6590 int i;
6591
6592 for (i = 0; i < obj->nr_programs; i++) {
6593 struct bpf_program *p = &obj->programs[i];
6594
6595 if (!p->nr_reloc)
6596 continue;
6597
6598 qsort(p->reloc_desc, p->nr_reloc, sizeof(*p->reloc_desc), cmp_relocs);
6599 }
6600 }
6601
6602 static int bpf_prog_assign_exc_cb(struct bpf_object *obj, struct bpf_program *prog)
6603 {
6604 const char *str = "exception_callback:";
6605 size_t pfx_len = strlen(str);
6606 int i, j, n;
6607
6608 if (!obj->btf || !kernel_supports(obj, FEAT_BTF_DECL_TAG))
6609 return 0;
6610
6611 n = btf__type_cnt(obj->btf);
6612 for (i = 1; i < n; i++) {
6613 const char *name;
6614 struct btf_type *t;
6615
6616 t = btf_type_by_id(obj->btf, i);
6617 if (!btf_is_decl_tag(t) || btf_decl_tag(t)->component_idx != -1)
6618 continue;
6619
6620 name = btf__str_by_offset(obj->btf, t->name_off);
6621 if (strncmp(name, str, pfx_len) != 0)
6622 continue;
6623
6624 t = btf_type_by_id(obj->btf, t->type);
6625 if (!btf_is_func(t) || btf_func_linkage(t) != BTF_FUNC_GLOBAL) {
6626 pr_warn("prog '%s': exception_callback:<value> decl tag not applied to the main program\n",
6627 prog->name);
6628 return -EINVAL;
6629 }
6630 if (strcmp(prog->name, btf__str_by_offset(obj->btf, t->name_off)) != 0)
6631 continue;
6632 /* Multiple callbacks are specified for the same prog,
6633 * the verifier will eventually return an error for this
6634 * case, hence simply skip appending a subprog.
6635 */
6636 if (prog->exception_cb_idx >= 0) {
6637 prog->exception_cb_idx = -1;
6638 break;
6639 }
6640
6641 name += pfx_len;
6642 if (str_is_empty(name)) {
6643 pr_warn("prog '%s': exception_callback:<value> decl tag contains empty value\n",
6644 prog->name);
6645 return -EINVAL;
6646 }
6647
6648 for (j = 0; j < obj->nr_programs; j++) {
6649 struct bpf_program *subprog = &obj->programs[j];
6650
6651 if (!prog_is_subprog(obj, subprog))
6652 continue;
6653 if (strcmp(name, subprog->name) != 0)
6654 continue;
6655 /* Enforce non-hidden, as from verifier point of
6656 * view it expects global functions, whereas the
6657 * mark_btf_static fixes up linkage as static.
6658 */
6659 if (!subprog->sym_global || subprog->mark_btf_static) {
6660 pr_warn("prog '%s': exception callback %s must be a global non-hidden function\n",
6661 prog->name, subprog->name);
6662 return -EINVAL;
6663 }
6664 /* Let's see if we already saw a static exception callback with the same name */
6665 if (prog->exception_cb_idx >= 0) {
6666 pr_warn("prog '%s': multiple subprogs with same name as exception callback '%s'\n",
6667 prog->name, subprog->name);
6668 return -EINVAL;
6669 }
6670 prog->exception_cb_idx = j;
6671 break;
6672 }
6673
6674 if (prog->exception_cb_idx >= 0)
6675 continue;
6676
6677 pr_warn("prog '%s': cannot find exception callback '%s'\n", prog->name, name);
6678 return -ENOENT;
6679 }
6680
6681 return 0;
6682 }
6683
6684 static struct {
6685 enum bpf_prog_type prog_type;
6686 const char *ctx_name;
6687 } global_ctx_map[] = {
6688 { BPF_PROG_TYPE_CGROUP_DEVICE, "bpf_cgroup_dev_ctx" },
6689 { BPF_PROG_TYPE_CGROUP_SKB, "__sk_buff" },
6690 { BPF_PROG_TYPE_CGROUP_SOCK, "bpf_sock" },
6691 { BPF_PROG_TYPE_CGROUP_SOCK_ADDR, "bpf_sock_addr" },
6692 { BPF_PROG_TYPE_CGROUP_SOCKOPT, "bpf_sockopt" },
6693 { BPF_PROG_TYPE_CGROUP_SYSCTL, "bpf_sysctl" },
6694 { BPF_PROG_TYPE_FLOW_DISSECTOR, "__sk_buff" },
6695 { BPF_PROG_TYPE_KPROBE, "bpf_user_pt_regs_t" },
6696 { BPF_PROG_TYPE_LWT_IN, "__sk_buff" },
6697 { BPF_PROG_TYPE_LWT_OUT, "__sk_buff" },
6698 { BPF_PROG_TYPE_LWT_SEG6LOCAL, "__sk_buff" },
6699 { BPF_PROG_TYPE_LWT_XMIT, "__sk_buff" },
6700 { BPF_PROG_TYPE_NETFILTER, "bpf_nf_ctx" },
6701 { BPF_PROG_TYPE_PERF_EVENT, "bpf_perf_event_data" },
6702 { BPF_PROG_TYPE_RAW_TRACEPOINT, "bpf_raw_tracepoint_args" },
6703 { BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE, "bpf_raw_tracepoint_args" },
6704 { BPF_PROG_TYPE_SCHED_ACT, "__sk_buff" },
6705 { BPF_PROG_TYPE_SCHED_CLS, "__sk_buff" },
6706 { BPF_PROG_TYPE_SK_LOOKUP, "bpf_sk_lookup" },
6707 { BPF_PROG_TYPE_SK_MSG, "sk_msg_md" },
6708 { BPF_PROG_TYPE_SK_REUSEPORT, "sk_reuseport_md" },
6709 { BPF_PROG_TYPE_SK_SKB, "__sk_buff" },
6710 { BPF_PROG_TYPE_SOCK_OPS, "bpf_sock_ops" },
6711 { BPF_PROG_TYPE_SOCKET_FILTER, "__sk_buff" },
6712 { BPF_PROG_TYPE_XDP, "xdp_md" },
6713 /* all other program types don't have "named" context structs */
6714 };
6715
6716 /* forward declarations for arch-specific underlying types of bpf_user_pt_regs_t typedef,
6717 * for below __builtin_types_compatible_p() checks;
6718 * with this approach we don't need any extra arch-specific #ifdef guards
6719 */
6720 struct pt_regs;
6721 struct user_pt_regs;
6722 struct user_regs_struct;
6723
6724 static bool need_func_arg_type_fixup(const struct btf *btf, const struct bpf_program *prog,
6725 const char *subprog_name, int arg_idx,
6726 int arg_type_id, const char *ctx_name)
6727 {
6728 const struct btf_type *t;
6729 const char *tname;
6730
6731 /* check if existing parameter already matches verifier expectations */
6732 t = skip_mods_and_typedefs(btf, arg_type_id, NULL);
6733 if (!btf_is_ptr(t))
6734 goto out_warn;
6735
6736 /* typedef bpf_user_pt_regs_t is a special PITA case, valid for kprobe
6737 * and perf_event programs, so check this case early on and forget
6738 * about it for subsequent checks
6739 */
6740 while (btf_is_mod(t))
6741 t = btf__type_by_id(btf, t->type);
6742 if (btf_is_typedef(t) &&
6743 (prog->type == BPF_PROG_TYPE_KPROBE || prog->type == BPF_PROG_TYPE_PERF_EVENT)) {
6744 tname = btf__str_by_offset(btf, t->name_off) ?: "<anon>";
6745 if (strcmp(tname, "bpf_user_pt_regs_t") == 0)
6746 return false; /* canonical type for kprobe/perf_event */
6747 }
6748
6749 /* now we can ignore typedefs moving forward */
6750 t = skip_mods_and_typedefs(btf, t->type, NULL);
6751
6752 /* if it's `void *`, definitely fix up BTF info */
6753 if (btf_is_void(t))
6754 return true;
6755
6756 /* if it's already proper canonical type, no need to fix up */
6757 tname = btf__str_by_offset(btf, t->name_off) ?: "<anon>";
6758 if (btf_is_struct(t) && strcmp(tname, ctx_name) == 0)
6759 return false;
6760
6761 /* special cases */
6762 switch (prog->type) {
6763 case BPF_PROG_TYPE_KPROBE:
6764 /* `struct pt_regs *` is expected, but we need to fix up */
6765 if (btf_is_struct(t) && strcmp(tname, "pt_regs") == 0)
6766 return true;
6767 break;
6768 case BPF_PROG_TYPE_PERF_EVENT:
6769 if (__builtin_types_compatible_p(bpf_user_pt_regs_t, struct pt_regs) &&
6770 btf_is_struct(t) && strcmp(tname, "pt_regs") == 0)
6771 return true;
6772 if (__builtin_types_compatible_p(bpf_user_pt_regs_t, struct user_pt_regs) &&
6773 btf_is_struct(t) && strcmp(tname, "user_pt_regs") == 0)
6774 return true;
6775 if (__builtin_types_compatible_p(bpf_user_pt_regs_t, struct user_regs_struct) &&
6776 btf_is_struct(t) && strcmp(tname, "user_regs_struct") == 0)
6777 return true;
6778 break;
6779 case BPF_PROG_TYPE_RAW_TRACEPOINT:
6780 case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE:
6781 /* allow u64* as ctx */
6782 if (btf_is_int(t) && t->size == 8)
6783 return true;
6784 break;
6785 default:
6786 break;
6787 }
6788
6789 out_warn:
6790 pr_warn("prog '%s': subprog '%s' arg#%d is expected to be of `struct %s *` type\n",
6791 prog->name, subprog_name, arg_idx, ctx_name);
6792 return false;
6793 }
6794
6795 static int clone_func_btf_info(struct btf *btf, int orig_fn_id, struct bpf_program *prog)
6796 {
6797 int fn_id, fn_proto_id, ret_type_id, orig_proto_id;
6798 int i, err, arg_cnt, fn_name_off, linkage;
6799 struct btf_type *fn_t, *fn_proto_t, *t;
6800 struct btf_param *p;
6801
6802 /* caller already validated FUNC -> FUNC_PROTO validity */
6803 fn_t = btf_type_by_id(btf, orig_fn_id);
6804 fn_proto_t = btf_type_by_id(btf, fn_t->type);
6805
6806 /* Note that each btf__add_xxx() operation invalidates
6807 * all btf_type and string pointers, so we need to be
6808 * very careful when cloning BTF types. BTF type
6809 * pointers have to be always refetched. And to avoid
6810 * problems with invalidated string pointers, we
6811 * add empty strings initially, then just fix up
6812 * name_off offsets in place. Offsets are stable for
6813 * existing strings, so that works out.
6814 */
6815 fn_name_off = fn_t->name_off; /* we are about to invalidate fn_t */
6816 linkage = btf_func_linkage(fn_t);
6817 orig_proto_id = fn_t->type; /* original FUNC_PROTO ID */
6818 ret_type_id = fn_proto_t->type; /* fn_proto_t will be invalidated */
6819 arg_cnt = btf_vlen(fn_proto_t);
6820
6821 /* clone FUNC_PROTO and its params */
6822 fn_proto_id = btf__add_func_proto(btf, ret_type_id);
6823 if (fn_proto_id < 0)
6824 return -EINVAL;
6825
6826 for (i = 0; i < arg_cnt; i++) {
6827 int name_off;
6828
6829 /* copy original parameter data */
6830 t = btf_type_by_id(btf, orig_proto_id);
6831 p = &btf_params(t)[i];
6832 name_off = p->name_off;
6833
6834 err = btf__add_func_param(btf, "", p->type);
6835 if (err)
6836 return err;
6837
6838 fn_proto_t = btf_type_by_id(btf, fn_proto_id);
6839 p = &btf_params(fn_proto_t)[i];
6840 p->name_off = name_off; /* use remembered str offset */
6841 }
6842
6843 /* clone FUNC now, btf__add_func() enforces non-empty name, so use
6844 * entry program's name as a placeholder, which we replace immediately
6845 * with original name_off
6846 */
6847 fn_id = btf__add_func(btf, prog->name, linkage, fn_proto_id);
6848 if (fn_id < 0)
6849 return -EINVAL;
6850
6851 fn_t = btf_type_by_id(btf, fn_id);
6852 fn_t->name_off = fn_name_off; /* reuse original string */
6853
6854 return fn_id;
6855 }
6856
6857 /* Check if main program or global subprog's function prototype has `arg:ctx`
6858 * argument tags, and, if necessary, substitute correct type to match what BPF
6859 * verifier would expect, taking into account specific program type. This
6860 * allows to support __arg_ctx tag transparently on old kernels that don't yet
6861 * have a native support for it in the verifier, making user's life much
6862 * easier.
6863 */
6864 static int bpf_program_fixup_func_info(struct bpf_object *obj, struct bpf_program *prog)
6865 {
6866 const char *ctx_name = NULL, *ctx_tag = "arg:ctx", *fn_name;
6867 struct bpf_func_info_min *func_rec;
6868 struct btf_type *fn_t, *fn_proto_t;
6869 struct btf *btf = obj->btf;
6870 const struct btf_type *t;
6871 struct btf_param *p;
6872 int ptr_id = 0, struct_id, tag_id, orig_fn_id;
6873 int i, n, arg_idx, arg_cnt, err, rec_idx;
6874 int *orig_ids;
6875
6876 /* no .BTF.ext, no problem */
6877 if (!obj->btf_ext || !prog->func_info)
6878 return 0;
6879
6880 /* don't do any fix ups if kernel natively supports __arg_ctx */
6881 if (kernel_supports(obj, FEAT_ARG_CTX_TAG))
6882 return 0;
6883
6884 /* some BPF program types just don't have named context structs, so
6885 * this fallback mechanism doesn't work for them
6886 */
6887 for (i = 0; i < ARRAY_SIZE(global_ctx_map); i++) {
6888 if (global_ctx_map[i].prog_type != prog->type)
6889 continue;
6890 ctx_name = global_ctx_map[i].ctx_name;
6891 break;
6892 }
6893 if (!ctx_name)
6894 return 0;
6895
6896 /* remember original func BTF IDs to detect if we already cloned them */
6897 orig_ids = calloc(prog->func_info_cnt, sizeof(*orig_ids));
6898 if (!orig_ids)
6899 return -ENOMEM;
6900 for (i = 0; i < prog->func_info_cnt; i++) {
6901 func_rec = prog->func_info + prog->func_info_rec_size * i;
6902 orig_ids[i] = func_rec->type_id;
6903 }
6904
6905 /* go through each DECL_TAG with "arg:ctx" and see if it points to one
6906 * of our subprogs; if yes and subprog is global and needs adjustment,
6907 * clone and adjust FUNC -> FUNC_PROTO combo
6908 */
6909 for (i = 1, n = btf__type_cnt(btf); i < n; i++) {
6910 /* only DECL_TAG with "arg:ctx" value are interesting */
6911 t = btf__type_by_id(btf, i);
6912 if (!btf_is_decl_tag(t))
6913 continue;
6914 if (strcmp(btf__str_by_offset(btf, t->name_off), ctx_tag) != 0)
6915 continue;
6916
6917 /* only global funcs need adjustment, if at all */
6918 orig_fn_id = t->type;
6919 fn_t = btf_type_by_id(btf, orig_fn_id);
6920 if (!btf_is_func(fn_t) || btf_func_linkage(fn_t) != BTF_FUNC_GLOBAL)
6921 continue;
6922
6923 /* sanity check FUNC -> FUNC_PROTO chain, just in case */
6924 fn_proto_t = btf_type_by_id(btf, fn_t->type);
6925 if (!fn_proto_t || !btf_is_func_proto(fn_proto_t))
6926 continue;
6927
6928 /* find corresponding func_info record */
6929 func_rec = NULL;
6930 for (rec_idx = 0; rec_idx < prog->func_info_cnt; rec_idx++) {
6931 if (orig_ids[rec_idx] == t->type) {
6932 func_rec = prog->func_info + prog->func_info_rec_size * rec_idx;
6933 break;
6934 }
6935 }
6936 /* current main program doesn't call into this subprog */
6937 if (!func_rec)
6938 continue;
6939
6940 /* some more sanity checking of DECL_TAG */
6941 arg_cnt = btf_vlen(fn_proto_t);
6942 arg_idx = btf_decl_tag(t)->component_idx;
6943 if (arg_idx < 0 || arg_idx >= arg_cnt)
6944 continue;
6945
6946 /* check if we should fix up argument type */
6947 p = &btf_params(fn_proto_t)[arg_idx];
6948 fn_name = btf__str_by_offset(btf, fn_t->name_off) ?: "<anon>";
6949 if (!need_func_arg_type_fixup(btf, prog, fn_name, arg_idx, p->type, ctx_name))
6950 continue;
6951
6952 /* clone fn/fn_proto, unless we already did it for another arg */
6953 if (func_rec->type_id == orig_fn_id) {
6954 int fn_id;
6955
6956 fn_id = clone_func_btf_info(btf, orig_fn_id, prog);
6957 if (fn_id < 0) {
6958 err = fn_id;
6959 goto err_out;
6960 }
6961
6962 /* point func_info record to a cloned FUNC type */
6963 func_rec->type_id = fn_id;
6964 }
6965
6966 /* create PTR -> STRUCT type chain to mark PTR_TO_CTX argument;
6967 * we do it just once per main BPF program, as all global
6968 * funcs share the same program type, so need only PTR ->
6969 * STRUCT type chain
6970 */
6971 if (ptr_id == 0) {
6972 struct_id = btf__add_struct(btf, ctx_name, 0);
6973 ptr_id = btf__add_ptr(btf, struct_id);
6974 if (ptr_id < 0 || struct_id < 0) {
6975 err = -EINVAL;
6976 goto err_out;
6977 }
6978 }
6979
6980 /* for completeness, clone DECL_TAG and point it to cloned param */
6981 tag_id = btf__add_decl_tag(btf, ctx_tag, func_rec->type_id, arg_idx);
6982 if (tag_id < 0) {
6983 err = -EINVAL;
6984 goto err_out;
6985 }
6986
6987 /* all the BTF manipulations invalidated pointers, refetch them */
6988 fn_t = btf_type_by_id(btf, func_rec->type_id);
6989 fn_proto_t = btf_type_by_id(btf, fn_t->type);
6990
6991 /* fix up type ID pointed to by param */
6992 p = &btf_params(fn_proto_t)[arg_idx];
6993 p->type = ptr_id;
6994 }
6995
6996 free(orig_ids);
6997 return 0;
6998 err_out:
6999 free(orig_ids);
7000 return err;
7001 }
7002
7003 static int bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_path)
7004 {
7005 struct bpf_program *prog;
7006 size_t i, j;
7007 int err;
7008
7009 if (obj->btf_ext) {
7010 err = bpf_object__relocate_core(obj, targ_btf_path);
7011 if (err) {
7012 pr_warn("failed to perform CO-RE relocations: %d\n",
7013 err);
7014 return err;
7015 }
7016 bpf_object__sort_relos(obj);
7017 }
7018
7019 /* Before relocating calls pre-process relocations and mark
7020 * few ld_imm64 instructions that points to subprogs.
7021 * Otherwise bpf_object__reloc_code() later would have to consider
7022 * all ld_imm64 insns as relocation candidates. That would
7023 * reduce relocation speed, since amount of find_prog_insn_relo()
7024 * would increase and most of them will fail to find a relo.
7025 */
7026 for (i = 0; i < obj->nr_programs; i++) {
7027 prog = &obj->programs[i];
7028 for (j = 0; j < prog->nr_reloc; j++) {
7029 struct reloc_desc *relo = &prog->reloc_desc[j];
7030 struct bpf_insn *insn = &prog->insns[relo->insn_idx];
7031
7032 /* mark the insn, so it's recognized by insn_is_pseudo_func() */
7033 if (relo->type == RELO_SUBPROG_ADDR)
7034 insn[0].src_reg = BPF_PSEUDO_FUNC;
7035 }
7036 }
7037
7038 /* relocate subprogram calls and append used subprograms to main
7039 * programs; each copy of subprogram code needs to be relocated
7040 * differently for each main program, because its code location might
7041 * have changed.
7042 * Append subprog relos to main programs to allow data relos to be
7043 * processed after text is completely relocated.
7044 */
7045 for (i = 0; i < obj->nr_programs; i++) {
7046 prog = &obj->programs[i];
7047 /* sub-program's sub-calls are relocated within the context of
7048 * its main program only
7049 */
7050 if (prog_is_subprog(obj, prog))
7051 continue;
7052 if (!prog->autoload)
7053 continue;
7054
7055 err = bpf_object__relocate_calls(obj, prog);
7056 if (err) {
7057 pr_warn("prog '%s': failed to relocate calls: %d\n",
7058 prog->name, err);
7059 return err;
7060 }
7061
7062 err = bpf_prog_assign_exc_cb(obj, prog);
7063 if (err)
7064 return err;
7065 /* Now, also append exception callback if it has not been done already. */
7066 if (prog->exception_cb_idx >= 0) {
7067 struct bpf_program *subprog = &obj->programs[prog->exception_cb_idx];
7068
7069 /* Calling exception callback directly is disallowed, which the
7070 * verifier will reject later. In case it was processed already,
7071 * we can skip this step, otherwise for all other valid cases we
7072 * have to append exception callback now.
7073 */
7074 if (subprog->sub_insn_off == 0) {
7075 err = bpf_object__append_subprog_code(obj, prog, subprog);
7076 if (err)
7077 return err;
7078 err = bpf_object__reloc_code(obj, prog, subprog);
7079 if (err)
7080 return err;
7081 }
7082 }
7083 }
7084 for (i = 0; i < obj->nr_programs; i++) {
7085 prog = &obj->programs[i];
7086 if (prog_is_subprog(obj, prog))
7087 continue;
7088 if (!prog->autoload)
7089 continue;
7090
7091 /* Process data relos for main programs */
7092 err = bpf_object__relocate_data(obj, prog);
7093 if (err) {
7094 pr_warn("prog '%s': failed to relocate data references: %d\n",
7095 prog->name, err);
7096 return err;
7097 }
7098
7099 /* Fix up .BTF.ext information, if necessary */
7100 err = bpf_program_fixup_func_info(obj, prog);
7101 if (err) {
7102 pr_warn("prog '%s': failed to perform .BTF.ext fix ups: %d\n",
7103 prog->name, err);
7104 return err;
7105 }
7106 }
7107
7108 return 0;
7109 }
7110
7111 static int bpf_object__collect_st_ops_relos(struct bpf_object *obj,
7112 Elf64_Shdr *shdr, Elf_Data *data);
7113
7114 static int bpf_object__collect_map_relos(struct bpf_object *obj,
7115 Elf64_Shdr *shdr, Elf_Data *data)
7116 {
7117 const int bpf_ptr_sz = 8, host_ptr_sz = sizeof(void *);
7118 int i, j, nrels, new_sz;
7119 const struct btf_var_secinfo *vi = NULL;
7120 const struct btf_type *sec, *var, *def;
7121 struct bpf_map *map = NULL, *targ_map = NULL;
7122 struct bpf_program *targ_prog = NULL;
7123 bool is_prog_array, is_map_in_map;
7124 const struct btf_member *member;
7125 const char *name, *mname, *type;
7126 unsigned int moff;
7127 Elf64_Sym *sym;
7128 Elf64_Rel *rel;
7129 void *tmp;
7130
7131 if (!obj->efile.btf_maps_sec_btf_id || !obj->btf)
7132 return -EINVAL;
7133 sec = btf__type_by_id(obj->btf, obj->efile.btf_maps_sec_btf_id);
7134 if (!sec)
7135 return -EINVAL;
7136
7137 nrels = shdr->sh_size / shdr->sh_entsize;
7138 for (i = 0; i < nrels; i++) {
7139 rel = elf_rel_by_idx(data, i);
7140 if (!rel) {
7141 pr_warn(".maps relo #%d: failed to get ELF relo\n", i);
7142 return -LIBBPF_ERRNO__FORMAT;
7143 }
7144
7145 sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info));
7146 if (!sym) {
7147 pr_warn(".maps relo #%d: symbol %zx not found\n",
7148 i, (size_t)ELF64_R_SYM(rel->r_info));
7149 return -LIBBPF_ERRNO__FORMAT;
7150 }
7151 name = elf_sym_str(obj, sym->st_name) ?: "<?>";
7152
7153 pr_debug(".maps relo #%d: for %zd value %zd rel->r_offset %zu name %d ('%s')\n",
7154 i, (ssize_t)(rel->r_info >> 32), (size_t)sym->st_value,
7155 (size_t)rel->r_offset, sym->st_name, name);
7156
7157 for (j = 0; j < obj->nr_maps; j++) {
7158 map = &obj->maps[j];
7159 if (map->sec_idx != obj->efile.btf_maps_shndx)
7160 continue;
7161
7162 vi = btf_var_secinfos(sec) + map->btf_var_idx;
7163 if (vi->offset <= rel->r_offset &&
7164 rel->r_offset + bpf_ptr_sz <= vi->offset + vi->size)
7165 break;
7166 }
7167 if (j == obj->nr_maps) {
7168 pr_warn(".maps relo #%d: cannot find map '%s' at rel->r_offset %zu\n",
7169 i, name, (size_t)rel->r_offset);
7170 return -EINVAL;
7171 }
7172
7173 is_map_in_map = bpf_map_type__is_map_in_map(map->def.type);
7174 is_prog_array = map->def.type == BPF_MAP_TYPE_PROG_ARRAY;
7175 type = is_map_in_map ? "map" : "prog";
7176 if (is_map_in_map) {
7177 if (sym->st_shndx != obj->efile.btf_maps_shndx) {
7178 pr_warn(".maps relo #%d: '%s' isn't a BTF-defined map\n",
7179 i, name);
7180 return -LIBBPF_ERRNO__RELOC;
7181 }
7182 if (map->def.type == BPF_MAP_TYPE_HASH_OF_MAPS &&
7183 map->def.key_size != sizeof(int)) {
7184 pr_warn(".maps relo #%d: hash-of-maps '%s' should have key size %zu.\n",
7185 i, map->name, sizeof(int));
7186 return -EINVAL;
7187 }
7188 targ_map = bpf_object__find_map_by_name(obj, name);
7189 if (!targ_map) {
7190 pr_warn(".maps relo #%d: '%s' isn't a valid map reference\n",
7191 i, name);
7192 return -ESRCH;
7193 }
7194 } else if (is_prog_array) {
7195 targ_prog = bpf_object__find_program_by_name(obj, name);
7196 if (!targ_prog) {
7197 pr_warn(".maps relo #%d: '%s' isn't a valid program reference\n",
7198 i, name);
7199 return -ESRCH;
7200 }
7201 if (targ_prog->sec_idx != sym->st_shndx ||
7202 targ_prog->sec_insn_off * 8 != sym->st_value ||
7203 prog_is_subprog(obj, targ_prog)) {
7204 pr_warn(".maps relo #%d: '%s' isn't an entry-point program\n",
7205 i, name);
7206 return -LIBBPF_ERRNO__RELOC;
7207 }
7208 } else {
7209 return -EINVAL;
7210 }
7211
7212 var = btf__type_by_id(obj->btf, vi->type);
7213 def = skip_mods_and_typedefs(obj->btf, var->type, NULL);
7214 if (btf_vlen(def) == 0)
7215 return -EINVAL;
7216 member = btf_members(def) + btf_vlen(def) - 1;
7217 mname = btf__name_by_offset(obj->btf, member->name_off);
7218 if (strcmp(mname, "values"))
7219 return -EINVAL;
7220
7221 moff = btf_member_bit_offset(def, btf_vlen(def) - 1) / 8;
7222 if (rel->r_offset - vi->offset < moff)
7223 return -EINVAL;
7224
7225 moff = rel->r_offset - vi->offset - moff;
7226 /* here we use BPF pointer size, which is always 64 bit, as we
7227 * are parsing ELF that was built for BPF target
7228 */
7229 if (moff % bpf_ptr_sz)
7230 return -EINVAL;
7231 moff /= bpf_ptr_sz;
7232 if (moff >= map->init_slots_sz) {
7233 new_sz = moff + 1;
7234 tmp = libbpf_reallocarray(map->init_slots, new_sz, host_ptr_sz);
7235 if (!tmp)
7236 return -ENOMEM;
7237 map->init_slots = tmp;
7238 memset(map->init_slots + map->init_slots_sz, 0,
7239 (new_sz - map->init_slots_sz) * host_ptr_sz);
7240 map->init_slots_sz = new_sz;
7241 }
7242 map->init_slots[moff] = is_map_in_map ? (void *)targ_map : (void *)targ_prog;
7243
7244 pr_debug(".maps relo #%d: map '%s' slot [%d] points to %s '%s'\n",
7245 i, map->name, moff, type, name);
7246 }
7247
7248 return 0;
7249 }
7250
7251 static int bpf_object__collect_relos(struct bpf_object *obj)
7252 {
7253 int i, err;
7254
7255 for (i = 0; i < obj->efile.sec_cnt; i++) {
7256 struct elf_sec_desc *sec_desc = &obj->efile.secs[i];
7257 Elf64_Shdr *shdr;
7258 Elf_Data *data;
7259 int idx;
7260
7261 if (sec_desc->sec_type != SEC_RELO)
7262 continue;
7263
7264 shdr = sec_desc->shdr;
7265 data = sec_desc->data;
7266 idx = shdr->sh_info;
7267
7268 if (shdr->sh_type != SHT_REL || idx < 0 || idx >= obj->efile.sec_cnt) {
7269 pr_warn("internal error at %d\n", __LINE__);
7270 return -LIBBPF_ERRNO__INTERNAL;
7271 }
7272
7273 if (obj->efile.secs[idx].sec_type == SEC_ST_OPS)
7274 err = bpf_object__collect_st_ops_relos(obj, shdr, data);
7275 else if (idx == obj->efile.btf_maps_shndx)
7276 err = bpf_object__collect_map_relos(obj, shdr, data);
7277 else
7278 err = bpf_object__collect_prog_relos(obj, shdr, data);
7279 if (err)
7280 return err;
7281 }
7282
7283 bpf_object__sort_relos(obj);
7284 return 0;
7285 }
7286
7287 static bool insn_is_helper_call(struct bpf_insn *insn, enum bpf_func_id *func_id)
7288 {
7289 if (BPF_CLASS(insn->code) == BPF_JMP &&
7290 BPF_OP(insn->code) == BPF_CALL &&
7291 BPF_SRC(insn->code) == BPF_K &&
7292 insn->src_reg == 0 &&
7293 insn->dst_reg == 0) {
7294 *func_id = insn->imm;
7295 return true;
7296 }
7297 return false;
7298 }
7299
7300 static int bpf_object__sanitize_prog(struct bpf_object *obj, struct bpf_program *prog)
7301 {
7302 struct bpf_insn *insn = prog->insns;
7303 enum bpf_func_id func_id;
7304 int i;
7305
7306 if (obj->gen_loader)
7307 return 0;
7308
7309 for (i = 0; i < prog->insns_cnt; i++, insn++) {
7310 if (!insn_is_helper_call(insn, &func_id))
7311 continue;
7312
7313 /* on kernels that don't yet support
7314 * bpf_probe_read_{kernel,user}[_str] helpers, fall back
7315 * to bpf_probe_read() which works well for old kernels
7316 */
7317 switch (func_id) {
7318 case BPF_FUNC_probe_read_kernel:
7319 case BPF_FUNC_probe_read_user:
7320 if (!kernel_supports(obj, FEAT_PROBE_READ_KERN))
7321 insn->imm = BPF_FUNC_probe_read;
7322 break;
7323 case BPF_FUNC_probe_read_kernel_str:
7324 case BPF_FUNC_probe_read_user_str:
7325 if (!kernel_supports(obj, FEAT_PROBE_READ_KERN))
7326 insn->imm = BPF_FUNC_probe_read_str;
7327 break;
7328 default:
7329 break;
7330 }
7331 }
7332 return 0;
7333 }
7334
7335 static int libbpf_find_attach_btf_id(struct bpf_program *prog, const char *attach_name,
7336 int *btf_obj_fd, int *btf_type_id);
7337
7338 /* this is called as prog->sec_def->prog_prepare_load_fn for libbpf-supported sec_defs */
7339 static int libbpf_prepare_prog_load(struct bpf_program *prog,
7340 struct bpf_prog_load_opts *opts, long cookie)
7341 {
7342 enum sec_def_flags def = cookie;
7343
7344 /* old kernels might not support specifying expected_attach_type */
7345 if ((def & SEC_EXP_ATTACH_OPT) && !kernel_supports(prog->obj, FEAT_EXP_ATTACH_TYPE))
7346 opts->expected_attach_type = 0;
7347
7348 if (def & SEC_SLEEPABLE)
7349 opts->prog_flags |= BPF_F_SLEEPABLE;
7350
7351 if (prog->type == BPF_PROG_TYPE_XDP && (def & SEC_XDP_FRAGS))
7352 opts->prog_flags |= BPF_F_XDP_HAS_FRAGS;
7353
7354 /* special check for usdt to use uprobe_multi link */
7355 if ((def & SEC_USDT) && kernel_supports(prog->obj, FEAT_UPROBE_MULTI_LINK))
7356 prog->expected_attach_type = BPF_TRACE_UPROBE_MULTI;
7357
7358 if ((def & SEC_ATTACH_BTF) && !prog->attach_btf_id) {
7359 int btf_obj_fd = 0, btf_type_id = 0, err;
7360 const char *attach_name;
7361
7362 attach_name = strchr(prog->sec_name, '/');
7363 if (!attach_name) {
7364 /* if BPF program is annotated with just SEC("fentry")
7365 * (or similar) without declaratively specifying
7366 * target, then it is expected that target will be
7367 * specified with bpf_program__set_attach_target() at
7368 * runtime before BPF object load step. If not, then
7369 * there is nothing to load into the kernel as BPF
7370 * verifier won't be able to validate BPF program
7371 * correctness anyways.
7372 */
7373 pr_warn("prog '%s': no BTF-based attach target is specified, use bpf_program__set_attach_target()\n",
7374 prog->name);
7375 return -EINVAL;
7376 }
7377 attach_name++; /* skip over / */
7378
7379 err = libbpf_find_attach_btf_id(prog, attach_name, &btf_obj_fd, &btf_type_id);
7380 if (err)
7381 return err;
7382
7383 /* cache resolved BTF FD and BTF type ID in the prog */
7384 prog->attach_btf_obj_fd = btf_obj_fd;
7385 prog->attach_btf_id = btf_type_id;
7386
7387 /* but by now libbpf common logic is not utilizing
7388 * prog->atach_btf_obj_fd/prog->attach_btf_id anymore because
7389 * this callback is called after opts were populated by
7390 * libbpf, so this callback has to update opts explicitly here
7391 */
7392 opts->attach_btf_obj_fd = btf_obj_fd;
7393 opts->attach_btf_id = btf_type_id;
7394 }
7395 return 0;
7396 }
7397
7398 static void fixup_verifier_log(struct bpf_program *prog, char *buf, size_t buf_sz);
7399
7400 static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog,
7401 struct bpf_insn *insns, int insns_cnt,
7402 const char *license, __u32 kern_version, int *prog_fd)
7403 {
7404 LIBBPF_OPTS(bpf_prog_load_opts, load_attr);
7405 const char *prog_name = NULL;
7406 char *cp, errmsg[STRERR_BUFSIZE];
7407 size_t log_buf_size = 0;
7408 char *log_buf = NULL, *tmp;
7409 bool own_log_buf = true;
7410 __u32 log_level = prog->log_level;
7411 int ret, err;
7412
7413 /* Be more helpful by rejecting programs that can't be validated early
7414 * with more meaningful and actionable error message.
7415 */
7416 switch (prog->type) {
7417 case BPF_PROG_TYPE_UNSPEC:
7418 /*
7419 * The program type must be set. Most likely we couldn't find a proper
7420 * section definition at load time, and thus we didn't infer the type.
7421 */
7422 pr_warn("prog '%s': missing BPF prog type, check ELF section name '%s'\n",
7423 prog->name, prog->sec_name);
7424 return -EINVAL;
7425 case BPF_PROG_TYPE_STRUCT_OPS:
7426 if (prog->attach_btf_id == 0) {
7427 pr_warn("prog '%s': SEC(\"struct_ops\") program isn't referenced anywhere, did you forget to use it?\n",
7428 prog->name);
7429 return -EINVAL;
7430 }
7431 break;
7432 default:
7433 break;
7434 }
7435
7436 if (!insns || !insns_cnt)
7437 return -EINVAL;
7438
7439 if (kernel_supports(obj, FEAT_PROG_NAME))
7440 prog_name = prog->name;
7441 load_attr.attach_prog_fd = prog->attach_prog_fd;
7442 load_attr.attach_btf_obj_fd = prog->attach_btf_obj_fd;
7443 load_attr.attach_btf_id = prog->attach_btf_id;
7444 load_attr.kern_version = kern_version;
7445 load_attr.prog_ifindex = prog->prog_ifindex;
7446
7447 /* specify func_info/line_info only if kernel supports them */
7448 if (obj->btf && btf__fd(obj->btf) >= 0 && kernel_supports(obj, FEAT_BTF_FUNC)) {
7449 load_attr.prog_btf_fd = btf__fd(obj->btf);
7450 load_attr.func_info = prog->func_info;
7451 load_attr.func_info_rec_size = prog->func_info_rec_size;
7452 load_attr.func_info_cnt = prog->func_info_cnt;
7453 load_attr.line_info = prog->line_info;
7454 load_attr.line_info_rec_size = prog->line_info_rec_size;
7455 load_attr.line_info_cnt = prog->line_info_cnt;
7456 }
7457 load_attr.log_level = log_level;
7458 load_attr.prog_flags = prog->prog_flags;
7459 load_attr.fd_array = obj->fd_array;
7460
7461 load_attr.token_fd = obj->token_fd;
7462 if (obj->token_fd)
7463 load_attr.prog_flags |= BPF_F_TOKEN_FD;
7464
7465 /* adjust load_attr if sec_def provides custom preload callback */
7466 if (prog->sec_def && prog->sec_def->prog_prepare_load_fn) {
7467 err = prog->sec_def->prog_prepare_load_fn(prog, &load_attr, prog->sec_def->cookie);
7468 if (err < 0) {
7469 pr_warn("prog '%s': failed to prepare load attributes: %d\n",
7470 prog->name, err);
7471 return err;
7472 }
7473 insns = prog->insns;
7474 insns_cnt = prog->insns_cnt;
7475 }
7476
7477 /* allow prog_prepare_load_fn to change expected_attach_type */
7478 load_attr.expected_attach_type = prog->expected_attach_type;
7479
7480 if (obj->gen_loader) {
7481 bpf_gen__prog_load(obj->gen_loader, prog->type, prog->name,
7482 license, insns, insns_cnt, &load_attr,
7483 prog - obj->programs);
7484 *prog_fd = -1;
7485 return 0;
7486 }
7487
7488 retry_load:
7489 /* if log_level is zero, we don't request logs initially even if
7490 * custom log_buf is specified; if the program load fails, then we'll
7491 * bump log_level to 1 and use either custom log_buf or we'll allocate
7492 * our own and retry the load to get details on what failed
7493 */
7494 if (log_level) {
7495 if (prog->log_buf) {
7496 log_buf = prog->log_buf;
7497 log_buf_size = prog->log_size;
7498 own_log_buf = false;
7499 } else if (obj->log_buf) {
7500 log_buf = obj->log_buf;
7501 log_buf_size = obj->log_size;
7502 own_log_buf = false;
7503 } else {
7504 log_buf_size = max((size_t)BPF_LOG_BUF_SIZE, log_buf_size * 2);
7505 tmp = realloc(log_buf, log_buf_size);
7506 if (!tmp) {
7507 ret = -ENOMEM;
7508 goto out;
7509 }
7510 log_buf = tmp;
7511 log_buf[0] = '\0';
7512 own_log_buf = true;
7513 }
7514 }
7515
7516 load_attr.log_buf = log_buf;
7517 load_attr.log_size = log_buf_size;
7518 load_attr.log_level = log_level;
7519
7520 ret = bpf_prog_load(prog->type, prog_name, license, insns, insns_cnt, &load_attr);
7521 if (ret >= 0) {
7522 if (log_level && own_log_buf) {
7523 pr_debug("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n",
7524 prog->name, log_buf);
7525 }
7526
7527 if (obj->has_rodata && kernel_supports(obj, FEAT_PROG_BIND_MAP)) {
7528 struct bpf_map *map;
7529 int i;
7530
7531 for (i = 0; i < obj->nr_maps; i++) {
7532 map = &prog->obj->maps[i];
7533 if (map->libbpf_type != LIBBPF_MAP_RODATA)
7534 continue;
7535
7536 if (bpf_prog_bind_map(ret, map->fd, NULL)) {
7537 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
7538 pr_warn("prog '%s': failed to bind map '%s': %s\n",
7539 prog->name, map->real_name, cp);
7540 /* Don't fail hard if can't bind rodata. */
7541 }
7542 }
7543 }
7544
7545 *prog_fd = ret;
7546 ret = 0;
7547 goto out;
7548 }
7549
7550 if (log_level == 0) {
7551 log_level = 1;
7552 goto retry_load;
7553 }
7554 /* On ENOSPC, increase log buffer size and retry, unless custom
7555 * log_buf is specified.
7556 * Be careful to not overflow u32, though. Kernel's log buf size limit
7557 * isn't part of UAPI so it can always be bumped to full 4GB. So don't
7558 * multiply by 2 unless we are sure we'll fit within 32 bits.
7559 * Currently, we'll get -EINVAL when we reach (UINT_MAX >> 2).
7560 */
7561 if (own_log_buf && errno == ENOSPC && log_buf_size <= UINT_MAX / 2)
7562 goto retry_load;
7563
7564 ret = -errno;
7565
7566 /* post-process verifier log to improve error descriptions */
7567 fixup_verifier_log(prog, log_buf, log_buf_size);
7568
7569 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
7570 pr_warn("prog '%s': BPF program load failed: %s\n", prog->name, cp);
7571 pr_perm_msg(ret);
7572
7573 if (own_log_buf && log_buf && log_buf[0] != '\0') {
7574 pr_warn("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n",
7575 prog->name, log_buf);
7576 }
7577
7578 out:
7579 if (own_log_buf)
7580 free(log_buf);
7581 return ret;
7582 }
7583
7584 static char *find_prev_line(char *buf, char *cur)
7585 {
7586 char *p;
7587
7588 if (cur == buf) /* end of a log buf */
7589 return NULL;
7590
7591 p = cur - 1;
7592 while (p - 1 >= buf && *(p - 1) != '\n')
7593 p--;
7594
7595 return p;
7596 }
7597
7598 static void patch_log(char *buf, size_t buf_sz, size_t log_sz,
7599 char *orig, size_t orig_sz, const char *patch)
7600 {
7601 /* size of the remaining log content to the right from the to-be-replaced part */
7602 size_t rem_sz = (buf + log_sz) - (orig + orig_sz);
7603 size_t patch_sz = strlen(patch);
7604
7605 if (patch_sz != orig_sz) {
7606 /* If patch line(s) are longer than original piece of verifier log,
7607 * shift log contents by (patch_sz - orig_sz) bytes to the right
7608 * starting from after to-be-replaced part of the log.
7609 *
7610 * If patch line(s) are shorter than original piece of verifier log,
7611 * shift log contents by (orig_sz - patch_sz) bytes to the left
7612 * starting from after to-be-replaced part of the log
7613 *
7614 * We need to be careful about not overflowing available
7615 * buf_sz capacity. If that's the case, we'll truncate the end
7616 * of the original log, as necessary.
7617 */
7618 if (patch_sz > orig_sz) {
7619 if (orig + patch_sz >= buf + buf_sz) {
7620 /* patch is big enough to cover remaining space completely */
7621 patch_sz -= (orig + patch_sz) - (buf + buf_sz) + 1;
7622 rem_sz = 0;
7623 } else if (patch_sz - orig_sz > buf_sz - log_sz) {
7624 /* patch causes part of remaining log to be truncated */
7625 rem_sz -= (patch_sz - orig_sz) - (buf_sz - log_sz);
7626 }
7627 }
7628 /* shift remaining log to the right by calculated amount */
7629 memmove(orig + patch_sz, orig + orig_sz, rem_sz);
7630 }
7631
7632 memcpy(orig, patch, patch_sz);
7633 }
7634
7635 static void fixup_log_failed_core_relo(struct bpf_program *prog,
7636 char *buf, size_t buf_sz, size_t log_sz,
7637 char *line1, char *line2, char *line3)
7638 {
7639 /* Expected log for failed and not properly guarded CO-RE relocation:
7640 * line1 -> 123: (85) call unknown#195896080
7641 * line2 -> invalid func unknown#195896080
7642 * line3 -> <anything else or end of buffer>
7643 *
7644 * "123" is the index of the instruction that was poisoned. We extract
7645 * instruction index to find corresponding CO-RE relocation and
7646 * replace this part of the log with more relevant information about
7647 * failed CO-RE relocation.
7648 */
7649 const struct bpf_core_relo *relo;
7650 struct bpf_core_spec spec;
7651 char patch[512], spec_buf[256];
7652 int insn_idx, err, spec_len;
7653
7654 if (sscanf(line1, "%d: (%*d) call unknown#195896080\n", &insn_idx) != 1)
7655 return;
7656
7657 relo = find_relo_core(prog, insn_idx);
7658 if (!relo)
7659 return;
7660
7661 err = bpf_core_parse_spec(prog->name, prog->obj->btf, relo, &spec);
7662 if (err)
7663 return;
7664
7665 spec_len = bpf_core_format_spec(spec_buf, sizeof(spec_buf), &spec);
7666 snprintf(patch, sizeof(patch),
7667 "%d: <invalid CO-RE relocation>\n"
7668 "failed to resolve CO-RE relocation %s%s\n",
7669 insn_idx, spec_buf, spec_len >= sizeof(spec_buf) ? "..." : "");
7670
7671 patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch);
7672 }
7673
7674 static void fixup_log_missing_map_load(struct bpf_program *prog,
7675 char *buf, size_t buf_sz, size_t log_sz,
7676 char *line1, char *line2, char *line3)
7677 {
7678 /* Expected log for failed and not properly guarded map reference:
7679 * line1 -> 123: (85) call unknown#2001000345
7680 * line2 -> invalid func unknown#2001000345
7681 * line3 -> <anything else or end of buffer>
7682 *
7683 * "123" is the index of the instruction that was poisoned.
7684 * "345" in "2001000345" is a map index in obj->maps to fetch map name.
7685 */
7686 struct bpf_object *obj = prog->obj;
7687 const struct bpf_map *map;
7688 int insn_idx, map_idx;
7689 char patch[128];
7690
7691 if (sscanf(line1, "%d: (%*d) call unknown#%d\n", &insn_idx, &map_idx) != 2)
7692 return;
7693
7694 map_idx -= POISON_LDIMM64_MAP_BASE;
7695 if (map_idx < 0 || map_idx >= obj->nr_maps)
7696 return;
7697 map = &obj->maps[map_idx];
7698
7699 snprintf(patch, sizeof(patch),
7700 "%d: <invalid BPF map reference>\n"
7701 "BPF map '%s' is referenced but wasn't created\n",
7702 insn_idx, map->name);
7703
7704 patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch);
7705 }
7706
7707 static void fixup_log_missing_kfunc_call(struct bpf_program *prog,
7708 char *buf, size_t buf_sz, size_t log_sz,
7709 char *line1, char *line2, char *line3)
7710 {
7711 /* Expected log for failed and not properly guarded kfunc call:
7712 * line1 -> 123: (85) call unknown#2002000345
7713 * line2 -> invalid func unknown#2002000345
7714 * line3 -> <anything else or end of buffer>
7715 *
7716 * "123" is the index of the instruction that was poisoned.
7717 * "345" in "2002000345" is an extern index in obj->externs to fetch kfunc name.
7718 */
7719 struct bpf_object *obj = prog->obj;
7720 const struct extern_desc *ext;
7721 int insn_idx, ext_idx;
7722 char patch[128];
7723
7724 if (sscanf(line1, "%d: (%*d) call unknown#%d\n", &insn_idx, &ext_idx) != 2)
7725 return;
7726
7727 ext_idx -= POISON_CALL_KFUNC_BASE;
7728 if (ext_idx < 0 || ext_idx >= obj->nr_extern)
7729 return;
7730 ext = &obj->externs[ext_idx];
7731
7732 snprintf(patch, sizeof(patch),
7733 "%d: <invalid kfunc call>\n"
7734 "kfunc '%s' is referenced but wasn't resolved\n",
7735 insn_idx, ext->name);
7736
7737 patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch);
7738 }
7739
7740 static void fixup_verifier_log(struct bpf_program *prog, char *buf, size_t buf_sz)
7741 {
7742 /* look for familiar error patterns in last N lines of the log */
7743 const size_t max_last_line_cnt = 10;
7744 char *prev_line, *cur_line, *next_line;
7745 size_t log_sz;
7746 int i;
7747
7748 if (!buf)
7749 return;
7750
7751 log_sz = strlen(buf) + 1;
7752 next_line = buf + log_sz - 1;
7753
7754 for (i = 0; i < max_last_line_cnt; i++, next_line = cur_line) {
7755 cur_line = find_prev_line(buf, next_line);
7756 if (!cur_line)
7757 return;
7758
7759 if (str_has_pfx(cur_line, "invalid func unknown#195896080\n")) {
7760 prev_line = find_prev_line(buf, cur_line);
7761 if (!prev_line)
7762 continue;
7763
7764 /* failed CO-RE relocation case */
7765 fixup_log_failed_core_relo(prog, buf, buf_sz, log_sz,
7766 prev_line, cur_line, next_line);
7767 return;
7768 } else if (str_has_pfx(cur_line, "invalid func unknown#"POISON_LDIMM64_MAP_PFX)) {
7769 prev_line = find_prev_line(buf, cur_line);
7770 if (!prev_line)
7771 continue;
7772
7773 /* reference to uncreated BPF map */
7774 fixup_log_missing_map_load(prog, buf, buf_sz, log_sz,
7775 prev_line, cur_line, next_line);
7776 return;
7777 } else if (str_has_pfx(cur_line, "invalid func unknown#"POISON_CALL_KFUNC_PFX)) {
7778 prev_line = find_prev_line(buf, cur_line);
7779 if (!prev_line)
7780 continue;
7781
7782 /* reference to unresolved kfunc */
7783 fixup_log_missing_kfunc_call(prog, buf, buf_sz, log_sz,
7784 prev_line, cur_line, next_line);
7785 return;
7786 }
7787 }
7788 }
7789
7790 static int bpf_program_record_relos(struct bpf_program *prog)
7791 {
7792 struct bpf_object *obj = prog->obj;
7793 int i;
7794
7795 for (i = 0; i < prog->nr_reloc; i++) {
7796 struct reloc_desc *relo = &prog->reloc_desc[i];
7797 struct extern_desc *ext = &obj->externs[relo->ext_idx];
7798 int kind;
7799
7800 switch (relo->type) {
7801 case RELO_EXTERN_LD64:
7802 if (ext->type != EXT_KSYM)
7803 continue;
7804 kind = btf_is_var(btf__type_by_id(obj->btf, ext->btf_id)) ?
7805 BTF_KIND_VAR : BTF_KIND_FUNC;
7806 bpf_gen__record_extern(obj->gen_loader, ext->name,
7807 ext->is_weak, !ext->ksym.type_id,
7808 true, kind, relo->insn_idx);
7809 break;
7810 case RELO_EXTERN_CALL:
7811 bpf_gen__record_extern(obj->gen_loader, ext->name,
7812 ext->is_weak, false, false, BTF_KIND_FUNC,
7813 relo->insn_idx);
7814 break;
7815 case RELO_CORE: {
7816 struct bpf_core_relo cr = {
7817 .insn_off = relo->insn_idx * 8,
7818 .type_id = relo->core_relo->type_id,
7819 .access_str_off = relo->core_relo->access_str_off,
7820 .kind = relo->core_relo->kind,
7821 };
7822
7823 bpf_gen__record_relo_core(obj->gen_loader, &cr);
7824 break;
7825 }
7826 default:
7827 continue;
7828 }
7829 }
7830 return 0;
7831 }
7832
7833 static int
7834 bpf_object__load_progs(struct bpf_object *obj, int log_level)
7835 {
7836 struct bpf_program *prog;
7837 size_t i;
7838 int err;
7839
7840 for (i = 0; i < obj->nr_programs; i++) {
7841 prog = &obj->programs[i];
7842 err = bpf_object__sanitize_prog(obj, prog);
7843 if (err)
7844 return err;
7845 }
7846
7847 for (i = 0; i < obj->nr_programs; i++) {
7848 prog = &obj->programs[i];
7849 if (prog_is_subprog(obj, prog))
7850 continue;
7851 if (!prog->autoload) {
7852 pr_debug("prog '%s': skipped loading\n", prog->name);
7853 continue;
7854 }
7855 prog->log_level |= log_level;
7856
7857 if (obj->gen_loader)
7858 bpf_program_record_relos(prog);
7859
7860 err = bpf_object_load_prog(obj, prog, prog->insns, prog->insns_cnt,
7861 obj->license, obj->kern_version, &prog->fd);
7862 if (err) {
7863 pr_warn("prog '%s': failed to load: %d\n", prog->name, err);
7864 return err;
7865 }
7866 }
7867
7868 bpf_object__free_relocs(obj);
7869 return 0;
7870 }
7871
7872 static const struct bpf_sec_def *find_sec_def(const char *sec_name);
7873
7874 static int bpf_object_init_progs(struct bpf_object *obj, const struct bpf_object_open_opts *opts)
7875 {
7876 struct bpf_program *prog;
7877 int err;
7878
7879 bpf_object__for_each_program(prog, obj) {
7880 prog->sec_def = find_sec_def(prog->sec_name);
7881 if (!prog->sec_def) {
7882 /* couldn't guess, but user might manually specify */
7883 pr_debug("prog '%s': unrecognized ELF section name '%s'\n",
7884 prog->name, prog->sec_name);
7885 continue;
7886 }
7887
7888 prog->type = prog->sec_def->prog_type;
7889 prog->expected_attach_type = prog->sec_def->expected_attach_type;
7890
7891 /* sec_def can have custom callback which should be called
7892 * after bpf_program is initialized to adjust its properties
7893 */
7894 if (prog->sec_def->prog_setup_fn) {
7895 err = prog->sec_def->prog_setup_fn(prog, prog->sec_def->cookie);
7896 if (err < 0) {
7897 pr_warn("prog '%s': failed to initialize: %d\n",
7898 prog->name, err);
7899 return err;
7900 }
7901 }
7902 }
7903
7904 return 0;
7905 }
7906
7907 static struct bpf_object *bpf_object_open(const char *path, const void *obj_buf, size_t obj_buf_sz,
7908 const char *obj_name,
7909 const struct bpf_object_open_opts *opts)
7910 {
7911 const char *kconfig, *btf_tmp_path, *token_path;
7912 struct bpf_object *obj;
7913 int err;
7914 char *log_buf;
7915 size_t log_size;
7916 __u32 log_level;
7917
7918 if (obj_buf && !obj_name)
7919 return ERR_PTR(-EINVAL);
7920
7921 if (elf_version(EV_CURRENT) == EV_NONE) {
7922 pr_warn("failed to init libelf for %s\n",
7923 path ? : "(mem buf)");
7924 return ERR_PTR(-LIBBPF_ERRNO__LIBELF);
7925 }
7926
7927 if (!OPTS_VALID(opts, bpf_object_open_opts))
7928 return ERR_PTR(-EINVAL);
7929
7930 obj_name = OPTS_GET(opts, object_name, NULL) ?: obj_name;
7931 if (obj_buf) {
7932 path = obj_name;
7933 pr_debug("loading object '%s' from buffer\n", obj_name);
7934 } else {
7935 pr_debug("loading object from %s\n", path);
7936 }
7937
7938 log_buf = OPTS_GET(opts, kernel_log_buf, NULL);
7939 log_size = OPTS_GET(opts, kernel_log_size, 0);
7940 log_level = OPTS_GET(opts, kernel_log_level, 0);
7941 if (log_size > UINT_MAX)
7942 return ERR_PTR(-EINVAL);
7943 if (log_size && !log_buf)
7944 return ERR_PTR(-EINVAL);
7945
7946 token_path = OPTS_GET(opts, bpf_token_path, NULL);
7947 /* if user didn't specify bpf_token_path explicitly, check if
7948 * LIBBPF_BPF_TOKEN_PATH envvar was set and treat it as bpf_token_path
7949 * option
7950 */
7951 if (!token_path)
7952 token_path = getenv("LIBBPF_BPF_TOKEN_PATH");
7953 if (token_path && strlen(token_path) >= PATH_MAX)
7954 return ERR_PTR(-ENAMETOOLONG);
7955
7956 obj = bpf_object__new(path, obj_buf, obj_buf_sz, obj_name);
7957 if (IS_ERR(obj))
7958 return obj;
7959
7960 obj->log_buf = log_buf;
7961 obj->log_size = log_size;
7962 obj->log_level = log_level;
7963
7964 if (token_path) {
7965 obj->token_path = strdup(token_path);
7966 if (!obj->token_path) {
7967 err = -ENOMEM;
7968 goto out;
7969 }
7970 }
7971
7972 btf_tmp_path = OPTS_GET(opts, btf_custom_path, NULL);
7973 if (btf_tmp_path) {
7974 if (strlen(btf_tmp_path) >= PATH_MAX) {
7975 err = -ENAMETOOLONG;
7976 goto out;
7977 }
7978 obj->btf_custom_path = strdup(btf_tmp_path);
7979 if (!obj->btf_custom_path) {
7980 err = -ENOMEM;
7981 goto out;
7982 }
7983 }
7984
7985 kconfig = OPTS_GET(opts, kconfig, NULL);
7986 if (kconfig) {
7987 obj->kconfig = strdup(kconfig);
7988 if (!obj->kconfig) {
7989 err = -ENOMEM;
7990 goto out;
7991 }
7992 }
7993
7994 err = bpf_object__elf_init(obj);
7995 err = err ? : bpf_object__check_endianness(obj);
7996 err = err ? : bpf_object__elf_collect(obj);
7997 err = err ? : bpf_object__collect_externs(obj);
7998 err = err ? : bpf_object_fixup_btf(obj);
7999 err = err ? : bpf_object__init_maps(obj, opts);
8000 err = err ? : bpf_object_init_progs(obj, opts);
8001 err = err ? : bpf_object__collect_relos(obj);
8002 if (err)
8003 goto out;
8004
8005 bpf_object__elf_finish(obj);
8006
8007 return obj;
8008 out:
8009 bpf_object__close(obj);
8010 return ERR_PTR(err);
8011 }
8012
8013 struct bpf_object *
8014 bpf_object__open_file(const char *path, const struct bpf_object_open_opts *opts)
8015 {
8016 if (!path)
8017 return libbpf_err_ptr(-EINVAL);
8018
8019 return libbpf_ptr(bpf_object_open(path, NULL, 0, NULL, opts));
8020 }
8021
8022 struct bpf_object *bpf_object__open(const char *path)
8023 {
8024 return bpf_object__open_file(path, NULL);
8025 }
8026
8027 struct bpf_object *
8028 bpf_object__open_mem(const void *obj_buf, size_t obj_buf_sz,
8029 const struct bpf_object_open_opts *opts)
8030 {
8031 char tmp_name[64];
8032
8033 if (!obj_buf || obj_buf_sz == 0)
8034 return libbpf_err_ptr(-EINVAL);
8035
8036 /* create a (quite useless) default "name" for this memory buffer object */
8037 snprintf(tmp_name, sizeof(tmp_name), "%lx-%zx", (unsigned long)obj_buf, obj_buf_sz);
8038
8039 return libbpf_ptr(bpf_object_open(NULL, obj_buf, obj_buf_sz, tmp_name, opts));
8040 }
8041
8042 static int bpf_object_unload(struct bpf_object *obj)
8043 {
8044 size_t i;
8045
8046 if (!obj)
8047 return libbpf_err(-EINVAL);
8048
8049 for (i = 0; i < obj->nr_maps; i++) {
8050 zclose(obj->maps[i].fd);
8051 if (obj->maps[i].st_ops)
8052 zfree(&obj->maps[i].st_ops->kern_vdata);
8053 }
8054
8055 for (i = 0; i < obj->nr_programs; i++)
8056 bpf_program__unload(&obj->programs[i]);
8057
8058 return 0;
8059 }
8060
8061 static int bpf_object__sanitize_maps(struct bpf_object *obj)
8062 {
8063 struct bpf_map *m;
8064
8065 bpf_object__for_each_map(m, obj) {
8066 if (!bpf_map__is_internal(m))
8067 continue;
8068 if (!kernel_supports(obj, FEAT_ARRAY_MMAP))
8069 m->def.map_flags &= ~BPF_F_MMAPABLE;
8070 }
8071
8072 return 0;
8073 }
8074
8075 typedef int (*kallsyms_cb_t)(unsigned long long sym_addr, char sym_type,
8076 const char *sym_name, void *ctx);
8077
8078 static int libbpf_kallsyms_parse(kallsyms_cb_t cb, void *ctx)
8079 {
8080 char sym_type, sym_name[500];
8081 unsigned long long sym_addr;
8082 int ret, err = 0;
8083 FILE *f;
8084
8085 f = fopen("/proc/kallsyms", "re");
8086 if (!f) {
8087 err = -errno;
8088 pr_warn("failed to open /proc/kallsyms: %d\n", err);
8089 return err;
8090 }
8091
8092 while (true) {
8093 ret = fscanf(f, "%llx %c %499s%*[^\n]\n",
8094 &sym_addr, &sym_type, sym_name);
8095 if (ret == EOF && feof(f))
8096 break;
8097 if (ret != 3) {
8098 pr_warn("failed to read kallsyms entry: %d\n", ret);
8099 err = -EINVAL;
8100 break;
8101 }
8102
8103 err = cb(sym_addr, sym_type, sym_name, ctx);
8104 if (err)
8105 break;
8106 }
8107
8108 fclose(f);
8109 return err;
8110 }
8111
8112 static int kallsyms_cb(unsigned long long sym_addr, char sym_type,
8113 const char *sym_name, void *ctx)
8114 {
8115 struct bpf_object *obj = ctx;
8116 const struct btf_type *t;
8117 struct extern_desc *ext;
8118 char *res;
8119
8120 res = strstr(sym_name, ".llvm.");
8121 if (sym_type == 'd' && res)
8122 ext = find_extern_by_name_with_len(obj, sym_name, res - sym_name);
8123 else
8124 ext = find_extern_by_name(obj, sym_name);
8125 if (!ext || ext->type != EXT_KSYM)
8126 return 0;
8127
8128 t = btf__type_by_id(obj->btf, ext->btf_id);
8129 if (!btf_is_var(t))
8130 return 0;
8131
8132 if (ext->is_set && ext->ksym.addr != sym_addr) {
8133 pr_warn("extern (ksym) '%s': resolution is ambiguous: 0x%llx or 0x%llx\n",
8134 sym_name, ext->ksym.addr, sym_addr);
8135 return -EINVAL;
8136 }
8137 if (!ext->is_set) {
8138 ext->is_set = true;
8139 ext->ksym.addr = sym_addr;
8140 pr_debug("extern (ksym) '%s': set to 0x%llx\n", sym_name, sym_addr);
8141 }
8142 return 0;
8143 }
8144
8145 static int bpf_object__read_kallsyms_file(struct bpf_object *obj)
8146 {
8147 return libbpf_kallsyms_parse(kallsyms_cb, obj);
8148 }
8149
8150 static int find_ksym_btf_id(struct bpf_object *obj, const char *ksym_name,
8151 __u16 kind, struct btf **res_btf,
8152 struct module_btf **res_mod_btf)
8153 {
8154 struct module_btf *mod_btf;
8155 struct btf *btf;
8156 int i, id, err;
8157
8158 btf = obj->btf_vmlinux;
8159 mod_btf = NULL;
8160 id = btf__find_by_name_kind(btf, ksym_name, kind);
8161
8162 if (id == -ENOENT) {
8163 err = load_module_btfs(obj);
8164 if (err)
8165 return err;
8166
8167 for (i = 0; i < obj->btf_module_cnt; i++) {
8168 /* we assume module_btf's BTF FD is always >0 */
8169 mod_btf = &obj->btf_modules[i];
8170 btf = mod_btf->btf;
8171 id = btf__find_by_name_kind_own(btf, ksym_name, kind);
8172 if (id != -ENOENT)
8173 break;
8174 }
8175 }
8176 if (id <= 0)
8177 return -ESRCH;
8178
8179 *res_btf = btf;
8180 *res_mod_btf = mod_btf;
8181 return id;
8182 }
8183
8184 static int bpf_object__resolve_ksym_var_btf_id(struct bpf_object *obj,
8185 struct extern_desc *ext)
8186 {
8187 const struct btf_type *targ_var, *targ_type;
8188 __u32 targ_type_id, local_type_id;
8189 struct module_btf *mod_btf = NULL;
8190 const char *targ_var_name;
8191 struct btf *btf = NULL;
8192 int id, err;
8193
8194 id = find_ksym_btf_id(obj, ext->name, BTF_KIND_VAR, &btf, &mod_btf);
8195 if (id < 0) {
8196 if (id == -ESRCH && ext->is_weak)
8197 return 0;
8198 pr_warn("extern (var ksym) '%s': not found in kernel BTF\n",
8199 ext->name);
8200 return id;
8201 }
8202
8203 /* find local type_id */
8204 local_type_id = ext->ksym.type_id;
8205
8206 /* find target type_id */
8207 targ_var = btf__type_by_id(btf, id);
8208 targ_var_name = btf__name_by_offset(btf, targ_var->name_off);
8209 targ_type = skip_mods_and_typedefs(btf, targ_var->type, &targ_type_id);
8210
8211 err = bpf_core_types_are_compat(obj->btf, local_type_id,
8212 btf, targ_type_id);
8213 if (err <= 0) {
8214 const struct btf_type *local_type;
8215 const char *targ_name, *local_name;
8216
8217 local_type = btf__type_by_id(obj->btf, local_type_id);
8218 local_name = btf__name_by_offset(obj->btf, local_type->name_off);
8219 targ_name = btf__name_by_offset(btf, targ_type->name_off);
8220
8221 pr_warn("extern (var ksym) '%s': incompatible types, expected [%d] %s %s, but kernel has [%d] %s %s\n",
8222 ext->name, local_type_id,
8223 btf_kind_str(local_type), local_name, targ_type_id,
8224 btf_kind_str(targ_type), targ_name);
8225 return -EINVAL;
8226 }
8227
8228 ext->is_set = true;
8229 ext->ksym.kernel_btf_obj_fd = mod_btf ? mod_btf->fd : 0;
8230 ext->ksym.kernel_btf_id = id;
8231 pr_debug("extern (var ksym) '%s': resolved to [%d] %s %s\n",
8232 ext->name, id, btf_kind_str(targ_var), targ_var_name);
8233
8234 return 0;
8235 }
8236
8237 static int bpf_object__resolve_ksym_func_btf_id(struct bpf_object *obj,
8238 struct extern_desc *ext)
8239 {
8240 int local_func_proto_id, kfunc_proto_id, kfunc_id;
8241 struct module_btf *mod_btf = NULL;
8242 const struct btf_type *kern_func;
8243 struct btf *kern_btf = NULL;
8244 int ret;
8245
8246 local_func_proto_id = ext->ksym.type_id;
8247
8248 kfunc_id = find_ksym_btf_id(obj, ext->essent_name ?: ext->name, BTF_KIND_FUNC, &kern_btf,
8249 &mod_btf);
8250 if (kfunc_id < 0) {
8251 if (kfunc_id == -ESRCH && ext->is_weak)
8252 return 0;
8253 pr_warn("extern (func ksym) '%s': not found in kernel or module BTFs\n",
8254 ext->name);
8255 return kfunc_id;
8256 }
8257
8258 kern_func = btf__type_by_id(kern_btf, kfunc_id);
8259 kfunc_proto_id = kern_func->type;
8260
8261 ret = bpf_core_types_are_compat(obj->btf, local_func_proto_id,
8262 kern_btf, kfunc_proto_id);
8263 if (ret <= 0) {
8264 if (ext->is_weak)
8265 return 0;
8266
8267 pr_warn("extern (func ksym) '%s': func_proto [%d] incompatible with %s [%d]\n",
8268 ext->name, local_func_proto_id,
8269 mod_btf ? mod_btf->name : "vmlinux", kfunc_proto_id);
8270 return -EINVAL;
8271 }
8272
8273 /* set index for module BTF fd in fd_array, if unset */
8274 if (mod_btf && !mod_btf->fd_array_idx) {
8275 /* insn->off is s16 */
8276 if (obj->fd_array_cnt == INT16_MAX) {
8277 pr_warn("extern (func ksym) '%s': module BTF fd index %d too big to fit in bpf_insn offset\n",
8278 ext->name, mod_btf->fd_array_idx);
8279 return -E2BIG;
8280 }
8281 /* Cannot use index 0 for module BTF fd */
8282 if (!obj->fd_array_cnt)
8283 obj->fd_array_cnt = 1;
8284
8285 ret = libbpf_ensure_mem((void **)&obj->fd_array, &obj->fd_array_cap, sizeof(int),
8286 obj->fd_array_cnt + 1);
8287 if (ret)
8288 return ret;
8289 mod_btf->fd_array_idx = obj->fd_array_cnt;
8290 /* we assume module BTF FD is always >0 */
8291 obj->fd_array[obj->fd_array_cnt++] = mod_btf->fd;
8292 }
8293
8294 ext->is_set = true;
8295 ext->ksym.kernel_btf_id = kfunc_id;
8296 ext->ksym.btf_fd_idx = mod_btf ? mod_btf->fd_array_idx : 0;
8297 /* Also set kernel_btf_obj_fd to make sure that bpf_object__relocate_data()
8298 * populates FD into ld_imm64 insn when it's used to point to kfunc.
8299 * {kernel_btf_id, btf_fd_idx} -> fixup bpf_call.
8300 * {kernel_btf_id, kernel_btf_obj_fd} -> fixup ld_imm64.
8301 */
8302 ext->ksym.kernel_btf_obj_fd = mod_btf ? mod_btf->fd : 0;
8303 pr_debug("extern (func ksym) '%s': resolved to %s [%d]\n",
8304 ext->name, mod_btf ? mod_btf->name : "vmlinux", kfunc_id);
8305
8306 return 0;
8307 }
8308
8309 static int bpf_object__resolve_ksyms_btf_id(struct bpf_object *obj)
8310 {
8311 const struct btf_type *t;
8312 struct extern_desc *ext;
8313 int i, err;
8314
8315 for (i = 0; i < obj->nr_extern; i++) {
8316 ext = &obj->externs[i];
8317 if (ext->type != EXT_KSYM || !ext->ksym.type_id)
8318 continue;
8319
8320 if (obj->gen_loader) {
8321 ext->is_set = true;
8322 ext->ksym.kernel_btf_obj_fd = 0;
8323 ext->ksym.kernel_btf_id = 0;
8324 continue;
8325 }
8326 t = btf__type_by_id(obj->btf, ext->btf_id);
8327 if (btf_is_var(t))
8328 err = bpf_object__resolve_ksym_var_btf_id(obj, ext);
8329 else
8330 err = bpf_object__resolve_ksym_func_btf_id(obj, ext);
8331 if (err)
8332 return err;
8333 }
8334 return 0;
8335 }
8336
8337 static int bpf_object__resolve_externs(struct bpf_object *obj,
8338 const char *extra_kconfig)
8339 {
8340 bool need_config = false, need_kallsyms = false;
8341 bool need_vmlinux_btf = false;
8342 struct extern_desc *ext;
8343 void *kcfg_data = NULL;
8344 int err, i;
8345
8346 if (obj->nr_extern == 0)
8347 return 0;
8348
8349 if (obj->kconfig_map_idx >= 0)
8350 kcfg_data = obj->maps[obj->kconfig_map_idx].mmaped;
8351
8352 for (i = 0; i < obj->nr_extern; i++) {
8353 ext = &obj->externs[i];
8354
8355 if (ext->type == EXT_KSYM) {
8356 if (ext->ksym.type_id)
8357 need_vmlinux_btf = true;
8358 else
8359 need_kallsyms = true;
8360 continue;
8361 } else if (ext->type == EXT_KCFG) {
8362 void *ext_ptr = kcfg_data + ext->kcfg.data_off;
8363 __u64 value = 0;
8364
8365 /* Kconfig externs need actual /proc/config.gz */
8366 if (str_has_pfx(ext->name, "CONFIG_")) {
8367 need_config = true;
8368 continue;
8369 }
8370
8371 /* Virtual kcfg externs are customly handled by libbpf */
8372 if (strcmp(ext->name, "LINUX_KERNEL_VERSION") == 0) {
8373 value = get_kernel_version();
8374 if (!value) {
8375 pr_warn("extern (kcfg) '%s': failed to get kernel version\n", ext->name);
8376 return -EINVAL;
8377 }
8378 } else if (strcmp(ext->name, "LINUX_HAS_BPF_COOKIE") == 0) {
8379 value = kernel_supports(obj, FEAT_BPF_COOKIE);
8380 } else if (strcmp(ext->name, "LINUX_HAS_SYSCALL_WRAPPER") == 0) {
8381 value = kernel_supports(obj, FEAT_SYSCALL_WRAPPER);
8382 } else if (!str_has_pfx(ext->name, "LINUX_") || !ext->is_weak) {
8383 /* Currently libbpf supports only CONFIG_ and LINUX_ prefixed
8384 * __kconfig externs, where LINUX_ ones are virtual and filled out
8385 * customly by libbpf (their values don't come from Kconfig).
8386 * If LINUX_xxx variable is not recognized by libbpf, but is marked
8387 * __weak, it defaults to zero value, just like for CONFIG_xxx
8388 * externs.
8389 */
8390 pr_warn("extern (kcfg) '%s': unrecognized virtual extern\n", ext->name);
8391 return -EINVAL;
8392 }
8393
8394 err = set_kcfg_value_num(ext, ext_ptr, value);
8395 if (err)
8396 return err;
8397 pr_debug("extern (kcfg) '%s': set to 0x%llx\n",
8398 ext->name, (long long)value);
8399 } else {
8400 pr_warn("extern '%s': unrecognized extern kind\n", ext->name);
8401 return -EINVAL;
8402 }
8403 }
8404 if (need_config && extra_kconfig) {
8405 err = bpf_object__read_kconfig_mem(obj, extra_kconfig, kcfg_data);
8406 if (err)
8407 return -EINVAL;
8408 need_config = false;
8409 for (i = 0; i < obj->nr_extern; i++) {
8410 ext = &obj->externs[i];
8411 if (ext->type == EXT_KCFG && !ext->is_set) {
8412 need_config = true;
8413 break;
8414 }
8415 }
8416 }
8417 if (need_config) {
8418 err = bpf_object__read_kconfig_file(obj, kcfg_data);
8419 if (err)
8420 return -EINVAL;
8421 }
8422 if (need_kallsyms) {
8423 err = bpf_object__read_kallsyms_file(obj);
8424 if (err)
8425 return -EINVAL;
8426 }
8427 if (need_vmlinux_btf) {
8428 err = bpf_object__resolve_ksyms_btf_id(obj);
8429 if (err)
8430 return -EINVAL;
8431 }
8432 for (i = 0; i < obj->nr_extern; i++) {
8433 ext = &obj->externs[i];
8434
8435 if (!ext->is_set && !ext->is_weak) {
8436 pr_warn("extern '%s' (strong): not resolved\n", ext->name);
8437 return -ESRCH;
8438 } else if (!ext->is_set) {
8439 pr_debug("extern '%s' (weak): not resolved, defaulting to zero\n",
8440 ext->name);
8441 }
8442 }
8443
8444 return 0;
8445 }
8446
8447 static void bpf_map_prepare_vdata(const struct bpf_map *map)
8448 {
8449 const struct btf_type *type;
8450 struct bpf_struct_ops *st_ops;
8451 __u32 i;
8452
8453 st_ops = map->st_ops;
8454 type = btf__type_by_id(map->obj->btf, st_ops->type_id);
8455 for (i = 0; i < btf_vlen(type); i++) {
8456 struct bpf_program *prog = st_ops->progs[i];
8457 void *kern_data;
8458 int prog_fd;
8459
8460 if (!prog)
8461 continue;
8462
8463 prog_fd = bpf_program__fd(prog);
8464 kern_data = st_ops->kern_vdata + st_ops->kern_func_off[i];
8465 *(unsigned long *)kern_data = prog_fd;
8466 }
8467 }
8468
8469 static int bpf_object_prepare_struct_ops(struct bpf_object *obj)
8470 {
8471 struct bpf_map *map;
8472 int i;
8473
8474 for (i = 0; i < obj->nr_maps; i++) {
8475 map = &obj->maps[i];
8476
8477 if (!bpf_map__is_struct_ops(map))
8478 continue;
8479
8480 if (!map->autocreate)
8481 continue;
8482
8483 bpf_map_prepare_vdata(map);
8484 }
8485
8486 return 0;
8487 }
8488
8489 static int bpf_object_load(struct bpf_object *obj, int extra_log_level, const char *target_btf_path)
8490 {
8491 int err, i;
8492
8493 if (!obj)
8494 return libbpf_err(-EINVAL);
8495
8496 if (obj->loaded) {
8497 pr_warn("object '%s': load can't be attempted twice\n", obj->name);
8498 return libbpf_err(-EINVAL);
8499 }
8500
8501 if (obj->gen_loader)
8502 bpf_gen__init(obj->gen_loader, extra_log_level, obj->nr_programs, obj->nr_maps);
8503
8504 err = bpf_object_prepare_token(obj);
8505 err = err ? : bpf_object__probe_loading(obj);
8506 err = err ? : bpf_object__load_vmlinux_btf(obj, false);
8507 err = err ? : bpf_object__resolve_externs(obj, obj->kconfig);
8508 err = err ? : bpf_object__sanitize_maps(obj);
8509 err = err ? : bpf_object__init_kern_struct_ops_maps(obj);
8510 err = err ? : bpf_object_adjust_struct_ops_autoload(obj);
8511 err = err ? : bpf_object__relocate(obj, obj->btf_custom_path ? : target_btf_path);
8512 err = err ? : bpf_object__sanitize_and_load_btf(obj);
8513 err = err ? : bpf_object__create_maps(obj);
8514 err = err ? : bpf_object__load_progs(obj, extra_log_level);
8515 err = err ? : bpf_object_init_prog_arrays(obj);
8516 err = err ? : bpf_object_prepare_struct_ops(obj);
8517
8518 if (obj->gen_loader) {
8519 /* reset FDs */
8520 if (obj->btf)
8521 btf__set_fd(obj->btf, -1);
8522 if (!err)
8523 err = bpf_gen__finish(obj->gen_loader, obj->nr_programs, obj->nr_maps);
8524 }
8525
8526 /* clean up fd_array */
8527 zfree(&obj->fd_array);
8528
8529 /* clean up module BTFs */
8530 for (i = 0; i < obj->btf_module_cnt; i++) {
8531 close(obj->btf_modules[i].fd);
8532 btf__free(obj->btf_modules[i].btf);
8533 free(obj->btf_modules[i].name);
8534 }
8535 free(obj->btf_modules);
8536
8537 /* clean up vmlinux BTF */
8538 btf__free(obj->btf_vmlinux);
8539 obj->btf_vmlinux = NULL;
8540
8541 obj->loaded = true; /* doesn't matter if successfully or not */
8542
8543 if (err)
8544 goto out;
8545
8546 return 0;
8547 out:
8548 /* unpin any maps that were auto-pinned during load */
8549 for (i = 0; i < obj->nr_maps; i++)
8550 if (obj->maps[i].pinned && !obj->maps[i].reused)
8551 bpf_map__unpin(&obj->maps[i], NULL);
8552
8553 bpf_object_unload(obj);
8554 pr_warn("failed to load object '%s'\n", obj->path);
8555 return libbpf_err(err);
8556 }
8557
8558 int bpf_object__load(struct bpf_object *obj)
8559 {
8560 return bpf_object_load(obj, 0, NULL);
8561 }
8562
8563 static int make_parent_dir(const char *path)
8564 {
8565 char *cp, errmsg[STRERR_BUFSIZE];
8566 char *dname, *dir;
8567 int err = 0;
8568
8569 dname = strdup(path);
8570 if (dname == NULL)
8571 return -ENOMEM;
8572
8573 dir = dirname(dname);
8574 if (mkdir(dir, 0700) && errno != EEXIST)
8575 err = -errno;
8576
8577 free(dname);
8578 if (err) {
8579 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
8580 pr_warn("failed to mkdir %s: %s\n", path, cp);
8581 }
8582 return err;
8583 }
8584
8585 static int check_path(const char *path)
8586 {
8587 char *cp, errmsg[STRERR_BUFSIZE];
8588 struct statfs st_fs;
8589 char *dname, *dir;
8590 int err = 0;
8591
8592 if (path == NULL)
8593 return -EINVAL;
8594
8595 dname = strdup(path);
8596 if (dname == NULL)
8597 return -ENOMEM;
8598
8599 dir = dirname(dname);
8600 if (statfs(dir, &st_fs)) {
8601 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
8602 pr_warn("failed to statfs %s: %s\n", dir, cp);
8603 err = -errno;
8604 }
8605 free(dname);
8606
8607 if (!err && st_fs.f_type != BPF_FS_MAGIC) {
8608 pr_warn("specified path %s is not on BPF FS\n", path);
8609 err = -EINVAL;
8610 }
8611
8612 return err;
8613 }
8614
8615 int bpf_program__pin(struct bpf_program *prog, const char *path)
8616 {
8617 char *cp, errmsg[STRERR_BUFSIZE];
8618 int err;
8619
8620 if (prog->fd < 0) {
8621 pr_warn("prog '%s': can't pin program that wasn't loaded\n", prog->name);
8622 return libbpf_err(-EINVAL);
8623 }
8624
8625 err = make_parent_dir(path);
8626 if (err)
8627 return libbpf_err(err);
8628
8629 err = check_path(path);
8630 if (err)
8631 return libbpf_err(err);
8632
8633 if (bpf_obj_pin(prog->fd, path)) {
8634 err = -errno;
8635 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
8636 pr_warn("prog '%s': failed to pin at '%s': %s\n", prog->name, path, cp);
8637 return libbpf_err(err);
8638 }
8639
8640 pr_debug("prog '%s': pinned at '%s'\n", prog->name, path);
8641 return 0;
8642 }
8643
8644 int bpf_program__unpin(struct bpf_program *prog, const char *path)
8645 {
8646 int err;
8647
8648 if (prog->fd < 0) {
8649 pr_warn("prog '%s': can't unpin program that wasn't loaded\n", prog->name);
8650 return libbpf_err(-EINVAL);
8651 }
8652
8653 err = check_path(path);
8654 if (err)
8655 return libbpf_err(err);
8656
8657 err = unlink(path);
8658 if (err)
8659 return libbpf_err(-errno);
8660
8661 pr_debug("prog '%s': unpinned from '%s'\n", prog->name, path);
8662 return 0;
8663 }
8664
8665 int bpf_map__pin(struct bpf_map *map, const char *path)
8666 {
8667 char *cp, errmsg[STRERR_BUFSIZE];
8668 int err;
8669
8670 if (map == NULL) {
8671 pr_warn("invalid map pointer\n");
8672 return libbpf_err(-EINVAL);
8673 }
8674
8675 if (map->fd < 0) {
8676 pr_warn("map '%s': can't pin BPF map without FD (was it created?)\n", map->name);
8677 return libbpf_err(-EINVAL);
8678 }
8679
8680 if (map->pin_path) {
8681 if (path && strcmp(path, map->pin_path)) {
8682 pr_warn("map '%s' already has pin path '%s' different from '%s'\n",
8683 bpf_map__name(map), map->pin_path, path);
8684 return libbpf_err(-EINVAL);
8685 } else if (map->pinned) {
8686 pr_debug("map '%s' already pinned at '%s'; not re-pinning\n",
8687 bpf_map__name(map), map->pin_path);
8688 return 0;
8689 }
8690 } else {
8691 if (!path) {
8692 pr_warn("missing a path to pin map '%s' at\n",
8693 bpf_map__name(map));
8694 return libbpf_err(-EINVAL);
8695 } else if (map->pinned) {
8696 pr_warn("map '%s' already pinned\n", bpf_map__name(map));
8697 return libbpf_err(-EEXIST);
8698 }
8699
8700 map->pin_path = strdup(path);
8701 if (!map->pin_path) {
8702 err = -errno;
8703 goto out_err;
8704 }
8705 }
8706
8707 err = make_parent_dir(map->pin_path);
8708 if (err)
8709 return libbpf_err(err);
8710
8711 err = check_path(map->pin_path);
8712 if (err)
8713 return libbpf_err(err);
8714
8715 if (bpf_obj_pin(map->fd, map->pin_path)) {
8716 err = -errno;
8717 goto out_err;
8718 }
8719
8720 map->pinned = true;
8721 pr_debug("pinned map '%s'\n", map->pin_path);
8722
8723 return 0;
8724
8725 out_err:
8726 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
8727 pr_warn("failed to pin map: %s\n", cp);
8728 return libbpf_err(err);
8729 }
8730
8731 int bpf_map__unpin(struct bpf_map *map, const char *path)
8732 {
8733 int err;
8734
8735 if (map == NULL) {
8736 pr_warn("invalid map pointer\n");
8737 return libbpf_err(-EINVAL);
8738 }
8739
8740 if (map->pin_path) {
8741 if (path && strcmp(path, map->pin_path)) {
8742 pr_warn("map '%s' already has pin path '%s' different from '%s'\n",
8743 bpf_map__name(map), map->pin_path, path);
8744 return libbpf_err(-EINVAL);
8745 }
8746 path = map->pin_path;
8747 } else if (!path) {
8748 pr_warn("no path to unpin map '%s' from\n",
8749 bpf_map__name(map));
8750 return libbpf_err(-EINVAL);
8751 }
8752
8753 err = check_path(path);
8754 if (err)
8755 return libbpf_err(err);
8756
8757 err = unlink(path);
8758 if (err != 0)
8759 return libbpf_err(-errno);
8760
8761 map->pinned = false;
8762 pr_debug("unpinned map '%s' from '%s'\n", bpf_map__name(map), path);
8763
8764 return 0;
8765 }
8766
8767 int bpf_map__set_pin_path(struct bpf_map *map, const char *path)
8768 {
8769 char *new = NULL;
8770
8771 if (path) {
8772 new = strdup(path);
8773 if (!new)
8774 return libbpf_err(-errno);
8775 }
8776
8777 free(map->pin_path);
8778 map->pin_path = new;
8779 return 0;
8780 }
8781
8782 __alias(bpf_map__pin_path)
8783 const char *bpf_map__get_pin_path(const struct bpf_map *map);
8784
8785 const char *bpf_map__pin_path(const struct bpf_map *map)
8786 {
8787 return map->pin_path;
8788 }
8789
8790 bool bpf_map__is_pinned(const struct bpf_map *map)
8791 {
8792 return map->pinned;
8793 }
8794
8795 static void sanitize_pin_path(char *s)
8796 {
8797 /* bpffs disallows periods in path names */
8798 while (*s) {
8799 if (*s == '.')
8800 *s = '_';
8801 s++;
8802 }
8803 }
8804
8805 int bpf_object__pin_maps(struct bpf_object *obj, const char *path)
8806 {
8807 struct bpf_map *map;
8808 int err;
8809
8810 if (!obj)
8811 return libbpf_err(-ENOENT);
8812
8813 if (!obj->loaded) {
8814 pr_warn("object not yet loaded; load it first\n");
8815 return libbpf_err(-ENOENT);
8816 }
8817
8818 bpf_object__for_each_map(map, obj) {
8819 char *pin_path = NULL;
8820 char buf[PATH_MAX];
8821
8822 if (!map->autocreate)
8823 continue;
8824
8825 if (path) {
8826 err = pathname_concat(buf, sizeof(buf), path, bpf_map__name(map));
8827 if (err)
8828 goto err_unpin_maps;
8829 sanitize_pin_path(buf);
8830 pin_path = buf;
8831 } else if (!map->pin_path) {
8832 continue;
8833 }
8834
8835 err = bpf_map__pin(map, pin_path);
8836 if (err)
8837 goto err_unpin_maps;
8838 }
8839
8840 return 0;
8841
8842 err_unpin_maps:
8843 while ((map = bpf_object__prev_map(obj, map))) {
8844 if (!map->pin_path)
8845 continue;
8846
8847 bpf_map__unpin(map, NULL);
8848 }
8849
8850 return libbpf_err(err);
8851 }
8852
8853 int bpf_object__unpin_maps(struct bpf_object *obj, const char *path)
8854 {
8855 struct bpf_map *map;
8856 int err;
8857
8858 if (!obj)
8859 return libbpf_err(-ENOENT);
8860
8861 bpf_object__for_each_map(map, obj) {
8862 char *pin_path = NULL;
8863 char buf[PATH_MAX];
8864
8865 if (path) {
8866 err = pathname_concat(buf, sizeof(buf), path, bpf_map__name(map));
8867 if (err)
8868 return libbpf_err(err);
8869 sanitize_pin_path(buf);
8870 pin_path = buf;
8871 } else if (!map->pin_path) {
8872 continue;
8873 }
8874
8875 err = bpf_map__unpin(map, pin_path);
8876 if (err)
8877 return libbpf_err(err);
8878 }
8879
8880 return 0;
8881 }
8882
8883 int bpf_object__pin_programs(struct bpf_object *obj, const char *path)
8884 {
8885 struct bpf_program *prog;
8886 char buf[PATH_MAX];
8887 int err;
8888
8889 if (!obj)
8890 return libbpf_err(-ENOENT);
8891
8892 if (!obj->loaded) {
8893 pr_warn("object not yet loaded; load it first\n");
8894 return libbpf_err(-ENOENT);
8895 }
8896
8897 bpf_object__for_each_program(prog, obj) {
8898 err = pathname_concat(buf, sizeof(buf), path, prog->name);
8899 if (err)
8900 goto err_unpin_programs;
8901
8902 err = bpf_program__pin(prog, buf);
8903 if (err)
8904 goto err_unpin_programs;
8905 }
8906
8907 return 0;
8908
8909 err_unpin_programs:
8910 while ((prog = bpf_object__prev_program(obj, prog))) {
8911 if (pathname_concat(buf, sizeof(buf), path, prog->name))
8912 continue;
8913
8914 bpf_program__unpin(prog, buf);
8915 }
8916
8917 return libbpf_err(err);
8918 }
8919
8920 int bpf_object__unpin_programs(struct bpf_object *obj, const char *path)
8921 {
8922 struct bpf_program *prog;
8923 int err;
8924
8925 if (!obj)
8926 return libbpf_err(-ENOENT);
8927
8928 bpf_object__for_each_program(prog, obj) {
8929 char buf[PATH_MAX];
8930
8931 err = pathname_concat(buf, sizeof(buf), path, prog->name);
8932 if (err)
8933 return libbpf_err(err);
8934
8935 err = bpf_program__unpin(prog, buf);
8936 if (err)
8937 return libbpf_err(err);
8938 }
8939
8940 return 0;
8941 }
8942
8943 int bpf_object__pin(struct bpf_object *obj, const char *path)
8944 {
8945 int err;
8946
8947 err = bpf_object__pin_maps(obj, path);
8948 if (err)
8949 return libbpf_err(err);
8950
8951 err = bpf_object__pin_programs(obj, path);
8952 if (err) {
8953 bpf_object__unpin_maps(obj, path);
8954 return libbpf_err(err);
8955 }
8956
8957 return 0;
8958 }
8959
8960 int bpf_object__unpin(struct bpf_object *obj, const char *path)
8961 {
8962 int err;
8963
8964 err = bpf_object__unpin_programs(obj, path);
8965 if (err)
8966 return libbpf_err(err);
8967
8968 err = bpf_object__unpin_maps(obj, path);
8969 if (err)
8970 return libbpf_err(err);
8971
8972 return 0;
8973 }
8974
8975 static void bpf_map__destroy(struct bpf_map *map)
8976 {
8977 if (map->inner_map) {
8978 bpf_map__destroy(map->inner_map);
8979 zfree(&map->inner_map);
8980 }
8981
8982 zfree(&map->init_slots);
8983 map->init_slots_sz = 0;
8984
8985 if (map->mmaped && map->mmaped != map->obj->arena_data)
8986 munmap(map->mmaped, bpf_map_mmap_sz(map));
8987 map->mmaped = NULL;
8988
8989 if (map->st_ops) {
8990 zfree(&map->st_ops->data);
8991 zfree(&map->st_ops->progs);
8992 zfree(&map->st_ops->kern_func_off);
8993 zfree(&map->st_ops);
8994 }
8995
8996 zfree(&map->name);
8997 zfree(&map->real_name);
8998 zfree(&map->pin_path);
8999
9000 if (map->fd >= 0)
9001 zclose(map->fd);
9002 }
9003
9004 void bpf_object__close(struct bpf_object *obj)
9005 {
9006 size_t i;
9007
9008 if (IS_ERR_OR_NULL(obj))
9009 return;
9010
9011 usdt_manager_free(obj->usdt_man);
9012 obj->usdt_man = NULL;
9013
9014 bpf_gen__free(obj->gen_loader);
9015 bpf_object__elf_finish(obj);
9016 bpf_object_unload(obj);
9017 btf__free(obj->btf);
9018 btf__free(obj->btf_vmlinux);
9019 btf_ext__free(obj->btf_ext);
9020
9021 for (i = 0; i < obj->nr_maps; i++)
9022 bpf_map__destroy(&obj->maps[i]);
9023
9024 zfree(&obj->btf_custom_path);
9025 zfree(&obj->kconfig);
9026
9027 for (i = 0; i < obj->nr_extern; i++)
9028 zfree(&obj->externs[i].essent_name);
9029
9030 zfree(&obj->externs);
9031 obj->nr_extern = 0;
9032
9033 zfree(&obj->maps);
9034 obj->nr_maps = 0;
9035
9036 if (obj->programs && obj->nr_programs) {
9037 for (i = 0; i < obj->nr_programs; i++)
9038 bpf_program__exit(&obj->programs[i]);
9039 }
9040 zfree(&obj->programs);
9041
9042 zfree(&obj->feat_cache);
9043 zfree(&obj->token_path);
9044 if (obj->token_fd > 0)
9045 close(obj->token_fd);
9046
9047 zfree(&obj->arena_data);
9048
9049 free(obj);
9050 }
9051
9052 const char *bpf_object__name(const struct bpf_object *obj)
9053 {
9054 return obj ? obj->name : libbpf_err_ptr(-EINVAL);
9055 }
9056
9057 unsigned int bpf_object__kversion(const struct bpf_object *obj)
9058 {
9059 return obj ? obj->kern_version : 0;
9060 }
9061
9062 struct btf *bpf_object__btf(const struct bpf_object *obj)
9063 {
9064 return obj ? obj->btf : NULL;
9065 }
9066
9067 int bpf_object__btf_fd(const struct bpf_object *obj)
9068 {
9069 return obj->btf ? btf__fd(obj->btf) : -1;
9070 }
9071
9072 int bpf_object__set_kversion(struct bpf_object *obj, __u32 kern_version)
9073 {
9074 if (obj->loaded)
9075 return libbpf_err(-EINVAL);
9076
9077 obj->kern_version = kern_version;
9078
9079 return 0;
9080 }
9081
9082 int bpf_object__gen_loader(struct bpf_object *obj, struct gen_loader_opts *opts)
9083 {
9084 struct bpf_gen *gen;
9085
9086 if (!opts)
9087 return -EFAULT;
9088 if (!OPTS_VALID(opts, gen_loader_opts))
9089 return -EINVAL;
9090 gen = calloc(sizeof(*gen), 1);
9091 if (!gen)
9092 return -ENOMEM;
9093 gen->opts = opts;
9094 obj->gen_loader = gen;
9095 return 0;
9096 }
9097
9098 static struct bpf_program *
9099 __bpf_program__iter(const struct bpf_program *p, const struct bpf_object *obj,
9100 bool forward)
9101 {
9102 size_t nr_programs = obj->nr_programs;
9103 ssize_t idx;
9104
9105 if (!nr_programs)
9106 return NULL;
9107
9108 if (!p)
9109 /* Iter from the beginning */
9110 return forward ? &obj->programs[0] :
9111 &obj->programs[nr_programs - 1];
9112
9113 if (p->obj != obj) {
9114 pr_warn("error: program handler doesn't match object\n");
9115 return errno = EINVAL, NULL;
9116 }
9117
9118 idx = (p - obj->programs) + (forward ? 1 : -1);
9119 if (idx >= obj->nr_programs || idx < 0)
9120 return NULL;
9121 return &obj->programs[idx];
9122 }
9123
9124 struct bpf_program *
9125 bpf_object__next_program(const struct bpf_object *obj, struct bpf_program *prev)
9126 {
9127 struct bpf_program *prog = prev;
9128
9129 do {
9130 prog = __bpf_program__iter(prog, obj, true);
9131 } while (prog && prog_is_subprog(obj, prog));
9132
9133 return prog;
9134 }
9135
9136 struct bpf_program *
9137 bpf_object__prev_program(const struct bpf_object *obj, struct bpf_program *next)
9138 {
9139 struct bpf_program *prog = next;
9140
9141 do {
9142 prog = __bpf_program__iter(prog, obj, false);
9143 } while (prog && prog_is_subprog(obj, prog));
9144
9145 return prog;
9146 }
9147
9148 void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex)
9149 {
9150 prog->prog_ifindex = ifindex;
9151 }
9152
9153 const char *bpf_program__name(const struct bpf_program *prog)
9154 {
9155 return prog->name;
9156 }
9157
9158 const char *bpf_program__section_name(const struct bpf_program *prog)
9159 {
9160 return prog->sec_name;
9161 }
9162
9163 bool bpf_program__autoload(const struct bpf_program *prog)
9164 {
9165 return prog->autoload;
9166 }
9167
9168 int bpf_program__set_autoload(struct bpf_program *prog, bool autoload)
9169 {
9170 if (prog->obj->loaded)
9171 return libbpf_err(-EINVAL);
9172
9173 prog->autoload = autoload;
9174 return 0;
9175 }
9176
9177 bool bpf_program__autoattach(const struct bpf_program *prog)
9178 {
9179 return prog->autoattach;
9180 }
9181
9182 void bpf_program__set_autoattach(struct bpf_program *prog, bool autoattach)
9183 {
9184 prog->autoattach = autoattach;
9185 }
9186
9187 const struct bpf_insn *bpf_program__insns(const struct bpf_program *prog)
9188 {
9189 return prog->insns;
9190 }
9191
9192 size_t bpf_program__insn_cnt(const struct bpf_program *prog)
9193 {
9194 return prog->insns_cnt;
9195 }
9196
9197 int bpf_program__set_insns(struct bpf_program *prog,
9198 struct bpf_insn *new_insns, size_t new_insn_cnt)
9199 {
9200 struct bpf_insn *insns;
9201
9202 if (prog->obj->loaded)
9203 return -EBUSY;
9204
9205 insns = libbpf_reallocarray(prog->insns, new_insn_cnt, sizeof(*insns));
9206 /* NULL is a valid return from reallocarray if the new count is zero */
9207 if (!insns && new_insn_cnt) {
9208 pr_warn("prog '%s': failed to realloc prog code\n", prog->name);
9209 return -ENOMEM;
9210 }
9211 memcpy(insns, new_insns, new_insn_cnt * sizeof(*insns));
9212
9213 prog->insns = insns;
9214 prog->insns_cnt = new_insn_cnt;
9215 return 0;
9216 }
9217
9218 int bpf_program__fd(const struct bpf_program *prog)
9219 {
9220 if (!prog)
9221 return libbpf_err(-EINVAL);
9222
9223 if (prog->fd < 0)
9224 return libbpf_err(-ENOENT);
9225
9226 return prog->fd;
9227 }
9228
9229 __alias(bpf_program__type)
9230 enum bpf_prog_type bpf_program__get_type(const struct bpf_program *prog);
9231
9232 enum bpf_prog_type bpf_program__type(const struct bpf_program *prog)
9233 {
9234 return prog->type;
9235 }
9236
9237 static size_t custom_sec_def_cnt;
9238 static struct bpf_sec_def *custom_sec_defs;
9239 static struct bpf_sec_def custom_fallback_def;
9240 static bool has_custom_fallback_def;
9241 static int last_custom_sec_def_handler_id;
9242
9243 int bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type)
9244 {
9245 if (prog->obj->loaded)
9246 return libbpf_err(-EBUSY);
9247
9248 /* if type is not changed, do nothing */
9249 if (prog->type == type)
9250 return 0;
9251
9252 prog->type = type;
9253
9254 /* If a program type was changed, we need to reset associated SEC()
9255 * handler, as it will be invalid now. The only exception is a generic
9256 * fallback handler, which by definition is program type-agnostic and
9257 * is a catch-all custom handler, optionally set by the application,
9258 * so should be able to handle any type of BPF program.
9259 */
9260 if (prog->sec_def != &custom_fallback_def)
9261 prog->sec_def = NULL;
9262 return 0;
9263 }
9264
9265 __alias(bpf_program__expected_attach_type)
9266 enum bpf_attach_type bpf_program__get_expected_attach_type(const struct bpf_program *prog);
9267
9268 enum bpf_attach_type bpf_program__expected_attach_type(const struct bpf_program *prog)
9269 {
9270 return prog->expected_attach_type;
9271 }
9272
9273 int bpf_program__set_expected_attach_type(struct bpf_program *prog,
9274 enum bpf_attach_type type)
9275 {
9276 if (prog->obj->loaded)
9277 return libbpf_err(-EBUSY);
9278
9279 prog->expected_attach_type = type;
9280 return 0;
9281 }
9282
9283 __u32 bpf_program__flags(const struct bpf_program *prog)
9284 {
9285 return prog->prog_flags;
9286 }
9287
9288 int bpf_program__set_flags(struct bpf_program *prog, __u32 flags)
9289 {
9290 if (prog->obj->loaded)
9291 return libbpf_err(-EBUSY);
9292
9293 prog->prog_flags = flags;
9294 return 0;
9295 }
9296
9297 __u32 bpf_program__log_level(const struct bpf_program *prog)
9298 {
9299 return prog->log_level;
9300 }
9301
9302 int bpf_program__set_log_level(struct bpf_program *prog, __u32 log_level)
9303 {
9304 if (prog->obj->loaded)
9305 return libbpf_err(-EBUSY);
9306
9307 prog->log_level = log_level;
9308 return 0;
9309 }
9310
9311 const char *bpf_program__log_buf(const struct bpf_program *prog, size_t *log_size)
9312 {
9313 *log_size = prog->log_size;
9314 return prog->log_buf;
9315 }
9316
9317 int bpf_program__set_log_buf(struct bpf_program *prog, char *log_buf, size_t log_size)
9318 {
9319 if (log_size && !log_buf)
9320 return -EINVAL;
9321 if (prog->log_size > UINT_MAX)
9322 return -EINVAL;
9323 if (prog->obj->loaded)
9324 return -EBUSY;
9325
9326 prog->log_buf = log_buf;
9327 prog->log_size = log_size;
9328 return 0;
9329 }
9330
9331 #define SEC_DEF(sec_pfx, ptype, atype, flags, ...) { \
9332 .sec = (char *)sec_pfx, \
9333 .prog_type = BPF_PROG_TYPE_##ptype, \
9334 .expected_attach_type = atype, \
9335 .cookie = (long)(flags), \
9336 .prog_prepare_load_fn = libbpf_prepare_prog_load, \
9337 __VA_ARGS__ \
9338 }
9339
9340 static int attach_kprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9341 static int attach_uprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9342 static int attach_ksyscall(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9343 static int attach_usdt(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9344 static int attach_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9345 static int attach_raw_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9346 static int attach_trace(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9347 static int attach_kprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9348 static int attach_kprobe_session(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9349 static int attach_uprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9350 static int attach_lsm(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9351 static int attach_iter(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9352
9353 static const struct bpf_sec_def section_defs[] = {
9354 SEC_DEF("socket", SOCKET_FILTER, 0, SEC_NONE),
9355 SEC_DEF("sk_reuseport/migrate", SK_REUSEPORT, BPF_SK_REUSEPORT_SELECT_OR_MIGRATE, SEC_ATTACHABLE),
9356 SEC_DEF("sk_reuseport", SK_REUSEPORT, BPF_SK_REUSEPORT_SELECT, SEC_ATTACHABLE),
9357 SEC_DEF("kprobe+", KPROBE, 0, SEC_NONE, attach_kprobe),
9358 SEC_DEF("uprobe+", KPROBE, 0, SEC_NONE, attach_uprobe),
9359 SEC_DEF("uprobe.s+", KPROBE, 0, SEC_SLEEPABLE, attach_uprobe),
9360 SEC_DEF("kretprobe+", KPROBE, 0, SEC_NONE, attach_kprobe),
9361 SEC_DEF("uretprobe+", KPROBE, 0, SEC_NONE, attach_uprobe),
9362 SEC_DEF("uretprobe.s+", KPROBE, 0, SEC_SLEEPABLE, attach_uprobe),
9363 SEC_DEF("kprobe.multi+", KPROBE, BPF_TRACE_KPROBE_MULTI, SEC_NONE, attach_kprobe_multi),
9364 SEC_DEF("kretprobe.multi+", KPROBE, BPF_TRACE_KPROBE_MULTI, SEC_NONE, attach_kprobe_multi),
9365 SEC_DEF("kprobe.session+", KPROBE, BPF_TRACE_KPROBE_SESSION, SEC_NONE, attach_kprobe_session),
9366 SEC_DEF("uprobe.multi+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_NONE, attach_uprobe_multi),
9367 SEC_DEF("uretprobe.multi+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_NONE, attach_uprobe_multi),
9368 SEC_DEF("uprobe.multi.s+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_SLEEPABLE, attach_uprobe_multi),
9369 SEC_DEF("uretprobe.multi.s+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_SLEEPABLE, attach_uprobe_multi),
9370 SEC_DEF("ksyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall),
9371 SEC_DEF("kretsyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall),
9372 SEC_DEF("usdt+", KPROBE, 0, SEC_USDT, attach_usdt),
9373 SEC_DEF("usdt.s+", KPROBE, 0, SEC_USDT | SEC_SLEEPABLE, attach_usdt),
9374 SEC_DEF("tc/ingress", SCHED_CLS, BPF_TCX_INGRESS, SEC_NONE), /* alias for tcx */
9375 SEC_DEF("tc/egress", SCHED_CLS, BPF_TCX_EGRESS, SEC_NONE), /* alias for tcx */
9376 SEC_DEF("tcx/ingress", SCHED_CLS, BPF_TCX_INGRESS, SEC_NONE),
9377 SEC_DEF("tcx/egress", SCHED_CLS, BPF_TCX_EGRESS, SEC_NONE),
9378 SEC_DEF("tc", SCHED_CLS, 0, SEC_NONE), /* deprecated / legacy, use tcx */
9379 SEC_DEF("classifier", SCHED_CLS, 0, SEC_NONE), /* deprecated / legacy, use tcx */
9380 SEC_DEF("action", SCHED_ACT, 0, SEC_NONE), /* deprecated / legacy, use tcx */
9381 SEC_DEF("netkit/primary", SCHED_CLS, BPF_NETKIT_PRIMARY, SEC_NONE),
9382 SEC_DEF("netkit/peer", SCHED_CLS, BPF_NETKIT_PEER, SEC_NONE),
9383 SEC_DEF("tracepoint+", TRACEPOINT, 0, SEC_NONE, attach_tp),
9384 SEC_DEF("tp+", TRACEPOINT, 0, SEC_NONE, attach_tp),
9385 SEC_DEF("raw_tracepoint+", RAW_TRACEPOINT, 0, SEC_NONE, attach_raw_tp),
9386 SEC_DEF("raw_tp+", RAW_TRACEPOINT, 0, SEC_NONE, attach_raw_tp),
9387 SEC_DEF("raw_tracepoint.w+", RAW_TRACEPOINT_WRITABLE, 0, SEC_NONE, attach_raw_tp),
9388 SEC_DEF("raw_tp.w+", RAW_TRACEPOINT_WRITABLE, 0, SEC_NONE, attach_raw_tp),
9389 SEC_DEF("tp_btf+", TRACING, BPF_TRACE_RAW_TP, SEC_ATTACH_BTF, attach_trace),
9390 SEC_DEF("fentry+", TRACING, BPF_TRACE_FENTRY, SEC_ATTACH_BTF, attach_trace),
9391 SEC_DEF("fmod_ret+", TRACING, BPF_MODIFY_RETURN, SEC_ATTACH_BTF, attach_trace),
9392 SEC_DEF("fexit+", TRACING, BPF_TRACE_FEXIT, SEC_ATTACH_BTF, attach_trace),
9393 SEC_DEF("fentry.s+", TRACING, BPF_TRACE_FENTRY, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_trace),
9394 SEC_DEF("fmod_ret.s+", TRACING, BPF_MODIFY_RETURN, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_trace),
9395 SEC_DEF("fexit.s+", TRACING, BPF_TRACE_FEXIT, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_trace),
9396 SEC_DEF("freplace+", EXT, 0, SEC_ATTACH_BTF, attach_trace),
9397 SEC_DEF("lsm+", LSM, BPF_LSM_MAC, SEC_ATTACH_BTF, attach_lsm),
9398 SEC_DEF("lsm.s+", LSM, BPF_LSM_MAC, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_lsm),
9399 SEC_DEF("lsm_cgroup+", LSM, BPF_LSM_CGROUP, SEC_ATTACH_BTF),
9400 SEC_DEF("iter+", TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF, attach_iter),
9401 SEC_DEF("iter.s+", TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_iter),
9402 SEC_DEF("syscall", SYSCALL, 0, SEC_SLEEPABLE),
9403 SEC_DEF("xdp.frags/devmap", XDP, BPF_XDP_DEVMAP, SEC_XDP_FRAGS),
9404 SEC_DEF("xdp/devmap", XDP, BPF_XDP_DEVMAP, SEC_ATTACHABLE),
9405 SEC_DEF("xdp.frags/cpumap", XDP, BPF_XDP_CPUMAP, SEC_XDP_FRAGS),
9406 SEC_DEF("xdp/cpumap", XDP, BPF_XDP_CPUMAP, SEC_ATTACHABLE),
9407 SEC_DEF("xdp.frags", XDP, BPF_XDP, SEC_XDP_FRAGS),
9408 SEC_DEF("xdp", XDP, BPF_XDP, SEC_ATTACHABLE_OPT),
9409 SEC_DEF("perf_event", PERF_EVENT, 0, SEC_NONE),
9410 SEC_DEF("lwt_in", LWT_IN, 0, SEC_NONE),
9411 SEC_DEF("lwt_out", LWT_OUT, 0, SEC_NONE),
9412 SEC_DEF("lwt_xmit", LWT_XMIT, 0, SEC_NONE),
9413 SEC_DEF("lwt_seg6local", LWT_SEG6LOCAL, 0, SEC_NONE),
9414 SEC_DEF("sockops", SOCK_OPS, BPF_CGROUP_SOCK_OPS, SEC_ATTACHABLE_OPT),
9415 SEC_DEF("sk_skb/stream_parser", SK_SKB, BPF_SK_SKB_STREAM_PARSER, SEC_ATTACHABLE_OPT),
9416 SEC_DEF("sk_skb/stream_verdict",SK_SKB, BPF_SK_SKB_STREAM_VERDICT, SEC_ATTACHABLE_OPT),
9417 SEC_DEF("sk_skb/verdict", SK_SKB, BPF_SK_SKB_VERDICT, SEC_ATTACHABLE_OPT),
9418 SEC_DEF("sk_skb", SK_SKB, 0, SEC_NONE),
9419 SEC_DEF("sk_msg", SK_MSG, BPF_SK_MSG_VERDICT, SEC_ATTACHABLE_OPT),
9420 SEC_DEF("lirc_mode2", LIRC_MODE2, BPF_LIRC_MODE2, SEC_ATTACHABLE_OPT),
9421 SEC_DEF("flow_dissector", FLOW_DISSECTOR, BPF_FLOW_DISSECTOR, SEC_ATTACHABLE_OPT),
9422 SEC_DEF("cgroup_skb/ingress", CGROUP_SKB, BPF_CGROUP_INET_INGRESS, SEC_ATTACHABLE_OPT),
9423 SEC_DEF("cgroup_skb/egress", CGROUP_SKB, BPF_CGROUP_INET_EGRESS, SEC_ATTACHABLE_OPT),
9424 SEC_DEF("cgroup/skb", CGROUP_SKB, 0, SEC_NONE),
9425 SEC_DEF("cgroup/sock_create", CGROUP_SOCK, BPF_CGROUP_INET_SOCK_CREATE, SEC_ATTACHABLE),
9426 SEC_DEF("cgroup/sock_release", CGROUP_SOCK, BPF_CGROUP_INET_SOCK_RELEASE, SEC_ATTACHABLE),
9427 SEC_DEF("cgroup/sock", CGROUP_SOCK, BPF_CGROUP_INET_SOCK_CREATE, SEC_ATTACHABLE_OPT),
9428 SEC_DEF("cgroup/post_bind4", CGROUP_SOCK, BPF_CGROUP_INET4_POST_BIND, SEC_ATTACHABLE),
9429 SEC_DEF("cgroup/post_bind6", CGROUP_SOCK, BPF_CGROUP_INET6_POST_BIND, SEC_ATTACHABLE),
9430 SEC_DEF("cgroup/bind4", CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_BIND, SEC_ATTACHABLE),
9431 SEC_DEF("cgroup/bind6", CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_BIND, SEC_ATTACHABLE),
9432 SEC_DEF("cgroup/connect4", CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_CONNECT, SEC_ATTACHABLE),
9433 SEC_DEF("cgroup/connect6", CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_CONNECT, SEC_ATTACHABLE),
9434 SEC_DEF("cgroup/connect_unix", CGROUP_SOCK_ADDR, BPF_CGROUP_UNIX_CONNECT, SEC_ATTACHABLE),
9435 SEC_DEF("cgroup/sendmsg4", CGROUP_SOCK_ADDR, BPF_CGROUP_UDP4_SENDMSG, SEC_ATTACHABLE),
9436 SEC_DEF("cgroup/sendmsg6", CGROUP_SOCK_ADDR, BPF_CGROUP_UDP6_SENDMSG, SEC_ATTACHABLE),
9437 SEC_DEF("cgroup/sendmsg_unix", CGROUP_SOCK_ADDR, BPF_CGROUP_UNIX_SENDMSG, SEC_ATTACHABLE),
9438 SEC_DEF("cgroup/recvmsg4", CGROUP_SOCK_ADDR, BPF_CGROUP_UDP4_RECVMSG, SEC_ATTACHABLE),
9439 SEC_DEF("cgroup/recvmsg6", CGROUP_SOCK_ADDR, BPF_CGROUP_UDP6_RECVMSG, SEC_ATTACHABLE),
9440 SEC_DEF("cgroup/recvmsg_unix", CGROUP_SOCK_ADDR, BPF_CGROUP_UNIX_RECVMSG, SEC_ATTACHABLE),
9441 SEC_DEF("cgroup/getpeername4", CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_GETPEERNAME, SEC_ATTACHABLE),
9442 SEC_DEF("cgroup/getpeername6", CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_GETPEERNAME, SEC_ATTACHABLE),
9443 SEC_DEF("cgroup/getpeername_unix", CGROUP_SOCK_ADDR, BPF_CGROUP_UNIX_GETPEERNAME, SEC_ATTACHABLE),
9444 SEC_DEF("cgroup/getsockname4", CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_GETSOCKNAME, SEC_ATTACHABLE),
9445 SEC_DEF("cgroup/getsockname6", CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_GETSOCKNAME, SEC_ATTACHABLE),
9446 SEC_DEF("cgroup/getsockname_unix", CGROUP_SOCK_ADDR, BPF_CGROUP_UNIX_GETSOCKNAME, SEC_ATTACHABLE),
9447 SEC_DEF("cgroup/sysctl", CGROUP_SYSCTL, BPF_CGROUP_SYSCTL, SEC_ATTACHABLE),
9448 SEC_DEF("cgroup/getsockopt", CGROUP_SOCKOPT, BPF_CGROUP_GETSOCKOPT, SEC_ATTACHABLE),
9449 SEC_DEF("cgroup/setsockopt", CGROUP_SOCKOPT, BPF_CGROUP_SETSOCKOPT, SEC_ATTACHABLE),
9450 SEC_DEF("cgroup/dev", CGROUP_DEVICE, BPF_CGROUP_DEVICE, SEC_ATTACHABLE_OPT),
9451 SEC_DEF("struct_ops+", STRUCT_OPS, 0, SEC_NONE),
9452 SEC_DEF("struct_ops.s+", STRUCT_OPS, 0, SEC_SLEEPABLE),
9453 SEC_DEF("sk_lookup", SK_LOOKUP, BPF_SK_LOOKUP, SEC_ATTACHABLE),
9454 SEC_DEF("netfilter", NETFILTER, BPF_NETFILTER, SEC_NONE),
9455 };
9456
9457 int libbpf_register_prog_handler(const char *sec,
9458 enum bpf_prog_type prog_type,
9459 enum bpf_attach_type exp_attach_type,
9460 const struct libbpf_prog_handler_opts *opts)
9461 {
9462 struct bpf_sec_def *sec_def;
9463
9464 if (!OPTS_VALID(opts, libbpf_prog_handler_opts))
9465 return libbpf_err(-EINVAL);
9466
9467 if (last_custom_sec_def_handler_id == INT_MAX) /* prevent overflow */
9468 return libbpf_err(-E2BIG);
9469
9470 if (sec) {
9471 sec_def = libbpf_reallocarray(custom_sec_defs, custom_sec_def_cnt + 1,
9472 sizeof(*sec_def));
9473 if (!sec_def)
9474 return libbpf_err(-ENOMEM);
9475
9476 custom_sec_defs = sec_def;
9477 sec_def = &custom_sec_defs[custom_sec_def_cnt];
9478 } else {
9479 if (has_custom_fallback_def)
9480 return libbpf_err(-EBUSY);
9481
9482 sec_def = &custom_fallback_def;
9483 }
9484
9485 sec_def->sec = sec ? strdup(sec) : NULL;
9486 if (sec && !sec_def->sec)
9487 return libbpf_err(-ENOMEM);
9488
9489 sec_def->prog_type = prog_type;
9490 sec_def->expected_attach_type = exp_attach_type;
9491 sec_def->cookie = OPTS_GET(opts, cookie, 0);
9492
9493 sec_def->prog_setup_fn = OPTS_GET(opts, prog_setup_fn, NULL);
9494 sec_def->prog_prepare_load_fn = OPTS_GET(opts, prog_prepare_load_fn, NULL);
9495 sec_def->prog_attach_fn = OPTS_GET(opts, prog_attach_fn, NULL);
9496
9497 sec_def->handler_id = ++last_custom_sec_def_handler_id;
9498
9499 if (sec)
9500 custom_sec_def_cnt++;
9501 else
9502 has_custom_fallback_def = true;
9503
9504 return sec_def->handler_id;
9505 }
9506
9507 int libbpf_unregister_prog_handler(int handler_id)
9508 {
9509 struct bpf_sec_def *sec_defs;
9510 int i;
9511
9512 if (handler_id <= 0)
9513 return libbpf_err(-EINVAL);
9514
9515 if (has_custom_fallback_def && custom_fallback_def.handler_id == handler_id) {
9516 memset(&custom_fallback_def, 0, sizeof(custom_fallback_def));
9517 has_custom_fallback_def = false;
9518 return 0;
9519 }
9520
9521 for (i = 0; i < custom_sec_def_cnt; i++) {
9522 if (custom_sec_defs[i].handler_id == handler_id)
9523 break;
9524 }
9525
9526 if (i == custom_sec_def_cnt)
9527 return libbpf_err(-ENOENT);
9528
9529 free(custom_sec_defs[i].sec);
9530 for (i = i + 1; i < custom_sec_def_cnt; i++)
9531 custom_sec_defs[i - 1] = custom_sec_defs[i];
9532 custom_sec_def_cnt--;
9533
9534 /* try to shrink the array, but it's ok if we couldn't */
9535 sec_defs = libbpf_reallocarray(custom_sec_defs, custom_sec_def_cnt, sizeof(*sec_defs));
9536 /* if new count is zero, reallocarray can return a valid NULL result;
9537 * in this case the previous pointer will be freed, so we *have to*
9538 * reassign old pointer to the new value (even if it's NULL)
9539 */
9540 if (sec_defs || custom_sec_def_cnt == 0)
9541 custom_sec_defs = sec_defs;
9542
9543 return 0;
9544 }
9545
9546 static bool sec_def_matches(const struct bpf_sec_def *sec_def, const char *sec_name)
9547 {
9548 size_t len = strlen(sec_def->sec);
9549
9550 /* "type/" always has to have proper SEC("type/extras") form */
9551 if (sec_def->sec[len - 1] == '/') {
9552 if (str_has_pfx(sec_name, sec_def->sec))
9553 return true;
9554 return false;
9555 }
9556
9557 /* "type+" means it can be either exact SEC("type") or
9558 * well-formed SEC("type/extras") with proper '/' separator
9559 */
9560 if (sec_def->sec[len - 1] == '+') {
9561 len--;
9562 /* not even a prefix */
9563 if (strncmp(sec_name, sec_def->sec, len) != 0)
9564 return false;
9565 /* exact match or has '/' separator */
9566 if (sec_name[len] == '\0' || sec_name[len] == '/')
9567 return true;
9568 return false;
9569 }
9570
9571 return strcmp(sec_name, sec_def->sec) == 0;
9572 }
9573
9574 static const struct bpf_sec_def *find_sec_def(const char *sec_name)
9575 {
9576 const struct bpf_sec_def *sec_def;
9577 int i, n;
9578
9579 n = custom_sec_def_cnt;
9580 for (i = 0; i < n; i++) {
9581 sec_def = &custom_sec_defs[i];
9582 if (sec_def_matches(sec_def, sec_name))
9583 return sec_def;
9584 }
9585
9586 n = ARRAY_SIZE(section_defs);
9587 for (i = 0; i < n; i++) {
9588 sec_def = §ion_defs[i];
9589 if (sec_def_matches(sec_def, sec_name))
9590 return sec_def;
9591 }
9592
9593 if (has_custom_fallback_def)
9594 return &custom_fallback_def;
9595
9596 return NULL;
9597 }
9598
9599 #define MAX_TYPE_NAME_SIZE 32
9600
9601 static char *libbpf_get_type_names(bool attach_type)
9602 {
9603 int i, len = ARRAY_SIZE(section_defs) * MAX_TYPE_NAME_SIZE;
9604 char *buf;
9605
9606 buf = malloc(len);
9607 if (!buf)
9608 return NULL;
9609
9610 buf[0] = '\0';
9611 /* Forge string buf with all available names */
9612 for (i = 0; i < ARRAY_SIZE(section_defs); i++) {
9613 const struct bpf_sec_def *sec_def = §ion_defs[i];
9614
9615 if (attach_type) {
9616 if (sec_def->prog_prepare_load_fn != libbpf_prepare_prog_load)
9617 continue;
9618
9619 if (!(sec_def->cookie & SEC_ATTACHABLE))
9620 continue;
9621 }
9622
9623 if (strlen(buf) + strlen(section_defs[i].sec) + 2 > len) {
9624 free(buf);
9625 return NULL;
9626 }
9627 strcat(buf, " ");
9628 strcat(buf, section_defs[i].sec);
9629 }
9630
9631 return buf;
9632 }
9633
9634 int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type,
9635 enum bpf_attach_type *expected_attach_type)
9636 {
9637 const struct bpf_sec_def *sec_def;
9638 char *type_names;
9639
9640 if (!name)
9641 return libbpf_err(-EINVAL);
9642
9643 sec_def = find_sec_def(name);
9644 if (sec_def) {
9645 *prog_type = sec_def->prog_type;
9646 *expected_attach_type = sec_def->expected_attach_type;
9647 return 0;
9648 }
9649
9650 pr_debug("failed to guess program type from ELF section '%s'\n", name);
9651 type_names = libbpf_get_type_names(false);
9652 if (type_names != NULL) {
9653 pr_debug("supported section(type) names are:%s\n", type_names);
9654 free(type_names);
9655 }
9656
9657 return libbpf_err(-ESRCH);
9658 }
9659
9660 const char *libbpf_bpf_attach_type_str(enum bpf_attach_type t)
9661 {
9662 if (t < 0 || t >= ARRAY_SIZE(attach_type_name))
9663 return NULL;
9664
9665 return attach_type_name[t];
9666 }
9667
9668 const char *libbpf_bpf_link_type_str(enum bpf_link_type t)
9669 {
9670 if (t < 0 || t >= ARRAY_SIZE(link_type_name))
9671 return NULL;
9672
9673 return link_type_name[t];
9674 }
9675
9676 const char *libbpf_bpf_map_type_str(enum bpf_map_type t)
9677 {
9678 if (t < 0 || t >= ARRAY_SIZE(map_type_name))
9679 return NULL;
9680
9681 return map_type_name[t];
9682 }
9683
9684 const char *libbpf_bpf_prog_type_str(enum bpf_prog_type t)
9685 {
9686 if (t < 0 || t >= ARRAY_SIZE(prog_type_name))
9687 return NULL;
9688
9689 return prog_type_name[t];
9690 }
9691
9692 static struct bpf_map *find_struct_ops_map_by_offset(struct bpf_object *obj,
9693 int sec_idx,
9694 size_t offset)
9695 {
9696 struct bpf_map *map;
9697 size_t i;
9698
9699 for (i = 0; i < obj->nr_maps; i++) {
9700 map = &obj->maps[i];
9701 if (!bpf_map__is_struct_ops(map))
9702 continue;
9703 if (map->sec_idx == sec_idx &&
9704 map->sec_offset <= offset &&
9705 offset - map->sec_offset < map->def.value_size)
9706 return map;
9707 }
9708
9709 return NULL;
9710 }
9711
9712 /* Collect the reloc from ELF, populate the st_ops->progs[], and update
9713 * st_ops->data for shadow type.
9714 */
9715 static int bpf_object__collect_st_ops_relos(struct bpf_object *obj,
9716 Elf64_Shdr *shdr, Elf_Data *data)
9717 {
9718 const struct btf_type *type;
9719 const struct btf_member *member;
9720 struct bpf_struct_ops *st_ops;
9721 struct bpf_program *prog;
9722 unsigned int shdr_idx;
9723 const struct btf *btf;
9724 struct bpf_map *map;
9725 unsigned int moff, insn_idx;
9726 const char *name;
9727 __u32 member_idx;
9728 Elf64_Sym *sym;
9729 Elf64_Rel *rel;
9730 int i, nrels;
9731
9732 btf = obj->btf;
9733 nrels = shdr->sh_size / shdr->sh_entsize;
9734 for (i = 0; i < nrels; i++) {
9735 rel = elf_rel_by_idx(data, i);
9736 if (!rel) {
9737 pr_warn("struct_ops reloc: failed to get %d reloc\n", i);
9738 return -LIBBPF_ERRNO__FORMAT;
9739 }
9740
9741 sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info));
9742 if (!sym) {
9743 pr_warn("struct_ops reloc: symbol %zx not found\n",
9744 (size_t)ELF64_R_SYM(rel->r_info));
9745 return -LIBBPF_ERRNO__FORMAT;
9746 }
9747
9748 name = elf_sym_str(obj, sym->st_name) ?: "<?>";
9749 map = find_struct_ops_map_by_offset(obj, shdr->sh_info, rel->r_offset);
9750 if (!map) {
9751 pr_warn("struct_ops reloc: cannot find map at rel->r_offset %zu\n",
9752 (size_t)rel->r_offset);
9753 return -EINVAL;
9754 }
9755
9756 moff = rel->r_offset - map->sec_offset;
9757 shdr_idx = sym->st_shndx;
9758 st_ops = map->st_ops;
9759 pr_debug("struct_ops reloc %s: for %lld value %lld shdr_idx %u rel->r_offset %zu map->sec_offset %zu name %d (\'%s\')\n",
9760 map->name,
9761 (long long)(rel->r_info >> 32),
9762 (long long)sym->st_value,
9763 shdr_idx, (size_t)rel->r_offset,
9764 map->sec_offset, sym->st_name, name);
9765
9766 if (shdr_idx >= SHN_LORESERVE) {
9767 pr_warn("struct_ops reloc %s: rel->r_offset %zu shdr_idx %u unsupported non-static function\n",
9768 map->name, (size_t)rel->r_offset, shdr_idx);
9769 return -LIBBPF_ERRNO__RELOC;
9770 }
9771 if (sym->st_value % BPF_INSN_SZ) {
9772 pr_warn("struct_ops reloc %s: invalid target program offset %llu\n",
9773 map->name, (unsigned long long)sym->st_value);
9774 return -LIBBPF_ERRNO__FORMAT;
9775 }
9776 insn_idx = sym->st_value / BPF_INSN_SZ;
9777
9778 type = btf__type_by_id(btf, st_ops->type_id);
9779 member = find_member_by_offset(type, moff * 8);
9780 if (!member) {
9781 pr_warn("struct_ops reloc %s: cannot find member at moff %u\n",
9782 map->name, moff);
9783 return -EINVAL;
9784 }
9785 member_idx = member - btf_members(type);
9786 name = btf__name_by_offset(btf, member->name_off);
9787
9788 if (!resolve_func_ptr(btf, member->type, NULL)) {
9789 pr_warn("struct_ops reloc %s: cannot relocate non func ptr %s\n",
9790 map->name, name);
9791 return -EINVAL;
9792 }
9793
9794 prog = find_prog_by_sec_insn(obj, shdr_idx, insn_idx);
9795 if (!prog) {
9796 pr_warn("struct_ops reloc %s: cannot find prog at shdr_idx %u to relocate func ptr %s\n",
9797 map->name, shdr_idx, name);
9798 return -EINVAL;
9799 }
9800
9801 /* prevent the use of BPF prog with invalid type */
9802 if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) {
9803 pr_warn("struct_ops reloc %s: prog %s is not struct_ops BPF program\n",
9804 map->name, prog->name);
9805 return -EINVAL;
9806 }
9807
9808 st_ops->progs[member_idx] = prog;
9809
9810 /* st_ops->data will be exposed to users, being returned by
9811 * bpf_map__initial_value() as a pointer to the shadow
9812 * type. All function pointers in the original struct type
9813 * should be converted to a pointer to struct bpf_program
9814 * in the shadow type.
9815 */
9816 *((struct bpf_program **)(st_ops->data + moff)) = prog;
9817 }
9818
9819 return 0;
9820 }
9821
9822 #define BTF_TRACE_PREFIX "btf_trace_"
9823 #define BTF_LSM_PREFIX "bpf_lsm_"
9824 #define BTF_ITER_PREFIX "bpf_iter_"
9825 #define BTF_MAX_NAME_SIZE 128
9826
9827 void btf_get_kernel_prefix_kind(enum bpf_attach_type attach_type,
9828 const char **prefix, int *kind)
9829 {
9830 switch (attach_type) {
9831 case BPF_TRACE_RAW_TP:
9832 *prefix = BTF_TRACE_PREFIX;
9833 *kind = BTF_KIND_TYPEDEF;
9834 break;
9835 case BPF_LSM_MAC:
9836 case BPF_LSM_CGROUP:
9837 *prefix = BTF_LSM_PREFIX;
9838 *kind = BTF_KIND_FUNC;
9839 break;
9840 case BPF_TRACE_ITER:
9841 *prefix = BTF_ITER_PREFIX;
9842 *kind = BTF_KIND_FUNC;
9843 break;
9844 default:
9845 *prefix = "";
9846 *kind = BTF_KIND_FUNC;
9847 }
9848 }
9849
9850 static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix,
9851 const char *name, __u32 kind)
9852 {
9853 char btf_type_name[BTF_MAX_NAME_SIZE];
9854 int ret;
9855
9856 ret = snprintf(btf_type_name, sizeof(btf_type_name),
9857 "%s%s", prefix, name);
9858 /* snprintf returns the number of characters written excluding the
9859 * terminating null. So, if >= BTF_MAX_NAME_SIZE are written, it
9860 * indicates truncation.
9861 */
9862 if (ret < 0 || ret >= sizeof(btf_type_name))
9863 return -ENAMETOOLONG;
9864 return btf__find_by_name_kind(btf, btf_type_name, kind);
9865 }
9866
9867 static inline int find_attach_btf_id(struct btf *btf, const char *name,
9868 enum bpf_attach_type attach_type)
9869 {
9870 const char *prefix;
9871 int kind;
9872
9873 btf_get_kernel_prefix_kind(attach_type, &prefix, &kind);
9874 return find_btf_by_prefix_kind(btf, prefix, name, kind);
9875 }
9876
9877 int libbpf_find_vmlinux_btf_id(const char *name,
9878 enum bpf_attach_type attach_type)
9879 {
9880 struct btf *btf;
9881 int err;
9882
9883 btf = btf__load_vmlinux_btf();
9884 err = libbpf_get_error(btf);
9885 if (err) {
9886 pr_warn("vmlinux BTF is not found\n");
9887 return libbpf_err(err);
9888 }
9889
9890 err = find_attach_btf_id(btf, name, attach_type);
9891 if (err <= 0)
9892 pr_warn("%s is not found in vmlinux BTF\n", name);
9893
9894 btf__free(btf);
9895 return libbpf_err(err);
9896 }
9897
9898 static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd)
9899 {
9900 struct bpf_prog_info info;
9901 __u32 info_len = sizeof(info);
9902 struct btf *btf;
9903 int err;
9904
9905 memset(&info, 0, info_len);
9906 err = bpf_prog_get_info_by_fd(attach_prog_fd, &info, &info_len);
9907 if (err) {
9908 pr_warn("failed bpf_prog_get_info_by_fd for FD %d: %d\n",
9909 attach_prog_fd, err);
9910 return err;
9911 }
9912
9913 err = -EINVAL;
9914 if (!info.btf_id) {
9915 pr_warn("The target program doesn't have BTF\n");
9916 goto out;
9917 }
9918 btf = btf__load_from_kernel_by_id(info.btf_id);
9919 err = libbpf_get_error(btf);
9920 if (err) {
9921 pr_warn("Failed to get BTF %d of the program: %d\n", info.btf_id, err);
9922 goto out;
9923 }
9924 err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC);
9925 btf__free(btf);
9926 if (err <= 0) {
9927 pr_warn("%s is not found in prog's BTF\n", name);
9928 goto out;
9929 }
9930 out:
9931 return err;
9932 }
9933
9934 static int find_kernel_btf_id(struct bpf_object *obj, const char *attach_name,
9935 enum bpf_attach_type attach_type,
9936 int *btf_obj_fd, int *btf_type_id)
9937 {
9938 int ret, i, mod_len;
9939 const char *fn_name, *mod_name = NULL;
9940
9941 fn_name = strchr(attach_name, ':');
9942 if (fn_name) {
9943 mod_name = attach_name;
9944 mod_len = fn_name - mod_name;
9945 fn_name++;
9946 }
9947
9948 if (!mod_name || strncmp(mod_name, "vmlinux", mod_len) == 0) {
9949 ret = find_attach_btf_id(obj->btf_vmlinux,
9950 mod_name ? fn_name : attach_name,
9951 attach_type);
9952 if (ret > 0) {
9953 *btf_obj_fd = 0; /* vmlinux BTF */
9954 *btf_type_id = ret;
9955 return 0;
9956 }
9957 if (ret != -ENOENT)
9958 return ret;
9959 }
9960
9961 ret = load_module_btfs(obj);
9962 if (ret)
9963 return ret;
9964
9965 for (i = 0; i < obj->btf_module_cnt; i++) {
9966 const struct module_btf *mod = &obj->btf_modules[i];
9967
9968 if (mod_name && strncmp(mod->name, mod_name, mod_len) != 0)
9969 continue;
9970
9971 ret = find_attach_btf_id(mod->btf,
9972 mod_name ? fn_name : attach_name,
9973 attach_type);
9974 if (ret > 0) {
9975 *btf_obj_fd = mod->fd;
9976 *btf_type_id = ret;
9977 return 0;
9978 }
9979 if (ret == -ENOENT)
9980 continue;
9981
9982 return ret;
9983 }
9984
9985 return -ESRCH;
9986 }
9987
9988 static int libbpf_find_attach_btf_id(struct bpf_program *prog, const char *attach_name,
9989 int *btf_obj_fd, int *btf_type_id)
9990 {
9991 enum bpf_attach_type attach_type = prog->expected_attach_type;
9992 __u32 attach_prog_fd = prog->attach_prog_fd;
9993 int err = 0;
9994
9995 /* BPF program's BTF ID */
9996 if (prog->type == BPF_PROG_TYPE_EXT || attach_prog_fd) {
9997 if (!attach_prog_fd) {
9998 pr_warn("prog '%s': attach program FD is not set\n", prog->name);
9999 return -EINVAL;
10000 }
10001 err = libbpf_find_prog_btf_id(attach_name, attach_prog_fd);
10002 if (err < 0) {
10003 pr_warn("prog '%s': failed to find BPF program (FD %d) BTF ID for '%s': %d\n",
10004 prog->name, attach_prog_fd, attach_name, err);
10005 return err;
10006 }
10007 *btf_obj_fd = 0;
10008 *btf_type_id = err;
10009 return 0;
10010 }
10011
10012 /* kernel/module BTF ID */
10013 if (prog->obj->gen_loader) {
10014 bpf_gen__record_attach_target(prog->obj->gen_loader, attach_name, attach_type);
10015 *btf_obj_fd = 0;
10016 *btf_type_id = 1;
10017 } else {
10018 err = find_kernel_btf_id(prog->obj, attach_name,
10019 attach_type, btf_obj_fd,
10020 btf_type_id);
10021 }
10022 if (err) {
10023 pr_warn("prog '%s': failed to find kernel BTF type ID of '%s': %d\n",
10024 prog->name, attach_name, err);
10025 return err;
10026 }
10027 return 0;
10028 }
10029
10030 int libbpf_attach_type_by_name(const char *name,
10031 enum bpf_attach_type *attach_type)
10032 {
10033 char *type_names;
10034 const struct bpf_sec_def *sec_def;
10035
10036 if (!name)
10037 return libbpf_err(-EINVAL);
10038
10039 sec_def = find_sec_def(name);
10040 if (!sec_def) {
10041 pr_debug("failed to guess attach type based on ELF section name '%s'\n", name);
10042 type_names = libbpf_get_type_names(true);
10043 if (type_names != NULL) {
10044 pr_debug("attachable section(type) names are:%s\n", type_names);
10045 free(type_names);
10046 }
10047
10048 return libbpf_err(-EINVAL);
10049 }
10050
10051 if (sec_def->prog_prepare_load_fn != libbpf_prepare_prog_load)
10052 return libbpf_err(-EINVAL);
10053 if (!(sec_def->cookie & SEC_ATTACHABLE))
10054 return libbpf_err(-EINVAL);
10055
10056 *attach_type = sec_def->expected_attach_type;
10057 return 0;
10058 }
10059
10060 int bpf_map__fd(const struct bpf_map *map)
10061 {
10062 if (!map)
10063 return libbpf_err(-EINVAL);
10064 if (!map_is_created(map))
10065 return -1;
10066 return map->fd;
10067 }
10068
10069 static bool map_uses_real_name(const struct bpf_map *map)
10070 {
10071 /* Since libbpf started to support custom .data.* and .rodata.* maps,
10072 * their user-visible name differs from kernel-visible name. Users see
10073 * such map's corresponding ELF section name as a map name.
10074 * This check distinguishes .data/.rodata from .data.* and .rodata.*
10075 * maps to know which name has to be returned to the user.
10076 */
10077 if (map->libbpf_type == LIBBPF_MAP_DATA && strcmp(map->real_name, DATA_SEC) != 0)
10078 return true;
10079 if (map->libbpf_type == LIBBPF_MAP_RODATA && strcmp(map->real_name, RODATA_SEC) != 0)
10080 return true;
10081 return false;
10082 }
10083
10084 const char *bpf_map__name(const struct bpf_map *map)
10085 {
10086 if (!map)
10087 return NULL;
10088
10089 if (map_uses_real_name(map))
10090 return map->real_name;
10091
10092 return map->name;
10093 }
10094
10095 enum bpf_map_type bpf_map__type(const struct bpf_map *map)
10096 {
10097 return map->def.type;
10098 }
10099
10100 int bpf_map__set_type(struct bpf_map *map, enum bpf_map_type type)
10101 {
10102 if (map_is_created(map))
10103 return libbpf_err(-EBUSY);
10104 map->def.type = type;
10105 return 0;
10106 }
10107
10108 __u32 bpf_map__map_flags(const struct bpf_map *map)
10109 {
10110 return map->def.map_flags;
10111 }
10112
10113 int bpf_map__set_map_flags(struct bpf_map *map, __u32 flags)
10114 {
10115 if (map_is_created(map))
10116 return libbpf_err(-EBUSY);
10117 map->def.map_flags = flags;
10118 return 0;
10119 }
10120
10121 __u64 bpf_map__map_extra(const struct bpf_map *map)
10122 {
10123 return map->map_extra;
10124 }
10125
10126 int bpf_map__set_map_extra(struct bpf_map *map, __u64 map_extra)
10127 {
10128 if (map_is_created(map))
10129 return libbpf_err(-EBUSY);
10130 map->map_extra = map_extra;
10131 return 0;
10132 }
10133
10134 __u32 bpf_map__numa_node(const struct bpf_map *map)
10135 {
10136 return map->numa_node;
10137 }
10138
10139 int bpf_map__set_numa_node(struct bpf_map *map, __u32 numa_node)
10140 {
10141 if (map_is_created(map))
10142 return libbpf_err(-EBUSY);
10143 map->numa_node = numa_node;
10144 return 0;
10145 }
10146
10147 __u32 bpf_map__key_size(const struct bpf_map *map)
10148 {
10149 return map->def.key_size;
10150 }
10151
10152 int bpf_map__set_key_size(struct bpf_map *map, __u32 size)
10153 {
10154 if (map_is_created(map))
10155 return libbpf_err(-EBUSY);
10156 map->def.key_size = size;
10157 return 0;
10158 }
10159
10160 __u32 bpf_map__value_size(const struct bpf_map *map)
10161 {
10162 return map->def.value_size;
10163 }
10164
10165 static int map_btf_datasec_resize(struct bpf_map *map, __u32 size)
10166 {
10167 struct btf *btf;
10168 struct btf_type *datasec_type, *var_type;
10169 struct btf_var_secinfo *var;
10170 const struct btf_type *array_type;
10171 const struct btf_array *array;
10172 int vlen, element_sz, new_array_id;
10173 __u32 nr_elements;
10174
10175 /* check btf existence */
10176 btf = bpf_object__btf(map->obj);
10177 if (!btf)
10178 return -ENOENT;
10179
10180 /* verify map is datasec */
10181 datasec_type = btf_type_by_id(btf, bpf_map__btf_value_type_id(map));
10182 if (!btf_is_datasec(datasec_type)) {
10183 pr_warn("map '%s': cannot be resized, map value type is not a datasec\n",
10184 bpf_map__name(map));
10185 return -EINVAL;
10186 }
10187
10188 /* verify datasec has at least one var */
10189 vlen = btf_vlen(datasec_type);
10190 if (vlen == 0) {
10191 pr_warn("map '%s': cannot be resized, map value datasec is empty\n",
10192 bpf_map__name(map));
10193 return -EINVAL;
10194 }
10195
10196 /* verify last var in the datasec is an array */
10197 var = &btf_var_secinfos(datasec_type)[vlen - 1];
10198 var_type = btf_type_by_id(btf, var->type);
10199 array_type = skip_mods_and_typedefs(btf, var_type->type, NULL);
10200 if (!btf_is_array(array_type)) {
10201 pr_warn("map '%s': cannot be resized, last var must be an array\n",
10202 bpf_map__name(map));
10203 return -EINVAL;
10204 }
10205
10206 /* verify request size aligns with array */
10207 array = btf_array(array_type);
10208 element_sz = btf__resolve_size(btf, array->type);
10209 if (element_sz <= 0 || (size - var->offset) % element_sz != 0) {
10210 pr_warn("map '%s': cannot be resized, element size (%d) doesn't align with new total size (%u)\n",
10211 bpf_map__name(map), element_sz, size);
10212 return -EINVAL;
10213 }
10214
10215 /* create a new array based on the existing array, but with new length */
10216 nr_elements = (size - var->offset) / element_sz;
10217 new_array_id = btf__add_array(btf, array->index_type, array->type, nr_elements);
10218 if (new_array_id < 0)
10219 return new_array_id;
10220
10221 /* adding a new btf type invalidates existing pointers to btf objects,
10222 * so refresh pointers before proceeding
10223 */
10224 datasec_type = btf_type_by_id(btf, map->btf_value_type_id);
10225 var = &btf_var_secinfos(datasec_type)[vlen - 1];
10226 var_type = btf_type_by_id(btf, var->type);
10227
10228 /* finally update btf info */
10229 datasec_type->size = size;
10230 var->size = size - var->offset;
10231 var_type->type = new_array_id;
10232
10233 return 0;
10234 }
10235
10236 int bpf_map__set_value_size(struct bpf_map *map, __u32 size)
10237 {
10238 if (map->obj->loaded || map->reused)
10239 return libbpf_err(-EBUSY);
10240
10241 if (map->mmaped) {
10242 size_t mmap_old_sz, mmap_new_sz;
10243 int err;
10244
10245 if (map->def.type != BPF_MAP_TYPE_ARRAY)
10246 return -EOPNOTSUPP;
10247
10248 mmap_old_sz = bpf_map_mmap_sz(map);
10249 mmap_new_sz = array_map_mmap_sz(size, map->def.max_entries);
10250 err = bpf_map_mmap_resize(map, mmap_old_sz, mmap_new_sz);
10251 if (err) {
10252 pr_warn("map '%s': failed to resize memory-mapped region: %d\n",
10253 bpf_map__name(map), err);
10254 return err;
10255 }
10256 err = map_btf_datasec_resize(map, size);
10257 if (err && err != -ENOENT) {
10258 pr_warn("map '%s': failed to adjust resized BTF, clearing BTF key/value info: %d\n",
10259 bpf_map__name(map), err);
10260 map->btf_value_type_id = 0;
10261 map->btf_key_type_id = 0;
10262 }
10263 }
10264
10265 map->def.value_size = size;
10266 return 0;
10267 }
10268
10269 __u32 bpf_map__btf_key_type_id(const struct bpf_map *map)
10270 {
10271 return map ? map->btf_key_type_id : 0;
10272 }
10273
10274 __u32 bpf_map__btf_value_type_id(const struct bpf_map *map)
10275 {
10276 return map ? map->btf_value_type_id : 0;
10277 }
10278
10279 int bpf_map__set_initial_value(struct bpf_map *map,
10280 const void *data, size_t size)
10281 {
10282 size_t actual_sz;
10283
10284 if (map->obj->loaded || map->reused)
10285 return libbpf_err(-EBUSY);
10286
10287 if (!map->mmaped || map->libbpf_type == LIBBPF_MAP_KCONFIG)
10288 return libbpf_err(-EINVAL);
10289
10290 if (map->def.type == BPF_MAP_TYPE_ARENA)
10291 actual_sz = map->obj->arena_data_sz;
10292 else
10293 actual_sz = map->def.value_size;
10294 if (size != actual_sz)
10295 return libbpf_err(-EINVAL);
10296
10297 memcpy(map->mmaped, data, size);
10298 return 0;
10299 }
10300
10301 void *bpf_map__initial_value(const struct bpf_map *map, size_t *psize)
10302 {
10303 if (bpf_map__is_struct_ops(map)) {
10304 if (psize)
10305 *psize = map->def.value_size;
10306 return map->st_ops->data;
10307 }
10308
10309 if (!map->mmaped)
10310 return NULL;
10311
10312 if (map->def.type == BPF_MAP_TYPE_ARENA)
10313 *psize = map->obj->arena_data_sz;
10314 else
10315 *psize = map->def.value_size;
10316
10317 return map->mmaped;
10318 }
10319
10320 bool bpf_map__is_internal(const struct bpf_map *map)
10321 {
10322 return map->libbpf_type != LIBBPF_MAP_UNSPEC;
10323 }
10324
10325 __u32 bpf_map__ifindex(const struct bpf_map *map)
10326 {
10327 return map->map_ifindex;
10328 }
10329
10330 int bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex)
10331 {
10332 if (map_is_created(map))
10333 return libbpf_err(-EBUSY);
10334 map->map_ifindex = ifindex;
10335 return 0;
10336 }
10337
10338 int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd)
10339 {
10340 if (!bpf_map_type__is_map_in_map(map->def.type)) {
10341 pr_warn("error: unsupported map type\n");
10342 return libbpf_err(-EINVAL);
10343 }
10344 if (map->inner_map_fd != -1) {
10345 pr_warn("error: inner_map_fd already specified\n");
10346 return libbpf_err(-EINVAL);
10347 }
10348 if (map->inner_map) {
10349 bpf_map__destroy(map->inner_map);
10350 zfree(&map->inner_map);
10351 }
10352 map->inner_map_fd = fd;
10353 return 0;
10354 }
10355
10356 static struct bpf_map *
10357 __bpf_map__iter(const struct bpf_map *m, const struct bpf_object *obj, int i)
10358 {
10359 ssize_t idx;
10360 struct bpf_map *s, *e;
10361
10362 if (!obj || !obj->maps)
10363 return errno = EINVAL, NULL;
10364
10365 s = obj->maps;
10366 e = obj->maps + obj->nr_maps;
10367
10368 if ((m < s) || (m >= e)) {
10369 pr_warn("error in %s: map handler doesn't belong to object\n",
10370 __func__);
10371 return errno = EINVAL, NULL;
10372 }
10373
10374 idx = (m - obj->maps) + i;
10375 if (idx >= obj->nr_maps || idx < 0)
10376 return NULL;
10377 return &obj->maps[idx];
10378 }
10379
10380 struct bpf_map *
10381 bpf_object__next_map(const struct bpf_object *obj, const struct bpf_map *prev)
10382 {
10383 if (prev == NULL && obj != NULL)
10384 return obj->maps;
10385
10386 return __bpf_map__iter(prev, obj, 1);
10387 }
10388
10389 struct bpf_map *
10390 bpf_object__prev_map(const struct bpf_object *obj, const struct bpf_map *next)
10391 {
10392 if (next == NULL && obj != NULL) {
10393 if (!obj->nr_maps)
10394 return NULL;
10395 return obj->maps + obj->nr_maps - 1;
10396 }
10397
10398 return __bpf_map__iter(next, obj, -1);
10399 }
10400
10401 struct bpf_map *
10402 bpf_object__find_map_by_name(const struct bpf_object *obj, const char *name)
10403 {
10404 struct bpf_map *pos;
10405
10406 bpf_object__for_each_map(pos, obj) {
10407 /* if it's a special internal map name (which always starts
10408 * with dot) then check if that special name matches the
10409 * real map name (ELF section name)
10410 */
10411 if (name[0] == '.') {
10412 if (pos->real_name && strcmp(pos->real_name, name) == 0)
10413 return pos;
10414 continue;
10415 }
10416 /* otherwise map name has to be an exact match */
10417 if (map_uses_real_name(pos)) {
10418 if (strcmp(pos->real_name, name) == 0)
10419 return pos;
10420 continue;
10421 }
10422 if (strcmp(pos->name, name) == 0)
10423 return pos;
10424 }
10425 return errno = ENOENT, NULL;
10426 }
10427
10428 int
10429 bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name)
10430 {
10431 return bpf_map__fd(bpf_object__find_map_by_name(obj, name));
10432 }
10433
10434 static int validate_map_op(const struct bpf_map *map, size_t key_sz,
10435 size_t value_sz, bool check_value_sz)
10436 {
10437 if (!map_is_created(map)) /* map is not yet created */
10438 return -ENOENT;
10439
10440 if (map->def.key_size != key_sz) {
10441 pr_warn("map '%s': unexpected key size %zu provided, expected %u\n",
10442 map->name, key_sz, map->def.key_size);
10443 return -EINVAL;
10444 }
10445
10446 if (map->fd < 0) {
10447 pr_warn("map '%s': can't use BPF map without FD (was it created?)\n", map->name);
10448 return -EINVAL;
10449 }
10450
10451 if (!check_value_sz)
10452 return 0;
10453
10454 switch (map->def.type) {
10455 case BPF_MAP_TYPE_PERCPU_ARRAY:
10456 case BPF_MAP_TYPE_PERCPU_HASH:
10457 case BPF_MAP_TYPE_LRU_PERCPU_HASH:
10458 case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE: {
10459 int num_cpu = libbpf_num_possible_cpus();
10460 size_t elem_sz = roundup(map->def.value_size, 8);
10461
10462 if (value_sz != num_cpu * elem_sz) {
10463 pr_warn("map '%s': unexpected value size %zu provided for per-CPU map, expected %d * %zu = %zd\n",
10464 map->name, value_sz, num_cpu, elem_sz, num_cpu * elem_sz);
10465 return -EINVAL;
10466 }
10467 break;
10468 }
10469 default:
10470 if (map->def.value_size != value_sz) {
10471 pr_warn("map '%s': unexpected value size %zu provided, expected %u\n",
10472 map->name, value_sz, map->def.value_size);
10473 return -EINVAL;
10474 }
10475 break;
10476 }
10477 return 0;
10478 }
10479
10480 int bpf_map__lookup_elem(const struct bpf_map *map,
10481 const void *key, size_t key_sz,
10482 void *value, size_t value_sz, __u64 flags)
10483 {
10484 int err;
10485
10486 err = validate_map_op(map, key_sz, value_sz, true);
10487 if (err)
10488 return libbpf_err(err);
10489
10490 return bpf_map_lookup_elem_flags(map->fd, key, value, flags);
10491 }
10492
10493 int bpf_map__update_elem(const struct bpf_map *map,
10494 const void *key, size_t key_sz,
10495 const void *value, size_t value_sz, __u64 flags)
10496 {
10497 int err;
10498
10499 err = validate_map_op(map, key_sz, value_sz, true);
10500 if (err)
10501 return libbpf_err(err);
10502
10503 return bpf_map_update_elem(map->fd, key, value, flags);
10504 }
10505
10506 int bpf_map__delete_elem(const struct bpf_map *map,
10507 const void *key, size_t key_sz, __u64 flags)
10508 {
10509 int err;
10510
10511 err = validate_map_op(map, key_sz, 0, false /* check_value_sz */);
10512 if (err)
10513 return libbpf_err(err);
10514
10515 return bpf_map_delete_elem_flags(map->fd, key, flags);
10516 }
10517
10518 int bpf_map__lookup_and_delete_elem(const struct bpf_map *map,
10519 const void *key, size_t key_sz,
10520 void *value, size_t value_sz, __u64 flags)
10521 {
10522 int err;
10523
10524 err = validate_map_op(map, key_sz, value_sz, true);
10525 if (err)
10526 return libbpf_err(err);
10527
10528 return bpf_map_lookup_and_delete_elem_flags(map->fd, key, value, flags);
10529 }
10530
10531 int bpf_map__get_next_key(const struct bpf_map *map,
10532 const void *cur_key, void *next_key, size_t key_sz)
10533 {
10534 int err;
10535
10536 err = validate_map_op(map, key_sz, 0, false /* check_value_sz */);
10537 if (err)
10538 return libbpf_err(err);
10539
10540 return bpf_map_get_next_key(map->fd, cur_key, next_key);
10541 }
10542
10543 long libbpf_get_error(const void *ptr)
10544 {
10545 if (!IS_ERR_OR_NULL(ptr))
10546 return 0;
10547
10548 if (IS_ERR(ptr))
10549 errno = -PTR_ERR(ptr);
10550
10551 /* If ptr == NULL, then errno should be already set by the failing
10552 * API, because libbpf never returns NULL on success and it now always
10553 * sets errno on error. So no extra errno handling for ptr == NULL
10554 * case.
10555 */
10556 return -errno;
10557 }
10558
10559 /* Replace link's underlying BPF program with the new one */
10560 int bpf_link__update_program(struct bpf_link *link, struct bpf_program *prog)
10561 {
10562 int ret;
10563 int prog_fd = bpf_program__fd(prog);
10564
10565 if (prog_fd < 0) {
10566 pr_warn("prog '%s': can't use BPF program without FD (was it loaded?)\n",
10567 prog->name);
10568 return libbpf_err(-EINVAL);
10569 }
10570
10571 ret = bpf_link_update(bpf_link__fd(link), prog_fd, NULL);
10572 return libbpf_err_errno(ret);
10573 }
10574
10575 /* Release "ownership" of underlying BPF resource (typically, BPF program
10576 * attached to some BPF hook, e.g., tracepoint, kprobe, etc). Disconnected
10577 * link, when destructed through bpf_link__destroy() call won't attempt to
10578 * detach/unregisted that BPF resource. This is useful in situations where,
10579 * say, attached BPF program has to outlive userspace program that attached it
10580 * in the system. Depending on type of BPF program, though, there might be
10581 * additional steps (like pinning BPF program in BPF FS) necessary to ensure
10582 * exit of userspace program doesn't trigger automatic detachment and clean up
10583 * inside the kernel.
10584 */
10585 void bpf_link__disconnect(struct bpf_link *link)
10586 {
10587 link->disconnected = true;
10588 }
10589
10590 int bpf_link__destroy(struct bpf_link *link)
10591 {
10592 int err = 0;
10593
10594 if (IS_ERR_OR_NULL(link))
10595 return 0;
10596
10597 if (!link->disconnected && link->detach)
10598 err = link->detach(link);
10599 if (link->pin_path)
10600 free(link->pin_path);
10601 if (link->dealloc)
10602 link->dealloc(link);
10603 else
10604 free(link);
10605
10606 return libbpf_err(err);
10607 }
10608
10609 int bpf_link__fd(const struct bpf_link *link)
10610 {
10611 return link->fd;
10612 }
10613
10614 const char *bpf_link__pin_path(const struct bpf_link *link)
10615 {
10616 return link->pin_path;
10617 }
10618
10619 static int bpf_link__detach_fd(struct bpf_link *link)
10620 {
10621 return libbpf_err_errno(close(link->fd));
10622 }
10623
10624 struct bpf_link *bpf_link__open(const char *path)
10625 {
10626 struct bpf_link *link;
10627 int fd;
10628
10629 fd = bpf_obj_get(path);
10630 if (fd < 0) {
10631 fd = -errno;
10632 pr_warn("failed to open link at %s: %d\n", path, fd);
10633 return libbpf_err_ptr(fd);
10634 }
10635
10636 link = calloc(1, sizeof(*link));
10637 if (!link) {
10638 close(fd);
10639 return libbpf_err_ptr(-ENOMEM);
10640 }
10641 link->detach = &bpf_link__detach_fd;
10642 link->fd = fd;
10643
10644 link->pin_path = strdup(path);
10645 if (!link->pin_path) {
10646 bpf_link__destroy(link);
10647 return libbpf_err_ptr(-ENOMEM);
10648 }
10649
10650 return link;
10651 }
10652
10653 int bpf_link__detach(struct bpf_link *link)
10654 {
10655 return bpf_link_detach(link->fd) ? -errno : 0;
10656 }
10657
10658 int bpf_link__pin(struct bpf_link *link, const char *path)
10659 {
10660 int err;
10661
10662 if (link->pin_path)
10663 return libbpf_err(-EBUSY);
10664 err = make_parent_dir(path);
10665 if (err)
10666 return libbpf_err(err);
10667 err = check_path(path);
10668 if (err)
10669 return libbpf_err(err);
10670
10671 link->pin_path = strdup(path);
10672 if (!link->pin_path)
10673 return libbpf_err(-ENOMEM);
10674
10675 if (bpf_obj_pin(link->fd, link->pin_path)) {
10676 err = -errno;
10677 zfree(&link->pin_path);
10678 return libbpf_err(err);
10679 }
10680
10681 pr_debug("link fd=%d: pinned at %s\n", link->fd, link->pin_path);
10682 return 0;
10683 }
10684
10685 int bpf_link__unpin(struct bpf_link *link)
10686 {
10687 int err;
10688
10689 if (!link->pin_path)
10690 return libbpf_err(-EINVAL);
10691
10692 err = unlink(link->pin_path);
10693 if (err != 0)
10694 return -errno;
10695
10696 pr_debug("link fd=%d: unpinned from %s\n", link->fd, link->pin_path);
10697 zfree(&link->pin_path);
10698 return 0;
10699 }
10700
10701 struct bpf_link_perf {
10702 struct bpf_link link;
10703 int perf_event_fd;
10704 /* legacy kprobe support: keep track of probe identifier and type */
10705 char *legacy_probe_name;
10706 bool legacy_is_kprobe;
10707 bool legacy_is_retprobe;
10708 };
10709
10710 static int remove_kprobe_event_legacy(const char *probe_name, bool retprobe);
10711 static int remove_uprobe_event_legacy(const char *probe_name, bool retprobe);
10712
10713 static int bpf_link_perf_detach(struct bpf_link *link)
10714 {
10715 struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link);
10716 int err = 0;
10717
10718 if (ioctl(perf_link->perf_event_fd, PERF_EVENT_IOC_DISABLE, 0) < 0)
10719 err = -errno;
10720
10721 if (perf_link->perf_event_fd != link->fd)
10722 close(perf_link->perf_event_fd);
10723 close(link->fd);
10724
10725 /* legacy uprobe/kprobe needs to be removed after perf event fd closure */
10726 if (perf_link->legacy_probe_name) {
10727 if (perf_link->legacy_is_kprobe) {
10728 err = remove_kprobe_event_legacy(perf_link->legacy_probe_name,
10729 perf_link->legacy_is_retprobe);
10730 } else {
10731 err = remove_uprobe_event_legacy(perf_link->legacy_probe_name,
10732 perf_link->legacy_is_retprobe);
10733 }
10734 }
10735
10736 return err;
10737 }
10738
10739 static void bpf_link_perf_dealloc(struct bpf_link *link)
10740 {
10741 struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link);
10742
10743 free(perf_link->legacy_probe_name);
10744 free(perf_link);
10745 }
10746
10747 struct bpf_link *bpf_program__attach_perf_event_opts(const struct bpf_program *prog, int pfd,
10748 const struct bpf_perf_event_opts *opts)
10749 {
10750 char errmsg[STRERR_BUFSIZE];
10751 struct bpf_link_perf *link;
10752 int prog_fd, link_fd = -1, err;
10753 bool force_ioctl_attach;
10754
10755 if (!OPTS_VALID(opts, bpf_perf_event_opts))
10756 return libbpf_err_ptr(-EINVAL);
10757
10758 if (pfd < 0) {
10759 pr_warn("prog '%s': invalid perf event FD %d\n",
10760 prog->name, pfd);
10761 return libbpf_err_ptr(-EINVAL);
10762 }
10763 prog_fd = bpf_program__fd(prog);
10764 if (prog_fd < 0) {
10765 pr_warn("prog '%s': can't attach BPF program without FD (was it loaded?)\n",
10766 prog->name);
10767 return libbpf_err_ptr(-EINVAL);
10768 }
10769
10770 link = calloc(1, sizeof(*link));
10771 if (!link)
10772 return libbpf_err_ptr(-ENOMEM);
10773 link->link.detach = &bpf_link_perf_detach;
10774 link->link.dealloc = &bpf_link_perf_dealloc;
10775 link->perf_event_fd = pfd;
10776
10777 force_ioctl_attach = OPTS_GET(opts, force_ioctl_attach, false);
10778 if (kernel_supports(prog->obj, FEAT_PERF_LINK) && !force_ioctl_attach) {
10779 DECLARE_LIBBPF_OPTS(bpf_link_create_opts, link_opts,
10780 .perf_event.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0));
10781
10782 link_fd = bpf_link_create(prog_fd, pfd, BPF_PERF_EVENT, &link_opts);
10783 if (link_fd < 0) {
10784 err = -errno;
10785 pr_warn("prog '%s': failed to create BPF link for perf_event FD %d: %d (%s)\n",
10786 prog->name, pfd,
10787 err, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10788 goto err_out;
10789 }
10790 link->link.fd = link_fd;
10791 } else {
10792 if (OPTS_GET(opts, bpf_cookie, 0)) {
10793 pr_warn("prog '%s': user context value is not supported\n", prog->name);
10794 err = -EOPNOTSUPP;
10795 goto err_out;
10796 }
10797
10798 if (ioctl(pfd, PERF_EVENT_IOC_SET_BPF, prog_fd) < 0) {
10799 err = -errno;
10800 pr_warn("prog '%s': failed to attach to perf_event FD %d: %s\n",
10801 prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10802 if (err == -EPROTO)
10803 pr_warn("prog '%s': try add PERF_SAMPLE_CALLCHAIN to or remove exclude_callchain_[kernel|user] from pfd %d\n",
10804 prog->name, pfd);
10805 goto err_out;
10806 }
10807 link->link.fd = pfd;
10808 }
10809 if (ioctl(pfd, PERF_EVENT_IOC_ENABLE, 0) < 0) {
10810 err = -errno;
10811 pr_warn("prog '%s': failed to enable perf_event FD %d: %s\n",
10812 prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10813 goto err_out;
10814 }
10815
10816 return &link->link;
10817 err_out:
10818 if (link_fd >= 0)
10819 close(link_fd);
10820 free(link);
10821 return libbpf_err_ptr(err);
10822 }
10823
10824 struct bpf_link *bpf_program__attach_perf_event(const struct bpf_program *prog, int pfd)
10825 {
10826 return bpf_program__attach_perf_event_opts(prog, pfd, NULL);
10827 }
10828
10829 /*
10830 * this function is expected to parse integer in the range of [0, 2^31-1] from
10831 * given file using scanf format string fmt. If actual parsed value is
10832 * negative, the result might be indistinguishable from error
10833 */
10834 static int parse_uint_from_file(const char *file, const char *fmt)
10835 {
10836 char buf[STRERR_BUFSIZE];
10837 int err, ret;
10838 FILE *f;
10839
10840 f = fopen(file, "re");
10841 if (!f) {
10842 err = -errno;
10843 pr_debug("failed to open '%s': %s\n", file,
10844 libbpf_strerror_r(err, buf, sizeof(buf)));
10845 return err;
10846 }
10847 err = fscanf(f, fmt, &ret);
10848 if (err != 1) {
10849 err = err == EOF ? -EIO : -errno;
10850 pr_debug("failed to parse '%s': %s\n", file,
10851 libbpf_strerror_r(err, buf, sizeof(buf)));
10852 fclose(f);
10853 return err;
10854 }
10855 fclose(f);
10856 return ret;
10857 }
10858
10859 static int determine_kprobe_perf_type(void)
10860 {
10861 const char *file = "/sys/bus/event_source/devices/kprobe/type";
10862
10863 return parse_uint_from_file(file, "%d\n");
10864 }
10865
10866 static int determine_uprobe_perf_type(void)
10867 {
10868 const char *file = "/sys/bus/event_source/devices/uprobe/type";
10869
10870 return parse_uint_from_file(file, "%d\n");
10871 }
10872
10873 static int determine_kprobe_retprobe_bit(void)
10874 {
10875 const char *file = "/sys/bus/event_source/devices/kprobe/format/retprobe";
10876
10877 return parse_uint_from_file(file, "config:%d\n");
10878 }
10879
10880 static int determine_uprobe_retprobe_bit(void)
10881 {
10882 const char *file = "/sys/bus/event_source/devices/uprobe/format/retprobe";
10883
10884 return parse_uint_from_file(file, "config:%d\n");
10885 }
10886
10887 #define PERF_UPROBE_REF_CTR_OFFSET_BITS 32
10888 #define PERF_UPROBE_REF_CTR_OFFSET_SHIFT 32
10889
10890 static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name,
10891 uint64_t offset, int pid, size_t ref_ctr_off)
10892 {
10893 const size_t attr_sz = sizeof(struct perf_event_attr);
10894 struct perf_event_attr attr;
10895 char errmsg[STRERR_BUFSIZE];
10896 int type, pfd;
10897
10898 if ((__u64)ref_ctr_off >= (1ULL << PERF_UPROBE_REF_CTR_OFFSET_BITS))
10899 return -EINVAL;
10900
10901 memset(&attr, 0, attr_sz);
10902
10903 type = uprobe ? determine_uprobe_perf_type()
10904 : determine_kprobe_perf_type();
10905 if (type < 0) {
10906 pr_warn("failed to determine %s perf type: %s\n",
10907 uprobe ? "uprobe" : "kprobe",
10908 libbpf_strerror_r(type, errmsg, sizeof(errmsg)));
10909 return type;
10910 }
10911 if (retprobe) {
10912 int bit = uprobe ? determine_uprobe_retprobe_bit()
10913 : determine_kprobe_retprobe_bit();
10914
10915 if (bit < 0) {
10916 pr_warn("failed to determine %s retprobe bit: %s\n",
10917 uprobe ? "uprobe" : "kprobe",
10918 libbpf_strerror_r(bit, errmsg, sizeof(errmsg)));
10919 return bit;
10920 }
10921 attr.config |= 1 << bit;
10922 }
10923 attr.size = attr_sz;
10924 attr.type = type;
10925 attr.config |= (__u64)ref_ctr_off << PERF_UPROBE_REF_CTR_OFFSET_SHIFT;
10926 attr.config1 = ptr_to_u64(name); /* kprobe_func or uprobe_path */
10927 attr.config2 = offset; /* kprobe_addr or probe_offset */
10928
10929 /* pid filter is meaningful only for uprobes */
10930 pfd = syscall(__NR_perf_event_open, &attr,
10931 pid < 0 ? -1 : pid /* pid */,
10932 pid == -1 ? 0 : -1 /* cpu */,
10933 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
10934 return pfd >= 0 ? pfd : -errno;
10935 }
10936
10937 static int append_to_file(const char *file, const char *fmt, ...)
10938 {
10939 int fd, n, err = 0;
10940 va_list ap;
10941 char buf[1024];
10942
10943 va_start(ap, fmt);
10944 n = vsnprintf(buf, sizeof(buf), fmt, ap);
10945 va_end(ap);
10946
10947 if (n < 0 || n >= sizeof(buf))
10948 return -EINVAL;
10949
10950 fd = open(file, O_WRONLY | O_APPEND | O_CLOEXEC, 0);
10951 if (fd < 0)
10952 return -errno;
10953
10954 if (write(fd, buf, n) < 0)
10955 err = -errno;
10956
10957 close(fd);
10958 return err;
10959 }
10960
10961 #define DEBUGFS "/sys/kernel/debug/tracing"
10962 #define TRACEFS "/sys/kernel/tracing"
10963
10964 static bool use_debugfs(void)
10965 {
10966 static int has_debugfs = -1;
10967
10968 if (has_debugfs < 0)
10969 has_debugfs = faccessat(AT_FDCWD, DEBUGFS, F_OK, AT_EACCESS) == 0;
10970
10971 return has_debugfs == 1;
10972 }
10973
10974 static const char *tracefs_path(void)
10975 {
10976 return use_debugfs() ? DEBUGFS : TRACEFS;
10977 }
10978
10979 static const char *tracefs_kprobe_events(void)
10980 {
10981 return use_debugfs() ? DEBUGFS"/kprobe_events" : TRACEFS"/kprobe_events";
10982 }
10983
10984 static const char *tracefs_uprobe_events(void)
10985 {
10986 return use_debugfs() ? DEBUGFS"/uprobe_events" : TRACEFS"/uprobe_events";
10987 }
10988
10989 static const char *tracefs_available_filter_functions(void)
10990 {
10991 return use_debugfs() ? DEBUGFS"/available_filter_functions"
10992 : TRACEFS"/available_filter_functions";
10993 }
10994
10995 static const char *tracefs_available_filter_functions_addrs(void)
10996 {
10997 return use_debugfs() ? DEBUGFS"/available_filter_functions_addrs"
10998 : TRACEFS"/available_filter_functions_addrs";
10999 }
11000
11001 static void gen_kprobe_legacy_event_name(char *buf, size_t buf_sz,
11002 const char *kfunc_name, size_t offset)
11003 {
11004 static int index = 0;
11005 int i;
11006
11007 snprintf(buf, buf_sz, "libbpf_%u_%s_0x%zx_%d", getpid(), kfunc_name, offset,
11008 __sync_fetch_and_add(&index, 1));
11009
11010 /* sanitize binary_path in the probe name */
11011 for (i = 0; buf[i]; i++) {
11012 if (!isalnum(buf[i]))
11013 buf[i] = '_';
11014 }
11015 }
11016
11017 static int add_kprobe_event_legacy(const char *probe_name, bool retprobe,
11018 const char *kfunc_name, size_t offset)
11019 {
11020 return append_to_file(tracefs_kprobe_events(), "%c:%s/%s %s+0x%zx",
11021 retprobe ? 'r' : 'p',
11022 retprobe ? "kretprobes" : "kprobes",
11023 probe_name, kfunc_name, offset);
11024 }
11025
11026 static int remove_kprobe_event_legacy(const char *probe_name, bool retprobe)
11027 {
11028 return append_to_file(tracefs_kprobe_events(), "-:%s/%s",
11029 retprobe ? "kretprobes" : "kprobes", probe_name);
11030 }
11031
11032 static int determine_kprobe_perf_type_legacy(const char *probe_name, bool retprobe)
11033 {
11034 char file[256];
11035
11036 snprintf(file, sizeof(file), "%s/events/%s/%s/id",
11037 tracefs_path(), retprobe ? "kretprobes" : "kprobes", probe_name);
11038
11039 return parse_uint_from_file(file, "%d\n");
11040 }
11041
11042 static int perf_event_kprobe_open_legacy(const char *probe_name, bool retprobe,
11043 const char *kfunc_name, size_t offset, int pid)
11044 {
11045 const size_t attr_sz = sizeof(struct perf_event_attr);
11046 struct perf_event_attr attr;
11047 char errmsg[STRERR_BUFSIZE];
11048 int type, pfd, err;
11049
11050 err = add_kprobe_event_legacy(probe_name, retprobe, kfunc_name, offset);
11051 if (err < 0) {
11052 pr_warn("failed to add legacy kprobe event for '%s+0x%zx': %s\n",
11053 kfunc_name, offset,
11054 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
11055 return err;
11056 }
11057 type = determine_kprobe_perf_type_legacy(probe_name, retprobe);
11058 if (type < 0) {
11059 err = type;
11060 pr_warn("failed to determine legacy kprobe event id for '%s+0x%zx': %s\n",
11061 kfunc_name, offset,
11062 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
11063 goto err_clean_legacy;
11064 }
11065
11066 memset(&attr, 0, attr_sz);
11067 attr.size = attr_sz;
11068 attr.config = type;
11069 attr.type = PERF_TYPE_TRACEPOINT;
11070
11071 pfd = syscall(__NR_perf_event_open, &attr,
11072 pid < 0 ? -1 : pid, /* pid */
11073 pid == -1 ? 0 : -1, /* cpu */
11074 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
11075 if (pfd < 0) {
11076 err = -errno;
11077 pr_warn("legacy kprobe perf_event_open() failed: %s\n",
11078 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
11079 goto err_clean_legacy;
11080 }
11081 return pfd;
11082
11083 err_clean_legacy:
11084 /* Clear the newly added legacy kprobe_event */
11085 remove_kprobe_event_legacy(probe_name, retprobe);
11086 return err;
11087 }
11088
11089 static const char *arch_specific_syscall_pfx(void)
11090 {
11091 #if defined(__x86_64__)
11092 return "x64";
11093 #elif defined(__i386__)
11094 return "ia32";
11095 #elif defined(__s390x__)
11096 return "s390x";
11097 #elif defined(__s390__)
11098 return "s390";
11099 #elif defined(__arm__)
11100 return "arm";
11101 #elif defined(__aarch64__)
11102 return "arm64";
11103 #elif defined(__mips__)
11104 return "mips";
11105 #elif defined(__riscv)
11106 return "riscv";
11107 #elif defined(__powerpc__)
11108 return "powerpc";
11109 #elif defined(__powerpc64__)
11110 return "powerpc64";
11111 #else
11112 return NULL;
11113 #endif
11114 }
11115
11116 int probe_kern_syscall_wrapper(int token_fd)
11117 {
11118 char syscall_name[64];
11119 const char *ksys_pfx;
11120
11121 ksys_pfx = arch_specific_syscall_pfx();
11122 if (!ksys_pfx)
11123 return 0;
11124
11125 snprintf(syscall_name, sizeof(syscall_name), "__%s_sys_bpf", ksys_pfx);
11126
11127 if (determine_kprobe_perf_type() >= 0) {
11128 int pfd;
11129
11130 pfd = perf_event_open_probe(false, false, syscall_name, 0, getpid(), 0);
11131 if (pfd >= 0)
11132 close(pfd);
11133
11134 return pfd >= 0 ? 1 : 0;
11135 } else { /* legacy mode */
11136 char probe_name[128];
11137
11138 gen_kprobe_legacy_event_name(probe_name, sizeof(probe_name), syscall_name, 0);
11139 if (add_kprobe_event_legacy(probe_name, false, syscall_name, 0) < 0)
11140 return 0;
11141
11142 (void)remove_kprobe_event_legacy(probe_name, false);
11143 return 1;
11144 }
11145 }
11146
11147 struct bpf_link *
11148 bpf_program__attach_kprobe_opts(const struct bpf_program *prog,
11149 const char *func_name,
11150 const struct bpf_kprobe_opts *opts)
11151 {
11152 DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts);
11153 enum probe_attach_mode attach_mode;
11154 char errmsg[STRERR_BUFSIZE];
11155 char *legacy_probe = NULL;
11156 struct bpf_link *link;
11157 size_t offset;
11158 bool retprobe, legacy;
11159 int pfd, err;
11160
11161 if (!OPTS_VALID(opts, bpf_kprobe_opts))
11162 return libbpf_err_ptr(-EINVAL);
11163
11164 attach_mode = OPTS_GET(opts, attach_mode, PROBE_ATTACH_MODE_DEFAULT);
11165 retprobe = OPTS_GET(opts, retprobe, false);
11166 offset = OPTS_GET(opts, offset, 0);
11167 pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0);
11168
11169 legacy = determine_kprobe_perf_type() < 0;
11170 switch (attach_mode) {
11171 case PROBE_ATTACH_MODE_LEGACY:
11172 legacy = true;
11173 pe_opts.force_ioctl_attach = true;
11174 break;
11175 case PROBE_ATTACH_MODE_PERF:
11176 if (legacy)
11177 return libbpf_err_ptr(-ENOTSUP);
11178 pe_opts.force_ioctl_attach = true;
11179 break;
11180 case PROBE_ATTACH_MODE_LINK:
11181 if (legacy || !kernel_supports(prog->obj, FEAT_PERF_LINK))
11182 return libbpf_err_ptr(-ENOTSUP);
11183 break;
11184 case PROBE_ATTACH_MODE_DEFAULT:
11185 break;
11186 default:
11187 return libbpf_err_ptr(-EINVAL);
11188 }
11189
11190 if (!legacy) {
11191 pfd = perf_event_open_probe(false /* uprobe */, retprobe,
11192 func_name, offset,
11193 -1 /* pid */, 0 /* ref_ctr_off */);
11194 } else {
11195 char probe_name[256];
11196
11197 gen_kprobe_legacy_event_name(probe_name, sizeof(probe_name),
11198 func_name, offset);
11199
11200 legacy_probe = strdup(probe_name);
11201 if (!legacy_probe)
11202 return libbpf_err_ptr(-ENOMEM);
11203
11204 pfd = perf_event_kprobe_open_legacy(legacy_probe, retprobe, func_name,
11205 offset, -1 /* pid */);
11206 }
11207 if (pfd < 0) {
11208 err = -errno;
11209 pr_warn("prog '%s': failed to create %s '%s+0x%zx' perf event: %s\n",
11210 prog->name, retprobe ? "kretprobe" : "kprobe",
11211 func_name, offset,
11212 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
11213 goto err_out;
11214 }
11215 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts);
11216 err = libbpf_get_error(link);
11217 if (err) {
11218 close(pfd);
11219 pr_warn("prog '%s': failed to attach to %s '%s+0x%zx': %s\n",
11220 prog->name, retprobe ? "kretprobe" : "kprobe",
11221 func_name, offset,
11222 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
11223 goto err_clean_legacy;
11224 }
11225 if (legacy) {
11226 struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link);
11227
11228 perf_link->legacy_probe_name = legacy_probe;
11229 perf_link->legacy_is_kprobe = true;
11230 perf_link->legacy_is_retprobe = retprobe;
11231 }
11232
11233 return link;
11234
11235 err_clean_legacy:
11236 if (legacy)
11237 remove_kprobe_event_legacy(legacy_probe, retprobe);
11238 err_out:
11239 free(legacy_probe);
11240 return libbpf_err_ptr(err);
11241 }
11242
11243 struct bpf_link *bpf_program__attach_kprobe(const struct bpf_program *prog,
11244 bool retprobe,
11245 const char *func_name)
11246 {
11247 DECLARE_LIBBPF_OPTS(bpf_kprobe_opts, opts,
11248 .retprobe = retprobe,
11249 );
11250
11251 return bpf_program__attach_kprobe_opts(prog, func_name, &opts);
11252 }
11253
11254 struct bpf_link *bpf_program__attach_ksyscall(const struct bpf_program *prog,
11255 const char *syscall_name,
11256 const struct bpf_ksyscall_opts *opts)
11257 {
11258 LIBBPF_OPTS(bpf_kprobe_opts, kprobe_opts);
11259 char func_name[128];
11260
11261 if (!OPTS_VALID(opts, bpf_ksyscall_opts))
11262 return libbpf_err_ptr(-EINVAL);
11263
11264 if (kernel_supports(prog->obj, FEAT_SYSCALL_WRAPPER)) {
11265 /* arch_specific_syscall_pfx() should never return NULL here
11266 * because it is guarded by kernel_supports(). However, since
11267 * compiler does not know that we have an explicit conditional
11268 * as well.
11269 */
11270 snprintf(func_name, sizeof(func_name), "__%s_sys_%s",
11271 arch_specific_syscall_pfx() ? : "", syscall_name);
11272 } else {
11273 snprintf(func_name, sizeof(func_name), "__se_sys_%s", syscall_name);
11274 }
11275
11276 kprobe_opts.retprobe = OPTS_GET(opts, retprobe, false);
11277 kprobe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0);
11278
11279 return bpf_program__attach_kprobe_opts(prog, func_name, &kprobe_opts);
11280 }
11281
11282 /* Adapted from perf/util/string.c */
11283 bool glob_match(const char *str, const char *pat)
11284 {
11285 while (*str && *pat && *pat != '*') {
11286 if (*pat == '?') { /* Matches any single character */
11287 str++;
11288 pat++;
11289 continue;
11290 }
11291 if (*str != *pat)
11292 return false;
11293 str++;
11294 pat++;
11295 }
11296 /* Check wild card */
11297 if (*pat == '*') {
11298 while (*pat == '*')
11299 pat++;
11300 if (!*pat) /* Tail wild card matches all */
11301 return true;
11302 while (*str)
11303 if (glob_match(str++, pat))
11304 return true;
11305 }
11306 return !*str && !*pat;
11307 }
11308
11309 struct kprobe_multi_resolve {
11310 const char *pattern;
11311 unsigned long *addrs;
11312 size_t cap;
11313 size_t cnt;
11314 };
11315
11316 struct avail_kallsyms_data {
11317 char **syms;
11318 size_t cnt;
11319 struct kprobe_multi_resolve *res;
11320 };
11321
11322 static int avail_func_cmp(const void *a, const void *b)
11323 {
11324 return strcmp(*(const char **)a, *(const char **)b);
11325 }
11326
11327 static int avail_kallsyms_cb(unsigned long long sym_addr, char sym_type,
11328 const char *sym_name, void *ctx)
11329 {
11330 struct avail_kallsyms_data *data = ctx;
11331 struct kprobe_multi_resolve *res = data->res;
11332 int err;
11333
11334 if (!bsearch(&sym_name, data->syms, data->cnt, sizeof(*data->syms), avail_func_cmp))
11335 return 0;
11336
11337 err = libbpf_ensure_mem((void **)&res->addrs, &res->cap, sizeof(*res->addrs), res->cnt + 1);
11338 if (err)
11339 return err;
11340
11341 res->addrs[res->cnt++] = (unsigned long)sym_addr;
11342 return 0;
11343 }
11344
11345 static int libbpf_available_kallsyms_parse(struct kprobe_multi_resolve *res)
11346 {
11347 const char *available_functions_file = tracefs_available_filter_functions();
11348 struct avail_kallsyms_data data;
11349 char sym_name[500];
11350 FILE *f;
11351 int err = 0, ret, i;
11352 char **syms = NULL;
11353 size_t cap = 0, cnt = 0;
11354
11355 f = fopen(available_functions_file, "re");
11356 if (!f) {
11357 err = -errno;
11358 pr_warn("failed to open %s: %d\n", available_functions_file, err);
11359 return err;
11360 }
11361
11362 while (true) {
11363 char *name;
11364
11365 ret = fscanf(f, "%499s%*[^\n]\n", sym_name);
11366 if (ret == EOF && feof(f))
11367 break;
11368
11369 if (ret != 1) {
11370 pr_warn("failed to parse available_filter_functions entry: %d\n", ret);
11371 err = -EINVAL;
11372 goto cleanup;
11373 }
11374
11375 if (!glob_match(sym_name, res->pattern))
11376 continue;
11377
11378 err = libbpf_ensure_mem((void **)&syms, &cap, sizeof(*syms), cnt + 1);
11379 if (err)
11380 goto cleanup;
11381
11382 name = strdup(sym_name);
11383 if (!name) {
11384 err = -errno;
11385 goto cleanup;
11386 }
11387
11388 syms[cnt++] = name;
11389 }
11390
11391 /* no entries found, bail out */
11392 if (cnt == 0) {
11393 err = -ENOENT;
11394 goto cleanup;
11395 }
11396
11397 /* sort available functions */
11398 qsort(syms, cnt, sizeof(*syms), avail_func_cmp);
11399
11400 data.syms = syms;
11401 data.res = res;
11402 data.cnt = cnt;
11403 libbpf_kallsyms_parse(avail_kallsyms_cb, &data);
11404
11405 if (res->cnt == 0)
11406 err = -ENOENT;
11407
11408 cleanup:
11409 for (i = 0; i < cnt; i++)
11410 free((char *)syms[i]);
11411 free(syms);
11412
11413 fclose(f);
11414 return err;
11415 }
11416
11417 static bool has_available_filter_functions_addrs(void)
11418 {
11419 return access(tracefs_available_filter_functions_addrs(), R_OK) != -1;
11420 }
11421
11422 static int libbpf_available_kprobes_parse(struct kprobe_multi_resolve *res)
11423 {
11424 const char *available_path = tracefs_available_filter_functions_addrs();
11425 char sym_name[500];
11426 FILE *f;
11427 int ret, err = 0;
11428 unsigned long long sym_addr;
11429
11430 f = fopen(available_path, "re");
11431 if (!f) {
11432 err = -errno;
11433 pr_warn("failed to open %s: %d\n", available_path, err);
11434 return err;
11435 }
11436
11437 while (true) {
11438 ret = fscanf(f, "%llx %499s%*[^\n]\n", &sym_addr, sym_name);
11439 if (ret == EOF && feof(f))
11440 break;
11441
11442 if (ret != 2) {
11443 pr_warn("failed to parse available_filter_functions_addrs entry: %d\n",
11444 ret);
11445 err = -EINVAL;
11446 goto cleanup;
11447 }
11448
11449 if (!glob_match(sym_name, res->pattern))
11450 continue;
11451
11452 err = libbpf_ensure_mem((void **)&res->addrs, &res->cap,
11453 sizeof(*res->addrs), res->cnt + 1);
11454 if (err)
11455 goto cleanup;
11456
11457 res->addrs[res->cnt++] = (unsigned long)sym_addr;
11458 }
11459
11460 if (res->cnt == 0)
11461 err = -ENOENT;
11462
11463 cleanup:
11464 fclose(f);
11465 return err;
11466 }
11467
11468 struct bpf_link *
11469 bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog,
11470 const char *pattern,
11471 const struct bpf_kprobe_multi_opts *opts)
11472 {
11473 LIBBPF_OPTS(bpf_link_create_opts, lopts);
11474 struct kprobe_multi_resolve res = {
11475 .pattern = pattern,
11476 };
11477 enum bpf_attach_type attach_type;
11478 struct bpf_link *link = NULL;
11479 char errmsg[STRERR_BUFSIZE];
11480 const unsigned long *addrs;
11481 int err, link_fd, prog_fd;
11482 bool retprobe, session;
11483 const __u64 *cookies;
11484 const char **syms;
11485 size_t cnt;
11486
11487 if (!OPTS_VALID(opts, bpf_kprobe_multi_opts))
11488 return libbpf_err_ptr(-EINVAL);
11489
11490 prog_fd = bpf_program__fd(prog);
11491 if (prog_fd < 0) {
11492 pr_warn("prog '%s': can't attach BPF program without FD (was it loaded?)\n",
11493 prog->name);
11494 return libbpf_err_ptr(-EINVAL);
11495 }
11496
11497 syms = OPTS_GET(opts, syms, false);
11498 addrs = OPTS_GET(opts, addrs, false);
11499 cnt = OPTS_GET(opts, cnt, false);
11500 cookies = OPTS_GET(opts, cookies, false);
11501
11502 if (!pattern && !addrs && !syms)
11503 return libbpf_err_ptr(-EINVAL);
11504 if (pattern && (addrs || syms || cookies || cnt))
11505 return libbpf_err_ptr(-EINVAL);
11506 if (!pattern && !cnt)
11507 return libbpf_err_ptr(-EINVAL);
11508 if (addrs && syms)
11509 return libbpf_err_ptr(-EINVAL);
11510
11511 if (pattern) {
11512 if (has_available_filter_functions_addrs())
11513 err = libbpf_available_kprobes_parse(&res);
11514 else
11515 err = libbpf_available_kallsyms_parse(&res);
11516 if (err)
11517 goto error;
11518 addrs = res.addrs;
11519 cnt = res.cnt;
11520 }
11521
11522 retprobe = OPTS_GET(opts, retprobe, false);
11523 session = OPTS_GET(opts, session, false);
11524
11525 if (retprobe && session)
11526 return libbpf_err_ptr(-EINVAL);
11527
11528 attach_type = session ? BPF_TRACE_KPROBE_SESSION : BPF_TRACE_KPROBE_MULTI;
11529
11530 lopts.kprobe_multi.syms = syms;
11531 lopts.kprobe_multi.addrs = addrs;
11532 lopts.kprobe_multi.cookies = cookies;
11533 lopts.kprobe_multi.cnt = cnt;
11534 lopts.kprobe_multi.flags = retprobe ? BPF_F_KPROBE_MULTI_RETURN : 0;
11535
11536 link = calloc(1, sizeof(*link));
11537 if (!link) {
11538 err = -ENOMEM;
11539 goto error;
11540 }
11541 link->detach = &bpf_link__detach_fd;
11542
11543 link_fd = bpf_link_create(prog_fd, 0, attach_type, &lopts);
11544 if (link_fd < 0) {
11545 err = -errno;
11546 pr_warn("prog '%s': failed to attach: %s\n",
11547 prog->name, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
11548 goto error;
11549 }
11550 link->fd = link_fd;
11551 free(res.addrs);
11552 return link;
11553
11554 error:
11555 free(link);
11556 free(res.addrs);
11557 return libbpf_err_ptr(err);
11558 }
11559
11560 static int attach_kprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link)
11561 {
11562 DECLARE_LIBBPF_OPTS(bpf_kprobe_opts, opts);
11563 unsigned long offset = 0;
11564 const char *func_name;
11565 char *func;
11566 int n;
11567
11568 *link = NULL;
11569
11570 /* no auto-attach for SEC("kprobe") and SEC("kretprobe") */
11571 if (strcmp(prog->sec_name, "kprobe") == 0 || strcmp(prog->sec_name, "kretprobe") == 0)
11572 return 0;
11573
11574 opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe/");
11575 if (opts.retprobe)
11576 func_name = prog->sec_name + sizeof("kretprobe/") - 1;
11577 else
11578 func_name = prog->sec_name + sizeof("kprobe/") - 1;
11579
11580 n = sscanf(func_name, "%m[a-zA-Z0-9_.]+%li", &func, &offset);
11581 if (n < 1) {
11582 pr_warn("kprobe name is invalid: %s\n", func_name);
11583 return -EINVAL;
11584 }
11585 if (opts.retprobe && offset != 0) {
11586 free(func);
11587 pr_warn("kretprobes do not support offset specification\n");
11588 return -EINVAL;
11589 }
11590
11591 opts.offset = offset;
11592 *link = bpf_program__attach_kprobe_opts(prog, func, &opts);
11593 free(func);
11594 return libbpf_get_error(*link);
11595 }
11596
11597 static int attach_ksyscall(const struct bpf_program *prog, long cookie, struct bpf_link **link)
11598 {
11599 LIBBPF_OPTS(bpf_ksyscall_opts, opts);
11600 const char *syscall_name;
11601
11602 *link = NULL;
11603
11604 /* no auto-attach for SEC("ksyscall") and SEC("kretsyscall") */
11605 if (strcmp(prog->sec_name, "ksyscall") == 0 || strcmp(prog->sec_name, "kretsyscall") == 0)
11606 return 0;
11607
11608 opts.retprobe = str_has_pfx(prog->sec_name, "kretsyscall/");
11609 if (opts.retprobe)
11610 syscall_name = prog->sec_name + sizeof("kretsyscall/") - 1;
11611 else
11612 syscall_name = prog->sec_name + sizeof("ksyscall/") - 1;
11613
11614 *link = bpf_program__attach_ksyscall(prog, syscall_name, &opts);
11615 return *link ? 0 : -errno;
11616 }
11617
11618 static int attach_kprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link)
11619 {
11620 LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
11621 const char *spec;
11622 char *pattern;
11623 int n;
11624
11625 *link = NULL;
11626
11627 /* no auto-attach for SEC("kprobe.multi") and SEC("kretprobe.multi") */
11628 if (strcmp(prog->sec_name, "kprobe.multi") == 0 ||
11629 strcmp(prog->sec_name, "kretprobe.multi") == 0)
11630 return 0;
11631
11632 opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe.multi/");
11633 if (opts.retprobe)
11634 spec = prog->sec_name + sizeof("kretprobe.multi/") - 1;
11635 else
11636 spec = prog->sec_name + sizeof("kprobe.multi/") - 1;
11637
11638 n = sscanf(spec, "%m[a-zA-Z0-9_.*?]", &pattern);
11639 if (n < 1) {
11640 pr_warn("kprobe multi pattern is invalid: %s\n", spec);
11641 return -EINVAL;
11642 }
11643
11644 *link = bpf_program__attach_kprobe_multi_opts(prog, pattern, &opts);
11645 free(pattern);
11646 return libbpf_get_error(*link);
11647 }
11648
11649 static int attach_kprobe_session(const struct bpf_program *prog, long cookie,
11650 struct bpf_link **link)
11651 {
11652 LIBBPF_OPTS(bpf_kprobe_multi_opts, opts, .session = true);
11653 const char *spec;
11654 char *pattern;
11655 int n;
11656
11657 *link = NULL;
11658
11659 /* no auto-attach for SEC("kprobe.session") */
11660 if (strcmp(prog->sec_name, "kprobe.session") == 0)
11661 return 0;
11662
11663 spec = prog->sec_name + sizeof("kprobe.session/") - 1;
11664 n = sscanf(spec, "%m[a-zA-Z0-9_.*?]", &pattern);
11665 if (n < 1) {
11666 pr_warn("kprobe session pattern is invalid: %s\n", spec);
11667 return -EINVAL;
11668 }
11669
11670 *link = bpf_program__attach_kprobe_multi_opts(prog, pattern, &opts);
11671 free(pattern);
11672 return *link ? 0 : -errno;
11673 }
11674
11675 static int attach_uprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link)
11676 {
11677 char *probe_type = NULL, *binary_path = NULL, *func_name = NULL;
11678 LIBBPF_OPTS(bpf_uprobe_multi_opts, opts);
11679 int n, ret = -EINVAL;
11680
11681 *link = NULL;
11682
11683 n = sscanf(prog->sec_name, "%m[^/]/%m[^:]:%m[^\n]",
11684 &probe_type, &binary_path, &func_name);
11685 switch (n) {
11686 case 1:
11687 /* handle SEC("u[ret]probe") - format is valid, but auto-attach is impossible. */
11688 ret = 0;
11689 break;
11690 case 3:
11691 opts.retprobe = strcmp(probe_type, "uretprobe.multi") == 0;
11692 *link = bpf_program__attach_uprobe_multi(prog, -1, binary_path, func_name, &opts);
11693 ret = libbpf_get_error(*link);
11694 break;
11695 default:
11696 pr_warn("prog '%s': invalid format of section definition '%s'\n", prog->name,
11697 prog->sec_name);
11698 break;
11699 }
11700 free(probe_type);
11701 free(binary_path);
11702 free(func_name);
11703 return ret;
11704 }
11705
11706 static void gen_uprobe_legacy_event_name(char *buf, size_t buf_sz,
11707 const char *binary_path, uint64_t offset)
11708 {
11709 int i;
11710
11711 snprintf(buf, buf_sz, "libbpf_%u_%s_0x%zx", getpid(), binary_path, (size_t)offset);
11712
11713 /* sanitize binary_path in the probe name */
11714 for (i = 0; buf[i]; i++) {
11715 if (!isalnum(buf[i]))
11716 buf[i] = '_';
11717 }
11718 }
11719
11720 static inline int add_uprobe_event_legacy(const char *probe_name, bool retprobe,
11721 const char *binary_path, size_t offset)
11722 {
11723 return append_to_file(tracefs_uprobe_events(), "%c:%s/%s %s:0x%zx",
11724 retprobe ? 'r' : 'p',
11725 retprobe ? "uretprobes" : "uprobes",
11726 probe_name, binary_path, offset);
11727 }
11728
11729 static inline int remove_uprobe_event_legacy(const char *probe_name, bool retprobe)
11730 {
11731 return append_to_file(tracefs_uprobe_events(), "-:%s/%s",
11732 retprobe ? "uretprobes" : "uprobes", probe_name);
11733 }
11734
11735 static int determine_uprobe_perf_type_legacy(const char *probe_name, bool retprobe)
11736 {
11737 char file[512];
11738
11739 snprintf(file, sizeof(file), "%s/events/%s/%s/id",
11740 tracefs_path(), retprobe ? "uretprobes" : "uprobes", probe_name);
11741
11742 return parse_uint_from_file(file, "%d\n");
11743 }
11744
11745 static int perf_event_uprobe_open_legacy(const char *probe_name, bool retprobe,
11746 const char *binary_path, size_t offset, int pid)
11747 {
11748 const size_t attr_sz = sizeof(struct perf_event_attr);
11749 struct perf_event_attr attr;
11750 int type, pfd, err;
11751
11752 err = add_uprobe_event_legacy(probe_name, retprobe, binary_path, offset);
11753 if (err < 0) {
11754 pr_warn("failed to add legacy uprobe event for %s:0x%zx: %d\n",
11755 binary_path, (size_t)offset, err);
11756 return err;
11757 }
11758 type = determine_uprobe_perf_type_legacy(probe_name, retprobe);
11759 if (type < 0) {
11760 err = type;
11761 pr_warn("failed to determine legacy uprobe event id for %s:0x%zx: %d\n",
11762 binary_path, offset, err);
11763 goto err_clean_legacy;
11764 }
11765
11766 memset(&attr, 0, attr_sz);
11767 attr.size = attr_sz;
11768 attr.config = type;
11769 attr.type = PERF_TYPE_TRACEPOINT;
11770
11771 pfd = syscall(__NR_perf_event_open, &attr,
11772 pid < 0 ? -1 : pid, /* pid */
11773 pid == -1 ? 0 : -1, /* cpu */
11774 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
11775 if (pfd < 0) {
11776 err = -errno;
11777 pr_warn("legacy uprobe perf_event_open() failed: %d\n", err);
11778 goto err_clean_legacy;
11779 }
11780 return pfd;
11781
11782 err_clean_legacy:
11783 /* Clear the newly added legacy uprobe_event */
11784 remove_uprobe_event_legacy(probe_name, retprobe);
11785 return err;
11786 }
11787
11788 /* Find offset of function name in archive specified by path. Currently
11789 * supported are .zip files that do not compress their contents, as used on
11790 * Android in the form of APKs, for example. "file_name" is the name of the ELF
11791 * file inside the archive. "func_name" matches symbol name or name@@LIB for
11792 * library functions.
11793 *
11794 * An overview of the APK format specifically provided here:
11795 * https://en.wikipedia.org/w/index.php?title=Apk_(file_format)&oldid=1139099120#Package_contents
11796 */
11797 static long elf_find_func_offset_from_archive(const char *archive_path, const char *file_name,
11798 const char *func_name)
11799 {
11800 struct zip_archive *archive;
11801 struct zip_entry entry;
11802 long ret;
11803 Elf *elf;
11804
11805 archive = zip_archive_open(archive_path);
11806 if (IS_ERR(archive)) {
11807 ret = PTR_ERR(archive);
11808 pr_warn("zip: failed to open %s: %ld\n", archive_path, ret);
11809 return ret;
11810 }
11811
11812 ret = zip_archive_find_entry(archive, file_name, &entry);
11813 if (ret) {
11814 pr_warn("zip: could not find archive member %s in %s: %ld\n", file_name,
11815 archive_path, ret);
11816 goto out;
11817 }
11818 pr_debug("zip: found entry for %s in %s at 0x%lx\n", file_name, archive_path,
11819 (unsigned long)entry.data_offset);
11820
11821 if (entry.compression) {
11822 pr_warn("zip: entry %s of %s is compressed and cannot be handled\n", file_name,
11823 archive_path);
11824 ret = -LIBBPF_ERRNO__FORMAT;
11825 goto out;
11826 }
11827
11828 elf = elf_memory((void *)entry.data, entry.data_length);
11829 if (!elf) {
11830 pr_warn("elf: could not read elf file %s from %s: %s\n", file_name, archive_path,
11831 elf_errmsg(-1));
11832 ret = -LIBBPF_ERRNO__LIBELF;
11833 goto out;
11834 }
11835
11836 ret = elf_find_func_offset(elf, file_name, func_name);
11837 if (ret > 0) {
11838 pr_debug("elf: symbol address match for %s of %s in %s: 0x%x + 0x%lx = 0x%lx\n",
11839 func_name, file_name, archive_path, entry.data_offset, ret,
11840 ret + entry.data_offset);
11841 ret += entry.data_offset;
11842 }
11843 elf_end(elf);
11844
11845 out:
11846 zip_archive_close(archive);
11847 return ret;
11848 }
11849
11850 static const char *arch_specific_lib_paths(void)
11851 {
11852 /*
11853 * Based on https://packages.debian.org/sid/libc6.
11854 *
11855 * Assume that the traced program is built for the same architecture
11856 * as libbpf, which should cover the vast majority of cases.
11857 */
11858 #if defined(__x86_64__)
11859 return "/lib/x86_64-linux-gnu";
11860 #elif defined(__i386__)
11861 return "/lib/i386-linux-gnu";
11862 #elif defined(__s390x__)
11863 return "/lib/s390x-linux-gnu";
11864 #elif defined(__s390__)
11865 return "/lib/s390-linux-gnu";
11866 #elif defined(__arm__) && defined(__SOFTFP__)
11867 return "/lib/arm-linux-gnueabi";
11868 #elif defined(__arm__) && !defined(__SOFTFP__)
11869 return "/lib/arm-linux-gnueabihf";
11870 #elif defined(__aarch64__)
11871 return "/lib/aarch64-linux-gnu";
11872 #elif defined(__mips__) && defined(__MIPSEL__) && _MIPS_SZLONG == 64
11873 return "/lib/mips64el-linux-gnuabi64";
11874 #elif defined(__mips__) && defined(__MIPSEL__) && _MIPS_SZLONG == 32
11875 return "/lib/mipsel-linux-gnu";
11876 #elif defined(__powerpc64__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
11877 return "/lib/powerpc64le-linux-gnu";
11878 #elif defined(__sparc__) && defined(__arch64__)
11879 return "/lib/sparc64-linux-gnu";
11880 #elif defined(__riscv) && __riscv_xlen == 64
11881 return "/lib/riscv64-linux-gnu";
11882 #else
11883 return NULL;
11884 #endif
11885 }
11886
11887 /* Get full path to program/shared library. */
11888 static int resolve_full_path(const char *file, char *result, size_t result_sz)
11889 {
11890 const char *search_paths[3] = {};
11891 int i, perm;
11892
11893 if (str_has_sfx(file, ".so") || strstr(file, ".so.")) {
11894 search_paths[0] = getenv("LD_LIBRARY_PATH");
11895 search_paths[1] = "/usr/lib64:/usr/lib";
11896 search_paths[2] = arch_specific_lib_paths();
11897 perm = R_OK;
11898 } else {
11899 search_paths[0] = getenv("PATH");
11900 search_paths[1] = "/usr/bin:/usr/sbin";
11901 perm = R_OK | X_OK;
11902 }
11903
11904 for (i = 0; i < ARRAY_SIZE(search_paths); i++) {
11905 const char *s;
11906
11907 if (!search_paths[i])
11908 continue;
11909 for (s = search_paths[i]; s != NULL; s = strchr(s, ':')) {
11910 char *next_path;
11911 int seg_len;
11912
11913 if (s[0] == ':')
11914 s++;
11915 next_path = strchr(s, ':');
11916 seg_len = next_path ? next_path - s : strlen(s);
11917 if (!seg_len)
11918 continue;
11919 snprintf(result, result_sz, "%.*s/%s", seg_len, s, file);
11920 /* ensure it has required permissions */
11921 if (faccessat(AT_FDCWD, result, perm, AT_EACCESS) < 0)
11922 continue;
11923 pr_debug("resolved '%s' to '%s'\n", file, result);
11924 return 0;
11925 }
11926 }
11927 return -ENOENT;
11928 }
11929
11930 struct bpf_link *
11931 bpf_program__attach_uprobe_multi(const struct bpf_program *prog,
11932 pid_t pid,
11933 const char *path,
11934 const char *func_pattern,
11935 const struct bpf_uprobe_multi_opts *opts)
11936 {
11937 const unsigned long *ref_ctr_offsets = NULL, *offsets = NULL;
11938 LIBBPF_OPTS(bpf_link_create_opts, lopts);
11939 unsigned long *resolved_offsets = NULL;
11940 int err = 0, link_fd, prog_fd;
11941 struct bpf_link *link = NULL;
11942 char errmsg[STRERR_BUFSIZE];
11943 char full_path[PATH_MAX];
11944 const __u64 *cookies;
11945 const char **syms;
11946 size_t cnt;
11947
11948 if (!OPTS_VALID(opts, bpf_uprobe_multi_opts))
11949 return libbpf_err_ptr(-EINVAL);
11950
11951 prog_fd = bpf_program__fd(prog);
11952 if (prog_fd < 0) {
11953 pr_warn("prog '%s': can't attach BPF program without FD (was it loaded?)\n",
11954 prog->name);
11955 return libbpf_err_ptr(-EINVAL);
11956 }
11957
11958 syms = OPTS_GET(opts, syms, NULL);
11959 offsets = OPTS_GET(opts, offsets, NULL);
11960 ref_ctr_offsets = OPTS_GET(opts, ref_ctr_offsets, NULL);
11961 cookies = OPTS_GET(opts, cookies, NULL);
11962 cnt = OPTS_GET(opts, cnt, 0);
11963
11964 /*
11965 * User can specify 2 mutually exclusive set of inputs:
11966 *
11967 * 1) use only path/func_pattern/pid arguments
11968 *
11969 * 2) use path/pid with allowed combinations of:
11970 * syms/offsets/ref_ctr_offsets/cookies/cnt
11971 *
11972 * - syms and offsets are mutually exclusive
11973 * - ref_ctr_offsets and cookies are optional
11974 *
11975 * Any other usage results in error.
11976 */
11977
11978 if (!path)
11979 return libbpf_err_ptr(-EINVAL);
11980 if (!func_pattern && cnt == 0)
11981 return libbpf_err_ptr(-EINVAL);
11982
11983 if (func_pattern) {
11984 if (syms || offsets || ref_ctr_offsets || cookies || cnt)
11985 return libbpf_err_ptr(-EINVAL);
11986 } else {
11987 if (!!syms == !!offsets)
11988 return libbpf_err_ptr(-EINVAL);
11989 }
11990
11991 if (func_pattern) {
11992 if (!strchr(path, '/')) {
11993 err = resolve_full_path(path, full_path, sizeof(full_path));
11994 if (err) {
11995 pr_warn("prog '%s': failed to resolve full path for '%s': %d\n",
11996 prog->name, path, err);
11997 return libbpf_err_ptr(err);
11998 }
11999 path = full_path;
12000 }
12001
12002 err = elf_resolve_pattern_offsets(path, func_pattern,
12003 &resolved_offsets, &cnt);
12004 if (err < 0)
12005 return libbpf_err_ptr(err);
12006 offsets = resolved_offsets;
12007 } else if (syms) {
12008 err = elf_resolve_syms_offsets(path, cnt, syms, &resolved_offsets, STT_FUNC);
12009 if (err < 0)
12010 return libbpf_err_ptr(err);
12011 offsets = resolved_offsets;
12012 }
12013
12014 lopts.uprobe_multi.path = path;
12015 lopts.uprobe_multi.offsets = offsets;
12016 lopts.uprobe_multi.ref_ctr_offsets = ref_ctr_offsets;
12017 lopts.uprobe_multi.cookies = cookies;
12018 lopts.uprobe_multi.cnt = cnt;
12019 lopts.uprobe_multi.flags = OPTS_GET(opts, retprobe, false) ? BPF_F_UPROBE_MULTI_RETURN : 0;
12020
12021 if (pid == 0)
12022 pid = getpid();
12023 if (pid > 0)
12024 lopts.uprobe_multi.pid = pid;
12025
12026 link = calloc(1, sizeof(*link));
12027 if (!link) {
12028 err = -ENOMEM;
12029 goto error;
12030 }
12031 link->detach = &bpf_link__detach_fd;
12032
12033 link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_UPROBE_MULTI, &lopts);
12034 if (link_fd < 0) {
12035 err = -errno;
12036 pr_warn("prog '%s': failed to attach multi-uprobe: %s\n",
12037 prog->name, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
12038 goto error;
12039 }
12040 link->fd = link_fd;
12041 free(resolved_offsets);
12042 return link;
12043
12044 error:
12045 free(resolved_offsets);
12046 free(link);
12047 return libbpf_err_ptr(err);
12048 }
12049
12050 LIBBPF_API struct bpf_link *
12051 bpf_program__attach_uprobe_opts(const struct bpf_program *prog, pid_t pid,
12052 const char *binary_path, size_t func_offset,
12053 const struct bpf_uprobe_opts *opts)
12054 {
12055 const char *archive_path = NULL, *archive_sep = NULL;
12056 char errmsg[STRERR_BUFSIZE], *legacy_probe = NULL;
12057 DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts);
12058 enum probe_attach_mode attach_mode;
12059 char full_path[PATH_MAX];
12060 struct bpf_link *link;
12061 size_t ref_ctr_off;
12062 int pfd, err;
12063 bool retprobe, legacy;
12064 const char *func_name;
12065
12066 if (!OPTS_VALID(opts, bpf_uprobe_opts))
12067 return libbpf_err_ptr(-EINVAL);
12068
12069 attach_mode = OPTS_GET(opts, attach_mode, PROBE_ATTACH_MODE_DEFAULT);
12070 retprobe = OPTS_GET(opts, retprobe, false);
12071 ref_ctr_off = OPTS_GET(opts, ref_ctr_offset, 0);
12072 pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0);
12073
12074 if (!binary_path)
12075 return libbpf_err_ptr(-EINVAL);
12076
12077 /* Check if "binary_path" refers to an archive. */
12078 archive_sep = strstr(binary_path, "!/");
12079 if (archive_sep) {
12080 full_path[0] = '\0';
12081 libbpf_strlcpy(full_path, binary_path,
12082 min(sizeof(full_path), (size_t)(archive_sep - binary_path + 1)));
12083 archive_path = full_path;
12084 binary_path = archive_sep + 2;
12085 } else if (!strchr(binary_path, '/')) {
12086 err = resolve_full_path(binary_path, full_path, sizeof(full_path));
12087 if (err) {
12088 pr_warn("prog '%s': failed to resolve full path for '%s': %d\n",
12089 prog->name, binary_path, err);
12090 return libbpf_err_ptr(err);
12091 }
12092 binary_path = full_path;
12093 }
12094 func_name = OPTS_GET(opts, func_name, NULL);
12095 if (func_name) {
12096 long sym_off;
12097
12098 if (archive_path) {
12099 sym_off = elf_find_func_offset_from_archive(archive_path, binary_path,
12100 func_name);
12101 binary_path = archive_path;
12102 } else {
12103 sym_off = elf_find_func_offset_from_file(binary_path, func_name);
12104 }
12105 if (sym_off < 0)
12106 return libbpf_err_ptr(sym_off);
12107 func_offset += sym_off;
12108 }
12109
12110 legacy = determine_uprobe_perf_type() < 0;
12111 switch (attach_mode) {
12112 case PROBE_ATTACH_MODE_LEGACY:
12113 legacy = true;
12114 pe_opts.force_ioctl_attach = true;
12115 break;
12116 case PROBE_ATTACH_MODE_PERF:
12117 if (legacy)
12118 return libbpf_err_ptr(-ENOTSUP);
12119 pe_opts.force_ioctl_attach = true;
12120 break;
12121 case PROBE_ATTACH_MODE_LINK:
12122 if (legacy || !kernel_supports(prog->obj, FEAT_PERF_LINK))
12123 return libbpf_err_ptr(-ENOTSUP);
12124 break;
12125 case PROBE_ATTACH_MODE_DEFAULT:
12126 break;
12127 default:
12128 return libbpf_err_ptr(-EINVAL);
12129 }
12130
12131 if (!legacy) {
12132 pfd = perf_event_open_probe(true /* uprobe */, retprobe, binary_path,
12133 func_offset, pid, ref_ctr_off);
12134 } else {
12135 char probe_name[PATH_MAX + 64];
12136
12137 if (ref_ctr_off)
12138 return libbpf_err_ptr(-EINVAL);
12139
12140 gen_uprobe_legacy_event_name(probe_name, sizeof(probe_name),
12141 binary_path, func_offset);
12142
12143 legacy_probe = strdup(probe_name);
12144 if (!legacy_probe)
12145 return libbpf_err_ptr(-ENOMEM);
12146
12147 pfd = perf_event_uprobe_open_legacy(legacy_probe, retprobe,
12148 binary_path, func_offset, pid);
12149 }
12150 if (pfd < 0) {
12151 err = -errno;
12152 pr_warn("prog '%s': failed to create %s '%s:0x%zx' perf event: %s\n",
12153 prog->name, retprobe ? "uretprobe" : "uprobe",
12154 binary_path, func_offset,
12155 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
12156 goto err_out;
12157 }
12158
12159 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts);
12160 err = libbpf_get_error(link);
12161 if (err) {
12162 close(pfd);
12163 pr_warn("prog '%s': failed to attach to %s '%s:0x%zx': %s\n",
12164 prog->name, retprobe ? "uretprobe" : "uprobe",
12165 binary_path, func_offset,
12166 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
12167 goto err_clean_legacy;
12168 }
12169 if (legacy) {
12170 struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link);
12171
12172 perf_link->legacy_probe_name = legacy_probe;
12173 perf_link->legacy_is_kprobe = false;
12174 perf_link->legacy_is_retprobe = retprobe;
12175 }
12176 return link;
12177
12178 err_clean_legacy:
12179 if (legacy)
12180 remove_uprobe_event_legacy(legacy_probe, retprobe);
12181 err_out:
12182 free(legacy_probe);
12183 return libbpf_err_ptr(err);
12184 }
12185
12186 /* Format of u[ret]probe section definition supporting auto-attach:
12187 * u[ret]probe/binary:function[+offset]
12188 *
12189 * binary can be an absolute/relative path or a filename; the latter is resolved to a
12190 * full binary path via bpf_program__attach_uprobe_opts.
12191 *
12192 * Specifying uprobe+ ensures we carry out strict matching; either "uprobe" must be
12193 * specified (and auto-attach is not possible) or the above format is specified for
12194 * auto-attach.
12195 */
12196 static int attach_uprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link)
12197 {
12198 DECLARE_LIBBPF_OPTS(bpf_uprobe_opts, opts);
12199 char *probe_type = NULL, *binary_path = NULL, *func_name = NULL, *func_off;
12200 int n, c, ret = -EINVAL;
12201 long offset = 0;
12202
12203 *link = NULL;
12204
12205 n = sscanf(prog->sec_name, "%m[^/]/%m[^:]:%m[^\n]",
12206 &probe_type, &binary_path, &func_name);
12207 switch (n) {
12208 case 1:
12209 /* handle SEC("u[ret]probe") - format is valid, but auto-attach is impossible. */
12210 ret = 0;
12211 break;
12212 case 2:
12213 pr_warn("prog '%s': section '%s' missing ':function[+offset]' specification\n",
12214 prog->name, prog->sec_name);
12215 break;
12216 case 3:
12217 /* check if user specifies `+offset`, if yes, this should be
12218 * the last part of the string, make sure sscanf read to EOL
12219 */
12220 func_off = strrchr(func_name, '+');
12221 if (func_off) {
12222 n = sscanf(func_off, "+%li%n", &offset, &c);
12223 if (n == 1 && *(func_off + c) == '\0')
12224 func_off[0] = '\0';
12225 else
12226 offset = 0;
12227 }
12228 opts.retprobe = strcmp(probe_type, "uretprobe") == 0 ||
12229 strcmp(probe_type, "uretprobe.s") == 0;
12230 if (opts.retprobe && offset != 0) {
12231 pr_warn("prog '%s': uretprobes do not support offset specification\n",
12232 prog->name);
12233 break;
12234 }
12235 opts.func_name = func_name;
12236 *link = bpf_program__attach_uprobe_opts(prog, -1, binary_path, offset, &opts);
12237 ret = libbpf_get_error(*link);
12238 break;
12239 default:
12240 pr_warn("prog '%s': invalid format of section definition '%s'\n", prog->name,
12241 prog->sec_name);
12242 break;
12243 }
12244 free(probe_type);
12245 free(binary_path);
12246 free(func_name);
12247
12248 return ret;
12249 }
12250
12251 struct bpf_link *bpf_program__attach_uprobe(const struct bpf_program *prog,
12252 bool retprobe, pid_t pid,
12253 const char *binary_path,
12254 size_t func_offset)
12255 {
12256 DECLARE_LIBBPF_OPTS(bpf_uprobe_opts, opts, .retprobe = retprobe);
12257
12258 return bpf_program__attach_uprobe_opts(prog, pid, binary_path, func_offset, &opts);
12259 }
12260
12261 struct bpf_link *bpf_program__attach_usdt(const struct bpf_program *prog,
12262 pid_t pid, const char *binary_path,
12263 const char *usdt_provider, const char *usdt_name,
12264 const struct bpf_usdt_opts *opts)
12265 {
12266 char resolved_path[512];
12267 struct bpf_object *obj = prog->obj;
12268 struct bpf_link *link;
12269 __u64 usdt_cookie;
12270 int err;
12271
12272 if (!OPTS_VALID(opts, bpf_uprobe_opts))
12273 return libbpf_err_ptr(-EINVAL);
12274
12275 if (bpf_program__fd(prog) < 0) {
12276 pr_warn("prog '%s': can't attach BPF program without FD (was it loaded?)\n",
12277 prog->name);
12278 return libbpf_err_ptr(-EINVAL);
12279 }
12280
12281 if (!binary_path)
12282 return libbpf_err_ptr(-EINVAL);
12283
12284 if (!strchr(binary_path, '/')) {
12285 err = resolve_full_path(binary_path, resolved_path, sizeof(resolved_path));
12286 if (err) {
12287 pr_warn("prog '%s': failed to resolve full path for '%s': %d\n",
12288 prog->name, binary_path, err);
12289 return libbpf_err_ptr(err);
12290 }
12291 binary_path = resolved_path;
12292 }
12293
12294 /* USDT manager is instantiated lazily on first USDT attach. It will
12295 * be destroyed together with BPF object in bpf_object__close().
12296 */
12297 if (IS_ERR(obj->usdt_man))
12298 return libbpf_ptr(obj->usdt_man);
12299 if (!obj->usdt_man) {
12300 obj->usdt_man = usdt_manager_new(obj);
12301 if (IS_ERR(obj->usdt_man))
12302 return libbpf_ptr(obj->usdt_man);
12303 }
12304
12305 usdt_cookie = OPTS_GET(opts, usdt_cookie, 0);
12306 link = usdt_manager_attach_usdt(obj->usdt_man, prog, pid, binary_path,
12307 usdt_provider, usdt_name, usdt_cookie);
12308 err = libbpf_get_error(link);
12309 if (err)
12310 return libbpf_err_ptr(err);
12311 return link;
12312 }
12313
12314 static int attach_usdt(const struct bpf_program *prog, long cookie, struct bpf_link **link)
12315 {
12316 char *path = NULL, *provider = NULL, *name = NULL;
12317 const char *sec_name;
12318 int n, err;
12319
12320 sec_name = bpf_program__section_name(prog);
12321 if (strcmp(sec_name, "usdt") == 0) {
12322 /* no auto-attach for just SEC("usdt") */
12323 *link = NULL;
12324 return 0;
12325 }
12326
12327 n = sscanf(sec_name, "usdt/%m[^:]:%m[^:]:%m[^:]", &path, &provider, &name);
12328 if (n != 3) {
12329 pr_warn("invalid section '%s', expected SEC(\"usdt/<path>:<provider>:<name>\")\n",
12330 sec_name);
12331 err = -EINVAL;
12332 } else {
12333 *link = bpf_program__attach_usdt(prog, -1 /* any process */, path,
12334 provider, name, NULL);
12335 err = libbpf_get_error(*link);
12336 }
12337 free(path);
12338 free(provider);
12339 free(name);
12340 return err;
12341 }
12342
12343 static int determine_tracepoint_id(const char *tp_category,
12344 const char *tp_name)
12345 {
12346 char file[PATH_MAX];
12347 int ret;
12348
12349 ret = snprintf(file, sizeof(file), "%s/events/%s/%s/id",
12350 tracefs_path(), tp_category, tp_name);
12351 if (ret < 0)
12352 return -errno;
12353 if (ret >= sizeof(file)) {
12354 pr_debug("tracepoint %s/%s path is too long\n",
12355 tp_category, tp_name);
12356 return -E2BIG;
12357 }
12358 return parse_uint_from_file(file, "%d\n");
12359 }
12360
12361 static int perf_event_open_tracepoint(const char *tp_category,
12362 const char *tp_name)
12363 {
12364 const size_t attr_sz = sizeof(struct perf_event_attr);
12365 struct perf_event_attr attr;
12366 char errmsg[STRERR_BUFSIZE];
12367 int tp_id, pfd, err;
12368
12369 tp_id = determine_tracepoint_id(tp_category, tp_name);
12370 if (tp_id < 0) {
12371 pr_warn("failed to determine tracepoint '%s/%s' perf event ID: %s\n",
12372 tp_category, tp_name,
12373 libbpf_strerror_r(tp_id, errmsg, sizeof(errmsg)));
12374 return tp_id;
12375 }
12376
12377 memset(&attr, 0, attr_sz);
12378 attr.type = PERF_TYPE_TRACEPOINT;
12379 attr.size = attr_sz;
12380 attr.config = tp_id;
12381
12382 pfd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 0 /* cpu */,
12383 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
12384 if (pfd < 0) {
12385 err = -errno;
12386 pr_warn("tracepoint '%s/%s' perf_event_open() failed: %s\n",
12387 tp_category, tp_name,
12388 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
12389 return err;
12390 }
12391 return pfd;
12392 }
12393
12394 struct bpf_link *bpf_program__attach_tracepoint_opts(const struct bpf_program *prog,
12395 const char *tp_category,
12396 const char *tp_name,
12397 const struct bpf_tracepoint_opts *opts)
12398 {
12399 DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts);
12400 char errmsg[STRERR_BUFSIZE];
12401 struct bpf_link *link;
12402 int pfd, err;
12403
12404 if (!OPTS_VALID(opts, bpf_tracepoint_opts))
12405 return libbpf_err_ptr(-EINVAL);
12406
12407 pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0);
12408
12409 pfd = perf_event_open_tracepoint(tp_category, tp_name);
12410 if (pfd < 0) {
12411 pr_warn("prog '%s': failed to create tracepoint '%s/%s' perf event: %s\n",
12412 prog->name, tp_category, tp_name,
12413 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
12414 return libbpf_err_ptr(pfd);
12415 }
12416 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts);
12417 err = libbpf_get_error(link);
12418 if (err) {
12419 close(pfd);
12420 pr_warn("prog '%s': failed to attach to tracepoint '%s/%s': %s\n",
12421 prog->name, tp_category, tp_name,
12422 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
12423 return libbpf_err_ptr(err);
12424 }
12425 return link;
12426 }
12427
12428 struct bpf_link *bpf_program__attach_tracepoint(const struct bpf_program *prog,
12429 const char *tp_category,
12430 const char *tp_name)
12431 {
12432 return bpf_program__attach_tracepoint_opts(prog, tp_category, tp_name, NULL);
12433 }
12434
12435 static int attach_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link)
12436 {
12437 char *sec_name, *tp_cat, *tp_name;
12438
12439 *link = NULL;
12440
12441 /* no auto-attach for SEC("tp") or SEC("tracepoint") */
12442 if (strcmp(prog->sec_name, "tp") == 0 || strcmp(prog->sec_name, "tracepoint") == 0)
12443 return 0;
12444
12445 sec_name = strdup(prog->sec_name);
12446 if (!sec_name)
12447 return -ENOMEM;
12448
12449 /* extract "tp/<category>/<name>" or "tracepoint/<category>/<name>" */
12450 if (str_has_pfx(prog->sec_name, "tp/"))
12451 tp_cat = sec_name + sizeof("tp/") - 1;
12452 else
12453 tp_cat = sec_name + sizeof("tracepoint/") - 1;
12454 tp_name = strchr(tp_cat, '/');
12455 if (!tp_name) {
12456 free(sec_name);
12457 return -EINVAL;
12458 }
12459 *tp_name = '\0';
12460 tp_name++;
12461
12462 *link = bpf_program__attach_tracepoint(prog, tp_cat, tp_name);
12463 free(sec_name);
12464 return libbpf_get_error(*link);
12465 }
12466
12467 struct bpf_link *
12468 bpf_program__attach_raw_tracepoint_opts(const struct bpf_program *prog,
12469 const char *tp_name,
12470 struct bpf_raw_tracepoint_opts *opts)
12471 {
12472 LIBBPF_OPTS(bpf_raw_tp_opts, raw_opts);
12473 char errmsg[STRERR_BUFSIZE];
12474 struct bpf_link *link;
12475 int prog_fd, pfd;
12476
12477 if (!OPTS_VALID(opts, bpf_raw_tracepoint_opts))
12478 return libbpf_err_ptr(-EINVAL);
12479
12480 prog_fd = bpf_program__fd(prog);
12481 if (prog_fd < 0) {
12482 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
12483 return libbpf_err_ptr(-EINVAL);
12484 }
12485
12486 link = calloc(1, sizeof(*link));
12487 if (!link)
12488 return libbpf_err_ptr(-ENOMEM);
12489 link->detach = &bpf_link__detach_fd;
12490
12491 raw_opts.tp_name = tp_name;
12492 raw_opts.cookie = OPTS_GET(opts, cookie, 0);
12493 pfd = bpf_raw_tracepoint_open_opts(prog_fd, &raw_opts);
12494 if (pfd < 0) {
12495 pfd = -errno;
12496 free(link);
12497 pr_warn("prog '%s': failed to attach to raw tracepoint '%s': %s\n",
12498 prog->name, tp_name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
12499 return libbpf_err_ptr(pfd);
12500 }
12501 link->fd = pfd;
12502 return link;
12503 }
12504
12505 struct bpf_link *bpf_program__attach_raw_tracepoint(const struct bpf_program *prog,
12506 const char *tp_name)
12507 {
12508 return bpf_program__attach_raw_tracepoint_opts(prog, tp_name, NULL);
12509 }
12510
12511 static int attach_raw_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link)
12512 {
12513 static const char *const prefixes[] = {
12514 "raw_tp",
12515 "raw_tracepoint",
12516 "raw_tp.w",
12517 "raw_tracepoint.w",
12518 };
12519 size_t i;
12520 const char *tp_name = NULL;
12521
12522 *link = NULL;
12523
12524 for (i = 0; i < ARRAY_SIZE(prefixes); i++) {
12525 size_t pfx_len;
12526
12527 if (!str_has_pfx(prog->sec_name, prefixes[i]))
12528 continue;
12529
12530 pfx_len = strlen(prefixes[i]);
12531 /* no auto-attach case of, e.g., SEC("raw_tp") */
12532 if (prog->sec_name[pfx_len] == '\0')
12533 return 0;
12534
12535 if (prog->sec_name[pfx_len] != '/')
12536 continue;
12537
12538 tp_name = prog->sec_name + pfx_len + 1;
12539 break;
12540 }
12541
12542 if (!tp_name) {
12543 pr_warn("prog '%s': invalid section name '%s'\n",
12544 prog->name, prog->sec_name);
12545 return -EINVAL;
12546 }
12547
12548 *link = bpf_program__attach_raw_tracepoint(prog, tp_name);
12549 return libbpf_get_error(*link);
12550 }
12551
12552 /* Common logic for all BPF program types that attach to a btf_id */
12553 static struct bpf_link *bpf_program__attach_btf_id(const struct bpf_program *prog,
12554 const struct bpf_trace_opts *opts)
12555 {
12556 LIBBPF_OPTS(bpf_link_create_opts, link_opts);
12557 char errmsg[STRERR_BUFSIZE];
12558 struct bpf_link *link;
12559 int prog_fd, pfd;
12560
12561 if (!OPTS_VALID(opts, bpf_trace_opts))
12562 return libbpf_err_ptr(-EINVAL);
12563
12564 prog_fd = bpf_program__fd(prog);
12565 if (prog_fd < 0) {
12566 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
12567 return libbpf_err_ptr(-EINVAL);
12568 }
12569
12570 link = calloc(1, sizeof(*link));
12571 if (!link)
12572 return libbpf_err_ptr(-ENOMEM);
12573 link->detach = &bpf_link__detach_fd;
12574
12575 /* libbpf is smart enough to redirect to BPF_RAW_TRACEPOINT_OPEN on old kernels */
12576 link_opts.tracing.cookie = OPTS_GET(opts, cookie, 0);
12577 pfd = bpf_link_create(prog_fd, 0, bpf_program__expected_attach_type(prog), &link_opts);
12578 if (pfd < 0) {
12579 pfd = -errno;
12580 free(link);
12581 pr_warn("prog '%s': failed to attach: %s\n",
12582 prog->name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
12583 return libbpf_err_ptr(pfd);
12584 }
12585 link->fd = pfd;
12586 return link;
12587 }
12588
12589 struct bpf_link *bpf_program__attach_trace(const struct bpf_program *prog)
12590 {
12591 return bpf_program__attach_btf_id(prog, NULL);
12592 }
12593
12594 struct bpf_link *bpf_program__attach_trace_opts(const struct bpf_program *prog,
12595 const struct bpf_trace_opts *opts)
12596 {
12597 return bpf_program__attach_btf_id(prog, opts);
12598 }
12599
12600 struct bpf_link *bpf_program__attach_lsm(const struct bpf_program *prog)
12601 {
12602 return bpf_program__attach_btf_id(prog, NULL);
12603 }
12604
12605 static int attach_trace(const struct bpf_program *prog, long cookie, struct bpf_link **link)
12606 {
12607 *link = bpf_program__attach_trace(prog);
12608 return libbpf_get_error(*link);
12609 }
12610
12611 static int attach_lsm(const struct bpf_program *prog, long cookie, struct bpf_link **link)
12612 {
12613 *link = bpf_program__attach_lsm(prog);
12614 return libbpf_get_error(*link);
12615 }
12616
12617 static struct bpf_link *
12618 bpf_program_attach_fd(const struct bpf_program *prog,
12619 int target_fd, const char *target_name,
12620 const struct bpf_link_create_opts *opts)
12621 {
12622 enum bpf_attach_type attach_type;
12623 char errmsg[STRERR_BUFSIZE];
12624 struct bpf_link *link;
12625 int prog_fd, link_fd;
12626
12627 prog_fd = bpf_program__fd(prog);
12628 if (prog_fd < 0) {
12629 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
12630 return libbpf_err_ptr(-EINVAL);
12631 }
12632
12633 link = calloc(1, sizeof(*link));
12634 if (!link)
12635 return libbpf_err_ptr(-ENOMEM);
12636 link->detach = &bpf_link__detach_fd;
12637
12638 attach_type = bpf_program__expected_attach_type(prog);
12639 link_fd = bpf_link_create(prog_fd, target_fd, attach_type, opts);
12640 if (link_fd < 0) {
12641 link_fd = -errno;
12642 free(link);
12643 pr_warn("prog '%s': failed to attach to %s: %s\n",
12644 prog->name, target_name,
12645 libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));
12646 return libbpf_err_ptr(link_fd);
12647 }
12648 link->fd = link_fd;
12649 return link;
12650 }
12651
12652 struct bpf_link *
12653 bpf_program__attach_cgroup(const struct bpf_program *prog, int cgroup_fd)
12654 {
12655 return bpf_program_attach_fd(prog, cgroup_fd, "cgroup", NULL);
12656 }
12657
12658 struct bpf_link *
12659 bpf_program__attach_netns(const struct bpf_program *prog, int netns_fd)
12660 {
12661 return bpf_program_attach_fd(prog, netns_fd, "netns", NULL);
12662 }
12663
12664 struct bpf_link *
12665 bpf_program__attach_sockmap(const struct bpf_program *prog, int map_fd)
12666 {
12667 return bpf_program_attach_fd(prog, map_fd, "sockmap", NULL);
12668 }
12669
12670 struct bpf_link *bpf_program__attach_xdp(const struct bpf_program *prog, int ifindex)
12671 {
12672 /* target_fd/target_ifindex use the same field in LINK_CREATE */
12673 return bpf_program_attach_fd(prog, ifindex, "xdp", NULL);
12674 }
12675
12676 struct bpf_link *
12677 bpf_program__attach_tcx(const struct bpf_program *prog, int ifindex,
12678 const struct bpf_tcx_opts *opts)
12679 {
12680 LIBBPF_OPTS(bpf_link_create_opts, link_create_opts);
12681 __u32 relative_id;
12682 int relative_fd;
12683
12684 if (!OPTS_VALID(opts, bpf_tcx_opts))
12685 return libbpf_err_ptr(-EINVAL);
12686
12687 relative_id = OPTS_GET(opts, relative_id, 0);
12688 relative_fd = OPTS_GET(opts, relative_fd, 0);
12689
12690 /* validate we don't have unexpected combinations of non-zero fields */
12691 if (!ifindex) {
12692 pr_warn("prog '%s': target netdevice ifindex cannot be zero\n",
12693 prog->name);
12694 return libbpf_err_ptr(-EINVAL);
12695 }
12696 if (relative_fd && relative_id) {
12697 pr_warn("prog '%s': relative_fd and relative_id cannot be set at the same time\n",
12698 prog->name);
12699 return libbpf_err_ptr(-EINVAL);
12700 }
12701
12702 link_create_opts.tcx.expected_revision = OPTS_GET(opts, expected_revision, 0);
12703 link_create_opts.tcx.relative_fd = relative_fd;
12704 link_create_opts.tcx.relative_id = relative_id;
12705 link_create_opts.flags = OPTS_GET(opts, flags, 0);
12706
12707 /* target_fd/target_ifindex use the same field in LINK_CREATE */
12708 return bpf_program_attach_fd(prog, ifindex, "tcx", &link_create_opts);
12709 }
12710
12711 struct bpf_link *
12712 bpf_program__attach_netkit(const struct bpf_program *prog, int ifindex,
12713 const struct bpf_netkit_opts *opts)
12714 {
12715 LIBBPF_OPTS(bpf_link_create_opts, link_create_opts);
12716 __u32 relative_id;
12717 int relative_fd;
12718
12719 if (!OPTS_VALID(opts, bpf_netkit_opts))
12720 return libbpf_err_ptr(-EINVAL);
12721
12722 relative_id = OPTS_GET(opts, relative_id, 0);
12723 relative_fd = OPTS_GET(opts, relative_fd, 0);
12724
12725 /* validate we don't have unexpected combinations of non-zero fields */
12726 if (!ifindex) {
12727 pr_warn("prog '%s': target netdevice ifindex cannot be zero\n",
12728 prog->name);
12729 return libbpf_err_ptr(-EINVAL);
12730 }
12731 if (relative_fd && relative_id) {
12732 pr_warn("prog '%s': relative_fd and relative_id cannot be set at the same time\n",
12733 prog->name);
12734 return libbpf_err_ptr(-EINVAL);
12735 }
12736
12737 link_create_opts.netkit.expected_revision = OPTS_GET(opts, expected_revision, 0);
12738 link_create_opts.netkit.relative_fd = relative_fd;
12739 link_create_opts.netkit.relative_id = relative_id;
12740 link_create_opts.flags = OPTS_GET(opts, flags, 0);
12741
12742 return bpf_program_attach_fd(prog, ifindex, "netkit", &link_create_opts);
12743 }
12744
12745 struct bpf_link *bpf_program__attach_freplace(const struct bpf_program *prog,
12746 int target_fd,
12747 const char *attach_func_name)
12748 {
12749 int btf_id;
12750
12751 if (!!target_fd != !!attach_func_name) {
12752 pr_warn("prog '%s': supply none or both of target_fd and attach_func_name\n",
12753 prog->name);
12754 return libbpf_err_ptr(-EINVAL);
12755 }
12756
12757 if (prog->type != BPF_PROG_TYPE_EXT) {
12758 pr_warn("prog '%s': only BPF_PROG_TYPE_EXT can attach as freplace",
12759 prog->name);
12760 return libbpf_err_ptr(-EINVAL);
12761 }
12762
12763 if (target_fd) {
12764 LIBBPF_OPTS(bpf_link_create_opts, target_opts);
12765
12766 btf_id = libbpf_find_prog_btf_id(attach_func_name, target_fd);
12767 if (btf_id < 0)
12768 return libbpf_err_ptr(btf_id);
12769
12770 target_opts.target_btf_id = btf_id;
12771
12772 return bpf_program_attach_fd(prog, target_fd, "freplace",
12773 &target_opts);
12774 } else {
12775 /* no target, so use raw_tracepoint_open for compatibility
12776 * with old kernels
12777 */
12778 return bpf_program__attach_trace(prog);
12779 }
12780 }
12781
12782 struct bpf_link *
12783 bpf_program__attach_iter(const struct bpf_program *prog,
12784 const struct bpf_iter_attach_opts *opts)
12785 {
12786 DECLARE_LIBBPF_OPTS(bpf_link_create_opts, link_create_opts);
12787 char errmsg[STRERR_BUFSIZE];
12788 struct bpf_link *link;
12789 int prog_fd, link_fd;
12790 __u32 target_fd = 0;
12791
12792 if (!OPTS_VALID(opts, bpf_iter_attach_opts))
12793 return libbpf_err_ptr(-EINVAL);
12794
12795 link_create_opts.iter_info = OPTS_GET(opts, link_info, (void *)0);
12796 link_create_opts.iter_info_len = OPTS_GET(opts, link_info_len, 0);
12797
12798 prog_fd = bpf_program__fd(prog);
12799 if (prog_fd < 0) {
12800 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
12801 return libbpf_err_ptr(-EINVAL);
12802 }
12803
12804 link = calloc(1, sizeof(*link));
12805 if (!link)
12806 return libbpf_err_ptr(-ENOMEM);
12807 link->detach = &bpf_link__detach_fd;
12808
12809 link_fd = bpf_link_create(prog_fd, target_fd, BPF_TRACE_ITER,
12810 &link_create_opts);
12811 if (link_fd < 0) {
12812 link_fd = -errno;
12813 free(link);
12814 pr_warn("prog '%s': failed to attach to iterator: %s\n",
12815 prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));
12816 return libbpf_err_ptr(link_fd);
12817 }
12818 link->fd = link_fd;
12819 return link;
12820 }
12821
12822 static int attach_iter(const struct bpf_program *prog, long cookie, struct bpf_link **link)
12823 {
12824 *link = bpf_program__attach_iter(prog, NULL);
12825 return libbpf_get_error(*link);
12826 }
12827
12828 struct bpf_link *bpf_program__attach_netfilter(const struct bpf_program *prog,
12829 const struct bpf_netfilter_opts *opts)
12830 {
12831 LIBBPF_OPTS(bpf_link_create_opts, lopts);
12832 struct bpf_link *link;
12833 int prog_fd, link_fd;
12834
12835 if (!OPTS_VALID(opts, bpf_netfilter_opts))
12836 return libbpf_err_ptr(-EINVAL);
12837
12838 prog_fd = bpf_program__fd(prog);
12839 if (prog_fd < 0) {
12840 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
12841 return libbpf_err_ptr(-EINVAL);
12842 }
12843
12844 link = calloc(1, sizeof(*link));
12845 if (!link)
12846 return libbpf_err_ptr(-ENOMEM);
12847
12848 link->detach = &bpf_link__detach_fd;
12849
12850 lopts.netfilter.pf = OPTS_GET(opts, pf, 0);
12851 lopts.netfilter.hooknum = OPTS_GET(opts, hooknum, 0);
12852 lopts.netfilter.priority = OPTS_GET(opts, priority, 0);
12853 lopts.netfilter.flags = OPTS_GET(opts, flags, 0);
12854
12855 link_fd = bpf_link_create(prog_fd, 0, BPF_NETFILTER, &lopts);
12856 if (link_fd < 0) {
12857 char errmsg[STRERR_BUFSIZE];
12858
12859 link_fd = -errno;
12860 free(link);
12861 pr_warn("prog '%s': failed to attach to netfilter: %s\n",
12862 prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));
12863 return libbpf_err_ptr(link_fd);
12864 }
12865 link->fd = link_fd;
12866
12867 return link;
12868 }
12869
12870 struct bpf_link *bpf_program__attach(const struct bpf_program *prog)
12871 {
12872 struct bpf_link *link = NULL;
12873 int err;
12874
12875 if (!prog->sec_def || !prog->sec_def->prog_attach_fn)
12876 return libbpf_err_ptr(-EOPNOTSUPP);
12877
12878 if (bpf_program__fd(prog) < 0) {
12879 pr_warn("prog '%s': can't attach BPF program without FD (was it loaded?)\n",
12880 prog->name);
12881 return libbpf_err_ptr(-EINVAL);
12882 }
12883
12884 err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, &link);
12885 if (err)
12886 return libbpf_err_ptr(err);
12887
12888 /* When calling bpf_program__attach() explicitly, auto-attach support
12889 * is expected to work, so NULL returned link is considered an error.
12890 * This is different for skeleton's attach, see comment in
12891 * bpf_object__attach_skeleton().
12892 */
12893 if (!link)
12894 return libbpf_err_ptr(-EOPNOTSUPP);
12895
12896 return link;
12897 }
12898
12899 struct bpf_link_struct_ops {
12900 struct bpf_link link;
12901 int map_fd;
12902 };
12903
12904 static int bpf_link__detach_struct_ops(struct bpf_link *link)
12905 {
12906 struct bpf_link_struct_ops *st_link;
12907 __u32 zero = 0;
12908
12909 st_link = container_of(link, struct bpf_link_struct_ops, link);
12910
12911 if (st_link->map_fd < 0)
12912 /* w/o a real link */
12913 return bpf_map_delete_elem(link->fd, &zero);
12914
12915 return close(link->fd);
12916 }
12917
12918 struct bpf_link *bpf_map__attach_struct_ops(const struct bpf_map *map)
12919 {
12920 struct bpf_link_struct_ops *link;
12921 __u32 zero = 0;
12922 int err, fd;
12923
12924 if (!bpf_map__is_struct_ops(map)) {
12925 pr_warn("map '%s': can't attach non-struct_ops map\n", map->name);
12926 return libbpf_err_ptr(-EINVAL);
12927 }
12928
12929 if (map->fd < 0) {
12930 pr_warn("map '%s': can't attach BPF map without FD (was it created?)\n", map->name);
12931 return libbpf_err_ptr(-EINVAL);
12932 }
12933
12934 link = calloc(1, sizeof(*link));
12935 if (!link)
12936 return libbpf_err_ptr(-EINVAL);
12937
12938 /* kern_vdata should be prepared during the loading phase. */
12939 err = bpf_map_update_elem(map->fd, &zero, map->st_ops->kern_vdata, 0);
12940 /* It can be EBUSY if the map has been used to create or
12941 * update a link before. We don't allow updating the value of
12942 * a struct_ops once it is set. That ensures that the value
12943 * never changed. So, it is safe to skip EBUSY.
12944 */
12945 if (err && (!(map->def.map_flags & BPF_F_LINK) || err != -EBUSY)) {
12946 free(link);
12947 return libbpf_err_ptr(err);
12948 }
12949
12950 link->link.detach = bpf_link__detach_struct_ops;
12951
12952 if (!(map->def.map_flags & BPF_F_LINK)) {
12953 /* w/o a real link */
12954 link->link.fd = map->fd;
12955 link->map_fd = -1;
12956 return &link->link;
12957 }
12958
12959 fd = bpf_link_create(map->fd, 0, BPF_STRUCT_OPS, NULL);
12960 if (fd < 0) {
12961 free(link);
12962 return libbpf_err_ptr(fd);
12963 }
12964
12965 link->link.fd = fd;
12966 link->map_fd = map->fd;
12967
12968 return &link->link;
12969 }
12970
12971 /*
12972 * Swap the back struct_ops of a link with a new struct_ops map.
12973 */
12974 int bpf_link__update_map(struct bpf_link *link, const struct bpf_map *map)
12975 {
12976 struct bpf_link_struct_ops *st_ops_link;
12977 __u32 zero = 0;
12978 int err;
12979
12980 if (!bpf_map__is_struct_ops(map))
12981 return -EINVAL;
12982
12983 if (map->fd < 0) {
12984 pr_warn("map '%s': can't use BPF map without FD (was it created?)\n", map->name);
12985 return -EINVAL;
12986 }
12987
12988 st_ops_link = container_of(link, struct bpf_link_struct_ops, link);
12989 /* Ensure the type of a link is correct */
12990 if (st_ops_link->map_fd < 0)
12991 return -EINVAL;
12992
12993 err = bpf_map_update_elem(map->fd, &zero, map->st_ops->kern_vdata, 0);
12994 /* It can be EBUSY if the map has been used to create or
12995 * update a link before. We don't allow updating the value of
12996 * a struct_ops once it is set. That ensures that the value
12997 * never changed. So, it is safe to skip EBUSY.
12998 */
12999 if (err && err != -EBUSY)
13000 return err;
13001
13002 err = bpf_link_update(link->fd, map->fd, NULL);
13003 if (err < 0)
13004 return err;
13005
13006 st_ops_link->map_fd = map->fd;
13007
13008 return 0;
13009 }
13010
13011 typedef enum bpf_perf_event_ret (*bpf_perf_event_print_t)(struct perf_event_header *hdr,
13012 void *private_data);
13013
13014 static enum bpf_perf_event_ret
13015 perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size,
13016 void **copy_mem, size_t *copy_size,
13017 bpf_perf_event_print_t fn, void *private_data)
13018 {
13019 struct perf_event_mmap_page *header = mmap_mem;
13020 __u64 data_head = ring_buffer_read_head(header);
13021 __u64 data_tail = header->data_tail;
13022 void *base = ((__u8 *)header) + page_size;
13023 int ret = LIBBPF_PERF_EVENT_CONT;
13024 struct perf_event_header *ehdr;
13025 size_t ehdr_size;
13026
13027 while (data_head != data_tail) {
13028 ehdr = base + (data_tail & (mmap_size - 1));
13029 ehdr_size = ehdr->size;
13030
13031 if (((void *)ehdr) + ehdr_size > base + mmap_size) {
13032 void *copy_start = ehdr;
13033 size_t len_first = base + mmap_size - copy_start;
13034 size_t len_secnd = ehdr_size - len_first;
13035
13036 if (*copy_size < ehdr_size) {
13037 free(*copy_mem);
13038 *copy_mem = malloc(ehdr_size);
13039 if (!*copy_mem) {
13040 *copy_size = 0;
13041 ret = LIBBPF_PERF_EVENT_ERROR;
13042 break;
13043 }
13044 *copy_size = ehdr_size;
13045 }
13046
13047 memcpy(*copy_mem, copy_start, len_first);
13048 memcpy(*copy_mem + len_first, base, len_secnd);
13049 ehdr = *copy_mem;
13050 }
13051
13052 ret = fn(ehdr, private_data);
13053 data_tail += ehdr_size;
13054 if (ret != LIBBPF_PERF_EVENT_CONT)
13055 break;
13056 }
13057
13058 ring_buffer_write_tail(header, data_tail);
13059 return libbpf_err(ret);
13060 }
13061
13062 struct perf_buffer;
13063
13064 struct perf_buffer_params {
13065 struct perf_event_attr *attr;
13066 /* if event_cb is specified, it takes precendence */
13067 perf_buffer_event_fn event_cb;
13068 /* sample_cb and lost_cb are higher-level common-case callbacks */
13069 perf_buffer_sample_fn sample_cb;
13070 perf_buffer_lost_fn lost_cb;
13071 void *ctx;
13072 int cpu_cnt;
13073 int *cpus;
13074 int *map_keys;
13075 };
13076
13077 struct perf_cpu_buf {
13078 struct perf_buffer *pb;
13079 void *base; /* mmap()'ed memory */
13080 void *buf; /* for reconstructing segmented data */
13081 size_t buf_size;
13082 int fd;
13083 int cpu;
13084 int map_key;
13085 };
13086
13087 struct perf_buffer {
13088 perf_buffer_event_fn event_cb;
13089 perf_buffer_sample_fn sample_cb;
13090 perf_buffer_lost_fn lost_cb;
13091 void *ctx; /* passed into callbacks */
13092
13093 size_t page_size;
13094 size_t mmap_size;
13095 struct perf_cpu_buf **cpu_bufs;
13096 struct epoll_event *events;
13097 int cpu_cnt; /* number of allocated CPU buffers */
13098 int epoll_fd; /* perf event FD */
13099 int map_fd; /* BPF_MAP_TYPE_PERF_EVENT_ARRAY BPF map FD */
13100 };
13101
13102 static void perf_buffer__free_cpu_buf(struct perf_buffer *pb,
13103 struct perf_cpu_buf *cpu_buf)
13104 {
13105 if (!cpu_buf)
13106 return;
13107 if (cpu_buf->base &&
13108 munmap(cpu_buf->base, pb->mmap_size + pb->page_size))
13109 pr_warn("failed to munmap cpu_buf #%d\n", cpu_buf->cpu);
13110 if (cpu_buf->fd >= 0) {
13111 ioctl(cpu_buf->fd, PERF_EVENT_IOC_DISABLE, 0);
13112 close(cpu_buf->fd);
13113 }
13114 free(cpu_buf->buf);
13115 free(cpu_buf);
13116 }
13117
13118 void perf_buffer__free(struct perf_buffer *pb)
13119 {
13120 int i;
13121
13122 if (IS_ERR_OR_NULL(pb))
13123 return;
13124 if (pb->cpu_bufs) {
13125 for (i = 0; i < pb->cpu_cnt; i++) {
13126 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];
13127
13128 if (!cpu_buf)
13129 continue;
13130
13131 bpf_map_delete_elem(pb->map_fd, &cpu_buf->map_key);
13132 perf_buffer__free_cpu_buf(pb, cpu_buf);
13133 }
13134 free(pb->cpu_bufs);
13135 }
13136 if (pb->epoll_fd >= 0)
13137 close(pb->epoll_fd);
13138 free(pb->events);
13139 free(pb);
13140 }
13141
13142 static struct perf_cpu_buf *
13143 perf_buffer__open_cpu_buf(struct perf_buffer *pb, struct perf_event_attr *attr,
13144 int cpu, int map_key)
13145 {
13146 struct perf_cpu_buf *cpu_buf;
13147 char msg[STRERR_BUFSIZE];
13148 int err;
13149
13150 cpu_buf = calloc(1, sizeof(*cpu_buf));
13151 if (!cpu_buf)
13152 return ERR_PTR(-ENOMEM);
13153
13154 cpu_buf->pb = pb;
13155 cpu_buf->cpu = cpu;
13156 cpu_buf->map_key = map_key;
13157
13158 cpu_buf->fd = syscall(__NR_perf_event_open, attr, -1 /* pid */, cpu,
13159 -1, PERF_FLAG_FD_CLOEXEC);
13160 if (cpu_buf->fd < 0) {
13161 err = -errno;
13162 pr_warn("failed to open perf buffer event on cpu #%d: %s\n",
13163 cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
13164 goto error;
13165 }
13166
13167 cpu_buf->base = mmap(NULL, pb->mmap_size + pb->page_size,
13168 PROT_READ | PROT_WRITE, MAP_SHARED,
13169 cpu_buf->fd, 0);
13170 if (cpu_buf->base == MAP_FAILED) {
13171 cpu_buf->base = NULL;
13172 err = -errno;
13173 pr_warn("failed to mmap perf buffer on cpu #%d: %s\n",
13174 cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
13175 goto error;
13176 }
13177
13178 if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) {
13179 err = -errno;
13180 pr_warn("failed to enable perf buffer event on cpu #%d: %s\n",
13181 cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
13182 goto error;
13183 }
13184
13185 return cpu_buf;
13186
13187 error:
13188 perf_buffer__free_cpu_buf(pb, cpu_buf);
13189 return (struct perf_cpu_buf *)ERR_PTR(err);
13190 }
13191
13192 static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt,
13193 struct perf_buffer_params *p);
13194
13195 struct perf_buffer *perf_buffer__new(int map_fd, size_t page_cnt,
13196 perf_buffer_sample_fn sample_cb,
13197 perf_buffer_lost_fn lost_cb,
13198 void *ctx,
13199 const struct perf_buffer_opts *opts)
13200 {
13201 const size_t attr_sz = sizeof(struct perf_event_attr);
13202 struct perf_buffer_params p = {};
13203 struct perf_event_attr attr;
13204 __u32 sample_period;
13205
13206 if (!OPTS_VALID(opts, perf_buffer_opts))
13207 return libbpf_err_ptr(-EINVAL);
13208
13209 sample_period = OPTS_GET(opts, sample_period, 1);
13210 if (!sample_period)
13211 sample_period = 1;
13212
13213 memset(&attr, 0, attr_sz);
13214 attr.size = attr_sz;
13215 attr.config = PERF_COUNT_SW_BPF_OUTPUT;
13216 attr.type = PERF_TYPE_SOFTWARE;
13217 attr.sample_type = PERF_SAMPLE_RAW;
13218 attr.sample_period = sample_period;
13219 attr.wakeup_events = sample_period;
13220
13221 p.attr = &attr;
13222 p.sample_cb = sample_cb;
13223 p.lost_cb = lost_cb;
13224 p.ctx = ctx;
13225
13226 return libbpf_ptr(__perf_buffer__new(map_fd, page_cnt, &p));
13227 }
13228
13229 struct perf_buffer *perf_buffer__new_raw(int map_fd, size_t page_cnt,
13230 struct perf_event_attr *attr,
13231 perf_buffer_event_fn event_cb, void *ctx,
13232 const struct perf_buffer_raw_opts *opts)
13233 {
13234 struct perf_buffer_params p = {};
13235
13236 if (!attr)
13237 return libbpf_err_ptr(-EINVAL);
13238
13239 if (!OPTS_VALID(opts, perf_buffer_raw_opts))
13240 return libbpf_err_ptr(-EINVAL);
13241
13242 p.attr = attr;
13243 p.event_cb = event_cb;
13244 p.ctx = ctx;
13245 p.cpu_cnt = OPTS_GET(opts, cpu_cnt, 0);
13246 p.cpus = OPTS_GET(opts, cpus, NULL);
13247 p.map_keys = OPTS_GET(opts, map_keys, NULL);
13248
13249 return libbpf_ptr(__perf_buffer__new(map_fd, page_cnt, &p));
13250 }
13251
13252 static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt,
13253 struct perf_buffer_params *p)
13254 {
13255 const char *online_cpus_file = "/sys/devices/system/cpu/online";
13256 struct bpf_map_info map;
13257 char msg[STRERR_BUFSIZE];
13258 struct perf_buffer *pb;
13259 bool *online = NULL;
13260 __u32 map_info_len;
13261 int err, i, j, n;
13262
13263 if (page_cnt == 0 || (page_cnt & (page_cnt - 1))) {
13264 pr_warn("page count should be power of two, but is %zu\n",
13265 page_cnt);
13266 return ERR_PTR(-EINVAL);
13267 }
13268
13269 /* best-effort sanity checks */
13270 memset(&map, 0, sizeof(map));
13271 map_info_len = sizeof(map);
13272 err = bpf_map_get_info_by_fd(map_fd, &map, &map_info_len);
13273 if (err) {
13274 err = -errno;
13275 /* if BPF_OBJ_GET_INFO_BY_FD is supported, will return
13276 * -EBADFD, -EFAULT, or -E2BIG on real error
13277 */
13278 if (err != -EINVAL) {
13279 pr_warn("failed to get map info for map FD %d: %s\n",
13280 map_fd, libbpf_strerror_r(err, msg, sizeof(msg)));
13281 return ERR_PTR(err);
13282 }
13283 pr_debug("failed to get map info for FD %d; API not supported? Ignoring...\n",
13284 map_fd);
13285 } else {
13286 if (map.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) {
13287 pr_warn("map '%s' should be BPF_MAP_TYPE_PERF_EVENT_ARRAY\n",
13288 map.name);
13289 return ERR_PTR(-EINVAL);
13290 }
13291 }
13292
13293 pb = calloc(1, sizeof(*pb));
13294 if (!pb)
13295 return ERR_PTR(-ENOMEM);
13296
13297 pb->event_cb = p->event_cb;
13298 pb->sample_cb = p->sample_cb;
13299 pb->lost_cb = p->lost_cb;
13300 pb->ctx = p->ctx;
13301
13302 pb->page_size = getpagesize();
13303 pb->mmap_size = pb->page_size * page_cnt;
13304 pb->map_fd = map_fd;
13305
13306 pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC);
13307 if (pb->epoll_fd < 0) {
13308 err = -errno;
13309 pr_warn("failed to create epoll instance: %s\n",
13310 libbpf_strerror_r(err, msg, sizeof(msg)));
13311 goto error;
13312 }
13313
13314 if (p->cpu_cnt > 0) {
13315 pb->cpu_cnt = p->cpu_cnt;
13316 } else {
13317 pb->cpu_cnt = libbpf_num_possible_cpus();
13318 if (pb->cpu_cnt < 0) {
13319 err = pb->cpu_cnt;
13320 goto error;
13321 }
13322 if (map.max_entries && map.max_entries < pb->cpu_cnt)
13323 pb->cpu_cnt = map.max_entries;
13324 }
13325
13326 pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events));
13327 if (!pb->events) {
13328 err = -ENOMEM;
13329 pr_warn("failed to allocate events: out of memory\n");
13330 goto error;
13331 }
13332 pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs));
13333 if (!pb->cpu_bufs) {
13334 err = -ENOMEM;
13335 pr_warn("failed to allocate buffers: out of memory\n");
13336 goto error;
13337 }
13338
13339 err = parse_cpu_mask_file(online_cpus_file, &online, &n);
13340 if (err) {
13341 pr_warn("failed to get online CPU mask: %d\n", err);
13342 goto error;
13343 }
13344
13345 for (i = 0, j = 0; i < pb->cpu_cnt; i++) {
13346 struct perf_cpu_buf *cpu_buf;
13347 int cpu, map_key;
13348
13349 cpu = p->cpu_cnt > 0 ? p->cpus[i] : i;
13350 map_key = p->cpu_cnt > 0 ? p->map_keys[i] : i;
13351
13352 /* in case user didn't explicitly requested particular CPUs to
13353 * be attached to, skip offline/not present CPUs
13354 */
13355 if (p->cpu_cnt <= 0 && (cpu >= n || !online[cpu]))
13356 continue;
13357
13358 cpu_buf = perf_buffer__open_cpu_buf(pb, p->attr, cpu, map_key);
13359 if (IS_ERR(cpu_buf)) {
13360 err = PTR_ERR(cpu_buf);
13361 goto error;
13362 }
13363
13364 pb->cpu_bufs[j] = cpu_buf;
13365
13366 err = bpf_map_update_elem(pb->map_fd, &map_key,
13367 &cpu_buf->fd, 0);
13368 if (err) {
13369 err = -errno;
13370 pr_warn("failed to set cpu #%d, key %d -> perf FD %d: %s\n",
13371 cpu, map_key, cpu_buf->fd,
13372 libbpf_strerror_r(err, msg, sizeof(msg)));
13373 goto error;
13374 }
13375
13376 pb->events[j].events = EPOLLIN;
13377 pb->events[j].data.ptr = cpu_buf;
13378 if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd,
13379 &pb->events[j]) < 0) {
13380 err = -errno;
13381 pr_warn("failed to epoll_ctl cpu #%d perf FD %d: %s\n",
13382 cpu, cpu_buf->fd,
13383 libbpf_strerror_r(err, msg, sizeof(msg)));
13384 goto error;
13385 }
13386 j++;
13387 }
13388 pb->cpu_cnt = j;
13389 free(online);
13390
13391 return pb;
13392
13393 error:
13394 free(online);
13395 if (pb)
13396 perf_buffer__free(pb);
13397 return ERR_PTR(err);
13398 }
13399
13400 struct perf_sample_raw {
13401 struct perf_event_header header;
13402 uint32_t size;
13403 char data[];
13404 };
13405
13406 struct perf_sample_lost {
13407 struct perf_event_header header;
13408 uint64_t id;
13409 uint64_t lost;
13410 uint64_t sample_id;
13411 };
13412
13413 static enum bpf_perf_event_ret
13414 perf_buffer__process_record(struct perf_event_header *e, void *ctx)
13415 {
13416 struct perf_cpu_buf *cpu_buf = ctx;
13417 struct perf_buffer *pb = cpu_buf->pb;
13418 void *data = e;
13419
13420 /* user wants full control over parsing perf event */
13421 if (pb->event_cb)
13422 return pb->event_cb(pb->ctx, cpu_buf->cpu, e);
13423
13424 switch (e->type) {
13425 case PERF_RECORD_SAMPLE: {
13426 struct perf_sample_raw *s = data;
13427
13428 if (pb->sample_cb)
13429 pb->sample_cb(pb->ctx, cpu_buf->cpu, s->data, s->size);
13430 break;
13431 }
13432 case PERF_RECORD_LOST: {
13433 struct perf_sample_lost *s = data;
13434
13435 if (pb->lost_cb)
13436 pb->lost_cb(pb->ctx, cpu_buf->cpu, s->lost);
13437 break;
13438 }
13439 default:
13440 pr_warn("unknown perf sample type %d\n", e->type);
13441 return LIBBPF_PERF_EVENT_ERROR;
13442 }
13443 return LIBBPF_PERF_EVENT_CONT;
13444 }
13445
13446 static int perf_buffer__process_records(struct perf_buffer *pb,
13447 struct perf_cpu_buf *cpu_buf)
13448 {
13449 enum bpf_perf_event_ret ret;
13450
13451 ret = perf_event_read_simple(cpu_buf->base, pb->mmap_size,
13452 pb->page_size, &cpu_buf->buf,
13453 &cpu_buf->buf_size,
13454 perf_buffer__process_record, cpu_buf);
13455 if (ret != LIBBPF_PERF_EVENT_CONT)
13456 return ret;
13457 return 0;
13458 }
13459
13460 int perf_buffer__epoll_fd(const struct perf_buffer *pb)
13461 {
13462 return pb->epoll_fd;
13463 }
13464
13465 int perf_buffer__poll(struct perf_buffer *pb, int timeout_ms)
13466 {
13467 int i, cnt, err;
13468
13469 cnt = epoll_wait(pb->epoll_fd, pb->events, pb->cpu_cnt, timeout_ms);
13470 if (cnt < 0)
13471 return -errno;
13472
13473 for (i = 0; i < cnt; i++) {
13474 struct perf_cpu_buf *cpu_buf = pb->events[i].data.ptr;
13475
13476 err = perf_buffer__process_records(pb, cpu_buf);
13477 if (err) {
13478 pr_warn("error while processing records: %d\n", err);
13479 return libbpf_err(err);
13480 }
13481 }
13482 return cnt;
13483 }
13484
13485 /* Return number of PERF_EVENT_ARRAY map slots set up by this perf_buffer
13486 * manager.
13487 */
13488 size_t perf_buffer__buffer_cnt(const struct perf_buffer *pb)
13489 {
13490 return pb->cpu_cnt;
13491 }
13492
13493 /*
13494 * Return perf_event FD of a ring buffer in *buf_idx* slot of
13495 * PERF_EVENT_ARRAY BPF map. This FD can be polled for new data using
13496 * select()/poll()/epoll() Linux syscalls.
13497 */
13498 int perf_buffer__buffer_fd(const struct perf_buffer *pb, size_t buf_idx)
13499 {
13500 struct perf_cpu_buf *cpu_buf;
13501
13502 if (buf_idx >= pb->cpu_cnt)
13503 return libbpf_err(-EINVAL);
13504
13505 cpu_buf = pb->cpu_bufs[buf_idx];
13506 if (!cpu_buf)
13507 return libbpf_err(-ENOENT);
13508
13509 return cpu_buf->fd;
13510 }
13511
13512 int perf_buffer__buffer(struct perf_buffer *pb, int buf_idx, void **buf, size_t *buf_size)
13513 {
13514 struct perf_cpu_buf *cpu_buf;
13515
13516 if (buf_idx >= pb->cpu_cnt)
13517 return libbpf_err(-EINVAL);
13518
13519 cpu_buf = pb->cpu_bufs[buf_idx];
13520 if (!cpu_buf)
13521 return libbpf_err(-ENOENT);
13522
13523 *buf = cpu_buf->base;
13524 *buf_size = pb->mmap_size;
13525 return 0;
13526 }
13527
13528 /*
13529 * Consume data from perf ring buffer corresponding to slot *buf_idx* in
13530 * PERF_EVENT_ARRAY BPF map without waiting/polling. If there is no data to
13531 * consume, do nothing and return success.
13532 * Returns:
13533 * - 0 on success;
13534 * - <0 on failure.
13535 */
13536 int perf_buffer__consume_buffer(struct perf_buffer *pb, size_t buf_idx)
13537 {
13538 struct perf_cpu_buf *cpu_buf;
13539
13540 if (buf_idx >= pb->cpu_cnt)
13541 return libbpf_err(-EINVAL);
13542
13543 cpu_buf = pb->cpu_bufs[buf_idx];
13544 if (!cpu_buf)
13545 return libbpf_err(-ENOENT);
13546
13547 return perf_buffer__process_records(pb, cpu_buf);
13548 }
13549
13550 int perf_buffer__consume(struct perf_buffer *pb)
13551 {
13552 int i, err;
13553
13554 for (i = 0; i < pb->cpu_cnt; i++) {
13555 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];
13556
13557 if (!cpu_buf)
13558 continue;
13559
13560 err = perf_buffer__process_records(pb, cpu_buf);
13561 if (err) {
13562 pr_warn("perf_buffer: failed to process records in buffer #%d: %d\n", i, err);
13563 return libbpf_err(err);
13564 }
13565 }
13566 return 0;
13567 }
13568
13569 int bpf_program__set_attach_target(struct bpf_program *prog,
13570 int attach_prog_fd,
13571 const char *attach_func_name)
13572 {
13573 int btf_obj_fd = 0, btf_id = 0, err;
13574
13575 if (!prog || attach_prog_fd < 0)
13576 return libbpf_err(-EINVAL);
13577
13578 if (prog->obj->loaded)
13579 return libbpf_err(-EINVAL);
13580
13581 if (attach_prog_fd && !attach_func_name) {
13582 /* remember attach_prog_fd and let bpf_program__load() find
13583 * BTF ID during the program load
13584 */
13585 prog->attach_prog_fd = attach_prog_fd;
13586 return 0;
13587 }
13588
13589 if (attach_prog_fd) {
13590 btf_id = libbpf_find_prog_btf_id(attach_func_name,
13591 attach_prog_fd);
13592 if (btf_id < 0)
13593 return libbpf_err(btf_id);
13594 } else {
13595 if (!attach_func_name)
13596 return libbpf_err(-EINVAL);
13597
13598 /* load btf_vmlinux, if not yet */
13599 err = bpf_object__load_vmlinux_btf(prog->obj, true);
13600 if (err)
13601 return libbpf_err(err);
13602 err = find_kernel_btf_id(prog->obj, attach_func_name,
13603 prog->expected_attach_type,
13604 &btf_obj_fd, &btf_id);
13605 if (err)
13606 return libbpf_err(err);
13607 }
13608
13609 prog->attach_btf_id = btf_id;
13610 prog->attach_btf_obj_fd = btf_obj_fd;
13611 prog->attach_prog_fd = attach_prog_fd;
13612 return 0;
13613 }
13614
13615 int parse_cpu_mask_str(const char *s, bool **mask, int *mask_sz)
13616 {
13617 int err = 0, n, len, start, end = -1;
13618 bool *tmp;
13619
13620 *mask = NULL;
13621 *mask_sz = 0;
13622
13623 /* Each sub string separated by ',' has format \d+-\d+ or \d+ */
13624 while (*s) {
13625 if (*s == ',' || *s == '\n') {
13626 s++;
13627 continue;
13628 }
13629 n = sscanf(s, "%d%n-%d%n", &start, &len, &end, &len);
13630 if (n <= 0 || n > 2) {
13631 pr_warn("Failed to get CPU range %s: %d\n", s, n);
13632 err = -EINVAL;
13633 goto cleanup;
13634 } else if (n == 1) {
13635 end = start;
13636 }
13637 if (start < 0 || start > end) {
13638 pr_warn("Invalid CPU range [%d,%d] in %s\n",
13639 start, end, s);
13640 err = -EINVAL;
13641 goto cleanup;
13642 }
13643 tmp = realloc(*mask, end + 1);
13644 if (!tmp) {
13645 err = -ENOMEM;
13646 goto cleanup;
13647 }
13648 *mask = tmp;
13649 memset(tmp + *mask_sz, 0, start - *mask_sz);
13650 memset(tmp + start, 1, end - start + 1);
13651 *mask_sz = end + 1;
13652 s += len;
13653 }
13654 if (!*mask_sz) {
13655 pr_warn("Empty CPU range\n");
13656 return -EINVAL;
13657 }
13658 return 0;
13659 cleanup:
13660 free(*mask);
13661 *mask = NULL;
13662 return err;
13663 }
13664
13665 int parse_cpu_mask_file(const char *fcpu, bool **mask, int *mask_sz)
13666 {
13667 int fd, err = 0, len;
13668 char buf[128];
13669
13670 fd = open(fcpu, O_RDONLY | O_CLOEXEC);
13671 if (fd < 0) {
13672 err = -errno;
13673 pr_warn("Failed to open cpu mask file %s: %d\n", fcpu, err);
13674 return err;
13675 }
13676 len = read(fd, buf, sizeof(buf));
13677 close(fd);
13678 if (len <= 0) {
13679 err = len ? -errno : -EINVAL;
13680 pr_warn("Failed to read cpu mask from %s: %d\n", fcpu, err);
13681 return err;
13682 }
13683 if (len >= sizeof(buf)) {
13684 pr_warn("CPU mask is too big in file %s\n", fcpu);
13685 return -E2BIG;
13686 }
13687 buf[len] = '\0';
13688
13689 return parse_cpu_mask_str(buf, mask, mask_sz);
13690 }
13691
13692 int libbpf_num_possible_cpus(void)
13693 {
13694 static const char *fcpu = "/sys/devices/system/cpu/possible";
13695 static int cpus;
13696 int err, n, i, tmp_cpus;
13697 bool *mask;
13698
13699 tmp_cpus = READ_ONCE(cpus);
13700 if (tmp_cpus > 0)
13701 return tmp_cpus;
13702
13703 err = parse_cpu_mask_file(fcpu, &mask, &n);
13704 if (err)
13705 return libbpf_err(err);
13706
13707 tmp_cpus = 0;
13708 for (i = 0; i < n; i++) {
13709 if (mask[i])
13710 tmp_cpus++;
13711 }
13712 free(mask);
13713
13714 WRITE_ONCE(cpus, tmp_cpus);
13715 return tmp_cpus;
13716 }
13717
13718 static int populate_skeleton_maps(const struct bpf_object *obj,
13719 struct bpf_map_skeleton *maps,
13720 size_t map_cnt, size_t map_skel_sz)
13721 {
13722 int i;
13723
13724 for (i = 0; i < map_cnt; i++) {
13725 struct bpf_map_skeleton *map_skel = (void *)maps + i * map_skel_sz;
13726 struct bpf_map **map = map_skel->map;
13727 const char *name = map_skel->name;
13728 void **mmaped = map_skel->mmaped;
13729
13730 *map = bpf_object__find_map_by_name(obj, name);
13731 if (!*map) {
13732 pr_warn("failed to find skeleton map '%s'\n", name);
13733 return -ESRCH;
13734 }
13735
13736 /* externs shouldn't be pre-setup from user code */
13737 if (mmaped && (*map)->libbpf_type != LIBBPF_MAP_KCONFIG)
13738 *mmaped = (*map)->mmaped;
13739 }
13740 return 0;
13741 }
13742
13743 static int populate_skeleton_progs(const struct bpf_object *obj,
13744 struct bpf_prog_skeleton *progs,
13745 size_t prog_cnt, size_t prog_skel_sz)
13746 {
13747 int i;
13748
13749 for (i = 0; i < prog_cnt; i++) {
13750 struct bpf_prog_skeleton *prog_skel = (void *)progs + i * prog_skel_sz;
13751 struct bpf_program **prog = prog_skel->prog;
13752 const char *name = prog_skel->name;
13753
13754 *prog = bpf_object__find_program_by_name(obj, name);
13755 if (!*prog) {
13756 pr_warn("failed to find skeleton program '%s'\n", name);
13757 return -ESRCH;
13758 }
13759 }
13760 return 0;
13761 }
13762
13763 int bpf_object__open_skeleton(struct bpf_object_skeleton *s,
13764 const struct bpf_object_open_opts *opts)
13765 {
13766 struct bpf_object *obj;
13767 int err;
13768
13769 obj = bpf_object_open(NULL, s->data, s->data_sz, s->name, opts);
13770 if (IS_ERR(obj)) {
13771 err = PTR_ERR(obj);
13772 pr_warn("failed to initialize skeleton BPF object '%s': %d\n", s->name, err);
13773 return libbpf_err(err);
13774 }
13775
13776 *s->obj = obj;
13777 err = populate_skeleton_maps(obj, s->maps, s->map_cnt, s->map_skel_sz);
13778 if (err) {
13779 pr_warn("failed to populate skeleton maps for '%s': %d\n", s->name, err);
13780 return libbpf_err(err);
13781 }
13782
13783 err = populate_skeleton_progs(obj, s->progs, s->prog_cnt, s->prog_skel_sz);
13784 if (err) {
13785 pr_warn("failed to populate skeleton progs for '%s': %d\n", s->name, err);
13786 return libbpf_err(err);
13787 }
13788
13789 return 0;
13790 }
13791
13792 int bpf_object__open_subskeleton(struct bpf_object_subskeleton *s)
13793 {
13794 int err, len, var_idx, i;
13795 const char *var_name;
13796 const struct bpf_map *map;
13797 struct btf *btf;
13798 __u32 map_type_id;
13799 const struct btf_type *map_type, *var_type;
13800 const struct bpf_var_skeleton *var_skel;
13801 struct btf_var_secinfo *var;
13802
13803 if (!s->obj)
13804 return libbpf_err(-EINVAL);
13805
13806 btf = bpf_object__btf(s->obj);
13807 if (!btf) {
13808 pr_warn("subskeletons require BTF at runtime (object %s)\n",
13809 bpf_object__name(s->obj));
13810 return libbpf_err(-errno);
13811 }
13812
13813 err = populate_skeleton_maps(s->obj, s->maps, s->map_cnt, s->map_skel_sz);
13814 if (err) {
13815 pr_warn("failed to populate subskeleton maps: %d\n", err);
13816 return libbpf_err(err);
13817 }
13818
13819 err = populate_skeleton_progs(s->obj, s->progs, s->prog_cnt, s->prog_skel_sz);
13820 if (err) {
13821 pr_warn("failed to populate subskeleton maps: %d\n", err);
13822 return libbpf_err(err);
13823 }
13824
13825 for (var_idx = 0; var_idx < s->var_cnt; var_idx++) {
13826 var_skel = (void *)s->vars + var_idx * s->var_skel_sz;
13827 map = *var_skel->map;
13828 map_type_id = bpf_map__btf_value_type_id(map);
13829 map_type = btf__type_by_id(btf, map_type_id);
13830
13831 if (!btf_is_datasec(map_type)) {
13832 pr_warn("type for map '%1$s' is not a datasec: %2$s",
13833 bpf_map__name(map),
13834 __btf_kind_str(btf_kind(map_type)));
13835 return libbpf_err(-EINVAL);
13836 }
13837
13838 len = btf_vlen(map_type);
13839 var = btf_var_secinfos(map_type);
13840 for (i = 0; i < len; i++, var++) {
13841 var_type = btf__type_by_id(btf, var->type);
13842 var_name = btf__name_by_offset(btf, var_type->name_off);
13843 if (strcmp(var_name, var_skel->name) == 0) {
13844 *var_skel->addr = map->mmaped + var->offset;
13845 break;
13846 }
13847 }
13848 }
13849 return 0;
13850 }
13851
13852 void bpf_object__destroy_subskeleton(struct bpf_object_subskeleton *s)
13853 {
13854 if (!s)
13855 return;
13856 free(s->maps);
13857 free(s->progs);
13858 free(s->vars);
13859 free(s);
13860 }
13861
13862 int bpf_object__load_skeleton(struct bpf_object_skeleton *s)
13863 {
13864 int i, err;
13865
13866 err = bpf_object__load(*s->obj);
13867 if (err) {
13868 pr_warn("failed to load BPF skeleton '%s': %d\n", s->name, err);
13869 return libbpf_err(err);
13870 }
13871
13872 for (i = 0; i < s->map_cnt; i++) {
13873 struct bpf_map_skeleton *map_skel = (void *)s->maps + i * s->map_skel_sz;
13874 struct bpf_map *map = *map_skel->map;
13875 size_t mmap_sz = bpf_map_mmap_sz(map);
13876 int prot, map_fd = map->fd;
13877 void **mmaped = map_skel->mmaped;
13878
13879 if (!mmaped)
13880 continue;
13881
13882 if (!(map->def.map_flags & BPF_F_MMAPABLE)) {
13883 *mmaped = NULL;
13884 continue;
13885 }
13886
13887 if (map->def.type == BPF_MAP_TYPE_ARENA) {
13888 *mmaped = map->mmaped;
13889 continue;
13890 }
13891
13892 if (map->def.map_flags & BPF_F_RDONLY_PROG)
13893 prot = PROT_READ;
13894 else
13895 prot = PROT_READ | PROT_WRITE;
13896
13897 /* Remap anonymous mmap()-ed "map initialization image" as
13898 * a BPF map-backed mmap()-ed memory, but preserving the same
13899 * memory address. This will cause kernel to change process'
13900 * page table to point to a different piece of kernel memory,
13901 * but from userspace point of view memory address (and its
13902 * contents, being identical at this point) will stay the
13903 * same. This mapping will be released by bpf_object__close()
13904 * as per normal clean up procedure, so we don't need to worry
13905 * about it from skeleton's clean up perspective.
13906 */
13907 *mmaped = mmap(map->mmaped, mmap_sz, prot, MAP_SHARED | MAP_FIXED, map_fd, 0);
13908 if (*mmaped == MAP_FAILED) {
13909 err = -errno;
13910 *mmaped = NULL;
13911 pr_warn("failed to re-mmap() map '%s': %d\n",
13912 bpf_map__name(map), err);
13913 return libbpf_err(err);
13914 }
13915 }
13916
13917 return 0;
13918 }
13919
13920 int bpf_object__attach_skeleton(struct bpf_object_skeleton *s)
13921 {
13922 int i, err;
13923
13924 for (i = 0; i < s->prog_cnt; i++) {
13925 struct bpf_prog_skeleton *prog_skel = (void *)s->progs + i * s->prog_skel_sz;
13926 struct bpf_program *prog = *prog_skel->prog;
13927 struct bpf_link **link = prog_skel->link;
13928
13929 if (!prog->autoload || !prog->autoattach)
13930 continue;
13931
13932 /* auto-attaching not supported for this program */
13933 if (!prog->sec_def || !prog->sec_def->prog_attach_fn)
13934 continue;
13935
13936 /* if user already set the link manually, don't attempt auto-attach */
13937 if (*link)
13938 continue;
13939
13940 err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, link);
13941 if (err) {
13942 pr_warn("prog '%s': failed to auto-attach: %d\n",
13943 bpf_program__name(prog), err);
13944 return libbpf_err(err);
13945 }
13946
13947 /* It's possible that for some SEC() definitions auto-attach
13948 * is supported in some cases (e.g., if definition completely
13949 * specifies target information), but is not in other cases.
13950 * SEC("uprobe") is one such case. If user specified target
13951 * binary and function name, such BPF program can be
13952 * auto-attached. But if not, it shouldn't trigger skeleton's
13953 * attach to fail. It should just be skipped.
13954 * attach_fn signals such case with returning 0 (no error) and
13955 * setting link to NULL.
13956 */
13957 }
13958
13959
13960 for (i = 0; i < s->map_cnt; i++) {
13961 struct bpf_map_skeleton *map_skel = (void *)s->maps + i * s->map_skel_sz;
13962 struct bpf_map *map = *map_skel->map;
13963 struct bpf_link **link;
13964
13965 if (!map->autocreate || !map->autoattach)
13966 continue;
13967
13968 /* only struct_ops maps can be attached */
13969 if (!bpf_map__is_struct_ops(map))
13970 continue;
13971
13972 /* skeleton is created with earlier version of bpftool, notify user */
13973 if (s->map_skel_sz < offsetofend(struct bpf_map_skeleton, link)) {
13974 pr_warn("map '%s': BPF skeleton version is old, skipping map auto-attachment...\n",
13975 bpf_map__name(map));
13976 continue;
13977 }
13978
13979 link = map_skel->link;
13980 if (*link)
13981 continue;
13982
13983 *link = bpf_map__attach_struct_ops(map);
13984 if (!*link) {
13985 err = -errno;
13986 pr_warn("map '%s': failed to auto-attach: %d\n", bpf_map__name(map), err);
13987 return libbpf_err(err);
13988 }
13989 }
13990
13991 return 0;
13992 }
13993
13994 void bpf_object__detach_skeleton(struct bpf_object_skeleton *s)
13995 {
13996 int i;
13997
13998 for (i = 0; i < s->prog_cnt; i++) {
13999 struct bpf_prog_skeleton *prog_skel = (void *)s->progs + i * s->prog_skel_sz;
14000 struct bpf_link **link = prog_skel->link;
14001
14002 bpf_link__destroy(*link);
14003 *link = NULL;
14004 }
14005
14006 if (s->map_skel_sz < sizeof(struct bpf_map_skeleton))
14007 return;
14008
14009 for (i = 0; i < s->map_cnt; i++) {
14010 struct bpf_map_skeleton *map_skel = (void *)s->maps + i * s->map_skel_sz;
14011 struct bpf_link **link = map_skel->link;
14012
14013 if (link) {
14014 bpf_link__destroy(*link);
14015 *link = NULL;
14016 }
14017 }
14018 }
14019
14020 void bpf_object__destroy_skeleton(struct bpf_object_skeleton *s)
14021 {
14022 if (!s)
14023 return;
14024
14025 bpf_object__detach_skeleton(s);
14026 if (s->obj)
14027 bpf_object__close(*s->obj);
14028 free(s->maps);
14029 free(s->progs);
14030 free(s);
14031 }
14032
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.