1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
2 /*
3 * BPF static linker
4 *
5 * Copyright (c) 2021 Facebook
6 */
7 #include <stdbool.h>
8 #include <stddef.h>
9 #include <stdio.h>
10 #include <stdlib.h>
11 #include <string.h>
12 #include <unistd.h>
13 #include <errno.h>
14 #include <linux/err.h>
15 #include <linux/btf.h>
16 #include <elf.h>
17 #include <libelf.h>
18 #include <fcntl.h>
19 #include "libbpf.h"
20 #include "btf.h"
21 #include "libbpf_internal.h"
22 #include "strset.h"
23
24 #define BTF_EXTERN_SEC ".extern"
25
26 struct src_sec {
27 const char *sec_name;
28 /* positional (not necessarily ELF) index in an array of sections */
29 int id;
30 /* positional (not necessarily ELF) index of a matching section in a final object file */
31 int dst_id;
32 /* section data offset in a matching output section */
33 int dst_off;
34 /* whether section is omitted from the final ELF file */
35 bool skipped;
36 /* whether section is an ephemeral section, not mapped to an ELF section */
37 bool ephemeral;
38
39 /* ELF info */
40 size_t sec_idx;
41 Elf_Scn *scn;
42 Elf64_Shdr *shdr;
43 Elf_Data *data;
44
45 /* corresponding BTF DATASEC type ID */
46 int sec_type_id;
47 };
48
49 struct src_obj {
50 const char *filename;
51 int fd;
52 Elf *elf;
53 /* Section header strings section index */
54 size_t shstrs_sec_idx;
55 /* SYMTAB section index */
56 size_t symtab_sec_idx;
57
58 struct btf *btf;
59 struct btf_ext *btf_ext;
60
61 /* List of sections (including ephemeral). Slot zero is unused. */
62 struct src_sec *secs;
63 int sec_cnt;
64
65 /* mapping of symbol indices from src to dst ELF */
66 int *sym_map;
67 /* mapping from the src BTF type IDs to dst ones */
68 int *btf_type_map;
69 };
70
71 /* single .BTF.ext data section */
72 struct btf_ext_sec_data {
73 size_t rec_cnt;
74 __u32 rec_sz;
75 void *recs;
76 };
77
78 struct glob_sym {
79 /* ELF symbol index */
80 int sym_idx;
81 /* associated section id for .ksyms, .kconfig, etc, but not .extern */
82 int sec_id;
83 /* extern name offset in STRTAB */
84 int name_off;
85 /* optional associated BTF type ID */
86 int btf_id;
87 /* BTF type ID to which VAR/FUNC type is pointing to; used for
88 * rewriting types when extern VAR/FUNC is resolved to a concrete
89 * definition
90 */
91 int underlying_btf_id;
92 /* sec_var index in the corresponding dst_sec, if exists */
93 int var_idx;
94
95 /* extern or resolved/global symbol */
96 bool is_extern;
97 /* weak or strong symbol, never goes back from strong to weak */
98 bool is_weak;
99 };
100
101 struct dst_sec {
102 char *sec_name;
103 /* positional (not necessarily ELF) index in an array of sections */
104 int id;
105
106 bool ephemeral;
107
108 /* ELF info */
109 size_t sec_idx;
110 Elf_Scn *scn;
111 Elf64_Shdr *shdr;
112 Elf_Data *data;
113
114 /* final output section size */
115 int sec_sz;
116 /* final output contents of the section */
117 void *raw_data;
118
119 /* corresponding STT_SECTION symbol index in SYMTAB */
120 int sec_sym_idx;
121
122 /* section's DATASEC variable info, emitted on BTF finalization */
123 bool has_btf;
124 int sec_var_cnt;
125 struct btf_var_secinfo *sec_vars;
126
127 /* section's .BTF.ext data */
128 struct btf_ext_sec_data func_info;
129 struct btf_ext_sec_data line_info;
130 struct btf_ext_sec_data core_relo_info;
131 };
132
133 struct bpf_linker {
134 char *filename;
135 int fd;
136 Elf *elf;
137 Elf64_Ehdr *elf_hdr;
138
139 /* Output sections metadata */
140 struct dst_sec *secs;
141 int sec_cnt;
142
143 struct strset *strtab_strs; /* STRTAB unique strings */
144 size_t strtab_sec_idx; /* STRTAB section index */
145 size_t symtab_sec_idx; /* SYMTAB section index */
146
147 struct btf *btf;
148 struct btf_ext *btf_ext;
149
150 /* global (including extern) ELF symbols */
151 int glob_sym_cnt;
152 struct glob_sym *glob_syms;
153 };
154
155 #define pr_warn_elf(fmt, ...) \
156 libbpf_print(LIBBPF_WARN, "libbpf: " fmt ": %s\n", ##__VA_ARGS__, elf_errmsg(-1))
157
158 static int init_output_elf(struct bpf_linker *linker, const char *file);
159
160 static int linker_load_obj_file(struct bpf_linker *linker, const char *filename,
161 const struct bpf_linker_file_opts *opts,
162 struct src_obj *obj);
163 static int linker_sanity_check_elf(struct src_obj *obj);
164 static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec);
165 static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec);
166 static int linker_sanity_check_btf(struct src_obj *obj);
167 static int linker_sanity_check_btf_ext(struct src_obj *obj);
168 static int linker_fixup_btf(struct src_obj *obj);
169 static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj);
170 static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj);
171 static int linker_append_elf_sym(struct bpf_linker *linker, struct src_obj *obj,
172 Elf64_Sym *sym, const char *sym_name, int src_sym_idx);
173 static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj);
174 static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj);
175 static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj);
176
177 static int finalize_btf(struct bpf_linker *linker);
178 static int finalize_btf_ext(struct bpf_linker *linker);
179
180 void bpf_linker__free(struct bpf_linker *linker)
181 {
182 int i;
183
184 if (!linker)
185 return;
186
187 free(linker->filename);
188
189 if (linker->elf)
190 elf_end(linker->elf);
191
192 if (linker->fd >= 0)
193 close(linker->fd);
194
195 strset__free(linker->strtab_strs);
196
197 btf__free(linker->btf);
198 btf_ext__free(linker->btf_ext);
199
200 for (i = 1; i < linker->sec_cnt; i++) {
201 struct dst_sec *sec = &linker->secs[i];
202
203 free(sec->sec_name);
204 free(sec->raw_data);
205 free(sec->sec_vars);
206
207 free(sec->func_info.recs);
208 free(sec->line_info.recs);
209 free(sec->core_relo_info.recs);
210 }
211 free(linker->secs);
212
213 free(linker->glob_syms);
214 free(linker);
215 }
216
217 struct bpf_linker *bpf_linker__new(const char *filename, struct bpf_linker_opts *opts)
218 {
219 struct bpf_linker *linker;
220 int err;
221
222 if (!OPTS_VALID(opts, bpf_linker_opts))
223 return errno = EINVAL, NULL;
224
225 if (elf_version(EV_CURRENT) == EV_NONE) {
226 pr_warn_elf("libelf initialization failed");
227 return errno = EINVAL, NULL;
228 }
229
230 linker = calloc(1, sizeof(*linker));
231 if (!linker)
232 return errno = ENOMEM, NULL;
233
234 linker->fd = -1;
235
236 err = init_output_elf(linker, filename);
237 if (err)
238 goto err_out;
239
240 return linker;
241
242 err_out:
243 bpf_linker__free(linker);
244 return errno = -err, NULL;
245 }
246
247 static struct dst_sec *add_dst_sec(struct bpf_linker *linker, const char *sec_name)
248 {
249 struct dst_sec *secs = linker->secs, *sec;
250 size_t new_cnt = linker->sec_cnt ? linker->sec_cnt + 1 : 2;
251
252 secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs));
253 if (!secs)
254 return NULL;
255
256 /* zero out newly allocated memory */
257 memset(secs + linker->sec_cnt, 0, (new_cnt - linker->sec_cnt) * sizeof(*secs));
258
259 linker->secs = secs;
260 linker->sec_cnt = new_cnt;
261
262 sec = &linker->secs[new_cnt - 1];
263 sec->id = new_cnt - 1;
264 sec->sec_name = strdup(sec_name);
265 if (!sec->sec_name)
266 return NULL;
267
268 return sec;
269 }
270
271 static Elf64_Sym *add_new_sym(struct bpf_linker *linker, size_t *sym_idx)
272 {
273 struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx];
274 Elf64_Sym *syms, *sym;
275 size_t sym_cnt = symtab->sec_sz / sizeof(*sym);
276
277 syms = libbpf_reallocarray(symtab->raw_data, sym_cnt + 1, sizeof(*sym));
278 if (!syms)
279 return NULL;
280
281 sym = &syms[sym_cnt];
282 memset(sym, 0, sizeof(*sym));
283
284 symtab->raw_data = syms;
285 symtab->sec_sz += sizeof(*sym);
286 symtab->shdr->sh_size += sizeof(*sym);
287 symtab->data->d_size += sizeof(*sym);
288
289 if (sym_idx)
290 *sym_idx = sym_cnt;
291
292 return sym;
293 }
294
295 static int init_output_elf(struct bpf_linker *linker, const char *file)
296 {
297 int err, str_off;
298 Elf64_Sym *init_sym;
299 struct dst_sec *sec;
300
301 linker->filename = strdup(file);
302 if (!linker->filename)
303 return -ENOMEM;
304
305 linker->fd = open(file, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0644);
306 if (linker->fd < 0) {
307 err = -errno;
308 pr_warn("failed to create '%s': %d\n", file, err);
309 return err;
310 }
311
312 linker->elf = elf_begin(linker->fd, ELF_C_WRITE, NULL);
313 if (!linker->elf) {
314 pr_warn_elf("failed to create ELF object");
315 return -EINVAL;
316 }
317
318 /* ELF header */
319 linker->elf_hdr = elf64_newehdr(linker->elf);
320 if (!linker->elf_hdr) {
321 pr_warn_elf("failed to create ELF header");
322 return -EINVAL;
323 }
324
325 linker->elf_hdr->e_machine = EM_BPF;
326 linker->elf_hdr->e_type = ET_REL;
327 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
328 linker->elf_hdr->e_ident[EI_DATA] = ELFDATA2LSB;
329 #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
330 linker->elf_hdr->e_ident[EI_DATA] = ELFDATA2MSB;
331 #else
332 #error "Unknown __BYTE_ORDER__"
333 #endif
334
335 /* STRTAB */
336 /* initialize strset with an empty string to conform to ELF */
337 linker->strtab_strs = strset__new(INT_MAX, "", sizeof(""));
338 if (libbpf_get_error(linker->strtab_strs))
339 return libbpf_get_error(linker->strtab_strs);
340
341 sec = add_dst_sec(linker, ".strtab");
342 if (!sec)
343 return -ENOMEM;
344
345 sec->scn = elf_newscn(linker->elf);
346 if (!sec->scn) {
347 pr_warn_elf("failed to create STRTAB section");
348 return -EINVAL;
349 }
350
351 sec->shdr = elf64_getshdr(sec->scn);
352 if (!sec->shdr)
353 return -EINVAL;
354
355 sec->data = elf_newdata(sec->scn);
356 if (!sec->data) {
357 pr_warn_elf("failed to create STRTAB data");
358 return -EINVAL;
359 }
360
361 str_off = strset__add_str(linker->strtab_strs, sec->sec_name);
362 if (str_off < 0)
363 return str_off;
364
365 sec->sec_idx = elf_ndxscn(sec->scn);
366 linker->elf_hdr->e_shstrndx = sec->sec_idx;
367 linker->strtab_sec_idx = sec->sec_idx;
368
369 sec->shdr->sh_name = str_off;
370 sec->shdr->sh_type = SHT_STRTAB;
371 sec->shdr->sh_flags = SHF_STRINGS;
372 sec->shdr->sh_offset = 0;
373 sec->shdr->sh_link = 0;
374 sec->shdr->sh_info = 0;
375 sec->shdr->sh_addralign = 1;
376 sec->shdr->sh_size = sec->sec_sz = 0;
377 sec->shdr->sh_entsize = 0;
378
379 /* SYMTAB */
380 sec = add_dst_sec(linker, ".symtab");
381 if (!sec)
382 return -ENOMEM;
383
384 sec->scn = elf_newscn(linker->elf);
385 if (!sec->scn) {
386 pr_warn_elf("failed to create SYMTAB section");
387 return -EINVAL;
388 }
389
390 sec->shdr = elf64_getshdr(sec->scn);
391 if (!sec->shdr)
392 return -EINVAL;
393
394 sec->data = elf_newdata(sec->scn);
395 if (!sec->data) {
396 pr_warn_elf("failed to create SYMTAB data");
397 return -EINVAL;
398 }
399
400 str_off = strset__add_str(linker->strtab_strs, sec->sec_name);
401 if (str_off < 0)
402 return str_off;
403
404 sec->sec_idx = elf_ndxscn(sec->scn);
405 linker->symtab_sec_idx = sec->sec_idx;
406
407 sec->shdr->sh_name = str_off;
408 sec->shdr->sh_type = SHT_SYMTAB;
409 sec->shdr->sh_flags = 0;
410 sec->shdr->sh_offset = 0;
411 sec->shdr->sh_link = linker->strtab_sec_idx;
412 /* sh_info should be one greater than the index of the last local
413 * symbol (i.e., binding is STB_LOCAL). But why and who cares?
414 */
415 sec->shdr->sh_info = 0;
416 sec->shdr->sh_addralign = 8;
417 sec->shdr->sh_entsize = sizeof(Elf64_Sym);
418
419 /* .BTF */
420 linker->btf = btf__new_empty();
421 err = libbpf_get_error(linker->btf);
422 if (err)
423 return err;
424
425 /* add the special all-zero symbol */
426 init_sym = add_new_sym(linker, NULL);
427 if (!init_sym)
428 return -EINVAL;
429
430 init_sym->st_name = 0;
431 init_sym->st_info = 0;
432 init_sym->st_other = 0;
433 init_sym->st_shndx = SHN_UNDEF;
434 init_sym->st_value = 0;
435 init_sym->st_size = 0;
436
437 return 0;
438 }
439
440 int bpf_linker__add_file(struct bpf_linker *linker, const char *filename,
441 const struct bpf_linker_file_opts *opts)
442 {
443 struct src_obj obj = {};
444 int err = 0;
445
446 if (!OPTS_VALID(opts, bpf_linker_file_opts))
447 return libbpf_err(-EINVAL);
448
449 if (!linker->elf)
450 return libbpf_err(-EINVAL);
451
452 err = err ?: linker_load_obj_file(linker, filename, opts, &obj);
453 err = err ?: linker_append_sec_data(linker, &obj);
454 err = err ?: linker_append_elf_syms(linker, &obj);
455 err = err ?: linker_append_elf_relos(linker, &obj);
456 err = err ?: linker_append_btf(linker, &obj);
457 err = err ?: linker_append_btf_ext(linker, &obj);
458
459 /* free up src_obj resources */
460 free(obj.btf_type_map);
461 btf__free(obj.btf);
462 btf_ext__free(obj.btf_ext);
463 free(obj.secs);
464 free(obj.sym_map);
465 if (obj.elf)
466 elf_end(obj.elf);
467 if (obj.fd >= 0)
468 close(obj.fd);
469
470 return libbpf_err(err);
471 }
472
473 static bool is_dwarf_sec_name(const char *name)
474 {
475 /* approximation, but the actual list is too long */
476 return strncmp(name, ".debug_", sizeof(".debug_") - 1) == 0;
477 }
478
479 static bool is_ignored_sec(struct src_sec *sec)
480 {
481 Elf64_Shdr *shdr = sec->shdr;
482 const char *name = sec->sec_name;
483
484 /* no special handling of .strtab */
485 if (shdr->sh_type == SHT_STRTAB)
486 return true;
487
488 /* ignore .llvm_addrsig section as well */
489 if (shdr->sh_type == SHT_LLVM_ADDRSIG)
490 return true;
491
492 /* no subprograms will lead to an empty .text section, ignore it */
493 if (shdr->sh_type == SHT_PROGBITS && shdr->sh_size == 0 &&
494 strcmp(sec->sec_name, ".text") == 0)
495 return true;
496
497 /* DWARF sections */
498 if (is_dwarf_sec_name(sec->sec_name))
499 return true;
500
501 if (strncmp(name, ".rel", sizeof(".rel") - 1) == 0) {
502 name += sizeof(".rel") - 1;
503 /* DWARF section relocations */
504 if (is_dwarf_sec_name(name))
505 return true;
506
507 /* .BTF and .BTF.ext don't need relocations */
508 if (strcmp(name, BTF_ELF_SEC) == 0 ||
509 strcmp(name, BTF_EXT_ELF_SEC) == 0)
510 return true;
511 }
512
513 return false;
514 }
515
516 static struct src_sec *add_src_sec(struct src_obj *obj, const char *sec_name)
517 {
518 struct src_sec *secs = obj->secs, *sec;
519 size_t new_cnt = obj->sec_cnt ? obj->sec_cnt + 1 : 2;
520
521 secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs));
522 if (!secs)
523 return NULL;
524
525 /* zero out newly allocated memory */
526 memset(secs + obj->sec_cnt, 0, (new_cnt - obj->sec_cnt) * sizeof(*secs));
527
528 obj->secs = secs;
529 obj->sec_cnt = new_cnt;
530
531 sec = &obj->secs[new_cnt - 1];
532 sec->id = new_cnt - 1;
533 sec->sec_name = sec_name;
534
535 return sec;
536 }
537
538 static int linker_load_obj_file(struct bpf_linker *linker, const char *filename,
539 const struct bpf_linker_file_opts *opts,
540 struct src_obj *obj)
541 {
542 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
543 const int host_endianness = ELFDATA2LSB;
544 #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
545 const int host_endianness = ELFDATA2MSB;
546 #else
547 #error "Unknown __BYTE_ORDER__"
548 #endif
549 int err = 0;
550 Elf_Scn *scn;
551 Elf_Data *data;
552 Elf64_Ehdr *ehdr;
553 Elf64_Shdr *shdr;
554 struct src_sec *sec;
555
556 pr_debug("linker: adding object file '%s'...\n", filename);
557
558 obj->filename = filename;
559
560 obj->fd = open(filename, O_RDONLY | O_CLOEXEC);
561 if (obj->fd < 0) {
562 err = -errno;
563 pr_warn("failed to open file '%s': %d\n", filename, err);
564 return err;
565 }
566 obj->elf = elf_begin(obj->fd, ELF_C_READ_MMAP, NULL);
567 if (!obj->elf) {
568 err = -errno;
569 pr_warn_elf("failed to parse ELF file '%s'", filename);
570 return err;
571 }
572
573 /* Sanity check ELF file high-level properties */
574 ehdr = elf64_getehdr(obj->elf);
575 if (!ehdr) {
576 err = -errno;
577 pr_warn_elf("failed to get ELF header for %s", filename);
578 return err;
579 }
580 if (ehdr->e_ident[EI_DATA] != host_endianness) {
581 err = -EOPNOTSUPP;
582 pr_warn_elf("unsupported byte order of ELF file %s", filename);
583 return err;
584 }
585 if (ehdr->e_type != ET_REL
586 || ehdr->e_machine != EM_BPF
587 || ehdr->e_ident[EI_CLASS] != ELFCLASS64) {
588 err = -EOPNOTSUPP;
589 pr_warn_elf("unsupported kind of ELF file %s", filename);
590 return err;
591 }
592
593 if (elf_getshdrstrndx(obj->elf, &obj->shstrs_sec_idx)) {
594 err = -errno;
595 pr_warn_elf("failed to get SHSTRTAB section index for %s", filename);
596 return err;
597 }
598
599 scn = NULL;
600 while ((scn = elf_nextscn(obj->elf, scn)) != NULL) {
601 size_t sec_idx = elf_ndxscn(scn);
602 const char *sec_name;
603
604 shdr = elf64_getshdr(scn);
605 if (!shdr) {
606 err = -errno;
607 pr_warn_elf("failed to get section #%zu header for %s",
608 sec_idx, filename);
609 return err;
610 }
611
612 sec_name = elf_strptr(obj->elf, obj->shstrs_sec_idx, shdr->sh_name);
613 if (!sec_name) {
614 err = -errno;
615 pr_warn_elf("failed to get section #%zu name for %s",
616 sec_idx, filename);
617 return err;
618 }
619
620 data = elf_getdata(scn, 0);
621 if (!data) {
622 err = -errno;
623 pr_warn_elf("failed to get section #%zu (%s) data from %s",
624 sec_idx, sec_name, filename);
625 return err;
626 }
627
628 sec = add_src_sec(obj, sec_name);
629 if (!sec)
630 return -ENOMEM;
631
632 sec->scn = scn;
633 sec->shdr = shdr;
634 sec->data = data;
635 sec->sec_idx = elf_ndxscn(scn);
636
637 if (is_ignored_sec(sec)) {
638 sec->skipped = true;
639 continue;
640 }
641
642 switch (shdr->sh_type) {
643 case SHT_SYMTAB:
644 if (obj->symtab_sec_idx) {
645 err = -EOPNOTSUPP;
646 pr_warn("multiple SYMTAB sections found, not supported\n");
647 return err;
648 }
649 obj->symtab_sec_idx = sec_idx;
650 break;
651 case SHT_STRTAB:
652 /* we'll construct our own string table */
653 break;
654 case SHT_PROGBITS:
655 if (strcmp(sec_name, BTF_ELF_SEC) == 0) {
656 obj->btf = btf__new(data->d_buf, shdr->sh_size);
657 err = libbpf_get_error(obj->btf);
658 if (err) {
659 pr_warn("failed to parse .BTF from %s: %d\n", filename, err);
660 return err;
661 }
662 sec->skipped = true;
663 continue;
664 }
665 if (strcmp(sec_name, BTF_EXT_ELF_SEC) == 0) {
666 obj->btf_ext = btf_ext__new(data->d_buf, shdr->sh_size);
667 err = libbpf_get_error(obj->btf_ext);
668 if (err) {
669 pr_warn("failed to parse .BTF.ext from '%s': %d\n", filename, err);
670 return err;
671 }
672 sec->skipped = true;
673 continue;
674 }
675
676 /* data & code */
677 break;
678 case SHT_NOBITS:
679 /* BSS */
680 break;
681 case SHT_REL:
682 /* relocations */
683 break;
684 default:
685 pr_warn("unrecognized section #%zu (%s) in %s\n",
686 sec_idx, sec_name, filename);
687 err = -EINVAL;
688 return err;
689 }
690 }
691
692 err = err ?: linker_sanity_check_elf(obj);
693 err = err ?: linker_sanity_check_btf(obj);
694 err = err ?: linker_sanity_check_btf_ext(obj);
695 err = err ?: linker_fixup_btf(obj);
696
697 return err;
698 }
699
700 static int linker_sanity_check_elf(struct src_obj *obj)
701 {
702 struct src_sec *sec;
703 int i, err;
704
705 if (!obj->symtab_sec_idx) {
706 pr_warn("ELF is missing SYMTAB section in %s\n", obj->filename);
707 return -EINVAL;
708 }
709 if (!obj->shstrs_sec_idx) {
710 pr_warn("ELF is missing section headers STRTAB section in %s\n", obj->filename);
711 return -EINVAL;
712 }
713
714 for (i = 1; i < obj->sec_cnt; i++) {
715 sec = &obj->secs[i];
716
717 if (sec->sec_name[0] == '\0') {
718 pr_warn("ELF section #%zu has empty name in %s\n", sec->sec_idx, obj->filename);
719 return -EINVAL;
720 }
721
722 if (is_dwarf_sec_name(sec->sec_name))
723 continue;
724
725 if (sec->shdr->sh_addralign && !is_pow_of_2(sec->shdr->sh_addralign)) {
726 pr_warn("ELF section #%zu alignment %llu is non pow-of-2 alignment in %s\n",
727 sec->sec_idx, (long long unsigned)sec->shdr->sh_addralign,
728 obj->filename);
729 return -EINVAL;
730 }
731 if (sec->shdr->sh_addralign != sec->data->d_align) {
732 pr_warn("ELF section #%zu has inconsistent alignment addr=%llu != d=%llu in %s\n",
733 sec->sec_idx, (long long unsigned)sec->shdr->sh_addralign,
734 (long long unsigned)sec->data->d_align, obj->filename);
735 return -EINVAL;
736 }
737
738 if (sec->shdr->sh_size != sec->data->d_size) {
739 pr_warn("ELF section #%zu has inconsistent section size sh=%llu != d=%llu in %s\n",
740 sec->sec_idx, (long long unsigned)sec->shdr->sh_size,
741 (long long unsigned)sec->data->d_size, obj->filename);
742 return -EINVAL;
743 }
744
745 switch (sec->shdr->sh_type) {
746 case SHT_SYMTAB:
747 err = linker_sanity_check_elf_symtab(obj, sec);
748 if (err)
749 return err;
750 break;
751 case SHT_STRTAB:
752 break;
753 case SHT_PROGBITS:
754 if (sec->shdr->sh_flags & SHF_EXECINSTR) {
755 if (sec->shdr->sh_size % sizeof(struct bpf_insn) != 0) {
756 pr_warn("ELF section #%zu has unexpected size alignment %llu in %s\n",
757 sec->sec_idx, (long long unsigned)sec->shdr->sh_size,
758 obj->filename);
759 return -EINVAL;
760 }
761 }
762 break;
763 case SHT_NOBITS:
764 break;
765 case SHT_REL:
766 err = linker_sanity_check_elf_relos(obj, sec);
767 if (err)
768 return err;
769 break;
770 case SHT_LLVM_ADDRSIG:
771 break;
772 default:
773 pr_warn("ELF section #%zu (%s) has unrecognized type %zu in %s\n",
774 sec->sec_idx, sec->sec_name, (size_t)sec->shdr->sh_type, obj->filename);
775 return -EINVAL;
776 }
777 }
778
779 return 0;
780 }
781
782 static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec)
783 {
784 struct src_sec *link_sec;
785 Elf64_Sym *sym;
786 int i, n;
787
788 if (sec->shdr->sh_entsize != sizeof(Elf64_Sym))
789 return -EINVAL;
790 if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0)
791 return -EINVAL;
792
793 if (!sec->shdr->sh_link || sec->shdr->sh_link >= obj->sec_cnt) {
794 pr_warn("ELF SYMTAB section #%zu points to missing STRTAB section #%zu in %s\n",
795 sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
796 return -EINVAL;
797 }
798 link_sec = &obj->secs[sec->shdr->sh_link];
799 if (link_sec->shdr->sh_type != SHT_STRTAB) {
800 pr_warn("ELF SYMTAB section #%zu points to invalid STRTAB section #%zu in %s\n",
801 sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
802 return -EINVAL;
803 }
804
805 n = sec->shdr->sh_size / sec->shdr->sh_entsize;
806 sym = sec->data->d_buf;
807 for (i = 0; i < n; i++, sym++) {
808 int sym_type = ELF64_ST_TYPE(sym->st_info);
809 int sym_bind = ELF64_ST_BIND(sym->st_info);
810 int sym_vis = ELF64_ST_VISIBILITY(sym->st_other);
811
812 if (i == 0) {
813 if (sym->st_name != 0 || sym->st_info != 0
814 || sym->st_other != 0 || sym->st_shndx != 0
815 || sym->st_value != 0 || sym->st_size != 0) {
816 pr_warn("ELF sym #0 is invalid in %s\n", obj->filename);
817 return -EINVAL;
818 }
819 continue;
820 }
821 if (sym_bind != STB_LOCAL && sym_bind != STB_GLOBAL && sym_bind != STB_WEAK) {
822 pr_warn("ELF sym #%d in section #%zu has unsupported symbol binding %d\n",
823 i, sec->sec_idx, sym_bind);
824 return -EINVAL;
825 }
826 if (sym_vis != STV_DEFAULT && sym_vis != STV_HIDDEN) {
827 pr_warn("ELF sym #%d in section #%zu has unsupported symbol visibility %d\n",
828 i, sec->sec_idx, sym_vis);
829 return -EINVAL;
830 }
831 if (sym->st_shndx == 0) {
832 if (sym_type != STT_NOTYPE || sym_bind == STB_LOCAL
833 || sym->st_value != 0 || sym->st_size != 0) {
834 pr_warn("ELF sym #%d is invalid extern symbol in %s\n",
835 i, obj->filename);
836
837 return -EINVAL;
838 }
839 continue;
840 }
841 if (sym->st_shndx < SHN_LORESERVE && sym->st_shndx >= obj->sec_cnt) {
842 pr_warn("ELF sym #%d in section #%zu points to missing section #%zu in %s\n",
843 i, sec->sec_idx, (size_t)sym->st_shndx, obj->filename);
844 return -EINVAL;
845 }
846 if (sym_type == STT_SECTION) {
847 if (sym->st_value != 0)
848 return -EINVAL;
849 continue;
850 }
851 }
852
853 return 0;
854 }
855
856 static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec)
857 {
858 struct src_sec *link_sec, *sym_sec;
859 Elf64_Rel *relo;
860 int i, n;
861
862 if (sec->shdr->sh_entsize != sizeof(Elf64_Rel))
863 return -EINVAL;
864 if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0)
865 return -EINVAL;
866
867 /* SHT_REL's sh_link should point to SYMTAB */
868 if (sec->shdr->sh_link != obj->symtab_sec_idx) {
869 pr_warn("ELF relo section #%zu points to invalid SYMTAB section #%zu in %s\n",
870 sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
871 return -EINVAL;
872 }
873
874 /* SHT_REL's sh_info points to relocated section */
875 if (!sec->shdr->sh_info || sec->shdr->sh_info >= obj->sec_cnt) {
876 pr_warn("ELF relo section #%zu points to missing section #%zu in %s\n",
877 sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename);
878 return -EINVAL;
879 }
880 link_sec = &obj->secs[sec->shdr->sh_info];
881
882 /* .rel<secname> -> <secname> pattern is followed */
883 if (strncmp(sec->sec_name, ".rel", sizeof(".rel") - 1) != 0
884 || strcmp(sec->sec_name + sizeof(".rel") - 1, link_sec->sec_name) != 0) {
885 pr_warn("ELF relo section #%zu name has invalid name in %s\n",
886 sec->sec_idx, obj->filename);
887 return -EINVAL;
888 }
889
890 /* don't further validate relocations for ignored sections */
891 if (link_sec->skipped)
892 return 0;
893
894 /* relocatable section is data or instructions */
895 if (link_sec->shdr->sh_type != SHT_PROGBITS && link_sec->shdr->sh_type != SHT_NOBITS) {
896 pr_warn("ELF relo section #%zu points to invalid section #%zu in %s\n",
897 sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename);
898 return -EINVAL;
899 }
900
901 /* check sanity of each relocation */
902 n = sec->shdr->sh_size / sec->shdr->sh_entsize;
903 relo = sec->data->d_buf;
904 sym_sec = &obj->secs[obj->symtab_sec_idx];
905 for (i = 0; i < n; i++, relo++) {
906 size_t sym_idx = ELF64_R_SYM(relo->r_info);
907 size_t sym_type = ELF64_R_TYPE(relo->r_info);
908
909 if (sym_type != R_BPF_64_64 && sym_type != R_BPF_64_32 &&
910 sym_type != R_BPF_64_ABS64 && sym_type != R_BPF_64_ABS32) {
911 pr_warn("ELF relo #%d in section #%zu has unexpected type %zu in %s\n",
912 i, sec->sec_idx, sym_type, obj->filename);
913 return -EINVAL;
914 }
915
916 if (!sym_idx || sym_idx * sizeof(Elf64_Sym) >= sym_sec->shdr->sh_size) {
917 pr_warn("ELF relo #%d in section #%zu points to invalid symbol #%zu in %s\n",
918 i, sec->sec_idx, sym_idx, obj->filename);
919 return -EINVAL;
920 }
921
922 if (link_sec->shdr->sh_flags & SHF_EXECINSTR) {
923 if (relo->r_offset % sizeof(struct bpf_insn) != 0) {
924 pr_warn("ELF relo #%d in section #%zu points to missing symbol #%zu in %s\n",
925 i, sec->sec_idx, sym_idx, obj->filename);
926 return -EINVAL;
927 }
928 }
929 }
930
931 return 0;
932 }
933
934 static int check_btf_type_id(__u32 *type_id, void *ctx)
935 {
936 struct btf *btf = ctx;
937
938 if (*type_id >= btf__type_cnt(btf))
939 return -EINVAL;
940
941 return 0;
942 }
943
944 static int check_btf_str_off(__u32 *str_off, void *ctx)
945 {
946 struct btf *btf = ctx;
947 const char *s;
948
949 s = btf__str_by_offset(btf, *str_off);
950
951 if (!s)
952 return -EINVAL;
953
954 return 0;
955 }
956
957 static int linker_sanity_check_btf(struct src_obj *obj)
958 {
959 struct btf_type *t;
960 int i, n, err;
961
962 if (!obj->btf)
963 return 0;
964
965 n = btf__type_cnt(obj->btf);
966 for (i = 1; i < n; i++) {
967 struct btf_field_iter it;
968 __u32 *type_id, *str_off;
969
970 t = btf_type_by_id(obj->btf, i);
971
972 err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_IDS);
973 if (err)
974 return err;
975 while ((type_id = btf_field_iter_next(&it))) {
976 if (*type_id >= n)
977 return -EINVAL;
978 }
979
980 err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_STRS);
981 if (err)
982 return err;
983 while ((str_off = btf_field_iter_next(&it))) {
984 if (!btf__str_by_offset(obj->btf, *str_off))
985 return -EINVAL;
986 }
987 }
988
989 return 0;
990 }
991
992 static int linker_sanity_check_btf_ext(struct src_obj *obj)
993 {
994 int err = 0;
995
996 if (!obj->btf_ext)
997 return 0;
998
999 /* can't use .BTF.ext without .BTF */
1000 if (!obj->btf)
1001 return -EINVAL;
1002
1003 err = err ?: btf_ext_visit_type_ids(obj->btf_ext, check_btf_type_id, obj->btf);
1004 err = err ?: btf_ext_visit_str_offs(obj->btf_ext, check_btf_str_off, obj->btf);
1005 if (err)
1006 return err;
1007
1008 return 0;
1009 }
1010
1011 static int init_sec(struct bpf_linker *linker, struct dst_sec *dst_sec, struct src_sec *src_sec)
1012 {
1013 Elf_Scn *scn;
1014 Elf_Data *data;
1015 Elf64_Shdr *shdr;
1016 int name_off;
1017
1018 dst_sec->sec_sz = 0;
1019 dst_sec->sec_idx = 0;
1020 dst_sec->ephemeral = src_sec->ephemeral;
1021
1022 /* ephemeral sections are just thin section shells lacking most parts */
1023 if (src_sec->ephemeral)
1024 return 0;
1025
1026 scn = elf_newscn(linker->elf);
1027 if (!scn)
1028 return -ENOMEM;
1029 data = elf_newdata(scn);
1030 if (!data)
1031 return -ENOMEM;
1032 shdr = elf64_getshdr(scn);
1033 if (!shdr)
1034 return -ENOMEM;
1035
1036 dst_sec->scn = scn;
1037 dst_sec->shdr = shdr;
1038 dst_sec->data = data;
1039 dst_sec->sec_idx = elf_ndxscn(scn);
1040
1041 name_off = strset__add_str(linker->strtab_strs, src_sec->sec_name);
1042 if (name_off < 0)
1043 return name_off;
1044
1045 shdr->sh_name = name_off;
1046 shdr->sh_type = src_sec->shdr->sh_type;
1047 shdr->sh_flags = src_sec->shdr->sh_flags;
1048 shdr->sh_size = 0;
1049 /* sh_link and sh_info have different meaning for different types of
1050 * sections, so we leave it up to the caller code to fill them in, if
1051 * necessary
1052 */
1053 shdr->sh_link = 0;
1054 shdr->sh_info = 0;
1055 shdr->sh_addralign = src_sec->shdr->sh_addralign;
1056 shdr->sh_entsize = src_sec->shdr->sh_entsize;
1057
1058 data->d_type = src_sec->data->d_type;
1059 data->d_size = 0;
1060 data->d_buf = NULL;
1061 data->d_align = src_sec->data->d_align;
1062 data->d_off = 0;
1063
1064 return 0;
1065 }
1066
1067 static struct dst_sec *find_dst_sec_by_name(struct bpf_linker *linker, const char *sec_name)
1068 {
1069 struct dst_sec *sec;
1070 int i;
1071
1072 for (i = 1; i < linker->sec_cnt; i++) {
1073 sec = &linker->secs[i];
1074
1075 if (strcmp(sec->sec_name, sec_name) == 0)
1076 return sec;
1077 }
1078
1079 return NULL;
1080 }
1081
1082 static bool secs_match(struct dst_sec *dst, struct src_sec *src)
1083 {
1084 if (dst->ephemeral || src->ephemeral)
1085 return true;
1086
1087 if (dst->shdr->sh_type != src->shdr->sh_type) {
1088 pr_warn("sec %s types mismatch\n", dst->sec_name);
1089 return false;
1090 }
1091 if (dst->shdr->sh_flags != src->shdr->sh_flags) {
1092 pr_warn("sec %s flags mismatch\n", dst->sec_name);
1093 return false;
1094 }
1095 if (dst->shdr->sh_entsize != src->shdr->sh_entsize) {
1096 pr_warn("sec %s entsize mismatch\n", dst->sec_name);
1097 return false;
1098 }
1099
1100 return true;
1101 }
1102
1103 static bool sec_content_is_same(struct dst_sec *dst_sec, struct src_sec *src_sec)
1104 {
1105 if (dst_sec->sec_sz != src_sec->shdr->sh_size)
1106 return false;
1107 if (memcmp(dst_sec->raw_data, src_sec->data->d_buf, dst_sec->sec_sz) != 0)
1108 return false;
1109 return true;
1110 }
1111
1112 static int extend_sec(struct bpf_linker *linker, struct dst_sec *dst, struct src_sec *src)
1113 {
1114 void *tmp;
1115 size_t dst_align, src_align;
1116 size_t dst_align_sz, dst_final_sz;
1117 int err;
1118
1119 /* Ephemeral source section doesn't contribute anything to ELF
1120 * section data.
1121 */
1122 if (src->ephemeral)
1123 return 0;
1124
1125 /* Some sections (like .maps) can contain both externs (and thus be
1126 * ephemeral) and non-externs (map definitions). So it's possible that
1127 * it has to be "upgraded" from ephemeral to non-ephemeral when the
1128 * first non-ephemeral entity appears. In such case, we add ELF
1129 * section, data, etc.
1130 */
1131 if (dst->ephemeral) {
1132 err = init_sec(linker, dst, src);
1133 if (err)
1134 return err;
1135 }
1136
1137 dst_align = dst->shdr->sh_addralign;
1138 src_align = src->shdr->sh_addralign;
1139 if (dst_align == 0)
1140 dst_align = 1;
1141 if (dst_align < src_align)
1142 dst_align = src_align;
1143
1144 dst_align_sz = (dst->sec_sz + dst_align - 1) / dst_align * dst_align;
1145
1146 /* no need to re-align final size */
1147 dst_final_sz = dst_align_sz + src->shdr->sh_size;
1148
1149 if (src->shdr->sh_type != SHT_NOBITS) {
1150 tmp = realloc(dst->raw_data, dst_final_sz);
1151 /* If dst_align_sz == 0, realloc() behaves in a special way:
1152 * 1. When dst->raw_data is NULL it returns:
1153 * "either NULL or a pointer suitable to be passed to free()" [1].
1154 * 2. When dst->raw_data is not-NULL it frees dst->raw_data and returns NULL,
1155 * thus invalidating any "pointer suitable to be passed to free()" obtained
1156 * at step (1).
1157 *
1158 * The dst_align_sz > 0 check avoids error exit after (2), otherwise
1159 * dst->raw_data would be freed again in bpf_linker__free().
1160 *
1161 * [1] man 3 realloc
1162 */
1163 if (!tmp && dst_align_sz > 0)
1164 return -ENOMEM;
1165 dst->raw_data = tmp;
1166
1167 /* pad dst section, if it's alignment forced size increase */
1168 memset(dst->raw_data + dst->sec_sz, 0, dst_align_sz - dst->sec_sz);
1169 /* now copy src data at a properly aligned offset */
1170 memcpy(dst->raw_data + dst_align_sz, src->data->d_buf, src->shdr->sh_size);
1171 }
1172
1173 dst->sec_sz = dst_final_sz;
1174 dst->shdr->sh_size = dst_final_sz;
1175 dst->data->d_size = dst_final_sz;
1176
1177 dst->shdr->sh_addralign = dst_align;
1178 dst->data->d_align = dst_align;
1179
1180 src->dst_off = dst_align_sz;
1181
1182 return 0;
1183 }
1184
1185 static bool is_data_sec(struct src_sec *sec)
1186 {
1187 if (!sec || sec->skipped)
1188 return false;
1189 /* ephemeral sections are data sections, e.g., .kconfig, .ksyms */
1190 if (sec->ephemeral)
1191 return true;
1192 return sec->shdr->sh_type == SHT_PROGBITS || sec->shdr->sh_type == SHT_NOBITS;
1193 }
1194
1195 static bool is_relo_sec(struct src_sec *sec)
1196 {
1197 if (!sec || sec->skipped || sec->ephemeral)
1198 return false;
1199 return sec->shdr->sh_type == SHT_REL;
1200 }
1201
1202 static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj)
1203 {
1204 int i, err;
1205
1206 for (i = 1; i < obj->sec_cnt; i++) {
1207 struct src_sec *src_sec;
1208 struct dst_sec *dst_sec;
1209
1210 src_sec = &obj->secs[i];
1211 if (!is_data_sec(src_sec))
1212 continue;
1213
1214 dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name);
1215 if (!dst_sec) {
1216 dst_sec = add_dst_sec(linker, src_sec->sec_name);
1217 if (!dst_sec)
1218 return -ENOMEM;
1219 err = init_sec(linker, dst_sec, src_sec);
1220 if (err) {
1221 pr_warn("failed to init section '%s'\n", src_sec->sec_name);
1222 return err;
1223 }
1224 } else {
1225 if (!secs_match(dst_sec, src_sec)) {
1226 pr_warn("ELF sections %s are incompatible\n", src_sec->sec_name);
1227 return -1;
1228 }
1229
1230 /* "license" and "version" sections are deduped */
1231 if (strcmp(src_sec->sec_name, "license") == 0
1232 || strcmp(src_sec->sec_name, "version") == 0) {
1233 if (!sec_content_is_same(dst_sec, src_sec)) {
1234 pr_warn("non-identical contents of section '%s' are not supported\n", src_sec->sec_name);
1235 return -EINVAL;
1236 }
1237 src_sec->skipped = true;
1238 src_sec->dst_id = dst_sec->id;
1239 continue;
1240 }
1241 }
1242
1243 /* record mapped section index */
1244 src_sec->dst_id = dst_sec->id;
1245
1246 err = extend_sec(linker, dst_sec, src_sec);
1247 if (err)
1248 return err;
1249 }
1250
1251 return 0;
1252 }
1253
1254 static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj)
1255 {
1256 struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx];
1257 Elf64_Sym *sym = symtab->data->d_buf;
1258 int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize, err;
1259 int str_sec_idx = symtab->shdr->sh_link;
1260 const char *sym_name;
1261
1262 obj->sym_map = calloc(n + 1, sizeof(*obj->sym_map));
1263 if (!obj->sym_map)
1264 return -ENOMEM;
1265
1266 for (i = 0; i < n; i++, sym++) {
1267 /* We already validated all-zero symbol #0 and we already
1268 * appended it preventively to the final SYMTAB, so skip it.
1269 */
1270 if (i == 0)
1271 continue;
1272
1273 sym_name = elf_strptr(obj->elf, str_sec_idx, sym->st_name);
1274 if (!sym_name) {
1275 pr_warn("can't fetch symbol name for symbol #%d in '%s'\n", i, obj->filename);
1276 return -EINVAL;
1277 }
1278
1279 err = linker_append_elf_sym(linker, obj, sym, sym_name, i);
1280 if (err)
1281 return err;
1282 }
1283
1284 return 0;
1285 }
1286
1287 static Elf64_Sym *get_sym_by_idx(struct bpf_linker *linker, size_t sym_idx)
1288 {
1289 struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx];
1290 Elf64_Sym *syms = symtab->raw_data;
1291
1292 return &syms[sym_idx];
1293 }
1294
1295 static struct glob_sym *find_glob_sym(struct bpf_linker *linker, const char *sym_name)
1296 {
1297 struct glob_sym *glob_sym;
1298 const char *name;
1299 int i;
1300
1301 for (i = 0; i < linker->glob_sym_cnt; i++) {
1302 glob_sym = &linker->glob_syms[i];
1303 name = strset__data(linker->strtab_strs) + glob_sym->name_off;
1304
1305 if (strcmp(name, sym_name) == 0)
1306 return glob_sym;
1307 }
1308
1309 return NULL;
1310 }
1311
1312 static struct glob_sym *add_glob_sym(struct bpf_linker *linker)
1313 {
1314 struct glob_sym *syms, *sym;
1315
1316 syms = libbpf_reallocarray(linker->glob_syms, linker->glob_sym_cnt + 1,
1317 sizeof(*linker->glob_syms));
1318 if (!syms)
1319 return NULL;
1320
1321 sym = &syms[linker->glob_sym_cnt];
1322 memset(sym, 0, sizeof(*sym));
1323 sym->var_idx = -1;
1324
1325 linker->glob_syms = syms;
1326 linker->glob_sym_cnt++;
1327
1328 return sym;
1329 }
1330
1331 static bool glob_sym_btf_matches(const char *sym_name, bool exact,
1332 const struct btf *btf1, __u32 id1,
1333 const struct btf *btf2, __u32 id2)
1334 {
1335 const struct btf_type *t1, *t2;
1336 bool is_static1, is_static2;
1337 const char *n1, *n2;
1338 int i, n;
1339
1340 recur:
1341 n1 = n2 = NULL;
1342 t1 = skip_mods_and_typedefs(btf1, id1, &id1);
1343 t2 = skip_mods_and_typedefs(btf2, id2, &id2);
1344
1345 /* check if only one side is FWD, otherwise handle with common logic */
1346 if (!exact && btf_is_fwd(t1) != btf_is_fwd(t2)) {
1347 n1 = btf__str_by_offset(btf1, t1->name_off);
1348 n2 = btf__str_by_offset(btf2, t2->name_off);
1349 if (strcmp(n1, n2) != 0) {
1350 pr_warn("global '%s': incompatible forward declaration names '%s' and '%s'\n",
1351 sym_name, n1, n2);
1352 return false;
1353 }
1354 /* validate if FWD kind matches concrete kind */
1355 if (btf_is_fwd(t1)) {
1356 if (btf_kflag(t1) && btf_is_union(t2))
1357 return true;
1358 if (!btf_kflag(t1) && btf_is_struct(t2))
1359 return true;
1360 pr_warn("global '%s': incompatible %s forward declaration and concrete kind %s\n",
1361 sym_name, btf_kflag(t1) ? "union" : "struct", btf_kind_str(t2));
1362 } else {
1363 if (btf_kflag(t2) && btf_is_union(t1))
1364 return true;
1365 if (!btf_kflag(t2) && btf_is_struct(t1))
1366 return true;
1367 pr_warn("global '%s': incompatible %s forward declaration and concrete kind %s\n",
1368 sym_name, btf_kflag(t2) ? "union" : "struct", btf_kind_str(t1));
1369 }
1370 return false;
1371 }
1372
1373 if (btf_kind(t1) != btf_kind(t2)) {
1374 pr_warn("global '%s': incompatible BTF kinds %s and %s\n",
1375 sym_name, btf_kind_str(t1), btf_kind_str(t2));
1376 return false;
1377 }
1378
1379 switch (btf_kind(t1)) {
1380 case BTF_KIND_STRUCT:
1381 case BTF_KIND_UNION:
1382 case BTF_KIND_ENUM:
1383 case BTF_KIND_ENUM64:
1384 case BTF_KIND_FWD:
1385 case BTF_KIND_FUNC:
1386 case BTF_KIND_VAR:
1387 n1 = btf__str_by_offset(btf1, t1->name_off);
1388 n2 = btf__str_by_offset(btf2, t2->name_off);
1389 if (strcmp(n1, n2) != 0) {
1390 pr_warn("global '%s': incompatible %s names '%s' and '%s'\n",
1391 sym_name, btf_kind_str(t1), n1, n2);
1392 return false;
1393 }
1394 break;
1395 default:
1396 break;
1397 }
1398
1399 switch (btf_kind(t1)) {
1400 case BTF_KIND_UNKN: /* void */
1401 case BTF_KIND_FWD:
1402 return true;
1403 case BTF_KIND_INT:
1404 case BTF_KIND_FLOAT:
1405 case BTF_KIND_ENUM:
1406 case BTF_KIND_ENUM64:
1407 /* ignore encoding for int and enum values for enum */
1408 if (t1->size != t2->size) {
1409 pr_warn("global '%s': incompatible %s '%s' size %u and %u\n",
1410 sym_name, btf_kind_str(t1), n1, t1->size, t2->size);
1411 return false;
1412 }
1413 return true;
1414 case BTF_KIND_PTR:
1415 /* just validate overall shape of the referenced type, so no
1416 * contents comparison for struct/union, and allowd fwd vs
1417 * struct/union
1418 */
1419 exact = false;
1420 id1 = t1->type;
1421 id2 = t2->type;
1422 goto recur;
1423 case BTF_KIND_ARRAY:
1424 /* ignore index type and array size */
1425 id1 = btf_array(t1)->type;
1426 id2 = btf_array(t2)->type;
1427 goto recur;
1428 case BTF_KIND_FUNC:
1429 /* extern and global linkages are compatible */
1430 is_static1 = btf_func_linkage(t1) == BTF_FUNC_STATIC;
1431 is_static2 = btf_func_linkage(t2) == BTF_FUNC_STATIC;
1432 if (is_static1 != is_static2) {
1433 pr_warn("global '%s': incompatible func '%s' linkage\n", sym_name, n1);
1434 return false;
1435 }
1436
1437 id1 = t1->type;
1438 id2 = t2->type;
1439 goto recur;
1440 case BTF_KIND_VAR:
1441 /* extern and global linkages are compatible */
1442 is_static1 = btf_var(t1)->linkage == BTF_VAR_STATIC;
1443 is_static2 = btf_var(t2)->linkage == BTF_VAR_STATIC;
1444 if (is_static1 != is_static2) {
1445 pr_warn("global '%s': incompatible var '%s' linkage\n", sym_name, n1);
1446 return false;
1447 }
1448
1449 id1 = t1->type;
1450 id2 = t2->type;
1451 goto recur;
1452 case BTF_KIND_STRUCT:
1453 case BTF_KIND_UNION: {
1454 const struct btf_member *m1, *m2;
1455
1456 if (!exact)
1457 return true;
1458
1459 if (btf_vlen(t1) != btf_vlen(t2)) {
1460 pr_warn("global '%s': incompatible number of %s fields %u and %u\n",
1461 sym_name, btf_kind_str(t1), btf_vlen(t1), btf_vlen(t2));
1462 return false;
1463 }
1464
1465 n = btf_vlen(t1);
1466 m1 = btf_members(t1);
1467 m2 = btf_members(t2);
1468 for (i = 0; i < n; i++, m1++, m2++) {
1469 n1 = btf__str_by_offset(btf1, m1->name_off);
1470 n2 = btf__str_by_offset(btf2, m2->name_off);
1471 if (strcmp(n1, n2) != 0) {
1472 pr_warn("global '%s': incompatible field #%d names '%s' and '%s'\n",
1473 sym_name, i, n1, n2);
1474 return false;
1475 }
1476 if (m1->offset != m2->offset) {
1477 pr_warn("global '%s': incompatible field #%d ('%s') offsets\n",
1478 sym_name, i, n1);
1479 return false;
1480 }
1481 if (!glob_sym_btf_matches(sym_name, exact, btf1, m1->type, btf2, m2->type))
1482 return false;
1483 }
1484
1485 return true;
1486 }
1487 case BTF_KIND_FUNC_PROTO: {
1488 const struct btf_param *m1, *m2;
1489
1490 if (btf_vlen(t1) != btf_vlen(t2)) {
1491 pr_warn("global '%s': incompatible number of %s params %u and %u\n",
1492 sym_name, btf_kind_str(t1), btf_vlen(t1), btf_vlen(t2));
1493 return false;
1494 }
1495
1496 n = btf_vlen(t1);
1497 m1 = btf_params(t1);
1498 m2 = btf_params(t2);
1499 for (i = 0; i < n; i++, m1++, m2++) {
1500 /* ignore func arg names */
1501 if (!glob_sym_btf_matches(sym_name, exact, btf1, m1->type, btf2, m2->type))
1502 return false;
1503 }
1504
1505 /* now check return type as well */
1506 id1 = t1->type;
1507 id2 = t2->type;
1508 goto recur;
1509 }
1510
1511 /* skip_mods_and_typedefs() make this impossible */
1512 case BTF_KIND_TYPEDEF:
1513 case BTF_KIND_VOLATILE:
1514 case BTF_KIND_CONST:
1515 case BTF_KIND_RESTRICT:
1516 /* DATASECs are never compared with each other */
1517 case BTF_KIND_DATASEC:
1518 default:
1519 pr_warn("global '%s': unsupported BTF kind %s\n",
1520 sym_name, btf_kind_str(t1));
1521 return false;
1522 }
1523 }
1524
1525 static bool map_defs_match(const char *sym_name,
1526 const struct btf *main_btf,
1527 const struct btf_map_def *main_def,
1528 const struct btf_map_def *main_inner_def,
1529 const struct btf *extra_btf,
1530 const struct btf_map_def *extra_def,
1531 const struct btf_map_def *extra_inner_def)
1532 {
1533 const char *reason;
1534
1535 if (main_def->map_type != extra_def->map_type) {
1536 reason = "type";
1537 goto mismatch;
1538 }
1539
1540 /* check key type/size match */
1541 if (main_def->key_size != extra_def->key_size) {
1542 reason = "key_size";
1543 goto mismatch;
1544 }
1545 if (!!main_def->key_type_id != !!extra_def->key_type_id) {
1546 reason = "key type";
1547 goto mismatch;
1548 }
1549 if ((main_def->parts & MAP_DEF_KEY_TYPE)
1550 && !glob_sym_btf_matches(sym_name, true /*exact*/,
1551 main_btf, main_def->key_type_id,
1552 extra_btf, extra_def->key_type_id)) {
1553 reason = "key type";
1554 goto mismatch;
1555 }
1556
1557 /* validate value type/size match */
1558 if (main_def->value_size != extra_def->value_size) {
1559 reason = "value_size";
1560 goto mismatch;
1561 }
1562 if (!!main_def->value_type_id != !!extra_def->value_type_id) {
1563 reason = "value type";
1564 goto mismatch;
1565 }
1566 if ((main_def->parts & MAP_DEF_VALUE_TYPE)
1567 && !glob_sym_btf_matches(sym_name, true /*exact*/,
1568 main_btf, main_def->value_type_id,
1569 extra_btf, extra_def->value_type_id)) {
1570 reason = "key type";
1571 goto mismatch;
1572 }
1573
1574 if (main_def->max_entries != extra_def->max_entries) {
1575 reason = "max_entries";
1576 goto mismatch;
1577 }
1578 if (main_def->map_flags != extra_def->map_flags) {
1579 reason = "map_flags";
1580 goto mismatch;
1581 }
1582 if (main_def->numa_node != extra_def->numa_node) {
1583 reason = "numa_node";
1584 goto mismatch;
1585 }
1586 if (main_def->pinning != extra_def->pinning) {
1587 reason = "pinning";
1588 goto mismatch;
1589 }
1590
1591 if ((main_def->parts & MAP_DEF_INNER_MAP) != (extra_def->parts & MAP_DEF_INNER_MAP)) {
1592 reason = "inner map";
1593 goto mismatch;
1594 }
1595
1596 if (main_def->parts & MAP_DEF_INNER_MAP) {
1597 char inner_map_name[128];
1598
1599 snprintf(inner_map_name, sizeof(inner_map_name), "%s.inner", sym_name);
1600
1601 return map_defs_match(inner_map_name,
1602 main_btf, main_inner_def, NULL,
1603 extra_btf, extra_inner_def, NULL);
1604 }
1605
1606 return true;
1607
1608 mismatch:
1609 pr_warn("global '%s': map %s mismatch\n", sym_name, reason);
1610 return false;
1611 }
1612
1613 static bool glob_map_defs_match(const char *sym_name,
1614 struct bpf_linker *linker, struct glob_sym *glob_sym,
1615 struct src_obj *obj, Elf64_Sym *sym, int btf_id)
1616 {
1617 struct btf_map_def dst_def = {}, dst_inner_def = {};
1618 struct btf_map_def src_def = {}, src_inner_def = {};
1619 const struct btf_type *t;
1620 int err;
1621
1622 t = btf__type_by_id(obj->btf, btf_id);
1623 if (!btf_is_var(t)) {
1624 pr_warn("global '%s': invalid map definition type [%d]\n", sym_name, btf_id);
1625 return false;
1626 }
1627 t = skip_mods_and_typedefs(obj->btf, t->type, NULL);
1628
1629 err = parse_btf_map_def(sym_name, obj->btf, t, true /*strict*/, &src_def, &src_inner_def);
1630 if (err) {
1631 pr_warn("global '%s': invalid map definition\n", sym_name);
1632 return false;
1633 }
1634
1635 /* re-parse existing map definition */
1636 t = btf__type_by_id(linker->btf, glob_sym->btf_id);
1637 t = skip_mods_and_typedefs(linker->btf, t->type, NULL);
1638 err = parse_btf_map_def(sym_name, linker->btf, t, true /*strict*/, &dst_def, &dst_inner_def);
1639 if (err) {
1640 /* this should not happen, because we already validated it */
1641 pr_warn("global '%s': invalid dst map definition\n", sym_name);
1642 return false;
1643 }
1644
1645 /* Currently extern map definition has to be complete and match
1646 * concrete map definition exactly. This restriction might be lifted
1647 * in the future.
1648 */
1649 return map_defs_match(sym_name, linker->btf, &dst_def, &dst_inner_def,
1650 obj->btf, &src_def, &src_inner_def);
1651 }
1652
1653 static bool glob_syms_match(const char *sym_name,
1654 struct bpf_linker *linker, struct glob_sym *glob_sym,
1655 struct src_obj *obj, Elf64_Sym *sym, size_t sym_idx, int btf_id)
1656 {
1657 const struct btf_type *src_t;
1658
1659 /* if we are dealing with externs, BTF types describing both global
1660 * and extern VARs/FUNCs should be completely present in all files
1661 */
1662 if (!glob_sym->btf_id || !btf_id) {
1663 pr_warn("BTF info is missing for global symbol '%s'\n", sym_name);
1664 return false;
1665 }
1666
1667 src_t = btf__type_by_id(obj->btf, btf_id);
1668 if (!btf_is_var(src_t) && !btf_is_func(src_t)) {
1669 pr_warn("only extern variables and functions are supported, but got '%s' for '%s'\n",
1670 btf_kind_str(src_t), sym_name);
1671 return false;
1672 }
1673
1674 /* deal with .maps definitions specially */
1675 if (glob_sym->sec_id && strcmp(linker->secs[glob_sym->sec_id].sec_name, MAPS_ELF_SEC) == 0)
1676 return glob_map_defs_match(sym_name, linker, glob_sym, obj, sym, btf_id);
1677
1678 if (!glob_sym_btf_matches(sym_name, true /*exact*/,
1679 linker->btf, glob_sym->btf_id, obj->btf, btf_id))
1680 return false;
1681
1682 return true;
1683 }
1684
1685 static bool btf_is_non_static(const struct btf_type *t)
1686 {
1687 return (btf_is_var(t) && btf_var(t)->linkage != BTF_VAR_STATIC)
1688 || (btf_is_func(t) && btf_func_linkage(t) != BTF_FUNC_STATIC);
1689 }
1690
1691 static int find_glob_sym_btf(struct src_obj *obj, Elf64_Sym *sym, const char *sym_name,
1692 int *out_btf_sec_id, int *out_btf_id)
1693 {
1694 int i, j, n, m, btf_id = 0;
1695 const struct btf_type *t;
1696 const struct btf_var_secinfo *vi;
1697 const char *name;
1698
1699 if (!obj->btf) {
1700 pr_warn("failed to find BTF info for object '%s'\n", obj->filename);
1701 return -EINVAL;
1702 }
1703
1704 n = btf__type_cnt(obj->btf);
1705 for (i = 1; i < n; i++) {
1706 t = btf__type_by_id(obj->btf, i);
1707
1708 /* some global and extern FUNCs and VARs might not be associated with any
1709 * DATASEC, so try to detect them in the same pass
1710 */
1711 if (btf_is_non_static(t)) {
1712 name = btf__str_by_offset(obj->btf, t->name_off);
1713 if (strcmp(name, sym_name) != 0)
1714 continue;
1715
1716 /* remember and still try to find DATASEC */
1717 btf_id = i;
1718 continue;
1719 }
1720
1721 if (!btf_is_datasec(t))
1722 continue;
1723
1724 vi = btf_var_secinfos(t);
1725 for (j = 0, m = btf_vlen(t); j < m; j++, vi++) {
1726 t = btf__type_by_id(obj->btf, vi->type);
1727 name = btf__str_by_offset(obj->btf, t->name_off);
1728
1729 if (strcmp(name, sym_name) != 0)
1730 continue;
1731 if (btf_is_var(t) && btf_var(t)->linkage == BTF_VAR_STATIC)
1732 continue;
1733 if (btf_is_func(t) && btf_func_linkage(t) == BTF_FUNC_STATIC)
1734 continue;
1735
1736 if (btf_id && btf_id != vi->type) {
1737 pr_warn("global/extern '%s' BTF is ambiguous: both types #%d and #%u match\n",
1738 sym_name, btf_id, vi->type);
1739 return -EINVAL;
1740 }
1741
1742 *out_btf_sec_id = i;
1743 *out_btf_id = vi->type;
1744
1745 return 0;
1746 }
1747 }
1748
1749 /* free-floating extern or global FUNC */
1750 if (btf_id) {
1751 *out_btf_sec_id = 0;
1752 *out_btf_id = btf_id;
1753 return 0;
1754 }
1755
1756 pr_warn("failed to find BTF info for global/extern symbol '%s'\n", sym_name);
1757 return -ENOENT;
1758 }
1759
1760 static struct src_sec *find_src_sec_by_name(struct src_obj *obj, const char *sec_name)
1761 {
1762 struct src_sec *sec;
1763 int i;
1764
1765 for (i = 1; i < obj->sec_cnt; i++) {
1766 sec = &obj->secs[i];
1767
1768 if (strcmp(sec->sec_name, sec_name) == 0)
1769 return sec;
1770 }
1771
1772 return NULL;
1773 }
1774
1775 static int complete_extern_btf_info(struct btf *dst_btf, int dst_id,
1776 struct btf *src_btf, int src_id)
1777 {
1778 struct btf_type *dst_t = btf_type_by_id(dst_btf, dst_id);
1779 struct btf_type *src_t = btf_type_by_id(src_btf, src_id);
1780 struct btf_param *src_p, *dst_p;
1781 const char *s;
1782 int i, n, off;
1783
1784 /* We already made sure that source and destination types (FUNC or
1785 * VAR) match in terms of types and argument names.
1786 */
1787 if (btf_is_var(dst_t)) {
1788 btf_var(dst_t)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
1789 return 0;
1790 }
1791
1792 dst_t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_GLOBAL, 0);
1793
1794 /* now onto FUNC_PROTO types */
1795 src_t = btf_type_by_id(src_btf, src_t->type);
1796 dst_t = btf_type_by_id(dst_btf, dst_t->type);
1797
1798 /* Fill in all the argument names, which for extern FUNCs are missing.
1799 * We'll end up with two copies of FUNCs/VARs for externs, but that
1800 * will be taken care of by BTF dedup at the very end.
1801 * It might be that BTF types for extern in one file has less/more BTF
1802 * information (e.g., FWD instead of full STRUCT/UNION information),
1803 * but that should be (in most cases, subject to BTF dedup rules)
1804 * handled and resolved by BTF dedup algorithm as well, so we won't
1805 * worry about it. Our only job is to make sure that argument names
1806 * are populated on both sides, otherwise BTF dedup will pedantically
1807 * consider them different.
1808 */
1809 src_p = btf_params(src_t);
1810 dst_p = btf_params(dst_t);
1811 for (i = 0, n = btf_vlen(dst_t); i < n; i++, src_p++, dst_p++) {
1812 if (!src_p->name_off)
1813 continue;
1814
1815 /* src_btf has more complete info, so add name to dst_btf */
1816 s = btf__str_by_offset(src_btf, src_p->name_off);
1817 off = btf__add_str(dst_btf, s);
1818 if (off < 0)
1819 return off;
1820 dst_p->name_off = off;
1821 }
1822 return 0;
1823 }
1824
1825 static void sym_update_bind(Elf64_Sym *sym, int sym_bind)
1826 {
1827 sym->st_info = ELF64_ST_INFO(sym_bind, ELF64_ST_TYPE(sym->st_info));
1828 }
1829
1830 static void sym_update_type(Elf64_Sym *sym, int sym_type)
1831 {
1832 sym->st_info = ELF64_ST_INFO(ELF64_ST_BIND(sym->st_info), sym_type);
1833 }
1834
1835 static void sym_update_visibility(Elf64_Sym *sym, int sym_vis)
1836 {
1837 /* libelf doesn't provide setters for ST_VISIBILITY,
1838 * but it is stored in the lower 2 bits of st_other
1839 */
1840 sym->st_other &= ~0x03;
1841 sym->st_other |= sym_vis;
1842 }
1843
1844 static int linker_append_elf_sym(struct bpf_linker *linker, struct src_obj *obj,
1845 Elf64_Sym *sym, const char *sym_name, int src_sym_idx)
1846 {
1847 struct src_sec *src_sec = NULL;
1848 struct dst_sec *dst_sec = NULL;
1849 struct glob_sym *glob_sym = NULL;
1850 int name_off, sym_type, sym_bind, sym_vis, err;
1851 int btf_sec_id = 0, btf_id = 0;
1852 size_t dst_sym_idx;
1853 Elf64_Sym *dst_sym;
1854 bool sym_is_extern;
1855
1856 sym_type = ELF64_ST_TYPE(sym->st_info);
1857 sym_bind = ELF64_ST_BIND(sym->st_info);
1858 sym_vis = ELF64_ST_VISIBILITY(sym->st_other);
1859 sym_is_extern = sym->st_shndx == SHN_UNDEF;
1860
1861 if (sym_is_extern) {
1862 if (!obj->btf) {
1863 pr_warn("externs without BTF info are not supported\n");
1864 return -ENOTSUP;
1865 }
1866 } else if (sym->st_shndx < SHN_LORESERVE) {
1867 src_sec = &obj->secs[sym->st_shndx];
1868 if (src_sec->skipped)
1869 return 0;
1870 dst_sec = &linker->secs[src_sec->dst_id];
1871
1872 /* allow only one STT_SECTION symbol per section */
1873 if (sym_type == STT_SECTION && dst_sec->sec_sym_idx) {
1874 obj->sym_map[src_sym_idx] = dst_sec->sec_sym_idx;
1875 return 0;
1876 }
1877 }
1878
1879 if (sym_bind == STB_LOCAL)
1880 goto add_sym;
1881
1882 /* find matching BTF info */
1883 err = find_glob_sym_btf(obj, sym, sym_name, &btf_sec_id, &btf_id);
1884 if (err)
1885 return err;
1886
1887 if (sym_is_extern && btf_sec_id) {
1888 const char *sec_name = NULL;
1889 const struct btf_type *t;
1890
1891 t = btf__type_by_id(obj->btf, btf_sec_id);
1892 sec_name = btf__str_by_offset(obj->btf, t->name_off);
1893
1894 /* Clang puts unannotated extern vars into
1895 * '.extern' BTF DATASEC. Treat them the same
1896 * as unannotated extern funcs (which are
1897 * currently not put into any DATASECs).
1898 * Those don't have associated src_sec/dst_sec.
1899 */
1900 if (strcmp(sec_name, BTF_EXTERN_SEC) != 0) {
1901 src_sec = find_src_sec_by_name(obj, sec_name);
1902 if (!src_sec) {
1903 pr_warn("failed to find matching ELF sec '%s'\n", sec_name);
1904 return -ENOENT;
1905 }
1906 dst_sec = &linker->secs[src_sec->dst_id];
1907 }
1908 }
1909
1910 glob_sym = find_glob_sym(linker, sym_name);
1911 if (glob_sym) {
1912 /* Preventively resolve to existing symbol. This is
1913 * needed for further relocation symbol remapping in
1914 * the next step of linking.
1915 */
1916 obj->sym_map[src_sym_idx] = glob_sym->sym_idx;
1917
1918 /* If both symbols are non-externs, at least one of
1919 * them has to be STB_WEAK, otherwise they are in
1920 * a conflict with each other.
1921 */
1922 if (!sym_is_extern && !glob_sym->is_extern
1923 && !glob_sym->is_weak && sym_bind != STB_WEAK) {
1924 pr_warn("conflicting non-weak symbol #%d (%s) definition in '%s'\n",
1925 src_sym_idx, sym_name, obj->filename);
1926 return -EINVAL;
1927 }
1928
1929 if (!glob_syms_match(sym_name, linker, glob_sym, obj, sym, src_sym_idx, btf_id))
1930 return -EINVAL;
1931
1932 dst_sym = get_sym_by_idx(linker, glob_sym->sym_idx);
1933
1934 /* If new symbol is strong, then force dst_sym to be strong as
1935 * well; this way a mix of weak and non-weak extern
1936 * definitions will end up being strong.
1937 */
1938 if (sym_bind == STB_GLOBAL) {
1939 /* We still need to preserve type (NOTYPE or
1940 * OBJECT/FUNC, depending on whether the symbol is
1941 * extern or not)
1942 */
1943 sym_update_bind(dst_sym, STB_GLOBAL);
1944 glob_sym->is_weak = false;
1945 }
1946
1947 /* Non-default visibility is "contaminating", with stricter
1948 * visibility overwriting more permissive ones, even if more
1949 * permissive visibility comes from just an extern definition.
1950 * Currently only STV_DEFAULT and STV_HIDDEN are allowed and
1951 * ensured by ELF symbol sanity checks above.
1952 */
1953 if (sym_vis > ELF64_ST_VISIBILITY(dst_sym->st_other))
1954 sym_update_visibility(dst_sym, sym_vis);
1955
1956 /* If the new symbol is extern, then regardless if
1957 * existing symbol is extern or resolved global, just
1958 * keep the existing one untouched.
1959 */
1960 if (sym_is_extern)
1961 return 0;
1962
1963 /* If existing symbol is a strong resolved symbol, bail out,
1964 * because we lost resolution battle have nothing to
1965 * contribute. We already checked abover that there is no
1966 * strong-strong conflict. We also already tightened binding
1967 * and visibility, so nothing else to contribute at that point.
1968 */
1969 if (!glob_sym->is_extern && sym_bind == STB_WEAK)
1970 return 0;
1971
1972 /* At this point, new symbol is strong non-extern,
1973 * so overwrite glob_sym with new symbol information.
1974 * Preserve binding and visibility.
1975 */
1976 sym_update_type(dst_sym, sym_type);
1977 dst_sym->st_shndx = dst_sec->sec_idx;
1978 dst_sym->st_value = src_sec->dst_off + sym->st_value;
1979 dst_sym->st_size = sym->st_size;
1980
1981 /* see comment below about dst_sec->id vs dst_sec->sec_idx */
1982 glob_sym->sec_id = dst_sec->id;
1983 glob_sym->is_extern = false;
1984
1985 if (complete_extern_btf_info(linker->btf, glob_sym->btf_id,
1986 obj->btf, btf_id))
1987 return -EINVAL;
1988
1989 /* request updating VAR's/FUNC's underlying BTF type when appending BTF type */
1990 glob_sym->underlying_btf_id = 0;
1991
1992 obj->sym_map[src_sym_idx] = glob_sym->sym_idx;
1993 return 0;
1994 }
1995
1996 add_sym:
1997 name_off = strset__add_str(linker->strtab_strs, sym_name);
1998 if (name_off < 0)
1999 return name_off;
2000
2001 dst_sym = add_new_sym(linker, &dst_sym_idx);
2002 if (!dst_sym)
2003 return -ENOMEM;
2004
2005 dst_sym->st_name = name_off;
2006 dst_sym->st_info = sym->st_info;
2007 dst_sym->st_other = sym->st_other;
2008 dst_sym->st_shndx = dst_sec ? dst_sec->sec_idx : sym->st_shndx;
2009 dst_sym->st_value = (src_sec ? src_sec->dst_off : 0) + sym->st_value;
2010 dst_sym->st_size = sym->st_size;
2011
2012 obj->sym_map[src_sym_idx] = dst_sym_idx;
2013
2014 if (sym_type == STT_SECTION && dst_sym) {
2015 dst_sec->sec_sym_idx = dst_sym_idx;
2016 dst_sym->st_value = 0;
2017 }
2018
2019 if (sym_bind != STB_LOCAL) {
2020 glob_sym = add_glob_sym(linker);
2021 if (!glob_sym)
2022 return -ENOMEM;
2023
2024 glob_sym->sym_idx = dst_sym_idx;
2025 /* we use dst_sec->id (and not dst_sec->sec_idx), because
2026 * ephemeral sections (.kconfig, .ksyms, etc) don't have
2027 * sec_idx (as they don't have corresponding ELF section), but
2028 * still have id. .extern doesn't have even ephemeral section
2029 * associated with it, so dst_sec->id == dst_sec->sec_idx == 0.
2030 */
2031 glob_sym->sec_id = dst_sec ? dst_sec->id : 0;
2032 glob_sym->name_off = name_off;
2033 /* we will fill btf_id in during BTF merging step */
2034 glob_sym->btf_id = 0;
2035 glob_sym->is_extern = sym_is_extern;
2036 glob_sym->is_weak = sym_bind == STB_WEAK;
2037 }
2038
2039 return 0;
2040 }
2041
2042 static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj)
2043 {
2044 struct src_sec *src_symtab = &obj->secs[obj->symtab_sec_idx];
2045 int i, err;
2046
2047 for (i = 1; i < obj->sec_cnt; i++) {
2048 struct src_sec *src_sec, *src_linked_sec;
2049 struct dst_sec *dst_sec, *dst_linked_sec;
2050 Elf64_Rel *src_rel, *dst_rel;
2051 int j, n;
2052
2053 src_sec = &obj->secs[i];
2054 if (!is_relo_sec(src_sec))
2055 continue;
2056
2057 /* shdr->sh_info points to relocatable section */
2058 src_linked_sec = &obj->secs[src_sec->shdr->sh_info];
2059 if (src_linked_sec->skipped)
2060 continue;
2061
2062 dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name);
2063 if (!dst_sec) {
2064 dst_sec = add_dst_sec(linker, src_sec->sec_name);
2065 if (!dst_sec)
2066 return -ENOMEM;
2067 err = init_sec(linker, dst_sec, src_sec);
2068 if (err) {
2069 pr_warn("failed to init section '%s'\n", src_sec->sec_name);
2070 return err;
2071 }
2072 } else if (!secs_match(dst_sec, src_sec)) {
2073 pr_warn("sections %s are not compatible\n", src_sec->sec_name);
2074 return -1;
2075 }
2076
2077 /* shdr->sh_link points to SYMTAB */
2078 dst_sec->shdr->sh_link = linker->symtab_sec_idx;
2079
2080 /* shdr->sh_info points to relocated section */
2081 dst_linked_sec = &linker->secs[src_linked_sec->dst_id];
2082 dst_sec->shdr->sh_info = dst_linked_sec->sec_idx;
2083
2084 src_sec->dst_id = dst_sec->id;
2085 err = extend_sec(linker, dst_sec, src_sec);
2086 if (err)
2087 return err;
2088
2089 src_rel = src_sec->data->d_buf;
2090 dst_rel = dst_sec->raw_data + src_sec->dst_off;
2091 n = src_sec->shdr->sh_size / src_sec->shdr->sh_entsize;
2092 for (j = 0; j < n; j++, src_rel++, dst_rel++) {
2093 size_t src_sym_idx, dst_sym_idx, sym_type;
2094 Elf64_Sym *src_sym;
2095
2096 src_sym_idx = ELF64_R_SYM(src_rel->r_info);
2097 src_sym = src_symtab->data->d_buf + sizeof(*src_sym) * src_sym_idx;
2098
2099 dst_sym_idx = obj->sym_map[src_sym_idx];
2100 dst_rel->r_offset += src_linked_sec->dst_off;
2101 sym_type = ELF64_R_TYPE(src_rel->r_info);
2102 dst_rel->r_info = ELF64_R_INFO(dst_sym_idx, sym_type);
2103
2104 if (ELF64_ST_TYPE(src_sym->st_info) == STT_SECTION) {
2105 struct src_sec *sec = &obj->secs[src_sym->st_shndx];
2106 struct bpf_insn *insn;
2107
2108 if (src_linked_sec->shdr->sh_flags & SHF_EXECINSTR) {
2109 /* calls to the very first static function inside
2110 * .text section at offset 0 will
2111 * reference section symbol, not the
2112 * function symbol. Fix that up,
2113 * otherwise it won't be possible to
2114 * relocate calls to two different
2115 * static functions with the same name
2116 * (rom two different object files)
2117 */
2118 insn = dst_linked_sec->raw_data + dst_rel->r_offset;
2119 if (insn->code == (BPF_JMP | BPF_CALL))
2120 insn->imm += sec->dst_off / sizeof(struct bpf_insn);
2121 else
2122 insn->imm += sec->dst_off;
2123 } else {
2124 pr_warn("relocation against STT_SECTION in non-exec section is not supported!\n");
2125 return -EINVAL;
2126 }
2127 }
2128
2129 }
2130 }
2131
2132 return 0;
2133 }
2134
2135 static Elf64_Sym *find_sym_by_name(struct src_obj *obj, size_t sec_idx,
2136 int sym_type, const char *sym_name)
2137 {
2138 struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx];
2139 Elf64_Sym *sym = symtab->data->d_buf;
2140 int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize;
2141 int str_sec_idx = symtab->shdr->sh_link;
2142 const char *name;
2143
2144 for (i = 0; i < n; i++, sym++) {
2145 if (sym->st_shndx != sec_idx)
2146 continue;
2147 if (ELF64_ST_TYPE(sym->st_info) != sym_type)
2148 continue;
2149
2150 name = elf_strptr(obj->elf, str_sec_idx, sym->st_name);
2151 if (!name)
2152 return NULL;
2153
2154 if (strcmp(sym_name, name) != 0)
2155 continue;
2156
2157 return sym;
2158 }
2159
2160 return NULL;
2161 }
2162
2163 static int linker_fixup_btf(struct src_obj *obj)
2164 {
2165 const char *sec_name;
2166 struct src_sec *sec;
2167 int i, j, n, m;
2168
2169 if (!obj->btf)
2170 return 0;
2171
2172 n = btf__type_cnt(obj->btf);
2173 for (i = 1; i < n; i++) {
2174 struct btf_var_secinfo *vi;
2175 struct btf_type *t;
2176
2177 t = btf_type_by_id(obj->btf, i);
2178 if (btf_kind(t) != BTF_KIND_DATASEC)
2179 continue;
2180
2181 sec_name = btf__str_by_offset(obj->btf, t->name_off);
2182 sec = find_src_sec_by_name(obj, sec_name);
2183 if (sec) {
2184 /* record actual section size, unless ephemeral */
2185 if (sec->shdr)
2186 t->size = sec->shdr->sh_size;
2187 } else {
2188 /* BTF can have some sections that are not represented
2189 * in ELF, e.g., .kconfig, .ksyms, .extern, which are used
2190 * for special extern variables.
2191 *
2192 * For all but one such special (ephemeral)
2193 * sections, we pre-create "section shells" to be able
2194 * to keep track of extra per-section metadata later
2195 * (e.g., those BTF extern variables).
2196 *
2197 * .extern is even more special, though, because it
2198 * contains extern variables that need to be resolved
2199 * by static linker, not libbpf and kernel. When such
2200 * externs are resolved, we are going to remove them
2201 * from .extern BTF section and might end up not
2202 * needing it at all. Each resolved extern should have
2203 * matching non-extern VAR/FUNC in other sections.
2204 *
2205 * We do support leaving some of the externs
2206 * unresolved, though, to support cases of building
2207 * libraries, which will later be linked against final
2208 * BPF applications. So if at finalization we still
2209 * see unresolved externs, we'll create .extern
2210 * section on our own.
2211 */
2212 if (strcmp(sec_name, BTF_EXTERN_SEC) == 0)
2213 continue;
2214
2215 sec = add_src_sec(obj, sec_name);
2216 if (!sec)
2217 return -ENOMEM;
2218
2219 sec->ephemeral = true;
2220 sec->sec_idx = 0; /* will match UNDEF shndx in ELF */
2221 }
2222
2223 /* remember ELF section and its BTF type ID match */
2224 sec->sec_type_id = i;
2225
2226 /* fix up variable offsets */
2227 vi = btf_var_secinfos(t);
2228 for (j = 0, m = btf_vlen(t); j < m; j++, vi++) {
2229 const struct btf_type *vt = btf__type_by_id(obj->btf, vi->type);
2230 const char *var_name;
2231 int var_linkage;
2232 Elf64_Sym *sym;
2233
2234 /* could be a variable or function */
2235 if (!btf_is_var(vt))
2236 continue;
2237
2238 var_name = btf__str_by_offset(obj->btf, vt->name_off);
2239 var_linkage = btf_var(vt)->linkage;
2240
2241 /* no need to patch up static or extern vars */
2242 if (var_linkage != BTF_VAR_GLOBAL_ALLOCATED)
2243 continue;
2244
2245 sym = find_sym_by_name(obj, sec->sec_idx, STT_OBJECT, var_name);
2246 if (!sym) {
2247 pr_warn("failed to find symbol for variable '%s' in section '%s'\n", var_name, sec_name);
2248 return -ENOENT;
2249 }
2250
2251 vi->offset = sym->st_value;
2252 }
2253 }
2254
2255 return 0;
2256 }
2257
2258 static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj)
2259 {
2260 const struct btf_type *t;
2261 int i, j, n, start_id, id, err;
2262 const char *name;
2263
2264 if (!obj->btf)
2265 return 0;
2266
2267 start_id = btf__type_cnt(linker->btf);
2268 n = btf__type_cnt(obj->btf);
2269
2270 obj->btf_type_map = calloc(n + 1, sizeof(int));
2271 if (!obj->btf_type_map)
2272 return -ENOMEM;
2273
2274 for (i = 1; i < n; i++) {
2275 struct glob_sym *glob_sym = NULL;
2276
2277 t = btf__type_by_id(obj->btf, i);
2278
2279 /* DATASECs are handled specially below */
2280 if (btf_kind(t) == BTF_KIND_DATASEC)
2281 continue;
2282
2283 if (btf_is_non_static(t)) {
2284 /* there should be glob_sym already */
2285 name = btf__str_by_offset(obj->btf, t->name_off);
2286 glob_sym = find_glob_sym(linker, name);
2287
2288 /* VARs without corresponding glob_sym are those that
2289 * belong to skipped/deduplicated sections (i.e.,
2290 * license and version), so just skip them
2291 */
2292 if (!glob_sym)
2293 continue;
2294
2295 /* linker_append_elf_sym() might have requested
2296 * updating underlying type ID, if extern was resolved
2297 * to strong symbol or weak got upgraded to non-weak
2298 */
2299 if (glob_sym->underlying_btf_id == 0)
2300 glob_sym->underlying_btf_id = -t->type;
2301
2302 /* globals from previous object files that match our
2303 * VAR/FUNC already have a corresponding associated
2304 * BTF type, so just make sure to use it
2305 */
2306 if (glob_sym->btf_id) {
2307 /* reuse existing BTF type for global var/func */
2308 obj->btf_type_map[i] = glob_sym->btf_id;
2309 continue;
2310 }
2311 }
2312
2313 id = btf__add_type(linker->btf, obj->btf, t);
2314 if (id < 0) {
2315 pr_warn("failed to append BTF type #%d from file '%s'\n", i, obj->filename);
2316 return id;
2317 }
2318
2319 obj->btf_type_map[i] = id;
2320
2321 /* record just appended BTF type for var/func */
2322 if (glob_sym) {
2323 glob_sym->btf_id = id;
2324 glob_sym->underlying_btf_id = -t->type;
2325 }
2326 }
2327
2328 /* remap all the types except DATASECs */
2329 n = btf__type_cnt(linker->btf);
2330 for (i = start_id; i < n; i++) {
2331 struct btf_type *dst_t = btf_type_by_id(linker->btf, i);
2332 struct btf_field_iter it;
2333 __u32 *type_id;
2334
2335 err = btf_field_iter_init(&it, dst_t, BTF_FIELD_ITER_IDS);
2336 if (err)
2337 return err;
2338
2339 while ((type_id = btf_field_iter_next(&it))) {
2340 int new_id = obj->btf_type_map[*type_id];
2341
2342 /* Error out if the type wasn't remapped. Ignore VOID which stays VOID. */
2343 if (new_id == 0 && *type_id != 0) {
2344 pr_warn("failed to find new ID mapping for original BTF type ID %u\n",
2345 *type_id);
2346 return -EINVAL;
2347 }
2348
2349 *type_id = obj->btf_type_map[*type_id];
2350 }
2351 }
2352
2353 /* Rewrite VAR/FUNC underlying types (i.e., FUNC's FUNC_PROTO and VAR's
2354 * actual type), if necessary
2355 */
2356 for (i = 0; i < linker->glob_sym_cnt; i++) {
2357 struct glob_sym *glob_sym = &linker->glob_syms[i];
2358 struct btf_type *glob_t;
2359
2360 if (glob_sym->underlying_btf_id >= 0)
2361 continue;
2362
2363 glob_sym->underlying_btf_id = obj->btf_type_map[-glob_sym->underlying_btf_id];
2364
2365 glob_t = btf_type_by_id(linker->btf, glob_sym->btf_id);
2366 glob_t->type = glob_sym->underlying_btf_id;
2367 }
2368
2369 /* append DATASEC info */
2370 for (i = 1; i < obj->sec_cnt; i++) {
2371 struct src_sec *src_sec;
2372 struct dst_sec *dst_sec;
2373 const struct btf_var_secinfo *src_var;
2374 struct btf_var_secinfo *dst_var;
2375
2376 src_sec = &obj->secs[i];
2377 if (!src_sec->sec_type_id || src_sec->skipped)
2378 continue;
2379 dst_sec = &linker->secs[src_sec->dst_id];
2380
2381 /* Mark section as having BTF regardless of the presence of
2382 * variables. In some cases compiler might generate empty BTF
2383 * with no variables information. E.g., when promoting local
2384 * array/structure variable initial values and BPF object
2385 * file otherwise has no read-only static variables in
2386 * .rodata. We need to preserve such empty BTF and just set
2387 * correct section size.
2388 */
2389 dst_sec->has_btf = true;
2390
2391 t = btf__type_by_id(obj->btf, src_sec->sec_type_id);
2392 src_var = btf_var_secinfos(t);
2393 n = btf_vlen(t);
2394 for (j = 0; j < n; j++, src_var++) {
2395 void *sec_vars = dst_sec->sec_vars;
2396 int new_id = obj->btf_type_map[src_var->type];
2397 struct glob_sym *glob_sym = NULL;
2398
2399 t = btf_type_by_id(linker->btf, new_id);
2400 if (btf_is_non_static(t)) {
2401 name = btf__str_by_offset(linker->btf, t->name_off);
2402 glob_sym = find_glob_sym(linker, name);
2403 if (glob_sym->sec_id != dst_sec->id) {
2404 pr_warn("global '%s': section mismatch %d vs %d\n",
2405 name, glob_sym->sec_id, dst_sec->id);
2406 return -EINVAL;
2407 }
2408 }
2409
2410 /* If there is already a member (VAR or FUNC) mapped
2411 * to the same type, don't add a duplicate entry.
2412 * This will happen when multiple object files define
2413 * the same extern VARs/FUNCs.
2414 */
2415 if (glob_sym && glob_sym->var_idx >= 0) {
2416 __s64 sz;
2417
2418 dst_var = &dst_sec->sec_vars[glob_sym->var_idx];
2419 /* Because underlying BTF type might have
2420 * changed, so might its size have changed, so
2421 * re-calculate and update it in sec_var.
2422 */
2423 sz = btf__resolve_size(linker->btf, glob_sym->underlying_btf_id);
2424 if (sz < 0) {
2425 pr_warn("global '%s': failed to resolve size of underlying type: %d\n",
2426 name, (int)sz);
2427 return -EINVAL;
2428 }
2429 dst_var->size = sz;
2430 continue;
2431 }
2432
2433 sec_vars = libbpf_reallocarray(sec_vars,
2434 dst_sec->sec_var_cnt + 1,
2435 sizeof(*dst_sec->sec_vars));
2436 if (!sec_vars)
2437 return -ENOMEM;
2438
2439 dst_sec->sec_vars = sec_vars;
2440 dst_sec->sec_var_cnt++;
2441
2442 dst_var = &dst_sec->sec_vars[dst_sec->sec_var_cnt - 1];
2443 dst_var->type = obj->btf_type_map[src_var->type];
2444 dst_var->size = src_var->size;
2445 dst_var->offset = src_sec->dst_off + src_var->offset;
2446
2447 if (glob_sym)
2448 glob_sym->var_idx = dst_sec->sec_var_cnt - 1;
2449 }
2450 }
2451
2452 return 0;
2453 }
2454
2455 static void *add_btf_ext_rec(struct btf_ext_sec_data *ext_data, const void *src_rec)
2456 {
2457 void *tmp;
2458
2459 tmp = libbpf_reallocarray(ext_data->recs, ext_data->rec_cnt + 1, ext_data->rec_sz);
2460 if (!tmp)
2461 return NULL;
2462 ext_data->recs = tmp;
2463
2464 tmp += ext_data->rec_cnt * ext_data->rec_sz;
2465 memcpy(tmp, src_rec, ext_data->rec_sz);
2466
2467 ext_data->rec_cnt++;
2468
2469 return tmp;
2470 }
2471
2472 static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj)
2473 {
2474 const struct btf_ext_info_sec *ext_sec;
2475 const char *sec_name, *s;
2476 struct src_sec *src_sec;
2477 struct dst_sec *dst_sec;
2478 int rec_sz, str_off, i;
2479
2480 if (!obj->btf_ext)
2481 return 0;
2482
2483 rec_sz = obj->btf_ext->func_info.rec_size;
2484 for_each_btf_ext_sec(&obj->btf_ext->func_info, ext_sec) {
2485 struct bpf_func_info_min *src_rec, *dst_rec;
2486
2487 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2488 src_sec = find_src_sec_by_name(obj, sec_name);
2489 if (!src_sec) {
2490 pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2491 return -EINVAL;
2492 }
2493 dst_sec = &linker->secs[src_sec->dst_id];
2494
2495 if (dst_sec->func_info.rec_sz == 0)
2496 dst_sec->func_info.rec_sz = rec_sz;
2497 if (dst_sec->func_info.rec_sz != rec_sz) {
2498 pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2499 return -EINVAL;
2500 }
2501
2502 for_each_btf_ext_rec(&obj->btf_ext->func_info, ext_sec, i, src_rec) {
2503 dst_rec = add_btf_ext_rec(&dst_sec->func_info, src_rec);
2504 if (!dst_rec)
2505 return -ENOMEM;
2506
2507 dst_rec->insn_off += src_sec->dst_off;
2508 dst_rec->type_id = obj->btf_type_map[dst_rec->type_id];
2509 }
2510 }
2511
2512 rec_sz = obj->btf_ext->line_info.rec_size;
2513 for_each_btf_ext_sec(&obj->btf_ext->line_info, ext_sec) {
2514 struct bpf_line_info_min *src_rec, *dst_rec;
2515
2516 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2517 src_sec = find_src_sec_by_name(obj, sec_name);
2518 if (!src_sec) {
2519 pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2520 return -EINVAL;
2521 }
2522 dst_sec = &linker->secs[src_sec->dst_id];
2523
2524 if (dst_sec->line_info.rec_sz == 0)
2525 dst_sec->line_info.rec_sz = rec_sz;
2526 if (dst_sec->line_info.rec_sz != rec_sz) {
2527 pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2528 return -EINVAL;
2529 }
2530
2531 for_each_btf_ext_rec(&obj->btf_ext->line_info, ext_sec, i, src_rec) {
2532 dst_rec = add_btf_ext_rec(&dst_sec->line_info, src_rec);
2533 if (!dst_rec)
2534 return -ENOMEM;
2535
2536 dst_rec->insn_off += src_sec->dst_off;
2537
2538 s = btf__str_by_offset(obj->btf, src_rec->file_name_off);
2539 str_off = btf__add_str(linker->btf, s);
2540 if (str_off < 0)
2541 return -ENOMEM;
2542 dst_rec->file_name_off = str_off;
2543
2544 s = btf__str_by_offset(obj->btf, src_rec->line_off);
2545 str_off = btf__add_str(linker->btf, s);
2546 if (str_off < 0)
2547 return -ENOMEM;
2548 dst_rec->line_off = str_off;
2549
2550 /* dst_rec->line_col is fine */
2551 }
2552 }
2553
2554 rec_sz = obj->btf_ext->core_relo_info.rec_size;
2555 for_each_btf_ext_sec(&obj->btf_ext->core_relo_info, ext_sec) {
2556 struct bpf_core_relo *src_rec, *dst_rec;
2557
2558 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2559 src_sec = find_src_sec_by_name(obj, sec_name);
2560 if (!src_sec) {
2561 pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2562 return -EINVAL;
2563 }
2564 dst_sec = &linker->secs[src_sec->dst_id];
2565
2566 if (dst_sec->core_relo_info.rec_sz == 0)
2567 dst_sec->core_relo_info.rec_sz = rec_sz;
2568 if (dst_sec->core_relo_info.rec_sz != rec_sz) {
2569 pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2570 return -EINVAL;
2571 }
2572
2573 for_each_btf_ext_rec(&obj->btf_ext->core_relo_info, ext_sec, i, src_rec) {
2574 dst_rec = add_btf_ext_rec(&dst_sec->core_relo_info, src_rec);
2575 if (!dst_rec)
2576 return -ENOMEM;
2577
2578 dst_rec->insn_off += src_sec->dst_off;
2579 dst_rec->type_id = obj->btf_type_map[dst_rec->type_id];
2580
2581 s = btf__str_by_offset(obj->btf, src_rec->access_str_off);
2582 str_off = btf__add_str(linker->btf, s);
2583 if (str_off < 0)
2584 return -ENOMEM;
2585 dst_rec->access_str_off = str_off;
2586
2587 /* dst_rec->kind is fine */
2588 }
2589 }
2590
2591 return 0;
2592 }
2593
2594 int bpf_linker__finalize(struct bpf_linker *linker)
2595 {
2596 struct dst_sec *sec;
2597 size_t strs_sz;
2598 const void *strs;
2599 int err, i;
2600
2601 if (!linker->elf)
2602 return libbpf_err(-EINVAL);
2603
2604 err = finalize_btf(linker);
2605 if (err)
2606 return libbpf_err(err);
2607
2608 /* Finalize strings */
2609 strs_sz = strset__data_size(linker->strtab_strs);
2610 strs = strset__data(linker->strtab_strs);
2611
2612 sec = &linker->secs[linker->strtab_sec_idx];
2613 sec->data->d_align = 1;
2614 sec->data->d_off = 0LL;
2615 sec->data->d_buf = (void *)strs;
2616 sec->data->d_type = ELF_T_BYTE;
2617 sec->data->d_size = strs_sz;
2618 sec->shdr->sh_size = strs_sz;
2619
2620 for (i = 1; i < linker->sec_cnt; i++) {
2621 sec = &linker->secs[i];
2622
2623 /* STRTAB is handled specially above */
2624 if (sec->sec_idx == linker->strtab_sec_idx)
2625 continue;
2626
2627 /* special ephemeral sections (.ksyms, .kconfig, etc) */
2628 if (!sec->scn)
2629 continue;
2630
2631 sec->data->d_buf = sec->raw_data;
2632 }
2633
2634 /* Finalize ELF layout */
2635 if (elf_update(linker->elf, ELF_C_NULL) < 0) {
2636 err = -errno;
2637 pr_warn_elf("failed to finalize ELF layout");
2638 return libbpf_err(err);
2639 }
2640
2641 /* Write out final ELF contents */
2642 if (elf_update(linker->elf, ELF_C_WRITE) < 0) {
2643 err = -errno;
2644 pr_warn_elf("failed to write ELF contents");
2645 return libbpf_err(err);
2646 }
2647
2648 elf_end(linker->elf);
2649 close(linker->fd);
2650
2651 linker->elf = NULL;
2652 linker->fd = -1;
2653
2654 return 0;
2655 }
2656
2657 static int emit_elf_data_sec(struct bpf_linker *linker, const char *sec_name,
2658 size_t align, const void *raw_data, size_t raw_sz)
2659 {
2660 Elf_Scn *scn;
2661 Elf_Data *data;
2662 Elf64_Shdr *shdr;
2663 int name_off;
2664
2665 name_off = strset__add_str(linker->strtab_strs, sec_name);
2666 if (name_off < 0)
2667 return name_off;
2668
2669 scn = elf_newscn(linker->elf);
2670 if (!scn)
2671 return -ENOMEM;
2672 data = elf_newdata(scn);
2673 if (!data)
2674 return -ENOMEM;
2675 shdr = elf64_getshdr(scn);
2676 if (!shdr)
2677 return -EINVAL;
2678
2679 shdr->sh_name = name_off;
2680 shdr->sh_type = SHT_PROGBITS;
2681 shdr->sh_flags = 0;
2682 shdr->sh_size = raw_sz;
2683 shdr->sh_link = 0;
2684 shdr->sh_info = 0;
2685 shdr->sh_addralign = align;
2686 shdr->sh_entsize = 0;
2687
2688 data->d_type = ELF_T_BYTE;
2689 data->d_size = raw_sz;
2690 data->d_buf = (void *)raw_data;
2691 data->d_align = align;
2692 data->d_off = 0;
2693
2694 return 0;
2695 }
2696
2697 static int finalize_btf(struct bpf_linker *linker)
2698 {
2699 LIBBPF_OPTS(btf_dedup_opts, opts);
2700 struct btf *btf = linker->btf;
2701 const void *raw_data;
2702 int i, j, id, err;
2703 __u32 raw_sz;
2704
2705 /* bail out if no BTF data was produced */
2706 if (btf__type_cnt(linker->btf) == 1)
2707 return 0;
2708
2709 for (i = 1; i < linker->sec_cnt; i++) {
2710 struct dst_sec *sec = &linker->secs[i];
2711
2712 if (!sec->has_btf)
2713 continue;
2714
2715 id = btf__add_datasec(btf, sec->sec_name, sec->sec_sz);
2716 if (id < 0) {
2717 pr_warn("failed to add consolidated BTF type for datasec '%s': %d\n",
2718 sec->sec_name, id);
2719 return id;
2720 }
2721
2722 for (j = 0; j < sec->sec_var_cnt; j++) {
2723 struct btf_var_secinfo *vi = &sec->sec_vars[j];
2724
2725 if (btf__add_datasec_var_info(btf, vi->type, vi->offset, vi->size))
2726 return -EINVAL;
2727 }
2728 }
2729
2730 err = finalize_btf_ext(linker);
2731 if (err) {
2732 pr_warn(".BTF.ext generation failed: %d\n", err);
2733 return err;
2734 }
2735
2736 opts.btf_ext = linker->btf_ext;
2737 err = btf__dedup(linker->btf, &opts);
2738 if (err) {
2739 pr_warn("BTF dedup failed: %d\n", err);
2740 return err;
2741 }
2742
2743 /* Emit .BTF section */
2744 raw_data = btf__raw_data(linker->btf, &raw_sz);
2745 if (!raw_data)
2746 return -ENOMEM;
2747
2748 err = emit_elf_data_sec(linker, BTF_ELF_SEC, 8, raw_data, raw_sz);
2749 if (err) {
2750 pr_warn("failed to write out .BTF ELF section: %d\n", err);
2751 return err;
2752 }
2753
2754 /* Emit .BTF.ext section */
2755 if (linker->btf_ext) {
2756 raw_data = btf_ext__raw_data(linker->btf_ext, &raw_sz);
2757 if (!raw_data)
2758 return -ENOMEM;
2759
2760 err = emit_elf_data_sec(linker, BTF_EXT_ELF_SEC, 8, raw_data, raw_sz);
2761 if (err) {
2762 pr_warn("failed to write out .BTF.ext ELF section: %d\n", err);
2763 return err;
2764 }
2765 }
2766
2767 return 0;
2768 }
2769
2770 static int emit_btf_ext_data(struct bpf_linker *linker, void *output,
2771 const char *sec_name, struct btf_ext_sec_data *sec_data)
2772 {
2773 struct btf_ext_info_sec *sec_info;
2774 void *cur = output;
2775 int str_off;
2776 size_t sz;
2777
2778 if (!sec_data->rec_cnt)
2779 return 0;
2780
2781 str_off = btf__add_str(linker->btf, sec_name);
2782 if (str_off < 0)
2783 return -ENOMEM;
2784
2785 sec_info = cur;
2786 sec_info->sec_name_off = str_off;
2787 sec_info->num_info = sec_data->rec_cnt;
2788 cur += sizeof(struct btf_ext_info_sec);
2789
2790 sz = sec_data->rec_cnt * sec_data->rec_sz;
2791 memcpy(cur, sec_data->recs, sz);
2792 cur += sz;
2793
2794 return cur - output;
2795 }
2796
2797 static int finalize_btf_ext(struct bpf_linker *linker)
2798 {
2799 size_t funcs_sz = 0, lines_sz = 0, core_relos_sz = 0, total_sz = 0;
2800 size_t func_rec_sz = 0, line_rec_sz = 0, core_relo_rec_sz = 0;
2801 struct btf_ext_header *hdr;
2802 void *data, *cur;
2803 int i, err, sz;
2804
2805 /* validate that all sections have the same .BTF.ext record sizes
2806 * and calculate total data size for each type of data (func info,
2807 * line info, core relos)
2808 */
2809 for (i = 1; i < linker->sec_cnt; i++) {
2810 struct dst_sec *sec = &linker->secs[i];
2811
2812 if (sec->func_info.rec_cnt) {
2813 if (func_rec_sz == 0)
2814 func_rec_sz = sec->func_info.rec_sz;
2815 if (func_rec_sz != sec->func_info.rec_sz) {
2816 pr_warn("mismatch in func_info record size %zu != %u\n",
2817 func_rec_sz, sec->func_info.rec_sz);
2818 return -EINVAL;
2819 }
2820
2821 funcs_sz += sizeof(struct btf_ext_info_sec) + func_rec_sz * sec->func_info.rec_cnt;
2822 }
2823 if (sec->line_info.rec_cnt) {
2824 if (line_rec_sz == 0)
2825 line_rec_sz = sec->line_info.rec_sz;
2826 if (line_rec_sz != sec->line_info.rec_sz) {
2827 pr_warn("mismatch in line_info record size %zu != %u\n",
2828 line_rec_sz, sec->line_info.rec_sz);
2829 return -EINVAL;
2830 }
2831
2832 lines_sz += sizeof(struct btf_ext_info_sec) + line_rec_sz * sec->line_info.rec_cnt;
2833 }
2834 if (sec->core_relo_info.rec_cnt) {
2835 if (core_relo_rec_sz == 0)
2836 core_relo_rec_sz = sec->core_relo_info.rec_sz;
2837 if (core_relo_rec_sz != sec->core_relo_info.rec_sz) {
2838 pr_warn("mismatch in core_relo_info record size %zu != %u\n",
2839 core_relo_rec_sz, sec->core_relo_info.rec_sz);
2840 return -EINVAL;
2841 }
2842
2843 core_relos_sz += sizeof(struct btf_ext_info_sec) + core_relo_rec_sz * sec->core_relo_info.rec_cnt;
2844 }
2845 }
2846
2847 if (!funcs_sz && !lines_sz && !core_relos_sz)
2848 return 0;
2849
2850 total_sz += sizeof(struct btf_ext_header);
2851 if (funcs_sz) {
2852 funcs_sz += sizeof(__u32); /* record size prefix */
2853 total_sz += funcs_sz;
2854 }
2855 if (lines_sz) {
2856 lines_sz += sizeof(__u32); /* record size prefix */
2857 total_sz += lines_sz;
2858 }
2859 if (core_relos_sz) {
2860 core_relos_sz += sizeof(__u32); /* record size prefix */
2861 total_sz += core_relos_sz;
2862 }
2863
2864 cur = data = calloc(1, total_sz);
2865 if (!data)
2866 return -ENOMEM;
2867
2868 hdr = cur;
2869 hdr->magic = BTF_MAGIC;
2870 hdr->version = BTF_VERSION;
2871 hdr->flags = 0;
2872 hdr->hdr_len = sizeof(struct btf_ext_header);
2873 cur += sizeof(struct btf_ext_header);
2874
2875 /* All offsets are in bytes relative to the end of this header */
2876 hdr->func_info_off = 0;
2877 hdr->func_info_len = funcs_sz;
2878 hdr->line_info_off = funcs_sz;
2879 hdr->line_info_len = lines_sz;
2880 hdr->core_relo_off = funcs_sz + lines_sz;
2881 hdr->core_relo_len = core_relos_sz;
2882
2883 if (funcs_sz) {
2884 *(__u32 *)cur = func_rec_sz;
2885 cur += sizeof(__u32);
2886
2887 for (i = 1; i < linker->sec_cnt; i++) {
2888 struct dst_sec *sec = &linker->secs[i];
2889
2890 sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->func_info);
2891 if (sz < 0) {
2892 err = sz;
2893 goto out;
2894 }
2895
2896 cur += sz;
2897 }
2898 }
2899
2900 if (lines_sz) {
2901 *(__u32 *)cur = line_rec_sz;
2902 cur += sizeof(__u32);
2903
2904 for (i = 1; i < linker->sec_cnt; i++) {
2905 struct dst_sec *sec = &linker->secs[i];
2906
2907 sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->line_info);
2908 if (sz < 0) {
2909 err = sz;
2910 goto out;
2911 }
2912
2913 cur += sz;
2914 }
2915 }
2916
2917 if (core_relos_sz) {
2918 *(__u32 *)cur = core_relo_rec_sz;
2919 cur += sizeof(__u32);
2920
2921 for (i = 1; i < linker->sec_cnt; i++) {
2922 struct dst_sec *sec = &linker->secs[i];
2923
2924 sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->core_relo_info);
2925 if (sz < 0) {
2926 err = sz;
2927 goto out;
2928 }
2929
2930 cur += sz;
2931 }
2932 }
2933
2934 linker->btf_ext = btf_ext__new(data, total_sz);
2935 err = libbpf_get_error(linker->btf_ext);
2936 if (err) {
2937 linker->btf_ext = NULL;
2938 pr_warn("failed to parse final .BTF.ext data: %d\n", err);
2939 goto out;
2940 }
2941
2942 out:
2943 free(data);
2944 return err;
2945 }
2946
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.