1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Copyright (C) 2015-2017 Josh Poimboeuf <jpoimboe@redhat.com>
4 */
5
6 #include <string.h>
7 #include <stdlib.h>
8 #include <inttypes.h>
9 #include <sys/mman.h>
10
11 #include <objtool/builtin.h>
12 #include <objtool/cfi.h>
13 #include <objtool/arch.h>
14 #include <objtool/check.h>
15 #include <objtool/special.h>
16 #include <objtool/warn.h>
17 #include <objtool/endianness.h>
18
19 #include <linux/objtool_types.h>
20 #include <linux/hashtable.h>
21 #include <linux/kernel.h>
22 #include <linux/static_call_types.h>
23 #include <linux/string.h>
24
25 struct alternative {
26 struct alternative *next;
27 struct instruction *insn;
28 bool skip_orig;
29 };
30
31 static unsigned long nr_cfi, nr_cfi_reused, nr_cfi_cache;
32
33 static struct cfi_init_state initial_func_cfi;
34 static struct cfi_state init_cfi;
35 static struct cfi_state func_cfi;
36 static struct cfi_state force_undefined_cfi;
37
38 struct instruction *find_insn(struct objtool_file *file,
39 struct section *sec, unsigned long offset)
40 {
41 struct instruction *insn;
42
43 hash_for_each_possible(file->insn_hash, insn, hash, sec_offset_hash(sec, offset)) {
44 if (insn->sec == sec && insn->offset == offset)
45 return insn;
46 }
47
48 return NULL;
49 }
50
51 struct instruction *next_insn_same_sec(struct objtool_file *file,
52 struct instruction *insn)
53 {
54 if (insn->idx == INSN_CHUNK_MAX)
55 return find_insn(file, insn->sec, insn->offset + insn->len);
56
57 insn++;
58 if (!insn->len)
59 return NULL;
60
61 return insn;
62 }
63
64 static struct instruction *next_insn_same_func(struct objtool_file *file,
65 struct instruction *insn)
66 {
67 struct instruction *next = next_insn_same_sec(file, insn);
68 struct symbol *func = insn_func(insn);
69
70 if (!func)
71 return NULL;
72
73 if (next && insn_func(next) == func)
74 return next;
75
76 /* Check if we're already in the subfunction: */
77 if (func == func->cfunc)
78 return NULL;
79
80 /* Move to the subfunction: */
81 return find_insn(file, func->cfunc->sec, func->cfunc->offset);
82 }
83
84 static struct instruction *prev_insn_same_sec(struct objtool_file *file,
85 struct instruction *insn)
86 {
87 if (insn->idx == 0) {
88 if (insn->prev_len)
89 return find_insn(file, insn->sec, insn->offset - insn->prev_len);
90 return NULL;
91 }
92
93 return insn - 1;
94 }
95
96 static struct instruction *prev_insn_same_sym(struct objtool_file *file,
97 struct instruction *insn)
98 {
99 struct instruction *prev = prev_insn_same_sec(file, insn);
100
101 if (prev && insn_func(prev) == insn_func(insn))
102 return prev;
103
104 return NULL;
105 }
106
107 #define for_each_insn(file, insn) \
108 for (struct section *__sec, *__fake = (struct section *)1; \
109 __fake; __fake = NULL) \
110 for_each_sec(file, __sec) \
111 sec_for_each_insn(file, __sec, insn)
112
113 #define func_for_each_insn(file, func, insn) \
114 for (insn = find_insn(file, func->sec, func->offset); \
115 insn; \
116 insn = next_insn_same_func(file, insn))
117
118 #define sym_for_each_insn(file, sym, insn) \
119 for (insn = find_insn(file, sym->sec, sym->offset); \
120 insn && insn->offset < sym->offset + sym->len; \
121 insn = next_insn_same_sec(file, insn))
122
123 #define sym_for_each_insn_continue_reverse(file, sym, insn) \
124 for (insn = prev_insn_same_sec(file, insn); \
125 insn && insn->offset >= sym->offset; \
126 insn = prev_insn_same_sec(file, insn))
127
128 #define sec_for_each_insn_from(file, insn) \
129 for (; insn; insn = next_insn_same_sec(file, insn))
130
131 #define sec_for_each_insn_continue(file, insn) \
132 for (insn = next_insn_same_sec(file, insn); insn; \
133 insn = next_insn_same_sec(file, insn))
134
135 static inline struct symbol *insn_call_dest(struct instruction *insn)
136 {
137 if (insn->type == INSN_JUMP_DYNAMIC ||
138 insn->type == INSN_CALL_DYNAMIC)
139 return NULL;
140
141 return insn->_call_dest;
142 }
143
144 static inline struct reloc *insn_jump_table(struct instruction *insn)
145 {
146 if (insn->type == INSN_JUMP_DYNAMIC ||
147 insn->type == INSN_CALL_DYNAMIC)
148 return insn->_jump_table;
149
150 return NULL;
151 }
152
153 static bool is_jump_table_jump(struct instruction *insn)
154 {
155 struct alt_group *alt_group = insn->alt_group;
156
157 if (insn_jump_table(insn))
158 return true;
159
160 /* Retpoline alternative for a jump table? */
161 return alt_group && alt_group->orig_group &&
162 insn_jump_table(alt_group->orig_group->first_insn);
163 }
164
165 static bool is_sibling_call(struct instruction *insn)
166 {
167 /*
168 * Assume only STT_FUNC calls have jump-tables.
169 */
170 if (insn_func(insn)) {
171 /* An indirect jump is either a sibling call or a jump to a table. */
172 if (insn->type == INSN_JUMP_DYNAMIC)
173 return !is_jump_table_jump(insn);
174 }
175
176 /* add_jump_destinations() sets insn_call_dest(insn) for sibling calls. */
177 return (is_static_jump(insn) && insn_call_dest(insn));
178 }
179
180 /*
181 * This checks to see if the given function is a "noreturn" function.
182 *
183 * For global functions which are outside the scope of this object file, we
184 * have to keep a manual list of them.
185 *
186 * For local functions, we have to detect them manually by simply looking for
187 * the lack of a return instruction.
188 */
189 static bool __dead_end_function(struct objtool_file *file, struct symbol *func,
190 int recursion)
191 {
192 int i;
193 struct instruction *insn;
194 bool empty = true;
195
196 #define NORETURN(func) __stringify(func),
197 static const char * const global_noreturns[] = {
198 #include "noreturns.h"
199 };
200 #undef NORETURN
201
202 if (!func)
203 return false;
204
205 if (func->bind == STB_GLOBAL || func->bind == STB_WEAK)
206 for (i = 0; i < ARRAY_SIZE(global_noreturns); i++)
207 if (!strcmp(func->name, global_noreturns[i]))
208 return true;
209
210 if (func->bind == STB_WEAK)
211 return false;
212
213 if (!func->len)
214 return false;
215
216 insn = find_insn(file, func->sec, func->offset);
217 if (!insn || !insn_func(insn))
218 return false;
219
220 func_for_each_insn(file, func, insn) {
221 empty = false;
222
223 if (insn->type == INSN_RETURN)
224 return false;
225 }
226
227 if (empty)
228 return false;
229
230 /*
231 * A function can have a sibling call instead of a return. In that
232 * case, the function's dead-end status depends on whether the target
233 * of the sibling call returns.
234 */
235 func_for_each_insn(file, func, insn) {
236 if (is_sibling_call(insn)) {
237 struct instruction *dest = insn->jump_dest;
238
239 if (!dest)
240 /* sibling call to another file */
241 return false;
242
243 /* local sibling call */
244 if (recursion == 5) {
245 /*
246 * Infinite recursion: two functions have
247 * sibling calls to each other. This is a very
248 * rare case. It means they aren't dead ends.
249 */
250 return false;
251 }
252
253 return __dead_end_function(file, insn_func(dest), recursion+1);
254 }
255 }
256
257 return true;
258 }
259
260 static bool dead_end_function(struct objtool_file *file, struct symbol *func)
261 {
262 return __dead_end_function(file, func, 0);
263 }
264
265 static void init_cfi_state(struct cfi_state *cfi)
266 {
267 int i;
268
269 for (i = 0; i < CFI_NUM_REGS; i++) {
270 cfi->regs[i].base = CFI_UNDEFINED;
271 cfi->vals[i].base = CFI_UNDEFINED;
272 }
273 cfi->cfa.base = CFI_UNDEFINED;
274 cfi->drap_reg = CFI_UNDEFINED;
275 cfi->drap_offset = -1;
276 }
277
278 static void init_insn_state(struct objtool_file *file, struct insn_state *state,
279 struct section *sec)
280 {
281 memset(state, 0, sizeof(*state));
282 init_cfi_state(&state->cfi);
283
284 /*
285 * We need the full vmlinux for noinstr validation, otherwise we can
286 * not correctly determine insn_call_dest(insn)->sec (external symbols
287 * do not have a section).
288 */
289 if (opts.link && opts.noinstr && sec)
290 state->noinstr = sec->noinstr;
291 }
292
293 static struct cfi_state *cfi_alloc(void)
294 {
295 struct cfi_state *cfi = calloc(1, sizeof(struct cfi_state));
296 if (!cfi) {
297 WARN("calloc failed");
298 exit(1);
299 }
300 nr_cfi++;
301 return cfi;
302 }
303
304 static int cfi_bits;
305 static struct hlist_head *cfi_hash;
306
307 static inline bool cficmp(struct cfi_state *cfi1, struct cfi_state *cfi2)
308 {
309 return memcmp((void *)cfi1 + sizeof(cfi1->hash),
310 (void *)cfi2 + sizeof(cfi2->hash),
311 sizeof(struct cfi_state) - sizeof(struct hlist_node));
312 }
313
314 static inline u32 cfi_key(struct cfi_state *cfi)
315 {
316 return jhash((void *)cfi + sizeof(cfi->hash),
317 sizeof(*cfi) - sizeof(cfi->hash), 0);
318 }
319
320 static struct cfi_state *cfi_hash_find_or_add(struct cfi_state *cfi)
321 {
322 struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
323 struct cfi_state *obj;
324
325 hlist_for_each_entry(obj, head, hash) {
326 if (!cficmp(cfi, obj)) {
327 nr_cfi_cache++;
328 return obj;
329 }
330 }
331
332 obj = cfi_alloc();
333 *obj = *cfi;
334 hlist_add_head(&obj->hash, head);
335
336 return obj;
337 }
338
339 static void cfi_hash_add(struct cfi_state *cfi)
340 {
341 struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
342
343 hlist_add_head(&cfi->hash, head);
344 }
345
346 static void *cfi_hash_alloc(unsigned long size)
347 {
348 cfi_bits = max(10, ilog2(size));
349 cfi_hash = mmap(NULL, sizeof(struct hlist_head) << cfi_bits,
350 PROT_READ|PROT_WRITE,
351 MAP_PRIVATE|MAP_ANON, -1, 0);
352 if (cfi_hash == (void *)-1L) {
353 WARN("mmap fail cfi_hash");
354 cfi_hash = NULL;
355 } else if (opts.stats) {
356 printf("cfi_bits: %d\n", cfi_bits);
357 }
358
359 return cfi_hash;
360 }
361
362 static unsigned long nr_insns;
363 static unsigned long nr_insns_visited;
364
365 /*
366 * Call the arch-specific instruction decoder for all the instructions and add
367 * them to the global instruction list.
368 */
369 static int decode_instructions(struct objtool_file *file)
370 {
371 struct section *sec;
372 struct symbol *func;
373 unsigned long offset;
374 struct instruction *insn;
375 int ret;
376
377 for_each_sec(file, sec) {
378 struct instruction *insns = NULL;
379 u8 prev_len = 0;
380 u8 idx = 0;
381
382 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
383 continue;
384
385 if (strcmp(sec->name, ".altinstr_replacement") &&
386 strcmp(sec->name, ".altinstr_aux") &&
387 strncmp(sec->name, ".discard.", 9))
388 sec->text = true;
389
390 if (!strcmp(sec->name, ".noinstr.text") ||
391 !strcmp(sec->name, ".entry.text") ||
392 !strcmp(sec->name, ".cpuidle.text") ||
393 !strncmp(sec->name, ".text..__x86.", 13))
394 sec->noinstr = true;
395
396 /*
397 * .init.text code is ran before userspace and thus doesn't
398 * strictly need retpolines, except for modules which are
399 * loaded late, they very much do need retpoline in their
400 * .init.text
401 */
402 if (!strcmp(sec->name, ".init.text") && !opts.module)
403 sec->init = true;
404
405 for (offset = 0; offset < sec->sh.sh_size; offset += insn->len) {
406 if (!insns || idx == INSN_CHUNK_MAX) {
407 insns = calloc(sizeof(*insn), INSN_CHUNK_SIZE);
408 if (!insns) {
409 WARN("malloc failed");
410 return -1;
411 }
412 idx = 0;
413 } else {
414 idx++;
415 }
416 insn = &insns[idx];
417 insn->idx = idx;
418
419 INIT_LIST_HEAD(&insn->call_node);
420 insn->sec = sec;
421 insn->offset = offset;
422 insn->prev_len = prev_len;
423
424 ret = arch_decode_instruction(file, sec, offset,
425 sec->sh.sh_size - offset,
426 insn);
427 if (ret)
428 return ret;
429
430 prev_len = insn->len;
431
432 /*
433 * By default, "ud2" is a dead end unless otherwise
434 * annotated, because GCC 7 inserts it for certain
435 * divide-by-zero cases.
436 */
437 if (insn->type == INSN_BUG)
438 insn->dead_end = true;
439
440 hash_add(file->insn_hash, &insn->hash, sec_offset_hash(sec, insn->offset));
441 nr_insns++;
442 }
443
444 // printf("%s: last chunk used: %d\n", sec->name, (int)idx);
445
446 sec_for_each_sym(sec, func) {
447 if (func->type != STT_NOTYPE && func->type != STT_FUNC)
448 continue;
449
450 if (func->offset == sec->sh.sh_size) {
451 /* Heuristic: likely an "end" symbol */
452 if (func->type == STT_NOTYPE)
453 continue;
454 WARN("%s(): STT_FUNC at end of section",
455 func->name);
456 return -1;
457 }
458
459 if (func->embedded_insn || func->alias != func)
460 continue;
461
462 if (!find_insn(file, sec, func->offset)) {
463 WARN("%s(): can't find starting instruction",
464 func->name);
465 return -1;
466 }
467
468 sym_for_each_insn(file, func, insn) {
469 insn->sym = func;
470 if (func->type == STT_FUNC &&
471 insn->type == INSN_ENDBR &&
472 list_empty(&insn->call_node)) {
473 if (insn->offset == func->offset) {
474 list_add_tail(&insn->call_node, &file->endbr_list);
475 file->nr_endbr++;
476 } else {
477 file->nr_endbr_int++;
478 }
479 }
480 }
481 }
482 }
483
484 if (opts.stats)
485 printf("nr_insns: %lu\n", nr_insns);
486
487 return 0;
488 }
489
490 /*
491 * Read the pv_ops[] .data table to find the static initialized values.
492 */
493 static int add_pv_ops(struct objtool_file *file, const char *symname)
494 {
495 struct symbol *sym, *func;
496 unsigned long off, end;
497 struct reloc *reloc;
498 int idx;
499
500 sym = find_symbol_by_name(file->elf, symname);
501 if (!sym)
502 return 0;
503
504 off = sym->offset;
505 end = off + sym->len;
506 for (;;) {
507 reloc = find_reloc_by_dest_range(file->elf, sym->sec, off, end - off);
508 if (!reloc)
509 break;
510
511 func = reloc->sym;
512 if (func->type == STT_SECTION)
513 func = find_symbol_by_offset(reloc->sym->sec,
514 reloc_addend(reloc));
515
516 idx = (reloc_offset(reloc) - sym->offset) / sizeof(unsigned long);
517
518 objtool_pv_add(file, idx, func);
519
520 off = reloc_offset(reloc) + 1;
521 if (off > end)
522 break;
523 }
524
525 return 0;
526 }
527
528 /*
529 * Allocate and initialize file->pv_ops[].
530 */
531 static int init_pv_ops(struct objtool_file *file)
532 {
533 static const char *pv_ops_tables[] = {
534 "pv_ops",
535 "xen_cpu_ops",
536 "xen_irq_ops",
537 "xen_mmu_ops",
538 NULL,
539 };
540 const char *pv_ops;
541 struct symbol *sym;
542 int idx, nr;
543
544 if (!opts.noinstr)
545 return 0;
546
547 file->pv_ops = NULL;
548
549 sym = find_symbol_by_name(file->elf, "pv_ops");
550 if (!sym)
551 return 0;
552
553 nr = sym->len / sizeof(unsigned long);
554 file->pv_ops = calloc(sizeof(struct pv_state), nr);
555 if (!file->pv_ops)
556 return -1;
557
558 for (idx = 0; idx < nr; idx++)
559 INIT_LIST_HEAD(&file->pv_ops[idx].targets);
560
561 for (idx = 0; (pv_ops = pv_ops_tables[idx]); idx++)
562 add_pv_ops(file, pv_ops);
563
564 return 0;
565 }
566
567 static struct instruction *find_last_insn(struct objtool_file *file,
568 struct section *sec)
569 {
570 struct instruction *insn = NULL;
571 unsigned int offset;
572 unsigned int end = (sec->sh.sh_size > 10) ? sec->sh.sh_size - 10 : 0;
573
574 for (offset = sec->sh.sh_size - 1; offset >= end && !insn; offset--)
575 insn = find_insn(file, sec, offset);
576
577 return insn;
578 }
579
580 /*
581 * Mark "ud2" instructions and manually annotated dead ends.
582 */
583 static int add_dead_ends(struct objtool_file *file)
584 {
585 struct section *rsec;
586 struct reloc *reloc;
587 struct instruction *insn;
588 uint64_t offset;
589
590 /*
591 * Check for manually annotated dead ends.
592 */
593 rsec = find_section_by_name(file->elf, ".rela.discard.unreachable");
594 if (!rsec)
595 goto reachable;
596
597 for_each_reloc(rsec, reloc) {
598 if (reloc->sym->type == STT_SECTION) {
599 offset = reloc_addend(reloc);
600 } else if (reloc->sym->local_label) {
601 offset = reloc->sym->offset;
602 } else {
603 WARN("unexpected relocation symbol type in %s", rsec->name);
604 return -1;
605 }
606
607 insn = find_insn(file, reloc->sym->sec, offset);
608 if (insn)
609 insn = prev_insn_same_sec(file, insn);
610 else if (offset == reloc->sym->sec->sh.sh_size) {
611 insn = find_last_insn(file, reloc->sym->sec);
612 if (!insn) {
613 WARN("can't find unreachable insn at %s+0x%" PRIx64,
614 reloc->sym->sec->name, offset);
615 return -1;
616 }
617 } else {
618 WARN("can't find unreachable insn at %s+0x%" PRIx64,
619 reloc->sym->sec->name, offset);
620 return -1;
621 }
622
623 insn->dead_end = true;
624 }
625
626 reachable:
627 /*
628 * These manually annotated reachable checks are needed for GCC 4.4,
629 * where the Linux unreachable() macro isn't supported. In that case
630 * GCC doesn't know the "ud2" is fatal, so it generates code as if it's
631 * not a dead end.
632 */
633 rsec = find_section_by_name(file->elf, ".rela.discard.reachable");
634 if (!rsec)
635 return 0;
636
637 for_each_reloc(rsec, reloc) {
638 if (reloc->sym->type == STT_SECTION) {
639 offset = reloc_addend(reloc);
640 } else if (reloc->sym->local_label) {
641 offset = reloc->sym->offset;
642 } else {
643 WARN("unexpected relocation symbol type in %s", rsec->name);
644 return -1;
645 }
646
647 insn = find_insn(file, reloc->sym->sec, offset);
648 if (insn)
649 insn = prev_insn_same_sec(file, insn);
650 else if (offset == reloc->sym->sec->sh.sh_size) {
651 insn = find_last_insn(file, reloc->sym->sec);
652 if (!insn) {
653 WARN("can't find reachable insn at %s+0x%" PRIx64,
654 reloc->sym->sec->name, offset);
655 return -1;
656 }
657 } else {
658 WARN("can't find reachable insn at %s+0x%" PRIx64,
659 reloc->sym->sec->name, offset);
660 return -1;
661 }
662
663 insn->dead_end = false;
664 }
665
666 return 0;
667 }
668
669 static int create_static_call_sections(struct objtool_file *file)
670 {
671 struct static_call_site *site;
672 struct section *sec;
673 struct instruction *insn;
674 struct symbol *key_sym;
675 char *key_name, *tmp;
676 int idx;
677
678 sec = find_section_by_name(file->elf, ".static_call_sites");
679 if (sec) {
680 INIT_LIST_HEAD(&file->static_call_list);
681 WARN("file already has .static_call_sites section, skipping");
682 return 0;
683 }
684
685 if (list_empty(&file->static_call_list))
686 return 0;
687
688 idx = 0;
689 list_for_each_entry(insn, &file->static_call_list, call_node)
690 idx++;
691
692 sec = elf_create_section_pair(file->elf, ".static_call_sites",
693 sizeof(*site), idx, idx * 2);
694 if (!sec)
695 return -1;
696
697 /* Allow modules to modify the low bits of static_call_site::key */
698 sec->sh.sh_flags |= SHF_WRITE;
699
700 idx = 0;
701 list_for_each_entry(insn, &file->static_call_list, call_node) {
702
703 /* populate reloc for 'addr' */
704 if (!elf_init_reloc_text_sym(file->elf, sec,
705 idx * sizeof(*site), idx * 2,
706 insn->sec, insn->offset))
707 return -1;
708
709 /* find key symbol */
710 key_name = strdup(insn_call_dest(insn)->name);
711 if (!key_name) {
712 perror("strdup");
713 return -1;
714 }
715 if (strncmp(key_name, STATIC_CALL_TRAMP_PREFIX_STR,
716 STATIC_CALL_TRAMP_PREFIX_LEN)) {
717 WARN("static_call: trampoline name malformed: %s", key_name);
718 free(key_name);
719 return -1;
720 }
721 tmp = key_name + STATIC_CALL_TRAMP_PREFIX_LEN - STATIC_CALL_KEY_PREFIX_LEN;
722 memcpy(tmp, STATIC_CALL_KEY_PREFIX_STR, STATIC_CALL_KEY_PREFIX_LEN);
723
724 key_sym = find_symbol_by_name(file->elf, tmp);
725 if (!key_sym) {
726 if (!opts.module) {
727 WARN("static_call: can't find static_call_key symbol: %s", tmp);
728 free(key_name);
729 return -1;
730 }
731
732 /*
733 * For modules(), the key might not be exported, which
734 * means the module can make static calls but isn't
735 * allowed to change them.
736 *
737 * In that case we temporarily set the key to be the
738 * trampoline address. This is fixed up in
739 * static_call_add_module().
740 */
741 key_sym = insn_call_dest(insn);
742 }
743 free(key_name);
744
745 /* populate reloc for 'key' */
746 if (!elf_init_reloc_data_sym(file->elf, sec,
747 idx * sizeof(*site) + 4,
748 (idx * 2) + 1, key_sym,
749 is_sibling_call(insn) * STATIC_CALL_SITE_TAIL))
750 return -1;
751
752 idx++;
753 }
754
755 return 0;
756 }
757
758 static int create_retpoline_sites_sections(struct objtool_file *file)
759 {
760 struct instruction *insn;
761 struct section *sec;
762 int idx;
763
764 sec = find_section_by_name(file->elf, ".retpoline_sites");
765 if (sec) {
766 WARN("file already has .retpoline_sites, skipping");
767 return 0;
768 }
769
770 idx = 0;
771 list_for_each_entry(insn, &file->retpoline_call_list, call_node)
772 idx++;
773
774 if (!idx)
775 return 0;
776
777 sec = elf_create_section_pair(file->elf, ".retpoline_sites",
778 sizeof(int), idx, idx);
779 if (!sec)
780 return -1;
781
782 idx = 0;
783 list_for_each_entry(insn, &file->retpoline_call_list, call_node) {
784
785 if (!elf_init_reloc_text_sym(file->elf, sec,
786 idx * sizeof(int), idx,
787 insn->sec, insn->offset))
788 return -1;
789
790 idx++;
791 }
792
793 return 0;
794 }
795
796 static int create_return_sites_sections(struct objtool_file *file)
797 {
798 struct instruction *insn;
799 struct section *sec;
800 int idx;
801
802 sec = find_section_by_name(file->elf, ".return_sites");
803 if (sec) {
804 WARN("file already has .return_sites, skipping");
805 return 0;
806 }
807
808 idx = 0;
809 list_for_each_entry(insn, &file->return_thunk_list, call_node)
810 idx++;
811
812 if (!idx)
813 return 0;
814
815 sec = elf_create_section_pair(file->elf, ".return_sites",
816 sizeof(int), idx, idx);
817 if (!sec)
818 return -1;
819
820 idx = 0;
821 list_for_each_entry(insn, &file->return_thunk_list, call_node) {
822
823 if (!elf_init_reloc_text_sym(file->elf, sec,
824 idx * sizeof(int), idx,
825 insn->sec, insn->offset))
826 return -1;
827
828 idx++;
829 }
830
831 return 0;
832 }
833
834 static int create_ibt_endbr_seal_sections(struct objtool_file *file)
835 {
836 struct instruction *insn;
837 struct section *sec;
838 int idx;
839
840 sec = find_section_by_name(file->elf, ".ibt_endbr_seal");
841 if (sec) {
842 WARN("file already has .ibt_endbr_seal, skipping");
843 return 0;
844 }
845
846 idx = 0;
847 list_for_each_entry(insn, &file->endbr_list, call_node)
848 idx++;
849
850 if (opts.stats) {
851 printf("ibt: ENDBR at function start: %d\n", file->nr_endbr);
852 printf("ibt: ENDBR inside functions: %d\n", file->nr_endbr_int);
853 printf("ibt: superfluous ENDBR: %d\n", idx);
854 }
855
856 if (!idx)
857 return 0;
858
859 sec = elf_create_section_pair(file->elf, ".ibt_endbr_seal",
860 sizeof(int), idx, idx);
861 if (!sec)
862 return -1;
863
864 idx = 0;
865 list_for_each_entry(insn, &file->endbr_list, call_node) {
866
867 int *site = (int *)sec->data->d_buf + idx;
868 struct symbol *sym = insn->sym;
869 *site = 0;
870
871 if (opts.module && sym && sym->type == STT_FUNC &&
872 insn->offset == sym->offset &&
873 (!strcmp(sym->name, "init_module") ||
874 !strcmp(sym->name, "cleanup_module")))
875 WARN("%s(): not an indirect call target", sym->name);
876
877 if (!elf_init_reloc_text_sym(file->elf, sec,
878 idx * sizeof(int), idx,
879 insn->sec, insn->offset))
880 return -1;
881
882 idx++;
883 }
884
885 return 0;
886 }
887
888 static int create_cfi_sections(struct objtool_file *file)
889 {
890 struct section *sec;
891 struct symbol *sym;
892 int idx;
893
894 sec = find_section_by_name(file->elf, ".cfi_sites");
895 if (sec) {
896 INIT_LIST_HEAD(&file->call_list);
897 WARN("file already has .cfi_sites section, skipping");
898 return 0;
899 }
900
901 idx = 0;
902 for_each_sym(file, sym) {
903 if (sym->type != STT_FUNC)
904 continue;
905
906 if (strncmp(sym->name, "__cfi_", 6))
907 continue;
908
909 idx++;
910 }
911
912 sec = elf_create_section_pair(file->elf, ".cfi_sites",
913 sizeof(unsigned int), idx, idx);
914 if (!sec)
915 return -1;
916
917 idx = 0;
918 for_each_sym(file, sym) {
919 if (sym->type != STT_FUNC)
920 continue;
921
922 if (strncmp(sym->name, "__cfi_", 6))
923 continue;
924
925 if (!elf_init_reloc_text_sym(file->elf, sec,
926 idx * sizeof(unsigned int), idx,
927 sym->sec, sym->offset))
928 return -1;
929
930 idx++;
931 }
932
933 return 0;
934 }
935
936 static int create_mcount_loc_sections(struct objtool_file *file)
937 {
938 size_t addr_size = elf_addr_size(file->elf);
939 struct instruction *insn;
940 struct section *sec;
941 int idx;
942
943 sec = find_section_by_name(file->elf, "__mcount_loc");
944 if (sec) {
945 INIT_LIST_HEAD(&file->mcount_loc_list);
946 WARN("file already has __mcount_loc section, skipping");
947 return 0;
948 }
949
950 if (list_empty(&file->mcount_loc_list))
951 return 0;
952
953 idx = 0;
954 list_for_each_entry(insn, &file->mcount_loc_list, call_node)
955 idx++;
956
957 sec = elf_create_section_pair(file->elf, "__mcount_loc", addr_size,
958 idx, idx);
959 if (!sec)
960 return -1;
961
962 sec->sh.sh_addralign = addr_size;
963
964 idx = 0;
965 list_for_each_entry(insn, &file->mcount_loc_list, call_node) {
966
967 struct reloc *reloc;
968
969 reloc = elf_init_reloc_text_sym(file->elf, sec, idx * addr_size, idx,
970 insn->sec, insn->offset);
971 if (!reloc)
972 return -1;
973
974 set_reloc_type(file->elf, reloc, addr_size == 8 ? R_ABS64 : R_ABS32);
975
976 idx++;
977 }
978
979 return 0;
980 }
981
982 static int create_direct_call_sections(struct objtool_file *file)
983 {
984 struct instruction *insn;
985 struct section *sec;
986 int idx;
987
988 sec = find_section_by_name(file->elf, ".call_sites");
989 if (sec) {
990 INIT_LIST_HEAD(&file->call_list);
991 WARN("file already has .call_sites section, skipping");
992 return 0;
993 }
994
995 if (list_empty(&file->call_list))
996 return 0;
997
998 idx = 0;
999 list_for_each_entry(insn, &file->call_list, call_node)
1000 idx++;
1001
1002 sec = elf_create_section_pair(file->elf, ".call_sites",
1003 sizeof(unsigned int), idx, idx);
1004 if (!sec)
1005 return -1;
1006
1007 idx = 0;
1008 list_for_each_entry(insn, &file->call_list, call_node) {
1009
1010 if (!elf_init_reloc_text_sym(file->elf, sec,
1011 idx * sizeof(unsigned int), idx,
1012 insn->sec, insn->offset))
1013 return -1;
1014
1015 idx++;
1016 }
1017
1018 return 0;
1019 }
1020
1021 /*
1022 * Warnings shouldn't be reported for ignored functions.
1023 */
1024 static void add_ignores(struct objtool_file *file)
1025 {
1026 struct instruction *insn;
1027 struct section *rsec;
1028 struct symbol *func;
1029 struct reloc *reloc;
1030
1031 rsec = find_section_by_name(file->elf, ".rela.discard.func_stack_frame_non_standard");
1032 if (!rsec)
1033 return;
1034
1035 for_each_reloc(rsec, reloc) {
1036 switch (reloc->sym->type) {
1037 case STT_FUNC:
1038 func = reloc->sym;
1039 break;
1040
1041 case STT_SECTION:
1042 func = find_func_by_offset(reloc->sym->sec, reloc_addend(reloc));
1043 if (!func)
1044 continue;
1045 break;
1046
1047 default:
1048 WARN("unexpected relocation symbol type in %s: %d",
1049 rsec->name, reloc->sym->type);
1050 continue;
1051 }
1052
1053 func_for_each_insn(file, func, insn)
1054 insn->ignore = true;
1055 }
1056 }
1057
1058 /*
1059 * This is a whitelist of functions that is allowed to be called with AC set.
1060 * The list is meant to be minimal and only contains compiler instrumentation
1061 * ABI and a few functions used to implement *_{to,from}_user() functions.
1062 *
1063 * These functions must not directly change AC, but may PUSHF/POPF.
1064 */
1065 static const char *uaccess_safe_builtin[] = {
1066 /* KASAN */
1067 "kasan_report",
1068 "kasan_check_range",
1069 /* KASAN out-of-line */
1070 "__asan_loadN_noabort",
1071 "__asan_load1_noabort",
1072 "__asan_load2_noabort",
1073 "__asan_load4_noabort",
1074 "__asan_load8_noabort",
1075 "__asan_load16_noabort",
1076 "__asan_storeN_noabort",
1077 "__asan_store1_noabort",
1078 "__asan_store2_noabort",
1079 "__asan_store4_noabort",
1080 "__asan_store8_noabort",
1081 "__asan_store16_noabort",
1082 "__kasan_check_read",
1083 "__kasan_check_write",
1084 /* KASAN in-line */
1085 "__asan_report_load_n_noabort",
1086 "__asan_report_load1_noabort",
1087 "__asan_report_load2_noabort",
1088 "__asan_report_load4_noabort",
1089 "__asan_report_load8_noabort",
1090 "__asan_report_load16_noabort",
1091 "__asan_report_store_n_noabort",
1092 "__asan_report_store1_noabort",
1093 "__asan_report_store2_noabort",
1094 "__asan_report_store4_noabort",
1095 "__asan_report_store8_noabort",
1096 "__asan_report_store16_noabort",
1097 /* KCSAN */
1098 "__kcsan_check_access",
1099 "__kcsan_mb",
1100 "__kcsan_wmb",
1101 "__kcsan_rmb",
1102 "__kcsan_release",
1103 "kcsan_found_watchpoint",
1104 "kcsan_setup_watchpoint",
1105 "kcsan_check_scoped_accesses",
1106 "kcsan_disable_current",
1107 "kcsan_enable_current_nowarn",
1108 /* KCSAN/TSAN */
1109 "__tsan_func_entry",
1110 "__tsan_func_exit",
1111 "__tsan_read_range",
1112 "__tsan_write_range",
1113 "__tsan_read1",
1114 "__tsan_read2",
1115 "__tsan_read4",
1116 "__tsan_read8",
1117 "__tsan_read16",
1118 "__tsan_write1",
1119 "__tsan_write2",
1120 "__tsan_write4",
1121 "__tsan_write8",
1122 "__tsan_write16",
1123 "__tsan_read_write1",
1124 "__tsan_read_write2",
1125 "__tsan_read_write4",
1126 "__tsan_read_write8",
1127 "__tsan_read_write16",
1128 "__tsan_volatile_read1",
1129 "__tsan_volatile_read2",
1130 "__tsan_volatile_read4",
1131 "__tsan_volatile_read8",
1132 "__tsan_volatile_read16",
1133 "__tsan_volatile_write1",
1134 "__tsan_volatile_write2",
1135 "__tsan_volatile_write4",
1136 "__tsan_volatile_write8",
1137 "__tsan_volatile_write16",
1138 "__tsan_atomic8_load",
1139 "__tsan_atomic16_load",
1140 "__tsan_atomic32_load",
1141 "__tsan_atomic64_load",
1142 "__tsan_atomic8_store",
1143 "__tsan_atomic16_store",
1144 "__tsan_atomic32_store",
1145 "__tsan_atomic64_store",
1146 "__tsan_atomic8_exchange",
1147 "__tsan_atomic16_exchange",
1148 "__tsan_atomic32_exchange",
1149 "__tsan_atomic64_exchange",
1150 "__tsan_atomic8_fetch_add",
1151 "__tsan_atomic16_fetch_add",
1152 "__tsan_atomic32_fetch_add",
1153 "__tsan_atomic64_fetch_add",
1154 "__tsan_atomic8_fetch_sub",
1155 "__tsan_atomic16_fetch_sub",
1156 "__tsan_atomic32_fetch_sub",
1157 "__tsan_atomic64_fetch_sub",
1158 "__tsan_atomic8_fetch_and",
1159 "__tsan_atomic16_fetch_and",
1160 "__tsan_atomic32_fetch_and",
1161 "__tsan_atomic64_fetch_and",
1162 "__tsan_atomic8_fetch_or",
1163 "__tsan_atomic16_fetch_or",
1164 "__tsan_atomic32_fetch_or",
1165 "__tsan_atomic64_fetch_or",
1166 "__tsan_atomic8_fetch_xor",
1167 "__tsan_atomic16_fetch_xor",
1168 "__tsan_atomic32_fetch_xor",
1169 "__tsan_atomic64_fetch_xor",
1170 "__tsan_atomic8_fetch_nand",
1171 "__tsan_atomic16_fetch_nand",
1172 "__tsan_atomic32_fetch_nand",
1173 "__tsan_atomic64_fetch_nand",
1174 "__tsan_atomic8_compare_exchange_strong",
1175 "__tsan_atomic16_compare_exchange_strong",
1176 "__tsan_atomic32_compare_exchange_strong",
1177 "__tsan_atomic64_compare_exchange_strong",
1178 "__tsan_atomic8_compare_exchange_weak",
1179 "__tsan_atomic16_compare_exchange_weak",
1180 "__tsan_atomic32_compare_exchange_weak",
1181 "__tsan_atomic64_compare_exchange_weak",
1182 "__tsan_atomic8_compare_exchange_val",
1183 "__tsan_atomic16_compare_exchange_val",
1184 "__tsan_atomic32_compare_exchange_val",
1185 "__tsan_atomic64_compare_exchange_val",
1186 "__tsan_atomic_thread_fence",
1187 "__tsan_atomic_signal_fence",
1188 "__tsan_unaligned_read16",
1189 "__tsan_unaligned_write16",
1190 /* KCOV */
1191 "write_comp_data",
1192 "check_kcov_mode",
1193 "__sanitizer_cov_trace_pc",
1194 "__sanitizer_cov_trace_const_cmp1",
1195 "__sanitizer_cov_trace_const_cmp2",
1196 "__sanitizer_cov_trace_const_cmp4",
1197 "__sanitizer_cov_trace_const_cmp8",
1198 "__sanitizer_cov_trace_cmp1",
1199 "__sanitizer_cov_trace_cmp2",
1200 "__sanitizer_cov_trace_cmp4",
1201 "__sanitizer_cov_trace_cmp8",
1202 "__sanitizer_cov_trace_switch",
1203 /* KMSAN */
1204 "kmsan_copy_to_user",
1205 "kmsan_disable_current",
1206 "kmsan_enable_current",
1207 "kmsan_report",
1208 "kmsan_unpoison_entry_regs",
1209 "kmsan_unpoison_memory",
1210 "__msan_chain_origin",
1211 "__msan_get_context_state",
1212 "__msan_instrument_asm_store",
1213 "__msan_metadata_ptr_for_load_1",
1214 "__msan_metadata_ptr_for_load_2",
1215 "__msan_metadata_ptr_for_load_4",
1216 "__msan_metadata_ptr_for_load_8",
1217 "__msan_metadata_ptr_for_load_n",
1218 "__msan_metadata_ptr_for_store_1",
1219 "__msan_metadata_ptr_for_store_2",
1220 "__msan_metadata_ptr_for_store_4",
1221 "__msan_metadata_ptr_for_store_8",
1222 "__msan_metadata_ptr_for_store_n",
1223 "__msan_poison_alloca",
1224 "__msan_warning",
1225 /* UBSAN */
1226 "ubsan_type_mismatch_common",
1227 "__ubsan_handle_type_mismatch",
1228 "__ubsan_handle_type_mismatch_v1",
1229 "__ubsan_handle_shift_out_of_bounds",
1230 "__ubsan_handle_load_invalid_value",
1231 /* STACKLEAK */
1232 "stackleak_track_stack",
1233 /* misc */
1234 "csum_partial_copy_generic",
1235 "copy_mc_fragile",
1236 "copy_mc_fragile_handle_tail",
1237 "copy_mc_enhanced_fast_string",
1238 "ftrace_likely_update", /* CONFIG_TRACE_BRANCH_PROFILING */
1239 "rep_stos_alternative",
1240 "rep_movs_alternative",
1241 "__copy_user_nocache",
1242 NULL
1243 };
1244
1245 static void add_uaccess_safe(struct objtool_file *file)
1246 {
1247 struct symbol *func;
1248 const char **name;
1249
1250 if (!opts.uaccess)
1251 return;
1252
1253 for (name = uaccess_safe_builtin; *name; name++) {
1254 func = find_symbol_by_name(file->elf, *name);
1255 if (!func)
1256 continue;
1257
1258 func->uaccess_safe = true;
1259 }
1260 }
1261
1262 /*
1263 * FIXME: For now, just ignore any alternatives which add retpolines. This is
1264 * a temporary hack, as it doesn't allow ORC to unwind from inside a retpoline.
1265 * But it at least allows objtool to understand the control flow *around* the
1266 * retpoline.
1267 */
1268 static int add_ignore_alternatives(struct objtool_file *file)
1269 {
1270 struct section *rsec;
1271 struct reloc *reloc;
1272 struct instruction *insn;
1273
1274 rsec = find_section_by_name(file->elf, ".rela.discard.ignore_alts");
1275 if (!rsec)
1276 return 0;
1277
1278 for_each_reloc(rsec, reloc) {
1279 if (reloc->sym->type != STT_SECTION) {
1280 WARN("unexpected relocation symbol type in %s", rsec->name);
1281 return -1;
1282 }
1283
1284 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
1285 if (!insn) {
1286 WARN("bad .discard.ignore_alts entry");
1287 return -1;
1288 }
1289
1290 insn->ignore_alts = true;
1291 }
1292
1293 return 0;
1294 }
1295
1296 /*
1297 * Symbols that replace INSN_CALL_DYNAMIC, every (tail) call to such a symbol
1298 * will be added to the .retpoline_sites section.
1299 */
1300 __weak bool arch_is_retpoline(struct symbol *sym)
1301 {
1302 return false;
1303 }
1304
1305 /*
1306 * Symbols that replace INSN_RETURN, every (tail) call to such a symbol
1307 * will be added to the .return_sites section.
1308 */
1309 __weak bool arch_is_rethunk(struct symbol *sym)
1310 {
1311 return false;
1312 }
1313
1314 /*
1315 * Symbols that are embedded inside other instructions, because sometimes crazy
1316 * code exists. These are mostly ignored for validation purposes.
1317 */
1318 __weak bool arch_is_embedded_insn(struct symbol *sym)
1319 {
1320 return false;
1321 }
1322
1323 static struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn)
1324 {
1325 struct reloc *reloc;
1326
1327 if (insn->no_reloc)
1328 return NULL;
1329
1330 if (!file)
1331 return NULL;
1332
1333 reloc = find_reloc_by_dest_range(file->elf, insn->sec,
1334 insn->offset, insn->len);
1335 if (!reloc) {
1336 insn->no_reloc = 1;
1337 return NULL;
1338 }
1339
1340 return reloc;
1341 }
1342
1343 static void remove_insn_ops(struct instruction *insn)
1344 {
1345 struct stack_op *op, *next;
1346
1347 for (op = insn->stack_ops; op; op = next) {
1348 next = op->next;
1349 free(op);
1350 }
1351 insn->stack_ops = NULL;
1352 }
1353
1354 static void annotate_call_site(struct objtool_file *file,
1355 struct instruction *insn, bool sibling)
1356 {
1357 struct reloc *reloc = insn_reloc(file, insn);
1358 struct symbol *sym = insn_call_dest(insn);
1359
1360 if (!sym)
1361 sym = reloc->sym;
1362
1363 /*
1364 * Alternative replacement code is just template code which is
1365 * sometimes copied to the original instruction. For now, don't
1366 * annotate it. (In the future we might consider annotating the
1367 * original instruction if/when it ever makes sense to do so.)
1368 */
1369 if (!strcmp(insn->sec->name, ".altinstr_replacement"))
1370 return;
1371
1372 if (sym->static_call_tramp) {
1373 list_add_tail(&insn->call_node, &file->static_call_list);
1374 return;
1375 }
1376
1377 if (sym->retpoline_thunk) {
1378 list_add_tail(&insn->call_node, &file->retpoline_call_list);
1379 return;
1380 }
1381
1382 /*
1383 * Many compilers cannot disable KCOV or sanitizer calls with a function
1384 * attribute so they need a little help, NOP out any such calls from
1385 * noinstr text.
1386 */
1387 if (opts.hack_noinstr && insn->sec->noinstr && sym->profiling_func) {
1388 if (reloc)
1389 set_reloc_type(file->elf, reloc, R_NONE);
1390
1391 elf_write_insn(file->elf, insn->sec,
1392 insn->offset, insn->len,
1393 sibling ? arch_ret_insn(insn->len)
1394 : arch_nop_insn(insn->len));
1395
1396 insn->type = sibling ? INSN_RETURN : INSN_NOP;
1397
1398 if (sibling) {
1399 /*
1400 * We've replaced the tail-call JMP insn by two new
1401 * insn: RET; INT3, except we only have a single struct
1402 * insn here. Mark it retpoline_safe to avoid the SLS
1403 * warning, instead of adding another insn.
1404 */
1405 insn->retpoline_safe = true;
1406 }
1407
1408 return;
1409 }
1410
1411 if (opts.mcount && sym->fentry) {
1412 if (sibling)
1413 WARN_INSN(insn, "tail call to __fentry__ !?!?");
1414 if (opts.mnop) {
1415 if (reloc)
1416 set_reloc_type(file->elf, reloc, R_NONE);
1417
1418 elf_write_insn(file->elf, insn->sec,
1419 insn->offset, insn->len,
1420 arch_nop_insn(insn->len));
1421
1422 insn->type = INSN_NOP;
1423 }
1424
1425 list_add_tail(&insn->call_node, &file->mcount_loc_list);
1426 return;
1427 }
1428
1429 if (insn->type == INSN_CALL && !insn->sec->init)
1430 list_add_tail(&insn->call_node, &file->call_list);
1431
1432 if (!sibling && dead_end_function(file, sym))
1433 insn->dead_end = true;
1434 }
1435
1436 static void add_call_dest(struct objtool_file *file, struct instruction *insn,
1437 struct symbol *dest, bool sibling)
1438 {
1439 insn->_call_dest = dest;
1440 if (!dest)
1441 return;
1442
1443 /*
1444 * Whatever stack impact regular CALLs have, should be undone
1445 * by the RETURN of the called function.
1446 *
1447 * Annotated intra-function calls retain the stack_ops but
1448 * are converted to JUMP, see read_intra_function_calls().
1449 */
1450 remove_insn_ops(insn);
1451
1452 annotate_call_site(file, insn, sibling);
1453 }
1454
1455 static void add_retpoline_call(struct objtool_file *file, struct instruction *insn)
1456 {
1457 /*
1458 * Retpoline calls/jumps are really dynamic calls/jumps in disguise,
1459 * so convert them accordingly.
1460 */
1461 switch (insn->type) {
1462 case INSN_CALL:
1463 insn->type = INSN_CALL_DYNAMIC;
1464 break;
1465 case INSN_JUMP_UNCONDITIONAL:
1466 insn->type = INSN_JUMP_DYNAMIC;
1467 break;
1468 case INSN_JUMP_CONDITIONAL:
1469 insn->type = INSN_JUMP_DYNAMIC_CONDITIONAL;
1470 break;
1471 default:
1472 return;
1473 }
1474
1475 insn->retpoline_safe = true;
1476
1477 /*
1478 * Whatever stack impact regular CALLs have, should be undone
1479 * by the RETURN of the called function.
1480 *
1481 * Annotated intra-function calls retain the stack_ops but
1482 * are converted to JUMP, see read_intra_function_calls().
1483 */
1484 remove_insn_ops(insn);
1485
1486 annotate_call_site(file, insn, false);
1487 }
1488
1489 static void add_return_call(struct objtool_file *file, struct instruction *insn, bool add)
1490 {
1491 /*
1492 * Return thunk tail calls are really just returns in disguise,
1493 * so convert them accordingly.
1494 */
1495 insn->type = INSN_RETURN;
1496 insn->retpoline_safe = true;
1497
1498 if (add)
1499 list_add_tail(&insn->call_node, &file->return_thunk_list);
1500 }
1501
1502 static bool is_first_func_insn(struct objtool_file *file,
1503 struct instruction *insn, struct symbol *sym)
1504 {
1505 if (insn->offset == sym->offset)
1506 return true;
1507
1508 /* Allow direct CALL/JMP past ENDBR */
1509 if (opts.ibt) {
1510 struct instruction *prev = prev_insn_same_sym(file, insn);
1511
1512 if (prev && prev->type == INSN_ENDBR &&
1513 insn->offset == sym->offset + prev->len)
1514 return true;
1515 }
1516
1517 return false;
1518 }
1519
1520 /*
1521 * A sibling call is a tail-call to another symbol -- to differentiate from a
1522 * recursive tail-call which is to the same symbol.
1523 */
1524 static bool jump_is_sibling_call(struct objtool_file *file,
1525 struct instruction *from, struct instruction *to)
1526 {
1527 struct symbol *fs = from->sym;
1528 struct symbol *ts = to->sym;
1529
1530 /* Not a sibling call if from/to a symbol hole */
1531 if (!fs || !ts)
1532 return false;
1533
1534 /* Not a sibling call if not targeting the start of a symbol. */
1535 if (!is_first_func_insn(file, to, ts))
1536 return false;
1537
1538 /* Disallow sibling calls into STT_NOTYPE */
1539 if (ts->type == STT_NOTYPE)
1540 return false;
1541
1542 /* Must not be self to be a sibling */
1543 return fs->pfunc != ts->pfunc;
1544 }
1545
1546 /*
1547 * Find the destination instructions for all jumps.
1548 */
1549 static int add_jump_destinations(struct objtool_file *file)
1550 {
1551 struct instruction *insn, *jump_dest;
1552 struct reloc *reloc;
1553 struct section *dest_sec;
1554 unsigned long dest_off;
1555
1556 for_each_insn(file, insn) {
1557 if (insn->jump_dest) {
1558 /*
1559 * handle_group_alt() may have previously set
1560 * 'jump_dest' for some alternatives.
1561 */
1562 continue;
1563 }
1564 if (!is_static_jump(insn))
1565 continue;
1566
1567 reloc = insn_reloc(file, insn);
1568 if (!reloc) {
1569 dest_sec = insn->sec;
1570 dest_off = arch_jump_destination(insn);
1571 } else if (reloc->sym->type == STT_SECTION) {
1572 dest_sec = reloc->sym->sec;
1573 dest_off = arch_dest_reloc_offset(reloc_addend(reloc));
1574 } else if (reloc->sym->retpoline_thunk) {
1575 add_retpoline_call(file, insn);
1576 continue;
1577 } else if (reloc->sym->return_thunk) {
1578 add_return_call(file, insn, true);
1579 continue;
1580 } else if (insn_func(insn)) {
1581 /*
1582 * External sibling call or internal sibling call with
1583 * STT_FUNC reloc.
1584 */
1585 add_call_dest(file, insn, reloc->sym, true);
1586 continue;
1587 } else if (reloc->sym->sec->idx) {
1588 dest_sec = reloc->sym->sec;
1589 dest_off = reloc->sym->sym.st_value +
1590 arch_dest_reloc_offset(reloc_addend(reloc));
1591 } else {
1592 /* non-func asm code jumping to another file */
1593 continue;
1594 }
1595
1596 jump_dest = find_insn(file, dest_sec, dest_off);
1597 if (!jump_dest) {
1598 struct symbol *sym = find_symbol_by_offset(dest_sec, dest_off);
1599
1600 /*
1601 * This is a special case for retbleed_untrain_ret().
1602 * It jumps to __x86_return_thunk(), but objtool
1603 * can't find the thunk's starting RET
1604 * instruction, because the RET is also in the
1605 * middle of another instruction. Objtool only
1606 * knows about the outer instruction.
1607 */
1608 if (sym && sym->embedded_insn) {
1609 add_return_call(file, insn, false);
1610 continue;
1611 }
1612
1613 WARN_INSN(insn, "can't find jump dest instruction at %s+0x%lx",
1614 dest_sec->name, dest_off);
1615 return -1;
1616 }
1617
1618 /*
1619 * An intra-TU jump in retpoline.o might not have a relocation
1620 * for its jump dest, in which case the above
1621 * add_{retpoline,return}_call() didn't happen.
1622 */
1623 if (jump_dest->sym && jump_dest->offset == jump_dest->sym->offset) {
1624 if (jump_dest->sym->retpoline_thunk) {
1625 add_retpoline_call(file, insn);
1626 continue;
1627 }
1628 if (jump_dest->sym->return_thunk) {
1629 add_return_call(file, insn, true);
1630 continue;
1631 }
1632 }
1633
1634 /*
1635 * Cross-function jump.
1636 */
1637 if (insn_func(insn) && insn_func(jump_dest) &&
1638 insn_func(insn) != insn_func(jump_dest)) {
1639
1640 /*
1641 * For GCC 8+, create parent/child links for any cold
1642 * subfunctions. This is _mostly_ redundant with a
1643 * similar initialization in read_symbols().
1644 *
1645 * If a function has aliases, we want the *first* such
1646 * function in the symbol table to be the subfunction's
1647 * parent. In that case we overwrite the
1648 * initialization done in read_symbols().
1649 *
1650 * However this code can't completely replace the
1651 * read_symbols() code because this doesn't detect the
1652 * case where the parent function's only reference to a
1653 * subfunction is through a jump table.
1654 */
1655 if (!strstr(insn_func(insn)->name, ".cold") &&
1656 strstr(insn_func(jump_dest)->name, ".cold")) {
1657 insn_func(insn)->cfunc = insn_func(jump_dest);
1658 insn_func(jump_dest)->pfunc = insn_func(insn);
1659 }
1660 }
1661
1662 if (jump_is_sibling_call(file, insn, jump_dest)) {
1663 /*
1664 * Internal sibling call without reloc or with
1665 * STT_SECTION reloc.
1666 */
1667 add_call_dest(file, insn, insn_func(jump_dest), true);
1668 continue;
1669 }
1670
1671 insn->jump_dest = jump_dest;
1672 }
1673
1674 return 0;
1675 }
1676
1677 static struct symbol *find_call_destination(struct section *sec, unsigned long offset)
1678 {
1679 struct symbol *call_dest;
1680
1681 call_dest = find_func_by_offset(sec, offset);
1682 if (!call_dest)
1683 call_dest = find_symbol_by_offset(sec, offset);
1684
1685 return call_dest;
1686 }
1687
1688 /*
1689 * Find the destination instructions for all calls.
1690 */
1691 static int add_call_destinations(struct objtool_file *file)
1692 {
1693 struct instruction *insn;
1694 unsigned long dest_off;
1695 struct symbol *dest;
1696 struct reloc *reloc;
1697
1698 for_each_insn(file, insn) {
1699 if (insn->type != INSN_CALL)
1700 continue;
1701
1702 reloc = insn_reloc(file, insn);
1703 if (!reloc) {
1704 dest_off = arch_jump_destination(insn);
1705 dest = find_call_destination(insn->sec, dest_off);
1706
1707 add_call_dest(file, insn, dest, false);
1708
1709 if (insn->ignore)
1710 continue;
1711
1712 if (!insn_call_dest(insn)) {
1713 WARN_INSN(insn, "unannotated intra-function call");
1714 return -1;
1715 }
1716
1717 if (insn_func(insn) && insn_call_dest(insn)->type != STT_FUNC) {
1718 WARN_INSN(insn, "unsupported call to non-function");
1719 return -1;
1720 }
1721
1722 } else if (reloc->sym->type == STT_SECTION) {
1723 dest_off = arch_dest_reloc_offset(reloc_addend(reloc));
1724 dest = find_call_destination(reloc->sym->sec, dest_off);
1725 if (!dest) {
1726 WARN_INSN(insn, "can't find call dest symbol at %s+0x%lx",
1727 reloc->sym->sec->name, dest_off);
1728 return -1;
1729 }
1730
1731 add_call_dest(file, insn, dest, false);
1732
1733 } else if (reloc->sym->retpoline_thunk) {
1734 add_retpoline_call(file, insn);
1735
1736 } else
1737 add_call_dest(file, insn, reloc->sym, false);
1738 }
1739
1740 return 0;
1741 }
1742
1743 /*
1744 * The .alternatives section requires some extra special care over and above
1745 * other special sections because alternatives are patched in place.
1746 */
1747 static int handle_group_alt(struct objtool_file *file,
1748 struct special_alt *special_alt,
1749 struct instruction *orig_insn,
1750 struct instruction **new_insn)
1751 {
1752 struct instruction *last_new_insn = NULL, *insn, *nop = NULL;
1753 struct alt_group *orig_alt_group, *new_alt_group;
1754 unsigned long dest_off;
1755
1756 orig_alt_group = orig_insn->alt_group;
1757 if (!orig_alt_group) {
1758 struct instruction *last_orig_insn = NULL;
1759
1760 orig_alt_group = malloc(sizeof(*orig_alt_group));
1761 if (!orig_alt_group) {
1762 WARN("malloc failed");
1763 return -1;
1764 }
1765 orig_alt_group->cfi = calloc(special_alt->orig_len,
1766 sizeof(struct cfi_state *));
1767 if (!orig_alt_group->cfi) {
1768 WARN("calloc failed");
1769 return -1;
1770 }
1771
1772 insn = orig_insn;
1773 sec_for_each_insn_from(file, insn) {
1774 if (insn->offset >= special_alt->orig_off + special_alt->orig_len)
1775 break;
1776
1777 insn->alt_group = orig_alt_group;
1778 last_orig_insn = insn;
1779 }
1780 orig_alt_group->orig_group = NULL;
1781 orig_alt_group->first_insn = orig_insn;
1782 orig_alt_group->last_insn = last_orig_insn;
1783 orig_alt_group->nop = NULL;
1784 } else {
1785 if (orig_alt_group->last_insn->offset + orig_alt_group->last_insn->len -
1786 orig_alt_group->first_insn->offset != special_alt->orig_len) {
1787 WARN_INSN(orig_insn, "weirdly overlapping alternative! %ld != %d",
1788 orig_alt_group->last_insn->offset +
1789 orig_alt_group->last_insn->len -
1790 orig_alt_group->first_insn->offset,
1791 special_alt->orig_len);
1792 return -1;
1793 }
1794 }
1795
1796 new_alt_group = malloc(sizeof(*new_alt_group));
1797 if (!new_alt_group) {
1798 WARN("malloc failed");
1799 return -1;
1800 }
1801
1802 if (special_alt->new_len < special_alt->orig_len) {
1803 /*
1804 * Insert a fake nop at the end to make the replacement
1805 * alt_group the same size as the original. This is needed to
1806 * allow propagate_alt_cfi() to do its magic. When the last
1807 * instruction affects the stack, the instruction after it (the
1808 * nop) will propagate the new state to the shared CFI array.
1809 */
1810 nop = malloc(sizeof(*nop));
1811 if (!nop) {
1812 WARN("malloc failed");
1813 return -1;
1814 }
1815 memset(nop, 0, sizeof(*nop));
1816
1817 nop->sec = special_alt->new_sec;
1818 nop->offset = special_alt->new_off + special_alt->new_len;
1819 nop->len = special_alt->orig_len - special_alt->new_len;
1820 nop->type = INSN_NOP;
1821 nop->sym = orig_insn->sym;
1822 nop->alt_group = new_alt_group;
1823 nop->ignore = orig_insn->ignore_alts;
1824 }
1825
1826 if (!special_alt->new_len) {
1827 *new_insn = nop;
1828 goto end;
1829 }
1830
1831 insn = *new_insn;
1832 sec_for_each_insn_from(file, insn) {
1833 struct reloc *alt_reloc;
1834
1835 if (insn->offset >= special_alt->new_off + special_alt->new_len)
1836 break;
1837
1838 last_new_insn = insn;
1839
1840 insn->ignore = orig_insn->ignore_alts;
1841 insn->sym = orig_insn->sym;
1842 insn->alt_group = new_alt_group;
1843
1844 /*
1845 * Since alternative replacement code is copy/pasted by the
1846 * kernel after applying relocations, generally such code can't
1847 * have relative-address relocation references to outside the
1848 * .altinstr_replacement section, unless the arch's
1849 * alternatives code can adjust the relative offsets
1850 * accordingly.
1851 */
1852 alt_reloc = insn_reloc(file, insn);
1853 if (alt_reloc && arch_pc_relative_reloc(alt_reloc) &&
1854 !arch_support_alt_relocation(special_alt, insn, alt_reloc)) {
1855
1856 WARN_INSN(insn, "unsupported relocation in alternatives section");
1857 return -1;
1858 }
1859
1860 if (!is_static_jump(insn))
1861 continue;
1862
1863 if (!insn->immediate)
1864 continue;
1865
1866 dest_off = arch_jump_destination(insn);
1867 if (dest_off == special_alt->new_off + special_alt->new_len) {
1868 insn->jump_dest = next_insn_same_sec(file, orig_alt_group->last_insn);
1869 if (!insn->jump_dest) {
1870 WARN_INSN(insn, "can't find alternative jump destination");
1871 return -1;
1872 }
1873 }
1874 }
1875
1876 if (!last_new_insn) {
1877 WARN_FUNC("can't find last new alternative instruction",
1878 special_alt->new_sec, special_alt->new_off);
1879 return -1;
1880 }
1881
1882 end:
1883 new_alt_group->orig_group = orig_alt_group;
1884 new_alt_group->first_insn = *new_insn;
1885 new_alt_group->last_insn = last_new_insn;
1886 new_alt_group->nop = nop;
1887 new_alt_group->cfi = orig_alt_group->cfi;
1888 return 0;
1889 }
1890
1891 /*
1892 * A jump table entry can either convert a nop to a jump or a jump to a nop.
1893 * If the original instruction is a jump, make the alt entry an effective nop
1894 * by just skipping the original instruction.
1895 */
1896 static int handle_jump_alt(struct objtool_file *file,
1897 struct special_alt *special_alt,
1898 struct instruction *orig_insn,
1899 struct instruction **new_insn)
1900 {
1901 if (orig_insn->type != INSN_JUMP_UNCONDITIONAL &&
1902 orig_insn->type != INSN_NOP) {
1903
1904 WARN_INSN(orig_insn, "unsupported instruction at jump label");
1905 return -1;
1906 }
1907
1908 if (opts.hack_jump_label && special_alt->key_addend & 2) {
1909 struct reloc *reloc = insn_reloc(file, orig_insn);
1910
1911 if (reloc)
1912 set_reloc_type(file->elf, reloc, R_NONE);
1913 elf_write_insn(file->elf, orig_insn->sec,
1914 orig_insn->offset, orig_insn->len,
1915 arch_nop_insn(orig_insn->len));
1916 orig_insn->type = INSN_NOP;
1917 }
1918
1919 if (orig_insn->type == INSN_NOP) {
1920 if (orig_insn->len == 2)
1921 file->jl_nop_short++;
1922 else
1923 file->jl_nop_long++;
1924
1925 return 0;
1926 }
1927
1928 if (orig_insn->len == 2)
1929 file->jl_short++;
1930 else
1931 file->jl_long++;
1932
1933 *new_insn = next_insn_same_sec(file, orig_insn);
1934 return 0;
1935 }
1936
1937 /*
1938 * Read all the special sections which have alternate instructions which can be
1939 * patched in or redirected to at runtime. Each instruction having alternate
1940 * instruction(s) has them added to its insn->alts list, which will be
1941 * traversed in validate_branch().
1942 */
1943 static int add_special_section_alts(struct objtool_file *file)
1944 {
1945 struct list_head special_alts;
1946 struct instruction *orig_insn, *new_insn;
1947 struct special_alt *special_alt, *tmp;
1948 struct alternative *alt;
1949 int ret;
1950
1951 ret = special_get_alts(file->elf, &special_alts);
1952 if (ret)
1953 return ret;
1954
1955 list_for_each_entry_safe(special_alt, tmp, &special_alts, list) {
1956
1957 orig_insn = find_insn(file, special_alt->orig_sec,
1958 special_alt->orig_off);
1959 if (!orig_insn) {
1960 WARN_FUNC("special: can't find orig instruction",
1961 special_alt->orig_sec, special_alt->orig_off);
1962 ret = -1;
1963 goto out;
1964 }
1965
1966 new_insn = NULL;
1967 if (!special_alt->group || special_alt->new_len) {
1968 new_insn = find_insn(file, special_alt->new_sec,
1969 special_alt->new_off);
1970 if (!new_insn) {
1971 WARN_FUNC("special: can't find new instruction",
1972 special_alt->new_sec,
1973 special_alt->new_off);
1974 ret = -1;
1975 goto out;
1976 }
1977 }
1978
1979 if (special_alt->group) {
1980 if (!special_alt->orig_len) {
1981 WARN_INSN(orig_insn, "empty alternative entry");
1982 continue;
1983 }
1984
1985 ret = handle_group_alt(file, special_alt, orig_insn,
1986 &new_insn);
1987 if (ret)
1988 goto out;
1989 } else if (special_alt->jump_or_nop) {
1990 ret = handle_jump_alt(file, special_alt, orig_insn,
1991 &new_insn);
1992 if (ret)
1993 goto out;
1994 }
1995
1996 alt = malloc(sizeof(*alt));
1997 if (!alt) {
1998 WARN("malloc failed");
1999 ret = -1;
2000 goto out;
2001 }
2002
2003 alt->insn = new_insn;
2004 alt->skip_orig = special_alt->skip_orig;
2005 orig_insn->ignore_alts |= special_alt->skip_alt;
2006 alt->next = orig_insn->alts;
2007 orig_insn->alts = alt;
2008
2009 list_del(&special_alt->list);
2010 free(special_alt);
2011 }
2012
2013 if (opts.stats) {
2014 printf("jl\\\tNOP\tJMP\n");
2015 printf("short:\t%ld\t%ld\n", file->jl_nop_short, file->jl_short);
2016 printf("long:\t%ld\t%ld\n", file->jl_nop_long, file->jl_long);
2017 }
2018
2019 out:
2020 return ret;
2021 }
2022
2023 static int add_jump_table(struct objtool_file *file, struct instruction *insn,
2024 struct reloc *next_table)
2025 {
2026 struct symbol *pfunc = insn_func(insn)->pfunc;
2027 struct reloc *table = insn_jump_table(insn);
2028 struct instruction *dest_insn;
2029 unsigned int prev_offset = 0;
2030 struct reloc *reloc = table;
2031 struct alternative *alt;
2032
2033 /*
2034 * Each @reloc is a switch table relocation which points to the target
2035 * instruction.
2036 */
2037 for_each_reloc_from(table->sec, reloc) {
2038
2039 /* Check for the end of the table: */
2040 if (reloc != table && reloc == next_table)
2041 break;
2042
2043 /* Make sure the table entries are consecutive: */
2044 if (prev_offset && reloc_offset(reloc) != prev_offset + 8)
2045 break;
2046
2047 /* Detect function pointers from contiguous objects: */
2048 if (reloc->sym->sec == pfunc->sec &&
2049 reloc_addend(reloc) == pfunc->offset)
2050 break;
2051
2052 dest_insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2053 if (!dest_insn)
2054 break;
2055
2056 /* Make sure the destination is in the same function: */
2057 if (!insn_func(dest_insn) || insn_func(dest_insn)->pfunc != pfunc)
2058 break;
2059
2060 alt = malloc(sizeof(*alt));
2061 if (!alt) {
2062 WARN("malloc failed");
2063 return -1;
2064 }
2065
2066 alt->insn = dest_insn;
2067 alt->next = insn->alts;
2068 insn->alts = alt;
2069 prev_offset = reloc_offset(reloc);
2070 }
2071
2072 if (!prev_offset) {
2073 WARN_INSN(insn, "can't find switch jump table");
2074 return -1;
2075 }
2076
2077 return 0;
2078 }
2079
2080 /*
2081 * find_jump_table() - Given a dynamic jump, find the switch jump table
2082 * associated with it.
2083 */
2084 static struct reloc *find_jump_table(struct objtool_file *file,
2085 struct symbol *func,
2086 struct instruction *insn)
2087 {
2088 struct reloc *table_reloc;
2089 struct instruction *dest_insn, *orig_insn = insn;
2090
2091 /*
2092 * Backward search using the @first_jump_src links, these help avoid
2093 * much of the 'in between' code. Which avoids us getting confused by
2094 * it.
2095 */
2096 for (;
2097 insn && insn_func(insn) && insn_func(insn)->pfunc == func;
2098 insn = insn->first_jump_src ?: prev_insn_same_sym(file, insn)) {
2099
2100 if (insn != orig_insn && insn->type == INSN_JUMP_DYNAMIC)
2101 break;
2102
2103 /* allow small jumps within the range */
2104 if (insn->type == INSN_JUMP_UNCONDITIONAL &&
2105 insn->jump_dest &&
2106 (insn->jump_dest->offset <= insn->offset ||
2107 insn->jump_dest->offset > orig_insn->offset))
2108 break;
2109
2110 table_reloc = arch_find_switch_table(file, insn);
2111 if (!table_reloc)
2112 continue;
2113 dest_insn = find_insn(file, table_reloc->sym->sec, reloc_addend(table_reloc));
2114 if (!dest_insn || !insn_func(dest_insn) || insn_func(dest_insn)->pfunc != func)
2115 continue;
2116
2117 return table_reloc;
2118 }
2119
2120 return NULL;
2121 }
2122
2123 /*
2124 * First pass: Mark the head of each jump table so that in the next pass,
2125 * we know when a given jump table ends and the next one starts.
2126 */
2127 static void mark_func_jump_tables(struct objtool_file *file,
2128 struct symbol *func)
2129 {
2130 struct instruction *insn, *last = NULL;
2131 struct reloc *reloc;
2132
2133 func_for_each_insn(file, func, insn) {
2134 if (!last)
2135 last = insn;
2136
2137 /*
2138 * Store back-pointers for unconditional forward jumps such
2139 * that find_jump_table() can back-track using those and
2140 * avoid some potentially confusing code.
2141 */
2142 if (insn->type == INSN_JUMP_UNCONDITIONAL && insn->jump_dest &&
2143 insn->offset > last->offset &&
2144 insn->jump_dest->offset > insn->offset &&
2145 !insn->jump_dest->first_jump_src) {
2146
2147 insn->jump_dest->first_jump_src = insn;
2148 last = insn->jump_dest;
2149 }
2150
2151 if (insn->type != INSN_JUMP_DYNAMIC)
2152 continue;
2153
2154 reloc = find_jump_table(file, func, insn);
2155 if (reloc)
2156 insn->_jump_table = reloc;
2157 }
2158 }
2159
2160 static int add_func_jump_tables(struct objtool_file *file,
2161 struct symbol *func)
2162 {
2163 struct instruction *insn, *insn_t1 = NULL, *insn_t2;
2164 int ret = 0;
2165
2166 func_for_each_insn(file, func, insn) {
2167 if (!insn_jump_table(insn))
2168 continue;
2169
2170 if (!insn_t1) {
2171 insn_t1 = insn;
2172 continue;
2173 }
2174
2175 insn_t2 = insn;
2176
2177 ret = add_jump_table(file, insn_t1, insn_jump_table(insn_t2));
2178 if (ret)
2179 return ret;
2180
2181 insn_t1 = insn_t2;
2182 }
2183
2184 if (insn_t1)
2185 ret = add_jump_table(file, insn_t1, NULL);
2186
2187 return ret;
2188 }
2189
2190 /*
2191 * For some switch statements, gcc generates a jump table in the .rodata
2192 * section which contains a list of addresses within the function to jump to.
2193 * This finds these jump tables and adds them to the insn->alts lists.
2194 */
2195 static int add_jump_table_alts(struct objtool_file *file)
2196 {
2197 struct symbol *func;
2198 int ret;
2199
2200 if (!file->rodata)
2201 return 0;
2202
2203 for_each_sym(file, func) {
2204 if (func->type != STT_FUNC)
2205 continue;
2206
2207 mark_func_jump_tables(file, func);
2208 ret = add_func_jump_tables(file, func);
2209 if (ret)
2210 return ret;
2211 }
2212
2213 return 0;
2214 }
2215
2216 static void set_func_state(struct cfi_state *state)
2217 {
2218 state->cfa = initial_func_cfi.cfa;
2219 memcpy(&state->regs, &initial_func_cfi.regs,
2220 CFI_NUM_REGS * sizeof(struct cfi_reg));
2221 state->stack_size = initial_func_cfi.cfa.offset;
2222 state->type = UNWIND_HINT_TYPE_CALL;
2223 }
2224
2225 static int read_unwind_hints(struct objtool_file *file)
2226 {
2227 struct cfi_state cfi = init_cfi;
2228 struct section *sec;
2229 struct unwind_hint *hint;
2230 struct instruction *insn;
2231 struct reloc *reloc;
2232 unsigned long offset;
2233 int i;
2234
2235 sec = find_section_by_name(file->elf, ".discard.unwind_hints");
2236 if (!sec)
2237 return 0;
2238
2239 if (!sec->rsec) {
2240 WARN("missing .rela.discard.unwind_hints section");
2241 return -1;
2242 }
2243
2244 if (sec->sh.sh_size % sizeof(struct unwind_hint)) {
2245 WARN("struct unwind_hint size mismatch");
2246 return -1;
2247 }
2248
2249 file->hints = true;
2250
2251 for (i = 0; i < sec->sh.sh_size / sizeof(struct unwind_hint); i++) {
2252 hint = (struct unwind_hint *)sec->data->d_buf + i;
2253
2254 reloc = find_reloc_by_dest(file->elf, sec, i * sizeof(*hint));
2255 if (!reloc) {
2256 WARN("can't find reloc for unwind_hints[%d]", i);
2257 return -1;
2258 }
2259
2260 if (reloc->sym->type == STT_SECTION) {
2261 offset = reloc_addend(reloc);
2262 } else if (reloc->sym->local_label) {
2263 offset = reloc->sym->offset;
2264 } else {
2265 WARN("unexpected relocation symbol type in %s", sec->rsec->name);
2266 return -1;
2267 }
2268
2269 insn = find_insn(file, reloc->sym->sec, offset);
2270 if (!insn) {
2271 WARN("can't find insn for unwind_hints[%d]", i);
2272 return -1;
2273 }
2274
2275 insn->hint = true;
2276
2277 if (hint->type == UNWIND_HINT_TYPE_UNDEFINED) {
2278 insn->cfi = &force_undefined_cfi;
2279 continue;
2280 }
2281
2282 if (hint->type == UNWIND_HINT_TYPE_SAVE) {
2283 insn->hint = false;
2284 insn->save = true;
2285 continue;
2286 }
2287
2288 if (hint->type == UNWIND_HINT_TYPE_RESTORE) {
2289 insn->restore = true;
2290 continue;
2291 }
2292
2293 if (hint->type == UNWIND_HINT_TYPE_REGS_PARTIAL) {
2294 struct symbol *sym = find_symbol_by_offset(insn->sec, insn->offset);
2295
2296 if (sym && sym->bind == STB_GLOBAL) {
2297 if (opts.ibt && insn->type != INSN_ENDBR && !insn->noendbr) {
2298 WARN_INSN(insn, "UNWIND_HINT_IRET_REGS without ENDBR");
2299 }
2300 }
2301 }
2302
2303 if (hint->type == UNWIND_HINT_TYPE_FUNC) {
2304 insn->cfi = &func_cfi;
2305 continue;
2306 }
2307
2308 if (insn->cfi)
2309 cfi = *(insn->cfi);
2310
2311 if (arch_decode_hint_reg(hint->sp_reg, &cfi.cfa.base)) {
2312 WARN_INSN(insn, "unsupported unwind_hint sp base reg %d", hint->sp_reg);
2313 return -1;
2314 }
2315
2316 cfi.cfa.offset = bswap_if_needed(file->elf, hint->sp_offset);
2317 cfi.type = hint->type;
2318 cfi.signal = hint->signal;
2319
2320 insn->cfi = cfi_hash_find_or_add(&cfi);
2321 }
2322
2323 return 0;
2324 }
2325
2326 static int read_noendbr_hints(struct objtool_file *file)
2327 {
2328 struct instruction *insn;
2329 struct section *rsec;
2330 struct reloc *reloc;
2331
2332 rsec = find_section_by_name(file->elf, ".rela.discard.noendbr");
2333 if (!rsec)
2334 return 0;
2335
2336 for_each_reloc(rsec, reloc) {
2337 insn = find_insn(file, reloc->sym->sec,
2338 reloc->sym->offset + reloc_addend(reloc));
2339 if (!insn) {
2340 WARN("bad .discard.noendbr entry");
2341 return -1;
2342 }
2343
2344 insn->noendbr = 1;
2345 }
2346
2347 return 0;
2348 }
2349
2350 static int read_retpoline_hints(struct objtool_file *file)
2351 {
2352 struct section *rsec;
2353 struct instruction *insn;
2354 struct reloc *reloc;
2355
2356 rsec = find_section_by_name(file->elf, ".rela.discard.retpoline_safe");
2357 if (!rsec)
2358 return 0;
2359
2360 for_each_reloc(rsec, reloc) {
2361 if (reloc->sym->type != STT_SECTION) {
2362 WARN("unexpected relocation symbol type in %s", rsec->name);
2363 return -1;
2364 }
2365
2366 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2367 if (!insn) {
2368 WARN("bad .discard.retpoline_safe entry");
2369 return -1;
2370 }
2371
2372 if (insn->type != INSN_JUMP_DYNAMIC &&
2373 insn->type != INSN_CALL_DYNAMIC &&
2374 insn->type != INSN_RETURN &&
2375 insn->type != INSN_NOP) {
2376 WARN_INSN(insn, "retpoline_safe hint not an indirect jump/call/ret/nop");
2377 return -1;
2378 }
2379
2380 insn->retpoline_safe = true;
2381 }
2382
2383 return 0;
2384 }
2385
2386 static int read_instr_hints(struct objtool_file *file)
2387 {
2388 struct section *rsec;
2389 struct instruction *insn;
2390 struct reloc *reloc;
2391
2392 rsec = find_section_by_name(file->elf, ".rela.discard.instr_end");
2393 if (!rsec)
2394 return 0;
2395
2396 for_each_reloc(rsec, reloc) {
2397 if (reloc->sym->type != STT_SECTION) {
2398 WARN("unexpected relocation symbol type in %s", rsec->name);
2399 return -1;
2400 }
2401
2402 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2403 if (!insn) {
2404 WARN("bad .discard.instr_end entry");
2405 return -1;
2406 }
2407
2408 insn->instr--;
2409 }
2410
2411 rsec = find_section_by_name(file->elf, ".rela.discard.instr_begin");
2412 if (!rsec)
2413 return 0;
2414
2415 for_each_reloc(rsec, reloc) {
2416 if (reloc->sym->type != STT_SECTION) {
2417 WARN("unexpected relocation symbol type in %s", rsec->name);
2418 return -1;
2419 }
2420
2421 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2422 if (!insn) {
2423 WARN("bad .discard.instr_begin entry");
2424 return -1;
2425 }
2426
2427 insn->instr++;
2428 }
2429
2430 return 0;
2431 }
2432
2433 static int read_validate_unret_hints(struct objtool_file *file)
2434 {
2435 struct section *rsec;
2436 struct instruction *insn;
2437 struct reloc *reloc;
2438
2439 rsec = find_section_by_name(file->elf, ".rela.discard.validate_unret");
2440 if (!rsec)
2441 return 0;
2442
2443 for_each_reloc(rsec, reloc) {
2444 if (reloc->sym->type != STT_SECTION) {
2445 WARN("unexpected relocation symbol type in %s", rsec->name);
2446 return -1;
2447 }
2448
2449 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2450 if (!insn) {
2451 WARN("bad .discard.instr_end entry");
2452 return -1;
2453 }
2454 insn->unret = 1;
2455 }
2456
2457 return 0;
2458 }
2459
2460
2461 static int read_intra_function_calls(struct objtool_file *file)
2462 {
2463 struct instruction *insn;
2464 struct section *rsec;
2465 struct reloc *reloc;
2466
2467 rsec = find_section_by_name(file->elf, ".rela.discard.intra_function_calls");
2468 if (!rsec)
2469 return 0;
2470
2471 for_each_reloc(rsec, reloc) {
2472 unsigned long dest_off;
2473
2474 if (reloc->sym->type != STT_SECTION) {
2475 WARN("unexpected relocation symbol type in %s",
2476 rsec->name);
2477 return -1;
2478 }
2479
2480 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2481 if (!insn) {
2482 WARN("bad .discard.intra_function_call entry");
2483 return -1;
2484 }
2485
2486 if (insn->type != INSN_CALL) {
2487 WARN_INSN(insn, "intra_function_call not a direct call");
2488 return -1;
2489 }
2490
2491 /*
2492 * Treat intra-function CALLs as JMPs, but with a stack_op.
2493 * See add_call_destinations(), which strips stack_ops from
2494 * normal CALLs.
2495 */
2496 insn->type = INSN_JUMP_UNCONDITIONAL;
2497
2498 dest_off = arch_jump_destination(insn);
2499 insn->jump_dest = find_insn(file, insn->sec, dest_off);
2500 if (!insn->jump_dest) {
2501 WARN_INSN(insn, "can't find call dest at %s+0x%lx",
2502 insn->sec->name, dest_off);
2503 return -1;
2504 }
2505 }
2506
2507 return 0;
2508 }
2509
2510 /*
2511 * Return true if name matches an instrumentation function, where calls to that
2512 * function from noinstr code can safely be removed, but compilers won't do so.
2513 */
2514 static bool is_profiling_func(const char *name)
2515 {
2516 /*
2517 * Many compilers cannot disable KCOV with a function attribute.
2518 */
2519 if (!strncmp(name, "__sanitizer_cov_", 16))
2520 return true;
2521
2522 /*
2523 * Some compilers currently do not remove __tsan_func_entry/exit nor
2524 * __tsan_atomic_signal_fence (used for barrier instrumentation) with
2525 * the __no_sanitize_thread attribute, remove them. Once the kernel's
2526 * minimum Clang version is 14.0, this can be removed.
2527 */
2528 if (!strncmp(name, "__tsan_func_", 12) ||
2529 !strcmp(name, "__tsan_atomic_signal_fence"))
2530 return true;
2531
2532 return false;
2533 }
2534
2535 static int classify_symbols(struct objtool_file *file)
2536 {
2537 struct symbol *func;
2538
2539 for_each_sym(file, func) {
2540 if (func->type == STT_NOTYPE && strstarts(func->name, ".L"))
2541 func->local_label = true;
2542
2543 if (func->bind != STB_GLOBAL)
2544 continue;
2545
2546 if (!strncmp(func->name, STATIC_CALL_TRAMP_PREFIX_STR,
2547 strlen(STATIC_CALL_TRAMP_PREFIX_STR)))
2548 func->static_call_tramp = true;
2549
2550 if (arch_is_retpoline(func))
2551 func->retpoline_thunk = true;
2552
2553 if (arch_is_rethunk(func))
2554 func->return_thunk = true;
2555
2556 if (arch_is_embedded_insn(func))
2557 func->embedded_insn = true;
2558
2559 if (arch_ftrace_match(func->name))
2560 func->fentry = true;
2561
2562 if (is_profiling_func(func->name))
2563 func->profiling_func = true;
2564 }
2565
2566 return 0;
2567 }
2568
2569 static void mark_rodata(struct objtool_file *file)
2570 {
2571 struct section *sec;
2572 bool found = false;
2573
2574 /*
2575 * Search for the following rodata sections, each of which can
2576 * potentially contain jump tables:
2577 *
2578 * - .rodata: can contain GCC switch tables
2579 * - .rodata.<func>: same, if -fdata-sections is being used
2580 * - .rodata..c_jump_table: contains C annotated jump tables
2581 *
2582 * .rodata.str1.* sections are ignored; they don't contain jump tables.
2583 */
2584 for_each_sec(file, sec) {
2585 if (!strncmp(sec->name, ".rodata", 7) &&
2586 !strstr(sec->name, ".str1.")) {
2587 sec->rodata = true;
2588 found = true;
2589 }
2590 }
2591
2592 file->rodata = found;
2593 }
2594
2595 static int decode_sections(struct objtool_file *file)
2596 {
2597 int ret;
2598
2599 mark_rodata(file);
2600
2601 ret = init_pv_ops(file);
2602 if (ret)
2603 return ret;
2604
2605 /*
2606 * Must be before add_{jump_call}_destination.
2607 */
2608 ret = classify_symbols(file);
2609 if (ret)
2610 return ret;
2611
2612 ret = decode_instructions(file);
2613 if (ret)
2614 return ret;
2615
2616 add_ignores(file);
2617 add_uaccess_safe(file);
2618
2619 ret = add_ignore_alternatives(file);
2620 if (ret)
2621 return ret;
2622
2623 /*
2624 * Must be before read_unwind_hints() since that needs insn->noendbr.
2625 */
2626 ret = read_noendbr_hints(file);
2627 if (ret)
2628 return ret;
2629
2630 /*
2631 * Must be before add_jump_destinations(), which depends on 'func'
2632 * being set for alternatives, to enable proper sibling call detection.
2633 */
2634 if (opts.stackval || opts.orc || opts.uaccess || opts.noinstr) {
2635 ret = add_special_section_alts(file);
2636 if (ret)
2637 return ret;
2638 }
2639
2640 ret = add_jump_destinations(file);
2641 if (ret)
2642 return ret;
2643
2644 /*
2645 * Must be before add_call_destination(); it changes INSN_CALL to
2646 * INSN_JUMP.
2647 */
2648 ret = read_intra_function_calls(file);
2649 if (ret)
2650 return ret;
2651
2652 ret = add_call_destinations(file);
2653 if (ret)
2654 return ret;
2655
2656 /*
2657 * Must be after add_call_destinations() such that it can override
2658 * dead_end_function() marks.
2659 */
2660 ret = add_dead_ends(file);
2661 if (ret)
2662 return ret;
2663
2664 ret = add_jump_table_alts(file);
2665 if (ret)
2666 return ret;
2667
2668 ret = read_unwind_hints(file);
2669 if (ret)
2670 return ret;
2671
2672 ret = read_retpoline_hints(file);
2673 if (ret)
2674 return ret;
2675
2676 ret = read_instr_hints(file);
2677 if (ret)
2678 return ret;
2679
2680 ret = read_validate_unret_hints(file);
2681 if (ret)
2682 return ret;
2683
2684 return 0;
2685 }
2686
2687 static bool is_special_call(struct instruction *insn)
2688 {
2689 if (insn->type == INSN_CALL) {
2690 struct symbol *dest = insn_call_dest(insn);
2691
2692 if (!dest)
2693 return false;
2694
2695 if (dest->fentry || dest->embedded_insn)
2696 return true;
2697 }
2698
2699 return false;
2700 }
2701
2702 static bool has_modified_stack_frame(struct instruction *insn, struct insn_state *state)
2703 {
2704 struct cfi_state *cfi = &state->cfi;
2705 int i;
2706
2707 if (cfi->cfa.base != initial_func_cfi.cfa.base || cfi->drap)
2708 return true;
2709
2710 if (cfi->cfa.offset != initial_func_cfi.cfa.offset)
2711 return true;
2712
2713 if (cfi->stack_size != initial_func_cfi.cfa.offset)
2714 return true;
2715
2716 for (i = 0; i < CFI_NUM_REGS; i++) {
2717 if (cfi->regs[i].base != initial_func_cfi.regs[i].base ||
2718 cfi->regs[i].offset != initial_func_cfi.regs[i].offset)
2719 return true;
2720 }
2721
2722 return false;
2723 }
2724
2725 static bool check_reg_frame_pos(const struct cfi_reg *reg,
2726 int expected_offset)
2727 {
2728 return reg->base == CFI_CFA &&
2729 reg->offset == expected_offset;
2730 }
2731
2732 static bool has_valid_stack_frame(struct insn_state *state)
2733 {
2734 struct cfi_state *cfi = &state->cfi;
2735
2736 if (cfi->cfa.base == CFI_BP &&
2737 check_reg_frame_pos(&cfi->regs[CFI_BP], -cfi->cfa.offset) &&
2738 check_reg_frame_pos(&cfi->regs[CFI_RA], -cfi->cfa.offset + 8))
2739 return true;
2740
2741 if (cfi->drap && cfi->regs[CFI_BP].base == CFI_BP)
2742 return true;
2743
2744 return false;
2745 }
2746
2747 static int update_cfi_state_regs(struct instruction *insn,
2748 struct cfi_state *cfi,
2749 struct stack_op *op)
2750 {
2751 struct cfi_reg *cfa = &cfi->cfa;
2752
2753 if (cfa->base != CFI_SP && cfa->base != CFI_SP_INDIRECT)
2754 return 0;
2755
2756 /* push */
2757 if (op->dest.type == OP_DEST_PUSH || op->dest.type == OP_DEST_PUSHF)
2758 cfa->offset += 8;
2759
2760 /* pop */
2761 if (op->src.type == OP_SRC_POP || op->src.type == OP_SRC_POPF)
2762 cfa->offset -= 8;
2763
2764 /* add immediate to sp */
2765 if (op->dest.type == OP_DEST_REG && op->src.type == OP_SRC_ADD &&
2766 op->dest.reg == CFI_SP && op->src.reg == CFI_SP)
2767 cfa->offset -= op->src.offset;
2768
2769 return 0;
2770 }
2771
2772 static void save_reg(struct cfi_state *cfi, unsigned char reg, int base, int offset)
2773 {
2774 if (arch_callee_saved_reg(reg) &&
2775 cfi->regs[reg].base == CFI_UNDEFINED) {
2776 cfi->regs[reg].base = base;
2777 cfi->regs[reg].offset = offset;
2778 }
2779 }
2780
2781 static void restore_reg(struct cfi_state *cfi, unsigned char reg)
2782 {
2783 cfi->regs[reg].base = initial_func_cfi.regs[reg].base;
2784 cfi->regs[reg].offset = initial_func_cfi.regs[reg].offset;
2785 }
2786
2787 /*
2788 * A note about DRAP stack alignment:
2789 *
2790 * GCC has the concept of a DRAP register, which is used to help keep track of
2791 * the stack pointer when aligning the stack. r10 or r13 is used as the DRAP
2792 * register. The typical DRAP pattern is:
2793 *
2794 * 4c 8d 54 24 08 lea 0x8(%rsp),%r10
2795 * 48 83 e4 c0 and $0xffffffffffffffc0,%rsp
2796 * 41 ff 72 f8 pushq -0x8(%r10)
2797 * 55 push %rbp
2798 * 48 89 e5 mov %rsp,%rbp
2799 * (more pushes)
2800 * 41 52 push %r10
2801 * ...
2802 * 41 5a pop %r10
2803 * (more pops)
2804 * 5d pop %rbp
2805 * 49 8d 62 f8 lea -0x8(%r10),%rsp
2806 * c3 retq
2807 *
2808 * There are some variations in the epilogues, like:
2809 *
2810 * 5b pop %rbx
2811 * 41 5a pop %r10
2812 * 41 5c pop %r12
2813 * 41 5d pop %r13
2814 * 41 5e pop %r14
2815 * c9 leaveq
2816 * 49 8d 62 f8 lea -0x8(%r10),%rsp
2817 * c3 retq
2818 *
2819 * and:
2820 *
2821 * 4c 8b 55 e8 mov -0x18(%rbp),%r10
2822 * 48 8b 5d e0 mov -0x20(%rbp),%rbx
2823 * 4c 8b 65 f0 mov -0x10(%rbp),%r12
2824 * 4c 8b 6d f8 mov -0x8(%rbp),%r13
2825 * c9 leaveq
2826 * 49 8d 62 f8 lea -0x8(%r10),%rsp
2827 * c3 retq
2828 *
2829 * Sometimes r13 is used as the DRAP register, in which case it's saved and
2830 * restored beforehand:
2831 *
2832 * 41 55 push %r13
2833 * 4c 8d 6c 24 10 lea 0x10(%rsp),%r13
2834 * 48 83 e4 f0 and $0xfffffffffffffff0,%rsp
2835 * ...
2836 * 49 8d 65 f0 lea -0x10(%r13),%rsp
2837 * 41 5d pop %r13
2838 * c3 retq
2839 */
2840 static int update_cfi_state(struct instruction *insn,
2841 struct instruction *next_insn,
2842 struct cfi_state *cfi, struct stack_op *op)
2843 {
2844 struct cfi_reg *cfa = &cfi->cfa;
2845 struct cfi_reg *regs = cfi->regs;
2846
2847 /* ignore UNWIND_HINT_UNDEFINED regions */
2848 if (cfi->force_undefined)
2849 return 0;
2850
2851 /* stack operations don't make sense with an undefined CFA */
2852 if (cfa->base == CFI_UNDEFINED) {
2853 if (insn_func(insn)) {
2854 WARN_INSN(insn, "undefined stack state");
2855 return -1;
2856 }
2857 return 0;
2858 }
2859
2860 if (cfi->type == UNWIND_HINT_TYPE_REGS ||
2861 cfi->type == UNWIND_HINT_TYPE_REGS_PARTIAL)
2862 return update_cfi_state_regs(insn, cfi, op);
2863
2864 switch (op->dest.type) {
2865
2866 case OP_DEST_REG:
2867 switch (op->src.type) {
2868
2869 case OP_SRC_REG:
2870 if (op->src.reg == CFI_SP && op->dest.reg == CFI_BP &&
2871 cfa->base == CFI_SP &&
2872 check_reg_frame_pos(®s[CFI_BP], -cfa->offset)) {
2873
2874 /* mov %rsp, %rbp */
2875 cfa->base = op->dest.reg;
2876 cfi->bp_scratch = false;
2877 }
2878
2879 else if (op->src.reg == CFI_SP &&
2880 op->dest.reg == CFI_BP && cfi->drap) {
2881
2882 /* drap: mov %rsp, %rbp */
2883 regs[CFI_BP].base = CFI_BP;
2884 regs[CFI_BP].offset = -cfi->stack_size;
2885 cfi->bp_scratch = false;
2886 }
2887
2888 else if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
2889
2890 /*
2891 * mov %rsp, %reg
2892 *
2893 * This is needed for the rare case where GCC
2894 * does:
2895 *
2896 * mov %rsp, %rax
2897 * ...
2898 * mov %rax, %rsp
2899 */
2900 cfi->vals[op->dest.reg].base = CFI_CFA;
2901 cfi->vals[op->dest.reg].offset = -cfi->stack_size;
2902 }
2903
2904 else if (op->src.reg == CFI_BP && op->dest.reg == CFI_SP &&
2905 (cfa->base == CFI_BP || cfa->base == cfi->drap_reg)) {
2906
2907 /*
2908 * mov %rbp, %rsp
2909 *
2910 * Restore the original stack pointer (Clang).
2911 */
2912 cfi->stack_size = -cfi->regs[CFI_BP].offset;
2913 }
2914
2915 else if (op->dest.reg == cfa->base) {
2916
2917 /* mov %reg, %rsp */
2918 if (cfa->base == CFI_SP &&
2919 cfi->vals[op->src.reg].base == CFI_CFA) {
2920
2921 /*
2922 * This is needed for the rare case
2923 * where GCC does something dumb like:
2924 *
2925 * lea 0x8(%rsp), %rcx
2926 * ...
2927 * mov %rcx, %rsp
2928 */
2929 cfa->offset = -cfi->vals[op->src.reg].offset;
2930 cfi->stack_size = cfa->offset;
2931
2932 } else if (cfa->base == CFI_SP &&
2933 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2934 cfi->vals[op->src.reg].offset == cfa->offset) {
2935
2936 /*
2937 * Stack swizzle:
2938 *
2939 * 1: mov %rsp, (%[tos])
2940 * 2: mov %[tos], %rsp
2941 * ...
2942 * 3: pop %rsp
2943 *
2944 * Where:
2945 *
2946 * 1 - places a pointer to the previous
2947 * stack at the Top-of-Stack of the
2948 * new stack.
2949 *
2950 * 2 - switches to the new stack.
2951 *
2952 * 3 - pops the Top-of-Stack to restore
2953 * the original stack.
2954 *
2955 * Note: we set base to SP_INDIRECT
2956 * here and preserve offset. Therefore
2957 * when the unwinder reaches ToS it
2958 * will dereference SP and then add the
2959 * offset to find the next frame, IOW:
2960 * (%rsp) + offset.
2961 */
2962 cfa->base = CFI_SP_INDIRECT;
2963
2964 } else {
2965 cfa->base = CFI_UNDEFINED;
2966 cfa->offset = 0;
2967 }
2968 }
2969
2970 else if (op->dest.reg == CFI_SP &&
2971 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2972 cfi->vals[op->src.reg].offset == cfa->offset) {
2973
2974 /*
2975 * The same stack swizzle case 2) as above. But
2976 * because we can't change cfa->base, case 3)
2977 * will become a regular POP. Pretend we're a
2978 * PUSH so things don't go unbalanced.
2979 */
2980 cfi->stack_size += 8;
2981 }
2982
2983
2984 break;
2985
2986 case OP_SRC_ADD:
2987 if (op->dest.reg == CFI_SP && op->src.reg == CFI_SP) {
2988
2989 /* add imm, %rsp */
2990 cfi->stack_size -= op->src.offset;
2991 if (cfa->base == CFI_SP)
2992 cfa->offset -= op->src.offset;
2993 break;
2994 }
2995
2996 if (op->dest.reg == CFI_BP && op->src.reg == CFI_SP &&
2997 insn->sym->frame_pointer) {
2998 /* addi.d fp,sp,imm on LoongArch */
2999 if (cfa->base == CFI_SP && cfa->offset == op->src.offset) {
3000 cfa->base = CFI_BP;
3001 cfa->offset = 0;
3002 }
3003 break;
3004 }
3005
3006 if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) {
3007 /* addi.d sp,fp,imm on LoongArch */
3008 if (cfa->base == CFI_BP && cfa->offset == 0) {
3009 if (insn->sym->frame_pointer) {
3010 cfa->base = CFI_SP;
3011 cfa->offset = -op->src.offset;
3012 }
3013 } else {
3014 /* lea disp(%rbp), %rsp */
3015 cfi->stack_size = -(op->src.offset + regs[CFI_BP].offset);
3016 }
3017 break;
3018 }
3019
3020 if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
3021
3022 /* drap: lea disp(%rsp), %drap */
3023 cfi->drap_reg = op->dest.reg;
3024
3025 /*
3026 * lea disp(%rsp), %reg
3027 *
3028 * This is needed for the rare case where GCC
3029 * does something dumb like:
3030 *
3031 * lea 0x8(%rsp), %rcx
3032 * ...
3033 * mov %rcx, %rsp
3034 */
3035 cfi->vals[op->dest.reg].base = CFI_CFA;
3036 cfi->vals[op->dest.reg].offset = \
3037 -cfi->stack_size + op->src.offset;
3038
3039 break;
3040 }
3041
3042 if (cfi->drap && op->dest.reg == CFI_SP &&
3043 op->src.reg == cfi->drap_reg) {
3044
3045 /* drap: lea disp(%drap), %rsp */
3046 cfa->base = CFI_SP;
3047 cfa->offset = cfi->stack_size = -op->src.offset;
3048 cfi->drap_reg = CFI_UNDEFINED;
3049 cfi->drap = false;
3050 break;
3051 }
3052
3053 if (op->dest.reg == cfi->cfa.base && !(next_insn && next_insn->hint)) {
3054 WARN_INSN(insn, "unsupported stack register modification");
3055 return -1;
3056 }
3057
3058 break;
3059
3060 case OP_SRC_AND:
3061 if (op->dest.reg != CFI_SP ||
3062 (cfi->drap_reg != CFI_UNDEFINED && cfa->base != CFI_SP) ||
3063 (cfi->drap_reg == CFI_UNDEFINED && cfa->base != CFI_BP)) {
3064 WARN_INSN(insn, "unsupported stack pointer realignment");
3065 return -1;
3066 }
3067
3068 if (cfi->drap_reg != CFI_UNDEFINED) {
3069 /* drap: and imm, %rsp */
3070 cfa->base = cfi->drap_reg;
3071 cfa->offset = cfi->stack_size = 0;
3072 cfi->drap = true;
3073 }
3074
3075 /*
3076 * Older versions of GCC (4.8ish) realign the stack
3077 * without DRAP, with a frame pointer.
3078 */
3079
3080 break;
3081
3082 case OP_SRC_POP:
3083 case OP_SRC_POPF:
3084 if (op->dest.reg == CFI_SP && cfa->base == CFI_SP_INDIRECT) {
3085
3086 /* pop %rsp; # restore from a stack swizzle */
3087 cfa->base = CFI_SP;
3088 break;
3089 }
3090
3091 if (!cfi->drap && op->dest.reg == cfa->base) {
3092
3093 /* pop %rbp */
3094 cfa->base = CFI_SP;
3095 }
3096
3097 if (cfi->drap && cfa->base == CFI_BP_INDIRECT &&
3098 op->dest.reg == cfi->drap_reg &&
3099 cfi->drap_offset == -cfi->stack_size) {
3100
3101 /* drap: pop %drap */
3102 cfa->base = cfi->drap_reg;
3103 cfa->offset = 0;
3104 cfi->drap_offset = -1;
3105
3106 } else if (cfi->stack_size == -regs[op->dest.reg].offset) {
3107
3108 /* pop %reg */
3109 restore_reg(cfi, op->dest.reg);
3110 }
3111
3112 cfi->stack_size -= 8;
3113 if (cfa->base == CFI_SP)
3114 cfa->offset -= 8;
3115
3116 break;
3117
3118 case OP_SRC_REG_INDIRECT:
3119 if (!cfi->drap && op->dest.reg == cfa->base &&
3120 op->dest.reg == CFI_BP) {
3121
3122 /* mov disp(%rsp), %rbp */
3123 cfa->base = CFI_SP;
3124 cfa->offset = cfi->stack_size;
3125 }
3126
3127 if (cfi->drap && op->src.reg == CFI_BP &&
3128 op->src.offset == cfi->drap_offset) {
3129
3130 /* drap: mov disp(%rbp), %drap */
3131 cfa->base = cfi->drap_reg;
3132 cfa->offset = 0;
3133 cfi->drap_offset = -1;
3134 }
3135
3136 if (cfi->drap && op->src.reg == CFI_BP &&
3137 op->src.offset == regs[op->dest.reg].offset) {
3138
3139 /* drap: mov disp(%rbp), %reg */
3140 restore_reg(cfi, op->dest.reg);
3141
3142 } else if (op->src.reg == cfa->base &&
3143 op->src.offset == regs[op->dest.reg].offset + cfa->offset) {
3144
3145 /* mov disp(%rbp), %reg */
3146 /* mov disp(%rsp), %reg */
3147 restore_reg(cfi, op->dest.reg);
3148
3149 } else if (op->src.reg == CFI_SP &&
3150 op->src.offset == regs[op->dest.reg].offset + cfi->stack_size) {
3151
3152 /* mov disp(%rsp), %reg */
3153 restore_reg(cfi, op->dest.reg);
3154 }
3155
3156 break;
3157
3158 default:
3159 WARN_INSN(insn, "unknown stack-related instruction");
3160 return -1;
3161 }
3162
3163 break;
3164
3165 case OP_DEST_PUSH:
3166 case OP_DEST_PUSHF:
3167 cfi->stack_size += 8;
3168 if (cfa->base == CFI_SP)
3169 cfa->offset += 8;
3170
3171 if (op->src.type != OP_SRC_REG)
3172 break;
3173
3174 if (cfi->drap) {
3175 if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
3176
3177 /* drap: push %drap */
3178 cfa->base = CFI_BP_INDIRECT;
3179 cfa->offset = -cfi->stack_size;
3180
3181 /* save drap so we know when to restore it */
3182 cfi->drap_offset = -cfi->stack_size;
3183
3184 } else if (op->src.reg == CFI_BP && cfa->base == cfi->drap_reg) {
3185
3186 /* drap: push %rbp */
3187 cfi->stack_size = 0;
3188
3189 } else {
3190
3191 /* drap: push %reg */
3192 save_reg(cfi, op->src.reg, CFI_BP, -cfi->stack_size);
3193 }
3194
3195 } else {
3196
3197 /* push %reg */
3198 save_reg(cfi, op->src.reg, CFI_CFA, -cfi->stack_size);
3199 }
3200
3201 /* detect when asm code uses rbp as a scratch register */
3202 if (opts.stackval && insn_func(insn) && op->src.reg == CFI_BP &&
3203 cfa->base != CFI_BP)
3204 cfi->bp_scratch = true;
3205 break;
3206
3207 case OP_DEST_REG_INDIRECT:
3208
3209 if (cfi->drap) {
3210 if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
3211
3212 /* drap: mov %drap, disp(%rbp) */
3213 cfa->base = CFI_BP_INDIRECT;
3214 cfa->offset = op->dest.offset;
3215
3216 /* save drap offset so we know when to restore it */
3217 cfi->drap_offset = op->dest.offset;
3218 } else {
3219
3220 /* drap: mov reg, disp(%rbp) */
3221 save_reg(cfi, op->src.reg, CFI_BP, op->dest.offset);
3222 }
3223
3224 } else if (op->dest.reg == cfa->base) {
3225
3226 /* mov reg, disp(%rbp) */
3227 /* mov reg, disp(%rsp) */
3228 save_reg(cfi, op->src.reg, CFI_CFA,
3229 op->dest.offset - cfi->cfa.offset);
3230
3231 } else if (op->dest.reg == CFI_SP) {
3232
3233 /* mov reg, disp(%rsp) */
3234 save_reg(cfi, op->src.reg, CFI_CFA,
3235 op->dest.offset - cfi->stack_size);
3236
3237 } else if (op->src.reg == CFI_SP && op->dest.offset == 0) {
3238
3239 /* mov %rsp, (%reg); # setup a stack swizzle. */
3240 cfi->vals[op->dest.reg].base = CFI_SP_INDIRECT;
3241 cfi->vals[op->dest.reg].offset = cfa->offset;
3242 }
3243
3244 break;
3245
3246 case OP_DEST_MEM:
3247 if (op->src.type != OP_SRC_POP && op->src.type != OP_SRC_POPF) {
3248 WARN_INSN(insn, "unknown stack-related memory operation");
3249 return -1;
3250 }
3251
3252 /* pop mem */
3253 cfi->stack_size -= 8;
3254 if (cfa->base == CFI_SP)
3255 cfa->offset -= 8;
3256
3257 break;
3258
3259 default:
3260 WARN_INSN(insn, "unknown stack-related instruction");
3261 return -1;
3262 }
3263
3264 return 0;
3265 }
3266
3267 /*
3268 * The stack layouts of alternatives instructions can sometimes diverge when
3269 * they have stack modifications. That's fine as long as the potential stack
3270 * layouts don't conflict at any given potential instruction boundary.
3271 *
3272 * Flatten the CFIs of the different alternative code streams (both original
3273 * and replacement) into a single shared CFI array which can be used to detect
3274 * conflicts and nicely feed a linear array of ORC entries to the unwinder.
3275 */
3276 static int propagate_alt_cfi(struct objtool_file *file, struct instruction *insn)
3277 {
3278 struct cfi_state **alt_cfi;
3279 int group_off;
3280
3281 if (!insn->alt_group)
3282 return 0;
3283
3284 if (!insn->cfi) {
3285 WARN("CFI missing");
3286 return -1;
3287 }
3288
3289 alt_cfi = insn->alt_group->cfi;
3290 group_off = insn->offset - insn->alt_group->first_insn->offset;
3291
3292 if (!alt_cfi[group_off]) {
3293 alt_cfi[group_off] = insn->cfi;
3294 } else {
3295 if (cficmp(alt_cfi[group_off], insn->cfi)) {
3296 struct alt_group *orig_group = insn->alt_group->orig_group ?: insn->alt_group;
3297 struct instruction *orig = orig_group->first_insn;
3298 char *where = offstr(insn->sec, insn->offset);
3299 WARN_INSN(orig, "stack layout conflict in alternatives: %s", where);
3300 free(where);
3301 return -1;
3302 }
3303 }
3304
3305 return 0;
3306 }
3307
3308 static int handle_insn_ops(struct instruction *insn,
3309 struct instruction *next_insn,
3310 struct insn_state *state)
3311 {
3312 struct stack_op *op;
3313
3314 for (op = insn->stack_ops; op; op = op->next) {
3315
3316 if (update_cfi_state(insn, next_insn, &state->cfi, op))
3317 return 1;
3318
3319 if (!insn->alt_group)
3320 continue;
3321
3322 if (op->dest.type == OP_DEST_PUSHF) {
3323 if (!state->uaccess_stack) {
3324 state->uaccess_stack = 1;
3325 } else if (state->uaccess_stack >> 31) {
3326 WARN_INSN(insn, "PUSHF stack exhausted");
3327 return 1;
3328 }
3329 state->uaccess_stack <<= 1;
3330 state->uaccess_stack |= state->uaccess;
3331 }
3332
3333 if (op->src.type == OP_SRC_POPF) {
3334 if (state->uaccess_stack) {
3335 state->uaccess = state->uaccess_stack & 1;
3336 state->uaccess_stack >>= 1;
3337 if (state->uaccess_stack == 1)
3338 state->uaccess_stack = 0;
3339 }
3340 }
3341 }
3342
3343 return 0;
3344 }
3345
3346 static bool insn_cfi_match(struct instruction *insn, struct cfi_state *cfi2)
3347 {
3348 struct cfi_state *cfi1 = insn->cfi;
3349 int i;
3350
3351 if (!cfi1) {
3352 WARN("CFI missing");
3353 return false;
3354 }
3355
3356 if (memcmp(&cfi1->cfa, &cfi2->cfa, sizeof(cfi1->cfa))) {
3357
3358 WARN_INSN(insn, "stack state mismatch: cfa1=%d%+d cfa2=%d%+d",
3359 cfi1->cfa.base, cfi1->cfa.offset,
3360 cfi2->cfa.base, cfi2->cfa.offset);
3361
3362 } else if (memcmp(&cfi1->regs, &cfi2->regs, sizeof(cfi1->regs))) {
3363 for (i = 0; i < CFI_NUM_REGS; i++) {
3364 if (!memcmp(&cfi1->regs[i], &cfi2->regs[i],
3365 sizeof(struct cfi_reg)))
3366 continue;
3367
3368 WARN_INSN(insn, "stack state mismatch: reg1[%d]=%d%+d reg2[%d]=%d%+d",
3369 i, cfi1->regs[i].base, cfi1->regs[i].offset,
3370 i, cfi2->regs[i].base, cfi2->regs[i].offset);
3371 break;
3372 }
3373
3374 } else if (cfi1->type != cfi2->type) {
3375
3376 WARN_INSN(insn, "stack state mismatch: type1=%d type2=%d",
3377 cfi1->type, cfi2->type);
3378
3379 } else if (cfi1->drap != cfi2->drap ||
3380 (cfi1->drap && cfi1->drap_reg != cfi2->drap_reg) ||
3381 (cfi1->drap && cfi1->drap_offset != cfi2->drap_offset)) {
3382
3383 WARN_INSN(insn, "stack state mismatch: drap1=%d(%d,%d) drap2=%d(%d,%d)",
3384 cfi1->drap, cfi1->drap_reg, cfi1->drap_offset,
3385 cfi2->drap, cfi2->drap_reg, cfi2->drap_offset);
3386
3387 } else
3388 return true;
3389
3390 return false;
3391 }
3392
3393 static inline bool func_uaccess_safe(struct symbol *func)
3394 {
3395 if (func)
3396 return func->uaccess_safe;
3397
3398 return false;
3399 }
3400
3401 static inline const char *call_dest_name(struct instruction *insn)
3402 {
3403 static char pvname[19];
3404 struct reloc *reloc;
3405 int idx;
3406
3407 if (insn_call_dest(insn))
3408 return insn_call_dest(insn)->name;
3409
3410 reloc = insn_reloc(NULL, insn);
3411 if (reloc && !strcmp(reloc->sym->name, "pv_ops")) {
3412 idx = (reloc_addend(reloc) / sizeof(void *));
3413 snprintf(pvname, sizeof(pvname), "pv_ops[%d]", idx);
3414 return pvname;
3415 }
3416
3417 return "{dynamic}";
3418 }
3419
3420 static bool pv_call_dest(struct objtool_file *file, struct instruction *insn)
3421 {
3422 struct symbol *target;
3423 struct reloc *reloc;
3424 int idx;
3425
3426 reloc = insn_reloc(file, insn);
3427 if (!reloc || strcmp(reloc->sym->name, "pv_ops"))
3428 return false;
3429
3430 idx = (arch_dest_reloc_offset(reloc_addend(reloc)) / sizeof(void *));
3431
3432 if (file->pv_ops[idx].clean)
3433 return true;
3434
3435 file->pv_ops[idx].clean = true;
3436
3437 list_for_each_entry(target, &file->pv_ops[idx].targets, pv_target) {
3438 if (!target->sec->noinstr) {
3439 WARN("pv_ops[%d]: %s", idx, target->name);
3440 file->pv_ops[idx].clean = false;
3441 }
3442 }
3443
3444 return file->pv_ops[idx].clean;
3445 }
3446
3447 static inline bool noinstr_call_dest(struct objtool_file *file,
3448 struct instruction *insn,
3449 struct symbol *func)
3450 {
3451 /*
3452 * We can't deal with indirect function calls at present;
3453 * assume they're instrumented.
3454 */
3455 if (!func) {
3456 if (file->pv_ops)
3457 return pv_call_dest(file, insn);
3458
3459 return false;
3460 }
3461
3462 /*
3463 * If the symbol is from a noinstr section; we good.
3464 */
3465 if (func->sec->noinstr)
3466 return true;
3467
3468 /*
3469 * If the symbol is a static_call trampoline, we can't tell.
3470 */
3471 if (func->static_call_tramp)
3472 return true;
3473
3474 /*
3475 * The __ubsan_handle_*() calls are like WARN(), they only happen when
3476 * something 'BAD' happened. At the risk of taking the machine down,
3477 * let them proceed to get the message out.
3478 */
3479 if (!strncmp(func->name, "__ubsan_handle_", 15))
3480 return true;
3481
3482 return false;
3483 }
3484
3485 static int validate_call(struct objtool_file *file,
3486 struct instruction *insn,
3487 struct insn_state *state)
3488 {
3489 if (state->noinstr && state->instr <= 0 &&
3490 !noinstr_call_dest(file, insn, insn_call_dest(insn))) {
3491 WARN_INSN(insn, "call to %s() leaves .noinstr.text section", call_dest_name(insn));
3492 return 1;
3493 }
3494
3495 if (state->uaccess && !func_uaccess_safe(insn_call_dest(insn))) {
3496 WARN_INSN(insn, "call to %s() with UACCESS enabled", call_dest_name(insn));
3497 return 1;
3498 }
3499
3500 if (state->df) {
3501 WARN_INSN(insn, "call to %s() with DF set", call_dest_name(insn));
3502 return 1;
3503 }
3504
3505 return 0;
3506 }
3507
3508 static int validate_sibling_call(struct objtool_file *file,
3509 struct instruction *insn,
3510 struct insn_state *state)
3511 {
3512 if (insn_func(insn) && has_modified_stack_frame(insn, state)) {
3513 WARN_INSN(insn, "sibling call from callable instruction with modified stack frame");
3514 return 1;
3515 }
3516
3517 return validate_call(file, insn, state);
3518 }
3519
3520 static int validate_return(struct symbol *func, struct instruction *insn, struct insn_state *state)
3521 {
3522 if (state->noinstr && state->instr > 0) {
3523 WARN_INSN(insn, "return with instrumentation enabled");
3524 return 1;
3525 }
3526
3527 if (state->uaccess && !func_uaccess_safe(func)) {
3528 WARN_INSN(insn, "return with UACCESS enabled");
3529 return 1;
3530 }
3531
3532 if (!state->uaccess && func_uaccess_safe(func)) {
3533 WARN_INSN(insn, "return with UACCESS disabled from a UACCESS-safe function");
3534 return 1;
3535 }
3536
3537 if (state->df) {
3538 WARN_INSN(insn, "return with DF set");
3539 return 1;
3540 }
3541
3542 if (func && has_modified_stack_frame(insn, state)) {
3543 WARN_INSN(insn, "return with modified stack frame");
3544 return 1;
3545 }
3546
3547 if (state->cfi.bp_scratch) {
3548 WARN_INSN(insn, "BP used as a scratch register");
3549 return 1;
3550 }
3551
3552 return 0;
3553 }
3554
3555 static struct instruction *next_insn_to_validate(struct objtool_file *file,
3556 struct instruction *insn)
3557 {
3558 struct alt_group *alt_group = insn->alt_group;
3559
3560 /*
3561 * Simulate the fact that alternatives are patched in-place. When the
3562 * end of a replacement alt_group is reached, redirect objtool flow to
3563 * the end of the original alt_group.
3564 *
3565 * insn->alts->insn -> alt_group->first_insn
3566 * ...
3567 * alt_group->last_insn
3568 * [alt_group->nop] -> next(orig_group->last_insn)
3569 */
3570 if (alt_group) {
3571 if (alt_group->nop) {
3572 /* ->nop implies ->orig_group */
3573 if (insn == alt_group->last_insn)
3574 return alt_group->nop;
3575 if (insn == alt_group->nop)
3576 goto next_orig;
3577 }
3578 if (insn == alt_group->last_insn && alt_group->orig_group)
3579 goto next_orig;
3580 }
3581
3582 return next_insn_same_sec(file, insn);
3583
3584 next_orig:
3585 return next_insn_same_sec(file, alt_group->orig_group->last_insn);
3586 }
3587
3588 /*
3589 * Follow the branch starting at the given instruction, and recursively follow
3590 * any other branches (jumps). Meanwhile, track the frame pointer state at
3591 * each instruction and validate all the rules described in
3592 * tools/objtool/Documentation/objtool.txt.
3593 */
3594 static int validate_branch(struct objtool_file *file, struct symbol *func,
3595 struct instruction *insn, struct insn_state state)
3596 {
3597 struct alternative *alt;
3598 struct instruction *next_insn, *prev_insn = NULL;
3599 struct section *sec;
3600 u8 visited;
3601 int ret;
3602
3603 sec = insn->sec;
3604
3605 while (1) {
3606 next_insn = next_insn_to_validate(file, insn);
3607
3608 if (func && insn_func(insn) && func != insn_func(insn)->pfunc) {
3609 /* Ignore KCFI type preambles, which always fall through */
3610 if (!strncmp(func->name, "__cfi_", 6) ||
3611 !strncmp(func->name, "__pfx_", 6))
3612 return 0;
3613
3614 WARN("%s() falls through to next function %s()",
3615 func->name, insn_func(insn)->name);
3616 return 1;
3617 }
3618
3619 if (func && insn->ignore) {
3620 WARN_INSN(insn, "BUG: why am I validating an ignored function?");
3621 return 1;
3622 }
3623
3624 visited = VISITED_BRANCH << state.uaccess;
3625 if (insn->visited & VISITED_BRANCH_MASK) {
3626 if (!insn->hint && !insn_cfi_match(insn, &state.cfi))
3627 return 1;
3628
3629 if (insn->visited & visited)
3630 return 0;
3631 } else {
3632 nr_insns_visited++;
3633 }
3634
3635 if (state.noinstr)
3636 state.instr += insn->instr;
3637
3638 if (insn->hint) {
3639 if (insn->restore) {
3640 struct instruction *save_insn, *i;
3641
3642 i = insn;
3643 save_insn = NULL;
3644
3645 sym_for_each_insn_continue_reverse(file, func, i) {
3646 if (i->save) {
3647 save_insn = i;
3648 break;
3649 }
3650 }
3651
3652 if (!save_insn) {
3653 WARN_INSN(insn, "no corresponding CFI save for CFI restore");
3654 return 1;
3655 }
3656
3657 if (!save_insn->visited) {
3658 /*
3659 * If the restore hint insn is at the
3660 * beginning of a basic block and was
3661 * branched to from elsewhere, and the
3662 * save insn hasn't been visited yet,
3663 * defer following this branch for now.
3664 * It will be seen later via the
3665 * straight-line path.
3666 */
3667 if (!prev_insn)
3668 return 0;
3669
3670 WARN_INSN(insn, "objtool isn't smart enough to handle this CFI save/restore combo");
3671 return 1;
3672 }
3673
3674 insn->cfi = save_insn->cfi;
3675 nr_cfi_reused++;
3676 }
3677
3678 state.cfi = *insn->cfi;
3679 } else {
3680 /* XXX track if we actually changed state.cfi */
3681
3682 if (prev_insn && !cficmp(prev_insn->cfi, &state.cfi)) {
3683 insn->cfi = prev_insn->cfi;
3684 nr_cfi_reused++;
3685 } else {
3686 insn->cfi = cfi_hash_find_or_add(&state.cfi);
3687 }
3688 }
3689
3690 insn->visited |= visited;
3691
3692 if (propagate_alt_cfi(file, insn))
3693 return 1;
3694
3695 if (!insn->ignore_alts && insn->alts) {
3696 bool skip_orig = false;
3697
3698 for (alt = insn->alts; alt; alt = alt->next) {
3699 if (alt->skip_orig)
3700 skip_orig = true;
3701
3702 ret = validate_branch(file, func, alt->insn, state);
3703 if (ret) {
3704 BT_INSN(insn, "(alt)");
3705 return ret;
3706 }
3707 }
3708
3709 if (skip_orig)
3710 return 0;
3711 }
3712
3713 if (handle_insn_ops(insn, next_insn, &state))
3714 return 1;
3715
3716 switch (insn->type) {
3717
3718 case INSN_RETURN:
3719 return validate_return(func, insn, &state);
3720
3721 case INSN_CALL:
3722 case INSN_CALL_DYNAMIC:
3723 ret = validate_call(file, insn, &state);
3724 if (ret)
3725 return ret;
3726
3727 if (opts.stackval && func && !is_special_call(insn) &&
3728 !has_valid_stack_frame(&state)) {
3729 WARN_INSN(insn, "call without frame pointer save/setup");
3730 return 1;
3731 }
3732
3733 if (insn->dead_end)
3734 return 0;
3735
3736 break;
3737
3738 case INSN_JUMP_CONDITIONAL:
3739 case INSN_JUMP_UNCONDITIONAL:
3740 if (is_sibling_call(insn)) {
3741 ret = validate_sibling_call(file, insn, &state);
3742 if (ret)
3743 return ret;
3744
3745 } else if (insn->jump_dest) {
3746 ret = validate_branch(file, func,
3747 insn->jump_dest, state);
3748 if (ret) {
3749 BT_INSN(insn, "(branch)");
3750 return ret;
3751 }
3752 }
3753
3754 if (insn->type == INSN_JUMP_UNCONDITIONAL)
3755 return 0;
3756
3757 break;
3758
3759 case INSN_JUMP_DYNAMIC:
3760 case INSN_JUMP_DYNAMIC_CONDITIONAL:
3761 if (is_sibling_call(insn)) {
3762 ret = validate_sibling_call(file, insn, &state);
3763 if (ret)
3764 return ret;
3765 }
3766
3767 if (insn->type == INSN_JUMP_DYNAMIC)
3768 return 0;
3769
3770 break;
3771
3772 case INSN_CONTEXT_SWITCH:
3773 if (func && (!next_insn || !next_insn->hint)) {
3774 WARN_INSN(insn, "unsupported instruction in callable function");
3775 return 1;
3776 }
3777 return 0;
3778
3779 case INSN_STAC:
3780 if (state.uaccess) {
3781 WARN_INSN(insn, "recursive UACCESS enable");
3782 return 1;
3783 }
3784
3785 state.uaccess = true;
3786 break;
3787
3788 case INSN_CLAC:
3789 if (!state.uaccess && func) {
3790 WARN_INSN(insn, "redundant UACCESS disable");
3791 return 1;
3792 }
3793
3794 if (func_uaccess_safe(func) && !state.uaccess_stack) {
3795 WARN_INSN(insn, "UACCESS-safe disables UACCESS");
3796 return 1;
3797 }
3798
3799 state.uaccess = false;
3800 break;
3801
3802 case INSN_STD:
3803 if (state.df) {
3804 WARN_INSN(insn, "recursive STD");
3805 return 1;
3806 }
3807
3808 state.df = true;
3809 break;
3810
3811 case INSN_CLD:
3812 if (!state.df && func) {
3813 WARN_INSN(insn, "redundant CLD");
3814 return 1;
3815 }
3816
3817 state.df = false;
3818 break;
3819
3820 default:
3821 break;
3822 }
3823
3824 if (insn->dead_end)
3825 return 0;
3826
3827 if (!next_insn) {
3828 if (state.cfi.cfa.base == CFI_UNDEFINED)
3829 return 0;
3830 WARN("%s: unexpected end of section", sec->name);
3831 return 1;
3832 }
3833
3834 prev_insn = insn;
3835 insn = next_insn;
3836 }
3837
3838 return 0;
3839 }
3840
3841 static int validate_unwind_hint(struct objtool_file *file,
3842 struct instruction *insn,
3843 struct insn_state *state)
3844 {
3845 if (insn->hint && !insn->visited && !insn->ignore) {
3846 int ret = validate_branch(file, insn_func(insn), insn, *state);
3847 if (ret)
3848 BT_INSN(insn, "<=== (hint)");
3849 return ret;
3850 }
3851
3852 return 0;
3853 }
3854
3855 static int validate_unwind_hints(struct objtool_file *file, struct section *sec)
3856 {
3857 struct instruction *insn;
3858 struct insn_state state;
3859 int warnings = 0;
3860
3861 if (!file->hints)
3862 return 0;
3863
3864 init_insn_state(file, &state, sec);
3865
3866 if (sec) {
3867 sec_for_each_insn(file, sec, insn)
3868 warnings += validate_unwind_hint(file, insn, &state);
3869 } else {
3870 for_each_insn(file, insn)
3871 warnings += validate_unwind_hint(file, insn, &state);
3872 }
3873
3874 return warnings;
3875 }
3876
3877 /*
3878 * Validate rethunk entry constraint: must untrain RET before the first RET.
3879 *
3880 * Follow every branch (intra-function) and ensure VALIDATE_UNRET_END comes
3881 * before an actual RET instruction.
3882 */
3883 static int validate_unret(struct objtool_file *file, struct instruction *insn)
3884 {
3885 struct instruction *next, *dest;
3886 int ret;
3887
3888 for (;;) {
3889 next = next_insn_to_validate(file, insn);
3890
3891 if (insn->visited & VISITED_UNRET)
3892 return 0;
3893
3894 insn->visited |= VISITED_UNRET;
3895
3896 if (!insn->ignore_alts && insn->alts) {
3897 struct alternative *alt;
3898 bool skip_orig = false;
3899
3900 for (alt = insn->alts; alt; alt = alt->next) {
3901 if (alt->skip_orig)
3902 skip_orig = true;
3903
3904 ret = validate_unret(file, alt->insn);
3905 if (ret) {
3906 BT_INSN(insn, "(alt)");
3907 return ret;
3908 }
3909 }
3910
3911 if (skip_orig)
3912 return 0;
3913 }
3914
3915 switch (insn->type) {
3916
3917 case INSN_CALL_DYNAMIC:
3918 case INSN_JUMP_DYNAMIC:
3919 case INSN_JUMP_DYNAMIC_CONDITIONAL:
3920 WARN_INSN(insn, "early indirect call");
3921 return 1;
3922
3923 case INSN_JUMP_UNCONDITIONAL:
3924 case INSN_JUMP_CONDITIONAL:
3925 if (!is_sibling_call(insn)) {
3926 if (!insn->jump_dest) {
3927 WARN_INSN(insn, "unresolved jump target after linking?!?");
3928 return -1;
3929 }
3930 ret = validate_unret(file, insn->jump_dest);
3931 if (ret) {
3932 BT_INSN(insn, "(branch%s)",
3933 insn->type == INSN_JUMP_CONDITIONAL ? "-cond" : "");
3934 return ret;
3935 }
3936
3937 if (insn->type == INSN_JUMP_UNCONDITIONAL)
3938 return 0;
3939
3940 break;
3941 }
3942
3943 /* fallthrough */
3944 case INSN_CALL:
3945 dest = find_insn(file, insn_call_dest(insn)->sec,
3946 insn_call_dest(insn)->offset);
3947 if (!dest) {
3948 WARN("Unresolved function after linking!?: %s",
3949 insn_call_dest(insn)->name);
3950 return -1;
3951 }
3952
3953 ret = validate_unret(file, dest);
3954 if (ret) {
3955 BT_INSN(insn, "(call)");
3956 return ret;
3957 }
3958 /*
3959 * If a call returns without error, it must have seen UNTRAIN_RET.
3960 * Therefore any non-error return is a success.
3961 */
3962 return 0;
3963
3964 case INSN_RETURN:
3965 WARN_INSN(insn, "RET before UNTRAIN");
3966 return 1;
3967
3968 case INSN_NOP:
3969 if (insn->retpoline_safe)
3970 return 0;
3971 break;
3972
3973 default:
3974 break;
3975 }
3976
3977 if (!next) {
3978 WARN_INSN(insn, "teh end!");
3979 return -1;
3980 }
3981 insn = next;
3982 }
3983
3984 return 0;
3985 }
3986
3987 /*
3988 * Validate that all branches starting at VALIDATE_UNRET_BEGIN encounter
3989 * VALIDATE_UNRET_END before RET.
3990 */
3991 static int validate_unrets(struct objtool_file *file)
3992 {
3993 struct instruction *insn;
3994 int ret, warnings = 0;
3995
3996 for_each_insn(file, insn) {
3997 if (!insn->unret)
3998 continue;
3999
4000 ret = validate_unret(file, insn);
4001 if (ret < 0) {
4002 WARN_INSN(insn, "Failed UNRET validation");
4003 return ret;
4004 }
4005 warnings += ret;
4006 }
4007
4008 return warnings;
4009 }
4010
4011 static int validate_retpoline(struct objtool_file *file)
4012 {
4013 struct instruction *insn;
4014 int warnings = 0;
4015
4016 for_each_insn(file, insn) {
4017 if (insn->type != INSN_JUMP_DYNAMIC &&
4018 insn->type != INSN_CALL_DYNAMIC &&
4019 insn->type != INSN_RETURN)
4020 continue;
4021
4022 if (insn->retpoline_safe)
4023 continue;
4024
4025 if (insn->sec->init)
4026 continue;
4027
4028 if (insn->type == INSN_RETURN) {
4029 if (opts.rethunk) {
4030 WARN_INSN(insn, "'naked' return found in MITIGATION_RETHUNK build");
4031 } else
4032 continue;
4033 } else {
4034 WARN_INSN(insn, "indirect %s found in MITIGATION_RETPOLINE build",
4035 insn->type == INSN_JUMP_DYNAMIC ? "jump" : "call");
4036 }
4037
4038 warnings++;
4039 }
4040
4041 return warnings;
4042 }
4043
4044 static bool is_kasan_insn(struct instruction *insn)
4045 {
4046 return (insn->type == INSN_CALL &&
4047 !strcmp(insn_call_dest(insn)->name, "__asan_handle_no_return"));
4048 }
4049
4050 static bool is_ubsan_insn(struct instruction *insn)
4051 {
4052 return (insn->type == INSN_CALL &&
4053 !strcmp(insn_call_dest(insn)->name,
4054 "__ubsan_handle_builtin_unreachable"));
4055 }
4056
4057 static bool ignore_unreachable_insn(struct objtool_file *file, struct instruction *insn)
4058 {
4059 int i;
4060 struct instruction *prev_insn;
4061
4062 if (insn->ignore || insn->type == INSN_NOP || insn->type == INSN_TRAP)
4063 return true;
4064
4065 /*
4066 * Ignore alternative replacement instructions. This can happen
4067 * when a whitelisted function uses one of the ALTERNATIVE macros.
4068 */
4069 if (!strcmp(insn->sec->name, ".altinstr_replacement") ||
4070 !strcmp(insn->sec->name, ".altinstr_aux"))
4071 return true;
4072
4073 /*
4074 * Whole archive runs might encounter dead code from weak symbols.
4075 * This is where the linker will have dropped the weak symbol in
4076 * favour of a regular symbol, but leaves the code in place.
4077 *
4078 * In this case we'll find a piece of code (whole function) that is not
4079 * covered by a !section symbol. Ignore them.
4080 */
4081 if (opts.link && !insn_func(insn)) {
4082 int size = find_symbol_hole_containing(insn->sec, insn->offset);
4083 unsigned long end = insn->offset + size;
4084
4085 if (!size) /* not a hole */
4086 return false;
4087
4088 if (size < 0) /* hole until the end */
4089 return true;
4090
4091 sec_for_each_insn_continue(file, insn) {
4092 /*
4093 * If we reach a visited instruction at or before the
4094 * end of the hole, ignore the unreachable.
4095 */
4096 if (insn->visited)
4097 return true;
4098
4099 if (insn->offset >= end)
4100 break;
4101
4102 /*
4103 * If this hole jumps to a .cold function, mark it ignore too.
4104 */
4105 if (insn->jump_dest && insn_func(insn->jump_dest) &&
4106 strstr(insn_func(insn->jump_dest)->name, ".cold")) {
4107 struct instruction *dest = insn->jump_dest;
4108 func_for_each_insn(file, insn_func(dest), dest)
4109 dest->ignore = true;
4110 }
4111 }
4112
4113 return false;
4114 }
4115
4116 if (!insn_func(insn))
4117 return false;
4118
4119 if (insn_func(insn)->static_call_tramp)
4120 return true;
4121
4122 /*
4123 * CONFIG_UBSAN_TRAP inserts a UD2 when it sees
4124 * __builtin_unreachable(). The BUG() macro has an unreachable() after
4125 * the UD2, which causes GCC's undefined trap logic to emit another UD2
4126 * (or occasionally a JMP to UD2).
4127 *
4128 * It may also insert a UD2 after calling a __noreturn function.
4129 */
4130 prev_insn = prev_insn_same_sec(file, insn);
4131 if (prev_insn->dead_end &&
4132 (insn->type == INSN_BUG ||
4133 (insn->type == INSN_JUMP_UNCONDITIONAL &&
4134 insn->jump_dest && insn->jump_dest->type == INSN_BUG)))
4135 return true;
4136
4137 /*
4138 * Check if this (or a subsequent) instruction is related to
4139 * CONFIG_UBSAN or CONFIG_KASAN.
4140 *
4141 * End the search at 5 instructions to avoid going into the weeds.
4142 */
4143 for (i = 0; i < 5; i++) {
4144
4145 if (is_kasan_insn(insn) || is_ubsan_insn(insn))
4146 return true;
4147
4148 if (insn->type == INSN_JUMP_UNCONDITIONAL) {
4149 if (insn->jump_dest &&
4150 insn_func(insn->jump_dest) == insn_func(insn)) {
4151 insn = insn->jump_dest;
4152 continue;
4153 }
4154
4155 break;
4156 }
4157
4158 if (insn->offset + insn->len >= insn_func(insn)->offset + insn_func(insn)->len)
4159 break;
4160
4161 insn = next_insn_same_sec(file, insn);
4162 }
4163
4164 return false;
4165 }
4166
4167 static int add_prefix_symbol(struct objtool_file *file, struct symbol *func)
4168 {
4169 struct instruction *insn, *prev;
4170 struct cfi_state *cfi;
4171
4172 insn = find_insn(file, func->sec, func->offset);
4173 if (!insn)
4174 return -1;
4175
4176 for (prev = prev_insn_same_sec(file, insn);
4177 prev;
4178 prev = prev_insn_same_sec(file, prev)) {
4179 u64 offset;
4180
4181 if (prev->type != INSN_NOP)
4182 return -1;
4183
4184 offset = func->offset - prev->offset;
4185
4186 if (offset > opts.prefix)
4187 return -1;
4188
4189 if (offset < opts.prefix)
4190 continue;
4191
4192 elf_create_prefix_symbol(file->elf, func, opts.prefix);
4193 break;
4194 }
4195
4196 if (!prev)
4197 return -1;
4198
4199 if (!insn->cfi) {
4200 /*
4201 * This can happen if stack validation isn't enabled or the
4202 * function is annotated with STACK_FRAME_NON_STANDARD.
4203 */
4204 return 0;
4205 }
4206
4207 /* Propagate insn->cfi to the prefix code */
4208 cfi = cfi_hash_find_or_add(insn->cfi);
4209 for (; prev != insn; prev = next_insn_same_sec(file, prev))
4210 prev->cfi = cfi;
4211
4212 return 0;
4213 }
4214
4215 static int add_prefix_symbols(struct objtool_file *file)
4216 {
4217 struct section *sec;
4218 struct symbol *func;
4219
4220 for_each_sec(file, sec) {
4221 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
4222 continue;
4223
4224 sec_for_each_sym(sec, func) {
4225 if (func->type != STT_FUNC)
4226 continue;
4227
4228 add_prefix_symbol(file, func);
4229 }
4230 }
4231
4232 return 0;
4233 }
4234
4235 static int validate_symbol(struct objtool_file *file, struct section *sec,
4236 struct symbol *sym, struct insn_state *state)
4237 {
4238 struct instruction *insn;
4239 int ret;
4240
4241 if (!sym->len) {
4242 WARN("%s() is missing an ELF size annotation", sym->name);
4243 return 1;
4244 }
4245
4246 if (sym->pfunc != sym || sym->alias != sym)
4247 return 0;
4248
4249 insn = find_insn(file, sec, sym->offset);
4250 if (!insn || insn->ignore || insn->visited)
4251 return 0;
4252
4253 state->uaccess = sym->uaccess_safe;
4254
4255 ret = validate_branch(file, insn_func(insn), insn, *state);
4256 if (ret)
4257 BT_INSN(insn, "<=== (sym)");
4258 return ret;
4259 }
4260
4261 static int validate_section(struct objtool_file *file, struct section *sec)
4262 {
4263 struct insn_state state;
4264 struct symbol *func;
4265 int warnings = 0;
4266
4267 sec_for_each_sym(sec, func) {
4268 if (func->type != STT_FUNC)
4269 continue;
4270
4271 init_insn_state(file, &state, sec);
4272 set_func_state(&state.cfi);
4273
4274 warnings += validate_symbol(file, sec, func, &state);
4275 }
4276
4277 return warnings;
4278 }
4279
4280 static int validate_noinstr_sections(struct objtool_file *file)
4281 {
4282 struct section *sec;
4283 int warnings = 0;
4284
4285 sec = find_section_by_name(file->elf, ".noinstr.text");
4286 if (sec) {
4287 warnings += validate_section(file, sec);
4288 warnings += validate_unwind_hints(file, sec);
4289 }
4290
4291 sec = find_section_by_name(file->elf, ".entry.text");
4292 if (sec) {
4293 warnings += validate_section(file, sec);
4294 warnings += validate_unwind_hints(file, sec);
4295 }
4296
4297 sec = find_section_by_name(file->elf, ".cpuidle.text");
4298 if (sec) {
4299 warnings += validate_section(file, sec);
4300 warnings += validate_unwind_hints(file, sec);
4301 }
4302
4303 return warnings;
4304 }
4305
4306 static int validate_functions(struct objtool_file *file)
4307 {
4308 struct section *sec;
4309 int warnings = 0;
4310
4311 for_each_sec(file, sec) {
4312 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
4313 continue;
4314
4315 warnings += validate_section(file, sec);
4316 }
4317
4318 return warnings;
4319 }
4320
4321 static void mark_endbr_used(struct instruction *insn)
4322 {
4323 if (!list_empty(&insn->call_node))
4324 list_del_init(&insn->call_node);
4325 }
4326
4327 static bool noendbr_range(struct objtool_file *file, struct instruction *insn)
4328 {
4329 struct symbol *sym = find_symbol_containing(insn->sec, insn->offset-1);
4330 struct instruction *first;
4331
4332 if (!sym)
4333 return false;
4334
4335 first = find_insn(file, sym->sec, sym->offset);
4336 if (!first)
4337 return false;
4338
4339 if (first->type != INSN_ENDBR && !first->noendbr)
4340 return false;
4341
4342 return insn->offset == sym->offset + sym->len;
4343 }
4344
4345 static int validate_ibt_insn(struct objtool_file *file, struct instruction *insn)
4346 {
4347 struct instruction *dest;
4348 struct reloc *reloc;
4349 unsigned long off;
4350 int warnings = 0;
4351
4352 /*
4353 * Looking for function pointer load relocations. Ignore
4354 * direct/indirect branches:
4355 */
4356 switch (insn->type) {
4357 case INSN_CALL:
4358 case INSN_CALL_DYNAMIC:
4359 case INSN_JUMP_CONDITIONAL:
4360 case INSN_JUMP_UNCONDITIONAL:
4361 case INSN_JUMP_DYNAMIC:
4362 case INSN_JUMP_DYNAMIC_CONDITIONAL:
4363 case INSN_RETURN:
4364 case INSN_NOP:
4365 return 0;
4366 default:
4367 break;
4368 }
4369
4370 for (reloc = insn_reloc(file, insn);
4371 reloc;
4372 reloc = find_reloc_by_dest_range(file->elf, insn->sec,
4373 reloc_offset(reloc) + 1,
4374 (insn->offset + insn->len) - (reloc_offset(reloc) + 1))) {
4375
4376 /*
4377 * static_call_update() references the trampoline, which
4378 * doesn't have (or need) ENDBR. Skip warning in that case.
4379 */
4380 if (reloc->sym->static_call_tramp)
4381 continue;
4382
4383 off = reloc->sym->offset;
4384 if (reloc_type(reloc) == R_X86_64_PC32 ||
4385 reloc_type(reloc) == R_X86_64_PLT32)
4386 off += arch_dest_reloc_offset(reloc_addend(reloc));
4387 else
4388 off += reloc_addend(reloc);
4389
4390 dest = find_insn(file, reloc->sym->sec, off);
4391 if (!dest)
4392 continue;
4393
4394 if (dest->type == INSN_ENDBR) {
4395 mark_endbr_used(dest);
4396 continue;
4397 }
4398
4399 if (insn_func(dest) && insn_func(insn) &&
4400 insn_func(dest)->pfunc == insn_func(insn)->pfunc) {
4401 /*
4402 * Anything from->to self is either _THIS_IP_ or
4403 * IRET-to-self.
4404 *
4405 * There is no sane way to annotate _THIS_IP_ since the
4406 * compiler treats the relocation as a constant and is
4407 * happy to fold in offsets, skewing any annotation we
4408 * do, leading to vast amounts of false-positives.
4409 *
4410 * There's also compiler generated _THIS_IP_ through
4411 * KCOV and such which we have no hope of annotating.
4412 *
4413 * As such, blanket accept self-references without
4414 * issue.
4415 */
4416 continue;
4417 }
4418
4419 /*
4420 * Accept anything ANNOTATE_NOENDBR.
4421 */
4422 if (dest->noendbr)
4423 continue;
4424
4425 /*
4426 * Accept if this is the instruction after a symbol
4427 * that is (no)endbr -- typical code-range usage.
4428 */
4429 if (noendbr_range(file, dest))
4430 continue;
4431
4432 WARN_INSN(insn, "relocation to !ENDBR: %s", offstr(dest->sec, dest->offset));
4433
4434 warnings++;
4435 }
4436
4437 return warnings;
4438 }
4439
4440 static int validate_ibt_data_reloc(struct objtool_file *file,
4441 struct reloc *reloc)
4442 {
4443 struct instruction *dest;
4444
4445 dest = find_insn(file, reloc->sym->sec,
4446 reloc->sym->offset + reloc_addend(reloc));
4447 if (!dest)
4448 return 0;
4449
4450 if (dest->type == INSN_ENDBR) {
4451 mark_endbr_used(dest);
4452 return 0;
4453 }
4454
4455 if (dest->noendbr)
4456 return 0;
4457
4458 WARN_FUNC("data relocation to !ENDBR: %s",
4459 reloc->sec->base, reloc_offset(reloc),
4460 offstr(dest->sec, dest->offset));
4461
4462 return 1;
4463 }
4464
4465 /*
4466 * Validate IBT rules and remove used ENDBR instructions from the seal list.
4467 * Unused ENDBR instructions will be annotated for sealing (i.e., replaced with
4468 * NOPs) later, in create_ibt_endbr_seal_sections().
4469 */
4470 static int validate_ibt(struct objtool_file *file)
4471 {
4472 struct section *sec;
4473 struct reloc *reloc;
4474 struct instruction *insn;
4475 int warnings = 0;
4476
4477 for_each_insn(file, insn)
4478 warnings += validate_ibt_insn(file, insn);
4479
4480 for_each_sec(file, sec) {
4481
4482 /* Already done by validate_ibt_insn() */
4483 if (sec->sh.sh_flags & SHF_EXECINSTR)
4484 continue;
4485
4486 if (!sec->rsec)
4487 continue;
4488
4489 /*
4490 * These sections can reference text addresses, but not with
4491 * the intent to indirect branch to them.
4492 */
4493 if ((!strncmp(sec->name, ".discard", 8) &&
4494 strcmp(sec->name, ".discard.ibt_endbr_noseal")) ||
4495 !strncmp(sec->name, ".debug", 6) ||
4496 !strcmp(sec->name, ".altinstructions") ||
4497 !strcmp(sec->name, ".ibt_endbr_seal") ||
4498 !strcmp(sec->name, ".orc_unwind_ip") ||
4499 !strcmp(sec->name, ".parainstructions") ||
4500 !strcmp(sec->name, ".retpoline_sites") ||
4501 !strcmp(sec->name, ".smp_locks") ||
4502 !strcmp(sec->name, ".static_call_sites") ||
4503 !strcmp(sec->name, "_error_injection_whitelist") ||
4504 !strcmp(sec->name, "_kprobe_blacklist") ||
4505 !strcmp(sec->name, "__bug_table") ||
4506 !strcmp(sec->name, "__ex_table") ||
4507 !strcmp(sec->name, "__jump_table") ||
4508 !strcmp(sec->name, "__mcount_loc") ||
4509 !strcmp(sec->name, ".kcfi_traps") ||
4510 strstr(sec->name, "__patchable_function_entries"))
4511 continue;
4512
4513 for_each_reloc(sec->rsec, reloc)
4514 warnings += validate_ibt_data_reloc(file, reloc);
4515 }
4516
4517 return warnings;
4518 }
4519
4520 static int validate_sls(struct objtool_file *file)
4521 {
4522 struct instruction *insn, *next_insn;
4523 int warnings = 0;
4524
4525 for_each_insn(file, insn) {
4526 next_insn = next_insn_same_sec(file, insn);
4527
4528 if (insn->retpoline_safe)
4529 continue;
4530
4531 switch (insn->type) {
4532 case INSN_RETURN:
4533 if (!next_insn || next_insn->type != INSN_TRAP) {
4534 WARN_INSN(insn, "missing int3 after ret");
4535 warnings++;
4536 }
4537
4538 break;
4539 case INSN_JUMP_DYNAMIC:
4540 if (!next_insn || next_insn->type != INSN_TRAP) {
4541 WARN_INSN(insn, "missing int3 after indirect jump");
4542 warnings++;
4543 }
4544 break;
4545 default:
4546 break;
4547 }
4548 }
4549
4550 return warnings;
4551 }
4552
4553 static bool ignore_noreturn_call(struct instruction *insn)
4554 {
4555 struct symbol *call_dest = insn_call_dest(insn);
4556
4557 /*
4558 * FIXME: hack, we need a real noreturn solution
4559 *
4560 * Problem is, exc_double_fault() may or may not return, depending on
4561 * whether CONFIG_X86_ESPFIX64 is set. But objtool has no visibility
4562 * to the kernel config.
4563 *
4564 * Other potential ways to fix it:
4565 *
4566 * - have compiler communicate __noreturn functions somehow
4567 * - remove CONFIG_X86_ESPFIX64
4568 * - read the .config file
4569 * - add a cmdline option
4570 * - create a generic objtool annotation format (vs a bunch of custom
4571 * formats) and annotate it
4572 */
4573 if (!strcmp(call_dest->name, "exc_double_fault")) {
4574 /* prevent further unreachable warnings for the caller */
4575 insn->sym->warned = 1;
4576 return true;
4577 }
4578
4579 return false;
4580 }
4581
4582 static int validate_reachable_instructions(struct objtool_file *file)
4583 {
4584 struct instruction *insn, *prev_insn;
4585 struct symbol *call_dest;
4586 int warnings = 0;
4587
4588 if (file->ignore_unreachables)
4589 return 0;
4590
4591 for_each_insn(file, insn) {
4592 if (insn->visited || ignore_unreachable_insn(file, insn))
4593 continue;
4594
4595 prev_insn = prev_insn_same_sec(file, insn);
4596 if (prev_insn && prev_insn->dead_end) {
4597 call_dest = insn_call_dest(prev_insn);
4598 if (call_dest && !ignore_noreturn_call(prev_insn)) {
4599 WARN_INSN(insn, "%s() is missing a __noreturn annotation",
4600 call_dest->name);
4601 warnings++;
4602 continue;
4603 }
4604 }
4605
4606 WARN_INSN(insn, "unreachable instruction");
4607 warnings++;
4608 }
4609
4610 return warnings;
4611 }
4612
4613 /* 'funcs' is a space-separated list of function names */
4614 static int disas_funcs(const char *funcs)
4615 {
4616 const char *objdump_str, *cross_compile;
4617 int size, ret;
4618 char *cmd;
4619
4620 cross_compile = getenv("CROSS_COMPILE");
4621
4622 objdump_str = "%sobjdump -wdr %s | gawk -M -v _funcs='%s' '"
4623 "BEGIN { split(_funcs, funcs); }"
4624 "/^$/ { func_match = 0; }"
4625 "/<.*>:/ { "
4626 "f = gensub(/.*<(.*)>:/, \"\\\\1\", 1);"
4627 "for (i in funcs) {"
4628 "if (funcs[i] == f) {"
4629 "func_match = 1;"
4630 "base = strtonum(\"0x\" $1);"
4631 "break;"
4632 "}"
4633 "}"
4634 "}"
4635 "{"
4636 "if (func_match) {"
4637 "addr = strtonum(\"0x\" $1);"
4638 "printf(\"%%04x \", addr - base);"
4639 "print;"
4640 "}"
4641 "}' 1>&2";
4642
4643 /* fake snprintf() to calculate the size */
4644 size = snprintf(NULL, 0, objdump_str, cross_compile, objname, funcs) + 1;
4645 if (size <= 0) {
4646 WARN("objdump string size calculation failed");
4647 return -1;
4648 }
4649
4650 cmd = malloc(size);
4651
4652 /* real snprintf() */
4653 snprintf(cmd, size, objdump_str, cross_compile, objname, funcs);
4654 ret = system(cmd);
4655 if (ret) {
4656 WARN("disassembly failed: %d", ret);
4657 return -1;
4658 }
4659
4660 return 0;
4661 }
4662
4663 static int disas_warned_funcs(struct objtool_file *file)
4664 {
4665 struct symbol *sym;
4666 char *funcs = NULL, *tmp;
4667
4668 for_each_sym(file, sym) {
4669 if (sym->warned) {
4670 if (!funcs) {
4671 funcs = malloc(strlen(sym->name) + 1);
4672 strcpy(funcs, sym->name);
4673 } else {
4674 tmp = malloc(strlen(funcs) + strlen(sym->name) + 2);
4675 sprintf(tmp, "%s %s", funcs, sym->name);
4676 free(funcs);
4677 funcs = tmp;
4678 }
4679 }
4680 }
4681
4682 if (funcs)
4683 disas_funcs(funcs);
4684
4685 return 0;
4686 }
4687
4688 struct insn_chunk {
4689 void *addr;
4690 struct insn_chunk *next;
4691 };
4692
4693 /*
4694 * Reduce peak RSS usage by freeing insns memory before writing the ELF file,
4695 * which can trigger more allocations for .debug_* sections whose data hasn't
4696 * been read yet.
4697 */
4698 static void free_insns(struct objtool_file *file)
4699 {
4700 struct instruction *insn;
4701 struct insn_chunk *chunks = NULL, *chunk;
4702
4703 for_each_insn(file, insn) {
4704 if (!insn->idx) {
4705 chunk = malloc(sizeof(*chunk));
4706 chunk->addr = insn;
4707 chunk->next = chunks;
4708 chunks = chunk;
4709 }
4710 }
4711
4712 for (chunk = chunks; chunk; chunk = chunk->next)
4713 free(chunk->addr);
4714 }
4715
4716 int check(struct objtool_file *file)
4717 {
4718 int ret, warnings = 0;
4719
4720 arch_initial_func_cfi_state(&initial_func_cfi);
4721 init_cfi_state(&init_cfi);
4722 init_cfi_state(&func_cfi);
4723 set_func_state(&func_cfi);
4724 init_cfi_state(&force_undefined_cfi);
4725 force_undefined_cfi.force_undefined = true;
4726
4727 if (!cfi_hash_alloc(1UL << (file->elf->symbol_bits - 3)))
4728 goto out;
4729
4730 cfi_hash_add(&init_cfi);
4731 cfi_hash_add(&func_cfi);
4732
4733 ret = decode_sections(file);
4734 if (ret < 0)
4735 goto out;
4736
4737 warnings += ret;
4738
4739 if (!nr_insns)
4740 goto out;
4741
4742 if (opts.retpoline) {
4743 ret = validate_retpoline(file);
4744 if (ret < 0)
4745 return ret;
4746 warnings += ret;
4747 }
4748
4749 if (opts.stackval || opts.orc || opts.uaccess) {
4750 ret = validate_functions(file);
4751 if (ret < 0)
4752 goto out;
4753 warnings += ret;
4754
4755 ret = validate_unwind_hints(file, NULL);
4756 if (ret < 0)
4757 goto out;
4758 warnings += ret;
4759
4760 if (!warnings) {
4761 ret = validate_reachable_instructions(file);
4762 if (ret < 0)
4763 goto out;
4764 warnings += ret;
4765 }
4766
4767 } else if (opts.noinstr) {
4768 ret = validate_noinstr_sections(file);
4769 if (ret < 0)
4770 goto out;
4771 warnings += ret;
4772 }
4773
4774 if (opts.unret) {
4775 /*
4776 * Must be after validate_branch() and friends, it plays
4777 * further games with insn->visited.
4778 */
4779 ret = validate_unrets(file);
4780 if (ret < 0)
4781 return ret;
4782 warnings += ret;
4783 }
4784
4785 if (opts.ibt) {
4786 ret = validate_ibt(file);
4787 if (ret < 0)
4788 goto out;
4789 warnings += ret;
4790 }
4791
4792 if (opts.sls) {
4793 ret = validate_sls(file);
4794 if (ret < 0)
4795 goto out;
4796 warnings += ret;
4797 }
4798
4799 if (opts.static_call) {
4800 ret = create_static_call_sections(file);
4801 if (ret < 0)
4802 goto out;
4803 warnings += ret;
4804 }
4805
4806 if (opts.retpoline) {
4807 ret = create_retpoline_sites_sections(file);
4808 if (ret < 0)
4809 goto out;
4810 warnings += ret;
4811 }
4812
4813 if (opts.cfi) {
4814 ret = create_cfi_sections(file);
4815 if (ret < 0)
4816 goto out;
4817 warnings += ret;
4818 }
4819
4820 if (opts.rethunk) {
4821 ret = create_return_sites_sections(file);
4822 if (ret < 0)
4823 goto out;
4824 warnings += ret;
4825
4826 if (opts.hack_skylake) {
4827 ret = create_direct_call_sections(file);
4828 if (ret < 0)
4829 goto out;
4830 warnings += ret;
4831 }
4832 }
4833
4834 if (opts.mcount) {
4835 ret = create_mcount_loc_sections(file);
4836 if (ret < 0)
4837 goto out;
4838 warnings += ret;
4839 }
4840
4841 if (opts.prefix) {
4842 ret = add_prefix_symbols(file);
4843 if (ret < 0)
4844 return ret;
4845 warnings += ret;
4846 }
4847
4848 if (opts.ibt) {
4849 ret = create_ibt_endbr_seal_sections(file);
4850 if (ret < 0)
4851 goto out;
4852 warnings += ret;
4853 }
4854
4855 if (opts.orc && nr_insns) {
4856 ret = orc_create(file);
4857 if (ret < 0)
4858 goto out;
4859 warnings += ret;
4860 }
4861
4862 free_insns(file);
4863
4864 if (opts.verbose)
4865 disas_warned_funcs(file);
4866
4867 if (opts.stats) {
4868 printf("nr_insns_visited: %ld\n", nr_insns_visited);
4869 printf("nr_cfi: %ld\n", nr_cfi);
4870 printf("nr_cfi_reused: %ld\n", nr_cfi_reused);
4871 printf("nr_cfi_cache: %ld\n", nr_cfi_cache);
4872 }
4873
4874 out:
4875 /*
4876 * For now, don't fail the kernel build on fatal warnings. These
4877 * errors are still fairly common due to the growing matrix of
4878 * supported toolchains and their recent pace of change.
4879 */
4880 return 0;
4881 }
4882
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.