1 // SPDX-License-Identifier: GPL-2.0-or-later 1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /* 2 /*
3 * Copyright (C) 2015-2017 Josh Poimboeuf <jpo 3 * Copyright (C) 2015-2017 Josh Poimboeuf <jpoimboe@redhat.com>
4 */ 4 */
5 5
6 #include <string.h> 6 #include <string.h>
7 #include <stdlib.h> 7 #include <stdlib.h>
8 #include <inttypes.h> 8 #include <inttypes.h>
9 #include <sys/mman.h> 9 #include <sys/mman.h>
10 10
11 #include <objtool/builtin.h> 11 #include <objtool/builtin.h>
12 #include <objtool/cfi.h> 12 #include <objtool/cfi.h>
13 #include <objtool/arch.h> 13 #include <objtool/arch.h>
14 #include <objtool/check.h> 14 #include <objtool/check.h>
15 #include <objtool/special.h> 15 #include <objtool/special.h>
16 #include <objtool/warn.h> 16 #include <objtool/warn.h>
17 #include <objtool/endianness.h> 17 #include <objtool/endianness.h>
18 18
19 #include <linux/objtool_types.h> 19 #include <linux/objtool_types.h>
20 #include <linux/hashtable.h> 20 #include <linux/hashtable.h>
21 #include <linux/kernel.h> 21 #include <linux/kernel.h>
22 #include <linux/static_call_types.h> 22 #include <linux/static_call_types.h>
23 #include <linux/string.h> 23 #include <linux/string.h>
24 24
25 struct alternative { 25 struct alternative {
26 struct alternative *next; 26 struct alternative *next;
27 struct instruction *insn; 27 struct instruction *insn;
28 bool skip_orig; 28 bool skip_orig;
29 }; 29 };
30 30
31 static unsigned long nr_cfi, nr_cfi_reused, nr 31 static unsigned long nr_cfi, nr_cfi_reused, nr_cfi_cache;
32 32
33 static struct cfi_init_state initial_func_cfi; 33 static struct cfi_init_state initial_func_cfi;
34 static struct cfi_state init_cfi; 34 static struct cfi_state init_cfi;
35 static struct cfi_state func_cfi; 35 static struct cfi_state func_cfi;
36 static struct cfi_state force_undefined_cfi; 36 static struct cfi_state force_undefined_cfi;
37 37
38 struct instruction *find_insn(struct objtool_f 38 struct instruction *find_insn(struct objtool_file *file,
39 struct section * 39 struct section *sec, unsigned long offset)
40 { 40 {
41 struct instruction *insn; 41 struct instruction *insn;
42 42
43 hash_for_each_possible(file->insn_hash 43 hash_for_each_possible(file->insn_hash, insn, hash, sec_offset_hash(sec, offset)) {
44 if (insn->sec == sec && insn-> 44 if (insn->sec == sec && insn->offset == offset)
45 return insn; 45 return insn;
46 } 46 }
47 47
48 return NULL; 48 return NULL;
49 } 49 }
50 50
51 struct instruction *next_insn_same_sec(struct 51 struct instruction *next_insn_same_sec(struct objtool_file *file,
52 struct 52 struct instruction *insn)
53 { 53 {
54 if (insn->idx == INSN_CHUNK_MAX) 54 if (insn->idx == INSN_CHUNK_MAX)
55 return find_insn(file, insn->s 55 return find_insn(file, insn->sec, insn->offset + insn->len);
56 56
57 insn++; 57 insn++;
58 if (!insn->len) 58 if (!insn->len)
59 return NULL; 59 return NULL;
60 60
61 return insn; 61 return insn;
62 } 62 }
63 63
64 static struct instruction *next_insn_same_func 64 static struct instruction *next_insn_same_func(struct objtool_file *file,
65 65 struct instruction *insn)
66 { 66 {
67 struct instruction *next = next_insn_s 67 struct instruction *next = next_insn_same_sec(file, insn);
68 struct symbol *func = insn_func(insn); 68 struct symbol *func = insn_func(insn);
69 69
70 if (!func) 70 if (!func)
71 return NULL; 71 return NULL;
72 72
73 if (next && insn_func(next) == func) 73 if (next && insn_func(next) == func)
74 return next; 74 return next;
75 75
76 /* Check if we're already in the subfu 76 /* Check if we're already in the subfunction: */
77 if (func == func->cfunc) 77 if (func == func->cfunc)
78 return NULL; 78 return NULL;
79 79
80 /* Move to the subfunction: */ 80 /* Move to the subfunction: */
81 return find_insn(file, func->cfunc->se 81 return find_insn(file, func->cfunc->sec, func->cfunc->offset);
82 } 82 }
83 83
84 static struct instruction *prev_insn_same_sec( 84 static struct instruction *prev_insn_same_sec(struct objtool_file *file,
85 85 struct instruction *insn)
86 { 86 {
87 if (insn->idx == 0) { 87 if (insn->idx == 0) {
88 if (insn->prev_len) 88 if (insn->prev_len)
89 return find_insn(file, 89 return find_insn(file, insn->sec, insn->offset - insn->prev_len);
90 return NULL; 90 return NULL;
91 } 91 }
92 92
93 return insn - 1; 93 return insn - 1;
94 } 94 }
95 95
96 static struct instruction *prev_insn_same_sym( 96 static struct instruction *prev_insn_same_sym(struct objtool_file *file,
97 97 struct instruction *insn)
98 { 98 {
99 struct instruction *prev = prev_insn_s 99 struct instruction *prev = prev_insn_same_sec(file, insn);
100 100
101 if (prev && insn_func(prev) == insn_fu 101 if (prev && insn_func(prev) == insn_func(insn))
102 return prev; 102 return prev;
103 103
104 return NULL; 104 return NULL;
105 } 105 }
106 106
107 #define for_each_insn(file, insn) 107 #define for_each_insn(file, insn) \
108 for (struct section *__sec, *__fake = 108 for (struct section *__sec, *__fake = (struct section *)1; \
109 __fake; __fake = NULL) 109 __fake; __fake = NULL) \
110 for_each_sec(file, __sec) 110 for_each_sec(file, __sec) \
111 sec_for_each_insn(file 111 sec_for_each_insn(file, __sec, insn)
112 112
113 #define func_for_each_insn(file, func, insn) 113 #define func_for_each_insn(file, func, insn) \
114 for (insn = find_insn(file, func->sec, 114 for (insn = find_insn(file, func->sec, func->offset); \
115 insn; 115 insn; \
116 insn = next_insn_same_func(file, 116 insn = next_insn_same_func(file, insn))
117 117
118 #define sym_for_each_insn(file, sym, insn) 118 #define sym_for_each_insn(file, sym, insn) \
119 for (insn = find_insn(file, sym->sec, 119 for (insn = find_insn(file, sym->sec, sym->offset); \
120 insn && insn->offset < sym->offse 120 insn && insn->offset < sym->offset + sym->len; \
121 insn = next_insn_same_sec(file, i 121 insn = next_insn_same_sec(file, insn))
122 122
123 #define sym_for_each_insn_continue_reverse(fil 123 #define sym_for_each_insn_continue_reverse(file, sym, insn) \
124 for (insn = prev_insn_same_sec(file, i 124 for (insn = prev_insn_same_sec(file, insn); \
125 insn && insn->offset >= sym->offs 125 insn && insn->offset >= sym->offset; \
126 insn = prev_insn_same_sec(file, i 126 insn = prev_insn_same_sec(file, insn))
127 127
128 #define sec_for_each_insn_from(file, insn) 128 #define sec_for_each_insn_from(file, insn) \
129 for (; insn; insn = next_insn_same_sec 129 for (; insn; insn = next_insn_same_sec(file, insn))
130 130
131 #define sec_for_each_insn_continue(file, insn) 131 #define sec_for_each_insn_continue(file, insn) \
132 for (insn = next_insn_same_sec(file, i 132 for (insn = next_insn_same_sec(file, insn); insn; \
133 insn = next_insn_same_sec(file, i 133 insn = next_insn_same_sec(file, insn))
134 134
135 static inline struct symbol *insn_call_dest(st 135 static inline struct symbol *insn_call_dest(struct instruction *insn)
136 { 136 {
137 if (insn->type == INSN_JUMP_DYNAMIC || 137 if (insn->type == INSN_JUMP_DYNAMIC ||
138 insn->type == INSN_CALL_DYNAMIC) 138 insn->type == INSN_CALL_DYNAMIC)
139 return NULL; 139 return NULL;
140 140
141 return insn->_call_dest; 141 return insn->_call_dest;
142 } 142 }
143 143
144 static inline struct reloc *insn_jump_table(st 144 static inline struct reloc *insn_jump_table(struct instruction *insn)
145 { 145 {
146 if (insn->type == INSN_JUMP_DYNAMIC || 146 if (insn->type == INSN_JUMP_DYNAMIC ||
147 insn->type == INSN_CALL_DYNAMIC) 147 insn->type == INSN_CALL_DYNAMIC)
148 return insn->_jump_table; 148 return insn->_jump_table;
149 149
150 return NULL; 150 return NULL;
151 } 151 }
152 152
153 static bool is_jump_table_jump(struct instruct 153 static bool is_jump_table_jump(struct instruction *insn)
154 { 154 {
155 struct alt_group *alt_group = insn->al 155 struct alt_group *alt_group = insn->alt_group;
156 156
157 if (insn_jump_table(insn)) 157 if (insn_jump_table(insn))
158 return true; 158 return true;
159 159
160 /* Retpoline alternative for a jump ta 160 /* Retpoline alternative for a jump table? */
161 return alt_group && alt_group->orig_gr 161 return alt_group && alt_group->orig_group &&
162 insn_jump_table(alt_group->orig 162 insn_jump_table(alt_group->orig_group->first_insn);
163 } 163 }
164 164
165 static bool is_sibling_call(struct instruction 165 static bool is_sibling_call(struct instruction *insn)
166 { 166 {
167 /* 167 /*
168 * Assume only STT_FUNC calls have jum 168 * Assume only STT_FUNC calls have jump-tables.
169 */ 169 */
170 if (insn_func(insn)) { 170 if (insn_func(insn)) {
171 /* An indirect jump is either 171 /* An indirect jump is either a sibling call or a jump to a table. */
172 if (insn->type == INSN_JUMP_DY 172 if (insn->type == INSN_JUMP_DYNAMIC)
173 return !is_jump_table_ 173 return !is_jump_table_jump(insn);
174 } 174 }
175 175
176 /* add_jump_destinations() sets insn_c 176 /* add_jump_destinations() sets insn_call_dest(insn) for sibling calls. */
177 return (is_static_jump(insn) && insn_c 177 return (is_static_jump(insn) && insn_call_dest(insn));
178 } 178 }
179 179
180 /* 180 /*
181 * Checks if a string ends with another. 181 * Checks if a string ends with another.
182 */ 182 */
183 static bool str_ends_with(const char *s, const 183 static bool str_ends_with(const char *s, const char *sub)
184 { 184 {
185 const int slen = strlen(s); 185 const int slen = strlen(s);
186 const int sublen = strlen(sub); 186 const int sublen = strlen(sub);
187 187
188 if (sublen > slen) 188 if (sublen > slen)
189 return 0; 189 return 0;
190 190
191 return !memcmp(s + slen - sublen, sub, 191 return !memcmp(s + slen - sublen, sub, sublen);
192 } 192 }
193 193
194 /* 194 /*
195 * Checks if a function is a Rust "noreturn" o 195 * Checks if a function is a Rust "noreturn" one.
196 */ 196 */
197 static bool is_rust_noreturn(const struct symb 197 static bool is_rust_noreturn(const struct symbol *func)
198 { 198 {
199 /* 199 /*
200 * If it does not start with "_R", the 200 * If it does not start with "_R", then it is not a Rust symbol.
201 */ 201 */
202 if (strncmp(func->name, "_R", 2)) 202 if (strncmp(func->name, "_R", 2))
203 return false; 203 return false;
204 204
205 /* 205 /*
206 * These are just heuristics -- we do 206 * These are just heuristics -- we do not control the precise symbol
207 * name, due to the crate disambiguato 207 * name, due to the crate disambiguators (which depend on the compiler)
208 * as well as changes to the source co 208 * as well as changes to the source code itself between versions (since
209 * these come from the Rust standard l 209 * these come from the Rust standard library).
210 */ 210 */
211 return str_ends_with(func->name, "_4co 211 return str_ends_with(func->name, "_4core5sliceSp15copy_from_slice17len_mismatch_fail") ||
212 str_ends_with(func->name, "_4co 212 str_ends_with(func->name, "_4core6option13unwrap_failed") ||
213 str_ends_with(func->name, "_4co 213 str_ends_with(func->name, "_4core6result13unwrap_failed") ||
214 str_ends_with(func->name, "_4co 214 str_ends_with(func->name, "_4core9panicking5panic") ||
215 str_ends_with(func->name, "_4co 215 str_ends_with(func->name, "_4core9panicking9panic_fmt") ||
216 str_ends_with(func->name, "_4co 216 str_ends_with(func->name, "_4core9panicking14panic_explicit") ||
217 str_ends_with(func->name, "_4co 217 str_ends_with(func->name, "_4core9panicking14panic_nounwind") ||
218 str_ends_with(func->name, "_4co 218 str_ends_with(func->name, "_4core9panicking18panic_bounds_check") ||
219 str_ends_with(func->name, "_4co 219 str_ends_with(func->name, "_4core9panicking19assert_failed_inner") ||
220 str_ends_with(func->name, "_4co 220 str_ends_with(func->name, "_4core9panicking36panic_misaligned_pointer_dereference") ||
221 strstr(func->name, "_4core9pani 221 strstr(func->name, "_4core9panicking11panic_const24panic_const_") ||
222 (strstr(func->name, "_4core5sli 222 (strstr(func->name, "_4core5slice5index24slice_") &&
223 str_ends_with(func->name, "_fa 223 str_ends_with(func->name, "_fail"));
224 } 224 }
225 225
226 /* 226 /*
227 * This checks to see if the given function is 227 * This checks to see if the given function is a "noreturn" function.
228 * 228 *
229 * For global functions which are outside the 229 * For global functions which are outside the scope of this object file, we
230 * have to keep a manual list of them. 230 * have to keep a manual list of them.
231 * 231 *
232 * For local functions, we have to detect them 232 * For local functions, we have to detect them manually by simply looking for
233 * the lack of a return instruction. 233 * the lack of a return instruction.
234 */ 234 */
235 static bool __dead_end_function(struct objtool 235 static bool __dead_end_function(struct objtool_file *file, struct symbol *func,
236 int recursion) 236 int recursion)
237 { 237 {
238 int i; 238 int i;
239 struct instruction *insn; 239 struct instruction *insn;
240 bool empty = true; 240 bool empty = true;
241 241
242 #define NORETURN(func) __stringify(func), 242 #define NORETURN(func) __stringify(func),
243 static const char * const global_noret 243 static const char * const global_noreturns[] = {
244 #include "noreturns.h" 244 #include "noreturns.h"
245 }; 245 };
246 #undef NORETURN 246 #undef NORETURN
247 247
248 if (!func) 248 if (!func)
249 return false; 249 return false;
250 250
251 if (func->bind == STB_GLOBAL || func-> 251 if (func->bind == STB_GLOBAL || func->bind == STB_WEAK) {
252 if (is_rust_noreturn(func)) 252 if (is_rust_noreturn(func))
253 return true; 253 return true;
254 254
255 for (i = 0; i < ARRAY_SIZE(glo 255 for (i = 0; i < ARRAY_SIZE(global_noreturns); i++)
256 if (!strcmp(func->name 256 if (!strcmp(func->name, global_noreturns[i]))
257 return true; 257 return true;
258 } 258 }
259 259
260 if (func->bind == STB_WEAK) 260 if (func->bind == STB_WEAK)
261 return false; 261 return false;
262 262
263 if (!func->len) 263 if (!func->len)
264 return false; 264 return false;
265 265
266 insn = find_insn(file, func->sec, func 266 insn = find_insn(file, func->sec, func->offset);
267 if (!insn || !insn_func(insn)) 267 if (!insn || !insn_func(insn))
268 return false; 268 return false;
269 269
270 func_for_each_insn(file, func, insn) { 270 func_for_each_insn(file, func, insn) {
271 empty = false; 271 empty = false;
272 272
273 if (insn->type == INSN_RETURN) 273 if (insn->type == INSN_RETURN)
274 return false; 274 return false;
275 } 275 }
276 276
277 if (empty) 277 if (empty)
278 return false; 278 return false;
279 279
280 /* 280 /*
281 * A function can have a sibling call 281 * A function can have a sibling call instead of a return. In that
282 * case, the function's dead-end statu 282 * case, the function's dead-end status depends on whether the target
283 * of the sibling call returns. 283 * of the sibling call returns.
284 */ 284 */
285 func_for_each_insn(file, func, insn) { 285 func_for_each_insn(file, func, insn) {
286 if (is_sibling_call(insn)) { 286 if (is_sibling_call(insn)) {
287 struct instruction *de 287 struct instruction *dest = insn->jump_dest;
288 288
289 if (!dest) 289 if (!dest)
290 /* sibling cal 290 /* sibling call to another file */
291 return false; 291 return false;
292 292
293 /* local sibling call 293 /* local sibling call */
294 if (recursion == 5) { 294 if (recursion == 5) {
295 /* 295 /*
296 * Infinite re 296 * Infinite recursion: two functions have
297 * sibling cal 297 * sibling calls to each other. This is a very
298 * rare case. 298 * rare case. It means they aren't dead ends.
299 */ 299 */
300 return false; 300 return false;
301 } 301 }
302 302
303 return __dead_end_func 303 return __dead_end_function(file, insn_func(dest), recursion+1);
304 } 304 }
305 } 305 }
306 306
307 return true; 307 return true;
308 } 308 }
309 309
310 static bool dead_end_function(struct objtool_f 310 static bool dead_end_function(struct objtool_file *file, struct symbol *func)
311 { 311 {
312 return __dead_end_function(file, func, 312 return __dead_end_function(file, func, 0);
313 } 313 }
314 314
315 static void init_cfi_state(struct cfi_state *c 315 static void init_cfi_state(struct cfi_state *cfi)
316 { 316 {
317 int i; 317 int i;
318 318
319 for (i = 0; i < CFI_NUM_REGS; i++) { 319 for (i = 0; i < CFI_NUM_REGS; i++) {
320 cfi->regs[i].base = CFI_UNDEFI 320 cfi->regs[i].base = CFI_UNDEFINED;
321 cfi->vals[i].base = CFI_UNDEFI 321 cfi->vals[i].base = CFI_UNDEFINED;
322 } 322 }
323 cfi->cfa.base = CFI_UNDEFINED; 323 cfi->cfa.base = CFI_UNDEFINED;
324 cfi->drap_reg = CFI_UNDEFINED; 324 cfi->drap_reg = CFI_UNDEFINED;
325 cfi->drap_offset = -1; 325 cfi->drap_offset = -1;
326 } 326 }
327 327
328 static void init_insn_state(struct objtool_fil 328 static void init_insn_state(struct objtool_file *file, struct insn_state *state,
329 struct section *se 329 struct section *sec)
330 { 330 {
331 memset(state, 0, sizeof(*state)); 331 memset(state, 0, sizeof(*state));
332 init_cfi_state(&state->cfi); 332 init_cfi_state(&state->cfi);
333 333
334 /* 334 /*
335 * We need the full vmlinux for noinst 335 * We need the full vmlinux for noinstr validation, otherwise we can
336 * not correctly determine insn_call_d 336 * not correctly determine insn_call_dest(insn)->sec (external symbols
337 * do not have a section). 337 * do not have a section).
338 */ 338 */
339 if (opts.link && opts.noinstr && sec) 339 if (opts.link && opts.noinstr && sec)
340 state->noinstr = sec->noinstr; 340 state->noinstr = sec->noinstr;
341 } 341 }
342 342
343 static struct cfi_state *cfi_alloc(void) 343 static struct cfi_state *cfi_alloc(void)
344 { 344 {
345 struct cfi_state *cfi = calloc(1, size 345 struct cfi_state *cfi = calloc(1, sizeof(struct cfi_state));
346 if (!cfi) { 346 if (!cfi) {
347 WARN("calloc failed"); 347 WARN("calloc failed");
348 exit(1); 348 exit(1);
349 } 349 }
350 nr_cfi++; 350 nr_cfi++;
351 return cfi; 351 return cfi;
352 } 352 }
353 353
354 static int cfi_bits; 354 static int cfi_bits;
355 static struct hlist_head *cfi_hash; 355 static struct hlist_head *cfi_hash;
356 356
357 static inline bool cficmp(struct cfi_state *cf 357 static inline bool cficmp(struct cfi_state *cfi1, struct cfi_state *cfi2)
358 { 358 {
359 return memcmp((void *)cfi1 + sizeof(cf 359 return memcmp((void *)cfi1 + sizeof(cfi1->hash),
360 (void *)cfi2 + sizeof(cf 360 (void *)cfi2 + sizeof(cfi2->hash),
361 sizeof(struct cfi_state) 361 sizeof(struct cfi_state) - sizeof(struct hlist_node));
362 } 362 }
363 363
364 static inline u32 cfi_key(struct cfi_state *cf 364 static inline u32 cfi_key(struct cfi_state *cfi)
365 { 365 {
366 return jhash((void *)cfi + sizeof(cfi- 366 return jhash((void *)cfi + sizeof(cfi->hash),
367 sizeof(*cfi) - sizeof(cfi 367 sizeof(*cfi) - sizeof(cfi->hash), 0);
368 } 368 }
369 369
370 static struct cfi_state *cfi_hash_find_or_add( 370 static struct cfi_state *cfi_hash_find_or_add(struct cfi_state *cfi)
371 { 371 {
372 struct hlist_head *head = &cfi_hash[ha 372 struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
373 struct cfi_state *obj; 373 struct cfi_state *obj;
374 374
375 hlist_for_each_entry(obj, head, hash) 375 hlist_for_each_entry(obj, head, hash) {
376 if (!cficmp(cfi, obj)) { 376 if (!cficmp(cfi, obj)) {
377 nr_cfi_cache++; 377 nr_cfi_cache++;
378 return obj; 378 return obj;
379 } 379 }
380 } 380 }
381 381
382 obj = cfi_alloc(); 382 obj = cfi_alloc();
383 *obj = *cfi; 383 *obj = *cfi;
384 hlist_add_head(&obj->hash, head); 384 hlist_add_head(&obj->hash, head);
385 385
386 return obj; 386 return obj;
387 } 387 }
388 388
389 static void cfi_hash_add(struct cfi_state *cfi 389 static void cfi_hash_add(struct cfi_state *cfi)
390 { 390 {
391 struct hlist_head *head = &cfi_hash[ha 391 struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
392 392
393 hlist_add_head(&cfi->hash, head); 393 hlist_add_head(&cfi->hash, head);
394 } 394 }
395 395
396 static void *cfi_hash_alloc(unsigned long size 396 static void *cfi_hash_alloc(unsigned long size)
397 { 397 {
398 cfi_bits = max(10, ilog2(size)); 398 cfi_bits = max(10, ilog2(size));
399 cfi_hash = mmap(NULL, sizeof(struct hl 399 cfi_hash = mmap(NULL, sizeof(struct hlist_head) << cfi_bits,
400 PROT_READ|PROT_WRITE, 400 PROT_READ|PROT_WRITE,
401 MAP_PRIVATE|MAP_ANON, 401 MAP_PRIVATE|MAP_ANON, -1, 0);
402 if (cfi_hash == (void *)-1L) { 402 if (cfi_hash == (void *)-1L) {
403 WARN("mmap fail cfi_hash"); 403 WARN("mmap fail cfi_hash");
404 cfi_hash = NULL; 404 cfi_hash = NULL;
405 } else if (opts.stats) { 405 } else if (opts.stats) {
406 printf("cfi_bits: %d\n", cfi_b 406 printf("cfi_bits: %d\n", cfi_bits);
407 } 407 }
408 408
409 return cfi_hash; 409 return cfi_hash;
410 } 410 }
411 411
412 static unsigned long nr_insns; 412 static unsigned long nr_insns;
413 static unsigned long nr_insns_visited; 413 static unsigned long nr_insns_visited;
414 414
415 /* 415 /*
416 * Call the arch-specific instruction decoder 416 * Call the arch-specific instruction decoder for all the instructions and add
417 * them to the global instruction list. 417 * them to the global instruction list.
418 */ 418 */
419 static int decode_instructions(struct objtool_ 419 static int decode_instructions(struct objtool_file *file)
420 { 420 {
421 struct section *sec; 421 struct section *sec;
422 struct symbol *func; 422 struct symbol *func;
423 unsigned long offset; 423 unsigned long offset;
424 struct instruction *insn; 424 struct instruction *insn;
425 int ret; 425 int ret;
426 426
427 for_each_sec(file, sec) { 427 for_each_sec(file, sec) {
428 struct instruction *insns = NU 428 struct instruction *insns = NULL;
429 u8 prev_len = 0; 429 u8 prev_len = 0;
430 u8 idx = 0; 430 u8 idx = 0;
431 431
432 if (!(sec->sh.sh_flags & SHF_E 432 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
433 continue; 433 continue;
434 434
435 if (strcmp(sec->name, ".altins 435 if (strcmp(sec->name, ".altinstr_replacement") &&
436 strcmp(sec->name, ".altins 436 strcmp(sec->name, ".altinstr_aux") &&
437 strncmp(sec->name, ".disca 437 strncmp(sec->name, ".discard.", 9))
438 sec->text = true; 438 sec->text = true;
439 439
440 if (!strcmp(sec->name, ".noins 440 if (!strcmp(sec->name, ".noinstr.text") ||
441 !strcmp(sec->name, ".entry 441 !strcmp(sec->name, ".entry.text") ||
442 !strcmp(sec->name, ".cpuid 442 !strcmp(sec->name, ".cpuidle.text") ||
443 !strncmp(sec->name, ".text 443 !strncmp(sec->name, ".text..__x86.", 13))
444 sec->noinstr = true; 444 sec->noinstr = true;
445 445
446 /* 446 /*
447 * .init.text code is ran befo 447 * .init.text code is ran before userspace and thus doesn't
448 * strictly need retpolines, e 448 * strictly need retpolines, except for modules which are
449 * loaded late, they very much 449 * loaded late, they very much do need retpoline in their
450 * .init.text 450 * .init.text
451 */ 451 */
452 if (!strcmp(sec->name, ".init. 452 if (!strcmp(sec->name, ".init.text") && !opts.module)
453 sec->init = true; 453 sec->init = true;
454 454
455 for (offset = 0; offset < sec- 455 for (offset = 0; offset < sec->sh.sh_size; offset += insn->len) {
456 if (!insns || idx == I 456 if (!insns || idx == INSN_CHUNK_MAX) {
457 insns = calloc 457 insns = calloc(sizeof(*insn), INSN_CHUNK_SIZE);
458 if (!insns) { 458 if (!insns) {
459 WARN(" 459 WARN("malloc failed");
460 return 460 return -1;
461 } 461 }
462 idx = 0; 462 idx = 0;
463 } else { 463 } else {
464 idx++; 464 idx++;
465 } 465 }
466 insn = &insns[idx]; 466 insn = &insns[idx];
467 insn->idx = idx; 467 insn->idx = idx;
468 468
469 INIT_LIST_HEAD(&insn-> 469 INIT_LIST_HEAD(&insn->call_node);
470 insn->sec = sec; 470 insn->sec = sec;
471 insn->offset = offset; 471 insn->offset = offset;
472 insn->prev_len = prev_ 472 insn->prev_len = prev_len;
473 473
474 ret = arch_decode_inst 474 ret = arch_decode_instruction(file, sec, offset,
475 475 sec->sh.sh_size - offset,
476 476 insn);
477 if (ret) 477 if (ret)
478 return ret; 478 return ret;
479 479
480 prev_len = insn->len; 480 prev_len = insn->len;
481 481
482 /* 482 /*
483 * By default, "ud2" i 483 * By default, "ud2" is a dead end unless otherwise
484 * annotated, because 484 * annotated, because GCC 7 inserts it for certain
485 * divide-by-zero case 485 * divide-by-zero cases.
486 */ 486 */
487 if (insn->type == INSN 487 if (insn->type == INSN_BUG)
488 insn->dead_end 488 insn->dead_end = true;
489 489
490 hash_add(file->insn_ha 490 hash_add(file->insn_hash, &insn->hash, sec_offset_hash(sec, insn->offset));
491 nr_insns++; 491 nr_insns++;
492 } 492 }
493 493
494 // printf("%s: last chunk used: % 494 // printf("%s: last chunk used: %d\n", sec->name, (int)idx);
495 495
496 sec_for_each_sym(sec, func) { 496 sec_for_each_sym(sec, func) {
497 if (func->type != STT_ 497 if (func->type != STT_NOTYPE && func->type != STT_FUNC)
498 continue; 498 continue;
499 499
500 if (func->offset == se 500 if (func->offset == sec->sh.sh_size) {
501 /* Heuristic: 501 /* Heuristic: likely an "end" symbol */
502 if (func->type 502 if (func->type == STT_NOTYPE)
503 contin 503 continue;
504 WARN("%s(): ST 504 WARN("%s(): STT_FUNC at end of section",
505 func->nam 505 func->name);
506 return -1; 506 return -1;
507 } 507 }
508 508
509 if (func->embedded_ins 509 if (func->embedded_insn || func->alias != func)
510 continue; 510 continue;
511 511
512 if (!find_insn(file, s 512 if (!find_insn(file, sec, func->offset)) {
513 WARN("%s(): ca 513 WARN("%s(): can't find starting instruction",
514 func->nam 514 func->name);
515 return -1; 515 return -1;
516 } 516 }
517 517
518 sym_for_each_insn(file 518 sym_for_each_insn(file, func, insn) {
519 insn->sym = fu 519 insn->sym = func;
520 if (func->type 520 if (func->type == STT_FUNC &&
521 insn->type 521 insn->type == INSN_ENDBR &&
522 list_empty 522 list_empty(&insn->call_node)) {
523 if (in 523 if (insn->offset == func->offset) {
524 524 list_add_tail(&insn->call_node, &file->endbr_list);
525 525 file->nr_endbr++;
526 } else 526 } else {
527 527 file->nr_endbr_int++;
528 } 528 }
529 } 529 }
530 } 530 }
531 } 531 }
532 } 532 }
533 533
534 if (opts.stats) 534 if (opts.stats)
535 printf("nr_insns: %lu\n", nr_i 535 printf("nr_insns: %lu\n", nr_insns);
536 536
537 return 0; 537 return 0;
538 } 538 }
539 539
540 /* 540 /*
541 * Read the pv_ops[] .data table to find the s 541 * Read the pv_ops[] .data table to find the static initialized values.
542 */ 542 */
543 static int add_pv_ops(struct objtool_file *fil 543 static int add_pv_ops(struct objtool_file *file, const char *symname)
544 { 544 {
545 struct symbol *sym, *func; 545 struct symbol *sym, *func;
546 unsigned long off, end; 546 unsigned long off, end;
547 struct reloc *reloc; 547 struct reloc *reloc;
548 int idx; 548 int idx;
549 549
550 sym = find_symbol_by_name(file->elf, s 550 sym = find_symbol_by_name(file->elf, symname);
551 if (!sym) 551 if (!sym)
552 return 0; 552 return 0;
553 553
554 off = sym->offset; 554 off = sym->offset;
555 end = off + sym->len; 555 end = off + sym->len;
556 for (;;) { 556 for (;;) {
557 reloc = find_reloc_by_dest_ran 557 reloc = find_reloc_by_dest_range(file->elf, sym->sec, off, end - off);
558 if (!reloc) 558 if (!reloc)
559 break; 559 break;
560 560
561 func = reloc->sym; 561 func = reloc->sym;
562 if (func->type == STT_SECTION) 562 if (func->type == STT_SECTION)
563 func = find_symbol_by_ 563 func = find_symbol_by_offset(reloc->sym->sec,
564 564 reloc_addend(reloc));
565 565
566 idx = (reloc_offset(reloc) - s 566 idx = (reloc_offset(reloc) - sym->offset) / sizeof(unsigned long);
567 567
568 objtool_pv_add(file, idx, func 568 objtool_pv_add(file, idx, func);
569 569
570 off = reloc_offset(reloc) + 1; 570 off = reloc_offset(reloc) + 1;
571 if (off > end) 571 if (off > end)
572 break; 572 break;
573 } 573 }
574 574
575 return 0; 575 return 0;
576 } 576 }
577 577
578 /* 578 /*
579 * Allocate and initialize file->pv_ops[]. 579 * Allocate and initialize file->pv_ops[].
580 */ 580 */
581 static int init_pv_ops(struct objtool_file *fi 581 static int init_pv_ops(struct objtool_file *file)
582 { 582 {
583 static const char *pv_ops_tables[] = { 583 static const char *pv_ops_tables[] = {
584 "pv_ops", 584 "pv_ops",
585 "xen_cpu_ops", 585 "xen_cpu_ops",
586 "xen_irq_ops", 586 "xen_irq_ops",
587 "xen_mmu_ops", 587 "xen_mmu_ops",
588 NULL, 588 NULL,
589 }; 589 };
590 const char *pv_ops; 590 const char *pv_ops;
591 struct symbol *sym; 591 struct symbol *sym;
592 int idx, nr; 592 int idx, nr;
593 593
594 if (!opts.noinstr) 594 if (!opts.noinstr)
595 return 0; 595 return 0;
596 596
597 file->pv_ops = NULL; 597 file->pv_ops = NULL;
598 598
599 sym = find_symbol_by_name(file->elf, " 599 sym = find_symbol_by_name(file->elf, "pv_ops");
600 if (!sym) 600 if (!sym)
601 return 0; 601 return 0;
602 602
603 nr = sym->len / sizeof(unsigned long); 603 nr = sym->len / sizeof(unsigned long);
604 file->pv_ops = calloc(sizeof(struct pv 604 file->pv_ops = calloc(sizeof(struct pv_state), nr);
605 if (!file->pv_ops) 605 if (!file->pv_ops)
606 return -1; 606 return -1;
607 607
608 for (idx = 0; idx < nr; idx++) 608 for (idx = 0; idx < nr; idx++)
609 INIT_LIST_HEAD(&file->pv_ops[i 609 INIT_LIST_HEAD(&file->pv_ops[idx].targets);
610 610
611 for (idx = 0; (pv_ops = pv_ops_tables[ 611 for (idx = 0; (pv_ops = pv_ops_tables[idx]); idx++)
612 add_pv_ops(file, pv_ops); 612 add_pv_ops(file, pv_ops);
613 613
614 return 0; 614 return 0;
615 } 615 }
616 616
617 static struct instruction *find_last_insn(stru 617 static struct instruction *find_last_insn(struct objtool_file *file,
618 stru 618 struct section *sec)
619 { 619 {
620 struct instruction *insn = NULL; 620 struct instruction *insn = NULL;
621 unsigned int offset; 621 unsigned int offset;
622 unsigned int end = (sec->sh.sh_size > 622 unsigned int end = (sec->sh.sh_size > 10) ? sec->sh.sh_size - 10 : 0;
623 623
624 for (offset = sec->sh.sh_size - 1; off 624 for (offset = sec->sh.sh_size - 1; offset >= end && !insn; offset--)
625 insn = find_insn(file, sec, of 625 insn = find_insn(file, sec, offset);
626 626
627 return insn; 627 return insn;
628 } 628 }
629 629
630 /* 630 /*
631 * Mark "ud2" instructions and manually annota 631 * Mark "ud2" instructions and manually annotated dead ends.
632 */ 632 */
633 static int add_dead_ends(struct objtool_file * 633 static int add_dead_ends(struct objtool_file *file)
634 { 634 {
635 struct section *rsec; 635 struct section *rsec;
636 struct reloc *reloc; 636 struct reloc *reloc;
637 struct instruction *insn; 637 struct instruction *insn;
638 uint64_t offset; 638 uint64_t offset;
639 639
640 /* 640 /*
641 * Check for manually annotated dead e 641 * Check for manually annotated dead ends.
642 */ 642 */
643 rsec = find_section_by_name(file->elf, 643 rsec = find_section_by_name(file->elf, ".rela.discard.unreachable");
644 if (!rsec) 644 if (!rsec)
645 goto reachable; 645 goto reachable;
646 646
647 for_each_reloc(rsec, reloc) { 647 for_each_reloc(rsec, reloc) {
648 if (reloc->sym->type == STT_SE 648 if (reloc->sym->type == STT_SECTION) {
649 offset = reloc_addend( 649 offset = reloc_addend(reloc);
650 } else if (reloc->sym->local_l 650 } else if (reloc->sym->local_label) {
651 offset = reloc->sym->o 651 offset = reloc->sym->offset;
652 } else { 652 } else {
653 WARN("unexpected reloc 653 WARN("unexpected relocation symbol type in %s", rsec->name);
654 return -1; 654 return -1;
655 } 655 }
656 656
657 insn = find_insn(file, reloc-> 657 insn = find_insn(file, reloc->sym->sec, offset);
658 if (insn) 658 if (insn)
659 insn = prev_insn_same_ 659 insn = prev_insn_same_sec(file, insn);
660 else if (offset == reloc->sym- 660 else if (offset == reloc->sym->sec->sh.sh_size) {
661 insn = find_last_insn( 661 insn = find_last_insn(file, reloc->sym->sec);
662 if (!insn) { 662 if (!insn) {
663 WARN("can't fi 663 WARN("can't find unreachable insn at %s+0x%" PRIx64,
664 reloc->sy 664 reloc->sym->sec->name, offset);
665 return -1; 665 return -1;
666 } 666 }
667 } else { 667 } else {
668 WARN("can't find unrea 668 WARN("can't find unreachable insn at %s+0x%" PRIx64,
669 reloc->sym->sec-> 669 reloc->sym->sec->name, offset);
670 return -1; 670 return -1;
671 } 671 }
672 672
673 insn->dead_end = true; 673 insn->dead_end = true;
674 } 674 }
675 675
676 reachable: 676 reachable:
677 /* 677 /*
678 * These manually annotated reachable 678 * These manually annotated reachable checks are needed for GCC 4.4,
679 * where the Linux unreachable() macro 679 * where the Linux unreachable() macro isn't supported. In that case
680 * GCC doesn't know the "ud2" is fatal 680 * GCC doesn't know the "ud2" is fatal, so it generates code as if it's
681 * not a dead end. 681 * not a dead end.
682 */ 682 */
683 rsec = find_section_by_name(file->elf, 683 rsec = find_section_by_name(file->elf, ".rela.discard.reachable");
684 if (!rsec) 684 if (!rsec)
685 return 0; 685 return 0;
686 686
687 for_each_reloc(rsec, reloc) { 687 for_each_reloc(rsec, reloc) {
688 if (reloc->sym->type == STT_SE 688 if (reloc->sym->type == STT_SECTION) {
689 offset = reloc_addend( 689 offset = reloc_addend(reloc);
690 } else if (reloc->sym->local_l 690 } else if (reloc->sym->local_label) {
691 offset = reloc->sym->o 691 offset = reloc->sym->offset;
692 } else { 692 } else {
693 WARN("unexpected reloc 693 WARN("unexpected relocation symbol type in %s", rsec->name);
694 return -1; 694 return -1;
695 } 695 }
696 696
697 insn = find_insn(file, reloc-> 697 insn = find_insn(file, reloc->sym->sec, offset);
698 if (insn) 698 if (insn)
699 insn = prev_insn_same_ 699 insn = prev_insn_same_sec(file, insn);
700 else if (offset == reloc->sym- 700 else if (offset == reloc->sym->sec->sh.sh_size) {
701 insn = find_last_insn( 701 insn = find_last_insn(file, reloc->sym->sec);
702 if (!insn) { 702 if (!insn) {
703 WARN("can't fi 703 WARN("can't find reachable insn at %s+0x%" PRIx64,
704 reloc->sy 704 reloc->sym->sec->name, offset);
705 return -1; 705 return -1;
706 } 706 }
707 } else { 707 } else {
708 WARN("can't find reach 708 WARN("can't find reachable insn at %s+0x%" PRIx64,
709 reloc->sym->sec-> 709 reloc->sym->sec->name, offset);
710 return -1; 710 return -1;
711 } 711 }
712 712
713 insn->dead_end = false; 713 insn->dead_end = false;
714 } 714 }
715 715
716 return 0; 716 return 0;
717 } 717 }
718 718
719 static int create_static_call_sections(struct 719 static int create_static_call_sections(struct objtool_file *file)
720 { 720 {
721 struct static_call_site *site; 721 struct static_call_site *site;
722 struct section *sec; 722 struct section *sec;
723 struct instruction *insn; 723 struct instruction *insn;
724 struct symbol *key_sym; 724 struct symbol *key_sym;
725 char *key_name, *tmp; 725 char *key_name, *tmp;
726 int idx; 726 int idx;
727 727
728 sec = find_section_by_name(file->elf, 728 sec = find_section_by_name(file->elf, ".static_call_sites");
729 if (sec) { 729 if (sec) {
730 INIT_LIST_HEAD(&file->static_c 730 INIT_LIST_HEAD(&file->static_call_list);
731 WARN("file already has .static 731 WARN("file already has .static_call_sites section, skipping");
732 return 0; 732 return 0;
733 } 733 }
734 734
735 if (list_empty(&file->static_call_list 735 if (list_empty(&file->static_call_list))
736 return 0; 736 return 0;
737 737
738 idx = 0; 738 idx = 0;
739 list_for_each_entry(insn, &file->stati 739 list_for_each_entry(insn, &file->static_call_list, call_node)
740 idx++; 740 idx++;
741 741
742 sec = elf_create_section_pair(file->el 742 sec = elf_create_section_pair(file->elf, ".static_call_sites",
743 sizeof(* 743 sizeof(*site), idx, idx * 2);
744 if (!sec) 744 if (!sec)
745 return -1; 745 return -1;
746 746
747 /* Allow modules to modify the low bit 747 /* Allow modules to modify the low bits of static_call_site::key */
748 sec->sh.sh_flags |= SHF_WRITE; 748 sec->sh.sh_flags |= SHF_WRITE;
749 749
750 idx = 0; 750 idx = 0;
751 list_for_each_entry(insn, &file->stati 751 list_for_each_entry(insn, &file->static_call_list, call_node) {
752 752
753 /* populate reloc for 'addr' * 753 /* populate reloc for 'addr' */
754 if (!elf_init_reloc_text_sym(f 754 if (!elf_init_reloc_text_sym(file->elf, sec,
755 i 755 idx * sizeof(*site), idx * 2,
756 i 756 insn->sec, insn->offset))
757 return -1; 757 return -1;
758 758
759 /* find key symbol */ 759 /* find key symbol */
760 key_name = strdup(insn_call_de 760 key_name = strdup(insn_call_dest(insn)->name);
761 if (!key_name) { 761 if (!key_name) {
762 perror("strdup"); 762 perror("strdup");
763 return -1; 763 return -1;
764 } 764 }
765 if (strncmp(key_name, STATIC_C 765 if (strncmp(key_name, STATIC_CALL_TRAMP_PREFIX_STR,
766 STATIC_CALL_TRAMP_ 766 STATIC_CALL_TRAMP_PREFIX_LEN)) {
767 WARN("static_call: tra 767 WARN("static_call: trampoline name malformed: %s", key_name);
768 free(key_name); 768 free(key_name);
769 return -1; 769 return -1;
770 } 770 }
771 tmp = key_name + STATIC_CALL_T 771 tmp = key_name + STATIC_CALL_TRAMP_PREFIX_LEN - STATIC_CALL_KEY_PREFIX_LEN;
772 memcpy(tmp, STATIC_CALL_KEY_PR 772 memcpy(tmp, STATIC_CALL_KEY_PREFIX_STR, STATIC_CALL_KEY_PREFIX_LEN);
773 773
774 key_sym = find_symbol_by_name( 774 key_sym = find_symbol_by_name(file->elf, tmp);
775 if (!key_sym) { 775 if (!key_sym) {
776 if (!opts.module) { 776 if (!opts.module) {
777 WARN("static_c 777 WARN("static_call: can't find static_call_key symbol: %s", tmp);
778 free(key_name) 778 free(key_name);
779 return -1; 779 return -1;
780 } 780 }
781 781
782 /* 782 /*
783 * For modules(), the 783 * For modules(), the key might not be exported, which
784 * means the module ca 784 * means the module can make static calls but isn't
785 * allowed to change t 785 * allowed to change them.
786 * 786 *
787 * In that case we tem 787 * In that case we temporarily set the key to be the
788 * trampoline address. 788 * trampoline address. This is fixed up in
789 * static_call_add_mod 789 * static_call_add_module().
790 */ 790 */
791 key_sym = insn_call_de 791 key_sym = insn_call_dest(insn);
792 } 792 }
793 free(key_name); 793 free(key_name);
794 794
795 /* populate reloc for 'key' */ 795 /* populate reloc for 'key' */
796 if (!elf_init_reloc_data_sym(f 796 if (!elf_init_reloc_data_sym(file->elf, sec,
797 i 797 idx * sizeof(*site) + 4,
798 ( 798 (idx * 2) + 1, key_sym,
799 i 799 is_sibling_call(insn) * STATIC_CALL_SITE_TAIL))
800 return -1; 800 return -1;
801 801
802 idx++; 802 idx++;
803 } 803 }
804 804
805 return 0; 805 return 0;
806 } 806 }
807 807
808 static int create_retpoline_sites_sections(str 808 static int create_retpoline_sites_sections(struct objtool_file *file)
809 { 809 {
810 struct instruction *insn; 810 struct instruction *insn;
811 struct section *sec; 811 struct section *sec;
812 int idx; 812 int idx;
813 813
814 sec = find_section_by_name(file->elf, 814 sec = find_section_by_name(file->elf, ".retpoline_sites");
815 if (sec) { 815 if (sec) {
816 WARN("file already has .retpol 816 WARN("file already has .retpoline_sites, skipping");
817 return 0; 817 return 0;
818 } 818 }
819 819
820 idx = 0; 820 idx = 0;
821 list_for_each_entry(insn, &file->retpo 821 list_for_each_entry(insn, &file->retpoline_call_list, call_node)
822 idx++; 822 idx++;
823 823
824 if (!idx) 824 if (!idx)
825 return 0; 825 return 0;
826 826
827 sec = elf_create_section_pair(file->el 827 sec = elf_create_section_pair(file->elf, ".retpoline_sites",
828 sizeof(i 828 sizeof(int), idx, idx);
829 if (!sec) 829 if (!sec)
830 return -1; 830 return -1;
831 831
832 idx = 0; 832 idx = 0;
833 list_for_each_entry(insn, &file->retpo 833 list_for_each_entry(insn, &file->retpoline_call_list, call_node) {
834 834
835 if (!elf_init_reloc_text_sym(f 835 if (!elf_init_reloc_text_sym(file->elf, sec,
836 i 836 idx * sizeof(int), idx,
837 i 837 insn->sec, insn->offset))
838 return -1; 838 return -1;
839 839
840 idx++; 840 idx++;
841 } 841 }
842 842
843 return 0; 843 return 0;
844 } 844 }
845 845
846 static int create_return_sites_sections(struct 846 static int create_return_sites_sections(struct objtool_file *file)
847 { 847 {
848 struct instruction *insn; 848 struct instruction *insn;
849 struct section *sec; 849 struct section *sec;
850 int idx; 850 int idx;
851 851
852 sec = find_section_by_name(file->elf, 852 sec = find_section_by_name(file->elf, ".return_sites");
853 if (sec) { 853 if (sec) {
854 WARN("file already has .return 854 WARN("file already has .return_sites, skipping");
855 return 0; 855 return 0;
856 } 856 }
857 857
858 idx = 0; 858 idx = 0;
859 list_for_each_entry(insn, &file->retur 859 list_for_each_entry(insn, &file->return_thunk_list, call_node)
860 idx++; 860 idx++;
861 861
862 if (!idx) 862 if (!idx)
863 return 0; 863 return 0;
864 864
865 sec = elf_create_section_pair(file->el 865 sec = elf_create_section_pair(file->elf, ".return_sites",
866 sizeof(i 866 sizeof(int), idx, idx);
867 if (!sec) 867 if (!sec)
868 return -1; 868 return -1;
869 869
870 idx = 0; 870 idx = 0;
871 list_for_each_entry(insn, &file->retur 871 list_for_each_entry(insn, &file->return_thunk_list, call_node) {
872 872
873 if (!elf_init_reloc_text_sym(f 873 if (!elf_init_reloc_text_sym(file->elf, sec,
874 i 874 idx * sizeof(int), idx,
875 i 875 insn->sec, insn->offset))
876 return -1; 876 return -1;
877 877
878 idx++; 878 idx++;
879 } 879 }
880 880
881 return 0; 881 return 0;
882 } 882 }
883 883
884 static int create_ibt_endbr_seal_sections(stru 884 static int create_ibt_endbr_seal_sections(struct objtool_file *file)
885 { 885 {
886 struct instruction *insn; 886 struct instruction *insn;
887 struct section *sec; 887 struct section *sec;
888 int idx; 888 int idx;
889 889
890 sec = find_section_by_name(file->elf, 890 sec = find_section_by_name(file->elf, ".ibt_endbr_seal");
891 if (sec) { 891 if (sec) {
892 WARN("file already has .ibt_en 892 WARN("file already has .ibt_endbr_seal, skipping");
893 return 0; 893 return 0;
894 } 894 }
895 895
896 idx = 0; 896 idx = 0;
897 list_for_each_entry(insn, &file->endbr 897 list_for_each_entry(insn, &file->endbr_list, call_node)
898 idx++; 898 idx++;
899 899
900 if (opts.stats) { 900 if (opts.stats) {
901 printf("ibt: ENDBR at function 901 printf("ibt: ENDBR at function start: %d\n", file->nr_endbr);
902 printf("ibt: ENDBR inside func 902 printf("ibt: ENDBR inside functions: %d\n", file->nr_endbr_int);
903 printf("ibt: superfluous ENDBR 903 printf("ibt: superfluous ENDBR: %d\n", idx);
904 } 904 }
905 905
906 if (!idx) 906 if (!idx)
907 return 0; 907 return 0;
908 908
909 sec = elf_create_section_pair(file->el 909 sec = elf_create_section_pair(file->elf, ".ibt_endbr_seal",
910 sizeof(i 910 sizeof(int), idx, idx);
911 if (!sec) 911 if (!sec)
912 return -1; 912 return -1;
913 913
914 idx = 0; 914 idx = 0;
915 list_for_each_entry(insn, &file->endbr 915 list_for_each_entry(insn, &file->endbr_list, call_node) {
916 916
917 int *site = (int *)sec->data-> 917 int *site = (int *)sec->data->d_buf + idx;
918 struct symbol *sym = insn->sym 918 struct symbol *sym = insn->sym;
919 *site = 0; 919 *site = 0;
920 920
921 if (opts.module && sym && sym- 921 if (opts.module && sym && sym->type == STT_FUNC &&
922 insn->offset == sym->offse 922 insn->offset == sym->offset &&
923 (!strcmp(sym->name, "init_ 923 (!strcmp(sym->name, "init_module") ||
924 !strcmp(sym->name, "clean 924 !strcmp(sym->name, "cleanup_module")))
925 WARN("%s(): not an ind 925 WARN("%s(): not an indirect call target", sym->name);
926 926
927 if (!elf_init_reloc_text_sym(f 927 if (!elf_init_reloc_text_sym(file->elf, sec,
928 i 928 idx * sizeof(int), idx,
929 i 929 insn->sec, insn->offset))
930 return -1; 930 return -1;
931 931
932 idx++; 932 idx++;
933 } 933 }
934 934
935 return 0; 935 return 0;
936 } 936 }
937 937
938 static int create_cfi_sections(struct objtool_ 938 static int create_cfi_sections(struct objtool_file *file)
939 { 939 {
940 struct section *sec; 940 struct section *sec;
941 struct symbol *sym; 941 struct symbol *sym;
942 int idx; 942 int idx;
943 943
944 sec = find_section_by_name(file->elf, 944 sec = find_section_by_name(file->elf, ".cfi_sites");
945 if (sec) { 945 if (sec) {
946 INIT_LIST_HEAD(&file->call_lis 946 INIT_LIST_HEAD(&file->call_list);
947 WARN("file already has .cfi_si 947 WARN("file already has .cfi_sites section, skipping");
948 return 0; 948 return 0;
949 } 949 }
950 950
951 idx = 0; 951 idx = 0;
952 for_each_sym(file, sym) { 952 for_each_sym(file, sym) {
953 if (sym->type != STT_FUNC) 953 if (sym->type != STT_FUNC)
954 continue; 954 continue;
955 955
956 if (strncmp(sym->name, "__cfi_ 956 if (strncmp(sym->name, "__cfi_", 6))
957 continue; 957 continue;
958 958
959 idx++; 959 idx++;
960 } 960 }
961 961
962 sec = elf_create_section_pair(file->el 962 sec = elf_create_section_pair(file->elf, ".cfi_sites",
963 sizeof(u 963 sizeof(unsigned int), idx, idx);
964 if (!sec) 964 if (!sec)
965 return -1; 965 return -1;
966 966
967 idx = 0; 967 idx = 0;
968 for_each_sym(file, sym) { 968 for_each_sym(file, sym) {
969 if (sym->type != STT_FUNC) 969 if (sym->type != STT_FUNC)
970 continue; 970 continue;
971 971
972 if (strncmp(sym->name, "__cfi_ 972 if (strncmp(sym->name, "__cfi_", 6))
973 continue; 973 continue;
974 974
975 if (!elf_init_reloc_text_sym(f 975 if (!elf_init_reloc_text_sym(file->elf, sec,
976 i 976 idx * sizeof(unsigned int), idx,
977 s 977 sym->sec, sym->offset))
978 return -1; 978 return -1;
979 979
980 idx++; 980 idx++;
981 } 981 }
982 982
983 return 0; 983 return 0;
984 } 984 }
985 985
986 static int create_mcount_loc_sections(struct o 986 static int create_mcount_loc_sections(struct objtool_file *file)
987 { 987 {
988 size_t addr_size = elf_addr_size(file- 988 size_t addr_size = elf_addr_size(file->elf);
989 struct instruction *insn; 989 struct instruction *insn;
990 struct section *sec; 990 struct section *sec;
991 int idx; 991 int idx;
992 992
993 sec = find_section_by_name(file->elf, 993 sec = find_section_by_name(file->elf, "__mcount_loc");
994 if (sec) { 994 if (sec) {
995 INIT_LIST_HEAD(&file->mcount_l 995 INIT_LIST_HEAD(&file->mcount_loc_list);
996 WARN("file already has __mcoun 996 WARN("file already has __mcount_loc section, skipping");
997 return 0; 997 return 0;
998 } 998 }
999 999
1000 if (list_empty(&file->mcount_loc_list 1000 if (list_empty(&file->mcount_loc_list))
1001 return 0; 1001 return 0;
1002 1002
1003 idx = 0; 1003 idx = 0;
1004 list_for_each_entry(insn, &file->mcou 1004 list_for_each_entry(insn, &file->mcount_loc_list, call_node)
1005 idx++; 1005 idx++;
1006 1006
1007 sec = elf_create_section_pair(file->e 1007 sec = elf_create_section_pair(file->elf, "__mcount_loc", addr_size,
1008 idx, id 1008 idx, idx);
1009 if (!sec) 1009 if (!sec)
1010 return -1; 1010 return -1;
1011 1011
1012 sec->sh.sh_addralign = addr_size; 1012 sec->sh.sh_addralign = addr_size;
1013 1013
1014 idx = 0; 1014 idx = 0;
1015 list_for_each_entry(insn, &file->mcou 1015 list_for_each_entry(insn, &file->mcount_loc_list, call_node) {
1016 1016
1017 struct reloc *reloc; 1017 struct reloc *reloc;
1018 1018
1019 reloc = elf_init_reloc_text_s 1019 reloc = elf_init_reloc_text_sym(file->elf, sec, idx * addr_size, idx,
1020 1020 insn->sec, insn->offset);
1021 if (!reloc) 1021 if (!reloc)
1022 return -1; 1022 return -1;
1023 1023
1024 set_reloc_type(file->elf, rel 1024 set_reloc_type(file->elf, reloc, addr_size == 8 ? R_ABS64 : R_ABS32);
1025 1025
1026 idx++; 1026 idx++;
1027 } 1027 }
1028 1028
1029 return 0; 1029 return 0;
1030 } 1030 }
1031 1031
1032 static int create_direct_call_sections(struct 1032 static int create_direct_call_sections(struct objtool_file *file)
1033 { 1033 {
1034 struct instruction *insn; 1034 struct instruction *insn;
1035 struct section *sec; 1035 struct section *sec;
1036 int idx; 1036 int idx;
1037 1037
1038 sec = find_section_by_name(file->elf, 1038 sec = find_section_by_name(file->elf, ".call_sites");
1039 if (sec) { 1039 if (sec) {
1040 INIT_LIST_HEAD(&file->call_li 1040 INIT_LIST_HEAD(&file->call_list);
1041 WARN("file already has .call_ 1041 WARN("file already has .call_sites section, skipping");
1042 return 0; 1042 return 0;
1043 } 1043 }
1044 1044
1045 if (list_empty(&file->call_list)) 1045 if (list_empty(&file->call_list))
1046 return 0; 1046 return 0;
1047 1047
1048 idx = 0; 1048 idx = 0;
1049 list_for_each_entry(insn, &file->call 1049 list_for_each_entry(insn, &file->call_list, call_node)
1050 idx++; 1050 idx++;
1051 1051
1052 sec = elf_create_section_pair(file->e 1052 sec = elf_create_section_pair(file->elf, ".call_sites",
1053 sizeof( 1053 sizeof(unsigned int), idx, idx);
1054 if (!sec) 1054 if (!sec)
1055 return -1; 1055 return -1;
1056 1056
1057 idx = 0; 1057 idx = 0;
1058 list_for_each_entry(insn, &file->call 1058 list_for_each_entry(insn, &file->call_list, call_node) {
1059 1059
1060 if (!elf_init_reloc_text_sym( 1060 if (!elf_init_reloc_text_sym(file->elf, sec,
1061 1061 idx * sizeof(unsigned int), idx,
1062 1062 insn->sec, insn->offset))
1063 return -1; 1063 return -1;
1064 1064
1065 idx++; 1065 idx++;
1066 } 1066 }
1067 1067
1068 return 0; 1068 return 0;
1069 } 1069 }
1070 1070
1071 /* 1071 /*
1072 * Warnings shouldn't be reported for ignored 1072 * Warnings shouldn't be reported for ignored functions.
1073 */ 1073 */
1074 static void add_ignores(struct objtool_file * 1074 static void add_ignores(struct objtool_file *file)
1075 { 1075 {
1076 struct instruction *insn; 1076 struct instruction *insn;
1077 struct section *rsec; 1077 struct section *rsec;
1078 struct symbol *func; 1078 struct symbol *func;
1079 struct reloc *reloc; 1079 struct reloc *reloc;
1080 1080
1081 rsec = find_section_by_name(file->elf 1081 rsec = find_section_by_name(file->elf, ".rela.discard.func_stack_frame_non_standard");
1082 if (!rsec) 1082 if (!rsec)
1083 return; 1083 return;
1084 1084
1085 for_each_reloc(rsec, reloc) { 1085 for_each_reloc(rsec, reloc) {
1086 switch (reloc->sym->type) { 1086 switch (reloc->sym->type) {
1087 case STT_FUNC: 1087 case STT_FUNC:
1088 func = reloc->sym; 1088 func = reloc->sym;
1089 break; 1089 break;
1090 1090
1091 case STT_SECTION: 1091 case STT_SECTION:
1092 func = find_func_by_o 1092 func = find_func_by_offset(reloc->sym->sec, reloc_addend(reloc));
1093 if (!func) 1093 if (!func)
1094 continue; 1094 continue;
1095 break; 1095 break;
1096 1096
1097 default: 1097 default:
1098 WARN("unexpected relo 1098 WARN("unexpected relocation symbol type in %s: %d",
1099 rsec->name, relo 1099 rsec->name, reloc->sym->type);
1100 continue; 1100 continue;
1101 } 1101 }
1102 1102
1103 func_for_each_insn(file, func 1103 func_for_each_insn(file, func, insn)
1104 insn->ignore = true; 1104 insn->ignore = true;
1105 } 1105 }
1106 } 1106 }
1107 1107
1108 /* 1108 /*
1109 * This is a whitelist of functions that is a 1109 * This is a whitelist of functions that is allowed to be called with AC set.
1110 * The list is meant to be minimal and only c 1110 * The list is meant to be minimal and only contains compiler instrumentation
1111 * ABI and a few functions used to implement 1111 * ABI and a few functions used to implement *_{to,from}_user() functions.
1112 * 1112 *
1113 * These functions must not directly change A 1113 * These functions must not directly change AC, but may PUSHF/POPF.
1114 */ 1114 */
1115 static const char *uaccess_safe_builtin[] = { 1115 static const char *uaccess_safe_builtin[] = {
1116 /* KASAN */ 1116 /* KASAN */
1117 "kasan_report", 1117 "kasan_report",
1118 "kasan_check_range", 1118 "kasan_check_range",
1119 /* KASAN out-of-line */ 1119 /* KASAN out-of-line */
1120 "__asan_loadN_noabort", 1120 "__asan_loadN_noabort",
1121 "__asan_load1_noabort", 1121 "__asan_load1_noabort",
1122 "__asan_load2_noabort", 1122 "__asan_load2_noabort",
1123 "__asan_load4_noabort", 1123 "__asan_load4_noabort",
1124 "__asan_load8_noabort", 1124 "__asan_load8_noabort",
1125 "__asan_load16_noabort", 1125 "__asan_load16_noabort",
1126 "__asan_storeN_noabort", 1126 "__asan_storeN_noabort",
1127 "__asan_store1_noabort", 1127 "__asan_store1_noabort",
1128 "__asan_store2_noabort", 1128 "__asan_store2_noabort",
1129 "__asan_store4_noabort", 1129 "__asan_store4_noabort",
1130 "__asan_store8_noabort", 1130 "__asan_store8_noabort",
1131 "__asan_store16_noabort", 1131 "__asan_store16_noabort",
1132 "__kasan_check_read", 1132 "__kasan_check_read",
1133 "__kasan_check_write", 1133 "__kasan_check_write",
1134 /* KASAN in-line */ 1134 /* KASAN in-line */
1135 "__asan_report_load_n_noabort", 1135 "__asan_report_load_n_noabort",
1136 "__asan_report_load1_noabort", 1136 "__asan_report_load1_noabort",
1137 "__asan_report_load2_noabort", 1137 "__asan_report_load2_noabort",
1138 "__asan_report_load4_noabort", 1138 "__asan_report_load4_noabort",
1139 "__asan_report_load8_noabort", 1139 "__asan_report_load8_noabort",
1140 "__asan_report_load16_noabort", 1140 "__asan_report_load16_noabort",
1141 "__asan_report_store_n_noabort", 1141 "__asan_report_store_n_noabort",
1142 "__asan_report_store1_noabort", 1142 "__asan_report_store1_noabort",
1143 "__asan_report_store2_noabort", 1143 "__asan_report_store2_noabort",
1144 "__asan_report_store4_noabort", 1144 "__asan_report_store4_noabort",
1145 "__asan_report_store8_noabort", 1145 "__asan_report_store8_noabort",
1146 "__asan_report_store16_noabort", 1146 "__asan_report_store16_noabort",
1147 /* KCSAN */ 1147 /* KCSAN */
1148 "__kcsan_check_access", 1148 "__kcsan_check_access",
1149 "__kcsan_mb", 1149 "__kcsan_mb",
1150 "__kcsan_wmb", 1150 "__kcsan_wmb",
1151 "__kcsan_rmb", 1151 "__kcsan_rmb",
1152 "__kcsan_release", 1152 "__kcsan_release",
1153 "kcsan_found_watchpoint", 1153 "kcsan_found_watchpoint",
1154 "kcsan_setup_watchpoint", 1154 "kcsan_setup_watchpoint",
1155 "kcsan_check_scoped_accesses", 1155 "kcsan_check_scoped_accesses",
1156 "kcsan_disable_current", 1156 "kcsan_disable_current",
1157 "kcsan_enable_current_nowarn", 1157 "kcsan_enable_current_nowarn",
1158 /* KCSAN/TSAN */ 1158 /* KCSAN/TSAN */
1159 "__tsan_func_entry", 1159 "__tsan_func_entry",
1160 "__tsan_func_exit", 1160 "__tsan_func_exit",
1161 "__tsan_read_range", 1161 "__tsan_read_range",
1162 "__tsan_write_range", 1162 "__tsan_write_range",
1163 "__tsan_read1", 1163 "__tsan_read1",
1164 "__tsan_read2", 1164 "__tsan_read2",
1165 "__tsan_read4", 1165 "__tsan_read4",
1166 "__tsan_read8", 1166 "__tsan_read8",
1167 "__tsan_read16", 1167 "__tsan_read16",
1168 "__tsan_write1", 1168 "__tsan_write1",
1169 "__tsan_write2", 1169 "__tsan_write2",
1170 "__tsan_write4", 1170 "__tsan_write4",
1171 "__tsan_write8", 1171 "__tsan_write8",
1172 "__tsan_write16", 1172 "__tsan_write16",
1173 "__tsan_read_write1", 1173 "__tsan_read_write1",
1174 "__tsan_read_write2", 1174 "__tsan_read_write2",
1175 "__tsan_read_write4", 1175 "__tsan_read_write4",
1176 "__tsan_read_write8", 1176 "__tsan_read_write8",
1177 "__tsan_read_write16", 1177 "__tsan_read_write16",
1178 "__tsan_volatile_read1", 1178 "__tsan_volatile_read1",
1179 "__tsan_volatile_read2", 1179 "__tsan_volatile_read2",
1180 "__tsan_volatile_read4", 1180 "__tsan_volatile_read4",
1181 "__tsan_volatile_read8", 1181 "__tsan_volatile_read8",
1182 "__tsan_volatile_read16", 1182 "__tsan_volatile_read16",
1183 "__tsan_volatile_write1", 1183 "__tsan_volatile_write1",
1184 "__tsan_volatile_write2", 1184 "__tsan_volatile_write2",
1185 "__tsan_volatile_write4", 1185 "__tsan_volatile_write4",
1186 "__tsan_volatile_write8", 1186 "__tsan_volatile_write8",
1187 "__tsan_volatile_write16", 1187 "__tsan_volatile_write16",
1188 "__tsan_atomic8_load", 1188 "__tsan_atomic8_load",
1189 "__tsan_atomic16_load", 1189 "__tsan_atomic16_load",
1190 "__tsan_atomic32_load", 1190 "__tsan_atomic32_load",
1191 "__tsan_atomic64_load", 1191 "__tsan_atomic64_load",
1192 "__tsan_atomic8_store", 1192 "__tsan_atomic8_store",
1193 "__tsan_atomic16_store", 1193 "__tsan_atomic16_store",
1194 "__tsan_atomic32_store", 1194 "__tsan_atomic32_store",
1195 "__tsan_atomic64_store", 1195 "__tsan_atomic64_store",
1196 "__tsan_atomic8_exchange", 1196 "__tsan_atomic8_exchange",
1197 "__tsan_atomic16_exchange", 1197 "__tsan_atomic16_exchange",
1198 "__tsan_atomic32_exchange", 1198 "__tsan_atomic32_exchange",
1199 "__tsan_atomic64_exchange", 1199 "__tsan_atomic64_exchange",
1200 "__tsan_atomic8_fetch_add", 1200 "__tsan_atomic8_fetch_add",
1201 "__tsan_atomic16_fetch_add", 1201 "__tsan_atomic16_fetch_add",
1202 "__tsan_atomic32_fetch_add", 1202 "__tsan_atomic32_fetch_add",
1203 "__tsan_atomic64_fetch_add", 1203 "__tsan_atomic64_fetch_add",
1204 "__tsan_atomic8_fetch_sub", 1204 "__tsan_atomic8_fetch_sub",
1205 "__tsan_atomic16_fetch_sub", 1205 "__tsan_atomic16_fetch_sub",
1206 "__tsan_atomic32_fetch_sub", 1206 "__tsan_atomic32_fetch_sub",
1207 "__tsan_atomic64_fetch_sub", 1207 "__tsan_atomic64_fetch_sub",
1208 "__tsan_atomic8_fetch_and", 1208 "__tsan_atomic8_fetch_and",
1209 "__tsan_atomic16_fetch_and", 1209 "__tsan_atomic16_fetch_and",
1210 "__tsan_atomic32_fetch_and", 1210 "__tsan_atomic32_fetch_and",
1211 "__tsan_atomic64_fetch_and", 1211 "__tsan_atomic64_fetch_and",
1212 "__tsan_atomic8_fetch_or", 1212 "__tsan_atomic8_fetch_or",
1213 "__tsan_atomic16_fetch_or", 1213 "__tsan_atomic16_fetch_or",
1214 "__tsan_atomic32_fetch_or", 1214 "__tsan_atomic32_fetch_or",
1215 "__tsan_atomic64_fetch_or", 1215 "__tsan_atomic64_fetch_or",
1216 "__tsan_atomic8_fetch_xor", 1216 "__tsan_atomic8_fetch_xor",
1217 "__tsan_atomic16_fetch_xor", 1217 "__tsan_atomic16_fetch_xor",
1218 "__tsan_atomic32_fetch_xor", 1218 "__tsan_atomic32_fetch_xor",
1219 "__tsan_atomic64_fetch_xor", 1219 "__tsan_atomic64_fetch_xor",
1220 "__tsan_atomic8_fetch_nand", 1220 "__tsan_atomic8_fetch_nand",
1221 "__tsan_atomic16_fetch_nand", 1221 "__tsan_atomic16_fetch_nand",
1222 "__tsan_atomic32_fetch_nand", 1222 "__tsan_atomic32_fetch_nand",
1223 "__tsan_atomic64_fetch_nand", 1223 "__tsan_atomic64_fetch_nand",
1224 "__tsan_atomic8_compare_exchange_stro 1224 "__tsan_atomic8_compare_exchange_strong",
1225 "__tsan_atomic16_compare_exchange_str 1225 "__tsan_atomic16_compare_exchange_strong",
1226 "__tsan_atomic32_compare_exchange_str 1226 "__tsan_atomic32_compare_exchange_strong",
1227 "__tsan_atomic64_compare_exchange_str 1227 "__tsan_atomic64_compare_exchange_strong",
1228 "__tsan_atomic8_compare_exchange_weak 1228 "__tsan_atomic8_compare_exchange_weak",
1229 "__tsan_atomic16_compare_exchange_wea 1229 "__tsan_atomic16_compare_exchange_weak",
1230 "__tsan_atomic32_compare_exchange_wea 1230 "__tsan_atomic32_compare_exchange_weak",
1231 "__tsan_atomic64_compare_exchange_wea 1231 "__tsan_atomic64_compare_exchange_weak",
1232 "__tsan_atomic8_compare_exchange_val" 1232 "__tsan_atomic8_compare_exchange_val",
1233 "__tsan_atomic16_compare_exchange_val 1233 "__tsan_atomic16_compare_exchange_val",
1234 "__tsan_atomic32_compare_exchange_val 1234 "__tsan_atomic32_compare_exchange_val",
1235 "__tsan_atomic64_compare_exchange_val 1235 "__tsan_atomic64_compare_exchange_val",
1236 "__tsan_atomic_thread_fence", 1236 "__tsan_atomic_thread_fence",
1237 "__tsan_atomic_signal_fence", 1237 "__tsan_atomic_signal_fence",
1238 "__tsan_unaligned_read16", 1238 "__tsan_unaligned_read16",
1239 "__tsan_unaligned_write16", 1239 "__tsan_unaligned_write16",
1240 /* KCOV */ 1240 /* KCOV */
1241 "write_comp_data", 1241 "write_comp_data",
1242 "check_kcov_mode", 1242 "check_kcov_mode",
1243 "__sanitizer_cov_trace_pc", 1243 "__sanitizer_cov_trace_pc",
1244 "__sanitizer_cov_trace_const_cmp1", 1244 "__sanitizer_cov_trace_const_cmp1",
1245 "__sanitizer_cov_trace_const_cmp2", 1245 "__sanitizer_cov_trace_const_cmp2",
1246 "__sanitizer_cov_trace_const_cmp4", 1246 "__sanitizer_cov_trace_const_cmp4",
1247 "__sanitizer_cov_trace_const_cmp8", 1247 "__sanitizer_cov_trace_const_cmp8",
1248 "__sanitizer_cov_trace_cmp1", 1248 "__sanitizer_cov_trace_cmp1",
1249 "__sanitizer_cov_trace_cmp2", 1249 "__sanitizer_cov_trace_cmp2",
1250 "__sanitizer_cov_trace_cmp4", 1250 "__sanitizer_cov_trace_cmp4",
1251 "__sanitizer_cov_trace_cmp8", 1251 "__sanitizer_cov_trace_cmp8",
1252 "__sanitizer_cov_trace_switch", 1252 "__sanitizer_cov_trace_switch",
1253 /* KMSAN */ 1253 /* KMSAN */
1254 "kmsan_copy_to_user", 1254 "kmsan_copy_to_user",
1255 "kmsan_disable_current", 1255 "kmsan_disable_current",
1256 "kmsan_enable_current", 1256 "kmsan_enable_current",
1257 "kmsan_report", 1257 "kmsan_report",
1258 "kmsan_unpoison_entry_regs", 1258 "kmsan_unpoison_entry_regs",
1259 "kmsan_unpoison_memory", 1259 "kmsan_unpoison_memory",
1260 "__msan_chain_origin", 1260 "__msan_chain_origin",
1261 "__msan_get_context_state", 1261 "__msan_get_context_state",
1262 "__msan_instrument_asm_store", 1262 "__msan_instrument_asm_store",
1263 "__msan_metadata_ptr_for_load_1", 1263 "__msan_metadata_ptr_for_load_1",
1264 "__msan_metadata_ptr_for_load_2", 1264 "__msan_metadata_ptr_for_load_2",
1265 "__msan_metadata_ptr_for_load_4", 1265 "__msan_metadata_ptr_for_load_4",
1266 "__msan_metadata_ptr_for_load_8", 1266 "__msan_metadata_ptr_for_load_8",
1267 "__msan_metadata_ptr_for_load_n", 1267 "__msan_metadata_ptr_for_load_n",
1268 "__msan_metadata_ptr_for_store_1", 1268 "__msan_metadata_ptr_for_store_1",
1269 "__msan_metadata_ptr_for_store_2", 1269 "__msan_metadata_ptr_for_store_2",
1270 "__msan_metadata_ptr_for_store_4", 1270 "__msan_metadata_ptr_for_store_4",
1271 "__msan_metadata_ptr_for_store_8", 1271 "__msan_metadata_ptr_for_store_8",
1272 "__msan_metadata_ptr_for_store_n", 1272 "__msan_metadata_ptr_for_store_n",
1273 "__msan_poison_alloca", 1273 "__msan_poison_alloca",
1274 "__msan_warning", 1274 "__msan_warning",
1275 /* UBSAN */ 1275 /* UBSAN */
1276 "ubsan_type_mismatch_common", 1276 "ubsan_type_mismatch_common",
1277 "__ubsan_handle_type_mismatch", 1277 "__ubsan_handle_type_mismatch",
1278 "__ubsan_handle_type_mismatch_v1", 1278 "__ubsan_handle_type_mismatch_v1",
1279 "__ubsan_handle_shift_out_of_bounds", 1279 "__ubsan_handle_shift_out_of_bounds",
1280 "__ubsan_handle_load_invalid_value", 1280 "__ubsan_handle_load_invalid_value",
1281 /* STACKLEAK */ 1281 /* STACKLEAK */
1282 "stackleak_track_stack", 1282 "stackleak_track_stack",
1283 /* misc */ 1283 /* misc */
1284 "csum_partial_copy_generic", 1284 "csum_partial_copy_generic",
1285 "copy_mc_fragile", 1285 "copy_mc_fragile",
1286 "copy_mc_fragile_handle_tail", 1286 "copy_mc_fragile_handle_tail",
1287 "copy_mc_enhanced_fast_string", 1287 "copy_mc_enhanced_fast_string",
1288 "ftrace_likely_update", /* CONFIG_TRA 1288 "ftrace_likely_update", /* CONFIG_TRACE_BRANCH_PROFILING */
1289 "rep_stos_alternative", 1289 "rep_stos_alternative",
1290 "rep_movs_alternative", 1290 "rep_movs_alternative",
1291 "__copy_user_nocache", 1291 "__copy_user_nocache",
1292 NULL 1292 NULL
1293 }; 1293 };
1294 1294
1295 static void add_uaccess_safe(struct objtool_f 1295 static void add_uaccess_safe(struct objtool_file *file)
1296 { 1296 {
1297 struct symbol *func; 1297 struct symbol *func;
1298 const char **name; 1298 const char **name;
1299 1299
1300 if (!opts.uaccess) 1300 if (!opts.uaccess)
1301 return; 1301 return;
1302 1302
1303 for (name = uaccess_safe_builtin; *na 1303 for (name = uaccess_safe_builtin; *name; name++) {
1304 func = find_symbol_by_name(fi 1304 func = find_symbol_by_name(file->elf, *name);
1305 if (!func) 1305 if (!func)
1306 continue; 1306 continue;
1307 1307
1308 func->uaccess_safe = true; 1308 func->uaccess_safe = true;
1309 } 1309 }
1310 } 1310 }
1311 1311
1312 /* 1312 /*
1313 * FIXME: For now, just ignore any alternativ 1313 * FIXME: For now, just ignore any alternatives which add retpolines. This is
1314 * a temporary hack, as it doesn't allow ORC 1314 * a temporary hack, as it doesn't allow ORC to unwind from inside a retpoline.
1315 * But it at least allows objtool to understa 1315 * But it at least allows objtool to understand the control flow *around* the
1316 * retpoline. 1316 * retpoline.
1317 */ 1317 */
1318 static int add_ignore_alternatives(struct obj 1318 static int add_ignore_alternatives(struct objtool_file *file)
1319 { 1319 {
1320 struct section *rsec; 1320 struct section *rsec;
1321 struct reloc *reloc; 1321 struct reloc *reloc;
1322 struct instruction *insn; 1322 struct instruction *insn;
1323 1323
1324 rsec = find_section_by_name(file->elf 1324 rsec = find_section_by_name(file->elf, ".rela.discard.ignore_alts");
1325 if (!rsec) 1325 if (!rsec)
1326 return 0; 1326 return 0;
1327 1327
1328 for_each_reloc(rsec, reloc) { 1328 for_each_reloc(rsec, reloc) {
1329 if (reloc->sym->type != STT_S 1329 if (reloc->sym->type != STT_SECTION) {
1330 WARN("unexpected relo 1330 WARN("unexpected relocation symbol type in %s", rsec->name);
1331 return -1; 1331 return -1;
1332 } 1332 }
1333 1333
1334 insn = find_insn(file, reloc- 1334 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
1335 if (!insn) { 1335 if (!insn) {
1336 WARN("bad .discard.ig 1336 WARN("bad .discard.ignore_alts entry");
1337 return -1; 1337 return -1;
1338 } 1338 }
1339 1339
1340 insn->ignore_alts = true; 1340 insn->ignore_alts = true;
1341 } 1341 }
1342 1342
1343 return 0; 1343 return 0;
1344 } 1344 }
1345 1345
1346 /* 1346 /*
1347 * Symbols that replace INSN_CALL_DYNAMIC, ev 1347 * Symbols that replace INSN_CALL_DYNAMIC, every (tail) call to such a symbol
1348 * will be added to the .retpoline_sites sect 1348 * will be added to the .retpoline_sites section.
1349 */ 1349 */
1350 __weak bool arch_is_retpoline(struct symbol * 1350 __weak bool arch_is_retpoline(struct symbol *sym)
1351 { 1351 {
1352 return false; 1352 return false;
1353 } 1353 }
1354 1354
1355 /* 1355 /*
1356 * Symbols that replace INSN_RETURN, every (t 1356 * Symbols that replace INSN_RETURN, every (tail) call to such a symbol
1357 * will be added to the .return_sites section 1357 * will be added to the .return_sites section.
1358 */ 1358 */
1359 __weak bool arch_is_rethunk(struct symbol *sy 1359 __weak bool arch_is_rethunk(struct symbol *sym)
1360 { 1360 {
1361 return false; 1361 return false;
1362 } 1362 }
1363 1363
1364 /* 1364 /*
1365 * Symbols that are embedded inside other ins 1365 * Symbols that are embedded inside other instructions, because sometimes crazy
1366 * code exists. These are mostly ignored for 1366 * code exists. These are mostly ignored for validation purposes.
1367 */ 1367 */
1368 __weak bool arch_is_embedded_insn(struct symb 1368 __weak bool arch_is_embedded_insn(struct symbol *sym)
1369 { 1369 {
1370 return false; 1370 return false;
1371 } 1371 }
1372 1372
1373 static struct reloc *insn_reloc(struct objtoo 1373 static struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn)
1374 { 1374 {
1375 struct reloc *reloc; 1375 struct reloc *reloc;
1376 1376
1377 if (insn->no_reloc) 1377 if (insn->no_reloc)
1378 return NULL; 1378 return NULL;
1379 1379
1380 if (!file) 1380 if (!file)
1381 return NULL; 1381 return NULL;
1382 1382
1383 reloc = find_reloc_by_dest_range(file 1383 reloc = find_reloc_by_dest_range(file->elf, insn->sec,
1384 insn 1384 insn->offset, insn->len);
1385 if (!reloc) { 1385 if (!reloc) {
1386 insn->no_reloc = 1; 1386 insn->no_reloc = 1;
1387 return NULL; 1387 return NULL;
1388 } 1388 }
1389 1389
1390 return reloc; 1390 return reloc;
1391 } 1391 }
1392 1392
1393 static void remove_insn_ops(struct instructio 1393 static void remove_insn_ops(struct instruction *insn)
1394 { 1394 {
1395 struct stack_op *op, *next; 1395 struct stack_op *op, *next;
1396 1396
1397 for (op = insn->stack_ops; op; op = n 1397 for (op = insn->stack_ops; op; op = next) {
1398 next = op->next; 1398 next = op->next;
1399 free(op); 1399 free(op);
1400 } 1400 }
1401 insn->stack_ops = NULL; 1401 insn->stack_ops = NULL;
1402 } 1402 }
1403 1403
1404 static void annotate_call_site(struct objtool 1404 static void annotate_call_site(struct objtool_file *file,
1405 struct instruc 1405 struct instruction *insn, bool sibling)
1406 { 1406 {
1407 struct reloc *reloc = insn_reloc(file 1407 struct reloc *reloc = insn_reloc(file, insn);
1408 struct symbol *sym = insn_call_dest(i 1408 struct symbol *sym = insn_call_dest(insn);
1409 1409
1410 if (!sym) 1410 if (!sym)
1411 sym = reloc->sym; 1411 sym = reloc->sym;
1412 1412
1413 /* 1413 /*
1414 * Alternative replacement code is ju 1414 * Alternative replacement code is just template code which is
1415 * sometimes copied to the original i 1415 * sometimes copied to the original instruction. For now, don't
1416 * annotate it. (In the future we mig 1416 * annotate it. (In the future we might consider annotating the
1417 * original instruction if/when it ev 1417 * original instruction if/when it ever makes sense to do so.)
1418 */ 1418 */
1419 if (!strcmp(insn->sec->name, ".altins 1419 if (!strcmp(insn->sec->name, ".altinstr_replacement"))
1420 return; 1420 return;
1421 1421
1422 if (sym->static_call_tramp) { 1422 if (sym->static_call_tramp) {
1423 list_add_tail(&insn->call_nod 1423 list_add_tail(&insn->call_node, &file->static_call_list);
1424 return; 1424 return;
1425 } 1425 }
1426 1426
1427 if (sym->retpoline_thunk) { 1427 if (sym->retpoline_thunk) {
1428 list_add_tail(&insn->call_nod 1428 list_add_tail(&insn->call_node, &file->retpoline_call_list);
1429 return; 1429 return;
1430 } 1430 }
1431 1431
1432 /* 1432 /*
1433 * Many compilers cannot disable KCOV 1433 * Many compilers cannot disable KCOV or sanitizer calls with a function
1434 * attribute so they need a little he 1434 * attribute so they need a little help, NOP out any such calls from
1435 * noinstr text. 1435 * noinstr text.
1436 */ 1436 */
1437 if (opts.hack_noinstr && insn->sec->n 1437 if (opts.hack_noinstr && insn->sec->noinstr && sym->profiling_func) {
1438 if (reloc) 1438 if (reloc)
1439 set_reloc_type(file-> 1439 set_reloc_type(file->elf, reloc, R_NONE);
1440 1440
1441 elf_write_insn(file->elf, ins 1441 elf_write_insn(file->elf, insn->sec,
1442 insn->offset, 1442 insn->offset, insn->len,
1443 sibling ? arch 1443 sibling ? arch_ret_insn(insn->len)
1444 : arch 1444 : arch_nop_insn(insn->len));
1445 1445
1446 insn->type = sibling ? INSN_R 1446 insn->type = sibling ? INSN_RETURN : INSN_NOP;
1447 1447
1448 if (sibling) { 1448 if (sibling) {
1449 /* 1449 /*
1450 * We've replaced the 1450 * We've replaced the tail-call JMP insn by two new
1451 * insn: RET; INT3, e 1451 * insn: RET; INT3, except we only have a single struct
1452 * insn here. Mark it 1452 * insn here. Mark it retpoline_safe to avoid the SLS
1453 * warning, instead o 1453 * warning, instead of adding another insn.
1454 */ 1454 */
1455 insn->retpoline_safe 1455 insn->retpoline_safe = true;
1456 } 1456 }
1457 1457
1458 return; 1458 return;
1459 } 1459 }
1460 1460
1461 if (opts.mcount && sym->fentry) { 1461 if (opts.mcount && sym->fentry) {
1462 if (sibling) 1462 if (sibling)
1463 WARN_INSN(insn, "tail 1463 WARN_INSN(insn, "tail call to __fentry__ !?!?");
1464 if (opts.mnop) { 1464 if (opts.mnop) {
1465 if (reloc) 1465 if (reloc)
1466 set_reloc_typ 1466 set_reloc_type(file->elf, reloc, R_NONE);
1467 1467
1468 elf_write_insn(file-> 1468 elf_write_insn(file->elf, insn->sec,
1469 insn-> 1469 insn->offset, insn->len,
1470 arch_n 1470 arch_nop_insn(insn->len));
1471 1471
1472 insn->type = INSN_NOP 1472 insn->type = INSN_NOP;
1473 } 1473 }
1474 1474
1475 list_add_tail(&insn->call_nod 1475 list_add_tail(&insn->call_node, &file->mcount_loc_list);
1476 return; 1476 return;
1477 } 1477 }
1478 1478
1479 if (insn->type == INSN_CALL && !insn- 1479 if (insn->type == INSN_CALL && !insn->sec->init)
1480 list_add_tail(&insn->call_nod 1480 list_add_tail(&insn->call_node, &file->call_list);
1481 1481
1482 if (!sibling && dead_end_function(fil 1482 if (!sibling && dead_end_function(file, sym))
1483 insn->dead_end = true; 1483 insn->dead_end = true;
1484 } 1484 }
1485 1485
1486 static void add_call_dest(struct objtool_file 1486 static void add_call_dest(struct objtool_file *file, struct instruction *insn,
1487 struct symbol *dest 1487 struct symbol *dest, bool sibling)
1488 { 1488 {
1489 insn->_call_dest = dest; 1489 insn->_call_dest = dest;
1490 if (!dest) 1490 if (!dest)
1491 return; 1491 return;
1492 1492
1493 /* 1493 /*
1494 * Whatever stack impact regular CALL 1494 * Whatever stack impact regular CALLs have, should be undone
1495 * by the RETURN of the called functi 1495 * by the RETURN of the called function.
1496 * 1496 *
1497 * Annotated intra-function calls ret 1497 * Annotated intra-function calls retain the stack_ops but
1498 * are converted to JUMP, see read_in 1498 * are converted to JUMP, see read_intra_function_calls().
1499 */ 1499 */
1500 remove_insn_ops(insn); 1500 remove_insn_ops(insn);
1501 1501
1502 annotate_call_site(file, insn, siblin 1502 annotate_call_site(file, insn, sibling);
1503 } 1503 }
1504 1504
1505 static void add_retpoline_call(struct objtool 1505 static void add_retpoline_call(struct objtool_file *file, struct instruction *insn)
1506 { 1506 {
1507 /* 1507 /*
1508 * Retpoline calls/jumps are really d 1508 * Retpoline calls/jumps are really dynamic calls/jumps in disguise,
1509 * so convert them accordingly. 1509 * so convert them accordingly.
1510 */ 1510 */
1511 switch (insn->type) { 1511 switch (insn->type) {
1512 case INSN_CALL: 1512 case INSN_CALL:
1513 insn->type = INSN_CALL_DYNAMI 1513 insn->type = INSN_CALL_DYNAMIC;
1514 break; 1514 break;
1515 case INSN_JUMP_UNCONDITIONAL: 1515 case INSN_JUMP_UNCONDITIONAL:
1516 insn->type = INSN_JUMP_DYNAMI 1516 insn->type = INSN_JUMP_DYNAMIC;
1517 break; 1517 break;
1518 case INSN_JUMP_CONDITIONAL: 1518 case INSN_JUMP_CONDITIONAL:
1519 insn->type = INSN_JUMP_DYNAMI 1519 insn->type = INSN_JUMP_DYNAMIC_CONDITIONAL;
1520 break; 1520 break;
1521 default: 1521 default:
1522 return; 1522 return;
1523 } 1523 }
1524 1524
1525 insn->retpoline_safe = true; 1525 insn->retpoline_safe = true;
1526 1526
1527 /* 1527 /*
1528 * Whatever stack impact regular CALL 1528 * Whatever stack impact regular CALLs have, should be undone
1529 * by the RETURN of the called functi 1529 * by the RETURN of the called function.
1530 * 1530 *
1531 * Annotated intra-function calls ret 1531 * Annotated intra-function calls retain the stack_ops but
1532 * are converted to JUMP, see read_in 1532 * are converted to JUMP, see read_intra_function_calls().
1533 */ 1533 */
1534 remove_insn_ops(insn); 1534 remove_insn_ops(insn);
1535 1535
1536 annotate_call_site(file, insn, false) 1536 annotate_call_site(file, insn, false);
1537 } 1537 }
1538 1538
1539 static void add_return_call(struct objtool_fi 1539 static void add_return_call(struct objtool_file *file, struct instruction *insn, bool add)
1540 { 1540 {
1541 /* 1541 /*
1542 * Return thunk tail calls are really 1542 * Return thunk tail calls are really just returns in disguise,
1543 * so convert them accordingly. 1543 * so convert them accordingly.
1544 */ 1544 */
1545 insn->type = INSN_RETURN; 1545 insn->type = INSN_RETURN;
1546 insn->retpoline_safe = true; 1546 insn->retpoline_safe = true;
1547 1547
1548 if (add) 1548 if (add)
1549 list_add_tail(&insn->call_nod 1549 list_add_tail(&insn->call_node, &file->return_thunk_list);
1550 } 1550 }
1551 1551
1552 static bool is_first_func_insn(struct objtool 1552 static bool is_first_func_insn(struct objtool_file *file,
1553 struct instruc 1553 struct instruction *insn, struct symbol *sym)
1554 { 1554 {
1555 if (insn->offset == sym->offset) 1555 if (insn->offset == sym->offset)
1556 return true; 1556 return true;
1557 1557
1558 /* Allow direct CALL/JMP past ENDBR * 1558 /* Allow direct CALL/JMP past ENDBR */
1559 if (opts.ibt) { 1559 if (opts.ibt) {
1560 struct instruction *prev = pr 1560 struct instruction *prev = prev_insn_same_sym(file, insn);
1561 1561
1562 if (prev && prev->type == INS 1562 if (prev && prev->type == INSN_ENDBR &&
1563 insn->offset == sym->offs 1563 insn->offset == sym->offset + prev->len)
1564 return true; 1564 return true;
1565 } 1565 }
1566 1566
1567 return false; 1567 return false;
1568 } 1568 }
1569 1569
1570 /* 1570 /*
1571 * A sibling call is a tail-call to another s 1571 * A sibling call is a tail-call to another symbol -- to differentiate from a
1572 * recursive tail-call which is to the same s 1572 * recursive tail-call which is to the same symbol.
1573 */ 1573 */
1574 static bool jump_is_sibling_call(struct objto 1574 static bool jump_is_sibling_call(struct objtool_file *file,
1575 struct instr 1575 struct instruction *from, struct instruction *to)
1576 { 1576 {
1577 struct symbol *fs = from->sym; 1577 struct symbol *fs = from->sym;
1578 struct symbol *ts = to->sym; 1578 struct symbol *ts = to->sym;
1579 1579
1580 /* Not a sibling call if from/to a sy 1580 /* Not a sibling call if from/to a symbol hole */
1581 if (!fs || !ts) 1581 if (!fs || !ts)
1582 return false; 1582 return false;
1583 1583
1584 /* Not a sibling call if not targetin 1584 /* Not a sibling call if not targeting the start of a symbol. */
1585 if (!is_first_func_insn(file, to, ts) 1585 if (!is_first_func_insn(file, to, ts))
1586 return false; 1586 return false;
1587 1587
1588 /* Disallow sibling calls into STT_NO 1588 /* Disallow sibling calls into STT_NOTYPE */
1589 if (ts->type == STT_NOTYPE) 1589 if (ts->type == STT_NOTYPE)
1590 return false; 1590 return false;
1591 1591
1592 /* Must not be self to be a sibling * 1592 /* Must not be self to be a sibling */
1593 return fs->pfunc != ts->pfunc; 1593 return fs->pfunc != ts->pfunc;
1594 } 1594 }
1595 1595
1596 /* 1596 /*
1597 * Find the destination instructions for all 1597 * Find the destination instructions for all jumps.
1598 */ 1598 */
1599 static int add_jump_destinations(struct objto 1599 static int add_jump_destinations(struct objtool_file *file)
1600 { 1600 {
1601 struct instruction *insn, *jump_dest; 1601 struct instruction *insn, *jump_dest;
1602 struct reloc *reloc; 1602 struct reloc *reloc;
1603 struct section *dest_sec; 1603 struct section *dest_sec;
1604 unsigned long dest_off; 1604 unsigned long dest_off;
1605 1605
1606 for_each_insn(file, insn) { 1606 for_each_insn(file, insn) {
1607 if (insn->jump_dest) { 1607 if (insn->jump_dest) {
1608 /* 1608 /*
1609 * handle_group_alt() 1609 * handle_group_alt() may have previously set
1610 * 'jump_dest' for so 1610 * 'jump_dest' for some alternatives.
1611 */ 1611 */
1612 continue; 1612 continue;
1613 } 1613 }
1614 if (!is_static_jump(insn)) 1614 if (!is_static_jump(insn))
1615 continue; 1615 continue;
1616 1616
1617 reloc = insn_reloc(file, insn 1617 reloc = insn_reloc(file, insn);
1618 if (!reloc) { 1618 if (!reloc) {
1619 dest_sec = insn->sec; 1619 dest_sec = insn->sec;
1620 dest_off = arch_jump_ 1620 dest_off = arch_jump_destination(insn);
1621 } else if (reloc->sym->type = 1621 } else if (reloc->sym->type == STT_SECTION) {
1622 dest_sec = reloc->sym 1622 dest_sec = reloc->sym->sec;
1623 dest_off = arch_dest_ 1623 dest_off = arch_dest_reloc_offset(reloc_addend(reloc));
1624 } else if (reloc->sym->retpol 1624 } else if (reloc->sym->retpoline_thunk) {
1625 add_retpoline_call(fi 1625 add_retpoline_call(file, insn);
1626 continue; 1626 continue;
1627 } else if (reloc->sym->return 1627 } else if (reloc->sym->return_thunk) {
1628 add_return_call(file, 1628 add_return_call(file, insn, true);
1629 continue; 1629 continue;
1630 } else if (insn_func(insn)) { 1630 } else if (insn_func(insn)) {
1631 /* 1631 /*
1632 * External sibling c 1632 * External sibling call or internal sibling call with
1633 * STT_FUNC reloc. 1633 * STT_FUNC reloc.
1634 */ 1634 */
1635 add_call_dest(file, i 1635 add_call_dest(file, insn, reloc->sym, true);
1636 continue; 1636 continue;
1637 } else if (reloc->sym->sec->i 1637 } else if (reloc->sym->sec->idx) {
1638 dest_sec = reloc->sym 1638 dest_sec = reloc->sym->sec;
1639 dest_off = reloc->sym 1639 dest_off = reloc->sym->sym.st_value +
1640 arch_dest_ 1640 arch_dest_reloc_offset(reloc_addend(reloc));
1641 } else { 1641 } else {
1642 /* non-func asm code 1642 /* non-func asm code jumping to another file */
1643 continue; 1643 continue;
1644 } 1644 }
1645 1645
1646 jump_dest = find_insn(file, d 1646 jump_dest = find_insn(file, dest_sec, dest_off);
1647 if (!jump_dest) { 1647 if (!jump_dest) {
1648 struct symbol *sym = 1648 struct symbol *sym = find_symbol_by_offset(dest_sec, dest_off);
1649 1649
1650 /* 1650 /*
1651 * This is a special 1651 * This is a special case for retbleed_untrain_ret().
1652 * It jumps to __x86_ 1652 * It jumps to __x86_return_thunk(), but objtool
1653 * can't find the thu 1653 * can't find the thunk's starting RET
1654 * instruction, becau 1654 * instruction, because the RET is also in the
1655 * middle of another 1655 * middle of another instruction. Objtool only
1656 * knows about the ou 1656 * knows about the outer instruction.
1657 */ 1657 */
1658 if (sym && sym->embed 1658 if (sym && sym->embedded_insn) {
1659 add_return_ca 1659 add_return_call(file, insn, false);
1660 continue; 1660 continue;
1661 } 1661 }
1662 1662
1663 WARN_INSN(insn, "can' 1663 WARN_INSN(insn, "can't find jump dest instruction at %s+0x%lx",
1664 dest_sec->n 1664 dest_sec->name, dest_off);
1665 return -1; 1665 return -1;
1666 } 1666 }
1667 1667
1668 /* 1668 /*
1669 * An intra-TU jump in retpol 1669 * An intra-TU jump in retpoline.o might not have a relocation
1670 * for its jump dest, in whic 1670 * for its jump dest, in which case the above
1671 * add_{retpoline,return}_cal 1671 * add_{retpoline,return}_call() didn't happen.
1672 */ 1672 */
1673 if (jump_dest->sym && jump_de 1673 if (jump_dest->sym && jump_dest->offset == jump_dest->sym->offset) {
1674 if (jump_dest->sym->r 1674 if (jump_dest->sym->retpoline_thunk) {
1675 add_retpoline 1675 add_retpoline_call(file, insn);
1676 continue; 1676 continue;
1677 } 1677 }
1678 if (jump_dest->sym->r 1678 if (jump_dest->sym->return_thunk) {
1679 add_return_ca 1679 add_return_call(file, insn, true);
1680 continue; 1680 continue;
1681 } 1681 }
1682 } 1682 }
1683 1683
1684 /* 1684 /*
1685 * Cross-function jump. 1685 * Cross-function jump.
1686 */ 1686 */
1687 if (insn_func(insn) && insn_f 1687 if (insn_func(insn) && insn_func(jump_dest) &&
1688 insn_func(insn) != insn_f 1688 insn_func(insn) != insn_func(jump_dest)) {
1689 1689
1690 /* 1690 /*
1691 * For GCC 8+, create 1691 * For GCC 8+, create parent/child links for any cold
1692 * subfunctions. Thi 1692 * subfunctions. This is _mostly_ redundant with a
1693 * similar initializa 1693 * similar initialization in read_symbols().
1694 * 1694 *
1695 * If a function has 1695 * If a function has aliases, we want the *first* such
1696 * function in the sy 1696 * function in the symbol table to be the subfunction's
1697 * parent. In that c 1697 * parent. In that case we overwrite the
1698 * initialization don 1698 * initialization done in read_symbols().
1699 * 1699 *
1700 * However this code 1700 * However this code can't completely replace the
1701 * read_symbols() cod 1701 * read_symbols() code because this doesn't detect the
1702 * case where the par 1702 * case where the parent function's only reference to a
1703 * subfunction is thr 1703 * subfunction is through a jump table.
1704 */ 1704 */
1705 if (!strstr(insn_func 1705 if (!strstr(insn_func(insn)->name, ".cold") &&
1706 strstr(insn_func( 1706 strstr(insn_func(jump_dest)->name, ".cold")) {
1707 insn_func(ins 1707 insn_func(insn)->cfunc = insn_func(jump_dest);
1708 insn_func(jum 1708 insn_func(jump_dest)->pfunc = insn_func(insn);
1709 } 1709 }
1710 } 1710 }
1711 1711
1712 if (jump_is_sibling_call(file 1712 if (jump_is_sibling_call(file, insn, jump_dest)) {
1713 /* 1713 /*
1714 * Internal sibling c 1714 * Internal sibling call without reloc or with
1715 * STT_SECTION reloc. 1715 * STT_SECTION reloc.
1716 */ 1716 */
1717 add_call_dest(file, i 1717 add_call_dest(file, insn, insn_func(jump_dest), true);
1718 continue; 1718 continue;
1719 } 1719 }
1720 1720
1721 insn->jump_dest = jump_dest; 1721 insn->jump_dest = jump_dest;
1722 } 1722 }
1723 1723
1724 return 0; 1724 return 0;
1725 } 1725 }
1726 1726
1727 static struct symbol *find_call_destination(s 1727 static struct symbol *find_call_destination(struct section *sec, unsigned long offset)
1728 { 1728 {
1729 struct symbol *call_dest; 1729 struct symbol *call_dest;
1730 1730
1731 call_dest = find_func_by_offset(sec, 1731 call_dest = find_func_by_offset(sec, offset);
1732 if (!call_dest) 1732 if (!call_dest)
1733 call_dest = find_symbol_by_of 1733 call_dest = find_symbol_by_offset(sec, offset);
1734 1734
1735 return call_dest; 1735 return call_dest;
1736 } 1736 }
1737 1737
1738 /* 1738 /*
1739 * Find the destination instructions for all 1739 * Find the destination instructions for all calls.
1740 */ 1740 */
1741 static int add_call_destinations(struct objto 1741 static int add_call_destinations(struct objtool_file *file)
1742 { 1742 {
1743 struct instruction *insn; 1743 struct instruction *insn;
1744 unsigned long dest_off; 1744 unsigned long dest_off;
1745 struct symbol *dest; 1745 struct symbol *dest;
1746 struct reloc *reloc; 1746 struct reloc *reloc;
1747 1747
1748 for_each_insn(file, insn) { 1748 for_each_insn(file, insn) {
1749 if (insn->type != INSN_CALL) 1749 if (insn->type != INSN_CALL)
1750 continue; 1750 continue;
1751 1751
1752 reloc = insn_reloc(file, insn 1752 reloc = insn_reloc(file, insn);
1753 if (!reloc) { 1753 if (!reloc) {
1754 dest_off = arch_jump_ 1754 dest_off = arch_jump_destination(insn);
1755 dest = find_call_dest 1755 dest = find_call_destination(insn->sec, dest_off);
1756 1756
1757 add_call_dest(file, i 1757 add_call_dest(file, insn, dest, false);
1758 1758
1759 if (insn->ignore) 1759 if (insn->ignore)
1760 continue; 1760 continue;
1761 1761
1762 if (!insn_call_dest(i 1762 if (!insn_call_dest(insn)) {
1763 WARN_INSN(ins 1763 WARN_INSN(insn, "unannotated intra-function call");
1764 return -1; 1764 return -1;
1765 } 1765 }
1766 1766
1767 if (insn_func(insn) & 1767 if (insn_func(insn) && insn_call_dest(insn)->type != STT_FUNC) {
1768 WARN_INSN(ins 1768 WARN_INSN(insn, "unsupported call to non-function");
1769 return -1; 1769 return -1;
1770 } 1770 }
1771 1771
1772 } else if (reloc->sym->type = 1772 } else if (reloc->sym->type == STT_SECTION) {
1773 dest_off = arch_dest_ 1773 dest_off = arch_dest_reloc_offset(reloc_addend(reloc));
1774 dest = find_call_dest 1774 dest = find_call_destination(reloc->sym->sec, dest_off);
1775 if (!dest) { 1775 if (!dest) {
1776 WARN_INSN(ins 1776 WARN_INSN(insn, "can't find call dest symbol at %s+0x%lx",
1777 rel 1777 reloc->sym->sec->name, dest_off);
1778 return -1; 1778 return -1;
1779 } 1779 }
1780 1780
1781 add_call_dest(file, i 1781 add_call_dest(file, insn, dest, false);
1782 1782
1783 } else if (reloc->sym->retpol 1783 } else if (reloc->sym->retpoline_thunk) {
1784 add_retpoline_call(fi 1784 add_retpoline_call(file, insn);
1785 1785
1786 } else 1786 } else
1787 add_call_dest(file, i 1787 add_call_dest(file, insn, reloc->sym, false);
1788 } 1788 }
1789 1789
1790 return 0; 1790 return 0;
1791 } 1791 }
1792 1792
1793 /* 1793 /*
1794 * The .alternatives section requires some ex 1794 * The .alternatives section requires some extra special care over and above
1795 * other special sections because alternative 1795 * other special sections because alternatives are patched in place.
1796 */ 1796 */
1797 static int handle_group_alt(struct objtool_fi 1797 static int handle_group_alt(struct objtool_file *file,
1798 struct special_al 1798 struct special_alt *special_alt,
1799 struct instructio 1799 struct instruction *orig_insn,
1800 struct instructio 1800 struct instruction **new_insn)
1801 { 1801 {
1802 struct instruction *last_new_insn = N 1802 struct instruction *last_new_insn = NULL, *insn, *nop = NULL;
1803 struct alt_group *orig_alt_group, *ne 1803 struct alt_group *orig_alt_group, *new_alt_group;
1804 unsigned long dest_off; 1804 unsigned long dest_off;
1805 1805
1806 orig_alt_group = orig_insn->alt_group 1806 orig_alt_group = orig_insn->alt_group;
1807 if (!orig_alt_group) { 1807 if (!orig_alt_group) {
1808 struct instruction *last_orig 1808 struct instruction *last_orig_insn = NULL;
1809 1809
1810 orig_alt_group = malloc(sizeo 1810 orig_alt_group = malloc(sizeof(*orig_alt_group));
1811 if (!orig_alt_group) { 1811 if (!orig_alt_group) {
1812 WARN("malloc failed") 1812 WARN("malloc failed");
1813 return -1; 1813 return -1;
1814 } 1814 }
1815 orig_alt_group->cfi = calloc( 1815 orig_alt_group->cfi = calloc(special_alt->orig_len,
1816 1816 sizeof(struct cfi_state *));
1817 if (!orig_alt_group->cfi) { 1817 if (!orig_alt_group->cfi) {
1818 WARN("calloc failed") 1818 WARN("calloc failed");
1819 return -1; 1819 return -1;
1820 } 1820 }
1821 1821
1822 insn = orig_insn; 1822 insn = orig_insn;
1823 sec_for_each_insn_from(file, 1823 sec_for_each_insn_from(file, insn) {
1824 if (insn->offset >= s 1824 if (insn->offset >= special_alt->orig_off + special_alt->orig_len)
1825 break; 1825 break;
1826 1826
1827 insn->alt_group = ori 1827 insn->alt_group = orig_alt_group;
1828 last_orig_insn = insn 1828 last_orig_insn = insn;
1829 } 1829 }
1830 orig_alt_group->orig_group = 1830 orig_alt_group->orig_group = NULL;
1831 orig_alt_group->first_insn = 1831 orig_alt_group->first_insn = orig_insn;
1832 orig_alt_group->last_insn = l 1832 orig_alt_group->last_insn = last_orig_insn;
1833 orig_alt_group->nop = NULL; 1833 orig_alt_group->nop = NULL;
1834 } else { 1834 } else {
1835 if (orig_alt_group->last_insn 1835 if (orig_alt_group->last_insn->offset + orig_alt_group->last_insn->len -
1836 orig_alt_group->first_ins 1836 orig_alt_group->first_insn->offset != special_alt->orig_len) {
1837 WARN_INSN(orig_insn, 1837 WARN_INSN(orig_insn, "weirdly overlapping alternative! %ld != %d",
1838 orig_alt_gr 1838 orig_alt_group->last_insn->offset +
1839 orig_alt_gr 1839 orig_alt_group->last_insn->len -
1840 orig_alt_gr 1840 orig_alt_group->first_insn->offset,
1841 special_alt 1841 special_alt->orig_len);
1842 return -1; 1842 return -1;
1843 } 1843 }
1844 } 1844 }
1845 1845
1846 new_alt_group = malloc(sizeof(*new_al 1846 new_alt_group = malloc(sizeof(*new_alt_group));
1847 if (!new_alt_group) { 1847 if (!new_alt_group) {
1848 WARN("malloc failed"); 1848 WARN("malloc failed");
1849 return -1; 1849 return -1;
1850 } 1850 }
1851 1851
1852 if (special_alt->new_len < special_al 1852 if (special_alt->new_len < special_alt->orig_len) {
1853 /* 1853 /*
1854 * Insert a fake nop at the e 1854 * Insert a fake nop at the end to make the replacement
1855 * alt_group the same size as 1855 * alt_group the same size as the original. This is needed to
1856 * allow propagate_alt_cfi() 1856 * allow propagate_alt_cfi() to do its magic. When the last
1857 * instruction affects the st 1857 * instruction affects the stack, the instruction after it (the
1858 * nop) will propagate the ne 1858 * nop) will propagate the new state to the shared CFI array.
1859 */ 1859 */
1860 nop = malloc(sizeof(*nop)); 1860 nop = malloc(sizeof(*nop));
1861 if (!nop) { 1861 if (!nop) {
1862 WARN("malloc failed") 1862 WARN("malloc failed");
1863 return -1; 1863 return -1;
1864 } 1864 }
1865 memset(nop, 0, sizeof(*nop)); 1865 memset(nop, 0, sizeof(*nop));
1866 1866
1867 nop->sec = special_alt->new_s 1867 nop->sec = special_alt->new_sec;
1868 nop->offset = special_alt->ne 1868 nop->offset = special_alt->new_off + special_alt->new_len;
1869 nop->len = special_alt->orig_ 1869 nop->len = special_alt->orig_len - special_alt->new_len;
1870 nop->type = INSN_NOP; 1870 nop->type = INSN_NOP;
1871 nop->sym = orig_insn->sym; 1871 nop->sym = orig_insn->sym;
1872 nop->alt_group = new_alt_grou 1872 nop->alt_group = new_alt_group;
1873 nop->ignore = orig_insn->igno 1873 nop->ignore = orig_insn->ignore_alts;
1874 } 1874 }
1875 1875
1876 if (!special_alt->new_len) { 1876 if (!special_alt->new_len) {
1877 *new_insn = nop; 1877 *new_insn = nop;
1878 goto end; 1878 goto end;
1879 } 1879 }
1880 1880
1881 insn = *new_insn; 1881 insn = *new_insn;
1882 sec_for_each_insn_from(file, insn) { 1882 sec_for_each_insn_from(file, insn) {
1883 struct reloc *alt_reloc; 1883 struct reloc *alt_reloc;
1884 1884
1885 if (insn->offset >= special_a 1885 if (insn->offset >= special_alt->new_off + special_alt->new_len)
1886 break; 1886 break;
1887 1887
1888 last_new_insn = insn; 1888 last_new_insn = insn;
1889 1889
1890 insn->ignore = orig_insn->ign 1890 insn->ignore = orig_insn->ignore_alts;
1891 insn->sym = orig_insn->sym; 1891 insn->sym = orig_insn->sym;
1892 insn->alt_group = new_alt_gro 1892 insn->alt_group = new_alt_group;
1893 1893
1894 /* 1894 /*
1895 * Since alternative replacem 1895 * Since alternative replacement code is copy/pasted by the
1896 * kernel after applying relo 1896 * kernel after applying relocations, generally such code can't
1897 * have relative-address relo 1897 * have relative-address relocation references to outside the
1898 * .altinstr_replacement sect 1898 * .altinstr_replacement section, unless the arch's
1899 * alternatives code can adju 1899 * alternatives code can adjust the relative offsets
1900 * accordingly. 1900 * accordingly.
1901 */ 1901 */
1902 alt_reloc = insn_reloc(file, 1902 alt_reloc = insn_reloc(file, insn);
1903 if (alt_reloc && arch_pc_rela 1903 if (alt_reloc && arch_pc_relative_reloc(alt_reloc) &&
1904 !arch_support_alt_relocat 1904 !arch_support_alt_relocation(special_alt, insn, alt_reloc)) {
1905 1905
1906 WARN_INSN(insn, "unsu 1906 WARN_INSN(insn, "unsupported relocation in alternatives section");
1907 return -1; 1907 return -1;
1908 } 1908 }
1909 1909
1910 if (!is_static_jump(insn)) 1910 if (!is_static_jump(insn))
1911 continue; 1911 continue;
1912 1912
1913 if (!insn->immediate) 1913 if (!insn->immediate)
1914 continue; 1914 continue;
1915 1915
1916 dest_off = arch_jump_destinat 1916 dest_off = arch_jump_destination(insn);
1917 if (dest_off == special_alt-> 1917 if (dest_off == special_alt->new_off + special_alt->new_len) {
1918 insn->jump_dest = nex 1918 insn->jump_dest = next_insn_same_sec(file, orig_alt_group->last_insn);
1919 if (!insn->jump_dest) 1919 if (!insn->jump_dest) {
1920 WARN_INSN(ins 1920 WARN_INSN(insn, "can't find alternative jump destination");
1921 return -1; 1921 return -1;
1922 } 1922 }
1923 } 1923 }
1924 } 1924 }
1925 1925
1926 if (!last_new_insn) { 1926 if (!last_new_insn) {
1927 WARN_FUNC("can't find last ne 1927 WARN_FUNC("can't find last new alternative instruction",
1928 special_alt->new_se 1928 special_alt->new_sec, special_alt->new_off);
1929 return -1; 1929 return -1;
1930 } 1930 }
1931 1931
1932 end: 1932 end:
1933 new_alt_group->orig_group = orig_alt_ 1933 new_alt_group->orig_group = orig_alt_group;
1934 new_alt_group->first_insn = *new_insn 1934 new_alt_group->first_insn = *new_insn;
1935 new_alt_group->last_insn = last_new_i 1935 new_alt_group->last_insn = last_new_insn;
1936 new_alt_group->nop = nop; 1936 new_alt_group->nop = nop;
1937 new_alt_group->cfi = orig_alt_group-> 1937 new_alt_group->cfi = orig_alt_group->cfi;
1938 return 0; 1938 return 0;
1939 } 1939 }
1940 1940
1941 /* 1941 /*
1942 * A jump table entry can either convert a no 1942 * A jump table entry can either convert a nop to a jump or a jump to a nop.
1943 * If the original instruction is a jump, mak 1943 * If the original instruction is a jump, make the alt entry an effective nop
1944 * by just skipping the original instruction. 1944 * by just skipping the original instruction.
1945 */ 1945 */
1946 static int handle_jump_alt(struct objtool_fil 1946 static int handle_jump_alt(struct objtool_file *file,
1947 struct special_alt 1947 struct special_alt *special_alt,
1948 struct instruction 1948 struct instruction *orig_insn,
1949 struct instruction 1949 struct instruction **new_insn)
1950 { 1950 {
1951 if (orig_insn->type != INSN_JUMP_UNCO 1951 if (orig_insn->type != INSN_JUMP_UNCONDITIONAL &&
1952 orig_insn->type != INSN_NOP) { 1952 orig_insn->type != INSN_NOP) {
1953 1953
1954 WARN_INSN(orig_insn, "unsuppo 1954 WARN_INSN(orig_insn, "unsupported instruction at jump label");
1955 return -1; 1955 return -1;
1956 } 1956 }
1957 1957
1958 if (opts.hack_jump_label && special_a 1958 if (opts.hack_jump_label && special_alt->key_addend & 2) {
1959 struct reloc *reloc = insn_re 1959 struct reloc *reloc = insn_reloc(file, orig_insn);
1960 1960
1961 if (reloc) 1961 if (reloc)
1962 set_reloc_type(file-> 1962 set_reloc_type(file->elf, reloc, R_NONE);
1963 elf_write_insn(file->elf, ori 1963 elf_write_insn(file->elf, orig_insn->sec,
1964 orig_insn->off 1964 orig_insn->offset, orig_insn->len,
1965 arch_nop_insn( 1965 arch_nop_insn(orig_insn->len));
1966 orig_insn->type = INSN_NOP; 1966 orig_insn->type = INSN_NOP;
1967 } 1967 }
1968 1968
1969 if (orig_insn->type == INSN_NOP) { 1969 if (orig_insn->type == INSN_NOP) {
1970 if (orig_insn->len == 2) 1970 if (orig_insn->len == 2)
1971 file->jl_nop_short++; 1971 file->jl_nop_short++;
1972 else 1972 else
1973 file->jl_nop_long++; 1973 file->jl_nop_long++;
1974 1974
1975 return 0; 1975 return 0;
1976 } 1976 }
1977 1977
1978 if (orig_insn->len == 2) 1978 if (orig_insn->len == 2)
1979 file->jl_short++; 1979 file->jl_short++;
1980 else 1980 else
1981 file->jl_long++; 1981 file->jl_long++;
1982 1982
1983 *new_insn = next_insn_same_sec(file, 1983 *new_insn = next_insn_same_sec(file, orig_insn);
1984 return 0; 1984 return 0;
1985 } 1985 }
1986 1986
1987 /* 1987 /*
1988 * Read all the special sections which have a 1988 * Read all the special sections which have alternate instructions which can be
1989 * patched in or redirected to at runtime. E 1989 * patched in or redirected to at runtime. Each instruction having alternate
1990 * instruction(s) has them added to its insn- 1990 * instruction(s) has them added to its insn->alts list, which will be
1991 * traversed in validate_branch(). 1991 * traversed in validate_branch().
1992 */ 1992 */
1993 static int add_special_section_alts(struct ob 1993 static int add_special_section_alts(struct objtool_file *file)
1994 { 1994 {
1995 struct list_head special_alts; 1995 struct list_head special_alts;
1996 struct instruction *orig_insn, *new_i 1996 struct instruction *orig_insn, *new_insn;
1997 struct special_alt *special_alt, *tmp 1997 struct special_alt *special_alt, *tmp;
1998 struct alternative *alt; 1998 struct alternative *alt;
1999 int ret; 1999 int ret;
2000 2000
2001 ret = special_get_alts(file->elf, &sp 2001 ret = special_get_alts(file->elf, &special_alts);
2002 if (ret) 2002 if (ret)
2003 return ret; 2003 return ret;
2004 2004
2005 list_for_each_entry_safe(special_alt, 2005 list_for_each_entry_safe(special_alt, tmp, &special_alts, list) {
2006 2006
2007 orig_insn = find_insn(file, s 2007 orig_insn = find_insn(file, special_alt->orig_sec,
2008 special 2008 special_alt->orig_off);
2009 if (!orig_insn) { 2009 if (!orig_insn) {
2010 WARN_FUNC("special: c 2010 WARN_FUNC("special: can't find orig instruction",
2011 special_alt 2011 special_alt->orig_sec, special_alt->orig_off);
2012 ret = -1; 2012 ret = -1;
2013 goto out; 2013 goto out;
2014 } 2014 }
2015 2015
2016 new_insn = NULL; 2016 new_insn = NULL;
2017 if (!special_alt->group || sp 2017 if (!special_alt->group || special_alt->new_len) {
2018 new_insn = find_insn( 2018 new_insn = find_insn(file, special_alt->new_sec,
2019 2019 special_alt->new_off);
2020 if (!new_insn) { 2020 if (!new_insn) {
2021 WARN_FUNC("sp 2021 WARN_FUNC("special: can't find new instruction",
2022 spe 2022 special_alt->new_sec,
2023 spe 2023 special_alt->new_off);
2024 ret = -1; 2024 ret = -1;
2025 goto out; 2025 goto out;
2026 } 2026 }
2027 } 2027 }
2028 2028
2029 if (special_alt->group) { 2029 if (special_alt->group) {
2030 if (!special_alt->ori 2030 if (!special_alt->orig_len) {
2031 WARN_INSN(ori 2031 WARN_INSN(orig_insn, "empty alternative entry");
2032 continue; 2032 continue;
2033 } 2033 }
2034 2034
2035 ret = handle_group_al 2035 ret = handle_group_alt(file, special_alt, orig_insn,
2036 2036 &new_insn);
2037 if (ret) 2037 if (ret)
2038 goto out; 2038 goto out;
2039 } else if (special_alt->jump_ 2039 } else if (special_alt->jump_or_nop) {
2040 ret = handle_jump_alt 2040 ret = handle_jump_alt(file, special_alt, orig_insn,
2041 2041 &new_insn);
2042 if (ret) 2042 if (ret)
2043 goto out; 2043 goto out;
2044 } 2044 }
2045 2045
2046 alt = malloc(sizeof(*alt)); 2046 alt = malloc(sizeof(*alt));
2047 if (!alt) { 2047 if (!alt) {
2048 WARN("malloc failed") 2048 WARN("malloc failed");
2049 ret = -1; 2049 ret = -1;
2050 goto out; 2050 goto out;
2051 } 2051 }
2052 2052
2053 alt->insn = new_insn; 2053 alt->insn = new_insn;
2054 alt->skip_orig = special_alt- 2054 alt->skip_orig = special_alt->skip_orig;
2055 orig_insn->ignore_alts |= spe 2055 orig_insn->ignore_alts |= special_alt->skip_alt;
2056 alt->next = orig_insn->alts; 2056 alt->next = orig_insn->alts;
2057 orig_insn->alts = alt; 2057 orig_insn->alts = alt;
2058 2058
2059 list_del(&special_alt->list); 2059 list_del(&special_alt->list);
2060 free(special_alt); 2060 free(special_alt);
2061 } 2061 }
2062 2062
2063 if (opts.stats) { 2063 if (opts.stats) {
2064 printf("jl\\\tNOP\tJMP\n"); 2064 printf("jl\\\tNOP\tJMP\n");
2065 printf("short:\t%ld\t%ld\n", 2065 printf("short:\t%ld\t%ld\n", file->jl_nop_short, file->jl_short);
2066 printf("long:\t%ld\t%ld\n", f 2066 printf("long:\t%ld\t%ld\n", file->jl_nop_long, file->jl_long);
2067 } 2067 }
2068 2068
2069 out: 2069 out:
2070 return ret; 2070 return ret;
2071 } 2071 }
2072 2072
2073 static int add_jump_table(struct objtool_file 2073 static int add_jump_table(struct objtool_file *file, struct instruction *insn,
2074 struct reloc *next_ 2074 struct reloc *next_table)
2075 { 2075 {
2076 struct symbol *pfunc = insn_func(insn 2076 struct symbol *pfunc = insn_func(insn)->pfunc;
2077 struct reloc *table = insn_jump_table 2077 struct reloc *table = insn_jump_table(insn);
2078 struct instruction *dest_insn; 2078 struct instruction *dest_insn;
2079 unsigned int prev_offset = 0; 2079 unsigned int prev_offset = 0;
2080 struct reloc *reloc = table; 2080 struct reloc *reloc = table;
2081 struct alternative *alt; 2081 struct alternative *alt;
2082 2082
2083 /* 2083 /*
2084 * Each @reloc is a switch table relo 2084 * Each @reloc is a switch table relocation which points to the target
2085 * instruction. 2085 * instruction.
2086 */ 2086 */
2087 for_each_reloc_from(table->sec, reloc 2087 for_each_reloc_from(table->sec, reloc) {
2088 2088
2089 /* Check for the end of the t 2089 /* Check for the end of the table: */
2090 if (reloc != table && reloc = 2090 if (reloc != table && reloc == next_table)
2091 break; 2091 break;
2092 2092
2093 /* Make sure the table entrie 2093 /* Make sure the table entries are consecutive: */
2094 if (prev_offset && reloc_offs 2094 if (prev_offset && reloc_offset(reloc) != prev_offset + 8)
2095 break; 2095 break;
2096 2096
2097 /* Detect function pointers f 2097 /* Detect function pointers from contiguous objects: */
2098 if (reloc->sym->sec == pfunc- 2098 if (reloc->sym->sec == pfunc->sec &&
2099 reloc_addend(reloc) == pf 2099 reloc_addend(reloc) == pfunc->offset)
2100 break; 2100 break;
2101 2101
2102 dest_insn = find_insn(file, r 2102 dest_insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2103 if (!dest_insn) 2103 if (!dest_insn)
2104 break; 2104 break;
2105 2105
2106 /* Make sure the destination 2106 /* Make sure the destination is in the same function: */
2107 if (!insn_func(dest_insn) || 2107 if (!insn_func(dest_insn) || insn_func(dest_insn)->pfunc != pfunc)
2108 break; 2108 break;
2109 2109
2110 alt = malloc(sizeof(*alt)); 2110 alt = malloc(sizeof(*alt));
2111 if (!alt) { 2111 if (!alt) {
2112 WARN("malloc failed") 2112 WARN("malloc failed");
2113 return -1; 2113 return -1;
2114 } 2114 }
2115 2115
2116 alt->insn = dest_insn; 2116 alt->insn = dest_insn;
2117 alt->next = insn->alts; 2117 alt->next = insn->alts;
2118 insn->alts = alt; 2118 insn->alts = alt;
2119 prev_offset = reloc_offset(re 2119 prev_offset = reloc_offset(reloc);
2120 } 2120 }
2121 2121
2122 if (!prev_offset) { 2122 if (!prev_offset) {
2123 WARN_INSN(insn, "can't find s 2123 WARN_INSN(insn, "can't find switch jump table");
2124 return -1; 2124 return -1;
2125 } 2125 }
2126 2126
2127 return 0; 2127 return 0;
2128 } 2128 }
2129 2129
2130 /* 2130 /*
2131 * find_jump_table() - Given a dynamic jump, 2131 * find_jump_table() - Given a dynamic jump, find the switch jump table
2132 * associated with it. 2132 * associated with it.
2133 */ 2133 */
2134 static struct reloc *find_jump_table(struct o 2134 static struct reloc *find_jump_table(struct objtool_file *file,
2135 struct 2135 struct symbol *func,
2136 struct 2136 struct instruction *insn)
2137 { 2137 {
2138 struct reloc *table_reloc; 2138 struct reloc *table_reloc;
2139 struct instruction *dest_insn, *orig_ 2139 struct instruction *dest_insn, *orig_insn = insn;
2140 2140
2141 /* 2141 /*
2142 * Backward search using the @first_j 2142 * Backward search using the @first_jump_src links, these help avoid
2143 * much of the 'in between' code. Whi 2143 * much of the 'in between' code. Which avoids us getting confused by
2144 * it. 2144 * it.
2145 */ 2145 */
2146 for (; 2146 for (;
2147 insn && insn_func(insn) && insn_ 2147 insn && insn_func(insn) && insn_func(insn)->pfunc == func;
2148 insn = insn->first_jump_src ?: p 2148 insn = insn->first_jump_src ?: prev_insn_same_sym(file, insn)) {
2149 2149
2150 if (insn != orig_insn && insn 2150 if (insn != orig_insn && insn->type == INSN_JUMP_DYNAMIC)
2151 break; 2151 break;
2152 2152
2153 /* allow small jumps within t 2153 /* allow small jumps within the range */
2154 if (insn->type == INSN_JUMP_U 2154 if (insn->type == INSN_JUMP_UNCONDITIONAL &&
2155 insn->jump_dest && 2155 insn->jump_dest &&
2156 (insn->jump_dest->offset 2156 (insn->jump_dest->offset <= insn->offset ||
2157 insn->jump_dest->offset 2157 insn->jump_dest->offset > orig_insn->offset))
2158 break; 2158 break;
2159 2159
2160 table_reloc = arch_find_switc 2160 table_reloc = arch_find_switch_table(file, insn);
2161 if (!table_reloc) 2161 if (!table_reloc)
2162 continue; 2162 continue;
2163 dest_insn = find_insn(file, t 2163 dest_insn = find_insn(file, table_reloc->sym->sec, reloc_addend(table_reloc));
2164 if (!dest_insn || !insn_func( 2164 if (!dest_insn || !insn_func(dest_insn) || insn_func(dest_insn)->pfunc != func)
2165 continue; 2165 continue;
2166 2166
2167 return table_reloc; 2167 return table_reloc;
2168 } 2168 }
2169 2169
2170 return NULL; 2170 return NULL;
2171 } 2171 }
2172 2172
2173 /* 2173 /*
2174 * First pass: Mark the head of each jump tab 2174 * First pass: Mark the head of each jump table so that in the next pass,
2175 * we know when a given jump table ends and t 2175 * we know when a given jump table ends and the next one starts.
2176 */ 2176 */
2177 static void mark_func_jump_tables(struct objt 2177 static void mark_func_jump_tables(struct objtool_file *file,
2178 struct sy 2178 struct symbol *func)
2179 { 2179 {
2180 struct instruction *insn, *last = NUL 2180 struct instruction *insn, *last = NULL;
2181 struct reloc *reloc; 2181 struct reloc *reloc;
2182 2182
2183 func_for_each_insn(file, func, insn) 2183 func_for_each_insn(file, func, insn) {
2184 if (!last) 2184 if (!last)
2185 last = insn; 2185 last = insn;
2186 2186
2187 /* 2187 /*
2188 * Store back-pointers for un 2188 * Store back-pointers for unconditional forward jumps such
2189 * that find_jump_table() can 2189 * that find_jump_table() can back-track using those and
2190 * avoid some potentially con 2190 * avoid some potentially confusing code.
2191 */ 2191 */
2192 if (insn->type == INSN_JUMP_U 2192 if (insn->type == INSN_JUMP_UNCONDITIONAL && insn->jump_dest &&
2193 insn->offset > last->offs 2193 insn->offset > last->offset &&
2194 insn->jump_dest->offset > 2194 insn->jump_dest->offset > insn->offset &&
2195 !insn->jump_dest->first_j 2195 !insn->jump_dest->first_jump_src) {
2196 2196
2197 insn->jump_dest->firs 2197 insn->jump_dest->first_jump_src = insn;
2198 last = insn->jump_des 2198 last = insn->jump_dest;
2199 } 2199 }
2200 2200
2201 if (insn->type != INSN_JUMP_D 2201 if (insn->type != INSN_JUMP_DYNAMIC)
2202 continue; 2202 continue;
2203 2203
2204 reloc = find_jump_table(file, 2204 reloc = find_jump_table(file, func, insn);
2205 if (reloc) 2205 if (reloc)
2206 insn->_jump_table = r 2206 insn->_jump_table = reloc;
2207 } 2207 }
2208 } 2208 }
2209 2209
2210 static int add_func_jump_tables(struct objtoo 2210 static int add_func_jump_tables(struct objtool_file *file,
2211 struct symb 2211 struct symbol *func)
2212 { 2212 {
2213 struct instruction *insn, *insn_t1 = 2213 struct instruction *insn, *insn_t1 = NULL, *insn_t2;
2214 int ret = 0; 2214 int ret = 0;
2215 2215
2216 func_for_each_insn(file, func, insn) 2216 func_for_each_insn(file, func, insn) {
2217 if (!insn_jump_table(insn)) 2217 if (!insn_jump_table(insn))
2218 continue; 2218 continue;
2219 2219
2220 if (!insn_t1) { 2220 if (!insn_t1) {
2221 insn_t1 = insn; 2221 insn_t1 = insn;
2222 continue; 2222 continue;
2223 } 2223 }
2224 2224
2225 insn_t2 = insn; 2225 insn_t2 = insn;
2226 2226
2227 ret = add_jump_table(file, in 2227 ret = add_jump_table(file, insn_t1, insn_jump_table(insn_t2));
2228 if (ret) 2228 if (ret)
2229 return ret; 2229 return ret;
2230 2230
2231 insn_t1 = insn_t2; 2231 insn_t1 = insn_t2;
2232 } 2232 }
2233 2233
2234 if (insn_t1) 2234 if (insn_t1)
2235 ret = add_jump_table(file, in 2235 ret = add_jump_table(file, insn_t1, NULL);
2236 2236
2237 return ret; 2237 return ret;
2238 } 2238 }
2239 2239
2240 /* 2240 /*
2241 * For some switch statements, gcc generates 2241 * For some switch statements, gcc generates a jump table in the .rodata
2242 * section which contains a list of addresses 2242 * section which contains a list of addresses within the function to jump to.
2243 * This finds these jump tables and adds them 2243 * This finds these jump tables and adds them to the insn->alts lists.
2244 */ 2244 */
2245 static int add_jump_table_alts(struct objtool 2245 static int add_jump_table_alts(struct objtool_file *file)
2246 { 2246 {
2247 struct symbol *func; 2247 struct symbol *func;
2248 int ret; 2248 int ret;
2249 2249
2250 if (!file->rodata) 2250 if (!file->rodata)
2251 return 0; 2251 return 0;
2252 2252
2253 for_each_sym(file, func) { 2253 for_each_sym(file, func) {
2254 if (func->type != STT_FUNC) 2254 if (func->type != STT_FUNC)
2255 continue; 2255 continue;
2256 2256
2257 mark_func_jump_tables(file, f 2257 mark_func_jump_tables(file, func);
2258 ret = add_func_jump_tables(fi 2258 ret = add_func_jump_tables(file, func);
2259 if (ret) 2259 if (ret)
2260 return ret; 2260 return ret;
2261 } 2261 }
2262 2262
2263 return 0; 2263 return 0;
2264 } 2264 }
2265 2265
2266 static void set_func_state(struct cfi_state * 2266 static void set_func_state(struct cfi_state *state)
2267 { 2267 {
2268 state->cfa = initial_func_cfi.cfa; 2268 state->cfa = initial_func_cfi.cfa;
2269 memcpy(&state->regs, &initial_func_cf 2269 memcpy(&state->regs, &initial_func_cfi.regs,
2270 CFI_NUM_REGS * sizeof(struct c 2270 CFI_NUM_REGS * sizeof(struct cfi_reg));
2271 state->stack_size = initial_func_cfi. 2271 state->stack_size = initial_func_cfi.cfa.offset;
2272 state->type = UNWIND_HINT_TYPE_CALL; 2272 state->type = UNWIND_HINT_TYPE_CALL;
2273 } 2273 }
2274 2274
2275 static int read_unwind_hints(struct objtool_f 2275 static int read_unwind_hints(struct objtool_file *file)
2276 { 2276 {
2277 struct cfi_state cfi = init_cfi; 2277 struct cfi_state cfi = init_cfi;
2278 struct section *sec; 2278 struct section *sec;
2279 struct unwind_hint *hint; 2279 struct unwind_hint *hint;
2280 struct instruction *insn; 2280 struct instruction *insn;
2281 struct reloc *reloc; 2281 struct reloc *reloc;
2282 unsigned long offset; 2282 unsigned long offset;
2283 int i; 2283 int i;
2284 2284
2285 sec = find_section_by_name(file->elf, 2285 sec = find_section_by_name(file->elf, ".discard.unwind_hints");
2286 if (!sec) 2286 if (!sec)
2287 return 0; 2287 return 0;
2288 2288
2289 if (!sec->rsec) { 2289 if (!sec->rsec) {
2290 WARN("missing .rela.discard.u 2290 WARN("missing .rela.discard.unwind_hints section");
2291 return -1; 2291 return -1;
2292 } 2292 }
2293 2293
2294 if (sec->sh.sh_size % sizeof(struct u 2294 if (sec->sh.sh_size % sizeof(struct unwind_hint)) {
2295 WARN("struct unwind_hint size 2295 WARN("struct unwind_hint size mismatch");
2296 return -1; 2296 return -1;
2297 } 2297 }
2298 2298
2299 file->hints = true; 2299 file->hints = true;
2300 2300
2301 for (i = 0; i < sec->sh.sh_size / siz 2301 for (i = 0; i < sec->sh.sh_size / sizeof(struct unwind_hint); i++) {
2302 hint = (struct unwind_hint *) 2302 hint = (struct unwind_hint *)sec->data->d_buf + i;
2303 2303
2304 reloc = find_reloc_by_dest(fi 2304 reloc = find_reloc_by_dest(file->elf, sec, i * sizeof(*hint));
2305 if (!reloc) { 2305 if (!reloc) {
2306 WARN("can't find relo 2306 WARN("can't find reloc for unwind_hints[%d]", i);
2307 return -1; 2307 return -1;
2308 } 2308 }
2309 2309
2310 if (reloc->sym->type == STT_S 2310 if (reloc->sym->type == STT_SECTION) {
2311 offset = reloc_addend 2311 offset = reloc_addend(reloc);
2312 } else if (reloc->sym->local_ 2312 } else if (reloc->sym->local_label) {
2313 offset = reloc->sym-> 2313 offset = reloc->sym->offset;
2314 } else { 2314 } else {
2315 WARN("unexpected relo 2315 WARN("unexpected relocation symbol type in %s", sec->rsec->name);
2316 return -1; 2316 return -1;
2317 } 2317 }
2318 2318
2319 insn = find_insn(file, reloc- 2319 insn = find_insn(file, reloc->sym->sec, offset);
2320 if (!insn) { 2320 if (!insn) {
2321 WARN("can't find insn 2321 WARN("can't find insn for unwind_hints[%d]", i);
2322 return -1; 2322 return -1;
2323 } 2323 }
2324 2324
2325 insn->hint = true; 2325 insn->hint = true;
2326 2326
2327 if (hint->type == UNWIND_HINT 2327 if (hint->type == UNWIND_HINT_TYPE_UNDEFINED) {
2328 insn->cfi = &force_un 2328 insn->cfi = &force_undefined_cfi;
2329 continue; 2329 continue;
2330 } 2330 }
2331 2331
2332 if (hint->type == UNWIND_HINT 2332 if (hint->type == UNWIND_HINT_TYPE_SAVE) {
2333 insn->hint = false; 2333 insn->hint = false;
2334 insn->save = true; 2334 insn->save = true;
2335 continue; 2335 continue;
2336 } 2336 }
2337 2337
2338 if (hint->type == UNWIND_HINT 2338 if (hint->type == UNWIND_HINT_TYPE_RESTORE) {
2339 insn->restore = true; 2339 insn->restore = true;
2340 continue; 2340 continue;
2341 } 2341 }
2342 2342
2343 if (hint->type == UNWIND_HINT 2343 if (hint->type == UNWIND_HINT_TYPE_REGS_PARTIAL) {
2344 struct symbol *sym = 2344 struct symbol *sym = find_symbol_by_offset(insn->sec, insn->offset);
2345 2345
2346 if (sym && sym->bind 2346 if (sym && sym->bind == STB_GLOBAL) {
2347 if (opts.ibt 2347 if (opts.ibt && insn->type != INSN_ENDBR && !insn->noendbr) {
2348 WARN_ 2348 WARN_INSN(insn, "UNWIND_HINT_IRET_REGS without ENDBR");
2349 } 2349 }
2350 } 2350 }
2351 } 2351 }
2352 2352
2353 if (hint->type == UNWIND_HINT 2353 if (hint->type == UNWIND_HINT_TYPE_FUNC) {
2354 insn->cfi = &func_cfi 2354 insn->cfi = &func_cfi;
2355 continue; 2355 continue;
2356 } 2356 }
2357 2357
2358 if (insn->cfi) 2358 if (insn->cfi)
2359 cfi = *(insn->cfi); 2359 cfi = *(insn->cfi);
2360 2360
2361 if (arch_decode_hint_reg(hint 2361 if (arch_decode_hint_reg(hint->sp_reg, &cfi.cfa.base)) {
2362 WARN_INSN(insn, "unsu 2362 WARN_INSN(insn, "unsupported unwind_hint sp base reg %d", hint->sp_reg);
2363 return -1; 2363 return -1;
2364 } 2364 }
2365 2365
2366 cfi.cfa.offset = bswap_if_nee 2366 cfi.cfa.offset = bswap_if_needed(file->elf, hint->sp_offset);
2367 cfi.type = hint->type; 2367 cfi.type = hint->type;
2368 cfi.signal = hint->signal; 2368 cfi.signal = hint->signal;
2369 2369
2370 insn->cfi = cfi_hash_find_or_ 2370 insn->cfi = cfi_hash_find_or_add(&cfi);
2371 } 2371 }
2372 2372
2373 return 0; 2373 return 0;
2374 } 2374 }
2375 2375
2376 static int read_noendbr_hints(struct objtool_ 2376 static int read_noendbr_hints(struct objtool_file *file)
2377 { 2377 {
2378 struct instruction *insn; 2378 struct instruction *insn;
2379 struct section *rsec; 2379 struct section *rsec;
2380 struct reloc *reloc; 2380 struct reloc *reloc;
2381 2381
2382 rsec = find_section_by_name(file->elf 2382 rsec = find_section_by_name(file->elf, ".rela.discard.noendbr");
2383 if (!rsec) 2383 if (!rsec)
2384 return 0; 2384 return 0;
2385 2385
2386 for_each_reloc(rsec, reloc) { 2386 for_each_reloc(rsec, reloc) {
2387 insn = find_insn(file, reloc- 2387 insn = find_insn(file, reloc->sym->sec,
2388 reloc->sym-> 2388 reloc->sym->offset + reloc_addend(reloc));
2389 if (!insn) { 2389 if (!insn) {
2390 WARN("bad .discard.no 2390 WARN("bad .discard.noendbr entry");
2391 return -1; 2391 return -1;
2392 } 2392 }
2393 2393
2394 insn->noendbr = 1; 2394 insn->noendbr = 1;
2395 } 2395 }
2396 2396
2397 return 0; 2397 return 0;
2398 } 2398 }
2399 2399
2400 static int read_retpoline_hints(struct objtoo 2400 static int read_retpoline_hints(struct objtool_file *file)
2401 { 2401 {
2402 struct section *rsec; 2402 struct section *rsec;
2403 struct instruction *insn; 2403 struct instruction *insn;
2404 struct reloc *reloc; 2404 struct reloc *reloc;
2405 2405
2406 rsec = find_section_by_name(file->elf 2406 rsec = find_section_by_name(file->elf, ".rela.discard.retpoline_safe");
2407 if (!rsec) 2407 if (!rsec)
2408 return 0; 2408 return 0;
2409 2409
2410 for_each_reloc(rsec, reloc) { 2410 for_each_reloc(rsec, reloc) {
2411 if (reloc->sym->type != STT_S 2411 if (reloc->sym->type != STT_SECTION) {
2412 WARN("unexpected relo 2412 WARN("unexpected relocation symbol type in %s", rsec->name);
2413 return -1; 2413 return -1;
2414 } 2414 }
2415 2415
2416 insn = find_insn(file, reloc- 2416 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2417 if (!insn) { 2417 if (!insn) {
2418 WARN("bad .discard.re 2418 WARN("bad .discard.retpoline_safe entry");
2419 return -1; 2419 return -1;
2420 } 2420 }
2421 2421
2422 if (insn->type != INSN_JUMP_D 2422 if (insn->type != INSN_JUMP_DYNAMIC &&
2423 insn->type != INSN_CALL_D 2423 insn->type != INSN_CALL_DYNAMIC &&
2424 insn->type != INSN_RETURN 2424 insn->type != INSN_RETURN &&
2425 insn->type != INSN_NOP) { 2425 insn->type != INSN_NOP) {
2426 WARN_INSN(insn, "retp 2426 WARN_INSN(insn, "retpoline_safe hint not an indirect jump/call/ret/nop");
2427 return -1; 2427 return -1;
2428 } 2428 }
2429 2429
2430 insn->retpoline_safe = true; 2430 insn->retpoline_safe = true;
2431 } 2431 }
2432 2432
2433 return 0; 2433 return 0;
2434 } 2434 }
2435 2435
2436 static int read_instr_hints(struct objtool_fi 2436 static int read_instr_hints(struct objtool_file *file)
2437 { 2437 {
2438 struct section *rsec; 2438 struct section *rsec;
2439 struct instruction *insn; 2439 struct instruction *insn;
2440 struct reloc *reloc; 2440 struct reloc *reloc;
2441 2441
2442 rsec = find_section_by_name(file->elf 2442 rsec = find_section_by_name(file->elf, ".rela.discard.instr_end");
2443 if (!rsec) 2443 if (!rsec)
2444 return 0; 2444 return 0;
2445 2445
2446 for_each_reloc(rsec, reloc) { 2446 for_each_reloc(rsec, reloc) {
2447 if (reloc->sym->type != STT_S 2447 if (reloc->sym->type != STT_SECTION) {
2448 WARN("unexpected relo 2448 WARN("unexpected relocation symbol type in %s", rsec->name);
2449 return -1; 2449 return -1;
2450 } 2450 }
2451 2451
2452 insn = find_insn(file, reloc- 2452 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2453 if (!insn) { 2453 if (!insn) {
2454 WARN("bad .discard.in 2454 WARN("bad .discard.instr_end entry");
2455 return -1; 2455 return -1;
2456 } 2456 }
2457 2457
2458 insn->instr--; 2458 insn->instr--;
2459 } 2459 }
2460 2460
2461 rsec = find_section_by_name(file->elf 2461 rsec = find_section_by_name(file->elf, ".rela.discard.instr_begin");
2462 if (!rsec) 2462 if (!rsec)
2463 return 0; 2463 return 0;
2464 2464
2465 for_each_reloc(rsec, reloc) { 2465 for_each_reloc(rsec, reloc) {
2466 if (reloc->sym->type != STT_S 2466 if (reloc->sym->type != STT_SECTION) {
2467 WARN("unexpected relo 2467 WARN("unexpected relocation symbol type in %s", rsec->name);
2468 return -1; 2468 return -1;
2469 } 2469 }
2470 2470
2471 insn = find_insn(file, reloc- 2471 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2472 if (!insn) { 2472 if (!insn) {
2473 WARN("bad .discard.in 2473 WARN("bad .discard.instr_begin entry");
2474 return -1; 2474 return -1;
2475 } 2475 }
2476 2476
2477 insn->instr++; 2477 insn->instr++;
2478 } 2478 }
2479 2479
2480 return 0; 2480 return 0;
2481 } 2481 }
2482 2482
2483 static int read_validate_unret_hints(struct o 2483 static int read_validate_unret_hints(struct objtool_file *file)
2484 { 2484 {
2485 struct section *rsec; 2485 struct section *rsec;
2486 struct instruction *insn; 2486 struct instruction *insn;
2487 struct reloc *reloc; 2487 struct reloc *reloc;
2488 2488
2489 rsec = find_section_by_name(file->elf 2489 rsec = find_section_by_name(file->elf, ".rela.discard.validate_unret");
2490 if (!rsec) 2490 if (!rsec)
2491 return 0; 2491 return 0;
2492 2492
2493 for_each_reloc(rsec, reloc) { 2493 for_each_reloc(rsec, reloc) {
2494 if (reloc->sym->type != STT_S 2494 if (reloc->sym->type != STT_SECTION) {
2495 WARN("unexpected relo 2495 WARN("unexpected relocation symbol type in %s", rsec->name);
2496 return -1; 2496 return -1;
2497 } 2497 }
2498 2498
2499 insn = find_insn(file, reloc- 2499 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2500 if (!insn) { 2500 if (!insn) {
2501 WARN("bad .discard.in 2501 WARN("bad .discard.instr_end entry");
2502 return -1; 2502 return -1;
2503 } 2503 }
2504 insn->unret = 1; 2504 insn->unret = 1;
2505 } 2505 }
2506 2506
2507 return 0; 2507 return 0;
2508 } 2508 }
2509 2509
2510 2510
2511 static int read_intra_function_calls(struct o 2511 static int read_intra_function_calls(struct objtool_file *file)
2512 { 2512 {
2513 struct instruction *insn; 2513 struct instruction *insn;
2514 struct section *rsec; 2514 struct section *rsec;
2515 struct reloc *reloc; 2515 struct reloc *reloc;
2516 2516
2517 rsec = find_section_by_name(file->elf 2517 rsec = find_section_by_name(file->elf, ".rela.discard.intra_function_calls");
2518 if (!rsec) 2518 if (!rsec)
2519 return 0; 2519 return 0;
2520 2520
2521 for_each_reloc(rsec, reloc) { 2521 for_each_reloc(rsec, reloc) {
2522 unsigned long dest_off; 2522 unsigned long dest_off;
2523 2523
2524 if (reloc->sym->type != STT_S 2524 if (reloc->sym->type != STT_SECTION) {
2525 WARN("unexpected relo 2525 WARN("unexpected relocation symbol type in %s",
2526 rsec->name); 2526 rsec->name);
2527 return -1; 2527 return -1;
2528 } 2528 }
2529 2529
2530 insn = find_insn(file, reloc- 2530 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2531 if (!insn) { 2531 if (!insn) {
2532 WARN("bad .discard.in 2532 WARN("bad .discard.intra_function_call entry");
2533 return -1; 2533 return -1;
2534 } 2534 }
2535 2535
2536 if (insn->type != INSN_CALL) 2536 if (insn->type != INSN_CALL) {
2537 WARN_INSN(insn, "intr 2537 WARN_INSN(insn, "intra_function_call not a direct call");
2538 return -1; 2538 return -1;
2539 } 2539 }
2540 2540
2541 /* 2541 /*
2542 * Treat intra-function CALLs 2542 * Treat intra-function CALLs as JMPs, but with a stack_op.
2543 * See add_call_destinations( 2543 * See add_call_destinations(), which strips stack_ops from
2544 * normal CALLs. 2544 * normal CALLs.
2545 */ 2545 */
2546 insn->type = INSN_JUMP_UNCOND 2546 insn->type = INSN_JUMP_UNCONDITIONAL;
2547 2547
2548 dest_off = arch_jump_destinat 2548 dest_off = arch_jump_destination(insn);
2549 insn->jump_dest = find_insn(f 2549 insn->jump_dest = find_insn(file, insn->sec, dest_off);
2550 if (!insn->jump_dest) { 2550 if (!insn->jump_dest) {
2551 WARN_INSN(insn, "can' 2551 WARN_INSN(insn, "can't find call dest at %s+0x%lx",
2552 insn->sec-> 2552 insn->sec->name, dest_off);
2553 return -1; 2553 return -1;
2554 } 2554 }
2555 } 2555 }
2556 2556
2557 return 0; 2557 return 0;
2558 } 2558 }
2559 2559
2560 /* 2560 /*
2561 * Return true if name matches an instrumenta 2561 * Return true if name matches an instrumentation function, where calls to that
2562 * function from noinstr code can safely be r 2562 * function from noinstr code can safely be removed, but compilers won't do so.
2563 */ 2563 */
2564 static bool is_profiling_func(const char *nam 2564 static bool is_profiling_func(const char *name)
2565 { 2565 {
2566 /* 2566 /*
2567 * Many compilers cannot disable KCOV 2567 * Many compilers cannot disable KCOV with a function attribute.
2568 */ 2568 */
2569 if (!strncmp(name, "__sanitizer_cov_" 2569 if (!strncmp(name, "__sanitizer_cov_", 16))
2570 return true; 2570 return true;
2571 2571
2572 /* 2572 /*
2573 * Some compilers currently do not re 2573 * Some compilers currently do not remove __tsan_func_entry/exit nor
2574 * __tsan_atomic_signal_fence (used f 2574 * __tsan_atomic_signal_fence (used for barrier instrumentation) with
2575 * the __no_sanitize_thread attribute 2575 * the __no_sanitize_thread attribute, remove them. Once the kernel's
2576 * minimum Clang version is 14.0, thi 2576 * minimum Clang version is 14.0, this can be removed.
2577 */ 2577 */
2578 if (!strncmp(name, "__tsan_func_", 12 2578 if (!strncmp(name, "__tsan_func_", 12) ||
2579 !strcmp(name, "__tsan_atomic_sign 2579 !strcmp(name, "__tsan_atomic_signal_fence"))
2580 return true; 2580 return true;
2581 2581
2582 return false; 2582 return false;
2583 } 2583 }
2584 2584
2585 static int classify_symbols(struct objtool_fi 2585 static int classify_symbols(struct objtool_file *file)
2586 { 2586 {
2587 struct symbol *func; 2587 struct symbol *func;
2588 2588
2589 for_each_sym(file, func) { 2589 for_each_sym(file, func) {
2590 if (func->type == STT_NOTYPE 2590 if (func->type == STT_NOTYPE && strstarts(func->name, ".L"))
2591 func->local_label = t 2591 func->local_label = true;
2592 2592
2593 if (func->bind != STB_GLOBAL) 2593 if (func->bind != STB_GLOBAL)
2594 continue; 2594 continue;
2595 2595
2596 if (!strncmp(func->name, STAT 2596 if (!strncmp(func->name, STATIC_CALL_TRAMP_PREFIX_STR,
2597 strlen(STATIC_CA 2597 strlen(STATIC_CALL_TRAMP_PREFIX_STR)))
2598 func->static_call_tra 2598 func->static_call_tramp = true;
2599 2599
2600 if (arch_is_retpoline(func)) 2600 if (arch_is_retpoline(func))
2601 func->retpoline_thunk 2601 func->retpoline_thunk = true;
2602 2602
2603 if (arch_is_rethunk(func)) 2603 if (arch_is_rethunk(func))
2604 func->return_thunk = 2604 func->return_thunk = true;
2605 2605
2606 if (arch_is_embedded_insn(fun 2606 if (arch_is_embedded_insn(func))
2607 func->embedded_insn = 2607 func->embedded_insn = true;
2608 2608
2609 if (arch_ftrace_match(func->n 2609 if (arch_ftrace_match(func->name))
2610 func->fentry = true; 2610 func->fentry = true;
2611 2611
2612 if (is_profiling_func(func->n 2612 if (is_profiling_func(func->name))
2613 func->profiling_func 2613 func->profiling_func = true;
2614 } 2614 }
2615 2615
2616 return 0; 2616 return 0;
2617 } 2617 }
2618 2618
2619 static void mark_rodata(struct objtool_file * 2619 static void mark_rodata(struct objtool_file *file)
2620 { 2620 {
2621 struct section *sec; 2621 struct section *sec;
2622 bool found = false; 2622 bool found = false;
2623 2623
2624 /* 2624 /*
2625 * Search for the following rodata se 2625 * Search for the following rodata sections, each of which can
2626 * potentially contain jump tables: 2626 * potentially contain jump tables:
2627 * 2627 *
2628 * - .rodata: can contain GCC switch 2628 * - .rodata: can contain GCC switch tables
2629 * - .rodata.<func>: same, if -fdata- 2629 * - .rodata.<func>: same, if -fdata-sections is being used
2630 * - .rodata..c_jump_table: contains 2630 * - .rodata..c_jump_table: contains C annotated jump tables
2631 * 2631 *
2632 * .rodata.str1.* sections are ignore 2632 * .rodata.str1.* sections are ignored; they don't contain jump tables.
2633 */ 2633 */
2634 for_each_sec(file, sec) { 2634 for_each_sec(file, sec) {
2635 if (!strncmp(sec->name, ".rod 2635 if (!strncmp(sec->name, ".rodata", 7) &&
2636 !strstr(sec->name, ".str1 2636 !strstr(sec->name, ".str1.")) {
2637 sec->rodata = true; 2637 sec->rodata = true;
2638 found = true; 2638 found = true;
2639 } 2639 }
2640 } 2640 }
2641 2641
2642 file->rodata = found; 2642 file->rodata = found;
2643 } 2643 }
2644 2644
2645 static int decode_sections(struct objtool_fil 2645 static int decode_sections(struct objtool_file *file)
2646 { 2646 {
2647 int ret; 2647 int ret;
2648 2648
2649 mark_rodata(file); 2649 mark_rodata(file);
2650 2650
2651 ret = init_pv_ops(file); 2651 ret = init_pv_ops(file);
2652 if (ret) 2652 if (ret)
2653 return ret; 2653 return ret;
2654 2654
2655 /* 2655 /*
2656 * Must be before add_{jump_call}_des 2656 * Must be before add_{jump_call}_destination.
2657 */ 2657 */
2658 ret = classify_symbols(file); 2658 ret = classify_symbols(file);
2659 if (ret) 2659 if (ret)
2660 return ret; 2660 return ret;
2661 2661
2662 ret = decode_instructions(file); 2662 ret = decode_instructions(file);
2663 if (ret) 2663 if (ret)
2664 return ret; 2664 return ret;
2665 2665
2666 add_ignores(file); 2666 add_ignores(file);
2667 add_uaccess_safe(file); 2667 add_uaccess_safe(file);
2668 2668
2669 ret = add_ignore_alternatives(file); 2669 ret = add_ignore_alternatives(file);
2670 if (ret) 2670 if (ret)
2671 return ret; 2671 return ret;
2672 2672
2673 /* 2673 /*
2674 * Must be before read_unwind_hints() 2674 * Must be before read_unwind_hints() since that needs insn->noendbr.
2675 */ 2675 */
2676 ret = read_noendbr_hints(file); 2676 ret = read_noendbr_hints(file);
2677 if (ret) 2677 if (ret)
2678 return ret; 2678 return ret;
2679 2679
2680 /* 2680 /*
2681 * Must be before add_jump_destinatio 2681 * Must be before add_jump_destinations(), which depends on 'func'
2682 * being set for alternatives, to ena 2682 * being set for alternatives, to enable proper sibling call detection.
2683 */ 2683 */
2684 if (opts.stackval || opts.orc || opts 2684 if (opts.stackval || opts.orc || opts.uaccess || opts.noinstr) {
2685 ret = add_special_section_alt 2685 ret = add_special_section_alts(file);
2686 if (ret) 2686 if (ret)
2687 return ret; 2687 return ret;
2688 } 2688 }
2689 2689
2690 ret = add_jump_destinations(file); 2690 ret = add_jump_destinations(file);
2691 if (ret) 2691 if (ret)
2692 return ret; 2692 return ret;
2693 2693
2694 /* 2694 /*
2695 * Must be before add_call_destinatio 2695 * Must be before add_call_destination(); it changes INSN_CALL to
2696 * INSN_JUMP. 2696 * INSN_JUMP.
2697 */ 2697 */
2698 ret = read_intra_function_calls(file) 2698 ret = read_intra_function_calls(file);
2699 if (ret) 2699 if (ret)
2700 return ret; 2700 return ret;
2701 2701
2702 ret = add_call_destinations(file); 2702 ret = add_call_destinations(file);
2703 if (ret) 2703 if (ret)
2704 return ret; 2704 return ret;
2705 2705
2706 /* 2706 /*
2707 * Must be after add_call_destination 2707 * Must be after add_call_destinations() such that it can override
2708 * dead_end_function() marks. 2708 * dead_end_function() marks.
2709 */ 2709 */
2710 ret = add_dead_ends(file); 2710 ret = add_dead_ends(file);
2711 if (ret) 2711 if (ret)
2712 return ret; 2712 return ret;
2713 2713
2714 ret = add_jump_table_alts(file); 2714 ret = add_jump_table_alts(file);
2715 if (ret) 2715 if (ret)
2716 return ret; 2716 return ret;
2717 2717
2718 ret = read_unwind_hints(file); 2718 ret = read_unwind_hints(file);
2719 if (ret) 2719 if (ret)
2720 return ret; 2720 return ret;
2721 2721
2722 ret = read_retpoline_hints(file); 2722 ret = read_retpoline_hints(file);
2723 if (ret) 2723 if (ret)
2724 return ret; 2724 return ret;
2725 2725
2726 ret = read_instr_hints(file); 2726 ret = read_instr_hints(file);
2727 if (ret) 2727 if (ret)
2728 return ret; 2728 return ret;
2729 2729
2730 ret = read_validate_unret_hints(file) 2730 ret = read_validate_unret_hints(file);
2731 if (ret) 2731 if (ret)
2732 return ret; 2732 return ret;
2733 2733
2734 return 0; 2734 return 0;
2735 } 2735 }
2736 2736
2737 static bool is_special_call(struct instructio 2737 static bool is_special_call(struct instruction *insn)
2738 { 2738 {
2739 if (insn->type == INSN_CALL) { 2739 if (insn->type == INSN_CALL) {
2740 struct symbol *dest = insn_ca 2740 struct symbol *dest = insn_call_dest(insn);
2741 2741
2742 if (!dest) 2742 if (!dest)
2743 return false; 2743 return false;
2744 2744
2745 if (dest->fentry || dest->emb 2745 if (dest->fentry || dest->embedded_insn)
2746 return true; 2746 return true;
2747 } 2747 }
2748 2748
2749 return false; 2749 return false;
2750 } 2750 }
2751 2751
2752 static bool has_modified_stack_frame(struct i 2752 static bool has_modified_stack_frame(struct instruction *insn, struct insn_state *state)
2753 { 2753 {
2754 struct cfi_state *cfi = &state->cfi; 2754 struct cfi_state *cfi = &state->cfi;
2755 int i; 2755 int i;
2756 2756
2757 if (cfi->cfa.base != initial_func_cfi 2757 if (cfi->cfa.base != initial_func_cfi.cfa.base || cfi->drap)
2758 return true; 2758 return true;
2759 2759
2760 if (cfi->cfa.offset != initial_func_c 2760 if (cfi->cfa.offset != initial_func_cfi.cfa.offset)
2761 return true; 2761 return true;
2762 2762
2763 if (cfi->stack_size != initial_func_c 2763 if (cfi->stack_size != initial_func_cfi.cfa.offset)
2764 return true; 2764 return true;
2765 2765
2766 for (i = 0; i < CFI_NUM_REGS; i++) { 2766 for (i = 0; i < CFI_NUM_REGS; i++) {
2767 if (cfi->regs[i].base != init 2767 if (cfi->regs[i].base != initial_func_cfi.regs[i].base ||
2768 cfi->regs[i].offset != in 2768 cfi->regs[i].offset != initial_func_cfi.regs[i].offset)
2769 return true; 2769 return true;
2770 } 2770 }
2771 2771
2772 return false; 2772 return false;
2773 } 2773 }
2774 2774
2775 static bool check_reg_frame_pos(const struct 2775 static bool check_reg_frame_pos(const struct cfi_reg *reg,
2776 int expected_ 2776 int expected_offset)
2777 { 2777 {
2778 return reg->base == CFI_CFA && 2778 return reg->base == CFI_CFA &&
2779 reg->offset == expected_offset 2779 reg->offset == expected_offset;
2780 } 2780 }
2781 2781
2782 static bool has_valid_stack_frame(struct insn 2782 static bool has_valid_stack_frame(struct insn_state *state)
2783 { 2783 {
2784 struct cfi_state *cfi = &state->cfi; 2784 struct cfi_state *cfi = &state->cfi;
2785 2785
2786 if (cfi->cfa.base == CFI_BP && 2786 if (cfi->cfa.base == CFI_BP &&
2787 check_reg_frame_pos(&cfi->regs[CF 2787 check_reg_frame_pos(&cfi->regs[CFI_BP], -cfi->cfa.offset) &&
2788 check_reg_frame_pos(&cfi->regs[CF 2788 check_reg_frame_pos(&cfi->regs[CFI_RA], -cfi->cfa.offset + 8))
2789 return true; 2789 return true;
2790 2790
2791 if (cfi->drap && cfi->regs[CFI_BP].ba 2791 if (cfi->drap && cfi->regs[CFI_BP].base == CFI_BP)
2792 return true; 2792 return true;
2793 2793
2794 return false; 2794 return false;
2795 } 2795 }
2796 2796
2797 static int update_cfi_state_regs(struct instr 2797 static int update_cfi_state_regs(struct instruction *insn,
2798 struct cfi_ 2798 struct cfi_state *cfi,
2799 struct stac 2799 struct stack_op *op)
2800 { 2800 {
2801 struct cfi_reg *cfa = &cfi->cfa; 2801 struct cfi_reg *cfa = &cfi->cfa;
2802 2802
2803 if (cfa->base != CFI_SP && cfa->base 2803 if (cfa->base != CFI_SP && cfa->base != CFI_SP_INDIRECT)
2804 return 0; 2804 return 0;
2805 2805
2806 /* push */ 2806 /* push */
2807 if (op->dest.type == OP_DEST_PUSH || 2807 if (op->dest.type == OP_DEST_PUSH || op->dest.type == OP_DEST_PUSHF)
2808 cfa->offset += 8; 2808 cfa->offset += 8;
2809 2809
2810 /* pop */ 2810 /* pop */
2811 if (op->src.type == OP_SRC_POP || op- 2811 if (op->src.type == OP_SRC_POP || op->src.type == OP_SRC_POPF)
2812 cfa->offset -= 8; 2812 cfa->offset -= 8;
2813 2813
2814 /* add immediate to sp */ 2814 /* add immediate to sp */
2815 if (op->dest.type == OP_DEST_REG && o 2815 if (op->dest.type == OP_DEST_REG && op->src.type == OP_SRC_ADD &&
2816 op->dest.reg == CFI_SP && op->src 2816 op->dest.reg == CFI_SP && op->src.reg == CFI_SP)
2817 cfa->offset -= op->src.offset 2817 cfa->offset -= op->src.offset;
2818 2818
2819 return 0; 2819 return 0;
2820 } 2820 }
2821 2821
2822 static void save_reg(struct cfi_state *cfi, u 2822 static void save_reg(struct cfi_state *cfi, unsigned char reg, int base, int offset)
2823 { 2823 {
2824 if (arch_callee_saved_reg(reg) && 2824 if (arch_callee_saved_reg(reg) &&
2825 cfi->regs[reg].base == CFI_UNDEFI 2825 cfi->regs[reg].base == CFI_UNDEFINED) {
2826 cfi->regs[reg].base = base; 2826 cfi->regs[reg].base = base;
2827 cfi->regs[reg].offset = offse 2827 cfi->regs[reg].offset = offset;
2828 } 2828 }
2829 } 2829 }
2830 2830
2831 static void restore_reg(struct cfi_state *cfi 2831 static void restore_reg(struct cfi_state *cfi, unsigned char reg)
2832 { 2832 {
2833 cfi->regs[reg].base = initial_func_cf 2833 cfi->regs[reg].base = initial_func_cfi.regs[reg].base;
2834 cfi->regs[reg].offset = initial_func_ 2834 cfi->regs[reg].offset = initial_func_cfi.regs[reg].offset;
2835 } 2835 }
2836 2836
2837 /* 2837 /*
2838 * A note about DRAP stack alignment: 2838 * A note about DRAP stack alignment:
2839 * 2839 *
2840 * GCC has the concept of a DRAP register, wh 2840 * GCC has the concept of a DRAP register, which is used to help keep track of
2841 * the stack pointer when aligning the stack. 2841 * the stack pointer when aligning the stack. r10 or r13 is used as the DRAP
2842 * register. The typical DRAP pattern is: 2842 * register. The typical DRAP pattern is:
2843 * 2843 *
2844 * 4c 8d 54 24 08 lea 0x8(%r 2844 * 4c 8d 54 24 08 lea 0x8(%rsp),%r10
2845 * 48 83 e4 c0 and $0xfff 2845 * 48 83 e4 c0 and $0xffffffffffffffc0,%rsp
2846 * 41 ff 72 f8 pushq -0x8(% 2846 * 41 ff 72 f8 pushq -0x8(%r10)
2847 * 55 push %rbp 2847 * 55 push %rbp
2848 * 48 89 e5 mov %rsp,% 2848 * 48 89 e5 mov %rsp,%rbp
2849 * (more pushes) 2849 * (more pushes)
2850 * 41 52 push %r10 2850 * 41 52 push %r10
2851 * ... 2851 * ...
2852 * 41 5a pop %r10 2852 * 41 5a pop %r10
2853 * (more pops) 2853 * (more pops)
2854 * 5d pop %rbp 2854 * 5d pop %rbp
2855 * 49 8d 62 f8 lea -0x8(% 2855 * 49 8d 62 f8 lea -0x8(%r10),%rsp
2856 * c3 retq 2856 * c3 retq
2857 * 2857 *
2858 * There are some variations in the epilogues 2858 * There are some variations in the epilogues, like:
2859 * 2859 *
2860 * 5b pop %rbx 2860 * 5b pop %rbx
2861 * 41 5a pop %r10 2861 * 41 5a pop %r10
2862 * 41 5c pop %r12 2862 * 41 5c pop %r12
2863 * 41 5d pop %r13 2863 * 41 5d pop %r13
2864 * 41 5e pop %r14 2864 * 41 5e pop %r14
2865 * c9 leaveq 2865 * c9 leaveq
2866 * 49 8d 62 f8 lea -0x8(% 2866 * 49 8d 62 f8 lea -0x8(%r10),%rsp
2867 * c3 retq 2867 * c3 retq
2868 * 2868 *
2869 * and: 2869 * and:
2870 * 2870 *
2871 * 4c 8b 55 e8 mov -0x18( 2871 * 4c 8b 55 e8 mov -0x18(%rbp),%r10
2872 * 48 8b 5d e0 mov -0x20( 2872 * 48 8b 5d e0 mov -0x20(%rbp),%rbx
2873 * 4c 8b 65 f0 mov -0x10( 2873 * 4c 8b 65 f0 mov -0x10(%rbp),%r12
2874 * 4c 8b 6d f8 mov -0x8(% 2874 * 4c 8b 6d f8 mov -0x8(%rbp),%r13
2875 * c9 leaveq 2875 * c9 leaveq
2876 * 49 8d 62 f8 lea -0x8(% 2876 * 49 8d 62 f8 lea -0x8(%r10),%rsp
2877 * c3 retq 2877 * c3 retq
2878 * 2878 *
2879 * Sometimes r13 is used as the DRAP register 2879 * Sometimes r13 is used as the DRAP register, in which case it's saved and
2880 * restored beforehand: 2880 * restored beforehand:
2881 * 2881 *
2882 * 41 55 push %r13 2882 * 41 55 push %r13
2883 * 4c 8d 6c 24 10 lea 0x10(% 2883 * 4c 8d 6c 24 10 lea 0x10(%rsp),%r13
2884 * 48 83 e4 f0 and $0xfff 2884 * 48 83 e4 f0 and $0xfffffffffffffff0,%rsp
2885 * ... 2885 * ...
2886 * 49 8d 65 f0 lea -0x10( 2886 * 49 8d 65 f0 lea -0x10(%r13),%rsp
2887 * 41 5d pop %r13 2887 * 41 5d pop %r13
2888 * c3 retq 2888 * c3 retq
2889 */ 2889 */
2890 static int update_cfi_state(struct instructio 2890 static int update_cfi_state(struct instruction *insn,
2891 struct instructio 2891 struct instruction *next_insn,
2892 struct cfi_state 2892 struct cfi_state *cfi, struct stack_op *op)
2893 { 2893 {
2894 struct cfi_reg *cfa = &cfi->cfa; 2894 struct cfi_reg *cfa = &cfi->cfa;
2895 struct cfi_reg *regs = cfi->regs; 2895 struct cfi_reg *regs = cfi->regs;
2896 2896
2897 /* ignore UNWIND_HINT_UNDEFINED regio 2897 /* ignore UNWIND_HINT_UNDEFINED regions */
2898 if (cfi->force_undefined) 2898 if (cfi->force_undefined)
2899 return 0; 2899 return 0;
2900 2900
2901 /* stack operations don't make sense 2901 /* stack operations don't make sense with an undefined CFA */
2902 if (cfa->base == CFI_UNDEFINED) { 2902 if (cfa->base == CFI_UNDEFINED) {
2903 if (insn_func(insn)) { 2903 if (insn_func(insn)) {
2904 WARN_INSN(insn, "unde 2904 WARN_INSN(insn, "undefined stack state");
2905 return -1; 2905 return -1;
2906 } 2906 }
2907 return 0; 2907 return 0;
2908 } 2908 }
2909 2909
2910 if (cfi->type == UNWIND_HINT_TYPE_REG 2910 if (cfi->type == UNWIND_HINT_TYPE_REGS ||
2911 cfi->type == UNWIND_HINT_TYPE_REG 2911 cfi->type == UNWIND_HINT_TYPE_REGS_PARTIAL)
2912 return update_cfi_state_regs( 2912 return update_cfi_state_regs(insn, cfi, op);
2913 2913
2914 switch (op->dest.type) { 2914 switch (op->dest.type) {
2915 2915
2916 case OP_DEST_REG: 2916 case OP_DEST_REG:
2917 switch (op->src.type) { 2917 switch (op->src.type) {
2918 2918
2919 case OP_SRC_REG: 2919 case OP_SRC_REG:
2920 if (op->src.reg == CF 2920 if (op->src.reg == CFI_SP && op->dest.reg == CFI_BP &&
2921 cfa->base == CFI_ 2921 cfa->base == CFI_SP &&
2922 check_reg_frame_p 2922 check_reg_frame_pos(®s[CFI_BP], -cfa->offset)) {
2923 2923
2924 /* mov %rsp, 2924 /* mov %rsp, %rbp */
2925 cfa->base = o 2925 cfa->base = op->dest.reg;
2926 cfi->bp_scrat 2926 cfi->bp_scratch = false;
2927 } 2927 }
2928 2928
2929 else if (op->src.reg 2929 else if (op->src.reg == CFI_SP &&
2930 op->dest.reg 2930 op->dest.reg == CFI_BP && cfi->drap) {
2931 2931
2932 /* drap: mov 2932 /* drap: mov %rsp, %rbp */
2933 regs[CFI_BP]. 2933 regs[CFI_BP].base = CFI_BP;
2934 regs[CFI_BP]. 2934 regs[CFI_BP].offset = -cfi->stack_size;
2935 cfi->bp_scrat 2935 cfi->bp_scratch = false;
2936 } 2936 }
2937 2937
2938 else if (op->src.reg 2938 else if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
2939 2939
2940 /* 2940 /*
2941 * mov %rsp, 2941 * mov %rsp, %reg
2942 * 2942 *
2943 * This is ne 2943 * This is needed for the rare case where GCC
2944 * does: 2944 * does:
2945 * 2945 *
2946 * mov % 2946 * mov %rsp, %rax
2947 * ... 2947 * ...
2948 * mov % 2948 * mov %rax, %rsp
2949 */ 2949 */
2950 cfi->vals[op- 2950 cfi->vals[op->dest.reg].base = CFI_CFA;
2951 cfi->vals[op- 2951 cfi->vals[op->dest.reg].offset = -cfi->stack_size;
2952 } 2952 }
2953 2953
2954 else if (op->src.reg 2954 else if (op->src.reg == CFI_BP && op->dest.reg == CFI_SP &&
2955 (cfa->base = 2955 (cfa->base == CFI_BP || cfa->base == cfi->drap_reg)) {
2956 2956
2957 /* 2957 /*
2958 * mov %rbp, 2958 * mov %rbp, %rsp
2959 * 2959 *
2960 * Restore th 2960 * Restore the original stack pointer (Clang).
2961 */ 2961 */
2962 cfi->stack_si 2962 cfi->stack_size = -cfi->regs[CFI_BP].offset;
2963 } 2963 }
2964 2964
2965 else if (op->dest.reg 2965 else if (op->dest.reg == cfa->base) {
2966 2966
2967 /* mov %reg, 2967 /* mov %reg, %rsp */
2968 if (cfa->base 2968 if (cfa->base == CFI_SP &&
2969 cfi->vals 2969 cfi->vals[op->src.reg].base == CFI_CFA) {
2970 2970
2971 /* 2971 /*
2972 * Th 2972 * This is needed for the rare case
2973 * wh 2973 * where GCC does something dumb like:
2974 * 2974 *
2975 * 2975 * lea 0x8(%rsp), %rcx
2976 * 2976 * ...
2977 * 2977 * mov %rcx, %rsp
2978 */ 2978 */
2979 cfa-> 2979 cfa->offset = -cfi->vals[op->src.reg].offset;
2980 cfi-> 2980 cfi->stack_size = cfa->offset;
2981 2981
2982 } else if (cf 2982 } else if (cfa->base == CFI_SP &&
2983 cf 2983 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2984 cf 2984 cfi->vals[op->src.reg].offset == cfa->offset) {
2985 2985
2986 /* 2986 /*
2987 * St 2987 * Stack swizzle:
2988 * 2988 *
2989 * 1: 2989 * 1: mov %rsp, (%[tos])
2990 * 2: 2990 * 2: mov %[tos], %rsp
2991 * 2991 * ...
2992 * 3: 2992 * 3: pop %rsp
2993 * 2993 *
2994 * Wh 2994 * Where:
2995 * 2995 *
2996 * 1 2996 * 1 - places a pointer to the previous
2997 * 2997 * stack at the Top-of-Stack of the
2998 * 2998 * new stack.
2999 * 2999 *
3000 * 2 3000 * 2 - switches to the new stack.
3001 * 3001 *
3002 * 3 3002 * 3 - pops the Top-of-Stack to restore
3003 * 3003 * the original stack.
3004 * 3004 *
3005 * No 3005 * Note: we set base to SP_INDIRECT
3006 * he 3006 * here and preserve offset. Therefore
3007 * wh 3007 * when the unwinder reaches ToS it
3008 * wi 3008 * will dereference SP and then add the
3009 * of 3009 * offset to find the next frame, IOW:
3010 * (% 3010 * (%rsp) + offset.
3011 */ 3011 */
3012 cfa-> 3012 cfa->base = CFI_SP_INDIRECT;
3013 3013
3014 } else { 3014 } else {
3015 cfa-> 3015 cfa->base = CFI_UNDEFINED;
3016 cfa-> 3016 cfa->offset = 0;
3017 } 3017 }
3018 } 3018 }
3019 3019
3020 else if (op->dest.reg 3020 else if (op->dest.reg == CFI_SP &&
3021 cfi->vals[op 3021 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
3022 cfi->vals[op 3022 cfi->vals[op->src.reg].offset == cfa->offset) {
3023 3023
3024 /* 3024 /*
3025 * The same s 3025 * The same stack swizzle case 2) as above. But
3026 * because we 3026 * because we can't change cfa->base, case 3)
3027 * will becom 3027 * will become a regular POP. Pretend we're a
3028 * PUSH so th 3028 * PUSH so things don't go unbalanced.
3029 */ 3029 */
3030 cfi->stack_si 3030 cfi->stack_size += 8;
3031 } 3031 }
3032 3032
3033 3033
3034 break; 3034 break;
3035 3035
3036 case OP_SRC_ADD: 3036 case OP_SRC_ADD:
3037 if (op->dest.reg == C 3037 if (op->dest.reg == CFI_SP && op->src.reg == CFI_SP) {
3038 3038
3039 /* add imm, % 3039 /* add imm, %rsp */
3040 cfi->stack_si 3040 cfi->stack_size -= op->src.offset;
3041 if (cfa->base 3041 if (cfa->base == CFI_SP)
3042 cfa-> 3042 cfa->offset -= op->src.offset;
3043 break; 3043 break;
3044 } 3044 }
3045 3045
3046 if (op->dest.reg == C 3046 if (op->dest.reg == CFI_BP && op->src.reg == CFI_SP &&
3047 insn->sym->frame_ 3047 insn->sym->frame_pointer) {
3048 /* addi.d fp, 3048 /* addi.d fp,sp,imm on LoongArch */
3049 if (cfa->base 3049 if (cfa->base == CFI_SP && cfa->offset == op->src.offset) {
3050 cfa-> 3050 cfa->base = CFI_BP;
3051 cfa-> 3051 cfa->offset = 0;
3052 } 3052 }
3053 break; 3053 break;
3054 } 3054 }
3055 3055
3056 if (op->dest.reg == C 3056 if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) {
3057 /* addi.d sp, 3057 /* addi.d sp,fp,imm on LoongArch */
3058 if (cfa->base 3058 if (cfa->base == CFI_BP && cfa->offset == 0) {
3059 if (i 3059 if (insn->sym->frame_pointer) {
3060 3060 cfa->base = CFI_SP;
3061 3061 cfa->offset = -op->src.offset;
3062 } 3062 }
3063 } else { 3063 } else {
3064 /* le 3064 /* lea disp(%rbp), %rsp */
3065 cfi-> 3065 cfi->stack_size = -(op->src.offset + regs[CFI_BP].offset);
3066 } 3066 }
3067 break; 3067 break;
3068 } 3068 }
3069 3069
3070 if (op->src.reg == CF 3070 if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
3071 3071
3072 /* drap: lea 3072 /* drap: lea disp(%rsp), %drap */
3073 cfi->drap_reg 3073 cfi->drap_reg = op->dest.reg;
3074 3074
3075 /* 3075 /*
3076 * lea disp(% 3076 * lea disp(%rsp), %reg
3077 * 3077 *
3078 * This is ne 3078 * This is needed for the rare case where GCC
3079 * does somet 3079 * does something dumb like:
3080 * 3080 *
3081 * lea 0 3081 * lea 0x8(%rsp), %rcx
3082 * ... 3082 * ...
3083 * mov % 3083 * mov %rcx, %rsp
3084 */ 3084 */
3085 cfi->vals[op- 3085 cfi->vals[op->dest.reg].base = CFI_CFA;
3086 cfi->vals[op- 3086 cfi->vals[op->dest.reg].offset = \
3087 -cfi- 3087 -cfi->stack_size + op->src.offset;
3088 3088
3089 break; 3089 break;
3090 } 3090 }
3091 3091
3092 if (cfi->drap && op-> 3092 if (cfi->drap && op->dest.reg == CFI_SP &&
3093 op->src.reg == cf 3093 op->src.reg == cfi->drap_reg) {
3094 3094
3095 /* drap: lea 3095 /* drap: lea disp(%drap), %rsp */
3096 cfa->base = C 3096 cfa->base = CFI_SP;
3097 cfa->offset = 3097 cfa->offset = cfi->stack_size = -op->src.offset;
3098 cfi->drap_reg 3098 cfi->drap_reg = CFI_UNDEFINED;
3099 cfi->drap = f 3099 cfi->drap = false;
3100 break; 3100 break;
3101 } 3101 }
3102 3102
3103 if (op->dest.reg == c 3103 if (op->dest.reg == cfi->cfa.base && !(next_insn && next_insn->hint)) {
3104 WARN_INSN(ins 3104 WARN_INSN(insn, "unsupported stack register modification");
3105 return -1; 3105 return -1;
3106 } 3106 }
3107 3107
3108 break; 3108 break;
3109 3109
3110 case OP_SRC_AND: 3110 case OP_SRC_AND:
3111 if (op->dest.reg != C 3111 if (op->dest.reg != CFI_SP ||
3112 (cfi->drap_reg != 3112 (cfi->drap_reg != CFI_UNDEFINED && cfa->base != CFI_SP) ||
3113 (cfi->drap_reg == 3113 (cfi->drap_reg == CFI_UNDEFINED && cfa->base != CFI_BP)) {
3114 WARN_INSN(ins 3114 WARN_INSN(insn, "unsupported stack pointer realignment");
3115 return -1; 3115 return -1;
3116 } 3116 }
3117 3117
3118 if (cfi->drap_reg != 3118 if (cfi->drap_reg != CFI_UNDEFINED) {
3119 /* drap: and 3119 /* drap: and imm, %rsp */
3120 cfa->base = c 3120 cfa->base = cfi->drap_reg;
3121 cfa->offset = 3121 cfa->offset = cfi->stack_size = 0;
3122 cfi->drap = t 3122 cfi->drap = true;
3123 } 3123 }
3124 3124
3125 /* 3125 /*
3126 * Older versions of 3126 * Older versions of GCC (4.8ish) realign the stack
3127 * without DRAP, with 3127 * without DRAP, with a frame pointer.
3128 */ 3128 */
3129 3129
3130 break; 3130 break;
3131 3131
3132 case OP_SRC_POP: 3132 case OP_SRC_POP:
3133 case OP_SRC_POPF: 3133 case OP_SRC_POPF:
3134 if (op->dest.reg == C 3134 if (op->dest.reg == CFI_SP && cfa->base == CFI_SP_INDIRECT) {
3135 3135
3136 /* pop %rsp; 3136 /* pop %rsp; # restore from a stack swizzle */
3137 cfa->base = C 3137 cfa->base = CFI_SP;
3138 break; 3138 break;
3139 } 3139 }
3140 3140
3141 if (!cfi->drap && op- 3141 if (!cfi->drap && op->dest.reg == cfa->base) {
3142 3142
3143 /* pop %rbp * 3143 /* pop %rbp */
3144 cfa->base = C 3144 cfa->base = CFI_SP;
3145 } 3145 }
3146 3146
3147 if (cfi->drap && cfa- 3147 if (cfi->drap && cfa->base == CFI_BP_INDIRECT &&
3148 op->dest.reg == c 3148 op->dest.reg == cfi->drap_reg &&
3149 cfi->drap_offset 3149 cfi->drap_offset == -cfi->stack_size) {
3150 3150
3151 /* drap: pop 3151 /* drap: pop %drap */
3152 cfa->base = c 3152 cfa->base = cfi->drap_reg;
3153 cfa->offset = 3153 cfa->offset = 0;
3154 cfi->drap_off 3154 cfi->drap_offset = -1;
3155 3155
3156 } else if (cfi->stack 3156 } else if (cfi->stack_size == -regs[op->dest.reg].offset) {
3157 3157
3158 /* pop %reg * 3158 /* pop %reg */
3159 restore_reg(c 3159 restore_reg(cfi, op->dest.reg);
3160 } 3160 }
3161 3161
3162 cfi->stack_size -= 8; 3162 cfi->stack_size -= 8;
3163 if (cfa->base == CFI_ 3163 if (cfa->base == CFI_SP)
3164 cfa->offset - 3164 cfa->offset -= 8;
3165 3165
3166 break; 3166 break;
3167 3167
3168 case OP_SRC_REG_INDIRECT: 3168 case OP_SRC_REG_INDIRECT:
3169 if (!cfi->drap && op- 3169 if (!cfi->drap && op->dest.reg == cfa->base &&
3170 op->dest.reg == C 3170 op->dest.reg == CFI_BP) {
3171 3171
3172 /* mov disp(% 3172 /* mov disp(%rsp), %rbp */
3173 cfa->base = C 3173 cfa->base = CFI_SP;
3174 cfa->offset = 3174 cfa->offset = cfi->stack_size;
3175 } 3175 }
3176 3176
3177 if (cfi->drap && op-> 3177 if (cfi->drap && op->src.reg == CFI_BP &&
3178 op->src.offset == 3178 op->src.offset == cfi->drap_offset) {
3179 3179
3180 /* drap: mov 3180 /* drap: mov disp(%rbp), %drap */
3181 cfa->base = c 3181 cfa->base = cfi->drap_reg;
3182 cfa->offset = 3182 cfa->offset = 0;
3183 cfi->drap_off 3183 cfi->drap_offset = -1;
3184 } 3184 }
3185 3185
3186 if (cfi->drap && op-> 3186 if (cfi->drap && op->src.reg == CFI_BP &&
3187 op->src.offset == 3187 op->src.offset == regs[op->dest.reg].offset) {
3188 3188
3189 /* drap: mov 3189 /* drap: mov disp(%rbp), %reg */
3190 restore_reg(c 3190 restore_reg(cfi, op->dest.reg);
3191 3191
3192 } else if (op->src.re 3192 } else if (op->src.reg == cfa->base &&
3193 op->src.offset == 3193 op->src.offset == regs[op->dest.reg].offset + cfa->offset) {
3194 3194
3195 /* mov disp(% 3195 /* mov disp(%rbp), %reg */
3196 /* mov disp(% 3196 /* mov disp(%rsp), %reg */
3197 restore_reg(c 3197 restore_reg(cfi, op->dest.reg);
3198 3198
3199 } else if (op->src.re 3199 } else if (op->src.reg == CFI_SP &&
3200 op->src.of 3200 op->src.offset == regs[op->dest.reg].offset + cfi->stack_size) {
3201 3201
3202 /* mov disp(% 3202 /* mov disp(%rsp), %reg */
3203 restore_reg(c 3203 restore_reg(cfi, op->dest.reg);
3204 } 3204 }
3205 3205
3206 break; 3206 break;
3207 3207
3208 default: 3208 default:
3209 WARN_INSN(insn, "unkn 3209 WARN_INSN(insn, "unknown stack-related instruction");
3210 return -1; 3210 return -1;
3211 } 3211 }
3212 3212
3213 break; 3213 break;
3214 3214
3215 case OP_DEST_PUSH: 3215 case OP_DEST_PUSH:
3216 case OP_DEST_PUSHF: 3216 case OP_DEST_PUSHF:
3217 cfi->stack_size += 8; 3217 cfi->stack_size += 8;
3218 if (cfa->base == CFI_SP) 3218 if (cfa->base == CFI_SP)
3219 cfa->offset += 8; 3219 cfa->offset += 8;
3220 3220
3221 if (op->src.type != OP_SRC_RE 3221 if (op->src.type != OP_SRC_REG)
3222 break; 3222 break;
3223 3223
3224 if (cfi->drap) { 3224 if (cfi->drap) {
3225 if (op->src.reg == cf 3225 if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
3226 3226
3227 /* drap: push 3227 /* drap: push %drap */
3228 cfa->base = C 3228 cfa->base = CFI_BP_INDIRECT;
3229 cfa->offset = 3229 cfa->offset = -cfi->stack_size;
3230 3230
3231 /* save drap 3231 /* save drap so we know when to restore it */
3232 cfi->drap_off 3232 cfi->drap_offset = -cfi->stack_size;
3233 3233
3234 } else if (op->src.re 3234 } else if (op->src.reg == CFI_BP && cfa->base == cfi->drap_reg) {
3235 3235
3236 /* drap: push 3236 /* drap: push %rbp */
3237 cfi->stack_si 3237 cfi->stack_size = 0;
3238 3238
3239 } else { 3239 } else {
3240 3240
3241 /* drap: push 3241 /* drap: push %reg */
3242 save_reg(cfi, 3242 save_reg(cfi, op->src.reg, CFI_BP, -cfi->stack_size);
3243 } 3243 }
3244 3244
3245 } else { 3245 } else {
3246 3246
3247 /* push %reg */ 3247 /* push %reg */
3248 save_reg(cfi, op->src 3248 save_reg(cfi, op->src.reg, CFI_CFA, -cfi->stack_size);
3249 } 3249 }
3250 3250
3251 /* detect when asm code uses 3251 /* detect when asm code uses rbp as a scratch register */
3252 if (opts.stackval && insn_fun 3252 if (opts.stackval && insn_func(insn) && op->src.reg == CFI_BP &&
3253 cfa->base != CFI_BP) 3253 cfa->base != CFI_BP)
3254 cfi->bp_scratch = tru 3254 cfi->bp_scratch = true;
3255 break; 3255 break;
3256 3256
3257 case OP_DEST_REG_INDIRECT: 3257 case OP_DEST_REG_INDIRECT:
3258 3258
3259 if (cfi->drap) { 3259 if (cfi->drap) {
3260 if (op->src.reg == cf 3260 if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
3261 3261
3262 /* drap: mov 3262 /* drap: mov %drap, disp(%rbp) */
3263 cfa->base = C 3263 cfa->base = CFI_BP_INDIRECT;
3264 cfa->offset = 3264 cfa->offset = op->dest.offset;
3265 3265
3266 /* save drap 3266 /* save drap offset so we know when to restore it */
3267 cfi->drap_off 3267 cfi->drap_offset = op->dest.offset;
3268 } else { 3268 } else {
3269 3269
3270 /* drap: mov 3270 /* drap: mov reg, disp(%rbp) */
3271 save_reg(cfi, 3271 save_reg(cfi, op->src.reg, CFI_BP, op->dest.offset);
3272 } 3272 }
3273 3273
3274 } else if (op->dest.reg == cf 3274 } else if (op->dest.reg == cfa->base) {
3275 3275
3276 /* mov reg, disp(%rbp 3276 /* mov reg, disp(%rbp) */
3277 /* mov reg, disp(%rsp 3277 /* mov reg, disp(%rsp) */
3278 save_reg(cfi, op->src 3278 save_reg(cfi, op->src.reg, CFI_CFA,
3279 op->dest.off 3279 op->dest.offset - cfi->cfa.offset);
3280 3280
3281 } else if (op->dest.reg == CF 3281 } else if (op->dest.reg == CFI_SP) {
3282 3282
3283 /* mov reg, disp(%rsp 3283 /* mov reg, disp(%rsp) */
3284 save_reg(cfi, op->src 3284 save_reg(cfi, op->src.reg, CFI_CFA,
3285 op->dest.off 3285 op->dest.offset - cfi->stack_size);
3286 3286
3287 } else if (op->src.reg == CFI 3287 } else if (op->src.reg == CFI_SP && op->dest.offset == 0) {
3288 3288
3289 /* mov %rsp, (%reg); 3289 /* mov %rsp, (%reg); # setup a stack swizzle. */
3290 cfi->vals[op->dest.re 3290 cfi->vals[op->dest.reg].base = CFI_SP_INDIRECT;
3291 cfi->vals[op->dest.re 3291 cfi->vals[op->dest.reg].offset = cfa->offset;
3292 } 3292 }
3293 3293
3294 break; 3294 break;
3295 3295
3296 case OP_DEST_MEM: 3296 case OP_DEST_MEM:
3297 if (op->src.type != OP_SRC_PO 3297 if (op->src.type != OP_SRC_POP && op->src.type != OP_SRC_POPF) {
3298 WARN_INSN(insn, "unkn 3298 WARN_INSN(insn, "unknown stack-related memory operation");
3299 return -1; 3299 return -1;
3300 } 3300 }
3301 3301
3302 /* pop mem */ 3302 /* pop mem */
3303 cfi->stack_size -= 8; 3303 cfi->stack_size -= 8;
3304 if (cfa->base == CFI_SP) 3304 if (cfa->base == CFI_SP)
3305 cfa->offset -= 8; 3305 cfa->offset -= 8;
3306 3306
3307 break; 3307 break;
3308 3308
3309 default: 3309 default:
3310 WARN_INSN(insn, "unknown stac 3310 WARN_INSN(insn, "unknown stack-related instruction");
3311 return -1; 3311 return -1;
3312 } 3312 }
3313 3313
3314 return 0; 3314 return 0;
3315 } 3315 }
3316 3316
3317 /* 3317 /*
3318 * The stack layouts of alternatives instruct 3318 * The stack layouts of alternatives instructions can sometimes diverge when
3319 * they have stack modifications. That's fin 3319 * they have stack modifications. That's fine as long as the potential stack
3320 * layouts don't conflict at any given potent 3320 * layouts don't conflict at any given potential instruction boundary.
3321 * 3321 *
3322 * Flatten the CFIs of the different alternat 3322 * Flatten the CFIs of the different alternative code streams (both original
3323 * and replacement) into a single shared CFI 3323 * and replacement) into a single shared CFI array which can be used to detect
3324 * conflicts and nicely feed a linear array o 3324 * conflicts and nicely feed a linear array of ORC entries to the unwinder.
3325 */ 3325 */
3326 static int propagate_alt_cfi(struct objtool_f 3326 static int propagate_alt_cfi(struct objtool_file *file, struct instruction *insn)
3327 { 3327 {
3328 struct cfi_state **alt_cfi; 3328 struct cfi_state **alt_cfi;
3329 int group_off; 3329 int group_off;
3330 3330
3331 if (!insn->alt_group) 3331 if (!insn->alt_group)
3332 return 0; 3332 return 0;
3333 3333
3334 if (!insn->cfi) { 3334 if (!insn->cfi) {
3335 WARN("CFI missing"); 3335 WARN("CFI missing");
3336 return -1; 3336 return -1;
3337 } 3337 }
3338 3338
3339 alt_cfi = insn->alt_group->cfi; 3339 alt_cfi = insn->alt_group->cfi;
3340 group_off = insn->offset - insn->alt_ 3340 group_off = insn->offset - insn->alt_group->first_insn->offset;
3341 3341
3342 if (!alt_cfi[group_off]) { 3342 if (!alt_cfi[group_off]) {
3343 alt_cfi[group_off] = insn->cf 3343 alt_cfi[group_off] = insn->cfi;
3344 } else { 3344 } else {
3345 if (cficmp(alt_cfi[group_off] 3345 if (cficmp(alt_cfi[group_off], insn->cfi)) {
3346 struct alt_group *ori 3346 struct alt_group *orig_group = insn->alt_group->orig_group ?: insn->alt_group;
3347 struct instruction *o 3347 struct instruction *orig = orig_group->first_insn;
3348 char *where = offstr( 3348 char *where = offstr(insn->sec, insn->offset);
3349 WARN_INSN(orig, "stac 3349 WARN_INSN(orig, "stack layout conflict in alternatives: %s", where);
3350 free(where); 3350 free(where);
3351 return -1; 3351 return -1;
3352 } 3352 }
3353 } 3353 }
3354 3354
3355 return 0; 3355 return 0;
3356 } 3356 }
3357 3357
3358 static int handle_insn_ops(struct instruction 3358 static int handle_insn_ops(struct instruction *insn,
3359 struct instruction 3359 struct instruction *next_insn,
3360 struct insn_state 3360 struct insn_state *state)
3361 { 3361 {
3362 struct stack_op *op; 3362 struct stack_op *op;
3363 3363
3364 for (op = insn->stack_ops; op; op = o 3364 for (op = insn->stack_ops; op; op = op->next) {
3365 3365
3366 if (update_cfi_state(insn, ne 3366 if (update_cfi_state(insn, next_insn, &state->cfi, op))
3367 return 1; 3367 return 1;
3368 3368
3369 if (!insn->alt_group) 3369 if (!insn->alt_group)
3370 continue; 3370 continue;
3371 3371
3372 if (op->dest.type == OP_DEST_ 3372 if (op->dest.type == OP_DEST_PUSHF) {
3373 if (!state->uaccess_s 3373 if (!state->uaccess_stack) {
3374 state->uacces 3374 state->uaccess_stack = 1;
3375 } else if (state->uac 3375 } else if (state->uaccess_stack >> 31) {
3376 WARN_INSN(ins 3376 WARN_INSN(insn, "PUSHF stack exhausted");
3377 return 1; 3377 return 1;
3378 } 3378 }
3379 state->uaccess_stack 3379 state->uaccess_stack <<= 1;
3380 state->uaccess_stack 3380 state->uaccess_stack |= state->uaccess;
3381 } 3381 }
3382 3382
3383 if (op->src.type == OP_SRC_PO 3383 if (op->src.type == OP_SRC_POPF) {
3384 if (state->uaccess_st 3384 if (state->uaccess_stack) {
3385 state->uacces 3385 state->uaccess = state->uaccess_stack & 1;
3386 state->uacces 3386 state->uaccess_stack >>= 1;
3387 if (state->ua 3387 if (state->uaccess_stack == 1)
3388 state 3388 state->uaccess_stack = 0;
3389 } 3389 }
3390 } 3390 }
3391 } 3391 }
3392 3392
3393 return 0; 3393 return 0;
3394 } 3394 }
3395 3395
3396 static bool insn_cfi_match(struct instruction 3396 static bool insn_cfi_match(struct instruction *insn, struct cfi_state *cfi2)
3397 { 3397 {
3398 struct cfi_state *cfi1 = insn->cfi; 3398 struct cfi_state *cfi1 = insn->cfi;
3399 int i; 3399 int i;
3400 3400
3401 if (!cfi1) { 3401 if (!cfi1) {
3402 WARN("CFI missing"); 3402 WARN("CFI missing");
3403 return false; 3403 return false;
3404 } 3404 }
3405 3405
3406 if (memcmp(&cfi1->cfa, &cfi2->cfa, si 3406 if (memcmp(&cfi1->cfa, &cfi2->cfa, sizeof(cfi1->cfa))) {
3407 3407
3408 WARN_INSN(insn, "stack state 3408 WARN_INSN(insn, "stack state mismatch: cfa1=%d%+d cfa2=%d%+d",
3409 cfi1->cfa.base, cfi 3409 cfi1->cfa.base, cfi1->cfa.offset,
3410 cfi2->cfa.base, cfi 3410 cfi2->cfa.base, cfi2->cfa.offset);
3411 3411
3412 } else if (memcmp(&cfi1->regs, &cfi2- 3412 } else if (memcmp(&cfi1->regs, &cfi2->regs, sizeof(cfi1->regs))) {
3413 for (i = 0; i < CFI_NUM_REGS; 3413 for (i = 0; i < CFI_NUM_REGS; i++) {
3414 if (!memcmp(&cfi1->re 3414 if (!memcmp(&cfi1->regs[i], &cfi2->regs[i],
3415 sizeof(st 3415 sizeof(struct cfi_reg)))
3416 continue; 3416 continue;
3417 3417
3418 WARN_INSN(insn, "stac 3418 WARN_INSN(insn, "stack state mismatch: reg1[%d]=%d%+d reg2[%d]=%d%+d",
3419 i, cfi1->re 3419 i, cfi1->regs[i].base, cfi1->regs[i].offset,
3420 i, cfi2->re 3420 i, cfi2->regs[i].base, cfi2->regs[i].offset);
3421 break; 3421 break;
3422 } 3422 }
3423 3423
3424 } else if (cfi1->type != cfi2->type) 3424 } else if (cfi1->type != cfi2->type) {
3425 3425
3426 WARN_INSN(insn, "stack state 3426 WARN_INSN(insn, "stack state mismatch: type1=%d type2=%d",
3427 cfi1->type, cfi2->t 3427 cfi1->type, cfi2->type);
3428 3428
3429 } else if (cfi1->drap != cfi2->drap | 3429 } else if (cfi1->drap != cfi2->drap ||
3430 (cfi1->drap && cfi1->drap_ 3430 (cfi1->drap && cfi1->drap_reg != cfi2->drap_reg) ||
3431 (cfi1->drap && cfi1->drap_ 3431 (cfi1->drap && cfi1->drap_offset != cfi2->drap_offset)) {
3432 3432
3433 WARN_INSN(insn, "stack state 3433 WARN_INSN(insn, "stack state mismatch: drap1=%d(%d,%d) drap2=%d(%d,%d)",
3434 cfi1->drap, cfi1->d 3434 cfi1->drap, cfi1->drap_reg, cfi1->drap_offset,
3435 cfi2->drap, cfi2->d 3435 cfi2->drap, cfi2->drap_reg, cfi2->drap_offset);
3436 3436
3437 } else 3437 } else
3438 return true; 3438 return true;
3439 3439
3440 return false; 3440 return false;
3441 } 3441 }
3442 3442
3443 static inline bool func_uaccess_safe(struct s 3443 static inline bool func_uaccess_safe(struct symbol *func)
3444 { 3444 {
3445 if (func) 3445 if (func)
3446 return func->uaccess_safe; 3446 return func->uaccess_safe;
3447 3447
3448 return false; 3448 return false;
3449 } 3449 }
3450 3450
3451 static inline const char *call_dest_name(stru 3451 static inline const char *call_dest_name(struct instruction *insn)
3452 { 3452 {
3453 static char pvname[19]; 3453 static char pvname[19];
3454 struct reloc *reloc; 3454 struct reloc *reloc;
3455 int idx; 3455 int idx;
3456 3456
3457 if (insn_call_dest(insn)) 3457 if (insn_call_dest(insn))
3458 return insn_call_dest(insn)-> 3458 return insn_call_dest(insn)->name;
3459 3459
3460 reloc = insn_reloc(NULL, insn); 3460 reloc = insn_reloc(NULL, insn);
3461 if (reloc && !strcmp(reloc->sym->name 3461 if (reloc && !strcmp(reloc->sym->name, "pv_ops")) {
3462 idx = (reloc_addend(reloc) / 3462 idx = (reloc_addend(reloc) / sizeof(void *));
3463 snprintf(pvname, sizeof(pvnam 3463 snprintf(pvname, sizeof(pvname), "pv_ops[%d]", idx);
3464 return pvname; 3464 return pvname;
3465 } 3465 }
3466 3466
3467 return "{dynamic}"; 3467 return "{dynamic}";
3468 } 3468 }
3469 3469
3470 static bool pv_call_dest(struct objtool_file 3470 static bool pv_call_dest(struct objtool_file *file, struct instruction *insn)
3471 { 3471 {
3472 struct symbol *target; 3472 struct symbol *target;
3473 struct reloc *reloc; 3473 struct reloc *reloc;
3474 int idx; 3474 int idx;
3475 3475
3476 reloc = insn_reloc(file, insn); 3476 reloc = insn_reloc(file, insn);
3477 if (!reloc || strcmp(reloc->sym->name 3477 if (!reloc || strcmp(reloc->sym->name, "pv_ops"))
3478 return false; 3478 return false;
3479 3479
3480 idx = (arch_dest_reloc_offset(reloc_a 3480 idx = (arch_dest_reloc_offset(reloc_addend(reloc)) / sizeof(void *));
3481 3481
3482 if (file->pv_ops[idx].clean) 3482 if (file->pv_ops[idx].clean)
3483 return true; 3483 return true;
3484 3484
3485 file->pv_ops[idx].clean = true; 3485 file->pv_ops[idx].clean = true;
3486 3486
3487 list_for_each_entry(target, &file->pv 3487 list_for_each_entry(target, &file->pv_ops[idx].targets, pv_target) {
3488 if (!target->sec->noinstr) { 3488 if (!target->sec->noinstr) {
3489 WARN("pv_ops[%d]: %s" 3489 WARN("pv_ops[%d]: %s", idx, target->name);
3490 file->pv_ops[idx].cle 3490 file->pv_ops[idx].clean = false;
3491 } 3491 }
3492 } 3492 }
3493 3493
3494 return file->pv_ops[idx].clean; 3494 return file->pv_ops[idx].clean;
3495 } 3495 }
3496 3496
3497 static inline bool noinstr_call_dest(struct o 3497 static inline bool noinstr_call_dest(struct objtool_file *file,
3498 struct i 3498 struct instruction *insn,
3499 struct s 3499 struct symbol *func)
3500 { 3500 {
3501 /* 3501 /*
3502 * We can't deal with indirect functi 3502 * We can't deal with indirect function calls at present;
3503 * assume they're instrumented. 3503 * assume they're instrumented.
3504 */ 3504 */
3505 if (!func) { 3505 if (!func) {
3506 if (file->pv_ops) 3506 if (file->pv_ops)
3507 return pv_call_dest(f 3507 return pv_call_dest(file, insn);
3508 3508
3509 return false; 3509 return false;
3510 } 3510 }
3511 3511
3512 /* 3512 /*
3513 * If the symbol is from a noinstr se 3513 * If the symbol is from a noinstr section; we good.
3514 */ 3514 */
3515 if (func->sec->noinstr) 3515 if (func->sec->noinstr)
3516 return true; 3516 return true;
3517 3517
3518 /* 3518 /*
3519 * If the symbol is a static_call tra 3519 * If the symbol is a static_call trampoline, we can't tell.
3520 */ 3520 */
3521 if (func->static_call_tramp) 3521 if (func->static_call_tramp)
3522 return true; 3522 return true;
3523 3523
3524 /* 3524 /*
3525 * The __ubsan_handle_*() calls are l 3525 * The __ubsan_handle_*() calls are like WARN(), they only happen when
3526 * something 'BAD' happened. At the r 3526 * something 'BAD' happened. At the risk of taking the machine down,
3527 * let them proceed to get the messag 3527 * let them proceed to get the message out.
3528 */ 3528 */
3529 if (!strncmp(func->name, "__ubsan_han 3529 if (!strncmp(func->name, "__ubsan_handle_", 15))
3530 return true; 3530 return true;
3531 3531
3532 return false; 3532 return false;
3533 } 3533 }
3534 3534
3535 static int validate_call(struct objtool_file 3535 static int validate_call(struct objtool_file *file,
3536 struct instruction * 3536 struct instruction *insn,
3537 struct insn_state *s 3537 struct insn_state *state)
3538 { 3538 {
3539 if (state->noinstr && state->instr <= 3539 if (state->noinstr && state->instr <= 0 &&
3540 !noinstr_call_dest(file, insn, in 3540 !noinstr_call_dest(file, insn, insn_call_dest(insn))) {
3541 WARN_INSN(insn, "call to %s() 3541 WARN_INSN(insn, "call to %s() leaves .noinstr.text section", call_dest_name(insn));
3542 return 1; 3542 return 1;
3543 } 3543 }
3544 3544
3545 if (state->uaccess && !func_uaccess_s 3545 if (state->uaccess && !func_uaccess_safe(insn_call_dest(insn))) {
3546 WARN_INSN(insn, "call to %s() 3546 WARN_INSN(insn, "call to %s() with UACCESS enabled", call_dest_name(insn));
3547 return 1; 3547 return 1;
3548 } 3548 }
3549 3549
3550 if (state->df) { 3550 if (state->df) {
3551 WARN_INSN(insn, "call to %s() 3551 WARN_INSN(insn, "call to %s() with DF set", call_dest_name(insn));
3552 return 1; 3552 return 1;
3553 } 3553 }
3554 3554
3555 return 0; 3555 return 0;
3556 } 3556 }
3557 3557
3558 static int validate_sibling_call(struct objto 3558 static int validate_sibling_call(struct objtool_file *file,
3559 struct instr 3559 struct instruction *insn,
3560 struct insn_ 3560 struct insn_state *state)
3561 { 3561 {
3562 if (insn_func(insn) && has_modified_s 3562 if (insn_func(insn) && has_modified_stack_frame(insn, state)) {
3563 WARN_INSN(insn, "sibling call 3563 WARN_INSN(insn, "sibling call from callable instruction with modified stack frame");
3564 return 1; 3564 return 1;
3565 } 3565 }
3566 3566
3567 return validate_call(file, insn, stat 3567 return validate_call(file, insn, state);
3568 } 3568 }
3569 3569
3570 static int validate_return(struct symbol *fun 3570 static int validate_return(struct symbol *func, struct instruction *insn, struct insn_state *state)
3571 { 3571 {
3572 if (state->noinstr && state->instr > 3572 if (state->noinstr && state->instr > 0) {
3573 WARN_INSN(insn, "return with 3573 WARN_INSN(insn, "return with instrumentation enabled");
3574 return 1; 3574 return 1;
3575 } 3575 }
3576 3576
3577 if (state->uaccess && !func_uaccess_s 3577 if (state->uaccess && !func_uaccess_safe(func)) {
3578 WARN_INSN(insn, "return with 3578 WARN_INSN(insn, "return with UACCESS enabled");
3579 return 1; 3579 return 1;
3580 } 3580 }
3581 3581
3582 if (!state->uaccess && func_uaccess_s 3582 if (!state->uaccess && func_uaccess_safe(func)) {
3583 WARN_INSN(insn, "return with 3583 WARN_INSN(insn, "return with UACCESS disabled from a UACCESS-safe function");
3584 return 1; 3584 return 1;
3585 } 3585 }
3586 3586
3587 if (state->df) { 3587 if (state->df) {
3588 WARN_INSN(insn, "return with 3588 WARN_INSN(insn, "return with DF set");
3589 return 1; 3589 return 1;
3590 } 3590 }
3591 3591
3592 if (func && has_modified_stack_frame( 3592 if (func && has_modified_stack_frame(insn, state)) {
3593 WARN_INSN(insn, "return with 3593 WARN_INSN(insn, "return with modified stack frame");
3594 return 1; 3594 return 1;
3595 } 3595 }
3596 3596
3597 if (state->cfi.bp_scratch) { 3597 if (state->cfi.bp_scratch) {
3598 WARN_INSN(insn, "BP used as a 3598 WARN_INSN(insn, "BP used as a scratch register");
3599 return 1; 3599 return 1;
3600 } 3600 }
3601 3601
3602 return 0; 3602 return 0;
3603 } 3603 }
3604 3604
3605 static struct instruction *next_insn_to_valid 3605 static struct instruction *next_insn_to_validate(struct objtool_file *file,
3606 3606 struct instruction *insn)
3607 { 3607 {
3608 struct alt_group *alt_group = insn->a 3608 struct alt_group *alt_group = insn->alt_group;
3609 3609
3610 /* 3610 /*
3611 * Simulate the fact that alternative 3611 * Simulate the fact that alternatives are patched in-place. When the
3612 * end of a replacement alt_group is 3612 * end of a replacement alt_group is reached, redirect objtool flow to
3613 * the end of the original alt_group. 3613 * the end of the original alt_group.
3614 * 3614 *
3615 * insn->alts->insn -> alt_group->fir 3615 * insn->alts->insn -> alt_group->first_insn
3616 * ... 3616 * ...
3617 * alt_group->las 3617 * alt_group->last_insn
3618 * [alt_group->no 3618 * [alt_group->nop] -> next(orig_group->last_insn)
3619 */ 3619 */
3620 if (alt_group) { 3620 if (alt_group) {
3621 if (alt_group->nop) { 3621 if (alt_group->nop) {
3622 /* ->nop implies ->or 3622 /* ->nop implies ->orig_group */
3623 if (insn == alt_group 3623 if (insn == alt_group->last_insn)
3624 return alt_gr 3624 return alt_group->nop;
3625 if (insn == alt_group 3625 if (insn == alt_group->nop)
3626 goto next_ori 3626 goto next_orig;
3627 } 3627 }
3628 if (insn == alt_group->last_i 3628 if (insn == alt_group->last_insn && alt_group->orig_group)
3629 goto next_orig; 3629 goto next_orig;
3630 } 3630 }
3631 3631
3632 return next_insn_same_sec(file, insn) 3632 return next_insn_same_sec(file, insn);
3633 3633
3634 next_orig: 3634 next_orig:
3635 return next_insn_same_sec(file, alt_g 3635 return next_insn_same_sec(file, alt_group->orig_group->last_insn);
3636 } 3636 }
3637 3637
3638 /* 3638 /*
3639 * Follow the branch starting at the given in 3639 * Follow the branch starting at the given instruction, and recursively follow
3640 * any other branches (jumps). Meanwhile, tr 3640 * any other branches (jumps). Meanwhile, track the frame pointer state at
3641 * each instruction and validate all the rule 3641 * each instruction and validate all the rules described in
3642 * tools/objtool/Documentation/objtool.txt. 3642 * tools/objtool/Documentation/objtool.txt.
3643 */ 3643 */
3644 static int validate_branch(struct objtool_fil 3644 static int validate_branch(struct objtool_file *file, struct symbol *func,
3645 struct instruction 3645 struct instruction *insn, struct insn_state state)
3646 { 3646 {
3647 struct alternative *alt; 3647 struct alternative *alt;
3648 struct instruction *next_insn, *prev_ 3648 struct instruction *next_insn, *prev_insn = NULL;
3649 struct section *sec; 3649 struct section *sec;
3650 u8 visited; 3650 u8 visited;
3651 int ret; 3651 int ret;
3652 3652
3653 sec = insn->sec; 3653 sec = insn->sec;
3654 3654
3655 while (1) { 3655 while (1) {
3656 next_insn = next_insn_to_vali 3656 next_insn = next_insn_to_validate(file, insn);
3657 3657
3658 if (func && insn_func(insn) & 3658 if (func && insn_func(insn) && func != insn_func(insn)->pfunc) {
3659 /* Ignore KCFI type p 3659 /* Ignore KCFI type preambles, which always fall through */
3660 if (!strncmp(func->na 3660 if (!strncmp(func->name, "__cfi_", 6) ||
3661 !strncmp(func->na 3661 !strncmp(func->name, "__pfx_", 6))
3662 return 0; 3662 return 0;
3663 3663
3664 WARN("%s() falls thro 3664 WARN("%s() falls through to next function %s()",
3665 func->name, insn 3665 func->name, insn_func(insn)->name);
3666 return 1; 3666 return 1;
3667 } 3667 }
3668 3668
3669 if (func && insn->ignore) { 3669 if (func && insn->ignore) {
3670 WARN_INSN(insn, "BUG: 3670 WARN_INSN(insn, "BUG: why am I validating an ignored function?");
3671 return 1; 3671 return 1;
3672 } 3672 }
3673 3673
3674 visited = VISITED_BRANCH << s 3674 visited = VISITED_BRANCH << state.uaccess;
3675 if (insn->visited & VISITED_B 3675 if (insn->visited & VISITED_BRANCH_MASK) {
3676 if (!insn->hint && !i 3676 if (!insn->hint && !insn_cfi_match(insn, &state.cfi))
3677 return 1; 3677 return 1;
3678 3678
3679 if (insn->visited & v 3679 if (insn->visited & visited)
3680 return 0; 3680 return 0;
3681 } else { 3681 } else {
3682 nr_insns_visited++; 3682 nr_insns_visited++;
3683 } 3683 }
3684 3684
3685 if (state.noinstr) 3685 if (state.noinstr)
3686 state.instr += insn-> 3686 state.instr += insn->instr;
3687 3687
3688 if (insn->hint) { 3688 if (insn->hint) {
3689 if (insn->restore) { 3689 if (insn->restore) {
3690 struct instru 3690 struct instruction *save_insn, *i;
3691 3691
3692 i = insn; 3692 i = insn;
3693 save_insn = N 3693 save_insn = NULL;
3694 3694
3695 sym_for_each_ 3695 sym_for_each_insn_continue_reverse(file, func, i) {
3696 if (i 3696 if (i->save) {
3697 3697 save_insn = i;
3698 3698 break;
3699 } 3699 }
3700 } 3700 }
3701 3701
3702 if (!save_ins 3702 if (!save_insn) {
3703 WARN_ 3703 WARN_INSN(insn, "no corresponding CFI save for CFI restore");
3704 retur 3704 return 1;
3705 } 3705 }
3706 3706
3707 if (!save_ins 3707 if (!save_insn->visited) {
3708 /* 3708 /*
3709 * If 3709 * If the restore hint insn is at the
3710 * be 3710 * beginning of a basic block and was
3711 * br 3711 * branched to from elsewhere, and the
3712 * sa 3712 * save insn hasn't been visited yet,
3713 * de 3713 * defer following this branch for now.
3714 * It 3714 * It will be seen later via the
3715 * st 3715 * straight-line path.
3716 */ 3716 */
3717 if (! 3717 if (!prev_insn)
3718 3718 return 0;
3719 3719
3720 WARN_ 3720 WARN_INSN(insn, "objtool isn't smart enough to handle this CFI save/restore combo");
3721 retur 3721 return 1;
3722 } 3722 }
3723 3723
3724 insn->cfi = s 3724 insn->cfi = save_insn->cfi;
3725 nr_cfi_reused 3725 nr_cfi_reused++;
3726 } 3726 }
3727 3727
3728 state.cfi = *insn->cf 3728 state.cfi = *insn->cfi;
3729 } else { 3729 } else {
3730 /* XXX track if we ac 3730 /* XXX track if we actually changed state.cfi */
3731 3731
3732 if (prev_insn && !cfi 3732 if (prev_insn && !cficmp(prev_insn->cfi, &state.cfi)) {
3733 insn->cfi = p 3733 insn->cfi = prev_insn->cfi;
3734 nr_cfi_reused 3734 nr_cfi_reused++;
3735 } else { 3735 } else {
3736 insn->cfi = c 3736 insn->cfi = cfi_hash_find_or_add(&state.cfi);
3737 } 3737 }
3738 } 3738 }
3739 3739
3740 insn->visited |= visited; 3740 insn->visited |= visited;
3741 3741
3742 if (propagate_alt_cfi(file, i 3742 if (propagate_alt_cfi(file, insn))
3743 return 1; 3743 return 1;
3744 3744
3745 if (!insn->ignore_alts && ins 3745 if (!insn->ignore_alts && insn->alts) {
3746 bool skip_orig = fals 3746 bool skip_orig = false;
3747 3747
3748 for (alt = insn->alts 3748 for (alt = insn->alts; alt; alt = alt->next) {
3749 if (alt->skip 3749 if (alt->skip_orig)
3750 skip_ 3750 skip_orig = true;
3751 3751
3752 ret = validat 3752 ret = validate_branch(file, func, alt->insn, state);
3753 if (ret) { 3753 if (ret) {
3754 BT_IN 3754 BT_INSN(insn, "(alt)");
3755 retur 3755 return ret;
3756 } 3756 }
3757 } 3757 }
3758 3758
3759 if (skip_orig) 3759 if (skip_orig)
3760 return 0; 3760 return 0;
3761 } 3761 }
3762 3762
3763 if (handle_insn_ops(insn, nex 3763 if (handle_insn_ops(insn, next_insn, &state))
3764 return 1; 3764 return 1;
3765 3765
3766 switch (insn->type) { 3766 switch (insn->type) {
3767 3767
3768 case INSN_RETURN: 3768 case INSN_RETURN:
3769 return validate_retur 3769 return validate_return(func, insn, &state);
3770 3770
3771 case INSN_CALL: 3771 case INSN_CALL:
3772 case INSN_CALL_DYNAMIC: 3772 case INSN_CALL_DYNAMIC:
3773 ret = validate_call(f 3773 ret = validate_call(file, insn, &state);
3774 if (ret) 3774 if (ret)
3775 return ret; 3775 return ret;
3776 3776
3777 if (opts.stackval && 3777 if (opts.stackval && func && !is_special_call(insn) &&
3778 !has_valid_stack_ 3778 !has_valid_stack_frame(&state)) {
3779 WARN_INSN(ins 3779 WARN_INSN(insn, "call without frame pointer save/setup");
3780 return 1; 3780 return 1;
3781 } 3781 }
3782 3782
3783 if (insn->dead_end) 3783 if (insn->dead_end)
3784 return 0; 3784 return 0;
3785 3785
3786 break; 3786 break;
3787 3787
3788 case INSN_JUMP_CONDITIONAL: 3788 case INSN_JUMP_CONDITIONAL:
3789 case INSN_JUMP_UNCONDITIONAL: 3789 case INSN_JUMP_UNCONDITIONAL:
3790 if (is_sibling_call(i 3790 if (is_sibling_call(insn)) {
3791 ret = validat 3791 ret = validate_sibling_call(file, insn, &state);
3792 if (ret) 3792 if (ret)
3793 retur 3793 return ret;
3794 3794
3795 } else if (insn->jump 3795 } else if (insn->jump_dest) {
3796 ret = validat 3796 ret = validate_branch(file, func,
3797 3797 insn->jump_dest, state);
3798 if (ret) { 3798 if (ret) {
3799 BT_IN 3799 BT_INSN(insn, "(branch)");
3800 retur 3800 return ret;
3801 } 3801 }
3802 } 3802 }
3803 3803
3804 if (insn->type == INS 3804 if (insn->type == INSN_JUMP_UNCONDITIONAL)
3805 return 0; 3805 return 0;
3806 3806
3807 break; 3807 break;
3808 3808
3809 case INSN_JUMP_DYNAMIC: 3809 case INSN_JUMP_DYNAMIC:
3810 case INSN_JUMP_DYNAMIC_CONDIT 3810 case INSN_JUMP_DYNAMIC_CONDITIONAL:
3811 if (is_sibling_call(i 3811 if (is_sibling_call(insn)) {
3812 ret = validat 3812 ret = validate_sibling_call(file, insn, &state);
3813 if (ret) 3813 if (ret)
3814 retur 3814 return ret;
3815 } 3815 }
3816 3816
3817 if (insn->type == INS 3817 if (insn->type == INSN_JUMP_DYNAMIC)
3818 return 0; 3818 return 0;
3819 3819
3820 break; 3820 break;
3821 3821
3822 case INSN_CONTEXT_SWITCH: 3822 case INSN_CONTEXT_SWITCH:
3823 if (func && (!next_in 3823 if (func && (!next_insn || !next_insn->hint)) {
3824 WARN_INSN(ins 3824 WARN_INSN(insn, "unsupported instruction in callable function");
3825 return 1; 3825 return 1;
3826 } 3826 }
3827 return 0; 3827 return 0;
3828 3828
3829 case INSN_STAC: 3829 case INSN_STAC:
3830 if (state.uaccess) { 3830 if (state.uaccess) {
3831 WARN_INSN(ins 3831 WARN_INSN(insn, "recursive UACCESS enable");
3832 return 1; 3832 return 1;
3833 } 3833 }
3834 3834
3835 state.uaccess = true; 3835 state.uaccess = true;
3836 break; 3836 break;
3837 3837
3838 case INSN_CLAC: 3838 case INSN_CLAC:
3839 if (!state.uaccess && 3839 if (!state.uaccess && func) {
3840 WARN_INSN(ins 3840 WARN_INSN(insn, "redundant UACCESS disable");
3841 return 1; 3841 return 1;
3842 } 3842 }
3843 3843
3844 if (func_uaccess_safe 3844 if (func_uaccess_safe(func) && !state.uaccess_stack) {
3845 WARN_INSN(ins 3845 WARN_INSN(insn, "UACCESS-safe disables UACCESS");
3846 return 1; 3846 return 1;
3847 } 3847 }
3848 3848
3849 state.uaccess = false 3849 state.uaccess = false;
3850 break; 3850 break;
3851 3851
3852 case INSN_STD: 3852 case INSN_STD:
3853 if (state.df) { 3853 if (state.df) {
3854 WARN_INSN(ins 3854 WARN_INSN(insn, "recursive STD");
3855 return 1; 3855 return 1;
3856 } 3856 }
3857 3857
3858 state.df = true; 3858 state.df = true;
3859 break; 3859 break;
3860 3860
3861 case INSN_CLD: 3861 case INSN_CLD:
3862 if (!state.df && func 3862 if (!state.df && func) {
3863 WARN_INSN(ins 3863 WARN_INSN(insn, "redundant CLD");
3864 return 1; 3864 return 1;
3865 } 3865 }
3866 3866
3867 state.df = false; 3867 state.df = false;
3868 break; 3868 break;
3869 3869
3870 default: 3870 default:
3871 break; 3871 break;
3872 } 3872 }
3873 3873
3874 if (insn->dead_end) 3874 if (insn->dead_end)
3875 return 0; 3875 return 0;
3876 3876
3877 if (!next_insn) { 3877 if (!next_insn) {
3878 if (state.cfi.cfa.bas 3878 if (state.cfi.cfa.base == CFI_UNDEFINED)
3879 return 0; 3879 return 0;
3880 WARN("%s: unexpected 3880 WARN("%s: unexpected end of section", sec->name);
3881 return 1; 3881 return 1;
3882 } 3882 }
3883 3883
3884 prev_insn = insn; 3884 prev_insn = insn;
3885 insn = next_insn; 3885 insn = next_insn;
3886 } 3886 }
3887 3887
3888 return 0; 3888 return 0;
3889 } 3889 }
3890 3890
3891 static int validate_unwind_hint(struct objtoo 3891 static int validate_unwind_hint(struct objtool_file *file,
3892 struct inst 3892 struct instruction *insn,
3893 struct insn 3893 struct insn_state *state)
3894 { 3894 {
3895 if (insn->hint && !insn->visited && ! 3895 if (insn->hint && !insn->visited && !insn->ignore) {
3896 int ret = validate_branch(fil 3896 int ret = validate_branch(file, insn_func(insn), insn, *state);
3897 if (ret) 3897 if (ret)
3898 BT_INSN(insn, "<=== ( 3898 BT_INSN(insn, "<=== (hint)");
3899 return ret; 3899 return ret;
3900 } 3900 }
3901 3901
3902 return 0; 3902 return 0;
3903 } 3903 }
3904 3904
3905 static int validate_unwind_hints(struct objto 3905 static int validate_unwind_hints(struct objtool_file *file, struct section *sec)
3906 { 3906 {
3907 struct instruction *insn; 3907 struct instruction *insn;
3908 struct insn_state state; 3908 struct insn_state state;
3909 int warnings = 0; 3909 int warnings = 0;
3910 3910
3911 if (!file->hints) 3911 if (!file->hints)
3912 return 0; 3912 return 0;
3913 3913
3914 init_insn_state(file, &state, sec); 3914 init_insn_state(file, &state, sec);
3915 3915
3916 if (sec) { 3916 if (sec) {
3917 sec_for_each_insn(file, sec, 3917 sec_for_each_insn(file, sec, insn)
3918 warnings += validate_ 3918 warnings += validate_unwind_hint(file, insn, &state);
3919 } else { 3919 } else {
3920 for_each_insn(file, insn) 3920 for_each_insn(file, insn)
3921 warnings += validate_ 3921 warnings += validate_unwind_hint(file, insn, &state);
3922 } 3922 }
3923 3923
3924 return warnings; 3924 return warnings;
3925 } 3925 }
3926 3926
3927 /* 3927 /*
3928 * Validate rethunk entry constraint: must un 3928 * Validate rethunk entry constraint: must untrain RET before the first RET.
3929 * 3929 *
3930 * Follow every branch (intra-function) and e 3930 * Follow every branch (intra-function) and ensure VALIDATE_UNRET_END comes
3931 * before an actual RET instruction. 3931 * before an actual RET instruction.
3932 */ 3932 */
3933 static int validate_unret(struct objtool_file 3933 static int validate_unret(struct objtool_file *file, struct instruction *insn)
3934 { 3934 {
3935 struct instruction *next, *dest; 3935 struct instruction *next, *dest;
3936 int ret; 3936 int ret;
3937 3937
3938 for (;;) { 3938 for (;;) {
3939 next = next_insn_to_validate( 3939 next = next_insn_to_validate(file, insn);
3940 3940
3941 if (insn->visited & VISITED_U 3941 if (insn->visited & VISITED_UNRET)
3942 return 0; 3942 return 0;
3943 3943
3944 insn->visited |= VISITED_UNRE 3944 insn->visited |= VISITED_UNRET;
3945 3945
3946 if (!insn->ignore_alts && ins 3946 if (!insn->ignore_alts && insn->alts) {
3947 struct alternative *a 3947 struct alternative *alt;
3948 bool skip_orig = fals 3948 bool skip_orig = false;
3949 3949
3950 for (alt = insn->alts 3950 for (alt = insn->alts; alt; alt = alt->next) {
3951 if (alt->skip 3951 if (alt->skip_orig)
3952 skip_ 3952 skip_orig = true;
3953 3953
3954 ret = validat 3954 ret = validate_unret(file, alt->insn);
3955 if (ret) { 3955 if (ret) {
3956 BT_IN 3956 BT_INSN(insn, "(alt)");
3957 retur 3957 return ret;
3958 } 3958 }
3959 } 3959 }
3960 3960
3961 if (skip_orig) 3961 if (skip_orig)
3962 return 0; 3962 return 0;
3963 } 3963 }
3964 3964
3965 switch (insn->type) { 3965 switch (insn->type) {
3966 3966
3967 case INSN_CALL_DYNAMIC: 3967 case INSN_CALL_DYNAMIC:
3968 case INSN_JUMP_DYNAMIC: 3968 case INSN_JUMP_DYNAMIC:
3969 case INSN_JUMP_DYNAMIC_CONDIT 3969 case INSN_JUMP_DYNAMIC_CONDITIONAL:
3970 WARN_INSN(insn, "earl 3970 WARN_INSN(insn, "early indirect call");
3971 return 1; 3971 return 1;
3972 3972
3973 case INSN_JUMP_UNCONDITIONAL: 3973 case INSN_JUMP_UNCONDITIONAL:
3974 case INSN_JUMP_CONDITIONAL: 3974 case INSN_JUMP_CONDITIONAL:
3975 if (!is_sibling_call( 3975 if (!is_sibling_call(insn)) {
3976 if (!insn->ju 3976 if (!insn->jump_dest) {
3977 WARN_ 3977 WARN_INSN(insn, "unresolved jump target after linking?!?");
3978 retur 3978 return -1;
3979 } 3979 }
3980 ret = validat 3980 ret = validate_unret(file, insn->jump_dest);
3981 if (ret) { 3981 if (ret) {
3982 BT_IN 3982 BT_INSN(insn, "(branch%s)",
3983 3983 insn->type == INSN_JUMP_CONDITIONAL ? "-cond" : "");
3984 retur 3984 return ret;
3985 } 3985 }
3986 3986
3987 if (insn->typ 3987 if (insn->type == INSN_JUMP_UNCONDITIONAL)
3988 retur 3988 return 0;
3989 3989
3990 break; 3990 break;
3991 } 3991 }
3992 3992
3993 /* fallthrough */ 3993 /* fallthrough */
3994 case INSN_CALL: 3994 case INSN_CALL:
3995 dest = find_insn(file 3995 dest = find_insn(file, insn_call_dest(insn)->sec,
3996 insn 3996 insn_call_dest(insn)->offset);
3997 if (!dest) { 3997 if (!dest) {
3998 WARN("Unresol 3998 WARN("Unresolved function after linking!?: %s",
3999 insn_cal 3999 insn_call_dest(insn)->name);
4000 return -1; 4000 return -1;
4001 } 4001 }
4002 4002
4003 ret = validate_unret( 4003 ret = validate_unret(file, dest);
4004 if (ret) { 4004 if (ret) {
4005 BT_INSN(insn, 4005 BT_INSN(insn, "(call)");
4006 return ret; 4006 return ret;
4007 } 4007 }
4008 /* 4008 /*
4009 * If a call returns 4009 * If a call returns without error, it must have seen UNTRAIN_RET.
4010 * Therefore any non- 4010 * Therefore any non-error return is a success.
4011 */ 4011 */
4012 return 0; 4012 return 0;
4013 4013
4014 case INSN_RETURN: 4014 case INSN_RETURN:
4015 WARN_INSN(insn, "RET 4015 WARN_INSN(insn, "RET before UNTRAIN");
4016 return 1; 4016 return 1;
4017 4017
4018 case INSN_NOP: 4018 case INSN_NOP:
4019 if (insn->retpoline_s 4019 if (insn->retpoline_safe)
4020 return 0; 4020 return 0;
4021 break; 4021 break;
4022 4022
4023 default: 4023 default:
4024 break; 4024 break;
4025 } 4025 }
4026 4026
4027 if (!next) { 4027 if (!next) {
4028 WARN_INSN(insn, "teh 4028 WARN_INSN(insn, "teh end!");
4029 return -1; 4029 return -1;
4030 } 4030 }
4031 insn = next; 4031 insn = next;
4032 } 4032 }
4033 4033
4034 return 0; 4034 return 0;
4035 } 4035 }
4036 4036
4037 /* 4037 /*
4038 * Validate that all branches starting at VAL 4038 * Validate that all branches starting at VALIDATE_UNRET_BEGIN encounter
4039 * VALIDATE_UNRET_END before RET. 4039 * VALIDATE_UNRET_END before RET.
4040 */ 4040 */
4041 static int validate_unrets(struct objtool_fil 4041 static int validate_unrets(struct objtool_file *file)
4042 { 4042 {
4043 struct instruction *insn; 4043 struct instruction *insn;
4044 int ret, warnings = 0; 4044 int ret, warnings = 0;
4045 4045
4046 for_each_insn(file, insn) { 4046 for_each_insn(file, insn) {
4047 if (!insn->unret) 4047 if (!insn->unret)
4048 continue; 4048 continue;
4049 4049
4050 ret = validate_unret(file, in 4050 ret = validate_unret(file, insn);
4051 if (ret < 0) { 4051 if (ret < 0) {
4052 WARN_INSN(insn, "Fail 4052 WARN_INSN(insn, "Failed UNRET validation");
4053 return ret; 4053 return ret;
4054 } 4054 }
4055 warnings += ret; 4055 warnings += ret;
4056 } 4056 }
4057 4057
4058 return warnings; 4058 return warnings;
4059 } 4059 }
4060 4060
4061 static int validate_retpoline(struct objtool_ 4061 static int validate_retpoline(struct objtool_file *file)
4062 { 4062 {
4063 struct instruction *insn; 4063 struct instruction *insn;
4064 int warnings = 0; 4064 int warnings = 0;
4065 4065
4066 for_each_insn(file, insn) { 4066 for_each_insn(file, insn) {
4067 if (insn->type != INSN_JUMP_D 4067 if (insn->type != INSN_JUMP_DYNAMIC &&
4068 insn->type != INSN_CALL_D 4068 insn->type != INSN_CALL_DYNAMIC &&
4069 insn->type != INSN_RETURN 4069 insn->type != INSN_RETURN)
4070 continue; 4070 continue;
4071 4071
4072 if (insn->retpoline_safe) 4072 if (insn->retpoline_safe)
4073 continue; 4073 continue;
4074 4074
4075 if (insn->sec->init) 4075 if (insn->sec->init)
4076 continue; 4076 continue;
4077 4077
4078 if (insn->type == INSN_RETURN 4078 if (insn->type == INSN_RETURN) {
4079 if (opts.rethunk) { 4079 if (opts.rethunk) {
4080 WARN_INSN(ins 4080 WARN_INSN(insn, "'naked' return found in MITIGATION_RETHUNK build");
4081 } else 4081 } else
4082 continue; 4082 continue;
4083 } else { 4083 } else {
4084 WARN_INSN(insn, "indi 4084 WARN_INSN(insn, "indirect %s found in MITIGATION_RETPOLINE build",
4085 insn->type 4085 insn->type == INSN_JUMP_DYNAMIC ? "jump" : "call");
4086 } 4086 }
4087 4087
4088 warnings++; 4088 warnings++;
4089 } 4089 }
4090 4090
4091 return warnings; 4091 return warnings;
4092 } 4092 }
4093 4093
4094 static bool is_kasan_insn(struct instruction 4094 static bool is_kasan_insn(struct instruction *insn)
4095 { 4095 {
4096 return (insn->type == INSN_CALL && 4096 return (insn->type == INSN_CALL &&
4097 !strcmp(insn_call_dest(insn)- 4097 !strcmp(insn_call_dest(insn)->name, "__asan_handle_no_return"));
4098 } 4098 }
4099 4099
4100 static bool is_ubsan_insn(struct instruction 4100 static bool is_ubsan_insn(struct instruction *insn)
4101 { 4101 {
4102 return (insn->type == INSN_CALL && 4102 return (insn->type == INSN_CALL &&
4103 !strcmp(insn_call_dest(insn)- 4103 !strcmp(insn_call_dest(insn)->name,
4104 "__ubsan_handle_built 4104 "__ubsan_handle_builtin_unreachable"));
4105 } 4105 }
4106 4106
4107 static bool ignore_unreachable_insn(struct ob 4107 static bool ignore_unreachable_insn(struct objtool_file *file, struct instruction *insn)
4108 { 4108 {
4109 int i; 4109 int i;
4110 struct instruction *prev_insn; 4110 struct instruction *prev_insn;
4111 4111
4112 if (insn->ignore || insn->type == INS 4112 if (insn->ignore || insn->type == INSN_NOP || insn->type == INSN_TRAP)
4113 return true; 4113 return true;
4114 4114
4115 /* 4115 /*
4116 * Ignore alternative replacement ins 4116 * Ignore alternative replacement instructions. This can happen
4117 * when a whitelisted function uses o 4117 * when a whitelisted function uses one of the ALTERNATIVE macros.
4118 */ 4118 */
4119 if (!strcmp(insn->sec->name, ".altins 4119 if (!strcmp(insn->sec->name, ".altinstr_replacement") ||
4120 !strcmp(insn->sec->name, ".altins 4120 !strcmp(insn->sec->name, ".altinstr_aux"))
4121 return true; 4121 return true;
4122 4122
4123 /* 4123 /*
4124 * Whole archive runs might encounter 4124 * Whole archive runs might encounter dead code from weak symbols.
4125 * This is where the linker will have 4125 * This is where the linker will have dropped the weak symbol in
4126 * favour of a regular symbol, but le 4126 * favour of a regular symbol, but leaves the code in place.
4127 * 4127 *
4128 * In this case we'll find a piece of 4128 * In this case we'll find a piece of code (whole function) that is not
4129 * covered by a !section symbol. Igno 4129 * covered by a !section symbol. Ignore them.
4130 */ 4130 */
4131 if (opts.link && !insn_func(insn)) { 4131 if (opts.link && !insn_func(insn)) {
4132 int size = find_symbol_hole_c 4132 int size = find_symbol_hole_containing(insn->sec, insn->offset);
4133 unsigned long end = insn->off 4133 unsigned long end = insn->offset + size;
4134 4134
4135 if (!size) /* not a hole */ 4135 if (!size) /* not a hole */
4136 return false; 4136 return false;
4137 4137
4138 if (size < 0) /* hole until t 4138 if (size < 0) /* hole until the end */
4139 return true; 4139 return true;
4140 4140
4141 sec_for_each_insn_continue(fi 4141 sec_for_each_insn_continue(file, insn) {
4142 /* 4142 /*
4143 * If we reach a visi 4143 * If we reach a visited instruction at or before the
4144 * end of the hole, i 4144 * end of the hole, ignore the unreachable.
4145 */ 4145 */
4146 if (insn->visited) 4146 if (insn->visited)
4147 return true; 4147 return true;
4148 4148
4149 if (insn->offset >= e 4149 if (insn->offset >= end)
4150 break; 4150 break;
4151 4151
4152 /* 4152 /*
4153 * If this hole jumps 4153 * If this hole jumps to a .cold function, mark it ignore too.
4154 */ 4154 */
4155 if (insn->jump_dest & 4155 if (insn->jump_dest && insn_func(insn->jump_dest) &&
4156 strstr(insn_func( 4156 strstr(insn_func(insn->jump_dest)->name, ".cold")) {
4157 struct instru 4157 struct instruction *dest = insn->jump_dest;
4158 func_for_each 4158 func_for_each_insn(file, insn_func(dest), dest)
4159 dest- 4159 dest->ignore = true;
4160 } 4160 }
4161 } 4161 }
4162 4162
4163 return false; 4163 return false;
4164 } 4164 }
4165 4165
4166 if (!insn_func(insn)) 4166 if (!insn_func(insn))
4167 return false; 4167 return false;
4168 4168
4169 if (insn_func(insn)->static_call_tram 4169 if (insn_func(insn)->static_call_tramp)
4170 return true; 4170 return true;
4171 4171
4172 /* 4172 /*
4173 * CONFIG_UBSAN_TRAP inserts a UD2 wh 4173 * CONFIG_UBSAN_TRAP inserts a UD2 when it sees
4174 * __builtin_unreachable(). The BUG( 4174 * __builtin_unreachable(). The BUG() macro has an unreachable() after
4175 * the UD2, which causes GCC's undefi 4175 * the UD2, which causes GCC's undefined trap logic to emit another UD2
4176 * (or occasionally a JMP to UD2). 4176 * (or occasionally a JMP to UD2).
4177 * 4177 *
4178 * It may also insert a UD2 after cal 4178 * It may also insert a UD2 after calling a __noreturn function.
4179 */ 4179 */
4180 prev_insn = prev_insn_same_sec(file, 4180 prev_insn = prev_insn_same_sec(file, insn);
4181 if (prev_insn->dead_end && 4181 if (prev_insn->dead_end &&
4182 (insn->type == INSN_BUG || 4182 (insn->type == INSN_BUG ||
4183 (insn->type == INSN_JUMP_UNCONDI 4183 (insn->type == INSN_JUMP_UNCONDITIONAL &&
4184 insn->jump_dest && insn->jump_d 4184 insn->jump_dest && insn->jump_dest->type == INSN_BUG)))
4185 return true; 4185 return true;
4186 4186
4187 /* 4187 /*
4188 * Check if this (or a subsequent) in 4188 * Check if this (or a subsequent) instruction is related to
4189 * CONFIG_UBSAN or CONFIG_KASAN. 4189 * CONFIG_UBSAN or CONFIG_KASAN.
4190 * 4190 *
4191 * End the search at 5 instructions t 4191 * End the search at 5 instructions to avoid going into the weeds.
4192 */ 4192 */
4193 for (i = 0; i < 5; i++) { 4193 for (i = 0; i < 5; i++) {
4194 4194
4195 if (is_kasan_insn(insn) || is 4195 if (is_kasan_insn(insn) || is_ubsan_insn(insn))
4196 return true; 4196 return true;
4197 4197
4198 if (insn->type == INSN_JUMP_U 4198 if (insn->type == INSN_JUMP_UNCONDITIONAL) {
4199 if (insn->jump_dest & 4199 if (insn->jump_dest &&
4200 insn_func(insn->j 4200 insn_func(insn->jump_dest) == insn_func(insn)) {
4201 insn = insn-> 4201 insn = insn->jump_dest;
4202 continue; 4202 continue;
4203 } 4203 }
4204 4204
4205 break; 4205 break;
4206 } 4206 }
4207 4207
4208 if (insn->offset + insn->len 4208 if (insn->offset + insn->len >= insn_func(insn)->offset + insn_func(insn)->len)
4209 break; 4209 break;
4210 4210
4211 insn = next_insn_same_sec(fil 4211 insn = next_insn_same_sec(file, insn);
4212 } 4212 }
4213 4213
4214 return false; 4214 return false;
4215 } 4215 }
4216 4216
4217 static int add_prefix_symbol(struct objtool_f 4217 static int add_prefix_symbol(struct objtool_file *file, struct symbol *func)
4218 { 4218 {
4219 struct instruction *insn, *prev; 4219 struct instruction *insn, *prev;
4220 struct cfi_state *cfi; 4220 struct cfi_state *cfi;
4221 4221
4222 insn = find_insn(file, func->sec, fun 4222 insn = find_insn(file, func->sec, func->offset);
4223 if (!insn) 4223 if (!insn)
4224 return -1; 4224 return -1;
4225 4225
4226 for (prev = prev_insn_same_sec(file, 4226 for (prev = prev_insn_same_sec(file, insn);
4227 prev; 4227 prev;
4228 prev = prev_insn_same_sec(file, 4228 prev = prev_insn_same_sec(file, prev)) {
4229 u64 offset; 4229 u64 offset;
4230 4230
4231 if (prev->type != INSN_NOP) 4231 if (prev->type != INSN_NOP)
4232 return -1; 4232 return -1;
4233 4233
4234 offset = func->offset - prev- 4234 offset = func->offset - prev->offset;
4235 4235
4236 if (offset > opts.prefix) 4236 if (offset > opts.prefix)
4237 return -1; 4237 return -1;
4238 4238
4239 if (offset < opts.prefix) 4239 if (offset < opts.prefix)
4240 continue; 4240 continue;
4241 4241
4242 elf_create_prefix_symbol(file 4242 elf_create_prefix_symbol(file->elf, func, opts.prefix);
4243 break; 4243 break;
4244 } 4244 }
4245 4245
4246 if (!prev) 4246 if (!prev)
4247 return -1; 4247 return -1;
4248 4248
4249 if (!insn->cfi) { 4249 if (!insn->cfi) {
4250 /* 4250 /*
4251 * This can happen if stack v 4251 * This can happen if stack validation isn't enabled or the
4252 * function is annotated with 4252 * function is annotated with STACK_FRAME_NON_STANDARD.
4253 */ 4253 */
4254 return 0; 4254 return 0;
4255 } 4255 }
4256 4256
4257 /* Propagate insn->cfi to the prefix 4257 /* Propagate insn->cfi to the prefix code */
4258 cfi = cfi_hash_find_or_add(insn->cfi) 4258 cfi = cfi_hash_find_or_add(insn->cfi);
4259 for (; prev != insn; prev = next_insn 4259 for (; prev != insn; prev = next_insn_same_sec(file, prev))
4260 prev->cfi = cfi; 4260 prev->cfi = cfi;
4261 4261
4262 return 0; 4262 return 0;
4263 } 4263 }
4264 4264
4265 static int add_prefix_symbols(struct objtool_ 4265 static int add_prefix_symbols(struct objtool_file *file)
4266 { 4266 {
4267 struct section *sec; 4267 struct section *sec;
4268 struct symbol *func; 4268 struct symbol *func;
4269 4269
4270 for_each_sec(file, sec) { 4270 for_each_sec(file, sec) {
4271 if (!(sec->sh.sh_flags & SHF_ 4271 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
4272 continue; 4272 continue;
4273 4273
4274 sec_for_each_sym(sec, func) { 4274 sec_for_each_sym(sec, func) {
4275 if (func->type != STT 4275 if (func->type != STT_FUNC)
4276 continue; 4276 continue;
4277 4277
4278 add_prefix_symbol(fil 4278 add_prefix_symbol(file, func);
4279 } 4279 }
4280 } 4280 }
4281 4281
4282 return 0; 4282 return 0;
4283 } 4283 }
4284 4284
4285 static int validate_symbol(struct objtool_fil 4285 static int validate_symbol(struct objtool_file *file, struct section *sec,
4286 struct symbol *sym 4286 struct symbol *sym, struct insn_state *state)
4287 { 4287 {
4288 struct instruction *insn; 4288 struct instruction *insn;
4289 int ret; 4289 int ret;
4290 4290
4291 if (!sym->len) { 4291 if (!sym->len) {
4292 WARN("%s() is missing an ELF 4292 WARN("%s() is missing an ELF size annotation", sym->name);
4293 return 1; 4293 return 1;
4294 } 4294 }
4295 4295
4296 if (sym->pfunc != sym || sym->alias ! 4296 if (sym->pfunc != sym || sym->alias != sym)
4297 return 0; 4297 return 0;
4298 4298
4299 insn = find_insn(file, sec, sym->offs 4299 insn = find_insn(file, sec, sym->offset);
4300 if (!insn || insn->ignore || insn->vi 4300 if (!insn || insn->ignore || insn->visited)
4301 return 0; 4301 return 0;
4302 4302
4303 state->uaccess = sym->uaccess_safe; 4303 state->uaccess = sym->uaccess_safe;
4304 4304
4305 ret = validate_branch(file, insn_func 4305 ret = validate_branch(file, insn_func(insn), insn, *state);
4306 if (ret) 4306 if (ret)
4307 BT_INSN(insn, "<=== (sym)"); 4307 BT_INSN(insn, "<=== (sym)");
4308 return ret; 4308 return ret;
4309 } 4309 }
4310 4310
4311 static int validate_section(struct objtool_fi 4311 static int validate_section(struct objtool_file *file, struct section *sec)
4312 { 4312 {
4313 struct insn_state state; 4313 struct insn_state state;
4314 struct symbol *func; 4314 struct symbol *func;
4315 int warnings = 0; 4315 int warnings = 0;
4316 4316
4317 sec_for_each_sym(sec, func) { 4317 sec_for_each_sym(sec, func) {
4318 if (func->type != STT_FUNC) 4318 if (func->type != STT_FUNC)
4319 continue; 4319 continue;
4320 4320
4321 init_insn_state(file, &state, 4321 init_insn_state(file, &state, sec);
4322 set_func_state(&state.cfi); 4322 set_func_state(&state.cfi);
4323 4323
4324 warnings += validate_symbol(f 4324 warnings += validate_symbol(file, sec, func, &state);
4325 } 4325 }
4326 4326
4327 return warnings; 4327 return warnings;
4328 } 4328 }
4329 4329
4330 static int validate_noinstr_sections(struct o 4330 static int validate_noinstr_sections(struct objtool_file *file)
4331 { 4331 {
4332 struct section *sec; 4332 struct section *sec;
4333 int warnings = 0; 4333 int warnings = 0;
4334 4334
4335 sec = find_section_by_name(file->elf, 4335 sec = find_section_by_name(file->elf, ".noinstr.text");
4336 if (sec) { 4336 if (sec) {
4337 warnings += validate_section( 4337 warnings += validate_section(file, sec);
4338 warnings += validate_unwind_h 4338 warnings += validate_unwind_hints(file, sec);
4339 } 4339 }
4340 4340
4341 sec = find_section_by_name(file->elf, 4341 sec = find_section_by_name(file->elf, ".entry.text");
4342 if (sec) { 4342 if (sec) {
4343 warnings += validate_section( 4343 warnings += validate_section(file, sec);
4344 warnings += validate_unwind_h 4344 warnings += validate_unwind_hints(file, sec);
4345 } 4345 }
4346 4346
4347 sec = find_section_by_name(file->elf, 4347 sec = find_section_by_name(file->elf, ".cpuidle.text");
4348 if (sec) { 4348 if (sec) {
4349 warnings += validate_section( 4349 warnings += validate_section(file, sec);
4350 warnings += validate_unwind_h 4350 warnings += validate_unwind_hints(file, sec);
4351 } 4351 }
4352 4352
4353 return warnings; 4353 return warnings;
4354 } 4354 }
4355 4355
4356 static int validate_functions(struct objtool_ 4356 static int validate_functions(struct objtool_file *file)
4357 { 4357 {
4358 struct section *sec; 4358 struct section *sec;
4359 int warnings = 0; 4359 int warnings = 0;
4360 4360
4361 for_each_sec(file, sec) { 4361 for_each_sec(file, sec) {
4362 if (!(sec->sh.sh_flags & SHF_ 4362 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
4363 continue; 4363 continue;
4364 4364
4365 warnings += validate_section( 4365 warnings += validate_section(file, sec);
4366 } 4366 }
4367 4367
4368 return warnings; 4368 return warnings;
4369 } 4369 }
4370 4370
4371 static void mark_endbr_used(struct instructio 4371 static void mark_endbr_used(struct instruction *insn)
4372 { 4372 {
4373 if (!list_empty(&insn->call_node)) 4373 if (!list_empty(&insn->call_node))
4374 list_del_init(&insn->call_nod 4374 list_del_init(&insn->call_node);
4375 } 4375 }
4376 4376
4377 static bool noendbr_range(struct objtool_file 4377 static bool noendbr_range(struct objtool_file *file, struct instruction *insn)
4378 { 4378 {
4379 struct symbol *sym = find_symbol_cont 4379 struct symbol *sym = find_symbol_containing(insn->sec, insn->offset-1);
4380 struct instruction *first; 4380 struct instruction *first;
4381 4381
4382 if (!sym) 4382 if (!sym)
4383 return false; 4383 return false;
4384 4384
4385 first = find_insn(file, sym->sec, sym 4385 first = find_insn(file, sym->sec, sym->offset);
4386 if (!first) 4386 if (!first)
4387 return false; 4387 return false;
4388 4388
4389 if (first->type != INSN_ENDBR && !fir 4389 if (first->type != INSN_ENDBR && !first->noendbr)
4390 return false; 4390 return false;
4391 4391
4392 return insn->offset == sym->offset + 4392 return insn->offset == sym->offset + sym->len;
4393 } 4393 }
4394 4394
4395 static int validate_ibt_insn(struct objtool_f 4395 static int validate_ibt_insn(struct objtool_file *file, struct instruction *insn)
4396 { 4396 {
4397 struct instruction *dest; 4397 struct instruction *dest;
4398 struct reloc *reloc; 4398 struct reloc *reloc;
4399 unsigned long off; 4399 unsigned long off;
4400 int warnings = 0; 4400 int warnings = 0;
4401 4401
4402 /* 4402 /*
4403 * Looking for function pointer load 4403 * Looking for function pointer load relocations. Ignore
4404 * direct/indirect branches: 4404 * direct/indirect branches:
4405 */ 4405 */
4406 switch (insn->type) { 4406 switch (insn->type) {
4407 case INSN_CALL: 4407 case INSN_CALL:
4408 case INSN_CALL_DYNAMIC: 4408 case INSN_CALL_DYNAMIC:
4409 case INSN_JUMP_CONDITIONAL: 4409 case INSN_JUMP_CONDITIONAL:
4410 case INSN_JUMP_UNCONDITIONAL: 4410 case INSN_JUMP_UNCONDITIONAL:
4411 case INSN_JUMP_DYNAMIC: 4411 case INSN_JUMP_DYNAMIC:
4412 case INSN_JUMP_DYNAMIC_CONDITIONAL: 4412 case INSN_JUMP_DYNAMIC_CONDITIONAL:
4413 case INSN_RETURN: 4413 case INSN_RETURN:
4414 case INSN_NOP: 4414 case INSN_NOP:
4415 return 0; 4415 return 0;
4416 default: 4416 default:
4417 break; 4417 break;
4418 } 4418 }
4419 4419
4420 for (reloc = insn_reloc(file, insn); 4420 for (reloc = insn_reloc(file, insn);
4421 reloc; 4421 reloc;
4422 reloc = find_reloc_by_dest_range 4422 reloc = find_reloc_by_dest_range(file->elf, insn->sec,
4423 4423 reloc_offset(reloc) + 1,
4424 4424 (insn->offset + insn->len) - (reloc_offset(reloc) + 1))) {
4425 4425
4426 /* 4426 /*
4427 * static_call_update() refer 4427 * static_call_update() references the trampoline, which
4428 * doesn't have (or need) END 4428 * doesn't have (or need) ENDBR. Skip warning in that case.
4429 */ 4429 */
4430 if (reloc->sym->static_call_t 4430 if (reloc->sym->static_call_tramp)
4431 continue; 4431 continue;
4432 4432
4433 off = reloc->sym->offset; 4433 off = reloc->sym->offset;
4434 if (reloc_type(reloc) == R_X8 4434 if (reloc_type(reloc) == R_X86_64_PC32 ||
4435 reloc_type(reloc) == R_X8 4435 reloc_type(reloc) == R_X86_64_PLT32)
4436 off += arch_dest_relo 4436 off += arch_dest_reloc_offset(reloc_addend(reloc));
4437 else 4437 else
4438 off += reloc_addend(r 4438 off += reloc_addend(reloc);
4439 4439
4440 dest = find_insn(file, reloc- 4440 dest = find_insn(file, reloc->sym->sec, off);
4441 if (!dest) 4441 if (!dest)
4442 continue; 4442 continue;
4443 4443
4444 if (dest->type == INSN_ENDBR) 4444 if (dest->type == INSN_ENDBR) {
4445 mark_endbr_used(dest) 4445 mark_endbr_used(dest);
4446 continue; 4446 continue;
4447 } 4447 }
4448 4448
4449 if (insn_func(dest) && insn_f 4449 if (insn_func(dest) && insn_func(insn) &&
4450 insn_func(dest)->pfunc == 4450 insn_func(dest)->pfunc == insn_func(insn)->pfunc) {
4451 /* 4451 /*
4452 * Anything from->to 4452 * Anything from->to self is either _THIS_IP_ or
4453 * IRET-to-self. 4453 * IRET-to-self.
4454 * 4454 *
4455 * There is no sane w 4455 * There is no sane way to annotate _THIS_IP_ since the
4456 * compiler treats th 4456 * compiler treats the relocation as a constant and is
4457 * happy to fold in o 4457 * happy to fold in offsets, skewing any annotation we
4458 * do, leading to vas 4458 * do, leading to vast amounts of false-positives.
4459 * 4459 *
4460 * There's also compi 4460 * There's also compiler generated _THIS_IP_ through
4461 * KCOV and such whic 4461 * KCOV and such which we have no hope of annotating.
4462 * 4462 *
4463 * As such, blanket a 4463 * As such, blanket accept self-references without
4464 * issue. 4464 * issue.
4465 */ 4465 */
4466 continue; 4466 continue;
4467 } 4467 }
4468 4468
4469 /* 4469 /*
4470 * Accept anything ANNOTATE_N 4470 * Accept anything ANNOTATE_NOENDBR.
4471 */ 4471 */
4472 if (dest->noendbr) 4472 if (dest->noendbr)
4473 continue; 4473 continue;
4474 4474
4475 /* 4475 /*
4476 * Accept if this is the inst 4476 * Accept if this is the instruction after a symbol
4477 * that is (no)endbr -- typic 4477 * that is (no)endbr -- typical code-range usage.
4478 */ 4478 */
4479 if (noendbr_range(file, dest) 4479 if (noendbr_range(file, dest))
4480 continue; 4480 continue;
4481 4481
4482 WARN_INSN(insn, "relocation t 4482 WARN_INSN(insn, "relocation to !ENDBR: %s", offstr(dest->sec, dest->offset));
4483 4483
4484 warnings++; 4484 warnings++;
4485 } 4485 }
4486 4486
4487 return warnings; 4487 return warnings;
4488 } 4488 }
4489 4489
4490 static int validate_ibt_data_reloc(struct obj 4490 static int validate_ibt_data_reloc(struct objtool_file *file,
4491 struct rel 4491 struct reloc *reloc)
4492 { 4492 {
4493 struct instruction *dest; 4493 struct instruction *dest;
4494 4494
4495 dest = find_insn(file, reloc->sym->se 4495 dest = find_insn(file, reloc->sym->sec,
4496 reloc->sym->offset + 4496 reloc->sym->offset + reloc_addend(reloc));
4497 if (!dest) 4497 if (!dest)
4498 return 0; 4498 return 0;
4499 4499
4500 if (dest->type == INSN_ENDBR) { 4500 if (dest->type == INSN_ENDBR) {
4501 mark_endbr_used(dest); 4501 mark_endbr_used(dest);
4502 return 0; 4502 return 0;
4503 } 4503 }
4504 4504
4505 if (dest->noendbr) 4505 if (dest->noendbr)
4506 return 0; 4506 return 0;
4507 4507
4508 WARN_FUNC("data relocation to !ENDBR: 4508 WARN_FUNC("data relocation to !ENDBR: %s",
4509 reloc->sec->base, reloc_off 4509 reloc->sec->base, reloc_offset(reloc),
4510 offstr(dest->sec, dest->off 4510 offstr(dest->sec, dest->offset));
4511 4511
4512 return 1; 4512 return 1;
4513 } 4513 }
4514 4514
4515 /* 4515 /*
4516 * Validate IBT rules and remove used ENDBR i 4516 * Validate IBT rules and remove used ENDBR instructions from the seal list.
4517 * Unused ENDBR instructions will be annotate 4517 * Unused ENDBR instructions will be annotated for sealing (i.e., replaced with
4518 * NOPs) later, in create_ibt_endbr_seal_sect 4518 * NOPs) later, in create_ibt_endbr_seal_sections().
4519 */ 4519 */
4520 static int validate_ibt(struct objtool_file * 4520 static int validate_ibt(struct objtool_file *file)
4521 { 4521 {
4522 struct section *sec; 4522 struct section *sec;
4523 struct reloc *reloc; 4523 struct reloc *reloc;
4524 struct instruction *insn; 4524 struct instruction *insn;
4525 int warnings = 0; 4525 int warnings = 0;
4526 4526
4527 for_each_insn(file, insn) 4527 for_each_insn(file, insn)
4528 warnings += validate_ibt_insn 4528 warnings += validate_ibt_insn(file, insn);
4529 4529
4530 for_each_sec(file, sec) { 4530 for_each_sec(file, sec) {
4531 4531
4532 /* Already done by validate_i 4532 /* Already done by validate_ibt_insn() */
4533 if (sec->sh.sh_flags & SHF_EX 4533 if (sec->sh.sh_flags & SHF_EXECINSTR)
4534 continue; 4534 continue;
4535 4535
4536 if (!sec->rsec) 4536 if (!sec->rsec)
4537 continue; 4537 continue;
4538 4538
4539 /* 4539 /*
4540 * These sections can referen 4540 * These sections can reference text addresses, but not with
4541 * the intent to indirect bra 4541 * the intent to indirect branch to them.
4542 */ 4542 */
4543 if ((!strncmp(sec->name, ".di 4543 if ((!strncmp(sec->name, ".discard", 8) &&
4544 strcmp(sec->name, ".disc 4544 strcmp(sec->name, ".discard.ibt_endbr_noseal")) ||
4545 !strncmp(sec->name, ".deb 4545 !strncmp(sec->name, ".debug", 6) ||
4546 !strcmp(sec->name, ".alti 4546 !strcmp(sec->name, ".altinstructions") ||
4547 !strcmp(sec->name, ".ibt_ 4547 !strcmp(sec->name, ".ibt_endbr_seal") ||
4548 !strcmp(sec->name, ".orc_ 4548 !strcmp(sec->name, ".orc_unwind_ip") ||
4549 !strcmp(sec->name, ".para 4549 !strcmp(sec->name, ".parainstructions") ||
4550 !strcmp(sec->name, ".retp 4550 !strcmp(sec->name, ".retpoline_sites") ||
4551 !strcmp(sec->name, ".smp_ 4551 !strcmp(sec->name, ".smp_locks") ||
4552 !strcmp(sec->name, ".stat 4552 !strcmp(sec->name, ".static_call_sites") ||
4553 !strcmp(sec->name, "_erro 4553 !strcmp(sec->name, "_error_injection_whitelist") ||
4554 !strcmp(sec->name, "_kpro 4554 !strcmp(sec->name, "_kprobe_blacklist") ||
4555 !strcmp(sec->name, "__bug 4555 !strcmp(sec->name, "__bug_table") ||
4556 !strcmp(sec->name, "__ex_ 4556 !strcmp(sec->name, "__ex_table") ||
4557 !strcmp(sec->name, "__jum 4557 !strcmp(sec->name, "__jump_table") ||
4558 !strcmp(sec->name, "__mco 4558 !strcmp(sec->name, "__mcount_loc") ||
4559 !strcmp(sec->name, ".kcfi 4559 !strcmp(sec->name, ".kcfi_traps") ||
4560 strstr(sec->name, "__patc 4560 strstr(sec->name, "__patchable_function_entries"))
4561 continue; 4561 continue;
4562 4562
4563 for_each_reloc(sec->rsec, rel 4563 for_each_reloc(sec->rsec, reloc)
4564 warnings += validate_ 4564 warnings += validate_ibt_data_reloc(file, reloc);
4565 } 4565 }
4566 4566
4567 return warnings; 4567 return warnings;
4568 } 4568 }
4569 4569
4570 static int validate_sls(struct objtool_file * 4570 static int validate_sls(struct objtool_file *file)
4571 { 4571 {
4572 struct instruction *insn, *next_insn; 4572 struct instruction *insn, *next_insn;
4573 int warnings = 0; 4573 int warnings = 0;
4574 4574
4575 for_each_insn(file, insn) { 4575 for_each_insn(file, insn) {
4576 next_insn = next_insn_same_se 4576 next_insn = next_insn_same_sec(file, insn);
4577 4577
4578 if (insn->retpoline_safe) 4578 if (insn->retpoline_safe)
4579 continue; 4579 continue;
4580 4580
4581 switch (insn->type) { 4581 switch (insn->type) {
4582 case INSN_RETURN: 4582 case INSN_RETURN:
4583 if (!next_insn || nex 4583 if (!next_insn || next_insn->type != INSN_TRAP) {
4584 WARN_INSN(ins 4584 WARN_INSN(insn, "missing int3 after ret");
4585 warnings++; 4585 warnings++;
4586 } 4586 }
4587 4587
4588 break; 4588 break;
4589 case INSN_JUMP_DYNAMIC: 4589 case INSN_JUMP_DYNAMIC:
4590 if (!next_insn || nex 4590 if (!next_insn || next_insn->type != INSN_TRAP) {
4591 WARN_INSN(ins 4591 WARN_INSN(insn, "missing int3 after indirect jump");
4592 warnings++; 4592 warnings++;
4593 } 4593 }
4594 break; 4594 break;
4595 default: 4595 default:
4596 break; 4596 break;
4597 } 4597 }
4598 } 4598 }
4599 4599
4600 return warnings; 4600 return warnings;
4601 } 4601 }
4602 4602
4603 static bool ignore_noreturn_call(struct instr 4603 static bool ignore_noreturn_call(struct instruction *insn)
4604 { 4604 {
4605 struct symbol *call_dest = insn_call_ 4605 struct symbol *call_dest = insn_call_dest(insn);
4606 4606
4607 /* 4607 /*
4608 * FIXME: hack, we need a real noretu 4608 * FIXME: hack, we need a real noreturn solution
4609 * 4609 *
4610 * Problem is, exc_double_fault() may 4610 * Problem is, exc_double_fault() may or may not return, depending on
4611 * whether CONFIG_X86_ESPFIX64 is set 4611 * whether CONFIG_X86_ESPFIX64 is set. But objtool has no visibility
4612 * to the kernel config. 4612 * to the kernel config.
4613 * 4613 *
4614 * Other potential ways to fix it: 4614 * Other potential ways to fix it:
4615 * 4615 *
4616 * - have compiler communicate __no 4616 * - have compiler communicate __noreturn functions somehow
4617 * - remove CONFIG_X86_ESPFIX64 4617 * - remove CONFIG_X86_ESPFIX64
4618 * - read the .config file 4618 * - read the .config file
4619 * - add a cmdline option 4619 * - add a cmdline option
4620 * - create a generic objtool annot 4620 * - create a generic objtool annotation format (vs a bunch of custom
4621 * formats) and annotate it 4621 * formats) and annotate it
4622 */ 4622 */
4623 if (!strcmp(call_dest->name, "exc_dou 4623 if (!strcmp(call_dest->name, "exc_double_fault")) {
4624 /* prevent further unreachabl 4624 /* prevent further unreachable warnings for the caller */
4625 insn->sym->warned = 1; 4625 insn->sym->warned = 1;
4626 return true; 4626 return true;
4627 } 4627 }
4628 4628
4629 return false; 4629 return false;
4630 } 4630 }
4631 4631
4632 static int validate_reachable_instructions(st 4632 static int validate_reachable_instructions(struct objtool_file *file)
4633 { 4633 {
4634 struct instruction *insn, *prev_insn; 4634 struct instruction *insn, *prev_insn;
4635 struct symbol *call_dest; 4635 struct symbol *call_dest;
4636 int warnings = 0; 4636 int warnings = 0;
4637 4637
4638 if (file->ignore_unreachables) 4638 if (file->ignore_unreachables)
4639 return 0; 4639 return 0;
4640 4640
4641 for_each_insn(file, insn) { 4641 for_each_insn(file, insn) {
4642 if (insn->visited || ignore_u 4642 if (insn->visited || ignore_unreachable_insn(file, insn))
4643 continue; 4643 continue;
4644 4644
4645 prev_insn = prev_insn_same_se 4645 prev_insn = prev_insn_same_sec(file, insn);
4646 if (prev_insn && prev_insn->d 4646 if (prev_insn && prev_insn->dead_end) {
4647 call_dest = insn_call 4647 call_dest = insn_call_dest(prev_insn);
4648 if (call_dest && !ign 4648 if (call_dest && !ignore_noreturn_call(prev_insn)) {
4649 WARN_INSN(ins 4649 WARN_INSN(insn, "%s() is missing a __noreturn annotation",
4650 cal 4650 call_dest->name);
4651 warnings++; 4651 warnings++;
4652 continue; 4652 continue;
4653 } 4653 }
4654 } 4654 }
4655 4655
4656 WARN_INSN(insn, "unreachable 4656 WARN_INSN(insn, "unreachable instruction");
4657 warnings++; 4657 warnings++;
4658 } 4658 }
4659 4659
4660 return warnings; 4660 return warnings;
4661 } 4661 }
4662 4662
4663 /* 'funcs' is a space-separated list of funct 4663 /* 'funcs' is a space-separated list of function names */
4664 static int disas_funcs(const char *funcs) 4664 static int disas_funcs(const char *funcs)
4665 { 4665 {
4666 const char *objdump_str, *cross_compi 4666 const char *objdump_str, *cross_compile;
4667 int size, ret; 4667 int size, ret;
4668 char *cmd; 4668 char *cmd;
4669 4669
4670 cross_compile = getenv("CROSS_COMPILE 4670 cross_compile = getenv("CROSS_COMPILE");
4671 4671
4672 objdump_str = "%sobjdump -wdr %s | ga 4672 objdump_str = "%sobjdump -wdr %s | gawk -M -v _funcs='%s' '"
4673 "BEGIN { split(_funcs 4673 "BEGIN { split(_funcs, funcs); }"
4674 "/^$/ { func_match = 4674 "/^$/ { func_match = 0; }"
4675 "/<.*>:/ { " 4675 "/<.*>:/ { "
4676 "f = gensub(/ 4676 "f = gensub(/.*<(.*)>:/, \"\\\\1\", 1);"
4677 "for (i in fu 4677 "for (i in funcs) {"
4678 "if ( 4678 "if (funcs[i] == f) {"
4679 4679 "func_match = 1;"
4680 4680 "base = strtonum(\"0x\" $1);"
4681 4681 "break;"
4682 "}" 4682 "}"
4683 "}" 4683 "}"
4684 "}" 4684 "}"
4685 "{" 4685 "{"
4686 "if (func_mat 4686 "if (func_match) {"
4687 "addr 4687 "addr = strtonum(\"0x\" $1);"
4688 "prin 4688 "printf(\"%%04x \", addr - base);"
4689 "prin 4689 "print;"
4690 "}" 4690 "}"
4691 "}' 1>&2"; 4691 "}' 1>&2";
4692 4692
4693 /* fake snprintf() to calculate the s 4693 /* fake snprintf() to calculate the size */
4694 size = snprintf(NULL, 0, objdump_str, 4694 size = snprintf(NULL, 0, objdump_str, cross_compile, objname, funcs) + 1;
4695 if (size <= 0) { 4695 if (size <= 0) {
4696 WARN("objdump string size cal 4696 WARN("objdump string size calculation failed");
4697 return -1; 4697 return -1;
4698 } 4698 }
4699 4699
4700 cmd = malloc(size); 4700 cmd = malloc(size);
4701 4701
4702 /* real snprintf() */ 4702 /* real snprintf() */
4703 snprintf(cmd, size, objdump_str, cros 4703 snprintf(cmd, size, objdump_str, cross_compile, objname, funcs);
4704 ret = system(cmd); 4704 ret = system(cmd);
4705 if (ret) { 4705 if (ret) {
4706 WARN("disassembly failed: %d" 4706 WARN("disassembly failed: %d", ret);
4707 return -1; 4707 return -1;
4708 } 4708 }
4709 4709
4710 return 0; 4710 return 0;
4711 } 4711 }
4712 4712
4713 static int disas_warned_funcs(struct objtool_ 4713 static int disas_warned_funcs(struct objtool_file *file)
4714 { 4714 {
4715 struct symbol *sym; 4715 struct symbol *sym;
4716 char *funcs = NULL, *tmp; 4716 char *funcs = NULL, *tmp;
4717 4717
4718 for_each_sym(file, sym) { 4718 for_each_sym(file, sym) {
4719 if (sym->warned) { 4719 if (sym->warned) {
4720 if (!funcs) { 4720 if (!funcs) {
4721 funcs = mallo 4721 funcs = malloc(strlen(sym->name) + 1);
4722 strcpy(funcs, 4722 strcpy(funcs, sym->name);
4723 } else { 4723 } else {
4724 tmp = malloc( 4724 tmp = malloc(strlen(funcs) + strlen(sym->name) + 2);
4725 sprintf(tmp, 4725 sprintf(tmp, "%s %s", funcs, sym->name);
4726 free(funcs); 4726 free(funcs);
4727 funcs = tmp; 4727 funcs = tmp;
4728 } 4728 }
4729 } 4729 }
4730 } 4730 }
4731 4731
4732 if (funcs) 4732 if (funcs)
4733 disas_funcs(funcs); 4733 disas_funcs(funcs);
4734 4734
4735 return 0; 4735 return 0;
4736 } 4736 }
4737 4737
4738 struct insn_chunk { 4738 struct insn_chunk {
4739 void *addr; 4739 void *addr;
4740 struct insn_chunk *next; 4740 struct insn_chunk *next;
4741 }; 4741 };
4742 4742
4743 /* 4743 /*
4744 * Reduce peak RSS usage by freeing insns mem 4744 * Reduce peak RSS usage by freeing insns memory before writing the ELF file,
4745 * which can trigger more allocations for .de 4745 * which can trigger more allocations for .debug_* sections whose data hasn't
4746 * been read yet. 4746 * been read yet.
4747 */ 4747 */
4748 static void free_insns(struct objtool_file *f 4748 static void free_insns(struct objtool_file *file)
4749 { 4749 {
4750 struct instruction *insn; 4750 struct instruction *insn;
4751 struct insn_chunk *chunks = NULL, *ch 4751 struct insn_chunk *chunks = NULL, *chunk;
4752 4752
4753 for_each_insn(file, insn) { 4753 for_each_insn(file, insn) {
4754 if (!insn->idx) { 4754 if (!insn->idx) {
4755 chunk = malloc(sizeof 4755 chunk = malloc(sizeof(*chunk));
4756 chunk->addr = insn; 4756 chunk->addr = insn;
4757 chunk->next = chunks; 4757 chunk->next = chunks;
4758 chunks = chunk; 4758 chunks = chunk;
4759 } 4759 }
4760 } 4760 }
4761 4761
4762 for (chunk = chunks; chunk; chunk = c 4762 for (chunk = chunks; chunk; chunk = chunk->next)
4763 free(chunk->addr); 4763 free(chunk->addr);
4764 } 4764 }
4765 4765
4766 int check(struct objtool_file *file) 4766 int check(struct objtool_file *file)
4767 { 4767 {
4768 int ret, warnings = 0; 4768 int ret, warnings = 0;
4769 4769
4770 arch_initial_func_cfi_state(&initial_ 4770 arch_initial_func_cfi_state(&initial_func_cfi);
4771 init_cfi_state(&init_cfi); 4771 init_cfi_state(&init_cfi);
4772 init_cfi_state(&func_cfi); 4772 init_cfi_state(&func_cfi);
4773 set_func_state(&func_cfi); 4773 set_func_state(&func_cfi);
4774 init_cfi_state(&force_undefined_cfi); 4774 init_cfi_state(&force_undefined_cfi);
4775 force_undefined_cfi.force_undefined = 4775 force_undefined_cfi.force_undefined = true;
4776 4776
4777 if (!cfi_hash_alloc(1UL << (file->elf 4777 if (!cfi_hash_alloc(1UL << (file->elf->symbol_bits - 3)))
4778 goto out; 4778 goto out;
4779 4779
4780 cfi_hash_add(&init_cfi); 4780 cfi_hash_add(&init_cfi);
4781 cfi_hash_add(&func_cfi); 4781 cfi_hash_add(&func_cfi);
4782 4782
4783 ret = decode_sections(file); 4783 ret = decode_sections(file);
4784 if (ret < 0) 4784 if (ret < 0)
4785 goto out; 4785 goto out;
4786 4786
4787 warnings += ret; 4787 warnings += ret;
4788 4788
4789 if (!nr_insns) 4789 if (!nr_insns)
4790 goto out; 4790 goto out;
4791 4791
4792 if (opts.retpoline) { 4792 if (opts.retpoline) {
4793 ret = validate_retpoline(file 4793 ret = validate_retpoline(file);
4794 if (ret < 0) 4794 if (ret < 0)
4795 return ret; 4795 return ret;
4796 warnings += ret; 4796 warnings += ret;
4797 } 4797 }
4798 4798
4799 if (opts.stackval || opts.orc || opts 4799 if (opts.stackval || opts.orc || opts.uaccess) {
4800 ret = validate_functions(file 4800 ret = validate_functions(file);
4801 if (ret < 0) 4801 if (ret < 0)
4802 goto out; 4802 goto out;
4803 warnings += ret; 4803 warnings += ret;
4804 4804
4805 ret = validate_unwind_hints(f 4805 ret = validate_unwind_hints(file, NULL);
4806 if (ret < 0) 4806 if (ret < 0)
4807 goto out; 4807 goto out;
4808 warnings += ret; 4808 warnings += ret;
4809 4809
4810 if (!warnings) { 4810 if (!warnings) {
4811 ret = validate_reacha 4811 ret = validate_reachable_instructions(file);
4812 if (ret < 0) 4812 if (ret < 0)
4813 goto out; 4813 goto out;
4814 warnings += ret; 4814 warnings += ret;
4815 } 4815 }
4816 4816
4817 } else if (opts.noinstr) { 4817 } else if (opts.noinstr) {
4818 ret = validate_noinstr_sectio 4818 ret = validate_noinstr_sections(file);
4819 if (ret < 0) 4819 if (ret < 0)
4820 goto out; 4820 goto out;
4821 warnings += ret; 4821 warnings += ret;
4822 } 4822 }
4823 4823
4824 if (opts.unret) { 4824 if (opts.unret) {
4825 /* 4825 /*
4826 * Must be after validate_bra 4826 * Must be after validate_branch() and friends, it plays
4827 * further games with insn->v 4827 * further games with insn->visited.
4828 */ 4828 */
4829 ret = validate_unrets(file); 4829 ret = validate_unrets(file);
4830 if (ret < 0) 4830 if (ret < 0)
4831 return ret; 4831 return ret;
4832 warnings += ret; 4832 warnings += ret;
4833 } 4833 }
4834 4834
4835 if (opts.ibt) { 4835 if (opts.ibt) {
4836 ret = validate_ibt(file); 4836 ret = validate_ibt(file);
4837 if (ret < 0) 4837 if (ret < 0)
4838 goto out; 4838 goto out;
4839 warnings += ret; 4839 warnings += ret;
4840 } 4840 }
4841 4841
4842 if (opts.sls) { 4842 if (opts.sls) {
4843 ret = validate_sls(file); 4843 ret = validate_sls(file);
4844 if (ret < 0) 4844 if (ret < 0)
4845 goto out; 4845 goto out;
4846 warnings += ret; 4846 warnings += ret;
4847 } 4847 }
4848 4848
4849 if (opts.static_call) { 4849 if (opts.static_call) {
4850 ret = create_static_call_sect 4850 ret = create_static_call_sections(file);
4851 if (ret < 0) 4851 if (ret < 0)
4852 goto out; 4852 goto out;
4853 warnings += ret; 4853 warnings += ret;
4854 } 4854 }
4855 4855
4856 if (opts.retpoline) { 4856 if (opts.retpoline) {
4857 ret = create_retpoline_sites_ 4857 ret = create_retpoline_sites_sections(file);
4858 if (ret < 0) 4858 if (ret < 0)
4859 goto out; 4859 goto out;
4860 warnings += ret; 4860 warnings += ret;
4861 } 4861 }
4862 4862
4863 if (opts.cfi) { 4863 if (opts.cfi) {
4864 ret = create_cfi_sections(fil 4864 ret = create_cfi_sections(file);
4865 if (ret < 0) 4865 if (ret < 0)
4866 goto out; 4866 goto out;
4867 warnings += ret; 4867 warnings += ret;
4868 } 4868 }
4869 4869
4870 if (opts.rethunk) { 4870 if (opts.rethunk) {
4871 ret = create_return_sites_sec 4871 ret = create_return_sites_sections(file);
4872 if (ret < 0) 4872 if (ret < 0)
4873 goto out; 4873 goto out;
4874 warnings += ret; 4874 warnings += ret;
4875 4875
4876 if (opts.hack_skylake) { 4876 if (opts.hack_skylake) {
4877 ret = create_direct_c 4877 ret = create_direct_call_sections(file);
4878 if (ret < 0) 4878 if (ret < 0)
4879 goto out; 4879 goto out;
4880 warnings += ret; 4880 warnings += ret;
4881 } 4881 }
4882 } 4882 }
4883 4883
4884 if (opts.mcount) { 4884 if (opts.mcount) {
4885 ret = create_mcount_loc_secti 4885 ret = create_mcount_loc_sections(file);
4886 if (ret < 0) 4886 if (ret < 0)
4887 goto out; 4887 goto out;
4888 warnings += ret; 4888 warnings += ret;
4889 } 4889 }
4890 4890
4891 if (opts.prefix) { 4891 if (opts.prefix) {
4892 ret = add_prefix_symbols(file 4892 ret = add_prefix_symbols(file);
4893 if (ret < 0) 4893 if (ret < 0)
4894 return ret; 4894 return ret;
4895 warnings += ret; 4895 warnings += ret;
4896 } 4896 }
4897 4897
4898 if (opts.ibt) { 4898 if (opts.ibt) {
4899 ret = create_ibt_endbr_seal_s 4899 ret = create_ibt_endbr_seal_sections(file);
4900 if (ret < 0) 4900 if (ret < 0)
4901 goto out; 4901 goto out;
4902 warnings += ret; 4902 warnings += ret;
4903 } 4903 }
4904 4904
4905 if (opts.orc && nr_insns) { 4905 if (opts.orc && nr_insns) {
4906 ret = orc_create(file); 4906 ret = orc_create(file);
4907 if (ret < 0) 4907 if (ret < 0)
4908 goto out; 4908 goto out;
4909 warnings += ret; 4909 warnings += ret;
4910 } 4910 }
4911 4911
4912 free_insns(file); 4912 free_insns(file);
4913 4913
4914 if (opts.verbose) 4914 if (opts.verbose)
4915 disas_warned_funcs(file); 4915 disas_warned_funcs(file);
4916 4916
4917 if (opts.stats) { 4917 if (opts.stats) {
4918 printf("nr_insns_visited: %ld 4918 printf("nr_insns_visited: %ld\n", nr_insns_visited);
4919 printf("nr_cfi: %ld\n", nr_cf 4919 printf("nr_cfi: %ld\n", nr_cfi);
4920 printf("nr_cfi_reused: %ld\n" 4920 printf("nr_cfi_reused: %ld\n", nr_cfi_reused);
4921 printf("nr_cfi_cache: %ld\n", 4921 printf("nr_cfi_cache: %ld\n", nr_cfi_cache);
4922 } 4922 }
4923 4923
4924 out: 4924 out:
4925 /* 4925 /*
4926 * For now, don't fail the kernel bu 4926 * For now, don't fail the kernel build on fatal warnings. These
4927 * errors are still fairly common du 4927 * errors are still fairly common due to the growing matrix of
4928 * supported toolchains and their re 4928 * supported toolchains and their recent pace of change.
4929 */ 4929 */
4930 return 0; 4930 return 0;
4931 } 4931 }
4932 4932
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.