1 // SPDX-License-Identifier: GPL-2.0-or-later 1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /* 2 /*
3 * Copyright (C) 2015-2017 Josh Poimboeuf <jpo 3 * Copyright (C) 2015-2017 Josh Poimboeuf <jpoimboe@redhat.com>
4 */ 4 */
5 5
6 #include <string.h> 6 #include <string.h>
7 #include <stdlib.h> 7 #include <stdlib.h>
8 #include <inttypes.h> 8 #include <inttypes.h>
9 #include <sys/mman.h> 9 #include <sys/mman.h>
10 10
11 #include <objtool/builtin.h> 11 #include <objtool/builtin.h>
12 #include <objtool/cfi.h> 12 #include <objtool/cfi.h>
13 #include <objtool/arch.h> 13 #include <objtool/arch.h>
14 #include <objtool/check.h> 14 #include <objtool/check.h>
15 #include <objtool/special.h> 15 #include <objtool/special.h>
16 #include <objtool/warn.h> 16 #include <objtool/warn.h>
17 #include <objtool/endianness.h> 17 #include <objtool/endianness.h>
18 18
19 #include <linux/objtool_types.h> 19 #include <linux/objtool_types.h>
20 #include <linux/hashtable.h> 20 #include <linux/hashtable.h>
21 #include <linux/kernel.h> 21 #include <linux/kernel.h>
22 #include <linux/static_call_types.h> 22 #include <linux/static_call_types.h>
23 #include <linux/string.h> 23 #include <linux/string.h>
24 24
25 struct alternative { 25 struct alternative {
26 struct alternative *next; 26 struct alternative *next;
27 struct instruction *insn; 27 struct instruction *insn;
28 bool skip_orig; 28 bool skip_orig;
29 }; 29 };
30 30
31 static unsigned long nr_cfi, nr_cfi_reused, nr 31 static unsigned long nr_cfi, nr_cfi_reused, nr_cfi_cache;
32 32
33 static struct cfi_init_state initial_func_cfi; 33 static struct cfi_init_state initial_func_cfi;
34 static struct cfi_state init_cfi; 34 static struct cfi_state init_cfi;
35 static struct cfi_state func_cfi; 35 static struct cfi_state func_cfi;
36 static struct cfi_state force_undefined_cfi; 36 static struct cfi_state force_undefined_cfi;
37 37
38 struct instruction *find_insn(struct objtool_f 38 struct instruction *find_insn(struct objtool_file *file,
39 struct section * 39 struct section *sec, unsigned long offset)
40 { 40 {
41 struct instruction *insn; 41 struct instruction *insn;
42 42
43 hash_for_each_possible(file->insn_hash 43 hash_for_each_possible(file->insn_hash, insn, hash, sec_offset_hash(sec, offset)) {
44 if (insn->sec == sec && insn-> 44 if (insn->sec == sec && insn->offset == offset)
45 return insn; 45 return insn;
46 } 46 }
47 47
48 return NULL; 48 return NULL;
49 } 49 }
50 50
51 struct instruction *next_insn_same_sec(struct 51 struct instruction *next_insn_same_sec(struct objtool_file *file,
52 struct 52 struct instruction *insn)
53 { 53 {
54 if (insn->idx == INSN_CHUNK_MAX) 54 if (insn->idx == INSN_CHUNK_MAX)
55 return find_insn(file, insn->s 55 return find_insn(file, insn->sec, insn->offset + insn->len);
56 56
57 insn++; 57 insn++;
58 if (!insn->len) 58 if (!insn->len)
59 return NULL; 59 return NULL;
60 60
61 return insn; 61 return insn;
62 } 62 }
63 63
64 static struct instruction *next_insn_same_func 64 static struct instruction *next_insn_same_func(struct objtool_file *file,
65 65 struct instruction *insn)
66 { 66 {
67 struct instruction *next = next_insn_s 67 struct instruction *next = next_insn_same_sec(file, insn);
68 struct symbol *func = insn_func(insn); 68 struct symbol *func = insn_func(insn);
69 69
70 if (!func) 70 if (!func)
71 return NULL; 71 return NULL;
72 72
73 if (next && insn_func(next) == func) 73 if (next && insn_func(next) == func)
74 return next; 74 return next;
75 75
76 /* Check if we're already in the subfu 76 /* Check if we're already in the subfunction: */
77 if (func == func->cfunc) 77 if (func == func->cfunc)
78 return NULL; 78 return NULL;
79 79
80 /* Move to the subfunction: */ 80 /* Move to the subfunction: */
81 return find_insn(file, func->cfunc->se 81 return find_insn(file, func->cfunc->sec, func->cfunc->offset);
82 } 82 }
83 83
84 static struct instruction *prev_insn_same_sec( 84 static struct instruction *prev_insn_same_sec(struct objtool_file *file,
85 85 struct instruction *insn)
86 { 86 {
87 if (insn->idx == 0) { 87 if (insn->idx == 0) {
88 if (insn->prev_len) 88 if (insn->prev_len)
89 return find_insn(file, 89 return find_insn(file, insn->sec, insn->offset - insn->prev_len);
90 return NULL; 90 return NULL;
91 } 91 }
92 92
93 return insn - 1; 93 return insn - 1;
94 } 94 }
95 95
96 static struct instruction *prev_insn_same_sym( 96 static struct instruction *prev_insn_same_sym(struct objtool_file *file,
97 97 struct instruction *insn)
98 { 98 {
99 struct instruction *prev = prev_insn_s 99 struct instruction *prev = prev_insn_same_sec(file, insn);
100 100
101 if (prev && insn_func(prev) == insn_fu 101 if (prev && insn_func(prev) == insn_func(insn))
102 return prev; 102 return prev;
103 103
104 return NULL; 104 return NULL;
105 } 105 }
106 106
107 #define for_each_insn(file, insn) 107 #define for_each_insn(file, insn) \
108 for (struct section *__sec, *__fake = 108 for (struct section *__sec, *__fake = (struct section *)1; \
109 __fake; __fake = NULL) 109 __fake; __fake = NULL) \
110 for_each_sec(file, __sec) 110 for_each_sec(file, __sec) \
111 sec_for_each_insn(file 111 sec_for_each_insn(file, __sec, insn)
112 112
113 #define func_for_each_insn(file, func, insn) 113 #define func_for_each_insn(file, func, insn) \
114 for (insn = find_insn(file, func->sec, 114 for (insn = find_insn(file, func->sec, func->offset); \
115 insn; 115 insn; \
116 insn = next_insn_same_func(file, 116 insn = next_insn_same_func(file, insn))
117 117
118 #define sym_for_each_insn(file, sym, insn) 118 #define sym_for_each_insn(file, sym, insn) \
119 for (insn = find_insn(file, sym->sec, 119 for (insn = find_insn(file, sym->sec, sym->offset); \
120 insn && insn->offset < sym->offse 120 insn && insn->offset < sym->offset + sym->len; \
121 insn = next_insn_same_sec(file, i 121 insn = next_insn_same_sec(file, insn))
122 122
123 #define sym_for_each_insn_continue_reverse(fil 123 #define sym_for_each_insn_continue_reverse(file, sym, insn) \
124 for (insn = prev_insn_same_sec(file, i 124 for (insn = prev_insn_same_sec(file, insn); \
125 insn && insn->offset >= sym->offs 125 insn && insn->offset >= sym->offset; \
126 insn = prev_insn_same_sec(file, i 126 insn = prev_insn_same_sec(file, insn))
127 127
128 #define sec_for_each_insn_from(file, insn) 128 #define sec_for_each_insn_from(file, insn) \
129 for (; insn; insn = next_insn_same_sec 129 for (; insn; insn = next_insn_same_sec(file, insn))
130 130
131 #define sec_for_each_insn_continue(file, insn) 131 #define sec_for_each_insn_continue(file, insn) \
132 for (insn = next_insn_same_sec(file, i 132 for (insn = next_insn_same_sec(file, insn); insn; \
133 insn = next_insn_same_sec(file, i 133 insn = next_insn_same_sec(file, insn))
134 134
135 static inline struct symbol *insn_call_dest(st 135 static inline struct symbol *insn_call_dest(struct instruction *insn)
136 { 136 {
137 if (insn->type == INSN_JUMP_DYNAMIC || 137 if (insn->type == INSN_JUMP_DYNAMIC ||
138 insn->type == INSN_CALL_DYNAMIC) 138 insn->type == INSN_CALL_DYNAMIC)
139 return NULL; 139 return NULL;
140 140
141 return insn->_call_dest; 141 return insn->_call_dest;
142 } 142 }
143 143
144 static inline struct reloc *insn_jump_table(st 144 static inline struct reloc *insn_jump_table(struct instruction *insn)
145 { 145 {
146 if (insn->type == INSN_JUMP_DYNAMIC || 146 if (insn->type == INSN_JUMP_DYNAMIC ||
147 insn->type == INSN_CALL_DYNAMIC) 147 insn->type == INSN_CALL_DYNAMIC)
148 return insn->_jump_table; 148 return insn->_jump_table;
149 149
150 return NULL; 150 return NULL;
151 } 151 }
152 152
153 static bool is_jump_table_jump(struct instruct 153 static bool is_jump_table_jump(struct instruction *insn)
154 { 154 {
155 struct alt_group *alt_group = insn->al 155 struct alt_group *alt_group = insn->alt_group;
156 156
157 if (insn_jump_table(insn)) 157 if (insn_jump_table(insn))
158 return true; 158 return true;
159 159
160 /* Retpoline alternative for a jump ta 160 /* Retpoline alternative for a jump table? */
161 return alt_group && alt_group->orig_gr 161 return alt_group && alt_group->orig_group &&
162 insn_jump_table(alt_group->orig 162 insn_jump_table(alt_group->orig_group->first_insn);
163 } 163 }
164 164
165 static bool is_sibling_call(struct instruction 165 static bool is_sibling_call(struct instruction *insn)
166 { 166 {
167 /* 167 /*
168 * Assume only STT_FUNC calls have jum 168 * Assume only STT_FUNC calls have jump-tables.
169 */ 169 */
170 if (insn_func(insn)) { 170 if (insn_func(insn)) {
171 /* An indirect jump is either 171 /* An indirect jump is either a sibling call or a jump to a table. */
172 if (insn->type == INSN_JUMP_DY 172 if (insn->type == INSN_JUMP_DYNAMIC)
173 return !is_jump_table_ 173 return !is_jump_table_jump(insn);
174 } 174 }
175 175
176 /* add_jump_destinations() sets insn_c 176 /* add_jump_destinations() sets insn_call_dest(insn) for sibling calls. */
177 return (is_static_jump(insn) && insn_c 177 return (is_static_jump(insn) && insn_call_dest(insn));
178 } 178 }
179 179
180 /* 180 /*
181 * Checks if a string ends with another. <<
182 */ <<
183 static bool str_ends_with(const char *s, const <<
184 { <<
185 const int slen = strlen(s); <<
186 const int sublen = strlen(sub); <<
187 <<
188 if (sublen > slen) <<
189 return 0; <<
190 <<
191 return !memcmp(s + slen - sublen, sub, <<
192 } <<
193 <<
194 /* <<
195 * Checks if a function is a Rust "noreturn" o <<
196 */ <<
197 static bool is_rust_noreturn(const struct symb <<
198 { <<
199 /* <<
200 * If it does not start with "_R", the <<
201 */ <<
202 if (strncmp(func->name, "_R", 2)) <<
203 return false; <<
204 <<
205 /* <<
206 * These are just heuristics -- we do <<
207 * name, due to the crate disambiguato <<
208 * as well as changes to the source co <<
209 * these come from the Rust standard l <<
210 */ <<
211 return str_ends_with(func->name, "_4co <<
212 str_ends_with(func->name, "_4co <<
213 str_ends_with(func->name, "_4co <<
214 str_ends_with(func->name, "_4co <<
215 str_ends_with(func->name, "_4co <<
216 str_ends_with(func->name, "_4co <<
217 str_ends_with(func->name, "_4co <<
218 str_ends_with(func->name, "_4co <<
219 str_ends_with(func->name, "_4co <<
220 str_ends_with(func->name, "_4co <<
221 strstr(func->name, "_4core9pani <<
222 (strstr(func->name, "_4core5sli <<
223 str_ends_with(func->name, "_fa <<
224 } <<
225 <<
226 /* <<
227 * This checks to see if the given function is 181 * This checks to see if the given function is a "noreturn" function.
228 * 182 *
229 * For global functions which are outside the 183 * For global functions which are outside the scope of this object file, we
230 * have to keep a manual list of them. 184 * have to keep a manual list of them.
231 * 185 *
232 * For local functions, we have to detect them 186 * For local functions, we have to detect them manually by simply looking for
233 * the lack of a return instruction. 187 * the lack of a return instruction.
234 */ 188 */
235 static bool __dead_end_function(struct objtool 189 static bool __dead_end_function(struct objtool_file *file, struct symbol *func,
236 int recursion) 190 int recursion)
237 { 191 {
238 int i; 192 int i;
239 struct instruction *insn; 193 struct instruction *insn;
240 bool empty = true; 194 bool empty = true;
241 195
242 #define NORETURN(func) __stringify(func), 196 #define NORETURN(func) __stringify(func),
243 static const char * const global_noret 197 static const char * const global_noreturns[] = {
244 #include "noreturns.h" 198 #include "noreturns.h"
245 }; 199 };
246 #undef NORETURN 200 #undef NORETURN
247 201
248 if (!func) 202 if (!func)
249 return false; 203 return false;
250 204
251 if (func->bind == STB_GLOBAL || func-> !! 205 if (func->bind == STB_GLOBAL || func->bind == STB_WEAK)
252 if (is_rust_noreturn(func)) <<
253 return true; <<
254 <<
255 for (i = 0; i < ARRAY_SIZE(glo 206 for (i = 0; i < ARRAY_SIZE(global_noreturns); i++)
256 if (!strcmp(func->name 207 if (!strcmp(func->name, global_noreturns[i]))
257 return true; 208 return true;
258 } <<
259 209
260 if (func->bind == STB_WEAK) 210 if (func->bind == STB_WEAK)
261 return false; 211 return false;
262 212
263 if (!func->len) 213 if (!func->len)
264 return false; 214 return false;
265 215
266 insn = find_insn(file, func->sec, func 216 insn = find_insn(file, func->sec, func->offset);
267 if (!insn || !insn_func(insn)) 217 if (!insn || !insn_func(insn))
268 return false; 218 return false;
269 219
270 func_for_each_insn(file, func, insn) { 220 func_for_each_insn(file, func, insn) {
271 empty = false; 221 empty = false;
272 222
273 if (insn->type == INSN_RETURN) 223 if (insn->type == INSN_RETURN)
274 return false; 224 return false;
275 } 225 }
276 226
277 if (empty) 227 if (empty)
278 return false; 228 return false;
279 229
280 /* 230 /*
281 * A function can have a sibling call 231 * A function can have a sibling call instead of a return. In that
282 * case, the function's dead-end statu 232 * case, the function's dead-end status depends on whether the target
283 * of the sibling call returns. 233 * of the sibling call returns.
284 */ 234 */
285 func_for_each_insn(file, func, insn) { 235 func_for_each_insn(file, func, insn) {
286 if (is_sibling_call(insn)) { 236 if (is_sibling_call(insn)) {
287 struct instruction *de 237 struct instruction *dest = insn->jump_dest;
288 238
289 if (!dest) 239 if (!dest)
290 /* sibling cal 240 /* sibling call to another file */
291 return false; 241 return false;
292 242
293 /* local sibling call 243 /* local sibling call */
294 if (recursion == 5) { 244 if (recursion == 5) {
295 /* 245 /*
296 * Infinite re 246 * Infinite recursion: two functions have
297 * sibling cal 247 * sibling calls to each other. This is a very
298 * rare case. 248 * rare case. It means they aren't dead ends.
299 */ 249 */
300 return false; 250 return false;
301 } 251 }
302 252
303 return __dead_end_func 253 return __dead_end_function(file, insn_func(dest), recursion+1);
304 } 254 }
305 } 255 }
306 256
307 return true; 257 return true;
308 } 258 }
309 259
310 static bool dead_end_function(struct objtool_f 260 static bool dead_end_function(struct objtool_file *file, struct symbol *func)
311 { 261 {
312 return __dead_end_function(file, func, 262 return __dead_end_function(file, func, 0);
313 } 263 }
314 264
315 static void init_cfi_state(struct cfi_state *c 265 static void init_cfi_state(struct cfi_state *cfi)
316 { 266 {
317 int i; 267 int i;
318 268
319 for (i = 0; i < CFI_NUM_REGS; i++) { 269 for (i = 0; i < CFI_NUM_REGS; i++) {
320 cfi->regs[i].base = CFI_UNDEFI 270 cfi->regs[i].base = CFI_UNDEFINED;
321 cfi->vals[i].base = CFI_UNDEFI 271 cfi->vals[i].base = CFI_UNDEFINED;
322 } 272 }
323 cfi->cfa.base = CFI_UNDEFINED; 273 cfi->cfa.base = CFI_UNDEFINED;
324 cfi->drap_reg = CFI_UNDEFINED; 274 cfi->drap_reg = CFI_UNDEFINED;
325 cfi->drap_offset = -1; 275 cfi->drap_offset = -1;
326 } 276 }
327 277
328 static void init_insn_state(struct objtool_fil 278 static void init_insn_state(struct objtool_file *file, struct insn_state *state,
329 struct section *se 279 struct section *sec)
330 { 280 {
331 memset(state, 0, sizeof(*state)); 281 memset(state, 0, sizeof(*state));
332 init_cfi_state(&state->cfi); 282 init_cfi_state(&state->cfi);
333 283
334 /* 284 /*
335 * We need the full vmlinux for noinst 285 * We need the full vmlinux for noinstr validation, otherwise we can
336 * not correctly determine insn_call_d 286 * not correctly determine insn_call_dest(insn)->sec (external symbols
337 * do not have a section). 287 * do not have a section).
338 */ 288 */
339 if (opts.link && opts.noinstr && sec) 289 if (opts.link && opts.noinstr && sec)
340 state->noinstr = sec->noinstr; 290 state->noinstr = sec->noinstr;
341 } 291 }
342 292
343 static struct cfi_state *cfi_alloc(void) 293 static struct cfi_state *cfi_alloc(void)
344 { 294 {
345 struct cfi_state *cfi = calloc(1, size 295 struct cfi_state *cfi = calloc(1, sizeof(struct cfi_state));
346 if (!cfi) { 296 if (!cfi) {
347 WARN("calloc failed"); 297 WARN("calloc failed");
348 exit(1); 298 exit(1);
349 } 299 }
350 nr_cfi++; 300 nr_cfi++;
351 return cfi; 301 return cfi;
352 } 302 }
353 303
354 static int cfi_bits; 304 static int cfi_bits;
355 static struct hlist_head *cfi_hash; 305 static struct hlist_head *cfi_hash;
356 306
357 static inline bool cficmp(struct cfi_state *cf 307 static inline bool cficmp(struct cfi_state *cfi1, struct cfi_state *cfi2)
358 { 308 {
359 return memcmp((void *)cfi1 + sizeof(cf 309 return memcmp((void *)cfi1 + sizeof(cfi1->hash),
360 (void *)cfi2 + sizeof(cf 310 (void *)cfi2 + sizeof(cfi2->hash),
361 sizeof(struct cfi_state) 311 sizeof(struct cfi_state) - sizeof(struct hlist_node));
362 } 312 }
363 313
364 static inline u32 cfi_key(struct cfi_state *cf 314 static inline u32 cfi_key(struct cfi_state *cfi)
365 { 315 {
366 return jhash((void *)cfi + sizeof(cfi- 316 return jhash((void *)cfi + sizeof(cfi->hash),
367 sizeof(*cfi) - sizeof(cfi 317 sizeof(*cfi) - sizeof(cfi->hash), 0);
368 } 318 }
369 319
370 static struct cfi_state *cfi_hash_find_or_add( 320 static struct cfi_state *cfi_hash_find_or_add(struct cfi_state *cfi)
371 { 321 {
372 struct hlist_head *head = &cfi_hash[ha 322 struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
373 struct cfi_state *obj; 323 struct cfi_state *obj;
374 324
375 hlist_for_each_entry(obj, head, hash) 325 hlist_for_each_entry(obj, head, hash) {
376 if (!cficmp(cfi, obj)) { 326 if (!cficmp(cfi, obj)) {
377 nr_cfi_cache++; 327 nr_cfi_cache++;
378 return obj; 328 return obj;
379 } 329 }
380 } 330 }
381 331
382 obj = cfi_alloc(); 332 obj = cfi_alloc();
383 *obj = *cfi; 333 *obj = *cfi;
384 hlist_add_head(&obj->hash, head); 334 hlist_add_head(&obj->hash, head);
385 335
386 return obj; 336 return obj;
387 } 337 }
388 338
389 static void cfi_hash_add(struct cfi_state *cfi 339 static void cfi_hash_add(struct cfi_state *cfi)
390 { 340 {
391 struct hlist_head *head = &cfi_hash[ha 341 struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
392 342
393 hlist_add_head(&cfi->hash, head); 343 hlist_add_head(&cfi->hash, head);
394 } 344 }
395 345
396 static void *cfi_hash_alloc(unsigned long size 346 static void *cfi_hash_alloc(unsigned long size)
397 { 347 {
398 cfi_bits = max(10, ilog2(size)); 348 cfi_bits = max(10, ilog2(size));
399 cfi_hash = mmap(NULL, sizeof(struct hl 349 cfi_hash = mmap(NULL, sizeof(struct hlist_head) << cfi_bits,
400 PROT_READ|PROT_WRITE, 350 PROT_READ|PROT_WRITE,
401 MAP_PRIVATE|MAP_ANON, 351 MAP_PRIVATE|MAP_ANON, -1, 0);
402 if (cfi_hash == (void *)-1L) { 352 if (cfi_hash == (void *)-1L) {
403 WARN("mmap fail cfi_hash"); 353 WARN("mmap fail cfi_hash");
404 cfi_hash = NULL; 354 cfi_hash = NULL;
405 } else if (opts.stats) { 355 } else if (opts.stats) {
406 printf("cfi_bits: %d\n", cfi_b 356 printf("cfi_bits: %d\n", cfi_bits);
407 } 357 }
408 358
409 return cfi_hash; 359 return cfi_hash;
410 } 360 }
411 361
412 static unsigned long nr_insns; 362 static unsigned long nr_insns;
413 static unsigned long nr_insns_visited; 363 static unsigned long nr_insns_visited;
414 364
415 /* 365 /*
416 * Call the arch-specific instruction decoder 366 * Call the arch-specific instruction decoder for all the instructions and add
417 * them to the global instruction list. 367 * them to the global instruction list.
418 */ 368 */
419 static int decode_instructions(struct objtool_ 369 static int decode_instructions(struct objtool_file *file)
420 { 370 {
421 struct section *sec; 371 struct section *sec;
422 struct symbol *func; 372 struct symbol *func;
423 unsigned long offset; 373 unsigned long offset;
424 struct instruction *insn; 374 struct instruction *insn;
425 int ret; 375 int ret;
426 376
427 for_each_sec(file, sec) { 377 for_each_sec(file, sec) {
428 struct instruction *insns = NU 378 struct instruction *insns = NULL;
429 u8 prev_len = 0; 379 u8 prev_len = 0;
430 u8 idx = 0; 380 u8 idx = 0;
431 381
432 if (!(sec->sh.sh_flags & SHF_E 382 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
433 continue; 383 continue;
434 384
435 if (strcmp(sec->name, ".altins 385 if (strcmp(sec->name, ".altinstr_replacement") &&
436 strcmp(sec->name, ".altins 386 strcmp(sec->name, ".altinstr_aux") &&
437 strncmp(sec->name, ".disca 387 strncmp(sec->name, ".discard.", 9))
438 sec->text = true; 388 sec->text = true;
439 389
440 if (!strcmp(sec->name, ".noins 390 if (!strcmp(sec->name, ".noinstr.text") ||
441 !strcmp(sec->name, ".entry 391 !strcmp(sec->name, ".entry.text") ||
442 !strcmp(sec->name, ".cpuid 392 !strcmp(sec->name, ".cpuidle.text") ||
443 !strncmp(sec->name, ".text 393 !strncmp(sec->name, ".text..__x86.", 13))
444 sec->noinstr = true; 394 sec->noinstr = true;
445 395
446 /* 396 /*
447 * .init.text code is ran befo 397 * .init.text code is ran before userspace and thus doesn't
448 * strictly need retpolines, e 398 * strictly need retpolines, except for modules which are
449 * loaded late, they very much 399 * loaded late, they very much do need retpoline in their
450 * .init.text 400 * .init.text
451 */ 401 */
452 if (!strcmp(sec->name, ".init. 402 if (!strcmp(sec->name, ".init.text") && !opts.module)
453 sec->init = true; 403 sec->init = true;
454 404
455 for (offset = 0; offset < sec- 405 for (offset = 0; offset < sec->sh.sh_size; offset += insn->len) {
456 if (!insns || idx == I 406 if (!insns || idx == INSN_CHUNK_MAX) {
457 insns = calloc 407 insns = calloc(sizeof(*insn), INSN_CHUNK_SIZE);
458 if (!insns) { 408 if (!insns) {
459 WARN(" 409 WARN("malloc failed");
460 return 410 return -1;
461 } 411 }
462 idx = 0; 412 idx = 0;
463 } else { 413 } else {
464 idx++; 414 idx++;
465 } 415 }
466 insn = &insns[idx]; 416 insn = &insns[idx];
467 insn->idx = idx; 417 insn->idx = idx;
468 418
469 INIT_LIST_HEAD(&insn-> 419 INIT_LIST_HEAD(&insn->call_node);
470 insn->sec = sec; 420 insn->sec = sec;
471 insn->offset = offset; 421 insn->offset = offset;
472 insn->prev_len = prev_ 422 insn->prev_len = prev_len;
473 423
474 ret = arch_decode_inst 424 ret = arch_decode_instruction(file, sec, offset,
475 425 sec->sh.sh_size - offset,
476 426 insn);
477 if (ret) 427 if (ret)
478 return ret; 428 return ret;
479 429
480 prev_len = insn->len; 430 prev_len = insn->len;
481 431
482 /* 432 /*
483 * By default, "ud2" i 433 * By default, "ud2" is a dead end unless otherwise
484 * annotated, because 434 * annotated, because GCC 7 inserts it for certain
485 * divide-by-zero case 435 * divide-by-zero cases.
486 */ 436 */
487 if (insn->type == INSN 437 if (insn->type == INSN_BUG)
488 insn->dead_end 438 insn->dead_end = true;
489 439
490 hash_add(file->insn_ha 440 hash_add(file->insn_hash, &insn->hash, sec_offset_hash(sec, insn->offset));
491 nr_insns++; 441 nr_insns++;
492 } 442 }
493 443
494 // printf("%s: last chunk used: % 444 // printf("%s: last chunk used: %d\n", sec->name, (int)idx);
495 445
496 sec_for_each_sym(sec, func) { 446 sec_for_each_sym(sec, func) {
497 if (func->type != STT_ 447 if (func->type != STT_NOTYPE && func->type != STT_FUNC)
498 continue; 448 continue;
499 449
500 if (func->offset == se 450 if (func->offset == sec->sh.sh_size) {
501 /* Heuristic: 451 /* Heuristic: likely an "end" symbol */
502 if (func->type 452 if (func->type == STT_NOTYPE)
503 contin 453 continue;
504 WARN("%s(): ST 454 WARN("%s(): STT_FUNC at end of section",
505 func->nam 455 func->name);
506 return -1; 456 return -1;
507 } 457 }
508 458
509 if (func->embedded_ins 459 if (func->embedded_insn || func->alias != func)
510 continue; 460 continue;
511 461
512 if (!find_insn(file, s 462 if (!find_insn(file, sec, func->offset)) {
513 WARN("%s(): ca 463 WARN("%s(): can't find starting instruction",
514 func->nam 464 func->name);
515 return -1; 465 return -1;
516 } 466 }
517 467
518 sym_for_each_insn(file 468 sym_for_each_insn(file, func, insn) {
519 insn->sym = fu 469 insn->sym = func;
520 if (func->type 470 if (func->type == STT_FUNC &&
521 insn->type 471 insn->type == INSN_ENDBR &&
522 list_empty 472 list_empty(&insn->call_node)) {
523 if (in 473 if (insn->offset == func->offset) {
524 474 list_add_tail(&insn->call_node, &file->endbr_list);
525 475 file->nr_endbr++;
526 } else 476 } else {
527 477 file->nr_endbr_int++;
528 } 478 }
529 } 479 }
530 } 480 }
531 } 481 }
532 } 482 }
533 483
534 if (opts.stats) 484 if (opts.stats)
535 printf("nr_insns: %lu\n", nr_i 485 printf("nr_insns: %lu\n", nr_insns);
536 486
537 return 0; 487 return 0;
538 } 488 }
539 489
540 /* 490 /*
541 * Read the pv_ops[] .data table to find the s 491 * Read the pv_ops[] .data table to find the static initialized values.
542 */ 492 */
543 static int add_pv_ops(struct objtool_file *fil 493 static int add_pv_ops(struct objtool_file *file, const char *symname)
544 { 494 {
545 struct symbol *sym, *func; 495 struct symbol *sym, *func;
546 unsigned long off, end; 496 unsigned long off, end;
547 struct reloc *reloc; 497 struct reloc *reloc;
548 int idx; 498 int idx;
549 499
550 sym = find_symbol_by_name(file->elf, s 500 sym = find_symbol_by_name(file->elf, symname);
551 if (!sym) 501 if (!sym)
552 return 0; 502 return 0;
553 503
554 off = sym->offset; 504 off = sym->offset;
555 end = off + sym->len; 505 end = off + sym->len;
556 for (;;) { 506 for (;;) {
557 reloc = find_reloc_by_dest_ran 507 reloc = find_reloc_by_dest_range(file->elf, sym->sec, off, end - off);
558 if (!reloc) 508 if (!reloc)
559 break; 509 break;
560 510
561 func = reloc->sym; 511 func = reloc->sym;
562 if (func->type == STT_SECTION) 512 if (func->type == STT_SECTION)
563 func = find_symbol_by_ 513 func = find_symbol_by_offset(reloc->sym->sec,
564 514 reloc_addend(reloc));
565 515
566 idx = (reloc_offset(reloc) - s 516 idx = (reloc_offset(reloc) - sym->offset) / sizeof(unsigned long);
567 517
568 objtool_pv_add(file, idx, func 518 objtool_pv_add(file, idx, func);
569 519
570 off = reloc_offset(reloc) + 1; 520 off = reloc_offset(reloc) + 1;
571 if (off > end) 521 if (off > end)
572 break; 522 break;
573 } 523 }
574 524
575 return 0; 525 return 0;
576 } 526 }
577 527
578 /* 528 /*
579 * Allocate and initialize file->pv_ops[]. 529 * Allocate and initialize file->pv_ops[].
580 */ 530 */
581 static int init_pv_ops(struct objtool_file *fi 531 static int init_pv_ops(struct objtool_file *file)
582 { 532 {
583 static const char *pv_ops_tables[] = { 533 static const char *pv_ops_tables[] = {
584 "pv_ops", 534 "pv_ops",
585 "xen_cpu_ops", 535 "xen_cpu_ops",
586 "xen_irq_ops", 536 "xen_irq_ops",
587 "xen_mmu_ops", 537 "xen_mmu_ops",
588 NULL, 538 NULL,
589 }; 539 };
590 const char *pv_ops; 540 const char *pv_ops;
591 struct symbol *sym; 541 struct symbol *sym;
592 int idx, nr; 542 int idx, nr;
593 543
594 if (!opts.noinstr) 544 if (!opts.noinstr)
595 return 0; 545 return 0;
596 546
597 file->pv_ops = NULL; 547 file->pv_ops = NULL;
598 548
599 sym = find_symbol_by_name(file->elf, " 549 sym = find_symbol_by_name(file->elf, "pv_ops");
600 if (!sym) 550 if (!sym)
601 return 0; 551 return 0;
602 552
603 nr = sym->len / sizeof(unsigned long); 553 nr = sym->len / sizeof(unsigned long);
604 file->pv_ops = calloc(sizeof(struct pv 554 file->pv_ops = calloc(sizeof(struct pv_state), nr);
605 if (!file->pv_ops) 555 if (!file->pv_ops)
606 return -1; 556 return -1;
607 557
608 for (idx = 0; idx < nr; idx++) 558 for (idx = 0; idx < nr; idx++)
609 INIT_LIST_HEAD(&file->pv_ops[i 559 INIT_LIST_HEAD(&file->pv_ops[idx].targets);
610 560
611 for (idx = 0; (pv_ops = pv_ops_tables[ 561 for (idx = 0; (pv_ops = pv_ops_tables[idx]); idx++)
612 add_pv_ops(file, pv_ops); 562 add_pv_ops(file, pv_ops);
613 563
614 return 0; 564 return 0;
615 } 565 }
616 566
617 static struct instruction *find_last_insn(stru 567 static struct instruction *find_last_insn(struct objtool_file *file,
618 stru 568 struct section *sec)
619 { 569 {
620 struct instruction *insn = NULL; 570 struct instruction *insn = NULL;
621 unsigned int offset; 571 unsigned int offset;
622 unsigned int end = (sec->sh.sh_size > 572 unsigned int end = (sec->sh.sh_size > 10) ? sec->sh.sh_size - 10 : 0;
623 573
624 for (offset = sec->sh.sh_size - 1; off 574 for (offset = sec->sh.sh_size - 1; offset >= end && !insn; offset--)
625 insn = find_insn(file, sec, of 575 insn = find_insn(file, sec, offset);
626 576
627 return insn; 577 return insn;
628 } 578 }
629 579
630 /* 580 /*
631 * Mark "ud2" instructions and manually annota 581 * Mark "ud2" instructions and manually annotated dead ends.
632 */ 582 */
633 static int add_dead_ends(struct objtool_file * 583 static int add_dead_ends(struct objtool_file *file)
634 { 584 {
635 struct section *rsec; 585 struct section *rsec;
636 struct reloc *reloc; 586 struct reloc *reloc;
637 struct instruction *insn; 587 struct instruction *insn;
638 uint64_t offset; 588 uint64_t offset;
639 589
640 /* 590 /*
641 * Check for manually annotated dead e 591 * Check for manually annotated dead ends.
642 */ 592 */
643 rsec = find_section_by_name(file->elf, 593 rsec = find_section_by_name(file->elf, ".rela.discard.unreachable");
644 if (!rsec) 594 if (!rsec)
645 goto reachable; 595 goto reachable;
646 596
647 for_each_reloc(rsec, reloc) { 597 for_each_reloc(rsec, reloc) {
648 if (reloc->sym->type == STT_SE 598 if (reloc->sym->type == STT_SECTION) {
649 offset = reloc_addend( 599 offset = reloc_addend(reloc);
650 } else if (reloc->sym->local_l 600 } else if (reloc->sym->local_label) {
651 offset = reloc->sym->o 601 offset = reloc->sym->offset;
652 } else { 602 } else {
653 WARN("unexpected reloc 603 WARN("unexpected relocation symbol type in %s", rsec->name);
654 return -1; 604 return -1;
655 } 605 }
656 606
657 insn = find_insn(file, reloc-> 607 insn = find_insn(file, reloc->sym->sec, offset);
658 if (insn) 608 if (insn)
659 insn = prev_insn_same_ 609 insn = prev_insn_same_sec(file, insn);
660 else if (offset == reloc->sym- 610 else if (offset == reloc->sym->sec->sh.sh_size) {
661 insn = find_last_insn( 611 insn = find_last_insn(file, reloc->sym->sec);
662 if (!insn) { 612 if (!insn) {
663 WARN("can't fi 613 WARN("can't find unreachable insn at %s+0x%" PRIx64,
664 reloc->sy 614 reloc->sym->sec->name, offset);
665 return -1; 615 return -1;
666 } 616 }
667 } else { 617 } else {
668 WARN("can't find unrea 618 WARN("can't find unreachable insn at %s+0x%" PRIx64,
669 reloc->sym->sec-> 619 reloc->sym->sec->name, offset);
670 return -1; 620 return -1;
671 } 621 }
672 622
673 insn->dead_end = true; 623 insn->dead_end = true;
674 } 624 }
675 625
676 reachable: 626 reachable:
677 /* 627 /*
678 * These manually annotated reachable 628 * These manually annotated reachable checks are needed for GCC 4.4,
679 * where the Linux unreachable() macro 629 * where the Linux unreachable() macro isn't supported. In that case
680 * GCC doesn't know the "ud2" is fatal 630 * GCC doesn't know the "ud2" is fatal, so it generates code as if it's
681 * not a dead end. 631 * not a dead end.
682 */ 632 */
683 rsec = find_section_by_name(file->elf, 633 rsec = find_section_by_name(file->elf, ".rela.discard.reachable");
684 if (!rsec) 634 if (!rsec)
685 return 0; 635 return 0;
686 636
687 for_each_reloc(rsec, reloc) { 637 for_each_reloc(rsec, reloc) {
688 if (reloc->sym->type == STT_SE 638 if (reloc->sym->type == STT_SECTION) {
689 offset = reloc_addend( 639 offset = reloc_addend(reloc);
690 } else if (reloc->sym->local_l 640 } else if (reloc->sym->local_label) {
691 offset = reloc->sym->o 641 offset = reloc->sym->offset;
692 } else { 642 } else {
693 WARN("unexpected reloc 643 WARN("unexpected relocation symbol type in %s", rsec->name);
694 return -1; 644 return -1;
695 } 645 }
696 646
697 insn = find_insn(file, reloc-> 647 insn = find_insn(file, reloc->sym->sec, offset);
698 if (insn) 648 if (insn)
699 insn = prev_insn_same_ 649 insn = prev_insn_same_sec(file, insn);
700 else if (offset == reloc->sym- 650 else if (offset == reloc->sym->sec->sh.sh_size) {
701 insn = find_last_insn( 651 insn = find_last_insn(file, reloc->sym->sec);
702 if (!insn) { 652 if (!insn) {
703 WARN("can't fi 653 WARN("can't find reachable insn at %s+0x%" PRIx64,
704 reloc->sy 654 reloc->sym->sec->name, offset);
705 return -1; 655 return -1;
706 } 656 }
707 } else { 657 } else {
708 WARN("can't find reach 658 WARN("can't find reachable insn at %s+0x%" PRIx64,
709 reloc->sym->sec-> 659 reloc->sym->sec->name, offset);
710 return -1; 660 return -1;
711 } 661 }
712 662
713 insn->dead_end = false; 663 insn->dead_end = false;
714 } 664 }
715 665
716 return 0; 666 return 0;
717 } 667 }
718 668
719 static int create_static_call_sections(struct 669 static int create_static_call_sections(struct objtool_file *file)
720 { 670 {
721 struct static_call_site *site; 671 struct static_call_site *site;
722 struct section *sec; 672 struct section *sec;
723 struct instruction *insn; 673 struct instruction *insn;
724 struct symbol *key_sym; 674 struct symbol *key_sym;
725 char *key_name, *tmp; 675 char *key_name, *tmp;
726 int idx; 676 int idx;
727 677
728 sec = find_section_by_name(file->elf, 678 sec = find_section_by_name(file->elf, ".static_call_sites");
729 if (sec) { 679 if (sec) {
730 INIT_LIST_HEAD(&file->static_c 680 INIT_LIST_HEAD(&file->static_call_list);
731 WARN("file already has .static 681 WARN("file already has .static_call_sites section, skipping");
732 return 0; 682 return 0;
733 } 683 }
734 684
735 if (list_empty(&file->static_call_list 685 if (list_empty(&file->static_call_list))
736 return 0; 686 return 0;
737 687
738 idx = 0; 688 idx = 0;
739 list_for_each_entry(insn, &file->stati 689 list_for_each_entry(insn, &file->static_call_list, call_node)
740 idx++; 690 idx++;
741 691
742 sec = elf_create_section_pair(file->el 692 sec = elf_create_section_pair(file->elf, ".static_call_sites",
743 sizeof(* 693 sizeof(*site), idx, idx * 2);
744 if (!sec) 694 if (!sec)
745 return -1; 695 return -1;
746 696
747 /* Allow modules to modify the low bit 697 /* Allow modules to modify the low bits of static_call_site::key */
748 sec->sh.sh_flags |= SHF_WRITE; 698 sec->sh.sh_flags |= SHF_WRITE;
749 699
750 idx = 0; 700 idx = 0;
751 list_for_each_entry(insn, &file->stati 701 list_for_each_entry(insn, &file->static_call_list, call_node) {
752 702
753 /* populate reloc for 'addr' * 703 /* populate reloc for 'addr' */
754 if (!elf_init_reloc_text_sym(f 704 if (!elf_init_reloc_text_sym(file->elf, sec,
755 i 705 idx * sizeof(*site), idx * 2,
756 i 706 insn->sec, insn->offset))
757 return -1; 707 return -1;
758 708
759 /* find key symbol */ 709 /* find key symbol */
760 key_name = strdup(insn_call_de 710 key_name = strdup(insn_call_dest(insn)->name);
761 if (!key_name) { 711 if (!key_name) {
762 perror("strdup"); 712 perror("strdup");
763 return -1; 713 return -1;
764 } 714 }
765 if (strncmp(key_name, STATIC_C 715 if (strncmp(key_name, STATIC_CALL_TRAMP_PREFIX_STR,
766 STATIC_CALL_TRAMP_ 716 STATIC_CALL_TRAMP_PREFIX_LEN)) {
767 WARN("static_call: tra 717 WARN("static_call: trampoline name malformed: %s", key_name);
768 free(key_name); 718 free(key_name);
769 return -1; 719 return -1;
770 } 720 }
771 tmp = key_name + STATIC_CALL_T 721 tmp = key_name + STATIC_CALL_TRAMP_PREFIX_LEN - STATIC_CALL_KEY_PREFIX_LEN;
772 memcpy(tmp, STATIC_CALL_KEY_PR 722 memcpy(tmp, STATIC_CALL_KEY_PREFIX_STR, STATIC_CALL_KEY_PREFIX_LEN);
773 723
774 key_sym = find_symbol_by_name( 724 key_sym = find_symbol_by_name(file->elf, tmp);
775 if (!key_sym) { 725 if (!key_sym) {
776 if (!opts.module) { 726 if (!opts.module) {
777 WARN("static_c 727 WARN("static_call: can't find static_call_key symbol: %s", tmp);
778 free(key_name) 728 free(key_name);
779 return -1; 729 return -1;
780 } 730 }
781 731
782 /* 732 /*
783 * For modules(), the 733 * For modules(), the key might not be exported, which
784 * means the module ca 734 * means the module can make static calls but isn't
785 * allowed to change t 735 * allowed to change them.
786 * 736 *
787 * In that case we tem 737 * In that case we temporarily set the key to be the
788 * trampoline address. 738 * trampoline address. This is fixed up in
789 * static_call_add_mod 739 * static_call_add_module().
790 */ 740 */
791 key_sym = insn_call_de 741 key_sym = insn_call_dest(insn);
792 } 742 }
793 free(key_name); 743 free(key_name);
794 744
795 /* populate reloc for 'key' */ 745 /* populate reloc for 'key' */
796 if (!elf_init_reloc_data_sym(f 746 if (!elf_init_reloc_data_sym(file->elf, sec,
797 i 747 idx * sizeof(*site) + 4,
798 ( 748 (idx * 2) + 1, key_sym,
799 i 749 is_sibling_call(insn) * STATIC_CALL_SITE_TAIL))
800 return -1; 750 return -1;
801 751
802 idx++; 752 idx++;
803 } 753 }
804 754
805 return 0; 755 return 0;
806 } 756 }
807 757
808 static int create_retpoline_sites_sections(str 758 static int create_retpoline_sites_sections(struct objtool_file *file)
809 { 759 {
810 struct instruction *insn; 760 struct instruction *insn;
811 struct section *sec; 761 struct section *sec;
812 int idx; 762 int idx;
813 763
814 sec = find_section_by_name(file->elf, 764 sec = find_section_by_name(file->elf, ".retpoline_sites");
815 if (sec) { 765 if (sec) {
816 WARN("file already has .retpol 766 WARN("file already has .retpoline_sites, skipping");
817 return 0; 767 return 0;
818 } 768 }
819 769
820 idx = 0; 770 idx = 0;
821 list_for_each_entry(insn, &file->retpo 771 list_for_each_entry(insn, &file->retpoline_call_list, call_node)
822 idx++; 772 idx++;
823 773
824 if (!idx) 774 if (!idx)
825 return 0; 775 return 0;
826 776
827 sec = elf_create_section_pair(file->el 777 sec = elf_create_section_pair(file->elf, ".retpoline_sites",
828 sizeof(i 778 sizeof(int), idx, idx);
829 if (!sec) 779 if (!sec)
830 return -1; 780 return -1;
831 781
832 idx = 0; 782 idx = 0;
833 list_for_each_entry(insn, &file->retpo 783 list_for_each_entry(insn, &file->retpoline_call_list, call_node) {
834 784
835 if (!elf_init_reloc_text_sym(f 785 if (!elf_init_reloc_text_sym(file->elf, sec,
836 i 786 idx * sizeof(int), idx,
837 i 787 insn->sec, insn->offset))
838 return -1; 788 return -1;
839 789
840 idx++; 790 idx++;
841 } 791 }
842 792
843 return 0; 793 return 0;
844 } 794 }
845 795
846 static int create_return_sites_sections(struct 796 static int create_return_sites_sections(struct objtool_file *file)
847 { 797 {
848 struct instruction *insn; 798 struct instruction *insn;
849 struct section *sec; 799 struct section *sec;
850 int idx; 800 int idx;
851 801
852 sec = find_section_by_name(file->elf, 802 sec = find_section_by_name(file->elf, ".return_sites");
853 if (sec) { 803 if (sec) {
854 WARN("file already has .return 804 WARN("file already has .return_sites, skipping");
855 return 0; 805 return 0;
856 } 806 }
857 807
858 idx = 0; 808 idx = 0;
859 list_for_each_entry(insn, &file->retur 809 list_for_each_entry(insn, &file->return_thunk_list, call_node)
860 idx++; 810 idx++;
861 811
862 if (!idx) 812 if (!idx)
863 return 0; 813 return 0;
864 814
865 sec = elf_create_section_pair(file->el 815 sec = elf_create_section_pair(file->elf, ".return_sites",
866 sizeof(i 816 sizeof(int), idx, idx);
867 if (!sec) 817 if (!sec)
868 return -1; 818 return -1;
869 819
870 idx = 0; 820 idx = 0;
871 list_for_each_entry(insn, &file->retur 821 list_for_each_entry(insn, &file->return_thunk_list, call_node) {
872 822
873 if (!elf_init_reloc_text_sym(f 823 if (!elf_init_reloc_text_sym(file->elf, sec,
874 i 824 idx * sizeof(int), idx,
875 i 825 insn->sec, insn->offset))
876 return -1; 826 return -1;
877 827
878 idx++; 828 idx++;
879 } 829 }
880 830
881 return 0; 831 return 0;
882 } 832 }
883 833
884 static int create_ibt_endbr_seal_sections(stru 834 static int create_ibt_endbr_seal_sections(struct objtool_file *file)
885 { 835 {
886 struct instruction *insn; 836 struct instruction *insn;
887 struct section *sec; 837 struct section *sec;
888 int idx; 838 int idx;
889 839
890 sec = find_section_by_name(file->elf, 840 sec = find_section_by_name(file->elf, ".ibt_endbr_seal");
891 if (sec) { 841 if (sec) {
892 WARN("file already has .ibt_en 842 WARN("file already has .ibt_endbr_seal, skipping");
893 return 0; 843 return 0;
894 } 844 }
895 845
896 idx = 0; 846 idx = 0;
897 list_for_each_entry(insn, &file->endbr 847 list_for_each_entry(insn, &file->endbr_list, call_node)
898 idx++; 848 idx++;
899 849
900 if (opts.stats) { 850 if (opts.stats) {
901 printf("ibt: ENDBR at function 851 printf("ibt: ENDBR at function start: %d\n", file->nr_endbr);
902 printf("ibt: ENDBR inside func 852 printf("ibt: ENDBR inside functions: %d\n", file->nr_endbr_int);
903 printf("ibt: superfluous ENDBR 853 printf("ibt: superfluous ENDBR: %d\n", idx);
904 } 854 }
905 855
906 if (!idx) 856 if (!idx)
907 return 0; 857 return 0;
908 858
909 sec = elf_create_section_pair(file->el 859 sec = elf_create_section_pair(file->elf, ".ibt_endbr_seal",
910 sizeof(i 860 sizeof(int), idx, idx);
911 if (!sec) 861 if (!sec)
912 return -1; 862 return -1;
913 863
914 idx = 0; 864 idx = 0;
915 list_for_each_entry(insn, &file->endbr 865 list_for_each_entry(insn, &file->endbr_list, call_node) {
916 866
917 int *site = (int *)sec->data-> 867 int *site = (int *)sec->data->d_buf + idx;
918 struct symbol *sym = insn->sym 868 struct symbol *sym = insn->sym;
919 *site = 0; 869 *site = 0;
920 870
921 if (opts.module && sym && sym- 871 if (opts.module && sym && sym->type == STT_FUNC &&
922 insn->offset == sym->offse 872 insn->offset == sym->offset &&
923 (!strcmp(sym->name, "init_ 873 (!strcmp(sym->name, "init_module") ||
924 !strcmp(sym->name, "clean 874 !strcmp(sym->name, "cleanup_module")))
925 WARN("%s(): not an ind 875 WARN("%s(): not an indirect call target", sym->name);
926 876
927 if (!elf_init_reloc_text_sym(f 877 if (!elf_init_reloc_text_sym(file->elf, sec,
928 i 878 idx * sizeof(int), idx,
929 i 879 insn->sec, insn->offset))
930 return -1; 880 return -1;
931 881
932 idx++; 882 idx++;
933 } 883 }
934 884
935 return 0; 885 return 0;
936 } 886 }
937 887
938 static int create_cfi_sections(struct objtool_ 888 static int create_cfi_sections(struct objtool_file *file)
939 { 889 {
940 struct section *sec; 890 struct section *sec;
941 struct symbol *sym; 891 struct symbol *sym;
942 int idx; 892 int idx;
943 893
944 sec = find_section_by_name(file->elf, 894 sec = find_section_by_name(file->elf, ".cfi_sites");
945 if (sec) { 895 if (sec) {
946 INIT_LIST_HEAD(&file->call_lis 896 INIT_LIST_HEAD(&file->call_list);
947 WARN("file already has .cfi_si 897 WARN("file already has .cfi_sites section, skipping");
948 return 0; 898 return 0;
949 } 899 }
950 900
951 idx = 0; 901 idx = 0;
952 for_each_sym(file, sym) { 902 for_each_sym(file, sym) {
953 if (sym->type != STT_FUNC) 903 if (sym->type != STT_FUNC)
954 continue; 904 continue;
955 905
956 if (strncmp(sym->name, "__cfi_ 906 if (strncmp(sym->name, "__cfi_", 6))
957 continue; 907 continue;
958 908
959 idx++; 909 idx++;
960 } 910 }
961 911
962 sec = elf_create_section_pair(file->el 912 sec = elf_create_section_pair(file->elf, ".cfi_sites",
963 sizeof(u 913 sizeof(unsigned int), idx, idx);
964 if (!sec) 914 if (!sec)
965 return -1; 915 return -1;
966 916
967 idx = 0; 917 idx = 0;
968 for_each_sym(file, sym) { 918 for_each_sym(file, sym) {
969 if (sym->type != STT_FUNC) 919 if (sym->type != STT_FUNC)
970 continue; 920 continue;
971 921
972 if (strncmp(sym->name, "__cfi_ 922 if (strncmp(sym->name, "__cfi_", 6))
973 continue; 923 continue;
974 924
975 if (!elf_init_reloc_text_sym(f 925 if (!elf_init_reloc_text_sym(file->elf, sec,
976 i 926 idx * sizeof(unsigned int), idx,
977 s 927 sym->sec, sym->offset))
978 return -1; 928 return -1;
979 929
980 idx++; 930 idx++;
981 } 931 }
982 932
983 return 0; 933 return 0;
984 } 934 }
985 935
986 static int create_mcount_loc_sections(struct o 936 static int create_mcount_loc_sections(struct objtool_file *file)
987 { 937 {
988 size_t addr_size = elf_addr_size(file- 938 size_t addr_size = elf_addr_size(file->elf);
989 struct instruction *insn; 939 struct instruction *insn;
990 struct section *sec; 940 struct section *sec;
991 int idx; 941 int idx;
992 942
993 sec = find_section_by_name(file->elf, 943 sec = find_section_by_name(file->elf, "__mcount_loc");
994 if (sec) { 944 if (sec) {
995 INIT_LIST_HEAD(&file->mcount_l 945 INIT_LIST_HEAD(&file->mcount_loc_list);
996 WARN("file already has __mcoun 946 WARN("file already has __mcount_loc section, skipping");
997 return 0; 947 return 0;
998 } 948 }
999 949
1000 if (list_empty(&file->mcount_loc_list 950 if (list_empty(&file->mcount_loc_list))
1001 return 0; 951 return 0;
1002 952
1003 idx = 0; 953 idx = 0;
1004 list_for_each_entry(insn, &file->mcou 954 list_for_each_entry(insn, &file->mcount_loc_list, call_node)
1005 idx++; 955 idx++;
1006 956
1007 sec = elf_create_section_pair(file->e 957 sec = elf_create_section_pair(file->elf, "__mcount_loc", addr_size,
1008 idx, id 958 idx, idx);
1009 if (!sec) 959 if (!sec)
1010 return -1; 960 return -1;
1011 961
1012 sec->sh.sh_addralign = addr_size; 962 sec->sh.sh_addralign = addr_size;
1013 963
1014 idx = 0; 964 idx = 0;
1015 list_for_each_entry(insn, &file->mcou 965 list_for_each_entry(insn, &file->mcount_loc_list, call_node) {
1016 966
1017 struct reloc *reloc; 967 struct reloc *reloc;
1018 968
1019 reloc = elf_init_reloc_text_s 969 reloc = elf_init_reloc_text_sym(file->elf, sec, idx * addr_size, idx,
1020 970 insn->sec, insn->offset);
1021 if (!reloc) 971 if (!reloc)
1022 return -1; 972 return -1;
1023 973
1024 set_reloc_type(file->elf, rel 974 set_reloc_type(file->elf, reloc, addr_size == 8 ? R_ABS64 : R_ABS32);
1025 975
1026 idx++; 976 idx++;
1027 } 977 }
1028 978
1029 return 0; 979 return 0;
1030 } 980 }
1031 981
1032 static int create_direct_call_sections(struct 982 static int create_direct_call_sections(struct objtool_file *file)
1033 { 983 {
1034 struct instruction *insn; 984 struct instruction *insn;
1035 struct section *sec; 985 struct section *sec;
1036 int idx; 986 int idx;
1037 987
1038 sec = find_section_by_name(file->elf, 988 sec = find_section_by_name(file->elf, ".call_sites");
1039 if (sec) { 989 if (sec) {
1040 INIT_LIST_HEAD(&file->call_li 990 INIT_LIST_HEAD(&file->call_list);
1041 WARN("file already has .call_ 991 WARN("file already has .call_sites section, skipping");
1042 return 0; 992 return 0;
1043 } 993 }
1044 994
1045 if (list_empty(&file->call_list)) 995 if (list_empty(&file->call_list))
1046 return 0; 996 return 0;
1047 997
1048 idx = 0; 998 idx = 0;
1049 list_for_each_entry(insn, &file->call 999 list_for_each_entry(insn, &file->call_list, call_node)
1050 idx++; 1000 idx++;
1051 1001
1052 sec = elf_create_section_pair(file->e 1002 sec = elf_create_section_pair(file->elf, ".call_sites",
1053 sizeof( 1003 sizeof(unsigned int), idx, idx);
1054 if (!sec) 1004 if (!sec)
1055 return -1; 1005 return -1;
1056 1006
1057 idx = 0; 1007 idx = 0;
1058 list_for_each_entry(insn, &file->call 1008 list_for_each_entry(insn, &file->call_list, call_node) {
1059 1009
1060 if (!elf_init_reloc_text_sym( 1010 if (!elf_init_reloc_text_sym(file->elf, sec,
1061 1011 idx * sizeof(unsigned int), idx,
1062 1012 insn->sec, insn->offset))
1063 return -1; 1013 return -1;
1064 1014
1065 idx++; 1015 idx++;
1066 } 1016 }
1067 1017
1068 return 0; 1018 return 0;
1069 } 1019 }
1070 1020
1071 /* 1021 /*
1072 * Warnings shouldn't be reported for ignored 1022 * Warnings shouldn't be reported for ignored functions.
1073 */ 1023 */
1074 static void add_ignores(struct objtool_file * 1024 static void add_ignores(struct objtool_file *file)
1075 { 1025 {
1076 struct instruction *insn; 1026 struct instruction *insn;
1077 struct section *rsec; 1027 struct section *rsec;
1078 struct symbol *func; 1028 struct symbol *func;
1079 struct reloc *reloc; 1029 struct reloc *reloc;
1080 1030
1081 rsec = find_section_by_name(file->elf 1031 rsec = find_section_by_name(file->elf, ".rela.discard.func_stack_frame_non_standard");
1082 if (!rsec) 1032 if (!rsec)
1083 return; 1033 return;
1084 1034
1085 for_each_reloc(rsec, reloc) { 1035 for_each_reloc(rsec, reloc) {
1086 switch (reloc->sym->type) { 1036 switch (reloc->sym->type) {
1087 case STT_FUNC: 1037 case STT_FUNC:
1088 func = reloc->sym; 1038 func = reloc->sym;
1089 break; 1039 break;
1090 1040
1091 case STT_SECTION: 1041 case STT_SECTION:
1092 func = find_func_by_o 1042 func = find_func_by_offset(reloc->sym->sec, reloc_addend(reloc));
1093 if (!func) 1043 if (!func)
1094 continue; 1044 continue;
1095 break; 1045 break;
1096 1046
1097 default: 1047 default:
1098 WARN("unexpected relo 1048 WARN("unexpected relocation symbol type in %s: %d",
1099 rsec->name, relo 1049 rsec->name, reloc->sym->type);
1100 continue; 1050 continue;
1101 } 1051 }
1102 1052
1103 func_for_each_insn(file, func 1053 func_for_each_insn(file, func, insn)
1104 insn->ignore = true; 1054 insn->ignore = true;
1105 } 1055 }
1106 } 1056 }
1107 1057
1108 /* 1058 /*
1109 * This is a whitelist of functions that is a 1059 * This is a whitelist of functions that is allowed to be called with AC set.
1110 * The list is meant to be minimal and only c 1060 * The list is meant to be minimal and only contains compiler instrumentation
1111 * ABI and a few functions used to implement 1061 * ABI and a few functions used to implement *_{to,from}_user() functions.
1112 * 1062 *
1113 * These functions must not directly change A 1063 * These functions must not directly change AC, but may PUSHF/POPF.
1114 */ 1064 */
1115 static const char *uaccess_safe_builtin[] = { 1065 static const char *uaccess_safe_builtin[] = {
1116 /* KASAN */ 1066 /* KASAN */
1117 "kasan_report", 1067 "kasan_report",
1118 "kasan_check_range", 1068 "kasan_check_range",
1119 /* KASAN out-of-line */ 1069 /* KASAN out-of-line */
1120 "__asan_loadN_noabort", 1070 "__asan_loadN_noabort",
1121 "__asan_load1_noabort", 1071 "__asan_load1_noabort",
1122 "__asan_load2_noabort", 1072 "__asan_load2_noabort",
1123 "__asan_load4_noabort", 1073 "__asan_load4_noabort",
1124 "__asan_load8_noabort", 1074 "__asan_load8_noabort",
1125 "__asan_load16_noabort", 1075 "__asan_load16_noabort",
1126 "__asan_storeN_noabort", 1076 "__asan_storeN_noabort",
1127 "__asan_store1_noabort", 1077 "__asan_store1_noabort",
1128 "__asan_store2_noabort", 1078 "__asan_store2_noabort",
1129 "__asan_store4_noabort", 1079 "__asan_store4_noabort",
1130 "__asan_store8_noabort", 1080 "__asan_store8_noabort",
1131 "__asan_store16_noabort", 1081 "__asan_store16_noabort",
1132 "__kasan_check_read", 1082 "__kasan_check_read",
1133 "__kasan_check_write", 1083 "__kasan_check_write",
1134 /* KASAN in-line */ 1084 /* KASAN in-line */
1135 "__asan_report_load_n_noabort", 1085 "__asan_report_load_n_noabort",
1136 "__asan_report_load1_noabort", 1086 "__asan_report_load1_noabort",
1137 "__asan_report_load2_noabort", 1087 "__asan_report_load2_noabort",
1138 "__asan_report_load4_noabort", 1088 "__asan_report_load4_noabort",
1139 "__asan_report_load8_noabort", 1089 "__asan_report_load8_noabort",
1140 "__asan_report_load16_noabort", 1090 "__asan_report_load16_noabort",
1141 "__asan_report_store_n_noabort", 1091 "__asan_report_store_n_noabort",
1142 "__asan_report_store1_noabort", 1092 "__asan_report_store1_noabort",
1143 "__asan_report_store2_noabort", 1093 "__asan_report_store2_noabort",
1144 "__asan_report_store4_noabort", 1094 "__asan_report_store4_noabort",
1145 "__asan_report_store8_noabort", 1095 "__asan_report_store8_noabort",
1146 "__asan_report_store16_noabort", 1096 "__asan_report_store16_noabort",
1147 /* KCSAN */ 1097 /* KCSAN */
1148 "__kcsan_check_access", 1098 "__kcsan_check_access",
1149 "__kcsan_mb", 1099 "__kcsan_mb",
1150 "__kcsan_wmb", 1100 "__kcsan_wmb",
1151 "__kcsan_rmb", 1101 "__kcsan_rmb",
1152 "__kcsan_release", 1102 "__kcsan_release",
1153 "kcsan_found_watchpoint", 1103 "kcsan_found_watchpoint",
1154 "kcsan_setup_watchpoint", 1104 "kcsan_setup_watchpoint",
1155 "kcsan_check_scoped_accesses", 1105 "kcsan_check_scoped_accesses",
1156 "kcsan_disable_current", 1106 "kcsan_disable_current",
1157 "kcsan_enable_current_nowarn", 1107 "kcsan_enable_current_nowarn",
1158 /* KCSAN/TSAN */ 1108 /* KCSAN/TSAN */
1159 "__tsan_func_entry", 1109 "__tsan_func_entry",
1160 "__tsan_func_exit", 1110 "__tsan_func_exit",
1161 "__tsan_read_range", 1111 "__tsan_read_range",
1162 "__tsan_write_range", 1112 "__tsan_write_range",
1163 "__tsan_read1", 1113 "__tsan_read1",
1164 "__tsan_read2", 1114 "__tsan_read2",
1165 "__tsan_read4", 1115 "__tsan_read4",
1166 "__tsan_read8", 1116 "__tsan_read8",
1167 "__tsan_read16", 1117 "__tsan_read16",
1168 "__tsan_write1", 1118 "__tsan_write1",
1169 "__tsan_write2", 1119 "__tsan_write2",
1170 "__tsan_write4", 1120 "__tsan_write4",
1171 "__tsan_write8", 1121 "__tsan_write8",
1172 "__tsan_write16", 1122 "__tsan_write16",
1173 "__tsan_read_write1", 1123 "__tsan_read_write1",
1174 "__tsan_read_write2", 1124 "__tsan_read_write2",
1175 "__tsan_read_write4", 1125 "__tsan_read_write4",
1176 "__tsan_read_write8", 1126 "__tsan_read_write8",
1177 "__tsan_read_write16", 1127 "__tsan_read_write16",
1178 "__tsan_volatile_read1", 1128 "__tsan_volatile_read1",
1179 "__tsan_volatile_read2", 1129 "__tsan_volatile_read2",
1180 "__tsan_volatile_read4", 1130 "__tsan_volatile_read4",
1181 "__tsan_volatile_read8", 1131 "__tsan_volatile_read8",
1182 "__tsan_volatile_read16", 1132 "__tsan_volatile_read16",
1183 "__tsan_volatile_write1", 1133 "__tsan_volatile_write1",
1184 "__tsan_volatile_write2", 1134 "__tsan_volatile_write2",
1185 "__tsan_volatile_write4", 1135 "__tsan_volatile_write4",
1186 "__tsan_volatile_write8", 1136 "__tsan_volatile_write8",
1187 "__tsan_volatile_write16", 1137 "__tsan_volatile_write16",
1188 "__tsan_atomic8_load", 1138 "__tsan_atomic8_load",
1189 "__tsan_atomic16_load", 1139 "__tsan_atomic16_load",
1190 "__tsan_atomic32_load", 1140 "__tsan_atomic32_load",
1191 "__tsan_atomic64_load", 1141 "__tsan_atomic64_load",
1192 "__tsan_atomic8_store", 1142 "__tsan_atomic8_store",
1193 "__tsan_atomic16_store", 1143 "__tsan_atomic16_store",
1194 "__tsan_atomic32_store", 1144 "__tsan_atomic32_store",
1195 "__tsan_atomic64_store", 1145 "__tsan_atomic64_store",
1196 "__tsan_atomic8_exchange", 1146 "__tsan_atomic8_exchange",
1197 "__tsan_atomic16_exchange", 1147 "__tsan_atomic16_exchange",
1198 "__tsan_atomic32_exchange", 1148 "__tsan_atomic32_exchange",
1199 "__tsan_atomic64_exchange", 1149 "__tsan_atomic64_exchange",
1200 "__tsan_atomic8_fetch_add", 1150 "__tsan_atomic8_fetch_add",
1201 "__tsan_atomic16_fetch_add", 1151 "__tsan_atomic16_fetch_add",
1202 "__tsan_atomic32_fetch_add", 1152 "__tsan_atomic32_fetch_add",
1203 "__tsan_atomic64_fetch_add", 1153 "__tsan_atomic64_fetch_add",
1204 "__tsan_atomic8_fetch_sub", 1154 "__tsan_atomic8_fetch_sub",
1205 "__tsan_atomic16_fetch_sub", 1155 "__tsan_atomic16_fetch_sub",
1206 "__tsan_atomic32_fetch_sub", 1156 "__tsan_atomic32_fetch_sub",
1207 "__tsan_atomic64_fetch_sub", 1157 "__tsan_atomic64_fetch_sub",
1208 "__tsan_atomic8_fetch_and", 1158 "__tsan_atomic8_fetch_and",
1209 "__tsan_atomic16_fetch_and", 1159 "__tsan_atomic16_fetch_and",
1210 "__tsan_atomic32_fetch_and", 1160 "__tsan_atomic32_fetch_and",
1211 "__tsan_atomic64_fetch_and", 1161 "__tsan_atomic64_fetch_and",
1212 "__tsan_atomic8_fetch_or", 1162 "__tsan_atomic8_fetch_or",
1213 "__tsan_atomic16_fetch_or", 1163 "__tsan_atomic16_fetch_or",
1214 "__tsan_atomic32_fetch_or", 1164 "__tsan_atomic32_fetch_or",
1215 "__tsan_atomic64_fetch_or", 1165 "__tsan_atomic64_fetch_or",
1216 "__tsan_atomic8_fetch_xor", 1166 "__tsan_atomic8_fetch_xor",
1217 "__tsan_atomic16_fetch_xor", 1167 "__tsan_atomic16_fetch_xor",
1218 "__tsan_atomic32_fetch_xor", 1168 "__tsan_atomic32_fetch_xor",
1219 "__tsan_atomic64_fetch_xor", 1169 "__tsan_atomic64_fetch_xor",
1220 "__tsan_atomic8_fetch_nand", 1170 "__tsan_atomic8_fetch_nand",
1221 "__tsan_atomic16_fetch_nand", 1171 "__tsan_atomic16_fetch_nand",
1222 "__tsan_atomic32_fetch_nand", 1172 "__tsan_atomic32_fetch_nand",
1223 "__tsan_atomic64_fetch_nand", 1173 "__tsan_atomic64_fetch_nand",
1224 "__tsan_atomic8_compare_exchange_stro 1174 "__tsan_atomic8_compare_exchange_strong",
1225 "__tsan_atomic16_compare_exchange_str 1175 "__tsan_atomic16_compare_exchange_strong",
1226 "__tsan_atomic32_compare_exchange_str 1176 "__tsan_atomic32_compare_exchange_strong",
1227 "__tsan_atomic64_compare_exchange_str 1177 "__tsan_atomic64_compare_exchange_strong",
1228 "__tsan_atomic8_compare_exchange_weak 1178 "__tsan_atomic8_compare_exchange_weak",
1229 "__tsan_atomic16_compare_exchange_wea 1179 "__tsan_atomic16_compare_exchange_weak",
1230 "__tsan_atomic32_compare_exchange_wea 1180 "__tsan_atomic32_compare_exchange_weak",
1231 "__tsan_atomic64_compare_exchange_wea 1181 "__tsan_atomic64_compare_exchange_weak",
1232 "__tsan_atomic8_compare_exchange_val" 1182 "__tsan_atomic8_compare_exchange_val",
1233 "__tsan_atomic16_compare_exchange_val 1183 "__tsan_atomic16_compare_exchange_val",
1234 "__tsan_atomic32_compare_exchange_val 1184 "__tsan_atomic32_compare_exchange_val",
1235 "__tsan_atomic64_compare_exchange_val 1185 "__tsan_atomic64_compare_exchange_val",
1236 "__tsan_atomic_thread_fence", 1186 "__tsan_atomic_thread_fence",
1237 "__tsan_atomic_signal_fence", 1187 "__tsan_atomic_signal_fence",
1238 "__tsan_unaligned_read16", 1188 "__tsan_unaligned_read16",
1239 "__tsan_unaligned_write16", 1189 "__tsan_unaligned_write16",
1240 /* KCOV */ 1190 /* KCOV */
1241 "write_comp_data", 1191 "write_comp_data",
1242 "check_kcov_mode", 1192 "check_kcov_mode",
1243 "__sanitizer_cov_trace_pc", 1193 "__sanitizer_cov_trace_pc",
1244 "__sanitizer_cov_trace_const_cmp1", 1194 "__sanitizer_cov_trace_const_cmp1",
1245 "__sanitizer_cov_trace_const_cmp2", 1195 "__sanitizer_cov_trace_const_cmp2",
1246 "__sanitizer_cov_trace_const_cmp4", 1196 "__sanitizer_cov_trace_const_cmp4",
1247 "__sanitizer_cov_trace_const_cmp8", 1197 "__sanitizer_cov_trace_const_cmp8",
1248 "__sanitizer_cov_trace_cmp1", 1198 "__sanitizer_cov_trace_cmp1",
1249 "__sanitizer_cov_trace_cmp2", 1199 "__sanitizer_cov_trace_cmp2",
1250 "__sanitizer_cov_trace_cmp4", 1200 "__sanitizer_cov_trace_cmp4",
1251 "__sanitizer_cov_trace_cmp8", 1201 "__sanitizer_cov_trace_cmp8",
1252 "__sanitizer_cov_trace_switch", 1202 "__sanitizer_cov_trace_switch",
1253 /* KMSAN */ 1203 /* KMSAN */
1254 "kmsan_copy_to_user", 1204 "kmsan_copy_to_user",
1255 "kmsan_disable_current", 1205 "kmsan_disable_current",
1256 "kmsan_enable_current", 1206 "kmsan_enable_current",
1257 "kmsan_report", 1207 "kmsan_report",
1258 "kmsan_unpoison_entry_regs", 1208 "kmsan_unpoison_entry_regs",
1259 "kmsan_unpoison_memory", 1209 "kmsan_unpoison_memory",
1260 "__msan_chain_origin", 1210 "__msan_chain_origin",
1261 "__msan_get_context_state", 1211 "__msan_get_context_state",
1262 "__msan_instrument_asm_store", 1212 "__msan_instrument_asm_store",
1263 "__msan_metadata_ptr_for_load_1", 1213 "__msan_metadata_ptr_for_load_1",
1264 "__msan_metadata_ptr_for_load_2", 1214 "__msan_metadata_ptr_for_load_2",
1265 "__msan_metadata_ptr_for_load_4", 1215 "__msan_metadata_ptr_for_load_4",
1266 "__msan_metadata_ptr_for_load_8", 1216 "__msan_metadata_ptr_for_load_8",
1267 "__msan_metadata_ptr_for_load_n", 1217 "__msan_metadata_ptr_for_load_n",
1268 "__msan_metadata_ptr_for_store_1", 1218 "__msan_metadata_ptr_for_store_1",
1269 "__msan_metadata_ptr_for_store_2", 1219 "__msan_metadata_ptr_for_store_2",
1270 "__msan_metadata_ptr_for_store_4", 1220 "__msan_metadata_ptr_for_store_4",
1271 "__msan_metadata_ptr_for_store_8", 1221 "__msan_metadata_ptr_for_store_8",
1272 "__msan_metadata_ptr_for_store_n", 1222 "__msan_metadata_ptr_for_store_n",
1273 "__msan_poison_alloca", 1223 "__msan_poison_alloca",
1274 "__msan_warning", 1224 "__msan_warning",
1275 /* UBSAN */ 1225 /* UBSAN */
1276 "ubsan_type_mismatch_common", 1226 "ubsan_type_mismatch_common",
1277 "__ubsan_handle_type_mismatch", 1227 "__ubsan_handle_type_mismatch",
1278 "__ubsan_handle_type_mismatch_v1", 1228 "__ubsan_handle_type_mismatch_v1",
1279 "__ubsan_handle_shift_out_of_bounds", 1229 "__ubsan_handle_shift_out_of_bounds",
1280 "__ubsan_handle_load_invalid_value", 1230 "__ubsan_handle_load_invalid_value",
1281 /* STACKLEAK */ 1231 /* STACKLEAK */
1282 "stackleak_track_stack", 1232 "stackleak_track_stack",
1283 /* misc */ 1233 /* misc */
1284 "csum_partial_copy_generic", 1234 "csum_partial_copy_generic",
1285 "copy_mc_fragile", 1235 "copy_mc_fragile",
1286 "copy_mc_fragile_handle_tail", 1236 "copy_mc_fragile_handle_tail",
1287 "copy_mc_enhanced_fast_string", 1237 "copy_mc_enhanced_fast_string",
1288 "ftrace_likely_update", /* CONFIG_TRA 1238 "ftrace_likely_update", /* CONFIG_TRACE_BRANCH_PROFILING */
1289 "rep_stos_alternative", 1239 "rep_stos_alternative",
1290 "rep_movs_alternative", 1240 "rep_movs_alternative",
1291 "__copy_user_nocache", 1241 "__copy_user_nocache",
1292 NULL 1242 NULL
1293 }; 1243 };
1294 1244
1295 static void add_uaccess_safe(struct objtool_f 1245 static void add_uaccess_safe(struct objtool_file *file)
1296 { 1246 {
1297 struct symbol *func; 1247 struct symbol *func;
1298 const char **name; 1248 const char **name;
1299 1249
1300 if (!opts.uaccess) 1250 if (!opts.uaccess)
1301 return; 1251 return;
1302 1252
1303 for (name = uaccess_safe_builtin; *na 1253 for (name = uaccess_safe_builtin; *name; name++) {
1304 func = find_symbol_by_name(fi 1254 func = find_symbol_by_name(file->elf, *name);
1305 if (!func) 1255 if (!func)
1306 continue; 1256 continue;
1307 1257
1308 func->uaccess_safe = true; 1258 func->uaccess_safe = true;
1309 } 1259 }
1310 } 1260 }
1311 1261
1312 /* 1262 /*
1313 * FIXME: For now, just ignore any alternativ 1263 * FIXME: For now, just ignore any alternatives which add retpolines. This is
1314 * a temporary hack, as it doesn't allow ORC 1264 * a temporary hack, as it doesn't allow ORC to unwind from inside a retpoline.
1315 * But it at least allows objtool to understa 1265 * But it at least allows objtool to understand the control flow *around* the
1316 * retpoline. 1266 * retpoline.
1317 */ 1267 */
1318 static int add_ignore_alternatives(struct obj 1268 static int add_ignore_alternatives(struct objtool_file *file)
1319 { 1269 {
1320 struct section *rsec; 1270 struct section *rsec;
1321 struct reloc *reloc; 1271 struct reloc *reloc;
1322 struct instruction *insn; 1272 struct instruction *insn;
1323 1273
1324 rsec = find_section_by_name(file->elf 1274 rsec = find_section_by_name(file->elf, ".rela.discard.ignore_alts");
1325 if (!rsec) 1275 if (!rsec)
1326 return 0; 1276 return 0;
1327 1277
1328 for_each_reloc(rsec, reloc) { 1278 for_each_reloc(rsec, reloc) {
1329 if (reloc->sym->type != STT_S 1279 if (reloc->sym->type != STT_SECTION) {
1330 WARN("unexpected relo 1280 WARN("unexpected relocation symbol type in %s", rsec->name);
1331 return -1; 1281 return -1;
1332 } 1282 }
1333 1283
1334 insn = find_insn(file, reloc- 1284 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
1335 if (!insn) { 1285 if (!insn) {
1336 WARN("bad .discard.ig 1286 WARN("bad .discard.ignore_alts entry");
1337 return -1; 1287 return -1;
1338 } 1288 }
1339 1289
1340 insn->ignore_alts = true; 1290 insn->ignore_alts = true;
1341 } 1291 }
1342 1292
1343 return 0; 1293 return 0;
1344 } 1294 }
1345 1295
1346 /* 1296 /*
1347 * Symbols that replace INSN_CALL_DYNAMIC, ev 1297 * Symbols that replace INSN_CALL_DYNAMIC, every (tail) call to such a symbol
1348 * will be added to the .retpoline_sites sect 1298 * will be added to the .retpoline_sites section.
1349 */ 1299 */
1350 __weak bool arch_is_retpoline(struct symbol * 1300 __weak bool arch_is_retpoline(struct symbol *sym)
1351 { 1301 {
1352 return false; 1302 return false;
1353 } 1303 }
1354 1304
1355 /* 1305 /*
1356 * Symbols that replace INSN_RETURN, every (t 1306 * Symbols that replace INSN_RETURN, every (tail) call to such a symbol
1357 * will be added to the .return_sites section 1307 * will be added to the .return_sites section.
1358 */ 1308 */
1359 __weak bool arch_is_rethunk(struct symbol *sy 1309 __weak bool arch_is_rethunk(struct symbol *sym)
1360 { 1310 {
1361 return false; 1311 return false;
1362 } 1312 }
1363 1313
1364 /* 1314 /*
1365 * Symbols that are embedded inside other ins 1315 * Symbols that are embedded inside other instructions, because sometimes crazy
1366 * code exists. These are mostly ignored for 1316 * code exists. These are mostly ignored for validation purposes.
1367 */ 1317 */
1368 __weak bool arch_is_embedded_insn(struct symb 1318 __weak bool arch_is_embedded_insn(struct symbol *sym)
1369 { 1319 {
1370 return false; 1320 return false;
1371 } 1321 }
1372 1322
1373 static struct reloc *insn_reloc(struct objtoo 1323 static struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn)
1374 { 1324 {
1375 struct reloc *reloc; 1325 struct reloc *reloc;
1376 1326
1377 if (insn->no_reloc) 1327 if (insn->no_reloc)
1378 return NULL; 1328 return NULL;
1379 1329
1380 if (!file) 1330 if (!file)
1381 return NULL; 1331 return NULL;
1382 1332
1383 reloc = find_reloc_by_dest_range(file 1333 reloc = find_reloc_by_dest_range(file->elf, insn->sec,
1384 insn 1334 insn->offset, insn->len);
1385 if (!reloc) { 1335 if (!reloc) {
1386 insn->no_reloc = 1; 1336 insn->no_reloc = 1;
1387 return NULL; 1337 return NULL;
1388 } 1338 }
1389 1339
1390 return reloc; 1340 return reloc;
1391 } 1341 }
1392 1342
1393 static void remove_insn_ops(struct instructio 1343 static void remove_insn_ops(struct instruction *insn)
1394 { 1344 {
1395 struct stack_op *op, *next; 1345 struct stack_op *op, *next;
1396 1346
1397 for (op = insn->stack_ops; op; op = n 1347 for (op = insn->stack_ops; op; op = next) {
1398 next = op->next; 1348 next = op->next;
1399 free(op); 1349 free(op);
1400 } 1350 }
1401 insn->stack_ops = NULL; 1351 insn->stack_ops = NULL;
1402 } 1352 }
1403 1353
1404 static void annotate_call_site(struct objtool 1354 static void annotate_call_site(struct objtool_file *file,
1405 struct instruc 1355 struct instruction *insn, bool sibling)
1406 { 1356 {
1407 struct reloc *reloc = insn_reloc(file 1357 struct reloc *reloc = insn_reloc(file, insn);
1408 struct symbol *sym = insn_call_dest(i 1358 struct symbol *sym = insn_call_dest(insn);
1409 1359
1410 if (!sym) 1360 if (!sym)
1411 sym = reloc->sym; 1361 sym = reloc->sym;
1412 1362
1413 /* 1363 /*
1414 * Alternative replacement code is ju 1364 * Alternative replacement code is just template code which is
1415 * sometimes copied to the original i 1365 * sometimes copied to the original instruction. For now, don't
1416 * annotate it. (In the future we mig 1366 * annotate it. (In the future we might consider annotating the
1417 * original instruction if/when it ev 1367 * original instruction if/when it ever makes sense to do so.)
1418 */ 1368 */
1419 if (!strcmp(insn->sec->name, ".altins 1369 if (!strcmp(insn->sec->name, ".altinstr_replacement"))
1420 return; 1370 return;
1421 1371
1422 if (sym->static_call_tramp) { 1372 if (sym->static_call_tramp) {
1423 list_add_tail(&insn->call_nod 1373 list_add_tail(&insn->call_node, &file->static_call_list);
1424 return; 1374 return;
1425 } 1375 }
1426 1376
1427 if (sym->retpoline_thunk) { 1377 if (sym->retpoline_thunk) {
1428 list_add_tail(&insn->call_nod 1378 list_add_tail(&insn->call_node, &file->retpoline_call_list);
1429 return; 1379 return;
1430 } 1380 }
1431 1381
1432 /* 1382 /*
1433 * Many compilers cannot disable KCOV 1383 * Many compilers cannot disable KCOV or sanitizer calls with a function
1434 * attribute so they need a little he 1384 * attribute so they need a little help, NOP out any such calls from
1435 * noinstr text. 1385 * noinstr text.
1436 */ 1386 */
1437 if (opts.hack_noinstr && insn->sec->n 1387 if (opts.hack_noinstr && insn->sec->noinstr && sym->profiling_func) {
1438 if (reloc) 1388 if (reloc)
1439 set_reloc_type(file-> 1389 set_reloc_type(file->elf, reloc, R_NONE);
1440 1390
1441 elf_write_insn(file->elf, ins 1391 elf_write_insn(file->elf, insn->sec,
1442 insn->offset, 1392 insn->offset, insn->len,
1443 sibling ? arch 1393 sibling ? arch_ret_insn(insn->len)
1444 : arch 1394 : arch_nop_insn(insn->len));
1445 1395
1446 insn->type = sibling ? INSN_R 1396 insn->type = sibling ? INSN_RETURN : INSN_NOP;
1447 1397
1448 if (sibling) { 1398 if (sibling) {
1449 /* 1399 /*
1450 * We've replaced the 1400 * We've replaced the tail-call JMP insn by two new
1451 * insn: RET; INT3, e 1401 * insn: RET; INT3, except we only have a single struct
1452 * insn here. Mark it 1402 * insn here. Mark it retpoline_safe to avoid the SLS
1453 * warning, instead o 1403 * warning, instead of adding another insn.
1454 */ 1404 */
1455 insn->retpoline_safe 1405 insn->retpoline_safe = true;
1456 } 1406 }
1457 1407
1458 return; 1408 return;
1459 } 1409 }
1460 1410
1461 if (opts.mcount && sym->fentry) { 1411 if (opts.mcount && sym->fentry) {
1462 if (sibling) 1412 if (sibling)
1463 WARN_INSN(insn, "tail 1413 WARN_INSN(insn, "tail call to __fentry__ !?!?");
1464 if (opts.mnop) { 1414 if (opts.mnop) {
1465 if (reloc) 1415 if (reloc)
1466 set_reloc_typ 1416 set_reloc_type(file->elf, reloc, R_NONE);
1467 1417
1468 elf_write_insn(file-> 1418 elf_write_insn(file->elf, insn->sec,
1469 insn-> 1419 insn->offset, insn->len,
1470 arch_n 1420 arch_nop_insn(insn->len));
1471 1421
1472 insn->type = INSN_NOP 1422 insn->type = INSN_NOP;
1473 } 1423 }
1474 1424
1475 list_add_tail(&insn->call_nod 1425 list_add_tail(&insn->call_node, &file->mcount_loc_list);
1476 return; 1426 return;
1477 } 1427 }
1478 1428
1479 if (insn->type == INSN_CALL && !insn- 1429 if (insn->type == INSN_CALL && !insn->sec->init)
1480 list_add_tail(&insn->call_nod 1430 list_add_tail(&insn->call_node, &file->call_list);
1481 1431
1482 if (!sibling && dead_end_function(fil 1432 if (!sibling && dead_end_function(file, sym))
1483 insn->dead_end = true; 1433 insn->dead_end = true;
1484 } 1434 }
1485 1435
1486 static void add_call_dest(struct objtool_file 1436 static void add_call_dest(struct objtool_file *file, struct instruction *insn,
1487 struct symbol *dest 1437 struct symbol *dest, bool sibling)
1488 { 1438 {
1489 insn->_call_dest = dest; 1439 insn->_call_dest = dest;
1490 if (!dest) 1440 if (!dest)
1491 return; 1441 return;
1492 1442
1493 /* 1443 /*
1494 * Whatever stack impact regular CALL 1444 * Whatever stack impact regular CALLs have, should be undone
1495 * by the RETURN of the called functi 1445 * by the RETURN of the called function.
1496 * 1446 *
1497 * Annotated intra-function calls ret 1447 * Annotated intra-function calls retain the stack_ops but
1498 * are converted to JUMP, see read_in 1448 * are converted to JUMP, see read_intra_function_calls().
1499 */ 1449 */
1500 remove_insn_ops(insn); 1450 remove_insn_ops(insn);
1501 1451
1502 annotate_call_site(file, insn, siblin 1452 annotate_call_site(file, insn, sibling);
1503 } 1453 }
1504 1454
1505 static void add_retpoline_call(struct objtool 1455 static void add_retpoline_call(struct objtool_file *file, struct instruction *insn)
1506 { 1456 {
1507 /* 1457 /*
1508 * Retpoline calls/jumps are really d 1458 * Retpoline calls/jumps are really dynamic calls/jumps in disguise,
1509 * so convert them accordingly. 1459 * so convert them accordingly.
1510 */ 1460 */
1511 switch (insn->type) { 1461 switch (insn->type) {
1512 case INSN_CALL: 1462 case INSN_CALL:
1513 insn->type = INSN_CALL_DYNAMI 1463 insn->type = INSN_CALL_DYNAMIC;
1514 break; 1464 break;
1515 case INSN_JUMP_UNCONDITIONAL: 1465 case INSN_JUMP_UNCONDITIONAL:
1516 insn->type = INSN_JUMP_DYNAMI 1466 insn->type = INSN_JUMP_DYNAMIC;
1517 break; 1467 break;
1518 case INSN_JUMP_CONDITIONAL: 1468 case INSN_JUMP_CONDITIONAL:
1519 insn->type = INSN_JUMP_DYNAMI 1469 insn->type = INSN_JUMP_DYNAMIC_CONDITIONAL;
1520 break; 1470 break;
1521 default: 1471 default:
1522 return; 1472 return;
1523 } 1473 }
1524 1474
1525 insn->retpoline_safe = true; 1475 insn->retpoline_safe = true;
1526 1476
1527 /* 1477 /*
1528 * Whatever stack impact regular CALL 1478 * Whatever stack impact regular CALLs have, should be undone
1529 * by the RETURN of the called functi 1479 * by the RETURN of the called function.
1530 * 1480 *
1531 * Annotated intra-function calls ret 1481 * Annotated intra-function calls retain the stack_ops but
1532 * are converted to JUMP, see read_in 1482 * are converted to JUMP, see read_intra_function_calls().
1533 */ 1483 */
1534 remove_insn_ops(insn); 1484 remove_insn_ops(insn);
1535 1485
1536 annotate_call_site(file, insn, false) 1486 annotate_call_site(file, insn, false);
1537 } 1487 }
1538 1488
1539 static void add_return_call(struct objtool_fi 1489 static void add_return_call(struct objtool_file *file, struct instruction *insn, bool add)
1540 { 1490 {
1541 /* 1491 /*
1542 * Return thunk tail calls are really 1492 * Return thunk tail calls are really just returns in disguise,
1543 * so convert them accordingly. 1493 * so convert them accordingly.
1544 */ 1494 */
1545 insn->type = INSN_RETURN; 1495 insn->type = INSN_RETURN;
1546 insn->retpoline_safe = true; 1496 insn->retpoline_safe = true;
1547 1497
1548 if (add) 1498 if (add)
1549 list_add_tail(&insn->call_nod 1499 list_add_tail(&insn->call_node, &file->return_thunk_list);
1550 } 1500 }
1551 1501
1552 static bool is_first_func_insn(struct objtool 1502 static bool is_first_func_insn(struct objtool_file *file,
1553 struct instruc 1503 struct instruction *insn, struct symbol *sym)
1554 { 1504 {
1555 if (insn->offset == sym->offset) 1505 if (insn->offset == sym->offset)
1556 return true; 1506 return true;
1557 1507
1558 /* Allow direct CALL/JMP past ENDBR * 1508 /* Allow direct CALL/JMP past ENDBR */
1559 if (opts.ibt) { 1509 if (opts.ibt) {
1560 struct instruction *prev = pr 1510 struct instruction *prev = prev_insn_same_sym(file, insn);
1561 1511
1562 if (prev && prev->type == INS 1512 if (prev && prev->type == INSN_ENDBR &&
1563 insn->offset == sym->offs 1513 insn->offset == sym->offset + prev->len)
1564 return true; 1514 return true;
1565 } 1515 }
1566 1516
1567 return false; 1517 return false;
1568 } 1518 }
1569 1519
1570 /* 1520 /*
1571 * A sibling call is a tail-call to another s 1521 * A sibling call is a tail-call to another symbol -- to differentiate from a
1572 * recursive tail-call which is to the same s 1522 * recursive tail-call which is to the same symbol.
1573 */ 1523 */
1574 static bool jump_is_sibling_call(struct objto 1524 static bool jump_is_sibling_call(struct objtool_file *file,
1575 struct instr 1525 struct instruction *from, struct instruction *to)
1576 { 1526 {
1577 struct symbol *fs = from->sym; 1527 struct symbol *fs = from->sym;
1578 struct symbol *ts = to->sym; 1528 struct symbol *ts = to->sym;
1579 1529
1580 /* Not a sibling call if from/to a sy 1530 /* Not a sibling call if from/to a symbol hole */
1581 if (!fs || !ts) 1531 if (!fs || !ts)
1582 return false; 1532 return false;
1583 1533
1584 /* Not a sibling call if not targetin 1534 /* Not a sibling call if not targeting the start of a symbol. */
1585 if (!is_first_func_insn(file, to, ts) 1535 if (!is_first_func_insn(file, to, ts))
1586 return false; 1536 return false;
1587 1537
1588 /* Disallow sibling calls into STT_NO 1538 /* Disallow sibling calls into STT_NOTYPE */
1589 if (ts->type == STT_NOTYPE) 1539 if (ts->type == STT_NOTYPE)
1590 return false; 1540 return false;
1591 1541
1592 /* Must not be self to be a sibling * 1542 /* Must not be self to be a sibling */
1593 return fs->pfunc != ts->pfunc; 1543 return fs->pfunc != ts->pfunc;
1594 } 1544 }
1595 1545
1596 /* 1546 /*
1597 * Find the destination instructions for all 1547 * Find the destination instructions for all jumps.
1598 */ 1548 */
1599 static int add_jump_destinations(struct objto 1549 static int add_jump_destinations(struct objtool_file *file)
1600 { 1550 {
1601 struct instruction *insn, *jump_dest; 1551 struct instruction *insn, *jump_dest;
1602 struct reloc *reloc; 1552 struct reloc *reloc;
1603 struct section *dest_sec; 1553 struct section *dest_sec;
1604 unsigned long dest_off; 1554 unsigned long dest_off;
1605 1555
1606 for_each_insn(file, insn) { 1556 for_each_insn(file, insn) {
1607 if (insn->jump_dest) { 1557 if (insn->jump_dest) {
1608 /* 1558 /*
1609 * handle_group_alt() 1559 * handle_group_alt() may have previously set
1610 * 'jump_dest' for so 1560 * 'jump_dest' for some alternatives.
1611 */ 1561 */
1612 continue; 1562 continue;
1613 } 1563 }
1614 if (!is_static_jump(insn)) 1564 if (!is_static_jump(insn))
1615 continue; 1565 continue;
1616 1566
1617 reloc = insn_reloc(file, insn 1567 reloc = insn_reloc(file, insn);
1618 if (!reloc) { 1568 if (!reloc) {
1619 dest_sec = insn->sec; 1569 dest_sec = insn->sec;
1620 dest_off = arch_jump_ 1570 dest_off = arch_jump_destination(insn);
1621 } else if (reloc->sym->type = 1571 } else if (reloc->sym->type == STT_SECTION) {
1622 dest_sec = reloc->sym 1572 dest_sec = reloc->sym->sec;
1623 dest_off = arch_dest_ 1573 dest_off = arch_dest_reloc_offset(reloc_addend(reloc));
1624 } else if (reloc->sym->retpol 1574 } else if (reloc->sym->retpoline_thunk) {
1625 add_retpoline_call(fi 1575 add_retpoline_call(file, insn);
1626 continue; 1576 continue;
1627 } else if (reloc->sym->return 1577 } else if (reloc->sym->return_thunk) {
1628 add_return_call(file, 1578 add_return_call(file, insn, true);
1629 continue; 1579 continue;
1630 } else if (insn_func(insn)) { 1580 } else if (insn_func(insn)) {
1631 /* 1581 /*
1632 * External sibling c 1582 * External sibling call or internal sibling call with
1633 * STT_FUNC reloc. 1583 * STT_FUNC reloc.
1634 */ 1584 */
1635 add_call_dest(file, i 1585 add_call_dest(file, insn, reloc->sym, true);
1636 continue; 1586 continue;
1637 } else if (reloc->sym->sec->i 1587 } else if (reloc->sym->sec->idx) {
1638 dest_sec = reloc->sym 1588 dest_sec = reloc->sym->sec;
1639 dest_off = reloc->sym 1589 dest_off = reloc->sym->sym.st_value +
1640 arch_dest_ 1590 arch_dest_reloc_offset(reloc_addend(reloc));
1641 } else { 1591 } else {
1642 /* non-func asm code 1592 /* non-func asm code jumping to another file */
1643 continue; 1593 continue;
1644 } 1594 }
1645 1595
1646 jump_dest = find_insn(file, d 1596 jump_dest = find_insn(file, dest_sec, dest_off);
1647 if (!jump_dest) { 1597 if (!jump_dest) {
1648 struct symbol *sym = 1598 struct symbol *sym = find_symbol_by_offset(dest_sec, dest_off);
1649 1599
1650 /* 1600 /*
1651 * This is a special 1601 * This is a special case for retbleed_untrain_ret().
1652 * It jumps to __x86_ 1602 * It jumps to __x86_return_thunk(), but objtool
1653 * can't find the thu 1603 * can't find the thunk's starting RET
1654 * instruction, becau 1604 * instruction, because the RET is also in the
1655 * middle of another 1605 * middle of another instruction. Objtool only
1656 * knows about the ou 1606 * knows about the outer instruction.
1657 */ 1607 */
1658 if (sym && sym->embed 1608 if (sym && sym->embedded_insn) {
1659 add_return_ca 1609 add_return_call(file, insn, false);
1660 continue; 1610 continue;
1661 } 1611 }
1662 1612
1663 WARN_INSN(insn, "can' 1613 WARN_INSN(insn, "can't find jump dest instruction at %s+0x%lx",
1664 dest_sec->n 1614 dest_sec->name, dest_off);
1665 return -1; 1615 return -1;
1666 } 1616 }
1667 1617
1668 /* 1618 /*
1669 * An intra-TU jump in retpol 1619 * An intra-TU jump in retpoline.o might not have a relocation
1670 * for its jump dest, in whic 1620 * for its jump dest, in which case the above
1671 * add_{retpoline,return}_cal 1621 * add_{retpoline,return}_call() didn't happen.
1672 */ 1622 */
1673 if (jump_dest->sym && jump_de 1623 if (jump_dest->sym && jump_dest->offset == jump_dest->sym->offset) {
1674 if (jump_dest->sym->r 1624 if (jump_dest->sym->retpoline_thunk) {
1675 add_retpoline 1625 add_retpoline_call(file, insn);
1676 continue; 1626 continue;
1677 } 1627 }
1678 if (jump_dest->sym->r 1628 if (jump_dest->sym->return_thunk) {
1679 add_return_ca 1629 add_return_call(file, insn, true);
1680 continue; 1630 continue;
1681 } 1631 }
1682 } 1632 }
1683 1633
1684 /* 1634 /*
1685 * Cross-function jump. 1635 * Cross-function jump.
1686 */ 1636 */
1687 if (insn_func(insn) && insn_f 1637 if (insn_func(insn) && insn_func(jump_dest) &&
1688 insn_func(insn) != insn_f 1638 insn_func(insn) != insn_func(jump_dest)) {
1689 1639
1690 /* 1640 /*
1691 * For GCC 8+, create 1641 * For GCC 8+, create parent/child links for any cold
1692 * subfunctions. Thi 1642 * subfunctions. This is _mostly_ redundant with a
1693 * similar initializa 1643 * similar initialization in read_symbols().
1694 * 1644 *
1695 * If a function has 1645 * If a function has aliases, we want the *first* such
1696 * function in the sy 1646 * function in the symbol table to be the subfunction's
1697 * parent. In that c 1647 * parent. In that case we overwrite the
1698 * initialization don 1648 * initialization done in read_symbols().
1699 * 1649 *
1700 * However this code 1650 * However this code can't completely replace the
1701 * read_symbols() cod 1651 * read_symbols() code because this doesn't detect the
1702 * case where the par 1652 * case where the parent function's only reference to a
1703 * subfunction is thr 1653 * subfunction is through a jump table.
1704 */ 1654 */
1705 if (!strstr(insn_func 1655 if (!strstr(insn_func(insn)->name, ".cold") &&
1706 strstr(insn_func( 1656 strstr(insn_func(jump_dest)->name, ".cold")) {
1707 insn_func(ins 1657 insn_func(insn)->cfunc = insn_func(jump_dest);
1708 insn_func(jum 1658 insn_func(jump_dest)->pfunc = insn_func(insn);
1709 } 1659 }
1710 } 1660 }
1711 1661
1712 if (jump_is_sibling_call(file 1662 if (jump_is_sibling_call(file, insn, jump_dest)) {
1713 /* 1663 /*
1714 * Internal sibling c 1664 * Internal sibling call without reloc or with
1715 * STT_SECTION reloc. 1665 * STT_SECTION reloc.
1716 */ 1666 */
1717 add_call_dest(file, i 1667 add_call_dest(file, insn, insn_func(jump_dest), true);
1718 continue; 1668 continue;
1719 } 1669 }
1720 1670
1721 insn->jump_dest = jump_dest; 1671 insn->jump_dest = jump_dest;
1722 } 1672 }
1723 1673
1724 return 0; 1674 return 0;
1725 } 1675 }
1726 1676
1727 static struct symbol *find_call_destination(s 1677 static struct symbol *find_call_destination(struct section *sec, unsigned long offset)
1728 { 1678 {
1729 struct symbol *call_dest; 1679 struct symbol *call_dest;
1730 1680
1731 call_dest = find_func_by_offset(sec, 1681 call_dest = find_func_by_offset(sec, offset);
1732 if (!call_dest) 1682 if (!call_dest)
1733 call_dest = find_symbol_by_of 1683 call_dest = find_symbol_by_offset(sec, offset);
1734 1684
1735 return call_dest; 1685 return call_dest;
1736 } 1686 }
1737 1687
1738 /* 1688 /*
1739 * Find the destination instructions for all 1689 * Find the destination instructions for all calls.
1740 */ 1690 */
1741 static int add_call_destinations(struct objto 1691 static int add_call_destinations(struct objtool_file *file)
1742 { 1692 {
1743 struct instruction *insn; 1693 struct instruction *insn;
1744 unsigned long dest_off; 1694 unsigned long dest_off;
1745 struct symbol *dest; 1695 struct symbol *dest;
1746 struct reloc *reloc; 1696 struct reloc *reloc;
1747 1697
1748 for_each_insn(file, insn) { 1698 for_each_insn(file, insn) {
1749 if (insn->type != INSN_CALL) 1699 if (insn->type != INSN_CALL)
1750 continue; 1700 continue;
1751 1701
1752 reloc = insn_reloc(file, insn 1702 reloc = insn_reloc(file, insn);
1753 if (!reloc) { 1703 if (!reloc) {
1754 dest_off = arch_jump_ 1704 dest_off = arch_jump_destination(insn);
1755 dest = find_call_dest 1705 dest = find_call_destination(insn->sec, dest_off);
1756 1706
1757 add_call_dest(file, i 1707 add_call_dest(file, insn, dest, false);
1758 1708
1759 if (insn->ignore) 1709 if (insn->ignore)
1760 continue; 1710 continue;
1761 1711
1762 if (!insn_call_dest(i 1712 if (!insn_call_dest(insn)) {
1763 WARN_INSN(ins 1713 WARN_INSN(insn, "unannotated intra-function call");
1764 return -1; 1714 return -1;
1765 } 1715 }
1766 1716
1767 if (insn_func(insn) & 1717 if (insn_func(insn) && insn_call_dest(insn)->type != STT_FUNC) {
1768 WARN_INSN(ins 1718 WARN_INSN(insn, "unsupported call to non-function");
1769 return -1; 1719 return -1;
1770 } 1720 }
1771 1721
1772 } else if (reloc->sym->type = 1722 } else if (reloc->sym->type == STT_SECTION) {
1773 dest_off = arch_dest_ 1723 dest_off = arch_dest_reloc_offset(reloc_addend(reloc));
1774 dest = find_call_dest 1724 dest = find_call_destination(reloc->sym->sec, dest_off);
1775 if (!dest) { 1725 if (!dest) {
1776 WARN_INSN(ins 1726 WARN_INSN(insn, "can't find call dest symbol at %s+0x%lx",
1777 rel 1727 reloc->sym->sec->name, dest_off);
1778 return -1; 1728 return -1;
1779 } 1729 }
1780 1730
1781 add_call_dest(file, i 1731 add_call_dest(file, insn, dest, false);
1782 1732
1783 } else if (reloc->sym->retpol 1733 } else if (reloc->sym->retpoline_thunk) {
1784 add_retpoline_call(fi 1734 add_retpoline_call(file, insn);
1785 1735
1786 } else 1736 } else
1787 add_call_dest(file, i 1737 add_call_dest(file, insn, reloc->sym, false);
1788 } 1738 }
1789 1739
1790 return 0; 1740 return 0;
1791 } 1741 }
1792 1742
1793 /* 1743 /*
1794 * The .alternatives section requires some ex 1744 * The .alternatives section requires some extra special care over and above
1795 * other special sections because alternative 1745 * other special sections because alternatives are patched in place.
1796 */ 1746 */
1797 static int handle_group_alt(struct objtool_fi 1747 static int handle_group_alt(struct objtool_file *file,
1798 struct special_al 1748 struct special_alt *special_alt,
1799 struct instructio 1749 struct instruction *orig_insn,
1800 struct instructio 1750 struct instruction **new_insn)
1801 { 1751 {
1802 struct instruction *last_new_insn = N 1752 struct instruction *last_new_insn = NULL, *insn, *nop = NULL;
1803 struct alt_group *orig_alt_group, *ne 1753 struct alt_group *orig_alt_group, *new_alt_group;
1804 unsigned long dest_off; 1754 unsigned long dest_off;
1805 1755
1806 orig_alt_group = orig_insn->alt_group 1756 orig_alt_group = orig_insn->alt_group;
1807 if (!orig_alt_group) { 1757 if (!orig_alt_group) {
1808 struct instruction *last_orig 1758 struct instruction *last_orig_insn = NULL;
1809 1759
1810 orig_alt_group = malloc(sizeo 1760 orig_alt_group = malloc(sizeof(*orig_alt_group));
1811 if (!orig_alt_group) { 1761 if (!orig_alt_group) {
1812 WARN("malloc failed") 1762 WARN("malloc failed");
1813 return -1; 1763 return -1;
1814 } 1764 }
1815 orig_alt_group->cfi = calloc( 1765 orig_alt_group->cfi = calloc(special_alt->orig_len,
1816 1766 sizeof(struct cfi_state *));
1817 if (!orig_alt_group->cfi) { 1767 if (!orig_alt_group->cfi) {
1818 WARN("calloc failed") 1768 WARN("calloc failed");
1819 return -1; 1769 return -1;
1820 } 1770 }
1821 1771
1822 insn = orig_insn; 1772 insn = orig_insn;
1823 sec_for_each_insn_from(file, 1773 sec_for_each_insn_from(file, insn) {
1824 if (insn->offset >= s 1774 if (insn->offset >= special_alt->orig_off + special_alt->orig_len)
1825 break; 1775 break;
1826 1776
1827 insn->alt_group = ori 1777 insn->alt_group = orig_alt_group;
1828 last_orig_insn = insn 1778 last_orig_insn = insn;
1829 } 1779 }
1830 orig_alt_group->orig_group = 1780 orig_alt_group->orig_group = NULL;
1831 orig_alt_group->first_insn = 1781 orig_alt_group->first_insn = orig_insn;
1832 orig_alt_group->last_insn = l 1782 orig_alt_group->last_insn = last_orig_insn;
1833 orig_alt_group->nop = NULL; 1783 orig_alt_group->nop = NULL;
1834 } else { 1784 } else {
1835 if (orig_alt_group->last_insn 1785 if (orig_alt_group->last_insn->offset + orig_alt_group->last_insn->len -
1836 orig_alt_group->first_ins 1786 orig_alt_group->first_insn->offset != special_alt->orig_len) {
1837 WARN_INSN(orig_insn, 1787 WARN_INSN(orig_insn, "weirdly overlapping alternative! %ld != %d",
1838 orig_alt_gr 1788 orig_alt_group->last_insn->offset +
1839 orig_alt_gr 1789 orig_alt_group->last_insn->len -
1840 orig_alt_gr 1790 orig_alt_group->first_insn->offset,
1841 special_alt 1791 special_alt->orig_len);
1842 return -1; 1792 return -1;
1843 } 1793 }
1844 } 1794 }
1845 1795
1846 new_alt_group = malloc(sizeof(*new_al 1796 new_alt_group = malloc(sizeof(*new_alt_group));
1847 if (!new_alt_group) { 1797 if (!new_alt_group) {
1848 WARN("malloc failed"); 1798 WARN("malloc failed");
1849 return -1; 1799 return -1;
1850 } 1800 }
1851 1801
1852 if (special_alt->new_len < special_al 1802 if (special_alt->new_len < special_alt->orig_len) {
1853 /* 1803 /*
1854 * Insert a fake nop at the e 1804 * Insert a fake nop at the end to make the replacement
1855 * alt_group the same size as 1805 * alt_group the same size as the original. This is needed to
1856 * allow propagate_alt_cfi() 1806 * allow propagate_alt_cfi() to do its magic. When the last
1857 * instruction affects the st 1807 * instruction affects the stack, the instruction after it (the
1858 * nop) will propagate the ne 1808 * nop) will propagate the new state to the shared CFI array.
1859 */ 1809 */
1860 nop = malloc(sizeof(*nop)); 1810 nop = malloc(sizeof(*nop));
1861 if (!nop) { 1811 if (!nop) {
1862 WARN("malloc failed") 1812 WARN("malloc failed");
1863 return -1; 1813 return -1;
1864 } 1814 }
1865 memset(nop, 0, sizeof(*nop)); 1815 memset(nop, 0, sizeof(*nop));
1866 1816
1867 nop->sec = special_alt->new_s 1817 nop->sec = special_alt->new_sec;
1868 nop->offset = special_alt->ne 1818 nop->offset = special_alt->new_off + special_alt->new_len;
1869 nop->len = special_alt->orig_ 1819 nop->len = special_alt->orig_len - special_alt->new_len;
1870 nop->type = INSN_NOP; 1820 nop->type = INSN_NOP;
1871 nop->sym = orig_insn->sym; 1821 nop->sym = orig_insn->sym;
1872 nop->alt_group = new_alt_grou 1822 nop->alt_group = new_alt_group;
1873 nop->ignore = orig_insn->igno 1823 nop->ignore = orig_insn->ignore_alts;
1874 } 1824 }
1875 1825
1876 if (!special_alt->new_len) { 1826 if (!special_alt->new_len) {
1877 *new_insn = nop; 1827 *new_insn = nop;
1878 goto end; 1828 goto end;
1879 } 1829 }
1880 1830
1881 insn = *new_insn; 1831 insn = *new_insn;
1882 sec_for_each_insn_from(file, insn) { 1832 sec_for_each_insn_from(file, insn) {
1883 struct reloc *alt_reloc; 1833 struct reloc *alt_reloc;
1884 1834
1885 if (insn->offset >= special_a 1835 if (insn->offset >= special_alt->new_off + special_alt->new_len)
1886 break; 1836 break;
1887 1837
1888 last_new_insn = insn; 1838 last_new_insn = insn;
1889 1839
1890 insn->ignore = orig_insn->ign 1840 insn->ignore = orig_insn->ignore_alts;
1891 insn->sym = orig_insn->sym; 1841 insn->sym = orig_insn->sym;
1892 insn->alt_group = new_alt_gro 1842 insn->alt_group = new_alt_group;
1893 1843
1894 /* 1844 /*
1895 * Since alternative replacem 1845 * Since alternative replacement code is copy/pasted by the
1896 * kernel after applying relo 1846 * kernel after applying relocations, generally such code can't
1897 * have relative-address relo 1847 * have relative-address relocation references to outside the
1898 * .altinstr_replacement sect 1848 * .altinstr_replacement section, unless the arch's
1899 * alternatives code can adju 1849 * alternatives code can adjust the relative offsets
1900 * accordingly. 1850 * accordingly.
1901 */ 1851 */
1902 alt_reloc = insn_reloc(file, 1852 alt_reloc = insn_reloc(file, insn);
1903 if (alt_reloc && arch_pc_rela 1853 if (alt_reloc && arch_pc_relative_reloc(alt_reloc) &&
1904 !arch_support_alt_relocat 1854 !arch_support_alt_relocation(special_alt, insn, alt_reloc)) {
1905 1855
1906 WARN_INSN(insn, "unsu 1856 WARN_INSN(insn, "unsupported relocation in alternatives section");
1907 return -1; 1857 return -1;
1908 } 1858 }
1909 1859
1910 if (!is_static_jump(insn)) 1860 if (!is_static_jump(insn))
1911 continue; 1861 continue;
1912 1862
1913 if (!insn->immediate) 1863 if (!insn->immediate)
1914 continue; 1864 continue;
1915 1865
1916 dest_off = arch_jump_destinat 1866 dest_off = arch_jump_destination(insn);
1917 if (dest_off == special_alt-> 1867 if (dest_off == special_alt->new_off + special_alt->new_len) {
1918 insn->jump_dest = nex 1868 insn->jump_dest = next_insn_same_sec(file, orig_alt_group->last_insn);
1919 if (!insn->jump_dest) 1869 if (!insn->jump_dest) {
1920 WARN_INSN(ins 1870 WARN_INSN(insn, "can't find alternative jump destination");
1921 return -1; 1871 return -1;
1922 } 1872 }
1923 } 1873 }
1924 } 1874 }
1925 1875
1926 if (!last_new_insn) { 1876 if (!last_new_insn) {
1927 WARN_FUNC("can't find last ne 1877 WARN_FUNC("can't find last new alternative instruction",
1928 special_alt->new_se 1878 special_alt->new_sec, special_alt->new_off);
1929 return -1; 1879 return -1;
1930 } 1880 }
1931 1881
1932 end: 1882 end:
1933 new_alt_group->orig_group = orig_alt_ 1883 new_alt_group->orig_group = orig_alt_group;
1934 new_alt_group->first_insn = *new_insn 1884 new_alt_group->first_insn = *new_insn;
1935 new_alt_group->last_insn = last_new_i 1885 new_alt_group->last_insn = last_new_insn;
1936 new_alt_group->nop = nop; 1886 new_alt_group->nop = nop;
1937 new_alt_group->cfi = orig_alt_group-> 1887 new_alt_group->cfi = orig_alt_group->cfi;
1938 return 0; 1888 return 0;
1939 } 1889 }
1940 1890
1941 /* 1891 /*
1942 * A jump table entry can either convert a no 1892 * A jump table entry can either convert a nop to a jump or a jump to a nop.
1943 * If the original instruction is a jump, mak 1893 * If the original instruction is a jump, make the alt entry an effective nop
1944 * by just skipping the original instruction. 1894 * by just skipping the original instruction.
1945 */ 1895 */
1946 static int handle_jump_alt(struct objtool_fil 1896 static int handle_jump_alt(struct objtool_file *file,
1947 struct special_alt 1897 struct special_alt *special_alt,
1948 struct instruction 1898 struct instruction *orig_insn,
1949 struct instruction 1899 struct instruction **new_insn)
1950 { 1900 {
1951 if (orig_insn->type != INSN_JUMP_UNCO 1901 if (orig_insn->type != INSN_JUMP_UNCONDITIONAL &&
1952 orig_insn->type != INSN_NOP) { 1902 orig_insn->type != INSN_NOP) {
1953 1903
1954 WARN_INSN(orig_insn, "unsuppo 1904 WARN_INSN(orig_insn, "unsupported instruction at jump label");
1955 return -1; 1905 return -1;
1956 } 1906 }
1957 1907
1958 if (opts.hack_jump_label && special_a 1908 if (opts.hack_jump_label && special_alt->key_addend & 2) {
1959 struct reloc *reloc = insn_re 1909 struct reloc *reloc = insn_reloc(file, orig_insn);
1960 1910
1961 if (reloc) 1911 if (reloc)
1962 set_reloc_type(file-> 1912 set_reloc_type(file->elf, reloc, R_NONE);
1963 elf_write_insn(file->elf, ori 1913 elf_write_insn(file->elf, orig_insn->sec,
1964 orig_insn->off 1914 orig_insn->offset, orig_insn->len,
1965 arch_nop_insn( 1915 arch_nop_insn(orig_insn->len));
1966 orig_insn->type = INSN_NOP; 1916 orig_insn->type = INSN_NOP;
1967 } 1917 }
1968 1918
1969 if (orig_insn->type == INSN_NOP) { 1919 if (orig_insn->type == INSN_NOP) {
1970 if (orig_insn->len == 2) 1920 if (orig_insn->len == 2)
1971 file->jl_nop_short++; 1921 file->jl_nop_short++;
1972 else 1922 else
1973 file->jl_nop_long++; 1923 file->jl_nop_long++;
1974 1924
1975 return 0; 1925 return 0;
1976 } 1926 }
1977 1927
1978 if (orig_insn->len == 2) 1928 if (orig_insn->len == 2)
1979 file->jl_short++; 1929 file->jl_short++;
1980 else 1930 else
1981 file->jl_long++; 1931 file->jl_long++;
1982 1932
1983 *new_insn = next_insn_same_sec(file, 1933 *new_insn = next_insn_same_sec(file, orig_insn);
1984 return 0; 1934 return 0;
1985 } 1935 }
1986 1936
1987 /* 1937 /*
1988 * Read all the special sections which have a 1938 * Read all the special sections which have alternate instructions which can be
1989 * patched in or redirected to at runtime. E 1939 * patched in or redirected to at runtime. Each instruction having alternate
1990 * instruction(s) has them added to its insn- 1940 * instruction(s) has them added to its insn->alts list, which will be
1991 * traversed in validate_branch(). 1941 * traversed in validate_branch().
1992 */ 1942 */
1993 static int add_special_section_alts(struct ob 1943 static int add_special_section_alts(struct objtool_file *file)
1994 { 1944 {
1995 struct list_head special_alts; 1945 struct list_head special_alts;
1996 struct instruction *orig_insn, *new_i 1946 struct instruction *orig_insn, *new_insn;
1997 struct special_alt *special_alt, *tmp 1947 struct special_alt *special_alt, *tmp;
1998 struct alternative *alt; 1948 struct alternative *alt;
1999 int ret; 1949 int ret;
2000 1950
2001 ret = special_get_alts(file->elf, &sp 1951 ret = special_get_alts(file->elf, &special_alts);
2002 if (ret) 1952 if (ret)
2003 return ret; 1953 return ret;
2004 1954
2005 list_for_each_entry_safe(special_alt, 1955 list_for_each_entry_safe(special_alt, tmp, &special_alts, list) {
2006 1956
2007 orig_insn = find_insn(file, s 1957 orig_insn = find_insn(file, special_alt->orig_sec,
2008 special 1958 special_alt->orig_off);
2009 if (!orig_insn) { 1959 if (!orig_insn) {
2010 WARN_FUNC("special: c 1960 WARN_FUNC("special: can't find orig instruction",
2011 special_alt 1961 special_alt->orig_sec, special_alt->orig_off);
2012 ret = -1; 1962 ret = -1;
2013 goto out; 1963 goto out;
2014 } 1964 }
2015 1965
2016 new_insn = NULL; 1966 new_insn = NULL;
2017 if (!special_alt->group || sp 1967 if (!special_alt->group || special_alt->new_len) {
2018 new_insn = find_insn( 1968 new_insn = find_insn(file, special_alt->new_sec,
2019 1969 special_alt->new_off);
2020 if (!new_insn) { 1970 if (!new_insn) {
2021 WARN_FUNC("sp 1971 WARN_FUNC("special: can't find new instruction",
2022 spe 1972 special_alt->new_sec,
2023 spe 1973 special_alt->new_off);
2024 ret = -1; 1974 ret = -1;
2025 goto out; 1975 goto out;
2026 } 1976 }
2027 } 1977 }
2028 1978
2029 if (special_alt->group) { 1979 if (special_alt->group) {
2030 if (!special_alt->ori 1980 if (!special_alt->orig_len) {
2031 WARN_INSN(ori 1981 WARN_INSN(orig_insn, "empty alternative entry");
2032 continue; 1982 continue;
2033 } 1983 }
2034 1984
2035 ret = handle_group_al 1985 ret = handle_group_alt(file, special_alt, orig_insn,
2036 1986 &new_insn);
2037 if (ret) 1987 if (ret)
2038 goto out; 1988 goto out;
2039 } else if (special_alt->jump_ 1989 } else if (special_alt->jump_or_nop) {
2040 ret = handle_jump_alt 1990 ret = handle_jump_alt(file, special_alt, orig_insn,
2041 1991 &new_insn);
2042 if (ret) 1992 if (ret)
2043 goto out; 1993 goto out;
2044 } 1994 }
2045 1995
2046 alt = malloc(sizeof(*alt)); 1996 alt = malloc(sizeof(*alt));
2047 if (!alt) { 1997 if (!alt) {
2048 WARN("malloc failed") 1998 WARN("malloc failed");
2049 ret = -1; 1999 ret = -1;
2050 goto out; 2000 goto out;
2051 } 2001 }
2052 2002
2053 alt->insn = new_insn; 2003 alt->insn = new_insn;
2054 alt->skip_orig = special_alt- 2004 alt->skip_orig = special_alt->skip_orig;
2055 orig_insn->ignore_alts |= spe 2005 orig_insn->ignore_alts |= special_alt->skip_alt;
2056 alt->next = orig_insn->alts; 2006 alt->next = orig_insn->alts;
2057 orig_insn->alts = alt; 2007 orig_insn->alts = alt;
2058 2008
2059 list_del(&special_alt->list); 2009 list_del(&special_alt->list);
2060 free(special_alt); 2010 free(special_alt);
2061 } 2011 }
2062 2012
2063 if (opts.stats) { 2013 if (opts.stats) {
2064 printf("jl\\\tNOP\tJMP\n"); 2014 printf("jl\\\tNOP\tJMP\n");
2065 printf("short:\t%ld\t%ld\n", 2015 printf("short:\t%ld\t%ld\n", file->jl_nop_short, file->jl_short);
2066 printf("long:\t%ld\t%ld\n", f 2016 printf("long:\t%ld\t%ld\n", file->jl_nop_long, file->jl_long);
2067 } 2017 }
2068 2018
2069 out: 2019 out:
2070 return ret; 2020 return ret;
2071 } 2021 }
2072 2022
2073 static int add_jump_table(struct objtool_file 2023 static int add_jump_table(struct objtool_file *file, struct instruction *insn,
2074 struct reloc *next_ 2024 struct reloc *next_table)
2075 { 2025 {
2076 struct symbol *pfunc = insn_func(insn 2026 struct symbol *pfunc = insn_func(insn)->pfunc;
2077 struct reloc *table = insn_jump_table 2027 struct reloc *table = insn_jump_table(insn);
2078 struct instruction *dest_insn; 2028 struct instruction *dest_insn;
2079 unsigned int prev_offset = 0; 2029 unsigned int prev_offset = 0;
2080 struct reloc *reloc = table; 2030 struct reloc *reloc = table;
2081 struct alternative *alt; 2031 struct alternative *alt;
2082 2032
2083 /* 2033 /*
2084 * Each @reloc is a switch table relo 2034 * Each @reloc is a switch table relocation which points to the target
2085 * instruction. 2035 * instruction.
2086 */ 2036 */
2087 for_each_reloc_from(table->sec, reloc 2037 for_each_reloc_from(table->sec, reloc) {
2088 2038
2089 /* Check for the end of the t 2039 /* Check for the end of the table: */
2090 if (reloc != table && reloc = 2040 if (reloc != table && reloc == next_table)
2091 break; 2041 break;
2092 2042
2093 /* Make sure the table entrie 2043 /* Make sure the table entries are consecutive: */
2094 if (prev_offset && reloc_offs 2044 if (prev_offset && reloc_offset(reloc) != prev_offset + 8)
2095 break; 2045 break;
2096 2046
2097 /* Detect function pointers f 2047 /* Detect function pointers from contiguous objects: */
2098 if (reloc->sym->sec == pfunc- 2048 if (reloc->sym->sec == pfunc->sec &&
2099 reloc_addend(reloc) == pf 2049 reloc_addend(reloc) == pfunc->offset)
2100 break; 2050 break;
2101 2051
2102 dest_insn = find_insn(file, r 2052 dest_insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2103 if (!dest_insn) 2053 if (!dest_insn)
2104 break; 2054 break;
2105 2055
2106 /* Make sure the destination 2056 /* Make sure the destination is in the same function: */
2107 if (!insn_func(dest_insn) || 2057 if (!insn_func(dest_insn) || insn_func(dest_insn)->pfunc != pfunc)
2108 break; 2058 break;
2109 2059
2110 alt = malloc(sizeof(*alt)); 2060 alt = malloc(sizeof(*alt));
2111 if (!alt) { 2061 if (!alt) {
2112 WARN("malloc failed") 2062 WARN("malloc failed");
2113 return -1; 2063 return -1;
2114 } 2064 }
2115 2065
2116 alt->insn = dest_insn; 2066 alt->insn = dest_insn;
2117 alt->next = insn->alts; 2067 alt->next = insn->alts;
2118 insn->alts = alt; 2068 insn->alts = alt;
2119 prev_offset = reloc_offset(re 2069 prev_offset = reloc_offset(reloc);
2120 } 2070 }
2121 2071
2122 if (!prev_offset) { 2072 if (!prev_offset) {
2123 WARN_INSN(insn, "can't find s 2073 WARN_INSN(insn, "can't find switch jump table");
2124 return -1; 2074 return -1;
2125 } 2075 }
2126 2076
2127 return 0; 2077 return 0;
2128 } 2078 }
2129 2079
2130 /* 2080 /*
2131 * find_jump_table() - Given a dynamic jump, 2081 * find_jump_table() - Given a dynamic jump, find the switch jump table
2132 * associated with it. 2082 * associated with it.
2133 */ 2083 */
2134 static struct reloc *find_jump_table(struct o 2084 static struct reloc *find_jump_table(struct objtool_file *file,
2135 struct 2085 struct symbol *func,
2136 struct 2086 struct instruction *insn)
2137 { 2087 {
2138 struct reloc *table_reloc; 2088 struct reloc *table_reloc;
2139 struct instruction *dest_insn, *orig_ 2089 struct instruction *dest_insn, *orig_insn = insn;
2140 2090
2141 /* 2091 /*
2142 * Backward search using the @first_j 2092 * Backward search using the @first_jump_src links, these help avoid
2143 * much of the 'in between' code. Whi 2093 * much of the 'in between' code. Which avoids us getting confused by
2144 * it. 2094 * it.
2145 */ 2095 */
2146 for (; 2096 for (;
2147 insn && insn_func(insn) && insn_ 2097 insn && insn_func(insn) && insn_func(insn)->pfunc == func;
2148 insn = insn->first_jump_src ?: p 2098 insn = insn->first_jump_src ?: prev_insn_same_sym(file, insn)) {
2149 2099
2150 if (insn != orig_insn && insn 2100 if (insn != orig_insn && insn->type == INSN_JUMP_DYNAMIC)
2151 break; 2101 break;
2152 2102
2153 /* allow small jumps within t 2103 /* allow small jumps within the range */
2154 if (insn->type == INSN_JUMP_U 2104 if (insn->type == INSN_JUMP_UNCONDITIONAL &&
2155 insn->jump_dest && 2105 insn->jump_dest &&
2156 (insn->jump_dest->offset 2106 (insn->jump_dest->offset <= insn->offset ||
2157 insn->jump_dest->offset 2107 insn->jump_dest->offset > orig_insn->offset))
2158 break; 2108 break;
2159 2109
2160 table_reloc = arch_find_switc 2110 table_reloc = arch_find_switch_table(file, insn);
2161 if (!table_reloc) 2111 if (!table_reloc)
2162 continue; 2112 continue;
2163 dest_insn = find_insn(file, t 2113 dest_insn = find_insn(file, table_reloc->sym->sec, reloc_addend(table_reloc));
2164 if (!dest_insn || !insn_func( 2114 if (!dest_insn || !insn_func(dest_insn) || insn_func(dest_insn)->pfunc != func)
2165 continue; 2115 continue;
2166 2116
2167 return table_reloc; 2117 return table_reloc;
2168 } 2118 }
2169 2119
2170 return NULL; 2120 return NULL;
2171 } 2121 }
2172 2122
2173 /* 2123 /*
2174 * First pass: Mark the head of each jump tab 2124 * First pass: Mark the head of each jump table so that in the next pass,
2175 * we know when a given jump table ends and t 2125 * we know when a given jump table ends and the next one starts.
2176 */ 2126 */
2177 static void mark_func_jump_tables(struct objt 2127 static void mark_func_jump_tables(struct objtool_file *file,
2178 struct sy 2128 struct symbol *func)
2179 { 2129 {
2180 struct instruction *insn, *last = NUL 2130 struct instruction *insn, *last = NULL;
2181 struct reloc *reloc; 2131 struct reloc *reloc;
2182 2132
2183 func_for_each_insn(file, func, insn) 2133 func_for_each_insn(file, func, insn) {
2184 if (!last) 2134 if (!last)
2185 last = insn; 2135 last = insn;
2186 2136
2187 /* 2137 /*
2188 * Store back-pointers for un 2138 * Store back-pointers for unconditional forward jumps such
2189 * that find_jump_table() can 2139 * that find_jump_table() can back-track using those and
2190 * avoid some potentially con 2140 * avoid some potentially confusing code.
2191 */ 2141 */
2192 if (insn->type == INSN_JUMP_U 2142 if (insn->type == INSN_JUMP_UNCONDITIONAL && insn->jump_dest &&
2193 insn->offset > last->offs 2143 insn->offset > last->offset &&
2194 insn->jump_dest->offset > 2144 insn->jump_dest->offset > insn->offset &&
2195 !insn->jump_dest->first_j 2145 !insn->jump_dest->first_jump_src) {
2196 2146
2197 insn->jump_dest->firs 2147 insn->jump_dest->first_jump_src = insn;
2198 last = insn->jump_des 2148 last = insn->jump_dest;
2199 } 2149 }
2200 2150
2201 if (insn->type != INSN_JUMP_D 2151 if (insn->type != INSN_JUMP_DYNAMIC)
2202 continue; 2152 continue;
2203 2153
2204 reloc = find_jump_table(file, 2154 reloc = find_jump_table(file, func, insn);
2205 if (reloc) 2155 if (reloc)
2206 insn->_jump_table = r 2156 insn->_jump_table = reloc;
2207 } 2157 }
2208 } 2158 }
2209 2159
2210 static int add_func_jump_tables(struct objtoo 2160 static int add_func_jump_tables(struct objtool_file *file,
2211 struct symb 2161 struct symbol *func)
2212 { 2162 {
2213 struct instruction *insn, *insn_t1 = 2163 struct instruction *insn, *insn_t1 = NULL, *insn_t2;
2214 int ret = 0; 2164 int ret = 0;
2215 2165
2216 func_for_each_insn(file, func, insn) 2166 func_for_each_insn(file, func, insn) {
2217 if (!insn_jump_table(insn)) 2167 if (!insn_jump_table(insn))
2218 continue; 2168 continue;
2219 2169
2220 if (!insn_t1) { 2170 if (!insn_t1) {
2221 insn_t1 = insn; 2171 insn_t1 = insn;
2222 continue; 2172 continue;
2223 } 2173 }
2224 2174
2225 insn_t2 = insn; 2175 insn_t2 = insn;
2226 2176
2227 ret = add_jump_table(file, in 2177 ret = add_jump_table(file, insn_t1, insn_jump_table(insn_t2));
2228 if (ret) 2178 if (ret)
2229 return ret; 2179 return ret;
2230 2180
2231 insn_t1 = insn_t2; 2181 insn_t1 = insn_t2;
2232 } 2182 }
2233 2183
2234 if (insn_t1) 2184 if (insn_t1)
2235 ret = add_jump_table(file, in 2185 ret = add_jump_table(file, insn_t1, NULL);
2236 2186
2237 return ret; 2187 return ret;
2238 } 2188 }
2239 2189
2240 /* 2190 /*
2241 * For some switch statements, gcc generates 2191 * For some switch statements, gcc generates a jump table in the .rodata
2242 * section which contains a list of addresses 2192 * section which contains a list of addresses within the function to jump to.
2243 * This finds these jump tables and adds them 2193 * This finds these jump tables and adds them to the insn->alts lists.
2244 */ 2194 */
2245 static int add_jump_table_alts(struct objtool 2195 static int add_jump_table_alts(struct objtool_file *file)
2246 { 2196 {
2247 struct symbol *func; 2197 struct symbol *func;
2248 int ret; 2198 int ret;
2249 2199
2250 if (!file->rodata) 2200 if (!file->rodata)
2251 return 0; 2201 return 0;
2252 2202
2253 for_each_sym(file, func) { 2203 for_each_sym(file, func) {
2254 if (func->type != STT_FUNC) 2204 if (func->type != STT_FUNC)
2255 continue; 2205 continue;
2256 2206
2257 mark_func_jump_tables(file, f 2207 mark_func_jump_tables(file, func);
2258 ret = add_func_jump_tables(fi 2208 ret = add_func_jump_tables(file, func);
2259 if (ret) 2209 if (ret)
2260 return ret; 2210 return ret;
2261 } 2211 }
2262 2212
2263 return 0; 2213 return 0;
2264 } 2214 }
2265 2215
2266 static void set_func_state(struct cfi_state * 2216 static void set_func_state(struct cfi_state *state)
2267 { 2217 {
2268 state->cfa = initial_func_cfi.cfa; 2218 state->cfa = initial_func_cfi.cfa;
2269 memcpy(&state->regs, &initial_func_cf 2219 memcpy(&state->regs, &initial_func_cfi.regs,
2270 CFI_NUM_REGS * sizeof(struct c 2220 CFI_NUM_REGS * sizeof(struct cfi_reg));
2271 state->stack_size = initial_func_cfi. 2221 state->stack_size = initial_func_cfi.cfa.offset;
2272 state->type = UNWIND_HINT_TYPE_CALL; 2222 state->type = UNWIND_HINT_TYPE_CALL;
2273 } 2223 }
2274 2224
2275 static int read_unwind_hints(struct objtool_f 2225 static int read_unwind_hints(struct objtool_file *file)
2276 { 2226 {
2277 struct cfi_state cfi = init_cfi; 2227 struct cfi_state cfi = init_cfi;
2278 struct section *sec; 2228 struct section *sec;
2279 struct unwind_hint *hint; 2229 struct unwind_hint *hint;
2280 struct instruction *insn; 2230 struct instruction *insn;
2281 struct reloc *reloc; 2231 struct reloc *reloc;
2282 unsigned long offset; 2232 unsigned long offset;
2283 int i; 2233 int i;
2284 2234
2285 sec = find_section_by_name(file->elf, 2235 sec = find_section_by_name(file->elf, ".discard.unwind_hints");
2286 if (!sec) 2236 if (!sec)
2287 return 0; 2237 return 0;
2288 2238
2289 if (!sec->rsec) { 2239 if (!sec->rsec) {
2290 WARN("missing .rela.discard.u 2240 WARN("missing .rela.discard.unwind_hints section");
2291 return -1; 2241 return -1;
2292 } 2242 }
2293 2243
2294 if (sec->sh.sh_size % sizeof(struct u 2244 if (sec->sh.sh_size % sizeof(struct unwind_hint)) {
2295 WARN("struct unwind_hint size 2245 WARN("struct unwind_hint size mismatch");
2296 return -1; 2246 return -1;
2297 } 2247 }
2298 2248
2299 file->hints = true; 2249 file->hints = true;
2300 2250
2301 for (i = 0; i < sec->sh.sh_size / siz 2251 for (i = 0; i < sec->sh.sh_size / sizeof(struct unwind_hint); i++) {
2302 hint = (struct unwind_hint *) 2252 hint = (struct unwind_hint *)sec->data->d_buf + i;
2303 2253
2304 reloc = find_reloc_by_dest(fi 2254 reloc = find_reloc_by_dest(file->elf, sec, i * sizeof(*hint));
2305 if (!reloc) { 2255 if (!reloc) {
2306 WARN("can't find relo 2256 WARN("can't find reloc for unwind_hints[%d]", i);
2307 return -1; 2257 return -1;
2308 } 2258 }
2309 2259
2310 if (reloc->sym->type == STT_S 2260 if (reloc->sym->type == STT_SECTION) {
2311 offset = reloc_addend 2261 offset = reloc_addend(reloc);
2312 } else if (reloc->sym->local_ 2262 } else if (reloc->sym->local_label) {
2313 offset = reloc->sym-> 2263 offset = reloc->sym->offset;
2314 } else { 2264 } else {
2315 WARN("unexpected relo 2265 WARN("unexpected relocation symbol type in %s", sec->rsec->name);
2316 return -1; 2266 return -1;
2317 } 2267 }
2318 2268
2319 insn = find_insn(file, reloc- 2269 insn = find_insn(file, reloc->sym->sec, offset);
2320 if (!insn) { 2270 if (!insn) {
2321 WARN("can't find insn 2271 WARN("can't find insn for unwind_hints[%d]", i);
2322 return -1; 2272 return -1;
2323 } 2273 }
2324 2274
2325 insn->hint = true; 2275 insn->hint = true;
2326 2276
2327 if (hint->type == UNWIND_HINT 2277 if (hint->type == UNWIND_HINT_TYPE_UNDEFINED) {
2328 insn->cfi = &force_un 2278 insn->cfi = &force_undefined_cfi;
2329 continue; 2279 continue;
2330 } 2280 }
2331 2281
2332 if (hint->type == UNWIND_HINT 2282 if (hint->type == UNWIND_HINT_TYPE_SAVE) {
2333 insn->hint = false; 2283 insn->hint = false;
2334 insn->save = true; 2284 insn->save = true;
2335 continue; 2285 continue;
2336 } 2286 }
2337 2287
2338 if (hint->type == UNWIND_HINT 2288 if (hint->type == UNWIND_HINT_TYPE_RESTORE) {
2339 insn->restore = true; 2289 insn->restore = true;
2340 continue; 2290 continue;
2341 } 2291 }
2342 2292
2343 if (hint->type == UNWIND_HINT 2293 if (hint->type == UNWIND_HINT_TYPE_REGS_PARTIAL) {
2344 struct symbol *sym = 2294 struct symbol *sym = find_symbol_by_offset(insn->sec, insn->offset);
2345 2295
2346 if (sym && sym->bind 2296 if (sym && sym->bind == STB_GLOBAL) {
2347 if (opts.ibt 2297 if (opts.ibt && insn->type != INSN_ENDBR && !insn->noendbr) {
2348 WARN_ 2298 WARN_INSN(insn, "UNWIND_HINT_IRET_REGS without ENDBR");
2349 } 2299 }
2350 } 2300 }
2351 } 2301 }
2352 2302
2353 if (hint->type == UNWIND_HINT 2303 if (hint->type == UNWIND_HINT_TYPE_FUNC) {
2354 insn->cfi = &func_cfi 2304 insn->cfi = &func_cfi;
2355 continue; 2305 continue;
2356 } 2306 }
2357 2307
2358 if (insn->cfi) 2308 if (insn->cfi)
2359 cfi = *(insn->cfi); 2309 cfi = *(insn->cfi);
2360 2310
2361 if (arch_decode_hint_reg(hint 2311 if (arch_decode_hint_reg(hint->sp_reg, &cfi.cfa.base)) {
2362 WARN_INSN(insn, "unsu 2312 WARN_INSN(insn, "unsupported unwind_hint sp base reg %d", hint->sp_reg);
2363 return -1; 2313 return -1;
2364 } 2314 }
2365 2315
2366 cfi.cfa.offset = bswap_if_nee 2316 cfi.cfa.offset = bswap_if_needed(file->elf, hint->sp_offset);
2367 cfi.type = hint->type; 2317 cfi.type = hint->type;
2368 cfi.signal = hint->signal; 2318 cfi.signal = hint->signal;
2369 2319
2370 insn->cfi = cfi_hash_find_or_ 2320 insn->cfi = cfi_hash_find_or_add(&cfi);
2371 } 2321 }
2372 2322
2373 return 0; 2323 return 0;
2374 } 2324 }
2375 2325
2376 static int read_noendbr_hints(struct objtool_ 2326 static int read_noendbr_hints(struct objtool_file *file)
2377 { 2327 {
2378 struct instruction *insn; 2328 struct instruction *insn;
2379 struct section *rsec; 2329 struct section *rsec;
2380 struct reloc *reloc; 2330 struct reloc *reloc;
2381 2331
2382 rsec = find_section_by_name(file->elf 2332 rsec = find_section_by_name(file->elf, ".rela.discard.noendbr");
2383 if (!rsec) 2333 if (!rsec)
2384 return 0; 2334 return 0;
2385 2335
2386 for_each_reloc(rsec, reloc) { 2336 for_each_reloc(rsec, reloc) {
2387 insn = find_insn(file, reloc- 2337 insn = find_insn(file, reloc->sym->sec,
2388 reloc->sym-> 2338 reloc->sym->offset + reloc_addend(reloc));
2389 if (!insn) { 2339 if (!insn) {
2390 WARN("bad .discard.no 2340 WARN("bad .discard.noendbr entry");
2391 return -1; 2341 return -1;
2392 } 2342 }
2393 2343
2394 insn->noendbr = 1; 2344 insn->noendbr = 1;
2395 } 2345 }
2396 2346
2397 return 0; 2347 return 0;
2398 } 2348 }
2399 2349
2400 static int read_retpoline_hints(struct objtoo 2350 static int read_retpoline_hints(struct objtool_file *file)
2401 { 2351 {
2402 struct section *rsec; 2352 struct section *rsec;
2403 struct instruction *insn; 2353 struct instruction *insn;
2404 struct reloc *reloc; 2354 struct reloc *reloc;
2405 2355
2406 rsec = find_section_by_name(file->elf 2356 rsec = find_section_by_name(file->elf, ".rela.discard.retpoline_safe");
2407 if (!rsec) 2357 if (!rsec)
2408 return 0; 2358 return 0;
2409 2359
2410 for_each_reloc(rsec, reloc) { 2360 for_each_reloc(rsec, reloc) {
2411 if (reloc->sym->type != STT_S 2361 if (reloc->sym->type != STT_SECTION) {
2412 WARN("unexpected relo 2362 WARN("unexpected relocation symbol type in %s", rsec->name);
2413 return -1; 2363 return -1;
2414 } 2364 }
2415 2365
2416 insn = find_insn(file, reloc- 2366 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2417 if (!insn) { 2367 if (!insn) {
2418 WARN("bad .discard.re 2368 WARN("bad .discard.retpoline_safe entry");
2419 return -1; 2369 return -1;
2420 } 2370 }
2421 2371
2422 if (insn->type != INSN_JUMP_D 2372 if (insn->type != INSN_JUMP_DYNAMIC &&
2423 insn->type != INSN_CALL_D 2373 insn->type != INSN_CALL_DYNAMIC &&
2424 insn->type != INSN_RETURN 2374 insn->type != INSN_RETURN &&
2425 insn->type != INSN_NOP) { 2375 insn->type != INSN_NOP) {
2426 WARN_INSN(insn, "retp 2376 WARN_INSN(insn, "retpoline_safe hint not an indirect jump/call/ret/nop");
2427 return -1; 2377 return -1;
2428 } 2378 }
2429 2379
2430 insn->retpoline_safe = true; 2380 insn->retpoline_safe = true;
2431 } 2381 }
2432 2382
2433 return 0; 2383 return 0;
2434 } 2384 }
2435 2385
2436 static int read_instr_hints(struct objtool_fi 2386 static int read_instr_hints(struct objtool_file *file)
2437 { 2387 {
2438 struct section *rsec; 2388 struct section *rsec;
2439 struct instruction *insn; 2389 struct instruction *insn;
2440 struct reloc *reloc; 2390 struct reloc *reloc;
2441 2391
2442 rsec = find_section_by_name(file->elf 2392 rsec = find_section_by_name(file->elf, ".rela.discard.instr_end");
2443 if (!rsec) 2393 if (!rsec)
2444 return 0; 2394 return 0;
2445 2395
2446 for_each_reloc(rsec, reloc) { 2396 for_each_reloc(rsec, reloc) {
2447 if (reloc->sym->type != STT_S 2397 if (reloc->sym->type != STT_SECTION) {
2448 WARN("unexpected relo 2398 WARN("unexpected relocation symbol type in %s", rsec->name);
2449 return -1; 2399 return -1;
2450 } 2400 }
2451 2401
2452 insn = find_insn(file, reloc- 2402 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2453 if (!insn) { 2403 if (!insn) {
2454 WARN("bad .discard.in 2404 WARN("bad .discard.instr_end entry");
2455 return -1; 2405 return -1;
2456 } 2406 }
2457 2407
2458 insn->instr--; 2408 insn->instr--;
2459 } 2409 }
2460 2410
2461 rsec = find_section_by_name(file->elf 2411 rsec = find_section_by_name(file->elf, ".rela.discard.instr_begin");
2462 if (!rsec) 2412 if (!rsec)
2463 return 0; 2413 return 0;
2464 2414
2465 for_each_reloc(rsec, reloc) { 2415 for_each_reloc(rsec, reloc) {
2466 if (reloc->sym->type != STT_S 2416 if (reloc->sym->type != STT_SECTION) {
2467 WARN("unexpected relo 2417 WARN("unexpected relocation symbol type in %s", rsec->name);
2468 return -1; 2418 return -1;
2469 } 2419 }
2470 2420
2471 insn = find_insn(file, reloc- 2421 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2472 if (!insn) { 2422 if (!insn) {
2473 WARN("bad .discard.in 2423 WARN("bad .discard.instr_begin entry");
2474 return -1; 2424 return -1;
2475 } 2425 }
2476 2426
2477 insn->instr++; 2427 insn->instr++;
2478 } 2428 }
2479 2429
2480 return 0; 2430 return 0;
2481 } 2431 }
2482 2432
2483 static int read_validate_unret_hints(struct o 2433 static int read_validate_unret_hints(struct objtool_file *file)
2484 { 2434 {
2485 struct section *rsec; 2435 struct section *rsec;
2486 struct instruction *insn; 2436 struct instruction *insn;
2487 struct reloc *reloc; 2437 struct reloc *reloc;
2488 2438
2489 rsec = find_section_by_name(file->elf 2439 rsec = find_section_by_name(file->elf, ".rela.discard.validate_unret");
2490 if (!rsec) 2440 if (!rsec)
2491 return 0; 2441 return 0;
2492 2442
2493 for_each_reloc(rsec, reloc) { 2443 for_each_reloc(rsec, reloc) {
2494 if (reloc->sym->type != STT_S 2444 if (reloc->sym->type != STT_SECTION) {
2495 WARN("unexpected relo 2445 WARN("unexpected relocation symbol type in %s", rsec->name);
2496 return -1; 2446 return -1;
2497 } 2447 }
2498 2448
2499 insn = find_insn(file, reloc- 2449 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2500 if (!insn) { 2450 if (!insn) {
2501 WARN("bad .discard.in 2451 WARN("bad .discard.instr_end entry");
2502 return -1; 2452 return -1;
2503 } 2453 }
2504 insn->unret = 1; 2454 insn->unret = 1;
2505 } 2455 }
2506 2456
2507 return 0; 2457 return 0;
2508 } 2458 }
2509 2459
2510 2460
2511 static int read_intra_function_calls(struct o 2461 static int read_intra_function_calls(struct objtool_file *file)
2512 { 2462 {
2513 struct instruction *insn; 2463 struct instruction *insn;
2514 struct section *rsec; 2464 struct section *rsec;
2515 struct reloc *reloc; 2465 struct reloc *reloc;
2516 2466
2517 rsec = find_section_by_name(file->elf 2467 rsec = find_section_by_name(file->elf, ".rela.discard.intra_function_calls");
2518 if (!rsec) 2468 if (!rsec)
2519 return 0; 2469 return 0;
2520 2470
2521 for_each_reloc(rsec, reloc) { 2471 for_each_reloc(rsec, reloc) {
2522 unsigned long dest_off; 2472 unsigned long dest_off;
2523 2473
2524 if (reloc->sym->type != STT_S 2474 if (reloc->sym->type != STT_SECTION) {
2525 WARN("unexpected relo 2475 WARN("unexpected relocation symbol type in %s",
2526 rsec->name); 2476 rsec->name);
2527 return -1; 2477 return -1;
2528 } 2478 }
2529 2479
2530 insn = find_insn(file, reloc- 2480 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2531 if (!insn) { 2481 if (!insn) {
2532 WARN("bad .discard.in 2482 WARN("bad .discard.intra_function_call entry");
2533 return -1; 2483 return -1;
2534 } 2484 }
2535 2485
2536 if (insn->type != INSN_CALL) 2486 if (insn->type != INSN_CALL) {
2537 WARN_INSN(insn, "intr 2487 WARN_INSN(insn, "intra_function_call not a direct call");
2538 return -1; 2488 return -1;
2539 } 2489 }
2540 2490
2541 /* 2491 /*
2542 * Treat intra-function CALLs 2492 * Treat intra-function CALLs as JMPs, but with a stack_op.
2543 * See add_call_destinations( 2493 * See add_call_destinations(), which strips stack_ops from
2544 * normal CALLs. 2494 * normal CALLs.
2545 */ 2495 */
2546 insn->type = INSN_JUMP_UNCOND 2496 insn->type = INSN_JUMP_UNCONDITIONAL;
2547 2497
2548 dest_off = arch_jump_destinat 2498 dest_off = arch_jump_destination(insn);
2549 insn->jump_dest = find_insn(f 2499 insn->jump_dest = find_insn(file, insn->sec, dest_off);
2550 if (!insn->jump_dest) { 2500 if (!insn->jump_dest) {
2551 WARN_INSN(insn, "can' 2501 WARN_INSN(insn, "can't find call dest at %s+0x%lx",
2552 insn->sec-> 2502 insn->sec->name, dest_off);
2553 return -1; 2503 return -1;
2554 } 2504 }
2555 } 2505 }
2556 2506
2557 return 0; 2507 return 0;
2558 } 2508 }
2559 2509
2560 /* 2510 /*
2561 * Return true if name matches an instrumenta 2511 * Return true if name matches an instrumentation function, where calls to that
2562 * function from noinstr code can safely be r 2512 * function from noinstr code can safely be removed, but compilers won't do so.
2563 */ 2513 */
2564 static bool is_profiling_func(const char *nam 2514 static bool is_profiling_func(const char *name)
2565 { 2515 {
2566 /* 2516 /*
2567 * Many compilers cannot disable KCOV 2517 * Many compilers cannot disable KCOV with a function attribute.
2568 */ 2518 */
2569 if (!strncmp(name, "__sanitizer_cov_" 2519 if (!strncmp(name, "__sanitizer_cov_", 16))
2570 return true; 2520 return true;
2571 2521
2572 /* 2522 /*
2573 * Some compilers currently do not re 2523 * Some compilers currently do not remove __tsan_func_entry/exit nor
2574 * __tsan_atomic_signal_fence (used f 2524 * __tsan_atomic_signal_fence (used for barrier instrumentation) with
2575 * the __no_sanitize_thread attribute 2525 * the __no_sanitize_thread attribute, remove them. Once the kernel's
2576 * minimum Clang version is 14.0, thi 2526 * minimum Clang version is 14.0, this can be removed.
2577 */ 2527 */
2578 if (!strncmp(name, "__tsan_func_", 12 2528 if (!strncmp(name, "__tsan_func_", 12) ||
2579 !strcmp(name, "__tsan_atomic_sign 2529 !strcmp(name, "__tsan_atomic_signal_fence"))
2580 return true; 2530 return true;
2581 2531
2582 return false; 2532 return false;
2583 } 2533 }
2584 2534
2585 static int classify_symbols(struct objtool_fi 2535 static int classify_symbols(struct objtool_file *file)
2586 { 2536 {
2587 struct symbol *func; 2537 struct symbol *func;
2588 2538
2589 for_each_sym(file, func) { 2539 for_each_sym(file, func) {
2590 if (func->type == STT_NOTYPE 2540 if (func->type == STT_NOTYPE && strstarts(func->name, ".L"))
2591 func->local_label = t 2541 func->local_label = true;
2592 2542
2593 if (func->bind != STB_GLOBAL) 2543 if (func->bind != STB_GLOBAL)
2594 continue; 2544 continue;
2595 2545
2596 if (!strncmp(func->name, STAT 2546 if (!strncmp(func->name, STATIC_CALL_TRAMP_PREFIX_STR,
2597 strlen(STATIC_CA 2547 strlen(STATIC_CALL_TRAMP_PREFIX_STR)))
2598 func->static_call_tra 2548 func->static_call_tramp = true;
2599 2549
2600 if (arch_is_retpoline(func)) 2550 if (arch_is_retpoline(func))
2601 func->retpoline_thunk 2551 func->retpoline_thunk = true;
2602 2552
2603 if (arch_is_rethunk(func)) 2553 if (arch_is_rethunk(func))
2604 func->return_thunk = 2554 func->return_thunk = true;
2605 2555
2606 if (arch_is_embedded_insn(fun 2556 if (arch_is_embedded_insn(func))
2607 func->embedded_insn = 2557 func->embedded_insn = true;
2608 2558
2609 if (arch_ftrace_match(func->n 2559 if (arch_ftrace_match(func->name))
2610 func->fentry = true; 2560 func->fentry = true;
2611 2561
2612 if (is_profiling_func(func->n 2562 if (is_profiling_func(func->name))
2613 func->profiling_func 2563 func->profiling_func = true;
2614 } 2564 }
2615 2565
2616 return 0; 2566 return 0;
2617 } 2567 }
2618 2568
2619 static void mark_rodata(struct objtool_file * 2569 static void mark_rodata(struct objtool_file *file)
2620 { 2570 {
2621 struct section *sec; 2571 struct section *sec;
2622 bool found = false; 2572 bool found = false;
2623 2573
2624 /* 2574 /*
2625 * Search for the following rodata se 2575 * Search for the following rodata sections, each of which can
2626 * potentially contain jump tables: 2576 * potentially contain jump tables:
2627 * 2577 *
2628 * - .rodata: can contain GCC switch 2578 * - .rodata: can contain GCC switch tables
2629 * - .rodata.<func>: same, if -fdata- 2579 * - .rodata.<func>: same, if -fdata-sections is being used
2630 * - .rodata..c_jump_table: contains 2580 * - .rodata..c_jump_table: contains C annotated jump tables
2631 * 2581 *
2632 * .rodata.str1.* sections are ignore 2582 * .rodata.str1.* sections are ignored; they don't contain jump tables.
2633 */ 2583 */
2634 for_each_sec(file, sec) { 2584 for_each_sec(file, sec) {
2635 if (!strncmp(sec->name, ".rod 2585 if (!strncmp(sec->name, ".rodata", 7) &&
2636 !strstr(sec->name, ".str1 2586 !strstr(sec->name, ".str1.")) {
2637 sec->rodata = true; 2587 sec->rodata = true;
2638 found = true; 2588 found = true;
2639 } 2589 }
2640 } 2590 }
2641 2591
2642 file->rodata = found; 2592 file->rodata = found;
2643 } 2593 }
2644 2594
2645 static int decode_sections(struct objtool_fil 2595 static int decode_sections(struct objtool_file *file)
2646 { 2596 {
2647 int ret; 2597 int ret;
2648 2598
2649 mark_rodata(file); 2599 mark_rodata(file);
2650 2600
2651 ret = init_pv_ops(file); 2601 ret = init_pv_ops(file);
2652 if (ret) 2602 if (ret)
2653 return ret; 2603 return ret;
2654 2604
2655 /* 2605 /*
2656 * Must be before add_{jump_call}_des 2606 * Must be before add_{jump_call}_destination.
2657 */ 2607 */
2658 ret = classify_symbols(file); 2608 ret = classify_symbols(file);
2659 if (ret) 2609 if (ret)
2660 return ret; 2610 return ret;
2661 2611
2662 ret = decode_instructions(file); 2612 ret = decode_instructions(file);
2663 if (ret) 2613 if (ret)
2664 return ret; 2614 return ret;
2665 2615
2666 add_ignores(file); 2616 add_ignores(file);
2667 add_uaccess_safe(file); 2617 add_uaccess_safe(file);
2668 2618
2669 ret = add_ignore_alternatives(file); 2619 ret = add_ignore_alternatives(file);
2670 if (ret) 2620 if (ret)
2671 return ret; 2621 return ret;
2672 2622
2673 /* 2623 /*
2674 * Must be before read_unwind_hints() 2624 * Must be before read_unwind_hints() since that needs insn->noendbr.
2675 */ 2625 */
2676 ret = read_noendbr_hints(file); 2626 ret = read_noendbr_hints(file);
2677 if (ret) 2627 if (ret)
2678 return ret; 2628 return ret;
2679 2629
2680 /* 2630 /*
2681 * Must be before add_jump_destinatio 2631 * Must be before add_jump_destinations(), which depends on 'func'
2682 * being set for alternatives, to ena 2632 * being set for alternatives, to enable proper sibling call detection.
2683 */ 2633 */
2684 if (opts.stackval || opts.orc || opts 2634 if (opts.stackval || opts.orc || opts.uaccess || opts.noinstr) {
2685 ret = add_special_section_alt 2635 ret = add_special_section_alts(file);
2686 if (ret) 2636 if (ret)
2687 return ret; 2637 return ret;
2688 } 2638 }
2689 2639
2690 ret = add_jump_destinations(file); 2640 ret = add_jump_destinations(file);
2691 if (ret) 2641 if (ret)
2692 return ret; 2642 return ret;
2693 2643
2694 /* 2644 /*
2695 * Must be before add_call_destinatio 2645 * Must be before add_call_destination(); it changes INSN_CALL to
2696 * INSN_JUMP. 2646 * INSN_JUMP.
2697 */ 2647 */
2698 ret = read_intra_function_calls(file) 2648 ret = read_intra_function_calls(file);
2699 if (ret) 2649 if (ret)
2700 return ret; 2650 return ret;
2701 2651
2702 ret = add_call_destinations(file); 2652 ret = add_call_destinations(file);
2703 if (ret) 2653 if (ret)
2704 return ret; 2654 return ret;
2705 2655
2706 /* 2656 /*
2707 * Must be after add_call_destination 2657 * Must be after add_call_destinations() such that it can override
2708 * dead_end_function() marks. 2658 * dead_end_function() marks.
2709 */ 2659 */
2710 ret = add_dead_ends(file); 2660 ret = add_dead_ends(file);
2711 if (ret) 2661 if (ret)
2712 return ret; 2662 return ret;
2713 2663
2714 ret = add_jump_table_alts(file); 2664 ret = add_jump_table_alts(file);
2715 if (ret) 2665 if (ret)
2716 return ret; 2666 return ret;
2717 2667
2718 ret = read_unwind_hints(file); 2668 ret = read_unwind_hints(file);
2719 if (ret) 2669 if (ret)
2720 return ret; 2670 return ret;
2721 2671
2722 ret = read_retpoline_hints(file); 2672 ret = read_retpoline_hints(file);
2723 if (ret) 2673 if (ret)
2724 return ret; 2674 return ret;
2725 2675
2726 ret = read_instr_hints(file); 2676 ret = read_instr_hints(file);
2727 if (ret) 2677 if (ret)
2728 return ret; 2678 return ret;
2729 2679
2730 ret = read_validate_unret_hints(file) 2680 ret = read_validate_unret_hints(file);
2731 if (ret) 2681 if (ret)
2732 return ret; 2682 return ret;
2733 2683
2734 return 0; 2684 return 0;
2735 } 2685 }
2736 2686
2737 static bool is_special_call(struct instructio 2687 static bool is_special_call(struct instruction *insn)
2738 { 2688 {
2739 if (insn->type == INSN_CALL) { 2689 if (insn->type == INSN_CALL) {
2740 struct symbol *dest = insn_ca 2690 struct symbol *dest = insn_call_dest(insn);
2741 2691
2742 if (!dest) 2692 if (!dest)
2743 return false; 2693 return false;
2744 2694
2745 if (dest->fentry || dest->emb 2695 if (dest->fentry || dest->embedded_insn)
2746 return true; 2696 return true;
2747 } 2697 }
2748 2698
2749 return false; 2699 return false;
2750 } 2700 }
2751 2701
2752 static bool has_modified_stack_frame(struct i 2702 static bool has_modified_stack_frame(struct instruction *insn, struct insn_state *state)
2753 { 2703 {
2754 struct cfi_state *cfi = &state->cfi; 2704 struct cfi_state *cfi = &state->cfi;
2755 int i; 2705 int i;
2756 2706
2757 if (cfi->cfa.base != initial_func_cfi 2707 if (cfi->cfa.base != initial_func_cfi.cfa.base || cfi->drap)
2758 return true; 2708 return true;
2759 2709
2760 if (cfi->cfa.offset != initial_func_c 2710 if (cfi->cfa.offset != initial_func_cfi.cfa.offset)
2761 return true; 2711 return true;
2762 2712
2763 if (cfi->stack_size != initial_func_c 2713 if (cfi->stack_size != initial_func_cfi.cfa.offset)
2764 return true; 2714 return true;
2765 2715
2766 for (i = 0; i < CFI_NUM_REGS; i++) { 2716 for (i = 0; i < CFI_NUM_REGS; i++) {
2767 if (cfi->regs[i].base != init 2717 if (cfi->regs[i].base != initial_func_cfi.regs[i].base ||
2768 cfi->regs[i].offset != in 2718 cfi->regs[i].offset != initial_func_cfi.regs[i].offset)
2769 return true; 2719 return true;
2770 } 2720 }
2771 2721
2772 return false; 2722 return false;
2773 } 2723 }
2774 2724
2775 static bool check_reg_frame_pos(const struct 2725 static bool check_reg_frame_pos(const struct cfi_reg *reg,
2776 int expected_ 2726 int expected_offset)
2777 { 2727 {
2778 return reg->base == CFI_CFA && 2728 return reg->base == CFI_CFA &&
2779 reg->offset == expected_offset 2729 reg->offset == expected_offset;
2780 } 2730 }
2781 2731
2782 static bool has_valid_stack_frame(struct insn 2732 static bool has_valid_stack_frame(struct insn_state *state)
2783 { 2733 {
2784 struct cfi_state *cfi = &state->cfi; 2734 struct cfi_state *cfi = &state->cfi;
2785 2735
2786 if (cfi->cfa.base == CFI_BP && 2736 if (cfi->cfa.base == CFI_BP &&
2787 check_reg_frame_pos(&cfi->regs[CF 2737 check_reg_frame_pos(&cfi->regs[CFI_BP], -cfi->cfa.offset) &&
2788 check_reg_frame_pos(&cfi->regs[CF 2738 check_reg_frame_pos(&cfi->regs[CFI_RA], -cfi->cfa.offset + 8))
2789 return true; 2739 return true;
2790 2740
2791 if (cfi->drap && cfi->regs[CFI_BP].ba 2741 if (cfi->drap && cfi->regs[CFI_BP].base == CFI_BP)
2792 return true; 2742 return true;
2793 2743
2794 return false; 2744 return false;
2795 } 2745 }
2796 2746
2797 static int update_cfi_state_regs(struct instr 2747 static int update_cfi_state_regs(struct instruction *insn,
2798 struct cfi_ 2748 struct cfi_state *cfi,
2799 struct stac 2749 struct stack_op *op)
2800 { 2750 {
2801 struct cfi_reg *cfa = &cfi->cfa; 2751 struct cfi_reg *cfa = &cfi->cfa;
2802 2752
2803 if (cfa->base != CFI_SP && cfa->base 2753 if (cfa->base != CFI_SP && cfa->base != CFI_SP_INDIRECT)
2804 return 0; 2754 return 0;
2805 2755
2806 /* push */ 2756 /* push */
2807 if (op->dest.type == OP_DEST_PUSH || 2757 if (op->dest.type == OP_DEST_PUSH || op->dest.type == OP_DEST_PUSHF)
2808 cfa->offset += 8; 2758 cfa->offset += 8;
2809 2759
2810 /* pop */ 2760 /* pop */
2811 if (op->src.type == OP_SRC_POP || op- 2761 if (op->src.type == OP_SRC_POP || op->src.type == OP_SRC_POPF)
2812 cfa->offset -= 8; 2762 cfa->offset -= 8;
2813 2763
2814 /* add immediate to sp */ 2764 /* add immediate to sp */
2815 if (op->dest.type == OP_DEST_REG && o 2765 if (op->dest.type == OP_DEST_REG && op->src.type == OP_SRC_ADD &&
2816 op->dest.reg == CFI_SP && op->src 2766 op->dest.reg == CFI_SP && op->src.reg == CFI_SP)
2817 cfa->offset -= op->src.offset 2767 cfa->offset -= op->src.offset;
2818 2768
2819 return 0; 2769 return 0;
2820 } 2770 }
2821 2771
2822 static void save_reg(struct cfi_state *cfi, u 2772 static void save_reg(struct cfi_state *cfi, unsigned char reg, int base, int offset)
2823 { 2773 {
2824 if (arch_callee_saved_reg(reg) && 2774 if (arch_callee_saved_reg(reg) &&
2825 cfi->regs[reg].base == CFI_UNDEFI 2775 cfi->regs[reg].base == CFI_UNDEFINED) {
2826 cfi->regs[reg].base = base; 2776 cfi->regs[reg].base = base;
2827 cfi->regs[reg].offset = offse 2777 cfi->regs[reg].offset = offset;
2828 } 2778 }
2829 } 2779 }
2830 2780
2831 static void restore_reg(struct cfi_state *cfi 2781 static void restore_reg(struct cfi_state *cfi, unsigned char reg)
2832 { 2782 {
2833 cfi->regs[reg].base = initial_func_cf 2783 cfi->regs[reg].base = initial_func_cfi.regs[reg].base;
2834 cfi->regs[reg].offset = initial_func_ 2784 cfi->regs[reg].offset = initial_func_cfi.regs[reg].offset;
2835 } 2785 }
2836 2786
2837 /* 2787 /*
2838 * A note about DRAP stack alignment: 2788 * A note about DRAP stack alignment:
2839 * 2789 *
2840 * GCC has the concept of a DRAP register, wh 2790 * GCC has the concept of a DRAP register, which is used to help keep track of
2841 * the stack pointer when aligning the stack. 2791 * the stack pointer when aligning the stack. r10 or r13 is used as the DRAP
2842 * register. The typical DRAP pattern is: 2792 * register. The typical DRAP pattern is:
2843 * 2793 *
2844 * 4c 8d 54 24 08 lea 0x8(%r 2794 * 4c 8d 54 24 08 lea 0x8(%rsp),%r10
2845 * 48 83 e4 c0 and $0xfff 2795 * 48 83 e4 c0 and $0xffffffffffffffc0,%rsp
2846 * 41 ff 72 f8 pushq -0x8(% 2796 * 41 ff 72 f8 pushq -0x8(%r10)
2847 * 55 push %rbp 2797 * 55 push %rbp
2848 * 48 89 e5 mov %rsp,% 2798 * 48 89 e5 mov %rsp,%rbp
2849 * (more pushes) 2799 * (more pushes)
2850 * 41 52 push %r10 2800 * 41 52 push %r10
2851 * ... 2801 * ...
2852 * 41 5a pop %r10 2802 * 41 5a pop %r10
2853 * (more pops) 2803 * (more pops)
2854 * 5d pop %rbp 2804 * 5d pop %rbp
2855 * 49 8d 62 f8 lea -0x8(% 2805 * 49 8d 62 f8 lea -0x8(%r10),%rsp
2856 * c3 retq 2806 * c3 retq
2857 * 2807 *
2858 * There are some variations in the epilogues 2808 * There are some variations in the epilogues, like:
2859 * 2809 *
2860 * 5b pop %rbx 2810 * 5b pop %rbx
2861 * 41 5a pop %r10 2811 * 41 5a pop %r10
2862 * 41 5c pop %r12 2812 * 41 5c pop %r12
2863 * 41 5d pop %r13 2813 * 41 5d pop %r13
2864 * 41 5e pop %r14 2814 * 41 5e pop %r14
2865 * c9 leaveq 2815 * c9 leaveq
2866 * 49 8d 62 f8 lea -0x8(% 2816 * 49 8d 62 f8 lea -0x8(%r10),%rsp
2867 * c3 retq 2817 * c3 retq
2868 * 2818 *
2869 * and: 2819 * and:
2870 * 2820 *
2871 * 4c 8b 55 e8 mov -0x18( 2821 * 4c 8b 55 e8 mov -0x18(%rbp),%r10
2872 * 48 8b 5d e0 mov -0x20( 2822 * 48 8b 5d e0 mov -0x20(%rbp),%rbx
2873 * 4c 8b 65 f0 mov -0x10( 2823 * 4c 8b 65 f0 mov -0x10(%rbp),%r12
2874 * 4c 8b 6d f8 mov -0x8(% 2824 * 4c 8b 6d f8 mov -0x8(%rbp),%r13
2875 * c9 leaveq 2825 * c9 leaveq
2876 * 49 8d 62 f8 lea -0x8(% 2826 * 49 8d 62 f8 lea -0x8(%r10),%rsp
2877 * c3 retq 2827 * c3 retq
2878 * 2828 *
2879 * Sometimes r13 is used as the DRAP register 2829 * Sometimes r13 is used as the DRAP register, in which case it's saved and
2880 * restored beforehand: 2830 * restored beforehand:
2881 * 2831 *
2882 * 41 55 push %r13 2832 * 41 55 push %r13
2883 * 4c 8d 6c 24 10 lea 0x10(% 2833 * 4c 8d 6c 24 10 lea 0x10(%rsp),%r13
2884 * 48 83 e4 f0 and $0xfff 2834 * 48 83 e4 f0 and $0xfffffffffffffff0,%rsp
2885 * ... 2835 * ...
2886 * 49 8d 65 f0 lea -0x10( 2836 * 49 8d 65 f0 lea -0x10(%r13),%rsp
2887 * 41 5d pop %r13 2837 * 41 5d pop %r13
2888 * c3 retq 2838 * c3 retq
2889 */ 2839 */
2890 static int update_cfi_state(struct instructio 2840 static int update_cfi_state(struct instruction *insn,
2891 struct instructio 2841 struct instruction *next_insn,
2892 struct cfi_state 2842 struct cfi_state *cfi, struct stack_op *op)
2893 { 2843 {
2894 struct cfi_reg *cfa = &cfi->cfa; 2844 struct cfi_reg *cfa = &cfi->cfa;
2895 struct cfi_reg *regs = cfi->regs; 2845 struct cfi_reg *regs = cfi->regs;
2896 2846
2897 /* ignore UNWIND_HINT_UNDEFINED regio 2847 /* ignore UNWIND_HINT_UNDEFINED regions */
2898 if (cfi->force_undefined) 2848 if (cfi->force_undefined)
2899 return 0; 2849 return 0;
2900 2850
2901 /* stack operations don't make sense 2851 /* stack operations don't make sense with an undefined CFA */
2902 if (cfa->base == CFI_UNDEFINED) { 2852 if (cfa->base == CFI_UNDEFINED) {
2903 if (insn_func(insn)) { 2853 if (insn_func(insn)) {
2904 WARN_INSN(insn, "unde 2854 WARN_INSN(insn, "undefined stack state");
2905 return -1; 2855 return -1;
2906 } 2856 }
2907 return 0; 2857 return 0;
2908 } 2858 }
2909 2859
2910 if (cfi->type == UNWIND_HINT_TYPE_REG 2860 if (cfi->type == UNWIND_HINT_TYPE_REGS ||
2911 cfi->type == UNWIND_HINT_TYPE_REG 2861 cfi->type == UNWIND_HINT_TYPE_REGS_PARTIAL)
2912 return update_cfi_state_regs( 2862 return update_cfi_state_regs(insn, cfi, op);
2913 2863
2914 switch (op->dest.type) { 2864 switch (op->dest.type) {
2915 2865
2916 case OP_DEST_REG: 2866 case OP_DEST_REG:
2917 switch (op->src.type) { 2867 switch (op->src.type) {
2918 2868
2919 case OP_SRC_REG: 2869 case OP_SRC_REG:
2920 if (op->src.reg == CF 2870 if (op->src.reg == CFI_SP && op->dest.reg == CFI_BP &&
2921 cfa->base == CFI_ 2871 cfa->base == CFI_SP &&
2922 check_reg_frame_p 2872 check_reg_frame_pos(®s[CFI_BP], -cfa->offset)) {
2923 2873
2924 /* mov %rsp, 2874 /* mov %rsp, %rbp */
2925 cfa->base = o 2875 cfa->base = op->dest.reg;
2926 cfi->bp_scrat 2876 cfi->bp_scratch = false;
2927 } 2877 }
2928 2878
2929 else if (op->src.reg 2879 else if (op->src.reg == CFI_SP &&
2930 op->dest.reg 2880 op->dest.reg == CFI_BP && cfi->drap) {
2931 2881
2932 /* drap: mov 2882 /* drap: mov %rsp, %rbp */
2933 regs[CFI_BP]. 2883 regs[CFI_BP].base = CFI_BP;
2934 regs[CFI_BP]. 2884 regs[CFI_BP].offset = -cfi->stack_size;
2935 cfi->bp_scrat 2885 cfi->bp_scratch = false;
2936 } 2886 }
2937 2887
2938 else if (op->src.reg 2888 else if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
2939 2889
2940 /* 2890 /*
2941 * mov %rsp, 2891 * mov %rsp, %reg
2942 * 2892 *
2943 * This is ne 2893 * This is needed for the rare case where GCC
2944 * does: 2894 * does:
2945 * 2895 *
2946 * mov % 2896 * mov %rsp, %rax
2947 * ... 2897 * ...
2948 * mov % 2898 * mov %rax, %rsp
2949 */ 2899 */
2950 cfi->vals[op- 2900 cfi->vals[op->dest.reg].base = CFI_CFA;
2951 cfi->vals[op- 2901 cfi->vals[op->dest.reg].offset = -cfi->stack_size;
2952 } 2902 }
2953 2903
2954 else if (op->src.reg 2904 else if (op->src.reg == CFI_BP && op->dest.reg == CFI_SP &&
2955 (cfa->base = 2905 (cfa->base == CFI_BP || cfa->base == cfi->drap_reg)) {
2956 2906
2957 /* 2907 /*
2958 * mov %rbp, 2908 * mov %rbp, %rsp
2959 * 2909 *
2960 * Restore th 2910 * Restore the original stack pointer (Clang).
2961 */ 2911 */
2962 cfi->stack_si 2912 cfi->stack_size = -cfi->regs[CFI_BP].offset;
2963 } 2913 }
2964 2914
2965 else if (op->dest.reg 2915 else if (op->dest.reg == cfa->base) {
2966 2916
2967 /* mov %reg, 2917 /* mov %reg, %rsp */
2968 if (cfa->base 2918 if (cfa->base == CFI_SP &&
2969 cfi->vals 2919 cfi->vals[op->src.reg].base == CFI_CFA) {
2970 2920
2971 /* 2921 /*
2972 * Th 2922 * This is needed for the rare case
2973 * wh 2923 * where GCC does something dumb like:
2974 * 2924 *
2975 * 2925 * lea 0x8(%rsp), %rcx
2976 * 2926 * ...
2977 * 2927 * mov %rcx, %rsp
2978 */ 2928 */
2979 cfa-> 2929 cfa->offset = -cfi->vals[op->src.reg].offset;
2980 cfi-> 2930 cfi->stack_size = cfa->offset;
2981 2931
2982 } else if (cf 2932 } else if (cfa->base == CFI_SP &&
2983 cf 2933 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2984 cf 2934 cfi->vals[op->src.reg].offset == cfa->offset) {
2985 2935
2986 /* 2936 /*
2987 * St 2937 * Stack swizzle:
2988 * 2938 *
2989 * 1: 2939 * 1: mov %rsp, (%[tos])
2990 * 2: 2940 * 2: mov %[tos], %rsp
2991 * 2941 * ...
2992 * 3: 2942 * 3: pop %rsp
2993 * 2943 *
2994 * Wh 2944 * Where:
2995 * 2945 *
2996 * 1 2946 * 1 - places a pointer to the previous
2997 * 2947 * stack at the Top-of-Stack of the
2998 * 2948 * new stack.
2999 * 2949 *
3000 * 2 2950 * 2 - switches to the new stack.
3001 * 2951 *
3002 * 3 2952 * 3 - pops the Top-of-Stack to restore
3003 * 2953 * the original stack.
3004 * 2954 *
3005 * No 2955 * Note: we set base to SP_INDIRECT
3006 * he 2956 * here and preserve offset. Therefore
3007 * wh 2957 * when the unwinder reaches ToS it
3008 * wi 2958 * will dereference SP and then add the
3009 * of 2959 * offset to find the next frame, IOW:
3010 * (% 2960 * (%rsp) + offset.
3011 */ 2961 */
3012 cfa-> 2962 cfa->base = CFI_SP_INDIRECT;
3013 2963
3014 } else { 2964 } else {
3015 cfa-> 2965 cfa->base = CFI_UNDEFINED;
3016 cfa-> 2966 cfa->offset = 0;
3017 } 2967 }
3018 } 2968 }
3019 2969
3020 else if (op->dest.reg 2970 else if (op->dest.reg == CFI_SP &&
3021 cfi->vals[op 2971 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
3022 cfi->vals[op 2972 cfi->vals[op->src.reg].offset == cfa->offset) {
3023 2973
3024 /* 2974 /*
3025 * The same s 2975 * The same stack swizzle case 2) as above. But
3026 * because we 2976 * because we can't change cfa->base, case 3)
3027 * will becom 2977 * will become a regular POP. Pretend we're a
3028 * PUSH so th 2978 * PUSH so things don't go unbalanced.
3029 */ 2979 */
3030 cfi->stack_si 2980 cfi->stack_size += 8;
3031 } 2981 }
3032 2982
3033 2983
3034 break; 2984 break;
3035 2985
3036 case OP_SRC_ADD: 2986 case OP_SRC_ADD:
3037 if (op->dest.reg == C 2987 if (op->dest.reg == CFI_SP && op->src.reg == CFI_SP) {
3038 2988
3039 /* add imm, % 2989 /* add imm, %rsp */
3040 cfi->stack_si 2990 cfi->stack_size -= op->src.offset;
3041 if (cfa->base 2991 if (cfa->base == CFI_SP)
3042 cfa-> 2992 cfa->offset -= op->src.offset;
3043 break; 2993 break;
3044 } 2994 }
3045 2995
3046 if (op->dest.reg == C 2996 if (op->dest.reg == CFI_BP && op->src.reg == CFI_SP &&
3047 insn->sym->frame_ 2997 insn->sym->frame_pointer) {
3048 /* addi.d fp, 2998 /* addi.d fp,sp,imm on LoongArch */
3049 if (cfa->base 2999 if (cfa->base == CFI_SP && cfa->offset == op->src.offset) {
3050 cfa-> 3000 cfa->base = CFI_BP;
3051 cfa-> 3001 cfa->offset = 0;
3052 } 3002 }
3053 break; 3003 break;
3054 } 3004 }
3055 3005
3056 if (op->dest.reg == C 3006 if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) {
3057 /* addi.d sp, 3007 /* addi.d sp,fp,imm on LoongArch */
3058 if (cfa->base 3008 if (cfa->base == CFI_BP && cfa->offset == 0) {
3059 if (i 3009 if (insn->sym->frame_pointer) {
3060 3010 cfa->base = CFI_SP;
3061 3011 cfa->offset = -op->src.offset;
3062 } 3012 }
3063 } else { 3013 } else {
3064 /* le 3014 /* lea disp(%rbp), %rsp */
3065 cfi-> 3015 cfi->stack_size = -(op->src.offset + regs[CFI_BP].offset);
3066 } 3016 }
3067 break; 3017 break;
3068 } 3018 }
3069 3019
3070 if (op->src.reg == CF 3020 if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
3071 3021
3072 /* drap: lea 3022 /* drap: lea disp(%rsp), %drap */
3073 cfi->drap_reg 3023 cfi->drap_reg = op->dest.reg;
3074 3024
3075 /* 3025 /*
3076 * lea disp(% 3026 * lea disp(%rsp), %reg
3077 * 3027 *
3078 * This is ne 3028 * This is needed for the rare case where GCC
3079 * does somet 3029 * does something dumb like:
3080 * 3030 *
3081 * lea 0 3031 * lea 0x8(%rsp), %rcx
3082 * ... 3032 * ...
3083 * mov % 3033 * mov %rcx, %rsp
3084 */ 3034 */
3085 cfi->vals[op- 3035 cfi->vals[op->dest.reg].base = CFI_CFA;
3086 cfi->vals[op- 3036 cfi->vals[op->dest.reg].offset = \
3087 -cfi- 3037 -cfi->stack_size + op->src.offset;
3088 3038
3089 break; 3039 break;
3090 } 3040 }
3091 3041
3092 if (cfi->drap && op-> 3042 if (cfi->drap && op->dest.reg == CFI_SP &&
3093 op->src.reg == cf 3043 op->src.reg == cfi->drap_reg) {
3094 3044
3095 /* drap: lea 3045 /* drap: lea disp(%drap), %rsp */
3096 cfa->base = C 3046 cfa->base = CFI_SP;
3097 cfa->offset = 3047 cfa->offset = cfi->stack_size = -op->src.offset;
3098 cfi->drap_reg 3048 cfi->drap_reg = CFI_UNDEFINED;
3099 cfi->drap = f 3049 cfi->drap = false;
3100 break; 3050 break;
3101 } 3051 }
3102 3052
3103 if (op->dest.reg == c 3053 if (op->dest.reg == cfi->cfa.base && !(next_insn && next_insn->hint)) {
3104 WARN_INSN(ins 3054 WARN_INSN(insn, "unsupported stack register modification");
3105 return -1; 3055 return -1;
3106 } 3056 }
3107 3057
3108 break; 3058 break;
3109 3059
3110 case OP_SRC_AND: 3060 case OP_SRC_AND:
3111 if (op->dest.reg != C 3061 if (op->dest.reg != CFI_SP ||
3112 (cfi->drap_reg != 3062 (cfi->drap_reg != CFI_UNDEFINED && cfa->base != CFI_SP) ||
3113 (cfi->drap_reg == 3063 (cfi->drap_reg == CFI_UNDEFINED && cfa->base != CFI_BP)) {
3114 WARN_INSN(ins 3064 WARN_INSN(insn, "unsupported stack pointer realignment");
3115 return -1; 3065 return -1;
3116 } 3066 }
3117 3067
3118 if (cfi->drap_reg != 3068 if (cfi->drap_reg != CFI_UNDEFINED) {
3119 /* drap: and 3069 /* drap: and imm, %rsp */
3120 cfa->base = c 3070 cfa->base = cfi->drap_reg;
3121 cfa->offset = 3071 cfa->offset = cfi->stack_size = 0;
3122 cfi->drap = t 3072 cfi->drap = true;
3123 } 3073 }
3124 3074
3125 /* 3075 /*
3126 * Older versions of 3076 * Older versions of GCC (4.8ish) realign the stack
3127 * without DRAP, with 3077 * without DRAP, with a frame pointer.
3128 */ 3078 */
3129 3079
3130 break; 3080 break;
3131 3081
3132 case OP_SRC_POP: 3082 case OP_SRC_POP:
3133 case OP_SRC_POPF: 3083 case OP_SRC_POPF:
3134 if (op->dest.reg == C 3084 if (op->dest.reg == CFI_SP && cfa->base == CFI_SP_INDIRECT) {
3135 3085
3136 /* pop %rsp; 3086 /* pop %rsp; # restore from a stack swizzle */
3137 cfa->base = C 3087 cfa->base = CFI_SP;
3138 break; 3088 break;
3139 } 3089 }
3140 3090
3141 if (!cfi->drap && op- 3091 if (!cfi->drap && op->dest.reg == cfa->base) {
3142 3092
3143 /* pop %rbp * 3093 /* pop %rbp */
3144 cfa->base = C 3094 cfa->base = CFI_SP;
3145 } 3095 }
3146 3096
3147 if (cfi->drap && cfa- 3097 if (cfi->drap && cfa->base == CFI_BP_INDIRECT &&
3148 op->dest.reg == c 3098 op->dest.reg == cfi->drap_reg &&
3149 cfi->drap_offset 3099 cfi->drap_offset == -cfi->stack_size) {
3150 3100
3151 /* drap: pop 3101 /* drap: pop %drap */
3152 cfa->base = c 3102 cfa->base = cfi->drap_reg;
3153 cfa->offset = 3103 cfa->offset = 0;
3154 cfi->drap_off 3104 cfi->drap_offset = -1;
3155 3105
3156 } else if (cfi->stack 3106 } else if (cfi->stack_size == -regs[op->dest.reg].offset) {
3157 3107
3158 /* pop %reg * 3108 /* pop %reg */
3159 restore_reg(c 3109 restore_reg(cfi, op->dest.reg);
3160 } 3110 }
3161 3111
3162 cfi->stack_size -= 8; 3112 cfi->stack_size -= 8;
3163 if (cfa->base == CFI_ 3113 if (cfa->base == CFI_SP)
3164 cfa->offset - 3114 cfa->offset -= 8;
3165 3115
3166 break; 3116 break;
3167 3117
3168 case OP_SRC_REG_INDIRECT: 3118 case OP_SRC_REG_INDIRECT:
3169 if (!cfi->drap && op- 3119 if (!cfi->drap && op->dest.reg == cfa->base &&
3170 op->dest.reg == C 3120 op->dest.reg == CFI_BP) {
3171 3121
3172 /* mov disp(% 3122 /* mov disp(%rsp), %rbp */
3173 cfa->base = C 3123 cfa->base = CFI_SP;
3174 cfa->offset = 3124 cfa->offset = cfi->stack_size;
3175 } 3125 }
3176 3126
3177 if (cfi->drap && op-> 3127 if (cfi->drap && op->src.reg == CFI_BP &&
3178 op->src.offset == 3128 op->src.offset == cfi->drap_offset) {
3179 3129
3180 /* drap: mov 3130 /* drap: mov disp(%rbp), %drap */
3181 cfa->base = c 3131 cfa->base = cfi->drap_reg;
3182 cfa->offset = 3132 cfa->offset = 0;
3183 cfi->drap_off 3133 cfi->drap_offset = -1;
3184 } 3134 }
3185 3135
3186 if (cfi->drap && op-> 3136 if (cfi->drap && op->src.reg == CFI_BP &&
3187 op->src.offset == 3137 op->src.offset == regs[op->dest.reg].offset) {
3188 3138
3189 /* drap: mov 3139 /* drap: mov disp(%rbp), %reg */
3190 restore_reg(c 3140 restore_reg(cfi, op->dest.reg);
3191 3141
3192 } else if (op->src.re 3142 } else if (op->src.reg == cfa->base &&
3193 op->src.offset == 3143 op->src.offset == regs[op->dest.reg].offset + cfa->offset) {
3194 3144
3195 /* mov disp(% 3145 /* mov disp(%rbp), %reg */
3196 /* mov disp(% 3146 /* mov disp(%rsp), %reg */
3197 restore_reg(c 3147 restore_reg(cfi, op->dest.reg);
3198 3148
3199 } else if (op->src.re 3149 } else if (op->src.reg == CFI_SP &&
3200 op->src.of 3150 op->src.offset == regs[op->dest.reg].offset + cfi->stack_size) {
3201 3151
3202 /* mov disp(% 3152 /* mov disp(%rsp), %reg */
3203 restore_reg(c 3153 restore_reg(cfi, op->dest.reg);
3204 } 3154 }
3205 3155
3206 break; 3156 break;
3207 3157
3208 default: 3158 default:
3209 WARN_INSN(insn, "unkn 3159 WARN_INSN(insn, "unknown stack-related instruction");
3210 return -1; 3160 return -1;
3211 } 3161 }
3212 3162
3213 break; 3163 break;
3214 3164
3215 case OP_DEST_PUSH: 3165 case OP_DEST_PUSH:
3216 case OP_DEST_PUSHF: 3166 case OP_DEST_PUSHF:
3217 cfi->stack_size += 8; 3167 cfi->stack_size += 8;
3218 if (cfa->base == CFI_SP) 3168 if (cfa->base == CFI_SP)
3219 cfa->offset += 8; 3169 cfa->offset += 8;
3220 3170
3221 if (op->src.type != OP_SRC_RE 3171 if (op->src.type != OP_SRC_REG)
3222 break; 3172 break;
3223 3173
3224 if (cfi->drap) { 3174 if (cfi->drap) {
3225 if (op->src.reg == cf 3175 if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
3226 3176
3227 /* drap: push 3177 /* drap: push %drap */
3228 cfa->base = C 3178 cfa->base = CFI_BP_INDIRECT;
3229 cfa->offset = 3179 cfa->offset = -cfi->stack_size;
3230 3180
3231 /* save drap 3181 /* save drap so we know when to restore it */
3232 cfi->drap_off 3182 cfi->drap_offset = -cfi->stack_size;
3233 3183
3234 } else if (op->src.re 3184 } else if (op->src.reg == CFI_BP && cfa->base == cfi->drap_reg) {
3235 3185
3236 /* drap: push 3186 /* drap: push %rbp */
3237 cfi->stack_si 3187 cfi->stack_size = 0;
3238 3188
3239 } else { 3189 } else {
3240 3190
3241 /* drap: push 3191 /* drap: push %reg */
3242 save_reg(cfi, 3192 save_reg(cfi, op->src.reg, CFI_BP, -cfi->stack_size);
3243 } 3193 }
3244 3194
3245 } else { 3195 } else {
3246 3196
3247 /* push %reg */ 3197 /* push %reg */
3248 save_reg(cfi, op->src 3198 save_reg(cfi, op->src.reg, CFI_CFA, -cfi->stack_size);
3249 } 3199 }
3250 3200
3251 /* detect when asm code uses 3201 /* detect when asm code uses rbp as a scratch register */
3252 if (opts.stackval && insn_fun 3202 if (opts.stackval && insn_func(insn) && op->src.reg == CFI_BP &&
3253 cfa->base != CFI_BP) 3203 cfa->base != CFI_BP)
3254 cfi->bp_scratch = tru 3204 cfi->bp_scratch = true;
3255 break; 3205 break;
3256 3206
3257 case OP_DEST_REG_INDIRECT: 3207 case OP_DEST_REG_INDIRECT:
3258 3208
3259 if (cfi->drap) { 3209 if (cfi->drap) {
3260 if (op->src.reg == cf 3210 if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
3261 3211
3262 /* drap: mov 3212 /* drap: mov %drap, disp(%rbp) */
3263 cfa->base = C 3213 cfa->base = CFI_BP_INDIRECT;
3264 cfa->offset = 3214 cfa->offset = op->dest.offset;
3265 3215
3266 /* save drap 3216 /* save drap offset so we know when to restore it */
3267 cfi->drap_off 3217 cfi->drap_offset = op->dest.offset;
3268 } else { 3218 } else {
3269 3219
3270 /* drap: mov 3220 /* drap: mov reg, disp(%rbp) */
3271 save_reg(cfi, 3221 save_reg(cfi, op->src.reg, CFI_BP, op->dest.offset);
3272 } 3222 }
3273 3223
3274 } else if (op->dest.reg == cf 3224 } else if (op->dest.reg == cfa->base) {
3275 3225
3276 /* mov reg, disp(%rbp 3226 /* mov reg, disp(%rbp) */
3277 /* mov reg, disp(%rsp 3227 /* mov reg, disp(%rsp) */
3278 save_reg(cfi, op->src 3228 save_reg(cfi, op->src.reg, CFI_CFA,
3279 op->dest.off 3229 op->dest.offset - cfi->cfa.offset);
3280 3230
3281 } else if (op->dest.reg == CF 3231 } else if (op->dest.reg == CFI_SP) {
3282 3232
3283 /* mov reg, disp(%rsp 3233 /* mov reg, disp(%rsp) */
3284 save_reg(cfi, op->src 3234 save_reg(cfi, op->src.reg, CFI_CFA,
3285 op->dest.off 3235 op->dest.offset - cfi->stack_size);
3286 3236
3287 } else if (op->src.reg == CFI 3237 } else if (op->src.reg == CFI_SP && op->dest.offset == 0) {
3288 3238
3289 /* mov %rsp, (%reg); 3239 /* mov %rsp, (%reg); # setup a stack swizzle. */
3290 cfi->vals[op->dest.re 3240 cfi->vals[op->dest.reg].base = CFI_SP_INDIRECT;
3291 cfi->vals[op->dest.re 3241 cfi->vals[op->dest.reg].offset = cfa->offset;
3292 } 3242 }
3293 3243
3294 break; 3244 break;
3295 3245
3296 case OP_DEST_MEM: 3246 case OP_DEST_MEM:
3297 if (op->src.type != OP_SRC_PO 3247 if (op->src.type != OP_SRC_POP && op->src.type != OP_SRC_POPF) {
3298 WARN_INSN(insn, "unkn 3248 WARN_INSN(insn, "unknown stack-related memory operation");
3299 return -1; 3249 return -1;
3300 } 3250 }
3301 3251
3302 /* pop mem */ 3252 /* pop mem */
3303 cfi->stack_size -= 8; 3253 cfi->stack_size -= 8;
3304 if (cfa->base == CFI_SP) 3254 if (cfa->base == CFI_SP)
3305 cfa->offset -= 8; 3255 cfa->offset -= 8;
3306 3256
3307 break; 3257 break;
3308 3258
3309 default: 3259 default:
3310 WARN_INSN(insn, "unknown stac 3260 WARN_INSN(insn, "unknown stack-related instruction");
3311 return -1; 3261 return -1;
3312 } 3262 }
3313 3263
3314 return 0; 3264 return 0;
3315 } 3265 }
3316 3266
3317 /* 3267 /*
3318 * The stack layouts of alternatives instruct 3268 * The stack layouts of alternatives instructions can sometimes diverge when
3319 * they have stack modifications. That's fin 3269 * they have stack modifications. That's fine as long as the potential stack
3320 * layouts don't conflict at any given potent 3270 * layouts don't conflict at any given potential instruction boundary.
3321 * 3271 *
3322 * Flatten the CFIs of the different alternat 3272 * Flatten the CFIs of the different alternative code streams (both original
3323 * and replacement) into a single shared CFI 3273 * and replacement) into a single shared CFI array which can be used to detect
3324 * conflicts and nicely feed a linear array o 3274 * conflicts and nicely feed a linear array of ORC entries to the unwinder.
3325 */ 3275 */
3326 static int propagate_alt_cfi(struct objtool_f 3276 static int propagate_alt_cfi(struct objtool_file *file, struct instruction *insn)
3327 { 3277 {
3328 struct cfi_state **alt_cfi; 3278 struct cfi_state **alt_cfi;
3329 int group_off; 3279 int group_off;
3330 3280
3331 if (!insn->alt_group) 3281 if (!insn->alt_group)
3332 return 0; 3282 return 0;
3333 3283
3334 if (!insn->cfi) { 3284 if (!insn->cfi) {
3335 WARN("CFI missing"); 3285 WARN("CFI missing");
3336 return -1; 3286 return -1;
3337 } 3287 }
3338 3288
3339 alt_cfi = insn->alt_group->cfi; 3289 alt_cfi = insn->alt_group->cfi;
3340 group_off = insn->offset - insn->alt_ 3290 group_off = insn->offset - insn->alt_group->first_insn->offset;
3341 3291
3342 if (!alt_cfi[group_off]) { 3292 if (!alt_cfi[group_off]) {
3343 alt_cfi[group_off] = insn->cf 3293 alt_cfi[group_off] = insn->cfi;
3344 } else { 3294 } else {
3345 if (cficmp(alt_cfi[group_off] 3295 if (cficmp(alt_cfi[group_off], insn->cfi)) {
3346 struct alt_group *ori 3296 struct alt_group *orig_group = insn->alt_group->orig_group ?: insn->alt_group;
3347 struct instruction *o 3297 struct instruction *orig = orig_group->first_insn;
3348 char *where = offstr( 3298 char *where = offstr(insn->sec, insn->offset);
3349 WARN_INSN(orig, "stac 3299 WARN_INSN(orig, "stack layout conflict in alternatives: %s", where);
3350 free(where); 3300 free(where);
3351 return -1; 3301 return -1;
3352 } 3302 }
3353 } 3303 }
3354 3304
3355 return 0; 3305 return 0;
3356 } 3306 }
3357 3307
3358 static int handle_insn_ops(struct instruction 3308 static int handle_insn_ops(struct instruction *insn,
3359 struct instruction 3309 struct instruction *next_insn,
3360 struct insn_state 3310 struct insn_state *state)
3361 { 3311 {
3362 struct stack_op *op; 3312 struct stack_op *op;
3363 3313
3364 for (op = insn->stack_ops; op; op = o 3314 for (op = insn->stack_ops; op; op = op->next) {
3365 3315
3366 if (update_cfi_state(insn, ne 3316 if (update_cfi_state(insn, next_insn, &state->cfi, op))
3367 return 1; 3317 return 1;
3368 3318
3369 if (!insn->alt_group) 3319 if (!insn->alt_group)
3370 continue; 3320 continue;
3371 3321
3372 if (op->dest.type == OP_DEST_ 3322 if (op->dest.type == OP_DEST_PUSHF) {
3373 if (!state->uaccess_s 3323 if (!state->uaccess_stack) {
3374 state->uacces 3324 state->uaccess_stack = 1;
3375 } else if (state->uac 3325 } else if (state->uaccess_stack >> 31) {
3376 WARN_INSN(ins 3326 WARN_INSN(insn, "PUSHF stack exhausted");
3377 return 1; 3327 return 1;
3378 } 3328 }
3379 state->uaccess_stack 3329 state->uaccess_stack <<= 1;
3380 state->uaccess_stack 3330 state->uaccess_stack |= state->uaccess;
3381 } 3331 }
3382 3332
3383 if (op->src.type == OP_SRC_PO 3333 if (op->src.type == OP_SRC_POPF) {
3384 if (state->uaccess_st 3334 if (state->uaccess_stack) {
3385 state->uacces 3335 state->uaccess = state->uaccess_stack & 1;
3386 state->uacces 3336 state->uaccess_stack >>= 1;
3387 if (state->ua 3337 if (state->uaccess_stack == 1)
3388 state 3338 state->uaccess_stack = 0;
3389 } 3339 }
3390 } 3340 }
3391 } 3341 }
3392 3342
3393 return 0; 3343 return 0;
3394 } 3344 }
3395 3345
3396 static bool insn_cfi_match(struct instruction 3346 static bool insn_cfi_match(struct instruction *insn, struct cfi_state *cfi2)
3397 { 3347 {
3398 struct cfi_state *cfi1 = insn->cfi; 3348 struct cfi_state *cfi1 = insn->cfi;
3399 int i; 3349 int i;
3400 3350
3401 if (!cfi1) { 3351 if (!cfi1) {
3402 WARN("CFI missing"); 3352 WARN("CFI missing");
3403 return false; 3353 return false;
3404 } 3354 }
3405 3355
3406 if (memcmp(&cfi1->cfa, &cfi2->cfa, si 3356 if (memcmp(&cfi1->cfa, &cfi2->cfa, sizeof(cfi1->cfa))) {
3407 3357
3408 WARN_INSN(insn, "stack state 3358 WARN_INSN(insn, "stack state mismatch: cfa1=%d%+d cfa2=%d%+d",
3409 cfi1->cfa.base, cfi 3359 cfi1->cfa.base, cfi1->cfa.offset,
3410 cfi2->cfa.base, cfi 3360 cfi2->cfa.base, cfi2->cfa.offset);
3411 3361
3412 } else if (memcmp(&cfi1->regs, &cfi2- 3362 } else if (memcmp(&cfi1->regs, &cfi2->regs, sizeof(cfi1->regs))) {
3413 for (i = 0; i < CFI_NUM_REGS; 3363 for (i = 0; i < CFI_NUM_REGS; i++) {
3414 if (!memcmp(&cfi1->re 3364 if (!memcmp(&cfi1->regs[i], &cfi2->regs[i],
3415 sizeof(st 3365 sizeof(struct cfi_reg)))
3416 continue; 3366 continue;
3417 3367
3418 WARN_INSN(insn, "stac 3368 WARN_INSN(insn, "stack state mismatch: reg1[%d]=%d%+d reg2[%d]=%d%+d",
3419 i, cfi1->re 3369 i, cfi1->regs[i].base, cfi1->regs[i].offset,
3420 i, cfi2->re 3370 i, cfi2->regs[i].base, cfi2->regs[i].offset);
3421 break; 3371 break;
3422 } 3372 }
3423 3373
3424 } else if (cfi1->type != cfi2->type) 3374 } else if (cfi1->type != cfi2->type) {
3425 3375
3426 WARN_INSN(insn, "stack state 3376 WARN_INSN(insn, "stack state mismatch: type1=%d type2=%d",
3427 cfi1->type, cfi2->t 3377 cfi1->type, cfi2->type);
3428 3378
3429 } else if (cfi1->drap != cfi2->drap | 3379 } else if (cfi1->drap != cfi2->drap ||
3430 (cfi1->drap && cfi1->drap_ 3380 (cfi1->drap && cfi1->drap_reg != cfi2->drap_reg) ||
3431 (cfi1->drap && cfi1->drap_ 3381 (cfi1->drap && cfi1->drap_offset != cfi2->drap_offset)) {
3432 3382
3433 WARN_INSN(insn, "stack state 3383 WARN_INSN(insn, "stack state mismatch: drap1=%d(%d,%d) drap2=%d(%d,%d)",
3434 cfi1->drap, cfi1->d 3384 cfi1->drap, cfi1->drap_reg, cfi1->drap_offset,
3435 cfi2->drap, cfi2->d 3385 cfi2->drap, cfi2->drap_reg, cfi2->drap_offset);
3436 3386
3437 } else 3387 } else
3438 return true; 3388 return true;
3439 3389
3440 return false; 3390 return false;
3441 } 3391 }
3442 3392
3443 static inline bool func_uaccess_safe(struct s 3393 static inline bool func_uaccess_safe(struct symbol *func)
3444 { 3394 {
3445 if (func) 3395 if (func)
3446 return func->uaccess_safe; 3396 return func->uaccess_safe;
3447 3397
3448 return false; 3398 return false;
3449 } 3399 }
3450 3400
3451 static inline const char *call_dest_name(stru 3401 static inline const char *call_dest_name(struct instruction *insn)
3452 { 3402 {
3453 static char pvname[19]; 3403 static char pvname[19];
3454 struct reloc *reloc; 3404 struct reloc *reloc;
3455 int idx; 3405 int idx;
3456 3406
3457 if (insn_call_dest(insn)) 3407 if (insn_call_dest(insn))
3458 return insn_call_dest(insn)-> 3408 return insn_call_dest(insn)->name;
3459 3409
3460 reloc = insn_reloc(NULL, insn); 3410 reloc = insn_reloc(NULL, insn);
3461 if (reloc && !strcmp(reloc->sym->name 3411 if (reloc && !strcmp(reloc->sym->name, "pv_ops")) {
3462 idx = (reloc_addend(reloc) / 3412 idx = (reloc_addend(reloc) / sizeof(void *));
3463 snprintf(pvname, sizeof(pvnam 3413 snprintf(pvname, sizeof(pvname), "pv_ops[%d]", idx);
3464 return pvname; 3414 return pvname;
3465 } 3415 }
3466 3416
3467 return "{dynamic}"; 3417 return "{dynamic}";
3468 } 3418 }
3469 3419
3470 static bool pv_call_dest(struct objtool_file 3420 static bool pv_call_dest(struct objtool_file *file, struct instruction *insn)
3471 { 3421 {
3472 struct symbol *target; 3422 struct symbol *target;
3473 struct reloc *reloc; 3423 struct reloc *reloc;
3474 int idx; 3424 int idx;
3475 3425
3476 reloc = insn_reloc(file, insn); 3426 reloc = insn_reloc(file, insn);
3477 if (!reloc || strcmp(reloc->sym->name 3427 if (!reloc || strcmp(reloc->sym->name, "pv_ops"))
3478 return false; 3428 return false;
3479 3429
3480 idx = (arch_dest_reloc_offset(reloc_a 3430 idx = (arch_dest_reloc_offset(reloc_addend(reloc)) / sizeof(void *));
3481 3431
3482 if (file->pv_ops[idx].clean) 3432 if (file->pv_ops[idx].clean)
3483 return true; 3433 return true;
3484 3434
3485 file->pv_ops[idx].clean = true; 3435 file->pv_ops[idx].clean = true;
3486 3436
3487 list_for_each_entry(target, &file->pv 3437 list_for_each_entry(target, &file->pv_ops[idx].targets, pv_target) {
3488 if (!target->sec->noinstr) { 3438 if (!target->sec->noinstr) {
3489 WARN("pv_ops[%d]: %s" 3439 WARN("pv_ops[%d]: %s", idx, target->name);
3490 file->pv_ops[idx].cle 3440 file->pv_ops[idx].clean = false;
3491 } 3441 }
3492 } 3442 }
3493 3443
3494 return file->pv_ops[idx].clean; 3444 return file->pv_ops[idx].clean;
3495 } 3445 }
3496 3446
3497 static inline bool noinstr_call_dest(struct o 3447 static inline bool noinstr_call_dest(struct objtool_file *file,
3498 struct i 3448 struct instruction *insn,
3499 struct s 3449 struct symbol *func)
3500 { 3450 {
3501 /* 3451 /*
3502 * We can't deal with indirect functi 3452 * We can't deal with indirect function calls at present;
3503 * assume they're instrumented. 3453 * assume they're instrumented.
3504 */ 3454 */
3505 if (!func) { 3455 if (!func) {
3506 if (file->pv_ops) 3456 if (file->pv_ops)
3507 return pv_call_dest(f 3457 return pv_call_dest(file, insn);
3508 3458
3509 return false; 3459 return false;
3510 } 3460 }
3511 3461
3512 /* 3462 /*
3513 * If the symbol is from a noinstr se 3463 * If the symbol is from a noinstr section; we good.
3514 */ 3464 */
3515 if (func->sec->noinstr) 3465 if (func->sec->noinstr)
3516 return true; 3466 return true;
3517 3467
3518 /* 3468 /*
3519 * If the symbol is a static_call tra 3469 * If the symbol is a static_call trampoline, we can't tell.
3520 */ 3470 */
3521 if (func->static_call_tramp) 3471 if (func->static_call_tramp)
3522 return true; 3472 return true;
3523 3473
3524 /* 3474 /*
3525 * The __ubsan_handle_*() calls are l 3475 * The __ubsan_handle_*() calls are like WARN(), they only happen when
3526 * something 'BAD' happened. At the r 3476 * something 'BAD' happened. At the risk of taking the machine down,
3527 * let them proceed to get the messag 3477 * let them proceed to get the message out.
3528 */ 3478 */
3529 if (!strncmp(func->name, "__ubsan_han 3479 if (!strncmp(func->name, "__ubsan_handle_", 15))
3530 return true; 3480 return true;
3531 3481
3532 return false; 3482 return false;
3533 } 3483 }
3534 3484
3535 static int validate_call(struct objtool_file 3485 static int validate_call(struct objtool_file *file,
3536 struct instruction * 3486 struct instruction *insn,
3537 struct insn_state *s 3487 struct insn_state *state)
3538 { 3488 {
3539 if (state->noinstr && state->instr <= 3489 if (state->noinstr && state->instr <= 0 &&
3540 !noinstr_call_dest(file, insn, in 3490 !noinstr_call_dest(file, insn, insn_call_dest(insn))) {
3541 WARN_INSN(insn, "call to %s() 3491 WARN_INSN(insn, "call to %s() leaves .noinstr.text section", call_dest_name(insn));
3542 return 1; 3492 return 1;
3543 } 3493 }
3544 3494
3545 if (state->uaccess && !func_uaccess_s 3495 if (state->uaccess && !func_uaccess_safe(insn_call_dest(insn))) {
3546 WARN_INSN(insn, "call to %s() 3496 WARN_INSN(insn, "call to %s() with UACCESS enabled", call_dest_name(insn));
3547 return 1; 3497 return 1;
3548 } 3498 }
3549 3499
3550 if (state->df) { 3500 if (state->df) {
3551 WARN_INSN(insn, "call to %s() 3501 WARN_INSN(insn, "call to %s() with DF set", call_dest_name(insn));
3552 return 1; 3502 return 1;
3553 } 3503 }
3554 3504
3555 return 0; 3505 return 0;
3556 } 3506 }
3557 3507
3558 static int validate_sibling_call(struct objto 3508 static int validate_sibling_call(struct objtool_file *file,
3559 struct instr 3509 struct instruction *insn,
3560 struct insn_ 3510 struct insn_state *state)
3561 { 3511 {
3562 if (insn_func(insn) && has_modified_s 3512 if (insn_func(insn) && has_modified_stack_frame(insn, state)) {
3563 WARN_INSN(insn, "sibling call 3513 WARN_INSN(insn, "sibling call from callable instruction with modified stack frame");
3564 return 1; 3514 return 1;
3565 } 3515 }
3566 3516
3567 return validate_call(file, insn, stat 3517 return validate_call(file, insn, state);
3568 } 3518 }
3569 3519
3570 static int validate_return(struct symbol *fun 3520 static int validate_return(struct symbol *func, struct instruction *insn, struct insn_state *state)
3571 { 3521 {
3572 if (state->noinstr && state->instr > 3522 if (state->noinstr && state->instr > 0) {
3573 WARN_INSN(insn, "return with 3523 WARN_INSN(insn, "return with instrumentation enabled");
3574 return 1; 3524 return 1;
3575 } 3525 }
3576 3526
3577 if (state->uaccess && !func_uaccess_s 3527 if (state->uaccess && !func_uaccess_safe(func)) {
3578 WARN_INSN(insn, "return with 3528 WARN_INSN(insn, "return with UACCESS enabled");
3579 return 1; 3529 return 1;
3580 } 3530 }
3581 3531
3582 if (!state->uaccess && func_uaccess_s 3532 if (!state->uaccess && func_uaccess_safe(func)) {
3583 WARN_INSN(insn, "return with 3533 WARN_INSN(insn, "return with UACCESS disabled from a UACCESS-safe function");
3584 return 1; 3534 return 1;
3585 } 3535 }
3586 3536
3587 if (state->df) { 3537 if (state->df) {
3588 WARN_INSN(insn, "return with 3538 WARN_INSN(insn, "return with DF set");
3589 return 1; 3539 return 1;
3590 } 3540 }
3591 3541
3592 if (func && has_modified_stack_frame( 3542 if (func && has_modified_stack_frame(insn, state)) {
3593 WARN_INSN(insn, "return with 3543 WARN_INSN(insn, "return with modified stack frame");
3594 return 1; 3544 return 1;
3595 } 3545 }
3596 3546
3597 if (state->cfi.bp_scratch) { 3547 if (state->cfi.bp_scratch) {
3598 WARN_INSN(insn, "BP used as a 3548 WARN_INSN(insn, "BP used as a scratch register");
3599 return 1; 3549 return 1;
3600 } 3550 }
3601 3551
3602 return 0; 3552 return 0;
3603 } 3553 }
3604 3554
3605 static struct instruction *next_insn_to_valid 3555 static struct instruction *next_insn_to_validate(struct objtool_file *file,
3606 3556 struct instruction *insn)
3607 { 3557 {
3608 struct alt_group *alt_group = insn->a 3558 struct alt_group *alt_group = insn->alt_group;
3609 3559
3610 /* 3560 /*
3611 * Simulate the fact that alternative 3561 * Simulate the fact that alternatives are patched in-place. When the
3612 * end of a replacement alt_group is 3562 * end of a replacement alt_group is reached, redirect objtool flow to
3613 * the end of the original alt_group. 3563 * the end of the original alt_group.
3614 * 3564 *
3615 * insn->alts->insn -> alt_group->fir 3565 * insn->alts->insn -> alt_group->first_insn
3616 * ... 3566 * ...
3617 * alt_group->las 3567 * alt_group->last_insn
3618 * [alt_group->no 3568 * [alt_group->nop] -> next(orig_group->last_insn)
3619 */ 3569 */
3620 if (alt_group) { 3570 if (alt_group) {
3621 if (alt_group->nop) { 3571 if (alt_group->nop) {
3622 /* ->nop implies ->or 3572 /* ->nop implies ->orig_group */
3623 if (insn == alt_group 3573 if (insn == alt_group->last_insn)
3624 return alt_gr 3574 return alt_group->nop;
3625 if (insn == alt_group 3575 if (insn == alt_group->nop)
3626 goto next_ori 3576 goto next_orig;
3627 } 3577 }
3628 if (insn == alt_group->last_i 3578 if (insn == alt_group->last_insn && alt_group->orig_group)
3629 goto next_orig; 3579 goto next_orig;
3630 } 3580 }
3631 3581
3632 return next_insn_same_sec(file, insn) 3582 return next_insn_same_sec(file, insn);
3633 3583
3634 next_orig: 3584 next_orig:
3635 return next_insn_same_sec(file, alt_g 3585 return next_insn_same_sec(file, alt_group->orig_group->last_insn);
3636 } 3586 }
3637 3587
3638 /* 3588 /*
3639 * Follow the branch starting at the given in 3589 * Follow the branch starting at the given instruction, and recursively follow
3640 * any other branches (jumps). Meanwhile, tr 3590 * any other branches (jumps). Meanwhile, track the frame pointer state at
3641 * each instruction and validate all the rule 3591 * each instruction and validate all the rules described in
3642 * tools/objtool/Documentation/objtool.txt. 3592 * tools/objtool/Documentation/objtool.txt.
3643 */ 3593 */
3644 static int validate_branch(struct objtool_fil 3594 static int validate_branch(struct objtool_file *file, struct symbol *func,
3645 struct instruction 3595 struct instruction *insn, struct insn_state state)
3646 { 3596 {
3647 struct alternative *alt; 3597 struct alternative *alt;
3648 struct instruction *next_insn, *prev_ 3598 struct instruction *next_insn, *prev_insn = NULL;
3649 struct section *sec; 3599 struct section *sec;
3650 u8 visited; 3600 u8 visited;
3651 int ret; 3601 int ret;
3652 3602
3653 sec = insn->sec; 3603 sec = insn->sec;
3654 3604
3655 while (1) { 3605 while (1) {
3656 next_insn = next_insn_to_vali 3606 next_insn = next_insn_to_validate(file, insn);
3657 3607
3658 if (func && insn_func(insn) & 3608 if (func && insn_func(insn) && func != insn_func(insn)->pfunc) {
3659 /* Ignore KCFI type p 3609 /* Ignore KCFI type preambles, which always fall through */
3660 if (!strncmp(func->na 3610 if (!strncmp(func->name, "__cfi_", 6) ||
3661 !strncmp(func->na 3611 !strncmp(func->name, "__pfx_", 6))
3662 return 0; 3612 return 0;
3663 3613
3664 WARN("%s() falls thro 3614 WARN("%s() falls through to next function %s()",
3665 func->name, insn 3615 func->name, insn_func(insn)->name);
3666 return 1; 3616 return 1;
3667 } 3617 }
3668 3618
3669 if (func && insn->ignore) { 3619 if (func && insn->ignore) {
3670 WARN_INSN(insn, "BUG: 3620 WARN_INSN(insn, "BUG: why am I validating an ignored function?");
3671 return 1; 3621 return 1;
3672 } 3622 }
3673 3623
3674 visited = VISITED_BRANCH << s 3624 visited = VISITED_BRANCH << state.uaccess;
3675 if (insn->visited & VISITED_B 3625 if (insn->visited & VISITED_BRANCH_MASK) {
3676 if (!insn->hint && !i 3626 if (!insn->hint && !insn_cfi_match(insn, &state.cfi))
3677 return 1; 3627 return 1;
3678 3628
3679 if (insn->visited & v 3629 if (insn->visited & visited)
3680 return 0; 3630 return 0;
3681 } else { 3631 } else {
3682 nr_insns_visited++; 3632 nr_insns_visited++;
3683 } 3633 }
3684 3634
3685 if (state.noinstr) 3635 if (state.noinstr)
3686 state.instr += insn-> 3636 state.instr += insn->instr;
3687 3637
3688 if (insn->hint) { 3638 if (insn->hint) {
3689 if (insn->restore) { 3639 if (insn->restore) {
3690 struct instru 3640 struct instruction *save_insn, *i;
3691 3641
3692 i = insn; 3642 i = insn;
3693 save_insn = N 3643 save_insn = NULL;
3694 3644
3695 sym_for_each_ 3645 sym_for_each_insn_continue_reverse(file, func, i) {
3696 if (i 3646 if (i->save) {
3697 3647 save_insn = i;
3698 3648 break;
3699 } 3649 }
3700 } 3650 }
3701 3651
3702 if (!save_ins 3652 if (!save_insn) {
3703 WARN_ 3653 WARN_INSN(insn, "no corresponding CFI save for CFI restore");
3704 retur 3654 return 1;
3705 } 3655 }
3706 3656
3707 if (!save_ins 3657 if (!save_insn->visited) {
3708 /* 3658 /*
3709 * If 3659 * If the restore hint insn is at the
3710 * be 3660 * beginning of a basic block and was
3711 * br 3661 * branched to from elsewhere, and the
3712 * sa 3662 * save insn hasn't been visited yet,
3713 * de 3663 * defer following this branch for now.
3714 * It 3664 * It will be seen later via the
3715 * st 3665 * straight-line path.
3716 */ 3666 */
3717 if (! 3667 if (!prev_insn)
3718 3668 return 0;
3719 3669
3720 WARN_ 3670 WARN_INSN(insn, "objtool isn't smart enough to handle this CFI save/restore combo");
3721 retur 3671 return 1;
3722 } 3672 }
3723 3673
3724 insn->cfi = s 3674 insn->cfi = save_insn->cfi;
3725 nr_cfi_reused 3675 nr_cfi_reused++;
3726 } 3676 }
3727 3677
3728 state.cfi = *insn->cf 3678 state.cfi = *insn->cfi;
3729 } else { 3679 } else {
3730 /* XXX track if we ac 3680 /* XXX track if we actually changed state.cfi */
3731 3681
3732 if (prev_insn && !cfi 3682 if (prev_insn && !cficmp(prev_insn->cfi, &state.cfi)) {
3733 insn->cfi = p 3683 insn->cfi = prev_insn->cfi;
3734 nr_cfi_reused 3684 nr_cfi_reused++;
3735 } else { 3685 } else {
3736 insn->cfi = c 3686 insn->cfi = cfi_hash_find_or_add(&state.cfi);
3737 } 3687 }
3738 } 3688 }
3739 3689
3740 insn->visited |= visited; 3690 insn->visited |= visited;
3741 3691
3742 if (propagate_alt_cfi(file, i 3692 if (propagate_alt_cfi(file, insn))
3743 return 1; 3693 return 1;
3744 3694
3745 if (!insn->ignore_alts && ins 3695 if (!insn->ignore_alts && insn->alts) {
3746 bool skip_orig = fals 3696 bool skip_orig = false;
3747 3697
3748 for (alt = insn->alts 3698 for (alt = insn->alts; alt; alt = alt->next) {
3749 if (alt->skip 3699 if (alt->skip_orig)
3750 skip_ 3700 skip_orig = true;
3751 3701
3752 ret = validat 3702 ret = validate_branch(file, func, alt->insn, state);
3753 if (ret) { 3703 if (ret) {
3754 BT_IN 3704 BT_INSN(insn, "(alt)");
3755 retur 3705 return ret;
3756 } 3706 }
3757 } 3707 }
3758 3708
3759 if (skip_orig) 3709 if (skip_orig)
3760 return 0; 3710 return 0;
3761 } 3711 }
3762 3712
3763 if (handle_insn_ops(insn, nex 3713 if (handle_insn_ops(insn, next_insn, &state))
3764 return 1; 3714 return 1;
3765 3715
3766 switch (insn->type) { 3716 switch (insn->type) {
3767 3717
3768 case INSN_RETURN: 3718 case INSN_RETURN:
3769 return validate_retur 3719 return validate_return(func, insn, &state);
3770 3720
3771 case INSN_CALL: 3721 case INSN_CALL:
3772 case INSN_CALL_DYNAMIC: 3722 case INSN_CALL_DYNAMIC:
3773 ret = validate_call(f 3723 ret = validate_call(file, insn, &state);
3774 if (ret) 3724 if (ret)
3775 return ret; 3725 return ret;
3776 3726
3777 if (opts.stackval && 3727 if (opts.stackval && func && !is_special_call(insn) &&
3778 !has_valid_stack_ 3728 !has_valid_stack_frame(&state)) {
3779 WARN_INSN(ins 3729 WARN_INSN(insn, "call without frame pointer save/setup");
3780 return 1; 3730 return 1;
3781 } 3731 }
3782 3732
3783 if (insn->dead_end) 3733 if (insn->dead_end)
3784 return 0; 3734 return 0;
3785 3735
3786 break; 3736 break;
3787 3737
3788 case INSN_JUMP_CONDITIONAL: 3738 case INSN_JUMP_CONDITIONAL:
3789 case INSN_JUMP_UNCONDITIONAL: 3739 case INSN_JUMP_UNCONDITIONAL:
3790 if (is_sibling_call(i 3740 if (is_sibling_call(insn)) {
3791 ret = validat 3741 ret = validate_sibling_call(file, insn, &state);
3792 if (ret) 3742 if (ret)
3793 retur 3743 return ret;
3794 3744
3795 } else if (insn->jump 3745 } else if (insn->jump_dest) {
3796 ret = validat 3746 ret = validate_branch(file, func,
3797 3747 insn->jump_dest, state);
3798 if (ret) { 3748 if (ret) {
3799 BT_IN 3749 BT_INSN(insn, "(branch)");
3800 retur 3750 return ret;
3801 } 3751 }
3802 } 3752 }
3803 3753
3804 if (insn->type == INS 3754 if (insn->type == INSN_JUMP_UNCONDITIONAL)
3805 return 0; 3755 return 0;
3806 3756
3807 break; 3757 break;
3808 3758
3809 case INSN_JUMP_DYNAMIC: 3759 case INSN_JUMP_DYNAMIC:
3810 case INSN_JUMP_DYNAMIC_CONDIT 3760 case INSN_JUMP_DYNAMIC_CONDITIONAL:
3811 if (is_sibling_call(i 3761 if (is_sibling_call(insn)) {
3812 ret = validat 3762 ret = validate_sibling_call(file, insn, &state);
3813 if (ret) 3763 if (ret)
3814 retur 3764 return ret;
3815 } 3765 }
3816 3766
3817 if (insn->type == INS 3767 if (insn->type == INSN_JUMP_DYNAMIC)
3818 return 0; 3768 return 0;
3819 3769
3820 break; 3770 break;
3821 3771
3822 case INSN_CONTEXT_SWITCH: 3772 case INSN_CONTEXT_SWITCH:
3823 if (func && (!next_in 3773 if (func && (!next_insn || !next_insn->hint)) {
3824 WARN_INSN(ins 3774 WARN_INSN(insn, "unsupported instruction in callable function");
3825 return 1; 3775 return 1;
3826 } 3776 }
3827 return 0; 3777 return 0;
3828 3778
3829 case INSN_STAC: 3779 case INSN_STAC:
3830 if (state.uaccess) { 3780 if (state.uaccess) {
3831 WARN_INSN(ins 3781 WARN_INSN(insn, "recursive UACCESS enable");
3832 return 1; 3782 return 1;
3833 } 3783 }
3834 3784
3835 state.uaccess = true; 3785 state.uaccess = true;
3836 break; 3786 break;
3837 3787
3838 case INSN_CLAC: 3788 case INSN_CLAC:
3839 if (!state.uaccess && 3789 if (!state.uaccess && func) {
3840 WARN_INSN(ins 3790 WARN_INSN(insn, "redundant UACCESS disable");
3841 return 1; 3791 return 1;
3842 } 3792 }
3843 3793
3844 if (func_uaccess_safe 3794 if (func_uaccess_safe(func) && !state.uaccess_stack) {
3845 WARN_INSN(ins 3795 WARN_INSN(insn, "UACCESS-safe disables UACCESS");
3846 return 1; 3796 return 1;
3847 } 3797 }
3848 3798
3849 state.uaccess = false 3799 state.uaccess = false;
3850 break; 3800 break;
3851 3801
3852 case INSN_STD: 3802 case INSN_STD:
3853 if (state.df) { 3803 if (state.df) {
3854 WARN_INSN(ins 3804 WARN_INSN(insn, "recursive STD");
3855 return 1; 3805 return 1;
3856 } 3806 }
3857 3807
3858 state.df = true; 3808 state.df = true;
3859 break; 3809 break;
3860 3810
3861 case INSN_CLD: 3811 case INSN_CLD:
3862 if (!state.df && func 3812 if (!state.df && func) {
3863 WARN_INSN(ins 3813 WARN_INSN(insn, "redundant CLD");
3864 return 1; 3814 return 1;
3865 } 3815 }
3866 3816
3867 state.df = false; 3817 state.df = false;
3868 break; 3818 break;
3869 3819
3870 default: 3820 default:
3871 break; 3821 break;
3872 } 3822 }
3873 3823
3874 if (insn->dead_end) 3824 if (insn->dead_end)
3875 return 0; 3825 return 0;
3876 3826
3877 if (!next_insn) { 3827 if (!next_insn) {
3878 if (state.cfi.cfa.bas 3828 if (state.cfi.cfa.base == CFI_UNDEFINED)
3879 return 0; 3829 return 0;
3880 WARN("%s: unexpected 3830 WARN("%s: unexpected end of section", sec->name);
3881 return 1; 3831 return 1;
3882 } 3832 }
3883 3833
3884 prev_insn = insn; 3834 prev_insn = insn;
3885 insn = next_insn; 3835 insn = next_insn;
3886 } 3836 }
3887 3837
3888 return 0; 3838 return 0;
3889 } 3839 }
3890 3840
3891 static int validate_unwind_hint(struct objtoo 3841 static int validate_unwind_hint(struct objtool_file *file,
3892 struct inst 3842 struct instruction *insn,
3893 struct insn 3843 struct insn_state *state)
3894 { 3844 {
3895 if (insn->hint && !insn->visited && ! 3845 if (insn->hint && !insn->visited && !insn->ignore) {
3896 int ret = validate_branch(fil 3846 int ret = validate_branch(file, insn_func(insn), insn, *state);
3897 if (ret) 3847 if (ret)
3898 BT_INSN(insn, "<=== ( 3848 BT_INSN(insn, "<=== (hint)");
3899 return ret; 3849 return ret;
3900 } 3850 }
3901 3851
3902 return 0; 3852 return 0;
3903 } 3853 }
3904 3854
3905 static int validate_unwind_hints(struct objto 3855 static int validate_unwind_hints(struct objtool_file *file, struct section *sec)
3906 { 3856 {
3907 struct instruction *insn; 3857 struct instruction *insn;
3908 struct insn_state state; 3858 struct insn_state state;
3909 int warnings = 0; 3859 int warnings = 0;
3910 3860
3911 if (!file->hints) 3861 if (!file->hints)
3912 return 0; 3862 return 0;
3913 3863
3914 init_insn_state(file, &state, sec); 3864 init_insn_state(file, &state, sec);
3915 3865
3916 if (sec) { 3866 if (sec) {
3917 sec_for_each_insn(file, sec, 3867 sec_for_each_insn(file, sec, insn)
3918 warnings += validate_ 3868 warnings += validate_unwind_hint(file, insn, &state);
3919 } else { 3869 } else {
3920 for_each_insn(file, insn) 3870 for_each_insn(file, insn)
3921 warnings += validate_ 3871 warnings += validate_unwind_hint(file, insn, &state);
3922 } 3872 }
3923 3873
3924 return warnings; 3874 return warnings;
3925 } 3875 }
3926 3876
3927 /* 3877 /*
3928 * Validate rethunk entry constraint: must un 3878 * Validate rethunk entry constraint: must untrain RET before the first RET.
3929 * 3879 *
3930 * Follow every branch (intra-function) and e 3880 * Follow every branch (intra-function) and ensure VALIDATE_UNRET_END comes
3931 * before an actual RET instruction. 3881 * before an actual RET instruction.
3932 */ 3882 */
3933 static int validate_unret(struct objtool_file 3883 static int validate_unret(struct objtool_file *file, struct instruction *insn)
3934 { 3884 {
3935 struct instruction *next, *dest; 3885 struct instruction *next, *dest;
3936 int ret; 3886 int ret;
3937 3887
3938 for (;;) { 3888 for (;;) {
3939 next = next_insn_to_validate( 3889 next = next_insn_to_validate(file, insn);
3940 3890
3941 if (insn->visited & VISITED_U 3891 if (insn->visited & VISITED_UNRET)
3942 return 0; 3892 return 0;
3943 3893
3944 insn->visited |= VISITED_UNRE 3894 insn->visited |= VISITED_UNRET;
3945 3895
3946 if (!insn->ignore_alts && ins 3896 if (!insn->ignore_alts && insn->alts) {
3947 struct alternative *a 3897 struct alternative *alt;
3948 bool skip_orig = fals 3898 bool skip_orig = false;
3949 3899
3950 for (alt = insn->alts 3900 for (alt = insn->alts; alt; alt = alt->next) {
3951 if (alt->skip 3901 if (alt->skip_orig)
3952 skip_ 3902 skip_orig = true;
3953 3903
3954 ret = validat 3904 ret = validate_unret(file, alt->insn);
3955 if (ret) { 3905 if (ret) {
3956 BT_IN 3906 BT_INSN(insn, "(alt)");
3957 retur 3907 return ret;
3958 } 3908 }
3959 } 3909 }
3960 3910
3961 if (skip_orig) 3911 if (skip_orig)
3962 return 0; 3912 return 0;
3963 } 3913 }
3964 3914
3965 switch (insn->type) { 3915 switch (insn->type) {
3966 3916
3967 case INSN_CALL_DYNAMIC: 3917 case INSN_CALL_DYNAMIC:
3968 case INSN_JUMP_DYNAMIC: 3918 case INSN_JUMP_DYNAMIC:
3969 case INSN_JUMP_DYNAMIC_CONDIT 3919 case INSN_JUMP_DYNAMIC_CONDITIONAL:
3970 WARN_INSN(insn, "earl 3920 WARN_INSN(insn, "early indirect call");
3971 return 1; 3921 return 1;
3972 3922
3973 case INSN_JUMP_UNCONDITIONAL: 3923 case INSN_JUMP_UNCONDITIONAL:
3974 case INSN_JUMP_CONDITIONAL: 3924 case INSN_JUMP_CONDITIONAL:
3975 if (!is_sibling_call( 3925 if (!is_sibling_call(insn)) {
3976 if (!insn->ju 3926 if (!insn->jump_dest) {
3977 WARN_ 3927 WARN_INSN(insn, "unresolved jump target after linking?!?");
3978 retur 3928 return -1;
3979 } 3929 }
3980 ret = validat 3930 ret = validate_unret(file, insn->jump_dest);
3981 if (ret) { 3931 if (ret) {
3982 BT_IN 3932 BT_INSN(insn, "(branch%s)",
3983 3933 insn->type == INSN_JUMP_CONDITIONAL ? "-cond" : "");
3984 retur 3934 return ret;
3985 } 3935 }
3986 3936
3987 if (insn->typ 3937 if (insn->type == INSN_JUMP_UNCONDITIONAL)
3988 retur 3938 return 0;
3989 3939
3990 break; 3940 break;
3991 } 3941 }
3992 3942
3993 /* fallthrough */ 3943 /* fallthrough */
3994 case INSN_CALL: 3944 case INSN_CALL:
3995 dest = find_insn(file 3945 dest = find_insn(file, insn_call_dest(insn)->sec,
3996 insn 3946 insn_call_dest(insn)->offset);
3997 if (!dest) { 3947 if (!dest) {
3998 WARN("Unresol 3948 WARN("Unresolved function after linking!?: %s",
3999 insn_cal 3949 insn_call_dest(insn)->name);
4000 return -1; 3950 return -1;
4001 } 3951 }
4002 3952
4003 ret = validate_unret( 3953 ret = validate_unret(file, dest);
4004 if (ret) { 3954 if (ret) {
4005 BT_INSN(insn, 3955 BT_INSN(insn, "(call)");
4006 return ret; 3956 return ret;
4007 } 3957 }
4008 /* 3958 /*
4009 * If a call returns 3959 * If a call returns without error, it must have seen UNTRAIN_RET.
4010 * Therefore any non- 3960 * Therefore any non-error return is a success.
4011 */ 3961 */
4012 return 0; 3962 return 0;
4013 3963
4014 case INSN_RETURN: 3964 case INSN_RETURN:
4015 WARN_INSN(insn, "RET 3965 WARN_INSN(insn, "RET before UNTRAIN");
4016 return 1; 3966 return 1;
4017 3967
4018 case INSN_NOP: 3968 case INSN_NOP:
4019 if (insn->retpoline_s 3969 if (insn->retpoline_safe)
4020 return 0; 3970 return 0;
4021 break; 3971 break;
4022 3972
4023 default: 3973 default:
4024 break; 3974 break;
4025 } 3975 }
4026 3976
4027 if (!next) { 3977 if (!next) {
4028 WARN_INSN(insn, "teh 3978 WARN_INSN(insn, "teh end!");
4029 return -1; 3979 return -1;
4030 } 3980 }
4031 insn = next; 3981 insn = next;
4032 } 3982 }
4033 3983
4034 return 0; 3984 return 0;
4035 } 3985 }
4036 3986
4037 /* 3987 /*
4038 * Validate that all branches starting at VAL 3988 * Validate that all branches starting at VALIDATE_UNRET_BEGIN encounter
4039 * VALIDATE_UNRET_END before RET. 3989 * VALIDATE_UNRET_END before RET.
4040 */ 3990 */
4041 static int validate_unrets(struct objtool_fil 3991 static int validate_unrets(struct objtool_file *file)
4042 { 3992 {
4043 struct instruction *insn; 3993 struct instruction *insn;
4044 int ret, warnings = 0; 3994 int ret, warnings = 0;
4045 3995
4046 for_each_insn(file, insn) { 3996 for_each_insn(file, insn) {
4047 if (!insn->unret) 3997 if (!insn->unret)
4048 continue; 3998 continue;
4049 3999
4050 ret = validate_unret(file, in 4000 ret = validate_unret(file, insn);
4051 if (ret < 0) { 4001 if (ret < 0) {
4052 WARN_INSN(insn, "Fail 4002 WARN_INSN(insn, "Failed UNRET validation");
4053 return ret; 4003 return ret;
4054 } 4004 }
4055 warnings += ret; 4005 warnings += ret;
4056 } 4006 }
4057 4007
4058 return warnings; 4008 return warnings;
4059 } 4009 }
4060 4010
4061 static int validate_retpoline(struct objtool_ 4011 static int validate_retpoline(struct objtool_file *file)
4062 { 4012 {
4063 struct instruction *insn; 4013 struct instruction *insn;
4064 int warnings = 0; 4014 int warnings = 0;
4065 4015
4066 for_each_insn(file, insn) { 4016 for_each_insn(file, insn) {
4067 if (insn->type != INSN_JUMP_D 4017 if (insn->type != INSN_JUMP_DYNAMIC &&
4068 insn->type != INSN_CALL_D 4018 insn->type != INSN_CALL_DYNAMIC &&
4069 insn->type != INSN_RETURN 4019 insn->type != INSN_RETURN)
4070 continue; 4020 continue;
4071 4021
4072 if (insn->retpoline_safe) 4022 if (insn->retpoline_safe)
4073 continue; 4023 continue;
4074 4024
4075 if (insn->sec->init) 4025 if (insn->sec->init)
4076 continue; 4026 continue;
4077 4027
4078 if (insn->type == INSN_RETURN 4028 if (insn->type == INSN_RETURN) {
4079 if (opts.rethunk) { 4029 if (opts.rethunk) {
4080 WARN_INSN(ins 4030 WARN_INSN(insn, "'naked' return found in MITIGATION_RETHUNK build");
4081 } else 4031 } else
4082 continue; 4032 continue;
4083 } else { 4033 } else {
4084 WARN_INSN(insn, "indi 4034 WARN_INSN(insn, "indirect %s found in MITIGATION_RETPOLINE build",
4085 insn->type 4035 insn->type == INSN_JUMP_DYNAMIC ? "jump" : "call");
4086 } 4036 }
4087 4037
4088 warnings++; 4038 warnings++;
4089 } 4039 }
4090 4040
4091 return warnings; 4041 return warnings;
4092 } 4042 }
4093 4043
4094 static bool is_kasan_insn(struct instruction 4044 static bool is_kasan_insn(struct instruction *insn)
4095 { 4045 {
4096 return (insn->type == INSN_CALL && 4046 return (insn->type == INSN_CALL &&
4097 !strcmp(insn_call_dest(insn)- 4047 !strcmp(insn_call_dest(insn)->name, "__asan_handle_no_return"));
4098 } 4048 }
4099 4049
4100 static bool is_ubsan_insn(struct instruction 4050 static bool is_ubsan_insn(struct instruction *insn)
4101 { 4051 {
4102 return (insn->type == INSN_CALL && 4052 return (insn->type == INSN_CALL &&
4103 !strcmp(insn_call_dest(insn)- 4053 !strcmp(insn_call_dest(insn)->name,
4104 "__ubsan_handle_built 4054 "__ubsan_handle_builtin_unreachable"));
4105 } 4055 }
4106 4056
4107 static bool ignore_unreachable_insn(struct ob 4057 static bool ignore_unreachable_insn(struct objtool_file *file, struct instruction *insn)
4108 { 4058 {
4109 int i; 4059 int i;
4110 struct instruction *prev_insn; 4060 struct instruction *prev_insn;
4111 4061
4112 if (insn->ignore || insn->type == INS 4062 if (insn->ignore || insn->type == INSN_NOP || insn->type == INSN_TRAP)
4113 return true; 4063 return true;
4114 4064
4115 /* 4065 /*
4116 * Ignore alternative replacement ins 4066 * Ignore alternative replacement instructions. This can happen
4117 * when a whitelisted function uses o 4067 * when a whitelisted function uses one of the ALTERNATIVE macros.
4118 */ 4068 */
4119 if (!strcmp(insn->sec->name, ".altins 4069 if (!strcmp(insn->sec->name, ".altinstr_replacement") ||
4120 !strcmp(insn->sec->name, ".altins 4070 !strcmp(insn->sec->name, ".altinstr_aux"))
4121 return true; 4071 return true;
4122 4072
4123 /* 4073 /*
4124 * Whole archive runs might encounter 4074 * Whole archive runs might encounter dead code from weak symbols.
4125 * This is where the linker will have 4075 * This is where the linker will have dropped the weak symbol in
4126 * favour of a regular symbol, but le 4076 * favour of a regular symbol, but leaves the code in place.
4127 * 4077 *
4128 * In this case we'll find a piece of 4078 * In this case we'll find a piece of code (whole function) that is not
4129 * covered by a !section symbol. Igno 4079 * covered by a !section symbol. Ignore them.
4130 */ 4080 */
4131 if (opts.link && !insn_func(insn)) { 4081 if (opts.link && !insn_func(insn)) {
4132 int size = find_symbol_hole_c 4082 int size = find_symbol_hole_containing(insn->sec, insn->offset);
4133 unsigned long end = insn->off 4083 unsigned long end = insn->offset + size;
4134 4084
4135 if (!size) /* not a hole */ 4085 if (!size) /* not a hole */
4136 return false; 4086 return false;
4137 4087
4138 if (size < 0) /* hole until t 4088 if (size < 0) /* hole until the end */
4139 return true; 4089 return true;
4140 4090
4141 sec_for_each_insn_continue(fi 4091 sec_for_each_insn_continue(file, insn) {
4142 /* 4092 /*
4143 * If we reach a visi 4093 * If we reach a visited instruction at or before the
4144 * end of the hole, i 4094 * end of the hole, ignore the unreachable.
4145 */ 4095 */
4146 if (insn->visited) 4096 if (insn->visited)
4147 return true; 4097 return true;
4148 4098
4149 if (insn->offset >= e 4099 if (insn->offset >= end)
4150 break; 4100 break;
4151 4101
4152 /* 4102 /*
4153 * If this hole jumps 4103 * If this hole jumps to a .cold function, mark it ignore too.
4154 */ 4104 */
4155 if (insn->jump_dest & 4105 if (insn->jump_dest && insn_func(insn->jump_dest) &&
4156 strstr(insn_func( 4106 strstr(insn_func(insn->jump_dest)->name, ".cold")) {
4157 struct instru 4107 struct instruction *dest = insn->jump_dest;
4158 func_for_each 4108 func_for_each_insn(file, insn_func(dest), dest)
4159 dest- 4109 dest->ignore = true;
4160 } 4110 }
4161 } 4111 }
4162 4112
4163 return false; 4113 return false;
4164 } 4114 }
4165 4115
4166 if (!insn_func(insn)) 4116 if (!insn_func(insn))
4167 return false; 4117 return false;
4168 4118
4169 if (insn_func(insn)->static_call_tram 4119 if (insn_func(insn)->static_call_tramp)
4170 return true; 4120 return true;
4171 4121
4172 /* 4122 /*
4173 * CONFIG_UBSAN_TRAP inserts a UD2 wh 4123 * CONFIG_UBSAN_TRAP inserts a UD2 when it sees
4174 * __builtin_unreachable(). The BUG( 4124 * __builtin_unreachable(). The BUG() macro has an unreachable() after
4175 * the UD2, which causes GCC's undefi 4125 * the UD2, which causes GCC's undefined trap logic to emit another UD2
4176 * (or occasionally a JMP to UD2). 4126 * (or occasionally a JMP to UD2).
4177 * 4127 *
4178 * It may also insert a UD2 after cal 4128 * It may also insert a UD2 after calling a __noreturn function.
4179 */ 4129 */
4180 prev_insn = prev_insn_same_sec(file, 4130 prev_insn = prev_insn_same_sec(file, insn);
4181 if (prev_insn->dead_end && 4131 if (prev_insn->dead_end &&
4182 (insn->type == INSN_BUG || 4132 (insn->type == INSN_BUG ||
4183 (insn->type == INSN_JUMP_UNCONDI 4133 (insn->type == INSN_JUMP_UNCONDITIONAL &&
4184 insn->jump_dest && insn->jump_d 4134 insn->jump_dest && insn->jump_dest->type == INSN_BUG)))
4185 return true; 4135 return true;
4186 4136
4187 /* 4137 /*
4188 * Check if this (or a subsequent) in 4138 * Check if this (or a subsequent) instruction is related to
4189 * CONFIG_UBSAN or CONFIG_KASAN. 4139 * CONFIG_UBSAN or CONFIG_KASAN.
4190 * 4140 *
4191 * End the search at 5 instructions t 4141 * End the search at 5 instructions to avoid going into the weeds.
4192 */ 4142 */
4193 for (i = 0; i < 5; i++) { 4143 for (i = 0; i < 5; i++) {
4194 4144
4195 if (is_kasan_insn(insn) || is 4145 if (is_kasan_insn(insn) || is_ubsan_insn(insn))
4196 return true; 4146 return true;
4197 4147
4198 if (insn->type == INSN_JUMP_U 4148 if (insn->type == INSN_JUMP_UNCONDITIONAL) {
4199 if (insn->jump_dest & 4149 if (insn->jump_dest &&
4200 insn_func(insn->j 4150 insn_func(insn->jump_dest) == insn_func(insn)) {
4201 insn = insn-> 4151 insn = insn->jump_dest;
4202 continue; 4152 continue;
4203 } 4153 }
4204 4154
4205 break; 4155 break;
4206 } 4156 }
4207 4157
4208 if (insn->offset + insn->len 4158 if (insn->offset + insn->len >= insn_func(insn)->offset + insn_func(insn)->len)
4209 break; 4159 break;
4210 4160
4211 insn = next_insn_same_sec(fil 4161 insn = next_insn_same_sec(file, insn);
4212 } 4162 }
4213 4163
4214 return false; 4164 return false;
4215 } 4165 }
4216 4166
4217 static int add_prefix_symbol(struct objtool_f 4167 static int add_prefix_symbol(struct objtool_file *file, struct symbol *func)
4218 { 4168 {
4219 struct instruction *insn, *prev; 4169 struct instruction *insn, *prev;
4220 struct cfi_state *cfi; 4170 struct cfi_state *cfi;
4221 4171
4222 insn = find_insn(file, func->sec, fun 4172 insn = find_insn(file, func->sec, func->offset);
4223 if (!insn) 4173 if (!insn)
4224 return -1; 4174 return -1;
4225 4175
4226 for (prev = prev_insn_same_sec(file, 4176 for (prev = prev_insn_same_sec(file, insn);
4227 prev; 4177 prev;
4228 prev = prev_insn_same_sec(file, 4178 prev = prev_insn_same_sec(file, prev)) {
4229 u64 offset; 4179 u64 offset;
4230 4180
4231 if (prev->type != INSN_NOP) 4181 if (prev->type != INSN_NOP)
4232 return -1; 4182 return -1;
4233 4183
4234 offset = func->offset - prev- 4184 offset = func->offset - prev->offset;
4235 4185
4236 if (offset > opts.prefix) 4186 if (offset > opts.prefix)
4237 return -1; 4187 return -1;
4238 4188
4239 if (offset < opts.prefix) 4189 if (offset < opts.prefix)
4240 continue; 4190 continue;
4241 4191
4242 elf_create_prefix_symbol(file 4192 elf_create_prefix_symbol(file->elf, func, opts.prefix);
4243 break; 4193 break;
4244 } 4194 }
4245 4195
4246 if (!prev) 4196 if (!prev)
4247 return -1; 4197 return -1;
4248 4198
4249 if (!insn->cfi) { 4199 if (!insn->cfi) {
4250 /* 4200 /*
4251 * This can happen if stack v 4201 * This can happen if stack validation isn't enabled or the
4252 * function is annotated with 4202 * function is annotated with STACK_FRAME_NON_STANDARD.
4253 */ 4203 */
4254 return 0; 4204 return 0;
4255 } 4205 }
4256 4206
4257 /* Propagate insn->cfi to the prefix 4207 /* Propagate insn->cfi to the prefix code */
4258 cfi = cfi_hash_find_or_add(insn->cfi) 4208 cfi = cfi_hash_find_or_add(insn->cfi);
4259 for (; prev != insn; prev = next_insn 4209 for (; prev != insn; prev = next_insn_same_sec(file, prev))
4260 prev->cfi = cfi; 4210 prev->cfi = cfi;
4261 4211
4262 return 0; 4212 return 0;
4263 } 4213 }
4264 4214
4265 static int add_prefix_symbols(struct objtool_ 4215 static int add_prefix_symbols(struct objtool_file *file)
4266 { 4216 {
4267 struct section *sec; 4217 struct section *sec;
4268 struct symbol *func; 4218 struct symbol *func;
4269 4219
4270 for_each_sec(file, sec) { 4220 for_each_sec(file, sec) {
4271 if (!(sec->sh.sh_flags & SHF_ 4221 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
4272 continue; 4222 continue;
4273 4223
4274 sec_for_each_sym(sec, func) { 4224 sec_for_each_sym(sec, func) {
4275 if (func->type != STT 4225 if (func->type != STT_FUNC)
4276 continue; 4226 continue;
4277 4227
4278 add_prefix_symbol(fil 4228 add_prefix_symbol(file, func);
4279 } 4229 }
4280 } 4230 }
4281 4231
4282 return 0; 4232 return 0;
4283 } 4233 }
4284 4234
4285 static int validate_symbol(struct objtool_fil 4235 static int validate_symbol(struct objtool_file *file, struct section *sec,
4286 struct symbol *sym 4236 struct symbol *sym, struct insn_state *state)
4287 { 4237 {
4288 struct instruction *insn; 4238 struct instruction *insn;
4289 int ret; 4239 int ret;
4290 4240
4291 if (!sym->len) { 4241 if (!sym->len) {
4292 WARN("%s() is missing an ELF 4242 WARN("%s() is missing an ELF size annotation", sym->name);
4293 return 1; 4243 return 1;
4294 } 4244 }
4295 4245
4296 if (sym->pfunc != sym || sym->alias ! 4246 if (sym->pfunc != sym || sym->alias != sym)
4297 return 0; 4247 return 0;
4298 4248
4299 insn = find_insn(file, sec, sym->offs 4249 insn = find_insn(file, sec, sym->offset);
4300 if (!insn || insn->ignore || insn->vi 4250 if (!insn || insn->ignore || insn->visited)
4301 return 0; 4251 return 0;
4302 4252
4303 state->uaccess = sym->uaccess_safe; 4253 state->uaccess = sym->uaccess_safe;
4304 4254
4305 ret = validate_branch(file, insn_func 4255 ret = validate_branch(file, insn_func(insn), insn, *state);
4306 if (ret) 4256 if (ret)
4307 BT_INSN(insn, "<=== (sym)"); 4257 BT_INSN(insn, "<=== (sym)");
4308 return ret; 4258 return ret;
4309 } 4259 }
4310 4260
4311 static int validate_section(struct objtool_fi 4261 static int validate_section(struct objtool_file *file, struct section *sec)
4312 { 4262 {
4313 struct insn_state state; 4263 struct insn_state state;
4314 struct symbol *func; 4264 struct symbol *func;
4315 int warnings = 0; 4265 int warnings = 0;
4316 4266
4317 sec_for_each_sym(sec, func) { 4267 sec_for_each_sym(sec, func) {
4318 if (func->type != STT_FUNC) 4268 if (func->type != STT_FUNC)
4319 continue; 4269 continue;
4320 4270
4321 init_insn_state(file, &state, 4271 init_insn_state(file, &state, sec);
4322 set_func_state(&state.cfi); 4272 set_func_state(&state.cfi);
4323 4273
4324 warnings += validate_symbol(f 4274 warnings += validate_symbol(file, sec, func, &state);
4325 } 4275 }
4326 4276
4327 return warnings; 4277 return warnings;
4328 } 4278 }
4329 4279
4330 static int validate_noinstr_sections(struct o 4280 static int validate_noinstr_sections(struct objtool_file *file)
4331 { 4281 {
4332 struct section *sec; 4282 struct section *sec;
4333 int warnings = 0; 4283 int warnings = 0;
4334 4284
4335 sec = find_section_by_name(file->elf, 4285 sec = find_section_by_name(file->elf, ".noinstr.text");
4336 if (sec) { 4286 if (sec) {
4337 warnings += validate_section( 4287 warnings += validate_section(file, sec);
4338 warnings += validate_unwind_h 4288 warnings += validate_unwind_hints(file, sec);
4339 } 4289 }
4340 4290
4341 sec = find_section_by_name(file->elf, 4291 sec = find_section_by_name(file->elf, ".entry.text");
4342 if (sec) { 4292 if (sec) {
4343 warnings += validate_section( 4293 warnings += validate_section(file, sec);
4344 warnings += validate_unwind_h 4294 warnings += validate_unwind_hints(file, sec);
4345 } 4295 }
4346 4296
4347 sec = find_section_by_name(file->elf, 4297 sec = find_section_by_name(file->elf, ".cpuidle.text");
4348 if (sec) { 4298 if (sec) {
4349 warnings += validate_section( 4299 warnings += validate_section(file, sec);
4350 warnings += validate_unwind_h 4300 warnings += validate_unwind_hints(file, sec);
4351 } 4301 }
4352 4302
4353 return warnings; 4303 return warnings;
4354 } 4304 }
4355 4305
4356 static int validate_functions(struct objtool_ 4306 static int validate_functions(struct objtool_file *file)
4357 { 4307 {
4358 struct section *sec; 4308 struct section *sec;
4359 int warnings = 0; 4309 int warnings = 0;
4360 4310
4361 for_each_sec(file, sec) { 4311 for_each_sec(file, sec) {
4362 if (!(sec->sh.sh_flags & SHF_ 4312 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
4363 continue; 4313 continue;
4364 4314
4365 warnings += validate_section( 4315 warnings += validate_section(file, sec);
4366 } 4316 }
4367 4317
4368 return warnings; 4318 return warnings;
4369 } 4319 }
4370 4320
4371 static void mark_endbr_used(struct instructio 4321 static void mark_endbr_used(struct instruction *insn)
4372 { 4322 {
4373 if (!list_empty(&insn->call_node)) 4323 if (!list_empty(&insn->call_node))
4374 list_del_init(&insn->call_nod 4324 list_del_init(&insn->call_node);
4375 } 4325 }
4376 4326
4377 static bool noendbr_range(struct objtool_file 4327 static bool noendbr_range(struct objtool_file *file, struct instruction *insn)
4378 { 4328 {
4379 struct symbol *sym = find_symbol_cont 4329 struct symbol *sym = find_symbol_containing(insn->sec, insn->offset-1);
4380 struct instruction *first; 4330 struct instruction *first;
4381 4331
4382 if (!sym) 4332 if (!sym)
4383 return false; 4333 return false;
4384 4334
4385 first = find_insn(file, sym->sec, sym 4335 first = find_insn(file, sym->sec, sym->offset);
4386 if (!first) 4336 if (!first)
4387 return false; 4337 return false;
4388 4338
4389 if (first->type != INSN_ENDBR && !fir 4339 if (first->type != INSN_ENDBR && !first->noendbr)
4390 return false; 4340 return false;
4391 4341
4392 return insn->offset == sym->offset + 4342 return insn->offset == sym->offset + sym->len;
4393 } 4343 }
4394 4344
4395 static int validate_ibt_insn(struct objtool_f 4345 static int validate_ibt_insn(struct objtool_file *file, struct instruction *insn)
4396 { 4346 {
4397 struct instruction *dest; 4347 struct instruction *dest;
4398 struct reloc *reloc; 4348 struct reloc *reloc;
4399 unsigned long off; 4349 unsigned long off;
4400 int warnings = 0; 4350 int warnings = 0;
4401 4351
4402 /* 4352 /*
4403 * Looking for function pointer load 4353 * Looking for function pointer load relocations. Ignore
4404 * direct/indirect branches: 4354 * direct/indirect branches:
4405 */ 4355 */
4406 switch (insn->type) { 4356 switch (insn->type) {
4407 case INSN_CALL: 4357 case INSN_CALL:
4408 case INSN_CALL_DYNAMIC: 4358 case INSN_CALL_DYNAMIC:
4409 case INSN_JUMP_CONDITIONAL: 4359 case INSN_JUMP_CONDITIONAL:
4410 case INSN_JUMP_UNCONDITIONAL: 4360 case INSN_JUMP_UNCONDITIONAL:
4411 case INSN_JUMP_DYNAMIC: 4361 case INSN_JUMP_DYNAMIC:
4412 case INSN_JUMP_DYNAMIC_CONDITIONAL: 4362 case INSN_JUMP_DYNAMIC_CONDITIONAL:
4413 case INSN_RETURN: 4363 case INSN_RETURN:
4414 case INSN_NOP: 4364 case INSN_NOP:
4415 return 0; 4365 return 0;
4416 default: 4366 default:
4417 break; 4367 break;
4418 } 4368 }
4419 4369
4420 for (reloc = insn_reloc(file, insn); 4370 for (reloc = insn_reloc(file, insn);
4421 reloc; 4371 reloc;
4422 reloc = find_reloc_by_dest_range 4372 reloc = find_reloc_by_dest_range(file->elf, insn->sec,
4423 4373 reloc_offset(reloc) + 1,
4424 4374 (insn->offset + insn->len) - (reloc_offset(reloc) + 1))) {
4425 4375
4426 /* 4376 /*
4427 * static_call_update() refer 4377 * static_call_update() references the trampoline, which
4428 * doesn't have (or need) END 4378 * doesn't have (or need) ENDBR. Skip warning in that case.
4429 */ 4379 */
4430 if (reloc->sym->static_call_t 4380 if (reloc->sym->static_call_tramp)
4431 continue; 4381 continue;
4432 4382
4433 off = reloc->sym->offset; 4383 off = reloc->sym->offset;
4434 if (reloc_type(reloc) == R_X8 4384 if (reloc_type(reloc) == R_X86_64_PC32 ||
4435 reloc_type(reloc) == R_X8 4385 reloc_type(reloc) == R_X86_64_PLT32)
4436 off += arch_dest_relo 4386 off += arch_dest_reloc_offset(reloc_addend(reloc));
4437 else 4387 else
4438 off += reloc_addend(r 4388 off += reloc_addend(reloc);
4439 4389
4440 dest = find_insn(file, reloc- 4390 dest = find_insn(file, reloc->sym->sec, off);
4441 if (!dest) 4391 if (!dest)
4442 continue; 4392 continue;
4443 4393
4444 if (dest->type == INSN_ENDBR) 4394 if (dest->type == INSN_ENDBR) {
4445 mark_endbr_used(dest) 4395 mark_endbr_used(dest);
4446 continue; 4396 continue;
4447 } 4397 }
4448 4398
4449 if (insn_func(dest) && insn_f 4399 if (insn_func(dest) && insn_func(insn) &&
4450 insn_func(dest)->pfunc == 4400 insn_func(dest)->pfunc == insn_func(insn)->pfunc) {
4451 /* 4401 /*
4452 * Anything from->to 4402 * Anything from->to self is either _THIS_IP_ or
4453 * IRET-to-self. 4403 * IRET-to-self.
4454 * 4404 *
4455 * There is no sane w 4405 * There is no sane way to annotate _THIS_IP_ since the
4456 * compiler treats th 4406 * compiler treats the relocation as a constant and is
4457 * happy to fold in o 4407 * happy to fold in offsets, skewing any annotation we
4458 * do, leading to vas 4408 * do, leading to vast amounts of false-positives.
4459 * 4409 *
4460 * There's also compi 4410 * There's also compiler generated _THIS_IP_ through
4461 * KCOV and such whic 4411 * KCOV and such which we have no hope of annotating.
4462 * 4412 *
4463 * As such, blanket a 4413 * As such, blanket accept self-references without
4464 * issue. 4414 * issue.
4465 */ 4415 */
4466 continue; 4416 continue;
4467 } 4417 }
4468 4418
4469 /* 4419 /*
4470 * Accept anything ANNOTATE_N 4420 * Accept anything ANNOTATE_NOENDBR.
4471 */ 4421 */
4472 if (dest->noendbr) 4422 if (dest->noendbr)
4473 continue; 4423 continue;
4474 4424
4475 /* 4425 /*
4476 * Accept if this is the inst 4426 * Accept if this is the instruction after a symbol
4477 * that is (no)endbr -- typic 4427 * that is (no)endbr -- typical code-range usage.
4478 */ 4428 */
4479 if (noendbr_range(file, dest) 4429 if (noendbr_range(file, dest))
4480 continue; 4430 continue;
4481 4431
4482 WARN_INSN(insn, "relocation t 4432 WARN_INSN(insn, "relocation to !ENDBR: %s", offstr(dest->sec, dest->offset));
4483 4433
4484 warnings++; 4434 warnings++;
4485 } 4435 }
4486 4436
4487 return warnings; 4437 return warnings;
4488 } 4438 }
4489 4439
4490 static int validate_ibt_data_reloc(struct obj 4440 static int validate_ibt_data_reloc(struct objtool_file *file,
4491 struct rel 4441 struct reloc *reloc)
4492 { 4442 {
4493 struct instruction *dest; 4443 struct instruction *dest;
4494 4444
4495 dest = find_insn(file, reloc->sym->se 4445 dest = find_insn(file, reloc->sym->sec,
4496 reloc->sym->offset + 4446 reloc->sym->offset + reloc_addend(reloc));
4497 if (!dest) 4447 if (!dest)
4498 return 0; 4448 return 0;
4499 4449
4500 if (dest->type == INSN_ENDBR) { 4450 if (dest->type == INSN_ENDBR) {
4501 mark_endbr_used(dest); 4451 mark_endbr_used(dest);
4502 return 0; 4452 return 0;
4503 } 4453 }
4504 4454
4505 if (dest->noendbr) 4455 if (dest->noendbr)
4506 return 0; 4456 return 0;
4507 4457
4508 WARN_FUNC("data relocation to !ENDBR: 4458 WARN_FUNC("data relocation to !ENDBR: %s",
4509 reloc->sec->base, reloc_off 4459 reloc->sec->base, reloc_offset(reloc),
4510 offstr(dest->sec, dest->off 4460 offstr(dest->sec, dest->offset));
4511 4461
4512 return 1; 4462 return 1;
4513 } 4463 }
4514 4464
4515 /* 4465 /*
4516 * Validate IBT rules and remove used ENDBR i 4466 * Validate IBT rules and remove used ENDBR instructions from the seal list.
4517 * Unused ENDBR instructions will be annotate 4467 * Unused ENDBR instructions will be annotated for sealing (i.e., replaced with
4518 * NOPs) later, in create_ibt_endbr_seal_sect 4468 * NOPs) later, in create_ibt_endbr_seal_sections().
4519 */ 4469 */
4520 static int validate_ibt(struct objtool_file * 4470 static int validate_ibt(struct objtool_file *file)
4521 { 4471 {
4522 struct section *sec; 4472 struct section *sec;
4523 struct reloc *reloc; 4473 struct reloc *reloc;
4524 struct instruction *insn; 4474 struct instruction *insn;
4525 int warnings = 0; 4475 int warnings = 0;
4526 4476
4527 for_each_insn(file, insn) 4477 for_each_insn(file, insn)
4528 warnings += validate_ibt_insn 4478 warnings += validate_ibt_insn(file, insn);
4529 4479
4530 for_each_sec(file, sec) { 4480 for_each_sec(file, sec) {
4531 4481
4532 /* Already done by validate_i 4482 /* Already done by validate_ibt_insn() */
4533 if (sec->sh.sh_flags & SHF_EX 4483 if (sec->sh.sh_flags & SHF_EXECINSTR)
4534 continue; 4484 continue;
4535 4485
4536 if (!sec->rsec) 4486 if (!sec->rsec)
4537 continue; 4487 continue;
4538 4488
4539 /* 4489 /*
4540 * These sections can referen 4490 * These sections can reference text addresses, but not with
4541 * the intent to indirect bra 4491 * the intent to indirect branch to them.
4542 */ 4492 */
4543 if ((!strncmp(sec->name, ".di 4493 if ((!strncmp(sec->name, ".discard", 8) &&
4544 strcmp(sec->name, ".disc 4494 strcmp(sec->name, ".discard.ibt_endbr_noseal")) ||
4545 !strncmp(sec->name, ".deb 4495 !strncmp(sec->name, ".debug", 6) ||
4546 !strcmp(sec->name, ".alti 4496 !strcmp(sec->name, ".altinstructions") ||
4547 !strcmp(sec->name, ".ibt_ 4497 !strcmp(sec->name, ".ibt_endbr_seal") ||
4548 !strcmp(sec->name, ".orc_ 4498 !strcmp(sec->name, ".orc_unwind_ip") ||
4549 !strcmp(sec->name, ".para 4499 !strcmp(sec->name, ".parainstructions") ||
4550 !strcmp(sec->name, ".retp 4500 !strcmp(sec->name, ".retpoline_sites") ||
4551 !strcmp(sec->name, ".smp_ 4501 !strcmp(sec->name, ".smp_locks") ||
4552 !strcmp(sec->name, ".stat 4502 !strcmp(sec->name, ".static_call_sites") ||
4553 !strcmp(sec->name, "_erro 4503 !strcmp(sec->name, "_error_injection_whitelist") ||
4554 !strcmp(sec->name, "_kpro 4504 !strcmp(sec->name, "_kprobe_blacklist") ||
4555 !strcmp(sec->name, "__bug 4505 !strcmp(sec->name, "__bug_table") ||
4556 !strcmp(sec->name, "__ex_ 4506 !strcmp(sec->name, "__ex_table") ||
4557 !strcmp(sec->name, "__jum 4507 !strcmp(sec->name, "__jump_table") ||
4558 !strcmp(sec->name, "__mco 4508 !strcmp(sec->name, "__mcount_loc") ||
4559 !strcmp(sec->name, ".kcfi 4509 !strcmp(sec->name, ".kcfi_traps") ||
4560 strstr(sec->name, "__patc 4510 strstr(sec->name, "__patchable_function_entries"))
4561 continue; 4511 continue;
4562 4512
4563 for_each_reloc(sec->rsec, rel 4513 for_each_reloc(sec->rsec, reloc)
4564 warnings += validate_ 4514 warnings += validate_ibt_data_reloc(file, reloc);
4565 } 4515 }
4566 4516
4567 return warnings; 4517 return warnings;
4568 } 4518 }
4569 4519
4570 static int validate_sls(struct objtool_file * 4520 static int validate_sls(struct objtool_file *file)
4571 { 4521 {
4572 struct instruction *insn, *next_insn; 4522 struct instruction *insn, *next_insn;
4573 int warnings = 0; 4523 int warnings = 0;
4574 4524
4575 for_each_insn(file, insn) { 4525 for_each_insn(file, insn) {
4576 next_insn = next_insn_same_se 4526 next_insn = next_insn_same_sec(file, insn);
4577 4527
4578 if (insn->retpoline_safe) 4528 if (insn->retpoline_safe)
4579 continue; 4529 continue;
4580 4530
4581 switch (insn->type) { 4531 switch (insn->type) {
4582 case INSN_RETURN: 4532 case INSN_RETURN:
4583 if (!next_insn || nex 4533 if (!next_insn || next_insn->type != INSN_TRAP) {
4584 WARN_INSN(ins 4534 WARN_INSN(insn, "missing int3 after ret");
4585 warnings++; 4535 warnings++;
4586 } 4536 }
4587 4537
4588 break; 4538 break;
4589 case INSN_JUMP_DYNAMIC: 4539 case INSN_JUMP_DYNAMIC:
4590 if (!next_insn || nex 4540 if (!next_insn || next_insn->type != INSN_TRAP) {
4591 WARN_INSN(ins 4541 WARN_INSN(insn, "missing int3 after indirect jump");
4592 warnings++; 4542 warnings++;
4593 } 4543 }
4594 break; 4544 break;
4595 default: 4545 default:
4596 break; 4546 break;
4597 } 4547 }
4598 } 4548 }
4599 4549
4600 return warnings; 4550 return warnings;
4601 } 4551 }
4602 4552
4603 static bool ignore_noreturn_call(struct instr 4553 static bool ignore_noreturn_call(struct instruction *insn)
4604 { 4554 {
4605 struct symbol *call_dest = insn_call_ 4555 struct symbol *call_dest = insn_call_dest(insn);
4606 4556
4607 /* 4557 /*
4608 * FIXME: hack, we need a real noretu 4558 * FIXME: hack, we need a real noreturn solution
4609 * 4559 *
4610 * Problem is, exc_double_fault() may 4560 * Problem is, exc_double_fault() may or may not return, depending on
4611 * whether CONFIG_X86_ESPFIX64 is set 4561 * whether CONFIG_X86_ESPFIX64 is set. But objtool has no visibility
4612 * to the kernel config. 4562 * to the kernel config.
4613 * 4563 *
4614 * Other potential ways to fix it: 4564 * Other potential ways to fix it:
4615 * 4565 *
4616 * - have compiler communicate __no 4566 * - have compiler communicate __noreturn functions somehow
4617 * - remove CONFIG_X86_ESPFIX64 4567 * - remove CONFIG_X86_ESPFIX64
4618 * - read the .config file 4568 * - read the .config file
4619 * - add a cmdline option 4569 * - add a cmdline option
4620 * - create a generic objtool annot 4570 * - create a generic objtool annotation format (vs a bunch of custom
4621 * formats) and annotate it 4571 * formats) and annotate it
4622 */ 4572 */
4623 if (!strcmp(call_dest->name, "exc_dou 4573 if (!strcmp(call_dest->name, "exc_double_fault")) {
4624 /* prevent further unreachabl 4574 /* prevent further unreachable warnings for the caller */
4625 insn->sym->warned = 1; 4575 insn->sym->warned = 1;
4626 return true; 4576 return true;
4627 } 4577 }
4628 4578
4629 return false; 4579 return false;
4630 } 4580 }
4631 4581
4632 static int validate_reachable_instructions(st 4582 static int validate_reachable_instructions(struct objtool_file *file)
4633 { 4583 {
4634 struct instruction *insn, *prev_insn; 4584 struct instruction *insn, *prev_insn;
4635 struct symbol *call_dest; 4585 struct symbol *call_dest;
4636 int warnings = 0; 4586 int warnings = 0;
4637 4587
4638 if (file->ignore_unreachables) 4588 if (file->ignore_unreachables)
4639 return 0; 4589 return 0;
4640 4590
4641 for_each_insn(file, insn) { 4591 for_each_insn(file, insn) {
4642 if (insn->visited || ignore_u 4592 if (insn->visited || ignore_unreachable_insn(file, insn))
4643 continue; 4593 continue;
4644 4594
4645 prev_insn = prev_insn_same_se 4595 prev_insn = prev_insn_same_sec(file, insn);
4646 if (prev_insn && prev_insn->d 4596 if (prev_insn && prev_insn->dead_end) {
4647 call_dest = insn_call 4597 call_dest = insn_call_dest(prev_insn);
4648 if (call_dest && !ign 4598 if (call_dest && !ignore_noreturn_call(prev_insn)) {
4649 WARN_INSN(ins 4599 WARN_INSN(insn, "%s() is missing a __noreturn annotation",
4650 cal 4600 call_dest->name);
4651 warnings++; 4601 warnings++;
4652 continue; 4602 continue;
4653 } 4603 }
4654 } 4604 }
4655 4605
4656 WARN_INSN(insn, "unreachable 4606 WARN_INSN(insn, "unreachable instruction");
4657 warnings++; 4607 warnings++;
4658 } 4608 }
4659 4609
4660 return warnings; 4610 return warnings;
4661 } 4611 }
4662 4612
4663 /* 'funcs' is a space-separated list of funct 4613 /* 'funcs' is a space-separated list of function names */
4664 static int disas_funcs(const char *funcs) 4614 static int disas_funcs(const char *funcs)
4665 { 4615 {
4666 const char *objdump_str, *cross_compi 4616 const char *objdump_str, *cross_compile;
4667 int size, ret; 4617 int size, ret;
4668 char *cmd; 4618 char *cmd;
4669 4619
4670 cross_compile = getenv("CROSS_COMPILE 4620 cross_compile = getenv("CROSS_COMPILE");
4671 4621
4672 objdump_str = "%sobjdump -wdr %s | ga 4622 objdump_str = "%sobjdump -wdr %s | gawk -M -v _funcs='%s' '"
4673 "BEGIN { split(_funcs 4623 "BEGIN { split(_funcs, funcs); }"
4674 "/^$/ { func_match = 4624 "/^$/ { func_match = 0; }"
4675 "/<.*>:/ { " 4625 "/<.*>:/ { "
4676 "f = gensub(/ 4626 "f = gensub(/.*<(.*)>:/, \"\\\\1\", 1);"
4677 "for (i in fu 4627 "for (i in funcs) {"
4678 "if ( 4628 "if (funcs[i] == f) {"
4679 4629 "func_match = 1;"
4680 4630 "base = strtonum(\"0x\" $1);"
4681 4631 "break;"
4682 "}" 4632 "}"
4683 "}" 4633 "}"
4684 "}" 4634 "}"
4685 "{" 4635 "{"
4686 "if (func_mat 4636 "if (func_match) {"
4687 "addr 4637 "addr = strtonum(\"0x\" $1);"
4688 "prin 4638 "printf(\"%%04x \", addr - base);"
4689 "prin 4639 "print;"
4690 "}" 4640 "}"
4691 "}' 1>&2"; 4641 "}' 1>&2";
4692 4642
4693 /* fake snprintf() to calculate the s 4643 /* fake snprintf() to calculate the size */
4694 size = snprintf(NULL, 0, objdump_str, 4644 size = snprintf(NULL, 0, objdump_str, cross_compile, objname, funcs) + 1;
4695 if (size <= 0) { 4645 if (size <= 0) {
4696 WARN("objdump string size cal 4646 WARN("objdump string size calculation failed");
4697 return -1; 4647 return -1;
4698 } 4648 }
4699 4649
4700 cmd = malloc(size); 4650 cmd = malloc(size);
4701 4651
4702 /* real snprintf() */ 4652 /* real snprintf() */
4703 snprintf(cmd, size, objdump_str, cros 4653 snprintf(cmd, size, objdump_str, cross_compile, objname, funcs);
4704 ret = system(cmd); 4654 ret = system(cmd);
4705 if (ret) { 4655 if (ret) {
4706 WARN("disassembly failed: %d" 4656 WARN("disassembly failed: %d", ret);
4707 return -1; 4657 return -1;
4708 } 4658 }
4709 4659
4710 return 0; 4660 return 0;
4711 } 4661 }
4712 4662
4713 static int disas_warned_funcs(struct objtool_ 4663 static int disas_warned_funcs(struct objtool_file *file)
4714 { 4664 {
4715 struct symbol *sym; 4665 struct symbol *sym;
4716 char *funcs = NULL, *tmp; 4666 char *funcs = NULL, *tmp;
4717 4667
4718 for_each_sym(file, sym) { 4668 for_each_sym(file, sym) {
4719 if (sym->warned) { 4669 if (sym->warned) {
4720 if (!funcs) { 4670 if (!funcs) {
4721 funcs = mallo 4671 funcs = malloc(strlen(sym->name) + 1);
4722 strcpy(funcs, 4672 strcpy(funcs, sym->name);
4723 } else { 4673 } else {
4724 tmp = malloc( 4674 tmp = malloc(strlen(funcs) + strlen(sym->name) + 2);
4725 sprintf(tmp, 4675 sprintf(tmp, "%s %s", funcs, sym->name);
4726 free(funcs); 4676 free(funcs);
4727 funcs = tmp; 4677 funcs = tmp;
4728 } 4678 }
4729 } 4679 }
4730 } 4680 }
4731 4681
4732 if (funcs) 4682 if (funcs)
4733 disas_funcs(funcs); 4683 disas_funcs(funcs);
4734 4684
4735 return 0; 4685 return 0;
4736 } 4686 }
4737 4687
4738 struct insn_chunk { 4688 struct insn_chunk {
4739 void *addr; 4689 void *addr;
4740 struct insn_chunk *next; 4690 struct insn_chunk *next;
4741 }; 4691 };
4742 4692
4743 /* 4693 /*
4744 * Reduce peak RSS usage by freeing insns mem 4694 * Reduce peak RSS usage by freeing insns memory before writing the ELF file,
4745 * which can trigger more allocations for .de 4695 * which can trigger more allocations for .debug_* sections whose data hasn't
4746 * been read yet. 4696 * been read yet.
4747 */ 4697 */
4748 static void free_insns(struct objtool_file *f 4698 static void free_insns(struct objtool_file *file)
4749 { 4699 {
4750 struct instruction *insn; 4700 struct instruction *insn;
4751 struct insn_chunk *chunks = NULL, *ch 4701 struct insn_chunk *chunks = NULL, *chunk;
4752 4702
4753 for_each_insn(file, insn) { 4703 for_each_insn(file, insn) {
4754 if (!insn->idx) { 4704 if (!insn->idx) {
4755 chunk = malloc(sizeof 4705 chunk = malloc(sizeof(*chunk));
4756 chunk->addr = insn; 4706 chunk->addr = insn;
4757 chunk->next = chunks; 4707 chunk->next = chunks;
4758 chunks = chunk; 4708 chunks = chunk;
4759 } 4709 }
4760 } 4710 }
4761 4711
4762 for (chunk = chunks; chunk; chunk = c 4712 for (chunk = chunks; chunk; chunk = chunk->next)
4763 free(chunk->addr); 4713 free(chunk->addr);
4764 } 4714 }
4765 4715
4766 int check(struct objtool_file *file) 4716 int check(struct objtool_file *file)
4767 { 4717 {
4768 int ret, warnings = 0; 4718 int ret, warnings = 0;
4769 4719
4770 arch_initial_func_cfi_state(&initial_ 4720 arch_initial_func_cfi_state(&initial_func_cfi);
4771 init_cfi_state(&init_cfi); 4721 init_cfi_state(&init_cfi);
4772 init_cfi_state(&func_cfi); 4722 init_cfi_state(&func_cfi);
4773 set_func_state(&func_cfi); 4723 set_func_state(&func_cfi);
4774 init_cfi_state(&force_undefined_cfi); 4724 init_cfi_state(&force_undefined_cfi);
4775 force_undefined_cfi.force_undefined = 4725 force_undefined_cfi.force_undefined = true;
4776 4726
4777 if (!cfi_hash_alloc(1UL << (file->elf 4727 if (!cfi_hash_alloc(1UL << (file->elf->symbol_bits - 3)))
4778 goto out; 4728 goto out;
4779 4729
4780 cfi_hash_add(&init_cfi); 4730 cfi_hash_add(&init_cfi);
4781 cfi_hash_add(&func_cfi); 4731 cfi_hash_add(&func_cfi);
4782 4732
4783 ret = decode_sections(file); 4733 ret = decode_sections(file);
4784 if (ret < 0) 4734 if (ret < 0)
4785 goto out; 4735 goto out;
4786 4736
4787 warnings += ret; 4737 warnings += ret;
4788 4738
4789 if (!nr_insns) 4739 if (!nr_insns)
4790 goto out; 4740 goto out;
4791 4741
4792 if (opts.retpoline) { 4742 if (opts.retpoline) {
4793 ret = validate_retpoline(file 4743 ret = validate_retpoline(file);
4794 if (ret < 0) 4744 if (ret < 0)
4795 return ret; 4745 return ret;
4796 warnings += ret; 4746 warnings += ret;
4797 } 4747 }
4798 4748
4799 if (opts.stackval || opts.orc || opts 4749 if (opts.stackval || opts.orc || opts.uaccess) {
4800 ret = validate_functions(file 4750 ret = validate_functions(file);
4801 if (ret < 0) 4751 if (ret < 0)
4802 goto out; 4752 goto out;
4803 warnings += ret; 4753 warnings += ret;
4804 4754
4805 ret = validate_unwind_hints(f 4755 ret = validate_unwind_hints(file, NULL);
4806 if (ret < 0) 4756 if (ret < 0)
4807 goto out; 4757 goto out;
4808 warnings += ret; 4758 warnings += ret;
4809 4759
4810 if (!warnings) { 4760 if (!warnings) {
4811 ret = validate_reacha 4761 ret = validate_reachable_instructions(file);
4812 if (ret < 0) 4762 if (ret < 0)
4813 goto out; 4763 goto out;
4814 warnings += ret; 4764 warnings += ret;
4815 } 4765 }
4816 4766
4817 } else if (opts.noinstr) { 4767 } else if (opts.noinstr) {
4818 ret = validate_noinstr_sectio 4768 ret = validate_noinstr_sections(file);
4819 if (ret < 0) 4769 if (ret < 0)
4820 goto out; 4770 goto out;
4821 warnings += ret; 4771 warnings += ret;
4822 } 4772 }
4823 4773
4824 if (opts.unret) { 4774 if (opts.unret) {
4825 /* 4775 /*
4826 * Must be after validate_bra 4776 * Must be after validate_branch() and friends, it plays
4827 * further games with insn->v 4777 * further games with insn->visited.
4828 */ 4778 */
4829 ret = validate_unrets(file); 4779 ret = validate_unrets(file);
4830 if (ret < 0) 4780 if (ret < 0)
4831 return ret; 4781 return ret;
4832 warnings += ret; 4782 warnings += ret;
4833 } 4783 }
4834 4784
4835 if (opts.ibt) { 4785 if (opts.ibt) {
4836 ret = validate_ibt(file); 4786 ret = validate_ibt(file);
4837 if (ret < 0) 4787 if (ret < 0)
4838 goto out; 4788 goto out;
4839 warnings += ret; 4789 warnings += ret;
4840 } 4790 }
4841 4791
4842 if (opts.sls) { 4792 if (opts.sls) {
4843 ret = validate_sls(file); 4793 ret = validate_sls(file);
4844 if (ret < 0) 4794 if (ret < 0)
4845 goto out; 4795 goto out;
4846 warnings += ret; 4796 warnings += ret;
4847 } 4797 }
4848 4798
4849 if (opts.static_call) { 4799 if (opts.static_call) {
4850 ret = create_static_call_sect 4800 ret = create_static_call_sections(file);
4851 if (ret < 0) 4801 if (ret < 0)
4852 goto out; 4802 goto out;
4853 warnings += ret; 4803 warnings += ret;
4854 } 4804 }
4855 4805
4856 if (opts.retpoline) { 4806 if (opts.retpoline) {
4857 ret = create_retpoline_sites_ 4807 ret = create_retpoline_sites_sections(file);
4858 if (ret < 0) 4808 if (ret < 0)
4859 goto out; 4809 goto out;
4860 warnings += ret; 4810 warnings += ret;
4861 } 4811 }
4862 4812
4863 if (opts.cfi) { 4813 if (opts.cfi) {
4864 ret = create_cfi_sections(fil 4814 ret = create_cfi_sections(file);
4865 if (ret < 0) 4815 if (ret < 0)
4866 goto out; 4816 goto out;
4867 warnings += ret; 4817 warnings += ret;
4868 } 4818 }
4869 4819
4870 if (opts.rethunk) { 4820 if (opts.rethunk) {
4871 ret = create_return_sites_sec 4821 ret = create_return_sites_sections(file);
4872 if (ret < 0) 4822 if (ret < 0)
4873 goto out; 4823 goto out;
4874 warnings += ret; 4824 warnings += ret;
4875 4825
4876 if (opts.hack_skylake) { 4826 if (opts.hack_skylake) {
4877 ret = create_direct_c 4827 ret = create_direct_call_sections(file);
4878 if (ret < 0) 4828 if (ret < 0)
4879 goto out; 4829 goto out;
4880 warnings += ret; 4830 warnings += ret;
4881 } 4831 }
4882 } 4832 }
4883 4833
4884 if (opts.mcount) { 4834 if (opts.mcount) {
4885 ret = create_mcount_loc_secti 4835 ret = create_mcount_loc_sections(file);
4886 if (ret < 0) 4836 if (ret < 0)
4887 goto out; 4837 goto out;
4888 warnings += ret; 4838 warnings += ret;
4889 } 4839 }
4890 4840
4891 if (opts.prefix) { 4841 if (opts.prefix) {
4892 ret = add_prefix_symbols(file 4842 ret = add_prefix_symbols(file);
4893 if (ret < 0) 4843 if (ret < 0)
4894 return ret; 4844 return ret;
4895 warnings += ret; 4845 warnings += ret;
4896 } 4846 }
4897 4847
4898 if (opts.ibt) { 4848 if (opts.ibt) {
4899 ret = create_ibt_endbr_seal_s 4849 ret = create_ibt_endbr_seal_sections(file);
4900 if (ret < 0) 4850 if (ret < 0)
4901 goto out; 4851 goto out;
4902 warnings += ret; 4852 warnings += ret;
4903 } 4853 }
4904 4854
4905 if (opts.orc && nr_insns) { 4855 if (opts.orc && nr_insns) {
4906 ret = orc_create(file); 4856 ret = orc_create(file);
4907 if (ret < 0) 4857 if (ret < 0)
4908 goto out; 4858 goto out;
4909 warnings += ret; 4859 warnings += ret;
4910 } 4860 }
4911 4861
4912 free_insns(file); 4862 free_insns(file);
4913 4863
4914 if (opts.verbose) 4864 if (opts.verbose)
4915 disas_warned_funcs(file); 4865 disas_warned_funcs(file);
4916 4866
4917 if (opts.stats) { 4867 if (opts.stats) {
4918 printf("nr_insns_visited: %ld 4868 printf("nr_insns_visited: %ld\n", nr_insns_visited);
4919 printf("nr_cfi: %ld\n", nr_cf 4869 printf("nr_cfi: %ld\n", nr_cfi);
4920 printf("nr_cfi_reused: %ld\n" 4870 printf("nr_cfi_reused: %ld\n", nr_cfi_reused);
4921 printf("nr_cfi_cache: %ld\n", 4871 printf("nr_cfi_cache: %ld\n", nr_cfi_cache);
4922 } 4872 }
4923 4873
4924 out: 4874 out:
4925 /* 4875 /*
4926 * For now, don't fail the kernel bu 4876 * For now, don't fail the kernel build on fatal warnings. These
4927 * errors are still fairly common du 4877 * errors are still fairly common due to the growing matrix of
4928 * supported toolchains and their re 4878 * supported toolchains and their recent pace of change.
4929 */ 4879 */
4930 return 0; 4880 return 0;
4931 } 4881 }
4932 4882
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.