1 // SPDX-License-Identifier: GPL-2.0-or-later 1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /* 2 /*
3 * Copyright (C) 2015-2017 Josh Poimboeuf <jpo 3 * Copyright (C) 2015-2017 Josh Poimboeuf <jpoimboe@redhat.com>
4 */ 4 */
5 5
6 #include <string.h> 6 #include <string.h>
7 #include <stdlib.h> 7 #include <stdlib.h>
8 #include <inttypes.h> 8 #include <inttypes.h>
9 #include <sys/mman.h> 9 #include <sys/mman.h>
10 10
11 #include <objtool/builtin.h> 11 #include <objtool/builtin.h>
12 #include <objtool/cfi.h> 12 #include <objtool/cfi.h>
13 #include <objtool/arch.h> 13 #include <objtool/arch.h>
14 #include <objtool/check.h> 14 #include <objtool/check.h>
15 #include <objtool/special.h> 15 #include <objtool/special.h>
16 #include <objtool/warn.h> 16 #include <objtool/warn.h>
17 #include <objtool/endianness.h> 17 #include <objtool/endianness.h>
18 18
19 #include <linux/objtool_types.h> 19 #include <linux/objtool_types.h>
20 #include <linux/hashtable.h> 20 #include <linux/hashtable.h>
21 #include <linux/kernel.h> 21 #include <linux/kernel.h>
22 #include <linux/static_call_types.h> 22 #include <linux/static_call_types.h>
23 #include <linux/string.h> <<
24 23
25 struct alternative { 24 struct alternative {
26 struct alternative *next; 25 struct alternative *next;
27 struct instruction *insn; 26 struct instruction *insn;
28 bool skip_orig; 27 bool skip_orig;
29 }; 28 };
30 29
31 static unsigned long nr_cfi, nr_cfi_reused, nr 30 static unsigned long nr_cfi, nr_cfi_reused, nr_cfi_cache;
32 31
33 static struct cfi_init_state initial_func_cfi; 32 static struct cfi_init_state initial_func_cfi;
34 static struct cfi_state init_cfi; 33 static struct cfi_state init_cfi;
35 static struct cfi_state func_cfi; 34 static struct cfi_state func_cfi;
36 static struct cfi_state force_undefined_cfi; 35 static struct cfi_state force_undefined_cfi;
37 36
38 struct instruction *find_insn(struct objtool_f 37 struct instruction *find_insn(struct objtool_file *file,
39 struct section * 38 struct section *sec, unsigned long offset)
40 { 39 {
41 struct instruction *insn; 40 struct instruction *insn;
42 41
43 hash_for_each_possible(file->insn_hash 42 hash_for_each_possible(file->insn_hash, insn, hash, sec_offset_hash(sec, offset)) {
44 if (insn->sec == sec && insn-> 43 if (insn->sec == sec && insn->offset == offset)
45 return insn; 44 return insn;
46 } 45 }
47 46
48 return NULL; 47 return NULL;
49 } 48 }
50 49
51 struct instruction *next_insn_same_sec(struct 50 struct instruction *next_insn_same_sec(struct objtool_file *file,
52 struct 51 struct instruction *insn)
53 { 52 {
54 if (insn->idx == INSN_CHUNK_MAX) 53 if (insn->idx == INSN_CHUNK_MAX)
55 return find_insn(file, insn->s 54 return find_insn(file, insn->sec, insn->offset + insn->len);
56 55
57 insn++; 56 insn++;
58 if (!insn->len) 57 if (!insn->len)
59 return NULL; 58 return NULL;
60 59
61 return insn; 60 return insn;
62 } 61 }
63 62
64 static struct instruction *next_insn_same_func 63 static struct instruction *next_insn_same_func(struct objtool_file *file,
65 64 struct instruction *insn)
66 { 65 {
67 struct instruction *next = next_insn_s 66 struct instruction *next = next_insn_same_sec(file, insn);
68 struct symbol *func = insn_func(insn); 67 struct symbol *func = insn_func(insn);
69 68
70 if (!func) 69 if (!func)
71 return NULL; 70 return NULL;
72 71
73 if (next && insn_func(next) == func) 72 if (next && insn_func(next) == func)
74 return next; 73 return next;
75 74
76 /* Check if we're already in the subfu 75 /* Check if we're already in the subfunction: */
77 if (func == func->cfunc) 76 if (func == func->cfunc)
78 return NULL; 77 return NULL;
79 78
80 /* Move to the subfunction: */ 79 /* Move to the subfunction: */
81 return find_insn(file, func->cfunc->se 80 return find_insn(file, func->cfunc->sec, func->cfunc->offset);
82 } 81 }
83 82
84 static struct instruction *prev_insn_same_sec( 83 static struct instruction *prev_insn_same_sec(struct objtool_file *file,
85 84 struct instruction *insn)
86 { 85 {
87 if (insn->idx == 0) { 86 if (insn->idx == 0) {
88 if (insn->prev_len) 87 if (insn->prev_len)
89 return find_insn(file, 88 return find_insn(file, insn->sec, insn->offset - insn->prev_len);
90 return NULL; 89 return NULL;
91 } 90 }
92 91
93 return insn - 1; 92 return insn - 1;
94 } 93 }
95 94
96 static struct instruction *prev_insn_same_sym( 95 static struct instruction *prev_insn_same_sym(struct objtool_file *file,
97 96 struct instruction *insn)
98 { 97 {
99 struct instruction *prev = prev_insn_s 98 struct instruction *prev = prev_insn_same_sec(file, insn);
100 99
101 if (prev && insn_func(prev) == insn_fu 100 if (prev && insn_func(prev) == insn_func(insn))
102 return prev; 101 return prev;
103 102
104 return NULL; 103 return NULL;
105 } 104 }
106 105
107 #define for_each_insn(file, insn) 106 #define for_each_insn(file, insn) \
108 for (struct section *__sec, *__fake = 107 for (struct section *__sec, *__fake = (struct section *)1; \
109 __fake; __fake = NULL) 108 __fake; __fake = NULL) \
110 for_each_sec(file, __sec) 109 for_each_sec(file, __sec) \
111 sec_for_each_insn(file 110 sec_for_each_insn(file, __sec, insn)
112 111
113 #define func_for_each_insn(file, func, insn) 112 #define func_for_each_insn(file, func, insn) \
114 for (insn = find_insn(file, func->sec, 113 for (insn = find_insn(file, func->sec, func->offset); \
115 insn; 114 insn; \
116 insn = next_insn_same_func(file, 115 insn = next_insn_same_func(file, insn))
117 116
118 #define sym_for_each_insn(file, sym, insn) 117 #define sym_for_each_insn(file, sym, insn) \
119 for (insn = find_insn(file, sym->sec, 118 for (insn = find_insn(file, sym->sec, sym->offset); \
120 insn && insn->offset < sym->offse 119 insn && insn->offset < sym->offset + sym->len; \
121 insn = next_insn_same_sec(file, i 120 insn = next_insn_same_sec(file, insn))
122 121
123 #define sym_for_each_insn_continue_reverse(fil 122 #define sym_for_each_insn_continue_reverse(file, sym, insn) \
124 for (insn = prev_insn_same_sec(file, i 123 for (insn = prev_insn_same_sec(file, insn); \
125 insn && insn->offset >= sym->offs 124 insn && insn->offset >= sym->offset; \
126 insn = prev_insn_same_sec(file, i 125 insn = prev_insn_same_sec(file, insn))
127 126
128 #define sec_for_each_insn_from(file, insn) 127 #define sec_for_each_insn_from(file, insn) \
129 for (; insn; insn = next_insn_same_sec 128 for (; insn; insn = next_insn_same_sec(file, insn))
130 129
131 #define sec_for_each_insn_continue(file, insn) 130 #define sec_for_each_insn_continue(file, insn) \
132 for (insn = next_insn_same_sec(file, i 131 for (insn = next_insn_same_sec(file, insn); insn; \
133 insn = next_insn_same_sec(file, i 132 insn = next_insn_same_sec(file, insn))
134 133
135 static inline struct symbol *insn_call_dest(st 134 static inline struct symbol *insn_call_dest(struct instruction *insn)
136 { 135 {
137 if (insn->type == INSN_JUMP_DYNAMIC || 136 if (insn->type == INSN_JUMP_DYNAMIC ||
138 insn->type == INSN_CALL_DYNAMIC) 137 insn->type == INSN_CALL_DYNAMIC)
139 return NULL; 138 return NULL;
140 139
141 return insn->_call_dest; 140 return insn->_call_dest;
142 } 141 }
143 142
144 static inline struct reloc *insn_jump_table(st 143 static inline struct reloc *insn_jump_table(struct instruction *insn)
145 { 144 {
146 if (insn->type == INSN_JUMP_DYNAMIC || 145 if (insn->type == INSN_JUMP_DYNAMIC ||
147 insn->type == INSN_CALL_DYNAMIC) 146 insn->type == INSN_CALL_DYNAMIC)
148 return insn->_jump_table; 147 return insn->_jump_table;
149 148
150 return NULL; 149 return NULL;
151 } 150 }
152 151
153 static bool is_jump_table_jump(struct instruct 152 static bool is_jump_table_jump(struct instruction *insn)
154 { 153 {
155 struct alt_group *alt_group = insn->al 154 struct alt_group *alt_group = insn->alt_group;
156 155
157 if (insn_jump_table(insn)) 156 if (insn_jump_table(insn))
158 return true; 157 return true;
159 158
160 /* Retpoline alternative for a jump ta 159 /* Retpoline alternative for a jump table? */
161 return alt_group && alt_group->orig_gr 160 return alt_group && alt_group->orig_group &&
162 insn_jump_table(alt_group->orig 161 insn_jump_table(alt_group->orig_group->first_insn);
163 } 162 }
164 163
165 static bool is_sibling_call(struct instruction 164 static bool is_sibling_call(struct instruction *insn)
166 { 165 {
167 /* 166 /*
168 * Assume only STT_FUNC calls have jum 167 * Assume only STT_FUNC calls have jump-tables.
169 */ 168 */
170 if (insn_func(insn)) { 169 if (insn_func(insn)) {
171 /* An indirect jump is either 170 /* An indirect jump is either a sibling call or a jump to a table. */
172 if (insn->type == INSN_JUMP_DY 171 if (insn->type == INSN_JUMP_DYNAMIC)
173 return !is_jump_table_ 172 return !is_jump_table_jump(insn);
174 } 173 }
175 174
176 /* add_jump_destinations() sets insn_c 175 /* add_jump_destinations() sets insn_call_dest(insn) for sibling calls. */
177 return (is_static_jump(insn) && insn_c 176 return (is_static_jump(insn) && insn_call_dest(insn));
178 } 177 }
179 178
180 /* 179 /*
181 * Checks if a string ends with another. <<
182 */ <<
183 static bool str_ends_with(const char *s, const <<
184 { <<
185 const int slen = strlen(s); <<
186 const int sublen = strlen(sub); <<
187 <<
188 if (sublen > slen) <<
189 return 0; <<
190 <<
191 return !memcmp(s + slen - sublen, sub, <<
192 } <<
193 <<
194 /* <<
195 * Checks if a function is a Rust "noreturn" o <<
196 */ <<
197 static bool is_rust_noreturn(const struct symb <<
198 { <<
199 /* <<
200 * If it does not start with "_R", the <<
201 */ <<
202 if (strncmp(func->name, "_R", 2)) <<
203 return false; <<
204 <<
205 /* <<
206 * These are just heuristics -- we do <<
207 * name, due to the crate disambiguato <<
208 * as well as changes to the source co <<
209 * these come from the Rust standard l <<
210 */ <<
211 return str_ends_with(func->name, "_4co <<
212 str_ends_with(func->name, "_4co <<
213 str_ends_with(func->name, "_4co <<
214 str_ends_with(func->name, "_4co <<
215 str_ends_with(func->name, "_4co <<
216 str_ends_with(func->name, "_4co <<
217 str_ends_with(func->name, "_4co <<
218 str_ends_with(func->name, "_4co <<
219 str_ends_with(func->name, "_4co <<
220 str_ends_with(func->name, "_4co <<
221 strstr(func->name, "_4core9pani <<
222 (strstr(func->name, "_4core5sli <<
223 str_ends_with(func->name, "_fa <<
224 } <<
225 <<
226 /* <<
227 * This checks to see if the given function is 180 * This checks to see if the given function is a "noreturn" function.
228 * 181 *
229 * For global functions which are outside the 182 * For global functions which are outside the scope of this object file, we
230 * have to keep a manual list of them. 183 * have to keep a manual list of them.
231 * 184 *
232 * For local functions, we have to detect them 185 * For local functions, we have to detect them manually by simply looking for
233 * the lack of a return instruction. 186 * the lack of a return instruction.
234 */ 187 */
235 static bool __dead_end_function(struct objtool 188 static bool __dead_end_function(struct objtool_file *file, struct symbol *func,
236 int recursion) 189 int recursion)
237 { 190 {
238 int i; 191 int i;
239 struct instruction *insn; 192 struct instruction *insn;
240 bool empty = true; 193 bool empty = true;
241 194
242 #define NORETURN(func) __stringify(func), 195 #define NORETURN(func) __stringify(func),
243 static const char * const global_noret 196 static const char * const global_noreturns[] = {
244 #include "noreturns.h" 197 #include "noreturns.h"
245 }; 198 };
246 #undef NORETURN 199 #undef NORETURN
247 200
248 if (!func) 201 if (!func)
249 return false; 202 return false;
250 203
251 if (func->bind == STB_GLOBAL || func-> !! 204 if (func->bind == STB_GLOBAL || func->bind == STB_WEAK)
252 if (is_rust_noreturn(func)) <<
253 return true; <<
254 <<
255 for (i = 0; i < ARRAY_SIZE(glo 205 for (i = 0; i < ARRAY_SIZE(global_noreturns); i++)
256 if (!strcmp(func->name 206 if (!strcmp(func->name, global_noreturns[i]))
257 return true; 207 return true;
258 } <<
259 208
260 if (func->bind == STB_WEAK) 209 if (func->bind == STB_WEAK)
261 return false; 210 return false;
262 211
263 if (!func->len) 212 if (!func->len)
264 return false; 213 return false;
265 214
266 insn = find_insn(file, func->sec, func 215 insn = find_insn(file, func->sec, func->offset);
267 if (!insn || !insn_func(insn)) 216 if (!insn || !insn_func(insn))
268 return false; 217 return false;
269 218
270 func_for_each_insn(file, func, insn) { 219 func_for_each_insn(file, func, insn) {
271 empty = false; 220 empty = false;
272 221
273 if (insn->type == INSN_RETURN) 222 if (insn->type == INSN_RETURN)
274 return false; 223 return false;
275 } 224 }
276 225
277 if (empty) 226 if (empty)
278 return false; 227 return false;
279 228
280 /* 229 /*
281 * A function can have a sibling call 230 * A function can have a sibling call instead of a return. In that
282 * case, the function's dead-end statu 231 * case, the function's dead-end status depends on whether the target
283 * of the sibling call returns. 232 * of the sibling call returns.
284 */ 233 */
285 func_for_each_insn(file, func, insn) { 234 func_for_each_insn(file, func, insn) {
286 if (is_sibling_call(insn)) { 235 if (is_sibling_call(insn)) {
287 struct instruction *de 236 struct instruction *dest = insn->jump_dest;
288 237
289 if (!dest) 238 if (!dest)
290 /* sibling cal 239 /* sibling call to another file */
291 return false; 240 return false;
292 241
293 /* local sibling call 242 /* local sibling call */
294 if (recursion == 5) { 243 if (recursion == 5) {
295 /* 244 /*
296 * Infinite re 245 * Infinite recursion: two functions have
297 * sibling cal 246 * sibling calls to each other. This is a very
298 * rare case. 247 * rare case. It means they aren't dead ends.
299 */ 248 */
300 return false; 249 return false;
301 } 250 }
302 251
303 return __dead_end_func 252 return __dead_end_function(file, insn_func(dest), recursion+1);
304 } 253 }
305 } 254 }
306 255
307 return true; 256 return true;
308 } 257 }
309 258
310 static bool dead_end_function(struct objtool_f 259 static bool dead_end_function(struct objtool_file *file, struct symbol *func)
311 { 260 {
312 return __dead_end_function(file, func, 261 return __dead_end_function(file, func, 0);
313 } 262 }
314 263
315 static void init_cfi_state(struct cfi_state *c 264 static void init_cfi_state(struct cfi_state *cfi)
316 { 265 {
317 int i; 266 int i;
318 267
319 for (i = 0; i < CFI_NUM_REGS; i++) { 268 for (i = 0; i < CFI_NUM_REGS; i++) {
320 cfi->regs[i].base = CFI_UNDEFI 269 cfi->regs[i].base = CFI_UNDEFINED;
321 cfi->vals[i].base = CFI_UNDEFI 270 cfi->vals[i].base = CFI_UNDEFINED;
322 } 271 }
323 cfi->cfa.base = CFI_UNDEFINED; 272 cfi->cfa.base = CFI_UNDEFINED;
324 cfi->drap_reg = CFI_UNDEFINED; 273 cfi->drap_reg = CFI_UNDEFINED;
325 cfi->drap_offset = -1; 274 cfi->drap_offset = -1;
326 } 275 }
327 276
328 static void init_insn_state(struct objtool_fil 277 static void init_insn_state(struct objtool_file *file, struct insn_state *state,
329 struct section *se 278 struct section *sec)
330 { 279 {
331 memset(state, 0, sizeof(*state)); 280 memset(state, 0, sizeof(*state));
332 init_cfi_state(&state->cfi); 281 init_cfi_state(&state->cfi);
333 282
334 /* 283 /*
335 * We need the full vmlinux for noinst 284 * We need the full vmlinux for noinstr validation, otherwise we can
336 * not correctly determine insn_call_d 285 * not correctly determine insn_call_dest(insn)->sec (external symbols
337 * do not have a section). 286 * do not have a section).
338 */ 287 */
339 if (opts.link && opts.noinstr && sec) 288 if (opts.link && opts.noinstr && sec)
340 state->noinstr = sec->noinstr; 289 state->noinstr = sec->noinstr;
341 } 290 }
342 291
343 static struct cfi_state *cfi_alloc(void) 292 static struct cfi_state *cfi_alloc(void)
344 { 293 {
345 struct cfi_state *cfi = calloc(1, size !! 294 struct cfi_state *cfi = calloc(sizeof(struct cfi_state), 1);
346 if (!cfi) { 295 if (!cfi) {
347 WARN("calloc failed"); 296 WARN("calloc failed");
348 exit(1); 297 exit(1);
349 } 298 }
350 nr_cfi++; 299 nr_cfi++;
351 return cfi; 300 return cfi;
352 } 301 }
353 302
354 static int cfi_bits; 303 static int cfi_bits;
355 static struct hlist_head *cfi_hash; 304 static struct hlist_head *cfi_hash;
356 305
357 static inline bool cficmp(struct cfi_state *cf 306 static inline bool cficmp(struct cfi_state *cfi1, struct cfi_state *cfi2)
358 { 307 {
359 return memcmp((void *)cfi1 + sizeof(cf 308 return memcmp((void *)cfi1 + sizeof(cfi1->hash),
360 (void *)cfi2 + sizeof(cf 309 (void *)cfi2 + sizeof(cfi2->hash),
361 sizeof(struct cfi_state) 310 sizeof(struct cfi_state) - sizeof(struct hlist_node));
362 } 311 }
363 312
364 static inline u32 cfi_key(struct cfi_state *cf 313 static inline u32 cfi_key(struct cfi_state *cfi)
365 { 314 {
366 return jhash((void *)cfi + sizeof(cfi- 315 return jhash((void *)cfi + sizeof(cfi->hash),
367 sizeof(*cfi) - sizeof(cfi 316 sizeof(*cfi) - sizeof(cfi->hash), 0);
368 } 317 }
369 318
370 static struct cfi_state *cfi_hash_find_or_add( 319 static struct cfi_state *cfi_hash_find_or_add(struct cfi_state *cfi)
371 { 320 {
372 struct hlist_head *head = &cfi_hash[ha 321 struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
373 struct cfi_state *obj; 322 struct cfi_state *obj;
374 323
375 hlist_for_each_entry(obj, head, hash) 324 hlist_for_each_entry(obj, head, hash) {
376 if (!cficmp(cfi, obj)) { 325 if (!cficmp(cfi, obj)) {
377 nr_cfi_cache++; 326 nr_cfi_cache++;
378 return obj; 327 return obj;
379 } 328 }
380 } 329 }
381 330
382 obj = cfi_alloc(); 331 obj = cfi_alloc();
383 *obj = *cfi; 332 *obj = *cfi;
384 hlist_add_head(&obj->hash, head); 333 hlist_add_head(&obj->hash, head);
385 334
386 return obj; 335 return obj;
387 } 336 }
388 337
389 static void cfi_hash_add(struct cfi_state *cfi 338 static void cfi_hash_add(struct cfi_state *cfi)
390 { 339 {
391 struct hlist_head *head = &cfi_hash[ha 340 struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
392 341
393 hlist_add_head(&cfi->hash, head); 342 hlist_add_head(&cfi->hash, head);
394 } 343 }
395 344
396 static void *cfi_hash_alloc(unsigned long size 345 static void *cfi_hash_alloc(unsigned long size)
397 { 346 {
398 cfi_bits = max(10, ilog2(size)); 347 cfi_bits = max(10, ilog2(size));
399 cfi_hash = mmap(NULL, sizeof(struct hl 348 cfi_hash = mmap(NULL, sizeof(struct hlist_head) << cfi_bits,
400 PROT_READ|PROT_WRITE, 349 PROT_READ|PROT_WRITE,
401 MAP_PRIVATE|MAP_ANON, 350 MAP_PRIVATE|MAP_ANON, -1, 0);
402 if (cfi_hash == (void *)-1L) { 351 if (cfi_hash == (void *)-1L) {
403 WARN("mmap fail cfi_hash"); 352 WARN("mmap fail cfi_hash");
404 cfi_hash = NULL; 353 cfi_hash = NULL;
405 } else if (opts.stats) { 354 } else if (opts.stats) {
406 printf("cfi_bits: %d\n", cfi_b 355 printf("cfi_bits: %d\n", cfi_bits);
407 } 356 }
408 357
409 return cfi_hash; 358 return cfi_hash;
410 } 359 }
411 360
412 static unsigned long nr_insns; 361 static unsigned long nr_insns;
413 static unsigned long nr_insns_visited; 362 static unsigned long nr_insns_visited;
414 363
415 /* 364 /*
416 * Call the arch-specific instruction decoder 365 * Call the arch-specific instruction decoder for all the instructions and add
417 * them to the global instruction list. 366 * them to the global instruction list.
418 */ 367 */
419 static int decode_instructions(struct objtool_ 368 static int decode_instructions(struct objtool_file *file)
420 { 369 {
421 struct section *sec; 370 struct section *sec;
422 struct symbol *func; 371 struct symbol *func;
423 unsigned long offset; 372 unsigned long offset;
424 struct instruction *insn; 373 struct instruction *insn;
425 int ret; 374 int ret;
426 375
427 for_each_sec(file, sec) { 376 for_each_sec(file, sec) {
428 struct instruction *insns = NU 377 struct instruction *insns = NULL;
429 u8 prev_len = 0; 378 u8 prev_len = 0;
430 u8 idx = 0; 379 u8 idx = 0;
431 380
432 if (!(sec->sh.sh_flags & SHF_E 381 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
433 continue; 382 continue;
434 383
435 if (strcmp(sec->name, ".altins 384 if (strcmp(sec->name, ".altinstr_replacement") &&
436 strcmp(sec->name, ".altins 385 strcmp(sec->name, ".altinstr_aux") &&
437 strncmp(sec->name, ".disca 386 strncmp(sec->name, ".discard.", 9))
438 sec->text = true; 387 sec->text = true;
439 388
440 if (!strcmp(sec->name, ".noins 389 if (!strcmp(sec->name, ".noinstr.text") ||
441 !strcmp(sec->name, ".entry 390 !strcmp(sec->name, ".entry.text") ||
442 !strcmp(sec->name, ".cpuid 391 !strcmp(sec->name, ".cpuidle.text") ||
443 !strncmp(sec->name, ".text 392 !strncmp(sec->name, ".text..__x86.", 13))
444 sec->noinstr = true; 393 sec->noinstr = true;
445 394
446 /* 395 /*
447 * .init.text code is ran befo 396 * .init.text code is ran before userspace and thus doesn't
448 * strictly need retpolines, e 397 * strictly need retpolines, except for modules which are
449 * loaded late, they very much 398 * loaded late, they very much do need retpoline in their
450 * .init.text 399 * .init.text
451 */ 400 */
452 if (!strcmp(sec->name, ".init. 401 if (!strcmp(sec->name, ".init.text") && !opts.module)
453 sec->init = true; 402 sec->init = true;
454 403
455 for (offset = 0; offset < sec- 404 for (offset = 0; offset < sec->sh.sh_size; offset += insn->len) {
456 if (!insns || idx == I 405 if (!insns || idx == INSN_CHUNK_MAX) {
457 insns = calloc 406 insns = calloc(sizeof(*insn), INSN_CHUNK_SIZE);
458 if (!insns) { 407 if (!insns) {
459 WARN(" 408 WARN("malloc failed");
460 return 409 return -1;
461 } 410 }
462 idx = 0; 411 idx = 0;
463 } else { 412 } else {
464 idx++; 413 idx++;
465 } 414 }
466 insn = &insns[idx]; 415 insn = &insns[idx];
467 insn->idx = idx; 416 insn->idx = idx;
468 417
469 INIT_LIST_HEAD(&insn-> 418 INIT_LIST_HEAD(&insn->call_node);
470 insn->sec = sec; 419 insn->sec = sec;
471 insn->offset = offset; 420 insn->offset = offset;
472 insn->prev_len = prev_ 421 insn->prev_len = prev_len;
473 422
474 ret = arch_decode_inst 423 ret = arch_decode_instruction(file, sec, offset,
475 424 sec->sh.sh_size - offset,
476 425 insn);
477 if (ret) 426 if (ret)
478 return ret; 427 return ret;
479 428
480 prev_len = insn->len; 429 prev_len = insn->len;
481 430
482 /* 431 /*
483 * By default, "ud2" i 432 * By default, "ud2" is a dead end unless otherwise
484 * annotated, because 433 * annotated, because GCC 7 inserts it for certain
485 * divide-by-zero case 434 * divide-by-zero cases.
486 */ 435 */
487 if (insn->type == INSN 436 if (insn->type == INSN_BUG)
488 insn->dead_end 437 insn->dead_end = true;
489 438
490 hash_add(file->insn_ha 439 hash_add(file->insn_hash, &insn->hash, sec_offset_hash(sec, insn->offset));
491 nr_insns++; 440 nr_insns++;
492 } 441 }
493 442
494 // printf("%s: last chunk used: % 443 // printf("%s: last chunk used: %d\n", sec->name, (int)idx);
495 444
496 sec_for_each_sym(sec, func) { 445 sec_for_each_sym(sec, func) {
497 if (func->type != STT_ 446 if (func->type != STT_NOTYPE && func->type != STT_FUNC)
498 continue; 447 continue;
499 448
500 if (func->offset == se 449 if (func->offset == sec->sh.sh_size) {
501 /* Heuristic: 450 /* Heuristic: likely an "end" symbol */
502 if (func->type 451 if (func->type == STT_NOTYPE)
503 contin 452 continue;
504 WARN("%s(): ST 453 WARN("%s(): STT_FUNC at end of section",
505 func->nam 454 func->name);
506 return -1; 455 return -1;
507 } 456 }
508 457
509 if (func->embedded_ins 458 if (func->embedded_insn || func->alias != func)
510 continue; 459 continue;
511 460
512 if (!find_insn(file, s 461 if (!find_insn(file, sec, func->offset)) {
513 WARN("%s(): ca 462 WARN("%s(): can't find starting instruction",
514 func->nam 463 func->name);
515 return -1; 464 return -1;
516 } 465 }
517 466
518 sym_for_each_insn(file 467 sym_for_each_insn(file, func, insn) {
519 insn->sym = fu 468 insn->sym = func;
520 if (func->type 469 if (func->type == STT_FUNC &&
521 insn->type 470 insn->type == INSN_ENDBR &&
522 list_empty 471 list_empty(&insn->call_node)) {
523 if (in 472 if (insn->offset == func->offset) {
524 473 list_add_tail(&insn->call_node, &file->endbr_list);
525 474 file->nr_endbr++;
526 } else 475 } else {
527 476 file->nr_endbr_int++;
528 } 477 }
529 } 478 }
530 } 479 }
531 } 480 }
532 } 481 }
533 482
534 if (opts.stats) 483 if (opts.stats)
535 printf("nr_insns: %lu\n", nr_i 484 printf("nr_insns: %lu\n", nr_insns);
536 485
537 return 0; 486 return 0;
538 } 487 }
539 488
540 /* 489 /*
541 * Read the pv_ops[] .data table to find the s 490 * Read the pv_ops[] .data table to find the static initialized values.
542 */ 491 */
543 static int add_pv_ops(struct objtool_file *fil 492 static int add_pv_ops(struct objtool_file *file, const char *symname)
544 { 493 {
545 struct symbol *sym, *func; 494 struct symbol *sym, *func;
546 unsigned long off, end; 495 unsigned long off, end;
547 struct reloc *reloc; 496 struct reloc *reloc;
548 int idx; 497 int idx;
549 498
550 sym = find_symbol_by_name(file->elf, s 499 sym = find_symbol_by_name(file->elf, symname);
551 if (!sym) 500 if (!sym)
552 return 0; 501 return 0;
553 502
554 off = sym->offset; 503 off = sym->offset;
555 end = off + sym->len; 504 end = off + sym->len;
556 for (;;) { 505 for (;;) {
557 reloc = find_reloc_by_dest_ran 506 reloc = find_reloc_by_dest_range(file->elf, sym->sec, off, end - off);
558 if (!reloc) 507 if (!reloc)
559 break; 508 break;
560 509
561 func = reloc->sym; 510 func = reloc->sym;
562 if (func->type == STT_SECTION) 511 if (func->type == STT_SECTION)
563 func = find_symbol_by_ 512 func = find_symbol_by_offset(reloc->sym->sec,
564 513 reloc_addend(reloc));
565 514
566 idx = (reloc_offset(reloc) - s 515 idx = (reloc_offset(reloc) - sym->offset) / sizeof(unsigned long);
567 516
568 objtool_pv_add(file, idx, func 517 objtool_pv_add(file, idx, func);
569 518
570 off = reloc_offset(reloc) + 1; 519 off = reloc_offset(reloc) + 1;
571 if (off > end) 520 if (off > end)
572 break; 521 break;
573 } 522 }
574 523
575 return 0; 524 return 0;
576 } 525 }
577 526
578 /* 527 /*
579 * Allocate and initialize file->pv_ops[]. 528 * Allocate and initialize file->pv_ops[].
580 */ 529 */
581 static int init_pv_ops(struct objtool_file *fi 530 static int init_pv_ops(struct objtool_file *file)
582 { 531 {
583 static const char *pv_ops_tables[] = { 532 static const char *pv_ops_tables[] = {
584 "pv_ops", 533 "pv_ops",
585 "xen_cpu_ops", 534 "xen_cpu_ops",
586 "xen_irq_ops", 535 "xen_irq_ops",
587 "xen_mmu_ops", 536 "xen_mmu_ops",
588 NULL, 537 NULL,
589 }; 538 };
590 const char *pv_ops; 539 const char *pv_ops;
591 struct symbol *sym; 540 struct symbol *sym;
592 int idx, nr; 541 int idx, nr;
593 542
594 if (!opts.noinstr) 543 if (!opts.noinstr)
595 return 0; 544 return 0;
596 545
597 file->pv_ops = NULL; 546 file->pv_ops = NULL;
598 547
599 sym = find_symbol_by_name(file->elf, " 548 sym = find_symbol_by_name(file->elf, "pv_ops");
600 if (!sym) 549 if (!sym)
601 return 0; 550 return 0;
602 551
603 nr = sym->len / sizeof(unsigned long); 552 nr = sym->len / sizeof(unsigned long);
604 file->pv_ops = calloc(sizeof(struct pv 553 file->pv_ops = calloc(sizeof(struct pv_state), nr);
605 if (!file->pv_ops) 554 if (!file->pv_ops)
606 return -1; 555 return -1;
607 556
608 for (idx = 0; idx < nr; idx++) 557 for (idx = 0; idx < nr; idx++)
609 INIT_LIST_HEAD(&file->pv_ops[i 558 INIT_LIST_HEAD(&file->pv_ops[idx].targets);
610 559
611 for (idx = 0; (pv_ops = pv_ops_tables[ 560 for (idx = 0; (pv_ops = pv_ops_tables[idx]); idx++)
612 add_pv_ops(file, pv_ops); 561 add_pv_ops(file, pv_ops);
613 562
614 return 0; 563 return 0;
615 } 564 }
616 565
617 static struct instruction *find_last_insn(stru 566 static struct instruction *find_last_insn(struct objtool_file *file,
618 stru 567 struct section *sec)
619 { 568 {
620 struct instruction *insn = NULL; 569 struct instruction *insn = NULL;
621 unsigned int offset; 570 unsigned int offset;
622 unsigned int end = (sec->sh.sh_size > 571 unsigned int end = (sec->sh.sh_size > 10) ? sec->sh.sh_size - 10 : 0;
623 572
624 for (offset = sec->sh.sh_size - 1; off 573 for (offset = sec->sh.sh_size - 1; offset >= end && !insn; offset--)
625 insn = find_insn(file, sec, of 574 insn = find_insn(file, sec, offset);
626 575
627 return insn; 576 return insn;
628 } 577 }
629 578
630 /* 579 /*
631 * Mark "ud2" instructions and manually annota 580 * Mark "ud2" instructions and manually annotated dead ends.
632 */ 581 */
633 static int add_dead_ends(struct objtool_file * 582 static int add_dead_ends(struct objtool_file *file)
634 { 583 {
635 struct section *rsec; 584 struct section *rsec;
636 struct reloc *reloc; 585 struct reloc *reloc;
637 struct instruction *insn; 586 struct instruction *insn;
638 uint64_t offset; !! 587 s64 addend;
639 588
640 /* 589 /*
641 * Check for manually annotated dead e 590 * Check for manually annotated dead ends.
642 */ 591 */
643 rsec = find_section_by_name(file->elf, 592 rsec = find_section_by_name(file->elf, ".rela.discard.unreachable");
644 if (!rsec) 593 if (!rsec)
645 goto reachable; 594 goto reachable;
646 595
647 for_each_reloc(rsec, reloc) { 596 for_each_reloc(rsec, reloc) {
648 if (reloc->sym->type == STT_SE !! 597
649 offset = reloc_addend( !! 598 if (reloc->sym->type != STT_SECTION) {
650 } else if (reloc->sym->local_l <<
651 offset = reloc->sym->o <<
652 } else { <<
653 WARN("unexpected reloc 599 WARN("unexpected relocation symbol type in %s", rsec->name);
654 return -1; 600 return -1;
655 } 601 }
656 602
657 insn = find_insn(file, reloc-> !! 603 addend = reloc_addend(reloc);
>> 604
>> 605 insn = find_insn(file, reloc->sym->sec, addend);
658 if (insn) 606 if (insn)
659 insn = prev_insn_same_ 607 insn = prev_insn_same_sec(file, insn);
660 else if (offset == reloc->sym- !! 608 else if (addend == reloc->sym->sec->sh.sh_size) {
661 insn = find_last_insn( 609 insn = find_last_insn(file, reloc->sym->sec);
662 if (!insn) { 610 if (!insn) {
663 WARN("can't fi 611 WARN("can't find unreachable insn at %s+0x%" PRIx64,
664 reloc->sy !! 612 reloc->sym->sec->name, addend);
665 return -1; 613 return -1;
666 } 614 }
667 } else { 615 } else {
668 WARN("can't find unrea 616 WARN("can't find unreachable insn at %s+0x%" PRIx64,
669 reloc->sym->sec-> !! 617 reloc->sym->sec->name, addend);
670 return -1; 618 return -1;
671 } 619 }
672 620
673 insn->dead_end = true; 621 insn->dead_end = true;
674 } 622 }
675 623
676 reachable: 624 reachable:
677 /* 625 /*
678 * These manually annotated reachable 626 * These manually annotated reachable checks are needed for GCC 4.4,
679 * where the Linux unreachable() macro 627 * where the Linux unreachable() macro isn't supported. In that case
680 * GCC doesn't know the "ud2" is fatal 628 * GCC doesn't know the "ud2" is fatal, so it generates code as if it's
681 * not a dead end. 629 * not a dead end.
682 */ 630 */
683 rsec = find_section_by_name(file->elf, 631 rsec = find_section_by_name(file->elf, ".rela.discard.reachable");
684 if (!rsec) 632 if (!rsec)
685 return 0; 633 return 0;
686 634
687 for_each_reloc(rsec, reloc) { 635 for_each_reloc(rsec, reloc) {
688 if (reloc->sym->type == STT_SE !! 636
689 offset = reloc_addend( !! 637 if (reloc->sym->type != STT_SECTION) {
690 } else if (reloc->sym->local_l <<
691 offset = reloc->sym->o <<
692 } else { <<
693 WARN("unexpected reloc 638 WARN("unexpected relocation symbol type in %s", rsec->name);
694 return -1; 639 return -1;
695 } 640 }
696 641
697 insn = find_insn(file, reloc-> !! 642 addend = reloc_addend(reloc);
>> 643
>> 644 insn = find_insn(file, reloc->sym->sec, addend);
698 if (insn) 645 if (insn)
699 insn = prev_insn_same_ 646 insn = prev_insn_same_sec(file, insn);
700 else if (offset == reloc->sym- !! 647 else if (addend == reloc->sym->sec->sh.sh_size) {
701 insn = find_last_insn( 648 insn = find_last_insn(file, reloc->sym->sec);
702 if (!insn) { 649 if (!insn) {
703 WARN("can't fi 650 WARN("can't find reachable insn at %s+0x%" PRIx64,
704 reloc->sy !! 651 reloc->sym->sec->name, addend);
705 return -1; 652 return -1;
706 } 653 }
707 } else { 654 } else {
708 WARN("can't find reach 655 WARN("can't find reachable insn at %s+0x%" PRIx64,
709 reloc->sym->sec-> !! 656 reloc->sym->sec->name, addend);
710 return -1; 657 return -1;
711 } 658 }
712 659
713 insn->dead_end = false; 660 insn->dead_end = false;
714 } 661 }
715 662
716 return 0; 663 return 0;
717 } 664 }
718 665
719 static int create_static_call_sections(struct 666 static int create_static_call_sections(struct objtool_file *file)
720 { 667 {
721 struct static_call_site *site; 668 struct static_call_site *site;
722 struct section *sec; 669 struct section *sec;
723 struct instruction *insn; 670 struct instruction *insn;
724 struct symbol *key_sym; 671 struct symbol *key_sym;
725 char *key_name, *tmp; 672 char *key_name, *tmp;
726 int idx; 673 int idx;
727 674
728 sec = find_section_by_name(file->elf, 675 sec = find_section_by_name(file->elf, ".static_call_sites");
729 if (sec) { 676 if (sec) {
730 INIT_LIST_HEAD(&file->static_c 677 INIT_LIST_HEAD(&file->static_call_list);
731 WARN("file already has .static 678 WARN("file already has .static_call_sites section, skipping");
732 return 0; 679 return 0;
733 } 680 }
734 681
735 if (list_empty(&file->static_call_list 682 if (list_empty(&file->static_call_list))
736 return 0; 683 return 0;
737 684
738 idx = 0; 685 idx = 0;
739 list_for_each_entry(insn, &file->stati 686 list_for_each_entry(insn, &file->static_call_list, call_node)
740 idx++; 687 idx++;
741 688
742 sec = elf_create_section_pair(file->el 689 sec = elf_create_section_pair(file->elf, ".static_call_sites",
743 sizeof(* 690 sizeof(*site), idx, idx * 2);
744 if (!sec) 691 if (!sec)
745 return -1; 692 return -1;
746 693
747 /* Allow modules to modify the low bit 694 /* Allow modules to modify the low bits of static_call_site::key */
748 sec->sh.sh_flags |= SHF_WRITE; 695 sec->sh.sh_flags |= SHF_WRITE;
749 696
750 idx = 0; 697 idx = 0;
751 list_for_each_entry(insn, &file->stati 698 list_for_each_entry(insn, &file->static_call_list, call_node) {
752 699
753 /* populate reloc for 'addr' * 700 /* populate reloc for 'addr' */
754 if (!elf_init_reloc_text_sym(f 701 if (!elf_init_reloc_text_sym(file->elf, sec,
755 i 702 idx * sizeof(*site), idx * 2,
756 i 703 insn->sec, insn->offset))
757 return -1; 704 return -1;
758 705
759 /* find key symbol */ 706 /* find key symbol */
760 key_name = strdup(insn_call_de 707 key_name = strdup(insn_call_dest(insn)->name);
761 if (!key_name) { 708 if (!key_name) {
762 perror("strdup"); 709 perror("strdup");
763 return -1; 710 return -1;
764 } 711 }
765 if (strncmp(key_name, STATIC_C 712 if (strncmp(key_name, STATIC_CALL_TRAMP_PREFIX_STR,
766 STATIC_CALL_TRAMP_ 713 STATIC_CALL_TRAMP_PREFIX_LEN)) {
767 WARN("static_call: tra 714 WARN("static_call: trampoline name malformed: %s", key_name);
768 free(key_name); 715 free(key_name);
769 return -1; 716 return -1;
770 } 717 }
771 tmp = key_name + STATIC_CALL_T 718 tmp = key_name + STATIC_CALL_TRAMP_PREFIX_LEN - STATIC_CALL_KEY_PREFIX_LEN;
772 memcpy(tmp, STATIC_CALL_KEY_PR 719 memcpy(tmp, STATIC_CALL_KEY_PREFIX_STR, STATIC_CALL_KEY_PREFIX_LEN);
773 720
774 key_sym = find_symbol_by_name( 721 key_sym = find_symbol_by_name(file->elf, tmp);
775 if (!key_sym) { 722 if (!key_sym) {
776 if (!opts.module) { 723 if (!opts.module) {
777 WARN("static_c 724 WARN("static_call: can't find static_call_key symbol: %s", tmp);
778 free(key_name) 725 free(key_name);
779 return -1; 726 return -1;
780 } 727 }
781 728
782 /* 729 /*
783 * For modules(), the 730 * For modules(), the key might not be exported, which
784 * means the module ca 731 * means the module can make static calls but isn't
785 * allowed to change t 732 * allowed to change them.
786 * 733 *
787 * In that case we tem 734 * In that case we temporarily set the key to be the
788 * trampoline address. 735 * trampoline address. This is fixed up in
789 * static_call_add_mod 736 * static_call_add_module().
790 */ 737 */
791 key_sym = insn_call_de 738 key_sym = insn_call_dest(insn);
792 } 739 }
793 free(key_name); 740 free(key_name);
794 741
795 /* populate reloc for 'key' */ 742 /* populate reloc for 'key' */
796 if (!elf_init_reloc_data_sym(f 743 if (!elf_init_reloc_data_sym(file->elf, sec,
797 i 744 idx * sizeof(*site) + 4,
798 ( 745 (idx * 2) + 1, key_sym,
799 i 746 is_sibling_call(insn) * STATIC_CALL_SITE_TAIL))
800 return -1; 747 return -1;
801 748
802 idx++; 749 idx++;
803 } 750 }
804 751
805 return 0; 752 return 0;
806 } 753 }
807 754
808 static int create_retpoline_sites_sections(str 755 static int create_retpoline_sites_sections(struct objtool_file *file)
809 { 756 {
810 struct instruction *insn; 757 struct instruction *insn;
811 struct section *sec; 758 struct section *sec;
812 int idx; 759 int idx;
813 760
814 sec = find_section_by_name(file->elf, 761 sec = find_section_by_name(file->elf, ".retpoline_sites");
815 if (sec) { 762 if (sec) {
816 WARN("file already has .retpol 763 WARN("file already has .retpoline_sites, skipping");
817 return 0; 764 return 0;
818 } 765 }
819 766
820 idx = 0; 767 idx = 0;
821 list_for_each_entry(insn, &file->retpo 768 list_for_each_entry(insn, &file->retpoline_call_list, call_node)
822 idx++; 769 idx++;
823 770
824 if (!idx) 771 if (!idx)
825 return 0; 772 return 0;
826 773
827 sec = elf_create_section_pair(file->el 774 sec = elf_create_section_pair(file->elf, ".retpoline_sites",
828 sizeof(i 775 sizeof(int), idx, idx);
829 if (!sec) 776 if (!sec)
830 return -1; 777 return -1;
831 778
832 idx = 0; 779 idx = 0;
833 list_for_each_entry(insn, &file->retpo 780 list_for_each_entry(insn, &file->retpoline_call_list, call_node) {
834 781
835 if (!elf_init_reloc_text_sym(f 782 if (!elf_init_reloc_text_sym(file->elf, sec,
836 i 783 idx * sizeof(int), idx,
837 i 784 insn->sec, insn->offset))
838 return -1; 785 return -1;
839 786
840 idx++; 787 idx++;
841 } 788 }
842 789
843 return 0; 790 return 0;
844 } 791 }
845 792
846 static int create_return_sites_sections(struct 793 static int create_return_sites_sections(struct objtool_file *file)
847 { 794 {
848 struct instruction *insn; 795 struct instruction *insn;
849 struct section *sec; 796 struct section *sec;
850 int idx; 797 int idx;
851 798
852 sec = find_section_by_name(file->elf, 799 sec = find_section_by_name(file->elf, ".return_sites");
853 if (sec) { 800 if (sec) {
854 WARN("file already has .return 801 WARN("file already has .return_sites, skipping");
855 return 0; 802 return 0;
856 } 803 }
857 804
858 idx = 0; 805 idx = 0;
859 list_for_each_entry(insn, &file->retur 806 list_for_each_entry(insn, &file->return_thunk_list, call_node)
860 idx++; 807 idx++;
861 808
862 if (!idx) 809 if (!idx)
863 return 0; 810 return 0;
864 811
865 sec = elf_create_section_pair(file->el 812 sec = elf_create_section_pair(file->elf, ".return_sites",
866 sizeof(i 813 sizeof(int), idx, idx);
867 if (!sec) 814 if (!sec)
868 return -1; 815 return -1;
869 816
870 idx = 0; 817 idx = 0;
871 list_for_each_entry(insn, &file->retur 818 list_for_each_entry(insn, &file->return_thunk_list, call_node) {
872 819
873 if (!elf_init_reloc_text_sym(f 820 if (!elf_init_reloc_text_sym(file->elf, sec,
874 i 821 idx * sizeof(int), idx,
875 i 822 insn->sec, insn->offset))
876 return -1; 823 return -1;
877 824
878 idx++; 825 idx++;
879 } 826 }
880 827
881 return 0; 828 return 0;
882 } 829 }
883 830
884 static int create_ibt_endbr_seal_sections(stru 831 static int create_ibt_endbr_seal_sections(struct objtool_file *file)
885 { 832 {
886 struct instruction *insn; 833 struct instruction *insn;
887 struct section *sec; 834 struct section *sec;
888 int idx; 835 int idx;
889 836
890 sec = find_section_by_name(file->elf, 837 sec = find_section_by_name(file->elf, ".ibt_endbr_seal");
891 if (sec) { 838 if (sec) {
892 WARN("file already has .ibt_en 839 WARN("file already has .ibt_endbr_seal, skipping");
893 return 0; 840 return 0;
894 } 841 }
895 842
896 idx = 0; 843 idx = 0;
897 list_for_each_entry(insn, &file->endbr 844 list_for_each_entry(insn, &file->endbr_list, call_node)
898 idx++; 845 idx++;
899 846
900 if (opts.stats) { 847 if (opts.stats) {
901 printf("ibt: ENDBR at function 848 printf("ibt: ENDBR at function start: %d\n", file->nr_endbr);
902 printf("ibt: ENDBR inside func 849 printf("ibt: ENDBR inside functions: %d\n", file->nr_endbr_int);
903 printf("ibt: superfluous ENDBR 850 printf("ibt: superfluous ENDBR: %d\n", idx);
904 } 851 }
905 852
906 if (!idx) 853 if (!idx)
907 return 0; 854 return 0;
908 855
909 sec = elf_create_section_pair(file->el 856 sec = elf_create_section_pair(file->elf, ".ibt_endbr_seal",
910 sizeof(i 857 sizeof(int), idx, idx);
911 if (!sec) 858 if (!sec)
912 return -1; 859 return -1;
913 860
914 idx = 0; 861 idx = 0;
915 list_for_each_entry(insn, &file->endbr 862 list_for_each_entry(insn, &file->endbr_list, call_node) {
916 863
917 int *site = (int *)sec->data-> 864 int *site = (int *)sec->data->d_buf + idx;
918 struct symbol *sym = insn->sym 865 struct symbol *sym = insn->sym;
919 *site = 0; 866 *site = 0;
920 867
921 if (opts.module && sym && sym- 868 if (opts.module && sym && sym->type == STT_FUNC &&
922 insn->offset == sym->offse 869 insn->offset == sym->offset &&
923 (!strcmp(sym->name, "init_ 870 (!strcmp(sym->name, "init_module") ||
924 !strcmp(sym->name, "clean 871 !strcmp(sym->name, "cleanup_module")))
925 WARN("%s(): not an ind 872 WARN("%s(): not an indirect call target", sym->name);
926 873
927 if (!elf_init_reloc_text_sym(f 874 if (!elf_init_reloc_text_sym(file->elf, sec,
928 i 875 idx * sizeof(int), idx,
929 i 876 insn->sec, insn->offset))
930 return -1; 877 return -1;
931 878
932 idx++; 879 idx++;
933 } 880 }
934 881
935 return 0; 882 return 0;
936 } 883 }
937 884
938 static int create_cfi_sections(struct objtool_ 885 static int create_cfi_sections(struct objtool_file *file)
939 { 886 {
940 struct section *sec; 887 struct section *sec;
941 struct symbol *sym; 888 struct symbol *sym;
942 int idx; 889 int idx;
943 890
944 sec = find_section_by_name(file->elf, 891 sec = find_section_by_name(file->elf, ".cfi_sites");
945 if (sec) { 892 if (sec) {
946 INIT_LIST_HEAD(&file->call_lis 893 INIT_LIST_HEAD(&file->call_list);
947 WARN("file already has .cfi_si 894 WARN("file already has .cfi_sites section, skipping");
948 return 0; 895 return 0;
949 } 896 }
950 897
951 idx = 0; 898 idx = 0;
952 for_each_sym(file, sym) { 899 for_each_sym(file, sym) {
953 if (sym->type != STT_FUNC) 900 if (sym->type != STT_FUNC)
954 continue; 901 continue;
955 902
956 if (strncmp(sym->name, "__cfi_ 903 if (strncmp(sym->name, "__cfi_", 6))
957 continue; 904 continue;
958 905
959 idx++; 906 idx++;
960 } 907 }
961 908
962 sec = elf_create_section_pair(file->el 909 sec = elf_create_section_pair(file->elf, ".cfi_sites",
963 sizeof(u 910 sizeof(unsigned int), idx, idx);
964 if (!sec) 911 if (!sec)
965 return -1; 912 return -1;
966 913
967 idx = 0; 914 idx = 0;
968 for_each_sym(file, sym) { 915 for_each_sym(file, sym) {
969 if (sym->type != STT_FUNC) 916 if (sym->type != STT_FUNC)
970 continue; 917 continue;
971 918
972 if (strncmp(sym->name, "__cfi_ 919 if (strncmp(sym->name, "__cfi_", 6))
973 continue; 920 continue;
974 921
975 if (!elf_init_reloc_text_sym(f 922 if (!elf_init_reloc_text_sym(file->elf, sec,
976 i 923 idx * sizeof(unsigned int), idx,
977 s 924 sym->sec, sym->offset))
978 return -1; 925 return -1;
979 926
980 idx++; 927 idx++;
981 } 928 }
982 929
983 return 0; 930 return 0;
984 } 931 }
985 932
986 static int create_mcount_loc_sections(struct o 933 static int create_mcount_loc_sections(struct objtool_file *file)
987 { 934 {
988 size_t addr_size = elf_addr_size(file- 935 size_t addr_size = elf_addr_size(file->elf);
989 struct instruction *insn; 936 struct instruction *insn;
990 struct section *sec; 937 struct section *sec;
991 int idx; 938 int idx;
992 939
993 sec = find_section_by_name(file->elf, 940 sec = find_section_by_name(file->elf, "__mcount_loc");
994 if (sec) { 941 if (sec) {
995 INIT_LIST_HEAD(&file->mcount_l 942 INIT_LIST_HEAD(&file->mcount_loc_list);
996 WARN("file already has __mcoun 943 WARN("file already has __mcount_loc section, skipping");
997 return 0; 944 return 0;
998 } 945 }
999 946
1000 if (list_empty(&file->mcount_loc_list 947 if (list_empty(&file->mcount_loc_list))
1001 return 0; 948 return 0;
1002 949
1003 idx = 0; 950 idx = 0;
1004 list_for_each_entry(insn, &file->mcou 951 list_for_each_entry(insn, &file->mcount_loc_list, call_node)
1005 idx++; 952 idx++;
1006 953
1007 sec = elf_create_section_pair(file->e 954 sec = elf_create_section_pair(file->elf, "__mcount_loc", addr_size,
1008 idx, id 955 idx, idx);
1009 if (!sec) 956 if (!sec)
1010 return -1; 957 return -1;
1011 958
1012 sec->sh.sh_addralign = addr_size; 959 sec->sh.sh_addralign = addr_size;
1013 960
1014 idx = 0; 961 idx = 0;
1015 list_for_each_entry(insn, &file->mcou 962 list_for_each_entry(insn, &file->mcount_loc_list, call_node) {
1016 963
1017 struct reloc *reloc; 964 struct reloc *reloc;
1018 965
1019 reloc = elf_init_reloc_text_s 966 reloc = elf_init_reloc_text_sym(file->elf, sec, idx * addr_size, idx,
1020 967 insn->sec, insn->offset);
1021 if (!reloc) 968 if (!reloc)
1022 return -1; 969 return -1;
1023 970
1024 set_reloc_type(file->elf, rel 971 set_reloc_type(file->elf, reloc, addr_size == 8 ? R_ABS64 : R_ABS32);
1025 972
1026 idx++; 973 idx++;
1027 } 974 }
1028 975
1029 return 0; 976 return 0;
1030 } 977 }
1031 978
1032 static int create_direct_call_sections(struct 979 static int create_direct_call_sections(struct objtool_file *file)
1033 { 980 {
1034 struct instruction *insn; 981 struct instruction *insn;
1035 struct section *sec; 982 struct section *sec;
1036 int idx; 983 int idx;
1037 984
1038 sec = find_section_by_name(file->elf, 985 sec = find_section_by_name(file->elf, ".call_sites");
1039 if (sec) { 986 if (sec) {
1040 INIT_LIST_HEAD(&file->call_li 987 INIT_LIST_HEAD(&file->call_list);
1041 WARN("file already has .call_ 988 WARN("file already has .call_sites section, skipping");
1042 return 0; 989 return 0;
1043 } 990 }
1044 991
1045 if (list_empty(&file->call_list)) 992 if (list_empty(&file->call_list))
1046 return 0; 993 return 0;
1047 994
1048 idx = 0; 995 idx = 0;
1049 list_for_each_entry(insn, &file->call 996 list_for_each_entry(insn, &file->call_list, call_node)
1050 idx++; 997 idx++;
1051 998
1052 sec = elf_create_section_pair(file->e 999 sec = elf_create_section_pair(file->elf, ".call_sites",
1053 sizeof( 1000 sizeof(unsigned int), idx, idx);
1054 if (!sec) 1001 if (!sec)
1055 return -1; 1002 return -1;
1056 1003
1057 idx = 0; 1004 idx = 0;
1058 list_for_each_entry(insn, &file->call 1005 list_for_each_entry(insn, &file->call_list, call_node) {
1059 1006
1060 if (!elf_init_reloc_text_sym( 1007 if (!elf_init_reloc_text_sym(file->elf, sec,
1061 1008 idx * sizeof(unsigned int), idx,
1062 1009 insn->sec, insn->offset))
1063 return -1; 1010 return -1;
1064 1011
1065 idx++; 1012 idx++;
1066 } 1013 }
1067 1014
1068 return 0; 1015 return 0;
1069 } 1016 }
1070 1017
1071 /* 1018 /*
1072 * Warnings shouldn't be reported for ignored 1019 * Warnings shouldn't be reported for ignored functions.
1073 */ 1020 */
1074 static void add_ignores(struct objtool_file * 1021 static void add_ignores(struct objtool_file *file)
1075 { 1022 {
1076 struct instruction *insn; 1023 struct instruction *insn;
1077 struct section *rsec; 1024 struct section *rsec;
1078 struct symbol *func; 1025 struct symbol *func;
1079 struct reloc *reloc; 1026 struct reloc *reloc;
1080 1027
1081 rsec = find_section_by_name(file->elf 1028 rsec = find_section_by_name(file->elf, ".rela.discard.func_stack_frame_non_standard");
1082 if (!rsec) 1029 if (!rsec)
1083 return; 1030 return;
1084 1031
1085 for_each_reloc(rsec, reloc) { 1032 for_each_reloc(rsec, reloc) {
1086 switch (reloc->sym->type) { 1033 switch (reloc->sym->type) {
1087 case STT_FUNC: 1034 case STT_FUNC:
1088 func = reloc->sym; 1035 func = reloc->sym;
1089 break; 1036 break;
1090 1037
1091 case STT_SECTION: 1038 case STT_SECTION:
1092 func = find_func_by_o 1039 func = find_func_by_offset(reloc->sym->sec, reloc_addend(reloc));
1093 if (!func) 1040 if (!func)
1094 continue; 1041 continue;
1095 break; 1042 break;
1096 1043
1097 default: 1044 default:
1098 WARN("unexpected relo 1045 WARN("unexpected relocation symbol type in %s: %d",
1099 rsec->name, relo 1046 rsec->name, reloc->sym->type);
1100 continue; 1047 continue;
1101 } 1048 }
1102 1049
1103 func_for_each_insn(file, func 1050 func_for_each_insn(file, func, insn)
1104 insn->ignore = true; 1051 insn->ignore = true;
1105 } 1052 }
1106 } 1053 }
1107 1054
1108 /* 1055 /*
1109 * This is a whitelist of functions that is a 1056 * This is a whitelist of functions that is allowed to be called with AC set.
1110 * The list is meant to be minimal and only c 1057 * The list is meant to be minimal and only contains compiler instrumentation
1111 * ABI and a few functions used to implement 1058 * ABI and a few functions used to implement *_{to,from}_user() functions.
1112 * 1059 *
1113 * These functions must not directly change A 1060 * These functions must not directly change AC, but may PUSHF/POPF.
1114 */ 1061 */
1115 static const char *uaccess_safe_builtin[] = { 1062 static const char *uaccess_safe_builtin[] = {
1116 /* KASAN */ 1063 /* KASAN */
1117 "kasan_report", 1064 "kasan_report",
1118 "kasan_check_range", 1065 "kasan_check_range",
1119 /* KASAN out-of-line */ 1066 /* KASAN out-of-line */
1120 "__asan_loadN_noabort", 1067 "__asan_loadN_noabort",
1121 "__asan_load1_noabort", 1068 "__asan_load1_noabort",
1122 "__asan_load2_noabort", 1069 "__asan_load2_noabort",
1123 "__asan_load4_noabort", 1070 "__asan_load4_noabort",
1124 "__asan_load8_noabort", 1071 "__asan_load8_noabort",
1125 "__asan_load16_noabort", 1072 "__asan_load16_noabort",
1126 "__asan_storeN_noabort", 1073 "__asan_storeN_noabort",
1127 "__asan_store1_noabort", 1074 "__asan_store1_noabort",
1128 "__asan_store2_noabort", 1075 "__asan_store2_noabort",
1129 "__asan_store4_noabort", 1076 "__asan_store4_noabort",
1130 "__asan_store8_noabort", 1077 "__asan_store8_noabort",
1131 "__asan_store16_noabort", 1078 "__asan_store16_noabort",
1132 "__kasan_check_read", 1079 "__kasan_check_read",
1133 "__kasan_check_write", 1080 "__kasan_check_write",
1134 /* KASAN in-line */ 1081 /* KASAN in-line */
1135 "__asan_report_load_n_noabort", 1082 "__asan_report_load_n_noabort",
1136 "__asan_report_load1_noabort", 1083 "__asan_report_load1_noabort",
1137 "__asan_report_load2_noabort", 1084 "__asan_report_load2_noabort",
1138 "__asan_report_load4_noabort", 1085 "__asan_report_load4_noabort",
1139 "__asan_report_load8_noabort", 1086 "__asan_report_load8_noabort",
1140 "__asan_report_load16_noabort", 1087 "__asan_report_load16_noabort",
1141 "__asan_report_store_n_noabort", 1088 "__asan_report_store_n_noabort",
1142 "__asan_report_store1_noabort", 1089 "__asan_report_store1_noabort",
1143 "__asan_report_store2_noabort", 1090 "__asan_report_store2_noabort",
1144 "__asan_report_store4_noabort", 1091 "__asan_report_store4_noabort",
1145 "__asan_report_store8_noabort", 1092 "__asan_report_store8_noabort",
1146 "__asan_report_store16_noabort", 1093 "__asan_report_store16_noabort",
1147 /* KCSAN */ 1094 /* KCSAN */
1148 "__kcsan_check_access", 1095 "__kcsan_check_access",
1149 "__kcsan_mb", 1096 "__kcsan_mb",
1150 "__kcsan_wmb", 1097 "__kcsan_wmb",
1151 "__kcsan_rmb", 1098 "__kcsan_rmb",
1152 "__kcsan_release", 1099 "__kcsan_release",
1153 "kcsan_found_watchpoint", 1100 "kcsan_found_watchpoint",
1154 "kcsan_setup_watchpoint", 1101 "kcsan_setup_watchpoint",
1155 "kcsan_check_scoped_accesses", 1102 "kcsan_check_scoped_accesses",
1156 "kcsan_disable_current", 1103 "kcsan_disable_current",
1157 "kcsan_enable_current_nowarn", 1104 "kcsan_enable_current_nowarn",
1158 /* KCSAN/TSAN */ 1105 /* KCSAN/TSAN */
1159 "__tsan_func_entry", 1106 "__tsan_func_entry",
1160 "__tsan_func_exit", 1107 "__tsan_func_exit",
1161 "__tsan_read_range", 1108 "__tsan_read_range",
1162 "__tsan_write_range", 1109 "__tsan_write_range",
1163 "__tsan_read1", 1110 "__tsan_read1",
1164 "__tsan_read2", 1111 "__tsan_read2",
1165 "__tsan_read4", 1112 "__tsan_read4",
1166 "__tsan_read8", 1113 "__tsan_read8",
1167 "__tsan_read16", 1114 "__tsan_read16",
1168 "__tsan_write1", 1115 "__tsan_write1",
1169 "__tsan_write2", 1116 "__tsan_write2",
1170 "__tsan_write4", 1117 "__tsan_write4",
1171 "__tsan_write8", 1118 "__tsan_write8",
1172 "__tsan_write16", 1119 "__tsan_write16",
1173 "__tsan_read_write1", 1120 "__tsan_read_write1",
1174 "__tsan_read_write2", 1121 "__tsan_read_write2",
1175 "__tsan_read_write4", 1122 "__tsan_read_write4",
1176 "__tsan_read_write8", 1123 "__tsan_read_write8",
1177 "__tsan_read_write16", 1124 "__tsan_read_write16",
1178 "__tsan_volatile_read1", 1125 "__tsan_volatile_read1",
1179 "__tsan_volatile_read2", 1126 "__tsan_volatile_read2",
1180 "__tsan_volatile_read4", 1127 "__tsan_volatile_read4",
1181 "__tsan_volatile_read8", 1128 "__tsan_volatile_read8",
1182 "__tsan_volatile_read16", 1129 "__tsan_volatile_read16",
1183 "__tsan_volatile_write1", 1130 "__tsan_volatile_write1",
1184 "__tsan_volatile_write2", 1131 "__tsan_volatile_write2",
1185 "__tsan_volatile_write4", 1132 "__tsan_volatile_write4",
1186 "__tsan_volatile_write8", 1133 "__tsan_volatile_write8",
1187 "__tsan_volatile_write16", 1134 "__tsan_volatile_write16",
1188 "__tsan_atomic8_load", 1135 "__tsan_atomic8_load",
1189 "__tsan_atomic16_load", 1136 "__tsan_atomic16_load",
1190 "__tsan_atomic32_load", 1137 "__tsan_atomic32_load",
1191 "__tsan_atomic64_load", 1138 "__tsan_atomic64_load",
1192 "__tsan_atomic8_store", 1139 "__tsan_atomic8_store",
1193 "__tsan_atomic16_store", 1140 "__tsan_atomic16_store",
1194 "__tsan_atomic32_store", 1141 "__tsan_atomic32_store",
1195 "__tsan_atomic64_store", 1142 "__tsan_atomic64_store",
1196 "__tsan_atomic8_exchange", 1143 "__tsan_atomic8_exchange",
1197 "__tsan_atomic16_exchange", 1144 "__tsan_atomic16_exchange",
1198 "__tsan_atomic32_exchange", 1145 "__tsan_atomic32_exchange",
1199 "__tsan_atomic64_exchange", 1146 "__tsan_atomic64_exchange",
1200 "__tsan_atomic8_fetch_add", 1147 "__tsan_atomic8_fetch_add",
1201 "__tsan_atomic16_fetch_add", 1148 "__tsan_atomic16_fetch_add",
1202 "__tsan_atomic32_fetch_add", 1149 "__tsan_atomic32_fetch_add",
1203 "__tsan_atomic64_fetch_add", 1150 "__tsan_atomic64_fetch_add",
1204 "__tsan_atomic8_fetch_sub", 1151 "__tsan_atomic8_fetch_sub",
1205 "__tsan_atomic16_fetch_sub", 1152 "__tsan_atomic16_fetch_sub",
1206 "__tsan_atomic32_fetch_sub", 1153 "__tsan_atomic32_fetch_sub",
1207 "__tsan_atomic64_fetch_sub", 1154 "__tsan_atomic64_fetch_sub",
1208 "__tsan_atomic8_fetch_and", 1155 "__tsan_atomic8_fetch_and",
1209 "__tsan_atomic16_fetch_and", 1156 "__tsan_atomic16_fetch_and",
1210 "__tsan_atomic32_fetch_and", 1157 "__tsan_atomic32_fetch_and",
1211 "__tsan_atomic64_fetch_and", 1158 "__tsan_atomic64_fetch_and",
1212 "__tsan_atomic8_fetch_or", 1159 "__tsan_atomic8_fetch_or",
1213 "__tsan_atomic16_fetch_or", 1160 "__tsan_atomic16_fetch_or",
1214 "__tsan_atomic32_fetch_or", 1161 "__tsan_atomic32_fetch_or",
1215 "__tsan_atomic64_fetch_or", 1162 "__tsan_atomic64_fetch_or",
1216 "__tsan_atomic8_fetch_xor", 1163 "__tsan_atomic8_fetch_xor",
1217 "__tsan_atomic16_fetch_xor", 1164 "__tsan_atomic16_fetch_xor",
1218 "__tsan_atomic32_fetch_xor", 1165 "__tsan_atomic32_fetch_xor",
1219 "__tsan_atomic64_fetch_xor", 1166 "__tsan_atomic64_fetch_xor",
1220 "__tsan_atomic8_fetch_nand", 1167 "__tsan_atomic8_fetch_nand",
1221 "__tsan_atomic16_fetch_nand", 1168 "__tsan_atomic16_fetch_nand",
1222 "__tsan_atomic32_fetch_nand", 1169 "__tsan_atomic32_fetch_nand",
1223 "__tsan_atomic64_fetch_nand", 1170 "__tsan_atomic64_fetch_nand",
1224 "__tsan_atomic8_compare_exchange_stro 1171 "__tsan_atomic8_compare_exchange_strong",
1225 "__tsan_atomic16_compare_exchange_str 1172 "__tsan_atomic16_compare_exchange_strong",
1226 "__tsan_atomic32_compare_exchange_str 1173 "__tsan_atomic32_compare_exchange_strong",
1227 "__tsan_atomic64_compare_exchange_str 1174 "__tsan_atomic64_compare_exchange_strong",
1228 "__tsan_atomic8_compare_exchange_weak 1175 "__tsan_atomic8_compare_exchange_weak",
1229 "__tsan_atomic16_compare_exchange_wea 1176 "__tsan_atomic16_compare_exchange_weak",
1230 "__tsan_atomic32_compare_exchange_wea 1177 "__tsan_atomic32_compare_exchange_weak",
1231 "__tsan_atomic64_compare_exchange_wea 1178 "__tsan_atomic64_compare_exchange_weak",
1232 "__tsan_atomic8_compare_exchange_val" 1179 "__tsan_atomic8_compare_exchange_val",
1233 "__tsan_atomic16_compare_exchange_val 1180 "__tsan_atomic16_compare_exchange_val",
1234 "__tsan_atomic32_compare_exchange_val 1181 "__tsan_atomic32_compare_exchange_val",
1235 "__tsan_atomic64_compare_exchange_val 1182 "__tsan_atomic64_compare_exchange_val",
1236 "__tsan_atomic_thread_fence", 1183 "__tsan_atomic_thread_fence",
1237 "__tsan_atomic_signal_fence", 1184 "__tsan_atomic_signal_fence",
1238 "__tsan_unaligned_read16", 1185 "__tsan_unaligned_read16",
1239 "__tsan_unaligned_write16", 1186 "__tsan_unaligned_write16",
1240 /* KCOV */ 1187 /* KCOV */
1241 "write_comp_data", 1188 "write_comp_data",
1242 "check_kcov_mode", 1189 "check_kcov_mode",
1243 "__sanitizer_cov_trace_pc", 1190 "__sanitizer_cov_trace_pc",
1244 "__sanitizer_cov_trace_const_cmp1", 1191 "__sanitizer_cov_trace_const_cmp1",
1245 "__sanitizer_cov_trace_const_cmp2", 1192 "__sanitizer_cov_trace_const_cmp2",
1246 "__sanitizer_cov_trace_const_cmp4", 1193 "__sanitizer_cov_trace_const_cmp4",
1247 "__sanitizer_cov_trace_const_cmp8", 1194 "__sanitizer_cov_trace_const_cmp8",
1248 "__sanitizer_cov_trace_cmp1", 1195 "__sanitizer_cov_trace_cmp1",
1249 "__sanitizer_cov_trace_cmp2", 1196 "__sanitizer_cov_trace_cmp2",
1250 "__sanitizer_cov_trace_cmp4", 1197 "__sanitizer_cov_trace_cmp4",
1251 "__sanitizer_cov_trace_cmp8", 1198 "__sanitizer_cov_trace_cmp8",
1252 "__sanitizer_cov_trace_switch", 1199 "__sanitizer_cov_trace_switch",
1253 /* KMSAN */ 1200 /* KMSAN */
1254 "kmsan_copy_to_user", 1201 "kmsan_copy_to_user",
1255 "kmsan_disable_current", <<
1256 "kmsan_enable_current", <<
1257 "kmsan_report", 1202 "kmsan_report",
1258 "kmsan_unpoison_entry_regs", 1203 "kmsan_unpoison_entry_regs",
1259 "kmsan_unpoison_memory", 1204 "kmsan_unpoison_memory",
1260 "__msan_chain_origin", 1205 "__msan_chain_origin",
1261 "__msan_get_context_state", 1206 "__msan_get_context_state",
1262 "__msan_instrument_asm_store", 1207 "__msan_instrument_asm_store",
1263 "__msan_metadata_ptr_for_load_1", 1208 "__msan_metadata_ptr_for_load_1",
1264 "__msan_metadata_ptr_for_load_2", 1209 "__msan_metadata_ptr_for_load_2",
1265 "__msan_metadata_ptr_for_load_4", 1210 "__msan_metadata_ptr_for_load_4",
1266 "__msan_metadata_ptr_for_load_8", 1211 "__msan_metadata_ptr_for_load_8",
1267 "__msan_metadata_ptr_for_load_n", 1212 "__msan_metadata_ptr_for_load_n",
1268 "__msan_metadata_ptr_for_store_1", 1213 "__msan_metadata_ptr_for_store_1",
1269 "__msan_metadata_ptr_for_store_2", 1214 "__msan_metadata_ptr_for_store_2",
1270 "__msan_metadata_ptr_for_store_4", 1215 "__msan_metadata_ptr_for_store_4",
1271 "__msan_metadata_ptr_for_store_8", 1216 "__msan_metadata_ptr_for_store_8",
1272 "__msan_metadata_ptr_for_store_n", 1217 "__msan_metadata_ptr_for_store_n",
1273 "__msan_poison_alloca", 1218 "__msan_poison_alloca",
1274 "__msan_warning", 1219 "__msan_warning",
1275 /* UBSAN */ 1220 /* UBSAN */
1276 "ubsan_type_mismatch_common", 1221 "ubsan_type_mismatch_common",
1277 "__ubsan_handle_type_mismatch", 1222 "__ubsan_handle_type_mismatch",
1278 "__ubsan_handle_type_mismatch_v1", 1223 "__ubsan_handle_type_mismatch_v1",
1279 "__ubsan_handle_shift_out_of_bounds", 1224 "__ubsan_handle_shift_out_of_bounds",
1280 "__ubsan_handle_load_invalid_value", 1225 "__ubsan_handle_load_invalid_value",
1281 /* STACKLEAK */ 1226 /* STACKLEAK */
1282 "stackleak_track_stack", 1227 "stackleak_track_stack",
1283 /* misc */ 1228 /* misc */
1284 "csum_partial_copy_generic", 1229 "csum_partial_copy_generic",
1285 "copy_mc_fragile", 1230 "copy_mc_fragile",
1286 "copy_mc_fragile_handle_tail", 1231 "copy_mc_fragile_handle_tail",
1287 "copy_mc_enhanced_fast_string", 1232 "copy_mc_enhanced_fast_string",
1288 "ftrace_likely_update", /* CONFIG_TRA 1233 "ftrace_likely_update", /* CONFIG_TRACE_BRANCH_PROFILING */
1289 "rep_stos_alternative", 1234 "rep_stos_alternative",
1290 "rep_movs_alternative", 1235 "rep_movs_alternative",
1291 "__copy_user_nocache", 1236 "__copy_user_nocache",
1292 NULL 1237 NULL
1293 }; 1238 };
1294 1239
1295 static void add_uaccess_safe(struct objtool_f 1240 static void add_uaccess_safe(struct objtool_file *file)
1296 { 1241 {
1297 struct symbol *func; 1242 struct symbol *func;
1298 const char **name; 1243 const char **name;
1299 1244
1300 if (!opts.uaccess) 1245 if (!opts.uaccess)
1301 return; 1246 return;
1302 1247
1303 for (name = uaccess_safe_builtin; *na 1248 for (name = uaccess_safe_builtin; *name; name++) {
1304 func = find_symbol_by_name(fi 1249 func = find_symbol_by_name(file->elf, *name);
1305 if (!func) 1250 if (!func)
1306 continue; 1251 continue;
1307 1252
1308 func->uaccess_safe = true; 1253 func->uaccess_safe = true;
1309 } 1254 }
1310 } 1255 }
1311 1256
1312 /* 1257 /*
1313 * FIXME: For now, just ignore any alternativ 1258 * FIXME: For now, just ignore any alternatives which add retpolines. This is
1314 * a temporary hack, as it doesn't allow ORC 1259 * a temporary hack, as it doesn't allow ORC to unwind from inside a retpoline.
1315 * But it at least allows objtool to understa 1260 * But it at least allows objtool to understand the control flow *around* the
1316 * retpoline. 1261 * retpoline.
1317 */ 1262 */
1318 static int add_ignore_alternatives(struct obj 1263 static int add_ignore_alternatives(struct objtool_file *file)
1319 { 1264 {
1320 struct section *rsec; 1265 struct section *rsec;
1321 struct reloc *reloc; 1266 struct reloc *reloc;
1322 struct instruction *insn; 1267 struct instruction *insn;
1323 1268
1324 rsec = find_section_by_name(file->elf 1269 rsec = find_section_by_name(file->elf, ".rela.discard.ignore_alts");
1325 if (!rsec) 1270 if (!rsec)
1326 return 0; 1271 return 0;
1327 1272
1328 for_each_reloc(rsec, reloc) { 1273 for_each_reloc(rsec, reloc) {
1329 if (reloc->sym->type != STT_S 1274 if (reloc->sym->type != STT_SECTION) {
1330 WARN("unexpected relo 1275 WARN("unexpected relocation symbol type in %s", rsec->name);
1331 return -1; 1276 return -1;
1332 } 1277 }
1333 1278
1334 insn = find_insn(file, reloc- 1279 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
1335 if (!insn) { 1280 if (!insn) {
1336 WARN("bad .discard.ig 1281 WARN("bad .discard.ignore_alts entry");
1337 return -1; 1282 return -1;
1338 } 1283 }
1339 1284
1340 insn->ignore_alts = true; 1285 insn->ignore_alts = true;
1341 } 1286 }
1342 1287
1343 return 0; 1288 return 0;
1344 } 1289 }
1345 1290
1346 /* 1291 /*
1347 * Symbols that replace INSN_CALL_DYNAMIC, ev 1292 * Symbols that replace INSN_CALL_DYNAMIC, every (tail) call to such a symbol
1348 * will be added to the .retpoline_sites sect 1293 * will be added to the .retpoline_sites section.
1349 */ 1294 */
1350 __weak bool arch_is_retpoline(struct symbol * 1295 __weak bool arch_is_retpoline(struct symbol *sym)
1351 { 1296 {
1352 return false; 1297 return false;
1353 } 1298 }
1354 1299
1355 /* 1300 /*
1356 * Symbols that replace INSN_RETURN, every (t 1301 * Symbols that replace INSN_RETURN, every (tail) call to such a symbol
1357 * will be added to the .return_sites section 1302 * will be added to the .return_sites section.
1358 */ 1303 */
1359 __weak bool arch_is_rethunk(struct symbol *sy 1304 __weak bool arch_is_rethunk(struct symbol *sym)
1360 { 1305 {
1361 return false; 1306 return false;
1362 } 1307 }
1363 1308
1364 /* 1309 /*
1365 * Symbols that are embedded inside other ins 1310 * Symbols that are embedded inside other instructions, because sometimes crazy
1366 * code exists. These are mostly ignored for 1311 * code exists. These are mostly ignored for validation purposes.
1367 */ 1312 */
1368 __weak bool arch_is_embedded_insn(struct symb 1313 __weak bool arch_is_embedded_insn(struct symbol *sym)
1369 { 1314 {
1370 return false; 1315 return false;
1371 } 1316 }
1372 1317
1373 static struct reloc *insn_reloc(struct objtoo 1318 static struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn)
1374 { 1319 {
1375 struct reloc *reloc; 1320 struct reloc *reloc;
1376 1321
1377 if (insn->no_reloc) 1322 if (insn->no_reloc)
1378 return NULL; 1323 return NULL;
1379 1324
1380 if (!file) 1325 if (!file)
1381 return NULL; 1326 return NULL;
1382 1327
1383 reloc = find_reloc_by_dest_range(file 1328 reloc = find_reloc_by_dest_range(file->elf, insn->sec,
1384 insn 1329 insn->offset, insn->len);
1385 if (!reloc) { 1330 if (!reloc) {
1386 insn->no_reloc = 1; 1331 insn->no_reloc = 1;
1387 return NULL; 1332 return NULL;
1388 } 1333 }
1389 1334
1390 return reloc; 1335 return reloc;
1391 } 1336 }
1392 1337
1393 static void remove_insn_ops(struct instructio 1338 static void remove_insn_ops(struct instruction *insn)
1394 { 1339 {
1395 struct stack_op *op, *next; 1340 struct stack_op *op, *next;
1396 1341
1397 for (op = insn->stack_ops; op; op = n 1342 for (op = insn->stack_ops; op; op = next) {
1398 next = op->next; 1343 next = op->next;
1399 free(op); 1344 free(op);
1400 } 1345 }
1401 insn->stack_ops = NULL; 1346 insn->stack_ops = NULL;
1402 } 1347 }
1403 1348
1404 static void annotate_call_site(struct objtool 1349 static void annotate_call_site(struct objtool_file *file,
1405 struct instruc 1350 struct instruction *insn, bool sibling)
1406 { 1351 {
1407 struct reloc *reloc = insn_reloc(file 1352 struct reloc *reloc = insn_reloc(file, insn);
1408 struct symbol *sym = insn_call_dest(i 1353 struct symbol *sym = insn_call_dest(insn);
1409 1354
1410 if (!sym) 1355 if (!sym)
1411 sym = reloc->sym; 1356 sym = reloc->sym;
1412 1357
1413 /* 1358 /*
1414 * Alternative replacement code is ju 1359 * Alternative replacement code is just template code which is
1415 * sometimes copied to the original i 1360 * sometimes copied to the original instruction. For now, don't
1416 * annotate it. (In the future we mig 1361 * annotate it. (In the future we might consider annotating the
1417 * original instruction if/when it ev 1362 * original instruction if/when it ever makes sense to do so.)
1418 */ 1363 */
1419 if (!strcmp(insn->sec->name, ".altins 1364 if (!strcmp(insn->sec->name, ".altinstr_replacement"))
1420 return; 1365 return;
1421 1366
1422 if (sym->static_call_tramp) { 1367 if (sym->static_call_tramp) {
1423 list_add_tail(&insn->call_nod 1368 list_add_tail(&insn->call_node, &file->static_call_list);
1424 return; 1369 return;
1425 } 1370 }
1426 1371
1427 if (sym->retpoline_thunk) { 1372 if (sym->retpoline_thunk) {
1428 list_add_tail(&insn->call_nod 1373 list_add_tail(&insn->call_node, &file->retpoline_call_list);
1429 return; 1374 return;
1430 } 1375 }
1431 1376
1432 /* 1377 /*
1433 * Many compilers cannot disable KCOV 1378 * Many compilers cannot disable KCOV or sanitizer calls with a function
1434 * attribute so they need a little he 1379 * attribute so they need a little help, NOP out any such calls from
1435 * noinstr text. 1380 * noinstr text.
1436 */ 1381 */
1437 if (opts.hack_noinstr && insn->sec->n 1382 if (opts.hack_noinstr && insn->sec->noinstr && sym->profiling_func) {
1438 if (reloc) 1383 if (reloc)
1439 set_reloc_type(file-> 1384 set_reloc_type(file->elf, reloc, R_NONE);
1440 1385
1441 elf_write_insn(file->elf, ins 1386 elf_write_insn(file->elf, insn->sec,
1442 insn->offset, 1387 insn->offset, insn->len,
1443 sibling ? arch 1388 sibling ? arch_ret_insn(insn->len)
1444 : arch 1389 : arch_nop_insn(insn->len));
1445 1390
1446 insn->type = sibling ? INSN_R 1391 insn->type = sibling ? INSN_RETURN : INSN_NOP;
1447 1392
1448 if (sibling) { 1393 if (sibling) {
1449 /* 1394 /*
1450 * We've replaced the 1395 * We've replaced the tail-call JMP insn by two new
1451 * insn: RET; INT3, e 1396 * insn: RET; INT3, except we only have a single struct
1452 * insn here. Mark it 1397 * insn here. Mark it retpoline_safe to avoid the SLS
1453 * warning, instead o 1398 * warning, instead of adding another insn.
1454 */ 1399 */
1455 insn->retpoline_safe 1400 insn->retpoline_safe = true;
1456 } 1401 }
1457 1402
1458 return; 1403 return;
1459 } 1404 }
1460 1405
1461 if (opts.mcount && sym->fentry) { 1406 if (opts.mcount && sym->fentry) {
1462 if (sibling) 1407 if (sibling)
1463 WARN_INSN(insn, "tail 1408 WARN_INSN(insn, "tail call to __fentry__ !?!?");
1464 if (opts.mnop) { 1409 if (opts.mnop) {
1465 if (reloc) 1410 if (reloc)
1466 set_reloc_typ 1411 set_reloc_type(file->elf, reloc, R_NONE);
1467 1412
1468 elf_write_insn(file-> 1413 elf_write_insn(file->elf, insn->sec,
1469 insn-> 1414 insn->offset, insn->len,
1470 arch_n 1415 arch_nop_insn(insn->len));
1471 1416
1472 insn->type = INSN_NOP 1417 insn->type = INSN_NOP;
1473 } 1418 }
1474 1419
1475 list_add_tail(&insn->call_nod 1420 list_add_tail(&insn->call_node, &file->mcount_loc_list);
1476 return; 1421 return;
1477 } 1422 }
1478 1423
1479 if (insn->type == INSN_CALL && !insn- 1424 if (insn->type == INSN_CALL && !insn->sec->init)
1480 list_add_tail(&insn->call_nod 1425 list_add_tail(&insn->call_node, &file->call_list);
1481 1426
1482 if (!sibling && dead_end_function(fil 1427 if (!sibling && dead_end_function(file, sym))
1483 insn->dead_end = true; 1428 insn->dead_end = true;
1484 } 1429 }
1485 1430
1486 static void add_call_dest(struct objtool_file 1431 static void add_call_dest(struct objtool_file *file, struct instruction *insn,
1487 struct symbol *dest 1432 struct symbol *dest, bool sibling)
1488 { 1433 {
1489 insn->_call_dest = dest; 1434 insn->_call_dest = dest;
1490 if (!dest) 1435 if (!dest)
1491 return; 1436 return;
1492 1437
1493 /* 1438 /*
1494 * Whatever stack impact regular CALL 1439 * Whatever stack impact regular CALLs have, should be undone
1495 * by the RETURN of the called functi 1440 * by the RETURN of the called function.
1496 * 1441 *
1497 * Annotated intra-function calls ret 1442 * Annotated intra-function calls retain the stack_ops but
1498 * are converted to JUMP, see read_in 1443 * are converted to JUMP, see read_intra_function_calls().
1499 */ 1444 */
1500 remove_insn_ops(insn); 1445 remove_insn_ops(insn);
1501 1446
1502 annotate_call_site(file, insn, siblin 1447 annotate_call_site(file, insn, sibling);
1503 } 1448 }
1504 1449
1505 static void add_retpoline_call(struct objtool 1450 static void add_retpoline_call(struct objtool_file *file, struct instruction *insn)
1506 { 1451 {
1507 /* 1452 /*
1508 * Retpoline calls/jumps are really d 1453 * Retpoline calls/jumps are really dynamic calls/jumps in disguise,
1509 * so convert them accordingly. 1454 * so convert them accordingly.
1510 */ 1455 */
1511 switch (insn->type) { 1456 switch (insn->type) {
1512 case INSN_CALL: 1457 case INSN_CALL:
1513 insn->type = INSN_CALL_DYNAMI 1458 insn->type = INSN_CALL_DYNAMIC;
1514 break; 1459 break;
1515 case INSN_JUMP_UNCONDITIONAL: 1460 case INSN_JUMP_UNCONDITIONAL:
1516 insn->type = INSN_JUMP_DYNAMI 1461 insn->type = INSN_JUMP_DYNAMIC;
1517 break; 1462 break;
1518 case INSN_JUMP_CONDITIONAL: 1463 case INSN_JUMP_CONDITIONAL:
1519 insn->type = INSN_JUMP_DYNAMI 1464 insn->type = INSN_JUMP_DYNAMIC_CONDITIONAL;
1520 break; 1465 break;
1521 default: 1466 default:
1522 return; 1467 return;
1523 } 1468 }
1524 1469
1525 insn->retpoline_safe = true; 1470 insn->retpoline_safe = true;
1526 1471
1527 /* 1472 /*
1528 * Whatever stack impact regular CALL 1473 * Whatever stack impact regular CALLs have, should be undone
1529 * by the RETURN of the called functi 1474 * by the RETURN of the called function.
1530 * 1475 *
1531 * Annotated intra-function calls ret 1476 * Annotated intra-function calls retain the stack_ops but
1532 * are converted to JUMP, see read_in 1477 * are converted to JUMP, see read_intra_function_calls().
1533 */ 1478 */
1534 remove_insn_ops(insn); 1479 remove_insn_ops(insn);
1535 1480
1536 annotate_call_site(file, insn, false) 1481 annotate_call_site(file, insn, false);
1537 } 1482 }
1538 1483
1539 static void add_return_call(struct objtool_fi 1484 static void add_return_call(struct objtool_file *file, struct instruction *insn, bool add)
1540 { 1485 {
1541 /* 1486 /*
1542 * Return thunk tail calls are really 1487 * Return thunk tail calls are really just returns in disguise,
1543 * so convert them accordingly. 1488 * so convert them accordingly.
1544 */ 1489 */
1545 insn->type = INSN_RETURN; 1490 insn->type = INSN_RETURN;
1546 insn->retpoline_safe = true; 1491 insn->retpoline_safe = true;
1547 1492
1548 if (add) 1493 if (add)
1549 list_add_tail(&insn->call_nod 1494 list_add_tail(&insn->call_node, &file->return_thunk_list);
1550 } 1495 }
1551 1496
1552 static bool is_first_func_insn(struct objtool 1497 static bool is_first_func_insn(struct objtool_file *file,
1553 struct instruc 1498 struct instruction *insn, struct symbol *sym)
1554 { 1499 {
1555 if (insn->offset == sym->offset) 1500 if (insn->offset == sym->offset)
1556 return true; 1501 return true;
1557 1502
1558 /* Allow direct CALL/JMP past ENDBR * 1503 /* Allow direct CALL/JMP past ENDBR */
1559 if (opts.ibt) { 1504 if (opts.ibt) {
1560 struct instruction *prev = pr 1505 struct instruction *prev = prev_insn_same_sym(file, insn);
1561 1506
1562 if (prev && prev->type == INS 1507 if (prev && prev->type == INSN_ENDBR &&
1563 insn->offset == sym->offs 1508 insn->offset == sym->offset + prev->len)
1564 return true; 1509 return true;
1565 } 1510 }
1566 1511
1567 return false; 1512 return false;
1568 } 1513 }
1569 1514
1570 /* 1515 /*
1571 * A sibling call is a tail-call to another s 1516 * A sibling call is a tail-call to another symbol -- to differentiate from a
1572 * recursive tail-call which is to the same s 1517 * recursive tail-call which is to the same symbol.
1573 */ 1518 */
1574 static bool jump_is_sibling_call(struct objto 1519 static bool jump_is_sibling_call(struct objtool_file *file,
1575 struct instr 1520 struct instruction *from, struct instruction *to)
1576 { 1521 {
1577 struct symbol *fs = from->sym; 1522 struct symbol *fs = from->sym;
1578 struct symbol *ts = to->sym; 1523 struct symbol *ts = to->sym;
1579 1524
1580 /* Not a sibling call if from/to a sy 1525 /* Not a sibling call if from/to a symbol hole */
1581 if (!fs || !ts) 1526 if (!fs || !ts)
1582 return false; 1527 return false;
1583 1528
1584 /* Not a sibling call if not targetin 1529 /* Not a sibling call if not targeting the start of a symbol. */
1585 if (!is_first_func_insn(file, to, ts) 1530 if (!is_first_func_insn(file, to, ts))
1586 return false; 1531 return false;
1587 1532
1588 /* Disallow sibling calls into STT_NO 1533 /* Disallow sibling calls into STT_NOTYPE */
1589 if (ts->type == STT_NOTYPE) 1534 if (ts->type == STT_NOTYPE)
1590 return false; 1535 return false;
1591 1536
1592 /* Must not be self to be a sibling * 1537 /* Must not be self to be a sibling */
1593 return fs->pfunc != ts->pfunc; 1538 return fs->pfunc != ts->pfunc;
1594 } 1539 }
1595 1540
1596 /* 1541 /*
1597 * Find the destination instructions for all 1542 * Find the destination instructions for all jumps.
1598 */ 1543 */
1599 static int add_jump_destinations(struct objto 1544 static int add_jump_destinations(struct objtool_file *file)
1600 { 1545 {
1601 struct instruction *insn, *jump_dest; 1546 struct instruction *insn, *jump_dest;
1602 struct reloc *reloc; 1547 struct reloc *reloc;
1603 struct section *dest_sec; 1548 struct section *dest_sec;
1604 unsigned long dest_off; 1549 unsigned long dest_off;
1605 1550
1606 for_each_insn(file, insn) { 1551 for_each_insn(file, insn) {
1607 if (insn->jump_dest) { 1552 if (insn->jump_dest) {
1608 /* 1553 /*
1609 * handle_group_alt() 1554 * handle_group_alt() may have previously set
1610 * 'jump_dest' for so 1555 * 'jump_dest' for some alternatives.
1611 */ 1556 */
1612 continue; 1557 continue;
1613 } 1558 }
1614 if (!is_static_jump(insn)) 1559 if (!is_static_jump(insn))
1615 continue; 1560 continue;
1616 1561
1617 reloc = insn_reloc(file, insn 1562 reloc = insn_reloc(file, insn);
1618 if (!reloc) { 1563 if (!reloc) {
1619 dest_sec = insn->sec; 1564 dest_sec = insn->sec;
1620 dest_off = arch_jump_ 1565 dest_off = arch_jump_destination(insn);
1621 } else if (reloc->sym->type = 1566 } else if (reloc->sym->type == STT_SECTION) {
1622 dest_sec = reloc->sym 1567 dest_sec = reloc->sym->sec;
1623 dest_off = arch_dest_ 1568 dest_off = arch_dest_reloc_offset(reloc_addend(reloc));
1624 } else if (reloc->sym->retpol 1569 } else if (reloc->sym->retpoline_thunk) {
1625 add_retpoline_call(fi 1570 add_retpoline_call(file, insn);
1626 continue; 1571 continue;
1627 } else if (reloc->sym->return 1572 } else if (reloc->sym->return_thunk) {
1628 add_return_call(file, 1573 add_return_call(file, insn, true);
1629 continue; 1574 continue;
1630 } else if (insn_func(insn)) { 1575 } else if (insn_func(insn)) {
1631 /* 1576 /*
1632 * External sibling c 1577 * External sibling call or internal sibling call with
1633 * STT_FUNC reloc. 1578 * STT_FUNC reloc.
1634 */ 1579 */
1635 add_call_dest(file, i 1580 add_call_dest(file, insn, reloc->sym, true);
1636 continue; 1581 continue;
1637 } else if (reloc->sym->sec->i 1582 } else if (reloc->sym->sec->idx) {
1638 dest_sec = reloc->sym 1583 dest_sec = reloc->sym->sec;
1639 dest_off = reloc->sym 1584 dest_off = reloc->sym->sym.st_value +
1640 arch_dest_ 1585 arch_dest_reloc_offset(reloc_addend(reloc));
1641 } else { 1586 } else {
1642 /* non-func asm code 1587 /* non-func asm code jumping to another file */
1643 continue; 1588 continue;
1644 } 1589 }
1645 1590
1646 jump_dest = find_insn(file, d 1591 jump_dest = find_insn(file, dest_sec, dest_off);
1647 if (!jump_dest) { 1592 if (!jump_dest) {
1648 struct symbol *sym = 1593 struct symbol *sym = find_symbol_by_offset(dest_sec, dest_off);
1649 1594
1650 /* 1595 /*
1651 * This is a special 1596 * This is a special case for retbleed_untrain_ret().
1652 * It jumps to __x86_ 1597 * It jumps to __x86_return_thunk(), but objtool
1653 * can't find the thu 1598 * can't find the thunk's starting RET
1654 * instruction, becau 1599 * instruction, because the RET is also in the
1655 * middle of another 1600 * middle of another instruction. Objtool only
1656 * knows about the ou 1601 * knows about the outer instruction.
1657 */ 1602 */
1658 if (sym && sym->embed 1603 if (sym && sym->embedded_insn) {
1659 add_return_ca 1604 add_return_call(file, insn, false);
1660 continue; 1605 continue;
1661 } 1606 }
1662 1607
1663 WARN_INSN(insn, "can' 1608 WARN_INSN(insn, "can't find jump dest instruction at %s+0x%lx",
1664 dest_sec->n 1609 dest_sec->name, dest_off);
1665 return -1; 1610 return -1;
1666 } 1611 }
1667 1612
1668 /* 1613 /*
1669 * An intra-TU jump in retpol <<
1670 * for its jump dest, in whic <<
1671 * add_{retpoline,return}_cal <<
1672 */ <<
1673 if (jump_dest->sym && jump_de <<
1674 if (jump_dest->sym->r <<
1675 add_retpoline <<
1676 continue; <<
1677 } <<
1678 if (jump_dest->sym->r <<
1679 add_return_ca <<
1680 continue; <<
1681 } <<
1682 } <<
1683 <<
1684 /* <<
1685 * Cross-function jump. 1614 * Cross-function jump.
1686 */ 1615 */
1687 if (insn_func(insn) && insn_f 1616 if (insn_func(insn) && insn_func(jump_dest) &&
1688 insn_func(insn) != insn_f 1617 insn_func(insn) != insn_func(jump_dest)) {
1689 1618
1690 /* 1619 /*
1691 * For GCC 8+, create 1620 * For GCC 8+, create parent/child links for any cold
1692 * subfunctions. Thi 1621 * subfunctions. This is _mostly_ redundant with a
1693 * similar initializa 1622 * similar initialization in read_symbols().
1694 * 1623 *
1695 * If a function has 1624 * If a function has aliases, we want the *first* such
1696 * function in the sy 1625 * function in the symbol table to be the subfunction's
1697 * parent. In that c 1626 * parent. In that case we overwrite the
1698 * initialization don 1627 * initialization done in read_symbols().
1699 * 1628 *
1700 * However this code 1629 * However this code can't completely replace the
1701 * read_symbols() cod 1630 * read_symbols() code because this doesn't detect the
1702 * case where the par 1631 * case where the parent function's only reference to a
1703 * subfunction is thr 1632 * subfunction is through a jump table.
1704 */ 1633 */
1705 if (!strstr(insn_func 1634 if (!strstr(insn_func(insn)->name, ".cold") &&
1706 strstr(insn_func( 1635 strstr(insn_func(jump_dest)->name, ".cold")) {
1707 insn_func(ins 1636 insn_func(insn)->cfunc = insn_func(jump_dest);
1708 insn_func(jum 1637 insn_func(jump_dest)->pfunc = insn_func(insn);
1709 } 1638 }
1710 } 1639 }
1711 1640
1712 if (jump_is_sibling_call(file 1641 if (jump_is_sibling_call(file, insn, jump_dest)) {
1713 /* 1642 /*
1714 * Internal sibling c 1643 * Internal sibling call without reloc or with
1715 * STT_SECTION reloc. 1644 * STT_SECTION reloc.
1716 */ 1645 */
1717 add_call_dest(file, i 1646 add_call_dest(file, insn, insn_func(jump_dest), true);
1718 continue; 1647 continue;
1719 } 1648 }
1720 1649
1721 insn->jump_dest = jump_dest; 1650 insn->jump_dest = jump_dest;
1722 } 1651 }
1723 1652
1724 return 0; 1653 return 0;
1725 } 1654 }
1726 1655
1727 static struct symbol *find_call_destination(s 1656 static struct symbol *find_call_destination(struct section *sec, unsigned long offset)
1728 { 1657 {
1729 struct symbol *call_dest; 1658 struct symbol *call_dest;
1730 1659
1731 call_dest = find_func_by_offset(sec, 1660 call_dest = find_func_by_offset(sec, offset);
1732 if (!call_dest) 1661 if (!call_dest)
1733 call_dest = find_symbol_by_of 1662 call_dest = find_symbol_by_offset(sec, offset);
1734 1663
1735 return call_dest; 1664 return call_dest;
1736 } 1665 }
1737 1666
1738 /* 1667 /*
1739 * Find the destination instructions for all 1668 * Find the destination instructions for all calls.
1740 */ 1669 */
1741 static int add_call_destinations(struct objto 1670 static int add_call_destinations(struct objtool_file *file)
1742 { 1671 {
1743 struct instruction *insn; 1672 struct instruction *insn;
1744 unsigned long dest_off; 1673 unsigned long dest_off;
1745 struct symbol *dest; 1674 struct symbol *dest;
1746 struct reloc *reloc; 1675 struct reloc *reloc;
1747 1676
1748 for_each_insn(file, insn) { 1677 for_each_insn(file, insn) {
1749 if (insn->type != INSN_CALL) 1678 if (insn->type != INSN_CALL)
1750 continue; 1679 continue;
1751 1680
1752 reloc = insn_reloc(file, insn 1681 reloc = insn_reloc(file, insn);
1753 if (!reloc) { 1682 if (!reloc) {
1754 dest_off = arch_jump_ 1683 dest_off = arch_jump_destination(insn);
1755 dest = find_call_dest 1684 dest = find_call_destination(insn->sec, dest_off);
1756 1685
1757 add_call_dest(file, i 1686 add_call_dest(file, insn, dest, false);
1758 1687
1759 if (insn->ignore) 1688 if (insn->ignore)
1760 continue; 1689 continue;
1761 1690
1762 if (!insn_call_dest(i 1691 if (!insn_call_dest(insn)) {
1763 WARN_INSN(ins 1692 WARN_INSN(insn, "unannotated intra-function call");
1764 return -1; 1693 return -1;
1765 } 1694 }
1766 1695
1767 if (insn_func(insn) & 1696 if (insn_func(insn) && insn_call_dest(insn)->type != STT_FUNC) {
1768 WARN_INSN(ins 1697 WARN_INSN(insn, "unsupported call to non-function");
1769 return -1; 1698 return -1;
1770 } 1699 }
1771 1700
1772 } else if (reloc->sym->type = 1701 } else if (reloc->sym->type == STT_SECTION) {
1773 dest_off = arch_dest_ 1702 dest_off = arch_dest_reloc_offset(reloc_addend(reloc));
1774 dest = find_call_dest 1703 dest = find_call_destination(reloc->sym->sec, dest_off);
1775 if (!dest) { 1704 if (!dest) {
1776 WARN_INSN(ins 1705 WARN_INSN(insn, "can't find call dest symbol at %s+0x%lx",
1777 rel 1706 reloc->sym->sec->name, dest_off);
1778 return -1; 1707 return -1;
1779 } 1708 }
1780 1709
1781 add_call_dest(file, i 1710 add_call_dest(file, insn, dest, false);
1782 1711
1783 } else if (reloc->sym->retpol 1712 } else if (reloc->sym->retpoline_thunk) {
1784 add_retpoline_call(fi 1713 add_retpoline_call(file, insn);
1785 1714
1786 } else 1715 } else
1787 add_call_dest(file, i 1716 add_call_dest(file, insn, reloc->sym, false);
1788 } 1717 }
1789 1718
1790 return 0; 1719 return 0;
1791 } 1720 }
1792 1721
1793 /* 1722 /*
1794 * The .alternatives section requires some ex 1723 * The .alternatives section requires some extra special care over and above
1795 * other special sections because alternative 1724 * other special sections because alternatives are patched in place.
1796 */ 1725 */
1797 static int handle_group_alt(struct objtool_fi 1726 static int handle_group_alt(struct objtool_file *file,
1798 struct special_al 1727 struct special_alt *special_alt,
1799 struct instructio 1728 struct instruction *orig_insn,
1800 struct instructio 1729 struct instruction **new_insn)
1801 { 1730 {
1802 struct instruction *last_new_insn = N 1731 struct instruction *last_new_insn = NULL, *insn, *nop = NULL;
1803 struct alt_group *orig_alt_group, *ne 1732 struct alt_group *orig_alt_group, *new_alt_group;
1804 unsigned long dest_off; 1733 unsigned long dest_off;
1805 1734
1806 orig_alt_group = orig_insn->alt_group 1735 orig_alt_group = orig_insn->alt_group;
1807 if (!orig_alt_group) { 1736 if (!orig_alt_group) {
1808 struct instruction *last_orig 1737 struct instruction *last_orig_insn = NULL;
1809 1738
1810 orig_alt_group = malloc(sizeo 1739 orig_alt_group = malloc(sizeof(*orig_alt_group));
1811 if (!orig_alt_group) { 1740 if (!orig_alt_group) {
1812 WARN("malloc failed") 1741 WARN("malloc failed");
1813 return -1; 1742 return -1;
1814 } 1743 }
1815 orig_alt_group->cfi = calloc( 1744 orig_alt_group->cfi = calloc(special_alt->orig_len,
1816 1745 sizeof(struct cfi_state *));
1817 if (!orig_alt_group->cfi) { 1746 if (!orig_alt_group->cfi) {
1818 WARN("calloc failed") 1747 WARN("calloc failed");
1819 return -1; 1748 return -1;
1820 } 1749 }
1821 1750
1822 insn = orig_insn; 1751 insn = orig_insn;
1823 sec_for_each_insn_from(file, 1752 sec_for_each_insn_from(file, insn) {
1824 if (insn->offset >= s 1753 if (insn->offset >= special_alt->orig_off + special_alt->orig_len)
1825 break; 1754 break;
1826 1755
1827 insn->alt_group = ori 1756 insn->alt_group = orig_alt_group;
1828 last_orig_insn = insn 1757 last_orig_insn = insn;
1829 } 1758 }
1830 orig_alt_group->orig_group = 1759 orig_alt_group->orig_group = NULL;
1831 orig_alt_group->first_insn = 1760 orig_alt_group->first_insn = orig_insn;
1832 orig_alt_group->last_insn = l 1761 orig_alt_group->last_insn = last_orig_insn;
1833 orig_alt_group->nop = NULL; 1762 orig_alt_group->nop = NULL;
1834 } else { 1763 } else {
1835 if (orig_alt_group->last_insn 1764 if (orig_alt_group->last_insn->offset + orig_alt_group->last_insn->len -
1836 orig_alt_group->first_ins 1765 orig_alt_group->first_insn->offset != special_alt->orig_len) {
1837 WARN_INSN(orig_insn, 1766 WARN_INSN(orig_insn, "weirdly overlapping alternative! %ld != %d",
1838 orig_alt_gr 1767 orig_alt_group->last_insn->offset +
1839 orig_alt_gr 1768 orig_alt_group->last_insn->len -
1840 orig_alt_gr 1769 orig_alt_group->first_insn->offset,
1841 special_alt 1770 special_alt->orig_len);
1842 return -1; 1771 return -1;
1843 } 1772 }
1844 } 1773 }
1845 1774
1846 new_alt_group = malloc(sizeof(*new_al 1775 new_alt_group = malloc(sizeof(*new_alt_group));
1847 if (!new_alt_group) { 1776 if (!new_alt_group) {
1848 WARN("malloc failed"); 1777 WARN("malloc failed");
1849 return -1; 1778 return -1;
1850 } 1779 }
1851 1780
1852 if (special_alt->new_len < special_al 1781 if (special_alt->new_len < special_alt->orig_len) {
1853 /* 1782 /*
1854 * Insert a fake nop at the e 1783 * Insert a fake nop at the end to make the replacement
1855 * alt_group the same size as 1784 * alt_group the same size as the original. This is needed to
1856 * allow propagate_alt_cfi() 1785 * allow propagate_alt_cfi() to do its magic. When the last
1857 * instruction affects the st 1786 * instruction affects the stack, the instruction after it (the
1858 * nop) will propagate the ne 1787 * nop) will propagate the new state to the shared CFI array.
1859 */ 1788 */
1860 nop = malloc(sizeof(*nop)); 1789 nop = malloc(sizeof(*nop));
1861 if (!nop) { 1790 if (!nop) {
1862 WARN("malloc failed") 1791 WARN("malloc failed");
1863 return -1; 1792 return -1;
1864 } 1793 }
1865 memset(nop, 0, sizeof(*nop)); 1794 memset(nop, 0, sizeof(*nop));
1866 1795
1867 nop->sec = special_alt->new_s 1796 nop->sec = special_alt->new_sec;
1868 nop->offset = special_alt->ne 1797 nop->offset = special_alt->new_off + special_alt->new_len;
1869 nop->len = special_alt->orig_ 1798 nop->len = special_alt->orig_len - special_alt->new_len;
1870 nop->type = INSN_NOP; 1799 nop->type = INSN_NOP;
1871 nop->sym = orig_insn->sym; 1800 nop->sym = orig_insn->sym;
1872 nop->alt_group = new_alt_grou 1801 nop->alt_group = new_alt_group;
1873 nop->ignore = orig_insn->igno 1802 nop->ignore = orig_insn->ignore_alts;
1874 } 1803 }
1875 1804
1876 if (!special_alt->new_len) { 1805 if (!special_alt->new_len) {
1877 *new_insn = nop; 1806 *new_insn = nop;
1878 goto end; 1807 goto end;
1879 } 1808 }
1880 1809
1881 insn = *new_insn; 1810 insn = *new_insn;
1882 sec_for_each_insn_from(file, insn) { 1811 sec_for_each_insn_from(file, insn) {
1883 struct reloc *alt_reloc; 1812 struct reloc *alt_reloc;
1884 1813
1885 if (insn->offset >= special_a 1814 if (insn->offset >= special_alt->new_off + special_alt->new_len)
1886 break; 1815 break;
1887 1816
1888 last_new_insn = insn; 1817 last_new_insn = insn;
1889 1818
1890 insn->ignore = orig_insn->ign 1819 insn->ignore = orig_insn->ignore_alts;
1891 insn->sym = orig_insn->sym; 1820 insn->sym = orig_insn->sym;
1892 insn->alt_group = new_alt_gro 1821 insn->alt_group = new_alt_group;
1893 1822
1894 /* 1823 /*
1895 * Since alternative replacem 1824 * Since alternative replacement code is copy/pasted by the
1896 * kernel after applying relo 1825 * kernel after applying relocations, generally such code can't
1897 * have relative-address relo 1826 * have relative-address relocation references to outside the
1898 * .altinstr_replacement sect 1827 * .altinstr_replacement section, unless the arch's
1899 * alternatives code can adju 1828 * alternatives code can adjust the relative offsets
1900 * accordingly. 1829 * accordingly.
1901 */ 1830 */
1902 alt_reloc = insn_reloc(file, 1831 alt_reloc = insn_reloc(file, insn);
1903 if (alt_reloc && arch_pc_rela 1832 if (alt_reloc && arch_pc_relative_reloc(alt_reloc) &&
1904 !arch_support_alt_relocat 1833 !arch_support_alt_relocation(special_alt, insn, alt_reloc)) {
1905 1834
1906 WARN_INSN(insn, "unsu 1835 WARN_INSN(insn, "unsupported relocation in alternatives section");
1907 return -1; 1836 return -1;
1908 } 1837 }
1909 1838
1910 if (!is_static_jump(insn)) 1839 if (!is_static_jump(insn))
1911 continue; 1840 continue;
1912 1841
1913 if (!insn->immediate) 1842 if (!insn->immediate)
1914 continue; 1843 continue;
1915 1844
1916 dest_off = arch_jump_destinat 1845 dest_off = arch_jump_destination(insn);
1917 if (dest_off == special_alt-> 1846 if (dest_off == special_alt->new_off + special_alt->new_len) {
1918 insn->jump_dest = nex 1847 insn->jump_dest = next_insn_same_sec(file, orig_alt_group->last_insn);
1919 if (!insn->jump_dest) 1848 if (!insn->jump_dest) {
1920 WARN_INSN(ins 1849 WARN_INSN(insn, "can't find alternative jump destination");
1921 return -1; 1850 return -1;
1922 } 1851 }
1923 } 1852 }
1924 } 1853 }
1925 1854
1926 if (!last_new_insn) { 1855 if (!last_new_insn) {
1927 WARN_FUNC("can't find last ne 1856 WARN_FUNC("can't find last new alternative instruction",
1928 special_alt->new_se 1857 special_alt->new_sec, special_alt->new_off);
1929 return -1; 1858 return -1;
1930 } 1859 }
1931 1860
1932 end: 1861 end:
1933 new_alt_group->orig_group = orig_alt_ 1862 new_alt_group->orig_group = orig_alt_group;
1934 new_alt_group->first_insn = *new_insn 1863 new_alt_group->first_insn = *new_insn;
1935 new_alt_group->last_insn = last_new_i 1864 new_alt_group->last_insn = last_new_insn;
1936 new_alt_group->nop = nop; 1865 new_alt_group->nop = nop;
1937 new_alt_group->cfi = orig_alt_group-> 1866 new_alt_group->cfi = orig_alt_group->cfi;
1938 return 0; 1867 return 0;
1939 } 1868 }
1940 1869
1941 /* 1870 /*
1942 * A jump table entry can either convert a no 1871 * A jump table entry can either convert a nop to a jump or a jump to a nop.
1943 * If the original instruction is a jump, mak 1872 * If the original instruction is a jump, make the alt entry an effective nop
1944 * by just skipping the original instruction. 1873 * by just skipping the original instruction.
1945 */ 1874 */
1946 static int handle_jump_alt(struct objtool_fil 1875 static int handle_jump_alt(struct objtool_file *file,
1947 struct special_alt 1876 struct special_alt *special_alt,
1948 struct instruction 1877 struct instruction *orig_insn,
1949 struct instruction 1878 struct instruction **new_insn)
1950 { 1879 {
1951 if (orig_insn->type != INSN_JUMP_UNCO 1880 if (orig_insn->type != INSN_JUMP_UNCONDITIONAL &&
1952 orig_insn->type != INSN_NOP) { 1881 orig_insn->type != INSN_NOP) {
1953 1882
1954 WARN_INSN(orig_insn, "unsuppo 1883 WARN_INSN(orig_insn, "unsupported instruction at jump label");
1955 return -1; 1884 return -1;
1956 } 1885 }
1957 1886
1958 if (opts.hack_jump_label && special_a 1887 if (opts.hack_jump_label && special_alt->key_addend & 2) {
1959 struct reloc *reloc = insn_re 1888 struct reloc *reloc = insn_reloc(file, orig_insn);
1960 1889
1961 if (reloc) 1890 if (reloc)
1962 set_reloc_type(file-> 1891 set_reloc_type(file->elf, reloc, R_NONE);
1963 elf_write_insn(file->elf, ori 1892 elf_write_insn(file->elf, orig_insn->sec,
1964 orig_insn->off 1893 orig_insn->offset, orig_insn->len,
1965 arch_nop_insn( 1894 arch_nop_insn(orig_insn->len));
1966 orig_insn->type = INSN_NOP; 1895 orig_insn->type = INSN_NOP;
1967 } 1896 }
1968 1897
1969 if (orig_insn->type == INSN_NOP) { 1898 if (orig_insn->type == INSN_NOP) {
1970 if (orig_insn->len == 2) 1899 if (orig_insn->len == 2)
1971 file->jl_nop_short++; 1900 file->jl_nop_short++;
1972 else 1901 else
1973 file->jl_nop_long++; 1902 file->jl_nop_long++;
1974 1903
1975 return 0; 1904 return 0;
1976 } 1905 }
1977 1906
1978 if (orig_insn->len == 2) 1907 if (orig_insn->len == 2)
1979 file->jl_short++; 1908 file->jl_short++;
1980 else 1909 else
1981 file->jl_long++; 1910 file->jl_long++;
1982 1911
1983 *new_insn = next_insn_same_sec(file, 1912 *new_insn = next_insn_same_sec(file, orig_insn);
1984 return 0; 1913 return 0;
1985 } 1914 }
1986 1915
1987 /* 1916 /*
1988 * Read all the special sections which have a 1917 * Read all the special sections which have alternate instructions which can be
1989 * patched in or redirected to at runtime. E 1918 * patched in or redirected to at runtime. Each instruction having alternate
1990 * instruction(s) has them added to its insn- 1919 * instruction(s) has them added to its insn->alts list, which will be
1991 * traversed in validate_branch(). 1920 * traversed in validate_branch().
1992 */ 1921 */
1993 static int add_special_section_alts(struct ob 1922 static int add_special_section_alts(struct objtool_file *file)
1994 { 1923 {
1995 struct list_head special_alts; 1924 struct list_head special_alts;
1996 struct instruction *orig_insn, *new_i 1925 struct instruction *orig_insn, *new_insn;
1997 struct special_alt *special_alt, *tmp 1926 struct special_alt *special_alt, *tmp;
1998 struct alternative *alt; 1927 struct alternative *alt;
1999 int ret; 1928 int ret;
2000 1929
2001 ret = special_get_alts(file->elf, &sp 1930 ret = special_get_alts(file->elf, &special_alts);
2002 if (ret) 1931 if (ret)
2003 return ret; 1932 return ret;
2004 1933
2005 list_for_each_entry_safe(special_alt, 1934 list_for_each_entry_safe(special_alt, tmp, &special_alts, list) {
2006 1935
2007 orig_insn = find_insn(file, s 1936 orig_insn = find_insn(file, special_alt->orig_sec,
2008 special 1937 special_alt->orig_off);
2009 if (!orig_insn) { 1938 if (!orig_insn) {
2010 WARN_FUNC("special: c 1939 WARN_FUNC("special: can't find orig instruction",
2011 special_alt 1940 special_alt->orig_sec, special_alt->orig_off);
2012 ret = -1; 1941 ret = -1;
2013 goto out; 1942 goto out;
2014 } 1943 }
2015 1944
2016 new_insn = NULL; 1945 new_insn = NULL;
2017 if (!special_alt->group || sp 1946 if (!special_alt->group || special_alt->new_len) {
2018 new_insn = find_insn( 1947 new_insn = find_insn(file, special_alt->new_sec,
2019 1948 special_alt->new_off);
2020 if (!new_insn) { 1949 if (!new_insn) {
2021 WARN_FUNC("sp 1950 WARN_FUNC("special: can't find new instruction",
2022 spe 1951 special_alt->new_sec,
2023 spe 1952 special_alt->new_off);
2024 ret = -1; 1953 ret = -1;
2025 goto out; 1954 goto out;
2026 } 1955 }
2027 } 1956 }
2028 1957
2029 if (special_alt->group) { 1958 if (special_alt->group) {
2030 if (!special_alt->ori 1959 if (!special_alt->orig_len) {
2031 WARN_INSN(ori 1960 WARN_INSN(orig_insn, "empty alternative entry");
2032 continue; 1961 continue;
2033 } 1962 }
2034 1963
2035 ret = handle_group_al 1964 ret = handle_group_alt(file, special_alt, orig_insn,
2036 1965 &new_insn);
2037 if (ret) 1966 if (ret)
2038 goto out; 1967 goto out;
2039 } else if (special_alt->jump_ 1968 } else if (special_alt->jump_or_nop) {
2040 ret = handle_jump_alt 1969 ret = handle_jump_alt(file, special_alt, orig_insn,
2041 1970 &new_insn);
2042 if (ret) 1971 if (ret)
2043 goto out; 1972 goto out;
2044 } 1973 }
2045 1974
2046 alt = malloc(sizeof(*alt)); 1975 alt = malloc(sizeof(*alt));
2047 if (!alt) { 1976 if (!alt) {
2048 WARN("malloc failed") 1977 WARN("malloc failed");
2049 ret = -1; 1978 ret = -1;
2050 goto out; 1979 goto out;
2051 } 1980 }
2052 1981
2053 alt->insn = new_insn; 1982 alt->insn = new_insn;
2054 alt->skip_orig = special_alt- 1983 alt->skip_orig = special_alt->skip_orig;
2055 orig_insn->ignore_alts |= spe 1984 orig_insn->ignore_alts |= special_alt->skip_alt;
2056 alt->next = orig_insn->alts; 1985 alt->next = orig_insn->alts;
2057 orig_insn->alts = alt; 1986 orig_insn->alts = alt;
2058 1987
2059 list_del(&special_alt->list); 1988 list_del(&special_alt->list);
2060 free(special_alt); 1989 free(special_alt);
2061 } 1990 }
2062 1991
2063 if (opts.stats) { 1992 if (opts.stats) {
2064 printf("jl\\\tNOP\tJMP\n"); 1993 printf("jl\\\tNOP\tJMP\n");
2065 printf("short:\t%ld\t%ld\n", 1994 printf("short:\t%ld\t%ld\n", file->jl_nop_short, file->jl_short);
2066 printf("long:\t%ld\t%ld\n", f 1995 printf("long:\t%ld\t%ld\n", file->jl_nop_long, file->jl_long);
2067 } 1996 }
2068 1997
2069 out: 1998 out:
2070 return ret; 1999 return ret;
2071 } 2000 }
2072 2001
2073 static int add_jump_table(struct objtool_file 2002 static int add_jump_table(struct objtool_file *file, struct instruction *insn,
2074 struct reloc *next_ 2003 struct reloc *next_table)
2075 { 2004 {
2076 struct symbol *pfunc = insn_func(insn 2005 struct symbol *pfunc = insn_func(insn)->pfunc;
2077 struct reloc *table = insn_jump_table 2006 struct reloc *table = insn_jump_table(insn);
2078 struct instruction *dest_insn; 2007 struct instruction *dest_insn;
2079 unsigned int prev_offset = 0; 2008 unsigned int prev_offset = 0;
2080 struct reloc *reloc = table; 2009 struct reloc *reloc = table;
2081 struct alternative *alt; 2010 struct alternative *alt;
2082 2011
2083 /* 2012 /*
2084 * Each @reloc is a switch table relo 2013 * Each @reloc is a switch table relocation which points to the target
2085 * instruction. 2014 * instruction.
2086 */ 2015 */
2087 for_each_reloc_from(table->sec, reloc 2016 for_each_reloc_from(table->sec, reloc) {
2088 2017
2089 /* Check for the end of the t 2018 /* Check for the end of the table: */
2090 if (reloc != table && reloc = 2019 if (reloc != table && reloc == next_table)
2091 break; 2020 break;
2092 2021
2093 /* Make sure the table entrie 2022 /* Make sure the table entries are consecutive: */
2094 if (prev_offset && reloc_offs 2023 if (prev_offset && reloc_offset(reloc) != prev_offset + 8)
2095 break; 2024 break;
2096 2025
2097 /* Detect function pointers f 2026 /* Detect function pointers from contiguous objects: */
2098 if (reloc->sym->sec == pfunc- 2027 if (reloc->sym->sec == pfunc->sec &&
2099 reloc_addend(reloc) == pf 2028 reloc_addend(reloc) == pfunc->offset)
2100 break; 2029 break;
2101 2030
2102 dest_insn = find_insn(file, r 2031 dest_insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2103 if (!dest_insn) 2032 if (!dest_insn)
2104 break; 2033 break;
2105 2034
2106 /* Make sure the destination 2035 /* Make sure the destination is in the same function: */
2107 if (!insn_func(dest_insn) || 2036 if (!insn_func(dest_insn) || insn_func(dest_insn)->pfunc != pfunc)
2108 break; 2037 break;
2109 2038
2110 alt = malloc(sizeof(*alt)); 2039 alt = malloc(sizeof(*alt));
2111 if (!alt) { 2040 if (!alt) {
2112 WARN("malloc failed") 2041 WARN("malloc failed");
2113 return -1; 2042 return -1;
2114 } 2043 }
2115 2044
2116 alt->insn = dest_insn; 2045 alt->insn = dest_insn;
2117 alt->next = insn->alts; 2046 alt->next = insn->alts;
2118 insn->alts = alt; 2047 insn->alts = alt;
2119 prev_offset = reloc_offset(re 2048 prev_offset = reloc_offset(reloc);
2120 } 2049 }
2121 2050
2122 if (!prev_offset) { 2051 if (!prev_offset) {
2123 WARN_INSN(insn, "can't find s 2052 WARN_INSN(insn, "can't find switch jump table");
2124 return -1; 2053 return -1;
2125 } 2054 }
2126 2055
2127 return 0; 2056 return 0;
2128 } 2057 }
2129 2058
2130 /* 2059 /*
2131 * find_jump_table() - Given a dynamic jump, 2060 * find_jump_table() - Given a dynamic jump, find the switch jump table
2132 * associated with it. 2061 * associated with it.
2133 */ 2062 */
2134 static struct reloc *find_jump_table(struct o 2063 static struct reloc *find_jump_table(struct objtool_file *file,
2135 struct 2064 struct symbol *func,
2136 struct 2065 struct instruction *insn)
2137 { 2066 {
2138 struct reloc *table_reloc; 2067 struct reloc *table_reloc;
2139 struct instruction *dest_insn, *orig_ 2068 struct instruction *dest_insn, *orig_insn = insn;
2140 2069
2141 /* 2070 /*
2142 * Backward search using the @first_j 2071 * Backward search using the @first_jump_src links, these help avoid
2143 * much of the 'in between' code. Whi 2072 * much of the 'in between' code. Which avoids us getting confused by
2144 * it. 2073 * it.
2145 */ 2074 */
2146 for (; 2075 for (;
2147 insn && insn_func(insn) && insn_ 2076 insn && insn_func(insn) && insn_func(insn)->pfunc == func;
2148 insn = insn->first_jump_src ?: p 2077 insn = insn->first_jump_src ?: prev_insn_same_sym(file, insn)) {
2149 2078
2150 if (insn != orig_insn && insn 2079 if (insn != orig_insn && insn->type == INSN_JUMP_DYNAMIC)
2151 break; 2080 break;
2152 2081
2153 /* allow small jumps within t 2082 /* allow small jumps within the range */
2154 if (insn->type == INSN_JUMP_U 2083 if (insn->type == INSN_JUMP_UNCONDITIONAL &&
2155 insn->jump_dest && 2084 insn->jump_dest &&
2156 (insn->jump_dest->offset 2085 (insn->jump_dest->offset <= insn->offset ||
2157 insn->jump_dest->offset 2086 insn->jump_dest->offset > orig_insn->offset))
2158 break; 2087 break;
2159 2088
2160 table_reloc = arch_find_switc 2089 table_reloc = arch_find_switch_table(file, insn);
2161 if (!table_reloc) 2090 if (!table_reloc)
2162 continue; 2091 continue;
2163 dest_insn = find_insn(file, t 2092 dest_insn = find_insn(file, table_reloc->sym->sec, reloc_addend(table_reloc));
2164 if (!dest_insn || !insn_func( 2093 if (!dest_insn || !insn_func(dest_insn) || insn_func(dest_insn)->pfunc != func)
2165 continue; 2094 continue;
2166 2095
2167 return table_reloc; 2096 return table_reloc;
2168 } 2097 }
2169 2098
2170 return NULL; 2099 return NULL;
2171 } 2100 }
2172 2101
2173 /* 2102 /*
2174 * First pass: Mark the head of each jump tab 2103 * First pass: Mark the head of each jump table so that in the next pass,
2175 * we know when a given jump table ends and t 2104 * we know when a given jump table ends and the next one starts.
2176 */ 2105 */
2177 static void mark_func_jump_tables(struct objt 2106 static void mark_func_jump_tables(struct objtool_file *file,
2178 struct sy 2107 struct symbol *func)
2179 { 2108 {
2180 struct instruction *insn, *last = NUL 2109 struct instruction *insn, *last = NULL;
2181 struct reloc *reloc; 2110 struct reloc *reloc;
2182 2111
2183 func_for_each_insn(file, func, insn) 2112 func_for_each_insn(file, func, insn) {
2184 if (!last) 2113 if (!last)
2185 last = insn; 2114 last = insn;
2186 2115
2187 /* 2116 /*
2188 * Store back-pointers for un 2117 * Store back-pointers for unconditional forward jumps such
2189 * that find_jump_table() can 2118 * that find_jump_table() can back-track using those and
2190 * avoid some potentially con 2119 * avoid some potentially confusing code.
2191 */ 2120 */
2192 if (insn->type == INSN_JUMP_U 2121 if (insn->type == INSN_JUMP_UNCONDITIONAL && insn->jump_dest &&
2193 insn->offset > last->offs 2122 insn->offset > last->offset &&
2194 insn->jump_dest->offset > 2123 insn->jump_dest->offset > insn->offset &&
2195 !insn->jump_dest->first_j 2124 !insn->jump_dest->first_jump_src) {
2196 2125
2197 insn->jump_dest->firs 2126 insn->jump_dest->first_jump_src = insn;
2198 last = insn->jump_des 2127 last = insn->jump_dest;
2199 } 2128 }
2200 2129
2201 if (insn->type != INSN_JUMP_D 2130 if (insn->type != INSN_JUMP_DYNAMIC)
2202 continue; 2131 continue;
2203 2132
2204 reloc = find_jump_table(file, 2133 reloc = find_jump_table(file, func, insn);
2205 if (reloc) 2134 if (reloc)
2206 insn->_jump_table = r 2135 insn->_jump_table = reloc;
2207 } 2136 }
2208 } 2137 }
2209 2138
2210 static int add_func_jump_tables(struct objtoo 2139 static int add_func_jump_tables(struct objtool_file *file,
2211 struct symb 2140 struct symbol *func)
2212 { 2141 {
2213 struct instruction *insn, *insn_t1 = 2142 struct instruction *insn, *insn_t1 = NULL, *insn_t2;
2214 int ret = 0; 2143 int ret = 0;
2215 2144
2216 func_for_each_insn(file, func, insn) 2145 func_for_each_insn(file, func, insn) {
2217 if (!insn_jump_table(insn)) 2146 if (!insn_jump_table(insn))
2218 continue; 2147 continue;
2219 2148
2220 if (!insn_t1) { 2149 if (!insn_t1) {
2221 insn_t1 = insn; 2150 insn_t1 = insn;
2222 continue; 2151 continue;
2223 } 2152 }
2224 2153
2225 insn_t2 = insn; 2154 insn_t2 = insn;
2226 2155
2227 ret = add_jump_table(file, in 2156 ret = add_jump_table(file, insn_t1, insn_jump_table(insn_t2));
2228 if (ret) 2157 if (ret)
2229 return ret; 2158 return ret;
2230 2159
2231 insn_t1 = insn_t2; 2160 insn_t1 = insn_t2;
2232 } 2161 }
2233 2162
2234 if (insn_t1) 2163 if (insn_t1)
2235 ret = add_jump_table(file, in 2164 ret = add_jump_table(file, insn_t1, NULL);
2236 2165
2237 return ret; 2166 return ret;
2238 } 2167 }
2239 2168
2240 /* 2169 /*
2241 * For some switch statements, gcc generates 2170 * For some switch statements, gcc generates a jump table in the .rodata
2242 * section which contains a list of addresses 2171 * section which contains a list of addresses within the function to jump to.
2243 * This finds these jump tables and adds them 2172 * This finds these jump tables and adds them to the insn->alts lists.
2244 */ 2173 */
2245 static int add_jump_table_alts(struct objtool 2174 static int add_jump_table_alts(struct objtool_file *file)
2246 { 2175 {
2247 struct symbol *func; 2176 struct symbol *func;
2248 int ret; 2177 int ret;
2249 2178
2250 if (!file->rodata) 2179 if (!file->rodata)
2251 return 0; 2180 return 0;
2252 2181
2253 for_each_sym(file, func) { 2182 for_each_sym(file, func) {
2254 if (func->type != STT_FUNC) 2183 if (func->type != STT_FUNC)
2255 continue; 2184 continue;
2256 2185
2257 mark_func_jump_tables(file, f 2186 mark_func_jump_tables(file, func);
2258 ret = add_func_jump_tables(fi 2187 ret = add_func_jump_tables(file, func);
2259 if (ret) 2188 if (ret)
2260 return ret; 2189 return ret;
2261 } 2190 }
2262 2191
2263 return 0; 2192 return 0;
2264 } 2193 }
2265 2194
2266 static void set_func_state(struct cfi_state * 2195 static void set_func_state(struct cfi_state *state)
2267 { 2196 {
2268 state->cfa = initial_func_cfi.cfa; 2197 state->cfa = initial_func_cfi.cfa;
2269 memcpy(&state->regs, &initial_func_cf 2198 memcpy(&state->regs, &initial_func_cfi.regs,
2270 CFI_NUM_REGS * sizeof(struct c 2199 CFI_NUM_REGS * sizeof(struct cfi_reg));
2271 state->stack_size = initial_func_cfi. 2200 state->stack_size = initial_func_cfi.cfa.offset;
2272 state->type = UNWIND_HINT_TYPE_CALL; 2201 state->type = UNWIND_HINT_TYPE_CALL;
2273 } 2202 }
2274 2203
2275 static int read_unwind_hints(struct objtool_f 2204 static int read_unwind_hints(struct objtool_file *file)
2276 { 2205 {
2277 struct cfi_state cfi = init_cfi; 2206 struct cfi_state cfi = init_cfi;
2278 struct section *sec; 2207 struct section *sec;
2279 struct unwind_hint *hint; 2208 struct unwind_hint *hint;
2280 struct instruction *insn; 2209 struct instruction *insn;
2281 struct reloc *reloc; 2210 struct reloc *reloc;
2282 unsigned long offset; <<
2283 int i; 2211 int i;
2284 2212
2285 sec = find_section_by_name(file->elf, 2213 sec = find_section_by_name(file->elf, ".discard.unwind_hints");
2286 if (!sec) 2214 if (!sec)
2287 return 0; 2215 return 0;
2288 2216
2289 if (!sec->rsec) { 2217 if (!sec->rsec) {
2290 WARN("missing .rela.discard.u 2218 WARN("missing .rela.discard.unwind_hints section");
2291 return -1; 2219 return -1;
2292 } 2220 }
2293 2221
2294 if (sec->sh.sh_size % sizeof(struct u 2222 if (sec->sh.sh_size % sizeof(struct unwind_hint)) {
2295 WARN("struct unwind_hint size 2223 WARN("struct unwind_hint size mismatch");
2296 return -1; 2224 return -1;
2297 } 2225 }
2298 2226
2299 file->hints = true; 2227 file->hints = true;
2300 2228
2301 for (i = 0; i < sec->sh.sh_size / siz 2229 for (i = 0; i < sec->sh.sh_size / sizeof(struct unwind_hint); i++) {
2302 hint = (struct unwind_hint *) 2230 hint = (struct unwind_hint *)sec->data->d_buf + i;
2303 2231
2304 reloc = find_reloc_by_dest(fi 2232 reloc = find_reloc_by_dest(file->elf, sec, i * sizeof(*hint));
2305 if (!reloc) { 2233 if (!reloc) {
2306 WARN("can't find relo 2234 WARN("can't find reloc for unwind_hints[%d]", i);
2307 return -1; 2235 return -1;
2308 } 2236 }
2309 2237
2310 if (reloc->sym->type == STT_S !! 2238 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2311 offset = reloc_addend <<
2312 } else if (reloc->sym->local_ <<
2313 offset = reloc->sym-> <<
2314 } else { <<
2315 WARN("unexpected relo <<
2316 return -1; <<
2317 } <<
2318 <<
2319 insn = find_insn(file, reloc- <<
2320 if (!insn) { 2239 if (!insn) {
2321 WARN("can't find insn 2240 WARN("can't find insn for unwind_hints[%d]", i);
2322 return -1; 2241 return -1;
2323 } 2242 }
2324 2243
2325 insn->hint = true; 2244 insn->hint = true;
2326 2245
2327 if (hint->type == UNWIND_HINT 2246 if (hint->type == UNWIND_HINT_TYPE_UNDEFINED) {
2328 insn->cfi = &force_un 2247 insn->cfi = &force_undefined_cfi;
2329 continue; 2248 continue;
2330 } 2249 }
2331 2250
2332 if (hint->type == UNWIND_HINT 2251 if (hint->type == UNWIND_HINT_TYPE_SAVE) {
2333 insn->hint = false; 2252 insn->hint = false;
2334 insn->save = true; 2253 insn->save = true;
2335 continue; 2254 continue;
2336 } 2255 }
2337 2256
2338 if (hint->type == UNWIND_HINT 2257 if (hint->type == UNWIND_HINT_TYPE_RESTORE) {
2339 insn->restore = true; 2258 insn->restore = true;
2340 continue; 2259 continue;
2341 } 2260 }
2342 2261
2343 if (hint->type == UNWIND_HINT 2262 if (hint->type == UNWIND_HINT_TYPE_REGS_PARTIAL) {
2344 struct symbol *sym = 2263 struct symbol *sym = find_symbol_by_offset(insn->sec, insn->offset);
2345 2264
2346 if (sym && sym->bind 2265 if (sym && sym->bind == STB_GLOBAL) {
2347 if (opts.ibt 2266 if (opts.ibt && insn->type != INSN_ENDBR && !insn->noendbr) {
2348 WARN_ 2267 WARN_INSN(insn, "UNWIND_HINT_IRET_REGS without ENDBR");
2349 } 2268 }
2350 } 2269 }
2351 } 2270 }
2352 2271
2353 if (hint->type == UNWIND_HINT 2272 if (hint->type == UNWIND_HINT_TYPE_FUNC) {
2354 insn->cfi = &func_cfi 2273 insn->cfi = &func_cfi;
2355 continue; 2274 continue;
2356 } 2275 }
2357 2276
2358 if (insn->cfi) 2277 if (insn->cfi)
2359 cfi = *(insn->cfi); 2278 cfi = *(insn->cfi);
2360 2279
2361 if (arch_decode_hint_reg(hint 2280 if (arch_decode_hint_reg(hint->sp_reg, &cfi.cfa.base)) {
2362 WARN_INSN(insn, "unsu 2281 WARN_INSN(insn, "unsupported unwind_hint sp base reg %d", hint->sp_reg);
2363 return -1; 2282 return -1;
2364 } 2283 }
2365 2284
2366 cfi.cfa.offset = bswap_if_nee 2285 cfi.cfa.offset = bswap_if_needed(file->elf, hint->sp_offset);
2367 cfi.type = hint->type; 2286 cfi.type = hint->type;
2368 cfi.signal = hint->signal; 2287 cfi.signal = hint->signal;
2369 2288
2370 insn->cfi = cfi_hash_find_or_ 2289 insn->cfi = cfi_hash_find_or_add(&cfi);
2371 } 2290 }
2372 2291
2373 return 0; 2292 return 0;
2374 } 2293 }
2375 2294
2376 static int read_noendbr_hints(struct objtool_ 2295 static int read_noendbr_hints(struct objtool_file *file)
2377 { 2296 {
2378 struct instruction *insn; 2297 struct instruction *insn;
2379 struct section *rsec; 2298 struct section *rsec;
2380 struct reloc *reloc; 2299 struct reloc *reloc;
2381 2300
2382 rsec = find_section_by_name(file->elf 2301 rsec = find_section_by_name(file->elf, ".rela.discard.noendbr");
2383 if (!rsec) 2302 if (!rsec)
2384 return 0; 2303 return 0;
2385 2304
2386 for_each_reloc(rsec, reloc) { 2305 for_each_reloc(rsec, reloc) {
2387 insn = find_insn(file, reloc- 2306 insn = find_insn(file, reloc->sym->sec,
2388 reloc->sym-> 2307 reloc->sym->offset + reloc_addend(reloc));
2389 if (!insn) { 2308 if (!insn) {
2390 WARN("bad .discard.no 2309 WARN("bad .discard.noendbr entry");
2391 return -1; 2310 return -1;
2392 } 2311 }
2393 2312
2394 insn->noendbr = 1; 2313 insn->noendbr = 1;
2395 } 2314 }
2396 2315
2397 return 0; 2316 return 0;
2398 } 2317 }
2399 2318
2400 static int read_retpoline_hints(struct objtoo 2319 static int read_retpoline_hints(struct objtool_file *file)
2401 { 2320 {
2402 struct section *rsec; 2321 struct section *rsec;
2403 struct instruction *insn; 2322 struct instruction *insn;
2404 struct reloc *reloc; 2323 struct reloc *reloc;
2405 2324
2406 rsec = find_section_by_name(file->elf 2325 rsec = find_section_by_name(file->elf, ".rela.discard.retpoline_safe");
2407 if (!rsec) 2326 if (!rsec)
2408 return 0; 2327 return 0;
2409 2328
2410 for_each_reloc(rsec, reloc) { 2329 for_each_reloc(rsec, reloc) {
2411 if (reloc->sym->type != STT_S 2330 if (reloc->sym->type != STT_SECTION) {
2412 WARN("unexpected relo 2331 WARN("unexpected relocation symbol type in %s", rsec->name);
2413 return -1; 2332 return -1;
2414 } 2333 }
2415 2334
2416 insn = find_insn(file, reloc- 2335 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2417 if (!insn) { 2336 if (!insn) {
2418 WARN("bad .discard.re 2337 WARN("bad .discard.retpoline_safe entry");
2419 return -1; 2338 return -1;
2420 } 2339 }
2421 2340
2422 if (insn->type != INSN_JUMP_D 2341 if (insn->type != INSN_JUMP_DYNAMIC &&
2423 insn->type != INSN_CALL_D 2342 insn->type != INSN_CALL_DYNAMIC &&
2424 insn->type != INSN_RETURN 2343 insn->type != INSN_RETURN &&
2425 insn->type != INSN_NOP) { 2344 insn->type != INSN_NOP) {
2426 WARN_INSN(insn, "retp 2345 WARN_INSN(insn, "retpoline_safe hint not an indirect jump/call/ret/nop");
2427 return -1; 2346 return -1;
2428 } 2347 }
2429 2348
2430 insn->retpoline_safe = true; 2349 insn->retpoline_safe = true;
2431 } 2350 }
2432 2351
2433 return 0; 2352 return 0;
2434 } 2353 }
2435 2354
2436 static int read_instr_hints(struct objtool_fi 2355 static int read_instr_hints(struct objtool_file *file)
2437 { 2356 {
2438 struct section *rsec; 2357 struct section *rsec;
2439 struct instruction *insn; 2358 struct instruction *insn;
2440 struct reloc *reloc; 2359 struct reloc *reloc;
2441 2360
2442 rsec = find_section_by_name(file->elf 2361 rsec = find_section_by_name(file->elf, ".rela.discard.instr_end");
2443 if (!rsec) 2362 if (!rsec)
2444 return 0; 2363 return 0;
2445 2364
2446 for_each_reloc(rsec, reloc) { 2365 for_each_reloc(rsec, reloc) {
2447 if (reloc->sym->type != STT_S 2366 if (reloc->sym->type != STT_SECTION) {
2448 WARN("unexpected relo 2367 WARN("unexpected relocation symbol type in %s", rsec->name);
2449 return -1; 2368 return -1;
2450 } 2369 }
2451 2370
2452 insn = find_insn(file, reloc- 2371 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2453 if (!insn) { 2372 if (!insn) {
2454 WARN("bad .discard.in 2373 WARN("bad .discard.instr_end entry");
2455 return -1; 2374 return -1;
2456 } 2375 }
2457 2376
2458 insn->instr--; 2377 insn->instr--;
2459 } 2378 }
2460 2379
2461 rsec = find_section_by_name(file->elf 2380 rsec = find_section_by_name(file->elf, ".rela.discard.instr_begin");
2462 if (!rsec) 2381 if (!rsec)
2463 return 0; 2382 return 0;
2464 2383
2465 for_each_reloc(rsec, reloc) { 2384 for_each_reloc(rsec, reloc) {
2466 if (reloc->sym->type != STT_S 2385 if (reloc->sym->type != STT_SECTION) {
2467 WARN("unexpected relo 2386 WARN("unexpected relocation symbol type in %s", rsec->name);
2468 return -1; 2387 return -1;
2469 } 2388 }
2470 2389
2471 insn = find_insn(file, reloc- 2390 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2472 if (!insn) { 2391 if (!insn) {
2473 WARN("bad .discard.in 2392 WARN("bad .discard.instr_begin entry");
2474 return -1; 2393 return -1;
2475 } 2394 }
2476 2395
2477 insn->instr++; 2396 insn->instr++;
2478 } 2397 }
2479 2398
2480 return 0; 2399 return 0;
2481 } 2400 }
2482 2401
2483 static int read_validate_unret_hints(struct o 2402 static int read_validate_unret_hints(struct objtool_file *file)
2484 { 2403 {
2485 struct section *rsec; 2404 struct section *rsec;
2486 struct instruction *insn; 2405 struct instruction *insn;
2487 struct reloc *reloc; 2406 struct reloc *reloc;
2488 2407
2489 rsec = find_section_by_name(file->elf 2408 rsec = find_section_by_name(file->elf, ".rela.discard.validate_unret");
2490 if (!rsec) 2409 if (!rsec)
2491 return 0; 2410 return 0;
2492 2411
2493 for_each_reloc(rsec, reloc) { 2412 for_each_reloc(rsec, reloc) {
2494 if (reloc->sym->type != STT_S 2413 if (reloc->sym->type != STT_SECTION) {
2495 WARN("unexpected relo 2414 WARN("unexpected relocation symbol type in %s", rsec->name);
2496 return -1; 2415 return -1;
2497 } 2416 }
2498 2417
2499 insn = find_insn(file, reloc- 2418 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2500 if (!insn) { 2419 if (!insn) {
2501 WARN("bad .discard.in 2420 WARN("bad .discard.instr_end entry");
2502 return -1; 2421 return -1;
2503 } 2422 }
2504 insn->unret = 1; 2423 insn->unret = 1;
2505 } 2424 }
2506 2425
2507 return 0; 2426 return 0;
2508 } 2427 }
2509 2428
2510 2429
2511 static int read_intra_function_calls(struct o 2430 static int read_intra_function_calls(struct objtool_file *file)
2512 { 2431 {
2513 struct instruction *insn; 2432 struct instruction *insn;
2514 struct section *rsec; 2433 struct section *rsec;
2515 struct reloc *reloc; 2434 struct reloc *reloc;
2516 2435
2517 rsec = find_section_by_name(file->elf 2436 rsec = find_section_by_name(file->elf, ".rela.discard.intra_function_calls");
2518 if (!rsec) 2437 if (!rsec)
2519 return 0; 2438 return 0;
2520 2439
2521 for_each_reloc(rsec, reloc) { 2440 for_each_reloc(rsec, reloc) {
2522 unsigned long dest_off; 2441 unsigned long dest_off;
2523 2442
2524 if (reloc->sym->type != STT_S 2443 if (reloc->sym->type != STT_SECTION) {
2525 WARN("unexpected relo 2444 WARN("unexpected relocation symbol type in %s",
2526 rsec->name); 2445 rsec->name);
2527 return -1; 2446 return -1;
2528 } 2447 }
2529 2448
2530 insn = find_insn(file, reloc- 2449 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2531 if (!insn) { 2450 if (!insn) {
2532 WARN("bad .discard.in 2451 WARN("bad .discard.intra_function_call entry");
2533 return -1; 2452 return -1;
2534 } 2453 }
2535 2454
2536 if (insn->type != INSN_CALL) 2455 if (insn->type != INSN_CALL) {
2537 WARN_INSN(insn, "intr 2456 WARN_INSN(insn, "intra_function_call not a direct call");
2538 return -1; 2457 return -1;
2539 } 2458 }
2540 2459
2541 /* 2460 /*
2542 * Treat intra-function CALLs 2461 * Treat intra-function CALLs as JMPs, but with a stack_op.
2543 * See add_call_destinations( 2462 * See add_call_destinations(), which strips stack_ops from
2544 * normal CALLs. 2463 * normal CALLs.
2545 */ 2464 */
2546 insn->type = INSN_JUMP_UNCOND 2465 insn->type = INSN_JUMP_UNCONDITIONAL;
2547 2466
2548 dest_off = arch_jump_destinat 2467 dest_off = arch_jump_destination(insn);
2549 insn->jump_dest = find_insn(f 2468 insn->jump_dest = find_insn(file, insn->sec, dest_off);
2550 if (!insn->jump_dest) { 2469 if (!insn->jump_dest) {
2551 WARN_INSN(insn, "can' 2470 WARN_INSN(insn, "can't find call dest at %s+0x%lx",
2552 insn->sec-> 2471 insn->sec->name, dest_off);
2553 return -1; 2472 return -1;
2554 } 2473 }
2555 } 2474 }
2556 2475
2557 return 0; 2476 return 0;
2558 } 2477 }
2559 2478
2560 /* 2479 /*
2561 * Return true if name matches an instrumenta 2480 * Return true if name matches an instrumentation function, where calls to that
2562 * function from noinstr code can safely be r 2481 * function from noinstr code can safely be removed, but compilers won't do so.
2563 */ 2482 */
2564 static bool is_profiling_func(const char *nam 2483 static bool is_profiling_func(const char *name)
2565 { 2484 {
2566 /* 2485 /*
2567 * Many compilers cannot disable KCOV 2486 * Many compilers cannot disable KCOV with a function attribute.
2568 */ 2487 */
2569 if (!strncmp(name, "__sanitizer_cov_" 2488 if (!strncmp(name, "__sanitizer_cov_", 16))
2570 return true; 2489 return true;
2571 2490
2572 /* 2491 /*
2573 * Some compilers currently do not re 2492 * Some compilers currently do not remove __tsan_func_entry/exit nor
2574 * __tsan_atomic_signal_fence (used f 2493 * __tsan_atomic_signal_fence (used for barrier instrumentation) with
2575 * the __no_sanitize_thread attribute 2494 * the __no_sanitize_thread attribute, remove them. Once the kernel's
2576 * minimum Clang version is 14.0, thi 2495 * minimum Clang version is 14.0, this can be removed.
2577 */ 2496 */
2578 if (!strncmp(name, "__tsan_func_", 12 2497 if (!strncmp(name, "__tsan_func_", 12) ||
2579 !strcmp(name, "__tsan_atomic_sign 2498 !strcmp(name, "__tsan_atomic_signal_fence"))
2580 return true; 2499 return true;
2581 2500
2582 return false; 2501 return false;
2583 } 2502 }
2584 2503
2585 static int classify_symbols(struct objtool_fi 2504 static int classify_symbols(struct objtool_file *file)
2586 { 2505 {
2587 struct symbol *func; 2506 struct symbol *func;
2588 2507
2589 for_each_sym(file, func) { 2508 for_each_sym(file, func) {
2590 if (func->type == STT_NOTYPE <<
2591 func->local_label = t <<
2592 <<
2593 if (func->bind != STB_GLOBAL) 2509 if (func->bind != STB_GLOBAL)
2594 continue; 2510 continue;
2595 2511
2596 if (!strncmp(func->name, STAT 2512 if (!strncmp(func->name, STATIC_CALL_TRAMP_PREFIX_STR,
2597 strlen(STATIC_CA 2513 strlen(STATIC_CALL_TRAMP_PREFIX_STR)))
2598 func->static_call_tra 2514 func->static_call_tramp = true;
2599 2515
2600 if (arch_is_retpoline(func)) 2516 if (arch_is_retpoline(func))
2601 func->retpoline_thunk 2517 func->retpoline_thunk = true;
2602 2518
2603 if (arch_is_rethunk(func)) 2519 if (arch_is_rethunk(func))
2604 func->return_thunk = 2520 func->return_thunk = true;
2605 2521
2606 if (arch_is_embedded_insn(fun 2522 if (arch_is_embedded_insn(func))
2607 func->embedded_insn = 2523 func->embedded_insn = true;
2608 2524
2609 if (arch_ftrace_match(func->n 2525 if (arch_ftrace_match(func->name))
2610 func->fentry = true; 2526 func->fentry = true;
2611 2527
2612 if (is_profiling_func(func->n 2528 if (is_profiling_func(func->name))
2613 func->profiling_func 2529 func->profiling_func = true;
2614 } 2530 }
2615 2531
2616 return 0; 2532 return 0;
2617 } 2533 }
2618 2534
2619 static void mark_rodata(struct objtool_file * 2535 static void mark_rodata(struct objtool_file *file)
2620 { 2536 {
2621 struct section *sec; 2537 struct section *sec;
2622 bool found = false; 2538 bool found = false;
2623 2539
2624 /* 2540 /*
2625 * Search for the following rodata se 2541 * Search for the following rodata sections, each of which can
2626 * potentially contain jump tables: 2542 * potentially contain jump tables:
2627 * 2543 *
2628 * - .rodata: can contain GCC switch 2544 * - .rodata: can contain GCC switch tables
2629 * - .rodata.<func>: same, if -fdata- 2545 * - .rodata.<func>: same, if -fdata-sections is being used
2630 * - .rodata..c_jump_table: contains 2546 * - .rodata..c_jump_table: contains C annotated jump tables
2631 * 2547 *
2632 * .rodata.str1.* sections are ignore 2548 * .rodata.str1.* sections are ignored; they don't contain jump tables.
2633 */ 2549 */
2634 for_each_sec(file, sec) { 2550 for_each_sec(file, sec) {
2635 if (!strncmp(sec->name, ".rod 2551 if (!strncmp(sec->name, ".rodata", 7) &&
2636 !strstr(sec->name, ".str1 2552 !strstr(sec->name, ".str1.")) {
2637 sec->rodata = true; 2553 sec->rodata = true;
2638 found = true; 2554 found = true;
2639 } 2555 }
2640 } 2556 }
2641 2557
2642 file->rodata = found; 2558 file->rodata = found;
2643 } 2559 }
2644 2560
2645 static int decode_sections(struct objtool_fil 2561 static int decode_sections(struct objtool_file *file)
2646 { 2562 {
2647 int ret; 2563 int ret;
2648 2564
2649 mark_rodata(file); 2565 mark_rodata(file);
2650 2566
2651 ret = init_pv_ops(file); 2567 ret = init_pv_ops(file);
2652 if (ret) 2568 if (ret)
2653 return ret; 2569 return ret;
2654 2570
2655 /* 2571 /*
2656 * Must be before add_{jump_call}_des 2572 * Must be before add_{jump_call}_destination.
2657 */ 2573 */
2658 ret = classify_symbols(file); 2574 ret = classify_symbols(file);
2659 if (ret) 2575 if (ret)
2660 return ret; 2576 return ret;
2661 2577
2662 ret = decode_instructions(file); 2578 ret = decode_instructions(file);
2663 if (ret) 2579 if (ret)
2664 return ret; 2580 return ret;
2665 2581
2666 add_ignores(file); 2582 add_ignores(file);
2667 add_uaccess_safe(file); 2583 add_uaccess_safe(file);
2668 2584
2669 ret = add_ignore_alternatives(file); 2585 ret = add_ignore_alternatives(file);
2670 if (ret) 2586 if (ret)
2671 return ret; 2587 return ret;
2672 2588
2673 /* 2589 /*
2674 * Must be before read_unwind_hints() 2590 * Must be before read_unwind_hints() since that needs insn->noendbr.
2675 */ 2591 */
2676 ret = read_noendbr_hints(file); 2592 ret = read_noendbr_hints(file);
2677 if (ret) 2593 if (ret)
2678 return ret; 2594 return ret;
2679 2595
2680 /* 2596 /*
2681 * Must be before add_jump_destinatio 2597 * Must be before add_jump_destinations(), which depends on 'func'
2682 * being set for alternatives, to ena 2598 * being set for alternatives, to enable proper sibling call detection.
2683 */ 2599 */
2684 if (opts.stackval || opts.orc || opts 2600 if (opts.stackval || opts.orc || opts.uaccess || opts.noinstr) {
2685 ret = add_special_section_alt 2601 ret = add_special_section_alts(file);
2686 if (ret) 2602 if (ret)
2687 return ret; 2603 return ret;
2688 } 2604 }
2689 2605
2690 ret = add_jump_destinations(file); 2606 ret = add_jump_destinations(file);
2691 if (ret) 2607 if (ret)
2692 return ret; 2608 return ret;
2693 2609
2694 /* 2610 /*
2695 * Must be before add_call_destinatio 2611 * Must be before add_call_destination(); it changes INSN_CALL to
2696 * INSN_JUMP. 2612 * INSN_JUMP.
2697 */ 2613 */
2698 ret = read_intra_function_calls(file) 2614 ret = read_intra_function_calls(file);
2699 if (ret) 2615 if (ret)
2700 return ret; 2616 return ret;
2701 2617
2702 ret = add_call_destinations(file); 2618 ret = add_call_destinations(file);
2703 if (ret) 2619 if (ret)
2704 return ret; 2620 return ret;
2705 2621
2706 /* 2622 /*
2707 * Must be after add_call_destination 2623 * Must be after add_call_destinations() such that it can override
2708 * dead_end_function() marks. 2624 * dead_end_function() marks.
2709 */ 2625 */
2710 ret = add_dead_ends(file); 2626 ret = add_dead_ends(file);
2711 if (ret) 2627 if (ret)
2712 return ret; 2628 return ret;
2713 2629
2714 ret = add_jump_table_alts(file); 2630 ret = add_jump_table_alts(file);
2715 if (ret) 2631 if (ret)
2716 return ret; 2632 return ret;
2717 2633
2718 ret = read_unwind_hints(file); 2634 ret = read_unwind_hints(file);
2719 if (ret) 2635 if (ret)
2720 return ret; 2636 return ret;
2721 2637
2722 ret = read_retpoline_hints(file); 2638 ret = read_retpoline_hints(file);
2723 if (ret) 2639 if (ret)
2724 return ret; 2640 return ret;
2725 2641
2726 ret = read_instr_hints(file); 2642 ret = read_instr_hints(file);
2727 if (ret) 2643 if (ret)
2728 return ret; 2644 return ret;
2729 2645
2730 ret = read_validate_unret_hints(file) 2646 ret = read_validate_unret_hints(file);
2731 if (ret) 2647 if (ret)
2732 return ret; 2648 return ret;
2733 2649
2734 return 0; 2650 return 0;
2735 } 2651 }
2736 2652
2737 static bool is_special_call(struct instructio 2653 static bool is_special_call(struct instruction *insn)
2738 { 2654 {
2739 if (insn->type == INSN_CALL) { 2655 if (insn->type == INSN_CALL) {
2740 struct symbol *dest = insn_ca 2656 struct symbol *dest = insn_call_dest(insn);
2741 2657
2742 if (!dest) 2658 if (!dest)
2743 return false; 2659 return false;
2744 2660
2745 if (dest->fentry || dest->emb 2661 if (dest->fentry || dest->embedded_insn)
2746 return true; 2662 return true;
2747 } 2663 }
2748 2664
2749 return false; 2665 return false;
2750 } 2666 }
2751 2667
2752 static bool has_modified_stack_frame(struct i 2668 static bool has_modified_stack_frame(struct instruction *insn, struct insn_state *state)
2753 { 2669 {
2754 struct cfi_state *cfi = &state->cfi; 2670 struct cfi_state *cfi = &state->cfi;
2755 int i; 2671 int i;
2756 2672
2757 if (cfi->cfa.base != initial_func_cfi 2673 if (cfi->cfa.base != initial_func_cfi.cfa.base || cfi->drap)
2758 return true; 2674 return true;
2759 2675
2760 if (cfi->cfa.offset != initial_func_c 2676 if (cfi->cfa.offset != initial_func_cfi.cfa.offset)
2761 return true; 2677 return true;
2762 2678
2763 if (cfi->stack_size != initial_func_c 2679 if (cfi->stack_size != initial_func_cfi.cfa.offset)
2764 return true; 2680 return true;
2765 2681
2766 for (i = 0; i < CFI_NUM_REGS; i++) { 2682 for (i = 0; i < CFI_NUM_REGS; i++) {
2767 if (cfi->regs[i].base != init 2683 if (cfi->regs[i].base != initial_func_cfi.regs[i].base ||
2768 cfi->regs[i].offset != in 2684 cfi->regs[i].offset != initial_func_cfi.regs[i].offset)
2769 return true; 2685 return true;
2770 } 2686 }
2771 2687
2772 return false; 2688 return false;
2773 } 2689 }
2774 2690
2775 static bool check_reg_frame_pos(const struct 2691 static bool check_reg_frame_pos(const struct cfi_reg *reg,
2776 int expected_ 2692 int expected_offset)
2777 { 2693 {
2778 return reg->base == CFI_CFA && 2694 return reg->base == CFI_CFA &&
2779 reg->offset == expected_offset 2695 reg->offset == expected_offset;
2780 } 2696 }
2781 2697
2782 static bool has_valid_stack_frame(struct insn 2698 static bool has_valid_stack_frame(struct insn_state *state)
2783 { 2699 {
2784 struct cfi_state *cfi = &state->cfi; 2700 struct cfi_state *cfi = &state->cfi;
2785 2701
2786 if (cfi->cfa.base == CFI_BP && 2702 if (cfi->cfa.base == CFI_BP &&
2787 check_reg_frame_pos(&cfi->regs[CF 2703 check_reg_frame_pos(&cfi->regs[CFI_BP], -cfi->cfa.offset) &&
2788 check_reg_frame_pos(&cfi->regs[CF 2704 check_reg_frame_pos(&cfi->regs[CFI_RA], -cfi->cfa.offset + 8))
2789 return true; 2705 return true;
2790 2706
2791 if (cfi->drap && cfi->regs[CFI_BP].ba 2707 if (cfi->drap && cfi->regs[CFI_BP].base == CFI_BP)
2792 return true; 2708 return true;
2793 2709
2794 return false; 2710 return false;
2795 } 2711 }
2796 2712
2797 static int update_cfi_state_regs(struct instr 2713 static int update_cfi_state_regs(struct instruction *insn,
2798 struct cfi_ 2714 struct cfi_state *cfi,
2799 struct stac 2715 struct stack_op *op)
2800 { 2716 {
2801 struct cfi_reg *cfa = &cfi->cfa; 2717 struct cfi_reg *cfa = &cfi->cfa;
2802 2718
2803 if (cfa->base != CFI_SP && cfa->base 2719 if (cfa->base != CFI_SP && cfa->base != CFI_SP_INDIRECT)
2804 return 0; 2720 return 0;
2805 2721
2806 /* push */ 2722 /* push */
2807 if (op->dest.type == OP_DEST_PUSH || 2723 if (op->dest.type == OP_DEST_PUSH || op->dest.type == OP_DEST_PUSHF)
2808 cfa->offset += 8; 2724 cfa->offset += 8;
2809 2725
2810 /* pop */ 2726 /* pop */
2811 if (op->src.type == OP_SRC_POP || op- 2727 if (op->src.type == OP_SRC_POP || op->src.type == OP_SRC_POPF)
2812 cfa->offset -= 8; 2728 cfa->offset -= 8;
2813 2729
2814 /* add immediate to sp */ 2730 /* add immediate to sp */
2815 if (op->dest.type == OP_DEST_REG && o 2731 if (op->dest.type == OP_DEST_REG && op->src.type == OP_SRC_ADD &&
2816 op->dest.reg == CFI_SP && op->src 2732 op->dest.reg == CFI_SP && op->src.reg == CFI_SP)
2817 cfa->offset -= op->src.offset 2733 cfa->offset -= op->src.offset;
2818 2734
2819 return 0; 2735 return 0;
2820 } 2736 }
2821 2737
2822 static void save_reg(struct cfi_state *cfi, u 2738 static void save_reg(struct cfi_state *cfi, unsigned char reg, int base, int offset)
2823 { 2739 {
2824 if (arch_callee_saved_reg(reg) && 2740 if (arch_callee_saved_reg(reg) &&
2825 cfi->regs[reg].base == CFI_UNDEFI 2741 cfi->regs[reg].base == CFI_UNDEFINED) {
2826 cfi->regs[reg].base = base; 2742 cfi->regs[reg].base = base;
2827 cfi->regs[reg].offset = offse 2743 cfi->regs[reg].offset = offset;
2828 } 2744 }
2829 } 2745 }
2830 2746
2831 static void restore_reg(struct cfi_state *cfi 2747 static void restore_reg(struct cfi_state *cfi, unsigned char reg)
2832 { 2748 {
2833 cfi->regs[reg].base = initial_func_cf 2749 cfi->regs[reg].base = initial_func_cfi.regs[reg].base;
2834 cfi->regs[reg].offset = initial_func_ 2750 cfi->regs[reg].offset = initial_func_cfi.regs[reg].offset;
2835 } 2751 }
2836 2752
2837 /* 2753 /*
2838 * A note about DRAP stack alignment: 2754 * A note about DRAP stack alignment:
2839 * 2755 *
2840 * GCC has the concept of a DRAP register, wh 2756 * GCC has the concept of a DRAP register, which is used to help keep track of
2841 * the stack pointer when aligning the stack. 2757 * the stack pointer when aligning the stack. r10 or r13 is used as the DRAP
2842 * register. The typical DRAP pattern is: 2758 * register. The typical DRAP pattern is:
2843 * 2759 *
2844 * 4c 8d 54 24 08 lea 0x8(%r 2760 * 4c 8d 54 24 08 lea 0x8(%rsp),%r10
2845 * 48 83 e4 c0 and $0xfff 2761 * 48 83 e4 c0 and $0xffffffffffffffc0,%rsp
2846 * 41 ff 72 f8 pushq -0x8(% 2762 * 41 ff 72 f8 pushq -0x8(%r10)
2847 * 55 push %rbp 2763 * 55 push %rbp
2848 * 48 89 e5 mov %rsp,% 2764 * 48 89 e5 mov %rsp,%rbp
2849 * (more pushes) 2765 * (more pushes)
2850 * 41 52 push %r10 2766 * 41 52 push %r10
2851 * ... 2767 * ...
2852 * 41 5a pop %r10 2768 * 41 5a pop %r10
2853 * (more pops) 2769 * (more pops)
2854 * 5d pop %rbp 2770 * 5d pop %rbp
2855 * 49 8d 62 f8 lea -0x8(% 2771 * 49 8d 62 f8 lea -0x8(%r10),%rsp
2856 * c3 retq 2772 * c3 retq
2857 * 2773 *
2858 * There are some variations in the epilogues 2774 * There are some variations in the epilogues, like:
2859 * 2775 *
2860 * 5b pop %rbx 2776 * 5b pop %rbx
2861 * 41 5a pop %r10 2777 * 41 5a pop %r10
2862 * 41 5c pop %r12 2778 * 41 5c pop %r12
2863 * 41 5d pop %r13 2779 * 41 5d pop %r13
2864 * 41 5e pop %r14 2780 * 41 5e pop %r14
2865 * c9 leaveq 2781 * c9 leaveq
2866 * 49 8d 62 f8 lea -0x8(% 2782 * 49 8d 62 f8 lea -0x8(%r10),%rsp
2867 * c3 retq 2783 * c3 retq
2868 * 2784 *
2869 * and: 2785 * and:
2870 * 2786 *
2871 * 4c 8b 55 e8 mov -0x18( 2787 * 4c 8b 55 e8 mov -0x18(%rbp),%r10
2872 * 48 8b 5d e0 mov -0x20( 2788 * 48 8b 5d e0 mov -0x20(%rbp),%rbx
2873 * 4c 8b 65 f0 mov -0x10( 2789 * 4c 8b 65 f0 mov -0x10(%rbp),%r12
2874 * 4c 8b 6d f8 mov -0x8(% 2790 * 4c 8b 6d f8 mov -0x8(%rbp),%r13
2875 * c9 leaveq 2791 * c9 leaveq
2876 * 49 8d 62 f8 lea -0x8(% 2792 * 49 8d 62 f8 lea -0x8(%r10),%rsp
2877 * c3 retq 2793 * c3 retq
2878 * 2794 *
2879 * Sometimes r13 is used as the DRAP register 2795 * Sometimes r13 is used as the DRAP register, in which case it's saved and
2880 * restored beforehand: 2796 * restored beforehand:
2881 * 2797 *
2882 * 41 55 push %r13 2798 * 41 55 push %r13
2883 * 4c 8d 6c 24 10 lea 0x10(% 2799 * 4c 8d 6c 24 10 lea 0x10(%rsp),%r13
2884 * 48 83 e4 f0 and $0xfff 2800 * 48 83 e4 f0 and $0xfffffffffffffff0,%rsp
2885 * ... 2801 * ...
2886 * 49 8d 65 f0 lea -0x10( 2802 * 49 8d 65 f0 lea -0x10(%r13),%rsp
2887 * 41 5d pop %r13 2803 * 41 5d pop %r13
2888 * c3 retq 2804 * c3 retq
2889 */ 2805 */
2890 static int update_cfi_state(struct instructio 2806 static int update_cfi_state(struct instruction *insn,
2891 struct instructio 2807 struct instruction *next_insn,
2892 struct cfi_state 2808 struct cfi_state *cfi, struct stack_op *op)
2893 { 2809 {
2894 struct cfi_reg *cfa = &cfi->cfa; 2810 struct cfi_reg *cfa = &cfi->cfa;
2895 struct cfi_reg *regs = cfi->regs; 2811 struct cfi_reg *regs = cfi->regs;
2896 2812
2897 /* ignore UNWIND_HINT_UNDEFINED regio 2813 /* ignore UNWIND_HINT_UNDEFINED regions */
2898 if (cfi->force_undefined) 2814 if (cfi->force_undefined)
2899 return 0; 2815 return 0;
2900 2816
2901 /* stack operations don't make sense 2817 /* stack operations don't make sense with an undefined CFA */
2902 if (cfa->base == CFI_UNDEFINED) { 2818 if (cfa->base == CFI_UNDEFINED) {
2903 if (insn_func(insn)) { 2819 if (insn_func(insn)) {
2904 WARN_INSN(insn, "unde 2820 WARN_INSN(insn, "undefined stack state");
2905 return -1; 2821 return -1;
2906 } 2822 }
2907 return 0; 2823 return 0;
2908 } 2824 }
2909 2825
2910 if (cfi->type == UNWIND_HINT_TYPE_REG 2826 if (cfi->type == UNWIND_HINT_TYPE_REGS ||
2911 cfi->type == UNWIND_HINT_TYPE_REG 2827 cfi->type == UNWIND_HINT_TYPE_REGS_PARTIAL)
2912 return update_cfi_state_regs( 2828 return update_cfi_state_regs(insn, cfi, op);
2913 2829
2914 switch (op->dest.type) { 2830 switch (op->dest.type) {
2915 2831
2916 case OP_DEST_REG: 2832 case OP_DEST_REG:
2917 switch (op->src.type) { 2833 switch (op->src.type) {
2918 2834
2919 case OP_SRC_REG: 2835 case OP_SRC_REG:
2920 if (op->src.reg == CF 2836 if (op->src.reg == CFI_SP && op->dest.reg == CFI_BP &&
2921 cfa->base == CFI_ 2837 cfa->base == CFI_SP &&
2922 check_reg_frame_p 2838 check_reg_frame_pos(®s[CFI_BP], -cfa->offset)) {
2923 2839
2924 /* mov %rsp, 2840 /* mov %rsp, %rbp */
2925 cfa->base = o 2841 cfa->base = op->dest.reg;
2926 cfi->bp_scrat 2842 cfi->bp_scratch = false;
2927 } 2843 }
2928 2844
2929 else if (op->src.reg 2845 else if (op->src.reg == CFI_SP &&
2930 op->dest.reg 2846 op->dest.reg == CFI_BP && cfi->drap) {
2931 2847
2932 /* drap: mov 2848 /* drap: mov %rsp, %rbp */
2933 regs[CFI_BP]. 2849 regs[CFI_BP].base = CFI_BP;
2934 regs[CFI_BP]. 2850 regs[CFI_BP].offset = -cfi->stack_size;
2935 cfi->bp_scrat 2851 cfi->bp_scratch = false;
2936 } 2852 }
2937 2853
2938 else if (op->src.reg 2854 else if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
2939 2855
2940 /* 2856 /*
2941 * mov %rsp, 2857 * mov %rsp, %reg
2942 * 2858 *
2943 * This is ne 2859 * This is needed for the rare case where GCC
2944 * does: 2860 * does:
2945 * 2861 *
2946 * mov % 2862 * mov %rsp, %rax
2947 * ... 2863 * ...
2948 * mov % 2864 * mov %rax, %rsp
2949 */ 2865 */
2950 cfi->vals[op- 2866 cfi->vals[op->dest.reg].base = CFI_CFA;
2951 cfi->vals[op- 2867 cfi->vals[op->dest.reg].offset = -cfi->stack_size;
2952 } 2868 }
2953 2869
2954 else if (op->src.reg 2870 else if (op->src.reg == CFI_BP && op->dest.reg == CFI_SP &&
2955 (cfa->base = 2871 (cfa->base == CFI_BP || cfa->base == cfi->drap_reg)) {
2956 2872
2957 /* 2873 /*
2958 * mov %rbp, 2874 * mov %rbp, %rsp
2959 * 2875 *
2960 * Restore th 2876 * Restore the original stack pointer (Clang).
2961 */ 2877 */
2962 cfi->stack_si 2878 cfi->stack_size = -cfi->regs[CFI_BP].offset;
2963 } 2879 }
2964 2880
2965 else if (op->dest.reg 2881 else if (op->dest.reg == cfa->base) {
2966 2882
2967 /* mov %reg, 2883 /* mov %reg, %rsp */
2968 if (cfa->base 2884 if (cfa->base == CFI_SP &&
2969 cfi->vals 2885 cfi->vals[op->src.reg].base == CFI_CFA) {
2970 2886
2971 /* 2887 /*
2972 * Th 2888 * This is needed for the rare case
2973 * wh 2889 * where GCC does something dumb like:
2974 * 2890 *
2975 * 2891 * lea 0x8(%rsp), %rcx
2976 * 2892 * ...
2977 * 2893 * mov %rcx, %rsp
2978 */ 2894 */
2979 cfa-> 2895 cfa->offset = -cfi->vals[op->src.reg].offset;
2980 cfi-> 2896 cfi->stack_size = cfa->offset;
2981 2897
2982 } else if (cf 2898 } else if (cfa->base == CFI_SP &&
2983 cf 2899 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2984 cf 2900 cfi->vals[op->src.reg].offset == cfa->offset) {
2985 2901
2986 /* 2902 /*
2987 * St 2903 * Stack swizzle:
2988 * 2904 *
2989 * 1: 2905 * 1: mov %rsp, (%[tos])
2990 * 2: 2906 * 2: mov %[tos], %rsp
2991 * 2907 * ...
2992 * 3: 2908 * 3: pop %rsp
2993 * 2909 *
2994 * Wh 2910 * Where:
2995 * 2911 *
2996 * 1 2912 * 1 - places a pointer to the previous
2997 * 2913 * stack at the Top-of-Stack of the
2998 * 2914 * new stack.
2999 * 2915 *
3000 * 2 2916 * 2 - switches to the new stack.
3001 * 2917 *
3002 * 3 2918 * 3 - pops the Top-of-Stack to restore
3003 * 2919 * the original stack.
3004 * 2920 *
3005 * No 2921 * Note: we set base to SP_INDIRECT
3006 * he 2922 * here and preserve offset. Therefore
3007 * wh 2923 * when the unwinder reaches ToS it
3008 * wi 2924 * will dereference SP and then add the
3009 * of 2925 * offset to find the next frame, IOW:
3010 * (% 2926 * (%rsp) + offset.
3011 */ 2927 */
3012 cfa-> 2928 cfa->base = CFI_SP_INDIRECT;
3013 2929
3014 } else { 2930 } else {
3015 cfa-> 2931 cfa->base = CFI_UNDEFINED;
3016 cfa-> 2932 cfa->offset = 0;
3017 } 2933 }
3018 } 2934 }
3019 2935
3020 else if (op->dest.reg 2936 else if (op->dest.reg == CFI_SP &&
3021 cfi->vals[op 2937 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
3022 cfi->vals[op 2938 cfi->vals[op->src.reg].offset == cfa->offset) {
3023 2939
3024 /* 2940 /*
3025 * The same s 2941 * The same stack swizzle case 2) as above. But
3026 * because we 2942 * because we can't change cfa->base, case 3)
3027 * will becom 2943 * will become a regular POP. Pretend we're a
3028 * PUSH so th 2944 * PUSH so things don't go unbalanced.
3029 */ 2945 */
3030 cfi->stack_si 2946 cfi->stack_size += 8;
3031 } 2947 }
3032 2948
3033 2949
3034 break; 2950 break;
3035 2951
3036 case OP_SRC_ADD: 2952 case OP_SRC_ADD:
3037 if (op->dest.reg == C 2953 if (op->dest.reg == CFI_SP && op->src.reg == CFI_SP) {
3038 2954
3039 /* add imm, % 2955 /* add imm, %rsp */
3040 cfi->stack_si 2956 cfi->stack_size -= op->src.offset;
3041 if (cfa->base 2957 if (cfa->base == CFI_SP)
3042 cfa-> 2958 cfa->offset -= op->src.offset;
3043 break; 2959 break;
3044 } 2960 }
3045 2961
3046 if (op->dest.reg == C <<
3047 insn->sym->frame_ <<
3048 /* addi.d fp, <<
3049 if (cfa->base <<
3050 cfa-> <<
3051 cfa-> <<
3052 } <<
3053 break; <<
3054 } <<
3055 <<
3056 if (op->dest.reg == C 2962 if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) {
3057 /* addi.d sp, !! 2963
3058 if (cfa->base !! 2964 /* lea disp(%rbp), %rsp */
3059 if (i !! 2965 cfi->stack_size = -(op->src.offset + regs[CFI_BP].offset);
3060 <<
3061 <<
3062 } <<
3063 } else { <<
3064 /* le <<
3065 cfi-> <<
3066 } <<
3067 break; 2966 break;
3068 } 2967 }
3069 2968
3070 if (op->src.reg == CF 2969 if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
3071 2970
3072 /* drap: lea 2971 /* drap: lea disp(%rsp), %drap */
3073 cfi->drap_reg 2972 cfi->drap_reg = op->dest.reg;
3074 2973
3075 /* 2974 /*
3076 * lea disp(% 2975 * lea disp(%rsp), %reg
3077 * 2976 *
3078 * This is ne 2977 * This is needed for the rare case where GCC
3079 * does somet 2978 * does something dumb like:
3080 * 2979 *
3081 * lea 0 2980 * lea 0x8(%rsp), %rcx
3082 * ... 2981 * ...
3083 * mov % 2982 * mov %rcx, %rsp
3084 */ 2983 */
3085 cfi->vals[op- 2984 cfi->vals[op->dest.reg].base = CFI_CFA;
3086 cfi->vals[op- 2985 cfi->vals[op->dest.reg].offset = \
3087 -cfi- 2986 -cfi->stack_size + op->src.offset;
3088 2987
3089 break; 2988 break;
3090 } 2989 }
3091 2990
3092 if (cfi->drap && op-> 2991 if (cfi->drap && op->dest.reg == CFI_SP &&
3093 op->src.reg == cf 2992 op->src.reg == cfi->drap_reg) {
3094 2993
3095 /* drap: lea 2994 /* drap: lea disp(%drap), %rsp */
3096 cfa->base = C 2995 cfa->base = CFI_SP;
3097 cfa->offset = 2996 cfa->offset = cfi->stack_size = -op->src.offset;
3098 cfi->drap_reg 2997 cfi->drap_reg = CFI_UNDEFINED;
3099 cfi->drap = f 2998 cfi->drap = false;
3100 break; 2999 break;
3101 } 3000 }
3102 3001
3103 if (op->dest.reg == c 3002 if (op->dest.reg == cfi->cfa.base && !(next_insn && next_insn->hint)) {
3104 WARN_INSN(ins 3003 WARN_INSN(insn, "unsupported stack register modification");
3105 return -1; 3004 return -1;
3106 } 3005 }
3107 3006
3108 break; 3007 break;
3109 3008
3110 case OP_SRC_AND: 3009 case OP_SRC_AND:
3111 if (op->dest.reg != C 3010 if (op->dest.reg != CFI_SP ||
3112 (cfi->drap_reg != 3011 (cfi->drap_reg != CFI_UNDEFINED && cfa->base != CFI_SP) ||
3113 (cfi->drap_reg == 3012 (cfi->drap_reg == CFI_UNDEFINED && cfa->base != CFI_BP)) {
3114 WARN_INSN(ins 3013 WARN_INSN(insn, "unsupported stack pointer realignment");
3115 return -1; 3014 return -1;
3116 } 3015 }
3117 3016
3118 if (cfi->drap_reg != 3017 if (cfi->drap_reg != CFI_UNDEFINED) {
3119 /* drap: and 3018 /* drap: and imm, %rsp */
3120 cfa->base = c 3019 cfa->base = cfi->drap_reg;
3121 cfa->offset = 3020 cfa->offset = cfi->stack_size = 0;
3122 cfi->drap = t 3021 cfi->drap = true;
3123 } 3022 }
3124 3023
3125 /* 3024 /*
3126 * Older versions of 3025 * Older versions of GCC (4.8ish) realign the stack
3127 * without DRAP, with 3026 * without DRAP, with a frame pointer.
3128 */ 3027 */
3129 3028
3130 break; 3029 break;
3131 3030
3132 case OP_SRC_POP: 3031 case OP_SRC_POP:
3133 case OP_SRC_POPF: 3032 case OP_SRC_POPF:
3134 if (op->dest.reg == C 3033 if (op->dest.reg == CFI_SP && cfa->base == CFI_SP_INDIRECT) {
3135 3034
3136 /* pop %rsp; 3035 /* pop %rsp; # restore from a stack swizzle */
3137 cfa->base = C 3036 cfa->base = CFI_SP;
3138 break; 3037 break;
3139 } 3038 }
3140 3039
3141 if (!cfi->drap && op- 3040 if (!cfi->drap && op->dest.reg == cfa->base) {
3142 3041
3143 /* pop %rbp * 3042 /* pop %rbp */
3144 cfa->base = C 3043 cfa->base = CFI_SP;
3145 } 3044 }
3146 3045
3147 if (cfi->drap && cfa- 3046 if (cfi->drap && cfa->base == CFI_BP_INDIRECT &&
3148 op->dest.reg == c 3047 op->dest.reg == cfi->drap_reg &&
3149 cfi->drap_offset 3048 cfi->drap_offset == -cfi->stack_size) {
3150 3049
3151 /* drap: pop 3050 /* drap: pop %drap */
3152 cfa->base = c 3051 cfa->base = cfi->drap_reg;
3153 cfa->offset = 3052 cfa->offset = 0;
3154 cfi->drap_off 3053 cfi->drap_offset = -1;
3155 3054
3156 } else if (cfi->stack 3055 } else if (cfi->stack_size == -regs[op->dest.reg].offset) {
3157 3056
3158 /* pop %reg * 3057 /* pop %reg */
3159 restore_reg(c 3058 restore_reg(cfi, op->dest.reg);
3160 } 3059 }
3161 3060
3162 cfi->stack_size -= 8; 3061 cfi->stack_size -= 8;
3163 if (cfa->base == CFI_ 3062 if (cfa->base == CFI_SP)
3164 cfa->offset - 3063 cfa->offset -= 8;
3165 3064
3166 break; 3065 break;
3167 3066
3168 case OP_SRC_REG_INDIRECT: 3067 case OP_SRC_REG_INDIRECT:
3169 if (!cfi->drap && op- 3068 if (!cfi->drap && op->dest.reg == cfa->base &&
3170 op->dest.reg == C 3069 op->dest.reg == CFI_BP) {
3171 3070
3172 /* mov disp(% 3071 /* mov disp(%rsp), %rbp */
3173 cfa->base = C 3072 cfa->base = CFI_SP;
3174 cfa->offset = 3073 cfa->offset = cfi->stack_size;
3175 } 3074 }
3176 3075
3177 if (cfi->drap && op-> 3076 if (cfi->drap && op->src.reg == CFI_BP &&
3178 op->src.offset == 3077 op->src.offset == cfi->drap_offset) {
3179 3078
3180 /* drap: mov 3079 /* drap: mov disp(%rbp), %drap */
3181 cfa->base = c 3080 cfa->base = cfi->drap_reg;
3182 cfa->offset = 3081 cfa->offset = 0;
3183 cfi->drap_off 3082 cfi->drap_offset = -1;
3184 } 3083 }
3185 3084
3186 if (cfi->drap && op-> 3085 if (cfi->drap && op->src.reg == CFI_BP &&
3187 op->src.offset == 3086 op->src.offset == regs[op->dest.reg].offset) {
3188 3087
3189 /* drap: mov 3088 /* drap: mov disp(%rbp), %reg */
3190 restore_reg(c 3089 restore_reg(cfi, op->dest.reg);
3191 3090
3192 } else if (op->src.re 3091 } else if (op->src.reg == cfa->base &&
3193 op->src.offset == 3092 op->src.offset == regs[op->dest.reg].offset + cfa->offset) {
3194 3093
3195 /* mov disp(% 3094 /* mov disp(%rbp), %reg */
3196 /* mov disp(% 3095 /* mov disp(%rsp), %reg */
3197 restore_reg(c 3096 restore_reg(cfi, op->dest.reg);
3198 3097
3199 } else if (op->src.re 3098 } else if (op->src.reg == CFI_SP &&
3200 op->src.of 3099 op->src.offset == regs[op->dest.reg].offset + cfi->stack_size) {
3201 3100
3202 /* mov disp(% 3101 /* mov disp(%rsp), %reg */
3203 restore_reg(c 3102 restore_reg(cfi, op->dest.reg);
3204 } 3103 }
3205 3104
3206 break; 3105 break;
3207 3106
3208 default: 3107 default:
3209 WARN_INSN(insn, "unkn 3108 WARN_INSN(insn, "unknown stack-related instruction");
3210 return -1; 3109 return -1;
3211 } 3110 }
3212 3111
3213 break; 3112 break;
3214 3113
3215 case OP_DEST_PUSH: 3114 case OP_DEST_PUSH:
3216 case OP_DEST_PUSHF: 3115 case OP_DEST_PUSHF:
3217 cfi->stack_size += 8; 3116 cfi->stack_size += 8;
3218 if (cfa->base == CFI_SP) 3117 if (cfa->base == CFI_SP)
3219 cfa->offset += 8; 3118 cfa->offset += 8;
3220 3119
3221 if (op->src.type != OP_SRC_RE 3120 if (op->src.type != OP_SRC_REG)
3222 break; 3121 break;
3223 3122
3224 if (cfi->drap) { 3123 if (cfi->drap) {
3225 if (op->src.reg == cf 3124 if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
3226 3125
3227 /* drap: push 3126 /* drap: push %drap */
3228 cfa->base = C 3127 cfa->base = CFI_BP_INDIRECT;
3229 cfa->offset = 3128 cfa->offset = -cfi->stack_size;
3230 3129
3231 /* save drap 3130 /* save drap so we know when to restore it */
3232 cfi->drap_off 3131 cfi->drap_offset = -cfi->stack_size;
3233 3132
3234 } else if (op->src.re 3133 } else if (op->src.reg == CFI_BP && cfa->base == cfi->drap_reg) {
3235 3134
3236 /* drap: push 3135 /* drap: push %rbp */
3237 cfi->stack_si 3136 cfi->stack_size = 0;
3238 3137
3239 } else { 3138 } else {
3240 3139
3241 /* drap: push 3140 /* drap: push %reg */
3242 save_reg(cfi, 3141 save_reg(cfi, op->src.reg, CFI_BP, -cfi->stack_size);
3243 } 3142 }
3244 3143
3245 } else { 3144 } else {
3246 3145
3247 /* push %reg */ 3146 /* push %reg */
3248 save_reg(cfi, op->src 3147 save_reg(cfi, op->src.reg, CFI_CFA, -cfi->stack_size);
3249 } 3148 }
3250 3149
3251 /* detect when asm code uses 3150 /* detect when asm code uses rbp as a scratch register */
3252 if (opts.stackval && insn_fun 3151 if (opts.stackval && insn_func(insn) && op->src.reg == CFI_BP &&
3253 cfa->base != CFI_BP) 3152 cfa->base != CFI_BP)
3254 cfi->bp_scratch = tru 3153 cfi->bp_scratch = true;
3255 break; 3154 break;
3256 3155
3257 case OP_DEST_REG_INDIRECT: 3156 case OP_DEST_REG_INDIRECT:
3258 3157
3259 if (cfi->drap) { 3158 if (cfi->drap) {
3260 if (op->src.reg == cf 3159 if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
3261 3160
3262 /* drap: mov 3161 /* drap: mov %drap, disp(%rbp) */
3263 cfa->base = C 3162 cfa->base = CFI_BP_INDIRECT;
3264 cfa->offset = 3163 cfa->offset = op->dest.offset;
3265 3164
3266 /* save drap 3165 /* save drap offset so we know when to restore it */
3267 cfi->drap_off 3166 cfi->drap_offset = op->dest.offset;
3268 } else { 3167 } else {
3269 3168
3270 /* drap: mov 3169 /* drap: mov reg, disp(%rbp) */
3271 save_reg(cfi, 3170 save_reg(cfi, op->src.reg, CFI_BP, op->dest.offset);
3272 } 3171 }
3273 3172
3274 } else if (op->dest.reg == cf 3173 } else if (op->dest.reg == cfa->base) {
3275 3174
3276 /* mov reg, disp(%rbp 3175 /* mov reg, disp(%rbp) */
3277 /* mov reg, disp(%rsp 3176 /* mov reg, disp(%rsp) */
3278 save_reg(cfi, op->src 3177 save_reg(cfi, op->src.reg, CFI_CFA,
3279 op->dest.off 3178 op->dest.offset - cfi->cfa.offset);
3280 3179
3281 } else if (op->dest.reg == CF 3180 } else if (op->dest.reg == CFI_SP) {
3282 3181
3283 /* mov reg, disp(%rsp 3182 /* mov reg, disp(%rsp) */
3284 save_reg(cfi, op->src 3183 save_reg(cfi, op->src.reg, CFI_CFA,
3285 op->dest.off 3184 op->dest.offset - cfi->stack_size);
3286 3185
3287 } else if (op->src.reg == CFI 3186 } else if (op->src.reg == CFI_SP && op->dest.offset == 0) {
3288 3187
3289 /* mov %rsp, (%reg); 3188 /* mov %rsp, (%reg); # setup a stack swizzle. */
3290 cfi->vals[op->dest.re 3189 cfi->vals[op->dest.reg].base = CFI_SP_INDIRECT;
3291 cfi->vals[op->dest.re 3190 cfi->vals[op->dest.reg].offset = cfa->offset;
3292 } 3191 }
3293 3192
3294 break; 3193 break;
3295 3194
3296 case OP_DEST_MEM: 3195 case OP_DEST_MEM:
3297 if (op->src.type != OP_SRC_PO 3196 if (op->src.type != OP_SRC_POP && op->src.type != OP_SRC_POPF) {
3298 WARN_INSN(insn, "unkn 3197 WARN_INSN(insn, "unknown stack-related memory operation");
3299 return -1; 3198 return -1;
3300 } 3199 }
3301 3200
3302 /* pop mem */ 3201 /* pop mem */
3303 cfi->stack_size -= 8; 3202 cfi->stack_size -= 8;
3304 if (cfa->base == CFI_SP) 3203 if (cfa->base == CFI_SP)
3305 cfa->offset -= 8; 3204 cfa->offset -= 8;
3306 3205
3307 break; 3206 break;
3308 3207
3309 default: 3208 default:
3310 WARN_INSN(insn, "unknown stac 3209 WARN_INSN(insn, "unknown stack-related instruction");
3311 return -1; 3210 return -1;
3312 } 3211 }
3313 3212
3314 return 0; 3213 return 0;
3315 } 3214 }
3316 3215
3317 /* 3216 /*
3318 * The stack layouts of alternatives instruct 3217 * The stack layouts of alternatives instructions can sometimes diverge when
3319 * they have stack modifications. That's fin 3218 * they have stack modifications. That's fine as long as the potential stack
3320 * layouts don't conflict at any given potent 3219 * layouts don't conflict at any given potential instruction boundary.
3321 * 3220 *
3322 * Flatten the CFIs of the different alternat 3221 * Flatten the CFIs of the different alternative code streams (both original
3323 * and replacement) into a single shared CFI 3222 * and replacement) into a single shared CFI array which can be used to detect
3324 * conflicts and nicely feed a linear array o 3223 * conflicts and nicely feed a linear array of ORC entries to the unwinder.
3325 */ 3224 */
3326 static int propagate_alt_cfi(struct objtool_f 3225 static int propagate_alt_cfi(struct objtool_file *file, struct instruction *insn)
3327 { 3226 {
3328 struct cfi_state **alt_cfi; 3227 struct cfi_state **alt_cfi;
3329 int group_off; 3228 int group_off;
3330 3229
3331 if (!insn->alt_group) 3230 if (!insn->alt_group)
3332 return 0; 3231 return 0;
3333 3232
3334 if (!insn->cfi) { 3233 if (!insn->cfi) {
3335 WARN("CFI missing"); 3234 WARN("CFI missing");
3336 return -1; 3235 return -1;
3337 } 3236 }
3338 3237
3339 alt_cfi = insn->alt_group->cfi; 3238 alt_cfi = insn->alt_group->cfi;
3340 group_off = insn->offset - insn->alt_ 3239 group_off = insn->offset - insn->alt_group->first_insn->offset;
3341 3240
3342 if (!alt_cfi[group_off]) { 3241 if (!alt_cfi[group_off]) {
3343 alt_cfi[group_off] = insn->cf 3242 alt_cfi[group_off] = insn->cfi;
3344 } else { 3243 } else {
3345 if (cficmp(alt_cfi[group_off] 3244 if (cficmp(alt_cfi[group_off], insn->cfi)) {
3346 struct alt_group *ori 3245 struct alt_group *orig_group = insn->alt_group->orig_group ?: insn->alt_group;
3347 struct instruction *o 3246 struct instruction *orig = orig_group->first_insn;
3348 char *where = offstr( 3247 char *where = offstr(insn->sec, insn->offset);
3349 WARN_INSN(orig, "stac 3248 WARN_INSN(orig, "stack layout conflict in alternatives: %s", where);
3350 free(where); 3249 free(where);
3351 return -1; 3250 return -1;
3352 } 3251 }
3353 } 3252 }
3354 3253
3355 return 0; 3254 return 0;
3356 } 3255 }
3357 3256
3358 static int handle_insn_ops(struct instruction 3257 static int handle_insn_ops(struct instruction *insn,
3359 struct instruction 3258 struct instruction *next_insn,
3360 struct insn_state 3259 struct insn_state *state)
3361 { 3260 {
3362 struct stack_op *op; 3261 struct stack_op *op;
3363 3262
3364 for (op = insn->stack_ops; op; op = o 3263 for (op = insn->stack_ops; op; op = op->next) {
3365 3264
3366 if (update_cfi_state(insn, ne 3265 if (update_cfi_state(insn, next_insn, &state->cfi, op))
3367 return 1; 3266 return 1;
3368 3267
3369 if (!insn->alt_group) 3268 if (!insn->alt_group)
3370 continue; 3269 continue;
3371 3270
3372 if (op->dest.type == OP_DEST_ 3271 if (op->dest.type == OP_DEST_PUSHF) {
3373 if (!state->uaccess_s 3272 if (!state->uaccess_stack) {
3374 state->uacces 3273 state->uaccess_stack = 1;
3375 } else if (state->uac 3274 } else if (state->uaccess_stack >> 31) {
3376 WARN_INSN(ins 3275 WARN_INSN(insn, "PUSHF stack exhausted");
3377 return 1; 3276 return 1;
3378 } 3277 }
3379 state->uaccess_stack 3278 state->uaccess_stack <<= 1;
3380 state->uaccess_stack 3279 state->uaccess_stack |= state->uaccess;
3381 } 3280 }
3382 3281
3383 if (op->src.type == OP_SRC_PO 3282 if (op->src.type == OP_SRC_POPF) {
3384 if (state->uaccess_st 3283 if (state->uaccess_stack) {
3385 state->uacces 3284 state->uaccess = state->uaccess_stack & 1;
3386 state->uacces 3285 state->uaccess_stack >>= 1;
3387 if (state->ua 3286 if (state->uaccess_stack == 1)
3388 state 3287 state->uaccess_stack = 0;
3389 } 3288 }
3390 } 3289 }
3391 } 3290 }
3392 3291
3393 return 0; 3292 return 0;
3394 } 3293 }
3395 3294
3396 static bool insn_cfi_match(struct instruction 3295 static bool insn_cfi_match(struct instruction *insn, struct cfi_state *cfi2)
3397 { 3296 {
3398 struct cfi_state *cfi1 = insn->cfi; 3297 struct cfi_state *cfi1 = insn->cfi;
3399 int i; 3298 int i;
3400 3299
3401 if (!cfi1) { 3300 if (!cfi1) {
3402 WARN("CFI missing"); 3301 WARN("CFI missing");
3403 return false; 3302 return false;
3404 } 3303 }
3405 3304
3406 if (memcmp(&cfi1->cfa, &cfi2->cfa, si 3305 if (memcmp(&cfi1->cfa, &cfi2->cfa, sizeof(cfi1->cfa))) {
3407 3306
3408 WARN_INSN(insn, "stack state 3307 WARN_INSN(insn, "stack state mismatch: cfa1=%d%+d cfa2=%d%+d",
3409 cfi1->cfa.base, cfi 3308 cfi1->cfa.base, cfi1->cfa.offset,
3410 cfi2->cfa.base, cfi 3309 cfi2->cfa.base, cfi2->cfa.offset);
3411 3310
3412 } else if (memcmp(&cfi1->regs, &cfi2- 3311 } else if (memcmp(&cfi1->regs, &cfi2->regs, sizeof(cfi1->regs))) {
3413 for (i = 0; i < CFI_NUM_REGS; 3312 for (i = 0; i < CFI_NUM_REGS; i++) {
3414 if (!memcmp(&cfi1->re 3313 if (!memcmp(&cfi1->regs[i], &cfi2->regs[i],
3415 sizeof(st 3314 sizeof(struct cfi_reg)))
3416 continue; 3315 continue;
3417 3316
3418 WARN_INSN(insn, "stac 3317 WARN_INSN(insn, "stack state mismatch: reg1[%d]=%d%+d reg2[%d]=%d%+d",
3419 i, cfi1->re 3318 i, cfi1->regs[i].base, cfi1->regs[i].offset,
3420 i, cfi2->re 3319 i, cfi2->regs[i].base, cfi2->regs[i].offset);
3421 break; 3320 break;
3422 } 3321 }
3423 3322
3424 } else if (cfi1->type != cfi2->type) 3323 } else if (cfi1->type != cfi2->type) {
3425 3324
3426 WARN_INSN(insn, "stack state 3325 WARN_INSN(insn, "stack state mismatch: type1=%d type2=%d",
3427 cfi1->type, cfi2->t 3326 cfi1->type, cfi2->type);
3428 3327
3429 } else if (cfi1->drap != cfi2->drap | 3328 } else if (cfi1->drap != cfi2->drap ||
3430 (cfi1->drap && cfi1->drap_ 3329 (cfi1->drap && cfi1->drap_reg != cfi2->drap_reg) ||
3431 (cfi1->drap && cfi1->drap_ 3330 (cfi1->drap && cfi1->drap_offset != cfi2->drap_offset)) {
3432 3331
3433 WARN_INSN(insn, "stack state 3332 WARN_INSN(insn, "stack state mismatch: drap1=%d(%d,%d) drap2=%d(%d,%d)",
3434 cfi1->drap, cfi1->d 3333 cfi1->drap, cfi1->drap_reg, cfi1->drap_offset,
3435 cfi2->drap, cfi2->d 3334 cfi2->drap, cfi2->drap_reg, cfi2->drap_offset);
3436 3335
3437 } else 3336 } else
3438 return true; 3337 return true;
3439 3338
3440 return false; 3339 return false;
3441 } 3340 }
3442 3341
3443 static inline bool func_uaccess_safe(struct s 3342 static inline bool func_uaccess_safe(struct symbol *func)
3444 { 3343 {
3445 if (func) 3344 if (func)
3446 return func->uaccess_safe; 3345 return func->uaccess_safe;
3447 3346
3448 return false; 3347 return false;
3449 } 3348 }
3450 3349
3451 static inline const char *call_dest_name(stru 3350 static inline const char *call_dest_name(struct instruction *insn)
3452 { 3351 {
3453 static char pvname[19]; 3352 static char pvname[19];
3454 struct reloc *reloc; 3353 struct reloc *reloc;
3455 int idx; 3354 int idx;
3456 3355
3457 if (insn_call_dest(insn)) 3356 if (insn_call_dest(insn))
3458 return insn_call_dest(insn)-> 3357 return insn_call_dest(insn)->name;
3459 3358
3460 reloc = insn_reloc(NULL, insn); 3359 reloc = insn_reloc(NULL, insn);
3461 if (reloc && !strcmp(reloc->sym->name 3360 if (reloc && !strcmp(reloc->sym->name, "pv_ops")) {
3462 idx = (reloc_addend(reloc) / 3361 idx = (reloc_addend(reloc) / sizeof(void *));
3463 snprintf(pvname, sizeof(pvnam 3362 snprintf(pvname, sizeof(pvname), "pv_ops[%d]", idx);
3464 return pvname; 3363 return pvname;
3465 } 3364 }
3466 3365
3467 return "{dynamic}"; 3366 return "{dynamic}";
3468 } 3367 }
3469 3368
3470 static bool pv_call_dest(struct objtool_file 3369 static bool pv_call_dest(struct objtool_file *file, struct instruction *insn)
3471 { 3370 {
3472 struct symbol *target; 3371 struct symbol *target;
3473 struct reloc *reloc; 3372 struct reloc *reloc;
3474 int idx; 3373 int idx;
3475 3374
3476 reloc = insn_reloc(file, insn); 3375 reloc = insn_reloc(file, insn);
3477 if (!reloc || strcmp(reloc->sym->name 3376 if (!reloc || strcmp(reloc->sym->name, "pv_ops"))
3478 return false; 3377 return false;
3479 3378
3480 idx = (arch_dest_reloc_offset(reloc_a 3379 idx = (arch_dest_reloc_offset(reloc_addend(reloc)) / sizeof(void *));
3481 3380
3482 if (file->pv_ops[idx].clean) 3381 if (file->pv_ops[idx].clean)
3483 return true; 3382 return true;
3484 3383
3485 file->pv_ops[idx].clean = true; 3384 file->pv_ops[idx].clean = true;
3486 3385
3487 list_for_each_entry(target, &file->pv 3386 list_for_each_entry(target, &file->pv_ops[idx].targets, pv_target) {
3488 if (!target->sec->noinstr) { 3387 if (!target->sec->noinstr) {
3489 WARN("pv_ops[%d]: %s" 3388 WARN("pv_ops[%d]: %s", idx, target->name);
3490 file->pv_ops[idx].cle 3389 file->pv_ops[idx].clean = false;
3491 } 3390 }
3492 } 3391 }
3493 3392
3494 return file->pv_ops[idx].clean; 3393 return file->pv_ops[idx].clean;
3495 } 3394 }
3496 3395
3497 static inline bool noinstr_call_dest(struct o 3396 static inline bool noinstr_call_dest(struct objtool_file *file,
3498 struct i 3397 struct instruction *insn,
3499 struct s 3398 struct symbol *func)
3500 { 3399 {
3501 /* 3400 /*
3502 * We can't deal with indirect functi 3401 * We can't deal with indirect function calls at present;
3503 * assume they're instrumented. 3402 * assume they're instrumented.
3504 */ 3403 */
3505 if (!func) { 3404 if (!func) {
3506 if (file->pv_ops) 3405 if (file->pv_ops)
3507 return pv_call_dest(f 3406 return pv_call_dest(file, insn);
3508 3407
3509 return false; 3408 return false;
3510 } 3409 }
3511 3410
3512 /* 3411 /*
3513 * If the symbol is from a noinstr se 3412 * If the symbol is from a noinstr section; we good.
3514 */ 3413 */
3515 if (func->sec->noinstr) 3414 if (func->sec->noinstr)
3516 return true; 3415 return true;
3517 3416
3518 /* 3417 /*
3519 * If the symbol is a static_call tra 3418 * If the symbol is a static_call trampoline, we can't tell.
3520 */ 3419 */
3521 if (func->static_call_tramp) 3420 if (func->static_call_tramp)
3522 return true; 3421 return true;
3523 3422
3524 /* 3423 /*
3525 * The __ubsan_handle_*() calls are l 3424 * The __ubsan_handle_*() calls are like WARN(), they only happen when
3526 * something 'BAD' happened. At the r 3425 * something 'BAD' happened. At the risk of taking the machine down,
3527 * let them proceed to get the messag 3426 * let them proceed to get the message out.
3528 */ 3427 */
3529 if (!strncmp(func->name, "__ubsan_han 3428 if (!strncmp(func->name, "__ubsan_handle_", 15))
3530 return true; 3429 return true;
3531 3430
3532 return false; 3431 return false;
3533 } 3432 }
3534 3433
3535 static int validate_call(struct objtool_file 3434 static int validate_call(struct objtool_file *file,
3536 struct instruction * 3435 struct instruction *insn,
3537 struct insn_state *s 3436 struct insn_state *state)
3538 { 3437 {
3539 if (state->noinstr && state->instr <= 3438 if (state->noinstr && state->instr <= 0 &&
3540 !noinstr_call_dest(file, insn, in 3439 !noinstr_call_dest(file, insn, insn_call_dest(insn))) {
3541 WARN_INSN(insn, "call to %s() 3440 WARN_INSN(insn, "call to %s() leaves .noinstr.text section", call_dest_name(insn));
3542 return 1; 3441 return 1;
3543 } 3442 }
3544 3443
3545 if (state->uaccess && !func_uaccess_s 3444 if (state->uaccess && !func_uaccess_safe(insn_call_dest(insn))) {
3546 WARN_INSN(insn, "call to %s() 3445 WARN_INSN(insn, "call to %s() with UACCESS enabled", call_dest_name(insn));
3547 return 1; 3446 return 1;
3548 } 3447 }
3549 3448
3550 if (state->df) { 3449 if (state->df) {
3551 WARN_INSN(insn, "call to %s() 3450 WARN_INSN(insn, "call to %s() with DF set", call_dest_name(insn));
3552 return 1; 3451 return 1;
3553 } 3452 }
3554 3453
3555 return 0; 3454 return 0;
3556 } 3455 }
3557 3456
3558 static int validate_sibling_call(struct objto 3457 static int validate_sibling_call(struct objtool_file *file,
3559 struct instr 3458 struct instruction *insn,
3560 struct insn_ 3459 struct insn_state *state)
3561 { 3460 {
3562 if (insn_func(insn) && has_modified_s 3461 if (insn_func(insn) && has_modified_stack_frame(insn, state)) {
3563 WARN_INSN(insn, "sibling call 3462 WARN_INSN(insn, "sibling call from callable instruction with modified stack frame");
3564 return 1; 3463 return 1;
3565 } 3464 }
3566 3465
3567 return validate_call(file, insn, stat 3466 return validate_call(file, insn, state);
3568 } 3467 }
3569 3468
3570 static int validate_return(struct symbol *fun 3469 static int validate_return(struct symbol *func, struct instruction *insn, struct insn_state *state)
3571 { 3470 {
3572 if (state->noinstr && state->instr > 3471 if (state->noinstr && state->instr > 0) {
3573 WARN_INSN(insn, "return with 3472 WARN_INSN(insn, "return with instrumentation enabled");
3574 return 1; 3473 return 1;
3575 } 3474 }
3576 3475
3577 if (state->uaccess && !func_uaccess_s 3476 if (state->uaccess && !func_uaccess_safe(func)) {
3578 WARN_INSN(insn, "return with 3477 WARN_INSN(insn, "return with UACCESS enabled");
3579 return 1; 3478 return 1;
3580 } 3479 }
3581 3480
3582 if (!state->uaccess && func_uaccess_s 3481 if (!state->uaccess && func_uaccess_safe(func)) {
3583 WARN_INSN(insn, "return with 3482 WARN_INSN(insn, "return with UACCESS disabled from a UACCESS-safe function");
3584 return 1; 3483 return 1;
3585 } 3484 }
3586 3485
3587 if (state->df) { 3486 if (state->df) {
3588 WARN_INSN(insn, "return with 3487 WARN_INSN(insn, "return with DF set");
3589 return 1; 3488 return 1;
3590 } 3489 }
3591 3490
3592 if (func && has_modified_stack_frame( 3491 if (func && has_modified_stack_frame(insn, state)) {
3593 WARN_INSN(insn, "return with 3492 WARN_INSN(insn, "return with modified stack frame");
3594 return 1; 3493 return 1;
3595 } 3494 }
3596 3495
3597 if (state->cfi.bp_scratch) { 3496 if (state->cfi.bp_scratch) {
3598 WARN_INSN(insn, "BP used as a 3497 WARN_INSN(insn, "BP used as a scratch register");
3599 return 1; 3498 return 1;
3600 } 3499 }
3601 3500
3602 return 0; 3501 return 0;
3603 } 3502 }
3604 3503
3605 static struct instruction *next_insn_to_valid 3504 static struct instruction *next_insn_to_validate(struct objtool_file *file,
3606 3505 struct instruction *insn)
3607 { 3506 {
3608 struct alt_group *alt_group = insn->a 3507 struct alt_group *alt_group = insn->alt_group;
3609 3508
3610 /* 3509 /*
3611 * Simulate the fact that alternative 3510 * Simulate the fact that alternatives are patched in-place. When the
3612 * end of a replacement alt_group is 3511 * end of a replacement alt_group is reached, redirect objtool flow to
3613 * the end of the original alt_group. 3512 * the end of the original alt_group.
3614 * 3513 *
3615 * insn->alts->insn -> alt_group->fir 3514 * insn->alts->insn -> alt_group->first_insn
3616 * ... 3515 * ...
3617 * alt_group->las 3516 * alt_group->last_insn
3618 * [alt_group->no 3517 * [alt_group->nop] -> next(orig_group->last_insn)
3619 */ 3518 */
3620 if (alt_group) { 3519 if (alt_group) {
3621 if (alt_group->nop) { 3520 if (alt_group->nop) {
3622 /* ->nop implies ->or 3521 /* ->nop implies ->orig_group */
3623 if (insn == alt_group 3522 if (insn == alt_group->last_insn)
3624 return alt_gr 3523 return alt_group->nop;
3625 if (insn == alt_group 3524 if (insn == alt_group->nop)
3626 goto next_ori 3525 goto next_orig;
3627 } 3526 }
3628 if (insn == alt_group->last_i 3527 if (insn == alt_group->last_insn && alt_group->orig_group)
3629 goto next_orig; 3528 goto next_orig;
3630 } 3529 }
3631 3530
3632 return next_insn_same_sec(file, insn) 3531 return next_insn_same_sec(file, insn);
3633 3532
3634 next_orig: 3533 next_orig:
3635 return next_insn_same_sec(file, alt_g 3534 return next_insn_same_sec(file, alt_group->orig_group->last_insn);
3636 } 3535 }
3637 3536
3638 /* 3537 /*
3639 * Follow the branch starting at the given in 3538 * Follow the branch starting at the given instruction, and recursively follow
3640 * any other branches (jumps). Meanwhile, tr 3539 * any other branches (jumps). Meanwhile, track the frame pointer state at
3641 * each instruction and validate all the rule 3540 * each instruction and validate all the rules described in
3642 * tools/objtool/Documentation/objtool.txt. 3541 * tools/objtool/Documentation/objtool.txt.
3643 */ 3542 */
3644 static int validate_branch(struct objtool_fil 3543 static int validate_branch(struct objtool_file *file, struct symbol *func,
3645 struct instruction 3544 struct instruction *insn, struct insn_state state)
3646 { 3545 {
3647 struct alternative *alt; 3546 struct alternative *alt;
3648 struct instruction *next_insn, *prev_ 3547 struct instruction *next_insn, *prev_insn = NULL;
3649 struct section *sec; 3548 struct section *sec;
3650 u8 visited; 3549 u8 visited;
3651 int ret; 3550 int ret;
3652 3551
3653 sec = insn->sec; 3552 sec = insn->sec;
3654 3553
3655 while (1) { 3554 while (1) {
3656 next_insn = next_insn_to_vali 3555 next_insn = next_insn_to_validate(file, insn);
3657 3556
3658 if (func && insn_func(insn) & 3557 if (func && insn_func(insn) && func != insn_func(insn)->pfunc) {
3659 /* Ignore KCFI type p 3558 /* Ignore KCFI type preambles, which always fall through */
3660 if (!strncmp(func->na 3559 if (!strncmp(func->name, "__cfi_", 6) ||
3661 !strncmp(func->na 3560 !strncmp(func->name, "__pfx_", 6))
3662 return 0; 3561 return 0;
3663 3562
3664 WARN("%s() falls thro 3563 WARN("%s() falls through to next function %s()",
3665 func->name, insn 3564 func->name, insn_func(insn)->name);
3666 return 1; 3565 return 1;
3667 } 3566 }
3668 3567
3669 if (func && insn->ignore) { 3568 if (func && insn->ignore) {
3670 WARN_INSN(insn, "BUG: 3569 WARN_INSN(insn, "BUG: why am I validating an ignored function?");
3671 return 1; 3570 return 1;
3672 } 3571 }
3673 3572
3674 visited = VISITED_BRANCH << s 3573 visited = VISITED_BRANCH << state.uaccess;
3675 if (insn->visited & VISITED_B 3574 if (insn->visited & VISITED_BRANCH_MASK) {
3676 if (!insn->hint && !i 3575 if (!insn->hint && !insn_cfi_match(insn, &state.cfi))
3677 return 1; 3576 return 1;
3678 3577
3679 if (insn->visited & v 3578 if (insn->visited & visited)
3680 return 0; 3579 return 0;
3681 } else { 3580 } else {
3682 nr_insns_visited++; 3581 nr_insns_visited++;
3683 } 3582 }
3684 3583
3685 if (state.noinstr) 3584 if (state.noinstr)
3686 state.instr += insn-> 3585 state.instr += insn->instr;
3687 3586
3688 if (insn->hint) { 3587 if (insn->hint) {
3689 if (insn->restore) { 3588 if (insn->restore) {
3690 struct instru 3589 struct instruction *save_insn, *i;
3691 3590
3692 i = insn; 3591 i = insn;
3693 save_insn = N 3592 save_insn = NULL;
3694 3593
3695 sym_for_each_ 3594 sym_for_each_insn_continue_reverse(file, func, i) {
3696 if (i 3595 if (i->save) {
3697 3596 save_insn = i;
3698 3597 break;
3699 } 3598 }
3700 } 3599 }
3701 3600
3702 if (!save_ins 3601 if (!save_insn) {
3703 WARN_ 3602 WARN_INSN(insn, "no corresponding CFI save for CFI restore");
3704 retur 3603 return 1;
3705 } 3604 }
3706 3605
3707 if (!save_ins 3606 if (!save_insn->visited) {
3708 /* 3607 /*
3709 * If 3608 * If the restore hint insn is at the
3710 * be 3609 * beginning of a basic block and was
3711 * br 3610 * branched to from elsewhere, and the
3712 * sa 3611 * save insn hasn't been visited yet,
3713 * de 3612 * defer following this branch for now.
3714 * It 3613 * It will be seen later via the
3715 * st 3614 * straight-line path.
3716 */ 3615 */
3717 if (! 3616 if (!prev_insn)
3718 3617 return 0;
3719 3618
3720 WARN_ 3619 WARN_INSN(insn, "objtool isn't smart enough to handle this CFI save/restore combo");
3721 retur 3620 return 1;
3722 } 3621 }
3723 3622
3724 insn->cfi = s 3623 insn->cfi = save_insn->cfi;
3725 nr_cfi_reused 3624 nr_cfi_reused++;
3726 } 3625 }
3727 3626
3728 state.cfi = *insn->cf 3627 state.cfi = *insn->cfi;
3729 } else { 3628 } else {
3730 /* XXX track if we ac 3629 /* XXX track if we actually changed state.cfi */
3731 3630
3732 if (prev_insn && !cfi 3631 if (prev_insn && !cficmp(prev_insn->cfi, &state.cfi)) {
3733 insn->cfi = p 3632 insn->cfi = prev_insn->cfi;
3734 nr_cfi_reused 3633 nr_cfi_reused++;
3735 } else { 3634 } else {
3736 insn->cfi = c 3635 insn->cfi = cfi_hash_find_or_add(&state.cfi);
3737 } 3636 }
3738 } 3637 }
3739 3638
3740 insn->visited |= visited; 3639 insn->visited |= visited;
3741 3640
3742 if (propagate_alt_cfi(file, i 3641 if (propagate_alt_cfi(file, insn))
3743 return 1; 3642 return 1;
3744 3643
3745 if (!insn->ignore_alts && ins 3644 if (!insn->ignore_alts && insn->alts) {
3746 bool skip_orig = fals 3645 bool skip_orig = false;
3747 3646
3748 for (alt = insn->alts 3647 for (alt = insn->alts; alt; alt = alt->next) {
3749 if (alt->skip 3648 if (alt->skip_orig)
3750 skip_ 3649 skip_orig = true;
3751 3650
3752 ret = validat 3651 ret = validate_branch(file, func, alt->insn, state);
3753 if (ret) { 3652 if (ret) {
3754 BT_IN 3653 BT_INSN(insn, "(alt)");
3755 retur 3654 return ret;
3756 } 3655 }
3757 } 3656 }
3758 3657
3759 if (skip_orig) 3658 if (skip_orig)
3760 return 0; 3659 return 0;
3761 } 3660 }
3762 3661
3763 if (handle_insn_ops(insn, nex 3662 if (handle_insn_ops(insn, next_insn, &state))
3764 return 1; 3663 return 1;
3765 3664
3766 switch (insn->type) { 3665 switch (insn->type) {
3767 3666
3768 case INSN_RETURN: 3667 case INSN_RETURN:
3769 return validate_retur 3668 return validate_return(func, insn, &state);
3770 3669
3771 case INSN_CALL: 3670 case INSN_CALL:
3772 case INSN_CALL_DYNAMIC: 3671 case INSN_CALL_DYNAMIC:
3773 ret = validate_call(f 3672 ret = validate_call(file, insn, &state);
3774 if (ret) 3673 if (ret)
3775 return ret; 3674 return ret;
3776 3675
3777 if (opts.stackval && 3676 if (opts.stackval && func && !is_special_call(insn) &&
3778 !has_valid_stack_ 3677 !has_valid_stack_frame(&state)) {
3779 WARN_INSN(ins 3678 WARN_INSN(insn, "call without frame pointer save/setup");
3780 return 1; 3679 return 1;
3781 } 3680 }
3782 3681
3783 if (insn->dead_end) 3682 if (insn->dead_end)
3784 return 0; 3683 return 0;
3785 3684
3786 break; 3685 break;
3787 3686
3788 case INSN_JUMP_CONDITIONAL: 3687 case INSN_JUMP_CONDITIONAL:
3789 case INSN_JUMP_UNCONDITIONAL: 3688 case INSN_JUMP_UNCONDITIONAL:
3790 if (is_sibling_call(i 3689 if (is_sibling_call(insn)) {
3791 ret = validat 3690 ret = validate_sibling_call(file, insn, &state);
3792 if (ret) 3691 if (ret)
3793 retur 3692 return ret;
3794 3693
3795 } else if (insn->jump 3694 } else if (insn->jump_dest) {
3796 ret = validat 3695 ret = validate_branch(file, func,
3797 3696 insn->jump_dest, state);
3798 if (ret) { 3697 if (ret) {
3799 BT_IN 3698 BT_INSN(insn, "(branch)");
3800 retur 3699 return ret;
3801 } 3700 }
3802 } 3701 }
3803 3702
3804 if (insn->type == INS 3703 if (insn->type == INSN_JUMP_UNCONDITIONAL)
3805 return 0; 3704 return 0;
3806 3705
3807 break; 3706 break;
3808 3707
3809 case INSN_JUMP_DYNAMIC: 3708 case INSN_JUMP_DYNAMIC:
3810 case INSN_JUMP_DYNAMIC_CONDIT 3709 case INSN_JUMP_DYNAMIC_CONDITIONAL:
3811 if (is_sibling_call(i 3710 if (is_sibling_call(insn)) {
3812 ret = validat 3711 ret = validate_sibling_call(file, insn, &state);
3813 if (ret) 3712 if (ret)
3814 retur 3713 return ret;
3815 } 3714 }
3816 3715
3817 if (insn->type == INS 3716 if (insn->type == INSN_JUMP_DYNAMIC)
3818 return 0; 3717 return 0;
3819 3718
3820 break; 3719 break;
3821 3720
3822 case INSN_CONTEXT_SWITCH: 3721 case INSN_CONTEXT_SWITCH:
3823 if (func && (!next_in 3722 if (func && (!next_insn || !next_insn->hint)) {
3824 WARN_INSN(ins 3723 WARN_INSN(insn, "unsupported instruction in callable function");
3825 return 1; 3724 return 1;
3826 } 3725 }
3827 return 0; 3726 return 0;
3828 3727
3829 case INSN_STAC: 3728 case INSN_STAC:
3830 if (state.uaccess) { 3729 if (state.uaccess) {
3831 WARN_INSN(ins 3730 WARN_INSN(insn, "recursive UACCESS enable");
3832 return 1; 3731 return 1;
3833 } 3732 }
3834 3733
3835 state.uaccess = true; 3734 state.uaccess = true;
3836 break; 3735 break;
3837 3736
3838 case INSN_CLAC: 3737 case INSN_CLAC:
3839 if (!state.uaccess && 3738 if (!state.uaccess && func) {
3840 WARN_INSN(ins 3739 WARN_INSN(insn, "redundant UACCESS disable");
3841 return 1; 3740 return 1;
3842 } 3741 }
3843 3742
3844 if (func_uaccess_safe 3743 if (func_uaccess_safe(func) && !state.uaccess_stack) {
3845 WARN_INSN(ins 3744 WARN_INSN(insn, "UACCESS-safe disables UACCESS");
3846 return 1; 3745 return 1;
3847 } 3746 }
3848 3747
3849 state.uaccess = false 3748 state.uaccess = false;
3850 break; 3749 break;
3851 3750
3852 case INSN_STD: 3751 case INSN_STD:
3853 if (state.df) { 3752 if (state.df) {
3854 WARN_INSN(ins 3753 WARN_INSN(insn, "recursive STD");
3855 return 1; 3754 return 1;
3856 } 3755 }
3857 3756
3858 state.df = true; 3757 state.df = true;
3859 break; 3758 break;
3860 3759
3861 case INSN_CLD: 3760 case INSN_CLD:
3862 if (!state.df && func 3761 if (!state.df && func) {
3863 WARN_INSN(ins 3762 WARN_INSN(insn, "redundant CLD");
3864 return 1; 3763 return 1;
3865 } 3764 }
3866 3765
3867 state.df = false; 3766 state.df = false;
3868 break; 3767 break;
3869 3768
3870 default: 3769 default:
3871 break; 3770 break;
3872 } 3771 }
3873 3772
3874 if (insn->dead_end) 3773 if (insn->dead_end)
3875 return 0; 3774 return 0;
3876 3775
3877 if (!next_insn) { 3776 if (!next_insn) {
3878 if (state.cfi.cfa.bas 3777 if (state.cfi.cfa.base == CFI_UNDEFINED)
3879 return 0; 3778 return 0;
3880 WARN("%s: unexpected 3779 WARN("%s: unexpected end of section", sec->name);
3881 return 1; 3780 return 1;
3882 } 3781 }
3883 3782
3884 prev_insn = insn; 3783 prev_insn = insn;
3885 insn = next_insn; 3784 insn = next_insn;
3886 } 3785 }
3887 3786
3888 return 0; 3787 return 0;
3889 } 3788 }
3890 3789
3891 static int validate_unwind_hint(struct objtoo 3790 static int validate_unwind_hint(struct objtool_file *file,
3892 struct inst 3791 struct instruction *insn,
3893 struct insn 3792 struct insn_state *state)
3894 { 3793 {
3895 if (insn->hint && !insn->visited && ! 3794 if (insn->hint && !insn->visited && !insn->ignore) {
3896 int ret = validate_branch(fil 3795 int ret = validate_branch(file, insn_func(insn), insn, *state);
3897 if (ret) 3796 if (ret)
3898 BT_INSN(insn, "<=== ( 3797 BT_INSN(insn, "<=== (hint)");
3899 return ret; 3798 return ret;
3900 } 3799 }
3901 3800
3902 return 0; 3801 return 0;
3903 } 3802 }
3904 3803
3905 static int validate_unwind_hints(struct objto 3804 static int validate_unwind_hints(struct objtool_file *file, struct section *sec)
3906 { 3805 {
3907 struct instruction *insn; 3806 struct instruction *insn;
3908 struct insn_state state; 3807 struct insn_state state;
3909 int warnings = 0; 3808 int warnings = 0;
3910 3809
3911 if (!file->hints) 3810 if (!file->hints)
3912 return 0; 3811 return 0;
3913 3812
3914 init_insn_state(file, &state, sec); 3813 init_insn_state(file, &state, sec);
3915 3814
3916 if (sec) { 3815 if (sec) {
3917 sec_for_each_insn(file, sec, 3816 sec_for_each_insn(file, sec, insn)
3918 warnings += validate_ 3817 warnings += validate_unwind_hint(file, insn, &state);
3919 } else { 3818 } else {
3920 for_each_insn(file, insn) 3819 for_each_insn(file, insn)
3921 warnings += validate_ 3820 warnings += validate_unwind_hint(file, insn, &state);
3922 } 3821 }
3923 3822
3924 return warnings; 3823 return warnings;
3925 } 3824 }
3926 3825
3927 /* 3826 /*
3928 * Validate rethunk entry constraint: must un 3827 * Validate rethunk entry constraint: must untrain RET before the first RET.
3929 * 3828 *
3930 * Follow every branch (intra-function) and e 3829 * Follow every branch (intra-function) and ensure VALIDATE_UNRET_END comes
3931 * before an actual RET instruction. 3830 * before an actual RET instruction.
3932 */ 3831 */
3933 static int validate_unret(struct objtool_file 3832 static int validate_unret(struct objtool_file *file, struct instruction *insn)
3934 { 3833 {
3935 struct instruction *next, *dest; 3834 struct instruction *next, *dest;
3936 int ret; 3835 int ret;
3937 3836
3938 for (;;) { 3837 for (;;) {
3939 next = next_insn_to_validate( 3838 next = next_insn_to_validate(file, insn);
3940 3839
3941 if (insn->visited & VISITED_U 3840 if (insn->visited & VISITED_UNRET)
3942 return 0; 3841 return 0;
3943 3842
3944 insn->visited |= VISITED_UNRE 3843 insn->visited |= VISITED_UNRET;
3945 3844
3946 if (!insn->ignore_alts && ins 3845 if (!insn->ignore_alts && insn->alts) {
3947 struct alternative *a 3846 struct alternative *alt;
3948 bool skip_orig = fals 3847 bool skip_orig = false;
3949 3848
3950 for (alt = insn->alts 3849 for (alt = insn->alts; alt; alt = alt->next) {
3951 if (alt->skip 3850 if (alt->skip_orig)
3952 skip_ 3851 skip_orig = true;
3953 3852
3954 ret = validat 3853 ret = validate_unret(file, alt->insn);
3955 if (ret) { 3854 if (ret) {
3956 BT_IN 3855 BT_INSN(insn, "(alt)");
3957 retur 3856 return ret;
3958 } 3857 }
3959 } 3858 }
3960 3859
3961 if (skip_orig) 3860 if (skip_orig)
3962 return 0; 3861 return 0;
3963 } 3862 }
3964 3863
3965 switch (insn->type) { 3864 switch (insn->type) {
3966 3865
3967 case INSN_CALL_DYNAMIC: 3866 case INSN_CALL_DYNAMIC:
3968 case INSN_JUMP_DYNAMIC: 3867 case INSN_JUMP_DYNAMIC:
3969 case INSN_JUMP_DYNAMIC_CONDIT 3868 case INSN_JUMP_DYNAMIC_CONDITIONAL:
3970 WARN_INSN(insn, "earl 3869 WARN_INSN(insn, "early indirect call");
3971 return 1; 3870 return 1;
3972 3871
3973 case INSN_JUMP_UNCONDITIONAL: 3872 case INSN_JUMP_UNCONDITIONAL:
3974 case INSN_JUMP_CONDITIONAL: 3873 case INSN_JUMP_CONDITIONAL:
3975 if (!is_sibling_call( 3874 if (!is_sibling_call(insn)) {
3976 if (!insn->ju 3875 if (!insn->jump_dest) {
3977 WARN_ 3876 WARN_INSN(insn, "unresolved jump target after linking?!?");
3978 retur 3877 return -1;
3979 } 3878 }
3980 ret = validat 3879 ret = validate_unret(file, insn->jump_dest);
3981 if (ret) { 3880 if (ret) {
3982 BT_IN 3881 BT_INSN(insn, "(branch%s)",
3983 3882 insn->type == INSN_JUMP_CONDITIONAL ? "-cond" : "");
3984 retur 3883 return ret;
3985 } 3884 }
3986 3885
3987 if (insn->typ 3886 if (insn->type == INSN_JUMP_UNCONDITIONAL)
3988 retur 3887 return 0;
3989 3888
3990 break; 3889 break;
3991 } 3890 }
3992 3891
3993 /* fallthrough */ 3892 /* fallthrough */
3994 case INSN_CALL: 3893 case INSN_CALL:
3995 dest = find_insn(file 3894 dest = find_insn(file, insn_call_dest(insn)->sec,
3996 insn 3895 insn_call_dest(insn)->offset);
3997 if (!dest) { 3896 if (!dest) {
3998 WARN("Unresol 3897 WARN("Unresolved function after linking!?: %s",
3999 insn_cal 3898 insn_call_dest(insn)->name);
4000 return -1; 3899 return -1;
4001 } 3900 }
4002 3901
4003 ret = validate_unret( 3902 ret = validate_unret(file, dest);
4004 if (ret) { 3903 if (ret) {
4005 BT_INSN(insn, 3904 BT_INSN(insn, "(call)");
4006 return ret; 3905 return ret;
4007 } 3906 }
4008 /* 3907 /*
4009 * If a call returns 3908 * If a call returns without error, it must have seen UNTRAIN_RET.
4010 * Therefore any non- 3909 * Therefore any non-error return is a success.
4011 */ 3910 */
4012 return 0; 3911 return 0;
4013 3912
4014 case INSN_RETURN: 3913 case INSN_RETURN:
4015 WARN_INSN(insn, "RET 3914 WARN_INSN(insn, "RET before UNTRAIN");
4016 return 1; 3915 return 1;
4017 3916
4018 case INSN_NOP: 3917 case INSN_NOP:
4019 if (insn->retpoline_s 3918 if (insn->retpoline_safe)
4020 return 0; 3919 return 0;
4021 break; 3920 break;
4022 3921
4023 default: 3922 default:
4024 break; 3923 break;
4025 } 3924 }
4026 3925
4027 if (!next) { 3926 if (!next) {
4028 WARN_INSN(insn, "teh 3927 WARN_INSN(insn, "teh end!");
4029 return -1; 3928 return -1;
4030 } 3929 }
4031 insn = next; 3930 insn = next;
4032 } 3931 }
4033 3932
4034 return 0; 3933 return 0;
4035 } 3934 }
4036 3935
4037 /* 3936 /*
4038 * Validate that all branches starting at VAL 3937 * Validate that all branches starting at VALIDATE_UNRET_BEGIN encounter
4039 * VALIDATE_UNRET_END before RET. 3938 * VALIDATE_UNRET_END before RET.
4040 */ 3939 */
4041 static int validate_unrets(struct objtool_fil 3940 static int validate_unrets(struct objtool_file *file)
4042 { 3941 {
4043 struct instruction *insn; 3942 struct instruction *insn;
4044 int ret, warnings = 0; 3943 int ret, warnings = 0;
4045 3944
4046 for_each_insn(file, insn) { 3945 for_each_insn(file, insn) {
4047 if (!insn->unret) 3946 if (!insn->unret)
4048 continue; 3947 continue;
4049 3948
4050 ret = validate_unret(file, in 3949 ret = validate_unret(file, insn);
4051 if (ret < 0) { 3950 if (ret < 0) {
4052 WARN_INSN(insn, "Fail 3951 WARN_INSN(insn, "Failed UNRET validation");
4053 return ret; 3952 return ret;
4054 } 3953 }
4055 warnings += ret; 3954 warnings += ret;
4056 } 3955 }
4057 3956
4058 return warnings; 3957 return warnings;
4059 } 3958 }
4060 3959
4061 static int validate_retpoline(struct objtool_ 3960 static int validate_retpoline(struct objtool_file *file)
4062 { 3961 {
4063 struct instruction *insn; 3962 struct instruction *insn;
4064 int warnings = 0; 3963 int warnings = 0;
4065 3964
4066 for_each_insn(file, insn) { 3965 for_each_insn(file, insn) {
4067 if (insn->type != INSN_JUMP_D 3966 if (insn->type != INSN_JUMP_DYNAMIC &&
4068 insn->type != INSN_CALL_D 3967 insn->type != INSN_CALL_DYNAMIC &&
4069 insn->type != INSN_RETURN 3968 insn->type != INSN_RETURN)
4070 continue; 3969 continue;
4071 3970
4072 if (insn->retpoline_safe) 3971 if (insn->retpoline_safe)
4073 continue; 3972 continue;
4074 3973
4075 if (insn->sec->init) 3974 if (insn->sec->init)
4076 continue; 3975 continue;
4077 3976
4078 if (insn->type == INSN_RETURN 3977 if (insn->type == INSN_RETURN) {
4079 if (opts.rethunk) { 3978 if (opts.rethunk) {
4080 WARN_INSN(ins !! 3979 WARN_INSN(insn, "'naked' return found in RETHUNK build");
4081 } else 3980 } else
4082 continue; 3981 continue;
4083 } else { 3982 } else {
4084 WARN_INSN(insn, "indi !! 3983 WARN_INSN(insn, "indirect %s found in RETPOLINE build",
4085 insn->type 3984 insn->type == INSN_JUMP_DYNAMIC ? "jump" : "call");
4086 } 3985 }
4087 3986
4088 warnings++; 3987 warnings++;
4089 } 3988 }
4090 3989
4091 return warnings; 3990 return warnings;
4092 } 3991 }
4093 3992
4094 static bool is_kasan_insn(struct instruction 3993 static bool is_kasan_insn(struct instruction *insn)
4095 { 3994 {
4096 return (insn->type == INSN_CALL && 3995 return (insn->type == INSN_CALL &&
4097 !strcmp(insn_call_dest(insn)- 3996 !strcmp(insn_call_dest(insn)->name, "__asan_handle_no_return"));
4098 } 3997 }
4099 3998
4100 static bool is_ubsan_insn(struct instruction 3999 static bool is_ubsan_insn(struct instruction *insn)
4101 { 4000 {
4102 return (insn->type == INSN_CALL && 4001 return (insn->type == INSN_CALL &&
4103 !strcmp(insn_call_dest(insn)- 4002 !strcmp(insn_call_dest(insn)->name,
4104 "__ubsan_handle_built 4003 "__ubsan_handle_builtin_unreachable"));
4105 } 4004 }
4106 4005
4107 static bool ignore_unreachable_insn(struct ob 4006 static bool ignore_unreachable_insn(struct objtool_file *file, struct instruction *insn)
4108 { 4007 {
4109 int i; 4008 int i;
4110 struct instruction *prev_insn; 4009 struct instruction *prev_insn;
4111 4010
4112 if (insn->ignore || insn->type == INS 4011 if (insn->ignore || insn->type == INSN_NOP || insn->type == INSN_TRAP)
4113 return true; 4012 return true;
4114 4013
4115 /* 4014 /*
4116 * Ignore alternative replacement ins 4015 * Ignore alternative replacement instructions. This can happen
4117 * when a whitelisted function uses o 4016 * when a whitelisted function uses one of the ALTERNATIVE macros.
4118 */ 4017 */
4119 if (!strcmp(insn->sec->name, ".altins 4018 if (!strcmp(insn->sec->name, ".altinstr_replacement") ||
4120 !strcmp(insn->sec->name, ".altins 4019 !strcmp(insn->sec->name, ".altinstr_aux"))
4121 return true; 4020 return true;
4122 4021
4123 /* 4022 /*
4124 * Whole archive runs might encounter 4023 * Whole archive runs might encounter dead code from weak symbols.
4125 * This is where the linker will have 4024 * This is where the linker will have dropped the weak symbol in
4126 * favour of a regular symbol, but le 4025 * favour of a regular symbol, but leaves the code in place.
4127 * 4026 *
4128 * In this case we'll find a piece of 4027 * In this case we'll find a piece of code (whole function) that is not
4129 * covered by a !section symbol. Igno 4028 * covered by a !section symbol. Ignore them.
4130 */ 4029 */
4131 if (opts.link && !insn_func(insn)) { 4030 if (opts.link && !insn_func(insn)) {
4132 int size = find_symbol_hole_c 4031 int size = find_symbol_hole_containing(insn->sec, insn->offset);
4133 unsigned long end = insn->off 4032 unsigned long end = insn->offset + size;
4134 4033
4135 if (!size) /* not a hole */ 4034 if (!size) /* not a hole */
4136 return false; 4035 return false;
4137 4036
4138 if (size < 0) /* hole until t 4037 if (size < 0) /* hole until the end */
4139 return true; 4038 return true;
4140 4039
4141 sec_for_each_insn_continue(fi 4040 sec_for_each_insn_continue(file, insn) {
4142 /* 4041 /*
4143 * If we reach a visi 4042 * If we reach a visited instruction at or before the
4144 * end of the hole, i 4043 * end of the hole, ignore the unreachable.
4145 */ 4044 */
4146 if (insn->visited) 4045 if (insn->visited)
4147 return true; 4046 return true;
4148 4047
4149 if (insn->offset >= e 4048 if (insn->offset >= end)
4150 break; 4049 break;
4151 4050
4152 /* 4051 /*
4153 * If this hole jumps 4052 * If this hole jumps to a .cold function, mark it ignore too.
4154 */ 4053 */
4155 if (insn->jump_dest & 4054 if (insn->jump_dest && insn_func(insn->jump_dest) &&
4156 strstr(insn_func( 4055 strstr(insn_func(insn->jump_dest)->name, ".cold")) {
4157 struct instru 4056 struct instruction *dest = insn->jump_dest;
4158 func_for_each 4057 func_for_each_insn(file, insn_func(dest), dest)
4159 dest- 4058 dest->ignore = true;
4160 } 4059 }
4161 } 4060 }
4162 4061
4163 return false; 4062 return false;
4164 } 4063 }
4165 4064
4166 if (!insn_func(insn)) 4065 if (!insn_func(insn))
4167 return false; 4066 return false;
4168 4067
4169 if (insn_func(insn)->static_call_tram 4068 if (insn_func(insn)->static_call_tramp)
4170 return true; 4069 return true;
4171 4070
4172 /* 4071 /*
4173 * CONFIG_UBSAN_TRAP inserts a UD2 wh 4072 * CONFIG_UBSAN_TRAP inserts a UD2 when it sees
4174 * __builtin_unreachable(). The BUG( 4073 * __builtin_unreachable(). The BUG() macro has an unreachable() after
4175 * the UD2, which causes GCC's undefi 4074 * the UD2, which causes GCC's undefined trap logic to emit another UD2
4176 * (or occasionally a JMP to UD2). 4075 * (or occasionally a JMP to UD2).
4177 * 4076 *
4178 * It may also insert a UD2 after cal 4077 * It may also insert a UD2 after calling a __noreturn function.
4179 */ 4078 */
4180 prev_insn = prev_insn_same_sec(file, 4079 prev_insn = prev_insn_same_sec(file, insn);
4181 if (prev_insn->dead_end && 4080 if (prev_insn->dead_end &&
4182 (insn->type == INSN_BUG || 4081 (insn->type == INSN_BUG ||
4183 (insn->type == INSN_JUMP_UNCONDI 4082 (insn->type == INSN_JUMP_UNCONDITIONAL &&
4184 insn->jump_dest && insn->jump_d 4083 insn->jump_dest && insn->jump_dest->type == INSN_BUG)))
4185 return true; 4084 return true;
4186 4085
4187 /* 4086 /*
4188 * Check if this (or a subsequent) in 4087 * Check if this (or a subsequent) instruction is related to
4189 * CONFIG_UBSAN or CONFIG_KASAN. 4088 * CONFIG_UBSAN or CONFIG_KASAN.
4190 * 4089 *
4191 * End the search at 5 instructions t 4090 * End the search at 5 instructions to avoid going into the weeds.
4192 */ 4091 */
4193 for (i = 0; i < 5; i++) { 4092 for (i = 0; i < 5; i++) {
4194 4093
4195 if (is_kasan_insn(insn) || is 4094 if (is_kasan_insn(insn) || is_ubsan_insn(insn))
4196 return true; 4095 return true;
4197 4096
4198 if (insn->type == INSN_JUMP_U 4097 if (insn->type == INSN_JUMP_UNCONDITIONAL) {
4199 if (insn->jump_dest & 4098 if (insn->jump_dest &&
4200 insn_func(insn->j 4099 insn_func(insn->jump_dest) == insn_func(insn)) {
4201 insn = insn-> 4100 insn = insn->jump_dest;
4202 continue; 4101 continue;
4203 } 4102 }
4204 4103
4205 break; 4104 break;
4206 } 4105 }
4207 4106
4208 if (insn->offset + insn->len 4107 if (insn->offset + insn->len >= insn_func(insn)->offset + insn_func(insn)->len)
4209 break; 4108 break;
4210 4109
4211 insn = next_insn_same_sec(fil 4110 insn = next_insn_same_sec(file, insn);
4212 } 4111 }
4213 4112
4214 return false; 4113 return false;
4215 } 4114 }
4216 4115
4217 static int add_prefix_symbol(struct objtool_f 4116 static int add_prefix_symbol(struct objtool_file *file, struct symbol *func)
4218 { 4117 {
4219 struct instruction *insn, *prev; 4118 struct instruction *insn, *prev;
4220 struct cfi_state *cfi; 4119 struct cfi_state *cfi;
4221 4120
4222 insn = find_insn(file, func->sec, fun 4121 insn = find_insn(file, func->sec, func->offset);
4223 if (!insn) 4122 if (!insn)
4224 return -1; 4123 return -1;
4225 4124
4226 for (prev = prev_insn_same_sec(file, 4125 for (prev = prev_insn_same_sec(file, insn);
4227 prev; 4126 prev;
4228 prev = prev_insn_same_sec(file, 4127 prev = prev_insn_same_sec(file, prev)) {
4229 u64 offset; 4128 u64 offset;
4230 4129
4231 if (prev->type != INSN_NOP) 4130 if (prev->type != INSN_NOP)
4232 return -1; 4131 return -1;
4233 4132
4234 offset = func->offset - prev- 4133 offset = func->offset - prev->offset;
4235 4134
4236 if (offset > opts.prefix) 4135 if (offset > opts.prefix)
4237 return -1; 4136 return -1;
4238 4137
4239 if (offset < opts.prefix) 4138 if (offset < opts.prefix)
4240 continue; 4139 continue;
4241 4140
4242 elf_create_prefix_symbol(file 4141 elf_create_prefix_symbol(file->elf, func, opts.prefix);
4243 break; 4142 break;
4244 } 4143 }
4245 4144
4246 if (!prev) 4145 if (!prev)
4247 return -1; 4146 return -1;
4248 4147
4249 if (!insn->cfi) { 4148 if (!insn->cfi) {
4250 /* 4149 /*
4251 * This can happen if stack v 4150 * This can happen if stack validation isn't enabled or the
4252 * function is annotated with 4151 * function is annotated with STACK_FRAME_NON_STANDARD.
4253 */ 4152 */
4254 return 0; 4153 return 0;
4255 } 4154 }
4256 4155
4257 /* Propagate insn->cfi to the prefix 4156 /* Propagate insn->cfi to the prefix code */
4258 cfi = cfi_hash_find_or_add(insn->cfi) 4157 cfi = cfi_hash_find_or_add(insn->cfi);
4259 for (; prev != insn; prev = next_insn 4158 for (; prev != insn; prev = next_insn_same_sec(file, prev))
4260 prev->cfi = cfi; 4159 prev->cfi = cfi;
4261 4160
4262 return 0; 4161 return 0;
4263 } 4162 }
4264 4163
4265 static int add_prefix_symbols(struct objtool_ 4164 static int add_prefix_symbols(struct objtool_file *file)
4266 { 4165 {
4267 struct section *sec; 4166 struct section *sec;
4268 struct symbol *func; 4167 struct symbol *func;
4269 4168
4270 for_each_sec(file, sec) { 4169 for_each_sec(file, sec) {
4271 if (!(sec->sh.sh_flags & SHF_ 4170 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
4272 continue; 4171 continue;
4273 4172
4274 sec_for_each_sym(sec, func) { 4173 sec_for_each_sym(sec, func) {
4275 if (func->type != STT 4174 if (func->type != STT_FUNC)
4276 continue; 4175 continue;
4277 4176
4278 add_prefix_symbol(fil 4177 add_prefix_symbol(file, func);
4279 } 4178 }
4280 } 4179 }
4281 4180
4282 return 0; 4181 return 0;
4283 } 4182 }
4284 4183
4285 static int validate_symbol(struct objtool_fil 4184 static int validate_symbol(struct objtool_file *file, struct section *sec,
4286 struct symbol *sym 4185 struct symbol *sym, struct insn_state *state)
4287 { 4186 {
4288 struct instruction *insn; 4187 struct instruction *insn;
4289 int ret; 4188 int ret;
4290 4189
4291 if (!sym->len) { 4190 if (!sym->len) {
4292 WARN("%s() is missing an ELF 4191 WARN("%s() is missing an ELF size annotation", sym->name);
4293 return 1; 4192 return 1;
4294 } 4193 }
4295 4194
4296 if (sym->pfunc != sym || sym->alias ! 4195 if (sym->pfunc != sym || sym->alias != sym)
4297 return 0; 4196 return 0;
4298 4197
4299 insn = find_insn(file, sec, sym->offs 4198 insn = find_insn(file, sec, sym->offset);
4300 if (!insn || insn->ignore || insn->vi 4199 if (!insn || insn->ignore || insn->visited)
4301 return 0; 4200 return 0;
4302 4201
4303 state->uaccess = sym->uaccess_safe; 4202 state->uaccess = sym->uaccess_safe;
4304 4203
4305 ret = validate_branch(file, insn_func 4204 ret = validate_branch(file, insn_func(insn), insn, *state);
4306 if (ret) 4205 if (ret)
4307 BT_INSN(insn, "<=== (sym)"); 4206 BT_INSN(insn, "<=== (sym)");
4308 return ret; 4207 return ret;
4309 } 4208 }
4310 4209
4311 static int validate_section(struct objtool_fi 4210 static int validate_section(struct objtool_file *file, struct section *sec)
4312 { 4211 {
4313 struct insn_state state; 4212 struct insn_state state;
4314 struct symbol *func; 4213 struct symbol *func;
4315 int warnings = 0; 4214 int warnings = 0;
4316 4215
4317 sec_for_each_sym(sec, func) { 4216 sec_for_each_sym(sec, func) {
4318 if (func->type != STT_FUNC) 4217 if (func->type != STT_FUNC)
4319 continue; 4218 continue;
4320 4219
4321 init_insn_state(file, &state, 4220 init_insn_state(file, &state, sec);
4322 set_func_state(&state.cfi); 4221 set_func_state(&state.cfi);
4323 4222
4324 warnings += validate_symbol(f 4223 warnings += validate_symbol(file, sec, func, &state);
4325 } 4224 }
4326 4225
4327 return warnings; 4226 return warnings;
4328 } 4227 }
4329 4228
4330 static int validate_noinstr_sections(struct o 4229 static int validate_noinstr_sections(struct objtool_file *file)
4331 { 4230 {
4332 struct section *sec; 4231 struct section *sec;
4333 int warnings = 0; 4232 int warnings = 0;
4334 4233
4335 sec = find_section_by_name(file->elf, 4234 sec = find_section_by_name(file->elf, ".noinstr.text");
4336 if (sec) { 4235 if (sec) {
4337 warnings += validate_section( 4236 warnings += validate_section(file, sec);
4338 warnings += validate_unwind_h 4237 warnings += validate_unwind_hints(file, sec);
4339 } 4238 }
4340 4239
4341 sec = find_section_by_name(file->elf, 4240 sec = find_section_by_name(file->elf, ".entry.text");
4342 if (sec) { 4241 if (sec) {
4343 warnings += validate_section( 4242 warnings += validate_section(file, sec);
4344 warnings += validate_unwind_h 4243 warnings += validate_unwind_hints(file, sec);
4345 } 4244 }
4346 4245
4347 sec = find_section_by_name(file->elf, 4246 sec = find_section_by_name(file->elf, ".cpuidle.text");
4348 if (sec) { 4247 if (sec) {
4349 warnings += validate_section( 4248 warnings += validate_section(file, sec);
4350 warnings += validate_unwind_h 4249 warnings += validate_unwind_hints(file, sec);
4351 } 4250 }
4352 4251
4353 return warnings; 4252 return warnings;
4354 } 4253 }
4355 4254
4356 static int validate_functions(struct objtool_ 4255 static int validate_functions(struct objtool_file *file)
4357 { 4256 {
4358 struct section *sec; 4257 struct section *sec;
4359 int warnings = 0; 4258 int warnings = 0;
4360 4259
4361 for_each_sec(file, sec) { 4260 for_each_sec(file, sec) {
4362 if (!(sec->sh.sh_flags & SHF_ 4261 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
4363 continue; 4262 continue;
4364 4263
4365 warnings += validate_section( 4264 warnings += validate_section(file, sec);
4366 } 4265 }
4367 4266
4368 return warnings; 4267 return warnings;
4369 } 4268 }
4370 4269
4371 static void mark_endbr_used(struct instructio 4270 static void mark_endbr_used(struct instruction *insn)
4372 { 4271 {
4373 if (!list_empty(&insn->call_node)) 4272 if (!list_empty(&insn->call_node))
4374 list_del_init(&insn->call_nod 4273 list_del_init(&insn->call_node);
4375 } 4274 }
4376 4275
4377 static bool noendbr_range(struct objtool_file 4276 static bool noendbr_range(struct objtool_file *file, struct instruction *insn)
4378 { 4277 {
4379 struct symbol *sym = find_symbol_cont 4278 struct symbol *sym = find_symbol_containing(insn->sec, insn->offset-1);
4380 struct instruction *first; 4279 struct instruction *first;
4381 4280
4382 if (!sym) 4281 if (!sym)
4383 return false; 4282 return false;
4384 4283
4385 first = find_insn(file, sym->sec, sym 4284 first = find_insn(file, sym->sec, sym->offset);
4386 if (!first) 4285 if (!first)
4387 return false; 4286 return false;
4388 4287
4389 if (first->type != INSN_ENDBR && !fir 4288 if (first->type != INSN_ENDBR && !first->noendbr)
4390 return false; 4289 return false;
4391 4290
4392 return insn->offset == sym->offset + 4291 return insn->offset == sym->offset + sym->len;
4393 } 4292 }
4394 4293
4395 static int validate_ibt_insn(struct objtool_f 4294 static int validate_ibt_insn(struct objtool_file *file, struct instruction *insn)
4396 { 4295 {
4397 struct instruction *dest; 4296 struct instruction *dest;
4398 struct reloc *reloc; 4297 struct reloc *reloc;
4399 unsigned long off; 4298 unsigned long off;
4400 int warnings = 0; 4299 int warnings = 0;
4401 4300
4402 /* 4301 /*
4403 * Looking for function pointer load 4302 * Looking for function pointer load relocations. Ignore
4404 * direct/indirect branches: 4303 * direct/indirect branches:
4405 */ 4304 */
4406 switch (insn->type) { 4305 switch (insn->type) {
4407 case INSN_CALL: 4306 case INSN_CALL:
4408 case INSN_CALL_DYNAMIC: 4307 case INSN_CALL_DYNAMIC:
4409 case INSN_JUMP_CONDITIONAL: 4308 case INSN_JUMP_CONDITIONAL:
4410 case INSN_JUMP_UNCONDITIONAL: 4309 case INSN_JUMP_UNCONDITIONAL:
4411 case INSN_JUMP_DYNAMIC: 4310 case INSN_JUMP_DYNAMIC:
4412 case INSN_JUMP_DYNAMIC_CONDITIONAL: 4311 case INSN_JUMP_DYNAMIC_CONDITIONAL:
4413 case INSN_RETURN: 4312 case INSN_RETURN:
4414 case INSN_NOP: 4313 case INSN_NOP:
4415 return 0; 4314 return 0;
4416 default: 4315 default:
4417 break; 4316 break;
4418 } 4317 }
4419 4318
4420 for (reloc = insn_reloc(file, insn); 4319 for (reloc = insn_reloc(file, insn);
4421 reloc; 4320 reloc;
4422 reloc = find_reloc_by_dest_range 4321 reloc = find_reloc_by_dest_range(file->elf, insn->sec,
4423 4322 reloc_offset(reloc) + 1,
4424 4323 (insn->offset + insn->len) - (reloc_offset(reloc) + 1))) {
4425 4324
4426 /* 4325 /*
4427 * static_call_update() refer 4326 * static_call_update() references the trampoline, which
4428 * doesn't have (or need) END 4327 * doesn't have (or need) ENDBR. Skip warning in that case.
4429 */ 4328 */
4430 if (reloc->sym->static_call_t 4329 if (reloc->sym->static_call_tramp)
4431 continue; 4330 continue;
4432 4331
4433 off = reloc->sym->offset; 4332 off = reloc->sym->offset;
4434 if (reloc_type(reloc) == R_X8 4333 if (reloc_type(reloc) == R_X86_64_PC32 ||
4435 reloc_type(reloc) == R_X8 4334 reloc_type(reloc) == R_X86_64_PLT32)
4436 off += arch_dest_relo 4335 off += arch_dest_reloc_offset(reloc_addend(reloc));
4437 else 4336 else
4438 off += reloc_addend(r 4337 off += reloc_addend(reloc);
4439 4338
4440 dest = find_insn(file, reloc- 4339 dest = find_insn(file, reloc->sym->sec, off);
4441 if (!dest) 4340 if (!dest)
4442 continue; 4341 continue;
4443 4342
4444 if (dest->type == INSN_ENDBR) 4343 if (dest->type == INSN_ENDBR) {
4445 mark_endbr_used(dest) 4344 mark_endbr_used(dest);
4446 continue; 4345 continue;
4447 } 4346 }
4448 4347
4449 if (insn_func(dest) && insn_f 4348 if (insn_func(dest) && insn_func(insn) &&
4450 insn_func(dest)->pfunc == 4349 insn_func(dest)->pfunc == insn_func(insn)->pfunc) {
4451 /* 4350 /*
4452 * Anything from->to 4351 * Anything from->to self is either _THIS_IP_ or
4453 * IRET-to-self. 4352 * IRET-to-self.
4454 * 4353 *
4455 * There is no sane w 4354 * There is no sane way to annotate _THIS_IP_ since the
4456 * compiler treats th 4355 * compiler treats the relocation as a constant and is
4457 * happy to fold in o 4356 * happy to fold in offsets, skewing any annotation we
4458 * do, leading to vas 4357 * do, leading to vast amounts of false-positives.
4459 * 4358 *
4460 * There's also compi 4359 * There's also compiler generated _THIS_IP_ through
4461 * KCOV and such whic 4360 * KCOV and such which we have no hope of annotating.
4462 * 4361 *
4463 * As such, blanket a 4362 * As such, blanket accept self-references without
4464 * issue. 4363 * issue.
4465 */ 4364 */
4466 continue; 4365 continue;
4467 } 4366 }
4468 4367
4469 /* 4368 /*
4470 * Accept anything ANNOTATE_N 4369 * Accept anything ANNOTATE_NOENDBR.
4471 */ 4370 */
4472 if (dest->noendbr) 4371 if (dest->noendbr)
4473 continue; 4372 continue;
4474 4373
4475 /* 4374 /*
4476 * Accept if this is the inst 4375 * Accept if this is the instruction after a symbol
4477 * that is (no)endbr -- typic 4376 * that is (no)endbr -- typical code-range usage.
4478 */ 4377 */
4479 if (noendbr_range(file, dest) 4378 if (noendbr_range(file, dest))
4480 continue; 4379 continue;
4481 4380
4482 WARN_INSN(insn, "relocation t 4381 WARN_INSN(insn, "relocation to !ENDBR: %s", offstr(dest->sec, dest->offset));
4483 4382
4484 warnings++; 4383 warnings++;
4485 } 4384 }
4486 4385
4487 return warnings; 4386 return warnings;
4488 } 4387 }
4489 4388
4490 static int validate_ibt_data_reloc(struct obj 4389 static int validate_ibt_data_reloc(struct objtool_file *file,
4491 struct rel 4390 struct reloc *reloc)
4492 { 4391 {
4493 struct instruction *dest; 4392 struct instruction *dest;
4494 4393
4495 dest = find_insn(file, reloc->sym->se 4394 dest = find_insn(file, reloc->sym->sec,
4496 reloc->sym->offset + 4395 reloc->sym->offset + reloc_addend(reloc));
4497 if (!dest) 4396 if (!dest)
4498 return 0; 4397 return 0;
4499 4398
4500 if (dest->type == INSN_ENDBR) { 4399 if (dest->type == INSN_ENDBR) {
4501 mark_endbr_used(dest); 4400 mark_endbr_used(dest);
4502 return 0; 4401 return 0;
4503 } 4402 }
4504 4403
4505 if (dest->noendbr) 4404 if (dest->noendbr)
4506 return 0; 4405 return 0;
4507 4406
4508 WARN_FUNC("data relocation to !ENDBR: 4407 WARN_FUNC("data relocation to !ENDBR: %s",
4509 reloc->sec->base, reloc_off 4408 reloc->sec->base, reloc_offset(reloc),
4510 offstr(dest->sec, dest->off 4409 offstr(dest->sec, dest->offset));
4511 4410
4512 return 1; 4411 return 1;
4513 } 4412 }
4514 4413
4515 /* 4414 /*
4516 * Validate IBT rules and remove used ENDBR i 4415 * Validate IBT rules and remove used ENDBR instructions from the seal list.
4517 * Unused ENDBR instructions will be annotate 4416 * Unused ENDBR instructions will be annotated for sealing (i.e., replaced with
4518 * NOPs) later, in create_ibt_endbr_seal_sect 4417 * NOPs) later, in create_ibt_endbr_seal_sections().
4519 */ 4418 */
4520 static int validate_ibt(struct objtool_file * 4419 static int validate_ibt(struct objtool_file *file)
4521 { 4420 {
4522 struct section *sec; 4421 struct section *sec;
4523 struct reloc *reloc; 4422 struct reloc *reloc;
4524 struct instruction *insn; 4423 struct instruction *insn;
4525 int warnings = 0; 4424 int warnings = 0;
4526 4425
4527 for_each_insn(file, insn) 4426 for_each_insn(file, insn)
4528 warnings += validate_ibt_insn 4427 warnings += validate_ibt_insn(file, insn);
4529 4428
4530 for_each_sec(file, sec) { 4429 for_each_sec(file, sec) {
4531 4430
4532 /* Already done by validate_i 4431 /* Already done by validate_ibt_insn() */
4533 if (sec->sh.sh_flags & SHF_EX 4432 if (sec->sh.sh_flags & SHF_EXECINSTR)
4534 continue; 4433 continue;
4535 4434
4536 if (!sec->rsec) 4435 if (!sec->rsec)
4537 continue; 4436 continue;
4538 4437
4539 /* 4438 /*
4540 * These sections can referen 4439 * These sections can reference text addresses, but not with
4541 * the intent to indirect bra 4440 * the intent to indirect branch to them.
4542 */ 4441 */
4543 if ((!strncmp(sec->name, ".di 4442 if ((!strncmp(sec->name, ".discard", 8) &&
4544 strcmp(sec->name, ".disc 4443 strcmp(sec->name, ".discard.ibt_endbr_noseal")) ||
4545 !strncmp(sec->name, ".deb 4444 !strncmp(sec->name, ".debug", 6) ||
4546 !strcmp(sec->name, ".alti 4445 !strcmp(sec->name, ".altinstructions") ||
4547 !strcmp(sec->name, ".ibt_ 4446 !strcmp(sec->name, ".ibt_endbr_seal") ||
4548 !strcmp(sec->name, ".orc_ 4447 !strcmp(sec->name, ".orc_unwind_ip") ||
4549 !strcmp(sec->name, ".para 4448 !strcmp(sec->name, ".parainstructions") ||
4550 !strcmp(sec->name, ".retp 4449 !strcmp(sec->name, ".retpoline_sites") ||
4551 !strcmp(sec->name, ".smp_ 4450 !strcmp(sec->name, ".smp_locks") ||
4552 !strcmp(sec->name, ".stat 4451 !strcmp(sec->name, ".static_call_sites") ||
4553 !strcmp(sec->name, "_erro 4452 !strcmp(sec->name, "_error_injection_whitelist") ||
4554 !strcmp(sec->name, "_kpro 4453 !strcmp(sec->name, "_kprobe_blacklist") ||
4555 !strcmp(sec->name, "__bug 4454 !strcmp(sec->name, "__bug_table") ||
4556 !strcmp(sec->name, "__ex_ 4455 !strcmp(sec->name, "__ex_table") ||
4557 !strcmp(sec->name, "__jum 4456 !strcmp(sec->name, "__jump_table") ||
4558 !strcmp(sec->name, "__mco 4457 !strcmp(sec->name, "__mcount_loc") ||
4559 !strcmp(sec->name, ".kcfi 4458 !strcmp(sec->name, ".kcfi_traps") ||
4560 strstr(sec->name, "__patc 4459 strstr(sec->name, "__patchable_function_entries"))
4561 continue; 4460 continue;
4562 4461
4563 for_each_reloc(sec->rsec, rel 4462 for_each_reloc(sec->rsec, reloc)
4564 warnings += validate_ 4463 warnings += validate_ibt_data_reloc(file, reloc);
4565 } 4464 }
4566 4465
4567 return warnings; 4466 return warnings;
4568 } 4467 }
4569 4468
4570 static int validate_sls(struct objtool_file * 4469 static int validate_sls(struct objtool_file *file)
4571 { 4470 {
4572 struct instruction *insn, *next_insn; 4471 struct instruction *insn, *next_insn;
4573 int warnings = 0; 4472 int warnings = 0;
4574 4473
4575 for_each_insn(file, insn) { 4474 for_each_insn(file, insn) {
4576 next_insn = next_insn_same_se 4475 next_insn = next_insn_same_sec(file, insn);
4577 4476
4578 if (insn->retpoline_safe) 4477 if (insn->retpoline_safe)
4579 continue; 4478 continue;
4580 4479
4581 switch (insn->type) { 4480 switch (insn->type) {
4582 case INSN_RETURN: 4481 case INSN_RETURN:
4583 if (!next_insn || nex 4482 if (!next_insn || next_insn->type != INSN_TRAP) {
4584 WARN_INSN(ins 4483 WARN_INSN(insn, "missing int3 after ret");
4585 warnings++; 4484 warnings++;
4586 } 4485 }
4587 4486
4588 break; 4487 break;
4589 case INSN_JUMP_DYNAMIC: 4488 case INSN_JUMP_DYNAMIC:
4590 if (!next_insn || nex 4489 if (!next_insn || next_insn->type != INSN_TRAP) {
4591 WARN_INSN(ins 4490 WARN_INSN(insn, "missing int3 after indirect jump");
4592 warnings++; 4491 warnings++;
4593 } 4492 }
4594 break; 4493 break;
4595 default: 4494 default:
4596 break; 4495 break;
4597 } 4496 }
4598 } 4497 }
4599 4498
4600 return warnings; 4499 return warnings;
4601 } 4500 }
4602 4501
4603 static bool ignore_noreturn_call(struct instr 4502 static bool ignore_noreturn_call(struct instruction *insn)
4604 { 4503 {
4605 struct symbol *call_dest = insn_call_ 4504 struct symbol *call_dest = insn_call_dest(insn);
4606 4505
4607 /* 4506 /*
4608 * FIXME: hack, we need a real noretu 4507 * FIXME: hack, we need a real noreturn solution
4609 * 4508 *
4610 * Problem is, exc_double_fault() may 4509 * Problem is, exc_double_fault() may or may not return, depending on
4611 * whether CONFIG_X86_ESPFIX64 is set 4510 * whether CONFIG_X86_ESPFIX64 is set. But objtool has no visibility
4612 * to the kernel config. 4511 * to the kernel config.
4613 * 4512 *
4614 * Other potential ways to fix it: 4513 * Other potential ways to fix it:
4615 * 4514 *
4616 * - have compiler communicate __no 4515 * - have compiler communicate __noreturn functions somehow
4617 * - remove CONFIG_X86_ESPFIX64 4516 * - remove CONFIG_X86_ESPFIX64
4618 * - read the .config file 4517 * - read the .config file
4619 * - add a cmdline option 4518 * - add a cmdline option
4620 * - create a generic objtool annot 4519 * - create a generic objtool annotation format (vs a bunch of custom
4621 * formats) and annotate it 4520 * formats) and annotate it
4622 */ 4521 */
4623 if (!strcmp(call_dest->name, "exc_dou 4522 if (!strcmp(call_dest->name, "exc_double_fault")) {
4624 /* prevent further unreachabl 4523 /* prevent further unreachable warnings for the caller */
4625 insn->sym->warned = 1; 4524 insn->sym->warned = 1;
4626 return true; 4525 return true;
4627 } 4526 }
4628 4527
4629 return false; 4528 return false;
4630 } 4529 }
4631 4530
4632 static int validate_reachable_instructions(st 4531 static int validate_reachable_instructions(struct objtool_file *file)
4633 { 4532 {
4634 struct instruction *insn, *prev_insn; 4533 struct instruction *insn, *prev_insn;
4635 struct symbol *call_dest; 4534 struct symbol *call_dest;
4636 int warnings = 0; 4535 int warnings = 0;
4637 4536
4638 if (file->ignore_unreachables) 4537 if (file->ignore_unreachables)
4639 return 0; 4538 return 0;
4640 4539
4641 for_each_insn(file, insn) { 4540 for_each_insn(file, insn) {
4642 if (insn->visited || ignore_u 4541 if (insn->visited || ignore_unreachable_insn(file, insn))
4643 continue; 4542 continue;
4644 4543
4645 prev_insn = prev_insn_same_se 4544 prev_insn = prev_insn_same_sec(file, insn);
4646 if (prev_insn && prev_insn->d 4545 if (prev_insn && prev_insn->dead_end) {
4647 call_dest = insn_call 4546 call_dest = insn_call_dest(prev_insn);
4648 if (call_dest && !ign 4547 if (call_dest && !ignore_noreturn_call(prev_insn)) {
4649 WARN_INSN(ins 4548 WARN_INSN(insn, "%s() is missing a __noreturn annotation",
4650 cal 4549 call_dest->name);
4651 warnings++; 4550 warnings++;
4652 continue; 4551 continue;
4653 } 4552 }
4654 } 4553 }
4655 4554
4656 WARN_INSN(insn, "unreachable 4555 WARN_INSN(insn, "unreachable instruction");
4657 warnings++; 4556 warnings++;
4658 } 4557 }
4659 4558
4660 return warnings; 4559 return warnings;
4661 } 4560 }
4662 4561
4663 /* 'funcs' is a space-separated list of funct 4562 /* 'funcs' is a space-separated list of function names */
4664 static int disas_funcs(const char *funcs) 4563 static int disas_funcs(const char *funcs)
4665 { 4564 {
4666 const char *objdump_str, *cross_compi 4565 const char *objdump_str, *cross_compile;
4667 int size, ret; 4566 int size, ret;
4668 char *cmd; 4567 char *cmd;
4669 4568
4670 cross_compile = getenv("CROSS_COMPILE 4569 cross_compile = getenv("CROSS_COMPILE");
4671 4570
4672 objdump_str = "%sobjdump -wdr %s | ga 4571 objdump_str = "%sobjdump -wdr %s | gawk -M -v _funcs='%s' '"
4673 "BEGIN { split(_funcs 4572 "BEGIN { split(_funcs, funcs); }"
4674 "/^$/ { func_match = 4573 "/^$/ { func_match = 0; }"
4675 "/<.*>:/ { " 4574 "/<.*>:/ { "
4676 "f = gensub(/ 4575 "f = gensub(/.*<(.*)>:/, \"\\\\1\", 1);"
4677 "for (i in fu 4576 "for (i in funcs) {"
4678 "if ( 4577 "if (funcs[i] == f) {"
4679 4578 "func_match = 1;"
4680 4579 "base = strtonum(\"0x\" $1);"
4681 4580 "break;"
4682 "}" 4581 "}"
4683 "}" 4582 "}"
4684 "}" 4583 "}"
4685 "{" 4584 "{"
4686 "if (func_mat 4585 "if (func_match) {"
4687 "addr 4586 "addr = strtonum(\"0x\" $1);"
4688 "prin 4587 "printf(\"%%04x \", addr - base);"
4689 "prin 4588 "print;"
4690 "}" 4589 "}"
4691 "}' 1>&2"; 4590 "}' 1>&2";
4692 4591
4693 /* fake snprintf() to calculate the s 4592 /* fake snprintf() to calculate the size */
4694 size = snprintf(NULL, 0, objdump_str, 4593 size = snprintf(NULL, 0, objdump_str, cross_compile, objname, funcs) + 1;
4695 if (size <= 0) { 4594 if (size <= 0) {
4696 WARN("objdump string size cal 4595 WARN("objdump string size calculation failed");
4697 return -1; 4596 return -1;
4698 } 4597 }
4699 4598
4700 cmd = malloc(size); 4599 cmd = malloc(size);
4701 4600
4702 /* real snprintf() */ 4601 /* real snprintf() */
4703 snprintf(cmd, size, objdump_str, cros 4602 snprintf(cmd, size, objdump_str, cross_compile, objname, funcs);
4704 ret = system(cmd); 4603 ret = system(cmd);
4705 if (ret) { 4604 if (ret) {
4706 WARN("disassembly failed: %d" 4605 WARN("disassembly failed: %d", ret);
4707 return -1; 4606 return -1;
4708 } 4607 }
4709 4608
4710 return 0; 4609 return 0;
4711 } 4610 }
4712 4611
4713 static int disas_warned_funcs(struct objtool_ 4612 static int disas_warned_funcs(struct objtool_file *file)
4714 { 4613 {
4715 struct symbol *sym; 4614 struct symbol *sym;
4716 char *funcs = NULL, *tmp; 4615 char *funcs = NULL, *tmp;
4717 4616
4718 for_each_sym(file, sym) { 4617 for_each_sym(file, sym) {
4719 if (sym->warned) { 4618 if (sym->warned) {
4720 if (!funcs) { 4619 if (!funcs) {
4721 funcs = mallo 4620 funcs = malloc(strlen(sym->name) + 1);
4722 strcpy(funcs, 4621 strcpy(funcs, sym->name);
4723 } else { 4622 } else {
4724 tmp = malloc( 4623 tmp = malloc(strlen(funcs) + strlen(sym->name) + 2);
4725 sprintf(tmp, 4624 sprintf(tmp, "%s %s", funcs, sym->name);
4726 free(funcs); 4625 free(funcs);
4727 funcs = tmp; 4626 funcs = tmp;
4728 } 4627 }
4729 } 4628 }
4730 } 4629 }
4731 4630
4732 if (funcs) 4631 if (funcs)
4733 disas_funcs(funcs); 4632 disas_funcs(funcs);
4734 4633
4735 return 0; 4634 return 0;
4736 } 4635 }
4737 4636
4738 struct insn_chunk { 4637 struct insn_chunk {
4739 void *addr; 4638 void *addr;
4740 struct insn_chunk *next; 4639 struct insn_chunk *next;
4741 }; 4640 };
4742 4641
4743 /* 4642 /*
4744 * Reduce peak RSS usage by freeing insns mem 4643 * Reduce peak RSS usage by freeing insns memory before writing the ELF file,
4745 * which can trigger more allocations for .de 4644 * which can trigger more allocations for .debug_* sections whose data hasn't
4746 * been read yet. 4645 * been read yet.
4747 */ 4646 */
4748 static void free_insns(struct objtool_file *f 4647 static void free_insns(struct objtool_file *file)
4749 { 4648 {
4750 struct instruction *insn; 4649 struct instruction *insn;
4751 struct insn_chunk *chunks = NULL, *ch 4650 struct insn_chunk *chunks = NULL, *chunk;
4752 4651
4753 for_each_insn(file, insn) { 4652 for_each_insn(file, insn) {
4754 if (!insn->idx) { 4653 if (!insn->idx) {
4755 chunk = malloc(sizeof 4654 chunk = malloc(sizeof(*chunk));
4756 chunk->addr = insn; 4655 chunk->addr = insn;
4757 chunk->next = chunks; 4656 chunk->next = chunks;
4758 chunks = chunk; 4657 chunks = chunk;
4759 } 4658 }
4760 } 4659 }
4761 4660
4762 for (chunk = chunks; chunk; chunk = c 4661 for (chunk = chunks; chunk; chunk = chunk->next)
4763 free(chunk->addr); 4662 free(chunk->addr);
4764 } 4663 }
4765 4664
4766 int check(struct objtool_file *file) 4665 int check(struct objtool_file *file)
4767 { 4666 {
4768 int ret, warnings = 0; 4667 int ret, warnings = 0;
4769 4668
4770 arch_initial_func_cfi_state(&initial_ 4669 arch_initial_func_cfi_state(&initial_func_cfi);
4771 init_cfi_state(&init_cfi); 4670 init_cfi_state(&init_cfi);
4772 init_cfi_state(&func_cfi); 4671 init_cfi_state(&func_cfi);
4773 set_func_state(&func_cfi); 4672 set_func_state(&func_cfi);
4774 init_cfi_state(&force_undefined_cfi); 4673 init_cfi_state(&force_undefined_cfi);
4775 force_undefined_cfi.force_undefined = 4674 force_undefined_cfi.force_undefined = true;
4776 4675
4777 if (!cfi_hash_alloc(1UL << (file->elf 4676 if (!cfi_hash_alloc(1UL << (file->elf->symbol_bits - 3)))
4778 goto out; 4677 goto out;
4779 4678
4780 cfi_hash_add(&init_cfi); 4679 cfi_hash_add(&init_cfi);
4781 cfi_hash_add(&func_cfi); 4680 cfi_hash_add(&func_cfi);
4782 4681
4783 ret = decode_sections(file); 4682 ret = decode_sections(file);
4784 if (ret < 0) 4683 if (ret < 0)
4785 goto out; 4684 goto out;
4786 4685
4787 warnings += ret; 4686 warnings += ret;
4788 4687
4789 if (!nr_insns) 4688 if (!nr_insns)
4790 goto out; 4689 goto out;
4791 4690
4792 if (opts.retpoline) { 4691 if (opts.retpoline) {
4793 ret = validate_retpoline(file 4692 ret = validate_retpoline(file);
4794 if (ret < 0) 4693 if (ret < 0)
4795 return ret; 4694 return ret;
4796 warnings += ret; 4695 warnings += ret;
4797 } 4696 }
4798 4697
4799 if (opts.stackval || opts.orc || opts 4698 if (opts.stackval || opts.orc || opts.uaccess) {
4800 ret = validate_functions(file 4699 ret = validate_functions(file);
4801 if (ret < 0) 4700 if (ret < 0)
4802 goto out; 4701 goto out;
4803 warnings += ret; 4702 warnings += ret;
4804 4703
4805 ret = validate_unwind_hints(f 4704 ret = validate_unwind_hints(file, NULL);
4806 if (ret < 0) 4705 if (ret < 0)
4807 goto out; 4706 goto out;
4808 warnings += ret; 4707 warnings += ret;
4809 4708
4810 if (!warnings) { 4709 if (!warnings) {
4811 ret = validate_reacha 4710 ret = validate_reachable_instructions(file);
4812 if (ret < 0) 4711 if (ret < 0)
4813 goto out; 4712 goto out;
4814 warnings += ret; 4713 warnings += ret;
4815 } 4714 }
4816 4715
4817 } else if (opts.noinstr) { 4716 } else if (opts.noinstr) {
4818 ret = validate_noinstr_sectio 4717 ret = validate_noinstr_sections(file);
4819 if (ret < 0) 4718 if (ret < 0)
4820 goto out; 4719 goto out;
4821 warnings += ret; 4720 warnings += ret;
4822 } 4721 }
4823 4722
4824 if (opts.unret) { 4723 if (opts.unret) {
4825 /* 4724 /*
4826 * Must be after validate_bra 4725 * Must be after validate_branch() and friends, it plays
4827 * further games with insn->v 4726 * further games with insn->visited.
4828 */ 4727 */
4829 ret = validate_unrets(file); 4728 ret = validate_unrets(file);
4830 if (ret < 0) 4729 if (ret < 0)
4831 return ret; 4730 return ret;
4832 warnings += ret; 4731 warnings += ret;
4833 } 4732 }
4834 4733
4835 if (opts.ibt) { 4734 if (opts.ibt) {
4836 ret = validate_ibt(file); 4735 ret = validate_ibt(file);
4837 if (ret < 0) 4736 if (ret < 0)
4838 goto out; 4737 goto out;
4839 warnings += ret; 4738 warnings += ret;
4840 } 4739 }
4841 4740
4842 if (opts.sls) { 4741 if (opts.sls) {
4843 ret = validate_sls(file); 4742 ret = validate_sls(file);
4844 if (ret < 0) 4743 if (ret < 0)
4845 goto out; 4744 goto out;
4846 warnings += ret; 4745 warnings += ret;
4847 } 4746 }
4848 4747
4849 if (opts.static_call) { 4748 if (opts.static_call) {
4850 ret = create_static_call_sect 4749 ret = create_static_call_sections(file);
4851 if (ret < 0) 4750 if (ret < 0)
4852 goto out; 4751 goto out;
4853 warnings += ret; 4752 warnings += ret;
4854 } 4753 }
4855 4754
4856 if (opts.retpoline) { 4755 if (opts.retpoline) {
4857 ret = create_retpoline_sites_ 4756 ret = create_retpoline_sites_sections(file);
4858 if (ret < 0) 4757 if (ret < 0)
4859 goto out; 4758 goto out;
4860 warnings += ret; 4759 warnings += ret;
4861 } 4760 }
4862 4761
4863 if (opts.cfi) { 4762 if (opts.cfi) {
4864 ret = create_cfi_sections(fil 4763 ret = create_cfi_sections(file);
4865 if (ret < 0) 4764 if (ret < 0)
4866 goto out; 4765 goto out;
4867 warnings += ret; 4766 warnings += ret;
4868 } 4767 }
4869 4768
4870 if (opts.rethunk) { 4769 if (opts.rethunk) {
4871 ret = create_return_sites_sec 4770 ret = create_return_sites_sections(file);
4872 if (ret < 0) 4771 if (ret < 0)
4873 goto out; 4772 goto out;
4874 warnings += ret; 4773 warnings += ret;
4875 4774
4876 if (opts.hack_skylake) { 4775 if (opts.hack_skylake) {
4877 ret = create_direct_c 4776 ret = create_direct_call_sections(file);
4878 if (ret < 0) 4777 if (ret < 0)
4879 goto out; 4778 goto out;
4880 warnings += ret; 4779 warnings += ret;
4881 } 4780 }
4882 } 4781 }
4883 4782
4884 if (opts.mcount) { 4783 if (opts.mcount) {
4885 ret = create_mcount_loc_secti 4784 ret = create_mcount_loc_sections(file);
4886 if (ret < 0) 4785 if (ret < 0)
4887 goto out; 4786 goto out;
4888 warnings += ret; 4787 warnings += ret;
4889 } 4788 }
4890 4789
4891 if (opts.prefix) { 4790 if (opts.prefix) {
4892 ret = add_prefix_symbols(file 4791 ret = add_prefix_symbols(file);
4893 if (ret < 0) 4792 if (ret < 0)
4894 return ret; 4793 return ret;
4895 warnings += ret; 4794 warnings += ret;
4896 } 4795 }
4897 4796
4898 if (opts.ibt) { 4797 if (opts.ibt) {
4899 ret = create_ibt_endbr_seal_s 4798 ret = create_ibt_endbr_seal_sections(file);
4900 if (ret < 0) 4799 if (ret < 0)
4901 goto out; 4800 goto out;
4902 warnings += ret; 4801 warnings += ret;
4903 } 4802 }
4904 4803
4905 if (opts.orc && nr_insns) { 4804 if (opts.orc && nr_insns) {
4906 ret = orc_create(file); 4805 ret = orc_create(file);
4907 if (ret < 0) 4806 if (ret < 0)
4908 goto out; 4807 goto out;
4909 warnings += ret; 4808 warnings += ret;
4910 } 4809 }
4911 4810
4912 free_insns(file); 4811 free_insns(file);
4913 4812
4914 if (opts.verbose) 4813 if (opts.verbose)
4915 disas_warned_funcs(file); 4814 disas_warned_funcs(file);
4916 4815
4917 if (opts.stats) { 4816 if (opts.stats) {
4918 printf("nr_insns_visited: %ld 4817 printf("nr_insns_visited: %ld\n", nr_insns_visited);
4919 printf("nr_cfi: %ld\n", nr_cf 4818 printf("nr_cfi: %ld\n", nr_cfi);
4920 printf("nr_cfi_reused: %ld\n" 4819 printf("nr_cfi_reused: %ld\n", nr_cfi_reused);
4921 printf("nr_cfi_cache: %ld\n", 4820 printf("nr_cfi_cache: %ld\n", nr_cfi_cache);
4922 } 4821 }
4923 4822
4924 out: 4823 out:
4925 /* 4824 /*
4926 * For now, don't fail the kernel bu 4825 * For now, don't fail the kernel build on fatal warnings. These
4927 * errors are still fairly common du 4826 * errors are still fairly common due to the growing matrix of
4928 * supported toolchains and their re 4827 * supported toolchains and their recent pace of change.
4929 */ 4828 */
4930 return 0; 4829 return 0;
4931 } 4830 }
4932 4831
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.