1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Kernel unwinding support
4 *
5 * (c) 2002-2004 Randolph Chung <tausq@debian.org>
6 *
7 * Derived partially from the IA64 implementation. The PA-RISC
8 * Runtime Architecture Document is also a useful reference to
9 * understand what is happening here
10 */
11
12 #include <linux/kernel.h>
13 #include <linux/init.h>
14 #include <linux/sched.h>
15 #include <linux/slab.h>
16 #include <linux/sort.h>
17 #include <linux/sched/task_stack.h>
18
19 #include <linux/uaccess.h>
20 #include <asm/assembly.h>
21 #include <asm/asm-offsets.h>
22 #include <asm/ptrace.h>
23
24 #include <asm/unwind.h>
25 #include <asm/switch_to.h>
26 #include <asm/sections.h>
27 #include <asm/ftrace.h>
28
29 /* #define DEBUG 1 */
30 #ifdef DEBUG
31 #define dbg(x...) pr_debug(x)
32 #else
33 #define dbg(x...) do { } while (0)
34 #endif
35
36 #define KERNEL_START (KERNEL_BINARY_TEXT_START)
37
38 extern struct unwind_table_entry __start___unwind[];
39 extern struct unwind_table_entry __stop___unwind[];
40
41 static DEFINE_SPINLOCK(unwind_lock);
42 /*
43 * the kernel unwind block is not dynamically allocated so that
44 * we can call unwind_init as early in the bootup process as
45 * possible (before the slab allocator is initialized)
46 */
47 static struct unwind_table kernel_unwind_table __ro_after_init;
48 static LIST_HEAD(unwind_tables);
49
50 static inline const struct unwind_table_entry *
51 find_unwind_entry_in_table(const struct unwind_table *table, unsigned long addr)
52 {
53 const struct unwind_table_entry *e = NULL;
54 unsigned long lo, hi, mid;
55
56 lo = 0;
57 hi = table->length - 1;
58
59 while (lo <= hi) {
60 mid = (hi - lo) / 2 + lo;
61 e = &table->table[mid];
62 if (addr < e->region_start)
63 hi = mid - 1;
64 else if (addr > e->region_end)
65 lo = mid + 1;
66 else
67 return e;
68 }
69
70 return NULL;
71 }
72
73 static const struct unwind_table_entry *
74 find_unwind_entry(unsigned long addr)
75 {
76 struct unwind_table *table;
77 const struct unwind_table_entry *e = NULL;
78
79 if (addr >= kernel_unwind_table.start &&
80 addr <= kernel_unwind_table.end)
81 e = find_unwind_entry_in_table(&kernel_unwind_table, addr);
82 else {
83 unsigned long flags;
84
85 spin_lock_irqsave(&unwind_lock, flags);
86 list_for_each_entry(table, &unwind_tables, list) {
87 if (addr >= table->start &&
88 addr <= table->end)
89 e = find_unwind_entry_in_table(table, addr);
90 if (e) {
91 /* Move-to-front to exploit common traces */
92 list_move(&table->list, &unwind_tables);
93 break;
94 }
95 }
96 spin_unlock_irqrestore(&unwind_lock, flags);
97 }
98
99 return e;
100 }
101
102 static void
103 unwind_table_init(struct unwind_table *table, const char *name,
104 unsigned long base_addr, unsigned long gp,
105 void *table_start, void *table_end)
106 {
107 struct unwind_table_entry *start = table_start;
108 struct unwind_table_entry *end =
109 (struct unwind_table_entry *)table_end - 1;
110
111 table->name = name;
112 table->base_addr = base_addr;
113 table->gp = gp;
114 table->start = base_addr + start->region_start;
115 table->end = base_addr + end->region_end;
116 table->table = (struct unwind_table_entry *)table_start;
117 table->length = end - start + 1;
118 INIT_LIST_HEAD(&table->list);
119
120 for (; start <= end; start++) {
121 if (start < end &&
122 start->region_end > (start+1)->region_start) {
123 pr_warn("Out of order unwind entry! %px and %px\n",
124 start, start+1);
125 }
126
127 start->region_start += base_addr;
128 start->region_end += base_addr;
129 }
130 }
131
132 static int cmp_unwind_table_entry(const void *a, const void *b)
133 {
134 return ((const struct unwind_table_entry *)a)->region_start
135 - ((const struct unwind_table_entry *)b)->region_start;
136 }
137
138 static void
139 unwind_table_sort(struct unwind_table_entry *start,
140 struct unwind_table_entry *finish)
141 {
142 sort(start, finish - start, sizeof(struct unwind_table_entry),
143 cmp_unwind_table_entry, NULL);
144 }
145
146 struct unwind_table *
147 unwind_table_add(const char *name, unsigned long base_addr,
148 unsigned long gp,
149 void *start, void *end)
150 {
151 struct unwind_table *table;
152 unsigned long flags;
153 struct unwind_table_entry *s = (struct unwind_table_entry *)start;
154 struct unwind_table_entry *e = (struct unwind_table_entry *)end;
155
156 unwind_table_sort(s, e);
157
158 table = kmalloc(sizeof(struct unwind_table), GFP_USER);
159 if (table == NULL)
160 return NULL;
161 unwind_table_init(table, name, base_addr, gp, start, end);
162 spin_lock_irqsave(&unwind_lock, flags);
163 list_add_tail(&table->list, &unwind_tables);
164 spin_unlock_irqrestore(&unwind_lock, flags);
165
166 return table;
167 }
168
169 void unwind_table_remove(struct unwind_table *table)
170 {
171 unsigned long flags;
172
173 spin_lock_irqsave(&unwind_lock, flags);
174 list_del(&table->list);
175 spin_unlock_irqrestore(&unwind_lock, flags);
176
177 kfree(table);
178 }
179
180 /* Called from setup_arch to import the kernel unwind info */
181 int __init unwind_init(void)
182 {
183 long start __maybe_unused, stop __maybe_unused;
184 register unsigned long gp __asm__ ("r27");
185
186 start = (long)&__start___unwind[0];
187 stop = (long)&__stop___unwind[0];
188
189 dbg("unwind_init: start = 0x%lx, end = 0x%lx, entries = %lu\n",
190 start, stop,
191 (stop - start) / sizeof(struct unwind_table_entry));
192
193 unwind_table_init(&kernel_unwind_table, "kernel", KERNEL_START,
194 gp,
195 &__start___unwind[0], &__stop___unwind[0]);
196 #if 0
197 {
198 int i;
199 for (i = 0; i < 10; i++)
200 {
201 printk("region 0x%x-0x%x\n",
202 __start___unwind[i].region_start,
203 __start___unwind[i].region_end);
204 }
205 }
206 #endif
207 return 0;
208 }
209
210 static bool pc_is_kernel_fn(unsigned long pc, void *fn)
211 {
212 return (unsigned long)dereference_kernel_function_descriptor(fn) == pc;
213 }
214
215 static int unwind_special(struct unwind_frame_info *info, unsigned long pc, int frame_size)
216 {
217 /*
218 * We have to use void * instead of a function pointer, because
219 * function pointers aren't a pointer to the function on 64-bit.
220 * Make them const so the compiler knows they live in .text
221 * Note: We could use dereference_kernel_function_descriptor()
222 * instead but we want to keep it simple here.
223 */
224 extern void * const ret_from_kernel_thread;
225 extern void * const syscall_exit;
226 extern void * const intr_return;
227 extern void * const _switch_to_ret;
228 #ifdef CONFIG_IRQSTACKS
229 extern void * const _call_on_stack;
230 #endif /* CONFIG_IRQSTACKS */
231
232 if (pc_is_kernel_fn(pc, handle_interruption)) {
233 struct pt_regs *regs = (struct pt_regs *)(info->sp - frame_size - PT_SZ_ALGN);
234 dbg("Unwinding through handle_interruption()\n");
235 info->prev_sp = regs->gr[30];
236 info->prev_ip = regs->iaoq[0];
237 return 1;
238 }
239
240 if (pc == (unsigned long)&ret_from_kernel_thread ||
241 pc == (unsigned long)&syscall_exit) {
242 info->prev_sp = info->prev_ip = 0;
243 return 1;
244 }
245
246 if (pc == (unsigned long)&intr_return) {
247 struct pt_regs *regs;
248
249 dbg("Found intr_return()\n");
250 regs = (struct pt_regs *)(info->sp - PT_SZ_ALGN);
251 info->prev_sp = regs->gr[30];
252 info->prev_ip = regs->iaoq[0];
253 info->rp = regs->gr[2];
254 return 1;
255 }
256
257 if (pc_is_kernel_fn(pc, _switch_to) ||
258 pc == (unsigned long)&_switch_to_ret) {
259 info->prev_sp = info->sp - CALLEE_SAVE_FRAME_SIZE;
260 info->prev_ip = *(unsigned long *)(info->prev_sp - RP_OFFSET);
261 return 1;
262 }
263
264 #ifdef CONFIG_IRQSTACKS
265 if (pc == (unsigned long)&_call_on_stack) {
266 info->prev_sp = *(unsigned long *)(info->sp - FRAME_SIZE - REG_SZ);
267 info->prev_ip = *(unsigned long *)(info->sp - FRAME_SIZE - RP_OFFSET);
268 return 1;
269 }
270 #endif
271 return 0;
272 }
273
274 static void unwind_frame_regs(struct unwind_frame_info *info)
275 {
276 const struct unwind_table_entry *e;
277 unsigned long npc;
278 unsigned int insn;
279 long frame_size = 0;
280 int looking_for_rp, rpoffset = 0;
281
282 e = find_unwind_entry(info->ip);
283 if (e == NULL) {
284 unsigned long sp;
285
286 dbg("Cannot find unwind entry for %pS; forced unwinding\n",
287 (void *) info->ip);
288
289 /* Since we are doing the unwinding blind, we don't know if
290 we are adjusting the stack correctly or extracting the rp
291 correctly. The rp is checked to see if it belongs to the
292 kernel text section, if not we assume we don't have a
293 correct stack frame and we continue to unwind the stack.
294 This is not quite correct, and will fail for loadable
295 modules. */
296 sp = info->sp & ~63;
297 do {
298 unsigned long tmp;
299
300 info->prev_sp = sp - 64;
301 info->prev_ip = 0;
302
303 /* Check if stack is inside kernel stack area */
304 if ((info->prev_sp - (unsigned long) task_stack_page(info->t))
305 >= THREAD_SIZE) {
306 info->prev_sp = 0;
307 break;
308 }
309
310 if (copy_from_kernel_nofault(&tmp,
311 (void *)info->prev_sp - RP_OFFSET, sizeof(tmp)))
312 break;
313 info->prev_ip = tmp;
314 sp = info->prev_sp;
315 } while (!kernel_text_address(info->prev_ip));
316
317 info->rp = 0;
318
319 dbg("analyzing func @ %lx with no unwind info, setting "
320 "prev_sp=%lx prev_ip=%lx\n", info->ip,
321 info->prev_sp, info->prev_ip);
322 } else {
323 dbg("e->start = 0x%x, e->end = 0x%x, Save_SP = %d, "
324 "Save_RP = %d, Millicode = %d size = %u\n",
325 e->region_start, e->region_end, e->Save_SP, e->Save_RP,
326 e->Millicode, e->Total_frame_size);
327
328 looking_for_rp = e->Save_RP;
329
330 for (npc = e->region_start;
331 (frame_size < (e->Total_frame_size << 3) ||
332 looking_for_rp) &&
333 npc < info->ip;
334 npc += 4) {
335
336 insn = *(unsigned int *)npc;
337
338 if ((insn & 0xffffc001) == 0x37de0000 ||
339 (insn & 0xffe00001) == 0x6fc00000) {
340 /* ldo X(sp), sp, or stwm X,D(sp) */
341 frame_size += (insn & 0x3fff) >> 1;
342 dbg("analyzing func @ %lx, insn=%08x @ "
343 "%lx, frame_size = %ld\n", info->ip,
344 insn, npc, frame_size);
345 } else if ((insn & 0xffe00009) == 0x73c00008) {
346 /* std,ma X,D(sp) */
347 frame_size += ((insn >> 4) & 0x3ff) << 3;
348 dbg("analyzing func @ %lx, insn=%08x @ "
349 "%lx, frame_size = %ld\n", info->ip,
350 insn, npc, frame_size);
351 } else if (insn == 0x6bc23fd9) {
352 /* stw rp,-20(sp) */
353 rpoffset = 20;
354 looking_for_rp = 0;
355 dbg("analyzing func @ %lx, insn=stw rp,"
356 "-20(sp) @ %lx\n", info->ip, npc);
357 } else if (insn == 0x0fc212c1) {
358 /* std rp,-16(sr0,sp) */
359 rpoffset = 16;
360 looking_for_rp = 0;
361 dbg("analyzing func @ %lx, insn=std rp,"
362 "-16(sp) @ %lx\n", info->ip, npc);
363 }
364 }
365
366 if (frame_size > e->Total_frame_size << 3)
367 frame_size = e->Total_frame_size << 3;
368
369 if (!unwind_special(info, e->region_start, frame_size)) {
370 info->prev_sp = info->sp - frame_size;
371 if (e->Millicode)
372 info->rp = info->r31;
373 else if (rpoffset)
374 info->rp = *(unsigned long *)(info->prev_sp - rpoffset);
375 info->prev_ip = info->rp;
376 info->rp = 0;
377 }
378
379 dbg("analyzing func @ %lx, setting prev_sp=%lx "
380 "prev_ip=%lx npc=%lx\n", info->ip, info->prev_sp,
381 info->prev_ip, npc);
382 }
383 }
384
385 void unwind_frame_init(struct unwind_frame_info *info, struct task_struct *t,
386 struct pt_regs *regs)
387 {
388 memset(info, 0, sizeof(struct unwind_frame_info));
389 info->t = t;
390 info->sp = regs->gr[30];
391 info->ip = regs->iaoq[0];
392 info->rp = regs->gr[2];
393 info->r31 = regs->gr[31];
394
395 dbg("(%d) Start unwind from sp=%08lx ip=%08lx\n",
396 t ? (int)t->pid : -1, info->sp, info->ip);
397 }
398
399 void unwind_frame_init_from_blocked_task(struct unwind_frame_info *info, struct task_struct *t)
400 {
401 struct pt_regs *r = &t->thread.regs;
402 struct pt_regs *r2;
403
404 r2 = kmalloc(sizeof(struct pt_regs), GFP_ATOMIC);
405 if (!r2)
406 return;
407 *r2 = *r;
408 r2->gr[30] = r->ksp;
409 r2->iaoq[0] = r->kpc;
410 unwind_frame_init(info, t, r2);
411 kfree(r2);
412 }
413
414 #define get_parisc_stackpointer() ({ \
415 unsigned long sp; \
416 __asm__("copy %%r30, %0" : "=r"(sp)); \
417 (sp); \
418 })
419
420 void unwind_frame_init_task(struct unwind_frame_info *info,
421 struct task_struct *task, struct pt_regs *regs)
422 {
423 task = task ? task : current;
424
425 if (task == current) {
426 struct pt_regs r;
427
428 if (!regs) {
429 memset(&r, 0, sizeof(r));
430 r.iaoq[0] = _THIS_IP_;
431 r.gr[2] = _RET_IP_;
432 r.gr[30] = get_parisc_stackpointer();
433 regs = &r;
434 }
435 unwind_frame_init(info, task, regs);
436 } else {
437 unwind_frame_init_from_blocked_task(info, task);
438 }
439 }
440
441 int unwind_once(struct unwind_frame_info *next_frame)
442 {
443 unwind_frame_regs(next_frame);
444
445 if (next_frame->prev_sp == 0 ||
446 next_frame->prev_ip == 0)
447 return -1;
448
449 next_frame->sp = next_frame->prev_sp;
450 next_frame->ip = next_frame->prev_ip;
451 next_frame->prev_sp = 0;
452 next_frame->prev_ip = 0;
453
454 dbg("(%d) Continue unwind to sp=%08lx ip=%08lx\n",
455 next_frame->t ? (int)next_frame->t->pid : -1,
456 next_frame->sp, next_frame->ip);
457
458 return 0;
459 }
460
461 int unwind_to_user(struct unwind_frame_info *info)
462 {
463 int ret;
464
465 do {
466 ret = unwind_once(info);
467 } while (!ret && !(info->ip & 3));
468
469 return ret;
470 }
471
472 unsigned long return_address(unsigned int level)
473 {
474 struct unwind_frame_info info;
475
476 /* initialize unwind info */
477 unwind_frame_init_task(&info, current, NULL);
478
479 /* unwind stack */
480 level += 2;
481 do {
482 if (unwind_once(&info) < 0 || info.ip == 0)
483 return 0;
484 if (!kernel_text_address(info.ip))
485 return 0;
486 } while (info.ip && level--);
487
488 return info.ip;
489 }
490
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.