1 /* SPDX-License-Identifier: GPL-2.0-only */ !! 1 /* -*- mode: asm -*-
2 /* <<
3 * Low-level exception handling code <<
4 * 2 *
5 * Copyright (C) 2012 ARM Ltd. !! 3 * linux/arch/m68k/kernel/entry.S
6 * Authors: Catalin Marinas <catalin.marina <<
7 * Will Deacon <will.deacon@arm.co <<
8 */ <<
9 <<
10 #include <linux/arm-smccc.h> <<
11 #include <linux/init.h> <<
12 #include <linux/linkage.h> <<
13 <<
14 #include <asm/alternative.h> <<
15 #include <asm/assembler.h> <<
16 #include <asm/asm-offsets.h> <<
17 #include <asm/asm_pointer_auth.h> <<
18 #include <asm/bug.h> <<
19 #include <asm/cpufeature.h> <<
20 #include <asm/errno.h> <<
21 #include <asm/esr.h> <<
22 #include <asm/irq.h> <<
23 #include <asm/memory.h> <<
24 #include <asm/mmu.h> <<
25 #include <asm/processor.h> <<
26 #include <asm/ptrace.h> <<
27 #include <asm/scs.h> <<
28 #include <asm/thread_info.h> <<
29 #include <asm/asm-uaccess.h> <<
30 #include <asm/unistd.h> <<
31 <<
32 .macro clear_gp_regs <<
33 .irp n,0,1,2,3,4,5,6,7,8,9,10,11,12 <<
34 mov x\n, xzr <<
35 .endr <<
36 .endm <<
37 <<
38 .macro kernel_ventry, el:req, ht:req, <<
39 .align 7 <<
40 .Lventry_start\@: <<
41 .if \el == 0 <<
42 /* <<
43 * This must be the first instruction <<
44 * skipped by the trampoline vectors, <<
45 */ <<
46 b .Lskip_tramp_vectors_cleanup\@ <<
47 .if \regsize == 64 <<
48 mrs x30, tpidrro_el0 <<
49 msr tpidrro_el0, xzr <<
50 .else <<
51 mov x30, xzr <<
52 .endif <<
53 .Lskip_tramp_vectors_cleanup\@: <<
54 .endif <<
55 <<
56 sub sp, sp, #PT_REGS_SIZE <<
57 #ifdef CONFIG_VMAP_STACK <<
58 /* <<
59 * Test whether the SP has overflowed, <<
60 * Task and IRQ stacks are aligned so <<
61 * should always be zero. <<
62 */ <<
63 add sp, sp, x0 <<
64 sub x0, sp, x0 <<
65 tbnz x0, #THREAD_SHIFT, 0f <<
66 sub x0, sp, x0 <<
67 sub sp, sp, x0 <<
68 b el\el\ht\()_\regsize\()_\label <<
69 <<
70 0: <<
71 /* <<
72 * Either we've just detected an overf <<
73 * while on the overflow stack. Either <<
74 * userspace, and can clobber EL0 regi <<
75 */ <<
76 <<
77 /* Stash the original SP (minus PT_REG <<
78 msr tpidr_el0, x0 <<
79 <<
80 /* Recover the original x0 value and s <<
81 sub x0, sp, x0 <<
82 msr tpidrro_el0, x0 <<
83 <<
84 /* Switch to the overflow stack */ <<
85 adr_this_cpu sp, overflow_stack + OVER <<
86 <<
87 /* <<
88 * Check whether we were already on th <<
89 * after panic() re-enables interrupts <<
90 */ <<
91 mrs x0, tpidr_el0 <<
92 sub x0, sp, x0 <<
93 tst x0, #~(OVERFLOW_STACK_SIZE - 1 <<
94 b.ne __bad_stack <<
95 <<
96 /* We were already on the overflow sta <<
97 sub sp, sp, x0 <<
98 mrs x0, tpidrro_el0 <<
99 #endif <<
100 b el\el\ht\()_\regsize\()_\label <<
101 .org .Lventry_start\@ + 128 // Did we over <<
102 .endm <<
103 <<
104 .macro tramp_alias, dst, sym <<
105 .set .Lalias\@, TRAMP_VALIAS + \sym <<
106 movz \dst, :abs_g2_s:.Lalias\@ <<
107 movk \dst, :abs_g1_nc:.Lalias\@ <<
108 movk \dst, :abs_g0_nc:.Lalias\@ <<
109 .endm <<
110 <<
111 /* <<
112 * This macro corrupts x0-x3. It is th <<
113 * them if required. <<
114 */ <<
115 .macro apply_ssbd, state, tmp1, tmp2 <<
116 alternative_cb ARM64_ALWAYS_SYSTEM, spectre_v <<
117 b .L__asm_ssbd_skip\@ <<
118 alternative_cb_end <<
119 ldr_this_cpu \tmp2, arm64_ssbd_call <<
120 cbz \tmp2, .L__asm_ssbd_skip\@ <<
121 ldr \tmp2, [tsk, #TSK_TI_FLAGS] <<
122 tbnz \tmp2, #TIF_SSBD, .L__asm_ssbd <<
123 mov w0, #ARM_SMCCC_ARCH_WORKAROUND <<
124 mov w1, #\state <<
125 alternative_cb ARM64_ALWAYS_SYSTEM, smccc_pat <<
126 nop <<
127 alternative_cb_end <<
128 .L__asm_ssbd_skip\@: <<
129 .endm <<
130 <<
131 /* Check for MTE asynchronous tag chec <<
132 .macro check_mte_async_tcf, tmp, ti_fl <<
133 #ifdef CONFIG_ARM64_MTE <<
134 .arch_extension lse <<
135 alternative_if_not ARM64_MTE <<
136 b 1f <<
137 alternative_else_nop_endif <<
138 /* <<
139 * Asynchronous tag check faults are o <<
140 * ASYM (3) modes. In each of these mo <<
141 * set, so skip the check if it is uns <<
142 */ <<
143 tbz \thread_sctlr, #(SCTLR_EL1_TCF <<
144 mrs_s \tmp, SYS_TFSRE0_EL1 <<
145 tbz \tmp, #SYS_TFSR_EL1_TF0_SHIFT, <<
146 /* Asynchronous TCF occurred for TTBR0 <<
147 mov \tmp, #_TIF_MTE_ASYNC_FAULT <<
148 add \ti_flags, tsk, #TSK_TI_FLAGS <<
149 stset \tmp, [\ti_flags] <<
150 1: <<
151 #endif <<
152 .endm <<
153 <<
154 /* Clear the MTE asynchronous tag chec <<
155 .macro clear_mte_async_tcf thread_sctl <<
156 #ifdef CONFIG_ARM64_MTE <<
157 alternative_if ARM64_MTE <<
158 /* See comment in check_mte_async_tcf <<
159 tbz \thread_sctlr, #(SCTLR_EL1_TCF <<
160 dsb ish <<
161 msr_s SYS_TFSRE0_EL1, xzr <<
162 1: <<
163 alternative_else_nop_endif <<
164 #endif <<
165 .endm <<
166 <<
167 .macro mte_set_gcr, mte_ctrl, tmp <<
168 #ifdef CONFIG_ARM64_MTE <<
169 ubfx \tmp, \mte_ctrl, #MTE_CTRL_GCR <<
170 orr \tmp, \tmp, #SYS_GCR_EL1_RRND <<
171 msr_s SYS_GCR_EL1, \tmp <<
172 #endif <<
173 .endm <<
174 <<
175 .macro mte_set_kernel_gcr, tmp, tmp2 <<
176 #ifdef CONFIG_KASAN_HW_TAGS <<
177 alternative_cb ARM64_ALWAYS_SYSTEM, kasan_hw_ <<
178 b 1f <<
179 alternative_cb_end <<
180 mov \tmp, KERNEL_GCR_EL1 <<
181 msr_s SYS_GCR_EL1, \tmp <<
182 1: <<
183 #endif <<
184 .endm <<
185 <<
186 .macro mte_set_user_gcr, tsk, tmp, tmp <<
187 #ifdef CONFIG_KASAN_HW_TAGS <<
188 alternative_cb ARM64_ALWAYS_SYSTEM, kasan_hw_ <<
189 b 1f <<
190 alternative_cb_end <<
191 ldr \tmp, [\tsk, #THREAD_MTE_CTRL] <<
192 <<
193 mte_set_gcr \tmp, \tmp2 <<
194 1: <<
195 #endif <<
196 .endm <<
197 <<
198 .macro kernel_entry, el, regsize = 64 <<
199 .if \el == 0 <<
200 alternative_insn nop, SET_PSTATE_DIT(1 <<
201 .endif <<
202 .if \regsize == 32 <<
203 mov w0, w0 <<
204 .endif <<
205 stp x0, x1, [sp, #16 * 0] <<
206 stp x2, x3, [sp, #16 * 1] <<
207 stp x4, x5, [sp, #16 * 2] <<
208 stp x6, x7, [sp, #16 * 3] <<
209 stp x8, x9, [sp, #16 * 4] <<
210 stp x10, x11, [sp, #16 * 5] <<
211 stp x12, x13, [sp, #16 * 6] <<
212 stp x14, x15, [sp, #16 * 7] <<
213 stp x16, x17, [sp, #16 * 8] <<
214 stp x18, x19, [sp, #16 * 9] <<
215 stp x20, x21, [sp, #16 * 10] <<
216 stp x22, x23, [sp, #16 * 11] <<
217 stp x24, x25, [sp, #16 * 12] <<
218 stp x26, x27, [sp, #16 * 13] <<
219 stp x28, x29, [sp, #16 * 14] <<
220 <<
221 .if \el == 0 <<
222 clear_gp_regs <<
223 mrs x21, sp_el0 <<
224 ldr_this_cpu tsk, __entry_task, x20 <<
225 msr sp_el0, tsk <<
226 <<
227 /* <<
228 * Ensure MDSCR_EL1.SS is clear, since <<
229 * when scheduling. <<
230 */ <<
231 ldr x19, [tsk, #TSK_TI_FLAGS] <<
232 disable_step_tsk x19, x20 <<
233 <<
234 /* Check for asynchronous tag check fa <<
235 ldr x0, [tsk, THREAD_SCTLR_USER] <<
236 check_mte_async_tcf x22, x23, x0 <<
237 <<
238 #ifdef CONFIG_ARM64_PTR_AUTH <<
239 alternative_if ARM64_HAS_ADDRESS_AUTH <<
240 /* <<
241 * Enable IA for in-kernel PAC if the <<
242 * this could be implemented with an u <<
243 * a load, this was measured to be slo <<
244 * <<
245 * Install the kernel IA key only if I <<
246 * was disabled on kernel exit then we <<
247 * installed so there is no need to in <<
248 */ <<
249 tbz x0, SCTLR_ELx_ENIA_SHIFT, 1f <<
250 __ptrauth_keys_install_kernel_nosync t <<
251 b 2f <<
252 1: <<
253 mrs x0, sctlr_el1 <<
254 orr x0, x0, SCTLR_ELx_ENIA <<
255 msr sctlr_el1, x0 <<
256 2: <<
257 alternative_else_nop_endif <<
258 #endif <<
259 <<
260 apply_ssbd 1, x22, x23 <<
261 <<
262 mte_set_kernel_gcr x22, x23 <<
263 <<
264 /* <<
265 * Any non-self-synchronizing system r <<
266 * kernel entry should be placed befor <<
267 */ <<
268 alternative_if ARM64_MTE <<
269 isb <<
270 b 1f <<
271 alternative_else_nop_endif <<
272 alternative_if ARM64_HAS_ADDRESS_AUTH <<
273 isb <<
274 alternative_else_nop_endif <<
275 1: <<
276 <<
277 scs_load_current <<
278 .else <<
279 add x21, sp, #PT_REGS_SIZE <<
280 get_current_task tsk <<
281 .endif /* \el == 0 */ <<
282 mrs x22, elr_el1 <<
283 mrs x23, spsr_el1 <<
284 stp lr, x21, [sp, #S_LR] <<
285 <<
286 /* <<
287 * For exceptions from EL0, create a f <<
288 * For exceptions from EL1, create a s <<
289 * interrupted code shows up in the ba <<
290 */ <<
291 .if \el == 0 <<
292 stp xzr, xzr, [sp, #S_STACKFRAME] <<
293 .else <<
294 stp x29, x22, [sp, #S_STACKFRAME] <<
295 .endif <<
296 add x29, sp, #S_STACKFRAME <<
297 <<
298 #ifdef CONFIG_ARM64_SW_TTBR0_PAN <<
299 alternative_if_not ARM64_HAS_PAN <<
300 bl __swpan_entry_el\el <<
301 alternative_else_nop_endif <<
302 #endif <<
303 <<
304 stp x22, x23, [sp, #S_PC] <<
305 <<
306 /* Not in a syscall by default (el0_sv <<
307 .if \el == 0 <<
308 mov w21, #NO_SYSCALL <<
309 str w21, [sp, #S_SYSCALLNO] <<
310 .endif <<
311 <<
312 #ifdef CONFIG_ARM64_PSEUDO_NMI <<
313 alternative_if_not ARM64_HAS_GIC_PRIO_MASKING <<
314 b .Lskip_pmr_save\@ <<
315 alternative_else_nop_endif <<
316 <<
317 mrs_s x20, SYS_ICC_PMR_EL1 <<
318 str x20, [sp, #S_PMR_SAVE] <<
319 mov x20, #GIC_PRIO_IRQON | GIC_PRI <<
320 msr_s SYS_ICC_PMR_EL1, x20 <<
321 <<
322 .Lskip_pmr_save\@: <<
323 #endif <<
324 <<
325 /* <<
326 * Registers that may be useful after <<
327 * <<
328 * x20 - ICC_PMR_EL1 <<
329 * x21 - aborted SP <<
330 * x22 - aborted PC <<
331 * x23 - aborted PSTATE <<
332 */ <<
333 .endm <<
334 <<
335 .macro kernel_exit, el <<
336 .if \el != 0 <<
337 disable_daif <<
338 .endif <<
339 <<
340 #ifdef CONFIG_ARM64_PSEUDO_NMI <<
341 alternative_if_not ARM64_HAS_GIC_PRIO_MASKING <<
342 b .Lskip_pmr_restore\@ <<
343 alternative_else_nop_endif <<
344 <<
345 ldr x20, [sp, #S_PMR_SAVE] <<
346 msr_s SYS_ICC_PMR_EL1, x20 <<
347 <<
348 /* Ensure priority change is seen by r <<
349 alternative_if_not ARM64_HAS_GIC_PRIO_RELAXED_ <<
350 dsb sy <<
351 alternative_else_nop_endif <<
352 <<
353 .Lskip_pmr_restore\@: <<
354 #endif <<
355 <<
356 ldp x21, x22, [sp, #S_PC] <<
357 <<
358 #ifdef CONFIG_ARM64_SW_TTBR0_PAN <<
359 alternative_if_not ARM64_HAS_PAN <<
360 bl __swpan_exit_el\el <<
361 alternative_else_nop_endif <<
362 #endif <<
363 <<
364 .if \el == 0 <<
365 ldr x23, [sp, #S_SP] <<
366 msr sp_el0, x23 <<
367 tst x22, #PSR_MODE32_BIT <<
368 b.eq 3f <<
369 <<
370 #ifdef CONFIG_ARM64_ERRATUM_845719 <<
371 alternative_if ARM64_WORKAROUND_845719 <<
372 #ifdef CONFIG_PID_IN_CONTEXTIDR <<
373 mrs x29, contextidr_el1 <<
374 msr contextidr_el1, x29 <<
375 #else <<
376 msr contextidr_el1, xzr <<
377 #endif <<
378 alternative_else_nop_endif <<
379 #endif <<
380 3: <<
381 scs_save tsk <<
382 <<
383 /* Ignore asynchronous tag check fault <<
384 ldr x0, [tsk, THREAD_SCTLR_USER] <<
385 clear_mte_async_tcf x0 <<
386 <<
387 #ifdef CONFIG_ARM64_PTR_AUTH <<
388 alternative_if ARM64_HAS_ADDRESS_AUTH <<
389 /* <<
390 * IA was enabled for in-kernel PAC. D <<
391 * alternatively install the user's IA <<
392 * SCTLR bits were updated on task swi <<
393 * <<
394 * No kernel C function calls after th <<
395 */ <<
396 tbz x0, SCTLR_ELx_ENIA_SHIFT, 1f <<
397 __ptrauth_keys_install_user tsk, x0, x <<
398 b 2f <<
399 1: <<
400 mrs x0, sctlr_el1 <<
401 bic x0, x0, SCTLR_ELx_ENIA <<
402 msr sctlr_el1, x0 <<
403 2: <<
404 alternative_else_nop_endif <<
405 #endif <<
406 <<
407 mte_set_user_gcr tsk, x0, x1 <<
408 <<
409 apply_ssbd 0, x0, x1 <<
410 .endif <<
411 <<
412 msr elr_el1, x21 <<
413 msr spsr_el1, x22 <<
414 ldp x0, x1, [sp, #16 * 0] <<
415 ldp x2, x3, [sp, #16 * 1] <<
416 ldp x4, x5, [sp, #16 * 2] <<
417 ldp x6, x7, [sp, #16 * 3] <<
418 ldp x8, x9, [sp, #16 * 4] <<
419 ldp x10, x11, [sp, #16 * 5] <<
420 ldp x12, x13, [sp, #16 * 6] <<
421 ldp x14, x15, [sp, #16 * 7] <<
422 ldp x16, x17, [sp, #16 * 8] <<
423 ldp x18, x19, [sp, #16 * 9] <<
424 ldp x20, x21, [sp, #16 * 10] <<
425 ldp x22, x23, [sp, #16 * 11] <<
426 ldp x24, x25, [sp, #16 * 12] <<
427 ldp x26, x27, [sp, #16 * 13] <<
428 ldp x28, x29, [sp, #16 * 14] <<
429 <<
430 .if \el == 0 <<
431 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 <<
432 alternative_insn "b .L_skip_tramp_exit <<
433 <<
434 msr far_el1, x29 <<
435 <<
436 ldr_this_cpu x30, this_cpu_vector, <<
437 tramp_alias x29, tramp_exit <<
438 msr vbar_el1, x30 <<
439 ldr lr, [sp, #S_LR] <<
440 add sp, sp, #PT_REGS_SIZE <<
441 br x29 <<
442 <<
443 .L_skip_tramp_exit_\@: <<
444 #endif <<
445 .endif <<
446 <<
447 ldr lr, [sp, #S_LR] <<
448 add sp, sp, #PT_REGS_SIZE <<
449 <<
450 .if \el == 0 <<
451 /* This must be after the last explici <<
452 alternative_if ARM64_WORKAROUND_SPECULATIVE_UN <<
453 tlbi vale1, xzr <<
454 dsb nsh <<
455 alternative_else_nop_endif <<
456 .else <<
457 /* Ensure any device/NC reads complete <<
458 alternative_insn nop, "dmb sy", ARM64_ <<
459 .endif <<
460 <<
461 eret <<
462 sb <<
463 .endm <<
464 <<
465 #ifdef CONFIG_ARM64_SW_TTBR0_PAN <<
466 /* <<
467 * Set the TTBR0 PAN bit in SPSR. When <<
468 * EL0, there is no need to check the <<
469 * accesses are always enabled. <<
470 * Note that the meaning of this bit d <<
471 * feature as all TTBR0_EL1 accesses a <<
472 * user mappings. <<
473 */ <<
474 SYM_CODE_START_LOCAL(__swpan_entry_el1) <<
475 mrs x21, ttbr0_el1 <<
476 tst x21, #TTBR_ASID_MASK <<
477 orr x23, x23, #PSR_PAN_BIT <<
478 b.eq 1f <<
479 and x23, x23, #~PSR_PAN_BIT <<
480 SYM_INNER_LABEL(__swpan_entry_el0, SYM_L_LOCAL <<
481 __uaccess_ttbr0_disable x21 <<
482 1: ret <<
483 SYM_CODE_END(__swpan_entry_el1) <<
484 <<
485 /* <<
486 * Restore access to TTBR0_EL1. If ret <<
487 * PAN bit checking. <<
488 */ <<
489 SYM_CODE_START_LOCAL(__swpan_exit_el1) <<
490 tbnz x22, #22, 1f <<
491 __uaccess_ttbr0_enable x0, x1 <<
492 1: and x22, x22, #~PSR_PAN_BIT <<
493 ret <<
494 SYM_CODE_END(__swpan_exit_el1) <<
495 <<
496 SYM_CODE_START_LOCAL(__swpan_exit_el0) <<
497 __uaccess_ttbr0_enable x0, x1 <<
498 /* <<
499 * Enable errata workarounds only if r <<
500 * workaround currently required for T <<
501 * Cavium erratum 27456 (broadcast TLB <<
502 * corruption). <<
503 */ <<
504 b post_ttbr_update_workaround <<
505 SYM_CODE_END(__swpan_exit_el0) <<
506 #endif <<
507 <<
508 /* GPRs used by entry code */ <<
509 tsk .req x28 // current thr <<
510 <<
511 .text <<
512 <<
513 /* <<
514 * Exception vectors. <<
515 */ <<
516 .pushsection ".entry.text", "ax" <<
517 <<
518 .align 11 <<
519 SYM_CODE_START(vectors) <<
520 kernel_ventry 1, t, 64, sync <<
521 kernel_ventry 1, t, 64, irq <<
522 kernel_ventry 1, t, 64, fiq <<
523 kernel_ventry 1, t, 64, error <<
524 <<
525 kernel_ventry 1, h, 64, sync <<
526 kernel_ventry 1, h, 64, irq <<
527 kernel_ventry 1, h, 64, fiq <<
528 kernel_ventry 1, h, 64, error <<
529 <<
530 kernel_ventry 0, t, 64, sync <<
531 kernel_ventry 0, t, 64, irq <<
532 kernel_ventry 0, t, 64, fiq <<
533 kernel_ventry 0, t, 64, error <<
534 <<
535 kernel_ventry 0, t, 32, sync <<
536 kernel_ventry 0, t, 32, irq <<
537 kernel_ventry 0, t, 32, fiq <<
538 kernel_ventry 0, t, 32, error <<
539 SYM_CODE_END(vectors) <<
540 <<
541 #ifdef CONFIG_VMAP_STACK <<
542 SYM_CODE_START_LOCAL(__bad_stack) <<
543 /* <<
544 * We detected an overflow in kernel_v <<
545 * overflow stack. Stash the exception <<
546 * handler. <<
547 */ <<
548 <<
549 /* Restore the original x0 value */ <<
550 mrs x0, tpidrro_el0 <<
551 <<
552 /* <<
553 * Store the original GPRs to the new <<
554 * PT_REGS_SIZE) was stashed in tpidr_ <<
555 */ <<
556 sub sp, sp, #PT_REGS_SIZE <<
557 kernel_entry 1 <<
558 mrs x0, tpidr_el0 <<
559 add x0, x0, #PT_REGS_SIZE <<
560 str x0, [sp, #S_SP] <<
561 <<
562 /* Stash the regs for handle_bad_stack <<
563 mov x0, sp <<
564 <<
565 /* Time to die */ <<
566 bl handle_bad_stack <<
567 ASM_BUG() <<
568 SYM_CODE_END(__bad_stack) <<
569 #endif /* CONFIG_VMAP_STACK */ <<
570 <<
571 <<
572 .macro entry_handler el:req, ht:req, r <<
573 SYM_CODE_START_LOCAL(el\el\ht\()_\regsize\()_\ <<
574 kernel_entry \el, \regsize <<
575 mov x0, sp <<
576 bl el\el\ht\()_\regsize\()_\label <<
577 .if \el == 0 <<
578 b ret_to_user <<
579 .else <<
580 b ret_to_kernel <<
581 .endif <<
582 SYM_CODE_END(el\el\ht\()_\regsize\()_\label) <<
583 .endm <<
584 <<
585 /* <<
586 * Early exception handlers <<
587 */ <<
588 entry_handler 1, t, 64, sync <<
589 entry_handler 1, t, 64, irq <<
590 entry_handler 1, t, 64, fiq <<
591 entry_handler 1, t, 64, error <<
592 <<
593 entry_handler 1, h, 64, sync <<
594 entry_handler 1, h, 64, irq <<
595 entry_handler 1, h, 64, fiq <<
596 entry_handler 1, h, 64, error <<
597 <<
598 entry_handler 0, t, 64, sync <<
599 entry_handler 0, t, 64, irq <<
600 entry_handler 0, t, 64, fiq <<
601 entry_handler 0, t, 64, error <<
602 <<
603 entry_handler 0, t, 32, sync <<
604 entry_handler 0, t, 32, irq <<
605 entry_handler 0, t, 32, fiq <<
606 entry_handler 0, t, 32, error <<
607 <<
608 SYM_CODE_START_LOCAL(ret_to_kernel) <<
609 kernel_exit 1 <<
610 SYM_CODE_END(ret_to_kernel) <<
611 <<
612 SYM_CODE_START_LOCAL(ret_to_user) <<
613 ldr x19, [tsk, #TSK_TI_FLAGS] <<
614 enable_step_tsk x19, x2 <<
615 #ifdef CONFIG_GCC_PLUGIN_STACKLEAK <<
616 bl stackleak_erase_on_task_stack <<
617 #endif <<
618 kernel_exit 0 <<
619 SYM_CODE_END(ret_to_user) <<
620 <<
621 .popsection <<
622 <<
623 // Move from tramp_pg_dir to swapper_p <<
624 .macro tramp_map_kernel, tmp <<
625 mrs \tmp, ttbr1_el1 <<
626 add \tmp, \tmp, #TRAMP_SWAPPER_OFF <<
627 bic \tmp, \tmp, #USER_ASID_FLAG <<
628 msr ttbr1_el1, \tmp <<
629 #ifdef CONFIG_QCOM_FALKOR_ERRATUM_1003 <<
630 alternative_if ARM64_WORKAROUND_QCOM_FALKOR_E1 <<
631 /* ASID already in \tmp[63:48] */ <<
632 movk \tmp, #:abs_g2_nc:(TRAMP_VALIA <<
633 movk \tmp, #:abs_g1_nc:(TRAMP_VALIA <<
634 /* 2MB boundary containing the vectors <<
635 movk \tmp, #:abs_g0_nc:((TRAMP_VALI <<
636 isb <<
637 tlbi vae1, \tmp <<
638 dsb nsh <<
639 alternative_else_nop_endif <<
640 #endif /* CONFIG_QCOM_FALKOR_ERRATUM_1003 */ <<
641 .endm <<
642 <<
643 // Move from swapper_pg_dir to tramp_p <<
644 .macro tramp_unmap_kernel, tmp <<
645 mrs \tmp, ttbr1_el1 <<
646 sub \tmp, \tmp, #TRAMP_SWAPPER_OFF <<
647 orr \tmp, \tmp, #USER_ASID_FLAG <<
648 msr ttbr1_el1, \tmp <<
649 /* <<
650 * We avoid running the post_ttbr_upda <<
651 * it's only needed by Cavium ThunderX <<
652 * disabled. <<
653 */ <<
654 .endm <<
655 <<
656 .macro tramp_data_read_var <<
657 #ifdef CONFIG_RELOCATABLE <<
658 ldr \dst, .L__tramp_data_\ <<
659 .ifndef .L__tramp_data_\var <<
660 .pushsection ".entry.tramp.rodata", <<
661 .align 3 <<
662 .L__tramp_data_\var: <<
663 .quad \var <<
664 .popsection <<
665 .endif <<
666 #else <<
667 /* <<
668 * As !RELOCATABLE implies !RANDOMIZE_ <<
669 * compile time constant (and hence no <<
670 * <<
671 * As statically allocated kernel code <<
672 * 47 bits of the address space we can <<
673 * instruction to load the upper 16 bi <<
674 */ <<
675 movz \dst, :abs_g2_s:\var <<
676 movk \dst, :abs_g1_nc:\var <<
677 movk \dst, :abs_g0_nc:\var <<
678 #endif <<
679 .endm <<
680 <<
681 #define BHB_MITIGATION_NONE 0 <<
682 #define BHB_MITIGATION_LOOP 1 <<
683 #define BHB_MITIGATION_FW 2 <<
684 #define BHB_MITIGATION_INSN 3 <<
685 <<
686 .macro tramp_ventry, vector_start, reg <<
687 .align 7 <<
688 1: <<
689 .if \regsize == 64 <<
690 msr tpidrro_el0, x30 // Res <<
691 .endif <<
692 <<
693 .if \bhb == BHB_MITIGATION_LOOP <<
694 /* <<
695 * This sequence must appear before th <<
696 * ret out of tramp_ventry. It appears <<
697 */ <<
698 __mitigate_spectre_bhb_loop x30 <<
699 .endif // \bhb == BHB_MITIGATION_LOOP <<
700 <<
701 .if \bhb == BHB_MITIGATION_INSN <<
702 clearbhb <<
703 isb <<
704 .endif // \bhb == BHB_MITIGATION_INSN <<
705 <<
706 .if \kpti == 1 <<
707 /* <<
708 * Defend against branch aliasing atta <<
709 * entry onto the return stack and usi <<
710 * enter the full-fat kernel vectors. <<
711 */ <<
712 bl 2f <<
713 b . <<
714 2: <<
715 tramp_map_kernel x30 <<
716 alternative_insn isb, nop, ARM64_WORKAROUND_QC <<
717 tramp_data_read_var x30, vectors <<
718 alternative_if_not ARM64_WORKAROUND_CAVIUM_TX2 <<
719 prfm plil1strm, [x30, #(1b - \vecto <<
720 alternative_else_nop_endif <<
721 <<
722 msr vbar_el1, x30 <<
723 isb <<
724 .else <<
725 adr_l x30, vectors <<
726 .endif // \kpti == 1 <<
727 <<
728 .if \bhb == BHB_MITIGATION_FW <<
729 /* <<
730 * The firmware sequence must appear b <<
731 * i.e. the ret out of tramp_ventry. B <<
732 * mapped to save/restore the register <<
733 */ <<
734 __mitigate_spectre_bhb_fw <<
735 .endif // \bhb == BHB_MITIGATION_FW <<
736 <<
737 add x30, x30, #(1b - \vector_start <<
738 ret <<
739 .org 1b + 128 // Did we overflow the ventry <<
740 .endm <<
741 <<
742 .macro generate_tramp_vector, kpti, <<
743 .Lvector_start\@: <<
744 .space 0x400 <<
745 <<
746 .rept 4 <<
747 tramp_ventry .Lvector_start\@, 64, <<
748 .endr <<
749 .rept 4 <<
750 tramp_ventry .Lvector_start\@, 32, <<
751 .endr <<
752 .endm <<
753 <<
754 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 <<
755 /* <<
756 * Exception vectors trampoline. <<
757 * The order must match __bp_harden_el1_vector <<
758 * arm64_bp_harden_el1_vectors enum. <<
759 */ <<
760 .pushsection ".entry.tramp.text", "ax" <<
761 .align 11 <<
762 SYM_CODE_START_LOCAL_NOALIGN(tramp_vectors) <<
763 #ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY <<
764 generate_tramp_vector kpti=1, bhb=BH <<
765 generate_tramp_vector kpti=1, bhb=BH <<
766 generate_tramp_vector kpti=1, bhb=BH <<
767 #endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTO <<
768 generate_tramp_vector kpti=1, bhb=BH <<
769 SYM_CODE_END(tramp_vectors) <<
770 <<
771 SYM_CODE_START_LOCAL(tramp_exit) <<
772 tramp_unmap_kernel x29 <<
773 mrs x29, far_el1 <<
774 eret <<
775 sb <<
776 SYM_CODE_END(tramp_exit) <<
777 .popsection <<
778 #endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */ <<
779 <<
780 /* <<
781 * Exception vectors for spectre mitigations o <<
782 * kpti is not in use. <<
783 */ <<
784 .macro generate_el1_vector, bhb <<
785 .Lvector_start\@: <<
786 kernel_ventry 1, t, 64, sync <<
787 kernel_ventry 1, t, 64, irq <<
788 kernel_ventry 1, t, 64, fiq <<
789 kernel_ventry 1, t, 64, error <<
790 <<
791 kernel_ventry 1, h, 64, sync <<
792 kernel_ventry 1, h, 64, irq <<
793 kernel_ventry 1, h, 64, fiq <<
794 kernel_ventry 1, h, 64, error <<
795 <<
796 .rept 4 <<
797 tramp_ventry .Lvector_start\@, 64, <<
798 .endr <<
799 .rept 4 <<
800 tramp_ventry .Lvector_start\@, 32, <<
801 .endr <<
802 .endm <<
803 <<
804 /* The order must match tramp_vecs and the arm <<
805 .pushsection ".entry.text", "ax" <<
806 .align 11 <<
807 SYM_CODE_START(__bp_harden_el1_vectors) <<
808 #ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY <<
809 generate_el1_vector bhb=BHB_MITIGA <<
810 generate_el1_vector bhb=BHB_MITIGA <<
811 generate_el1_vector bhb=BHB_MITIGA <<
812 #endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTO <<
813 SYM_CODE_END(__bp_harden_el1_vectors) <<
814 .popsection <<
815 <<
816 <<
817 /* <<
818 * Register switch for AArch64. The callee-sav <<
819 * and restored. On entry: <<
820 * x0 = previous task_struct (must be preser <<
821 * x1 = next task_struct <<
822 * Previous and next are guaranteed not to be <<
823 * 4 *
824 */ !! 5 * Copyright (C) 1991, 1992 Linus Torvalds
825 SYM_FUNC_START(cpu_switch_to) <<
826 mov x10, #THREAD_CPU_CONTEXT <<
827 add x8, x0, x10 <<
828 mov x9, sp <<
829 stp x19, x20, [x8], #16 <<
830 stp x21, x22, [x8], #16 <<
831 stp x23, x24, [x8], #16 <<
832 stp x25, x26, [x8], #16 <<
833 stp x27, x28, [x8], #16 <<
834 stp x29, x9, [x8], #16 <<
835 str lr, [x8] <<
836 add x8, x1, x10 <<
837 ldp x19, x20, [x8], #16 <<
838 ldp x21, x22, [x8], #16 <<
839 ldp x23, x24, [x8], #16 <<
840 ldp x25, x26, [x8], #16 <<
841 ldp x27, x28, [x8], #16 <<
842 ldp x29, x9, [x8], #16 <<
843 ldr lr, [x8] <<
844 mov sp, x9 <<
845 msr sp_el0, x1 <<
846 ptrauth_keys_install_kernel x1, x8, x9 <<
847 scs_save x0 <<
848 scs_load_current <<
849 ret <<
850 SYM_FUNC_END(cpu_switch_to) <<
851 NOKPROBE(cpu_switch_to) <<
852 <<
853 /* <<
854 * This is how we return from a fork. <<
855 */ <<
856 SYM_CODE_START(ret_from_fork) <<
857 bl schedule_tail <<
858 cbz x19, 1f <<
859 mov x0, x20 <<
860 blr x19 <<
861 1: get_current_task tsk <<
862 mov x0, sp <<
863 bl asm_exit_to_user_mode <<
864 b ret_to_user <<
865 SYM_CODE_END(ret_from_fork) <<
866 NOKPROBE(ret_from_fork) <<
867 <<
868 /* <<
869 * void call_on_irq_stack(struct pt_regs *regs <<
870 * void (*func)(struct <<
871 * 6 *
872 * Calls func(regs) using this CPU's irq stack !! 7 * This file is subject to the terms and conditions of the GNU General Public
873 */ !! 8 * License. See the file README.legal in the main directory of this archive
874 SYM_FUNC_START(call_on_irq_stack) !! 9 * for more details.
875 #ifdef CONFIG_SHADOW_CALL_STACK !! 10 *
876 get_current_task x16 !! 11 * Linux/m68k support by Hamish Macdonald
877 scs_save x16 !! 12 *
878 ldr_this_cpu scs_sp, irq_shadow_call_s !! 13 * 68060 fixes by Jesper Skov
879 #endif <<
880 <<
881 /* Create a frame record to save our L <<
882 stp x29, x30, [sp, #-16]! <<
883 mov x29, sp <<
884 <<
885 ldr_this_cpu x16, irq_stack_ptr, x17 <<
886 <<
887 /* Move to the new stack and call the <<
888 add sp, x16, #IRQ_STACK_SIZE <<
889 blr x1 <<
890 <<
891 /* <<
892 * Restore the SP from the FP, and res <<
893 * record. <<
894 */ <<
895 mov sp, x29 <<
896 ldp x29, x30, [sp], #16 <<
897 scs_load_current <<
898 ret <<
899 SYM_FUNC_END(call_on_irq_stack) <<
900 NOKPROBE(call_on_irq_stack) <<
901 <<
902 #ifdef CONFIG_ARM_SDE_INTERFACE <<
903 <<
904 #include <asm/sdei.h> <<
905 #include <uapi/linux/arm_sdei.h> <<
906 <<
907 .macro sdei_handler_exit exit_mode <<
908 /* On success, this call never returns <<
909 cmp \exit_mode, #SDEI_EXIT_SMC <<
910 b.ne 99f <<
911 smc #0 <<
912 b . <<
913 99: hvc #0 <<
914 b . <<
915 .endm <<
916 <<
917 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 <<
918 /* <<
919 * The regular SDEI entry point may have been <<
920 * the kernel. This trampoline restores the ke <<
921 * argument accessible. <<
922 * 14 *
923 * This clobbers x4, __sdei_handler() will res <<
924 * copy. <<
925 */ 15 */
926 .pushsection ".entry.tramp.text", "ax" <<
927 SYM_CODE_START(__sdei_asm_entry_trampoline) <<
928 mrs x4, ttbr1_el1 <<
929 tbz x4, #USER_ASID_BIT, 1f <<
930 <<
931 tramp_map_kernel tmp=x4 <<
932 isb <<
933 mov x4, xzr <<
934 <<
935 /* <<
936 * Remember whether to unmap the kerne <<
937 */ <<
938 1: str x4, [x1, #(SDEI_EVENT_INTREGS <<
939 tramp_data_read_var x4, __sdei_asm <<
940 br x4 <<
941 SYM_CODE_END(__sdei_asm_entry_trampoline) <<
942 NOKPROBE(__sdei_asm_entry_trampoline) <<
943 16
944 /* 17 /*
945 * Make the exit call and restore the original !! 18 * entry.S contains the system-call and fault low-level handling routines.
>> 19 * This also contains the timer-interrupt handler, as well as all interrupts
>> 20 * and faults that can result in a task-switch.
>> 21 *
>> 22 * NOTE: This code handles signal-recognition, which happens every time
>> 23 * after a timer-interrupt and after each system call.
946 * 24 *
947 * x0 & x1: setup for the exit API call <<
948 * x2: exit_mode <<
949 * x4: struct sdei_registered_event argument f <<
950 */ 25 */
951 SYM_CODE_START(__sdei_asm_exit_trampoline) <<
952 ldr x4, [x4, #(SDEI_EVENT_INTREGS <<
953 cbnz x4, 1f <<
954 <<
955 tramp_unmap_kernel tmp=x4 <<
956 <<
957 1: sdei_handler_exit exit_mode=x2 <<
958 SYM_CODE_END(__sdei_asm_exit_trampoline) <<
959 NOKPROBE(__sdei_asm_exit_trampoline) <<
960 .popsection // .entry.tramp.text <<
961 #endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */ <<
962 26
963 /* 27 /*
964 * Software Delegated Exception entry point. !! 28 * 12/03/96 Jes: Currently we only support m68k single-cpu systems, so
965 * !! 29 * all pointers that used to be 'current' are now entry
966 * x0: Event number !! 30 * number 0 in the 'current_set' list.
967 * x1: struct sdei_registered_event argument f <<
968 * x2: interrupted PC <<
969 * x3: interrupted PSTATE <<
970 * x4: maybe clobbered by the trampoline <<
971 * 31 *
972 * Firmware has preserved x0->x17 for us, we m !! 32 * 6/05/00 RZ: addedd writeback completion after return from sighandler
973 * follow SMC-CC. We save (or retrieve) all th !! 33 * for 68040
974 * want them. <<
975 */ 34 */
976 SYM_CODE_START(__sdei_asm_handler) <<
977 stp x2, x3, [x1, #SDEI_EVENT_INTRE <<
978 stp x4, x5, [x1, #SDEI_EVENT_INTRE <<
979 stp x6, x7, [x1, #SDEI_EVENT_INTRE <<
980 stp x8, x9, [x1, #SDEI_EVENT_INTRE <<
981 stp x10, x11, [x1, #SDEI_EVENT_INT <<
982 stp x12, x13, [x1, #SDEI_EVENT_INT <<
983 stp x14, x15, [x1, #SDEI_EVENT_INT <<
984 stp x16, x17, [x1, #SDEI_EVENT_INT <<
985 stp x18, x19, [x1, #SDEI_EVENT_INT <<
986 stp x20, x21, [x1, #SDEI_EVENT_INT <<
987 stp x22, x23, [x1, #SDEI_EVENT_INT <<
988 stp x24, x25, [x1, #SDEI_EVENT_INT <<
989 stp x26, x27, [x1, #SDEI_EVENT_INT <<
990 stp x28, x29, [x1, #SDEI_EVENT_INT <<
991 mov x4, sp <<
992 stp lr, x4, [x1, #SDEI_EVENT_INTRE <<
993 <<
994 mov x19, x1 <<
995 <<
996 /* Store the registered-event for cras <<
997 ldrb w4, [x19, #SDEI_EVENT_PRIORITY <<
998 cbnz w4, 1f <<
999 adr_this_cpu dst=x5, sym=sdei_active_n <<
1000 b 2f <<
1001 1: adr_this_cpu dst=x5, sym=sdei_active_ <<
1002 2: str x19, [x5] <<
1003 35
1004 #ifdef CONFIG_VMAP_STACK !! 36 #include <linux/config.h>
1005 /* !! 37 #include <linux/linkage.h>
1006 * entry.S may have been using sp as !! 38 #include <asm/entry.h>
1007 * this is a normal or critical event !! 39 #include <asm/errno.h>
1008 * stack for this CPU. !! 40 #include <asm/setup.h>
1009 */ !! 41 #include <asm/segment.h>
1010 cbnz w4, 1f !! 42 #include <asm/traps.h>
1011 ldr_this_cpu dst=x5, sym=sdei_stack_n !! 43 #include <asm/unistd.h>
1012 b 2f <<
1013 1: ldr_this_cpu dst=x5, sym=sdei_stack_c <<
1014 2: mov x6, #SDEI_STACK_SIZE <<
1015 add x5, x5, x6 <<
1016 mov sp, x5 <<
1017 #endif <<
1018 44
1019 #ifdef CONFIG_SHADOW_CALL_STACK !! 45 #include "m68k_defs.h"
1020 /* Use a separate shadow call stack f <<
1021 cbnz w4, 3f <<
1022 ldr_this_cpu dst=scs_sp, sym=sdei_sha <<
1023 b 4f <<
1024 3: ldr_this_cpu dst=scs_sp, sym=sdei_sha <<
1025 4: <<
1026 #endif <<
1027 46
1028 /* !! 47 .globl system_call, buserr, trap
1029 * We may have interrupted userspace, !! 48 .globl resume, ret_from_exception
1030 * return-to either of these. We can' !! 49 .globl ret_from_signal
1031 */ !! 50 .globl inthandler, sys_call_table
1032 mrs x28, sp_el0 !! 51 .globl sys_fork, sys_clone, sys_vfork
1033 ldr_this_cpu dst=x0, sym=__entry_t !! 52 .globl ret_from_interrupt, bad_interrupt
1034 msr sp_el0, x0 !! 53
1035 !! 54 .text
1036 /* If we interrupted the kernel point !! 55 ENTRY(buserr)
1037 and x0, x3, #0xc !! 56 SAVE_ALL_INT
1038 mrs x1, CurrentEL !! 57 GET_CURRENT(%d0)
1039 cmp x0, x1 !! 58 movel %sp,%sp@- | stack frame pointer argument
1040 csel x29, x29, xzr, eq // fp !! 59 bsrl buserr_c
1041 csel x4, x2, xzr, eq // el !! 60 addql #4,%sp
1042 !! 61 jra ret_from_exception
1043 stp x29, x4, [sp, #-16]! !! 62
1044 mov x29, sp !! 63 ENTRY(trap)
1045 !! 64 SAVE_ALL_INT
1046 add x0, x19, #SDEI_EVENT_INTREGS !! 65 GET_CURRENT(%d0)
1047 mov x1, x19 !! 66 movel %sp,%sp@- | stack frame pointer argument
1048 bl __sdei_handler !! 67 bsrl trap_c
1049 !! 68 addql #4,%sp
1050 msr sp_el0, x28 !! 69 jra ret_from_exception
1051 /* restore regs >x17 that we clobbere !! 70
1052 mov x4, x19 // keep x4 fo !! 71 | After a fork we jump here directly from resume,
1053 ldp x28, x29, [x4, #SDEI_EVENT_IN !! 72 | so that %d1 contains the previous task
1054 ldp x18, x19, [x4, #SDEI_EVENT_IN !! 73 | schedule_tail now used regardless of CONFIG_SMP
1055 ldp lr, x1, [x4, #SDEI_EVENT_INTR !! 74 ENTRY(ret_from_fork)
1056 mov sp, x1 !! 75 movel %d1,%sp@-
1057 !! 76 jsr schedule_tail
1058 mov x1, x0 // ad !! 77 addql #4,%sp
1059 /* x0 = (x0 <= SDEI_EV_FAILED) ? !! 78 jra ret_from_exception
1060 * EVENT_COMPLETE:EVENT_COMPLETE_AND_ !! 79
1061 */ !! 80 badsys:
1062 cmp x0, #SDEI_EV_FAILED !! 81 movel #-ENOSYS,%sp@(PT_D0)
1063 mov_q x2, SDEI_1_0_FN_SDEI_EVENT_CO !! 82 jra ret_from_exception
1064 mov_q x3, SDEI_1_0_FN_SDEI_EVENT_CO !! 83
1065 csel x0, x2, x3, ls !! 84 do_trace:
1066 !! 85 movel #-ENOSYS,%sp@(PT_D0) | needed for strace
1067 ldr_l x2, sdei_exit_mode !! 86 subql #4,%sp
1068 !! 87 SAVE_SWITCH_STACK
1069 /* Clear the registered-event seen by !! 88 jbsr syscall_trace
1070 ldrb w3, [x4, #SDEI_EVENT_PRIORITY !! 89 RESTORE_SWITCH_STACK
1071 cbnz w3, 1f !! 90 addql #4,%sp
1072 adr_this_cpu dst=x5, sym=sdei_active_ !! 91 movel %sp@(PT_ORIG_D0),%d1
1073 b 2f !! 92 movel #-ENOSYS,%d0
1074 1: adr_this_cpu dst=x5, sym=sdei_active_ !! 93 cmpl #NR_syscalls,%d1
1075 2: str xzr, [x5] !! 94 jcc 1f
1076 !! 95 jbsr @(sys_call_table,%d1:l:4)@(0)
1077 alternative_if_not ARM64_UNMAP_KERNEL_AT_EL0 !! 96 1: movel %d0,%sp@(PT_D0) | save the return value
1078 sdei_handler_exit exit_mode=x2 !! 97 subql #4,%sp | dummy return address
1079 alternative_else_nop_endif !! 98 SAVE_SWITCH_STACK
1080 !! 99 jbsr syscall_trace
1081 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 !! 100
1082 tramp_alias dst=x5, sym=__sdei_as !! 101 ret_from_signal:
1083 br x5 !! 102 RESTORE_SWITCH_STACK
>> 103 addql #4,%sp
>> 104 /* on 68040 complete pending writebacks if any */
>> 105 #ifdef CONFIG_M68040
>> 106 bfextu %sp@(PT_VECTOR){#0,#4},%d0
>> 107 subql #7,%d0 | bus error frame ?
>> 108 jbne 1f
>> 109 movel %sp,%sp@-
>> 110 jbsr berr_040cleanup
>> 111 addql #4,%sp
>> 112 1:
>> 113 #endif
>> 114 jra ret_from_exception
>> 115
>> 116 ENTRY(system_call)
>> 117 SAVE_ALL_SYS
>> 118
>> 119 GET_CURRENT(%d1)
>> 120 | save top of frame
>> 121 movel %sp,%curptr@(TASK_THREAD+THREAD_ESP0)
>> 122
>> 123 tstb %curptr@(TASK_SYSCALL_TRACE)
>> 124 jne do_trace
>> 125 cmpl #NR_syscalls,%d0
>> 126 jcc badsys
>> 127 jbsr @(sys_call_table,%d0:l:4)@(0)
>> 128 movel %d0,%sp@(PT_D0) | save the return value
>> 129
>> 130 |oriw #0x0700,%sr
>> 131 movel %curptr@(TASK_WORK),%d0
>> 132 jne syscall_exit_work
>> 133 1: RESTORE_ALL
>> 134
>> 135 syscall_exit_work:
>> 136 btst #5,%sp@(PT_SR) | check if returning to kernel
>> 137 bnes 1b | if so, skip resched, signals
>> 138 tstw %d0
>> 139 jeq do_signal_return
>> 140 tstb %d0
>> 141 jne do_delayed_trace
>> 142
>> 143 pea resume_userspace
>> 144 jmp schedule
>> 145
>> 146 ret_from_exception:
>> 147 btst #5,%sp@(PT_SR) | check if returning to kernel
>> 148 bnes 1f | if so, skip resched, signals
>> 149 | only allow interrupts when we are really the last one on the
>> 150 | kernel stack, otherwise stack overflow can occur during
>> 151 | heavy interrupt load
>> 152 andw #ALLOWINT,%sr
>> 153
>> 154 resume_userspace:
>> 155 movel %curptr@(TASK_WORK),%d0
>> 156 lsrl #8,%d0
>> 157 jne exit_work
>> 158 1: RESTORE_ALL
>> 159
>> 160 exit_work:
>> 161 | save top of frame
>> 162 movel %sp,%curptr@(TASK_THREAD+THREAD_ESP0)
>> 163 tstb %d0
>> 164 jeq do_signal_return
>> 165
>> 166 pea resume_userspace
>> 167 jmp schedule
>> 168
>> 169 do_signal_return:
>> 170 |andw #ALLOWINT,%sr
>> 171 subql #4,%sp | dummy return address
>> 172 SAVE_SWITCH_STACK
>> 173 pea %sp@(SWITCH_STACK_SIZE)
>> 174 clrl %sp@-
>> 175 bsrl do_signal
>> 176 addql #8,%sp
>> 177 RESTORE_SWITCH_STACK
>> 178 addql #4,%sp
>> 179 jbra resume_userspace
>> 180
>> 181 do_delayed_trace:
>> 182 bclr #7,%sp@(PT_SR) | clear trace bit in SR
>> 183 pea 1 | send SIGTRAP
>> 184 movel %curptr,%sp@-
>> 185 pea LSIGTRAP
>> 186 jbsr send_sig
>> 187 addql #8,%sp
>> 188 addql #4,%sp
>> 189 jbra resume_userspace
>> 190
>> 191
>> 192 #if 0
>> 193 #ifdef CONFIG_AMIGA
>> 194 ami_inthandler:
>> 195 addql #1,irq_stat+CPUSTAT_LOCAL_IRQ_COUNT
>> 196 SAVE_ALL_INT
>> 197 GET_CURRENT(%d0)
>> 198
>> 199 bfextu %sp@(PT_VECTOR){#4,#12},%d0
>> 200 movel %d0,%a0
>> 201 addql #1,%a0@(kstat+STAT_IRQ-VECOFF(VEC_SPUR))
>> 202 movel %a0@(autoirq_list-VECOFF(VEC_SPUR)),%a0
>> 203
>> 204 | amiga vector int handler get the req mask instead of irq vector
>> 205 lea CUSTOMBASE,%a1
>> 206 movew %a1@(C_INTREQR),%d0
>> 207 andw %a1@(C_INTENAR),%d0
>> 208
>> 209 | prepare stack (push frame pointer, dev_id & req mask)
>> 210 pea %sp@
>> 211 movel %a0@(IRQ_DEVID),%sp@-
>> 212 movel %d0,%sp@-
>> 213 pea %pc@(ret_from_interrupt:w)
>> 214 jbra @(IRQ_HANDLER,%a0)@(0)
>> 215
>> 216 ENTRY(nmi_handler)
>> 217 rte
>> 218 #endif
>> 219 #endif
>> 220
>> 221 /*
>> 222 ** This is the main interrupt handler, responsible for calling process_int()
>> 223 */
>> 224 inthandler:
>> 225 SAVE_ALL_INT
>> 226 GET_CURRENT(%d0)
>> 227 addqb #1,%curptr@(TASK_INFO+TINFO_PREEMPT+2)
>> 228 | put exception # in d0
>> 229 bfextu %sp@(PT_VECTOR){#4,#10},%d0
>> 230
>> 231 movel %sp,%sp@-
>> 232 movel %d0,%sp@- | put vector # on stack
>> 233 #if defined(MACH_Q40_ONLY) && defined(CONFIG_BLK_DEV_FD)
>> 234 btstb #4,0xff000000 | Q40 floppy needs very special treatment ...
>> 235 jbeq 1f
>> 236 btstb #3,0xff000004
>> 237 jbeq 1f
>> 238 jbsr floppy_hardint
>> 239 jbra 3f
>> 240 1:
>> 241 #endif
>> 242 jbsr process_int | process the IRQ
>> 243 3: addql #8,%sp | pop parameters off stack
>> 244
>> 245 ret_from_interrupt:
>> 246 subqb #1,%curptr@(TASK_INFO+TINFO_PREEMPT+2)
>> 247 jeq 1f
>> 248 2:
>> 249 RESTORE_ALL
>> 250 1:
>> 251 moveq #(~ALLOWINT>>8)&0xff,%d0
>> 252 andb %sp@(PT_SR),%d0
>> 253 jne 2b
>> 254
>> 255 /* check if we need to do software interrupts */
>> 256 tstl irq_stat+CPUSTAT_SOFTIRQ_PENDING
>> 257 jeq ret_from_exception
>> 258 pea ret_from_exception
>> 259 jra do_softirq
>> 260
>> 261
>> 262 /* Handler for uninitialized and spurious interrupts */
>> 263
>> 264 bad_interrupt:
>> 265 addql #1,num_spurious
>> 266 rte
>> 267
>> 268 ENTRY(sys_fork)
>> 269 SAVE_SWITCH_STACK
>> 270 pea %sp@(SWITCH_STACK_SIZE)
>> 271 jbsr m68k_fork
>> 272 addql #4,%sp
>> 273 RESTORE_SWITCH_STACK
>> 274 rts
>> 275
>> 276 ENTRY(sys_clone)
>> 277 SAVE_SWITCH_STACK
>> 278 pea %sp@(SWITCH_STACK_SIZE)
>> 279 jbsr m68k_clone
>> 280 addql #4,%sp
>> 281 RESTORE_SWITCH_STACK
>> 282 rts
>> 283
>> 284 ENTRY(sys_vfork)
>> 285 SAVE_SWITCH_STACK
>> 286 pea %sp@(SWITCH_STACK_SIZE)
>> 287 jbsr m68k_vfork
>> 288 addql #4,%sp
>> 289 RESTORE_SWITCH_STACK
>> 290 rts
>> 291
>> 292 ENTRY(sys_sigsuspend)
>> 293 SAVE_SWITCH_STACK
>> 294 pea %sp@(SWITCH_STACK_SIZE)
>> 295 jbsr do_sigsuspend
>> 296 addql #4,%sp
>> 297 RESTORE_SWITCH_STACK
>> 298 rts
>> 299
>> 300 ENTRY(sys_rt_sigsuspend)
>> 301 SAVE_SWITCH_STACK
>> 302 pea %sp@(SWITCH_STACK_SIZE)
>> 303 jbsr do_rt_sigsuspend
>> 304 addql #4,%sp
>> 305 RESTORE_SWITCH_STACK
>> 306 rts
>> 307
>> 308 ENTRY(sys_sigreturn)
>> 309 SAVE_SWITCH_STACK
>> 310 jbsr do_sigreturn
>> 311 RESTORE_SWITCH_STACK
>> 312 rts
>> 313
>> 314 ENTRY(sys_rt_sigreturn)
>> 315 SAVE_SWITCH_STACK
>> 316 jbsr do_rt_sigreturn
>> 317 RESTORE_SWITCH_STACK
>> 318 rts
>> 319
>> 320 resume:
>> 321 /*
>> 322 * Beware - when entering resume, prev (the current task) is
>> 323 * in a0, next (the new task) is in a1,so don't change these
>> 324 * registers until their contents are no longer needed.
>> 325 */
>> 326
>> 327 /* save sr */
>> 328 movew %sr,%a0@(TASK_THREAD+THREAD_SR)
>> 329
>> 330 /* save fs (sfc,%dfc) (may be pointing to kernel memory) */
>> 331 movec %sfc,%d0
>> 332 movew %d0,%a0@(TASK_THREAD+THREAD_FS)
>> 333
>> 334 /* save usp */
>> 335 /* it is better to use a movel here instead of a movew 8*) */
>> 336 movec %usp,%d0
>> 337 movel %d0,%a0@(TASK_THREAD+THREAD_USP)
>> 338
>> 339 /* save non-scratch registers on stack */
>> 340 SAVE_SWITCH_STACK
>> 341
>> 342 /* save current kernel stack pointer */
>> 343 movel %sp,%a0@(TASK_THREAD+THREAD_KSP)
>> 344
>> 345 /* save floating point context */
>> 346 #ifndef CONFIG_M68KFPU_EMU_ONLY
>> 347 #ifdef CONFIG_M68KFPU_EMU
>> 348 tstl m68k_fputype
>> 349 jeq 3f
>> 350 #endif
>> 351 fsave %a0@(TASK_THREAD+THREAD_FPSTATE)
>> 352
>> 353 #if defined(CONFIG_M68060)
>> 354 #if !defined(CPU_M68060_ONLY)
>> 355 btst #3,m68k_cputype+3
>> 356 beqs 1f
>> 357 #endif
>> 358 /* The 060 FPU keeps status in bits 15-8 of the first longword */
>> 359 tstb %a0@(TASK_THREAD+THREAD_FPSTATE+2)
>> 360 jeq 3f
>> 361 #if !defined(CPU_M68060_ONLY)
>> 362 jra 2f
>> 363 #endif
>> 364 #endif /* CONFIG_M68060 */
>> 365 #if !defined(CPU_M68060_ONLY)
>> 366 1: tstb %a0@(TASK_THREAD+THREAD_FPSTATE)
>> 367 jeq 3f
1084 #endif 368 #endif
1085 SYM_CODE_END(__sdei_asm_handler) !! 369 2: fmovemx %fp0-%fp7,%a0@(TASK_THREAD+THREAD_FPREG)
1086 NOKPROBE(__sdei_asm_handler) !! 370 fmoveml %fpcr/%fpsr/%fpiar,%a0@(TASK_THREAD+THREAD_FPCNTL)
>> 371 3:
>> 372 #endif /* CONFIG_M68KFPU_EMU_ONLY */
>> 373 /* Return previous task in %d1 */
>> 374 movel %curptr,%d1
>> 375
>> 376 /* switch to new task (a1 contains new task) */
>> 377 movel %a1,%curptr
>> 378
>> 379 /* restore floating point context */
>> 380 #ifndef CONFIG_M68KFPU_EMU_ONLY
>> 381 #ifdef CONFIG_M68KFPU_EMU
>> 382 tstl m68k_fputype
>> 383 jeq 4f
>> 384 #endif
>> 385 #if defined(CONFIG_M68060)
>> 386 #if !defined(CPU_M68060_ONLY)
>> 387 btst #3,m68k_cputype+3
>> 388 beqs 1f
>> 389 #endif
>> 390 /* The 060 FPU keeps status in bits 15-8 of the first longword */
>> 391 tstb %a1@(TASK_THREAD+THREAD_FPSTATE+2)
>> 392 jeq 3f
>> 393 #if !defined(CPU_M68060_ONLY)
>> 394 jra 2f
>> 395 #endif
>> 396 #endif /* CONFIG_M68060 */
>> 397 #if !defined(CPU_M68060_ONLY)
>> 398 1: tstb %a1@(TASK_THREAD+THREAD_FPSTATE)
>> 399 jeq 3f
>> 400 #endif
>> 401 2: fmovemx %a1@(TASK_THREAD+THREAD_FPREG),%fp0-%fp7
>> 402 fmoveml %a1@(TASK_THREAD+THREAD_FPCNTL),%fpcr/%fpsr/%fpiar
>> 403 3: frestore %a1@(TASK_THREAD+THREAD_FPSTATE)
>> 404 4:
>> 405 #endif /* CONFIG_M68KFPU_EMU_ONLY */
>> 406
>> 407 /* restore the kernel stack pointer */
>> 408 movel %a1@(TASK_THREAD+THREAD_KSP),%sp
>> 409
>> 410 /* restore non-scratch registers */
>> 411 RESTORE_SWITCH_STACK
1087 412
1088 SYM_CODE_START(__sdei_handler_abort) !! 413 /* restore user stack pointer */
1089 mov_q x0, SDEI_1_0_FN_SDEI_EVENT_CO !! 414 movel %a1@(TASK_THREAD+THREAD_USP),%a0
1090 adr x1, 1f !! 415 movel %a0,%usp
1091 ldr_l x2, sdei_exit_mode !! 416
1092 sdei_handler_exit exit_mode=x2 !! 417 /* restore fs (sfc,%dfc) */
1093 // exit the handler and jump to the n !! 418 movew %a1@(TASK_THREAD+THREAD_FS),%a0
1094 // Exit will stomp x0-x17, PSTATE, EL !! 419 movec %a0,%sfc
1095 1: ret !! 420 movec %a0,%dfc
1096 SYM_CODE_END(__sdei_handler_abort) !! 421
1097 NOKPROBE(__sdei_handler_abort) !! 422 /* restore status register */
1098 #endif /* CONFIG_ARM_SDE_INTERFACE */ !! 423 movew %a1@(TASK_THREAD+THREAD_SR),%sr
>> 424
>> 425 rts
>> 426
>> 427 .data
>> 428 ALIGN
>> 429 sys_call_table:
>> 430 .long sys_ni_syscall /* 0 - old "setup()" system call*/
>> 431 .long sys_exit
>> 432 .long sys_fork
>> 433 .long sys_read
>> 434 .long sys_write
>> 435 .long sys_open /* 5 */
>> 436 .long sys_close
>> 437 .long sys_waitpid
>> 438 .long sys_creat
>> 439 .long sys_link
>> 440 .long sys_unlink /* 10 */
>> 441 .long sys_execve
>> 442 .long sys_chdir
>> 443 .long sys_time
>> 444 .long sys_mknod
>> 445 .long sys_chmod /* 15 */
>> 446 .long sys_chown16
>> 447 .long sys_ni_syscall /* old break syscall holder */
>> 448 .long sys_stat
>> 449 .long sys_lseek
>> 450 .long sys_getpid /* 20 */
>> 451 .long sys_mount
>> 452 .long sys_oldumount
>> 453 .long sys_setuid16
>> 454 .long sys_getuid16
>> 455 .long sys_stime /* 25 */
>> 456 .long sys_ptrace
>> 457 .long sys_alarm
>> 458 .long sys_fstat
>> 459 .long sys_pause
>> 460 .long sys_utime /* 30 */
>> 461 .long sys_ni_syscall /* old stty syscall holder */
>> 462 .long sys_ni_syscall /* old gtty syscall holder */
>> 463 .long sys_access
>> 464 .long sys_nice
>> 465 .long sys_ni_syscall /* 35 */ /* old ftime syscall holder */
>> 466 .long sys_sync
>> 467 .long sys_kill
>> 468 .long sys_rename
>> 469 .long sys_mkdir
>> 470 .long sys_rmdir /* 40 */
>> 471 .long sys_dup
>> 472 .long sys_pipe
>> 473 .long sys_times
>> 474 .long sys_ni_syscall /* old prof syscall holder */
>> 475 .long sys_brk /* 45 */
>> 476 .long sys_setgid16
>> 477 .long sys_getgid16
>> 478 .long sys_signal
>> 479 .long sys_geteuid16
>> 480 .long sys_getegid16 /* 50 */
>> 481 .long sys_acct
>> 482 .long sys_umount /* recycled never used phys() */
>> 483 .long sys_ni_syscall /* old lock syscall holder */
>> 484 .long sys_ioctl
>> 485 .long sys_fcntl /* 55 */
>> 486 .long sys_ni_syscall /* old mpx syscall holder */
>> 487 .long sys_setpgid
>> 488 .long sys_ni_syscall /* old ulimit syscall holder */
>> 489 .long sys_ni_syscall
>> 490 .long sys_umask /* 60 */
>> 491 .long sys_chroot
>> 492 .long sys_ustat
>> 493 .long sys_dup2
>> 494 .long sys_getppid
>> 495 .long sys_getpgrp /* 65 */
>> 496 .long sys_setsid
>> 497 .long sys_sigaction
>> 498 .long sys_sgetmask
>> 499 .long sys_ssetmask
>> 500 .long sys_setreuid16 /* 70 */
>> 501 .long sys_setregid16
>> 502 .long sys_sigsuspend
>> 503 .long sys_sigpending
>> 504 .long sys_sethostname
>> 505 .long sys_setrlimit /* 75 */
>> 506 .long sys_old_getrlimit
>> 507 .long sys_getrusage
>> 508 .long sys_gettimeofday
>> 509 .long sys_settimeofday
>> 510 .long sys_getgroups16 /* 80 */
>> 511 .long sys_setgroups16
>> 512 .long old_select
>> 513 .long sys_symlink
>> 514 .long sys_lstat
>> 515 .long sys_readlink /* 85 */
>> 516 .long sys_uselib
>> 517 .long sys_swapon
>> 518 .long sys_reboot
>> 519 .long old_readdir
>> 520 .long old_mmap /* 90 */
>> 521 .long sys_munmap
>> 522 .long sys_truncate
>> 523 .long sys_ftruncate
>> 524 .long sys_fchmod
>> 525 .long sys_fchown16 /* 95 */
>> 526 .long sys_getpriority
>> 527 .long sys_setpriority
>> 528 .long sys_ni_syscall /* old profil syscall holder */
>> 529 .long sys_statfs
>> 530 .long sys_fstatfs /* 100 */
>> 531 .long sys_ioperm
>> 532 .long sys_socketcall
>> 533 .long sys_syslog
>> 534 .long sys_setitimer
>> 535 .long sys_getitimer /* 105 */
>> 536 .long sys_newstat
>> 537 .long sys_newlstat
>> 538 .long sys_newfstat
>> 539 .long sys_ni_syscall
>> 540 .long sys_ni_syscall /* 110 */ /* iopl for i386 */
>> 541 .long sys_vhangup
>> 542 .long sys_ni_syscall /* obsolete idle() syscall */
>> 543 .long sys_ni_syscall /* vm86old for i386 */
>> 544 .long sys_wait4
>> 545 .long sys_swapoff /* 115 */
>> 546 .long sys_sysinfo
>> 547 .long sys_ipc
>> 548 .long sys_fsync
>> 549 .long sys_sigreturn
>> 550 .long sys_clone /* 120 */
>> 551 .long sys_setdomainname
>> 552 .long sys_newuname
>> 553 .long sys_cacheflush /* modify_ldt for i386 */
>> 554 .long sys_adjtimex
>> 555 .long sys_mprotect /* 125 */
>> 556 .long sys_sigprocmask
>> 557 .long sys_ni_syscall /* old "create_module" */
>> 558 .long sys_init_module
>> 559 .long sys_delete_module
>> 560 .long sys_ni_syscall /* 130 - old "get_kernel_syms" */
>> 561 .long sys_quotactl
>> 562 .long sys_getpgid
>> 563 .long sys_fchdir
>> 564 .long sys_bdflush
>> 565 .long sys_sysfs /* 135 */
>> 566 .long sys_personality
>> 567 .long sys_ni_syscall /* for afs_syscall */
>> 568 .long sys_setfsuid16
>> 569 .long sys_setfsgid16
>> 570 .long sys_llseek /* 140 */
>> 571 .long sys_getdents
>> 572 .long sys_select
>> 573 .long sys_flock
>> 574 .long sys_msync
>> 575 .long sys_readv /* 145 */
>> 576 .long sys_writev
>> 577 .long sys_getsid
>> 578 .long sys_fdatasync
>> 579 .long sys_sysctl
>> 580 .long sys_mlock /* 150 */
>> 581 .long sys_munlock
>> 582 .long sys_mlockall
>> 583 .long sys_munlockall
>> 584 .long sys_sched_setparam
>> 585 .long sys_sched_getparam /* 155 */
>> 586 .long sys_sched_setscheduler
>> 587 .long sys_sched_getscheduler
>> 588 .long sys_sched_yield
>> 589 .long sys_sched_get_priority_max
>> 590 .long sys_sched_get_priority_min /* 160 */
>> 591 .long sys_sched_rr_get_interval
>> 592 .long sys_nanosleep
>> 593 .long sys_mremap
>> 594 .long sys_setresuid16
>> 595 .long sys_getresuid16 /* 165 */
>> 596 .long sys_getpagesize
>> 597 .long sys_ni_syscall /* old sys_query_module */
>> 598 .long sys_poll
>> 599 .long sys_nfsservctl
>> 600 .long sys_setresgid16 /* 170 */
>> 601 .long sys_getresgid16
>> 602 .long sys_prctl
>> 603 .long sys_rt_sigreturn
>> 604 .long sys_rt_sigaction
>> 605 .long sys_rt_sigprocmask /* 175 */
>> 606 .long sys_rt_sigpending
>> 607 .long sys_rt_sigtimedwait
>> 608 .long sys_rt_sigqueueinfo
>> 609 .long sys_rt_sigsuspend
>> 610 .long sys_pread64 /* 180 */
>> 611 .long sys_pwrite64
>> 612 .long sys_lchown16;
>> 613 .long sys_getcwd
>> 614 .long sys_capget
>> 615 .long sys_capset /* 185 */
>> 616 .long sys_sigaltstack
>> 617 .long sys_sendfile
>> 618 .long sys_ni_syscall /* streams1 */
>> 619 .long sys_ni_syscall /* streams2 */
>> 620 .long sys_vfork /* 190 */
>> 621 .long sys_getrlimit
>> 622 .long sys_mmap2
>> 623 .long sys_truncate64
>> 624 .long sys_ftruncate64
>> 625 .long sys_stat64 /* 195 */
>> 626 .long sys_lstat64
>> 627 .long sys_fstat64
>> 628 .long sys_chown
>> 629 .long sys_getuid
>> 630 .long sys_getgid /* 200 */
>> 631 .long sys_geteuid
>> 632 .long sys_getegid
>> 633 .long sys_setreuid
>> 634 .long sys_setregid
>> 635 .long sys_getgroups /* 205 */
>> 636 .long sys_setgroups
>> 637 .long sys_fchown
>> 638 .long sys_setresuid
>> 639 .long sys_getresuid
>> 640 .long sys_setresgid /* 210 */
>> 641 .long sys_getresgid
>> 642 .long sys_lchown
>> 643 .long sys_setuid
>> 644 .long sys_setgid
>> 645 .long sys_setfsuid /* 215 */
>> 646 .long sys_setfsgid
>> 647 .long sys_pivot_root
>> 648 .long sys_ni_syscall
>> 649 .long sys_ni_syscall
>> 650 .long sys_getdents64 /* 220 */
>> 651 .long sys_gettid
>> 652 .long sys_tkill
>> 653 .long sys_setxattr
>> 654 .long sys_lsetxattr
>> 655 .long sys_fsetxattr /* 225 */
>> 656 .long sys_getxattr
>> 657 .long sys_lgetxattr
>> 658 .long sys_fgetxattr
>> 659 .long sys_listxattr
>> 660 .long sys_llistxattr /* 230 */
>> 661 .long sys_flistxattr
>> 662 .long sys_removexattr
>> 663 .long sys_lremovexattr
>> 664 .long sys_fremovexattr
>> 665 .long sys_futex /* 235 */
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.