1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * This file contains the 64-bit "server" Powe
4 * of the low level exception handling includi
5 * vectors, exception return, part of the slb
6 * handling and other fixed offset specific th
7 *
8 * This file is meant to be #included from hea
9 * position dependent assembly.
10 *
11 * Most of this originates from head_64.S and
12 * copyright history.
13 *
14 */
15
16 #include <linux/linkage.h>
17 #include <asm/hw_irq.h>
18 #include <asm/exception-64s.h>
19 #include <asm/ptrace.h>
20 #include <asm/cpuidle.h>
21 #include <asm/head-64.h>
22 #include <asm/feature-fixups.h>
23 #include <asm/kup.h>
24
25 /*
26 * Following are fixed section helper macros.
27 *
28 * EXC_REAL_BEGIN/END - real, unrelocated exc
29 * EXC_VIRT_BEGIN/END - virt (AIL), unrelocat
30 * TRAMP_REAL_BEGIN - real, unrelocated hel
31 * TRAMP_VIRT_BEGIN - virt, unreloc helpers
32 * EXC_COMMON - After switching to vi
33 */
34
35 #define EXC_REAL_BEGIN(name, start, size)
36 FIXED_SECTION_ENTRY_BEGIN_LOCATION(rea
37
38 #define EXC_REAL_END(name, start, size)
39 FIXED_SECTION_ENTRY_END_LOCATION(real_
40
41 #define EXC_VIRT_BEGIN(name, start, size)
42 FIXED_SECTION_ENTRY_BEGIN_LOCATION(vir
43
44 #define EXC_VIRT_END(name, start, size)
45 FIXED_SECTION_ENTRY_END_LOCATION(virt_
46
47 #define EXC_COMMON_BEGIN(name)
48 USE_TEXT_SECTION();
49 .balign IFETCH_ALIGN_BYTES;
50 .global name;
51 _ASM_NOKPROBE_SYMBOL(name);
52 DEFINE_FIXED_SYMBOL(name, text);
53 name:
54
55 #define TRAMP_REAL_BEGIN(name)
56 FIXED_SECTION_ENTRY_BEGIN(real_trampol
57
58 #define TRAMP_VIRT_BEGIN(name)
59 FIXED_SECTION_ENTRY_BEGIN(virt_trampol
60
61 #define EXC_REAL_NONE(start, size)
62 FIXED_SECTION_ENTRY_BEGIN_LOCATION(rea
63 FIXED_SECTION_ENTRY_END_LOCATION(real_
64
65 #define EXC_VIRT_NONE(start, size)
66 FIXED_SECTION_ENTRY_BEGIN_LOCATION(vir
67 FIXED_SECTION_ENTRY_END_LOCATION(virt_
68
69 /*
70 * We're short on space and time in the except
71 * use the normal LOAD_REG_IMMEDIATE macro to
72 * Instead we get the base of the kernel from
73 * part of label. This requires that the label
74 * that kernelbase be 64K aligned.
75 */
76 #define LOAD_HANDLER(reg, label)
77 ld reg,PACAKBASE(r13); /* get
78 ori reg,reg,FIXED_SYMBOL_ABS_ADDR(
79
80 #define __LOAD_HANDLER(reg, label, section)
81 ld reg,PACAKBASE(r13);
82 ori reg,reg,(ABS_ADDR(label, secti
83
84 /*
85 * Branches from unrelocated code (e.g., inter
86 * head-y require >64K offsets.
87 */
88 #define __LOAD_FAR_HANDLER(reg, label, section
89 ld reg,PACAKBASE(r13);
90 ori reg,reg,(ABS_ADDR(label, secti
91 addis reg,reg,(ABS_ADDR(label, secti
92
93 /*
94 * Interrupt code generation macros
95 */
96 #define IVEC .L_IVEC_\name\()
97 #define IHSRR .L_IHSRR_\name\()
98 #define IHSRR_IF_HVMODE .L_IHSRR_IF_HVMODE_\na
99 #define IAREA .L_IAREA_\name\()
100 #define IVIRT .L_IVIRT_\name\()
101 #define IISIDE .L_IISIDE_\name\()
102 #define ICFAR .L_ICFAR_\name\()
103 #define ICFAR_IF_HVMODE .L_ICFAR_IF_HVMODE_\na
104 #define IDAR .L_IDAR_\name\()
105 #define IDSISR .L_IDSISR_\name\()
106 #define IBRANCH_TO_COMMON .L_IBRANCH_TO_
107 #define IREALMODE_COMMON .L_IREALMODE_C
108 #define IMASK .L_IMASK_\name\()
109 #define IKVM_REAL .L_IKVM_REAL_\name\()
110 #define __IKVM_REAL(name) .L_IKVM_REAL_
111 #define IKVM_VIRT .L_IKVM_VIRT_\name\()
112 #define ISTACK .L_ISTACK_\name\()
113 #define __ISTACK(name) .L_ISTACK_ ## name
114 #define IKUAP .L_IKUAP_\name\()
115 #define IMSR_R12 .L_IMSR_R12_\name\()
116
117 #define INT_DEFINE_BEGIN(n)
118 .macro int_define_ ## n name
119
120 #define INT_DEFINE_END(n)
121 .endm ;
122 int_define_ ## n n ;
123 do_define_int n
124
125 .macro do_define_int name
126 .ifndef IVEC
127 .error "IVEC not defined"
128 .endif
129 .ifndef IHSRR
130 IHSRR=0
131 .endif
132 .ifndef IHSRR_IF_HVMODE
133 IHSRR_IF_HVMODE=0
134 .endif
135 .ifndef IAREA
136 IAREA=PACA_EXGEN
137 .endif
138 .ifndef IVIRT
139 IVIRT=1
140 .endif
141 .ifndef IISIDE
142 IISIDE=0
143 .endif
144 .ifndef ICFAR
145 ICFAR=1
146 .endif
147 .ifndef ICFAR_IF_HVMODE
148 ICFAR_IF_HVMODE=0
149 .endif
150 .ifndef IDAR
151 IDAR=0
152 .endif
153 .ifndef IDSISR
154 IDSISR=0
155 .endif
156 .ifndef IBRANCH_TO_COMMON
157 IBRANCH_TO_COMMON=1
158 .endif
159 .ifndef IREALMODE_COMMON
160 IREALMODE_COMMON=0
161 .else
162 .if ! IBRANCH_TO_COMMON
163 .error "IREALMODE_COMM
164 .endif
165 .endif
166 .ifndef IMASK
167 IMASK=0
168 .endif
169 .ifndef IKVM_REAL
170 IKVM_REAL=0
171 .endif
172 .ifndef IKVM_VIRT
173 IKVM_VIRT=0
174 .endif
175 .ifndef ISTACK
176 ISTACK=1
177 .endif
178 .ifndef IKUAP
179 IKUAP=1
180 .endif
181 .ifndef IMSR_R12
182 IMSR_R12=0
183 .endif
184 .endm
185
186 /*
187 * All interrupts which set HSRR registers, as
188 * syscall when invoked with "sc 1" switch to
189 * so they all generally need to test whether
190 *
191 * Note: SRESET and MCE may also be sent to th
192 * taken with MSR[HV]=0.
193 *
194 * Interrupts which set SRR registers (with th
195 * elevate to MSR[HV]=1 mode, though most can
196 * MSR[HV]=1 (e.g., bare metal kernel and use
197 * not need to test whether a guest is running
198 * the guest directly, including nested HV KVM
199 *
200 * The exception is PR KVM, where the guest ru
201 * runs with MSR[HV]=0, so the host takes all
202 * guest. PR KVM runs with LPCR[AIL]=0 which c
203 * delivered to the real-mode entry point, the
204 * KVM in their real mode handlers, and only w
205 *
206 * Interrupts that are taken in MSR[HV]=0 and
207 * delivered in real-mode when the MMU is in h
208 * registers are not set appropriately to tran
209 * radix mode these can be delivered in virt-m
210 * used implicitly (see: effective LPID, effec
211 */
212
213 /*
214 * If an interrupt is taken while a guest is r
215 * to KVM to handle.
216 */
217
218 .macro KVMTEST name handler
219 #ifdef CONFIG_KVM_BOOK3S_64_HANDLER
220 lbz r10,HSTATE_IN_GUEST(r13)
221 cmpwi r10,0
222 /* HSRR variants have the 0x2 bit adde
223 .if IHSRR_IF_HVMODE
224 BEGIN_FTR_SECTION
225 li r10,(IVEC + 0x2)
226 FTR_SECTION_ELSE
227 li r10,(IVEC)
228 ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMO
229 .elseif IHSRR
230 li r10,(IVEC + 0x2)
231 .else
232 li r10,(IVEC)
233 .endif
234 bne \handler
235 #endif
236 .endm
237
238 /*
239 * This is the BOOK3S interrupt entry code mac
240 *
241 * This can result in one of several things ha
242 * - Branch to the _common handler, relocated,
243 * These are normal interrupts (synchronous
244 * the kernel.
245 * - Branch to KVM, relocated but real mode in
246 * These occur when HSTATE_IN_GUEST is set.
247 * / intended for host or guest kernel, but
248 * because the machine state is set for gues
249 * - Branch to the masked handler, unrelocated
250 * These occur when maskable asynchronous in
251 * irq_soft_mask set.
252 * - Branch to an "early" handler in real mode
253 * This is done if early=1. MCE and HMI use
254 * mode.
255 * - Fall through and continue executing in re
256 * This is done if early=2.
257 */
258
259 .macro GEN_BRANCH_TO_COMMON name, virt
260 .if IREALMODE_COMMON
261 LOAD_HANDLER(r10, \name\()_common)
262 mtctr r10
263 bctr
264 .else
265 .if \virt
266 #ifndef CONFIG_RELOCATABLE
267 b \name\()_common_virt
268 #else
269 LOAD_HANDLER(r10, \name\()_common_virt
270 mtctr r10
271 bctr
272 #endif
273 .else
274 LOAD_HANDLER(r10, \name\()_common_real
275 mtctr r10
276 bctr
277 .endif
278 .endif
279 .endm
280
281 .macro GEN_INT_ENTRY name, virt, ool=0
282 SET_SCRATCH0(r13)
283 GET_PACA(r13)
284 std r9,IAREA+EX_R9(r13)
285 BEGIN_FTR_SECTION
286 mfspr r9,SPRN_PPR
287 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
288 HMT_MEDIUM
289 std r10,IAREA+EX_R10(r13)
290 .if ICFAR
291 BEGIN_FTR_SECTION
292 mfspr r10,SPRN_CFAR
293 END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
294 .elseif ICFAR_IF_HVMODE
295 BEGIN_FTR_SECTION
296 BEGIN_FTR_SECTION_NESTED(69)
297 mfspr r10,SPRN_CFAR
298 END_FTR_SECTION_NESTED(CPU_FTR_CFAR, CPU_FTR
299 FTR_SECTION_ELSE
300 BEGIN_FTR_SECTION_NESTED(69)
301 li r10,0
302 END_FTR_SECTION_NESTED(CPU_FTR_CFAR, CPU_FTR
303 ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU
304 .endif
305 .if \ool
306 .if !\virt
307 b tramp_real_\name
308 .pushsection .text
309 TRAMP_REAL_BEGIN(tramp_real_\name)
310 .else
311 b tramp_virt_\name
312 .pushsection .text
313 TRAMP_VIRT_BEGIN(tramp_virt_\name)
314 .endif
315 .endif
316
317 BEGIN_FTR_SECTION
318 std r9,IAREA+EX_PPR(r13)
319 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
320 .if ICFAR || ICFAR_IF_HVMODE
321 BEGIN_FTR_SECTION
322 std r10,IAREA+EX_CFAR(r13)
323 END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
324 .endif
325 INTERRUPT_TO_KERNEL
326 mfctr r10
327 std r10,IAREA+EX_CTR(r13)
328 mfcr r9
329 std r11,IAREA+EX_R11(r13)
330 std r12,IAREA+EX_R12(r13)
331
332 /*
333 * DAR/DSISR, SCRATCH0 must be read be
334 * because a d-side MCE will clobber t
335 * not recoverable if they are live.
336 */
337 GET_SCRATCH0(r10)
338 std r10,IAREA+EX_R13(r13)
339 .if IDAR && !IISIDE
340 .if IHSRR
341 mfspr r10,SPRN_HDAR
342 .else
343 mfspr r10,SPRN_DAR
344 .endif
345 std r10,IAREA+EX_DAR(r13)
346 .endif
347 .if IDSISR && !IISIDE
348 .if IHSRR
349 mfspr r10,SPRN_HDSISR
350 .else
351 mfspr r10,SPRN_DSISR
352 .endif
353 stw r10,IAREA+EX_DSISR(r13)
354 .endif
355
356 .if IHSRR_IF_HVMODE
357 BEGIN_FTR_SECTION
358 mfspr r11,SPRN_HSRR0 /* sav
359 mfspr r12,SPRN_HSRR1 /* and
360 FTR_SECTION_ELSE
361 mfspr r11,SPRN_SRR0 /* sav
362 mfspr r12,SPRN_SRR1 /* and
363 ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMO
364 .elseif IHSRR
365 mfspr r11,SPRN_HSRR0 /* sav
366 mfspr r12,SPRN_HSRR1 /* and
367 .else
368 mfspr r11,SPRN_SRR0 /* sav
369 mfspr r12,SPRN_SRR1 /* and
370 .endif
371
372 .if IBRANCH_TO_COMMON
373 GEN_BRANCH_TO_COMMON \name \virt
374 .endif
375
376 .if \ool
377 .popsection
378 .endif
379 .endm
380
381 /*
382 * __GEN_COMMON_ENTRY is required to receive t
383 * entry, except in the case of the real-mode
384 * __GEN_REALMODE_COMMON_ENTRY.
385 *
386 * This switches to virtual mode and sets MSR[
387 */
388 .macro __GEN_COMMON_ENTRY name
389 DEFINE_FIXED_SYMBOL(\name\()_common_real, text
390 \name\()_common_real:
391 .if IKVM_REAL
392 KVMTEST \name kvm_interrupt
393 .endif
394
395 ld r10,PACAKMSR(r13) /* get
396 /* MSR[RI] is clear iff using SRR regs
397 .if IHSRR_IF_HVMODE
398 BEGIN_FTR_SECTION
399 xori r10,r10,MSR_RI
400 END_FTR_SECTION_IFCLR(CPU_FTR_HVMODE)
401 .elseif ! IHSRR
402 xori r10,r10,MSR_RI
403 .endif
404 mtmsrd r10
405
406 .if IVIRT
407 .if IKVM_VIRT
408 b 1f /* skip the virt test comin
409 .endif
410
411 .balign IFETCH_ALIGN_BYTES
412 DEFINE_FIXED_SYMBOL(\name\()_common_virt, text
413 \name\()_common_virt:
414 .if IKVM_VIRT
415 KVMTEST \name kvm_interrupt
416 1:
417 .endif
418 .endif /* IVIRT */
419 .endm
420
421 /*
422 * Don't switch to virt mode. Used for early M
423 * want to run in real mode.
424 */
425 .macro __GEN_REALMODE_COMMON_ENTRY name
426 DEFINE_FIXED_SYMBOL(\name\()_common_real, text
427 \name\()_common_real:
428 .if IKVM_REAL
429 KVMTEST \name kvm_interrupt
430 .endif
431 .endm
432
433 .macro __GEN_COMMON_BODY name
434 .if IMASK
435 .if ! ISTACK
436 .error "No support for masked
437 .endif
438
439 /* If coming from user, skip s
440 andi. r10,r12,MSR_PR
441 bne 3f
442
443 /*
444 * Kernel code running below _
445 * implicitly soft-masked if i
446 * in the soft mask table.
447 */
448 LOAD_HANDLER(r10, __end_soft_m
449 cmpld r11,r10
450 bge+ 1f
451
452 /* SEARCH_SOFT_MASK_TABLE clob
453 mtctr r12
454 stw r9,PACA_EXGEN+EX_CCR(r
455 SEARCH_SOFT_MASK_TABLE
456 cmpdi r12,0
457 mfctr r12 /* Res
458 lwz r9,PACA_EXGEN+EX_CCR(r
459 beq 1f /* Not
460 li r10,IMASK
461 b 2f /* In
462
463 /* Test the soft mask state ag
464 1: lbz r10,PACAIRQSOFTMASK(r1
465 2: andi. r10,r10,IMASK
466 /* Associate vector numbers wi
467 .if IVEC == 0x500 || IVEC == 0
468 li r10,PACA_IRQ_EE
469 .elseif IVEC == 0x900
470 li r10,PACA_IRQ_DEC
471 .elseif IVEC == 0xa00 || IVEC
472 li r10,PACA_IRQ_DBELL
473 .elseif IVEC == 0xe60
474 li r10,PACA_IRQ_HMI
475 .elseif IVEC == 0xf00
476 li r10,PACA_IRQ_PMI
477 .else
478 .abort "Bad maskable vector"
479 .endif
480
481 .if IHSRR_IF_HVMODE
482 BEGIN_FTR_SECTION
483 bne masked_Hinterrupt
484 FTR_SECTION_ELSE
485 bne masked_interrupt
486 ALT_FTR_SECTION_END_IFSET(CPU_
487 .elseif IHSRR
488 bne masked_Hinterrupt
489 .else
490 bne masked_interrupt
491 .endif
492 .endif
493
494 .if ISTACK
495 andi. r10,r12,MSR_PR /* See
496 3: mr r10,r1 /* Sav
497 subi r1,r1,INT_FRAME_SIZE /* all
498 beq- 100f
499 ld r1,PACAKSAVE(r13) /* ker
500 100: tdgei r1,-INT_FRAME_SIZE /* tra
501 EMIT_BUG_ENTRY 100b,__FILE__,__LINE__,
502 .endif
503
504 std r9,_CCR(r1) /* sav
505 std r11,_NIP(r1) /* sav
506 std r12,_MSR(r1) /* sav
507 std r10,0(r1) /* mak
508 std r0,GPR0(r1) /* sav
509 std r10,GPR1(r1) /* sav
510 SANITIZE_GPR(0)
511
512 /* Mark our [H]SRRs valid for return *
513 li r10,1
514 .if IHSRR_IF_HVMODE
515 BEGIN_FTR_SECTION
516 stb r10,PACAHSRR_VALID(r13)
517 FTR_SECTION_ELSE
518 stb r10,PACASRR_VALID(r13)
519 ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMO
520 .elseif IHSRR
521 stb r10,PACAHSRR_VALID(r13)
522 .else
523 stb r10,PACASRR_VALID(r13)
524 .endif
525
526 .if ISTACK
527 .if IKUAP
528 kuap_save_amr_and_lock r9, r10, cr1, c
529 .endif
530 beq 101f /* if
531 BEGIN_FTR_SECTION
532 ld r9,IAREA+EX_PPR(r13) /* Rea
533 std r9,_PPR(r1)
534 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
535 101:
536 .else
537 .if IKUAP
538 kuap_save_amr_and_lock r9, r10, cr1
539 .endif
540 .endif
541
542 /* Save original regs values from save
543 ld r9,IAREA+EX_R9(r13) /* mov
544 ld r10,IAREA+EX_R10(r13)
545 std r9,GPR9(r1)
546 std r10,GPR10(r1)
547 ld r9,IAREA+EX_R11(r13) /* mov
548 ld r10,IAREA+EX_R12(r13)
549 ld r11,IAREA+EX_R13(r13)
550 std r9,GPR11(r1)
551 std r10,GPR12(r1)
552 std r11,GPR13(r1)
553 .if !IMSR_R12
554 SANITIZE_GPRS(9, 12)
555 .else
556 SANITIZE_GPRS(9, 11)
557 .endif
558
559 SAVE_NVGPRS(r1)
560 SANITIZE_NVGPRS()
561
562 .if IDAR
563 .if IISIDE
564 ld r10,_NIP(r1)
565 .else
566 ld r10,IAREA+EX_DAR(r13)
567 .endif
568 std r10,_DAR(r1)
569 .endif
570
571 .if IDSISR
572 .if IISIDE
573 ld r10,_MSR(r1)
574 lis r11,DSISR_SRR1_MATCH_64S@h
575 and r10,r10,r11
576 .else
577 lwz r10,IAREA+EX_DSISR(r13)
578 .endif
579 std r10,_DSISR(r1)
580 .endif
581
582 BEGIN_FTR_SECTION
583 .if ICFAR || ICFAR_IF_HVMODE
584 ld r10,IAREA+EX_CFAR(r13)
585 .else
586 li r10,0
587 .endif
588 std r10,ORIG_GPR3(r1)
589 END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
590 ld r10,IAREA+EX_CTR(r13)
591 std r10,_CTR(r1)
592 SAVE_GPRS(2, 8, r1) /* sav
593 SANITIZE_GPRS(2, 8)
594 mflr r9 /* Get
595 LOAD_PACA_TOC() /* get
596 std r9,_LINK(r1)
597 lbz r10,PACAIRQSOFTMASK(r13)
598 mfspr r11,SPRN_XER /* sav
599 std r10,SOFTE(r1)
600 std r11,_XER(r1)
601 li r9,IVEC
602 std r9,_TRAP(r1) /* set
603 li r10,0
604 LOAD_REG_IMMEDIATE(r11, STACK_FRAME_RE
605 std r10,RESULT(r1) /* cle
606 std r11,STACK_INT_FRAME_MARKER(r1)
607 .endm
608
609 /*
610 * On entry r13 points to the paca, r9-r13 are
611 * r9 contains the saved CR, r11 and r12 conta
612 * SRR1, and relocation is on.
613 *
614 * If stack=0, then the stack is already set i
615 * PPR save and CPU accounting is not done for
616 */
617 .macro GEN_COMMON name
618 __GEN_COMMON_ENTRY \name
619 __GEN_COMMON_BODY \name
620 .endm
621
622 .macro SEARCH_RESTART_TABLE
623 #ifdef CONFIG_RELOCATABLE
624 mr r12,r2
625 LOAD_PACA_TOC()
626 LOAD_REG_ADDR(r9, __start___restart_ta
627 LOAD_REG_ADDR(r10, __stop___restart_ta
628 mr r2,r12
629 #else
630 LOAD_REG_IMMEDIATE_SYM(r9, r12, __star
631 LOAD_REG_IMMEDIATE_SYM(r10, r12, __sto
632 #endif
633 300:
634 cmpd r9,r10
635 beq 302f
636 ld r12,0(r9)
637 cmpld r11,r12
638 blt 301f
639 ld r12,8(r9)
640 cmpld r11,r12
641 bge 301f
642 ld r12,16(r9)
643 b 303f
644 301:
645 addi r9,r9,24
646 b 300b
647 302:
648 li r12,0
649 303:
650 .endm
651
652 .macro SEARCH_SOFT_MASK_TABLE
653 #ifdef CONFIG_RELOCATABLE
654 mr r12,r2
655 LOAD_PACA_TOC()
656 LOAD_REG_ADDR(r9, __start___soft_mask_
657 LOAD_REG_ADDR(r10, __stop___soft_mask_
658 mr r2,r12
659 #else
660 LOAD_REG_IMMEDIATE_SYM(r9, r12, __star
661 LOAD_REG_IMMEDIATE_SYM(r10, r12, __sto
662 #endif
663 300:
664 cmpd r9,r10
665 beq 302f
666 ld r12,0(r9)
667 cmpld r11,r12
668 blt 301f
669 ld r12,8(r9)
670 cmpld r11,r12
671 bge 301f
672 li r12,1
673 b 303f
674 301:
675 addi r9,r9,16
676 b 300b
677 302:
678 li r12,0
679 303:
680 .endm
681
682 /*
683 * Restore all registers including H/SRR0/1 sa
684 * standard exception.
685 */
686 .macro EXCEPTION_RESTORE_REGS hsrr=0
687 /* Move original SRR0 and SRR1 into th
688 ld r9,_MSR(r1)
689 li r10,0
690 .if \hsrr
691 mtspr SPRN_HSRR1,r9
692 stb r10,PACAHSRR_VALID(r13)
693 .else
694 mtspr SPRN_SRR1,r9
695 stb r10,PACASRR_VALID(r13)
696 .endif
697 ld r9,_NIP(r1)
698 .if \hsrr
699 mtspr SPRN_HSRR0,r9
700 .else
701 mtspr SPRN_SRR0,r9
702 .endif
703 ld r9,_CTR(r1)
704 mtctr r9
705 ld r9,_XER(r1)
706 mtxer r9
707 ld r9,_LINK(r1)
708 mtlr r9
709 ld r9,_CCR(r1)
710 mtcr r9
711 SANITIZE_RESTORE_NVGPRS()
712 REST_GPRS(2, 13, r1)
713 REST_GPR(0, r1)
714 /* restore original r1. */
715 ld r1,GPR1(r1)
716 .endm
717
718 /*
719 * EARLY_BOOT_FIXUP - Fix real-mode interrupt
720 *
721 * There's a short window during boot where al
722 * little endian, any exceptions will cause th
723 * endian. For example a WARN() boils down to
724 * cause a program check, and we end up here b
725 * mode. The first instruction of the program
726 * below) is an mtsprg, which when executed in
727 * a ~3GB displacement from r3. The content of
728 * from some random location, and depending on
729 * checkstop, or an infinitely recursive page
730 *
731 * So to handle that case we have a trampoline
732 * the wrong endian and flip us back to the co
733 * MSR[LE] using mtmsr, so we have to use rfid
734 * as well as a GPR. To do that we use SPRG0/2
735 * the paca. SPRG3 is user readable, but this
736 * early in boot, and SPRG3 will be reinitiali
737 * userspace starts.
738 */
739 .macro EARLY_BOOT_FIXUP
740 BEGIN_FTR_SECTION
741 #ifdef CONFIG_CPU_LITTLE_ENDIAN
742 tdi 0,0,0x48 // Trap never, or in
743 b 2f // Skip trampoline i
744 .long 0xa643707d // mtsprg 0, r11
745 .long 0xa6027a7d // mfsrr0 r11
746 .long 0xa643727d // mtsprg 2, r11
747 .long 0xa6027b7d // mfsrr1 r11
748 .long 0xa643737d // mtsprg 3, r11
749 .long 0xa600607d // mfmsr r11
750 .long 0x01006b69 // xori r11, r11,
751 .long 0xa6037b7d // mtsrr1 r11
752 /*
753 * This is 'li r11,1f' where 1f is th
754 * label, byteswapped into the SI fiel
755 */
756 .long 0x00006039 | \
757 ((ABS_ADDR(1f, real_vectors) &
758 ((ABS_ADDR(1f, real_vectors) &
759 .long 0xa6037a7d // mtsrr0 r11
760 .long 0x2400004c // rfid
761 1:
762 mfsprg r11, 3
763 mtsrr1 r11 // Restore SRR1
764 mfsprg r11, 2
765 mtsrr0 r11 // Restore SRR0
766 mfsprg r11, 0 // Restore r11
767 2:
768 #endif
769 /*
770 * program check could hit at any time
771 * MSR[ME] in early boot. So check if
772 * yet, and spin forever if not.
773 */
774 mtsprg 0, r11
775 mfcr r11
776 cmpdi r13, 0
777 beq .
778 mtcr r11
779 mfsprg r11, 0
780 END_FTR_SECTION(0, 1) // nop out after boo
781 .endm
782
783 /*
784 * There are a few constraints to be concerned
785 * - Real mode exceptions code/data must be lo
786 * - Virtual mode exceptions must be mapped at
787 * - Fixed location code must not call directl
788 * area when built with CONFIG_RELOCATABLE.
789 * must be used.
790 * - LOAD_HANDLER targets must be within first
791 * virtual 0xc00...
792 * - Conditional branch targets must be within
793 *
794 * "Virtual exceptions" run with relocation on
795 * therefore don't have to run in physically l
796 * virtual mode kernel code. However on reloca
797 * to branch to KERNELBASE offset because the
798 * the exception vectors) may be located elsew
799 *
800 * Virtual exceptions correspond with physical
801 * are offset by 0xc000000000000000 and also t
802 * offset applied. Virtual exceptions are enab
803 * Interrupt Location (AIL) bit set in the LPC
804 * guarantee they will be delivered virtually.
805 * cause exceptions to be delivered in real mo
806 *
807 * The scv instructions are a special case. Th
808 * scv exceptions have unique reentrancy prope
809 *
810 * It's impossible to receive interrupts below
811 *
812 * KVM: None of the virtual exceptions are fro
813 * escalated to HV=1 from HV=0 is delivered vi
814 *
815 *
816 * We layout physical memory as follows:
817 * 0x0000 - 0x00ff : Secondary processor spin
818 * 0x0100 - 0x18ff : Real mode pSeries interru
819 * 0x1900 - 0x2fff : Real mode trampolines
820 * 0x3000 - 0x58ff : Relon (IR=1,DR=1) mode pS
821 * 0x5900 - 0x6fff : Relon mode trampolines
822 * 0x7000 - 0x7fff : FWNMI data area
823 * 0x8000 - .... : Common interrupt handlers
824 * setup code, rest of kerne
825 *
826 * We could reclaim 0x4000-0x42ff for real mod
827 * is necessary. Until then it's more consiste
828 * vectors there.
829 */
830 OPEN_FIXED_SECTION(real_vectors, 0x0100
831 OPEN_FIXED_SECTION(real_trampolines, 0x1900
832 OPEN_FIXED_SECTION(virt_vectors, 0x3000
833 OPEN_FIXED_SECTION(virt_trampolines, 0x5900
834
835 #ifdef CONFIG_PPC_POWERNV
836 .globl start_real_trampolines
837 .globl end_real_trampolines
838 .globl start_virt_trampolines
839 .globl end_virt_trampolines
840 #endif
841
842 #if defined(CONFIG_PPC_PSERIES) || defined(CON
843 /*
844 * Data area reserved for FWNMI option.
845 * This address (0x7000) is fixed by the RPA.
846 * pseries and powernv need to keep the whole
847 * 0x7000 to 0x8000 free for use by the firmwa
848 */
849 ZERO_FIXED_SECTION(fwnmi_page, 0x7000
850 OPEN_TEXT_SECTION(0x8000)
851 #else
852 OPEN_TEXT_SECTION(0x7000)
853 #endif
854
855 USE_FIXED_SECTION(real_vectors)
856
857 /*
858 * This is the start of the interrupt handlers
859 * This code runs with relocation off.
860 * Code from here to __end_interrupts gets cop
861 * address 0x100 when we are running a relocat
862 * Therefore any relative branches in this sec
863 * branch to labels in this section.
864 */
865 .globl __start_interrupts
866 __start_interrupts:
867
868 /**
869 * Interrupt 0x3000 - System Call Vectored Int
870 * This is a synchronous interrupt invoked wit
871 * system call does not alter the HV bit, so i
872 *
873 * Handling:
874 * scv instructions enter the kernel without c
875 * In particular, this means we can take a mas
876 * in the scv handler, which is unlike any oth
877 * by treating the instruction addresses in th
878 * by adding a SOFT_MASK_TABLE entry for them.
879 *
880 * AIL-0 mode scv exceptions go to 0x17000-0x1
881 * ensure scv is never executed with relocatio
882 * should never happen.
883 *
884 * Before leaving the following inside-__end_s
885 * following must be true:
886 * - MSR[PR]=1 (i.e., return to userspace)
887 * - MSR_EE|MSR_RI is clear (no reentrant exce
888 * - Standard kernel environment is set up (st
889 *
890 * KVM:
891 * These interrupts do not elevate HV 0->1, so
892 * ensures that FSCR[SCV] is disabled whenever
893 *
894 * Call convention:
895 *
896 * syscall register convention is in Documenta
897 */
898 EXC_VIRT_BEGIN(system_call_vectored, 0x3000, 0
899 /* SCV 0 */
900 mr r9,r13
901 GET_PACA(r13)
902 mflr r11
903 mfctr r12
904 li r10,IRQS_ALL_DISABLED
905 stb r10,PACAIRQSOFTMASK(r13)
906 #ifdef CONFIG_RELOCATABLE
907 b system_call_vectored_tramp
908 #else
909 b system_call_vectored_common
910 #endif
911 nop
912
913 /* SCV 1 - 127 */
914 .rept 127
915 mr r9,r13
916 GET_PACA(r13)
917 mflr r11
918 mfctr r12
919 li r10,IRQS_ALL_DISABLED
920 stb r10,PACAIRQSOFTMASK(r13)
921 li r0,-1 /* cause failure */
922 #ifdef CONFIG_RELOCATABLE
923 b system_call_vectored_sigill_tr
924 #else
925 b system_call_vectored_sigill
926 #endif
927 .endr
928 EXC_VIRT_END(system_call_vectored, 0x3000, 0x1
929
930 // Treat scv vectors as soft-masked, see comme
931 // Use absolute values rather than labels here
932 // because this code runs unrelocated.
933 SOFT_MASK_TABLE(0xc000000000003000, 0xc0000000
934
935 #ifdef CONFIG_RELOCATABLE
936 TRAMP_VIRT_BEGIN(system_call_vectored_tramp)
937 __LOAD_HANDLER(r10, system_call_vector
938 mtctr r10
939 bctr
940
941 TRAMP_VIRT_BEGIN(system_call_vectored_sigill_t
942 __LOAD_HANDLER(r10, system_call_vector
943 mtctr r10
944 bctr
945 #endif
946
947
948 /* No virt vectors corresponding with 0x0..0x1
949 EXC_VIRT_NONE(0x4000, 0x100)
950
951
952 /**
953 * Interrupt 0x100 - System Reset Interrupt (S
954 * This is a non-maskable, asynchronous interr
955 * It is caused by:
956 * - Wake from power-saving state, on powernv.
957 * - An NMI from another CPU, triggered by fir
958 * - As crash/debug signal injected from BMC,
959 *
960 * Handling:
961 * Power-save wakeup is the only performance c
962 * determined quickly as possible first. In th
963 * can be discarded and SPRs like CFAR don't n
964 *
965 * If not a powersave wakeup, then it's run as
966 * it uses its own stack and PACA save area to
967 * environment for debugging.
968 *
969 * This interrupt is not maskable, so triggeri
970 * or SCRATCH0 is in use, etc. may cause a cra
971 * correct to switch to virtual mode to run th
972 * because it might be interrupted when the MM
973 * is clear).
974 *
975 * FWNMI:
976 * PAPR specifies a "fwnmi" facility which sen
977 * entry point with a different register set u
978 * send the sreset to 0x100 in the guest if it
979 *
980 * KVM:
981 * Unlike most SRR interrupts, this may be tak
982 * in a guest, so a KVM test is required. KVM
983 * mode and then raise the sreset.
984 */
985 INT_DEFINE_BEGIN(system_reset)
986 IVEC=0x100
987 IAREA=PACA_EXNMI
988 IVIRT=0 /* no virt entry point */
989 ISTACK=0
990 IKVM_REAL=1
991 INT_DEFINE_END(system_reset)
992
993 EXC_REAL_BEGIN(system_reset, 0x100, 0x100)
994 #ifdef CONFIG_PPC_P7_NAP
995 /*
996 * If running native on arch 2.06 or l
997 * from nap/sleep/winkle, and branch t
998 * bits 46:47. A non-0 value indicates
999 * saving state. The idle wakeup handl
1000 * but we branch to the 0xc000... add
1001 * with mtmsrd later, after SPRs are
1002 *
1003 * Careful to minimise cost for the f
1004 * also avoiding clobbering CFAR for
1005 *
1006 * For the idle wake case volatile re
1007 * is why we use those initially. If
1008 * wake, carefully put everything bac
1009 * common exception macros to handle
1010 */
1011 BEGIN_FTR_SECTION
1012 SET_SCRATCH0(r13)
1013 GET_PACA(r13)
1014 std r3,PACA_EXNMI+0*8(r13)
1015 std r4,PACA_EXNMI+1*8(r13)
1016 std r5,PACA_EXNMI+2*8(r13)
1017 mfspr r3,SPRN_SRR1
1018 mfocrf r4,0x80
1019 rlwinm. r5,r3,47-31,30,31
1020 bne+ system_reset_idle_wake
1021 /* Not powersave wakeup. Restore regs
1022 mtocrf 0x80,r4
1023 ld r3,PACA_EXNMI+0*8(r13)
1024 ld r4,PACA_EXNMI+1*8(r13)
1025 ld r5,PACA_EXNMI+2*8(r13)
1026 GET_SCRATCH0(r13)
1027 END_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FT
1028 #endif
1029
1030 GEN_INT_ENTRY system_reset, virt=0
1031 /*
1032 * In theory, we should not enable re
1033 * in SRR1, because the MMU may not b
1034 * SLB may have been cleared). In pra
1035 * small windows where that's the cas
1036 * be dangerous anyway.
1037 */
1038 EXC_REAL_END(system_reset, 0x100, 0x100)
1039 EXC_VIRT_NONE(0x4100, 0x100)
1040
1041 #ifdef CONFIG_PPC_P7_NAP
1042 TRAMP_REAL_BEGIN(system_reset_idle_wake)
1043 /* We are waking up from idle, so may
1044 cmpwi cr1,r5,2
1045 bltlr cr1 /* no state loss, ret
1046 __LOAD_FAR_HANDLER(r12, DOTSYM(idle_r
1047 mtctr r12
1048 bctr
1049 #endif
1050
1051 #ifdef CONFIG_PPC_PSERIES
1052 /*
1053 * Vectors for the FWNMI option. Share commo
1054 */
1055 TRAMP_REAL_BEGIN(system_reset_fwnmi)
1056 GEN_INT_ENTRY system_reset, virt=0
1057
1058 #endif /* CONFIG_PPC_PSERIES */
1059
1060 EXC_COMMON_BEGIN(system_reset_common)
1061 __GEN_COMMON_ENTRY system_reset
1062 /*
1063 * Increment paca->in_nmi. When the i
1064 * enable MSR_RI, then SLB or MCE wil
1065 * NMI will notice in_nmi and not rec
1066 * stack. in_nmi reentrancy is tested
1067 */
1068 lhz r10,PACA_IN_NMI(r13)
1069 addi r10,r10,1
1070 sth r10,PACA_IN_NMI(r13)
1071
1072 mr r10,r1
1073 ld r1,PACA_NMI_EMERG_SP(r13)
1074 subi r1,r1,INT_FRAME_SIZE
1075 __GEN_COMMON_BODY system_reset
1076
1077 addi r3,r1,STACK_INT_FRAME_REGS
1078 bl CFUNC(system_reset_exception)
1079
1080 /* Clear MSR_RI before setting SRR0 a
1081 li r9,0
1082 mtmsrd r9,1
1083
1084 /*
1085 * MSR_RI is clear, now we can decrem
1086 */
1087 lhz r10,PACA_IN_NMI(r13)
1088 subi r10,r10,1
1089 sth r10,PACA_IN_NMI(r13)
1090
1091 kuap_kernel_restore r9, r10
1092 EXCEPTION_RESTORE_REGS
1093 RFI_TO_USER_OR_KERNEL
1094
1095
1096 /**
1097 * Interrupt 0x200 - Machine Check Interrupt
1098 * This is a non-maskable interrupt always ta
1099 * synchronous or asynchronous, caused by har
1100 * taken in a power-saving state.
1101 *
1102 * Handling:
1103 * Similarly to system reset, this uses its o
1104 * the difference is re-entrancy is allowed o
1105 *
1106 * machine_check_early is run in real mode, a
1107 * machine check and tries to handle it (e.g.
1108 * error detected there), determines if it wa
1109 * event.
1110 *
1111 * This early code does not "reconcile" irq s
1112 * regular interrupts do, so irqs_disabled()
1113 * properly (irq disable/enable already doesn
1114 * not work in real mode).
1115 *
1116 * Then, depending on the execution context w
1117 * are 3 main actions:
1118 * - Executing in kernel mode. The event is q
1119 * it is handled when it is next safe to do
1120 * interrupts), which could be immediately
1121 * avoids nasty issues like switching to vi
1122 * bad state, or when executing OPAL code.
1123 * but it has different priorities). Check
1124 * save, and return via the wake up code if
1125 *
1126 * - Executing in user mode. machine_check_ex
1127 * interrupt handler, which processes the d
1128 *
1129 * - Executing in guest mode. The interrupt i
1130 * branches to KVM to deal with. KVM may qu
1131 * to report later.
1132 *
1133 * This interrupt is not maskable, so if it t
1134 * or SCRATCH0 is in use, it may cause a cras
1135 *
1136 * KVM:
1137 * See SRESET.
1138 */
1139 INT_DEFINE_BEGIN(machine_check_early)
1140 IVEC=0x200
1141 IAREA=PACA_EXMC
1142 IVIRT=0 /* no virt entry point */
1143 IREALMODE_COMMON=1
1144 ISTACK=0
1145 IDAR=1
1146 IDSISR=1
1147 IKUAP=0 /* We don't touch AMR here, w
1148 INT_DEFINE_END(machine_check_early)
1149
1150 INT_DEFINE_BEGIN(machine_check)
1151 IVEC=0x200
1152 IAREA=PACA_EXMC
1153 IVIRT=0 /* no virt entry point */
1154 IDAR=1
1155 IDSISR=1
1156 IKVM_REAL=1
1157 INT_DEFINE_END(machine_check)
1158
1159 EXC_REAL_BEGIN(machine_check, 0x200, 0x100)
1160 EARLY_BOOT_FIXUP
1161 GEN_INT_ENTRY machine_check_early, vi
1162 EXC_REAL_END(machine_check, 0x200, 0x100)
1163 EXC_VIRT_NONE(0x4200, 0x100)
1164
1165 #ifdef CONFIG_PPC_PSERIES
1166 TRAMP_REAL_BEGIN(machine_check_fwnmi)
1167 /* See comment at machine_check excep
1168 GEN_INT_ENTRY machine_check_early, vi
1169 #endif
1170
1171 #define MACHINE_CHECK_HANDLER_WINDUP
1172 /* Clear MSR_RI before setting SRR0 a
1173 li r9,0;
1174 mtmsrd r9,1; /* Clear MSR_
1175 /* Decrement paca->in_mce now RI is c
1176 lhz r12,PACA_IN_MCE(r13);
1177 subi r12,r12,1;
1178 sth r12,PACA_IN_MCE(r13);
1179 EXCEPTION_RESTORE_REGS
1180
1181 EXC_COMMON_BEGIN(machine_check_early_common)
1182 __GEN_REALMODE_COMMON_ENTRY machine_c
1183
1184 /*
1185 * Switch to mc_emergency stack and h
1186 * the nested MCE upto level 4 to avo
1187 * Save MCE registers srr1, srr0, dar
1188 *
1189 * We use paca->in_mce to check wheth
1190 * nested machine check. We increment
1191 * machine checks.
1192 *
1193 * If this is the first entry then se
1194 * paca->mc_emergency_sp, otherwise r
1195 * stack frame on mc_emergency stack.
1196 *
1197 * NOTE: We are here with MSR_ME=0 (o
1198 * checkstop if we get another machin
1199 * rfid with MSR_ME=1.
1200 *
1201 * This interrupt can wake directly f
1202 * the machine check is handled then
1203 * to restore state.
1204 */
1205 lhz r10,PACA_IN_MCE(r13)
1206 cmpwi r10,0 /* Ar
1207 cmpwi cr1,r10,MAX_MCE_DEPTH /* Ar
1208 addi r10,r10,1 /* in
1209 sth r10,PACA_IN_MCE(r13)
1210
1211 mr r10,r1 /* Sa
1212 bne 1f
1213 /* First machine check entry */
1214 ld r1,PACAMCEMERGSP(r13) /* Us
1215 1: /* Limit nested MCE to level 4 to avo
1216 bgt cr1,unrecoverable_mce /* Ch
1217 subi r1,r1,INT_FRAME_SIZE /* al
1218
1219 __GEN_COMMON_BODY machine_check_early
1220
1221 BEGIN_FTR_SECTION
1222 bl enable_machine_check
1223 END_FTR_SECTION_IFSET(CPU_FTR_HVMODE)
1224 addi r3,r1,STACK_INT_FRAME_REGS
1225 BEGIN_FTR_SECTION
1226 bl CFUNC(machine_check_early_boo
1227 END_FTR_SECTION(0, 1) // nop out after bo
1228 bl CFUNC(machine_check_early)
1229 std r3,RESULT(r1) /* Save resul
1230 ld r12,_MSR(r1)
1231
1232 #ifdef CONFIG_PPC_P7_NAP
1233 /*
1234 * Check if thread was in power savin
1235 * of the following is true:
1236 * a. thread wasn't in power saving m
1237 * b. thread was in power saving mode
1238 * supervisor state loss or hyperv
1239 *
1240 * Go back to nap/sleep/winkle mode a
1241 */
1242 BEGIN_FTR_SECTION
1243 rlwinm. r11,r12,47-31,30,31
1244 bne machine_check_idle_common
1245 END_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FT
1246 #endif
1247
1248 #ifdef CONFIG_KVM_BOOK3S_64_HANDLER
1249 /*
1250 * Check if we are coming from guest.
1251 * exception handler which will take
1252 * machine_check_kvm->kvm_interrupt b
1253 * to guest.
1254 */
1255 lbz r11,HSTATE_IN_GUEST(r13)
1256 cmpwi r11,0 /* Ch
1257 bne mce_deliver /* co
1258 #endif
1259
1260 /*
1261 * Check if we are coming from usersp
1262 * exception handler which will deliv
1263 */
1264 andi. r11,r12,MSR_PR /* Se
1265 bne mce_deliver /* co
1266
1267 /*
1268 * At this point we are coming from k
1269 * Queue up the MCE event and return
1270 * But before that, check if this is
1271 * If yes, then stay on emergency sta
1272 */
1273 andi. r11,r12,MSR_RI
1274 beq unrecoverable_mce
1275
1276 /*
1277 * Check if we have successfully hand
1278 * then stay on emergency stack and p
1279 */
1280 ld r3,RESULT(r1) /* Load resul
1281 cmpdi r3,0 /* see if we
1282 beq unrecoverable_mce /* if !hand
1283
1284 /*
1285 * Return from MC interrupt.
1286 * Queue up the MCE event so that we
1287 * returning from kernel or opal call
1288 */
1289 bl CFUNC(machine_check_queue_eve
1290 MACHINE_CHECK_HANDLER_WINDUP
1291 RFI_TO_KERNEL
1292
1293 mce_deliver:
1294 /*
1295 * This is a host user or guest MCE.
1296 * run the "late" handler. For host u
1297 * machine_check_exception handler in
1298 * interrupt handler. For guest, this
1299 * and branch to the KVM interrupt si
1300 */
1301 BEGIN_FTR_SECTION
1302 ld r10,ORIG_GPR3(r1)
1303 mtspr SPRN_CFAR,r10
1304 END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
1305 MACHINE_CHECK_HANDLER_WINDUP
1306 GEN_INT_ENTRY machine_check, virt=0
1307
1308 EXC_COMMON_BEGIN(machine_check_common)
1309 /*
1310 * Machine check is different because
1311 * save area: PACA_EXMC instead of PA
1312 */
1313 GEN_COMMON machine_check
1314 addi r3,r1,STACK_INT_FRAME_REGS
1315 bl CFUNC(machine_check_exception
1316 b interrupt_return_srr
1317
1318
1319 #ifdef CONFIG_PPC_P7_NAP
1320 /*
1321 * This is an idle wakeup. Low level machine
1322 * done. Queue the event then call the idle c
1323 */
1324 EXC_COMMON_BEGIN(machine_check_idle_common)
1325 bl CFUNC(machine_check_queue_eve
1326
1327 /*
1328 * GPR-loss wakeups are relatively st
1329 * idle sleep code has saved all non-
1330 * own stack, and r1 in PACAR1.
1331 *
1332 * For no-loss wakeups the r1 and lr
1333 * early machine check handler have t
1334 * the kernel TOC, so no need to rest
1335 *
1336 * Then decrement MCE nesting after f
1337 */
1338 ld r3,_MSR(r1)
1339 ld r4,_LINK(r1)
1340 ld r1,GPR1(r1)
1341
1342 lhz r11,PACA_IN_MCE(r13)
1343 subi r11,r11,1
1344 sth r11,PACA_IN_MCE(r13)
1345
1346 mtlr r4
1347 rlwinm r10,r3,47-31,30,31
1348 cmpwi cr1,r10,2
1349 bltlr cr1 /* no state loss, ret
1350 b idle_return_gpr_loss
1351 #endif
1352
1353 EXC_COMMON_BEGIN(unrecoverable_mce)
1354 /*
1355 * We are going down. But there are c
1356 * another MCE during panic path and
1357 * with no way out. Hence, turn ME bi
1358 * when another MCE is hit during pan
1359 * and hypervisor will get restarted
1360 */
1361 BEGIN_FTR_SECTION
1362 li r10,0 /* clear MSR_RI */
1363 mtmsrd r10,1
1364 bl CFUNC(disable_machine_check)
1365 END_FTR_SECTION_IFSET(CPU_FTR_HVMODE)
1366 ld r10,PACAKMSR(r13)
1367 li r3,MSR_ME
1368 andc r10,r10,r3
1369 mtmsrd r10
1370
1371 lhz r12,PACA_IN_MCE(r13)
1372 subi r12,r12,1
1373 sth r12,PACA_IN_MCE(r13)
1374
1375 /*
1376 * Invoke machine_check_exception to
1377 * This is the NMI version of the han
1378 * the early handler which is a true
1379 */
1380 addi r3,r1,STACK_INT_FRAME_REGS
1381 bl CFUNC(machine_check_exception
1382
1383 /*
1384 * We will not reach here. Even if we
1385 * Call unrecoverable_exception and d
1386 */
1387 addi r3,r1,STACK_INT_FRAME_REGS
1388 bl CFUNC(unrecoverable_exception
1389 b .
1390
1391
1392 /**
1393 * Interrupt 0x300 - Data Storage Interrupt (
1394 * This is a synchronous interrupt generated
1395 * e.g., a load orstore which does not have a
1396 * permissions. DAWR matches also fault here,
1397 * errors e.g., copy/paste, AMO, certain inva
1398 *
1399 * Handling:
1400 * - Hash MMU
1401 * Go to do_hash_fault, which attempts to f
1402 * Linux page table. Hash faults can hit in
1403 * arbitrary state (e.g., interrupts disabl
1404 * "non-bolted" regions, e.g., vmalloc spac
1405 * backed by Linux page table entries.
1406 *
1407 * If no entry is found the Linux page faul
1408 * do_hash_fault). Linux page faults can ha
1409 * copy operations of course.
1410 *
1411 * KVM: The KVM HDSI handler may perform a
1412 * MMU context, which may cause a DSI in th
1413 * KVM handler. MSR[IR] is not enabled, so
1414 * always be used regardless of AIL setting
1415 *
1416 * - Radix MMU
1417 * The hardware loads from the Linux page t
1418 * immediately to Linux page fault.
1419 *
1420 * Conditions like DAWR match are handled on
1421 */
1422 INT_DEFINE_BEGIN(data_access)
1423 IVEC=0x300
1424 IDAR=1
1425 IDSISR=1
1426 IKVM_REAL=1
1427 INT_DEFINE_END(data_access)
1428
1429 EXC_REAL_BEGIN(data_access, 0x300, 0x80)
1430 GEN_INT_ENTRY data_access, virt=0
1431 EXC_REAL_END(data_access, 0x300, 0x80)
1432 EXC_VIRT_BEGIN(data_access, 0x4300, 0x80)
1433 GEN_INT_ENTRY data_access, virt=1
1434 EXC_VIRT_END(data_access, 0x4300, 0x80)
1435 EXC_COMMON_BEGIN(data_access_common)
1436 GEN_COMMON data_access
1437 ld r4,_DSISR(r1)
1438 addi r3,r1,STACK_INT_FRAME_REGS
1439 andis. r0,r4,DSISR_DABRMATCH@h
1440 bne- 1f
1441 #ifdef CONFIG_PPC_64S_HASH_MMU
1442 BEGIN_MMU_FTR_SECTION
1443 bl CFUNC(do_hash_fault)
1444 MMU_FTR_SECTION_ELSE
1445 bl CFUNC(do_page_fault)
1446 ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RA
1447 #else
1448 bl CFUNC(do_page_fault)
1449 #endif
1450 b interrupt_return_srr
1451
1452 1: bl CFUNC(do_break)
1453 /*
1454 * do_break() may have changed the NV
1455 * If so, we need to restore them wit
1456 */
1457 HANDLER_RESTORE_NVGPRS()
1458 b interrupt_return_srr
1459
1460
1461 /**
1462 * Interrupt 0x380 - Data Segment Interrupt (
1463 * This is a synchronous interrupt in respons
1464 * entry for HPT, or an address outside RPT t
1465 *
1466 * Handling:
1467 * - HPT:
1468 * This refills the SLB, or reports an acce
1469 * fault. When coming from user-mode, the S
1470 * data, though it may itself take a DSLB.
1471 * recursive faults must be avoided so acce
1472 * image text/data, kernel stack, and any d
1473 * ppc64_bolted_size (first segment). The k
1474 * on user-handler data structures.
1475 *
1476 * KVM: Same as 0x300, DSLB must test for K
1477 */
1478 INT_DEFINE_BEGIN(data_access_slb)
1479 IVEC=0x380
1480 IDAR=1
1481 IKVM_REAL=1
1482 INT_DEFINE_END(data_access_slb)
1483
1484 EXC_REAL_BEGIN(data_access_slb, 0x380, 0x80)
1485 GEN_INT_ENTRY data_access_slb, virt=0
1486 EXC_REAL_END(data_access_slb, 0x380, 0x80)
1487 EXC_VIRT_BEGIN(data_access_slb, 0x4380, 0x80)
1488 GEN_INT_ENTRY data_access_slb, virt=1
1489 EXC_VIRT_END(data_access_slb, 0x4380, 0x80)
1490 EXC_COMMON_BEGIN(data_access_slb_common)
1491 GEN_COMMON data_access_slb
1492 #ifdef CONFIG_PPC_64S_HASH_MMU
1493 BEGIN_MMU_FTR_SECTION
1494 /* HPT case, do SLB fault */
1495 addi r3,r1,STACK_INT_FRAME_REGS
1496 bl CFUNC(do_slb_fault)
1497 cmpdi r3,0
1498 bne- 1f
1499 b fast_interrupt_return_srr
1500 1: /* Error case */
1501 MMU_FTR_SECTION_ELSE
1502 /* Radix case, access is outside page
1503 li r3,-EFAULT
1504 ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RA
1505 #else
1506 li r3,-EFAULT
1507 #endif
1508 std r3,RESULT(r1)
1509 addi r3,r1,STACK_INT_FRAME_REGS
1510 bl CFUNC(do_bad_segment_interrup
1511 b interrupt_return_srr
1512
1513
1514 /**
1515 * Interrupt 0x400 - Instruction Storage Inte
1516 * This is a synchronous interrupt in respons
1517 * instruction fetch.
1518 *
1519 * Handling:
1520 * Similar to DSI, though in response to fetc
1521 * in SRR0 (rather than DAR), and status in S
1522 */
1523 INT_DEFINE_BEGIN(instruction_access)
1524 IVEC=0x400
1525 IISIDE=1
1526 IDAR=1
1527 IDSISR=1
1528 #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
1529 IKVM_REAL=1
1530 #endif
1531 INT_DEFINE_END(instruction_access)
1532
1533 EXC_REAL_BEGIN(instruction_access, 0x400, 0x8
1534 GEN_INT_ENTRY instruction_access, vir
1535 EXC_REAL_END(instruction_access, 0x400, 0x80)
1536 EXC_VIRT_BEGIN(instruction_access, 0x4400, 0x
1537 GEN_INT_ENTRY instruction_access, vir
1538 EXC_VIRT_END(instruction_access, 0x4400, 0x80
1539 EXC_COMMON_BEGIN(instruction_access_common)
1540 GEN_COMMON instruction_access
1541 addi r3,r1,STACK_INT_FRAME_REGS
1542 #ifdef CONFIG_PPC_64S_HASH_MMU
1543 BEGIN_MMU_FTR_SECTION
1544 bl CFUNC(do_hash_fault)
1545 MMU_FTR_SECTION_ELSE
1546 bl CFUNC(do_page_fault)
1547 ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RA
1548 #else
1549 bl CFUNC(do_page_fault)
1550 #endif
1551 b interrupt_return_srr
1552
1553
1554 /**
1555 * Interrupt 0x480 - Instruction Segment Inte
1556 * This is a synchronous interrupt in respons
1557 * instruction fetch.
1558 *
1559 * Handling:
1560 * Similar to DSLB, though in response to fet
1561 * in SRR0 (rather than DAR).
1562 */
1563 INT_DEFINE_BEGIN(instruction_access_slb)
1564 IVEC=0x480
1565 IISIDE=1
1566 IDAR=1
1567 #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
1568 IKVM_REAL=1
1569 #endif
1570 INT_DEFINE_END(instruction_access_slb)
1571
1572 EXC_REAL_BEGIN(instruction_access_slb, 0x480,
1573 GEN_INT_ENTRY instruction_access_slb,
1574 EXC_REAL_END(instruction_access_slb, 0x480, 0
1575 EXC_VIRT_BEGIN(instruction_access_slb, 0x4480
1576 GEN_INT_ENTRY instruction_access_slb,
1577 EXC_VIRT_END(instruction_access_slb, 0x4480,
1578 EXC_COMMON_BEGIN(instruction_access_slb_commo
1579 GEN_COMMON instruction_access_slb
1580 #ifdef CONFIG_PPC_64S_HASH_MMU
1581 BEGIN_MMU_FTR_SECTION
1582 /* HPT case, do SLB fault */
1583 addi r3,r1,STACK_INT_FRAME_REGS
1584 bl CFUNC(do_slb_fault)
1585 cmpdi r3,0
1586 bne- 1f
1587 b fast_interrupt_return_srr
1588 1: /* Error case */
1589 MMU_FTR_SECTION_ELSE
1590 /* Radix case, access is outside page
1591 li r3,-EFAULT
1592 ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RA
1593 #else
1594 li r3,-EFAULT
1595 #endif
1596 std r3,RESULT(r1)
1597 addi r3,r1,STACK_INT_FRAME_REGS
1598 bl CFUNC(do_bad_segment_interrup
1599 b interrupt_return_srr
1600
1601
1602 /**
1603 * Interrupt 0x500 - External Interrupt.
1604 * This is an asynchronous maskable interrupt
1605 * exception" from the interrupt controller o
1606 * interrupt). It is maskable in hardware by
1607 * soft-maskable with IRQS_DISABLED mask (i.e
1608 *
1609 * When running in HV mode, Linux sets up the
1610 * interrupts are delivered with HSRR registe
1611 * reqiures IHSRR_IF_HVMODE.
1612 *
1613 * On bare metal POWER9 and later, Linux sets
1614 * external interrupts are delivered as Hyper
1615 * rather than External Interrupts.
1616 *
1617 * Handling:
1618 * This calls into Linux IRQ handler. NVGPRs
1619 * because registers at the time of the inter
1620 * asynchronous.
1621 *
1622 * If soft masked, the masked handler will no
1623 * replay, and clear MSR[EE] in the interrupt
1624 *
1625 * CFAR is not required because this is an as
1626 * general won't have much bearing on the sta
1627 * exception of crash/debug IPIs, but those a
1628 * IPIs. Unless this is an HV interrupt and K
1629 * it may be exiting the guest and need CFAR
1630 */
1631 INT_DEFINE_BEGIN(hardware_interrupt)
1632 IVEC=0x500
1633 IHSRR_IF_HVMODE=1
1634 IMASK=IRQS_DISABLED
1635 IKVM_REAL=1
1636 IKVM_VIRT=1
1637 ICFAR=0
1638 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
1639 ICFAR_IF_HVMODE=1
1640 #endif
1641 INT_DEFINE_END(hardware_interrupt)
1642
1643 EXC_REAL_BEGIN(hardware_interrupt, 0x500, 0x1
1644 GEN_INT_ENTRY hardware_interrupt, vir
1645 EXC_REAL_END(hardware_interrupt, 0x500, 0x100
1646 EXC_VIRT_BEGIN(hardware_interrupt, 0x4500, 0x
1647 GEN_INT_ENTRY hardware_interrupt, vir
1648 EXC_VIRT_END(hardware_interrupt, 0x4500, 0x10
1649 EXC_COMMON_BEGIN(hardware_interrupt_common)
1650 GEN_COMMON hardware_interrupt
1651 addi r3,r1,STACK_INT_FRAME_REGS
1652 bl CFUNC(do_IRQ)
1653 BEGIN_FTR_SECTION
1654 b interrupt_return_hsrr
1655 FTR_SECTION_ELSE
1656 b interrupt_return_srr
1657 ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVM
1658
1659
1660 /**
1661 * Interrupt 0x600 - Alignment Interrupt
1662 * This is a synchronous interrupt in respons
1663 */
1664 INT_DEFINE_BEGIN(alignment)
1665 IVEC=0x600
1666 IDAR=1
1667 IDSISR=1
1668 #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
1669 IKVM_REAL=1
1670 #endif
1671 INT_DEFINE_END(alignment)
1672
1673 EXC_REAL_BEGIN(alignment, 0x600, 0x100)
1674 GEN_INT_ENTRY alignment, virt=0
1675 EXC_REAL_END(alignment, 0x600, 0x100)
1676 EXC_VIRT_BEGIN(alignment, 0x4600, 0x100)
1677 GEN_INT_ENTRY alignment, virt=1
1678 EXC_VIRT_END(alignment, 0x4600, 0x100)
1679 EXC_COMMON_BEGIN(alignment_common)
1680 GEN_COMMON alignment
1681 addi r3,r1,STACK_INT_FRAME_REGS
1682 bl CFUNC(alignment_exception)
1683 HANDLER_RESTORE_NVGPRS() /* instructi
1684 b interrupt_return_srr
1685
1686
1687 /**
1688 * Interrupt 0x700 - Program Interrupt (progr
1689 * This is a synchronous interrupt in respons
1690 * traps, privilege errors, TM errors, floati
1691 *
1692 * Handling:
1693 * This interrupt may use the "emergency stac
1694 * from kernel context, which complicates han
1695 */
1696 INT_DEFINE_BEGIN(program_check)
1697 IVEC=0x700
1698 #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
1699 IKVM_REAL=1
1700 #endif
1701 INT_DEFINE_END(program_check)
1702
1703 EXC_REAL_BEGIN(program_check, 0x700, 0x100)
1704 EARLY_BOOT_FIXUP
1705 GEN_INT_ENTRY program_check, virt=0
1706 EXC_REAL_END(program_check, 0x700, 0x100)
1707 EXC_VIRT_BEGIN(program_check, 0x4700, 0x100)
1708 GEN_INT_ENTRY program_check, virt=1
1709 EXC_VIRT_END(program_check, 0x4700, 0x100)
1710 EXC_COMMON_BEGIN(program_check_common)
1711 __GEN_COMMON_ENTRY program_check
1712
1713 /*
1714 * It's possible to receive a TM Bad
1715 * userspace register values (in part
1716 * that we came from the kernel. Norm
1717 * stack logic, and we would report a
1718 * we switch to the emergency stack i
1719 * the kernel.
1720 */
1721
1722 andi. r10,r12,MSR_PR
1723 bne .Lnormal_stack /* If
1724
1725 andis. r10,r12,(SRR1_PROGTM)@h
1726 bne .Lemergency_stack /* If
1727
1728 cmpdi r1,-INT_FRAME_SIZE /* ch
1729 blt .Lnormal_stack /* no
1730
1731 /* Use the emergency stack
1732 .Lemergency_stack:
1733 andi. r10,r12,MSR_PR /* Se
1734 /* 3
1735 mr r10,r1 /* Sa
1736 ld r1,PACAEMERGSP(r13) /* Us
1737 subi r1,r1,INT_FRAME_SIZE /* al
1738 __ISTACK(program_check)=0
1739 __GEN_COMMON_BODY program_check
1740 b .Ldo_program_check
1741
1742 .Lnormal_stack:
1743 __ISTACK(program_check)=1
1744 __GEN_COMMON_BODY program_check
1745
1746 .Ldo_program_check:
1747 addi r3,r1,STACK_INT_FRAME_REGS
1748 bl CFUNC(program_check_exception
1749 HANDLER_RESTORE_NVGPRS() /* instructi
1750 b interrupt_return_srr
1751
1752
1753 /*
1754 * Interrupt 0x800 - Floating-Point Unavailab
1755 * This is a synchronous interrupt in respons
1756 * with MSR[FP]=0.
1757 *
1758 * Handling:
1759 * This will load FP registers and enable the
1760 * otherwise report a bad kernel use of FP.
1761 */
1762 INT_DEFINE_BEGIN(fp_unavailable)
1763 IVEC=0x800
1764 #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
1765 IKVM_REAL=1
1766 #endif
1767 IMSR_R12=1
1768 INT_DEFINE_END(fp_unavailable)
1769
1770 EXC_REAL_BEGIN(fp_unavailable, 0x800, 0x100)
1771 GEN_INT_ENTRY fp_unavailable, virt=0
1772 EXC_REAL_END(fp_unavailable, 0x800, 0x100)
1773 EXC_VIRT_BEGIN(fp_unavailable, 0x4800, 0x100)
1774 GEN_INT_ENTRY fp_unavailable, virt=1
1775 EXC_VIRT_END(fp_unavailable, 0x4800, 0x100)
1776 EXC_COMMON_BEGIN(fp_unavailable_common)
1777 GEN_COMMON fp_unavailable
1778 bne 1f /* if
1779 addi r3,r1,STACK_INT_FRAME_REGS
1780 bl CFUNC(kernel_fp_unavailable_e
1781 0: trap
1782 EMIT_BUG_ENTRY 0b, __FILE__, __LINE__
1783 1:
1784 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1785 BEGIN_FTR_SECTION
1786 /* Test if 2 TM state bits are zero.
1787 * transaction), go do TM stuff
1788 */
1789 rldicl. r0, r12, (64-MSR_TS_LG), (64-
1790 bne- 2f
1791 END_FTR_SECTION_IFSET(CPU_FTR_TM)
1792 #endif
1793 bl CFUNC(load_up_fpu)
1794 b fast_interrupt_return_srr
1795 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1796 2: /* User process was in a transaction
1797 addi r3,r1,STACK_INT_FRAME_REGS
1798 bl CFUNC(fp_unavailable_tm)
1799 b interrupt_return_srr
1800 #endif
1801
1802
1803 /**
1804 * Interrupt 0x900 - Decrementer Interrupt.
1805 * This is an asynchronous interrupt in respo
1806 * (e.g., DEC has wrapped below zero). It is
1807 * MSR[EE], and soft-maskable with IRQS_DISAB
1808 * local_irq_disable()).
1809 *
1810 * Handling:
1811 * This calls into Linux timer handler. NVGPR
1812 *
1813 * If soft masked, the masked handler will no
1814 * replay, and bump the decrementer to a high
1815 * in the interrupted context.
1816 * If PPC_WATCHDOG is configured, the soft ma
1817 * things back up to run soft_nmi_interrupt a
1818 * on the emergency stack.
1819 *
1820 * CFAR is not required because this is async
1821 * A watchdog interrupt may like to have CFAR
1822 * branch is long gone by that point (e.g., i
1823 */
1824 INT_DEFINE_BEGIN(decrementer)
1825 IVEC=0x900
1826 IMASK=IRQS_DISABLED
1827 #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
1828 IKVM_REAL=1
1829 #endif
1830 ICFAR=0
1831 INT_DEFINE_END(decrementer)
1832
1833 EXC_REAL_BEGIN(decrementer, 0x900, 0x80)
1834 GEN_INT_ENTRY decrementer, virt=0
1835 EXC_REAL_END(decrementer, 0x900, 0x80)
1836 EXC_VIRT_BEGIN(decrementer, 0x4900, 0x80)
1837 GEN_INT_ENTRY decrementer, virt=1
1838 EXC_VIRT_END(decrementer, 0x4900, 0x80)
1839 EXC_COMMON_BEGIN(decrementer_common)
1840 GEN_COMMON decrementer
1841 addi r3,r1,STACK_INT_FRAME_REGS
1842 bl CFUNC(timer_interrupt)
1843 b interrupt_return_srr
1844
1845
1846 /**
1847 * Interrupt 0x980 - Hypervisor Decrementer I
1848 * This is an asynchronous interrupt, similar
1849 * register.
1850 *
1851 * Handling:
1852 * Linux does not use this outside KVM where
1853 * while the guest is given control of DEC. I
1854 * the KVM test and routed there.
1855 */
1856 INT_DEFINE_BEGIN(hdecrementer)
1857 IVEC=0x980
1858 IHSRR=1
1859 ISTACK=0
1860 IKVM_REAL=1
1861 IKVM_VIRT=1
1862 INT_DEFINE_END(hdecrementer)
1863
1864 EXC_REAL_BEGIN(hdecrementer, 0x980, 0x80)
1865 GEN_INT_ENTRY hdecrementer, virt=0
1866 EXC_REAL_END(hdecrementer, 0x980, 0x80)
1867 EXC_VIRT_BEGIN(hdecrementer, 0x4980, 0x80)
1868 GEN_INT_ENTRY hdecrementer, virt=1
1869 EXC_VIRT_END(hdecrementer, 0x4980, 0x80)
1870 EXC_COMMON_BEGIN(hdecrementer_common)
1871 __GEN_COMMON_ENTRY hdecrementer
1872 /*
1873 * Hypervisor decrementer interrupts
1874 * shouldn't occur but are sometimes
1875 * guest. We don't need to do anythi
1876 * edge-triggered.
1877 *
1878 * Be careful to avoid touching the k
1879 */
1880 li r10,0
1881 stb r10,PACAHSRR_VALID(r13)
1882 ld r10,PACA_EXGEN+EX_CTR(r13)
1883 mtctr r10
1884 mtcrf 0x80,r9
1885 ld r9,PACA_EXGEN+EX_R9(r13)
1886 ld r10,PACA_EXGEN+EX_R10(r13)
1887 ld r11,PACA_EXGEN+EX_R11(r13)
1888 ld r12,PACA_EXGEN+EX_R12(r13)
1889 ld r13,PACA_EXGEN+EX_R13(r13)
1890 HRFI_TO_KERNEL
1891
1892
1893 /**
1894 * Interrupt 0xa00 - Directed Privileged Door
1895 * This is an asynchronous interrupt in respo
1896 * It is maskable in hardware by clearing MSR
1897 * IRQS_DISABLED mask (i.e., local_irq_disabl
1898 *
1899 * Handling:
1900 * Guests may use this for IPIs between threa
1901 * hypervisor supports it. NVGPRS are not sav
1902 *
1903 * If soft masked, the masked handler will no
1904 * replay, leaving MSR[EE] enabled in the int
1905 * doorbells are edge triggered.
1906 *
1907 * CFAR is not required, similarly to hardwar
1908 */
1909 INT_DEFINE_BEGIN(doorbell_super)
1910 IVEC=0xa00
1911 IMASK=IRQS_DISABLED
1912 #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
1913 IKVM_REAL=1
1914 #endif
1915 ICFAR=0
1916 INT_DEFINE_END(doorbell_super)
1917
1918 EXC_REAL_BEGIN(doorbell_super, 0xa00, 0x100)
1919 GEN_INT_ENTRY doorbell_super, virt=0
1920 EXC_REAL_END(doorbell_super, 0xa00, 0x100)
1921 EXC_VIRT_BEGIN(doorbell_super, 0x4a00, 0x100)
1922 GEN_INT_ENTRY doorbell_super, virt=1
1923 EXC_VIRT_END(doorbell_super, 0x4a00, 0x100)
1924 EXC_COMMON_BEGIN(doorbell_super_common)
1925 GEN_COMMON doorbell_super
1926 addi r3,r1,STACK_INT_FRAME_REGS
1927 #ifdef CONFIG_PPC_DOORBELL
1928 bl CFUNC(doorbell_exception)
1929 #else
1930 bl CFUNC(unknown_async_exception
1931 #endif
1932 b interrupt_return_srr
1933
1934
1935 EXC_REAL_NONE(0xb00, 0x100)
1936 EXC_VIRT_NONE(0x4b00, 0x100)
1937
1938 /**
1939 * Interrupt 0xc00 - System Call Interrupt (s
1940 * This is a synchronous interrupt invoked wi
1941 * system call is invoked with "sc 0" and doe
1942 * is directed to the currently running OS. T
1943 * "sc 1" and it sets HV=1, so it elevates to
1944 *
1945 * In HPT, sc 1 always goes to 0xc00 real mod
1946 * 0x4c00 virtual mode.
1947 *
1948 * Handling:
1949 * If the KVM test fires then it was due to a
1950 * routed to KVM. Otherwise this executes a n
1951 *
1952 * Call convention:
1953 *
1954 * syscall and hypercalls register convention
1955 * Documentation/arch/powerpc/syscall64-abi.r
1956 * Documentation/arch/powerpc/papr_hcalls.rst
1957 *
1958 * The intersection of volatile registers tha
1959 * inputs is: cr0, xer, ctr. We may use these
1960 * without saving, though xer is not a good i
1961 * interpret some bits so it may be costly to
1962 */
1963 INT_DEFINE_BEGIN(system_call)
1964 IVEC=0xc00
1965 IKVM_REAL=1
1966 IKVM_VIRT=1
1967 ICFAR=0
1968 INT_DEFINE_END(system_call)
1969
1970 .macro SYSTEM_CALL virt
1971 #ifdef CONFIG_KVM_BOOK3S_64_HANDLER
1972 /*
1973 * There is a little bit of juggling
1974 * working well. Save r13 in ctr to a
1975 * register.
1976 *
1977 * Userspace syscalls have already sa
1978 * it before setting HMT_MEDIUM.
1979 */
1980 mtctr r13
1981 GET_PACA(r13)
1982 std r10,PACA_EXGEN+EX_R10(r13)
1983 INTERRUPT_TO_KERNEL
1984 KVMTEST system_call kvm_hcall /* uses
1985 mfctr r9
1986 #else
1987 mr r9,r13
1988 GET_PACA(r13)
1989 INTERRUPT_TO_KERNEL
1990 #endif
1991
1992 /* We reach here with PACA in r13, r1
1993 mfspr r11,SPRN_SRR0
1994 mfspr r12,SPRN_SRR1
1995
1996 HMT_MEDIUM
1997
1998 .if ! \virt
1999 __LOAD_HANDLER(r10, system_call_commo
2000 mtctr r10
2001 bctr
2002 .else
2003 #ifdef CONFIG_RELOCATABLE
2004 __LOAD_HANDLER(r10, system_call_commo
2005 mtctr r10
2006 bctr
2007 #else
2008 b system_call_common
2009 #endif
2010 .endif
2011 .endm
2012
2013 EXC_REAL_BEGIN(system_call, 0xc00, 0x100)
2014 SYSTEM_CALL 0
2015 EXC_REAL_END(system_call, 0xc00, 0x100)
2016 EXC_VIRT_BEGIN(system_call, 0x4c00, 0x100)
2017 SYSTEM_CALL 1
2018 EXC_VIRT_END(system_call, 0x4c00, 0x100)
2019
2020 #ifdef CONFIG_KVM_BOOK3S_64_HANDLER
2021 TRAMP_REAL_BEGIN(kvm_hcall)
2022 std r9,PACA_EXGEN+EX_R9(r13)
2023 std r11,PACA_EXGEN+EX_R11(r13)
2024 std r12,PACA_EXGEN+EX_R12(r13)
2025 mfcr r9
2026 mfctr r10
2027 std r10,PACA_EXGEN+EX_R13(r13)
2028 li r10,0
2029 std r10,PACA_EXGEN+EX_CFAR(r13)
2030 std r10,PACA_EXGEN+EX_CTR(r13)
2031 /*
2032 * Save the PPR (on systems that sup
2033 * HMT_MEDIUM. That allows the KVM c
2034 * guest state (it is the guest's PP
2035 */
2036 BEGIN_FTR_SECTION
2037 mfspr r10,SPRN_PPR
2038 std r10,PACA_EXGEN+EX_PPR(r13)
2039 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
2040
2041 HMT_MEDIUM
2042
2043 #ifdef CONFIG_RELOCATABLE
2044 /*
2045 * Requires __LOAD_FAR_HANDLER beause
2046 * outside the head section.
2047 */
2048 __LOAD_FAR_HANDLER(r10, kvmppc_hcall,
2049 mtctr r10
2050 bctr
2051 #else
2052 b kvmppc_hcall
2053 #endif
2054 #endif
2055
2056 /**
2057 * Interrupt 0xd00 - Trace Interrupt.
2058 * This is a synchronous interrupt in respons
2059 * breakpoint faults.
2060 */
2061 INT_DEFINE_BEGIN(single_step)
2062 IVEC=0xd00
2063 #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
2064 IKVM_REAL=1
2065 #endif
2066 INT_DEFINE_END(single_step)
2067
2068 EXC_REAL_BEGIN(single_step, 0xd00, 0x100)
2069 GEN_INT_ENTRY single_step, virt=0
2070 EXC_REAL_END(single_step, 0xd00, 0x100)
2071 EXC_VIRT_BEGIN(single_step, 0x4d00, 0x100)
2072 GEN_INT_ENTRY single_step, virt=1
2073 EXC_VIRT_END(single_step, 0x4d00, 0x100)
2074 EXC_COMMON_BEGIN(single_step_common)
2075 GEN_COMMON single_step
2076 addi r3,r1,STACK_INT_FRAME_REGS
2077 bl CFUNC(single_step_exception)
2078 b interrupt_return_srr
2079
2080
2081 /**
2082 * Interrupt 0xe00 - Hypervisor Data Storage
2083 * This is a synchronous interrupt in respons
2084 * guest data access.
2085 *
2086 * Handling:
2087 * This should always get routed to KVM. In r
2088 * by a guest nested radix access that can't
2089 * partition scope page table. In hash mode,
2090 * running with translation disabled (virtual
2091 * KVM will update the page table structures
2092 */
2093 INT_DEFINE_BEGIN(h_data_storage)
2094 IVEC=0xe00
2095 IHSRR=1
2096 IDAR=1
2097 IDSISR=1
2098 IKVM_REAL=1
2099 IKVM_VIRT=1
2100 INT_DEFINE_END(h_data_storage)
2101
2102 EXC_REAL_BEGIN(h_data_storage, 0xe00, 0x20)
2103 GEN_INT_ENTRY h_data_storage, virt=0,
2104 EXC_REAL_END(h_data_storage, 0xe00, 0x20)
2105 EXC_VIRT_BEGIN(h_data_storage, 0x4e00, 0x20)
2106 GEN_INT_ENTRY h_data_storage, virt=1,
2107 EXC_VIRT_END(h_data_storage, 0x4e00, 0x20)
2108 EXC_COMMON_BEGIN(h_data_storage_common)
2109 GEN_COMMON h_data_storage
2110 addi r3,r1,STACK_INT_FRAME_REGS
2111 BEGIN_MMU_FTR_SECTION
2112 bl CFUNC(do_bad_page_fault_segv)
2113 MMU_FTR_SECTION_ELSE
2114 bl CFUNC(unknown_exception)
2115 ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_TYPE_RA
2116 b interrupt_return_hsrr
2117
2118
2119 /**
2120 * Interrupt 0xe20 - Hypervisor Instruction S
2121 * This is a synchronous interrupt in respons
2122 * guest instruction fetch, similar to HDSI.
2123 */
2124 INT_DEFINE_BEGIN(h_instr_storage)
2125 IVEC=0xe20
2126 IHSRR=1
2127 IKVM_REAL=1
2128 IKVM_VIRT=1
2129 INT_DEFINE_END(h_instr_storage)
2130
2131 EXC_REAL_BEGIN(h_instr_storage, 0xe20, 0x20)
2132 GEN_INT_ENTRY h_instr_storage, virt=0
2133 EXC_REAL_END(h_instr_storage, 0xe20, 0x20)
2134 EXC_VIRT_BEGIN(h_instr_storage, 0x4e20, 0x20)
2135 GEN_INT_ENTRY h_instr_storage, virt=1
2136 EXC_VIRT_END(h_instr_storage, 0x4e20, 0x20)
2137 EXC_COMMON_BEGIN(h_instr_storage_common)
2138 GEN_COMMON h_instr_storage
2139 addi r3,r1,STACK_INT_FRAME_REGS
2140 bl CFUNC(unknown_exception)
2141 b interrupt_return_hsrr
2142
2143
2144 /**
2145 * Interrupt 0xe40 - Hypervisor Emulation Ass
2146 */
2147 INT_DEFINE_BEGIN(emulation_assist)
2148 IVEC=0xe40
2149 IHSRR=1
2150 IKVM_REAL=1
2151 IKVM_VIRT=1
2152 INT_DEFINE_END(emulation_assist)
2153
2154 EXC_REAL_BEGIN(emulation_assist, 0xe40, 0x20)
2155 GEN_INT_ENTRY emulation_assist, virt=
2156 EXC_REAL_END(emulation_assist, 0xe40, 0x20)
2157 EXC_VIRT_BEGIN(emulation_assist, 0x4e40, 0x20
2158 GEN_INT_ENTRY emulation_assist, virt=
2159 EXC_VIRT_END(emulation_assist, 0x4e40, 0x20)
2160 EXC_COMMON_BEGIN(emulation_assist_common)
2161 GEN_COMMON emulation_assist
2162 addi r3,r1,STACK_INT_FRAME_REGS
2163 bl CFUNC(emulation_assist_interr
2164 HANDLER_RESTORE_NVGPRS() /* instructi
2165 b interrupt_return_hsrr
2166
2167
2168 /**
2169 * Interrupt 0xe60 - Hypervisor Maintenance I
2170 * This is an asynchronous interrupt caused b
2171 * Exception. It is always taken in real mode
2172 * unlike SRESET and MCE.
2173 *
2174 * It is maskable in hardware by clearing MSR
2175 * with IRQS_DISABLED mask (i.e., local_irq_d
2176 *
2177 * Handling:
2178 * This is a special case, this is handled si
2179 * initial real mode handler that is not soft
2180 * problem. Then a regular handler which is s
2181 * problem.
2182 *
2183 * The emergency stack is used for the early
2184 *
2185 * XXX: unclear why MCE and HMI schemes could
2186 * either use soft-masking for the MCE, or us
2187 *
2188 * KVM:
2189 * Unlike MCE, this calls into KVM without ca
2190 * first.
2191 */
2192 INT_DEFINE_BEGIN(hmi_exception_early)
2193 IVEC=0xe60
2194 IHSRR=1
2195 IREALMODE_COMMON=1
2196 ISTACK=0
2197 IKUAP=0 /* We don't touch AMR here, w
2198 IKVM_REAL=1
2199 INT_DEFINE_END(hmi_exception_early)
2200
2201 INT_DEFINE_BEGIN(hmi_exception)
2202 IVEC=0xe60
2203 IHSRR=1
2204 IMASK=IRQS_DISABLED
2205 IKVM_REAL=1
2206 INT_DEFINE_END(hmi_exception)
2207
2208 EXC_REAL_BEGIN(hmi_exception, 0xe60, 0x20)
2209 GEN_INT_ENTRY hmi_exception_early, vi
2210 EXC_REAL_END(hmi_exception, 0xe60, 0x20)
2211 EXC_VIRT_NONE(0x4e60, 0x20)
2212
2213 EXC_COMMON_BEGIN(hmi_exception_early_common)
2214 __GEN_REALMODE_COMMON_ENTRY hmi_excep
2215
2216 mr r10,r1 /* Sa
2217 ld r1,PACAEMERGSP(r13) /* Us
2218 subi r1,r1,INT_FRAME_SIZE /* al
2219
2220 __GEN_COMMON_BODY hmi_exception_early
2221
2222 addi r3,r1,STACK_INT_FRAME_REGS
2223 bl CFUNC(hmi_exception_realmode)
2224 cmpdi cr0,r3,0
2225 bne 1f
2226
2227 EXCEPTION_RESTORE_REGS hsrr=1
2228 HRFI_TO_USER_OR_KERNEL
2229
2230 1:
2231 /*
2232 * Go to virtual mode and pull the HM
2233 * firmware.
2234 */
2235 EXCEPTION_RESTORE_REGS hsrr=1
2236 GEN_INT_ENTRY hmi_exception, virt=0
2237
2238 EXC_COMMON_BEGIN(hmi_exception_common)
2239 GEN_COMMON hmi_exception
2240 addi r3,r1,STACK_INT_FRAME_REGS
2241 bl CFUNC(handle_hmi_exception)
2242 b interrupt_return_hsrr
2243
2244
2245 /**
2246 * Interrupt 0xe80 - Directed Hypervisor Door
2247 * This is an asynchronous interrupt in respo
2248 * Similar to the 0xa00 doorbell but for host
2249 *
2250 * CFAR is not required (similar to doorbell_
2251 * is enabled, in which case it may be a gues
2252 * include KVM support so it would be nice if
2253 * patched out if KVM was not currently runni
2254 */
2255 INT_DEFINE_BEGIN(h_doorbell)
2256 IVEC=0xe80
2257 IHSRR=1
2258 IMASK=IRQS_DISABLED
2259 IKVM_REAL=1
2260 IKVM_VIRT=1
2261 #ifndef CONFIG_KVM_BOOK3S_HV_POSSIBLE
2262 ICFAR=0
2263 #endif
2264 INT_DEFINE_END(h_doorbell)
2265
2266 EXC_REAL_BEGIN(h_doorbell, 0xe80, 0x20)
2267 GEN_INT_ENTRY h_doorbell, virt=0, ool
2268 EXC_REAL_END(h_doorbell, 0xe80, 0x20)
2269 EXC_VIRT_BEGIN(h_doorbell, 0x4e80, 0x20)
2270 GEN_INT_ENTRY h_doorbell, virt=1, ool
2271 EXC_VIRT_END(h_doorbell, 0x4e80, 0x20)
2272 EXC_COMMON_BEGIN(h_doorbell_common)
2273 GEN_COMMON h_doorbell
2274 addi r3,r1,STACK_INT_FRAME_REGS
2275 #ifdef CONFIG_PPC_DOORBELL
2276 bl CFUNC(doorbell_exception)
2277 #else
2278 bl CFUNC(unknown_async_exception
2279 #endif
2280 b interrupt_return_hsrr
2281
2282
2283 /**
2284 * Interrupt 0xea0 - Hypervisor Virtualizatio
2285 * This is an asynchronous interrupt in respo
2286 * Similar to 0x500 but for host only.
2287 *
2288 * Like h_doorbell, CFAR is only required for
2289 * a guest exit.
2290 */
2291 INT_DEFINE_BEGIN(h_virt_irq)
2292 IVEC=0xea0
2293 IHSRR=1
2294 IMASK=IRQS_DISABLED
2295 IKVM_REAL=1
2296 IKVM_VIRT=1
2297 #ifndef CONFIG_KVM_BOOK3S_HV_POSSIBLE
2298 ICFAR=0
2299 #endif
2300 INT_DEFINE_END(h_virt_irq)
2301
2302 EXC_REAL_BEGIN(h_virt_irq, 0xea0, 0x20)
2303 GEN_INT_ENTRY h_virt_irq, virt=0, ool
2304 EXC_REAL_END(h_virt_irq, 0xea0, 0x20)
2305 EXC_VIRT_BEGIN(h_virt_irq, 0x4ea0, 0x20)
2306 GEN_INT_ENTRY h_virt_irq, virt=1, ool
2307 EXC_VIRT_END(h_virt_irq, 0x4ea0, 0x20)
2308 EXC_COMMON_BEGIN(h_virt_irq_common)
2309 GEN_COMMON h_virt_irq
2310 addi r3,r1,STACK_INT_FRAME_REGS
2311 bl CFUNC(do_IRQ)
2312 b interrupt_return_hsrr
2313
2314
2315 EXC_REAL_NONE(0xec0, 0x20)
2316 EXC_VIRT_NONE(0x4ec0, 0x20)
2317 EXC_REAL_NONE(0xee0, 0x20)
2318 EXC_VIRT_NONE(0x4ee0, 0x20)
2319
2320
2321 /*
2322 * Interrupt 0xf00 - Performance Monitor Inte
2323 * This is an asynchronous interrupt in respo
2324 * It is maskable in hardware by clearing MSR
2325 * IRQS_PMI_DISABLED mask (NOTE: NOT local_ir
2326 *
2327 * Handling:
2328 * This calls into the perf subsystem.
2329 *
2330 * Like the watchdog soft-nmi, it appears an
2331 * runs under local_irq_disable. However it m
2332 * powerpc-specific code.
2333 *
2334 * If soft masked, the masked handler will no
2335 * replay, and clear MSR[EE] in the interrupt
2336 *
2337 * CFAR is not used by perf interrupts so not
2338 */
2339 INT_DEFINE_BEGIN(performance_monitor)
2340 IVEC=0xf00
2341 IMASK=IRQS_PMI_DISABLED
2342 #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
2343 IKVM_REAL=1
2344 #endif
2345 ICFAR=0
2346 INT_DEFINE_END(performance_monitor)
2347
2348 EXC_REAL_BEGIN(performance_monitor, 0xf00, 0x
2349 GEN_INT_ENTRY performance_monitor, vi
2350 EXC_REAL_END(performance_monitor, 0xf00, 0x20
2351 EXC_VIRT_BEGIN(performance_monitor, 0x4f00, 0
2352 GEN_INT_ENTRY performance_monitor, vi
2353 EXC_VIRT_END(performance_monitor, 0x4f00, 0x2
2354 EXC_COMMON_BEGIN(performance_monitor_common)
2355 GEN_COMMON performance_monitor
2356 addi r3,r1,STACK_INT_FRAME_REGS
2357 lbz r4,PACAIRQSOFTMASK(r13)
2358 cmpdi r4,IRQS_ENABLED
2359 bne 1f
2360 bl CFUNC(performance_monitor_exc
2361 b interrupt_return_srr
2362 1:
2363 bl CFUNC(performance_monitor_exc
2364 /* Clear MSR_RI before setting SRR0 a
2365 li r9,0
2366 mtmsrd r9,1
2367
2368 kuap_kernel_restore r9, r10
2369
2370 EXCEPTION_RESTORE_REGS hsrr=0
2371 RFI_TO_KERNEL
2372
2373 /**
2374 * Interrupt 0xf20 - Vector Unavailable Inter
2375 * This is a synchronous interrupt in respons
2376 * executing a vector (or altivec) instructio
2377 * Similar to FP unavailable.
2378 */
2379 INT_DEFINE_BEGIN(altivec_unavailable)
2380 IVEC=0xf20
2381 #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
2382 IKVM_REAL=1
2383 #endif
2384 IMSR_R12=1
2385 INT_DEFINE_END(altivec_unavailable)
2386
2387 EXC_REAL_BEGIN(altivec_unavailable, 0xf20, 0x
2388 GEN_INT_ENTRY altivec_unavailable, vi
2389 EXC_REAL_END(altivec_unavailable, 0xf20, 0x20
2390 EXC_VIRT_BEGIN(altivec_unavailable, 0x4f20, 0
2391 GEN_INT_ENTRY altivec_unavailable, vi
2392 EXC_VIRT_END(altivec_unavailable, 0x4f20, 0x2
2393 EXC_COMMON_BEGIN(altivec_unavailable_common)
2394 GEN_COMMON altivec_unavailable
2395 #ifdef CONFIG_ALTIVEC
2396 BEGIN_FTR_SECTION
2397 beq 1f
2398 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
2399 BEGIN_FTR_SECTION_NESTED(69)
2400 /* Test if 2 TM state bits are zero.
2401 * transaction), go do TM stuff
2402 */
2403 rldicl. r0, r12, (64-MSR_TS_LG), (64-
2404 bne- 2f
2405 END_FTR_SECTION_NESTED(CPU_FTR_TM, CPU_FTR_
2406 #endif
2407 bl CFUNC(load_up_altivec)
2408 b fast_interrupt_return_srr
2409 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
2410 2: /* User process was in a transaction
2411 addi r3,r1,STACK_INT_FRAME_REGS
2412 bl CFUNC(altivec_unavailable_tm)
2413 b interrupt_return_srr
2414 #endif
2415 1:
2416 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
2417 #endif
2418 addi r3,r1,STACK_INT_FRAME_REGS
2419 bl CFUNC(altivec_unavailable_exc
2420 b interrupt_return_srr
2421
2422
2423 /**
2424 * Interrupt 0xf40 - VSX Unavailable Interrup
2425 * This is a synchronous interrupt in respons
2426 * executing a VSX instruction with MSR[VSX]=
2427 * Similar to FP unavailable.
2428 */
2429 INT_DEFINE_BEGIN(vsx_unavailable)
2430 IVEC=0xf40
2431 #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
2432 IKVM_REAL=1
2433 #endif
2434 IMSR_R12=1
2435 INT_DEFINE_END(vsx_unavailable)
2436
2437 EXC_REAL_BEGIN(vsx_unavailable, 0xf40, 0x20)
2438 GEN_INT_ENTRY vsx_unavailable, virt=0
2439 EXC_REAL_END(vsx_unavailable, 0xf40, 0x20)
2440 EXC_VIRT_BEGIN(vsx_unavailable, 0x4f40, 0x20)
2441 GEN_INT_ENTRY vsx_unavailable, virt=1
2442 EXC_VIRT_END(vsx_unavailable, 0x4f40, 0x20)
2443 EXC_COMMON_BEGIN(vsx_unavailable_common)
2444 GEN_COMMON vsx_unavailable
2445 #ifdef CONFIG_VSX
2446 BEGIN_FTR_SECTION
2447 beq 1f
2448 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
2449 BEGIN_FTR_SECTION_NESTED(69)
2450 /* Test if 2 TM state bits are zero.
2451 * transaction), go do TM stuff
2452 */
2453 rldicl. r0, r12, (64-MSR_TS_LG), (64-
2454 bne- 2f
2455 END_FTR_SECTION_NESTED(CPU_FTR_TM, CPU_FTR_
2456 #endif
2457 b load_up_vsx
2458 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
2459 2: /* User process was in a transaction
2460 addi r3,r1,STACK_INT_FRAME_REGS
2461 bl CFUNC(vsx_unavailable_tm)
2462 b interrupt_return_srr
2463 #endif
2464 1:
2465 END_FTR_SECTION_IFSET(CPU_FTR_VSX)
2466 #endif
2467 addi r3,r1,STACK_INT_FRAME_REGS
2468 bl CFUNC(vsx_unavailable_excepti
2469 b interrupt_return_srr
2470
2471
2472 /**
2473 * Interrupt 0xf60 - Facility Unavailable Int
2474 * This is a synchronous interrupt in respons
2475 * executing an instruction without access to
2476 * resolved by the OS (e.g., FSCR, MSR).
2477 * Similar to FP unavailable.
2478 */
2479 INT_DEFINE_BEGIN(facility_unavailable)
2480 IVEC=0xf60
2481 #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
2482 IKVM_REAL=1
2483 #endif
2484 INT_DEFINE_END(facility_unavailable)
2485
2486 EXC_REAL_BEGIN(facility_unavailable, 0xf60, 0
2487 GEN_INT_ENTRY facility_unavailable, v
2488 EXC_REAL_END(facility_unavailable, 0xf60, 0x2
2489 EXC_VIRT_BEGIN(facility_unavailable, 0x4f60,
2490 GEN_INT_ENTRY facility_unavailable, v
2491 EXC_VIRT_END(facility_unavailable, 0x4f60, 0x
2492 EXC_COMMON_BEGIN(facility_unavailable_common)
2493 GEN_COMMON facility_unavailable
2494 addi r3,r1,STACK_INT_FRAME_REGS
2495 bl CFUNC(facility_unavailable_ex
2496 HANDLER_RESTORE_NVGPRS() /* instructi
2497 b interrupt_return_srr
2498
2499
2500 /**
2501 * Interrupt 0xf60 - Hypervisor Facility Unav
2502 * This is a synchronous interrupt in respons
2503 * executing an instruction without access to
2504 * be resolved in HV mode (e.g., HFSCR).
2505 * Similar to FP unavailable.
2506 */
2507 INT_DEFINE_BEGIN(h_facility_unavailable)
2508 IVEC=0xf80
2509 IHSRR=1
2510 IKVM_REAL=1
2511 IKVM_VIRT=1
2512 INT_DEFINE_END(h_facility_unavailable)
2513
2514 EXC_REAL_BEGIN(h_facility_unavailable, 0xf80,
2515 GEN_INT_ENTRY h_facility_unavailable,
2516 EXC_REAL_END(h_facility_unavailable, 0xf80, 0
2517 EXC_VIRT_BEGIN(h_facility_unavailable, 0x4f80
2518 GEN_INT_ENTRY h_facility_unavailable,
2519 EXC_VIRT_END(h_facility_unavailable, 0x4f80,
2520 EXC_COMMON_BEGIN(h_facility_unavailable_commo
2521 GEN_COMMON h_facility_unavailable
2522 addi r3,r1,STACK_INT_FRAME_REGS
2523 bl CFUNC(facility_unavailable_ex
2524 /* XXX Shouldn't be necessary in prac
2525 HANDLER_RESTORE_NVGPRS()
2526 b interrupt_return_hsrr
2527
2528
2529 EXC_REAL_NONE(0xfa0, 0x20)
2530 EXC_VIRT_NONE(0x4fa0, 0x20)
2531 EXC_REAL_NONE(0xfc0, 0x20)
2532 EXC_VIRT_NONE(0x4fc0, 0x20)
2533 EXC_REAL_NONE(0xfe0, 0x20)
2534 EXC_VIRT_NONE(0x4fe0, 0x20)
2535
2536 EXC_REAL_NONE(0x1000, 0x100)
2537 EXC_VIRT_NONE(0x5000, 0x100)
2538 EXC_REAL_NONE(0x1100, 0x100)
2539 EXC_VIRT_NONE(0x5100, 0x100)
2540
2541 #ifdef CONFIG_CBE_RAS
2542 INT_DEFINE_BEGIN(cbe_system_error)
2543 IVEC=0x1200
2544 IHSRR=1
2545 INT_DEFINE_END(cbe_system_error)
2546
2547 EXC_REAL_BEGIN(cbe_system_error, 0x1200, 0x10
2548 GEN_INT_ENTRY cbe_system_error, virt=
2549 EXC_REAL_END(cbe_system_error, 0x1200, 0x100)
2550 EXC_VIRT_NONE(0x5200, 0x100)
2551 EXC_COMMON_BEGIN(cbe_system_error_common)
2552 GEN_COMMON cbe_system_error
2553 addi r3,r1,STACK_INT_FRAME_REGS
2554 bl CFUNC(cbe_system_error_except
2555 b interrupt_return_hsrr
2556
2557 #else /* CONFIG_CBE_RAS */
2558 EXC_REAL_NONE(0x1200, 0x100)
2559 EXC_VIRT_NONE(0x5200, 0x100)
2560 #endif
2561
2562 /**
2563 * Interrupt 0x1300 - Instruction Address Bre
2564 * This has been removed from the ISA before
2565 * 64-bit BookS ISA supported, however the G5
2566 * interrupt with a non-architected feature a
2567 * processor interface.
2568 */
2569 INT_DEFINE_BEGIN(instruction_breakpoint)
2570 IVEC=0x1300
2571 #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
2572 IKVM_REAL=1
2573 #endif
2574 INT_DEFINE_END(instruction_breakpoint)
2575
2576 EXC_REAL_BEGIN(instruction_breakpoint, 0x1300
2577 GEN_INT_ENTRY instruction_breakpoint,
2578 EXC_REAL_END(instruction_breakpoint, 0x1300,
2579 EXC_VIRT_BEGIN(instruction_breakpoint, 0x5300
2580 GEN_INT_ENTRY instruction_breakpoint,
2581 EXC_VIRT_END(instruction_breakpoint, 0x5300,
2582 EXC_COMMON_BEGIN(instruction_breakpoint_commo
2583 GEN_COMMON instruction_breakpoint
2584 addi r3,r1,STACK_INT_FRAME_REGS
2585 bl CFUNC(instruction_breakpoint_
2586 b interrupt_return_srr
2587
2588
2589 EXC_REAL_NONE(0x1400, 0x100)
2590 EXC_VIRT_NONE(0x5400, 0x100)
2591
2592 /**
2593 * Interrupt 0x1500 - Soft Patch Interrupt
2594 *
2595 * Handling:
2596 * This is an implementation specific interru
2597 * range of exceptions.
2598 *
2599 * This interrupt handler is unique in that i
2600 * code even for guests (and even in guest co
2601 * for speed. POWER9 does not raise denorm ex
2602 * could be phased out in future to reduce sp
2603 */
2604 INT_DEFINE_BEGIN(denorm_exception)
2605 IVEC=0x1500
2606 IHSRR=1
2607 IBRANCH_TO_COMMON=0
2608 IKVM_REAL=1
2609 INT_DEFINE_END(denorm_exception)
2610
2611 EXC_REAL_BEGIN(denorm_exception, 0x1500, 0x10
2612 GEN_INT_ENTRY denorm_exception, virt=
2613 #ifdef CONFIG_PPC_DENORMALISATION
2614 andis. r10,r12,(HSRR1_DENORM)@h /* d
2615 bne+ denorm_assist
2616 #endif
2617 GEN_BRANCH_TO_COMMON denorm_exception
2618 EXC_REAL_END(denorm_exception, 0x1500, 0x100)
2619 #ifdef CONFIG_PPC_DENORMALISATION
2620 EXC_VIRT_BEGIN(denorm_exception, 0x5500, 0x10
2621 GEN_INT_ENTRY denorm_exception, virt=
2622 andis. r10,r12,(HSRR1_DENORM)@h /* d
2623 bne+ denorm_assist
2624 GEN_BRANCH_TO_COMMON denorm_exception
2625 EXC_VIRT_END(denorm_exception, 0x5500, 0x100)
2626 #else
2627 EXC_VIRT_NONE(0x5500, 0x100)
2628 #endif
2629
2630 #ifdef CONFIG_PPC_DENORMALISATION
2631 TRAMP_REAL_BEGIN(denorm_assist)
2632 BEGIN_FTR_SECTION
2633 /*
2634 * To denormalise we need to move a copy of t
2635 * For POWER6 do that here for all FP regs.
2636 */
2637 mfmsr r10
2638 ori r10,r10,(MSR_FP|MSR_FE0|MSR_F
2639 xori r10,r10,(MSR_FE0|MSR_FE1)
2640 mtmsrd r10
2641 sync
2642
2643 .Lreg=0
2644 .rept 32
2645 fmr .Lreg,.Lreg
2646 .Lreg=.Lreg+1
2647 .endr
2648
2649 FTR_SECTION_ELSE
2650 /*
2651 * To denormalise we need to move a copy of t
2652 * For POWER7 do that here for the first 32 V
2653 */
2654 mfmsr r10
2655 oris r10,r10,MSR_VSX@h
2656 mtmsrd r10
2657 sync
2658
2659 .Lreg=0
2660 .rept 32
2661 XVCPSGNDP(.Lreg,.Lreg,.Lreg)
2662 .Lreg=.Lreg+1
2663 .endr
2664
2665 ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_206)
2666
2667 BEGIN_FTR_SECTION
2668 b denorm_done
2669 END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
2670 /*
2671 * To denormalise we need to move a copy of t
2672 * For POWER8 we need to do that for all 64 V
2673 */
2674 .Lreg=32
2675 .rept 32
2676 XVCPSGNDP(.Lreg,.Lreg,.Lreg)
2677 .Lreg=.Lreg+1
2678 .endr
2679
2680 denorm_done:
2681 mfspr r11,SPRN_HSRR0
2682 subi r11,r11,4
2683 mtspr SPRN_HSRR0,r11
2684 mtcrf 0x80,r9
2685 ld r9,PACA_EXGEN+EX_R9(r13)
2686 BEGIN_FTR_SECTION
2687 ld r10,PACA_EXGEN+EX_PPR(r13)
2688 mtspr SPRN_PPR,r10
2689 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
2690 BEGIN_FTR_SECTION
2691 ld r10,PACA_EXGEN+EX_CFAR(r13)
2692 mtspr SPRN_CFAR,r10
2693 END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
2694 li r10,0
2695 stb r10,PACAHSRR_VALID(r13)
2696 ld r10,PACA_EXGEN+EX_R10(r13)
2697 ld r11,PACA_EXGEN+EX_R11(r13)
2698 ld r12,PACA_EXGEN+EX_R12(r13)
2699 ld r13,PACA_EXGEN+EX_R13(r13)
2700 HRFI_TO_UNKNOWN
2701 b .
2702 #endif
2703
2704 EXC_COMMON_BEGIN(denorm_exception_common)
2705 GEN_COMMON denorm_exception
2706 addi r3,r1,STACK_INT_FRAME_REGS
2707 bl CFUNC(unknown_exception)
2708 b interrupt_return_hsrr
2709
2710
2711 #ifdef CONFIG_CBE_RAS
2712 INT_DEFINE_BEGIN(cbe_maintenance)
2713 IVEC=0x1600
2714 IHSRR=1
2715 INT_DEFINE_END(cbe_maintenance)
2716
2717 EXC_REAL_BEGIN(cbe_maintenance, 0x1600, 0x100
2718 GEN_INT_ENTRY cbe_maintenance, virt=0
2719 EXC_REAL_END(cbe_maintenance, 0x1600, 0x100)
2720 EXC_VIRT_NONE(0x5600, 0x100)
2721 EXC_COMMON_BEGIN(cbe_maintenance_common)
2722 GEN_COMMON cbe_maintenance
2723 addi r3,r1,STACK_INT_FRAME_REGS
2724 bl CFUNC(cbe_maintenance_excepti
2725 b interrupt_return_hsrr
2726
2727 #else /* CONFIG_CBE_RAS */
2728 EXC_REAL_NONE(0x1600, 0x100)
2729 EXC_VIRT_NONE(0x5600, 0x100)
2730 #endif
2731
2732
2733 INT_DEFINE_BEGIN(altivec_assist)
2734 IVEC=0x1700
2735 #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
2736 IKVM_REAL=1
2737 #endif
2738 INT_DEFINE_END(altivec_assist)
2739
2740 EXC_REAL_BEGIN(altivec_assist, 0x1700, 0x100)
2741 GEN_INT_ENTRY altivec_assist, virt=0
2742 EXC_REAL_END(altivec_assist, 0x1700, 0x100)
2743 EXC_VIRT_BEGIN(altivec_assist, 0x5700, 0x100)
2744 GEN_INT_ENTRY altivec_assist, virt=1
2745 EXC_VIRT_END(altivec_assist, 0x5700, 0x100)
2746 EXC_COMMON_BEGIN(altivec_assist_common)
2747 GEN_COMMON altivec_assist
2748 addi r3,r1,STACK_INT_FRAME_REGS
2749 #ifdef CONFIG_ALTIVEC
2750 bl CFUNC(altivec_assist_exceptio
2751 HANDLER_RESTORE_NVGPRS() /* instructi
2752 #else
2753 bl CFUNC(unknown_exception)
2754 #endif
2755 b interrupt_return_srr
2756
2757
2758 #ifdef CONFIG_CBE_RAS
2759 INT_DEFINE_BEGIN(cbe_thermal)
2760 IVEC=0x1800
2761 IHSRR=1
2762 INT_DEFINE_END(cbe_thermal)
2763
2764 EXC_REAL_BEGIN(cbe_thermal, 0x1800, 0x100)
2765 GEN_INT_ENTRY cbe_thermal, virt=0
2766 EXC_REAL_END(cbe_thermal, 0x1800, 0x100)
2767 EXC_VIRT_NONE(0x5800, 0x100)
2768 EXC_COMMON_BEGIN(cbe_thermal_common)
2769 GEN_COMMON cbe_thermal
2770 addi r3,r1,STACK_INT_FRAME_REGS
2771 bl CFUNC(cbe_thermal_exception)
2772 b interrupt_return_hsrr
2773
2774 #else /* CONFIG_CBE_RAS */
2775 EXC_REAL_NONE(0x1800, 0x100)
2776 EXC_VIRT_NONE(0x5800, 0x100)
2777 #endif
2778
2779
2780 #ifdef CONFIG_PPC_WATCHDOG
2781
2782 INT_DEFINE_BEGIN(soft_nmi)
2783 IVEC=0x900
2784 ISTACK=0
2785 ICFAR=0
2786 INT_DEFINE_END(soft_nmi)
2787
2788 /*
2789 * Branch to soft_nmi_interrupt using the eme
2790 * stack is one that is usable by maskable in
2791 * remains off. It is used for recovery when
2792 * normal kernel stack, for example. The "sof
2793 * stack because we want irq disabled section
2794 * at all (other than PMU interrupts), so use
2795 * and run it entirely with interrupts hard d
2796 */
2797 EXC_COMMON_BEGIN(soft_nmi_common)
2798 mr r10,r1
2799 ld r1,PACAEMERGSP(r13)
2800 subi r1,r1,INT_FRAME_SIZE
2801 __GEN_COMMON_BODY soft_nmi
2802
2803 addi r3,r1,STACK_INT_FRAME_REGS
2804 bl CFUNC(soft_nmi_interrupt)
2805
2806 /* Clear MSR_RI before setting SRR0 a
2807 li r9,0
2808 mtmsrd r9,1
2809
2810 kuap_kernel_restore r9, r10
2811
2812 EXCEPTION_RESTORE_REGS hsrr=0
2813 RFI_TO_KERNEL
2814
2815 #endif /* CONFIG_PPC_WATCHDOG */
2816
2817 /*
2818 * An interrupt came in while soft-disabled.
2819 * - If it was a decrementer interrupt, we bu
2820 * - If it was a doorbell we return immediate
2821 * triggered and won't automatically refire
2822 * - If it was a HMI we return immediately si
2823 * and it won't refire.
2824 * - Else it is one of PACA_IRQ_MUST_HARD_MAS
2825 * This is called with r10 containing the val
2826 */
2827 .macro MASKED_INTERRUPT hsrr=0
2828 .if \hsrr
2829 masked_Hinterrupt:
2830 .else
2831 masked_interrupt:
2832 .endif
2833 stw r9,PACA_EXGEN+EX_CCR(r13)
2834 #ifdef CONFIG_PPC_IRQ_SOFT_MASK_DEBUG
2835 /*
2836 * Ensure there was no previous MUST_
2837 * HARD_DIS setting. If this does fir
2838 * masked and MSR[EE] will be cleared
2839 * panic, but somebody probably enabl
2840 * PACA_IRQ_HARD_DIS, mtmsr(mfmsr() |
2841 * cause.
2842 */
2843 lbz r9,PACAIRQHAPPENED(r13)
2844 andi. r9,r9,(PACA_IRQ_MUST_HARD_MAS
2845 0: tdnei r9,0
2846 EMIT_WARN_ENTRY 0b,__FILE__,__LINE__,
2847 #endif
2848 lbz r9,PACAIRQHAPPENED(r13)
2849 or r9,r9,r10
2850 stb r9,PACAIRQHAPPENED(r13)
2851
2852 .if ! \hsrr
2853 cmpwi r10,PACA_IRQ_DEC
2854 bne 1f
2855 LOAD_REG_IMMEDIATE(r9, 0x7fffffff)
2856 mtspr SPRN_DEC,r9
2857 #ifdef CONFIG_PPC_WATCHDOG
2858 lwz r9,PACA_EXGEN+EX_CCR(r13)
2859 b soft_nmi_common
2860 #else
2861 b 2f
2862 #endif
2863 .endif
2864
2865 1: andi. r10,r10,PACA_IRQ_MUST_HARD_MA
2866 beq 2f
2867 xori r12,r12,MSR_EE /* clear MSR_
2868 .if \hsrr
2869 mtspr SPRN_HSRR1,r12
2870 .else
2871 mtspr SPRN_SRR1,r12
2872 .endif
2873 ori r9,r9,PACA_IRQ_HARD_DIS
2874 stb r9,PACAIRQHAPPENED(r13)
2875 2: /* done */
2876 li r9,0
2877 .if \hsrr
2878 stb r9,PACAHSRR_VALID(r13)
2879 .else
2880 stb r9,PACASRR_VALID(r13)
2881 .endif
2882
2883 SEARCH_RESTART_TABLE
2884 cmpdi r12,0
2885 beq 3f
2886 .if \hsrr
2887 mtspr SPRN_HSRR0,r12
2888 .else
2889 mtspr SPRN_SRR0,r12
2890 .endif
2891 3:
2892
2893 ld r9,PACA_EXGEN+EX_CTR(r13)
2894 mtctr r9
2895 lwz r9,PACA_EXGEN+EX_CCR(r13)
2896 mtcrf 0x80,r9
2897 std r1,PACAR1(r13)
2898 ld r9,PACA_EXGEN+EX_R9(r13)
2899 ld r10,PACA_EXGEN+EX_R10(r13)
2900 ld r11,PACA_EXGEN+EX_R11(r13)
2901 ld r12,PACA_EXGEN+EX_R12(r13)
2902 ld r13,PACA_EXGEN+EX_R13(r13)
2903 /* May return to masked low address w
2904 .if \hsrr
2905 HRFI_TO_KERNEL
2906 .else
2907 RFI_TO_KERNEL
2908 .endif
2909 b .
2910 .endm
2911
2912 TRAMP_REAL_BEGIN(stf_barrier_fallback)
2913 std r9,PACA_EXRFI+EX_R9(r13)
2914 std r10,PACA_EXRFI+EX_R10(r13)
2915 sync
2916 ld r9,PACA_EXRFI+EX_R9(r13)
2917 ld r10,PACA_EXRFI+EX_R10(r13)
2918 ori 31,31,0
2919 .rept 14
2920 b 1f
2921 1:
2922 .endr
2923 blr
2924
2925 /* Clobbers r10, r11, ctr */
2926 .macro L1D_DISPLACEMENT_FLUSH
2927 ld r10,PACA_RFI_FLUSH_FALLBACK_A
2928 ld r11,PACA_L1D_FLUSH_SIZE(r13)
2929 srdi r11,r11,(7 + 3) /* 128 byte l
2930 mtctr r11
2931 DCBT_BOOK3S_STOP_ALL_STREAM_IDS(r11)
2932
2933 /* order ld/st prior to dcbt stop all
2934 sync
2935
2936 /*
2937 * The load addresses are at staggere
2938 * which suits some pipelines better
2939 * hurt).
2940 */
2941 1:
2942 ld r11,(0x80 + 8)*0(r10)
2943 ld r11,(0x80 + 8)*1(r10)
2944 ld r11,(0x80 + 8)*2(r10)
2945 ld r11,(0x80 + 8)*3(r10)
2946 ld r11,(0x80 + 8)*4(r10)
2947 ld r11,(0x80 + 8)*5(r10)
2948 ld r11,(0x80 + 8)*6(r10)
2949 ld r11,(0x80 + 8)*7(r10)
2950 addi r10,r10,0x80*8
2951 bdnz 1b
2952 .endm
2953
2954 TRAMP_REAL_BEGIN(entry_flush_fallback)
2955 std r9,PACA_EXRFI+EX_R9(r13)
2956 std r10,PACA_EXRFI+EX_R10(r13)
2957 std r11,PACA_EXRFI+EX_R11(r13)
2958 mfctr r9
2959 L1D_DISPLACEMENT_FLUSH
2960 mtctr r9
2961 ld r9,PACA_EXRFI+EX_R9(r13)
2962 ld r10,PACA_EXRFI+EX_R10(r13)
2963 ld r11,PACA_EXRFI+EX_R11(r13)
2964 blr
2965
2966 /*
2967 * The SCV entry flush happens with interrupt
2968 * to prevent EXRFI being clobbered by NMIs (
2969 * (containing LR) does not need to be preser
2970 * puts 0 in the pt_regs, CTR can be clobbere
2971 */
2972 TRAMP_REAL_BEGIN(scv_entry_flush_fallback)
2973 li r10,0
2974 mtmsrd r10,1
2975 lbz r10,PACAIRQHAPPENED(r13)
2976 ori r10,r10,PACA_IRQ_HARD_DIS
2977 stb r10,PACAIRQHAPPENED(r13)
2978 std r11,PACA_EXRFI+EX_R11(r13)
2979 L1D_DISPLACEMENT_FLUSH
2980 ld r11,PACA_EXRFI+EX_R11(r13)
2981 li r10,MSR_RI
2982 mtmsrd r10,1
2983 blr
2984
2985 TRAMP_REAL_BEGIN(rfi_flush_fallback)
2986 SET_SCRATCH0(r13);
2987 GET_PACA(r13);
2988 std r1,PACA_EXRFI+EX_R12(r13)
2989 ld r1,PACAKSAVE(r13)
2990 std r9,PACA_EXRFI+EX_R9(r13)
2991 std r10,PACA_EXRFI+EX_R10(r13)
2992 std r11,PACA_EXRFI+EX_R11(r13)
2993 mfctr r9
2994 L1D_DISPLACEMENT_FLUSH
2995 mtctr r9
2996 ld r9,PACA_EXRFI+EX_R9(r13)
2997 ld r10,PACA_EXRFI+EX_R10(r13)
2998 ld r11,PACA_EXRFI+EX_R11(r13)
2999 ld r1,PACA_EXRFI+EX_R12(r13)
3000 GET_SCRATCH0(r13);
3001 rfid
3002
3003 TRAMP_REAL_BEGIN(hrfi_flush_fallback)
3004 SET_SCRATCH0(r13);
3005 GET_PACA(r13);
3006 std r1,PACA_EXRFI+EX_R12(r13)
3007 ld r1,PACAKSAVE(r13)
3008 std r9,PACA_EXRFI+EX_R9(r13)
3009 std r10,PACA_EXRFI+EX_R10(r13)
3010 std r11,PACA_EXRFI+EX_R11(r13)
3011 mfctr r9
3012 L1D_DISPLACEMENT_FLUSH
3013 mtctr r9
3014 ld r9,PACA_EXRFI+EX_R9(r13)
3015 ld r10,PACA_EXRFI+EX_R10(r13)
3016 ld r11,PACA_EXRFI+EX_R11(r13)
3017 ld r1,PACA_EXRFI+EX_R12(r13)
3018 GET_SCRATCH0(r13);
3019 hrfid
3020
3021 TRAMP_REAL_BEGIN(rfscv_flush_fallback)
3022 /* system call volatile */
3023 mr r7,r13
3024 GET_PACA(r13);
3025 mr r8,r1
3026 ld r1,PACAKSAVE(r13)
3027 mfctr r9
3028 ld r10,PACA_RFI_FLUSH_FALLBACK_A
3029 ld r11,PACA_L1D_FLUSH_SIZE(r13)
3030 srdi r11,r11,(7 + 3) /* 128 byte l
3031 mtctr r11
3032 DCBT_BOOK3S_STOP_ALL_STREAM_IDS(r11)
3033
3034 /* order ld/st prior to dcbt stop all
3035 sync
3036
3037 /*
3038 * The load adresses are at staggered
3039 * which suits some pipelines better
3040 * hurt).
3041 */
3042 1:
3043 ld r11,(0x80 + 8)*0(r10)
3044 ld r11,(0x80 + 8)*1(r10)
3045 ld r11,(0x80 + 8)*2(r10)
3046 ld r11,(0x80 + 8)*3(r10)
3047 ld r11,(0x80 + 8)*4(r10)
3048 ld r11,(0x80 + 8)*5(r10)
3049 ld r11,(0x80 + 8)*6(r10)
3050 ld r11,(0x80 + 8)*7(r10)
3051 addi r10,r10,0x80*8
3052 bdnz 1b
3053
3054 mtctr r9
3055 li r9,0
3056 li r10,0
3057 li r11,0
3058 mr r1,r8
3059 mr r13,r7
3060 RFSCV
3061
3062 USE_TEXT_SECTION()
3063
3064 #ifdef CONFIG_KVM_BOOK3S_64_HANDLER
3065 kvm_interrupt:
3066 /*
3067 * The conditional branch in KVMTEST
3068 * make a stub.
3069 */
3070 b kvmppc_interrupt
3071 #endif
3072
3073 _GLOBAL(do_uaccess_flush)
3074 UACCESS_FLUSH_FIXUP_SECTION
3075 nop
3076 nop
3077 nop
3078 blr
3079 L1D_DISPLACEMENT_FLUSH
3080 blr
3081 _ASM_NOKPROBE_SYMBOL(do_uaccess_flush)
3082 EXPORT_SYMBOL(do_uaccess_flush)
3083
3084
3085 MASKED_INTERRUPT
3086 MASKED_INTERRUPT hsrr=1
3087
3088 USE_FIXED_SECTION(virt_trampolines)
3089 /*
3090 * All code below __end_soft_masked i
3091 * any code runs here with MSR[EE]=1,
3092 * soft interrupt being raised (i.e.,
3093 *
3094 * The __end_interrupts marker must b
3095 * handlers, so that they are copied
3096 * a relocatable kernel. This ensures
3097 * trampoline handlers (like 0x4f00,
3098 * directly, without using LOAD_HANDL
3099 */
3100 .align 7
3101 .globl __end_interrupts
3102 __end_interrupts:
3103 DEFINE_FIXED_SYMBOL(__end_interrupts, virt_tr
3104
3105 CLOSE_FIXED_SECTION(real_vectors);
3106 CLOSE_FIXED_SECTION(real_trampolines);
3107 CLOSE_FIXED_SECTION(virt_vectors);
3108 CLOSE_FIXED_SECTION(virt_trampolines);
3109
3110 USE_TEXT_SECTION()
3111
3112 /* MSR[RI] should be clear because this uses
3113 _GLOBAL(enable_machine_check)
3114 mflr r0
3115 bcl 20,31,$+4
3116 0: mflr r3
3117 addi r3,r3,(1f - 0b)
3118 mtspr SPRN_SRR0,r3
3119 mfmsr r3
3120 ori r3,r3,MSR_ME
3121 mtspr SPRN_SRR1,r3
3122 RFI_TO_KERNEL
3123 1: mtlr r0
3124 blr
3125
3126 /* MSR[RI] should be clear because this uses
3127 SYM_FUNC_START_LOCAL(disable_machine_check)
3128 mflr r0
3129 bcl 20,31,$+4
3130 0: mflr r3
3131 addi r3,r3,(1f - 0b)
3132 mtspr SPRN_SRR0,r3
3133 mfmsr r3
3134 li r4,MSR_ME
3135 andc r3,r3,r4
3136 mtspr SPRN_SRR1,r3
3137 RFI_TO_KERNEL
3138 1: mtlr r0
3139 blr
3140 SYM_FUNC_END(disable_machine_check)
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.