1 /* SPDX-License-Identifier: GPL-2.0 */ 1 /* SPDX-License-Identifier: GPL-2.0 */
2 /* 2 /*
3 * S390 low-level entry points. !! 3 * arch/alpha/kernel/entry.S
4 * 4 *
5 * Copyright IBM Corp. 1999, 2012 !! 5 * Kernel entry-points.
6 * Author(s): Martin Schwidefsky (schwidefs <<
7 * Hartmut Penner (hp@de.ibm.com <<
8 * Denis Joseph Barrow (djbarrow <<
9 */ 6 */
10 7
11 #include <linux/export.h> <<
12 #include <linux/init.h> <<
13 #include <linux/linkage.h> <<
14 #include <asm/asm-extable.h> <<
15 #include <asm/alternative.h> <<
16 #include <asm/processor.h> <<
17 #include <asm/cache.h> <<
18 #include <asm/dwarf.h> <<
19 #include <asm/errno.h> <<
20 #include <asm/ptrace.h> <<
21 #include <asm/thread_info.h> <<
22 #include <asm/asm-offsets.h> 8 #include <asm/asm-offsets.h>
>> 9 #include <asm/thread_info.h>
>> 10 #include <asm/pal.h>
>> 11 #include <asm/errno.h>
23 #include <asm/unistd.h> 12 #include <asm/unistd.h>
24 #include <asm/page.h> <<
25 #include <asm/sigp.h> <<
26 #include <asm/irq.h> <<
27 #include <asm/fpu-insn.h> <<
28 #include <asm/setup.h> <<
29 #include <asm/nmi.h> <<
30 #include <asm/nospec-insn.h> <<
31 #include <asm/lowcore.h> <<
32 <<
33 _LPP_OFFSET = __LC_LPP <<
34 <<
35 .macro STBEAR address <<
36 ALTERNATIVE "nop", ".insn s,0xb2010000 <<
37 .endm <<
38 <<
39 .macro LBEAR address <<
40 ALTERNATIVE "nop", ".insn s,0xb2000000 <<
41 .endm <<
42 <<
43 .macro LPSWEY address, lpswe <<
44 ALTERNATIVE_2 "b \lpswe;nopr", \ <<
45 ".insn siy,0xeb0000000071,\add <<
46 __stringify(.insn siy,0xeb0000 <<
47 ALT_LOWCORE <<
48 .endm <<
49 <<
50 .macro MBEAR reg, lowcore <<
51 ALTERNATIVE "brcl 0,0", __stringify(mv <<
52 ALT_FACILITY(193) <<
53 .endm <<
54 <<
55 .macro CHECK_STACK savearea, lowcore <<
56 #ifdef CONFIG_CHECK_STACK <<
57 tml %r15,THREAD_SIZE - CONFIG_STAC <<
58 la %r14,\savearea(\lowcore) <<
59 jz stack_overflow <<
60 #endif <<
61 .endm <<
62 13
63 .macro CHECK_VMAP_STACK savearea, low !! 14 .text
64 #ifdef CONFIG_VMAP_STACK !! 15 .set noat
65 lgr %r14,%r15 !! 16 .cfi_sections .debug_frame
66 nill %r14,0x10000 - THREAD_SIZE !! 17
67 oill %r14,STACK_INIT_OFFSET !! 18 /* Stack offsets. */
68 clg %r14,__LC_KERNEL_STACK(\lowcor !! 19 #define SP_OFF 184
69 je \oklabel !! 20 #define SWITCH_STACK_SIZE 64
70 clg %r14,__LC_ASYNC_STACK(\lowcore !! 21
71 je \oklabel !! 22 .macro CFI_START_OSF_FRAME func
72 clg %r14,__LC_MCCK_STACK(\lowcore) !! 23 .align 4
73 je \oklabel !! 24 .globl \func
74 clg %r14,__LC_NODAT_STACK(\lowcore !! 25 .type \func,@function
75 je \oklabel !! 26 \func:
76 clg %r14,__LC_RESTART_STACK(\lowco !! 27 .cfi_startproc simple
77 je \oklabel !! 28 .cfi_return_column 64
78 la %r14,\savearea(\lowcore) !! 29 .cfi_def_cfa $sp, 48
79 j stack_overflow !! 30 .cfi_rel_offset 64, 8
>> 31 .cfi_rel_offset $gp, 16
>> 32 .cfi_rel_offset $16, 24
>> 33 .cfi_rel_offset $17, 32
>> 34 .cfi_rel_offset $18, 40
>> 35 .endm
>> 36
>> 37 .macro CFI_END_OSF_FRAME func
>> 38 .cfi_endproc
>> 39 .size \func, . - \func
>> 40 .endm
>> 41
>> 42 /*
>> 43 * This defines the normal kernel pt-regs layout.
>> 44 *
>> 45 * regs 9-15 preserved by C code
>> 46 * regs 16-18 saved by PAL-code
>> 47 * regs 29-30 saved and set up by PAL-code
>> 48 * JRP - Save regs 16-18 in a special area of the stack, so that
>> 49 * the palcode-provided values are available to the signal handler.
>> 50 */
>> 51
>> 52 .macro SAVE_ALL
>> 53 subq $sp, SP_OFF, $sp
>> 54 .cfi_adjust_cfa_offset SP_OFF
>> 55 stq $0, 0($sp)
>> 56 stq $1, 8($sp)
>> 57 stq $2, 16($sp)
>> 58 stq $3, 24($sp)
>> 59 stq $4, 32($sp)
>> 60 stq $28, 144($sp)
>> 61 .cfi_rel_offset $0, 0
>> 62 .cfi_rel_offset $1, 8
>> 63 .cfi_rel_offset $2, 16
>> 64 .cfi_rel_offset $3, 24
>> 65 .cfi_rel_offset $4, 32
>> 66 .cfi_rel_offset $28, 144
>> 67 lda $2, alpha_mv
>> 68 stq $5, 40($sp)
>> 69 stq $6, 48($sp)
>> 70 stq $7, 56($sp)
>> 71 stq $8, 64($sp)
>> 72 stq $19, 72($sp)
>> 73 stq $20, 80($sp)
>> 74 stq $21, 88($sp)
>> 75 ldq $2, HAE_CACHE($2)
>> 76 stq $22, 96($sp)
>> 77 stq $23, 104($sp)
>> 78 stq $24, 112($sp)
>> 79 stq $25, 120($sp)
>> 80 stq $26, 128($sp)
>> 81 stq $27, 136($sp)
>> 82 stq $2, 152($sp)
>> 83 stq $16, 160($sp)
>> 84 stq $17, 168($sp)
>> 85 stq $18, 176($sp)
>> 86 .cfi_rel_offset $5, 40
>> 87 .cfi_rel_offset $6, 48
>> 88 .cfi_rel_offset $7, 56
>> 89 .cfi_rel_offset $8, 64
>> 90 .cfi_rel_offset $19, 72
>> 91 .cfi_rel_offset $20, 80
>> 92 .cfi_rel_offset $21, 88
>> 93 .cfi_rel_offset $22, 96
>> 94 .cfi_rel_offset $23, 104
>> 95 .cfi_rel_offset $24, 112
>> 96 .cfi_rel_offset $25, 120
>> 97 .cfi_rel_offset $26, 128
>> 98 .cfi_rel_offset $27, 136
>> 99 .endm
>> 100
>> 101 .macro RESTORE_ALL
>> 102 lda $19, alpha_mv
>> 103 ldq $0, 0($sp)
>> 104 ldq $1, 8($sp)
>> 105 ldq $2, 16($sp)
>> 106 ldq $3, 24($sp)
>> 107 ldq $21, 152($sp)
>> 108 ldq $20, HAE_CACHE($19)
>> 109 ldq $4, 32($sp)
>> 110 ldq $5, 40($sp)
>> 111 ldq $6, 48($sp)
>> 112 ldq $7, 56($sp)
>> 113 subq $20, $21, $20
>> 114 ldq $8, 64($sp)
>> 115 beq $20, 99f
>> 116 ldq $20, HAE_REG($19)
>> 117 stq $21, HAE_CACHE($19)
>> 118 stq $21, 0($20)
>> 119 99: ldq $19, 72($sp)
>> 120 ldq $20, 80($sp)
>> 121 ldq $21, 88($sp)
>> 122 ldq $22, 96($sp)
>> 123 ldq $23, 104($sp)
>> 124 ldq $24, 112($sp)
>> 125 ldq $25, 120($sp)
>> 126 ldq $26, 128($sp)
>> 127 ldq $27, 136($sp)
>> 128 ldq $28, 144($sp)
>> 129 addq $sp, SP_OFF, $sp
>> 130 .cfi_restore $0
>> 131 .cfi_restore $1
>> 132 .cfi_restore $2
>> 133 .cfi_restore $3
>> 134 .cfi_restore $4
>> 135 .cfi_restore $5
>> 136 .cfi_restore $6
>> 137 .cfi_restore $7
>> 138 .cfi_restore $8
>> 139 .cfi_restore $19
>> 140 .cfi_restore $20
>> 141 .cfi_restore $21
>> 142 .cfi_restore $22
>> 143 .cfi_restore $23
>> 144 .cfi_restore $24
>> 145 .cfi_restore $25
>> 146 .cfi_restore $26
>> 147 .cfi_restore $27
>> 148 .cfi_restore $28
>> 149 .cfi_adjust_cfa_offset -SP_OFF
>> 150 .endm
>> 151
>> 152 .macro DO_SWITCH_STACK
>> 153 bsr $1, do_switch_stack
>> 154 .cfi_adjust_cfa_offset SWITCH_STACK_SIZE
>> 155 .cfi_rel_offset $9, 0
>> 156 .cfi_rel_offset $10, 8
>> 157 .cfi_rel_offset $11, 16
>> 158 .cfi_rel_offset $12, 24
>> 159 .cfi_rel_offset $13, 32
>> 160 .cfi_rel_offset $14, 40
>> 161 .cfi_rel_offset $15, 48
>> 162 .endm
>> 163
>> 164 .macro UNDO_SWITCH_STACK
>> 165 bsr $1, undo_switch_stack
>> 166 .cfi_restore $9
>> 167 .cfi_restore $10
>> 168 .cfi_restore $11
>> 169 .cfi_restore $12
>> 170 .cfi_restore $13
>> 171 .cfi_restore $14
>> 172 .cfi_restore $15
>> 173 .cfi_adjust_cfa_offset -SWITCH_STACK_SIZE
>> 174 .endm
>> 175
>> 176 /*
>> 177 * Non-syscall kernel entry points.
>> 178 */
>> 179
>> 180 CFI_START_OSF_FRAME entInt
>> 181 SAVE_ALL
>> 182 lda $8, 0x3fff
>> 183 lda $26, ret_from_sys_call
>> 184 bic $sp, $8, $8
>> 185 mov $sp, $19
>> 186 jsr $31, do_entInt
>> 187 CFI_END_OSF_FRAME entInt
>> 188
>> 189 CFI_START_OSF_FRAME entArith
>> 190 SAVE_ALL
>> 191 lda $8, 0x3fff
>> 192 lda $26, ret_from_sys_call
>> 193 bic $sp, $8, $8
>> 194 mov $sp, $18
>> 195 jsr $31, do_entArith
>> 196 CFI_END_OSF_FRAME entArith
>> 197
>> 198 CFI_START_OSF_FRAME entMM
>> 199 SAVE_ALL
>> 200 /* save $9 - $15 so the inline exception code can manipulate them. */
>> 201 subq $sp, 56, $sp
>> 202 .cfi_adjust_cfa_offset 56
>> 203 stq $9, 0($sp)
>> 204 stq $10, 8($sp)
>> 205 stq $11, 16($sp)
>> 206 stq $12, 24($sp)
>> 207 stq $13, 32($sp)
>> 208 stq $14, 40($sp)
>> 209 stq $15, 48($sp)
>> 210 .cfi_rel_offset $9, 0
>> 211 .cfi_rel_offset $10, 8
>> 212 .cfi_rel_offset $11, 16
>> 213 .cfi_rel_offset $12, 24
>> 214 .cfi_rel_offset $13, 32
>> 215 .cfi_rel_offset $14, 40
>> 216 .cfi_rel_offset $15, 48
>> 217 addq $sp, 56, $19
>> 218 /* handle the fault */
>> 219 lda $8, 0x3fff
>> 220 bic $sp, $8, $8
>> 221 jsr $26, do_page_fault
>> 222 /* reload the registers after the exception code played. */
>> 223 ldq $9, 0($sp)
>> 224 ldq $10, 8($sp)
>> 225 ldq $11, 16($sp)
>> 226 ldq $12, 24($sp)
>> 227 ldq $13, 32($sp)
>> 228 ldq $14, 40($sp)
>> 229 ldq $15, 48($sp)
>> 230 addq $sp, 56, $sp
>> 231 .cfi_restore $9
>> 232 .cfi_restore $10
>> 233 .cfi_restore $11
>> 234 .cfi_restore $12
>> 235 .cfi_restore $13
>> 236 .cfi_restore $14
>> 237 .cfi_restore $15
>> 238 .cfi_adjust_cfa_offset -56
>> 239 /* finish up the syscall as normal. */
>> 240 br ret_from_sys_call
>> 241 CFI_END_OSF_FRAME entMM
>> 242
>> 243 CFI_START_OSF_FRAME entIF
>> 244 SAVE_ALL
>> 245 lda $8, 0x3fff
>> 246 lda $26, ret_from_sys_call
>> 247 bic $sp, $8, $8
>> 248 mov $sp, $17
>> 249 jsr $31, do_entIF
>> 250 CFI_END_OSF_FRAME entIF
>> 251
>> 252 CFI_START_OSF_FRAME entUna
>> 253 lda $sp, -256($sp)
>> 254 .cfi_adjust_cfa_offset 256
>> 255 stq $0, 0($sp)
>> 256 .cfi_rel_offset $0, 0
>> 257 .cfi_remember_state
>> 258 ldq $0, 256($sp) /* get PS */
>> 259 stq $1, 8($sp)
>> 260 stq $2, 16($sp)
>> 261 stq $3, 24($sp)
>> 262 and $0, 8, $0 /* user mode? */
>> 263 stq $4, 32($sp)
>> 264 bne $0, entUnaUser /* yup -> do user-level unaligned fault */
>> 265 stq $5, 40($sp)
>> 266 stq $6, 48($sp)
>> 267 stq $7, 56($sp)
>> 268 stq $8, 64($sp)
>> 269 stq $9, 72($sp)
>> 270 stq $10, 80($sp)
>> 271 stq $11, 88($sp)
>> 272 stq $12, 96($sp)
>> 273 stq $13, 104($sp)
>> 274 stq $14, 112($sp)
>> 275 stq $15, 120($sp)
>> 276 /* 16-18 PAL-saved */
>> 277 stq $19, 152($sp)
>> 278 stq $20, 160($sp)
>> 279 stq $21, 168($sp)
>> 280 stq $22, 176($sp)
>> 281 stq $23, 184($sp)
>> 282 stq $24, 192($sp)
>> 283 stq $25, 200($sp)
>> 284 stq $26, 208($sp)
>> 285 stq $27, 216($sp)
>> 286 stq $28, 224($sp)
>> 287 mov $sp, $19
>> 288 stq $gp, 232($sp)
>> 289 .cfi_rel_offset $1, 1*8
>> 290 .cfi_rel_offset $2, 2*8
>> 291 .cfi_rel_offset $3, 3*8
>> 292 .cfi_rel_offset $4, 4*8
>> 293 .cfi_rel_offset $5, 5*8
>> 294 .cfi_rel_offset $6, 6*8
>> 295 .cfi_rel_offset $7, 7*8
>> 296 .cfi_rel_offset $8, 8*8
>> 297 .cfi_rel_offset $9, 9*8
>> 298 .cfi_rel_offset $10, 10*8
>> 299 .cfi_rel_offset $11, 11*8
>> 300 .cfi_rel_offset $12, 12*8
>> 301 .cfi_rel_offset $13, 13*8
>> 302 .cfi_rel_offset $14, 14*8
>> 303 .cfi_rel_offset $15, 15*8
>> 304 .cfi_rel_offset $19, 19*8
>> 305 .cfi_rel_offset $20, 20*8
>> 306 .cfi_rel_offset $21, 21*8
>> 307 .cfi_rel_offset $22, 22*8
>> 308 .cfi_rel_offset $23, 23*8
>> 309 .cfi_rel_offset $24, 24*8
>> 310 .cfi_rel_offset $25, 25*8
>> 311 .cfi_rel_offset $26, 26*8
>> 312 .cfi_rel_offset $27, 27*8
>> 313 .cfi_rel_offset $28, 28*8
>> 314 .cfi_rel_offset $29, 29*8
>> 315 lda $8, 0x3fff
>> 316 stq $31, 248($sp)
>> 317 bic $sp, $8, $8
>> 318 jsr $26, do_entUna
>> 319 ldq $0, 0($sp)
>> 320 ldq $1, 8($sp)
>> 321 ldq $2, 16($sp)
>> 322 ldq $3, 24($sp)
>> 323 ldq $4, 32($sp)
>> 324 ldq $5, 40($sp)
>> 325 ldq $6, 48($sp)
>> 326 ldq $7, 56($sp)
>> 327 ldq $8, 64($sp)
>> 328 ldq $9, 72($sp)
>> 329 ldq $10, 80($sp)
>> 330 ldq $11, 88($sp)
>> 331 ldq $12, 96($sp)
>> 332 ldq $13, 104($sp)
>> 333 ldq $14, 112($sp)
>> 334 ldq $15, 120($sp)
>> 335 /* 16-18 PAL-saved */
>> 336 ldq $19, 152($sp)
>> 337 ldq $20, 160($sp)
>> 338 ldq $21, 168($sp)
>> 339 ldq $22, 176($sp)
>> 340 ldq $23, 184($sp)
>> 341 ldq $24, 192($sp)
>> 342 ldq $25, 200($sp)
>> 343 ldq $26, 208($sp)
>> 344 ldq $27, 216($sp)
>> 345 ldq $28, 224($sp)
>> 346 ldq $gp, 232($sp)
>> 347 lda $sp, 256($sp)
>> 348 .cfi_restore $1
>> 349 .cfi_restore $2
>> 350 .cfi_restore $3
>> 351 .cfi_restore $4
>> 352 .cfi_restore $5
>> 353 .cfi_restore $6
>> 354 .cfi_restore $7
>> 355 .cfi_restore $8
>> 356 .cfi_restore $9
>> 357 .cfi_restore $10
>> 358 .cfi_restore $11
>> 359 .cfi_restore $12
>> 360 .cfi_restore $13
>> 361 .cfi_restore $14
>> 362 .cfi_restore $15
>> 363 .cfi_restore $19
>> 364 .cfi_restore $20
>> 365 .cfi_restore $21
>> 366 .cfi_restore $22
>> 367 .cfi_restore $23
>> 368 .cfi_restore $24
>> 369 .cfi_restore $25
>> 370 .cfi_restore $26
>> 371 .cfi_restore $27
>> 372 .cfi_restore $28
>> 373 .cfi_restore $29
>> 374 .cfi_adjust_cfa_offset -256
>> 375 call_pal PAL_rti
>> 376
>> 377 .align 4
>> 378 entUnaUser:
>> 379 .cfi_restore_state
>> 380 ldq $0, 0($sp) /* restore original $0 */
>> 381 lda $sp, 256($sp) /* pop entUna's stack frame */
>> 382 .cfi_restore $0
>> 383 .cfi_adjust_cfa_offset -256
>> 384 SAVE_ALL /* setup normal kernel stack */
>> 385 lda $sp, -56($sp)
>> 386 .cfi_adjust_cfa_offset 56
>> 387 stq $9, 0($sp)
>> 388 stq $10, 8($sp)
>> 389 stq $11, 16($sp)
>> 390 stq $12, 24($sp)
>> 391 stq $13, 32($sp)
>> 392 stq $14, 40($sp)
>> 393 stq $15, 48($sp)
>> 394 .cfi_rel_offset $9, 0
>> 395 .cfi_rel_offset $10, 8
>> 396 .cfi_rel_offset $11, 16
>> 397 .cfi_rel_offset $12, 24
>> 398 .cfi_rel_offset $13, 32
>> 399 .cfi_rel_offset $14, 40
>> 400 .cfi_rel_offset $15, 48
>> 401 lda $8, 0x3fff
>> 402 addq $sp, 56, $19
>> 403 bic $sp, $8, $8
>> 404 jsr $26, do_entUnaUser
>> 405 ldq $9, 0($sp)
>> 406 ldq $10, 8($sp)
>> 407 ldq $11, 16($sp)
>> 408 ldq $12, 24($sp)
>> 409 ldq $13, 32($sp)
>> 410 ldq $14, 40($sp)
>> 411 ldq $15, 48($sp)
>> 412 lda $sp, 56($sp)
>> 413 .cfi_restore $9
>> 414 .cfi_restore $10
>> 415 .cfi_restore $11
>> 416 .cfi_restore $12
>> 417 .cfi_restore $13
>> 418 .cfi_restore $14
>> 419 .cfi_restore $15
>> 420 .cfi_adjust_cfa_offset -56
>> 421 br ret_from_sys_call
>> 422 CFI_END_OSF_FRAME entUna
>> 423
>> 424 CFI_START_OSF_FRAME entDbg
>> 425 SAVE_ALL
>> 426 lda $8, 0x3fff
>> 427 lda $26, ret_from_sys_call
>> 428 bic $sp, $8, $8
>> 429 mov $sp, $16
>> 430 jsr $31, do_entDbg
>> 431 CFI_END_OSF_FRAME entDbg
>> 432
>> 433 /*
>> 434 * The system call entry point is special. Most importantly, it looks
>> 435 * like a function call to userspace as far as clobbered registers. We
>> 436 * do preserve the argument registers (for syscall restarts) and $26
>> 437 * (for leaf syscall functions).
>> 438 *
>> 439 * So much for theory. We don't take advantage of this yet.
>> 440 *
>> 441 * Note that a0-a2 are not saved by PALcode as with the other entry points.
>> 442 */
>> 443
>> 444 .align 4
>> 445 .globl entSys
>> 446 .type entSys, @function
>> 447 .cfi_startproc simple
>> 448 .cfi_return_column 64
>> 449 .cfi_def_cfa $sp, 48
>> 450 .cfi_rel_offset 64, 8
>> 451 .cfi_rel_offset $gp, 16
>> 452 entSys:
>> 453 SAVE_ALL
>> 454 lda $8, 0x3fff
>> 455 bic $sp, $8, $8
>> 456 lda $4, NR_syscalls($31)
>> 457 stq $16, SP_OFF+24($sp)
>> 458 lda $5, sys_call_table
>> 459 lda $27, sys_ni_syscall
>> 460 cmpult $0, $4, $4
>> 461 ldl $3, TI_FLAGS($8)
>> 462 stq $17, SP_OFF+32($sp)
>> 463 s8addq $0, $5, $5
>> 464 stq $18, SP_OFF+40($sp)
>> 465 .cfi_rel_offset $16, SP_OFF+24
>> 466 .cfi_rel_offset $17, SP_OFF+32
>> 467 .cfi_rel_offset $18, SP_OFF+40
>> 468 #ifdef CONFIG_AUDITSYSCALL
>> 469 lda $6, _TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT
>> 470 and $3, $6, $3
>> 471 bne $3, strace
80 #else 472 #else
81 j \oklabel !! 473 blbs $3, strace /* check for SYSCALL_TRACE in disguise */
82 #endif 474 #endif
83 .endm !! 475 beq $4, 1f
>> 476 ldq $27, 0($5)
>> 477 1: jsr $26, ($27), sys_ni_syscall
>> 478 ldgp $gp, 0($26)
>> 479 blt $0, $syscall_error /* the call failed */
>> 480 $ret_success:
>> 481 stq $0, 0($sp)
>> 482 stq $31, 72($sp) /* a3=0 => no error */
>> 483
>> 484 .align 4
>> 485 .globl ret_from_sys_call
>> 486 ret_from_sys_call:
>> 487 cmovne $26, 0, $18 /* $18 = 0 => non-restartable */
>> 488 ldq $0, SP_OFF($sp)
>> 489 and $0, 8, $0
>> 490 beq $0, ret_to_kernel
>> 491 ret_to_user:
>> 492 /* Make sure need_resched and sigpending don't change between
>> 493 sampling and the rti. */
>> 494 lda $16, 7
>> 495 call_pal PAL_swpipl
>> 496 ldl $17, TI_FLAGS($8)
>> 497 and $17, _TIF_WORK_MASK, $2
>> 498 bne $2, work_pending
>> 499 restore_all:
>> 500 ldl $2, TI_STATUS($8)
>> 501 and $2, TS_SAVED_FP | TS_RESTORE_FP, $3
>> 502 bne $3, restore_fpu
>> 503 restore_other:
>> 504 .cfi_remember_state
>> 505 RESTORE_ALL
>> 506 call_pal PAL_rti
>> 507
>> 508 ret_to_kernel:
>> 509 .cfi_restore_state
>> 510 lda $16, 7
>> 511 call_pal PAL_swpipl
>> 512 br restore_other
84 513
>> 514 .align 3
>> 515 $syscall_error:
85 /* 516 /*
86 * The TSTMSK macro generates a test-u !! 517 * Some system calls (e.g., ptrace) can return arbitrary
87 * calculating the memory offset for t !! 518 * values which might normally be mistaken as error numbers.
88 * Mask value can be any constant. Th !! 519 * Those functions must zero $0 (v0) directly in the stack
89 * value to calculate the memory offse !! 520 * frame to indicate that a negative return value wasn't an
90 * instruction. !! 521 * error number..
91 */ 522 */
92 .macro TSTMSK addr, mask, size=8, byte !! 523 ldq $18, 0($sp) /* old syscall nr (zero if success) */
93 .if (\bytepos < \size) && (\ma !! 524 beq $18, $ret_success
94 .if (\mask & 0xff) <<
95 .error "Mask e <<
96 .endif <<
97 TSTMSK \addr, "(\mask <<
98 .exitm <<
99 .endif <<
100 .ifeq \mask <<
101 .error "Mask must not <<
102 .endif <<
103 off = \size - \bytepos - 1 <<
104 tm off+\addr, \mask <<
105 .endm <<
106 <<
107 .macro BPOFF <<
108 ALTERNATIVE "nop", ".insn rrf,0xb2e800 <<
109 .endm <<
110 <<
111 .macro BPON <<
112 ALTERNATIVE "nop", ".insn rrf,0xb2e800 <<
113 .endm <<
114 <<
115 .macro BPENTER tif_ptr,tif_mask <<
116 ALTERNATIVE "TSTMSK \tif_ptr,\tif_mask <<
117 "j .+12; nop; nop", ALT_SP <<
118 .endm <<
119 <<
120 .macro BPEXIT tif_ptr,tif_mask <<
121 TSTMSK \tif_ptr,\tif_mask <<
122 ALTERNATIVE "jz .+8; .insn rrf,0xb2e8 <<
123 "jnz .+8; .insn rrf,0xb2e8 <<
124 .endm <<
125 <<
126 #if IS_ENABLED(CONFIG_KVM) <<
127 .macro SIEEXIT sie_control,lowcore <<
128 lg %r9,\sie_control <<
129 ni __SIE_PROG0C+3(%r9),0xfe <<
130 lctlg %c1,%c1,__LC_KERNEL_ASCE(\lowc <<
131 lg %r9,__LC_CURRENT(\lowcore) <<
132 mvi __TI_sie(%r9),0 <<
133 larl %r9,sie_exit <<
134 .endm <<
135 #endif <<
136 525
137 .macro STACKLEAK_ERASE !! 526 ldq $19, 72($sp) /* .. and this a3 */
138 #ifdef CONFIG_GCC_PLUGIN_STACKLEAK !! 527 subq $31, $0, $0 /* with error in v0 */
139 brasl %r14,stackleak_erase_on_task_s !! 528 addq $31, 1, $1 /* set a3 for errno return */
140 #endif !! 529 stq $0, 0($sp)
141 .endm !! 530 mov $31, $26 /* tell "ret_from_sys_call" we can restart */
>> 531 stq $1, 72($sp) /* a3 for return */
>> 532 br ret_from_sys_call
142 533
143 GEN_BR_THUNK %r14 !! 534 /*
>> 535 * Do all cleanup when returning from all interrupts and system calls.
>> 536 *
>> 537 * Arguments:
>> 538 * $8: current.
>> 539 * $17: TI_FLAGS.
>> 540 * $18: The old syscall number, or zero if this is not a return
>> 541 * from a syscall that errored and is possibly restartable.
>> 542 * $19: The old a3 value
>> 543 */
>> 544
>> 545 .align 4
>> 546 .type work_pending, @function
>> 547 work_pending:
>> 548 and $17, _TIF_NOTIFY_RESUME | _TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL, $2
>> 549 bne $2, $work_notifysig
144 550
145 .section .kprobes.text, "ax" !! 551 $work_resched:
146 .Ldummy: <<
147 /* 552 /*
148 * The following nop exists only in or !! 553 * We can get here only if we returned from syscall without SIGPENDING
149 * symbol starts at the beginning of t !! 554 * or got through work_notifysig already. Either case means no syscall
150 * In that case there would be several !! 555 * restarts for us, so let $18 and $19 burn.
151 * E.g. objdump would take an arbitrar <<
152 * the code. <<
153 * With the added nop in between this <<
154 */ 556 */
155 nop 0 !! 557 jsr $26, schedule
>> 558 mov 0, $18
>> 559 br ret_to_user
>> 560
>> 561 $work_notifysig:
>> 562 mov $sp, $16
>> 563 DO_SWITCH_STACK
>> 564 jsr $26, do_work_pending
>> 565 UNDO_SWITCH_STACK
>> 566 br restore_all
156 567
157 /* 568 /*
158 * Scheduler resume function, called by __swit !! 569 * PTRACE syscall handler
159 * gpr2 = (task_struct *)prev !! 570 */
160 * gpr3 = (task_struct *)next <<
161 * Returns: <<
162 * gpr2 = prev <<
163 */ <<
164 SYM_FUNC_START(__switch_to_asm) <<
165 stmg %r6,%r15,__SF_GPRS(%r15) <<
166 lghi %r4,__TASK_stack <<
167 lghi %r1,__TASK_thread <<
168 llill %r5,STACK_INIT_OFFSET <<
169 stg %r15,__THREAD_ksp(%r1,%r2) <<
170 lg %r15,0(%r4,%r3) <<
171 agr %r15,%r5 <<
172 GET_LC %r13 <<
173 stg %r3,__LC_CURRENT(%r13) <<
174 stg %r15,__LC_KERNEL_STACK(%r13) <<
175 lg %r15,__THREAD_ksp(%r1,%r3) <<
176 aghi %r3,__TASK_pid <<
177 mvc __LC_CURRENT_PID(4,%r13),0(%r3 <<
178 ALTERNATIVE "nop", "lpp _LPP_OFFSET(%r <<
179 lmg %r6,%r15,__SF_GPRS(%r15) <<
180 BR_EX %r14 <<
181 SYM_FUNC_END(__switch_to_asm) <<
182 571
183 #if IS_ENABLED(CONFIG_KVM) !! 572 .align 4
>> 573 .type strace, @function
>> 574 strace:
>> 575 /* set up signal stack, call syscall_trace */
>> 576 // NB: if anyone adds preemption, this block will need to be protected
>> 577 ldl $1, TI_STATUS($8)
>> 578 and $1, TS_SAVED_FP, $3
>> 579 or $1, TS_SAVED_FP, $2
>> 580 bne $3, 1f
>> 581 stl $2, TI_STATUS($8)
>> 582 bsr $26, __save_fpu
>> 583 1:
>> 584 DO_SWITCH_STACK
>> 585 jsr $26, syscall_trace_enter /* returns the syscall number */
>> 586 UNDO_SWITCH_STACK
>> 587
>> 588 /* get the arguments back.. */
>> 589 ldq $16, SP_OFF+24($sp)
>> 590 ldq $17, SP_OFF+32($sp)
>> 591 ldq $18, SP_OFF+40($sp)
>> 592 ldq $19, 72($sp)
>> 593 ldq $20, 80($sp)
>> 594 ldq $21, 88($sp)
>> 595
>> 596 /* get the system call pointer.. */
>> 597 lda $1, NR_syscalls($31)
>> 598 lda $2, sys_call_table
>> 599 lda $27, sys_ni_syscall
>> 600 cmpult $0, $1, $1
>> 601 s8addq $0, $2, $2
>> 602 beq $1, 1f
>> 603 ldq $27, 0($2)
>> 604 1: jsr $26, ($27), sys_gettimeofday
>> 605 ret_from_straced:
>> 606 ldgp $gp, 0($26)
>> 607
>> 608 /* check return.. */
>> 609 blt $0, $strace_error /* the call failed */
>> 610 $strace_success:
>> 611 stq $31, 72($sp) /* a3=0 => no error */
>> 612 stq $0, 0($sp) /* save return value */
>> 613
>> 614 DO_SWITCH_STACK
>> 615 jsr $26, syscall_trace_leave
>> 616 UNDO_SWITCH_STACK
>> 617 br $31, ret_from_sys_call
>> 618
>> 619 .align 3
>> 620 $strace_error:
>> 621 ldq $18, 0($sp) /* old syscall nr (zero if success) */
>> 622 beq $18, $strace_success
>> 623 ldq $19, 72($sp) /* .. and this a3 */
>> 624
>> 625 subq $31, $0, $0 /* with error in v0 */
>> 626 addq $31, 1, $1 /* set a3 for errno return */
>> 627 stq $0, 0($sp)
>> 628 stq $1, 72($sp) /* a3 for return */
>> 629
>> 630 DO_SWITCH_STACK
>> 631 mov $18, $9 /* save old syscall number */
>> 632 mov $19, $10 /* save old a3 */
>> 633 jsr $26, syscall_trace_leave
>> 634 mov $9, $18
>> 635 mov $10, $19
>> 636 UNDO_SWITCH_STACK
>> 637
>> 638 mov $31, $26 /* tell "ret_from_sys_call" we can restart */
>> 639 br ret_from_sys_call
>> 640 CFI_END_OSF_FRAME entSys
>> 641
184 /* 642 /*
185 * __sie64a calling convention: !! 643 * Save and restore the switch stack -- aka the balance of the user context.
186 * %r2 pointer to sie control block phys !! 644 */
187 * %r3 pointer to sie control block virt !! 645
188 * %r4 guest register save area !! 646 .align 4
189 * %r5 guest asce !! 647 .type do_switch_stack, @function
190 */ !! 648 .cfi_startproc simple
191 SYM_FUNC_START(__sie64a) !! 649 .cfi_return_column 64
192 stmg %r6,%r14,__SF_GPRS(%r15) !! 650 .cfi_def_cfa $sp, 0
193 GET_LC %r13 !! 651 .cfi_register 64, $1
194 lg %r14,__LC_CURRENT(%r13) !! 652 do_switch_stack:
195 stg %r2,__SF_SIE_CONTROL_PHYS(%r15 !! 653 lda $sp, -SWITCH_STACK_SIZE($sp)
196 stg %r3,__SF_SIE_CONTROL(%r15) !! 654 .cfi_adjust_cfa_offset SWITCH_STACK_SIZE
197 stg %r4,__SF_SIE_SAVEAREA(%r15) !! 655 stq $9, 0($sp)
198 stg %r5,__SF_SIE_GUEST_ASCE(%r15) !! 656 stq $10, 8($sp)
199 xc __SF_SIE_REASON(8,%r15),__SF_S !! 657 stq $11, 16($sp)
200 mvc __SF_SIE_FLAGS(8,%r15),__TI_fl !! 658 stq $12, 24($sp)
201 lmg %r0,%r13,0(%r4) !! 659 stq $13, 32($sp)
202 mvi __TI_sie(%r14),1 !! 660 stq $14, 40($sp)
203 lctlg %c1,%c1,__SF_SIE_GUEST_ASCE(%r !! 661 stq $15, 48($sp)
204 lg %r14,__SF_SIE_CONTROL(%r15) !! 662 stq $26, 56($sp)
205 oi __SIE_PROG0C+3(%r14),1 !! 663 ret $31, ($1), 1
206 tm __SIE_PROG20+3(%r14),3 !! 664 .cfi_endproc
207 jnz .Lsie_skip !! 665 .size do_switch_stack, .-do_switch_stack
208 lg %r14,__SF_SIE_CONTROL_PHYS(%r1 !! 666
209 BPEXIT __SF_SIE_FLAGS(%r15),_TIF_ISOL !! 667 .align 4
210 .Lsie_entry: !! 668 .type undo_switch_stack, @function
211 sie 0(%r14) !! 669 .cfi_startproc simple
212 # Let the next instruction be NOP to avoid tri !! 670 .cfi_def_cfa $sp, 0
213 # and handling it in a guest as result of the !! 671 .cfi_register 64, $1
214 nopr 7 !! 672 undo_switch_stack:
215 .Lsie_leave: !! 673 ldq $9, 0($sp)
216 BPOFF !! 674 ldq $10, 8($sp)
217 BPENTER __SF_SIE_FLAGS(%r15),_TIF_ISOL !! 675 ldq $11, 16($sp)
218 .Lsie_skip: !! 676 ldq $12, 24($sp)
219 lg %r14,__SF_SIE_CONTROL(%r15) !! 677 ldq $13, 32($sp)
220 ni __SIE_PROG0C+3(%r14),0xfe !! 678 ldq $14, 40($sp)
221 GET_LC %r14 !! 679 ldq $15, 48($sp)
222 lctlg %c1,%c1,__LC_KERNEL_ASCE(%r14) !! 680 ldq $26, 56($sp)
223 lg %r14,__LC_CURRENT(%r14) !! 681 lda $sp, SWITCH_STACK_SIZE($sp)
224 mvi __TI_sie(%r14),0 !! 682 ret $31, ($1), 1
225 # some program checks are suppressing. C code !! 683 .cfi_endproc
226 # will rewind the PSW by the ILC, which is oft !! 684 .size undo_switch_stack, .-undo_switch_stack
227 # are some corner cases (e.g. runtime instrume !! 685
228 # Other instructions between __sie64a and .Lsi !! 686 #define FR(n) n * 8 + TI_FP($8)
229 # interrupts. So lets use 3 nops as a landing !! 687 .align 4
230 .Lrewind_pad6: !! 688 .globl __save_fpu
231 nopr 7 !! 689 .type __save_fpu, @function
232 .Lrewind_pad4: !! 690 __save_fpu:
233 nopr 7 !! 691 #define V(n) stt $f##n, FR(n)
234 .Lrewind_pad2: !! 692 V( 0); V( 1); V( 2); V( 3)
235 nopr 7 !! 693 V( 4); V( 5); V( 6); V( 7)
236 SYM_INNER_LABEL(sie_exit, SYM_L_GLOBAL) !! 694 V( 8); V( 9); V(10); V(11)
237 lg %r14,__SF_SIE_SAVEAREA(%r15) !! 695 V(12); V(13); V(14); V(15)
238 stmg %r0,%r13,0(%r14) !! 696 V(16); V(17); V(18); V(19)
239 xgr %r0,%r0 !! 697 V(20); V(21); V(22); V(23)
240 xgr %r1,%r1 !! 698 V(24); V(25); V(26); V(27)
241 xgr %r3,%r3 !! 699 mf_fpcr $f0 # get fpcr
242 xgr %r4,%r4 !! 700 V(28); V(29); V(30)
243 xgr %r5,%r5 !! 701 stt $f0, FR(31) # save fpcr in slot of $f31
244 lmg %r6,%r14,__SF_GPRS(%r15) !! 702 ldt $f0, FR(0) # don't let "__save_fpu" change fp state.
245 lg %r2,__SF_SIE_REASON(%r15) !! 703 ret
246 BR_EX %r14 !! 704 #undef V
247 .Lsie_fault: !! 705 .size __save_fpu, .-__save_fpu
248 lghi %r14,-EFAULT !! 706
249 stg %r14,__SF_SIE_REASON(%r15) !! 707 .align 4
250 j sie_exit !! 708 restore_fpu:
251 !! 709 and $3, TS_RESTORE_FP, $3
252 EX_TABLE(.Lrewind_pad6,.Lsie_fault) !! 710 bic $2, TS_SAVED_FP | TS_RESTORE_FP, $2
253 EX_TABLE(.Lrewind_pad4,.Lsie_fault) !! 711 beq $3, 1f
254 EX_TABLE(.Lrewind_pad2,.Lsie_fault) !! 712 #define V(n) ldt $f##n, FR(n)
255 EX_TABLE(sie_exit,.Lsie_fault) !! 713 ldt $f30, FR(31) # get saved fpcr
256 SYM_FUNC_END(__sie64a) !! 714 V( 0); V( 1); V( 2); V( 3)
257 EXPORT_SYMBOL(__sie64a) !! 715 mt_fpcr $f30 # install saved fpcr
258 EXPORT_SYMBOL(sie_exit) !! 716 V( 4); V( 5); V( 6); V( 7)
259 #endif !! 717 V( 8); V( 9); V(10); V(11)
>> 718 V(12); V(13); V(14); V(15)
>> 719 V(16); V(17); V(18); V(19)
>> 720 V(20); V(21); V(22); V(23)
>> 721 V(24); V(25); V(26); V(27)
>> 722 V(28); V(29); V(30)
>> 723 1: stl $2, TI_STATUS($8)
>> 724 br restore_other
>> 725 #undef V
260 726
>> 727
261 /* 728 /*
262 * SVC interrupt handler routine. System calls !! 729 * The meat of the context switch code.
263 * are entered with interrupts disabled. <<
264 */ 730 */
265 !! 731 .align 4
266 SYM_CODE_START(system_call) !! 732 .globl alpha_switch_to
267 STMG_LC %r8,%r15,__LC_SAVE_AREA !! 733 .type alpha_switch_to, @function
268 GET_LC %r13 !! 734 .cfi_startproc
269 stpt __LC_SYS_ENTER_TIMER(%r13) !! 735 alpha_switch_to:
270 BPOFF !! 736 DO_SWITCH_STACK
271 lghi %r14,0 !! 737 ldl $1, TI_STATUS($8)
272 .Lsysc_per: !! 738 and $1, TS_RESTORE_FP, $3
273 STBEAR __LC_LAST_BREAK(%r13) !! 739 bne $3, 1f
274 lctlg %c1,%c1,__LC_KERNEL_ASCE(%r13) !! 740 or $1, TS_RESTORE_FP | TS_SAVED_FP, $2
275 lg %r15,__LC_KERNEL_STACK(%r13) !! 741 and $1, TS_SAVED_FP, $3
276 xc __SF_BACKCHAIN(8,%r15),__SF_BA !! 742 stl $2, TI_STATUS($8)
277 stmg %r0,%r7,STACK_FRAME_OVERHEAD+_ !! 743 bne $3, 1f
278 # clear user controlled register to pr !! 744 bsr $26, __save_fpu
279 xgr %r0,%r0 !! 745 1:
280 xgr %r1,%r1 !! 746 call_pal PAL_swpctx
281 xgr %r4,%r4 !! 747 lda $8, 0x3fff
282 xgr %r5,%r5 !! 748 UNDO_SWITCH_STACK
283 xgr %r6,%r6 !! 749 bic $sp, $8, $8
284 xgr %r7,%r7 !! 750 mov $17, $0
285 xgr %r8,%r8 !! 751 ret
286 xgr %r9,%r9 !! 752 .cfi_endproc
287 xgr %r10,%r10 !! 753 .size alpha_switch_to, .-alpha_switch_to
288 xgr %r11,%r11 <<
289 la %r2,STACK_FRAME_OVERHEAD(%r15) <<
290 mvc __PT_R8(64,%r2),__LC_SAVE_AREA <<
291 MBEAR %r2,%r13 <<
292 lgr %r3,%r14 <<
293 brasl %r14,__do_syscall <<
294 STACKLEAK_ERASE <<
295 lctlg %c1,%c1,__LC_USER_ASCE(%r13) <<
296 mvc __LC_RETURN_PSW(16,%r13),STACK <<
297 BPON <<
298 LBEAR STACK_FRAME_OVERHEAD+__PT_LAST <<
299 stpt __LC_EXIT_TIMER(%r13) <<
300 lmg %r0,%r15,STACK_FRAME_OVERHEAD+ <<
301 LPSWEY __LC_RETURN_PSW,__LC_RETURN_LP <<
302 SYM_CODE_END(system_call) <<
303 <<
304 # <<
305 # a new process exits the kernel with ret_from <<
306 # <<
307 SYM_CODE_START(ret_from_fork) <<
308 lgr %r3,%r11 <<
309 brasl %r14,__ret_from_fork <<
310 STACKLEAK_ERASE <<
311 GET_LC %r13 <<
312 lctlg %c1,%c1,__LC_USER_ASCE(%r13) <<
313 mvc __LC_RETURN_PSW(16,%r13),STACK <<
314 BPON <<
315 LBEAR STACK_FRAME_OVERHEAD+__PT_LAST <<
316 stpt __LC_EXIT_TIMER(%r13) <<
317 lmg %r0,%r15,STACK_FRAME_OVERHEAD+ <<
318 LPSWEY __LC_RETURN_PSW,__LC_RETURN_LP <<
319 SYM_CODE_END(ret_from_fork) <<
320 754
321 /* 755 /*
322 * Program check handler routine !! 756 * New processes begin life here.
323 */ 757 */
324 758
325 SYM_CODE_START(pgm_check_handler) !! 759 .globl ret_from_fork
326 STMG_LC %r8,%r15,__LC_SAVE_AREA !! 760 .align 4
327 GET_LC %r13 !! 761 .ent ret_from_fork
328 stpt __LC_SYS_ENTER_TIMER(%r13) !! 762 ret_from_fork:
329 BPOFF !! 763 lda $26, ret_to_user
330 lgr %r10,%r15 !! 764 mov $17, $16
331 lmg %r8,%r9,__LC_PGM_OLD_PSW(%r13) !! 765 jmp $31, schedule_tail
332 tmhh %r8,0x0001 # comi !! 766 .end ret_from_fork
333 jno .Lpgm_skip_asce <<
334 lctlg %c1,%c1,__LC_KERNEL_ASCE(%r13) <<
335 j 3f # -> f <<
336 .Lpgm_skip_asce: <<
337 1: tmhh %r8,0x4000 # PER <<
338 jnz 2f # -> e <<
339 tm __LC_PGM_ILC+3(%r13),0x80 <<
340 jnz .Lpgm_svcper # -> s <<
341 2: CHECK_STACK __LC_SAVE_AREA,%r13 <<
342 aghi %r15,-(STACK_FRAME_OVERHEAD + <<
343 # CHECK_VMAP_STACK branches to stack_o <<
344 CHECK_VMAP_STACK __LC_SAVE_AREA,%r13,4 <<
345 3: lg %r15,__LC_KERNEL_STACK(%r13) <<
346 4: la %r11,STACK_FRAME_OVERHEAD(%r15 <<
347 xc __PT_FLAGS(8,%r11),__PT_FLAGS( <<
348 xc __SF_BACKCHAIN(8,%r15),__SF_BA <<
349 stmg %r0,%r7,__PT_R0(%r11) <<
350 mvc __PT_R8(64,%r11),__LC_SAVE_ARE <<
351 mvc __PT_LAST_BREAK(8,%r11),__LC_P <<
352 stctg %c1,%c1,__PT_CR1(%r11) <<
353 #if IS_ENABLED(CONFIG_KVM) <<
354 ltg %r12,__LC_GMAP(%r13) <<
355 jz 5f <<
356 clc __GMAP_ASCE(8,%r12), __PT_CR1( <<
357 jne 5f <<
358 BPENTER __SF_SIE_FLAGS(%r10),_TIF_ISOL <<
359 SIEEXIT __SF_SIE_CONTROL(%r10),%r13 <<
360 #endif <<
361 5: stmg %r8,%r9,__PT_PSW(%r11) <<
362 # clear user controlled registers to p <<
363 xgr %r0,%r0 <<
364 xgr %r1,%r1 <<
365 xgr %r3,%r3 <<
366 xgr %r4,%r4 <<
367 xgr %r5,%r5 <<
368 xgr %r6,%r6 <<
369 xgr %r7,%r7 <<
370 lgr %r2,%r11 <<
371 brasl %r14,__do_pgm_check <<
372 tmhh %r8,0x0001 # retu <<
373 jno .Lpgm_exit_kernel <<
374 STACKLEAK_ERASE <<
375 lctlg %c1,%c1,__LC_USER_ASCE(%r13) <<
376 BPON <<
377 stpt __LC_EXIT_TIMER(%r13) <<
378 .Lpgm_exit_kernel: <<
379 mvc __LC_RETURN_PSW(16,%r13),STACK <<
380 LBEAR STACK_FRAME_OVERHEAD+__PT_LAST <<
381 lmg %r0,%r15,STACK_FRAME_OVERHEAD+ <<
382 LPSWEY __LC_RETURN_PSW,__LC_RETURN_LP <<
383 <<
384 # <<
385 # single stepped system call <<
386 # <<
387 .Lpgm_svcper: <<
388 mvc __LC_RETURN_PSW(8,%r13),__LC_S <<
389 larl %r14,.Lsysc_per <<
390 stg %r14,__LC_RETURN_PSW+8(%r13) <<
391 lghi %r14,1 <<
392 LBEAR __LC_PGM_LAST_BREAK(%r13) <<
393 LPSWEY __LC_RETURN_PSW,__LC_RETURN_LP <<
394 SYM_CODE_END(pgm_check_handler) <<
395 767
396 /* 768 /*
397 * Interrupt handler macro used for external a !! 769 * ... and new kernel threads - here
398 */ 770 */
399 .macro INT_HANDLER name,lc_old_psw,handler !! 771 .align 4
400 SYM_CODE_START(\name) !! 772 .globl ret_from_kernel_thread
401 STMG_LC %r8,%r15,__LC_SAVE_AREA !! 773 .ent ret_from_kernel_thread
402 GET_LC %r13 !! 774 ret_from_kernel_thread:
403 stckf __LC_INT_CLOCK(%r13) !! 775 mov $17, $16
404 stpt __LC_SYS_ENTER_TIMER(%r13) !! 776 jsr $26, schedule_tail
405 STBEAR __LC_LAST_BREAK(%r13) !! 777 mov $9, $27
406 BPOFF !! 778 mov $10, $16
407 lmg %r8,%r9,\lc_old_psw(%r13) !! 779 jsr $26, ($9)
408 tmhh %r8,0x0001 !! 780 br $31, ret_to_user
409 jnz 1f !! 781 .end ret_from_kernel_thread
410 #if IS_ENABLED(CONFIG_KVM) <<
411 lg %r10,__LC_CURRENT(%r13) <<
412 tm __TI_sie(%r10),0xff <<
413 jz 0f <<
414 BPENTER __SF_SIE_FLAGS(%r15),_TIF_ISOL <<
415 SIEEXIT __SF_SIE_CONTROL(%r15),%r13 <<
416 #endif <<
417 0: CHECK_STACK __LC_SAVE_AREA,%r13 <<
418 aghi %r15,-(STACK_FRAME_OVERHEAD + <<
419 j 2f <<
420 1: lctlg %c1,%c1,__LC_KERNEL_ASCE(%r13) <<
421 lg %r15,__LC_KERNEL_STACK(%r13) <<
422 2: xc __SF_BACKCHAIN(8,%r15),__SF_BA <<
423 la %r11,STACK_FRAME_OVERHEAD(%r15 <<
424 stmg %r0,%r7,__PT_R0(%r11) <<
425 # clear user controlled registers to p <<
426 xgr %r0,%r0 <<
427 xgr %r1,%r1 <<
428 xgr %r3,%r3 <<
429 xgr %r4,%r4 <<
430 xgr %r5,%r5 <<
431 xgr %r6,%r6 <<
432 xgr %r7,%r7 <<
433 xgr %r10,%r10 <<
434 xc __PT_FLAGS(8,%r11),__PT_FLAGS( <<
435 mvc __PT_R8(64,%r11),__LC_SAVE_ARE <<
436 MBEAR %r11,%r13 <<
437 stmg %r8,%r9,__PT_PSW(%r11) <<
438 lgr %r2,%r11 # pass <<
439 brasl %r14,\handler <<
440 mvc __LC_RETURN_PSW(16,%r13),__PT_ <<
441 tmhh %r8,0x0001 # retu <<
442 jno 2f <<
443 STACKLEAK_ERASE <<
444 lctlg %c1,%c1,__LC_USER_ASCE(%r13) <<
445 BPON <<
446 stpt __LC_EXIT_TIMER(%r13) <<
447 2: LBEAR __PT_LAST_BREAK(%r11) <<
448 lmg %r0,%r15,__PT_R0(%r11) <<
449 LPSWEY __LC_RETURN_PSW,__LC_RETURN_LP <<
450 SYM_CODE_END(\name) <<
451 .endm <<
452 <<
453 INT_HANDLER ext_int_handler,__LC_EXT_OLD_PSW,d <<
454 INT_HANDLER io_int_handler,__LC_IO_OLD_PSW,do_ <<
455 782
>> 783
456 /* 784 /*
457 * Machine check handler routines !! 785 * Special system calls. Most of these are special in that they either
>> 786 * have to play switch_stack games.
458 */ 787 */
459 SYM_CODE_START(mcck_int_handler) <<
460 BPOFF <<
461 GET_LC %r13 <<
462 lmg %r8,%r9,__LC_MCK_OLD_PSW(%r13) <<
463 TSTMSK __LC_MCCK_CODE(%r13),MCCK_CODE <<
464 jo .Lmcck_panic # yes <<
465 TSTMSK __LC_MCCK_CODE(%r13),MCCK_CODE <<
466 jno .Lmcck_panic # cont <<
467 ptlb <<
468 lay %r14,__LC_CPU_TIMER_SAVE_AREA( <<
469 mvc __LC_MCCK_ENTER_TIMER(8,%r13), <<
470 TSTMSK __LC_MCCK_CODE(%r13),MCCK_CODE <<
471 jo 3f <<
472 la %r14,__LC_SYS_ENTER_TIMER(%r13 <<
473 clc 0(8,%r14),__LC_EXIT_TIMER(%r13 <<
474 jl 1f <<
475 la %r14,__LC_EXIT_TIMER(%r13) <<
476 1: clc 0(8,%r14),__LC_LAST_UPDATE_TIM <<
477 jl 2f <<
478 la %r14,__LC_LAST_UPDATE_TIMER(%r <<
479 2: spt 0(%r14) <<
480 mvc __LC_MCCK_ENTER_TIMER(8,%r13), <<
481 3: TSTMSK __LC_MCCK_CODE(%r13),MCCK_CODE <<
482 jno .Lmcck_panic <<
483 tmhh %r8,0x0001 # inte <<
484 jnz .Lmcck_user <<
485 TSTMSK __LC_MCCK_CODE(%r13),MCCK_CODE <<
486 jno .Lmcck_panic <<
487 #if IS_ENABLED(CONFIG_KVM) <<
488 lg %r10,__LC_CURRENT(%r13) <<
489 tm __TI_sie(%r10),0xff <<
490 jz .Lmcck_user <<
491 # Need to compare the address instead <<
492 # Otherwise there would be a race betw <<
493 # and entering SIE (or leaving and cle <<
494 # would cause machine checks targeted <<
495 # handled by the host. <<
496 larl %r14,.Lsie_entry <<
497 clgrjl %r9,%r14, 4f <<
498 larl %r14,.Lsie_leave <<
499 clgrjhe %r9,%r14, 4f <<
500 lg %r10,__LC_PCPU <<
501 oi __PCPU_FLAGS+7(%r10), _CIF_MCC <<
502 4: BPENTER __SF_SIE_FLAGS(%r15),_TIF_ISOL <<
503 SIEEXIT __SF_SIE_CONTROL(%r15),%r13 <<
504 #endif <<
505 .Lmcck_user: <<
506 lg %r15,__LC_MCCK_STACK(%r13) <<
507 la %r11,STACK_FRAME_OVERHEAD(%r15 <<
508 stctg %c1,%c1,__PT_CR1(%r11) <<
509 lctlg %c1,%c1,__LC_KERNEL_ASCE(%r13) <<
510 xc __SF_BACKCHAIN(8,%r15),__SF_BA <<
511 lay %r14,__LC_GPREGS_SAVE_AREA(%r1 <<
512 mvc __PT_R0(128,%r11),0(%r14) <<
513 # clear user controlled registers to p <<
514 xgr %r0,%r0 <<
515 xgr %r1,%r1 <<
516 xgr %r3,%r3 <<
517 xgr %r4,%r4 <<
518 xgr %r5,%r5 <<
519 xgr %r6,%r6 <<
520 xgr %r7,%r7 <<
521 xgr %r10,%r10 <<
522 stmg %r8,%r9,__PT_PSW(%r11) <<
523 xc __PT_FLAGS(8,%r11),__PT_FLAGS( <<
524 xc __SF_BACKCHAIN(8,%r15),__SF_BA <<
525 lgr %r2,%r11 # pass <<
526 brasl %r14,s390_do_machine_check <<
527 lctlg %c1,%c1,__PT_CR1(%r11) <<
528 lmg %r0,%r10,__PT_R0(%r11) <<
529 mvc __LC_RETURN_MCCK_PSW(16,%r13), <<
530 tm __LC_RETURN_MCCK_PSW+1(%r13),0 <<
531 jno 0f <<
532 BPON <<
533 stpt __LC_EXIT_TIMER(%r13) <<
534 0: ALTERNATIVE "brcl 0,0", __stringify(la <<
535 ALT_FACILITY(193) <<
536 LBEAR 0(%r12) <<
537 lmg %r11,%r15,__PT_R11(%r11) <<
538 LPSWEY __LC_RETURN_MCCK_PSW,__LC_RETU <<
539 788
540 .Lmcck_panic: !! 789 .macro fork_like name
541 /* !! 790 .align 4
542 * Iterate over all possible CPU addre !! 791 .globl alpha_\name
543 * and stop each CPU using signal proc !! 792 .ent alpha_\name
544 * to allow just one CPU-stopper and p !! 793 alpha_\name:
545 * stopping each other while leaving t !! 794 .prologue 0
546 */ !! 795 bsr $1, do_switch_stack
547 lhi %r5,0 !! 796 // NB: if anyone adds preemption, this block will need to be protected
548 lhi %r6,1 !! 797 ldl $1, TI_STATUS($8)
549 larl %r7,stop_lock !! 798 and $1, TS_SAVED_FP, $3
550 cs %r5,%r6,0(%r7) # sing !! 799 or $1, TS_SAVED_FP, $2
551 jnz 4f !! 800 bne $3, 1f
552 larl %r7,this_cpu !! 801 stl $2, TI_STATUS($8)
553 stap 0(%r7) # this !! 802 bsr $26, __save_fpu
554 lh %r4,0(%r7) !! 803 1:
555 nilh %r4,0 !! 804 jsr $26, sys_\name
556 lhi %r0,1 !! 805 ldq $26, 56($sp)
557 sll %r0,16 # CPU !! 806 lda $sp, SWITCH_STACK_SIZE($sp)
558 lhi %r3,0 # next !! 807 ret
559 0: cr %r3,%r4 !! 808 .end alpha_\name
560 je 2f !! 809 .endm
561 1: sigp %r1,%r3,SIGP_STOP # stop <<
562 brc SIGP_CC_BUSY,1b <<
563 2: ahi %r3,1 <<
564 brct %r0,0b <<
565 3: sigp %r1,%r4,SIGP_STOP # stop <<
566 brc SIGP_CC_BUSY,3b <<
567 4: j 4b <<
568 SYM_CODE_END(mcck_int_handler) <<
569 <<
570 SYM_CODE_START(restart_int_handler) <<
571 ALTERNATIVE "nop", "lpp _LPP_OFFSET", <<
572 stg %r15,__LC_SAVE_AREA_RESTART <<
573 TSTMSK __LC_RESTART_FLAGS,RESTART_FLA <<
574 jz 0f <<
575 lctlg %c0,%c15,__LC_CREGS_SAVE_AREA <<
576 0: larl %r15,daton_psw <<
577 lpswe 0(%r15) <<
578 .Ldaton: <<
579 GET_LC %r15 <<
580 lg %r15,__LC_RESTART_STACK(%r15) <<
581 xc STACK_FRAME_OVERHEAD(__PT_SIZE <<
582 stmg %r0,%r14,STACK_FRAME_OVERHEAD+ <<
583 GET_LC %r13 <<
584 mvc STACK_FRAME_OVERHEAD+__PT_R15( <<
585 mvc STACK_FRAME_OVERHEAD+__PT_PSW( <<
586 xc 0(STACK_FRAME_OVERHEAD,%r15),0 <<
587 lg %r1,__LC_RESTART_FN(%r13) <<
588 lg %r2,__LC_RESTART_DATA(%r13) <<
589 lgf %r3,__LC_RESTART_SOURCE(%r13) <<
590 ltgr %r3,%r3 <<
591 jm 1f <<
592 0: sigp %r4,%r3,SIGP_SENSE <<
593 brc 10,0b <<
594 1: basr %r14,%r1 <<
595 stap __SF_EMPTY(%r15) <<
596 llgh %r3,__SF_EMPTY(%r15) <<
597 2: sigp %r4,%r3,SIGP_STOP <<
598 brc 2,2b <<
599 3: j 3b <<
600 SYM_CODE_END(restart_int_handler) <<
601 <<
602 __INIT <<
603 SYM_CODE_START(early_pgm_check_handler) <<
604 STMG_LC %r8,%r15,__LC_SAVE_AREA <<
605 GET_LC %r13 <<
606 aghi %r15,-(STACK_FRAME_OVERHEAD+__ <<
607 la %r11,STACK_FRAME_OVERHEAD(%r15 <<
608 xc __SF_BACKCHAIN(8,%r15),__SF_BA <<
609 stmg %r0,%r7,__PT_R0(%r11) <<
610 mvc __PT_PSW(16,%r11),__LC_PGM_OLD <<
611 mvc __PT_R8(64,%r11),__LC_SAVE_ARE <<
612 lgr %r2,%r11 <<
613 brasl %r14,__do_early_pgm_check <<
614 mvc __LC_RETURN_PSW(16,%r13),STACK <<
615 lmg %r0,%r15,STACK_FRAME_OVERHEAD+ <<
616 LPSWEY __LC_RETURN_PSW,__LC_RETURN_LP <<
617 SYM_CODE_END(early_pgm_check_handler) <<
618 __FINIT <<
619 810
620 .section .kprobes.text, "ax" !! 811 fork_like fork
>> 812 fork_like vfork
>> 813 fork_like clone
>> 814 fork_like clone3
>> 815
>> 816 .macro sigreturn_like name
>> 817 .align 4
>> 818 .globl sys_\name
>> 819 .ent sys_\name
>> 820 sys_\name:
>> 821 .prologue 0
>> 822 lda $9, ret_from_straced
>> 823 cmpult $26, $9, $9
>> 824 lda $sp, -SWITCH_STACK_SIZE($sp)
>> 825 jsr $26, do_\name
>> 826 bne $9, 1f
>> 827 jsr $26, syscall_trace_leave
>> 828 1: br $1, undo_switch_stack
>> 829 br ret_from_sys_call
>> 830 .end sys_\name
>> 831 .endm
621 832
622 #if defined(CONFIG_CHECK_STACK) || defined(CON !! 833 sigreturn_like sigreturn
623 /* !! 834 sigreturn_like rt_sigreturn
624 * The synchronous or the asynchronous stack o <<
625 * No need to properly save the registers, we <<
626 * Setup a pt_regs so that show_trace can prov <<
627 */ <<
628 SYM_CODE_START(stack_overflow) <<
629 GET_LC %r15 <<
630 lg %r15,__LC_NODAT_STACK(%r15) # <<
631 la %r11,STACK_FRAME_OVERHEAD(%r15 <<
632 stmg %r0,%r7,__PT_R0(%r11) <<
633 stmg %r8,%r9,__PT_PSW(%r11) <<
634 mvc __PT_R8(64,%r11),0(%r14) <<
635 stg %r10,__PT_ORIG_GPR2(%r11) # st <<
636 xc __SF_BACKCHAIN(8,%r15),__SF_BA <<
637 lgr %r2,%r11 # pass <<
638 jg kernel_stack_overflow <<
639 SYM_CODE_END(stack_overflow) <<
640 #endif <<
641 835
642 .section .data, "aw" !! 836 .align 4
643 .balign 4 !! 837 .globl alpha_syscall_zero
644 SYM_DATA_LOCAL(stop_lock, .long 0) !! 838 .ent alpha_syscall_zero
645 SYM_DATA_LOCAL(this_cpu, .short 0) !! 839 alpha_syscall_zero:
646 .balign 8 !! 840 .prologue 0
647 SYM_DATA_START_LOCAL(daton_psw) !! 841 /* Special because it needs to do something opposite to
648 .quad PSW_KERNEL_BITS !! 842 force_successful_syscall_return(). We use the saved
649 .quad .Ldaton !! 843 syscall number for that, zero meaning "not an error".
650 SYM_DATA_END(daton_psw) !! 844 That works nicely, but for real syscall 0 we need to
651 !! 845 make sure that this logics doesn't get confused.
652 .section .rodata, "a" !! 846 Store a non-zero there - -ENOSYS we need in register
653 .balign 8 !! 847 for our return value will do just fine.
654 #define SYSCALL(esame,emu) .quad __s390x_ !! 848 */
655 SYM_DATA_START(sys_call_table) !! 849 lda $0, -ENOSYS
656 #include "asm/syscall_table.h" !! 850 unop
657 SYM_DATA_END(sys_call_table) !! 851 stq $0, 0($sp)
658 #undef SYSCALL !! 852 ret
659 !! 853 .end alpha_syscall_zero
660 #ifdef CONFIG_COMPAT <<
661 <<
662 #define SYSCALL(esame,emu) .quad __s390_ <<
663 SYM_DATA_START(sys_call_table_emu) <<
664 #include "asm/syscall_table.h" <<
665 SYM_DATA_END(sys_call_table_emu) <<
666 #undef SYSCALL <<
667 #endif <<
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.