1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * Transactional memory support routines to re
4 * transactional process state.
5 *
6 * Copyright 2012 Matt Evans & Michael Neuling
7 */
8
9 #include <linux/export.h>
10 #include <asm/asm-offsets.h>
11 #include <asm/ppc_asm.h>
12 #include <asm/ppc-opcode.h>
13 #include <asm/ptrace.h>
14 #include <asm/reg.h>
15 #include <asm/bug.h>
16 #include <asm/feature-fixups.h>
17
18 #ifdef CONFIG_VSX
19 /* See fpu.S, this is borrowed from there */
20 #define __SAVE_32FPRS_VSRS(n,c,base)
21 BEGIN_FTR_SECTION
22 b 2f;
23 END_FTR_SECTION_IFSET(CPU_FTR_VSX);
24 SAVE_32FPRS(n,base);
25 b 3f;
26 2: SAVE_32VSRS(n,c,base);
27 3:
28 #define __REST_32FPRS_VSRS(n,c,base)
29 BEGIN_FTR_SECTION
30 b 2f;
31 END_FTR_SECTION_IFSET(CPU_FTR_VSX);
32 REST_32FPRS(n,base);
33 b 3f;
34 2: REST_32VSRS(n,c,base);
35 3:
36 #else
37 #define __SAVE_32FPRS_VSRS(n,c,base) SAVE_3
38 #define __REST_32FPRS_VSRS(n,c,base) REST_3
39 #endif
40 #define SAVE_32FPRS_VSRS(n,c,base) \
41 __SAVE_32FPRS_VSRS(n,__REG_##c,__REG_#
42 #define REST_32FPRS_VSRS(n,c,base) \
43 __REST_32FPRS_VSRS(n,__REG_##c,__REG_#
44
45 /* Stack frame offsets for local variables. */
46 #define TM_FRAME_L0 TM_FRAME_SIZE-16
47 #define TM_FRAME_L1 TM_FRAME_SIZE-8
48
49
50 /* In order to access the TM SPRs, TM must be
51 _GLOBAL(tm_enable)
52 mfmsr r4
53 li r3, MSR_TM >> 32
54 sldi r3, r3, 32
55 and. r0, r4, r3
56 bne 1f
57 or r4, r4, r3
58 mtmsrd r4
59 1: blr
60 EXPORT_SYMBOL_GPL(tm_enable);
61
62 _GLOBAL(tm_disable)
63 mfmsr r4
64 li r3, MSR_TM >> 32
65 sldi r3, r3, 32
66 andc r4, r4, r3
67 mtmsrd r4
68 blr
69 EXPORT_SYMBOL_GPL(tm_disable);
70
71 _GLOBAL(tm_save_sprs)
72 mfspr r0, SPRN_TFHAR
73 std r0, THREAD_TM_TFHAR(r3)
74 mfspr r0, SPRN_TEXASR
75 std r0, THREAD_TM_TEXASR(r3)
76 mfspr r0, SPRN_TFIAR
77 std r0, THREAD_TM_TFIAR(r3)
78 blr
79
80 _GLOBAL(tm_restore_sprs)
81 ld r0, THREAD_TM_TFHAR(r3)
82 mtspr SPRN_TFHAR, r0
83 ld r0, THREAD_TM_TEXASR(r3)
84 mtspr SPRN_TEXASR, r0
85 ld r0, THREAD_TM_TFIAR(r3)
86 mtspr SPRN_TFIAR, r0
87 blr
88
89 /* Passed an 8-bit failure cause as fi
90 _GLOBAL(tm_abort)
91 TABORT(R3)
92 blr
93 EXPORT_SYMBOL_GPL(tm_abort);
94
95 /*
96 * void tm_reclaim(struct thread_struct *threa
97 * uint8_t cause)
98 *
99 * - Performs a full reclaim. This destr
100 * transactions and updates thread.ckpt
101 * thread.ckvr_state with the original
102 * thread->regs is unchanged.
103 *
104 * Purpose is to both abort transactions of, a
105 * a transactions at a context switch. We pres
106 * state to restore them when the thread's sch
107 * userland as though nothing happened, but wh
108 * they will abort back to the checkpointed st
109 *
110 * Call with IRQs off, stacks get all out of s
111 */
112 _GLOBAL(tm_reclaim)
113 mfcr r5
114 mflr r0
115 stw r5, 8(r1)
116 std r0, 16(r1)
117 std r2, STK_GOT(r1)
118 stdu r1, -TM_FRAME_SIZE(r1)
119
120 /* We've a struct pt_regs at [r1+STACK
121
122 std r3, STK_PARAM(R3)(r1)
123 SAVE_NVGPRS(r1)
124
125 /*
126 * Save kernel live AMR since it will
127 * but can be used elsewhere later in
128 */
129 mfspr r3, SPRN_AMR
130 std r3, TM_FRAME_L1(r1)
131
132 /* We need to setup MSR for VSX regist
133 mfmsr r14
134 mr r15, r14
135 ori r15, r15, MSR_FP
136 li r16, 0
137 ori r16, r16, MSR_EE /* IRQs hard
138 andc r15, r15, r16
139 oris r15, r15, MSR_VEC@h
140 #ifdef CONFIG_VSX
141 BEGIN_FTR_SECTION
142 oris r15,r15, MSR_VSX@h
143 END_FTR_SECTION_IFSET(CPU_FTR_VSX)
144 #endif
145 mtmsrd r15
146 std r14, TM_FRAME_L0(r1)
147
148 /* Do sanity check on MSR to make sure
149 li r7, (MSR_TS_S)@higher
150 srdi r6, r14, 32
151 and r6, r6, r7
152 1: tdeqi r6, 0
153 EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,0
154
155 /* Stash the stack pointer away for us
156 std r1, PACAR1(r13)
157
158 /* Clear MSR RI since we are about to
159 li r5, 0
160 mtmsrd r5, 1
161
162 /*
163 * BE CAREFUL HERE:
164 * At this point we can't take an SLB
165 * off. Load only to/from the stack/pa
166 * until we turn MSR RI back on.
167 *
168 * The moment we treclaim, ALL of our
169 * to user register state. (FPRs, CCR
170 * Use an sprg and a tm_scratch in the
171 */
172 TRECLAIM(R4)
173
174 /*
175 * ******************** GPRs *********
176 * Stash the checkpointed r13 in the s
177 */
178 SET_SCRATCH0(r13)
179 GET_PACA(r13)
180
181 /*
182 * Stash the checkpointed r1 away in p
183 * stack pointer back into r1.
184 */
185 std r1, PACATMSCRATCH(r13)
186 ld r1, PACAR1(r13)
187
188 std r11, GPR11(r1)
189
190 /*
191 * Move the saved user r1 to the kerne
192 * clobbered by an exception once we t
193 */
194 ld r11, PACATMSCRATCH(r13)
195 std r11, GPR1(r1)
196
197 /*
198 * Store r13 away so we can free up th
199 * handler (needed once we start acces
200 */
201 GET_SCRATCH0(r11)
202 std r11, GPR13(r1)
203
204 /* Reset MSR RI so we can take SLB fau
205 li r11, MSR_RI
206 mtmsrd r11, 1
207
208 /* Store the PPR in r11 and reset to d
209 mfspr r11, SPRN_PPR
210 HMT_MEDIUM
211
212 /* Now get some more GPRS free */
213 std r7, GPR7(r1)
214 std r12, GPR12(r1)
215 ld r12, STK_PARAM(R3)(r1)
216
217 std r11, THREAD_TM_PPR(r12)
218
219 addi r7, r12, PT_CKPT_REGS
220
221 /*
222 * Make r7 look like an exception fram
223 * GPRx(n) macros. r7 is NOT a pt_regs
224 */
225 subi r7, r7, STACK_INT_FRAME_REGS
226
227 /* Sync the userland GPRs 2-12, 14-31
228 SAVE_GPR(0, r7)
229 SAVE_GPRS(2, 6, r7)
230 SAVE_GPRS(8, 10, r7)
231 ld r3, GPR1(r1)
232 ld r4, GPR7(r1)
233 ld r5, GPR11(r1)
234 ld r6, GPR12(r1)
235 ld r8, GPR13(r1)
236 std r3, GPR1(r7)
237 std r4, GPR7(r7)
238 std r5, GPR11(r7)
239 std r6, GPR12(r7)
240 std r8, GPR13(r7)
241
242 SAVE_NVGPRS(r7)
243
244 /* ******************** NIP **********
245 mfspr r3, SPRN_TFHAR
246 std r3, _NIP(r7)
247 /*
248 * The checkpointed NIP is ignored whe
249 * but is used in signal return to 'wi
250 */
251
252 /* ***************** CTR, LR, CR, XER
253 mfctr r3
254 mflr r4
255 mfcr r5
256 mfxer r6
257
258 std r3, _CTR(r7)
259 std r4, _LINK(r7)
260 std r5, _CCR(r7)
261 std r6, _XER(r7)
262
263 /* ******************** TAR, DSCR ****
264 mfspr r3, SPRN_TAR
265 mfspr r4, SPRN_DSCR
266
267 std r3, THREAD_TM_TAR(r12)
268 std r4, THREAD_TM_DSCR(r12)
269
270 /* ******************** AMR **********
271 mfspr r3, SPRN_AMR
272 std r3, THREAD_TM_AMR(r12)
273
274 /*
275 * MSR and flags: We don't change CRs,
276 */
277
278
279 /*
280 * ******************** FPR/VR/VSRs **
281 * After reclaiming, capture the check
282 *
283 * We enabled VEC/FP/VSX in the msr ab
284 * instructions!
285 */
286 mr r3, r12
287
288 /* Altivec (VEC/VMX/VR)*/
289 addi r7, r3, THREAD_CKVRSTATE
290 SAVE_32VRS(0, r6, r7) /* r6 scratch,
291 mfvscr v0
292 li r6, VRSTATE_VSCR
293 stvx v0, r7, r6
294
295 /* VRSAVE */
296 mfspr r0, SPRN_VRSAVE
297 std r0, THREAD_CKVRSAVE(r3)
298
299 /* Floating Point (FP) */
300 addi r7, r3, THREAD_CKFPSTATE
301 SAVE_32FPRS_VSRS(0, R6, R7) /* r6
302 mffs fr0
303 stfd fr0,FPSTATE_FPSCR(r7)
304
305
306 /*
307 * TM regs, incl TEXASR -- these live
308 * been updated by the treclaim, to ex
309 * cause (aborted).
310 */
311 mfspr r0, SPRN_TEXASR
312 mfspr r3, SPRN_TFHAR
313 mfspr r4, SPRN_TFIAR
314 std r0, THREAD_TM_TEXASR(r12)
315 std r3, THREAD_TM_TFHAR(r12)
316 std r4, THREAD_TM_TFIAR(r12)
317
318 /* Restore kernel live AMR */
319 ld r8, TM_FRAME_L1(r1)
320 mtspr SPRN_AMR, r8
321
322 /* Restore original MSR/IRQ state & cl
323 ld r14, TM_FRAME_L0(r1)
324
325 li r15, 0
326 rldimi r14, r15, MSR_TS_LG, (63-MSR_T
327 mtmsrd r14
328
329 REST_NVGPRS(r1)
330
331 addi r1, r1, TM_FRAME_SIZE
332 lwz r4, 8(r1)
333 ld r0, 16(r1)
334 mtcr r4
335 mtlr r0
336 ld r2, STK_GOT(r1)
337
338 /* Load CPU's default DSCR */
339 ld r0, PACA_DSCR_DEFAULT(r13)
340 mtspr SPRN_DSCR, r0
341
342 blr
343
344
345 /*
346 * void __tm_recheckpoint(struct threa
347 * - Restore the checkpointed reg
348 * when we switch_to a process.
349 *
350 * Call with IRQs off, stacks get
351 * some periods in here!
352 */
353 _GLOBAL(__tm_recheckpoint)
354 mfcr r5
355 mflr r0
356 stw r5, 8(r1)
357 std r0, 16(r1)
358 std r2, STK_GOT(r1)
359 stdu r1, -TM_FRAME_SIZE(r1)
360
361 /*
362 * We've a struct pt_regs at [r1+STACK
363 * This is used for backing up the NVG
364 */
365 SAVE_NVGPRS(r1)
366
367 /*
368 * Save kernel live AMR since it will
369 * but can be used elsewhere later in
370 */
371 mfspr r8, SPRN_AMR
372 std r8, TM_FRAME_L0(r1)
373
374 /* Load complete register state from t
375
376 addi r7, r3, PT_CKPT_REGS
377
378 /*
379 * Make r7 look like an exception fram
380 * GPRx(n) macros. r7 is now NOT a pt_
381 */
382 subi r7, r7, STACK_INT_FRAME_REGS
383
384 /* We need to setup MSR for FP/VMX/VSX
385 mfmsr r6
386 mr r5, r6
387 ori r5, r5, MSR_FP
388 #ifdef CONFIG_ALTIVEC
389 oris r5, r5, MSR_VEC@h
390 #endif
391 #ifdef CONFIG_VSX
392 BEGIN_FTR_SECTION
393 oris r5,r5, MSR_VSX@h
394 END_FTR_SECTION_IFSET(CPU_FTR_VSX)
395 #endif
396 mtmsrd r5
397
398 #ifdef CONFIG_ALTIVEC
399 /*
400 * FP and VEC registers: These are rec
401 * thread.ckfp_state and thread.ckvr_s
402 * thread.fp_state[] version holds the
403 * and will be loaded subsequently by
404 */
405 addi r8, r3, THREAD_CKVRSTATE
406 li r5, VRSTATE_VSCR
407 lvx v0, r8, r5
408 mtvscr v0
409 REST_32VRS(0, r5, r8)
410 ld r5, THREAD_CKVRSAVE(r3)
411 mtspr SPRN_VRSAVE, r5
412 #endif
413
414 addi r8, r3, THREAD_CKFPSTATE
415 lfd fr0, FPSTATE_FPSCR(r8)
416 MTFSF_L(fr0)
417 REST_32FPRS_VSRS(0, R4, R8)
418
419 mtmsr r6
420
421 restore_gprs:
422
423 /* ****************** CTR, LR, XER ***
424 ld r4, _CTR(r7)
425 ld r5, _LINK(r7)
426 ld r8, _XER(r7)
427
428 mtctr r4
429 mtlr r5
430 mtxer r8
431
432 /* ******************** TAR **********
433 ld r4, THREAD_TM_TAR(r3)
434 mtspr SPRN_TAR, r4
435
436 /* ******************** AMR **********
437 ld r4, THREAD_TM_AMR(r3)
438 mtspr SPRN_AMR, r4
439
440 /* Load up the PPR and DSCR in GPRs on
441 ld r5, THREAD_TM_DSCR(r3)
442 ld r6, THREAD_TM_PPR(r3)
443
444 REST_GPR(0, r7)
445 REST_GPRS(2, 4, r7)
446 REST_GPRS(8, 12, r7)
447 REST_GPRS(14, 31, r7)
448
449 /* Load up PPR and DSCR here so we don
450 mtspr SPRN_DSCR, r5
451 mtspr SPRN_PPR, r6
452
453 /*
454 * Do final sanity check on TEXASR to
455 * here before we load up the userspac
456 * a call chain.
457 */
458 mfspr r5, SPRN_TEXASR
459 srdi r5, r5, 16
460 li r6, (TEXASR_FS)@h
461 and r6, r6, r5
462 1: tdeqi r6, 0
463 EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,0
464
465 /*
466 * Do final sanity check on MSR to mak
467 * or suspended.
468 */
469 mfmsr r6
470 li r5, (MSR_TS_MASK)@higher
471 srdi r6, r6, 32
472 and r6, r6, r5
473 1: tdnei r6, 0
474 EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,0
475
476 /* Restore CR */
477 ld r6, _CCR(r7)
478 mtcr r6
479
480 REST_GPR(6, r7)
481
482 /*
483 * Store user r1 and r5 and r13 on the
484 * areas / compiler reserved areas), s
485 * we clear MSR RI.
486 */
487
488 REST_GPR(5, r7)
489 std r5, -8(r1)
490 ld r5, GPR13(r7)
491 std r5, -16(r1)
492 ld r5, GPR1(r7)
493 std r5, -24(r1)
494
495 REST_GPR(7, r7)
496
497 /* Stash the stack pointer away for us
498 std r1, PACAR1(r13)
499
500 /* Clear MSR RI since we are about to
501 li r5, 0
502 mtmsrd r5, 1
503
504 /*
505 * BE CAREFUL HERE:
506 * At this point we can't take an SLB
507 * off. Load only to/from the stack/pa
508 * until we turn MSR RI back on.
509 */
510
511 ld r5, -8(r1)
512 ld r13, -16(r1)
513 ld r1, -24(r1)
514
515 /* Commit register state as checkpoint
516 TRECHKPT
517
518 HMT_MEDIUM
519
520 /*
521 * Our transactional state has now cha
522 *
523 * Now just get out of here. Transact
524 * updated once restore is called on t
525 * -to process.
526 */
527
528 GET_PACA(r13)
529 ld r1, PACAR1(r13)
530
531 /* R13, R1 is restored, so we are reco
532 li r4, MSR_RI
533 mtmsrd r4, 1
534
535 /* Restore kernel live AMR */
536 ld r8, TM_FRAME_L0(r1)
537 mtspr SPRN_AMR, r8
538
539 REST_NVGPRS(r1)
540
541 addi r1, r1, TM_FRAME_SIZE
542 lwz r4, 8(r1)
543 ld r0, 16(r1)
544 mtcr r4
545 mtlr r0
546 ld r2, STK_GOT(r1)
547
548 /* Load CPU's default DSCR */
549 ld r0, PACA_DSCR_DEFAULT(r13)
550 mtspr SPRN_DSCR, r0
551
552 blr
553
554 /* ***********************************
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.