1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * spu_switch.c
4 *
5 * (C) Copyright IBM Corp. 2005
6 *
7 * Author: Mark Nutter <mnutter@us.ibm.com>
8 *
9 * Host-side part of SPU context switch sequence outlined in
10 * Synergistic Processor Element, Book IV.
11 *
12 * A fully premptive switch of an SPE is very expensive in terms
13 * of time and system resources. SPE Book IV indicates that SPE
14 * allocation should follow a "serially reusable device" model,
15 * in which the SPE is assigned a task until it completes. When
16 * this is not possible, this sequence may be used to premptively
17 * save, and then later (optionally) restore the context of a
18 * program executing on an SPE.
19 */
20
21 #include <linux/export.h>
22 #include <linux/errno.h>
23 #include <linux/hardirq.h>
24 #include <linux/sched.h>
25 #include <linux/kernel.h>
26 #include <linux/mm.h>
27 #include <linux/vmalloc.h>
28 #include <linux/smp.h>
29 #include <linux/stddef.h>
30 #include <linux/unistd.h>
31
32 #include <asm/io.h>
33 #include <asm/spu.h>
34 #include <asm/spu_priv1.h>
35 #include <asm/spu_csa.h>
36 #include <asm/mmu_context.h>
37
38 #include "spufs.h"
39
40 #include "spu_save_dump.h"
41 #include "spu_restore_dump.h"
42
43 #if 0
44 #define POLL_WHILE_TRUE(_c) { \
45 do { \
46 } while (_c); \
47 }
48 #else
49 #define RELAX_SPIN_COUNT 1000
50 #define POLL_WHILE_TRUE(_c) { \
51 do { \
52 int _i; \
53 for (_i=0; _i<RELAX_SPIN_COUNT && (_c); _i++) { \
54 cpu_relax(); \
55 } \
56 if (unlikely(_c)) yield(); \
57 else break; \
58 } while (_c); \
59 }
60 #endif /* debug */
61
62 #define POLL_WHILE_FALSE(_c) POLL_WHILE_TRUE(!(_c))
63
64 static inline void acquire_spu_lock(struct spu *spu)
65 {
66 /* Save, Step 1:
67 * Restore, Step 1:
68 * Acquire SPU-specific mutual exclusion lock.
69 * TBD.
70 */
71 }
72
73 static inline void release_spu_lock(struct spu *spu)
74 {
75 /* Restore, Step 76:
76 * Release SPU-specific mutual exclusion lock.
77 * TBD.
78 */
79 }
80
81 static inline int check_spu_isolate(struct spu_state *csa, struct spu *spu)
82 {
83 struct spu_problem __iomem *prob = spu->problem;
84 u32 isolate_state;
85
86 /* Save, Step 2:
87 * Save, Step 6:
88 * If SPU_Status[E,L,IS] any field is '1', this
89 * SPU is in isolate state and cannot be context
90 * saved at this time.
91 */
92 isolate_state = SPU_STATUS_ISOLATED_STATE |
93 SPU_STATUS_ISOLATED_LOAD_STATUS | SPU_STATUS_ISOLATED_EXIT_STATUS;
94 return (in_be32(&prob->spu_status_R) & isolate_state) ? 1 : 0;
95 }
96
97 static inline void disable_interrupts(struct spu_state *csa, struct spu *spu)
98 {
99 /* Save, Step 3:
100 * Restore, Step 2:
101 * Save INT_Mask_class0 in CSA.
102 * Write INT_MASK_class0 with value of 0.
103 * Save INT_Mask_class1 in CSA.
104 * Write INT_MASK_class1 with value of 0.
105 * Save INT_Mask_class2 in CSA.
106 * Write INT_MASK_class2 with value of 0.
107 * Synchronize all three interrupts to be sure
108 * we no longer execute a handler on another CPU.
109 */
110 spin_lock_irq(&spu->register_lock);
111 if (csa) {
112 csa->priv1.int_mask_class0_RW = spu_int_mask_get(spu, 0);
113 csa->priv1.int_mask_class1_RW = spu_int_mask_get(spu, 1);
114 csa->priv1.int_mask_class2_RW = spu_int_mask_get(spu, 2);
115 }
116 spu_int_mask_set(spu, 0, 0ul);
117 spu_int_mask_set(spu, 1, 0ul);
118 spu_int_mask_set(spu, 2, 0ul);
119 eieio();
120 spin_unlock_irq(&spu->register_lock);
121
122 /*
123 * This flag needs to be set before calling synchronize_irq so
124 * that the update will be visible to the relevant handlers
125 * via a simple load.
126 */
127 set_bit(SPU_CONTEXT_SWITCH_PENDING, &spu->flags);
128 clear_bit(SPU_CONTEXT_FAULT_PENDING, &spu->flags);
129 synchronize_irq(spu->irqs[0]);
130 synchronize_irq(spu->irqs[1]);
131 synchronize_irq(spu->irqs[2]);
132 }
133
134 static inline void set_watchdog_timer(struct spu_state *csa, struct spu *spu)
135 {
136 /* Save, Step 4:
137 * Restore, Step 25.
138 * Set a software watchdog timer, which specifies the
139 * maximum allowable time for a context save sequence.
140 *
141 * For present, this implementation will not set a global
142 * watchdog timer, as virtualization & variable system load
143 * may cause unpredictable execution times.
144 */
145 }
146
147 static inline void inhibit_user_access(struct spu_state *csa, struct spu *spu)
148 {
149 /* Save, Step 5:
150 * Restore, Step 3:
151 * Inhibit user-space access (if provided) to this
152 * SPU by unmapping the virtual pages assigned to
153 * the SPU memory-mapped I/O (MMIO) for problem
154 * state. TBD.
155 */
156 }
157
158 static inline void set_switch_pending(struct spu_state *csa, struct spu *spu)
159 {
160 /* Save, Step 7:
161 * Restore, Step 5:
162 * Set a software context switch pending flag.
163 * Done above in Step 3 - disable_interrupts().
164 */
165 }
166
167 static inline void save_mfc_cntl(struct spu_state *csa, struct spu *spu)
168 {
169 struct spu_priv2 __iomem *priv2 = spu->priv2;
170
171 /* Save, Step 8:
172 * Suspend DMA and save MFC_CNTL.
173 */
174 switch (in_be64(&priv2->mfc_control_RW) &
175 MFC_CNTL_SUSPEND_DMA_STATUS_MASK) {
176 case MFC_CNTL_SUSPEND_IN_PROGRESS:
177 POLL_WHILE_FALSE((in_be64(&priv2->mfc_control_RW) &
178 MFC_CNTL_SUSPEND_DMA_STATUS_MASK) ==
179 MFC_CNTL_SUSPEND_COMPLETE);
180 fallthrough;
181 case MFC_CNTL_SUSPEND_COMPLETE:
182 if (csa)
183 csa->priv2.mfc_control_RW =
184 in_be64(&priv2->mfc_control_RW) |
185 MFC_CNTL_SUSPEND_DMA_QUEUE;
186 break;
187 case MFC_CNTL_NORMAL_DMA_QUEUE_OPERATION:
188 out_be64(&priv2->mfc_control_RW, MFC_CNTL_SUSPEND_DMA_QUEUE);
189 POLL_WHILE_FALSE((in_be64(&priv2->mfc_control_RW) &
190 MFC_CNTL_SUSPEND_DMA_STATUS_MASK) ==
191 MFC_CNTL_SUSPEND_COMPLETE);
192 if (csa)
193 csa->priv2.mfc_control_RW =
194 in_be64(&priv2->mfc_control_RW) &
195 ~MFC_CNTL_SUSPEND_DMA_QUEUE &
196 ~MFC_CNTL_SUSPEND_MASK;
197 break;
198 }
199 }
200
201 static inline void save_spu_runcntl(struct spu_state *csa, struct spu *spu)
202 {
203 struct spu_problem __iomem *prob = spu->problem;
204
205 /* Save, Step 9:
206 * Save SPU_Runcntl in the CSA. This value contains
207 * the "Application Desired State".
208 */
209 csa->prob.spu_runcntl_RW = in_be32(&prob->spu_runcntl_RW);
210 }
211
212 static inline void save_mfc_sr1(struct spu_state *csa, struct spu *spu)
213 {
214 /* Save, Step 10:
215 * Save MFC_SR1 in the CSA.
216 */
217 csa->priv1.mfc_sr1_RW = spu_mfc_sr1_get(spu);
218 }
219
220 static inline void save_spu_status(struct spu_state *csa, struct spu *spu)
221 {
222 struct spu_problem __iomem *prob = spu->problem;
223
224 /* Save, Step 11:
225 * Read SPU_Status[R], and save to CSA.
226 */
227 if ((in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING) == 0) {
228 csa->prob.spu_status_R = in_be32(&prob->spu_status_R);
229 } else {
230 u32 stopped;
231
232 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP);
233 eieio();
234 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
235 SPU_STATUS_RUNNING);
236 stopped =
237 SPU_STATUS_INVALID_INSTR | SPU_STATUS_SINGLE_STEP |
238 SPU_STATUS_STOPPED_BY_HALT | SPU_STATUS_STOPPED_BY_STOP;
239 if ((in_be32(&prob->spu_status_R) & stopped) == 0)
240 csa->prob.spu_status_R = SPU_STATUS_RUNNING;
241 else
242 csa->prob.spu_status_R = in_be32(&prob->spu_status_R);
243 }
244 }
245
246 static inline void save_mfc_stopped_status(struct spu_state *csa,
247 struct spu *spu)
248 {
249 struct spu_priv2 __iomem *priv2 = spu->priv2;
250 const u64 mask = MFC_CNTL_DECREMENTER_RUNNING |
251 MFC_CNTL_DMA_QUEUES_EMPTY;
252
253 /* Save, Step 12:
254 * Read MFC_CNTL[Ds]. Update saved copy of
255 * CSA.MFC_CNTL[Ds].
256 *
257 * update: do the same with MFC_CNTL[Q].
258 */
259 csa->priv2.mfc_control_RW &= ~mask;
260 csa->priv2.mfc_control_RW |= in_be64(&priv2->mfc_control_RW) & mask;
261 }
262
263 static inline void halt_mfc_decr(struct spu_state *csa, struct spu *spu)
264 {
265 struct spu_priv2 __iomem *priv2 = spu->priv2;
266
267 /* Save, Step 13:
268 * Write MFC_CNTL[Dh] set to a '1' to halt
269 * the decrementer.
270 */
271 out_be64(&priv2->mfc_control_RW,
272 MFC_CNTL_DECREMENTER_HALTED | MFC_CNTL_SUSPEND_MASK);
273 eieio();
274 }
275
276 static inline void save_timebase(struct spu_state *csa, struct spu *spu)
277 {
278 /* Save, Step 14:
279 * Read PPE Timebase High and Timebase low registers
280 * and save in CSA. TBD.
281 */
282 csa->suspend_time = get_cycles();
283 }
284
285 static inline void remove_other_spu_access(struct spu_state *csa,
286 struct spu *spu)
287 {
288 /* Save, Step 15:
289 * Remove other SPU access to this SPU by unmapping
290 * this SPU's pages from their address space. TBD.
291 */
292 }
293
294 static inline void do_mfc_mssync(struct spu_state *csa, struct spu *spu)
295 {
296 struct spu_problem __iomem *prob = spu->problem;
297
298 /* Save, Step 16:
299 * Restore, Step 11.
300 * Write SPU_MSSync register. Poll SPU_MSSync[P]
301 * for a value of 0.
302 */
303 out_be64(&prob->spc_mssync_RW, 1UL);
304 POLL_WHILE_TRUE(in_be64(&prob->spc_mssync_RW) & MS_SYNC_PENDING);
305 }
306
307 static inline void issue_mfc_tlbie(struct spu_state *csa, struct spu *spu)
308 {
309 /* Save, Step 17:
310 * Restore, Step 12.
311 * Restore, Step 48.
312 * Write TLB_Invalidate_Entry[IS,VPN,L,Lp]=0 register.
313 * Then issue a PPE sync instruction.
314 */
315 spu_tlb_invalidate(spu);
316 mb();
317 }
318
319 static inline void handle_pending_interrupts(struct spu_state *csa,
320 struct spu *spu)
321 {
322 /* Save, Step 18:
323 * Handle any pending interrupts from this SPU
324 * here. This is OS or hypervisor specific. One
325 * option is to re-enable interrupts to handle any
326 * pending interrupts, with the interrupt handlers
327 * recognizing the software Context Switch Pending
328 * flag, to ensure the SPU execution or MFC command
329 * queue is not restarted. TBD.
330 */
331 }
332
333 static inline void save_mfc_queues(struct spu_state *csa, struct spu *spu)
334 {
335 struct spu_priv2 __iomem *priv2 = spu->priv2;
336 int i;
337
338 /* Save, Step 19:
339 * If MFC_Cntl[Se]=0 then save
340 * MFC command queues.
341 */
342 if ((in_be64(&priv2->mfc_control_RW) & MFC_CNTL_DMA_QUEUES_EMPTY) == 0) {
343 for (i = 0; i < 8; i++) {
344 csa->priv2.puq[i].mfc_cq_data0_RW =
345 in_be64(&priv2->puq[i].mfc_cq_data0_RW);
346 csa->priv2.puq[i].mfc_cq_data1_RW =
347 in_be64(&priv2->puq[i].mfc_cq_data1_RW);
348 csa->priv2.puq[i].mfc_cq_data2_RW =
349 in_be64(&priv2->puq[i].mfc_cq_data2_RW);
350 csa->priv2.puq[i].mfc_cq_data3_RW =
351 in_be64(&priv2->puq[i].mfc_cq_data3_RW);
352 }
353 for (i = 0; i < 16; i++) {
354 csa->priv2.spuq[i].mfc_cq_data0_RW =
355 in_be64(&priv2->spuq[i].mfc_cq_data0_RW);
356 csa->priv2.spuq[i].mfc_cq_data1_RW =
357 in_be64(&priv2->spuq[i].mfc_cq_data1_RW);
358 csa->priv2.spuq[i].mfc_cq_data2_RW =
359 in_be64(&priv2->spuq[i].mfc_cq_data2_RW);
360 csa->priv2.spuq[i].mfc_cq_data3_RW =
361 in_be64(&priv2->spuq[i].mfc_cq_data3_RW);
362 }
363 }
364 }
365
366 static inline void save_ppu_querymask(struct spu_state *csa, struct spu *spu)
367 {
368 struct spu_problem __iomem *prob = spu->problem;
369
370 /* Save, Step 20:
371 * Save the PPU_QueryMask register
372 * in the CSA.
373 */
374 csa->prob.dma_querymask_RW = in_be32(&prob->dma_querymask_RW);
375 }
376
377 static inline void save_ppu_querytype(struct spu_state *csa, struct spu *spu)
378 {
379 struct spu_problem __iomem *prob = spu->problem;
380
381 /* Save, Step 21:
382 * Save the PPU_QueryType register
383 * in the CSA.
384 */
385 csa->prob.dma_querytype_RW = in_be32(&prob->dma_querytype_RW);
386 }
387
388 static inline void save_ppu_tagstatus(struct spu_state *csa, struct spu *spu)
389 {
390 struct spu_problem __iomem *prob = spu->problem;
391
392 /* Save the Prxy_TagStatus register in the CSA.
393 *
394 * It is unnecessary to restore dma_tagstatus_R, however,
395 * dma_tagstatus_R in the CSA is accessed via backing_ops, so
396 * we must save it.
397 */
398 csa->prob.dma_tagstatus_R = in_be32(&prob->dma_tagstatus_R);
399 }
400
401 static inline void save_mfc_csr_tsq(struct spu_state *csa, struct spu *spu)
402 {
403 struct spu_priv2 __iomem *priv2 = spu->priv2;
404
405 /* Save, Step 22:
406 * Save the MFC_CSR_TSQ register
407 * in the LSCSA.
408 */
409 csa->priv2.spu_tag_status_query_RW =
410 in_be64(&priv2->spu_tag_status_query_RW);
411 }
412
413 static inline void save_mfc_csr_cmd(struct spu_state *csa, struct spu *spu)
414 {
415 struct spu_priv2 __iomem *priv2 = spu->priv2;
416
417 /* Save, Step 23:
418 * Save the MFC_CSR_CMD1 and MFC_CSR_CMD2
419 * registers in the CSA.
420 */
421 csa->priv2.spu_cmd_buf1_RW = in_be64(&priv2->spu_cmd_buf1_RW);
422 csa->priv2.spu_cmd_buf2_RW = in_be64(&priv2->spu_cmd_buf2_RW);
423 }
424
425 static inline void save_mfc_csr_ato(struct spu_state *csa, struct spu *spu)
426 {
427 struct spu_priv2 __iomem *priv2 = spu->priv2;
428
429 /* Save, Step 24:
430 * Save the MFC_CSR_ATO register in
431 * the CSA.
432 */
433 csa->priv2.spu_atomic_status_RW = in_be64(&priv2->spu_atomic_status_RW);
434 }
435
436 static inline void save_mfc_tclass_id(struct spu_state *csa, struct spu *spu)
437 {
438 /* Save, Step 25:
439 * Save the MFC_TCLASS_ID register in
440 * the CSA.
441 */
442 csa->priv1.mfc_tclass_id_RW = spu_mfc_tclass_id_get(spu);
443 }
444
445 static inline void set_mfc_tclass_id(struct spu_state *csa, struct spu *spu)
446 {
447 /* Save, Step 26:
448 * Restore, Step 23.
449 * Write the MFC_TCLASS_ID register with
450 * the value 0x10000000.
451 */
452 spu_mfc_tclass_id_set(spu, 0x10000000);
453 eieio();
454 }
455
456 static inline void purge_mfc_queue(struct spu_state *csa, struct spu *spu)
457 {
458 struct spu_priv2 __iomem *priv2 = spu->priv2;
459
460 /* Save, Step 27:
461 * Restore, Step 14.
462 * Write MFC_CNTL[Pc]=1 (purge queue).
463 */
464 out_be64(&priv2->mfc_control_RW,
465 MFC_CNTL_PURGE_DMA_REQUEST |
466 MFC_CNTL_SUSPEND_MASK);
467 eieio();
468 }
469
470 static inline void wait_purge_complete(struct spu_state *csa, struct spu *spu)
471 {
472 struct spu_priv2 __iomem *priv2 = spu->priv2;
473
474 /* Save, Step 28:
475 * Poll MFC_CNTL[Ps] until value '11' is read
476 * (purge complete).
477 */
478 POLL_WHILE_FALSE((in_be64(&priv2->mfc_control_RW) &
479 MFC_CNTL_PURGE_DMA_STATUS_MASK) ==
480 MFC_CNTL_PURGE_DMA_COMPLETE);
481 }
482
483 static inline void setup_mfc_sr1(struct spu_state *csa, struct spu *spu)
484 {
485 /* Save, Step 30:
486 * Restore, Step 18:
487 * Write MFC_SR1 with MFC_SR1[D=0,S=1] and
488 * MFC_SR1[TL,R,Pr,T] set correctly for the
489 * OS specific environment.
490 *
491 * Implementation note: The SPU-side code
492 * for save/restore is privileged, so the
493 * MFC_SR1[Pr] bit is not set.
494 *
495 */
496 spu_mfc_sr1_set(spu, (MFC_STATE1_MASTER_RUN_CONTROL_MASK |
497 MFC_STATE1_RELOCATE_MASK |
498 MFC_STATE1_BUS_TLBIE_MASK));
499 }
500
501 static inline void save_spu_npc(struct spu_state *csa, struct spu *spu)
502 {
503 struct spu_problem __iomem *prob = spu->problem;
504
505 /* Save, Step 31:
506 * Save SPU_NPC in the CSA.
507 */
508 csa->prob.spu_npc_RW = in_be32(&prob->spu_npc_RW);
509 }
510
511 static inline void save_spu_privcntl(struct spu_state *csa, struct spu *spu)
512 {
513 struct spu_priv2 __iomem *priv2 = spu->priv2;
514
515 /* Save, Step 32:
516 * Save SPU_PrivCntl in the CSA.
517 */
518 csa->priv2.spu_privcntl_RW = in_be64(&priv2->spu_privcntl_RW);
519 }
520
521 static inline void reset_spu_privcntl(struct spu_state *csa, struct spu *spu)
522 {
523 struct spu_priv2 __iomem *priv2 = spu->priv2;
524
525 /* Save, Step 33:
526 * Restore, Step 16:
527 * Write SPU_PrivCntl[S,Le,A] fields reset to 0.
528 */
529 out_be64(&priv2->spu_privcntl_RW, 0UL);
530 eieio();
531 }
532
533 static inline void save_spu_lslr(struct spu_state *csa, struct spu *spu)
534 {
535 struct spu_priv2 __iomem *priv2 = spu->priv2;
536
537 /* Save, Step 34:
538 * Save SPU_LSLR in the CSA.
539 */
540 csa->priv2.spu_lslr_RW = in_be64(&priv2->spu_lslr_RW);
541 }
542
543 static inline void reset_spu_lslr(struct spu_state *csa, struct spu *spu)
544 {
545 struct spu_priv2 __iomem *priv2 = spu->priv2;
546
547 /* Save, Step 35:
548 * Restore, Step 17.
549 * Reset SPU_LSLR.
550 */
551 out_be64(&priv2->spu_lslr_RW, LS_ADDR_MASK);
552 eieio();
553 }
554
555 static inline void save_spu_cfg(struct spu_state *csa, struct spu *spu)
556 {
557 struct spu_priv2 __iomem *priv2 = spu->priv2;
558
559 /* Save, Step 36:
560 * Save SPU_Cfg in the CSA.
561 */
562 csa->priv2.spu_cfg_RW = in_be64(&priv2->spu_cfg_RW);
563 }
564
565 static inline void save_pm_trace(struct spu_state *csa, struct spu *spu)
566 {
567 /* Save, Step 37:
568 * Save PM_Trace_Tag_Wait_Mask in the CSA.
569 * Not performed by this implementation.
570 */
571 }
572
573 static inline void save_mfc_rag(struct spu_state *csa, struct spu *spu)
574 {
575 /* Save, Step 38:
576 * Save RA_GROUP_ID register and the
577 * RA_ENABLE reigster in the CSA.
578 */
579 csa->priv1.resource_allocation_groupID_RW =
580 spu_resource_allocation_groupID_get(spu);
581 csa->priv1.resource_allocation_enable_RW =
582 spu_resource_allocation_enable_get(spu);
583 }
584
585 static inline void save_ppu_mb_stat(struct spu_state *csa, struct spu *spu)
586 {
587 struct spu_problem __iomem *prob = spu->problem;
588
589 /* Save, Step 39:
590 * Save MB_Stat register in the CSA.
591 */
592 csa->prob.mb_stat_R = in_be32(&prob->mb_stat_R);
593 }
594
595 static inline void save_ppu_mb(struct spu_state *csa, struct spu *spu)
596 {
597 struct spu_problem __iomem *prob = spu->problem;
598
599 /* Save, Step 40:
600 * Save the PPU_MB register in the CSA.
601 */
602 csa->prob.pu_mb_R = in_be32(&prob->pu_mb_R);
603 }
604
605 static inline void save_ppuint_mb(struct spu_state *csa, struct spu *spu)
606 {
607 struct spu_priv2 __iomem *priv2 = spu->priv2;
608
609 /* Save, Step 41:
610 * Save the PPUINT_MB register in the CSA.
611 */
612 csa->priv2.puint_mb_R = in_be64(&priv2->puint_mb_R);
613 }
614
615 static inline void save_ch_part1(struct spu_state *csa, struct spu *spu)
616 {
617 struct spu_priv2 __iomem *priv2 = spu->priv2;
618 u64 idx, ch_indices[] = { 0UL, 3UL, 4UL, 24UL, 25UL, 27UL };
619 int i;
620
621 /* Save, Step 42:
622 */
623
624 /* Save CH 1, without channel count */
625 out_be64(&priv2->spu_chnlcntptr_RW, 1);
626 csa->spu_chnldata_RW[1] = in_be64(&priv2->spu_chnldata_RW);
627
628 /* Save the following CH: [0,3,4,24,25,27] */
629 for (i = 0; i < ARRAY_SIZE(ch_indices); i++) {
630 idx = ch_indices[i];
631 out_be64(&priv2->spu_chnlcntptr_RW, idx);
632 eieio();
633 csa->spu_chnldata_RW[idx] = in_be64(&priv2->spu_chnldata_RW);
634 csa->spu_chnlcnt_RW[idx] = in_be64(&priv2->spu_chnlcnt_RW);
635 out_be64(&priv2->spu_chnldata_RW, 0UL);
636 out_be64(&priv2->spu_chnlcnt_RW, 0UL);
637 eieio();
638 }
639 }
640
641 static inline void save_spu_mb(struct spu_state *csa, struct spu *spu)
642 {
643 struct spu_priv2 __iomem *priv2 = spu->priv2;
644 int i;
645
646 /* Save, Step 43:
647 * Save SPU Read Mailbox Channel.
648 */
649 out_be64(&priv2->spu_chnlcntptr_RW, 29UL);
650 eieio();
651 csa->spu_chnlcnt_RW[29] = in_be64(&priv2->spu_chnlcnt_RW);
652 for (i = 0; i < 4; i++) {
653 csa->spu_mailbox_data[i] = in_be64(&priv2->spu_chnldata_RW);
654 }
655 out_be64(&priv2->spu_chnlcnt_RW, 0UL);
656 eieio();
657 }
658
659 static inline void save_mfc_cmd(struct spu_state *csa, struct spu *spu)
660 {
661 struct spu_priv2 __iomem *priv2 = spu->priv2;
662
663 /* Save, Step 44:
664 * Save MFC_CMD Channel.
665 */
666 out_be64(&priv2->spu_chnlcntptr_RW, 21UL);
667 eieio();
668 csa->spu_chnlcnt_RW[21] = in_be64(&priv2->spu_chnlcnt_RW);
669 eieio();
670 }
671
672 static inline void reset_ch(struct spu_state *csa, struct spu *spu)
673 {
674 struct spu_priv2 __iomem *priv2 = spu->priv2;
675 u64 ch_indices[4] = { 21UL, 23UL, 28UL, 30UL };
676 u64 ch_counts[4] = { 16UL, 1UL, 1UL, 1UL };
677 u64 idx;
678 int i;
679
680 /* Save, Step 45:
681 * Reset the following CH: [21, 23, 28, 30]
682 */
683 for (i = 0; i < 4; i++) {
684 idx = ch_indices[i];
685 out_be64(&priv2->spu_chnlcntptr_RW, idx);
686 eieio();
687 out_be64(&priv2->spu_chnlcnt_RW, ch_counts[i]);
688 eieio();
689 }
690 }
691
692 static inline void resume_mfc_queue(struct spu_state *csa, struct spu *spu)
693 {
694 struct spu_priv2 __iomem *priv2 = spu->priv2;
695
696 /* Save, Step 46:
697 * Restore, Step 25.
698 * Write MFC_CNTL[Sc]=0 (resume queue processing).
699 */
700 out_be64(&priv2->mfc_control_RW, MFC_CNTL_RESUME_DMA_QUEUE);
701 }
702
703 static inline void setup_mfc_slbs(struct spu_state *csa, struct spu *spu,
704 unsigned int *code, int code_size)
705 {
706 /* Save, Step 47:
707 * Restore, Step 30.
708 * If MFC_SR1[R]=1, write 0 to SLB_Invalidate_All
709 * register, then initialize SLB_VSID and SLB_ESID
710 * to provide access to SPU context save code and
711 * LSCSA.
712 *
713 * This implementation places both the context
714 * switch code and LSCSA in kernel address space.
715 *
716 * Further this implementation assumes that the
717 * MFC_SR1[R]=1 (in other words, assume that
718 * translation is desired by OS environment).
719 */
720 spu_invalidate_slbs(spu);
721 spu_setup_kernel_slbs(spu, csa->lscsa, code, code_size);
722 }
723
724 static inline void set_switch_active(struct spu_state *csa, struct spu *spu)
725 {
726 /* Save, Step 48:
727 * Restore, Step 23.
728 * Change the software context switch pending flag
729 * to context switch active. This implementation does
730 * not uses a switch active flag.
731 *
732 * Now that we have saved the mfc in the csa, we can add in the
733 * restart command if an exception occurred.
734 */
735 if (test_bit(SPU_CONTEXT_FAULT_PENDING, &spu->flags))
736 csa->priv2.mfc_control_RW |= MFC_CNTL_RESTART_DMA_COMMAND;
737 clear_bit(SPU_CONTEXT_SWITCH_PENDING, &spu->flags);
738 mb();
739 }
740
741 static inline void enable_interrupts(struct spu_state *csa, struct spu *spu)
742 {
743 unsigned long class1_mask = CLASS1_ENABLE_SEGMENT_FAULT_INTR |
744 CLASS1_ENABLE_STORAGE_FAULT_INTR;
745
746 /* Save, Step 49:
747 * Restore, Step 22:
748 * Reset and then enable interrupts, as
749 * needed by OS.
750 *
751 * This implementation enables only class1
752 * (translation) interrupts.
753 */
754 spin_lock_irq(&spu->register_lock);
755 spu_int_stat_clear(spu, 0, CLASS0_INTR_MASK);
756 spu_int_stat_clear(spu, 1, CLASS1_INTR_MASK);
757 spu_int_stat_clear(spu, 2, CLASS2_INTR_MASK);
758 spu_int_mask_set(spu, 0, 0ul);
759 spu_int_mask_set(spu, 1, class1_mask);
760 spu_int_mask_set(spu, 2, 0ul);
761 spin_unlock_irq(&spu->register_lock);
762 }
763
764 static inline int send_mfc_dma(struct spu *spu, unsigned long ea,
765 unsigned int ls_offset, unsigned int size,
766 unsigned int tag, unsigned int rclass,
767 unsigned int cmd)
768 {
769 struct spu_problem __iomem *prob = spu->problem;
770 union mfc_tag_size_class_cmd command;
771 unsigned int transfer_size;
772 volatile unsigned int status = 0x0;
773
774 while (size > 0) {
775 transfer_size =
776 (size > MFC_MAX_DMA_SIZE) ? MFC_MAX_DMA_SIZE : size;
777 command.u.mfc_size = transfer_size;
778 command.u.mfc_tag = tag;
779 command.u.mfc_rclassid = rclass;
780 command.u.mfc_cmd = cmd;
781 do {
782 out_be32(&prob->mfc_lsa_W, ls_offset);
783 out_be64(&prob->mfc_ea_W, ea);
784 out_be64(&prob->mfc_union_W.all64, command.all64);
785 status =
786 in_be32(&prob->mfc_union_W.by32.mfc_class_cmd32);
787 if (unlikely(status & 0x2)) {
788 cpu_relax();
789 }
790 } while (status & 0x3);
791 size -= transfer_size;
792 ea += transfer_size;
793 ls_offset += transfer_size;
794 }
795 return 0;
796 }
797
798 static inline void save_ls_16kb(struct spu_state *csa, struct spu *spu)
799 {
800 unsigned long addr = (unsigned long)&csa->lscsa->ls[0];
801 unsigned int ls_offset = 0x0;
802 unsigned int size = 16384;
803 unsigned int tag = 0;
804 unsigned int rclass = 0;
805 unsigned int cmd = MFC_PUT_CMD;
806
807 /* Save, Step 50:
808 * Issue a DMA command to copy the first 16K bytes
809 * of local storage to the CSA.
810 */
811 send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd);
812 }
813
814 static inline void set_spu_npc(struct spu_state *csa, struct spu *spu)
815 {
816 struct spu_problem __iomem *prob = spu->problem;
817
818 /* Save, Step 51:
819 * Restore, Step 31.
820 * Write SPU_NPC[IE]=0 and SPU_NPC[LSA] to entry
821 * point address of context save code in local
822 * storage.
823 *
824 * This implementation uses SPU-side save/restore
825 * programs with entry points at LSA of 0.
826 */
827 out_be32(&prob->spu_npc_RW, 0);
828 eieio();
829 }
830
831 static inline void set_signot1(struct spu_state *csa, struct spu *spu)
832 {
833 struct spu_problem __iomem *prob = spu->problem;
834 union {
835 u64 ull;
836 u32 ui[2];
837 } addr64;
838
839 /* Save, Step 52:
840 * Restore, Step 32:
841 * Write SPU_Sig_Notify_1 register with upper 32-bits
842 * of the CSA.LSCSA effective address.
843 */
844 addr64.ull = (u64) csa->lscsa;
845 out_be32(&prob->signal_notify1, addr64.ui[0]);
846 eieio();
847 }
848
849 static inline void set_signot2(struct spu_state *csa, struct spu *spu)
850 {
851 struct spu_problem __iomem *prob = spu->problem;
852 union {
853 u64 ull;
854 u32 ui[2];
855 } addr64;
856
857 /* Save, Step 53:
858 * Restore, Step 33:
859 * Write SPU_Sig_Notify_2 register with lower 32-bits
860 * of the CSA.LSCSA effective address.
861 */
862 addr64.ull = (u64) csa->lscsa;
863 out_be32(&prob->signal_notify2, addr64.ui[1]);
864 eieio();
865 }
866
867 static inline void send_save_code(struct spu_state *csa, struct spu *spu)
868 {
869 unsigned long addr = (unsigned long)&spu_save_code[0];
870 unsigned int ls_offset = 0x0;
871 unsigned int size = sizeof(spu_save_code);
872 unsigned int tag = 0;
873 unsigned int rclass = 0;
874 unsigned int cmd = MFC_GETFS_CMD;
875
876 /* Save, Step 54:
877 * Issue a DMA command to copy context save code
878 * to local storage and start SPU.
879 */
880 send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd);
881 }
882
883 static inline void set_ppu_querymask(struct spu_state *csa, struct spu *spu)
884 {
885 struct spu_problem __iomem *prob = spu->problem;
886
887 /* Save, Step 55:
888 * Restore, Step 38.
889 * Write PPU_QueryMask=1 (enable Tag Group 0)
890 * and issue eieio instruction.
891 */
892 out_be32(&prob->dma_querymask_RW, MFC_TAGID_TO_TAGMASK(0));
893 eieio();
894 }
895
896 static inline void wait_tag_complete(struct spu_state *csa, struct spu *spu)
897 {
898 struct spu_problem __iomem *prob = spu->problem;
899 u32 mask = MFC_TAGID_TO_TAGMASK(0);
900 unsigned long flags;
901
902 /* Save, Step 56:
903 * Restore, Step 39.
904 * Restore, Step 39.
905 * Restore, Step 46.
906 * Poll PPU_TagStatus[gn] until 01 (Tag group 0 complete)
907 * or write PPU_QueryType[TS]=01 and wait for Tag Group
908 * Complete Interrupt. Write INT_Stat_Class0 or
909 * INT_Stat_Class2 with value of 'handled'.
910 */
911 POLL_WHILE_FALSE(in_be32(&prob->dma_tagstatus_R) & mask);
912
913 local_irq_save(flags);
914 spu_int_stat_clear(spu, 0, CLASS0_INTR_MASK);
915 spu_int_stat_clear(spu, 2, CLASS2_INTR_MASK);
916 local_irq_restore(flags);
917 }
918
919 static inline void wait_spu_stopped(struct spu_state *csa, struct spu *spu)
920 {
921 struct spu_problem __iomem *prob = spu->problem;
922 unsigned long flags;
923
924 /* Save, Step 57:
925 * Restore, Step 40.
926 * Poll until SPU_Status[R]=0 or wait for SPU Class 0
927 * or SPU Class 2 interrupt. Write INT_Stat_class0
928 * or INT_Stat_class2 with value of handled.
929 */
930 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING);
931
932 local_irq_save(flags);
933 spu_int_stat_clear(spu, 0, CLASS0_INTR_MASK);
934 spu_int_stat_clear(spu, 2, CLASS2_INTR_MASK);
935 local_irq_restore(flags);
936 }
937
938 static inline int check_save_status(struct spu_state *csa, struct spu *spu)
939 {
940 struct spu_problem __iomem *prob = spu->problem;
941 u32 complete;
942
943 /* Save, Step 54:
944 * If SPU_Status[P]=1 and SPU_Status[SC] = "success",
945 * context save succeeded, otherwise context save
946 * failed.
947 */
948 complete = ((SPU_SAVE_COMPLETE << SPU_STOP_STATUS_SHIFT) |
949 SPU_STATUS_STOPPED_BY_STOP);
950 return (in_be32(&prob->spu_status_R) != complete) ? 1 : 0;
951 }
952
953 static inline void terminate_spu_app(struct spu_state *csa, struct spu *spu)
954 {
955 /* Restore, Step 4:
956 * If required, notify the "using application" that
957 * the SPU task has been terminated. TBD.
958 */
959 }
960
961 static inline void suspend_mfc_and_halt_decr(struct spu_state *csa,
962 struct spu *spu)
963 {
964 struct spu_priv2 __iomem *priv2 = spu->priv2;
965
966 /* Restore, Step 7:
967 * Write MFC_Cntl[Dh,Sc,Sm]='1','1','' to suspend
968 * the queue and halt the decrementer.
969 */
970 out_be64(&priv2->mfc_control_RW, MFC_CNTL_SUSPEND_DMA_QUEUE |
971 MFC_CNTL_DECREMENTER_HALTED);
972 eieio();
973 }
974
975 static inline void wait_suspend_mfc_complete(struct spu_state *csa,
976 struct spu *spu)
977 {
978 struct spu_priv2 __iomem *priv2 = spu->priv2;
979
980 /* Restore, Step 8:
981 * Restore, Step 47.
982 * Poll MFC_CNTL[Ss] until 11 is returned.
983 */
984 POLL_WHILE_FALSE((in_be64(&priv2->mfc_control_RW) &
985 MFC_CNTL_SUSPEND_DMA_STATUS_MASK) ==
986 MFC_CNTL_SUSPEND_COMPLETE);
987 }
988
989 static inline int suspend_spe(struct spu_state *csa, struct spu *spu)
990 {
991 struct spu_problem __iomem *prob = spu->problem;
992
993 /* Restore, Step 9:
994 * If SPU_Status[R]=1, stop SPU execution
995 * and wait for stop to complete.
996 *
997 * Returns 1 if SPU_Status[R]=1 on entry.
998 * 0 otherwise
999 */
1000 if (in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING) {
1001 if (in_be32(&prob->spu_status_R) &
1002 SPU_STATUS_ISOLATED_EXIT_STATUS) {
1003 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
1004 SPU_STATUS_RUNNING);
1005 }
1006 if ((in_be32(&prob->spu_status_R) &
1007 SPU_STATUS_ISOLATED_LOAD_STATUS)
1008 || (in_be32(&prob->spu_status_R) &
1009 SPU_STATUS_ISOLATED_STATE)) {
1010 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP);
1011 eieio();
1012 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
1013 SPU_STATUS_RUNNING);
1014 out_be32(&prob->spu_runcntl_RW, 0x2);
1015 eieio();
1016 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
1017 SPU_STATUS_RUNNING);
1018 }
1019 if (in_be32(&prob->spu_status_R) &
1020 SPU_STATUS_WAITING_FOR_CHANNEL) {
1021 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP);
1022 eieio();
1023 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
1024 SPU_STATUS_RUNNING);
1025 }
1026 return 1;
1027 }
1028 return 0;
1029 }
1030
1031 static inline void clear_spu_status(struct spu_state *csa, struct spu *spu)
1032 {
1033 struct spu_problem __iomem *prob = spu->problem;
1034
1035 /* Restore, Step 10:
1036 * If SPU_Status[R]=0 and SPU_Status[E,L,IS]=1,
1037 * release SPU from isolate state.
1038 */
1039 if (!(in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING)) {
1040 if (in_be32(&prob->spu_status_R) &
1041 SPU_STATUS_ISOLATED_EXIT_STATUS) {
1042 spu_mfc_sr1_set(spu,
1043 MFC_STATE1_MASTER_RUN_CONTROL_MASK);
1044 eieio();
1045 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE);
1046 eieio();
1047 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
1048 SPU_STATUS_RUNNING);
1049 }
1050 if ((in_be32(&prob->spu_status_R) &
1051 SPU_STATUS_ISOLATED_LOAD_STATUS)
1052 || (in_be32(&prob->spu_status_R) &
1053 SPU_STATUS_ISOLATED_STATE)) {
1054 spu_mfc_sr1_set(spu,
1055 MFC_STATE1_MASTER_RUN_CONTROL_MASK);
1056 eieio();
1057 out_be32(&prob->spu_runcntl_RW, 0x2);
1058 eieio();
1059 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
1060 SPU_STATUS_RUNNING);
1061 }
1062 }
1063 }
1064
1065 static inline void reset_ch_part1(struct spu_state *csa, struct spu *spu)
1066 {
1067 struct spu_priv2 __iomem *priv2 = spu->priv2;
1068 u64 ch_indices[] = { 0UL, 3UL, 4UL, 24UL, 25UL, 27UL };
1069 u64 idx;
1070 int i;
1071
1072 /* Restore, Step 20:
1073 */
1074
1075 /* Reset CH 1 */
1076 out_be64(&priv2->spu_chnlcntptr_RW, 1);
1077 out_be64(&priv2->spu_chnldata_RW, 0UL);
1078
1079 /* Reset the following CH: [0,3,4,24,25,27] */
1080 for (i = 0; i < ARRAY_SIZE(ch_indices); i++) {
1081 idx = ch_indices[i];
1082 out_be64(&priv2->spu_chnlcntptr_RW, idx);
1083 eieio();
1084 out_be64(&priv2->spu_chnldata_RW, 0UL);
1085 out_be64(&priv2->spu_chnlcnt_RW, 0UL);
1086 eieio();
1087 }
1088 }
1089
1090 static inline void reset_ch_part2(struct spu_state *csa, struct spu *spu)
1091 {
1092 struct spu_priv2 __iomem *priv2 = spu->priv2;
1093 u64 ch_indices[5] = { 21UL, 23UL, 28UL, 29UL, 30UL };
1094 u64 ch_counts[5] = { 16UL, 1UL, 1UL, 0UL, 1UL };
1095 u64 idx;
1096 int i;
1097
1098 /* Restore, Step 21:
1099 * Reset the following CH: [21, 23, 28, 29, 30]
1100 */
1101 for (i = 0; i < 5; i++) {
1102 idx = ch_indices[i];
1103 out_be64(&priv2->spu_chnlcntptr_RW, idx);
1104 eieio();
1105 out_be64(&priv2->spu_chnlcnt_RW, ch_counts[i]);
1106 eieio();
1107 }
1108 }
1109
1110 static inline void setup_spu_status_part1(struct spu_state *csa,
1111 struct spu *spu)
1112 {
1113 u32 status_P = SPU_STATUS_STOPPED_BY_STOP;
1114 u32 status_I = SPU_STATUS_INVALID_INSTR;
1115 u32 status_H = SPU_STATUS_STOPPED_BY_HALT;
1116 u32 status_S = SPU_STATUS_SINGLE_STEP;
1117 u32 status_S_I = SPU_STATUS_SINGLE_STEP | SPU_STATUS_INVALID_INSTR;
1118 u32 status_S_P = SPU_STATUS_SINGLE_STEP | SPU_STATUS_STOPPED_BY_STOP;
1119 u32 status_P_H = SPU_STATUS_STOPPED_BY_HALT |SPU_STATUS_STOPPED_BY_STOP;
1120 u32 status_P_I = SPU_STATUS_STOPPED_BY_STOP |SPU_STATUS_INVALID_INSTR;
1121 u32 status_code;
1122
1123 /* Restore, Step 27:
1124 * If the CSA.SPU_Status[I,S,H,P]=1 then add the correct
1125 * instruction sequence to the end of the SPU based restore
1126 * code (after the "context restored" stop and signal) to
1127 * restore the correct SPU status.
1128 *
1129 * NOTE: Rather than modifying the SPU executable, we
1130 * instead add a new 'stopped_status' field to the
1131 * LSCSA. The SPU-side restore reads this field and
1132 * takes the appropriate action when exiting.
1133 */
1134
1135 status_code =
1136 (csa->prob.spu_status_R >> SPU_STOP_STATUS_SHIFT) & 0xFFFF;
1137 if ((csa->prob.spu_status_R & status_P_I) == status_P_I) {
1138
1139 /* SPU_Status[P,I]=1 - Illegal Instruction followed
1140 * by Stop and Signal instruction, followed by 'br -4'.
1141 *
1142 */
1143 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_P_I;
1144 csa->lscsa->stopped_status.slot[1] = status_code;
1145
1146 } else if ((csa->prob.spu_status_R & status_P_H) == status_P_H) {
1147
1148 /* SPU_Status[P,H]=1 - Halt Conditional, followed
1149 * by Stop and Signal instruction, followed by
1150 * 'br -4'.
1151 */
1152 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_P_H;
1153 csa->lscsa->stopped_status.slot[1] = status_code;
1154
1155 } else if ((csa->prob.spu_status_R & status_S_P) == status_S_P) {
1156
1157 /* SPU_Status[S,P]=1 - Stop and Signal instruction
1158 * followed by 'br -4'.
1159 */
1160 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_S_P;
1161 csa->lscsa->stopped_status.slot[1] = status_code;
1162
1163 } else if ((csa->prob.spu_status_R & status_S_I) == status_S_I) {
1164
1165 /* SPU_Status[S,I]=1 - Illegal instruction followed
1166 * by 'br -4'.
1167 */
1168 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_S_I;
1169 csa->lscsa->stopped_status.slot[1] = status_code;
1170
1171 } else if ((csa->prob.spu_status_R & status_P) == status_P) {
1172
1173 /* SPU_Status[P]=1 - Stop and Signal instruction
1174 * followed by 'br -4'.
1175 */
1176 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_P;
1177 csa->lscsa->stopped_status.slot[1] = status_code;
1178
1179 } else if ((csa->prob.spu_status_R & status_H) == status_H) {
1180
1181 /* SPU_Status[H]=1 - Halt Conditional, followed
1182 * by 'br -4'.
1183 */
1184 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_H;
1185
1186 } else if ((csa->prob.spu_status_R & status_S) == status_S) {
1187
1188 /* SPU_Status[S]=1 - Two nop instructions.
1189 */
1190 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_S;
1191
1192 } else if ((csa->prob.spu_status_R & status_I) == status_I) {
1193
1194 /* SPU_Status[I]=1 - Illegal instruction followed
1195 * by 'br -4'.
1196 */
1197 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_I;
1198
1199 }
1200 }
1201
1202 static inline void setup_spu_status_part2(struct spu_state *csa,
1203 struct spu *spu)
1204 {
1205 u32 mask;
1206
1207 /* Restore, Step 28:
1208 * If the CSA.SPU_Status[I,S,H,P,R]=0 then
1209 * add a 'br *' instruction to the end of
1210 * the SPU based restore code.
1211 *
1212 * NOTE: Rather than modifying the SPU executable, we
1213 * instead add a new 'stopped_status' field to the
1214 * LSCSA. The SPU-side restore reads this field and
1215 * takes the appropriate action when exiting.
1216 */
1217 mask = SPU_STATUS_INVALID_INSTR |
1218 SPU_STATUS_SINGLE_STEP |
1219 SPU_STATUS_STOPPED_BY_HALT |
1220 SPU_STATUS_STOPPED_BY_STOP | SPU_STATUS_RUNNING;
1221 if (!(csa->prob.spu_status_R & mask)) {
1222 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_R;
1223 }
1224 }
1225
1226 static inline void restore_mfc_rag(struct spu_state *csa, struct spu *spu)
1227 {
1228 /* Restore, Step 29:
1229 * Restore RA_GROUP_ID register and the
1230 * RA_ENABLE reigster from the CSA.
1231 */
1232 spu_resource_allocation_groupID_set(spu,
1233 csa->priv1.resource_allocation_groupID_RW);
1234 spu_resource_allocation_enable_set(spu,
1235 csa->priv1.resource_allocation_enable_RW);
1236 }
1237
1238 static inline void send_restore_code(struct spu_state *csa, struct spu *spu)
1239 {
1240 unsigned long addr = (unsigned long)&spu_restore_code[0];
1241 unsigned int ls_offset = 0x0;
1242 unsigned int size = sizeof(spu_restore_code);
1243 unsigned int tag = 0;
1244 unsigned int rclass = 0;
1245 unsigned int cmd = MFC_GETFS_CMD;
1246
1247 /* Restore, Step 37:
1248 * Issue MFC DMA command to copy context
1249 * restore code to local storage.
1250 */
1251 send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd);
1252 }
1253
1254 static inline void setup_decr(struct spu_state *csa, struct spu *spu)
1255 {
1256 /* Restore, Step 34:
1257 * If CSA.MFC_CNTL[Ds]=1 (decrementer was
1258 * running) then adjust decrementer, set
1259 * decrementer running status in LSCSA,
1260 * and set decrementer "wrapped" status
1261 * in LSCSA.
1262 */
1263 if (csa->priv2.mfc_control_RW & MFC_CNTL_DECREMENTER_RUNNING) {
1264 cycles_t resume_time = get_cycles();
1265 cycles_t delta_time = resume_time - csa->suspend_time;
1266
1267 csa->lscsa->decr_status.slot[0] = SPU_DECR_STATUS_RUNNING;
1268 if (csa->lscsa->decr.slot[0] < delta_time) {
1269 csa->lscsa->decr_status.slot[0] |=
1270 SPU_DECR_STATUS_WRAPPED;
1271 }
1272
1273 csa->lscsa->decr.slot[0] -= delta_time;
1274 } else {
1275 csa->lscsa->decr_status.slot[0] = 0;
1276 }
1277 }
1278
1279 static inline void setup_ppu_mb(struct spu_state *csa, struct spu *spu)
1280 {
1281 /* Restore, Step 35:
1282 * Copy the CSA.PU_MB data into the LSCSA.
1283 */
1284 csa->lscsa->ppu_mb.slot[0] = csa->prob.pu_mb_R;
1285 }
1286
1287 static inline void setup_ppuint_mb(struct spu_state *csa, struct spu *spu)
1288 {
1289 /* Restore, Step 36:
1290 * Copy the CSA.PUINT_MB data into the LSCSA.
1291 */
1292 csa->lscsa->ppuint_mb.slot[0] = csa->priv2.puint_mb_R;
1293 }
1294
1295 static inline int check_restore_status(struct spu_state *csa, struct spu *spu)
1296 {
1297 struct spu_problem __iomem *prob = spu->problem;
1298 u32 complete;
1299
1300 /* Restore, Step 40:
1301 * If SPU_Status[P]=1 and SPU_Status[SC] = "success",
1302 * context restore succeeded, otherwise context restore
1303 * failed.
1304 */
1305 complete = ((SPU_RESTORE_COMPLETE << SPU_STOP_STATUS_SHIFT) |
1306 SPU_STATUS_STOPPED_BY_STOP);
1307 return (in_be32(&prob->spu_status_R) != complete) ? 1 : 0;
1308 }
1309
1310 static inline void restore_spu_privcntl(struct spu_state *csa, struct spu *spu)
1311 {
1312 struct spu_priv2 __iomem *priv2 = spu->priv2;
1313
1314 /* Restore, Step 41:
1315 * Restore SPU_PrivCntl from the CSA.
1316 */
1317 out_be64(&priv2->spu_privcntl_RW, csa->priv2.spu_privcntl_RW);
1318 eieio();
1319 }
1320
1321 static inline void restore_status_part1(struct spu_state *csa, struct spu *spu)
1322 {
1323 struct spu_problem __iomem *prob = spu->problem;
1324 u32 mask;
1325
1326 /* Restore, Step 42:
1327 * If any CSA.SPU_Status[I,S,H,P]=1, then
1328 * restore the error or single step state.
1329 */
1330 mask = SPU_STATUS_INVALID_INSTR |
1331 SPU_STATUS_SINGLE_STEP |
1332 SPU_STATUS_STOPPED_BY_HALT | SPU_STATUS_STOPPED_BY_STOP;
1333 if (csa->prob.spu_status_R & mask) {
1334 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE);
1335 eieio();
1336 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
1337 SPU_STATUS_RUNNING);
1338 }
1339 }
1340
1341 static inline void restore_status_part2(struct spu_state *csa, struct spu *spu)
1342 {
1343 struct spu_problem __iomem *prob = spu->problem;
1344 u32 mask;
1345
1346 /* Restore, Step 43:
1347 * If all CSA.SPU_Status[I,S,H,P,R]=0 then write
1348 * SPU_RunCntl[R0R1]='01', wait for SPU_Status[R]=1,
1349 * then write '00' to SPU_RunCntl[R0R1] and wait
1350 * for SPU_Status[R]=0.
1351 */
1352 mask = SPU_STATUS_INVALID_INSTR |
1353 SPU_STATUS_SINGLE_STEP |
1354 SPU_STATUS_STOPPED_BY_HALT |
1355 SPU_STATUS_STOPPED_BY_STOP | SPU_STATUS_RUNNING;
1356 if (!(csa->prob.spu_status_R & mask)) {
1357 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE);
1358 eieio();
1359 POLL_WHILE_FALSE(in_be32(&prob->spu_status_R) &
1360 SPU_STATUS_RUNNING);
1361 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP);
1362 eieio();
1363 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
1364 SPU_STATUS_RUNNING);
1365 }
1366 }
1367
1368 static inline void restore_ls_16kb(struct spu_state *csa, struct spu *spu)
1369 {
1370 unsigned long addr = (unsigned long)&csa->lscsa->ls[0];
1371 unsigned int ls_offset = 0x0;
1372 unsigned int size = 16384;
1373 unsigned int tag = 0;
1374 unsigned int rclass = 0;
1375 unsigned int cmd = MFC_GET_CMD;
1376
1377 /* Restore, Step 44:
1378 * Issue a DMA command to restore the first
1379 * 16kb of local storage from CSA.
1380 */
1381 send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd);
1382 }
1383
1384 static inline void suspend_mfc(struct spu_state *csa, struct spu *spu)
1385 {
1386 struct spu_priv2 __iomem *priv2 = spu->priv2;
1387
1388 /* Restore, Step 47.
1389 * Write MFC_Cntl[Sc,Sm]='1','' to suspend
1390 * the queue.
1391 */
1392 out_be64(&priv2->mfc_control_RW, MFC_CNTL_SUSPEND_DMA_QUEUE);
1393 eieio();
1394 }
1395
1396 static inline void clear_interrupts(struct spu_state *csa, struct spu *spu)
1397 {
1398 /* Restore, Step 49:
1399 * Write INT_MASK_class0 with value of 0.
1400 * Write INT_MASK_class1 with value of 0.
1401 * Write INT_MASK_class2 with value of 0.
1402 * Write INT_STAT_class0 with value of -1.
1403 * Write INT_STAT_class1 with value of -1.
1404 * Write INT_STAT_class2 with value of -1.
1405 */
1406 spin_lock_irq(&spu->register_lock);
1407 spu_int_mask_set(spu, 0, 0ul);
1408 spu_int_mask_set(spu, 1, 0ul);
1409 spu_int_mask_set(spu, 2, 0ul);
1410 spu_int_stat_clear(spu, 0, CLASS0_INTR_MASK);
1411 spu_int_stat_clear(spu, 1, CLASS1_INTR_MASK);
1412 spu_int_stat_clear(spu, 2, CLASS2_INTR_MASK);
1413 spin_unlock_irq(&spu->register_lock);
1414 }
1415
1416 static inline void restore_mfc_queues(struct spu_state *csa, struct spu *spu)
1417 {
1418 struct spu_priv2 __iomem *priv2 = spu->priv2;
1419 int i;
1420
1421 /* Restore, Step 50:
1422 * If MFC_Cntl[Se]!=0 then restore
1423 * MFC command queues.
1424 */
1425 if ((csa->priv2.mfc_control_RW & MFC_CNTL_DMA_QUEUES_EMPTY_MASK) == 0) {
1426 for (i = 0; i < 8; i++) {
1427 out_be64(&priv2->puq[i].mfc_cq_data0_RW,
1428 csa->priv2.puq[i].mfc_cq_data0_RW);
1429 out_be64(&priv2->puq[i].mfc_cq_data1_RW,
1430 csa->priv2.puq[i].mfc_cq_data1_RW);
1431 out_be64(&priv2->puq[i].mfc_cq_data2_RW,
1432 csa->priv2.puq[i].mfc_cq_data2_RW);
1433 out_be64(&priv2->puq[i].mfc_cq_data3_RW,
1434 csa->priv2.puq[i].mfc_cq_data3_RW);
1435 }
1436 for (i = 0; i < 16; i++) {
1437 out_be64(&priv2->spuq[i].mfc_cq_data0_RW,
1438 csa->priv2.spuq[i].mfc_cq_data0_RW);
1439 out_be64(&priv2->spuq[i].mfc_cq_data1_RW,
1440 csa->priv2.spuq[i].mfc_cq_data1_RW);
1441 out_be64(&priv2->spuq[i].mfc_cq_data2_RW,
1442 csa->priv2.spuq[i].mfc_cq_data2_RW);
1443 out_be64(&priv2->spuq[i].mfc_cq_data3_RW,
1444 csa->priv2.spuq[i].mfc_cq_data3_RW);
1445 }
1446 }
1447 eieio();
1448 }
1449
1450 static inline void restore_ppu_querymask(struct spu_state *csa, struct spu *spu)
1451 {
1452 struct spu_problem __iomem *prob = spu->problem;
1453
1454 /* Restore, Step 51:
1455 * Restore the PPU_QueryMask register from CSA.
1456 */
1457 out_be32(&prob->dma_querymask_RW, csa->prob.dma_querymask_RW);
1458 eieio();
1459 }
1460
1461 static inline void restore_ppu_querytype(struct spu_state *csa, struct spu *spu)
1462 {
1463 struct spu_problem __iomem *prob = spu->problem;
1464
1465 /* Restore, Step 52:
1466 * Restore the PPU_QueryType register from CSA.
1467 */
1468 out_be32(&prob->dma_querytype_RW, csa->prob.dma_querytype_RW);
1469 eieio();
1470 }
1471
1472 static inline void restore_mfc_csr_tsq(struct spu_state *csa, struct spu *spu)
1473 {
1474 struct spu_priv2 __iomem *priv2 = spu->priv2;
1475
1476 /* Restore, Step 53:
1477 * Restore the MFC_CSR_TSQ register from CSA.
1478 */
1479 out_be64(&priv2->spu_tag_status_query_RW,
1480 csa->priv2.spu_tag_status_query_RW);
1481 eieio();
1482 }
1483
1484 static inline void restore_mfc_csr_cmd(struct spu_state *csa, struct spu *spu)
1485 {
1486 struct spu_priv2 __iomem *priv2 = spu->priv2;
1487
1488 /* Restore, Step 54:
1489 * Restore the MFC_CSR_CMD1 and MFC_CSR_CMD2
1490 * registers from CSA.
1491 */
1492 out_be64(&priv2->spu_cmd_buf1_RW, csa->priv2.spu_cmd_buf1_RW);
1493 out_be64(&priv2->spu_cmd_buf2_RW, csa->priv2.spu_cmd_buf2_RW);
1494 eieio();
1495 }
1496
1497 static inline void restore_mfc_csr_ato(struct spu_state *csa, struct spu *spu)
1498 {
1499 struct spu_priv2 __iomem *priv2 = spu->priv2;
1500
1501 /* Restore, Step 55:
1502 * Restore the MFC_CSR_ATO register from CSA.
1503 */
1504 out_be64(&priv2->spu_atomic_status_RW, csa->priv2.spu_atomic_status_RW);
1505 }
1506
1507 static inline void restore_mfc_tclass_id(struct spu_state *csa, struct spu *spu)
1508 {
1509 /* Restore, Step 56:
1510 * Restore the MFC_TCLASS_ID register from CSA.
1511 */
1512 spu_mfc_tclass_id_set(spu, csa->priv1.mfc_tclass_id_RW);
1513 eieio();
1514 }
1515
1516 static inline void set_llr_event(struct spu_state *csa, struct spu *spu)
1517 {
1518 u64 ch0_cnt, ch0_data;
1519 u64 ch1_data;
1520
1521 /* Restore, Step 57:
1522 * Set the Lock Line Reservation Lost Event by:
1523 * 1. OR CSA.SPU_Event_Status with bit 21 (Lr) set to 1.
1524 * 2. If CSA.SPU_Channel_0_Count=0 and
1525 * CSA.SPU_Wr_Event_Mask[Lr]=1 and
1526 * CSA.SPU_Event_Status[Lr]=0 then set
1527 * CSA.SPU_Event_Status_Count=1.
1528 */
1529 ch0_cnt = csa->spu_chnlcnt_RW[0];
1530 ch0_data = csa->spu_chnldata_RW[0];
1531 ch1_data = csa->spu_chnldata_RW[1];
1532 csa->spu_chnldata_RW[0] |= MFC_LLR_LOST_EVENT;
1533 if ((ch0_cnt == 0) && !(ch0_data & MFC_LLR_LOST_EVENT) &&
1534 (ch1_data & MFC_LLR_LOST_EVENT)) {
1535 csa->spu_chnlcnt_RW[0] = 1;
1536 }
1537 }
1538
1539 static inline void restore_decr_wrapped(struct spu_state *csa, struct spu *spu)
1540 {
1541 /* Restore, Step 58:
1542 * If the status of the CSA software decrementer
1543 * "wrapped" flag is set, OR in a '1' to
1544 * CSA.SPU_Event_Status[Tm].
1545 */
1546 if (!(csa->lscsa->decr_status.slot[0] & SPU_DECR_STATUS_WRAPPED))
1547 return;
1548
1549 if ((csa->spu_chnlcnt_RW[0] == 0) &&
1550 (csa->spu_chnldata_RW[1] & 0x20) &&
1551 !(csa->spu_chnldata_RW[0] & 0x20))
1552 csa->spu_chnlcnt_RW[0] = 1;
1553
1554 csa->spu_chnldata_RW[0] |= 0x20;
1555 }
1556
1557 static inline void restore_ch_part1(struct spu_state *csa, struct spu *spu)
1558 {
1559 struct spu_priv2 __iomem *priv2 = spu->priv2;
1560 u64 idx, ch_indices[] = { 0UL, 3UL, 4UL, 24UL, 25UL, 27UL };
1561 int i;
1562
1563 /* Restore, Step 59:
1564 * Restore the following CH: [0,3,4,24,25,27]
1565 */
1566 for (i = 0; i < ARRAY_SIZE(ch_indices); i++) {
1567 idx = ch_indices[i];
1568 out_be64(&priv2->spu_chnlcntptr_RW, idx);
1569 eieio();
1570 out_be64(&priv2->spu_chnldata_RW, csa->spu_chnldata_RW[idx]);
1571 out_be64(&priv2->spu_chnlcnt_RW, csa->spu_chnlcnt_RW[idx]);
1572 eieio();
1573 }
1574 }
1575
1576 static inline void restore_ch_part2(struct spu_state *csa, struct spu *spu)
1577 {
1578 struct spu_priv2 __iomem *priv2 = spu->priv2;
1579 u64 ch_indices[3] = { 9UL, 21UL, 23UL };
1580 u64 ch_counts[3] = { 1UL, 16UL, 1UL };
1581 u64 idx;
1582 int i;
1583
1584 /* Restore, Step 60:
1585 * Restore the following CH: [9,21,23].
1586 */
1587 ch_counts[0] = 1UL;
1588 ch_counts[1] = csa->spu_chnlcnt_RW[21];
1589 ch_counts[2] = 1UL;
1590 for (i = 0; i < 3; i++) {
1591 idx = ch_indices[i];
1592 out_be64(&priv2->spu_chnlcntptr_RW, idx);
1593 eieio();
1594 out_be64(&priv2->spu_chnlcnt_RW, ch_counts[i]);
1595 eieio();
1596 }
1597 }
1598
1599 static inline void restore_spu_lslr(struct spu_state *csa, struct spu *spu)
1600 {
1601 struct spu_priv2 __iomem *priv2 = spu->priv2;
1602
1603 /* Restore, Step 61:
1604 * Restore the SPU_LSLR register from CSA.
1605 */
1606 out_be64(&priv2->spu_lslr_RW, csa->priv2.spu_lslr_RW);
1607 eieio();
1608 }
1609
1610 static inline void restore_spu_cfg(struct spu_state *csa, struct spu *spu)
1611 {
1612 struct spu_priv2 __iomem *priv2 = spu->priv2;
1613
1614 /* Restore, Step 62:
1615 * Restore the SPU_Cfg register from CSA.
1616 */
1617 out_be64(&priv2->spu_cfg_RW, csa->priv2.spu_cfg_RW);
1618 eieio();
1619 }
1620
1621 static inline void restore_pm_trace(struct spu_state *csa, struct spu *spu)
1622 {
1623 /* Restore, Step 63:
1624 * Restore PM_Trace_Tag_Wait_Mask from CSA.
1625 * Not performed by this implementation.
1626 */
1627 }
1628
1629 static inline void restore_spu_npc(struct spu_state *csa, struct spu *spu)
1630 {
1631 struct spu_problem __iomem *prob = spu->problem;
1632
1633 /* Restore, Step 64:
1634 * Restore SPU_NPC from CSA.
1635 */
1636 out_be32(&prob->spu_npc_RW, csa->prob.spu_npc_RW);
1637 eieio();
1638 }
1639
1640 static inline void restore_spu_mb(struct spu_state *csa, struct spu *spu)
1641 {
1642 struct spu_priv2 __iomem *priv2 = spu->priv2;
1643 int i;
1644
1645 /* Restore, Step 65:
1646 * Restore MFC_RdSPU_MB from CSA.
1647 */
1648 out_be64(&priv2->spu_chnlcntptr_RW, 29UL);
1649 eieio();
1650 out_be64(&priv2->spu_chnlcnt_RW, csa->spu_chnlcnt_RW[29]);
1651 for (i = 0; i < 4; i++) {
1652 out_be64(&priv2->spu_chnldata_RW, csa->spu_mailbox_data[i]);
1653 }
1654 eieio();
1655 }
1656
1657 static inline void check_ppu_mb_stat(struct spu_state *csa, struct spu *spu)
1658 {
1659 struct spu_problem __iomem *prob = spu->problem;
1660
1661 /* Restore, Step 66:
1662 * If CSA.MB_Stat[P]=0 (mailbox empty) then
1663 * read from the PPU_MB register.
1664 */
1665 if ((csa->prob.mb_stat_R & 0xFF) == 0) {
1666 in_be32(&prob->pu_mb_R);
1667 eieio();
1668 }
1669 }
1670
1671 static inline void check_ppuint_mb_stat(struct spu_state *csa, struct spu *spu)
1672 {
1673 struct spu_priv2 __iomem *priv2 = spu->priv2;
1674
1675 /* Restore, Step 66:
1676 * If CSA.MB_Stat[I]=0 (mailbox empty) then
1677 * read from the PPUINT_MB register.
1678 */
1679 if ((csa->prob.mb_stat_R & 0xFF0000) == 0) {
1680 in_be64(&priv2->puint_mb_R);
1681 eieio();
1682 spu_int_stat_clear(spu, 2, CLASS2_ENABLE_MAILBOX_INTR);
1683 eieio();
1684 }
1685 }
1686
1687 static inline void restore_mfc_sr1(struct spu_state *csa, struct spu *spu)
1688 {
1689 /* Restore, Step 69:
1690 * Restore the MFC_SR1 register from CSA.
1691 */
1692 spu_mfc_sr1_set(spu, csa->priv1.mfc_sr1_RW);
1693 eieio();
1694 }
1695
1696 static inline void set_int_route(struct spu_state *csa, struct spu *spu)
1697 {
1698 struct spu_context *ctx = spu->ctx;
1699
1700 spu_cpu_affinity_set(spu, ctx->last_ran);
1701 }
1702
1703 static inline void restore_other_spu_access(struct spu_state *csa,
1704 struct spu *spu)
1705 {
1706 /* Restore, Step 70:
1707 * Restore other SPU mappings to this SPU. TBD.
1708 */
1709 }
1710
1711 static inline void restore_spu_runcntl(struct spu_state *csa, struct spu *spu)
1712 {
1713 struct spu_problem __iomem *prob = spu->problem;
1714
1715 /* Restore, Step 71:
1716 * If CSA.SPU_Status[R]=1 then write
1717 * SPU_RunCntl[R0R1]='01'.
1718 */
1719 if (csa->prob.spu_status_R & SPU_STATUS_RUNNING) {
1720 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE);
1721 eieio();
1722 }
1723 }
1724
1725 static inline void restore_mfc_cntl(struct spu_state *csa, struct spu *spu)
1726 {
1727 struct spu_priv2 __iomem *priv2 = spu->priv2;
1728
1729 /* Restore, Step 72:
1730 * Restore the MFC_CNTL register for the CSA.
1731 */
1732 out_be64(&priv2->mfc_control_RW, csa->priv2.mfc_control_RW);
1733 eieio();
1734
1735 /*
1736 * The queue is put back into the same state that was evident prior to
1737 * the context switch. The suspend flag is added to the saved state in
1738 * the csa, if the operational state was suspending or suspended. In
1739 * this case, the code that suspended the mfc is responsible for
1740 * continuing it. Note that SPE faults do not change the operational
1741 * state of the spu.
1742 */
1743 }
1744
1745 static inline void enable_user_access(struct spu_state *csa, struct spu *spu)
1746 {
1747 /* Restore, Step 73:
1748 * Enable user-space access (if provided) to this
1749 * SPU by mapping the virtual pages assigned to
1750 * the SPU memory-mapped I/O (MMIO) for problem
1751 * state. TBD.
1752 */
1753 }
1754
1755 static inline void reset_switch_active(struct spu_state *csa, struct spu *spu)
1756 {
1757 /* Restore, Step 74:
1758 * Reset the "context switch active" flag.
1759 * Not performed by this implementation.
1760 */
1761 }
1762
1763 static inline void reenable_interrupts(struct spu_state *csa, struct spu *spu)
1764 {
1765 /* Restore, Step 75:
1766 * Re-enable SPU interrupts.
1767 */
1768 spin_lock_irq(&spu->register_lock);
1769 spu_int_mask_set(spu, 0, csa->priv1.int_mask_class0_RW);
1770 spu_int_mask_set(spu, 1, csa->priv1.int_mask_class1_RW);
1771 spu_int_mask_set(spu, 2, csa->priv1.int_mask_class2_RW);
1772 spin_unlock_irq(&spu->register_lock);
1773 }
1774
1775 static int quiece_spu(struct spu_state *prev, struct spu *spu)
1776 {
1777 /*
1778 * Combined steps 2-18 of SPU context save sequence, which
1779 * quiesce the SPU state (disable SPU execution, MFC command
1780 * queues, decrementer, SPU interrupts, etc.).
1781 *
1782 * Returns 0 on success.
1783 * 2 if failed step 2.
1784 * 6 if failed step 6.
1785 */
1786
1787 if (check_spu_isolate(prev, spu)) { /* Step 2. */
1788 return 2;
1789 }
1790 disable_interrupts(prev, spu); /* Step 3. */
1791 set_watchdog_timer(prev, spu); /* Step 4. */
1792 inhibit_user_access(prev, spu); /* Step 5. */
1793 if (check_spu_isolate(prev, spu)) { /* Step 6. */
1794 return 6;
1795 }
1796 set_switch_pending(prev, spu); /* Step 7. */
1797 save_mfc_cntl(prev, spu); /* Step 8. */
1798 save_spu_runcntl(prev, spu); /* Step 9. */
1799 save_mfc_sr1(prev, spu); /* Step 10. */
1800 save_spu_status(prev, spu); /* Step 11. */
1801 save_mfc_stopped_status(prev, spu); /* Step 12. */
1802 halt_mfc_decr(prev, spu); /* Step 13. */
1803 save_timebase(prev, spu); /* Step 14. */
1804 remove_other_spu_access(prev, spu); /* Step 15. */
1805 do_mfc_mssync(prev, spu); /* Step 16. */
1806 issue_mfc_tlbie(prev, spu); /* Step 17. */
1807 handle_pending_interrupts(prev, spu); /* Step 18. */
1808
1809 return 0;
1810 }
1811
1812 static void save_csa(struct spu_state *prev, struct spu *spu)
1813 {
1814 /*
1815 * Combine steps 19-44 of SPU context save sequence, which
1816 * save regions of the privileged & problem state areas.
1817 */
1818
1819 save_mfc_queues(prev, spu); /* Step 19. */
1820 save_ppu_querymask(prev, spu); /* Step 20. */
1821 save_ppu_querytype(prev, spu); /* Step 21. */
1822 save_ppu_tagstatus(prev, spu); /* NEW. */
1823 save_mfc_csr_tsq(prev, spu); /* Step 22. */
1824 save_mfc_csr_cmd(prev, spu); /* Step 23. */
1825 save_mfc_csr_ato(prev, spu); /* Step 24. */
1826 save_mfc_tclass_id(prev, spu); /* Step 25. */
1827 set_mfc_tclass_id(prev, spu); /* Step 26. */
1828 save_mfc_cmd(prev, spu); /* Step 26a - moved from 44. */
1829 purge_mfc_queue(prev, spu); /* Step 27. */
1830 wait_purge_complete(prev, spu); /* Step 28. */
1831 setup_mfc_sr1(prev, spu); /* Step 30. */
1832 save_spu_npc(prev, spu); /* Step 31. */
1833 save_spu_privcntl(prev, spu); /* Step 32. */
1834 reset_spu_privcntl(prev, spu); /* Step 33. */
1835 save_spu_lslr(prev, spu); /* Step 34. */
1836 reset_spu_lslr(prev, spu); /* Step 35. */
1837 save_spu_cfg(prev, spu); /* Step 36. */
1838 save_pm_trace(prev, spu); /* Step 37. */
1839 save_mfc_rag(prev, spu); /* Step 38. */
1840 save_ppu_mb_stat(prev, spu); /* Step 39. */
1841 save_ppu_mb(prev, spu); /* Step 40. */
1842 save_ppuint_mb(prev, spu); /* Step 41. */
1843 save_ch_part1(prev, spu); /* Step 42. */
1844 save_spu_mb(prev, spu); /* Step 43. */
1845 reset_ch(prev, spu); /* Step 45. */
1846 }
1847
1848 static void save_lscsa(struct spu_state *prev, struct spu *spu)
1849 {
1850 /*
1851 * Perform steps 46-57 of SPU context save sequence,
1852 * which save regions of the local store and register
1853 * file.
1854 */
1855
1856 resume_mfc_queue(prev, spu); /* Step 46. */
1857 /* Step 47. */
1858 setup_mfc_slbs(prev, spu, spu_save_code, sizeof(spu_save_code));
1859 set_switch_active(prev, spu); /* Step 48. */
1860 enable_interrupts(prev, spu); /* Step 49. */
1861 save_ls_16kb(prev, spu); /* Step 50. */
1862 set_spu_npc(prev, spu); /* Step 51. */
1863 set_signot1(prev, spu); /* Step 52. */
1864 set_signot2(prev, spu); /* Step 53. */
1865 send_save_code(prev, spu); /* Step 54. */
1866 set_ppu_querymask(prev, spu); /* Step 55. */
1867 wait_tag_complete(prev, spu); /* Step 56. */
1868 wait_spu_stopped(prev, spu); /* Step 57. */
1869 }
1870
1871 static void force_spu_isolate_exit(struct spu *spu)
1872 {
1873 struct spu_problem __iomem *prob = spu->problem;
1874 struct spu_priv2 __iomem *priv2 = spu->priv2;
1875
1876 /* Stop SPE execution and wait for completion. */
1877 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP);
1878 iobarrier_rw();
1879 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING);
1880
1881 /* Restart SPE master runcntl. */
1882 spu_mfc_sr1_set(spu, MFC_STATE1_MASTER_RUN_CONTROL_MASK);
1883 iobarrier_w();
1884
1885 /* Initiate isolate exit request and wait for completion. */
1886 out_be64(&priv2->spu_privcntl_RW, 4LL);
1887 iobarrier_w();
1888 out_be32(&prob->spu_runcntl_RW, 2);
1889 iobarrier_rw();
1890 POLL_WHILE_FALSE((in_be32(&prob->spu_status_R)
1891 & SPU_STATUS_STOPPED_BY_STOP));
1892
1893 /* Reset load request to normal. */
1894 out_be64(&priv2->spu_privcntl_RW, SPU_PRIVCNT_LOAD_REQUEST_NORMAL);
1895 iobarrier_w();
1896 }
1897
1898 /**
1899 * stop_spu_isolate
1900 * Check SPU run-control state and force isolated
1901 * exit function as necessary.
1902 */
1903 static void stop_spu_isolate(struct spu *spu)
1904 {
1905 struct spu_problem __iomem *prob = spu->problem;
1906
1907 if (in_be32(&prob->spu_status_R) & SPU_STATUS_ISOLATED_STATE) {
1908 /* The SPU is in isolated state; the only way
1909 * to get it out is to perform an isolated
1910 * exit (clean) operation.
1911 */
1912 force_spu_isolate_exit(spu);
1913 }
1914 }
1915
1916 static void harvest(struct spu_state *prev, struct spu *spu)
1917 {
1918 /*
1919 * Perform steps 2-25 of SPU context restore sequence,
1920 * which resets an SPU either after a failed save, or
1921 * when using SPU for first time.
1922 */
1923
1924 disable_interrupts(prev, spu); /* Step 2. */
1925 inhibit_user_access(prev, spu); /* Step 3. */
1926 terminate_spu_app(prev, spu); /* Step 4. */
1927 set_switch_pending(prev, spu); /* Step 5. */
1928 stop_spu_isolate(spu); /* NEW. */
1929 remove_other_spu_access(prev, spu); /* Step 6. */
1930 suspend_mfc_and_halt_decr(prev, spu); /* Step 7. */
1931 wait_suspend_mfc_complete(prev, spu); /* Step 8. */
1932 if (!suspend_spe(prev, spu)) /* Step 9. */
1933 clear_spu_status(prev, spu); /* Step 10. */
1934 do_mfc_mssync(prev, spu); /* Step 11. */
1935 issue_mfc_tlbie(prev, spu); /* Step 12. */
1936 handle_pending_interrupts(prev, spu); /* Step 13. */
1937 purge_mfc_queue(prev, spu); /* Step 14. */
1938 wait_purge_complete(prev, spu); /* Step 15. */
1939 reset_spu_privcntl(prev, spu); /* Step 16. */
1940 reset_spu_lslr(prev, spu); /* Step 17. */
1941 setup_mfc_sr1(prev, spu); /* Step 18. */
1942 spu_invalidate_slbs(spu); /* Step 19. */
1943 reset_ch_part1(prev, spu); /* Step 20. */
1944 reset_ch_part2(prev, spu); /* Step 21. */
1945 enable_interrupts(prev, spu); /* Step 22. */
1946 set_switch_active(prev, spu); /* Step 23. */
1947 set_mfc_tclass_id(prev, spu); /* Step 24. */
1948 resume_mfc_queue(prev, spu); /* Step 25. */
1949 }
1950
1951 static void restore_lscsa(struct spu_state *next, struct spu *spu)
1952 {
1953 /*
1954 * Perform steps 26-40 of SPU context restore sequence,
1955 * which restores regions of the local store and register
1956 * file.
1957 */
1958
1959 set_watchdog_timer(next, spu); /* Step 26. */
1960 setup_spu_status_part1(next, spu); /* Step 27. */
1961 setup_spu_status_part2(next, spu); /* Step 28. */
1962 restore_mfc_rag(next, spu); /* Step 29. */
1963 /* Step 30. */
1964 setup_mfc_slbs(next, spu, spu_restore_code, sizeof(spu_restore_code));
1965 set_spu_npc(next, spu); /* Step 31. */
1966 set_signot1(next, spu); /* Step 32. */
1967 set_signot2(next, spu); /* Step 33. */
1968 setup_decr(next, spu); /* Step 34. */
1969 setup_ppu_mb(next, spu); /* Step 35. */
1970 setup_ppuint_mb(next, spu); /* Step 36. */
1971 send_restore_code(next, spu); /* Step 37. */
1972 set_ppu_querymask(next, spu); /* Step 38. */
1973 wait_tag_complete(next, spu); /* Step 39. */
1974 wait_spu_stopped(next, spu); /* Step 40. */
1975 }
1976
1977 static void restore_csa(struct spu_state *next, struct spu *spu)
1978 {
1979 /*
1980 * Combine steps 41-76 of SPU context restore sequence, which
1981 * restore regions of the privileged & problem state areas.
1982 */
1983
1984 restore_spu_privcntl(next, spu); /* Step 41. */
1985 restore_status_part1(next, spu); /* Step 42. */
1986 restore_status_part2(next, spu); /* Step 43. */
1987 restore_ls_16kb(next, spu); /* Step 44. */
1988 wait_tag_complete(next, spu); /* Step 45. */
1989 suspend_mfc(next, spu); /* Step 46. */
1990 wait_suspend_mfc_complete(next, spu); /* Step 47. */
1991 issue_mfc_tlbie(next, spu); /* Step 48. */
1992 clear_interrupts(next, spu); /* Step 49. */
1993 restore_mfc_queues(next, spu); /* Step 50. */
1994 restore_ppu_querymask(next, spu); /* Step 51. */
1995 restore_ppu_querytype(next, spu); /* Step 52. */
1996 restore_mfc_csr_tsq(next, spu); /* Step 53. */
1997 restore_mfc_csr_cmd(next, spu); /* Step 54. */
1998 restore_mfc_csr_ato(next, spu); /* Step 55. */
1999 restore_mfc_tclass_id(next, spu); /* Step 56. */
2000 set_llr_event(next, spu); /* Step 57. */
2001 restore_decr_wrapped(next, spu); /* Step 58. */
2002 restore_ch_part1(next, spu); /* Step 59. */
2003 restore_ch_part2(next, spu); /* Step 60. */
2004 restore_spu_lslr(next, spu); /* Step 61. */
2005 restore_spu_cfg(next, spu); /* Step 62. */
2006 restore_pm_trace(next, spu); /* Step 63. */
2007 restore_spu_npc(next, spu); /* Step 64. */
2008 restore_spu_mb(next, spu); /* Step 65. */
2009 check_ppu_mb_stat(next, spu); /* Step 66. */
2010 check_ppuint_mb_stat(next, spu); /* Step 67. */
2011 spu_invalidate_slbs(spu); /* Modified Step 68. */
2012 restore_mfc_sr1(next, spu); /* Step 69. */
2013 set_int_route(next, spu); /* NEW */
2014 restore_other_spu_access(next, spu); /* Step 70. */
2015 restore_spu_runcntl(next, spu); /* Step 71. */
2016 restore_mfc_cntl(next, spu); /* Step 72. */
2017 enable_user_access(next, spu); /* Step 73. */
2018 reset_switch_active(next, spu); /* Step 74. */
2019 reenable_interrupts(next, spu); /* Step 75. */
2020 }
2021
2022 static int __do_spu_save(struct spu_state *prev, struct spu *spu)
2023 {
2024 int rc;
2025
2026 /*
2027 * SPU context save can be broken into three phases:
2028 *
2029 * (a) quiesce [steps 2-16].
2030 * (b) save of CSA, performed by PPE [steps 17-42]
2031 * (c) save of LSCSA, mostly performed by SPU [steps 43-52].
2032 *
2033 * Returns 0 on success.
2034 * 2,6 if failed to quiece SPU
2035 * 53 if SPU-side of save failed.
2036 */
2037
2038 rc = quiece_spu(prev, spu); /* Steps 2-16. */
2039 switch (rc) {
2040 default:
2041 case 2:
2042 case 6:
2043 harvest(prev, spu);
2044 return rc;
2045 break;
2046 case 0:
2047 break;
2048 }
2049 save_csa(prev, spu); /* Steps 17-43. */
2050 save_lscsa(prev, spu); /* Steps 44-53. */
2051 return check_save_status(prev, spu); /* Step 54. */
2052 }
2053
2054 static int __do_spu_restore(struct spu_state *next, struct spu *spu)
2055 {
2056 int rc;
2057
2058 /*
2059 * SPU context restore can be broken into three phases:
2060 *
2061 * (a) harvest (or reset) SPU [steps 2-24].
2062 * (b) restore LSCSA [steps 25-40], mostly performed by SPU.
2063 * (c) restore CSA [steps 41-76], performed by PPE.
2064 *
2065 * The 'harvest' step is not performed here, but rather
2066 * as needed below.
2067 */
2068
2069 restore_lscsa(next, spu); /* Steps 24-39. */
2070 rc = check_restore_status(next, spu); /* Step 40. */
2071 switch (rc) {
2072 default:
2073 /* Failed. Return now. */
2074 return rc;
2075 break;
2076 case 0:
2077 /* Fall through to next step. */
2078 break;
2079 }
2080 restore_csa(next, spu);
2081
2082 return 0;
2083 }
2084
2085 /**
2086 * spu_save - SPU context save, with locking.
2087 * @prev: pointer to SPU context save area, to be saved.
2088 * @spu: pointer to SPU iomem structure.
2089 *
2090 * Acquire locks, perform the save operation then return.
2091 */
2092 int spu_save(struct spu_state *prev, struct spu *spu)
2093 {
2094 int rc;
2095
2096 acquire_spu_lock(spu); /* Step 1. */
2097 rc = __do_spu_save(prev, spu); /* Steps 2-53. */
2098 release_spu_lock(spu);
2099 if (rc != 0 && rc != 2 && rc != 6) {
2100 panic("%s failed on SPU[%d], rc=%d.\n",
2101 __func__, spu->number, rc);
2102 }
2103 return 0;
2104 }
2105 EXPORT_SYMBOL_GPL(spu_save);
2106
2107 /**
2108 * spu_restore - SPU context restore, with harvest and locking.
2109 * @new: pointer to SPU context save area, to be restored.
2110 * @spu: pointer to SPU iomem structure.
2111 *
2112 * Perform harvest + restore, as we may not be coming
2113 * from a previous successful save operation, and the
2114 * hardware state is unknown.
2115 */
2116 int spu_restore(struct spu_state *new, struct spu *spu)
2117 {
2118 int rc;
2119
2120 acquire_spu_lock(spu);
2121 harvest(NULL, spu);
2122 spu->slb_replace = 0;
2123 rc = __do_spu_restore(new, spu);
2124 release_spu_lock(spu);
2125 if (rc) {
2126 panic("%s failed on SPU[%d] rc=%d.\n",
2127 __func__, spu->number, rc);
2128 }
2129 return rc;
2130 }
2131 EXPORT_SYMBOL_GPL(spu_restore);
2132
2133 static void init_prob(struct spu_state *csa)
2134 {
2135 csa->spu_chnlcnt_RW[9] = 1;
2136 csa->spu_chnlcnt_RW[21] = 16;
2137 csa->spu_chnlcnt_RW[23] = 1;
2138 csa->spu_chnlcnt_RW[28] = 1;
2139 csa->spu_chnlcnt_RW[30] = 1;
2140 csa->prob.spu_runcntl_RW = SPU_RUNCNTL_STOP;
2141 csa->prob.mb_stat_R = 0x000400;
2142 }
2143
2144 static void init_priv1(struct spu_state *csa)
2145 {
2146 /* Enable decode, relocate, tlbie response, master runcntl. */
2147 csa->priv1.mfc_sr1_RW = MFC_STATE1_LOCAL_STORAGE_DECODE_MASK |
2148 MFC_STATE1_MASTER_RUN_CONTROL_MASK |
2149 MFC_STATE1_PROBLEM_STATE_MASK |
2150 MFC_STATE1_RELOCATE_MASK | MFC_STATE1_BUS_TLBIE_MASK;
2151
2152 /* Enable OS-specific set of interrupts. */
2153 csa->priv1.int_mask_class0_RW = CLASS0_ENABLE_DMA_ALIGNMENT_INTR |
2154 CLASS0_ENABLE_INVALID_DMA_COMMAND_INTR |
2155 CLASS0_ENABLE_SPU_ERROR_INTR;
2156 csa->priv1.int_mask_class1_RW = CLASS1_ENABLE_SEGMENT_FAULT_INTR |
2157 CLASS1_ENABLE_STORAGE_FAULT_INTR;
2158 csa->priv1.int_mask_class2_RW = CLASS2_ENABLE_SPU_STOP_INTR |
2159 CLASS2_ENABLE_SPU_HALT_INTR |
2160 CLASS2_ENABLE_SPU_DMA_TAG_GROUP_COMPLETE_INTR;
2161 }
2162
2163 static void init_priv2(struct spu_state *csa)
2164 {
2165 csa->priv2.spu_lslr_RW = LS_ADDR_MASK;
2166 csa->priv2.mfc_control_RW = MFC_CNTL_RESUME_DMA_QUEUE |
2167 MFC_CNTL_NORMAL_DMA_QUEUE_OPERATION |
2168 MFC_CNTL_DMA_QUEUES_EMPTY_MASK;
2169 }
2170
2171 /**
2172 * spu_alloc_csa - allocate and initialize an SPU context save area.
2173 *
2174 * Allocate and initialize the contents of an SPU context save area.
2175 * This includes enabling address translation, interrupt masks, etc.,
2176 * as appropriate for the given OS environment.
2177 *
2178 * Note that storage for the 'lscsa' is allocated separately,
2179 * as it is by far the largest of the context save regions,
2180 * and may need to be pinned or otherwise specially aligned.
2181 */
2182 int spu_init_csa(struct spu_state *csa)
2183 {
2184 int rc;
2185
2186 if (!csa)
2187 return -EINVAL;
2188 memset(csa, 0, sizeof(struct spu_state));
2189
2190 rc = spu_alloc_lscsa(csa);
2191 if (rc)
2192 return rc;
2193
2194 spin_lock_init(&csa->register_lock);
2195
2196 init_prob(csa);
2197 init_priv1(csa);
2198 init_priv2(csa);
2199
2200 return 0;
2201 }
2202
2203 void spu_fini_csa(struct spu_state *csa)
2204 {
2205 spu_free_lscsa(csa);
2206 }
2207
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.