1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * ultra.S: Don't expand these all over the pl
4 *
5 * Copyright (C) 1997, 2000, 2008 David S. Mil
6 */
7
8 #include <linux/pgtable.h>
9 #include <asm/asi.h>
10 #include <asm/page.h>
11 #include <asm/spitfire.h>
12 #include <asm/mmu_context.h>
13 #include <asm/mmu.h>
14 #include <asm/pil.h>
15 #include <asm/head.h>
16 #include <asm/thread_info.h>
17 #include <asm/cacheflush.h>
18 #include <asm/hypervisor.h>
19 #include <asm/cpudata.h>
20
21 /* Basically, most of the Spitfire vs.
22 * has to do with the fact that Cheeta
23 * IMMU flushes out of the secondary c
24 * to throw a south lake birthday part
25 * in Microelectronics who refused to
26 */
27
28 /* This file is meant to be read effic
29 * Staraj sie tego nikomu nie pierdoln
30 */
31 .text
32 .align 32
33 .globl __flush_tlb_mm
34 __flush_tlb_mm: /* 19 insns */
35 /* %o0=(ctx & TAG_CONTEXT_BITS), %o1=S
36 ldxa [%o1] ASI_DMMU, %g2
37 cmp %g2, %o0
38 bne,pn %icc, __spitfire_flush
39 mov 0x50, %g3
40 stxa %g0, [%g3] ASI_DMMU_DE
41 stxa %g0, [%g3] ASI_IMMU_DE
42 sethi %hi(KERNBASE), %g3
43 flush %g3
44 retl
45 nop
46 nop
47 nop
48 nop
49 nop
50 nop
51 nop
52 nop
53 nop
54 nop
55
56 .align 32
57 .globl __flush_tlb_page
58 __flush_tlb_page: /* 22 insns */
59 /* %o0 = context, %o1 = vaddr */
60 rdpr %pstate, %g7
61 andn %g7, PSTATE_IE, %g2
62 wrpr %g2, %pstate
63 mov SECONDARY_CONTEXT, %o4
64 ldxa [%o4] ASI_DMMU, %g2
65 stxa %o0, [%o4] ASI_DMMU
66 andcc %o1, 1, %g0
67 andn %o1, 1, %o3
68 be,pn %icc, 1f
69 or %o3, 0x10, %o3
70 stxa %g0, [%o3] ASI_IMMU_DE
71 1: stxa %g0, [%o3] ASI_DMMU_DE
72 membar #Sync
73 stxa %g2, [%o4] ASI_DMMU
74 sethi %hi(KERNBASE), %o4
75 flush %o4
76 retl
77 wrpr %g7, 0x0, %pstate
78 nop
79 nop
80 nop
81 nop
82
83 .align 32
84 .globl __flush_tlb_pending
85 __flush_tlb_pending: /* 27 insns */
86 /* %o0 = context, %o1 = nr, %o2 = vadd
87 rdpr %pstate, %g7
88 sllx %o1, 3, %o1
89 andn %g7, PSTATE_IE, %g2
90 wrpr %g2, %pstate
91 mov SECONDARY_CONTEXT, %o4
92 ldxa [%o4] ASI_DMMU, %g2
93 stxa %o0, [%o4] ASI_DMMU
94 1: sub %o1, (1 << 3), %o1
95 ldx [%o2 + %o1], %o3
96 andcc %o3, 1, %g0
97 andn %o3, 1, %o3
98 be,pn %icc, 2f
99 or %o3, 0x10, %o3
100 stxa %g0, [%o3] ASI_IMMU_DE
101 2: stxa %g0, [%o3] ASI_DMMU_DE
102 membar #Sync
103 brnz,pt %o1, 1b
104 nop
105 stxa %g2, [%o4] ASI_DMMU
106 sethi %hi(KERNBASE), %o4
107 flush %o4
108 retl
109 wrpr %g7, 0x0, %pstate
110 nop
111 nop
112 nop
113 nop
114
115 .align 32
116 .globl __flush_tlb_kernel_ran
117 __flush_tlb_kernel_range: /* 31 insns */
118 /* %o0=start, %o1=end */
119 cmp %o0, %o1
120 be,pn %xcc, 2f
121 sub %o1, %o0, %o3
122 srlx %o3, 18, %o4
123 brnz,pn %o4, __spitfire_flush_
124 sethi %hi(PAGE_SIZE), %o4
125 sub %o3, %o4, %o3
126 or %o0, 0x20, %o0
127 1: stxa %g0, [%o0 + %o3] ASI_D
128 stxa %g0, [%o0 + %o3] ASI_I
129 membar #Sync
130 brnz,pt %o3, 1b
131 sub %o3, %o4, %o3
132 2: sethi %hi(KERNBASE), %o3
133 flush %o3
134 retl
135 nop
136 nop
137 nop
138 nop
139 nop
140 nop
141 nop
142 nop
143 nop
144 nop
145 nop
146 nop
147 nop
148 nop
149 nop
150
151 __spitfire_flush_tlb_kernel_range_slow:
152 mov 63 * 8, %o4
153 1: ldxa [%o4] ASI_ITLB_DATA_AC
154 andcc %o3, 0x40, %g0
155 bne,pn %xcc, 2f
156 mov TLB_TAG_ACCESS, %o3
157 stxa %g0, [%o3] ASI_IMMU
158 stxa %g0, [%o4] ASI_ITLB_DA
159 membar #Sync
160 2: ldxa [%o4] ASI_DTLB_DATA_AC
161 andcc %o3, 0x40, %g0
162 bne,pn %xcc, 2f
163 mov TLB_TAG_ACCESS, %o3
164 stxa %g0, [%o3] ASI_DMMU
165 stxa %g0, [%o4] ASI_DTLB_DA
166 membar #Sync
167 2: sub %o4, 8, %o4
168 brgez,pt %o4, 1b
169 nop
170 retl
171 nop
172
173 __spitfire_flush_tlb_mm_slow:
174 rdpr %pstate, %g1
175 wrpr %g1, PSTATE_IE, %pstat
176 stxa %o0, [%o1] ASI_DMMU
177 stxa %g0, [%g3] ASI_DMMU_DE
178 stxa %g0, [%g3] ASI_IMMU_DE
179 flush %g6
180 stxa %g2, [%o1] ASI_DMMU
181 sethi %hi(KERNBASE), %o1
182 flush %o1
183 retl
184 wrpr %g1, 0, %pstate
185
186 /*
187 * The following code flushes one page_size wo
188 */
189 .section .kprobes.text, "ax"
190 .align 32
191 .globl __flush_icache_page
192 __flush_icache_page: /* %o0 = phys_page */
193 srlx %o0, PAGE_SHIFT, %o0
194 sethi %hi(PAGE_OFFSET), %g1
195 sllx %o0, PAGE_SHIFT, %o0
196 sethi %hi(PAGE_SIZE), %g2
197 ldx [%g1 + %lo(PAGE_OFFSET
198 add %o0, %g1, %o0
199 1: subcc %g2, 32, %g2
200 bne,pt %icc, 1b
201 flush %o0 + %g2
202 retl
203 nop
204
205 #ifdef DCACHE_ALIASING_POSSIBLE
206
207 #if (PAGE_SHIFT != 13)
208 #error only page shift of 13 is supported by d
209 #endif
210
211 #define DTAG_MASK 0x3
212
213 /* This routine is Spitfire specific s
214 * D-cache size and line-size are OK.
215 */
216 .align 64
217 .globl __flush_dcache_page
218 __flush_dcache_page: /* %o0=kaddr, %o1=flus
219 sethi %hi(PAGE_OFFSET), %g1
220 ldx [%g1 + %lo(PAGE_OFFSET
221 sub %o0, %g1, %o0
222 srlx %o0, 11, %o0
223 sethi %hi(1 << 14), %o2
224 sub %o2, (1 << 5), %o2
225 1: ldxa [%o2] ASI_DCACHE_TAG,
226 andcc %o3, DTAG_MASK, %g0
227 be,pn %xcc, 2f
228 andn %o3, DTAG_MASK, %o3
229 cmp %o3, %o0
230 bne,pt %xcc, 2f
231 nop
232 stxa %g0, [%o2] ASI_DCACHE_
233 membar #Sync
234 2: brnz,pt %o2, 1b
235 sub %o2, (1 << 5), %o2
236
237 /* The I-cache does not snoop local st
238 * better flush that too when necessar
239 */
240 brnz,pt %o1, __flush_icache_pa
241 sllx %o0, 11, %o0
242 retl
243 nop
244
245 #endif /* DCACHE_ALIASING_POSSIBLE */
246
247 .previous
248
249 /* Cheetah specific versions, patched
250 __cheetah_flush_tlb_mm: /* 19 insns */
251 rdpr %pstate, %g7
252 andn %g7, PSTATE_IE, %g2
253 wrpr %g2, 0x0, %pstate
254 wrpr %g0, 1, %tl
255 mov PRIMARY_CONTEXT, %o2
256 mov 0x40, %g3
257 ldxa [%o2] ASI_DMMU, %g2
258 srlx %g2, CTX_PGSZ1_NUC_SHI
259 sllx %o1, CTX_PGSZ1_NUC_SHI
260 or %o0, %o1, %o0 /* Pre
261 stxa %o0, [%o2] ASI_DMMU
262 stxa %g0, [%g3] ASI_DMMU_DE
263 stxa %g0, [%g3] ASI_IMMU_DE
264 stxa %g2, [%o2] ASI_DMMU
265 sethi %hi(KERNBASE), %o2
266 flush %o2
267 wrpr %g0, 0, %tl
268 retl
269 wrpr %g7, 0x0, %pstate
270
271 __cheetah_flush_tlb_page: /* 22 insns */
272 /* %o0 = context, %o1 = vaddr */
273 rdpr %pstate, %g7
274 andn %g7, PSTATE_IE, %g2
275 wrpr %g2, 0x0, %pstate
276 wrpr %g0, 1, %tl
277 mov PRIMARY_CONTEXT, %o4
278 ldxa [%o4] ASI_DMMU, %g2
279 srlx %g2, CTX_PGSZ1_NUC_SHI
280 sllx %o3, CTX_PGSZ1_NUC_SHI
281 or %o0, %o3, %o0 /* Pre
282 stxa %o0, [%o4] ASI_DMMU
283 andcc %o1, 1, %g0
284 be,pn %icc, 1f
285 andn %o1, 1, %o3
286 stxa %g0, [%o3] ASI_IMMU_DE
287 1: stxa %g0, [%o3] ASI_DMMU_DE
288 membar #Sync
289 stxa %g2, [%o4] ASI_DMMU
290 sethi %hi(KERNBASE), %o4
291 flush %o4
292 wrpr %g0, 0, %tl
293 retl
294 wrpr %g7, 0x0, %pstate
295
296 __cheetah_flush_tlb_pending: /* 27 insns */
297 /* %o0 = context, %o1 = nr, %o2 = vadd
298 rdpr %pstate, %g7
299 sllx %o1, 3, %o1
300 andn %g7, PSTATE_IE, %g2
301 wrpr %g2, 0x0, %pstate
302 wrpr %g0, 1, %tl
303 mov PRIMARY_CONTEXT, %o4
304 ldxa [%o4] ASI_DMMU, %g2
305 srlx %g2, CTX_PGSZ1_NUC_SHI
306 sllx %o3, CTX_PGSZ1_NUC_SHI
307 or %o0, %o3, %o0 /* Pre
308 stxa %o0, [%o4] ASI_DMMU
309 1: sub %o1, (1 << 3), %o1
310 ldx [%o2 + %o1], %o3
311 andcc %o3, 1, %g0
312 be,pn %icc, 2f
313 andn %o3, 1, %o3
314 stxa %g0, [%o3] ASI_IMMU_DE
315 2: stxa %g0, [%o3] ASI_DMMU_DE
316 membar #Sync
317 brnz,pt %o1, 1b
318 nop
319 stxa %g2, [%o4] ASI_DMMU
320 sethi %hi(KERNBASE), %o4
321 flush %o4
322 wrpr %g0, 0, %tl
323 retl
324 wrpr %g7, 0x0, %pstate
325
326 __cheetah_flush_tlb_kernel_range: /* 31
327 /* %o0=start, %o1=end */
328 cmp %o0, %o1
329 be,pn %xcc, 2f
330 sub %o1, %o0, %o3
331 srlx %o3, 18, %o4
332 brnz,pn %o4, 3f
333 sethi %hi(PAGE_SIZE), %o4
334 sub %o3, %o4, %o3
335 or %o0, 0x20, %o0
336 1: stxa %g0, [%o0 + %o3] ASI_D
337 stxa %g0, [%o0 + %o3] ASI_I
338 membar #Sync
339 brnz,pt %o3, 1b
340 sub %o3, %o4, %o3
341 2: sethi %hi(KERNBASE), %o3
342 flush %o3
343 retl
344 nop
345 3: mov 0x80, %o4
346 stxa %g0, [%o4] ASI_DMMU_DE
347 membar #Sync
348 stxa %g0, [%o4] ASI_IMMU_DE
349 membar #Sync
350 retl
351 nop
352 nop
353 nop
354 nop
355 nop
356 nop
357 nop
358 nop
359
360 #ifdef DCACHE_ALIASING_POSSIBLE
361 __cheetah_flush_dcache_page: /* 11 insns */
362 sethi %hi(PAGE_OFFSET), %g1
363 ldx [%g1 + %lo(PAGE_OFFSET
364 sub %o0, %g1, %o0
365 sethi %hi(PAGE_SIZE), %o4
366 1: subcc %o4, (1 << 5), %o4
367 stxa %g0, [%o0 + %o4] ASI_D
368 membar #Sync
369 bne,pt %icc, 1b
370 nop
371 retl /* I-cache flush never
372 nop
373 #endif /* DCACHE_ALIASING_POSSIBLE */
374
375 /* Hypervisor specific versions, patch
376 __hypervisor_tlb_tl0_error:
377 save %sp, -192, %sp
378 mov %i0, %o0
379 call hypervisor_tlbop_error
380 mov %i1, %o1
381 ret
382 restore
383
384 __hypervisor_flush_tlb_mm: /* 19 insns */
385 mov %o0, %o2 /* ARG
386 mov 0, %o0 /* ARG
387 mov 0, %o1 /* ARG
388 mov HV_MMU_ALL, %o3 /* ARG
389 mov HV_FAST_MMU_DEMAP_CTX,
390 ta HV_FAST_TRAP
391 brnz,pn %o0, 1f
392 mov HV_FAST_MMU_DEMAP_CTX,
393 retl
394 nop
395 1: sethi %hi(__hypervisor_tlb_t
396 jmpl %o5 + %lo(__hypervisor
397 nop
398 nop
399 nop
400 nop
401 nop
402 nop
403 nop
404
405 __hypervisor_flush_tlb_page: /* 22 insns */
406 /* %o0 = context, %o1 = vaddr */
407 mov %o0, %g2
408 mov %o1, %o0
409 mov %g2, %o1
410 mov HV_MMU_ALL, %o2
411 srlx %o0, PAGE_SHIFT, %o0
412 sllx %o0, PAGE_SHIFT, %o0
413 ta HV_MMU_UNMAP_ADDR_TRAP
414 brnz,pn %o0, 1f
415 mov HV_MMU_UNMAP_ADDR_TRAP
416 retl
417 nop
418 1: sethi %hi(__hypervisor_tlb_t
419 jmpl %o2 + %lo(__hypervisor
420 nop
421 nop
422 nop
423 nop
424 nop
425 nop
426 nop
427 nop
428 nop
429
430 __hypervisor_flush_tlb_pending: /* 27 insns */
431 /* %o0 = context, %o1 = nr, %o2 = vadd
432 sllx %o1, 3, %g1
433 mov %o2, %g2
434 mov %o0, %g3
435 1: sub %g1, (1 << 3), %g1
436 ldx [%g2 + %g1], %o0
437 mov %g3, %o1
438 mov HV_MMU_ALL, %o2
439 srlx %o0, PAGE_SHIFT, %o0
440 sllx %o0, PAGE_SHIFT, %o0
441 ta HV_MMU_UNMAP_ADDR_TRAP
442 brnz,pn %o0, 1f
443 mov HV_MMU_UNMAP_ADDR_TRAP
444 brnz,pt %g1, 1b
445 nop
446 retl
447 nop
448 1: sethi %hi(__hypervisor_tlb_t
449 jmpl %o2 + %lo(__hypervisor
450 nop
451 nop
452 nop
453 nop
454 nop
455 nop
456 nop
457 nop
458 nop
459
460 __hypervisor_flush_tlb_kernel_range: /* 31 ins
461 /* %o0=start, %o1=end */
462 cmp %o0, %o1
463 be,pn %xcc, 2f
464 sub %o1, %o0, %g2
465 srlx %g2, 18, %g3
466 brnz,pn %g3, 4f
467 mov %o0, %g1
468 sethi %hi(PAGE_SIZE), %g3
469 sub %g2, %g3, %g2
470 1: add %g1, %g2, %o0 /* ARG
471 mov 0, %o1 /* ARG
472 mov HV_MMU_ALL, %o2 /* ARG
473 ta HV_MMU_UNMAP_ADDR_TRAP
474 brnz,pn %o0, 3f
475 mov HV_MMU_UNMAP_ADDR_TRAP
476 brnz,pt %g2, 1b
477 sub %g2, %g3, %g2
478 2: retl
479 nop
480 3: sethi %hi(__hypervisor_tlb_t
481 jmpl %o2 + %lo(__hypervisor
482 nop
483 4: mov 0, %o0 /* ARG
484 mov 0, %o1 /* ARG
485 mov 0, %o2 /* ARG
486 mov HV_MMU_ALL, %o3 /* ARG
487 mov HV_FAST_MMU_DEMAP_CTX,
488 ta HV_FAST_TRAP
489 brnz,pn %o0, 3b
490 mov HV_FAST_MMU_DEMAP_CTX,
491 retl
492 nop
493
494 #ifdef DCACHE_ALIASING_POSSIBLE
495 /* XXX Niagara and friends have an 8K
496 * XXX possible, but nothing explicit
497 * XXX guarantees this.
498 */
499 __hypervisor_flush_dcache_page: /* 2 insns */
500 retl
501 nop
502 #endif
503
504 tlb_patch_one:
505 1: lduw [%o1], %g1
506 stw %g1, [%o0]
507 flush %o0
508 subcc %o2, 1, %o2
509 add %o1, 4, %o1
510 bne,pt %icc, 1b
511 add %o0, 4, %o0
512 retl
513 nop
514
515 #ifdef CONFIG_SMP
516 /* These are all called by the slaves
517 * trap level 1, with interrupts fully
518 *
519 * Register usage:
520 * %g5 mm->context (all t
521 * %g1 address arg 1 (tlb p
522 * %g7 address arg 2 (tlb r
523 *
524 * %g6 scratch 1
525 * %g2 scratch 2
526 * %g3 scratch 3
527 * %g4 scratch 4
528 */
529 .align 32
530 .globl xcall_flush_tlb_mm
531 xcall_flush_tlb_mm: /* 24 insns */
532 mov PRIMARY_CONTEXT, %g2
533 ldxa [%g2] ASI_DMMU, %g3
534 srlx %g3, CTX_PGSZ1_NUC_SHI
535 sllx %g4, CTX_PGSZ1_NUC_SHI
536 or %g5, %g4, %g5 /* Pre
537 stxa %g5, [%g2] ASI_DMMU
538 mov 0x40, %g4
539 stxa %g0, [%g4] ASI_DMMU_DE
540 stxa %g0, [%g4] ASI_IMMU_DE
541 stxa %g3, [%g2] ASI_DMMU
542 retry
543 nop
544 nop
545 nop
546 nop
547 nop
548 nop
549 nop
550 nop
551 nop
552 nop
553 nop
554 nop
555 nop
556
557 .globl xcall_flush_tlb_page
558 xcall_flush_tlb_page: /* 20 insns */
559 /* %g5=context, %g1=vaddr */
560 mov PRIMARY_CONTEXT, %g4
561 ldxa [%g4] ASI_DMMU, %g2
562 srlx %g2, CTX_PGSZ1_NUC_SHI
563 sllx %g4, CTX_PGSZ1_NUC_SHI
564 or %g5, %g4, %g5
565 mov PRIMARY_CONTEXT, %g4
566 stxa %g5, [%g4] ASI_DMMU
567 andcc %g1, 0x1, %g0
568 be,pn %icc, 2f
569 andn %g1, 0x1, %g5
570 stxa %g0, [%g5] ASI_IMMU_DE
571 2: stxa %g0, [%g5] ASI_DMMU_DE
572 membar #Sync
573 stxa %g2, [%g4] ASI_DMMU
574 retry
575 nop
576 nop
577 nop
578 nop
579 nop
580
581 .globl xcall_flush_tlb_kernel
582 xcall_flush_tlb_kernel_range: /* 44 insns */
583 sethi %hi(PAGE_SIZE - 1), %g
584 or %g2, %lo(PAGE_SIZE - 1
585 andn %g1, %g2, %g1
586 andn %g7, %g2, %g7
587 sub %g7, %g1, %g3
588 srlx %g3, 18, %g2
589 brnz,pn %g2, 2f
590 sethi %hi(PAGE_SIZE), %g2
591 sub %g3, %g2, %g3
592 or %g1, 0x20, %g1
593 1: stxa %g0, [%g1 + %g3] ASI_D
594 stxa %g0, [%g1 + %g3] ASI_I
595 membar #Sync
596 brnz,pt %g3, 1b
597 sub %g3, %g2, %g3
598 retry
599 2: mov 63 * 8, %g1
600 1: ldxa [%g1] ASI_ITLB_DATA_AC
601 andcc %g2, 0x40, %g0
602 bne,pn %xcc, 2f
603 mov TLB_TAG_ACCESS, %g2
604 stxa %g0, [%g2] ASI_IMMU
605 stxa %g0, [%g1] ASI_ITLB_DA
606 membar #Sync
607 2: ldxa [%g1] ASI_DTLB_DATA_AC
608 andcc %g2, 0x40, %g0
609 bne,pn %xcc, 2f
610 mov TLB_TAG_ACCESS, %g2
611 stxa %g0, [%g2] ASI_DMMU
612 stxa %g0, [%g1] ASI_DTLB_DA
613 membar #Sync
614 2: sub %g1, 8, %g1
615 brgez,pt %g1, 1b
616 nop
617 retry
618 nop
619 nop
620 nop
621 nop
622 nop
623 nop
624 nop
625 nop
626 nop
627
628 /* This runs in a very controlled envi
629 * not need to worry about BH races et
630 */
631 .globl xcall_sync_tick
632 xcall_sync_tick:
633
634 661: rdpr %pstate, %g2
635 wrpr %g2, PSTATE_IG | PSTAT
636 .section .sun4v_2insn_patch, "a
637 .word 661b
638 nop
639 nop
640 .previous
641
642 rdpr %pil, %g2
643 wrpr %g0, PIL_NORMAL_MAX, %
644 sethi %hi(109f), %g7
645 b,pt %xcc, etrap_irq
646 109: or %g7, %lo(109b), %g7
647 #ifdef CONFIG_TRACE_IRQFLAGS
648 call trace_hardirqs_off
649 nop
650 #endif
651 call smp_synchronize_tick_c
652 nop
653 b rtrap_xcall
654 ldx [%sp + PTREGS_OFF + PT
655
656 .globl xcall_fetch_glob_regs
657 xcall_fetch_glob_regs:
658 sethi %hi(global_cpu_snapsho
659 or %g1, %lo(global_cpu_sn
660 __GET_CPUID(%g2)
661 sllx %g2, 6, %g3
662 add %g1, %g3, %g1
663 rdpr %tstate, %g7
664 stx %g7, [%g1 + GR_SNAP_TS
665 rdpr %tpc, %g7
666 stx %g7, [%g1 + GR_SNAP_TP
667 rdpr %tnpc, %g7
668 stx %g7, [%g1 + GR_SNAP_TN
669 stx %o7, [%g1 + GR_SNAP_O7
670 stx %i7, [%g1 + GR_SNAP_I7
671 /* Don't try this at home kids... */
672 rdpr %cwp, %g3
673 sub %g3, 1, %g7
674 wrpr %g7, %cwp
675 mov %i7, %g7
676 wrpr %g3, %cwp
677 stx %g7, [%g1 + GR_SNAP_RP
678 sethi %hi(trap_block), %g7
679 or %g7, %lo(trap_block),
680 sllx %g2, TRAP_BLOCK_SZ_SHI
681 add %g7, %g2, %g7
682 ldx [%g7 + TRAP_PER_CPU_TH
683 stx %g3, [%g1 + GR_SNAP_TH
684 retry
685
686 .globl xcall_fetch_glob_pmu
687 xcall_fetch_glob_pmu:
688 sethi %hi(global_cpu_snapsho
689 or %g1, %lo(global_cpu_sn
690 __GET_CPUID(%g2)
691 sllx %g2, 6, %g3
692 add %g1, %g3, %g1
693 rd %pic, %g7
694 stx %g7, [%g1 + (4 * 8)]
695 rd %pcr, %g7
696 stx %g7, [%g1 + (0 * 8)]
697 retry
698
699 .globl xcall_fetch_glob_pmu_n
700 xcall_fetch_glob_pmu_n4:
701 sethi %hi(global_cpu_snapsho
702 or %g1, %lo(global_cpu_sn
703 __GET_CPUID(%g2)
704 sllx %g2, 6, %g3
705 add %g1, %g3, %g1
706
707 ldxa [%g0] ASI_PIC, %g7
708 stx %g7, [%g1 + (4 * 8)]
709 mov 0x08, %g3
710 ldxa [%g3] ASI_PIC, %g7
711 stx %g7, [%g1 + (5 * 8)]
712 mov 0x10, %g3
713 ldxa [%g3] ASI_PIC, %g7
714 stx %g7, [%g1 + (6 * 8)]
715 mov 0x18, %g3
716 ldxa [%g3] ASI_PIC, %g7
717 stx %g7, [%g1 + (7 * 8)]
718
719 mov %o0, %g2
720 mov %o1, %g3
721 mov %o5, %g7
722
723 mov HV_FAST_VT_GET_PERFREG
724 mov 3, %o0
725 ta HV_FAST_TRAP
726 stx %o1, [%g1 + (3 * 8)]
727 mov HV_FAST_VT_GET_PERFREG
728 mov 2, %o0
729 ta HV_FAST_TRAP
730 stx %o1, [%g1 + (2 * 8)]
731 mov HV_FAST_VT_GET_PERFREG
732 mov 1, %o0
733 ta HV_FAST_TRAP
734 stx %o1, [%g1 + (1 * 8)]
735 mov HV_FAST_VT_GET_PERFREG
736 mov 0, %o0
737 ta HV_FAST_TRAP
738 stx %o1, [%g1 + (0 * 8)]
739
740 mov %g2, %o0
741 mov %g3, %o1
742 mov %g7, %o5
743
744 retry
745
746 __cheetah_xcall_flush_tlb_kernel_range: /* 44
747 sethi %hi(PAGE_SIZE - 1), %g
748 or %g2, %lo(PAGE_SIZE - 1
749 andn %g1, %g2, %g1
750 andn %g7, %g2, %g7
751 sub %g7, %g1, %g3
752 srlx %g3, 18, %g2
753 brnz,pn %g2, 2f
754 sethi %hi(PAGE_SIZE), %g2
755 sub %g3, %g2, %g3
756 or %g1, 0x20, %g1
757 1: stxa %g0, [%g1 + %g3] ASI_D
758 stxa %g0, [%g1 + %g3] ASI_I
759 membar #Sync
760 brnz,pt %g3, 1b
761 sub %g3, %g2, %g3
762 retry
763 2: mov 0x80, %g2
764 stxa %g0, [%g2] ASI_DMMU_DE
765 membar #Sync
766 stxa %g0, [%g2] ASI_IMMU_DE
767 membar #Sync
768 retry
769 nop
770 nop
771 nop
772 nop
773 nop
774 nop
775 nop
776 nop
777 nop
778 nop
779 nop
780 nop
781 nop
782 nop
783 nop
784 nop
785 nop
786 nop
787 nop
788 nop
789 nop
790 nop
791
792 #ifdef DCACHE_ALIASING_POSSIBLE
793 .align 32
794 .globl xcall_flush_dcache_pag
795 xcall_flush_dcache_page_cheetah: /* %g1 == phy
796 sethi %hi(PAGE_SIZE), %g3
797 1: subcc %g3, (1 << 5), %g3
798 stxa %g0, [%g1 + %g3] ASI_D
799 membar #Sync
800 bne,pt %icc, 1b
801 nop
802 retry
803 nop
804 #endif /* DCACHE_ALIASING_POSSIBLE */
805
806 .globl xcall_flush_dcache_pag
807 xcall_flush_dcache_page_spitfire: /* %g1 == ph
808 %g7 == ke
809 %g5 == (p
810 #ifdef DCACHE_ALIASING_POSSIBLE
811 srlx %g1, (13 - 2), %g1
812 sethi %hi(L1DCACHE_SIZE), %g
813 sub %g3, (1 << 5), %g3
814 1: ldxa [%g3] ASI_DCACHE_TAG,
815 andcc %g2, 0x3, %g0
816 be,pn %xcc, 2f
817 andn %g2, 0x3, %g2
818 cmp %g2, %g1
819
820 bne,pt %xcc, 2f
821 nop
822 stxa %g0, [%g3] ASI_DCACHE_
823 membar #Sync
824 2: cmp %g3, 0
825 bne,pt %xcc, 1b
826 sub %g3, (1 << 5), %g3
827
828 brz,pn %g5, 2f
829 #endif /* DCACHE_ALIASING_POSSIBLE */
830 sethi %hi(PAGE_SIZE), %g3
831
832 1: flush %g7
833 subcc %g3, (1 << 5), %g3
834 bne,pt %icc, 1b
835 add %g7, (1 << 5), %g7
836
837 2: retry
838 nop
839 nop
840
841 /* %g5: error
842 * %g6: tlb op
843 */
844 __hypervisor_tlb_xcall_error:
845 mov %g5, %g4
846 mov %g6, %g5
847 ba,pt %xcc, etrap
848 rd %pc, %g7
849 mov %l4, %o0
850 call hypervisor_tlbop_error_xcall
851 mov %l5, %o1
852 ba,a,pt %xcc, rtrap
853
854 .globl __hypervisor_xcall_flu
855 __hypervisor_xcall_flush_tlb_mm: /* 24 insns *
856 /* %g5=ctx, g1,g2,g3,g4,g7=scratch, %g
857 mov %o0, %g2
858 mov %o1, %g3
859 mov %o2, %g4
860 mov %o3, %g1
861 mov %o5, %g7
862 clr %o0 /* ARG
863 clr %o1 /* ARG
864 mov %g5, %o2 /* ARG
865 mov HV_MMU_ALL, %o3 /* ARG
866 mov HV_FAST_MMU_DEMAP_CTX,
867 ta HV_FAST_TRAP
868 mov HV_FAST_MMU_DEMAP_CTX,
869 brnz,pn %o0, 1f
870 mov %o0, %g5
871 mov %g2, %o0
872 mov %g3, %o1
873 mov %g4, %o2
874 mov %g1, %o3
875 mov %g7, %o5
876 membar #Sync
877 retry
878 1: sethi %hi(__hypervisor_tlb_x
879 jmpl %g4 + %lo(__hypervisor
880 nop
881
882 .globl __hypervisor_xcall_flu
883 __hypervisor_xcall_flush_tlb_page: /* 20 insns
884 /* %g5=ctx, %g1=vaddr */
885 mov %o0, %g2
886 mov %o1, %g3
887 mov %o2, %g4
888 mov %g1, %o0
889 mov %g5, %o1
890 mov HV_MMU_ALL, %o2
891 srlx %o0, PAGE_SHIFT, %o0
892 sllx %o0, PAGE_SHIFT, %o0
893 ta HV_MMU_UNMAP_ADDR_TRAP
894 mov HV_MMU_UNMAP_ADDR_TRAP
895 brnz,a,pn %o0, 1f
896 mov %o0, %g5
897 mov %g2, %o0
898 mov %g3, %o1
899 mov %g4, %o2
900 membar #Sync
901 retry
902 1: sethi %hi(__hypervisor_tlb_x
903 jmpl %g4 + %lo(__hypervisor
904 nop
905
906 .globl __hypervisor_xcall_flu
907 __hypervisor_xcall_flush_tlb_kernel_range: /*
908 /* %g1=start, %g7=end, g2,g3,g4,g5,g6=
909 sethi %hi(PAGE_SIZE - 1), %g
910 or %g2, %lo(PAGE_SIZE - 1
911 andn %g1, %g2, %g1
912 andn %g7, %g2, %g7
913 sub %g7, %g1, %g3
914 srlx %g3, 18, %g7
915 add %g2, 1, %g2
916 sub %g3, %g2, %g3
917 mov %o0, %g2
918 mov %o1, %g4
919 brnz,pn %g7, 2f
920 mov %o2, %g7
921 1: add %g1, %g3, %o0 /* ARG
922 mov 0, %o1 /* ARG
923 mov HV_MMU_ALL, %o2 /* ARG
924 ta HV_MMU_UNMAP_ADDR_TRAP
925 mov HV_MMU_UNMAP_ADDR_TRAP
926 brnz,pn %o0, 1f
927 mov %o0, %g5
928 sethi %hi(PAGE_SIZE), %o2
929 brnz,pt %g3, 1b
930 sub %g3, %o2, %g3
931 5: mov %g2, %o0
932 mov %g4, %o1
933 mov %g7, %o2
934 membar #Sync
935 retry
936 1: sethi %hi(__hypervisor_tlb_x
937 jmpl %g4 + %lo(__hypervisor
938 nop
939 2: mov %o3, %g1
940 mov %o5, %g3
941 mov 0, %o0 /* ARG
942 mov 0, %o1 /* ARG
943 mov 0, %o2 /* ARG
944 mov HV_MMU_ALL, %o3 /* ARG
945 mov HV_FAST_MMU_DEMAP_CTX,
946 ta HV_FAST_TRAP
947 mov %g1, %o3
948 brz,pt %o0, 5b
949 mov %g3, %o5
950 mov HV_FAST_MMU_DEMAP_CTX,
951 ba,pt %xcc, 1b
952 clr %g5
953
954 /* These just get rescheduled to PIL v
955 .globl xcall_call_function
956 xcall_call_function:
957 wr %g0, (1 << PIL_SMP_CAL
958 retry
959
960 .globl xcall_call_function_si
961 xcall_call_function_single:
962 wr %g0, (1 << PIL_SMP_CAL
963 retry
964
965 .globl xcall_receive_signal
966 xcall_receive_signal:
967 wr %g0, (1 << PIL_SMP_REC
968 retry
969
970 .globl xcall_capture
971 xcall_capture:
972 wr %g0, (1 << PIL_SMP_CAP
973 retry
974
975 #ifdef CONFIG_KGDB
976 .globl xcall_kgdb_capture
977 xcall_kgdb_capture:
978 wr %g0, (1 << PIL_KGDB_CA
979 retry
980 #endif
981
982 #endif /* CONFIG_SMP */
983
984 .globl cheetah_patch_cachetlb
985 cheetah_patch_cachetlbops:
986 save %sp, -128, %sp
987
988 sethi %hi(__flush_tlb_mm), %
989 or %o0, %lo(__flush_tlb_m
990 sethi %hi(__cheetah_flush_tl
991 or %o1, %lo(__cheetah_flu
992 call tlb_patch_one
993 mov 19, %o2
994
995 sethi %hi(__flush_tlb_page),
996 or %o0, %lo(__flush_tlb_p
997 sethi %hi(__cheetah_flush_tl
998 or %o1, %lo(__cheetah_flu
999 call tlb_patch_one
1000 mov 22, %o2
1001
1002 sethi %hi(__flush_tlb_pendi
1003 or %o0, %lo(__flush_tlb_
1004 sethi %hi(__cheetah_flush_t
1005 or %o1, %lo(__cheetah_fl
1006 call tlb_patch_one
1007 mov 27, %o2
1008
1009 sethi %hi(__flush_tlb_kerne
1010 or %o0, %lo(__flush_tlb_
1011 sethi %hi(__cheetah_flush_t
1012 or %o1, %lo(__cheetah_fl
1013 call tlb_patch_one
1014 mov 31, %o2
1015
1016 #ifdef DCACHE_ALIASING_POSSIBLE
1017 sethi %hi(__flush_dcache_pa
1018 or %o0, %lo(__flush_dcac
1019 sethi %hi(__cheetah_flush_d
1020 or %o1, %lo(__cheetah_fl
1021 call tlb_patch_one
1022 mov 11, %o2
1023 #endif /* DCACHE_ALIASING_POSSIBLE */
1024
1025 #ifdef CONFIG_SMP
1026 sethi %hi(xcall_flush_tlb_k
1027 or %o0, %lo(xcall_flush_
1028 sethi %hi(__cheetah_xcall_f
1029 or %o1, %lo(__cheetah_xc
1030 call tlb_patch_one
1031 mov 44, %o2
1032 #endif /* CONFIG_SMP */
1033
1034 ret
1035 restore
1036
1037 .globl hypervisor_patch_cach
1038 hypervisor_patch_cachetlbops:
1039 save %sp, -128, %sp
1040
1041 sethi %hi(__flush_tlb_mm),
1042 or %o0, %lo(__flush_tlb_
1043 sethi %hi(__hypervisor_flus
1044 or %o1, %lo(__hypervisor
1045 call tlb_patch_one
1046 mov 19, %o2
1047
1048 sethi %hi(__flush_tlb_page)
1049 or %o0, %lo(__flush_tlb_
1050 sethi %hi(__hypervisor_flus
1051 or %o1, %lo(__hypervisor
1052 call tlb_patch_one
1053 mov 22, %o2
1054
1055 sethi %hi(__flush_tlb_pendi
1056 or %o0, %lo(__flush_tlb_
1057 sethi %hi(__hypervisor_flus
1058 or %o1, %lo(__hypervisor
1059 call tlb_patch_one
1060 mov 27, %o2
1061
1062 sethi %hi(__flush_tlb_kerne
1063 or %o0, %lo(__flush_tlb_
1064 sethi %hi(__hypervisor_flus
1065 or %o1, %lo(__hypervisor
1066 call tlb_patch_one
1067 mov 31, %o2
1068
1069 #ifdef DCACHE_ALIASING_POSSIBLE
1070 sethi %hi(__flush_dcache_pa
1071 or %o0, %lo(__flush_dcac
1072 sethi %hi(__hypervisor_flus
1073 or %o1, %lo(__hypervisor
1074 call tlb_patch_one
1075 mov 2, %o2
1076 #endif /* DCACHE_ALIASING_POSSIBLE */
1077
1078 #ifdef CONFIG_SMP
1079 sethi %hi(xcall_flush_tlb_m
1080 or %o0, %lo(xcall_flush_
1081 sethi %hi(__hypervisor_xcal
1082 or %o1, %lo(__hypervisor
1083 call tlb_patch_one
1084 mov 24, %o2
1085
1086 sethi %hi(xcall_flush_tlb_p
1087 or %o0, %lo(xcall_flush_
1088 sethi %hi(__hypervisor_xcal
1089 or %o1, %lo(__hypervisor
1090 call tlb_patch_one
1091 mov 20, %o2
1092
1093 sethi %hi(xcall_flush_tlb_k
1094 or %o0, %lo(xcall_flush_
1095 sethi %hi(__hypervisor_xcal
1096 or %o1, %lo(__hypervisor
1097 call tlb_patch_one
1098 mov 44, %o2
1099 #endif /* CONFIG_SMP */
1100
1101 ret
1102 restore
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.