1 /* SPDX-License-Identifier: GPL-2.0-or-later *
2 /*
3 * PowerPC version
4 * Copyright (C) 1995-1996 Gary Thomas (gdt
5 * Rewritten by Cort Dougan (cort@cs.nmt.edu)
6 * Copyright (C) 1996 Cort Dougan <cort@cs.n
7 * Adapted for Power Macintosh by Paul Macker
8 * Low-level exception handlers and MMU suppo
9 * rewritten by Paul Mackerras.
10 * Copyright (C) 1996 Paul Mackerras.
11 *
12 * This file contains low-level assembler rou
13 * the PowerPC MMU hash table. (PPC 8xx proc
14 * hash table, so this file is not used on th
15 */
16
17 #include <linux/export.h>
18 #include <linux/pgtable.h>
19 #include <linux/init.h>
20 #include <asm/reg.h>
21 #include <asm/page.h>
22 #include <asm/cputable.h>
23 #include <asm/ppc_asm.h>
24 #include <asm/thread_info.h>
25 #include <asm/asm-offsets.h>
26 #include <asm/feature-fixups.h>
27 #include <asm/code-patching-asm.h>
28
29 #ifdef CONFIG_PTE_64BIT
30 #define PTE_T_SIZE 8
31 #define PTE_FLAGS_OFFSET 4 /* off
32 #else
33 #define PTE_T_SIZE 4
34 #define PTE_FLAGS_OFFSET 0
35 #endif
36
37 /*
38 * Load a PTE into the hash table, if possible
39 * The address is in r4, and r3 contains requi
40 * - For ISI: _PAGE_PRESENT | _PAGE_EXEC
41 * - For DSI: _PAGE_PRESENT | _PAGE_READ | _
42 * r9 contains the SRR1 value, from which we u
43 * SPRG_THREAD contains the physical address o
44 *
45 * Returns to the caller if the access is ille
46 * mapping for the address. Otherwise it plac
47 * in the hash table and returns from the exce
48 * Uses r0, r3 - r6, r8, r10, ctr, lr.
49 */
50 .text
51 _GLOBAL(hash_page)
52 #ifdef CONFIG_SMP
53 lis r8, (mmu_hash_lock - PAGE_OFFS
54 ori r8, r8, (mmu_hash_lock - PAGE_
55 lis r0,0x0fff
56 b 10f
57 11: lwz r6,0(r8)
58 cmpwi 0,r6,0
59 bne 11b
60 10: lwarx r6,0,r8
61 cmpwi 0,r6,0
62 bne- 11b
63 stwcx. r0,0,r8
64 bne- 10b
65 isync
66 #endif
67 /* Get PTE (linux-style) and check acc
68 lis r0, TASK_SIZE@h /* che
69 cmplw 0,r4,r0
70 mfspr r8,SPRN_SPRG_THREAD /* cur
71 lwz r5,PGDIR(r8) /* vir
72 blt+ 112f /* ass
73 lis r5,swapper_pg_dir@ha /* if
74 andi. r0,r9,MSR_PR /* Che
75 addi r5,r5,swapper_pg_dir@l /* ker
76 #ifdef CONFIG_SMP
77 bne- .Lhash_page_out /* ret
78 #else
79 bnelr-
80 #endif
81 112: tophys(r5, r5)
82 #ifndef CONFIG_PTE_64BIT
83 rlwimi r5,r4,12,20,29 /* ins
84 lwz r8,0(r5) /* get
85 rlwinm. r8,r8,0,0,19 /* ext
86 #else
87 rlwinm r8,r4,13,19,29 /* Com
88 lwzx r8,r8,r5 /* Get
89 rlwinm. r8,r8,0,0,20 /* ext
90 #endif
91 #ifdef CONFIG_SMP
92 beq- .Lhash_page_out /* ret
93 #else
94 /* XXX it seems like the 601 will give
95 rfi if its alignment is wrong (bott
96 8 or 0xc) and we have had a not-tak
97 to the address following the rfi. *
98 beqlr-
99 #endif
100 #ifndef CONFIG_PTE_64BIT
101 rlwimi r8,r4,22,20,29 /* ins
102 #else
103 rlwimi r8,r4,23,20,28 /* com
104 /*
105 * If PTE_64BIT is set, the low word i
106 * word for locking since it contains
107 */
108 addi r8,r8,PTE_FLAGS_OFFSET
109 #endif
110
111 /*
112 * Update the linux PTE atomically. W
113 * because almost always, there won't
114 * and there won't already be an HPTE,
115 * to update the PTE to set _PAGE_HASH
116 */
117 .Lretry:
118 lwarx r6,0,r8 /* get
119 #ifdef CONFIG_PPC_KUAP
120 mfsrin r5,r4
121 rlwinm r0,r9,28,_PAGE_WRITE /* MSR
122 rlwinm r5,r5,12,_PAGE_WRITE /* Ks
123 andc r5,r5,r0 /* Ks
124 andc r5,r6,r5 /* Cle
125 andc. r5,r3,r5 /* che
126 #else
127 andc. r5,r3,r6 /* che
128 #endif
129 rlwinm r0,r3,32-3,24,24 /* _PA
130 ori r0,r0,_PAGE_ACCESSED|_PAGE_HAS
131 #ifdef CONFIG_SMP
132 bne- .Lhash_page_out /* ret
133 #else
134 bnelr-
135 #endif
136 or r5,r0,r6 /* set
137 #ifdef CONFIG_PTE_64BIT
138 #ifdef CONFIG_SMP
139 subf r10,r6,r8 /* cre
140 subi r10,r10,PTE_FLAGS_OFFSET
141 lwzx r10,r6,r10 /* Get
142 #else
143 lwz r10,-PTE_FLAGS_OFFSET(r8)
144 #endif /* CONFIG_SMP */
145 #endif /* CONFIG_PTE_64BIT */
146 stwcx. r5,0,r8 /* att
147 bne- .Lretry /* ret
148
149 mfsrin r3,r4 /* get
150 bl create_hpte /* add
151
152 #ifdef CONFIG_SMP
153 eieio
154 lis r8, (mmu_hash_lock - PAGE_OFFS
155 li r0,0
156 stw r0, (mmu_hash_lock - PAGE_OFFS
157 #endif
158 b fast_hash_page_return
159
160 #ifdef CONFIG_SMP
161 .Lhash_page_out:
162 eieio
163 lis r8, (mmu_hash_lock - PAGE_OFFS
164 li r0,0
165 stw r0, (mmu_hash_lock - PAGE_OFFS
166 blr
167 #endif /* CONFIG_SMP */
168 _ASM_NOKPROBE_SYMBOL(hash_page)
169
170 /*
171 * Add an entry for a particular page to the h
172 *
173 * add_hash_page(unsigned context, unsigned lo
174 *
175 * We assume any necessary modifications to th
176 * the accessed bit) have already been done an
177 * a hash table in use (i.e. we're not on a 60
178 */
179 _GLOBAL(add_hash_page)
180 mflr r0
181 stw r0,4(r1)
182
183 #ifdef CONFIG_SMP
184 lwz r8,TASK_CPU(r2) /* to
185 oris r8,r8,12
186 #endif /* CONFIG_SMP */
187
188 /*
189 * We disable interrupts here, even on
190 * want to race with hash_page, and be
191 * _PAGE_HASHPTE bit to be a reliable
192 * the HPTE exists (or at least whethe
193 * We also turn off the MMU for data a
194 * we can't take a hash table miss (as
195 * covered by a BAT). -- paulus
196 */
197 mfmsr r9
198 rlwinm r0,r9,0,17,15 /* cle
199 rlwinm r0,r0,0,28,26 /* cle
200 mtmsr r0
201 isync
202
203 #ifdef CONFIG_SMP
204 lis r6, (mmu_hash_lock - PAGE_OFFS
205 addi r6, r6, (mmu_hash_lock - PAGE_
206 10: lwarx r0,0,r6 /* tak
207 cmpwi 0,r0,0
208 bne- 11f
209 stwcx. r8,0,r6
210 beq+ 12f
211 11: lwz r0,0(r6)
212 cmpwi 0,r0,0
213 beq 10b
214 b 11b
215 12: isync
216 #endif
217
218 /*
219 * Fetch the linux pte and test and se
220 * If _PAGE_HASHPTE was already set, w
221 * HPTE, so we just unlock and return.
222 */
223 mr r8,r5
224 #ifndef CONFIG_PTE_64BIT
225 rlwimi r8,r4,22,20,29
226 #else
227 rlwimi r8,r4,23,20,28
228 addi r8,r8,PTE_FLAGS_OFFSET
229 #endif
230 1: lwarx r6,0,r8
231 andi. r0,r6,_PAGE_HASHPTE
232 bne 9f /* if
233 #ifdef CONFIG_PTE_64BIT
234 #ifdef CONFIG_SMP
235 subf r10,r6,r8 /* cre
236 subi r10,r10,PTE_FLAGS_OFFSET
237 lwzx r10,r6,r10 /* Get
238 #else
239 lwz r10,-PTE_FLAGS_OFFSET(r8)
240 #endif /* CONFIG_SMP */
241 #endif /* CONFIG_PTE_64BIT */
242 ori r5,r6,_PAGE_HASHPTE
243 stwcx. r5,0,r8
244 bne- 1b
245
246 /* Convert context and va to VSID */
247 mulli r3,r3,897*16 /* mul
248 rlwinm r0,r4,4,28,31 /* get
249 mulli r0,r0,0x111 /* mul
250 add r3,r3,r0 /* not
251
252 bl create_hpte
253
254 9:
255 #ifdef CONFIG_SMP
256 lis r6, (mmu_hash_lock - PAGE_OFFS
257 addi r6, r6, (mmu_hash_lock - PAGE_
258 eieio
259 li r0,0
260 stw r0,0(r6) /* cle
261 #endif
262
263 /* reenable interrupts and DR */
264 mtmsr r9
265 isync
266
267 lwz r0,4(r1)
268 mtlr r0
269 blr
270 _ASM_NOKPROBE_SYMBOL(add_hash_page)
271
272 /*
273 * This routine adds a hardware PTE to the has
274 * It is designed to be called with the MMU ei
275 * r3 contains the VSID, r4 contains the virtu
276 * r5 contains the linux PTE, r6 contains the
277 * linux PTE (before setting _PAGE_HASHPTE). r
278 * upper half of the PTE if CONFIG_PTE_64BIT.
279 * On SMP, the caller should have the mmu_hash
280 * We assume that the caller has (or will) set
281 * bit in the linux PTE in memory. The value
282 * be the old linux PTE value; if it doesn't h
283 * this routine will skip the search for an ex
284 * This procedure modifies r0, r3 - r6, r8, cr
285 * -- paulus.
286 *
287 * For speed, 4 of the instructions get patche
288 * physical address of the hash table are know
289 * of Hash_base and Hash_bits below are for th
290 */
291 Hash_base = early_hash
292 Hash_bits = 12 /* e.g
293 Hash_msk = (((1 << Hash_bits) - 1) * 64)
294
295 /* defines for the PTE format for 32-bit PPCs
296 #define HPTE_SIZE 8
297 #define PTEG_SIZE 64
298 #define LG_PTEG_SIZE 6
299 #define LDPTEu lwzu
300 #define LDPTE lwz
301 #define STPTE stw
302 #define CMPPTE cmpw
303 #define PTE_H 0x40
304 #define PTE_V 0x80000000
305 #define TST_V(r) rlwinm. r,r,0,0,0
306 #define SET_V(r) oris r,r,PTE_V@h
307 #define CLR_V(r,t) rlwinm r,r,0,1,31
308
309 #define HASH_LEFT 31-(LG_PTEG_SIZE+Hash_
310 #define HASH_RIGHT 31-LG_PTEG_SIZE
311
312 __REF
313 _GLOBAL(create_hpte)
314 /* Convert linux-style PTE (r5) to low
315 lis r0, TASK_SIZE@h
316 rlwinm r5,r5,0,~3 /* Cle
317 cmplw r4,r0
318 rlwinm r8,r5,32-9,30,30 /* _PA
319 rlwinm r0,r5,32-6,30,30 /* _PA
320 and r8,r8,r0 /* wri
321 bge- 1f /* Ker
322 ori r5,r5,3 /* Use
323 1: ori r8,r8,0xe04 /* cle
324 andc r8,r5,r8 /* PP
325 BEGIN_FTR_SECTION
326 rlwinm r8,r8,0,~_PAGE_COHERENT /* cle
327 END_FTR_SECTION_IFCLR(CPU_FTR_NEED_COHERENT)
328 #ifdef CONFIG_PTE_64BIT
329 /* Put the XPN bits into the PTE */
330 rlwimi r8,r10,8,20,22
331 rlwimi r8,r10,2,29,29
332 #endif
333
334 /* Construct the high word of the PPC-
335 rlwinm r5,r3,7,1,24 /* put
336 rlwimi r5,r4,10,26,31 /* put
337 SET_V(r5) /* set
338
339 patch_site 0f, patch__hash_page_A
340 patch_site 1f, patch__hash_page_A
341 patch_site 2f, patch__hash_page_A
342 /* Get the address of the primary PTE
343 0: lis r0, (Hash_base - PAGE_OFFSET)@
344 1: rlwimi r0,r3,LG_PTEG_SIZE,HASH_LEFT,H
345 2: rlwinm r3,r4,20+LG_PTEG_SIZE,HASH_LEF
346 xor r3,r3,r0 /* mak
347 li r0,8 /* PTE
348
349 /*
350 * Test the _PAGE_HASHPTE bit in the o
351 * if it is clear, meaning that the HP
352 */
353 andi. r6,r6,_PAGE_HASHPTE
354 beq+ 10f /* no
355 tlbie r4
356
357 /* Search the primary PTEG for a PTE w
358 mtctr r0
359 addi r4,r3,-HPTE_SIZE
360 1: LDPTEu r6,HPTE_SIZE(r4) /* get
361 CMPPTE 0,r6,r5
362 bdnzf 2,1b /* loo
363 beq+ .Lfound_slot
364
365 patch_site 0f, patch__hash_page_B
366 /* Search the secondary PTEG for a mat
367 ori r5,r5,PTE_H /* set
368 0: xoris r4,r3,Hash_msk>>16 /* com
369 xori r4,r4,(-PTEG_SIZE & 0xffff)
370 addi r4,r4,-HPTE_SIZE
371 mtctr r0
372 2: LDPTEu r6,HPTE_SIZE(r4)
373 CMPPTE 0,r6,r5
374 bdnzf 2,2b
375 beq+ .Lfound_slot
376 xori r5,r5,PTE_H /* cle
377
378 /* Search the primary PTEG for an empt
379 10: mtctr r0
380 addi r4,r3,-HPTE_SIZE /* sea
381 1: LDPTEu r6,HPTE_SIZE(r4) /* get
382 TST_V(r6) /* tes
383 bdnzf 2,1b /* loo
384 beq+ .Lfound_empty
385
386 patch_site 0f, patch__hash_page_C
387 /* Search the secondary PTEG for an em
388 ori r5,r5,PTE_H /* set
389 0: xoris r4,r3,Hash_msk>>16 /* com
390 xori r4,r4,(-PTEG_SIZE & 0xffff)
391 addi r4,r4,-HPTE_SIZE
392 mtctr r0
393 2: LDPTEu r6,HPTE_SIZE(r4)
394 TST_V(r6)
395 bdnzf 2,2b
396 beq+ .Lfound_empty
397 xori r5,r5,PTE_H /* cle
398
399 /*
400 * Choose an arbitrary slot in the pri
401 * Since both the primary and secondar
402 * have no information that the PTEs i
403 * more important or useful than those
404 * and we know there is a definite (al
405 * advantage to putting the PTE in the
406 * put the PTE in the primary PTEG.
407 */
408
409 lis r4, (next_slot - PAGE_OFFSET)@
410 lwz r6, (next_slot - PAGE_OFFSET)@
411 addi r6,r6,HPTE_SIZE
412 andi. r6,r6,7*HPTE_SIZE
413 stw r6,next_slot@l(r4)
414 add r4,r3,r6
415
416 #ifndef CONFIG_SMP
417 /* Store PTE in PTEG */
418 .Lfound_empty:
419 STPTE r5,0(r4)
420 .Lfound_slot:
421 STPTE r8,HPTE_SIZE/2(r4)
422
423 #else /* CONFIG_SMP */
424 /*
425 * Between the tlbie above and updating the ha
426 * another CPU could read the hash table entry
427 * There are 3 cases:
428 * 1. using an empty slot
429 * 2. updating an earlier entry to change perm
430 * 3. taking over the PTE for an unrelated add
431 *
432 * In each case it doesn't really matter if th
433 * PTE in their TLB. So we don't need to both
434 * which is convenient as we've overwritten th
435 * address. :-) The tlbie above is mainly to
436 * and gets the new PTE from the hash table.
437 *
438 * We do however have to make sure that the PT
439 * state with the V bit set.
440 */
441 .Lfound_empty:
442 .Lfound_slot:
443 CLR_V(r5,r0) /* clear V (va
444 STPTE r5,0(r4)
445 sync
446 TLBSYNC
447 STPTE r8,HPTE_SIZE/2(r4) /* put in c
448 sync
449 SET_V(r5)
450 STPTE r5,0(r4) /* finally set
451 #endif /* CONFIG_SMP */
452
453 sync /* make sure pte updat
454 blr
455 .previous
456 _ASM_NOKPROBE_SYMBOL(create_hpte)
457
458 .section .bss
459 .align 2
460 next_slot:
461 .space 4
462 .previous
463
464 /*
465 * Flush the entry for a particular page from
466 *
467 * flush_hash_pages(unsigned context, unsigned
468 * int count)
469 *
470 * We assume that there is a hash table in use
471 */
472 __REF
473 _GLOBAL(flush_hash_pages)
474 /*
475 * We disable interrupts here, even on
476 * the _PAGE_HASHPTE bit to be a relia
477 * whether the HPTE exists (or at leas
478 * We also turn off the MMU for data a
479 * we can't take a hash table miss (as
480 * covered by a BAT). -- paulus
481 */
482 mfmsr r10
483 rlwinm r0,r10,0,17,15 /* cle
484 rlwinm r0,r0,0,28,26 /* cle
485 mtmsr r0
486 isync
487
488 /* First find a PTE in the range that
489 #ifndef CONFIG_PTE_64BIT
490 rlwimi r5,r4,22,20,29
491 #else
492 rlwimi r5,r4,23,20,28
493 addi r5,r5,PTE_FLAGS_OFFSET
494 #endif
495 1: lwz r0,0(r5)
496 cmpwi cr1,r6,1
497 andi. r0,r0,_PAGE_HASHPTE
498 bne 2f
499 ble cr1,19f
500 addi r4,r4,0x1000
501 addi r5,r5,PTE_T_SIZE
502 addi r6,r6,-1
503 b 1b
504
505 /* Convert context and va to VSID */
506 2: mulli r3,r3,897*16 /* mul
507 rlwinm r0,r4,4,28,31 /* get
508 mulli r0,r0,0x111 /* mul
509 add r3,r3,r0 /* not
510
511 /* Construct the high word of the PPC-
512 rlwinm r11,r3,7,1,24 /* put
513 rlwimi r11,r4,10,26,31 /* put
514 SET_V(r11) /* set
515
516 #ifdef CONFIG_SMP
517 lis r9, (mmu_hash_lock - PAGE_OFFS
518 addi r9, r9, (mmu_hash_lock - PAGE_
519 tophys (r8, r2)
520 lwz r8, TASK_CPU(r8)
521 oris r8,r8,9
522 10: lwarx r0,0,r9
523 cmpwi 0,r0,0
524 bne- 11f
525 stwcx. r8,0,r9
526 beq+ 12f
527 11: lwz r0,0(r9)
528 cmpwi 0,r0,0
529 beq 10b
530 b 11b
531 12: isync
532 #endif
533
534 /*
535 * Check the _PAGE_HASHPTE bit in the
536 * already clear, we're done (for this
537 * clear it (atomically) and proceed.
538 */
539 33: lwarx r8,0,r5 /* fet
540 andi. r0,r8,_PAGE_HASHPTE
541 beq 8f /* don
542 rlwinm r8,r8,0,31,29 /* cle
543 stwcx. r8,0,r5 /* upd
544 bne- 33b
545
546 patch_site 0f, patch__flush_hash_
547 patch_site 1f, patch__flush_hash_
548 patch_site 2f, patch__flush_hash_
549 /* Get the address of the primary PTE
550 0: lis r8, (Hash_base - PAGE_OFFSET)@
551 1: rlwimi r8,r3,LG_PTEG_SIZE,HASH_LEFT,H
552 2: rlwinm r0,r4,20+LG_PTEG_SIZE,HASH_LEF
553 xor r8,r0,r8 /* mak
554
555 /* Search the primary PTEG for a PTE w
556 li r0,8 /* PTE
557 mtctr r0
558 addi r12,r8,-HPTE_SIZE
559 1: LDPTEu r0,HPTE_SIZE(r12) /* get
560 CMPPTE 0,r0,r11
561 bdnzf 2,1b /* loo
562 beq+ 3f
563
564 patch_site 0f, patch__flush_hash_
565 /* Search the secondary PTEG for a mat
566 ori r11,r11,PTE_H /* set
567 li r0,8 /* PTE
568 0: xoris r12,r8,Hash_msk>>16 /* com
569 xori r12,r12,(-PTEG_SIZE & 0xffff)
570 addi r12,r12,-HPTE_SIZE
571 mtctr r0
572 2: LDPTEu r0,HPTE_SIZE(r12)
573 CMPPTE 0,r0,r11
574 bdnzf 2,2b
575 xori r11,r11,PTE_H /* cle
576 bne- 4f /* sho
577
578 3: li r0,0
579 STPTE r0,0(r12) /* inv
580 4: sync
581 tlbie r4 /* in
582 sync
583
584 8: ble cr1,9f /* if
585 81: addi r6,r6,-1
586 addi r5,r5,PTE_T_SIZE
587 addi r4,r4,0x1000
588 lwz r0,0(r5) /* che
589 cmpwi cr1,r6,1
590 andi. r0,r0,_PAGE_HASHPTE
591 bne 33b
592 bgt cr1,81b
593
594 9:
595 #ifdef CONFIG_SMP
596 TLBSYNC
597 li r0,0
598 stw r0,0(r9) /* cle
599 #endif
600
601 19: mtmsr r10
602 isync
603 blr
604 .previous
605 EXPORT_SYMBOL(flush_hash_pages)
606 _ASM_NOKPROBE_SYMBOL(flush_hash_pages)
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.