1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * Copyright (C) 2012 ARM Ltd.
4 */
5 #ifndef __ASM_PGTABLE_H
6 #define __ASM_PGTABLE_H
7
8 #include <asm/bug.h>
9 #include <asm/proc-fns.h>
10
11 #include <asm/memory.h>
12 #include <asm/mte.h>
13 #include <asm/pgtable-hwdef.h>
14 #include <asm/pgtable-prot.h>
15 #include <asm/tlbflush.h>
16
17 /*
18 * VMALLOC range.
19 *
20 * VMALLOC_START: beginning of the kernel vmalloc space
21 * VMALLOC_END: extends to the available space below vmemmap
22 */
23 #define VMALLOC_START (MODULES_END)
24 #if VA_BITS == VA_BITS_MIN
25 #define VMALLOC_END (VMEMMAP_START - SZ_8M)
26 #else
27 #define VMEMMAP_UNUSED_NPAGES ((_PAGE_OFFSET(vabits_actual) - PAGE_OFFSET) >> PAGE_SHIFT)
28 #define VMALLOC_END (VMEMMAP_START + VMEMMAP_UNUSED_NPAGES * sizeof(struct page) - SZ_8M)
29 #endif
30
31 #define vmemmap ((struct page *)VMEMMAP_START - (memstart_addr >> PAGE_SHIFT))
32
33 #ifndef __ASSEMBLY__
34
35 #include <asm/cmpxchg.h>
36 #include <asm/fixmap.h>
37 #include <linux/mmdebug.h>
38 #include <linux/mm_types.h>
39 #include <linux/sched.h>
40 #include <linux/page_table_check.h>
41
42 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
43 #define __HAVE_ARCH_FLUSH_PMD_TLB_RANGE
44
45 /* Set stride and tlb_level in flush_*_tlb_range */
46 #define flush_pmd_tlb_range(vma, addr, end) \
47 __flush_tlb_range(vma, addr, end, PMD_SIZE, false, 2)
48 #define flush_pud_tlb_range(vma, addr, end) \
49 __flush_tlb_range(vma, addr, end, PUD_SIZE, false, 1)
50 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
51
52 /*
53 * Outside of a few very special situations (e.g. hibernation), we always
54 * use broadcast TLB invalidation instructions, therefore a spurious page
55 * fault on one CPU which has been handled concurrently by another CPU
56 * does not need to perform additional invalidation.
57 */
58 #define flush_tlb_fix_spurious_fault(vma, address, ptep) do { } while (0)
59
60 /*
61 * ZERO_PAGE is a global shared page that is always zero: used
62 * for zero-mapped memory areas etc..
63 */
64 extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)];
65 #define ZERO_PAGE(vaddr) phys_to_page(__pa_symbol(empty_zero_page))
66
67 #define pte_ERROR(e) \
68 pr_err("%s:%d: bad pte %016llx.\n", __FILE__, __LINE__, pte_val(e))
69
70 /*
71 * Macros to convert between a physical address and its placement in a
72 * page table entry, taking care of 52-bit addresses.
73 */
74 #ifdef CONFIG_ARM64_PA_BITS_52
75 static inline phys_addr_t __pte_to_phys(pte_t pte)
76 {
77 pte_val(pte) &= ~PTE_MAYBE_SHARED;
78 return (pte_val(pte) & PTE_ADDR_LOW) |
79 ((pte_val(pte) & PTE_ADDR_HIGH) << PTE_ADDR_HIGH_SHIFT);
80 }
81 static inline pteval_t __phys_to_pte_val(phys_addr_t phys)
82 {
83 return (phys | (phys >> PTE_ADDR_HIGH_SHIFT)) & PHYS_TO_PTE_ADDR_MASK;
84 }
85 #else
86 #define __pte_to_phys(pte) (pte_val(pte) & PTE_ADDR_LOW)
87 #define __phys_to_pte_val(phys) (phys)
88 #endif
89
90 #define pte_pfn(pte) (__pte_to_phys(pte) >> PAGE_SHIFT)
91 #define pfn_pte(pfn,prot) \
92 __pte(__phys_to_pte_val((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot))
93
94 #define pte_none(pte) (!pte_val(pte))
95 #define __pte_clear(mm, addr, ptep) \
96 __set_pte(ptep, __pte(0))
97 #define pte_page(pte) (pfn_to_page(pte_pfn(pte)))
98
99 /*
100 * The following only work if pte_present(). Undefined behaviour otherwise.
101 */
102 #define pte_present(pte) (pte_valid(pte) || pte_present_invalid(pte))
103 #define pte_young(pte) (!!(pte_val(pte) & PTE_AF))
104 #define pte_special(pte) (!!(pte_val(pte) & PTE_SPECIAL))
105 #define pte_write(pte) (!!(pte_val(pte) & PTE_WRITE))
106 #define pte_rdonly(pte) (!!(pte_val(pte) & PTE_RDONLY))
107 #define pte_user(pte) (!!(pte_val(pte) & PTE_USER))
108 #define pte_user_exec(pte) (!(pte_val(pte) & PTE_UXN))
109 #define pte_cont(pte) (!!(pte_val(pte) & PTE_CONT))
110 #define pte_devmap(pte) (!!(pte_val(pte) & PTE_DEVMAP))
111 #define pte_tagged(pte) ((pte_val(pte) & PTE_ATTRINDX_MASK) == \
112 PTE_ATTRINDX(MT_NORMAL_TAGGED))
113
114 #define pte_cont_addr_end(addr, end) \
115 ({ unsigned long __boundary = ((addr) + CONT_PTE_SIZE) & CONT_PTE_MASK; \
116 (__boundary - 1 < (end) - 1) ? __boundary : (end); \
117 })
118
119 #define pmd_cont_addr_end(addr, end) \
120 ({ unsigned long __boundary = ((addr) + CONT_PMD_SIZE) & CONT_PMD_MASK; \
121 (__boundary - 1 < (end) - 1) ? __boundary : (end); \
122 })
123
124 #define pte_hw_dirty(pte) (pte_write(pte) && !pte_rdonly(pte))
125 #define pte_sw_dirty(pte) (!!(pte_val(pte) & PTE_DIRTY))
126 #define pte_dirty(pte) (pte_sw_dirty(pte) || pte_hw_dirty(pte))
127
128 #define pte_valid(pte) (!!(pte_val(pte) & PTE_VALID))
129 #define pte_present_invalid(pte) \
130 ((pte_val(pte) & (PTE_VALID | PTE_PRESENT_INVALID)) == PTE_PRESENT_INVALID)
131 /*
132 * Execute-only user mappings do not have the PTE_USER bit set. All valid
133 * kernel mappings have the PTE_UXN bit set.
134 */
135 #define pte_valid_not_user(pte) \
136 ((pte_val(pte) & (PTE_VALID | PTE_USER | PTE_UXN)) == (PTE_VALID | PTE_UXN))
137 /*
138 * Returns true if the pte is valid and has the contiguous bit set.
139 */
140 #define pte_valid_cont(pte) (pte_valid(pte) && pte_cont(pte))
141 /*
142 * Could the pte be present in the TLB? We must check mm_tlb_flush_pending
143 * so that we don't erroneously return false for pages that have been
144 * remapped as PROT_NONE but are yet to be flushed from the TLB.
145 * Note that we can't make any assumptions based on the state of the access
146 * flag, since __ptep_clear_flush_young() elides a DSB when invalidating the
147 * TLB.
148 */
149 #define pte_accessible(mm, pte) \
150 (mm_tlb_flush_pending(mm) ? pte_present(pte) : pte_valid(pte))
151
152 /*
153 * p??_access_permitted() is true for valid user mappings (PTE_USER
154 * bit set, subject to the write permission check). For execute-only
155 * mappings, like PROT_EXEC with EPAN (both PTE_USER and PTE_UXN bits
156 * not set) must return false. PROT_NONE mappings do not have the
157 * PTE_VALID bit set.
158 */
159 #define pte_access_permitted(pte, write) \
160 (((pte_val(pte) & (PTE_VALID | PTE_USER)) == (PTE_VALID | PTE_USER)) && (!(write) || pte_write(pte)))
161 #define pmd_access_permitted(pmd, write) \
162 (pte_access_permitted(pmd_pte(pmd), (write)))
163 #define pud_access_permitted(pud, write) \
164 (pte_access_permitted(pud_pte(pud), (write)))
165
166 static inline pte_t clear_pte_bit(pte_t pte, pgprot_t prot)
167 {
168 pte_val(pte) &= ~pgprot_val(prot);
169 return pte;
170 }
171
172 static inline pte_t set_pte_bit(pte_t pte, pgprot_t prot)
173 {
174 pte_val(pte) |= pgprot_val(prot);
175 return pte;
176 }
177
178 static inline pmd_t clear_pmd_bit(pmd_t pmd, pgprot_t prot)
179 {
180 pmd_val(pmd) &= ~pgprot_val(prot);
181 return pmd;
182 }
183
184 static inline pmd_t set_pmd_bit(pmd_t pmd, pgprot_t prot)
185 {
186 pmd_val(pmd) |= pgprot_val(prot);
187 return pmd;
188 }
189
190 static inline pte_t pte_mkwrite_novma(pte_t pte)
191 {
192 pte = set_pte_bit(pte, __pgprot(PTE_WRITE));
193 pte = clear_pte_bit(pte, __pgprot(PTE_RDONLY));
194 return pte;
195 }
196
197 static inline pte_t pte_mkclean(pte_t pte)
198 {
199 pte = clear_pte_bit(pte, __pgprot(PTE_DIRTY));
200 pte = set_pte_bit(pte, __pgprot(PTE_RDONLY));
201
202 return pte;
203 }
204
205 static inline pte_t pte_mkdirty(pte_t pte)
206 {
207 pte = set_pte_bit(pte, __pgprot(PTE_DIRTY));
208
209 if (pte_write(pte))
210 pte = clear_pte_bit(pte, __pgprot(PTE_RDONLY));
211
212 return pte;
213 }
214
215 static inline pte_t pte_wrprotect(pte_t pte)
216 {
217 /*
218 * If hardware-dirty (PTE_WRITE/DBM bit set and PTE_RDONLY
219 * clear), set the PTE_DIRTY bit.
220 */
221 if (pte_hw_dirty(pte))
222 pte = set_pte_bit(pte, __pgprot(PTE_DIRTY));
223
224 pte = clear_pte_bit(pte, __pgprot(PTE_WRITE));
225 pte = set_pte_bit(pte, __pgprot(PTE_RDONLY));
226 return pte;
227 }
228
229 static inline pte_t pte_mkold(pte_t pte)
230 {
231 return clear_pte_bit(pte, __pgprot(PTE_AF));
232 }
233
234 static inline pte_t pte_mkyoung(pte_t pte)
235 {
236 return set_pte_bit(pte, __pgprot(PTE_AF));
237 }
238
239 static inline pte_t pte_mkspecial(pte_t pte)
240 {
241 return set_pte_bit(pte, __pgprot(PTE_SPECIAL));
242 }
243
244 static inline pte_t pte_mkcont(pte_t pte)
245 {
246 pte = set_pte_bit(pte, __pgprot(PTE_CONT));
247 return set_pte_bit(pte, __pgprot(PTE_TYPE_PAGE));
248 }
249
250 static inline pte_t pte_mknoncont(pte_t pte)
251 {
252 return clear_pte_bit(pte, __pgprot(PTE_CONT));
253 }
254
255 static inline pte_t pte_mkpresent(pte_t pte)
256 {
257 return set_pte_bit(pte, __pgprot(PTE_VALID));
258 }
259
260 static inline pte_t pte_mkinvalid(pte_t pte)
261 {
262 pte = set_pte_bit(pte, __pgprot(PTE_PRESENT_INVALID));
263 pte = clear_pte_bit(pte, __pgprot(PTE_VALID));
264 return pte;
265 }
266
267 static inline pmd_t pmd_mkcont(pmd_t pmd)
268 {
269 return __pmd(pmd_val(pmd) | PMD_SECT_CONT);
270 }
271
272 static inline pte_t pte_mkdevmap(pte_t pte)
273 {
274 return set_pte_bit(pte, __pgprot(PTE_DEVMAP | PTE_SPECIAL));
275 }
276
277 #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP
278 static inline int pte_uffd_wp(pte_t pte)
279 {
280 return !!(pte_val(pte) & PTE_UFFD_WP);
281 }
282
283 static inline pte_t pte_mkuffd_wp(pte_t pte)
284 {
285 return pte_wrprotect(set_pte_bit(pte, __pgprot(PTE_UFFD_WP)));
286 }
287
288 static inline pte_t pte_clear_uffd_wp(pte_t pte)
289 {
290 return clear_pte_bit(pte, __pgprot(PTE_UFFD_WP));
291 }
292 #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */
293
294 static inline void __set_pte_nosync(pte_t *ptep, pte_t pte)
295 {
296 WRITE_ONCE(*ptep, pte);
297 }
298
299 static inline void __set_pte(pte_t *ptep, pte_t pte)
300 {
301 __set_pte_nosync(ptep, pte);
302
303 /*
304 * Only if the new pte is valid and kernel, otherwise TLB maintenance
305 * or update_mmu_cache() have the necessary barriers.
306 */
307 if (pte_valid_not_user(pte)) {
308 dsb(ishst);
309 isb();
310 }
311 }
312
313 static inline pte_t __ptep_get(pte_t *ptep)
314 {
315 return READ_ONCE(*ptep);
316 }
317
318 extern void __sync_icache_dcache(pte_t pteval);
319 bool pgattr_change_is_safe(u64 old, u64 new);
320
321 /*
322 * PTE bits configuration in the presence of hardware Dirty Bit Management
323 * (PTE_WRITE == PTE_DBM):
324 *
325 * Dirty Writable | PTE_RDONLY PTE_WRITE PTE_DIRTY (sw)
326 * 0 0 | 1 0 0
327 * 0 1 | 1 1 0
328 * 1 0 | 1 0 1
329 * 1 1 | 0 1 x
330 *
331 * When hardware DBM is not present, the sofware PTE_DIRTY bit is updated via
332 * the page fault mechanism. Checking the dirty status of a pte becomes:
333 *
334 * PTE_DIRTY || (PTE_WRITE && !PTE_RDONLY)
335 */
336
337 static inline void __check_safe_pte_update(struct mm_struct *mm, pte_t *ptep,
338 pte_t pte)
339 {
340 pte_t old_pte;
341
342 if (!IS_ENABLED(CONFIG_DEBUG_VM))
343 return;
344
345 old_pte = __ptep_get(ptep);
346
347 if (!pte_valid(old_pte) || !pte_valid(pte))
348 return;
349 if (mm != current->active_mm && atomic_read(&mm->mm_users) <= 1)
350 return;
351
352 /*
353 * Check for potential race with hardware updates of the pte
354 * (__ptep_set_access_flags safely changes valid ptes without going
355 * through an invalid entry).
356 */
357 VM_WARN_ONCE(!pte_young(pte),
358 "%s: racy access flag clearing: 0x%016llx -> 0x%016llx",
359 __func__, pte_val(old_pte), pte_val(pte));
360 VM_WARN_ONCE(pte_write(old_pte) && !pte_dirty(pte),
361 "%s: racy dirty state clearing: 0x%016llx -> 0x%016llx",
362 __func__, pte_val(old_pte), pte_val(pte));
363 VM_WARN_ONCE(!pgattr_change_is_safe(pte_val(old_pte), pte_val(pte)),
364 "%s: unsafe attribute change: 0x%016llx -> 0x%016llx",
365 __func__, pte_val(old_pte), pte_val(pte));
366 }
367
368 static inline void __sync_cache_and_tags(pte_t pte, unsigned int nr_pages)
369 {
370 if (pte_present(pte) && pte_user_exec(pte) && !pte_special(pte))
371 __sync_icache_dcache(pte);
372
373 /*
374 * If the PTE would provide user space access to the tags associated
375 * with it then ensure that the MTE tags are synchronised. Although
376 * pte_access_permitted() returns false for exec only mappings, they
377 * don't expose tags (instruction fetches don't check tags).
378 */
379 if (system_supports_mte() && pte_access_permitted(pte, false) &&
380 !pte_special(pte) && pte_tagged(pte))
381 mte_sync_tags(pte, nr_pages);
382 }
383
384 /*
385 * Select all bits except the pfn
386 */
387 static inline pgprot_t pte_pgprot(pte_t pte)
388 {
389 unsigned long pfn = pte_pfn(pte);
390
391 return __pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^ pte_val(pte));
392 }
393
394 #define pte_advance_pfn pte_advance_pfn
395 static inline pte_t pte_advance_pfn(pte_t pte, unsigned long nr)
396 {
397 return pfn_pte(pte_pfn(pte) + nr, pte_pgprot(pte));
398 }
399
400 static inline void __set_ptes(struct mm_struct *mm,
401 unsigned long __always_unused addr,
402 pte_t *ptep, pte_t pte, unsigned int nr)
403 {
404 page_table_check_ptes_set(mm, ptep, pte, nr);
405 __sync_cache_and_tags(pte, nr);
406
407 for (;;) {
408 __check_safe_pte_update(mm, ptep, pte);
409 __set_pte(ptep, pte);
410 if (--nr == 0)
411 break;
412 ptep++;
413 pte = pte_advance_pfn(pte, 1);
414 }
415 }
416
417 /*
418 * Huge pte definitions.
419 */
420 #define pte_mkhuge(pte) (__pte(pte_val(pte) & ~PTE_TABLE_BIT))
421
422 /*
423 * Hugetlb definitions.
424 */
425 #define HUGE_MAX_HSTATE 4
426 #define HPAGE_SHIFT PMD_SHIFT
427 #define HPAGE_SIZE (_AC(1, UL) << HPAGE_SHIFT)
428 #define HPAGE_MASK (~(HPAGE_SIZE - 1))
429 #define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT)
430
431 static inline pte_t pgd_pte(pgd_t pgd)
432 {
433 return __pte(pgd_val(pgd));
434 }
435
436 static inline pte_t p4d_pte(p4d_t p4d)
437 {
438 return __pte(p4d_val(p4d));
439 }
440
441 static inline pte_t pud_pte(pud_t pud)
442 {
443 return __pte(pud_val(pud));
444 }
445
446 static inline pud_t pte_pud(pte_t pte)
447 {
448 return __pud(pte_val(pte));
449 }
450
451 static inline pmd_t pud_pmd(pud_t pud)
452 {
453 return __pmd(pud_val(pud));
454 }
455
456 static inline pte_t pmd_pte(pmd_t pmd)
457 {
458 return __pte(pmd_val(pmd));
459 }
460
461 static inline pmd_t pte_pmd(pte_t pte)
462 {
463 return __pmd(pte_val(pte));
464 }
465
466 static inline pgprot_t mk_pud_sect_prot(pgprot_t prot)
467 {
468 return __pgprot((pgprot_val(prot) & ~PUD_TABLE_BIT) | PUD_TYPE_SECT);
469 }
470
471 static inline pgprot_t mk_pmd_sect_prot(pgprot_t prot)
472 {
473 return __pgprot((pgprot_val(prot) & ~PMD_TABLE_BIT) | PMD_TYPE_SECT);
474 }
475
476 static inline pte_t pte_swp_mkexclusive(pte_t pte)
477 {
478 return set_pte_bit(pte, __pgprot(PTE_SWP_EXCLUSIVE));
479 }
480
481 static inline int pte_swp_exclusive(pte_t pte)
482 {
483 return pte_val(pte) & PTE_SWP_EXCLUSIVE;
484 }
485
486 static inline pte_t pte_swp_clear_exclusive(pte_t pte)
487 {
488 return clear_pte_bit(pte, __pgprot(PTE_SWP_EXCLUSIVE));
489 }
490
491 #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP
492 static inline pte_t pte_swp_mkuffd_wp(pte_t pte)
493 {
494 return set_pte_bit(pte, __pgprot(PTE_SWP_UFFD_WP));
495 }
496
497 static inline int pte_swp_uffd_wp(pte_t pte)
498 {
499 return !!(pte_val(pte) & PTE_SWP_UFFD_WP);
500 }
501
502 static inline pte_t pte_swp_clear_uffd_wp(pte_t pte)
503 {
504 return clear_pte_bit(pte, __pgprot(PTE_SWP_UFFD_WP));
505 }
506 #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */
507
508 #ifdef CONFIG_NUMA_BALANCING
509 /*
510 * See the comment in include/linux/pgtable.h
511 */
512 static inline int pte_protnone(pte_t pte)
513 {
514 /*
515 * pte_present_invalid() tells us that the pte is invalid from HW
516 * perspective but present from SW perspective, so the fields are to be
517 * interpretted as per the HW layout. The second 2 checks are the unique
518 * encoding that we use for PROT_NONE. It is insufficient to only use
519 * the first check because we share the same encoding scheme with pmds
520 * which support pmd_mkinvalid(), so can be present-invalid without
521 * being PROT_NONE.
522 */
523 return pte_present_invalid(pte) && !pte_user(pte) && !pte_user_exec(pte);
524 }
525
526 static inline int pmd_protnone(pmd_t pmd)
527 {
528 return pte_protnone(pmd_pte(pmd));
529 }
530 #endif
531
532 #define pmd_present(pmd) pte_present(pmd_pte(pmd))
533
534 /*
535 * THP definitions.
536 */
537
538 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
539 static inline int pmd_trans_huge(pmd_t pmd)
540 {
541 return pmd_val(pmd) && pmd_present(pmd) && !(pmd_val(pmd) & PMD_TABLE_BIT);
542 }
543 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
544
545 #define pmd_dirty(pmd) pte_dirty(pmd_pte(pmd))
546 #define pmd_young(pmd) pte_young(pmd_pte(pmd))
547 #define pmd_valid(pmd) pte_valid(pmd_pte(pmd))
548 #define pmd_user(pmd) pte_user(pmd_pte(pmd))
549 #define pmd_user_exec(pmd) pte_user_exec(pmd_pte(pmd))
550 #define pmd_cont(pmd) pte_cont(pmd_pte(pmd))
551 #define pmd_wrprotect(pmd) pte_pmd(pte_wrprotect(pmd_pte(pmd)))
552 #define pmd_mkold(pmd) pte_pmd(pte_mkold(pmd_pte(pmd)))
553 #define pmd_mkwrite_novma(pmd) pte_pmd(pte_mkwrite_novma(pmd_pte(pmd)))
554 #define pmd_mkclean(pmd) pte_pmd(pte_mkclean(pmd_pte(pmd)))
555 #define pmd_mkdirty(pmd) pte_pmd(pte_mkdirty(pmd_pte(pmd)))
556 #define pmd_mkyoung(pmd) pte_pmd(pte_mkyoung(pmd_pte(pmd)))
557 #define pmd_mkinvalid(pmd) pte_pmd(pte_mkinvalid(pmd_pte(pmd)))
558 #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP
559 #define pmd_uffd_wp(pmd) pte_uffd_wp(pmd_pte(pmd))
560 #define pmd_mkuffd_wp(pmd) pte_pmd(pte_mkuffd_wp(pmd_pte(pmd)))
561 #define pmd_clear_uffd_wp(pmd) pte_pmd(pte_clear_uffd_wp(pmd_pte(pmd)))
562 #define pmd_swp_uffd_wp(pmd) pte_swp_uffd_wp(pmd_pte(pmd))
563 #define pmd_swp_mkuffd_wp(pmd) pte_pmd(pte_swp_mkuffd_wp(pmd_pte(pmd)))
564 #define pmd_swp_clear_uffd_wp(pmd) \
565 pte_pmd(pte_swp_clear_uffd_wp(pmd_pte(pmd)))
566 #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */
567
568 #define pmd_write(pmd) pte_write(pmd_pte(pmd))
569
570 #define pmd_mkhuge(pmd) (__pmd(pmd_val(pmd) & ~PMD_TABLE_BIT))
571
572 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
573 #define pmd_devmap(pmd) pte_devmap(pmd_pte(pmd))
574 #endif
575 static inline pmd_t pmd_mkdevmap(pmd_t pmd)
576 {
577 return pte_pmd(set_pte_bit(pmd_pte(pmd), __pgprot(PTE_DEVMAP)));
578 }
579
580 #define __pmd_to_phys(pmd) __pte_to_phys(pmd_pte(pmd))
581 #define __phys_to_pmd_val(phys) __phys_to_pte_val(phys)
582 #define pmd_pfn(pmd) ((__pmd_to_phys(pmd) & PMD_MASK) >> PAGE_SHIFT)
583 #define pfn_pmd(pfn,prot) __pmd(__phys_to_pmd_val((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot))
584 #define mk_pmd(page,prot) pfn_pmd(page_to_pfn(page),prot)
585
586 #define pud_young(pud) pte_young(pud_pte(pud))
587 #define pud_mkyoung(pud) pte_pud(pte_mkyoung(pud_pte(pud)))
588 #define pud_write(pud) pte_write(pud_pte(pud))
589
590 #define pud_mkhuge(pud) (__pud(pud_val(pud) & ~PUD_TABLE_BIT))
591
592 #define __pud_to_phys(pud) __pte_to_phys(pud_pte(pud))
593 #define __phys_to_pud_val(phys) __phys_to_pte_val(phys)
594 #define pud_pfn(pud) ((__pud_to_phys(pud) & PUD_MASK) >> PAGE_SHIFT)
595 #define pfn_pud(pfn,prot) __pud(__phys_to_pud_val((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot))
596
597 static inline void __set_pte_at(struct mm_struct *mm,
598 unsigned long __always_unused addr,
599 pte_t *ptep, pte_t pte, unsigned int nr)
600 {
601 __sync_cache_and_tags(pte, nr);
602 __check_safe_pte_update(mm, ptep, pte);
603 __set_pte(ptep, pte);
604 }
605
606 static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
607 pmd_t *pmdp, pmd_t pmd)
608 {
609 page_table_check_pmd_set(mm, pmdp, pmd);
610 return __set_pte_at(mm, addr, (pte_t *)pmdp, pmd_pte(pmd),
611 PMD_SIZE >> PAGE_SHIFT);
612 }
613
614 static inline void set_pud_at(struct mm_struct *mm, unsigned long addr,
615 pud_t *pudp, pud_t pud)
616 {
617 page_table_check_pud_set(mm, pudp, pud);
618 return __set_pte_at(mm, addr, (pte_t *)pudp, pud_pte(pud),
619 PUD_SIZE >> PAGE_SHIFT);
620 }
621
622 #define __p4d_to_phys(p4d) __pte_to_phys(p4d_pte(p4d))
623 #define __phys_to_p4d_val(phys) __phys_to_pte_val(phys)
624
625 #define __pgd_to_phys(pgd) __pte_to_phys(pgd_pte(pgd))
626 #define __phys_to_pgd_val(phys) __phys_to_pte_val(phys)
627
628 #define __pgprot_modify(prot,mask,bits) \
629 __pgprot((pgprot_val(prot) & ~(mask)) | (bits))
630
631 #define pgprot_nx(prot) \
632 __pgprot_modify(prot, PTE_MAYBE_GP, PTE_PXN)
633
634 /*
635 * Mark the prot value as uncacheable and unbufferable.
636 */
637 #define pgprot_noncached(prot) \
638 __pgprot_modify(prot, PTE_ATTRINDX_MASK, PTE_ATTRINDX(MT_DEVICE_nGnRnE) | PTE_PXN | PTE_UXN)
639 #define pgprot_writecombine(prot) \
640 __pgprot_modify(prot, PTE_ATTRINDX_MASK, PTE_ATTRINDX(MT_NORMAL_NC) | PTE_PXN | PTE_UXN)
641 #define pgprot_device(prot) \
642 __pgprot_modify(prot, PTE_ATTRINDX_MASK, PTE_ATTRINDX(MT_DEVICE_nGnRE) | PTE_PXN | PTE_UXN)
643 #define pgprot_tagged(prot) \
644 __pgprot_modify(prot, PTE_ATTRINDX_MASK, PTE_ATTRINDX(MT_NORMAL_TAGGED))
645 #define pgprot_mhp pgprot_tagged
646 /*
647 * DMA allocations for non-coherent devices use what the Arm architecture calls
648 * "Normal non-cacheable" memory, which permits speculation, unaligned accesses
649 * and merging of writes. This is different from "Device-nGnR[nE]" memory which
650 * is intended for MMIO and thus forbids speculation, preserves access size,
651 * requires strict alignment and can also force write responses to come from the
652 * endpoint.
653 */
654 #define pgprot_dmacoherent(prot) \
655 __pgprot_modify(prot, PTE_ATTRINDX_MASK, \
656 PTE_ATTRINDX(MT_NORMAL_NC) | PTE_PXN | PTE_UXN)
657
658 #define __HAVE_PHYS_MEM_ACCESS_PROT
659 struct file;
660 extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
661 unsigned long size, pgprot_t vma_prot);
662
663 #define pmd_none(pmd) (!pmd_val(pmd))
664
665 #define pmd_table(pmd) ((pmd_val(pmd) & PMD_TYPE_MASK) == \
666 PMD_TYPE_TABLE)
667 #define pmd_sect(pmd) ((pmd_val(pmd) & PMD_TYPE_MASK) == \
668 PMD_TYPE_SECT)
669 #define pmd_leaf(pmd) (pmd_present(pmd) && !pmd_table(pmd))
670 #define pmd_bad(pmd) (!pmd_table(pmd))
671
672 #define pmd_leaf_size(pmd) (pmd_cont(pmd) ? CONT_PMD_SIZE : PMD_SIZE)
673 #define pte_leaf_size(pte) (pte_cont(pte) ? CONT_PTE_SIZE : PAGE_SIZE)
674
675 #if defined(CONFIG_ARM64_64K_PAGES) || CONFIG_PGTABLE_LEVELS < 3
676 static inline bool pud_sect(pud_t pud) { return false; }
677 static inline bool pud_table(pud_t pud) { return true; }
678 #else
679 #define pud_sect(pud) ((pud_val(pud) & PUD_TYPE_MASK) == \
680 PUD_TYPE_SECT)
681 #define pud_table(pud) ((pud_val(pud) & PUD_TYPE_MASK) == \
682 PUD_TYPE_TABLE)
683 #endif
684
685 extern pgd_t init_pg_dir[];
686 extern pgd_t init_pg_end[];
687 extern pgd_t swapper_pg_dir[];
688 extern pgd_t idmap_pg_dir[];
689 extern pgd_t tramp_pg_dir[];
690 extern pgd_t reserved_pg_dir[];
691
692 extern void set_swapper_pgd(pgd_t *pgdp, pgd_t pgd);
693
694 static inline bool in_swapper_pgdir(void *addr)
695 {
696 return ((unsigned long)addr & PAGE_MASK) ==
697 ((unsigned long)swapper_pg_dir & PAGE_MASK);
698 }
699
700 static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
701 {
702 #ifdef __PAGETABLE_PMD_FOLDED
703 if (in_swapper_pgdir(pmdp)) {
704 set_swapper_pgd((pgd_t *)pmdp, __pgd(pmd_val(pmd)));
705 return;
706 }
707 #endif /* __PAGETABLE_PMD_FOLDED */
708
709 WRITE_ONCE(*pmdp, pmd);
710
711 if (pmd_valid(pmd)) {
712 dsb(ishst);
713 isb();
714 }
715 }
716
717 static inline void pmd_clear(pmd_t *pmdp)
718 {
719 set_pmd(pmdp, __pmd(0));
720 }
721
722 static inline phys_addr_t pmd_page_paddr(pmd_t pmd)
723 {
724 return __pmd_to_phys(pmd);
725 }
726
727 static inline unsigned long pmd_page_vaddr(pmd_t pmd)
728 {
729 return (unsigned long)__va(pmd_page_paddr(pmd));
730 }
731
732 /* Find an entry in the third-level page table. */
733 #define pte_offset_phys(dir,addr) (pmd_page_paddr(READ_ONCE(*(dir))) + pte_index(addr) * sizeof(pte_t))
734
735 #define pte_set_fixmap(addr) ((pte_t *)set_fixmap_offset(FIX_PTE, addr))
736 #define pte_set_fixmap_offset(pmd, addr) pte_set_fixmap(pte_offset_phys(pmd, addr))
737 #define pte_clear_fixmap() clear_fixmap(FIX_PTE)
738
739 #define pmd_page(pmd) phys_to_page(__pmd_to_phys(pmd))
740
741 /* use ONLY for statically allocated translation tables */
742 #define pte_offset_kimg(dir,addr) ((pte_t *)__phys_to_kimg(pte_offset_phys((dir), (addr))))
743
744 /*
745 * Conversion functions: convert a page and protection to a page entry,
746 * and a page entry and page directory to the page they refer to.
747 */
748 #define mk_pte(page,prot) pfn_pte(page_to_pfn(page),prot)
749
750 #if CONFIG_PGTABLE_LEVELS > 2
751
752 #define pmd_ERROR(e) \
753 pr_err("%s:%d: bad pmd %016llx.\n", __FILE__, __LINE__, pmd_val(e))
754
755 #define pud_none(pud) (!pud_val(pud))
756 #define pud_bad(pud) (!pud_table(pud))
757 #define pud_present(pud) pte_present(pud_pte(pud))
758 #ifndef __PAGETABLE_PMD_FOLDED
759 #define pud_leaf(pud) (pud_present(pud) && !pud_table(pud))
760 #else
761 #define pud_leaf(pud) false
762 #endif
763 #define pud_valid(pud) pte_valid(pud_pte(pud))
764 #define pud_user(pud) pte_user(pud_pte(pud))
765 #define pud_user_exec(pud) pte_user_exec(pud_pte(pud))
766
767 static inline bool pgtable_l4_enabled(void);
768
769 static inline void set_pud(pud_t *pudp, pud_t pud)
770 {
771 if (!pgtable_l4_enabled() && in_swapper_pgdir(pudp)) {
772 set_swapper_pgd((pgd_t *)pudp, __pgd(pud_val(pud)));
773 return;
774 }
775
776 WRITE_ONCE(*pudp, pud);
777
778 if (pud_valid(pud)) {
779 dsb(ishst);
780 isb();
781 }
782 }
783
784 static inline void pud_clear(pud_t *pudp)
785 {
786 set_pud(pudp, __pud(0));
787 }
788
789 static inline phys_addr_t pud_page_paddr(pud_t pud)
790 {
791 return __pud_to_phys(pud);
792 }
793
794 static inline pmd_t *pud_pgtable(pud_t pud)
795 {
796 return (pmd_t *)__va(pud_page_paddr(pud));
797 }
798
799 /* Find an entry in the second-level page table. */
800 #define pmd_offset_phys(dir, addr) (pud_page_paddr(READ_ONCE(*(dir))) + pmd_index(addr) * sizeof(pmd_t))
801
802 #define pmd_set_fixmap(addr) ((pmd_t *)set_fixmap_offset(FIX_PMD, addr))
803 #define pmd_set_fixmap_offset(pud, addr) pmd_set_fixmap(pmd_offset_phys(pud, addr))
804 #define pmd_clear_fixmap() clear_fixmap(FIX_PMD)
805
806 #define pud_page(pud) phys_to_page(__pud_to_phys(pud))
807
808 /* use ONLY for statically allocated translation tables */
809 #define pmd_offset_kimg(dir,addr) ((pmd_t *)__phys_to_kimg(pmd_offset_phys((dir), (addr))))
810
811 #else
812
813 #define pud_valid(pud) false
814 #define pud_page_paddr(pud) ({ BUILD_BUG(); 0; })
815 #define pud_user_exec(pud) pud_user(pud) /* Always 0 with folding */
816
817 /* Match pmd_offset folding in <asm/generic/pgtable-nopmd.h> */
818 #define pmd_set_fixmap(addr) NULL
819 #define pmd_set_fixmap_offset(pudp, addr) ((pmd_t *)pudp)
820 #define pmd_clear_fixmap()
821
822 #define pmd_offset_kimg(dir,addr) ((pmd_t *)dir)
823
824 #endif /* CONFIG_PGTABLE_LEVELS > 2 */
825
826 #if CONFIG_PGTABLE_LEVELS > 3
827
828 static __always_inline bool pgtable_l4_enabled(void)
829 {
830 if (CONFIG_PGTABLE_LEVELS > 4 || !IS_ENABLED(CONFIG_ARM64_LPA2))
831 return true;
832 if (!alternative_has_cap_likely(ARM64_ALWAYS_BOOT))
833 return vabits_actual == VA_BITS;
834 return alternative_has_cap_unlikely(ARM64_HAS_VA52);
835 }
836
837 static inline bool mm_pud_folded(const struct mm_struct *mm)
838 {
839 return !pgtable_l4_enabled();
840 }
841 #define mm_pud_folded mm_pud_folded
842
843 #define pud_ERROR(e) \
844 pr_err("%s:%d: bad pud %016llx.\n", __FILE__, __LINE__, pud_val(e))
845
846 #define p4d_none(p4d) (pgtable_l4_enabled() && !p4d_val(p4d))
847 #define p4d_bad(p4d) (pgtable_l4_enabled() && !(p4d_val(p4d) & 2))
848 #define p4d_present(p4d) (!p4d_none(p4d))
849
850 static inline void set_p4d(p4d_t *p4dp, p4d_t p4d)
851 {
852 if (in_swapper_pgdir(p4dp)) {
853 set_swapper_pgd((pgd_t *)p4dp, __pgd(p4d_val(p4d)));
854 return;
855 }
856
857 WRITE_ONCE(*p4dp, p4d);
858 dsb(ishst);
859 isb();
860 }
861
862 static inline void p4d_clear(p4d_t *p4dp)
863 {
864 if (pgtable_l4_enabled())
865 set_p4d(p4dp, __p4d(0));
866 }
867
868 static inline phys_addr_t p4d_page_paddr(p4d_t p4d)
869 {
870 return __p4d_to_phys(p4d);
871 }
872
873 #define pud_index(addr) (((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1))
874
875 static inline pud_t *p4d_to_folded_pud(p4d_t *p4dp, unsigned long addr)
876 {
877 return (pud_t *)PTR_ALIGN_DOWN(p4dp, PAGE_SIZE) + pud_index(addr);
878 }
879
880 static inline pud_t *p4d_pgtable(p4d_t p4d)
881 {
882 return (pud_t *)__va(p4d_page_paddr(p4d));
883 }
884
885 static inline phys_addr_t pud_offset_phys(p4d_t *p4dp, unsigned long addr)
886 {
887 BUG_ON(!pgtable_l4_enabled());
888
889 return p4d_page_paddr(READ_ONCE(*p4dp)) + pud_index(addr) * sizeof(pud_t);
890 }
891
892 static inline
893 pud_t *pud_offset_lockless(p4d_t *p4dp, p4d_t p4d, unsigned long addr)
894 {
895 if (!pgtable_l4_enabled())
896 return p4d_to_folded_pud(p4dp, addr);
897 return (pud_t *)__va(p4d_page_paddr(p4d)) + pud_index(addr);
898 }
899 #define pud_offset_lockless pud_offset_lockless
900
901 static inline pud_t *pud_offset(p4d_t *p4dp, unsigned long addr)
902 {
903 return pud_offset_lockless(p4dp, READ_ONCE(*p4dp), addr);
904 }
905 #define pud_offset pud_offset
906
907 static inline pud_t *pud_set_fixmap(unsigned long addr)
908 {
909 if (!pgtable_l4_enabled())
910 return NULL;
911 return (pud_t *)set_fixmap_offset(FIX_PUD, addr);
912 }
913
914 static inline pud_t *pud_set_fixmap_offset(p4d_t *p4dp, unsigned long addr)
915 {
916 if (!pgtable_l4_enabled())
917 return p4d_to_folded_pud(p4dp, addr);
918 return pud_set_fixmap(pud_offset_phys(p4dp, addr));
919 }
920
921 static inline void pud_clear_fixmap(void)
922 {
923 if (pgtable_l4_enabled())
924 clear_fixmap(FIX_PUD);
925 }
926
927 /* use ONLY for statically allocated translation tables */
928 static inline pud_t *pud_offset_kimg(p4d_t *p4dp, u64 addr)
929 {
930 if (!pgtable_l4_enabled())
931 return p4d_to_folded_pud(p4dp, addr);
932 return (pud_t *)__phys_to_kimg(pud_offset_phys(p4dp, addr));
933 }
934
935 #define p4d_page(p4d) pfn_to_page(__phys_to_pfn(__p4d_to_phys(p4d)))
936
937 #else
938
939 static inline bool pgtable_l4_enabled(void) { return false; }
940
941 #define p4d_page_paddr(p4d) ({ BUILD_BUG(); 0;})
942
943 /* Match pud_offset folding in <asm/generic/pgtable-nopud.h> */
944 #define pud_set_fixmap(addr) NULL
945 #define pud_set_fixmap_offset(pgdp, addr) ((pud_t *)pgdp)
946 #define pud_clear_fixmap()
947
948 #define pud_offset_kimg(dir,addr) ((pud_t *)dir)
949
950 #endif /* CONFIG_PGTABLE_LEVELS > 3 */
951
952 #if CONFIG_PGTABLE_LEVELS > 4
953
954 static __always_inline bool pgtable_l5_enabled(void)
955 {
956 if (!alternative_has_cap_likely(ARM64_ALWAYS_BOOT))
957 return vabits_actual == VA_BITS;
958 return alternative_has_cap_unlikely(ARM64_HAS_VA52);
959 }
960
961 static inline bool mm_p4d_folded(const struct mm_struct *mm)
962 {
963 return !pgtable_l5_enabled();
964 }
965 #define mm_p4d_folded mm_p4d_folded
966
967 #define p4d_ERROR(e) \
968 pr_err("%s:%d: bad p4d %016llx.\n", __FILE__, __LINE__, p4d_val(e))
969
970 #define pgd_none(pgd) (pgtable_l5_enabled() && !pgd_val(pgd))
971 #define pgd_bad(pgd) (pgtable_l5_enabled() && !(pgd_val(pgd) & 2))
972 #define pgd_present(pgd) (!pgd_none(pgd))
973
974 static inline void set_pgd(pgd_t *pgdp, pgd_t pgd)
975 {
976 if (in_swapper_pgdir(pgdp)) {
977 set_swapper_pgd(pgdp, __pgd(pgd_val(pgd)));
978 return;
979 }
980
981 WRITE_ONCE(*pgdp, pgd);
982 dsb(ishst);
983 isb();
984 }
985
986 static inline void pgd_clear(pgd_t *pgdp)
987 {
988 if (pgtable_l5_enabled())
989 set_pgd(pgdp, __pgd(0));
990 }
991
992 static inline phys_addr_t pgd_page_paddr(pgd_t pgd)
993 {
994 return __pgd_to_phys(pgd);
995 }
996
997 #define p4d_index(addr) (((addr) >> P4D_SHIFT) & (PTRS_PER_P4D - 1))
998
999 static inline p4d_t *pgd_to_folded_p4d(pgd_t *pgdp, unsigned long addr)
1000 {
1001 return (p4d_t *)PTR_ALIGN_DOWN(pgdp, PAGE_SIZE) + p4d_index(addr);
1002 }
1003
1004 static inline phys_addr_t p4d_offset_phys(pgd_t *pgdp, unsigned long addr)
1005 {
1006 BUG_ON(!pgtable_l5_enabled());
1007
1008 return pgd_page_paddr(READ_ONCE(*pgdp)) + p4d_index(addr) * sizeof(p4d_t);
1009 }
1010
1011 static inline
1012 p4d_t *p4d_offset_lockless(pgd_t *pgdp, pgd_t pgd, unsigned long addr)
1013 {
1014 if (!pgtable_l5_enabled())
1015 return pgd_to_folded_p4d(pgdp, addr);
1016 return (p4d_t *)__va(pgd_page_paddr(pgd)) + p4d_index(addr);
1017 }
1018 #define p4d_offset_lockless p4d_offset_lockless
1019
1020 static inline p4d_t *p4d_offset(pgd_t *pgdp, unsigned long addr)
1021 {
1022 return p4d_offset_lockless(pgdp, READ_ONCE(*pgdp), addr);
1023 }
1024
1025 static inline p4d_t *p4d_set_fixmap(unsigned long addr)
1026 {
1027 if (!pgtable_l5_enabled())
1028 return NULL;
1029 return (p4d_t *)set_fixmap_offset(FIX_P4D, addr);
1030 }
1031
1032 static inline p4d_t *p4d_set_fixmap_offset(pgd_t *pgdp, unsigned long addr)
1033 {
1034 if (!pgtable_l5_enabled())
1035 return pgd_to_folded_p4d(pgdp, addr);
1036 return p4d_set_fixmap(p4d_offset_phys(pgdp, addr));
1037 }
1038
1039 static inline void p4d_clear_fixmap(void)
1040 {
1041 if (pgtable_l5_enabled())
1042 clear_fixmap(FIX_P4D);
1043 }
1044
1045 /* use ONLY for statically allocated translation tables */
1046 static inline p4d_t *p4d_offset_kimg(pgd_t *pgdp, u64 addr)
1047 {
1048 if (!pgtable_l5_enabled())
1049 return pgd_to_folded_p4d(pgdp, addr);
1050 return (p4d_t *)__phys_to_kimg(p4d_offset_phys(pgdp, addr));
1051 }
1052
1053 #define pgd_page(pgd) pfn_to_page(__phys_to_pfn(__pgd_to_phys(pgd)))
1054
1055 #else
1056
1057 static inline bool pgtable_l5_enabled(void) { return false; }
1058
1059 #define p4d_index(addr) (((addr) >> P4D_SHIFT) & (PTRS_PER_P4D - 1))
1060
1061 /* Match p4d_offset folding in <asm/generic/pgtable-nop4d.h> */
1062 #define p4d_set_fixmap(addr) NULL
1063 #define p4d_set_fixmap_offset(p4dp, addr) ((p4d_t *)p4dp)
1064 #define p4d_clear_fixmap()
1065
1066 #define p4d_offset_kimg(dir,addr) ((p4d_t *)dir)
1067
1068 static inline
1069 p4d_t *p4d_offset_lockless_folded(pgd_t *pgdp, pgd_t pgd, unsigned long addr)
1070 {
1071 /*
1072 * With runtime folding of the pud, pud_offset_lockless() passes
1073 * the 'pgd_t *' we return here to p4d_to_folded_pud(), which
1074 * will offset the pointer assuming that it points into
1075 * a page-table page. However, the fast GUP path passes us a
1076 * pgd_t allocated on the stack and so we must use the original
1077 * pointer in 'pgdp' to construct the p4d pointer instead of
1078 * using the generic p4d_offset_lockless() implementation.
1079 *
1080 * Note: reusing the original pointer means that we may
1081 * dereference the same (live) page-table entry multiple times.
1082 * This is safe because it is still only loaded once in the
1083 * context of each level and the CPU guarantees same-address
1084 * read-after-read ordering.
1085 */
1086 return p4d_offset(pgdp, addr);
1087 }
1088 #define p4d_offset_lockless p4d_offset_lockless_folded
1089
1090 #endif /* CONFIG_PGTABLE_LEVELS > 4 */
1091
1092 #define pgd_ERROR(e) \
1093 pr_err("%s:%d: bad pgd %016llx.\n", __FILE__, __LINE__, pgd_val(e))
1094
1095 #define pgd_set_fixmap(addr) ((pgd_t *)set_fixmap_offset(FIX_PGD, addr))
1096 #define pgd_clear_fixmap() clear_fixmap(FIX_PGD)
1097
1098 static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
1099 {
1100 /*
1101 * Normal and Normal-Tagged are two different memory types and indices
1102 * in MAIR_EL1. The mask below has to include PTE_ATTRINDX_MASK.
1103 */
1104 const pteval_t mask = PTE_USER | PTE_PXN | PTE_UXN | PTE_RDONLY |
1105 PTE_PRESENT_INVALID | PTE_VALID | PTE_WRITE |
1106 PTE_GP | PTE_ATTRINDX_MASK;
1107 /* preserve the hardware dirty information */
1108 if (pte_hw_dirty(pte))
1109 pte = set_pte_bit(pte, __pgprot(PTE_DIRTY));
1110
1111 pte_val(pte) = (pte_val(pte) & ~mask) | (pgprot_val(newprot) & mask);
1112 /*
1113 * If we end up clearing hw dirtiness for a sw-dirty PTE, set hardware
1114 * dirtiness again.
1115 */
1116 if (pte_sw_dirty(pte))
1117 pte = pte_mkdirty(pte);
1118 return pte;
1119 }
1120
1121 static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
1122 {
1123 return pte_pmd(pte_modify(pmd_pte(pmd), newprot));
1124 }
1125
1126 extern int __ptep_set_access_flags(struct vm_area_struct *vma,
1127 unsigned long address, pte_t *ptep,
1128 pte_t entry, int dirty);
1129
1130 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1131 #define __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
1132 static inline int pmdp_set_access_flags(struct vm_area_struct *vma,
1133 unsigned long address, pmd_t *pmdp,
1134 pmd_t entry, int dirty)
1135 {
1136 return __ptep_set_access_flags(vma, address, (pte_t *)pmdp,
1137 pmd_pte(entry), dirty);
1138 }
1139
1140 static inline int pud_devmap(pud_t pud)
1141 {
1142 return 0;
1143 }
1144
1145 static inline int pgd_devmap(pgd_t pgd)
1146 {
1147 return 0;
1148 }
1149 #endif
1150
1151 #ifdef CONFIG_PAGE_TABLE_CHECK
1152 static inline bool pte_user_accessible_page(pte_t pte)
1153 {
1154 return pte_valid(pte) && (pte_user(pte) || pte_user_exec(pte));
1155 }
1156
1157 static inline bool pmd_user_accessible_page(pmd_t pmd)
1158 {
1159 return pmd_valid(pmd) && !pmd_table(pmd) && (pmd_user(pmd) || pmd_user_exec(pmd));
1160 }
1161
1162 static inline bool pud_user_accessible_page(pud_t pud)
1163 {
1164 return pud_valid(pud) && !pud_table(pud) && (pud_user(pud) || pud_user_exec(pud));
1165 }
1166 #endif
1167
1168 /*
1169 * Atomic pte/pmd modifications.
1170 */
1171 static inline int __ptep_test_and_clear_young(struct vm_area_struct *vma,
1172 unsigned long address,
1173 pte_t *ptep)
1174 {
1175 pte_t old_pte, pte;
1176
1177 pte = __ptep_get(ptep);
1178 do {
1179 old_pte = pte;
1180 pte = pte_mkold(pte);
1181 pte_val(pte) = cmpxchg_relaxed(&pte_val(*ptep),
1182 pte_val(old_pte), pte_val(pte));
1183 } while (pte_val(pte) != pte_val(old_pte));
1184
1185 return pte_young(pte);
1186 }
1187
1188 static inline int __ptep_clear_flush_young(struct vm_area_struct *vma,
1189 unsigned long address, pte_t *ptep)
1190 {
1191 int young = __ptep_test_and_clear_young(vma, address, ptep);
1192
1193 if (young) {
1194 /*
1195 * We can elide the trailing DSB here since the worst that can
1196 * happen is that a CPU continues to use the young entry in its
1197 * TLB and we mistakenly reclaim the associated page. The
1198 * window for such an event is bounded by the next
1199 * context-switch, which provides a DSB to complete the TLB
1200 * invalidation.
1201 */
1202 flush_tlb_page_nosync(vma, address);
1203 }
1204
1205 return young;
1206 }
1207
1208 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1209 #define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
1210 static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
1211 unsigned long address,
1212 pmd_t *pmdp)
1213 {
1214 return __ptep_test_and_clear_young(vma, address, (pte_t *)pmdp);
1215 }
1216 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
1217
1218 static inline pte_t __ptep_get_and_clear(struct mm_struct *mm,
1219 unsigned long address, pte_t *ptep)
1220 {
1221 pte_t pte = __pte(xchg_relaxed(&pte_val(*ptep), 0));
1222
1223 page_table_check_pte_clear(mm, pte);
1224
1225 return pte;
1226 }
1227
1228 static inline void __clear_full_ptes(struct mm_struct *mm, unsigned long addr,
1229 pte_t *ptep, unsigned int nr, int full)
1230 {
1231 for (;;) {
1232 __ptep_get_and_clear(mm, addr, ptep);
1233 if (--nr == 0)
1234 break;
1235 ptep++;
1236 addr += PAGE_SIZE;
1237 }
1238 }
1239
1240 static inline pte_t __get_and_clear_full_ptes(struct mm_struct *mm,
1241 unsigned long addr, pte_t *ptep,
1242 unsigned int nr, int full)
1243 {
1244 pte_t pte, tmp_pte;
1245
1246 pte = __ptep_get_and_clear(mm, addr, ptep);
1247 while (--nr) {
1248 ptep++;
1249 addr += PAGE_SIZE;
1250 tmp_pte = __ptep_get_and_clear(mm, addr, ptep);
1251 if (pte_dirty(tmp_pte))
1252 pte = pte_mkdirty(pte);
1253 if (pte_young(tmp_pte))
1254 pte = pte_mkyoung(pte);
1255 }
1256 return pte;
1257 }
1258
1259 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1260 #define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
1261 static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
1262 unsigned long address, pmd_t *pmdp)
1263 {
1264 pmd_t pmd = __pmd(xchg_relaxed(&pmd_val(*pmdp), 0));
1265
1266 page_table_check_pmd_clear(mm, pmd);
1267
1268 return pmd;
1269 }
1270 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
1271
1272 static inline void ___ptep_set_wrprotect(struct mm_struct *mm,
1273 unsigned long address, pte_t *ptep,
1274 pte_t pte)
1275 {
1276 pte_t old_pte;
1277
1278 do {
1279 old_pte = pte;
1280 pte = pte_wrprotect(pte);
1281 pte_val(pte) = cmpxchg_relaxed(&pte_val(*ptep),
1282 pte_val(old_pte), pte_val(pte));
1283 } while (pte_val(pte) != pte_val(old_pte));
1284 }
1285
1286 /*
1287 * __ptep_set_wrprotect - mark read-only while trasferring potential hardware
1288 * dirty status (PTE_DBM && !PTE_RDONLY) to the software PTE_DIRTY bit.
1289 */
1290 static inline void __ptep_set_wrprotect(struct mm_struct *mm,
1291 unsigned long address, pte_t *ptep)
1292 {
1293 ___ptep_set_wrprotect(mm, address, ptep, __ptep_get(ptep));
1294 }
1295
1296 static inline void __wrprotect_ptes(struct mm_struct *mm, unsigned long address,
1297 pte_t *ptep, unsigned int nr)
1298 {
1299 unsigned int i;
1300
1301 for (i = 0; i < nr; i++, address += PAGE_SIZE, ptep++)
1302 __ptep_set_wrprotect(mm, address, ptep);
1303 }
1304
1305 static inline void __clear_young_dirty_pte(struct vm_area_struct *vma,
1306 unsigned long addr, pte_t *ptep,
1307 pte_t pte, cydp_t flags)
1308 {
1309 pte_t old_pte;
1310
1311 do {
1312 old_pte = pte;
1313
1314 if (flags & CYDP_CLEAR_YOUNG)
1315 pte = pte_mkold(pte);
1316 if (flags & CYDP_CLEAR_DIRTY)
1317 pte = pte_mkclean(pte);
1318
1319 pte_val(pte) = cmpxchg_relaxed(&pte_val(*ptep),
1320 pte_val(old_pte), pte_val(pte));
1321 } while (pte_val(pte) != pte_val(old_pte));
1322 }
1323
1324 static inline void __clear_young_dirty_ptes(struct vm_area_struct *vma,
1325 unsigned long addr, pte_t *ptep,
1326 unsigned int nr, cydp_t flags)
1327 {
1328 pte_t pte;
1329
1330 for (;;) {
1331 pte = __ptep_get(ptep);
1332
1333 if (flags == (CYDP_CLEAR_YOUNG | CYDP_CLEAR_DIRTY))
1334 __set_pte(ptep, pte_mkclean(pte_mkold(pte)));
1335 else
1336 __clear_young_dirty_pte(vma, addr, ptep, pte, flags);
1337
1338 if (--nr == 0)
1339 break;
1340 ptep++;
1341 addr += PAGE_SIZE;
1342 }
1343 }
1344
1345 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1346 #define __HAVE_ARCH_PMDP_SET_WRPROTECT
1347 static inline void pmdp_set_wrprotect(struct mm_struct *mm,
1348 unsigned long address, pmd_t *pmdp)
1349 {
1350 __ptep_set_wrprotect(mm, address, (pte_t *)pmdp);
1351 }
1352
1353 #define pmdp_establish pmdp_establish
1354 static inline pmd_t pmdp_establish(struct vm_area_struct *vma,
1355 unsigned long address, pmd_t *pmdp, pmd_t pmd)
1356 {
1357 page_table_check_pmd_set(vma->vm_mm, pmdp, pmd);
1358 return __pmd(xchg_relaxed(&pmd_val(*pmdp), pmd_val(pmd)));
1359 }
1360 #endif
1361
1362 /*
1363 * Encode and decode a swap entry:
1364 * bits 0-1: present (must be zero)
1365 * bits 2: remember PG_anon_exclusive
1366 * bit 3: remember uffd-wp state
1367 * bits 6-10: swap type
1368 * bit 11: PTE_PRESENT_INVALID (must be zero)
1369 * bits 12-61: swap offset
1370 */
1371 #define __SWP_TYPE_SHIFT 6
1372 #define __SWP_TYPE_BITS 5
1373 #define __SWP_TYPE_MASK ((1 << __SWP_TYPE_BITS) - 1)
1374 #define __SWP_OFFSET_SHIFT 12
1375 #define __SWP_OFFSET_BITS 50
1376 #define __SWP_OFFSET_MASK ((1UL << __SWP_OFFSET_BITS) - 1)
1377
1378 #define __swp_type(x) (((x).val >> __SWP_TYPE_SHIFT) & __SWP_TYPE_MASK)
1379 #define __swp_offset(x) (((x).val >> __SWP_OFFSET_SHIFT) & __SWP_OFFSET_MASK)
1380 #define __swp_entry(type,offset) ((swp_entry_t) { ((type) << __SWP_TYPE_SHIFT) | ((offset) << __SWP_OFFSET_SHIFT) })
1381
1382 #define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) })
1383 #define __swp_entry_to_pte(swp) ((pte_t) { (swp).val })
1384
1385 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
1386 #define __pmd_to_swp_entry(pmd) ((swp_entry_t) { pmd_val(pmd) })
1387 #define __swp_entry_to_pmd(swp) __pmd((swp).val)
1388 #endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */
1389
1390 /*
1391 * Ensure that there are not more swap files than can be encoded in the kernel
1392 * PTEs.
1393 */
1394 #define MAX_SWAPFILES_CHECK() BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > __SWP_TYPE_BITS)
1395
1396 #ifdef CONFIG_ARM64_MTE
1397
1398 #define __HAVE_ARCH_PREPARE_TO_SWAP
1399 extern int arch_prepare_to_swap(struct folio *folio);
1400
1401 #define __HAVE_ARCH_SWAP_INVALIDATE
1402 static inline void arch_swap_invalidate_page(int type, pgoff_t offset)
1403 {
1404 if (system_supports_mte())
1405 mte_invalidate_tags(type, offset);
1406 }
1407
1408 static inline void arch_swap_invalidate_area(int type)
1409 {
1410 if (system_supports_mte())
1411 mte_invalidate_tags_area(type);
1412 }
1413
1414 #define __HAVE_ARCH_SWAP_RESTORE
1415 extern void arch_swap_restore(swp_entry_t entry, struct folio *folio);
1416
1417 #endif /* CONFIG_ARM64_MTE */
1418
1419 /*
1420 * On AArch64, the cache coherency is handled via the __set_ptes() function.
1421 */
1422 static inline void update_mmu_cache_range(struct vm_fault *vmf,
1423 struct vm_area_struct *vma, unsigned long addr, pte_t *ptep,
1424 unsigned int nr)
1425 {
1426 /*
1427 * We don't do anything here, so there's a very small chance of
1428 * us retaking a user fault which we just fixed up. The alternative
1429 * is doing a dsb(ishst), but that penalises the fastpath.
1430 */
1431 }
1432
1433 #define update_mmu_cache(vma, addr, ptep) \
1434 update_mmu_cache_range(NULL, vma, addr, ptep, 1)
1435 #define update_mmu_cache_pmd(vma, address, pmd) do { } while (0)
1436
1437 #ifdef CONFIG_ARM64_PA_BITS_52
1438 #define phys_to_ttbr(addr) (((addr) | ((addr) >> 46)) & TTBR_BADDR_MASK_52)
1439 #else
1440 #define phys_to_ttbr(addr) (addr)
1441 #endif
1442
1443 /*
1444 * On arm64 without hardware Access Flag, copying from user will fail because
1445 * the pte is old and cannot be marked young. So we always end up with zeroed
1446 * page after fork() + CoW for pfn mappings. We don't always have a
1447 * hardware-managed access flag on arm64.
1448 */
1449 #define arch_has_hw_pte_young cpu_has_hw_af
1450
1451 /*
1452 * Experimentally, it's cheap to set the access flag in hardware and we
1453 * benefit from prefaulting mappings as 'old' to start with.
1454 */
1455 #define arch_wants_old_prefaulted_pte cpu_has_hw_af
1456
1457 static inline bool pud_sect_supported(void)
1458 {
1459 return PAGE_SIZE == SZ_4K;
1460 }
1461
1462
1463 #define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION
1464 #define ptep_modify_prot_start ptep_modify_prot_start
1465 extern pte_t ptep_modify_prot_start(struct vm_area_struct *vma,
1466 unsigned long addr, pte_t *ptep);
1467
1468 #define ptep_modify_prot_commit ptep_modify_prot_commit
1469 extern void ptep_modify_prot_commit(struct vm_area_struct *vma,
1470 unsigned long addr, pte_t *ptep,
1471 pte_t old_pte, pte_t new_pte);
1472
1473 #ifdef CONFIG_ARM64_CONTPTE
1474
1475 /*
1476 * The contpte APIs are used to transparently manage the contiguous bit in ptes
1477 * where it is possible and makes sense to do so. The PTE_CONT bit is considered
1478 * a private implementation detail of the public ptep API (see below).
1479 */
1480 extern void __contpte_try_fold(struct mm_struct *mm, unsigned long addr,
1481 pte_t *ptep, pte_t pte);
1482 extern void __contpte_try_unfold(struct mm_struct *mm, unsigned long addr,
1483 pte_t *ptep, pte_t pte);
1484 extern pte_t contpte_ptep_get(pte_t *ptep, pte_t orig_pte);
1485 extern pte_t contpte_ptep_get_lockless(pte_t *orig_ptep);
1486 extern void contpte_set_ptes(struct mm_struct *mm, unsigned long addr,
1487 pte_t *ptep, pte_t pte, unsigned int nr);
1488 extern void contpte_clear_full_ptes(struct mm_struct *mm, unsigned long addr,
1489 pte_t *ptep, unsigned int nr, int full);
1490 extern pte_t contpte_get_and_clear_full_ptes(struct mm_struct *mm,
1491 unsigned long addr, pte_t *ptep,
1492 unsigned int nr, int full);
1493 extern int contpte_ptep_test_and_clear_young(struct vm_area_struct *vma,
1494 unsigned long addr, pte_t *ptep);
1495 extern int contpte_ptep_clear_flush_young(struct vm_area_struct *vma,
1496 unsigned long addr, pte_t *ptep);
1497 extern void contpte_wrprotect_ptes(struct mm_struct *mm, unsigned long addr,
1498 pte_t *ptep, unsigned int nr);
1499 extern int contpte_ptep_set_access_flags(struct vm_area_struct *vma,
1500 unsigned long addr, pte_t *ptep,
1501 pte_t entry, int dirty);
1502 extern void contpte_clear_young_dirty_ptes(struct vm_area_struct *vma,
1503 unsigned long addr, pte_t *ptep,
1504 unsigned int nr, cydp_t flags);
1505
1506 static __always_inline void contpte_try_fold(struct mm_struct *mm,
1507 unsigned long addr, pte_t *ptep, pte_t pte)
1508 {
1509 /*
1510 * Only bother trying if both the virtual and physical addresses are
1511 * aligned and correspond to the last entry in a contig range. The core
1512 * code mostly modifies ranges from low to high, so this is the likely
1513 * the last modification in the contig range, so a good time to fold.
1514 * We can't fold special mappings, because there is no associated folio.
1515 */
1516
1517 const unsigned long contmask = CONT_PTES - 1;
1518 bool valign = ((addr >> PAGE_SHIFT) & contmask) == contmask;
1519
1520 if (unlikely(valign)) {
1521 bool palign = (pte_pfn(pte) & contmask) == contmask;
1522
1523 if (unlikely(palign &&
1524 pte_valid(pte) && !pte_cont(pte) && !pte_special(pte)))
1525 __contpte_try_fold(mm, addr, ptep, pte);
1526 }
1527 }
1528
1529 static __always_inline void contpte_try_unfold(struct mm_struct *mm,
1530 unsigned long addr, pte_t *ptep, pte_t pte)
1531 {
1532 if (unlikely(pte_valid_cont(pte)))
1533 __contpte_try_unfold(mm, addr, ptep, pte);
1534 }
1535
1536 #define pte_batch_hint pte_batch_hint
1537 static inline unsigned int pte_batch_hint(pte_t *ptep, pte_t pte)
1538 {
1539 if (!pte_valid_cont(pte))
1540 return 1;
1541
1542 return CONT_PTES - (((unsigned long)ptep >> 3) & (CONT_PTES - 1));
1543 }
1544
1545 /*
1546 * The below functions constitute the public API that arm64 presents to the
1547 * core-mm to manipulate PTE entries within their page tables (or at least this
1548 * is the subset of the API that arm64 needs to implement). These public
1549 * versions will automatically and transparently apply the contiguous bit where
1550 * it makes sense to do so. Therefore any users that are contig-aware (e.g.
1551 * hugetlb, kernel mapper) should NOT use these APIs, but instead use the
1552 * private versions, which are prefixed with double underscore. All of these
1553 * APIs except for ptep_get_lockless() are expected to be called with the PTL
1554 * held. Although the contiguous bit is considered private to the
1555 * implementation, it is deliberately allowed to leak through the getters (e.g.
1556 * ptep_get()), back to core code. This is required so that pte_leaf_size() can
1557 * provide an accurate size for perf_get_pgtable_size(). But this leakage means
1558 * its possible a pte will be passed to a setter with the contiguous bit set, so
1559 * we explicitly clear the contiguous bit in those cases to prevent accidentally
1560 * setting it in the pgtable.
1561 */
1562
1563 #define ptep_get ptep_get
1564 static inline pte_t ptep_get(pte_t *ptep)
1565 {
1566 pte_t pte = __ptep_get(ptep);
1567
1568 if (likely(!pte_valid_cont(pte)))
1569 return pte;
1570
1571 return contpte_ptep_get(ptep, pte);
1572 }
1573
1574 #define ptep_get_lockless ptep_get_lockless
1575 static inline pte_t ptep_get_lockless(pte_t *ptep)
1576 {
1577 pte_t pte = __ptep_get(ptep);
1578
1579 if (likely(!pte_valid_cont(pte)))
1580 return pte;
1581
1582 return contpte_ptep_get_lockless(ptep);
1583 }
1584
1585 static inline void set_pte(pte_t *ptep, pte_t pte)
1586 {
1587 /*
1588 * We don't have the mm or vaddr so cannot unfold contig entries (since
1589 * it requires tlb maintenance). set_pte() is not used in core code, so
1590 * this should never even be called. Regardless do our best to service
1591 * any call and emit a warning if there is any attempt to set a pte on
1592 * top of an existing contig range.
1593 */
1594 pte_t orig_pte = __ptep_get(ptep);
1595
1596 WARN_ON_ONCE(pte_valid_cont(orig_pte));
1597 __set_pte(ptep, pte_mknoncont(pte));
1598 }
1599
1600 #define set_ptes set_ptes
1601 static __always_inline void set_ptes(struct mm_struct *mm, unsigned long addr,
1602 pte_t *ptep, pte_t pte, unsigned int nr)
1603 {
1604 pte = pte_mknoncont(pte);
1605
1606 if (likely(nr == 1)) {
1607 contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep));
1608 __set_ptes(mm, addr, ptep, pte, 1);
1609 contpte_try_fold(mm, addr, ptep, pte);
1610 } else {
1611 contpte_set_ptes(mm, addr, ptep, pte, nr);
1612 }
1613 }
1614
1615 static inline void pte_clear(struct mm_struct *mm,
1616 unsigned long addr, pte_t *ptep)
1617 {
1618 contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep));
1619 __pte_clear(mm, addr, ptep);
1620 }
1621
1622 #define clear_full_ptes clear_full_ptes
1623 static inline void clear_full_ptes(struct mm_struct *mm, unsigned long addr,
1624 pte_t *ptep, unsigned int nr, int full)
1625 {
1626 if (likely(nr == 1)) {
1627 contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep));
1628 __clear_full_ptes(mm, addr, ptep, nr, full);
1629 } else {
1630 contpte_clear_full_ptes(mm, addr, ptep, nr, full);
1631 }
1632 }
1633
1634 #define get_and_clear_full_ptes get_and_clear_full_ptes
1635 static inline pte_t get_and_clear_full_ptes(struct mm_struct *mm,
1636 unsigned long addr, pte_t *ptep,
1637 unsigned int nr, int full)
1638 {
1639 pte_t pte;
1640
1641 if (likely(nr == 1)) {
1642 contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep));
1643 pte = __get_and_clear_full_ptes(mm, addr, ptep, nr, full);
1644 } else {
1645 pte = contpte_get_and_clear_full_ptes(mm, addr, ptep, nr, full);
1646 }
1647
1648 return pte;
1649 }
1650
1651 #define __HAVE_ARCH_PTEP_GET_AND_CLEAR
1652 static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
1653 unsigned long addr, pte_t *ptep)
1654 {
1655 contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep));
1656 return __ptep_get_and_clear(mm, addr, ptep);
1657 }
1658
1659 #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
1660 static inline int ptep_test_and_clear_young(struct vm_area_struct *vma,
1661 unsigned long addr, pte_t *ptep)
1662 {
1663 pte_t orig_pte = __ptep_get(ptep);
1664
1665 if (likely(!pte_valid_cont(orig_pte)))
1666 return __ptep_test_and_clear_young(vma, addr, ptep);
1667
1668 return contpte_ptep_test_and_clear_young(vma, addr, ptep);
1669 }
1670
1671 #define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
1672 static inline int ptep_clear_flush_young(struct vm_area_struct *vma,
1673 unsigned long addr, pte_t *ptep)
1674 {
1675 pte_t orig_pte = __ptep_get(ptep);
1676
1677 if (likely(!pte_valid_cont(orig_pte)))
1678 return __ptep_clear_flush_young(vma, addr, ptep);
1679
1680 return contpte_ptep_clear_flush_young(vma, addr, ptep);
1681 }
1682
1683 #define wrprotect_ptes wrprotect_ptes
1684 static __always_inline void wrprotect_ptes(struct mm_struct *mm,
1685 unsigned long addr, pte_t *ptep, unsigned int nr)
1686 {
1687 if (likely(nr == 1)) {
1688 /*
1689 * Optimization: wrprotect_ptes() can only be called for present
1690 * ptes so we only need to check contig bit as condition for
1691 * unfold, and we can remove the contig bit from the pte we read
1692 * to avoid re-reading. This speeds up fork() which is sensitive
1693 * for order-0 folios. Equivalent to contpte_try_unfold().
1694 */
1695 pte_t orig_pte = __ptep_get(ptep);
1696
1697 if (unlikely(pte_cont(orig_pte))) {
1698 __contpte_try_unfold(mm, addr, ptep, orig_pte);
1699 orig_pte = pte_mknoncont(orig_pte);
1700 }
1701 ___ptep_set_wrprotect(mm, addr, ptep, orig_pte);
1702 } else {
1703 contpte_wrprotect_ptes(mm, addr, ptep, nr);
1704 }
1705 }
1706
1707 #define __HAVE_ARCH_PTEP_SET_WRPROTECT
1708 static inline void ptep_set_wrprotect(struct mm_struct *mm,
1709 unsigned long addr, pte_t *ptep)
1710 {
1711 wrprotect_ptes(mm, addr, ptep, 1);
1712 }
1713
1714 #define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
1715 static inline int ptep_set_access_flags(struct vm_area_struct *vma,
1716 unsigned long addr, pte_t *ptep,
1717 pte_t entry, int dirty)
1718 {
1719 pte_t orig_pte = __ptep_get(ptep);
1720
1721 entry = pte_mknoncont(entry);
1722
1723 if (likely(!pte_valid_cont(orig_pte)))
1724 return __ptep_set_access_flags(vma, addr, ptep, entry, dirty);
1725
1726 return contpte_ptep_set_access_flags(vma, addr, ptep, entry, dirty);
1727 }
1728
1729 #define clear_young_dirty_ptes clear_young_dirty_ptes
1730 static inline void clear_young_dirty_ptes(struct vm_area_struct *vma,
1731 unsigned long addr, pte_t *ptep,
1732 unsigned int nr, cydp_t flags)
1733 {
1734 if (likely(nr == 1 && !pte_cont(__ptep_get(ptep))))
1735 __clear_young_dirty_ptes(vma, addr, ptep, nr, flags);
1736 else
1737 contpte_clear_young_dirty_ptes(vma, addr, ptep, nr, flags);
1738 }
1739
1740 #else /* CONFIG_ARM64_CONTPTE */
1741
1742 #define ptep_get __ptep_get
1743 #define set_pte __set_pte
1744 #define set_ptes __set_ptes
1745 #define pte_clear __pte_clear
1746 #define clear_full_ptes __clear_full_ptes
1747 #define get_and_clear_full_ptes __get_and_clear_full_ptes
1748 #define __HAVE_ARCH_PTEP_GET_AND_CLEAR
1749 #define ptep_get_and_clear __ptep_get_and_clear
1750 #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
1751 #define ptep_test_and_clear_young __ptep_test_and_clear_young
1752 #define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
1753 #define ptep_clear_flush_young __ptep_clear_flush_young
1754 #define __HAVE_ARCH_PTEP_SET_WRPROTECT
1755 #define ptep_set_wrprotect __ptep_set_wrprotect
1756 #define wrprotect_ptes __wrprotect_ptes
1757 #define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
1758 #define ptep_set_access_flags __ptep_set_access_flags
1759 #define clear_young_dirty_ptes __clear_young_dirty_ptes
1760
1761 #endif /* CONFIG_ARM64_CONTPTE */
1762
1763 #endif /* !__ASSEMBLY__ */
1764
1765 #endif /* __ASM_PGTABLE_H */
1766
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.