1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _LINUX_PGTABLE_H
3 #define _LINUX_PGTABLE_H
4
5 #include <linux/pfn.h>
6 #include <asm/pgtable.h>
7
8 #define PMD_ORDER (PMD_SHIFT - PAGE_SHIFT)
9 #define PUD_ORDER (PUD_SHIFT - PAGE_SHIFT)
10
11 #ifndef __ASSEMBLY__
12 #ifdef CONFIG_MMU
13
14 #include <linux/mm_types.h>
15 #include <linux/bug.h>
16 #include <linux/errno.h>
17 #include <asm-generic/pgtable_uffd.h>
18 #include <linux/page_table_check.h>
19
20 #if 5 - defined(__PAGETABLE_P4D_FOLDED) - defined(__PAGETABLE_PUD_FOLDED) - \
21 defined(__PAGETABLE_PMD_FOLDED) != CONFIG_PGTABLE_LEVELS
22 #error CONFIG_PGTABLE_LEVELS is not consistent with __PAGETABLE_{P4D,PUD,PMD}_FOLDED
23 #endif
24
25 /*
26 * On almost all architectures and configurations, 0 can be used as the
27 * upper ceiling to free_pgtables(): on many architectures it has the same
28 * effect as using TASK_SIZE. However, there is one configuration which
29 * must impose a more careful limit, to avoid freeing kernel pgtables.
30 */
31 #ifndef USER_PGTABLES_CEILING
32 #define USER_PGTABLES_CEILING 0UL
33 #endif
34
35 /*
36 * This defines the first usable user address. Platforms
37 * can override its value with custom FIRST_USER_ADDRESS
38 * defined in their respective <asm/pgtable.h>.
39 */
40 #ifndef FIRST_USER_ADDRESS
41 #define FIRST_USER_ADDRESS 0UL
42 #endif
43
44 /*
45 * This defines the generic helper for accessing PMD page
46 * table page. Although platforms can still override this
47 * via their respective <asm/pgtable.h>.
48 */
49 #ifndef pmd_pgtable
50 #define pmd_pgtable(pmd) pmd_page(pmd)
51 #endif
52
53 #define pmd_folio(pmd) page_folio(pmd_page(pmd))
54
55 /*
56 * A page table page can be thought of an array like this: pXd_t[PTRS_PER_PxD]
57 *
58 * The pXx_index() functions return the index of the entry in the page
59 * table page which would control the given virtual address
60 *
61 * As these functions may be used by the same code for different levels of
62 * the page table folding, they are always available, regardless of
63 * CONFIG_PGTABLE_LEVELS value. For the folded levels they simply return 0
64 * because in such cases PTRS_PER_PxD equals 1.
65 */
66
67 static inline unsigned long pte_index(unsigned long address)
68 {
69 return (address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1);
70 }
71
72 #ifndef pmd_index
73 static inline unsigned long pmd_index(unsigned long address)
74 {
75 return (address >> PMD_SHIFT) & (PTRS_PER_PMD - 1);
76 }
77 #define pmd_index pmd_index
78 #endif
79
80 #ifndef pud_index
81 static inline unsigned long pud_index(unsigned long address)
82 {
83 return (address >> PUD_SHIFT) & (PTRS_PER_PUD - 1);
84 }
85 #define pud_index pud_index
86 #endif
87
88 #ifndef pgd_index
89 /* Must be a compile-time constant, so implement it as a macro */
90 #define pgd_index(a) (((a) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1))
91 #endif
92
93 #ifndef pte_offset_kernel
94 static inline pte_t *pte_offset_kernel(pmd_t *pmd, unsigned long address)
95 {
96 return (pte_t *)pmd_page_vaddr(*pmd) + pte_index(address);
97 }
98 #define pte_offset_kernel pte_offset_kernel
99 #endif
100
101 #ifdef CONFIG_HIGHPTE
102 #define __pte_map(pmd, address) \
103 ((pte_t *)kmap_local_page(pmd_page(*(pmd))) + pte_index((address)))
104 #define pte_unmap(pte) do { \
105 kunmap_local((pte)); \
106 rcu_read_unlock(); \
107 } while (0)
108 #else
109 static inline pte_t *__pte_map(pmd_t *pmd, unsigned long address)
110 {
111 return pte_offset_kernel(pmd, address);
112 }
113 static inline void pte_unmap(pte_t *pte)
114 {
115 rcu_read_unlock();
116 }
117 #endif
118
119 void pte_free_defer(struct mm_struct *mm, pgtable_t pgtable);
120
121 /* Find an entry in the second-level page table.. */
122 #ifndef pmd_offset
123 static inline pmd_t *pmd_offset(pud_t *pud, unsigned long address)
124 {
125 return pud_pgtable(*pud) + pmd_index(address);
126 }
127 #define pmd_offset pmd_offset
128 #endif
129
130 #ifndef pud_offset
131 static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address)
132 {
133 return p4d_pgtable(*p4d) + pud_index(address);
134 }
135 #define pud_offset pud_offset
136 #endif
137
138 static inline pgd_t *pgd_offset_pgd(pgd_t *pgd, unsigned long address)
139 {
140 return (pgd + pgd_index(address));
141 };
142
143 /*
144 * a shortcut to get a pgd_t in a given mm
145 */
146 #ifndef pgd_offset
147 #define pgd_offset(mm, address) pgd_offset_pgd((mm)->pgd, (address))
148 #endif
149
150 /*
151 * a shortcut which implies the use of the kernel's pgd, instead
152 * of a process's
153 */
154 #define pgd_offset_k(address) pgd_offset(&init_mm, (address))
155
156 /*
157 * In many cases it is known that a virtual address is mapped at PMD or PTE
158 * level, so instead of traversing all the page table levels, we can get a
159 * pointer to the PMD entry in user or kernel page table or translate a virtual
160 * address to the pointer in the PTE in the kernel page tables with simple
161 * helpers.
162 */
163 static inline pmd_t *pmd_off(struct mm_struct *mm, unsigned long va)
164 {
165 return pmd_offset(pud_offset(p4d_offset(pgd_offset(mm, va), va), va), va);
166 }
167
168 static inline pmd_t *pmd_off_k(unsigned long va)
169 {
170 return pmd_offset(pud_offset(p4d_offset(pgd_offset_k(va), va), va), va);
171 }
172
173 static inline pte_t *virt_to_kpte(unsigned long vaddr)
174 {
175 pmd_t *pmd = pmd_off_k(vaddr);
176
177 return pmd_none(*pmd) ? NULL : pte_offset_kernel(pmd, vaddr);
178 }
179
180 #ifndef pmd_young
181 static inline int pmd_young(pmd_t pmd)
182 {
183 return 0;
184 }
185 #endif
186
187 #ifndef pmd_dirty
188 static inline int pmd_dirty(pmd_t pmd)
189 {
190 return 0;
191 }
192 #endif
193
194 /*
195 * A facility to provide lazy MMU batching. This allows PTE updates and
196 * page invalidations to be delayed until a call to leave lazy MMU mode
197 * is issued. Some architectures may benefit from doing this, and it is
198 * beneficial for both shadow and direct mode hypervisors, which may batch
199 * the PTE updates which happen during this window. Note that using this
200 * interface requires that read hazards be removed from the code. A read
201 * hazard could result in the direct mode hypervisor case, since the actual
202 * write to the page tables may not yet have taken place, so reads though
203 * a raw PTE pointer after it has been modified are not guaranteed to be
204 * up to date. This mode can only be entered and left under the protection of
205 * the page table locks for all page tables which may be modified. In the UP
206 * case, this is required so that preemption is disabled, and in the SMP case,
207 * it must synchronize the delayed page table writes properly on other CPUs.
208 */
209 #ifndef __HAVE_ARCH_ENTER_LAZY_MMU_MODE
210 #define arch_enter_lazy_mmu_mode() do {} while (0)
211 #define arch_leave_lazy_mmu_mode() do {} while (0)
212 #define arch_flush_lazy_mmu_mode() do {} while (0)
213 #endif
214
215 #ifndef pte_batch_hint
216 /**
217 * pte_batch_hint - Number of pages that can be added to batch without scanning.
218 * @ptep: Page table pointer for the entry.
219 * @pte: Page table entry.
220 *
221 * Some architectures know that a set of contiguous ptes all map the same
222 * contiguous memory with the same permissions. In this case, it can provide a
223 * hint to aid pte batching without the core code needing to scan every pte.
224 *
225 * An architecture implementation may ignore the PTE accessed state. Further,
226 * the dirty state must apply atomically to all the PTEs described by the hint.
227 *
228 * May be overridden by the architecture, else pte_batch_hint is always 1.
229 */
230 static inline unsigned int pte_batch_hint(pte_t *ptep, pte_t pte)
231 {
232 return 1;
233 }
234 #endif
235
236 #ifndef pte_advance_pfn
237 static inline pte_t pte_advance_pfn(pte_t pte, unsigned long nr)
238 {
239 return __pte(pte_val(pte) + (nr << PFN_PTE_SHIFT));
240 }
241 #endif
242
243 #define pte_next_pfn(pte) pte_advance_pfn(pte, 1)
244
245 #ifndef set_ptes
246 /**
247 * set_ptes - Map consecutive pages to a contiguous range of addresses.
248 * @mm: Address space to map the pages into.
249 * @addr: Address to map the first page at.
250 * @ptep: Page table pointer for the first entry.
251 * @pte: Page table entry for the first page.
252 * @nr: Number of pages to map.
253 *
254 * When nr==1, initial state of pte may be present or not present, and new state
255 * may be present or not present. When nr>1, initial state of all ptes must be
256 * not present, and new state must be present.
257 *
258 * May be overridden by the architecture, or the architecture can define
259 * set_pte() and PFN_PTE_SHIFT.
260 *
261 * Context: The caller holds the page table lock. The pages all belong
262 * to the same folio. The PTEs are all in the same PMD.
263 */
264 static inline void set_ptes(struct mm_struct *mm, unsigned long addr,
265 pte_t *ptep, pte_t pte, unsigned int nr)
266 {
267 page_table_check_ptes_set(mm, ptep, pte, nr);
268
269 arch_enter_lazy_mmu_mode();
270 for (;;) {
271 set_pte(ptep, pte);
272 if (--nr == 0)
273 break;
274 ptep++;
275 pte = pte_next_pfn(pte);
276 }
277 arch_leave_lazy_mmu_mode();
278 }
279 #endif
280 #define set_pte_at(mm, addr, ptep, pte) set_ptes(mm, addr, ptep, pte, 1)
281
282 #ifndef __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
283 extern int ptep_set_access_flags(struct vm_area_struct *vma,
284 unsigned long address, pte_t *ptep,
285 pte_t entry, int dirty);
286 #endif
287
288 #ifndef __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
289 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
290 extern int pmdp_set_access_flags(struct vm_area_struct *vma,
291 unsigned long address, pmd_t *pmdp,
292 pmd_t entry, int dirty);
293 extern int pudp_set_access_flags(struct vm_area_struct *vma,
294 unsigned long address, pud_t *pudp,
295 pud_t entry, int dirty);
296 #else
297 static inline int pmdp_set_access_flags(struct vm_area_struct *vma,
298 unsigned long address, pmd_t *pmdp,
299 pmd_t entry, int dirty)
300 {
301 BUILD_BUG();
302 return 0;
303 }
304 static inline int pudp_set_access_flags(struct vm_area_struct *vma,
305 unsigned long address, pud_t *pudp,
306 pud_t entry, int dirty)
307 {
308 BUILD_BUG();
309 return 0;
310 }
311 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
312 #endif
313
314 #ifndef ptep_get
315 static inline pte_t ptep_get(pte_t *ptep)
316 {
317 return READ_ONCE(*ptep);
318 }
319 #endif
320
321 #ifndef pmdp_get
322 static inline pmd_t pmdp_get(pmd_t *pmdp)
323 {
324 return READ_ONCE(*pmdp);
325 }
326 #endif
327
328 #ifndef pudp_get
329 static inline pud_t pudp_get(pud_t *pudp)
330 {
331 return READ_ONCE(*pudp);
332 }
333 #endif
334
335 #ifndef p4dp_get
336 static inline p4d_t p4dp_get(p4d_t *p4dp)
337 {
338 return READ_ONCE(*p4dp);
339 }
340 #endif
341
342 #ifndef pgdp_get
343 static inline pgd_t pgdp_get(pgd_t *pgdp)
344 {
345 return READ_ONCE(*pgdp);
346 }
347 #endif
348
349 #ifndef __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
350 static inline int ptep_test_and_clear_young(struct vm_area_struct *vma,
351 unsigned long address,
352 pte_t *ptep)
353 {
354 pte_t pte = ptep_get(ptep);
355 int r = 1;
356 if (!pte_young(pte))
357 r = 0;
358 else
359 set_pte_at(vma->vm_mm, address, ptep, pte_mkold(pte));
360 return r;
361 }
362 #endif
363
364 #ifndef __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
365 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG)
366 static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
367 unsigned long address,
368 pmd_t *pmdp)
369 {
370 pmd_t pmd = *pmdp;
371 int r = 1;
372 if (!pmd_young(pmd))
373 r = 0;
374 else
375 set_pmd_at(vma->vm_mm, address, pmdp, pmd_mkold(pmd));
376 return r;
377 }
378 #else
379 static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
380 unsigned long address,
381 pmd_t *pmdp)
382 {
383 BUILD_BUG();
384 return 0;
385 }
386 #endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG */
387 #endif
388
389 #ifndef __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
390 int ptep_clear_flush_young(struct vm_area_struct *vma,
391 unsigned long address, pte_t *ptep);
392 #endif
393
394 #ifndef __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH
395 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
396 extern int pmdp_clear_flush_young(struct vm_area_struct *vma,
397 unsigned long address, pmd_t *pmdp);
398 #else
399 /*
400 * Despite relevant to THP only, this API is called from generic rmap code
401 * under PageTransHuge(), hence needs a dummy implementation for !THP
402 */
403 static inline int pmdp_clear_flush_young(struct vm_area_struct *vma,
404 unsigned long address, pmd_t *pmdp)
405 {
406 BUILD_BUG();
407 return 0;
408 }
409 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
410 #endif
411
412 #ifndef arch_has_hw_nonleaf_pmd_young
413 /*
414 * Return whether the accessed bit in non-leaf PMD entries is supported on the
415 * local CPU.
416 */
417 static inline bool arch_has_hw_nonleaf_pmd_young(void)
418 {
419 return IS_ENABLED(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG);
420 }
421 #endif
422
423 #ifndef arch_has_hw_pte_young
424 /*
425 * Return whether the accessed bit is supported on the local CPU.
426 *
427 * This stub assumes accessing through an old PTE triggers a page fault.
428 * Architectures that automatically set the access bit should overwrite it.
429 */
430 static inline bool arch_has_hw_pte_young(void)
431 {
432 return IS_ENABLED(CONFIG_ARCH_HAS_HW_PTE_YOUNG);
433 }
434 #endif
435
436 #ifndef arch_check_zapped_pte
437 static inline void arch_check_zapped_pte(struct vm_area_struct *vma,
438 pte_t pte)
439 {
440 }
441 #endif
442
443 #ifndef arch_check_zapped_pmd
444 static inline void arch_check_zapped_pmd(struct vm_area_struct *vma,
445 pmd_t pmd)
446 {
447 }
448 #endif
449
450 #ifndef __HAVE_ARCH_PTEP_GET_AND_CLEAR
451 static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
452 unsigned long address,
453 pte_t *ptep)
454 {
455 pte_t pte = ptep_get(ptep);
456 pte_clear(mm, address, ptep);
457 page_table_check_pte_clear(mm, pte);
458 return pte;
459 }
460 #endif
461
462 #ifndef clear_young_dirty_ptes
463 /**
464 * clear_young_dirty_ptes - Mark PTEs that map consecutive pages of the
465 * same folio as old/clean.
466 * @mm: Address space the pages are mapped into.
467 * @addr: Address the first page is mapped at.
468 * @ptep: Page table pointer for the first entry.
469 * @nr: Number of entries to mark old/clean.
470 * @flags: Flags to modify the PTE batch semantics.
471 *
472 * May be overridden by the architecture; otherwise, implemented by
473 * get_and_clear/modify/set for each pte in the range.
474 *
475 * Note that PTE bits in the PTE range besides the PFN can differ. For example,
476 * some PTEs might be write-protected.
477 *
478 * Context: The caller holds the page table lock. The PTEs map consecutive
479 * pages that belong to the same folio. The PTEs are all in the same PMD.
480 */
481 static inline void clear_young_dirty_ptes(struct vm_area_struct *vma,
482 unsigned long addr, pte_t *ptep,
483 unsigned int nr, cydp_t flags)
484 {
485 pte_t pte;
486
487 for (;;) {
488 if (flags == CYDP_CLEAR_YOUNG)
489 ptep_test_and_clear_young(vma, addr, ptep);
490 else {
491 pte = ptep_get_and_clear(vma->vm_mm, addr, ptep);
492 if (flags & CYDP_CLEAR_YOUNG)
493 pte = pte_mkold(pte);
494 if (flags & CYDP_CLEAR_DIRTY)
495 pte = pte_mkclean(pte);
496 set_pte_at(vma->vm_mm, addr, ptep, pte);
497 }
498 if (--nr == 0)
499 break;
500 ptep++;
501 addr += PAGE_SIZE;
502 }
503 }
504 #endif
505
506 static inline void ptep_clear(struct mm_struct *mm, unsigned long addr,
507 pte_t *ptep)
508 {
509 ptep_get_and_clear(mm, addr, ptep);
510 }
511
512 #ifdef CONFIG_GUP_GET_PXX_LOW_HIGH
513 /*
514 * For walking the pagetables without holding any locks. Some architectures
515 * (eg x86-32 PAE) cannot load the entries atomically without using expensive
516 * instructions. We are guaranteed that a PTE will only either go from not
517 * present to present, or present to not present -- it will not switch to a
518 * completely different present page without a TLB flush inbetween; which we
519 * are blocking by holding interrupts off.
520 *
521 * Setting ptes from not present to present goes:
522 *
523 * ptep->pte_high = h;
524 * smp_wmb();
525 * ptep->pte_low = l;
526 *
527 * And present to not present goes:
528 *
529 * ptep->pte_low = 0;
530 * smp_wmb();
531 * ptep->pte_high = 0;
532 *
533 * We must ensure here that the load of pte_low sees 'l' IFF pte_high sees 'h'.
534 * We load pte_high *after* loading pte_low, which ensures we don't see an older
535 * value of pte_high. *Then* we recheck pte_low, which ensures that we haven't
536 * picked up a changed pte high. We might have gotten rubbish values from
537 * pte_low and pte_high, but we are guaranteed that pte_low will not have the
538 * present bit set *unless* it is 'l'. Because get_user_pages_fast() only
539 * operates on present ptes we're safe.
540 */
541 static inline pte_t ptep_get_lockless(pte_t *ptep)
542 {
543 pte_t pte;
544
545 do {
546 pte.pte_low = ptep->pte_low;
547 smp_rmb();
548 pte.pte_high = ptep->pte_high;
549 smp_rmb();
550 } while (unlikely(pte.pte_low != ptep->pte_low));
551
552 return pte;
553 }
554 #define ptep_get_lockless ptep_get_lockless
555
556 #if CONFIG_PGTABLE_LEVELS > 2
557 static inline pmd_t pmdp_get_lockless(pmd_t *pmdp)
558 {
559 pmd_t pmd;
560
561 do {
562 pmd.pmd_low = pmdp->pmd_low;
563 smp_rmb();
564 pmd.pmd_high = pmdp->pmd_high;
565 smp_rmb();
566 } while (unlikely(pmd.pmd_low != pmdp->pmd_low));
567
568 return pmd;
569 }
570 #define pmdp_get_lockless pmdp_get_lockless
571 #define pmdp_get_lockless_sync() tlb_remove_table_sync_one()
572 #endif /* CONFIG_PGTABLE_LEVELS > 2 */
573 #endif /* CONFIG_GUP_GET_PXX_LOW_HIGH */
574
575 /*
576 * We require that the PTE can be read atomically.
577 */
578 #ifndef ptep_get_lockless
579 static inline pte_t ptep_get_lockless(pte_t *ptep)
580 {
581 return ptep_get(ptep);
582 }
583 #endif
584
585 #ifndef pmdp_get_lockless
586 static inline pmd_t pmdp_get_lockless(pmd_t *pmdp)
587 {
588 return pmdp_get(pmdp);
589 }
590 static inline void pmdp_get_lockless_sync(void)
591 {
592 }
593 #endif
594
595 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
596 #ifndef __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
597 static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
598 unsigned long address,
599 pmd_t *pmdp)
600 {
601 pmd_t pmd = *pmdp;
602
603 pmd_clear(pmdp);
604 page_table_check_pmd_clear(mm, pmd);
605
606 return pmd;
607 }
608 #endif /* __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR */
609 #ifndef __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR
610 static inline pud_t pudp_huge_get_and_clear(struct mm_struct *mm,
611 unsigned long address,
612 pud_t *pudp)
613 {
614 pud_t pud = *pudp;
615
616 pud_clear(pudp);
617 page_table_check_pud_clear(mm, pud);
618
619 return pud;
620 }
621 #endif /* __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR */
622 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
623
624 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
625 #ifndef __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR_FULL
626 static inline pmd_t pmdp_huge_get_and_clear_full(struct vm_area_struct *vma,
627 unsigned long address, pmd_t *pmdp,
628 int full)
629 {
630 return pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp);
631 }
632 #endif
633
634 #ifndef __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR_FULL
635 static inline pud_t pudp_huge_get_and_clear_full(struct vm_area_struct *vma,
636 unsigned long address, pud_t *pudp,
637 int full)
638 {
639 return pudp_huge_get_and_clear(vma->vm_mm, address, pudp);
640 }
641 #endif
642 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
643
644 #ifndef __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL
645 static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,
646 unsigned long address, pte_t *ptep,
647 int full)
648 {
649 return ptep_get_and_clear(mm, address, ptep);
650 }
651 #endif
652
653 #ifndef get_and_clear_full_ptes
654 /**
655 * get_and_clear_full_ptes - Clear present PTEs that map consecutive pages of
656 * the same folio, collecting dirty/accessed bits.
657 * @mm: Address space the pages are mapped into.
658 * @addr: Address the first page is mapped at.
659 * @ptep: Page table pointer for the first entry.
660 * @nr: Number of entries to clear.
661 * @full: Whether we are clearing a full mm.
662 *
663 * May be overridden by the architecture; otherwise, implemented as a simple
664 * loop over ptep_get_and_clear_full(), merging dirty/accessed bits into the
665 * returned PTE.
666 *
667 * Note that PTE bits in the PTE range besides the PFN can differ. For example,
668 * some PTEs might be write-protected.
669 *
670 * Context: The caller holds the page table lock. The PTEs map consecutive
671 * pages that belong to the same folio. The PTEs are all in the same PMD.
672 */
673 static inline pte_t get_and_clear_full_ptes(struct mm_struct *mm,
674 unsigned long addr, pte_t *ptep, unsigned int nr, int full)
675 {
676 pte_t pte, tmp_pte;
677
678 pte = ptep_get_and_clear_full(mm, addr, ptep, full);
679 while (--nr) {
680 ptep++;
681 addr += PAGE_SIZE;
682 tmp_pte = ptep_get_and_clear_full(mm, addr, ptep, full);
683 if (pte_dirty(tmp_pte))
684 pte = pte_mkdirty(pte);
685 if (pte_young(tmp_pte))
686 pte = pte_mkyoung(pte);
687 }
688 return pte;
689 }
690 #endif
691
692 #ifndef clear_full_ptes
693 /**
694 * clear_full_ptes - Clear present PTEs that map consecutive pages of the same
695 * folio.
696 * @mm: Address space the pages are mapped into.
697 * @addr: Address the first page is mapped at.
698 * @ptep: Page table pointer for the first entry.
699 * @nr: Number of entries to clear.
700 * @full: Whether we are clearing a full mm.
701 *
702 * May be overridden by the architecture; otherwise, implemented as a simple
703 * loop over ptep_get_and_clear_full().
704 *
705 * Note that PTE bits in the PTE range besides the PFN can differ. For example,
706 * some PTEs might be write-protected.
707 *
708 * Context: The caller holds the page table lock. The PTEs map consecutive
709 * pages that belong to the same folio. The PTEs are all in the same PMD.
710 */
711 static inline void clear_full_ptes(struct mm_struct *mm, unsigned long addr,
712 pte_t *ptep, unsigned int nr, int full)
713 {
714 for (;;) {
715 ptep_get_and_clear_full(mm, addr, ptep, full);
716 if (--nr == 0)
717 break;
718 ptep++;
719 addr += PAGE_SIZE;
720 }
721 }
722 #endif
723
724 /*
725 * If two threads concurrently fault at the same page, the thread that
726 * won the race updates the PTE and its local TLB/Cache. The other thread
727 * gives up, simply does nothing, and continues; on architectures where
728 * software can update TLB, local TLB can be updated here to avoid next page
729 * fault. This function updates TLB only, do nothing with cache or others.
730 * It is the difference with function update_mmu_cache.
731 */
732 #ifndef update_mmu_tlb_range
733 static inline void update_mmu_tlb_range(struct vm_area_struct *vma,
734 unsigned long address, pte_t *ptep, unsigned int nr)
735 {
736 }
737 #endif
738
739 static inline void update_mmu_tlb(struct vm_area_struct *vma,
740 unsigned long address, pte_t *ptep)
741 {
742 update_mmu_tlb_range(vma, address, ptep, 1);
743 }
744
745 /*
746 * Some architectures may be able to avoid expensive synchronization
747 * primitives when modifications are made to PTE's which are already
748 * not present, or in the process of an address space destruction.
749 */
750 #ifndef __HAVE_ARCH_PTE_CLEAR_NOT_PRESENT_FULL
751 static inline void pte_clear_not_present_full(struct mm_struct *mm,
752 unsigned long address,
753 pte_t *ptep,
754 int full)
755 {
756 pte_clear(mm, address, ptep);
757 }
758 #endif
759
760 #ifndef clear_not_present_full_ptes
761 /**
762 * clear_not_present_full_ptes - Clear multiple not present PTEs which are
763 * consecutive in the pgtable.
764 * @mm: Address space the ptes represent.
765 * @addr: Address of the first pte.
766 * @ptep: Page table pointer for the first entry.
767 * @nr: Number of entries to clear.
768 * @full: Whether we are clearing a full mm.
769 *
770 * May be overridden by the architecture; otherwise, implemented as a simple
771 * loop over pte_clear_not_present_full().
772 *
773 * Context: The caller holds the page table lock. The PTEs are all not present.
774 * The PTEs are all in the same PMD.
775 */
776 static inline void clear_not_present_full_ptes(struct mm_struct *mm,
777 unsigned long addr, pte_t *ptep, unsigned int nr, int full)
778 {
779 for (;;) {
780 pte_clear_not_present_full(mm, addr, ptep, full);
781 if (--nr == 0)
782 break;
783 ptep++;
784 addr += PAGE_SIZE;
785 }
786 }
787 #endif
788
789 #ifndef __HAVE_ARCH_PTEP_CLEAR_FLUSH
790 extern pte_t ptep_clear_flush(struct vm_area_struct *vma,
791 unsigned long address,
792 pte_t *ptep);
793 #endif
794
795 #ifndef __HAVE_ARCH_PMDP_HUGE_CLEAR_FLUSH
796 extern pmd_t pmdp_huge_clear_flush(struct vm_area_struct *vma,
797 unsigned long address,
798 pmd_t *pmdp);
799 extern pud_t pudp_huge_clear_flush(struct vm_area_struct *vma,
800 unsigned long address,
801 pud_t *pudp);
802 #endif
803
804 #ifndef pte_mkwrite
805 static inline pte_t pte_mkwrite(pte_t pte, struct vm_area_struct *vma)
806 {
807 return pte_mkwrite_novma(pte);
808 }
809 #endif
810
811 #if defined(CONFIG_ARCH_WANT_PMD_MKWRITE) && !defined(pmd_mkwrite)
812 static inline pmd_t pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma)
813 {
814 return pmd_mkwrite_novma(pmd);
815 }
816 #endif
817
818 #ifndef __HAVE_ARCH_PTEP_SET_WRPROTECT
819 struct mm_struct;
820 static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long address, pte_t *ptep)
821 {
822 pte_t old_pte = ptep_get(ptep);
823 set_pte_at(mm, address, ptep, pte_wrprotect(old_pte));
824 }
825 #endif
826
827 #ifndef wrprotect_ptes
828 /**
829 * wrprotect_ptes - Write-protect PTEs that map consecutive pages of the same
830 * folio.
831 * @mm: Address space the pages are mapped into.
832 * @addr: Address the first page is mapped at.
833 * @ptep: Page table pointer for the first entry.
834 * @nr: Number of entries to write-protect.
835 *
836 * May be overridden by the architecture; otherwise, implemented as a simple
837 * loop over ptep_set_wrprotect().
838 *
839 * Note that PTE bits in the PTE range besides the PFN can differ. For example,
840 * some PTEs might be write-protected.
841 *
842 * Context: The caller holds the page table lock. The PTEs map consecutive
843 * pages that belong to the same folio. The PTEs are all in the same PMD.
844 */
845 static inline void wrprotect_ptes(struct mm_struct *mm, unsigned long addr,
846 pte_t *ptep, unsigned int nr)
847 {
848 for (;;) {
849 ptep_set_wrprotect(mm, addr, ptep);
850 if (--nr == 0)
851 break;
852 ptep++;
853 addr += PAGE_SIZE;
854 }
855 }
856 #endif
857
858 /*
859 * On some architectures hardware does not set page access bit when accessing
860 * memory page, it is responsibility of software setting this bit. It brings
861 * out extra page fault penalty to track page access bit. For optimization page
862 * access bit can be set during all page fault flow on these arches.
863 * To be differentiate with macro pte_mkyoung, this macro is used on platforms
864 * where software maintains page access bit.
865 */
866 #ifndef pte_sw_mkyoung
867 static inline pte_t pte_sw_mkyoung(pte_t pte)
868 {
869 return pte;
870 }
871 #define pte_sw_mkyoung pte_sw_mkyoung
872 #endif
873
874 #ifndef __HAVE_ARCH_PMDP_SET_WRPROTECT
875 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
876 static inline void pmdp_set_wrprotect(struct mm_struct *mm,
877 unsigned long address, pmd_t *pmdp)
878 {
879 pmd_t old_pmd = *pmdp;
880 set_pmd_at(mm, address, pmdp, pmd_wrprotect(old_pmd));
881 }
882 #else
883 static inline void pmdp_set_wrprotect(struct mm_struct *mm,
884 unsigned long address, pmd_t *pmdp)
885 {
886 BUILD_BUG();
887 }
888 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
889 #endif
890 #ifndef __HAVE_ARCH_PUDP_SET_WRPROTECT
891 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
892 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
893 static inline void pudp_set_wrprotect(struct mm_struct *mm,
894 unsigned long address, pud_t *pudp)
895 {
896 pud_t old_pud = *pudp;
897
898 set_pud_at(mm, address, pudp, pud_wrprotect(old_pud));
899 }
900 #else
901 static inline void pudp_set_wrprotect(struct mm_struct *mm,
902 unsigned long address, pud_t *pudp)
903 {
904 BUILD_BUG();
905 }
906 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
907 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
908 #endif
909
910 #ifndef pmdp_collapse_flush
911 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
912 extern pmd_t pmdp_collapse_flush(struct vm_area_struct *vma,
913 unsigned long address, pmd_t *pmdp);
914 #else
915 static inline pmd_t pmdp_collapse_flush(struct vm_area_struct *vma,
916 unsigned long address,
917 pmd_t *pmdp)
918 {
919 BUILD_BUG();
920 return *pmdp;
921 }
922 #define pmdp_collapse_flush pmdp_collapse_flush
923 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
924 #endif
925
926 #ifndef __HAVE_ARCH_PGTABLE_DEPOSIT
927 extern void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
928 pgtable_t pgtable);
929 #endif
930
931 #ifndef __HAVE_ARCH_PGTABLE_WITHDRAW
932 extern pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp);
933 #endif
934
935 #ifndef arch_needs_pgtable_deposit
936 #define arch_needs_pgtable_deposit() (false)
937 #endif
938
939 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
940 /*
941 * This is an implementation of pmdp_establish() that is only suitable for an
942 * architecture that doesn't have hardware dirty/accessed bits. In this case we
943 * can't race with CPU which sets these bits and non-atomic approach is fine.
944 */
945 static inline pmd_t generic_pmdp_establish(struct vm_area_struct *vma,
946 unsigned long address, pmd_t *pmdp, pmd_t pmd)
947 {
948 pmd_t old_pmd = *pmdp;
949 set_pmd_at(vma->vm_mm, address, pmdp, pmd);
950 return old_pmd;
951 }
952 #endif
953
954 #ifndef __HAVE_ARCH_PMDP_INVALIDATE
955 extern pmd_t pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
956 pmd_t *pmdp);
957 #endif
958
959 #ifndef __HAVE_ARCH_PMDP_INVALIDATE_AD
960
961 /*
962 * pmdp_invalidate_ad() invalidates the PMD while changing a transparent
963 * hugepage mapping in the page tables. This function is similar to
964 * pmdp_invalidate(), but should only be used if the access and dirty bits would
965 * not be cleared by the software in the new PMD value. The function ensures
966 * that hardware changes of the access and dirty bits updates would not be lost.
967 *
968 * Doing so can allow in certain architectures to avoid a TLB flush in most
969 * cases. Yet, another TLB flush might be necessary later if the PMD update
970 * itself requires such flush (e.g., if protection was set to be stricter). Yet,
971 * even when a TLB flush is needed because of the update, the caller may be able
972 * to batch these TLB flushing operations, so fewer TLB flush operations are
973 * needed.
974 */
975 extern pmd_t pmdp_invalidate_ad(struct vm_area_struct *vma,
976 unsigned long address, pmd_t *pmdp);
977 #endif
978
979 #ifndef __HAVE_ARCH_PTE_SAME
980 static inline int pte_same(pte_t pte_a, pte_t pte_b)
981 {
982 return pte_val(pte_a) == pte_val(pte_b);
983 }
984 #endif
985
986 #ifndef __HAVE_ARCH_PTE_UNUSED
987 /*
988 * Some architectures provide facilities to virtualization guests
989 * so that they can flag allocated pages as unused. This allows the
990 * host to transparently reclaim unused pages. This function returns
991 * whether the pte's page is unused.
992 */
993 static inline int pte_unused(pte_t pte)
994 {
995 return 0;
996 }
997 #endif
998
999 #ifndef pte_access_permitted
1000 #define pte_access_permitted(pte, write) \
1001 (pte_present(pte) && (!(write) || pte_write(pte)))
1002 #endif
1003
1004 #ifndef pmd_access_permitted
1005 #define pmd_access_permitted(pmd, write) \
1006 (pmd_present(pmd) && (!(write) || pmd_write(pmd)))
1007 #endif
1008
1009 #ifndef pud_access_permitted
1010 #define pud_access_permitted(pud, write) \
1011 (pud_present(pud) && (!(write) || pud_write(pud)))
1012 #endif
1013
1014 #ifndef p4d_access_permitted
1015 #define p4d_access_permitted(p4d, write) \
1016 (p4d_present(p4d) && (!(write) || p4d_write(p4d)))
1017 #endif
1018
1019 #ifndef pgd_access_permitted
1020 #define pgd_access_permitted(pgd, write) \
1021 (pgd_present(pgd) && (!(write) || pgd_write(pgd)))
1022 #endif
1023
1024 #ifndef __HAVE_ARCH_PMD_SAME
1025 static inline int pmd_same(pmd_t pmd_a, pmd_t pmd_b)
1026 {
1027 return pmd_val(pmd_a) == pmd_val(pmd_b);
1028 }
1029 #endif
1030
1031 #ifndef pud_same
1032 static inline int pud_same(pud_t pud_a, pud_t pud_b)
1033 {
1034 return pud_val(pud_a) == pud_val(pud_b);
1035 }
1036 #define pud_same pud_same
1037 #endif
1038
1039 #ifndef __HAVE_ARCH_P4D_SAME
1040 static inline int p4d_same(p4d_t p4d_a, p4d_t p4d_b)
1041 {
1042 return p4d_val(p4d_a) == p4d_val(p4d_b);
1043 }
1044 #endif
1045
1046 #ifndef __HAVE_ARCH_PGD_SAME
1047 static inline int pgd_same(pgd_t pgd_a, pgd_t pgd_b)
1048 {
1049 return pgd_val(pgd_a) == pgd_val(pgd_b);
1050 }
1051 #endif
1052
1053 /*
1054 * Use set_p*_safe(), and elide TLB flushing, when confident that *no*
1055 * TLB flush will be required as a result of the "set". For example, use
1056 * in scenarios where it is known ahead of time that the routine is
1057 * setting non-present entries, or re-setting an existing entry to the
1058 * same value. Otherwise, use the typical "set" helpers and flush the
1059 * TLB.
1060 */
1061 #define set_pte_safe(ptep, pte) \
1062 ({ \
1063 WARN_ON_ONCE(pte_present(*ptep) && !pte_same(*ptep, pte)); \
1064 set_pte(ptep, pte); \
1065 })
1066
1067 #define set_pmd_safe(pmdp, pmd) \
1068 ({ \
1069 WARN_ON_ONCE(pmd_present(*pmdp) && !pmd_same(*pmdp, pmd)); \
1070 set_pmd(pmdp, pmd); \
1071 })
1072
1073 #define set_pud_safe(pudp, pud) \
1074 ({ \
1075 WARN_ON_ONCE(pud_present(*pudp) && !pud_same(*pudp, pud)); \
1076 set_pud(pudp, pud); \
1077 })
1078
1079 #define set_p4d_safe(p4dp, p4d) \
1080 ({ \
1081 WARN_ON_ONCE(p4d_present(*p4dp) && !p4d_same(*p4dp, p4d)); \
1082 set_p4d(p4dp, p4d); \
1083 })
1084
1085 #define set_pgd_safe(pgdp, pgd) \
1086 ({ \
1087 WARN_ON_ONCE(pgd_present(*pgdp) && !pgd_same(*pgdp, pgd)); \
1088 set_pgd(pgdp, pgd); \
1089 })
1090
1091 #ifndef __HAVE_ARCH_DO_SWAP_PAGE
1092 static inline void arch_do_swap_page_nr(struct mm_struct *mm,
1093 struct vm_area_struct *vma,
1094 unsigned long addr,
1095 pte_t pte, pte_t oldpte,
1096 int nr)
1097 {
1098
1099 }
1100 #else
1101 /*
1102 * Some architectures support metadata associated with a page. When a
1103 * page is being swapped out, this metadata must be saved so it can be
1104 * restored when the page is swapped back in. SPARC M7 and newer
1105 * processors support an ADI (Application Data Integrity) tag for the
1106 * page as metadata for the page. arch_do_swap_page() can restore this
1107 * metadata when a page is swapped back in.
1108 */
1109 static inline void arch_do_swap_page_nr(struct mm_struct *mm,
1110 struct vm_area_struct *vma,
1111 unsigned long addr,
1112 pte_t pte, pte_t oldpte,
1113 int nr)
1114 {
1115 for (int i = 0; i < nr; i++) {
1116 arch_do_swap_page(vma->vm_mm, vma, addr + i * PAGE_SIZE,
1117 pte_advance_pfn(pte, i),
1118 pte_advance_pfn(oldpte, i));
1119 }
1120 }
1121 #endif
1122
1123 #ifndef __HAVE_ARCH_UNMAP_ONE
1124 /*
1125 * Some architectures support metadata associated with a page. When a
1126 * page is being swapped out, this metadata must be saved so it can be
1127 * restored when the page is swapped back in. SPARC M7 and newer
1128 * processors support an ADI (Application Data Integrity) tag for the
1129 * page as metadata for the page. arch_unmap_one() can save this
1130 * metadata on a swap-out of a page.
1131 */
1132 static inline int arch_unmap_one(struct mm_struct *mm,
1133 struct vm_area_struct *vma,
1134 unsigned long addr,
1135 pte_t orig_pte)
1136 {
1137 return 0;
1138 }
1139 #endif
1140
1141 /*
1142 * Allow architectures to preserve additional metadata associated with
1143 * swapped-out pages. The corresponding __HAVE_ARCH_SWAP_* macros and function
1144 * prototypes must be defined in the arch-specific asm/pgtable.h file.
1145 */
1146 #ifndef __HAVE_ARCH_PREPARE_TO_SWAP
1147 static inline int arch_prepare_to_swap(struct folio *folio)
1148 {
1149 return 0;
1150 }
1151 #endif
1152
1153 #ifndef __HAVE_ARCH_SWAP_INVALIDATE
1154 static inline void arch_swap_invalidate_page(int type, pgoff_t offset)
1155 {
1156 }
1157
1158 static inline void arch_swap_invalidate_area(int type)
1159 {
1160 }
1161 #endif
1162
1163 #ifndef __HAVE_ARCH_SWAP_RESTORE
1164 static inline void arch_swap_restore(swp_entry_t entry, struct folio *folio)
1165 {
1166 }
1167 #endif
1168
1169 #ifndef __HAVE_ARCH_PGD_OFFSET_GATE
1170 #define pgd_offset_gate(mm, addr) pgd_offset(mm, addr)
1171 #endif
1172
1173 #ifndef __HAVE_ARCH_MOVE_PTE
1174 #define move_pte(pte, old_addr, new_addr) (pte)
1175 #endif
1176
1177 #ifndef pte_accessible
1178 # define pte_accessible(mm, pte) ((void)(pte), 1)
1179 #endif
1180
1181 #ifndef flush_tlb_fix_spurious_fault
1182 #define flush_tlb_fix_spurious_fault(vma, address, ptep) flush_tlb_page(vma, address)
1183 #endif
1184
1185 /*
1186 * When walking page tables, get the address of the next boundary,
1187 * or the end address of the range if that comes earlier. Although no
1188 * vma end wraps to 0, rounded up __boundary may wrap to 0 throughout.
1189 */
1190
1191 #define pgd_addr_end(addr, end) \
1192 ({ unsigned long __boundary = ((addr) + PGDIR_SIZE) & PGDIR_MASK; \
1193 (__boundary - 1 < (end) - 1)? __boundary: (end); \
1194 })
1195
1196 #ifndef p4d_addr_end
1197 #define p4d_addr_end(addr, end) \
1198 ({ unsigned long __boundary = ((addr) + P4D_SIZE) & P4D_MASK; \
1199 (__boundary - 1 < (end) - 1)? __boundary: (end); \
1200 })
1201 #endif
1202
1203 #ifndef pud_addr_end
1204 #define pud_addr_end(addr, end) \
1205 ({ unsigned long __boundary = ((addr) + PUD_SIZE) & PUD_MASK; \
1206 (__boundary - 1 < (end) - 1)? __boundary: (end); \
1207 })
1208 #endif
1209
1210 #ifndef pmd_addr_end
1211 #define pmd_addr_end(addr, end) \
1212 ({ unsigned long __boundary = ((addr) + PMD_SIZE) & PMD_MASK; \
1213 (__boundary - 1 < (end) - 1)? __boundary: (end); \
1214 })
1215 #endif
1216
1217 /*
1218 * When walking page tables, we usually want to skip any p?d_none entries;
1219 * and any p?d_bad entries - reporting the error before resetting to none.
1220 * Do the tests inline, but report and clear the bad entry in mm/memory.c.
1221 */
1222 void pgd_clear_bad(pgd_t *);
1223
1224 #ifndef __PAGETABLE_P4D_FOLDED
1225 void p4d_clear_bad(p4d_t *);
1226 #else
1227 #define p4d_clear_bad(p4d) do { } while (0)
1228 #endif
1229
1230 #ifndef __PAGETABLE_PUD_FOLDED
1231 void pud_clear_bad(pud_t *);
1232 #else
1233 #define pud_clear_bad(p4d) do { } while (0)
1234 #endif
1235
1236 void pmd_clear_bad(pmd_t *);
1237
1238 static inline int pgd_none_or_clear_bad(pgd_t *pgd)
1239 {
1240 if (pgd_none(*pgd))
1241 return 1;
1242 if (unlikely(pgd_bad(*pgd))) {
1243 pgd_clear_bad(pgd);
1244 return 1;
1245 }
1246 return 0;
1247 }
1248
1249 static inline int p4d_none_or_clear_bad(p4d_t *p4d)
1250 {
1251 if (p4d_none(*p4d))
1252 return 1;
1253 if (unlikely(p4d_bad(*p4d))) {
1254 p4d_clear_bad(p4d);
1255 return 1;
1256 }
1257 return 0;
1258 }
1259
1260 static inline int pud_none_or_clear_bad(pud_t *pud)
1261 {
1262 if (pud_none(*pud))
1263 return 1;
1264 if (unlikely(pud_bad(*pud))) {
1265 pud_clear_bad(pud);
1266 return 1;
1267 }
1268 return 0;
1269 }
1270
1271 static inline int pmd_none_or_clear_bad(pmd_t *pmd)
1272 {
1273 if (pmd_none(*pmd))
1274 return 1;
1275 if (unlikely(pmd_bad(*pmd))) {
1276 pmd_clear_bad(pmd);
1277 return 1;
1278 }
1279 return 0;
1280 }
1281
1282 static inline pte_t __ptep_modify_prot_start(struct vm_area_struct *vma,
1283 unsigned long addr,
1284 pte_t *ptep)
1285 {
1286 /*
1287 * Get the current pte state, but zero it out to make it
1288 * non-present, preventing the hardware from asynchronously
1289 * updating it.
1290 */
1291 return ptep_get_and_clear(vma->vm_mm, addr, ptep);
1292 }
1293
1294 static inline void __ptep_modify_prot_commit(struct vm_area_struct *vma,
1295 unsigned long addr,
1296 pte_t *ptep, pte_t pte)
1297 {
1298 /*
1299 * The pte is non-present, so there's no hardware state to
1300 * preserve.
1301 */
1302 set_pte_at(vma->vm_mm, addr, ptep, pte);
1303 }
1304
1305 #ifndef __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION
1306 /*
1307 * Start a pte protection read-modify-write transaction, which
1308 * protects against asynchronous hardware modifications to the pte.
1309 * The intention is not to prevent the hardware from making pte
1310 * updates, but to prevent any updates it may make from being lost.
1311 *
1312 * This does not protect against other software modifications of the
1313 * pte; the appropriate pte lock must be held over the transaction.
1314 *
1315 * Note that this interface is intended to be batchable, meaning that
1316 * ptep_modify_prot_commit may not actually update the pte, but merely
1317 * queue the update to be done at some later time. The update must be
1318 * actually committed before the pte lock is released, however.
1319 */
1320 static inline pte_t ptep_modify_prot_start(struct vm_area_struct *vma,
1321 unsigned long addr,
1322 pte_t *ptep)
1323 {
1324 return __ptep_modify_prot_start(vma, addr, ptep);
1325 }
1326
1327 /*
1328 * Commit an update to a pte, leaving any hardware-controlled bits in
1329 * the PTE unmodified.
1330 */
1331 static inline void ptep_modify_prot_commit(struct vm_area_struct *vma,
1332 unsigned long addr,
1333 pte_t *ptep, pte_t old_pte, pte_t pte)
1334 {
1335 __ptep_modify_prot_commit(vma, addr, ptep, pte);
1336 }
1337 #endif /* __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION */
1338 #endif /* CONFIG_MMU */
1339
1340 /*
1341 * No-op macros that just return the current protection value. Defined here
1342 * because these macros can be used even if CONFIG_MMU is not defined.
1343 */
1344
1345 #ifndef pgprot_nx
1346 #define pgprot_nx(prot) (prot)
1347 #endif
1348
1349 #ifndef pgprot_noncached
1350 #define pgprot_noncached(prot) (prot)
1351 #endif
1352
1353 #ifndef pgprot_writecombine
1354 #define pgprot_writecombine pgprot_noncached
1355 #endif
1356
1357 #ifndef pgprot_writethrough
1358 #define pgprot_writethrough pgprot_noncached
1359 #endif
1360
1361 #ifndef pgprot_device
1362 #define pgprot_device pgprot_noncached
1363 #endif
1364
1365 #ifndef pgprot_mhp
1366 #define pgprot_mhp(prot) (prot)
1367 #endif
1368
1369 #ifdef CONFIG_MMU
1370 #ifndef pgprot_modify
1371 #define pgprot_modify pgprot_modify
1372 static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot)
1373 {
1374 if (pgprot_val(oldprot) == pgprot_val(pgprot_noncached(oldprot)))
1375 newprot = pgprot_noncached(newprot);
1376 if (pgprot_val(oldprot) == pgprot_val(pgprot_writecombine(oldprot)))
1377 newprot = pgprot_writecombine(newprot);
1378 if (pgprot_val(oldprot) == pgprot_val(pgprot_device(oldprot)))
1379 newprot = pgprot_device(newprot);
1380 return newprot;
1381 }
1382 #endif
1383 #endif /* CONFIG_MMU */
1384
1385 #ifndef pgprot_encrypted
1386 #define pgprot_encrypted(prot) (prot)
1387 #endif
1388
1389 #ifndef pgprot_decrypted
1390 #define pgprot_decrypted(prot) (prot)
1391 #endif
1392
1393 /*
1394 * A facility to provide batching of the reload of page tables and
1395 * other process state with the actual context switch code for
1396 * paravirtualized guests. By convention, only one of the batched
1397 * update (lazy) modes (CPU, MMU) should be active at any given time,
1398 * entry should never be nested, and entry and exits should always be
1399 * paired. This is for sanity of maintaining and reasoning about the
1400 * kernel code. In this case, the exit (end of the context switch) is
1401 * in architecture-specific code, and so doesn't need a generic
1402 * definition.
1403 */
1404 #ifndef __HAVE_ARCH_START_CONTEXT_SWITCH
1405 #define arch_start_context_switch(prev) do {} while (0)
1406 #endif
1407
1408 #ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY
1409 #ifndef CONFIG_ARCH_ENABLE_THP_MIGRATION
1410 static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd)
1411 {
1412 return pmd;
1413 }
1414
1415 static inline int pmd_swp_soft_dirty(pmd_t pmd)
1416 {
1417 return 0;
1418 }
1419
1420 static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd)
1421 {
1422 return pmd;
1423 }
1424 #endif
1425 #else /* !CONFIG_HAVE_ARCH_SOFT_DIRTY */
1426 static inline int pte_soft_dirty(pte_t pte)
1427 {
1428 return 0;
1429 }
1430
1431 static inline int pmd_soft_dirty(pmd_t pmd)
1432 {
1433 return 0;
1434 }
1435
1436 static inline pte_t pte_mksoft_dirty(pte_t pte)
1437 {
1438 return pte;
1439 }
1440
1441 static inline pmd_t pmd_mksoft_dirty(pmd_t pmd)
1442 {
1443 return pmd;
1444 }
1445
1446 static inline pte_t pte_clear_soft_dirty(pte_t pte)
1447 {
1448 return pte;
1449 }
1450
1451 static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd)
1452 {
1453 return pmd;
1454 }
1455
1456 static inline pte_t pte_swp_mksoft_dirty(pte_t pte)
1457 {
1458 return pte;
1459 }
1460
1461 static inline int pte_swp_soft_dirty(pte_t pte)
1462 {
1463 return 0;
1464 }
1465
1466 static inline pte_t pte_swp_clear_soft_dirty(pte_t pte)
1467 {
1468 return pte;
1469 }
1470
1471 static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd)
1472 {
1473 return pmd;
1474 }
1475
1476 static inline int pmd_swp_soft_dirty(pmd_t pmd)
1477 {
1478 return 0;
1479 }
1480
1481 static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd)
1482 {
1483 return pmd;
1484 }
1485 #endif
1486
1487 #ifndef __HAVE_PFNMAP_TRACKING
1488 /*
1489 * Interfaces that can be used by architecture code to keep track of
1490 * memory type of pfn mappings specified by the remap_pfn_range,
1491 * vmf_insert_pfn.
1492 */
1493
1494 /*
1495 * track_pfn_remap is called when a _new_ pfn mapping is being established
1496 * by remap_pfn_range() for physical range indicated by pfn and size.
1497 */
1498 static inline int track_pfn_remap(struct vm_area_struct *vma, pgprot_t *prot,
1499 unsigned long pfn, unsigned long addr,
1500 unsigned long size)
1501 {
1502 return 0;
1503 }
1504
1505 /*
1506 * track_pfn_insert is called when a _new_ single pfn is established
1507 * by vmf_insert_pfn().
1508 */
1509 static inline void track_pfn_insert(struct vm_area_struct *vma, pgprot_t *prot,
1510 pfn_t pfn)
1511 {
1512 }
1513
1514 /*
1515 * track_pfn_copy is called when vma that is covering the pfnmap gets
1516 * copied through copy_page_range().
1517 */
1518 static inline int track_pfn_copy(struct vm_area_struct *vma)
1519 {
1520 return 0;
1521 }
1522
1523 /*
1524 * untrack_pfn is called while unmapping a pfnmap for a region.
1525 * untrack can be called for a specific region indicated by pfn and size or
1526 * can be for the entire vma (in which case pfn, size are zero).
1527 */
1528 static inline void untrack_pfn(struct vm_area_struct *vma,
1529 unsigned long pfn, unsigned long size,
1530 bool mm_wr_locked)
1531 {
1532 }
1533
1534 /*
1535 * untrack_pfn_clear is called while mremapping a pfnmap for a new region
1536 * or fails to copy pgtable during duplicate vm area.
1537 */
1538 static inline void untrack_pfn_clear(struct vm_area_struct *vma)
1539 {
1540 }
1541 #else
1542 extern int track_pfn_remap(struct vm_area_struct *vma, pgprot_t *prot,
1543 unsigned long pfn, unsigned long addr,
1544 unsigned long size);
1545 extern void track_pfn_insert(struct vm_area_struct *vma, pgprot_t *prot,
1546 pfn_t pfn);
1547 extern int track_pfn_copy(struct vm_area_struct *vma);
1548 extern void untrack_pfn(struct vm_area_struct *vma, unsigned long pfn,
1549 unsigned long size, bool mm_wr_locked);
1550 extern void untrack_pfn_clear(struct vm_area_struct *vma);
1551 #endif
1552
1553 #ifdef CONFIG_MMU
1554 #ifdef __HAVE_COLOR_ZERO_PAGE
1555 static inline int is_zero_pfn(unsigned long pfn)
1556 {
1557 extern unsigned long zero_pfn;
1558 unsigned long offset_from_zero_pfn = pfn - zero_pfn;
1559 return offset_from_zero_pfn <= (zero_page_mask >> PAGE_SHIFT);
1560 }
1561
1562 #define my_zero_pfn(addr) page_to_pfn(ZERO_PAGE(addr))
1563
1564 #else
1565 static inline int is_zero_pfn(unsigned long pfn)
1566 {
1567 extern unsigned long zero_pfn;
1568 return pfn == zero_pfn;
1569 }
1570
1571 static inline unsigned long my_zero_pfn(unsigned long addr)
1572 {
1573 extern unsigned long zero_pfn;
1574 return zero_pfn;
1575 }
1576 #endif
1577 #else
1578 static inline int is_zero_pfn(unsigned long pfn)
1579 {
1580 return 0;
1581 }
1582
1583 static inline unsigned long my_zero_pfn(unsigned long addr)
1584 {
1585 return 0;
1586 }
1587 #endif /* CONFIG_MMU */
1588
1589 #ifdef CONFIG_MMU
1590
1591 #ifndef CONFIG_TRANSPARENT_HUGEPAGE
1592 static inline int pmd_trans_huge(pmd_t pmd)
1593 {
1594 return 0;
1595 }
1596 #ifndef pmd_write
1597 static inline int pmd_write(pmd_t pmd)
1598 {
1599 BUG();
1600 return 0;
1601 }
1602 #endif /* pmd_write */
1603 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
1604
1605 #ifndef pud_write
1606 static inline int pud_write(pud_t pud)
1607 {
1608 BUG();
1609 return 0;
1610 }
1611 #endif /* pud_write */
1612
1613 #if !defined(CONFIG_ARCH_HAS_PTE_DEVMAP) || !defined(CONFIG_TRANSPARENT_HUGEPAGE)
1614 static inline int pmd_devmap(pmd_t pmd)
1615 {
1616 return 0;
1617 }
1618 static inline int pud_devmap(pud_t pud)
1619 {
1620 return 0;
1621 }
1622 static inline int pgd_devmap(pgd_t pgd)
1623 {
1624 return 0;
1625 }
1626 #endif
1627
1628 #if !defined(CONFIG_TRANSPARENT_HUGEPAGE) || \
1629 !defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
1630 static inline int pud_trans_huge(pud_t pud)
1631 {
1632 return 0;
1633 }
1634 #endif
1635
1636 static inline int pud_trans_unstable(pud_t *pud)
1637 {
1638 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && \
1639 defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
1640 pud_t pudval = READ_ONCE(*pud);
1641
1642 if (pud_none(pudval) || pud_trans_huge(pudval) || pud_devmap(pudval))
1643 return 1;
1644 if (unlikely(pud_bad(pudval))) {
1645 pud_clear_bad(pud);
1646 return 1;
1647 }
1648 #endif
1649 return 0;
1650 }
1651
1652 #ifndef CONFIG_NUMA_BALANCING
1653 /*
1654 * In an inaccessible (PROT_NONE) VMA, pte_protnone() may indicate "yes". It is
1655 * perfectly valid to indicate "no" in that case, which is why our default
1656 * implementation defaults to "always no".
1657 *
1658 * In an accessible VMA, however, pte_protnone() reliably indicates PROT_NONE
1659 * page protection due to NUMA hinting. NUMA hinting faults only apply in
1660 * accessible VMAs.
1661 *
1662 * So, to reliably identify PROT_NONE PTEs that require a NUMA hinting fault,
1663 * looking at the VMA accessibility is sufficient.
1664 */
1665 static inline int pte_protnone(pte_t pte)
1666 {
1667 return 0;
1668 }
1669
1670 static inline int pmd_protnone(pmd_t pmd)
1671 {
1672 return 0;
1673 }
1674 #endif /* CONFIG_NUMA_BALANCING */
1675
1676 #endif /* CONFIG_MMU */
1677
1678 #ifdef CONFIG_HAVE_ARCH_HUGE_VMAP
1679
1680 #ifndef __PAGETABLE_P4D_FOLDED
1681 int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot);
1682 void p4d_clear_huge(p4d_t *p4d);
1683 #else
1684 static inline int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot)
1685 {
1686 return 0;
1687 }
1688 static inline void p4d_clear_huge(p4d_t *p4d) { }
1689 #endif /* !__PAGETABLE_P4D_FOLDED */
1690
1691 int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot);
1692 int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot);
1693 int pud_clear_huge(pud_t *pud);
1694 int pmd_clear_huge(pmd_t *pmd);
1695 int p4d_free_pud_page(p4d_t *p4d, unsigned long addr);
1696 int pud_free_pmd_page(pud_t *pud, unsigned long addr);
1697 int pmd_free_pte_page(pmd_t *pmd, unsigned long addr);
1698 #else /* !CONFIG_HAVE_ARCH_HUGE_VMAP */
1699 static inline int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot)
1700 {
1701 return 0;
1702 }
1703 static inline int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot)
1704 {
1705 return 0;
1706 }
1707 static inline int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot)
1708 {
1709 return 0;
1710 }
1711 static inline void p4d_clear_huge(p4d_t *p4d) { }
1712 static inline int pud_clear_huge(pud_t *pud)
1713 {
1714 return 0;
1715 }
1716 static inline int pmd_clear_huge(pmd_t *pmd)
1717 {
1718 return 0;
1719 }
1720 static inline int p4d_free_pud_page(p4d_t *p4d, unsigned long addr)
1721 {
1722 return 0;
1723 }
1724 static inline int pud_free_pmd_page(pud_t *pud, unsigned long addr)
1725 {
1726 return 0;
1727 }
1728 static inline int pmd_free_pte_page(pmd_t *pmd, unsigned long addr)
1729 {
1730 return 0;
1731 }
1732 #endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */
1733
1734 #ifndef __HAVE_ARCH_FLUSH_PMD_TLB_RANGE
1735 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1736 /*
1737 * ARCHes with special requirements for evicting THP backing TLB entries can
1738 * implement this. Otherwise also, it can help optimize normal TLB flush in
1739 * THP regime. Stock flush_tlb_range() typically has optimization to nuke the
1740 * entire TLB if flush span is greater than a threshold, which will
1741 * likely be true for a single huge page. Thus a single THP flush will
1742 * invalidate the entire TLB which is not desirable.
1743 * e.g. see arch/arc: flush_pmd_tlb_range
1744 */
1745 #define flush_pmd_tlb_range(vma, addr, end) flush_tlb_range(vma, addr, end)
1746 #define flush_pud_tlb_range(vma, addr, end) flush_tlb_range(vma, addr, end)
1747 #else
1748 #define flush_pmd_tlb_range(vma, addr, end) BUILD_BUG()
1749 #define flush_pud_tlb_range(vma, addr, end) BUILD_BUG()
1750 #endif
1751 #endif
1752
1753 struct file;
1754 int phys_mem_access_prot_allowed(struct file *file, unsigned long pfn,
1755 unsigned long size, pgprot_t *vma_prot);
1756
1757 #ifndef CONFIG_X86_ESPFIX64
1758 static inline void init_espfix_bsp(void) { }
1759 #endif
1760
1761 extern void __init pgtable_cache_init(void);
1762
1763 #ifndef __HAVE_ARCH_PFN_MODIFY_ALLOWED
1764 static inline bool pfn_modify_allowed(unsigned long pfn, pgprot_t prot)
1765 {
1766 return true;
1767 }
1768
1769 static inline bool arch_has_pfn_modify_check(void)
1770 {
1771 return false;
1772 }
1773 #endif /* !_HAVE_ARCH_PFN_MODIFY_ALLOWED */
1774
1775 /*
1776 * Architecture PAGE_KERNEL_* fallbacks
1777 *
1778 * Some architectures don't define certain PAGE_KERNEL_* flags. This is either
1779 * because they really don't support them, or the port needs to be updated to
1780 * reflect the required functionality. Below are a set of relatively safe
1781 * fallbacks, as best effort, which we can count on in lieu of the architectures
1782 * not defining them on their own yet.
1783 */
1784
1785 #ifndef PAGE_KERNEL_RO
1786 # define PAGE_KERNEL_RO PAGE_KERNEL
1787 #endif
1788
1789 #ifndef PAGE_KERNEL_EXEC
1790 # define PAGE_KERNEL_EXEC PAGE_KERNEL
1791 #endif
1792
1793 /*
1794 * Page Table Modification bits for pgtbl_mod_mask.
1795 *
1796 * These are used by the p?d_alloc_track*() set of functions an in the generic
1797 * vmalloc/ioremap code to track at which page-table levels entries have been
1798 * modified. Based on that the code can better decide when vmalloc and ioremap
1799 * mapping changes need to be synchronized to other page-tables in the system.
1800 */
1801 #define __PGTBL_PGD_MODIFIED 0
1802 #define __PGTBL_P4D_MODIFIED 1
1803 #define __PGTBL_PUD_MODIFIED 2
1804 #define __PGTBL_PMD_MODIFIED 3
1805 #define __PGTBL_PTE_MODIFIED 4
1806
1807 #define PGTBL_PGD_MODIFIED BIT(__PGTBL_PGD_MODIFIED)
1808 #define PGTBL_P4D_MODIFIED BIT(__PGTBL_P4D_MODIFIED)
1809 #define PGTBL_PUD_MODIFIED BIT(__PGTBL_PUD_MODIFIED)
1810 #define PGTBL_PMD_MODIFIED BIT(__PGTBL_PMD_MODIFIED)
1811 #define PGTBL_PTE_MODIFIED BIT(__PGTBL_PTE_MODIFIED)
1812
1813 /* Page-Table Modification Mask */
1814 typedef unsigned int pgtbl_mod_mask;
1815
1816 #endif /* !__ASSEMBLY__ */
1817
1818 #if !defined(MAX_POSSIBLE_PHYSMEM_BITS) && !defined(CONFIG_64BIT)
1819 #ifdef CONFIG_PHYS_ADDR_T_64BIT
1820 /*
1821 * ZSMALLOC needs to know the highest PFN on 32-bit architectures
1822 * with physical address space extension, but falls back to
1823 * BITS_PER_LONG otherwise.
1824 */
1825 #error Missing MAX_POSSIBLE_PHYSMEM_BITS definition
1826 #else
1827 #define MAX_POSSIBLE_PHYSMEM_BITS 32
1828 #endif
1829 #endif
1830
1831 #ifndef has_transparent_hugepage
1832 #define has_transparent_hugepage() IS_BUILTIN(CONFIG_TRANSPARENT_HUGEPAGE)
1833 #endif
1834
1835 #ifndef has_transparent_pud_hugepage
1836 #define has_transparent_pud_hugepage() IS_BUILTIN(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
1837 #endif
1838 /*
1839 * On some architectures it depends on the mm if the p4d/pud or pmd
1840 * layer of the page table hierarchy is folded or not.
1841 */
1842 #ifndef mm_p4d_folded
1843 #define mm_p4d_folded(mm) __is_defined(__PAGETABLE_P4D_FOLDED)
1844 #endif
1845
1846 #ifndef mm_pud_folded
1847 #define mm_pud_folded(mm) __is_defined(__PAGETABLE_PUD_FOLDED)
1848 #endif
1849
1850 #ifndef mm_pmd_folded
1851 #define mm_pmd_folded(mm) __is_defined(__PAGETABLE_PMD_FOLDED)
1852 #endif
1853
1854 #ifndef p4d_offset_lockless
1855 #define p4d_offset_lockless(pgdp, pgd, address) p4d_offset(&(pgd), address)
1856 #endif
1857 #ifndef pud_offset_lockless
1858 #define pud_offset_lockless(p4dp, p4d, address) pud_offset(&(p4d), address)
1859 #endif
1860 #ifndef pmd_offset_lockless
1861 #define pmd_offset_lockless(pudp, pud, address) pmd_offset(&(pud), address)
1862 #endif
1863
1864 /*
1865 * pXd_leaf() is the API to check whether a pgtable entry is a huge page
1866 * mapping. It should work globally across all archs, without any
1867 * dependency on CONFIG_* options. For architectures that do not support
1868 * huge mappings on specific levels, below fallbacks will be used.
1869 *
1870 * A leaf pgtable entry should always imply the following:
1871 *
1872 * - It is a "present" entry. IOW, before using this API, please check it
1873 * with pXd_present() first. NOTE: it may not always mean the "present
1874 * bit" is set. For example, PROT_NONE entries are always "present".
1875 *
1876 * - It should _never_ be a swap entry of any type. Above "present" check
1877 * should have guarded this, but let's be crystal clear on this.
1878 *
1879 * - It should contain a huge PFN, which points to a huge page larger than
1880 * PAGE_SIZE of the platform. The PFN format isn't important here.
1881 *
1882 * - It should cover all kinds of huge mappings (e.g., pXd_trans_huge(),
1883 * pXd_devmap(), or hugetlb mappings).
1884 */
1885 #ifndef pgd_leaf
1886 #define pgd_leaf(x) false
1887 #endif
1888 #ifndef p4d_leaf
1889 #define p4d_leaf(x) false
1890 #endif
1891 #ifndef pud_leaf
1892 #define pud_leaf(x) false
1893 #endif
1894 #ifndef pmd_leaf
1895 #define pmd_leaf(x) false
1896 #endif
1897
1898 #ifndef pgd_leaf_size
1899 #define pgd_leaf_size(x) (1ULL << PGDIR_SHIFT)
1900 #endif
1901 #ifndef p4d_leaf_size
1902 #define p4d_leaf_size(x) P4D_SIZE
1903 #endif
1904 #ifndef pud_leaf_size
1905 #define pud_leaf_size(x) PUD_SIZE
1906 #endif
1907 #ifndef pmd_leaf_size
1908 #define pmd_leaf_size(x) PMD_SIZE
1909 #endif
1910 #ifndef __pte_leaf_size
1911 #ifndef pte_leaf_size
1912 #define pte_leaf_size(x) PAGE_SIZE
1913 #endif
1914 #define __pte_leaf_size(x,y) pte_leaf_size(y)
1915 #endif
1916
1917 /*
1918 * We always define pmd_pfn for all archs as it's used in lots of generic
1919 * code. Now it happens too for pud_pfn (and can happen for larger
1920 * mappings too in the future; we're not there yet). Instead of defining
1921 * it for all archs (like pmd_pfn), provide a fallback.
1922 *
1923 * Note that returning 0 here means any arch that didn't define this can
1924 * get severely wrong when it hits a real pud leaf. It's arch's
1925 * responsibility to properly define it when a huge pud is possible.
1926 */
1927 #ifndef pud_pfn
1928 #define pud_pfn(x) 0
1929 #endif
1930
1931 /*
1932 * Some architectures have MMUs that are configurable or selectable at boot
1933 * time. These lead to variable PTRS_PER_x. For statically allocated arrays it
1934 * helps to have a static maximum value.
1935 */
1936
1937 #ifndef MAX_PTRS_PER_PTE
1938 #define MAX_PTRS_PER_PTE PTRS_PER_PTE
1939 #endif
1940
1941 #ifndef MAX_PTRS_PER_PMD
1942 #define MAX_PTRS_PER_PMD PTRS_PER_PMD
1943 #endif
1944
1945 #ifndef MAX_PTRS_PER_PUD
1946 #define MAX_PTRS_PER_PUD PTRS_PER_PUD
1947 #endif
1948
1949 #ifndef MAX_PTRS_PER_P4D
1950 #define MAX_PTRS_PER_P4D PTRS_PER_P4D
1951 #endif
1952
1953 /* description of effects of mapping type and prot in current implementation.
1954 * this is due to the limited x86 page protection hardware. The expected
1955 * behavior is in parens:
1956 *
1957 * map_type prot
1958 * PROT_NONE PROT_READ PROT_WRITE PROT_EXEC
1959 * MAP_SHARED r: (no) no r: (yes) yes r: (no) yes r: (no) yes
1960 * w: (no) no w: (no) no w: (yes) yes w: (no) no
1961 * x: (no) no x: (no) yes x: (no) yes x: (yes) yes
1962 *
1963 * MAP_PRIVATE r: (no) no r: (yes) yes r: (no) yes r: (no) yes
1964 * w: (no) no w: (no) no w: (copy) copy w: (no) no
1965 * x: (no) no x: (no) yes x: (no) yes x: (yes) yes
1966 *
1967 * On arm64, PROT_EXEC has the following behaviour for both MAP_SHARED and
1968 * MAP_PRIVATE (with Enhanced PAN supported):
1969 * r: (no) no
1970 * w: (no) no
1971 * x: (yes) yes
1972 */
1973 #define DECLARE_VM_GET_PAGE_PROT \
1974 pgprot_t vm_get_page_prot(unsigned long vm_flags) \
1975 { \
1976 return protection_map[vm_flags & \
1977 (VM_READ | VM_WRITE | VM_EXEC | VM_SHARED)]; \
1978 } \
1979 EXPORT_SYMBOL(vm_get_page_prot);
1980
1981 #endif /* _LINUX_PGTABLE_H */
1982
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.