Linux/mm/pgtable-generic.c

Version: ~ [ linux-6.12-rc7 ] ~ [ linux-6.11.7 ] ~ [ linux-6.10.14 ] ~ [ linux-6.9.12 ] ~ [ linux-6.8.12 ] ~ [ linux-6.7.12 ] ~ [ linux-6.6.60 ] ~ [ linux-6.5.13 ] ~ [ linux-6.4.16 ] ~ [ linux-6.3.13 ] ~ [ linux-6.2.16 ] ~ [ linux-6.1.116 ] ~ [ linux-6.0.19 ] ~ [ linux-5.19.17 ] ~ [ linux-5.18.19 ] ~ [ linux-5.17.15 ] ~ [ linux-5.16.20 ] ~ [ linux-5.15.171 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.229 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.285 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.323 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.336 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.337 ] ~ [ linux-4.4.302 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.12 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

Diff markup

1 // SPDX-License-Identifier: GPL-2.0 << 2 /* 1 /* 3 * mm/pgtable-generic.c 2 * mm/pgtable-generic.c 4 * 3 * 5 * Generic pgtable methods declared in linux/ !! 4 * Generic pgtable methods declared in asm-generic/pgtable.h 6 * 5 * 7 * Copyright (C) 2010 Linus Torvalds 6 * Copyright (C) 2010 Linus Torvalds 8 */ 7 */ 9 8 10 #include <linux/pagemap.h> 9 #include <linux/pagemap.h> 11 #include <linux/hugetlb.h> << 12 #include <linux/pgtable.h> << 13 #include <linux/swap.h> << 14 #include <linux/swapops.h> << 15 #include <linux/mm_inline.h> << 16 #include <asm/pgalloc.h> << 17 #include <asm/tlb.h> 10 #include <asm/tlb.h> >> 11 #include <asm-generic/pgtable.h> 18 12 19 /* 13 /* 20 * If a p?d_bad entry is found while walking p 14 * If a p?d_bad entry is found while walking page tables, report 21 * the error, before resetting entry to p?d_no 15 * the error, before resetting entry to p?d_none. Usually (but 22 * very seldom) called out from the p?d_none_o 16 * very seldom) called out from the p?d_none_or_clear_bad macros. 23 */ 17 */ 24 18 25 void pgd_clear_bad(pgd_t *pgd) 19 void pgd_clear_bad(pgd_t *pgd) 26 { 20 { 27 pgd_ERROR(*pgd); 21 pgd_ERROR(*pgd); 28 pgd_clear(pgd); 22 pgd_clear(pgd); 29 } 23 } 30 24 31 #ifndef __PAGETABLE_P4D_FOLDED << 32 void p4d_clear_bad(p4d_t *p4d) << 33 { << 34 p4d_ERROR(*p4d); << 35 p4d_clear(p4d); << 36 } << 37 #endif << 38 << 39 #ifndef __PAGETABLE_PUD_FOLDED << 40 void pud_clear_bad(pud_t *pud) 25 void pud_clear_bad(pud_t *pud) 41 { 26 { 42 pud_ERROR(*pud); 27 pud_ERROR(*pud); 43 pud_clear(pud); 28 pud_clear(pud); 44 } 29 } 45 #endif << 46 30 47 /* << 48 * Note that the pmd variant below can't be st << 49 * above. pmd folding is special and typically << 50 * level even when folded << 51 */ << 52 void pmd_clear_bad(pmd_t *pmd) 31 void pmd_clear_bad(pmd_t *pmd) 53 { 32 { 54 pmd_ERROR(*pmd); 33 pmd_ERROR(*pmd); 55 pmd_clear(pmd); 34 pmd_clear(pmd); 56 } 35 } 57 36 58 #ifndef __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS 37 #ifndef __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS 59 /* 38 /* 60 * Only sets the access flags (dirty, accessed !! 39 * Only sets the access flags (dirty, accessed), as well as write 61 * permission. Furthermore, we know it always 40 * permission. Furthermore, we know it always gets set to a "more 62 * permissive" setting, which allows most arch 41 * permissive" setting, which allows most architectures to optimize 63 * this. We return whether the PTE actually ch 42 * this. We return whether the PTE actually changed, which in turn 64 * instructs the caller to do things like upda 43 * instructs the caller to do things like update__mmu_cache. This 65 * used to be done in the caller, but sparc ne 44 * used to be done in the caller, but sparc needs minor faults to 66 * force that call on sun4c so we changed this 45 * force that call on sun4c so we changed this macro slightly 67 */ 46 */ 68 int ptep_set_access_flags(struct vm_area_struc 47 int ptep_set_access_flags(struct vm_area_struct *vma, 69 unsigned long addres 48 unsigned long address, pte_t *ptep, 70 pte_t entry, int dir 49 pte_t entry, int dirty) 71 { 50 { 72 int changed = !pte_same(ptep_get(ptep) !! 51 int changed = !pte_same(*ptep, entry); 73 if (changed) { 52 if (changed) { 74 set_pte_at(vma->vm_mm, address 53 set_pte_at(vma->vm_mm, address, ptep, entry); 75 flush_tlb_fix_spurious_fault(v !! 54 flush_tlb_fix_spurious_fault(vma, address); 76 } 55 } 77 return changed; 56 return changed; 78 } 57 } 79 #endif 58 #endif 80 59 81 #ifndef __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH 60 #ifndef __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH 82 int ptep_clear_flush_young(struct vm_area_stru 61 int ptep_clear_flush_young(struct vm_area_struct *vma, 83 unsigned long addre 62 unsigned long address, pte_t *ptep) 84 { 63 { 85 int young; 64 int young; 86 young = ptep_test_and_clear_young(vma, 65 young = ptep_test_and_clear_young(vma, address, ptep); 87 if (young) 66 if (young) 88 flush_tlb_page(vma, address); 67 flush_tlb_page(vma, address); 89 return young; 68 return young; 90 } 69 } 91 #endif 70 #endif 92 71 93 #ifndef __HAVE_ARCH_PTEP_CLEAR_FLUSH 72 #ifndef __HAVE_ARCH_PTEP_CLEAR_FLUSH 94 pte_t ptep_clear_flush(struct vm_area_struct * 73 pte_t ptep_clear_flush(struct vm_area_struct *vma, unsigned long address, 95 pte_t *ptep) 74 pte_t *ptep) 96 { 75 { 97 struct mm_struct *mm = (vma)->vm_mm; 76 struct mm_struct *mm = (vma)->vm_mm; 98 pte_t pte; 77 pte_t pte; 99 pte = ptep_get_and_clear(mm, address, 78 pte = ptep_get_and_clear(mm, address, ptep); 100 if (pte_accessible(mm, pte)) 79 if (pte_accessible(mm, pte)) 101 flush_tlb_page(vma, address); 80 flush_tlb_page(vma, address); 102 return pte; 81 return pte; 103 } 82 } 104 #endif 83 #endif 105 84 106 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 85 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 107 86 108 #ifndef __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS 87 #ifndef __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS 109 int pmdp_set_access_flags(struct vm_area_struc 88 int pmdp_set_access_flags(struct vm_area_struct *vma, 110 unsigned long addres 89 unsigned long address, pmd_t *pmdp, 111 pmd_t entry, int dir 90 pmd_t entry, int dirty) 112 { 91 { 113 int changed = !pmd_same(*pmdp, entry); 92 int changed = !pmd_same(*pmdp, entry); 114 VM_BUG_ON(address & ~HPAGE_PMD_MASK); 93 VM_BUG_ON(address & ~HPAGE_PMD_MASK); 115 if (changed) { 94 if (changed) { 116 set_pmd_at(vma->vm_mm, address 95 set_pmd_at(vma->vm_mm, address, pmdp, entry); 117 flush_pmd_tlb_range(vma, addre 96 flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE); 118 } 97 } 119 return changed; 98 return changed; 120 } 99 } 121 #endif 100 #endif 122 101 123 #ifndef __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH 102 #ifndef __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH 124 int pmdp_clear_flush_young(struct vm_area_stru 103 int pmdp_clear_flush_young(struct vm_area_struct *vma, 125 unsigned long addre 104 unsigned long address, pmd_t *pmdp) 126 { 105 { 127 int young; 106 int young; 128 VM_BUG_ON(address & ~HPAGE_PMD_MASK); 107 VM_BUG_ON(address & ~HPAGE_PMD_MASK); 129 young = pmdp_test_and_clear_young(vma, 108 young = pmdp_test_and_clear_young(vma, address, pmdp); 130 if (young) 109 if (young) 131 flush_pmd_tlb_range(vma, addre 110 flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE); 132 return young; 111 return young; 133 } 112 } 134 #endif 113 #endif 135 114 136 #ifndef __HAVE_ARCH_PMDP_HUGE_CLEAR_FLUSH 115 #ifndef __HAVE_ARCH_PMDP_HUGE_CLEAR_FLUSH 137 pmd_t pmdp_huge_clear_flush(struct vm_area_str 116 pmd_t pmdp_huge_clear_flush(struct vm_area_struct *vma, unsigned long address, 138 pmd_t *pmdp) 117 pmd_t *pmdp) 139 { 118 { 140 pmd_t pmd; 119 pmd_t pmd; 141 VM_BUG_ON(address & ~HPAGE_PMD_MASK); 120 VM_BUG_ON(address & ~HPAGE_PMD_MASK); 142 VM_BUG_ON(pmd_present(*pmdp) && !pmd_t !! 121 VM_BUG_ON(!pmd_trans_huge(*pmdp) && !pmd_devmap(*pmdp)); 143 !pmd_devmap(*pmdp)) << 144 pmd = pmdp_huge_get_and_clear(vma->vm_ 122 pmd = pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp); 145 flush_pmd_tlb_range(vma, address, addr 123 flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE); 146 return pmd; 124 return pmd; 147 } 125 } 148 << 149 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_P << 150 pud_t pudp_huge_clear_flush(struct vm_area_str << 151 pud_t *pudp) << 152 { << 153 pud_t pud; << 154 << 155 VM_BUG_ON(address & ~HPAGE_PUD_MASK); << 156 VM_BUG_ON(!pud_trans_huge(*pudp) && !p << 157 pud = pudp_huge_get_and_clear(vma->vm_ << 158 flush_pud_tlb_range(vma, address, addr << 159 return pud; << 160 } << 161 #endif << 162 #endif 126 #endif 163 127 164 #ifndef __HAVE_ARCH_PGTABLE_DEPOSIT 128 #ifndef __HAVE_ARCH_PGTABLE_DEPOSIT 165 void pgtable_trans_huge_deposit(struct mm_stru 129 void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, 166 pgtable_t pgta 130 pgtable_t pgtable) 167 { 131 { 168 assert_spin_locked(pmd_lockptr(mm, pmd 132 assert_spin_locked(pmd_lockptr(mm, pmdp)); 169 133 170 /* FIFO */ 134 /* FIFO */ 171 if (!pmd_huge_pte(mm, pmdp)) 135 if (!pmd_huge_pte(mm, pmdp)) 172 INIT_LIST_HEAD(&pgtable->lru); 136 INIT_LIST_HEAD(&pgtable->lru); 173 else 137 else 174 list_add(&pgtable->lru, &pmd_h 138 list_add(&pgtable->lru, &pmd_huge_pte(mm, pmdp)->lru); 175 pmd_huge_pte(mm, pmdp) = pgtable; 139 pmd_huge_pte(mm, pmdp) = pgtable; 176 } 140 } 177 #endif 141 #endif 178 142 179 #ifndef __HAVE_ARCH_PGTABLE_WITHDRAW 143 #ifndef __HAVE_ARCH_PGTABLE_WITHDRAW 180 /* no "address" argument so destroys page colo 144 /* no "address" argument so destroys page coloring of some arch */ 181 pgtable_t pgtable_trans_huge_withdraw(struct m 145 pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp) 182 { 146 { 183 pgtable_t pgtable; 147 pgtable_t pgtable; 184 148 185 assert_spin_locked(pmd_lockptr(mm, pmd 149 assert_spin_locked(pmd_lockptr(mm, pmdp)); 186 150 187 /* FIFO */ 151 /* FIFO */ 188 pgtable = pmd_huge_pte(mm, pmdp); 152 pgtable = pmd_huge_pte(mm, pmdp); 189 pmd_huge_pte(mm, pmdp) = list_first_en 153 pmd_huge_pte(mm, pmdp) = list_first_entry_or_null(&pgtable->lru, 190 154 struct page, lru); 191 if (pmd_huge_pte(mm, pmdp)) 155 if (pmd_huge_pte(mm, pmdp)) 192 list_del(&pgtable->lru); 156 list_del(&pgtable->lru); 193 return pgtable; 157 return pgtable; 194 } 158 } 195 #endif 159 #endif 196 160 197 #ifndef __HAVE_ARCH_PMDP_INVALIDATE 161 #ifndef __HAVE_ARCH_PMDP_INVALIDATE 198 pmd_t pmdp_invalidate(struct vm_area_struct *v !! 162 void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address, 199 pmd_t *pmdp) 163 pmd_t *pmdp) 200 { 164 { 201 VM_WARN_ON_ONCE(!pmd_present(*pmdp)); !! 165 pmd_t entry = *pmdp; 202 pmd_t old = pmdp_establish(vma, addres !! 166 set_pmd_at(vma->vm_mm, address, pmdp, pmd_mknotpresent(entry)); 203 flush_pmd_tlb_range(vma, address, addr 167 flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE); 204 return old; << 205 } << 206 #endif << 207 << 208 #ifndef __HAVE_ARCH_PMDP_INVALIDATE_AD << 209 pmd_t pmdp_invalidate_ad(struct vm_area_struct << 210 pmd_t *pmdp) << 211 { << 212 VM_WARN_ON_ONCE(!pmd_present(*pmdp)); << 213 return pmdp_invalidate(vma, address, p << 214 } 168 } 215 #endif 169 #endif 216 170 217 #ifndef pmdp_collapse_flush 171 #ifndef pmdp_collapse_flush 218 pmd_t pmdp_collapse_flush(struct vm_area_struc 172 pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long address, 219 pmd_t *pmdp) 173 pmd_t *pmdp) 220 { 174 { 221 /* 175 /* 222 * pmd and hugepage pte format are sam 176 * pmd and hugepage pte format are same. So we could 223 * use the same function. 177 * use the same function. 224 */ 178 */ 225 pmd_t pmd; 179 pmd_t pmd; 226 180 227 VM_BUG_ON(address & ~HPAGE_PMD_MASK); 181 VM_BUG_ON(address & ~HPAGE_PMD_MASK); 228 VM_BUG_ON(pmd_trans_huge(*pmdp)); 182 VM_BUG_ON(pmd_trans_huge(*pmdp)); 229 pmd = pmdp_huge_get_and_clear(vma->vm_ 183 pmd = pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp); 230 184 231 /* collapse entails shooting down ptes 185 /* collapse entails shooting down ptes not pmd */ 232 flush_tlb_range(vma, address, address 186 flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE); 233 return pmd; 187 return pmd; 234 } 188 } 235 #endif 189 #endif 236 << 237 /* arch define pte_free_defer in asm/pgalloc.h << 238 #ifndef pte_free_defer << 239 static void pte_free_now(struct rcu_head *head << 240 { << 241 struct page *page; << 242 << 243 page = container_of(head, struct page, << 244 pte_free(NULL /* mm not passed and not << 245 } << 246 << 247 void pte_free_defer(struct mm_struct *mm, pgta << 248 { << 249 struct page *page; << 250 << 251 page = pgtable; << 252 call_rcu(&page->rcu_head, pte_free_now << 253 } << 254 #endif /* pte_free_defer */ << 255 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 190 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 256 << 257 #if defined(CONFIG_GUP_GET_PXX_LOW_HIGH) && \ << 258 (defined(CONFIG_SMP) || defined(CONFIG << 259 /* << 260 * See the comment above ptep_get_lockless() i << 261 * the barriers in pmdp_get_lockless() cannot << 262 * pmd_high actually belongs with the value in << 263 * off blocks the TLB flush between present up << 264 * successful __pte_offset_map() points to a p << 265 */ << 266 static unsigned long pmdp_get_lockless_start(v << 267 { << 268 unsigned long irqflags; << 269 << 270 local_irq_save(irqflags); << 271 return irqflags; << 272 } << 273 static void pmdp_get_lockless_end(unsigned lon << 274 { << 275 local_irq_restore(irqflags); << 276 } << 277 #else << 278 static unsigned long pmdp_get_lockless_start(v << 279 static void pmdp_get_lockless_end(unsigned lon << 280 #endif << 281 << 282 pte_t *__pte_offset_map(pmd_t *pmd, unsigned l << 283 { << 284 unsigned long irqflags; << 285 pmd_t pmdval; << 286 << 287 rcu_read_lock(); << 288 irqflags = pmdp_get_lockless_start(); << 289 pmdval = pmdp_get_lockless(pmd); << 290 pmdp_get_lockless_end(irqflags); << 291 << 292 if (pmdvalp) << 293 *pmdvalp = pmdval; << 294 if (unlikely(pmd_none(pmdval) || is_pm << 295 goto nomap; << 296 if (unlikely(pmd_trans_huge(pmdval) || << 297 goto nomap; << 298 if (unlikely(pmd_bad(pmdval))) { << 299 pmd_clear_bad(pmd); << 300 goto nomap; << 301 } << 302 return __pte_map(&pmdval, addr); << 303 nomap: << 304 rcu_read_unlock(); << 305 return NULL; << 306 } << 307 << 308 pte_t *pte_offset_map_nolock(struct mm_struct << 309 unsigned long add << 310 { << 311 pmd_t pmdval; << 312 pte_t *pte; << 313 << 314 pte = __pte_offset_map(pmd, addr, &pmd << 315 if (likely(pte)) << 316 *ptlp = pte_lockptr(mm, &pmdva << 317 return pte; << 318 } << 319 << 320 /* << 321 * pte_offset_map_lock(mm, pmd, addr, ptlp), a << 322 * __pte_offset_map_lock() below, is usually c << 323 * addr, reached by walking down the mm's pgd, << 324 * holding mmap_lock or vma lock for read or f << 325 * context, while holding file's i_mmap_lock o << 326 * write). In a few cases, it may be used with << 327 * copied to or constructed on the stack. << 328 * << 329 * When successful, it returns the pte pointer << 330 * kmapped if necessary (when CONFIG_HIGHPTE), << 331 * modification by software, with a pointer to << 332 * configs mm->page_table_lock, in SPLIT_PTLOC << 333 * struct page). pte_unmap_unlock(pte, ptl) t << 334 * << 335 * But it is unsuccessful, returning NULL with << 336 * page table at *pmd: if, for example, the pa << 337 * or replaced by the huge pmd of a THP. (Whe << 338 * after acquiring the ptlock, and retried int << 339 * page table can be safely removed or replace << 340 * << 341 * pte_offset_map(pmd, addr), and its internal << 342 * just returns the pte pointer for addr, its << 343 * or NULL if there is no page table at *pmd. << 344 * page table, so cannot normally be used when << 345 * or when entries read must be stable. But i << 346 * that even when page table is racily removed << 347 * and disconnected table. Until pte_unmap(pt << 348 * afterwards. << 349 * << 350 * pte_offset_map_nolock(mm, pmd, addr, ptlp), << 351 * but when successful, it also outputs a poin << 352 * pte_offset_map_lock() does, but in this cas << 353 * the caller to avoid a later pte_lockptr(mm, << 354 * act on a changed *pmd: pte_offset_map_noloc << 355 * pointer for the page table that it returns. << 356 * recheck *pmd once the lock is taken; in pra << 357 * either the mmap_lock for write, or pte_same << 358 * << 359 * Note that free_pgtables(), used after unmap << 360 * exiting the whole mm, does not take page ta << 361 * table, and may not use RCU at all: "outside << 362 * pte_offset_map() and co once the vma is det << 363 */ << 364 pte_t *__pte_offset_map_lock(struct mm_struct << 365 unsigned long add << 366 { << 367 spinlock_t *ptl; << 368 pmd_t pmdval; << 369 pte_t *pte; << 370 again: << 371 pte = __pte_offset_map(pmd, addr, &pmd << 372 if (unlikely(!pte)) << 373 return pte; << 374 ptl = pte_lockptr(mm, &pmdval); << 375 spin_lock(ptl); << 376 if (likely(pmd_same(pmdval, pmdp_get_l << 377 *ptlp = ptl; << 378 return pte; << 379 } << 380 pte_unmap_unlock(pte, ptl); << 381 goto again; << 382 } << 383 191

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

TOMOYO Linux Cross Reference
Linux/mm/pgtable-generic.c

Diff markup

Differences between /mm/pgtable-generic.c (Version linux-6.12-rc7) and /mm/pgtable-generic.c (Version linux-4.10.17)

TOMOYO Linux Cross Reference Linux/mm/pgtable-generic.c

Diff markup

Differences between /mm/pgtable-generic.c (Version linux-6.12-rc7) and /mm/pgtable-generic.c (Version linux-4.10.17)

TOMOYO Linux Cross Reference
Linux/mm/pgtable-generic.c