~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/mm/pgtable-generic.c

Version: ~ [ linux-6.11-rc3 ] ~ [ linux-6.10.4 ] ~ [ linux-6.9.12 ] ~ [ linux-6.8.12 ] ~ [ linux-6.7.12 ] ~ [ linux-6.6.45 ] ~ [ linux-6.5.13 ] ~ [ linux-6.4.16 ] ~ [ linux-6.3.13 ] ~ [ linux-6.2.16 ] ~ [ linux-6.1.104 ] ~ [ linux-6.0.19 ] ~ [ linux-5.19.17 ] ~ [ linux-5.18.19 ] ~ [ linux-5.17.15 ] ~ [ linux-5.16.20 ] ~ [ linux-5.15.164 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.223 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.281 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.319 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.336 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.337 ] ~ [ linux-4.4.302 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.9 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

Diff markup

Differences between /mm/pgtable-generic.c (Version linux-6.11-rc3) and /mm/pgtable-generic.c (Version linux-6.5.13)


  1 // SPDX-License-Identifier: GPL-2.0                 1 // SPDX-License-Identifier: GPL-2.0
  2 /*                                                  2 /*
  3  *  mm/pgtable-generic.c                            3  *  mm/pgtable-generic.c
  4  *                                                  4  *
  5  *  Generic pgtable methods declared in linux/      5  *  Generic pgtable methods declared in linux/pgtable.h
  6  *                                                  6  *
  7  *  Copyright (C) 2010  Linus Torvalds              7  *  Copyright (C) 2010  Linus Torvalds
  8  */                                                 8  */
  9                                                     9 
 10 #include <linux/pagemap.h>                         10 #include <linux/pagemap.h>
 11 #include <linux/hugetlb.h>                         11 #include <linux/hugetlb.h>
 12 #include <linux/pgtable.h>                         12 #include <linux/pgtable.h>
 13 #include <linux/swap.h>                            13 #include <linux/swap.h>
 14 #include <linux/swapops.h>                         14 #include <linux/swapops.h>
 15 #include <linux/mm_inline.h>                       15 #include <linux/mm_inline.h>
 16 #include <asm/pgalloc.h>                       << 
 17 #include <asm/tlb.h>                               16 #include <asm/tlb.h>
 18                                                    17 
 19 /*                                                 18 /*
 20  * If a p?d_bad entry is found while walking p     19  * If a p?d_bad entry is found while walking page tables, report
 21  * the error, before resetting entry to p?d_no     20  * the error, before resetting entry to p?d_none.  Usually (but
 22  * very seldom) called out from the p?d_none_o     21  * very seldom) called out from the p?d_none_or_clear_bad macros.
 23  */                                                22  */
 24                                                    23 
 25 void pgd_clear_bad(pgd_t *pgd)                     24 void pgd_clear_bad(pgd_t *pgd)
 26 {                                                  25 {
 27         pgd_ERROR(*pgd);                           26         pgd_ERROR(*pgd);
 28         pgd_clear(pgd);                            27         pgd_clear(pgd);
 29 }                                                  28 }
 30                                                    29 
 31 #ifndef __PAGETABLE_P4D_FOLDED                     30 #ifndef __PAGETABLE_P4D_FOLDED
 32 void p4d_clear_bad(p4d_t *p4d)                     31 void p4d_clear_bad(p4d_t *p4d)
 33 {                                                  32 {
 34         p4d_ERROR(*p4d);                           33         p4d_ERROR(*p4d);
 35         p4d_clear(p4d);                            34         p4d_clear(p4d);
 36 }                                                  35 }
 37 #endif                                             36 #endif
 38                                                    37 
 39 #ifndef __PAGETABLE_PUD_FOLDED                     38 #ifndef __PAGETABLE_PUD_FOLDED
 40 void pud_clear_bad(pud_t *pud)                     39 void pud_clear_bad(pud_t *pud)
 41 {                                                  40 {
 42         pud_ERROR(*pud);                           41         pud_ERROR(*pud);
 43         pud_clear(pud);                            42         pud_clear(pud);
 44 }                                                  43 }
 45 #endif                                             44 #endif
 46                                                    45 
 47 /*                                                 46 /*
 48  * Note that the pmd variant below can't be st     47  * Note that the pmd variant below can't be stub'ed out just as for p4d/pud
 49  * above. pmd folding is special and typically     48  * above. pmd folding is special and typically pmd_* macros refer to upper
 50  * level even when folded                          49  * level even when folded
 51  */                                                50  */
 52 void pmd_clear_bad(pmd_t *pmd)                     51 void pmd_clear_bad(pmd_t *pmd)
 53 {                                                  52 {
 54         pmd_ERROR(*pmd);                           53         pmd_ERROR(*pmd);
 55         pmd_clear(pmd);                            54         pmd_clear(pmd);
 56 }                                                  55 }
 57                                                    56 
 58 #ifndef __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS          57 #ifndef __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
 59 /*                                                 58 /*
 60  * Only sets the access flags (dirty, accessed     59  * Only sets the access flags (dirty, accessed), as well as write
 61  * permission. Furthermore, we know it always      60  * permission. Furthermore, we know it always gets set to a "more
 62  * permissive" setting, which allows most arch     61  * permissive" setting, which allows most architectures to optimize
 63  * this. We return whether the PTE actually ch     62  * this. We return whether the PTE actually changed, which in turn
 64  * instructs the caller to do things like upda     63  * instructs the caller to do things like update__mmu_cache.  This
 65  * used to be done in the caller, but sparc ne     64  * used to be done in the caller, but sparc needs minor faults to
 66  * force that call on sun4c so we changed this     65  * force that call on sun4c so we changed this macro slightly
 67  */                                                66  */
 68 int ptep_set_access_flags(struct vm_area_struc     67 int ptep_set_access_flags(struct vm_area_struct *vma,
 69                           unsigned long addres     68                           unsigned long address, pte_t *ptep,
 70                           pte_t entry, int dir     69                           pte_t entry, int dirty)
 71 {                                                  70 {
 72         int changed = !pte_same(ptep_get(ptep)     71         int changed = !pte_same(ptep_get(ptep), entry);
 73         if (changed) {                             72         if (changed) {
 74                 set_pte_at(vma->vm_mm, address     73                 set_pte_at(vma->vm_mm, address, ptep, entry);
 75                 flush_tlb_fix_spurious_fault(v     74                 flush_tlb_fix_spurious_fault(vma, address, ptep);
 76         }                                          75         }
 77         return changed;                            76         return changed;
 78 }                                                  77 }
 79 #endif                                             78 #endif
 80                                                    79 
 81 #ifndef __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH         80 #ifndef __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
 82 int ptep_clear_flush_young(struct vm_area_stru     81 int ptep_clear_flush_young(struct vm_area_struct *vma,
 83                            unsigned long addre     82                            unsigned long address, pte_t *ptep)
 84 {                                                  83 {
 85         int young;                                 84         int young;
 86         young = ptep_test_and_clear_young(vma,     85         young = ptep_test_and_clear_young(vma, address, ptep);
 87         if (young)                                 86         if (young)
 88                 flush_tlb_page(vma, address);      87                 flush_tlb_page(vma, address);
 89         return young;                              88         return young;
 90 }                                                  89 }
 91 #endif                                             90 #endif
 92                                                    91 
 93 #ifndef __HAVE_ARCH_PTEP_CLEAR_FLUSH               92 #ifndef __HAVE_ARCH_PTEP_CLEAR_FLUSH
 94 pte_t ptep_clear_flush(struct vm_area_struct *     93 pte_t ptep_clear_flush(struct vm_area_struct *vma, unsigned long address,
 95                        pte_t *ptep)                94                        pte_t *ptep)
 96 {                                                  95 {
 97         struct mm_struct *mm = (vma)->vm_mm;       96         struct mm_struct *mm = (vma)->vm_mm;
 98         pte_t pte;                                 97         pte_t pte;
 99         pte = ptep_get_and_clear(mm, address,      98         pte = ptep_get_and_clear(mm, address, ptep);
100         if (pte_accessible(mm, pte))               99         if (pte_accessible(mm, pte))
101                 flush_tlb_page(vma, address);     100                 flush_tlb_page(vma, address);
102         return pte;                               101         return pte;
103 }                                                 102 }
104 #endif                                            103 #endif
105                                                   104 
106 #ifdef CONFIG_TRANSPARENT_HUGEPAGE                105 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
107                                                   106 
108 #ifndef __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS         107 #ifndef __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
109 int pmdp_set_access_flags(struct vm_area_struc    108 int pmdp_set_access_flags(struct vm_area_struct *vma,
110                           unsigned long addres    109                           unsigned long address, pmd_t *pmdp,
111                           pmd_t entry, int dir    110                           pmd_t entry, int dirty)
112 {                                                 111 {
113         int changed = !pmd_same(*pmdp, entry);    112         int changed = !pmd_same(*pmdp, entry);
114         VM_BUG_ON(address & ~HPAGE_PMD_MASK);     113         VM_BUG_ON(address & ~HPAGE_PMD_MASK);
115         if (changed) {                            114         if (changed) {
116                 set_pmd_at(vma->vm_mm, address    115                 set_pmd_at(vma->vm_mm, address, pmdp, entry);
117                 flush_pmd_tlb_range(vma, addre    116                 flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
118         }                                         117         }
119         return changed;                           118         return changed;
120 }                                                 119 }
121 #endif                                            120 #endif
122                                                   121 
123 #ifndef __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH        122 #ifndef __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH
124 int pmdp_clear_flush_young(struct vm_area_stru    123 int pmdp_clear_flush_young(struct vm_area_struct *vma,
125                            unsigned long addre    124                            unsigned long address, pmd_t *pmdp)
126 {                                                 125 {
127         int young;                                126         int young;
128         VM_BUG_ON(address & ~HPAGE_PMD_MASK);     127         VM_BUG_ON(address & ~HPAGE_PMD_MASK);
129         young = pmdp_test_and_clear_young(vma,    128         young = pmdp_test_and_clear_young(vma, address, pmdp);
130         if (young)                                129         if (young)
131                 flush_pmd_tlb_range(vma, addre    130                 flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
132         return young;                             131         return young;
133 }                                                 132 }
134 #endif                                            133 #endif
135                                                   134 
136 #ifndef __HAVE_ARCH_PMDP_HUGE_CLEAR_FLUSH         135 #ifndef __HAVE_ARCH_PMDP_HUGE_CLEAR_FLUSH
137 pmd_t pmdp_huge_clear_flush(struct vm_area_str    136 pmd_t pmdp_huge_clear_flush(struct vm_area_struct *vma, unsigned long address,
138                             pmd_t *pmdp)          137                             pmd_t *pmdp)
139 {                                                 138 {
140         pmd_t pmd;                                139         pmd_t pmd;
141         VM_BUG_ON(address & ~HPAGE_PMD_MASK);     140         VM_BUG_ON(address & ~HPAGE_PMD_MASK);
142         VM_BUG_ON(pmd_present(*pmdp) && !pmd_t    141         VM_BUG_ON(pmd_present(*pmdp) && !pmd_trans_huge(*pmdp) &&
143                            !pmd_devmap(*pmdp))    142                            !pmd_devmap(*pmdp));
144         pmd = pmdp_huge_get_and_clear(vma->vm_    143         pmd = pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp);
145         flush_pmd_tlb_range(vma, address, addr    144         flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
146         return pmd;                               145         return pmd;
147 }                                                 146 }
148                                                   147 
149 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_P    148 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
150 pud_t pudp_huge_clear_flush(struct vm_area_str    149 pud_t pudp_huge_clear_flush(struct vm_area_struct *vma, unsigned long address,
151                             pud_t *pudp)          150                             pud_t *pudp)
152 {                                                 151 {
153         pud_t pud;                                152         pud_t pud;
154                                                   153 
155         VM_BUG_ON(address & ~HPAGE_PUD_MASK);     154         VM_BUG_ON(address & ~HPAGE_PUD_MASK);
156         VM_BUG_ON(!pud_trans_huge(*pudp) && !p    155         VM_BUG_ON(!pud_trans_huge(*pudp) && !pud_devmap(*pudp));
157         pud = pudp_huge_get_and_clear(vma->vm_    156         pud = pudp_huge_get_and_clear(vma->vm_mm, address, pudp);
158         flush_pud_tlb_range(vma, address, addr    157         flush_pud_tlb_range(vma, address, address + HPAGE_PUD_SIZE);
159         return pud;                               158         return pud;
160 }                                                 159 }
161 #endif                                            160 #endif
162 #endif                                            161 #endif
163                                                   162 
164 #ifndef __HAVE_ARCH_PGTABLE_DEPOSIT               163 #ifndef __HAVE_ARCH_PGTABLE_DEPOSIT
165 void pgtable_trans_huge_deposit(struct mm_stru    164 void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
166                                 pgtable_t pgta    165                                 pgtable_t pgtable)
167 {                                                 166 {
168         assert_spin_locked(pmd_lockptr(mm, pmd    167         assert_spin_locked(pmd_lockptr(mm, pmdp));
169                                                   168 
170         /* FIFO */                                169         /* FIFO */
171         if (!pmd_huge_pte(mm, pmdp))              170         if (!pmd_huge_pte(mm, pmdp))
172                 INIT_LIST_HEAD(&pgtable->lru);    171                 INIT_LIST_HEAD(&pgtable->lru);
173         else                                      172         else
174                 list_add(&pgtable->lru, &pmd_h    173                 list_add(&pgtable->lru, &pmd_huge_pte(mm, pmdp)->lru);
175         pmd_huge_pte(mm, pmdp) = pgtable;         174         pmd_huge_pte(mm, pmdp) = pgtable;
176 }                                                 175 }
177 #endif                                            176 #endif
178                                                   177 
179 #ifndef __HAVE_ARCH_PGTABLE_WITHDRAW              178 #ifndef __HAVE_ARCH_PGTABLE_WITHDRAW
180 /* no "address" argument so destroys page colo    179 /* no "address" argument so destroys page coloring of some arch */
181 pgtable_t pgtable_trans_huge_withdraw(struct m    180 pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
182 {                                                 181 {
183         pgtable_t pgtable;                        182         pgtable_t pgtable;
184                                                   183 
185         assert_spin_locked(pmd_lockptr(mm, pmd    184         assert_spin_locked(pmd_lockptr(mm, pmdp));
186                                                   185 
187         /* FIFO */                                186         /* FIFO */
188         pgtable = pmd_huge_pte(mm, pmdp);         187         pgtable = pmd_huge_pte(mm, pmdp);
189         pmd_huge_pte(mm, pmdp) = list_first_en    188         pmd_huge_pte(mm, pmdp) = list_first_entry_or_null(&pgtable->lru,
190                                                   189                                                           struct page, lru);
191         if (pmd_huge_pte(mm, pmdp))               190         if (pmd_huge_pte(mm, pmdp))
192                 list_del(&pgtable->lru);          191                 list_del(&pgtable->lru);
193         return pgtable;                           192         return pgtable;
194 }                                                 193 }
195 #endif                                            194 #endif
196                                                   195 
197 #ifndef __HAVE_ARCH_PMDP_INVALIDATE               196 #ifndef __HAVE_ARCH_PMDP_INVALIDATE
198 pmd_t pmdp_invalidate(struct vm_area_struct *v    197 pmd_t pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
199                      pmd_t *pmdp)                 198                      pmd_t *pmdp)
200 {                                                 199 {
201         VM_WARN_ON_ONCE(!pmd_present(*pmdp));  << 
202         pmd_t old = pmdp_establish(vma, addres    200         pmd_t old = pmdp_establish(vma, address, pmdp, pmd_mkinvalid(*pmdp));
203         flush_pmd_tlb_range(vma, address, addr    201         flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
204         return old;                               202         return old;
205 }                                                 203 }
206 #endif                                            204 #endif
207                                                   205 
208 #ifndef __HAVE_ARCH_PMDP_INVALIDATE_AD            206 #ifndef __HAVE_ARCH_PMDP_INVALIDATE_AD
209 pmd_t pmdp_invalidate_ad(struct vm_area_struct    207 pmd_t pmdp_invalidate_ad(struct vm_area_struct *vma, unsigned long address,
210                          pmd_t *pmdp)             208                          pmd_t *pmdp)
211 {                                                 209 {
212         VM_WARN_ON_ONCE(!pmd_present(*pmdp));  << 
213         return pmdp_invalidate(vma, address, p    210         return pmdp_invalidate(vma, address, pmdp);
214 }                                                 211 }
215 #endif                                            212 #endif
216                                                   213 
217 #ifndef pmdp_collapse_flush                       214 #ifndef pmdp_collapse_flush
218 pmd_t pmdp_collapse_flush(struct vm_area_struc    215 pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long address,
219                           pmd_t *pmdp)            216                           pmd_t *pmdp)
220 {                                                 217 {
221         /*                                        218         /*
222          * pmd and hugepage pte format are sam    219          * pmd and hugepage pte format are same. So we could
223          * use the same function.                 220          * use the same function.
224          */                                       221          */
225         pmd_t pmd;                                222         pmd_t pmd;
226                                                   223 
227         VM_BUG_ON(address & ~HPAGE_PMD_MASK);     224         VM_BUG_ON(address & ~HPAGE_PMD_MASK);
228         VM_BUG_ON(pmd_trans_huge(*pmdp));         225         VM_BUG_ON(pmd_trans_huge(*pmdp));
229         pmd = pmdp_huge_get_and_clear(vma->vm_    226         pmd = pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp);
230                                                   227 
231         /* collapse entails shooting down ptes    228         /* collapse entails shooting down ptes not pmd */
232         flush_tlb_range(vma, address, address     229         flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
233         return pmd;                               230         return pmd;
234 }                                                 231 }
235 #endif                                            232 #endif
236                                                << 
237 /* arch define pte_free_defer in asm/pgalloc.h << 
238 #ifndef pte_free_defer                         << 
239 static void pte_free_now(struct rcu_head *head << 
240 {                                              << 
241         struct page *page;                     << 
242                                                << 
243         page = container_of(head, struct page, << 
244         pte_free(NULL /* mm not passed and not << 
245 }                                              << 
246                                                << 
247 void pte_free_defer(struct mm_struct *mm, pgta << 
248 {                                              << 
249         struct page *page;                     << 
250                                                << 
251         page = pgtable;                        << 
252         call_rcu(&page->rcu_head, pte_free_now << 
253 }                                              << 
254 #endif /* pte_free_defer */                    << 
255 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */          233 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
256                                                   234 
257 #if defined(CONFIG_GUP_GET_PXX_LOW_HIGH) && \  << 
258         (defined(CONFIG_SMP) || defined(CONFIG << 
259 /*                                             << 
260  * See the comment above ptep_get_lockless() i << 
261  * the barriers in pmdp_get_lockless() cannot  << 
262  * pmd_high actually belongs with the value in << 
263  * off blocks the TLB flush between present up << 
264  * successful __pte_offset_map() points to a p << 
265  */                                            << 
266 static unsigned long pmdp_get_lockless_start(v << 
267 {                                              << 
268         unsigned long irqflags;                << 
269                                                << 
270         local_irq_save(irqflags);              << 
271         return irqflags;                       << 
272 }                                              << 
273 static void pmdp_get_lockless_end(unsigned lon << 
274 {                                              << 
275         local_irq_restore(irqflags);           << 
276 }                                              << 
277 #else                                          << 
278 static unsigned long pmdp_get_lockless_start(v << 
279 static void pmdp_get_lockless_end(unsigned lon << 
280 #endif                                         << 
281                                                << 
282 pte_t *__pte_offset_map(pmd_t *pmd, unsigned l    235 pte_t *__pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp)
283 {                                                 236 {
284         unsigned long irqflags;                << 
285         pmd_t pmdval;                             237         pmd_t pmdval;
286                                                   238 
287         rcu_read_lock();                       !! 239         /* rcu_read_lock() to be added later */
288         irqflags = pmdp_get_lockless_start();  << 
289         pmdval = pmdp_get_lockless(pmd);          240         pmdval = pmdp_get_lockless(pmd);
290         pmdp_get_lockless_end(irqflags);       << 
291                                                << 
292         if (pmdvalp)                              241         if (pmdvalp)
293                 *pmdvalp = pmdval;                242                 *pmdvalp = pmdval;
294         if (unlikely(pmd_none(pmdval) || is_pm    243         if (unlikely(pmd_none(pmdval) || is_pmd_migration_entry(pmdval)))
295                 goto nomap;                       244                 goto nomap;
296         if (unlikely(pmd_trans_huge(pmdval) ||    245         if (unlikely(pmd_trans_huge(pmdval) || pmd_devmap(pmdval)))
297                 goto nomap;                       246                 goto nomap;
298         if (unlikely(pmd_bad(pmdval))) {          247         if (unlikely(pmd_bad(pmdval))) {
299                 pmd_clear_bad(pmd);               248                 pmd_clear_bad(pmd);
300                 goto nomap;                       249                 goto nomap;
301         }                                         250         }
302         return __pte_map(&pmdval, addr);          251         return __pte_map(&pmdval, addr);
303 nomap:                                            252 nomap:
304         rcu_read_unlock();                     !! 253         /* rcu_read_unlock() to be added later */
305         return NULL;                              254         return NULL;
306 }                                                 255 }
307                                                   256 
308 pte_t *pte_offset_map_nolock(struct mm_struct     257 pte_t *pte_offset_map_nolock(struct mm_struct *mm, pmd_t *pmd,
309                              unsigned long add    258                              unsigned long addr, spinlock_t **ptlp)
310 {                                                 259 {
311         pmd_t pmdval;                             260         pmd_t pmdval;
312         pte_t *pte;                               261         pte_t *pte;
313                                                   262 
314         pte = __pte_offset_map(pmd, addr, &pmd    263         pte = __pte_offset_map(pmd, addr, &pmdval);
315         if (likely(pte))                          264         if (likely(pte))
316                 *ptlp = pte_lockptr(mm, &pmdva    265                 *ptlp = pte_lockptr(mm, &pmdval);
317         return pte;                               266         return pte;
318 }                                                 267 }
319                                                   268 
320 /*                                             << 
321  * pte_offset_map_lock(mm, pmd, addr, ptlp), a << 
322  * __pte_offset_map_lock() below, is usually c << 
323  * addr, reached by walking down the mm's pgd, << 
324  * holding mmap_lock or vma lock for read or f << 
325  * context, while holding file's i_mmap_lock o << 
326  * write). In a few cases, it may be used with << 
327  * copied to or constructed on the stack.      << 
328  *                                             << 
329  * When successful, it returns the pte pointer << 
330  * kmapped if necessary (when CONFIG_HIGHPTE), << 
331  * modification by software, with a pointer to << 
332  * configs mm->page_table_lock, in SPLIT_PTLOC << 
333  * struct page).  pte_unmap_unlock(pte, ptl) t << 
334  *                                             << 
335  * But it is unsuccessful, returning NULL with << 
336  * page table at *pmd: if, for example, the pa << 
337  * or replaced by the huge pmd of a THP.  (Whe << 
338  * after acquiring the ptlock, and retried int << 
339  * page table can be safely removed or replace << 
340  *                                             << 
341  * pte_offset_map(pmd, addr), and its internal << 
342  * just returns the pte pointer for addr, its  << 
343  * or NULL if there is no page table at *pmd.  << 
344  * page table, so cannot normally be used when << 
345  * or when entries read must be stable.  But i << 
346  * that even when page table is racily removed << 
347  * and disconnected table.  Until pte_unmap(pt << 
348  * afterwards.                                 << 
349  *                                             << 
350  * pte_offset_map_nolock(mm, pmd, addr, ptlp), << 
351  * but when successful, it also outputs a poin << 
352  * pte_offset_map_lock() does, but in this cas << 
353  * the caller to avoid a later pte_lockptr(mm, << 
354  * act on a changed *pmd: pte_offset_map_noloc << 
355  * pointer for the page table that it returns. << 
356  * recheck *pmd once the lock is taken; in pra << 
357  * either the mmap_lock for write, or pte_same << 
358  *                                             << 
359  * Note that free_pgtables(), used after unmap << 
360  * exiting the whole mm, does not take page ta << 
361  * table, and may not use RCU at all: "outside << 
362  * pte_offset_map() and co once the vma is det << 
363  */                                            << 
364 pte_t *__pte_offset_map_lock(struct mm_struct     269 pte_t *__pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd,
365                              unsigned long add    270                              unsigned long addr, spinlock_t **ptlp)
366 {                                                 271 {
367         spinlock_t *ptl;                          272         spinlock_t *ptl;
368         pmd_t pmdval;                             273         pmd_t pmdval;
369         pte_t *pte;                               274         pte_t *pte;
370 again:                                            275 again:
371         pte = __pte_offset_map(pmd, addr, &pmd    276         pte = __pte_offset_map(pmd, addr, &pmdval);
372         if (unlikely(!pte))                       277         if (unlikely(!pte))
373                 return pte;                       278                 return pte;
374         ptl = pte_lockptr(mm, &pmdval);           279         ptl = pte_lockptr(mm, &pmdval);
375         spin_lock(ptl);                           280         spin_lock(ptl);
376         if (likely(pmd_same(pmdval, pmdp_get_l    281         if (likely(pmd_same(pmdval, pmdp_get_lockless(pmd)))) {
377                 *ptlp = ptl;                      282                 *ptlp = ptl;
378                 return pte;                       283                 return pte;
379         }                                         284         }
380         pte_unmap_unlock(pte, ptl);               285         pte_unmap_unlock(pte, ptl);
381         goto again;                               286         goto again;
382 }                                                 287 }
383                                                   288 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

sflogo.php