1 <<
2 // SPDX-License-Identifier: GPL-2.0-only 1 // SPDX-License-Identifier: GPL-2.0-only
3 /* 2 /*
4 * linux/mm/memory.c 3 * linux/mm/memory.c
5 * 4 *
6 * Copyright (C) 1991, 1992, 1993, 1994 Linu 5 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
7 */ 6 */
8 7
9 /* 8 /*
10 * demand-loading started 01.12.91 - seems it 9 * demand-loading started 01.12.91 - seems it is high on the list of
11 * things wanted, and it should be easy to imp 10 * things wanted, and it should be easy to implement. - Linus
12 */ 11 */
13 12
14 /* 13 /*
15 * Ok, demand-loading was easy, shared pages a 14 * Ok, demand-loading was easy, shared pages a little bit tricker. Shared
16 * pages started 02.12.91, seems to work. - Li 15 * pages started 02.12.91, seems to work. - Linus.
17 * 16 *
18 * Tested sharing by executing about 30 /bin/s 17 * Tested sharing by executing about 30 /bin/sh: under the old kernel it
19 * would have taken more than the 6M I have fr 18 * would have taken more than the 6M I have free, but it worked well as
20 * far as I could see. 19 * far as I could see.
21 * 20 *
22 * Also corrected some "invalidate()"s - I was 21 * Also corrected some "invalidate()"s - I wasn't doing enough of them.
23 */ 22 */
24 23
25 /* 24 /*
26 * Real VM (paging to/from disk) started 18.12 25 * Real VM (paging to/from disk) started 18.12.91. Much more work and
27 * thought has to go into this. Oh, well.. 26 * thought has to go into this. Oh, well..
28 * 19.12.91 - works, somewhat. Sometimes I g 27 * 19.12.91 - works, somewhat. Sometimes I get faults, don't know why.
29 * Found it. Everything seems to 28 * Found it. Everything seems to work now.
30 * 20.12.91 - Ok, making the swap-device cha 29 * 20.12.91 - Ok, making the swap-device changeable like the root.
31 */ 30 */
32 31
33 /* 32 /*
34 * 05.04.94 - Multi-page memory management a 33 * 05.04.94 - Multi-page memory management added for v1.1.
35 * Idea by Alex Bligh (alex@cconc 34 * Idea by Alex Bligh (alex@cconcepts.co.uk)
36 * 35 *
37 * 16.07.99 - Support of BIGMEM added by Ger 36 * 16.07.99 - Support of BIGMEM added by Gerhard Wichert, Siemens AG
38 * (Gerhard.Wichert@pdb.siemens.d 37 * (Gerhard.Wichert@pdb.siemens.de)
39 * 38 *
40 * Aug/Sep 2004 Changed to four level page tab 39 * Aug/Sep 2004 Changed to four level page tables (Andi Kleen)
41 */ 40 */
42 41
43 #include <linux/kernel_stat.h> 42 #include <linux/kernel_stat.h>
44 #include <linux/mm.h> 43 #include <linux/mm.h>
45 #include <linux/mm_inline.h> <<
46 #include <linux/sched/mm.h> 44 #include <linux/sched/mm.h>
47 #include <linux/sched/coredump.h> 45 #include <linux/sched/coredump.h>
48 #include <linux/sched/numa_balancing.h> 46 #include <linux/sched/numa_balancing.h>
49 #include <linux/sched/task.h> 47 #include <linux/sched/task.h>
50 #include <linux/hugetlb.h> 48 #include <linux/hugetlb.h>
51 #include <linux/mman.h> 49 #include <linux/mman.h>
52 #include <linux/swap.h> 50 #include <linux/swap.h>
53 #include <linux/highmem.h> 51 #include <linux/highmem.h>
54 #include <linux/pagemap.h> 52 #include <linux/pagemap.h>
55 #include <linux/memremap.h> 53 #include <linux/memremap.h>
56 #include <linux/kmsan.h> <<
57 #include <linux/ksm.h> 54 #include <linux/ksm.h>
58 #include <linux/rmap.h> 55 #include <linux/rmap.h>
59 #include <linux/export.h> 56 #include <linux/export.h>
60 #include <linux/delayacct.h> 57 #include <linux/delayacct.h>
61 #include <linux/init.h> 58 #include <linux/init.h>
62 #include <linux/pfn_t.h> 59 #include <linux/pfn_t.h>
63 #include <linux/writeback.h> 60 #include <linux/writeback.h>
64 #include <linux/memcontrol.h> 61 #include <linux/memcontrol.h>
65 #include <linux/mmu_notifier.h> 62 #include <linux/mmu_notifier.h>
66 #include <linux/swapops.h> 63 #include <linux/swapops.h>
67 #include <linux/elf.h> 64 #include <linux/elf.h>
68 #include <linux/gfp.h> 65 #include <linux/gfp.h>
69 #include <linux/migrate.h> 66 #include <linux/migrate.h>
70 #include <linux/string.h> 67 #include <linux/string.h>
71 #include <linux/memory-tiers.h> <<
72 #include <linux/debugfs.h> 68 #include <linux/debugfs.h>
73 #include <linux/userfaultfd_k.h> 69 #include <linux/userfaultfd_k.h>
74 #include <linux/dax.h> 70 #include <linux/dax.h>
75 #include <linux/oom.h> 71 #include <linux/oom.h>
76 #include <linux/numa.h> 72 #include <linux/numa.h>
77 #include <linux/perf_event.h> 73 #include <linux/perf_event.h>
78 #include <linux/ptrace.h> 74 #include <linux/ptrace.h>
79 #include <linux/vmalloc.h> 75 #include <linux/vmalloc.h>
80 #include <linux/sched/sysctl.h> <<
81 76
82 #include <trace/events/kmem.h> 77 #include <trace/events/kmem.h>
83 78
84 #include <asm/io.h> 79 #include <asm/io.h>
85 #include <asm/mmu_context.h> 80 #include <asm/mmu_context.h>
86 #include <asm/pgalloc.h> 81 #include <asm/pgalloc.h>
87 #include <linux/uaccess.h> 82 #include <linux/uaccess.h>
88 #include <asm/tlb.h> 83 #include <asm/tlb.h>
89 #include <asm/tlbflush.h> 84 #include <asm/tlbflush.h>
90 85
91 #include "pgalloc-track.h" 86 #include "pgalloc-track.h"
92 #include "internal.h" 87 #include "internal.h"
93 #include "swap.h" <<
94 88
95 #if defined(LAST_CPUPID_NOT_IN_PAGE_FLAGS) && 89 #if defined(LAST_CPUPID_NOT_IN_PAGE_FLAGS) && !defined(CONFIG_COMPILE_TEST)
96 #warning Unfortunate NUMA and NUMA Balancing c 90 #warning Unfortunate NUMA and NUMA Balancing config, growing page-frame for last_cpupid.
97 #endif 91 #endif
98 92
99 #ifndef CONFIG_NUMA !! 93 #ifndef CONFIG_NEED_MULTIPLE_NODES
>> 94 /* use the per-pgdat data instead for discontigmem - mbligh */
100 unsigned long max_mapnr; 95 unsigned long max_mapnr;
101 EXPORT_SYMBOL(max_mapnr); 96 EXPORT_SYMBOL(max_mapnr);
102 97
103 struct page *mem_map; 98 struct page *mem_map;
104 EXPORT_SYMBOL(mem_map); 99 EXPORT_SYMBOL(mem_map);
105 #endif 100 #endif
106 101
107 static vm_fault_t do_fault(struct vm_fault *vm <<
108 static vm_fault_t do_anonymous_page(struct vm_ <<
109 static bool vmf_pte_changed(struct vm_fault *v <<
110 <<
111 /* <<
112 * Return true if the original pte was a uffd- <<
113 * wr-protected). <<
114 */ <<
115 static __always_inline bool vmf_orig_pte_uffd_ <<
116 { <<
117 if (!userfaultfd_wp(vmf->vma)) <<
118 return false; <<
119 if (!(vmf->flags & FAULT_FLAG_ORIG_PTE <<
120 return false; <<
121 <<
122 return pte_marker_uffd_wp(vmf->orig_pt <<
123 } <<
124 <<
125 /* 102 /*
126 * A number of key systems in x86 including io 103 * A number of key systems in x86 including ioremap() rely on the assumption
127 * that high_memory defines the upper bound on 104 * that high_memory defines the upper bound on direct map memory, then end
128 * of ZONE_NORMAL. !! 105 * of ZONE_NORMAL. Under CONFIG_DISCONTIG this means that max_low_pfn and
>> 106 * highstart_pfn must be the same; there must be no gap between ZONE_NORMAL
>> 107 * and ZONE_HIGHMEM.
129 */ 108 */
130 void *high_memory; 109 void *high_memory;
131 EXPORT_SYMBOL(high_memory); 110 EXPORT_SYMBOL(high_memory);
132 111
133 /* 112 /*
134 * Randomize the address space (stacks, mmaps, 113 * Randomize the address space (stacks, mmaps, brk, etc.).
135 * 114 *
136 * ( When CONFIG_COMPAT_BRK=y we exclude brk f 115 * ( When CONFIG_COMPAT_BRK=y we exclude brk from randomization,
137 * as ancient (libc5 based) binaries can seg 116 * as ancient (libc5 based) binaries can segfault. )
138 */ 117 */
139 int randomize_va_space __read_mostly = 118 int randomize_va_space __read_mostly =
140 #ifdef CONFIG_COMPAT_BRK 119 #ifdef CONFIG_COMPAT_BRK
141 1; 120 1;
142 #else 121 #else
143 2; 122 2;
144 #endif 123 #endif
145 124
146 #ifndef arch_wants_old_prefaulted_pte !! 125 #ifndef arch_faults_on_old_pte
147 static inline bool arch_wants_old_prefaulted_p !! 126 static inline bool arch_faults_on_old_pte(void)
148 { 127 {
149 /* 128 /*
150 * Transitioning a PTE from 'old' to ' !! 129 * Those arches which don't have hw access flag feature need to
151 * some architectures, even if it's pe !! 130 * implement their own helper. By default, "true" means pagefault
152 * default, "false" means prefaulted e !! 131 * will be hit on old pte.
153 */ 132 */
154 return false; !! 133 return true;
155 } 134 }
156 #endif 135 #endif
157 136
158 static int __init disable_randmaps(char *s) 137 static int __init disable_randmaps(char *s)
159 { 138 {
160 randomize_va_space = 0; 139 randomize_va_space = 0;
161 return 1; 140 return 1;
162 } 141 }
163 __setup("norandmaps", disable_randmaps); 142 __setup("norandmaps", disable_randmaps);
164 143
165 unsigned long zero_pfn __read_mostly; 144 unsigned long zero_pfn __read_mostly;
166 EXPORT_SYMBOL(zero_pfn); 145 EXPORT_SYMBOL(zero_pfn);
167 146
168 unsigned long highest_memmap_pfn __read_mostly 147 unsigned long highest_memmap_pfn __read_mostly;
169 148
170 /* 149 /*
171 * CONFIG_MMU architectures set up ZERO_PAGE i 150 * CONFIG_MMU architectures set up ZERO_PAGE in their paging_init()
172 */ 151 */
173 static int __init init_zero_pfn(void) 152 static int __init init_zero_pfn(void)
174 { 153 {
175 zero_pfn = page_to_pfn(ZERO_PAGE(0)); 154 zero_pfn = page_to_pfn(ZERO_PAGE(0));
176 return 0; 155 return 0;
177 } 156 }
178 early_initcall(init_zero_pfn); 157 early_initcall(init_zero_pfn);
179 158
180 void mm_trace_rss_stat(struct mm_struct *mm, i !! 159 void mm_trace_rss_stat(struct mm_struct *mm, int member, long count)
181 { 160 {
182 trace_rss_stat(mm, member); !! 161 trace_rss_stat(mm, member, count);
183 } 162 }
184 163
>> 164 #if defined(SPLIT_RSS_COUNTING)
>> 165
>> 166 void sync_mm_rss(struct mm_struct *mm)
>> 167 {
>> 168 int i;
>> 169
>> 170 for (i = 0; i < NR_MM_COUNTERS; i++) {
>> 171 if (current->rss_stat.count[i]) {
>> 172 add_mm_counter(mm, i, current->rss_stat.count[i]);
>> 173 current->rss_stat.count[i] = 0;
>> 174 }
>> 175 }
>> 176 current->rss_stat.events = 0;
>> 177 }
>> 178
>> 179 static void add_mm_counter_fast(struct mm_struct *mm, int member, int val)
>> 180 {
>> 181 struct task_struct *task = current;
>> 182
>> 183 if (likely(task->mm == mm))
>> 184 task->rss_stat.count[member] += val;
>> 185 else
>> 186 add_mm_counter(mm, member, val);
>> 187 }
>> 188 #define inc_mm_counter_fast(mm, member) add_mm_counter_fast(mm, member, 1)
>> 189 #define dec_mm_counter_fast(mm, member) add_mm_counter_fast(mm, member, -1)
>> 190
>> 191 /* sync counter once per 64 page faults */
>> 192 #define TASK_RSS_EVENTS_THRESH (64)
>> 193 static void check_sync_rss_stat(struct task_struct *task)
>> 194 {
>> 195 if (unlikely(task != current))
>> 196 return;
>> 197 if (unlikely(task->rss_stat.events++ > TASK_RSS_EVENTS_THRESH))
>> 198 sync_mm_rss(task->mm);
>> 199 }
>> 200 #else /* SPLIT_RSS_COUNTING */
>> 201
>> 202 #define inc_mm_counter_fast(mm, member) inc_mm_counter(mm, member)
>> 203 #define dec_mm_counter_fast(mm, member) dec_mm_counter(mm, member)
>> 204
>> 205 static void check_sync_rss_stat(struct task_struct *task)
>> 206 {
>> 207 }
>> 208
>> 209 #endif /* SPLIT_RSS_COUNTING */
>> 210
185 /* 211 /*
186 * Note: this doesn't free the actual pages th 212 * Note: this doesn't free the actual pages themselves. That
187 * has been handled earlier when unmapping all 213 * has been handled earlier when unmapping all the memory regions.
188 */ 214 */
189 static void free_pte_range(struct mmu_gather * 215 static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd,
190 unsigned long addr) 216 unsigned long addr)
191 { 217 {
192 pgtable_t token = pmd_pgtable(*pmd); 218 pgtable_t token = pmd_pgtable(*pmd);
193 pmd_clear(pmd); 219 pmd_clear(pmd);
194 pte_free_tlb(tlb, token, addr); 220 pte_free_tlb(tlb, token, addr);
195 mm_dec_nr_ptes(tlb->mm); 221 mm_dec_nr_ptes(tlb->mm);
196 } 222 }
197 223
198 static inline void free_pmd_range(struct mmu_g 224 static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
199 unsigned long 225 unsigned long addr, unsigned long end,
200 unsigned long 226 unsigned long floor, unsigned long ceiling)
201 { 227 {
202 pmd_t *pmd; 228 pmd_t *pmd;
203 unsigned long next; 229 unsigned long next;
204 unsigned long start; 230 unsigned long start;
205 231
206 start = addr; 232 start = addr;
207 pmd = pmd_offset(pud, addr); 233 pmd = pmd_offset(pud, addr);
208 do { 234 do {
209 next = pmd_addr_end(addr, end) 235 next = pmd_addr_end(addr, end);
210 if (pmd_none_or_clear_bad(pmd) 236 if (pmd_none_or_clear_bad(pmd))
211 continue; 237 continue;
212 free_pte_range(tlb, pmd, addr) 238 free_pte_range(tlb, pmd, addr);
213 } while (pmd++, addr = next, addr != e 239 } while (pmd++, addr = next, addr != end);
214 240
215 start &= PUD_MASK; 241 start &= PUD_MASK;
216 if (start < floor) 242 if (start < floor)
217 return; 243 return;
218 if (ceiling) { 244 if (ceiling) {
219 ceiling &= PUD_MASK; 245 ceiling &= PUD_MASK;
220 if (!ceiling) 246 if (!ceiling)
221 return; 247 return;
222 } 248 }
223 if (end - 1 > ceiling - 1) 249 if (end - 1 > ceiling - 1)
224 return; 250 return;
225 251
226 pmd = pmd_offset(pud, start); 252 pmd = pmd_offset(pud, start);
227 pud_clear(pud); 253 pud_clear(pud);
228 pmd_free_tlb(tlb, pmd, start); 254 pmd_free_tlb(tlb, pmd, start);
229 mm_dec_nr_pmds(tlb->mm); 255 mm_dec_nr_pmds(tlb->mm);
230 } 256 }
231 257
232 static inline void free_pud_range(struct mmu_g 258 static inline void free_pud_range(struct mmu_gather *tlb, p4d_t *p4d,
233 unsigned long 259 unsigned long addr, unsigned long end,
234 unsigned long 260 unsigned long floor, unsigned long ceiling)
235 { 261 {
236 pud_t *pud; 262 pud_t *pud;
237 unsigned long next; 263 unsigned long next;
238 unsigned long start; 264 unsigned long start;
239 265
240 start = addr; 266 start = addr;
241 pud = pud_offset(p4d, addr); 267 pud = pud_offset(p4d, addr);
242 do { 268 do {
243 next = pud_addr_end(addr, end) 269 next = pud_addr_end(addr, end);
244 if (pud_none_or_clear_bad(pud) 270 if (pud_none_or_clear_bad(pud))
245 continue; 271 continue;
246 free_pmd_range(tlb, pud, addr, 272 free_pmd_range(tlb, pud, addr, next, floor, ceiling);
247 } while (pud++, addr = next, addr != e 273 } while (pud++, addr = next, addr != end);
248 274
249 start &= P4D_MASK; 275 start &= P4D_MASK;
250 if (start < floor) 276 if (start < floor)
251 return; 277 return;
252 if (ceiling) { 278 if (ceiling) {
253 ceiling &= P4D_MASK; 279 ceiling &= P4D_MASK;
254 if (!ceiling) 280 if (!ceiling)
255 return; 281 return;
256 } 282 }
257 if (end - 1 > ceiling - 1) 283 if (end - 1 > ceiling - 1)
258 return; 284 return;
259 285
260 pud = pud_offset(p4d, start); 286 pud = pud_offset(p4d, start);
261 p4d_clear(p4d); 287 p4d_clear(p4d);
262 pud_free_tlb(tlb, pud, start); 288 pud_free_tlb(tlb, pud, start);
263 mm_dec_nr_puds(tlb->mm); 289 mm_dec_nr_puds(tlb->mm);
264 } 290 }
265 291
266 static inline void free_p4d_range(struct mmu_g 292 static inline void free_p4d_range(struct mmu_gather *tlb, pgd_t *pgd,
267 unsigned long 293 unsigned long addr, unsigned long end,
268 unsigned long 294 unsigned long floor, unsigned long ceiling)
269 { 295 {
270 p4d_t *p4d; 296 p4d_t *p4d;
271 unsigned long next; 297 unsigned long next;
272 unsigned long start; 298 unsigned long start;
273 299
274 start = addr; 300 start = addr;
275 p4d = p4d_offset(pgd, addr); 301 p4d = p4d_offset(pgd, addr);
276 do { 302 do {
277 next = p4d_addr_end(addr, end) 303 next = p4d_addr_end(addr, end);
278 if (p4d_none_or_clear_bad(p4d) 304 if (p4d_none_or_clear_bad(p4d))
279 continue; 305 continue;
280 free_pud_range(tlb, p4d, addr, 306 free_pud_range(tlb, p4d, addr, next, floor, ceiling);
281 } while (p4d++, addr = next, addr != e 307 } while (p4d++, addr = next, addr != end);
282 308
283 start &= PGDIR_MASK; 309 start &= PGDIR_MASK;
284 if (start < floor) 310 if (start < floor)
285 return; 311 return;
286 if (ceiling) { 312 if (ceiling) {
287 ceiling &= PGDIR_MASK; 313 ceiling &= PGDIR_MASK;
288 if (!ceiling) 314 if (!ceiling)
289 return; 315 return;
290 } 316 }
291 if (end - 1 > ceiling - 1) 317 if (end - 1 > ceiling - 1)
292 return; 318 return;
293 319
294 p4d = p4d_offset(pgd, start); 320 p4d = p4d_offset(pgd, start);
295 pgd_clear(pgd); 321 pgd_clear(pgd);
296 p4d_free_tlb(tlb, p4d, start); 322 p4d_free_tlb(tlb, p4d, start);
297 } 323 }
298 324
299 /* 325 /*
300 * This function frees user-level page tables 326 * This function frees user-level page tables of a process.
301 */ 327 */
302 void free_pgd_range(struct mmu_gather *tlb, 328 void free_pgd_range(struct mmu_gather *tlb,
303 unsigned long addr, un 329 unsigned long addr, unsigned long end,
304 unsigned long floor, u 330 unsigned long floor, unsigned long ceiling)
305 { 331 {
306 pgd_t *pgd; 332 pgd_t *pgd;
307 unsigned long next; 333 unsigned long next;
308 334
309 /* 335 /*
310 * The next few lines have given us lo 336 * The next few lines have given us lots of grief...
311 * 337 *
312 * Why are we testing PMD* at this top 338 * Why are we testing PMD* at this top level? Because often
313 * there will be no work to do at all, 339 * there will be no work to do at all, and we'd prefer not to
314 * go all the way down to the bottom j 340 * go all the way down to the bottom just to discover that.
315 * 341 *
316 * Why all these "- 1"s? Because 0 re 342 * Why all these "- 1"s? Because 0 represents both the bottom
317 * of the address space and the top of 343 * of the address space and the top of it (using -1 for the
318 * top wouldn't help much: the masks w 344 * top wouldn't help much: the masks would do the wrong thing).
319 * The rule is that addr 0 and floor 0 345 * The rule is that addr 0 and floor 0 refer to the bottom of
320 * the address space, but end 0 and ce 346 * the address space, but end 0 and ceiling 0 refer to the top
321 * Comparisons need to use "end - 1" a 347 * Comparisons need to use "end - 1" and "ceiling - 1" (though
322 * that end 0 case should be mythical) 348 * that end 0 case should be mythical).
323 * 349 *
324 * Wherever addr is brought up or ceil 350 * Wherever addr is brought up or ceiling brought down, we must
325 * be careful to reject "the opposite 351 * be careful to reject "the opposite 0" before it confuses the
326 * subsequent tests. But what about w 352 * subsequent tests. But what about where end is brought down
327 * by PMD_SIZE below? no, end can't go 353 * by PMD_SIZE below? no, end can't go down to 0 there.
328 * 354 *
329 * Whereas we round start (addr) and c 355 * Whereas we round start (addr) and ceiling down, by different
330 * masks at different levels, in order 356 * masks at different levels, in order to test whether a table
331 * now has no other vmas using it, so 357 * now has no other vmas using it, so can be freed, we don't
332 * bother to round floor or end up - t 358 * bother to round floor or end up - the tests don't need that.
333 */ 359 */
334 360
335 addr &= PMD_MASK; 361 addr &= PMD_MASK;
336 if (addr < floor) { 362 if (addr < floor) {
337 addr += PMD_SIZE; 363 addr += PMD_SIZE;
338 if (!addr) 364 if (!addr)
339 return; 365 return;
340 } 366 }
341 if (ceiling) { 367 if (ceiling) {
342 ceiling &= PMD_MASK; 368 ceiling &= PMD_MASK;
343 if (!ceiling) 369 if (!ceiling)
344 return; 370 return;
345 } 371 }
346 if (end - 1 > ceiling - 1) 372 if (end - 1 > ceiling - 1)
347 end -= PMD_SIZE; 373 end -= PMD_SIZE;
348 if (addr > end - 1) 374 if (addr > end - 1)
349 return; 375 return;
350 /* 376 /*
351 * We add page table cache pages with 377 * We add page table cache pages with PAGE_SIZE,
352 * (see pte_free_tlb()), flush the tlb 378 * (see pte_free_tlb()), flush the tlb if we need
353 */ 379 */
354 tlb_change_page_size(tlb, PAGE_SIZE); 380 tlb_change_page_size(tlb, PAGE_SIZE);
355 pgd = pgd_offset(tlb->mm, addr); 381 pgd = pgd_offset(tlb->mm, addr);
356 do { 382 do {
357 next = pgd_addr_end(addr, end) 383 next = pgd_addr_end(addr, end);
358 if (pgd_none_or_clear_bad(pgd) 384 if (pgd_none_or_clear_bad(pgd))
359 continue; 385 continue;
360 free_p4d_range(tlb, pgd, addr, 386 free_p4d_range(tlb, pgd, addr, next, floor, ceiling);
361 } while (pgd++, addr = next, addr != e 387 } while (pgd++, addr = next, addr != end);
362 } 388 }
363 389
364 void free_pgtables(struct mmu_gather *tlb, str !! 390 void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *vma,
365 struct vm_area_struct *vma, !! 391 unsigned long floor, unsigned long ceiling)
366 unsigned long ceiling, bool <<
367 { 392 {
368 struct unlink_vma_file_batch vb; !! 393 while (vma) {
369 !! 394 struct vm_area_struct *next = vma->vm_next;
370 do { <<
371 unsigned long addr = vma->vm_s 395 unsigned long addr = vma->vm_start;
372 struct vm_area_struct *next; <<
373 <<
374 /* <<
375 * Note: USER_PGTABLES_CEILING <<
376 * be 0. This will underflow <<
377 */ <<
378 next = mas_find(mas, ceiling - <<
379 if (unlikely(xa_is_zero(next)) <<
380 next = NULL; <<
381 396
382 /* 397 /*
383 * Hide vma from rmap and trun 398 * Hide vma from rmap and truncate_pagecache before freeing
384 * pgtables 399 * pgtables
385 */ 400 */
386 if (mm_wr_locked) <<
387 vma_start_write(vma); <<
388 unlink_anon_vmas(vma); 401 unlink_anon_vmas(vma);
>> 402 unlink_file_vma(vma);
389 403
390 if (is_vm_hugetlb_page(vma)) { 404 if (is_vm_hugetlb_page(vma)) {
391 unlink_file_vma(vma); <<
392 hugetlb_free_pgd_range 405 hugetlb_free_pgd_range(tlb, addr, vma->vm_end,
393 floor, next ? 406 floor, next ? next->vm_start : ceiling);
394 } else { 407 } else {
395 unlink_file_vma_batch_ <<
396 unlink_file_vma_batch_ <<
397 <<
398 /* 408 /*
399 * Optimization: gathe 409 * Optimization: gather nearby vmas into one call down
400 */ 410 */
401 while (next && next->v 411 while (next && next->vm_start <= vma->vm_end + PMD_SIZE
402 && !is_vm_huget 412 && !is_vm_hugetlb_page(next)) {
403 vma = next; 413 vma = next;
404 next = mas_fin !! 414 next = vma->vm_next;
405 if (unlikely(x <<
406 next = <<
407 if (mm_wr_lock <<
408 vma_st <<
409 unlink_anon_vm 415 unlink_anon_vmas(vma);
410 unlink_file_vm !! 416 unlink_file_vma(vma);
411 } 417 }
412 unlink_file_vma_batch_ <<
413 free_pgd_range(tlb, ad 418 free_pgd_range(tlb, addr, vma->vm_end,
414 floor, next ? 419 floor, next ? next->vm_start : ceiling);
415 } 420 }
416 vma = next; 421 vma = next;
417 } while (vma); <<
418 } <<
419 <<
420 void pmd_install(struct mm_struct *mm, pmd_t * <<
421 { <<
422 spinlock_t *ptl = pmd_lock(mm, pmd); <<
423 <<
424 if (likely(pmd_none(*pmd))) { /* Has <<
425 mm_inc_nr_ptes(mm); <<
426 /* <<
427 * Ensure all pte setup (eg. p <<
428 * visible before the pte is m <<
429 * put into page tables. <<
430 * <<
431 * The other side of the story <<
432 * table walking code (when wa <<
433 * ie. most of the time). Fort <<
434 * of a chain of data-dependen <<
435 * being the notable exception <<
436 * seen in-order. See the alph <<
437 * smp_rmb() barriers in page <<
438 */ <<
439 smp_wmb(); /* Could be smp_wmb <<
440 pmd_populate(mm, pmd, *pte); <<
441 *pte = NULL; <<
442 } 422 }
443 spin_unlock(ptl); <<
444 } 423 }
445 424
446 int __pte_alloc(struct mm_struct *mm, pmd_t *p 425 int __pte_alloc(struct mm_struct *mm, pmd_t *pmd)
447 { 426 {
>> 427 spinlock_t *ptl;
448 pgtable_t new = pte_alloc_one(mm); 428 pgtable_t new = pte_alloc_one(mm);
449 if (!new) 429 if (!new)
450 return -ENOMEM; 430 return -ENOMEM;
451 431
452 pmd_install(mm, pmd, &new); !! 432 /*
>> 433 * Ensure all pte setup (eg. pte page lock and page clearing) are
>> 434 * visible before the pte is made visible to other CPUs by being
>> 435 * put into page tables.
>> 436 *
>> 437 * The other side of the story is the pointer chasing in the page
>> 438 * table walking code (when walking the page table without locking;
>> 439 * ie. most of the time). Fortunately, these data accesses consist
>> 440 * of a chain of data-dependent loads, meaning most CPUs (alpha
>> 441 * being the notable exception) will already guarantee loads are
>> 442 * seen in-order. See the alpha page table accessors for the
>> 443 * smp_rmb() barriers in page table walking code.
>> 444 */
>> 445 smp_wmb(); /* Could be smp_wmb__xxx(before|after)_spin_lock */
>> 446
>> 447 ptl = pmd_lock(mm, pmd);
>> 448 if (likely(pmd_none(*pmd))) { /* Has another populated it ? */
>> 449 mm_inc_nr_ptes(mm);
>> 450 pmd_populate(mm, pmd, new);
>> 451 new = NULL;
>> 452 }
>> 453 spin_unlock(ptl);
453 if (new) 454 if (new)
454 pte_free(mm, new); 455 pte_free(mm, new);
455 return 0; 456 return 0;
456 } 457 }
457 458
458 int __pte_alloc_kernel(pmd_t *pmd) 459 int __pte_alloc_kernel(pmd_t *pmd)
459 { 460 {
460 pte_t *new = pte_alloc_one_kernel(&ini 461 pte_t *new = pte_alloc_one_kernel(&init_mm);
461 if (!new) 462 if (!new)
462 return -ENOMEM; 463 return -ENOMEM;
463 464
>> 465 smp_wmb(); /* See comment in __pte_alloc */
>> 466
464 spin_lock(&init_mm.page_table_lock); 467 spin_lock(&init_mm.page_table_lock);
465 if (likely(pmd_none(*pmd))) { /* Has 468 if (likely(pmd_none(*pmd))) { /* Has another populated it ? */
466 smp_wmb(); /* See comment in p <<
467 pmd_populate_kernel(&init_mm, 469 pmd_populate_kernel(&init_mm, pmd, new);
468 new = NULL; 470 new = NULL;
469 } 471 }
470 spin_unlock(&init_mm.page_table_lock); 472 spin_unlock(&init_mm.page_table_lock);
471 if (new) 473 if (new)
472 pte_free_kernel(&init_mm, new) 474 pte_free_kernel(&init_mm, new);
473 return 0; 475 return 0;
474 } 476 }
475 477
476 static inline void init_rss_vec(int *rss) 478 static inline void init_rss_vec(int *rss)
477 { 479 {
478 memset(rss, 0, sizeof(int) * NR_MM_COU 480 memset(rss, 0, sizeof(int) * NR_MM_COUNTERS);
479 } 481 }
480 482
481 static inline void add_mm_rss_vec(struct mm_st 483 static inline void add_mm_rss_vec(struct mm_struct *mm, int *rss)
482 { 484 {
483 int i; 485 int i;
484 486
>> 487 if (current->mm == mm)
>> 488 sync_mm_rss(mm);
485 for (i = 0; i < NR_MM_COUNTERS; i++) 489 for (i = 0; i < NR_MM_COUNTERS; i++)
486 if (rss[i]) 490 if (rss[i])
487 add_mm_counter(mm, i, 491 add_mm_counter(mm, i, rss[i]);
488 } 492 }
489 493
490 /* 494 /*
491 * This function is called to print an error w 495 * This function is called to print an error when a bad pte
492 * is found. For example, we might have a PFN- 496 * is found. For example, we might have a PFN-mapped pte in
493 * a region that doesn't allow it. 497 * a region that doesn't allow it.
494 * 498 *
495 * The calling function must still handle the 499 * The calling function must still handle the error.
496 */ 500 */
497 static void print_bad_pte(struct vm_area_struc 501 static void print_bad_pte(struct vm_area_struct *vma, unsigned long addr,
498 pte_t pte, struct pa 502 pte_t pte, struct page *page)
499 { 503 {
500 pgd_t *pgd = pgd_offset(vma->vm_mm, ad 504 pgd_t *pgd = pgd_offset(vma->vm_mm, addr);
501 p4d_t *p4d = p4d_offset(pgd, addr); 505 p4d_t *p4d = p4d_offset(pgd, addr);
502 pud_t *pud = pud_offset(p4d, addr); 506 pud_t *pud = pud_offset(p4d, addr);
503 pmd_t *pmd = pmd_offset(pud, addr); 507 pmd_t *pmd = pmd_offset(pud, addr);
504 struct address_space *mapping; 508 struct address_space *mapping;
505 pgoff_t index; 509 pgoff_t index;
506 static unsigned long resume; 510 static unsigned long resume;
507 static unsigned long nr_shown; 511 static unsigned long nr_shown;
508 static unsigned long nr_unshown; 512 static unsigned long nr_unshown;
509 513
510 /* 514 /*
511 * Allow a burst of 60 reports, then k 515 * Allow a burst of 60 reports, then keep quiet for that minute;
512 * or allow a steady drip of one repor 516 * or allow a steady drip of one report per second.
513 */ 517 */
514 if (nr_shown == 60) { 518 if (nr_shown == 60) {
515 if (time_before(jiffies, resum 519 if (time_before(jiffies, resume)) {
516 nr_unshown++; 520 nr_unshown++;
517 return; 521 return;
518 } 522 }
519 if (nr_unshown) { 523 if (nr_unshown) {
520 pr_alert("BUG: Bad pag 524 pr_alert("BUG: Bad page map: %lu messages suppressed\n",
521 nr_unshown); 525 nr_unshown);
522 nr_unshown = 0; 526 nr_unshown = 0;
523 } 527 }
524 nr_shown = 0; 528 nr_shown = 0;
525 } 529 }
526 if (nr_shown++ == 0) 530 if (nr_shown++ == 0)
527 resume = jiffies + 60 * HZ; 531 resume = jiffies + 60 * HZ;
528 532
529 mapping = vma->vm_file ? vma->vm_file- 533 mapping = vma->vm_file ? vma->vm_file->f_mapping : NULL;
530 index = linear_page_index(vma, addr); 534 index = linear_page_index(vma, addr);
531 535
532 pr_alert("BUG: Bad page map in process 536 pr_alert("BUG: Bad page map in process %s pte:%08llx pmd:%08llx\n",
533 current->comm, 537 current->comm,
534 (long long)pte_val(pte), (lon 538 (long long)pte_val(pte), (long long)pmd_val(*pmd));
535 if (page) 539 if (page)
536 dump_page(page, "bad pte"); 540 dump_page(page, "bad pte");
537 pr_alert("addr:%px vm_flags:%08lx anon 541 pr_alert("addr:%px vm_flags:%08lx anon_vma:%px mapping:%px index:%lx\n",
538 (void *)addr, vma->vm_flags, 542 (void *)addr, vma->vm_flags, vma->anon_vma, mapping, index);
539 pr_alert("file:%pD fault:%ps mmap:%ps !! 543 pr_alert("file:%pD fault:%ps mmap:%ps readpage:%ps\n",
540 vma->vm_file, 544 vma->vm_file,
541 vma->vm_ops ? vma->vm_ops->fa 545 vma->vm_ops ? vma->vm_ops->fault : NULL,
542 vma->vm_file ? vma->vm_file-> 546 vma->vm_file ? vma->vm_file->f_op->mmap : NULL,
543 mapping ? mapping->a_ops->rea !! 547 mapping ? mapping->a_ops->readpage : NULL);
544 dump_stack(); 548 dump_stack();
545 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_ 549 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
546 } 550 }
547 551
548 /* 552 /*
549 * vm_normal_page -- This function gets the "s 553 * vm_normal_page -- This function gets the "struct page" associated with a pte.
550 * 554 *
551 * "Special" mappings do not wish to be associ 555 * "Special" mappings do not wish to be associated with a "struct page" (either
552 * it doesn't exist, or it exists but they don 556 * it doesn't exist, or it exists but they don't want to touch it). In this
553 * case, NULL is returned here. "Normal" mappi 557 * case, NULL is returned here. "Normal" mappings do have a struct page.
554 * 558 *
555 * There are 2 broad cases. Firstly, an archit 559 * There are 2 broad cases. Firstly, an architecture may define a pte_special()
556 * pte bit, in which case this function is tri 560 * pte bit, in which case this function is trivial. Secondly, an architecture
557 * may not have a spare pte bit, which require 561 * may not have a spare pte bit, which requires a more complicated scheme,
558 * described below. 562 * described below.
559 * 563 *
560 * A raw VM_PFNMAP mapping (ie. one that is no 564 * A raw VM_PFNMAP mapping (ie. one that is not COWed) is always considered a
561 * special mapping (even if there are underlyi 565 * special mapping (even if there are underlying and valid "struct pages").
562 * COWed pages of a VM_PFNMAP are always norma 566 * COWed pages of a VM_PFNMAP are always normal.
563 * 567 *
564 * The way we recognize COWed pages within VM_ 568 * The way we recognize COWed pages within VM_PFNMAP mappings is through the
565 * rules set up by "remap_pfn_range()": the vm 569 * rules set up by "remap_pfn_range()": the vma will have the VM_PFNMAP bit
566 * set, and the vm_pgoff will point to the fir 570 * set, and the vm_pgoff will point to the first PFN mapped: thus every special
567 * mapping will always honor the rule 571 * mapping will always honor the rule
568 * 572 *
569 * pfn_of_page == vma->vm_pgoff + ((addr 573 * pfn_of_page == vma->vm_pgoff + ((addr - vma->vm_start) >> PAGE_SHIFT)
570 * 574 *
571 * And for normal mappings this is false. 575 * And for normal mappings this is false.
572 * 576 *
573 * This restricts such mappings to be a linear 577 * This restricts such mappings to be a linear translation from virtual address
574 * to pfn. To get around this restriction, we 578 * to pfn. To get around this restriction, we allow arbitrary mappings so long
575 * as the vma is not a COW mapping; in that ca 579 * as the vma is not a COW mapping; in that case, we know that all ptes are
576 * special (because none can have been COWed). 580 * special (because none can have been COWed).
577 * 581 *
578 * 582 *
579 * In order to support COW of arbitrary specia 583 * In order to support COW of arbitrary special mappings, we have VM_MIXEDMAP.
580 * 584 *
581 * VM_MIXEDMAP mappings can likewise contain m 585 * VM_MIXEDMAP mappings can likewise contain memory with or without "struct
582 * page" backing, however the difference is th 586 * page" backing, however the difference is that _all_ pages with a struct
583 * page (that is, those where pfn_valid is tru 587 * page (that is, those where pfn_valid is true) are refcounted and considered
584 * normal pages by the VM. The only exception !! 588 * normal pages by the VM. The disadvantage is that pages are refcounted
585 * *never* refcounted. !! 589 * (which can be slower and simply not an option for some PFNMAP users). The
586 * !! 590 * advantage is that we don't have to follow the strict linearity rule of
587 * The disadvantage is that pages are refcount !! 591 * PFNMAP mappings in order to support COWable mappings.
588 * simply not an option for some PFNMAP users) <<
589 * don't have to follow the strict linearity r <<
590 * order to support COWable mappings. <<
591 * 592 *
592 */ 593 */
593 struct page *vm_normal_page(struct vm_area_str 594 struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
594 pte_t pte) 595 pte_t pte)
595 { 596 {
596 unsigned long pfn = pte_pfn(pte); 597 unsigned long pfn = pte_pfn(pte);
597 598
598 if (IS_ENABLED(CONFIG_ARCH_HAS_PTE_SPE 599 if (IS_ENABLED(CONFIG_ARCH_HAS_PTE_SPECIAL)) {
599 if (likely(!pte_special(pte))) 600 if (likely(!pte_special(pte)))
600 goto check_pfn; 601 goto check_pfn;
601 if (vma->vm_ops && vma->vm_ops 602 if (vma->vm_ops && vma->vm_ops->find_special_page)
602 return vma->vm_ops->fi 603 return vma->vm_ops->find_special_page(vma, addr);
603 if (vma->vm_flags & (VM_PFNMAP 604 if (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP))
604 return NULL; 605 return NULL;
605 if (is_zero_pfn(pfn)) 606 if (is_zero_pfn(pfn))
606 return NULL; 607 return NULL;
607 if (pte_devmap(pte)) 608 if (pte_devmap(pte))
608 /* <<
609 * NOTE: New users of ZONE_DEV <<
610 * and will have refcounts inc <<
611 * when they are inserted into <<
612 * return here. Legacy ZONE_DE <<
613 * do not have refcounts. Exam <<
614 * MEMORY_DEVICE_FS_DAX type i <<
615 */ <<
616 return NULL; 609 return NULL;
617 610
618 print_bad_pte(vma, addr, pte, 611 print_bad_pte(vma, addr, pte, NULL);
619 return NULL; 612 return NULL;
620 } 613 }
621 614
622 /* !CONFIG_ARCH_HAS_PTE_SPECIAL case f 615 /* !CONFIG_ARCH_HAS_PTE_SPECIAL case follows: */
623 616
624 if (unlikely(vma->vm_flags & (VM_PFNMA 617 if (unlikely(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))) {
625 if (vma->vm_flags & VM_MIXEDMA 618 if (vma->vm_flags & VM_MIXEDMAP) {
626 if (!pfn_valid(pfn)) 619 if (!pfn_valid(pfn))
627 return NULL; 620 return NULL;
628 if (is_zero_pfn(pfn)) <<
629 return NULL; <<
630 goto out; 621 goto out;
631 } else { 622 } else {
632 unsigned long off; 623 unsigned long off;
633 off = (addr - vma->vm_ 624 off = (addr - vma->vm_start) >> PAGE_SHIFT;
634 if (pfn == vma->vm_pgo 625 if (pfn == vma->vm_pgoff + off)
635 return NULL; 626 return NULL;
636 if (!is_cow_mapping(vm 627 if (!is_cow_mapping(vma->vm_flags))
637 return NULL; 628 return NULL;
638 } 629 }
639 } 630 }
640 631
641 if (is_zero_pfn(pfn)) 632 if (is_zero_pfn(pfn))
642 return NULL; 633 return NULL;
643 634
644 check_pfn: 635 check_pfn:
645 if (unlikely(pfn > highest_memmap_pfn) 636 if (unlikely(pfn > highest_memmap_pfn)) {
646 print_bad_pte(vma, addr, pte, 637 print_bad_pte(vma, addr, pte, NULL);
647 return NULL; 638 return NULL;
648 } 639 }
649 640
650 /* 641 /*
651 * NOTE! We still have PageReserved() 642 * NOTE! We still have PageReserved() pages in the page tables.
652 * eg. VDSO mappings can cause them to 643 * eg. VDSO mappings can cause them to exist.
653 */ 644 */
654 out: 645 out:
655 VM_WARN_ON_ONCE(is_zero_pfn(pfn)); <<
656 return pfn_to_page(pfn); 646 return pfn_to_page(pfn);
657 } 647 }
658 648
659 struct folio *vm_normal_folio(struct vm_area_s !! 649 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
660 pte_t pte) <<
661 { <<
662 struct page *page = vm_normal_page(vma <<
663 <<
664 if (page) <<
665 return page_folio(page); <<
666 return NULL; <<
667 } <<
668 <<
669 #ifdef CONFIG_PGTABLE_HAS_HUGE_LEAVES <<
670 struct page *vm_normal_page_pmd(struct vm_area 650 struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr,
671 pmd_t pmd) 651 pmd_t pmd)
672 { 652 {
673 unsigned long pfn = pmd_pfn(pmd); 653 unsigned long pfn = pmd_pfn(pmd);
674 654
675 /* Currently it's only used for huge p !! 655 /*
676 if (unlikely(pmd_special(pmd))) !! 656 * There is no pmd_special() but there may be special pmds, e.g.
677 return NULL; !! 657 * in a direct-access (dax) mapping, so let's just replicate the
678 !! 658 * !CONFIG_ARCH_HAS_PTE_SPECIAL case from vm_normal_page() here.
>> 659 */
679 if (unlikely(vma->vm_flags & (VM_PFNMA 660 if (unlikely(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))) {
680 if (vma->vm_flags & VM_MIXEDMA 661 if (vma->vm_flags & VM_MIXEDMAP) {
681 if (!pfn_valid(pfn)) 662 if (!pfn_valid(pfn))
682 return NULL; 663 return NULL;
683 goto out; 664 goto out;
684 } else { 665 } else {
685 unsigned long off; 666 unsigned long off;
686 off = (addr - vma->vm_ 667 off = (addr - vma->vm_start) >> PAGE_SHIFT;
687 if (pfn == vma->vm_pgo 668 if (pfn == vma->vm_pgoff + off)
688 return NULL; 669 return NULL;
689 if (!is_cow_mapping(vm 670 if (!is_cow_mapping(vma->vm_flags))
690 return NULL; 671 return NULL;
691 } 672 }
692 } 673 }
693 674
694 if (pmd_devmap(pmd)) 675 if (pmd_devmap(pmd))
695 return NULL; 676 return NULL;
696 if (is_huge_zero_pmd(pmd)) 677 if (is_huge_zero_pmd(pmd))
697 return NULL; 678 return NULL;
698 if (unlikely(pfn > highest_memmap_pfn) 679 if (unlikely(pfn > highest_memmap_pfn))
699 return NULL; 680 return NULL;
700 681
701 /* 682 /*
702 * NOTE! We still have PageReserved() 683 * NOTE! We still have PageReserved() pages in the page tables.
703 * eg. VDSO mappings can cause them to 684 * eg. VDSO mappings can cause them to exist.
704 */ 685 */
705 out: 686 out:
706 return pfn_to_page(pfn); 687 return pfn_to_page(pfn);
707 } 688 }
708 <<
709 struct folio *vm_normal_folio_pmd(struct vm_ar <<
710 unsigned lon <<
711 { <<
712 struct page *page = vm_normal_page_pmd <<
713 <<
714 if (page) <<
715 return page_folio(page); <<
716 return NULL; <<
717 } <<
718 #endif 689 #endif
719 690
720 static void restore_exclusive_pte(struct vm_ar <<
721 struct page <<
722 pte_t *ptep) <<
723 { <<
724 struct folio *folio = page_folio(page) <<
725 pte_t orig_pte; <<
726 pte_t pte; <<
727 swp_entry_t entry; <<
728 <<
729 orig_pte = ptep_get(ptep); <<
730 pte = pte_mkold(mk_pte(page, READ_ONCE <<
731 if (pte_swp_soft_dirty(orig_pte)) <<
732 pte = pte_mksoft_dirty(pte); <<
733 <<
734 entry = pte_to_swp_entry(orig_pte); <<
735 if (pte_swp_uffd_wp(orig_pte)) <<
736 pte = pte_mkuffd_wp(pte); <<
737 else if (is_writable_device_exclusive_ <<
738 pte = maybe_mkwrite(pte_mkdirt <<
739 <<
740 VM_BUG_ON_FOLIO(pte_write(pte) && (!fo <<
741 Pag <<
742 <<
743 /* <<
744 * No need to take a page reference as <<
745 * created when the swap entry was mad <<
746 */ <<
747 if (folio_test_anon(folio)) <<
748 folio_add_anon_rmap_pte(folio, <<
749 else <<
750 /* <<
751 * Currently device exclusive <<
752 * memory so the entry shouldn <<
753 */ <<
754 WARN_ON_ONCE(1); <<
755 <<
756 set_pte_at(vma->vm_mm, address, ptep, <<
757 <<
758 /* <<
759 * No need to invalidate - it was non- <<
760 * secondary CPUs may have mappings th <<
761 */ <<
762 update_mmu_cache(vma, address, ptep); <<
763 } <<
764 <<
765 /* <<
766 * Tries to restore an exclusive pte if the pa <<
767 * sleeping. <<
768 */ <<
769 static int <<
770 try_restore_exclusive_pte(pte_t *src_pte, stru <<
771 unsigned long addr) <<
772 { <<
773 swp_entry_t entry = pte_to_swp_entry(p <<
774 struct page *page = pfn_swap_entry_to_ <<
775 <<
776 if (trylock_page(page)) { <<
777 restore_exclusive_pte(vma, pag <<
778 unlock_page(page); <<
779 return 0; <<
780 } <<
781 <<
782 return -EBUSY; <<
783 } <<
784 <<
785 /* 691 /*
786 * copy one vm_area from one task to the other 692 * copy one vm_area from one task to the other. Assumes the page tables
787 * already present in the new task to be clear 693 * already present in the new task to be cleared in the whole range
788 * covered by this vma. 694 * covered by this vma.
789 */ 695 */
790 696
791 static unsigned long 697 static unsigned long
792 copy_nonpresent_pte(struct mm_struct *dst_mm, 698 copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
793 pte_t *dst_pte, pte_t *src_pte !! 699 pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *vma,
794 struct vm_area_struct *src_vma !! 700 unsigned long addr, int *rss)
795 { 701 {
796 unsigned long vm_flags = dst_vma->vm_f !! 702 unsigned long vm_flags = vma->vm_flags;
797 pte_t orig_pte = ptep_get(src_pte); !! 703 pte_t pte = *src_pte;
798 pte_t pte = orig_pte; <<
799 struct folio *folio; <<
800 struct page *page; 704 struct page *page;
801 swp_entry_t entry = pte_to_swp_entry(o !! 705 swp_entry_t entry = pte_to_swp_entry(pte);
802 706
803 if (likely(!non_swap_entry(entry))) { 707 if (likely(!non_swap_entry(entry))) {
804 if (swap_duplicate(entry) < 0) 708 if (swap_duplicate(entry) < 0)
805 return -EIO; !! 709 return entry.val;
806 710
807 /* make sure dst_mm is on swap 711 /* make sure dst_mm is on swapoff's mmlist. */
808 if (unlikely(list_empty(&dst_m 712 if (unlikely(list_empty(&dst_mm->mmlist))) {
809 spin_lock(&mmlist_lock 713 spin_lock(&mmlist_lock);
810 if (list_empty(&dst_mm 714 if (list_empty(&dst_mm->mmlist))
811 list_add(&dst_ 715 list_add(&dst_mm->mmlist,
812 716 &src_mm->mmlist);
813 spin_unlock(&mmlist_lo 717 spin_unlock(&mmlist_lock);
814 } 718 }
815 /* Mark the swap entry as shar <<
816 if (pte_swp_exclusive(orig_pte <<
817 pte = pte_swp_clear_ex <<
818 set_pte_at(src_mm, add <<
819 } <<
820 rss[MM_SWAPENTS]++; 719 rss[MM_SWAPENTS]++;
821 } else if (is_migration_entry(entry)) 720 } else if (is_migration_entry(entry)) {
822 folio = pfn_swap_entry_folio(e !! 721 page = migration_entry_to_page(entry);
823 722
824 rss[mm_counter(folio)]++; !! 723 rss[mm_counter(page)]++;
825 724
826 if (!is_readable_migration_ent !! 725 if (is_write_migration_entry(entry) &&
827 is_cow_mapping 726 is_cow_mapping(vm_flags)) {
828 /* 727 /*
829 * COW mappings requir !! 728 * COW mappings require pages in both
830 * to be set to read. !! 729 * parent and child to be set to read.
831 * now shared. <<
832 */ 730 */
833 entry = make_readable_ !! 731 make_migration_entry_read(&entry);
834 <<
835 pte = swp_entry_to_pte 732 pte = swp_entry_to_pte(entry);
836 if (pte_swp_soft_dirty !! 733 if (pte_swp_soft_dirty(*src_pte))
837 pte = pte_swp_ 734 pte = pte_swp_mksoft_dirty(pte);
838 if (pte_swp_uffd_wp(or !! 735 if (pte_swp_uffd_wp(*src_pte))
839 pte = pte_swp_ 736 pte = pte_swp_mkuffd_wp(pte);
840 set_pte_at(src_mm, add 737 set_pte_at(src_mm, addr, src_pte, pte);
841 } 738 }
842 } else if (is_device_private_entry(ent 739 } else if (is_device_private_entry(entry)) {
843 page = pfn_swap_entry_to_page( !! 740 page = device_private_entry_to_page(entry);
844 folio = page_folio(page); <<
845 741
846 /* 742 /*
847 * Update rss count even for u 743 * Update rss count even for unaddressable pages, as
848 * they should treated just li 744 * they should treated just like normal pages in this
849 * respect. 745 * respect.
850 * 746 *
851 * We will likely want to have 747 * We will likely want to have some new rss counters
852 * for unaddressable pages, at 748 * for unaddressable pages, at some point. But for now
853 * keep things as they are. 749 * keep things as they are.
854 */ 750 */
855 folio_get(folio); !! 751 get_page(page);
856 rss[mm_counter(folio)]++; !! 752 rss[mm_counter(page)]++;
857 /* Cannot fail as these pages !! 753 page_dup_rmap(page, false);
858 folio_try_dup_anon_rmap_pte(fo <<
859 754
860 /* 755 /*
861 * We do not preserve soft-dir 756 * We do not preserve soft-dirty information, because so
862 * far, checkpoint/restore is 757 * far, checkpoint/restore is the only feature that
863 * requires that. And checkpoi 758 * requires that. And checkpoint/restore does not work
864 * when a device driver is inv 759 * when a device driver is involved (you cannot easily
865 * save and restore device dri 760 * save and restore device driver state).
866 */ 761 */
867 if (is_writable_device_private !! 762 if (is_write_device_private_entry(entry) &&
868 is_cow_mapping(vm_flags)) 763 is_cow_mapping(vm_flags)) {
869 entry = make_readable_ !! 764 make_device_private_entry_read(&entry);
870 <<
871 pte = swp_entry_to_pte 765 pte = swp_entry_to_pte(entry);
872 if (pte_swp_uffd_wp(or !! 766 if (pte_swp_uffd_wp(*src_pte))
873 pte = pte_swp_ 767 pte = pte_swp_mkuffd_wp(pte);
874 set_pte_at(src_mm, add 768 set_pte_at(src_mm, addr, src_pte, pte);
875 } 769 }
876 } else if (is_device_exclusive_entry(e <<
877 /* <<
878 * Make device exclusive entri <<
879 * original entry then copying <<
880 * exclusive entries currently <<
881 * (ie. COW) mappings. <<
882 */ <<
883 VM_BUG_ON(!is_cow_mapping(src_ <<
884 if (try_restore_exclusive_pte( <<
885 return -EBUSY; <<
886 return -ENOENT; <<
887 } else if (is_pte_marker_entry(entry)) <<
888 pte_marker marker = copy_pte_m <<
889 <<
890 if (marker) <<
891 set_pte_at(dst_mm, add <<
892 make_pte_ma <<
893 return 0; <<
894 } 770 }
895 if (!userfaultfd_wp(dst_vma)) <<
896 pte = pte_swp_clear_uffd_wp(pt <<
897 set_pte_at(dst_mm, addr, dst_pte, pte) 771 set_pte_at(dst_mm, addr, dst_pte, pte);
898 return 0; 772 return 0;
899 } 773 }
900 774
901 /* 775 /*
902 * Copy a present and normal page. !! 776 * Copy a present and normal page if necessary.
903 * 777 *
904 * NOTE! The usual case is that this isn't req !! 778 * NOTE! The usual case is that this doesn't need to do
905 * instead, the caller can just increase the p !! 779 * anything, and can just return a positive value. That
906 * and re-use the pte the traditional way. !! 780 * will let the caller know that it can just increase
>> 781 * the page refcount and re-use the pte the traditional
>> 782 * way.
>> 783 *
>> 784 * But _if_ we need to copy it because it needs to be
>> 785 * pinned in the parent (and the child should get its own
>> 786 * copy rather than just a reference to the same page),
>> 787 * we'll do that here and return zero to let the caller
>> 788 * know we're done.
907 * 789 *
908 * And if we need a pre-allocated page but don 790 * And if we need a pre-allocated page but don't yet have
909 * one, return a negative error to let the pre 791 * one, return a negative error to let the preallocation
910 * code know so that it can do so outside the 792 * code know so that it can do so outside the page table
911 * lock. 793 * lock.
912 */ 794 */
913 static inline int 795 static inline int
914 copy_present_page(struct vm_area_struct *dst_v 796 copy_present_page(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
915 pte_t *dst_pte, pte_t *src_p 797 pte_t *dst_pte, pte_t *src_pte, unsigned long addr, int *rss,
916 struct folio **prealloc, str !! 798 struct page **prealloc, pte_t pte, struct page *page)
917 { 799 {
918 struct folio *new_folio; !! 800 struct mm_struct *src_mm = src_vma->vm_mm;
919 pte_t pte; !! 801 struct page *new_page;
>> 802
>> 803 if (!is_cow_mapping(src_vma->vm_flags))
>> 804 return 1;
920 805
921 new_folio = *prealloc; !! 806 /*
922 if (!new_folio) !! 807 * What we want to do is to check whether this page may
>> 808 * have been pinned by the parent process. If so,
>> 809 * instead of wrprotect the pte on both sides, we copy
>> 810 * the page immediately so that we'll always guarantee
>> 811 * the pinned page won't be randomly replaced in the
>> 812 * future.
>> 813 *
>> 814 * The page pinning checks are just "has this mm ever
>> 815 * seen pinning", along with the (inexact) check of
>> 816 * the page count. That might give false positives for
>> 817 * for pinning, but it will work correctly.
>> 818 */
>> 819 if (likely(!atomic_read(&src_mm->has_pinned)))
>> 820 return 1;
>> 821 if (likely(!page_maybe_dma_pinned(page)))
>> 822 return 1;
>> 823
>> 824 new_page = *prealloc;
>> 825 if (!new_page)
923 return -EAGAIN; 826 return -EAGAIN;
924 827
925 /* 828 /*
926 * We have a prealloc page, all good! 829 * We have a prealloc page, all good! Take it
927 * over and copy the page & arm it. 830 * over and copy the page & arm it.
928 */ 831 */
929 <<
930 if (copy_mc_user_highpage(&new_folio-> <<
931 return -EHWPOISON; <<
932 <<
933 *prealloc = NULL; 832 *prealloc = NULL;
934 __folio_mark_uptodate(new_folio); !! 833 copy_user_highpage(new_page, page, addr, src_vma);
935 folio_add_new_anon_rmap(new_folio, dst !! 834 __SetPageUptodate(new_page);
936 folio_add_lru_vma(new_folio, dst_vma); !! 835 page_add_new_anon_rmap(new_page, dst_vma, addr, false);
937 rss[MM_ANONPAGES]++; !! 836 lru_cache_add_inactive_or_unevictable(new_page, dst_vma);
>> 837 rss[mm_counter(new_page)]++;
938 838
939 /* All done, just insert the new page 839 /* All done, just insert the new page copy in the child */
940 pte = mk_pte(&new_folio->page, dst_vma !! 840 pte = mk_pte(new_page, dst_vma->vm_page_prot);
941 pte = maybe_mkwrite(pte_mkdirty(pte), 841 pte = maybe_mkwrite(pte_mkdirty(pte), dst_vma);
942 if (userfaultfd_pte_wp(dst_vma, ptep_g <<
943 /* Uffd-wp needs to be deliver <<
944 pte = pte_mkuffd_wp(pte); <<
945 set_pte_at(dst_vma->vm_mm, addr, dst_p 842 set_pte_at(dst_vma->vm_mm, addr, dst_pte, pte);
946 return 0; 843 return 0;
947 } 844 }
948 845
949 static __always_inline void __copy_present_pte <<
950 struct vm_area_struct *src_vma <<
951 pte_t pte, unsigned long addr, <<
952 { <<
953 struct mm_struct *src_mm = src_vma->vm <<
954 <<
955 /* If it's a COW mapping, write protec <<
956 if (is_cow_mapping(src_vma->vm_flags) <<
957 wrprotect_ptes(src_mm, addr, s <<
958 pte = pte_wrprotect(pte); <<
959 } <<
960 <<
961 /* If it's a shared mapping, mark it c <<
962 if (src_vma->vm_flags & VM_SHARED) <<
963 pte = pte_mkclean(pte); <<
964 pte = pte_mkold(pte); <<
965 <<
966 if (!userfaultfd_wp(dst_vma)) <<
967 pte = pte_clear_uffd_wp(pte); <<
968 <<
969 set_ptes(dst_vma->vm_mm, addr, dst_pte <<
970 } <<
971 <<
972 /* 846 /*
973 * Copy one present PTE, trying to batch-proce !! 847 * Copy one pte. Returns 0 if succeeded, or -EAGAIN if one preallocated page
974 * consecutive pages of the same folio by copy !! 848 * is required to copy this pte.
975 * <<
976 * Returns -EAGAIN if one preallocated page is <<
977 * Otherwise, returns the number of copied PTE <<
978 */ 849 */
979 static inline int 850 static inline int
980 copy_present_ptes(struct vm_area_struct *dst_v !! 851 copy_present_pte(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
981 pte_t *dst_pte, pte_t *src_pt !! 852 pte_t *dst_pte, pte_t *src_pte, unsigned long addr, int *rss,
982 int max_nr, int *rss, struct !! 853 struct page **prealloc)
983 { 854 {
>> 855 struct mm_struct *src_mm = src_vma->vm_mm;
>> 856 unsigned long vm_flags = src_vma->vm_flags;
>> 857 pte_t pte = *src_pte;
984 struct page *page; 858 struct page *page;
985 struct folio *folio; <<
986 bool any_writable; <<
987 fpb_t flags = 0; <<
988 int err, nr; <<
989 859
990 page = vm_normal_page(src_vma, addr, p 860 page = vm_normal_page(src_vma, addr, pte);
991 if (unlikely(!page)) !! 861 if (page) {
992 goto copy_pte; !! 862 int retval;
993 863
994 folio = page_folio(page); !! 864 retval = copy_present_page(dst_vma, src_vma, dst_pte, src_pte,
>> 865 addr, rss, prealloc, pte, page);
>> 866 if (retval <= 0)
>> 867 return retval;
995 868
996 /* !! 869 get_page(page);
997 * If we likely have to copy, just don !! 870 page_dup_rmap(page, false);
998 * sure that the common "small folio" !! 871 rss[mm_counter(page)]++;
999 * by keeping the batching logic separ <<
1000 */ <<
1001 if (unlikely(!*prealloc && folio_test <<
1002 if (src_vma->vm_flags & VM_SH <<
1003 flags |= FPB_IGNORE_D <<
1004 if (!vma_soft_dirty_enabled(s <<
1005 flags |= FPB_IGNORE_S <<
1006 <<
1007 nr = folio_pte_batch(folio, a <<
1008 &any_wri <<
1009 folio_ref_add(folio, nr); <<
1010 if (folio_test_anon(folio)) { <<
1011 if (unlikely(folio_tr <<
1012 <<
1013 folio_ref_sub <<
1014 return -EAGAI <<
1015 } <<
1016 rss[MM_ANONPAGES] += <<
1017 VM_WARN_ON_FOLIO(Page <<
1018 } else { <<
1019 folio_dup_file_rmap_p <<
1020 rss[mm_counter_file(f <<
1021 } <<
1022 if (any_writable) <<
1023 pte = pte_mkwrite(pte <<
1024 __copy_present_ptes(dst_vma, <<
1025 addr, nr) <<
1026 return nr; <<
1027 } 872 }
1028 873
1029 folio_get(folio); !! 874 /*
1030 if (folio_test_anon(folio)) { !! 875 * If it's a COW mapping, write protect it both
1031 /* !! 876 * in the parent and the child
1032 * If this page may have been !! 877 */
1033 * copy the page immediately !! 878 if (is_cow_mapping(vm_flags) && pte_write(pte)) {
1034 * guarantee the pinned page !! 879 ptep_set_wrprotect(src_mm, addr, src_pte);
1035 * future. !! 880 pte = pte_wrprotect(pte);
1036 */ <<
1037 if (unlikely(folio_try_dup_an <<
1038 /* Page may be pinned <<
1039 folio_put(folio); <<
1040 err = copy_present_pa <<
1041 <<
1042 return err ? err : 1; <<
1043 } <<
1044 rss[MM_ANONPAGES]++; <<
1045 VM_WARN_ON_FOLIO(PageAnonExcl <<
1046 } else { <<
1047 folio_dup_file_rmap_pte(folio <<
1048 rss[mm_counter_file(folio)]++ <<
1049 } 881 }
1050 882
1051 copy_pte: !! 883 /*
1052 __copy_present_ptes(dst_vma, src_vma, !! 884 * If it's a shared mapping, mark it clean in
1053 return 1; !! 885 * the child
>> 886 */
>> 887 if (vm_flags & VM_SHARED)
>> 888 pte = pte_mkclean(pte);
>> 889 pte = pte_mkold(pte);
>> 890
>> 891 /*
>> 892 * Make sure the _PAGE_UFFD_WP bit is cleared if the new VMA
>> 893 * does not have the VM_UFFD_WP, which means that the uffd
>> 894 * fork event is not enabled.
>> 895 */
>> 896 if (!(vm_flags & VM_UFFD_WP))
>> 897 pte = pte_clear_uffd_wp(pte);
>> 898
>> 899 set_pte_at(dst_vma->vm_mm, addr, dst_pte, pte);
>> 900 return 0;
1054 } 901 }
1055 902
1056 static inline struct folio *folio_prealloc(st !! 903 static inline struct page *
1057 struct vm_area_struct *vma, u !! 904 page_copy_prealloc(struct mm_struct *src_mm, struct vm_area_struct *vma,
>> 905 unsigned long addr)
1058 { 906 {
1059 struct folio *new_folio; !! 907 struct page *new_page;
1060 <<
1061 if (need_zero) <<
1062 new_folio = vma_alloc_zeroed_ <<
1063 else <<
1064 new_folio = vma_alloc_folio(G <<
1065 a <<
1066 908
1067 if (!new_folio) !! 909 new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, addr);
>> 910 if (!new_page)
1068 return NULL; 911 return NULL;
1069 912
1070 if (mem_cgroup_charge(new_folio, src_ !! 913 if (mem_cgroup_charge(new_page, src_mm, GFP_KERNEL)) {
1071 folio_put(new_folio); !! 914 put_page(new_page);
1072 return NULL; 915 return NULL;
1073 } 916 }
1074 folio_throttle_swaprate(new_folio, GF !! 917 cgroup_throttle_swaprate(new_page, GFP_KERNEL);
1075 918
1076 return new_folio; !! 919 return new_page;
1077 } 920 }
1078 921
1079 static int 922 static int
1080 copy_pte_range(struct vm_area_struct *dst_vma 923 copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
1081 pmd_t *dst_pmd, pmd_t *src_pmd 924 pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr,
1082 unsigned long end) 925 unsigned long end)
1083 { 926 {
1084 struct mm_struct *dst_mm = dst_vma->v 927 struct mm_struct *dst_mm = dst_vma->vm_mm;
1085 struct mm_struct *src_mm = src_vma->v 928 struct mm_struct *src_mm = src_vma->vm_mm;
1086 pte_t *orig_src_pte, *orig_dst_pte; 929 pte_t *orig_src_pte, *orig_dst_pte;
1087 pte_t *src_pte, *dst_pte; 930 pte_t *src_pte, *dst_pte;
1088 pte_t ptent; <<
1089 spinlock_t *src_ptl, *dst_ptl; 931 spinlock_t *src_ptl, *dst_ptl;
1090 int progress, max_nr, ret = 0; !! 932 int progress, ret = 0;
1091 int rss[NR_MM_COUNTERS]; 933 int rss[NR_MM_COUNTERS];
1092 swp_entry_t entry = (swp_entry_t){0}; 934 swp_entry_t entry = (swp_entry_t){0};
1093 struct folio *prealloc = NULL; !! 935 struct page *prealloc = NULL;
1094 int nr; <<
1095 936
1096 again: 937 again:
1097 progress = 0; 938 progress = 0;
1098 init_rss_vec(rss); 939 init_rss_vec(rss);
1099 940
1100 /* <<
1101 * copy_pmd_range()'s prior pmd_none_ <<
1102 * error handling here, assume that e <<
1103 * protects anon from unexpected THP <<
1104 * protected by mmap_lock-less collap <<
1105 * (whereas vma_needs_copy() skips ar <<
1106 * can remove such assumptions later, <<
1107 */ <<
1108 dst_pte = pte_alloc_map_lock(dst_mm, 941 dst_pte = pte_alloc_map_lock(dst_mm, dst_pmd, addr, &dst_ptl);
1109 if (!dst_pte) { 942 if (!dst_pte) {
1110 ret = -ENOMEM; 943 ret = -ENOMEM;
1111 goto out; 944 goto out;
1112 } 945 }
1113 src_pte = pte_offset_map_nolock(src_m !! 946 src_pte = pte_offset_map(src_pmd, addr);
1114 if (!src_pte) { !! 947 src_ptl = pte_lockptr(src_mm, src_pmd);
1115 pte_unmap_unlock(dst_pte, dst <<
1116 /* ret == 0 */ <<
1117 goto out; <<
1118 } <<
1119 spin_lock_nested(src_ptl, SINGLE_DEPT 948 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
1120 orig_src_pte = src_pte; 949 orig_src_pte = src_pte;
1121 orig_dst_pte = dst_pte; 950 orig_dst_pte = dst_pte;
1122 arch_enter_lazy_mmu_mode(); 951 arch_enter_lazy_mmu_mode();
1123 952
1124 do { 953 do {
1125 nr = 1; <<
1126 <<
1127 /* 954 /*
1128 * We are holding two locks a 955 * We are holding two locks at this point - either of them
1129 * could generate latencies i 956 * could generate latencies in another task on another CPU.
1130 */ 957 */
1131 if (progress >= 32) { 958 if (progress >= 32) {
1132 progress = 0; 959 progress = 0;
1133 if (need_resched() || 960 if (need_resched() ||
1134 spin_needbreak(sr 961 spin_needbreak(src_ptl) || spin_needbreak(dst_ptl))
1135 break; 962 break;
1136 } 963 }
1137 ptent = ptep_get(src_pte); !! 964 if (pte_none(*src_pte)) {
1138 if (pte_none(ptent)) { <<
1139 progress++; 965 progress++;
1140 continue; 966 continue;
1141 } 967 }
1142 if (unlikely(!pte_present(pte !! 968 if (unlikely(!pte_present(*src_pte))) {
1143 ret = copy_nonpresent !! 969 entry.val = copy_nonpresent_pte(dst_mm, src_mm,
1144 !! 970 dst_pte, src_pte,
1145 !! 971 src_vma, addr, rss);
1146 !! 972 if (entry.val)
1147 if (ret == -EIO) { <<
1148 entry = pte_t <<
1149 break; 973 break;
1150 } else if (ret == -EB !! 974 progress += 8;
1151 break; !! 975 continue;
1152 } else if (!ret) { <<
1153 progress += 8 <<
1154 continue; <<
1155 } <<
1156 ptent = ptep_get(src_ <<
1157 VM_WARN_ON_ONCE(!pte_ <<
1158 <<
1159 /* <<
1160 * Device exclusive e <<
1161 * the now present pt <<
1162 */ <<
1163 WARN_ON_ONCE(ret != - <<
1164 } 976 }
1165 /* copy_present_ptes() will c !! 977 /* copy_present_pte() will clear `*prealloc' if consumed */
1166 max_nr = (end - addr) / PAGE_ !! 978 ret = copy_present_pte(dst_vma, src_vma, dst_pte, src_pte,
1167 ret = copy_present_ptes(dst_v !! 979 addr, rss, &prealloc);
1168 ptent <<
1169 /* 980 /*
1170 * If we need a pre-allocated 981 * If we need a pre-allocated page for this pte, drop the
1171 * locks, allocate, and try a 982 * locks, allocate, and try again.
1172 * If copy failed due to hwpo <<
1173 */ 983 */
1174 if (unlikely(ret == -EAGAIN | !! 984 if (unlikely(ret == -EAGAIN))
1175 break; 985 break;
1176 if (unlikely(prealloc)) { 986 if (unlikely(prealloc)) {
1177 /* 987 /*
1178 * pre-alloc page can 988 * pre-alloc page cannot be reused by next time so as
1179 * to strictly follow 989 * to strictly follow mempolicy (e.g., alloc_page_vma()
1180 * will allocate page 990 * will allocate page according to address). This
1181 * could only happen 991 * could only happen if one pinned pte changed.
1182 */ 992 */
1183 folio_put(prealloc); !! 993 put_page(prealloc);
1184 prealloc = NULL; 994 prealloc = NULL;
1185 } 995 }
1186 nr = ret; !! 996 progress += 8;
1187 progress += 8 * nr; !! 997 } while (dst_pte++, src_pte++, addr += PAGE_SIZE, addr != end);
1188 } while (dst_pte += nr, src_pte += nr <<
1189 addr != end); <<
1190 998
1191 arch_leave_lazy_mmu_mode(); 999 arch_leave_lazy_mmu_mode();
1192 pte_unmap_unlock(orig_src_pte, src_pt !! 1000 spin_unlock(src_ptl);
>> 1001 pte_unmap(orig_src_pte);
1193 add_mm_rss_vec(dst_mm, rss); 1002 add_mm_rss_vec(dst_mm, rss);
1194 pte_unmap_unlock(orig_dst_pte, dst_pt 1003 pte_unmap_unlock(orig_dst_pte, dst_ptl);
1195 cond_resched(); 1004 cond_resched();
1196 1005
1197 if (ret == -EIO) { !! 1006 if (entry.val) {
1198 VM_WARN_ON_ONCE(!entry.val); <<
1199 if (add_swap_count_continuati 1007 if (add_swap_count_continuation(entry, GFP_KERNEL) < 0) {
1200 ret = -ENOMEM; 1008 ret = -ENOMEM;
1201 goto out; 1009 goto out;
1202 } 1010 }
1203 entry.val = 0; 1011 entry.val = 0;
1204 } else if (ret == -EBUSY || unlikely( !! 1012 } else if (ret) {
1205 goto out; !! 1013 WARN_ON_ONCE(ret != -EAGAIN);
1206 } else if (ret == -EAGAIN) { !! 1014 prealloc = page_copy_prealloc(src_mm, src_vma, addr);
1207 prealloc = folio_prealloc(src <<
1208 if (!prealloc) 1015 if (!prealloc)
1209 return -ENOMEM; 1016 return -ENOMEM;
1210 } else if (ret < 0) { !! 1017 /* We've captured and resolved the error. Reset, try again. */
1211 VM_WARN_ON_ONCE(1); !! 1018 ret = 0;
1212 } 1019 }
1213 <<
1214 /* We've captured and resolved the er <<
1215 ret = 0; <<
1216 <<
1217 if (addr != end) 1020 if (addr != end)
1218 goto again; 1021 goto again;
1219 out: 1022 out:
1220 if (unlikely(prealloc)) 1023 if (unlikely(prealloc))
1221 folio_put(prealloc); !! 1024 put_page(prealloc);
1222 return ret; 1025 return ret;
1223 } 1026 }
1224 1027
1225 static inline int 1028 static inline int
1226 copy_pmd_range(struct vm_area_struct *dst_vma 1029 copy_pmd_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
1227 pud_t *dst_pud, pud_t *src_pud 1030 pud_t *dst_pud, pud_t *src_pud, unsigned long addr,
1228 unsigned long end) 1031 unsigned long end)
1229 { 1032 {
1230 struct mm_struct *dst_mm = dst_vma->v 1033 struct mm_struct *dst_mm = dst_vma->vm_mm;
1231 struct mm_struct *src_mm = src_vma->v 1034 struct mm_struct *src_mm = src_vma->vm_mm;
1232 pmd_t *src_pmd, *dst_pmd; 1035 pmd_t *src_pmd, *dst_pmd;
1233 unsigned long next; 1036 unsigned long next;
1234 1037
1235 dst_pmd = pmd_alloc(dst_mm, dst_pud, 1038 dst_pmd = pmd_alloc(dst_mm, dst_pud, addr);
1236 if (!dst_pmd) 1039 if (!dst_pmd)
1237 return -ENOMEM; 1040 return -ENOMEM;
1238 src_pmd = pmd_offset(src_pud, addr); 1041 src_pmd = pmd_offset(src_pud, addr);
1239 do { 1042 do {
1240 next = pmd_addr_end(addr, end 1043 next = pmd_addr_end(addr, end);
1241 if (is_swap_pmd(*src_pmd) || 1044 if (is_swap_pmd(*src_pmd) || pmd_trans_huge(*src_pmd)
1242 || pmd_devmap(*src_pm 1045 || pmd_devmap(*src_pmd)) {
1243 int err; 1046 int err;
1244 VM_BUG_ON_VMA(next-ad 1047 VM_BUG_ON_VMA(next-addr != HPAGE_PMD_SIZE, src_vma);
1245 err = copy_huge_pmd(d !! 1048 err = copy_huge_pmd(dst_mm, src_mm,
1246 a !! 1049 dst_pmd, src_pmd, addr, src_vma);
1247 if (err == -ENOMEM) 1050 if (err == -ENOMEM)
1248 return -ENOME 1051 return -ENOMEM;
1249 if (!err) 1052 if (!err)
1250 continue; 1053 continue;
1251 /* fall through */ 1054 /* fall through */
1252 } 1055 }
1253 if (pmd_none_or_clear_bad(src 1056 if (pmd_none_or_clear_bad(src_pmd))
1254 continue; 1057 continue;
1255 if (copy_pte_range(dst_vma, s 1058 if (copy_pte_range(dst_vma, src_vma, dst_pmd, src_pmd,
1256 addr, next 1059 addr, next))
1257 return -ENOMEM; 1060 return -ENOMEM;
1258 } while (dst_pmd++, src_pmd++, addr = 1061 } while (dst_pmd++, src_pmd++, addr = next, addr != end);
1259 return 0; 1062 return 0;
1260 } 1063 }
1261 1064
1262 static inline int 1065 static inline int
1263 copy_pud_range(struct vm_area_struct *dst_vma 1066 copy_pud_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
1264 p4d_t *dst_p4d, p4d_t *src_p4d 1067 p4d_t *dst_p4d, p4d_t *src_p4d, unsigned long addr,
1265 unsigned long end) 1068 unsigned long end)
1266 { 1069 {
1267 struct mm_struct *dst_mm = dst_vma->v 1070 struct mm_struct *dst_mm = dst_vma->vm_mm;
1268 struct mm_struct *src_mm = src_vma->v 1071 struct mm_struct *src_mm = src_vma->vm_mm;
1269 pud_t *src_pud, *dst_pud; 1072 pud_t *src_pud, *dst_pud;
1270 unsigned long next; 1073 unsigned long next;
1271 1074
1272 dst_pud = pud_alloc(dst_mm, dst_p4d, 1075 dst_pud = pud_alloc(dst_mm, dst_p4d, addr);
1273 if (!dst_pud) 1076 if (!dst_pud)
1274 return -ENOMEM; 1077 return -ENOMEM;
1275 src_pud = pud_offset(src_p4d, addr); 1078 src_pud = pud_offset(src_p4d, addr);
1276 do { 1079 do {
1277 next = pud_addr_end(addr, end 1080 next = pud_addr_end(addr, end);
1278 if (pud_trans_huge(*src_pud) 1081 if (pud_trans_huge(*src_pud) || pud_devmap(*src_pud)) {
1279 int err; 1082 int err;
1280 1083
1281 VM_BUG_ON_VMA(next-ad 1084 VM_BUG_ON_VMA(next-addr != HPAGE_PUD_SIZE, src_vma);
1282 err = copy_huge_pud(d 1085 err = copy_huge_pud(dst_mm, src_mm,
1283 d 1086 dst_pud, src_pud, addr, src_vma);
1284 if (err == -ENOMEM) 1087 if (err == -ENOMEM)
1285 return -ENOME 1088 return -ENOMEM;
1286 if (!err) 1089 if (!err)
1287 continue; 1090 continue;
1288 /* fall through */ 1091 /* fall through */
1289 } 1092 }
1290 if (pud_none_or_clear_bad(src 1093 if (pud_none_or_clear_bad(src_pud))
1291 continue; 1094 continue;
1292 if (copy_pmd_range(dst_vma, s 1095 if (copy_pmd_range(dst_vma, src_vma, dst_pud, src_pud,
1293 addr, next 1096 addr, next))
1294 return -ENOMEM; 1097 return -ENOMEM;
1295 } while (dst_pud++, src_pud++, addr = 1098 } while (dst_pud++, src_pud++, addr = next, addr != end);
1296 return 0; 1099 return 0;
1297 } 1100 }
1298 1101
1299 static inline int 1102 static inline int
1300 copy_p4d_range(struct vm_area_struct *dst_vma 1103 copy_p4d_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
1301 pgd_t *dst_pgd, pgd_t *src_pgd 1104 pgd_t *dst_pgd, pgd_t *src_pgd, unsigned long addr,
1302 unsigned long end) 1105 unsigned long end)
1303 { 1106 {
1304 struct mm_struct *dst_mm = dst_vma->v 1107 struct mm_struct *dst_mm = dst_vma->vm_mm;
1305 p4d_t *src_p4d, *dst_p4d; 1108 p4d_t *src_p4d, *dst_p4d;
1306 unsigned long next; 1109 unsigned long next;
1307 1110
1308 dst_p4d = p4d_alloc(dst_mm, dst_pgd, 1111 dst_p4d = p4d_alloc(dst_mm, dst_pgd, addr);
1309 if (!dst_p4d) 1112 if (!dst_p4d)
1310 return -ENOMEM; 1113 return -ENOMEM;
1311 src_p4d = p4d_offset(src_pgd, addr); 1114 src_p4d = p4d_offset(src_pgd, addr);
1312 do { 1115 do {
1313 next = p4d_addr_end(addr, end 1116 next = p4d_addr_end(addr, end);
1314 if (p4d_none_or_clear_bad(src 1117 if (p4d_none_or_clear_bad(src_p4d))
1315 continue; 1118 continue;
1316 if (copy_pud_range(dst_vma, s 1119 if (copy_pud_range(dst_vma, src_vma, dst_p4d, src_p4d,
1317 addr, next 1120 addr, next))
1318 return -ENOMEM; 1121 return -ENOMEM;
1319 } while (dst_p4d++, src_p4d++, addr = 1122 } while (dst_p4d++, src_p4d++, addr = next, addr != end);
1320 return 0; 1123 return 0;
1321 } 1124 }
1322 1125
1323 /* <<
1324 * Return true if the vma needs to copy the p <<
1325 * false when we can speed up fork() by allow <<
1326 * when the child accesses the memory range. <<
1327 */ <<
1328 static bool <<
1329 vma_needs_copy(struct vm_area_struct *dst_vma <<
1330 { <<
1331 /* <<
1332 * Always copy pgtables when dst_vma <<
1333 * file-backed (e.g. shmem). Because <<
1334 * contains uffd-wp protection inform <<
1335 * retrieve from page cache, and skip <<
1336 */ <<
1337 if (userfaultfd_wp(dst_vma)) <<
1338 return true; <<
1339 <<
1340 if (src_vma->vm_flags & (VM_PFNMAP | <<
1341 return true; <<
1342 <<
1343 if (src_vma->anon_vma) <<
1344 return true; <<
1345 <<
1346 /* <<
1347 * Don't copy ptes where a page fault <<
1348 * becomes much lighter when there ar <<
1349 * mappings. The tradeoff is that cop <<
1350 * than faulting. <<
1351 */ <<
1352 return false; <<
1353 } <<
1354 <<
1355 int 1126 int
1356 copy_page_range(struct vm_area_struct *dst_vm 1127 copy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma)
1357 { 1128 {
1358 pgd_t *src_pgd, *dst_pgd; 1129 pgd_t *src_pgd, *dst_pgd;
1359 unsigned long next; 1130 unsigned long next;
1360 unsigned long addr = src_vma->vm_star 1131 unsigned long addr = src_vma->vm_start;
1361 unsigned long end = src_vma->vm_end; 1132 unsigned long end = src_vma->vm_end;
1362 struct mm_struct *dst_mm = dst_vma->v 1133 struct mm_struct *dst_mm = dst_vma->vm_mm;
1363 struct mm_struct *src_mm = src_vma->v 1134 struct mm_struct *src_mm = src_vma->vm_mm;
1364 struct mmu_notifier_range range; 1135 struct mmu_notifier_range range;
1365 bool is_cow; 1136 bool is_cow;
1366 int ret; 1137 int ret;
1367 1138
1368 if (!vma_needs_copy(dst_vma, src_vma) !! 1139 /*
>> 1140 * Don't copy ptes where a page fault will fill them correctly.
>> 1141 * Fork becomes much lighter when there are big shared or private
>> 1142 * readonly mappings. The tradeoff is that copy_page_range is more
>> 1143 * efficient than faulting.
>> 1144 */
>> 1145 if (!(src_vma->vm_flags & (VM_HUGETLB | VM_PFNMAP | VM_MIXEDMAP)) &&
>> 1146 !src_vma->anon_vma)
1369 return 0; 1147 return 0;
1370 1148
1371 if (is_vm_hugetlb_page(src_vma)) 1149 if (is_vm_hugetlb_page(src_vma))
1372 return copy_hugetlb_page_rang !! 1150 return copy_hugetlb_page_range(dst_mm, src_mm, src_vma);
1373 1151
1374 if (unlikely(src_vma->vm_flags & VM_P 1152 if (unlikely(src_vma->vm_flags & VM_PFNMAP)) {
1375 /* 1153 /*
1376 * We do not free on error ca 1154 * We do not free on error cases below as remove_vma
1377 * gets called on error from 1155 * gets called on error from higher level routine
1378 */ 1156 */
1379 ret = track_pfn_copy(src_vma) 1157 ret = track_pfn_copy(src_vma);
1380 if (ret) 1158 if (ret)
1381 return ret; 1159 return ret;
1382 } 1160 }
1383 1161
1384 /* 1162 /*
1385 * We need to invalidate the secondar 1163 * We need to invalidate the secondary MMU mappings only when
1386 * there could be a permission downgr 1164 * there could be a permission downgrade on the ptes of the
1387 * parent mm. And a permission downgr 1165 * parent mm. And a permission downgrade will only happen if
1388 * is_cow_mapping() returns true. 1166 * is_cow_mapping() returns true.
1389 */ 1167 */
1390 is_cow = is_cow_mapping(src_vma->vm_f 1168 is_cow = is_cow_mapping(src_vma->vm_flags);
1391 1169
1392 if (is_cow) { 1170 if (is_cow) {
1393 mmu_notifier_range_init(&rang 1171 mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_PAGE,
1394 0, sr !! 1172 0, src_vma, src_mm, addr, end);
1395 mmu_notifier_invalidate_range 1173 mmu_notifier_invalidate_range_start(&range);
1396 /* 1174 /*
1397 * Disabling preemption is no 1175 * Disabling preemption is not needed for the write side, as
1398 * the read side doesn't spin 1176 * the read side doesn't spin, but goes to the mmap_lock.
1399 * 1177 *
1400 * Use the raw variant of the 1178 * Use the raw variant of the seqcount_t write API to avoid
1401 * lockdep complaining about 1179 * lockdep complaining about preemptibility.
1402 */ 1180 */
1403 vma_assert_write_locked(src_v !! 1181 mmap_assert_write_locked(src_mm);
1404 raw_write_seqcount_begin(&src 1182 raw_write_seqcount_begin(&src_mm->write_protect_seq);
1405 } 1183 }
1406 1184
1407 ret = 0; 1185 ret = 0;
1408 dst_pgd = pgd_offset(dst_mm, addr); 1186 dst_pgd = pgd_offset(dst_mm, addr);
1409 src_pgd = pgd_offset(src_mm, addr); 1187 src_pgd = pgd_offset(src_mm, addr);
1410 do { 1188 do {
1411 next = pgd_addr_end(addr, end 1189 next = pgd_addr_end(addr, end);
1412 if (pgd_none_or_clear_bad(src 1190 if (pgd_none_or_clear_bad(src_pgd))
1413 continue; 1191 continue;
1414 if (unlikely(copy_p4d_range(d 1192 if (unlikely(copy_p4d_range(dst_vma, src_vma, dst_pgd, src_pgd,
1415 a 1193 addr, next))) {
1416 untrack_pfn_clear(dst <<
1417 ret = -ENOMEM; 1194 ret = -ENOMEM;
1418 break; 1195 break;
1419 } 1196 }
1420 } while (dst_pgd++, src_pgd++, addr = 1197 } while (dst_pgd++, src_pgd++, addr = next, addr != end);
1421 1198
1422 if (is_cow) { 1199 if (is_cow) {
1423 raw_write_seqcount_end(&src_m 1200 raw_write_seqcount_end(&src_mm->write_protect_seq);
1424 mmu_notifier_invalidate_range 1201 mmu_notifier_invalidate_range_end(&range);
1425 } 1202 }
1426 return ret; 1203 return ret;
1427 } 1204 }
1428 1205
1429 /* Whether we should zap all COWed (private) <<
1430 static inline bool should_zap_cows(struct zap <<
1431 { <<
1432 /* By default, zap all pages */ <<
1433 if (!details) <<
1434 return true; <<
1435 <<
1436 /* Or, we zap COWed pages only if the <<
1437 return details->even_cows; <<
1438 } <<
1439 <<
1440 /* Decides whether we should zap this folio w <<
1441 static inline bool should_zap_folio(struct za <<
1442 struct fo <<
1443 { <<
1444 /* If we can make a decision without <<
1445 if (should_zap_cows(details)) <<
1446 return true; <<
1447 <<
1448 /* Otherwise we should only zap non-a <<
1449 return !folio_test_anon(folio); <<
1450 } <<
1451 <<
1452 static inline bool zap_drop_file_uffd_wp(stru <<
1453 { <<
1454 if (!details) <<
1455 return false; <<
1456 <<
1457 return details->zap_flags & ZAP_FLAG_ <<
1458 } <<
1459 <<
1460 /* <<
1461 * This function makes sure that we'll replac <<
1462 * swap special pte marker when necessary. Mu <<
1463 */ <<
1464 static inline void <<
1465 zap_install_uffd_wp_if_needed(struct vm_area_ <<
1466 unsigned long a <<
1467 struct zap_deta <<
1468 { <<
1469 /* Zap on anonymous always means drop <<
1470 if (vma_is_anonymous(vma)) <<
1471 return; <<
1472 <<
1473 if (zap_drop_file_uffd_wp(details)) <<
1474 return; <<
1475 <<
1476 for (;;) { <<
1477 /* the PFN in the PTE is irre <<
1478 pte_install_uffd_wp_if_needed <<
1479 if (--nr == 0) <<
1480 break; <<
1481 pte++; <<
1482 addr += PAGE_SIZE; <<
1483 } <<
1484 } <<
1485 <<
1486 static __always_inline void zap_present_folio <<
1487 struct vm_area_struct *vma, s <<
1488 struct page *page, pte_t *pte <<
1489 unsigned long addr, struct za <<
1490 bool *force_flush, bool *forc <<
1491 { <<
1492 struct mm_struct *mm = tlb->mm; <<
1493 bool delay_rmap = false; <<
1494 <<
1495 if (!folio_test_anon(folio)) { <<
1496 ptent = get_and_clear_full_pt <<
1497 if (pte_dirty(ptent)) { <<
1498 folio_mark_dirty(foli <<
1499 if (tlb_delay_rmap(tl <<
1500 delay_rmap = <<
1501 *force_flush <<
1502 } <<
1503 } <<
1504 if (pte_young(ptent) && likel <<
1505 folio_mark_accessed(f <<
1506 rss[mm_counter(folio)] -= nr; <<
1507 } else { <<
1508 /* We don't need up-to-date a <<
1509 clear_full_ptes(mm, addr, pte <<
1510 rss[MM_ANONPAGES] -= nr; <<
1511 } <<
1512 /* Checking a single PTE in a batch i <<
1513 arch_check_zapped_pte(vma, ptent); <<
1514 tlb_remove_tlb_entries(tlb, pte, nr, <<
1515 if (unlikely(userfaultfd_pte_wp(vma, <<
1516 zap_install_uffd_wp_if_needed <<
1517 <<
1518 <<
1519 if (!delay_rmap) { <<
1520 folio_remove_rmap_ptes(folio, <<
1521 <<
1522 if (unlikely(folio_mapcount(f <<
1523 print_bad_pte(vma, ad <<
1524 } <<
1525 if (unlikely(__tlb_remove_folio_pages <<
1526 *force_flush = true; <<
1527 *force_break = true; <<
1528 } <<
1529 } <<
1530 <<
1531 /* <<
1532 * Zap or skip at least one present PTE, tryi <<
1533 * PTEs that map consecutive pages of the sam <<
1534 * <<
1535 * Returns the number of processed (skipped o <<
1536 */ <<
1537 static inline int zap_present_ptes(struct mmu <<
1538 struct vm_area_struct *vma, p <<
1539 unsigned int max_nr, unsigned <<
1540 struct zap_details *details, <<
1541 bool *force_break) <<
1542 { <<
1543 const fpb_t fpb_flags = FPB_IGNORE_DI <<
1544 struct mm_struct *mm = tlb->mm; <<
1545 struct folio *folio; <<
1546 struct page *page; <<
1547 int nr; <<
1548 <<
1549 page = vm_normal_page(vma, addr, pten <<
1550 if (!page) { <<
1551 /* We don't need up-to-date a <<
1552 ptep_get_and_clear_full(mm, a <<
1553 arch_check_zapped_pte(vma, pt <<
1554 tlb_remove_tlb_entry(tlb, pte <<
1555 if (userfaultfd_pte_wp(vma, p <<
1556 zap_install_uffd_wp_i <<
1557 <<
1558 ksm_might_unmap_zero_page(mm, <<
1559 return 1; <<
1560 } <<
1561 <<
1562 folio = page_folio(page); <<
1563 if (unlikely(!should_zap_folio(detail <<
1564 return 1; <<
1565 <<
1566 /* <<
1567 * Make sure that the common "small f <<
1568 * by keeping the batching logic sepa <<
1569 */ <<
1570 if (unlikely(folio_test_large(folio) <<
1571 nr = folio_pte_batch(folio, a <<
1572 NULL, NU <<
1573 <<
1574 zap_present_folio_ptes(tlb, v <<
1575 addr, <<
1576 force_ <<
1577 return nr; <<
1578 } <<
1579 zap_present_folio_ptes(tlb, vma, foli <<
1580 details, rss, <<
1581 return 1; <<
1582 } <<
1583 <<
1584 static unsigned long zap_pte_range(struct mmu 1206 static unsigned long zap_pte_range(struct mmu_gather *tlb,
1585 struct vm_are 1207 struct vm_area_struct *vma, pmd_t *pmd,
1586 unsigned long 1208 unsigned long addr, unsigned long end,
1587 struct zap_de 1209 struct zap_details *details)
1588 { 1210 {
1589 bool force_flush = false, force_break <<
1590 struct mm_struct *mm = tlb->mm; 1211 struct mm_struct *mm = tlb->mm;
>> 1212 int force_flush = 0;
1591 int rss[NR_MM_COUNTERS]; 1213 int rss[NR_MM_COUNTERS];
1592 spinlock_t *ptl; 1214 spinlock_t *ptl;
1593 pte_t *start_pte; 1215 pte_t *start_pte;
1594 pte_t *pte; 1216 pte_t *pte;
1595 swp_entry_t entry; 1217 swp_entry_t entry;
1596 int nr; <<
1597 1218
1598 tlb_change_page_size(tlb, PAGE_SIZE); 1219 tlb_change_page_size(tlb, PAGE_SIZE);
>> 1220 again:
1599 init_rss_vec(rss); 1221 init_rss_vec(rss);
1600 start_pte = pte = pte_offset_map_lock !! 1222 start_pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
1601 if (!pte) !! 1223 pte = start_pte;
1602 return addr; <<
1603 <<
1604 flush_tlb_batched_pending(mm); 1224 flush_tlb_batched_pending(mm);
1605 arch_enter_lazy_mmu_mode(); 1225 arch_enter_lazy_mmu_mode();
1606 do { 1226 do {
1607 pte_t ptent = ptep_get(pte); !! 1227 pte_t ptent = *pte;
1608 struct folio *folio; <<
1609 struct page *page; <<
1610 int max_nr; <<
1611 <<
1612 nr = 1; <<
1613 if (pte_none(ptent)) 1228 if (pte_none(ptent))
1614 continue; 1229 continue;
1615 1230
1616 if (need_resched()) 1231 if (need_resched())
1617 break; 1232 break;
1618 1233
1619 if (pte_present(ptent)) { 1234 if (pte_present(ptent)) {
1620 max_nr = (end - addr) !! 1235 struct page *page;
1621 nr = zap_present_ptes !! 1236
1622 !! 1237 page = vm_normal_page(vma, addr, ptent);
1623 !! 1238 if (unlikely(details) && page) {
1624 if (unlikely(force_br !! 1239 /*
1625 addr += nr * !! 1240 * unmap_shared_mapping_pages() wants to
>> 1241 * invalidate cache without truncating:
>> 1242 * unmap shared but keep private pages.
>> 1243 */
>> 1244 if (details->check_mapping &&
>> 1245 details->check_mapping != page_rmapping(page))
>> 1246 continue;
>> 1247 }
>> 1248 ptent = ptep_get_and_clear_full(mm, addr, pte,
>> 1249 tlb->fullmm);
>> 1250 tlb_remove_tlb_entry(tlb, pte, addr);
>> 1251 if (unlikely(!page))
>> 1252 continue;
>> 1253
>> 1254 if (!PageAnon(page)) {
>> 1255 if (pte_dirty(ptent)) {
>> 1256 force_flush = 1;
>> 1257 set_page_dirty(page);
>> 1258 }
>> 1259 if (pte_young(ptent) &&
>> 1260 likely(!(vma->vm_flags & VM_SEQ_READ)))
>> 1261 mark_page_accessed(page);
>> 1262 }
>> 1263 rss[mm_counter(page)]--;
>> 1264 page_remove_rmap(page, false);
>> 1265 if (unlikely(page_mapcount(page) < 0))
>> 1266 print_bad_pte(vma, addr, ptent, page);
>> 1267 if (unlikely(__tlb_remove_page(tlb, page))) {
>> 1268 force_flush = 1;
>> 1269 addr += PAGE_SIZE;
1626 break; 1270 break;
1627 } 1271 }
1628 continue; 1272 continue;
1629 } 1273 }
1630 1274
1631 entry = pte_to_swp_entry(pten 1275 entry = pte_to_swp_entry(ptent);
1632 if (is_device_private_entry(e !! 1276 if (is_device_private_entry(entry)) {
1633 is_device_exclusive_entry !! 1277 struct page *page = device_private_entry_to_page(entry);
1634 page = pfn_swap_entry !! 1278
1635 folio = page_folio(pa !! 1279 if (unlikely(details && details->check_mapping)) {
1636 if (unlikely(!should_ !! 1280 /*
1637 continue; !! 1281 * unmap_shared_mapping_pages() wants to
1638 /* !! 1282 * invalidate cache without truncating:
1639 * Both device privat !! 1283 * unmap shared but keep private pages.
1640 * work with anonymou !! 1284 */
1641 * consider uffd-wp b !! 1285 if (details->check_mapping !=
1642 * see zap_install_uf !! 1286 page_rmapping(page))
1643 */ !! 1287 continue;
1644 WARN_ON_ONCE(!vma_is_ !! 1288 }
1645 rss[mm_counter(folio) !! 1289
1646 if (is_device_private !! 1290 pte_clear_not_present_full(mm, addr, pte, tlb->fullmm);
1647 folio_remove_ !! 1291 rss[mm_counter(page)]--;
1648 folio_put(folio); !! 1292 page_remove_rmap(page, false);
1649 } else if (!non_swap_entry(en !! 1293 put_page(page);
1650 max_nr = (end - addr) !! 1294 continue;
1651 nr = swap_pte_batch(p <<
1652 /* Genuine swap entri <<
1653 if (!should_zap_cows( <<
1654 continue; <<
1655 rss[MM_SWAPENTS] -= n <<
1656 free_swap_and_cache_n <<
1657 } else if (is_migration_entry <<
1658 folio = pfn_swap_entr <<
1659 if (!should_zap_folio <<
1660 continue; <<
1661 rss[mm_counter(folio) <<
1662 } else if (pte_marker_entry_u <<
1663 /* <<
1664 * For anon: always d <<
1665 * drop the marker if <<
1666 */ <<
1667 if (!vma_is_anonymous <<
1668 !zap_drop_file_uf <<
1669 continue; <<
1670 } else if (is_hwpoison_entry( <<
1671 is_poisoned_swp_en <<
1672 if (!should_zap_cows( <<
1673 continue; <<
1674 } else { <<
1675 /* We should have cov <<
1676 pr_alert("unrecognize <<
1677 WARN_ON_ONCE(1); <<
1678 } 1295 }
1679 clear_not_present_full_ptes(m !! 1296
1680 zap_install_uffd_wp_if_needed !! 1297 /* If details->check_mapping, we leave swap entries. */
1681 } while (pte += nr, addr += PAGE_SIZE !! 1298 if (unlikely(details))
>> 1299 continue;
>> 1300
>> 1301 if (!non_swap_entry(entry))
>> 1302 rss[MM_SWAPENTS]--;
>> 1303 else if (is_migration_entry(entry)) {
>> 1304 struct page *page;
>> 1305
>> 1306 page = migration_entry_to_page(entry);
>> 1307 rss[mm_counter(page)]--;
>> 1308 }
>> 1309 if (unlikely(!free_swap_and_cache(entry)))
>> 1310 print_bad_pte(vma, addr, ptent, NULL);
>> 1311 pte_clear_not_present_full(mm, addr, pte, tlb->fullmm);
>> 1312 } while (pte++, addr += PAGE_SIZE, addr != end);
1682 1313
1683 add_mm_rss_vec(mm, rss); 1314 add_mm_rss_vec(mm, rss);
1684 arch_leave_lazy_mmu_mode(); 1315 arch_leave_lazy_mmu_mode();
1685 1316
1686 /* Do the actual TLB flush before dro 1317 /* Do the actual TLB flush before dropping ptl */
1687 if (force_flush) { !! 1318 if (force_flush)
1688 tlb_flush_mmu_tlbonly(tlb); 1319 tlb_flush_mmu_tlbonly(tlb);
1689 tlb_flush_rmaps(tlb, vma); <<
1690 } <<
1691 pte_unmap_unlock(start_pte, ptl); 1320 pte_unmap_unlock(start_pte, ptl);
1692 1321
1693 /* 1322 /*
1694 * If we forced a TLB flush (either d 1323 * If we forced a TLB flush (either due to running out of
1695 * batch buffers or because we needed 1324 * batch buffers or because we needed to flush dirty TLB
1696 * entries before releasing the ptl), 1325 * entries before releasing the ptl), free the batched
1697 * memory too. Come back again if we !! 1326 * memory too. Restart if we didn't do everything.
1698 */ 1327 */
1699 if (force_flush) !! 1328 if (force_flush) {
>> 1329 force_flush = 0;
1700 tlb_flush_mmu(tlb); 1330 tlb_flush_mmu(tlb);
>> 1331 }
>> 1332
>> 1333 if (addr != end) {
>> 1334 cond_resched();
>> 1335 goto again;
>> 1336 }
1701 1337
1702 return addr; 1338 return addr;
1703 } 1339 }
1704 1340
1705 static inline unsigned long zap_pmd_range(str 1341 static inline unsigned long zap_pmd_range(struct mmu_gather *tlb,
1706 struct vm_are 1342 struct vm_area_struct *vma, pud_t *pud,
1707 unsigned long 1343 unsigned long addr, unsigned long end,
1708 struct zap_de 1344 struct zap_details *details)
1709 { 1345 {
1710 pmd_t *pmd; 1346 pmd_t *pmd;
1711 unsigned long next; 1347 unsigned long next;
1712 1348
1713 pmd = pmd_offset(pud, addr); 1349 pmd = pmd_offset(pud, addr);
1714 do { 1350 do {
1715 next = pmd_addr_end(addr, end 1351 next = pmd_addr_end(addr, end);
1716 if (is_swap_pmd(*pmd) || pmd_ 1352 if (is_swap_pmd(*pmd) || pmd_trans_huge(*pmd) || pmd_devmap(*pmd)) {
1717 if (next - addr != HP 1353 if (next - addr != HPAGE_PMD_SIZE)
1718 __split_huge_ 1354 __split_huge_pmd(vma, pmd, addr, false, NULL);
1719 else if (zap_huge_pmd !! 1355 else if (zap_huge_pmd(tlb, vma, pmd, addr))
1720 addr = next; !! 1356 goto next;
1721 continue; <<
1722 } <<
1723 /* fall through */ 1357 /* fall through */
1724 } else if (details && details <<
1725 folio_test_pmd_map <<
1726 next - addr == HPA <<
1727 spinlock_t *ptl = pmd <<
1728 /* <<
1729 * Take and drop THP <<
1730 * prematurely, while <<
1731 * but not yet decrem <<
1732 */ <<
1733 spin_unlock(ptl); <<
1734 } 1358 }
1735 if (pmd_none(*pmd)) { !! 1359 /*
1736 addr = next; !! 1360 * Here there can be other concurrent MADV_DONTNEED or
1737 continue; !! 1361 * trans huge page faults running, and if the pmd is
1738 } !! 1362 * none or trans huge it can change under us. This is
1739 addr = zap_pte_range(tlb, vma !! 1363 * because MADV_DONTNEED holds the mmap_lock in read
1740 if (addr != next) !! 1364 * mode.
1741 pmd--; !! 1365 */
1742 } while (pmd++, cond_resched(), addr !! 1366 if (pmd_none_or_trans_huge_or_clear_bad(pmd))
>> 1367 goto next;
>> 1368 next = zap_pte_range(tlb, vma, pmd, addr, next, details);
>> 1369 next:
>> 1370 cond_resched();
>> 1371 } while (pmd++, addr = next, addr != end);
1743 1372
1744 return addr; 1373 return addr;
1745 } 1374 }
1746 1375
1747 static inline unsigned long zap_pud_range(str 1376 static inline unsigned long zap_pud_range(struct mmu_gather *tlb,
1748 struct vm_are 1377 struct vm_area_struct *vma, p4d_t *p4d,
1749 unsigned long 1378 unsigned long addr, unsigned long end,
1750 struct zap_de 1379 struct zap_details *details)
1751 { 1380 {
1752 pud_t *pud; 1381 pud_t *pud;
1753 unsigned long next; 1382 unsigned long next;
1754 1383
1755 pud = pud_offset(p4d, addr); 1384 pud = pud_offset(p4d, addr);
1756 do { 1385 do {
1757 next = pud_addr_end(addr, end 1386 next = pud_addr_end(addr, end);
1758 if (pud_trans_huge(*pud) || p 1387 if (pud_trans_huge(*pud) || pud_devmap(*pud)) {
1759 if (next - addr != HP 1388 if (next - addr != HPAGE_PUD_SIZE) {
1760 mmap_assert_l 1389 mmap_assert_locked(tlb->mm);
1761 split_huge_pu 1390 split_huge_pud(vma, pud, addr);
1762 } else if (zap_huge_p 1391 } else if (zap_huge_pud(tlb, vma, pud, addr))
1763 goto next; 1392 goto next;
1764 /* fall through */ 1393 /* fall through */
1765 } 1394 }
1766 if (pud_none_or_clear_bad(pud 1395 if (pud_none_or_clear_bad(pud))
1767 continue; 1396 continue;
1768 next = zap_pmd_range(tlb, vma 1397 next = zap_pmd_range(tlb, vma, pud, addr, next, details);
1769 next: 1398 next:
1770 cond_resched(); 1399 cond_resched();
1771 } while (pud++, addr = next, addr != 1400 } while (pud++, addr = next, addr != end);
1772 1401
1773 return addr; 1402 return addr;
1774 } 1403 }
1775 1404
1776 static inline unsigned long zap_p4d_range(str 1405 static inline unsigned long zap_p4d_range(struct mmu_gather *tlb,
1777 struct vm_are 1406 struct vm_area_struct *vma, pgd_t *pgd,
1778 unsigned long 1407 unsigned long addr, unsigned long end,
1779 struct zap_de 1408 struct zap_details *details)
1780 { 1409 {
1781 p4d_t *p4d; 1410 p4d_t *p4d;
1782 unsigned long next; 1411 unsigned long next;
1783 1412
1784 p4d = p4d_offset(pgd, addr); 1413 p4d = p4d_offset(pgd, addr);
1785 do { 1414 do {
1786 next = p4d_addr_end(addr, end 1415 next = p4d_addr_end(addr, end);
1787 if (p4d_none_or_clear_bad(p4d 1416 if (p4d_none_or_clear_bad(p4d))
1788 continue; 1417 continue;
1789 next = zap_pud_range(tlb, vma 1418 next = zap_pud_range(tlb, vma, p4d, addr, next, details);
1790 } while (p4d++, addr = next, addr != 1419 } while (p4d++, addr = next, addr != end);
1791 1420
1792 return addr; 1421 return addr;
1793 } 1422 }
1794 1423
1795 void unmap_page_range(struct mmu_gather *tlb, 1424 void unmap_page_range(struct mmu_gather *tlb,
1796 struct vm_area_s 1425 struct vm_area_struct *vma,
1797 unsigned long ad 1426 unsigned long addr, unsigned long end,
1798 struct zap_detai 1427 struct zap_details *details)
1799 { 1428 {
1800 pgd_t *pgd; 1429 pgd_t *pgd;
1801 unsigned long next; 1430 unsigned long next;
1802 1431
1803 BUG_ON(addr >= end); 1432 BUG_ON(addr >= end);
1804 tlb_start_vma(tlb, vma); 1433 tlb_start_vma(tlb, vma);
1805 pgd = pgd_offset(vma->vm_mm, addr); 1434 pgd = pgd_offset(vma->vm_mm, addr);
1806 do { 1435 do {
1807 next = pgd_addr_end(addr, end 1436 next = pgd_addr_end(addr, end);
1808 if (pgd_none_or_clear_bad(pgd 1437 if (pgd_none_or_clear_bad(pgd))
1809 continue; 1438 continue;
1810 next = zap_p4d_range(tlb, vma 1439 next = zap_p4d_range(tlb, vma, pgd, addr, next, details);
1811 } while (pgd++, addr = next, addr != 1440 } while (pgd++, addr = next, addr != end);
1812 tlb_end_vma(tlb, vma); 1441 tlb_end_vma(tlb, vma);
1813 } 1442 }
1814 1443
1815 1444
1816 static void unmap_single_vma(struct mmu_gathe 1445 static void unmap_single_vma(struct mmu_gather *tlb,
1817 struct vm_area_struct *vma, u 1446 struct vm_area_struct *vma, unsigned long start_addr,
1818 unsigned long end_addr, 1447 unsigned long end_addr,
1819 struct zap_details *details, !! 1448 struct zap_details *details)
1820 { 1449 {
1821 unsigned long start = max(vma->vm_sta 1450 unsigned long start = max(vma->vm_start, start_addr);
1822 unsigned long end; 1451 unsigned long end;
1823 1452
1824 if (start >= vma->vm_end) 1453 if (start >= vma->vm_end)
1825 return; 1454 return;
1826 end = min(vma->vm_end, end_addr); 1455 end = min(vma->vm_end, end_addr);
1827 if (end <= vma->vm_start) 1456 if (end <= vma->vm_start)
1828 return; 1457 return;
1829 1458
1830 if (vma->vm_file) 1459 if (vma->vm_file)
1831 uprobe_munmap(vma, start, end 1460 uprobe_munmap(vma, start, end);
1832 1461
1833 if (unlikely(vma->vm_flags & VM_PFNMA 1462 if (unlikely(vma->vm_flags & VM_PFNMAP))
1834 untrack_pfn(vma, 0, 0, mm_wr_ !! 1463 untrack_pfn(vma, 0, 0);
1835 1464
1836 if (start != end) { 1465 if (start != end) {
1837 if (unlikely(is_vm_hugetlb_pa 1466 if (unlikely(is_vm_hugetlb_page(vma))) {
1838 /* 1467 /*
1839 * It is undesirable 1468 * It is undesirable to test vma->vm_file as it
1840 * should be non-null 1469 * should be non-null for valid hugetlb area.
1841 * However, vm_file w 1470 * However, vm_file will be NULL in the error
1842 * cleanup path of mm 1471 * cleanup path of mmap_region. When
1843 * hugetlbfs ->mmap m 1472 * hugetlbfs ->mmap method fails,
1844 * mmap_region() null 1473 * mmap_region() nullifies vma->vm_file
1845 * before calling thi 1474 * before calling this function to clean up.
1846 * Since no pte has a 1475 * Since no pte has actually been setup, it is
1847 * safe to do nothing 1476 * safe to do nothing in this case.
1848 */ 1477 */
1849 if (vma->vm_file) { 1478 if (vma->vm_file) {
1850 zap_flags_t z !! 1479 i_mmap_lock_write(vma->vm_file->f_mapping);
1851 details-> !! 1480 __unmap_hugepage_range_final(tlb, vma, start, end, NULL);
1852 __unmap_hugep !! 1481 i_mmap_unlock_write(vma->vm_file->f_mapping);
1853 <<
1854 } 1482 }
1855 } else 1483 } else
1856 unmap_page_range(tlb, 1484 unmap_page_range(tlb, vma, start, end, details);
1857 } 1485 }
1858 } 1486 }
1859 1487
1860 /** 1488 /**
1861 * unmap_vmas - unmap a range of memory cover 1489 * unmap_vmas - unmap a range of memory covered by a list of vma's
1862 * @tlb: address of the caller's struct mmu_g 1490 * @tlb: address of the caller's struct mmu_gather
1863 * @mas: the maple state <<
1864 * @vma: the starting vma 1491 * @vma: the starting vma
1865 * @start_addr: virtual address at which to s 1492 * @start_addr: virtual address at which to start unmapping
1866 * @end_addr: virtual address at which to end 1493 * @end_addr: virtual address at which to end unmapping
1867 * @tree_end: The maximum index to check <<
1868 * @mm_wr_locked: lock flag <<
1869 * 1494 *
1870 * Unmap all pages in the vma list. 1495 * Unmap all pages in the vma list.
1871 * 1496 *
1872 * Only addresses between `start' and `end' w 1497 * Only addresses between `start' and `end' will be unmapped.
1873 * 1498 *
1874 * The VMA list must be sorted in ascending v 1499 * The VMA list must be sorted in ascending virtual address order.
1875 * 1500 *
1876 * unmap_vmas() assumes that the caller will 1501 * unmap_vmas() assumes that the caller will flush the whole unmapped address
1877 * range after unmap_vmas() returns. So the 1502 * range after unmap_vmas() returns. So the only responsibility here is to
1878 * ensure that any thus-far unmapped pages ar 1503 * ensure that any thus-far unmapped pages are flushed before unmap_vmas()
1879 * drops the lock and schedules. 1504 * drops the lock and schedules.
1880 */ 1505 */
1881 void unmap_vmas(struct mmu_gather *tlb, struc !! 1506 void unmap_vmas(struct mmu_gather *tlb,
1882 struct vm_area_struct *vma, u 1507 struct vm_area_struct *vma, unsigned long start_addr,
1883 unsigned long end_addr, unsig !! 1508 unsigned long end_addr)
1884 bool mm_wr_locked) <<
1885 { 1509 {
1886 struct mmu_notifier_range range; 1510 struct mmu_notifier_range range;
1887 struct zap_details details = { <<
1888 .zap_flags = ZAP_FLAG_DROP_MA <<
1889 /* Careful - we need to zap p <<
1890 .even_cows = true, <<
1891 }; <<
1892 1511
1893 mmu_notifier_range_init(&range, MMU_N !! 1512 mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma, vma->vm_mm,
1894 start_addr, e 1513 start_addr, end_addr);
1895 mmu_notifier_invalidate_range_start(& 1514 mmu_notifier_invalidate_range_start(&range);
1896 do { !! 1515 for ( ; vma && vma->vm_start < end_addr; vma = vma->vm_next)
1897 unsigned long start = start_a !! 1516 unmap_single_vma(tlb, vma, start_addr, end_addr, NULL);
1898 unsigned long end = end_addr; !! 1517 mmu_notifier_invalidate_range_end(&range);
1899 hugetlb_zap_begin(vma, &start !! 1518 }
1900 unmap_single_vma(tlb, vma, st !! 1519
1901 mm_wr_locked !! 1520 /**
1902 hugetlb_zap_end(vma, &details !! 1521 * zap_page_range - remove user pages in a given range
1903 vma = mas_find(mas, tree_end !! 1522 * @vma: vm_area_struct holding the applicable pages
1904 } while (vma && likely(!xa_is_zero(vm !! 1523 * @start: starting address of pages to zap
>> 1524 * @size: number of bytes to zap
>> 1525 *
>> 1526 * Caller must protect the VMA list
>> 1527 */
>> 1528 void zap_page_range(struct vm_area_struct *vma, unsigned long start,
>> 1529 unsigned long size)
>> 1530 {
>> 1531 struct mmu_notifier_range range;
>> 1532 struct mmu_gather tlb;
>> 1533
>> 1534 lru_add_drain();
>> 1535 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm,
>> 1536 start, start + size);
>> 1537 tlb_gather_mmu(&tlb, vma->vm_mm, start, range.end);
>> 1538 update_hiwater_rss(vma->vm_mm);
>> 1539 mmu_notifier_invalidate_range_start(&range);
>> 1540 for ( ; vma && vma->vm_start < range.end; vma = vma->vm_next)
>> 1541 unmap_single_vma(&tlb, vma, start, range.end, NULL);
1905 mmu_notifier_invalidate_range_end(&ra 1542 mmu_notifier_invalidate_range_end(&range);
>> 1543 tlb_finish_mmu(&tlb, start, range.end);
1906 } 1544 }
1907 1545
1908 /** 1546 /**
1909 * zap_page_range_single - remove user pages 1547 * zap_page_range_single - remove user pages in a given range
1910 * @vma: vm_area_struct holding the applicabl 1548 * @vma: vm_area_struct holding the applicable pages
1911 * @address: starting address of pages to zap 1549 * @address: starting address of pages to zap
1912 * @size: number of bytes to zap 1550 * @size: number of bytes to zap
1913 * @details: details of shared cache invalida 1551 * @details: details of shared cache invalidation
1914 * 1552 *
1915 * The range must fit into one VMA. 1553 * The range must fit into one VMA.
1916 */ 1554 */
1917 void zap_page_range_single(struct vm_area_str !! 1555 static void zap_page_range_single(struct vm_area_struct *vma, unsigned long address,
1918 unsigned long size, struct za 1556 unsigned long size, struct zap_details *details)
1919 { 1557 {
1920 const unsigned long end = address + s <<
1921 struct mmu_notifier_range range; 1558 struct mmu_notifier_range range;
1922 struct mmu_gather tlb; 1559 struct mmu_gather tlb;
1923 1560
1924 lru_add_drain(); 1561 lru_add_drain();
1925 mmu_notifier_range_init(&range, MMU_N !! 1562 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm,
1926 address, end) !! 1563 address, address + size);
1927 hugetlb_zap_begin(vma, &range.start, !! 1564 tlb_gather_mmu(&tlb, vma->vm_mm, address, range.end);
1928 tlb_gather_mmu(&tlb, vma->vm_mm); <<
1929 update_hiwater_rss(vma->vm_mm); 1565 update_hiwater_rss(vma->vm_mm);
1930 mmu_notifier_invalidate_range_start(& 1566 mmu_notifier_invalidate_range_start(&range);
1931 /* !! 1567 unmap_single_vma(&tlb, vma, address, range.end, details);
1932 * unmap 'address-end' not 'range.sta <<
1933 * could have been expanded for huget <<
1934 */ <<
1935 unmap_single_vma(&tlb, vma, address, <<
1936 mmu_notifier_invalidate_range_end(&ra 1568 mmu_notifier_invalidate_range_end(&range);
1937 tlb_finish_mmu(&tlb); !! 1569 tlb_finish_mmu(&tlb, address, range.end);
1938 hugetlb_zap_end(vma, details); <<
1939 } 1570 }
1940 1571
1941 /** 1572 /**
1942 * zap_vma_ptes - remove ptes mapping the vma 1573 * zap_vma_ptes - remove ptes mapping the vma
1943 * @vma: vm_area_struct holding ptes to be za 1574 * @vma: vm_area_struct holding ptes to be zapped
1944 * @address: starting address of pages to zap 1575 * @address: starting address of pages to zap
1945 * @size: number of bytes to zap 1576 * @size: number of bytes to zap
1946 * 1577 *
1947 * This function only unmaps ptes assigned to 1578 * This function only unmaps ptes assigned to VM_PFNMAP vmas.
1948 * 1579 *
1949 * The entire address range must be fully con 1580 * The entire address range must be fully contained within the vma.
1950 * 1581 *
1951 */ 1582 */
1952 void zap_vma_ptes(struct vm_area_struct *vma, 1583 void zap_vma_ptes(struct vm_area_struct *vma, unsigned long address,
1953 unsigned long size) 1584 unsigned long size)
1954 { 1585 {
1955 if (!range_in_vma(vma, address, addre !! 1586 if (address < vma->vm_start || address + size > vma->vm_end ||
1956 !(vma->vm_flags & VM_ 1587 !(vma->vm_flags & VM_PFNMAP))
1957 return; 1588 return;
1958 1589
1959 zap_page_range_single(vma, address, s 1590 zap_page_range_single(vma, address, size, NULL);
1960 } 1591 }
1961 EXPORT_SYMBOL_GPL(zap_vma_ptes); 1592 EXPORT_SYMBOL_GPL(zap_vma_ptes);
1962 1593
1963 static pmd_t *walk_to_pmd(struct mm_struct *m 1594 static pmd_t *walk_to_pmd(struct mm_struct *mm, unsigned long addr)
1964 { 1595 {
1965 pgd_t *pgd; 1596 pgd_t *pgd;
1966 p4d_t *p4d; 1597 p4d_t *p4d;
1967 pud_t *pud; 1598 pud_t *pud;
1968 pmd_t *pmd; 1599 pmd_t *pmd;
1969 1600
1970 pgd = pgd_offset(mm, addr); 1601 pgd = pgd_offset(mm, addr);
1971 p4d = p4d_alloc(mm, pgd, addr); 1602 p4d = p4d_alloc(mm, pgd, addr);
1972 if (!p4d) 1603 if (!p4d)
1973 return NULL; 1604 return NULL;
1974 pud = pud_alloc(mm, p4d, addr); 1605 pud = pud_alloc(mm, p4d, addr);
1975 if (!pud) 1606 if (!pud)
1976 return NULL; 1607 return NULL;
1977 pmd = pmd_alloc(mm, pud, addr); 1608 pmd = pmd_alloc(mm, pud, addr);
1978 if (!pmd) 1609 if (!pmd)
1979 return NULL; 1610 return NULL;
1980 1611
1981 VM_BUG_ON(pmd_trans_huge(*pmd)); 1612 VM_BUG_ON(pmd_trans_huge(*pmd));
1982 return pmd; 1613 return pmd;
1983 } 1614 }
1984 1615
1985 pte_t *__get_locked_pte(struct mm_struct *mm, 1616 pte_t *__get_locked_pte(struct mm_struct *mm, unsigned long addr,
1986 spinlock_t **ptl) 1617 spinlock_t **ptl)
1987 { 1618 {
1988 pmd_t *pmd = walk_to_pmd(mm, addr); 1619 pmd_t *pmd = walk_to_pmd(mm, addr);
1989 1620
1990 if (!pmd) 1621 if (!pmd)
1991 return NULL; 1622 return NULL;
1992 return pte_alloc_map_lock(mm, pmd, ad 1623 return pte_alloc_map_lock(mm, pmd, addr, ptl);
1993 } 1624 }
1994 1625
1995 static bool vm_mixed_zeropage_allowed(struct !! 1626 static int validate_page_before_insert(struct page *page)
1996 { <<
1997 VM_WARN_ON_ONCE(vma->vm_flags & VM_PF <<
1998 /* <<
1999 * Whoever wants to forbid the zeropa <<
2000 * might already have been mapped has <<
2001 * bail out on any zeropages. Zeropag <<
2002 * be unshared using FAULT_FLAG_UNSHA <<
2003 */ <<
2004 if (mm_forbids_zeropage(vma->vm_mm)) <<
2005 return false; <<
2006 /* zeropages in COW mappings are comm <<
2007 if (is_cow_mapping(vma->vm_flags)) <<
2008 return true; <<
2009 /* Mappings that do not allow for wri <<
2010 if (!(vma->vm_flags & (VM_WRITE | VM_ <<
2011 return true; <<
2012 /* <<
2013 * Why not allow any VMA that has vm_ <<
2014 * find the shared zeropage and longt <<
2015 * be problematic as soon as the zero <<
2016 * page due to vma->vm_ops->pfn_mkwri <<
2017 * now differ to what GUP looked up. <<
2018 * FOLL_LONGTERM and VM_IO is incompa <<
2019 * check_vma_flags). <<
2020 */ <<
2021 return vma->vm_ops && vma->vm_ops->pf <<
2022 (vma_is_fsdax(vma) || vma->vm_ <<
2023 } <<
2024 <<
2025 static int validate_page_before_insert(struct <<
2026 struct <<
2027 { 1627 {
2028 struct folio *folio = page_folio(page !! 1628 if (PageAnon(page) || PageSlab(page) || page_has_type(page))
2029 <<
2030 if (!folio_ref_count(folio)) <<
2031 return -EINVAL; 1629 return -EINVAL;
2032 if (unlikely(is_zero_folio(folio))) { !! 1630 flush_dcache_page(page);
2033 if (!vm_mixed_zeropage_allowe <<
2034 return -EINVAL; <<
2035 return 0; <<
2036 } <<
2037 if (folio_test_anon(folio) || folio_t <<
2038 page_has_type(page)) <<
2039 return -EINVAL; <<
2040 flush_dcache_folio(folio); <<
2041 return 0; 1631 return 0;
2042 } 1632 }
2043 1633
2044 static int insert_page_into_pte_locked(struct !! 1634 static int insert_page_into_pte_locked(struct mm_struct *mm, pte_t *pte,
2045 unsigned long addr, s 1635 unsigned long addr, struct page *page, pgprot_t prot)
2046 { 1636 {
2047 struct folio *folio = page_folio(page !! 1637 if (!pte_none(*pte))
2048 pte_t pteval; <<
2049 <<
2050 if (!pte_none(ptep_get(pte))) <<
2051 return -EBUSY; 1638 return -EBUSY;
2052 /* Ok, finally just insert the thing. 1639 /* Ok, finally just insert the thing.. */
2053 pteval = mk_pte(page, prot); !! 1640 get_page(page);
2054 if (unlikely(is_zero_folio(folio))) { !! 1641 inc_mm_counter_fast(mm, mm_counter_file(page));
2055 pteval = pte_mkspecial(pteval !! 1642 page_add_file_rmap(page, false);
2056 } else { !! 1643 set_pte_at(mm, addr, pte, mk_pte(page, prot));
2057 folio_get(folio); <<
2058 inc_mm_counter(vma->vm_mm, mm <<
2059 folio_add_file_rmap_pte(folio <<
2060 } <<
2061 set_pte_at(vma->vm_mm, addr, pte, pte <<
2062 return 0; 1644 return 0;
2063 } 1645 }
2064 1646
>> 1647 /*
>> 1648 * This is the old fallback for page remapping.
>> 1649 *
>> 1650 * For historical reasons, it only allows reserved pages. Only
>> 1651 * old drivers should use this, and they needed to mark their
>> 1652 * pages reserved for the old functions anyway.
>> 1653 */
2065 static int insert_page(struct vm_area_struct 1654 static int insert_page(struct vm_area_struct *vma, unsigned long addr,
2066 struct page *page, pg 1655 struct page *page, pgprot_t prot)
2067 { 1656 {
>> 1657 struct mm_struct *mm = vma->vm_mm;
2068 int retval; 1658 int retval;
2069 pte_t *pte; 1659 pte_t *pte;
2070 spinlock_t *ptl; 1660 spinlock_t *ptl;
2071 1661
2072 retval = validate_page_before_insert( !! 1662 retval = validate_page_before_insert(page);
2073 if (retval) 1663 if (retval)
2074 goto out; 1664 goto out;
2075 retval = -ENOMEM; 1665 retval = -ENOMEM;
2076 pte = get_locked_pte(vma->vm_mm, addr !! 1666 pte = get_locked_pte(mm, addr, &ptl);
2077 if (!pte) 1667 if (!pte)
2078 goto out; 1668 goto out;
2079 retval = insert_page_into_pte_locked( !! 1669 retval = insert_page_into_pte_locked(mm, pte, addr, page, prot);
2080 pte_unmap_unlock(pte, ptl); 1670 pte_unmap_unlock(pte, ptl);
2081 out: 1671 out:
2082 return retval; 1672 return retval;
2083 } 1673 }
2084 1674
2085 static int insert_page_in_batch_locked(struct !! 1675 #ifdef pte_index
>> 1676 static int insert_page_in_batch_locked(struct mm_struct *mm, pte_t *pte,
2086 unsigned long addr, s 1677 unsigned long addr, struct page *page, pgprot_t prot)
2087 { 1678 {
2088 int err; 1679 int err;
2089 1680
2090 err = validate_page_before_insert(vma !! 1681 if (!page_count(page))
>> 1682 return -EINVAL;
>> 1683 err = validate_page_before_insert(page);
2091 if (err) 1684 if (err)
2092 return err; 1685 return err;
2093 return insert_page_into_pte_locked(vm !! 1686 return insert_page_into_pte_locked(mm, pte, addr, page, prot);
2094 } 1687 }
2095 1688
2096 /* insert_pages() amortizes the cost of spinl 1689 /* insert_pages() amortizes the cost of spinlock operations
2097 * when inserting pages in a loop. !! 1690 * when inserting pages in a loop. Arch *must* define pte_index.
2098 */ 1691 */
2099 static int insert_pages(struct vm_area_struct 1692 static int insert_pages(struct vm_area_struct *vma, unsigned long addr,
2100 struct page **pages, 1693 struct page **pages, unsigned long *num, pgprot_t prot)
2101 { 1694 {
2102 pmd_t *pmd = NULL; 1695 pmd_t *pmd = NULL;
2103 pte_t *start_pte, *pte; 1696 pte_t *start_pte, *pte;
2104 spinlock_t *pte_lock; 1697 spinlock_t *pte_lock;
2105 struct mm_struct *const mm = vma->vm_ 1698 struct mm_struct *const mm = vma->vm_mm;
2106 unsigned long curr_page_idx = 0; 1699 unsigned long curr_page_idx = 0;
2107 unsigned long remaining_pages_total = 1700 unsigned long remaining_pages_total = *num;
2108 unsigned long pages_to_write_in_pmd; 1701 unsigned long pages_to_write_in_pmd;
2109 int ret; 1702 int ret;
2110 more: 1703 more:
2111 ret = -EFAULT; 1704 ret = -EFAULT;
2112 pmd = walk_to_pmd(mm, addr); 1705 pmd = walk_to_pmd(mm, addr);
2113 if (!pmd) 1706 if (!pmd)
2114 goto out; 1707 goto out;
2115 1708
2116 pages_to_write_in_pmd = min_t(unsigne 1709 pages_to_write_in_pmd = min_t(unsigned long,
2117 remaining_pages_total, PTRS_P 1710 remaining_pages_total, PTRS_PER_PTE - pte_index(addr));
2118 1711
2119 /* Allocate the PTE if necessary; tak 1712 /* Allocate the PTE if necessary; takes PMD lock once only. */
2120 ret = -ENOMEM; 1713 ret = -ENOMEM;
2121 if (pte_alloc(mm, pmd)) 1714 if (pte_alloc(mm, pmd))
2122 goto out; 1715 goto out;
2123 1716
2124 while (pages_to_write_in_pmd) { 1717 while (pages_to_write_in_pmd) {
2125 int pte_idx = 0; 1718 int pte_idx = 0;
2126 const int batch_size = min_t( 1719 const int batch_size = min_t(int, pages_to_write_in_pmd, 8);
2127 1720
2128 start_pte = pte_offset_map_lo 1721 start_pte = pte_offset_map_lock(mm, pmd, addr, &pte_lock);
2129 if (!start_pte) { <<
2130 ret = -EFAULT; <<
2131 goto out; <<
2132 } <<
2133 for (pte = start_pte; pte_idx 1722 for (pte = start_pte; pte_idx < batch_size; ++pte, ++pte_idx) {
2134 int err = insert_page !! 1723 int err = insert_page_in_batch_locked(mm, pte,
2135 addr, pages[c 1724 addr, pages[curr_page_idx], prot);
2136 if (unlikely(err)) { 1725 if (unlikely(err)) {
2137 pte_unmap_unl 1726 pte_unmap_unlock(start_pte, pte_lock);
2138 ret = err; 1727 ret = err;
2139 remaining_pag 1728 remaining_pages_total -= pte_idx;
2140 goto out; 1729 goto out;
2141 } 1730 }
2142 addr += PAGE_SIZE; 1731 addr += PAGE_SIZE;
2143 ++curr_page_idx; 1732 ++curr_page_idx;
2144 } 1733 }
2145 pte_unmap_unlock(start_pte, p 1734 pte_unmap_unlock(start_pte, pte_lock);
2146 pages_to_write_in_pmd -= batc 1735 pages_to_write_in_pmd -= batch_size;
2147 remaining_pages_total -= batc 1736 remaining_pages_total -= batch_size;
2148 } 1737 }
2149 if (remaining_pages_total) 1738 if (remaining_pages_total)
2150 goto more; 1739 goto more;
2151 ret = 0; 1740 ret = 0;
2152 out: 1741 out:
2153 *num = remaining_pages_total; 1742 *num = remaining_pages_total;
2154 return ret; 1743 return ret;
2155 } 1744 }
>> 1745 #endif /* ifdef pte_index */
2156 1746
2157 /** 1747 /**
2158 * vm_insert_pages - insert multiple pages in 1748 * vm_insert_pages - insert multiple pages into user vma, batching the pmd lock.
2159 * @vma: user vma to map to 1749 * @vma: user vma to map to
2160 * @addr: target start user address of these 1750 * @addr: target start user address of these pages
2161 * @pages: source kernel pages 1751 * @pages: source kernel pages
2162 * @num: in: number of pages to map. out: num 1752 * @num: in: number of pages to map. out: number of pages that were *not*
2163 * mapped. (0 means all pages were successful 1753 * mapped. (0 means all pages were successfully mapped).
2164 * 1754 *
2165 * Preferred over vm_insert_page() when inser 1755 * Preferred over vm_insert_page() when inserting multiple pages.
2166 * 1756 *
2167 * In case of error, we may have mapped a sub 1757 * In case of error, we may have mapped a subset of the provided
2168 * pages. It is the caller's responsibility t 1758 * pages. It is the caller's responsibility to account for this case.
2169 * 1759 *
2170 * The same restrictions apply as in vm_inser 1760 * The same restrictions apply as in vm_insert_page().
2171 */ 1761 */
2172 int vm_insert_pages(struct vm_area_struct *vm 1762 int vm_insert_pages(struct vm_area_struct *vma, unsigned long addr,
2173 struct page **pages, 1763 struct page **pages, unsigned long *num)
2174 { 1764 {
>> 1765 #ifdef pte_index
2175 const unsigned long end_addr = addr + 1766 const unsigned long end_addr = addr + (*num * PAGE_SIZE) - 1;
2176 1767
2177 if (addr < vma->vm_start || end_addr 1768 if (addr < vma->vm_start || end_addr >= vma->vm_end)
2178 return -EFAULT; 1769 return -EFAULT;
2179 if (!(vma->vm_flags & VM_MIXEDMAP)) { 1770 if (!(vma->vm_flags & VM_MIXEDMAP)) {
2180 BUG_ON(mmap_read_trylock(vma- 1771 BUG_ON(mmap_read_trylock(vma->vm_mm));
2181 BUG_ON(vma->vm_flags & VM_PFN 1772 BUG_ON(vma->vm_flags & VM_PFNMAP);
2182 vm_flags_set(vma, VM_MIXEDMAP !! 1773 vma->vm_flags |= VM_MIXEDMAP;
2183 } 1774 }
2184 /* Defer page refcount checking till 1775 /* Defer page refcount checking till we're about to map that page. */
2185 return insert_pages(vma, addr, pages, 1776 return insert_pages(vma, addr, pages, num, vma->vm_page_prot);
>> 1777 #else
>> 1778 unsigned long idx = 0, pgcount = *num;
>> 1779 int err = -EINVAL;
>> 1780
>> 1781 for (; idx < pgcount; ++idx) {
>> 1782 err = vm_insert_page(vma, addr + (PAGE_SIZE * idx), pages[idx]);
>> 1783 if (err)
>> 1784 break;
>> 1785 }
>> 1786 *num = pgcount - idx;
>> 1787 return err;
>> 1788 #endif /* ifdef pte_index */
2186 } 1789 }
2187 EXPORT_SYMBOL(vm_insert_pages); 1790 EXPORT_SYMBOL(vm_insert_pages);
2188 1791
2189 /** 1792 /**
2190 * vm_insert_page - insert single page into u 1793 * vm_insert_page - insert single page into user vma
2191 * @vma: user vma to map to 1794 * @vma: user vma to map to
2192 * @addr: target user address of this page 1795 * @addr: target user address of this page
2193 * @page: source kernel page 1796 * @page: source kernel page
2194 * 1797 *
2195 * This allows drivers to insert individual p 1798 * This allows drivers to insert individual pages they've allocated
2196 * into a user vma. The zeropage is supported !! 1799 * into a user vma.
2197 * see vm_mixed_zeropage_allowed(). <<
2198 * 1800 *
2199 * The page has to be a nice clean _individua 1801 * The page has to be a nice clean _individual_ kernel allocation.
2200 * If you allocate a compound page, you need 1802 * If you allocate a compound page, you need to have marked it as
2201 * such (__GFP_COMP), or manually just split 1803 * such (__GFP_COMP), or manually just split the page up yourself
2202 * (see split_page()). 1804 * (see split_page()).
2203 * 1805 *
2204 * NOTE! Traditionally this was done with "re 1806 * NOTE! Traditionally this was done with "remap_pfn_range()" which
2205 * took an arbitrary page protection paramete 1807 * took an arbitrary page protection parameter. This doesn't allow
2206 * that. Your vma protection will have to be 1808 * that. Your vma protection will have to be set up correctly, which
2207 * means that if you want a shared writable m 1809 * means that if you want a shared writable mapping, you'd better
2208 * ask for a shared writable mapping! 1810 * ask for a shared writable mapping!
2209 * 1811 *
2210 * The page does not need to be reserved. 1812 * The page does not need to be reserved.
2211 * 1813 *
2212 * Usually this function is called from f_op- 1814 * Usually this function is called from f_op->mmap() handler
2213 * under mm->mmap_lock write-lock, so it can 1815 * under mm->mmap_lock write-lock, so it can change vma->vm_flags.
2214 * Caller must set VM_MIXEDMAP on vma if it w 1816 * Caller must set VM_MIXEDMAP on vma if it wants to call this
2215 * function from other places, for example fr 1817 * function from other places, for example from page-fault handler.
2216 * 1818 *
2217 * Return: %0 on success, negative error code 1819 * Return: %0 on success, negative error code otherwise.
2218 */ 1820 */
2219 int vm_insert_page(struct vm_area_struct *vma 1821 int vm_insert_page(struct vm_area_struct *vma, unsigned long addr,
2220 struct page *page) 1822 struct page *page)
2221 { 1823 {
2222 if (addr < vma->vm_start || addr >= v 1824 if (addr < vma->vm_start || addr >= vma->vm_end)
2223 return -EFAULT; 1825 return -EFAULT;
>> 1826 if (!page_count(page))
>> 1827 return -EINVAL;
2224 if (!(vma->vm_flags & VM_MIXEDMAP)) { 1828 if (!(vma->vm_flags & VM_MIXEDMAP)) {
2225 BUG_ON(mmap_read_trylock(vma- 1829 BUG_ON(mmap_read_trylock(vma->vm_mm));
2226 BUG_ON(vma->vm_flags & VM_PFN 1830 BUG_ON(vma->vm_flags & VM_PFNMAP);
2227 vm_flags_set(vma, VM_MIXEDMAP !! 1831 vma->vm_flags |= VM_MIXEDMAP;
2228 } 1832 }
2229 return insert_page(vma, addr, page, v 1833 return insert_page(vma, addr, page, vma->vm_page_prot);
2230 } 1834 }
2231 EXPORT_SYMBOL(vm_insert_page); 1835 EXPORT_SYMBOL(vm_insert_page);
2232 1836
2233 /* 1837 /*
2234 * __vm_map_pages - maps range of kernel page 1838 * __vm_map_pages - maps range of kernel pages into user vma
2235 * @vma: user vma to map to 1839 * @vma: user vma to map to
2236 * @pages: pointer to array of source kernel 1840 * @pages: pointer to array of source kernel pages
2237 * @num: number of pages in page array 1841 * @num: number of pages in page array
2238 * @offset: user's requested vm_pgoff 1842 * @offset: user's requested vm_pgoff
2239 * 1843 *
2240 * This allows drivers to map range of kernel 1844 * This allows drivers to map range of kernel pages into a user vma.
2241 * The zeropage is supported in some VMAs, se <<
2242 * vm_mixed_zeropage_allowed(). <<
2243 * 1845 *
2244 * Return: 0 on success and error code otherw 1846 * Return: 0 on success and error code otherwise.
2245 */ 1847 */
2246 static int __vm_map_pages(struct vm_area_stru 1848 static int __vm_map_pages(struct vm_area_struct *vma, struct page **pages,
2247 unsigned long 1849 unsigned long num, unsigned long offset)
2248 { 1850 {
2249 unsigned long count = vma_pages(vma); 1851 unsigned long count = vma_pages(vma);
2250 unsigned long uaddr = vma->vm_start; 1852 unsigned long uaddr = vma->vm_start;
2251 int ret, i; 1853 int ret, i;
2252 1854
2253 /* Fail if the user requested offset 1855 /* Fail if the user requested offset is beyond the end of the object */
2254 if (offset >= num) 1856 if (offset >= num)
2255 return -ENXIO; 1857 return -ENXIO;
2256 1858
2257 /* Fail if the user requested size ex 1859 /* Fail if the user requested size exceeds available object size */
2258 if (count > num - offset) 1860 if (count > num - offset)
2259 return -ENXIO; 1861 return -ENXIO;
2260 1862
2261 for (i = 0; i < count; i++) { 1863 for (i = 0; i < count; i++) {
2262 ret = vm_insert_page(vma, uad 1864 ret = vm_insert_page(vma, uaddr, pages[offset + i]);
2263 if (ret < 0) 1865 if (ret < 0)
2264 return ret; 1866 return ret;
2265 uaddr += PAGE_SIZE; 1867 uaddr += PAGE_SIZE;
2266 } 1868 }
2267 1869
2268 return 0; 1870 return 0;
2269 } 1871 }
2270 1872
2271 /** 1873 /**
2272 * vm_map_pages - maps range of kernel pages 1874 * vm_map_pages - maps range of kernel pages starts with non zero offset
2273 * @vma: user vma to map to 1875 * @vma: user vma to map to
2274 * @pages: pointer to array of source kernel 1876 * @pages: pointer to array of source kernel pages
2275 * @num: number of pages in page array 1877 * @num: number of pages in page array
2276 * 1878 *
2277 * Maps an object consisting of @num pages, c 1879 * Maps an object consisting of @num pages, catering for the user's
2278 * requested vm_pgoff 1880 * requested vm_pgoff
2279 * 1881 *
2280 * If we fail to insert any page into the vma 1882 * If we fail to insert any page into the vma, the function will return
2281 * immediately leaving any previously inserte 1883 * immediately leaving any previously inserted pages present. Callers
2282 * from the mmap handler may immediately retu 1884 * from the mmap handler may immediately return the error as their caller
2283 * will destroy the vma, removing any success 1885 * will destroy the vma, removing any successfully inserted pages. Other
2284 * callers should make their own arrangements 1886 * callers should make their own arrangements for calling unmap_region().
2285 * 1887 *
2286 * Context: Process context. Called by mmap h 1888 * Context: Process context. Called by mmap handlers.
2287 * Return: 0 on success and error code otherw 1889 * Return: 0 on success and error code otherwise.
2288 */ 1890 */
2289 int vm_map_pages(struct vm_area_struct *vma, 1891 int vm_map_pages(struct vm_area_struct *vma, struct page **pages,
2290 unsigned long 1892 unsigned long num)
2291 { 1893 {
2292 return __vm_map_pages(vma, pages, num 1894 return __vm_map_pages(vma, pages, num, vma->vm_pgoff);
2293 } 1895 }
2294 EXPORT_SYMBOL(vm_map_pages); 1896 EXPORT_SYMBOL(vm_map_pages);
2295 1897
2296 /** 1898 /**
2297 * vm_map_pages_zero - map range of kernel pa 1899 * vm_map_pages_zero - map range of kernel pages starts with zero offset
2298 * @vma: user vma to map to 1900 * @vma: user vma to map to
2299 * @pages: pointer to array of source kernel 1901 * @pages: pointer to array of source kernel pages
2300 * @num: number of pages in page array 1902 * @num: number of pages in page array
2301 * 1903 *
2302 * Similar to vm_map_pages(), except that it 1904 * Similar to vm_map_pages(), except that it explicitly sets the offset
2303 * to 0. This function is intended for the dr 1905 * to 0. This function is intended for the drivers that did not consider
2304 * vm_pgoff. 1906 * vm_pgoff.
2305 * 1907 *
2306 * Context: Process context. Called by mmap h 1908 * Context: Process context. Called by mmap handlers.
2307 * Return: 0 on success and error code otherw 1909 * Return: 0 on success and error code otherwise.
2308 */ 1910 */
2309 int vm_map_pages_zero(struct vm_area_struct * 1911 int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages,
2310 unsigned long 1912 unsigned long num)
2311 { 1913 {
2312 return __vm_map_pages(vma, pages, num 1914 return __vm_map_pages(vma, pages, num, 0);
2313 } 1915 }
2314 EXPORT_SYMBOL(vm_map_pages_zero); 1916 EXPORT_SYMBOL(vm_map_pages_zero);
2315 1917
2316 static vm_fault_t insert_pfn(struct vm_area_s 1918 static vm_fault_t insert_pfn(struct vm_area_struct *vma, unsigned long addr,
2317 pfn_t pfn, pgprot_t p 1919 pfn_t pfn, pgprot_t prot, bool mkwrite)
2318 { 1920 {
2319 struct mm_struct *mm = vma->vm_mm; 1921 struct mm_struct *mm = vma->vm_mm;
2320 pte_t *pte, entry; 1922 pte_t *pte, entry;
2321 spinlock_t *ptl; 1923 spinlock_t *ptl;
2322 1924
2323 pte = get_locked_pte(mm, addr, &ptl); 1925 pte = get_locked_pte(mm, addr, &ptl);
2324 if (!pte) 1926 if (!pte)
2325 return VM_FAULT_OOM; 1927 return VM_FAULT_OOM;
2326 entry = ptep_get(pte); !! 1928 if (!pte_none(*pte)) {
2327 if (!pte_none(entry)) { <<
2328 if (mkwrite) { 1929 if (mkwrite) {
2329 /* 1930 /*
2330 * For read faults on 1931 * For read faults on private mappings the PFN passed
2331 * in may not match t 1932 * in may not match the PFN we have mapped if the
2332 * mapped PFN is a wr 1933 * mapped PFN is a writeable COW page. In the mkwrite
2333 * case we are creati 1934 * case we are creating a writable PTE for a shared
2334 * mapping and we exp 1935 * mapping and we expect the PFNs to match. If they
2335 * don't match, we ar 1936 * don't match, we are likely racing with block
2336 * allocation and map 1937 * allocation and mapping invalidation so just skip the
2337 * update. 1938 * update.
2338 */ 1939 */
2339 if (pte_pfn(entry) != !! 1940 if (pte_pfn(*pte) != pfn_t_to_pfn(pfn)) {
2340 WARN_ON_ONCE( !! 1941 WARN_ON_ONCE(!is_zero_pfn(pte_pfn(*pte)));
2341 goto out_unlo 1942 goto out_unlock;
2342 } 1943 }
2343 entry = pte_mkyoung(e !! 1944 entry = pte_mkyoung(*pte);
2344 entry = maybe_mkwrite 1945 entry = maybe_mkwrite(pte_mkdirty(entry), vma);
2345 if (ptep_set_access_f 1946 if (ptep_set_access_flags(vma, addr, pte, entry, 1))
2346 update_mmu_ca 1947 update_mmu_cache(vma, addr, pte);
2347 } 1948 }
2348 goto out_unlock; 1949 goto out_unlock;
2349 } 1950 }
2350 1951
2351 /* Ok, finally just insert the thing. 1952 /* Ok, finally just insert the thing.. */
2352 if (pfn_t_devmap(pfn)) 1953 if (pfn_t_devmap(pfn))
2353 entry = pte_mkdevmap(pfn_t_pt 1954 entry = pte_mkdevmap(pfn_t_pte(pfn, prot));
2354 else 1955 else
2355 entry = pte_mkspecial(pfn_t_p 1956 entry = pte_mkspecial(pfn_t_pte(pfn, prot));
2356 1957
2357 if (mkwrite) { 1958 if (mkwrite) {
2358 entry = pte_mkyoung(entry); 1959 entry = pte_mkyoung(entry);
2359 entry = maybe_mkwrite(pte_mkd 1960 entry = maybe_mkwrite(pte_mkdirty(entry), vma);
2360 } 1961 }
2361 1962
2362 set_pte_at(mm, addr, pte, entry); 1963 set_pte_at(mm, addr, pte, entry);
2363 update_mmu_cache(vma, addr, pte); /* 1964 update_mmu_cache(vma, addr, pte); /* XXX: why not for insert_page? */
2364 1965
2365 out_unlock: 1966 out_unlock:
2366 pte_unmap_unlock(pte, ptl); 1967 pte_unmap_unlock(pte, ptl);
2367 return VM_FAULT_NOPAGE; 1968 return VM_FAULT_NOPAGE;
2368 } 1969 }
2369 1970
2370 /** 1971 /**
2371 * vmf_insert_pfn_prot - insert single pfn in 1972 * vmf_insert_pfn_prot - insert single pfn into user vma with specified pgprot
2372 * @vma: user vma to map to 1973 * @vma: user vma to map to
2373 * @addr: target user address of this page 1974 * @addr: target user address of this page
2374 * @pfn: source kernel pfn 1975 * @pfn: source kernel pfn
2375 * @pgprot: pgprot flags for the inserted pag 1976 * @pgprot: pgprot flags for the inserted page
2376 * 1977 *
2377 * This is exactly like vmf_insert_pfn(), exc 1978 * This is exactly like vmf_insert_pfn(), except that it allows drivers
2378 * to override pgprot on a per-page basis. 1979 * to override pgprot on a per-page basis.
2379 * 1980 *
2380 * This only makes sense for IO mappings, and 1981 * This only makes sense for IO mappings, and it makes no sense for
2381 * COW mappings. In general, using multiple 1982 * COW mappings. In general, using multiple vmas is preferable;
2382 * vmf_insert_pfn_prot should only be used if 1983 * vmf_insert_pfn_prot should only be used if using multiple VMAs is
2383 * impractical. 1984 * impractical.
2384 * 1985 *
2385 * pgprot typically only differs from @vma->v !! 1986 * See vmf_insert_mixed_prot() for a discussion of the implication of using
2386 * caching- and encryption bits different tha !! 1987 * a value of @pgprot different from that of @vma->vm_page_prot.
2387 * because the caching- or encryption mode ma <<
2388 * <<
2389 * This is ok as long as @vma->vm_page_prot i <<
2390 * to set caching and encryption bits for tho <<
2391 * This is ensured by core vm only modifying <<
2392 * functions that don't touch caching- or enc <<
2393 * if needed. (See for example mprotect()). <<
2394 * <<
2395 * Also when new page-table entries are creat <<
2396 * fault() callback, and never using the valu <<
2397 * except for page-table entries that point t <<
2398 * of COW. <<
2399 * 1988 *
2400 * Context: Process context. May allocate us 1989 * Context: Process context. May allocate using %GFP_KERNEL.
2401 * Return: vm_fault_t value. 1990 * Return: vm_fault_t value.
2402 */ 1991 */
2403 vm_fault_t vmf_insert_pfn_prot(struct vm_area 1992 vm_fault_t vmf_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr,
2404 unsigned long pfn, pg 1993 unsigned long pfn, pgprot_t pgprot)
2405 { 1994 {
2406 /* 1995 /*
2407 * Technically, architectures with pt 1996 * Technically, architectures with pte_special can avoid all these
2408 * restrictions (same for remap_pfn_r 1997 * restrictions (same for remap_pfn_range). However we would like
2409 * consistency in testing and feature 1998 * consistency in testing and feature parity among all, so we should
2410 * try to keep these invariants in pl 1999 * try to keep these invariants in place for everybody.
2411 */ 2000 */
2412 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|V 2001 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)));
2413 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM 2002 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) ==
2414 2003 (VM_PFNMAP|VM_MIXEDMAP));
2415 BUG_ON((vma->vm_flags & VM_PFNMAP) && 2004 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags));
2416 BUG_ON((vma->vm_flags & VM_MIXEDMAP) 2005 BUG_ON((vma->vm_flags & VM_MIXEDMAP) && pfn_valid(pfn));
2417 2006
2418 if (addr < vma->vm_start || addr >= v 2007 if (addr < vma->vm_start || addr >= vma->vm_end)
2419 return VM_FAULT_SIGBUS; 2008 return VM_FAULT_SIGBUS;
2420 2009
2421 if (!pfn_modify_allowed(pfn, pgprot)) 2010 if (!pfn_modify_allowed(pfn, pgprot))
2422 return VM_FAULT_SIGBUS; 2011 return VM_FAULT_SIGBUS;
2423 2012
2424 track_pfn_insert(vma, &pgprot, __pfn_ 2013 track_pfn_insert(vma, &pgprot, __pfn_to_pfn_t(pfn, PFN_DEV));
2425 2014
2426 return insert_pfn(vma, addr, __pfn_to 2015 return insert_pfn(vma, addr, __pfn_to_pfn_t(pfn, PFN_DEV), pgprot,
2427 false); 2016 false);
2428 } 2017 }
2429 EXPORT_SYMBOL(vmf_insert_pfn_prot); 2018 EXPORT_SYMBOL(vmf_insert_pfn_prot);
2430 2019
2431 /** 2020 /**
2432 * vmf_insert_pfn - insert single pfn into us 2021 * vmf_insert_pfn - insert single pfn into user vma
2433 * @vma: user vma to map to 2022 * @vma: user vma to map to
2434 * @addr: target user address of this page 2023 * @addr: target user address of this page
2435 * @pfn: source kernel pfn 2024 * @pfn: source kernel pfn
2436 * 2025 *
2437 * Similar to vm_insert_page, this allows dri 2026 * Similar to vm_insert_page, this allows drivers to insert individual pages
2438 * they've allocated into a user vma. Same co 2027 * they've allocated into a user vma. Same comments apply.
2439 * 2028 *
2440 * This function should only be called from a 2029 * This function should only be called from a vm_ops->fault handler, and
2441 * in that case the handler should return the 2030 * in that case the handler should return the result of this function.
2442 * 2031 *
2443 * vma cannot be a COW mapping. 2032 * vma cannot be a COW mapping.
2444 * 2033 *
2445 * As this is called only for pages that do n 2034 * As this is called only for pages that do not currently exist, we
2446 * do not need to flush old virtual caches or 2035 * do not need to flush old virtual caches or the TLB.
2447 * 2036 *
2448 * Context: Process context. May allocate us 2037 * Context: Process context. May allocate using %GFP_KERNEL.
2449 * Return: vm_fault_t value. 2038 * Return: vm_fault_t value.
2450 */ 2039 */
2451 vm_fault_t vmf_insert_pfn(struct vm_area_stru 2040 vm_fault_t vmf_insert_pfn(struct vm_area_struct *vma, unsigned long addr,
2452 unsigned long pfn) 2041 unsigned long pfn)
2453 { 2042 {
2454 return vmf_insert_pfn_prot(vma, addr, 2043 return vmf_insert_pfn_prot(vma, addr, pfn, vma->vm_page_prot);
2455 } 2044 }
2456 EXPORT_SYMBOL(vmf_insert_pfn); 2045 EXPORT_SYMBOL(vmf_insert_pfn);
2457 2046
2458 static bool vm_mixed_ok(struct vm_area_struct !! 2047 static bool vm_mixed_ok(struct vm_area_struct *vma, pfn_t pfn)
2459 { 2048 {
2460 if (unlikely(is_zero_pfn(pfn_t_to_pfn <<
2461 (mkwrite || !vm_mixed_zeropage_al <<
2462 return false; <<
2463 /* these checks mirror the abort cond 2049 /* these checks mirror the abort conditions in vm_normal_page */
2464 if (vma->vm_flags & VM_MIXEDMAP) 2050 if (vma->vm_flags & VM_MIXEDMAP)
2465 return true; 2051 return true;
2466 if (pfn_t_devmap(pfn)) 2052 if (pfn_t_devmap(pfn))
2467 return true; 2053 return true;
2468 if (pfn_t_special(pfn)) 2054 if (pfn_t_special(pfn))
2469 return true; 2055 return true;
2470 if (is_zero_pfn(pfn_t_to_pfn(pfn))) 2056 if (is_zero_pfn(pfn_t_to_pfn(pfn)))
2471 return true; 2057 return true;
2472 return false; 2058 return false;
2473 } 2059 }
2474 2060
2475 static vm_fault_t __vm_insert_mixed(struct vm 2061 static vm_fault_t __vm_insert_mixed(struct vm_area_struct *vma,
2476 unsigned long addr, pfn_t pfn !! 2062 unsigned long addr, pfn_t pfn, pgprot_t pgprot,
>> 2063 bool mkwrite)
2477 { 2064 {
2478 pgprot_t pgprot = vma->vm_page_prot; <<
2479 int err; 2065 int err;
2480 2066
2481 if (!vm_mixed_ok(vma, pfn, mkwrite)) !! 2067 BUG_ON(!vm_mixed_ok(vma, pfn));
2482 return VM_FAULT_SIGBUS; <<
2483 2068
2484 if (addr < vma->vm_start || addr >= v 2069 if (addr < vma->vm_start || addr >= vma->vm_end)
2485 return VM_FAULT_SIGBUS; 2070 return VM_FAULT_SIGBUS;
2486 2071
2487 track_pfn_insert(vma, &pgprot, pfn); 2072 track_pfn_insert(vma, &pgprot, pfn);
2488 2073
2489 if (!pfn_modify_allowed(pfn_t_to_pfn( 2074 if (!pfn_modify_allowed(pfn_t_to_pfn(pfn), pgprot))
2490 return VM_FAULT_SIGBUS; 2075 return VM_FAULT_SIGBUS;
2491 2076
2492 /* 2077 /*
2493 * If we don't have pte special, then 2078 * If we don't have pte special, then we have to use the pfn_valid()
2494 * based VM_MIXEDMAP scheme (see vm_n 2079 * based VM_MIXEDMAP scheme (see vm_normal_page), and thus we *must*
2495 * refcount the page if pfn_valid is 2080 * refcount the page if pfn_valid is true (hence insert_page rather
2496 * than insert_pfn). If a zero_pfn w 2081 * than insert_pfn). If a zero_pfn were inserted into a VM_MIXEDMAP
2497 * without pte special, it would ther 2082 * without pte special, it would there be refcounted as a normal page.
2498 */ 2083 */
2499 if (!IS_ENABLED(CONFIG_ARCH_HAS_PTE_S 2084 if (!IS_ENABLED(CONFIG_ARCH_HAS_PTE_SPECIAL) &&
2500 !pfn_t_devmap(pfn) && pfn_t_valid 2085 !pfn_t_devmap(pfn) && pfn_t_valid(pfn)) {
2501 struct page *page; 2086 struct page *page;
2502 2087
2503 /* 2088 /*
2504 * At this point we are commi 2089 * At this point we are committed to insert_page()
2505 * regardless of whether the 2090 * regardless of whether the caller specified flags that
2506 * result in pfn_t_has_page() 2091 * result in pfn_t_has_page() == false.
2507 */ 2092 */
2508 page = pfn_to_page(pfn_t_to_p 2093 page = pfn_to_page(pfn_t_to_pfn(pfn));
2509 err = insert_page(vma, addr, 2094 err = insert_page(vma, addr, page, pgprot);
2510 } else { 2095 } else {
2511 return insert_pfn(vma, addr, 2096 return insert_pfn(vma, addr, pfn, pgprot, mkwrite);
2512 } 2097 }
2513 2098
2514 if (err == -ENOMEM) 2099 if (err == -ENOMEM)
2515 return VM_FAULT_OOM; 2100 return VM_FAULT_OOM;
2516 if (err < 0 && err != -EBUSY) 2101 if (err < 0 && err != -EBUSY)
2517 return VM_FAULT_SIGBUS; 2102 return VM_FAULT_SIGBUS;
2518 2103
2519 return VM_FAULT_NOPAGE; 2104 return VM_FAULT_NOPAGE;
2520 } 2105 }
2521 2106
>> 2107 /**
>> 2108 * vmf_insert_mixed_prot - insert single pfn into user vma with specified pgprot
>> 2109 * @vma: user vma to map to
>> 2110 * @addr: target user address of this page
>> 2111 * @pfn: source kernel pfn
>> 2112 * @pgprot: pgprot flags for the inserted page
>> 2113 *
>> 2114 * This is exactly like vmf_insert_mixed(), except that it allows drivers
>> 2115 * to override pgprot on a per-page basis.
>> 2116 *
>> 2117 * Typically this function should be used by drivers to set caching- and
>> 2118 * encryption bits different than those of @vma->vm_page_prot, because
>> 2119 * the caching- or encryption mode may not be known at mmap() time.
>> 2120 * This is ok as long as @vma->vm_page_prot is not used by the core vm
>> 2121 * to set caching and encryption bits for those vmas (except for COW pages).
>> 2122 * This is ensured by core vm only modifying these page table entries using
>> 2123 * functions that don't touch caching- or encryption bits, using pte_modify()
>> 2124 * if needed. (See for example mprotect()).
>> 2125 * Also when new page-table entries are created, this is only done using the
>> 2126 * fault() callback, and never using the value of vma->vm_page_prot,
>> 2127 * except for page-table entries that point to anonymous pages as the result
>> 2128 * of COW.
>> 2129 *
>> 2130 * Context: Process context. May allocate using %GFP_KERNEL.
>> 2131 * Return: vm_fault_t value.
>> 2132 */
>> 2133 vm_fault_t vmf_insert_mixed_prot(struct vm_area_struct *vma, unsigned long addr,
>> 2134 pfn_t pfn, pgprot_t pgprot)
>> 2135 {
>> 2136 return __vm_insert_mixed(vma, addr, pfn, pgprot, false);
>> 2137 }
>> 2138 EXPORT_SYMBOL(vmf_insert_mixed_prot);
>> 2139
2522 vm_fault_t vmf_insert_mixed(struct vm_area_st 2140 vm_fault_t vmf_insert_mixed(struct vm_area_struct *vma, unsigned long addr,
2523 pfn_t pfn) 2141 pfn_t pfn)
2524 { 2142 {
2525 return __vm_insert_mixed(vma, addr, p !! 2143 return __vm_insert_mixed(vma, addr, pfn, vma->vm_page_prot, false);
2526 } 2144 }
2527 EXPORT_SYMBOL(vmf_insert_mixed); 2145 EXPORT_SYMBOL(vmf_insert_mixed);
2528 2146
2529 /* 2147 /*
2530 * If the insertion of PTE failed because so 2148 * If the insertion of PTE failed because someone else already added a
2531 * different entry in the mean time, we trea 2149 * different entry in the mean time, we treat that as success as we assume
2532 * the same entry was actually inserted. 2150 * the same entry was actually inserted.
2533 */ 2151 */
2534 vm_fault_t vmf_insert_mixed_mkwrite(struct vm 2152 vm_fault_t vmf_insert_mixed_mkwrite(struct vm_area_struct *vma,
2535 unsigned long addr, pfn_t pfn 2153 unsigned long addr, pfn_t pfn)
2536 { 2154 {
2537 return __vm_insert_mixed(vma, addr, p !! 2155 return __vm_insert_mixed(vma, addr, pfn, vma->vm_page_prot, true);
2538 } 2156 }
>> 2157 EXPORT_SYMBOL(vmf_insert_mixed_mkwrite);
2539 2158
2540 /* 2159 /*
2541 * maps a range of physical memory into the r 2160 * maps a range of physical memory into the requested pages. the old
2542 * mappings are removed. any references to no 2161 * mappings are removed. any references to nonexistent pages results
2543 * in null mappings (currently treated as "co 2162 * in null mappings (currently treated as "copy-on-access")
2544 */ 2163 */
2545 static int remap_pte_range(struct mm_struct * 2164 static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd,
2546 unsigned long addr, u 2165 unsigned long addr, unsigned long end,
2547 unsigned long pfn, pg 2166 unsigned long pfn, pgprot_t prot)
2548 { 2167 {
2549 pte_t *pte, *mapped_pte; 2168 pte_t *pte, *mapped_pte;
2550 spinlock_t *ptl; 2169 spinlock_t *ptl;
2551 int err = 0; 2170 int err = 0;
2552 2171
2553 mapped_pte = pte = pte_alloc_map_lock 2172 mapped_pte = pte = pte_alloc_map_lock(mm, pmd, addr, &ptl);
2554 if (!pte) 2173 if (!pte)
2555 return -ENOMEM; 2174 return -ENOMEM;
2556 arch_enter_lazy_mmu_mode(); 2175 arch_enter_lazy_mmu_mode();
2557 do { 2176 do {
2558 BUG_ON(!pte_none(ptep_get(pte !! 2177 BUG_ON(!pte_none(*pte));
2559 if (!pfn_modify_allowed(pfn, 2178 if (!pfn_modify_allowed(pfn, prot)) {
2560 err = -EACCES; 2179 err = -EACCES;
2561 break; 2180 break;
2562 } 2181 }
2563 set_pte_at(mm, addr, pte, pte 2182 set_pte_at(mm, addr, pte, pte_mkspecial(pfn_pte(pfn, prot)));
2564 pfn++; 2183 pfn++;
2565 } while (pte++, addr += PAGE_SIZE, ad 2184 } while (pte++, addr += PAGE_SIZE, addr != end);
2566 arch_leave_lazy_mmu_mode(); 2185 arch_leave_lazy_mmu_mode();
2567 pte_unmap_unlock(mapped_pte, ptl); 2186 pte_unmap_unlock(mapped_pte, ptl);
2568 return err; 2187 return err;
2569 } 2188 }
2570 2189
2571 static inline int remap_pmd_range(struct mm_s 2190 static inline int remap_pmd_range(struct mm_struct *mm, pud_t *pud,
2572 unsigned long addr, u 2191 unsigned long addr, unsigned long end,
2573 unsigned long pfn, pg 2192 unsigned long pfn, pgprot_t prot)
2574 { 2193 {
2575 pmd_t *pmd; 2194 pmd_t *pmd;
2576 unsigned long next; 2195 unsigned long next;
2577 int err; 2196 int err;
2578 2197
2579 pfn -= addr >> PAGE_SHIFT; 2198 pfn -= addr >> PAGE_SHIFT;
2580 pmd = pmd_alloc(mm, pud, addr); 2199 pmd = pmd_alloc(mm, pud, addr);
2581 if (!pmd) 2200 if (!pmd)
2582 return -ENOMEM; 2201 return -ENOMEM;
2583 VM_BUG_ON(pmd_trans_huge(*pmd)); 2202 VM_BUG_ON(pmd_trans_huge(*pmd));
2584 do { 2203 do {
2585 next = pmd_addr_end(addr, end 2204 next = pmd_addr_end(addr, end);
2586 err = remap_pte_range(mm, pmd 2205 err = remap_pte_range(mm, pmd, addr, next,
2587 pfn + (addr > 2206 pfn + (addr >> PAGE_SHIFT), prot);
2588 if (err) 2207 if (err)
2589 return err; 2208 return err;
2590 } while (pmd++, addr = next, addr != 2209 } while (pmd++, addr = next, addr != end);
2591 return 0; 2210 return 0;
2592 } 2211 }
2593 2212
2594 static inline int remap_pud_range(struct mm_s 2213 static inline int remap_pud_range(struct mm_struct *mm, p4d_t *p4d,
2595 unsigned long addr, u 2214 unsigned long addr, unsigned long end,
2596 unsigned long pfn, pg 2215 unsigned long pfn, pgprot_t prot)
2597 { 2216 {
2598 pud_t *pud; 2217 pud_t *pud;
2599 unsigned long next; 2218 unsigned long next;
2600 int err; 2219 int err;
2601 2220
2602 pfn -= addr >> PAGE_SHIFT; 2221 pfn -= addr >> PAGE_SHIFT;
2603 pud = pud_alloc(mm, p4d, addr); 2222 pud = pud_alloc(mm, p4d, addr);
2604 if (!pud) 2223 if (!pud)
2605 return -ENOMEM; 2224 return -ENOMEM;
2606 do { 2225 do {
2607 next = pud_addr_end(addr, end 2226 next = pud_addr_end(addr, end);
2608 err = remap_pmd_range(mm, pud 2227 err = remap_pmd_range(mm, pud, addr, next,
2609 pfn + (addr > 2228 pfn + (addr >> PAGE_SHIFT), prot);
2610 if (err) 2229 if (err)
2611 return err; 2230 return err;
2612 } while (pud++, addr = next, addr != 2231 } while (pud++, addr = next, addr != end);
2613 return 0; 2232 return 0;
2614 } 2233 }
2615 2234
2616 static inline int remap_p4d_range(struct mm_s 2235 static inline int remap_p4d_range(struct mm_struct *mm, pgd_t *pgd,
2617 unsigned long addr, u 2236 unsigned long addr, unsigned long end,
2618 unsigned long pfn, pg 2237 unsigned long pfn, pgprot_t prot)
2619 { 2238 {
2620 p4d_t *p4d; 2239 p4d_t *p4d;
2621 unsigned long next; 2240 unsigned long next;
2622 int err; 2241 int err;
2623 2242
2624 pfn -= addr >> PAGE_SHIFT; 2243 pfn -= addr >> PAGE_SHIFT;
2625 p4d = p4d_alloc(mm, pgd, addr); 2244 p4d = p4d_alloc(mm, pgd, addr);
2626 if (!p4d) 2245 if (!p4d)
2627 return -ENOMEM; 2246 return -ENOMEM;
2628 do { 2247 do {
2629 next = p4d_addr_end(addr, end 2248 next = p4d_addr_end(addr, end);
2630 err = remap_pud_range(mm, p4d 2249 err = remap_pud_range(mm, p4d, addr, next,
2631 pfn + (addr > 2250 pfn + (addr >> PAGE_SHIFT), prot);
2632 if (err) 2251 if (err)
2633 return err; 2252 return err;
2634 } while (p4d++, addr = next, addr != 2253 } while (p4d++, addr = next, addr != end);
2635 return 0; 2254 return 0;
2636 } 2255 }
2637 2256
2638 static int remap_pfn_range_internal(struct vm !! 2257 /**
2639 unsigned long pfn, unsigned l !! 2258 * remap_pfn_range - remap kernel memory to userspace
>> 2259 * @vma: user vma to map to
>> 2260 * @addr: target page aligned user address to start at
>> 2261 * @pfn: page frame number of kernel physical memory address
>> 2262 * @size: size of mapping area
>> 2263 * @prot: page protection flags for this mapping
>> 2264 *
>> 2265 * Note: this is only safe if the mm semaphore is held when called.
>> 2266 *
>> 2267 * Return: %0 on success, negative error code otherwise.
>> 2268 */
>> 2269 int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr,
>> 2270 unsigned long pfn, unsigned long size, pgprot_t prot)
2640 { 2271 {
2641 pgd_t *pgd; 2272 pgd_t *pgd;
2642 unsigned long next; 2273 unsigned long next;
2643 unsigned long end = addr + PAGE_ALIGN 2274 unsigned long end = addr + PAGE_ALIGN(size);
2644 struct mm_struct *mm = vma->vm_mm; 2275 struct mm_struct *mm = vma->vm_mm;
>> 2276 unsigned long remap_pfn = pfn;
2645 int err; 2277 int err;
2646 2278
2647 if (WARN_ON_ONCE(!PAGE_ALIGNED(addr)) 2279 if (WARN_ON_ONCE(!PAGE_ALIGNED(addr)))
2648 return -EINVAL; 2280 return -EINVAL;
2649 2281
2650 /* 2282 /*
2651 * Physically remapped pages are spec 2283 * Physically remapped pages are special. Tell the
2652 * rest of the world about it: 2284 * rest of the world about it:
2653 * VM_IO tells people not to look a 2285 * VM_IO tells people not to look at these pages
2654 * (accesses can have side effec 2286 * (accesses can have side effects).
2655 * VM_PFNMAP tells the core MM that 2287 * VM_PFNMAP tells the core MM that the base pages are just
2656 * raw PFN mappings, and do not 2288 * raw PFN mappings, and do not have a "struct page" associated
2657 * with them. 2289 * with them.
2658 * VM_DONTEXPAND 2290 * VM_DONTEXPAND
2659 * Disable vma merging and expan 2291 * Disable vma merging and expanding with mremap().
2660 * VM_DONTDUMP 2292 * VM_DONTDUMP
2661 * Omit vma from core dump, even 2293 * Omit vma from core dump, even when VM_IO turned off.
2662 * 2294 *
2663 * There's a horrible special case to 2295 * There's a horrible special case to handle copy-on-write
2664 * behaviour that some programs depen 2296 * behaviour that some programs depend on. We mark the "original"
2665 * un-COW'ed pages by matching them u 2297 * un-COW'ed pages by matching them up with "vma->vm_pgoff".
2666 * See vm_normal_page() for details. 2298 * See vm_normal_page() for details.
2667 */ 2299 */
2668 if (is_cow_mapping(vma->vm_flags)) { 2300 if (is_cow_mapping(vma->vm_flags)) {
2669 if (addr != vma->vm_start || 2301 if (addr != vma->vm_start || end != vma->vm_end)
2670 return -EINVAL; 2302 return -EINVAL;
2671 vma->vm_pgoff = pfn; 2303 vma->vm_pgoff = pfn;
2672 } 2304 }
2673 2305
2674 vm_flags_set(vma, VM_IO | VM_PFNMAP | !! 2306 err = track_pfn_remap(vma, &prot, remap_pfn, addr, PAGE_ALIGN(size));
>> 2307 if (err)
>> 2308 return -EINVAL;
>> 2309
>> 2310 vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP;
2675 2311
2676 BUG_ON(addr >= end); 2312 BUG_ON(addr >= end);
2677 pfn -= addr >> PAGE_SHIFT; 2313 pfn -= addr >> PAGE_SHIFT;
2678 pgd = pgd_offset(mm, addr); 2314 pgd = pgd_offset(mm, addr);
2679 flush_cache_range(vma, addr, end); 2315 flush_cache_range(vma, addr, end);
2680 do { 2316 do {
2681 next = pgd_addr_end(addr, end 2317 next = pgd_addr_end(addr, end);
2682 err = remap_p4d_range(mm, pgd 2318 err = remap_p4d_range(mm, pgd, addr, next,
2683 pfn + (addr > 2319 pfn + (addr >> PAGE_SHIFT), prot);
2684 if (err) 2320 if (err)
2685 return err; !! 2321 break;
2686 } while (pgd++, addr = next, addr != 2322 } while (pgd++, addr = next, addr != end);
2687 2323
2688 return 0; <<
2689 } <<
2690 <<
2691 /* <<
2692 * Variant of remap_pfn_range that does not c <<
2693 * must have pre-validated the caching bits o <<
2694 */ <<
2695 int remap_pfn_range_notrack(struct vm_area_st <<
2696 unsigned long pfn, unsigned l <<
2697 { <<
2698 int error = remap_pfn_range_internal( <<
2699 <<
2700 if (!error) <<
2701 return 0; <<
2702 <<
2703 /* <<
2704 * A partial pfn range mapping is dan <<
2705 * maintain page reference counts, an <<
2706 * pages due to the error. So zap it <<
2707 */ <<
2708 zap_page_range_single(vma, addr, size <<
2709 return error; <<
2710 } <<
2711 <<
2712 /** <<
2713 * remap_pfn_range - remap kernel memory to u <<
2714 * @vma: user vma to map to <<
2715 * @addr: target page aligned user address to <<
2716 * @pfn: page frame number of kernel physical <<
2717 * @size: size of mapping area <<
2718 * @prot: page protection flags for this mapp <<
2719 * <<
2720 * Note: this is only safe if the mm semaphor <<
2721 * <<
2722 * Return: %0 on success, negative error code <<
2723 */ <<
2724 int remap_pfn_range(struct vm_area_struct *vm <<
2725 unsigned long pfn, unsign <<
2726 { <<
2727 int err; <<
2728 <<
2729 err = track_pfn_remap(vma, &prot, pfn <<
2730 if (err) 2324 if (err)
2731 return -EINVAL; !! 2325 untrack_pfn(vma, remap_pfn, PAGE_ALIGN(size));
2732 2326
2733 err = remap_pfn_range_notrack(vma, ad <<
2734 if (err) <<
2735 untrack_pfn(vma, pfn, PAGE_AL <<
2736 return err; 2327 return err;
2737 } 2328 }
2738 EXPORT_SYMBOL(remap_pfn_range); 2329 EXPORT_SYMBOL(remap_pfn_range);
2739 2330
2740 /** 2331 /**
2741 * vm_iomap_memory - remap memory to userspac 2332 * vm_iomap_memory - remap memory to userspace
2742 * @vma: user vma to map to 2333 * @vma: user vma to map to
2743 * @start: start of the physical memory to be 2334 * @start: start of the physical memory to be mapped
2744 * @len: size of area 2335 * @len: size of area
2745 * 2336 *
2746 * This is a simplified io_remap_pfn_range() 2337 * This is a simplified io_remap_pfn_range() for common driver use. The
2747 * driver just needs to give us the physical 2338 * driver just needs to give us the physical memory range to be mapped,
2748 * we'll figure out the rest from the vma inf 2339 * we'll figure out the rest from the vma information.
2749 * 2340 *
2750 * NOTE! Some drivers might want to tweak vma 2341 * NOTE! Some drivers might want to tweak vma->vm_page_prot first to get
2751 * whatever write-combining details or simila 2342 * whatever write-combining details or similar.
2752 * 2343 *
2753 * Return: %0 on success, negative error code 2344 * Return: %0 on success, negative error code otherwise.
2754 */ 2345 */
2755 int vm_iomap_memory(struct vm_area_struct *vm 2346 int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len)
2756 { 2347 {
2757 unsigned long vm_len, pfn, pages; 2348 unsigned long vm_len, pfn, pages;
2758 2349
2759 /* Check that the physical memory are 2350 /* Check that the physical memory area passed in looks valid */
2760 if (start + len < start) 2351 if (start + len < start)
2761 return -EINVAL; 2352 return -EINVAL;
2762 /* 2353 /*
2763 * You *really* shouldn't map things 2354 * You *really* shouldn't map things that aren't page-aligned,
2764 * but we've historically allowed it 2355 * but we've historically allowed it because IO memory might
2765 * just have smaller alignment. 2356 * just have smaller alignment.
2766 */ 2357 */
2767 len += start & ~PAGE_MASK; 2358 len += start & ~PAGE_MASK;
2768 pfn = start >> PAGE_SHIFT; 2359 pfn = start >> PAGE_SHIFT;
2769 pages = (len + ~PAGE_MASK) >> PAGE_SH 2360 pages = (len + ~PAGE_MASK) >> PAGE_SHIFT;
2770 if (pfn + pages < pfn) 2361 if (pfn + pages < pfn)
2771 return -EINVAL; 2362 return -EINVAL;
2772 2363
2773 /* We start the mapping 'vm_pgoff' pa 2364 /* We start the mapping 'vm_pgoff' pages into the area */
2774 if (vma->vm_pgoff > pages) 2365 if (vma->vm_pgoff > pages)
2775 return -EINVAL; 2366 return -EINVAL;
2776 pfn += vma->vm_pgoff; 2367 pfn += vma->vm_pgoff;
2777 pages -= vma->vm_pgoff; 2368 pages -= vma->vm_pgoff;
2778 2369
2779 /* Can we fit all of the mapping? */ 2370 /* Can we fit all of the mapping? */
2780 vm_len = vma->vm_end - vma->vm_start; 2371 vm_len = vma->vm_end - vma->vm_start;
2781 if (vm_len >> PAGE_SHIFT > pages) 2372 if (vm_len >> PAGE_SHIFT > pages)
2782 return -EINVAL; 2373 return -EINVAL;
2783 2374
2784 /* Ok, let it rip */ 2375 /* Ok, let it rip */
2785 return io_remap_pfn_range(vma, vma->v 2376 return io_remap_pfn_range(vma, vma->vm_start, pfn, vm_len, vma->vm_page_prot);
2786 } 2377 }
2787 EXPORT_SYMBOL(vm_iomap_memory); 2378 EXPORT_SYMBOL(vm_iomap_memory);
2788 2379
2789 static int apply_to_pte_range(struct mm_struc 2380 static int apply_to_pte_range(struct mm_struct *mm, pmd_t *pmd,
2790 unsigned 2381 unsigned long addr, unsigned long end,
2791 pte_fn_t 2382 pte_fn_t fn, void *data, bool create,
2792 pgtbl_mo 2383 pgtbl_mod_mask *mask)
2793 { 2384 {
2794 pte_t *pte, *mapped_pte; !! 2385 pte_t *pte;
2795 int err = 0; 2386 int err = 0;
2796 spinlock_t *ptl; 2387 spinlock_t *ptl;
2797 2388
2798 if (create) { 2389 if (create) {
2799 mapped_pte = pte = (mm == &in !! 2390 pte = (mm == &init_mm) ?
2800 pte_alloc_kernel_trac 2391 pte_alloc_kernel_track(pmd, addr, mask) :
2801 pte_alloc_map_lock(mm 2392 pte_alloc_map_lock(mm, pmd, addr, &ptl);
2802 if (!pte) 2393 if (!pte)
2803 return -ENOMEM; 2394 return -ENOMEM;
2804 } else { 2395 } else {
2805 mapped_pte = pte = (mm == &in !! 2396 pte = (mm == &init_mm) ?
2806 pte_offset_kernel(pmd 2397 pte_offset_kernel(pmd, addr) :
2807 pte_offset_map_lock(m 2398 pte_offset_map_lock(mm, pmd, addr, &ptl);
2808 if (!pte) <<
2809 return -EINVAL; <<
2810 } 2399 }
2811 2400
>> 2401 BUG_ON(pmd_huge(*pmd));
>> 2402
2812 arch_enter_lazy_mmu_mode(); 2403 arch_enter_lazy_mmu_mode();
2813 2404
2814 if (fn) { 2405 if (fn) {
2815 do { 2406 do {
2816 if (create || !pte_no !! 2407 if (create || !pte_none(*pte)) {
2817 err = fn(pte+ 2408 err = fn(pte++, addr, data);
2818 if (err) 2409 if (err)
2819 break 2410 break;
2820 } 2411 }
2821 } while (addr += PAGE_SIZE, a 2412 } while (addr += PAGE_SIZE, addr != end);
2822 } 2413 }
2823 *mask |= PGTBL_PTE_MODIFIED; 2414 *mask |= PGTBL_PTE_MODIFIED;
2824 2415
2825 arch_leave_lazy_mmu_mode(); 2416 arch_leave_lazy_mmu_mode();
2826 2417
2827 if (mm != &init_mm) 2418 if (mm != &init_mm)
2828 pte_unmap_unlock(mapped_pte, !! 2419 pte_unmap_unlock(pte-1, ptl);
2829 return err; 2420 return err;
2830 } 2421 }
2831 2422
2832 static int apply_to_pmd_range(struct mm_struc 2423 static int apply_to_pmd_range(struct mm_struct *mm, pud_t *pud,
2833 unsigned 2424 unsigned long addr, unsigned long end,
2834 pte_fn_t 2425 pte_fn_t fn, void *data, bool create,
2835 pgtbl_mo 2426 pgtbl_mod_mask *mask)
2836 { 2427 {
2837 pmd_t *pmd; 2428 pmd_t *pmd;
2838 unsigned long next; 2429 unsigned long next;
2839 int err = 0; 2430 int err = 0;
2840 2431
2841 BUG_ON(pud_leaf(*pud)); !! 2432 BUG_ON(pud_huge(*pud));
2842 2433
2843 if (create) { 2434 if (create) {
2844 pmd = pmd_alloc_track(mm, pud 2435 pmd = pmd_alloc_track(mm, pud, addr, mask);
2845 if (!pmd) 2436 if (!pmd)
2846 return -ENOMEM; 2437 return -ENOMEM;
2847 } else { 2438 } else {
2848 pmd = pmd_offset(pud, addr); 2439 pmd = pmd_offset(pud, addr);
2849 } 2440 }
2850 do { 2441 do {
2851 next = pmd_addr_end(addr, end 2442 next = pmd_addr_end(addr, end);
2852 if (pmd_none(*pmd) && !create !! 2443 if (create || !pmd_none_or_clear_bad(pmd)) {
2853 continue; !! 2444 err = apply_to_pte_range(mm, pmd, addr, next, fn, data,
2854 if (WARN_ON_ONCE(pmd_leaf(*pm !! 2445 create, mask);
2855 return -EINVAL; !! 2446 if (err)
2856 if (!pmd_none(*pmd) && WARN_O !! 2447 break;
2857 if (!create) <<
2858 continue; <<
2859 pmd_clear_bad(pmd); <<
2860 } 2448 }
2861 err = apply_to_pte_range(mm, <<
2862 fn, <<
2863 if (err) <<
2864 break; <<
2865 } while (pmd++, addr = next, addr != 2449 } while (pmd++, addr = next, addr != end);
2866 <<
2867 return err; 2450 return err;
2868 } 2451 }
2869 2452
2870 static int apply_to_pud_range(struct mm_struc 2453 static int apply_to_pud_range(struct mm_struct *mm, p4d_t *p4d,
2871 unsigned 2454 unsigned long addr, unsigned long end,
2872 pte_fn_t 2455 pte_fn_t fn, void *data, bool create,
2873 pgtbl_mo 2456 pgtbl_mod_mask *mask)
2874 { 2457 {
2875 pud_t *pud; 2458 pud_t *pud;
2876 unsigned long next; 2459 unsigned long next;
2877 int err = 0; 2460 int err = 0;
2878 2461
2879 if (create) { 2462 if (create) {
2880 pud = pud_alloc_track(mm, p4d 2463 pud = pud_alloc_track(mm, p4d, addr, mask);
2881 if (!pud) 2464 if (!pud)
2882 return -ENOMEM; 2465 return -ENOMEM;
2883 } else { 2466 } else {
2884 pud = pud_offset(p4d, addr); 2467 pud = pud_offset(p4d, addr);
2885 } 2468 }
2886 do { 2469 do {
2887 next = pud_addr_end(addr, end 2470 next = pud_addr_end(addr, end);
2888 if (pud_none(*pud) && !create !! 2471 if (create || !pud_none_or_clear_bad(pud)) {
2889 continue; !! 2472 err = apply_to_pmd_range(mm, pud, addr, next, fn, data,
2890 if (WARN_ON_ONCE(pud_leaf(*pu !! 2473 create, mask);
2891 return -EINVAL; !! 2474 if (err)
2892 if (!pud_none(*pud) && WARN_O !! 2475 break;
2893 if (!create) <<
2894 continue; <<
2895 pud_clear_bad(pud); <<
2896 } 2476 }
2897 err = apply_to_pmd_range(mm, <<
2898 fn, <<
2899 if (err) <<
2900 break; <<
2901 } while (pud++, addr = next, addr != 2477 } while (pud++, addr = next, addr != end);
2902 <<
2903 return err; 2478 return err;
2904 } 2479 }
2905 2480
2906 static int apply_to_p4d_range(struct mm_struc 2481 static int apply_to_p4d_range(struct mm_struct *mm, pgd_t *pgd,
2907 unsigned 2482 unsigned long addr, unsigned long end,
2908 pte_fn_t 2483 pte_fn_t fn, void *data, bool create,
2909 pgtbl_mo 2484 pgtbl_mod_mask *mask)
2910 { 2485 {
2911 p4d_t *p4d; 2486 p4d_t *p4d;
2912 unsigned long next; 2487 unsigned long next;
2913 int err = 0; 2488 int err = 0;
2914 2489
2915 if (create) { 2490 if (create) {
2916 p4d = p4d_alloc_track(mm, pgd 2491 p4d = p4d_alloc_track(mm, pgd, addr, mask);
2917 if (!p4d) 2492 if (!p4d)
2918 return -ENOMEM; 2493 return -ENOMEM;
2919 } else { 2494 } else {
2920 p4d = p4d_offset(pgd, addr); 2495 p4d = p4d_offset(pgd, addr);
2921 } 2496 }
2922 do { 2497 do {
2923 next = p4d_addr_end(addr, end 2498 next = p4d_addr_end(addr, end);
2924 if (p4d_none(*p4d) && !create !! 2499 if (create || !p4d_none_or_clear_bad(p4d)) {
2925 continue; !! 2500 err = apply_to_pud_range(mm, p4d, addr, next, fn, data,
2926 if (WARN_ON_ONCE(p4d_leaf(*p4 !! 2501 create, mask);
2927 return -EINVAL; !! 2502 if (err)
2928 if (!p4d_none(*p4d) && WARN_O !! 2503 break;
2929 if (!create) <<
2930 continue; <<
2931 p4d_clear_bad(p4d); <<
2932 } 2504 }
2933 err = apply_to_pud_range(mm, <<
2934 fn, <<
2935 if (err) <<
2936 break; <<
2937 } while (p4d++, addr = next, addr != 2505 } while (p4d++, addr = next, addr != end);
2938 <<
2939 return err; 2506 return err;
2940 } 2507 }
2941 2508
2942 static int __apply_to_page_range(struct mm_st 2509 static int __apply_to_page_range(struct mm_struct *mm, unsigned long addr,
2943 unsigned lon 2510 unsigned long size, pte_fn_t fn,
2944 void *data, 2511 void *data, bool create)
2945 { 2512 {
2946 pgd_t *pgd; 2513 pgd_t *pgd;
2947 unsigned long start = addr, next; 2514 unsigned long start = addr, next;
2948 unsigned long end = addr + size; 2515 unsigned long end = addr + size;
2949 pgtbl_mod_mask mask = 0; 2516 pgtbl_mod_mask mask = 0;
2950 int err = 0; 2517 int err = 0;
2951 2518
2952 if (WARN_ON(addr >= end)) 2519 if (WARN_ON(addr >= end))
2953 return -EINVAL; 2520 return -EINVAL;
2954 2521
2955 pgd = pgd_offset(mm, addr); 2522 pgd = pgd_offset(mm, addr);
2956 do { 2523 do {
2957 next = pgd_addr_end(addr, end 2524 next = pgd_addr_end(addr, end);
2958 if (pgd_none(*pgd) && !create !! 2525 if (!create && pgd_none_or_clear_bad(pgd))
2959 continue; 2526 continue;
2960 if (WARN_ON_ONCE(pgd_leaf(*pg !! 2527 err = apply_to_p4d_range(mm, pgd, addr, next, fn, data, create, &mask);
2961 return -EINVAL; <<
2962 if (!pgd_none(*pgd) && WARN_O <<
2963 if (!create) <<
2964 continue; <<
2965 pgd_clear_bad(pgd); <<
2966 } <<
2967 err = apply_to_p4d_range(mm, <<
2968 fn, <<
2969 if (err) 2528 if (err)
2970 break; 2529 break;
2971 } while (pgd++, addr = next, addr != 2530 } while (pgd++, addr = next, addr != end);
2972 2531
2973 if (mask & ARCH_PAGE_TABLE_SYNC_MASK) 2532 if (mask & ARCH_PAGE_TABLE_SYNC_MASK)
2974 arch_sync_kernel_mappings(sta 2533 arch_sync_kernel_mappings(start, start + size);
2975 2534
2976 return err; 2535 return err;
2977 } 2536 }
2978 2537
2979 /* 2538 /*
2980 * Scan a region of virtual memory, filling i 2539 * Scan a region of virtual memory, filling in page tables as necessary
2981 * and calling a provided function on each le 2540 * and calling a provided function on each leaf page table.
2982 */ 2541 */
2983 int apply_to_page_range(struct mm_struct *mm, 2542 int apply_to_page_range(struct mm_struct *mm, unsigned long addr,
2984 unsigned long size, p 2543 unsigned long size, pte_fn_t fn, void *data)
2985 { 2544 {
2986 return __apply_to_page_range(mm, addr 2545 return __apply_to_page_range(mm, addr, size, fn, data, true);
2987 } 2546 }
2988 EXPORT_SYMBOL_GPL(apply_to_page_range); 2547 EXPORT_SYMBOL_GPL(apply_to_page_range);
2989 2548
2990 /* 2549 /*
2991 * Scan a region of virtual memory, calling a 2550 * Scan a region of virtual memory, calling a provided function on
2992 * each leaf page table where it exists. 2551 * each leaf page table where it exists.
2993 * 2552 *
2994 * Unlike apply_to_page_range, this does _not 2553 * Unlike apply_to_page_range, this does _not_ fill in page tables
2995 * where they are absent. 2554 * where they are absent.
2996 */ 2555 */
2997 int apply_to_existing_page_range(struct mm_st 2556 int apply_to_existing_page_range(struct mm_struct *mm, unsigned long addr,
2998 unsigned lon 2557 unsigned long size, pte_fn_t fn, void *data)
2999 { 2558 {
3000 return __apply_to_page_range(mm, addr 2559 return __apply_to_page_range(mm, addr, size, fn, data, false);
3001 } 2560 }
3002 EXPORT_SYMBOL_GPL(apply_to_existing_page_rang 2561 EXPORT_SYMBOL_GPL(apply_to_existing_page_range);
3003 2562
3004 /* 2563 /*
3005 * handle_pte_fault chooses page fault handle 2564 * handle_pte_fault chooses page fault handler according to an entry which was
3006 * read non-atomically. Before making any co 2565 * read non-atomically. Before making any commitment, on those architectures
3007 * or configurations (e.g. i386 with PAE) whi 2566 * or configurations (e.g. i386 with PAE) which might give a mix of unmatched
3008 * parts, do_swap_page must check under lock 2567 * parts, do_swap_page must check under lock before unmapping the pte and
3009 * proceeding (but do_wp_page is only called 2568 * proceeding (but do_wp_page is only called after already making such a check;
3010 * and do_anonymous_page can safely check lat 2569 * and do_anonymous_page can safely check later on).
3011 */ 2570 */
3012 static inline int pte_unmap_same(struct vm_fa !! 2571 static inline int pte_unmap_same(struct mm_struct *mm, pmd_t *pmd,
>> 2572 pte_t *page_table, pte_t orig_pte)
3013 { 2573 {
3014 int same = 1; 2574 int same = 1;
3015 #if defined(CONFIG_SMP) || defined(CONFIG_PRE 2575 #if defined(CONFIG_SMP) || defined(CONFIG_PREEMPTION)
3016 if (sizeof(pte_t) > sizeof(unsigned l 2576 if (sizeof(pte_t) > sizeof(unsigned long)) {
3017 spin_lock(vmf->ptl); !! 2577 spinlock_t *ptl = pte_lockptr(mm, pmd);
3018 same = pte_same(ptep_get(vmf- !! 2578 spin_lock(ptl);
3019 spin_unlock(vmf->ptl); !! 2579 same = pte_same(*page_table, orig_pte);
>> 2580 spin_unlock(ptl);
3020 } 2581 }
3021 #endif 2582 #endif
3022 pte_unmap(vmf->pte); !! 2583 pte_unmap(page_table);
3023 vmf->pte = NULL; <<
3024 return same; 2584 return same;
3025 } 2585 }
3026 2586
3027 /* !! 2587 static inline bool cow_user_page(struct page *dst, struct page *src,
3028 * Return: !! 2588 struct vm_fault *vmf)
3029 * 0: copied succeeded <<
3030 * -EHWPOISON: copy failed due to hw <<
3031 * -EAGAIN: copied failed (some o <<
3032 */ <<
3033 static inline int __wp_page_copy_user(struct <<
3034 struct <<
3035 { 2589 {
3036 int ret; !! 2590 bool ret;
3037 void *kaddr; 2591 void *kaddr;
3038 void __user *uaddr; 2592 void __user *uaddr;
>> 2593 bool locked = false;
3039 struct vm_area_struct *vma = vmf->vma 2594 struct vm_area_struct *vma = vmf->vma;
3040 struct mm_struct *mm = vma->vm_mm; 2595 struct mm_struct *mm = vma->vm_mm;
3041 unsigned long addr = vmf->address; 2596 unsigned long addr = vmf->address;
3042 2597
3043 if (likely(src)) { 2598 if (likely(src)) {
3044 if (copy_mc_user_highpage(dst !! 2599 copy_user_highpage(dst, src, addr, vma);
3045 return -EHWPOISON; !! 2600 return true;
3046 return 0; <<
3047 } 2601 }
3048 2602
3049 /* 2603 /*
3050 * If the source page was a PFN mappi 2604 * If the source page was a PFN mapping, we don't have
3051 * a "struct page" for it. We do a be 2605 * a "struct page" for it. We do a best-effort copy by
3052 * just copying from the original use 2606 * just copying from the original user address. If that
3053 * fails, we just zero-fill it. Live 2607 * fails, we just zero-fill it. Live with it.
3054 */ 2608 */
3055 kaddr = kmap_local_page(dst); !! 2609 kaddr = kmap_atomic(dst);
3056 pagefault_disable(); <<
3057 uaddr = (void __user *)(addr & PAGE_M 2610 uaddr = (void __user *)(addr & PAGE_MASK);
3058 2611
3059 /* 2612 /*
3060 * On architectures with software "ac 2613 * On architectures with software "accessed" bits, we would
3061 * take a double page fault, so mark 2614 * take a double page fault, so mark it accessed here.
3062 */ 2615 */
3063 vmf->pte = NULL; !! 2616 if (arch_faults_on_old_pte() && !pte_young(vmf->orig_pte)) {
3064 if (!arch_has_hw_pte_young() && !pte_ <<
3065 pte_t entry; 2617 pte_t entry;
3066 2618
3067 vmf->pte = pte_offset_map_loc 2619 vmf->pte = pte_offset_map_lock(mm, vmf->pmd, addr, &vmf->ptl);
3068 if (unlikely(!vmf->pte || !pt !! 2620 locked = true;
>> 2621 if (!likely(pte_same(*vmf->pte, vmf->orig_pte))) {
3069 /* 2622 /*
3070 * Other thread has a 2623 * Other thread has already handled the fault
3071 * and update local t 2624 * and update local tlb only
3072 */ 2625 */
3073 if (vmf->pte) !! 2626 update_mmu_tlb(vma, addr, vmf->pte);
3074 update_mmu_tl !! 2627 ret = false;
3075 ret = -EAGAIN; <<
3076 goto pte_unlock; 2628 goto pte_unlock;
3077 } 2629 }
3078 2630
3079 entry = pte_mkyoung(vmf->orig 2631 entry = pte_mkyoung(vmf->orig_pte);
3080 if (ptep_set_access_flags(vma 2632 if (ptep_set_access_flags(vma, addr, vmf->pte, entry, 0))
3081 update_mmu_cache_rang !! 2633 update_mmu_cache(vma, addr, vmf->pte);
3082 } 2634 }
3083 2635
3084 /* 2636 /*
3085 * This really shouldn't fail, becaus 2637 * This really shouldn't fail, because the page is there
3086 * in the page tables. But it might j 2638 * in the page tables. But it might just be unreadable,
3087 * in which case we just give up and 2639 * in which case we just give up and fill the result with
3088 * zeroes. 2640 * zeroes.
3089 */ 2641 */
3090 if (__copy_from_user_inatomic(kaddr, 2642 if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE)) {
3091 if (vmf->pte) !! 2643 if (locked)
3092 goto warn; 2644 goto warn;
3093 2645
3094 /* Re-validate under PTL if t 2646 /* Re-validate under PTL if the page is still mapped */
3095 vmf->pte = pte_offset_map_loc 2647 vmf->pte = pte_offset_map_lock(mm, vmf->pmd, addr, &vmf->ptl);
3096 if (unlikely(!vmf->pte || !pt !! 2648 locked = true;
>> 2649 if (!likely(pte_same(*vmf->pte, vmf->orig_pte))) {
3097 /* The PTE changed un 2650 /* The PTE changed under us, update local tlb */
3098 if (vmf->pte) !! 2651 update_mmu_tlb(vma, addr, vmf->pte);
3099 update_mmu_tl !! 2652 ret = false;
3100 ret = -EAGAIN; <<
3101 goto pte_unlock; 2653 goto pte_unlock;
3102 } 2654 }
3103 2655
3104 /* 2656 /*
3105 * The same page can be mappe 2657 * The same page can be mapped back since last copy attempt.
3106 * Try to copy again under PT 2658 * Try to copy again under PTL.
3107 */ 2659 */
3108 if (__copy_from_user_inatomic 2660 if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE)) {
3109 /* 2661 /*
3110 * Give a warn in cas 2662 * Give a warn in case there can be some obscure
3111 * use-case 2663 * use-case
3112 */ 2664 */
3113 warn: 2665 warn:
3114 WARN_ON_ONCE(1); 2666 WARN_ON_ONCE(1);
3115 clear_page(kaddr); 2667 clear_page(kaddr);
3116 } 2668 }
3117 } 2669 }
3118 2670
3119 ret = 0; !! 2671 ret = true;
3120 2672
3121 pte_unlock: 2673 pte_unlock:
3122 if (vmf->pte) !! 2674 if (locked)
3123 pte_unmap_unlock(vmf->pte, vm 2675 pte_unmap_unlock(vmf->pte, vmf->ptl);
3124 pagefault_enable(); !! 2676 kunmap_atomic(kaddr);
3125 kunmap_local(kaddr); <<
3126 flush_dcache_page(dst); 2677 flush_dcache_page(dst);
3127 2678
3128 return ret; 2679 return ret;
3129 } 2680 }
3130 2681
3131 static gfp_t __get_fault_gfp_mask(struct vm_a 2682 static gfp_t __get_fault_gfp_mask(struct vm_area_struct *vma)
3132 { 2683 {
3133 struct file *vm_file = vma->vm_file; 2684 struct file *vm_file = vma->vm_file;
3134 2685
3135 if (vm_file) 2686 if (vm_file)
3136 return mapping_gfp_mask(vm_fi 2687 return mapping_gfp_mask(vm_file->f_mapping) | __GFP_FS | __GFP_IO;
3137 2688
3138 /* 2689 /*
3139 * Special mappings (e.g. VDSO) do no 2690 * Special mappings (e.g. VDSO) do not have any file so fake
3140 * a default GFP_KERNEL for them. 2691 * a default GFP_KERNEL for them.
3141 */ 2692 */
3142 return GFP_KERNEL; 2693 return GFP_KERNEL;
3143 } 2694 }
3144 2695
3145 /* 2696 /*
3146 * Notify the address space that the page is 2697 * Notify the address space that the page is about to become writable so that
3147 * it can prohibit this or wait for the page 2698 * it can prohibit this or wait for the page to get into an appropriate state.
3148 * 2699 *
3149 * We do this without the lock held, so that 2700 * We do this without the lock held, so that it can sleep if it needs to.
3150 */ 2701 */
3151 static vm_fault_t do_page_mkwrite(struct vm_f !! 2702 static vm_fault_t do_page_mkwrite(struct vm_fault *vmf)
3152 { 2703 {
3153 vm_fault_t ret; 2704 vm_fault_t ret;
>> 2705 struct page *page = vmf->page;
3154 unsigned int old_flags = vmf->flags; 2706 unsigned int old_flags = vmf->flags;
3155 2707
3156 vmf->flags = FAULT_FLAG_WRITE|FAULT_F 2708 vmf->flags = FAULT_FLAG_WRITE|FAULT_FLAG_MKWRITE;
3157 2709
3158 if (vmf->vma->vm_file && 2710 if (vmf->vma->vm_file &&
3159 IS_SWAPFILE(vmf->vma->vm_file->f_ 2711 IS_SWAPFILE(vmf->vma->vm_file->f_mapping->host))
3160 return VM_FAULT_SIGBUS; 2712 return VM_FAULT_SIGBUS;
3161 2713
3162 ret = vmf->vma->vm_ops->page_mkwrite( 2714 ret = vmf->vma->vm_ops->page_mkwrite(vmf);
3163 /* Restore original flags so that cal 2715 /* Restore original flags so that caller is not surprised */
3164 vmf->flags = old_flags; 2716 vmf->flags = old_flags;
3165 if (unlikely(ret & (VM_FAULT_ERROR | 2717 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))
3166 return ret; 2718 return ret;
3167 if (unlikely(!(ret & VM_FAULT_LOCKED) 2719 if (unlikely(!(ret & VM_FAULT_LOCKED))) {
3168 folio_lock(folio); !! 2720 lock_page(page);
3169 if (!folio->mapping) { !! 2721 if (!page->mapping) {
3170 folio_unlock(folio); !! 2722 unlock_page(page);
3171 return 0; /* retry */ 2723 return 0; /* retry */
3172 } 2724 }
3173 ret |= VM_FAULT_LOCKED; 2725 ret |= VM_FAULT_LOCKED;
3174 } else 2726 } else
3175 VM_BUG_ON_FOLIO(!folio_test_l !! 2727 VM_BUG_ON_PAGE(!PageLocked(page), page);
3176 return ret; 2728 return ret;
3177 } 2729 }
3178 2730
3179 /* 2731 /*
3180 * Handle dirtying of a page in shared file m 2732 * Handle dirtying of a page in shared file mapping on a write fault.
3181 * 2733 *
3182 * The function expects the page to be locked 2734 * The function expects the page to be locked and unlocks it.
3183 */ 2735 */
3184 static vm_fault_t fault_dirty_shared_page(str 2736 static vm_fault_t fault_dirty_shared_page(struct vm_fault *vmf)
3185 { 2737 {
3186 struct vm_area_struct *vma = vmf->vma 2738 struct vm_area_struct *vma = vmf->vma;
3187 struct address_space *mapping; 2739 struct address_space *mapping;
3188 struct folio *folio = page_folio(vmf- !! 2740 struct page *page = vmf->page;
3189 bool dirtied; 2741 bool dirtied;
3190 bool page_mkwrite = vma->vm_ops && vm 2742 bool page_mkwrite = vma->vm_ops && vma->vm_ops->page_mkwrite;
3191 2743
3192 dirtied = folio_mark_dirty(folio); !! 2744 dirtied = set_page_dirty(page);
3193 VM_BUG_ON_FOLIO(folio_test_anon(folio !! 2745 VM_BUG_ON_PAGE(PageAnon(page), page);
3194 /* 2746 /*
3195 * Take a local copy of the address_s !! 2747 * Take a local copy of the address_space - page.mapping may be zeroed
3196 * by truncate after folio_unlock(). !! 2748 * by truncate after unlock_page(). The address_space itself remains
3197 * pinned by vma->vm_file's reference !! 2749 * pinned by vma->vm_file's reference. We rely on unlock_page()'s
3198 * release semantics to prevent the c 2750 * release semantics to prevent the compiler from undoing this copying.
3199 */ 2751 */
3200 mapping = folio_raw_mapping(folio); !! 2752 mapping = page_rmapping(page);
3201 folio_unlock(folio); !! 2753 unlock_page(page);
3202 2754
3203 if (!page_mkwrite) 2755 if (!page_mkwrite)
3204 file_update_time(vma->vm_file 2756 file_update_time(vma->vm_file);
3205 2757
3206 /* 2758 /*
3207 * Throttle page dirtying rate down t 2759 * Throttle page dirtying rate down to writeback speed.
3208 * 2760 *
3209 * mapping may be NULL here because s 2761 * mapping may be NULL here because some device drivers do not
3210 * set page.mapping but still dirty t 2762 * set page.mapping but still dirty their pages
3211 * 2763 *
3212 * Drop the mmap_lock before waiting 2764 * Drop the mmap_lock before waiting on IO, if we can. The file
3213 * is pinning the mapping, as per abo 2765 * is pinning the mapping, as per above.
3214 */ 2766 */
3215 if ((dirtied || page_mkwrite) && mapp 2767 if ((dirtied || page_mkwrite) && mapping) {
3216 struct file *fpin; 2768 struct file *fpin;
3217 2769
3218 fpin = maybe_unlock_mmap_for_ 2770 fpin = maybe_unlock_mmap_for_io(vmf, NULL);
3219 balance_dirty_pages_ratelimit 2771 balance_dirty_pages_ratelimited(mapping);
3220 if (fpin) { 2772 if (fpin) {
3221 fput(fpin); 2773 fput(fpin);
3222 return VM_FAULT_COMPL !! 2774 return VM_FAULT_RETRY;
3223 } 2775 }
3224 } 2776 }
3225 2777
3226 return 0; 2778 return 0;
3227 } 2779 }
3228 2780
3229 /* 2781 /*
3230 * Handle write page faults for pages that ca 2782 * Handle write page faults for pages that can be reused in the current vma
3231 * 2783 *
3232 * This can happen either due to the mapping 2784 * This can happen either due to the mapping being with the VM_SHARED flag,
3233 * or due to us being the last reference stan 2785 * or due to us being the last reference standing to the page. In either
3234 * case, all we need to do here is to mark th 2786 * case, all we need to do here is to mark the page as writable and update
3235 * any related book-keeping. 2787 * any related book-keeping.
3236 */ 2788 */
3237 static inline void wp_page_reuse(struct vm_fa !! 2789 static inline void wp_page_reuse(struct vm_fault *vmf)
3238 __releases(vmf->ptl) 2790 __releases(vmf->ptl)
3239 { 2791 {
3240 struct vm_area_struct *vma = vmf->vma 2792 struct vm_area_struct *vma = vmf->vma;
>> 2793 struct page *page = vmf->page;
3241 pte_t entry; 2794 pte_t entry;
3242 !! 2795 /*
3243 VM_BUG_ON(!(vmf->flags & FAULT_FLAG_W !! 2796 * Clear the pages cpupid information as the existing
3244 VM_WARN_ON(is_zero_pfn(pte_pfn(vmf->o !! 2797 * information potentially belongs to a now completely
3245 !! 2798 * unrelated process.
3246 if (folio) { !! 2799 */
3247 VM_BUG_ON(folio_test_anon(fol !! 2800 if (page)
3248 !PageAnonExclusive( !! 2801 page_cpupid_xchg_last(page, (1 << LAST_CPUPID_SHIFT) - 1);
3249 /* <<
3250 * Clear the folio's cpupid i <<
3251 * information potentially be <<
3252 * unrelated process. <<
3253 */ <<
3254 folio_xchg_last_cpupid(folio, <<
3255 } <<
3256 2802
3257 flush_cache_page(vma, vmf->address, p 2803 flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte));
3258 entry = pte_mkyoung(vmf->orig_pte); 2804 entry = pte_mkyoung(vmf->orig_pte);
3259 entry = maybe_mkwrite(pte_mkdirty(ent 2805 entry = maybe_mkwrite(pte_mkdirty(entry), vma);
3260 if (ptep_set_access_flags(vma, vmf->a 2806 if (ptep_set_access_flags(vma, vmf->address, vmf->pte, entry, 1))
3261 update_mmu_cache_range(vmf, v !! 2807 update_mmu_cache(vma, vmf->address, vmf->pte);
3262 pte_unmap_unlock(vmf->pte, vmf->ptl); 2808 pte_unmap_unlock(vmf->pte, vmf->ptl);
3263 count_vm_event(PGREUSE); 2809 count_vm_event(PGREUSE);
3264 } 2810 }
3265 2811
3266 /* 2812 /*
3267 * We could add a bitflag somewhere, but for !! 2813 * Handle the case of a page which we actually need to copy to a new page.
3268 * vm_ops that have a ->map_pages have been a <<
3269 * the mmap_lock to be held. <<
3270 */ <<
3271 static inline vm_fault_t vmf_can_call_fault(c <<
3272 { <<
3273 struct vm_area_struct *vma = vmf->vma <<
3274 <<
3275 if (vma->vm_ops->map_pages || !(vmf-> <<
3276 return 0; <<
3277 vma_end_read(vma); <<
3278 return VM_FAULT_RETRY; <<
3279 } <<
3280 <<
3281 /** <<
3282 * __vmf_anon_prepare - Prepare to handle an <<
3283 * @vmf: The vm_fault descriptor passed from <<
3284 * <<
3285 * When preparing to insert an anonymous page <<
3286 * fault handler, call this function rather t <<
3287 * If this vma does not already have an assoc <<
3288 * only protected by the per-VMA lock, the ca <<
3289 * mmap_lock held. __anon_vma_prepare() will <<
3290 * determine if this VMA can share its anon_v <<
3291 * do with only the per-VMA lock held for thi <<
3292 * <<
3293 * Return: 0 if fault handling can proceed. <<
3294 * returned to the caller. <<
3295 */ <<
3296 vm_fault_t __vmf_anon_prepare(struct vm_fault <<
3297 { <<
3298 struct vm_area_struct *vma = vmf->vma <<
3299 vm_fault_t ret = 0; <<
3300 <<
3301 if (likely(vma->anon_vma)) <<
3302 return 0; <<
3303 if (vmf->flags & FAULT_FLAG_VMA_LOCK) <<
3304 if (!mmap_read_trylock(vma->v <<
3305 return VM_FAULT_RETRY <<
3306 } <<
3307 if (__anon_vma_prepare(vma)) <<
3308 ret = VM_FAULT_OOM; <<
3309 if (vmf->flags & FAULT_FLAG_VMA_LOCK) <<
3310 mmap_read_unlock(vma->vm_mm); <<
3311 return ret; <<
3312 } <<
3313 <<
3314 /* <<
3315 * Handle the case of a page which we actuall <<
3316 * either due to COW or unsharing. <<
3317 * 2814 *
3318 * Called with mmap_lock locked and the old p 2815 * Called with mmap_lock locked and the old page referenced, but
3319 * without the ptl held. 2816 * without the ptl held.
3320 * 2817 *
3321 * High level logic flow: 2818 * High level logic flow:
3322 * 2819 *
3323 * - Allocate a page, copy the content of the 2820 * - Allocate a page, copy the content of the old page to the new one.
3324 * - Handle book keeping and accounting - cgr 2821 * - Handle book keeping and accounting - cgroups, mmu-notifiers, etc.
3325 * - Take the PTL. If the pte changed, bail o 2822 * - Take the PTL. If the pte changed, bail out and release the allocated page
3326 * - If the pte is still the way we remember 2823 * - If the pte is still the way we remember it, update the page table and all
3327 * relevant references. This includes dropp 2824 * relevant references. This includes dropping the reference the page-table
3328 * held to the old page, as well as updatin 2825 * held to the old page, as well as updating the rmap.
3329 * - In any case, unlock the PTL and drop the 2826 * - In any case, unlock the PTL and drop the reference we took to the old page.
3330 */ 2827 */
3331 static vm_fault_t wp_page_copy(struct vm_faul 2828 static vm_fault_t wp_page_copy(struct vm_fault *vmf)
3332 { 2829 {
3333 const bool unshare = vmf->flags & FAU <<
3334 struct vm_area_struct *vma = vmf->vma 2830 struct vm_area_struct *vma = vmf->vma;
3335 struct mm_struct *mm = vma->vm_mm; 2831 struct mm_struct *mm = vma->vm_mm;
3336 struct folio *old_folio = NULL; !! 2832 struct page *old_page = vmf->page;
3337 struct folio *new_folio = NULL; !! 2833 struct page *new_page = NULL;
3338 pte_t entry; 2834 pte_t entry;
3339 int page_copied = 0; 2835 int page_copied = 0;
3340 struct mmu_notifier_range range; 2836 struct mmu_notifier_range range;
3341 vm_fault_t ret; <<
3342 bool pfn_is_zero; <<
3343 <<
3344 delayacct_wpcopy_start(); <<
3345 <<
3346 if (vmf->page) <<
3347 old_folio = page_folio(vmf->p <<
3348 ret = vmf_anon_prepare(vmf); <<
3349 if (unlikely(ret)) <<
3350 goto out; <<
3351 2837
3352 pfn_is_zero = is_zero_pfn(pte_pfn(vmf !! 2838 if (unlikely(anon_vma_prepare(vma)))
3353 new_folio = folio_prealloc(mm, vma, v <<
3354 if (!new_folio) <<
3355 goto oom; 2839 goto oom;
3356 2840
3357 if (!pfn_is_zero) { !! 2841 if (is_zero_pfn(pte_pfn(vmf->orig_pte))) {
3358 int err; !! 2842 new_page = alloc_zeroed_user_highpage_movable(vma,
>> 2843 vmf->address);
>> 2844 if (!new_page)
>> 2845 goto oom;
>> 2846 } else {
>> 2847 new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma,
>> 2848 vmf->address);
>> 2849 if (!new_page)
>> 2850 goto oom;
3359 2851
3360 err = __wp_page_copy_user(&ne !! 2852 if (!cow_user_page(new_page, old_page, vmf)) {
3361 if (err) { <<
3362 /* 2853 /*
3363 * COW failed, if the 2854 * COW failed, if the fault was solved by other,
3364 * it's fine. If not, 2855 * it's fine. If not, userspace would re-fault on
3365 * the same address a 2856 * the same address and we will handle the fault
3366 * from the second at 2857 * from the second attempt.
3367 * The -EHWPOISON cas <<
3368 */ 2858 */
3369 folio_put(new_folio); !! 2859 put_page(new_page);
3370 if (old_folio) !! 2860 if (old_page)
3371 folio_put(old !! 2861 put_page(old_page);
3372 !! 2862 return 0;
3373 delayacct_wpcopy_end( <<
3374 return err == -EHWPOI <<
3375 } 2863 }
3376 kmsan_copy_page_meta(&new_fol <<
3377 } 2864 }
3378 2865
3379 __folio_mark_uptodate(new_folio); !! 2866 if (mem_cgroup_charge(new_page, mm, GFP_KERNEL))
>> 2867 goto oom_free_new;
>> 2868 cgroup_throttle_swaprate(new_page, GFP_KERNEL);
3380 2869
3381 mmu_notifier_range_init(&range, MMU_N !! 2870 __SetPageUptodate(new_page);
>> 2871
>> 2872 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm,
3382 vmf->address 2873 vmf->address & PAGE_MASK,
3383 (vmf->address 2874 (vmf->address & PAGE_MASK) + PAGE_SIZE);
3384 mmu_notifier_invalidate_range_start(& 2875 mmu_notifier_invalidate_range_start(&range);
3385 2876
3386 /* 2877 /*
3387 * Re-check the pte - we dropped the 2878 * Re-check the pte - we dropped the lock
3388 */ 2879 */
3389 vmf->pte = pte_offset_map_lock(mm, vm 2880 vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl);
3390 if (likely(vmf->pte && pte_same(ptep_ !! 2881 if (likely(pte_same(*vmf->pte, vmf->orig_pte))) {
3391 if (old_folio) { !! 2882 if (old_page) {
3392 if (!folio_test_anon( !! 2883 if (!PageAnon(old_page)) {
3393 dec_mm_counte !! 2884 dec_mm_counter_fast(mm,
3394 inc_mm_counte !! 2885 mm_counter_file(old_page));
>> 2886 inc_mm_counter_fast(mm, MM_ANONPAGES);
3395 } 2887 }
3396 } else { 2888 } else {
3397 ksm_might_unmap_zero_ !! 2889 inc_mm_counter_fast(mm, MM_ANONPAGES);
3398 inc_mm_counter(mm, MM <<
3399 } 2890 }
3400 flush_cache_page(vma, vmf->ad 2891 flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte));
3401 entry = mk_pte(&new_folio->pa !! 2892 entry = mk_pte(new_page, vma->vm_page_prot);
3402 entry = pte_sw_mkyoung(entry) 2893 entry = pte_sw_mkyoung(entry);
3403 if (unlikely(unshare)) { !! 2894 entry = maybe_mkwrite(pte_mkdirty(entry), vma);
3404 if (pte_soft_dirty(vm <<
3405 entry = pte_m <<
3406 if (pte_uffd_wp(vmf-> <<
3407 entry = pte_m <<
3408 } else { <<
3409 entry = maybe_mkwrite <<
3410 } <<
3411 2895
3412 /* 2896 /*
3413 * Clear the pte entry and fl 2897 * Clear the pte entry and flush it first, before updating the
3414 * pte with the new entry, to 2898 * pte with the new entry, to keep TLBs on different CPUs in
3415 * sync. This code used to se 2899 * sync. This code used to set the new PTE then flush TLBs, but
3416 * that left a window where t 2900 * that left a window where the new PTE could be loaded into
3417 * some TLBs while the old PT 2901 * some TLBs while the old PTE remains in others.
3418 */ 2902 */
3419 ptep_clear_flush(vma, vmf->ad !! 2903 ptep_clear_flush_notify(vma, vmf->address, vmf->pte);
3420 folio_add_new_anon_rmap(new_f !! 2904 page_add_new_anon_rmap(new_page, vma, vmf->address, false);
3421 folio_add_lru_vma(new_folio, !! 2905 lru_cache_add_inactive_or_unevictable(new_page, vma);
3422 BUG_ON(unshare && pte_write(e !! 2906 /*
3423 set_pte_at(mm, vmf->address, !! 2907 * We call the notify macro here because, when using secondary
3424 update_mmu_cache_range(vmf, v !! 2908 * mmu page tables (such as kvm shadow page tables), we want the
3425 if (old_folio) { !! 2909 * new page to be mapped directly into the secondary page table.
>> 2910 */
>> 2911 set_pte_at_notify(mm, vmf->address, vmf->pte, entry);
>> 2912 update_mmu_cache(vma, vmf->address, vmf->pte);
>> 2913 if (old_page) {
3426 /* 2914 /*
3427 * Only after switchi 2915 * Only after switching the pte to the new page may
3428 * we remove the mapc 2916 * we remove the mapcount here. Otherwise another
3429 * process may come a 2917 * process may come and find the rmap count decremented
3430 * before the pte is 2918 * before the pte is switched to the new page, and
3431 * "reuse" the old pa 2919 * "reuse" the old page writing into it while our pte
3432 * here still points 2920 * here still points into it and can be read by other
3433 * threads. 2921 * threads.
3434 * 2922 *
3435 * The critical issue 2923 * The critical issue is to order this
3436 * folio_remove_rmap_ !! 2924 * page_remove_rmap with the ptp_clear_flush above.
3437 * above. Those store !! 2925 * Those stores are ordered by (if nothing else,)
3438 * the barrier presen 2926 * the barrier present in the atomic_add_negative
3439 * in folio_remove_rm !! 2927 * in page_remove_rmap.
3440 * 2928 *
3441 * Then the TLB flush 2929 * Then the TLB flush in ptep_clear_flush ensures that
3442 * no process can acc 2930 * no process can access the old page before the
3443 * decremented mapcou 2931 * decremented mapcount is visible. And the old page
3444 * cannot be reused u 2932 * cannot be reused until after the decremented
3445 * mapcount is visibl 2933 * mapcount is visible. So transitively, TLBs to
3446 * old page will be f 2934 * old page will be flushed before it can be reused.
3447 */ 2935 */
3448 folio_remove_rmap_pte !! 2936 page_remove_rmap(old_page, false);
3449 } 2937 }
3450 2938
3451 /* Free the old page.. */ 2939 /* Free the old page.. */
3452 new_folio = old_folio; !! 2940 new_page = old_page;
3453 page_copied = 1; 2941 page_copied = 1;
3454 pte_unmap_unlock(vmf->pte, vm !! 2942 } else {
3455 } else if (vmf->pte) { <<
3456 update_mmu_tlb(vma, vmf->addr 2943 update_mmu_tlb(vma, vmf->address, vmf->pte);
3457 pte_unmap_unlock(vmf->pte, vm <<
3458 } 2944 }
3459 2945
3460 mmu_notifier_invalidate_range_end(&ra !! 2946 if (new_page)
3461 !! 2947 put_page(new_page);
3462 if (new_folio) <<
3463 folio_put(new_folio); <<
3464 if (old_folio) { <<
3465 if (page_copied) <<
3466 free_swap_cache(old_f <<
3467 folio_put(old_folio); <<
3468 } <<
3469 2948
3470 delayacct_wpcopy_end(); !! 2949 pte_unmap_unlock(vmf->pte, vmf->ptl);
3471 return 0; !! 2950 /*
>> 2951 * No need to double call mmu_notifier->invalidate_range() callback as
>> 2952 * the above ptep_clear_flush_notify() did already call it.
>> 2953 */
>> 2954 mmu_notifier_invalidate_range_only_end(&range);
>> 2955 if (old_page) {
>> 2956 /*
>> 2957 * Don't let another task, with possibly unlocked vma,
>> 2958 * keep the mlocked page.
>> 2959 */
>> 2960 if (page_copied && (vma->vm_flags & VM_LOCKED)) {
>> 2961 lock_page(old_page); /* LRU manipulation */
>> 2962 if (PageMlocked(old_page))
>> 2963 munlock_vma_page(old_page);
>> 2964 unlock_page(old_page);
>> 2965 }
>> 2966 put_page(old_page);
>> 2967 }
>> 2968 return page_copied ? VM_FAULT_WRITE : 0;
>> 2969 oom_free_new:
>> 2970 put_page(new_page);
3472 oom: 2971 oom:
3473 ret = VM_FAULT_OOM; !! 2972 if (old_page)
3474 out: !! 2973 put_page(old_page);
3475 if (old_folio) !! 2974 return VM_FAULT_OOM;
3476 folio_put(old_folio); <<
3477 <<
3478 delayacct_wpcopy_end(); <<
3479 return ret; <<
3480 } 2975 }
3481 2976
3482 /** 2977 /**
3483 * finish_mkwrite_fault - finish page fault f 2978 * finish_mkwrite_fault - finish page fault for a shared mapping, making PTE
3484 * writeable once the 2979 * writeable once the page is prepared
3485 * 2980 *
3486 * @vmf: structure describing the fault 2981 * @vmf: structure describing the fault
3487 * @folio: the folio of vmf->page <<
3488 * 2982 *
3489 * This function handles all that is needed t 2983 * This function handles all that is needed to finish a write page fault in a
3490 * shared mapping due to PTE being read-only 2984 * shared mapping due to PTE being read-only once the mapped page is prepared.
3491 * It handles locking of PTE and modifying it 2985 * It handles locking of PTE and modifying it.
3492 * 2986 *
3493 * The function expects the page to be locked 2987 * The function expects the page to be locked or other protection against
3494 * concurrent faults / writeback (such as DAX 2988 * concurrent faults / writeback (such as DAX radix tree locks).
3495 * 2989 *
3496 * Return: %0 on success, %VM_FAULT_NOPAGE wh !! 2990 * Return: %VM_FAULT_WRITE on success, %0 when PTE got changed before
3497 * we acquired PTE lock. 2991 * we acquired PTE lock.
3498 */ 2992 */
3499 static vm_fault_t finish_mkwrite_fault(struct !! 2993 vm_fault_t finish_mkwrite_fault(struct vm_fault *vmf)
3500 { 2994 {
3501 WARN_ON_ONCE(!(vmf->vma->vm_flags & V 2995 WARN_ON_ONCE(!(vmf->vma->vm_flags & VM_SHARED));
3502 vmf->pte = pte_offset_map_lock(vmf->v 2996 vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd, vmf->address,
3503 &vmf-> 2997 &vmf->ptl);
3504 if (!vmf->pte) <<
3505 return VM_FAULT_NOPAGE; <<
3506 /* 2998 /*
3507 * We might have raced with another p 2999 * We might have raced with another page fault while we released the
3508 * pte_offset_map_lock. 3000 * pte_offset_map_lock.
3509 */ 3001 */
3510 if (!pte_same(ptep_get(vmf->pte), vmf !! 3002 if (!pte_same(*vmf->pte, vmf->orig_pte)) {
3511 update_mmu_tlb(vmf->vma, vmf- 3003 update_mmu_tlb(vmf->vma, vmf->address, vmf->pte);
3512 pte_unmap_unlock(vmf->pte, vm 3004 pte_unmap_unlock(vmf->pte, vmf->ptl);
3513 return VM_FAULT_NOPAGE; 3005 return VM_FAULT_NOPAGE;
3514 } 3006 }
3515 wp_page_reuse(vmf, folio); !! 3007 wp_page_reuse(vmf);
3516 return 0; 3008 return 0;
3517 } 3009 }
3518 3010
3519 /* 3011 /*
3520 * Handle write page faults for VM_MIXEDMAP o 3012 * Handle write page faults for VM_MIXEDMAP or VM_PFNMAP for a VM_SHARED
3521 * mapping 3013 * mapping
3522 */ 3014 */
3523 static vm_fault_t wp_pfn_shared(struct vm_fau 3015 static vm_fault_t wp_pfn_shared(struct vm_fault *vmf)
3524 { 3016 {
3525 struct vm_area_struct *vma = vmf->vma 3017 struct vm_area_struct *vma = vmf->vma;
3526 3018
3527 if (vma->vm_ops && vma->vm_ops->pfn_m 3019 if (vma->vm_ops && vma->vm_ops->pfn_mkwrite) {
3528 vm_fault_t ret; 3020 vm_fault_t ret;
3529 3021
3530 pte_unmap_unlock(vmf->pte, vm 3022 pte_unmap_unlock(vmf->pte, vmf->ptl);
3531 ret = vmf_can_call_fault(vmf) <<
3532 if (ret) <<
3533 return ret; <<
3534 <<
3535 vmf->flags |= FAULT_FLAG_MKWR 3023 vmf->flags |= FAULT_FLAG_MKWRITE;
3536 ret = vma->vm_ops->pfn_mkwrit 3024 ret = vma->vm_ops->pfn_mkwrite(vmf);
3537 if (ret & (VM_FAULT_ERROR | V 3025 if (ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE))
3538 return ret; 3026 return ret;
3539 return finish_mkwrite_fault(v !! 3027 return finish_mkwrite_fault(vmf);
3540 } 3028 }
3541 wp_page_reuse(vmf, NULL); !! 3029 wp_page_reuse(vmf);
3542 return 0; !! 3030 return VM_FAULT_WRITE;
3543 } 3031 }
3544 3032
3545 static vm_fault_t wp_page_shared(struct vm_fa !! 3033 static vm_fault_t wp_page_shared(struct vm_fault *vmf)
3546 __releases(vmf->ptl) 3034 __releases(vmf->ptl)
3547 { 3035 {
3548 struct vm_area_struct *vma = vmf->vma 3036 struct vm_area_struct *vma = vmf->vma;
3549 vm_fault_t ret = 0; !! 3037 vm_fault_t ret = VM_FAULT_WRITE;
3550 3038
3551 folio_get(folio); !! 3039 get_page(vmf->page);
3552 3040
3553 if (vma->vm_ops && vma->vm_ops->page_ 3041 if (vma->vm_ops && vma->vm_ops->page_mkwrite) {
3554 vm_fault_t tmp; 3042 vm_fault_t tmp;
3555 3043
3556 pte_unmap_unlock(vmf->pte, vm 3044 pte_unmap_unlock(vmf->pte, vmf->ptl);
3557 tmp = vmf_can_call_fault(vmf) !! 3045 tmp = do_page_mkwrite(vmf);
3558 if (tmp) { <<
3559 folio_put(folio); <<
3560 return tmp; <<
3561 } <<
3562 <<
3563 tmp = do_page_mkwrite(vmf, fo <<
3564 if (unlikely(!tmp || (tmp & 3046 if (unlikely(!tmp || (tmp &
3565 (VM_FAU 3047 (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))) {
3566 folio_put(folio); !! 3048 put_page(vmf->page);
3567 return tmp; 3049 return tmp;
3568 } 3050 }
3569 tmp = finish_mkwrite_fault(vm !! 3051 tmp = finish_mkwrite_fault(vmf);
3570 if (unlikely(tmp & (VM_FAULT_ 3052 if (unlikely(tmp & (VM_FAULT_ERROR | VM_FAULT_NOPAGE))) {
3571 folio_unlock(folio); !! 3053 unlock_page(vmf->page);
3572 folio_put(folio); !! 3054 put_page(vmf->page);
3573 return tmp; 3055 return tmp;
3574 } 3056 }
3575 } else { 3057 } else {
3576 wp_page_reuse(vmf, folio); !! 3058 wp_page_reuse(vmf);
3577 folio_lock(folio); !! 3059 lock_page(vmf->page);
3578 } 3060 }
3579 ret |= fault_dirty_shared_page(vmf); 3061 ret |= fault_dirty_shared_page(vmf);
3580 folio_put(folio); !! 3062 put_page(vmf->page);
3581 3063
3582 return ret; 3064 return ret;
3583 } 3065 }
3584 3066
3585 static bool wp_can_reuse_anon_folio(struct fo <<
3586 struct vm <<
3587 { <<
3588 /* <<
3589 * We could currently only reuse a su <<
3590 * other subpages of the large folios <<
3591 * let's just consistently not reuse <<
3592 * reuse in that scenario, and give b <<
3593 * sooner. <<
3594 */ <<
3595 if (folio_test_large(folio)) <<
3596 return false; <<
3597 <<
3598 /* <<
3599 * We have to verify under folio lock <<
3600 * just an optimization to avoid lock <<
3601 * the swapcache if there is little h <<
3602 * <<
3603 * KSM doesn't necessarily raise the <<
3604 */ <<
3605 if (folio_test_ksm(folio) || folio_re <<
3606 return false; <<
3607 if (!folio_test_lru(folio)) <<
3608 /* <<
3609 * We cannot easily detect+ha <<
3610 * remote LRU caches or refer <<
3611 */ <<
3612 lru_add_drain(); <<
3613 if (folio_ref_count(folio) > 1 + foli <<
3614 return false; <<
3615 if (!folio_trylock(folio)) <<
3616 return false; <<
3617 if (folio_test_swapcache(folio)) <<
3618 folio_free_swap(folio); <<
3619 if (folio_test_ksm(folio) || folio_re <<
3620 folio_unlock(folio); <<
3621 return false; <<
3622 } <<
3623 /* <<
3624 * Ok, we've got the only folio refer <<
3625 * and the folio is locked, it's dark <<
3626 * sunglasses. Hit it. <<
3627 */ <<
3628 folio_move_anon_rmap(folio, vma); <<
3629 folio_unlock(folio); <<
3630 return true; <<
3631 } <<
3632 <<
3633 /* 3067 /*
3634 * This routine handles present pages, when !! 3068 * This routine handles present pages, when users try to write
3635 * * users try to write to a shared page (FAU !! 3069 * to a shared page. It is done by copying the page to a new address
3636 * * GUP wants to take a R/O pin on a possibl !! 3070 * and decrementing the shared-page counter for the old page.
3637 * (FAULT_FLAG_UNSHARE) <<
3638 * <<
3639 * It is done by copying the page to a new ad <<
3640 * shared-page counter for the old page. <<
3641 * 3071 *
3642 * Note that this routine assumes that the pr 3072 * Note that this routine assumes that the protection checks have been
3643 * done by the caller (the low-level page fau 3073 * done by the caller (the low-level page fault routine in most cases).
3644 * Thus, with FAULT_FLAG_WRITE, we can safely !! 3074 * Thus we can safely just mark it writable once we've done any necessary
3645 * done any necessary COW. !! 3075 * COW.
3646 * 3076 *
3647 * In case of FAULT_FLAG_WRITE, we also mark !! 3077 * We also mark the page dirty at this point even though the page will
3648 * though the page will change only once the !! 3078 * change only once the write actually happens. This avoids a few races,
3649 * avoids a few races, and potentially makes !! 3079 * and potentially makes it more efficient.
3650 * 3080 *
3651 * We enter with non-exclusive mmap_lock (to 3081 * We enter with non-exclusive mmap_lock (to exclude vma changes,
3652 * but allow concurrent faults), with pte bot 3082 * but allow concurrent faults), with pte both mapped and locked.
3653 * We return with mmap_lock still held, but p 3083 * We return with mmap_lock still held, but pte unmapped and unlocked.
3654 */ 3084 */
3655 static vm_fault_t do_wp_page(struct vm_fault 3085 static vm_fault_t do_wp_page(struct vm_fault *vmf)
3656 __releases(vmf->ptl) 3086 __releases(vmf->ptl)
3657 { 3087 {
3658 const bool unshare = vmf->flags & FAU <<
3659 struct vm_area_struct *vma = vmf->vma 3088 struct vm_area_struct *vma = vmf->vma;
3660 struct folio *folio = NULL; <<
3661 pte_t pte; <<
3662 <<
3663 if (likely(!unshare)) { <<
3664 if (userfaultfd_pte_wp(vma, p <<
3665 if (!userfaultfd_wp_a <<
3666 pte_unmap_unl <<
3667 return handle <<
3668 } <<
3669 <<
3670 /* <<
3671 * Nothing needed (ca <<
3672 * etc.) because we'r <<
3673 * which is completel <<
3674 */ <<
3675 pte = pte_clear_uffd_ <<
3676 3089
3677 set_pte_at(vma->vm_mm !! 3090 if (userfaultfd_pte_wp(vma, *vmf->pte)) {
3678 /* !! 3091 pte_unmap_unlock(vmf->pte, vmf->ptl);
3679 * Update this to be !! 3092 return handle_userfault(vmf, VM_UFFD_WP);
3680 * handling <<
3681 */ <<
3682 vmf->orig_pte = pte; <<
3683 } <<
3684 <<
3685 /* <<
3686 * Userfaultfd write-protect <<
3687 * is flushed in this case be <<
3688 */ <<
3689 if (unlikely(userfaultfd_wp(v <<
3690 mm_tlb_flush_pen <<
3691 flush_tlb_page(vmf->v <<
3692 } 3093 }
3693 3094
3694 vmf->page = vm_normal_page(vma, vmf-> <<
3695 <<
3696 if (vmf->page) <<
3697 folio = page_folio(vmf->page) <<
3698 <<
3699 /* 3095 /*
3700 * Shared mapping: we are guaranteed !! 3096 * Userfaultfd write-protect can defer flushes. Ensure the TLB
3701 * FAULT_FLAG_WRITE set at this point !! 3097 * is flushed in this case before copying.
3702 */ 3098 */
3703 if (vma->vm_flags & (VM_SHARED | VM_M !! 3099 if (unlikely(userfaultfd_wp(vmf->vma) &&
>> 3100 mm_tlb_flush_pending(vmf->vma->vm_mm)))
>> 3101 flush_tlb_page(vmf->vma, vmf->address);
>> 3102
>> 3103 vmf->page = vm_normal_page(vma, vmf->address, vmf->orig_pte);
>> 3104 if (!vmf->page) {
3704 /* 3105 /*
3705 * VM_MIXEDMAP !pfn_valid() c 3106 * VM_MIXEDMAP !pfn_valid() case, or VM_SOFTDIRTY clear on a
3706 * VM_PFNMAP VMA. 3107 * VM_PFNMAP VMA.
3707 * 3108 *
3708 * We should not cow pages in 3109 * We should not cow pages in a shared writeable mapping.
3709 * Just mark the pages writab 3110 * Just mark the pages writable and/or call ops->pfn_mkwrite.
3710 */ 3111 */
3711 if (!vmf->page) !! 3112 if ((vma->vm_flags & (VM_WRITE|VM_SHARED)) ==
>> 3113 (VM_WRITE|VM_SHARED))
3712 return wp_pfn_shared( 3114 return wp_pfn_shared(vmf);
3713 return wp_page_shared(vmf, fo !! 3115
>> 3116 pte_unmap_unlock(vmf->pte, vmf->ptl);
>> 3117 return wp_page_copy(vmf);
3714 } 3118 }
3715 3119
3716 /* 3120 /*
3717 * Private mapping: create an exclusi !! 3121 * Take out anonymous pages first, anonymous shared vmas are
3718 * is impossible. We might miss VM_WR !! 3122 * not dirty accountable.
3719 * <<
3720 * If we encounter a page that is mar <<
3721 * the page without further checks. <<
3722 */ 3123 */
3723 if (folio && folio_test_anon(folio) & !! 3124 if (PageAnon(vmf->page)) {
3724 (PageAnonExclusive(vmf->page) || !! 3125 struct page *page = vmf->page;
3725 if (!PageAnonExclusive(vmf->p !! 3126
3726 SetPageAnonExclusive( !! 3127 /* PageKsm() doesn't necessarily raise the page refcount */
3727 if (unlikely(unshare)) { !! 3128 if (PageKsm(page) || page_count(page) != 1)
3728 pte_unmap_unlock(vmf- !! 3129 goto copy;
3729 return 0; !! 3130 if (!trylock_page(page))
>> 3131 goto copy;
>> 3132 if (PageKsm(page) || page_mapcount(page) != 1 || page_count(page) != 1) {
>> 3133 unlock_page(page);
>> 3134 goto copy;
3730 } 3135 }
3731 wp_page_reuse(vmf, folio); !! 3136 /*
3732 return 0; !! 3137 * Ok, we've got the only map reference, and the only
>> 3138 * page count reference, and the page is locked,
>> 3139 * it's dark out, and we're wearing sunglasses. Hit it.
>> 3140 */
>> 3141 unlock_page(page);
>> 3142 wp_page_reuse(vmf);
>> 3143 return VM_FAULT_WRITE;
>> 3144 } else if (unlikely((vma->vm_flags & (VM_WRITE|VM_SHARED)) ==
>> 3145 (VM_WRITE|VM_SHARED))) {
>> 3146 return wp_page_shared(vmf);
3733 } 3147 }
>> 3148 copy:
3734 /* 3149 /*
3735 * Ok, we need to copy. Oh, well.. 3150 * Ok, we need to copy. Oh, well..
3736 */ 3151 */
3737 if (folio) !! 3152 get_page(vmf->page);
3738 folio_get(folio); <<
3739 3153
3740 pte_unmap_unlock(vmf->pte, vmf->ptl); 3154 pte_unmap_unlock(vmf->pte, vmf->ptl);
3741 #ifdef CONFIG_KSM <<
3742 if (folio && folio_test_ksm(folio)) <<
3743 count_vm_event(COW_KSM); <<
3744 #endif <<
3745 return wp_page_copy(vmf); 3155 return wp_page_copy(vmf);
3746 } 3156 }
3747 3157
3748 static void unmap_mapping_range_vma(struct vm 3158 static void unmap_mapping_range_vma(struct vm_area_struct *vma,
3749 unsigned long start_addr, uns 3159 unsigned long start_addr, unsigned long end_addr,
3750 struct zap_details *details) 3160 struct zap_details *details)
3751 { 3161 {
3752 zap_page_range_single(vma, start_addr 3162 zap_page_range_single(vma, start_addr, end_addr - start_addr, details);
3753 } 3163 }
3754 3164
3755 static inline void unmap_mapping_range_tree(s 3165 static inline void unmap_mapping_range_tree(struct rb_root_cached *root,
3756 p <<
3757 p <<
3758 s 3166 struct zap_details *details)
3759 { 3167 {
3760 struct vm_area_struct *vma; 3168 struct vm_area_struct *vma;
3761 pgoff_t vba, vea, zba, zea; 3169 pgoff_t vba, vea, zba, zea;
3762 3170
3763 vma_interval_tree_foreach(vma, root, !! 3171 vma_interval_tree_foreach(vma, root,
>> 3172 details->first_index, details->last_index) {
>> 3173
3764 vba = vma->vm_pgoff; 3174 vba = vma->vm_pgoff;
3765 vea = vba + vma_pages(vma) - 3175 vea = vba + vma_pages(vma) - 1;
3766 zba = max(first_index, vba); !! 3176 zba = details->first_index;
3767 zea = min(last_index, vea); !! 3177 if (zba < vba)
>> 3178 zba = vba;
>> 3179 zea = details->last_index;
>> 3180 if (zea > vea)
>> 3181 zea = vea;
3768 3182
3769 unmap_mapping_range_vma(vma, 3183 unmap_mapping_range_vma(vma,
3770 ((zba - vba) << PAGE_ 3184 ((zba - vba) << PAGE_SHIFT) + vma->vm_start,
3771 ((zea - vba + 1) << P 3185 ((zea - vba + 1) << PAGE_SHIFT) + vma->vm_start,
3772 details); 3186 details);
3773 } 3187 }
3774 } 3188 }
3775 3189
3776 /** 3190 /**
3777 * unmap_mapping_folio() - Unmap single folio <<
3778 * @folio: The locked folio to be unmapped. <<
3779 * <<
3780 * Unmap this folio from any userspace proces <<
3781 * Typically, for efficiency, the range of ne <<
3782 * unmapped by unmap_mapping_pages() or unmap <<
3783 * truncation or invalidation holds the lock <<
3784 * the page has been remapped again: and then <<
3785 * to unmap it finally. <<
3786 */ <<
3787 void unmap_mapping_folio(struct folio *folio) <<
3788 { <<
3789 struct address_space *mapping = folio <<
3790 struct zap_details details = { }; <<
3791 pgoff_t first_index; <<
3792 pgoff_t last_index; <<
3793 <<
3794 VM_BUG_ON(!folio_test_locked(folio)); <<
3795 <<
3796 first_index = folio->index; <<
3797 last_index = folio_next_index(folio) <<
3798 <<
3799 details.even_cows = false; <<
3800 details.single_folio = folio; <<
3801 details.zap_flags = ZAP_FLAG_DROP_MAR <<
3802 <<
3803 i_mmap_lock_read(mapping); <<
3804 if (unlikely(!RB_EMPTY_ROOT(&mapping- <<
3805 unmap_mapping_range_tree(&map <<
3806 last <<
3807 i_mmap_unlock_read(mapping); <<
3808 } <<
3809 <<
3810 /** <<
3811 * unmap_mapping_pages() - Unmap pages from p 3191 * unmap_mapping_pages() - Unmap pages from processes.
3812 * @mapping: The address space containing pag 3192 * @mapping: The address space containing pages to be unmapped.
3813 * @start: Index of first page to be unmapped 3193 * @start: Index of first page to be unmapped.
3814 * @nr: Number of pages to be unmapped. 0 to 3194 * @nr: Number of pages to be unmapped. 0 to unmap to end of file.
3815 * @even_cows: Whether to unmap even private 3195 * @even_cows: Whether to unmap even private COWed pages.
3816 * 3196 *
3817 * Unmap the pages in this address space from 3197 * Unmap the pages in this address space from any userspace process which
3818 * has them mmaped. Generally, you want to r 3198 * has them mmaped. Generally, you want to remove COWed pages as well when
3819 * a file is being truncated, but not when in 3199 * a file is being truncated, but not when invalidating pages from the page
3820 * cache. 3200 * cache.
3821 */ 3201 */
3822 void unmap_mapping_pages(struct address_space 3202 void unmap_mapping_pages(struct address_space *mapping, pgoff_t start,
3823 pgoff_t nr, bool even_cows) 3203 pgoff_t nr, bool even_cows)
3824 { 3204 {
3825 struct zap_details details = { }; 3205 struct zap_details details = { };
3826 pgoff_t first_index = start; <<
3827 pgoff_t last_index = start + nr - 1; <<
3828 3206
3829 details.even_cows = even_cows; !! 3207 details.check_mapping = even_cows ? NULL : mapping;
3830 if (last_index < first_index) !! 3208 details.first_index = start;
3831 last_index = ULONG_MAX; !! 3209 details.last_index = start + nr - 1;
>> 3210 if (details.last_index < details.first_index)
>> 3211 details.last_index = ULONG_MAX;
3832 3212
3833 i_mmap_lock_read(mapping); !! 3213 i_mmap_lock_write(mapping);
3834 if (unlikely(!RB_EMPTY_ROOT(&mapping- 3214 if (unlikely(!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root)))
3835 unmap_mapping_range_tree(&map !! 3215 unmap_mapping_range_tree(&mapping->i_mmap, &details);
3836 last !! 3216 i_mmap_unlock_write(mapping);
3837 i_mmap_unlock_read(mapping); <<
3838 } 3217 }
3839 EXPORT_SYMBOL_GPL(unmap_mapping_pages); <<
3840 3218
3841 /** 3219 /**
3842 * unmap_mapping_range - unmap the portion of 3220 * unmap_mapping_range - unmap the portion of all mmaps in the specified
3843 * address_space corresponding to the specifi 3221 * address_space corresponding to the specified byte range in the underlying
3844 * file. 3222 * file.
3845 * 3223 *
3846 * @mapping: the address space containing mma 3224 * @mapping: the address space containing mmaps to be unmapped.
3847 * @holebegin: byte in first page to unmap, r 3225 * @holebegin: byte in first page to unmap, relative to the start of
3848 * the underlying file. This will be rounded 3226 * the underlying file. This will be rounded down to a PAGE_SIZE
3849 * boundary. Note that this is different fro 3227 * boundary. Note that this is different from truncate_pagecache(), which
3850 * must keep the partial page. In contrast, 3228 * must keep the partial page. In contrast, we must get rid of
3851 * partial pages. 3229 * partial pages.
3852 * @holelen: size of prospective hole in byte 3230 * @holelen: size of prospective hole in bytes. This will be rounded
3853 * up to a PAGE_SIZE boundary. A holelen of 3231 * up to a PAGE_SIZE boundary. A holelen of zero truncates to the
3854 * end of the file. 3232 * end of the file.
3855 * @even_cows: 1 when truncating a file, unma 3233 * @even_cows: 1 when truncating a file, unmap even private COWed pages;
3856 * but 0 when invalidating pagecache, don't t 3234 * but 0 when invalidating pagecache, don't throw away private data.
3857 */ 3235 */
3858 void unmap_mapping_range(struct address_space 3236 void unmap_mapping_range(struct address_space *mapping,
3859 loff_t const holebegin, loff_ 3237 loff_t const holebegin, loff_t const holelen, int even_cows)
3860 { 3238 {
3861 pgoff_t hba = (pgoff_t)(holebegin) >> !! 3239 pgoff_t hba = holebegin >> PAGE_SHIFT;
3862 pgoff_t hlen = ((pgoff_t)(holelen) + !! 3240 pgoff_t hlen = (holelen + PAGE_SIZE - 1) >> PAGE_SHIFT;
3863 3241
3864 /* Check for overflow. */ 3242 /* Check for overflow. */
3865 if (sizeof(holelen) > sizeof(hlen)) { 3243 if (sizeof(holelen) > sizeof(hlen)) {
3866 long long holeend = 3244 long long holeend =
3867 (holebegin + holelen 3245 (holebegin + holelen + PAGE_SIZE - 1) >> PAGE_SHIFT;
3868 if (holeend & ~(long long)ULO 3246 if (holeend & ~(long long)ULONG_MAX)
3869 hlen = ULONG_MAX - hb 3247 hlen = ULONG_MAX - hba + 1;
3870 } 3248 }
3871 3249
3872 unmap_mapping_pages(mapping, hba, hle 3250 unmap_mapping_pages(mapping, hba, hlen, even_cows);
3873 } 3251 }
3874 EXPORT_SYMBOL(unmap_mapping_range); 3252 EXPORT_SYMBOL(unmap_mapping_range);
3875 3253
3876 /* 3254 /*
3877 * Restore a potential device exclusive pte t <<
3878 */ <<
3879 static vm_fault_t remove_device_exclusive_ent <<
3880 { <<
3881 struct folio *folio = page_folio(vmf- <<
3882 struct vm_area_struct *vma = vmf->vma <<
3883 struct mmu_notifier_range range; <<
3884 vm_fault_t ret; <<
3885 <<
3886 /* <<
3887 * We need a reference to lock the fo <<
3888 * the PTL so a racing thread can rem <<
3889 * entry and unmap it. If the folio i <<
3890 * have been removed already. If it h <<
3891 * been re-allocated after being free <<
3892 * unlock it. <<
3893 */ <<
3894 if (!folio_try_get(folio)) <<
3895 return 0; <<
3896 <<
3897 ret = folio_lock_or_retry(folio, vmf) <<
3898 if (ret) { <<
3899 folio_put(folio); <<
3900 return ret; <<
3901 } <<
3902 mmu_notifier_range_init_owner(&range, <<
3903 vma->vm_mm, v <<
3904 (vmf->address <<
3905 mmu_notifier_invalidate_range_start(& <<
3906 <<
3907 vmf->pte = pte_offset_map_lock(vma->v <<
3908 &vmf->ptl); <<
3909 if (likely(vmf->pte && pte_same(ptep_ <<
3910 restore_exclusive_pte(vma, vm <<
3911 <<
3912 if (vmf->pte) <<
3913 pte_unmap_unlock(vmf->pte, vm <<
3914 folio_unlock(folio); <<
3915 folio_put(folio); <<
3916 <<
3917 mmu_notifier_invalidate_range_end(&ra <<
3918 return 0; <<
3919 } <<
3920 <<
3921 static inline bool should_try_to_free_swap(st <<
3922 st <<
3923 un <<
3924 { <<
3925 if (!folio_test_swapcache(folio)) <<
3926 return false; <<
3927 if (mem_cgroup_swap_full(folio) || (v <<
3928 folio_test_mlocked(folio)) <<
3929 return true; <<
3930 /* <<
3931 * If we want to map a page that's in <<
3932 * have to detect via the refcount if <<
3933 * user. Try freeing the swapcache to <<
3934 * reference only in case it's likely <<
3935 */ <<
3936 return (fault_flags & FAULT_FLAG_WRIT <<
3937 folio_ref_count(folio) == (1 <<
3938 } <<
3939 <<
3940 static vm_fault_t pte_marker_clear(struct vm_ <<
3941 { <<
3942 vmf->pte = pte_offset_map_lock(vmf->v <<
3943 vmf->a <<
3944 if (!vmf->pte) <<
3945 return 0; <<
3946 /* <<
3947 * Be careful so that we will only re <<
3948 * none pte. Otherwise it means the <<
3949 * <<
3950 * This should also cover the case wh <<
3951 * quickly from a PTE_MARKER_UFFD_WP <<
3952 * So is_pte_marker() check is not en <<
3953 */ <<
3954 if (pte_same(vmf->orig_pte, ptep_get( <<
3955 pte_clear(vmf->vma->vm_mm, vm <<
3956 pte_unmap_unlock(vmf->pte, vmf->ptl); <<
3957 return 0; <<
3958 } <<
3959 <<
3960 static vm_fault_t do_pte_missing(struct vm_fa <<
3961 { <<
3962 if (vma_is_anonymous(vmf->vma)) <<
3963 return do_anonymous_page(vmf) <<
3964 else <<
3965 return do_fault(vmf); <<
3966 } <<
3967 <<
3968 /* <<
3969 * This is actually a page-missing access, bu <<
3970 * installed. It means this pte was wr-prote <<
3971 */ <<
3972 static vm_fault_t pte_marker_handle_uffd_wp(s <<
3973 { <<
3974 /* <<
3975 * Just in case there're leftover spe <<
3976 * got unregistered - we can simply c <<
3977 */ <<
3978 if (unlikely(!userfaultfd_wp(vmf->vma <<
3979 return pte_marker_clear(vmf); <<
3980 <<
3981 return do_pte_missing(vmf); <<
3982 } <<
3983 <<
3984 static vm_fault_t handle_pte_marker(struct vm <<
3985 { <<
3986 swp_entry_t entry = pte_to_swp_entry( <<
3987 unsigned long marker = pte_marker_get <<
3988 <<
3989 /* <<
3990 * PTE markers should never be empty. <<
3991 * the best thing to do is to kill th <<
3992 */ <<
3993 if (WARN_ON_ONCE(!marker)) <<
3994 return VM_FAULT_SIGBUS; <<
3995 <<
3996 /* Higher priority than uffd-wp when <<
3997 if (marker & PTE_MARKER_POISONED) <<
3998 return VM_FAULT_HWPOISON; <<
3999 <<
4000 if (pte_marker_entry_uffd_wp(entry)) <<
4001 return pte_marker_handle_uffd <<
4002 <<
4003 /* This is an unknown pte marker */ <<
4004 return VM_FAULT_SIGBUS; <<
4005 } <<
4006 <<
4007 static struct folio *__alloc_swap_folio(struc <<
4008 { <<
4009 struct vm_area_struct *vma = vmf->vma <<
4010 struct folio *folio; <<
4011 swp_entry_t entry; <<
4012 <<
4013 folio = vma_alloc_folio(GFP_HIGHUSER_ <<
4014 vmf->address, <<
4015 if (!folio) <<
4016 return NULL; <<
4017 <<
4018 entry = pte_to_swp_entry(vmf->orig_pt <<
4019 if (mem_cgroup_swapin_charge_folio(fo <<
4020 GF <<
4021 folio_put(folio); <<
4022 return NULL; <<
4023 } <<
4024 <<
4025 return folio; <<
4026 } <<
4027 <<
4028 #ifdef CONFIG_TRANSPARENT_HUGEPAGE <<
4029 static inline int non_swapcache_batch(swp_ent <<
4030 { <<
4031 struct swap_info_struct *si = swp_swa <<
4032 pgoff_t offset = swp_offset(entry); <<
4033 int i; <<
4034 <<
4035 /* <<
4036 * While allocating a large folio and <<
4037 * the case the being faulted pte doe <<
4038 * ensure all PTEs have no cache as w <<
4039 * swap devices while the content is <<
4040 */ <<
4041 for (i = 0; i < max_nr; i++) { <<
4042 if ((si->swap_map[offset + i] <<
4043 return i; <<
4044 } <<
4045 <<
4046 return i; <<
4047 } <<
4048 <<
4049 /* <<
4050 * Check if the PTEs within a range are conti <<
4051 * and have consistent swapcache, zeromap. <<
4052 */ <<
4053 static bool can_swapin_thp(struct vm_fault *v <<
4054 { <<
4055 unsigned long addr; <<
4056 swp_entry_t entry; <<
4057 int idx; <<
4058 pte_t pte; <<
4059 <<
4060 addr = ALIGN_DOWN(vmf->address, nr_pa <<
4061 idx = (vmf->address - addr) / PAGE_SI <<
4062 pte = ptep_get(ptep); <<
4063 <<
4064 if (!pte_same(pte, pte_move_swp_offse <<
4065 return false; <<
4066 entry = pte_to_swp_entry(pte); <<
4067 if (swap_pte_batch(ptep, nr_pages, pt <<
4068 return false; <<
4069 <<
4070 /* <<
4071 * swap_read_folio() can't handle the <<
4072 * from different backends. And they <<
4073 * things might be added once zswap s <<
4074 */ <<
4075 if (unlikely(swap_zeromap_batch(entry <<
4076 return false; <<
4077 if (unlikely(non_swapcache_batch(entr <<
4078 return false; <<
4079 <<
4080 return true; <<
4081 } <<
4082 <<
4083 static inline unsigned long thp_swap_suitable <<
4084 <<
4085 <<
4086 { <<
4087 int order, nr; <<
4088 <<
4089 order = highest_order(orders); <<
4090 <<
4091 /* <<
4092 * To swap in a THP with nr pages, we <<
4093 * is aligned with that number, as it <<
4094 * This helps filter out most invalid <<
4095 */ <<
4096 while (orders) { <<
4097 nr = 1 << order; <<
4098 if ((addr >> PAGE_SHIFT) % nr <<
4099 break; <<
4100 order = next_order(&orders, o <<
4101 } <<
4102 <<
4103 return orders; <<
4104 } <<
4105 <<
4106 static struct folio *alloc_swap_folio(struct <<
4107 { <<
4108 struct vm_area_struct *vma = vmf->vma <<
4109 unsigned long orders; <<
4110 struct folio *folio; <<
4111 unsigned long addr; <<
4112 swp_entry_t entry; <<
4113 spinlock_t *ptl; <<
4114 pte_t *pte; <<
4115 gfp_t gfp; <<
4116 int order; <<
4117 <<
4118 /* <<
4119 * If uffd is active for the vma we n <<
4120 * maintain the uffd semantics. <<
4121 */ <<
4122 if (unlikely(userfaultfd_armed(vma))) <<
4123 goto fallback; <<
4124 <<
4125 /* <<
4126 * A large swapped out folio could be <<
4127 * lack handling for such cases, so f <<
4128 * folio. <<
4129 */ <<
4130 if (!zswap_never_enabled()) <<
4131 goto fallback; <<
4132 <<
4133 entry = pte_to_swp_entry(vmf->orig_pt <<
4134 /* <<
4135 * Get a list of all the (large) orde <<
4136 * and suitable for swapping THP. <<
4137 */ <<
4138 orders = thp_vma_allowable_orders(vma <<
4139 TVA_IN_PF | TVA_ENFOR <<
4140 orders = thp_vma_suitable_orders(vma, <<
4141 orders = thp_swap_suitable_orders(swp <<
4142 vmf <<
4143 <<
4144 if (!orders) <<
4145 goto fallback; <<
4146 <<
4147 pte = pte_offset_map_lock(vmf->vma->v <<
4148 vmf->addres <<
4149 if (unlikely(!pte)) <<
4150 goto fallback; <<
4151 <<
4152 /* <<
4153 * For do_swap_page, find the highest <<
4154 * completely swap entries with conti <<
4155 */ <<
4156 order = highest_order(orders); <<
4157 while (orders) { <<
4158 addr = ALIGN_DOWN(vmf->addres <<
4159 if (can_swapin_thp(vmf, pte + <<
4160 break; <<
4161 order = next_order(&orders, o <<
4162 } <<
4163 <<
4164 pte_unmap_unlock(pte, ptl); <<
4165 <<
4166 /* Try allocating the highest of the <<
4167 gfp = vma_thp_gfp_mask(vma); <<
4168 while (orders) { <<
4169 addr = ALIGN_DOWN(vmf->addres <<
4170 folio = vma_alloc_folio(gfp, <<
4171 if (folio) { <<
4172 if (!mem_cgroup_swapi <<
4173 <<
4174 return folio; <<
4175 folio_put(folio); <<
4176 } <<
4177 order = next_order(&orders, o <<
4178 } <<
4179 <<
4180 fallback: <<
4181 return __alloc_swap_folio(vmf); <<
4182 } <<
4183 #else /* !CONFIG_TRANSPARENT_HUGEPAGE */ <<
4184 static struct folio *alloc_swap_folio(struct <<
4185 { <<
4186 return __alloc_swap_folio(vmf); <<
4187 } <<
4188 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ <<
4189 <<
4190 static DECLARE_WAIT_QUEUE_HEAD(swapcache_wq); <<
4191 <<
4192 /* <<
4193 * We enter with non-exclusive mmap_lock (to 3255 * We enter with non-exclusive mmap_lock (to exclude vma changes,
4194 * but allow concurrent faults), and pte mapp 3256 * but allow concurrent faults), and pte mapped but not yet locked.
4195 * We return with pte unmapped and unlocked. 3257 * We return with pte unmapped and unlocked.
4196 * 3258 *
4197 * We return with the mmap_lock locked or unl 3259 * We return with the mmap_lock locked or unlocked in the same cases
4198 * as does filemap_fault(). 3260 * as does filemap_fault().
4199 */ 3261 */
4200 vm_fault_t do_swap_page(struct vm_fault *vmf) 3262 vm_fault_t do_swap_page(struct vm_fault *vmf)
4201 { 3263 {
4202 struct vm_area_struct *vma = vmf->vma 3264 struct vm_area_struct *vma = vmf->vma;
4203 struct folio *swapcache, *folio = NUL !! 3265 struct page *page = NULL, *swapcache;
4204 DECLARE_WAITQUEUE(wait, current); <<
4205 struct page *page; <<
4206 struct swap_info_struct *si = NULL; <<
4207 rmap_t rmap_flags = RMAP_NONE; <<
4208 bool need_clear_cache = false; <<
4209 bool exclusive = false; <<
4210 swp_entry_t entry; 3266 swp_entry_t entry;
4211 pte_t pte; 3267 pte_t pte;
>> 3268 int locked;
>> 3269 int exclusive = 0;
4212 vm_fault_t ret = 0; 3270 vm_fault_t ret = 0;
4213 void *shadow = NULL; 3271 void *shadow = NULL;
4214 int nr_pages; <<
4215 unsigned long page_idx; <<
4216 unsigned long address; <<
4217 pte_t *ptep; <<
4218 3272
4219 if (!pte_unmap_same(vmf)) !! 3273 if (!pte_unmap_same(vma->vm_mm, vmf->pmd, vmf->pte, vmf->orig_pte))
4220 goto out; 3274 goto out;
4221 3275
4222 entry = pte_to_swp_entry(vmf->orig_pt 3276 entry = pte_to_swp_entry(vmf->orig_pte);
4223 if (unlikely(non_swap_entry(entry))) 3277 if (unlikely(non_swap_entry(entry))) {
4224 if (is_migration_entry(entry) 3278 if (is_migration_entry(entry)) {
4225 migration_entry_wait( 3279 migration_entry_wait(vma->vm_mm, vmf->pmd,
4226 3280 vmf->address);
4227 } else if (is_device_exclusiv <<
4228 vmf->page = pfn_swap_ <<
4229 ret = remove_device_e <<
4230 } else if (is_device_private_ 3281 } else if (is_device_private_entry(entry)) {
4231 if (vmf->flags & FAUL !! 3282 vmf->page = device_private_entry_to_page(entry);
4232 /* <<
4233 * migrate_to <<
4234 * under VMA <<
4235 */ <<
4236 vma_end_read( <<
4237 ret = VM_FAUL <<
4238 goto out; <<
4239 } <<
4240 <<
4241 vmf->page = pfn_swap_ <<
4242 vmf->pte = pte_offset <<
4243 vmf-> <<
4244 if (unlikely(!vmf->pt <<
4245 !pte_sam <<
4246 <<
4247 goto unlock; <<
4248 <<
4249 /* <<
4250 * Get a page referen <<
4251 * freed. <<
4252 */ <<
4253 get_page(vmf->page); <<
4254 pte_unmap_unlock(vmf- <<
4255 ret = vmf->page->pgma 3283 ret = vmf->page->pgmap->ops->migrate_to_ram(vmf);
4256 put_page(vmf->page); <<
4257 } else if (is_hwpoison_entry( 3284 } else if (is_hwpoison_entry(entry)) {
4258 ret = VM_FAULT_HWPOIS 3285 ret = VM_FAULT_HWPOISON;
4259 } else if (is_pte_marker_entr <<
4260 ret = handle_pte_mark <<
4261 } else { 3286 } else {
4262 print_bad_pte(vma, vm 3287 print_bad_pte(vma, vmf->address, vmf->orig_pte, NULL);
4263 ret = VM_FAULT_SIGBUS 3288 ret = VM_FAULT_SIGBUS;
4264 } 3289 }
4265 goto out; 3290 goto out;
4266 } 3291 }
4267 3292
4268 /* Prevent swapoff from happening to <<
4269 si = get_swap_device(entry); <<
4270 if (unlikely(!si)) <<
4271 goto out; <<
4272 3293
4273 folio = swap_cache_get_folio(entry, v !! 3294 delayacct_set_flag(DELAYACCT_PF_SWAPIN);
4274 if (folio) !! 3295 page = lookup_swap_cache(entry, vma, vmf->address);
4275 page = folio_file_page(folio, !! 3296 swapcache = page;
4276 swapcache = folio; !! 3297
>> 3298 if (!page) {
>> 3299 struct swap_info_struct *si = swp_swap_info(entry);
4277 3300
4278 if (!folio) { <<
4279 if (data_race(si->flags & SWP 3301 if (data_race(si->flags & SWP_SYNCHRONOUS_IO) &&
4280 __swap_count(entry) == 1) 3302 __swap_count(entry) == 1) {
4281 /* skip swapcache */ 3303 /* skip swapcache */
4282 folio = alloc_swap_fo !! 3304 page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma,
4283 if (folio) { !! 3305 vmf->address);
4284 __folio_set_l !! 3306 if (page) {
4285 __folio_set_s !! 3307 int err;
4286 !! 3308
4287 nr_pages = fo !! 3309 __SetPageLocked(page);
4288 if (folio_tes !! 3310 __SetPageSwapBacked(page);
4289 entry !! 3311 set_page_private(page, entry.val);
4290 /* !! 3312
4291 * Prevent pa !! 3313 /* Tell memcg to use swap ownership records */
4292 * the cache !! 3314 SetPageSwapCache(page);
4293 * may finish !! 3315 err = mem_cgroup_charge(page, vma->vm_mm,
4294 * swapout re !! 3316 GFP_KERNEL);
4295 * undetectab !! 3317 ClearPageSwapCache(page);
4296 * to entry r !! 3318 if (err) {
4297 */ !! 3319 ret = VM_FAULT_OOM;
4298 if (swapcache <<
4299 /* <<
4300 * Re <<
4301 * re <<
4302 */ <<
4303 add_w <<
4304 sched <<
4305 remov <<
4306 goto 3320 goto out_page;
4307 } 3321 }
4308 need_clear_ca <<
4309 <<
4310 mem_cgroup_sw <<
4311 3322
4312 shadow = get_ 3323 shadow = get_shadow_from_swap_cache(entry);
4313 if (shadow) 3324 if (shadow)
4314 worki !! 3325 workingset_refault(page, shadow);
4315 3326
4316 folio_add_lru !! 3327 lru_cache_add(page);
4317 !! 3328 swap_readpage(page, true);
4318 /* To provide <<
4319 folio->swap = <<
4320 swap_read_fol <<
4321 folio->privat <<
4322 } 3329 }
4323 } else { 3330 } else {
4324 folio = swapin_readah !! 3331 page = swapin_readahead(entry, GFP_HIGHUSER_MOVABLE,
4325 3332 vmf);
4326 swapcache = folio; !! 3333 swapcache = page;
4327 } 3334 }
4328 3335
4329 if (!folio) { !! 3336 if (!page) {
4330 /* 3337 /*
4331 * Back out if somebo 3338 * Back out if somebody else faulted in this pte
4332 * while we released 3339 * while we released the pte lock.
4333 */ 3340 */
4334 vmf->pte = pte_offset 3341 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
4335 vmf-> 3342 vmf->address, &vmf->ptl);
4336 if (likely(vmf->pte & !! 3343 if (likely(pte_same(*vmf->pte, vmf->orig_pte)))
4337 pte_same(p <<
4338 ret = VM_FAUL 3344 ret = VM_FAULT_OOM;
>> 3345 delayacct_clear_flag(DELAYACCT_PF_SWAPIN);
4339 goto unlock; 3346 goto unlock;
4340 } 3347 }
4341 3348
4342 /* Had to read the page from 3349 /* Had to read the page from swap area: Major fault */
4343 ret = VM_FAULT_MAJOR; 3350 ret = VM_FAULT_MAJOR;
4344 count_vm_event(PGMAJFAULT); 3351 count_vm_event(PGMAJFAULT);
4345 count_memcg_event_mm(vma->vm_ 3352 count_memcg_event_mm(vma->vm_mm, PGMAJFAULT);
4346 page = folio_file_page(folio, <<
4347 } else if (PageHWPoison(page)) { 3353 } else if (PageHWPoison(page)) {
4348 /* 3354 /*
4349 * hwpoisoned dirty swapcache 3355 * hwpoisoned dirty swapcache pages are kept for killing
4350 * owner processes (which may 3356 * owner processes (which may be unknown at hwpoison time)
4351 */ 3357 */
4352 ret = VM_FAULT_HWPOISON; 3358 ret = VM_FAULT_HWPOISON;
>> 3359 delayacct_clear_flag(DELAYACCT_PF_SWAPIN);
4353 goto out_release; 3360 goto out_release;
4354 } 3361 }
4355 3362
4356 ret |= folio_lock_or_retry(folio, vmf !! 3363 locked = lock_page_or_retry(page, vma->vm_mm, vmf->flags);
4357 if (ret & VM_FAULT_RETRY) <<
4358 goto out_release; <<
4359 3364
4360 if (swapcache) { !! 3365 delayacct_clear_flag(DELAYACCT_PF_SWAPIN);
4361 /* !! 3366 if (!locked) {
4362 * Make sure folio_free_swap( !! 3367 ret |= VM_FAULT_RETRY;
4363 * swapcache from under us. !! 3368 goto out_release;
4364 * below, are not enough to e !! 3369 }
4365 * swapcache, we need to chec <<
4366 * changed. <<
4367 */ <<
4368 if (unlikely(!folio_test_swap <<
4369 page_swap_entry( <<
4370 goto out_page; <<
4371 3370
4372 /* !! 3371 /*
4373 * KSM sometimes has to copy !! 3372 * Make sure try_to_free_swap or reuse_swap_page or swapoff did not
4374 * page->index of !PageKSM() !! 3373 * release the swapcache from under us. The page pin, and pte_same
4375 * anon VMA -- PageKSM() is l !! 3374 * test below, are not enough to exclude that. Even if it is still
4376 */ !! 3375 * swapcache, we need to check that the page's swap has not changed.
4377 folio = ksm_might_need_to_cop !! 3376 */
4378 if (unlikely(!folio)) { !! 3377 if (unlikely((!PageSwapCache(page) ||
4379 ret = VM_FAULT_OOM; !! 3378 page_private(page) != entry.val)) && swapcache)
4380 folio = swapcache; !! 3379 goto out_page;
4381 goto out_page; <<
4382 } else if (unlikely(folio == <<
4383 ret = VM_FAULT_HWPOIS <<
4384 folio = swapcache; <<
4385 goto out_page; <<
4386 } <<
4387 if (folio != swapcache) <<
4388 page = folio_page(fol <<
4389 3380
4390 /* !! 3381 page = ksm_might_need_to_copy(page, vma, vmf->address);
4391 * If we want to map a page t !! 3382 if (unlikely(!page)) {
4392 * have to detect via the ref !! 3383 ret = VM_FAULT_OOM;
4393 * owner. Try removing the ex !! 3384 page = swapcache;
4394 * caches if required. !! 3385 goto out_page;
4395 */ <<
4396 if ((vmf->flags & FAULT_FLAG_ <<
4397 !folio_test_ksm(folio) && <<
4398 lru_add_drain(); <<
4399 } 3386 }
4400 3387
4401 folio_throttle_swaprate(folio, GFP_KE !! 3388 cgroup_throttle_swaprate(page, GFP_KERNEL);
4402 3389
4403 /* 3390 /*
4404 * Back out if somebody else already 3391 * Back out if somebody else already faulted in this pte.
4405 */ 3392 */
4406 vmf->pte = pte_offset_map_lock(vma->v 3393 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
4407 &vmf->ptl); 3394 &vmf->ptl);
4408 if (unlikely(!vmf->pte || !pte_same(p !! 3395 if (unlikely(!pte_same(*vmf->pte, vmf->orig_pte)))
4409 goto out_nomap; 3396 goto out_nomap;
4410 3397
4411 if (unlikely(!folio_test_uptodate(fol !! 3398 if (unlikely(!PageUptodate(page))) {
4412 ret = VM_FAULT_SIGBUS; 3399 ret = VM_FAULT_SIGBUS;
4413 goto out_nomap; 3400 goto out_nomap;
4414 } 3401 }
4415 3402
4416 /* allocated large folios for SWP_SYN <<
4417 if (folio_test_large(folio) && !folio <<
4418 unsigned long nr = folio_nr_p <<
4419 unsigned long folio_start = A <<
4420 unsigned long idx = (vmf->add <<
4421 pte_t *folio_ptep = vmf->pte <<
4422 pte_t folio_pte = ptep_get(fo <<
4423 <<
4424 if (!pte_same(folio_pte, pte_ <<
4425 swap_pte_batch(folio_ptep <<
4426 goto out_nomap; <<
4427 <<
4428 page_idx = idx; <<
4429 address = folio_start; <<
4430 ptep = folio_ptep; <<
4431 goto check_folio; <<
4432 } <<
4433 <<
4434 nr_pages = 1; <<
4435 page_idx = 0; <<
4436 address = vmf->address; <<
4437 ptep = vmf->pte; <<
4438 if (folio_test_large(folio) && folio_ <<
4439 int nr = folio_nr_pages(folio <<
4440 unsigned long idx = folio_pag <<
4441 unsigned long folio_start = a <<
4442 unsigned long folio_end = fol <<
4443 pte_t *folio_ptep; <<
4444 pte_t folio_pte; <<
4445 <<
4446 if (unlikely(folio_start < ma <<
4447 goto check_folio; <<
4448 if (unlikely(folio_end > pmd_ <<
4449 goto check_folio; <<
4450 <<
4451 folio_ptep = vmf->pte - idx; <<
4452 folio_pte = ptep_get(folio_pt <<
4453 if (!pte_same(folio_pte, pte_ <<
4454 swap_pte_batch(folio_ptep <<
4455 goto check_folio; <<
4456 <<
4457 page_idx = idx; <<
4458 address = folio_start; <<
4459 ptep = folio_ptep; <<
4460 nr_pages = nr; <<
4461 entry = folio->swap; <<
4462 page = &folio->page; <<
4463 } <<
4464 <<
4465 check_folio: <<
4466 /* <<
4467 * PG_anon_exclusive reuses PG_mapped <<
4468 * must never point at an anonymous p <<
4469 * PG_anon_exclusive. Sanity check th <<
4470 * no filesystem set PG_mappedtodisk <<
4471 * check after taking the PT lock and <<
4472 * concurrently faulted in this page <<
4473 */ <<
4474 BUG_ON(!folio_test_anon(folio) && fol <<
4475 BUG_ON(folio_test_anon(folio) && Page <<
4476 <<
4477 /* <<
4478 * Check under PT lock (to protect ag <<
4479 * the swap entry concurrently) for c <<
4480 */ <<
4481 if (!folio_test_ksm(folio)) { <<
4482 exclusive = pte_swp_exclusive <<
4483 if (folio != swapcache) { <<
4484 /* <<
4485 * We have a fresh pa <<
4486 * swapcache -> certa <<
4487 */ <<
4488 exclusive = true; <<
4489 } else if (exclusive && folio <<
4490 data_race(si->flags <<
4491 /* <<
4492 * This is tricky: no <<
4493 * concurrent page mo <<
4494 * <<
4495 * So if we stumble o <<
4496 * we must not set th <<
4497 * map it writable wi <<
4498 * while still under <<
4499 * <<
4500 * For these problema <<
4501 * exclusive marker: <<
4502 * writeback only if <<
4503 * there are no unexp <<
4504 * unmapping succeede <<
4505 * further GUP refere <<
4506 * appear, so droppin <<
4507 * it only R/O is fin <<
4508 */ <<
4509 exclusive = false; <<
4510 } <<
4511 } <<
4512 <<
4513 /* 3403 /*
4514 * Some architectures may have to res !! 3404 * The page isn't present yet, go ahead with the fault.
4515 * when reading from swap. This metad !! 3405 *
4516 * so this must be called before swap !! 3406 * Be careful about the sequence of operations here.
4517 */ !! 3407 * To get its accounting right, reuse_swap_page() must be called
4518 arch_swap_restore(folio_swap(entry, f !! 3408 * while the page is counted on swap but not yet in mapcount i.e.
4519 !! 3409 * before page_add_anon_rmap() and swap_free(); try_to_free_swap()
4520 /* !! 3410 * must be called after the swap_free(), or it will never succeed.
4521 * Remove the swap entry and conditio <<
4522 * We're already holding a reference <<
4523 * yet. <<
4524 */ 3411 */
4525 swap_free_nr(entry, nr_pages); <<
4526 if (should_try_to_free_swap(folio, vm <<
4527 folio_free_swap(folio); <<
4528 3412
4529 add_mm_counter(vma->vm_mm, MM_ANONPAG !! 3413 inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES);
4530 add_mm_counter(vma->vm_mm, MM_SWAPENT !! 3414 dec_mm_counter_fast(vma->vm_mm, MM_SWAPENTS);
4531 pte = mk_pte(page, vma->vm_page_prot) 3415 pte = mk_pte(page, vma->vm_page_prot);
>> 3416 if ((vmf->flags & FAULT_FLAG_WRITE) && reuse_swap_page(page, NULL)) {
>> 3417 pte = maybe_mkwrite(pte_mkdirty(pte), vma);
>> 3418 vmf->flags &= ~FAULT_FLAG_WRITE;
>> 3419 ret |= VM_FAULT_WRITE;
>> 3420 exclusive = RMAP_EXCLUSIVE;
>> 3421 }
>> 3422 flush_icache_page(vma, page);
4532 if (pte_swp_soft_dirty(vmf->orig_pte) 3423 if (pte_swp_soft_dirty(vmf->orig_pte))
4533 pte = pte_mksoft_dirty(pte); 3424 pte = pte_mksoft_dirty(pte);
4534 if (pte_swp_uffd_wp(vmf->orig_pte)) !! 3425 if (pte_swp_uffd_wp(vmf->orig_pte)) {
4535 pte = pte_mkuffd_wp(pte); 3426 pte = pte_mkuffd_wp(pte);
4536 !! 3427 pte = pte_wrprotect(pte);
4537 /* <<
4538 * Same logic as in do_wp_page(); how <<
4539 * certainly not shared either becaus <<
4540 * exposing them to the swapcache or <<
4541 * exclusivity. <<
4542 */ <<
4543 if (!folio_test_ksm(folio) && <<
4544 (exclusive || folio_ref_count(fol <<
4545 if ((vma->vm_flags & VM_WRITE <<
4546 !pte_needs_soft_dirty_wp( <<
4547 pte = pte_mkwrite(pte <<
4548 if (vmf->flags & FAUL <<
4549 pte = pte_mkd <<
4550 vmf->flags &= <<
4551 } <<
4552 } <<
4553 rmap_flags |= RMAP_EXCLUSIVE; <<
4554 } 3428 }
4555 folio_ref_add(folio, nr_pages - 1); !! 3429 set_pte_at(vma->vm_mm, vmf->address, vmf->pte, pte);
4556 flush_icache_pages(vma, page, nr_page !! 3430 arch_do_swap_page(vma->vm_mm, vma, vmf->address, pte, vmf->orig_pte);
4557 vmf->orig_pte = pte_advance_pfn(pte, !! 3431 vmf->orig_pte = pte;
4558 3432
4559 /* ksm created a completely new copy 3433 /* ksm created a completely new copy */
4560 if (unlikely(folio != swapcache && sw !! 3434 if (unlikely(page != swapcache && swapcache)) {
4561 folio_add_new_anon_rmap(folio !! 3435 page_add_new_anon_rmap(page, vma, vmf->address, false);
4562 folio_add_lru_vma(folio, vma) !! 3436 lru_cache_add_inactive_or_unevictable(page, vma);
4563 } else if (!folio_test_anon(folio)) { <<
4564 /* <<
4565 * We currently only expect s <<
4566 * fully exclusive or fully s <<
4567 * folios which are fully exc <<
4568 * folios within swapcache he <<
4569 */ <<
4570 VM_WARN_ON_ONCE(folio_test_la <<
4571 VM_WARN_ON_FOLIO(!folio_test_ <<
4572 folio_add_new_anon_rmap(folio <<
4573 } else { 3437 } else {
4574 folio_add_anon_rmap_ptes(foli !! 3438 do_page_add_anon_rmap(page, vma, vmf->address, exclusive);
4575 rmap_ <<
4576 } 3439 }
4577 3440
4578 VM_BUG_ON(!folio_test_anon(folio) || !! 3441 swap_free(entry);
4579 (pte_write(pte) && !P !! 3442 if (mem_cgroup_swap_full(page) ||
4580 set_ptes(vma->vm_mm, address, ptep, p !! 3443 (vma->vm_flags & VM_LOCKED) || PageMlocked(page))
4581 arch_do_swap_page_nr(vma->vm_mm, vma, !! 3444 try_to_free_swap(page);
4582 pte, pte, nr_pages); !! 3445 unlock_page(page);
4583 !! 3446 if (page != swapcache && swapcache) {
4584 folio_unlock(folio); <<
4585 if (folio != swapcache && swapcache) <<
4586 /* 3447 /*
4587 * Hold the lock to avoid the 3448 * Hold the lock to avoid the swap entry to be reused
4588 * until we take the PT lock 3449 * until we take the PT lock for the pte_same() check
4589 * (to avoid false positives 3450 * (to avoid false positives from pte_same). For
4590 * further safety release the 3451 * further safety release the lock after the swap_free
4591 * so that the swap count won 3452 * so that the swap count won't change under a
4592 * parallel locked swapcache. 3453 * parallel locked swapcache.
4593 */ 3454 */
4594 folio_unlock(swapcache); !! 3455 unlock_page(swapcache);
4595 folio_put(swapcache); !! 3456 put_page(swapcache);
4596 } 3457 }
4597 3458
4598 if (vmf->flags & FAULT_FLAG_WRITE) { 3459 if (vmf->flags & FAULT_FLAG_WRITE) {
4599 ret |= do_wp_page(vmf); 3460 ret |= do_wp_page(vmf);
4600 if (ret & VM_FAULT_ERROR) 3461 if (ret & VM_FAULT_ERROR)
4601 ret &= VM_FAULT_ERROR 3462 ret &= VM_FAULT_ERROR;
4602 goto out; 3463 goto out;
4603 } 3464 }
4604 3465
4605 /* No need to invalidate - it was non 3466 /* No need to invalidate - it was non-present before */
4606 update_mmu_cache_range(vmf, vma, addr !! 3467 update_mmu_cache(vma, vmf->address, vmf->pte);
4607 unlock: 3468 unlock:
4608 if (vmf->pte) !! 3469 pte_unmap_unlock(vmf->pte, vmf->ptl);
4609 pte_unmap_unlock(vmf->pte, vm <<
4610 out: 3470 out:
4611 /* Clear the swap cache pin for direc <<
4612 if (need_clear_cache) { <<
4613 swapcache_clear(si, entry, nr <<
4614 if (waitqueue_active(&swapcac <<
4615 wake_up(&swapcache_wq <<
4616 } <<
4617 if (si) <<
4618 put_swap_device(si); <<
4619 return ret; 3471 return ret;
4620 out_nomap: 3472 out_nomap:
4621 if (vmf->pte) !! 3473 pte_unmap_unlock(vmf->pte, vmf->ptl);
4622 pte_unmap_unlock(vmf->pte, vm <<
4623 out_page: 3474 out_page:
4624 folio_unlock(folio); !! 3475 unlock_page(page);
4625 out_release: 3476 out_release:
4626 folio_put(folio); !! 3477 put_page(page);
4627 if (folio != swapcache && swapcache) !! 3478 if (page != swapcache && swapcache) {
4628 folio_unlock(swapcache); !! 3479 unlock_page(swapcache);
4629 folio_put(swapcache); !! 3480 put_page(swapcache);
4630 } <<
4631 if (need_clear_cache) { <<
4632 swapcache_clear(si, entry, nr <<
4633 if (waitqueue_active(&swapcac <<
4634 wake_up(&swapcache_wq <<
4635 } 3481 }
4636 if (si) <<
4637 put_swap_device(si); <<
4638 return ret; 3482 return ret;
4639 } 3483 }
4640 3484
4641 static bool pte_range_none(pte_t *pte, int nr <<
4642 { <<
4643 int i; <<
4644 <<
4645 for (i = 0; i < nr_pages; i++) { <<
4646 if (!pte_none(ptep_get_lockle <<
4647 return false; <<
4648 } <<
4649 <<
4650 return true; <<
4651 } <<
4652 <<
4653 static struct folio *alloc_anon_folio(struct <<
4654 { <<
4655 struct vm_area_struct *vma = vmf->vma <<
4656 #ifdef CONFIG_TRANSPARENT_HUGEPAGE <<
4657 unsigned long orders; <<
4658 struct folio *folio; <<
4659 unsigned long addr; <<
4660 pte_t *pte; <<
4661 gfp_t gfp; <<
4662 int order; <<
4663 <<
4664 /* <<
4665 * If uffd is active for the vma we n <<
4666 * maintain the uffd semantics. <<
4667 */ <<
4668 if (unlikely(userfaultfd_armed(vma))) <<
4669 goto fallback; <<
4670 <<
4671 /* <<
4672 * Get a list of all the (large) orde <<
4673 * for this vma. Then filter out the <<
4674 * the faulting address and still be <<
4675 */ <<
4676 orders = thp_vma_allowable_orders(vma <<
4677 TVA_IN_PF | TVA_ENFOR <<
4678 orders = thp_vma_suitable_orders(vma, <<
4679 <<
4680 if (!orders) <<
4681 goto fallback; <<
4682 <<
4683 pte = pte_offset_map(vmf->pmd, vmf->a <<
4684 if (!pte) <<
4685 return ERR_PTR(-EAGAIN); <<
4686 <<
4687 /* <<
4688 * Find the highest order where the a <<
4689 * pte_none(). Note that all remainin <<
4690 * pte_none(). <<
4691 */ <<
4692 order = highest_order(orders); <<
4693 while (orders) { <<
4694 addr = ALIGN_DOWN(vmf->addres <<
4695 if (pte_range_none(pte + pte_ <<
4696 break; <<
4697 order = next_order(&orders, o <<
4698 } <<
4699 <<
4700 pte_unmap(pte); <<
4701 <<
4702 if (!orders) <<
4703 goto fallback; <<
4704 <<
4705 /* Try allocating the highest of the <<
4706 gfp = vma_thp_gfp_mask(vma); <<
4707 while (orders) { <<
4708 addr = ALIGN_DOWN(vmf->addres <<
4709 folio = vma_alloc_folio(gfp, <<
4710 if (folio) { <<
4711 if (mem_cgroup_charge <<
4712 count_mthp_st <<
4713 folio_put(fol <<
4714 goto next; <<
4715 } <<
4716 folio_throttle_swapra <<
4717 folio_zero_user(folio <<
4718 return folio; <<
4719 } <<
4720 next: <<
4721 count_mthp_stat(order, MTHP_S <<
4722 order = next_order(&orders, o <<
4723 } <<
4724 <<
4725 fallback: <<
4726 #endif <<
4727 return folio_prealloc(vma->vm_mm, vma <<
4728 } <<
4729 <<
4730 /* 3485 /*
4731 * We enter with non-exclusive mmap_lock (to 3486 * We enter with non-exclusive mmap_lock (to exclude vma changes,
4732 * but allow concurrent faults), and pte mapp 3487 * but allow concurrent faults), and pte mapped but not yet locked.
4733 * We return with mmap_lock still held, but p 3488 * We return with mmap_lock still held, but pte unmapped and unlocked.
4734 */ 3489 */
4735 static vm_fault_t do_anonymous_page(struct vm 3490 static vm_fault_t do_anonymous_page(struct vm_fault *vmf)
4736 { 3491 {
4737 struct vm_area_struct *vma = vmf->vma 3492 struct vm_area_struct *vma = vmf->vma;
4738 unsigned long addr = vmf->address; !! 3493 struct page *page;
4739 struct folio *folio; <<
4740 vm_fault_t ret = 0; 3494 vm_fault_t ret = 0;
4741 int nr_pages = 1; <<
4742 pte_t entry; 3495 pte_t entry;
4743 3496
4744 /* File mapping without ->vm_ops ? */ 3497 /* File mapping without ->vm_ops ? */
4745 if (vma->vm_flags & VM_SHARED) 3498 if (vma->vm_flags & VM_SHARED)
4746 return VM_FAULT_SIGBUS; 3499 return VM_FAULT_SIGBUS;
4747 3500
4748 /* 3501 /*
4749 * Use pte_alloc() instead of pte_all !! 3502 * Use pte_alloc() instead of pte_alloc_map(). We can't run
4750 * be distinguished from a transient !! 3503 * pte_offset_map() on pmds where a huge pmd might be created
>> 3504 * from a different thread.
>> 3505 *
>> 3506 * pte_alloc_map() is safe to use under mmap_write_lock(mm) or when
>> 3507 * parallel threads are excluded by other means.
>> 3508 *
>> 3509 * Here we only have mmap_read_lock(mm).
4751 */ 3510 */
4752 if (pte_alloc(vma->vm_mm, vmf->pmd)) 3511 if (pte_alloc(vma->vm_mm, vmf->pmd))
4753 return VM_FAULT_OOM; 3512 return VM_FAULT_OOM;
4754 3513
>> 3514 /* See the comment in pte_alloc_one_map() */
>> 3515 if (unlikely(pmd_trans_unstable(vmf->pmd)))
>> 3516 return 0;
>> 3517
4755 /* Use the zero-page for reads */ 3518 /* Use the zero-page for reads */
4756 if (!(vmf->flags & FAULT_FLAG_WRITE) 3519 if (!(vmf->flags & FAULT_FLAG_WRITE) &&
4757 !mm_forbids_zeropage( 3520 !mm_forbids_zeropage(vma->vm_mm)) {
4758 entry = pte_mkspecial(pfn_pte 3521 entry = pte_mkspecial(pfn_pte(my_zero_pfn(vmf->address),
4759 3522 vma->vm_page_prot));
4760 vmf->pte = pte_offset_map_loc 3523 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
4761 vmf->address, 3524 vmf->address, &vmf->ptl);
4762 if (!vmf->pte) !! 3525 if (!pte_none(*vmf->pte)) {
4763 goto unlock; <<
4764 if (vmf_pte_changed(vmf)) { <<
4765 update_mmu_tlb(vma, v 3526 update_mmu_tlb(vma, vmf->address, vmf->pte);
4766 goto unlock; 3527 goto unlock;
4767 } 3528 }
4768 ret = check_stable_address_sp 3529 ret = check_stable_address_space(vma->vm_mm);
4769 if (ret) 3530 if (ret)
4770 goto unlock; 3531 goto unlock;
4771 /* Deliver the page fault to 3532 /* Deliver the page fault to userland, check inside PT lock */
4772 if (userfaultfd_missing(vma)) 3533 if (userfaultfd_missing(vma)) {
4773 pte_unmap_unlock(vmf- 3534 pte_unmap_unlock(vmf->pte, vmf->ptl);
4774 return handle_userfau 3535 return handle_userfault(vmf, VM_UFFD_MISSING);
4775 } 3536 }
4776 goto setpte; 3537 goto setpte;
4777 } 3538 }
4778 3539
4779 /* Allocate our own private page. */ 3540 /* Allocate our own private page. */
4780 ret = vmf_anon_prepare(vmf); !! 3541 if (unlikely(anon_vma_prepare(vma)))
4781 if (ret) !! 3542 goto oom;
4782 return ret; !! 3543 page = alloc_zeroed_user_highpage_movable(vma, vmf->address);
4783 /* Returns NULL on OOM or ERR_PTR(-EA !! 3544 if (!page)
4784 folio = alloc_anon_folio(vmf); <<
4785 if (IS_ERR(folio)) <<
4786 return 0; <<
4787 if (!folio) <<
4788 goto oom; 3545 goto oom;
4789 3546
4790 nr_pages = folio_nr_pages(folio); !! 3547 if (mem_cgroup_charge(page, vma->vm_mm, GFP_KERNEL))
4791 addr = ALIGN_DOWN(vmf->address, nr_pa !! 3548 goto oom_free_page;
>> 3549 cgroup_throttle_swaprate(page, GFP_KERNEL);
4792 3550
4793 /* 3551 /*
4794 * The memory barrier inside __folio_ !! 3552 * The memory barrier inside __SetPageUptodate makes sure that
4795 * preceding stores to the page conte 3553 * preceding stores to the page contents become visible before
4796 * the set_pte_at() write. 3554 * the set_pte_at() write.
4797 */ 3555 */
4798 __folio_mark_uptodate(folio); !! 3556 __SetPageUptodate(page);
4799 3557
4800 entry = mk_pte(&folio->page, vma->vm_ !! 3558 entry = mk_pte(page, vma->vm_page_prot);
4801 entry = pte_sw_mkyoung(entry); 3559 entry = pte_sw_mkyoung(entry);
4802 if (vma->vm_flags & VM_WRITE) 3560 if (vma->vm_flags & VM_WRITE)
4803 entry = pte_mkwrite(pte_mkdir !! 3561 entry = pte_mkwrite(pte_mkdirty(entry));
4804 3562
4805 vmf->pte = pte_offset_map_lock(vma->v !! 3563 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
4806 if (!vmf->pte) !! 3564 &vmf->ptl);
4807 goto release; !! 3565 if (!pte_none(*vmf->pte)) {
4808 if (nr_pages == 1 && vmf_pte_changed( !! 3566 update_mmu_cache(vma, vmf->address, vmf->pte);
4809 update_mmu_tlb(vma, addr, vmf <<
4810 goto release; <<
4811 } else if (nr_pages > 1 && !pte_range <<
4812 update_mmu_tlb_range(vma, add <<
4813 goto release; 3567 goto release;
4814 } 3568 }
4815 3569
4816 ret = check_stable_address_space(vma- 3570 ret = check_stable_address_space(vma->vm_mm);
4817 if (ret) 3571 if (ret)
4818 goto release; 3572 goto release;
4819 3573
4820 /* Deliver the page fault to userland 3574 /* Deliver the page fault to userland, check inside PT lock */
4821 if (userfaultfd_missing(vma)) { 3575 if (userfaultfd_missing(vma)) {
4822 pte_unmap_unlock(vmf->pte, vm 3576 pte_unmap_unlock(vmf->pte, vmf->ptl);
4823 folio_put(folio); !! 3577 put_page(page);
4824 return handle_userfault(vmf, 3578 return handle_userfault(vmf, VM_UFFD_MISSING);
4825 } 3579 }
4826 3580
4827 folio_ref_add(folio, nr_pages - 1); !! 3581 inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES);
4828 add_mm_counter(vma->vm_mm, MM_ANONPAG !! 3582 page_add_new_anon_rmap(page, vma, vmf->address, false);
4829 count_mthp_stat(folio_order(folio), M !! 3583 lru_cache_add_inactive_or_unevictable(page, vma);
4830 folio_add_new_anon_rmap(folio, vma, a <<
4831 folio_add_lru_vma(folio, vma); <<
4832 setpte: 3584 setpte:
4833 if (vmf_orig_pte_uffd_wp(vmf)) !! 3585 set_pte_at(vma->vm_mm, vmf->address, vmf->pte, entry);
4834 entry = pte_mkuffd_wp(entry); <<
4835 set_ptes(vma->vm_mm, addr, vmf->pte, <<
4836 3586
4837 /* No need to invalidate - it was non 3587 /* No need to invalidate - it was non-present before */
4838 update_mmu_cache_range(vmf, vma, addr !! 3588 update_mmu_cache(vma, vmf->address, vmf->pte);
4839 unlock: 3589 unlock:
4840 if (vmf->pte) !! 3590 pte_unmap_unlock(vmf->pte, vmf->ptl);
4841 pte_unmap_unlock(vmf->pte, vm <<
4842 return ret; 3591 return ret;
4843 release: 3592 release:
4844 folio_put(folio); !! 3593 put_page(page);
4845 goto unlock; 3594 goto unlock;
>> 3595 oom_free_page:
>> 3596 put_page(page);
4846 oom: 3597 oom:
4847 return VM_FAULT_OOM; 3598 return VM_FAULT_OOM;
4848 } 3599 }
4849 3600
4850 /* 3601 /*
4851 * The mmap_lock must have been held on entry 3602 * The mmap_lock must have been held on entry, and may have been
4852 * released depending on flags and vma->vm_op 3603 * released depending on flags and vma->vm_ops->fault() return value.
4853 * See filemap_fault() and __lock_page_retry( 3604 * See filemap_fault() and __lock_page_retry().
4854 */ 3605 */
4855 static vm_fault_t __do_fault(struct vm_fault 3606 static vm_fault_t __do_fault(struct vm_fault *vmf)
4856 { 3607 {
4857 struct vm_area_struct *vma = vmf->vma 3608 struct vm_area_struct *vma = vmf->vma;
4858 struct folio *folio; <<
4859 vm_fault_t ret; 3609 vm_fault_t ret;
4860 3610
4861 /* 3611 /*
4862 * Preallocate pte before we take pag 3612 * Preallocate pte before we take page_lock because this might lead to
4863 * deadlocks for memcg reclaim which 3613 * deadlocks for memcg reclaim which waits for pages under writeback:
4864 * lock_ 3614 * lock_page(A)
4865 * SetPa 3615 * SetPageWriteback(A)
4866 * unloc 3616 * unlock_page(A)
4867 * lock_page(B) 3617 * lock_page(B)
4868 * lock_ 3618 * lock_page(B)
4869 * pte_alloc_one 3619 * pte_alloc_one
4870 * shrink_folio_list !! 3620 * shrink_page_list
4871 * wait_on_page_writeback(A) 3621 * wait_on_page_writeback(A)
4872 * SetPa 3622 * SetPageWriteback(B)
4873 * unloc 3623 * unlock_page(B)
4874 * # flu 3624 * # flush A, B to clear the writeback
4875 */ 3625 */
4876 if (pmd_none(*vmf->pmd) && !vmf->prea 3626 if (pmd_none(*vmf->pmd) && !vmf->prealloc_pte) {
4877 vmf->prealloc_pte = pte_alloc 3627 vmf->prealloc_pte = pte_alloc_one(vma->vm_mm);
4878 if (!vmf->prealloc_pte) 3628 if (!vmf->prealloc_pte)
4879 return VM_FAULT_OOM; 3629 return VM_FAULT_OOM;
>> 3630 smp_wmb(); /* See comment in __pte_alloc() */
4880 } 3631 }
4881 3632
4882 ret = vma->vm_ops->fault(vmf); 3633 ret = vma->vm_ops->fault(vmf);
4883 if (unlikely(ret & (VM_FAULT_ERROR | 3634 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY |
4884 VM_FAULT_DONE_COW 3635 VM_FAULT_DONE_COW)))
4885 return ret; 3636 return ret;
4886 3637
4887 folio = page_folio(vmf->page); <<
4888 if (unlikely(PageHWPoison(vmf->page)) 3638 if (unlikely(PageHWPoison(vmf->page))) {
4889 vm_fault_t poisonret = VM_FAU !! 3639 if (ret & VM_FAULT_LOCKED)
4890 if (ret & VM_FAULT_LOCKED) { !! 3640 unlock_page(vmf->page);
4891 if (page_mapped(vmf-> !! 3641 put_page(vmf->page);
4892 unmap_mapping <<
4893 /* Retry if a clean f <<
4894 if (mapping_evict_fol <<
4895 poisonret = V <<
4896 folio_unlock(folio); <<
4897 } <<
4898 folio_put(folio); <<
4899 vmf->page = NULL; 3642 vmf->page = NULL;
4900 return poisonret; !! 3643 return VM_FAULT_HWPOISON;
4901 } 3644 }
4902 3645
4903 if (unlikely(!(ret & VM_FAULT_LOCKED) 3646 if (unlikely(!(ret & VM_FAULT_LOCKED)))
4904 folio_lock(folio); !! 3647 lock_page(vmf->page);
4905 else 3648 else
4906 VM_BUG_ON_PAGE(!folio_test_lo !! 3649 VM_BUG_ON_PAGE(!PageLocked(vmf->page), vmf->page);
4907 3650
4908 return ret; 3651 return ret;
4909 } 3652 }
4910 3653
>> 3654 /*
>> 3655 * The ordering of these checks is important for pmds with _PAGE_DEVMAP set.
>> 3656 * If we check pmd_trans_unstable() first we will trip the bad_pmd() check
>> 3657 * inside of pmd_none_or_trans_huge_or_clear_bad(). This will end up correctly
>> 3658 * returning 1 but not before it spams dmesg with the pmd_clear_bad() output.
>> 3659 */
>> 3660 static int pmd_devmap_trans_unstable(pmd_t *pmd)
>> 3661 {
>> 3662 return pmd_devmap(*pmd) || pmd_trans_unstable(pmd);
>> 3663 }
>> 3664
>> 3665 static vm_fault_t pte_alloc_one_map(struct vm_fault *vmf)
>> 3666 {
>> 3667 struct vm_area_struct *vma = vmf->vma;
>> 3668
>> 3669 if (!pmd_none(*vmf->pmd))
>> 3670 goto map_pte;
>> 3671 if (vmf->prealloc_pte) {
>> 3672 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
>> 3673 if (unlikely(!pmd_none(*vmf->pmd))) {
>> 3674 spin_unlock(vmf->ptl);
>> 3675 goto map_pte;
>> 3676 }
>> 3677
>> 3678 mm_inc_nr_ptes(vma->vm_mm);
>> 3679 pmd_populate(vma->vm_mm, vmf->pmd, vmf->prealloc_pte);
>> 3680 spin_unlock(vmf->ptl);
>> 3681 vmf->prealloc_pte = NULL;
>> 3682 } else if (unlikely(pte_alloc(vma->vm_mm, vmf->pmd))) {
>> 3683 return VM_FAULT_OOM;
>> 3684 }
>> 3685 map_pte:
>> 3686 /*
>> 3687 * If a huge pmd materialized under us just retry later. Use
>> 3688 * pmd_trans_unstable() via pmd_devmap_trans_unstable() instead of
>> 3689 * pmd_trans_huge() to ensure the pmd didn't become pmd_trans_huge
>> 3690 * under us and then back to pmd_none, as a result of MADV_DONTNEED
>> 3691 * running immediately after a huge pmd fault in a different thread of
>> 3692 * this mm, in turn leading to a misleading pmd_trans_huge() retval.
>> 3693 * All we have to ensure is that it is a regular pmd that we can walk
>> 3694 * with pte_offset_map() and we can do that through an atomic read in
>> 3695 * C, which is what pmd_trans_unstable() provides.
>> 3696 */
>> 3697 if (pmd_devmap_trans_unstable(vmf->pmd))
>> 3698 return VM_FAULT_NOPAGE;
>> 3699
>> 3700 /*
>> 3701 * At this point we know that our vmf->pmd points to a page of ptes
>> 3702 * and it cannot become pmd_none(), pmd_devmap() or pmd_trans_huge()
>> 3703 * for the duration of the fault. If a racing MADV_DONTNEED runs and
>> 3704 * we zap the ptes pointed to by our vmf->pmd, the vmf->ptl will still
>> 3705 * be valid and we will re-check to make sure the vmf->pte isn't
>> 3706 * pte_none() under vmf->ptl protection when we return to
>> 3707 * alloc_set_pte().
>> 3708 */
>> 3709 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
>> 3710 &vmf->ptl);
>> 3711 return 0;
>> 3712 }
>> 3713
4911 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 3714 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
4912 static void deposit_prealloc_pte(struct vm_fa 3715 static void deposit_prealloc_pte(struct vm_fault *vmf)
4913 { 3716 {
4914 struct vm_area_struct *vma = vmf->vma 3717 struct vm_area_struct *vma = vmf->vma;
4915 3718
4916 pgtable_trans_huge_deposit(vma->vm_mm 3719 pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, vmf->prealloc_pte);
4917 /* 3720 /*
4918 * We are going to consume the preall 3721 * We are going to consume the prealloc table,
4919 * count that as nr_ptes. 3722 * count that as nr_ptes.
4920 */ 3723 */
4921 mm_inc_nr_ptes(vma->vm_mm); 3724 mm_inc_nr_ptes(vma->vm_mm);
4922 vmf->prealloc_pte = NULL; 3725 vmf->prealloc_pte = NULL;
4923 } 3726 }
4924 3727
4925 vm_fault_t do_set_pmd(struct vm_fault *vmf, s !! 3728 static vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page)
4926 { 3729 {
4927 struct folio *folio = page_folio(page <<
4928 struct vm_area_struct *vma = vmf->vma 3730 struct vm_area_struct *vma = vmf->vma;
4929 bool write = vmf->flags & FAULT_FLAG_ 3731 bool write = vmf->flags & FAULT_FLAG_WRITE;
4930 unsigned long haddr = vmf->address & 3732 unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
4931 pmd_t entry; 3733 pmd_t entry;
>> 3734 int i;
4932 vm_fault_t ret = VM_FAULT_FALLBACK; 3735 vm_fault_t ret = VM_FAULT_FALLBACK;
4933 3736
4934 /* !! 3737 if (!transhuge_vma_suitable(vma, haddr))
4935 * It is too late to allocate a small <<
4936 * folio in the pagecache: especially <<
4937 * PMD mappings, but PTE-mapped THP a <<
4938 * PMD mappings if THPs are disabled. <<
4939 */ <<
4940 if (thp_disabled_by_hw() || vma_thp_d <<
4941 return ret; 3738 return ret;
4942 3739
4943 if (!thp_vma_suitable_order(vma, hadd !! 3740 page = compound_head(page);
4944 return ret; !! 3741 if (compound_order(page) != HPAGE_PMD_ORDER)
4945 <<
4946 if (folio_order(folio) != HPAGE_PMD_O <<
4947 return ret; <<
4948 page = &folio->page; <<
4949 <<
4950 /* <<
4951 * Just backoff if any subpage of a T <<
4952 * the corrupted page may mapped by P <<
4953 * check. This kind of THP just can <<
4954 * the corrupted subpage should trigg <<
4955 */ <<
4956 if (unlikely(folio_test_has_hwpoisone <<
4957 return ret; 3742 return ret;
4958 3743
4959 /* 3744 /*
4960 * Archs like ppc64 need additional s !! 3745 * Archs like ppc64 need additonal space to store information
4961 * related to pte entry. Use the prea 3746 * related to pte entry. Use the preallocated table for that.
4962 */ 3747 */
4963 if (arch_needs_pgtable_deposit() && ! 3748 if (arch_needs_pgtable_deposit() && !vmf->prealloc_pte) {
4964 vmf->prealloc_pte = pte_alloc 3749 vmf->prealloc_pte = pte_alloc_one(vma->vm_mm);
4965 if (!vmf->prealloc_pte) 3750 if (!vmf->prealloc_pte)
4966 return VM_FAULT_OOM; 3751 return VM_FAULT_OOM;
>> 3752 smp_wmb(); /* See comment in __pte_alloc() */
4967 } 3753 }
4968 3754
4969 vmf->ptl = pmd_lock(vma->vm_mm, vmf-> 3755 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
4970 if (unlikely(!pmd_none(*vmf->pmd))) 3756 if (unlikely(!pmd_none(*vmf->pmd)))
4971 goto out; 3757 goto out;
4972 3758
4973 flush_icache_pages(vma, page, HPAGE_P !! 3759 for (i = 0; i < HPAGE_PMD_NR; i++)
>> 3760 flush_icache_page(vma, page + i);
4974 3761
4975 entry = mk_huge_pmd(page, vma->vm_pag 3762 entry = mk_huge_pmd(page, vma->vm_page_prot);
4976 if (write) 3763 if (write)
4977 entry = maybe_pmd_mkwrite(pmd 3764 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
4978 3765
4979 add_mm_counter(vma->vm_mm, mm_counter !! 3766 add_mm_counter(vma->vm_mm, mm_counter_file(page), HPAGE_PMD_NR);
4980 folio_add_file_rmap_pmd(folio, page, !! 3767 page_add_file_rmap(page, true);
4981 <<
4982 /* 3768 /*
4983 * deposit and withdraw with pmd lock 3769 * deposit and withdraw with pmd lock held
4984 */ 3770 */
4985 if (arch_needs_pgtable_deposit()) 3771 if (arch_needs_pgtable_deposit())
4986 deposit_prealloc_pte(vmf); 3772 deposit_prealloc_pte(vmf);
4987 3773
4988 set_pmd_at(vma->vm_mm, haddr, vmf->pm 3774 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry);
4989 3775
4990 update_mmu_cache_pmd(vma, haddr, vmf- 3776 update_mmu_cache_pmd(vma, haddr, vmf->pmd);
4991 3777
4992 /* fault is handled */ 3778 /* fault is handled */
4993 ret = 0; 3779 ret = 0;
4994 count_vm_event(THP_FILE_MAPPED); 3780 count_vm_event(THP_FILE_MAPPED);
4995 out: 3781 out:
4996 spin_unlock(vmf->ptl); 3782 spin_unlock(vmf->ptl);
4997 return ret; 3783 return ret;
4998 } 3784 }
4999 #else 3785 #else
5000 vm_fault_t do_set_pmd(struct vm_fault *vmf, s !! 3786 static vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page)
5001 { 3787 {
5002 return VM_FAULT_FALLBACK; !! 3788 BUILD_BUG();
>> 3789 return 0;
5003 } 3790 }
5004 #endif 3791 #endif
5005 3792
5006 /** 3793 /**
5007 * set_pte_range - Set a range of PTEs to poi !! 3794 * alloc_set_pte - setup new PTE entry for given page and add reverse page
5008 * @vmf: Fault decription. !! 3795 * mapping. If needed, the function allocates page table or use pre-allocated.
5009 * @folio: The folio that contains @page. !! 3796 *
5010 * @page: The first page to create a PTE for. !! 3797 * @vmf: fault environment
5011 * @nr: The number of PTEs to create. !! 3798 * @page: page to map
5012 * @addr: The first address to create a PTE f !! 3799 *
>> 3800 * Caller must take care of unlocking vmf->ptl, if vmf->pte is non-NULL on
>> 3801 * return.
>> 3802 *
>> 3803 * Target users are page handler itself and implementations of
>> 3804 * vm_ops->map_pages.
>> 3805 *
>> 3806 * Return: %0 on success, %VM_FAULT_ code in case of error.
5013 */ 3807 */
5014 void set_pte_range(struct vm_fault *vmf, stru !! 3808 vm_fault_t alloc_set_pte(struct vm_fault *vmf, struct page *page)
5015 struct page *page, unsigned i <<
5016 { 3809 {
5017 struct vm_area_struct *vma = vmf->vma 3810 struct vm_area_struct *vma = vmf->vma;
5018 bool write = vmf->flags & FAULT_FLAG_ 3811 bool write = vmf->flags & FAULT_FLAG_WRITE;
5019 bool prefault = !in_range(vmf->addres <<
5020 pte_t entry; 3812 pte_t entry;
>> 3813 vm_fault_t ret;
5021 3814
5022 flush_icache_pages(vma, page, nr); !! 3815 if (pmd_none(*vmf->pmd) && PageTransCompound(page)) {
5023 entry = mk_pte(page, vma->vm_page_pro !! 3816 ret = do_set_pmd(vmf, page);
>> 3817 if (ret != VM_FAULT_FALLBACK)
>> 3818 return ret;
>> 3819 }
5024 3820
5025 if (prefault && arch_wants_old_prefau !! 3821 if (!vmf->pte) {
5026 entry = pte_mkold(entry); !! 3822 ret = pte_alloc_one_map(vmf);
5027 else !! 3823 if (ret)
5028 entry = pte_sw_mkyoung(entry) !! 3824 return ret;
>> 3825 }
>> 3826
>> 3827 /* Re-check under ptl */
>> 3828 if (unlikely(!pte_none(*vmf->pte))) {
>> 3829 update_mmu_tlb(vma, vmf->address, vmf->pte);
>> 3830 return VM_FAULT_NOPAGE;
>> 3831 }
5029 3832
>> 3833 flush_icache_page(vma, page);
>> 3834 entry = mk_pte(page, vma->vm_page_prot);
>> 3835 entry = pte_sw_mkyoung(entry);
5030 if (write) 3836 if (write)
5031 entry = maybe_mkwrite(pte_mkd 3837 entry = maybe_mkwrite(pte_mkdirty(entry), vma);
5032 if (unlikely(vmf_orig_pte_uffd_wp(vmf <<
5033 entry = pte_mkuffd_wp(entry); <<
5034 /* copy-on-write page */ 3838 /* copy-on-write page */
5035 if (write && !(vma->vm_flags & VM_SHA 3839 if (write && !(vma->vm_flags & VM_SHARED)) {
5036 VM_BUG_ON_FOLIO(nr != 1, foli !! 3840 inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES);
5037 folio_add_new_anon_rmap(folio !! 3841 page_add_new_anon_rmap(page, vma, vmf->address, false);
5038 folio_add_lru_vma(folio, vma) !! 3842 lru_cache_add_inactive_or_unevictable(page, vma);
5039 } else { 3843 } else {
5040 folio_add_file_rmap_ptes(foli !! 3844 inc_mm_counter_fast(vma->vm_mm, mm_counter_file(page));
>> 3845 page_add_file_rmap(page, false);
5041 } 3846 }
5042 set_ptes(vma->vm_mm, addr, vmf->pte, !! 3847 set_pte_at(vma->vm_mm, vmf->address, vmf->pte, entry);
5043 3848
5044 /* no need to invalidate: a not-prese 3849 /* no need to invalidate: a not-present page won't be cached */
5045 update_mmu_cache_range(vmf, vma, addr !! 3850 update_mmu_cache(vma, vmf->address, vmf->pte);
5046 } <<
5047 <<
5048 static bool vmf_pte_changed(struct vm_fault * <<
5049 { <<
5050 if (vmf->flags & FAULT_FLAG_ORIG_PTE_ <<
5051 return !pte_same(ptep_get(vmf <<
5052 3851
5053 return !pte_none(ptep_get(vmf->pte)); !! 3852 return 0;
5054 } 3853 }
5055 3854
>> 3855
5056 /** 3856 /**
5057 * finish_fault - finish page fault once we h 3857 * finish_fault - finish page fault once we have prepared the page to fault
5058 * 3858 *
5059 * @vmf: structure describing the fault 3859 * @vmf: structure describing the fault
5060 * 3860 *
5061 * This function handles all that is needed t 3861 * This function handles all that is needed to finish a page fault once the
5062 * page to fault in is prepared. It handles l 3862 * page to fault in is prepared. It handles locking of PTEs, inserts PTE for
5063 * given page, adds reverse page mapping, han 3863 * given page, adds reverse page mapping, handles memcg charges and LRU
5064 * addition. 3864 * addition.
5065 * 3865 *
5066 * The function expects the page to be locked 3866 * The function expects the page to be locked and on success it consumes a
5067 * reference of a page being mapped (for the 3867 * reference of a page being mapped (for the PTE which maps it).
5068 * 3868 *
5069 * Return: %0 on success, %VM_FAULT_ code in 3869 * Return: %0 on success, %VM_FAULT_ code in case of error.
5070 */ 3870 */
5071 vm_fault_t finish_fault(struct vm_fault *vmf) 3871 vm_fault_t finish_fault(struct vm_fault *vmf)
5072 { 3872 {
5073 struct vm_area_struct *vma = vmf->vma <<
5074 struct page *page; 3873 struct page *page;
5075 struct folio *folio; !! 3874 vm_fault_t ret = 0;
5076 vm_fault_t ret; <<
5077 bool is_cow = (vmf->flags & FAULT_FLA <<
5078 !(vma->vm_flags & VM_SH <<
5079 int type, nr_pages; <<
5080 unsigned long addr = vmf->address; <<
5081 3875
5082 /* Did we COW the page? */ 3876 /* Did we COW the page? */
5083 if (is_cow) !! 3877 if ((vmf->flags & FAULT_FLAG_WRITE) &&
>> 3878 !(vmf->vma->vm_flags & VM_SHARED))
5084 page = vmf->cow_page; 3879 page = vmf->cow_page;
5085 else 3880 else
5086 page = vmf->page; 3881 page = vmf->page;
5087 3882
5088 /* 3883 /*
5089 * check even for read faults because 3884 * check even for read faults because we might have lost our CoWed
5090 * page 3885 * page
5091 */ 3886 */
5092 if (!(vma->vm_flags & VM_SHARED)) { !! 3887 if (!(vmf->vma->vm_flags & VM_SHARED))
5093 ret = check_stable_address_sp !! 3888 ret = check_stable_address_space(vmf->vma->vm_mm);
5094 if (ret) !! 3889 if (!ret)
5095 return ret; !! 3890 ret = alloc_set_pte(vmf, page);
5096 } !! 3891 if (vmf->pte)
5097 !! 3892 pte_unmap_unlock(vmf->pte, vmf->ptl);
5098 if (pmd_none(*vmf->pmd)) { <<
5099 if (PageTransCompound(page)) <<
5100 ret = do_set_pmd(vmf, <<
5101 if (ret != VM_FAULT_F <<
5102 return ret; <<
5103 } <<
5104 <<
5105 if (vmf->prealloc_pte) <<
5106 pmd_install(vma->vm_m <<
5107 else if (unlikely(pte_alloc(v <<
5108 return VM_FAULT_OOM; <<
5109 } <<
5110 <<
5111 folio = page_folio(page); <<
5112 nr_pages = folio_nr_pages(folio); <<
5113 <<
5114 /* <<
5115 * Using per-page fault to maintain t <<
5116 * approach also applies to non-anony <<
5117 * inflating the RSS of the process. <<
5118 */ <<
5119 if (!vma_is_anon_shmem(vma) || unlike <<
5120 nr_pages = 1; <<
5121 } else if (nr_pages > 1) { <<
5122 pgoff_t idx = folio_page_idx( <<
5123 /* The page offset of vmf->ad <<
5124 pgoff_t vma_off = vmf->pgoff <<
5125 /* The index of the entry in <<
5126 pgoff_t pte_off = pte_index(v <<
5127 <<
5128 /* <<
5129 * Fallback to per-page fault <<
5130 * cache beyond the VMA limit <<
5131 */ <<
5132 if (unlikely(vma_off < idx || <<
5133 vma_off + (nr_pag <<
5134 pte_off < idx || <<
5135 pte_off + (nr_pag <<
5136 nr_pages = 1; <<
5137 } else { <<
5138 /* Now we can set map <<
5139 addr = vmf->address - <<
5140 page = &folio->page; <<
5141 } <<
5142 } <<
5143 <<
5144 vmf->pte = pte_offset_map_lock(vma->v <<
5145 addr, <<
5146 if (!vmf->pte) <<
5147 return VM_FAULT_NOPAGE; <<
5148 <<
5149 /* Re-check under ptl */ <<
5150 if (nr_pages == 1 && unlikely(vmf_pte <<
5151 update_mmu_tlb(vma, addr, vmf <<
5152 ret = VM_FAULT_NOPAGE; <<
5153 goto unlock; <<
5154 } else if (nr_pages > 1 && !pte_range <<
5155 update_mmu_tlb_range(vma, add <<
5156 ret = VM_FAULT_NOPAGE; <<
5157 goto unlock; <<
5158 } <<
5159 <<
5160 folio_ref_add(folio, nr_pages - 1); <<
5161 set_pte_range(vmf, folio, page, nr_pa <<
5162 type = is_cow ? MM_ANONPAGES : mm_cou <<
5163 add_mm_counter(vma->vm_mm, type, nr_p <<
5164 ret = 0; <<
5165 <<
5166 unlock: <<
5167 pte_unmap_unlock(vmf->pte, vmf->ptl); <<
5168 return ret; 3893 return ret;
5169 } 3894 }
5170 3895
5171 static unsigned long fault_around_pages __rea !! 3896 static unsigned long fault_around_bytes __read_mostly =
5172 65536 >> PAGE_SHIFT; !! 3897 rounddown_pow_of_two(65536);
5173 3898
5174 #ifdef CONFIG_DEBUG_FS 3899 #ifdef CONFIG_DEBUG_FS
5175 static int fault_around_bytes_get(void *data, 3900 static int fault_around_bytes_get(void *data, u64 *val)
5176 { 3901 {
5177 *val = fault_around_pages << PAGE_SHI !! 3902 *val = fault_around_bytes;
5178 return 0; 3903 return 0;
5179 } 3904 }
5180 3905
5181 /* 3906 /*
5182 * fault_around_bytes must be rounded down to 3907 * fault_around_bytes must be rounded down to the nearest page order as it's
5183 * what do_fault_around() expects to see. 3908 * what do_fault_around() expects to see.
5184 */ 3909 */
5185 static int fault_around_bytes_set(void *data, 3910 static int fault_around_bytes_set(void *data, u64 val)
5186 { 3911 {
5187 if (val / PAGE_SIZE > PTRS_PER_PTE) 3912 if (val / PAGE_SIZE > PTRS_PER_PTE)
5188 return -EINVAL; 3913 return -EINVAL;
5189 !! 3914 if (val > PAGE_SIZE)
5190 /* !! 3915 fault_around_bytes = rounddown_pow_of_two(val);
5191 * The minimum value is 1 page, howev !! 3916 else
5192 * at all. See should_fault_around(). !! 3917 fault_around_bytes = PAGE_SIZE; /* rounddown_pow_of_two(0) is undefined */
5193 */ <<
5194 val = max(val, PAGE_SIZE); <<
5195 fault_around_pages = rounddown_pow_of <<
5196 <<
5197 return 0; 3918 return 0;
5198 } 3919 }
5199 DEFINE_DEBUGFS_ATTRIBUTE(fault_around_bytes_f 3920 DEFINE_DEBUGFS_ATTRIBUTE(fault_around_bytes_fops,
5200 fault_around_bytes_get, fault 3921 fault_around_bytes_get, fault_around_bytes_set, "%llu\n");
5201 3922
5202 static int __init fault_around_debugfs(void) 3923 static int __init fault_around_debugfs(void)
5203 { 3924 {
5204 debugfs_create_file_unsafe("fault_aro 3925 debugfs_create_file_unsafe("fault_around_bytes", 0644, NULL, NULL,
5205 &fault_aro 3926 &fault_around_bytes_fops);
5206 return 0; 3927 return 0;
5207 } 3928 }
5208 late_initcall(fault_around_debugfs); 3929 late_initcall(fault_around_debugfs);
5209 #endif 3930 #endif
5210 3931
5211 /* 3932 /*
5212 * do_fault_around() tries to map few pages a 3933 * do_fault_around() tries to map few pages around the fault address. The hope
5213 * is that the pages will be needed soon and 3934 * is that the pages will be needed soon and this will lower the number of
5214 * faults to handle. 3935 * faults to handle.
5215 * 3936 *
5216 * It uses vm_ops->map_pages() to map the pag 3937 * It uses vm_ops->map_pages() to map the pages, which skips the page if it's
5217 * not ready to be mapped: not up-to-date, lo 3938 * not ready to be mapped: not up-to-date, locked, etc.
5218 * 3939 *
5219 * This function doesn't cross VMA or page ta !! 3940 * This function is called with the page table lock taken. In the split ptlock
5220 * map_pages() and acquire a PTE lock only on !! 3941 * case the page table lock only protects only those entries which belong to
>> 3942 * the page table corresponding to the fault address.
>> 3943 *
>> 3944 * This function doesn't cross the VMA boundaries, in order to call map_pages()
>> 3945 * only once.
5221 * 3946 *
5222 * fault_around_pages defines how many pages !! 3947 * fault_around_bytes defines how many bytes we'll try to map.
5223 * do_fault_around() expects it to be set to 3948 * do_fault_around() expects it to be set to a power of two less than or equal
5224 * to PTRS_PER_PTE. 3949 * to PTRS_PER_PTE.
5225 * 3950 *
5226 * The virtual address of the area that we ma 3951 * The virtual address of the area that we map is naturally aligned to
5227 * fault_around_pages * PAGE_SIZE rounded dow !! 3952 * fault_around_bytes rounded down to the machine page size
5228 * (and therefore to page order). This way i 3953 * (and therefore to page order). This way it's easier to guarantee
5229 * that we don't cross page table boundaries. 3954 * that we don't cross page table boundaries.
5230 */ 3955 */
5231 static vm_fault_t do_fault_around(struct vm_f 3956 static vm_fault_t do_fault_around(struct vm_fault *vmf)
5232 { 3957 {
5233 pgoff_t nr_pages = READ_ONCE(fault_ar !! 3958 unsigned long address = vmf->address, nr_pages, mask;
5234 pgoff_t pte_off = pte_index(vmf->addr !! 3959 pgoff_t start_pgoff = vmf->pgoff;
5235 /* The page offset of vmf->address wi !! 3960 pgoff_t end_pgoff;
5236 pgoff_t vma_off = vmf->pgoff - vmf->v !! 3961 int off;
5237 pgoff_t from_pte, to_pte; !! 3962 vm_fault_t ret = 0;
5238 vm_fault_t ret; !! 3963
>> 3964 nr_pages = READ_ONCE(fault_around_bytes) >> PAGE_SHIFT;
>> 3965 mask = ~(nr_pages * PAGE_SIZE - 1) & PAGE_MASK;
5239 3966
5240 /* The PTE offset of the start addres !! 3967 vmf->address = max(address & mask, vmf->vma->vm_start);
5241 from_pte = max(ALIGN_DOWN(pte_off, nr !! 3968 off = ((address - vmf->address) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1);
5242 pte_off - min(pte_off, !! 3969 start_pgoff -= off;
5243 !! 3970
5244 /* The PTE offset of the end address, !! 3971 /*
5245 to_pte = min3(from_pte + nr_pages, (p !! 3972 * end_pgoff is either the end of the page table, the end of
5246 pte_off + vma_pages(vmf !! 3973 * the vma or nr_pages from start_pgoff, depending what is nearest.
>> 3974 */
>> 3975 end_pgoff = start_pgoff -
>> 3976 ((vmf->address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) +
>> 3977 PTRS_PER_PTE - 1;
>> 3978 end_pgoff = min3(end_pgoff, vma_pages(vmf->vma) + vmf->vma->vm_pgoff - 1,
>> 3979 start_pgoff + nr_pages - 1);
5247 3980
5248 if (pmd_none(*vmf->pmd)) { 3981 if (pmd_none(*vmf->pmd)) {
5249 vmf->prealloc_pte = pte_alloc 3982 vmf->prealloc_pte = pte_alloc_one(vmf->vma->vm_mm);
5250 if (!vmf->prealloc_pte) 3983 if (!vmf->prealloc_pte)
5251 return VM_FAULT_OOM; !! 3984 goto out;
>> 3985 smp_wmb(); /* See comment in __pte_alloc() */
5252 } 3986 }
5253 3987
5254 rcu_read_lock(); !! 3988 vmf->vma->vm_ops->map_pages(vmf, start_pgoff, end_pgoff);
5255 ret = vmf->vma->vm_ops->map_pages(vmf <<
5256 vmf->pgoff + from_pte <<
5257 vmf->pgoff + to_pte - <<
5258 rcu_read_unlock(); <<
5259 3989
5260 return ret; !! 3990 /* Huge page is mapped? Page fault is solved */
5261 } !! 3991 if (pmd_trans_huge(*vmf->pmd)) {
5262 !! 3992 ret = VM_FAULT_NOPAGE;
5263 /* Return true if we should do read fault-aro !! 3993 goto out;
5264 static inline bool should_fault_around(struct !! 3994 }
5265 { <<
5266 /* No ->map_pages? No way to fault a <<
5267 if (!vmf->vma->vm_ops->map_pages) <<
5268 return false; <<
5269 3995
5270 if (uffd_disable_fault_around(vmf->vm !! 3996 /* ->map_pages() haven't done anything useful. Cold page cache? */
5271 return false; !! 3997 if (!vmf->pte)
>> 3998 goto out;
5272 3999
5273 /* A single page implies no faulting !! 4000 /* check if the page fault is solved */
5274 return fault_around_pages > 1; !! 4001 vmf->pte -= (vmf->address >> PAGE_SHIFT) - (address >> PAGE_SHIFT);
>> 4002 if (!pte_none(*vmf->pte))
>> 4003 ret = VM_FAULT_NOPAGE;
>> 4004 pte_unmap_unlock(vmf->pte, vmf->ptl);
>> 4005 out:
>> 4006 vmf->address = address;
>> 4007 vmf->pte = NULL;
>> 4008 return ret;
5275 } 4009 }
5276 4010
5277 static vm_fault_t do_read_fault(struct vm_fau 4011 static vm_fault_t do_read_fault(struct vm_fault *vmf)
5278 { 4012 {
>> 4013 struct vm_area_struct *vma = vmf->vma;
5279 vm_fault_t ret = 0; 4014 vm_fault_t ret = 0;
5280 struct folio *folio; <<
5281 4015
5282 /* 4016 /*
5283 * Let's call ->map_pages() first and 4017 * Let's call ->map_pages() first and use ->fault() as fallback
5284 * if page by the offset is not ready 4018 * if page by the offset is not ready to be mapped (cold cache or
5285 * something). 4019 * something).
5286 */ 4020 */
5287 if (should_fault_around(vmf)) { !! 4021 if (vma->vm_ops->map_pages && fault_around_bytes >> PAGE_SHIFT > 1) {
5288 ret = do_fault_around(vmf); 4022 ret = do_fault_around(vmf);
5289 if (ret) 4023 if (ret)
5290 return ret; 4024 return ret;
5291 } 4025 }
5292 4026
5293 ret = vmf_can_call_fault(vmf); <<
5294 if (ret) <<
5295 return ret; <<
5296 <<
5297 ret = __do_fault(vmf); 4027 ret = __do_fault(vmf);
5298 if (unlikely(ret & (VM_FAULT_ERROR | 4028 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
5299 return ret; 4029 return ret;
5300 4030
5301 ret |= finish_fault(vmf); 4031 ret |= finish_fault(vmf);
5302 folio = page_folio(vmf->page); !! 4032 unlock_page(vmf->page);
5303 folio_unlock(folio); <<
5304 if (unlikely(ret & (VM_FAULT_ERROR | 4033 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
5305 folio_put(folio); !! 4034 put_page(vmf->page);
5306 return ret; 4035 return ret;
5307 } 4036 }
5308 4037
5309 static vm_fault_t do_cow_fault(struct vm_faul 4038 static vm_fault_t do_cow_fault(struct vm_fault *vmf)
5310 { 4039 {
5311 struct vm_area_struct *vma = vmf->vma 4040 struct vm_area_struct *vma = vmf->vma;
5312 struct folio *folio; <<
5313 vm_fault_t ret; 4041 vm_fault_t ret;
5314 4042
5315 ret = vmf_can_call_fault(vmf); !! 4043 if (unlikely(anon_vma_prepare(vma)))
5316 if (!ret) !! 4044 return VM_FAULT_OOM;
5317 ret = vmf_anon_prepare(vmf); <<
5318 if (ret) <<
5319 return ret; <<
5320 4045
5321 folio = folio_prealloc(vma->vm_mm, vm !! 4046 vmf->cow_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, vmf->address);
5322 if (!folio) !! 4047 if (!vmf->cow_page)
5323 return VM_FAULT_OOM; 4048 return VM_FAULT_OOM;
5324 4049
5325 vmf->cow_page = &folio->page; !! 4050 if (mem_cgroup_charge(vmf->cow_page, vma->vm_mm, GFP_KERNEL)) {
>> 4051 put_page(vmf->cow_page);
>> 4052 return VM_FAULT_OOM;
>> 4053 }
>> 4054 cgroup_throttle_swaprate(vmf->cow_page, GFP_KERNEL);
5326 4055
5327 ret = __do_fault(vmf); 4056 ret = __do_fault(vmf);
5328 if (unlikely(ret & (VM_FAULT_ERROR | 4057 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
5329 goto uncharge_out; 4058 goto uncharge_out;
5330 if (ret & VM_FAULT_DONE_COW) 4059 if (ret & VM_FAULT_DONE_COW)
5331 return ret; 4060 return ret;
5332 4061
5333 if (copy_mc_user_highpage(vmf->cow_pa !! 4062 copy_user_highpage(vmf->cow_page, vmf->page, vmf->address, vma);
5334 ret = VM_FAULT_HWPOISON; !! 4063 __SetPageUptodate(vmf->cow_page);
5335 goto unlock; <<
5336 } <<
5337 __folio_mark_uptodate(folio); <<
5338 4064
5339 ret |= finish_fault(vmf); 4065 ret |= finish_fault(vmf);
5340 unlock: <<
5341 unlock_page(vmf->page); 4066 unlock_page(vmf->page);
5342 put_page(vmf->page); 4067 put_page(vmf->page);
5343 if (unlikely(ret & (VM_FAULT_ERROR | 4068 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
5344 goto uncharge_out; 4069 goto uncharge_out;
5345 return ret; 4070 return ret;
5346 uncharge_out: 4071 uncharge_out:
5347 folio_put(folio); !! 4072 put_page(vmf->cow_page);
5348 return ret; 4073 return ret;
5349 } 4074 }
5350 4075
5351 static vm_fault_t do_shared_fault(struct vm_f 4076 static vm_fault_t do_shared_fault(struct vm_fault *vmf)
5352 { 4077 {
5353 struct vm_area_struct *vma = vmf->vma 4078 struct vm_area_struct *vma = vmf->vma;
5354 vm_fault_t ret, tmp; 4079 vm_fault_t ret, tmp;
5355 struct folio *folio; <<
5356 <<
5357 ret = vmf_can_call_fault(vmf); <<
5358 if (ret) <<
5359 return ret; <<
5360 4080
5361 ret = __do_fault(vmf); 4081 ret = __do_fault(vmf);
5362 if (unlikely(ret & (VM_FAULT_ERROR | 4082 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
5363 return ret; 4083 return ret;
5364 4084
5365 folio = page_folio(vmf->page); <<
5366 <<
5367 /* 4085 /*
5368 * Check if the backing address space 4086 * Check if the backing address space wants to know that the page is
5369 * about to become writable 4087 * about to become writable
5370 */ 4088 */
5371 if (vma->vm_ops->page_mkwrite) { 4089 if (vma->vm_ops->page_mkwrite) {
5372 folio_unlock(folio); !! 4090 unlock_page(vmf->page);
5373 tmp = do_page_mkwrite(vmf, fo !! 4091 tmp = do_page_mkwrite(vmf);
5374 if (unlikely(!tmp || 4092 if (unlikely(!tmp ||
5375 (tmp & (VM_FA 4093 (tmp & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))) {
5376 folio_put(folio); !! 4094 put_page(vmf->page);
5377 return tmp; 4095 return tmp;
5378 } 4096 }
5379 } 4097 }
5380 4098
5381 ret |= finish_fault(vmf); 4099 ret |= finish_fault(vmf);
5382 if (unlikely(ret & (VM_FAULT_ERROR | 4100 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE |
5383 VM_FA 4101 VM_FAULT_RETRY))) {
5384 folio_unlock(folio); !! 4102 unlock_page(vmf->page);
5385 folio_put(folio); !! 4103 put_page(vmf->page);
5386 return ret; 4104 return ret;
5387 } 4105 }
5388 4106
5389 ret |= fault_dirty_shared_page(vmf); 4107 ret |= fault_dirty_shared_page(vmf);
5390 return ret; 4108 return ret;
5391 } 4109 }
5392 4110
5393 /* 4111 /*
5394 * We enter with non-exclusive mmap_lock (to 4112 * We enter with non-exclusive mmap_lock (to exclude vma changes,
5395 * but allow concurrent faults). 4113 * but allow concurrent faults).
5396 * The mmap_lock may have been released depen 4114 * The mmap_lock may have been released depending on flags and our
5397 * return value. See filemap_fault() and __f !! 4115 * return value. See filemap_fault() and __lock_page_or_retry().
5398 * If mmap_lock is released, vma may become i 4116 * If mmap_lock is released, vma may become invalid (for example
5399 * by other thread calling munmap()). 4117 * by other thread calling munmap()).
5400 */ 4118 */
5401 static vm_fault_t do_fault(struct vm_fault *v 4119 static vm_fault_t do_fault(struct vm_fault *vmf)
5402 { 4120 {
5403 struct vm_area_struct *vma = vmf->vma 4121 struct vm_area_struct *vma = vmf->vma;
5404 struct mm_struct *vm_mm = vma->vm_mm; 4122 struct mm_struct *vm_mm = vma->vm_mm;
5405 vm_fault_t ret; 4123 vm_fault_t ret;
5406 4124
5407 /* 4125 /*
5408 * The VMA was not fully populated on 4126 * The VMA was not fully populated on mmap() or missing VM_DONTEXPAND
5409 */ 4127 */
5410 if (!vma->vm_ops->fault) { 4128 if (!vma->vm_ops->fault) {
5411 vmf->pte = pte_offset_map_loc !! 4129 /*
5412 !! 4130 * If we find a migration pmd entry or a none pmd entry, which
5413 if (unlikely(!vmf->pte)) !! 4131 * should never happen, return SIGBUS
>> 4132 */
>> 4133 if (unlikely(!pmd_present(*vmf->pmd)))
5414 ret = VM_FAULT_SIGBUS 4134 ret = VM_FAULT_SIGBUS;
5415 else { 4135 else {
>> 4136 vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm,
>> 4137 vmf->pmd,
>> 4138 vmf->address,
>> 4139 &vmf->ptl);
5416 /* 4140 /*
5417 * Make sure this is 4141 * Make sure this is not a temporary clearing of pte
5418 * by holding ptl and 4142 * by holding ptl and checking again. A R/M/W update
5419 * of pte involves: t 4143 * of pte involves: take ptl, clearing the pte so that
5420 * we don't have conc 4144 * we don't have concurrent modification by hardware
5421 * followed by an upd 4145 * followed by an update.
5422 */ 4146 */
5423 if (unlikely(pte_none !! 4147 if (unlikely(pte_none(*vmf->pte)))
5424 ret = VM_FAUL 4148 ret = VM_FAULT_SIGBUS;
5425 else 4149 else
5426 ret = VM_FAUL 4150 ret = VM_FAULT_NOPAGE;
5427 4151
5428 pte_unmap_unlock(vmf- 4152 pte_unmap_unlock(vmf->pte, vmf->ptl);
5429 } 4153 }
5430 } else if (!(vmf->flags & FAULT_FLAG_ 4154 } else if (!(vmf->flags & FAULT_FLAG_WRITE))
5431 ret = do_read_fault(vmf); 4155 ret = do_read_fault(vmf);
5432 else if (!(vma->vm_flags & VM_SHARED) 4156 else if (!(vma->vm_flags & VM_SHARED))
5433 ret = do_cow_fault(vmf); 4157 ret = do_cow_fault(vmf);
5434 else 4158 else
5435 ret = do_shared_fault(vmf); 4159 ret = do_shared_fault(vmf);
5436 4160
5437 /* preallocated pagetable is unused: 4161 /* preallocated pagetable is unused: free it */
5438 if (vmf->prealloc_pte) { 4162 if (vmf->prealloc_pte) {
5439 pte_free(vm_mm, vmf->prealloc 4163 pte_free(vm_mm, vmf->prealloc_pte);
5440 vmf->prealloc_pte = NULL; 4164 vmf->prealloc_pte = NULL;
5441 } 4165 }
5442 return ret; 4166 return ret;
5443 } 4167 }
5444 4168
5445 int numa_migrate_check(struct folio *folio, s !! 4169 static int numa_migrate_prep(struct page *page, struct vm_area_struct *vma,
5446 unsigned long addr, int !! 4170 unsigned long addr, int page_nid,
5447 bool writable, int *las !! 4171 int *flags)
5448 { 4172 {
5449 struct vm_area_struct *vma = vmf->vma !! 4173 get_page(page);
5450 <<
5451 /* <<
5452 * Avoid grouping on RO pages in gene <<
5453 * much anyway since they can be in s <<
5454 * the case where a mapping is writab <<
5455 * to it but pte_write gets cleared d <<
5456 * pte_dirty has unpredictable behavi <<
5457 * background writeback, dirty balanc <<
5458 */ <<
5459 if (!writable) <<
5460 *flags |= TNF_NO_GROUP; <<
5461 <<
5462 /* <<
5463 * Flag if the folio is shared betwee <<
5464 * is later used when determining whe <<
5465 */ <<
5466 if (folio_likely_mapped_shared(folio) <<
5467 *flags |= TNF_SHARED; <<
5468 /* <<
5469 * For memory tiering mode, cpupid of <<
5470 * to record page access time. So us <<
5471 */ <<
5472 if (folio_use_access_time(folio)) <<
5473 *last_cpupid = (-1 & LAST_CPU <<
5474 else <<
5475 *last_cpupid = folio_last_cpu <<
5476 <<
5477 /* Record the current PID acceesing V <<
5478 vma_set_access_pid_bit(vma); <<
5479 4174
5480 count_vm_numa_event(NUMA_HINT_FAULTS) 4175 count_vm_numa_event(NUMA_HINT_FAULTS);
5481 #ifdef CONFIG_NUMA_BALANCING !! 4176 if (page_nid == numa_node_id()) {
5482 count_memcg_folio_events(folio, NUMA_ <<
5483 #endif <<
5484 if (folio_nid(folio) == numa_node_id( <<
5485 count_vm_numa_event(NUMA_HINT 4177 count_vm_numa_event(NUMA_HINT_FAULTS_LOCAL);
5486 *flags |= TNF_FAULT_LOCAL; 4178 *flags |= TNF_FAULT_LOCAL;
5487 } 4179 }
5488 4180
5489 return mpol_misplaced(folio, vmf, add !! 4181 return mpol_misplaced(page, vma, addr);
5490 } <<
5491 <<
5492 static void numa_rebuild_single_mapping(struc <<
5493 unsig <<
5494 bool <<
5495 { <<
5496 pte_t pte, old_pte; <<
5497 <<
5498 old_pte = ptep_modify_prot_start(vma, <<
5499 pte = pte_modify(old_pte, vma->vm_pag <<
5500 pte = pte_mkyoung(pte); <<
5501 if (writable) <<
5502 pte = pte_mkwrite(pte, vma); <<
5503 ptep_modify_prot_commit(vma, fault_ad <<
5504 update_mmu_cache_range(vmf, vma, faul <<
5505 } <<
5506 <<
5507 static void numa_rebuild_large_mapping(struct <<
5508 struct <<
5509 bool i <<
5510 { <<
5511 int nr = pte_pfn(fault_pte) - folio_p <<
5512 unsigned long start, end, addr = vmf- <<
5513 unsigned long addr_start = addr - (nr <<
5514 unsigned long pt_start = ALIGN_DOWN(a <<
5515 pte_t *start_ptep; <<
5516 <<
5517 /* Stay within the VMA and within the <<
5518 start = max3(addr_start, pt_start, vm <<
5519 end = min3(addr_start + folio_size(fo <<
5520 vma->vm_end); <<
5521 start_ptep = vmf->pte - ((addr - star <<
5522 <<
5523 /* Restore all PTEs' mapping of the l <<
5524 for (addr = start; addr != end; start <<
5525 pte_t ptent = ptep_get(start_ <<
5526 bool writable = false; <<
5527 <<
5528 if (!pte_present(ptent) || !p <<
5529 continue; <<
5530 <<
5531 if (pfn_folio(pte_pfn(ptent)) <<
5532 continue; <<
5533 <<
5534 if (!ignore_writable) { <<
5535 ptent = pte_modify(pt <<
5536 writable = pte_write( <<
5537 if (!writable && pte_ <<
5538 can_change_pte_wr <<
5539 writable = tr <<
5540 } <<
5541 <<
5542 numa_rebuild_single_mapping(v <<
5543 } <<
5544 } 4182 }
5545 4183
5546 static vm_fault_t do_numa_page(struct vm_faul 4184 static vm_fault_t do_numa_page(struct vm_fault *vmf)
5547 { 4185 {
5548 struct vm_area_struct *vma = vmf->vma 4186 struct vm_area_struct *vma = vmf->vma;
5549 struct folio *folio = NULL; !! 4187 struct page *page = NULL;
5550 int nid = NUMA_NO_NODE; !! 4188 int page_nid = NUMA_NO_NODE;
5551 bool writable = false, ignore_writabl <<
5552 bool pte_write_upgrade = vma_wants_ma <<
5553 int last_cpupid; 4189 int last_cpupid;
5554 int target_nid; 4190 int target_nid;
>> 4191 bool migrated = false;
5555 pte_t pte, old_pte; 4192 pte_t pte, old_pte;
5556 int flags = 0, nr_pages; !! 4193 bool was_writable = pte_savedwrite(vmf->orig_pte);
>> 4194 int flags = 0;
5557 4195
5558 /* 4196 /*
5559 * The pte cannot be used safely unti !! 4197 * The "pte" at this point cannot be used safely without
5560 * table lock, that its contents have !! 4198 * validation through pte_unmap_same(). It's of NUMA type but
>> 4199 * the pfn may be screwed if the read is non atomic.
5561 */ 4200 */
>> 4201 vmf->ptl = pte_lockptr(vma->vm_mm, vmf->pmd);
5562 spin_lock(vmf->ptl); 4202 spin_lock(vmf->ptl);
5563 /* Read the live PTE from the page ta !! 4203 if (unlikely(!pte_same(*vmf->pte, vmf->orig_pte))) {
5564 old_pte = ptep_get(vmf->pte); <<
5565 <<
5566 if (unlikely(!pte_same(old_pte, vmf-> <<
5567 pte_unmap_unlock(vmf->pte, vm 4204 pte_unmap_unlock(vmf->pte, vmf->ptl);
5568 return 0; !! 4205 goto out;
5569 } 4206 }
5570 4207
5571 pte = pte_modify(old_pte, vma->vm_pag <<
5572 <<
5573 /* 4208 /*
5574 * Detect now whether the PTE could b !! 4209 * Make it present again, Depending on how arch implementes non
5575 * is only valid while holding the PT !! 4210 * accessible ptes, some can allow access by kernel mode.
5576 */ 4211 */
5577 writable = pte_write(pte); !! 4212 old_pte = ptep_modify_prot_start(vma, vmf->address, vmf->pte);
5578 if (!writable && pte_write_upgrade && !! 4213 pte = pte_modify(old_pte, vma->vm_page_prot);
5579 can_change_pte_writable(vma, vmf- !! 4214 pte = pte_mkyoung(pte);
5580 writable = true; !! 4215 if (was_writable)
5581 !! 4216 pte = pte_mkwrite(pte);
5582 folio = vm_normal_folio(vma, vmf->add !! 4217 ptep_modify_prot_commit(vma, vmf->address, vmf->pte, old_pte, pte);
5583 if (!folio || folio_is_zone_device(fo !! 4218 update_mmu_cache(vma, vmf->address, vmf->pte);
5584 goto out_map; <<
5585 <<
5586 nid = folio_nid(folio); <<
5587 nr_pages = folio_nr_pages(folio); <<
5588 <<
5589 target_nid = numa_migrate_check(folio <<
5590 writa <<
5591 if (target_nid == NUMA_NO_NODE) <<
5592 goto out_map; <<
5593 if (migrate_misplaced_folio_prepare(f <<
5594 flags |= TNF_MIGRATE_FAIL; <<
5595 goto out_map; <<
5596 } <<
5597 /* The folio is isolated and isolatio <<
5598 pte_unmap_unlock(vmf->pte, vmf->ptl); <<
5599 writable = false; <<
5600 ignore_writable = true; <<
5601 4219
5602 /* Migrate to the requested node */ !! 4220 page = vm_normal_page(vma, vmf->address, pte);
5603 if (!migrate_misplaced_folio(folio, v !! 4221 if (!page) {
5604 nid = target_nid; !! 4222 pte_unmap_unlock(vmf->pte, vmf->ptl);
5605 flags |= TNF_MIGRATED; <<
5606 task_numa_fault(last_cpupid, <<
5607 return 0; 4223 return 0;
5608 } 4224 }
5609 4225
5610 flags |= TNF_MIGRATE_FAIL; !! 4226 /* TODO: handle PTE-mapped THP */
5611 vmf->pte = pte_offset_map_lock(vma->v !! 4227 if (PageCompound(page)) {
5612 vmf->a <<
5613 if (unlikely(!vmf->pte)) <<
5614 return 0; <<
5615 if (unlikely(!pte_same(ptep_get(vmf-> <<
5616 pte_unmap_unlock(vmf->pte, vm 4228 pte_unmap_unlock(vmf->pte, vmf->ptl);
5617 return 0; 4229 return 0;
5618 } 4230 }
5619 out_map: !! 4231
5620 /* 4232 /*
5621 * Make it present again, depending o !! 4233 * Avoid grouping on RO pages in general. RO pages shouldn't hurt as
5622 * non-accessible ptes, some can allo !! 4234 * much anyway since they can be in shared cache state. This misses
>> 4235 * the case where a mapping is writable but the process never writes
>> 4236 * to it but pte_write gets cleared during protection updates and
>> 4237 * pte_dirty has unpredictable behaviour between PTE scan updates,
>> 4238 * background writeback, dirty balancing and application behaviour.
5623 */ 4239 */
5624 if (folio && folio_test_large(folio)) !! 4240 if (!pte_write(pte))
5625 numa_rebuild_large_mapping(vm !! 4241 flags |= TNF_NO_GROUP;
5626 pt !! 4242
5627 else !! 4243 /*
5628 numa_rebuild_single_mapping(v !! 4244 * Flag if the page is shared between multiple address spaces. This
5629 w !! 4245 * is later used when determining whether to group tasks together
>> 4246 */
>> 4247 if (page_mapcount(page) > 1 && (vma->vm_flags & VM_SHARED))
>> 4248 flags |= TNF_SHARED;
>> 4249
>> 4250 last_cpupid = page_cpupid_last(page);
>> 4251 page_nid = page_to_nid(page);
>> 4252 target_nid = numa_migrate_prep(page, vma, vmf->address, page_nid,
>> 4253 &flags);
5630 pte_unmap_unlock(vmf->pte, vmf->ptl); 4254 pte_unmap_unlock(vmf->pte, vmf->ptl);
>> 4255 if (target_nid == NUMA_NO_NODE) {
>> 4256 put_page(page);
>> 4257 goto out;
>> 4258 }
>> 4259
>> 4260 /* Migrate to the requested node */
>> 4261 migrated = migrate_misplaced_page(page, vma, target_nid);
>> 4262 if (migrated) {
>> 4263 page_nid = target_nid;
>> 4264 flags |= TNF_MIGRATED;
>> 4265 } else
>> 4266 flags |= TNF_MIGRATE_FAIL;
5631 4267
5632 if (nid != NUMA_NO_NODE) !! 4268 out:
5633 task_numa_fault(last_cpupid, !! 4269 if (page_nid != NUMA_NO_NODE)
>> 4270 task_numa_fault(last_cpupid, page_nid, 1, flags);
5634 return 0; 4271 return 0;
5635 } 4272 }
5636 4273
5637 static inline vm_fault_t create_huge_pmd(stru 4274 static inline vm_fault_t create_huge_pmd(struct vm_fault *vmf)
5638 { 4275 {
5639 struct vm_area_struct *vma = vmf->vma !! 4276 if (vma_is_anonymous(vmf->vma))
5640 if (vma_is_anonymous(vma)) <<
5641 return do_huge_pmd_anonymous_ 4277 return do_huge_pmd_anonymous_page(vmf);
5642 if (vma->vm_ops->huge_fault) !! 4278 if (vmf->vma->vm_ops->huge_fault)
5643 return vma->vm_ops->huge_faul !! 4279 return vmf->vma->vm_ops->huge_fault(vmf, PE_SIZE_PMD);
5644 return VM_FAULT_FALLBACK; 4280 return VM_FAULT_FALLBACK;
5645 } 4281 }
5646 4282
5647 /* `inline' is required to avoid gcc 4.1.2 bu 4283 /* `inline' is required to avoid gcc 4.1.2 build error */
5648 static inline vm_fault_t wp_huge_pmd(struct v !! 4284 static inline vm_fault_t wp_huge_pmd(struct vm_fault *vmf, pmd_t orig_pmd)
5649 { 4285 {
5650 struct vm_area_struct *vma = vmf->vma !! 4286 if (vma_is_anonymous(vmf->vma)) {
5651 const bool unshare = vmf->flags & FAU !! 4287 if (userfaultfd_huge_pmd_wp(vmf->vma, orig_pmd))
5652 vm_fault_t ret; <<
5653 <<
5654 if (vma_is_anonymous(vma)) { <<
5655 if (likely(!unshare) && <<
5656 userfaultfd_huge_pmd_wp(v <<
5657 if (userfaultfd_wp_as <<
5658 goto split; <<
5659 return handle_userfau 4288 return handle_userfault(vmf, VM_UFFD_WP);
5660 } !! 4289 return do_huge_pmd_wp_page(vmf, orig_pmd);
5661 return do_huge_pmd_wp_page(vm <<
5662 } 4290 }
>> 4291 if (vmf->vma->vm_ops->huge_fault) {
>> 4292 vm_fault_t ret = vmf->vma->vm_ops->huge_fault(vmf, PE_SIZE_PMD);
5663 4293
5664 if (vma->vm_flags & (VM_SHARED | VM_M !! 4294 if (!(ret & VM_FAULT_FALLBACK))
5665 if (vma->vm_ops->huge_fault) !! 4295 return ret;
5666 ret = vma->vm_ops->hu <<
5667 if (!(ret & VM_FAULT_ <<
5668 return ret; <<
5669 } <<
5670 } 4296 }
5671 4297
5672 split: <<
5673 /* COW or write-notify handled on pte 4298 /* COW or write-notify handled on pte level: split pmd. */
5674 __split_huge_pmd(vma, vmf->pmd, vmf-> !! 4299 __split_huge_pmd(vmf->vma, vmf->pmd, vmf->address, false, NULL);
5675 4300
5676 return VM_FAULT_FALLBACK; 4301 return VM_FAULT_FALLBACK;
5677 } 4302 }
5678 4303
5679 static vm_fault_t create_huge_pud(struct vm_f 4304 static vm_fault_t create_huge_pud(struct vm_fault *vmf)
5680 { 4305 {
5681 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && 4306 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && \
5682 defined(CONFIG_HAVE_ARCH_TRANSPARENT_ 4307 defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
5683 struct vm_area_struct *vma = vmf->vma <<
5684 /* No support for anonymous transpare 4308 /* No support for anonymous transparent PUD pages yet */
5685 if (vma_is_anonymous(vma)) !! 4309 if (vma_is_anonymous(vmf->vma))
5686 return VM_FAULT_FALLBACK; !! 4310 goto split;
5687 if (vma->vm_ops->huge_fault) !! 4311 if (vmf->vma->vm_ops->huge_fault) {
5688 return vma->vm_ops->huge_faul !! 4312 vm_fault_t ret = vmf->vma->vm_ops->huge_fault(vmf, PE_SIZE_PUD);
>> 4313
>> 4314 if (!(ret & VM_FAULT_FALLBACK))
>> 4315 return ret;
>> 4316 }
>> 4317 split:
>> 4318 /* COW or write-notify not handled on PUD level: split pud.*/
>> 4319 __split_huge_pud(vmf->vma, vmf->pud, vmf->address);
5689 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 4320 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
5690 return VM_FAULT_FALLBACK; 4321 return VM_FAULT_FALLBACK;
5691 } 4322 }
5692 4323
5693 static vm_fault_t wp_huge_pud(struct vm_fault 4324 static vm_fault_t wp_huge_pud(struct vm_fault *vmf, pud_t orig_pud)
5694 { 4325 {
5695 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !! 4326 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
5696 defined(CONFIG_HAVE_ARCH_TRANSPARENT_ <<
5697 struct vm_area_struct *vma = vmf->vma <<
5698 vm_fault_t ret; <<
5699 <<
5700 /* No support for anonymous transpare 4327 /* No support for anonymous transparent PUD pages yet */
5701 if (vma_is_anonymous(vma)) !! 4328 if (vma_is_anonymous(vmf->vma))
5702 goto split; !! 4329 return VM_FAULT_FALLBACK;
5703 if (vma->vm_flags & (VM_SHARED | VM_M !! 4330 if (vmf->vma->vm_ops->huge_fault)
5704 if (vma->vm_ops->huge_fault) !! 4331 return vmf->vma->vm_ops->huge_fault(vmf, PE_SIZE_PUD);
5705 ret = vma->vm_ops->hu !! 4332 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
5706 if (!(ret & VM_FAULT_ <<
5707 return ret; <<
5708 } <<
5709 } <<
5710 split: <<
5711 /* COW or write-notify not handled on <<
5712 __split_huge_pud(vma, vmf->pud, vmf-> <<
5713 #endif /* CONFIG_TRANSPARENT_HUGEPAGE && CONF <<
5714 return VM_FAULT_FALLBACK; 4333 return VM_FAULT_FALLBACK;
5715 } 4334 }
5716 4335
5717 /* 4336 /*
5718 * These routines also need to handle stuff l 4337 * These routines also need to handle stuff like marking pages dirty
5719 * and/or accessed for architectures that don 4338 * and/or accessed for architectures that don't do it in hardware (most
5720 * RISC architectures). The early dirtying i 4339 * RISC architectures). The early dirtying is also good on the i386.
5721 * 4340 *
5722 * There is also a hook called "update_mmu_ca 4341 * There is also a hook called "update_mmu_cache()" that architectures
5723 * with external mmu caches can use to update 4342 * with external mmu caches can use to update those (ie the Sparc or
5724 * PowerPC hashed page tables that act as ext 4343 * PowerPC hashed page tables that act as extended TLBs).
5725 * 4344 *
5726 * We enter with non-exclusive mmap_lock (to 4345 * We enter with non-exclusive mmap_lock (to exclude vma changes, but allow
5727 * concurrent faults). 4346 * concurrent faults).
5728 * 4347 *
5729 * The mmap_lock may have been released depen 4348 * The mmap_lock may have been released depending on flags and our return value.
5730 * See filemap_fault() and __folio_lock_or_re !! 4349 * See filemap_fault() and __lock_page_or_retry().
5731 */ 4350 */
5732 static vm_fault_t handle_pte_fault(struct vm_ 4351 static vm_fault_t handle_pte_fault(struct vm_fault *vmf)
5733 { 4352 {
5734 pte_t entry; 4353 pte_t entry;
5735 4354
5736 if (unlikely(pmd_none(*vmf->pmd))) { 4355 if (unlikely(pmd_none(*vmf->pmd))) {
5737 /* 4356 /*
5738 * Leave __pte_alloc() until 4357 * Leave __pte_alloc() until later: because vm_ops->fault may
5739 * want to allocate huge page 4358 * want to allocate huge page, and if we expose page table
5740 * for an instant, it will be 4359 * for an instant, it will be difficult to retract from
5741 * concurrent faults and from 4360 * concurrent faults and from rmap lookups.
5742 */ 4361 */
5743 vmf->pte = NULL; 4362 vmf->pte = NULL;
5744 vmf->flags &= ~FAULT_FLAG_ORI <<
5745 } else { 4363 } else {
>> 4364 /* See comment in pte_alloc_one_map() */
>> 4365 if (pmd_devmap_trans_unstable(vmf->pmd))
>> 4366 return 0;
5746 /* 4367 /*
5747 * A regular pmd is establish 4368 * A regular pmd is established and it can't morph into a huge
5748 * pmd by anon khugepaged, si !! 4369 * pmd from under us anymore at this point because we hold the
5749 * mode; but shmem or file co !! 4370 * mmap_lock read mode and khugepaged takes it in write mode.
5750 * it into a huge pmd: just r !! 4371 * So now it's safe to run pte_offset_map().
5751 */ 4372 */
5752 vmf->pte = pte_offset_map_nol !! 4373 vmf->pte = pte_offset_map(vmf->pmd, vmf->address);
5753 !! 4374 vmf->orig_pte = *vmf->pte;
5754 if (unlikely(!vmf->pte)) <<
5755 return 0; <<
5756 vmf->orig_pte = ptep_get_lock <<
5757 vmf->flags |= FAULT_FLAG_ORIG <<
5758 4375
>> 4376 /*
>> 4377 * some architectures can have larger ptes than wordsize,
>> 4378 * e.g.ppc44x-defconfig has CONFIG_PTE_64BIT=y and
>> 4379 * CONFIG_32BIT=y, so READ_ONCE cannot guarantee atomic
>> 4380 * accesses. The code below just needs a consistent view
>> 4381 * for the ifs and we later double check anyway with the
>> 4382 * ptl lock held. So here a barrier will do.
>> 4383 */
>> 4384 barrier();
5759 if (pte_none(vmf->orig_pte)) 4385 if (pte_none(vmf->orig_pte)) {
5760 pte_unmap(vmf->pte); 4386 pte_unmap(vmf->pte);
5761 vmf->pte = NULL; 4387 vmf->pte = NULL;
5762 } 4388 }
5763 } 4389 }
5764 4390
5765 if (!vmf->pte) !! 4391 if (!vmf->pte) {
5766 return do_pte_missing(vmf); !! 4392 if (vma_is_anonymous(vmf->vma))
>> 4393 return do_anonymous_page(vmf);
>> 4394 else
>> 4395 return do_fault(vmf);
>> 4396 }
5767 4397
5768 if (!pte_present(vmf->orig_pte)) 4398 if (!pte_present(vmf->orig_pte))
5769 return do_swap_page(vmf); 4399 return do_swap_page(vmf);
5770 4400
5771 if (pte_protnone(vmf->orig_pte) && vm 4401 if (pte_protnone(vmf->orig_pte) && vma_is_accessible(vmf->vma))
5772 return do_numa_page(vmf); 4402 return do_numa_page(vmf);
5773 4403
>> 4404 vmf->ptl = pte_lockptr(vmf->vma->vm_mm, vmf->pmd);
5774 spin_lock(vmf->ptl); 4405 spin_lock(vmf->ptl);
5775 entry = vmf->orig_pte; 4406 entry = vmf->orig_pte;
5776 if (unlikely(!pte_same(ptep_get(vmf-> !! 4407 if (unlikely(!pte_same(*vmf->pte, entry))) {
5777 update_mmu_tlb(vmf->vma, vmf- 4408 update_mmu_tlb(vmf->vma, vmf->address, vmf->pte);
5778 goto unlock; 4409 goto unlock;
5779 } 4410 }
5780 if (vmf->flags & (FAULT_FLAG_WRITE|FA !! 4411 if (vmf->flags & FAULT_FLAG_WRITE) {
5781 if (!pte_write(entry)) 4412 if (!pte_write(entry))
5782 return do_wp_page(vmf 4413 return do_wp_page(vmf);
5783 else if (likely(vmf->flags & !! 4414 entry = pte_mkdirty(entry);
5784 entry = pte_mkdirty(e <<
5785 } 4415 }
5786 entry = pte_mkyoung(entry); 4416 entry = pte_mkyoung(entry);
5787 if (ptep_set_access_flags(vmf->vma, v 4417 if (ptep_set_access_flags(vmf->vma, vmf->address, vmf->pte, entry,
5788 vmf->flags & 4418 vmf->flags & FAULT_FLAG_WRITE)) {
5789 update_mmu_cache_range(vmf, v !! 4419 update_mmu_cache(vmf->vma, vmf->address, vmf->pte);
5790 vmf->pte, 1); <<
5791 } else { 4420 } else {
5792 /* Skip spurious TLB flush fo 4421 /* Skip spurious TLB flush for retried page fault */
5793 if (vmf->flags & FAULT_FLAG_T 4422 if (vmf->flags & FAULT_FLAG_TRIED)
5794 goto unlock; 4423 goto unlock;
5795 /* 4424 /*
5796 * This is needed only for pr 4425 * This is needed only for protection faults but the arch code
5797 * is not yet telling us if t 4426 * is not yet telling us if this is a protection fault or not.
5798 * This still avoids useless 4427 * This still avoids useless tlb flushes for .text page faults
5799 * with threads. 4428 * with threads.
5800 */ 4429 */
5801 if (vmf->flags & FAULT_FLAG_W 4430 if (vmf->flags & FAULT_FLAG_WRITE)
5802 flush_tlb_fix_spuriou !! 4431 flush_tlb_fix_spurious_fault(vmf->vma, vmf->address);
5803 <<
5804 } 4432 }
5805 unlock: 4433 unlock:
5806 pte_unmap_unlock(vmf->pte, vmf->ptl); 4434 pte_unmap_unlock(vmf->pte, vmf->ptl);
5807 return 0; 4435 return 0;
5808 } 4436 }
5809 4437
5810 /* 4438 /*
5811 * On entry, we hold either the VMA lock or t !! 4439 * By the time we get here, we already hold the mm semaphore
5812 * (FAULT_FLAG_VMA_LOCK tells you which). If !! 4440 *
5813 * the result, the mmap_lock is not held on e !! 4441 * The mmap_lock may have been released depending on flags and our
5814 * and __folio_lock_or_retry(). !! 4442 * return value. See filemap_fault() and __lock_page_or_retry().
5815 */ 4443 */
5816 static vm_fault_t __handle_mm_fault(struct vm 4444 static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma,
5817 unsigned long address, unsign 4445 unsigned long address, unsigned int flags)
5818 { 4446 {
5819 struct vm_fault vmf = { 4447 struct vm_fault vmf = {
5820 .vma = vma, 4448 .vma = vma,
5821 .address = address & PAGE_MAS 4449 .address = address & PAGE_MASK,
5822 .real_address = address, <<
5823 .flags = flags, 4450 .flags = flags,
5824 .pgoff = linear_page_index(vm 4451 .pgoff = linear_page_index(vma, address),
5825 .gfp_mask = __get_fault_gfp_m 4452 .gfp_mask = __get_fault_gfp_mask(vma),
5826 }; 4453 };
>> 4454 unsigned int dirty = flags & FAULT_FLAG_WRITE;
5827 struct mm_struct *mm = vma->vm_mm; 4455 struct mm_struct *mm = vma->vm_mm;
5828 unsigned long vm_flags = vma->vm_flag <<
5829 pgd_t *pgd; 4456 pgd_t *pgd;
5830 p4d_t *p4d; 4457 p4d_t *p4d;
5831 vm_fault_t ret; 4458 vm_fault_t ret;
5832 4459
5833 pgd = pgd_offset(mm, address); 4460 pgd = pgd_offset(mm, address);
5834 p4d = p4d_alloc(mm, pgd, address); 4461 p4d = p4d_alloc(mm, pgd, address);
5835 if (!p4d) 4462 if (!p4d)
5836 return VM_FAULT_OOM; 4463 return VM_FAULT_OOM;
5837 4464
5838 vmf.pud = pud_alloc(mm, p4d, address) 4465 vmf.pud = pud_alloc(mm, p4d, address);
5839 if (!vmf.pud) 4466 if (!vmf.pud)
5840 return VM_FAULT_OOM; 4467 return VM_FAULT_OOM;
5841 retry_pud: 4468 retry_pud:
5842 if (pud_none(*vmf.pud) && !! 4469 if (pud_none(*vmf.pud) && __transparent_hugepage_enabled(vma)) {
5843 thp_vma_allowable_order(vma, vm_f <<
5844 TVA_IN_PF | T <<
5845 ret = create_huge_pud(&vmf); 4470 ret = create_huge_pud(&vmf);
5846 if (!(ret & VM_FAULT_FALLBACK 4471 if (!(ret & VM_FAULT_FALLBACK))
5847 return ret; 4472 return ret;
5848 } else { 4473 } else {
5849 pud_t orig_pud = *vmf.pud; 4474 pud_t orig_pud = *vmf.pud;
5850 4475
5851 barrier(); 4476 barrier();
5852 if (pud_trans_huge(orig_pud) 4477 if (pud_trans_huge(orig_pud) || pud_devmap(orig_pud)) {
5853 4478
5854 /* !! 4479 /* NUMA case for anonymous PUDs would go here */
5855 * TODO once we suppo !! 4480
5856 * FAULT_FLAG_UNSHARE !! 4481 if (dirty && !pud_write(orig_pud)) {
5857 */ <<
5858 if ((flags & FAULT_FL <<
5859 ret = wp_huge 4482 ret = wp_huge_pud(&vmf, orig_pud);
5860 if (!(ret & V 4483 if (!(ret & VM_FAULT_FALLBACK))
5861 retur 4484 return ret;
5862 } else { 4485 } else {
5863 huge_pud_set_ 4486 huge_pud_set_accessed(&vmf, orig_pud);
5864 return 0; 4487 return 0;
5865 } 4488 }
5866 } 4489 }
5867 } 4490 }
5868 4491
5869 vmf.pmd = pmd_alloc(mm, vmf.pud, addr 4492 vmf.pmd = pmd_alloc(mm, vmf.pud, address);
5870 if (!vmf.pmd) 4493 if (!vmf.pmd)
5871 return VM_FAULT_OOM; 4494 return VM_FAULT_OOM;
5872 4495
5873 /* Huge pud page fault raced with pmd 4496 /* Huge pud page fault raced with pmd_alloc? */
5874 if (pud_trans_unstable(vmf.pud)) 4497 if (pud_trans_unstable(vmf.pud))
5875 goto retry_pud; 4498 goto retry_pud;
5876 4499
5877 if (pmd_none(*vmf.pmd) && !! 4500 if (pmd_none(*vmf.pmd) && __transparent_hugepage_enabled(vma)) {
5878 thp_vma_allowable_order(vma, vm_f <<
5879 TVA_IN_PF | T <<
5880 ret = create_huge_pmd(&vmf); 4501 ret = create_huge_pmd(&vmf);
5881 if (!(ret & VM_FAULT_FALLBACK 4502 if (!(ret & VM_FAULT_FALLBACK))
5882 return ret; 4503 return ret;
5883 } else { 4504 } else {
5884 vmf.orig_pmd = pmdp_get_lockl !! 4505 pmd_t orig_pmd = *vmf.pmd;
5885 4506
5886 if (unlikely(is_swap_pmd(vmf. !! 4507 barrier();
>> 4508 if (unlikely(is_swap_pmd(orig_pmd))) {
5887 VM_BUG_ON(thp_migrati 4509 VM_BUG_ON(thp_migration_supported() &&
5888 !is !! 4510 !is_pmd_migration_entry(orig_pmd));
5889 if (is_pmd_migration_ !! 4511 if (is_pmd_migration_entry(orig_pmd))
5890 pmd_migration 4512 pmd_migration_entry_wait(mm, vmf.pmd);
5891 return 0; 4513 return 0;
5892 } 4514 }
5893 if (pmd_trans_huge(vmf.orig_p !! 4515 if (pmd_trans_huge(orig_pmd) || pmd_devmap(orig_pmd)) {
5894 if (pmd_protnone(vmf. !! 4516 if (pmd_protnone(orig_pmd) && vma_is_accessible(vma))
5895 return do_hug !! 4517 return do_huge_pmd_numa_page(&vmf, orig_pmd);
5896 !! 4518
5897 if ((flags & (FAULT_F !! 4519 if (dirty && !pmd_write(orig_pmd)) {
5898 !pmd_write(vmf.or !! 4520 ret = wp_huge_pmd(&vmf, orig_pmd);
5899 ret = wp_huge <<
5900 if (!(ret & V 4521 if (!(ret & VM_FAULT_FALLBACK))
5901 retur 4522 return ret;
5902 } else { 4523 } else {
5903 huge_pmd_set_ !! 4524 huge_pmd_set_accessed(&vmf, orig_pmd);
5904 return 0; 4525 return 0;
5905 } 4526 }
5906 } 4527 }
5907 } 4528 }
5908 4529
5909 return handle_pte_fault(&vmf); 4530 return handle_pte_fault(&vmf);
5910 } 4531 }
5911 4532
5912 /** 4533 /**
5913 * mm_account_fault - Do page fault accountin !! 4534 * mm_account_fault - Do page fault accountings
5914 * @mm: mm from which memcg should be extract !! 4535 *
5915 * @regs: the pt_regs struct pointer. When s 4536 * @regs: the pt_regs struct pointer. When set to NULL, will skip accounting
5916 * of perf event counters, but we'll s 4537 * of perf event counters, but we'll still do the per-task accounting to
5917 * the task who triggered this page fa 4538 * the task who triggered this page fault.
5918 * @address: the faulted address. 4539 * @address: the faulted address.
5919 * @flags: the fault flags. 4540 * @flags: the fault flags.
5920 * @ret: the fault retcode. 4541 * @ret: the fault retcode.
5921 * 4542 *
5922 * This will take care of most of the page fa !! 4543 * This will take care of most of the page fault accountings. Meanwhile, it
5923 * will also include the PERF_COUNT_SW_PAGE_F 4544 * will also include the PERF_COUNT_SW_PAGE_FAULTS_[MAJ|MIN] perf counter
5924 * updates. However, note that the handling !! 4545 * updates. However note that the handling of PERF_COUNT_SW_PAGE_FAULTS should
5925 * still be in per-arch page fault handlers a 4546 * still be in per-arch page fault handlers at the entry of page fault.
5926 */ 4547 */
5927 static inline void mm_account_fault(struct mm !! 4548 static inline void mm_account_fault(struct pt_regs *regs,
5928 unsigned 4549 unsigned long address, unsigned int flags,
5929 vm_fault_ 4550 vm_fault_t ret)
5930 { 4551 {
5931 bool major; 4552 bool major;
5932 4553
5933 /* Incomplete faults will be accounte <<
5934 if (ret & VM_FAULT_RETRY) <<
5935 return; <<
5936 <<
5937 /* <<
5938 * To preserve the behavior of older <<
5939 * both successful and failed faults, <<
5940 * which ignore failed cases. <<
5941 */ <<
5942 count_vm_event(PGFAULT); <<
5943 count_memcg_event_mm(mm, PGFAULT); <<
5944 <<
5945 /* 4554 /*
5946 * Do not account for unsuccessful fa !! 4555 * We don't do accounting for some specific faults:
5947 * valid). That includes arch_vma_ac !! 4556 *
5948 * reaching here. So this is not a "t !! 4557 * - Unsuccessful faults (e.g. when the address wasn't valid). That
5949 * counter. We should use the hw pro !! 4558 * includes arch_vma_access_permitted() failing before reaching here.
>> 4559 * So this is not a "this many hardware page faults" counter. We
>> 4560 * should use the hw profiling for that.
>> 4561 *
>> 4562 * - Incomplete faults (VM_FAULT_RETRY). They will only be counted
>> 4563 * once they're completed.
5950 */ 4564 */
5951 if (ret & VM_FAULT_ERROR) !! 4565 if (ret & (VM_FAULT_ERROR | VM_FAULT_RETRY))
5952 return; 4566 return;
5953 4567
5954 /* 4568 /*
5955 * We define the fault as a major fau 4569 * We define the fault as a major fault when the final successful fault
5956 * is VM_FAULT_MAJOR, or if it retrie 4570 * is VM_FAULT_MAJOR, or if it retried (which implies that we couldn't
5957 * handle it immediately previously). 4571 * handle it immediately previously).
5958 */ 4572 */
5959 major = (ret & VM_FAULT_MAJOR) || (fl 4573 major = (ret & VM_FAULT_MAJOR) || (flags & FAULT_FLAG_TRIED);
5960 4574
5961 if (major) 4575 if (major)
5962 current->maj_flt++; 4576 current->maj_flt++;
5963 else 4577 else
5964 current->min_flt++; 4578 current->min_flt++;
5965 4579
5966 /* 4580 /*
5967 * If the fault is done for GUP, regs 4581 * If the fault is done for GUP, regs will be NULL. We only do the
5968 * accounting for the per thread faul 4582 * accounting for the per thread fault counters who triggered the
5969 * fault, and we skip the perf event 4583 * fault, and we skip the perf event updates.
5970 */ 4584 */
5971 if (!regs) 4585 if (!regs)
5972 return; 4586 return;
5973 4587
5974 if (major) 4588 if (major)
5975 perf_sw_event(PERF_COUNT_SW_P 4589 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs, address);
5976 else 4590 else
5977 perf_sw_event(PERF_COUNT_SW_P 4591 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, address);
5978 } 4592 }
5979 4593
5980 #ifdef CONFIG_LRU_GEN <<
5981 static void lru_gen_enter_fault(struct vm_are <<
5982 { <<
5983 /* the LRU algorithm only applies to <<
5984 current->in_lru_fault = vma_has_recen <<
5985 } <<
5986 <<
5987 static void lru_gen_exit_fault(void) <<
5988 { <<
5989 current->in_lru_fault = false; <<
5990 } <<
5991 #else <<
5992 static void lru_gen_enter_fault(struct vm_are <<
5993 { <<
5994 } <<
5995 <<
5996 static void lru_gen_exit_fault(void) <<
5997 { <<
5998 } <<
5999 #endif /* CONFIG_LRU_GEN */ <<
6000 <<
6001 static vm_fault_t sanitize_fault_flags(struct <<
6002 unsign <<
6003 { <<
6004 if (unlikely(*flags & FAULT_FLAG_UNSH <<
6005 if (WARN_ON_ONCE(*flags & FAU <<
6006 return VM_FAULT_SIGSE <<
6007 /* <<
6008 * FAULT_FLAG_UNSHARE only ap <<
6009 * just treat it like an ordi <<
6010 */ <<
6011 if (!is_cow_mapping(vma->vm_f <<
6012 *flags &= ~FAULT_FLAG <<
6013 } else if (*flags & FAULT_FLAG_WRITE) <<
6014 /* Write faults on read-only <<
6015 if (WARN_ON_ONCE(!(vma->vm_fl <<
6016 return VM_FAULT_SIGSE <<
6017 /* ... and FOLL_FORCE only ap <<
6018 if (WARN_ON_ONCE(!(vma->vm_fl <<
6019 !is_cow_mapp <<
6020 return VM_FAULT_SIGSE <<
6021 } <<
6022 #ifdef CONFIG_PER_VMA_LOCK <<
6023 /* <<
6024 * Per-VMA locks can't be used with F <<
6025 * the assumption that lock is droppe <<
6026 */ <<
6027 if (WARN_ON_ONCE((*flags & <<
6028 (FAULT_FLAG_VMA_LOCK <<
6029 (FAULT_FLAG_VMA_LOCK <<
6030 return VM_FAULT_SIGSEGV; <<
6031 #endif <<
6032 <<
6033 return 0; <<
6034 } <<
6035 <<
6036 /* 4594 /*
6037 * By the time we get here, we already hold t 4595 * By the time we get here, we already hold the mm semaphore
6038 * 4596 *
6039 * The mmap_lock may have been released depen 4597 * The mmap_lock may have been released depending on flags and our
6040 * return value. See filemap_fault() and __f !! 4598 * return value. See filemap_fault() and __lock_page_or_retry().
6041 */ 4599 */
6042 vm_fault_t handle_mm_fault(struct vm_area_str 4600 vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
6043 unsigned int flags 4601 unsigned int flags, struct pt_regs *regs)
6044 { 4602 {
6045 /* If the fault handler drops the mma <<
6046 struct mm_struct *mm = vma->vm_mm; <<
6047 vm_fault_t ret; 4603 vm_fault_t ret;
6048 bool is_droppable; <<
6049 4604
6050 __set_current_state(TASK_RUNNING); 4605 __set_current_state(TASK_RUNNING);
6051 4606
6052 ret = sanitize_fault_flags(vma, &flag !! 4607 count_vm_event(PGFAULT);
6053 if (ret) !! 4608 count_memcg_event_mm(vma->vm_mm, PGFAULT);
6054 goto out; !! 4609
>> 4610 /* do counter updates before entering really critical section. */
>> 4611 check_sync_rss_stat(current);
6055 4612
6056 if (!arch_vma_access_permitted(vma, f 4613 if (!arch_vma_access_permitted(vma, flags & FAULT_FLAG_WRITE,
6057 f 4614 flags & FAULT_FLAG_INSTRUCTION,
6058 f !! 4615 flags & FAULT_FLAG_REMOTE))
6059 ret = VM_FAULT_SIGSEGV; !! 4616 return VM_FAULT_SIGSEGV;
6060 goto out; <<
6061 } <<
6062 <<
6063 is_droppable = !!(vma->vm_flags & VM_ <<
6064 4617
6065 /* 4618 /*
6066 * Enable the memcg OOM handling for 4619 * Enable the memcg OOM handling for faults triggered in user
6067 * space. Kernel faults are handled 4620 * space. Kernel faults are handled more gracefully.
6068 */ 4621 */
6069 if (flags & FAULT_FLAG_USER) 4622 if (flags & FAULT_FLAG_USER)
6070 mem_cgroup_enter_user_fault() 4623 mem_cgroup_enter_user_fault();
6071 4624
6072 lru_gen_enter_fault(vma); <<
6073 <<
6074 if (unlikely(is_vm_hugetlb_page(vma)) 4625 if (unlikely(is_vm_hugetlb_page(vma)))
6075 ret = hugetlb_fault(vma->vm_m 4626 ret = hugetlb_fault(vma->vm_mm, vma, address, flags);
6076 else 4627 else
6077 ret = __handle_mm_fault(vma, 4628 ret = __handle_mm_fault(vma, address, flags);
6078 4629
6079 /* <<
6080 * Warning: It is no longer safe to d <<
6081 * because mmap_lock might have been <<
6082 * vma might be destroyed from undern <<
6083 */ <<
6084 <<
6085 lru_gen_exit_fault(); <<
6086 <<
6087 /* If the mapping is droppable, then <<
6088 if (is_droppable) <<
6089 ret &= ~VM_FAULT_OOM; <<
6090 <<
6091 if (flags & FAULT_FLAG_USER) { 4630 if (flags & FAULT_FLAG_USER) {
6092 mem_cgroup_exit_user_fault(); 4631 mem_cgroup_exit_user_fault();
6093 /* 4632 /*
6094 * The task may have entered 4633 * The task may have entered a memcg OOM situation but
6095 * if the allocation error wa 4634 * if the allocation error was handled gracefully (no
6096 * VM_FAULT_OOM), there is no 4635 * VM_FAULT_OOM), there is no need to kill anything.
6097 * Just clean up the OOM stat 4636 * Just clean up the OOM state peacefully.
6098 */ 4637 */
6099 if (task_in_memcg_oom(current 4638 if (task_in_memcg_oom(current) && !(ret & VM_FAULT_OOM))
6100 mem_cgroup_oom_synchr 4639 mem_cgroup_oom_synchronize(false);
6101 } 4640 }
6102 out: !! 4641
6103 mm_account_fault(mm, regs, address, f !! 4642 mm_account_fault(regs, address, flags, ret);
6104 4643
6105 return ret; 4644 return ret;
6106 } 4645 }
6107 EXPORT_SYMBOL_GPL(handle_mm_fault); 4646 EXPORT_SYMBOL_GPL(handle_mm_fault);
6108 4647
6109 #ifdef CONFIG_LOCK_MM_AND_FIND_VMA <<
6110 #include <linux/extable.h> <<
6111 <<
6112 static inline bool get_mmap_lock_carefully(st <<
6113 { <<
6114 if (likely(mmap_read_trylock(mm))) <<
6115 return true; <<
6116 <<
6117 if (regs && !user_mode(regs)) { <<
6118 unsigned long ip = exception_ <<
6119 if (!search_exception_tables( <<
6120 return false; <<
6121 } <<
6122 <<
6123 return !mmap_read_lock_killable(mm); <<
6124 } <<
6125 <<
6126 static inline bool mmap_upgrade_trylock(struc <<
6127 { <<
6128 /* <<
6129 * We don't have this operation yet. <<
6130 * <<
6131 * It should be easy enough to do: it <<
6132 * atomic_long_try_cmpxchg_acquire <<
6133 * from RWSEM_READER_BIAS -> RWSEM_WR <<
6134 * it also needs the proper lockdep m <<
6135 */ <<
6136 return false; <<
6137 } <<
6138 <<
6139 static inline bool upgrade_mmap_lock_carefull <<
6140 { <<
6141 mmap_read_unlock(mm); <<
6142 if (regs && !user_mode(regs)) { <<
6143 unsigned long ip = exception_ <<
6144 if (!search_exception_tables( <<
6145 return false; <<
6146 } <<
6147 return !mmap_write_lock_killable(mm); <<
6148 } <<
6149 <<
6150 /* <<
6151 * Helper for page fault handling. <<
6152 * <<
6153 * This is kind of equivalend to "mmap_read_l <<
6154 * by "find_extend_vma()", except it's a lot <<
6155 * the locking (and will drop the lock on fai <<
6156 * <<
6157 * For example, if we have a kernel bug that <<
6158 * fault, we don't want to just use mmap_read <<
6159 * the mm lock, because that would deadlock i <<
6160 * to happen while we're holding the mm lock <<
6161 * <<
6162 * So this checks the exception tables on ker <<
6163 * order to only do this all for instructions <<
6164 * expected to fault. <<
6165 * <<
6166 * We can also actually take the mm lock for <<
6167 * need to extend the vma, which helps the VM <<
6168 */ <<
6169 struct vm_area_struct *lock_mm_and_find_vma(s <<
6170 unsigned long addr, s <<
6171 { <<
6172 struct vm_area_struct *vma; <<
6173 <<
6174 if (!get_mmap_lock_carefully(mm, regs <<
6175 return NULL; <<
6176 <<
6177 vma = find_vma(mm, addr); <<
6178 if (likely(vma && (vma->vm_start <= a <<
6179 return vma; <<
6180 <<
6181 /* <<
6182 * Well, dang. We might still be succ <<
6183 * if we can extend a vma to do so. <<
6184 */ <<
6185 if (!vma || !(vma->vm_flags & VM_GROW <<
6186 mmap_read_unlock(mm); <<
6187 return NULL; <<
6188 } <<
6189 <<
6190 /* <<
6191 * We can try to upgrade the mmap loc <<
6192 * in which case we can continue to u <<
6193 * we already looked up. <<
6194 * <<
6195 * Otherwise we'll have to drop the m <<
6196 * re-take it, and also look up the v <<
6197 * re-checking it. <<
6198 */ <<
6199 if (!mmap_upgrade_trylock(mm)) { <<
6200 if (!upgrade_mmap_lock_carefu <<
6201 return NULL; <<
6202 <<
6203 vma = find_vma(mm, addr); <<
6204 if (!vma) <<
6205 goto fail; <<
6206 if (vma->vm_start <= addr) <<
6207 goto success; <<
6208 if (!(vma->vm_flags & VM_GROW <<
6209 goto fail; <<
6210 } <<
6211 <<
6212 if (expand_stack_locked(vma, addr)) <<
6213 goto fail; <<
6214 <<
6215 success: <<
6216 mmap_write_downgrade(mm); <<
6217 return vma; <<
6218 <<
6219 fail: <<
6220 mmap_write_unlock(mm); <<
6221 return NULL; <<
6222 } <<
6223 #endif <<
6224 <<
6225 #ifdef CONFIG_PER_VMA_LOCK <<
6226 /* <<
6227 * Lookup and lock a VMA under RCU protection <<
6228 * stable and not isolated. If the VMA is not <<
6229 * function returns NULL. <<
6230 */ <<
6231 struct vm_area_struct *lock_vma_under_rcu(str <<
6232 uns <<
6233 { <<
6234 MA_STATE(mas, &mm->mm_mt, address, ad <<
6235 struct vm_area_struct *vma; <<
6236 <<
6237 rcu_read_lock(); <<
6238 retry: <<
6239 vma = mas_walk(&mas); <<
6240 if (!vma) <<
6241 goto inval; <<
6242 <<
6243 if (!vma_start_read(vma)) <<
6244 goto inval; <<
6245 <<
6246 /* Check if the VMA got isolated afte <<
6247 if (vma->detached) { <<
6248 vma_end_read(vma); <<
6249 count_vm_vma_lock_event(VMA_L <<
6250 /* The area was replaced with <<
6251 goto retry; <<
6252 } <<
6253 /* <<
6254 * At this point, we have a stable re <<
6255 * locked and we know it hasn't alrea <<
6256 * From here on, we can access the VM <<
6257 * fields are accessible for RCU read <<
6258 */ <<
6259 <<
6260 /* Check since vm_start/vm_end might <<
6261 if (unlikely(address < vma->vm_start <<
6262 goto inval_end_read; <<
6263 <<
6264 rcu_read_unlock(); <<
6265 return vma; <<
6266 <<
6267 inval_end_read: <<
6268 vma_end_read(vma); <<
6269 inval: <<
6270 rcu_read_unlock(); <<
6271 count_vm_vma_lock_event(VMA_LOCK_ABOR <<
6272 return NULL; <<
6273 } <<
6274 #endif /* CONFIG_PER_VMA_LOCK */ <<
6275 <<
6276 #ifndef __PAGETABLE_P4D_FOLDED 4648 #ifndef __PAGETABLE_P4D_FOLDED
6277 /* 4649 /*
6278 * Allocate p4d page table. 4650 * Allocate p4d page table.
6279 * We've already handled the fast-path in-lin 4651 * We've already handled the fast-path in-line.
6280 */ 4652 */
6281 int __p4d_alloc(struct mm_struct *mm, pgd_t * 4653 int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
6282 { 4654 {
6283 p4d_t *new = p4d_alloc_one(mm, addres 4655 p4d_t *new = p4d_alloc_one(mm, address);
6284 if (!new) 4656 if (!new)
6285 return -ENOMEM; 4657 return -ENOMEM;
6286 4658
>> 4659 smp_wmb(); /* See comment in __pte_alloc */
>> 4660
6287 spin_lock(&mm->page_table_lock); 4661 spin_lock(&mm->page_table_lock);
6288 if (pgd_present(*pgd)) { /* An !! 4662 if (pgd_present(*pgd)) /* Another has populated it */
6289 p4d_free(mm, new); 4663 p4d_free(mm, new);
6290 } else { !! 4664 else
6291 smp_wmb(); /* See comment in <<
6292 pgd_populate(mm, pgd, new); 4665 pgd_populate(mm, pgd, new);
6293 } <<
6294 spin_unlock(&mm->page_table_lock); 4666 spin_unlock(&mm->page_table_lock);
6295 return 0; 4667 return 0;
6296 } 4668 }
6297 #endif /* __PAGETABLE_P4D_FOLDED */ 4669 #endif /* __PAGETABLE_P4D_FOLDED */
6298 4670
6299 #ifndef __PAGETABLE_PUD_FOLDED 4671 #ifndef __PAGETABLE_PUD_FOLDED
6300 /* 4672 /*
6301 * Allocate page upper directory. 4673 * Allocate page upper directory.
6302 * We've already handled the fast-path in-lin 4674 * We've already handled the fast-path in-line.
6303 */ 4675 */
6304 int __pud_alloc(struct mm_struct *mm, p4d_t * 4676 int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address)
6305 { 4677 {
6306 pud_t *new = pud_alloc_one(mm, addres 4678 pud_t *new = pud_alloc_one(mm, address);
6307 if (!new) 4679 if (!new)
6308 return -ENOMEM; 4680 return -ENOMEM;
6309 4681
>> 4682 smp_wmb(); /* See comment in __pte_alloc */
>> 4683
6310 spin_lock(&mm->page_table_lock); 4684 spin_lock(&mm->page_table_lock);
6311 if (!p4d_present(*p4d)) { 4685 if (!p4d_present(*p4d)) {
6312 mm_inc_nr_puds(mm); 4686 mm_inc_nr_puds(mm);
6313 smp_wmb(); /* See comment in <<
6314 p4d_populate(mm, p4d, new); 4687 p4d_populate(mm, p4d, new);
6315 } else /* Another has populated it * 4688 } else /* Another has populated it */
6316 pud_free(mm, new); 4689 pud_free(mm, new);
6317 spin_unlock(&mm->page_table_lock); 4690 spin_unlock(&mm->page_table_lock);
6318 return 0; 4691 return 0;
6319 } 4692 }
6320 #endif /* __PAGETABLE_PUD_FOLDED */ 4693 #endif /* __PAGETABLE_PUD_FOLDED */
6321 4694
6322 #ifndef __PAGETABLE_PMD_FOLDED 4695 #ifndef __PAGETABLE_PMD_FOLDED
6323 /* 4696 /*
6324 * Allocate page middle directory. 4697 * Allocate page middle directory.
6325 * We've already handled the fast-path in-lin 4698 * We've already handled the fast-path in-line.
6326 */ 4699 */
6327 int __pmd_alloc(struct mm_struct *mm, pud_t * 4700 int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
6328 { 4701 {
6329 spinlock_t *ptl; 4702 spinlock_t *ptl;
6330 pmd_t *new = pmd_alloc_one(mm, addres 4703 pmd_t *new = pmd_alloc_one(mm, address);
6331 if (!new) 4704 if (!new)
6332 return -ENOMEM; 4705 return -ENOMEM;
6333 4706
>> 4707 smp_wmb(); /* See comment in __pte_alloc */
>> 4708
6334 ptl = pud_lock(mm, pud); 4709 ptl = pud_lock(mm, pud);
6335 if (!pud_present(*pud)) { 4710 if (!pud_present(*pud)) {
6336 mm_inc_nr_pmds(mm); 4711 mm_inc_nr_pmds(mm);
6337 smp_wmb(); /* See comment in <<
6338 pud_populate(mm, pud, new); 4712 pud_populate(mm, pud, new);
6339 } else { /* Another has popula !! 4713 } else /* Another has populated it */
6340 pmd_free(mm, new); 4714 pmd_free(mm, new);
6341 } <<
6342 spin_unlock(ptl); 4715 spin_unlock(ptl);
6343 return 0; 4716 return 0;
6344 } 4717 }
6345 #endif /* __PAGETABLE_PMD_FOLDED */ 4718 #endif /* __PAGETABLE_PMD_FOLDED */
6346 4719
6347 static inline void pfnmap_args_setup(struct f !! 4720 int follow_invalidate_pte(struct mm_struct *mm, unsigned long address,
6348 spinlock !! 4721 struct mmu_notifier_range *range, pte_t **ptepp,
6349 pgprot_t !! 4722 pmd_t **pmdpp, spinlock_t **ptlp)
6350 unsigned <<
6351 bool spe <<
6352 { <<
6353 args->lock = lock; <<
6354 args->ptep = ptep; <<
6355 args->pfn = pfn_base + ((args->addres <<
6356 args->pgprot = pgprot; <<
6357 args->writable = writable; <<
6358 args->special = special; <<
6359 } <<
6360 <<
6361 static inline void pfnmap_lockdep_assert(stru <<
6362 { <<
6363 #ifdef CONFIG_LOCKDEP <<
6364 struct file *file = vma->vm_file; <<
6365 struct address_space *mapping = file <<
6366 <<
6367 if (mapping) <<
6368 lockdep_assert(lockdep_is_hel <<
6369 lockdep_is_hel <<
6370 else <<
6371 lockdep_assert(lockdep_is_hel <<
6372 #endif <<
6373 } <<
6374 <<
6375 /** <<
6376 * follow_pfnmap_start() - Look up a pfn mapp <<
6377 * @args: Pointer to struct @follow_pfnmap_ar <<
6378 * <<
6379 * The caller needs to setup args->vma and ar <<
6380 * virtual address as the target of such look <<
6381 * the results will be put into other output <<
6382 * <<
6383 * After the caller finished using the fields <<
6384 * another follow_pfnmap_end() to proper rele <<
6385 * of such look up request. <<
6386 * <<
6387 * During the start() and end() calls, the re <<
6388 * as proper locks will be held. After the e <<
6389 * in @follow_pfnmap_args will be invalid to <<
6390 * use of such information after end() may re <<
6391 * by the caller with page table updates, oth <<
6392 * security bug. <<
6393 * <<
6394 * If the PTE maps a refcounted page, callers <<
6395 * against invalidation with MMU notifiers; o <<
6396 * a later point in time can trigger use-afte <<
6397 * <<
6398 * Only IO mappings and raw PFN mappings are <<
6399 * should be taken for read, and the mmap sem <<
6400 * before the end() is invoked. <<
6401 * <<
6402 * This function must not be used to modify P <<
6403 * <<
6404 * Return: zero on success, negative otherwis <<
6405 */ <<
6406 int follow_pfnmap_start(struct follow_pfnmap_ <<
6407 { 4723 {
6408 struct vm_area_struct *vma = args->vm !! 4724 pgd_t *pgd;
6409 unsigned long address = args->address !! 4725 p4d_t *p4d;
6410 struct mm_struct *mm = vma->vm_mm; !! 4726 pud_t *pud;
6411 spinlock_t *lock; !! 4727 pmd_t *pmd;
6412 pgd_t *pgdp; !! 4728 pte_t *ptep;
6413 p4d_t *p4dp, p4d; <<
6414 pud_t *pudp, pud; <<
6415 pmd_t *pmdp, pmd; <<
6416 pte_t *ptep, pte; <<
6417 <<
6418 pfnmap_lockdep_assert(vma); <<
6419 4729
6420 if (unlikely(address < vma->vm_start !! 4730 pgd = pgd_offset(mm, address);
>> 4731 if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
6421 goto out; 4732 goto out;
6422 4733
6423 if (!(vma->vm_flags & (VM_IO | VM_PFN !! 4734 p4d = p4d_offset(pgd, address);
6424 goto out; !! 4735 if (p4d_none(*p4d) || unlikely(p4d_bad(*p4d)))
6425 retry: <<
6426 pgdp = pgd_offset(mm, address); <<
6427 if (pgd_none(*pgdp) || unlikely(pgd_b <<
6428 goto out; 4736 goto out;
6429 4737
6430 p4dp = p4d_offset(pgdp, address); !! 4738 pud = pud_offset(p4d, address);
6431 p4d = READ_ONCE(*p4dp); !! 4739 if (pud_none(*pud) || unlikely(pud_bad(*pud)))
6432 if (p4d_none(p4d) || unlikely(p4d_bad <<
6433 goto out; 4740 goto out;
6434 4741
6435 pudp = pud_offset(p4dp, address); !! 4742 pmd = pmd_offset(pud, address);
6436 pud = READ_ONCE(*pudp); !! 4743 VM_BUG_ON(pmd_trans_huge(*pmd));
6437 if (pud_none(pud)) <<
6438 goto out; <<
6439 if (pud_leaf(pud)) { <<
6440 lock = pud_lock(mm, pudp); <<
6441 if (!unlikely(pud_leaf(pud))) <<
6442 spin_unlock(lock); <<
6443 goto retry; <<
6444 } <<
6445 pfnmap_args_setup(args, lock, <<
6446 pud_pfn(pud <<
6447 pud_special <<
6448 return 0; <<
6449 } <<
6450 4744
6451 pmdp = pmd_offset(pudp, address); !! 4745 if (pmd_huge(*pmd)) {
6452 pmd = pmdp_get_lockless(pmdp); !! 4746 if (!pmdpp)
6453 if (pmd_leaf(pmd)) { !! 4747 goto out;
6454 lock = pmd_lock(mm, pmdp); !! 4748
6455 if (!unlikely(pmd_leaf(pmd))) !! 4749 if (range) {
6456 spin_unlock(lock); !! 4750 mmu_notifier_range_init(range, MMU_NOTIFY_CLEAR, 0,
6457 goto retry; !! 4751 NULL, mm, address & PMD_MASK,
6458 } !! 4752 (address & PMD_MASK) + PMD_SIZE);
6459 pfnmap_args_setup(args, lock, !! 4753 mmu_notifier_invalidate_range_start(range);
6460 pmd_pfn(pmd !! 4754 }
6461 pmd_special !! 4755 *ptlp = pmd_lock(mm, pmd);
6462 return 0; !! 4756 if (pmd_huge(*pmd)) {
>> 4757 *pmdpp = pmd;
>> 4758 return 0;
>> 4759 }
>> 4760 spin_unlock(*ptlp);
>> 4761 if (range)
>> 4762 mmu_notifier_invalidate_range_end(range);
6463 } 4763 }
6464 4764
6465 ptep = pte_offset_map_lock(mm, pmdp, !! 4765 if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd)))
6466 if (!ptep) <<
6467 goto out; 4766 goto out;
6468 pte = ptep_get(ptep); !! 4767
6469 if (!pte_present(pte)) !! 4768 if (range) {
>> 4769 mmu_notifier_range_init(range, MMU_NOTIFY_CLEAR, 0, NULL, mm,
>> 4770 address & PAGE_MASK,
>> 4771 (address & PAGE_MASK) + PAGE_SIZE);
>> 4772 mmu_notifier_invalidate_range_start(range);
>> 4773 }
>> 4774 ptep = pte_offset_map_lock(mm, pmd, address, ptlp);
>> 4775 if (!pte_present(*ptep))
6470 goto unlock; 4776 goto unlock;
6471 pfnmap_args_setup(args, lock, ptep, p !! 4777 *ptepp = ptep;
6472 pte_pfn(pte), PAGE_ <<
6473 pte_special(pte)); <<
6474 return 0; 4778 return 0;
6475 unlock: 4779 unlock:
6476 pte_unmap_unlock(ptep, lock); !! 4780 pte_unmap_unlock(ptep, *ptlp);
>> 4781 if (range)
>> 4782 mmu_notifier_invalidate_range_end(range);
6477 out: 4783 out:
6478 return -EINVAL; 4784 return -EINVAL;
6479 } 4785 }
6480 EXPORT_SYMBOL_GPL(follow_pfnmap_start); <<
6481 4786
6482 /** 4787 /**
6483 * follow_pfnmap_end(): End a follow_pfnmap_s !! 4788 * follow_pte - look up PTE at a user virtual address
6484 * @args: Pointer to struct @follow_pfnmap_ar !! 4789 * @mm: the mm_struct of the target address space
>> 4790 * @address: user virtual address
>> 4791 * @ptepp: location to store found PTE
>> 4792 * @ptlp: location to store the lock for the PTE
>> 4793 *
>> 4794 * On a successful return, the pointer to the PTE is stored in @ptepp;
>> 4795 * the corresponding lock is taken and its location is stored in @ptlp.
>> 4796 * The contents of the PTE are only stable until @ptlp is released;
>> 4797 * any further use, if any, must be protected against invalidation
>> 4798 * with MMU notifiers.
6485 * 4799 *
6486 * Must be used in pair of follow_pfnmap_star !! 4800 * Only IO mappings and raw PFN mappings are allowed. The mmap semaphore
6487 * above for more information. !! 4801 * should be taken for read.
>> 4802 *
>> 4803 * KVM uses this function. While it is arguably less bad than ``follow_pfn``,
>> 4804 * it is not a good general-purpose API.
>> 4805 *
>> 4806 * Return: zero on success, -ve otherwise.
6488 */ 4807 */
6489 void follow_pfnmap_end(struct follow_pfnmap_a !! 4808 int follow_pte(struct mm_struct *mm, unsigned long address,
>> 4809 pte_t **ptepp, spinlock_t **ptlp)
6490 { 4810 {
6491 if (args->lock) !! 4811 return follow_invalidate_pte(mm, address, NULL, ptepp, NULL, ptlp);
6492 spin_unlock(args->lock); <<
6493 if (args->ptep) <<
6494 pte_unmap(args->ptep); <<
6495 } 4812 }
6496 EXPORT_SYMBOL_GPL(follow_pfnmap_end); !! 4813 EXPORT_SYMBOL_GPL(follow_pte);
6497 4814
6498 #ifdef CONFIG_HAVE_IOREMAP_PROT <<
6499 /** 4815 /**
6500 * generic_access_phys - generic implementati !! 4816 * follow_pfn - look up PFN at a user virtual address
6501 * @vma: the vma to access !! 4817 * @vma: memory mapping
6502 * @addr: userspace address, not relative off !! 4818 * @address: user virtual address
6503 * @buf: buffer to read/write !! 4819 * @pfn: location to store found PFN
6504 * @len: length of transfer !! 4820 *
6505 * @write: set to FOLL_WRITE when writing, ot !! 4821 * Only IO mappings and raw PFN mappings are allowed.
6506 * !! 4822 *
6507 * This is a generic implementation for &vm_o !! 4823 * This function does not allow the caller to read the permissions
6508 * iomem mapping. This callback is used by ac !! 4824 * of the PTE. Do not use it.
6509 * not page based. !! 4825 *
>> 4826 * Return: zero and the pfn at @pfn on success, -ve otherwise.
6510 */ 4827 */
>> 4828 int follow_pfn(struct vm_area_struct *vma, unsigned long address,
>> 4829 unsigned long *pfn)
>> 4830 {
>> 4831 int ret = -EINVAL;
>> 4832 spinlock_t *ptl;
>> 4833 pte_t *ptep;
>> 4834
>> 4835 if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
>> 4836 return ret;
>> 4837
>> 4838 ret = follow_pte(vma->vm_mm, address, &ptep, &ptl);
>> 4839 if (ret)
>> 4840 return ret;
>> 4841 *pfn = pte_pfn(*ptep);
>> 4842 pte_unmap_unlock(ptep, ptl);
>> 4843 return 0;
>> 4844 }
>> 4845 EXPORT_SYMBOL(follow_pfn);
>> 4846
>> 4847 #ifdef CONFIG_HAVE_IOREMAP_PROT
>> 4848 int follow_phys(struct vm_area_struct *vma,
>> 4849 unsigned long address, unsigned int flags,
>> 4850 unsigned long *prot, resource_size_t *phys)
>> 4851 {
>> 4852 int ret = -EINVAL;
>> 4853 pte_t *ptep, pte;
>> 4854 spinlock_t *ptl;
>> 4855
>> 4856 if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
>> 4857 goto out;
>> 4858
>> 4859 if (follow_pte(vma->vm_mm, address, &ptep, &ptl))
>> 4860 goto out;
>> 4861 pte = *ptep;
>> 4862
>> 4863 if ((flags & FOLL_WRITE) && !pte_write(pte))
>> 4864 goto unlock;
>> 4865
>> 4866 *prot = pgprot_val(pte_pgprot(pte));
>> 4867 *phys = (resource_size_t)pte_pfn(pte) << PAGE_SHIFT;
>> 4868
>> 4869 ret = 0;
>> 4870 unlock:
>> 4871 pte_unmap_unlock(ptep, ptl);
>> 4872 out:
>> 4873 return ret;
>> 4874 }
>> 4875
6511 int generic_access_phys(struct vm_area_struct 4876 int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
6512 void *buf, int len, i 4877 void *buf, int len, int write)
6513 { 4878 {
6514 resource_size_t phys_addr; 4879 resource_size_t phys_addr;
6515 unsigned long prot = 0; 4880 unsigned long prot = 0;
6516 void __iomem *maddr; 4881 void __iomem *maddr;
6517 int offset = offset_in_page(addr); !! 4882 int offset = addr & (PAGE_SIZE-1);
6518 int ret = -EINVAL; <<
6519 bool writable; <<
6520 struct follow_pfnmap_args args = { .v <<
6521 4883
6522 retry: !! 4884 if (follow_phys(vma, addr, write, &prot, &phys_addr))
6523 if (follow_pfnmap_start(&args)) <<
6524 return -EINVAL; <<
6525 prot = pgprot_val(args.pgprot); <<
6526 phys_addr = (resource_size_t)args.pfn <<
6527 writable = args.writable; <<
6528 follow_pfnmap_end(&args); <<
6529 <<
6530 if ((write & FOLL_WRITE) && !writable <<
6531 return -EINVAL; 4885 return -EINVAL;
6532 4886
6533 maddr = ioremap_prot(phys_addr, PAGE_ 4887 maddr = ioremap_prot(phys_addr, PAGE_ALIGN(len + offset), prot);
6534 if (!maddr) 4888 if (!maddr)
6535 return -ENOMEM; 4889 return -ENOMEM;
6536 4890
6537 if (follow_pfnmap_start(&args)) <<
6538 goto out_unmap; <<
6539 <<
6540 if ((prot != pgprot_val(args.pgprot)) <<
6541 (phys_addr != (args.pfn << PAGE_S <<
6542 (writable != args.writable)) { <<
6543 follow_pfnmap_end(&args); <<
6544 iounmap(maddr); <<
6545 goto retry; <<
6546 } <<
6547 <<
6548 if (write) 4891 if (write)
6549 memcpy_toio(maddr + offset, b 4892 memcpy_toio(maddr + offset, buf, len);
6550 else 4893 else
6551 memcpy_fromio(buf, maddr + of 4894 memcpy_fromio(buf, maddr + offset, len);
6552 ret = len; <<
6553 follow_pfnmap_end(&args); <<
6554 out_unmap: <<
6555 iounmap(maddr); 4895 iounmap(maddr);
6556 4896
6557 return ret; !! 4897 return len;
6558 } 4898 }
6559 EXPORT_SYMBOL_GPL(generic_access_phys); 4899 EXPORT_SYMBOL_GPL(generic_access_phys);
6560 #endif 4900 #endif
6561 4901
6562 /* 4902 /*
6563 * Access another process' address space as g 4903 * Access another process' address space as given in mm.
6564 */ 4904 */
6565 static int __access_remote_vm(struct mm_struc !! 4905 int __access_remote_vm(struct mm_struct *mm, unsigned long addr, void *buf,
6566 void *buf, int !! 4906 int len, unsigned int gup_flags)
6567 { 4907 {
>> 4908 struct vm_area_struct *vma;
6568 void *old_buf = buf; 4909 void *old_buf = buf;
6569 int write = gup_flags & FOLL_WRITE; 4910 int write = gup_flags & FOLL_WRITE;
6570 4911
6571 if (mmap_read_lock_killable(mm)) 4912 if (mmap_read_lock_killable(mm))
6572 return 0; 4913 return 0;
6573 4914
6574 /* Untag the address before looking u <<
6575 addr = untagged_addr_remote(mm, addr) <<
6576 <<
6577 /* Avoid triggering the temporary war <<
6578 if (!vma_lookup(mm, addr) && !expand_ <<
6579 return 0; <<
6580 <<
6581 /* ignore errors, just check how much 4915 /* ignore errors, just check how much was successfully transferred */
6582 while (len) { 4916 while (len) {
6583 int bytes, offset; !! 4917 int bytes, ret, offset;
6584 void *maddr; 4918 void *maddr;
6585 struct vm_area_struct *vma = !! 4919 struct page *page = NULL;
6586 struct page *page = get_user_ <<
6587 <<
6588 <<
6589 if (IS_ERR(page)) { <<
6590 /* We might need to e <<
6591 vma = vma_lookup(mm, <<
6592 if (!vma) { <<
6593 vma = expand_ <<
6594 <<
6595 /* mmap_lock <<
6596 if (!vma) <<
6597 retur <<
6598 <<
6599 /* Try again <<
6600 continue; <<
6601 } <<
6602 4920
>> 4921 ret = get_user_pages_remote(mm, addr, 1,
>> 4922 gup_flags, &page, &vma, NULL);
>> 4923 if (ret <= 0) {
>> 4924 #ifndef CONFIG_HAVE_IOREMAP_PROT
>> 4925 break;
>> 4926 #else
6603 /* 4927 /*
6604 * Check if this is a 4928 * Check if this is a VM_IO | VM_PFNMAP VMA, which
6605 * we can access usin 4929 * we can access using slightly different code.
6606 */ 4930 */
6607 bytes = 0; !! 4931 vma = find_vma(mm, addr);
6608 #ifdef CONFIG_HAVE_IOREMAP_PROT !! 4932 if (!vma || vma->vm_start > addr)
>> 4933 break;
6609 if (vma->vm_ops && vm 4934 if (vma->vm_ops && vma->vm_ops->access)
6610 bytes = vma-> !! 4935 ret = vma->vm_ops->access(vma, addr, buf,
6611 !! 4936 len, write);
6612 #endif !! 4937 if (ret <= 0)
6613 if (bytes <= 0) <<
6614 break; 4938 break;
>> 4939 bytes = ret;
>> 4940 #endif
6615 } else { 4941 } else {
6616 bytes = len; 4942 bytes = len;
6617 offset = addr & (PAGE 4943 offset = addr & (PAGE_SIZE-1);
6618 if (bytes > PAGE_SIZE 4944 if (bytes > PAGE_SIZE-offset)
6619 bytes = PAGE_ 4945 bytes = PAGE_SIZE-offset;
6620 4946
6621 maddr = kmap_local_pa !! 4947 maddr = kmap(page);
6622 if (write) { 4948 if (write) {
6623 copy_to_user_ 4949 copy_to_user_page(vma, page, addr,
6624 4950 maddr + offset, buf, bytes);
6625 set_page_dirt 4951 set_page_dirty_lock(page);
6626 } else { 4952 } else {
6627 copy_from_use 4953 copy_from_user_page(vma, page, addr,
6628 4954 buf, maddr + offset, bytes);
6629 } 4955 }
6630 unmap_and_put_page(pa !! 4956 kunmap(page);
>> 4957 put_page(page);
6631 } 4958 }
6632 len -= bytes; 4959 len -= bytes;
6633 buf += bytes; 4960 buf += bytes;
6634 addr += bytes; 4961 addr += bytes;
6635 } 4962 }
6636 mmap_read_unlock(mm); 4963 mmap_read_unlock(mm);
6637 4964
6638 return buf - old_buf; 4965 return buf - old_buf;
6639 } 4966 }
6640 4967
6641 /** 4968 /**
6642 * access_remote_vm - access another process' 4969 * access_remote_vm - access another process' address space
6643 * @mm: the mm_struct of the target a 4970 * @mm: the mm_struct of the target address space
6644 * @addr: start address to access 4971 * @addr: start address to access
6645 * @buf: source or destination buffer 4972 * @buf: source or destination buffer
6646 * @len: number of bytes to transfer 4973 * @len: number of bytes to transfer
6647 * @gup_flags: flags modifying lookup behavi 4974 * @gup_flags: flags modifying lookup behaviour
6648 * 4975 *
6649 * The caller must hold a reference on @mm. 4976 * The caller must hold a reference on @mm.
6650 * 4977 *
6651 * Return: number of bytes copied from source 4978 * Return: number of bytes copied from source to destination.
6652 */ 4979 */
6653 int access_remote_vm(struct mm_struct *mm, un 4980 int access_remote_vm(struct mm_struct *mm, unsigned long addr,
6654 void *buf, int len, unsigned 4981 void *buf, int len, unsigned int gup_flags)
6655 { 4982 {
6656 return __access_remote_vm(mm, addr, b 4983 return __access_remote_vm(mm, addr, buf, len, gup_flags);
6657 } 4984 }
6658 4985
6659 /* 4986 /*
6660 * Access another process' address space. 4987 * Access another process' address space.
6661 * Source/target buffer must be kernel space, 4988 * Source/target buffer must be kernel space,
6662 * Do not walk the page table directly, use g 4989 * Do not walk the page table directly, use get_user_pages
6663 */ 4990 */
6664 int access_process_vm(struct task_struct *tsk 4991 int access_process_vm(struct task_struct *tsk, unsigned long addr,
6665 void *buf, int len, unsigned 4992 void *buf, int len, unsigned int gup_flags)
6666 { 4993 {
6667 struct mm_struct *mm; 4994 struct mm_struct *mm;
6668 int ret; 4995 int ret;
6669 4996
6670 mm = get_task_mm(tsk); 4997 mm = get_task_mm(tsk);
6671 if (!mm) 4998 if (!mm)
6672 return 0; 4999 return 0;
6673 5000
6674 ret = __access_remote_vm(mm, addr, bu 5001 ret = __access_remote_vm(mm, addr, buf, len, gup_flags);
6675 5002
6676 mmput(mm); 5003 mmput(mm);
6677 5004
6678 return ret; 5005 return ret;
6679 } 5006 }
6680 EXPORT_SYMBOL_GPL(access_process_vm); 5007 EXPORT_SYMBOL_GPL(access_process_vm);
6681 5008
6682 /* 5009 /*
6683 * Print the name of a VMA. 5010 * Print the name of a VMA.
6684 */ 5011 */
6685 void print_vma_addr(char *prefix, unsigned lo 5012 void print_vma_addr(char *prefix, unsigned long ip)
6686 { 5013 {
6687 struct mm_struct *mm = current->mm; 5014 struct mm_struct *mm = current->mm;
6688 struct vm_area_struct *vma; 5015 struct vm_area_struct *vma;
6689 5016
6690 /* 5017 /*
6691 * we might be running from an atomic 5018 * we might be running from an atomic context so we cannot sleep
6692 */ 5019 */
6693 if (!mmap_read_trylock(mm)) 5020 if (!mmap_read_trylock(mm))
6694 return; 5021 return;
6695 5022
6696 vma = vma_lookup(mm, ip); !! 5023 vma = find_vma(mm, ip);
6697 if (vma && vma->vm_file) { 5024 if (vma && vma->vm_file) {
6698 struct file *f = vma->vm_file 5025 struct file *f = vma->vm_file;
6699 ip -= vma->vm_start; !! 5026 char *buf = (char *)__get_free_page(GFP_NOWAIT);
6700 ip += vma->vm_pgoff << PAGE_S !! 5027 if (buf) {
6701 printk("%s%pD[%lx,%lx+%lx]", !! 5028 char *p;
6702 vma->vm_start !! 5029
6703 vma->vm_end - !! 5030 p = file_path(f, buf, PAGE_SIZE);
>> 5031 if (IS_ERR(p))
>> 5032 p = "?";
>> 5033 printk("%s%s[%lx+%lx]", prefix, kbasename(p),
>> 5034 vma->vm_start,
>> 5035 vma->vm_end - vma->vm_start);
>> 5036 free_page((unsigned long)buf);
>> 5037 }
6704 } 5038 }
6705 mmap_read_unlock(mm); 5039 mmap_read_unlock(mm);
6706 } 5040 }
6707 5041
6708 #if defined(CONFIG_PROVE_LOCKING) || defined( 5042 #if defined(CONFIG_PROVE_LOCKING) || defined(CONFIG_DEBUG_ATOMIC_SLEEP)
6709 void __might_fault(const char *file, int line 5043 void __might_fault(const char *file, int line)
6710 { 5044 {
>> 5045 /*
>> 5046 * Some code (nfs/sunrpc) uses socket ops on kernel memory while
>> 5047 * holding the mmap_lock, this is safe because kernel memory doesn't
>> 5048 * get paged out, therefore we'll never actually fault, and the
>> 5049 * below annotations will generate false positives.
>> 5050 */
>> 5051 if (uaccess_kernel())
>> 5052 return;
6711 if (pagefault_disabled()) 5053 if (pagefault_disabled())
6712 return; 5054 return;
6713 __might_sleep(file, line); !! 5055 __might_sleep(file, line, 0);
6714 #if defined(CONFIG_DEBUG_ATOMIC_SLEEP) 5056 #if defined(CONFIG_DEBUG_ATOMIC_SLEEP)
6715 if (current->mm) 5057 if (current->mm)
6716 might_lock_read(¤t->mm- 5058 might_lock_read(¤t->mm->mmap_lock);
6717 #endif 5059 #endif
6718 } 5060 }
6719 EXPORT_SYMBOL(__might_fault); 5061 EXPORT_SYMBOL(__might_fault);
6720 #endif 5062 #endif
6721 5063
6722 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || d 5064 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS)
6723 /* 5065 /*
6724 * Process all subpages of the specified huge 5066 * Process all subpages of the specified huge page with the specified
6725 * operation. The target subpage will be pro 5067 * operation. The target subpage will be processed last to keep its
6726 * cache lines hot. 5068 * cache lines hot.
6727 */ 5069 */
6728 static inline int process_huge_page( !! 5070 static inline void process_huge_page(
6729 unsigned long addr_hint, unsigned int !! 5071 unsigned long addr_hint, unsigned int pages_per_huge_page,
6730 int (*process_subpage)(unsigned long !! 5072 void (*process_subpage)(unsigned long addr, int idx, void *arg),
6731 void *arg) 5073 void *arg)
6732 { 5074 {
6733 int i, n, base, l, ret; !! 5075 int i, n, base, l;
6734 unsigned long addr = addr_hint & 5076 unsigned long addr = addr_hint &
6735 ~(((unsigned long)nr_pages << !! 5077 ~(((unsigned long)pages_per_huge_page << PAGE_SHIFT) - 1);
6736 5078
6737 /* Process target subpage last to kee 5079 /* Process target subpage last to keep its cache lines hot */
6738 might_sleep(); 5080 might_sleep();
6739 n = (addr_hint - addr) / PAGE_SIZE; 5081 n = (addr_hint - addr) / PAGE_SIZE;
6740 if (2 * n <= nr_pages) { !! 5082 if (2 * n <= pages_per_huge_page) {
6741 /* If target subpage in first 5083 /* If target subpage in first half of huge page */
6742 base = 0; 5084 base = 0;
6743 l = n; 5085 l = n;
6744 /* Process subpages at the en 5086 /* Process subpages at the end of huge page */
6745 for (i = nr_pages - 1; i >= 2 !! 5087 for (i = pages_per_huge_page - 1; i >= 2 * n; i--) {
6746 cond_resched(); 5088 cond_resched();
6747 ret = process_subpage !! 5089 process_subpage(addr + i * PAGE_SIZE, i, arg);
6748 if (ret) <<
6749 return ret; <<
6750 } 5090 }
6751 } else { 5091 } else {
6752 /* If target subpage in secon 5092 /* If target subpage in second half of huge page */
6753 base = nr_pages - 2 * (nr_pag !! 5093 base = pages_per_huge_page - 2 * (pages_per_huge_page - n);
6754 l = nr_pages - n; !! 5094 l = pages_per_huge_page - n;
6755 /* Process subpages at the be 5095 /* Process subpages at the begin of huge page */
6756 for (i = 0; i < base; i++) { 5096 for (i = 0; i < base; i++) {
6757 cond_resched(); 5097 cond_resched();
6758 ret = process_subpage !! 5098 process_subpage(addr + i * PAGE_SIZE, i, arg);
6759 if (ret) <<
6760 return ret; <<
6761 } 5099 }
6762 } 5100 }
6763 /* 5101 /*
6764 * Process remaining subpages in left 5102 * Process remaining subpages in left-right-left-right pattern
6765 * towards the target subpage 5103 * towards the target subpage
6766 */ 5104 */
6767 for (i = 0; i < l; i++) { 5105 for (i = 0; i < l; i++) {
6768 int left_idx = base + i; 5106 int left_idx = base + i;
6769 int right_idx = base + 2 * l 5107 int right_idx = base + 2 * l - 1 - i;
6770 5108
6771 cond_resched(); 5109 cond_resched();
6772 ret = process_subpage(addr + !! 5110 process_subpage(addr + left_idx * PAGE_SIZE, left_idx, arg);
6773 if (ret) <<
6774 return ret; <<
6775 cond_resched(); 5111 cond_resched();
6776 ret = process_subpage(addr + !! 5112 process_subpage(addr + right_idx * PAGE_SIZE, right_idx, arg);
6777 if (ret) <<
6778 return ret; <<
6779 } 5113 }
6780 return 0; <<
6781 } 5114 }
6782 5115
6783 static void clear_gigantic_page(struct folio !! 5116 static void clear_gigantic_page(struct page *page,
6784 unsigned int !! 5117 unsigned long addr,
>> 5118 unsigned int pages_per_huge_page)
6785 { 5119 {
6786 int i; 5120 int i;
>> 5121 struct page *p = page;
6787 5122
6788 might_sleep(); 5123 might_sleep();
6789 for (i = 0; i < nr_pages; i++) { !! 5124 for (i = 0; i < pages_per_huge_page;
>> 5125 i++, p = mem_map_next(p, page, i)) {
6790 cond_resched(); 5126 cond_resched();
6791 clear_user_highpage(folio_pag !! 5127 clear_user_highpage(p, addr + i * PAGE_SIZE);
6792 } 5128 }
6793 } 5129 }
6794 5130
6795 static int clear_subpage(unsigned long addr, !! 5131 static void clear_subpage(unsigned long addr, int idx, void *arg)
6796 { 5132 {
6797 struct folio *folio = arg; !! 5133 struct page *page = arg;
6798 5134
6799 clear_user_highpage(folio_page(folio, !! 5135 clear_user_highpage(page + idx, addr);
6800 return 0; <<
6801 } 5136 }
6802 5137
6803 /** !! 5138 void clear_huge_page(struct page *page,
6804 * folio_zero_user - Zero a folio which will !! 5139 unsigned long addr_hint, unsigned int pages_per_huge_page)
6805 * @folio: The folio to zero. <<
6806 * @addr_hint: The address will be accessed o <<
6807 */ <<
6808 void folio_zero_user(struct folio *folio, uns <<
6809 { 5140 {
6810 unsigned int nr_pages = folio_nr_page !! 5141 unsigned long addr = addr_hint &
>> 5142 ~(((unsigned long)pages_per_huge_page << PAGE_SHIFT) - 1);
6811 5143
6812 if (unlikely(nr_pages > MAX_ORDER_NR_ !! 5144 if (unlikely(pages_per_huge_page > MAX_ORDER_NR_PAGES)) {
6813 clear_gigantic_page(folio, ad !! 5145 clear_gigantic_page(page, addr, pages_per_huge_page);
6814 else !! 5146 return;
6815 process_huge_page(addr_hint, !! 5147 }
>> 5148
>> 5149 process_huge_page(addr_hint, pages_per_huge_page, clear_subpage, page);
6816 } 5150 }
6817 5151
6818 static int copy_user_gigantic_page(struct fol !! 5152 static void copy_user_gigantic_page(struct page *dst, struct page *src,
6819 unsigned l !! 5153 unsigned long addr,
6820 struct vm_ !! 5154 struct vm_area_struct *vma,
6821 unsigned i !! 5155 unsigned int pages_per_huge_page)
6822 { 5156 {
6823 int i; 5157 int i;
6824 struct page *dst_page; !! 5158 struct page *dst_base = dst;
6825 struct page *src_page; !! 5159 struct page *src_base = src;
6826 <<
6827 for (i = 0; i < nr_pages; i++) { <<
6828 dst_page = folio_page(dst, i) <<
6829 src_page = folio_page(src, i) <<
6830 5160
>> 5161 for (i = 0; i < pages_per_huge_page; ) {
6831 cond_resched(); 5162 cond_resched();
6832 if (copy_mc_user_highpage(dst !! 5163 copy_user_highpage(dst, src, addr + i*PAGE_SIZE, vma);
6833 add !! 5164
6834 return -EHWPOISON; !! 5165 i++;
>> 5166 dst = mem_map_next(dst, dst_base, i);
>> 5167 src = mem_map_next(src, src_base, i);
6835 } 5168 }
6836 return 0; <<
6837 } 5169 }
6838 5170
6839 struct copy_subpage_arg { 5171 struct copy_subpage_arg {
6840 struct folio *dst; !! 5172 struct page *dst;
6841 struct folio *src; !! 5173 struct page *src;
6842 struct vm_area_struct *vma; 5174 struct vm_area_struct *vma;
6843 }; 5175 };
6844 5176
6845 static int copy_subpage(unsigned long addr, i !! 5177 static void copy_subpage(unsigned long addr, int idx, void *arg)
6846 { 5178 {
6847 struct copy_subpage_arg *copy_arg = a 5179 struct copy_subpage_arg *copy_arg = arg;
6848 struct page *dst = folio_page(copy_ar <<
6849 struct page *src = folio_page(copy_ar <<
6850 5180
6851 if (copy_mc_user_highpage(dst, src, a !! 5181 copy_user_highpage(copy_arg->dst + idx, copy_arg->src + idx,
6852 return -EHWPOISON; !! 5182 addr, copy_arg->vma);
6853 return 0; <<
6854 } 5183 }
6855 5184
6856 int copy_user_large_folio(struct folio *dst, !! 5185 void copy_user_huge_page(struct page *dst, struct page *src,
6857 unsigned long addr_ !! 5186 unsigned long addr_hint, struct vm_area_struct *vma,
>> 5187 unsigned int pages_per_huge_page)
6858 { 5188 {
6859 unsigned int nr_pages = folio_nr_page !! 5189 unsigned long addr = addr_hint &
>> 5190 ~(((unsigned long)pages_per_huge_page << PAGE_SHIFT) - 1);
6860 struct copy_subpage_arg arg = { 5191 struct copy_subpage_arg arg = {
6861 .dst = dst, 5192 .dst = dst,
6862 .src = src, 5193 .src = src,
6863 .vma = vma, 5194 .vma = vma,
6864 }; 5195 };
6865 5196
6866 if (unlikely(nr_pages > MAX_ORDER_NR_ !! 5197 if (unlikely(pages_per_huge_page > MAX_ORDER_NR_PAGES)) {
6867 return copy_user_gigantic_pag !! 5198 copy_user_gigantic_page(dst, src, addr, vma,
>> 5199 pages_per_huge_page);
>> 5200 return;
>> 5201 }
6868 5202
6869 return process_huge_page(addr_hint, n !! 5203 process_huge_page(addr_hint, pages_per_huge_page, copy_subpage, &arg);
6870 } 5204 }
6871 5205
6872 long copy_folio_from_user(struct folio *dst_f !! 5206 long copy_huge_page_from_user(struct page *dst_page,
6873 const void __user !! 5207 const void __user *usr_src,
6874 bool allow_pagefau !! 5208 unsigned int pages_per_huge_page,
>> 5209 bool allow_pagefault)
6875 { 5210 {
6876 void *kaddr; !! 5211 void *src = (void *)usr_src;
>> 5212 void *page_kaddr;
6877 unsigned long i, rc = 0; 5213 unsigned long i, rc = 0;
6878 unsigned int nr_pages = folio_nr_page !! 5214 unsigned long ret_val = pages_per_huge_page * PAGE_SIZE;
6879 unsigned long ret_val = nr_pages * PA !! 5215 struct page *subpage = dst_page;
6880 struct page *subpage; !! 5216
6881 !! 5217 for (i = 0; i < pages_per_huge_page;
6882 for (i = 0; i < nr_pages; i++) { !! 5218 i++, subpage = mem_map_next(subpage, dst_page, i)) {
6883 subpage = folio_page(dst_foli !! 5219 if (allow_pagefault)
6884 kaddr = kmap_local_page(subpa !! 5220 page_kaddr = kmap(subpage);
6885 if (!allow_pagefault) !! 5221 else
6886 pagefault_disable(); !! 5222 page_kaddr = kmap_atomic(subpage);
6887 rc = copy_from_user(kaddr, us !! 5223 rc = copy_from_user(page_kaddr,
6888 if (!allow_pagefault) !! 5224 (const void __user *)(src + i * PAGE_SIZE),
6889 pagefault_enable(); !! 5225 PAGE_SIZE);
6890 kunmap_local(kaddr); !! 5226 if (allow_pagefault)
>> 5227 kunmap(subpage);
>> 5228 else
>> 5229 kunmap_atomic(page_kaddr);
6891 5230
6892 ret_val -= (PAGE_SIZE - rc); 5231 ret_val -= (PAGE_SIZE - rc);
6893 if (rc) 5232 if (rc)
6894 break; 5233 break;
6895 5234
6896 flush_dcache_page(subpage); <<
6897 <<
6898 cond_resched(); 5235 cond_resched();
6899 } 5236 }
6900 return ret_val; 5237 return ret_val;
6901 } 5238 }
6902 #endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONF 5239 #endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */
6903 5240
6904 #if defined(CONFIG_SPLIT_PTE_PTLOCKS) && ALLO !! 5241 #if USE_SPLIT_PTE_PTLOCKS && ALLOC_SPLIT_PTLOCKS
6905 5242
6906 static struct kmem_cache *page_ptl_cachep; 5243 static struct kmem_cache *page_ptl_cachep;
6907 5244
6908 void __init ptlock_cache_init(void) 5245 void __init ptlock_cache_init(void)
6909 { 5246 {
6910 page_ptl_cachep = kmem_cache_create(" 5247 page_ptl_cachep = kmem_cache_create("page->ptl", sizeof(spinlock_t), 0,
6911 SLAB_PANIC, NULL); 5248 SLAB_PANIC, NULL);
6912 } 5249 }
6913 5250
6914 bool ptlock_alloc(struct ptdesc *ptdesc) !! 5251 bool ptlock_alloc(struct page *page)
6915 { 5252 {
6916 spinlock_t *ptl; 5253 spinlock_t *ptl;
6917 5254
6918 ptl = kmem_cache_alloc(page_ptl_cache 5255 ptl = kmem_cache_alloc(page_ptl_cachep, GFP_KERNEL);
6919 if (!ptl) 5256 if (!ptl)
6920 return false; 5257 return false;
6921 ptdesc->ptl = ptl; !! 5258 page->ptl = ptl;
6922 return true; 5259 return true;
6923 } 5260 }
6924 5261
6925 void ptlock_free(struct ptdesc *ptdesc) !! 5262 void ptlock_free(struct page *page)
6926 { 5263 {
6927 kmem_cache_free(page_ptl_cachep, ptde !! 5264 kmem_cache_free(page_ptl_cachep, page->ptl);
6928 } 5265 }
6929 #endif 5266 #endif
6930 <<
6931 void vma_pgtable_walk_begin(struct vm_area_st <<
6932 { <<
6933 if (is_vm_hugetlb_page(vma)) <<
6934 hugetlb_vma_lock_read(vma); <<
6935 } <<
6936 <<
6937 void vma_pgtable_walk_end(struct vm_area_stru <<
6938 { <<
6939 if (is_vm_hugetlb_page(vma)) <<
6940 hugetlb_vma_unlock_read(vma); <<
6941 } <<
6942 5267
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.