1 /* 1 /*
2 * mm/rmap.c - physical to virtual reverse map 2 * mm/rmap.c - physical to virtual reverse mappings
3 * 3 *
4 * Copyright 2001, Rik van Riel <riel@conectiv 4 * Copyright 2001, Rik van Riel <riel@conectiva.com.br>
5 * Released under the General Public License ( 5 * Released under the General Public License (GPL).
6 * 6 *
7 * Simple, low overhead reverse mapping scheme 7 * Simple, low overhead reverse mapping scheme.
8 * Please try to keep this thing as modular as 8 * Please try to keep this thing as modular as possible.
9 * 9 *
10 * Provides methods for unmapping each kind of 10 * Provides methods for unmapping each kind of mapped page:
11 * the anon methods track anonymous pages, and 11 * the anon methods track anonymous pages, and
12 * the file methods track pages belonging to a 12 * the file methods track pages belonging to an inode.
13 * 13 *
14 * Original design by Rik van Riel <riel@conec 14 * Original design by Rik van Riel <riel@conectiva.com.br> 2001
15 * File methods by Dave McCracken <dmccr@us.ib 15 * File methods by Dave McCracken <dmccr@us.ibm.com> 2003, 2004
16 * Anonymous methods by Andrea Arcangeli <andr 16 * Anonymous methods by Andrea Arcangeli <andrea@suse.de> 2004
17 * Contributions by Hugh Dickins 2003, 2004 17 * Contributions by Hugh Dickins 2003, 2004
18 */ 18 */
19 19
20 /* 20 /*
21 * Lock ordering in mm: 21 * Lock ordering in mm:
22 * 22 *
23 * inode->i_rwsem (while writing or trun 23 * inode->i_rwsem (while writing or truncating, not reading or faulting)
24 * mm->mmap_lock 24 * mm->mmap_lock
25 * mapping->invalidate_lock (in filemap_fa 25 * mapping->invalidate_lock (in filemap_fault)
26 * folio_lock 26 * folio_lock
27 * hugetlbfs_i_mmap_rwsem_key (in huge 27 * hugetlbfs_i_mmap_rwsem_key (in huge_pmd_share, see hugetlbfs below)
28 * vma_start_write 28 * vma_start_write
29 * mapping->i_mmap_rwsem 29 * mapping->i_mmap_rwsem
30 * anon_vma->rwsem 30 * anon_vma->rwsem
31 * mm->page_table_lock or pte_ 31 * mm->page_table_lock or pte_lock
32 * swap_lock (in swap_duplic 32 * swap_lock (in swap_duplicate, swap_info_get)
33 * mmlist_lock (in mmput, 33 * mmlist_lock (in mmput, drain_mmlist and others)
34 * mapping->private_lock ( 34 * mapping->private_lock (in block_dirty_folio)
35 * folio_lock_memcg move 35 * folio_lock_memcg move_lock (in block_dirty_folio)
36 * i_pages lock (widel 36 * i_pages lock (widely used)
37 * lruvec->lru_lock 37 * lruvec->lru_lock (in folio_lruvec_lock_irq)
38 * inode->i_lock (in set_p 38 * inode->i_lock (in set_page_dirty's __mark_inode_dirty)
39 * bdi.wb->list_lock (in s 39 * bdi.wb->list_lock (in set_page_dirty's __mark_inode_dirty)
40 * sb_lock (within inode 40 * sb_lock (within inode_lock in fs/fs-writeback.c)
41 * i_pages lock (widely 41 * i_pages lock (widely used, in set_page_dirty,
42 * in arch-dep 42 * in arch-dependent flush_dcache_mmap_lock,
43 * within bdi. 43 * within bdi.wb->list_lock in __sync_single_inode)
44 * 44 *
45 * anon_vma->rwsem,mapping->i_mmap_rwsem (me 45 * anon_vma->rwsem,mapping->i_mmap_rwsem (memory_failure, collect_procs_anon)
46 * ->tasklist_lock 46 * ->tasklist_lock
47 * pte map lock 47 * pte map lock
48 * 48 *
49 * hugetlbfs PageHuge() take locks in this ord 49 * hugetlbfs PageHuge() take locks in this order:
50 * hugetlb_fault_mutex (hugetlbfs specific p 50 * hugetlb_fault_mutex (hugetlbfs specific page fault mutex)
51 * vma_lock (hugetlb specific lock for pmd 51 * vma_lock (hugetlb specific lock for pmd_sharing)
52 * mapping->i_mmap_rwsem (also used for 52 * mapping->i_mmap_rwsem (also used for hugetlb pmd sharing)
53 * folio_lock 53 * folio_lock
54 */ 54 */
55 55
56 #include <linux/mm.h> 56 #include <linux/mm.h>
57 #include <linux/sched/mm.h> 57 #include <linux/sched/mm.h>
58 #include <linux/sched/task.h> 58 #include <linux/sched/task.h>
59 #include <linux/pagemap.h> 59 #include <linux/pagemap.h>
60 #include <linux/swap.h> 60 #include <linux/swap.h>
61 #include <linux/swapops.h> 61 #include <linux/swapops.h>
62 #include <linux/slab.h> 62 #include <linux/slab.h>
63 #include <linux/init.h> 63 #include <linux/init.h>
64 #include <linux/ksm.h> 64 #include <linux/ksm.h>
65 #include <linux/rmap.h> 65 #include <linux/rmap.h>
66 #include <linux/rcupdate.h> 66 #include <linux/rcupdate.h>
67 #include <linux/export.h> 67 #include <linux/export.h>
68 #include <linux/memcontrol.h> 68 #include <linux/memcontrol.h>
69 #include <linux/mmu_notifier.h> 69 #include <linux/mmu_notifier.h>
70 #include <linux/migrate.h> 70 #include <linux/migrate.h>
71 #include <linux/hugetlb.h> 71 #include <linux/hugetlb.h>
72 #include <linux/huge_mm.h> 72 #include <linux/huge_mm.h>
73 #include <linux/backing-dev.h> 73 #include <linux/backing-dev.h>
74 #include <linux/page_idle.h> 74 #include <linux/page_idle.h>
75 #include <linux/memremap.h> 75 #include <linux/memremap.h>
76 #include <linux/userfaultfd_k.h> 76 #include <linux/userfaultfd_k.h>
77 #include <linux/mm_inline.h> 77 #include <linux/mm_inline.h>
78 #include <linux/oom.h> 78 #include <linux/oom.h>
79 79
80 #include <asm/tlbflush.h> 80 #include <asm/tlbflush.h>
81 81
82 #define CREATE_TRACE_POINTS 82 #define CREATE_TRACE_POINTS
83 #include <trace/events/tlb.h> 83 #include <trace/events/tlb.h>
84 #include <trace/events/migrate.h> 84 #include <trace/events/migrate.h>
85 85
86 #include "internal.h" 86 #include "internal.h"
87 87
88 static struct kmem_cache *anon_vma_cachep; 88 static struct kmem_cache *anon_vma_cachep;
89 static struct kmem_cache *anon_vma_chain_cache 89 static struct kmem_cache *anon_vma_chain_cachep;
90 90
91 static inline struct anon_vma *anon_vma_alloc( 91 static inline struct anon_vma *anon_vma_alloc(void)
92 { 92 {
93 struct anon_vma *anon_vma; 93 struct anon_vma *anon_vma;
94 94
95 anon_vma = kmem_cache_alloc(anon_vma_c 95 anon_vma = kmem_cache_alloc(anon_vma_cachep, GFP_KERNEL);
96 if (anon_vma) { 96 if (anon_vma) {
97 atomic_set(&anon_vma->refcount 97 atomic_set(&anon_vma->refcount, 1);
98 anon_vma->num_children = 0; 98 anon_vma->num_children = 0;
99 anon_vma->num_active_vmas = 0; 99 anon_vma->num_active_vmas = 0;
100 anon_vma->parent = anon_vma; 100 anon_vma->parent = anon_vma;
101 /* 101 /*
102 * Initialise the anon_vma roo 102 * Initialise the anon_vma root to point to itself. If called
103 * from fork, the root will be 103 * from fork, the root will be reset to the parents anon_vma.
104 */ 104 */
105 anon_vma->root = anon_vma; 105 anon_vma->root = anon_vma;
106 } 106 }
107 107
108 return anon_vma; 108 return anon_vma;
109 } 109 }
110 110
111 static inline void anon_vma_free(struct anon_v 111 static inline void anon_vma_free(struct anon_vma *anon_vma)
112 { 112 {
113 VM_BUG_ON(atomic_read(&anon_vma->refco 113 VM_BUG_ON(atomic_read(&anon_vma->refcount));
114 114
115 /* 115 /*
116 * Synchronize against folio_lock_anon 116 * Synchronize against folio_lock_anon_vma_read() such that
117 * we can safely hold the lock without 117 * we can safely hold the lock without the anon_vma getting
118 * freed. 118 * freed.
119 * 119 *
120 * Relies on the full mb implied by th 120 * Relies on the full mb implied by the atomic_dec_and_test() from
121 * put_anon_vma() against the acquire 121 * put_anon_vma() against the acquire barrier implied by
122 * down_read_trylock() from folio_lock 122 * down_read_trylock() from folio_lock_anon_vma_read(). This orders:
123 * 123 *
124 * folio_lock_anon_vma_read() VS 124 * folio_lock_anon_vma_read() VS put_anon_vma()
125 * down_read_trylock() 125 * down_read_trylock() atomic_dec_and_test()
126 * LOCK 126 * LOCK MB
127 * atomic_read() 127 * atomic_read() rwsem_is_locked()
128 * 128 *
129 * LOCK should suffice since the actua 129 * LOCK should suffice since the actual taking of the lock must
130 * happen _before_ what follows. 130 * happen _before_ what follows.
131 */ 131 */
132 might_sleep(); 132 might_sleep();
133 if (rwsem_is_locked(&anon_vma->root->r 133 if (rwsem_is_locked(&anon_vma->root->rwsem)) {
134 anon_vma_lock_write(anon_vma); 134 anon_vma_lock_write(anon_vma);
135 anon_vma_unlock_write(anon_vma 135 anon_vma_unlock_write(anon_vma);
136 } 136 }
137 137
138 kmem_cache_free(anon_vma_cachep, anon_ 138 kmem_cache_free(anon_vma_cachep, anon_vma);
139 } 139 }
140 140
141 static inline struct anon_vma_chain *anon_vma_ 141 static inline struct anon_vma_chain *anon_vma_chain_alloc(gfp_t gfp)
142 { 142 {
143 return kmem_cache_alloc(anon_vma_chain 143 return kmem_cache_alloc(anon_vma_chain_cachep, gfp);
144 } 144 }
145 145
146 static void anon_vma_chain_free(struct anon_vm 146 static void anon_vma_chain_free(struct anon_vma_chain *anon_vma_chain)
147 { 147 {
148 kmem_cache_free(anon_vma_chain_cachep, 148 kmem_cache_free(anon_vma_chain_cachep, anon_vma_chain);
149 } 149 }
150 150
151 static void anon_vma_chain_link(struct vm_area 151 static void anon_vma_chain_link(struct vm_area_struct *vma,
152 struct anon_vm 152 struct anon_vma_chain *avc,
153 struct anon_vm 153 struct anon_vma *anon_vma)
154 { 154 {
155 avc->vma = vma; 155 avc->vma = vma;
156 avc->anon_vma = anon_vma; 156 avc->anon_vma = anon_vma;
157 list_add(&avc->same_vma, &vma->anon_vm 157 list_add(&avc->same_vma, &vma->anon_vma_chain);
158 anon_vma_interval_tree_insert(avc, &an 158 anon_vma_interval_tree_insert(avc, &anon_vma->rb_root);
159 } 159 }
160 160
161 /** 161 /**
162 * __anon_vma_prepare - attach an anon_vma to 162 * __anon_vma_prepare - attach an anon_vma to a memory region
163 * @vma: the memory region in question 163 * @vma: the memory region in question
164 * 164 *
165 * This makes sure the memory mapping describe 165 * This makes sure the memory mapping described by 'vma' has
166 * an 'anon_vma' attached to it, so that we ca 166 * an 'anon_vma' attached to it, so that we can associate the
167 * anonymous pages mapped into it with that an 167 * anonymous pages mapped into it with that anon_vma.
168 * 168 *
169 * The common case will be that we already hav 169 * The common case will be that we already have one, which
170 * is handled inline by anon_vma_prepare(). Bu 170 * is handled inline by anon_vma_prepare(). But if
171 * not we either need to find an adjacent mapp 171 * not we either need to find an adjacent mapping that we
172 * can re-use the anon_vma from (very common w 172 * can re-use the anon_vma from (very common when the only
173 * reason for splitting a vma has been mprotec 173 * reason for splitting a vma has been mprotect()), or we
174 * allocate a new one. 174 * allocate a new one.
175 * 175 *
176 * Anon-vma allocations are very subtle, becau 176 * Anon-vma allocations are very subtle, because we may have
177 * optimistically looked up an anon_vma in fol 177 * optimistically looked up an anon_vma in folio_lock_anon_vma_read()
178 * and that may actually touch the rwsem even 178 * and that may actually touch the rwsem even in the newly
179 * allocated vma (it depends on RCU to make su 179 * allocated vma (it depends on RCU to make sure that the
180 * anon_vma isn't actually destroyed). 180 * anon_vma isn't actually destroyed).
181 * 181 *
182 * As a result, we need to do proper anon_vma 182 * As a result, we need to do proper anon_vma locking even
183 * for the new allocation. At the same time, w 183 * for the new allocation. At the same time, we do not want
184 * to do any locking for the common case of al 184 * to do any locking for the common case of already having
185 * an anon_vma. 185 * an anon_vma.
186 */ 186 */
187 int __anon_vma_prepare(struct vm_area_struct * 187 int __anon_vma_prepare(struct vm_area_struct *vma)
188 { 188 {
189 struct mm_struct *mm = vma->vm_mm; 189 struct mm_struct *mm = vma->vm_mm;
190 struct anon_vma *anon_vma, *allocated; 190 struct anon_vma *anon_vma, *allocated;
191 struct anon_vma_chain *avc; 191 struct anon_vma_chain *avc;
192 192
193 mmap_assert_locked(mm); 193 mmap_assert_locked(mm);
194 might_sleep(); 194 might_sleep();
195 195
196 avc = anon_vma_chain_alloc(GFP_KERNEL) 196 avc = anon_vma_chain_alloc(GFP_KERNEL);
197 if (!avc) 197 if (!avc)
198 goto out_enomem; 198 goto out_enomem;
199 199
200 anon_vma = find_mergeable_anon_vma(vma 200 anon_vma = find_mergeable_anon_vma(vma);
201 allocated = NULL; 201 allocated = NULL;
202 if (!anon_vma) { 202 if (!anon_vma) {
203 anon_vma = anon_vma_alloc(); 203 anon_vma = anon_vma_alloc();
204 if (unlikely(!anon_vma)) 204 if (unlikely(!anon_vma))
205 goto out_enomem_free_a 205 goto out_enomem_free_avc;
206 anon_vma->num_children++; /* s 206 anon_vma->num_children++; /* self-parent link for new root */
207 allocated = anon_vma; 207 allocated = anon_vma;
208 } 208 }
209 209
210 anon_vma_lock_write(anon_vma); 210 anon_vma_lock_write(anon_vma);
211 /* page_table_lock to protect against 211 /* page_table_lock to protect against threads */
212 spin_lock(&mm->page_table_lock); 212 spin_lock(&mm->page_table_lock);
213 if (likely(!vma->anon_vma)) { 213 if (likely(!vma->anon_vma)) {
214 vma->anon_vma = anon_vma; 214 vma->anon_vma = anon_vma;
215 anon_vma_chain_link(vma, avc, 215 anon_vma_chain_link(vma, avc, anon_vma);
216 anon_vma->num_active_vmas++; 216 anon_vma->num_active_vmas++;
217 allocated = NULL; 217 allocated = NULL;
218 avc = NULL; 218 avc = NULL;
219 } 219 }
220 spin_unlock(&mm->page_table_lock); 220 spin_unlock(&mm->page_table_lock);
221 anon_vma_unlock_write(anon_vma); 221 anon_vma_unlock_write(anon_vma);
222 222
223 if (unlikely(allocated)) 223 if (unlikely(allocated))
224 put_anon_vma(allocated); 224 put_anon_vma(allocated);
225 if (unlikely(avc)) 225 if (unlikely(avc))
226 anon_vma_chain_free(avc); 226 anon_vma_chain_free(avc);
227 227
228 return 0; 228 return 0;
229 229
230 out_enomem_free_avc: 230 out_enomem_free_avc:
231 anon_vma_chain_free(avc); 231 anon_vma_chain_free(avc);
232 out_enomem: 232 out_enomem:
233 return -ENOMEM; 233 return -ENOMEM;
234 } 234 }
235 235
236 /* 236 /*
237 * This is a useful helper function for lockin 237 * This is a useful helper function for locking the anon_vma root as
238 * we traverse the vma->anon_vma_chain, loopin 238 * we traverse the vma->anon_vma_chain, looping over anon_vma's that
239 * have the same vma. 239 * have the same vma.
240 * 240 *
241 * Such anon_vma's should have the same root, 241 * Such anon_vma's should have the same root, so you'd expect to see
242 * just a single mutex_lock for the whole trav 242 * just a single mutex_lock for the whole traversal.
243 */ 243 */
244 static inline struct anon_vma *lock_anon_vma_r 244 static inline struct anon_vma *lock_anon_vma_root(struct anon_vma *root, struct anon_vma *anon_vma)
245 { 245 {
246 struct anon_vma *new_root = anon_vma-> 246 struct anon_vma *new_root = anon_vma->root;
247 if (new_root != root) { 247 if (new_root != root) {
248 if (WARN_ON_ONCE(root)) 248 if (WARN_ON_ONCE(root))
249 up_write(&root->rwsem) 249 up_write(&root->rwsem);
250 root = new_root; 250 root = new_root;
251 down_write(&root->rwsem); 251 down_write(&root->rwsem);
252 } 252 }
253 return root; 253 return root;
254 } 254 }
255 255
256 static inline void unlock_anon_vma_root(struct 256 static inline void unlock_anon_vma_root(struct anon_vma *root)
257 { 257 {
258 if (root) 258 if (root)
259 up_write(&root->rwsem); 259 up_write(&root->rwsem);
260 } 260 }
261 261
262 /* 262 /*
263 * Attach the anon_vmas from src to dst. 263 * Attach the anon_vmas from src to dst.
264 * Returns 0 on success, -ENOMEM on failure. 264 * Returns 0 on success, -ENOMEM on failure.
265 * 265 *
266 * anon_vma_clone() is called by vma_expand(), 266 * anon_vma_clone() is called by vma_expand(), vma_merge(), __split_vma(),
267 * copy_vma() and anon_vma_fork(). The first f 267 * copy_vma() and anon_vma_fork(). The first four want an exact copy of src,
268 * while the last one, anon_vma_fork(), may tr 268 * while the last one, anon_vma_fork(), may try to reuse an existing anon_vma to
269 * prevent endless growth of anon_vma. Since d 269 * prevent endless growth of anon_vma. Since dst->anon_vma is set to NULL before
270 * call, we can identify this case by checking 270 * call, we can identify this case by checking (!dst->anon_vma &&
271 * src->anon_vma). 271 * src->anon_vma).
272 * 272 *
273 * If (!dst->anon_vma && src->anon_vma) is tru 273 * If (!dst->anon_vma && src->anon_vma) is true, this function tries to find
274 * and reuse existing anon_vma which has no vm 274 * and reuse existing anon_vma which has no vmas and only one child anon_vma.
275 * This prevents degradation of anon_vma hiera 275 * This prevents degradation of anon_vma hierarchy to endless linear chain in
276 * case of constantly forking task. On the oth 276 * case of constantly forking task. On the other hand, an anon_vma with more
277 * than one child isn't reused even if there w 277 * than one child isn't reused even if there was no alive vma, thus rmap
278 * walker has a good chance of avoiding scanni 278 * walker has a good chance of avoiding scanning the whole hierarchy when it
279 * searches where page is mapped. 279 * searches where page is mapped.
280 */ 280 */
281 int anon_vma_clone(struct vm_area_struct *dst, 281 int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src)
282 { 282 {
283 struct anon_vma_chain *avc, *pavc; 283 struct anon_vma_chain *avc, *pavc;
284 struct anon_vma *root = NULL; 284 struct anon_vma *root = NULL;
285 285
286 list_for_each_entry_reverse(pavc, &src 286 list_for_each_entry_reverse(pavc, &src->anon_vma_chain, same_vma) {
287 struct anon_vma *anon_vma; 287 struct anon_vma *anon_vma;
288 288
289 avc = anon_vma_chain_alloc(GFP 289 avc = anon_vma_chain_alloc(GFP_NOWAIT | __GFP_NOWARN);
290 if (unlikely(!avc)) { 290 if (unlikely(!avc)) {
291 unlock_anon_vma_root(r 291 unlock_anon_vma_root(root);
292 root = NULL; 292 root = NULL;
293 avc = anon_vma_chain_a 293 avc = anon_vma_chain_alloc(GFP_KERNEL);
294 if (!avc) 294 if (!avc)
295 goto enomem_fa 295 goto enomem_failure;
296 } 296 }
297 anon_vma = pavc->anon_vma; 297 anon_vma = pavc->anon_vma;
298 root = lock_anon_vma_root(root 298 root = lock_anon_vma_root(root, anon_vma);
299 anon_vma_chain_link(dst, avc, 299 anon_vma_chain_link(dst, avc, anon_vma);
300 300
301 /* 301 /*
302 * Reuse existing anon_vma if 302 * Reuse existing anon_vma if it has no vma and only one
303 * anon_vma child. 303 * anon_vma child.
304 * 304 *
305 * Root anon_vma is never reus 305 * Root anon_vma is never reused:
306 * it has self-parent referenc 306 * it has self-parent reference and at least one child.
307 */ 307 */
308 if (!dst->anon_vma && src->ano 308 if (!dst->anon_vma && src->anon_vma &&
309 anon_vma->num_children < 2 309 anon_vma->num_children < 2 &&
310 anon_vma->num_active_vmas 310 anon_vma->num_active_vmas == 0)
311 dst->anon_vma = anon_v 311 dst->anon_vma = anon_vma;
312 } 312 }
313 if (dst->anon_vma) 313 if (dst->anon_vma)
314 dst->anon_vma->num_active_vmas 314 dst->anon_vma->num_active_vmas++;
315 unlock_anon_vma_root(root); 315 unlock_anon_vma_root(root);
316 return 0; 316 return 0;
317 317
318 enomem_failure: 318 enomem_failure:
319 /* 319 /*
320 * dst->anon_vma is dropped here other 320 * dst->anon_vma is dropped here otherwise its num_active_vmas can
321 * be incorrectly decremented in unlin 321 * be incorrectly decremented in unlink_anon_vmas().
322 * We can safely do this because calle 322 * We can safely do this because callers of anon_vma_clone() don't care
323 * about dst->anon_vma if anon_vma_clo 323 * about dst->anon_vma if anon_vma_clone() failed.
324 */ 324 */
325 dst->anon_vma = NULL; 325 dst->anon_vma = NULL;
326 unlink_anon_vmas(dst); 326 unlink_anon_vmas(dst);
327 return -ENOMEM; 327 return -ENOMEM;
328 } 328 }
329 329
330 /* 330 /*
331 * Attach vma to its own anon_vma, as well as 331 * Attach vma to its own anon_vma, as well as to the anon_vmas that
332 * the corresponding VMA in the parent process 332 * the corresponding VMA in the parent process is attached to.
333 * Returns 0 on success, non-zero on failure. 333 * Returns 0 on success, non-zero on failure.
334 */ 334 */
335 int anon_vma_fork(struct vm_area_struct *vma, 335 int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma)
336 { 336 {
337 struct anon_vma_chain *avc; 337 struct anon_vma_chain *avc;
338 struct anon_vma *anon_vma; 338 struct anon_vma *anon_vma;
339 int error; 339 int error;
340 340
341 /* Don't bother if the parent process 341 /* Don't bother if the parent process has no anon_vma here. */
342 if (!pvma->anon_vma) 342 if (!pvma->anon_vma)
343 return 0; 343 return 0;
344 344
345 /* Drop inherited anon_vma, we'll reus 345 /* Drop inherited anon_vma, we'll reuse existing or allocate new. */
346 vma->anon_vma = NULL; 346 vma->anon_vma = NULL;
347 347
348 /* 348 /*
349 * First, attach the new VMA to the pa 349 * First, attach the new VMA to the parent VMA's anon_vmas,
350 * so rmap can find non-COWed pages in 350 * so rmap can find non-COWed pages in child processes.
351 */ 351 */
352 error = anon_vma_clone(vma, pvma); 352 error = anon_vma_clone(vma, pvma);
353 if (error) 353 if (error)
354 return error; 354 return error;
355 355
356 /* An existing anon_vma has been reuse 356 /* An existing anon_vma has been reused, all done then. */
357 if (vma->anon_vma) 357 if (vma->anon_vma)
358 return 0; 358 return 0;
359 359
360 /* Then add our own anon_vma. */ 360 /* Then add our own anon_vma. */
361 anon_vma = anon_vma_alloc(); 361 anon_vma = anon_vma_alloc();
362 if (!anon_vma) 362 if (!anon_vma)
363 goto out_error; 363 goto out_error;
364 anon_vma->num_active_vmas++; 364 anon_vma->num_active_vmas++;
365 avc = anon_vma_chain_alloc(GFP_KERNEL) 365 avc = anon_vma_chain_alloc(GFP_KERNEL);
366 if (!avc) 366 if (!avc)
367 goto out_error_free_anon_vma; 367 goto out_error_free_anon_vma;
368 368
369 /* 369 /*
370 * The root anon_vma's rwsem is the lo 370 * The root anon_vma's rwsem is the lock actually used when we
371 * lock any of the anon_vmas in this a 371 * lock any of the anon_vmas in this anon_vma tree.
372 */ 372 */
373 anon_vma->root = pvma->anon_vma->root; 373 anon_vma->root = pvma->anon_vma->root;
374 anon_vma->parent = pvma->anon_vma; 374 anon_vma->parent = pvma->anon_vma;
375 /* 375 /*
376 * With refcounts, an anon_vma can sta 376 * With refcounts, an anon_vma can stay around longer than the
377 * process it belongs to. The root ano 377 * process it belongs to. The root anon_vma needs to be pinned until
378 * this anon_vma is freed, because the 378 * this anon_vma is freed, because the lock lives in the root.
379 */ 379 */
380 get_anon_vma(anon_vma->root); 380 get_anon_vma(anon_vma->root);
381 /* Mark this anon_vma as the one where 381 /* Mark this anon_vma as the one where our new (COWed) pages go. */
382 vma->anon_vma = anon_vma; 382 vma->anon_vma = anon_vma;
383 anon_vma_lock_write(anon_vma); 383 anon_vma_lock_write(anon_vma);
384 anon_vma_chain_link(vma, avc, anon_vma 384 anon_vma_chain_link(vma, avc, anon_vma);
385 anon_vma->parent->num_children++; 385 anon_vma->parent->num_children++;
386 anon_vma_unlock_write(anon_vma); 386 anon_vma_unlock_write(anon_vma);
387 387
388 return 0; 388 return 0;
389 389
390 out_error_free_anon_vma: 390 out_error_free_anon_vma:
391 put_anon_vma(anon_vma); 391 put_anon_vma(anon_vma);
392 out_error: 392 out_error:
393 unlink_anon_vmas(vma); 393 unlink_anon_vmas(vma);
394 return -ENOMEM; 394 return -ENOMEM;
395 } 395 }
396 396
397 void unlink_anon_vmas(struct vm_area_struct *v 397 void unlink_anon_vmas(struct vm_area_struct *vma)
398 { 398 {
399 struct anon_vma_chain *avc, *next; 399 struct anon_vma_chain *avc, *next;
400 struct anon_vma *root = NULL; 400 struct anon_vma *root = NULL;
401 401
402 /* 402 /*
403 * Unlink each anon_vma chained to the 403 * Unlink each anon_vma chained to the VMA. This list is ordered
404 * from newest to oldest, ensuring the 404 * from newest to oldest, ensuring the root anon_vma gets freed last.
405 */ 405 */
406 list_for_each_entry_safe(avc, next, &v 406 list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) {
407 struct anon_vma *anon_vma = av 407 struct anon_vma *anon_vma = avc->anon_vma;
408 408
409 root = lock_anon_vma_root(root 409 root = lock_anon_vma_root(root, anon_vma);
410 anon_vma_interval_tree_remove( 410 anon_vma_interval_tree_remove(avc, &anon_vma->rb_root);
411 411
412 /* 412 /*
413 * Leave empty anon_vmas on th 413 * Leave empty anon_vmas on the list - we'll need
414 * to free them outside the lo 414 * to free them outside the lock.
415 */ 415 */
416 if (RB_EMPTY_ROOT(&anon_vma->r 416 if (RB_EMPTY_ROOT(&anon_vma->rb_root.rb_root)) {
417 anon_vma->parent->num_ 417 anon_vma->parent->num_children--;
418 continue; 418 continue;
419 } 419 }
420 420
421 list_del(&avc->same_vma); 421 list_del(&avc->same_vma);
422 anon_vma_chain_free(avc); 422 anon_vma_chain_free(avc);
423 } 423 }
424 if (vma->anon_vma) { 424 if (vma->anon_vma) {
425 vma->anon_vma->num_active_vmas 425 vma->anon_vma->num_active_vmas--;
426 426
427 /* 427 /*
428 * vma would still be needed a 428 * vma would still be needed after unlink, and anon_vma will be prepared
429 * when handle fault. 429 * when handle fault.
430 */ 430 */
431 vma->anon_vma = NULL; 431 vma->anon_vma = NULL;
432 } 432 }
433 unlock_anon_vma_root(root); 433 unlock_anon_vma_root(root);
434 434
435 /* 435 /*
436 * Iterate the list once more, it now 436 * Iterate the list once more, it now only contains empty and unlinked
437 * anon_vmas, destroy them. Could not 437 * anon_vmas, destroy them. Could not do before due to __put_anon_vma()
438 * needing to write-acquire the anon_v 438 * needing to write-acquire the anon_vma->root->rwsem.
439 */ 439 */
440 list_for_each_entry_safe(avc, next, &v 440 list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) {
441 struct anon_vma *anon_vma = av 441 struct anon_vma *anon_vma = avc->anon_vma;
442 442
443 VM_WARN_ON(anon_vma->num_child 443 VM_WARN_ON(anon_vma->num_children);
444 VM_WARN_ON(anon_vma->num_activ 444 VM_WARN_ON(anon_vma->num_active_vmas);
445 put_anon_vma(anon_vma); 445 put_anon_vma(anon_vma);
446 446
447 list_del(&avc->same_vma); 447 list_del(&avc->same_vma);
448 anon_vma_chain_free(avc); 448 anon_vma_chain_free(avc);
449 } 449 }
450 } 450 }
451 451
452 static void anon_vma_ctor(void *data) 452 static void anon_vma_ctor(void *data)
453 { 453 {
454 struct anon_vma *anon_vma = data; 454 struct anon_vma *anon_vma = data;
455 455
456 init_rwsem(&anon_vma->rwsem); 456 init_rwsem(&anon_vma->rwsem);
457 atomic_set(&anon_vma->refcount, 0); 457 atomic_set(&anon_vma->refcount, 0);
458 anon_vma->rb_root = RB_ROOT_CACHED; 458 anon_vma->rb_root = RB_ROOT_CACHED;
459 } 459 }
460 460
461 void __init anon_vma_init(void) 461 void __init anon_vma_init(void)
462 { 462 {
463 anon_vma_cachep = kmem_cache_create("a 463 anon_vma_cachep = kmem_cache_create("anon_vma", sizeof(struct anon_vma),
464 0, SLAB_TYPESAFE_BY_RC 464 0, SLAB_TYPESAFE_BY_RCU|SLAB_PANIC|SLAB_ACCOUNT,
465 anon_vma_ctor); 465 anon_vma_ctor);
466 anon_vma_chain_cachep = KMEM_CACHE(ano 466 anon_vma_chain_cachep = KMEM_CACHE(anon_vma_chain,
467 SLAB_PANIC|SLAB_ACCOUN 467 SLAB_PANIC|SLAB_ACCOUNT);
468 } 468 }
469 469
470 /* 470 /*
471 * Getting a lock on a stable anon_vma from a 471 * Getting a lock on a stable anon_vma from a page off the LRU is tricky!
472 * 472 *
473 * Since there is no serialization what so eve 473 * Since there is no serialization what so ever against folio_remove_rmap_*()
474 * the best this function can do is return a r 474 * the best this function can do is return a refcount increased anon_vma
475 * that might have been relevant to this page. 475 * that might have been relevant to this page.
476 * 476 *
477 * The page might have been remapped to a diff 477 * The page might have been remapped to a different anon_vma or the anon_vma
478 * returned may already be freed (and even reu 478 * returned may already be freed (and even reused).
479 * 479 *
480 * In case it was remapped to a different anon 480 * In case it was remapped to a different anon_vma, the new anon_vma will be a
481 * child of the old anon_vma, and the anon_vma 481 * child of the old anon_vma, and the anon_vma lifetime rules will therefore
482 * ensure that any anon_vma obtained from the 482 * ensure that any anon_vma obtained from the page will still be valid for as
483 * long as we observe page_mapped() [ hence al 483 * long as we observe page_mapped() [ hence all those page_mapped() tests ].
484 * 484 *
485 * All users of this function must be very car 485 * All users of this function must be very careful when walking the anon_vma
486 * chain and verify that the page in question 486 * chain and verify that the page in question is indeed mapped in it
487 * [ something equivalent to page_mapped_in_vm 487 * [ something equivalent to page_mapped_in_vma() ].
488 * 488 *
489 * Since anon_vma's slab is SLAB_TYPESAFE_BY_R 489 * Since anon_vma's slab is SLAB_TYPESAFE_BY_RCU and we know from
490 * folio_remove_rmap_*() that the anon_vma poi 490 * folio_remove_rmap_*() that the anon_vma pointer from page->mapping is valid
491 * if there is a mapcount, we can dereference 491 * if there is a mapcount, we can dereference the anon_vma after observing
492 * those. 492 * those.
493 * 493 *
494 * NOTE: the caller should normally hold folio 494 * NOTE: the caller should normally hold folio lock when calling this. If
495 * not, the caller needs to double check the a 495 * not, the caller needs to double check the anon_vma didn't change after
496 * taking the anon_vma lock for either read or 496 * taking the anon_vma lock for either read or write (UFFDIO_MOVE can modify it
497 * concurrently without folio lock protection) 497 * concurrently without folio lock protection). See folio_lock_anon_vma_read()
498 * which has already covered that, and comment 498 * which has already covered that, and comment above remap_pages().
499 */ 499 */
500 struct anon_vma *folio_get_anon_vma(struct fol 500 struct anon_vma *folio_get_anon_vma(struct folio *folio)
501 { 501 {
502 struct anon_vma *anon_vma = NULL; 502 struct anon_vma *anon_vma = NULL;
503 unsigned long anon_mapping; 503 unsigned long anon_mapping;
504 504
505 rcu_read_lock(); 505 rcu_read_lock();
506 anon_mapping = (unsigned long)READ_ONC 506 anon_mapping = (unsigned long)READ_ONCE(folio->mapping);
507 if ((anon_mapping & PAGE_MAPPING_FLAGS 507 if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON)
508 goto out; 508 goto out;
509 if (!folio_mapped(folio)) 509 if (!folio_mapped(folio))
510 goto out; 510 goto out;
511 511
512 anon_vma = (struct anon_vma *) (anon_m 512 anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON);
513 if (!atomic_inc_not_zero(&anon_vma->re 513 if (!atomic_inc_not_zero(&anon_vma->refcount)) {
514 anon_vma = NULL; 514 anon_vma = NULL;
515 goto out; 515 goto out;
516 } 516 }
517 517
518 /* 518 /*
519 * If this folio is still mapped, then 519 * If this folio is still mapped, then its anon_vma cannot have been
520 * freed. But if it has been unmapped 520 * freed. But if it has been unmapped, we have no security against the
521 * anon_vma structure being freed and 521 * anon_vma structure being freed and reused (for another anon_vma:
522 * SLAB_TYPESAFE_BY_RCU guarantees tha 522 * SLAB_TYPESAFE_BY_RCU guarantees that - so the atomic_inc_not_zero()
523 * above cannot corrupt). 523 * above cannot corrupt).
524 */ 524 */
525 if (!folio_mapped(folio)) { 525 if (!folio_mapped(folio)) {
526 rcu_read_unlock(); 526 rcu_read_unlock();
527 put_anon_vma(anon_vma); 527 put_anon_vma(anon_vma);
528 return NULL; 528 return NULL;
529 } 529 }
530 out: 530 out:
531 rcu_read_unlock(); 531 rcu_read_unlock();
532 532
533 return anon_vma; 533 return anon_vma;
534 } 534 }
535 535
536 /* 536 /*
537 * Similar to folio_get_anon_vma() except it l 537 * Similar to folio_get_anon_vma() except it locks the anon_vma.
538 * 538 *
539 * Its a little more complex as it tries to ke 539 * Its a little more complex as it tries to keep the fast path to a single
540 * atomic op -- the trylock. If we fail the tr 540 * atomic op -- the trylock. If we fail the trylock, we fall back to getting a
541 * reference like with folio_get_anon_vma() an 541 * reference like with folio_get_anon_vma() and then block on the mutex
542 * on !rwc->try_lock case. 542 * on !rwc->try_lock case.
543 */ 543 */
544 struct anon_vma *folio_lock_anon_vma_read(stru 544 struct anon_vma *folio_lock_anon_vma_read(struct folio *folio,
545 stru 545 struct rmap_walk_control *rwc)
546 { 546 {
547 struct anon_vma *anon_vma = NULL; 547 struct anon_vma *anon_vma = NULL;
548 struct anon_vma *root_anon_vma; 548 struct anon_vma *root_anon_vma;
549 unsigned long anon_mapping; 549 unsigned long anon_mapping;
550 550
551 retry: 551 retry:
552 rcu_read_lock(); 552 rcu_read_lock();
553 anon_mapping = (unsigned long)READ_ONC 553 anon_mapping = (unsigned long)READ_ONCE(folio->mapping);
554 if ((anon_mapping & PAGE_MAPPING_FLAGS 554 if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON)
555 goto out; 555 goto out;
556 if (!folio_mapped(folio)) 556 if (!folio_mapped(folio))
557 goto out; 557 goto out;
558 558
559 anon_vma = (struct anon_vma *) (anon_m 559 anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON);
560 root_anon_vma = READ_ONCE(anon_vma->ro 560 root_anon_vma = READ_ONCE(anon_vma->root);
561 if (down_read_trylock(&root_anon_vma-> 561 if (down_read_trylock(&root_anon_vma->rwsem)) {
562 /* 562 /*
563 * folio_move_anon_rmap() migh 563 * folio_move_anon_rmap() might have changed the anon_vma as we
564 * might not hold the folio lo 564 * might not hold the folio lock here.
565 */ 565 */
566 if (unlikely((unsigned long)RE 566 if (unlikely((unsigned long)READ_ONCE(folio->mapping) !=
567 anon_mapping)) { 567 anon_mapping)) {
568 up_read(&root_anon_vma 568 up_read(&root_anon_vma->rwsem);
569 rcu_read_unlock(); 569 rcu_read_unlock();
570 goto retry; 570 goto retry;
571 } 571 }
572 572
573 /* 573 /*
574 * If the folio is still mappe 574 * If the folio is still mapped, then this anon_vma is still
575 * its anon_vma, and holding t 575 * its anon_vma, and holding the mutex ensures that it will
576 * not go away, see anon_vma_f 576 * not go away, see anon_vma_free().
577 */ 577 */
578 if (!folio_mapped(folio)) { 578 if (!folio_mapped(folio)) {
579 up_read(&root_anon_vma 579 up_read(&root_anon_vma->rwsem);
580 anon_vma = NULL; 580 anon_vma = NULL;
581 } 581 }
582 goto out; 582 goto out;
583 } 583 }
584 584
585 if (rwc && rwc->try_lock) { 585 if (rwc && rwc->try_lock) {
586 anon_vma = NULL; 586 anon_vma = NULL;
587 rwc->contended = true; 587 rwc->contended = true;
588 goto out; 588 goto out;
589 } 589 }
590 590
591 /* trylock failed, we got to sleep */ 591 /* trylock failed, we got to sleep */
592 if (!atomic_inc_not_zero(&anon_vma->re 592 if (!atomic_inc_not_zero(&anon_vma->refcount)) {
593 anon_vma = NULL; 593 anon_vma = NULL;
594 goto out; 594 goto out;
595 } 595 }
596 596
597 if (!folio_mapped(folio)) { 597 if (!folio_mapped(folio)) {
598 rcu_read_unlock(); 598 rcu_read_unlock();
599 put_anon_vma(anon_vma); 599 put_anon_vma(anon_vma);
600 return NULL; 600 return NULL;
601 } 601 }
602 602
603 /* we pinned the anon_vma, its safe to 603 /* we pinned the anon_vma, its safe to sleep */
604 rcu_read_unlock(); 604 rcu_read_unlock();
605 anon_vma_lock_read(anon_vma); 605 anon_vma_lock_read(anon_vma);
606 606
607 /* 607 /*
608 * folio_move_anon_rmap() might have c 608 * folio_move_anon_rmap() might have changed the anon_vma as we might
609 * not hold the folio lock here. 609 * not hold the folio lock here.
610 */ 610 */
611 if (unlikely((unsigned long)READ_ONCE( 611 if (unlikely((unsigned long)READ_ONCE(folio->mapping) !=
612 anon_mapping)) { 612 anon_mapping)) {
613 anon_vma_unlock_read(anon_vma) 613 anon_vma_unlock_read(anon_vma);
614 put_anon_vma(anon_vma); 614 put_anon_vma(anon_vma);
615 anon_vma = NULL; 615 anon_vma = NULL;
616 goto retry; 616 goto retry;
617 } 617 }
618 618
619 if (atomic_dec_and_test(&anon_vma->ref 619 if (atomic_dec_and_test(&anon_vma->refcount)) {
620 /* 620 /*
621 * Oops, we held the last refc 621 * Oops, we held the last refcount, release the lock
622 * and bail -- can't simply us 622 * and bail -- can't simply use put_anon_vma() because
623 * we'll deadlock on the anon_ 623 * we'll deadlock on the anon_vma_lock_write() recursion.
624 */ 624 */
625 anon_vma_unlock_read(anon_vma) 625 anon_vma_unlock_read(anon_vma);
626 __put_anon_vma(anon_vma); 626 __put_anon_vma(anon_vma);
627 anon_vma = NULL; 627 anon_vma = NULL;
628 } 628 }
629 629
630 return anon_vma; 630 return anon_vma;
631 631
632 out: 632 out:
633 rcu_read_unlock(); 633 rcu_read_unlock();
634 return anon_vma; 634 return anon_vma;
635 } 635 }
636 636
637 #ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUS 637 #ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
638 /* 638 /*
639 * Flush TLB entries for recently unmapped pag 639 * Flush TLB entries for recently unmapped pages from remote CPUs. It is
640 * important if a PTE was dirty when it was un 640 * important if a PTE was dirty when it was unmapped that it's flushed
641 * before any IO is initiated on the page to p 641 * before any IO is initiated on the page to prevent lost writes. Similarly,
642 * it must be flushed before freeing to preven 642 * it must be flushed before freeing to prevent data leakage.
643 */ 643 */
644 void try_to_unmap_flush(void) 644 void try_to_unmap_flush(void)
645 { 645 {
646 struct tlbflush_unmap_batch *tlb_ubc = 646 struct tlbflush_unmap_batch *tlb_ubc = ¤t->tlb_ubc;
647 647
648 if (!tlb_ubc->flush_required) 648 if (!tlb_ubc->flush_required)
649 return; 649 return;
650 650
651 arch_tlbbatch_flush(&tlb_ubc->arch); 651 arch_tlbbatch_flush(&tlb_ubc->arch);
652 tlb_ubc->flush_required = false; 652 tlb_ubc->flush_required = false;
653 tlb_ubc->writable = false; 653 tlb_ubc->writable = false;
654 } 654 }
655 655
656 /* Flush iff there are potentially writable TL 656 /* Flush iff there are potentially writable TLB entries that can race with IO */
657 void try_to_unmap_flush_dirty(void) 657 void try_to_unmap_flush_dirty(void)
658 { 658 {
659 struct tlbflush_unmap_batch *tlb_ubc = 659 struct tlbflush_unmap_batch *tlb_ubc = ¤t->tlb_ubc;
660 660
661 if (tlb_ubc->writable) 661 if (tlb_ubc->writable)
662 try_to_unmap_flush(); 662 try_to_unmap_flush();
663 } 663 }
664 664
665 /* 665 /*
666 * Bits 0-14 of mm->tlb_flush_batched record p 666 * Bits 0-14 of mm->tlb_flush_batched record pending generations.
667 * Bits 16-30 of mm->tlb_flush_batched bit rec 667 * Bits 16-30 of mm->tlb_flush_batched bit record flushed generations.
668 */ 668 */
669 #define TLB_FLUSH_BATCH_FLUSHED_SHIFT 16 669 #define TLB_FLUSH_BATCH_FLUSHED_SHIFT 16
670 #define TLB_FLUSH_BATCH_PENDING_MASK 670 #define TLB_FLUSH_BATCH_PENDING_MASK \
671 ((1 << (TLB_FLUSH_BATCH_FLUSHED_SHIFT 671 ((1 << (TLB_FLUSH_BATCH_FLUSHED_SHIFT - 1)) - 1)
672 #define TLB_FLUSH_BATCH_PENDING_LARGE 672 #define TLB_FLUSH_BATCH_PENDING_LARGE \
673 (TLB_FLUSH_BATCH_PENDING_MASK / 2) 673 (TLB_FLUSH_BATCH_PENDING_MASK / 2)
674 674
675 static void set_tlb_ubc_flush_pending(struct m 675 static void set_tlb_ubc_flush_pending(struct mm_struct *mm, pte_t pteval,
676 unsigned 676 unsigned long uaddr)
677 { 677 {
678 struct tlbflush_unmap_batch *tlb_ubc = 678 struct tlbflush_unmap_batch *tlb_ubc = ¤t->tlb_ubc;
679 int batch; 679 int batch;
680 bool writable = pte_dirty(pteval); 680 bool writable = pte_dirty(pteval);
681 681
682 if (!pte_accessible(mm, pteval)) 682 if (!pte_accessible(mm, pteval))
683 return; 683 return;
684 684
685 arch_tlbbatch_add_pending(&tlb_ubc->ar 685 arch_tlbbatch_add_pending(&tlb_ubc->arch, mm, uaddr);
686 tlb_ubc->flush_required = true; 686 tlb_ubc->flush_required = true;
687 687
688 /* 688 /*
689 * Ensure compiler does not re-order t 689 * Ensure compiler does not re-order the setting of tlb_flush_batched
690 * before the PTE is cleared. 690 * before the PTE is cleared.
691 */ 691 */
692 barrier(); 692 barrier();
693 batch = atomic_read(&mm->tlb_flush_bat 693 batch = atomic_read(&mm->tlb_flush_batched);
694 retry: 694 retry:
695 if ((batch & TLB_FLUSH_BATCH_PENDING_M 695 if ((batch & TLB_FLUSH_BATCH_PENDING_MASK) > TLB_FLUSH_BATCH_PENDING_LARGE) {
696 /* 696 /*
697 * Prevent `pending' from catc 697 * Prevent `pending' from catching up with `flushed' because of
698 * overflow. Reset `pending' 698 * overflow. Reset `pending' and `flushed' to be 1 and 0 if
699 * `pending' becomes large. 699 * `pending' becomes large.
700 */ 700 */
701 if (!atomic_try_cmpxchg(&mm->t 701 if (!atomic_try_cmpxchg(&mm->tlb_flush_batched, &batch, 1))
702 goto retry; 702 goto retry;
703 } else { 703 } else {
704 atomic_inc(&mm->tlb_flush_batc 704 atomic_inc(&mm->tlb_flush_batched);
705 } 705 }
706 706
707 /* 707 /*
708 * If the PTE was dirty then it's best 708 * If the PTE was dirty then it's best to assume it's writable. The
709 * caller must use try_to_unmap_flush_ 709 * caller must use try_to_unmap_flush_dirty() or try_to_unmap_flush()
710 * before the page is queued for IO. 710 * before the page is queued for IO.
711 */ 711 */
712 if (writable) 712 if (writable)
713 tlb_ubc->writable = true; 713 tlb_ubc->writable = true;
714 } 714 }
715 715
716 /* 716 /*
717 * Returns true if the TLB flush should be def 717 * Returns true if the TLB flush should be deferred to the end of a batch of
718 * unmap operations to reduce IPIs. 718 * unmap operations to reduce IPIs.
719 */ 719 */
720 static bool should_defer_flush(struct mm_struc 720 static bool should_defer_flush(struct mm_struct *mm, enum ttu_flags flags)
721 { 721 {
722 if (!(flags & TTU_BATCH_FLUSH)) 722 if (!(flags & TTU_BATCH_FLUSH))
723 return false; 723 return false;
724 724
725 return arch_tlbbatch_should_defer(mm); 725 return arch_tlbbatch_should_defer(mm);
726 } 726 }
727 727
728 /* 728 /*
729 * Reclaim unmaps pages under the PTL but do n 729 * Reclaim unmaps pages under the PTL but do not flush the TLB prior to
730 * releasing the PTL if TLB flushes are batche 730 * releasing the PTL if TLB flushes are batched. It's possible for a parallel
731 * operation such as mprotect or munmap to rac 731 * operation such as mprotect or munmap to race between reclaim unmapping
732 * the page and flushing the page. If this rac 732 * the page and flushing the page. If this race occurs, it potentially allows
733 * access to data via a stale TLB entry. Track 733 * access to data via a stale TLB entry. Tracking all mm's that have TLB
734 * batching in flight would be expensive durin 734 * batching in flight would be expensive during reclaim so instead track
735 * whether TLB batching occurred in the past a 735 * whether TLB batching occurred in the past and if so then do a flush here
736 * if required. This will cost one additional 736 * if required. This will cost one additional flush per reclaim cycle paid
737 * by the first operation at risk such as mpro 737 * by the first operation at risk such as mprotect and mumap.
738 * 738 *
739 * This must be called under the PTL so that a 739 * This must be called under the PTL so that an access to tlb_flush_batched
740 * that is potentially a "reclaim vs mprotect/ 740 * that is potentially a "reclaim vs mprotect/munmap/etc" race will synchronise
741 * via the PTL. 741 * via the PTL.
742 */ 742 */
743 void flush_tlb_batched_pending(struct mm_struc 743 void flush_tlb_batched_pending(struct mm_struct *mm)
744 { 744 {
745 int batch = atomic_read(&mm->tlb_flush 745 int batch = atomic_read(&mm->tlb_flush_batched);
746 int pending = batch & TLB_FLUSH_BATCH_ 746 int pending = batch & TLB_FLUSH_BATCH_PENDING_MASK;
747 int flushed = batch >> TLB_FLUSH_BATCH 747 int flushed = batch >> TLB_FLUSH_BATCH_FLUSHED_SHIFT;
748 748
749 if (pending != flushed) { 749 if (pending != flushed) {
750 arch_flush_tlb_batched_pending 750 arch_flush_tlb_batched_pending(mm);
751 /* 751 /*
752 * If the new TLB flushing is 752 * If the new TLB flushing is pending during flushing, leave
753 * mm->tlb_flush_batched as is 753 * mm->tlb_flush_batched as is, to avoid losing flushing.
754 */ 754 */
755 atomic_cmpxchg(&mm->tlb_flush_ 755 atomic_cmpxchg(&mm->tlb_flush_batched, batch,
756 pending | (pend 756 pending | (pending << TLB_FLUSH_BATCH_FLUSHED_SHIFT));
757 } 757 }
758 } 758 }
759 #else 759 #else
760 static void set_tlb_ubc_flush_pending(struct m 760 static void set_tlb_ubc_flush_pending(struct mm_struct *mm, pte_t pteval,
761 unsigned 761 unsigned long uaddr)
762 { 762 {
763 } 763 }
764 764
765 static bool should_defer_flush(struct mm_struc 765 static bool should_defer_flush(struct mm_struct *mm, enum ttu_flags flags)
766 { 766 {
767 return false; 767 return false;
768 } 768 }
769 #endif /* CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_F 769 #endif /* CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH */
770 770
771 /* 771 /*
772 * At what user virtual address is page expect 772 * At what user virtual address is page expected in vma?
773 * Caller should check the page is actually pa 773 * Caller should check the page is actually part of the vma.
774 */ 774 */
775 unsigned long page_address_in_vma(struct page 775 unsigned long page_address_in_vma(struct page *page, struct vm_area_struct *vma)
776 { 776 {
777 struct folio *folio = page_folio(page) 777 struct folio *folio = page_folio(page);
778 pgoff_t pgoff; 778 pgoff_t pgoff;
779 779
780 if (folio_test_anon(folio)) { 780 if (folio_test_anon(folio)) {
781 struct anon_vma *page__anon_vm 781 struct anon_vma *page__anon_vma = folio_anon_vma(folio);
782 /* 782 /*
783 * Note: swapoff's unuse_vma() 783 * Note: swapoff's unuse_vma() is more efficient with this
784 * check, and needs it to matc 784 * check, and needs it to match anon_vma when KSM is active.
785 */ 785 */
786 if (!vma->anon_vma || !page__a 786 if (!vma->anon_vma || !page__anon_vma ||
787 vma->anon_vma->root != pag 787 vma->anon_vma->root != page__anon_vma->root)
788 return -EFAULT; 788 return -EFAULT;
789 } else if (!vma->vm_file) { 789 } else if (!vma->vm_file) {
790 return -EFAULT; 790 return -EFAULT;
791 } else if (vma->vm_file->f_mapping != 791 } else if (vma->vm_file->f_mapping != folio->mapping) {
792 return -EFAULT; 792 return -EFAULT;
793 } 793 }
794 794
795 /* The !page__anon_vma above handles K 795 /* The !page__anon_vma above handles KSM folios */
796 pgoff = folio->index + folio_page_idx( 796 pgoff = folio->index + folio_page_idx(folio, page);
797 return vma_address(vma, pgoff, 1); 797 return vma_address(vma, pgoff, 1);
798 } 798 }
799 799
800 /* 800 /*
801 * Returns the actual pmd_t* where we expect ' 801 * Returns the actual pmd_t* where we expect 'address' to be mapped from, or
802 * NULL if it doesn't exist. No guarantees / 802 * NULL if it doesn't exist. No guarantees / checks on what the pmd_t*
803 * represents. 803 * represents.
804 */ 804 */
805 pmd_t *mm_find_pmd(struct mm_struct *mm, unsig 805 pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address)
806 { 806 {
807 pgd_t *pgd; 807 pgd_t *pgd;
808 p4d_t *p4d; 808 p4d_t *p4d;
809 pud_t *pud; 809 pud_t *pud;
810 pmd_t *pmd = NULL; 810 pmd_t *pmd = NULL;
811 811
812 pgd = pgd_offset(mm, address); 812 pgd = pgd_offset(mm, address);
813 if (!pgd_present(*pgd)) 813 if (!pgd_present(*pgd))
814 goto out; 814 goto out;
815 815
816 p4d = p4d_offset(pgd, address); 816 p4d = p4d_offset(pgd, address);
817 if (!p4d_present(*p4d)) 817 if (!p4d_present(*p4d))
818 goto out; 818 goto out;
819 819
820 pud = pud_offset(p4d, address); 820 pud = pud_offset(p4d, address);
821 if (!pud_present(*pud)) 821 if (!pud_present(*pud))
822 goto out; 822 goto out;
823 823
824 pmd = pmd_offset(pud, address); 824 pmd = pmd_offset(pud, address);
825 out: 825 out:
826 return pmd; 826 return pmd;
827 } 827 }
828 828
829 struct folio_referenced_arg { 829 struct folio_referenced_arg {
830 int mapcount; 830 int mapcount;
831 int referenced; 831 int referenced;
832 unsigned long vm_flags; 832 unsigned long vm_flags;
833 struct mem_cgroup *memcg; 833 struct mem_cgroup *memcg;
834 }; 834 };
835 835
836 /* 836 /*
837 * arg: folio_referenced_arg will be passed 837 * arg: folio_referenced_arg will be passed
838 */ 838 */
839 static bool folio_referenced_one(struct folio 839 static bool folio_referenced_one(struct folio *folio,
840 struct vm_area_struct *vma, un 840 struct vm_area_struct *vma, unsigned long address, void *arg)
841 { 841 {
842 struct folio_referenced_arg *pra = arg 842 struct folio_referenced_arg *pra = arg;
843 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma 843 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);
844 int referenced = 0; 844 int referenced = 0;
845 unsigned long start = address, ptes = 845 unsigned long start = address, ptes = 0;
846 846
847 while (page_vma_mapped_walk(&pvmw)) { 847 while (page_vma_mapped_walk(&pvmw)) {
848 address = pvmw.address; 848 address = pvmw.address;
849 849
850 if (vma->vm_flags & VM_LOCKED) 850 if (vma->vm_flags & VM_LOCKED) {
851 if (!folio_test_large( 851 if (!folio_test_large(folio) || !pvmw.pte) {
852 /* Restore the 852 /* Restore the mlock which got missed */
853 mlock_vma_foli 853 mlock_vma_folio(folio, vma);
854 page_vma_mappe 854 page_vma_mapped_walk_done(&pvmw);
855 pra->vm_flags 855 pra->vm_flags |= VM_LOCKED;
856 return false; 856 return false; /* To break the loop */
857 } 857 }
858 /* 858 /*
859 * For large folio ful 859 * For large folio fully mapped to VMA, will
860 * be handled after th 860 * be handled after the pvmw loop.
861 * 861 *
862 * For large folio cro 862 * For large folio cross VMA boundaries, it's
863 * expected to be pick 863 * expected to be picked by page reclaim. But
864 * should skip referen 864 * should skip reference of pages which are in
865 * the range of VM_LOC 865 * the range of VM_LOCKED vma. As page reclaim
866 * should just count t 866 * should just count the reference of pages out
867 * the range of VM_LOC 867 * the range of VM_LOCKED vma.
868 */ 868 */
869 ptes++; 869 ptes++;
870 pra->mapcount--; 870 pra->mapcount--;
871 continue; 871 continue;
872 } 872 }
873 873
874 /* 874 /*
875 * Skip the non-shared swapbac 875 * Skip the non-shared swapbacked folio mapped solely by
876 * the exiting or OOM-reaped p 876 * the exiting or OOM-reaped process. This avoids redundant
877 * swap-out followed by an imm 877 * swap-out followed by an immediate unmap.
878 */ 878 */
879 if ((!atomic_read(&vma->vm_mm- 879 if ((!atomic_read(&vma->vm_mm->mm_users) ||
880 check_stable_address_space 880 check_stable_address_space(vma->vm_mm)) &&
881 folio_test_anon(folio) && 881 folio_test_anon(folio) && folio_test_swapbacked(folio) &&
882 !folio_likely_mapped_share 882 !folio_likely_mapped_shared(folio)) {
883 pra->referenced = -1; 883 pra->referenced = -1;
884 page_vma_mapped_walk_d 884 page_vma_mapped_walk_done(&pvmw);
885 return false; 885 return false;
886 } 886 }
887 887
888 if (lru_gen_enabled() && pvmw. 888 if (lru_gen_enabled() && pvmw.pte) {
889 if (lru_gen_look_aroun 889 if (lru_gen_look_around(&pvmw))
890 referenced++; 890 referenced++;
891 } else if (pvmw.pte) { 891 } else if (pvmw.pte) {
892 if (ptep_clear_flush_y 892 if (ptep_clear_flush_young_notify(vma, address,
893 893 pvmw.pte))
894 referenced++; 894 referenced++;
895 } else if (IS_ENABLED(CONFIG_T 895 } else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) {
896 if (pmdp_clear_flush_y 896 if (pmdp_clear_flush_young_notify(vma, address,
897 897 pvmw.pmd))
898 referenced++; 898 referenced++;
899 } else { 899 } else {
900 /* unexpected pmd-mapp 900 /* unexpected pmd-mapped folio? */
901 WARN_ON_ONCE(1); 901 WARN_ON_ONCE(1);
902 } 902 }
903 903
904 pra->mapcount--; 904 pra->mapcount--;
905 } 905 }
906 906
907 if ((vma->vm_flags & VM_LOCKED) && 907 if ((vma->vm_flags & VM_LOCKED) &&
908 folio_test_large(folio 908 folio_test_large(folio) &&
909 folio_within_vma(folio 909 folio_within_vma(folio, vma)) {
910 unsigned long s_align, e_align 910 unsigned long s_align, e_align;
911 911
912 s_align = ALIGN_DOWN(start, PM 912 s_align = ALIGN_DOWN(start, PMD_SIZE);
913 e_align = ALIGN_DOWN(start + f 913 e_align = ALIGN_DOWN(start + folio_size(folio) - 1, PMD_SIZE);
914 914
915 /* folio doesn't cross page ta 915 /* folio doesn't cross page table boundary and fully mapped */
916 if ((s_align == e_align) && (p 916 if ((s_align == e_align) && (ptes == folio_nr_pages(folio))) {
917 /* Restore the mlock w 917 /* Restore the mlock which got missed */
918 mlock_vma_folio(folio, 918 mlock_vma_folio(folio, vma);
919 pra->vm_flags |= VM_LO 919 pra->vm_flags |= VM_LOCKED;
920 return false; /* To br 920 return false; /* To break the loop */
921 } 921 }
922 } 922 }
923 923
924 if (referenced) 924 if (referenced)
925 folio_clear_idle(folio); 925 folio_clear_idle(folio);
926 if (folio_test_clear_young(folio)) 926 if (folio_test_clear_young(folio))
927 referenced++; 927 referenced++;
928 928
929 if (referenced) { 929 if (referenced) {
930 pra->referenced++; 930 pra->referenced++;
931 pra->vm_flags |= vma->vm_flags 931 pra->vm_flags |= vma->vm_flags & ~VM_LOCKED;
932 } 932 }
933 933
934 if (!pra->mapcount) 934 if (!pra->mapcount)
935 return false; /* To break the 935 return false; /* To break the loop */
936 936
937 return true; 937 return true;
938 } 938 }
939 939
940 static bool invalid_folio_referenced_vma(struc 940 static bool invalid_folio_referenced_vma(struct vm_area_struct *vma, void *arg)
941 { 941 {
942 struct folio_referenced_arg *pra = arg 942 struct folio_referenced_arg *pra = arg;
943 struct mem_cgroup *memcg = pra->memcg; 943 struct mem_cgroup *memcg = pra->memcg;
944 944
945 /* 945 /*
946 * Ignore references from this mapping 946 * Ignore references from this mapping if it has no recency. If the
947 * folio has been used in another mapp 947 * folio has been used in another mapping, we will catch it; if this
948 * other mapping is already gone, the 948 * other mapping is already gone, the unmap path will have set the
949 * referenced flag or activated the fo 949 * referenced flag or activated the folio in zap_pte_range().
950 */ 950 */
951 if (!vma_has_recency(vma)) 951 if (!vma_has_recency(vma))
952 return true; 952 return true;
953 953
954 /* 954 /*
955 * If we are reclaiming on behalf of a 955 * If we are reclaiming on behalf of a cgroup, skip counting on behalf
956 * of references from different cgroup 956 * of references from different cgroups.
957 */ 957 */
958 if (memcg && !mm_match_cgroup(vma->vm_ 958 if (memcg && !mm_match_cgroup(vma->vm_mm, memcg))
959 return true; 959 return true;
960 960
961 return false; 961 return false;
962 } 962 }
963 963
964 /** 964 /**
965 * folio_referenced() - Test if the folio was 965 * folio_referenced() - Test if the folio was referenced.
966 * @folio: The folio to test. 966 * @folio: The folio to test.
967 * @is_locked: Caller holds lock on the folio. 967 * @is_locked: Caller holds lock on the folio.
968 * @memcg: target memory cgroup 968 * @memcg: target memory cgroup
969 * @vm_flags: A combination of all the vma->vm 969 * @vm_flags: A combination of all the vma->vm_flags which referenced the folio.
970 * 970 *
971 * Quick test_and_clear_referenced for all map 971 * Quick test_and_clear_referenced for all mappings of a folio,
972 * 972 *
973 * Return: The number of mappings which refere 973 * Return: The number of mappings which referenced the folio. Return -1 if
974 * the function bailed out due to rmap lock co 974 * the function bailed out due to rmap lock contention.
975 */ 975 */
976 int folio_referenced(struct folio *folio, int 976 int folio_referenced(struct folio *folio, int is_locked,
977 struct mem_cgroup *memcg, 977 struct mem_cgroup *memcg, unsigned long *vm_flags)
978 { 978 {
979 bool we_locked = false; 979 bool we_locked = false;
980 struct folio_referenced_arg pra = { 980 struct folio_referenced_arg pra = {
981 .mapcount = folio_mapcount(fol 981 .mapcount = folio_mapcount(folio),
982 .memcg = memcg, 982 .memcg = memcg,
983 }; 983 };
984 struct rmap_walk_control rwc = { 984 struct rmap_walk_control rwc = {
985 .rmap_one = folio_referenced_o 985 .rmap_one = folio_referenced_one,
986 .arg = (void *)&pra, 986 .arg = (void *)&pra,
987 .anon_lock = folio_lock_anon_v 987 .anon_lock = folio_lock_anon_vma_read,
988 .try_lock = true, 988 .try_lock = true,
989 .invalid_vma = invalid_folio_r 989 .invalid_vma = invalid_folio_referenced_vma,
990 }; 990 };
991 991
992 *vm_flags = 0; 992 *vm_flags = 0;
993 if (!pra.mapcount) 993 if (!pra.mapcount)
994 return 0; 994 return 0;
995 995
996 if (!folio_raw_mapping(folio)) 996 if (!folio_raw_mapping(folio))
997 return 0; 997 return 0;
998 998
999 if (!is_locked && (!folio_test_anon(fo 999 if (!is_locked && (!folio_test_anon(folio) || folio_test_ksm(folio))) {
1000 we_locked = folio_trylock(fol 1000 we_locked = folio_trylock(folio);
1001 if (!we_locked) 1001 if (!we_locked)
1002 return 1; 1002 return 1;
1003 } 1003 }
1004 1004
1005 rmap_walk(folio, &rwc); 1005 rmap_walk(folio, &rwc);
1006 *vm_flags = pra.vm_flags; 1006 *vm_flags = pra.vm_flags;
1007 1007
1008 if (we_locked) 1008 if (we_locked)
1009 folio_unlock(folio); 1009 folio_unlock(folio);
1010 1010
1011 return rwc.contended ? -1 : pra.refer 1011 return rwc.contended ? -1 : pra.referenced;
1012 } 1012 }
1013 1013
1014 static int page_vma_mkclean_one(struct page_v 1014 static int page_vma_mkclean_one(struct page_vma_mapped_walk *pvmw)
1015 { 1015 {
1016 int cleaned = 0; 1016 int cleaned = 0;
1017 struct vm_area_struct *vma = pvmw->vm 1017 struct vm_area_struct *vma = pvmw->vma;
1018 struct mmu_notifier_range range; 1018 struct mmu_notifier_range range;
1019 unsigned long address = pvmw->address 1019 unsigned long address = pvmw->address;
1020 1020
1021 /* 1021 /*
1022 * We have to assume the worse case i 1022 * We have to assume the worse case ie pmd for invalidation. Note that
1023 * the folio can not be freed from th 1023 * the folio can not be freed from this function.
1024 */ 1024 */
1025 mmu_notifier_range_init(&range, MMU_N 1025 mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_PAGE, 0,
1026 vma->vm_mm, a 1026 vma->vm_mm, address, vma_address_end(pvmw));
1027 mmu_notifier_invalidate_range_start(& 1027 mmu_notifier_invalidate_range_start(&range);
1028 1028
1029 while (page_vma_mapped_walk(pvmw)) { 1029 while (page_vma_mapped_walk(pvmw)) {
1030 int ret = 0; 1030 int ret = 0;
1031 1031
1032 address = pvmw->address; 1032 address = pvmw->address;
1033 if (pvmw->pte) { 1033 if (pvmw->pte) {
1034 pte_t *pte = pvmw->pt 1034 pte_t *pte = pvmw->pte;
1035 pte_t entry = ptep_ge 1035 pte_t entry = ptep_get(pte);
1036 1036
1037 if (!pte_dirty(entry) 1037 if (!pte_dirty(entry) && !pte_write(entry))
1038 continue; 1038 continue;
1039 1039
1040 flush_cache_page(vma, 1040 flush_cache_page(vma, address, pte_pfn(entry));
1041 entry = ptep_clear_fl 1041 entry = ptep_clear_flush(vma, address, pte);
1042 entry = pte_wrprotect 1042 entry = pte_wrprotect(entry);
1043 entry = pte_mkclean(e 1043 entry = pte_mkclean(entry);
1044 set_pte_at(vma->vm_mm 1044 set_pte_at(vma->vm_mm, address, pte, entry);
1045 ret = 1; 1045 ret = 1;
1046 } else { 1046 } else {
1047 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1047 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1048 pmd_t *pmd = pvmw->pm 1048 pmd_t *pmd = pvmw->pmd;
1049 pmd_t entry; 1049 pmd_t entry;
1050 1050
1051 if (!pmd_dirty(*pmd) 1051 if (!pmd_dirty(*pmd) && !pmd_write(*pmd))
1052 continue; 1052 continue;
1053 1053
1054 flush_cache_range(vma 1054 flush_cache_range(vma, address,
1055 add 1055 address + HPAGE_PMD_SIZE);
1056 entry = pmdp_invalida 1056 entry = pmdp_invalidate(vma, address, pmd);
1057 entry = pmd_wrprotect 1057 entry = pmd_wrprotect(entry);
1058 entry = pmd_mkclean(e 1058 entry = pmd_mkclean(entry);
1059 set_pmd_at(vma->vm_mm 1059 set_pmd_at(vma->vm_mm, address, pmd, entry);
1060 ret = 1; 1060 ret = 1;
1061 #else 1061 #else
1062 /* unexpected pmd-map 1062 /* unexpected pmd-mapped folio? */
1063 WARN_ON_ONCE(1); 1063 WARN_ON_ONCE(1);
1064 #endif 1064 #endif
1065 } 1065 }
1066 1066
1067 if (ret) 1067 if (ret)
1068 cleaned++; 1068 cleaned++;
1069 } 1069 }
1070 1070
1071 mmu_notifier_invalidate_range_end(&ra 1071 mmu_notifier_invalidate_range_end(&range);
1072 1072
1073 return cleaned; 1073 return cleaned;
1074 } 1074 }
1075 1075
1076 static bool page_mkclean_one(struct folio *fo 1076 static bool page_mkclean_one(struct folio *folio, struct vm_area_struct *vma,
1077 unsigned long ad 1077 unsigned long address, void *arg)
1078 { 1078 {
1079 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vm 1079 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, PVMW_SYNC);
1080 int *cleaned = arg; 1080 int *cleaned = arg;
1081 1081
1082 *cleaned += page_vma_mkclean_one(&pvm 1082 *cleaned += page_vma_mkclean_one(&pvmw);
1083 1083
1084 return true; 1084 return true;
1085 } 1085 }
1086 1086
1087 static bool invalid_mkclean_vma(struct vm_are 1087 static bool invalid_mkclean_vma(struct vm_area_struct *vma, void *arg)
1088 { 1088 {
1089 if (vma->vm_flags & VM_SHARED) 1089 if (vma->vm_flags & VM_SHARED)
1090 return false; 1090 return false;
1091 1091
1092 return true; 1092 return true;
1093 } 1093 }
1094 1094
1095 int folio_mkclean(struct folio *folio) 1095 int folio_mkclean(struct folio *folio)
1096 { 1096 {
1097 int cleaned = 0; 1097 int cleaned = 0;
1098 struct address_space *mapping; 1098 struct address_space *mapping;
1099 struct rmap_walk_control rwc = { 1099 struct rmap_walk_control rwc = {
1100 .arg = (void *)&cleaned, 1100 .arg = (void *)&cleaned,
1101 .rmap_one = page_mkclean_one, 1101 .rmap_one = page_mkclean_one,
1102 .invalid_vma = invalid_mkclea 1102 .invalid_vma = invalid_mkclean_vma,
1103 }; 1103 };
1104 1104
1105 BUG_ON(!folio_test_locked(folio)); 1105 BUG_ON(!folio_test_locked(folio));
1106 1106
1107 if (!folio_mapped(folio)) 1107 if (!folio_mapped(folio))
1108 return 0; 1108 return 0;
1109 1109
1110 mapping = folio_mapping(folio); 1110 mapping = folio_mapping(folio);
1111 if (!mapping) 1111 if (!mapping)
1112 return 0; 1112 return 0;
1113 1113
1114 rmap_walk(folio, &rwc); 1114 rmap_walk(folio, &rwc);
1115 1115
1116 return cleaned; 1116 return cleaned;
1117 } 1117 }
1118 EXPORT_SYMBOL_GPL(folio_mkclean); 1118 EXPORT_SYMBOL_GPL(folio_mkclean);
1119 1119
1120 /** 1120 /**
1121 * pfn_mkclean_range - Cleans the PTEs (inclu 1121 * pfn_mkclean_range - Cleans the PTEs (including PMDs) mapped with range of
1122 * [@pfn, @pfn + @nr_page 1122 * [@pfn, @pfn + @nr_pages) at the specific offset (@pgoff)
1123 * within the @vma of sha 1123 * within the @vma of shared mappings. And since clean PTEs
1124 * should also be readonl 1124 * should also be readonly, write protects them too.
1125 * @pfn: start pfn. 1125 * @pfn: start pfn.
1126 * @nr_pages: number of physically contiguous 1126 * @nr_pages: number of physically contiguous pages srarting with @pfn.
1127 * @pgoff: page offset that the @pfn mapped w 1127 * @pgoff: page offset that the @pfn mapped with.
1128 * @vma: vma that @pfn mapped within. 1128 * @vma: vma that @pfn mapped within.
1129 * 1129 *
1130 * Returns the number of cleaned PTEs (includ 1130 * Returns the number of cleaned PTEs (including PMDs).
1131 */ 1131 */
1132 int pfn_mkclean_range(unsigned long pfn, unsi 1132 int pfn_mkclean_range(unsigned long pfn, unsigned long nr_pages, pgoff_t pgoff,
1133 struct vm_area_struct * 1133 struct vm_area_struct *vma)
1134 { 1134 {
1135 struct page_vma_mapped_walk pvmw = { 1135 struct page_vma_mapped_walk pvmw = {
1136 .pfn = pfn, 1136 .pfn = pfn,
1137 .nr_pages = nr_pages, 1137 .nr_pages = nr_pages,
1138 .pgoff = pgoff, 1138 .pgoff = pgoff,
1139 .vma = vma, 1139 .vma = vma,
1140 .flags = PVMW_SYNC, 1140 .flags = PVMW_SYNC,
1141 }; 1141 };
1142 1142
1143 if (invalid_mkclean_vma(vma, NULL)) 1143 if (invalid_mkclean_vma(vma, NULL))
1144 return 0; 1144 return 0;
1145 1145
1146 pvmw.address = vma_address(vma, pgoff 1146 pvmw.address = vma_address(vma, pgoff, nr_pages);
1147 VM_BUG_ON_VMA(pvmw.address == -EFAULT 1147 VM_BUG_ON_VMA(pvmw.address == -EFAULT, vma);
1148 1148
1149 return page_vma_mkclean_one(&pvmw); 1149 return page_vma_mkclean_one(&pvmw);
1150 } 1150 }
1151 1151
1152 static __always_inline unsigned int __folio_a 1152 static __always_inline unsigned int __folio_add_rmap(struct folio *folio,
1153 struct page *page, int nr_pag 1153 struct page *page, int nr_pages, enum rmap_level level,
1154 int *nr_pmdmapped) 1154 int *nr_pmdmapped)
1155 { 1155 {
1156 atomic_t *mapped = &folio->_nr_pages_ 1156 atomic_t *mapped = &folio->_nr_pages_mapped;
1157 const int orig_nr_pages = nr_pages; 1157 const int orig_nr_pages = nr_pages;
1158 int first = 0, nr = 0; 1158 int first = 0, nr = 0;
1159 1159
1160 __folio_rmap_sanity_checks(folio, pag 1160 __folio_rmap_sanity_checks(folio, page, nr_pages, level);
1161 1161
1162 switch (level) { 1162 switch (level) {
1163 case RMAP_LEVEL_PTE: 1163 case RMAP_LEVEL_PTE:
1164 if (!folio_test_large(folio)) 1164 if (!folio_test_large(folio)) {
1165 nr = atomic_inc_and_t 1165 nr = atomic_inc_and_test(&folio->_mapcount);
1166 break; 1166 break;
1167 } 1167 }
1168 1168
1169 do { 1169 do {
1170 first += atomic_inc_a 1170 first += atomic_inc_and_test(&page->_mapcount);
1171 } while (page++, --nr_pages > 1171 } while (page++, --nr_pages > 0);
1172 1172
1173 if (first && 1173 if (first &&
1174 atomic_add_return_relaxed 1174 atomic_add_return_relaxed(first, mapped) < ENTIRELY_MAPPED)
1175 nr = first; 1175 nr = first;
1176 1176
1177 atomic_add(orig_nr_pages, &fo 1177 atomic_add(orig_nr_pages, &folio->_large_mapcount);
1178 break; 1178 break;
1179 case RMAP_LEVEL_PMD: 1179 case RMAP_LEVEL_PMD:
1180 first = atomic_inc_and_test(& 1180 first = atomic_inc_and_test(&folio->_entire_mapcount);
1181 if (first) { 1181 if (first) {
1182 nr = atomic_add_retur 1182 nr = atomic_add_return_relaxed(ENTIRELY_MAPPED, mapped);
1183 if (likely(nr < ENTIR 1183 if (likely(nr < ENTIRELY_MAPPED + ENTIRELY_MAPPED)) {
1184 *nr_pmdmapped 1184 *nr_pmdmapped = folio_nr_pages(folio);
1185 nr = *nr_pmdm 1185 nr = *nr_pmdmapped - (nr & FOLIO_PAGES_MAPPED);
1186 /* Raced ahea 1186 /* Raced ahead of a remove and another add? */
1187 if (unlikely( 1187 if (unlikely(nr < 0))
1188 nr = 1188 nr = 0;
1189 } else { 1189 } else {
1190 /* Raced ahea 1190 /* Raced ahead of a remove of ENTIRELY_MAPPED */
1191 nr = 0; 1191 nr = 0;
1192 } 1192 }
1193 } 1193 }
1194 atomic_inc(&folio->_large_map 1194 atomic_inc(&folio->_large_mapcount);
1195 break; 1195 break;
1196 } 1196 }
1197 return nr; 1197 return nr;
1198 } 1198 }
1199 1199
1200 /** 1200 /**
1201 * folio_move_anon_rmap - move a folio to our 1201 * folio_move_anon_rmap - move a folio to our anon_vma
1202 * @folio: The folio to move to our anon 1202 * @folio: The folio to move to our anon_vma
1203 * @vma: The vma the folio belongs to 1203 * @vma: The vma the folio belongs to
1204 * 1204 *
1205 * When a folio belongs exclusively to one pr 1205 * When a folio belongs exclusively to one process after a COW event,
1206 * that folio can be moved into the anon_vma 1206 * that folio can be moved into the anon_vma that belongs to just that
1207 * process, so the rmap code will not search 1207 * process, so the rmap code will not search the parent or sibling processes.
1208 */ 1208 */
1209 void folio_move_anon_rmap(struct folio *folio 1209 void folio_move_anon_rmap(struct folio *folio, struct vm_area_struct *vma)
1210 { 1210 {
1211 void *anon_vma = vma->anon_vma; 1211 void *anon_vma = vma->anon_vma;
1212 1212
1213 VM_BUG_ON_FOLIO(!folio_test_locked(fo 1213 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
1214 VM_BUG_ON_VMA(!anon_vma, vma); 1214 VM_BUG_ON_VMA(!anon_vma, vma);
1215 1215
1216 anon_vma += PAGE_MAPPING_ANON; 1216 anon_vma += PAGE_MAPPING_ANON;
1217 /* 1217 /*
1218 * Ensure that anon_vma and the PAGE_ 1218 * Ensure that anon_vma and the PAGE_MAPPING_ANON bit are written
1219 * simultaneously, so a concurrent re 1219 * simultaneously, so a concurrent reader (eg folio_referenced()'s
1220 * folio_test_anon()) will not see on 1220 * folio_test_anon()) will not see one without the other.
1221 */ 1221 */
1222 WRITE_ONCE(folio->mapping, anon_vma); 1222 WRITE_ONCE(folio->mapping, anon_vma);
1223 } 1223 }
1224 1224
1225 /** 1225 /**
1226 * __folio_set_anon - set up a new anonymous 1226 * __folio_set_anon - set up a new anonymous rmap for a folio
1227 * @folio: The folio to set up the new a 1227 * @folio: The folio to set up the new anonymous rmap for.
1228 * @vma: VM area to add the folio to. 1228 * @vma: VM area to add the folio to.
1229 * @address: User virtual address of the m 1229 * @address: User virtual address of the mapping
1230 * @exclusive: Whether the folio is exclusiv 1230 * @exclusive: Whether the folio is exclusive to the process.
1231 */ 1231 */
1232 static void __folio_set_anon(struct folio *fo 1232 static void __folio_set_anon(struct folio *folio, struct vm_area_struct *vma,
1233 unsigned long ad 1233 unsigned long address, bool exclusive)
1234 { 1234 {
1235 struct anon_vma *anon_vma = vma->anon 1235 struct anon_vma *anon_vma = vma->anon_vma;
1236 1236
1237 BUG_ON(!anon_vma); 1237 BUG_ON(!anon_vma);
1238 1238
1239 /* 1239 /*
1240 * If the folio isn't exclusive to th 1240 * If the folio isn't exclusive to this vma, we must use the _oldest_
1241 * possible anon_vma for the folio ma 1241 * possible anon_vma for the folio mapping!
1242 */ 1242 */
1243 if (!exclusive) 1243 if (!exclusive)
1244 anon_vma = anon_vma->root; 1244 anon_vma = anon_vma->root;
1245 1245
1246 /* 1246 /*
1247 * page_idle does a lockless/optimist 1247 * page_idle does a lockless/optimistic rmap scan on folio->mapping.
1248 * Make sure the compiler doesn't spl 1248 * Make sure the compiler doesn't split the stores of anon_vma and
1249 * the PAGE_MAPPING_ANON type identif 1249 * the PAGE_MAPPING_ANON type identifier, otherwise the rmap code
1250 * could mistake the mapping for a st 1250 * could mistake the mapping for a struct address_space and crash.
1251 */ 1251 */
1252 anon_vma = (void *) anon_vma + PAGE_M 1252 anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON;
1253 WRITE_ONCE(folio->mapping, (struct ad 1253 WRITE_ONCE(folio->mapping, (struct address_space *) anon_vma);
1254 folio->index = linear_page_index(vma, 1254 folio->index = linear_page_index(vma, address);
1255 } 1255 }
1256 1256
1257 /** 1257 /**
1258 * __page_check_anon_rmap - sanity check anon 1258 * __page_check_anon_rmap - sanity check anonymous rmap addition
1259 * @folio: The folio containing @page. 1259 * @folio: The folio containing @page.
1260 * @page: the page to check the mapping 1260 * @page: the page to check the mapping of
1261 * @vma: the vm area in which the mapp 1261 * @vma: the vm area in which the mapping is added
1262 * @address: the user virtual address mapp 1262 * @address: the user virtual address mapped
1263 */ 1263 */
1264 static void __page_check_anon_rmap(struct fol 1264 static void __page_check_anon_rmap(struct folio *folio, struct page *page,
1265 struct vm_area_struct *vma, unsigned 1265 struct vm_area_struct *vma, unsigned long address)
1266 { 1266 {
1267 /* 1267 /*
1268 * The page's anon-rmap details (mapp 1268 * The page's anon-rmap details (mapping and index) are guaranteed to
1269 * be set up correctly at this point. 1269 * be set up correctly at this point.
1270 * 1270 *
1271 * We have exclusion against folio_ad 1271 * We have exclusion against folio_add_anon_rmap_*() because the caller
1272 * always holds the page locked. 1272 * always holds the page locked.
1273 * 1273 *
1274 * We have exclusion against folio_ad 1274 * We have exclusion against folio_add_new_anon_rmap because those pages
1275 * are initially only visible via the 1275 * are initially only visible via the pagetables, and the pte is locked
1276 * over the call to folio_add_new_ano 1276 * over the call to folio_add_new_anon_rmap.
1277 */ 1277 */
1278 VM_BUG_ON_FOLIO(folio_anon_vma(folio) 1278 VM_BUG_ON_FOLIO(folio_anon_vma(folio)->root != vma->anon_vma->root,
1279 folio); 1279 folio);
1280 VM_BUG_ON_PAGE(page_to_pgoff(page) != 1280 VM_BUG_ON_PAGE(page_to_pgoff(page) != linear_page_index(vma, address),
1281 page); 1281 page);
1282 } 1282 }
1283 1283
1284 static void __folio_mod_stat(struct folio *fo 1284 static void __folio_mod_stat(struct folio *folio, int nr, int nr_pmdmapped)
1285 { 1285 {
1286 int idx; 1286 int idx;
1287 1287
1288 if (nr) { 1288 if (nr) {
1289 idx = folio_test_anon(folio) 1289 idx = folio_test_anon(folio) ? NR_ANON_MAPPED : NR_FILE_MAPPED;
1290 __lruvec_stat_mod_folio(folio 1290 __lruvec_stat_mod_folio(folio, idx, nr);
1291 } 1291 }
1292 if (nr_pmdmapped) { 1292 if (nr_pmdmapped) {
1293 if (folio_test_anon(folio)) { 1293 if (folio_test_anon(folio)) {
1294 idx = NR_ANON_THPS; 1294 idx = NR_ANON_THPS;
1295 __lruvec_stat_mod_fol 1295 __lruvec_stat_mod_folio(folio, idx, nr_pmdmapped);
1296 } else { 1296 } else {
1297 /* NR_*_PMDMAPPED are 1297 /* NR_*_PMDMAPPED are not maintained per-memcg */
1298 idx = folio_test_swap 1298 idx = folio_test_swapbacked(folio) ?
1299 NR_SHMEM_PMDM 1299 NR_SHMEM_PMDMAPPED : NR_FILE_PMDMAPPED;
1300 __mod_node_page_state 1300 __mod_node_page_state(folio_pgdat(folio), idx,
1301 1301 nr_pmdmapped);
1302 } 1302 }
1303 } 1303 }
1304 } 1304 }
1305 1305
1306 static __always_inline void __folio_add_anon_ 1306 static __always_inline void __folio_add_anon_rmap(struct folio *folio,
1307 struct page *page, int nr_pag 1307 struct page *page, int nr_pages, struct vm_area_struct *vma,
1308 unsigned long address, rmap_t 1308 unsigned long address, rmap_t flags, enum rmap_level level)
1309 { 1309 {
1310 int i, nr, nr_pmdmapped = 0; 1310 int i, nr, nr_pmdmapped = 0;
1311 1311
1312 VM_WARN_ON_FOLIO(!folio_test_anon(fol 1312 VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio);
1313 1313
1314 nr = __folio_add_rmap(folio, page, nr 1314 nr = __folio_add_rmap(folio, page, nr_pages, level, &nr_pmdmapped);
1315 1315
1316 if (likely(!folio_test_ksm(folio))) 1316 if (likely(!folio_test_ksm(folio)))
1317 __page_check_anon_rmap(folio, 1317 __page_check_anon_rmap(folio, page, vma, address);
1318 1318
1319 __folio_mod_stat(folio, nr, nr_pmdmap 1319 __folio_mod_stat(folio, nr, nr_pmdmapped);
1320 1320
1321 if (flags & RMAP_EXCLUSIVE) { 1321 if (flags & RMAP_EXCLUSIVE) {
1322 switch (level) { 1322 switch (level) {
1323 case RMAP_LEVEL_PTE: 1323 case RMAP_LEVEL_PTE:
1324 for (i = 0; i < nr_pa 1324 for (i = 0; i < nr_pages; i++)
1325 SetPageAnonEx 1325 SetPageAnonExclusive(page + i);
1326 break; 1326 break;
1327 case RMAP_LEVEL_PMD: 1327 case RMAP_LEVEL_PMD:
1328 SetPageAnonExclusive( 1328 SetPageAnonExclusive(page);
1329 break; 1329 break;
1330 } 1330 }
1331 } 1331 }
1332 for (i = 0; i < nr_pages; i++) { 1332 for (i = 0; i < nr_pages; i++) {
1333 struct page *cur_page = page 1333 struct page *cur_page = page + i;
1334 1334
1335 /* While PTE-mapping a THP we 1335 /* While PTE-mapping a THP we have a PMD and a PTE mapping. */
1336 VM_WARN_ON_FOLIO((atomic_read 1336 VM_WARN_ON_FOLIO((atomic_read(&cur_page->_mapcount) > 0 ||
1337 (folio_test 1337 (folio_test_large(folio) &&
1338 folio_enti 1338 folio_entire_mapcount(folio) > 1)) &&
1339 PageAnonExcl 1339 PageAnonExclusive(cur_page), folio);
1340 } 1340 }
1341 1341
1342 /* 1342 /*
1343 * For large folio, only mlock it if 1343 * For large folio, only mlock it if it's fully mapped to VMA. It's
1344 * not easy to check whether the larg 1344 * not easy to check whether the large folio is fully mapped to VMA
1345 * here. Only mlock normal 4K folio a 1345 * here. Only mlock normal 4K folio and leave page reclaim to handle
1346 * large folio. 1346 * large folio.
1347 */ 1347 */
1348 if (!folio_test_large(folio)) 1348 if (!folio_test_large(folio))
1349 mlock_vma_folio(folio, vma); 1349 mlock_vma_folio(folio, vma);
1350 } 1350 }
1351 1351
1352 /** 1352 /**
1353 * folio_add_anon_rmap_ptes - add PTE mapping 1353 * folio_add_anon_rmap_ptes - add PTE mappings to a page range of an anon folio
1354 * @folio: The folio to add the mappings 1354 * @folio: The folio to add the mappings to
1355 * @page: The first page to add 1355 * @page: The first page to add
1356 * @nr_pages: The number of pages which wil 1356 * @nr_pages: The number of pages which will be mapped
1357 * @vma: The vm area in which the mapp 1357 * @vma: The vm area in which the mappings are added
1358 * @address: The user virtual address of t 1358 * @address: The user virtual address of the first page to map
1359 * @flags: The rmap flags 1359 * @flags: The rmap flags
1360 * 1360 *
1361 * The page range of folio is defined by [fir 1361 * The page range of folio is defined by [first_page, first_page + nr_pages)
1362 * 1362 *
1363 * The caller needs to hold the page table lo 1363 * The caller needs to hold the page table lock, and the page must be locked in
1364 * the anon_vma case: to serialize mapping,in 1364 * the anon_vma case: to serialize mapping,index checking after setting,
1365 * and to ensure that an anon folio is not be 1365 * and to ensure that an anon folio is not being upgraded racily to a KSM folio
1366 * (but KSM folios are never downgraded). 1366 * (but KSM folios are never downgraded).
1367 */ 1367 */
1368 void folio_add_anon_rmap_ptes(struct folio *f 1368 void folio_add_anon_rmap_ptes(struct folio *folio, struct page *page,
1369 int nr_pages, struct vm_area_ 1369 int nr_pages, struct vm_area_struct *vma, unsigned long address,
1370 rmap_t flags) 1370 rmap_t flags)
1371 { 1371 {
1372 __folio_add_anon_rmap(folio, page, nr 1372 __folio_add_anon_rmap(folio, page, nr_pages, vma, address, flags,
1373 RMAP_LEVEL_PTE) 1373 RMAP_LEVEL_PTE);
1374 } 1374 }
1375 1375
1376 /** 1376 /**
1377 * folio_add_anon_rmap_pmd - add a PMD mappin 1377 * folio_add_anon_rmap_pmd - add a PMD mapping to a page range of an anon folio
1378 * @folio: The folio to add the mapping 1378 * @folio: The folio to add the mapping to
1379 * @page: The first page to add 1379 * @page: The first page to add
1380 * @vma: The vm area in which the mapp 1380 * @vma: The vm area in which the mapping is added
1381 * @address: The user virtual address of t 1381 * @address: The user virtual address of the first page to map
1382 * @flags: The rmap flags 1382 * @flags: The rmap flags
1383 * 1383 *
1384 * The page range of folio is defined by [fir 1384 * The page range of folio is defined by [first_page, first_page + HPAGE_PMD_NR)
1385 * 1385 *
1386 * The caller needs to hold the page table lo 1386 * The caller needs to hold the page table lock, and the page must be locked in
1387 * the anon_vma case: to serialize mapping,in 1387 * the anon_vma case: to serialize mapping,index checking after setting.
1388 */ 1388 */
1389 void folio_add_anon_rmap_pmd(struct folio *fo 1389 void folio_add_anon_rmap_pmd(struct folio *folio, struct page *page,
1390 struct vm_area_struct *vma, u 1390 struct vm_area_struct *vma, unsigned long address, rmap_t flags)
1391 { 1391 {
1392 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1392 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1393 __folio_add_anon_rmap(folio, page, HP 1393 __folio_add_anon_rmap(folio, page, HPAGE_PMD_NR, vma, address, flags,
1394 RMAP_LEVEL_PMD) 1394 RMAP_LEVEL_PMD);
1395 #else 1395 #else
1396 WARN_ON_ONCE(true); 1396 WARN_ON_ONCE(true);
1397 #endif 1397 #endif
1398 } 1398 }
1399 1399
1400 /** 1400 /**
1401 * folio_add_new_anon_rmap - Add mapping to a 1401 * folio_add_new_anon_rmap - Add mapping to a new anonymous folio.
1402 * @folio: The folio to add the mapping 1402 * @folio: The folio to add the mapping to.
1403 * @vma: the vm area in which the mapp 1403 * @vma: the vm area in which the mapping is added
1404 * @address: the user virtual address mapp 1404 * @address: the user virtual address mapped
1405 * @flags: The rmap flags 1405 * @flags: The rmap flags
1406 * 1406 *
1407 * Like folio_add_anon_rmap_*() but must only 1407 * Like folio_add_anon_rmap_*() but must only be called on *new* folios.
1408 * This means the inc-and-test can be bypasse 1408 * This means the inc-and-test can be bypassed.
1409 * The folio doesn't necessarily need to be l 1409 * The folio doesn't necessarily need to be locked while it's exclusive
1410 * unless two threads map it concurrently. Ho 1410 * unless two threads map it concurrently. However, the folio must be
1411 * locked if it's shared. 1411 * locked if it's shared.
1412 * 1412 *
1413 * If the folio is pmd-mappable, it is accoun 1413 * If the folio is pmd-mappable, it is accounted as a THP.
1414 */ 1414 */
1415 void folio_add_new_anon_rmap(struct folio *fo 1415 void folio_add_new_anon_rmap(struct folio *folio, struct vm_area_struct *vma,
1416 unsigned long address, rmap_t 1416 unsigned long address, rmap_t flags)
1417 { 1417 {
1418 const int nr = folio_nr_pages(folio); 1418 const int nr = folio_nr_pages(folio);
1419 const bool exclusive = flags & RMAP_E 1419 const bool exclusive = flags & RMAP_EXCLUSIVE;
1420 int nr_pmdmapped = 0; 1420 int nr_pmdmapped = 0;
1421 1421
1422 VM_WARN_ON_FOLIO(folio_test_hugetlb(f 1422 VM_WARN_ON_FOLIO(folio_test_hugetlb(folio), folio);
1423 VM_WARN_ON_FOLIO(!exclusive && !folio 1423 VM_WARN_ON_FOLIO(!exclusive && !folio_test_locked(folio), folio);
1424 VM_BUG_ON_VMA(address < vma->vm_start 1424 VM_BUG_ON_VMA(address < vma->vm_start ||
1425 address + (nr << PAGE 1425 address + (nr << PAGE_SHIFT) > vma->vm_end, vma);
1426 1426
1427 /* 1427 /*
1428 * VM_DROPPABLE mappings don't swap; 1428 * VM_DROPPABLE mappings don't swap; instead they're just dropped when
1429 * under memory pressure. 1429 * under memory pressure.
1430 */ 1430 */
1431 if (!folio_test_swapbacked(folio) && 1431 if (!folio_test_swapbacked(folio) && !(vma->vm_flags & VM_DROPPABLE))
1432 __folio_set_swapbacked(folio) 1432 __folio_set_swapbacked(folio);
1433 __folio_set_anon(folio, vma, address, 1433 __folio_set_anon(folio, vma, address, exclusive);
1434 1434
1435 if (likely(!folio_test_large(folio))) 1435 if (likely(!folio_test_large(folio))) {
1436 /* increment count (starts at 1436 /* increment count (starts at -1) */
1437 atomic_set(&folio->_mapcount, 1437 atomic_set(&folio->_mapcount, 0);
1438 if (exclusive) 1438 if (exclusive)
1439 SetPageAnonExclusive( 1439 SetPageAnonExclusive(&folio->page);
1440 } else if (!folio_test_pmd_mappable(f 1440 } else if (!folio_test_pmd_mappable(folio)) {
1441 int i; 1441 int i;
1442 1442
1443 for (i = 0; i < nr; i++) { 1443 for (i = 0; i < nr; i++) {
1444 struct page *page = f 1444 struct page *page = folio_page(folio, i);
1445 1445
1446 /* increment count (s 1446 /* increment count (starts at -1) */
1447 atomic_set(&page->_ma 1447 atomic_set(&page->_mapcount, 0);
1448 if (exclusive) 1448 if (exclusive)
1449 SetPageAnonEx 1449 SetPageAnonExclusive(page);
1450 } 1450 }
1451 1451
1452 /* increment count (starts at 1452 /* increment count (starts at -1) */
1453 atomic_set(&folio->_large_map 1453 atomic_set(&folio->_large_mapcount, nr - 1);
1454 atomic_set(&folio->_nr_pages_ 1454 atomic_set(&folio->_nr_pages_mapped, nr);
1455 } else { 1455 } else {
1456 /* increment count (starts at 1456 /* increment count (starts at -1) */
1457 atomic_set(&folio->_entire_ma 1457 atomic_set(&folio->_entire_mapcount, 0);
1458 /* increment count (starts at 1458 /* increment count (starts at -1) */
1459 atomic_set(&folio->_large_map 1459 atomic_set(&folio->_large_mapcount, 0);
1460 atomic_set(&folio->_nr_pages_ 1460 atomic_set(&folio->_nr_pages_mapped, ENTIRELY_MAPPED);
1461 if (exclusive) 1461 if (exclusive)
1462 SetPageAnonExclusive( 1462 SetPageAnonExclusive(&folio->page);
1463 nr_pmdmapped = nr; 1463 nr_pmdmapped = nr;
1464 } 1464 }
1465 1465
1466 __folio_mod_stat(folio, nr, nr_pmdmap 1466 __folio_mod_stat(folio, nr, nr_pmdmapped);
1467 mod_mthp_stat(folio_order(folio), MTH 1467 mod_mthp_stat(folio_order(folio), MTHP_STAT_NR_ANON, 1);
1468 } 1468 }
1469 1469
1470 static __always_inline void __folio_add_file_ 1470 static __always_inline void __folio_add_file_rmap(struct folio *folio,
1471 struct page *page, int nr_pag 1471 struct page *page, int nr_pages, struct vm_area_struct *vma,
1472 enum rmap_level level) 1472 enum rmap_level level)
1473 { 1473 {
1474 int nr, nr_pmdmapped = 0; 1474 int nr, nr_pmdmapped = 0;
1475 1475
1476 VM_WARN_ON_FOLIO(folio_test_anon(foli 1476 VM_WARN_ON_FOLIO(folio_test_anon(folio), folio);
1477 1477
1478 nr = __folio_add_rmap(folio, page, nr 1478 nr = __folio_add_rmap(folio, page, nr_pages, level, &nr_pmdmapped);
1479 __folio_mod_stat(folio, nr, nr_pmdmap 1479 __folio_mod_stat(folio, nr, nr_pmdmapped);
1480 1480
1481 /* See comments in folio_add_anon_rma 1481 /* See comments in folio_add_anon_rmap_*() */
1482 if (!folio_test_large(folio)) 1482 if (!folio_test_large(folio))
1483 mlock_vma_folio(folio, vma); 1483 mlock_vma_folio(folio, vma);
1484 } 1484 }
1485 1485
1486 /** 1486 /**
1487 * folio_add_file_rmap_ptes - add PTE mapping 1487 * folio_add_file_rmap_ptes - add PTE mappings to a page range of a folio
1488 * @folio: The folio to add the mappings 1488 * @folio: The folio to add the mappings to
1489 * @page: The first page to add 1489 * @page: The first page to add
1490 * @nr_pages: The number of pages that will 1490 * @nr_pages: The number of pages that will be mapped using PTEs
1491 * @vma: The vm area in which the mapp 1491 * @vma: The vm area in which the mappings are added
1492 * 1492 *
1493 * The page range of the folio is defined by 1493 * The page range of the folio is defined by [page, page + nr_pages)
1494 * 1494 *
1495 * The caller needs to hold the page table lo 1495 * The caller needs to hold the page table lock.
1496 */ 1496 */
1497 void folio_add_file_rmap_ptes(struct folio *f 1497 void folio_add_file_rmap_ptes(struct folio *folio, struct page *page,
1498 int nr_pages, struct vm_area_ 1498 int nr_pages, struct vm_area_struct *vma)
1499 { 1499 {
1500 __folio_add_file_rmap(folio, page, nr 1500 __folio_add_file_rmap(folio, page, nr_pages, vma, RMAP_LEVEL_PTE);
1501 } 1501 }
1502 1502
1503 /** 1503 /**
1504 * folio_add_file_rmap_pmd - add a PMD mappin 1504 * folio_add_file_rmap_pmd - add a PMD mapping to a page range of a folio
1505 * @folio: The folio to add the mapping 1505 * @folio: The folio to add the mapping to
1506 * @page: The first page to add 1506 * @page: The first page to add
1507 * @vma: The vm area in which the mapp 1507 * @vma: The vm area in which the mapping is added
1508 * 1508 *
1509 * The page range of the folio is defined by 1509 * The page range of the folio is defined by [page, page + HPAGE_PMD_NR)
1510 * 1510 *
1511 * The caller needs to hold the page table lo 1511 * The caller needs to hold the page table lock.
1512 */ 1512 */
1513 void folio_add_file_rmap_pmd(struct folio *fo 1513 void folio_add_file_rmap_pmd(struct folio *folio, struct page *page,
1514 struct vm_area_struct *vma) 1514 struct vm_area_struct *vma)
1515 { 1515 {
1516 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1516 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1517 __folio_add_file_rmap(folio, page, HP 1517 __folio_add_file_rmap(folio, page, HPAGE_PMD_NR, vma, RMAP_LEVEL_PMD);
1518 #else 1518 #else
1519 WARN_ON_ONCE(true); 1519 WARN_ON_ONCE(true);
1520 #endif 1520 #endif
1521 } 1521 }
1522 1522
1523 static __always_inline void __folio_remove_rm 1523 static __always_inline void __folio_remove_rmap(struct folio *folio,
1524 struct page *page, int nr_pag 1524 struct page *page, int nr_pages, struct vm_area_struct *vma,
1525 enum rmap_level level) 1525 enum rmap_level level)
1526 { 1526 {
1527 atomic_t *mapped = &folio->_nr_pages_ 1527 atomic_t *mapped = &folio->_nr_pages_mapped;
1528 int last = 0, nr = 0, nr_pmdmapped = 1528 int last = 0, nr = 0, nr_pmdmapped = 0;
1529 bool partially_mapped = false; 1529 bool partially_mapped = false;
1530 1530
1531 __folio_rmap_sanity_checks(folio, pag 1531 __folio_rmap_sanity_checks(folio, page, nr_pages, level);
1532 1532
1533 switch (level) { 1533 switch (level) {
1534 case RMAP_LEVEL_PTE: 1534 case RMAP_LEVEL_PTE:
1535 if (!folio_test_large(folio)) 1535 if (!folio_test_large(folio)) {
1536 nr = atomic_add_negat 1536 nr = atomic_add_negative(-1, &folio->_mapcount);
1537 break; 1537 break;
1538 } 1538 }
1539 1539
1540 atomic_sub(nr_pages, &folio-> 1540 atomic_sub(nr_pages, &folio->_large_mapcount);
1541 do { 1541 do {
1542 last += atomic_add_ne 1542 last += atomic_add_negative(-1, &page->_mapcount);
1543 } while (page++, --nr_pages > 1543 } while (page++, --nr_pages > 0);
1544 1544
1545 if (last && 1545 if (last &&
1546 atomic_sub_return_relaxed 1546 atomic_sub_return_relaxed(last, mapped) < ENTIRELY_MAPPED)
1547 nr = last; 1547 nr = last;
1548 1548
1549 partially_mapped = nr && atom 1549 partially_mapped = nr && atomic_read(mapped);
1550 break; 1550 break;
1551 case RMAP_LEVEL_PMD: 1551 case RMAP_LEVEL_PMD:
1552 atomic_dec(&folio->_large_map 1552 atomic_dec(&folio->_large_mapcount);
1553 last = atomic_add_negative(-1 1553 last = atomic_add_negative(-1, &folio->_entire_mapcount);
1554 if (last) { 1554 if (last) {
1555 nr = atomic_sub_retur 1555 nr = atomic_sub_return_relaxed(ENTIRELY_MAPPED, mapped);
1556 if (likely(nr < ENTIR 1556 if (likely(nr < ENTIRELY_MAPPED)) {
1557 nr_pmdmapped 1557 nr_pmdmapped = folio_nr_pages(folio);
1558 nr = nr_pmdma 1558 nr = nr_pmdmapped - (nr & FOLIO_PAGES_MAPPED);
1559 /* Raced ahea 1559 /* Raced ahead of another remove and an add? */
1560 if (unlikely( 1560 if (unlikely(nr < 0))
1561 nr = 1561 nr = 0;
1562 } else { 1562 } else {
1563 /* An add of 1563 /* An add of ENTIRELY_MAPPED raced ahead */
1564 nr = 0; 1564 nr = 0;
1565 } 1565 }
1566 } 1566 }
1567 1567
1568 partially_mapped = nr && nr < 1568 partially_mapped = nr && nr < nr_pmdmapped;
1569 break; 1569 break;
1570 } 1570 }
1571 1571
1572 /* 1572 /*
1573 * Queue anon large folio for deferre 1573 * Queue anon large folio for deferred split if at least one page of
1574 * the folio is unmapped and at least 1574 * the folio is unmapped and at least one page is still mapped.
1575 * 1575 *
1576 * Check partially_mapped first to en 1576 * Check partially_mapped first to ensure it is a large folio.
1577 */ 1577 */
1578 if (partially_mapped && folio_test_an 1578 if (partially_mapped && folio_test_anon(folio) &&
1579 !folio_test_partially_mapped(foli 1579 !folio_test_partially_mapped(folio))
1580 deferred_split_folio(folio, t 1580 deferred_split_folio(folio, true);
1581 1581
1582 __folio_mod_stat(folio, -nr, -nr_pmdm 1582 __folio_mod_stat(folio, -nr, -nr_pmdmapped);
1583 1583
1584 /* 1584 /*
1585 * It would be tidy to reset folio_te 1585 * It would be tidy to reset folio_test_anon mapping when fully
1586 * unmapped, but that might overwrite 1586 * unmapped, but that might overwrite a racing folio_add_anon_rmap_*()
1587 * which increments mapcount after us 1587 * which increments mapcount after us but sets mapping before us:
1588 * so leave the reset to free_pages_p 1588 * so leave the reset to free_pages_prepare, and remember that
1589 * it's only reliable while mapped. 1589 * it's only reliable while mapped.
1590 */ 1590 */
1591 1591
1592 munlock_vma_folio(folio, vma); 1592 munlock_vma_folio(folio, vma);
1593 } 1593 }
1594 1594
1595 /** 1595 /**
1596 * folio_remove_rmap_ptes - remove PTE mappin 1596 * folio_remove_rmap_ptes - remove PTE mappings from a page range of a folio
1597 * @folio: The folio to remove the mappi 1597 * @folio: The folio to remove the mappings from
1598 * @page: The first page to remove 1598 * @page: The first page to remove
1599 * @nr_pages: The number of pages that will 1599 * @nr_pages: The number of pages that will be removed from the mapping
1600 * @vma: The vm area from which the ma 1600 * @vma: The vm area from which the mappings are removed
1601 * 1601 *
1602 * The page range of the folio is defined by 1602 * The page range of the folio is defined by [page, page + nr_pages)
1603 * 1603 *
1604 * The caller needs to hold the page table lo 1604 * The caller needs to hold the page table lock.
1605 */ 1605 */
1606 void folio_remove_rmap_ptes(struct folio *fol 1606 void folio_remove_rmap_ptes(struct folio *folio, struct page *page,
1607 int nr_pages, struct vm_area_ 1607 int nr_pages, struct vm_area_struct *vma)
1608 { 1608 {
1609 __folio_remove_rmap(folio, page, nr_p 1609 __folio_remove_rmap(folio, page, nr_pages, vma, RMAP_LEVEL_PTE);
1610 } 1610 }
1611 1611
1612 /** 1612 /**
1613 * folio_remove_rmap_pmd - remove a PMD mappi 1613 * folio_remove_rmap_pmd - remove a PMD mapping from a page range of a folio
1614 * @folio: The folio to remove the mappi 1614 * @folio: The folio to remove the mapping from
1615 * @page: The first page to remove 1615 * @page: The first page to remove
1616 * @vma: The vm area from which the ma 1616 * @vma: The vm area from which the mapping is removed
1617 * 1617 *
1618 * The page range of the folio is defined by 1618 * The page range of the folio is defined by [page, page + HPAGE_PMD_NR)
1619 * 1619 *
1620 * The caller needs to hold the page table lo 1620 * The caller needs to hold the page table lock.
1621 */ 1621 */
1622 void folio_remove_rmap_pmd(struct folio *foli 1622 void folio_remove_rmap_pmd(struct folio *folio, struct page *page,
1623 struct vm_area_struct *vma) 1623 struct vm_area_struct *vma)
1624 { 1624 {
1625 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1625 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1626 __folio_remove_rmap(folio, page, HPAG 1626 __folio_remove_rmap(folio, page, HPAGE_PMD_NR, vma, RMAP_LEVEL_PMD);
1627 #else 1627 #else
1628 WARN_ON_ONCE(true); 1628 WARN_ON_ONCE(true);
1629 #endif 1629 #endif
1630 } 1630 }
1631 1631
1632 /* 1632 /*
1633 * @arg: enum ttu_flags will be passed to thi 1633 * @arg: enum ttu_flags will be passed to this argument
1634 */ 1634 */
1635 static bool try_to_unmap_one(struct folio *fo 1635 static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
1636 unsigned long address, v 1636 unsigned long address, void *arg)
1637 { 1637 {
1638 struct mm_struct *mm = vma->vm_mm; 1638 struct mm_struct *mm = vma->vm_mm;
1639 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vm 1639 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);
1640 pte_t pteval; 1640 pte_t pteval;
1641 struct page *subpage; 1641 struct page *subpage;
1642 bool anon_exclusive, ret = true; 1642 bool anon_exclusive, ret = true;
1643 struct mmu_notifier_range range; 1643 struct mmu_notifier_range range;
1644 enum ttu_flags flags = (enum ttu_flag 1644 enum ttu_flags flags = (enum ttu_flags)(long)arg;
1645 unsigned long pfn; 1645 unsigned long pfn;
1646 unsigned long hsz = 0; 1646 unsigned long hsz = 0;
1647 1647
1648 /* 1648 /*
1649 * When racing against e.g. zap_pte_r 1649 * When racing against e.g. zap_pte_range() on another cpu,
1650 * in between its ptep_get_and_clear_ 1650 * in between its ptep_get_and_clear_full() and folio_remove_rmap_*(),
1651 * try_to_unmap() may return before p 1651 * try_to_unmap() may return before page_mapped() has become false,
1652 * if page table locking is skipped: 1652 * if page table locking is skipped: use TTU_SYNC to wait for that.
1653 */ 1653 */
1654 if (flags & TTU_SYNC) 1654 if (flags & TTU_SYNC)
1655 pvmw.flags = PVMW_SYNC; 1655 pvmw.flags = PVMW_SYNC;
1656 1656
1657 /* 1657 /*
1658 * For THP, we have to assume the wor 1658 * For THP, we have to assume the worse case ie pmd for invalidation.
1659 * For hugetlb, it could be much wors 1659 * For hugetlb, it could be much worse if we need to do pud
1660 * invalidation in the case of pmd sh 1660 * invalidation in the case of pmd sharing.
1661 * 1661 *
1662 * Note that the folio can not be fre 1662 * Note that the folio can not be freed in this function as call of
1663 * try_to_unmap() must hold a referen 1663 * try_to_unmap() must hold a reference on the folio.
1664 */ 1664 */
1665 range.end = vma_address_end(&pvmw); 1665 range.end = vma_address_end(&pvmw);
1666 mmu_notifier_range_init(&range, MMU_N 1666 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm,
1667 address, rang 1667 address, range.end);
1668 if (folio_test_hugetlb(folio)) { 1668 if (folio_test_hugetlb(folio)) {
1669 /* 1669 /*
1670 * If sharing is possible, st 1670 * If sharing is possible, start and end will be adjusted
1671 * accordingly. 1671 * accordingly.
1672 */ 1672 */
1673 adjust_range_if_pmd_sharing_p 1673 adjust_range_if_pmd_sharing_possible(vma, &range.start,
1674 1674 &range.end);
1675 1675
1676 /* We need the huge page size 1676 /* We need the huge page size for set_huge_pte_at() */
1677 hsz = huge_page_size(hstate_v 1677 hsz = huge_page_size(hstate_vma(vma));
1678 } 1678 }
1679 mmu_notifier_invalidate_range_start(& 1679 mmu_notifier_invalidate_range_start(&range);
1680 1680
1681 while (page_vma_mapped_walk(&pvmw)) { 1681 while (page_vma_mapped_walk(&pvmw)) {
1682 /* 1682 /*
1683 * If the folio is in an mloc 1683 * If the folio is in an mlock()d vma, we must not swap it out.
1684 */ 1684 */
1685 if (!(flags & TTU_IGNORE_MLOC 1685 if (!(flags & TTU_IGNORE_MLOCK) &&
1686 (vma->vm_flags & VM_LOCKE 1686 (vma->vm_flags & VM_LOCKED)) {
1687 /* Restore the mlock 1687 /* Restore the mlock which got missed */
1688 if (!folio_test_large 1688 if (!folio_test_large(folio))
1689 mlock_vma_fol 1689 mlock_vma_folio(folio, vma);
1690 goto walk_abort; 1690 goto walk_abort;
1691 } 1691 }
1692 1692
1693 if (!pvmw.pte) { 1693 if (!pvmw.pte) {
1694 if (unmap_huge_pmd_lo 1694 if (unmap_huge_pmd_locked(vma, pvmw.address, pvmw.pmd,
1695 1695 folio))
1696 goto walk_don 1696 goto walk_done;
1697 1697
1698 if (flags & TTU_SPLIT 1698 if (flags & TTU_SPLIT_HUGE_PMD) {
1699 /* 1699 /*
1700 * We tempora 1700 * We temporarily have to drop the PTL and
1701 * restart so 1701 * restart so we can process the PTE-mapped THP.
1702 */ 1702 */
1703 split_huge_pm 1703 split_huge_pmd_locked(vma, pvmw.address,
1704 1704 pvmw.pmd, false, folio);
1705 flags &= ~TTU 1705 flags &= ~TTU_SPLIT_HUGE_PMD;
1706 page_vma_mapp 1706 page_vma_mapped_walk_restart(&pvmw);
1707 continue; 1707 continue;
1708 } 1708 }
1709 } 1709 }
1710 1710
1711 /* Unexpected PMD-mapped THP? 1711 /* Unexpected PMD-mapped THP? */
1712 VM_BUG_ON_FOLIO(!pvmw.pte, fo 1712 VM_BUG_ON_FOLIO(!pvmw.pte, folio);
1713 1713
1714 pfn = pte_pfn(ptep_get(pvmw.p 1714 pfn = pte_pfn(ptep_get(pvmw.pte));
1715 subpage = folio_page(folio, p 1715 subpage = folio_page(folio, pfn - folio_pfn(folio));
1716 address = pvmw.address; 1716 address = pvmw.address;
1717 anon_exclusive = folio_test_a 1717 anon_exclusive = folio_test_anon(folio) &&
1718 PageAnonExcl 1718 PageAnonExclusive(subpage);
1719 1719
1720 if (folio_test_hugetlb(folio) 1720 if (folio_test_hugetlb(folio)) {
1721 bool anon = folio_tes 1721 bool anon = folio_test_anon(folio);
1722 1722
1723 /* 1723 /*
1724 * The try_to_unmap() 1724 * The try_to_unmap() is only passed a hugetlb page
1725 * in the case where 1725 * in the case where the hugetlb page is poisoned.
1726 */ 1726 */
1727 VM_BUG_ON_PAGE(!PageH 1727 VM_BUG_ON_PAGE(!PageHWPoison(subpage), subpage);
1728 /* 1728 /*
1729 * huge_pmd_unshare m 1729 * huge_pmd_unshare may unmap an entire PMD page.
1730 * There is no way of 1730 * There is no way of knowing exactly which PMDs may
1731 * be cached for this 1731 * be cached for this mm, so we must flush them all.
1732 * start/end were alr 1732 * start/end were already adjusted above to cover this
1733 * range. 1733 * range.
1734 */ 1734 */
1735 flush_cache_range(vma 1735 flush_cache_range(vma, range.start, range.end);
1736 1736
1737 /* 1737 /*
1738 * To call huge_pmd_u 1738 * To call huge_pmd_unshare, i_mmap_rwsem must be
1739 * held in write mode 1739 * held in write mode. Caller needs to explicitly
1740 * do this outside rm 1740 * do this outside rmap routines.
1741 * 1741 *
1742 * We also must hold 1742 * We also must hold hugetlb vma_lock in write mode.
1743 * Lock order dictate 1743 * Lock order dictates acquiring vma_lock BEFORE
1744 * i_mmap_rwsem. We 1744 * i_mmap_rwsem. We can only try lock here and fail
1745 * if unsuccessful. 1745 * if unsuccessful.
1746 */ 1746 */
1747 if (!anon) { 1747 if (!anon) {
1748 VM_BUG_ON(!(f 1748 VM_BUG_ON(!(flags & TTU_RMAP_LOCKED));
1749 if (!hugetlb_ 1749 if (!hugetlb_vma_trylock_write(vma))
1750 goto 1750 goto walk_abort;
1751 if (huge_pmd_ 1751 if (huge_pmd_unshare(mm, vma, address, pvmw.pte)) {
1752 huget 1752 hugetlb_vma_unlock_write(vma);
1753 flush 1753 flush_tlb_range(vma,
1754 1754 range.start, range.end);
1755 /* 1755 /*
1756 * Th 1756 * The ref count of the PMD page was
1757 * dr 1757 * dropped which is part of the way map
1758 * co 1758 * counting is done for shared PMDs.
1759 * Re 1759 * Return 'true' here. When there is
1760 * no 1760 * no other sharing, huge_pmd_unshare
1761 * re 1761 * returns false and we will unmap the
1762 * ac 1762 * actual page and drop map count
1763 * to 1763 * to zero.
1764 */ 1764 */
1765 goto 1765 goto walk_done;
1766 } 1766 }
1767 hugetlb_vma_u 1767 hugetlb_vma_unlock_write(vma);
1768 } 1768 }
1769 pteval = huge_ptep_cl 1769 pteval = huge_ptep_clear_flush(vma, address, pvmw.pte);
1770 } else { 1770 } else {
1771 flush_cache_page(vma, 1771 flush_cache_page(vma, address, pfn);
1772 /* Nuke the page tabl 1772 /* Nuke the page table entry. */
1773 if (should_defer_flus 1773 if (should_defer_flush(mm, flags)) {
1774 /* 1774 /*
1775 * We clear t 1775 * We clear the PTE but do not flush so potentially
1776 * a remote C 1776 * a remote CPU could still be writing to the folio.
1777 * If the ent 1777 * If the entry was previously clean then the
1778 * architectu 1778 * architecture must guarantee that a clear->dirty
1779 * transition 1779 * transition on a cached TLB entry is written through
1780 * and traps 1780 * and traps if the PTE is unmapped.
1781 */ 1781 */
1782 pteval = ptep 1782 pteval = ptep_get_and_clear(mm, address, pvmw.pte);
1783 1783
1784 set_tlb_ubc_f 1784 set_tlb_ubc_flush_pending(mm, pteval, address);
1785 } else { 1785 } else {
1786 pteval = ptep 1786 pteval = ptep_clear_flush(vma, address, pvmw.pte);
1787 } 1787 }
1788 } 1788 }
1789 1789
1790 /* 1790 /*
1791 * Now the pte is cleared. If 1791 * Now the pte is cleared. If this pte was uffd-wp armed,
1792 * we may want to replace a n 1792 * we may want to replace a none pte with a marker pte if
1793 * it's file-backed, so we do 1793 * it's file-backed, so we don't lose the tracking info.
1794 */ 1794 */
1795 pte_install_uffd_wp_if_needed 1795 pte_install_uffd_wp_if_needed(vma, address, pvmw.pte, pteval);
1796 1796
1797 /* Set the dirty flag on the 1797 /* Set the dirty flag on the folio now the pte is gone. */
1798 if (pte_dirty(pteval)) 1798 if (pte_dirty(pteval))
1799 folio_mark_dirty(foli 1799 folio_mark_dirty(folio);
1800 1800
1801 /* Update high watermark befo 1801 /* Update high watermark before we lower rss */
1802 update_hiwater_rss(mm); 1802 update_hiwater_rss(mm);
1803 1803
1804 if (PageHWPoison(subpage) && 1804 if (PageHWPoison(subpage) && (flags & TTU_HWPOISON)) {
1805 pteval = swp_entry_to 1805 pteval = swp_entry_to_pte(make_hwpoison_entry(subpage));
1806 if (folio_test_hugetl 1806 if (folio_test_hugetlb(folio)) {
1807 hugetlb_count 1807 hugetlb_count_sub(folio_nr_pages(folio), mm);
1808 set_huge_pte_ 1808 set_huge_pte_at(mm, address, pvmw.pte, pteval,
1809 1809 hsz);
1810 } else { 1810 } else {
1811 dec_mm_counte 1811 dec_mm_counter(mm, mm_counter(folio));
1812 set_pte_at(mm 1812 set_pte_at(mm, address, pvmw.pte, pteval);
1813 } 1813 }
1814 1814
1815 } else if (pte_unused(pteval) 1815 } else if (pte_unused(pteval) && !userfaultfd_armed(vma)) {
1816 /* 1816 /*
1817 * The guest indicate 1817 * The guest indicated that the page content is of no
1818 * interest anymore. 1818 * interest anymore. Simply discard the pte, vmscan
1819 * will take care of 1819 * will take care of the rest.
1820 * A future reference 1820 * A future reference will then fault in a new zero
1821 * page. When userfau 1821 * page. When userfaultfd is active, we must not drop
1822 * this page though, 1822 * this page though, as its main user (postcopy
1823 * migration) will no 1823 * migration) will not expect userfaults on already
1824 * copied pages. 1824 * copied pages.
1825 */ 1825 */
1826 dec_mm_counter(mm, mm 1826 dec_mm_counter(mm, mm_counter(folio));
1827 } else if (folio_test_anon(fo 1827 } else if (folio_test_anon(folio)) {
1828 swp_entry_t entry = p 1828 swp_entry_t entry = page_swap_entry(subpage);
1829 pte_t swp_pte; 1829 pte_t swp_pte;
1830 /* 1830 /*
1831 * Store the swap loc 1831 * Store the swap location in the pte.
1832 * See handle_pte_fau 1832 * See handle_pte_fault() ...
1833 */ 1833 */
1834 if (unlikely(folio_te 1834 if (unlikely(folio_test_swapbacked(folio) !=
1835 folio 1835 folio_test_swapcache(folio))) {
1836 WARN_ON_ONCE( 1836 WARN_ON_ONCE(1);
1837 goto walk_abo 1837 goto walk_abort;
1838 } 1838 }
1839 1839
1840 /* MADV_FREE page che 1840 /* MADV_FREE page check */
1841 if (!folio_test_swapb 1841 if (!folio_test_swapbacked(folio)) {
1842 int ref_count 1842 int ref_count, map_count;
1843 1843
1844 /* 1844 /*
1845 * Synchroniz 1845 * Synchronize with gup_pte_range():
1846 * - clear PT 1846 * - clear PTE; barrier; read refcount
1847 * - inc refc 1847 * - inc refcount; barrier; read PTE
1848 */ 1848 */
1849 smp_mb(); 1849 smp_mb();
1850 1850
1851 ref_count = f 1851 ref_count = folio_ref_count(folio);
1852 map_count = f 1852 map_count = folio_mapcount(folio);
1853 1853
1854 /* 1854 /*
1855 * Order read 1855 * Order reads for page refcount and dirty flag
1856 * (see comme 1856 * (see comments in __remove_mapping()).
1857 */ 1857 */
1858 smp_rmb(); 1858 smp_rmb();
1859 1859
1860 /* 1860 /*
1861 * The only p 1861 * The only page refs must be one from isolation
1862 * plus the r 1862 * plus the rmap(s) (dropped by discard:).
1863 */ 1863 */
1864 if (ref_count 1864 if (ref_count == 1 + map_count &&
1865 (!folio_t 1865 (!folio_test_dirty(folio) ||
1866 /* 1866 /*
1867 * Unlik 1867 * Unlike MADV_FREE mappings, VM_DROPPABLE
1868 * ones 1868 * ones can be dropped even if they've
1869 * been 1869 * been dirtied.
1870 */ 1870 */
1871 (vma->vm 1871 (vma->vm_flags & VM_DROPPABLE))) {
1872 dec_m 1872 dec_mm_counter(mm, MM_ANONPAGES);
1873 goto 1873 goto discard;
1874 } 1874 }
1875 1875
1876 /* 1876 /*
1877 * If the fol 1877 * If the folio was redirtied, it cannot be
1878 * discarded. 1878 * discarded. Remap the page to page table.
1879 */ 1879 */
1880 set_pte_at(mm 1880 set_pte_at(mm, address, pvmw.pte, pteval);
1881 /* 1881 /*
1882 * Unlike MAD 1882 * Unlike MADV_FREE mappings, VM_DROPPABLE ones
1883 * never get 1883 * never get swap backed on failure to drop.
1884 */ 1884 */
1885 if (!(vma->vm 1885 if (!(vma->vm_flags & VM_DROPPABLE))
1886 folio 1886 folio_set_swapbacked(folio);
1887 goto walk_abo 1887 goto walk_abort;
1888 } 1888 }
1889 1889
1890 if (swap_duplicate(en 1890 if (swap_duplicate(entry) < 0) {
1891 set_pte_at(mm 1891 set_pte_at(mm, address, pvmw.pte, pteval);
1892 goto walk_abo 1892 goto walk_abort;
1893 } 1893 }
1894 if (arch_unmap_one(mm 1894 if (arch_unmap_one(mm, vma, address, pteval) < 0) {
1895 swap_free(ent 1895 swap_free(entry);
1896 set_pte_at(mm 1896 set_pte_at(mm, address, pvmw.pte, pteval);
1897 goto walk_abo 1897 goto walk_abort;
1898 } 1898 }
1899 1899
1900 /* See folio_try_shar 1900 /* See folio_try_share_anon_rmap(): clear PTE first. */
1901 if (anon_exclusive && 1901 if (anon_exclusive &&
1902 folio_try_share_a 1902 folio_try_share_anon_rmap_pte(folio, subpage)) {
1903 swap_free(ent 1903 swap_free(entry);
1904 set_pte_at(mm 1904 set_pte_at(mm, address, pvmw.pte, pteval);
1905 goto walk_abo 1905 goto walk_abort;
1906 } 1906 }
1907 if (list_empty(&mm->m 1907 if (list_empty(&mm->mmlist)) {
1908 spin_lock(&mm 1908 spin_lock(&mmlist_lock);
1909 if (list_empt 1909 if (list_empty(&mm->mmlist))
1910 list_ 1910 list_add(&mm->mmlist, &init_mm.mmlist);
1911 spin_unlock(& 1911 spin_unlock(&mmlist_lock);
1912 } 1912 }
1913 dec_mm_counter(mm, MM 1913 dec_mm_counter(mm, MM_ANONPAGES);
1914 inc_mm_counter(mm, MM 1914 inc_mm_counter(mm, MM_SWAPENTS);
1915 swp_pte = swp_entry_t 1915 swp_pte = swp_entry_to_pte(entry);
1916 if (anon_exclusive) 1916 if (anon_exclusive)
1917 swp_pte = pte 1917 swp_pte = pte_swp_mkexclusive(swp_pte);
1918 if (pte_soft_dirty(pt 1918 if (pte_soft_dirty(pteval))
1919 swp_pte = pte 1919 swp_pte = pte_swp_mksoft_dirty(swp_pte);
1920 if (pte_uffd_wp(pteva 1920 if (pte_uffd_wp(pteval))
1921 swp_pte = pte 1921 swp_pte = pte_swp_mkuffd_wp(swp_pte);
1922 set_pte_at(mm, addres 1922 set_pte_at(mm, address, pvmw.pte, swp_pte);
1923 } else { 1923 } else {
1924 /* 1924 /*
1925 * This is a locked f 1925 * This is a locked file-backed folio,
1926 * so it cannot be re 1926 * so it cannot be removed from the page
1927 * cache and replaced 1927 * cache and replaced by a new folio before
1928 * mmu_notifier_inval 1928 * mmu_notifier_invalidate_range_end, so no
1929 * concurrent thread 1929 * concurrent thread might update its page table
1930 * to point at a new 1930 * to point at a new folio while a device is
1931 * still using this f 1931 * still using this folio.
1932 * 1932 *
1933 * See Documentation/ 1933 * See Documentation/mm/mmu_notifier.rst
1934 */ 1934 */
1935 dec_mm_counter(mm, mm 1935 dec_mm_counter(mm, mm_counter_file(folio));
1936 } 1936 }
1937 discard: 1937 discard:
1938 if (unlikely(folio_test_huget 1938 if (unlikely(folio_test_hugetlb(folio)))
1939 hugetlb_remove_rmap(f 1939 hugetlb_remove_rmap(folio);
1940 else 1940 else
1941 folio_remove_rmap_pte 1941 folio_remove_rmap_pte(folio, subpage, vma);
1942 if (vma->vm_flags & VM_LOCKED 1942 if (vma->vm_flags & VM_LOCKED)
1943 mlock_drain_local(); 1943 mlock_drain_local();
1944 folio_put(folio); 1944 folio_put(folio);
1945 continue; 1945 continue;
1946 walk_abort: 1946 walk_abort:
1947 ret = false; 1947 ret = false;
1948 walk_done: 1948 walk_done:
1949 page_vma_mapped_walk_done(&pv 1949 page_vma_mapped_walk_done(&pvmw);
1950 break; 1950 break;
1951 } 1951 }
1952 1952
1953 mmu_notifier_invalidate_range_end(&ra 1953 mmu_notifier_invalidate_range_end(&range);
1954 1954
1955 return ret; 1955 return ret;
1956 } 1956 }
1957 1957
1958 static bool invalid_migration_vma(struct vm_a 1958 static bool invalid_migration_vma(struct vm_area_struct *vma, void *arg)
1959 { 1959 {
1960 return vma_is_temporary_stack(vma); 1960 return vma_is_temporary_stack(vma);
1961 } 1961 }
1962 1962
1963 static int folio_not_mapped(struct folio *fol 1963 static int folio_not_mapped(struct folio *folio)
1964 { 1964 {
1965 return !folio_mapped(folio); 1965 return !folio_mapped(folio);
1966 } 1966 }
1967 1967
1968 /** 1968 /**
1969 * try_to_unmap - Try to remove all page tabl 1969 * try_to_unmap - Try to remove all page table mappings to a folio.
1970 * @folio: The folio to unmap. 1970 * @folio: The folio to unmap.
1971 * @flags: action and flags 1971 * @flags: action and flags
1972 * 1972 *
1973 * Tries to remove all the page table entries 1973 * Tries to remove all the page table entries which are mapping this
1974 * folio. It is the caller's responsibility 1974 * folio. It is the caller's responsibility to check if the folio is
1975 * still mapped if needed (use TTU_SYNC to pr 1975 * still mapped if needed (use TTU_SYNC to prevent accounting races).
1976 * 1976 *
1977 * Context: Caller must hold the folio lock. 1977 * Context: Caller must hold the folio lock.
1978 */ 1978 */
1979 void try_to_unmap(struct folio *folio, enum t 1979 void try_to_unmap(struct folio *folio, enum ttu_flags flags)
1980 { 1980 {
1981 struct rmap_walk_control rwc = { 1981 struct rmap_walk_control rwc = {
1982 .rmap_one = try_to_unmap_one, 1982 .rmap_one = try_to_unmap_one,
1983 .arg = (void *)flags, 1983 .arg = (void *)flags,
1984 .done = folio_not_mapped, 1984 .done = folio_not_mapped,
1985 .anon_lock = folio_lock_anon_ 1985 .anon_lock = folio_lock_anon_vma_read,
1986 }; 1986 };
1987 1987
1988 if (flags & TTU_RMAP_LOCKED) 1988 if (flags & TTU_RMAP_LOCKED)
1989 rmap_walk_locked(folio, &rwc) 1989 rmap_walk_locked(folio, &rwc);
1990 else 1990 else
1991 rmap_walk(folio, &rwc); 1991 rmap_walk(folio, &rwc);
1992 } 1992 }
1993 1993
1994 /* 1994 /*
1995 * @arg: enum ttu_flags will be passed to thi 1995 * @arg: enum ttu_flags will be passed to this argument.
1996 * 1996 *
1997 * If TTU_SPLIT_HUGE_PMD is specified any PMD 1997 * If TTU_SPLIT_HUGE_PMD is specified any PMD mappings will be split into PTEs
1998 * containing migration entries. 1998 * containing migration entries.
1999 */ 1999 */
2000 static bool try_to_migrate_one(struct folio * 2000 static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma,
2001 unsigned long address, v 2001 unsigned long address, void *arg)
2002 { 2002 {
2003 struct mm_struct *mm = vma->vm_mm; 2003 struct mm_struct *mm = vma->vm_mm;
2004 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vm 2004 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);
2005 pte_t pteval; 2005 pte_t pteval;
2006 struct page *subpage; 2006 struct page *subpage;
2007 bool anon_exclusive, ret = true; 2007 bool anon_exclusive, ret = true;
2008 struct mmu_notifier_range range; 2008 struct mmu_notifier_range range;
2009 enum ttu_flags flags = (enum ttu_flag 2009 enum ttu_flags flags = (enum ttu_flags)(long)arg;
2010 unsigned long pfn; 2010 unsigned long pfn;
2011 unsigned long hsz = 0; 2011 unsigned long hsz = 0;
2012 2012
2013 /* 2013 /*
2014 * When racing against e.g. zap_pte_r 2014 * When racing against e.g. zap_pte_range() on another cpu,
2015 * in between its ptep_get_and_clear_ 2015 * in between its ptep_get_and_clear_full() and folio_remove_rmap_*(),
2016 * try_to_migrate() may return before 2016 * try_to_migrate() may return before page_mapped() has become false,
2017 * if page table locking is skipped: 2017 * if page table locking is skipped: use TTU_SYNC to wait for that.
2018 */ 2018 */
2019 if (flags & TTU_SYNC) 2019 if (flags & TTU_SYNC)
2020 pvmw.flags = PVMW_SYNC; 2020 pvmw.flags = PVMW_SYNC;
2021 2021
2022 /* 2022 /*
2023 * unmap_page() in mm/huge_memory.c i 2023 * unmap_page() in mm/huge_memory.c is the only user of migration with
2024 * TTU_SPLIT_HUGE_PMD and it wants to 2024 * TTU_SPLIT_HUGE_PMD and it wants to freeze.
2025 */ 2025 */
2026 if (flags & TTU_SPLIT_HUGE_PMD) 2026 if (flags & TTU_SPLIT_HUGE_PMD)
2027 split_huge_pmd_address(vma, a 2027 split_huge_pmd_address(vma, address, true, folio);
2028 2028
2029 /* 2029 /*
2030 * For THP, we have to assume the wor 2030 * For THP, we have to assume the worse case ie pmd for invalidation.
2031 * For hugetlb, it could be much wors 2031 * For hugetlb, it could be much worse if we need to do pud
2032 * invalidation in the case of pmd sh 2032 * invalidation in the case of pmd sharing.
2033 * 2033 *
2034 * Note that the page can not be free 2034 * Note that the page can not be free in this function as call of
2035 * try_to_unmap() must hold a referen 2035 * try_to_unmap() must hold a reference on the page.
2036 */ 2036 */
2037 range.end = vma_address_end(&pvmw); 2037 range.end = vma_address_end(&pvmw);
2038 mmu_notifier_range_init(&range, MMU_N 2038 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm,
2039 address, rang 2039 address, range.end);
2040 if (folio_test_hugetlb(folio)) { 2040 if (folio_test_hugetlb(folio)) {
2041 /* 2041 /*
2042 * If sharing is possible, st 2042 * If sharing is possible, start and end will be adjusted
2043 * accordingly. 2043 * accordingly.
2044 */ 2044 */
2045 adjust_range_if_pmd_sharing_p 2045 adjust_range_if_pmd_sharing_possible(vma, &range.start,
2046 2046 &range.end);
2047 2047
2048 /* We need the huge page size 2048 /* We need the huge page size for set_huge_pte_at() */
2049 hsz = huge_page_size(hstate_v 2049 hsz = huge_page_size(hstate_vma(vma));
2050 } 2050 }
2051 mmu_notifier_invalidate_range_start(& 2051 mmu_notifier_invalidate_range_start(&range);
2052 2052
2053 while (page_vma_mapped_walk(&pvmw)) { 2053 while (page_vma_mapped_walk(&pvmw)) {
2054 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION 2054 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
2055 /* PMD-mapped THP migration e 2055 /* PMD-mapped THP migration entry */
2056 if (!pvmw.pte) { 2056 if (!pvmw.pte) {
2057 subpage = folio_page( 2057 subpage = folio_page(folio,
2058 pmd_pfn(*pvmw 2058 pmd_pfn(*pvmw.pmd) - folio_pfn(folio));
2059 VM_BUG_ON_FOLIO(folio 2059 VM_BUG_ON_FOLIO(folio_test_hugetlb(folio) ||
2060 !foli 2060 !folio_test_pmd_mappable(folio), folio);
2061 2061
2062 if (set_pmd_migration 2062 if (set_pmd_migration_entry(&pvmw, subpage)) {
2063 ret = false; 2063 ret = false;
2064 page_vma_mapp 2064 page_vma_mapped_walk_done(&pvmw);
2065 break; 2065 break;
2066 } 2066 }
2067 continue; 2067 continue;
2068 } 2068 }
2069 #endif 2069 #endif
2070 2070
2071 /* Unexpected PMD-mapped THP? 2071 /* Unexpected PMD-mapped THP? */
2072 VM_BUG_ON_FOLIO(!pvmw.pte, fo 2072 VM_BUG_ON_FOLIO(!pvmw.pte, folio);
2073 2073
2074 pfn = pte_pfn(ptep_get(pvmw.p 2074 pfn = pte_pfn(ptep_get(pvmw.pte));
2075 2075
2076 if (folio_is_zone_device(foli 2076 if (folio_is_zone_device(folio)) {
2077 /* 2077 /*
2078 * Our PTE is a non-p 2078 * Our PTE is a non-present device exclusive entry and
2079 * calculating the su 2079 * calculating the subpage as for the common case would
2080 * result in an inval 2080 * result in an invalid pointer.
2081 * 2081 *
2082 * Since only PAGE_SI 2082 * Since only PAGE_SIZE pages can currently be
2083 * migrated, just set 2083 * migrated, just set it to page. This will need to be
2084 * changed when hugep 2084 * changed when hugepage migrations to device private
2085 * memory are support 2085 * memory are supported.
2086 */ 2086 */
2087 VM_BUG_ON_FOLIO(folio 2087 VM_BUG_ON_FOLIO(folio_nr_pages(folio) > 1, folio);
2088 subpage = &folio->pag 2088 subpage = &folio->page;
2089 } else { 2089 } else {
2090 subpage = folio_page( 2090 subpage = folio_page(folio, pfn - folio_pfn(folio));
2091 } 2091 }
2092 address = pvmw.address; 2092 address = pvmw.address;
2093 anon_exclusive = folio_test_a 2093 anon_exclusive = folio_test_anon(folio) &&
2094 PageAnonExcl 2094 PageAnonExclusive(subpage);
2095 2095
2096 if (folio_test_hugetlb(folio) 2096 if (folio_test_hugetlb(folio)) {
2097 bool anon = folio_tes 2097 bool anon = folio_test_anon(folio);
2098 2098
2099 /* 2099 /*
2100 * huge_pmd_unshare m 2100 * huge_pmd_unshare may unmap an entire PMD page.
2101 * There is no way of 2101 * There is no way of knowing exactly which PMDs may
2102 * be cached for this 2102 * be cached for this mm, so we must flush them all.
2103 * start/end were alr 2103 * start/end were already adjusted above to cover this
2104 * range. 2104 * range.
2105 */ 2105 */
2106 flush_cache_range(vma 2106 flush_cache_range(vma, range.start, range.end);
2107 2107
2108 /* 2108 /*
2109 * To call huge_pmd_u 2109 * To call huge_pmd_unshare, i_mmap_rwsem must be
2110 * held in write mode 2110 * held in write mode. Caller needs to explicitly
2111 * do this outside rm 2111 * do this outside rmap routines.
2112 * 2112 *
2113 * We also must hold 2113 * We also must hold hugetlb vma_lock in write mode.
2114 * Lock order dictate 2114 * Lock order dictates acquiring vma_lock BEFORE
2115 * i_mmap_rwsem. We 2115 * i_mmap_rwsem. We can only try lock here and
2116 * fail if unsuccessf 2116 * fail if unsuccessful.
2117 */ 2117 */
2118 if (!anon) { 2118 if (!anon) {
2119 VM_BUG_ON(!(f 2119 VM_BUG_ON(!(flags & TTU_RMAP_LOCKED));
2120 if (!hugetlb_ 2120 if (!hugetlb_vma_trylock_write(vma)) {
2121 page_ 2121 page_vma_mapped_walk_done(&pvmw);
2122 ret = 2122 ret = false;
2123 break 2123 break;
2124 } 2124 }
2125 if (huge_pmd_ 2125 if (huge_pmd_unshare(mm, vma, address, pvmw.pte)) {
2126 huget 2126 hugetlb_vma_unlock_write(vma);
2127 flush 2127 flush_tlb_range(vma,
2128 2128 range.start, range.end);
2129 2129
2130 /* 2130 /*
2131 * Th 2131 * The ref count of the PMD page was
2132 * dr 2132 * dropped which is part of the way map
2133 * co 2133 * counting is done for shared PMDs.
2134 * Re 2134 * Return 'true' here. When there is
2135 * no 2135 * no other sharing, huge_pmd_unshare
2136 * re 2136 * returns false and we will unmap the
2137 * ac 2137 * actual page and drop map count
2138 * to 2138 * to zero.
2139 */ 2139 */
2140 page_ 2140 page_vma_mapped_walk_done(&pvmw);
2141 break 2141 break;
2142 } 2142 }
2143 hugetlb_vma_u 2143 hugetlb_vma_unlock_write(vma);
2144 } 2144 }
2145 /* Nuke the hugetlb p 2145 /* Nuke the hugetlb page table entry */
2146 pteval = huge_ptep_cl 2146 pteval = huge_ptep_clear_flush(vma, address, pvmw.pte);
2147 } else { 2147 } else {
2148 flush_cache_page(vma, 2148 flush_cache_page(vma, address, pfn);
2149 /* Nuke the page tabl 2149 /* Nuke the page table entry. */
2150 if (should_defer_flus 2150 if (should_defer_flush(mm, flags)) {
2151 /* 2151 /*
2152 * We clear t 2152 * We clear the PTE but do not flush so potentially
2153 * a remote C 2153 * a remote CPU could still be writing to the folio.
2154 * If the ent 2154 * If the entry was previously clean then the
2155 * architectu 2155 * architecture must guarantee that a clear->dirty
2156 * transition 2156 * transition on a cached TLB entry is written through
2157 * and traps 2157 * and traps if the PTE is unmapped.
2158 */ 2158 */
2159 pteval = ptep 2159 pteval = ptep_get_and_clear(mm, address, pvmw.pte);
2160 2160
2161 set_tlb_ubc_f 2161 set_tlb_ubc_flush_pending(mm, pteval, address);
2162 } else { 2162 } else {
2163 pteval = ptep 2163 pteval = ptep_clear_flush(vma, address, pvmw.pte);
2164 } 2164 }
2165 } 2165 }
2166 2166
2167 /* Set the dirty flag on the 2167 /* Set the dirty flag on the folio now the pte is gone. */
2168 if (pte_dirty(pteval)) 2168 if (pte_dirty(pteval))
2169 folio_mark_dirty(foli 2169 folio_mark_dirty(folio);
2170 2170
2171 /* Update high watermark befo 2171 /* Update high watermark before we lower rss */
2172 update_hiwater_rss(mm); 2172 update_hiwater_rss(mm);
2173 2173
2174 if (folio_is_device_private(f 2174 if (folio_is_device_private(folio)) {
2175 unsigned long pfn = f 2175 unsigned long pfn = folio_pfn(folio);
2176 swp_entry_t entry; 2176 swp_entry_t entry;
2177 pte_t swp_pte; 2177 pte_t swp_pte;
2178 2178
2179 if (anon_exclusive) 2179 if (anon_exclusive)
2180 WARN_ON_ONCE( 2180 WARN_ON_ONCE(folio_try_share_anon_rmap_pte(folio,
2181 2181 subpage));
2182 2182
2183 /* 2183 /*
2184 * Store the pfn of t 2184 * Store the pfn of the page in a special migration
2185 * pte. do_swap_page( 2185 * pte. do_swap_page() will wait until the migration
2186 * pte is removed and 2186 * pte is removed and then restart fault handling.
2187 */ 2187 */
2188 entry = pte_to_swp_en 2188 entry = pte_to_swp_entry(pteval);
2189 if (is_writable_devic 2189 if (is_writable_device_private_entry(entry))
2190 entry = make_ 2190 entry = make_writable_migration_entry(pfn);
2191 else if (anon_exclusi 2191 else if (anon_exclusive)
2192 entry = make_ 2192 entry = make_readable_exclusive_migration_entry(pfn);
2193 else 2193 else
2194 entry = make_ 2194 entry = make_readable_migration_entry(pfn);
2195 swp_pte = swp_entry_t 2195 swp_pte = swp_entry_to_pte(entry);
2196 2196
2197 /* 2197 /*
2198 * pteval maps a zone 2198 * pteval maps a zone device page and is therefore
2199 * a swap pte. 2199 * a swap pte.
2200 */ 2200 */
2201 if (pte_swp_soft_dirt 2201 if (pte_swp_soft_dirty(pteval))
2202 swp_pte = pte 2202 swp_pte = pte_swp_mksoft_dirty(swp_pte);
2203 if (pte_swp_uffd_wp(p 2203 if (pte_swp_uffd_wp(pteval))
2204 swp_pte = pte 2204 swp_pte = pte_swp_mkuffd_wp(swp_pte);
2205 set_pte_at(mm, pvmw.a 2205 set_pte_at(mm, pvmw.address, pvmw.pte, swp_pte);
2206 trace_set_migration_p 2206 trace_set_migration_pte(pvmw.address, pte_val(swp_pte),
2207 2207 folio_order(folio));
2208 /* 2208 /*
2209 * No need to invalid 2209 * No need to invalidate here it will synchronize on
2210 * against the specia 2210 * against the special swap migration pte.
2211 */ 2211 */
2212 } else if (PageHWPoison(subpa 2212 } else if (PageHWPoison(subpage)) {
2213 pteval = swp_entry_to 2213 pteval = swp_entry_to_pte(make_hwpoison_entry(subpage));
2214 if (folio_test_hugetl 2214 if (folio_test_hugetlb(folio)) {
2215 hugetlb_count 2215 hugetlb_count_sub(folio_nr_pages(folio), mm);
2216 set_huge_pte_ 2216 set_huge_pte_at(mm, address, pvmw.pte, pteval,
2217 2217 hsz);
2218 } else { 2218 } else {
2219 dec_mm_counte 2219 dec_mm_counter(mm, mm_counter(folio));
2220 set_pte_at(mm 2220 set_pte_at(mm, address, pvmw.pte, pteval);
2221 } 2221 }
2222 2222
2223 } else if (pte_unused(pteval) 2223 } else if (pte_unused(pteval) && !userfaultfd_armed(vma)) {
2224 /* 2224 /*
2225 * The guest indicate 2225 * The guest indicated that the page content is of no
2226 * interest anymore. 2226 * interest anymore. Simply discard the pte, vmscan
2227 * will take care of 2227 * will take care of the rest.
2228 * A future reference 2228 * A future reference will then fault in a new zero
2229 * page. When userfau 2229 * page. When userfaultfd is active, we must not drop
2230 * this page though, 2230 * this page though, as its main user (postcopy
2231 * migration) will no 2231 * migration) will not expect userfaults on already
2232 * copied pages. 2232 * copied pages.
2233 */ 2233 */
2234 dec_mm_counter(mm, mm 2234 dec_mm_counter(mm, mm_counter(folio));
2235 } else { 2235 } else {
2236 swp_entry_t entry; 2236 swp_entry_t entry;
2237 pte_t swp_pte; 2237 pte_t swp_pte;
2238 2238
2239 if (arch_unmap_one(mm 2239 if (arch_unmap_one(mm, vma, address, pteval) < 0) {
2240 if (folio_tes 2240 if (folio_test_hugetlb(folio))
2241 set_h 2241 set_huge_pte_at(mm, address, pvmw.pte,
2242 2242 pteval, hsz);
2243 else 2243 else
2244 set_p 2244 set_pte_at(mm, address, pvmw.pte, pteval);
2245 ret = false; 2245 ret = false;
2246 page_vma_mapp 2246 page_vma_mapped_walk_done(&pvmw);
2247 break; 2247 break;
2248 } 2248 }
2249 VM_BUG_ON_PAGE(pte_wr 2249 VM_BUG_ON_PAGE(pte_write(pteval) && folio_test_anon(folio) &&
2250 !anon_ 2250 !anon_exclusive, subpage);
2251 2251
2252 /* See folio_try_shar 2252 /* See folio_try_share_anon_rmap_pte(): clear PTE first. */
2253 if (folio_test_hugetl 2253 if (folio_test_hugetlb(folio)) {
2254 if (anon_excl 2254 if (anon_exclusive &&
2255 hugetlb_t 2255 hugetlb_try_share_anon_rmap(folio)) {
2256 set_h 2256 set_huge_pte_at(mm, address, pvmw.pte,
2257 2257 pteval, hsz);
2258 ret = 2258 ret = false;
2259 page_ 2259 page_vma_mapped_walk_done(&pvmw);
2260 break 2260 break;
2261 } 2261 }
2262 } else if (anon_exclu 2262 } else if (anon_exclusive &&
2263 folio_try_ 2263 folio_try_share_anon_rmap_pte(folio, subpage)) {
2264 set_pte_at(mm 2264 set_pte_at(mm, address, pvmw.pte, pteval);
2265 ret = false; 2265 ret = false;
2266 page_vma_mapp 2266 page_vma_mapped_walk_done(&pvmw);
2267 break; 2267 break;
2268 } 2268 }
2269 2269
2270 /* 2270 /*
2271 * Store the pfn of t 2271 * Store the pfn of the page in a special migration
2272 * pte. do_swap_page( 2272 * pte. do_swap_page() will wait until the migration
2273 * pte is removed and 2273 * pte is removed and then restart fault handling.
2274 */ 2274 */
2275 if (pte_write(pteval) 2275 if (pte_write(pteval))
2276 entry = make_ 2276 entry = make_writable_migration_entry(
2277 2277 page_to_pfn(subpage));
2278 else if (anon_exclusi 2278 else if (anon_exclusive)
2279 entry = make_ 2279 entry = make_readable_exclusive_migration_entry(
2280 2280 page_to_pfn(subpage));
2281 else 2281 else
2282 entry = make_ 2282 entry = make_readable_migration_entry(
2283 2283 page_to_pfn(subpage));
2284 if (pte_young(pteval) 2284 if (pte_young(pteval))
2285 entry = make_ 2285 entry = make_migration_entry_young(entry);
2286 if (pte_dirty(pteval) 2286 if (pte_dirty(pteval))
2287 entry = make_ 2287 entry = make_migration_entry_dirty(entry);
2288 swp_pte = swp_entry_t 2288 swp_pte = swp_entry_to_pte(entry);
2289 if (pte_soft_dirty(pt 2289 if (pte_soft_dirty(pteval))
2290 swp_pte = pte 2290 swp_pte = pte_swp_mksoft_dirty(swp_pte);
2291 if (pte_uffd_wp(pteva 2291 if (pte_uffd_wp(pteval))
2292 swp_pte = pte 2292 swp_pte = pte_swp_mkuffd_wp(swp_pte);
2293 if (folio_test_hugetl 2293 if (folio_test_hugetlb(folio))
2294 set_huge_pte_ 2294 set_huge_pte_at(mm, address, pvmw.pte, swp_pte,
2295 2295 hsz);
2296 else 2296 else
2297 set_pte_at(mm 2297 set_pte_at(mm, address, pvmw.pte, swp_pte);
2298 trace_set_migration_p 2298 trace_set_migration_pte(address, pte_val(swp_pte),
2299 2299 folio_order(folio));
2300 /* 2300 /*
2301 * No need to invalid 2301 * No need to invalidate here it will synchronize on
2302 * against the specia 2302 * against the special swap migration pte.
2303 */ 2303 */
2304 } 2304 }
2305 2305
2306 if (unlikely(folio_test_huget 2306 if (unlikely(folio_test_hugetlb(folio)))
2307 hugetlb_remove_rmap(f 2307 hugetlb_remove_rmap(folio);
2308 else 2308 else
2309 folio_remove_rmap_pte 2309 folio_remove_rmap_pte(folio, subpage, vma);
2310 if (vma->vm_flags & VM_LOCKED 2310 if (vma->vm_flags & VM_LOCKED)
2311 mlock_drain_local(); 2311 mlock_drain_local();
2312 folio_put(folio); 2312 folio_put(folio);
2313 } 2313 }
2314 2314
2315 mmu_notifier_invalidate_range_end(&ra 2315 mmu_notifier_invalidate_range_end(&range);
2316 2316
2317 return ret; 2317 return ret;
2318 } 2318 }
2319 2319
2320 /** 2320 /**
2321 * try_to_migrate - try to replace all page t 2321 * try_to_migrate - try to replace all page table mappings with swap entries
2322 * @folio: the folio to replace page table en 2322 * @folio: the folio to replace page table entries for
2323 * @flags: action and flags 2323 * @flags: action and flags
2324 * 2324 *
2325 * Tries to remove all the page table entries 2325 * Tries to remove all the page table entries which are mapping this folio and
2326 * replace them with special swap entries. Ca 2326 * replace them with special swap entries. Caller must hold the folio lock.
2327 */ 2327 */
2328 void try_to_migrate(struct folio *folio, enum 2328 void try_to_migrate(struct folio *folio, enum ttu_flags flags)
2329 { 2329 {
2330 struct rmap_walk_control rwc = { 2330 struct rmap_walk_control rwc = {
2331 .rmap_one = try_to_migrate_on 2331 .rmap_one = try_to_migrate_one,
2332 .arg = (void *)flags, 2332 .arg = (void *)flags,
2333 .done = folio_not_mapped, 2333 .done = folio_not_mapped,
2334 .anon_lock = folio_lock_anon_ 2334 .anon_lock = folio_lock_anon_vma_read,
2335 }; 2335 };
2336 2336
2337 /* 2337 /*
2338 * Migration always ignores mlock and 2338 * Migration always ignores mlock and only supports TTU_RMAP_LOCKED and
2339 * TTU_SPLIT_HUGE_PMD, TTU_SYNC, and 2339 * TTU_SPLIT_HUGE_PMD, TTU_SYNC, and TTU_BATCH_FLUSH flags.
2340 */ 2340 */
2341 if (WARN_ON_ONCE(flags & ~(TTU_RMAP_L 2341 if (WARN_ON_ONCE(flags & ~(TTU_RMAP_LOCKED | TTU_SPLIT_HUGE_PMD |
2342 TTU_S 2342 TTU_SYNC | TTU_BATCH_FLUSH)))
2343 return; 2343 return;
2344 2344
2345 if (folio_is_zone_device(folio) && 2345 if (folio_is_zone_device(folio) &&
2346 (!folio_is_device_private(folio) 2346 (!folio_is_device_private(folio) && !folio_is_device_coherent(folio)))
2347 return; 2347 return;
2348 2348
2349 /* 2349 /*
2350 * During exec, a temporary VMA is se 2350 * During exec, a temporary VMA is setup and later moved.
2351 * The VMA is moved under the anon_vm 2351 * The VMA is moved under the anon_vma lock but not the
2352 * page tables leading to a race wher 2352 * page tables leading to a race where migration cannot
2353 * find the migration ptes. Rather th 2353 * find the migration ptes. Rather than increasing the
2354 * locking requirements of exec(), mi 2354 * locking requirements of exec(), migration skips
2355 * temporary VMAs until after exec() 2355 * temporary VMAs until after exec() completes.
2356 */ 2356 */
2357 if (!folio_test_ksm(folio) && folio_t 2357 if (!folio_test_ksm(folio) && folio_test_anon(folio))
2358 rwc.invalid_vma = invalid_mig 2358 rwc.invalid_vma = invalid_migration_vma;
2359 2359
2360 if (flags & TTU_RMAP_LOCKED) 2360 if (flags & TTU_RMAP_LOCKED)
2361 rmap_walk_locked(folio, &rwc) 2361 rmap_walk_locked(folio, &rwc);
2362 else 2362 else
2363 rmap_walk(folio, &rwc); 2363 rmap_walk(folio, &rwc);
2364 } 2364 }
2365 2365
2366 #ifdef CONFIG_DEVICE_PRIVATE 2366 #ifdef CONFIG_DEVICE_PRIVATE
2367 struct make_exclusive_args { 2367 struct make_exclusive_args {
2368 struct mm_struct *mm; 2368 struct mm_struct *mm;
2369 unsigned long address; 2369 unsigned long address;
2370 void *owner; 2370 void *owner;
2371 bool valid; 2371 bool valid;
2372 }; 2372 };
2373 2373
2374 static bool page_make_device_exclusive_one(st 2374 static bool page_make_device_exclusive_one(struct folio *folio,
2375 struct vm_area_struct *vma, u 2375 struct vm_area_struct *vma, unsigned long address, void *priv)
2376 { 2376 {
2377 struct mm_struct *mm = vma->vm_mm; 2377 struct mm_struct *mm = vma->vm_mm;
2378 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vm 2378 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);
2379 struct make_exclusive_args *args = pr 2379 struct make_exclusive_args *args = priv;
2380 pte_t pteval; 2380 pte_t pteval;
2381 struct page *subpage; 2381 struct page *subpage;
2382 bool ret = true; 2382 bool ret = true;
2383 struct mmu_notifier_range range; 2383 struct mmu_notifier_range range;
2384 swp_entry_t entry; 2384 swp_entry_t entry;
2385 pte_t swp_pte; 2385 pte_t swp_pte;
2386 pte_t ptent; 2386 pte_t ptent;
2387 2387
2388 mmu_notifier_range_init_owner(&range, 2388 mmu_notifier_range_init_owner(&range, MMU_NOTIFY_EXCLUSIVE, 0,
2389 vma->vm 2389 vma->vm_mm, address, min(vma->vm_end,
2390 address 2390 address + folio_size(folio)),
2391 args->o 2391 args->owner);
2392 mmu_notifier_invalidate_range_start(& 2392 mmu_notifier_invalidate_range_start(&range);
2393 2393
2394 while (page_vma_mapped_walk(&pvmw)) { 2394 while (page_vma_mapped_walk(&pvmw)) {
2395 /* Unexpected PMD-mapped THP? 2395 /* Unexpected PMD-mapped THP? */
2396 VM_BUG_ON_FOLIO(!pvmw.pte, fo 2396 VM_BUG_ON_FOLIO(!pvmw.pte, folio);
2397 2397
2398 ptent = ptep_get(pvmw.pte); 2398 ptent = ptep_get(pvmw.pte);
2399 if (!pte_present(ptent)) { 2399 if (!pte_present(ptent)) {
2400 ret = false; 2400 ret = false;
2401 page_vma_mapped_walk_ 2401 page_vma_mapped_walk_done(&pvmw);
2402 break; 2402 break;
2403 } 2403 }
2404 2404
2405 subpage = folio_page(folio, 2405 subpage = folio_page(folio,
2406 pte_pfn(ptent 2406 pte_pfn(ptent) - folio_pfn(folio));
2407 address = pvmw.address; 2407 address = pvmw.address;
2408 2408
2409 /* Nuke the page table entry. 2409 /* Nuke the page table entry. */
2410 flush_cache_page(vma, address 2410 flush_cache_page(vma, address, pte_pfn(ptent));
2411 pteval = ptep_clear_flush(vma 2411 pteval = ptep_clear_flush(vma, address, pvmw.pte);
2412 2412
2413 /* Set the dirty flag on the 2413 /* Set the dirty flag on the folio now the pte is gone. */
2414 if (pte_dirty(pteval)) 2414 if (pte_dirty(pteval))
2415 folio_mark_dirty(foli 2415 folio_mark_dirty(folio);
2416 2416
2417 /* 2417 /*
2418 * Check that our target page 2418 * Check that our target page is still mapped at the expected
2419 * address. 2419 * address.
2420 */ 2420 */
2421 if (args->mm == mm && args->a 2421 if (args->mm == mm && args->address == address &&
2422 pte_write(pteval)) 2422 pte_write(pteval))
2423 args->valid = true; 2423 args->valid = true;
2424 2424
2425 /* 2425 /*
2426 * Store the pfn of the page 2426 * Store the pfn of the page in a special migration
2427 * pte. do_swap_page() will w 2427 * pte. do_swap_page() will wait until the migration
2428 * pte is removed and then re 2428 * pte is removed and then restart fault handling.
2429 */ 2429 */
2430 if (pte_write(pteval)) 2430 if (pte_write(pteval))
2431 entry = make_writable 2431 entry = make_writable_device_exclusive_entry(
2432 2432 page_to_pfn(subpage));
2433 else 2433 else
2434 entry = make_readable 2434 entry = make_readable_device_exclusive_entry(
2435 2435 page_to_pfn(subpage));
2436 swp_pte = swp_entry_to_pte(en 2436 swp_pte = swp_entry_to_pte(entry);
2437 if (pte_soft_dirty(pteval)) 2437 if (pte_soft_dirty(pteval))
2438 swp_pte = pte_swp_mks 2438 swp_pte = pte_swp_mksoft_dirty(swp_pte);
2439 if (pte_uffd_wp(pteval)) 2439 if (pte_uffd_wp(pteval))
2440 swp_pte = pte_swp_mku 2440 swp_pte = pte_swp_mkuffd_wp(swp_pte);
2441 2441
2442 set_pte_at(mm, address, pvmw. 2442 set_pte_at(mm, address, pvmw.pte, swp_pte);
2443 2443
2444 /* 2444 /*
2445 * There is a reference on th 2445 * There is a reference on the page for the swap entry which has
2446 * been removed, so shouldn't 2446 * been removed, so shouldn't take another.
2447 */ 2447 */
2448 folio_remove_rmap_pte(folio, 2448 folio_remove_rmap_pte(folio, subpage, vma);
2449 } 2449 }
2450 2450
2451 mmu_notifier_invalidate_range_end(&ra 2451 mmu_notifier_invalidate_range_end(&range);
2452 2452
2453 return ret; 2453 return ret;
2454 } 2454 }
2455 2455
2456 /** 2456 /**
2457 * folio_make_device_exclusive - Mark the fol 2457 * folio_make_device_exclusive - Mark the folio exclusively owned by a device.
2458 * @folio: The folio to replace page table en 2458 * @folio: The folio to replace page table entries for.
2459 * @mm: The mm_struct where the folio is expe 2459 * @mm: The mm_struct where the folio is expected to be mapped.
2460 * @address: Address where the folio is expec 2460 * @address: Address where the folio is expected to be mapped.
2461 * @owner: passed to MMU_NOTIFY_EXCLUSIVE ran 2461 * @owner: passed to MMU_NOTIFY_EXCLUSIVE range notifier callbacks
2462 * 2462 *
2463 * Tries to remove all the page table entries 2463 * Tries to remove all the page table entries which are mapping this
2464 * folio and replace them with special device 2464 * folio and replace them with special device exclusive swap entries to
2465 * grant a device exclusive access to the fol 2465 * grant a device exclusive access to the folio.
2466 * 2466 *
2467 * Context: Caller must hold the folio lock. 2467 * Context: Caller must hold the folio lock.
2468 * Return: false if the page is still mapped, 2468 * Return: false if the page is still mapped, or if it could not be unmapped
2469 * from the expected address. Otherwise retur 2469 * from the expected address. Otherwise returns true (success).
2470 */ 2470 */
2471 static bool folio_make_device_exclusive(struc 2471 static bool folio_make_device_exclusive(struct folio *folio,
2472 struct mm_struct *mm, unsigne 2472 struct mm_struct *mm, unsigned long address, void *owner)
2473 { 2473 {
2474 struct make_exclusive_args args = { 2474 struct make_exclusive_args args = {
2475 .mm = mm, 2475 .mm = mm,
2476 .address = address, 2476 .address = address,
2477 .owner = owner, 2477 .owner = owner,
2478 .valid = false, 2478 .valid = false,
2479 }; 2479 };
2480 struct rmap_walk_control rwc = { 2480 struct rmap_walk_control rwc = {
2481 .rmap_one = page_make_device_ 2481 .rmap_one = page_make_device_exclusive_one,
2482 .done = folio_not_mapped, 2482 .done = folio_not_mapped,
2483 .anon_lock = folio_lock_anon_ 2483 .anon_lock = folio_lock_anon_vma_read,
2484 .arg = &args, 2484 .arg = &args,
2485 }; 2485 };
2486 2486
2487 /* 2487 /*
2488 * Restrict to anonymous folios for n 2488 * Restrict to anonymous folios for now to avoid potential writeback
2489 * issues. 2489 * issues.
2490 */ 2490 */
2491 if (!folio_test_anon(folio)) 2491 if (!folio_test_anon(folio))
2492 return false; 2492 return false;
2493 2493
2494 rmap_walk(folio, &rwc); 2494 rmap_walk(folio, &rwc);
2495 2495
2496 return args.valid && !folio_mapcount( 2496 return args.valid && !folio_mapcount(folio);
2497 } 2497 }
2498 2498
2499 /** 2499 /**
2500 * make_device_exclusive_range() - Mark a ran 2500 * make_device_exclusive_range() - Mark a range for exclusive use by a device
2501 * @mm: mm_struct of associated target proces 2501 * @mm: mm_struct of associated target process
2502 * @start: start of the region to mark for ex 2502 * @start: start of the region to mark for exclusive device access
2503 * @end: end address of region 2503 * @end: end address of region
2504 * @pages: returns the pages which were succe 2504 * @pages: returns the pages which were successfully marked for exclusive access
2505 * @owner: passed to MMU_NOTIFY_EXCLUSIVE ran 2505 * @owner: passed to MMU_NOTIFY_EXCLUSIVE range notifier to allow filtering
2506 * 2506 *
2507 * Returns: number of pages found in the rang 2507 * Returns: number of pages found in the range by GUP. A page is marked for
2508 * exclusive access only if the page pointer 2508 * exclusive access only if the page pointer is non-NULL.
2509 * 2509 *
2510 * This function finds ptes mapping page(s) t 2510 * This function finds ptes mapping page(s) to the given address range, locks
2511 * them and replaces mappings with special sw 2511 * them and replaces mappings with special swap entries preventing userspace CPU
2512 * access. On fault these entries are replace 2512 * access. On fault these entries are replaced with the original mapping after
2513 * calling MMU notifiers. 2513 * calling MMU notifiers.
2514 * 2514 *
2515 * A driver using this to program access from 2515 * A driver using this to program access from a device must use a mmu notifier
2516 * critical section to hold a device specific 2516 * critical section to hold a device specific lock during programming. Once
2517 * programming is complete it should drop the 2517 * programming is complete it should drop the page lock and reference after
2518 * which point CPU access to the page will re 2518 * which point CPU access to the page will revoke the exclusive access.
2519 */ 2519 */
2520 int make_device_exclusive_range(struct mm_str 2520 int make_device_exclusive_range(struct mm_struct *mm, unsigned long start,
2521 unsigned long 2521 unsigned long end, struct page **pages,
2522 void *owner) 2522 void *owner)
2523 { 2523 {
2524 long npages = (end - start) >> PAGE_S 2524 long npages = (end - start) >> PAGE_SHIFT;
2525 long i; 2525 long i;
2526 2526
2527 npages = get_user_pages_remote(mm, st 2527 npages = get_user_pages_remote(mm, start, npages,
2528 FOLL_G 2528 FOLL_GET | FOLL_WRITE | FOLL_SPLIT_PMD,
2529 pages, 2529 pages, NULL);
2530 if (npages < 0) 2530 if (npages < 0)
2531 return npages; 2531 return npages;
2532 2532
2533 for (i = 0; i < npages; i++, start += 2533 for (i = 0; i < npages; i++, start += PAGE_SIZE) {
2534 struct folio *folio = page_fo 2534 struct folio *folio = page_folio(pages[i]);
2535 if (PageTail(pages[i]) || !fo 2535 if (PageTail(pages[i]) || !folio_trylock(folio)) {
2536 folio_put(folio); 2536 folio_put(folio);
2537 pages[i] = NULL; 2537 pages[i] = NULL;
2538 continue; 2538 continue;
2539 } 2539 }
2540 2540
2541 if (!folio_make_device_exclus 2541 if (!folio_make_device_exclusive(folio, mm, start, owner)) {
2542 folio_unlock(folio); 2542 folio_unlock(folio);
2543 folio_put(folio); 2543 folio_put(folio);
2544 pages[i] = NULL; 2544 pages[i] = NULL;
2545 } 2545 }
2546 } 2546 }
2547 2547
2548 return npages; 2548 return npages;
2549 } 2549 }
2550 EXPORT_SYMBOL_GPL(make_device_exclusive_range 2550 EXPORT_SYMBOL_GPL(make_device_exclusive_range);
2551 #endif 2551 #endif
2552 2552
2553 void __put_anon_vma(struct anon_vma *anon_vma 2553 void __put_anon_vma(struct anon_vma *anon_vma)
2554 { 2554 {
2555 struct anon_vma *root = anon_vma->roo 2555 struct anon_vma *root = anon_vma->root;
2556 2556
2557 anon_vma_free(anon_vma); 2557 anon_vma_free(anon_vma);
2558 if (root != anon_vma && atomic_dec_an 2558 if (root != anon_vma && atomic_dec_and_test(&root->refcount))
2559 anon_vma_free(root); 2559 anon_vma_free(root);
2560 } 2560 }
2561 2561
2562 static struct anon_vma *rmap_walk_anon_lock(s 2562 static struct anon_vma *rmap_walk_anon_lock(struct folio *folio,
2563 s 2563 struct rmap_walk_control *rwc)
2564 { 2564 {
2565 struct anon_vma *anon_vma; 2565 struct anon_vma *anon_vma;
2566 2566
2567 if (rwc->anon_lock) 2567 if (rwc->anon_lock)
2568 return rwc->anon_lock(folio, 2568 return rwc->anon_lock(folio, rwc);
2569 2569
2570 /* 2570 /*
2571 * Note: remove_migration_ptes() cann 2571 * Note: remove_migration_ptes() cannot use folio_lock_anon_vma_read()
2572 * because that depends on page_mappe 2572 * because that depends on page_mapped(); but not all its usages
2573 * are holding mmap_lock. Users witho 2573 * are holding mmap_lock. Users without mmap_lock are required to
2574 * take a reference count to prevent 2574 * take a reference count to prevent the anon_vma disappearing
2575 */ 2575 */
2576 anon_vma = folio_anon_vma(folio); 2576 anon_vma = folio_anon_vma(folio);
2577 if (!anon_vma) 2577 if (!anon_vma)
2578 return NULL; 2578 return NULL;
2579 2579
2580 if (anon_vma_trylock_read(anon_vma)) 2580 if (anon_vma_trylock_read(anon_vma))
2581 goto out; 2581 goto out;
2582 2582
2583 if (rwc->try_lock) { 2583 if (rwc->try_lock) {
2584 anon_vma = NULL; 2584 anon_vma = NULL;
2585 rwc->contended = true; 2585 rwc->contended = true;
2586 goto out; 2586 goto out;
2587 } 2587 }
2588 2588
2589 anon_vma_lock_read(anon_vma); 2589 anon_vma_lock_read(anon_vma);
2590 out: 2590 out:
2591 return anon_vma; 2591 return anon_vma;
2592 } 2592 }
2593 2593
2594 /* 2594 /*
2595 * rmap_walk_anon - do something to anonymous 2595 * rmap_walk_anon - do something to anonymous page using the object-based
2596 * rmap method 2596 * rmap method
2597 * @folio: the folio to be handled 2597 * @folio: the folio to be handled
2598 * @rwc: control variable according to each w 2598 * @rwc: control variable according to each walk type
2599 * @locked: caller holds relevant rmap lock 2599 * @locked: caller holds relevant rmap lock
2600 * 2600 *
2601 * Find all the mappings of a folio using the 2601 * Find all the mappings of a folio using the mapping pointer and the vma
2602 * chains contained in the anon_vma struct it 2602 * chains contained in the anon_vma struct it points to.
2603 */ 2603 */
2604 static void rmap_walk_anon(struct folio *foli 2604 static void rmap_walk_anon(struct folio *folio,
2605 struct rmap_walk_control *rwc 2605 struct rmap_walk_control *rwc, bool locked)
2606 { 2606 {
2607 struct anon_vma *anon_vma; 2607 struct anon_vma *anon_vma;
2608 pgoff_t pgoff_start, pgoff_end; 2608 pgoff_t pgoff_start, pgoff_end;
2609 struct anon_vma_chain *avc; 2609 struct anon_vma_chain *avc;
2610 2610
2611 if (locked) { 2611 if (locked) {
2612 anon_vma = folio_anon_vma(fol 2612 anon_vma = folio_anon_vma(folio);
2613 /* anon_vma disappear under u 2613 /* anon_vma disappear under us? */
2614 VM_BUG_ON_FOLIO(!anon_vma, fo 2614 VM_BUG_ON_FOLIO(!anon_vma, folio);
2615 } else { 2615 } else {
2616 anon_vma = rmap_walk_anon_loc 2616 anon_vma = rmap_walk_anon_lock(folio, rwc);
2617 } 2617 }
2618 if (!anon_vma) 2618 if (!anon_vma)
2619 return; 2619 return;
2620 2620
2621 pgoff_start = folio_pgoff(folio); 2621 pgoff_start = folio_pgoff(folio);
2622 pgoff_end = pgoff_start + folio_nr_pa 2622 pgoff_end = pgoff_start + folio_nr_pages(folio) - 1;
2623 anon_vma_interval_tree_foreach(avc, & 2623 anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root,
2624 pgoff_start, pgoff_en 2624 pgoff_start, pgoff_end) {
2625 struct vm_area_struct *vma = 2625 struct vm_area_struct *vma = avc->vma;
2626 unsigned long address = vma_a 2626 unsigned long address = vma_address(vma, pgoff_start,
2627 folio_nr_page 2627 folio_nr_pages(folio));
2628 2628
2629 VM_BUG_ON_VMA(address == -EFA 2629 VM_BUG_ON_VMA(address == -EFAULT, vma);
2630 cond_resched(); 2630 cond_resched();
2631 2631
2632 if (rwc->invalid_vma && rwc-> 2632 if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg))
2633 continue; 2633 continue;
2634 2634
2635 if (!rwc->rmap_one(folio, vma 2635 if (!rwc->rmap_one(folio, vma, address, rwc->arg))
2636 break; 2636 break;
2637 if (rwc->done && rwc->done(fo 2637 if (rwc->done && rwc->done(folio))
2638 break; 2638 break;
2639 } 2639 }
2640 2640
2641 if (!locked) 2641 if (!locked)
2642 anon_vma_unlock_read(anon_vma 2642 anon_vma_unlock_read(anon_vma);
2643 } 2643 }
2644 2644
2645 /* 2645 /*
2646 * rmap_walk_file - do something to file page 2646 * rmap_walk_file - do something to file page using the object-based rmap method
2647 * @folio: the folio to be handled 2647 * @folio: the folio to be handled
2648 * @rwc: control variable according to each w 2648 * @rwc: control variable according to each walk type
2649 * @locked: caller holds relevant rmap lock 2649 * @locked: caller holds relevant rmap lock
2650 * 2650 *
2651 * Find all the mappings of a folio using the 2651 * Find all the mappings of a folio using the mapping pointer and the vma chains
2652 * contained in the address_space struct it p 2652 * contained in the address_space struct it points to.
2653 */ 2653 */
2654 static void rmap_walk_file(struct folio *foli 2654 static void rmap_walk_file(struct folio *folio,
2655 struct rmap_walk_control *rwc 2655 struct rmap_walk_control *rwc, bool locked)
2656 { 2656 {
2657 struct address_space *mapping = folio 2657 struct address_space *mapping = folio_mapping(folio);
2658 pgoff_t pgoff_start, pgoff_end; 2658 pgoff_t pgoff_start, pgoff_end;
2659 struct vm_area_struct *vma; 2659 struct vm_area_struct *vma;
2660 2660
2661 /* 2661 /*
2662 * The page lock not only makes sure 2662 * The page lock not only makes sure that page->mapping cannot
2663 * suddenly be NULLified by truncatio 2663 * suddenly be NULLified by truncation, it makes sure that the
2664 * structure at mapping cannot be fre 2664 * structure at mapping cannot be freed and reused yet,
2665 * so we can safely take mapping->i_m 2665 * so we can safely take mapping->i_mmap_rwsem.
2666 */ 2666 */
2667 VM_BUG_ON_FOLIO(!folio_test_locked(fo 2667 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
2668 2668
2669 if (!mapping) 2669 if (!mapping)
2670 return; 2670 return;
2671 2671
2672 pgoff_start = folio_pgoff(folio); 2672 pgoff_start = folio_pgoff(folio);
2673 pgoff_end = pgoff_start + folio_nr_pa 2673 pgoff_end = pgoff_start + folio_nr_pages(folio) - 1;
2674 if (!locked) { 2674 if (!locked) {
2675 if (i_mmap_trylock_read(mappi 2675 if (i_mmap_trylock_read(mapping))
2676 goto lookup; 2676 goto lookup;
2677 2677
2678 if (rwc->try_lock) { 2678 if (rwc->try_lock) {
2679 rwc->contended = true 2679 rwc->contended = true;
2680 return; 2680 return;
2681 } 2681 }
2682 2682
2683 i_mmap_lock_read(mapping); 2683 i_mmap_lock_read(mapping);
2684 } 2684 }
2685 lookup: 2685 lookup:
2686 vma_interval_tree_foreach(vma, &mappi 2686 vma_interval_tree_foreach(vma, &mapping->i_mmap,
2687 pgoff_start, pgoff_en 2687 pgoff_start, pgoff_end) {
2688 unsigned long address = vma_a 2688 unsigned long address = vma_address(vma, pgoff_start,
2689 folio_nr_pages 2689 folio_nr_pages(folio));
2690 2690
2691 VM_BUG_ON_VMA(address == -EFA 2691 VM_BUG_ON_VMA(address == -EFAULT, vma);
2692 cond_resched(); 2692 cond_resched();
2693 2693
2694 if (rwc->invalid_vma && rwc-> 2694 if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg))
2695 continue; 2695 continue;
2696 2696
2697 if (!rwc->rmap_one(folio, vma 2697 if (!rwc->rmap_one(folio, vma, address, rwc->arg))
2698 goto done; 2698 goto done;
2699 if (rwc->done && rwc->done(fo 2699 if (rwc->done && rwc->done(folio))
2700 goto done; 2700 goto done;
2701 } 2701 }
2702 2702
2703 done: 2703 done:
2704 if (!locked) 2704 if (!locked)
2705 i_mmap_unlock_read(mapping); 2705 i_mmap_unlock_read(mapping);
2706 } 2706 }
2707 2707
2708 void rmap_walk(struct folio *folio, struct rm 2708 void rmap_walk(struct folio *folio, struct rmap_walk_control *rwc)
2709 { 2709 {
2710 if (unlikely(folio_test_ksm(folio))) 2710 if (unlikely(folio_test_ksm(folio)))
2711 rmap_walk_ksm(folio, rwc); 2711 rmap_walk_ksm(folio, rwc);
2712 else if (folio_test_anon(folio)) 2712 else if (folio_test_anon(folio))
2713 rmap_walk_anon(folio, rwc, fa 2713 rmap_walk_anon(folio, rwc, false);
2714 else 2714 else
2715 rmap_walk_file(folio, rwc, fa 2715 rmap_walk_file(folio, rwc, false);
2716 } 2716 }
2717 2717
2718 /* Like rmap_walk, but caller holds relevant 2718 /* Like rmap_walk, but caller holds relevant rmap lock */
2719 void rmap_walk_locked(struct folio *folio, st 2719 void rmap_walk_locked(struct folio *folio, struct rmap_walk_control *rwc)
2720 { 2720 {
2721 /* no ksm support for now */ 2721 /* no ksm support for now */
2722 VM_BUG_ON_FOLIO(folio_test_ksm(folio) 2722 VM_BUG_ON_FOLIO(folio_test_ksm(folio), folio);
2723 if (folio_test_anon(folio)) 2723 if (folio_test_anon(folio))
2724 rmap_walk_anon(folio, rwc, tr 2724 rmap_walk_anon(folio, rwc, true);
2725 else 2725 else
2726 rmap_walk_file(folio, rwc, tr 2726 rmap_walk_file(folio, rwc, true);
2727 } 2727 }
2728 2728
2729 #ifdef CONFIG_HUGETLB_PAGE 2729 #ifdef CONFIG_HUGETLB_PAGE
2730 /* 2730 /*
2731 * The following two functions are for anonym 2731 * The following two functions are for anonymous (private mapped) hugepages.
2732 * Unlike common anonymous pages, anonymous h 2732 * Unlike common anonymous pages, anonymous hugepages have no accounting code
2733 * and no lru code, because we handle hugepag 2733 * and no lru code, because we handle hugepages differently from common pages.
2734 */ 2734 */
2735 void hugetlb_add_anon_rmap(struct folio *foli 2735 void hugetlb_add_anon_rmap(struct folio *folio, struct vm_area_struct *vma,
2736 unsigned long address, rmap_t 2736 unsigned long address, rmap_t flags)
2737 { 2737 {
2738 VM_WARN_ON_FOLIO(!folio_test_hugetlb( 2738 VM_WARN_ON_FOLIO(!folio_test_hugetlb(folio), folio);
2739 VM_WARN_ON_FOLIO(!folio_test_anon(fol 2739 VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio);
2740 2740
2741 atomic_inc(&folio->_entire_mapcount); 2741 atomic_inc(&folio->_entire_mapcount);
2742 atomic_inc(&folio->_large_mapcount); 2742 atomic_inc(&folio->_large_mapcount);
2743 if (flags & RMAP_EXCLUSIVE) 2743 if (flags & RMAP_EXCLUSIVE)
2744 SetPageAnonExclusive(&folio-> 2744 SetPageAnonExclusive(&folio->page);
2745 VM_WARN_ON_FOLIO(folio_entire_mapcoun 2745 VM_WARN_ON_FOLIO(folio_entire_mapcount(folio) > 1 &&
2746 PageAnonExclusive(&f 2746 PageAnonExclusive(&folio->page), folio);
2747 } 2747 }
2748 2748
2749 void hugetlb_add_new_anon_rmap(struct folio * 2749 void hugetlb_add_new_anon_rmap(struct folio *folio,
2750 struct vm_area_struct *vma, u 2750 struct vm_area_struct *vma, unsigned long address)
2751 { 2751 {
2752 VM_WARN_ON_FOLIO(!folio_test_hugetlb( 2752 VM_WARN_ON_FOLIO(!folio_test_hugetlb(folio), folio);
2753 2753
2754 BUG_ON(address < vma->vm_start || add 2754 BUG_ON(address < vma->vm_start || address >= vma->vm_end);
2755 /* increment count (starts at -1) */ 2755 /* increment count (starts at -1) */
2756 atomic_set(&folio->_entire_mapcount, 2756 atomic_set(&folio->_entire_mapcount, 0);
2757 atomic_set(&folio->_large_mapcount, 0 2757 atomic_set(&folio->_large_mapcount, 0);
2758 folio_clear_hugetlb_restore_reserve(f 2758 folio_clear_hugetlb_restore_reserve(folio);
2759 __folio_set_anon(folio, vma, address, 2759 __folio_set_anon(folio, vma, address, true);
2760 SetPageAnonExclusive(&folio->page); 2760 SetPageAnonExclusive(&folio->page);
2761 } 2761 }
2762 #endif /* CONFIG_HUGETLB_PAGE */ 2762 #endif /* CONFIG_HUGETLB_PAGE */
2763 2763
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.