1 // SPDX-License-Identifier: GPL-2.0-or-later 1 // SPDX-License-Identifier: GPL-2.0-or-later
2 2
3 /* 3 /*
4 * VMA-specific functions. 4 * VMA-specific functions.
5 */ 5 */
6 6
7 #include "vma_internal.h" 7 #include "vma_internal.h"
8 #include "vma.h" 8 #include "vma.h"
9 9
10 static inline bool is_mergeable_vma(struct vma 10 static inline bool is_mergeable_vma(struct vma_merge_struct *vmg, bool merge_next)
11 { 11 {
12 struct vm_area_struct *vma = merge_nex 12 struct vm_area_struct *vma = merge_next ? vmg->next : vmg->prev;
13 13
14 if (!mpol_equal(vmg->policy, vma_polic 14 if (!mpol_equal(vmg->policy, vma_policy(vma)))
15 return false; 15 return false;
16 /* 16 /*
17 * VM_SOFTDIRTY should not prevent fro 17 * VM_SOFTDIRTY should not prevent from VMA merging, if we
18 * match the flags but dirty bit -- th 18 * match the flags but dirty bit -- the caller should mark
19 * merged VMA as dirty. If dirty bit w 19 * merged VMA as dirty. If dirty bit won't be excluded from
20 * comparison, we increase pressure on 20 * comparison, we increase pressure on the memory system forcing
21 * the kernel to generate new VMAs whe 21 * the kernel to generate new VMAs when old one could be
22 * extended instead. 22 * extended instead.
23 */ 23 */
24 if ((vma->vm_flags ^ vmg->flags) & ~VM 24 if ((vma->vm_flags ^ vmg->flags) & ~VM_SOFTDIRTY)
25 return false; 25 return false;
26 if (vma->vm_file != vmg->file) 26 if (vma->vm_file != vmg->file)
27 return false; 27 return false;
28 if (!is_mergeable_vm_userfaultfd_ctx(v 28 if (!is_mergeable_vm_userfaultfd_ctx(vma, vmg->uffd_ctx))
29 return false; 29 return false;
30 if (!anon_vma_name_eq(anon_vma_name(vm 30 if (!anon_vma_name_eq(anon_vma_name(vma), vmg->anon_name))
31 return false; 31 return false;
32 return true; 32 return true;
33 } 33 }
34 34
35 static inline bool is_mergeable_anon_vma(struc 35 static inline bool is_mergeable_anon_vma(struct anon_vma *anon_vma1,
36 struct anon_vma *anon_vma2, s 36 struct anon_vma *anon_vma2, struct vm_area_struct *vma)
37 { 37 {
38 /* 38 /*
39 * The list_is_singular() test is to a 39 * The list_is_singular() test is to avoid merging VMA cloned from
40 * parents. This can improve scalabili 40 * parents. This can improve scalability caused by anon_vma lock.
41 */ 41 */
42 if ((!anon_vma1 || !anon_vma2) && (!vm 42 if ((!anon_vma1 || !anon_vma2) && (!vma ||
43 list_is_singular(&vma->anon_vm 43 list_is_singular(&vma->anon_vma_chain)))
44 return true; 44 return true;
45 return anon_vma1 == anon_vma2; 45 return anon_vma1 == anon_vma2;
46 } 46 }
47 47
48 /* Are the anon_vma's belonging to each VMA co 48 /* Are the anon_vma's belonging to each VMA compatible with one another? */
49 static inline bool are_anon_vmas_compatible(st 49 static inline bool are_anon_vmas_compatible(struct vm_area_struct *vma1,
50 st 50 struct vm_area_struct *vma2)
51 { 51 {
52 return is_mergeable_anon_vma(vma1->ano 52 return is_mergeable_anon_vma(vma1->anon_vma, vma2->anon_vma, NULL);
53 } 53 }
54 54
55 /* 55 /*
56 * init_multi_vma_prep() - Initializer for str 56 * init_multi_vma_prep() - Initializer for struct vma_prepare
57 * @vp: The vma_prepare struct 57 * @vp: The vma_prepare struct
58 * @vma: The vma that will be altered once loc 58 * @vma: The vma that will be altered once locked
59 * @next: The next vma if it is to be adjusted 59 * @next: The next vma if it is to be adjusted
60 * @remove: The first vma to be removed 60 * @remove: The first vma to be removed
61 * @remove2: The second vma to be removed 61 * @remove2: The second vma to be removed
62 */ 62 */
63 static void init_multi_vma_prep(struct vma_pre 63 static void init_multi_vma_prep(struct vma_prepare *vp,
64 struct vm_area 64 struct vm_area_struct *vma,
65 struct vm_area 65 struct vm_area_struct *next,
66 struct vm_area 66 struct vm_area_struct *remove,
67 struct vm_area 67 struct vm_area_struct *remove2)
68 { 68 {
69 memset(vp, 0, sizeof(struct vma_prepar 69 memset(vp, 0, sizeof(struct vma_prepare));
70 vp->vma = vma; 70 vp->vma = vma;
71 vp->anon_vma = vma->anon_vma; 71 vp->anon_vma = vma->anon_vma;
72 vp->remove = remove; 72 vp->remove = remove;
73 vp->remove2 = remove2; 73 vp->remove2 = remove2;
74 vp->adj_next = next; 74 vp->adj_next = next;
75 if (!vp->anon_vma && next) 75 if (!vp->anon_vma && next)
76 vp->anon_vma = next->anon_vma; 76 vp->anon_vma = next->anon_vma;
77 77
78 vp->file = vma->vm_file; 78 vp->file = vma->vm_file;
79 if (vp->file) 79 if (vp->file)
80 vp->mapping = vma->vm_file->f_ 80 vp->mapping = vma->vm_file->f_mapping;
81 81
82 } 82 }
83 83
84 /* 84 /*
85 * Return true if we can merge this (vm_flags, 85 * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff)
86 * in front of (at a lower virtual address and 86 * in front of (at a lower virtual address and file offset than) the vma.
87 * 87 *
88 * We cannot merge two vmas if they have diffe 88 * We cannot merge two vmas if they have differently assigned (non-NULL)
89 * anon_vmas, nor if same anon_vma is assigned 89 * anon_vmas, nor if same anon_vma is assigned but offsets incompatible.
90 * 90 *
91 * We don't check here for the merged mmap wra 91 * We don't check here for the merged mmap wrapping around the end of pagecache
92 * indices (16TB on ia32) because do_mmap() do 92 * indices (16TB on ia32) because do_mmap() does not permit mmap's which
93 * wrap, nor mmaps which cover the final page 93 * wrap, nor mmaps which cover the final page at index -1UL.
94 * 94 *
95 * We assume the vma may be removed as part of 95 * We assume the vma may be removed as part of the merge.
96 */ 96 */
97 static bool can_vma_merge_before(struct vma_me 97 static bool can_vma_merge_before(struct vma_merge_struct *vmg)
98 { 98 {
99 pgoff_t pglen = PHYS_PFN(vmg->end - vm 99 pgoff_t pglen = PHYS_PFN(vmg->end - vmg->start);
100 100
101 if (is_mergeable_vma(vmg, /* merge_nex 101 if (is_mergeable_vma(vmg, /* merge_next = */ true) &&
102 is_mergeable_anon_vma(vmg->anon_vm 102 is_mergeable_anon_vma(vmg->anon_vma, vmg->next->anon_vma, vmg->next)) {
103 if (vmg->next->vm_pgoff == vmg 103 if (vmg->next->vm_pgoff == vmg->pgoff + pglen)
104 return true; 104 return true;
105 } 105 }
106 106
107 return false; 107 return false;
108 } 108 }
109 109
110 /* 110 /*
111 * Return true if we can merge this (vm_flags, 111 * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff)
112 * beyond (at a higher virtual address and fil 112 * beyond (at a higher virtual address and file offset than) the vma.
113 * 113 *
114 * We cannot merge two vmas if they have diffe 114 * We cannot merge two vmas if they have differently assigned (non-NULL)
115 * anon_vmas, nor if same anon_vma is assigned 115 * anon_vmas, nor if same anon_vma is assigned but offsets incompatible.
116 * 116 *
117 * We assume that vma is not removed as part o 117 * We assume that vma is not removed as part of the merge.
118 */ 118 */
119 static bool can_vma_merge_after(struct vma_mer 119 static bool can_vma_merge_after(struct vma_merge_struct *vmg)
120 { 120 {
121 if (is_mergeable_vma(vmg, /* merge_nex 121 if (is_mergeable_vma(vmg, /* merge_next = */ false) &&
122 is_mergeable_anon_vma(vmg->anon_vm 122 is_mergeable_anon_vma(vmg->anon_vma, vmg->prev->anon_vma, vmg->prev)) {
123 if (vmg->prev->vm_pgoff + vma_ 123 if (vmg->prev->vm_pgoff + vma_pages(vmg->prev) == vmg->pgoff)
124 return true; 124 return true;
125 } 125 }
126 return false; 126 return false;
127 } 127 }
128 128
129 static void __vma_link_file(struct vm_area_str 129 static void __vma_link_file(struct vm_area_struct *vma,
130 struct address_spa 130 struct address_space *mapping)
131 { 131 {
132 if (vma_is_shared_maywrite(vma)) 132 if (vma_is_shared_maywrite(vma))
133 mapping_allow_writable(mapping 133 mapping_allow_writable(mapping);
134 134
135 flush_dcache_mmap_lock(mapping); 135 flush_dcache_mmap_lock(mapping);
136 vma_interval_tree_insert(vma, &mapping 136 vma_interval_tree_insert(vma, &mapping->i_mmap);
137 flush_dcache_mmap_unlock(mapping); 137 flush_dcache_mmap_unlock(mapping);
138 } 138 }
139 139
140 /* 140 /*
141 * Requires inode->i_mapping->i_mmap_rwsem 141 * Requires inode->i_mapping->i_mmap_rwsem
142 */ 142 */
143 static void __remove_shared_vm_struct(struct v 143 static void __remove_shared_vm_struct(struct vm_area_struct *vma,
144 struct a 144 struct address_space *mapping)
145 { 145 {
146 if (vma_is_shared_maywrite(vma)) 146 if (vma_is_shared_maywrite(vma))
147 mapping_unmap_writable(mapping 147 mapping_unmap_writable(mapping);
148 148
149 flush_dcache_mmap_lock(mapping); 149 flush_dcache_mmap_lock(mapping);
150 vma_interval_tree_remove(vma, &mapping 150 vma_interval_tree_remove(vma, &mapping->i_mmap);
151 flush_dcache_mmap_unlock(mapping); 151 flush_dcache_mmap_unlock(mapping);
152 } 152 }
153 153
154 /* 154 /*
155 * vma_prepare() - Helper function for handlin 155 * vma_prepare() - Helper function for handling locking VMAs prior to altering
156 * @vp: The initialized vma_prepare struct 156 * @vp: The initialized vma_prepare struct
157 */ 157 */
158 static void vma_prepare(struct vma_prepare *vp 158 static void vma_prepare(struct vma_prepare *vp)
159 { 159 {
160 if (vp->file) { 160 if (vp->file) {
161 uprobe_munmap(vp->vma, vp->vma 161 uprobe_munmap(vp->vma, vp->vma->vm_start, vp->vma->vm_end);
162 162
163 if (vp->adj_next) 163 if (vp->adj_next)
164 uprobe_munmap(vp->adj_ 164 uprobe_munmap(vp->adj_next, vp->adj_next->vm_start,
165 vp->adj_ 165 vp->adj_next->vm_end);
166 166
167 i_mmap_lock_write(vp->mapping) 167 i_mmap_lock_write(vp->mapping);
168 if (vp->insert && vp->insert-> 168 if (vp->insert && vp->insert->vm_file) {
169 /* 169 /*
170 * Put into interval t 170 * Put into interval tree now, so instantiated pages
171 * are visible to arm/ 171 * are visible to arm/parisc __flush_dcache_page
172 * throughout; but we 172 * throughout; but we cannot insert into address
173 * space until vma sta 173 * space until vma start or end is updated.
174 */ 174 */
175 __vma_link_file(vp->in 175 __vma_link_file(vp->insert,
176 vp->in 176 vp->insert->vm_file->f_mapping);
177 } 177 }
178 } 178 }
179 179
180 if (vp->anon_vma) { 180 if (vp->anon_vma) {
181 anon_vma_lock_write(vp->anon_v 181 anon_vma_lock_write(vp->anon_vma);
182 anon_vma_interval_tree_pre_upd 182 anon_vma_interval_tree_pre_update_vma(vp->vma);
183 if (vp->adj_next) 183 if (vp->adj_next)
184 anon_vma_interval_tree 184 anon_vma_interval_tree_pre_update_vma(vp->adj_next);
185 } 185 }
186 186
187 if (vp->file) { 187 if (vp->file) {
188 flush_dcache_mmap_lock(vp->map 188 flush_dcache_mmap_lock(vp->mapping);
189 vma_interval_tree_remove(vp->v 189 vma_interval_tree_remove(vp->vma, &vp->mapping->i_mmap);
190 if (vp->adj_next) 190 if (vp->adj_next)
191 vma_interval_tree_remo 191 vma_interval_tree_remove(vp->adj_next,
192 192 &vp->mapping->i_mmap);
193 } 193 }
194 194
195 } 195 }
196 196
197 /* 197 /*
198 * vma_complete- Helper function for handling 198 * vma_complete- Helper function for handling the unlocking after altering VMAs,
199 * or for inserting a VMA. 199 * or for inserting a VMA.
200 * 200 *
201 * @vp: The vma_prepare struct 201 * @vp: The vma_prepare struct
202 * @vmi: The vma iterator 202 * @vmi: The vma iterator
203 * @mm: The mm_struct 203 * @mm: The mm_struct
204 */ 204 */
205 static void vma_complete(struct vma_prepare *v 205 static void vma_complete(struct vma_prepare *vp, struct vma_iterator *vmi,
206 struct mm_struct *mm) 206 struct mm_struct *mm)
207 { 207 {
208 if (vp->file) { 208 if (vp->file) {
209 if (vp->adj_next) 209 if (vp->adj_next)
210 vma_interval_tree_inse 210 vma_interval_tree_insert(vp->adj_next,
211 211 &vp->mapping->i_mmap);
212 vma_interval_tree_insert(vp->v 212 vma_interval_tree_insert(vp->vma, &vp->mapping->i_mmap);
213 flush_dcache_mmap_unlock(vp->m 213 flush_dcache_mmap_unlock(vp->mapping);
214 } 214 }
215 215
216 if (vp->remove && vp->file) { 216 if (vp->remove && vp->file) {
217 __remove_shared_vm_struct(vp-> 217 __remove_shared_vm_struct(vp->remove, vp->mapping);
218 if (vp->remove2) 218 if (vp->remove2)
219 __remove_shared_vm_str 219 __remove_shared_vm_struct(vp->remove2, vp->mapping);
220 } else if (vp->insert) { 220 } else if (vp->insert) {
221 /* 221 /*
222 * split_vma has split insert 222 * split_vma has split insert from vma, and needs
223 * us to insert it before drop 223 * us to insert it before dropping the locks
224 * (it may either follow vma o 224 * (it may either follow vma or precede it).
225 */ 225 */
226 vma_iter_store(vmi, vp->insert 226 vma_iter_store(vmi, vp->insert);
227 mm->map_count++; 227 mm->map_count++;
228 } 228 }
229 229
230 if (vp->anon_vma) { 230 if (vp->anon_vma) {
231 anon_vma_interval_tree_post_up 231 anon_vma_interval_tree_post_update_vma(vp->vma);
232 if (vp->adj_next) 232 if (vp->adj_next)
233 anon_vma_interval_tree 233 anon_vma_interval_tree_post_update_vma(vp->adj_next);
234 anon_vma_unlock_write(vp->anon 234 anon_vma_unlock_write(vp->anon_vma);
235 } 235 }
236 236
237 if (vp->file) { 237 if (vp->file) {
238 i_mmap_unlock_write(vp->mappin 238 i_mmap_unlock_write(vp->mapping);
239 uprobe_mmap(vp->vma); 239 uprobe_mmap(vp->vma);
240 240
241 if (vp->adj_next) 241 if (vp->adj_next)
242 uprobe_mmap(vp->adj_ne 242 uprobe_mmap(vp->adj_next);
243 } 243 }
244 244
245 if (vp->remove) { 245 if (vp->remove) {
246 again: 246 again:
247 vma_mark_detached(vp->remove, 247 vma_mark_detached(vp->remove, true);
248 if (vp->file) { 248 if (vp->file) {
249 uprobe_munmap(vp->remo 249 uprobe_munmap(vp->remove, vp->remove->vm_start,
250 vp->remo 250 vp->remove->vm_end);
251 fput(vp->file); 251 fput(vp->file);
252 } 252 }
253 if (vp->remove->anon_vma) 253 if (vp->remove->anon_vma)
254 anon_vma_merge(vp->vma 254 anon_vma_merge(vp->vma, vp->remove);
255 mm->map_count--; 255 mm->map_count--;
256 mpol_put(vma_policy(vp->remove 256 mpol_put(vma_policy(vp->remove));
257 if (!vp->remove2) 257 if (!vp->remove2)
258 WARN_ON_ONCE(vp->vma-> 258 WARN_ON_ONCE(vp->vma->vm_end < vp->remove->vm_end);
259 vm_area_free(vp->remove); 259 vm_area_free(vp->remove);
260 260
261 /* 261 /*
262 * In mprotect's case 6 (see c 262 * In mprotect's case 6 (see comments on vma_merge),
263 * we are removing both mid an 263 * we are removing both mid and next vmas
264 */ 264 */
265 if (vp->remove2) { 265 if (vp->remove2) {
266 vp->remove = vp->remov 266 vp->remove = vp->remove2;
267 vp->remove2 = NULL; 267 vp->remove2 = NULL;
268 goto again; 268 goto again;
269 } 269 }
270 } 270 }
271 if (vp->insert && vp->file) 271 if (vp->insert && vp->file)
272 uprobe_mmap(vp->insert); 272 uprobe_mmap(vp->insert);
273 } 273 }
274 274
275 /* 275 /*
276 * init_vma_prep() - Initializer wrapper for v 276 * init_vma_prep() - Initializer wrapper for vma_prepare struct
277 * @vp: The vma_prepare struct 277 * @vp: The vma_prepare struct
278 * @vma: The vma that will be altered once loc 278 * @vma: The vma that will be altered once locked
279 */ 279 */
280 static void init_vma_prep(struct vma_prepare * 280 static void init_vma_prep(struct vma_prepare *vp, struct vm_area_struct *vma)
281 { 281 {
282 init_multi_vma_prep(vp, vma, NULL, NUL 282 init_multi_vma_prep(vp, vma, NULL, NULL, NULL);
283 } 283 }
284 284
285 /* 285 /*
286 * Can the proposed VMA be merged with the lef 286 * Can the proposed VMA be merged with the left (previous) VMA taking into
287 * account the start position of the proposed 287 * account the start position of the proposed range.
288 */ 288 */
289 static bool can_vma_merge_left(struct vma_merg 289 static bool can_vma_merge_left(struct vma_merge_struct *vmg)
290 290
291 { 291 {
292 return vmg->prev && vmg->prev->vm_end 292 return vmg->prev && vmg->prev->vm_end == vmg->start &&
293 can_vma_merge_after(vmg); 293 can_vma_merge_after(vmg);
294 } 294 }
295 295
296 /* 296 /*
297 * Can the proposed VMA be merged with the rig 297 * Can the proposed VMA be merged with the right (next) VMA taking into
298 * account the end position of the proposed ra 298 * account the end position of the proposed range.
299 * 299 *
300 * In addition, if we can merge with the left 300 * In addition, if we can merge with the left VMA, ensure that left and right
301 * anon_vma's are also compatible. 301 * anon_vma's are also compatible.
302 */ 302 */
303 static bool can_vma_merge_right(struct vma_mer 303 static bool can_vma_merge_right(struct vma_merge_struct *vmg,
304 bool can_merge 304 bool can_merge_left)
305 { 305 {
306 if (!vmg->next || vmg->end != vmg->nex 306 if (!vmg->next || vmg->end != vmg->next->vm_start ||
307 !can_vma_merge_before(vmg)) 307 !can_vma_merge_before(vmg))
308 return false; 308 return false;
309 309
310 if (!can_merge_left) 310 if (!can_merge_left)
311 return true; 311 return true;
312 312
313 /* 313 /*
314 * If we can merge with prev (left) an 314 * If we can merge with prev (left) and next (right), indicating that
315 * each VMA's anon_vma is compatible w 315 * each VMA's anon_vma is compatible with the proposed anon_vma, this
316 * does not mean prev and next are com 316 * does not mean prev and next are compatible with EACH OTHER.
317 * 317 *
318 * We therefore check this in addition 318 * We therefore check this in addition to mergeability to either side.
319 */ 319 */
320 return are_anon_vmas_compatible(vmg->p 320 return are_anon_vmas_compatible(vmg->prev, vmg->next);
321 } 321 }
322 322
323 /* 323 /*
324 * Close a vm structure and free it. 324 * Close a vm structure and free it.
325 */ 325 */
326 void remove_vma(struct vm_area_struct *vma, bo 326 void remove_vma(struct vm_area_struct *vma, bool unreachable)
327 { 327 {
328 might_sleep(); 328 might_sleep();
329 vma_close(vma); 329 vma_close(vma);
330 if (vma->vm_file) 330 if (vma->vm_file)
331 fput(vma->vm_file); 331 fput(vma->vm_file);
332 mpol_put(vma_policy(vma)); 332 mpol_put(vma_policy(vma));
333 if (unreachable) 333 if (unreachable)
334 __vm_area_free(vma); 334 __vm_area_free(vma);
335 else 335 else
336 vm_area_free(vma); 336 vm_area_free(vma);
337 } 337 }
338 338
339 /* 339 /*
340 * Get rid of page table information in the in 340 * Get rid of page table information in the indicated region.
341 * 341 *
342 * Called with the mm semaphore held. 342 * Called with the mm semaphore held.
343 */ 343 */
344 void unmap_region(struct ma_state *mas, struct 344 void unmap_region(struct ma_state *mas, struct vm_area_struct *vma,
345 struct vm_area_struct *prev, s 345 struct vm_area_struct *prev, struct vm_area_struct *next)
346 { 346 {
347 struct mm_struct *mm = vma->vm_mm; 347 struct mm_struct *mm = vma->vm_mm;
348 struct mmu_gather tlb; 348 struct mmu_gather tlb;
349 349
350 lru_add_drain(); 350 lru_add_drain();
351 tlb_gather_mmu(&tlb, mm); 351 tlb_gather_mmu(&tlb, mm);
352 update_hiwater_rss(mm); 352 update_hiwater_rss(mm);
353 unmap_vmas(&tlb, mas, vma, vma->vm_sta 353 unmap_vmas(&tlb, mas, vma, vma->vm_start, vma->vm_end, vma->vm_end,
354 /* mm_wr_locked = */ true); 354 /* mm_wr_locked = */ true);
355 mas_set(mas, vma->vm_end); 355 mas_set(mas, vma->vm_end);
356 free_pgtables(&tlb, mas, vma, prev ? p 356 free_pgtables(&tlb, mas, vma, prev ? prev->vm_end : FIRST_USER_ADDRESS,
357 next ? next->vm_start : 357 next ? next->vm_start : USER_PGTABLES_CEILING,
358 /* mm_wr_locked = */ tru 358 /* mm_wr_locked = */ true);
359 tlb_finish_mmu(&tlb); 359 tlb_finish_mmu(&tlb);
360 } 360 }
361 361
362 /* 362 /*
363 * __split_vma() bypasses sysctl_max_map_count 363 * __split_vma() bypasses sysctl_max_map_count checking. We use this where it
364 * has already been checked or doesn't make se 364 * has already been checked or doesn't make sense to fail.
365 * VMA Iterator will point to the original VMA 365 * VMA Iterator will point to the original VMA.
366 */ 366 */
367 static int __split_vma(struct vma_iterator *vm 367 static int __split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma,
368 unsigned long addr, int 368 unsigned long addr, int new_below)
369 { 369 {
370 struct vma_prepare vp; 370 struct vma_prepare vp;
371 struct vm_area_struct *new; 371 struct vm_area_struct *new;
372 int err; 372 int err;
373 373
374 WARN_ON(vma->vm_start >= addr); 374 WARN_ON(vma->vm_start >= addr);
375 WARN_ON(vma->vm_end <= addr); 375 WARN_ON(vma->vm_end <= addr);
376 376
377 if (vma->vm_ops && vma->vm_ops->may_sp 377 if (vma->vm_ops && vma->vm_ops->may_split) {
378 err = vma->vm_ops->may_split(v 378 err = vma->vm_ops->may_split(vma, addr);
379 if (err) 379 if (err)
380 return err; 380 return err;
381 } 381 }
382 382
383 new = vm_area_dup(vma); 383 new = vm_area_dup(vma);
384 if (!new) 384 if (!new)
385 return -ENOMEM; 385 return -ENOMEM;
386 386
387 if (new_below) { 387 if (new_below) {
388 new->vm_end = addr; 388 new->vm_end = addr;
389 } else { 389 } else {
390 new->vm_start = addr; 390 new->vm_start = addr;
391 new->vm_pgoff += ((addr - vma- 391 new->vm_pgoff += ((addr - vma->vm_start) >> PAGE_SHIFT);
392 } 392 }
393 393
394 err = -ENOMEM; 394 err = -ENOMEM;
395 vma_iter_config(vmi, new->vm_start, ne 395 vma_iter_config(vmi, new->vm_start, new->vm_end);
396 if (vma_iter_prealloc(vmi, new)) 396 if (vma_iter_prealloc(vmi, new))
397 goto out_free_vma; 397 goto out_free_vma;
398 398
399 err = vma_dup_policy(vma, new); 399 err = vma_dup_policy(vma, new);
400 if (err) 400 if (err)
401 goto out_free_vmi; 401 goto out_free_vmi;
402 402
403 err = anon_vma_clone(new, vma); 403 err = anon_vma_clone(new, vma);
404 if (err) 404 if (err)
405 goto out_free_mpol; 405 goto out_free_mpol;
406 406
407 if (new->vm_file) 407 if (new->vm_file)
408 get_file(new->vm_file); 408 get_file(new->vm_file);
409 409
410 if (new->vm_ops && new->vm_ops->open) 410 if (new->vm_ops && new->vm_ops->open)
411 new->vm_ops->open(new); 411 new->vm_ops->open(new);
412 412
413 vma_start_write(vma); 413 vma_start_write(vma);
414 vma_start_write(new); 414 vma_start_write(new);
415 415
416 init_vma_prep(&vp, vma); 416 init_vma_prep(&vp, vma);
417 vp.insert = new; 417 vp.insert = new;
418 vma_prepare(&vp); 418 vma_prepare(&vp);
419 vma_adjust_trans_huge(vma, vma->vm_sta 419 vma_adjust_trans_huge(vma, vma->vm_start, addr, 0);
420 420
421 if (new_below) { 421 if (new_below) {
422 vma->vm_start = addr; 422 vma->vm_start = addr;
423 vma->vm_pgoff += (addr - new-> 423 vma->vm_pgoff += (addr - new->vm_start) >> PAGE_SHIFT;
424 } else { 424 } else {
425 vma->vm_end = addr; 425 vma->vm_end = addr;
426 } 426 }
427 427
428 /* vma_complete stores the new vma */ 428 /* vma_complete stores the new vma */
429 vma_complete(&vp, vmi, vma->vm_mm); 429 vma_complete(&vp, vmi, vma->vm_mm);
430 validate_mm(vma->vm_mm); 430 validate_mm(vma->vm_mm);
431 431
432 /* Success. */ 432 /* Success. */
433 if (new_below) 433 if (new_below)
434 vma_next(vmi); 434 vma_next(vmi);
435 else 435 else
436 vma_prev(vmi); 436 vma_prev(vmi);
437 437
438 return 0; 438 return 0;
439 439
440 out_free_mpol: 440 out_free_mpol:
441 mpol_put(vma_policy(new)); 441 mpol_put(vma_policy(new));
442 out_free_vmi: 442 out_free_vmi:
443 vma_iter_free(vmi); 443 vma_iter_free(vmi);
444 out_free_vma: 444 out_free_vma:
445 vm_area_free(new); 445 vm_area_free(new);
446 return err; 446 return err;
447 } 447 }
448 448
449 /* 449 /*
450 * Split a vma into two pieces at address 'add 450 * Split a vma into two pieces at address 'addr', a new vma is allocated
451 * either for the first part or the tail. 451 * either for the first part or the tail.
452 */ 452 */
453 static int split_vma(struct vma_iterator *vmi, 453 static int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma,
454 unsigned long addr, int n 454 unsigned long addr, int new_below)
455 { 455 {
456 if (vma->vm_mm->map_count >= sysctl_ma 456 if (vma->vm_mm->map_count >= sysctl_max_map_count)
457 return -ENOMEM; 457 return -ENOMEM;
458 458
459 return __split_vma(vmi, vma, addr, new 459 return __split_vma(vmi, vma, addr, new_below);
460 } 460 }
461 461
462 /* 462 /*
463 * vma has some anon_vma assigned, and is alre 463 * vma has some anon_vma assigned, and is already inserted on that
464 * anon_vma's interval trees. 464 * anon_vma's interval trees.
465 * 465 *
466 * Before updating the vma's vm_start / vm_end 466 * Before updating the vma's vm_start / vm_end / vm_pgoff fields, the
467 * vma must be removed from the anon_vma's int 467 * vma must be removed from the anon_vma's interval trees using
468 * anon_vma_interval_tree_pre_update_vma(). 468 * anon_vma_interval_tree_pre_update_vma().
469 * 469 *
470 * After the update, the vma will be reinserte 470 * After the update, the vma will be reinserted using
471 * anon_vma_interval_tree_post_update_vma(). 471 * anon_vma_interval_tree_post_update_vma().
472 * 472 *
473 * The entire update must be protected by excl 473 * The entire update must be protected by exclusive mmap_lock and by
474 * the root anon_vma's mutex. 474 * the root anon_vma's mutex.
475 */ 475 */
476 void 476 void
477 anon_vma_interval_tree_pre_update_vma(struct v 477 anon_vma_interval_tree_pre_update_vma(struct vm_area_struct *vma)
478 { 478 {
479 struct anon_vma_chain *avc; 479 struct anon_vma_chain *avc;
480 480
481 list_for_each_entry(avc, &vma->anon_vm 481 list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
482 anon_vma_interval_tree_remove( 482 anon_vma_interval_tree_remove(avc, &avc->anon_vma->rb_root);
483 } 483 }
484 484
485 void 485 void
486 anon_vma_interval_tree_post_update_vma(struct 486 anon_vma_interval_tree_post_update_vma(struct vm_area_struct *vma)
487 { 487 {
488 struct anon_vma_chain *avc; 488 struct anon_vma_chain *avc;
489 489
490 list_for_each_entry(avc, &vma->anon_vm 490 list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
491 anon_vma_interval_tree_insert( 491 anon_vma_interval_tree_insert(avc, &avc->anon_vma->rb_root);
492 } 492 }
493 493
494 /* 494 /*
495 * dup_anon_vma() - Helper function to duplica 495 * dup_anon_vma() - Helper function to duplicate anon_vma
496 * @dst: The destination VMA 496 * @dst: The destination VMA
497 * @src: The source VMA 497 * @src: The source VMA
498 * @dup: Pointer to the destination VMA when s 498 * @dup: Pointer to the destination VMA when successful.
499 * 499 *
500 * Returns: 0 on success. 500 * Returns: 0 on success.
501 */ 501 */
502 static int dup_anon_vma(struct vm_area_struct 502 static int dup_anon_vma(struct vm_area_struct *dst,
503 struct vm_area_struct 503 struct vm_area_struct *src, struct vm_area_struct **dup)
504 { 504 {
505 /* 505 /*
506 * Easily overlooked: when mprotect sh 506 * Easily overlooked: when mprotect shifts the boundary, make sure the
507 * expanding vma has anon_vma set if t 507 * expanding vma has anon_vma set if the shrinking vma had, to cover any
508 * anon pages imported. 508 * anon pages imported.
509 */ 509 */
510 if (src->anon_vma && !dst->anon_vma) { 510 if (src->anon_vma && !dst->anon_vma) {
511 int ret; 511 int ret;
512 512
513 vma_assert_write_locked(dst); 513 vma_assert_write_locked(dst);
514 dst->anon_vma = src->anon_vma; 514 dst->anon_vma = src->anon_vma;
515 ret = anon_vma_clone(dst, src) 515 ret = anon_vma_clone(dst, src);
516 if (ret) 516 if (ret)
517 return ret; 517 return ret;
518 518
519 *dup = dst; 519 *dup = dst;
520 } 520 }
521 521
522 return 0; 522 return 0;
523 } 523 }
524 524
525 #ifdef CONFIG_DEBUG_VM_MAPLE_TREE 525 #ifdef CONFIG_DEBUG_VM_MAPLE_TREE
526 void validate_mm(struct mm_struct *mm) 526 void validate_mm(struct mm_struct *mm)
527 { 527 {
528 int bug = 0; 528 int bug = 0;
529 int i = 0; 529 int i = 0;
530 struct vm_area_struct *vma; 530 struct vm_area_struct *vma;
531 VMA_ITERATOR(vmi, mm, 0); 531 VMA_ITERATOR(vmi, mm, 0);
532 532
533 mt_validate(&mm->mm_mt); 533 mt_validate(&mm->mm_mt);
534 for_each_vma(vmi, vma) { 534 for_each_vma(vmi, vma) {
535 #ifdef CONFIG_DEBUG_VM_RB 535 #ifdef CONFIG_DEBUG_VM_RB
536 struct anon_vma *anon_vma = vm 536 struct anon_vma *anon_vma = vma->anon_vma;
537 struct anon_vma_chain *avc; 537 struct anon_vma_chain *avc;
538 #endif 538 #endif
539 unsigned long vmi_start, vmi_e 539 unsigned long vmi_start, vmi_end;
540 bool warn = 0; 540 bool warn = 0;
541 541
542 vmi_start = vma_iter_addr(&vmi 542 vmi_start = vma_iter_addr(&vmi);
543 vmi_end = vma_iter_end(&vmi); 543 vmi_end = vma_iter_end(&vmi);
544 if (VM_WARN_ON_ONCE_MM(vma->vm 544 if (VM_WARN_ON_ONCE_MM(vma->vm_end != vmi_end, mm))
545 warn = 1; 545 warn = 1;
546 546
547 if (VM_WARN_ON_ONCE_MM(vma->vm 547 if (VM_WARN_ON_ONCE_MM(vma->vm_start != vmi_start, mm))
548 warn = 1; 548 warn = 1;
549 549
550 if (warn) { 550 if (warn) {
551 pr_emerg("issue in %s\ 551 pr_emerg("issue in %s\n", current->comm);
552 dump_stack(); 552 dump_stack();
553 dump_vma(vma); 553 dump_vma(vma);
554 pr_emerg("tree range: 554 pr_emerg("tree range: %px start %lx end %lx\n", vma,
555 vmi_start, vm 555 vmi_start, vmi_end - 1);
556 vma_iter_dump_tree(&vm 556 vma_iter_dump_tree(&vmi);
557 } 557 }
558 558
559 #ifdef CONFIG_DEBUG_VM_RB 559 #ifdef CONFIG_DEBUG_VM_RB
560 if (anon_vma) { 560 if (anon_vma) {
561 anon_vma_lock_read(ano 561 anon_vma_lock_read(anon_vma);
562 list_for_each_entry(av 562 list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
563 anon_vma_inter 563 anon_vma_interval_tree_verify(avc);
564 anon_vma_unlock_read(a 564 anon_vma_unlock_read(anon_vma);
565 } 565 }
566 #endif 566 #endif
567 i++; 567 i++;
568 } 568 }
569 if (i != mm->map_count) { 569 if (i != mm->map_count) {
570 pr_emerg("map_count %d vma ite 570 pr_emerg("map_count %d vma iterator %d\n", mm->map_count, i);
571 bug = 1; 571 bug = 1;
572 } 572 }
573 VM_BUG_ON_MM(bug, mm); 573 VM_BUG_ON_MM(bug, mm);
574 } 574 }
575 #endif /* CONFIG_DEBUG_VM_MAPLE_TREE */ 575 #endif /* CONFIG_DEBUG_VM_MAPLE_TREE */
576 576
577 /* Actually perform the VMA merge operation. * 577 /* Actually perform the VMA merge operation. */
578 static int commit_merge(struct vma_merge_struc 578 static int commit_merge(struct vma_merge_struct *vmg,
579 struct vm_area_struct 579 struct vm_area_struct *adjust,
580 struct vm_area_struct 580 struct vm_area_struct *remove,
581 struct vm_area_struct 581 struct vm_area_struct *remove2,
582 long adj_start, 582 long adj_start,
583 bool expanded) 583 bool expanded)
584 { 584 {
585 struct vma_prepare vp; 585 struct vma_prepare vp;
586 586
587 init_multi_vma_prep(&vp, vmg->vma, adj 587 init_multi_vma_prep(&vp, vmg->vma, adjust, remove, remove2);
588 588
589 VM_WARN_ON(vp.anon_vma && adjust && ad 589 VM_WARN_ON(vp.anon_vma && adjust && adjust->anon_vma &&
590 vp.anon_vma != adjust->anon 590 vp.anon_vma != adjust->anon_vma);
591 591
592 if (expanded) { 592 if (expanded) {
593 /* Note: vma iterator must be 593 /* Note: vma iterator must be pointing to 'start'. */
594 vma_iter_config(vmg->vmi, vmg- 594 vma_iter_config(vmg->vmi, vmg->start, vmg->end);
595 } else { 595 } else {
596 vma_iter_config(vmg->vmi, adju 596 vma_iter_config(vmg->vmi, adjust->vm_start + adj_start,
597 adjust->vm_end 597 adjust->vm_end);
598 } 598 }
599 599
600 if (vma_iter_prealloc(vmg->vmi, vmg->v 600 if (vma_iter_prealloc(vmg->vmi, vmg->vma))
601 return -ENOMEM; 601 return -ENOMEM;
602 602
603 vma_prepare(&vp); 603 vma_prepare(&vp);
604 vma_adjust_trans_huge(vmg->vma, vmg->s 604 vma_adjust_trans_huge(vmg->vma, vmg->start, vmg->end, adj_start);
605 vma_set_range(vmg->vma, vmg->start, vm 605 vma_set_range(vmg->vma, vmg->start, vmg->end, vmg->pgoff);
606 606
607 if (expanded) 607 if (expanded)
608 vma_iter_store(vmg->vmi, vmg-> 608 vma_iter_store(vmg->vmi, vmg->vma);
609 609
610 if (adj_start) { 610 if (adj_start) {
611 adjust->vm_start += adj_start; 611 adjust->vm_start += adj_start;
612 adjust->vm_pgoff += PHYS_PFN(a 612 adjust->vm_pgoff += PHYS_PFN(adj_start);
613 if (adj_start < 0) { 613 if (adj_start < 0) {
614 WARN_ON(expanded); 614 WARN_ON(expanded);
615 vma_iter_store(vmg->vm 615 vma_iter_store(vmg->vmi, adjust);
616 } 616 }
617 } 617 }
618 618
619 vma_complete(&vp, vmg->vmi, vmg->vma-> 619 vma_complete(&vp, vmg->vmi, vmg->vma->vm_mm);
620 620
621 return 0; 621 return 0;
622 } 622 }
623 623
624 /* We can only remove VMAs when merging if the 624 /* We can only remove VMAs when merging if they do not have a close hook. */
625 static bool can_merge_remove_vma(struct vm_are 625 static bool can_merge_remove_vma(struct vm_area_struct *vma)
626 { 626 {
627 return !vma->vm_ops || !vma->vm_ops->c 627 return !vma->vm_ops || !vma->vm_ops->close;
628 } 628 }
629 629
630 /* 630 /*
631 * vma_merge_existing_range - Attempt to merge 631 * vma_merge_existing_range - Attempt to merge VMAs based on a VMA having its
632 * attributes modified. 632 * attributes modified.
633 * 633 *
634 * @vmg: Describes the modifications being mad 634 * @vmg: Describes the modifications being made to a VMA and associated
635 * metadata. 635 * metadata.
636 * 636 *
637 * When the attributes of a range within a VMA 637 * When the attributes of a range within a VMA change, then it might be possible
638 * for immediately adjacent VMAs to be merged 638 * for immediately adjacent VMAs to be merged into that VMA due to having
639 * identical properties. 639 * identical properties.
640 * 640 *
641 * This function checks for the existence of a 641 * This function checks for the existence of any such mergeable VMAs and updates
642 * the maple tree describing the @vmg->vma->vm 642 * the maple tree describing the @vmg->vma->vm_mm address space to account for
643 * this, as well as any VMAs shrunk/expanded/d 643 * this, as well as any VMAs shrunk/expanded/deleted as a result of this merge.
644 * 644 *
645 * As part of this operation, if a merge occur 645 * As part of this operation, if a merge occurs, the @vmg object will have its
646 * vma, start, end, and pgoff fields modified 646 * vma, start, end, and pgoff fields modified to execute the merge. Subsequent
647 * calls to this function should reset these f 647 * calls to this function should reset these fields.
648 * 648 *
649 * Returns: The merged VMA if merge succeeds, 649 * Returns: The merged VMA if merge succeeds, or NULL otherwise.
650 * 650 *
651 * ASSUMPTIONS: 651 * ASSUMPTIONS:
652 * - The caller must assign the VMA to be modi 652 * - The caller must assign the VMA to be modifed to @vmg->vma.
653 * - The caller must have set @vmg->prev to th 653 * - The caller must have set @vmg->prev to the previous VMA, if there is one.
654 * - The caller must not set @vmg->next, as we 654 * - The caller must not set @vmg->next, as we determine this.
655 * - The caller must hold a WRITE lock on the 655 * - The caller must hold a WRITE lock on the mm_struct->mmap_lock.
656 * - vmi must be positioned within [@vmg->vma- 656 * - vmi must be positioned within [@vmg->vma->vm_start, @vmg->vma->vm_end).
657 */ 657 */
658 static struct vm_area_struct *vma_merge_existi 658 static struct vm_area_struct *vma_merge_existing_range(struct vma_merge_struct *vmg)
659 { 659 {
660 struct vm_area_struct *vma = vmg->vma; 660 struct vm_area_struct *vma = vmg->vma;
661 struct vm_area_struct *prev = vmg->pre 661 struct vm_area_struct *prev = vmg->prev;
662 struct vm_area_struct *next, *res; 662 struct vm_area_struct *next, *res;
663 struct vm_area_struct *anon_dup = NULL 663 struct vm_area_struct *anon_dup = NULL;
664 struct vm_area_struct *adjust = NULL; 664 struct vm_area_struct *adjust = NULL;
665 unsigned long start = vmg->start; 665 unsigned long start = vmg->start;
666 unsigned long end = vmg->end; 666 unsigned long end = vmg->end;
667 bool left_side = vma && start == vma-> 667 bool left_side = vma && start == vma->vm_start;
668 bool right_side = vma && end == vma->v 668 bool right_side = vma && end == vma->vm_end;
669 int err = 0; 669 int err = 0;
670 long adj_start = 0; 670 long adj_start = 0;
671 bool merge_will_delete_vma, merge_will 671 bool merge_will_delete_vma, merge_will_delete_next;
672 bool merge_left, merge_right, merge_bo 672 bool merge_left, merge_right, merge_both;
673 bool expanded; 673 bool expanded;
674 674
675 mmap_assert_write_locked(vmg->mm); 675 mmap_assert_write_locked(vmg->mm);
676 VM_WARN_ON(!vma); /* We are modifying 676 VM_WARN_ON(!vma); /* We are modifying a VMA, so caller must specify. */
677 VM_WARN_ON(vmg->next); /* We set this. 677 VM_WARN_ON(vmg->next); /* We set this. */
678 VM_WARN_ON(prev && start <= prev->vm_s 678 VM_WARN_ON(prev && start <= prev->vm_start);
679 VM_WARN_ON(start >= end); 679 VM_WARN_ON(start >= end);
680 /* 680 /*
681 * If vma == prev, then we are offset 681 * If vma == prev, then we are offset into a VMA. Otherwise, if we are
682 * not, we must span a portion of the 682 * not, we must span a portion of the VMA.
683 */ 683 */
684 VM_WARN_ON(vma && ((vma != prev && vmg 684 VM_WARN_ON(vma && ((vma != prev && vmg->start != vma->vm_start) ||
685 vmg->end > vma->vm_ 685 vmg->end > vma->vm_end));
686 /* The vmi must be positioned within v 686 /* The vmi must be positioned within vmg->vma. */
687 VM_WARN_ON(vma && !(vma_iter_addr(vmg- 687 VM_WARN_ON(vma && !(vma_iter_addr(vmg->vmi) >= vma->vm_start &&
688 vma_iter_addr(vmg- 688 vma_iter_addr(vmg->vmi) < vma->vm_end));
689 689
690 vmg->state = VMA_MERGE_NOMERGE; 690 vmg->state = VMA_MERGE_NOMERGE;
691 691
692 /* 692 /*
693 * If a special mapping or if the rang 693 * If a special mapping or if the range being modified is neither at the
694 * furthermost left or right side of t 694 * furthermost left or right side of the VMA, then we have no chance of
695 * merging and should abort. 695 * merging and should abort.
696 */ 696 */
697 if (vmg->flags & VM_SPECIAL || (!left_ 697 if (vmg->flags & VM_SPECIAL || (!left_side && !right_side))
698 return NULL; 698 return NULL;
699 699
700 if (left_side) 700 if (left_side)
701 merge_left = can_vma_merge_lef 701 merge_left = can_vma_merge_left(vmg);
702 else 702 else
703 merge_left = false; 703 merge_left = false;
704 704
705 if (right_side) { 705 if (right_side) {
706 next = vmg->next = vma_iter_ne 706 next = vmg->next = vma_iter_next_range(vmg->vmi);
707 vma_iter_prev_range(vmg->vmi); 707 vma_iter_prev_range(vmg->vmi);
708 708
709 merge_right = can_vma_merge_ri 709 merge_right = can_vma_merge_right(vmg, merge_left);
710 } else { 710 } else {
711 merge_right = false; 711 merge_right = false;
712 next = NULL; 712 next = NULL;
713 } 713 }
714 714
715 if (merge_left) /* If merging 715 if (merge_left) /* If merging prev, position iterator there. */
716 vma_prev(vmg->vmi); 716 vma_prev(vmg->vmi);
717 else if (!merge_right) /* If we have 717 else if (!merge_right) /* If we have nothing to merge, abort. */
718 return NULL; 718 return NULL;
719 719
720 merge_both = merge_left && merge_right 720 merge_both = merge_left && merge_right;
721 /* If we span the entire VMA, a merge 721 /* If we span the entire VMA, a merge implies it will be deleted. */
722 merge_will_delete_vma = left_side && r 722 merge_will_delete_vma = left_side && right_side;
723 723
724 /* 724 /*
725 * If we need to remove vma in its ent 725 * If we need to remove vma in its entirety but are unable to do so,
726 * we have no sensible recourse but to 726 * we have no sensible recourse but to abort the merge.
727 */ 727 */
728 if (merge_will_delete_vma && !can_merg 728 if (merge_will_delete_vma && !can_merge_remove_vma(vma))
729 return NULL; 729 return NULL;
730 730
731 /* 731 /*
732 * If we merge both VMAs, then next is 732 * If we merge both VMAs, then next is also deleted. This implies
733 * merge_will_delete_vma also. 733 * merge_will_delete_vma also.
734 */ 734 */
735 merge_will_delete_next = merge_both; 735 merge_will_delete_next = merge_both;
736 736
737 /* 737 /*
738 * If we cannot delete next, then we c 738 * If we cannot delete next, then we can reduce the operation to merging
739 * prev and vma (thereby deleting vma) 739 * prev and vma (thereby deleting vma).
740 */ 740 */
741 if (merge_will_delete_next && !can_mer 741 if (merge_will_delete_next && !can_merge_remove_vma(next)) {
742 merge_will_delete_next = false 742 merge_will_delete_next = false;
743 merge_right = false; 743 merge_right = false;
744 merge_both = false; 744 merge_both = false;
745 } 745 }
746 746
747 /* No matter what happens, we will be 747 /* No matter what happens, we will be adjusting vma. */
748 vma_start_write(vma); 748 vma_start_write(vma);
749 749
750 if (merge_left) 750 if (merge_left)
751 vma_start_write(prev); 751 vma_start_write(prev);
752 752
753 if (merge_right) 753 if (merge_right)
754 vma_start_write(next); 754 vma_start_write(next);
755 755
756 if (merge_both) { 756 if (merge_both) {
757 /* 757 /*
758 * |<----->| 758 * |<----->|
759 * |-------*********-------| 759 * |-------*********-------|
760 * prev vma next 760 * prev vma next
761 * extend delete delete 761 * extend delete delete
762 */ 762 */
763 763
764 vmg->vma = prev; 764 vmg->vma = prev;
765 vmg->start = prev->vm_start; 765 vmg->start = prev->vm_start;
766 vmg->end = next->vm_end; 766 vmg->end = next->vm_end;
767 vmg->pgoff = prev->vm_pgoff; 767 vmg->pgoff = prev->vm_pgoff;
768 768
769 /* 769 /*
770 * We already ensured anon_vma 770 * We already ensured anon_vma compatibility above, so now it's
771 * simply a case of, if prev h 771 * simply a case of, if prev has no anon_vma object, which of
772 * next or vma contains the an 772 * next or vma contains the anon_vma we must duplicate.
773 */ 773 */
774 err = dup_anon_vma(prev, next- 774 err = dup_anon_vma(prev, next->anon_vma ? next : vma, &anon_dup);
775 } else if (merge_left) { 775 } else if (merge_left) {
776 /* 776 /*
777 * |<----->| OR 777 * |<----->| OR
778 * |<--------->| 778 * |<--------->|
779 * |-------************* 779 * |-------*************
780 * prev vma 780 * prev vma
781 * extend shrink/delete 781 * extend shrink/delete
782 */ 782 */
783 783
784 vmg->vma = prev; 784 vmg->vma = prev;
785 vmg->start = prev->vm_start; 785 vmg->start = prev->vm_start;
786 vmg->pgoff = prev->vm_pgoff; 786 vmg->pgoff = prev->vm_pgoff;
787 787
788 if (!merge_will_delete_vma) { 788 if (!merge_will_delete_vma) {
789 adjust = vma; 789 adjust = vma;
790 adj_start = vmg->end - 790 adj_start = vmg->end - vma->vm_start;
791 } 791 }
792 792
793 err = dup_anon_vma(prev, vma, 793 err = dup_anon_vma(prev, vma, &anon_dup);
794 } else { /* merge_right */ 794 } else { /* merge_right */
795 /* 795 /*
796 * |<----->| OR 796 * |<----->| OR
797 * |<--------->| 797 * |<--------->|
798 * *************-------| 798 * *************-------|
799 * vma next 799 * vma next
800 * shrink/delete extend 800 * shrink/delete extend
801 */ 801 */
802 802
803 pgoff_t pglen = PHYS_PFN(vmg-> 803 pgoff_t pglen = PHYS_PFN(vmg->end - vmg->start);
804 804
805 VM_WARN_ON(!merge_right); 805 VM_WARN_ON(!merge_right);
806 /* If we are offset into a VMA 806 /* If we are offset into a VMA, then prev must be vma. */
807 VM_WARN_ON(vmg->start > vma->v 807 VM_WARN_ON(vmg->start > vma->vm_start && prev && vma != prev);
808 808
809 if (merge_will_delete_vma) { 809 if (merge_will_delete_vma) {
810 vmg->vma = next; 810 vmg->vma = next;
811 vmg->end = next->vm_en 811 vmg->end = next->vm_end;
812 vmg->pgoff = next->vm_ 812 vmg->pgoff = next->vm_pgoff - pglen;
813 } else { 813 } else {
814 /* 814 /*
815 * We shrink vma and e 815 * We shrink vma and expand next.
816 * 816 *
817 * IMPORTANT: This is 817 * IMPORTANT: This is the ONLY case where the final
818 * merged VMA is NOT v 818 * merged VMA is NOT vmg->vma, but rather vmg->next.
819 */ 819 */
820 820
821 vmg->start = vma->vm_s 821 vmg->start = vma->vm_start;
822 vmg->end = start; 822 vmg->end = start;
823 vmg->pgoff = vma->vm_p 823 vmg->pgoff = vma->vm_pgoff;
824 824
825 adjust = next; 825 adjust = next;
826 adj_start = -(vma->vm_ 826 adj_start = -(vma->vm_end - start);
827 } 827 }
828 828
829 err = dup_anon_vma(next, vma, 829 err = dup_anon_vma(next, vma, &anon_dup);
830 } 830 }
831 831
832 if (err) 832 if (err)
833 goto abort; 833 goto abort;
834 834
835 /* 835 /*
836 * In nearly all cases, we expand vmg- 836 * In nearly all cases, we expand vmg->vma. There is one exception -
837 * merge_right where we partially span 837 * merge_right where we partially span the VMA. In this case we shrink
838 * the end of vmg->vma and adjust the 838 * the end of vmg->vma and adjust the start of vmg->next accordingly.
839 */ 839 */
840 expanded = !merge_right || merge_will_ 840 expanded = !merge_right || merge_will_delete_vma;
841 841
842 if (commit_merge(vmg, adjust, 842 if (commit_merge(vmg, adjust,
843 merge_will_delete_vma 843 merge_will_delete_vma ? vma : NULL,
844 merge_will_delete_nex 844 merge_will_delete_next ? next : NULL,
845 adj_start, expanded)) 845 adj_start, expanded)) {
846 if (anon_dup) 846 if (anon_dup)
847 unlink_anon_vmas(anon_ 847 unlink_anon_vmas(anon_dup);
848 848
849 vmg->state = VMA_MERGE_ERROR_N 849 vmg->state = VMA_MERGE_ERROR_NOMEM;
850 return NULL; 850 return NULL;
851 } 851 }
852 852
853 res = merge_left ? prev : next; 853 res = merge_left ? prev : next;
854 khugepaged_enter_vma(res, vmg->flags); 854 khugepaged_enter_vma(res, vmg->flags);
855 855
856 vmg->state = VMA_MERGE_SUCCESS; 856 vmg->state = VMA_MERGE_SUCCESS;
857 return res; 857 return res;
858 858
859 abort: 859 abort:
860 vma_iter_set(vmg->vmi, start); 860 vma_iter_set(vmg->vmi, start);
861 vma_iter_load(vmg->vmi); 861 vma_iter_load(vmg->vmi);
862 vmg->state = VMA_MERGE_ERROR_NOMEM; 862 vmg->state = VMA_MERGE_ERROR_NOMEM;
863 return NULL; 863 return NULL;
864 } 864 }
865 865
866 /* 866 /*
867 * vma_merge_new_range - Attempt to merge a ne 867 * vma_merge_new_range - Attempt to merge a new VMA into address space
868 * 868 *
869 * @vmg: Describes the VMA we are adding, in t 869 * @vmg: Describes the VMA we are adding, in the range @vmg->start to @vmg->end
870 * (exclusive), which we try to merge wi 870 * (exclusive), which we try to merge with any adjacent VMAs if possible.
871 * 871 *
872 * We are about to add a VMA to the address sp 872 * We are about to add a VMA to the address space starting at @vmg->start and
873 * ending at @vmg->end. There are three differ 873 * ending at @vmg->end. There are three different possible scenarios:
874 * 874 *
875 * 1. There is a VMA with identical properties 875 * 1. There is a VMA with identical properties immediately adjacent to the
876 * proposed new VMA [@vmg->start, @vmg->end 876 * proposed new VMA [@vmg->start, @vmg->end) either before or after it -
877 * EXPAND that VMA: 877 * EXPAND that VMA:
878 * 878 *
879 * Proposed: |-----| or |-----| 879 * Proposed: |-----| or |-----|
880 * Existing: |----| |----| 880 * Existing: |----| |----|
881 * 881 *
882 * 2. There are VMAs with identical properties 882 * 2. There are VMAs with identical properties immediately adjacent to the
883 * proposed new VMA [@vmg->start, @vmg->end 883 * proposed new VMA [@vmg->start, @vmg->end) both before AND after it -
884 * EXPAND the former and REMOVE the latter: 884 * EXPAND the former and REMOVE the latter:
885 * 885 *
886 * Proposed: |-----| 886 * Proposed: |-----|
887 * Existing: |----| |----| 887 * Existing: |----| |----|
888 * 888 *
889 * 3. There are no VMAs immediately adjacent t 889 * 3. There are no VMAs immediately adjacent to the proposed new VMA or those
890 * VMAs do not have identical attributes - 890 * VMAs do not have identical attributes - NO MERGE POSSIBLE.
891 * 891 *
892 * In instances where we can merge, this funct 892 * In instances where we can merge, this function returns the expanded VMA which
893 * will have its range adjusted accordingly an 893 * will have its range adjusted accordingly and the underlying maple tree also
894 * adjusted. 894 * adjusted.
895 * 895 *
896 * Returns: In instances where no merge was po 896 * Returns: In instances where no merge was possible, NULL. Otherwise, a pointer
897 * to the VMA we expanded. 897 * to the VMA we expanded.
898 * 898 *
899 * This function adjusts @vmg to provide @vmg- 899 * This function adjusts @vmg to provide @vmg->next if not already specified,
900 * and adjusts [@vmg->start, @vmg->end) to spa 900 * and adjusts [@vmg->start, @vmg->end) to span the expanded range.
901 * 901 *
902 * ASSUMPTIONS: 902 * ASSUMPTIONS:
903 * - The caller must hold a WRITE lock on the 903 * - The caller must hold a WRITE lock on the mm_struct->mmap_lock.
904 * - The caller must have determined that [@vm 904 * - The caller must have determined that [@vmg->start, @vmg->end) is empty,
905 other than VMAs that will be unmapped sho 905 other than VMAs that will be unmapped should the operation succeed.
906 * - The caller must have specified the previo 906 * - The caller must have specified the previous vma in @vmg->prev.
907 * - The caller must have specified the next v 907 * - The caller must have specified the next vma in @vmg->next.
908 * - The caller must have positioned the vmi a 908 * - The caller must have positioned the vmi at or before the gap.
909 */ 909 */
910 struct vm_area_struct *vma_merge_new_range(str 910 struct vm_area_struct *vma_merge_new_range(struct vma_merge_struct *vmg)
911 { 911 {
912 struct vm_area_struct *prev = vmg->pre 912 struct vm_area_struct *prev = vmg->prev;
913 struct vm_area_struct *next = vmg->nex 913 struct vm_area_struct *next = vmg->next;
914 unsigned long start = vmg->start; 914 unsigned long start = vmg->start;
915 unsigned long end = vmg->end; 915 unsigned long end = vmg->end;
916 pgoff_t pgoff = vmg->pgoff; 916 pgoff_t pgoff = vmg->pgoff;
917 pgoff_t pglen = PHYS_PFN(end - start); 917 pgoff_t pglen = PHYS_PFN(end - start);
918 bool can_merge_left, can_merge_right; 918 bool can_merge_left, can_merge_right;
919 bool just_expand = vmg->merge_flags & 919 bool just_expand = vmg->merge_flags & VMG_FLAG_JUST_EXPAND;
920 920
921 mmap_assert_write_locked(vmg->mm); 921 mmap_assert_write_locked(vmg->mm);
922 VM_WARN_ON(vmg->vma); 922 VM_WARN_ON(vmg->vma);
923 /* vmi must point at or before the gap 923 /* vmi must point at or before the gap. */
924 VM_WARN_ON(vma_iter_addr(vmg->vmi) > e 924 VM_WARN_ON(vma_iter_addr(vmg->vmi) > end);
925 925
926 vmg->state = VMA_MERGE_NOMERGE; 926 vmg->state = VMA_MERGE_NOMERGE;
927 927
928 /* Special VMAs are unmergeable, also 928 /* Special VMAs are unmergeable, also if no prev/next. */
929 if ((vmg->flags & VM_SPECIAL) || (!pre 929 if ((vmg->flags & VM_SPECIAL) || (!prev && !next))
930 return NULL; 930 return NULL;
931 931
932 can_merge_left = can_vma_merge_left(vm 932 can_merge_left = can_vma_merge_left(vmg);
933 can_merge_right = !just_expand && can_ 933 can_merge_right = !just_expand && can_vma_merge_right(vmg, can_merge_left);
934 934
935 /* If we can merge with the next VMA, 935 /* If we can merge with the next VMA, adjust vmg accordingly. */
936 if (can_merge_right) { 936 if (can_merge_right) {
937 vmg->end = next->vm_end; 937 vmg->end = next->vm_end;
938 vmg->vma = next; 938 vmg->vma = next;
939 vmg->pgoff = next->vm_pgoff - 939 vmg->pgoff = next->vm_pgoff - pglen;
940 } 940 }
941 941
942 /* If we can merge with the previous V 942 /* If we can merge with the previous VMA, adjust vmg accordingly. */
943 if (can_merge_left) { 943 if (can_merge_left) {
944 vmg->start = prev->vm_start; 944 vmg->start = prev->vm_start;
945 vmg->vma = prev; 945 vmg->vma = prev;
946 vmg->pgoff = prev->vm_pgoff; 946 vmg->pgoff = prev->vm_pgoff;
947 947
948 /* 948 /*
949 * If this merge would result 949 * If this merge would result in removal of the next VMA but we
950 * are not permitted to do so, 950 * are not permitted to do so, reduce the operation to merging
951 * prev and vma. 951 * prev and vma.
952 */ 952 */
953 if (can_merge_right && !can_me 953 if (can_merge_right && !can_merge_remove_vma(next))
954 vmg->end = end; 954 vmg->end = end;
955 955
956 /* In expand-only case we are 956 /* In expand-only case we are already positioned at prev. */
957 if (!just_expand) { 957 if (!just_expand) {
958 /* Equivalent to going 958 /* Equivalent to going to the previous range. */
959 vma_prev(vmg->vmi); 959 vma_prev(vmg->vmi);
960 } 960 }
961 } 961 }
962 962
963 /* 963 /*
964 * Now try to expand adjacent VMA(s). 964 * Now try to expand adjacent VMA(s). This takes care of removing the
965 * following VMA if we have VMAs on bo 965 * following VMA if we have VMAs on both sides.
966 */ 966 */
967 if (vmg->vma && !vma_expand(vmg)) { 967 if (vmg->vma && !vma_expand(vmg)) {
968 khugepaged_enter_vma(vmg->vma, 968 khugepaged_enter_vma(vmg->vma, vmg->flags);
969 vmg->state = VMA_MERGE_SUCCESS 969 vmg->state = VMA_MERGE_SUCCESS;
970 return vmg->vma; 970 return vmg->vma;
971 } 971 }
972 972
973 /* If expansion failed, reset state. A 973 /* If expansion failed, reset state. Allows us to retry merge later. */
974 if (!just_expand) { 974 if (!just_expand) {
975 vmg->vma = NULL; 975 vmg->vma = NULL;
976 vmg->start = start; 976 vmg->start = start;
977 vmg->end = end; 977 vmg->end = end;
978 vmg->pgoff = pgoff; 978 vmg->pgoff = pgoff;
979 if (vmg->vma == prev) 979 if (vmg->vma == prev)
980 vma_iter_set(vmg->vmi, 980 vma_iter_set(vmg->vmi, start);
981 } 981 }
982 982
983 return NULL; 983 return NULL;
984 } 984 }
985 985
986 /* 986 /*
987 * vma_expand - Expand an existing VMA 987 * vma_expand - Expand an existing VMA
988 * 988 *
989 * @vmg: Describes a VMA expansion operation. 989 * @vmg: Describes a VMA expansion operation.
990 * 990 *
991 * Expand @vma to vmg->start and vmg->end. Ca 991 * Expand @vma to vmg->start and vmg->end. Can expand off the start and end.
992 * Will expand over vmg->next if it's differen 992 * Will expand over vmg->next if it's different from vmg->vma and vmg->end ==
993 * vmg->next->vm_end. Checking if the vmg->vm 993 * vmg->next->vm_end. Checking if the vmg->vma can expand and merge with
994 * vmg->next needs to be handled by the caller 994 * vmg->next needs to be handled by the caller.
995 * 995 *
996 * Returns: 0 on success. 996 * Returns: 0 on success.
997 * 997 *
998 * ASSUMPTIONS: 998 * ASSUMPTIONS:
999 * - The caller must hold a WRITE lock on vmg- 999 * - The caller must hold a WRITE lock on vmg->vma->mm->mmap_lock.
1000 * - The caller must have set @vmg->vma and @ 1000 * - The caller must have set @vmg->vma and @vmg->next.
1001 */ 1001 */
1002 int vma_expand(struct vma_merge_struct *vmg) 1002 int vma_expand(struct vma_merge_struct *vmg)
1003 { 1003 {
1004 struct vm_area_struct *anon_dup = NUL 1004 struct vm_area_struct *anon_dup = NULL;
1005 bool remove_next = false; 1005 bool remove_next = false;
1006 struct vm_area_struct *vma = vmg->vma 1006 struct vm_area_struct *vma = vmg->vma;
1007 struct vm_area_struct *next = vmg->ne 1007 struct vm_area_struct *next = vmg->next;
1008 1008
1009 mmap_assert_write_locked(vmg->mm); 1009 mmap_assert_write_locked(vmg->mm);
1010 1010
1011 vma_start_write(vma); 1011 vma_start_write(vma);
1012 if (next && (vma != next) && (vmg->en 1012 if (next && (vma != next) && (vmg->end == next->vm_end)) {
1013 int ret; 1013 int ret;
1014 1014
1015 remove_next = true; 1015 remove_next = true;
1016 /* This should already have b 1016 /* This should already have been checked by this point. */
1017 VM_WARN_ON(!can_merge_remove_ 1017 VM_WARN_ON(!can_merge_remove_vma(next));
1018 vma_start_write(next); 1018 vma_start_write(next);
1019 ret = dup_anon_vma(vma, next, 1019 ret = dup_anon_vma(vma, next, &anon_dup);
1020 if (ret) 1020 if (ret)
1021 return ret; 1021 return ret;
1022 } 1022 }
1023 1023
1024 /* Not merging but overwriting any pa 1024 /* Not merging but overwriting any part of next is not handled. */
1025 VM_WARN_ON(next && !remove_next && 1025 VM_WARN_ON(next && !remove_next &&
1026 next != vma && vmg->end > n 1026 next != vma && vmg->end > next->vm_start);
1027 /* Only handles expanding */ 1027 /* Only handles expanding */
1028 VM_WARN_ON(vma->vm_start < vmg->start 1028 VM_WARN_ON(vma->vm_start < vmg->start || vma->vm_end > vmg->end);
1029 1029
1030 if (commit_merge(vmg, NULL, remove_ne 1030 if (commit_merge(vmg, NULL, remove_next ? next : NULL, NULL, 0, true))
1031 goto nomem; 1031 goto nomem;
1032 1032
1033 return 0; 1033 return 0;
1034 1034
1035 nomem: 1035 nomem:
1036 vmg->state = VMA_MERGE_ERROR_NOMEM; 1036 vmg->state = VMA_MERGE_ERROR_NOMEM;
1037 if (anon_dup) 1037 if (anon_dup)
1038 unlink_anon_vmas(anon_dup); 1038 unlink_anon_vmas(anon_dup);
1039 return -ENOMEM; 1039 return -ENOMEM;
1040 } 1040 }
1041 1041
1042 /* 1042 /*
1043 * vma_shrink() - Reduce an existing VMAs mem 1043 * vma_shrink() - Reduce an existing VMAs memory area
1044 * @vmi: The vma iterator 1044 * @vmi: The vma iterator
1045 * @vma: The VMA to modify 1045 * @vma: The VMA to modify
1046 * @start: The new start 1046 * @start: The new start
1047 * @end: The new end 1047 * @end: The new end
1048 * 1048 *
1049 * Returns: 0 on success, -ENOMEM otherwise 1049 * Returns: 0 on success, -ENOMEM otherwise
1050 */ 1050 */
1051 int vma_shrink(struct vma_iterator *vmi, stru 1051 int vma_shrink(struct vma_iterator *vmi, struct vm_area_struct *vma,
1052 unsigned long start, unsigned 1052 unsigned long start, unsigned long end, pgoff_t pgoff)
1053 { 1053 {
1054 struct vma_prepare vp; 1054 struct vma_prepare vp;
1055 1055
1056 WARN_ON((vma->vm_start != start) && ( 1056 WARN_ON((vma->vm_start != start) && (vma->vm_end != end));
1057 1057
1058 if (vma->vm_start < start) 1058 if (vma->vm_start < start)
1059 vma_iter_config(vmi, vma->vm_ 1059 vma_iter_config(vmi, vma->vm_start, start);
1060 else 1060 else
1061 vma_iter_config(vmi, end, vma 1061 vma_iter_config(vmi, end, vma->vm_end);
1062 1062
1063 if (vma_iter_prealloc(vmi, NULL)) 1063 if (vma_iter_prealloc(vmi, NULL))
1064 return -ENOMEM; 1064 return -ENOMEM;
1065 1065
1066 vma_start_write(vma); 1066 vma_start_write(vma);
1067 1067
1068 init_vma_prep(&vp, vma); 1068 init_vma_prep(&vp, vma);
1069 vma_prepare(&vp); 1069 vma_prepare(&vp);
1070 vma_adjust_trans_huge(vma, start, end 1070 vma_adjust_trans_huge(vma, start, end, 0);
1071 1071
1072 vma_iter_clear(vmi); 1072 vma_iter_clear(vmi);
1073 vma_set_range(vma, start, end, pgoff) 1073 vma_set_range(vma, start, end, pgoff);
1074 vma_complete(&vp, vmi, vma->vm_mm); 1074 vma_complete(&vp, vmi, vma->vm_mm);
1075 validate_mm(vma->vm_mm); 1075 validate_mm(vma->vm_mm);
1076 return 0; 1076 return 0;
1077 } 1077 }
1078 1078
1079 static inline void vms_clear_ptes(struct vma_ 1079 static inline void vms_clear_ptes(struct vma_munmap_struct *vms,
1080 struct ma_state *mas_deta 1080 struct ma_state *mas_detach, bool mm_wr_locked)
1081 { 1081 {
1082 struct mmu_gather tlb; 1082 struct mmu_gather tlb;
1083 1083
1084 if (!vms->clear_ptes) /* Nothing to d 1084 if (!vms->clear_ptes) /* Nothing to do */
1085 return; 1085 return;
1086 1086
1087 /* 1087 /*
1088 * We can free page tables without wr 1088 * We can free page tables without write-locking mmap_lock because VMAs
1089 * were isolated before we downgraded 1089 * were isolated before we downgraded mmap_lock.
1090 */ 1090 */
1091 mas_set(mas_detach, 1); 1091 mas_set(mas_detach, 1);
1092 lru_add_drain(); 1092 lru_add_drain();
1093 tlb_gather_mmu(&tlb, vms->vma->vm_mm) 1093 tlb_gather_mmu(&tlb, vms->vma->vm_mm);
1094 update_hiwater_rss(vms->vma->vm_mm); 1094 update_hiwater_rss(vms->vma->vm_mm);
1095 unmap_vmas(&tlb, mas_detach, vms->vma 1095 unmap_vmas(&tlb, mas_detach, vms->vma, vms->start, vms->end,
1096 vms->vma_count, mm_wr_lock 1096 vms->vma_count, mm_wr_locked);
1097 1097
1098 mas_set(mas_detach, 1); 1098 mas_set(mas_detach, 1);
1099 /* start and end may be different if 1099 /* start and end may be different if there is no prev or next vma. */
1100 free_pgtables(&tlb, mas_detach, vms-> 1100 free_pgtables(&tlb, mas_detach, vms->vma, vms->unmap_start,
1101 vms->unmap_end, mm_wr_l 1101 vms->unmap_end, mm_wr_locked);
1102 tlb_finish_mmu(&tlb); 1102 tlb_finish_mmu(&tlb);
1103 vms->clear_ptes = false; 1103 vms->clear_ptes = false;
1104 } 1104 }
1105 1105
1106 void vms_clean_up_area(struct vma_munmap_stru 1106 void vms_clean_up_area(struct vma_munmap_struct *vms,
1107 struct ma_state *mas_detach) 1107 struct ma_state *mas_detach)
1108 { 1108 {
1109 struct vm_area_struct *vma; 1109 struct vm_area_struct *vma;
1110 1110
1111 if (!vms->nr_pages) 1111 if (!vms->nr_pages)
1112 return; 1112 return;
1113 1113
1114 vms_clear_ptes(vms, mas_detach, true) 1114 vms_clear_ptes(vms, mas_detach, true);
1115 mas_set(mas_detach, 0); 1115 mas_set(mas_detach, 0);
1116 mas_for_each(mas_detach, vma, ULONG_M 1116 mas_for_each(mas_detach, vma, ULONG_MAX)
1117 vma_close(vma); 1117 vma_close(vma);
1118 } 1118 }
1119 1119
1120 /* 1120 /*
1121 * vms_complete_munmap_vmas() - Finish the mu 1121 * vms_complete_munmap_vmas() - Finish the munmap() operation
1122 * @vms: The vma munmap struct 1122 * @vms: The vma munmap struct
1123 * @mas_detach: The maple state of the detach 1123 * @mas_detach: The maple state of the detached vmas
1124 * 1124 *
1125 * This updates the mm_struct, unmaps the reg 1125 * This updates the mm_struct, unmaps the region, frees the resources
1126 * used for the munmap() and may downgrade th 1126 * used for the munmap() and may downgrade the lock - if requested. Everything
1127 * needed to be done once the vma maple tree 1127 * needed to be done once the vma maple tree is updated.
1128 */ 1128 */
1129 void vms_complete_munmap_vmas(struct vma_munm 1129 void vms_complete_munmap_vmas(struct vma_munmap_struct *vms,
1130 struct ma_state *mas_detach) 1130 struct ma_state *mas_detach)
1131 { 1131 {
1132 struct vm_area_struct *vma; 1132 struct vm_area_struct *vma;
1133 struct mm_struct *mm; 1133 struct mm_struct *mm;
1134 1134
1135 mm = current->mm; 1135 mm = current->mm;
1136 mm->map_count -= vms->vma_count; 1136 mm->map_count -= vms->vma_count;
1137 mm->locked_vm -= vms->locked_vm; 1137 mm->locked_vm -= vms->locked_vm;
1138 if (vms->unlock) 1138 if (vms->unlock)
1139 mmap_write_downgrade(mm); 1139 mmap_write_downgrade(mm);
1140 1140
1141 if (!vms->nr_pages) 1141 if (!vms->nr_pages)
1142 return; 1142 return;
1143 1143
1144 vms_clear_ptes(vms, mas_detach, !vms- 1144 vms_clear_ptes(vms, mas_detach, !vms->unlock);
1145 /* Update high watermark before we lo 1145 /* Update high watermark before we lower total_vm */
1146 update_hiwater_vm(mm); 1146 update_hiwater_vm(mm);
1147 /* Stat accounting */ 1147 /* Stat accounting */
1148 WRITE_ONCE(mm->total_vm, READ_ONCE(mm 1148 WRITE_ONCE(mm->total_vm, READ_ONCE(mm->total_vm) - vms->nr_pages);
1149 /* Paranoid bookkeeping */ 1149 /* Paranoid bookkeeping */
1150 VM_WARN_ON(vms->exec_vm > mm->exec_vm 1150 VM_WARN_ON(vms->exec_vm > mm->exec_vm);
1151 VM_WARN_ON(vms->stack_vm > mm->stack_ 1151 VM_WARN_ON(vms->stack_vm > mm->stack_vm);
1152 VM_WARN_ON(vms->data_vm > mm->data_vm 1152 VM_WARN_ON(vms->data_vm > mm->data_vm);
1153 mm->exec_vm -= vms->exec_vm; 1153 mm->exec_vm -= vms->exec_vm;
1154 mm->stack_vm -= vms->stack_vm; 1154 mm->stack_vm -= vms->stack_vm;
1155 mm->data_vm -= vms->data_vm; 1155 mm->data_vm -= vms->data_vm;
1156 1156
1157 /* Remove and clean up vmas */ 1157 /* Remove and clean up vmas */
1158 mas_set(mas_detach, 0); 1158 mas_set(mas_detach, 0);
1159 mas_for_each(mas_detach, vma, ULONG_M 1159 mas_for_each(mas_detach, vma, ULONG_MAX)
1160 remove_vma(vma, /* unreachabl 1160 remove_vma(vma, /* unreachable = */ false);
1161 1161
1162 vm_unacct_memory(vms->nr_accounted); 1162 vm_unacct_memory(vms->nr_accounted);
1163 validate_mm(mm); 1163 validate_mm(mm);
1164 if (vms->unlock) 1164 if (vms->unlock)
1165 mmap_read_unlock(mm); 1165 mmap_read_unlock(mm);
1166 1166
1167 __mt_destroy(mas_detach->tree); 1167 __mt_destroy(mas_detach->tree);
1168 } 1168 }
1169 1169
1170 /* 1170 /*
1171 * vms_gather_munmap_vmas() - Put all VMAs wi 1171 * vms_gather_munmap_vmas() - Put all VMAs within a range into a maple tree
1172 * for removal at a later date. Handles spli 1172 * for removal at a later date. Handles splitting first and last if necessary
1173 * and marking the vmas as isolated. 1173 * and marking the vmas as isolated.
1174 * 1174 *
1175 * @vms: The vma munmap struct 1175 * @vms: The vma munmap struct
1176 * @mas_detach: The maple state tracking the 1176 * @mas_detach: The maple state tracking the detached tree
1177 * 1177 *
1178 * Return: 0 on success, error otherwise 1178 * Return: 0 on success, error otherwise
1179 */ 1179 */
1180 int vms_gather_munmap_vmas(struct vma_munmap_ 1180 int vms_gather_munmap_vmas(struct vma_munmap_struct *vms,
1181 struct ma_state *mas_detach) 1181 struct ma_state *mas_detach)
1182 { 1182 {
1183 struct vm_area_struct *next = NULL; 1183 struct vm_area_struct *next = NULL;
1184 int error; 1184 int error;
1185 1185
1186 /* 1186 /*
1187 * If we need to split any vma, do it 1187 * If we need to split any vma, do it now to save pain later.
1188 * Does it split the first one? 1188 * Does it split the first one?
1189 */ 1189 */
1190 if (vms->start > vms->vma->vm_start) 1190 if (vms->start > vms->vma->vm_start) {
1191 1191
1192 /* 1192 /*
1193 * Make sure that map_count o 1193 * Make sure that map_count on return from munmap() will
1194 * not exceed its limit; but 1194 * not exceed its limit; but let map_count go just above
1195 * its limit temporarily, to 1195 * its limit temporarily, to help free resources as expected.
1196 */ 1196 */
1197 if (vms->end < vms->vma->vm_e 1197 if (vms->end < vms->vma->vm_end &&
1198 vms->vma->vm_mm->map_coun 1198 vms->vma->vm_mm->map_count >= sysctl_max_map_count) {
1199 error = -ENOMEM; 1199 error = -ENOMEM;
1200 goto map_count_exceed 1200 goto map_count_exceeded;
1201 } 1201 }
1202 1202
1203 /* Don't bother splitting the 1203 /* Don't bother splitting the VMA if we can't unmap it anyway */
1204 if (!can_modify_vma(vms->vma) 1204 if (!can_modify_vma(vms->vma)) {
1205 error = -EPERM; 1205 error = -EPERM;
1206 goto start_split_fail 1206 goto start_split_failed;
1207 } 1207 }
1208 1208
1209 error = __split_vma(vms->vmi, 1209 error = __split_vma(vms->vmi, vms->vma, vms->start, 1);
1210 if (error) 1210 if (error)
1211 goto start_split_fail 1211 goto start_split_failed;
1212 } 1212 }
1213 vms->prev = vma_prev(vms->vmi); 1213 vms->prev = vma_prev(vms->vmi);
1214 if (vms->prev) 1214 if (vms->prev)
1215 vms->unmap_start = vms->prev- 1215 vms->unmap_start = vms->prev->vm_end;
1216 1216
1217 /* 1217 /*
1218 * Detach a range of VMAs from the mm 1218 * Detach a range of VMAs from the mm. Using next as a temp variable as
1219 * it is always overwritten. 1219 * it is always overwritten.
1220 */ 1220 */
1221 for_each_vma_range(*(vms->vmi), next, 1221 for_each_vma_range(*(vms->vmi), next, vms->end) {
1222 long nrpages; 1222 long nrpages;
1223 1223
1224 if (!can_modify_vma(next)) { 1224 if (!can_modify_vma(next)) {
1225 error = -EPERM; 1225 error = -EPERM;
1226 goto modify_vma_faile 1226 goto modify_vma_failed;
1227 } 1227 }
1228 /* Does it split the end? */ 1228 /* Does it split the end? */
1229 if (next->vm_end > vms->end) 1229 if (next->vm_end > vms->end) {
1230 error = __split_vma(v 1230 error = __split_vma(vms->vmi, next, vms->end, 0);
1231 if (error) 1231 if (error)
1232 goto end_spli 1232 goto end_split_failed;
1233 } 1233 }
1234 vma_start_write(next); 1234 vma_start_write(next);
1235 mas_set(mas_detach, vms->vma_ 1235 mas_set(mas_detach, vms->vma_count++);
1236 error = mas_store_gfp(mas_det 1236 error = mas_store_gfp(mas_detach, next, GFP_KERNEL);
1237 if (error) 1237 if (error)
1238 goto munmap_gather_fa 1238 goto munmap_gather_failed;
1239 1239
1240 vma_mark_detached(next, true) 1240 vma_mark_detached(next, true);
1241 nrpages = vma_pages(next); 1241 nrpages = vma_pages(next);
1242 1242
1243 vms->nr_pages += nrpages; 1243 vms->nr_pages += nrpages;
1244 if (next->vm_flags & VM_LOCKE 1244 if (next->vm_flags & VM_LOCKED)
1245 vms->locked_vm += nrp 1245 vms->locked_vm += nrpages;
1246 1246
1247 if (next->vm_flags & VM_ACCOU 1247 if (next->vm_flags & VM_ACCOUNT)
1248 vms->nr_accounted += 1248 vms->nr_accounted += nrpages;
1249 1249
1250 if (is_exec_mapping(next->vm_ 1250 if (is_exec_mapping(next->vm_flags))
1251 vms->exec_vm += nrpag 1251 vms->exec_vm += nrpages;
1252 else if (is_stack_mapping(nex 1252 else if (is_stack_mapping(next->vm_flags))
1253 vms->stack_vm += nrpa 1253 vms->stack_vm += nrpages;
1254 else if (is_data_mapping(next 1254 else if (is_data_mapping(next->vm_flags))
1255 vms->data_vm += nrpag 1255 vms->data_vm += nrpages;
1256 1256
1257 if (unlikely(vms->uf)) { 1257 if (unlikely(vms->uf)) {
1258 /* 1258 /*
1259 * If userfaultfd_unm 1259 * If userfaultfd_unmap_prep returns an error the vmas
1260 * will remain split, 1260 * will remain split, but userland will get a
1261 * highly unexpected 1261 * highly unexpected error anyway. This is no
1262 * different than the 1262 * different than the case where the first of the two
1263 * __split_vma fails, 1263 * __split_vma fails, but we don't undo the first
1264 * split, despite we 1264 * split, despite we could. This is unlikely enough
1265 * failure that it's 1265 * failure that it's not worth optimizing it for.
1266 */ 1266 */
1267 error = userfaultfd_u 1267 error = userfaultfd_unmap_prep(next, vms->start,
1268 1268 vms->end, vms->uf);
1269 if (error) 1269 if (error)
1270 goto userfaul 1270 goto userfaultfd_error;
1271 } 1271 }
1272 #ifdef CONFIG_DEBUG_VM_MAPLE_TREE 1272 #ifdef CONFIG_DEBUG_VM_MAPLE_TREE
1273 BUG_ON(next->vm_start < vms-> 1273 BUG_ON(next->vm_start < vms->start);
1274 BUG_ON(next->vm_start > vms-> 1274 BUG_ON(next->vm_start > vms->end);
1275 #endif 1275 #endif
1276 } 1276 }
1277 1277
1278 vms->next = vma_next(vms->vmi); 1278 vms->next = vma_next(vms->vmi);
1279 if (vms->next) 1279 if (vms->next)
1280 vms->unmap_end = vms->next->v 1280 vms->unmap_end = vms->next->vm_start;
1281 1281
1282 #if defined(CONFIG_DEBUG_VM_MAPLE_TREE) 1282 #if defined(CONFIG_DEBUG_VM_MAPLE_TREE)
1283 /* Make sure no VMAs are about to be 1283 /* Make sure no VMAs are about to be lost. */
1284 { 1284 {
1285 MA_STATE(test, mas_detach->tr 1285 MA_STATE(test, mas_detach->tree, 0, 0);
1286 struct vm_area_struct *vma_ma 1286 struct vm_area_struct *vma_mas, *vma_test;
1287 int test_count = 0; 1287 int test_count = 0;
1288 1288
1289 vma_iter_set(vms->vmi, vms->s 1289 vma_iter_set(vms->vmi, vms->start);
1290 rcu_read_lock(); 1290 rcu_read_lock();
1291 vma_test = mas_find(&test, vm 1291 vma_test = mas_find(&test, vms->vma_count - 1);
1292 for_each_vma_range(*(vms->vmi 1292 for_each_vma_range(*(vms->vmi), vma_mas, vms->end) {
1293 BUG_ON(vma_mas != vma 1293 BUG_ON(vma_mas != vma_test);
1294 test_count++; 1294 test_count++;
1295 vma_test = mas_next(& 1295 vma_test = mas_next(&test, vms->vma_count - 1);
1296 } 1296 }
1297 rcu_read_unlock(); 1297 rcu_read_unlock();
1298 BUG_ON(vms->vma_count != test 1298 BUG_ON(vms->vma_count != test_count);
1299 } 1299 }
1300 #endif 1300 #endif
1301 1301
1302 while (vma_iter_addr(vms->vmi) > vms- 1302 while (vma_iter_addr(vms->vmi) > vms->start)
1303 vma_iter_prev_range(vms->vmi) 1303 vma_iter_prev_range(vms->vmi);
1304 1304
1305 vms->clear_ptes = true; 1305 vms->clear_ptes = true;
1306 return 0; 1306 return 0;
1307 1307
1308 userfaultfd_error: 1308 userfaultfd_error:
1309 munmap_gather_failed: 1309 munmap_gather_failed:
1310 end_split_failed: 1310 end_split_failed:
1311 modify_vma_failed: 1311 modify_vma_failed:
1312 reattach_vmas(mas_detach); 1312 reattach_vmas(mas_detach);
1313 start_split_failed: 1313 start_split_failed:
1314 map_count_exceeded: 1314 map_count_exceeded:
1315 return error; 1315 return error;
1316 } 1316 }
1317 1317
1318 /* 1318 /*
1319 * do_vmi_align_munmap() - munmap the aligned 1319 * do_vmi_align_munmap() - munmap the aligned region from @start to @end.
1320 * @vmi: The vma iterator 1320 * @vmi: The vma iterator
1321 * @vma: The starting vm_area_struct 1321 * @vma: The starting vm_area_struct
1322 * @mm: The mm_struct 1322 * @mm: The mm_struct
1323 * @start: The aligned start address to munma 1323 * @start: The aligned start address to munmap.
1324 * @end: The aligned end address to munmap. 1324 * @end: The aligned end address to munmap.
1325 * @uf: The userfaultfd list_head 1325 * @uf: The userfaultfd list_head
1326 * @unlock: Set to true to drop the mmap_lock 1326 * @unlock: Set to true to drop the mmap_lock. unlocking only happens on
1327 * success. 1327 * success.
1328 * 1328 *
1329 * Return: 0 on success and drops the lock if 1329 * Return: 0 on success and drops the lock if so directed, error and leaves the
1330 * lock held otherwise. 1330 * lock held otherwise.
1331 */ 1331 */
1332 int do_vmi_align_munmap(struct vma_iterator * 1332 int do_vmi_align_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma,
1333 struct mm_struct *mm, unsigne 1333 struct mm_struct *mm, unsigned long start, unsigned long end,
1334 struct list_head *uf, bool un 1334 struct list_head *uf, bool unlock)
1335 { 1335 {
1336 struct maple_tree mt_detach; 1336 struct maple_tree mt_detach;
1337 MA_STATE(mas_detach, &mt_detach, 0, 0 1337 MA_STATE(mas_detach, &mt_detach, 0, 0);
1338 mt_init_flags(&mt_detach, vmi->mas.tr 1338 mt_init_flags(&mt_detach, vmi->mas.tree->ma_flags & MT_FLAGS_LOCK_MASK);
1339 mt_on_stack(mt_detach); 1339 mt_on_stack(mt_detach);
1340 struct vma_munmap_struct vms; 1340 struct vma_munmap_struct vms;
1341 int error; 1341 int error;
1342 1342
1343 init_vma_munmap(&vms, vmi, vma, start 1343 init_vma_munmap(&vms, vmi, vma, start, end, uf, unlock);
1344 error = vms_gather_munmap_vmas(&vms, 1344 error = vms_gather_munmap_vmas(&vms, &mas_detach);
1345 if (error) 1345 if (error)
1346 goto gather_failed; 1346 goto gather_failed;
1347 1347
1348 error = vma_iter_clear_gfp(vmi, start 1348 error = vma_iter_clear_gfp(vmi, start, end, GFP_KERNEL);
1349 if (error) 1349 if (error)
1350 goto clear_tree_failed; 1350 goto clear_tree_failed;
1351 1351
1352 /* Point of no return */ 1352 /* Point of no return */
1353 vms_complete_munmap_vmas(&vms, &mas_d 1353 vms_complete_munmap_vmas(&vms, &mas_detach);
1354 return 0; 1354 return 0;
1355 1355
1356 clear_tree_failed: 1356 clear_tree_failed:
1357 reattach_vmas(&mas_detach); 1357 reattach_vmas(&mas_detach);
1358 gather_failed: 1358 gather_failed:
1359 validate_mm(mm); 1359 validate_mm(mm);
1360 return error; 1360 return error;
1361 } 1361 }
1362 1362
1363 /* 1363 /*
1364 * do_vmi_munmap() - munmap a given range. 1364 * do_vmi_munmap() - munmap a given range.
1365 * @vmi: The vma iterator 1365 * @vmi: The vma iterator
1366 * @mm: The mm_struct 1366 * @mm: The mm_struct
1367 * @start: The start address to munmap 1367 * @start: The start address to munmap
1368 * @len: The length of the range to munmap 1368 * @len: The length of the range to munmap
1369 * @uf: The userfaultfd list_head 1369 * @uf: The userfaultfd list_head
1370 * @unlock: set to true if the user wants to 1370 * @unlock: set to true if the user wants to drop the mmap_lock on success
1371 * 1371 *
1372 * This function takes a @mas that is either 1372 * This function takes a @mas that is either pointing to the previous VMA or set
1373 * to MA_START and sets it up to remove the m 1373 * to MA_START and sets it up to remove the mapping(s). The @len will be
1374 * aligned. 1374 * aligned.
1375 * 1375 *
1376 * Return: 0 on success and drops the lock if 1376 * Return: 0 on success and drops the lock if so directed, error and leaves the
1377 * lock held otherwise. 1377 * lock held otherwise.
1378 */ 1378 */
1379 int do_vmi_munmap(struct vma_iterator *vmi, s 1379 int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm,
1380 unsigned long start, size_t 1380 unsigned long start, size_t len, struct list_head *uf,
1381 bool unlock) 1381 bool unlock)
1382 { 1382 {
1383 unsigned long end; 1383 unsigned long end;
1384 struct vm_area_struct *vma; 1384 struct vm_area_struct *vma;
1385 1385
1386 if ((offset_in_page(start)) || start 1386 if ((offset_in_page(start)) || start > TASK_SIZE || len > TASK_SIZE-start)
1387 return -EINVAL; 1387 return -EINVAL;
1388 1388
1389 end = start + PAGE_ALIGN(len); 1389 end = start + PAGE_ALIGN(len);
1390 if (end == start) 1390 if (end == start)
1391 return -EINVAL; 1391 return -EINVAL;
1392 1392
1393 /* Find the first overlapping VMA */ 1393 /* Find the first overlapping VMA */
1394 vma = vma_find(vmi, end); 1394 vma = vma_find(vmi, end);
1395 if (!vma) { 1395 if (!vma) {
1396 if (unlock) 1396 if (unlock)
1397 mmap_write_unlock(mm) 1397 mmap_write_unlock(mm);
1398 return 0; 1398 return 0;
1399 } 1399 }
1400 1400
1401 return do_vmi_align_munmap(vmi, vma, 1401 return do_vmi_align_munmap(vmi, vma, mm, start, end, uf, unlock);
1402 } 1402 }
1403 1403
1404 /* 1404 /*
1405 * We are about to modify one or multiple of 1405 * We are about to modify one or multiple of a VMA's flags, policy, userfaultfd
1406 * context and anonymous VMA name within the 1406 * context and anonymous VMA name within the range [start, end).
1407 * 1407 *
1408 * As a result, we might be able to merge the 1408 * As a result, we might be able to merge the newly modified VMA range with an
1409 * adjacent VMA with identical properties. 1409 * adjacent VMA with identical properties.
1410 * 1410 *
1411 * If no merge is possible and the range does 1411 * If no merge is possible and the range does not span the entirety of the VMA,
1412 * we then need to split the VMA to accommoda 1412 * we then need to split the VMA to accommodate the change.
1413 * 1413 *
1414 * The function returns either the merged VMA 1414 * The function returns either the merged VMA, the original VMA if a split was
1415 * required instead, or an error if the split 1415 * required instead, or an error if the split failed.
1416 */ 1416 */
1417 static struct vm_area_struct *vma_modify(stru 1417 static struct vm_area_struct *vma_modify(struct vma_merge_struct *vmg)
1418 { 1418 {
1419 struct vm_area_struct *vma = vmg->vma 1419 struct vm_area_struct *vma = vmg->vma;
1420 struct vm_area_struct *merged; 1420 struct vm_area_struct *merged;
1421 1421
1422 /* First, try to merge. */ 1422 /* First, try to merge. */
1423 merged = vma_merge_existing_range(vmg 1423 merged = vma_merge_existing_range(vmg);
1424 if (merged) 1424 if (merged)
1425 return merged; 1425 return merged;
1426 1426
1427 /* Split any preceding portion of the 1427 /* Split any preceding portion of the VMA. */
1428 if (vma->vm_start < vmg->start) { 1428 if (vma->vm_start < vmg->start) {
1429 int err = split_vma(vmg->vmi, 1429 int err = split_vma(vmg->vmi, vma, vmg->start, 1);
1430 1430
1431 if (err) 1431 if (err)
1432 return ERR_PTR(err); 1432 return ERR_PTR(err);
1433 } 1433 }
1434 1434
1435 /* Split any trailing portion of the 1435 /* Split any trailing portion of the VMA. */
1436 if (vma->vm_end > vmg->end) { 1436 if (vma->vm_end > vmg->end) {
1437 int err = split_vma(vmg->vmi, 1437 int err = split_vma(vmg->vmi, vma, vmg->end, 0);
1438 1438
1439 if (err) 1439 if (err)
1440 return ERR_PTR(err); 1440 return ERR_PTR(err);
1441 } 1441 }
1442 1442
1443 return vma; 1443 return vma;
1444 } 1444 }
1445 1445
1446 struct vm_area_struct *vma_modify_flags( 1446 struct vm_area_struct *vma_modify_flags(
1447 struct vma_iterator *vmi, struct vm_a 1447 struct vma_iterator *vmi, struct vm_area_struct *prev,
1448 struct vm_area_struct *vma, unsigned 1448 struct vm_area_struct *vma, unsigned long start, unsigned long end,
1449 unsigned long new_flags) 1449 unsigned long new_flags)
1450 { 1450 {
1451 VMG_VMA_STATE(vmg, vmi, prev, vma, st 1451 VMG_VMA_STATE(vmg, vmi, prev, vma, start, end);
1452 1452
1453 vmg.flags = new_flags; 1453 vmg.flags = new_flags;
1454 1454
1455 return vma_modify(&vmg); 1455 return vma_modify(&vmg);
1456 } 1456 }
1457 1457
1458 struct vm_area_struct 1458 struct vm_area_struct
1459 *vma_modify_flags_name(struct vma_iterator *v 1459 *vma_modify_flags_name(struct vma_iterator *vmi,
1460 struct vm_area_struct 1460 struct vm_area_struct *prev,
1461 struct vm_area_struct 1461 struct vm_area_struct *vma,
1462 unsigned long start, 1462 unsigned long start,
1463 unsigned long end, 1463 unsigned long end,
1464 unsigned long new_flag 1464 unsigned long new_flags,
1465 struct anon_vma_name * 1465 struct anon_vma_name *new_name)
1466 { 1466 {
1467 VMG_VMA_STATE(vmg, vmi, prev, vma, st 1467 VMG_VMA_STATE(vmg, vmi, prev, vma, start, end);
1468 1468
1469 vmg.flags = new_flags; 1469 vmg.flags = new_flags;
1470 vmg.anon_name = new_name; 1470 vmg.anon_name = new_name;
1471 1471
1472 return vma_modify(&vmg); 1472 return vma_modify(&vmg);
1473 } 1473 }
1474 1474
1475 struct vm_area_struct 1475 struct vm_area_struct
1476 *vma_modify_policy(struct vma_iterator *vmi, 1476 *vma_modify_policy(struct vma_iterator *vmi,
1477 struct vm_area_struct *pre 1477 struct vm_area_struct *prev,
1478 struct vm_area_struct *vma 1478 struct vm_area_struct *vma,
1479 unsigned long start, unsig 1479 unsigned long start, unsigned long end,
1480 struct mempolicy *new_pol) 1480 struct mempolicy *new_pol)
1481 { 1481 {
1482 VMG_VMA_STATE(vmg, vmi, prev, vma, st 1482 VMG_VMA_STATE(vmg, vmi, prev, vma, start, end);
1483 1483
1484 vmg.policy = new_pol; 1484 vmg.policy = new_pol;
1485 1485
1486 return vma_modify(&vmg); 1486 return vma_modify(&vmg);
1487 } 1487 }
1488 1488
1489 struct vm_area_struct 1489 struct vm_area_struct
1490 *vma_modify_flags_uffd(struct vma_iterator *v 1490 *vma_modify_flags_uffd(struct vma_iterator *vmi,
1491 struct vm_area_struct 1491 struct vm_area_struct *prev,
1492 struct vm_area_struct 1492 struct vm_area_struct *vma,
1493 unsigned long start, u 1493 unsigned long start, unsigned long end,
1494 unsigned long new_flag 1494 unsigned long new_flags,
1495 struct vm_userfaultfd_ 1495 struct vm_userfaultfd_ctx new_ctx)
1496 { 1496 {
1497 VMG_VMA_STATE(vmg, vmi, prev, vma, st 1497 VMG_VMA_STATE(vmg, vmi, prev, vma, start, end);
1498 1498
1499 vmg.flags = new_flags; 1499 vmg.flags = new_flags;
1500 vmg.uffd_ctx = new_ctx; 1500 vmg.uffd_ctx = new_ctx;
1501 1501
1502 return vma_modify(&vmg); 1502 return vma_modify(&vmg);
1503 } 1503 }
1504 1504
1505 /* 1505 /*
1506 * Expand vma by delta bytes, potentially mer 1506 * Expand vma by delta bytes, potentially merging with an immediately adjacent
1507 * VMA with identical properties. 1507 * VMA with identical properties.
1508 */ 1508 */
1509 struct vm_area_struct *vma_merge_extend(struc 1509 struct vm_area_struct *vma_merge_extend(struct vma_iterator *vmi,
1510 struc 1510 struct vm_area_struct *vma,
1511 unsig 1511 unsigned long delta)
1512 { 1512 {
1513 VMG_VMA_STATE(vmg, vmi, vma, vma, vma 1513 VMG_VMA_STATE(vmg, vmi, vma, vma, vma->vm_end, vma->vm_end + delta);
1514 1514
1515 vmg.next = vma_iter_next_rewind(vmi, 1515 vmg.next = vma_iter_next_rewind(vmi, NULL);
1516 vmg.vma = NULL; /* We use the VMA to 1516 vmg.vma = NULL; /* We use the VMA to populate VMG fields only. */
1517 1517
1518 return vma_merge_new_range(&vmg); 1518 return vma_merge_new_range(&vmg);
1519 } 1519 }
1520 1520
1521 void unlink_file_vma_batch_init(struct unlink 1521 void unlink_file_vma_batch_init(struct unlink_vma_file_batch *vb)
1522 { 1522 {
1523 vb->count = 0; 1523 vb->count = 0;
1524 } 1524 }
1525 1525
1526 static void unlink_file_vma_batch_process(str 1526 static void unlink_file_vma_batch_process(struct unlink_vma_file_batch *vb)
1527 { 1527 {
1528 struct address_space *mapping; 1528 struct address_space *mapping;
1529 int i; 1529 int i;
1530 1530
1531 mapping = vb->vmas[0]->vm_file->f_map 1531 mapping = vb->vmas[0]->vm_file->f_mapping;
1532 i_mmap_lock_write(mapping); 1532 i_mmap_lock_write(mapping);
1533 for (i = 0; i < vb->count; i++) { 1533 for (i = 0; i < vb->count; i++) {
1534 VM_WARN_ON_ONCE(vb->vmas[i]-> 1534 VM_WARN_ON_ONCE(vb->vmas[i]->vm_file->f_mapping != mapping);
1535 __remove_shared_vm_struct(vb- 1535 __remove_shared_vm_struct(vb->vmas[i], mapping);
1536 } 1536 }
1537 i_mmap_unlock_write(mapping); 1537 i_mmap_unlock_write(mapping);
1538 1538
1539 unlink_file_vma_batch_init(vb); 1539 unlink_file_vma_batch_init(vb);
1540 } 1540 }
1541 1541
1542 void unlink_file_vma_batch_add(struct unlink_ 1542 void unlink_file_vma_batch_add(struct unlink_vma_file_batch *vb,
1543 struct vm_area 1543 struct vm_area_struct *vma)
1544 { 1544 {
1545 if (vma->vm_file == NULL) 1545 if (vma->vm_file == NULL)
1546 return; 1546 return;
1547 1547
1548 if ((vb->count > 0 && vb->vmas[0]->vm 1548 if ((vb->count > 0 && vb->vmas[0]->vm_file != vma->vm_file) ||
1549 vb->count == ARRAY_SIZE(vb->vmas) 1549 vb->count == ARRAY_SIZE(vb->vmas))
1550 unlink_file_vma_batch_process 1550 unlink_file_vma_batch_process(vb);
1551 1551
1552 vb->vmas[vb->count] = vma; 1552 vb->vmas[vb->count] = vma;
1553 vb->count++; 1553 vb->count++;
1554 } 1554 }
1555 1555
1556 void unlink_file_vma_batch_final(struct unlin 1556 void unlink_file_vma_batch_final(struct unlink_vma_file_batch *vb)
1557 { 1557 {
1558 if (vb->count > 0) 1558 if (vb->count > 0)
1559 unlink_file_vma_batch_process 1559 unlink_file_vma_batch_process(vb);
1560 } 1560 }
1561 1561
1562 /* 1562 /*
1563 * Unlink a file-based vm structure from its 1563 * Unlink a file-based vm structure from its interval tree, to hide
1564 * vma from rmap and vmtruncate before freein 1564 * vma from rmap and vmtruncate before freeing its page tables.
1565 */ 1565 */
1566 void unlink_file_vma(struct vm_area_struct *v 1566 void unlink_file_vma(struct vm_area_struct *vma)
1567 { 1567 {
1568 struct file *file = vma->vm_file; 1568 struct file *file = vma->vm_file;
1569 1569
1570 if (file) { 1570 if (file) {
1571 struct address_space *mapping 1571 struct address_space *mapping = file->f_mapping;
1572 1572
1573 i_mmap_lock_write(mapping); 1573 i_mmap_lock_write(mapping);
1574 __remove_shared_vm_struct(vma 1574 __remove_shared_vm_struct(vma, mapping);
1575 i_mmap_unlock_write(mapping); 1575 i_mmap_unlock_write(mapping);
1576 } 1576 }
1577 } 1577 }
1578 1578
1579 void vma_link_file(struct vm_area_struct *vma 1579 void vma_link_file(struct vm_area_struct *vma)
1580 { 1580 {
1581 struct file *file = vma->vm_file; 1581 struct file *file = vma->vm_file;
1582 struct address_space *mapping; 1582 struct address_space *mapping;
1583 1583
1584 if (file) { 1584 if (file) {
1585 mapping = file->f_mapping; 1585 mapping = file->f_mapping;
1586 i_mmap_lock_write(mapping); 1586 i_mmap_lock_write(mapping);
1587 __vma_link_file(vma, mapping) 1587 __vma_link_file(vma, mapping);
1588 i_mmap_unlock_write(mapping); 1588 i_mmap_unlock_write(mapping);
1589 } 1589 }
1590 } 1590 }
1591 1591
1592 int vma_link(struct mm_struct *mm, struct vm_ 1592 int vma_link(struct mm_struct *mm, struct vm_area_struct *vma)
1593 { 1593 {
1594 VMA_ITERATOR(vmi, mm, 0); 1594 VMA_ITERATOR(vmi, mm, 0);
1595 1595
1596 vma_iter_config(&vmi, vma->vm_start, 1596 vma_iter_config(&vmi, vma->vm_start, vma->vm_end);
1597 if (vma_iter_prealloc(&vmi, vma)) 1597 if (vma_iter_prealloc(&vmi, vma))
1598 return -ENOMEM; 1598 return -ENOMEM;
1599 1599
1600 vma_start_write(vma); 1600 vma_start_write(vma);
1601 vma_iter_store(&vmi, vma); 1601 vma_iter_store(&vmi, vma);
1602 vma_link_file(vma); 1602 vma_link_file(vma);
1603 mm->map_count++; 1603 mm->map_count++;
1604 validate_mm(mm); 1604 validate_mm(mm);
1605 return 0; 1605 return 0;
1606 } 1606 }
1607 1607
1608 /* 1608 /*
1609 * Copy the vma structure to a new location i 1609 * Copy the vma structure to a new location in the same mm,
1610 * prior to moving page table entries, to eff 1610 * prior to moving page table entries, to effect an mremap move.
1611 */ 1611 */
1612 struct vm_area_struct *copy_vma(struct vm_are 1612 struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
1613 unsigned long addr, unsigned long len 1613 unsigned long addr, unsigned long len, pgoff_t pgoff,
1614 bool *need_rmap_locks) 1614 bool *need_rmap_locks)
1615 { 1615 {
1616 struct vm_area_struct *vma = *vmap; 1616 struct vm_area_struct *vma = *vmap;
1617 unsigned long vma_start = vma->vm_sta 1617 unsigned long vma_start = vma->vm_start;
1618 struct mm_struct *mm = vma->vm_mm; 1618 struct mm_struct *mm = vma->vm_mm;
1619 struct vm_area_struct *new_vma; 1619 struct vm_area_struct *new_vma;
1620 bool faulted_in_anon_vma = true; 1620 bool faulted_in_anon_vma = true;
1621 VMA_ITERATOR(vmi, mm, addr); 1621 VMA_ITERATOR(vmi, mm, addr);
1622 VMG_VMA_STATE(vmg, &vmi, NULL, vma, a 1622 VMG_VMA_STATE(vmg, &vmi, NULL, vma, addr, addr + len);
1623 1623
1624 /* 1624 /*
1625 * If anonymous vma has not yet been 1625 * If anonymous vma has not yet been faulted, update new pgoff
1626 * to match new location, to increase 1626 * to match new location, to increase its chance of merging.
1627 */ 1627 */
1628 if (unlikely(vma_is_anonymous(vma) && 1628 if (unlikely(vma_is_anonymous(vma) && !vma->anon_vma)) {
1629 pgoff = addr >> PAGE_SHIFT; 1629 pgoff = addr >> PAGE_SHIFT;
1630 faulted_in_anon_vma = false; 1630 faulted_in_anon_vma = false;
1631 } 1631 }
1632 1632
1633 new_vma = find_vma_prev(mm, addr, &vm 1633 new_vma = find_vma_prev(mm, addr, &vmg.prev);
1634 if (new_vma && new_vma->vm_start < ad 1634 if (new_vma && new_vma->vm_start < addr + len)
1635 return NULL; /* should nev 1635 return NULL; /* should never get here */
1636 1636
1637 vmg.vma = NULL; /* New VMA range. */ 1637 vmg.vma = NULL; /* New VMA range. */
1638 vmg.pgoff = pgoff; 1638 vmg.pgoff = pgoff;
1639 vmg.next = vma_iter_next_rewind(&vmi, 1639 vmg.next = vma_iter_next_rewind(&vmi, NULL);
1640 new_vma = vma_merge_new_range(&vmg); 1640 new_vma = vma_merge_new_range(&vmg);
1641 1641
1642 if (new_vma) { 1642 if (new_vma) {
1643 /* 1643 /*
1644 * Source vma may have been m 1644 * Source vma may have been merged into new_vma
1645 */ 1645 */
1646 if (unlikely(vma_start >= new 1646 if (unlikely(vma_start >= new_vma->vm_start &&
1647 vma_start < new_ 1647 vma_start < new_vma->vm_end)) {
1648 /* 1648 /*
1649 * The only way we ca 1649 * The only way we can get a vma_merge with
1650 * self during an mre 1650 * self during an mremap is if the vma hasn't
1651 * been faulted in ye 1651 * been faulted in yet and we were allowed to
1652 * reset the dst vma- 1652 * reset the dst vma->vm_pgoff to the
1653 * destination addres 1653 * destination address of the mremap to allow
1654 * the merge to happe 1654 * the merge to happen. mremap must change the
1655 * vm_pgoff linearity 1655 * vm_pgoff linearity between src and dst vmas
1656 * (in turn preventin 1656 * (in turn preventing a vma_merge) to be
1657 * safe. It is only s 1657 * safe. It is only safe to keep the vm_pgoff
1658 * linear if there ar 1658 * linear if there are no pages mapped yet.
1659 */ 1659 */
1660 VM_BUG_ON_VMA(faulted 1660 VM_BUG_ON_VMA(faulted_in_anon_vma, new_vma);
1661 *vmap = vma = new_vma 1661 *vmap = vma = new_vma;
1662 } 1662 }
1663 *need_rmap_locks = (new_vma-> 1663 *need_rmap_locks = (new_vma->vm_pgoff <= vma->vm_pgoff);
1664 } else { 1664 } else {
1665 new_vma = vm_area_dup(vma); 1665 new_vma = vm_area_dup(vma);
1666 if (!new_vma) 1666 if (!new_vma)
1667 goto out; 1667 goto out;
1668 vma_set_range(new_vma, addr, 1668 vma_set_range(new_vma, addr, addr + len, pgoff);
1669 if (vma_dup_policy(vma, new_v 1669 if (vma_dup_policy(vma, new_vma))
1670 goto out_free_vma; 1670 goto out_free_vma;
1671 if (anon_vma_clone(new_vma, v 1671 if (anon_vma_clone(new_vma, vma))
1672 goto out_free_mempol; 1672 goto out_free_mempol;
1673 if (new_vma->vm_file) 1673 if (new_vma->vm_file)
1674 get_file(new_vma->vm_ 1674 get_file(new_vma->vm_file);
1675 if (new_vma->vm_ops && new_vm 1675 if (new_vma->vm_ops && new_vma->vm_ops->open)
1676 new_vma->vm_ops->open 1676 new_vma->vm_ops->open(new_vma);
1677 if (vma_link(mm, new_vma)) 1677 if (vma_link(mm, new_vma))
1678 goto out_vma_link; 1678 goto out_vma_link;
1679 *need_rmap_locks = false; 1679 *need_rmap_locks = false;
1680 } 1680 }
1681 return new_vma; 1681 return new_vma;
1682 1682
1683 out_vma_link: 1683 out_vma_link:
1684 vma_close(new_vma); 1684 vma_close(new_vma);
1685 1685
1686 if (new_vma->vm_file) 1686 if (new_vma->vm_file)
1687 fput(new_vma->vm_file); 1687 fput(new_vma->vm_file);
1688 1688
1689 unlink_anon_vmas(new_vma); 1689 unlink_anon_vmas(new_vma);
1690 out_free_mempol: 1690 out_free_mempol:
1691 mpol_put(vma_policy(new_vma)); 1691 mpol_put(vma_policy(new_vma));
1692 out_free_vma: 1692 out_free_vma:
1693 vm_area_free(new_vma); 1693 vm_area_free(new_vma);
1694 out: 1694 out:
1695 return NULL; 1695 return NULL;
1696 } 1696 }
1697 1697
1698 /* 1698 /*
1699 * Rough compatibility check to quickly see i 1699 * Rough compatibility check to quickly see if it's even worth looking
1700 * at sharing an anon_vma. 1700 * at sharing an anon_vma.
1701 * 1701 *
1702 * They need to have the same vm_file, and th 1702 * They need to have the same vm_file, and the flags can only differ
1703 * in things that mprotect may change. 1703 * in things that mprotect may change.
1704 * 1704 *
1705 * NOTE! The fact that we share an anon_vma d 1705 * NOTE! The fact that we share an anon_vma doesn't _have_ to mean that
1706 * we can merge the two vma's. For example, w 1706 * we can merge the two vma's. For example, we refuse to merge a vma if
1707 * there is a vm_ops->close() function, becau 1707 * there is a vm_ops->close() function, because that indicates that the
1708 * driver is doing some kind of reference cou 1708 * driver is doing some kind of reference counting. But that doesn't
1709 * really matter for the anon_vma sharing cas 1709 * really matter for the anon_vma sharing case.
1710 */ 1710 */
1711 static int anon_vma_compatible(struct vm_area 1711 static int anon_vma_compatible(struct vm_area_struct *a, struct vm_area_struct *b)
1712 { 1712 {
1713 return a->vm_end == b->vm_start && 1713 return a->vm_end == b->vm_start &&
1714 mpol_equal(vma_policy(a), vma 1714 mpol_equal(vma_policy(a), vma_policy(b)) &&
1715 a->vm_file == b->vm_file && 1715 a->vm_file == b->vm_file &&
1716 !((a->vm_flags ^ b->vm_flags) 1716 !((a->vm_flags ^ b->vm_flags) & ~(VM_ACCESS_FLAGS | VM_SOFTDIRTY)) &&
1717 b->vm_pgoff == a->vm_pgoff + 1717 b->vm_pgoff == a->vm_pgoff + ((b->vm_start - a->vm_start) >> PAGE_SHIFT);
1718 } 1718 }
1719 1719
1720 /* 1720 /*
1721 * Do some basic sanity checking to see if we 1721 * Do some basic sanity checking to see if we can re-use the anon_vma
1722 * from 'old'. The 'a'/'b' vma's are in VM or 1722 * from 'old'. The 'a'/'b' vma's are in VM order - one of them will be
1723 * the same as 'old', the other will be the n 1723 * the same as 'old', the other will be the new one that is trying
1724 * to share the anon_vma. 1724 * to share the anon_vma.
1725 * 1725 *
1726 * NOTE! This runs with mmap_lock held for re 1726 * NOTE! This runs with mmap_lock held for reading, so it is possible that
1727 * the anon_vma of 'old' is concurrently in t 1727 * the anon_vma of 'old' is concurrently in the process of being set up
1728 * by another page fault trying to merge _tha 1728 * by another page fault trying to merge _that_. But that's ok: if it
1729 * is being set up, that automatically means 1729 * is being set up, that automatically means that it will be a singleton
1730 * acceptable for merging, so we can do all o 1730 * acceptable for merging, so we can do all of this optimistically. But
1731 * we do that READ_ONCE() to make sure that w 1731 * we do that READ_ONCE() to make sure that we never re-load the pointer.
1732 * 1732 *
1733 * IOW: that the "list_is_singular()" test on 1733 * IOW: that the "list_is_singular()" test on the anon_vma_chain only
1734 * matters for the 'stable anon_vma' case (ie 1734 * matters for the 'stable anon_vma' case (ie the thing we want to avoid
1735 * is to return an anon_vma that is "complex" 1735 * is to return an anon_vma that is "complex" due to having gone through
1736 * a fork). 1736 * a fork).
1737 * 1737 *
1738 * We also make sure that the two vma's are c 1738 * We also make sure that the two vma's are compatible (adjacent,
1739 * and with the same memory policies). That's 1739 * and with the same memory policies). That's all stable, even with just
1740 * a read lock on the mmap_lock. 1740 * a read lock on the mmap_lock.
1741 */ 1741 */
1742 static struct anon_vma *reusable_anon_vma(str 1742 static struct anon_vma *reusable_anon_vma(struct vm_area_struct *old,
1743 str 1743 struct vm_area_struct *a,
1744 str 1744 struct vm_area_struct *b)
1745 { 1745 {
1746 if (anon_vma_compatible(a, b)) { 1746 if (anon_vma_compatible(a, b)) {
1747 struct anon_vma *anon_vma = R 1747 struct anon_vma *anon_vma = READ_ONCE(old->anon_vma);
1748 1748
1749 if (anon_vma && list_is_singu 1749 if (anon_vma && list_is_singular(&old->anon_vma_chain))
1750 return anon_vma; 1750 return anon_vma;
1751 } 1751 }
1752 return NULL; 1752 return NULL;
1753 } 1753 }
1754 1754
1755 /* 1755 /*
1756 * find_mergeable_anon_vma is used by anon_vm 1756 * find_mergeable_anon_vma is used by anon_vma_prepare, to check
1757 * neighbouring vmas for a suitable anon_vma, 1757 * neighbouring vmas for a suitable anon_vma, before it goes off
1758 * to allocate a new anon_vma. It checks bec 1758 * to allocate a new anon_vma. It checks because a repetitive
1759 * sequence of mprotects and faults may other 1759 * sequence of mprotects and faults may otherwise lead to distinct
1760 * anon_vmas being allocated, preventing vma 1760 * anon_vmas being allocated, preventing vma merge in subsequent
1761 * mprotect. 1761 * mprotect.
1762 */ 1762 */
1763 struct anon_vma *find_mergeable_anon_vma(stru 1763 struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma)
1764 { 1764 {
1765 struct anon_vma *anon_vma = NULL; 1765 struct anon_vma *anon_vma = NULL;
1766 struct vm_area_struct *prev, *next; 1766 struct vm_area_struct *prev, *next;
1767 VMA_ITERATOR(vmi, vma->vm_mm, vma->vm 1767 VMA_ITERATOR(vmi, vma->vm_mm, vma->vm_end);
1768 1768
1769 /* Try next first. */ 1769 /* Try next first. */
1770 next = vma_iter_load(&vmi); 1770 next = vma_iter_load(&vmi);
1771 if (next) { 1771 if (next) {
1772 anon_vma = reusable_anon_vma( 1772 anon_vma = reusable_anon_vma(next, vma, next);
1773 if (anon_vma) 1773 if (anon_vma)
1774 return anon_vma; 1774 return anon_vma;
1775 } 1775 }
1776 1776
1777 prev = vma_prev(&vmi); 1777 prev = vma_prev(&vmi);
1778 VM_BUG_ON_VMA(prev != vma, vma); 1778 VM_BUG_ON_VMA(prev != vma, vma);
1779 prev = vma_prev(&vmi); 1779 prev = vma_prev(&vmi);
1780 /* Try prev next. */ 1780 /* Try prev next. */
1781 if (prev) 1781 if (prev)
1782 anon_vma = reusable_anon_vma( 1782 anon_vma = reusable_anon_vma(prev, prev, vma);
1783 1783
1784 /* 1784 /*
1785 * We might reach here with anon_vma 1785 * We might reach here with anon_vma == NULL if we can't find
1786 * any reusable anon_vma. 1786 * any reusable anon_vma.
1787 * There's no absolute need to look o 1787 * There's no absolute need to look only at touching neighbours:
1788 * we could search further afield for 1788 * we could search further afield for "compatible" anon_vmas.
1789 * But it would probably just be a wa 1789 * But it would probably just be a waste of time searching,
1790 * or lead to too many vmas hanging o 1790 * or lead to too many vmas hanging off the same anon_vma.
1791 * We're trying to allow mprotect rem 1791 * We're trying to allow mprotect remerging later on,
1792 * not trying to minimize memory used 1792 * not trying to minimize memory used for anon_vmas.
1793 */ 1793 */
1794 return anon_vma; 1794 return anon_vma;
1795 } 1795 }
1796 1796
1797 static bool vm_ops_needs_writenotify(const st 1797 static bool vm_ops_needs_writenotify(const struct vm_operations_struct *vm_ops)
1798 { 1798 {
1799 return vm_ops && (vm_ops->page_mkwrit 1799 return vm_ops && (vm_ops->page_mkwrite || vm_ops->pfn_mkwrite);
1800 } 1800 }
1801 1801
1802 static bool vma_is_shared_writable(struct vm_ 1802 static bool vma_is_shared_writable(struct vm_area_struct *vma)
1803 { 1803 {
1804 return (vma->vm_flags & (VM_WRITE | V 1804 return (vma->vm_flags & (VM_WRITE | VM_SHARED)) ==
1805 (VM_WRITE | VM_SHARED); 1805 (VM_WRITE | VM_SHARED);
1806 } 1806 }
1807 1807
1808 static bool vma_fs_can_writeback(struct vm_ar 1808 static bool vma_fs_can_writeback(struct vm_area_struct *vma)
1809 { 1809 {
1810 /* No managed pages to writeback. */ 1810 /* No managed pages to writeback. */
1811 if (vma->vm_flags & VM_PFNMAP) 1811 if (vma->vm_flags & VM_PFNMAP)
1812 return false; 1812 return false;
1813 1813
1814 return vma->vm_file && vma->vm_file-> 1814 return vma->vm_file && vma->vm_file->f_mapping &&
1815 mapping_can_writeback(vma->vm 1815 mapping_can_writeback(vma->vm_file->f_mapping);
1816 } 1816 }
1817 1817
1818 /* 1818 /*
1819 * Does this VMA require the underlying folio 1819 * Does this VMA require the underlying folios to have their dirty state
1820 * tracked? 1820 * tracked?
1821 */ 1821 */
1822 bool vma_needs_dirty_tracking(struct vm_area_ 1822 bool vma_needs_dirty_tracking(struct vm_area_struct *vma)
1823 { 1823 {
1824 /* Only shared, writable VMAs require 1824 /* Only shared, writable VMAs require dirty tracking. */
1825 if (!vma_is_shared_writable(vma)) 1825 if (!vma_is_shared_writable(vma))
1826 return false; 1826 return false;
1827 1827
1828 /* Does the filesystem need to be not 1828 /* Does the filesystem need to be notified? */
1829 if (vm_ops_needs_writenotify(vma->vm_ 1829 if (vm_ops_needs_writenotify(vma->vm_ops))
1830 return true; 1830 return true;
1831 1831
1832 /* 1832 /*
1833 * Even if the filesystem doesn't ind 1833 * Even if the filesystem doesn't indicate a need for writenotify, if it
1834 * can writeback, dirty tracking is s 1834 * can writeback, dirty tracking is still required.
1835 */ 1835 */
1836 return vma_fs_can_writeback(vma); 1836 return vma_fs_can_writeback(vma);
1837 } 1837 }
1838 1838
1839 /* 1839 /*
1840 * Some shared mappings will want the pages m 1840 * Some shared mappings will want the pages marked read-only
1841 * to track write events. If so, we'll downgr 1841 * to track write events. If so, we'll downgrade vm_page_prot
1842 * to the private version (using protection_m 1842 * to the private version (using protection_map[] without the
1843 * VM_SHARED bit). 1843 * VM_SHARED bit).
1844 */ 1844 */
1845 bool vma_wants_writenotify(struct vm_area_str 1845 bool vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot)
1846 { 1846 {
1847 /* If it was private or non-writable, 1847 /* If it was private or non-writable, the write bit is already clear */
1848 if (!vma_is_shared_writable(vma)) 1848 if (!vma_is_shared_writable(vma))
1849 return false; 1849 return false;
1850 1850
1851 /* The backer wishes to know when pag 1851 /* The backer wishes to know when pages are first written to? */
1852 if (vm_ops_needs_writenotify(vma->vm_ 1852 if (vm_ops_needs_writenotify(vma->vm_ops))
1853 return true; 1853 return true;
1854 1854
1855 /* The open routine did something to 1855 /* The open routine did something to the protections that pgprot_modify
1856 * won't preserve? */ 1856 * won't preserve? */
1857 if (pgprot_val(vm_page_prot) != 1857 if (pgprot_val(vm_page_prot) !=
1858 pgprot_val(vm_pgprot_modify(vm_pa 1858 pgprot_val(vm_pgprot_modify(vm_page_prot, vma->vm_flags)))
1859 return false; 1859 return false;
1860 1860
1861 /* 1861 /*
1862 * Do we need to track softdirty? hug 1862 * Do we need to track softdirty? hugetlb does not support softdirty
1863 * tracking yet. 1863 * tracking yet.
1864 */ 1864 */
1865 if (vma_soft_dirty_enabled(vma) && !i 1865 if (vma_soft_dirty_enabled(vma) && !is_vm_hugetlb_page(vma))
1866 return true; 1866 return true;
1867 1867
1868 /* Do we need write faults for uffd-w 1868 /* Do we need write faults for uffd-wp tracking? */
1869 if (userfaultfd_wp(vma)) 1869 if (userfaultfd_wp(vma))
1870 return true; 1870 return true;
1871 1871
1872 /* Can the mapping track the dirty pa 1872 /* Can the mapping track the dirty pages? */
1873 return vma_fs_can_writeback(vma); 1873 return vma_fs_can_writeback(vma);
1874 } 1874 }
1875 1875
1876 static DEFINE_MUTEX(mm_all_locks_mutex); 1876 static DEFINE_MUTEX(mm_all_locks_mutex);
1877 1877
1878 static void vm_lock_anon_vma(struct mm_struct 1878 static void vm_lock_anon_vma(struct mm_struct *mm, struct anon_vma *anon_vma)
1879 { 1879 {
1880 if (!test_bit(0, (unsigned long *) &a 1880 if (!test_bit(0, (unsigned long *) &anon_vma->root->rb_root.rb_root.rb_node)) {
1881 /* 1881 /*
1882 * The LSB of head.next can't 1882 * The LSB of head.next can't change from under us
1883 * because we hold the mm_all 1883 * because we hold the mm_all_locks_mutex.
1884 */ 1884 */
1885 down_write_nest_lock(&anon_vm 1885 down_write_nest_lock(&anon_vma->root->rwsem, &mm->mmap_lock);
1886 /* 1886 /*
1887 * We can safely modify head. 1887 * We can safely modify head.next after taking the
1888 * anon_vma->root->rwsem. If 1888 * anon_vma->root->rwsem. If some other vma in this mm shares
1889 * the same anon_vma we won't 1889 * the same anon_vma we won't take it again.
1890 * 1890 *
1891 * No need of atomic instruct 1891 * No need of atomic instructions here, head.next
1892 * can't change from under us 1892 * can't change from under us thanks to the
1893 * anon_vma->root->rwsem. 1893 * anon_vma->root->rwsem.
1894 */ 1894 */
1895 if (__test_and_set_bit(0, (un 1895 if (__test_and_set_bit(0, (unsigned long *)
1896 &anon_ 1896 &anon_vma->root->rb_root.rb_root.rb_node))
1897 BUG(); 1897 BUG();
1898 } 1898 }
1899 } 1899 }
1900 1900
1901 static void vm_lock_mapping(struct mm_struct 1901 static void vm_lock_mapping(struct mm_struct *mm, struct address_space *mapping)
1902 { 1902 {
1903 if (!test_bit(AS_MM_ALL_LOCKS, &mappi 1903 if (!test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) {
1904 /* 1904 /*
1905 * AS_MM_ALL_LOCKS can't chan 1905 * AS_MM_ALL_LOCKS can't change from under us because
1906 * we hold the mm_all_locks_m 1906 * we hold the mm_all_locks_mutex.
1907 * 1907 *
1908 * Operations on ->flags have 1908 * Operations on ->flags have to be atomic because
1909 * even if AS_MM_ALL_LOCKS is 1909 * even if AS_MM_ALL_LOCKS is stable thanks to the
1910 * mm_all_locks_mutex, there 1910 * mm_all_locks_mutex, there may be other cpus
1911 * changing other bitflags in 1911 * changing other bitflags in parallel to us.
1912 */ 1912 */
1913 if (test_and_set_bit(AS_MM_AL 1913 if (test_and_set_bit(AS_MM_ALL_LOCKS, &mapping->flags))
1914 BUG(); 1914 BUG();
1915 down_write_nest_lock(&mapping 1915 down_write_nest_lock(&mapping->i_mmap_rwsem, &mm->mmap_lock);
1916 } 1916 }
1917 } 1917 }
1918 1918
1919 /* 1919 /*
1920 * This operation locks against the VM for al 1920 * This operation locks against the VM for all pte/vma/mm related
1921 * operations that could ever happen on a cer 1921 * operations that could ever happen on a certain mm. This includes
1922 * vmtruncate, try_to_unmap, and all page fau 1922 * vmtruncate, try_to_unmap, and all page faults.
1923 * 1923 *
1924 * The caller must take the mmap_lock in writ 1924 * The caller must take the mmap_lock in write mode before calling
1925 * mm_take_all_locks(). The caller isn't allo 1925 * mm_take_all_locks(). The caller isn't allowed to release the
1926 * mmap_lock until mm_drop_all_locks() return 1926 * mmap_lock until mm_drop_all_locks() returns.
1927 * 1927 *
1928 * mmap_lock in write mode is required in ord 1928 * mmap_lock in write mode is required in order to block all operations
1929 * that could modify pagetables and free page 1929 * that could modify pagetables and free pages without need of
1930 * altering the vma layout. It's also needed 1930 * altering the vma layout. It's also needed in write mode to avoid new
1931 * anon_vmas to be associated with existing v 1931 * anon_vmas to be associated with existing vmas.
1932 * 1932 *
1933 * A single task can't take more than one mm_ 1933 * A single task can't take more than one mm_take_all_locks() in a row
1934 * or it would deadlock. 1934 * or it would deadlock.
1935 * 1935 *
1936 * The LSB in anon_vma->rb_root.rb_node and t 1936 * The LSB in anon_vma->rb_root.rb_node and the AS_MM_ALL_LOCKS bitflag in
1937 * mapping->flags avoid to take the same lock 1937 * mapping->flags avoid to take the same lock twice, if more than one
1938 * vma in this mm is backed by the same anon_ 1938 * vma in this mm is backed by the same anon_vma or address_space.
1939 * 1939 *
1940 * We take locks in following order, accordin 1940 * We take locks in following order, accordingly to comment at beginning
1941 * of mm/rmap.c: 1941 * of mm/rmap.c:
1942 * - all hugetlbfs_i_mmap_rwsem_key locks ( 1942 * - all hugetlbfs_i_mmap_rwsem_key locks (aka mapping->i_mmap_rwsem for
1943 * hugetlb mapping); 1943 * hugetlb mapping);
1944 * - all vmas marked locked 1944 * - all vmas marked locked
1945 * - all i_mmap_rwsem locks; 1945 * - all i_mmap_rwsem locks;
1946 * - all anon_vma->rwseml 1946 * - all anon_vma->rwseml
1947 * 1947 *
1948 * We can take all locks within these types r 1948 * We can take all locks within these types randomly because the VM code
1949 * doesn't nest them and we protected from pa 1949 * doesn't nest them and we protected from parallel mm_take_all_locks() by
1950 * mm_all_locks_mutex. 1950 * mm_all_locks_mutex.
1951 * 1951 *
1952 * mm_take_all_locks() and mm_drop_all_locks 1952 * mm_take_all_locks() and mm_drop_all_locks are expensive operations
1953 * that may have to take thousand of locks. 1953 * that may have to take thousand of locks.
1954 * 1954 *
1955 * mm_take_all_locks() can fail if it's inter 1955 * mm_take_all_locks() can fail if it's interrupted by signals.
1956 */ 1956 */
1957 int mm_take_all_locks(struct mm_struct *mm) 1957 int mm_take_all_locks(struct mm_struct *mm)
1958 { 1958 {
1959 struct vm_area_struct *vma; 1959 struct vm_area_struct *vma;
1960 struct anon_vma_chain *avc; 1960 struct anon_vma_chain *avc;
1961 VMA_ITERATOR(vmi, mm, 0); 1961 VMA_ITERATOR(vmi, mm, 0);
1962 1962
1963 mmap_assert_write_locked(mm); 1963 mmap_assert_write_locked(mm);
1964 1964
1965 mutex_lock(&mm_all_locks_mutex); 1965 mutex_lock(&mm_all_locks_mutex);
1966 1966
1967 /* 1967 /*
1968 * vma_start_write() does not have a 1968 * vma_start_write() does not have a complement in mm_drop_all_locks()
1969 * because vma_start_write() is alway 1969 * because vma_start_write() is always asymmetrical; it marks a VMA as
1970 * being written to until mmap_write_ 1970 * being written to until mmap_write_unlock() or mmap_write_downgrade()
1971 * is reached. 1971 * is reached.
1972 */ 1972 */
1973 for_each_vma(vmi, vma) { 1973 for_each_vma(vmi, vma) {
1974 if (signal_pending(current)) 1974 if (signal_pending(current))
1975 goto out_unlock; 1975 goto out_unlock;
1976 vma_start_write(vma); 1976 vma_start_write(vma);
1977 } 1977 }
1978 1978
1979 vma_iter_init(&vmi, mm, 0); 1979 vma_iter_init(&vmi, mm, 0);
1980 for_each_vma(vmi, vma) { 1980 for_each_vma(vmi, vma) {
1981 if (signal_pending(current)) 1981 if (signal_pending(current))
1982 goto out_unlock; 1982 goto out_unlock;
1983 if (vma->vm_file && vma->vm_f 1983 if (vma->vm_file && vma->vm_file->f_mapping &&
1984 is_vm_hugetlb 1984 is_vm_hugetlb_page(vma))
1985 vm_lock_mapping(mm, v 1985 vm_lock_mapping(mm, vma->vm_file->f_mapping);
1986 } 1986 }
1987 1987
1988 vma_iter_init(&vmi, mm, 0); 1988 vma_iter_init(&vmi, mm, 0);
1989 for_each_vma(vmi, vma) { 1989 for_each_vma(vmi, vma) {
1990 if (signal_pending(current)) 1990 if (signal_pending(current))
1991 goto out_unlock; 1991 goto out_unlock;
1992 if (vma->vm_file && vma->vm_f 1992 if (vma->vm_file && vma->vm_file->f_mapping &&
1993 !is_vm_hugetl 1993 !is_vm_hugetlb_page(vma))
1994 vm_lock_mapping(mm, v 1994 vm_lock_mapping(mm, vma->vm_file->f_mapping);
1995 } 1995 }
1996 1996
1997 vma_iter_init(&vmi, mm, 0); 1997 vma_iter_init(&vmi, mm, 0);
1998 for_each_vma(vmi, vma) { 1998 for_each_vma(vmi, vma) {
1999 if (signal_pending(current)) 1999 if (signal_pending(current))
2000 goto out_unlock; 2000 goto out_unlock;
2001 if (vma->anon_vma) 2001 if (vma->anon_vma)
2002 list_for_each_entry(a 2002 list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
2003 vm_lock_anon_ 2003 vm_lock_anon_vma(mm, avc->anon_vma);
2004 } 2004 }
2005 2005
2006 return 0; 2006 return 0;
2007 2007
2008 out_unlock: 2008 out_unlock:
2009 mm_drop_all_locks(mm); 2009 mm_drop_all_locks(mm);
2010 return -EINTR; 2010 return -EINTR;
2011 } 2011 }
2012 2012
2013 static void vm_unlock_anon_vma(struct anon_vm 2013 static void vm_unlock_anon_vma(struct anon_vma *anon_vma)
2014 { 2014 {
2015 if (test_bit(0, (unsigned long *) &an 2015 if (test_bit(0, (unsigned long *) &anon_vma->root->rb_root.rb_root.rb_node)) {
2016 /* 2016 /*
2017 * The LSB of head.next can't 2017 * The LSB of head.next can't change to 0 from under
2018 * us because we hold the mm_ 2018 * us because we hold the mm_all_locks_mutex.
2019 * 2019 *
2020 * We must however clear the 2020 * We must however clear the bitflag before unlocking
2021 * the vma so the users using 2021 * the vma so the users using the anon_vma->rb_root will
2022 * never see our bitflag. 2022 * never see our bitflag.
2023 * 2023 *
2024 * No need of atomic instruct 2024 * No need of atomic instructions here, head.next
2025 * can't change from under us 2025 * can't change from under us until we release the
2026 * anon_vma->root->rwsem. 2026 * anon_vma->root->rwsem.
2027 */ 2027 */
2028 if (!__test_and_clear_bit(0, 2028 if (!__test_and_clear_bit(0, (unsigned long *)
2029 &an 2029 &anon_vma->root->rb_root.rb_root.rb_node))
2030 BUG(); 2030 BUG();
2031 anon_vma_unlock_write(anon_vm 2031 anon_vma_unlock_write(anon_vma);
2032 } 2032 }
2033 } 2033 }
2034 2034
2035 static void vm_unlock_mapping(struct address_ 2035 static void vm_unlock_mapping(struct address_space *mapping)
2036 { 2036 {
2037 if (test_bit(AS_MM_ALL_LOCKS, &mappin 2037 if (test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) {
2038 /* 2038 /*
2039 * AS_MM_ALL_LOCKS can't chan 2039 * AS_MM_ALL_LOCKS can't change to 0 from under us
2040 * because we hold the mm_all 2040 * because we hold the mm_all_locks_mutex.
2041 */ 2041 */
2042 i_mmap_unlock_write(mapping); 2042 i_mmap_unlock_write(mapping);
2043 if (!test_and_clear_bit(AS_MM 2043 if (!test_and_clear_bit(AS_MM_ALL_LOCKS,
2044 &mapp 2044 &mapping->flags))
2045 BUG(); 2045 BUG();
2046 } 2046 }
2047 } 2047 }
2048 2048
2049 /* 2049 /*
2050 * The mmap_lock cannot be released by the ca 2050 * The mmap_lock cannot be released by the caller until
2051 * mm_drop_all_locks() returns. 2051 * mm_drop_all_locks() returns.
2052 */ 2052 */
2053 void mm_drop_all_locks(struct mm_struct *mm) 2053 void mm_drop_all_locks(struct mm_struct *mm)
2054 { 2054 {
2055 struct vm_area_struct *vma; 2055 struct vm_area_struct *vma;
2056 struct anon_vma_chain *avc; 2056 struct anon_vma_chain *avc;
2057 VMA_ITERATOR(vmi, mm, 0); 2057 VMA_ITERATOR(vmi, mm, 0);
2058 2058
2059 mmap_assert_write_locked(mm); 2059 mmap_assert_write_locked(mm);
2060 BUG_ON(!mutex_is_locked(&mm_all_locks 2060 BUG_ON(!mutex_is_locked(&mm_all_locks_mutex));
2061 2061
2062 for_each_vma(vmi, vma) { 2062 for_each_vma(vmi, vma) {
2063 if (vma->anon_vma) 2063 if (vma->anon_vma)
2064 list_for_each_entry(a 2064 list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
2065 vm_unlock_ano 2065 vm_unlock_anon_vma(avc->anon_vma);
2066 if (vma->vm_file && vma->vm_f 2066 if (vma->vm_file && vma->vm_file->f_mapping)
2067 vm_unlock_mapping(vma 2067 vm_unlock_mapping(vma->vm_file->f_mapping);
2068 } 2068 }
2069 2069
2070 mutex_unlock(&mm_all_locks_mutex); 2070 mutex_unlock(&mm_all_locks_mutex);
2071 } 2071 }
2072 2072
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.