1 // SPDX-License-Identifier: GPL-2.0-or-later 1
2
3 /*
4 * VMA-specific functions.
5 */
6
7 #include "vma_internal.h"
8 #include "vma.h"
9
10 static inline bool is_mergeable_vma(struct vma
11 {
12 struct vm_area_struct *vma = merge_nex
13
14 if (!mpol_equal(vmg->policy, vma_polic
15 return false;
16 /*
17 * VM_SOFTDIRTY should not prevent fro
18 * match the flags but dirty bit -- th
19 * merged VMA as dirty. If dirty bit w
20 * comparison, we increase pressure on
21 * the kernel to generate new VMAs whe
22 * extended instead.
23 */
24 if ((vma->vm_flags ^ vmg->flags) & ~VM
25 return false;
26 if (vma->vm_file != vmg->file)
27 return false;
28 if (!is_mergeable_vm_userfaultfd_ctx(v
29 return false;
30 if (!anon_vma_name_eq(anon_vma_name(vm
31 return false;
32 return true;
33 }
34
35 static inline bool is_mergeable_anon_vma(struc
36 struct anon_vma *anon_vma2, s
37 {
38 /*
39 * The list_is_singular() test is to a
40 * parents. This can improve scalabili
41 */
42 if ((!anon_vma1 || !anon_vma2) && (!vm
43 list_is_singular(&vma->anon_vm
44 return true;
45 return anon_vma1 == anon_vma2;
46 }
47
48 /* Are the anon_vma's belonging to each VMA co
49 static inline bool are_anon_vmas_compatible(st
50 st
51 {
52 return is_mergeable_anon_vma(vma1->ano
53 }
54
55 /*
56 * init_multi_vma_prep() - Initializer for str
57 * @vp: The vma_prepare struct
58 * @vma: The vma that will be altered once loc
59 * @next: The next vma if it is to be adjusted
60 * @remove: The first vma to be removed
61 * @remove2: The second vma to be removed
62 */
63 static void init_multi_vma_prep(struct vma_pre
64 struct vm_area
65 struct vm_area
66 struct vm_area
67 struct vm_area
68 {
69 memset(vp, 0, sizeof(struct vma_prepar
70 vp->vma = vma;
71 vp->anon_vma = vma->anon_vma;
72 vp->remove = remove;
73 vp->remove2 = remove2;
74 vp->adj_next = next;
75 if (!vp->anon_vma && next)
76 vp->anon_vma = next->anon_vma;
77
78 vp->file = vma->vm_file;
79 if (vp->file)
80 vp->mapping = vma->vm_file->f_
81
82 }
83
84 /*
85 * Return true if we can merge this (vm_flags,
86 * in front of (at a lower virtual address and
87 *
88 * We cannot merge two vmas if they have diffe
89 * anon_vmas, nor if same anon_vma is assigned
90 *
91 * We don't check here for the merged mmap wra
92 * indices (16TB on ia32) because do_mmap() do
93 * wrap, nor mmaps which cover the final page
94 *
95 * We assume the vma may be removed as part of
96 */
97 static bool can_vma_merge_before(struct vma_me
98 {
99 pgoff_t pglen = PHYS_PFN(vmg->end - vm
100
101 if (is_mergeable_vma(vmg, /* merge_nex
102 is_mergeable_anon_vma(vmg->anon_vm
103 if (vmg->next->vm_pgoff == vmg
104 return true;
105 }
106
107 return false;
108 }
109
110 /*
111 * Return true if we can merge this (vm_flags,
112 * beyond (at a higher virtual address and fil
113 *
114 * We cannot merge two vmas if they have diffe
115 * anon_vmas, nor if same anon_vma is assigned
116 *
117 * We assume that vma is not removed as part o
118 */
119 static bool can_vma_merge_after(struct vma_mer
120 {
121 if (is_mergeable_vma(vmg, /* merge_nex
122 is_mergeable_anon_vma(vmg->anon_vm
123 if (vmg->prev->vm_pgoff + vma_
124 return true;
125 }
126 return false;
127 }
128
129 static void __vma_link_file(struct vm_area_str
130 struct address_spa
131 {
132 if (vma_is_shared_maywrite(vma))
133 mapping_allow_writable(mapping
134
135 flush_dcache_mmap_lock(mapping);
136 vma_interval_tree_insert(vma, &mapping
137 flush_dcache_mmap_unlock(mapping);
138 }
139
140 /*
141 * Requires inode->i_mapping->i_mmap_rwsem
142 */
143 static void __remove_shared_vm_struct(struct v
144 struct a
145 {
146 if (vma_is_shared_maywrite(vma))
147 mapping_unmap_writable(mapping
148
149 flush_dcache_mmap_lock(mapping);
150 vma_interval_tree_remove(vma, &mapping
151 flush_dcache_mmap_unlock(mapping);
152 }
153
154 /*
155 * vma_prepare() - Helper function for handlin
156 * @vp: The initialized vma_prepare struct
157 */
158 static void vma_prepare(struct vma_prepare *vp
159 {
160 if (vp->file) {
161 uprobe_munmap(vp->vma, vp->vma
162
163 if (vp->adj_next)
164 uprobe_munmap(vp->adj_
165 vp->adj_
166
167 i_mmap_lock_write(vp->mapping)
168 if (vp->insert && vp->insert->
169 /*
170 * Put into interval t
171 * are visible to arm/
172 * throughout; but we
173 * space until vma sta
174 */
175 __vma_link_file(vp->in
176 vp->in
177 }
178 }
179
180 if (vp->anon_vma) {
181 anon_vma_lock_write(vp->anon_v
182 anon_vma_interval_tree_pre_upd
183 if (vp->adj_next)
184 anon_vma_interval_tree
185 }
186
187 if (vp->file) {
188 flush_dcache_mmap_lock(vp->map
189 vma_interval_tree_remove(vp->v
190 if (vp->adj_next)
191 vma_interval_tree_remo
192
193 }
194
195 }
196
197 /*
198 * vma_complete- Helper function for handling
199 * or for inserting a VMA.
200 *
201 * @vp: The vma_prepare struct
202 * @vmi: The vma iterator
203 * @mm: The mm_struct
204 */
205 static void vma_complete(struct vma_prepare *v
206 struct mm_struct *mm)
207 {
208 if (vp->file) {
209 if (vp->adj_next)
210 vma_interval_tree_inse
211
212 vma_interval_tree_insert(vp->v
213 flush_dcache_mmap_unlock(vp->m
214 }
215
216 if (vp->remove && vp->file) {
217 __remove_shared_vm_struct(vp->
218 if (vp->remove2)
219 __remove_shared_vm_str
220 } else if (vp->insert) {
221 /*
222 * split_vma has split insert
223 * us to insert it before drop
224 * (it may either follow vma o
225 */
226 vma_iter_store(vmi, vp->insert
227 mm->map_count++;
228 }
229
230 if (vp->anon_vma) {
231 anon_vma_interval_tree_post_up
232 if (vp->adj_next)
233 anon_vma_interval_tree
234 anon_vma_unlock_write(vp->anon
235 }
236
237 if (vp->file) {
238 i_mmap_unlock_write(vp->mappin
239 uprobe_mmap(vp->vma);
240
241 if (vp->adj_next)
242 uprobe_mmap(vp->adj_ne
243 }
244
245 if (vp->remove) {
246 again:
247 vma_mark_detached(vp->remove,
248 if (vp->file) {
249 uprobe_munmap(vp->remo
250 vp->remo
251 fput(vp->file);
252 }
253 if (vp->remove->anon_vma)
254 anon_vma_merge(vp->vma
255 mm->map_count--;
256 mpol_put(vma_policy(vp->remove
257 if (!vp->remove2)
258 WARN_ON_ONCE(vp->vma->
259 vm_area_free(vp->remove);
260
261 /*
262 * In mprotect's case 6 (see c
263 * we are removing both mid an
264 */
265 if (vp->remove2) {
266 vp->remove = vp->remov
267 vp->remove2 = NULL;
268 goto again;
269 }
270 }
271 if (vp->insert && vp->file)
272 uprobe_mmap(vp->insert);
273 }
274
275 /*
276 * init_vma_prep() - Initializer wrapper for v
277 * @vp: The vma_prepare struct
278 * @vma: The vma that will be altered once loc
279 */
280 static void init_vma_prep(struct vma_prepare *
281 {
282 init_multi_vma_prep(vp, vma, NULL, NUL
283 }
284
285 /*
286 * Can the proposed VMA be merged with the lef
287 * account the start position of the proposed
288 */
289 static bool can_vma_merge_left(struct vma_merg
290
291 {
292 return vmg->prev && vmg->prev->vm_end
293 can_vma_merge_after(vmg);
294 }
295
296 /*
297 * Can the proposed VMA be merged with the rig
298 * account the end position of the proposed ra
299 *
300 * In addition, if we can merge with the left
301 * anon_vma's are also compatible.
302 */
303 static bool can_vma_merge_right(struct vma_mer
304 bool can_merge
305 {
306 if (!vmg->next || vmg->end != vmg->nex
307 !can_vma_merge_before(vmg))
308 return false;
309
310 if (!can_merge_left)
311 return true;
312
313 /*
314 * If we can merge with prev (left) an
315 * each VMA's anon_vma is compatible w
316 * does not mean prev and next are com
317 *
318 * We therefore check this in addition
319 */
320 return are_anon_vmas_compatible(vmg->p
321 }
322
323 /*
324 * Close a vm structure and free it.
325 */
326 void remove_vma(struct vm_area_struct *vma, bo
327 {
328 might_sleep();
329 vma_close(vma);
330 if (vma->vm_file)
331 fput(vma->vm_file);
332 mpol_put(vma_policy(vma));
333 if (unreachable)
334 __vm_area_free(vma);
335 else
336 vm_area_free(vma);
337 }
338
339 /*
340 * Get rid of page table information in the in
341 *
342 * Called with the mm semaphore held.
343 */
344 void unmap_region(struct ma_state *mas, struct
345 struct vm_area_struct *prev, s
346 {
347 struct mm_struct *mm = vma->vm_mm;
348 struct mmu_gather tlb;
349
350 lru_add_drain();
351 tlb_gather_mmu(&tlb, mm);
352 update_hiwater_rss(mm);
353 unmap_vmas(&tlb, mas, vma, vma->vm_sta
354 /* mm_wr_locked = */ true);
355 mas_set(mas, vma->vm_end);
356 free_pgtables(&tlb, mas, vma, prev ? p
357 next ? next->vm_start :
358 /* mm_wr_locked = */ tru
359 tlb_finish_mmu(&tlb);
360 }
361
362 /*
363 * __split_vma() bypasses sysctl_max_map_count
364 * has already been checked or doesn't make se
365 * VMA Iterator will point to the original VMA
366 */
367 static int __split_vma(struct vma_iterator *vm
368 unsigned long addr, int
369 {
370 struct vma_prepare vp;
371 struct vm_area_struct *new;
372 int err;
373
374 WARN_ON(vma->vm_start >= addr);
375 WARN_ON(vma->vm_end <= addr);
376
377 if (vma->vm_ops && vma->vm_ops->may_sp
378 err = vma->vm_ops->may_split(v
379 if (err)
380 return err;
381 }
382
383 new = vm_area_dup(vma);
384 if (!new)
385 return -ENOMEM;
386
387 if (new_below) {
388 new->vm_end = addr;
389 } else {
390 new->vm_start = addr;
391 new->vm_pgoff += ((addr - vma-
392 }
393
394 err = -ENOMEM;
395 vma_iter_config(vmi, new->vm_start, ne
396 if (vma_iter_prealloc(vmi, new))
397 goto out_free_vma;
398
399 err = vma_dup_policy(vma, new);
400 if (err)
401 goto out_free_vmi;
402
403 err = anon_vma_clone(new, vma);
404 if (err)
405 goto out_free_mpol;
406
407 if (new->vm_file)
408 get_file(new->vm_file);
409
410 if (new->vm_ops && new->vm_ops->open)
411 new->vm_ops->open(new);
412
413 vma_start_write(vma);
414 vma_start_write(new);
415
416 init_vma_prep(&vp, vma);
417 vp.insert = new;
418 vma_prepare(&vp);
419 vma_adjust_trans_huge(vma, vma->vm_sta
420
421 if (new_below) {
422 vma->vm_start = addr;
423 vma->vm_pgoff += (addr - new->
424 } else {
425 vma->vm_end = addr;
426 }
427
428 /* vma_complete stores the new vma */
429 vma_complete(&vp, vmi, vma->vm_mm);
430 validate_mm(vma->vm_mm);
431
432 /* Success. */
433 if (new_below)
434 vma_next(vmi);
435 else
436 vma_prev(vmi);
437
438 return 0;
439
440 out_free_mpol:
441 mpol_put(vma_policy(new));
442 out_free_vmi:
443 vma_iter_free(vmi);
444 out_free_vma:
445 vm_area_free(new);
446 return err;
447 }
448
449 /*
450 * Split a vma into two pieces at address 'add
451 * either for the first part or the tail.
452 */
453 static int split_vma(struct vma_iterator *vmi,
454 unsigned long addr, int n
455 {
456 if (vma->vm_mm->map_count >= sysctl_ma
457 return -ENOMEM;
458
459 return __split_vma(vmi, vma, addr, new
460 }
461
462 /*
463 * vma has some anon_vma assigned, and is alre
464 * anon_vma's interval trees.
465 *
466 * Before updating the vma's vm_start / vm_end
467 * vma must be removed from the anon_vma's int
468 * anon_vma_interval_tree_pre_update_vma().
469 *
470 * After the update, the vma will be reinserte
471 * anon_vma_interval_tree_post_update_vma().
472 *
473 * The entire update must be protected by excl
474 * the root anon_vma's mutex.
475 */
476 void
477 anon_vma_interval_tree_pre_update_vma(struct v
478 {
479 struct anon_vma_chain *avc;
480
481 list_for_each_entry(avc, &vma->anon_vm
482 anon_vma_interval_tree_remove(
483 }
484
485 void
486 anon_vma_interval_tree_post_update_vma(struct
487 {
488 struct anon_vma_chain *avc;
489
490 list_for_each_entry(avc, &vma->anon_vm
491 anon_vma_interval_tree_insert(
492 }
493
494 /*
495 * dup_anon_vma() - Helper function to duplica
496 * @dst: The destination VMA
497 * @src: The source VMA
498 * @dup: Pointer to the destination VMA when s
499 *
500 * Returns: 0 on success.
501 */
502 static int dup_anon_vma(struct vm_area_struct
503 struct vm_area_struct
504 {
505 /*
506 * Easily overlooked: when mprotect sh
507 * expanding vma has anon_vma set if t
508 * anon pages imported.
509 */
510 if (src->anon_vma && !dst->anon_vma) {
511 int ret;
512
513 vma_assert_write_locked(dst);
514 dst->anon_vma = src->anon_vma;
515 ret = anon_vma_clone(dst, src)
516 if (ret)
517 return ret;
518
519 *dup = dst;
520 }
521
522 return 0;
523 }
524
525 #ifdef CONFIG_DEBUG_VM_MAPLE_TREE
526 void validate_mm(struct mm_struct *mm)
527 {
528 int bug = 0;
529 int i = 0;
530 struct vm_area_struct *vma;
531 VMA_ITERATOR(vmi, mm, 0);
532
533 mt_validate(&mm->mm_mt);
534 for_each_vma(vmi, vma) {
535 #ifdef CONFIG_DEBUG_VM_RB
536 struct anon_vma *anon_vma = vm
537 struct anon_vma_chain *avc;
538 #endif
539 unsigned long vmi_start, vmi_e
540 bool warn = 0;
541
542 vmi_start = vma_iter_addr(&vmi
543 vmi_end = vma_iter_end(&vmi);
544 if (VM_WARN_ON_ONCE_MM(vma->vm
545 warn = 1;
546
547 if (VM_WARN_ON_ONCE_MM(vma->vm
548 warn = 1;
549
550 if (warn) {
551 pr_emerg("issue in %s\
552 dump_stack();
553 dump_vma(vma);
554 pr_emerg("tree range:
555 vmi_start, vm
556 vma_iter_dump_tree(&vm
557 }
558
559 #ifdef CONFIG_DEBUG_VM_RB
560 if (anon_vma) {
561 anon_vma_lock_read(ano
562 list_for_each_entry(av
563 anon_vma_inter
564 anon_vma_unlock_read(a
565 }
566 #endif
567 i++;
568 }
569 if (i != mm->map_count) {
570 pr_emerg("map_count %d vma ite
571 bug = 1;
572 }
573 VM_BUG_ON_MM(bug, mm);
574 }
575 #endif /* CONFIG_DEBUG_VM_MAPLE_TREE */
576
577 /* Actually perform the VMA merge operation. *
578 static int commit_merge(struct vma_merge_struc
579 struct vm_area_struct
580 struct vm_area_struct
581 struct vm_area_struct
582 long adj_start,
583 bool expanded)
584 {
585 struct vma_prepare vp;
586
587 init_multi_vma_prep(&vp, vmg->vma, adj
588
589 VM_WARN_ON(vp.anon_vma && adjust && ad
590 vp.anon_vma != adjust->anon
591
592 if (expanded) {
593 /* Note: vma iterator must be
594 vma_iter_config(vmg->vmi, vmg-
595 } else {
596 vma_iter_config(vmg->vmi, adju
597 adjust->vm_end
598 }
599
600 if (vma_iter_prealloc(vmg->vmi, vmg->v
601 return -ENOMEM;
602
603 vma_prepare(&vp);
604 vma_adjust_trans_huge(vmg->vma, vmg->s
605 vma_set_range(vmg->vma, vmg->start, vm
606
607 if (expanded)
608 vma_iter_store(vmg->vmi, vmg->
609
610 if (adj_start) {
611 adjust->vm_start += adj_start;
612 adjust->vm_pgoff += PHYS_PFN(a
613 if (adj_start < 0) {
614 WARN_ON(expanded);
615 vma_iter_store(vmg->vm
616 }
617 }
618
619 vma_complete(&vp, vmg->vmi, vmg->vma->
620
621 return 0;
622 }
623
624 /* We can only remove VMAs when merging if the
625 static bool can_merge_remove_vma(struct vm_are
626 {
627 return !vma->vm_ops || !vma->vm_ops->c
628 }
629
630 /*
631 * vma_merge_existing_range - Attempt to merge
632 * attributes modified.
633 *
634 * @vmg: Describes the modifications being mad
635 * metadata.
636 *
637 * When the attributes of a range within a VMA
638 * for immediately adjacent VMAs to be merged
639 * identical properties.
640 *
641 * This function checks for the existence of a
642 * the maple tree describing the @vmg->vma->vm
643 * this, as well as any VMAs shrunk/expanded/d
644 *
645 * As part of this operation, if a merge occur
646 * vma, start, end, and pgoff fields modified
647 * calls to this function should reset these f
648 *
649 * Returns: The merged VMA if merge succeeds,
650 *
651 * ASSUMPTIONS:
652 * - The caller must assign the VMA to be modi
653 * - The caller must have set @vmg->prev to th
654 * - The caller must not set @vmg->next, as we
655 * - The caller must hold a WRITE lock on the
656 * - vmi must be positioned within [@vmg->vma-
657 */
658 static struct vm_area_struct *vma_merge_existi
659 {
660 struct vm_area_struct *vma = vmg->vma;
661 struct vm_area_struct *prev = vmg->pre
662 struct vm_area_struct *next, *res;
663 struct vm_area_struct *anon_dup = NULL
664 struct vm_area_struct *adjust = NULL;
665 unsigned long start = vmg->start;
666 unsigned long end = vmg->end;
667 bool left_side = vma && start == vma->
668 bool right_side = vma && end == vma->v
669 int err = 0;
670 long adj_start = 0;
671 bool merge_will_delete_vma, merge_will
672 bool merge_left, merge_right, merge_bo
673 bool expanded;
674
675 mmap_assert_write_locked(vmg->mm);
676 VM_WARN_ON(!vma); /* We are modifying
677 VM_WARN_ON(vmg->next); /* We set this.
678 VM_WARN_ON(prev && start <= prev->vm_s
679 VM_WARN_ON(start >= end);
680 /*
681 * If vma == prev, then we are offset
682 * not, we must span a portion of the
683 */
684 VM_WARN_ON(vma && ((vma != prev && vmg
685 vmg->end > vma->vm_
686 /* The vmi must be positioned within v
687 VM_WARN_ON(vma && !(vma_iter_addr(vmg-
688 vma_iter_addr(vmg-
689
690 vmg->state = VMA_MERGE_NOMERGE;
691
692 /*
693 * If a special mapping or if the rang
694 * furthermost left or right side of t
695 * merging and should abort.
696 */
697 if (vmg->flags & VM_SPECIAL || (!left_
698 return NULL;
699
700 if (left_side)
701 merge_left = can_vma_merge_lef
702 else
703 merge_left = false;
704
705 if (right_side) {
706 next = vmg->next = vma_iter_ne
707 vma_iter_prev_range(vmg->vmi);
708
709 merge_right = can_vma_merge_ri
710 } else {
711 merge_right = false;
712 next = NULL;
713 }
714
715 if (merge_left) /* If merging
716 vma_prev(vmg->vmi);
717 else if (!merge_right) /* If we have
718 return NULL;
719
720 merge_both = merge_left && merge_right
721 /* If we span the entire VMA, a merge
722 merge_will_delete_vma = left_side && r
723
724 /*
725 * If we need to remove vma in its ent
726 * we have no sensible recourse but to
727 */
728 if (merge_will_delete_vma && !can_merg
729 return NULL;
730
731 /*
732 * If we merge both VMAs, then next is
733 * merge_will_delete_vma also.
734 */
735 merge_will_delete_next = merge_both;
736
737 /*
738 * If we cannot delete next, then we c
739 * prev and vma (thereby deleting vma)
740 */
741 if (merge_will_delete_next && !can_mer
742 merge_will_delete_next = false
743 merge_right = false;
744 merge_both = false;
745 }
746
747 /* No matter what happens, we will be
748 vma_start_write(vma);
749
750 if (merge_left)
751 vma_start_write(prev);
752
753 if (merge_right)
754 vma_start_write(next);
755
756 if (merge_both) {
757 /*
758 * |<----->|
759 * |-------*********-------|
760 * prev vma next
761 * extend delete delete
762 */
763
764 vmg->vma = prev;
765 vmg->start = prev->vm_start;
766 vmg->end = next->vm_end;
767 vmg->pgoff = prev->vm_pgoff;
768
769 /*
770 * We already ensured anon_vma
771 * simply a case of, if prev h
772 * next or vma contains the an
773 */
774 err = dup_anon_vma(prev, next-
775 } else if (merge_left) {
776 /*
777 * |<----->| OR
778 * |<--------->|
779 * |-------*************
780 * prev vma
781 * extend shrink/delete
782 */
783
784 vmg->vma = prev;
785 vmg->start = prev->vm_start;
786 vmg->pgoff = prev->vm_pgoff;
787
788 if (!merge_will_delete_vma) {
789 adjust = vma;
790 adj_start = vmg->end -
791 }
792
793 err = dup_anon_vma(prev, vma,
794 } else { /* merge_right */
795 /*
796 * |<----->| OR
797 * |<--------->|
798 * *************-------|
799 * vma next
800 * shrink/delete extend
801 */
802
803 pgoff_t pglen = PHYS_PFN(vmg->
804
805 VM_WARN_ON(!merge_right);
806 /* If we are offset into a VMA
807 VM_WARN_ON(vmg->start > vma->v
808
809 if (merge_will_delete_vma) {
810 vmg->vma = next;
811 vmg->end = next->vm_en
812 vmg->pgoff = next->vm_
813 } else {
814 /*
815 * We shrink vma and e
816 *
817 * IMPORTANT: This is
818 * merged VMA is NOT v
819 */
820
821 vmg->start = vma->vm_s
822 vmg->end = start;
823 vmg->pgoff = vma->vm_p
824
825 adjust = next;
826 adj_start = -(vma->vm_
827 }
828
829 err = dup_anon_vma(next, vma,
830 }
831
832 if (err)
833 goto abort;
834
835 /*
836 * In nearly all cases, we expand vmg-
837 * merge_right where we partially span
838 * the end of vmg->vma and adjust the
839 */
840 expanded = !merge_right || merge_will_
841
842 if (commit_merge(vmg, adjust,
843 merge_will_delete_vma
844 merge_will_delete_nex
845 adj_start, expanded))
846 if (anon_dup)
847 unlink_anon_vmas(anon_
848
849 vmg->state = VMA_MERGE_ERROR_N
850 return NULL;
851 }
852
853 res = merge_left ? prev : next;
854 khugepaged_enter_vma(res, vmg->flags);
855
856 vmg->state = VMA_MERGE_SUCCESS;
857 return res;
858
859 abort:
860 vma_iter_set(vmg->vmi, start);
861 vma_iter_load(vmg->vmi);
862 vmg->state = VMA_MERGE_ERROR_NOMEM;
863 return NULL;
864 }
865
866 /*
867 * vma_merge_new_range - Attempt to merge a ne
868 *
869 * @vmg: Describes the VMA we are adding, in t
870 * (exclusive), which we try to merge wi
871 *
872 * We are about to add a VMA to the address sp
873 * ending at @vmg->end. There are three differ
874 *
875 * 1. There is a VMA with identical properties
876 * proposed new VMA [@vmg->start, @vmg->end
877 * EXPAND that VMA:
878 *
879 * Proposed: |-----| or |-----|
880 * Existing: |----| |----|
881 *
882 * 2. There are VMAs with identical properties
883 * proposed new VMA [@vmg->start, @vmg->end
884 * EXPAND the former and REMOVE the latter:
885 *
886 * Proposed: |-----|
887 * Existing: |----| |----|
888 *
889 * 3. There are no VMAs immediately adjacent t
890 * VMAs do not have identical attributes -
891 *
892 * In instances where we can merge, this funct
893 * will have its range adjusted accordingly an
894 * adjusted.
895 *
896 * Returns: In instances where no merge was po
897 * to the VMA we expanded.
898 *
899 * This function adjusts @vmg to provide @vmg-
900 * and adjusts [@vmg->start, @vmg->end) to spa
901 *
902 * ASSUMPTIONS:
903 * - The caller must hold a WRITE lock on the
904 * - The caller must have determined that [@vm
905 other than VMAs that will be unmapped sho
906 * - The caller must have specified the previo
907 * - The caller must have specified the next v
908 * - The caller must have positioned the vmi a
909 */
910 struct vm_area_struct *vma_merge_new_range(str
911 {
912 struct vm_area_struct *prev = vmg->pre
913 struct vm_area_struct *next = vmg->nex
914 unsigned long start = vmg->start;
915 unsigned long end = vmg->end;
916 pgoff_t pgoff = vmg->pgoff;
917 pgoff_t pglen = PHYS_PFN(end - start);
918 bool can_merge_left, can_merge_right;
919 bool just_expand = vmg->merge_flags &
920
921 mmap_assert_write_locked(vmg->mm);
922 VM_WARN_ON(vmg->vma);
923 /* vmi must point at or before the gap
924 VM_WARN_ON(vma_iter_addr(vmg->vmi) > e
925
926 vmg->state = VMA_MERGE_NOMERGE;
927
928 /* Special VMAs are unmergeable, also
929 if ((vmg->flags & VM_SPECIAL) || (!pre
930 return NULL;
931
932 can_merge_left = can_vma_merge_left(vm
933 can_merge_right = !just_expand && can_
934
935 /* If we can merge with the next VMA,
936 if (can_merge_right) {
937 vmg->end = next->vm_end;
938 vmg->vma = next;
939 vmg->pgoff = next->vm_pgoff -
940 }
941
942 /* If we can merge with the previous V
943 if (can_merge_left) {
944 vmg->start = prev->vm_start;
945 vmg->vma = prev;
946 vmg->pgoff = prev->vm_pgoff;
947
948 /*
949 * If this merge would result
950 * are not permitted to do so,
951 * prev and vma.
952 */
953 if (can_merge_right && !can_me
954 vmg->end = end;
955
956 /* In expand-only case we are
957 if (!just_expand) {
958 /* Equivalent to going
959 vma_prev(vmg->vmi);
960 }
961 }
962
963 /*
964 * Now try to expand adjacent VMA(s).
965 * following VMA if we have VMAs on bo
966 */
967 if (vmg->vma && !vma_expand(vmg)) {
968 khugepaged_enter_vma(vmg->vma,
969 vmg->state = VMA_MERGE_SUCCESS
970 return vmg->vma;
971 }
972
973 /* If expansion failed, reset state. A
974 if (!just_expand) {
975 vmg->vma = NULL;
976 vmg->start = start;
977 vmg->end = end;
978 vmg->pgoff = pgoff;
979 if (vmg->vma == prev)
980 vma_iter_set(vmg->vmi,
981 }
982
983 return NULL;
984 }
985
986 /*
987 * vma_expand - Expand an existing VMA
988 *
989 * @vmg: Describes a VMA expansion operation.
990 *
991 * Expand @vma to vmg->start and vmg->end. Ca
992 * Will expand over vmg->next if it's differen
993 * vmg->next->vm_end. Checking if the vmg->vm
994 * vmg->next needs to be handled by the caller
995 *
996 * Returns: 0 on success.
997 *
998 * ASSUMPTIONS:
999 * - The caller must hold a WRITE lock on vmg-
1000 * - The caller must have set @vmg->vma and @
1001 */
1002 int vma_expand(struct vma_merge_struct *vmg)
1003 {
1004 struct vm_area_struct *anon_dup = NUL
1005 bool remove_next = false;
1006 struct vm_area_struct *vma = vmg->vma
1007 struct vm_area_struct *next = vmg->ne
1008
1009 mmap_assert_write_locked(vmg->mm);
1010
1011 vma_start_write(vma);
1012 if (next && (vma != next) && (vmg->en
1013 int ret;
1014
1015 remove_next = true;
1016 /* This should already have b
1017 VM_WARN_ON(!can_merge_remove_
1018 vma_start_write(next);
1019 ret = dup_anon_vma(vma, next,
1020 if (ret)
1021 return ret;
1022 }
1023
1024 /* Not merging but overwriting any pa
1025 VM_WARN_ON(next && !remove_next &&
1026 next != vma && vmg->end > n
1027 /* Only handles expanding */
1028 VM_WARN_ON(vma->vm_start < vmg->start
1029
1030 if (commit_merge(vmg, NULL, remove_ne
1031 goto nomem;
1032
1033 return 0;
1034
1035 nomem:
1036 vmg->state = VMA_MERGE_ERROR_NOMEM;
1037 if (anon_dup)
1038 unlink_anon_vmas(anon_dup);
1039 return -ENOMEM;
1040 }
1041
1042 /*
1043 * vma_shrink() - Reduce an existing VMAs mem
1044 * @vmi: The vma iterator
1045 * @vma: The VMA to modify
1046 * @start: The new start
1047 * @end: The new end
1048 *
1049 * Returns: 0 on success, -ENOMEM otherwise
1050 */
1051 int vma_shrink(struct vma_iterator *vmi, stru
1052 unsigned long start, unsigned
1053 {
1054 struct vma_prepare vp;
1055
1056 WARN_ON((vma->vm_start != start) && (
1057
1058 if (vma->vm_start < start)
1059 vma_iter_config(vmi, vma->vm_
1060 else
1061 vma_iter_config(vmi, end, vma
1062
1063 if (vma_iter_prealloc(vmi, NULL))
1064 return -ENOMEM;
1065
1066 vma_start_write(vma);
1067
1068 init_vma_prep(&vp, vma);
1069 vma_prepare(&vp);
1070 vma_adjust_trans_huge(vma, start, end
1071
1072 vma_iter_clear(vmi);
1073 vma_set_range(vma, start, end, pgoff)
1074 vma_complete(&vp, vmi, vma->vm_mm);
1075 validate_mm(vma->vm_mm);
1076 return 0;
1077 }
1078
1079 static inline void vms_clear_ptes(struct vma_
1080 struct ma_state *mas_deta
1081 {
1082 struct mmu_gather tlb;
1083
1084 if (!vms->clear_ptes) /* Nothing to d
1085 return;
1086
1087 /*
1088 * We can free page tables without wr
1089 * were isolated before we downgraded
1090 */
1091 mas_set(mas_detach, 1);
1092 lru_add_drain();
1093 tlb_gather_mmu(&tlb, vms->vma->vm_mm)
1094 update_hiwater_rss(vms->vma->vm_mm);
1095 unmap_vmas(&tlb, mas_detach, vms->vma
1096 vms->vma_count, mm_wr_lock
1097
1098 mas_set(mas_detach, 1);
1099 /* start and end may be different if
1100 free_pgtables(&tlb, mas_detach, vms->
1101 vms->unmap_end, mm_wr_l
1102 tlb_finish_mmu(&tlb);
1103 vms->clear_ptes = false;
1104 }
1105
1106 void vms_clean_up_area(struct vma_munmap_stru
1107 struct ma_state *mas_detach)
1108 {
1109 struct vm_area_struct *vma;
1110
1111 if (!vms->nr_pages)
1112 return;
1113
1114 vms_clear_ptes(vms, mas_detach, true)
1115 mas_set(mas_detach, 0);
1116 mas_for_each(mas_detach, vma, ULONG_M
1117 vma_close(vma);
1118 }
1119
1120 /*
1121 * vms_complete_munmap_vmas() - Finish the mu
1122 * @vms: The vma munmap struct
1123 * @mas_detach: The maple state of the detach
1124 *
1125 * This updates the mm_struct, unmaps the reg
1126 * used for the munmap() and may downgrade th
1127 * needed to be done once the vma maple tree
1128 */
1129 void vms_complete_munmap_vmas(struct vma_munm
1130 struct ma_state *mas_detach)
1131 {
1132 struct vm_area_struct *vma;
1133 struct mm_struct *mm;
1134
1135 mm = current->mm;
1136 mm->map_count -= vms->vma_count;
1137 mm->locked_vm -= vms->locked_vm;
1138 if (vms->unlock)
1139 mmap_write_downgrade(mm);
1140
1141 if (!vms->nr_pages)
1142 return;
1143
1144 vms_clear_ptes(vms, mas_detach, !vms-
1145 /* Update high watermark before we lo
1146 update_hiwater_vm(mm);
1147 /* Stat accounting */
1148 WRITE_ONCE(mm->total_vm, READ_ONCE(mm
1149 /* Paranoid bookkeeping */
1150 VM_WARN_ON(vms->exec_vm > mm->exec_vm
1151 VM_WARN_ON(vms->stack_vm > mm->stack_
1152 VM_WARN_ON(vms->data_vm > mm->data_vm
1153 mm->exec_vm -= vms->exec_vm;
1154 mm->stack_vm -= vms->stack_vm;
1155 mm->data_vm -= vms->data_vm;
1156
1157 /* Remove and clean up vmas */
1158 mas_set(mas_detach, 0);
1159 mas_for_each(mas_detach, vma, ULONG_M
1160 remove_vma(vma, /* unreachabl
1161
1162 vm_unacct_memory(vms->nr_accounted);
1163 validate_mm(mm);
1164 if (vms->unlock)
1165 mmap_read_unlock(mm);
1166
1167 __mt_destroy(mas_detach->tree);
1168 }
1169
1170 /*
1171 * vms_gather_munmap_vmas() - Put all VMAs wi
1172 * for removal at a later date. Handles spli
1173 * and marking the vmas as isolated.
1174 *
1175 * @vms: The vma munmap struct
1176 * @mas_detach: The maple state tracking the
1177 *
1178 * Return: 0 on success, error otherwise
1179 */
1180 int vms_gather_munmap_vmas(struct vma_munmap_
1181 struct ma_state *mas_detach)
1182 {
1183 struct vm_area_struct *next = NULL;
1184 int error;
1185
1186 /*
1187 * If we need to split any vma, do it
1188 * Does it split the first one?
1189 */
1190 if (vms->start > vms->vma->vm_start)
1191
1192 /*
1193 * Make sure that map_count o
1194 * not exceed its limit; but
1195 * its limit temporarily, to
1196 */
1197 if (vms->end < vms->vma->vm_e
1198 vms->vma->vm_mm->map_coun
1199 error = -ENOMEM;
1200 goto map_count_exceed
1201 }
1202
1203 /* Don't bother splitting the
1204 if (!can_modify_vma(vms->vma)
1205 error = -EPERM;
1206 goto start_split_fail
1207 }
1208
1209 error = __split_vma(vms->vmi,
1210 if (error)
1211 goto start_split_fail
1212 }
1213 vms->prev = vma_prev(vms->vmi);
1214 if (vms->prev)
1215 vms->unmap_start = vms->prev-
1216
1217 /*
1218 * Detach a range of VMAs from the mm
1219 * it is always overwritten.
1220 */
1221 for_each_vma_range(*(vms->vmi), next,
1222 long nrpages;
1223
1224 if (!can_modify_vma(next)) {
1225 error = -EPERM;
1226 goto modify_vma_faile
1227 }
1228 /* Does it split the end? */
1229 if (next->vm_end > vms->end)
1230 error = __split_vma(v
1231 if (error)
1232 goto end_spli
1233 }
1234 vma_start_write(next);
1235 mas_set(mas_detach, vms->vma_
1236 error = mas_store_gfp(mas_det
1237 if (error)
1238 goto munmap_gather_fa
1239
1240 vma_mark_detached(next, true)
1241 nrpages = vma_pages(next);
1242
1243 vms->nr_pages += nrpages;
1244 if (next->vm_flags & VM_LOCKE
1245 vms->locked_vm += nrp
1246
1247 if (next->vm_flags & VM_ACCOU
1248 vms->nr_accounted +=
1249
1250 if (is_exec_mapping(next->vm_
1251 vms->exec_vm += nrpag
1252 else if (is_stack_mapping(nex
1253 vms->stack_vm += nrpa
1254 else if (is_data_mapping(next
1255 vms->data_vm += nrpag
1256
1257 if (unlikely(vms->uf)) {
1258 /*
1259 * If userfaultfd_unm
1260 * will remain split,
1261 * highly unexpected
1262 * different than the
1263 * __split_vma fails,
1264 * split, despite we
1265 * failure that it's
1266 */
1267 error = userfaultfd_u
1268
1269 if (error)
1270 goto userfaul
1271 }
1272 #ifdef CONFIG_DEBUG_VM_MAPLE_TREE
1273 BUG_ON(next->vm_start < vms->
1274 BUG_ON(next->vm_start > vms->
1275 #endif
1276 }
1277
1278 vms->next = vma_next(vms->vmi);
1279 if (vms->next)
1280 vms->unmap_end = vms->next->v
1281
1282 #if defined(CONFIG_DEBUG_VM_MAPLE_TREE)
1283 /* Make sure no VMAs are about to be
1284 {
1285 MA_STATE(test, mas_detach->tr
1286 struct vm_area_struct *vma_ma
1287 int test_count = 0;
1288
1289 vma_iter_set(vms->vmi, vms->s
1290 rcu_read_lock();
1291 vma_test = mas_find(&test, vm
1292 for_each_vma_range(*(vms->vmi
1293 BUG_ON(vma_mas != vma
1294 test_count++;
1295 vma_test = mas_next(&
1296 }
1297 rcu_read_unlock();
1298 BUG_ON(vms->vma_count != test
1299 }
1300 #endif
1301
1302 while (vma_iter_addr(vms->vmi) > vms-
1303 vma_iter_prev_range(vms->vmi)
1304
1305 vms->clear_ptes = true;
1306 return 0;
1307
1308 userfaultfd_error:
1309 munmap_gather_failed:
1310 end_split_failed:
1311 modify_vma_failed:
1312 reattach_vmas(mas_detach);
1313 start_split_failed:
1314 map_count_exceeded:
1315 return error;
1316 }
1317
1318 /*
1319 * do_vmi_align_munmap() - munmap the aligned
1320 * @vmi: The vma iterator
1321 * @vma: The starting vm_area_struct
1322 * @mm: The mm_struct
1323 * @start: The aligned start address to munma
1324 * @end: The aligned end address to munmap.
1325 * @uf: The userfaultfd list_head
1326 * @unlock: Set to true to drop the mmap_lock
1327 * success.
1328 *
1329 * Return: 0 on success and drops the lock if
1330 * lock held otherwise.
1331 */
1332 int do_vmi_align_munmap(struct vma_iterator *
1333 struct mm_struct *mm, unsigne
1334 struct list_head *uf, bool un
1335 {
1336 struct maple_tree mt_detach;
1337 MA_STATE(mas_detach, &mt_detach, 0, 0
1338 mt_init_flags(&mt_detach, vmi->mas.tr
1339 mt_on_stack(mt_detach);
1340 struct vma_munmap_struct vms;
1341 int error;
1342
1343 init_vma_munmap(&vms, vmi, vma, start
1344 error = vms_gather_munmap_vmas(&vms,
1345 if (error)
1346 goto gather_failed;
1347
1348 error = vma_iter_clear_gfp(vmi, start
1349 if (error)
1350 goto clear_tree_failed;
1351
1352 /* Point of no return */
1353 vms_complete_munmap_vmas(&vms, &mas_d
1354 return 0;
1355
1356 clear_tree_failed:
1357 reattach_vmas(&mas_detach);
1358 gather_failed:
1359 validate_mm(mm);
1360 return error;
1361 }
1362
1363 /*
1364 * do_vmi_munmap() - munmap a given range.
1365 * @vmi: The vma iterator
1366 * @mm: The mm_struct
1367 * @start: The start address to munmap
1368 * @len: The length of the range to munmap
1369 * @uf: The userfaultfd list_head
1370 * @unlock: set to true if the user wants to
1371 *
1372 * This function takes a @mas that is either
1373 * to MA_START and sets it up to remove the m
1374 * aligned.
1375 *
1376 * Return: 0 on success and drops the lock if
1377 * lock held otherwise.
1378 */
1379 int do_vmi_munmap(struct vma_iterator *vmi, s
1380 unsigned long start, size_t
1381 bool unlock)
1382 {
1383 unsigned long end;
1384 struct vm_area_struct *vma;
1385
1386 if ((offset_in_page(start)) || start
1387 return -EINVAL;
1388
1389 end = start + PAGE_ALIGN(len);
1390 if (end == start)
1391 return -EINVAL;
1392
1393 /* Find the first overlapping VMA */
1394 vma = vma_find(vmi, end);
1395 if (!vma) {
1396 if (unlock)
1397 mmap_write_unlock(mm)
1398 return 0;
1399 }
1400
1401 return do_vmi_align_munmap(vmi, vma,
1402 }
1403
1404 /*
1405 * We are about to modify one or multiple of
1406 * context and anonymous VMA name within the
1407 *
1408 * As a result, we might be able to merge the
1409 * adjacent VMA with identical properties.
1410 *
1411 * If no merge is possible and the range does
1412 * we then need to split the VMA to accommoda
1413 *
1414 * The function returns either the merged VMA
1415 * required instead, or an error if the split
1416 */
1417 static struct vm_area_struct *vma_modify(stru
1418 {
1419 struct vm_area_struct *vma = vmg->vma
1420 struct vm_area_struct *merged;
1421
1422 /* First, try to merge. */
1423 merged = vma_merge_existing_range(vmg
1424 if (merged)
1425 return merged;
1426
1427 /* Split any preceding portion of the
1428 if (vma->vm_start < vmg->start) {
1429 int err = split_vma(vmg->vmi,
1430
1431 if (err)
1432 return ERR_PTR(err);
1433 }
1434
1435 /* Split any trailing portion of the
1436 if (vma->vm_end > vmg->end) {
1437 int err = split_vma(vmg->vmi,
1438
1439 if (err)
1440 return ERR_PTR(err);
1441 }
1442
1443 return vma;
1444 }
1445
1446 struct vm_area_struct *vma_modify_flags(
1447 struct vma_iterator *vmi, struct vm_a
1448 struct vm_area_struct *vma, unsigned
1449 unsigned long new_flags)
1450 {
1451 VMG_VMA_STATE(vmg, vmi, prev, vma, st
1452
1453 vmg.flags = new_flags;
1454
1455 return vma_modify(&vmg);
1456 }
1457
1458 struct vm_area_struct
1459 *vma_modify_flags_name(struct vma_iterator *v
1460 struct vm_area_struct
1461 struct vm_area_struct
1462 unsigned long start,
1463 unsigned long end,
1464 unsigned long new_flag
1465 struct anon_vma_name *
1466 {
1467 VMG_VMA_STATE(vmg, vmi, prev, vma, st
1468
1469 vmg.flags = new_flags;
1470 vmg.anon_name = new_name;
1471
1472 return vma_modify(&vmg);
1473 }
1474
1475 struct vm_area_struct
1476 *vma_modify_policy(struct vma_iterator *vmi,
1477 struct vm_area_struct *pre
1478 struct vm_area_struct *vma
1479 unsigned long start, unsig
1480 struct mempolicy *new_pol)
1481 {
1482 VMG_VMA_STATE(vmg, vmi, prev, vma, st
1483
1484 vmg.policy = new_pol;
1485
1486 return vma_modify(&vmg);
1487 }
1488
1489 struct vm_area_struct
1490 *vma_modify_flags_uffd(struct vma_iterator *v
1491 struct vm_area_struct
1492 struct vm_area_struct
1493 unsigned long start, u
1494 unsigned long new_flag
1495 struct vm_userfaultfd_
1496 {
1497 VMG_VMA_STATE(vmg, vmi, prev, vma, st
1498
1499 vmg.flags = new_flags;
1500 vmg.uffd_ctx = new_ctx;
1501
1502 return vma_modify(&vmg);
1503 }
1504
1505 /*
1506 * Expand vma by delta bytes, potentially mer
1507 * VMA with identical properties.
1508 */
1509 struct vm_area_struct *vma_merge_extend(struc
1510 struc
1511 unsig
1512 {
1513 VMG_VMA_STATE(vmg, vmi, vma, vma, vma
1514
1515 vmg.next = vma_iter_next_rewind(vmi,
1516 vmg.vma = NULL; /* We use the VMA to
1517
1518 return vma_merge_new_range(&vmg);
1519 }
1520
1521 void unlink_file_vma_batch_init(struct unlink
1522 {
1523 vb->count = 0;
1524 }
1525
1526 static void unlink_file_vma_batch_process(str
1527 {
1528 struct address_space *mapping;
1529 int i;
1530
1531 mapping = vb->vmas[0]->vm_file->f_map
1532 i_mmap_lock_write(mapping);
1533 for (i = 0; i < vb->count; i++) {
1534 VM_WARN_ON_ONCE(vb->vmas[i]->
1535 __remove_shared_vm_struct(vb-
1536 }
1537 i_mmap_unlock_write(mapping);
1538
1539 unlink_file_vma_batch_init(vb);
1540 }
1541
1542 void unlink_file_vma_batch_add(struct unlink_
1543 struct vm_area
1544 {
1545 if (vma->vm_file == NULL)
1546 return;
1547
1548 if ((vb->count > 0 && vb->vmas[0]->vm
1549 vb->count == ARRAY_SIZE(vb->vmas)
1550 unlink_file_vma_batch_process
1551
1552 vb->vmas[vb->count] = vma;
1553 vb->count++;
1554 }
1555
1556 void unlink_file_vma_batch_final(struct unlin
1557 {
1558 if (vb->count > 0)
1559 unlink_file_vma_batch_process
1560 }
1561
1562 /*
1563 * Unlink a file-based vm structure from its
1564 * vma from rmap and vmtruncate before freein
1565 */
1566 void unlink_file_vma(struct vm_area_struct *v
1567 {
1568 struct file *file = vma->vm_file;
1569
1570 if (file) {
1571 struct address_space *mapping
1572
1573 i_mmap_lock_write(mapping);
1574 __remove_shared_vm_struct(vma
1575 i_mmap_unlock_write(mapping);
1576 }
1577 }
1578
1579 void vma_link_file(struct vm_area_struct *vma
1580 {
1581 struct file *file = vma->vm_file;
1582 struct address_space *mapping;
1583
1584 if (file) {
1585 mapping = file->f_mapping;
1586 i_mmap_lock_write(mapping);
1587 __vma_link_file(vma, mapping)
1588 i_mmap_unlock_write(mapping);
1589 }
1590 }
1591
1592 int vma_link(struct mm_struct *mm, struct vm_
1593 {
1594 VMA_ITERATOR(vmi, mm, 0);
1595
1596 vma_iter_config(&vmi, vma->vm_start,
1597 if (vma_iter_prealloc(&vmi, vma))
1598 return -ENOMEM;
1599
1600 vma_start_write(vma);
1601 vma_iter_store(&vmi, vma);
1602 vma_link_file(vma);
1603 mm->map_count++;
1604 validate_mm(mm);
1605 return 0;
1606 }
1607
1608 /*
1609 * Copy the vma structure to a new location i
1610 * prior to moving page table entries, to eff
1611 */
1612 struct vm_area_struct *copy_vma(struct vm_are
1613 unsigned long addr, unsigned long len
1614 bool *need_rmap_locks)
1615 {
1616 struct vm_area_struct *vma = *vmap;
1617 unsigned long vma_start = vma->vm_sta
1618 struct mm_struct *mm = vma->vm_mm;
1619 struct vm_area_struct *new_vma;
1620 bool faulted_in_anon_vma = true;
1621 VMA_ITERATOR(vmi, mm, addr);
1622 VMG_VMA_STATE(vmg, &vmi, NULL, vma, a
1623
1624 /*
1625 * If anonymous vma has not yet been
1626 * to match new location, to increase
1627 */
1628 if (unlikely(vma_is_anonymous(vma) &&
1629 pgoff = addr >> PAGE_SHIFT;
1630 faulted_in_anon_vma = false;
1631 }
1632
1633 new_vma = find_vma_prev(mm, addr, &vm
1634 if (new_vma && new_vma->vm_start < ad
1635 return NULL; /* should nev
1636
1637 vmg.vma = NULL; /* New VMA range. */
1638 vmg.pgoff = pgoff;
1639 vmg.next = vma_iter_next_rewind(&vmi,
1640 new_vma = vma_merge_new_range(&vmg);
1641
1642 if (new_vma) {
1643 /*
1644 * Source vma may have been m
1645 */
1646 if (unlikely(vma_start >= new
1647 vma_start < new_
1648 /*
1649 * The only way we ca
1650 * self during an mre
1651 * been faulted in ye
1652 * reset the dst vma-
1653 * destination addres
1654 * the merge to happe
1655 * vm_pgoff linearity
1656 * (in turn preventin
1657 * safe. It is only s
1658 * linear if there ar
1659 */
1660 VM_BUG_ON_VMA(faulted
1661 *vmap = vma = new_vma
1662 }
1663 *need_rmap_locks = (new_vma->
1664 } else {
1665 new_vma = vm_area_dup(vma);
1666 if (!new_vma)
1667 goto out;
1668 vma_set_range(new_vma, addr,
1669 if (vma_dup_policy(vma, new_v
1670 goto out_free_vma;
1671 if (anon_vma_clone(new_vma, v
1672 goto out_free_mempol;
1673 if (new_vma->vm_file)
1674 get_file(new_vma->vm_
1675 if (new_vma->vm_ops && new_vm
1676 new_vma->vm_ops->open
1677 if (vma_link(mm, new_vma))
1678 goto out_vma_link;
1679 *need_rmap_locks = false;
1680 }
1681 return new_vma;
1682
1683 out_vma_link:
1684 vma_close(new_vma);
1685
1686 if (new_vma->vm_file)
1687 fput(new_vma->vm_file);
1688
1689 unlink_anon_vmas(new_vma);
1690 out_free_mempol:
1691 mpol_put(vma_policy(new_vma));
1692 out_free_vma:
1693 vm_area_free(new_vma);
1694 out:
1695 return NULL;
1696 }
1697
1698 /*
1699 * Rough compatibility check to quickly see i
1700 * at sharing an anon_vma.
1701 *
1702 * They need to have the same vm_file, and th
1703 * in things that mprotect may change.
1704 *
1705 * NOTE! The fact that we share an anon_vma d
1706 * we can merge the two vma's. For example, w
1707 * there is a vm_ops->close() function, becau
1708 * driver is doing some kind of reference cou
1709 * really matter for the anon_vma sharing cas
1710 */
1711 static int anon_vma_compatible(struct vm_area
1712 {
1713 return a->vm_end == b->vm_start &&
1714 mpol_equal(vma_policy(a), vma
1715 a->vm_file == b->vm_file &&
1716 !((a->vm_flags ^ b->vm_flags)
1717 b->vm_pgoff == a->vm_pgoff +
1718 }
1719
1720 /*
1721 * Do some basic sanity checking to see if we
1722 * from 'old'. The 'a'/'b' vma's are in VM or
1723 * the same as 'old', the other will be the n
1724 * to share the anon_vma.
1725 *
1726 * NOTE! This runs with mmap_lock held for re
1727 * the anon_vma of 'old' is concurrently in t
1728 * by another page fault trying to merge _tha
1729 * is being set up, that automatically means
1730 * acceptable for merging, so we can do all o
1731 * we do that READ_ONCE() to make sure that w
1732 *
1733 * IOW: that the "list_is_singular()" test on
1734 * matters for the 'stable anon_vma' case (ie
1735 * is to return an anon_vma that is "complex"
1736 * a fork).
1737 *
1738 * We also make sure that the two vma's are c
1739 * and with the same memory policies). That's
1740 * a read lock on the mmap_lock.
1741 */
1742 static struct anon_vma *reusable_anon_vma(str
1743 str
1744 str
1745 {
1746 if (anon_vma_compatible(a, b)) {
1747 struct anon_vma *anon_vma = R
1748
1749 if (anon_vma && list_is_singu
1750 return anon_vma;
1751 }
1752 return NULL;
1753 }
1754
1755 /*
1756 * find_mergeable_anon_vma is used by anon_vm
1757 * neighbouring vmas for a suitable anon_vma,
1758 * to allocate a new anon_vma. It checks bec
1759 * sequence of mprotects and faults may other
1760 * anon_vmas being allocated, preventing vma
1761 * mprotect.
1762 */
1763 struct anon_vma *find_mergeable_anon_vma(stru
1764 {
1765 struct anon_vma *anon_vma = NULL;
1766 struct vm_area_struct *prev, *next;
1767 VMA_ITERATOR(vmi, vma->vm_mm, vma->vm
1768
1769 /* Try next first. */
1770 next = vma_iter_load(&vmi);
1771 if (next) {
1772 anon_vma = reusable_anon_vma(
1773 if (anon_vma)
1774 return anon_vma;
1775 }
1776
1777 prev = vma_prev(&vmi);
1778 VM_BUG_ON_VMA(prev != vma, vma);
1779 prev = vma_prev(&vmi);
1780 /* Try prev next. */
1781 if (prev)
1782 anon_vma = reusable_anon_vma(
1783
1784 /*
1785 * We might reach here with anon_vma
1786 * any reusable anon_vma.
1787 * There's no absolute need to look o
1788 * we could search further afield for
1789 * But it would probably just be a wa
1790 * or lead to too many vmas hanging o
1791 * We're trying to allow mprotect rem
1792 * not trying to minimize memory used
1793 */
1794 return anon_vma;
1795 }
1796
1797 static bool vm_ops_needs_writenotify(const st
1798 {
1799 return vm_ops && (vm_ops->page_mkwrit
1800 }
1801
1802 static bool vma_is_shared_writable(struct vm_
1803 {
1804 return (vma->vm_flags & (VM_WRITE | V
1805 (VM_WRITE | VM_SHARED);
1806 }
1807
1808 static bool vma_fs_can_writeback(struct vm_ar
1809 {
1810 /* No managed pages to writeback. */
1811 if (vma->vm_flags & VM_PFNMAP)
1812 return false;
1813
1814 return vma->vm_file && vma->vm_file->
1815 mapping_can_writeback(vma->vm
1816 }
1817
1818 /*
1819 * Does this VMA require the underlying folio
1820 * tracked?
1821 */
1822 bool vma_needs_dirty_tracking(struct vm_area_
1823 {
1824 /* Only shared, writable VMAs require
1825 if (!vma_is_shared_writable(vma))
1826 return false;
1827
1828 /* Does the filesystem need to be not
1829 if (vm_ops_needs_writenotify(vma->vm_
1830 return true;
1831
1832 /*
1833 * Even if the filesystem doesn't ind
1834 * can writeback, dirty tracking is s
1835 */
1836 return vma_fs_can_writeback(vma);
1837 }
1838
1839 /*
1840 * Some shared mappings will want the pages m
1841 * to track write events. If so, we'll downgr
1842 * to the private version (using protection_m
1843 * VM_SHARED bit).
1844 */
1845 bool vma_wants_writenotify(struct vm_area_str
1846 {
1847 /* If it was private or non-writable,
1848 if (!vma_is_shared_writable(vma))
1849 return false;
1850
1851 /* The backer wishes to know when pag
1852 if (vm_ops_needs_writenotify(vma->vm_
1853 return true;
1854
1855 /* The open routine did something to
1856 * won't preserve? */
1857 if (pgprot_val(vm_page_prot) !=
1858 pgprot_val(vm_pgprot_modify(vm_pa
1859 return false;
1860
1861 /*
1862 * Do we need to track softdirty? hug
1863 * tracking yet.
1864 */
1865 if (vma_soft_dirty_enabled(vma) && !i
1866 return true;
1867
1868 /* Do we need write faults for uffd-w
1869 if (userfaultfd_wp(vma))
1870 return true;
1871
1872 /* Can the mapping track the dirty pa
1873 return vma_fs_can_writeback(vma);
1874 }
1875
1876 static DEFINE_MUTEX(mm_all_locks_mutex);
1877
1878 static void vm_lock_anon_vma(struct mm_struct
1879 {
1880 if (!test_bit(0, (unsigned long *) &a
1881 /*
1882 * The LSB of head.next can't
1883 * because we hold the mm_all
1884 */
1885 down_write_nest_lock(&anon_vm
1886 /*
1887 * We can safely modify head.
1888 * anon_vma->root->rwsem. If
1889 * the same anon_vma we won't
1890 *
1891 * No need of atomic instruct
1892 * can't change from under us
1893 * anon_vma->root->rwsem.
1894 */
1895 if (__test_and_set_bit(0, (un
1896 &anon_
1897 BUG();
1898 }
1899 }
1900
1901 static void vm_lock_mapping(struct mm_struct
1902 {
1903 if (!test_bit(AS_MM_ALL_LOCKS, &mappi
1904 /*
1905 * AS_MM_ALL_LOCKS can't chan
1906 * we hold the mm_all_locks_m
1907 *
1908 * Operations on ->flags have
1909 * even if AS_MM_ALL_LOCKS is
1910 * mm_all_locks_mutex, there
1911 * changing other bitflags in
1912 */
1913 if (test_and_set_bit(AS_MM_AL
1914 BUG();
1915 down_write_nest_lock(&mapping
1916 }
1917 }
1918
1919 /*
1920 * This operation locks against the VM for al
1921 * operations that could ever happen on a cer
1922 * vmtruncate, try_to_unmap, and all page fau
1923 *
1924 * The caller must take the mmap_lock in writ
1925 * mm_take_all_locks(). The caller isn't allo
1926 * mmap_lock until mm_drop_all_locks() return
1927 *
1928 * mmap_lock in write mode is required in ord
1929 * that could modify pagetables and free page
1930 * altering the vma layout. It's also needed
1931 * anon_vmas to be associated with existing v
1932 *
1933 * A single task can't take more than one mm_
1934 * or it would deadlock.
1935 *
1936 * The LSB in anon_vma->rb_root.rb_node and t
1937 * mapping->flags avoid to take the same lock
1938 * vma in this mm is backed by the same anon_
1939 *
1940 * We take locks in following order, accordin
1941 * of mm/rmap.c:
1942 * - all hugetlbfs_i_mmap_rwsem_key locks (
1943 * hugetlb mapping);
1944 * - all vmas marked locked
1945 * - all i_mmap_rwsem locks;
1946 * - all anon_vma->rwseml
1947 *
1948 * We can take all locks within these types r
1949 * doesn't nest them and we protected from pa
1950 * mm_all_locks_mutex.
1951 *
1952 * mm_take_all_locks() and mm_drop_all_locks
1953 * that may have to take thousand of locks.
1954 *
1955 * mm_take_all_locks() can fail if it's inter
1956 */
1957 int mm_take_all_locks(struct mm_struct *mm)
1958 {
1959 struct vm_area_struct *vma;
1960 struct anon_vma_chain *avc;
1961 VMA_ITERATOR(vmi, mm, 0);
1962
1963 mmap_assert_write_locked(mm);
1964
1965 mutex_lock(&mm_all_locks_mutex);
1966
1967 /*
1968 * vma_start_write() does not have a
1969 * because vma_start_write() is alway
1970 * being written to until mmap_write_
1971 * is reached.
1972 */
1973 for_each_vma(vmi, vma) {
1974 if (signal_pending(current))
1975 goto out_unlock;
1976 vma_start_write(vma);
1977 }
1978
1979 vma_iter_init(&vmi, mm, 0);
1980 for_each_vma(vmi, vma) {
1981 if (signal_pending(current))
1982 goto out_unlock;
1983 if (vma->vm_file && vma->vm_f
1984 is_vm_hugetlb
1985 vm_lock_mapping(mm, v
1986 }
1987
1988 vma_iter_init(&vmi, mm, 0);
1989 for_each_vma(vmi, vma) {
1990 if (signal_pending(current))
1991 goto out_unlock;
1992 if (vma->vm_file && vma->vm_f
1993 !is_vm_hugetl
1994 vm_lock_mapping(mm, v
1995 }
1996
1997 vma_iter_init(&vmi, mm, 0);
1998 for_each_vma(vmi, vma) {
1999 if (signal_pending(current))
2000 goto out_unlock;
2001 if (vma->anon_vma)
2002 list_for_each_entry(a
2003 vm_lock_anon_
2004 }
2005
2006 return 0;
2007
2008 out_unlock:
2009 mm_drop_all_locks(mm);
2010 return -EINTR;
2011 }
2012
2013 static void vm_unlock_anon_vma(struct anon_vm
2014 {
2015 if (test_bit(0, (unsigned long *) &an
2016 /*
2017 * The LSB of head.next can't
2018 * us because we hold the mm_
2019 *
2020 * We must however clear the
2021 * the vma so the users using
2022 * never see our bitflag.
2023 *
2024 * No need of atomic instruct
2025 * can't change from under us
2026 * anon_vma->root->rwsem.
2027 */
2028 if (!__test_and_clear_bit(0,
2029 &an
2030 BUG();
2031 anon_vma_unlock_write(anon_vm
2032 }
2033 }
2034
2035 static void vm_unlock_mapping(struct address_
2036 {
2037 if (test_bit(AS_MM_ALL_LOCKS, &mappin
2038 /*
2039 * AS_MM_ALL_LOCKS can't chan
2040 * because we hold the mm_all
2041 */
2042 i_mmap_unlock_write(mapping);
2043 if (!test_and_clear_bit(AS_MM
2044 &mapp
2045 BUG();
2046 }
2047 }
2048
2049 /*
2050 * The mmap_lock cannot be released by the ca
2051 * mm_drop_all_locks() returns.
2052 */
2053 void mm_drop_all_locks(struct mm_struct *mm)
2054 {
2055 struct vm_area_struct *vma;
2056 struct anon_vma_chain *avc;
2057 VMA_ITERATOR(vmi, mm, 0);
2058
2059 mmap_assert_write_locked(mm);
2060 BUG_ON(!mutex_is_locked(&mm_all_locks
2061
2062 for_each_vma(vmi, vma) {
2063 if (vma->anon_vma)
2064 list_for_each_entry(a
2065 vm_unlock_ano
2066 if (vma->vm_file && vma->vm_f
2067 vm_unlock_mapping(vma
2068 }
2069
2070 mutex_unlock(&mm_all_locks_mutex);
2071 }
2072
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.