1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * linux/mm/madvise.c
4 *
5 * Copyright (C) 1999 Linus Torvalds
6 * Copyright (C) 2002 Christoph Hellwig
7 */
8
9 #include <linux/mman.h>
10 #include <linux/pagemap.h>
11 #include <linux/syscalls.h>
12 #include <linux/mempolicy.h>
13 #include <linux/page-isolation.h>
14 #include <linux/page_idle.h>
15 #include <linux/userfaultfd_k.h>
16 #include <linux/hugetlb.h>
17 #include <linux/falloc.h>
18 #include <linux/fadvise.h>
19 #include <linux/sched.h>
20 #include <linux/sched/mm.h>
21 #include <linux/mm_inline.h>
22 #include <linux/string.h>
23 #include <linux/uio.h>
24 #include <linux/ksm.h>
25 #include <linux/fs.h>
26 #include <linux/file.h>
27 #include <linux/blkdev.h>
28 #include <linux/backing-dev.h>
29 #include <linux/pagewalk.h>
30 #include <linux/swap.h>
31 #include <linux/swapops.h>
32 #include <linux/shmem_fs.h>
33 #include <linux/mmu_notifier.h>
34
35 #include <asm/tlb.h>
36
37 #include "internal.h"
38 #include "swap.h"
39
40 struct madvise_walk_private {
41 struct mmu_gather *tlb;
42 bool pageout;
43 };
44
45 /*
46 * Any behaviour which results in changes to the vma->vm_flags needs to
47 * take mmap_lock for writing. Others, which simply traverse vmas, need
48 * to only take it for reading.
49 */
50 static int madvise_need_mmap_write(int behavior)
51 {
52 switch (behavior) {
53 case MADV_REMOVE:
54 case MADV_WILLNEED:
55 case MADV_DONTNEED:
56 case MADV_DONTNEED_LOCKED:
57 case MADV_COLD:
58 case MADV_PAGEOUT:
59 case MADV_FREE:
60 case MADV_POPULATE_READ:
61 case MADV_POPULATE_WRITE:
62 case MADV_COLLAPSE:
63 return 0;
64 default:
65 /* be safe, default to 1. list exceptions explicitly */
66 return 1;
67 }
68 }
69
70 #ifdef CONFIG_ANON_VMA_NAME
71 struct anon_vma_name *anon_vma_name_alloc(const char *name)
72 {
73 struct anon_vma_name *anon_name;
74 size_t count;
75
76 /* Add 1 for NUL terminator at the end of the anon_name->name */
77 count = strlen(name) + 1;
78 anon_name = kmalloc(struct_size(anon_name, name, count), GFP_KERNEL);
79 if (anon_name) {
80 kref_init(&anon_name->kref);
81 memcpy(anon_name->name, name, count);
82 }
83
84 return anon_name;
85 }
86
87 void anon_vma_name_free(struct kref *kref)
88 {
89 struct anon_vma_name *anon_name =
90 container_of(kref, struct anon_vma_name, kref);
91 kfree(anon_name);
92 }
93
94 struct anon_vma_name *anon_vma_name(struct vm_area_struct *vma)
95 {
96 mmap_assert_locked(vma->vm_mm);
97
98 return vma->anon_name;
99 }
100
101 /* mmap_lock should be write-locked */
102 static int replace_anon_vma_name(struct vm_area_struct *vma,
103 struct anon_vma_name *anon_name)
104 {
105 struct anon_vma_name *orig_name = anon_vma_name(vma);
106
107 if (!anon_name) {
108 vma->anon_name = NULL;
109 anon_vma_name_put(orig_name);
110 return 0;
111 }
112
113 if (anon_vma_name_eq(orig_name, anon_name))
114 return 0;
115
116 vma->anon_name = anon_vma_name_reuse(anon_name);
117 anon_vma_name_put(orig_name);
118
119 return 0;
120 }
121 #else /* CONFIG_ANON_VMA_NAME */
122 static int replace_anon_vma_name(struct vm_area_struct *vma,
123 struct anon_vma_name *anon_name)
124 {
125 if (anon_name)
126 return -EINVAL;
127
128 return 0;
129 }
130 #endif /* CONFIG_ANON_VMA_NAME */
131 /*
132 * Update the vm_flags on region of a vma, splitting it or merging it as
133 * necessary. Must be called with mmap_lock held for writing;
134 * Caller should ensure anon_name stability by raising its refcount even when
135 * anon_name belongs to a valid vma because this function might free that vma.
136 */
137 static int madvise_update_vma(struct vm_area_struct *vma,
138 struct vm_area_struct **prev, unsigned long start,
139 unsigned long end, unsigned long new_flags,
140 struct anon_vma_name *anon_name)
141 {
142 struct mm_struct *mm = vma->vm_mm;
143 int error;
144 VMA_ITERATOR(vmi, mm, start);
145
146 if (new_flags == vma->vm_flags && anon_vma_name_eq(anon_vma_name(vma), anon_name)) {
147 *prev = vma;
148 return 0;
149 }
150
151 vma = vma_modify_flags_name(&vmi, *prev, vma, start, end, new_flags,
152 anon_name);
153 if (IS_ERR(vma))
154 return PTR_ERR(vma);
155
156 *prev = vma;
157
158 /* vm_flags is protected by the mmap_lock held in write mode. */
159 vma_start_write(vma);
160 vm_flags_reset(vma, new_flags);
161 if (!vma->vm_file || vma_is_anon_shmem(vma)) {
162 error = replace_anon_vma_name(vma, anon_name);
163 if (error)
164 return error;
165 }
166
167 return 0;
168 }
169
170 #ifdef CONFIG_SWAP
171 static int swapin_walk_pmd_entry(pmd_t *pmd, unsigned long start,
172 unsigned long end, struct mm_walk *walk)
173 {
174 struct vm_area_struct *vma = walk->private;
175 struct swap_iocb *splug = NULL;
176 pte_t *ptep = NULL;
177 spinlock_t *ptl;
178 unsigned long addr;
179
180 for (addr = start; addr < end; addr += PAGE_SIZE) {
181 pte_t pte;
182 swp_entry_t entry;
183 struct folio *folio;
184
185 if (!ptep++) {
186 ptep = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
187 if (!ptep)
188 break;
189 }
190
191 pte = ptep_get(ptep);
192 if (!is_swap_pte(pte))
193 continue;
194 entry = pte_to_swp_entry(pte);
195 if (unlikely(non_swap_entry(entry)))
196 continue;
197
198 pte_unmap_unlock(ptep, ptl);
199 ptep = NULL;
200
201 folio = read_swap_cache_async(entry, GFP_HIGHUSER_MOVABLE,
202 vma, addr, &splug);
203 if (folio)
204 folio_put(folio);
205 }
206
207 if (ptep)
208 pte_unmap_unlock(ptep, ptl);
209 swap_read_unplug(splug);
210 cond_resched();
211
212 return 0;
213 }
214
215 static const struct mm_walk_ops swapin_walk_ops = {
216 .pmd_entry = swapin_walk_pmd_entry,
217 .walk_lock = PGWALK_RDLOCK,
218 };
219
220 static void shmem_swapin_range(struct vm_area_struct *vma,
221 unsigned long start, unsigned long end,
222 struct address_space *mapping)
223 {
224 XA_STATE(xas, &mapping->i_pages, linear_page_index(vma, start));
225 pgoff_t end_index = linear_page_index(vma, end) - 1;
226 struct folio *folio;
227 struct swap_iocb *splug = NULL;
228
229 rcu_read_lock();
230 xas_for_each(&xas, folio, end_index) {
231 unsigned long addr;
232 swp_entry_t entry;
233
234 if (!xa_is_value(folio))
235 continue;
236 entry = radix_to_swp_entry(folio);
237 /* There might be swapin error entries in shmem mapping. */
238 if (non_swap_entry(entry))
239 continue;
240
241 addr = vma->vm_start +
242 ((xas.xa_index - vma->vm_pgoff) << PAGE_SHIFT);
243 xas_pause(&xas);
244 rcu_read_unlock();
245
246 folio = read_swap_cache_async(entry, mapping_gfp_mask(mapping),
247 vma, addr, &splug);
248 if (folio)
249 folio_put(folio);
250
251 rcu_read_lock();
252 }
253 rcu_read_unlock();
254 swap_read_unplug(splug);
255 }
256 #endif /* CONFIG_SWAP */
257
258 /*
259 * Schedule all required I/O operations. Do not wait for completion.
260 */
261 static long madvise_willneed(struct vm_area_struct *vma,
262 struct vm_area_struct **prev,
263 unsigned long start, unsigned long end)
264 {
265 struct mm_struct *mm = vma->vm_mm;
266 struct file *file = vma->vm_file;
267 loff_t offset;
268
269 *prev = vma;
270 #ifdef CONFIG_SWAP
271 if (!file) {
272 walk_page_range(vma->vm_mm, start, end, &swapin_walk_ops, vma);
273 lru_add_drain(); /* Push any new pages onto the LRU now */
274 return 0;
275 }
276
277 if (shmem_mapping(file->f_mapping)) {
278 shmem_swapin_range(vma, start, end, file->f_mapping);
279 lru_add_drain(); /* Push any new pages onto the LRU now */
280 return 0;
281 }
282 #else
283 if (!file)
284 return -EBADF;
285 #endif
286
287 if (IS_DAX(file_inode(file))) {
288 /* no bad return value, but ignore advice */
289 return 0;
290 }
291
292 /*
293 * Filesystem's fadvise may need to take various locks. We need to
294 * explicitly grab a reference because the vma (and hence the
295 * vma's reference to the file) can go away as soon as we drop
296 * mmap_lock.
297 */
298 *prev = NULL; /* tell sys_madvise we drop mmap_lock */
299 get_file(file);
300 offset = (loff_t)(start - vma->vm_start)
301 + ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
302 mmap_read_unlock(mm);
303 vfs_fadvise(file, offset, end - start, POSIX_FADV_WILLNEED);
304 fput(file);
305 mmap_read_lock(mm);
306 return 0;
307 }
308
309 static inline bool can_do_file_pageout(struct vm_area_struct *vma)
310 {
311 if (!vma->vm_file)
312 return false;
313 /*
314 * paging out pagecache only for non-anonymous mappings that correspond
315 * to the files the calling process could (if tried) open for writing;
316 * otherwise we'd be including shared non-exclusive mappings, which
317 * opens a side channel.
318 */
319 return inode_owner_or_capable(&nop_mnt_idmap,
320 file_inode(vma->vm_file)) ||
321 file_permission(vma->vm_file, MAY_WRITE) == 0;
322 }
323
324 static inline int madvise_folio_pte_batch(unsigned long addr, unsigned long end,
325 struct folio *folio, pte_t *ptep,
326 pte_t pte, bool *any_young,
327 bool *any_dirty)
328 {
329 const fpb_t fpb_flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
330 int max_nr = (end - addr) / PAGE_SIZE;
331
332 return folio_pte_batch(folio, addr, ptep, pte, max_nr, fpb_flags, NULL,
333 any_young, any_dirty);
334 }
335
336 static int madvise_cold_or_pageout_pte_range(pmd_t *pmd,
337 unsigned long addr, unsigned long end,
338 struct mm_walk *walk)
339 {
340 struct madvise_walk_private *private = walk->private;
341 struct mmu_gather *tlb = private->tlb;
342 bool pageout = private->pageout;
343 struct mm_struct *mm = tlb->mm;
344 struct vm_area_struct *vma = walk->vma;
345 pte_t *start_pte, *pte, ptent;
346 spinlock_t *ptl;
347 struct folio *folio = NULL;
348 LIST_HEAD(folio_list);
349 bool pageout_anon_only_filter;
350 unsigned int batch_count = 0;
351 int nr;
352
353 if (fatal_signal_pending(current))
354 return -EINTR;
355
356 pageout_anon_only_filter = pageout && !vma_is_anonymous(vma) &&
357 !can_do_file_pageout(vma);
358
359 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
360 if (pmd_trans_huge(*pmd)) {
361 pmd_t orig_pmd;
362 unsigned long next = pmd_addr_end(addr, end);
363
364 tlb_change_page_size(tlb, HPAGE_PMD_SIZE);
365 ptl = pmd_trans_huge_lock(pmd, vma);
366 if (!ptl)
367 return 0;
368
369 orig_pmd = *pmd;
370 if (is_huge_zero_pmd(orig_pmd))
371 goto huge_unlock;
372
373 if (unlikely(!pmd_present(orig_pmd))) {
374 VM_BUG_ON(thp_migration_supported() &&
375 !is_pmd_migration_entry(orig_pmd));
376 goto huge_unlock;
377 }
378
379 folio = pmd_folio(orig_pmd);
380
381 /* Do not interfere with other mappings of this folio */
382 if (folio_likely_mapped_shared(folio))
383 goto huge_unlock;
384
385 if (pageout_anon_only_filter && !folio_test_anon(folio))
386 goto huge_unlock;
387
388 if (next - addr != HPAGE_PMD_SIZE) {
389 int err;
390
391 folio_get(folio);
392 spin_unlock(ptl);
393 folio_lock(folio);
394 err = split_folio(folio);
395 folio_unlock(folio);
396 folio_put(folio);
397 if (!err)
398 goto regular_folio;
399 return 0;
400 }
401
402 if (!pageout && pmd_young(orig_pmd)) {
403 pmdp_invalidate(vma, addr, pmd);
404 orig_pmd = pmd_mkold(orig_pmd);
405
406 set_pmd_at(mm, addr, pmd, orig_pmd);
407 tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
408 }
409
410 folio_clear_referenced(folio);
411 folio_test_clear_young(folio);
412 if (folio_test_active(folio))
413 folio_set_workingset(folio);
414 if (pageout) {
415 if (folio_isolate_lru(folio)) {
416 if (folio_test_unevictable(folio))
417 folio_putback_lru(folio);
418 else
419 list_add(&folio->lru, &folio_list);
420 }
421 } else
422 folio_deactivate(folio);
423 huge_unlock:
424 spin_unlock(ptl);
425 if (pageout)
426 reclaim_pages(&folio_list);
427 return 0;
428 }
429
430 regular_folio:
431 #endif
432 tlb_change_page_size(tlb, PAGE_SIZE);
433 restart:
434 start_pte = pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
435 if (!start_pte)
436 return 0;
437 flush_tlb_batched_pending(mm);
438 arch_enter_lazy_mmu_mode();
439 for (; addr < end; pte += nr, addr += nr * PAGE_SIZE) {
440 nr = 1;
441 ptent = ptep_get(pte);
442
443 if (++batch_count == SWAP_CLUSTER_MAX) {
444 batch_count = 0;
445 if (need_resched()) {
446 arch_leave_lazy_mmu_mode();
447 pte_unmap_unlock(start_pte, ptl);
448 cond_resched();
449 goto restart;
450 }
451 }
452
453 if (pte_none(ptent))
454 continue;
455
456 if (!pte_present(ptent))
457 continue;
458
459 folio = vm_normal_folio(vma, addr, ptent);
460 if (!folio || folio_is_zone_device(folio))
461 continue;
462
463 /*
464 * If we encounter a large folio, only split it if it is not
465 * fully mapped within the range we are operating on. Otherwise
466 * leave it as is so that it can be swapped out whole. If we
467 * fail to split a folio, leave it in place and advance to the
468 * next pte in the range.
469 */
470 if (folio_test_large(folio)) {
471 bool any_young;
472
473 nr = madvise_folio_pte_batch(addr, end, folio, pte,
474 ptent, &any_young, NULL);
475 if (any_young)
476 ptent = pte_mkyoung(ptent);
477
478 if (nr < folio_nr_pages(folio)) {
479 int err;
480
481 if (folio_likely_mapped_shared(folio))
482 continue;
483 if (pageout_anon_only_filter && !folio_test_anon(folio))
484 continue;
485 if (!folio_trylock(folio))
486 continue;
487 folio_get(folio);
488 arch_leave_lazy_mmu_mode();
489 pte_unmap_unlock(start_pte, ptl);
490 start_pte = NULL;
491 err = split_folio(folio);
492 folio_unlock(folio);
493 folio_put(folio);
494 start_pte = pte =
495 pte_offset_map_lock(mm, pmd, addr, &ptl);
496 if (!start_pte)
497 break;
498 arch_enter_lazy_mmu_mode();
499 if (!err)
500 nr = 0;
501 continue;
502 }
503 }
504
505 /*
506 * Do not interfere with other mappings of this folio and
507 * non-LRU folio. If we have a large folio at this point, we
508 * know it is fully mapped so if its mapcount is the same as its
509 * number of pages, it must be exclusive.
510 */
511 if (!folio_test_lru(folio) ||
512 folio_mapcount(folio) != folio_nr_pages(folio))
513 continue;
514
515 if (pageout_anon_only_filter && !folio_test_anon(folio))
516 continue;
517
518 if (!pageout && pte_young(ptent)) {
519 clear_young_dirty_ptes(vma, addr, pte, nr,
520 CYDP_CLEAR_YOUNG);
521 tlb_remove_tlb_entries(tlb, pte, nr, addr);
522 }
523
524 /*
525 * We are deactivating a folio for accelerating reclaiming.
526 * VM couldn't reclaim the folio unless we clear PG_young.
527 * As a side effect, it makes confuse idle-page tracking
528 * because they will miss recent referenced history.
529 */
530 folio_clear_referenced(folio);
531 folio_test_clear_young(folio);
532 if (folio_test_active(folio))
533 folio_set_workingset(folio);
534 if (pageout) {
535 if (folio_isolate_lru(folio)) {
536 if (folio_test_unevictable(folio))
537 folio_putback_lru(folio);
538 else
539 list_add(&folio->lru, &folio_list);
540 }
541 } else
542 folio_deactivate(folio);
543 }
544
545 if (start_pte) {
546 arch_leave_lazy_mmu_mode();
547 pte_unmap_unlock(start_pte, ptl);
548 }
549 if (pageout)
550 reclaim_pages(&folio_list);
551 cond_resched();
552
553 return 0;
554 }
555
556 static const struct mm_walk_ops cold_walk_ops = {
557 .pmd_entry = madvise_cold_or_pageout_pte_range,
558 .walk_lock = PGWALK_RDLOCK,
559 };
560
561 static void madvise_cold_page_range(struct mmu_gather *tlb,
562 struct vm_area_struct *vma,
563 unsigned long addr, unsigned long end)
564 {
565 struct madvise_walk_private walk_private = {
566 .pageout = false,
567 .tlb = tlb,
568 };
569
570 tlb_start_vma(tlb, vma);
571 walk_page_range(vma->vm_mm, addr, end, &cold_walk_ops, &walk_private);
572 tlb_end_vma(tlb, vma);
573 }
574
575 static inline bool can_madv_lru_vma(struct vm_area_struct *vma)
576 {
577 return !(vma->vm_flags & (VM_LOCKED|VM_PFNMAP|VM_HUGETLB));
578 }
579
580 static long madvise_cold(struct vm_area_struct *vma,
581 struct vm_area_struct **prev,
582 unsigned long start_addr, unsigned long end_addr)
583 {
584 struct mm_struct *mm = vma->vm_mm;
585 struct mmu_gather tlb;
586
587 *prev = vma;
588 if (!can_madv_lru_vma(vma))
589 return -EINVAL;
590
591 lru_add_drain();
592 tlb_gather_mmu(&tlb, mm);
593 madvise_cold_page_range(&tlb, vma, start_addr, end_addr);
594 tlb_finish_mmu(&tlb);
595
596 return 0;
597 }
598
599 static void madvise_pageout_page_range(struct mmu_gather *tlb,
600 struct vm_area_struct *vma,
601 unsigned long addr, unsigned long end)
602 {
603 struct madvise_walk_private walk_private = {
604 .pageout = true,
605 .tlb = tlb,
606 };
607
608 tlb_start_vma(tlb, vma);
609 walk_page_range(vma->vm_mm, addr, end, &cold_walk_ops, &walk_private);
610 tlb_end_vma(tlb, vma);
611 }
612
613 static long madvise_pageout(struct vm_area_struct *vma,
614 struct vm_area_struct **prev,
615 unsigned long start_addr, unsigned long end_addr)
616 {
617 struct mm_struct *mm = vma->vm_mm;
618 struct mmu_gather tlb;
619
620 *prev = vma;
621 if (!can_madv_lru_vma(vma))
622 return -EINVAL;
623
624 /*
625 * If the VMA belongs to a private file mapping, there can be private
626 * dirty pages which can be paged out if even this process is neither
627 * owner nor write capable of the file. We allow private file mappings
628 * further to pageout dirty anon pages.
629 */
630 if (!vma_is_anonymous(vma) && (!can_do_file_pageout(vma) &&
631 (vma->vm_flags & VM_MAYSHARE)))
632 return 0;
633
634 lru_add_drain();
635 tlb_gather_mmu(&tlb, mm);
636 madvise_pageout_page_range(&tlb, vma, start_addr, end_addr);
637 tlb_finish_mmu(&tlb);
638
639 return 0;
640 }
641
642 static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr,
643 unsigned long end, struct mm_walk *walk)
644
645 {
646 const cydp_t cydp_flags = CYDP_CLEAR_YOUNG | CYDP_CLEAR_DIRTY;
647 struct mmu_gather *tlb = walk->private;
648 struct mm_struct *mm = tlb->mm;
649 struct vm_area_struct *vma = walk->vma;
650 spinlock_t *ptl;
651 pte_t *start_pte, *pte, ptent;
652 struct folio *folio;
653 int nr_swap = 0;
654 unsigned long next;
655 int nr, max_nr;
656
657 next = pmd_addr_end(addr, end);
658 if (pmd_trans_huge(*pmd))
659 if (madvise_free_huge_pmd(tlb, vma, pmd, addr, next))
660 return 0;
661
662 tlb_change_page_size(tlb, PAGE_SIZE);
663 start_pte = pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
664 if (!start_pte)
665 return 0;
666 flush_tlb_batched_pending(mm);
667 arch_enter_lazy_mmu_mode();
668 for (; addr != end; pte += nr, addr += PAGE_SIZE * nr) {
669 nr = 1;
670 ptent = ptep_get(pte);
671
672 if (pte_none(ptent))
673 continue;
674 /*
675 * If the pte has swp_entry, just clear page table to
676 * prevent swap-in which is more expensive rather than
677 * (page allocation + zeroing).
678 */
679 if (!pte_present(ptent)) {
680 swp_entry_t entry;
681
682 entry = pte_to_swp_entry(ptent);
683 if (!non_swap_entry(entry)) {
684 max_nr = (end - addr) / PAGE_SIZE;
685 nr = swap_pte_batch(pte, max_nr, ptent);
686 nr_swap -= nr;
687 free_swap_and_cache_nr(entry, nr);
688 clear_not_present_full_ptes(mm, addr, pte, nr, tlb->fullmm);
689 } else if (is_hwpoison_entry(entry) ||
690 is_poisoned_swp_entry(entry)) {
691 pte_clear_not_present_full(mm, addr, pte, tlb->fullmm);
692 }
693 continue;
694 }
695
696 folio = vm_normal_folio(vma, addr, ptent);
697 if (!folio || folio_is_zone_device(folio))
698 continue;
699
700 /*
701 * If we encounter a large folio, only split it if it is not
702 * fully mapped within the range we are operating on. Otherwise
703 * leave it as is so that it can be marked as lazyfree. If we
704 * fail to split a folio, leave it in place and advance to the
705 * next pte in the range.
706 */
707 if (folio_test_large(folio)) {
708 bool any_young, any_dirty;
709
710 nr = madvise_folio_pte_batch(addr, end, folio, pte,
711 ptent, &any_young, &any_dirty);
712
713 if (nr < folio_nr_pages(folio)) {
714 int err;
715
716 if (folio_likely_mapped_shared(folio))
717 continue;
718 if (!folio_trylock(folio))
719 continue;
720 folio_get(folio);
721 arch_leave_lazy_mmu_mode();
722 pte_unmap_unlock(start_pte, ptl);
723 start_pte = NULL;
724 err = split_folio(folio);
725 folio_unlock(folio);
726 folio_put(folio);
727 pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
728 start_pte = pte;
729 if (!start_pte)
730 break;
731 arch_enter_lazy_mmu_mode();
732 if (!err)
733 nr = 0;
734 continue;
735 }
736
737 if (any_young)
738 ptent = pte_mkyoung(ptent);
739 if (any_dirty)
740 ptent = pte_mkdirty(ptent);
741 }
742
743 if (folio_test_swapcache(folio) || folio_test_dirty(folio)) {
744 if (!folio_trylock(folio))
745 continue;
746 /*
747 * If we have a large folio at this point, we know it is
748 * fully mapped so if its mapcount is the same as its
749 * number of pages, it must be exclusive.
750 */
751 if (folio_mapcount(folio) != folio_nr_pages(folio)) {
752 folio_unlock(folio);
753 continue;
754 }
755
756 if (folio_test_swapcache(folio) &&
757 !folio_free_swap(folio)) {
758 folio_unlock(folio);
759 continue;
760 }
761
762 folio_clear_dirty(folio);
763 folio_unlock(folio);
764 }
765
766 if (pte_young(ptent) || pte_dirty(ptent)) {
767 clear_young_dirty_ptes(vma, addr, pte, nr, cydp_flags);
768 tlb_remove_tlb_entries(tlb, pte, nr, addr);
769 }
770 folio_mark_lazyfree(folio);
771 }
772
773 if (nr_swap)
774 add_mm_counter(mm, MM_SWAPENTS, nr_swap);
775 if (start_pte) {
776 arch_leave_lazy_mmu_mode();
777 pte_unmap_unlock(start_pte, ptl);
778 }
779 cond_resched();
780
781 return 0;
782 }
783
784 static const struct mm_walk_ops madvise_free_walk_ops = {
785 .pmd_entry = madvise_free_pte_range,
786 .walk_lock = PGWALK_RDLOCK,
787 };
788
789 static int madvise_free_single_vma(struct vm_area_struct *vma,
790 unsigned long start_addr, unsigned long end_addr)
791 {
792 struct mm_struct *mm = vma->vm_mm;
793 struct mmu_notifier_range range;
794 struct mmu_gather tlb;
795
796 /* MADV_FREE works for only anon vma at the moment */
797 if (!vma_is_anonymous(vma))
798 return -EINVAL;
799
800 range.start = max(vma->vm_start, start_addr);
801 if (range.start >= vma->vm_end)
802 return -EINVAL;
803 range.end = min(vma->vm_end, end_addr);
804 if (range.end <= vma->vm_start)
805 return -EINVAL;
806 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm,
807 range.start, range.end);
808
809 lru_add_drain();
810 tlb_gather_mmu(&tlb, mm);
811 update_hiwater_rss(mm);
812
813 mmu_notifier_invalidate_range_start(&range);
814 tlb_start_vma(&tlb, vma);
815 walk_page_range(vma->vm_mm, range.start, range.end,
816 &madvise_free_walk_ops, &tlb);
817 tlb_end_vma(&tlb, vma);
818 mmu_notifier_invalidate_range_end(&range);
819 tlb_finish_mmu(&tlb);
820
821 return 0;
822 }
823
824 /*
825 * Application no longer needs these pages. If the pages are dirty,
826 * it's OK to just throw them away. The app will be more careful about
827 * data it wants to keep. Be sure to free swap resources too. The
828 * zap_page_range_single call sets things up for shrink_active_list to actually
829 * free these pages later if no one else has touched them in the meantime,
830 * although we could add these pages to a global reuse list for
831 * shrink_active_list to pick up before reclaiming other pages.
832 *
833 * NB: This interface discards data rather than pushes it out to swap,
834 * as some implementations do. This has performance implications for
835 * applications like large transactional databases which want to discard
836 * pages in anonymous maps after committing to backing store the data
837 * that was kept in them. There is no reason to write this data out to
838 * the swap area if the application is discarding it.
839 *
840 * An interface that causes the system to free clean pages and flush
841 * dirty pages is already available as msync(MS_INVALIDATE).
842 */
843 static long madvise_dontneed_single_vma(struct vm_area_struct *vma,
844 unsigned long start, unsigned long end)
845 {
846 zap_page_range_single(vma, start, end - start, NULL);
847 return 0;
848 }
849
850 static bool madvise_dontneed_free_valid_vma(struct vm_area_struct *vma,
851 unsigned long start,
852 unsigned long *end,
853 int behavior)
854 {
855 if (!is_vm_hugetlb_page(vma)) {
856 unsigned int forbidden = VM_PFNMAP;
857
858 if (behavior != MADV_DONTNEED_LOCKED)
859 forbidden |= VM_LOCKED;
860
861 return !(vma->vm_flags & forbidden);
862 }
863
864 if (behavior != MADV_DONTNEED && behavior != MADV_DONTNEED_LOCKED)
865 return false;
866 if (start & ~huge_page_mask(hstate_vma(vma)))
867 return false;
868
869 /*
870 * Madvise callers expect the length to be rounded up to PAGE_SIZE
871 * boundaries, and may be unaware that this VMA uses huge pages.
872 * Avoid unexpected data loss by rounding down the number of
873 * huge pages freed.
874 */
875 *end = ALIGN_DOWN(*end, huge_page_size(hstate_vma(vma)));
876
877 return true;
878 }
879
880 static long madvise_dontneed_free(struct vm_area_struct *vma,
881 struct vm_area_struct **prev,
882 unsigned long start, unsigned long end,
883 int behavior)
884 {
885 struct mm_struct *mm = vma->vm_mm;
886
887 *prev = vma;
888 if (!madvise_dontneed_free_valid_vma(vma, start, &end, behavior))
889 return -EINVAL;
890
891 if (start == end)
892 return 0;
893
894 if (!userfaultfd_remove(vma, start, end)) {
895 *prev = NULL; /* mmap_lock has been dropped, prev is stale */
896
897 mmap_read_lock(mm);
898 vma = vma_lookup(mm, start);
899 if (!vma)
900 return -ENOMEM;
901 /*
902 * Potential end adjustment for hugetlb vma is OK as
903 * the check below keeps end within vma.
904 */
905 if (!madvise_dontneed_free_valid_vma(vma, start, &end,
906 behavior))
907 return -EINVAL;
908 if (end > vma->vm_end) {
909 /*
910 * Don't fail if end > vma->vm_end. If the old
911 * vma was split while the mmap_lock was
912 * released the effect of the concurrent
913 * operation may not cause madvise() to
914 * have an undefined result. There may be an
915 * adjacent next vma that we'll walk
916 * next. userfaultfd_remove() will generate an
917 * UFFD_EVENT_REMOVE repetition on the
918 * end-vma->vm_end range, but the manager can
919 * handle a repetition fine.
920 */
921 end = vma->vm_end;
922 }
923 VM_WARN_ON(start >= end);
924 }
925
926 if (behavior == MADV_DONTNEED || behavior == MADV_DONTNEED_LOCKED)
927 return madvise_dontneed_single_vma(vma, start, end);
928 else if (behavior == MADV_FREE)
929 return madvise_free_single_vma(vma, start, end);
930 else
931 return -EINVAL;
932 }
933
934 static long madvise_populate(struct mm_struct *mm, unsigned long start,
935 unsigned long end, int behavior)
936 {
937 const bool write = behavior == MADV_POPULATE_WRITE;
938 int locked = 1;
939 long pages;
940
941 while (start < end) {
942 /* Populate (prefault) page tables readable/writable. */
943 pages = faultin_page_range(mm, start, end, write, &locked);
944 if (!locked) {
945 mmap_read_lock(mm);
946 locked = 1;
947 }
948 if (pages < 0) {
949 switch (pages) {
950 case -EINTR:
951 return -EINTR;
952 case -EINVAL: /* Incompatible mappings / permissions. */
953 return -EINVAL;
954 case -EHWPOISON:
955 return -EHWPOISON;
956 case -EFAULT: /* VM_FAULT_SIGBUS or VM_FAULT_SIGSEGV */
957 return -EFAULT;
958 default:
959 pr_warn_once("%s: unhandled return value: %ld\n",
960 __func__, pages);
961 fallthrough;
962 case -ENOMEM: /* No VMA or out of memory. */
963 return -ENOMEM;
964 }
965 }
966 start += pages * PAGE_SIZE;
967 }
968 return 0;
969 }
970
971 /*
972 * Application wants to free up the pages and associated backing store.
973 * This is effectively punching a hole into the middle of a file.
974 */
975 static long madvise_remove(struct vm_area_struct *vma,
976 struct vm_area_struct **prev,
977 unsigned long start, unsigned long end)
978 {
979 loff_t offset;
980 int error;
981 struct file *f;
982 struct mm_struct *mm = vma->vm_mm;
983
984 *prev = NULL; /* tell sys_madvise we drop mmap_lock */
985
986 if (vma->vm_flags & VM_LOCKED)
987 return -EINVAL;
988
989 f = vma->vm_file;
990
991 if (!f || !f->f_mapping || !f->f_mapping->host) {
992 return -EINVAL;
993 }
994
995 if (!vma_is_shared_maywrite(vma))
996 return -EACCES;
997
998 offset = (loff_t)(start - vma->vm_start)
999 + ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
1000
1001 /*
1002 * Filesystem's fallocate may need to take i_rwsem. We need to
1003 * explicitly grab a reference because the vma (and hence the
1004 * vma's reference to the file) can go away as soon as we drop
1005 * mmap_lock.
1006 */
1007 get_file(f);
1008 if (userfaultfd_remove(vma, start, end)) {
1009 /* mmap_lock was not released by userfaultfd_remove() */
1010 mmap_read_unlock(mm);
1011 }
1012 error = vfs_fallocate(f,
1013 FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
1014 offset, end - start);
1015 fput(f);
1016 mmap_read_lock(mm);
1017 return error;
1018 }
1019
1020 /*
1021 * Apply an madvise behavior to a region of a vma. madvise_update_vma
1022 * will handle splitting a vm area into separate areas, each area with its own
1023 * behavior.
1024 */
1025 static int madvise_vma_behavior(struct vm_area_struct *vma,
1026 struct vm_area_struct **prev,
1027 unsigned long start, unsigned long end,
1028 unsigned long behavior)
1029 {
1030 int error;
1031 struct anon_vma_name *anon_name;
1032 unsigned long new_flags = vma->vm_flags;
1033
1034 switch (behavior) {
1035 case MADV_REMOVE:
1036 return madvise_remove(vma, prev, start, end);
1037 case MADV_WILLNEED:
1038 return madvise_willneed(vma, prev, start, end);
1039 case MADV_COLD:
1040 return madvise_cold(vma, prev, start, end);
1041 case MADV_PAGEOUT:
1042 return madvise_pageout(vma, prev, start, end);
1043 case MADV_FREE:
1044 case MADV_DONTNEED:
1045 case MADV_DONTNEED_LOCKED:
1046 return madvise_dontneed_free(vma, prev, start, end, behavior);
1047 case MADV_NORMAL:
1048 new_flags = new_flags & ~VM_RAND_READ & ~VM_SEQ_READ;
1049 break;
1050 case MADV_SEQUENTIAL:
1051 new_flags = (new_flags & ~VM_RAND_READ) | VM_SEQ_READ;
1052 break;
1053 case MADV_RANDOM:
1054 new_flags = (new_flags & ~VM_SEQ_READ) | VM_RAND_READ;
1055 break;
1056 case MADV_DONTFORK:
1057 new_flags |= VM_DONTCOPY;
1058 break;
1059 case MADV_DOFORK:
1060 if (vma->vm_flags & VM_IO)
1061 return -EINVAL;
1062 new_flags &= ~VM_DONTCOPY;
1063 break;
1064 case MADV_WIPEONFORK:
1065 /* MADV_WIPEONFORK is only supported on anonymous memory. */
1066 if (vma->vm_file || vma->vm_flags & VM_SHARED)
1067 return -EINVAL;
1068 new_flags |= VM_WIPEONFORK;
1069 break;
1070 case MADV_KEEPONFORK:
1071 if (vma->vm_flags & VM_DROPPABLE)
1072 return -EINVAL;
1073 new_flags &= ~VM_WIPEONFORK;
1074 break;
1075 case MADV_DONTDUMP:
1076 new_flags |= VM_DONTDUMP;
1077 break;
1078 case MADV_DODUMP:
1079 if ((!is_vm_hugetlb_page(vma) && new_flags & VM_SPECIAL) ||
1080 (vma->vm_flags & VM_DROPPABLE))
1081 return -EINVAL;
1082 new_flags &= ~VM_DONTDUMP;
1083 break;
1084 case MADV_MERGEABLE:
1085 case MADV_UNMERGEABLE:
1086 error = ksm_madvise(vma, start, end, behavior, &new_flags);
1087 if (error)
1088 goto out;
1089 break;
1090 case MADV_HUGEPAGE:
1091 case MADV_NOHUGEPAGE:
1092 error = hugepage_madvise(vma, &new_flags, behavior);
1093 if (error)
1094 goto out;
1095 break;
1096 case MADV_COLLAPSE:
1097 return madvise_collapse(vma, prev, start, end);
1098 }
1099
1100 anon_name = anon_vma_name(vma);
1101 anon_vma_name_get(anon_name);
1102 error = madvise_update_vma(vma, prev, start, end, new_flags,
1103 anon_name);
1104 anon_vma_name_put(anon_name);
1105
1106 out:
1107 /*
1108 * madvise() returns EAGAIN if kernel resources, such as
1109 * slab, are temporarily unavailable.
1110 */
1111 if (error == -ENOMEM)
1112 error = -EAGAIN;
1113 return error;
1114 }
1115
1116 #ifdef CONFIG_MEMORY_FAILURE
1117 /*
1118 * Error injection support for memory error handling.
1119 */
1120 static int madvise_inject_error(int behavior,
1121 unsigned long start, unsigned long end)
1122 {
1123 unsigned long size;
1124
1125 if (!capable(CAP_SYS_ADMIN))
1126 return -EPERM;
1127
1128
1129 for (; start < end; start += size) {
1130 unsigned long pfn;
1131 struct page *page;
1132 int ret;
1133
1134 ret = get_user_pages_fast(start, 1, 0, &page);
1135 if (ret != 1)
1136 return ret;
1137 pfn = page_to_pfn(page);
1138
1139 /*
1140 * When soft offlining hugepages, after migrating the page
1141 * we dissolve it, therefore in the second loop "page" will
1142 * no longer be a compound page.
1143 */
1144 size = page_size(compound_head(page));
1145
1146 if (behavior == MADV_SOFT_OFFLINE) {
1147 pr_info("Soft offlining pfn %#lx at process virtual address %#lx\n",
1148 pfn, start);
1149 ret = soft_offline_page(pfn, MF_COUNT_INCREASED);
1150 } else {
1151 pr_info("Injecting memory failure for pfn %#lx at process virtual address %#lx\n",
1152 pfn, start);
1153 ret = memory_failure(pfn, MF_ACTION_REQUIRED | MF_COUNT_INCREASED | MF_SW_SIMULATED);
1154 if (ret == -EOPNOTSUPP)
1155 ret = 0;
1156 }
1157
1158 if (ret)
1159 return ret;
1160 }
1161
1162 return 0;
1163 }
1164 #endif
1165
1166 static bool
1167 madvise_behavior_valid(int behavior)
1168 {
1169 switch (behavior) {
1170 case MADV_DOFORK:
1171 case MADV_DONTFORK:
1172 case MADV_NORMAL:
1173 case MADV_SEQUENTIAL:
1174 case MADV_RANDOM:
1175 case MADV_REMOVE:
1176 case MADV_WILLNEED:
1177 case MADV_DONTNEED:
1178 case MADV_DONTNEED_LOCKED:
1179 case MADV_FREE:
1180 case MADV_COLD:
1181 case MADV_PAGEOUT:
1182 case MADV_POPULATE_READ:
1183 case MADV_POPULATE_WRITE:
1184 #ifdef CONFIG_KSM
1185 case MADV_MERGEABLE:
1186 case MADV_UNMERGEABLE:
1187 #endif
1188 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1189 case MADV_HUGEPAGE:
1190 case MADV_NOHUGEPAGE:
1191 case MADV_COLLAPSE:
1192 #endif
1193 case MADV_DONTDUMP:
1194 case MADV_DODUMP:
1195 case MADV_WIPEONFORK:
1196 case MADV_KEEPONFORK:
1197 #ifdef CONFIG_MEMORY_FAILURE
1198 case MADV_SOFT_OFFLINE:
1199 case MADV_HWPOISON:
1200 #endif
1201 return true;
1202
1203 default:
1204 return false;
1205 }
1206 }
1207
1208 static bool process_madvise_behavior_valid(int behavior)
1209 {
1210 switch (behavior) {
1211 case MADV_COLD:
1212 case MADV_PAGEOUT:
1213 case MADV_WILLNEED:
1214 case MADV_COLLAPSE:
1215 return true;
1216 default:
1217 return false;
1218 }
1219 }
1220
1221 /*
1222 * Walk the vmas in range [start,end), and call the visit function on each one.
1223 * The visit function will get start and end parameters that cover the overlap
1224 * between the current vma and the original range. Any unmapped regions in the
1225 * original range will result in this function returning -ENOMEM while still
1226 * calling the visit function on all of the existing vmas in the range.
1227 * Must be called with the mmap_lock held for reading or writing.
1228 */
1229 static
1230 int madvise_walk_vmas(struct mm_struct *mm, unsigned long start,
1231 unsigned long end, unsigned long arg,
1232 int (*visit)(struct vm_area_struct *vma,
1233 struct vm_area_struct **prev, unsigned long start,
1234 unsigned long end, unsigned long arg))
1235 {
1236 struct vm_area_struct *vma;
1237 struct vm_area_struct *prev;
1238 unsigned long tmp;
1239 int unmapped_error = 0;
1240
1241 /*
1242 * If the interval [start,end) covers some unmapped address
1243 * ranges, just ignore them, but return -ENOMEM at the end.
1244 * - different from the way of handling in mlock etc.
1245 */
1246 vma = find_vma_prev(mm, start, &prev);
1247 if (vma && start > vma->vm_start)
1248 prev = vma;
1249
1250 for (;;) {
1251 int error;
1252
1253 /* Still start < end. */
1254 if (!vma)
1255 return -ENOMEM;
1256
1257 /* Here start < (end|vma->vm_end). */
1258 if (start < vma->vm_start) {
1259 unmapped_error = -ENOMEM;
1260 start = vma->vm_start;
1261 if (start >= end)
1262 break;
1263 }
1264
1265 /* Here vma->vm_start <= start < (end|vma->vm_end) */
1266 tmp = vma->vm_end;
1267 if (end < tmp)
1268 tmp = end;
1269
1270 /* Here vma->vm_start <= start < tmp <= (end|vma->vm_end). */
1271 error = visit(vma, &prev, start, tmp, arg);
1272 if (error)
1273 return error;
1274 start = tmp;
1275 if (prev && start < prev->vm_end)
1276 start = prev->vm_end;
1277 if (start >= end)
1278 break;
1279 if (prev)
1280 vma = find_vma(mm, prev->vm_end);
1281 else /* madvise_remove dropped mmap_lock */
1282 vma = find_vma(mm, start);
1283 }
1284
1285 return unmapped_error;
1286 }
1287
1288 #ifdef CONFIG_ANON_VMA_NAME
1289 static int madvise_vma_anon_name(struct vm_area_struct *vma,
1290 struct vm_area_struct **prev,
1291 unsigned long start, unsigned long end,
1292 unsigned long anon_name)
1293 {
1294 int error;
1295
1296 /* Only anonymous mappings can be named */
1297 if (vma->vm_file && !vma_is_anon_shmem(vma))
1298 return -EBADF;
1299
1300 error = madvise_update_vma(vma, prev, start, end, vma->vm_flags,
1301 (struct anon_vma_name *)anon_name);
1302
1303 /*
1304 * madvise() returns EAGAIN if kernel resources, such as
1305 * slab, are temporarily unavailable.
1306 */
1307 if (error == -ENOMEM)
1308 error = -EAGAIN;
1309 return error;
1310 }
1311
1312 int madvise_set_anon_name(struct mm_struct *mm, unsigned long start,
1313 unsigned long len_in, struct anon_vma_name *anon_name)
1314 {
1315 unsigned long end;
1316 unsigned long len;
1317
1318 if (start & ~PAGE_MASK)
1319 return -EINVAL;
1320 len = (len_in + ~PAGE_MASK) & PAGE_MASK;
1321
1322 /* Check to see whether len was rounded up from small -ve to zero */
1323 if (len_in && !len)
1324 return -EINVAL;
1325
1326 end = start + len;
1327 if (end < start)
1328 return -EINVAL;
1329
1330 if (end == start)
1331 return 0;
1332
1333 return madvise_walk_vmas(mm, start, end, (unsigned long)anon_name,
1334 madvise_vma_anon_name);
1335 }
1336 #endif /* CONFIG_ANON_VMA_NAME */
1337 /*
1338 * The madvise(2) system call.
1339 *
1340 * Applications can use madvise() to advise the kernel how it should
1341 * handle paging I/O in this VM area. The idea is to help the kernel
1342 * use appropriate read-ahead and caching techniques. The information
1343 * provided is advisory only, and can be safely disregarded by the
1344 * kernel without affecting the correct operation of the application.
1345 *
1346 * behavior values:
1347 * MADV_NORMAL - the default behavior is to read clusters. This
1348 * results in some read-ahead and read-behind.
1349 * MADV_RANDOM - the system should read the minimum amount of data
1350 * on any access, since it is unlikely that the appli-
1351 * cation will need more than what it asks for.
1352 * MADV_SEQUENTIAL - pages in the given range will probably be accessed
1353 * once, so they can be aggressively read ahead, and
1354 * can be freed soon after they are accessed.
1355 * MADV_WILLNEED - the application is notifying the system to read
1356 * some pages ahead.
1357 * MADV_DONTNEED - the application is finished with the given range,
1358 * so the kernel can free resources associated with it.
1359 * MADV_FREE - the application marks pages in the given range as lazy free,
1360 * where actual purges are postponed until memory pressure happens.
1361 * MADV_REMOVE - the application wants to free up the given range of
1362 * pages and associated backing store.
1363 * MADV_DONTFORK - omit this area from child's address space when forking:
1364 * typically, to avoid COWing pages pinned by get_user_pages().
1365 * MADV_DOFORK - cancel MADV_DONTFORK: no longer omit this area when forking.
1366 * MADV_WIPEONFORK - present the child process with zero-filled memory in this
1367 * range after a fork.
1368 * MADV_KEEPONFORK - undo the effect of MADV_WIPEONFORK
1369 * MADV_HWPOISON - trigger memory error handler as if the given memory range
1370 * were corrupted by unrecoverable hardware memory failure.
1371 * MADV_SOFT_OFFLINE - try to soft-offline the given range of memory.
1372 * MADV_MERGEABLE - the application recommends that KSM try to merge pages in
1373 * this area with pages of identical content from other such areas.
1374 * MADV_UNMERGEABLE- cancel MADV_MERGEABLE: no longer merge pages with others.
1375 * MADV_HUGEPAGE - the application wants to back the given range by transparent
1376 * huge pages in the future. Existing pages might be coalesced and
1377 * new pages might be allocated as THP.
1378 * MADV_NOHUGEPAGE - mark the given range as not worth being backed by
1379 * transparent huge pages so the existing pages will not be
1380 * coalesced into THP and new pages will not be allocated as THP.
1381 * MADV_COLLAPSE - synchronously coalesce pages into new THP.
1382 * MADV_DONTDUMP - the application wants to prevent pages in the given range
1383 * from being included in its core dump.
1384 * MADV_DODUMP - cancel MADV_DONTDUMP: no longer exclude from core dump.
1385 * MADV_COLD - the application is not expected to use this memory soon,
1386 * deactivate pages in this range so that they can be reclaimed
1387 * easily if memory pressure happens.
1388 * MADV_PAGEOUT - the application is not expected to use this memory soon,
1389 * page out the pages in this range immediately.
1390 * MADV_POPULATE_READ - populate (prefault) page tables readable by
1391 * triggering read faults if required
1392 * MADV_POPULATE_WRITE - populate (prefault) page tables writable by
1393 * triggering write faults if required
1394 *
1395 * return values:
1396 * zero - success
1397 * -EINVAL - start + len < 0, start is not page-aligned,
1398 * "behavior" is not a valid value, or application
1399 * is attempting to release locked or shared pages,
1400 * or the specified address range includes file, Huge TLB,
1401 * MAP_SHARED or VMPFNMAP range.
1402 * -ENOMEM - addresses in the specified range are not currently
1403 * mapped, or are outside the AS of the process.
1404 * -EIO - an I/O error occurred while paging in data.
1405 * -EBADF - map exists, but area maps something that isn't a file.
1406 * -EAGAIN - a kernel resource was temporarily unavailable.
1407 * -EPERM - memory is sealed.
1408 */
1409 int do_madvise(struct mm_struct *mm, unsigned long start, size_t len_in, int behavior)
1410 {
1411 unsigned long end;
1412 int error;
1413 int write;
1414 size_t len;
1415 struct blk_plug plug;
1416
1417 if (!madvise_behavior_valid(behavior))
1418 return -EINVAL;
1419
1420 if (!PAGE_ALIGNED(start))
1421 return -EINVAL;
1422 len = PAGE_ALIGN(len_in);
1423
1424 /* Check to see whether len was rounded up from small -ve to zero */
1425 if (len_in && !len)
1426 return -EINVAL;
1427
1428 end = start + len;
1429 if (end < start)
1430 return -EINVAL;
1431
1432 if (end == start)
1433 return 0;
1434
1435 #ifdef CONFIG_MEMORY_FAILURE
1436 if (behavior == MADV_HWPOISON || behavior == MADV_SOFT_OFFLINE)
1437 return madvise_inject_error(behavior, start, start + len_in);
1438 #endif
1439
1440 write = madvise_need_mmap_write(behavior);
1441 if (write) {
1442 if (mmap_write_lock_killable(mm))
1443 return -EINTR;
1444 } else {
1445 mmap_read_lock(mm);
1446 }
1447
1448 start = untagged_addr_remote(mm, start);
1449 end = start + len;
1450
1451 /*
1452 * Check if the address range is sealed for do_madvise().
1453 * can_modify_mm_madv assumes we have acquired the lock on MM.
1454 */
1455 if (unlikely(!can_modify_mm_madv(mm, start, end, behavior))) {
1456 error = -EPERM;
1457 goto out;
1458 }
1459
1460 blk_start_plug(&plug);
1461 switch (behavior) {
1462 case MADV_POPULATE_READ:
1463 case MADV_POPULATE_WRITE:
1464 error = madvise_populate(mm, start, end, behavior);
1465 break;
1466 default:
1467 error = madvise_walk_vmas(mm, start, end, behavior,
1468 madvise_vma_behavior);
1469 break;
1470 }
1471 blk_finish_plug(&plug);
1472
1473 out:
1474 if (write)
1475 mmap_write_unlock(mm);
1476 else
1477 mmap_read_unlock(mm);
1478
1479 return error;
1480 }
1481
1482 SYSCALL_DEFINE3(madvise, unsigned long, start, size_t, len_in, int, behavior)
1483 {
1484 return do_madvise(current->mm, start, len_in, behavior);
1485 }
1486
1487 SYSCALL_DEFINE5(process_madvise, int, pidfd, const struct iovec __user *, vec,
1488 size_t, vlen, int, behavior, unsigned int, flags)
1489 {
1490 ssize_t ret;
1491 struct iovec iovstack[UIO_FASTIOV];
1492 struct iovec *iov = iovstack;
1493 struct iov_iter iter;
1494 struct task_struct *task;
1495 struct mm_struct *mm;
1496 size_t total_len;
1497 unsigned int f_flags;
1498
1499 if (flags != 0) {
1500 ret = -EINVAL;
1501 goto out;
1502 }
1503
1504 ret = import_iovec(ITER_DEST, vec, vlen, ARRAY_SIZE(iovstack), &iov, &iter);
1505 if (ret < 0)
1506 goto out;
1507
1508 task = pidfd_get_task(pidfd, &f_flags);
1509 if (IS_ERR(task)) {
1510 ret = PTR_ERR(task);
1511 goto free_iov;
1512 }
1513
1514 if (!process_madvise_behavior_valid(behavior)) {
1515 ret = -EINVAL;
1516 goto release_task;
1517 }
1518
1519 /* Require PTRACE_MODE_READ to avoid leaking ASLR metadata. */
1520 mm = mm_access(task, PTRACE_MODE_READ_FSCREDS);
1521 if (IS_ERR_OR_NULL(mm)) {
1522 ret = IS_ERR(mm) ? PTR_ERR(mm) : -ESRCH;
1523 goto release_task;
1524 }
1525
1526 /*
1527 * Require CAP_SYS_NICE for influencing process performance. Note that
1528 * only non-destructive hints are currently supported.
1529 */
1530 if (!capable(CAP_SYS_NICE)) {
1531 ret = -EPERM;
1532 goto release_mm;
1533 }
1534
1535 total_len = iov_iter_count(&iter);
1536
1537 while (iov_iter_count(&iter)) {
1538 ret = do_madvise(mm, (unsigned long)iter_iov_addr(&iter),
1539 iter_iov_len(&iter), behavior);
1540 if (ret < 0)
1541 break;
1542 iov_iter_advance(&iter, iter_iov_len(&iter));
1543 }
1544
1545 ret = (total_len - iov_iter_count(&iter)) ? : ret;
1546
1547 release_mm:
1548 mmput(mm);
1549 release_task:
1550 put_task_struct(task);
1551 free_iov:
1552 kfree(iov);
1553 out:
1554 return ret;
1555 }
1556
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.