1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/pagewalk.h>
3 #include <linux/mm_inline.h>
4 #include <linux/hugetlb.h>
5 #include <linux/huge_mm.h>
6 #include <linux/mount.h>
7 #include <linux/ksm.h>
8 #include <linux/seq_file.h>
9 #include <linux/highmem.h>
10 #include <linux/ptrace.h>
11 #include <linux/slab.h>
12 #include <linux/pagemap.h>
13 #include <linux/mempolicy.h>
14 #include <linux/rmap.h>
15 #include <linux/swap.h>
16 #include <linux/sched/mm.h>
17 #include <linux/swapops.h>
18 #include <linux/mmu_notifier.h>
19 #include <linux/page_idle.h>
20 #include <linux/shmem_fs.h>
21 #include <linux/uaccess.h>
22 #include <linux/pkeys.h>
23 #include <linux/minmax.h>
24 #include <linux/overflow.h>
25 #include <linux/buildid.h>
26
27 #include <asm/elf.h>
28 #include <asm/tlb.h>
29 #include <asm/tlbflush.h>
30 #include "internal.h"
31
32 #define SEQ_PUT_DEC(str, val) \
33 seq_put_decimal_ull_width(m, str, (val) << (PAGE_SHIFT-10), 8)
34 void task_mem(struct seq_file *m, struct mm_struct *mm)
35 {
36 unsigned long text, lib, swap, anon, file, shmem;
37 unsigned long hiwater_vm, total_vm, hiwater_rss, total_rss;
38
39 anon = get_mm_counter(mm, MM_ANONPAGES);
40 file = get_mm_counter(mm, MM_FILEPAGES);
41 shmem = get_mm_counter(mm, MM_SHMEMPAGES);
42
43 /*
44 * Note: to minimize their overhead, mm maintains hiwater_vm and
45 * hiwater_rss only when about to *lower* total_vm or rss. Any
46 * collector of these hiwater stats must therefore get total_vm
47 * and rss too, which will usually be the higher. Barriers? not
48 * worth the effort, such snapshots can always be inconsistent.
49 */
50 hiwater_vm = total_vm = mm->total_vm;
51 if (hiwater_vm < mm->hiwater_vm)
52 hiwater_vm = mm->hiwater_vm;
53 hiwater_rss = total_rss = anon + file + shmem;
54 if (hiwater_rss < mm->hiwater_rss)
55 hiwater_rss = mm->hiwater_rss;
56
57 /* split executable areas between text and lib */
58 text = PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK);
59 text = min(text, mm->exec_vm << PAGE_SHIFT);
60 lib = (mm->exec_vm << PAGE_SHIFT) - text;
61
62 swap = get_mm_counter(mm, MM_SWAPENTS);
63 SEQ_PUT_DEC("VmPeak:\t", hiwater_vm);
64 SEQ_PUT_DEC(" kB\nVmSize:\t", total_vm);
65 SEQ_PUT_DEC(" kB\nVmLck:\t", mm->locked_vm);
66 SEQ_PUT_DEC(" kB\nVmPin:\t", atomic64_read(&mm->pinned_vm));
67 SEQ_PUT_DEC(" kB\nVmHWM:\t", hiwater_rss);
68 SEQ_PUT_DEC(" kB\nVmRSS:\t", total_rss);
69 SEQ_PUT_DEC(" kB\nRssAnon:\t", anon);
70 SEQ_PUT_DEC(" kB\nRssFile:\t", file);
71 SEQ_PUT_DEC(" kB\nRssShmem:\t", shmem);
72 SEQ_PUT_DEC(" kB\nVmData:\t", mm->data_vm);
73 SEQ_PUT_DEC(" kB\nVmStk:\t", mm->stack_vm);
74 seq_put_decimal_ull_width(m,
75 " kB\nVmExe:\t", text >> 10, 8);
76 seq_put_decimal_ull_width(m,
77 " kB\nVmLib:\t", lib >> 10, 8);
78 seq_put_decimal_ull_width(m,
79 " kB\nVmPTE:\t", mm_pgtables_bytes(mm) >> 10, 8);
80 SEQ_PUT_DEC(" kB\nVmSwap:\t", swap);
81 seq_puts(m, " kB\n");
82 hugetlb_report_usage(m, mm);
83 }
84 #undef SEQ_PUT_DEC
85
86 unsigned long task_vsize(struct mm_struct *mm)
87 {
88 return PAGE_SIZE * mm->total_vm;
89 }
90
91 unsigned long task_statm(struct mm_struct *mm,
92 unsigned long *shared, unsigned long *text,
93 unsigned long *data, unsigned long *resident)
94 {
95 *shared = get_mm_counter(mm, MM_FILEPAGES) +
96 get_mm_counter(mm, MM_SHMEMPAGES);
97 *text = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK))
98 >> PAGE_SHIFT;
99 *data = mm->data_vm + mm->stack_vm;
100 *resident = *shared + get_mm_counter(mm, MM_ANONPAGES);
101 return mm->total_vm;
102 }
103
104 #ifdef CONFIG_NUMA
105 /*
106 * Save get_task_policy() for show_numa_map().
107 */
108 static void hold_task_mempolicy(struct proc_maps_private *priv)
109 {
110 struct task_struct *task = priv->task;
111
112 task_lock(task);
113 priv->task_mempolicy = get_task_policy(task);
114 mpol_get(priv->task_mempolicy);
115 task_unlock(task);
116 }
117 static void release_task_mempolicy(struct proc_maps_private *priv)
118 {
119 mpol_put(priv->task_mempolicy);
120 }
121 #else
122 static void hold_task_mempolicy(struct proc_maps_private *priv)
123 {
124 }
125 static void release_task_mempolicy(struct proc_maps_private *priv)
126 {
127 }
128 #endif
129
130 static struct vm_area_struct *proc_get_vma(struct proc_maps_private *priv,
131 loff_t *ppos)
132 {
133 struct vm_area_struct *vma = vma_next(&priv->iter);
134
135 if (vma) {
136 *ppos = vma->vm_start;
137 } else {
138 *ppos = -2UL;
139 vma = get_gate_vma(priv->mm);
140 }
141
142 return vma;
143 }
144
145 static void *m_start(struct seq_file *m, loff_t *ppos)
146 {
147 struct proc_maps_private *priv = m->private;
148 unsigned long last_addr = *ppos;
149 struct mm_struct *mm;
150
151 /* See m_next(). Zero at the start or after lseek. */
152 if (last_addr == -1UL)
153 return NULL;
154
155 priv->task = get_proc_task(priv->inode);
156 if (!priv->task)
157 return ERR_PTR(-ESRCH);
158
159 mm = priv->mm;
160 if (!mm || !mmget_not_zero(mm)) {
161 put_task_struct(priv->task);
162 priv->task = NULL;
163 return NULL;
164 }
165
166 if (mmap_read_lock_killable(mm)) {
167 mmput(mm);
168 put_task_struct(priv->task);
169 priv->task = NULL;
170 return ERR_PTR(-EINTR);
171 }
172
173 vma_iter_init(&priv->iter, mm, last_addr);
174 hold_task_mempolicy(priv);
175 if (last_addr == -2UL)
176 return get_gate_vma(mm);
177
178 return proc_get_vma(priv, ppos);
179 }
180
181 static void *m_next(struct seq_file *m, void *v, loff_t *ppos)
182 {
183 if (*ppos == -2UL) {
184 *ppos = -1UL;
185 return NULL;
186 }
187 return proc_get_vma(m->private, ppos);
188 }
189
190 static void m_stop(struct seq_file *m, void *v)
191 {
192 struct proc_maps_private *priv = m->private;
193 struct mm_struct *mm = priv->mm;
194
195 if (!priv->task)
196 return;
197
198 release_task_mempolicy(priv);
199 mmap_read_unlock(mm);
200 mmput(mm);
201 put_task_struct(priv->task);
202 priv->task = NULL;
203 }
204
205 static int proc_maps_open(struct inode *inode, struct file *file,
206 const struct seq_operations *ops, int psize)
207 {
208 struct proc_maps_private *priv = __seq_open_private(file, ops, psize);
209
210 if (!priv)
211 return -ENOMEM;
212
213 priv->inode = inode;
214 priv->mm = proc_mem_open(inode, PTRACE_MODE_READ);
215 if (IS_ERR(priv->mm)) {
216 int err = PTR_ERR(priv->mm);
217
218 seq_release_private(inode, file);
219 return err;
220 }
221
222 return 0;
223 }
224
225 static int proc_map_release(struct inode *inode, struct file *file)
226 {
227 struct seq_file *seq = file->private_data;
228 struct proc_maps_private *priv = seq->private;
229
230 if (priv->mm)
231 mmdrop(priv->mm);
232
233 return seq_release_private(inode, file);
234 }
235
236 static int do_maps_open(struct inode *inode, struct file *file,
237 const struct seq_operations *ops)
238 {
239 return proc_maps_open(inode, file, ops,
240 sizeof(struct proc_maps_private));
241 }
242
243 static void get_vma_name(struct vm_area_struct *vma,
244 const struct path **path,
245 const char **name,
246 const char **name_fmt)
247 {
248 struct anon_vma_name *anon_name = vma->vm_mm ? anon_vma_name(vma) : NULL;
249
250 *name = NULL;
251 *path = NULL;
252 *name_fmt = NULL;
253
254 /*
255 * Print the dentry name for named mappings, and a
256 * special [heap] marker for the heap:
257 */
258 if (vma->vm_file) {
259 /*
260 * If user named this anon shared memory via
261 * prctl(PR_SET_VMA ..., use the provided name.
262 */
263 if (anon_name) {
264 *name_fmt = "[anon_shmem:%s]";
265 *name = anon_name->name;
266 } else {
267 *path = file_user_path(vma->vm_file);
268 }
269 return;
270 }
271
272 if (vma->vm_ops && vma->vm_ops->name) {
273 *name = vma->vm_ops->name(vma);
274 if (*name)
275 return;
276 }
277
278 *name = arch_vma_name(vma);
279 if (*name)
280 return;
281
282 if (!vma->vm_mm) {
283 *name = "[vdso]";
284 return;
285 }
286
287 if (vma_is_initial_heap(vma)) {
288 *name = "[heap]";
289 return;
290 }
291
292 if (vma_is_initial_stack(vma)) {
293 *name = "[stack]";
294 return;
295 }
296
297 if (anon_name) {
298 *name_fmt = "[anon:%s]";
299 *name = anon_name->name;
300 return;
301 }
302 }
303
304 static void show_vma_header_prefix(struct seq_file *m,
305 unsigned long start, unsigned long end,
306 vm_flags_t flags, unsigned long long pgoff,
307 dev_t dev, unsigned long ino)
308 {
309 seq_setwidth(m, 25 + sizeof(void *) * 6 - 1);
310 seq_put_hex_ll(m, NULL, start, 8);
311 seq_put_hex_ll(m, "-", end, 8);
312 seq_putc(m, ' ');
313 seq_putc(m, flags & VM_READ ? 'r' : '-');
314 seq_putc(m, flags & VM_WRITE ? 'w' : '-');
315 seq_putc(m, flags & VM_EXEC ? 'x' : '-');
316 seq_putc(m, flags & VM_MAYSHARE ? 's' : 'p');
317 seq_put_hex_ll(m, " ", pgoff, 8);
318 seq_put_hex_ll(m, " ", MAJOR(dev), 2);
319 seq_put_hex_ll(m, ":", MINOR(dev), 2);
320 seq_put_decimal_ull(m, " ", ino);
321 seq_putc(m, ' ');
322 }
323
324 static void
325 show_map_vma(struct seq_file *m, struct vm_area_struct *vma)
326 {
327 const struct path *path;
328 const char *name_fmt, *name;
329 vm_flags_t flags = vma->vm_flags;
330 unsigned long ino = 0;
331 unsigned long long pgoff = 0;
332 unsigned long start, end;
333 dev_t dev = 0;
334
335 if (vma->vm_file) {
336 const struct inode *inode = file_user_inode(vma->vm_file);
337
338 dev = inode->i_sb->s_dev;
339 ino = inode->i_ino;
340 pgoff = ((loff_t)vma->vm_pgoff) << PAGE_SHIFT;
341 }
342
343 start = vma->vm_start;
344 end = vma->vm_end;
345 show_vma_header_prefix(m, start, end, flags, pgoff, dev, ino);
346
347 get_vma_name(vma, &path, &name, &name_fmt);
348 if (path) {
349 seq_pad(m, ' ');
350 seq_path(m, path, "\n");
351 } else if (name_fmt) {
352 seq_pad(m, ' ');
353 seq_printf(m, name_fmt, name);
354 } else if (name) {
355 seq_pad(m, ' ');
356 seq_puts(m, name);
357 }
358 seq_putc(m, '\n');
359 }
360
361 static int show_map(struct seq_file *m, void *v)
362 {
363 show_map_vma(m, v);
364 return 0;
365 }
366
367 static const struct seq_operations proc_pid_maps_op = {
368 .start = m_start,
369 .next = m_next,
370 .stop = m_stop,
371 .show = show_map
372 };
373
374 static int pid_maps_open(struct inode *inode, struct file *file)
375 {
376 return do_maps_open(inode, file, &proc_pid_maps_op);
377 }
378
379 #define PROCMAP_QUERY_VMA_FLAGS ( \
380 PROCMAP_QUERY_VMA_READABLE | \
381 PROCMAP_QUERY_VMA_WRITABLE | \
382 PROCMAP_QUERY_VMA_EXECUTABLE | \
383 PROCMAP_QUERY_VMA_SHARED \
384 )
385
386 #define PROCMAP_QUERY_VALID_FLAGS_MASK ( \
387 PROCMAP_QUERY_COVERING_OR_NEXT_VMA | \
388 PROCMAP_QUERY_FILE_BACKED_VMA | \
389 PROCMAP_QUERY_VMA_FLAGS \
390 )
391
392 static int query_vma_setup(struct mm_struct *mm)
393 {
394 return mmap_read_lock_killable(mm);
395 }
396
397 static void query_vma_teardown(struct mm_struct *mm, struct vm_area_struct *vma)
398 {
399 mmap_read_unlock(mm);
400 }
401
402 static struct vm_area_struct *query_vma_find_by_addr(struct mm_struct *mm, unsigned long addr)
403 {
404 return find_vma(mm, addr);
405 }
406
407 static struct vm_area_struct *query_matching_vma(struct mm_struct *mm,
408 unsigned long addr, u32 flags)
409 {
410 struct vm_area_struct *vma;
411
412 next_vma:
413 vma = query_vma_find_by_addr(mm, addr);
414 if (!vma)
415 goto no_vma;
416
417 /* user requested only file-backed VMA, keep iterating */
418 if ((flags & PROCMAP_QUERY_FILE_BACKED_VMA) && !vma->vm_file)
419 goto skip_vma;
420
421 /* VMA permissions should satisfy query flags */
422 if (flags & PROCMAP_QUERY_VMA_FLAGS) {
423 u32 perm = 0;
424
425 if (flags & PROCMAP_QUERY_VMA_READABLE)
426 perm |= VM_READ;
427 if (flags & PROCMAP_QUERY_VMA_WRITABLE)
428 perm |= VM_WRITE;
429 if (flags & PROCMAP_QUERY_VMA_EXECUTABLE)
430 perm |= VM_EXEC;
431 if (flags & PROCMAP_QUERY_VMA_SHARED)
432 perm |= VM_MAYSHARE;
433
434 if ((vma->vm_flags & perm) != perm)
435 goto skip_vma;
436 }
437
438 /* found covering VMA or user is OK with the matching next VMA */
439 if ((flags & PROCMAP_QUERY_COVERING_OR_NEXT_VMA) || vma->vm_start <= addr)
440 return vma;
441
442 skip_vma:
443 /*
444 * If the user needs closest matching VMA, keep iterating.
445 */
446 addr = vma->vm_end;
447 if (flags & PROCMAP_QUERY_COVERING_OR_NEXT_VMA)
448 goto next_vma;
449
450 no_vma:
451 return ERR_PTR(-ENOENT);
452 }
453
454 static int do_procmap_query(struct proc_maps_private *priv, void __user *uarg)
455 {
456 struct procmap_query karg;
457 struct vm_area_struct *vma;
458 struct mm_struct *mm;
459 const char *name = NULL;
460 char build_id_buf[BUILD_ID_SIZE_MAX], *name_buf = NULL;
461 __u64 usize;
462 int err;
463
464 if (copy_from_user(&usize, (void __user *)uarg, sizeof(usize)))
465 return -EFAULT;
466 /* argument struct can never be that large, reject abuse */
467 if (usize > PAGE_SIZE)
468 return -E2BIG;
469 /* argument struct should have at least query_flags and query_addr fields */
470 if (usize < offsetofend(struct procmap_query, query_addr))
471 return -EINVAL;
472 err = copy_struct_from_user(&karg, sizeof(karg), uarg, usize);
473 if (err)
474 return err;
475
476 /* reject unknown flags */
477 if (karg.query_flags & ~PROCMAP_QUERY_VALID_FLAGS_MASK)
478 return -EINVAL;
479 /* either both buffer address and size are set, or both should be zero */
480 if (!!karg.vma_name_size != !!karg.vma_name_addr)
481 return -EINVAL;
482 if (!!karg.build_id_size != !!karg.build_id_addr)
483 return -EINVAL;
484
485 mm = priv->mm;
486 if (!mm || !mmget_not_zero(mm))
487 return -ESRCH;
488
489 err = query_vma_setup(mm);
490 if (err) {
491 mmput(mm);
492 return err;
493 }
494
495 vma = query_matching_vma(mm, karg.query_addr, karg.query_flags);
496 if (IS_ERR(vma)) {
497 err = PTR_ERR(vma);
498 vma = NULL;
499 goto out;
500 }
501
502 karg.vma_start = vma->vm_start;
503 karg.vma_end = vma->vm_end;
504
505 karg.vma_flags = 0;
506 if (vma->vm_flags & VM_READ)
507 karg.vma_flags |= PROCMAP_QUERY_VMA_READABLE;
508 if (vma->vm_flags & VM_WRITE)
509 karg.vma_flags |= PROCMAP_QUERY_VMA_WRITABLE;
510 if (vma->vm_flags & VM_EXEC)
511 karg.vma_flags |= PROCMAP_QUERY_VMA_EXECUTABLE;
512 if (vma->vm_flags & VM_MAYSHARE)
513 karg.vma_flags |= PROCMAP_QUERY_VMA_SHARED;
514
515 karg.vma_page_size = vma_kernel_pagesize(vma);
516
517 if (vma->vm_file) {
518 const struct inode *inode = file_user_inode(vma->vm_file);
519
520 karg.vma_offset = ((__u64)vma->vm_pgoff) << PAGE_SHIFT;
521 karg.dev_major = MAJOR(inode->i_sb->s_dev);
522 karg.dev_minor = MINOR(inode->i_sb->s_dev);
523 karg.inode = inode->i_ino;
524 } else {
525 karg.vma_offset = 0;
526 karg.dev_major = 0;
527 karg.dev_minor = 0;
528 karg.inode = 0;
529 }
530
531 if (karg.build_id_size) {
532 __u32 build_id_sz;
533
534 err = build_id_parse(vma, build_id_buf, &build_id_sz);
535 if (err) {
536 karg.build_id_size = 0;
537 } else {
538 if (karg.build_id_size < build_id_sz) {
539 err = -ENAMETOOLONG;
540 goto out;
541 }
542 karg.build_id_size = build_id_sz;
543 }
544 }
545
546 if (karg.build_id_size) {
547 __u32 build_id_sz;
548
549 err = build_id_parse(vma, build_id_buf, &build_id_sz);
550 if (err) {
551 karg.build_id_size = 0;
552 } else {
553 if (karg.build_id_size < build_id_sz) {
554 err = -ENAMETOOLONG;
555 goto out;
556 }
557 karg.build_id_size = build_id_sz;
558 }
559 }
560
561 if (karg.vma_name_size) {
562 size_t name_buf_sz = min_t(size_t, PATH_MAX, karg.vma_name_size);
563 const struct path *path;
564 const char *name_fmt;
565 size_t name_sz = 0;
566
567 get_vma_name(vma, &path, &name, &name_fmt);
568
569 if (path || name_fmt || name) {
570 name_buf = kmalloc(name_buf_sz, GFP_KERNEL);
571 if (!name_buf) {
572 err = -ENOMEM;
573 goto out;
574 }
575 }
576 if (path) {
577 name = d_path(path, name_buf, name_buf_sz);
578 if (IS_ERR(name)) {
579 err = PTR_ERR(name);
580 goto out;
581 }
582 name_sz = name_buf + name_buf_sz - name;
583 } else if (name || name_fmt) {
584 name_sz = 1 + snprintf(name_buf, name_buf_sz, name_fmt ?: "%s", name);
585 name = name_buf;
586 }
587 if (name_sz > name_buf_sz) {
588 err = -ENAMETOOLONG;
589 goto out;
590 }
591 karg.vma_name_size = name_sz;
592 }
593
594 /* unlock vma or mmap_lock, and put mm_struct before copying data to user */
595 query_vma_teardown(mm, vma);
596 mmput(mm);
597
598 if (karg.vma_name_size && copy_to_user(u64_to_user_ptr(karg.vma_name_addr),
599 name, karg.vma_name_size)) {
600 kfree(name_buf);
601 return -EFAULT;
602 }
603 kfree(name_buf);
604
605 if (karg.build_id_size && copy_to_user(u64_to_user_ptr(karg.build_id_addr),
606 build_id_buf, karg.build_id_size))
607 return -EFAULT;
608
609 if (copy_to_user(uarg, &karg, min_t(size_t, sizeof(karg), usize)))
610 return -EFAULT;
611
612 return 0;
613
614 out:
615 query_vma_teardown(mm, vma);
616 mmput(mm);
617 kfree(name_buf);
618 return err;
619 }
620
621 static long procfs_procmap_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
622 {
623 struct seq_file *seq = file->private_data;
624 struct proc_maps_private *priv = seq->private;
625
626 switch (cmd) {
627 case PROCMAP_QUERY:
628 return do_procmap_query(priv, (void __user *)arg);
629 default:
630 return -ENOIOCTLCMD;
631 }
632 }
633
634 const struct file_operations proc_pid_maps_operations = {
635 .open = pid_maps_open,
636 .read = seq_read,
637 .llseek = seq_lseek,
638 .release = proc_map_release,
639 .unlocked_ioctl = procfs_procmap_ioctl,
640 .compat_ioctl = compat_ptr_ioctl,
641 };
642
643 /*
644 * Proportional Set Size(PSS): my share of RSS.
645 *
646 * PSS of a process is the count of pages it has in memory, where each
647 * page is divided by the number of processes sharing it. So if a
648 * process has 1000 pages all to itself, and 1000 shared with one other
649 * process, its PSS will be 1500.
650 *
651 * To keep (accumulated) division errors low, we adopt a 64bit
652 * fixed-point pss counter to minimize division errors. So (pss >>
653 * PSS_SHIFT) would be the real byte count.
654 *
655 * A shift of 12 before division means (assuming 4K page size):
656 * - 1M 3-user-pages add up to 8KB errors;
657 * - supports mapcount up to 2^24, or 16M;
658 * - supports PSS up to 2^52 bytes, or 4PB.
659 */
660 #define PSS_SHIFT 12
661
662 #ifdef CONFIG_PROC_PAGE_MONITOR
663 struct mem_size_stats {
664 unsigned long resident;
665 unsigned long shared_clean;
666 unsigned long shared_dirty;
667 unsigned long private_clean;
668 unsigned long private_dirty;
669 unsigned long referenced;
670 unsigned long anonymous;
671 unsigned long lazyfree;
672 unsigned long anonymous_thp;
673 unsigned long shmem_thp;
674 unsigned long file_thp;
675 unsigned long swap;
676 unsigned long shared_hugetlb;
677 unsigned long private_hugetlb;
678 unsigned long ksm;
679 u64 pss;
680 u64 pss_anon;
681 u64 pss_file;
682 u64 pss_shmem;
683 u64 pss_dirty;
684 u64 pss_locked;
685 u64 swap_pss;
686 };
687
688 static void smaps_page_accumulate(struct mem_size_stats *mss,
689 struct folio *folio, unsigned long size, unsigned long pss,
690 bool dirty, bool locked, bool private)
691 {
692 mss->pss += pss;
693
694 if (folio_test_anon(folio))
695 mss->pss_anon += pss;
696 else if (folio_test_swapbacked(folio))
697 mss->pss_shmem += pss;
698 else
699 mss->pss_file += pss;
700
701 if (locked)
702 mss->pss_locked += pss;
703
704 if (dirty || folio_test_dirty(folio)) {
705 mss->pss_dirty += pss;
706 if (private)
707 mss->private_dirty += size;
708 else
709 mss->shared_dirty += size;
710 } else {
711 if (private)
712 mss->private_clean += size;
713 else
714 mss->shared_clean += size;
715 }
716 }
717
718 static void smaps_account(struct mem_size_stats *mss, struct page *page,
719 bool compound, bool young, bool dirty, bool locked,
720 bool present)
721 {
722 struct folio *folio = page_folio(page);
723 int i, nr = compound ? compound_nr(page) : 1;
724 unsigned long size = nr * PAGE_SIZE;
725
726 /*
727 * First accumulate quantities that depend only on |size| and the type
728 * of the compound page.
729 */
730 if (folio_test_anon(folio)) {
731 mss->anonymous += size;
732 if (!folio_test_swapbacked(folio) && !dirty &&
733 !folio_test_dirty(folio))
734 mss->lazyfree += size;
735 }
736
737 if (folio_test_ksm(folio))
738 mss->ksm += size;
739
740 mss->resident += size;
741 /* Accumulate the size in pages that have been accessed. */
742 if (young || folio_test_young(folio) || folio_test_referenced(folio))
743 mss->referenced += size;
744
745 /*
746 * Then accumulate quantities that may depend on sharing, or that may
747 * differ page-by-page.
748 *
749 * refcount == 1 for present entries guarantees that the folio is mapped
750 * exactly once. For large folios this implies that exactly one
751 * PTE/PMD/... maps (a part of) this folio.
752 *
753 * Treat all non-present entries (where relying on the mapcount and
754 * refcount doesn't make sense) as "maybe shared, but not sure how
755 * often". We treat device private entries as being fake-present.
756 *
757 * Note that it would not be safe to read the mapcount especially for
758 * pages referenced by migration entries, even with the PTL held.
759 */
760 if (folio_ref_count(folio) == 1 || !present) {
761 smaps_page_accumulate(mss, folio, size, size << PSS_SHIFT,
762 dirty, locked, present);
763 return;
764 }
765 /*
766 * We obtain a snapshot of the mapcount. Without holding the folio lock
767 * this snapshot can be slightly wrong as we cannot always read the
768 * mapcount atomically.
769 */
770 for (i = 0; i < nr; i++, page++) {
771 int mapcount = folio_precise_page_mapcount(folio, page);
772 unsigned long pss = PAGE_SIZE << PSS_SHIFT;
773 if (mapcount >= 2)
774 pss /= mapcount;
775 smaps_page_accumulate(mss, folio, PAGE_SIZE, pss,
776 dirty, locked, mapcount < 2);
777 }
778 }
779
780 #ifdef CONFIG_SHMEM
781 static int smaps_pte_hole(unsigned long addr, unsigned long end,
782 __always_unused int depth, struct mm_walk *walk)
783 {
784 struct mem_size_stats *mss = walk->private;
785 struct vm_area_struct *vma = walk->vma;
786
787 mss->swap += shmem_partial_swap_usage(walk->vma->vm_file->f_mapping,
788 linear_page_index(vma, addr),
789 linear_page_index(vma, end));
790
791 return 0;
792 }
793 #else
794 #define smaps_pte_hole NULL
795 #endif /* CONFIG_SHMEM */
796
797 static void smaps_pte_hole_lookup(unsigned long addr, struct mm_walk *walk)
798 {
799 #ifdef CONFIG_SHMEM
800 if (walk->ops->pte_hole) {
801 /* depth is not used */
802 smaps_pte_hole(addr, addr + PAGE_SIZE, 0, walk);
803 }
804 #endif
805 }
806
807 static void smaps_pte_entry(pte_t *pte, unsigned long addr,
808 struct mm_walk *walk)
809 {
810 struct mem_size_stats *mss = walk->private;
811 struct vm_area_struct *vma = walk->vma;
812 bool locked = !!(vma->vm_flags & VM_LOCKED);
813 struct page *page = NULL;
814 bool present = false, young = false, dirty = false;
815 pte_t ptent = ptep_get(pte);
816
817 if (pte_present(ptent)) {
818 page = vm_normal_page(vma, addr, ptent);
819 young = pte_young(ptent);
820 dirty = pte_dirty(ptent);
821 present = true;
822 } else if (is_swap_pte(ptent)) {
823 swp_entry_t swpent = pte_to_swp_entry(ptent);
824
825 if (!non_swap_entry(swpent)) {
826 int mapcount;
827
828 mss->swap += PAGE_SIZE;
829 mapcount = swp_swapcount(swpent);
830 if (mapcount >= 2) {
831 u64 pss_delta = (u64)PAGE_SIZE << PSS_SHIFT;
832
833 do_div(pss_delta, mapcount);
834 mss->swap_pss += pss_delta;
835 } else {
836 mss->swap_pss += (u64)PAGE_SIZE << PSS_SHIFT;
837 }
838 } else if (is_pfn_swap_entry(swpent)) {
839 if (is_device_private_entry(swpent))
840 present = true;
841 page = pfn_swap_entry_to_page(swpent);
842 }
843 } else {
844 smaps_pte_hole_lookup(addr, walk);
845 return;
846 }
847
848 if (!page)
849 return;
850
851 smaps_account(mss, page, false, young, dirty, locked, present);
852 }
853
854 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
855 static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr,
856 struct mm_walk *walk)
857 {
858 struct mem_size_stats *mss = walk->private;
859 struct vm_area_struct *vma = walk->vma;
860 bool locked = !!(vma->vm_flags & VM_LOCKED);
861 struct page *page = NULL;
862 bool present = false;
863 struct folio *folio;
864
865 if (pmd_present(*pmd)) {
866 page = vm_normal_page_pmd(vma, addr, *pmd);
867 present = true;
868 } else if (unlikely(thp_migration_supported() && is_swap_pmd(*pmd))) {
869 swp_entry_t entry = pmd_to_swp_entry(*pmd);
870
871 if (is_pfn_swap_entry(entry))
872 page = pfn_swap_entry_to_page(entry);
873 }
874 if (IS_ERR_OR_NULL(page))
875 return;
876 folio = page_folio(page);
877 if (folio_test_anon(folio))
878 mss->anonymous_thp += HPAGE_PMD_SIZE;
879 else if (folio_test_swapbacked(folio))
880 mss->shmem_thp += HPAGE_PMD_SIZE;
881 else if (folio_is_zone_device(folio))
882 /* pass */;
883 else
884 mss->file_thp += HPAGE_PMD_SIZE;
885
886 smaps_account(mss, page, true, pmd_young(*pmd), pmd_dirty(*pmd),
887 locked, present);
888 }
889 #else
890 static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr,
891 struct mm_walk *walk)
892 {
893 }
894 #endif
895
896 static int smaps_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
897 struct mm_walk *walk)
898 {
899 struct vm_area_struct *vma = walk->vma;
900 pte_t *pte;
901 spinlock_t *ptl;
902
903 ptl = pmd_trans_huge_lock(pmd, vma);
904 if (ptl) {
905 smaps_pmd_entry(pmd, addr, walk);
906 spin_unlock(ptl);
907 goto out;
908 }
909
910 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
911 if (!pte) {
912 walk->action = ACTION_AGAIN;
913 return 0;
914 }
915 for (; addr != end; pte++, addr += PAGE_SIZE)
916 smaps_pte_entry(pte, addr, walk);
917 pte_unmap_unlock(pte - 1, ptl);
918 out:
919 cond_resched();
920 return 0;
921 }
922
923 static void show_smap_vma_flags(struct seq_file *m, struct vm_area_struct *vma)
924 {
925 /*
926 * Don't forget to update Documentation/ on changes.
927 */
928 static const char mnemonics[BITS_PER_LONG][2] = {
929 /*
930 * In case if we meet a flag we don't know about.
931 */
932 [0 ... (BITS_PER_LONG-1)] = "??",
933
934 [ilog2(VM_READ)] = "rd",
935 [ilog2(VM_WRITE)] = "wr",
936 [ilog2(VM_EXEC)] = "ex",
937 [ilog2(VM_SHARED)] = "sh",
938 [ilog2(VM_MAYREAD)] = "mr",
939 [ilog2(VM_MAYWRITE)] = "mw",
940 [ilog2(VM_MAYEXEC)] = "me",
941 [ilog2(VM_MAYSHARE)] = "ms",
942 [ilog2(VM_GROWSDOWN)] = "gd",
943 [ilog2(VM_PFNMAP)] = "pf",
944 [ilog2(VM_LOCKED)] = "lo",
945 [ilog2(VM_IO)] = "io",
946 [ilog2(VM_SEQ_READ)] = "sr",
947 [ilog2(VM_RAND_READ)] = "rr",
948 [ilog2(VM_DONTCOPY)] = "dc",
949 [ilog2(VM_DONTEXPAND)] = "de",
950 [ilog2(VM_LOCKONFAULT)] = "lf",
951 [ilog2(VM_ACCOUNT)] = "ac",
952 [ilog2(VM_NORESERVE)] = "nr",
953 [ilog2(VM_HUGETLB)] = "ht",
954 [ilog2(VM_SYNC)] = "sf",
955 [ilog2(VM_ARCH_1)] = "ar",
956 [ilog2(VM_WIPEONFORK)] = "wf",
957 [ilog2(VM_DONTDUMP)] = "dd",
958 #ifdef CONFIG_ARM64_BTI
959 [ilog2(VM_ARM64_BTI)] = "bt",
960 #endif
961 #ifdef CONFIG_MEM_SOFT_DIRTY
962 [ilog2(VM_SOFTDIRTY)] = "sd",
963 #endif
964 [ilog2(VM_MIXEDMAP)] = "mm",
965 [ilog2(VM_HUGEPAGE)] = "hg",
966 [ilog2(VM_NOHUGEPAGE)] = "nh",
967 [ilog2(VM_MERGEABLE)] = "mg",
968 [ilog2(VM_UFFD_MISSING)]= "um",
969 [ilog2(VM_UFFD_WP)] = "uw",
970 #ifdef CONFIG_ARM64_MTE
971 [ilog2(VM_MTE)] = "mt",
972 [ilog2(VM_MTE_ALLOWED)] = "",
973 #endif
974 #ifdef CONFIG_ARCH_HAS_PKEYS
975 /* These come out via ProtectionKey: */
976 [ilog2(VM_PKEY_BIT0)] = "",
977 [ilog2(VM_PKEY_BIT1)] = "",
978 [ilog2(VM_PKEY_BIT2)] = "",
979 [ilog2(VM_PKEY_BIT3)] = "",
980 #if VM_PKEY_BIT4
981 [ilog2(VM_PKEY_BIT4)] = "",
982 #endif
983 #endif /* CONFIG_ARCH_HAS_PKEYS */
984 #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR
985 [ilog2(VM_UFFD_MINOR)] = "ui",
986 #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_MINOR */
987 #ifdef CONFIG_X86_USER_SHADOW_STACK
988 [ilog2(VM_SHADOW_STACK)] = "ss",
989 #endif
990 #ifdef CONFIG_64BIT
991 [ilog2(VM_DROPPABLE)] = "dp",
992 [ilog2(VM_SEALED)] = "sl",
993 #endif
994 };
995 size_t i;
996
997 seq_puts(m, "VmFlags: ");
998 for (i = 0; i < BITS_PER_LONG; i++) {
999 if (!mnemonics[i][0])
1000 continue;
1001 if (vma->vm_flags & (1UL << i)) {
1002 seq_putc(m, mnemonics[i][0]);
1003 seq_putc(m, mnemonics[i][1]);
1004 seq_putc(m, ' ');
1005 }
1006 }
1007 seq_putc(m, '\n');
1008 }
1009
1010 #ifdef CONFIG_HUGETLB_PAGE
1011 static int smaps_hugetlb_range(pte_t *pte, unsigned long hmask,
1012 unsigned long addr, unsigned long end,
1013 struct mm_walk *walk)
1014 {
1015 struct mem_size_stats *mss = walk->private;
1016 struct vm_area_struct *vma = walk->vma;
1017 pte_t ptent = huge_ptep_get(walk->mm, addr, pte);
1018 struct folio *folio = NULL;
1019 bool present = false;
1020
1021 if (pte_present(ptent)) {
1022 folio = page_folio(pte_page(ptent));
1023 present = true;
1024 } else if (is_swap_pte(ptent)) {
1025 swp_entry_t swpent = pte_to_swp_entry(ptent);
1026
1027 if (is_pfn_swap_entry(swpent))
1028 folio = pfn_swap_entry_folio(swpent);
1029 }
1030
1031 if (folio) {
1032 /* We treat non-present entries as "maybe shared". */
1033 if (!present || folio_likely_mapped_shared(folio) ||
1034 hugetlb_pmd_shared(pte))
1035 mss->shared_hugetlb += huge_page_size(hstate_vma(vma));
1036 else
1037 mss->private_hugetlb += huge_page_size(hstate_vma(vma));
1038 }
1039 return 0;
1040 }
1041 #else
1042 #define smaps_hugetlb_range NULL
1043 #endif /* HUGETLB_PAGE */
1044
1045 static const struct mm_walk_ops smaps_walk_ops = {
1046 .pmd_entry = smaps_pte_range,
1047 .hugetlb_entry = smaps_hugetlb_range,
1048 .walk_lock = PGWALK_RDLOCK,
1049 };
1050
1051 static const struct mm_walk_ops smaps_shmem_walk_ops = {
1052 .pmd_entry = smaps_pte_range,
1053 .hugetlb_entry = smaps_hugetlb_range,
1054 .pte_hole = smaps_pte_hole,
1055 .walk_lock = PGWALK_RDLOCK,
1056 };
1057
1058 /*
1059 * Gather mem stats from @vma with the indicated beginning
1060 * address @start, and keep them in @mss.
1061 *
1062 * Use vm_start of @vma as the beginning address if @start is 0.
1063 */
1064 static void smap_gather_stats(struct vm_area_struct *vma,
1065 struct mem_size_stats *mss, unsigned long start)
1066 {
1067 const struct mm_walk_ops *ops = &smaps_walk_ops;
1068
1069 /* Invalid start */
1070 if (start >= vma->vm_end)
1071 return;
1072
1073 if (vma->vm_file && shmem_mapping(vma->vm_file->f_mapping)) {
1074 /*
1075 * For shared or readonly shmem mappings we know that all
1076 * swapped out pages belong to the shmem object, and we can
1077 * obtain the swap value much more efficiently. For private
1078 * writable mappings, we might have COW pages that are
1079 * not affected by the parent swapped out pages of the shmem
1080 * object, so we have to distinguish them during the page walk.
1081 * Unless we know that the shmem object (or the part mapped by
1082 * our VMA) has no swapped out pages at all.
1083 */
1084 unsigned long shmem_swapped = shmem_swap_usage(vma);
1085
1086 if (!start && (!shmem_swapped || (vma->vm_flags & VM_SHARED) ||
1087 !(vma->vm_flags & VM_WRITE))) {
1088 mss->swap += shmem_swapped;
1089 } else {
1090 ops = &smaps_shmem_walk_ops;
1091 }
1092 }
1093
1094 /* mmap_lock is held in m_start */
1095 if (!start)
1096 walk_page_vma(vma, ops, mss);
1097 else
1098 walk_page_range(vma->vm_mm, start, vma->vm_end, ops, mss);
1099 }
1100
1101 #define SEQ_PUT_DEC(str, val) \
1102 seq_put_decimal_ull_width(m, str, (val) >> 10, 8)
1103
1104 /* Show the contents common for smaps and smaps_rollup */
1105 static void __show_smap(struct seq_file *m, const struct mem_size_stats *mss,
1106 bool rollup_mode)
1107 {
1108 SEQ_PUT_DEC("Rss: ", mss->resident);
1109 SEQ_PUT_DEC(" kB\nPss: ", mss->pss >> PSS_SHIFT);
1110 SEQ_PUT_DEC(" kB\nPss_Dirty: ", mss->pss_dirty >> PSS_SHIFT);
1111 if (rollup_mode) {
1112 /*
1113 * These are meaningful only for smaps_rollup, otherwise two of
1114 * them are zero, and the other one is the same as Pss.
1115 */
1116 SEQ_PUT_DEC(" kB\nPss_Anon: ",
1117 mss->pss_anon >> PSS_SHIFT);
1118 SEQ_PUT_DEC(" kB\nPss_File: ",
1119 mss->pss_file >> PSS_SHIFT);
1120 SEQ_PUT_DEC(" kB\nPss_Shmem: ",
1121 mss->pss_shmem >> PSS_SHIFT);
1122 }
1123 SEQ_PUT_DEC(" kB\nShared_Clean: ", mss->shared_clean);
1124 SEQ_PUT_DEC(" kB\nShared_Dirty: ", mss->shared_dirty);
1125 SEQ_PUT_DEC(" kB\nPrivate_Clean: ", mss->private_clean);
1126 SEQ_PUT_DEC(" kB\nPrivate_Dirty: ", mss->private_dirty);
1127 SEQ_PUT_DEC(" kB\nReferenced: ", mss->referenced);
1128 SEQ_PUT_DEC(" kB\nAnonymous: ", mss->anonymous);
1129 SEQ_PUT_DEC(" kB\nKSM: ", mss->ksm);
1130 SEQ_PUT_DEC(" kB\nLazyFree: ", mss->lazyfree);
1131 SEQ_PUT_DEC(" kB\nAnonHugePages: ", mss->anonymous_thp);
1132 SEQ_PUT_DEC(" kB\nShmemPmdMapped: ", mss->shmem_thp);
1133 SEQ_PUT_DEC(" kB\nFilePmdMapped: ", mss->file_thp);
1134 SEQ_PUT_DEC(" kB\nShared_Hugetlb: ", mss->shared_hugetlb);
1135 seq_put_decimal_ull_width(m, " kB\nPrivate_Hugetlb: ",
1136 mss->private_hugetlb >> 10, 7);
1137 SEQ_PUT_DEC(" kB\nSwap: ", mss->swap);
1138 SEQ_PUT_DEC(" kB\nSwapPss: ",
1139 mss->swap_pss >> PSS_SHIFT);
1140 SEQ_PUT_DEC(" kB\nLocked: ",
1141 mss->pss_locked >> PSS_SHIFT);
1142 seq_puts(m, " kB\n");
1143 }
1144
1145 static int show_smap(struct seq_file *m, void *v)
1146 {
1147 struct vm_area_struct *vma = v;
1148 struct mem_size_stats mss = {};
1149
1150 smap_gather_stats(vma, &mss, 0);
1151
1152 show_map_vma(m, vma);
1153
1154 SEQ_PUT_DEC("Size: ", vma->vm_end - vma->vm_start);
1155 SEQ_PUT_DEC(" kB\nKernelPageSize: ", vma_kernel_pagesize(vma));
1156 SEQ_PUT_DEC(" kB\nMMUPageSize: ", vma_mmu_pagesize(vma));
1157 seq_puts(m, " kB\n");
1158
1159 __show_smap(m, &mss, false);
1160
1161 seq_printf(m, "THPeligible: %8u\n",
1162 !!thp_vma_allowable_orders(vma, vma->vm_flags,
1163 TVA_SMAPS | TVA_ENFORCE_SYSFS, THP_ORDERS_ALL));
1164
1165 if (arch_pkeys_enabled())
1166 seq_printf(m, "ProtectionKey: %8u\n", vma_pkey(vma));
1167 show_smap_vma_flags(m, vma);
1168
1169 return 0;
1170 }
1171
1172 static int show_smaps_rollup(struct seq_file *m, void *v)
1173 {
1174 struct proc_maps_private *priv = m->private;
1175 struct mem_size_stats mss = {};
1176 struct mm_struct *mm = priv->mm;
1177 struct vm_area_struct *vma;
1178 unsigned long vma_start = 0, last_vma_end = 0;
1179 int ret = 0;
1180 VMA_ITERATOR(vmi, mm, 0);
1181
1182 priv->task = get_proc_task(priv->inode);
1183 if (!priv->task)
1184 return -ESRCH;
1185
1186 if (!mm || !mmget_not_zero(mm)) {
1187 ret = -ESRCH;
1188 goto out_put_task;
1189 }
1190
1191 ret = mmap_read_lock_killable(mm);
1192 if (ret)
1193 goto out_put_mm;
1194
1195 hold_task_mempolicy(priv);
1196 vma = vma_next(&vmi);
1197
1198 if (unlikely(!vma))
1199 goto empty_set;
1200
1201 vma_start = vma->vm_start;
1202 do {
1203 smap_gather_stats(vma, &mss, 0);
1204 last_vma_end = vma->vm_end;
1205
1206 /*
1207 * Release mmap_lock temporarily if someone wants to
1208 * access it for write request.
1209 */
1210 if (mmap_lock_is_contended(mm)) {
1211 vma_iter_invalidate(&vmi);
1212 mmap_read_unlock(mm);
1213 ret = mmap_read_lock_killable(mm);
1214 if (ret) {
1215 release_task_mempolicy(priv);
1216 goto out_put_mm;
1217 }
1218
1219 /*
1220 * After dropping the lock, there are four cases to
1221 * consider. See the following example for explanation.
1222 *
1223 * +------+------+-----------+
1224 * | VMA1 | VMA2 | VMA3 |
1225 * +------+------+-----------+
1226 * | | | |
1227 * 4k 8k 16k 400k
1228 *
1229 * Suppose we drop the lock after reading VMA2 due to
1230 * contention, then we get:
1231 *
1232 * last_vma_end = 16k
1233 *
1234 * 1) VMA2 is freed, but VMA3 exists:
1235 *
1236 * vma_next(vmi) will return VMA3.
1237 * In this case, just continue from VMA3.
1238 *
1239 * 2) VMA2 still exists:
1240 *
1241 * vma_next(vmi) will return VMA3.
1242 * In this case, just continue from VMA3.
1243 *
1244 * 3) No more VMAs can be found:
1245 *
1246 * vma_next(vmi) will return NULL.
1247 * No more things to do, just break.
1248 *
1249 * 4) (last_vma_end - 1) is the middle of a vma (VMA'):
1250 *
1251 * vma_next(vmi) will return VMA' whose range
1252 * contains last_vma_end.
1253 * Iterate VMA' from last_vma_end.
1254 */
1255 vma = vma_next(&vmi);
1256 /* Case 3 above */
1257 if (!vma)
1258 break;
1259
1260 /* Case 1 and 2 above */
1261 if (vma->vm_start >= last_vma_end) {
1262 smap_gather_stats(vma, &mss, 0);
1263 last_vma_end = vma->vm_end;
1264 continue;
1265 }
1266
1267 /* Case 4 above */
1268 if (vma->vm_end > last_vma_end) {
1269 smap_gather_stats(vma, &mss, last_vma_end);
1270 last_vma_end = vma->vm_end;
1271 }
1272 }
1273 } for_each_vma(vmi, vma);
1274
1275 empty_set:
1276 show_vma_header_prefix(m, vma_start, last_vma_end, 0, 0, 0, 0);
1277 seq_pad(m, ' ');
1278 seq_puts(m, "[rollup]\n");
1279
1280 __show_smap(m, &mss, true);
1281
1282 release_task_mempolicy(priv);
1283 mmap_read_unlock(mm);
1284
1285 out_put_mm:
1286 mmput(mm);
1287 out_put_task:
1288 put_task_struct(priv->task);
1289 priv->task = NULL;
1290
1291 return ret;
1292 }
1293 #undef SEQ_PUT_DEC
1294
1295 static const struct seq_operations proc_pid_smaps_op = {
1296 .start = m_start,
1297 .next = m_next,
1298 .stop = m_stop,
1299 .show = show_smap
1300 };
1301
1302 static int pid_smaps_open(struct inode *inode, struct file *file)
1303 {
1304 return do_maps_open(inode, file, &proc_pid_smaps_op);
1305 }
1306
1307 static int smaps_rollup_open(struct inode *inode, struct file *file)
1308 {
1309 int ret;
1310 struct proc_maps_private *priv;
1311
1312 priv = kzalloc(sizeof(*priv), GFP_KERNEL_ACCOUNT);
1313 if (!priv)
1314 return -ENOMEM;
1315
1316 ret = single_open(file, show_smaps_rollup, priv);
1317 if (ret)
1318 goto out_free;
1319
1320 priv->inode = inode;
1321 priv->mm = proc_mem_open(inode, PTRACE_MODE_READ);
1322 if (IS_ERR(priv->mm)) {
1323 ret = PTR_ERR(priv->mm);
1324
1325 single_release(inode, file);
1326 goto out_free;
1327 }
1328
1329 return 0;
1330
1331 out_free:
1332 kfree(priv);
1333 return ret;
1334 }
1335
1336 static int smaps_rollup_release(struct inode *inode, struct file *file)
1337 {
1338 struct seq_file *seq = file->private_data;
1339 struct proc_maps_private *priv = seq->private;
1340
1341 if (priv->mm)
1342 mmdrop(priv->mm);
1343
1344 kfree(priv);
1345 return single_release(inode, file);
1346 }
1347
1348 const struct file_operations proc_pid_smaps_operations = {
1349 .open = pid_smaps_open,
1350 .read = seq_read,
1351 .llseek = seq_lseek,
1352 .release = proc_map_release,
1353 };
1354
1355 const struct file_operations proc_pid_smaps_rollup_operations = {
1356 .open = smaps_rollup_open,
1357 .read = seq_read,
1358 .llseek = seq_lseek,
1359 .release = smaps_rollup_release,
1360 };
1361
1362 enum clear_refs_types {
1363 CLEAR_REFS_ALL = 1,
1364 CLEAR_REFS_ANON,
1365 CLEAR_REFS_MAPPED,
1366 CLEAR_REFS_SOFT_DIRTY,
1367 CLEAR_REFS_MM_HIWATER_RSS,
1368 CLEAR_REFS_LAST,
1369 };
1370
1371 struct clear_refs_private {
1372 enum clear_refs_types type;
1373 };
1374
1375 #ifdef CONFIG_MEM_SOFT_DIRTY
1376
1377 static inline bool pte_is_pinned(struct vm_area_struct *vma, unsigned long addr, pte_t pte)
1378 {
1379 struct folio *folio;
1380
1381 if (!pte_write(pte))
1382 return false;
1383 if (!is_cow_mapping(vma->vm_flags))
1384 return false;
1385 if (likely(!test_bit(MMF_HAS_PINNED, &vma->vm_mm->flags)))
1386 return false;
1387 folio = vm_normal_folio(vma, addr, pte);
1388 if (!folio)
1389 return false;
1390 return folio_maybe_dma_pinned(folio);
1391 }
1392
1393 static inline void clear_soft_dirty(struct vm_area_struct *vma,
1394 unsigned long addr, pte_t *pte)
1395 {
1396 /*
1397 * The soft-dirty tracker uses #PF-s to catch writes
1398 * to pages, so write-protect the pte as well. See the
1399 * Documentation/admin-guide/mm/soft-dirty.rst for full description
1400 * of how soft-dirty works.
1401 */
1402 pte_t ptent = ptep_get(pte);
1403
1404 if (pte_present(ptent)) {
1405 pte_t old_pte;
1406
1407 if (pte_is_pinned(vma, addr, ptent))
1408 return;
1409 old_pte = ptep_modify_prot_start(vma, addr, pte);
1410 ptent = pte_wrprotect(old_pte);
1411 ptent = pte_clear_soft_dirty(ptent);
1412 ptep_modify_prot_commit(vma, addr, pte, old_pte, ptent);
1413 } else if (is_swap_pte(ptent)) {
1414 ptent = pte_swp_clear_soft_dirty(ptent);
1415 set_pte_at(vma->vm_mm, addr, pte, ptent);
1416 }
1417 }
1418 #else
1419 static inline void clear_soft_dirty(struct vm_area_struct *vma,
1420 unsigned long addr, pte_t *pte)
1421 {
1422 }
1423 #endif
1424
1425 #if defined(CONFIG_MEM_SOFT_DIRTY) && defined(CONFIG_TRANSPARENT_HUGEPAGE)
1426 static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma,
1427 unsigned long addr, pmd_t *pmdp)
1428 {
1429 pmd_t old, pmd = *pmdp;
1430
1431 if (pmd_present(pmd)) {
1432 /* See comment in change_huge_pmd() */
1433 old = pmdp_invalidate(vma, addr, pmdp);
1434 if (pmd_dirty(old))
1435 pmd = pmd_mkdirty(pmd);
1436 if (pmd_young(old))
1437 pmd = pmd_mkyoung(pmd);
1438
1439 pmd = pmd_wrprotect(pmd);
1440 pmd = pmd_clear_soft_dirty(pmd);
1441
1442 set_pmd_at(vma->vm_mm, addr, pmdp, pmd);
1443 } else if (is_migration_entry(pmd_to_swp_entry(pmd))) {
1444 pmd = pmd_swp_clear_soft_dirty(pmd);
1445 set_pmd_at(vma->vm_mm, addr, pmdp, pmd);
1446 }
1447 }
1448 #else
1449 static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma,
1450 unsigned long addr, pmd_t *pmdp)
1451 {
1452 }
1453 #endif
1454
1455 static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr,
1456 unsigned long end, struct mm_walk *walk)
1457 {
1458 struct clear_refs_private *cp = walk->private;
1459 struct vm_area_struct *vma = walk->vma;
1460 pte_t *pte, ptent;
1461 spinlock_t *ptl;
1462 struct folio *folio;
1463
1464 ptl = pmd_trans_huge_lock(pmd, vma);
1465 if (ptl) {
1466 if (cp->type == CLEAR_REFS_SOFT_DIRTY) {
1467 clear_soft_dirty_pmd(vma, addr, pmd);
1468 goto out;
1469 }
1470
1471 if (!pmd_present(*pmd))
1472 goto out;
1473
1474 folio = pmd_folio(*pmd);
1475
1476 /* Clear accessed and referenced bits. */
1477 pmdp_test_and_clear_young(vma, addr, pmd);
1478 folio_test_clear_young(folio);
1479 folio_clear_referenced(folio);
1480 out:
1481 spin_unlock(ptl);
1482 return 0;
1483 }
1484
1485 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
1486 if (!pte) {
1487 walk->action = ACTION_AGAIN;
1488 return 0;
1489 }
1490 for (; addr != end; pte++, addr += PAGE_SIZE) {
1491 ptent = ptep_get(pte);
1492
1493 if (cp->type == CLEAR_REFS_SOFT_DIRTY) {
1494 clear_soft_dirty(vma, addr, pte);
1495 continue;
1496 }
1497
1498 if (!pte_present(ptent))
1499 continue;
1500
1501 folio = vm_normal_folio(vma, addr, ptent);
1502 if (!folio)
1503 continue;
1504
1505 /* Clear accessed and referenced bits. */
1506 ptep_test_and_clear_young(vma, addr, pte);
1507 folio_test_clear_young(folio);
1508 folio_clear_referenced(folio);
1509 }
1510 pte_unmap_unlock(pte - 1, ptl);
1511 cond_resched();
1512 return 0;
1513 }
1514
1515 static int clear_refs_test_walk(unsigned long start, unsigned long end,
1516 struct mm_walk *walk)
1517 {
1518 struct clear_refs_private *cp = walk->private;
1519 struct vm_area_struct *vma = walk->vma;
1520
1521 if (vma->vm_flags & VM_PFNMAP)
1522 return 1;
1523
1524 /*
1525 * Writing 1 to /proc/pid/clear_refs affects all pages.
1526 * Writing 2 to /proc/pid/clear_refs only affects anonymous pages.
1527 * Writing 3 to /proc/pid/clear_refs only affects file mapped pages.
1528 * Writing 4 to /proc/pid/clear_refs affects all pages.
1529 */
1530 if (cp->type == CLEAR_REFS_ANON && vma->vm_file)
1531 return 1;
1532 if (cp->type == CLEAR_REFS_MAPPED && !vma->vm_file)
1533 return 1;
1534 return 0;
1535 }
1536
1537 static const struct mm_walk_ops clear_refs_walk_ops = {
1538 .pmd_entry = clear_refs_pte_range,
1539 .test_walk = clear_refs_test_walk,
1540 .walk_lock = PGWALK_WRLOCK,
1541 };
1542
1543 static ssize_t clear_refs_write(struct file *file, const char __user *buf,
1544 size_t count, loff_t *ppos)
1545 {
1546 struct task_struct *task;
1547 char buffer[PROC_NUMBUF] = {};
1548 struct mm_struct *mm;
1549 struct vm_area_struct *vma;
1550 enum clear_refs_types type;
1551 int itype;
1552 int rv;
1553
1554 if (count > sizeof(buffer) - 1)
1555 count = sizeof(buffer) - 1;
1556 if (copy_from_user(buffer, buf, count))
1557 return -EFAULT;
1558 rv = kstrtoint(strstrip(buffer), 10, &itype);
1559 if (rv < 0)
1560 return rv;
1561 type = (enum clear_refs_types)itype;
1562 if (type < CLEAR_REFS_ALL || type >= CLEAR_REFS_LAST)
1563 return -EINVAL;
1564
1565 task = get_proc_task(file_inode(file));
1566 if (!task)
1567 return -ESRCH;
1568 mm = get_task_mm(task);
1569 if (mm) {
1570 VMA_ITERATOR(vmi, mm, 0);
1571 struct mmu_notifier_range range;
1572 struct clear_refs_private cp = {
1573 .type = type,
1574 };
1575
1576 if (mmap_write_lock_killable(mm)) {
1577 count = -EINTR;
1578 goto out_mm;
1579 }
1580 if (type == CLEAR_REFS_MM_HIWATER_RSS) {
1581 /*
1582 * Writing 5 to /proc/pid/clear_refs resets the peak
1583 * resident set size to this mm's current rss value.
1584 */
1585 reset_mm_hiwater_rss(mm);
1586 goto out_unlock;
1587 }
1588
1589 if (type == CLEAR_REFS_SOFT_DIRTY) {
1590 for_each_vma(vmi, vma) {
1591 if (!(vma->vm_flags & VM_SOFTDIRTY))
1592 continue;
1593 vm_flags_clear(vma, VM_SOFTDIRTY);
1594 vma_set_page_prot(vma);
1595 }
1596
1597 inc_tlb_flush_pending(mm);
1598 mmu_notifier_range_init(&range, MMU_NOTIFY_SOFT_DIRTY,
1599 0, mm, 0, -1UL);
1600 mmu_notifier_invalidate_range_start(&range);
1601 }
1602 walk_page_range(mm, 0, -1, &clear_refs_walk_ops, &cp);
1603 if (type == CLEAR_REFS_SOFT_DIRTY) {
1604 mmu_notifier_invalidate_range_end(&range);
1605 flush_tlb_mm(mm);
1606 dec_tlb_flush_pending(mm);
1607 }
1608 out_unlock:
1609 mmap_write_unlock(mm);
1610 out_mm:
1611 mmput(mm);
1612 }
1613 put_task_struct(task);
1614
1615 return count;
1616 }
1617
1618 const struct file_operations proc_clear_refs_operations = {
1619 .write = clear_refs_write,
1620 .llseek = noop_llseek,
1621 };
1622
1623 typedef struct {
1624 u64 pme;
1625 } pagemap_entry_t;
1626
1627 struct pagemapread {
1628 int pos, len; /* units: PM_ENTRY_BYTES, not bytes */
1629 pagemap_entry_t *buffer;
1630 bool show_pfn;
1631 };
1632
1633 #define PAGEMAP_WALK_SIZE (PMD_SIZE)
1634 #define PAGEMAP_WALK_MASK (PMD_MASK)
1635
1636 #define PM_ENTRY_BYTES sizeof(pagemap_entry_t)
1637 #define PM_PFRAME_BITS 55
1638 #define PM_PFRAME_MASK GENMASK_ULL(PM_PFRAME_BITS - 1, 0)
1639 #define PM_SOFT_DIRTY BIT_ULL(55)
1640 #define PM_MMAP_EXCLUSIVE BIT_ULL(56)
1641 #define PM_UFFD_WP BIT_ULL(57)
1642 #define PM_FILE BIT_ULL(61)
1643 #define PM_SWAP BIT_ULL(62)
1644 #define PM_PRESENT BIT_ULL(63)
1645
1646 #define PM_END_OF_BUFFER 1
1647
1648 static inline pagemap_entry_t make_pme(u64 frame, u64 flags)
1649 {
1650 return (pagemap_entry_t) { .pme = (frame & PM_PFRAME_MASK) | flags };
1651 }
1652
1653 static int add_to_pagemap(pagemap_entry_t *pme, struct pagemapread *pm)
1654 {
1655 pm->buffer[pm->pos++] = *pme;
1656 if (pm->pos >= pm->len)
1657 return PM_END_OF_BUFFER;
1658 return 0;
1659 }
1660
1661 static int pagemap_pte_hole(unsigned long start, unsigned long end,
1662 __always_unused int depth, struct mm_walk *walk)
1663 {
1664 struct pagemapread *pm = walk->private;
1665 unsigned long addr = start;
1666 int err = 0;
1667
1668 while (addr < end) {
1669 struct vm_area_struct *vma = find_vma(walk->mm, addr);
1670 pagemap_entry_t pme = make_pme(0, 0);
1671 /* End of address space hole, which we mark as non-present. */
1672 unsigned long hole_end;
1673
1674 if (vma)
1675 hole_end = min(end, vma->vm_start);
1676 else
1677 hole_end = end;
1678
1679 for (; addr < hole_end; addr += PAGE_SIZE) {
1680 err = add_to_pagemap(&pme, pm);
1681 if (err)
1682 goto out;
1683 }
1684
1685 if (!vma)
1686 break;
1687
1688 /* Addresses in the VMA. */
1689 if (vma->vm_flags & VM_SOFTDIRTY)
1690 pme = make_pme(0, PM_SOFT_DIRTY);
1691 for (; addr < min(end, vma->vm_end); addr += PAGE_SIZE) {
1692 err = add_to_pagemap(&pme, pm);
1693 if (err)
1694 goto out;
1695 }
1696 }
1697 out:
1698 return err;
1699 }
1700
1701 static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm,
1702 struct vm_area_struct *vma, unsigned long addr, pte_t pte)
1703 {
1704 u64 frame = 0, flags = 0;
1705 struct page *page = NULL;
1706 struct folio *folio;
1707
1708 if (pte_present(pte)) {
1709 if (pm->show_pfn)
1710 frame = pte_pfn(pte);
1711 flags |= PM_PRESENT;
1712 page = vm_normal_page(vma, addr, pte);
1713 if (pte_soft_dirty(pte))
1714 flags |= PM_SOFT_DIRTY;
1715 if (pte_uffd_wp(pte))
1716 flags |= PM_UFFD_WP;
1717 } else if (is_swap_pte(pte)) {
1718 swp_entry_t entry;
1719 if (pte_swp_soft_dirty(pte))
1720 flags |= PM_SOFT_DIRTY;
1721 if (pte_swp_uffd_wp(pte))
1722 flags |= PM_UFFD_WP;
1723 entry = pte_to_swp_entry(pte);
1724 if (pm->show_pfn) {
1725 pgoff_t offset;
1726 /*
1727 * For PFN swap offsets, keeping the offset field
1728 * to be PFN only to be compatible with old smaps.
1729 */
1730 if (is_pfn_swap_entry(entry))
1731 offset = swp_offset_pfn(entry);
1732 else
1733 offset = swp_offset(entry);
1734 frame = swp_type(entry) |
1735 (offset << MAX_SWAPFILES_SHIFT);
1736 }
1737 flags |= PM_SWAP;
1738 if (is_pfn_swap_entry(entry))
1739 page = pfn_swap_entry_to_page(entry);
1740 if (pte_marker_entry_uffd_wp(entry))
1741 flags |= PM_UFFD_WP;
1742 }
1743
1744 if (page) {
1745 folio = page_folio(page);
1746 if (!folio_test_anon(folio))
1747 flags |= PM_FILE;
1748 if ((flags & PM_PRESENT) &&
1749 folio_precise_page_mapcount(folio, page) == 1)
1750 flags |= PM_MMAP_EXCLUSIVE;
1751 }
1752 if (vma->vm_flags & VM_SOFTDIRTY)
1753 flags |= PM_SOFT_DIRTY;
1754
1755 return make_pme(frame, flags);
1756 }
1757
1758 static int pagemap_pmd_range(pmd_t *pmdp, unsigned long addr, unsigned long end,
1759 struct mm_walk *walk)
1760 {
1761 struct vm_area_struct *vma = walk->vma;
1762 struct pagemapread *pm = walk->private;
1763 spinlock_t *ptl;
1764 pte_t *pte, *orig_pte;
1765 int err = 0;
1766 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1767
1768 ptl = pmd_trans_huge_lock(pmdp, vma);
1769 if (ptl) {
1770 unsigned int idx = (addr & ~PMD_MASK) >> PAGE_SHIFT;
1771 u64 flags = 0, frame = 0;
1772 pmd_t pmd = *pmdp;
1773 struct page *page = NULL;
1774 struct folio *folio = NULL;
1775
1776 if (vma->vm_flags & VM_SOFTDIRTY)
1777 flags |= PM_SOFT_DIRTY;
1778
1779 if (pmd_present(pmd)) {
1780 page = pmd_page(pmd);
1781
1782 flags |= PM_PRESENT;
1783 if (pmd_soft_dirty(pmd))
1784 flags |= PM_SOFT_DIRTY;
1785 if (pmd_uffd_wp(pmd))
1786 flags |= PM_UFFD_WP;
1787 if (pm->show_pfn)
1788 frame = pmd_pfn(pmd) + idx;
1789 }
1790 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
1791 else if (is_swap_pmd(pmd)) {
1792 swp_entry_t entry = pmd_to_swp_entry(pmd);
1793 unsigned long offset;
1794
1795 if (pm->show_pfn) {
1796 if (is_pfn_swap_entry(entry))
1797 offset = swp_offset_pfn(entry) + idx;
1798 else
1799 offset = swp_offset(entry) + idx;
1800 frame = swp_type(entry) |
1801 (offset << MAX_SWAPFILES_SHIFT);
1802 }
1803 flags |= PM_SWAP;
1804 if (pmd_swp_soft_dirty(pmd))
1805 flags |= PM_SOFT_DIRTY;
1806 if (pmd_swp_uffd_wp(pmd))
1807 flags |= PM_UFFD_WP;
1808 VM_BUG_ON(!is_pmd_migration_entry(pmd));
1809 page = pfn_swap_entry_to_page(entry);
1810 }
1811 #endif
1812
1813 if (page) {
1814 folio = page_folio(page);
1815 if (!folio_test_anon(folio))
1816 flags |= PM_FILE;
1817 }
1818
1819 for (; addr != end; addr += PAGE_SIZE, idx++) {
1820 unsigned long cur_flags = flags;
1821 pagemap_entry_t pme;
1822
1823 if (folio && (flags & PM_PRESENT) &&
1824 folio_precise_page_mapcount(folio, page + idx) == 1)
1825 cur_flags |= PM_MMAP_EXCLUSIVE;
1826
1827 pme = make_pme(frame, cur_flags);
1828 err = add_to_pagemap(&pme, pm);
1829 if (err)
1830 break;
1831 if (pm->show_pfn) {
1832 if (flags & PM_PRESENT)
1833 frame++;
1834 else if (flags & PM_SWAP)
1835 frame += (1 << MAX_SWAPFILES_SHIFT);
1836 }
1837 }
1838 spin_unlock(ptl);
1839 return err;
1840 }
1841 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
1842
1843 /*
1844 * We can assume that @vma always points to a valid one and @end never
1845 * goes beyond vma->vm_end.
1846 */
1847 orig_pte = pte = pte_offset_map_lock(walk->mm, pmdp, addr, &ptl);
1848 if (!pte) {
1849 walk->action = ACTION_AGAIN;
1850 return err;
1851 }
1852 for (; addr < end; pte++, addr += PAGE_SIZE) {
1853 pagemap_entry_t pme;
1854
1855 pme = pte_to_pagemap_entry(pm, vma, addr, ptep_get(pte));
1856 err = add_to_pagemap(&pme, pm);
1857 if (err)
1858 break;
1859 }
1860 pte_unmap_unlock(orig_pte, ptl);
1861
1862 cond_resched();
1863
1864 return err;
1865 }
1866
1867 #ifdef CONFIG_HUGETLB_PAGE
1868 /* This function walks within one hugetlb entry in the single call */
1869 static int pagemap_hugetlb_range(pte_t *ptep, unsigned long hmask,
1870 unsigned long addr, unsigned long end,
1871 struct mm_walk *walk)
1872 {
1873 struct pagemapread *pm = walk->private;
1874 struct vm_area_struct *vma = walk->vma;
1875 u64 flags = 0, frame = 0;
1876 int err = 0;
1877 pte_t pte;
1878
1879 if (vma->vm_flags & VM_SOFTDIRTY)
1880 flags |= PM_SOFT_DIRTY;
1881
1882 pte = huge_ptep_get(walk->mm, addr, ptep);
1883 if (pte_present(pte)) {
1884 struct folio *folio = page_folio(pte_page(pte));
1885
1886 if (!folio_test_anon(folio))
1887 flags |= PM_FILE;
1888
1889 if (!folio_likely_mapped_shared(folio) &&
1890 !hugetlb_pmd_shared(ptep))
1891 flags |= PM_MMAP_EXCLUSIVE;
1892
1893 if (huge_pte_uffd_wp(pte))
1894 flags |= PM_UFFD_WP;
1895
1896 flags |= PM_PRESENT;
1897 if (pm->show_pfn)
1898 frame = pte_pfn(pte) +
1899 ((addr & ~hmask) >> PAGE_SHIFT);
1900 } else if (pte_swp_uffd_wp_any(pte)) {
1901 flags |= PM_UFFD_WP;
1902 }
1903
1904 for (; addr != end; addr += PAGE_SIZE) {
1905 pagemap_entry_t pme = make_pme(frame, flags);
1906
1907 err = add_to_pagemap(&pme, pm);
1908 if (err)
1909 return err;
1910 if (pm->show_pfn && (flags & PM_PRESENT))
1911 frame++;
1912 }
1913
1914 cond_resched();
1915
1916 return err;
1917 }
1918 #else
1919 #define pagemap_hugetlb_range NULL
1920 #endif /* HUGETLB_PAGE */
1921
1922 static const struct mm_walk_ops pagemap_ops = {
1923 .pmd_entry = pagemap_pmd_range,
1924 .pte_hole = pagemap_pte_hole,
1925 .hugetlb_entry = pagemap_hugetlb_range,
1926 .walk_lock = PGWALK_RDLOCK,
1927 };
1928
1929 /*
1930 * /proc/pid/pagemap - an array mapping virtual pages to pfns
1931 *
1932 * For each page in the address space, this file contains one 64-bit entry
1933 * consisting of the following:
1934 *
1935 * Bits 0-54 page frame number (PFN) if present
1936 * Bits 0-4 swap type if swapped
1937 * Bits 5-54 swap offset if swapped
1938 * Bit 55 pte is soft-dirty (see Documentation/admin-guide/mm/soft-dirty.rst)
1939 * Bit 56 page exclusively mapped
1940 * Bit 57 pte is uffd-wp write-protected
1941 * Bits 58-60 zero
1942 * Bit 61 page is file-page or shared-anon
1943 * Bit 62 page swapped
1944 * Bit 63 page present
1945 *
1946 * If the page is not present but in swap, then the PFN contains an
1947 * encoding of the swap file number and the page's offset into the
1948 * swap. Unmapped pages return a null PFN. This allows determining
1949 * precisely which pages are mapped (or in swap) and comparing mapped
1950 * pages between processes.
1951 *
1952 * Efficient users of this interface will use /proc/pid/maps to
1953 * determine which areas of memory are actually mapped and llseek to
1954 * skip over unmapped regions.
1955 */
1956 static ssize_t pagemap_read(struct file *file, char __user *buf,
1957 size_t count, loff_t *ppos)
1958 {
1959 struct mm_struct *mm = file->private_data;
1960 struct pagemapread pm;
1961 unsigned long src;
1962 unsigned long svpfn;
1963 unsigned long start_vaddr;
1964 unsigned long end_vaddr;
1965 int ret = 0, copied = 0;
1966
1967 if (!mm || !mmget_not_zero(mm))
1968 goto out;
1969
1970 ret = -EINVAL;
1971 /* file position must be aligned */
1972 if ((*ppos % PM_ENTRY_BYTES) || (count % PM_ENTRY_BYTES))
1973 goto out_mm;
1974
1975 ret = 0;
1976 if (!count)
1977 goto out_mm;
1978
1979 /* do not disclose physical addresses: attack vector */
1980 pm.show_pfn = file_ns_capable(file, &init_user_ns, CAP_SYS_ADMIN);
1981
1982 pm.len = (PAGEMAP_WALK_SIZE >> PAGE_SHIFT);
1983 pm.buffer = kmalloc_array(pm.len, PM_ENTRY_BYTES, GFP_KERNEL);
1984 ret = -ENOMEM;
1985 if (!pm.buffer)
1986 goto out_mm;
1987
1988 src = *ppos;
1989 svpfn = src / PM_ENTRY_BYTES;
1990 end_vaddr = mm->task_size;
1991
1992 /* watch out for wraparound */
1993 start_vaddr = end_vaddr;
1994 if (svpfn <= (ULONG_MAX >> PAGE_SHIFT)) {
1995 unsigned long end;
1996
1997 ret = mmap_read_lock_killable(mm);
1998 if (ret)
1999 goto out_free;
2000 start_vaddr = untagged_addr_remote(mm, svpfn << PAGE_SHIFT);
2001 mmap_read_unlock(mm);
2002
2003 end = start_vaddr + ((count / PM_ENTRY_BYTES) << PAGE_SHIFT);
2004 if (end >= start_vaddr && end < mm->task_size)
2005 end_vaddr = end;
2006 }
2007
2008 /* Ensure the address is inside the task */
2009 if (start_vaddr > mm->task_size)
2010 start_vaddr = end_vaddr;
2011
2012 ret = 0;
2013 while (count && (start_vaddr < end_vaddr)) {
2014 int len;
2015 unsigned long end;
2016
2017 pm.pos = 0;
2018 end = (start_vaddr + PAGEMAP_WALK_SIZE) & PAGEMAP_WALK_MASK;
2019 /* overflow ? */
2020 if (end < start_vaddr || end > end_vaddr)
2021 end = end_vaddr;
2022 ret = mmap_read_lock_killable(mm);
2023 if (ret)
2024 goto out_free;
2025 ret = walk_page_range(mm, start_vaddr, end, &pagemap_ops, &pm);
2026 mmap_read_unlock(mm);
2027 start_vaddr = end;
2028
2029 len = min(count, PM_ENTRY_BYTES * pm.pos);
2030 if (copy_to_user(buf, pm.buffer, len)) {
2031 ret = -EFAULT;
2032 goto out_free;
2033 }
2034 copied += len;
2035 buf += len;
2036 count -= len;
2037 }
2038 *ppos += copied;
2039 if (!ret || ret == PM_END_OF_BUFFER)
2040 ret = copied;
2041
2042 out_free:
2043 kfree(pm.buffer);
2044 out_mm:
2045 mmput(mm);
2046 out:
2047 return ret;
2048 }
2049
2050 static int pagemap_open(struct inode *inode, struct file *file)
2051 {
2052 struct mm_struct *mm;
2053
2054 mm = proc_mem_open(inode, PTRACE_MODE_READ);
2055 if (IS_ERR(mm))
2056 return PTR_ERR(mm);
2057 file->private_data = mm;
2058 return 0;
2059 }
2060
2061 static int pagemap_release(struct inode *inode, struct file *file)
2062 {
2063 struct mm_struct *mm = file->private_data;
2064
2065 if (mm)
2066 mmdrop(mm);
2067 return 0;
2068 }
2069
2070 #define PM_SCAN_CATEGORIES (PAGE_IS_WPALLOWED | PAGE_IS_WRITTEN | \
2071 PAGE_IS_FILE | PAGE_IS_PRESENT | \
2072 PAGE_IS_SWAPPED | PAGE_IS_PFNZERO | \
2073 PAGE_IS_HUGE | PAGE_IS_SOFT_DIRTY)
2074 #define PM_SCAN_FLAGS (PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC)
2075
2076 struct pagemap_scan_private {
2077 struct pm_scan_arg arg;
2078 unsigned long masks_of_interest, cur_vma_category;
2079 struct page_region *vec_buf;
2080 unsigned long vec_buf_len, vec_buf_index, found_pages;
2081 struct page_region __user *vec_out;
2082 };
2083
2084 static unsigned long pagemap_page_category(struct pagemap_scan_private *p,
2085 struct vm_area_struct *vma,
2086 unsigned long addr, pte_t pte)
2087 {
2088 unsigned long categories = 0;
2089
2090 if (pte_present(pte)) {
2091 struct page *page;
2092
2093 categories |= PAGE_IS_PRESENT;
2094 if (!pte_uffd_wp(pte))
2095 categories |= PAGE_IS_WRITTEN;
2096
2097 if (p->masks_of_interest & PAGE_IS_FILE) {
2098 page = vm_normal_page(vma, addr, pte);
2099 if (page && !PageAnon(page))
2100 categories |= PAGE_IS_FILE;
2101 }
2102
2103 if (is_zero_pfn(pte_pfn(pte)))
2104 categories |= PAGE_IS_PFNZERO;
2105 if (pte_soft_dirty(pte))
2106 categories |= PAGE_IS_SOFT_DIRTY;
2107 } else if (is_swap_pte(pte)) {
2108 swp_entry_t swp;
2109
2110 categories |= PAGE_IS_SWAPPED;
2111 if (!pte_swp_uffd_wp_any(pte))
2112 categories |= PAGE_IS_WRITTEN;
2113
2114 if (p->masks_of_interest & PAGE_IS_FILE) {
2115 swp = pte_to_swp_entry(pte);
2116 if (is_pfn_swap_entry(swp) &&
2117 !folio_test_anon(pfn_swap_entry_folio(swp)))
2118 categories |= PAGE_IS_FILE;
2119 }
2120 if (pte_swp_soft_dirty(pte))
2121 categories |= PAGE_IS_SOFT_DIRTY;
2122 }
2123
2124 return categories;
2125 }
2126
2127 static void make_uffd_wp_pte(struct vm_area_struct *vma,
2128 unsigned long addr, pte_t *pte, pte_t ptent)
2129 {
2130 if (pte_present(ptent)) {
2131 pte_t old_pte;
2132
2133 old_pte = ptep_modify_prot_start(vma, addr, pte);
2134 ptent = pte_mkuffd_wp(old_pte);
2135 ptep_modify_prot_commit(vma, addr, pte, old_pte, ptent);
2136 } else if (is_swap_pte(ptent)) {
2137 ptent = pte_swp_mkuffd_wp(ptent);
2138 set_pte_at(vma->vm_mm, addr, pte, ptent);
2139 } else {
2140 set_pte_at(vma->vm_mm, addr, pte,
2141 make_pte_marker(PTE_MARKER_UFFD_WP));
2142 }
2143 }
2144
2145 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
2146 static unsigned long pagemap_thp_category(struct pagemap_scan_private *p,
2147 struct vm_area_struct *vma,
2148 unsigned long addr, pmd_t pmd)
2149 {
2150 unsigned long categories = PAGE_IS_HUGE;
2151
2152 if (pmd_present(pmd)) {
2153 struct page *page;
2154
2155 categories |= PAGE_IS_PRESENT;
2156 if (!pmd_uffd_wp(pmd))
2157 categories |= PAGE_IS_WRITTEN;
2158
2159 if (p->masks_of_interest & PAGE_IS_FILE) {
2160 page = vm_normal_page_pmd(vma, addr, pmd);
2161 if (page && !PageAnon(page))
2162 categories |= PAGE_IS_FILE;
2163 }
2164
2165 if (is_zero_pfn(pmd_pfn(pmd)))
2166 categories |= PAGE_IS_PFNZERO;
2167 if (pmd_soft_dirty(pmd))
2168 categories |= PAGE_IS_SOFT_DIRTY;
2169 } else if (is_swap_pmd(pmd)) {
2170 swp_entry_t swp;
2171
2172 categories |= PAGE_IS_SWAPPED;
2173 if (!pmd_swp_uffd_wp(pmd))
2174 categories |= PAGE_IS_WRITTEN;
2175 if (pmd_swp_soft_dirty(pmd))
2176 categories |= PAGE_IS_SOFT_DIRTY;
2177
2178 if (p->masks_of_interest & PAGE_IS_FILE) {
2179 swp = pmd_to_swp_entry(pmd);
2180 if (is_pfn_swap_entry(swp) &&
2181 !folio_test_anon(pfn_swap_entry_folio(swp)))
2182 categories |= PAGE_IS_FILE;
2183 }
2184 }
2185
2186 return categories;
2187 }
2188
2189 static void make_uffd_wp_pmd(struct vm_area_struct *vma,
2190 unsigned long addr, pmd_t *pmdp)
2191 {
2192 pmd_t old, pmd = *pmdp;
2193
2194 if (pmd_present(pmd)) {
2195 old = pmdp_invalidate_ad(vma, addr, pmdp);
2196 pmd = pmd_mkuffd_wp(old);
2197 set_pmd_at(vma->vm_mm, addr, pmdp, pmd);
2198 } else if (is_migration_entry(pmd_to_swp_entry(pmd))) {
2199 pmd = pmd_swp_mkuffd_wp(pmd);
2200 set_pmd_at(vma->vm_mm, addr, pmdp, pmd);
2201 }
2202 }
2203 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
2204
2205 #ifdef CONFIG_HUGETLB_PAGE
2206 static unsigned long pagemap_hugetlb_category(pte_t pte)
2207 {
2208 unsigned long categories = PAGE_IS_HUGE;
2209
2210 /*
2211 * According to pagemap_hugetlb_range(), file-backed HugeTLB
2212 * page cannot be swapped. So PAGE_IS_FILE is not checked for
2213 * swapped pages.
2214 */
2215 if (pte_present(pte)) {
2216 categories |= PAGE_IS_PRESENT;
2217 if (!huge_pte_uffd_wp(pte))
2218 categories |= PAGE_IS_WRITTEN;
2219 if (!PageAnon(pte_page(pte)))
2220 categories |= PAGE_IS_FILE;
2221 if (is_zero_pfn(pte_pfn(pte)))
2222 categories |= PAGE_IS_PFNZERO;
2223 if (pte_soft_dirty(pte))
2224 categories |= PAGE_IS_SOFT_DIRTY;
2225 } else if (is_swap_pte(pte)) {
2226 categories |= PAGE_IS_SWAPPED;
2227 if (!pte_swp_uffd_wp_any(pte))
2228 categories |= PAGE_IS_WRITTEN;
2229 if (pte_swp_soft_dirty(pte))
2230 categories |= PAGE_IS_SOFT_DIRTY;
2231 }
2232
2233 return categories;
2234 }
2235
2236 static void make_uffd_wp_huge_pte(struct vm_area_struct *vma,
2237 unsigned long addr, pte_t *ptep,
2238 pte_t ptent)
2239 {
2240 unsigned long psize;
2241
2242 if (is_hugetlb_entry_hwpoisoned(ptent) || is_pte_marker(ptent))
2243 return;
2244
2245 psize = huge_page_size(hstate_vma(vma));
2246
2247 if (is_hugetlb_entry_migration(ptent))
2248 set_huge_pte_at(vma->vm_mm, addr, ptep,
2249 pte_swp_mkuffd_wp(ptent), psize);
2250 else if (!huge_pte_none(ptent))
2251 huge_ptep_modify_prot_commit(vma, addr, ptep, ptent,
2252 huge_pte_mkuffd_wp(ptent));
2253 else
2254 set_huge_pte_at(vma->vm_mm, addr, ptep,
2255 make_pte_marker(PTE_MARKER_UFFD_WP), psize);
2256 }
2257 #endif /* CONFIG_HUGETLB_PAGE */
2258
2259 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLB_PAGE)
2260 static void pagemap_scan_backout_range(struct pagemap_scan_private *p,
2261 unsigned long addr, unsigned long end)
2262 {
2263 struct page_region *cur_buf = &p->vec_buf[p->vec_buf_index];
2264
2265 if (cur_buf->start != addr)
2266 cur_buf->end = addr;
2267 else
2268 cur_buf->start = cur_buf->end = 0;
2269
2270 p->found_pages -= (end - addr) / PAGE_SIZE;
2271 }
2272 #endif
2273
2274 static bool pagemap_scan_is_interesting_page(unsigned long categories,
2275 const struct pagemap_scan_private *p)
2276 {
2277 categories ^= p->arg.category_inverted;
2278 if ((categories & p->arg.category_mask) != p->arg.category_mask)
2279 return false;
2280 if (p->arg.category_anyof_mask && !(categories & p->arg.category_anyof_mask))
2281 return false;
2282
2283 return true;
2284 }
2285
2286 static bool pagemap_scan_is_interesting_vma(unsigned long categories,
2287 const struct pagemap_scan_private *p)
2288 {
2289 unsigned long required = p->arg.category_mask & PAGE_IS_WPALLOWED;
2290
2291 categories ^= p->arg.category_inverted;
2292 if ((categories & required) != required)
2293 return false;
2294
2295 return true;
2296 }
2297
2298 static int pagemap_scan_test_walk(unsigned long start, unsigned long end,
2299 struct mm_walk *walk)
2300 {
2301 struct pagemap_scan_private *p = walk->private;
2302 struct vm_area_struct *vma = walk->vma;
2303 unsigned long vma_category = 0;
2304 bool wp_allowed = userfaultfd_wp_async(vma) &&
2305 userfaultfd_wp_use_markers(vma);
2306
2307 if (!wp_allowed) {
2308 /* User requested explicit failure over wp-async capability */
2309 if (p->arg.flags & PM_SCAN_CHECK_WPASYNC)
2310 return -EPERM;
2311 /*
2312 * User requires wr-protect, and allows silently skipping
2313 * unsupported vmas.
2314 */
2315 if (p->arg.flags & PM_SCAN_WP_MATCHING)
2316 return 1;
2317 /*
2318 * Then the request doesn't involve wr-protects at all,
2319 * fall through to the rest checks, and allow vma walk.
2320 */
2321 }
2322
2323 if (vma->vm_flags & VM_PFNMAP)
2324 return 1;
2325
2326 if (wp_allowed)
2327 vma_category |= PAGE_IS_WPALLOWED;
2328
2329 if (vma->vm_flags & VM_SOFTDIRTY)
2330 vma_category |= PAGE_IS_SOFT_DIRTY;
2331
2332 if (!pagemap_scan_is_interesting_vma(vma_category, p))
2333 return 1;
2334
2335 p->cur_vma_category = vma_category;
2336
2337 return 0;
2338 }
2339
2340 static bool pagemap_scan_push_range(unsigned long categories,
2341 struct pagemap_scan_private *p,
2342 unsigned long addr, unsigned long end)
2343 {
2344 struct page_region *cur_buf = &p->vec_buf[p->vec_buf_index];
2345
2346 /*
2347 * When there is no output buffer provided at all, the sentinel values
2348 * won't match here. There is no other way for `cur_buf->end` to be
2349 * non-zero other than it being non-empty.
2350 */
2351 if (addr == cur_buf->end && categories == cur_buf->categories) {
2352 cur_buf->end = end;
2353 return true;
2354 }
2355
2356 if (cur_buf->end) {
2357 if (p->vec_buf_index >= p->vec_buf_len - 1)
2358 return false;
2359
2360 cur_buf = &p->vec_buf[++p->vec_buf_index];
2361 }
2362
2363 cur_buf->start = addr;
2364 cur_buf->end = end;
2365 cur_buf->categories = categories;
2366
2367 return true;
2368 }
2369
2370 static int pagemap_scan_output(unsigned long categories,
2371 struct pagemap_scan_private *p,
2372 unsigned long addr, unsigned long *end)
2373 {
2374 unsigned long n_pages, total_pages;
2375 int ret = 0;
2376
2377 if (!p->vec_buf)
2378 return 0;
2379
2380 categories &= p->arg.return_mask;
2381
2382 n_pages = (*end - addr) / PAGE_SIZE;
2383 if (check_add_overflow(p->found_pages, n_pages, &total_pages) ||
2384 total_pages > p->arg.max_pages) {
2385 size_t n_too_much = total_pages - p->arg.max_pages;
2386 *end -= n_too_much * PAGE_SIZE;
2387 n_pages -= n_too_much;
2388 ret = -ENOSPC;
2389 }
2390
2391 if (!pagemap_scan_push_range(categories, p, addr, *end)) {
2392 *end = addr;
2393 n_pages = 0;
2394 ret = -ENOSPC;
2395 }
2396
2397 p->found_pages += n_pages;
2398 if (ret)
2399 p->arg.walk_end = *end;
2400
2401 return ret;
2402 }
2403
2404 static int pagemap_scan_thp_entry(pmd_t *pmd, unsigned long start,
2405 unsigned long end, struct mm_walk *walk)
2406 {
2407 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
2408 struct pagemap_scan_private *p = walk->private;
2409 struct vm_area_struct *vma = walk->vma;
2410 unsigned long categories;
2411 spinlock_t *ptl;
2412 int ret = 0;
2413
2414 ptl = pmd_trans_huge_lock(pmd, vma);
2415 if (!ptl)
2416 return -ENOENT;
2417
2418 categories = p->cur_vma_category |
2419 pagemap_thp_category(p, vma, start, *pmd);
2420
2421 if (!pagemap_scan_is_interesting_page(categories, p))
2422 goto out_unlock;
2423
2424 ret = pagemap_scan_output(categories, p, start, &end);
2425 if (start == end)
2426 goto out_unlock;
2427
2428 if (~p->arg.flags & PM_SCAN_WP_MATCHING)
2429 goto out_unlock;
2430 if (~categories & PAGE_IS_WRITTEN)
2431 goto out_unlock;
2432
2433 /*
2434 * Break huge page into small pages if the WP operation
2435 * needs to be performed on a portion of the huge page.
2436 */
2437 if (end != start + HPAGE_SIZE) {
2438 spin_unlock(ptl);
2439 split_huge_pmd(vma, pmd, start);
2440 pagemap_scan_backout_range(p, start, end);
2441 /* Report as if there was no THP */
2442 return -ENOENT;
2443 }
2444
2445 make_uffd_wp_pmd(vma, start, pmd);
2446 flush_tlb_range(vma, start, end);
2447 out_unlock:
2448 spin_unlock(ptl);
2449 return ret;
2450 #else /* !CONFIG_TRANSPARENT_HUGEPAGE */
2451 return -ENOENT;
2452 #endif
2453 }
2454
2455 static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start,
2456 unsigned long end, struct mm_walk *walk)
2457 {
2458 struct pagemap_scan_private *p = walk->private;
2459 struct vm_area_struct *vma = walk->vma;
2460 unsigned long addr, flush_end = 0;
2461 pte_t *pte, *start_pte;
2462 spinlock_t *ptl;
2463 int ret;
2464
2465 arch_enter_lazy_mmu_mode();
2466
2467 ret = pagemap_scan_thp_entry(pmd, start, end, walk);
2468 if (ret != -ENOENT) {
2469 arch_leave_lazy_mmu_mode();
2470 return ret;
2471 }
2472
2473 ret = 0;
2474 start_pte = pte = pte_offset_map_lock(vma->vm_mm, pmd, start, &ptl);
2475 if (!pte) {
2476 arch_leave_lazy_mmu_mode();
2477 walk->action = ACTION_AGAIN;
2478 return 0;
2479 }
2480
2481 if ((p->arg.flags & PM_SCAN_WP_MATCHING) && !p->vec_out) {
2482 /* Fast path for performing exclusive WP */
2483 for (addr = start; addr != end; pte++, addr += PAGE_SIZE) {
2484 pte_t ptent = ptep_get(pte);
2485
2486 if ((pte_present(ptent) && pte_uffd_wp(ptent)) ||
2487 pte_swp_uffd_wp_any(ptent))
2488 continue;
2489 make_uffd_wp_pte(vma, addr, pte, ptent);
2490 if (!flush_end)
2491 start = addr;
2492 flush_end = addr + PAGE_SIZE;
2493 }
2494 goto flush_and_return;
2495 }
2496
2497 if (!p->arg.category_anyof_mask && !p->arg.category_inverted &&
2498 p->arg.category_mask == PAGE_IS_WRITTEN &&
2499 p->arg.return_mask == PAGE_IS_WRITTEN) {
2500 for (addr = start; addr < end; pte++, addr += PAGE_SIZE) {
2501 unsigned long next = addr + PAGE_SIZE;
2502 pte_t ptent = ptep_get(pte);
2503
2504 if ((pte_present(ptent) && pte_uffd_wp(ptent)) ||
2505 pte_swp_uffd_wp_any(ptent))
2506 continue;
2507 ret = pagemap_scan_output(p->cur_vma_category | PAGE_IS_WRITTEN,
2508 p, addr, &next);
2509 if (next == addr)
2510 break;
2511 if (~p->arg.flags & PM_SCAN_WP_MATCHING)
2512 continue;
2513 make_uffd_wp_pte(vma, addr, pte, ptent);
2514 if (!flush_end)
2515 start = addr;
2516 flush_end = next;
2517 }
2518 goto flush_and_return;
2519 }
2520
2521 for (addr = start; addr != end; pte++, addr += PAGE_SIZE) {
2522 pte_t ptent = ptep_get(pte);
2523 unsigned long categories = p->cur_vma_category |
2524 pagemap_page_category(p, vma, addr, ptent);
2525 unsigned long next = addr + PAGE_SIZE;
2526
2527 if (!pagemap_scan_is_interesting_page(categories, p))
2528 continue;
2529
2530 ret = pagemap_scan_output(categories, p, addr, &next);
2531 if (next == addr)
2532 break;
2533
2534 if (~p->arg.flags & PM_SCAN_WP_MATCHING)
2535 continue;
2536 if (~categories & PAGE_IS_WRITTEN)
2537 continue;
2538
2539 make_uffd_wp_pte(vma, addr, pte, ptent);
2540 if (!flush_end)
2541 start = addr;
2542 flush_end = next;
2543 }
2544
2545 flush_and_return:
2546 if (flush_end)
2547 flush_tlb_range(vma, start, addr);
2548
2549 pte_unmap_unlock(start_pte, ptl);
2550 arch_leave_lazy_mmu_mode();
2551
2552 cond_resched();
2553 return ret;
2554 }
2555
2556 #ifdef CONFIG_HUGETLB_PAGE
2557 static int pagemap_scan_hugetlb_entry(pte_t *ptep, unsigned long hmask,
2558 unsigned long start, unsigned long end,
2559 struct mm_walk *walk)
2560 {
2561 struct pagemap_scan_private *p = walk->private;
2562 struct vm_area_struct *vma = walk->vma;
2563 unsigned long categories;
2564 spinlock_t *ptl;
2565 int ret = 0;
2566 pte_t pte;
2567
2568 if (~p->arg.flags & PM_SCAN_WP_MATCHING) {
2569 /* Go the short route when not write-protecting pages. */
2570
2571 pte = huge_ptep_get(walk->mm, start, ptep);
2572 categories = p->cur_vma_category | pagemap_hugetlb_category(pte);
2573
2574 if (!pagemap_scan_is_interesting_page(categories, p))
2575 return 0;
2576
2577 return pagemap_scan_output(categories, p, start, &end);
2578 }
2579
2580 i_mmap_lock_write(vma->vm_file->f_mapping);
2581 ptl = huge_pte_lock(hstate_vma(vma), vma->vm_mm, ptep);
2582
2583 pte = huge_ptep_get(walk->mm, start, ptep);
2584 categories = p->cur_vma_category | pagemap_hugetlb_category(pte);
2585
2586 if (!pagemap_scan_is_interesting_page(categories, p))
2587 goto out_unlock;
2588
2589 ret = pagemap_scan_output(categories, p, start, &end);
2590 if (start == end)
2591 goto out_unlock;
2592
2593 if (~categories & PAGE_IS_WRITTEN)
2594 goto out_unlock;
2595
2596 if (end != start + HPAGE_SIZE) {
2597 /* Partial HugeTLB page WP isn't possible. */
2598 pagemap_scan_backout_range(p, start, end);
2599 p->arg.walk_end = start;
2600 ret = 0;
2601 goto out_unlock;
2602 }
2603
2604 make_uffd_wp_huge_pte(vma, start, ptep, pte);
2605 flush_hugetlb_tlb_range(vma, start, end);
2606
2607 out_unlock:
2608 spin_unlock(ptl);
2609 i_mmap_unlock_write(vma->vm_file->f_mapping);
2610
2611 return ret;
2612 }
2613 #else
2614 #define pagemap_scan_hugetlb_entry NULL
2615 #endif
2616
2617 static int pagemap_scan_pte_hole(unsigned long addr, unsigned long end,
2618 int depth, struct mm_walk *walk)
2619 {
2620 struct pagemap_scan_private *p = walk->private;
2621 struct vm_area_struct *vma = walk->vma;
2622 int ret, err;
2623
2624 if (!vma || !pagemap_scan_is_interesting_page(p->cur_vma_category, p))
2625 return 0;
2626
2627 ret = pagemap_scan_output(p->cur_vma_category, p, addr, &end);
2628 if (addr == end)
2629 return ret;
2630
2631 if (~p->arg.flags & PM_SCAN_WP_MATCHING)
2632 return ret;
2633
2634 err = uffd_wp_range(vma, addr, end - addr, true);
2635 if (err < 0)
2636 ret = err;
2637
2638 return ret;
2639 }
2640
2641 static const struct mm_walk_ops pagemap_scan_ops = {
2642 .test_walk = pagemap_scan_test_walk,
2643 .pmd_entry = pagemap_scan_pmd_entry,
2644 .pte_hole = pagemap_scan_pte_hole,
2645 .hugetlb_entry = pagemap_scan_hugetlb_entry,
2646 };
2647
2648 static int pagemap_scan_get_args(struct pm_scan_arg *arg,
2649 unsigned long uarg)
2650 {
2651 if (copy_from_user(arg, (void __user *)uarg, sizeof(*arg)))
2652 return -EFAULT;
2653
2654 if (arg->size != sizeof(struct pm_scan_arg))
2655 return -EINVAL;
2656
2657 /* Validate requested features */
2658 if (arg->flags & ~PM_SCAN_FLAGS)
2659 return -EINVAL;
2660 if ((arg->category_inverted | arg->category_mask |
2661 arg->category_anyof_mask | arg->return_mask) & ~PM_SCAN_CATEGORIES)
2662 return -EINVAL;
2663
2664 arg->start = untagged_addr((unsigned long)arg->start);
2665 arg->end = untagged_addr((unsigned long)arg->end);
2666 arg->vec = untagged_addr((unsigned long)arg->vec);
2667
2668 /* Validate memory pointers */
2669 if (!IS_ALIGNED(arg->start, PAGE_SIZE))
2670 return -EINVAL;
2671 if (!access_ok((void __user *)(long)arg->start, arg->end - arg->start))
2672 return -EFAULT;
2673 if (!arg->vec && arg->vec_len)
2674 return -EINVAL;
2675 if (arg->vec && !access_ok((void __user *)(long)arg->vec,
2676 arg->vec_len * sizeof(struct page_region)))
2677 return -EFAULT;
2678
2679 /* Fixup default values */
2680 arg->end = ALIGN(arg->end, PAGE_SIZE);
2681 arg->walk_end = 0;
2682 if (!arg->max_pages)
2683 arg->max_pages = ULONG_MAX;
2684
2685 return 0;
2686 }
2687
2688 static int pagemap_scan_writeback_args(struct pm_scan_arg *arg,
2689 unsigned long uargl)
2690 {
2691 struct pm_scan_arg __user *uarg = (void __user *)uargl;
2692
2693 if (copy_to_user(&uarg->walk_end, &arg->walk_end, sizeof(arg->walk_end)))
2694 return -EFAULT;
2695
2696 return 0;
2697 }
2698
2699 static int pagemap_scan_init_bounce_buffer(struct pagemap_scan_private *p)
2700 {
2701 if (!p->arg.vec_len)
2702 return 0;
2703
2704 p->vec_buf_len = min_t(size_t, PAGEMAP_WALK_SIZE >> PAGE_SHIFT,
2705 p->arg.vec_len);
2706 p->vec_buf = kmalloc_array(p->vec_buf_len, sizeof(*p->vec_buf),
2707 GFP_KERNEL);
2708 if (!p->vec_buf)
2709 return -ENOMEM;
2710
2711 p->vec_buf->start = p->vec_buf->end = 0;
2712 p->vec_out = (struct page_region __user *)(long)p->arg.vec;
2713
2714 return 0;
2715 }
2716
2717 static long pagemap_scan_flush_buffer(struct pagemap_scan_private *p)
2718 {
2719 const struct page_region *buf = p->vec_buf;
2720 long n = p->vec_buf_index;
2721
2722 if (!p->vec_buf)
2723 return 0;
2724
2725 if (buf[n].end != buf[n].start)
2726 n++;
2727
2728 if (!n)
2729 return 0;
2730
2731 if (copy_to_user(p->vec_out, buf, n * sizeof(*buf)))
2732 return -EFAULT;
2733
2734 p->arg.vec_len -= n;
2735 p->vec_out += n;
2736
2737 p->vec_buf_index = 0;
2738 p->vec_buf_len = min_t(size_t, p->vec_buf_len, p->arg.vec_len);
2739 p->vec_buf->start = p->vec_buf->end = 0;
2740
2741 return n;
2742 }
2743
2744 static long do_pagemap_scan(struct mm_struct *mm, unsigned long uarg)
2745 {
2746 struct pagemap_scan_private p = {0};
2747 unsigned long walk_start;
2748 size_t n_ranges_out = 0;
2749 int ret;
2750
2751 ret = pagemap_scan_get_args(&p.arg, uarg);
2752 if (ret)
2753 return ret;
2754
2755 p.masks_of_interest = p.arg.category_mask | p.arg.category_anyof_mask |
2756 p.arg.return_mask;
2757 ret = pagemap_scan_init_bounce_buffer(&p);
2758 if (ret)
2759 return ret;
2760
2761 for (walk_start = p.arg.start; walk_start < p.arg.end;
2762 walk_start = p.arg.walk_end) {
2763 struct mmu_notifier_range range;
2764 long n_out;
2765
2766 if (fatal_signal_pending(current)) {
2767 ret = -EINTR;
2768 break;
2769 }
2770
2771 ret = mmap_read_lock_killable(mm);
2772 if (ret)
2773 break;
2774
2775 /* Protection change for the range is going to happen. */
2776 if (p.arg.flags & PM_SCAN_WP_MATCHING) {
2777 mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_VMA, 0,
2778 mm, walk_start, p.arg.end);
2779 mmu_notifier_invalidate_range_start(&range);
2780 }
2781
2782 ret = walk_page_range(mm, walk_start, p.arg.end,
2783 &pagemap_scan_ops, &p);
2784
2785 if (p.arg.flags & PM_SCAN_WP_MATCHING)
2786 mmu_notifier_invalidate_range_end(&range);
2787
2788 mmap_read_unlock(mm);
2789
2790 n_out = pagemap_scan_flush_buffer(&p);
2791 if (n_out < 0)
2792 ret = n_out;
2793 else
2794 n_ranges_out += n_out;
2795
2796 if (ret != -ENOSPC)
2797 break;
2798
2799 if (p.arg.vec_len == 0 || p.found_pages == p.arg.max_pages)
2800 break;
2801 }
2802
2803 /* ENOSPC signifies early stop (buffer full) from the walk. */
2804 if (!ret || ret == -ENOSPC)
2805 ret = n_ranges_out;
2806
2807 /* The walk_end isn't set when ret is zero */
2808 if (!p.arg.walk_end)
2809 p.arg.walk_end = p.arg.end;
2810 if (pagemap_scan_writeback_args(&p.arg, uarg))
2811 ret = -EFAULT;
2812
2813 kfree(p.vec_buf);
2814 return ret;
2815 }
2816
2817 static long do_pagemap_cmd(struct file *file, unsigned int cmd,
2818 unsigned long arg)
2819 {
2820 struct mm_struct *mm = file->private_data;
2821
2822 switch (cmd) {
2823 case PAGEMAP_SCAN:
2824 return do_pagemap_scan(mm, arg);
2825
2826 default:
2827 return -EINVAL;
2828 }
2829 }
2830
2831 const struct file_operations proc_pagemap_operations = {
2832 .llseek = mem_lseek, /* borrow this */
2833 .read = pagemap_read,
2834 .open = pagemap_open,
2835 .release = pagemap_release,
2836 .unlocked_ioctl = do_pagemap_cmd,
2837 .compat_ioctl = do_pagemap_cmd,
2838 };
2839 #endif /* CONFIG_PROC_PAGE_MONITOR */
2840
2841 #ifdef CONFIG_NUMA
2842
2843 struct numa_maps {
2844 unsigned long pages;
2845 unsigned long anon;
2846 unsigned long active;
2847 unsigned long writeback;
2848 unsigned long mapcount_max;
2849 unsigned long dirty;
2850 unsigned long swapcache;
2851 unsigned long node[MAX_NUMNODES];
2852 };
2853
2854 struct numa_maps_private {
2855 struct proc_maps_private proc_maps;
2856 struct numa_maps md;
2857 };
2858
2859 static void gather_stats(struct page *page, struct numa_maps *md, int pte_dirty,
2860 unsigned long nr_pages)
2861 {
2862 struct folio *folio = page_folio(page);
2863 int count = folio_precise_page_mapcount(folio, page);
2864
2865 md->pages += nr_pages;
2866 if (pte_dirty || folio_test_dirty(folio))
2867 md->dirty += nr_pages;
2868
2869 if (folio_test_swapcache(folio))
2870 md->swapcache += nr_pages;
2871
2872 if (folio_test_active(folio) || folio_test_unevictable(folio))
2873 md->active += nr_pages;
2874
2875 if (folio_test_writeback(folio))
2876 md->writeback += nr_pages;
2877
2878 if (folio_test_anon(folio))
2879 md->anon += nr_pages;
2880
2881 if (count > md->mapcount_max)
2882 md->mapcount_max = count;
2883
2884 md->node[folio_nid(folio)] += nr_pages;
2885 }
2886
2887 static struct page *can_gather_numa_stats(pte_t pte, struct vm_area_struct *vma,
2888 unsigned long addr)
2889 {
2890 struct page *page;
2891 int nid;
2892
2893 if (!pte_present(pte))
2894 return NULL;
2895
2896 page = vm_normal_page(vma, addr, pte);
2897 if (!page || is_zone_device_page(page))
2898 return NULL;
2899
2900 if (PageReserved(page))
2901 return NULL;
2902
2903 nid = page_to_nid(page);
2904 if (!node_isset(nid, node_states[N_MEMORY]))
2905 return NULL;
2906
2907 return page;
2908 }
2909
2910 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
2911 static struct page *can_gather_numa_stats_pmd(pmd_t pmd,
2912 struct vm_area_struct *vma,
2913 unsigned long addr)
2914 {
2915 struct page *page;
2916 int nid;
2917
2918 if (!pmd_present(pmd))
2919 return NULL;
2920
2921 page = vm_normal_page_pmd(vma, addr, pmd);
2922 if (!page)
2923 return NULL;
2924
2925 if (PageReserved(page))
2926 return NULL;
2927
2928 nid = page_to_nid(page);
2929 if (!node_isset(nid, node_states[N_MEMORY]))
2930 return NULL;
2931
2932 return page;
2933 }
2934 #endif
2935
2936 static int gather_pte_stats(pmd_t *pmd, unsigned long addr,
2937 unsigned long end, struct mm_walk *walk)
2938 {
2939 struct numa_maps *md = walk->private;
2940 struct vm_area_struct *vma = walk->vma;
2941 spinlock_t *ptl;
2942 pte_t *orig_pte;
2943 pte_t *pte;
2944
2945 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
2946 ptl = pmd_trans_huge_lock(pmd, vma);
2947 if (ptl) {
2948 struct page *page;
2949
2950 page = can_gather_numa_stats_pmd(*pmd, vma, addr);
2951 if (page)
2952 gather_stats(page, md, pmd_dirty(*pmd),
2953 HPAGE_PMD_SIZE/PAGE_SIZE);
2954 spin_unlock(ptl);
2955 return 0;
2956 }
2957 #endif
2958 orig_pte = pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl);
2959 if (!pte) {
2960 walk->action = ACTION_AGAIN;
2961 return 0;
2962 }
2963 do {
2964 pte_t ptent = ptep_get(pte);
2965 struct page *page = can_gather_numa_stats(ptent, vma, addr);
2966 if (!page)
2967 continue;
2968 gather_stats(page, md, pte_dirty(ptent), 1);
2969
2970 } while (pte++, addr += PAGE_SIZE, addr != end);
2971 pte_unmap_unlock(orig_pte, ptl);
2972 cond_resched();
2973 return 0;
2974 }
2975 #ifdef CONFIG_HUGETLB_PAGE
2976 static int gather_hugetlb_stats(pte_t *pte, unsigned long hmask,
2977 unsigned long addr, unsigned long end, struct mm_walk *walk)
2978 {
2979 pte_t huge_pte = huge_ptep_get(walk->mm, addr, pte);
2980 struct numa_maps *md;
2981 struct page *page;
2982
2983 if (!pte_present(huge_pte))
2984 return 0;
2985
2986 page = pte_page(huge_pte);
2987
2988 md = walk->private;
2989 gather_stats(page, md, pte_dirty(huge_pte), 1);
2990 return 0;
2991 }
2992
2993 #else
2994 static int gather_hugetlb_stats(pte_t *pte, unsigned long hmask,
2995 unsigned long addr, unsigned long end, struct mm_walk *walk)
2996 {
2997 return 0;
2998 }
2999 #endif
3000
3001 static const struct mm_walk_ops show_numa_ops = {
3002 .hugetlb_entry = gather_hugetlb_stats,
3003 .pmd_entry = gather_pte_stats,
3004 .walk_lock = PGWALK_RDLOCK,
3005 };
3006
3007 /*
3008 * Display pages allocated per node and memory policy via /proc.
3009 */
3010 static int show_numa_map(struct seq_file *m, void *v)
3011 {
3012 struct numa_maps_private *numa_priv = m->private;
3013 struct proc_maps_private *proc_priv = &numa_priv->proc_maps;
3014 struct vm_area_struct *vma = v;
3015 struct numa_maps *md = &numa_priv->md;
3016 struct file *file = vma->vm_file;
3017 struct mm_struct *mm = vma->vm_mm;
3018 char buffer[64];
3019 struct mempolicy *pol;
3020 pgoff_t ilx;
3021 int nid;
3022
3023 if (!mm)
3024 return 0;
3025
3026 /* Ensure we start with an empty set of numa_maps statistics. */
3027 memset(md, 0, sizeof(*md));
3028
3029 pol = __get_vma_policy(vma, vma->vm_start, &ilx);
3030 if (pol) {
3031 mpol_to_str(buffer, sizeof(buffer), pol);
3032 mpol_cond_put(pol);
3033 } else {
3034 mpol_to_str(buffer, sizeof(buffer), proc_priv->task_mempolicy);
3035 }
3036
3037 seq_printf(m, "%08lx %s", vma->vm_start, buffer);
3038
3039 if (file) {
3040 seq_puts(m, " file=");
3041 seq_path(m, file_user_path(file), "\n\t= ");
3042 } else if (vma_is_initial_heap(vma)) {
3043 seq_puts(m, " heap");
3044 } else if (vma_is_initial_stack(vma)) {
3045 seq_puts(m, " stack");
3046 }
3047
3048 if (is_vm_hugetlb_page(vma))
3049 seq_puts(m, " huge");
3050
3051 /* mmap_lock is held by m_start */
3052 walk_page_vma(vma, &show_numa_ops, md);
3053
3054 if (!md->pages)
3055 goto out;
3056
3057 if (md->anon)
3058 seq_printf(m, " anon=%lu", md->anon);
3059
3060 if (md->dirty)
3061 seq_printf(m, " dirty=%lu", md->dirty);
3062
3063 if (md->pages != md->anon && md->pages != md->dirty)
3064 seq_printf(m, " mapped=%lu", md->pages);
3065
3066 if (md->mapcount_max > 1)
3067 seq_printf(m, " mapmax=%lu", md->mapcount_max);
3068
3069 if (md->swapcache)
3070 seq_printf(m, " swapcache=%lu", md->swapcache);
3071
3072 if (md->active < md->pages && !is_vm_hugetlb_page(vma))
3073 seq_printf(m, " active=%lu", md->active);
3074
3075 if (md->writeback)
3076 seq_printf(m, " writeback=%lu", md->writeback);
3077
3078 for_each_node_state(nid, N_MEMORY)
3079 if (md->node[nid])
3080 seq_printf(m, " N%d=%lu", nid, md->node[nid]);
3081
3082 seq_printf(m, " kernelpagesize_kB=%lu", vma_kernel_pagesize(vma) >> 10);
3083 out:
3084 seq_putc(m, '\n');
3085 return 0;
3086 }
3087
3088 static const struct seq_operations proc_pid_numa_maps_op = {
3089 .start = m_start,
3090 .next = m_next,
3091 .stop = m_stop,
3092 .show = show_numa_map,
3093 };
3094
3095 static int pid_numa_maps_open(struct inode *inode, struct file *file)
3096 {
3097 return proc_maps_open(inode, file, &proc_pid_numa_maps_op,
3098 sizeof(struct numa_maps_private));
3099 }
3100
3101 const struct file_operations proc_pid_numa_maps_operations = {
3102 .open = pid_numa_maps_open,
3103 .read = seq_read,
3104 .llseek = seq_lseek,
3105 .release = proc_map_release,
3106 };
3107
3108 #endif /* CONFIG_NUMA */
3109
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.