1 /* 1
2 * Resizable virtual memory filesystem for Lin
3 *
4 * Copyright (C) 2000 Linus Torvalds.
5 * 2000 Transmeta Corp.
6 * 2000-2001 Christoph Rohland
7 * 2000-2001 SAP AG
8 * 2002 Red Hat Inc.
9 * Copyright (C) 2002-2011 Hugh Dickins.
10 * Copyright (C) 2011 Google Inc.
11 * Copyright (C) 2002-2005 VERITAS Software Co
12 * Copyright (C) 2004 Andi Kleen, SuSE Labs
13 *
14 * Extended attribute support for tmpfs:
15 * Copyright (c) 2004, Luke Kenneth Casson Lei
16 * Copyright (c) 2004 Red Hat, Inc., James Mor
17 *
18 * tiny-shmem:
19 * Copyright (c) 2004, 2008 Matt Mackall <mpm@
20 *
21 * This file is released under the GPL.
22 */
23
24 #include <linux/fs.h>
25 #include <linux/init.h>
26 #include <linux/vfs.h>
27 #include <linux/mount.h>
28 #include <linux/ramfs.h>
29 #include <linux/pagemap.h>
30 #include <linux/file.h>
31 #include <linux/fileattr.h>
32 #include <linux/mm.h>
33 #include <linux/random.h>
34 #include <linux/sched/signal.h>
35 #include <linux/export.h>
36 #include <linux/shmem_fs.h>
37 #include <linux/swap.h>
38 #include <linux/uio.h>
39 #include <linux/hugetlb.h>
40 #include <linux/fs_parser.h>
41 #include <linux/swapfile.h>
42 #include <linux/iversion.h>
43 #include "swap.h"
44
45 static struct vfsmount *shm_mnt __ro_after_ini
46
47 #ifdef CONFIG_SHMEM
48 /*
49 * This virtual memory filesystem is heavily b
50 * extends ramfs by the ability to use swap an
51 * which makes it a completely usable filesyst
52 */
53
54 #include <linux/xattr.h>
55 #include <linux/exportfs.h>
56 #include <linux/posix_acl.h>
57 #include <linux/posix_acl_xattr.h>
58 #include <linux/mman.h>
59 #include <linux/string.h>
60 #include <linux/slab.h>
61 #include <linux/backing-dev.h>
62 #include <linux/writeback.h>
63 #include <linux/pagevec.h>
64 #include <linux/percpu_counter.h>
65 #include <linux/falloc.h>
66 #include <linux/splice.h>
67 #include <linux/security.h>
68 #include <linux/swapops.h>
69 #include <linux/mempolicy.h>
70 #include <linux/namei.h>
71 #include <linux/ctype.h>
72 #include <linux/migrate.h>
73 #include <linux/highmem.h>
74 #include <linux/seq_file.h>
75 #include <linux/magic.h>
76 #include <linux/syscalls.h>
77 #include <linux/fcntl.h>
78 #include <uapi/linux/memfd.h>
79 #include <linux/rmap.h>
80 #include <linux/uuid.h>
81 #include <linux/quotaops.h>
82 #include <linux/rcupdate_wait.h>
83
84 #include <linux/uaccess.h>
85
86 #include "internal.h"
87
88 #define BLOCKS_PER_PAGE (PAGE_SIZE/512)
89 #define VM_ACCT(size) (PAGE_ALIGN(size) >>
90
91 /* Pretend that each entry is of this size in
92 #define BOGO_DIRENT_SIZE 20
93
94 /* Pretend that one inode + its dentry occupy
95 #define BOGO_INODE_SIZE 1024
96
97 /* Symlink up to this size is kmalloc'ed inste
98 #define SHORT_SYMLINK_LEN 128
99
100 /*
101 * shmem_fallocate communicates with shmem_fau
102 * inode->i_private (with i_rwsem making sure
103 * a time): we would prefer not to enlarge the
104 */
105 struct shmem_falloc {
106 wait_queue_head_t *waitq; /* faults in
107 pgoff_t start; /* start of ra
108 pgoff_t next; /* the next pa
109 pgoff_t nr_falloced; /* how many ne
110 pgoff_t nr_unswapped; /* how often w
111 };
112
113 struct shmem_options {
114 unsigned long long blocks;
115 unsigned long long inodes;
116 struct mempolicy *mpol;
117 kuid_t uid;
118 kgid_t gid;
119 umode_t mode;
120 bool full_inums;
121 int huge;
122 int seen;
123 bool noswap;
124 unsigned short quota_types;
125 struct shmem_quota_limits qlimits;
126 #define SHMEM_SEEN_BLOCKS 1
127 #define SHMEM_SEEN_INODES 2
128 #define SHMEM_SEEN_HUGE 4
129 #define SHMEM_SEEN_INUMS 8
130 #define SHMEM_SEEN_NOSWAP 16
131 #define SHMEM_SEEN_QUOTA 32
132 };
133
134 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
135 static unsigned long huge_shmem_orders_always
136 static unsigned long huge_shmem_orders_madvise
137 static unsigned long huge_shmem_orders_inherit
138 static unsigned long huge_shmem_orders_within_
139 #endif
140
141 #ifdef CONFIG_TMPFS
142 static unsigned long shmem_default_max_blocks(
143 {
144 return totalram_pages() / 2;
145 }
146
147 static unsigned long shmem_default_max_inodes(
148 {
149 unsigned long nr_pages = totalram_page
150
151 return min3(nr_pages - totalhigh_pages
152 ULONG_MAX / BOGO_INODE
153 }
154 #endif
155
156 static int shmem_swapin_folio(struct inode *in
157 struct folio **foliop,
158 struct vm_area_struct
159
160 static inline struct shmem_sb_info *SHMEM_SB(s
161 {
162 return sb->s_fs_info;
163 }
164
165 /*
166 * shmem_file_setup pre-accounts the whole fix
167 * for shared memory and for shared anonymous
168 * (unless MAP_NORESERVE and sysctl_overcommit
169 * consistent with the pre-accounting of priva
170 */
171 static inline int shmem_acct_size(unsigned lon
172 {
173 return (flags & VM_NORESERVE) ?
174 0 : security_vm_enough_memory_
175 }
176
177 static inline void shmem_unacct_size(unsigned
178 {
179 if (!(flags & VM_NORESERVE))
180 vm_unacct_memory(VM_ACCT(size)
181 }
182
183 static inline int shmem_reacct_size(unsigned l
184 loff_t oldsize, loff_t newsize
185 {
186 if (!(flags & VM_NORESERVE)) {
187 if (VM_ACCT(newsize) > VM_ACCT
188 return security_vm_eno
189 VM_ACC
190 else if (VM_ACCT(newsize) < VM
191 vm_unacct_memory(VM_AC
192 }
193 return 0;
194 }
195
196 /*
197 * ... whereas tmpfs objects are accounted inc
198 * pages are allocated, in order to allow larg
199 * shmem_get_folio reports shmem_acct_blocks f
200 * so that a failure on a sparse tmpfs mapping
201 */
202 static inline int shmem_acct_blocks(unsigned l
203 {
204 if (!(flags & VM_NORESERVE))
205 return 0;
206
207 return security_vm_enough_memory_mm(cu
208 pages * VM_ACCT(PAGE_S
209 }
210
211 static inline void shmem_unacct_blocks(unsigne
212 {
213 if (flags & VM_NORESERVE)
214 vm_unacct_memory(pages * VM_AC
215 }
216
217 static int shmem_inode_acct_blocks(struct inod
218 {
219 struct shmem_inode_info *info = SHMEM_
220 struct shmem_sb_info *sbinfo = SHMEM_S
221 int err = -ENOSPC;
222
223 if (shmem_acct_blocks(info->flags, pag
224 return err;
225
226 might_sleep(); /* when quotas */
227 if (sbinfo->max_blocks) {
228 if (!percpu_counter_limited_ad
229
230 goto unacct;
231
232 err = dquot_alloc_block_nodirt
233 if (err) {
234 percpu_counter_sub(&sb
235 goto unacct;
236 }
237 } else {
238 err = dquot_alloc_block_nodirt
239 if (err)
240 goto unacct;
241 }
242
243 return 0;
244
245 unacct:
246 shmem_unacct_blocks(info->flags, pages
247 return err;
248 }
249
250 static void shmem_inode_unacct_blocks(struct i
251 {
252 struct shmem_inode_info *info = SHMEM_
253 struct shmem_sb_info *sbinfo = SHMEM_S
254
255 might_sleep(); /* when quotas */
256 dquot_free_block_nodirty(inode, pages)
257
258 if (sbinfo->max_blocks)
259 percpu_counter_sub(&sbinfo->us
260 shmem_unacct_blocks(info->flags, pages
261 }
262
263 static const struct super_operations shmem_ops
264 static const struct address_space_operations s
265 static const struct file_operations shmem_file
266 static const struct inode_operations shmem_ino
267 static const struct inode_operations shmem_dir
268 static const struct inode_operations shmem_spe
269 static const struct vm_operations_struct shmem
270 static const struct vm_operations_struct shmem
271 static struct file_system_type shmem_fs_type;
272
273 bool shmem_mapping(struct address_space *mappi
274 {
275 return mapping->a_ops == &shmem_aops;
276 }
277 EXPORT_SYMBOL_GPL(shmem_mapping);
278
279 bool vma_is_anon_shmem(struct vm_area_struct *
280 {
281 return vma->vm_ops == &shmem_anon_vm_o
282 }
283
284 bool vma_is_shmem(struct vm_area_struct *vma)
285 {
286 return vma_is_anon_shmem(vma) || vma->
287 }
288
289 static LIST_HEAD(shmem_swaplist);
290 static DEFINE_MUTEX(shmem_swaplist_mutex);
291
292 #ifdef CONFIG_TMPFS_QUOTA
293
294 static int shmem_enable_quotas(struct super_bl
295 unsigned short
296 {
297 int type, err = 0;
298
299 sb_dqopt(sb)->flags |= DQUOT_QUOTA_SYS
300 for (type = 0; type < SHMEM_MAXQUOTAS;
301 if (!(quota_types & (1 << type
302 continue;
303 err = dquot_load_quota_sb(sb,
304 DQUO
305 DQUO
306 if (err)
307 goto out_err;
308 }
309 return 0;
310
311 out_err:
312 pr_warn("tmpfs: failed to enable quota
313 type, err);
314 for (type--; type >= 0; type--)
315 dquot_quota_off(sb, type);
316 return err;
317 }
318
319 static void shmem_disable_quotas(struct super_
320 {
321 int type;
322
323 for (type = 0; type < SHMEM_MAXQUOTAS;
324 dquot_quota_off(sb, type);
325 }
326
327 static struct dquot __rcu **shmem_get_dquots(s
328 {
329 return SHMEM_I(inode)->i_dquot;
330 }
331 #endif /* CONFIG_TMPFS_QUOTA */
332
333 /*
334 * shmem_reserve_inode() performs bookkeeping
335 * produces a novel ino for the newly allocate
336 *
337 * It may also be called when making a hard li
338 * each dentry. However, in that case, no new
339 * internally draws from another pool of inode
340 * get_next_ino()). This case is indicated by
341 */
342 #define SHMEM_INO_BATCH 1024
343 static int shmem_reserve_inode(struct super_bl
344 {
345 struct shmem_sb_info *sbinfo = SHMEM_S
346 ino_t ino;
347
348 if (!(sb->s_flags & SB_KERNMOUNT)) {
349 raw_spin_lock(&sbinfo->stat_lo
350 if (sbinfo->max_inodes) {
351 if (sbinfo->free_ispac
352 raw_spin_unloc
353 return -ENOSPC
354 }
355 sbinfo->free_ispace -=
356 }
357 if (inop) {
358 ino = sbinfo->next_ino
359 if (unlikely(is_zero_i
360 ino = sbinfo->
361 if (unlikely(!sbinfo->
362 ino > UIN
363 /*
364 * Emulate get
365 * compatibili
366 */
367 if (IS_ENABLED
368 pr_war
369
370 sbinfo->next_i
371 ino = sbinfo->
372 }
373 *inop = ino;
374 }
375 raw_spin_unlock(&sbinfo->stat_
376 } else if (inop) {
377 /*
378 * __shmem_file_setup, one of
379 * doesn't hold stat_lock in s
380 * max_inodes is always 0, and
381 * unknown contexts. As such,
382 * which doesn't require the p
383 * the batch boundary.
384 *
385 * We don't need to worry abou
386 * shmem mounts are not expose
387 * to worry about things like
388 */
389 ino_t *next_ino;
390
391 next_ino = per_cpu_ptr(sbinfo-
392 ino = *next_ino;
393 if (unlikely(ino % SHMEM_INO_B
394 raw_spin_lock(&sbinfo-
395 ino = sbinfo->next_ino
396 sbinfo->next_ino += SH
397 raw_spin_unlock(&sbinf
398 if (unlikely(is_zero_i
399 ino++;
400 }
401 *inop = ino;
402 *next_ino = ++ino;
403 put_cpu();
404 }
405
406 return 0;
407 }
408
409 static void shmem_free_inode(struct super_bloc
410 {
411 struct shmem_sb_info *sbinfo = SHMEM_S
412 if (sbinfo->max_inodes) {
413 raw_spin_lock(&sbinfo->stat_lo
414 sbinfo->free_ispace += BOGO_IN
415 raw_spin_unlock(&sbinfo->stat_
416 }
417 }
418
419 /**
420 * shmem_recalc_inode - recalculate the block
421 * @inode: inode to recalc
422 * @alloced: the change in number of pages all
423 * @swapped: the change in number of pages swa
424 *
425 * We have to calculate the free blocks since
426 * undirtied hole pages behind our back.
427 *
428 * But normally info->alloced == inode->i_ma
429 * So mm freed is info->alloced - (inode->i_ma
430 */
431 static void shmem_recalc_inode(struct inode *i
432 {
433 struct shmem_inode_info *info = SHMEM_
434 long freed;
435
436 spin_lock(&info->lock);
437 info->alloced += alloced;
438 info->swapped += swapped;
439 freed = info->alloced - info->swapped
440 READ_ONCE(inode->i_mapping->nr
441 /*
442 * Special case: whereas normally shme
443 * after i_mapping->nrpages has alread
444 * shmem_writepage() has to raise swap
445 * to stop a racing shmem_recalc_inode
446 * been freed. Compensate here, to av
447 */
448 if (swapped > 0)
449 freed += swapped;
450 if (freed > 0)
451 info->alloced -= freed;
452 spin_unlock(&info->lock);
453
454 /* The quota case may block */
455 if (freed > 0)
456 shmem_inode_unacct_blocks(inod
457 }
458
459 bool shmem_charge(struct inode *inode, long pa
460 {
461 struct address_space *mapping = inode-
462
463 if (shmem_inode_acct_blocks(inode, pag
464 return false;
465
466 /* nrpages adjustment first, then shme
467 xa_lock_irq(&mapping->i_pages);
468 mapping->nrpages += pages;
469 xa_unlock_irq(&mapping->i_pages);
470
471 shmem_recalc_inode(inode, pages, 0);
472 return true;
473 }
474
475 void shmem_uncharge(struct inode *inode, long
476 {
477 /* pages argument is currently unused:
478 /* nrpages adjustment done by __filema
479
480 shmem_recalc_inode(inode, 0, 0);
481 }
482
483 /*
484 * Replace item expected in xarray by a new it
485 */
486 static int shmem_replace_entry(struct address_
487 pgoff_t index, void *e
488 {
489 XA_STATE(xas, &mapping->i_pages, index
490 void *item;
491
492 VM_BUG_ON(!expected);
493 VM_BUG_ON(!replacement);
494 item = xas_load(&xas);
495 if (item != expected)
496 return -ENOENT;
497 xas_store(&xas, replacement);
498 return 0;
499 }
500
501 /*
502 * Sometimes, before we decide whether to proc
503 * that an entry was not already brought back
504 *
505 * Checking folio is not enough: by the time a
506 * might be reused, and again be swapcache, us
507 */
508 static bool shmem_confirm_swap(struct address_
509 pgoff_t index,
510 {
511 return xa_load(&mapping->i_pages, inde
512 }
513
514 /*
515 * Definitions for "huge tmpfs": tmpfs mounted
516 *
517 * SHMEM_HUGE_NEVER:
518 * disables huge pages for the mount;
519 * SHMEM_HUGE_ALWAYS:
520 * enables huge pages for the mount;
521 * SHMEM_HUGE_WITHIN_SIZE:
522 * only allocate huge pages if the page w
523 * also respect fadvise()/madvise() hints
524 * SHMEM_HUGE_ADVISE:
525 * only allocate huge pages if requested
526 */
527
528 #define SHMEM_HUGE_NEVER 0
529 #define SHMEM_HUGE_ALWAYS 1
530 #define SHMEM_HUGE_WITHIN_SIZE 2
531 #define SHMEM_HUGE_ADVISE 3
532
533 /*
534 * Special values.
535 * Only can be set via /sys/kernel/mm/transpar
536 *
537 * SHMEM_HUGE_DENY:
538 * disables huge on shm_mnt and all mount
539 * SHMEM_HUGE_FORCE:
540 * enables huge on shm_mnt and all mounts
541 *
542 */
543 #define SHMEM_HUGE_DENY (-1)
544 #define SHMEM_HUGE_FORCE (-2)
545
546 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
547 /* ifdef here to avoid bloating shmem.o when n
548
549 static int shmem_huge __read_mostly = SHMEM_HU
550
551 static bool __shmem_huge_global_enabled(struct
552 loff_t
553 struct
554 unsign
555 {
556 struct mm_struct *mm = vma ? vma->vm_m
557 loff_t i_size;
558
559 if (!S_ISREG(inode->i_mode))
560 return false;
561 if (mm && ((vm_flags & VM_NOHUGEPAGE)
562 return false;
563 if (shmem_huge == SHMEM_HUGE_DENY)
564 return false;
565 if (shmem_huge_force || shmem_huge ==
566 return true;
567
568 switch (SHMEM_SB(inode->i_sb)->huge) {
569 case SHMEM_HUGE_ALWAYS:
570 return true;
571 case SHMEM_HUGE_WITHIN_SIZE:
572 index = round_up(index + 1, HP
573 i_size = max(write_end, i_size
574 i_size = round_up(i_size, PAGE
575 if (i_size >> PAGE_SHIFT >= in
576 return true;
577 fallthrough;
578 case SHMEM_HUGE_ADVISE:
579 if (mm && (vm_flags & VM_HUGEP
580 return true;
581 fallthrough;
582 default:
583 return false;
584 }
585 }
586
587 static bool shmem_huge_global_enabled(struct i
588 loff_t write_end, bool shme
589 struct vm_area_struct *vma,
590 {
591 if (HPAGE_PMD_ORDER > MAX_PAGECACHE_OR
592 return false;
593
594 return __shmem_huge_global_enabled(ino
595 shm
596 }
597
598 #if defined(CONFIG_SYSFS)
599 static int shmem_parse_huge(const char *str)
600 {
601 if (!strcmp(str, "never"))
602 return SHMEM_HUGE_NEVER;
603 if (!strcmp(str, "always"))
604 return SHMEM_HUGE_ALWAYS;
605 if (!strcmp(str, "within_size"))
606 return SHMEM_HUGE_WITHIN_SIZE;
607 if (!strcmp(str, "advise"))
608 return SHMEM_HUGE_ADVISE;
609 if (!strcmp(str, "deny"))
610 return SHMEM_HUGE_DENY;
611 if (!strcmp(str, "force"))
612 return SHMEM_HUGE_FORCE;
613 return -EINVAL;
614 }
615 #endif
616
617 #if defined(CONFIG_SYSFS) || defined(CONFIG_TM
618 static const char *shmem_format_huge(int huge)
619 {
620 switch (huge) {
621 case SHMEM_HUGE_NEVER:
622 return "never";
623 case SHMEM_HUGE_ALWAYS:
624 return "always";
625 case SHMEM_HUGE_WITHIN_SIZE:
626 return "within_size";
627 case SHMEM_HUGE_ADVISE:
628 return "advise";
629 case SHMEM_HUGE_DENY:
630 return "deny";
631 case SHMEM_HUGE_FORCE:
632 return "force";
633 default:
634 VM_BUG_ON(1);
635 return "bad_val";
636 }
637 }
638 #endif
639
640 static unsigned long shmem_unused_huge_shrink(
641 struct shrink_control *sc, uns
642 {
643 LIST_HEAD(list), *pos, *next;
644 struct inode *inode;
645 struct shmem_inode_info *info;
646 struct folio *folio;
647 unsigned long batch = sc ? sc->nr_to_s
648 unsigned long split = 0, freed = 0;
649
650 if (list_empty(&sbinfo->shrinklist))
651 return SHRINK_STOP;
652
653 spin_lock(&sbinfo->shrinklist_lock);
654 list_for_each_safe(pos, next, &sbinfo-
655 info = list_entry(pos, struct
656
657 /* pin the inode */
658 inode = igrab(&info->vfs_inode
659
660 /* inode is about to be evicte
661 if (!inode) {
662 list_del_init(&info->s
663 goto next;
664 }
665
666 list_move(&info->shrinklist, &
667 next:
668 sbinfo->shrinklist_len--;
669 if (!--batch)
670 break;
671 }
672 spin_unlock(&sbinfo->shrinklist_lock);
673
674 list_for_each_safe(pos, next, &list) {
675 pgoff_t next, end;
676 loff_t i_size;
677 int ret;
678
679 info = list_entry(pos, struct
680 inode = &info->vfs_inode;
681
682 if (nr_to_free && freed >= nr_
683 goto move_back;
684
685 i_size = i_size_read(inode);
686 folio = filemap_get_entry(inod
687 if (!folio || xa_is_value(foli
688 goto drop;
689
690 /* No large folio at the end o
691 if (!folio_test_large(folio))
692 folio_put(folio);
693 goto drop;
694 }
695
696 /* Check if there is anything
697 next = folio_next_index(folio)
698 end = shmem_fallocend(inode, D
699 if (end <= folio->index || end
700 folio_put(folio);
701 goto drop;
702 }
703
704 /*
705 * Move the inode on the list
706 * to lock the page at this ti
707 *
708 * Waiting for the lock may le
709 * reclaim path.
710 */
711 if (!folio_trylock(folio)) {
712 folio_put(folio);
713 goto move_back;
714 }
715
716 ret = split_folio(folio);
717 folio_unlock(folio);
718 folio_put(folio);
719
720 /* If split failed move the in
721 if (ret)
722 goto move_back;
723
724 freed += next - end;
725 split++;
726 drop:
727 list_del_init(&info->shrinklis
728 goto put;
729 move_back:
730 /*
731 * Make sure the inode is eith
732 * from any local list before
733 * in another thread once we p
734 * is corrupted).
735 */
736 spin_lock(&sbinfo->shrinklist_
737 list_move(&info->shrinklist, &
738 sbinfo->shrinklist_len++;
739 spin_unlock(&sbinfo->shrinklis
740 put:
741 iput(inode);
742 }
743
744 return split;
745 }
746
747 static long shmem_unused_huge_scan(struct supe
748 struct shrink_control *sc)
749 {
750 struct shmem_sb_info *sbinfo = SHMEM_S
751
752 if (!READ_ONCE(sbinfo->shrinklist_len)
753 return SHRINK_STOP;
754
755 return shmem_unused_huge_shrink(sbinfo
756 }
757
758 static long shmem_unused_huge_count(struct sup
759 struct shrink_control *sc)
760 {
761 struct shmem_sb_info *sbinfo = SHMEM_S
762 return READ_ONCE(sbinfo->shrinklist_le
763 }
764 #else /* !CONFIG_TRANSPARENT_HUGEPAGE */
765
766 #define shmem_huge SHMEM_HUGE_DENY
767
768 static unsigned long shmem_unused_huge_shrink(
769 struct shrink_control *sc, uns
770 {
771 return 0;
772 }
773
774 static bool shmem_huge_global_enabled(struct i
775 loff_t write_end, bool shmem_h
776 struct vm_area_struct *vma, un
777 {
778 return false;
779 }
780 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
781
782 /*
783 * Somewhat like filemap_add_folio, but error
784 */
785 static int shmem_add_to_page_cache(struct foli
786 struct addr
787 pgoff_t ind
788 {
789 XA_STATE_ORDER(xas, &mapping->i_pages,
790 long nr = folio_nr_pages(folio);
791
792 VM_BUG_ON_FOLIO(index != round_down(in
793 VM_BUG_ON_FOLIO(!folio_test_locked(fol
794 VM_BUG_ON_FOLIO(!folio_test_swapbacked
795
796 folio_ref_add(folio, nr);
797 folio->mapping = mapping;
798 folio->index = index;
799
800 gfp &= GFP_RECLAIM_MASK;
801 folio_throttle_swaprate(folio, gfp);
802
803 do {
804 xas_lock_irq(&xas);
805 if (expected != xas_find_confl
806 xas_set_err(&xas, -EEX
807 goto unlock;
808 }
809 if (expected && xas_find_confl
810 xas_set_err(&xas, -EEX
811 goto unlock;
812 }
813 xas_store(&xas, folio);
814 if (xas_error(&xas))
815 goto unlock;
816 if (folio_test_pmd_mappable(fo
817 __lruvec_stat_mod_foli
818 __lruvec_stat_mod_folio(folio,
819 __lruvec_stat_mod_folio(folio,
820 mapping->nrpages += nr;
821 unlock:
822 xas_unlock_irq(&xas);
823 } while (xas_nomem(&xas, gfp));
824
825 if (xas_error(&xas)) {
826 folio->mapping = NULL;
827 folio_ref_sub(folio, nr);
828 return xas_error(&xas);
829 }
830
831 return 0;
832 }
833
834 /*
835 * Somewhat like filemap_remove_folio, but sub
836 */
837 static void shmem_delete_from_page_cache(struc
838 {
839 struct address_space *mapping = folio-
840 long nr = folio_nr_pages(folio);
841 int error;
842
843 xa_lock_irq(&mapping->i_pages);
844 error = shmem_replace_entry(mapping, f
845 folio->mapping = NULL;
846 mapping->nrpages -= nr;
847 __lruvec_stat_mod_folio(folio, NR_FILE
848 __lruvec_stat_mod_folio(folio, NR_SHME
849 xa_unlock_irq(&mapping->i_pages);
850 folio_put_refs(folio, nr);
851 BUG_ON(error);
852 }
853
854 /*
855 * Remove swap entry from page cache, free the
856 * the number of pages being freed. 0 means en
857 * being freed).
858 */
859 static long shmem_free_swap(struct address_spa
860 pgoff_t index, voi
861 {
862 int order = xa_get_order(&mapping->i_p
863 void *old;
864
865 old = xa_cmpxchg_irq(&mapping->i_pages
866 if (old != radswap)
867 return 0;
868 free_swap_and_cache_nr(radix_to_swp_en
869
870 return 1 << order;
871 }
872
873 /*
874 * Determine (in bytes) how many of the shmem
875 * given offsets are swapped out.
876 *
877 * This is safe to call without i_rwsem or the
878 * as long as the inode doesn't go away and ra
879 */
880 unsigned long shmem_partial_swap_usage(struct
881
882 {
883 XA_STATE(xas, &mapping->i_pages, start
884 struct page *page;
885 unsigned long swapped = 0;
886 unsigned long max = end - 1;
887
888 rcu_read_lock();
889 xas_for_each(&xas, page, max) {
890 if (xas_retry(&xas, page))
891 continue;
892 if (xa_is_value(page))
893 swapped += 1 << xas_ge
894 if (xas.xa_index == max)
895 break;
896 if (need_resched()) {
897 xas_pause(&xas);
898 cond_resched_rcu();
899 }
900 }
901 rcu_read_unlock();
902
903 return swapped << PAGE_SHIFT;
904 }
905
906 /*
907 * Determine (in bytes) how many of the shmem
908 * given vma is swapped out.
909 *
910 * This is safe to call without i_rwsem or the
911 * as long as the inode doesn't go away and ra
912 */
913 unsigned long shmem_swap_usage(struct vm_area_
914 {
915 struct inode *inode = file_inode(vma->
916 struct shmem_inode_info *info = SHMEM_
917 struct address_space *mapping = inode-
918 unsigned long swapped;
919
920 /* Be careful as we don't hold info->l
921 swapped = READ_ONCE(info->swapped);
922
923 /*
924 * The easier cases are when the shmem
925 * the vma maps it whole. Then we can
926 * already track.
927 */
928 if (!swapped)
929 return 0;
930
931 if (!vma->vm_pgoff && vma->vm_end - vm
932 return swapped << PAGE_SHIFT;
933
934 /* Here comes the more involved part *
935 return shmem_partial_swap_usage(mappin
936 vma->v
937 }
938
939 /*
940 * SysV IPC SHM_UNLOCK restore Unevictable pag
941 */
942 void shmem_unlock_mapping(struct address_space
943 {
944 struct folio_batch fbatch;
945 pgoff_t index = 0;
946
947 folio_batch_init(&fbatch);
948 /*
949 * Minor point, but we might as well s
950 */
951 while (!mapping_unevictable(mapping) &
952 filemap_get_folios(mapping, &in
953 check_move_unevictable_folios(
954 folio_batch_release(&fbatch);
955 cond_resched();
956 }
957 }
958
959 static struct folio *shmem_get_partial_folio(s
960 {
961 struct folio *folio;
962
963 /*
964 * At first avoid shmem_get_folio(,,,S
965 * beyond i_size, and reports fallocat
966 */
967 folio = filemap_get_entry(inode->i_map
968 if (!folio)
969 return folio;
970 if (!xa_is_value(folio)) {
971 folio_lock(folio);
972 if (folio->mapping == inode->i
973 return folio;
974 /* The folio has been swapped
975 folio_unlock(folio);
976 folio_put(folio);
977 }
978 /*
979 * But read a folio back from swap if
980 * (although in some cases this is jus
981 */
982 folio = NULL;
983 shmem_get_folio(inode, index, 0, &foli
984 return folio;
985 }
986
987 /*
988 * Remove range of pages and swap entries from
989 * If !unfalloc, truncate or punch hole; if un
990 */
991 static void shmem_undo_range(struct inode *ino
992
993 {
994 struct address_space *mapping = inode-
995 struct shmem_inode_info *info = SHMEM_
996 pgoff_t start = (lstart + PAGE_SIZE -
997 pgoff_t end = (lend + 1) >> PAGE_SHIFT
998 struct folio_batch fbatch;
999 pgoff_t indices[PAGEVEC_SIZE];
1000 struct folio *folio;
1001 bool same_folio;
1002 long nr_swaps_freed = 0;
1003 pgoff_t index;
1004 int i;
1005
1006 if (lend == -1)
1007 end = -1; /* unsigned,
1008
1009 if (info->fallocend > start && info->
1010 info->fallocend = start;
1011
1012 folio_batch_init(&fbatch);
1013 index = start;
1014 while (index < end && find_lock_entri
1015 &fbatch, indices)) {
1016 for (i = 0; i < folio_batch_c
1017 folio = fbatch.folios
1018
1019 if (xa_is_value(folio
1020 if (unfalloc)
1021 conti
1022 nr_swaps_free
1023
1024 continue;
1025 }
1026
1027 if (!unfalloc || !fol
1028 truncate_inod
1029 folio_unlock(folio);
1030 }
1031 folio_batch_remove_exceptiona
1032 folio_batch_release(&fbatch);
1033 cond_resched();
1034 }
1035
1036 /*
1037 * When undoing a failed fallocate, w
1038 * zeroing and splitting below, but s
1039 * folio when !uptodate indicates tha
1040 * even when [lstart, lend] covers on
1041 */
1042 if (unfalloc)
1043 goto whole_folios;
1044
1045 same_folio = (lstart >> PAGE_SHIFT) =
1046 folio = shmem_get_partial_folio(inode
1047 if (folio) {
1048 same_folio = lend < folio_pos
1049 folio_mark_dirty(folio);
1050 if (!truncate_inode_partial_f
1051 start = folio_next_in
1052 if (same_folio)
1053 end = folio->
1054 }
1055 folio_unlock(folio);
1056 folio_put(folio);
1057 folio = NULL;
1058 }
1059
1060 if (!same_folio)
1061 folio = shmem_get_partial_fol
1062 if (folio) {
1063 folio_mark_dirty(folio);
1064 if (!truncate_inode_partial_f
1065 end = folio->index;
1066 folio_unlock(folio);
1067 folio_put(folio);
1068 }
1069
1070 whole_folios:
1071
1072 index = start;
1073 while (index < end) {
1074 cond_resched();
1075
1076 if (!find_get_entries(mapping
1077 indices)) {
1078 /* If all gone or hol
1079 if (index == start ||
1080 break;
1081 /* But if truncating,
1082 index = start;
1083 continue;
1084 }
1085 for (i = 0; i < folio_batch_c
1086 folio = fbatch.folios
1087
1088 if (xa_is_value(folio
1089 long swaps_fr
1090
1091 if (unfalloc)
1092 conti
1093 swaps_freed =
1094 if (!swaps_fr
1095 /* Sw
1096 index
1097 break
1098 }
1099 nr_swaps_free
1100 continue;
1101 }
1102
1103 folio_lock(folio);
1104
1105 if (!unfalloc || !fol
1106 if (folio_map
1107 /* Pa
1108 folio
1109 index
1110 break
1111 }
1112 VM_BUG_ON_FOL
1113
1114
1115 if (!folio_te
1116 trunc
1117 } else if (tr
1118 /*
1119 * If
1120 * th
1121 * Ot
1122 * dr
1123 * ze
1124 * is
1125 */
1126 if (!
1127
1128
1129
1130 }
1131 }
1132 }
1133 folio_unlock(folio);
1134 }
1135 folio_batch_remove_exceptiona
1136 folio_batch_release(&fbatch);
1137 }
1138
1139 shmem_recalc_inode(inode, 0, -nr_swap
1140 }
1141
1142 void shmem_truncate_range(struct inode *inode
1143 {
1144 shmem_undo_range(inode, lstart, lend,
1145 inode_set_mtime_to_ts(inode, inode_se
1146 inode_inc_iversion(inode);
1147 }
1148 EXPORT_SYMBOL_GPL(shmem_truncate_range);
1149
1150 static int shmem_getattr(struct mnt_idmap *id
1151 const struct path *p
1152 u32 request_mask, un
1153 {
1154 struct inode *inode = path->dentry->d
1155 struct shmem_inode_info *info = SHMEM
1156
1157 if (info->alloced - info->swapped !=
1158 shmem_recalc_inode(inode, 0,
1159
1160 if (info->fsflags & FS_APPEND_FL)
1161 stat->attributes |= STATX_ATT
1162 if (info->fsflags & FS_IMMUTABLE_FL)
1163 stat->attributes |= STATX_ATT
1164 if (info->fsflags & FS_NODUMP_FL)
1165 stat->attributes |= STATX_ATT
1166 stat->attributes_mask |= (STATX_ATTR_
1167 STATX_ATTR_IMMUTABLE
1168 STATX_ATTR_NODUMP);
1169 inode_lock_shared(inode);
1170 generic_fillattr(idmap, request_mask,
1171 inode_unlock_shared(inode);
1172
1173 if (shmem_huge_global_enabled(inode,
1174 stat->blksize = HPAGE_PMD_SIZ
1175
1176 if (request_mask & STATX_BTIME) {
1177 stat->result_mask |= STATX_BT
1178 stat->btime.tv_sec = info->i_
1179 stat->btime.tv_nsec = info->i
1180 }
1181
1182 return 0;
1183 }
1184
1185 static int shmem_setattr(struct mnt_idmap *id
1186 struct dentry *dentr
1187 {
1188 struct inode *inode = d_inode(dentry)
1189 struct shmem_inode_info *info = SHMEM
1190 int error;
1191 bool update_mtime = false;
1192 bool update_ctime = true;
1193
1194 error = setattr_prepare(idmap, dentry
1195 if (error)
1196 return error;
1197
1198 if ((info->seals & F_SEAL_EXEC) && (a
1199 if ((inode->i_mode ^ attr->ia
1200 return -EPERM;
1201 }
1202 }
1203
1204 if (S_ISREG(inode->i_mode) && (attr->
1205 loff_t oldsize = inode->i_siz
1206 loff_t newsize = attr->ia_siz
1207
1208 /* protected by i_rwsem */
1209 if ((newsize < oldsize && (in
1210 (newsize > oldsize && (in
1211 return -EPERM;
1212
1213 if (newsize != oldsize) {
1214 error = shmem_reacct_
1215 oldsi
1216 if (error)
1217 return error;
1218 i_size_write(inode, n
1219 update_mtime = true;
1220 } else {
1221 update_ctime = false;
1222 }
1223 if (newsize <= oldsize) {
1224 loff_t holebegin = ro
1225 if (oldsize > holebeg
1226 unmap_mapping
1227
1228 if (info->alloced)
1229 shmem_truncat
1230
1231 /* unmap again to rem
1232 if (oldsize > holebeg
1233 unmap_mapping
1234
1235 }
1236 }
1237
1238 if (is_quota_modification(idmap, inod
1239 error = dquot_initialize(inod
1240 if (error)
1241 return error;
1242 }
1243
1244 /* Transfer quota accounting */
1245 if (i_uid_needs_update(idmap, attr, i
1246 i_gid_needs_update(idmap, attr, i
1247 error = dquot_transfer(idmap,
1248 if (error)
1249 return error;
1250 }
1251
1252 setattr_copy(idmap, inode, attr);
1253 if (attr->ia_valid & ATTR_MODE)
1254 error = posix_acl_chmod(idmap
1255 if (!error && update_ctime) {
1256 inode_set_ctime_current(inode
1257 if (update_mtime)
1258 inode_set_mtime_to_ts
1259 inode_inc_iversion(inode);
1260 }
1261 return error;
1262 }
1263
1264 static void shmem_evict_inode(struct inode *i
1265 {
1266 struct shmem_inode_info *info = SHMEM
1267 struct shmem_sb_info *sbinfo = SHMEM_
1268 size_t freed = 0;
1269
1270 if (shmem_mapping(inode->i_mapping))
1271 shmem_unacct_size(info->flags
1272 inode->i_size = 0;
1273 mapping_set_exiting(inode->i_
1274 shmem_truncate_range(inode, 0
1275 if (!list_empty(&info->shrink
1276 spin_lock(&sbinfo->sh
1277 if (!list_empty(&info
1278 list_del_init
1279 sbinfo->shrin
1280 }
1281 spin_unlock(&sbinfo->
1282 }
1283 while (!list_empty(&info->swa
1284 /* Wait while shmem_u
1285 wait_var_event(&info-
1286 !atomi
1287 mutex_lock(&shmem_swa
1288 /* ...but beware of t
1289 if (!atomic_read(&inf
1290 list_del_init
1291 mutex_unlock(&shmem_s
1292 }
1293 }
1294
1295 simple_xattrs_free(&info->xattrs, sbi
1296 shmem_free_inode(inode->i_sb, freed);
1297 WARN_ON(inode->i_blocks);
1298 clear_inode(inode);
1299 #ifdef CONFIG_TMPFS_QUOTA
1300 dquot_free_inode(inode);
1301 dquot_drop(inode);
1302 #endif
1303 }
1304
1305 static int shmem_find_swap_entries(struct add
1306 pgoff_t st
1307 pgoff_t *i
1308 {
1309 XA_STATE(xas, &mapping->i_pages, star
1310 struct folio *folio;
1311 swp_entry_t entry;
1312
1313 rcu_read_lock();
1314 xas_for_each(&xas, folio, ULONG_MAX)
1315 if (xas_retry(&xas, folio))
1316 continue;
1317
1318 if (!xa_is_value(folio))
1319 continue;
1320
1321 entry = radix_to_swp_entry(fo
1322 /*
1323 * swapin error entries can b
1324 * deliberately ignored here
1325 */
1326 if (swp_type(entry) != type)
1327 continue;
1328
1329 indices[folio_batch_count(fba
1330 if (!folio_batch_add(fbatch,
1331 break;
1332
1333 if (need_resched()) {
1334 xas_pause(&xas);
1335 cond_resched_rcu();
1336 }
1337 }
1338 rcu_read_unlock();
1339
1340 return xas.xa_index;
1341 }
1342
1343 /*
1344 * Move the swapped pages for an inode to pag
1345 * of pages swapped in, or the error in case
1346 */
1347 static int shmem_unuse_swap_entries(struct in
1348 struct folio_batch *fbatch, p
1349 {
1350 int i = 0;
1351 int ret = 0;
1352 int error = 0;
1353 struct address_space *mapping = inode
1354
1355 for (i = 0; i < folio_batch_count(fba
1356 struct folio *folio = fbatch-
1357
1358 if (!xa_is_value(folio))
1359 continue;
1360 error = shmem_swapin_folio(in
1361 mappi
1362 if (error == 0) {
1363 folio_unlock(folio);
1364 folio_put(folio);
1365 ret++;
1366 }
1367 if (error == -ENOMEM)
1368 break;
1369 error = 0;
1370 }
1371 return error ? error : ret;
1372 }
1373
1374 /*
1375 * If swap found in inode, free it and move p
1376 */
1377 static int shmem_unuse_inode(struct inode *in
1378 {
1379 struct address_space *mapping = inode
1380 pgoff_t start = 0;
1381 struct folio_batch fbatch;
1382 pgoff_t indices[PAGEVEC_SIZE];
1383 int ret = 0;
1384
1385 do {
1386 folio_batch_init(&fbatch);
1387 shmem_find_swap_entries(mappi
1388 if (folio_batch_count(&fbatch
1389 ret = 0;
1390 break;
1391 }
1392
1393 ret = shmem_unuse_swap_entrie
1394 if (ret < 0)
1395 break;
1396
1397 start = indices[folio_batch_c
1398 } while (true);
1399
1400 return ret;
1401 }
1402
1403 /*
1404 * Read all the shared memory data that resid
1405 * device 'type' back into memory, so the swa
1406 * unused.
1407 */
1408 int shmem_unuse(unsigned int type)
1409 {
1410 struct shmem_inode_info *info, *next;
1411 int error = 0;
1412
1413 if (list_empty(&shmem_swaplist))
1414 return 0;
1415
1416 mutex_lock(&shmem_swaplist_mutex);
1417 list_for_each_entry_safe(info, next,
1418 if (!info->swapped) {
1419 list_del_init(&info->
1420 continue;
1421 }
1422 /*
1423 * Drop the swaplist mutex wh
1424 * but before doing so, make
1425 * remove placeholder inode f
1426 * (igrab() would protect fro
1427 */
1428 atomic_inc(&info->stop_evicti
1429 mutex_unlock(&shmem_swaplist_
1430
1431 error = shmem_unuse_inode(&in
1432 cond_resched();
1433
1434 mutex_lock(&shmem_swaplist_mu
1435 next = list_next_entry(info,
1436 if (!info->swapped)
1437 list_del_init(&info->
1438 if (atomic_dec_and_test(&info
1439 wake_up_var(&info->st
1440 if (error)
1441 break;
1442 }
1443 mutex_unlock(&shmem_swaplist_mutex);
1444
1445 return error;
1446 }
1447
1448 /*
1449 * Move the page from the page cache to the s
1450 */
1451 static int shmem_writepage(struct page *page,
1452 {
1453 struct folio *folio = page_folio(page
1454 struct address_space *mapping = folio
1455 struct inode *inode = mapping->host;
1456 struct shmem_inode_info *info = SHMEM
1457 struct shmem_sb_info *sbinfo = SHMEM_
1458 swp_entry_t swap;
1459 pgoff_t index;
1460 int nr_pages;
1461 bool split = false;
1462
1463 /*
1464 * Our capabilities prevent regular w
1465 * shmem_writepage; but a stacking fi
1466 * its underlying filesystem, in whic
1467 * swap only in response to memory pr
1468 * threads or sync.
1469 */
1470 if (WARN_ON_ONCE(!wbc->for_reclaim))
1471 goto redirty;
1472
1473 if (WARN_ON_ONCE((info->flags & VM_LO
1474 goto redirty;
1475
1476 if (!total_swap_pages)
1477 goto redirty;
1478
1479 /*
1480 * If CONFIG_THP_SWAP is not enabled,
1481 * split when swapping.
1482 *
1483 * And shrinkage of pages beyond i_si
1484 * swapout of a large folio crossing
1485 * (unless fallocate has been used to
1486 */
1487 if (folio_test_large(folio)) {
1488 index = shmem_fallocend(inode
1489 DIV_ROUND_UP(i_size_r
1490 if ((index > folio->index &&
1491 !IS_ENABLED(CONFIG_THP_SW
1492 split = true;
1493 }
1494
1495 if (split) {
1496 try_split:
1497 /* Ensure the subpages are st
1498 folio_test_set_dirty(folio);
1499 if (split_huge_page_to_list_t
1500 goto redirty;
1501 folio = page_folio(page);
1502 folio_clear_dirty(folio);
1503 }
1504
1505 index = folio->index;
1506 nr_pages = folio_nr_pages(folio);
1507
1508 /*
1509 * This is somewhat ridiculous, but w
1510 * value into swapfile.c, the only wa
1511 * fallocated folio arriving here is
1512 *
1513 * That's okay for a folio already fa
1514 * not yet completed the fallocation,
1515 * of this folio in case we have to u
1516 * good idea to continue anyway, once
1517 * reactivate the folio, and let shme
1518 */
1519 if (!folio_test_uptodate(folio)) {
1520 if (inode->i_private) {
1521 struct shmem_falloc *
1522 spin_lock(&inode->i_l
1523 shmem_falloc = inode-
1524 if (shmem_falloc &&
1525 !shmem_falloc->wa
1526 index >= shmem_fa
1527 index < shmem_fal
1528 shmem_falloc-
1529 else
1530 shmem_falloc
1531 spin_unlock(&inode->i
1532 if (shmem_falloc)
1533 goto redirty;
1534 }
1535 folio_zero_range(folio, 0, fo
1536 flush_dcache_folio(folio);
1537 folio_mark_uptodate(folio);
1538 }
1539
1540 swap = folio_alloc_swap(folio);
1541 if (!swap.val) {
1542 if (nr_pages > 1)
1543 goto try_split;
1544
1545 goto redirty;
1546 }
1547
1548 /*
1549 * Add inode to shmem_unuse()'s list
1550 * if it's not already there. Do it
1551 * moved to swap cache, when its page
1552 * the inode from eviction. But don'
1553 * we've incremented swapped, because
1554 * prune a !swapped inode from the sw
1555 */
1556 mutex_lock(&shmem_swaplist_mutex);
1557 if (list_empty(&info->swaplist))
1558 list_add(&info->swaplist, &sh
1559
1560 if (add_to_swap_cache(folio, swap,
1561 __GFP_HIGH | __GFP_NO
1562 NULL) == 0) {
1563 shmem_recalc_inode(inode, 0,
1564 swap_shmem_alloc(swap, nr_pag
1565 shmem_delete_from_page_cache(
1566
1567 mutex_unlock(&shmem_swaplist_
1568 BUG_ON(folio_mapped(folio));
1569 return swap_writepage(&folio-
1570 }
1571
1572 mutex_unlock(&shmem_swaplist_mutex);
1573 put_swap_folio(folio, swap);
1574 redirty:
1575 folio_mark_dirty(folio);
1576 if (wbc->for_reclaim)
1577 return AOP_WRITEPAGE_ACTIVATE
1578 folio_unlock(folio);
1579 return 0;
1580 }
1581
1582 #if defined(CONFIG_NUMA) && defined(CONFIG_TM
1583 static void shmem_show_mpol(struct seq_file *
1584 {
1585 char buffer[64];
1586
1587 if (!mpol || mpol->mode == MPOL_DEFAU
1588 return; /* show nothi
1589
1590 mpol_to_str(buffer, sizeof(buffer), m
1591
1592 seq_printf(seq, ",mpol=%s", buffer);
1593 }
1594
1595 static struct mempolicy *shmem_get_sbmpol(str
1596 {
1597 struct mempolicy *mpol = NULL;
1598 if (sbinfo->mpol) {
1599 raw_spin_lock(&sbinfo->stat_l
1600 mpol = sbinfo->mpol;
1601 mpol_get(mpol);
1602 raw_spin_unlock(&sbinfo->stat
1603 }
1604 return mpol;
1605 }
1606 #else /* !CONFIG_NUMA || !CONFIG_TMPFS */
1607 static inline void shmem_show_mpol(struct seq
1608 {
1609 }
1610 static inline struct mempolicy *shmem_get_sbm
1611 {
1612 return NULL;
1613 }
1614 #endif /* CONFIG_NUMA && CONFIG_TMPFS */
1615
1616 static struct mempolicy *shmem_get_pgoff_poli
1617 pgoff_t index, unsign
1618
1619 static struct folio *shmem_swapin_cluster(swp
1620 struct shmem_inode_in
1621 {
1622 struct mempolicy *mpol;
1623 pgoff_t ilx;
1624 struct folio *folio;
1625
1626 mpol = shmem_get_pgoff_policy(info, i
1627 folio = swap_cluster_readahead(swap,
1628 mpol_cond_put(mpol);
1629
1630 return folio;
1631 }
1632
1633 /*
1634 * Make sure huge_gfp is always more limited
1635 * Some of the flags set permissions, while o
1636 */
1637 static gfp_t limit_gfp_mask(gfp_t huge_gfp, g
1638 {
1639 gfp_t allowflags = __GFP_IO | __GFP_F
1640 gfp_t denyflags = __GFP_NOWARN | __GF
1641 gfp_t zoneflags = limit_gfp & GFP_ZON
1642 gfp_t result = huge_gfp & ~(allowflag
1643
1644 /* Allow allocations only from the or
1645 result |= zoneflags;
1646
1647 /*
1648 * Minimize the result gfp by taking
1649 * and the intersection of the allow
1650 */
1651 result |= (limit_gfp & denyflags);
1652 result |= (huge_gfp & limit_gfp) & al
1653
1654 return result;
1655 }
1656
1657 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1658 unsigned long shmem_allowable_huge_orders(str
1659 struct vm_are
1660 loff_t write_
1661 {
1662 unsigned long mask = READ_ONCE(huge_s
1663 unsigned long within_size_orders = RE
1664 unsigned long vm_flags = vma ? vma->v
1665 bool global_huge;
1666 loff_t i_size;
1667 int order;
1668
1669 if (thp_disabled_by_hw() || (vma && v
1670 return 0;
1671
1672 global_huge = shmem_huge_global_enabl
1673 shmem
1674 if (!vma || !vma_is_anon_shmem(vma))
1675 /*
1676 * For tmpfs, we now only sup
1677 * is enabled, otherwise fall
1678 */
1679 return global_huge ? BIT(HPAG
1680 }
1681
1682 /*
1683 * Following the 'deny' semantics of
1684 * option off from all mounts.
1685 */
1686 if (shmem_huge == SHMEM_HUGE_DENY)
1687 return 0;
1688
1689 /*
1690 * Only allow inherit orders if the t
1691 * means non-PMD sized THP can not ov
1692 */
1693 if (shmem_huge == SHMEM_HUGE_FORCE)
1694 return READ_ONCE(huge_shmem_o
1695
1696 /* Allow mTHP that will be fully with
1697 order = highest_order(within_size_ord
1698 while (within_size_orders) {
1699 index = round_up(index + 1, o
1700 i_size = round_up(i_size_read
1701 if (i_size >> PAGE_SHIFT >= i
1702 mask |= within_size_o
1703 break;
1704 }
1705
1706 order = next_order(&within_si
1707 }
1708
1709 if (vm_flags & VM_HUGEPAGE)
1710 mask |= READ_ONCE(huge_shmem_
1711
1712 if (global_huge)
1713 mask |= READ_ONCE(huge_shmem_
1714
1715 return THP_ORDERS_ALL_FILE_DEFAULT &
1716 }
1717
1718 static unsigned long shmem_suitable_orders(st
1719 st
1720 un
1721 {
1722 struct vm_area_struct *vma = vmf ? vm
1723 pgoff_t aligned_index;
1724 unsigned long pages;
1725 int order;
1726
1727 if (vma) {
1728 orders = thp_vma_suitable_ord
1729 if (!orders)
1730 return 0;
1731 }
1732
1733 /* Find the highest order that can ad
1734 order = highest_order(orders);
1735 while (orders) {
1736 pages = 1UL << order;
1737 aligned_index = round_down(in
1738 /*
1739 * Check for conflict before
1740 * Conflict might be that a h
1741 * and added to page cache by
1742 * is already at least one sm
1743 * Be careful to retry when a
1744 * Elsewhere -EEXIST would be
1745 */
1746 if (!xa_find(&mapping->i_page
1747 aligned_index +
1748 break;
1749 order = next_order(&orders, o
1750 }
1751
1752 return orders;
1753 }
1754 #else
1755 static unsigned long shmem_suitable_orders(st
1756 st
1757 un
1758 {
1759 return 0;
1760 }
1761 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
1762
1763 static struct folio *shmem_alloc_folio(gfp_t
1764 struct shmem_inode_info *info
1765 {
1766 struct mempolicy *mpol;
1767 pgoff_t ilx;
1768 struct folio *folio;
1769
1770 mpol = shmem_get_pgoff_policy(info, i
1771 folio = folio_alloc_mpol(gfp, order,
1772 mpol_cond_put(mpol);
1773
1774 return folio;
1775 }
1776
1777 static struct folio *shmem_alloc_and_add_foli
1778 gfp_t gfp, struct inode *inod
1779 struct mm_struct *fault_mm, u
1780 {
1781 struct address_space *mapping = inode
1782 struct shmem_inode_info *info = SHMEM
1783 unsigned long suitable_orders = 0;
1784 struct folio *folio = NULL;
1785 long pages;
1786 int error, order;
1787
1788 if (!IS_ENABLED(CONFIG_TRANSPARENT_HU
1789 orders = 0;
1790
1791 if (orders > 0) {
1792 suitable_orders = shmem_suita
1793
1794
1795 order = highest_order(suitabl
1796 while (suitable_orders) {
1797 pages = 1UL << order;
1798 index = round_down(in
1799 folio = shmem_alloc_f
1800 if (folio)
1801 goto allocate
1802
1803 if (pages == HPAGE_PM
1804 count_vm_even
1805 count_mthp_stat(order
1806 order = next_order(&s
1807 }
1808 } else {
1809 pages = 1;
1810 folio = shmem_alloc_folio(gfp
1811 }
1812 if (!folio)
1813 return ERR_PTR(-ENOMEM);
1814
1815 allocated:
1816 __folio_set_locked(folio);
1817 __folio_set_swapbacked(folio);
1818
1819 gfp &= GFP_RECLAIM_MASK;
1820 error = mem_cgroup_charge(folio, faul
1821 if (error) {
1822 if (xa_find(&mapping->i_pages
1823 index + pages
1824 error = -EEXIST;
1825 } else if (pages > 1) {
1826 if (pages == HPAGE_PM
1827 count_vm_even
1828 count_vm_even
1829 }
1830 count_mthp_stat(folio
1831 count_mthp_stat(folio
1832 }
1833 goto unlock;
1834 }
1835
1836 error = shmem_add_to_page_cache(folio
1837 if (error)
1838 goto unlock;
1839
1840 error = shmem_inode_acct_blocks(inode
1841 if (error) {
1842 struct shmem_sb_info *sbinfo
1843 long freed;
1844 /*
1845 * Try to reclaim some space
1846 * large folios beyond i_size
1847 */
1848 shmem_unused_huge_shrink(sbin
1849 /*
1850 * And do a shmem_recalc_inod
1851 * except our folio is there
1852 */
1853 spin_lock(&info->lock);
1854 freed = pages + info->alloced
1855 READ_ONCE(mapping->nr
1856 if (freed > 0)
1857 info->alloced -= free
1858 spin_unlock(&info->lock);
1859 if (freed > 0)
1860 shmem_inode_unacct_bl
1861 error = shmem_inode_acct_bloc
1862 if (error) {
1863 filemap_remove_folio(
1864 goto unlock;
1865 }
1866 }
1867
1868 shmem_recalc_inode(inode, pages, 0);
1869 folio_add_lru(folio);
1870 return folio;
1871
1872 unlock:
1873 folio_unlock(folio);
1874 folio_put(folio);
1875 return ERR_PTR(error);
1876 }
1877
1878 /*
1879 * When a page is moved from swapcache to shm
1880 * usual swapin of shmem_get_folio_gfp(), or
1881 * shmem_unuse_inode()), it may have been rea
1882 * ignorance of the mapping it belongs to. I
1883 * constraints (like the gma500 GEM driver, w
1884 * we may need to copy to a suitable page bef
1885 *
1886 * In a future release, this may well be exte
1887 * NUMA mempolicy, and applied also to anonym
1888 * but for now it is a simple matter of zone.
1889 */
1890 static bool shmem_should_replace_folio(struct
1891 {
1892 return folio_zonenum(folio) > gfp_zon
1893 }
1894
1895 static int shmem_replace_folio(struct folio *
1896 struct shmem_
1897 struct vm_are
1898 {
1899 struct folio *new, *old = *foliop;
1900 swp_entry_t entry = old->swap;
1901 struct address_space *swap_mapping =
1902 pgoff_t swap_index = swap_cache_index
1903 XA_STATE(xas, &swap_mapping->i_pages,
1904 int nr_pages = folio_nr_pages(old);
1905 int error = 0, i;
1906
1907 /*
1908 * We have arrived here because our z
1909 * limit chance of success by further
1910 */
1911 gfp &= ~GFP_CONSTRAINT_MASK;
1912 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1913 if (nr_pages > 1) {
1914 gfp_t huge_gfp = vma_thp_gfp_
1915
1916 gfp = limit_gfp_mask(huge_gfp
1917 }
1918 #endif
1919
1920 new = shmem_alloc_folio(gfp, folio_or
1921 if (!new)
1922 return -ENOMEM;
1923
1924 folio_ref_add(new, nr_pages);
1925 folio_copy(new, old);
1926 flush_dcache_folio(new);
1927
1928 __folio_set_locked(new);
1929 __folio_set_swapbacked(new);
1930 folio_mark_uptodate(new);
1931 new->swap = entry;
1932 folio_set_swapcache(new);
1933
1934 /* Swap cache still stores N entries
1935 xa_lock_irq(&swap_mapping->i_pages);
1936 for (i = 0; i < nr_pages; i++) {
1937 void *item = xas_load(&xas);
1938
1939 if (item != old) {
1940 error = -ENOENT;
1941 break;
1942 }
1943
1944 xas_store(&xas, new);
1945 xas_next(&xas);
1946 }
1947 if (!error) {
1948 mem_cgroup_replace_folio(old,
1949 __lruvec_stat_mod_folio(new,
1950 __lruvec_stat_mod_folio(new,
1951 __lruvec_stat_mod_folio(old,
1952 __lruvec_stat_mod_folio(old,
1953 }
1954 xa_unlock_irq(&swap_mapping->i_pages)
1955
1956 if (unlikely(error)) {
1957 /*
1958 * Is this possible? I think
1959 * check both the swapcache f
1960 * after getting the folio lo
1961 * Reverse old to newpage for
1962 */
1963 old = new;
1964 } else {
1965 folio_add_lru(new);
1966 *foliop = new;
1967 }
1968
1969 folio_clear_swapcache(old);
1970 old->private = NULL;
1971
1972 folio_unlock(old);
1973 /*
1974 * The old folio are removed from swa
1975 * reference, as well as one temporar
1976 * cache.
1977 */
1978 folio_put_refs(old, nr_pages + 1);
1979 return error;
1980 }
1981
1982 static void shmem_set_folio_swapin_error(stru
1983 stru
1984 {
1985 struct address_space *mapping = inode
1986 swp_entry_t swapin_error;
1987 void *old;
1988 int nr_pages;
1989
1990 swapin_error = make_poisoned_swp_entr
1991 old = xa_cmpxchg_irq(&mapping->i_page
1992 swp_to_radix_ent
1993 swp_to_radix_ent
1994 if (old != swp_to_radix_entry(swap))
1995 return;
1996
1997 nr_pages = folio_nr_pages(folio);
1998 folio_wait_writeback(folio);
1999 delete_from_swap_cache(folio);
2000 /*
2001 * Don't treat swapin error folio as
2002 * won't be 0 when inode is released
2003 * in shmem_evict_inode().
2004 */
2005 shmem_recalc_inode(inode, -nr_pages,
2006 swap_free_nr(swap, nr_pages);
2007 }
2008
2009 static int shmem_split_large_entry(struct ino
2010 swp_entry_
2011 {
2012 struct address_space *mapping = inode
2013 XA_STATE_ORDER(xas, &mapping->i_pages
2014 void *alloced_shadow = NULL;
2015 int alloced_order = 0, i;
2016
2017 /* Convert user data gfp flags to xar
2018 gfp &= GFP_RECLAIM_MASK;
2019
2020 for (;;) {
2021 int order = -1, split_order =
2022 void *old = NULL;
2023
2024 xas_lock_irq(&xas);
2025 old = xas_load(&xas);
2026 if (!xa_is_value(old) || swp_
2027 xas_set_err(&xas, -EE
2028 goto unlock;
2029 }
2030
2031 order = xas_get_order(&xas);
2032
2033 /* Swap entry may have change
2034 if (alloced_order &&
2035 (old != alloced_shadow ||
2036 xas_destroy(&xas);
2037 alloced_order = 0;
2038 }
2039
2040 /* Try to split large swap en
2041 if (order > 0) {
2042 if (!alloced_order) {
2043 split_order =
2044 goto unlock;
2045 }
2046 xas_split(&xas, old,
2047
2048 /*
2049 * Re-set the swap en
2050 * offset of the orig
2051 */
2052 for (i = 0; i < 1 <<
2053 pgoff_t align
2054 swp_entry_t t
2055
2056 tmp = swp_ent
2057 __xa_store(&m
2058 sw
2059 }
2060 }
2061
2062 unlock:
2063 xas_unlock_irq(&xas);
2064
2065 /* split needed, alloc here a
2066 if (split_order) {
2067 xas_split_alloc(&xas,
2068 if (xas_error(&xas))
2069 goto error;
2070 alloced_shadow = old;
2071 alloced_order = split
2072 xas_reset(&xas);
2073 continue;
2074 }
2075
2076 if (!xas_nomem(&xas, gfp))
2077 break;
2078 }
2079
2080 error:
2081 if (xas_error(&xas))
2082 return xas_error(&xas);
2083
2084 return alloced_order;
2085 }
2086
2087 /*
2088 * Swap in the folio pointed to by *foliop.
2089 * Caller has to make sure that *foliop conta
2090 * Returns 0 and the folio in foliop if succe
2091 * error code and NULL in *foliop.
2092 */
2093 static int shmem_swapin_folio(struct inode *i
2094 struct folio **f
2095 gfp_t gfp, struc
2096 vm_fault_t *faul
2097 {
2098 struct address_space *mapping = inode
2099 struct mm_struct *fault_mm = vma ? vm
2100 struct shmem_inode_info *info = SHMEM
2101 struct swap_info_struct *si;
2102 struct folio *folio = NULL;
2103 swp_entry_t swap;
2104 int error, nr_pages;
2105
2106 VM_BUG_ON(!*foliop || !xa_is_value(*f
2107 swap = radix_to_swp_entry(*foliop);
2108 *foliop = NULL;
2109
2110 if (is_poisoned_swp_entry(swap))
2111 return -EIO;
2112
2113 si = get_swap_device(swap);
2114 if (!si) {
2115 if (!shmem_confirm_swap(mappi
2116 return -EEXIST;
2117 else
2118 return -EINVAL;
2119 }
2120
2121 /* Look it up and read it in.. */
2122 folio = swap_cache_get_folio(swap, NU
2123 if (!folio) {
2124 int split_order;
2125
2126 /* Or update major stats only
2127 if (fault_type) {
2128 *fault_type |= VM_FAU
2129 count_vm_event(PGMAJF
2130 count_memcg_event_mm(
2131 }
2132
2133 /*
2134 * Now swap device can only s
2135 * should split the large swa
2136 * if necessary.
2137 */
2138 split_order = shmem_split_lar
2139 if (split_order < 0) {
2140 error = split_order;
2141 goto failed;
2142 }
2143
2144 /*
2145 * If the large swap entry ha
2146 * necessary to recalculate t
2147 * the old order alignment.
2148 */
2149 if (split_order > 0) {
2150 pgoff_t offset = inde
2151
2152 swap = swp_entry(swp_
2153 }
2154
2155 /* Here we actually start the
2156 folio = shmem_swapin_cluster(
2157 if (!folio) {
2158 error = -ENOMEM;
2159 goto failed;
2160 }
2161 }
2162
2163 /* We have to do this with folio lock
2164 folio_lock(folio);
2165 if (!folio_test_swapcache(folio) ||
2166 folio->swap.val != swap.val ||
2167 !shmem_confirm_swap(mapping, inde
2168 error = -EEXIST;
2169 goto unlock;
2170 }
2171 if (!folio_test_uptodate(folio)) {
2172 error = -EIO;
2173 goto failed;
2174 }
2175 folio_wait_writeback(folio);
2176 nr_pages = folio_nr_pages(folio);
2177
2178 /*
2179 * Some architectures may have to res
2180 * folio after reading from swap.
2181 */
2182 arch_swap_restore(folio_swap(swap, fo
2183
2184 if (shmem_should_replace_folio(folio,
2185 error = shmem_replace_folio(&
2186 if (error)
2187 goto failed;
2188 }
2189
2190 error = shmem_add_to_page_cache(folio
2191 round
2192 swp_t
2193 if (error)
2194 goto failed;
2195
2196 shmem_recalc_inode(inode, 0, -nr_page
2197
2198 if (sgp == SGP_WRITE)
2199 folio_mark_accessed(folio);
2200
2201 delete_from_swap_cache(folio);
2202 folio_mark_dirty(folio);
2203 swap_free_nr(swap, nr_pages);
2204 put_swap_device(si);
2205
2206 *foliop = folio;
2207 return 0;
2208 failed:
2209 if (!shmem_confirm_swap(mapping, inde
2210 error = -EEXIST;
2211 if (error == -EIO)
2212 shmem_set_folio_swapin_error(
2213 unlock:
2214 if (folio) {
2215 folio_unlock(folio);
2216 folio_put(folio);
2217 }
2218 put_swap_device(si);
2219
2220 return error;
2221 }
2222
2223 /*
2224 * shmem_get_folio_gfp - find page in cache,
2225 *
2226 * If we allocate a new one we do not mark it
2227 * vm. If we swap it in we mark it dirty sinc
2228 * entry since a page cannot live in both the
2229 *
2230 * vmf and fault_type are only supplied by sh
2231 */
2232 static int shmem_get_folio_gfp(struct inode *
2233 loff_t write_end, struct foli
2234 gfp_t gfp, struct vm_fault *v
2235 {
2236 struct vm_area_struct *vma = vmf ? vm
2237 struct mm_struct *fault_mm;
2238 struct folio *folio;
2239 int error;
2240 bool alloced;
2241 unsigned long orders = 0;
2242
2243 if (WARN_ON_ONCE(!shmem_mapping(inode
2244 return -EINVAL;
2245
2246 if (index > (MAX_LFS_FILESIZE >> PAGE
2247 return -EFBIG;
2248 repeat:
2249 if (sgp <= SGP_CACHE &&
2250 ((loff_t)index << PAGE_SHIFT) >=
2251 return -EINVAL;
2252
2253 alloced = false;
2254 fault_mm = vma ? vma->vm_mm : NULL;
2255
2256 folio = filemap_get_entry(inode->i_ma
2257 if (folio && vma && userfaultfd_minor
2258 if (!xa_is_value(folio))
2259 folio_put(folio);
2260 *fault_type = handle_userfaul
2261 return 0;
2262 }
2263
2264 if (xa_is_value(folio)) {
2265 error = shmem_swapin_folio(in
2266 sg
2267 if (error == -EEXIST)
2268 goto repeat;
2269
2270 *foliop = folio;
2271 return error;
2272 }
2273
2274 if (folio) {
2275 folio_lock(folio);
2276
2277 /* Has the folio been truncat
2278 if (unlikely(folio->mapping !
2279 folio_unlock(folio);
2280 folio_put(folio);
2281 goto repeat;
2282 }
2283 if (sgp == SGP_WRITE)
2284 folio_mark_accessed(f
2285 if (folio_test_uptodate(folio
2286 goto out;
2287 /* fallocated folio */
2288 if (sgp != SGP_READ)
2289 goto clear;
2290 folio_unlock(folio);
2291 folio_put(folio);
2292 }
2293
2294 /*
2295 * SGP_READ: succeed on hole, with NU
2296 * SGP_NOALLOC: fail on hole, with NU
2297 */
2298 *foliop = NULL;
2299 if (sgp == SGP_READ)
2300 return 0;
2301 if (sgp == SGP_NOALLOC)
2302 return -ENOENT;
2303
2304 /*
2305 * Fast cache lookup and swap lookup
2306 */
2307
2308 if (vma && userfaultfd_missing(vma))
2309 *fault_type = handle_userfaul
2310 return 0;
2311 }
2312
2313 /* Find hugepage orders that are allo
2314 orders = shmem_allowable_huge_orders(
2315 if (orders > 0) {
2316 gfp_t huge_gfp;
2317
2318 huge_gfp = vma_thp_gfp_mask(v
2319 huge_gfp = limit_gfp_mask(hug
2320 folio = shmem_alloc_and_add_f
2321 inode, index,
2322 if (!IS_ERR(folio)) {
2323 if (folio_test_pmd_ma
2324 count_vm_even
2325 count_mthp_stat(folio
2326 goto alloced;
2327 }
2328 if (PTR_ERR(folio) == -EEXIST
2329 goto repeat;
2330 }
2331
2332 folio = shmem_alloc_and_add_folio(vmf
2333 if (IS_ERR(folio)) {
2334 error = PTR_ERR(folio);
2335 if (error == -EEXIST)
2336 goto repeat;
2337 folio = NULL;
2338 goto unlock;
2339 }
2340
2341 alloced:
2342 alloced = true;
2343 if (folio_test_large(folio) &&
2344 DIV_ROUND_UP(i_size_read(inode),
2345 folio
2346 struct shmem_sb_info *sbinfo
2347 struct shmem_inode_info *info
2348 /*
2349 * Part of the large folio is
2350 * to shrink under memory pre
2351 */
2352 spin_lock(&sbinfo->shrinklist
2353 /*
2354 * _careful to defend against
2355 * ->shrink_list in shmem_unu
2356 */
2357 if (list_empty_careful(&info-
2358 list_add_tail(&info->
2359 &sbinfo
2360 sbinfo->shrinklist_le
2361 }
2362 spin_unlock(&sbinfo->shrinkli
2363 }
2364
2365 if (sgp == SGP_WRITE)
2366 folio_set_referenced(folio);
2367 /*
2368 * Let SGP_FALLOC use the SGP_WRITE o
2369 */
2370 if (sgp == SGP_FALLOC)
2371 sgp = SGP_WRITE;
2372 clear:
2373 /*
2374 * Let SGP_WRITE caller clear ends if
2375 * but SGP_FALLOC on a folio fallocat
2376 * it now, lest undo on failure cance
2377 */
2378 if (sgp != SGP_WRITE && !folio_test_u
2379 long i, n = folio_nr_pages(fo
2380
2381 for (i = 0; i < n; i++)
2382 clear_highpage(folio_
2383 flush_dcache_folio(folio);
2384 folio_mark_uptodate(folio);
2385 }
2386
2387 /* Perhaps the file has been truncate
2388 if (sgp <= SGP_CACHE &&
2389 ((loff_t)index << PAGE_SHIFT) >=
2390 error = -EINVAL;
2391 goto unlock;
2392 }
2393 out:
2394 *foliop = folio;
2395 return 0;
2396
2397 /*
2398 * Error recovery.
2399 */
2400 unlock:
2401 if (alloced)
2402 filemap_remove_folio(folio);
2403 shmem_recalc_inode(inode, 0, 0);
2404 if (folio) {
2405 folio_unlock(folio);
2406 folio_put(folio);
2407 }
2408 return error;
2409 }
2410
2411 /**
2412 * shmem_get_folio - find, and lock a shmem f
2413 * @inode: inode to search
2414 * @index: the page index.
2415 * @write_end: end of a write, could extend
2416 * @foliop: pointer to the folio if found
2417 * @sgp: SGP_* flags to control behavi
2418 *
2419 * Looks up the page cache entry at @inode &
2420 * present, it is returned locked with an inc
2421 *
2422 * If the caller modifies data in the folio,
2423 * before unlocking the folio to ensure that
2424 * There is no need to reserve space before c
2425 *
2426 * When no folio is found, the behavior depen
2427 * - for SGP_READ, *@foliop is %NULL and 0 i
2428 * - for SGP_NOALLOC, *@foliop is %NULL and
2429 * - for all other flags a new folio is allo
2430 * page cache and returned locked in @foli
2431 *
2432 * Context: May sleep.
2433 * Return: 0 if successful, else a negative e
2434 */
2435 int shmem_get_folio(struct inode *inode, pgof
2436 struct folio **foliop, en
2437 {
2438 return shmem_get_folio_gfp(inode, ind
2439 mapping_gfp_mask(inod
2440 }
2441 EXPORT_SYMBOL_GPL(shmem_get_folio);
2442
2443 /*
2444 * This is like autoremove_wake_function, but
2445 * entry unconditionally - even if something
2446 * target.
2447 */
2448 static int synchronous_wake_function(wait_que
2449 unsigned int mode, in
2450 {
2451 int ret = default_wake_function(wait,
2452 list_del_init(&wait->entry);
2453 return ret;
2454 }
2455
2456 /*
2457 * Trinity finds that probing a hole which tm
2458 * prevent the hole-punch from ever completin
2459 * locks writers out with its hold on i_rwsem
2460 * faulting pages into the hole while it's be
2461 * shmem_undo_range() does remove the additio
2462 * keep up, as each new page needs its own un
2463 * and the i_mmap tree grows ever slower to s
2464 *
2465 * It does not matter if we sometimes reach t
2466 * hole-punch begins, so that one fault then
2467 * we just need to make racing faults a rare
2468 *
2469 * The implementation below would be much sim
2470 * standard mutex or completion: but we canno
2471 * and bloating every shmem inode for this un
2472 */
2473 static vm_fault_t shmem_falloc_wait(struct vm
2474 {
2475 struct shmem_falloc *shmem_falloc;
2476 struct file *fpin = NULL;
2477 vm_fault_t ret = 0;
2478
2479 spin_lock(&inode->i_lock);
2480 shmem_falloc = inode->i_private;
2481 if (shmem_falloc &&
2482 shmem_falloc->waitq &&
2483 vmf->pgoff >= shmem_falloc->start
2484 vmf->pgoff < shmem_falloc->next)
2485 wait_queue_head_t *shmem_fall
2486 DEFINE_WAIT_FUNC(shmem_fault_
2487
2488 ret = VM_FAULT_NOPAGE;
2489 fpin = maybe_unlock_mmap_for_
2490 shmem_falloc_waitq = shmem_fa
2491 prepare_to_wait(shmem_falloc_
2492 TASK_UNINTERR
2493 spin_unlock(&inode->i_lock);
2494 schedule();
2495
2496 /*
2497 * shmem_falloc_waitq points
2498 * stack of the hole-punching
2499 * is usually invalid by the
2500 * finish_wait() does not der
2501 * though i_lock needed lest
2502 */
2503 spin_lock(&inode->i_lock);
2504 finish_wait(shmem_falloc_wait
2505 }
2506 spin_unlock(&inode->i_lock);
2507 if (fpin) {
2508 fput(fpin);
2509 ret = VM_FAULT_RETRY;
2510 }
2511 return ret;
2512 }
2513
2514 static vm_fault_t shmem_fault(struct vm_fault
2515 {
2516 struct inode *inode = file_inode(vmf-
2517 gfp_t gfp = mapping_gfp_mask(inode->i
2518 struct folio *folio = NULL;
2519 vm_fault_t ret = 0;
2520 int err;
2521
2522 /*
2523 * Trinity finds that probing a hole
2524 * prevent the hole-punch from ever c
2525 */
2526 if (unlikely(inode->i_private)) {
2527 ret = shmem_falloc_wait(vmf,
2528 if (ret)
2529 return ret;
2530 }
2531
2532 WARN_ON_ONCE(vmf->page != NULL);
2533 err = shmem_get_folio_gfp(inode, vmf-
2534 gfp, vmf, &
2535 if (err)
2536 return vmf_error(err);
2537 if (folio) {
2538 vmf->page = folio_file_page(f
2539 ret |= VM_FAULT_LOCKED;
2540 }
2541 return ret;
2542 }
2543
2544 unsigned long shmem_get_unmapped_area(struct
2545 unsigne
2546 unsigne
2547 {
2548 unsigned long addr;
2549 unsigned long offset;
2550 unsigned long inflated_len;
2551 unsigned long inflated_addr;
2552 unsigned long inflated_offset;
2553 unsigned long hpage_size;
2554
2555 if (len > TASK_SIZE)
2556 return -ENOMEM;
2557
2558 addr = mm_get_unmapped_area(current->
2559 flags);
2560
2561 if (!IS_ENABLED(CONFIG_TRANSPARENT_HU
2562 return addr;
2563 if (IS_ERR_VALUE(addr))
2564 return addr;
2565 if (addr & ~PAGE_MASK)
2566 return addr;
2567 if (addr > TASK_SIZE - len)
2568 return addr;
2569
2570 if (shmem_huge == SHMEM_HUGE_DENY)
2571 return addr;
2572 if (flags & MAP_FIXED)
2573 return addr;
2574 /*
2575 * Our priority is to support MAP_SHA
2576 * and support MAP_PRIVATE mapped hug
2577 * But if caller specified an address
2578 * successfully, respect that as befo
2579 */
2580 if (uaddr == addr)
2581 return addr;
2582
2583 hpage_size = HPAGE_PMD_SIZE;
2584 if (shmem_huge != SHMEM_HUGE_FORCE) {
2585 struct super_block *sb;
2586 unsigned long __maybe_unused
2587 int order = 0;
2588
2589 if (file) {
2590 VM_BUG_ON(file->f_op
2591 sb = file_inode(file)
2592 } else {
2593 /*
2594 * Called directly fr
2595 * for "/dev/zero", t
2596 */
2597 if (IS_ERR(shm_mnt))
2598 return addr;
2599 sb = shm_mnt->mnt_sb;
2600
2601 /*
2602 * Find the highest m
2603 * provide a suitable
2604 */
2605 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
2606 hpage_orders = READ_O
2607 hpage_orders |= READ_
2608 hpage_orders |= READ_
2609 if (SHMEM_SB(sb)->hug
2610 hpage_orders
2611
2612 if (hpage_orders > 0)
2613 order = highe
2614 hpage_size =
2615 }
2616 #endif
2617 }
2618 if (SHMEM_SB(sb)->huge == SHM
2619 return addr;
2620 }
2621
2622 if (len < hpage_size)
2623 return addr;
2624
2625 offset = (pgoff << PAGE_SHIFT) & (hpa
2626 if (offset && offset + len < 2 * hpag
2627 return addr;
2628 if ((addr & (hpage_size - 1)) == offs
2629 return addr;
2630
2631 inflated_len = len + hpage_size - PAG
2632 if (inflated_len > TASK_SIZE)
2633 return addr;
2634 if (inflated_len < len)
2635 return addr;
2636
2637 inflated_addr = mm_get_unmapped_area(
2638
2639 if (IS_ERR_VALUE(inflated_addr))
2640 return addr;
2641 if (inflated_addr & ~PAGE_MASK)
2642 return addr;
2643
2644 inflated_offset = inflated_addr & (hp
2645 inflated_addr += offset - inflated_of
2646 if (inflated_offset > offset)
2647 inflated_addr += hpage_size;
2648
2649 if (inflated_addr > TASK_SIZE - len)
2650 return addr;
2651 return inflated_addr;
2652 }
2653
2654 #ifdef CONFIG_NUMA
2655 static int shmem_set_policy(struct vm_area_st
2656 {
2657 struct inode *inode = file_inode(vma-
2658 return mpol_set_shared_policy(&SHMEM_
2659 }
2660
2661 static struct mempolicy *shmem_get_policy(str
2662 uns
2663 {
2664 struct inode *inode = file_inode(vma-
2665 pgoff_t index;
2666
2667 /*
2668 * Bias interleave by inode number to
2669 * but this interface is independent
2670 * supplies only that bias, letting c
2671 * by page order, as in shmem_get_pgo
2672 */
2673 *ilx = inode->i_ino;
2674 index = ((addr - vma->vm_start) >> PA
2675 return mpol_shared_policy_lookup(&SHM
2676 }
2677
2678 static struct mempolicy *shmem_get_pgoff_poli
2679 pgoff_t index, unsign
2680 {
2681 struct mempolicy *mpol;
2682
2683 /* Bias interleave by inode number to
2684 *ilx = info->vfs_inode.i_ino + (index
2685
2686 mpol = mpol_shared_policy_lookup(&inf
2687 return mpol ? mpol : get_task_policy(
2688 }
2689 #else
2690 static struct mempolicy *shmem_get_pgoff_poli
2691 pgoff_t index, unsign
2692 {
2693 *ilx = 0;
2694 return NULL;
2695 }
2696 #endif /* CONFIG_NUMA */
2697
2698 int shmem_lock(struct file *file, int lock, s
2699 {
2700 struct inode *inode = file_inode(file
2701 struct shmem_inode_info *info = SHMEM
2702 int retval = -ENOMEM;
2703
2704 /*
2705 * What serializes the accesses to in
2706 * ipc_lock_object() when called from
2707 * no serialization needed when calle
2708 */
2709 if (lock && !(info->flags & VM_LOCKED
2710 if (!user_shm_lock(inode->i_s
2711 goto out_nomem;
2712 info->flags |= VM_LOCKED;
2713 mapping_set_unevictable(file-
2714 }
2715 if (!lock && (info->flags & VM_LOCKED
2716 user_shm_unlock(inode->i_size
2717 info->flags &= ~VM_LOCKED;
2718 mapping_clear_unevictable(fil
2719 }
2720 retval = 0;
2721
2722 out_nomem:
2723 return retval;
2724 }
2725
2726 static int shmem_mmap(struct file *file, stru
2727 {
2728 struct inode *inode = file_inode(file
2729 struct shmem_inode_info *info = SHMEM
2730 int ret;
2731
2732 ret = seal_check_write(info->seals, v
2733 if (ret)
2734 return ret;
2735
2736 file_accessed(file);
2737 /* This is anonymous shared memory if
2738 if (inode->i_nlink)
2739 vma->vm_ops = &shmem_vm_ops;
2740 else
2741 vma->vm_ops = &shmem_anon_vm_
2742 return 0;
2743 }
2744
2745 static int shmem_file_open(struct inode *inod
2746 {
2747 file->f_mode |= FMODE_CAN_ODIRECT;
2748 return generic_file_open(inode, file)
2749 }
2750
2751 #ifdef CONFIG_TMPFS_XATTR
2752 static int shmem_initxattrs(struct inode *, c
2753
2754 /*
2755 * chattr's fsflags are unrelated to extended
2756 * but tmpfs has chosen to enable them under
2757 */
2758 static void shmem_set_inode_flags(struct inod
2759 {
2760 unsigned int i_flags = 0;
2761
2762 if (fsflags & FS_NOATIME_FL)
2763 i_flags |= S_NOATIME;
2764 if (fsflags & FS_APPEND_FL)
2765 i_flags |= S_APPEND;
2766 if (fsflags & FS_IMMUTABLE_FL)
2767 i_flags |= S_IMMUTABLE;
2768 /*
2769 * But FS_NODUMP_FL does not require
2770 */
2771 inode_set_flags(inode, i_flags, S_NOA
2772 }
2773 #else
2774 static void shmem_set_inode_flags(struct inod
2775 {
2776 }
2777 #define shmem_initxattrs NULL
2778 #endif
2779
2780 static struct offset_ctx *shmem_get_offset_ct
2781 {
2782 return &SHMEM_I(inode)->dir_offsets;
2783 }
2784
2785 static struct inode *__shmem_get_inode(struct
2786
2787
2788
2789 {
2790 struct inode *inode;
2791 struct shmem_inode_info *info;
2792 struct shmem_sb_info *sbinfo = SHMEM_
2793 ino_t ino;
2794 int err;
2795
2796 err = shmem_reserve_inode(sb, &ino);
2797 if (err)
2798 return ERR_PTR(err);
2799
2800 inode = new_inode(sb);
2801 if (!inode) {
2802 shmem_free_inode(sb, 0);
2803 return ERR_PTR(-ENOSPC);
2804 }
2805
2806 inode->i_ino = ino;
2807 inode_init_owner(idmap, inode, dir, m
2808 inode->i_blocks = 0;
2809 simple_inode_init_ts(inode);
2810 inode->i_generation = get_random_u32(
2811 info = SHMEM_I(inode);
2812 memset(info, 0, (char *)inode - (char
2813 spin_lock_init(&info->lock);
2814 atomic_set(&info->stop_eviction, 0);
2815 info->seals = F_SEAL_SEAL;
2816 info->flags = flags & VM_NORESERVE;
2817 info->i_crtime = inode_get_mtime(inod
2818 info->fsflags = (dir == NULL) ? 0 :
2819 SHMEM_I(dir)->fsflags & SHMEM
2820 if (info->fsflags)
2821 shmem_set_inode_flags(inode,
2822 INIT_LIST_HEAD(&info->shrinklist);
2823 INIT_LIST_HEAD(&info->swaplist);
2824 simple_xattrs_init(&info->xattrs);
2825 cache_no_acl(inode);
2826 if (sbinfo->noswap)
2827 mapping_set_unevictable(inode
2828 mapping_set_large_folios(inode->i_map
2829
2830 switch (mode & S_IFMT) {
2831 default:
2832 inode->i_op = &shmem_special_
2833 init_special_inode(inode, mod
2834 break;
2835 case S_IFREG:
2836 inode->i_mapping->a_ops = &sh
2837 inode->i_op = &shmem_inode_op
2838 inode->i_fop = &shmem_file_op
2839 mpol_shared_policy_init(&info
2840 shme
2841 break;
2842 case S_IFDIR:
2843 inc_nlink(inode);
2844 /* Some things misbehave if s
2845 inode->i_size = 2 * BOGO_DIRE
2846 inode->i_op = &shmem_dir_inod
2847 inode->i_fop = &simple_offset
2848 simple_offset_init(shmem_get_
2849 break;
2850 case S_IFLNK:
2851 /*
2852 * Must not load anything in
2853 * mpol_free_shared_policy wi
2854 */
2855 mpol_shared_policy_init(&info
2856 break;
2857 }
2858
2859 lockdep_annotate_inode_mutex_key(inod
2860 return inode;
2861 }
2862
2863 #ifdef CONFIG_TMPFS_QUOTA
2864 static struct inode *shmem_get_inode(struct m
2865 struct s
2866 umode_t
2867 {
2868 int err;
2869 struct inode *inode;
2870
2871 inode = __shmem_get_inode(idmap, sb,
2872 if (IS_ERR(inode))
2873 return inode;
2874
2875 err = dquot_initialize(inode);
2876 if (err)
2877 goto errout;
2878
2879 err = dquot_alloc_inode(inode);
2880 if (err) {
2881 dquot_drop(inode);
2882 goto errout;
2883 }
2884 return inode;
2885
2886 errout:
2887 inode->i_flags |= S_NOQUOTA;
2888 iput(inode);
2889 return ERR_PTR(err);
2890 }
2891 #else
2892 static inline struct inode *shmem_get_inode(s
2893 struct s
2894 umode_t
2895 {
2896 return __shmem_get_inode(idmap, sb, d
2897 }
2898 #endif /* CONFIG_TMPFS_QUOTA */
2899
2900 #ifdef CONFIG_USERFAULTFD
2901 int shmem_mfill_atomic_pte(pmd_t *dst_pmd,
2902 struct vm_area_str
2903 unsigned long dst_
2904 unsigned long src_
2905 uffd_flags_t flags
2906 struct folio **fol
2907 {
2908 struct inode *inode = file_inode(dst_
2909 struct shmem_inode_info *info = SHMEM
2910 struct address_space *mapping = inode
2911 gfp_t gfp = mapping_gfp_mask(mapping)
2912 pgoff_t pgoff = linear_page_index(dst
2913 void *page_kaddr;
2914 struct folio *folio;
2915 int ret;
2916 pgoff_t max_off;
2917
2918 if (shmem_inode_acct_blocks(inode, 1)
2919 /*
2920 * We may have got a page, re
2921 * and now we find ourselves
2922 * avoid a BUG_ON in our call
2923 */
2924 if (unlikely(*foliop)) {
2925 folio_put(*foliop);
2926 *foliop = NULL;
2927 }
2928 return -ENOMEM;
2929 }
2930
2931 if (!*foliop) {
2932 ret = -ENOMEM;
2933 folio = shmem_alloc_folio(gfp
2934 if (!folio)
2935 goto out_unacct_block
2936
2937 if (uffd_flags_mode_is(flags,
2938 page_kaddr = kmap_loc
2939 /*
2940 * The read mmap_lock
2941 * mmap_lock being re
2942 * possible if a writ
2943 *
2944 * process A thread 1
2945 * process A thread 2
2946 * process B thread 1
2947 * process B thread 2
2948 * process A thread 1
2949 * process B thread 1
2950 *
2951 * Disable page fault
2952 * and retry the copy
2953 */
2954 pagefault_disable();
2955 ret = copy_from_user(
2956
2957
2958 pagefault_enable();
2959 kunmap_local(page_kad
2960
2961 /* fallback to copy_f
2962 if (unlikely(ret)) {
2963 *foliop = fol
2964 ret = -ENOENT
2965 /* don't free
2966 goto out_unac
2967 }
2968
2969 flush_dcache_folio(fo
2970 } else { /* ZE
2971 clear_user_highpage(&
2972 }
2973 } else {
2974 folio = *foliop;
2975 VM_BUG_ON_FOLIO(folio_test_la
2976 *foliop = NULL;
2977 }
2978
2979 VM_BUG_ON(folio_test_locked(folio));
2980 VM_BUG_ON(folio_test_swapbacked(folio
2981 __folio_set_locked(folio);
2982 __folio_set_swapbacked(folio);
2983 __folio_mark_uptodate(folio);
2984
2985 ret = -EFAULT;
2986 max_off = DIV_ROUND_UP(i_size_read(in
2987 if (unlikely(pgoff >= max_off))
2988 goto out_release;
2989
2990 ret = mem_cgroup_charge(folio, dst_vm
2991 if (ret)
2992 goto out_release;
2993 ret = shmem_add_to_page_cache(folio,
2994 if (ret)
2995 goto out_release;
2996
2997 ret = mfill_atomic_install_pte(dst_pm
2998 &folio
2999 if (ret)
3000 goto out_delete_from_cache;
3001
3002 shmem_recalc_inode(inode, 1, 0);
3003 folio_unlock(folio);
3004 return 0;
3005 out_delete_from_cache:
3006 filemap_remove_folio(folio);
3007 out_release:
3008 folio_unlock(folio);
3009 folio_put(folio);
3010 out_unacct_blocks:
3011 shmem_inode_unacct_blocks(inode, 1);
3012 return ret;
3013 }
3014 #endif /* CONFIG_USERFAULTFD */
3015
3016 #ifdef CONFIG_TMPFS
3017 static const struct inode_operations shmem_sy
3018 static const struct inode_operations shmem_sh
3019
3020 static int
3021 shmem_write_begin(struct file *file, struct a
3022 loff_t pos, unsigned
3023 struct folio **foliop
3024 {
3025 struct inode *inode = mapping->host;
3026 struct shmem_inode_info *info = SHMEM
3027 pgoff_t index = pos >> PAGE_SHIFT;
3028 struct folio *folio;
3029 int ret = 0;
3030
3031 /* i_rwsem is held by caller */
3032 if (unlikely(info->seals & (F_SEAL_GR
3033 F_SEAL_WRI
3034 if (info->seals & (F_SEAL_WRI
3035 return -EPERM;
3036 if ((info->seals & F_SEAL_GRO
3037 return -EPERM;
3038 }
3039
3040 ret = shmem_get_folio(inode, index, p
3041 if (ret)
3042 return ret;
3043
3044 if (folio_test_hwpoison(folio) ||
3045 (folio_test_large(folio) && folio
3046 folio_unlock(folio);
3047 folio_put(folio);
3048 return -EIO;
3049 }
3050
3051 *foliop = folio;
3052 return 0;
3053 }
3054
3055 static int
3056 shmem_write_end(struct file *file, struct add
3057 loff_t pos, unsigned
3058 struct folio *folio,
3059 {
3060 struct inode *inode = mapping->host;
3061
3062 if (pos + copied > inode->i_size)
3063 i_size_write(inode, pos + cop
3064
3065 if (!folio_test_uptodate(folio)) {
3066 if (copied < folio_size(folio
3067 size_t from = offset_
3068 folio_zero_segments(f
3069 from
3070 }
3071 folio_mark_uptodate(folio);
3072 }
3073 folio_mark_dirty(folio);
3074 folio_unlock(folio);
3075 folio_put(folio);
3076
3077 return copied;
3078 }
3079
3080 static ssize_t shmem_file_read_iter(struct ki
3081 {
3082 struct file *file = iocb->ki_filp;
3083 struct inode *inode = file_inode(file
3084 struct address_space *mapping = inode
3085 pgoff_t index;
3086 unsigned long offset;
3087 int error = 0;
3088 ssize_t retval = 0;
3089 loff_t *ppos = &iocb->ki_pos;
3090
3091 index = *ppos >> PAGE_SHIFT;
3092 offset = *ppos & ~PAGE_MASK;
3093
3094 for (;;) {
3095 struct folio *folio = NULL;
3096 struct page *page = NULL;
3097 pgoff_t end_index;
3098 unsigned long nr, ret;
3099 loff_t i_size = i_size_read(i
3100
3101 end_index = i_size >> PAGE_SH
3102 if (index > end_index)
3103 break;
3104 if (index == end_index) {
3105 nr = i_size & ~PAGE_M
3106 if (nr <= offset)
3107 break;
3108 }
3109
3110 error = shmem_get_folio(inode
3111 if (error) {
3112 if (error == -EINVAL)
3113 error = 0;
3114 break;
3115 }
3116 if (folio) {
3117 folio_unlock(folio);
3118
3119 page = folio_file_pag
3120 if (PageHWPoison(page
3121 folio_put(fol
3122 error = -EIO;
3123 break;
3124 }
3125 }
3126
3127 /*
3128 * We must evaluate after, si
3129 * are called without i_rwsem
3130 */
3131 nr = PAGE_SIZE;
3132 i_size = i_size_read(inode);
3133 end_index = i_size >> PAGE_SH
3134 if (index == end_index) {
3135 nr = i_size & ~PAGE_M
3136 if (nr <= offset) {
3137 if (folio)
3138 folio
3139 break;
3140 }
3141 }
3142 nr -= offset;
3143
3144 if (folio) {
3145 /*
3146 * If users can be wr
3147 * virtual addresses,
3148 * before reading the
3149 */
3150 if (mapping_writably_
3151 flush_dcache_
3152 /*
3153 * Mark the page acce
3154 */
3155 if (!offset)
3156 folio_mark_ac
3157 /*
3158 * Ok, we have the pa
3159 * now we can copy it
3160 */
3161 ret = copy_page_to_it
3162 folio_put(folio);
3163
3164 } else if (user_backed_iter(t
3165 /*
3166 * Copy to user tends
3167 * clear_user() not s
3168 * faster to copy the
3169 */
3170 ret = copy_page_to_it
3171 } else {
3172 /*
3173 * But submitting the
3174 * splice() - or othe
3175 * so don't attempt t
3176 */
3177 ret = iov_iter_zero(n
3178 }
3179
3180 retval += ret;
3181 offset += ret;
3182 index += offset >> PAGE_SHIFT
3183 offset &= ~PAGE_MASK;
3184
3185 if (!iov_iter_count(to))
3186 break;
3187 if (ret < nr) {
3188 error = -EFAULT;
3189 break;
3190 }
3191 cond_resched();
3192 }
3193
3194 *ppos = ((loff_t) index << PAGE_SHIFT
3195 file_accessed(file);
3196 return retval ? retval : error;
3197 }
3198
3199 static ssize_t shmem_file_write_iter(struct k
3200 {
3201 struct file *file = iocb->ki_filp;
3202 struct inode *inode = file->f_mapping
3203 ssize_t ret;
3204
3205 inode_lock(inode);
3206 ret = generic_write_checks(iocb, from
3207 if (ret <= 0)
3208 goto unlock;
3209 ret = file_remove_privs(file);
3210 if (ret)
3211 goto unlock;
3212 ret = file_update_time(file);
3213 if (ret)
3214 goto unlock;
3215 ret = generic_perform_write(iocb, fro
3216 unlock:
3217 inode_unlock(inode);
3218 return ret;
3219 }
3220
3221 static bool zero_pipe_buf_get(struct pipe_ino
3222 struct pipe_buf
3223 {
3224 return true;
3225 }
3226
3227 static void zero_pipe_buf_release(struct pipe
3228 struct pipe
3229 {
3230 }
3231
3232 static bool zero_pipe_buf_try_steal(struct pi
3233 struct pi
3234 {
3235 return false;
3236 }
3237
3238 static const struct pipe_buf_operations zero_
3239 .release = zero_pipe_buf_relea
3240 .try_steal = zero_pipe_buf_try_s
3241 .get = zero_pipe_buf_get,
3242 };
3243
3244 static size_t splice_zeropage_into_pipe(struc
3245 loff_
3246 {
3247 size_t offset = fpos & ~PAGE_MASK;
3248
3249 size = min_t(size_t, size, PAGE_SIZE
3250
3251 if (!pipe_full(pipe->head, pipe->tail
3252 struct pipe_buffer *buf = pip
3253
3254 *buf = (struct pipe_buffer) {
3255 .ops = &zero_pipe_
3256 .page = ZERO_PAGE(0
3257 .offset = offset,
3258 .len = size,
3259 };
3260 pipe->head++;
3261 }
3262
3263 return size;
3264 }
3265
3266 static ssize_t shmem_file_splice_read(struct
3267 struct
3268 size_t
3269 {
3270 struct inode *inode = file_inode(in);
3271 struct address_space *mapping = inode
3272 struct folio *folio = NULL;
3273 size_t total_spliced = 0, used, npage
3274 loff_t isize;
3275 int error = 0;
3276
3277 /* Work out how much data we can actu
3278 used = pipe_occupancy(pipe->head, pip
3279 npages = max_t(ssize_t, pipe->max_usa
3280 len = min_t(size_t, len, npages * PAG
3281
3282 do {
3283 if (*ppos >= i_size_read(inod
3284 break;
3285
3286 error = shmem_get_folio(inode
3287 SGP_R
3288 if (error) {
3289 if (error == -EINVAL)
3290 error = 0;
3291 break;
3292 }
3293 if (folio) {
3294 folio_unlock(folio);
3295
3296 if (folio_test_hwpois
3297 (folio_test_large
3298 folio_test_has_h
3299 error = -EIO;
3300 break;
3301 }
3302 }
3303
3304 /*
3305 * i_size must be checked aft
3306 *
3307 * Checking i_size after the
3308 * the correct value for "nr"
3309 * part of the page is not co
3310 * another truncate extends t
3311 */
3312 isize = i_size_read(inode);
3313 if (unlikely(*ppos >= isize))
3314 break;
3315 part = min_t(loff_t, isize -
3316
3317 if (folio) {
3318 /*
3319 * If users can be wr
3320 * virtual addresses,
3321 * before reading the
3322 */
3323 if (mapping_writably_
3324 flush_dcache_
3325 folio_mark_accessed(f
3326 /*
3327 * Ok, we have the pa
3328 * now splice it into
3329 */
3330 n = splice_folio_into
3331 folio_put(folio);
3332 folio = NULL;
3333 } else {
3334 n = splice_zeropage_i
3335 }
3336
3337 if (!n)
3338 break;
3339 len -= n;
3340 total_spliced += n;
3341 *ppos += n;
3342 in->f_ra.prev_pos = *ppos;
3343 if (pipe_full(pipe->head, pip
3344 break;
3345
3346 cond_resched();
3347 } while (len);
3348
3349 if (folio)
3350 folio_put(folio);
3351
3352 file_accessed(in);
3353 return total_spliced ? total_spliced
3354 }
3355
3356 static loff_t shmem_file_llseek(struct file *
3357 {
3358 struct address_space *mapping = file-
3359 struct inode *inode = mapping->host;
3360
3361 if (whence != SEEK_DATA && whence !=
3362 return generic_file_llseek_si
3363 MAX_L
3364 if (offset < 0)
3365 return -ENXIO;
3366
3367 inode_lock(inode);
3368 /* We're holding i_rwsem so we can ac
3369 offset = mapping_seek_hole_data(mappi
3370 if (offset >= 0)
3371 offset = vfs_setpos(file, off
3372 inode_unlock(inode);
3373 return offset;
3374 }
3375
3376 static long shmem_fallocate(struct file *file
3377
3378 {
3379 struct inode *inode = file_inode(file
3380 struct shmem_sb_info *sbinfo = SHMEM_
3381 struct shmem_inode_info *info = SHMEM
3382 struct shmem_falloc shmem_falloc;
3383 pgoff_t start, index, end, undo_fallo
3384 int error;
3385
3386 if (mode & ~(FALLOC_FL_KEEP_SIZE | FA
3387 return -EOPNOTSUPP;
3388
3389 inode_lock(inode);
3390
3391 if (mode & FALLOC_FL_PUNCH_HOLE) {
3392 struct address_space *mapping
3393 loff_t unmap_start = round_up
3394 loff_t unmap_end = round_down
3395 DECLARE_WAIT_QUEUE_HEAD_ONSTA
3396
3397 /* protected by i_rwsem */
3398 if (info->seals & (F_SEAL_WRI
3399 error = -EPERM;
3400 goto out;
3401 }
3402
3403 shmem_falloc.waitq = &shmem_f
3404 shmem_falloc.start = (u64)unm
3405 shmem_falloc.next = (unmap_en
3406 spin_lock(&inode->i_lock);
3407 inode->i_private = &shmem_fal
3408 spin_unlock(&inode->i_lock);
3409
3410 if ((u64)unmap_end > (u64)unm
3411 unmap_mapping_range(m
3412 1
3413 shmem_truncate_range(inode, o
3414 /* No need to unmap again: ho
3415
3416 spin_lock(&inode->i_lock);
3417 inode->i_private = NULL;
3418 wake_up_all(&shmem_falloc_wai
3419 WARN_ON_ONCE(!list_empty(&shm
3420 spin_unlock(&inode->i_lock);
3421 error = 0;
3422 goto out;
3423 }
3424
3425 /* We need to check rlimit even when
3426 error = inode_newsize_ok(inode, offse
3427 if (error)
3428 goto out;
3429
3430 if ((info->seals & F_SEAL_GROW) && of
3431 error = -EPERM;
3432 goto out;
3433 }
3434
3435 start = offset >> PAGE_SHIFT;
3436 end = (offset + len + PAGE_SIZE - 1)
3437 /* Try to avoid a swapstorm if len is
3438 if (sbinfo->max_blocks && end - start
3439 error = -ENOSPC;
3440 goto out;
3441 }
3442
3443 shmem_falloc.waitq = NULL;
3444 shmem_falloc.start = start;
3445 shmem_falloc.next = start;
3446 shmem_falloc.nr_falloced = 0;
3447 shmem_falloc.nr_unswapped = 0;
3448 spin_lock(&inode->i_lock);
3449 inode->i_private = &shmem_falloc;
3450 spin_unlock(&inode->i_lock);
3451
3452 /*
3453 * info->fallocend is only relevant w
3454 * involved: to prevent split_huge_pa
3455 * pages when FALLOC_FL_KEEP_SIZE com
3456 */
3457 undo_fallocend = info->fallocend;
3458 if (info->fallocend < end)
3459 info->fallocend = end;
3460
3461 for (index = start; index < end; ) {
3462 struct folio *folio;
3463
3464 /*
3465 * Check for fatal signal so
3466 * situations. We don't want
3467 * signals as large fallocate
3468 * e.g. periodic timers may r
3469 * restarting.
3470 */
3471 if (fatal_signal_pending(curr
3472 error = -EINTR;
3473 else if (shmem_falloc.nr_unsw
3474 error = -ENOMEM;
3475 else
3476 error = shmem_get_fol
3477
3478 if (error) {
3479 info->fallocend = und
3480 /* Remove the !uptoda
3481 if (index > start) {
3482 shmem_undo_ra
3483 (loff_t)s
3484 ((loff_t)
3485 }
3486 goto undone;
3487 }
3488
3489 /*
3490 * Here is a more important o
3491 * a second SGP_FALLOC on the
3492 * making it uptodate and un-
3493 */
3494 index = folio_next_index(foli
3495 /* Beware 32-bit wraparound *
3496 if (!index)
3497 index--;
3498
3499 /*
3500 * Inform shmem_writepage() h
3501 * No need for lock or barrie
3502 */
3503 if (!folio_test_uptodate(foli
3504 shmem_falloc.nr_fallo
3505 shmem_falloc.next = index;
3506
3507 /*
3508 * If !uptodate, leave it tha
3509 * can be recognized if we ne
3510 * But mark it dirty so that
3511 * than free the folios we ar
3512 * might still be clean: we n
3513 */
3514 folio_mark_dirty(folio);
3515 folio_unlock(folio);
3516 folio_put(folio);
3517 cond_resched();
3518 }
3519
3520 if (!(mode & FALLOC_FL_KEEP_SIZE) &&
3521 i_size_write(inode, offset +
3522 undone:
3523 spin_lock(&inode->i_lock);
3524 inode->i_private = NULL;
3525 spin_unlock(&inode->i_lock);
3526 out:
3527 if (!error)
3528 file_modified(file);
3529 inode_unlock(inode);
3530 return error;
3531 }
3532
3533 static int shmem_statfs(struct dentry *dentry
3534 {
3535 struct shmem_sb_info *sbinfo = SHMEM_
3536
3537 buf->f_type = TMPFS_MAGIC;
3538 buf->f_bsize = PAGE_SIZE;
3539 buf->f_namelen = NAME_MAX;
3540 if (sbinfo->max_blocks) {
3541 buf->f_blocks = sbinfo->max_b
3542 buf->f_bavail =
3543 buf->f_bfree = sbinfo->max_b
3544 percpu_counte
3545 }
3546 if (sbinfo->max_inodes) {
3547 buf->f_files = sbinfo->max_in
3548 buf->f_ffree = sbinfo->free_i
3549 }
3550 /* else leave those fields 0 like sim
3551
3552 buf->f_fsid = uuid_to_fsid(dentry->d_
3553
3554 return 0;
3555 }
3556
3557 /*
3558 * File creation. Allocate an inode, and we'r
3559 */
3560 static int
3561 shmem_mknod(struct mnt_idmap *idmap, struct i
3562 struct dentry *dentry, umode_t mo
3563 {
3564 struct inode *inode;
3565 int error;
3566
3567 inode = shmem_get_inode(idmap, dir->i
3568 if (IS_ERR(inode))
3569 return PTR_ERR(inode);
3570
3571 error = simple_acl_create(dir, inode)
3572 if (error)
3573 goto out_iput;
3574 error = security_inode_init_security(
3575
3576 if (error && error != -EOPNOTSUPP)
3577 goto out_iput;
3578
3579 error = simple_offset_add(shmem_get_o
3580 if (error)
3581 goto out_iput;
3582
3583 dir->i_size += BOGO_DIRENT_SIZE;
3584 inode_set_mtime_to_ts(dir, inode_set_
3585 inode_inc_iversion(dir);
3586 d_instantiate(dentry, inode);
3587 dget(dentry); /* Extra count - pin th
3588 return error;
3589
3590 out_iput:
3591 iput(inode);
3592 return error;
3593 }
3594
3595 static int
3596 shmem_tmpfile(struct mnt_idmap *idmap, struct
3597 struct file *file, umode_t mode
3598 {
3599 struct inode *inode;
3600 int error;
3601
3602 inode = shmem_get_inode(idmap, dir->i
3603 if (IS_ERR(inode)) {
3604 error = PTR_ERR(inode);
3605 goto err_out;
3606 }
3607 error = security_inode_init_security(
3608
3609 if (error && error != -EOPNOTSUPP)
3610 goto out_iput;
3611 error = simple_acl_create(dir, inode)
3612 if (error)
3613 goto out_iput;
3614 d_tmpfile(file, inode);
3615
3616 err_out:
3617 return finish_open_simple(file, error
3618 out_iput:
3619 iput(inode);
3620 return error;
3621 }
3622
3623 static int shmem_mkdir(struct mnt_idmap *idma
3624 struct dentry *dentry,
3625 {
3626 int error;
3627
3628 error = shmem_mknod(idmap, dir, dentr
3629 if (error)
3630 return error;
3631 inc_nlink(dir);
3632 return 0;
3633 }
3634
3635 static int shmem_create(struct mnt_idmap *idm
3636 struct dentry *dentry
3637 {
3638 return shmem_mknod(idmap, dir, dentry
3639 }
3640
3641 /*
3642 * Link a file..
3643 */
3644 static int shmem_link(struct dentry *old_dent
3645 struct dentry *dentry)
3646 {
3647 struct inode *inode = d_inode(old_den
3648 int ret = 0;
3649
3650 /*
3651 * No ordinary (disk based) filesyste
3652 * but each new link needs a new dent
3653 * tmpfs dentries cannot be pruned un
3654 * But if an O_TMPFILE file is linked
3655 * first link must skip that, to get
3656 */
3657 if (inode->i_nlink) {
3658 ret = shmem_reserve_inode(ino
3659 if (ret)
3660 goto out;
3661 }
3662
3663 ret = simple_offset_add(shmem_get_off
3664 if (ret) {
3665 if (inode->i_nlink)
3666 shmem_free_inode(inod
3667 goto out;
3668 }
3669
3670 dir->i_size += BOGO_DIRENT_SIZE;
3671 inode_set_mtime_to_ts(dir,
3672 inode_set_ctime
3673 inode_inc_iversion(dir);
3674 inc_nlink(inode);
3675 ihold(inode); /* New dentry referen
3676 dget(dentry); /* Extra pinning coun
3677 d_instantiate(dentry, inode);
3678 out:
3679 return ret;
3680 }
3681
3682 static int shmem_unlink(struct inode *dir, st
3683 {
3684 struct inode *inode = d_inode(dentry)
3685
3686 if (inode->i_nlink > 1 && !S_ISDIR(in
3687 shmem_free_inode(inode->i_sb,
3688
3689 simple_offset_remove(shmem_get_offset
3690
3691 dir->i_size -= BOGO_DIRENT_SIZE;
3692 inode_set_mtime_to_ts(dir,
3693 inode_set_ctime
3694 inode_inc_iversion(dir);
3695 drop_nlink(inode);
3696 dput(dentry); /* Undo the count fro
3697 return 0;
3698 }
3699
3700 static int shmem_rmdir(struct inode *dir, str
3701 {
3702 if (!simple_offset_empty(dentry))
3703 return -ENOTEMPTY;
3704
3705 drop_nlink(d_inode(dentry));
3706 drop_nlink(dir);
3707 return shmem_unlink(dir, dentry);
3708 }
3709
3710 static int shmem_whiteout(struct mnt_idmap *i
3711 struct inode *old_d
3712 {
3713 struct dentry *whiteout;
3714 int error;
3715
3716 whiteout = d_alloc(old_dentry->d_pare
3717 if (!whiteout)
3718 return -ENOMEM;
3719
3720 error = shmem_mknod(idmap, old_dir, w
3721 S_IFCHR | WHITEOU
3722 dput(whiteout);
3723 if (error)
3724 return error;
3725
3726 /*
3727 * Cheat and hash the whiteout while
3728 * place, instead of playing games wi
3729 *
3730 * d_lookup() will consistently find
3731 * not sure which one, but that isn't
3732 */
3733 d_rehash(whiteout);
3734 return 0;
3735 }
3736
3737 /*
3738 * The VFS layer already does all the dentry
3739 * we just have to decrement the usage count
3740 * it exists so that the VFS layer correctly
3741 * gets overwritten.
3742 */
3743 static int shmem_rename2(struct mnt_idmap *id
3744 struct inode *old_di
3745 struct inode *new_di
3746 unsigned int flags)
3747 {
3748 struct inode *inode = d_inode(old_den
3749 int they_are_dirs = S_ISDIR(inode->i_
3750 int error;
3751
3752 if (flags & ~(RENAME_NOREPLACE | RENA
3753 return -EINVAL;
3754
3755 if (flags & RENAME_EXCHANGE)
3756 return simple_offset_rename_e
3757
3758
3759 if (!simple_offset_empty(new_dentry))
3760 return -ENOTEMPTY;
3761
3762 if (flags & RENAME_WHITEOUT) {
3763 error = shmem_whiteout(idmap,
3764 if (error)
3765 return error;
3766 }
3767
3768 error = simple_offset_rename(old_dir,
3769 if (error)
3770 return error;
3771
3772 if (d_really_is_positive(new_dentry))
3773 (void) shmem_unlink(new_dir,
3774 if (they_are_dirs) {
3775 drop_nlink(d_inode(ne
3776 drop_nlink(old_dir);
3777 }
3778 } else if (they_are_dirs) {
3779 drop_nlink(old_dir);
3780 inc_nlink(new_dir);
3781 }
3782
3783 old_dir->i_size -= BOGO_DIRENT_SIZE;
3784 new_dir->i_size += BOGO_DIRENT_SIZE;
3785 simple_rename_timestamp(old_dir, old_
3786 inode_inc_iversion(old_dir);
3787 inode_inc_iversion(new_dir);
3788 return 0;
3789 }
3790
3791 static int shmem_symlink(struct mnt_idmap *id
3792 struct dentry *dentr
3793 {
3794 int error;
3795 int len;
3796 struct inode *inode;
3797 struct folio *folio;
3798
3799 len = strlen(symname) + 1;
3800 if (len > PAGE_SIZE)
3801 return -ENAMETOOLONG;
3802
3803 inode = shmem_get_inode(idmap, dir->i
3804 VM_NORESERVE)
3805 if (IS_ERR(inode))
3806 return PTR_ERR(inode);
3807
3808 error = security_inode_init_security(
3809
3810 if (error && error != -EOPNOTSUPP)
3811 goto out_iput;
3812
3813 error = simple_offset_add(shmem_get_o
3814 if (error)
3815 goto out_iput;
3816
3817 inode->i_size = len-1;
3818 if (len <= SHORT_SYMLINK_LEN) {
3819 inode->i_link = kmemdup(symna
3820 if (!inode->i_link) {
3821 error = -ENOMEM;
3822 goto out_remove_offse
3823 }
3824 inode->i_op = &shmem_short_sy
3825 } else {
3826 inode_nohighmem(inode);
3827 inode->i_mapping->a_ops = &sh
3828 error = shmem_get_folio(inode
3829 if (error)
3830 goto out_remove_offse
3831 inode->i_op = &shmem_symlink_
3832 memcpy(folio_address(folio),
3833 folio_mark_uptodate(folio);
3834 folio_mark_dirty(folio);
3835 folio_unlock(folio);
3836 folio_put(folio);
3837 }
3838 dir->i_size += BOGO_DIRENT_SIZE;
3839 inode_set_mtime_to_ts(dir, inode_set_
3840 inode_inc_iversion(dir);
3841 d_instantiate(dentry, inode);
3842 dget(dentry);
3843 return 0;
3844
3845 out_remove_offset:
3846 simple_offset_remove(shmem_get_offset
3847 out_iput:
3848 iput(inode);
3849 return error;
3850 }
3851
3852 static void shmem_put_link(void *arg)
3853 {
3854 folio_mark_accessed(arg);
3855 folio_put(arg);
3856 }
3857
3858 static const char *shmem_get_link(struct dent
3859 struct dela
3860 {
3861 struct folio *folio = NULL;
3862 int error;
3863
3864 if (!dentry) {
3865 folio = filemap_get_folio(ino
3866 if (IS_ERR(folio))
3867 return ERR_PTR(-ECHIL
3868 if (PageHWPoison(folio_page(f
3869 !folio_test_uptodate(foli
3870 folio_put(folio);
3871 return ERR_PTR(-ECHIL
3872 }
3873 } else {
3874 error = shmem_get_folio(inode
3875 if (error)
3876 return ERR_PTR(error)
3877 if (!folio)
3878 return ERR_PTR(-ECHIL
3879 if (PageHWPoison(folio_page(f
3880 folio_unlock(folio);
3881 folio_put(folio);
3882 return ERR_PTR(-ECHIL
3883 }
3884 folio_unlock(folio);
3885 }
3886 set_delayed_call(done, shmem_put_link
3887 return folio_address(folio);
3888 }
3889
3890 #ifdef CONFIG_TMPFS_XATTR
3891
3892 static int shmem_fileattr_get(struct dentry *
3893 {
3894 struct shmem_inode_info *info = SHMEM
3895
3896 fileattr_fill_flags(fa, info->fsflags
3897
3898 return 0;
3899 }
3900
3901 static int shmem_fileattr_set(struct mnt_idma
3902 struct dentry *
3903 {
3904 struct inode *inode = d_inode(dentry)
3905 struct shmem_inode_info *info = SHMEM
3906
3907 if (fileattr_has_fsx(fa))
3908 return -EOPNOTSUPP;
3909 if (fa->flags & ~SHMEM_FL_USER_MODIFI
3910 return -EOPNOTSUPP;
3911
3912 info->fsflags = (info->fsflags & ~SHM
3913 (fa->flags & SHMEM_FL_USER_MO
3914
3915 shmem_set_inode_flags(inode, info->fs
3916 inode_set_ctime_current(inode);
3917 inode_inc_iversion(inode);
3918 return 0;
3919 }
3920
3921 /*
3922 * Superblocks without xattr inode operations
3923 * support from the LSM "for free". As soon a
3924 * like ACLs, we also need to implement the s
3925 * filesystem level, though.
3926 */
3927
3928 /*
3929 * Callback for security_inode_init_security(
3930 */
3931 static int shmem_initxattrs(struct inode *ino
3932 const struct xatt
3933 {
3934 struct shmem_inode_info *info = SHMEM
3935 struct shmem_sb_info *sbinfo = SHMEM_
3936 const struct xattr *xattr;
3937 struct simple_xattr *new_xattr;
3938 size_t ispace = 0;
3939 size_t len;
3940
3941 if (sbinfo->max_inodes) {
3942 for (xattr = xattr_array; xat
3943 ispace += simple_xatt
3944 xattr->value_
3945 }
3946 if (ispace) {
3947 raw_spin_lock(&sbinfo
3948 if (sbinfo->free_ispa
3949 ispace = 0;
3950 else
3951 sbinfo->free_
3952 raw_spin_unlock(&sbin
3953 if (!ispace)
3954 return -ENOSP
3955 }
3956 }
3957
3958 for (xattr = xattr_array; xattr->name
3959 new_xattr = simple_xattr_allo
3960 if (!new_xattr)
3961 break;
3962
3963 len = strlen(xattr->name) + 1
3964 new_xattr->name = kmalloc(XAT
3965 GFP
3966 if (!new_xattr->name) {
3967 kvfree(new_xattr);
3968 break;
3969 }
3970
3971 memcpy(new_xattr->name, XATTR
3972 XATTR_SECURITY_PREFIX_
3973 memcpy(new_xattr->name + XATT
3974 xattr->name, len);
3975
3976 simple_xattr_add(&info->xattr
3977 }
3978
3979 if (xattr->name != NULL) {
3980 if (ispace) {
3981 raw_spin_lock(&sbinfo
3982 sbinfo->free_ispace +
3983 raw_spin_unlock(&sbin
3984 }
3985 simple_xattrs_free(&info->xat
3986 return -ENOMEM;
3987 }
3988
3989 return 0;
3990 }
3991
3992 static int shmem_xattr_handler_get(const stru
3993 struct den
3994 const char
3995 {
3996 struct shmem_inode_info *info = SHMEM
3997
3998 name = xattr_full_name(handler, name)
3999 return simple_xattr_get(&info->xattrs
4000 }
4001
4002 static int shmem_xattr_handler_set(const stru
4003 struct mnt
4004 struct den
4005 const char
4006 size_t siz
4007 {
4008 struct shmem_inode_info *info = SHMEM
4009 struct shmem_sb_info *sbinfo = SHMEM_
4010 struct simple_xattr *old_xattr;
4011 size_t ispace = 0;
4012
4013 name = xattr_full_name(handler, name)
4014 if (value && sbinfo->max_inodes) {
4015 ispace = simple_xattr_space(n
4016 raw_spin_lock(&sbinfo->stat_l
4017 if (sbinfo->free_ispace < isp
4018 ispace = 0;
4019 else
4020 sbinfo->free_ispace -
4021 raw_spin_unlock(&sbinfo->stat
4022 if (!ispace)
4023 return -ENOSPC;
4024 }
4025
4026 old_xattr = simple_xattr_set(&info->x
4027 if (!IS_ERR(old_xattr)) {
4028 ispace = 0;
4029 if (old_xattr && sbinfo->max_
4030 ispace = simple_xattr
4031
4032 simple_xattr_free(old_xattr);
4033 old_xattr = NULL;
4034 inode_set_ctime_current(inode
4035 inode_inc_iversion(inode);
4036 }
4037 if (ispace) {
4038 raw_spin_lock(&sbinfo->stat_l
4039 sbinfo->free_ispace += ispace
4040 raw_spin_unlock(&sbinfo->stat
4041 }
4042 return PTR_ERR(old_xattr);
4043 }
4044
4045 static const struct xattr_handler shmem_secur
4046 .prefix = XATTR_SECURITY_PREFIX,
4047 .get = shmem_xattr_handler_get,
4048 .set = shmem_xattr_handler_set,
4049 };
4050
4051 static const struct xattr_handler shmem_trust
4052 .prefix = XATTR_TRUSTED_PREFIX,
4053 .get = shmem_xattr_handler_get,
4054 .set = shmem_xattr_handler_set,
4055 };
4056
4057 static const struct xattr_handler shmem_user_
4058 .prefix = XATTR_USER_PREFIX,
4059 .get = shmem_xattr_handler_get,
4060 .set = shmem_xattr_handler_set,
4061 };
4062
4063 static const struct xattr_handler * const shm
4064 &shmem_security_xattr_handler,
4065 &shmem_trusted_xattr_handler,
4066 &shmem_user_xattr_handler,
4067 NULL
4068 };
4069
4070 static ssize_t shmem_listxattr(struct dentry
4071 {
4072 struct shmem_inode_info *info = SHMEM
4073 return simple_xattr_list(d_inode(dent
4074 }
4075 #endif /* CONFIG_TMPFS_XATTR */
4076
4077 static const struct inode_operations shmem_sh
4078 .getattr = shmem_getattr,
4079 .setattr = shmem_setattr,
4080 .get_link = simple_get_link,
4081 #ifdef CONFIG_TMPFS_XATTR
4082 .listxattr = shmem_listxattr,
4083 #endif
4084 };
4085
4086 static const struct inode_operations shmem_sy
4087 .getattr = shmem_getattr,
4088 .setattr = shmem_setattr,
4089 .get_link = shmem_get_link,
4090 #ifdef CONFIG_TMPFS_XATTR
4091 .listxattr = shmem_listxattr,
4092 #endif
4093 };
4094
4095 static struct dentry *shmem_get_parent(struct
4096 {
4097 return ERR_PTR(-ESTALE);
4098 }
4099
4100 static int shmem_match(struct inode *ino, voi
4101 {
4102 __u32 *fh = vfh;
4103 __u64 inum = fh[2];
4104 inum = (inum << 32) | fh[1];
4105 return ino->i_ino == inum && fh[0] ==
4106 }
4107
4108 /* Find any alias of inode, but prefer a hash
4109 static struct dentry *shmem_find_alias(struct
4110 {
4111 struct dentry *alias = d_find_alias(i
4112
4113 return alias ?: d_find_any_alias(inod
4114 }
4115
4116 static struct dentry *shmem_fh_to_dentry(stru
4117 struct fid *fid, int fh_len,
4118 {
4119 struct inode *inode;
4120 struct dentry *dentry = NULL;
4121 u64 inum;
4122
4123 if (fh_len < 3)
4124 return NULL;
4125
4126 inum = fid->raw[2];
4127 inum = (inum << 32) | fid->raw[1];
4128
4129 inode = ilookup5(sb, (unsigned long)(
4130 shmem_match, fid->raw
4131 if (inode) {
4132 dentry = shmem_find_alias(ino
4133 iput(inode);
4134 }
4135
4136 return dentry;
4137 }
4138
4139 static int shmem_encode_fh(struct inode *inod
4140 struct inode
4141 {
4142 if (*len < 3) {
4143 *len = 3;
4144 return FILEID_INVALID;
4145 }
4146
4147 if (inode_unhashed(inode)) {
4148 /* Unfortunately insert_inode
4149 * so as we hash inodes here
4150 * time, we need a lock to en
4151 * to do it once
4152 */
4153 static DEFINE_SPINLOCK(lock);
4154 spin_lock(&lock);
4155 if (inode_unhashed(inode))
4156 __insert_inode_hash(i
4157 i
4158 spin_unlock(&lock);
4159 }
4160
4161 fh[0] = inode->i_generation;
4162 fh[1] = inode->i_ino;
4163 fh[2] = ((__u64)inode->i_ino) >> 32;
4164
4165 *len = 3;
4166 return 1;
4167 }
4168
4169 static const struct export_operations shmem_e
4170 .get_parent = shmem_get_parent,
4171 .encode_fh = shmem_encode_fh,
4172 .fh_to_dentry = shmem_fh_to_dentry,
4173 };
4174
4175 enum shmem_param {
4176 Opt_gid,
4177 Opt_huge,
4178 Opt_mode,
4179 Opt_mpol,
4180 Opt_nr_blocks,
4181 Opt_nr_inodes,
4182 Opt_size,
4183 Opt_uid,
4184 Opt_inode32,
4185 Opt_inode64,
4186 Opt_noswap,
4187 Opt_quota,
4188 Opt_usrquota,
4189 Opt_grpquota,
4190 Opt_usrquota_block_hardlimit,
4191 Opt_usrquota_inode_hardlimit,
4192 Opt_grpquota_block_hardlimit,
4193 Opt_grpquota_inode_hardlimit,
4194 };
4195
4196 static const struct constant_table shmem_para
4197 {"never", SHMEM_HUGE_NEVER },
4198 {"always", SHMEM_HUGE_ALWAYS },
4199 {"within_size", SHMEM_HUGE_WITHIN_SIZ
4200 {"advise", SHMEM_HUGE_ADVISE },
4201 {}
4202 };
4203
4204 const struct fs_parameter_spec shmem_fs_param
4205 fsparam_gid ("gid", Opt_g
4206 fsparam_enum ("huge", Opt_h
4207 fsparam_u32oct("mode", Opt_m
4208 fsparam_string("mpol", Opt_m
4209 fsparam_string("nr_blocks", Opt_n
4210 fsparam_string("nr_inodes", Opt_n
4211 fsparam_string("size", Opt_s
4212 fsparam_uid ("uid", Opt_u
4213 fsparam_flag ("inode32", Opt_i
4214 fsparam_flag ("inode64", Opt_i
4215 fsparam_flag ("noswap", Opt_n
4216 #ifdef CONFIG_TMPFS_QUOTA
4217 fsparam_flag ("quota", Opt_q
4218 fsparam_flag ("usrquota", Opt_u
4219 fsparam_flag ("grpquota", Opt_g
4220 fsparam_string("usrquota_block_hardli
4221 fsparam_string("usrquota_inode_hardli
4222 fsparam_string("grpquota_block_hardli
4223 fsparam_string("grpquota_inode_hardli
4224 #endif
4225 {}
4226 };
4227
4228 static int shmem_parse_one(struct fs_context
4229 {
4230 struct shmem_options *ctx = fc->fs_pr
4231 struct fs_parse_result result;
4232 unsigned long long size;
4233 char *rest;
4234 int opt;
4235 kuid_t kuid;
4236 kgid_t kgid;
4237
4238 opt = fs_parse(fc, shmem_fs_parameter
4239 if (opt < 0)
4240 return opt;
4241
4242 switch (opt) {
4243 case Opt_size:
4244 size = memparse(param->string
4245 if (*rest == '%') {
4246 size <<= PAGE_SHIFT;
4247 size *= totalram_page
4248 do_div(size, 100);
4249 rest++;
4250 }
4251 if (*rest)
4252 goto bad_value;
4253 ctx->blocks = DIV_ROUND_UP(si
4254 ctx->seen |= SHMEM_SEEN_BLOCK
4255 break;
4256 case Opt_nr_blocks:
4257 ctx->blocks = memparse(param-
4258 if (*rest || ctx->blocks > LO
4259 goto bad_value;
4260 ctx->seen |= SHMEM_SEEN_BLOCK
4261 break;
4262 case Opt_nr_inodes:
4263 ctx->inodes = memparse(param-
4264 if (*rest || ctx->inodes > UL
4265 goto bad_value;
4266 ctx->seen |= SHMEM_SEEN_INODE
4267 break;
4268 case Opt_mode:
4269 ctx->mode = result.uint_32 &
4270 break;
4271 case Opt_uid:
4272 kuid = result.uid;
4273
4274 /*
4275 * The requested uid must be
4276 * filesystem's idmapping.
4277 */
4278 if (!kuid_has_mapping(fc->use
4279 goto bad_value;
4280
4281 ctx->uid = kuid;
4282 break;
4283 case Opt_gid:
4284 kgid = result.gid;
4285
4286 /*
4287 * The requested gid must be
4288 * filesystem's idmapping.
4289 */
4290 if (!kgid_has_mapping(fc->use
4291 goto bad_value;
4292
4293 ctx->gid = kgid;
4294 break;
4295 case Opt_huge:
4296 ctx->huge = result.uint_32;
4297 if (ctx->huge != SHMEM_HUGE_N
4298 !(IS_ENABLED(CONFIG_TRANS
4299 has_transparent_hugepag
4300 goto unsupported_para
4301 ctx->seen |= SHMEM_SEEN_HUGE;
4302 break;
4303 case Opt_mpol:
4304 if (IS_ENABLED(CONFIG_NUMA))
4305 mpol_put(ctx->mpol);
4306 ctx->mpol = NULL;
4307 if (mpol_parse_str(pa
4308 goto bad_valu
4309 break;
4310 }
4311 goto unsupported_parameter;
4312 case Opt_inode32:
4313 ctx->full_inums = false;
4314 ctx->seen |= SHMEM_SEEN_INUMS
4315 break;
4316 case Opt_inode64:
4317 if (sizeof(ino_t) < 8) {
4318 return invalfc(fc,
4319 "Canno
4320 }
4321 ctx->full_inums = true;
4322 ctx->seen |= SHMEM_SEEN_INUMS
4323 break;
4324 case Opt_noswap:
4325 if ((fc->user_ns != &init_use
4326 return invalfc(fc,
4327 "Turni
4328 }
4329 ctx->noswap = true;
4330 ctx->seen |= SHMEM_SEEN_NOSWA
4331 break;
4332 case Opt_quota:
4333 if (fc->user_ns != &init_user
4334 return invalfc(fc, "Q
4335 ctx->seen |= SHMEM_SEEN_QUOTA
4336 ctx->quota_types |= (QTYPE_MA
4337 break;
4338 case Opt_usrquota:
4339 if (fc->user_ns != &init_user
4340 return invalfc(fc, "Q
4341 ctx->seen |= SHMEM_SEEN_QUOTA
4342 ctx->quota_types |= QTYPE_MAS
4343 break;
4344 case Opt_grpquota:
4345 if (fc->user_ns != &init_user
4346 return invalfc(fc, "Q
4347 ctx->seen |= SHMEM_SEEN_QUOTA
4348 ctx->quota_types |= QTYPE_MAS
4349 break;
4350 case Opt_usrquota_block_hardlimit:
4351 size = memparse(param->string
4352 if (*rest || !size)
4353 goto bad_value;
4354 if (size > SHMEM_QUOTA_MAX_SP
4355 return invalfc(fc,
4356 "User
4357 ctx->qlimits.usrquota_bhardli
4358 break;
4359 case Opt_grpquota_block_hardlimit:
4360 size = memparse(param->string
4361 if (*rest || !size)
4362 goto bad_value;
4363 if (size > SHMEM_QUOTA_MAX_SP
4364 return invalfc(fc,
4365 "Group
4366 ctx->qlimits.grpquota_bhardli
4367 break;
4368 case Opt_usrquota_inode_hardlimit:
4369 size = memparse(param->string
4370 if (*rest || !size)
4371 goto bad_value;
4372 if (size > SHMEM_QUOTA_MAX_IN
4373 return invalfc(fc,
4374 "User
4375 ctx->qlimits.usrquota_ihardli
4376 break;
4377 case Opt_grpquota_inode_hardlimit:
4378 size = memparse(param->string
4379 if (*rest || !size)
4380 goto bad_value;
4381 if (size > SHMEM_QUOTA_MAX_IN
4382 return invalfc(fc,
4383 "Group
4384 ctx->qlimits.grpquota_ihardli
4385 break;
4386 }
4387 return 0;
4388
4389 unsupported_parameter:
4390 return invalfc(fc, "Unsupported param
4391 bad_value:
4392 return invalfc(fc, "Bad value for '%s
4393 }
4394
4395 static int shmem_parse_options(struct fs_cont
4396 {
4397 char *options = data;
4398
4399 if (options) {
4400 int err = security_sb_eat_lsm
4401 if (err)
4402 return err;
4403 }
4404
4405 while (options != NULL) {
4406 char *this_char = options;
4407 for (;;) {
4408 /*
4409 * NUL-terminate this
4410 * mount options form
4411 * but mpol's nodelis
4412 */
4413 options = strchr(opti
4414 if (options == NULL)
4415 break;
4416 options++;
4417 if (!isdigit(*options
4418 options[-1] =
4419 break;
4420 }
4421 }
4422 if (*this_char) {
4423 char *value = strchr(
4424 size_t len = 0;
4425 int err;
4426
4427 if (value) {
4428 *value++ = '\
4429 len = strlen(
4430 }
4431 err = vfs_parse_fs_st
4432 if (err < 0)
4433 return err;
4434 }
4435 }
4436 return 0;
4437 }
4438
4439 /*
4440 * Reconfigure a shmem filesystem.
4441 */
4442 static int shmem_reconfigure(struct fs_contex
4443 {
4444 struct shmem_options *ctx = fc->fs_pr
4445 struct shmem_sb_info *sbinfo = SHMEM_
4446 unsigned long used_isp;
4447 struct mempolicy *mpol = NULL;
4448 const char *err;
4449
4450 raw_spin_lock(&sbinfo->stat_lock);
4451 used_isp = sbinfo->max_inodes * BOGO_
4452
4453 if ((ctx->seen & SHMEM_SEEN_BLOCKS) &
4454 if (!sbinfo->max_blocks) {
4455 err = "Cannot retroac
4456 goto out;
4457 }
4458 if (percpu_counter_compare(&s
4459 ct
4460 err = "Too small a si
4461 goto out;
4462 }
4463 }
4464 if ((ctx->seen & SHMEM_SEEN_INODES) &
4465 if (!sbinfo->max_inodes) {
4466 err = "Cannot retroac
4467 goto out;
4468 }
4469 if (ctx->inodes * BOGO_INODE_
4470 err = "Too few inodes
4471 goto out;
4472 }
4473 }
4474
4475 if ((ctx->seen & SHMEM_SEEN_INUMS) &&
4476 sbinfo->next_ino > UINT_MAX) {
4477 err = "Current inum too high
4478 goto out;
4479 }
4480 if ((ctx->seen & SHMEM_SEEN_NOSWAP) &
4481 err = "Cannot disable swap on
4482 goto out;
4483 }
4484 if (!(ctx->seen & SHMEM_SEEN_NOSWAP)
4485 err = "Cannot enable swap on
4486 goto out;
4487 }
4488
4489 if (ctx->seen & SHMEM_SEEN_QUOTA &&
4490 !sb_any_quota_loaded(fc->root->d_
4491 err = "Cannot enable quota on
4492 goto out;
4493 }
4494
4495 #ifdef CONFIG_TMPFS_QUOTA
4496 #define CHANGED_LIMIT(name)
4497 (ctx->qlimits.name## hardlimit &&
4498 (ctx->qlimits.name## hardlimit != sbi
4499
4500 if (CHANGED_LIMIT(usrquota_b) || CHAN
4501 CHANGED_LIMIT(grpquota_b) || CHAN
4502 err = "Cannot change global q
4503 goto out;
4504 }
4505 #endif /* CONFIG_TMPFS_QUOTA */
4506
4507 if (ctx->seen & SHMEM_SEEN_HUGE)
4508 sbinfo->huge = ctx->huge;
4509 if (ctx->seen & SHMEM_SEEN_INUMS)
4510 sbinfo->full_inums = ctx->ful
4511 if (ctx->seen & SHMEM_SEEN_BLOCKS)
4512 sbinfo->max_blocks = ctx->bl
4513 if (ctx->seen & SHMEM_SEEN_INODES) {
4514 sbinfo->max_inodes = ctx->in
4515 sbinfo->free_ispace = ctx->in
4516 }
4517
4518 /*
4519 * Preserve previous mempolicy unless
4520 */
4521 if (ctx->mpol) {
4522 mpol = sbinfo->mpol;
4523 sbinfo->mpol = ctx->mpol;
4524 ctx->mpol = NULL;
4525 }
4526
4527 if (ctx->noswap)
4528 sbinfo->noswap = true;
4529
4530 raw_spin_unlock(&sbinfo->stat_lock);
4531 mpol_put(mpol);
4532 return 0;
4533 out:
4534 raw_spin_unlock(&sbinfo->stat_lock);
4535 return invalfc(fc, "%s", err);
4536 }
4537
4538 static int shmem_show_options(struct seq_file
4539 {
4540 struct shmem_sb_info *sbinfo = SHMEM_
4541 struct mempolicy *mpol;
4542
4543 if (sbinfo->max_blocks != shmem_defau
4544 seq_printf(seq, ",size=%luk",
4545 if (sbinfo->max_inodes != shmem_defau
4546 seq_printf(seq, ",nr_inodes=%
4547 if (sbinfo->mode != (0777 | S_ISVTX))
4548 seq_printf(seq, ",mode=%03ho"
4549 if (!uid_eq(sbinfo->uid, GLOBAL_ROOT_
4550 seq_printf(seq, ",uid=%u",
4551 from_kuid_mun
4552 if (!gid_eq(sbinfo->gid, GLOBAL_ROOT_
4553 seq_printf(seq, ",gid=%u",
4554 from_kgid_mun
4555
4556 /*
4557 * Showing inode{64,32} might be usef
4558 * since then people don't have to re
4559 * /proc/config.gz to confirm 64-bit
4560 * (which may not even exist if IKCON
4561 *
4562 * We hide it when inode64 isn't the
4563 * inodes, since that probably just m
4564 * consideration.
4565 *
4566 * As such:
4567 *
4568 * +-------------
4569 * | TMPFS_INODE6
4570 * +------------------+-------------
4571 * | full_inums=true | show
4572 * | full_inums=false | show
4573 * +------------------+-------------
4574 *
4575 */
4576 if (IS_ENABLED(CONFIG_TMPFS_INODE64)
4577 seq_printf(seq, ",inode%d", (
4578 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
4579 /* Rightly or wrongly, show huge moun
4580 if (sbinfo->huge)
4581 seq_printf(seq, ",huge=%s", s
4582 #endif
4583 mpol = shmem_get_sbmpol(sbinfo);
4584 shmem_show_mpol(seq, mpol);
4585 mpol_put(mpol);
4586 if (sbinfo->noswap)
4587 seq_printf(seq, ",noswap");
4588 #ifdef CONFIG_TMPFS_QUOTA
4589 if (sb_has_quota_active(root->d_sb, U
4590 seq_printf(seq, ",usrquota");
4591 if (sb_has_quota_active(root->d_sb, G
4592 seq_printf(seq, ",grpquota");
4593 if (sbinfo->qlimits.usrquota_bhardlim
4594 seq_printf(seq, ",usrquota_bl
4595 sbinfo->qlimits.us
4596 if (sbinfo->qlimits.grpquota_bhardlim
4597 seq_printf(seq, ",grpquota_bl
4598 sbinfo->qlimits.gr
4599 if (sbinfo->qlimits.usrquota_ihardlim
4600 seq_printf(seq, ",usrquota_in
4601 sbinfo->qlimits.us
4602 if (sbinfo->qlimits.grpquota_ihardlim
4603 seq_printf(seq, ",grpquota_in
4604 sbinfo->qlimits.gr
4605 #endif
4606 return 0;
4607 }
4608
4609 #endif /* CONFIG_TMPFS */
4610
4611 static void shmem_put_super(struct super_bloc
4612 {
4613 struct shmem_sb_info *sbinfo = SHMEM_
4614
4615 #ifdef CONFIG_TMPFS_QUOTA
4616 shmem_disable_quotas(sb);
4617 #endif
4618 free_percpu(sbinfo->ino_batch);
4619 percpu_counter_destroy(&sbinfo->used_
4620 mpol_put(sbinfo->mpol);
4621 kfree(sbinfo);
4622 sb->s_fs_info = NULL;
4623 }
4624
4625 static int shmem_fill_super(struct super_bloc
4626 {
4627 struct shmem_options *ctx = fc->fs_pr
4628 struct inode *inode;
4629 struct shmem_sb_info *sbinfo;
4630 int error = -ENOMEM;
4631
4632 /* Round up to L1_CACHE_BYTES to resi
4633 sbinfo = kzalloc(max((int)sizeof(stru
4634 L1_CACHE_BYTE
4635 if (!sbinfo)
4636 return error;
4637
4638 sb->s_fs_info = sbinfo;
4639
4640 #ifdef CONFIG_TMPFS
4641 /*
4642 * Per default we only allow half of
4643 * tmpfs instance, limiting inodes to
4644 * but the internal instance is left
4645 */
4646 if (!(sb->s_flags & SB_KERNMOUNT)) {
4647 if (!(ctx->seen & SHMEM_SEEN_
4648 ctx->blocks = shmem_d
4649 if (!(ctx->seen & SHMEM_SEEN_
4650 ctx->inodes = shmem_d
4651 if (!(ctx->seen & SHMEM_SEEN_
4652 ctx->full_inums = IS_
4653 sbinfo->noswap = ctx->noswap;
4654 } else {
4655 sb->s_flags |= SB_NOUSER;
4656 }
4657 sb->s_export_op = &shmem_export_ops;
4658 sb->s_flags |= SB_NOSEC | SB_I_VERSIO
4659 #else
4660 sb->s_flags |= SB_NOUSER;
4661 #endif
4662 sbinfo->max_blocks = ctx->blocks;
4663 sbinfo->max_inodes = ctx->inodes;
4664 sbinfo->free_ispace = sbinfo->max_ino
4665 if (sb->s_flags & SB_KERNMOUNT) {
4666 sbinfo->ino_batch = alloc_per
4667 if (!sbinfo->ino_batch)
4668 goto failed;
4669 }
4670 sbinfo->uid = ctx->uid;
4671 sbinfo->gid = ctx->gid;
4672 sbinfo->full_inums = ctx->full_inums;
4673 sbinfo->mode = ctx->mode;
4674 sbinfo->huge = ctx->huge;
4675 sbinfo->mpol = ctx->mpol;
4676 ctx->mpol = NULL;
4677
4678 raw_spin_lock_init(&sbinfo->stat_lock
4679 if (percpu_counter_init(&sbinfo->used
4680 goto failed;
4681 spin_lock_init(&sbinfo->shrinklist_lo
4682 INIT_LIST_HEAD(&sbinfo->shrinklist);
4683
4684 sb->s_maxbytes = MAX_LFS_FILESIZE;
4685 sb->s_blocksize = PAGE_SIZE;
4686 sb->s_blocksize_bits = PAGE_SHIFT;
4687 sb->s_magic = TMPFS_MAGIC;
4688 sb->s_op = &shmem_ops;
4689 sb->s_time_gran = 1;
4690 #ifdef CONFIG_TMPFS_XATTR
4691 sb->s_xattr = shmem_xattr_handlers;
4692 #endif
4693 #ifdef CONFIG_TMPFS_POSIX_ACL
4694 sb->s_flags |= SB_POSIXACL;
4695 #endif
4696 uuid_t uuid;
4697 uuid_gen(&uuid);
4698 super_set_uuid(sb, uuid.b, sizeof(uui
4699
4700 #ifdef CONFIG_TMPFS_QUOTA
4701 if (ctx->seen & SHMEM_SEEN_QUOTA) {
4702 sb->dq_op = &shmem_quota_oper
4703 sb->s_qcop = &dquot_quotactl_
4704 sb->s_quota_types = QTYPE_MAS
4705
4706 /* Copy the default limits fr
4707 memcpy(&sbinfo->qlimits, &ctx
4708 sizeof(struct shmem_qu
4709
4710 if (shmem_enable_quotas(sb, c
4711 goto failed;
4712 }
4713 #endif /* CONFIG_TMPFS_QUOTA */
4714
4715 inode = shmem_get_inode(&nop_mnt_idma
4716 S_IFDIR | sbi
4717 if (IS_ERR(inode)) {
4718 error = PTR_ERR(inode);
4719 goto failed;
4720 }
4721 inode->i_uid = sbinfo->uid;
4722 inode->i_gid = sbinfo->gid;
4723 sb->s_root = d_make_root(inode);
4724 if (!sb->s_root)
4725 goto failed;
4726 return 0;
4727
4728 failed:
4729 shmem_put_super(sb);
4730 return error;
4731 }
4732
4733 static int shmem_get_tree(struct fs_context *
4734 {
4735 return get_tree_nodev(fc, shmem_fill_
4736 }
4737
4738 static void shmem_free_fc(struct fs_context *
4739 {
4740 struct shmem_options *ctx = fc->fs_pr
4741
4742 if (ctx) {
4743 mpol_put(ctx->mpol);
4744 kfree(ctx);
4745 }
4746 }
4747
4748 static const struct fs_context_operations shm
4749 .free = shmem_free_
4750 .get_tree = shmem_get_t
4751 #ifdef CONFIG_TMPFS
4752 .parse_monolithic = shmem_parse
4753 .parse_param = shmem_parse
4754 .reconfigure = shmem_recon
4755 #endif
4756 };
4757
4758 static struct kmem_cache *shmem_inode_cachep
4759
4760 static struct inode *shmem_alloc_inode(struct
4761 {
4762 struct shmem_inode_info *info;
4763 info = alloc_inode_sb(sb, shmem_inode
4764 if (!info)
4765 return NULL;
4766 return &info->vfs_inode;
4767 }
4768
4769 static void shmem_free_in_core_inode(struct i
4770 {
4771 if (S_ISLNK(inode->i_mode))
4772 kfree(inode->i_link);
4773 kmem_cache_free(shmem_inode_cachep, S
4774 }
4775
4776 static void shmem_destroy_inode(struct inode
4777 {
4778 if (S_ISREG(inode->i_mode))
4779 mpol_free_shared_policy(&SHME
4780 if (S_ISDIR(inode->i_mode))
4781 simple_offset_destroy(shmem_g
4782 }
4783
4784 static void shmem_init_inode(void *foo)
4785 {
4786 struct shmem_inode_info *info = foo;
4787 inode_init_once(&info->vfs_inode);
4788 }
4789
4790 static void __init shmem_init_inodecache(void
4791 {
4792 shmem_inode_cachep = kmem_cache_creat
4793 sizeof(struct
4794 0, SLAB_PANIC
4795 }
4796
4797 static void __init shmem_destroy_inodecache(v
4798 {
4799 kmem_cache_destroy(shmem_inode_cachep
4800 }
4801
4802 /* Keep the page in page cache instead of tru
4803 static int shmem_error_remove_folio(struct ad
4804 struct fol
4805 {
4806 return 0;
4807 }
4808
4809 static const struct address_space_operations
4810 .writepage = shmem_writepage,
4811 .dirty_folio = noop_dirty_folio,
4812 #ifdef CONFIG_TMPFS
4813 .write_begin = shmem_write_begin,
4814 .write_end = shmem_write_end,
4815 #endif
4816 #ifdef CONFIG_MIGRATION
4817 .migrate_folio = migrate_folio,
4818 #endif
4819 .error_remove_folio = shmem_error_rem
4820 };
4821
4822 static const struct file_operations shmem_fil
4823 .mmap = shmem_mmap,
4824 .open = shmem_file_open,
4825 .get_unmapped_area = shmem_get_unmapp
4826 #ifdef CONFIG_TMPFS
4827 .llseek = shmem_file_llseek,
4828 .read_iter = shmem_file_read_ite
4829 .write_iter = shmem_file_write_it
4830 .fsync = noop_fsync,
4831 .splice_read = shmem_file_splice_r
4832 .splice_write = iter_file_splice_wr
4833 .fallocate = shmem_fallocate,
4834 #endif
4835 };
4836
4837 static const struct inode_operations shmem_in
4838 .getattr = shmem_getattr,
4839 .setattr = shmem_setattr,
4840 #ifdef CONFIG_TMPFS_XATTR
4841 .listxattr = shmem_listxattr,
4842 .set_acl = simple_set_acl,
4843 .fileattr_get = shmem_fileattr_get,
4844 .fileattr_set = shmem_fileattr_set,
4845 #endif
4846 };
4847
4848 static const struct inode_operations shmem_di
4849 #ifdef CONFIG_TMPFS
4850 .getattr = shmem_getattr,
4851 .create = shmem_create,
4852 .lookup = simple_lookup,
4853 .link = shmem_link,
4854 .unlink = shmem_unlink,
4855 .symlink = shmem_symlink,
4856 .mkdir = shmem_mkdir,
4857 .rmdir = shmem_rmdir,
4858 .mknod = shmem_mknod,
4859 .rename = shmem_rename2,
4860 .tmpfile = shmem_tmpfile,
4861 .get_offset_ctx = shmem_get_offset_ct
4862 #endif
4863 #ifdef CONFIG_TMPFS_XATTR
4864 .listxattr = shmem_listxattr,
4865 .fileattr_get = shmem_fileattr_get,
4866 .fileattr_set = shmem_fileattr_set,
4867 #endif
4868 #ifdef CONFIG_TMPFS_POSIX_ACL
4869 .setattr = shmem_setattr,
4870 .set_acl = simple_set_acl,
4871 #endif
4872 };
4873
4874 static const struct inode_operations shmem_sp
4875 .getattr = shmem_getattr,
4876 #ifdef CONFIG_TMPFS_XATTR
4877 .listxattr = shmem_listxattr,
4878 #endif
4879 #ifdef CONFIG_TMPFS_POSIX_ACL
4880 .setattr = shmem_setattr,
4881 .set_acl = simple_set_acl,
4882 #endif
4883 };
4884
4885 static const struct super_operations shmem_op
4886 .alloc_inode = shmem_alloc_inode,
4887 .free_inode = shmem_free_in_core_
4888 .destroy_inode = shmem_destroy_inode
4889 #ifdef CONFIG_TMPFS
4890 .statfs = shmem_statfs,
4891 .show_options = shmem_show_options,
4892 #endif
4893 #ifdef CONFIG_TMPFS_QUOTA
4894 .get_dquots = shmem_get_dquots,
4895 #endif
4896 .evict_inode = shmem_evict_inode,
4897 .drop_inode = generic_delete_inod
4898 .put_super = shmem_put_super,
4899 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
4900 .nr_cached_objects = shmem_unuse
4901 .free_cached_objects = shmem_unuse
4902 #endif
4903 };
4904
4905 static const struct vm_operations_struct shme
4906 .fault = shmem_fault,
4907 .map_pages = filemap_map_pages,
4908 #ifdef CONFIG_NUMA
4909 .set_policy = shmem_set_policy,
4910 .get_policy = shmem_get_policy,
4911 #endif
4912 };
4913
4914 static const struct vm_operations_struct shme
4915 .fault = shmem_fault,
4916 .map_pages = filemap_map_pages,
4917 #ifdef CONFIG_NUMA
4918 .set_policy = shmem_set_policy,
4919 .get_policy = shmem_get_policy,
4920 #endif
4921 };
4922
4923 int shmem_init_fs_context(struct fs_context *
4924 {
4925 struct shmem_options *ctx;
4926
4927 ctx = kzalloc(sizeof(struct shmem_opt
4928 if (!ctx)
4929 return -ENOMEM;
4930
4931 ctx->mode = 0777 | S_ISVTX;
4932 ctx->uid = current_fsuid();
4933 ctx->gid = current_fsgid();
4934
4935 fc->fs_private = ctx;
4936 fc->ops = &shmem_fs_context_ops;
4937 return 0;
4938 }
4939
4940 static struct file_system_type shmem_fs_type
4941 .owner = THIS_MODULE,
4942 .name = "tmpfs",
4943 .init_fs_context = shmem_init_fs_cont
4944 #ifdef CONFIG_TMPFS
4945 .parameters = shmem_fs_parameters
4946 #endif
4947 .kill_sb = kill_litter_super,
4948 .fs_flags = FS_USERNS_MOUNT | F
4949 };
4950
4951 void __init shmem_init(void)
4952 {
4953 int error;
4954
4955 shmem_init_inodecache();
4956
4957 #ifdef CONFIG_TMPFS_QUOTA
4958 register_quota_format(&shmem_quota_fo
4959 #endif
4960
4961 error = register_filesystem(&shmem_fs
4962 if (error) {
4963 pr_err("Could not register tm
4964 goto out2;
4965 }
4966
4967 shm_mnt = kern_mount(&shmem_fs_type);
4968 if (IS_ERR(shm_mnt)) {
4969 error = PTR_ERR(shm_mnt);
4970 pr_err("Could not kern_mount
4971 goto out1;
4972 }
4973
4974 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
4975 if (has_transparent_hugepage() && shm
4976 SHMEM_SB(shm_mnt->mnt_sb)->hu
4977 else
4978 shmem_huge = SHMEM_HUGE_NEVER
4979
4980 /*
4981 * Default to setting PMD-sized THP t
4982 * disable all other multi-size THPs.
4983 */
4984 huge_shmem_orders_inherit = BIT(HPAGE
4985 #endif
4986 return;
4987
4988 out1:
4989 unregister_filesystem(&shmem_fs_type)
4990 out2:
4991 #ifdef CONFIG_TMPFS_QUOTA
4992 unregister_quota_format(&shmem_quota_
4993 #endif
4994 shmem_destroy_inodecache();
4995 shm_mnt = ERR_PTR(error);
4996 }
4997
4998 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && d
4999 static ssize_t shmem_enabled_show(struct kobj
5000 struct kobj
5001 {
5002 static const int values[] = {
5003 SHMEM_HUGE_ALWAYS,
5004 SHMEM_HUGE_WITHIN_SIZE,
5005 SHMEM_HUGE_ADVISE,
5006 SHMEM_HUGE_NEVER,
5007 SHMEM_HUGE_DENY,
5008 SHMEM_HUGE_FORCE,
5009 };
5010 int len = 0;
5011 int i;
5012
5013 for (i = 0; i < ARRAY_SIZE(values); i
5014 len += sysfs_emit_at(buf, len
5015 shmem_huge ==
5016 i ? " " : "",
5017 }
5018 len += sysfs_emit_at(buf, len, "\n");
5019
5020 return len;
5021 }
5022
5023 static ssize_t shmem_enabled_store(struct kob
5024 struct kobj_attribute *attr,
5025 {
5026 char tmp[16];
5027 int huge;
5028
5029 if (count + 1 > sizeof(tmp))
5030 return -EINVAL;
5031 memcpy(tmp, buf, count);
5032 tmp[count] = '\0';
5033 if (count && tmp[count - 1] == '\n')
5034 tmp[count - 1] = '\0';
5035
5036 huge = shmem_parse_huge(tmp);
5037 if (huge == -EINVAL)
5038 return -EINVAL;
5039 if (!has_transparent_hugepage() &&
5040 huge != SHMEM_HUGE_NE
5041 return -EINVAL;
5042
5043 /* Do not override huge allocation po
5044 if (huge == SHMEM_HUGE_FORCE &&
5045 huge_shmem_orders_inherit != BIT(
5046 return -EINVAL;
5047
5048 shmem_huge = huge;
5049 if (shmem_huge > SHMEM_HUGE_DENY)
5050 SHMEM_SB(shm_mnt->mnt_sb)->hu
5051 return count;
5052 }
5053
5054 struct kobj_attribute shmem_enabled_attr = __
5055 static DEFINE_SPINLOCK(huge_shmem_orders_lock
5056
5057 static ssize_t thpsize_shmem_enabled_show(str
5058 str
5059 {
5060 int order = to_thpsize(kobj)->order;
5061 const char *output;
5062
5063 if (test_bit(order, &huge_shmem_order
5064 output = "[always] inherit wi
5065 else if (test_bit(order, &huge_shmem_
5066 output = "always [inherit] wi
5067 else if (test_bit(order, &huge_shmem_
5068 output = "always inherit [wit
5069 else if (test_bit(order, &huge_shmem_
5070 output = "always inherit with
5071 else
5072 output = "always inherit with
5073
5074 return sysfs_emit(buf, "%s\n", output
5075 }
5076
5077 static ssize_t thpsize_shmem_enabled_store(st
5078 st
5079 co
5080 {
5081 int order = to_thpsize(kobj)->order;
5082 ssize_t ret = count;
5083
5084 if (sysfs_streq(buf, "always")) {
5085 spin_lock(&huge_shmem_orders_
5086 clear_bit(order, &huge_shmem_
5087 clear_bit(order, &huge_shmem_
5088 clear_bit(order, &huge_shmem_
5089 set_bit(order, &huge_shmem_or
5090 spin_unlock(&huge_shmem_order
5091 } else if (sysfs_streq(buf, "inherit"
5092 /* Do not override huge alloc
5093 if (shmem_huge == SHMEM_HUGE_
5094 order != HPAGE_PMD_ORDER)
5095 return -EINVAL;
5096
5097 spin_lock(&huge_shmem_orders_
5098 clear_bit(order, &huge_shmem_
5099 clear_bit(order, &huge_shmem_
5100 clear_bit(order, &huge_shmem_
5101 set_bit(order, &huge_shmem_or
5102 spin_unlock(&huge_shmem_order
5103 } else if (sysfs_streq(buf, "within_s
5104 spin_lock(&huge_shmem_orders_
5105 clear_bit(order, &huge_shmem_
5106 clear_bit(order, &huge_shmem_
5107 clear_bit(order, &huge_shmem_
5108 set_bit(order, &huge_shmem_or
5109 spin_unlock(&huge_shmem_order
5110 } else if (sysfs_streq(buf, "advise")
5111 spin_lock(&huge_shmem_orders_
5112 clear_bit(order, &huge_shmem_
5113 clear_bit(order, &huge_shmem_
5114 clear_bit(order, &huge_shmem_
5115 set_bit(order, &huge_shmem_or
5116 spin_unlock(&huge_shmem_order
5117 } else if (sysfs_streq(buf, "never"))
5118 spin_lock(&huge_shmem_orders_
5119 clear_bit(order, &huge_shmem_
5120 clear_bit(order, &huge_shmem_
5121 clear_bit(order, &huge_shmem_
5122 clear_bit(order, &huge_shmem_
5123 spin_unlock(&huge_shmem_order
5124 } else {
5125 ret = -EINVAL;
5126 }
5127
5128 return ret;
5129 }
5130
5131 struct kobj_attribute thpsize_shmem_enabled_a
5132 __ATTR(shmem_enabled, 0644, thpsize_s
5133 #endif /* CONFIG_TRANSPARENT_HUGEPAGE && CONF
5134
5135 #else /* !CONFIG_SHMEM */
5136
5137 /*
5138 * tiny-shmem: simple shmemfs and tmpfs using
5139 *
5140 * This is intended for small system where th
5141 * shmem code (swap-backed and resource-limit
5142 * their complexity. On systems without swap
5143 * effectively equivalent, but much lighter w
5144 */
5145
5146 static struct file_system_type shmem_fs_type
5147 .name = "tmpfs",
5148 .init_fs_context = ramfs_init_fs_cont
5149 .parameters = ramfs_fs_parameters
5150 .kill_sb = ramfs_kill_sb,
5151 .fs_flags = FS_USERNS_MOUNT,
5152 };
5153
5154 void __init shmem_init(void)
5155 {
5156 BUG_ON(register_filesystem(&shmem_fs_
5157
5158 shm_mnt = kern_mount(&shmem_fs_type);
5159 BUG_ON(IS_ERR(shm_mnt));
5160 }
5161
5162 int shmem_unuse(unsigned int type)
5163 {
5164 return 0;
5165 }
5166
5167 int shmem_lock(struct file *file, int lock, s
5168 {
5169 return 0;
5170 }
5171
5172 void shmem_unlock_mapping(struct address_spac
5173 {
5174 }
5175
5176 #ifdef CONFIG_MMU
5177 unsigned long shmem_get_unmapped_area(struct
5178 unsigne
5179 unsigne
5180 {
5181 return mm_get_unmapped_area(current->
5182 }
5183 #endif
5184
5185 void shmem_truncate_range(struct inode *inode
5186 {
5187 truncate_inode_pages_range(inode->i_m
5188 }
5189 EXPORT_SYMBOL_GPL(shmem_truncate_range);
5190
5191 #define shmem_vm_ops
5192 #define shmem_anon_vm_ops
5193 #define shmem_file_operations
5194 #define shmem_acct_size(flags, size)
5195 #define shmem_unacct_size(flags, size)
5196
5197 static inline struct inode *shmem_get_inode(s
5198 struct super_
5199 umode_t mode,
5200 {
5201 struct inode *inode = ramfs_get_inode
5202 return inode ? inode : ERR_PTR(-ENOSP
5203 }
5204
5205 #endif /* CONFIG_SHMEM */
5206
5207 /* common code */
5208
5209 static struct file *__shmem_file_setup(struct
5210 loff_t size, unsigned
5211 {
5212 struct inode *inode;
5213 struct file *res;
5214
5215 if (IS_ERR(mnt))
5216 return ERR_CAST(mnt);
5217
5218 if (size < 0 || size > MAX_LFS_FILESI
5219 return ERR_PTR(-EINVAL);
5220
5221 if (shmem_acct_size(flags, size))
5222 return ERR_PTR(-ENOMEM);
5223
5224 if (is_idmapped_mnt(mnt))
5225 return ERR_PTR(-EINVAL);
5226
5227 inode = shmem_get_inode(&nop_mnt_idma
5228 S_IFREG | S_I
5229 if (IS_ERR(inode)) {
5230 shmem_unacct_size(flags, size
5231 return ERR_CAST(inode);
5232 }
5233 inode->i_flags |= i_flags;
5234 inode->i_size = size;
5235 clear_nlink(inode); /* It is unli
5236 res = ERR_PTR(ramfs_nommu_expand_for_
5237 if (!IS_ERR(res))
5238 res = alloc_file_pseudo(inode
5239 &shmem_file_o
5240 if (IS_ERR(res))
5241 iput(inode);
5242 return res;
5243 }
5244
5245 /**
5246 * shmem_kernel_file_setup - get an unlinked
5247 * kernel internal. There will be NO LS
5248 * underlying inode. So users of this i
5249 * higher layer. The users are the big_
5250 * checks are provided at the key or shm
5251 * @name: name for dentry (to be seen in /pro
5252 * @size: size to be set for the file
5253 * @flags: VM_NORESERVE suppresses pre-accoun
5254 */
5255 struct file *shmem_kernel_file_setup(const ch
5256 {
5257 return __shmem_file_setup(shm_mnt, na
5258 }
5259 EXPORT_SYMBOL_GPL(shmem_kernel_file_setup);
5260
5261 /**
5262 * shmem_file_setup - get an unlinked file li
5263 * @name: name for dentry (to be seen in /pro
5264 * @size: size to be set for the file
5265 * @flags: VM_NORESERVE suppresses pre-accoun
5266 */
5267 struct file *shmem_file_setup(const char *nam
5268 {
5269 return __shmem_file_setup(shm_mnt, na
5270 }
5271 EXPORT_SYMBOL_GPL(shmem_file_setup);
5272
5273 /**
5274 * shmem_file_setup_with_mnt - get an unlinke
5275 * @mnt: the tmpfs mount where the file will
5276 * @name: name for dentry (to be seen in /pro
5277 * @size: size to be set for the file
5278 * @flags: VM_NORESERVE suppresses pre-accoun
5279 */
5280 struct file *shmem_file_setup_with_mnt(struct
5281 loff_t
5282 {
5283 return __shmem_file_setup(mnt, name,
5284 }
5285 EXPORT_SYMBOL_GPL(shmem_file_setup_with_mnt);
5286
5287 /**
5288 * shmem_zero_setup - setup a shared anonymou
5289 * @vma: the vma to be mmapped is prepared by
5290 */
5291 int shmem_zero_setup(struct vm_area_struct *v
5292 {
5293 struct file *file;
5294 loff_t size = vma->vm_end - vma->vm_s
5295
5296 /*
5297 * Cloning a new file under mmap_lock
5298 * between XFS directory reading and
5299 * accessible to the user through its
5300 * bypass file security, in the same
5301 */
5302 file = shmem_kernel_file_setup("dev/z
5303 if (IS_ERR(file))
5304 return PTR_ERR(file);
5305
5306 if (vma->vm_file)
5307 fput(vma->vm_file);
5308 vma->vm_file = file;
5309 vma->vm_ops = &shmem_anon_vm_ops;
5310
5311 return 0;
5312 }
5313
5314 /**
5315 * shmem_read_folio_gfp - read into page cach
5316 * @mapping: the folio's address_space
5317 * @index: the folio index
5318 * @gfp: the page allocator flags to u
5319 *
5320 * This behaves as a tmpfs "read_cache_page_g
5321 * with any new page allocations done using t
5322 * But read_cache_page_gfp() uses the ->read_
5323 * suit tmpfs, since it may have pages in swa
5324 * for itself; although drivers/gpu/drm i915
5325 *
5326 * i915_gem_object_get_pages_gtt() mixes __GF
5327 * with the mapping_gfp_mask(), to avoid OOMi
5328 */
5329 struct folio *shmem_read_folio_gfp(struct add
5330 pgoff_t index, gfp_t gfp)
5331 {
5332 #ifdef CONFIG_SHMEM
5333 struct inode *inode = mapping->host;
5334 struct folio *folio;
5335 int error;
5336
5337 error = shmem_get_folio_gfp(inode, in
5338 gfp, NULL
5339 if (error)
5340 return ERR_PTR(error);
5341
5342 folio_unlock(folio);
5343 return folio;
5344 #else
5345 /*
5346 * The tiny !SHMEM case uses ramfs wi
5347 */
5348 return mapping_read_folio_gfp(mapping
5349 #endif
5350 }
5351 EXPORT_SYMBOL_GPL(shmem_read_folio_gfp);
5352
5353 struct page *shmem_read_mapping_page_gfp(stru
5354 pgof
5355 {
5356 struct folio *folio = shmem_read_foli
5357 struct page *page;
5358
5359 if (IS_ERR(folio))
5360 return &folio->page;
5361
5362 page = folio_file_page(folio, index);
5363 if (PageHWPoison(page)) {
5364 folio_put(folio);
5365 return ERR_PTR(-EIO);
5366 }
5367
5368 return page;
5369 }
5370 EXPORT_SYMBOL_GPL(shmem_read_mapping_page_gfp
5371
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.