1 /*
2 * Resizable virtual memory filesystem for Linux.
3 *
4 * Copyright (C) 2000 Linus Torvalds.
5 * 2000 Transmeta Corp.
6 * 2000-2001 Christoph Rohland
7 * 2000-2001 SAP AG
8 * 2002 Red Hat Inc.
9 * Copyright (C) 2002-2011 Hugh Dickins.
10 * Copyright (C) 2011 Google Inc.
11 * Copyright (C) 2002-2005 VERITAS Software Corporation.
12 * Copyright (C) 2004 Andi Kleen, SuSE Labs
13 *
14 * Extended attribute support for tmpfs:
15 * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net>
16 * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com>
17 *
18 * tiny-shmem:
19 * Copyright (c) 2004, 2008 Matt Mackall <mpm@selenic.com>
20 *
21 * This file is released under the GPL.
22 */
23
24 #include <linux/fs.h>
25 #include <linux/init.h>
26 #include <linux/vfs.h>
27 #include <linux/mount.h>
28 #include <linux/ramfs.h>
29 #include <linux/pagemap.h>
30 #include <linux/file.h>
31 #include <linux/fileattr.h>
32 #include <linux/mm.h>
33 #include <linux/random.h>
34 #include <linux/sched/signal.h>
35 #include <linux/export.h>
36 #include <linux/shmem_fs.h>
37 #include <linux/swap.h>
38 #include <linux/uio.h>
39 #include <linux/hugetlb.h>
40 #include <linux/fs_parser.h>
41 #include <linux/swapfile.h>
42 #include <linux/iversion.h>
43 #include "swap.h"
44
45 static struct vfsmount *shm_mnt __ro_after_init;
46
47 #ifdef CONFIG_SHMEM
48 /*
49 * This virtual memory filesystem is heavily based on the ramfs. It
50 * extends ramfs by the ability to use swap and honor resource limits
51 * which makes it a completely usable filesystem.
52 */
53
54 #include <linux/xattr.h>
55 #include <linux/exportfs.h>
56 #include <linux/posix_acl.h>
57 #include <linux/posix_acl_xattr.h>
58 #include <linux/mman.h>
59 #include <linux/string.h>
60 #include <linux/slab.h>
61 #include <linux/backing-dev.h>
62 #include <linux/writeback.h>
63 #include <linux/pagevec.h>
64 #include <linux/percpu_counter.h>
65 #include <linux/falloc.h>
66 #include <linux/splice.h>
67 #include <linux/security.h>
68 #include <linux/swapops.h>
69 #include <linux/mempolicy.h>
70 #include <linux/namei.h>
71 #include <linux/ctype.h>
72 #include <linux/migrate.h>
73 #include <linux/highmem.h>
74 #include <linux/seq_file.h>
75 #include <linux/magic.h>
76 #include <linux/syscalls.h>
77 #include <linux/fcntl.h>
78 #include <uapi/linux/memfd.h>
79 #include <linux/rmap.h>
80 #include <linux/uuid.h>
81 #include <linux/quotaops.h>
82 #include <linux/rcupdate_wait.h>
83
84 #include <linux/uaccess.h>
85
86 #include "internal.h"
87
88 #define BLOCKS_PER_PAGE (PAGE_SIZE/512)
89 #define VM_ACCT(size) (PAGE_ALIGN(size) >> PAGE_SHIFT)
90
91 /* Pretend that each entry is of this size in directory's i_size */
92 #define BOGO_DIRENT_SIZE 20
93
94 /* Pretend that one inode + its dentry occupy this much memory */
95 #define BOGO_INODE_SIZE 1024
96
97 /* Symlink up to this size is kmalloc'ed instead of using a swappable page */
98 #define SHORT_SYMLINK_LEN 128
99
100 /*
101 * shmem_fallocate communicates with shmem_fault or shmem_writepage via
102 * inode->i_private (with i_rwsem making sure that it has only one user at
103 * a time): we would prefer not to enlarge the shmem inode just for that.
104 */
105 struct shmem_falloc {
106 wait_queue_head_t *waitq; /* faults into hole wait for punch to end */
107 pgoff_t start; /* start of range currently being fallocated */
108 pgoff_t next; /* the next page offset to be fallocated */
109 pgoff_t nr_falloced; /* how many new pages have been fallocated */
110 pgoff_t nr_unswapped; /* how often writepage refused to swap out */
111 };
112
113 struct shmem_options {
114 unsigned long long blocks;
115 unsigned long long inodes;
116 struct mempolicy *mpol;
117 kuid_t uid;
118 kgid_t gid;
119 umode_t mode;
120 bool full_inums;
121 int huge;
122 int seen;
123 bool noswap;
124 unsigned short quota_types;
125 struct shmem_quota_limits qlimits;
126 #define SHMEM_SEEN_BLOCKS 1
127 #define SHMEM_SEEN_INODES 2
128 #define SHMEM_SEEN_HUGE 4
129 #define SHMEM_SEEN_INUMS 8
130 #define SHMEM_SEEN_NOSWAP 16
131 #define SHMEM_SEEN_QUOTA 32
132 };
133
134 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
135 static unsigned long huge_shmem_orders_always __read_mostly;
136 static unsigned long huge_shmem_orders_madvise __read_mostly;
137 static unsigned long huge_shmem_orders_inherit __read_mostly;
138 static unsigned long huge_shmem_orders_within_size __read_mostly;
139 #endif
140
141 #ifdef CONFIG_TMPFS
142 static unsigned long shmem_default_max_blocks(void)
143 {
144 return totalram_pages() / 2;
145 }
146
147 static unsigned long shmem_default_max_inodes(void)
148 {
149 unsigned long nr_pages = totalram_pages();
150
151 return min3(nr_pages - totalhigh_pages(), nr_pages / 2,
152 ULONG_MAX / BOGO_INODE_SIZE);
153 }
154 #endif
155
156 static int shmem_swapin_folio(struct inode *inode, pgoff_t index,
157 struct folio **foliop, enum sgp_type sgp, gfp_t gfp,
158 struct mm_struct *fault_mm, vm_fault_t *fault_type);
159
160 static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb)
161 {
162 return sb->s_fs_info;
163 }
164
165 /*
166 * shmem_file_setup pre-accounts the whole fixed size of a VM object,
167 * for shared memory and for shared anonymous (/dev/zero) mappings
168 * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1),
169 * consistent with the pre-accounting of private mappings ...
170 */
171 static inline int shmem_acct_size(unsigned long flags, loff_t size)
172 {
173 return (flags & VM_NORESERVE) ?
174 0 : security_vm_enough_memory_mm(current->mm, VM_ACCT(size));
175 }
176
177 static inline void shmem_unacct_size(unsigned long flags, loff_t size)
178 {
179 if (!(flags & VM_NORESERVE))
180 vm_unacct_memory(VM_ACCT(size));
181 }
182
183 static inline int shmem_reacct_size(unsigned long flags,
184 loff_t oldsize, loff_t newsize)
185 {
186 if (!(flags & VM_NORESERVE)) {
187 if (VM_ACCT(newsize) > VM_ACCT(oldsize))
188 return security_vm_enough_memory_mm(current->mm,
189 VM_ACCT(newsize) - VM_ACCT(oldsize));
190 else if (VM_ACCT(newsize) < VM_ACCT(oldsize))
191 vm_unacct_memory(VM_ACCT(oldsize) - VM_ACCT(newsize));
192 }
193 return 0;
194 }
195
196 /*
197 * ... whereas tmpfs objects are accounted incrementally as
198 * pages are allocated, in order to allow large sparse files.
199 * shmem_get_folio reports shmem_acct_blocks failure as -ENOSPC not -ENOMEM,
200 * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM.
201 */
202 static inline int shmem_acct_blocks(unsigned long flags, long pages)
203 {
204 if (!(flags & VM_NORESERVE))
205 return 0;
206
207 return security_vm_enough_memory_mm(current->mm,
208 pages * VM_ACCT(PAGE_SIZE));
209 }
210
211 static inline void shmem_unacct_blocks(unsigned long flags, long pages)
212 {
213 if (flags & VM_NORESERVE)
214 vm_unacct_memory(pages * VM_ACCT(PAGE_SIZE));
215 }
216
217 static int shmem_inode_acct_blocks(struct inode *inode, long pages)
218 {
219 struct shmem_inode_info *info = SHMEM_I(inode);
220 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
221 int err = -ENOSPC;
222
223 if (shmem_acct_blocks(info->flags, pages))
224 return err;
225
226 might_sleep(); /* when quotas */
227 if (sbinfo->max_blocks) {
228 if (!percpu_counter_limited_add(&sbinfo->used_blocks,
229 sbinfo->max_blocks, pages))
230 goto unacct;
231
232 err = dquot_alloc_block_nodirty(inode, pages);
233 if (err) {
234 percpu_counter_sub(&sbinfo->used_blocks, pages);
235 goto unacct;
236 }
237 } else {
238 err = dquot_alloc_block_nodirty(inode, pages);
239 if (err)
240 goto unacct;
241 }
242
243 return 0;
244
245 unacct:
246 shmem_unacct_blocks(info->flags, pages);
247 return err;
248 }
249
250 static void shmem_inode_unacct_blocks(struct inode *inode, long pages)
251 {
252 struct shmem_inode_info *info = SHMEM_I(inode);
253 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
254
255 might_sleep(); /* when quotas */
256 dquot_free_block_nodirty(inode, pages);
257
258 if (sbinfo->max_blocks)
259 percpu_counter_sub(&sbinfo->used_blocks, pages);
260 shmem_unacct_blocks(info->flags, pages);
261 }
262
263 static const struct super_operations shmem_ops;
264 static const struct address_space_operations shmem_aops;
265 static const struct file_operations shmem_file_operations;
266 static const struct inode_operations shmem_inode_operations;
267 static const struct inode_operations shmem_dir_inode_operations;
268 static const struct inode_operations shmem_special_inode_operations;
269 static const struct vm_operations_struct shmem_vm_ops;
270 static const struct vm_operations_struct shmem_anon_vm_ops;
271 static struct file_system_type shmem_fs_type;
272
273 bool shmem_mapping(struct address_space *mapping)
274 {
275 return mapping->a_ops == &shmem_aops;
276 }
277 EXPORT_SYMBOL_GPL(shmem_mapping);
278
279 bool vma_is_anon_shmem(struct vm_area_struct *vma)
280 {
281 return vma->vm_ops == &shmem_anon_vm_ops;
282 }
283
284 bool vma_is_shmem(struct vm_area_struct *vma)
285 {
286 return vma_is_anon_shmem(vma) || vma->vm_ops == &shmem_vm_ops;
287 }
288
289 static LIST_HEAD(shmem_swaplist);
290 static DEFINE_MUTEX(shmem_swaplist_mutex);
291
292 #ifdef CONFIG_TMPFS_QUOTA
293
294 static int shmem_enable_quotas(struct super_block *sb,
295 unsigned short quota_types)
296 {
297 int type, err = 0;
298
299 sb_dqopt(sb)->flags |= DQUOT_QUOTA_SYS_FILE | DQUOT_NOLIST_DIRTY;
300 for (type = 0; type < SHMEM_MAXQUOTAS; type++) {
301 if (!(quota_types & (1 << type)))
302 continue;
303 err = dquot_load_quota_sb(sb, type, QFMT_SHMEM,
304 DQUOT_USAGE_ENABLED |
305 DQUOT_LIMITS_ENABLED);
306 if (err)
307 goto out_err;
308 }
309 return 0;
310
311 out_err:
312 pr_warn("tmpfs: failed to enable quota tracking (type=%d, err=%d)\n",
313 type, err);
314 for (type--; type >= 0; type--)
315 dquot_quota_off(sb, type);
316 return err;
317 }
318
319 static void shmem_disable_quotas(struct super_block *sb)
320 {
321 int type;
322
323 for (type = 0; type < SHMEM_MAXQUOTAS; type++)
324 dquot_quota_off(sb, type);
325 }
326
327 static struct dquot __rcu **shmem_get_dquots(struct inode *inode)
328 {
329 return SHMEM_I(inode)->i_dquot;
330 }
331 #endif /* CONFIG_TMPFS_QUOTA */
332
333 /*
334 * shmem_reserve_inode() performs bookkeeping to reserve a shmem inode, and
335 * produces a novel ino for the newly allocated inode.
336 *
337 * It may also be called when making a hard link to permit the space needed by
338 * each dentry. However, in that case, no new inode number is needed since that
339 * internally draws from another pool of inode numbers (currently global
340 * get_next_ino()). This case is indicated by passing NULL as inop.
341 */
342 #define SHMEM_INO_BATCH 1024
343 static int shmem_reserve_inode(struct super_block *sb, ino_t *inop)
344 {
345 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
346 ino_t ino;
347
348 if (!(sb->s_flags & SB_KERNMOUNT)) {
349 raw_spin_lock(&sbinfo->stat_lock);
350 if (sbinfo->max_inodes) {
351 if (sbinfo->free_ispace < BOGO_INODE_SIZE) {
352 raw_spin_unlock(&sbinfo->stat_lock);
353 return -ENOSPC;
354 }
355 sbinfo->free_ispace -= BOGO_INODE_SIZE;
356 }
357 if (inop) {
358 ino = sbinfo->next_ino++;
359 if (unlikely(is_zero_ino(ino)))
360 ino = sbinfo->next_ino++;
361 if (unlikely(!sbinfo->full_inums &&
362 ino > UINT_MAX)) {
363 /*
364 * Emulate get_next_ino uint wraparound for
365 * compatibility
366 */
367 if (IS_ENABLED(CONFIG_64BIT))
368 pr_warn("%s: inode number overflow on device %d, consider using inode64 mount option\n",
369 __func__, MINOR(sb->s_dev));
370 sbinfo->next_ino = 1;
371 ino = sbinfo->next_ino++;
372 }
373 *inop = ino;
374 }
375 raw_spin_unlock(&sbinfo->stat_lock);
376 } else if (inop) {
377 /*
378 * __shmem_file_setup, one of our callers, is lock-free: it
379 * doesn't hold stat_lock in shmem_reserve_inode since
380 * max_inodes is always 0, and is called from potentially
381 * unknown contexts. As such, use a per-cpu batched allocator
382 * which doesn't require the per-sb stat_lock unless we are at
383 * the batch boundary.
384 *
385 * We don't need to worry about inode{32,64} since SB_KERNMOUNT
386 * shmem mounts are not exposed to userspace, so we don't need
387 * to worry about things like glibc compatibility.
388 */
389 ino_t *next_ino;
390
391 next_ino = per_cpu_ptr(sbinfo->ino_batch, get_cpu());
392 ino = *next_ino;
393 if (unlikely(ino % SHMEM_INO_BATCH == 0)) {
394 raw_spin_lock(&sbinfo->stat_lock);
395 ino = sbinfo->next_ino;
396 sbinfo->next_ino += SHMEM_INO_BATCH;
397 raw_spin_unlock(&sbinfo->stat_lock);
398 if (unlikely(is_zero_ino(ino)))
399 ino++;
400 }
401 *inop = ino;
402 *next_ino = ++ino;
403 put_cpu();
404 }
405
406 return 0;
407 }
408
409 static void shmem_free_inode(struct super_block *sb, size_t freed_ispace)
410 {
411 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
412 if (sbinfo->max_inodes) {
413 raw_spin_lock(&sbinfo->stat_lock);
414 sbinfo->free_ispace += BOGO_INODE_SIZE + freed_ispace;
415 raw_spin_unlock(&sbinfo->stat_lock);
416 }
417 }
418
419 /**
420 * shmem_recalc_inode - recalculate the block usage of an inode
421 * @inode: inode to recalc
422 * @alloced: the change in number of pages allocated to inode
423 * @swapped: the change in number of pages swapped from inode
424 *
425 * We have to calculate the free blocks since the mm can drop
426 * undirtied hole pages behind our back.
427 *
428 * But normally info->alloced == inode->i_mapping->nrpages + info->swapped
429 * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped)
430 */
431 static void shmem_recalc_inode(struct inode *inode, long alloced, long swapped)
432 {
433 struct shmem_inode_info *info = SHMEM_I(inode);
434 long freed;
435
436 spin_lock(&info->lock);
437 info->alloced += alloced;
438 info->swapped += swapped;
439 freed = info->alloced - info->swapped -
440 READ_ONCE(inode->i_mapping->nrpages);
441 /*
442 * Special case: whereas normally shmem_recalc_inode() is called
443 * after i_mapping->nrpages has already been adjusted (up or down),
444 * shmem_writepage() has to raise swapped before nrpages is lowered -
445 * to stop a racing shmem_recalc_inode() from thinking that a page has
446 * been freed. Compensate here, to avoid the need for a followup call.
447 */
448 if (swapped > 0)
449 freed += swapped;
450 if (freed > 0)
451 info->alloced -= freed;
452 spin_unlock(&info->lock);
453
454 /* The quota case may block */
455 if (freed > 0)
456 shmem_inode_unacct_blocks(inode, freed);
457 }
458
459 bool shmem_charge(struct inode *inode, long pages)
460 {
461 struct address_space *mapping = inode->i_mapping;
462
463 if (shmem_inode_acct_blocks(inode, pages))
464 return false;
465
466 /* nrpages adjustment first, then shmem_recalc_inode() when balanced */
467 xa_lock_irq(&mapping->i_pages);
468 mapping->nrpages += pages;
469 xa_unlock_irq(&mapping->i_pages);
470
471 shmem_recalc_inode(inode, pages, 0);
472 return true;
473 }
474
475 void shmem_uncharge(struct inode *inode, long pages)
476 {
477 /* pages argument is currently unused: keep it to help debugging */
478 /* nrpages adjustment done by __filemap_remove_folio() or caller */
479
480 shmem_recalc_inode(inode, 0, 0);
481 }
482
483 /*
484 * Replace item expected in xarray by a new item, while holding xa_lock.
485 */
486 static int shmem_replace_entry(struct address_space *mapping,
487 pgoff_t index, void *expected, void *replacement)
488 {
489 XA_STATE(xas, &mapping->i_pages, index);
490 void *item;
491
492 VM_BUG_ON(!expected);
493 VM_BUG_ON(!replacement);
494 item = xas_load(&xas);
495 if (item != expected)
496 return -ENOENT;
497 xas_store(&xas, replacement);
498 return 0;
499 }
500
501 /*
502 * Sometimes, before we decide whether to proceed or to fail, we must check
503 * that an entry was not already brought back from swap by a racing thread.
504 *
505 * Checking page is not enough: by the time a SwapCache page is locked, it
506 * might be reused, and again be SwapCache, using the same swap as before.
507 */
508 static bool shmem_confirm_swap(struct address_space *mapping,
509 pgoff_t index, swp_entry_t swap)
510 {
511 return xa_load(&mapping->i_pages, index) == swp_to_radix_entry(swap);
512 }
513
514 /*
515 * Definitions for "huge tmpfs": tmpfs mounted with the huge= option
516 *
517 * SHMEM_HUGE_NEVER:
518 * disables huge pages for the mount;
519 * SHMEM_HUGE_ALWAYS:
520 * enables huge pages for the mount;
521 * SHMEM_HUGE_WITHIN_SIZE:
522 * only allocate huge pages if the page will be fully within i_size,
523 * also respect fadvise()/madvise() hints;
524 * SHMEM_HUGE_ADVISE:
525 * only allocate huge pages if requested with fadvise()/madvise();
526 */
527
528 #define SHMEM_HUGE_NEVER 0
529 #define SHMEM_HUGE_ALWAYS 1
530 #define SHMEM_HUGE_WITHIN_SIZE 2
531 #define SHMEM_HUGE_ADVISE 3
532
533 /*
534 * Special values.
535 * Only can be set via /sys/kernel/mm/transparent_hugepage/shmem_enabled:
536 *
537 * SHMEM_HUGE_DENY:
538 * disables huge on shm_mnt and all mounts, for emergency use;
539 * SHMEM_HUGE_FORCE:
540 * enables huge on shm_mnt and all mounts, w/o needing option, for testing;
541 *
542 */
543 #define SHMEM_HUGE_DENY (-1)
544 #define SHMEM_HUGE_FORCE (-2)
545
546 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
547 /* ifdef here to avoid bloating shmem.o when not necessary */
548
549 static int shmem_huge __read_mostly = SHMEM_HUGE_NEVER;
550
551 static bool __shmem_is_huge(struct inode *inode, pgoff_t index,
552 bool shmem_huge_force, struct mm_struct *mm,
553 unsigned long vm_flags)
554 {
555 loff_t i_size;
556
557 if (!S_ISREG(inode->i_mode))
558 return false;
559 if (mm && ((vm_flags & VM_NOHUGEPAGE) || test_bit(MMF_DISABLE_THP, &mm->flags)))
560 return false;
561 if (shmem_huge == SHMEM_HUGE_DENY)
562 return false;
563 if (shmem_huge_force || shmem_huge == SHMEM_HUGE_FORCE)
564 return true;
565
566 switch (SHMEM_SB(inode->i_sb)->huge) {
567 case SHMEM_HUGE_ALWAYS:
568 return true;
569 case SHMEM_HUGE_WITHIN_SIZE:
570 index = round_up(index + 1, HPAGE_PMD_NR);
571 i_size = round_up(i_size_read(inode), PAGE_SIZE);
572 if (i_size >> PAGE_SHIFT >= index)
573 return true;
574 fallthrough;
575 case SHMEM_HUGE_ADVISE:
576 if (mm && (vm_flags & VM_HUGEPAGE))
577 return true;
578 fallthrough;
579 default:
580 return false;
581 }
582 }
583
584 bool shmem_is_huge(struct inode *inode, pgoff_t index,
585 bool shmem_huge_force, struct mm_struct *mm,
586 unsigned long vm_flags)
587 {
588 if (HPAGE_PMD_ORDER > MAX_PAGECACHE_ORDER)
589 return false;
590
591 return __shmem_is_huge(inode, index, shmem_huge_force, mm, vm_flags);
592 }
593
594 #if defined(CONFIG_SYSFS)
595 static int shmem_parse_huge(const char *str)
596 {
597 if (!strcmp(str, "never"))
598 return SHMEM_HUGE_NEVER;
599 if (!strcmp(str, "always"))
600 return SHMEM_HUGE_ALWAYS;
601 if (!strcmp(str, "within_size"))
602 return SHMEM_HUGE_WITHIN_SIZE;
603 if (!strcmp(str, "advise"))
604 return SHMEM_HUGE_ADVISE;
605 if (!strcmp(str, "deny"))
606 return SHMEM_HUGE_DENY;
607 if (!strcmp(str, "force"))
608 return SHMEM_HUGE_FORCE;
609 return -EINVAL;
610 }
611 #endif
612
613 #if defined(CONFIG_SYSFS) || defined(CONFIG_TMPFS)
614 static const char *shmem_format_huge(int huge)
615 {
616 switch (huge) {
617 case SHMEM_HUGE_NEVER:
618 return "never";
619 case SHMEM_HUGE_ALWAYS:
620 return "always";
621 case SHMEM_HUGE_WITHIN_SIZE:
622 return "within_size";
623 case SHMEM_HUGE_ADVISE:
624 return "advise";
625 case SHMEM_HUGE_DENY:
626 return "deny";
627 case SHMEM_HUGE_FORCE:
628 return "force";
629 default:
630 VM_BUG_ON(1);
631 return "bad_val";
632 }
633 }
634 #endif
635
636 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo,
637 struct shrink_control *sc, unsigned long nr_to_split)
638 {
639 LIST_HEAD(list), *pos, *next;
640 LIST_HEAD(to_remove);
641 struct inode *inode;
642 struct shmem_inode_info *info;
643 struct folio *folio;
644 unsigned long batch = sc ? sc->nr_to_scan : 128;
645 int split = 0;
646
647 if (list_empty(&sbinfo->shrinklist))
648 return SHRINK_STOP;
649
650 spin_lock(&sbinfo->shrinklist_lock);
651 list_for_each_safe(pos, next, &sbinfo->shrinklist) {
652 info = list_entry(pos, struct shmem_inode_info, shrinklist);
653
654 /* pin the inode */
655 inode = igrab(&info->vfs_inode);
656
657 /* inode is about to be evicted */
658 if (!inode) {
659 list_del_init(&info->shrinklist);
660 goto next;
661 }
662
663 /* Check if there's anything to gain */
664 if (round_up(inode->i_size, PAGE_SIZE) ==
665 round_up(inode->i_size, HPAGE_PMD_SIZE)) {
666 list_move(&info->shrinklist, &to_remove);
667 goto next;
668 }
669
670 list_move(&info->shrinklist, &list);
671 next:
672 sbinfo->shrinklist_len--;
673 if (!--batch)
674 break;
675 }
676 spin_unlock(&sbinfo->shrinklist_lock);
677
678 list_for_each_safe(pos, next, &to_remove) {
679 info = list_entry(pos, struct shmem_inode_info, shrinklist);
680 inode = &info->vfs_inode;
681 list_del_init(&info->shrinklist);
682 iput(inode);
683 }
684
685 list_for_each_safe(pos, next, &list) {
686 int ret;
687 pgoff_t index;
688
689 info = list_entry(pos, struct shmem_inode_info, shrinklist);
690 inode = &info->vfs_inode;
691
692 if (nr_to_split && split >= nr_to_split)
693 goto move_back;
694
695 index = (inode->i_size & HPAGE_PMD_MASK) >> PAGE_SHIFT;
696 folio = filemap_get_folio(inode->i_mapping, index);
697 if (IS_ERR(folio))
698 goto drop;
699
700 /* No huge page at the end of the file: nothing to split */
701 if (!folio_test_large(folio)) {
702 folio_put(folio);
703 goto drop;
704 }
705
706 /*
707 * Move the inode on the list back to shrinklist if we failed
708 * to lock the page at this time.
709 *
710 * Waiting for the lock may lead to deadlock in the
711 * reclaim path.
712 */
713 if (!folio_trylock(folio)) {
714 folio_put(folio);
715 goto move_back;
716 }
717
718 ret = split_folio(folio);
719 folio_unlock(folio);
720 folio_put(folio);
721
722 /* If split failed move the inode on the list back to shrinklist */
723 if (ret)
724 goto move_back;
725
726 split++;
727 drop:
728 list_del_init(&info->shrinklist);
729 goto put;
730 move_back:
731 /*
732 * Make sure the inode is either on the global list or deleted
733 * from any local list before iput() since it could be deleted
734 * in another thread once we put the inode (then the local list
735 * is corrupted).
736 */
737 spin_lock(&sbinfo->shrinklist_lock);
738 list_move(&info->shrinklist, &sbinfo->shrinklist);
739 sbinfo->shrinklist_len++;
740 spin_unlock(&sbinfo->shrinklist_lock);
741 put:
742 iput(inode);
743 }
744
745 return split;
746 }
747
748 static long shmem_unused_huge_scan(struct super_block *sb,
749 struct shrink_control *sc)
750 {
751 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
752
753 if (!READ_ONCE(sbinfo->shrinklist_len))
754 return SHRINK_STOP;
755
756 return shmem_unused_huge_shrink(sbinfo, sc, 0);
757 }
758
759 static long shmem_unused_huge_count(struct super_block *sb,
760 struct shrink_control *sc)
761 {
762 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
763 return READ_ONCE(sbinfo->shrinklist_len);
764 }
765 #else /* !CONFIG_TRANSPARENT_HUGEPAGE */
766
767 #define shmem_huge SHMEM_HUGE_DENY
768
769 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo,
770 struct shrink_control *sc, unsigned long nr_to_split)
771 {
772 return 0;
773 }
774 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
775
776 /*
777 * Somewhat like filemap_add_folio, but error if expected item has gone.
778 */
779 static int shmem_add_to_page_cache(struct folio *folio,
780 struct address_space *mapping,
781 pgoff_t index, void *expected, gfp_t gfp)
782 {
783 XA_STATE_ORDER(xas, &mapping->i_pages, index, folio_order(folio));
784 long nr = folio_nr_pages(folio);
785
786 VM_BUG_ON_FOLIO(index != round_down(index, nr), folio);
787 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
788 VM_BUG_ON_FOLIO(!folio_test_swapbacked(folio), folio);
789 VM_BUG_ON(expected && folio_test_large(folio));
790
791 folio_ref_add(folio, nr);
792 folio->mapping = mapping;
793 folio->index = index;
794
795 gfp &= GFP_RECLAIM_MASK;
796 folio_throttle_swaprate(folio, gfp);
797
798 do {
799 xas_lock_irq(&xas);
800 if (expected != xas_find_conflict(&xas)) {
801 xas_set_err(&xas, -EEXIST);
802 goto unlock;
803 }
804 if (expected && xas_find_conflict(&xas)) {
805 xas_set_err(&xas, -EEXIST);
806 goto unlock;
807 }
808 xas_store(&xas, folio);
809 if (xas_error(&xas))
810 goto unlock;
811 if (folio_test_pmd_mappable(folio))
812 __lruvec_stat_mod_folio(folio, NR_SHMEM_THPS, nr);
813 __lruvec_stat_mod_folio(folio, NR_FILE_PAGES, nr);
814 __lruvec_stat_mod_folio(folio, NR_SHMEM, nr);
815 mapping->nrpages += nr;
816 unlock:
817 xas_unlock_irq(&xas);
818 } while (xas_nomem(&xas, gfp));
819
820 if (xas_error(&xas)) {
821 folio->mapping = NULL;
822 folio_ref_sub(folio, nr);
823 return xas_error(&xas);
824 }
825
826 return 0;
827 }
828
829 /*
830 * Somewhat like filemap_remove_folio, but substitutes swap for @folio.
831 */
832 static void shmem_delete_from_page_cache(struct folio *folio, void *radswap)
833 {
834 struct address_space *mapping = folio->mapping;
835 long nr = folio_nr_pages(folio);
836 int error;
837
838 xa_lock_irq(&mapping->i_pages);
839 error = shmem_replace_entry(mapping, folio->index, folio, radswap);
840 folio->mapping = NULL;
841 mapping->nrpages -= nr;
842 __lruvec_stat_mod_folio(folio, NR_FILE_PAGES, -nr);
843 __lruvec_stat_mod_folio(folio, NR_SHMEM, -nr);
844 xa_unlock_irq(&mapping->i_pages);
845 folio_put(folio);
846 BUG_ON(error);
847 }
848
849 /*
850 * Remove swap entry from page cache, free the swap and its page cache.
851 */
852 static int shmem_free_swap(struct address_space *mapping,
853 pgoff_t index, void *radswap)
854 {
855 void *old;
856
857 old = xa_cmpxchg_irq(&mapping->i_pages, index, radswap, NULL, 0);
858 if (old != radswap)
859 return -ENOENT;
860 free_swap_and_cache(radix_to_swp_entry(radswap));
861 return 0;
862 }
863
864 /*
865 * Determine (in bytes) how many of the shmem object's pages mapped by the
866 * given offsets are swapped out.
867 *
868 * This is safe to call without i_rwsem or the i_pages lock thanks to RCU,
869 * as long as the inode doesn't go away and racy results are not a problem.
870 */
871 unsigned long shmem_partial_swap_usage(struct address_space *mapping,
872 pgoff_t start, pgoff_t end)
873 {
874 XA_STATE(xas, &mapping->i_pages, start);
875 struct page *page;
876 unsigned long swapped = 0;
877 unsigned long max = end - 1;
878
879 rcu_read_lock();
880 xas_for_each(&xas, page, max) {
881 if (xas_retry(&xas, page))
882 continue;
883 if (xa_is_value(page))
884 swapped++;
885 if (xas.xa_index == max)
886 break;
887 if (need_resched()) {
888 xas_pause(&xas);
889 cond_resched_rcu();
890 }
891 }
892 rcu_read_unlock();
893
894 return swapped << PAGE_SHIFT;
895 }
896
897 /*
898 * Determine (in bytes) how many of the shmem object's pages mapped by the
899 * given vma is swapped out.
900 *
901 * This is safe to call without i_rwsem or the i_pages lock thanks to RCU,
902 * as long as the inode doesn't go away and racy results are not a problem.
903 */
904 unsigned long shmem_swap_usage(struct vm_area_struct *vma)
905 {
906 struct inode *inode = file_inode(vma->vm_file);
907 struct shmem_inode_info *info = SHMEM_I(inode);
908 struct address_space *mapping = inode->i_mapping;
909 unsigned long swapped;
910
911 /* Be careful as we don't hold info->lock */
912 swapped = READ_ONCE(info->swapped);
913
914 /*
915 * The easier cases are when the shmem object has nothing in swap, or
916 * the vma maps it whole. Then we can simply use the stats that we
917 * already track.
918 */
919 if (!swapped)
920 return 0;
921
922 if (!vma->vm_pgoff && vma->vm_end - vma->vm_start >= inode->i_size)
923 return swapped << PAGE_SHIFT;
924
925 /* Here comes the more involved part */
926 return shmem_partial_swap_usage(mapping, vma->vm_pgoff,
927 vma->vm_pgoff + vma_pages(vma));
928 }
929
930 /*
931 * SysV IPC SHM_UNLOCK restore Unevictable pages to their evictable lists.
932 */
933 void shmem_unlock_mapping(struct address_space *mapping)
934 {
935 struct folio_batch fbatch;
936 pgoff_t index = 0;
937
938 folio_batch_init(&fbatch);
939 /*
940 * Minor point, but we might as well stop if someone else SHM_LOCKs it.
941 */
942 while (!mapping_unevictable(mapping) &&
943 filemap_get_folios(mapping, &index, ~0UL, &fbatch)) {
944 check_move_unevictable_folios(&fbatch);
945 folio_batch_release(&fbatch);
946 cond_resched();
947 }
948 }
949
950 static struct folio *shmem_get_partial_folio(struct inode *inode, pgoff_t index)
951 {
952 struct folio *folio;
953
954 /*
955 * At first avoid shmem_get_folio(,,,SGP_READ): that fails
956 * beyond i_size, and reports fallocated folios as holes.
957 */
958 folio = filemap_get_entry(inode->i_mapping, index);
959 if (!folio)
960 return folio;
961 if (!xa_is_value(folio)) {
962 folio_lock(folio);
963 if (folio->mapping == inode->i_mapping)
964 return folio;
965 /* The folio has been swapped out */
966 folio_unlock(folio);
967 folio_put(folio);
968 }
969 /*
970 * But read a folio back from swap if any of it is within i_size
971 * (although in some cases this is just a waste of time).
972 */
973 folio = NULL;
974 shmem_get_folio(inode, index, &folio, SGP_READ);
975 return folio;
976 }
977
978 /*
979 * Remove range of pages and swap entries from page cache, and free them.
980 * If !unfalloc, truncate or punch hole; if unfalloc, undo failed fallocate.
981 */
982 static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend,
983 bool unfalloc)
984 {
985 struct address_space *mapping = inode->i_mapping;
986 struct shmem_inode_info *info = SHMEM_I(inode);
987 pgoff_t start = (lstart + PAGE_SIZE - 1) >> PAGE_SHIFT;
988 pgoff_t end = (lend + 1) >> PAGE_SHIFT;
989 struct folio_batch fbatch;
990 pgoff_t indices[PAGEVEC_SIZE];
991 struct folio *folio;
992 bool same_folio;
993 long nr_swaps_freed = 0;
994 pgoff_t index;
995 int i;
996
997 if (lend == -1)
998 end = -1; /* unsigned, so actually very big */
999
1000 if (info->fallocend > start && info->fallocend <= end && !unfalloc)
1001 info->fallocend = start;
1002
1003 folio_batch_init(&fbatch);
1004 index = start;
1005 while (index < end && find_lock_entries(mapping, &index, end - 1,
1006 &fbatch, indices)) {
1007 for (i = 0; i < folio_batch_count(&fbatch); i++) {
1008 folio = fbatch.folios[i];
1009
1010 if (xa_is_value(folio)) {
1011 if (unfalloc)
1012 continue;
1013 nr_swaps_freed += !shmem_free_swap(mapping,
1014 indices[i], folio);
1015 continue;
1016 }
1017
1018 if (!unfalloc || !folio_test_uptodate(folio))
1019 truncate_inode_folio(mapping, folio);
1020 folio_unlock(folio);
1021 }
1022 folio_batch_remove_exceptionals(&fbatch);
1023 folio_batch_release(&fbatch);
1024 cond_resched();
1025 }
1026
1027 /*
1028 * When undoing a failed fallocate, we want none of the partial folio
1029 * zeroing and splitting below, but shall want to truncate the whole
1030 * folio when !uptodate indicates that it was added by this fallocate,
1031 * even when [lstart, lend] covers only a part of the folio.
1032 */
1033 if (unfalloc)
1034 goto whole_folios;
1035
1036 same_folio = (lstart >> PAGE_SHIFT) == (lend >> PAGE_SHIFT);
1037 folio = shmem_get_partial_folio(inode, lstart >> PAGE_SHIFT);
1038 if (folio) {
1039 same_folio = lend < folio_pos(folio) + folio_size(folio);
1040 folio_mark_dirty(folio);
1041 if (!truncate_inode_partial_folio(folio, lstart, lend)) {
1042 start = folio_next_index(folio);
1043 if (same_folio)
1044 end = folio->index;
1045 }
1046 folio_unlock(folio);
1047 folio_put(folio);
1048 folio = NULL;
1049 }
1050
1051 if (!same_folio)
1052 folio = shmem_get_partial_folio(inode, lend >> PAGE_SHIFT);
1053 if (folio) {
1054 folio_mark_dirty(folio);
1055 if (!truncate_inode_partial_folio(folio, lstart, lend))
1056 end = folio->index;
1057 folio_unlock(folio);
1058 folio_put(folio);
1059 }
1060
1061 whole_folios:
1062
1063 index = start;
1064 while (index < end) {
1065 cond_resched();
1066
1067 if (!find_get_entries(mapping, &index, end - 1, &fbatch,
1068 indices)) {
1069 /* If all gone or hole-punch or unfalloc, we're done */
1070 if (index == start || end != -1)
1071 break;
1072 /* But if truncating, restart to make sure all gone */
1073 index = start;
1074 continue;
1075 }
1076 for (i = 0; i < folio_batch_count(&fbatch); i++) {
1077 folio = fbatch.folios[i];
1078
1079 if (xa_is_value(folio)) {
1080 if (unfalloc)
1081 continue;
1082 if (shmem_free_swap(mapping, indices[i], folio)) {
1083 /* Swap was replaced by page: retry */
1084 index = indices[i];
1085 break;
1086 }
1087 nr_swaps_freed++;
1088 continue;
1089 }
1090
1091 folio_lock(folio);
1092
1093 if (!unfalloc || !folio_test_uptodate(folio)) {
1094 if (folio_mapping(folio) != mapping) {
1095 /* Page was replaced by swap: retry */
1096 folio_unlock(folio);
1097 index = indices[i];
1098 break;
1099 }
1100 VM_BUG_ON_FOLIO(folio_test_writeback(folio),
1101 folio);
1102
1103 if (!folio_test_large(folio)) {
1104 truncate_inode_folio(mapping, folio);
1105 } else if (truncate_inode_partial_folio(folio, lstart, lend)) {
1106 /*
1107 * If we split a page, reset the loop so
1108 * that we pick up the new sub pages.
1109 * Otherwise the THP was entirely
1110 * dropped or the target range was
1111 * zeroed, so just continue the loop as
1112 * is.
1113 */
1114 if (!folio_test_large(folio)) {
1115 folio_unlock(folio);
1116 index = start;
1117 break;
1118 }
1119 }
1120 }
1121 folio_unlock(folio);
1122 }
1123 folio_batch_remove_exceptionals(&fbatch);
1124 folio_batch_release(&fbatch);
1125 }
1126
1127 shmem_recalc_inode(inode, 0, -nr_swaps_freed);
1128 }
1129
1130 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
1131 {
1132 shmem_undo_range(inode, lstart, lend, false);
1133 inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode));
1134 inode_inc_iversion(inode);
1135 }
1136 EXPORT_SYMBOL_GPL(shmem_truncate_range);
1137
1138 static int shmem_getattr(struct mnt_idmap *idmap,
1139 const struct path *path, struct kstat *stat,
1140 u32 request_mask, unsigned int query_flags)
1141 {
1142 struct inode *inode = path->dentry->d_inode;
1143 struct shmem_inode_info *info = SHMEM_I(inode);
1144
1145 if (info->alloced - info->swapped != inode->i_mapping->nrpages)
1146 shmem_recalc_inode(inode, 0, 0);
1147
1148 if (info->fsflags & FS_APPEND_FL)
1149 stat->attributes |= STATX_ATTR_APPEND;
1150 if (info->fsflags & FS_IMMUTABLE_FL)
1151 stat->attributes |= STATX_ATTR_IMMUTABLE;
1152 if (info->fsflags & FS_NODUMP_FL)
1153 stat->attributes |= STATX_ATTR_NODUMP;
1154 stat->attributes_mask |= (STATX_ATTR_APPEND |
1155 STATX_ATTR_IMMUTABLE |
1156 STATX_ATTR_NODUMP);
1157 generic_fillattr(idmap, request_mask, inode, stat);
1158
1159 if (shmem_is_huge(inode, 0, false, NULL, 0))
1160 stat->blksize = HPAGE_PMD_SIZE;
1161
1162 if (request_mask & STATX_BTIME) {
1163 stat->result_mask |= STATX_BTIME;
1164 stat->btime.tv_sec = info->i_crtime.tv_sec;
1165 stat->btime.tv_nsec = info->i_crtime.tv_nsec;
1166 }
1167
1168 return 0;
1169 }
1170
1171 static int shmem_setattr(struct mnt_idmap *idmap,
1172 struct dentry *dentry, struct iattr *attr)
1173 {
1174 struct inode *inode = d_inode(dentry);
1175 struct shmem_inode_info *info = SHMEM_I(inode);
1176 int error;
1177 bool update_mtime = false;
1178 bool update_ctime = true;
1179
1180 error = setattr_prepare(idmap, dentry, attr);
1181 if (error)
1182 return error;
1183
1184 if ((info->seals & F_SEAL_EXEC) && (attr->ia_valid & ATTR_MODE)) {
1185 if ((inode->i_mode ^ attr->ia_mode) & 0111) {
1186 return -EPERM;
1187 }
1188 }
1189
1190 if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) {
1191 loff_t oldsize = inode->i_size;
1192 loff_t newsize = attr->ia_size;
1193
1194 /* protected by i_rwsem */
1195 if ((newsize < oldsize && (info->seals & F_SEAL_SHRINK)) ||
1196 (newsize > oldsize && (info->seals & F_SEAL_GROW)))
1197 return -EPERM;
1198
1199 if (newsize != oldsize) {
1200 error = shmem_reacct_size(SHMEM_I(inode)->flags,
1201 oldsize, newsize);
1202 if (error)
1203 return error;
1204 i_size_write(inode, newsize);
1205 update_mtime = true;
1206 } else {
1207 update_ctime = false;
1208 }
1209 if (newsize <= oldsize) {
1210 loff_t holebegin = round_up(newsize, PAGE_SIZE);
1211 if (oldsize > holebegin)
1212 unmap_mapping_range(inode->i_mapping,
1213 holebegin, 0, 1);
1214 if (info->alloced)
1215 shmem_truncate_range(inode,
1216 newsize, (loff_t)-1);
1217 /* unmap again to remove racily COWed private pages */
1218 if (oldsize > holebegin)
1219 unmap_mapping_range(inode->i_mapping,
1220 holebegin, 0, 1);
1221 }
1222 }
1223
1224 if (is_quota_modification(idmap, inode, attr)) {
1225 error = dquot_initialize(inode);
1226 if (error)
1227 return error;
1228 }
1229
1230 /* Transfer quota accounting */
1231 if (i_uid_needs_update(idmap, attr, inode) ||
1232 i_gid_needs_update(idmap, attr, inode)) {
1233 error = dquot_transfer(idmap, inode, attr);
1234 if (error)
1235 return error;
1236 }
1237
1238 setattr_copy(idmap, inode, attr);
1239 if (attr->ia_valid & ATTR_MODE)
1240 error = posix_acl_chmod(idmap, dentry, inode->i_mode);
1241 if (!error && update_ctime) {
1242 inode_set_ctime_current(inode);
1243 if (update_mtime)
1244 inode_set_mtime_to_ts(inode, inode_get_ctime(inode));
1245 inode_inc_iversion(inode);
1246 }
1247 return error;
1248 }
1249
1250 static void shmem_evict_inode(struct inode *inode)
1251 {
1252 struct shmem_inode_info *info = SHMEM_I(inode);
1253 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
1254 size_t freed = 0;
1255
1256 if (shmem_mapping(inode->i_mapping)) {
1257 shmem_unacct_size(info->flags, inode->i_size);
1258 inode->i_size = 0;
1259 mapping_set_exiting(inode->i_mapping);
1260 shmem_truncate_range(inode, 0, (loff_t)-1);
1261 if (!list_empty(&info->shrinklist)) {
1262 spin_lock(&sbinfo->shrinklist_lock);
1263 if (!list_empty(&info->shrinklist)) {
1264 list_del_init(&info->shrinklist);
1265 sbinfo->shrinklist_len--;
1266 }
1267 spin_unlock(&sbinfo->shrinklist_lock);
1268 }
1269 while (!list_empty(&info->swaplist)) {
1270 /* Wait while shmem_unuse() is scanning this inode... */
1271 wait_var_event(&info->stop_eviction,
1272 !atomic_read(&info->stop_eviction));
1273 mutex_lock(&shmem_swaplist_mutex);
1274 /* ...but beware of the race if we peeked too early */
1275 if (!atomic_read(&info->stop_eviction))
1276 list_del_init(&info->swaplist);
1277 mutex_unlock(&shmem_swaplist_mutex);
1278 }
1279 }
1280
1281 simple_xattrs_free(&info->xattrs, sbinfo->max_inodes ? &freed : NULL);
1282 shmem_free_inode(inode->i_sb, freed);
1283 WARN_ON(inode->i_blocks);
1284 clear_inode(inode);
1285 #ifdef CONFIG_TMPFS_QUOTA
1286 dquot_free_inode(inode);
1287 dquot_drop(inode);
1288 #endif
1289 }
1290
1291 static int shmem_find_swap_entries(struct address_space *mapping,
1292 pgoff_t start, struct folio_batch *fbatch,
1293 pgoff_t *indices, unsigned int type)
1294 {
1295 XA_STATE(xas, &mapping->i_pages, start);
1296 struct folio *folio;
1297 swp_entry_t entry;
1298
1299 rcu_read_lock();
1300 xas_for_each(&xas, folio, ULONG_MAX) {
1301 if (xas_retry(&xas, folio))
1302 continue;
1303
1304 if (!xa_is_value(folio))
1305 continue;
1306
1307 entry = radix_to_swp_entry(folio);
1308 /*
1309 * swapin error entries can be found in the mapping. But they're
1310 * deliberately ignored here as we've done everything we can do.
1311 */
1312 if (swp_type(entry) != type)
1313 continue;
1314
1315 indices[folio_batch_count(fbatch)] = xas.xa_index;
1316 if (!folio_batch_add(fbatch, folio))
1317 break;
1318
1319 if (need_resched()) {
1320 xas_pause(&xas);
1321 cond_resched_rcu();
1322 }
1323 }
1324 rcu_read_unlock();
1325
1326 return xas.xa_index;
1327 }
1328
1329 /*
1330 * Move the swapped pages for an inode to page cache. Returns the count
1331 * of pages swapped in, or the error in case of failure.
1332 */
1333 static int shmem_unuse_swap_entries(struct inode *inode,
1334 struct folio_batch *fbatch, pgoff_t *indices)
1335 {
1336 int i = 0;
1337 int ret = 0;
1338 int error = 0;
1339 struct address_space *mapping = inode->i_mapping;
1340
1341 for (i = 0; i < folio_batch_count(fbatch); i++) {
1342 struct folio *folio = fbatch->folios[i];
1343
1344 if (!xa_is_value(folio))
1345 continue;
1346 error = shmem_swapin_folio(inode, indices[i], &folio, SGP_CACHE,
1347 mapping_gfp_mask(mapping), NULL, NULL);
1348 if (error == 0) {
1349 folio_unlock(folio);
1350 folio_put(folio);
1351 ret++;
1352 }
1353 if (error == -ENOMEM)
1354 break;
1355 error = 0;
1356 }
1357 return error ? error : ret;
1358 }
1359
1360 /*
1361 * If swap found in inode, free it and move page from swapcache to filecache.
1362 */
1363 static int shmem_unuse_inode(struct inode *inode, unsigned int type)
1364 {
1365 struct address_space *mapping = inode->i_mapping;
1366 pgoff_t start = 0;
1367 struct folio_batch fbatch;
1368 pgoff_t indices[PAGEVEC_SIZE];
1369 int ret = 0;
1370
1371 do {
1372 folio_batch_init(&fbatch);
1373 shmem_find_swap_entries(mapping, start, &fbatch, indices, type);
1374 if (folio_batch_count(&fbatch) == 0) {
1375 ret = 0;
1376 break;
1377 }
1378
1379 ret = shmem_unuse_swap_entries(inode, &fbatch, indices);
1380 if (ret < 0)
1381 break;
1382
1383 start = indices[folio_batch_count(&fbatch) - 1];
1384 } while (true);
1385
1386 return ret;
1387 }
1388
1389 /*
1390 * Read all the shared memory data that resides in the swap
1391 * device 'type' back into memory, so the swap device can be
1392 * unused.
1393 */
1394 int shmem_unuse(unsigned int type)
1395 {
1396 struct shmem_inode_info *info, *next;
1397 int error = 0;
1398
1399 if (list_empty(&shmem_swaplist))
1400 return 0;
1401
1402 mutex_lock(&shmem_swaplist_mutex);
1403 list_for_each_entry_safe(info, next, &shmem_swaplist, swaplist) {
1404 if (!info->swapped) {
1405 list_del_init(&info->swaplist);
1406 continue;
1407 }
1408 /*
1409 * Drop the swaplist mutex while searching the inode for swap;
1410 * but before doing so, make sure shmem_evict_inode() will not
1411 * remove placeholder inode from swaplist, nor let it be freed
1412 * (igrab() would protect from unlink, but not from unmount).
1413 */
1414 atomic_inc(&info->stop_eviction);
1415 mutex_unlock(&shmem_swaplist_mutex);
1416
1417 error = shmem_unuse_inode(&info->vfs_inode, type);
1418 cond_resched();
1419
1420 mutex_lock(&shmem_swaplist_mutex);
1421 next = list_next_entry(info, swaplist);
1422 if (!info->swapped)
1423 list_del_init(&info->swaplist);
1424 if (atomic_dec_and_test(&info->stop_eviction))
1425 wake_up_var(&info->stop_eviction);
1426 if (error)
1427 break;
1428 }
1429 mutex_unlock(&shmem_swaplist_mutex);
1430
1431 return error;
1432 }
1433
1434 /*
1435 * Move the page from the page cache to the swap cache.
1436 */
1437 static int shmem_writepage(struct page *page, struct writeback_control *wbc)
1438 {
1439 struct folio *folio = page_folio(page);
1440 struct address_space *mapping = folio->mapping;
1441 struct inode *inode = mapping->host;
1442 struct shmem_inode_info *info = SHMEM_I(inode);
1443 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
1444 swp_entry_t swap;
1445 pgoff_t index;
1446
1447 /*
1448 * Our capabilities prevent regular writeback or sync from ever calling
1449 * shmem_writepage; but a stacking filesystem might use ->writepage of
1450 * its underlying filesystem, in which case tmpfs should write out to
1451 * swap only in response to memory pressure, and not for the writeback
1452 * threads or sync.
1453 */
1454 if (WARN_ON_ONCE(!wbc->for_reclaim))
1455 goto redirty;
1456
1457 if (WARN_ON_ONCE((info->flags & VM_LOCKED) || sbinfo->noswap))
1458 goto redirty;
1459
1460 if (!total_swap_pages)
1461 goto redirty;
1462
1463 /*
1464 * If /sys/kernel/mm/transparent_hugepage/shmem_enabled is "always" or
1465 * "force", drivers/gpu/drm/i915/gem/i915_gem_shmem.c gets huge pages,
1466 * and its shmem_writeback() needs them to be split when swapping.
1467 */
1468 if (folio_test_large(folio)) {
1469 /* Ensure the subpages are still dirty */
1470 folio_test_set_dirty(folio);
1471 if (split_huge_page(page) < 0)
1472 goto redirty;
1473 folio = page_folio(page);
1474 folio_clear_dirty(folio);
1475 }
1476
1477 index = folio->index;
1478
1479 /*
1480 * This is somewhat ridiculous, but without plumbing a SWAP_MAP_FALLOC
1481 * value into swapfile.c, the only way we can correctly account for a
1482 * fallocated folio arriving here is now to initialize it and write it.
1483 *
1484 * That's okay for a folio already fallocated earlier, but if we have
1485 * not yet completed the fallocation, then (a) we want to keep track
1486 * of this folio in case we have to undo it, and (b) it may not be a
1487 * good idea to continue anyway, once we're pushing into swap. So
1488 * reactivate the folio, and let shmem_fallocate() quit when too many.
1489 */
1490 if (!folio_test_uptodate(folio)) {
1491 if (inode->i_private) {
1492 struct shmem_falloc *shmem_falloc;
1493 spin_lock(&inode->i_lock);
1494 shmem_falloc = inode->i_private;
1495 if (shmem_falloc &&
1496 !shmem_falloc->waitq &&
1497 index >= shmem_falloc->start &&
1498 index < shmem_falloc->next)
1499 shmem_falloc->nr_unswapped++;
1500 else
1501 shmem_falloc = NULL;
1502 spin_unlock(&inode->i_lock);
1503 if (shmem_falloc)
1504 goto redirty;
1505 }
1506 folio_zero_range(folio, 0, folio_size(folio));
1507 flush_dcache_folio(folio);
1508 folio_mark_uptodate(folio);
1509 }
1510
1511 swap = folio_alloc_swap(folio);
1512 if (!swap.val)
1513 goto redirty;
1514
1515 /*
1516 * Add inode to shmem_unuse()'s list of swapped-out inodes,
1517 * if it's not already there. Do it now before the folio is
1518 * moved to swap cache, when its pagelock no longer protects
1519 * the inode from eviction. But don't unlock the mutex until
1520 * we've incremented swapped, because shmem_unuse_inode() will
1521 * prune a !swapped inode from the swaplist under this mutex.
1522 */
1523 mutex_lock(&shmem_swaplist_mutex);
1524 if (list_empty(&info->swaplist))
1525 list_add(&info->swaplist, &shmem_swaplist);
1526
1527 if (add_to_swap_cache(folio, swap,
1528 __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN,
1529 NULL) == 0) {
1530 shmem_recalc_inode(inode, 0, 1);
1531 swap_shmem_alloc(swap);
1532 shmem_delete_from_page_cache(folio, swp_to_radix_entry(swap));
1533
1534 mutex_unlock(&shmem_swaplist_mutex);
1535 BUG_ON(folio_mapped(folio));
1536 return swap_writepage(&folio->page, wbc);
1537 }
1538
1539 mutex_unlock(&shmem_swaplist_mutex);
1540 put_swap_folio(folio, swap);
1541 redirty:
1542 folio_mark_dirty(folio);
1543 if (wbc->for_reclaim)
1544 return AOP_WRITEPAGE_ACTIVATE; /* Return with folio locked */
1545 folio_unlock(folio);
1546 return 0;
1547 }
1548
1549 #if defined(CONFIG_NUMA) && defined(CONFIG_TMPFS)
1550 static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
1551 {
1552 char buffer[64];
1553
1554 if (!mpol || mpol->mode == MPOL_DEFAULT)
1555 return; /* show nothing */
1556
1557 mpol_to_str(buffer, sizeof(buffer), mpol);
1558
1559 seq_printf(seq, ",mpol=%s", buffer);
1560 }
1561
1562 static struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
1563 {
1564 struct mempolicy *mpol = NULL;
1565 if (sbinfo->mpol) {
1566 raw_spin_lock(&sbinfo->stat_lock); /* prevent replace/use races */
1567 mpol = sbinfo->mpol;
1568 mpol_get(mpol);
1569 raw_spin_unlock(&sbinfo->stat_lock);
1570 }
1571 return mpol;
1572 }
1573 #else /* !CONFIG_NUMA || !CONFIG_TMPFS */
1574 static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
1575 {
1576 }
1577 static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
1578 {
1579 return NULL;
1580 }
1581 #endif /* CONFIG_NUMA && CONFIG_TMPFS */
1582
1583 static struct mempolicy *shmem_get_pgoff_policy(struct shmem_inode_info *info,
1584 pgoff_t index, unsigned int order, pgoff_t *ilx);
1585
1586 static struct folio *shmem_swapin_cluster(swp_entry_t swap, gfp_t gfp,
1587 struct shmem_inode_info *info, pgoff_t index)
1588 {
1589 struct mempolicy *mpol;
1590 pgoff_t ilx;
1591 struct folio *folio;
1592
1593 mpol = shmem_get_pgoff_policy(info, index, 0, &ilx);
1594 folio = swap_cluster_readahead(swap, gfp, mpol, ilx);
1595 mpol_cond_put(mpol);
1596
1597 return folio;
1598 }
1599
1600 /*
1601 * Make sure huge_gfp is always more limited than limit_gfp.
1602 * Some of the flags set permissions, while others set limitations.
1603 */
1604 static gfp_t limit_gfp_mask(gfp_t huge_gfp, gfp_t limit_gfp)
1605 {
1606 gfp_t allowflags = __GFP_IO | __GFP_FS | __GFP_RECLAIM;
1607 gfp_t denyflags = __GFP_NOWARN | __GFP_NORETRY;
1608 gfp_t zoneflags = limit_gfp & GFP_ZONEMASK;
1609 gfp_t result = huge_gfp & ~(allowflags | GFP_ZONEMASK);
1610
1611 /* Allow allocations only from the originally specified zones. */
1612 result |= zoneflags;
1613
1614 /*
1615 * Minimize the result gfp by taking the union with the deny flags,
1616 * and the intersection of the allow flags.
1617 */
1618 result |= (limit_gfp & denyflags);
1619 result |= (huge_gfp & limit_gfp) & allowflags;
1620
1621 return result;
1622 }
1623
1624 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1625 unsigned long shmem_allowable_huge_orders(struct inode *inode,
1626 struct vm_area_struct *vma, pgoff_t index,
1627 bool global_huge)
1628 {
1629 unsigned long mask = READ_ONCE(huge_shmem_orders_always);
1630 unsigned long within_size_orders = READ_ONCE(huge_shmem_orders_within_size);
1631 unsigned long vm_flags = vma->vm_flags;
1632 loff_t i_size;
1633 int order;
1634
1635 if ((vm_flags & VM_NOHUGEPAGE) ||
1636 test_bit(MMF_DISABLE_THP, &vma->vm_mm->flags))
1637 return 0;
1638
1639 /* If the hardware/firmware marked hugepage support disabled. */
1640 if (transparent_hugepage_flags & (1 << TRANSPARENT_HUGEPAGE_UNSUPPORTED))
1641 return 0;
1642
1643 /*
1644 * Following the 'deny' semantics of the top level, force the huge
1645 * option off from all mounts.
1646 */
1647 if (shmem_huge == SHMEM_HUGE_DENY)
1648 return 0;
1649
1650 /*
1651 * Only allow inherit orders if the top-level value is 'force', which
1652 * means non-PMD sized THP can not override 'huge' mount option now.
1653 */
1654 if (shmem_huge == SHMEM_HUGE_FORCE)
1655 return READ_ONCE(huge_shmem_orders_inherit);
1656
1657 /* Allow mTHP that will be fully within i_size. */
1658 order = highest_order(within_size_orders);
1659 while (within_size_orders) {
1660 index = round_up(index + 1, order);
1661 i_size = round_up(i_size_read(inode), PAGE_SIZE);
1662 if (i_size >> PAGE_SHIFT >= index) {
1663 mask |= within_size_orders;
1664 break;
1665 }
1666
1667 order = next_order(&within_size_orders, order);
1668 }
1669
1670 if (vm_flags & VM_HUGEPAGE)
1671 mask |= READ_ONCE(huge_shmem_orders_madvise);
1672
1673 if (global_huge)
1674 mask |= READ_ONCE(huge_shmem_orders_inherit);
1675
1676 return THP_ORDERS_ALL_FILE_DEFAULT & mask;
1677 }
1678
1679 static unsigned long shmem_suitable_orders(struct inode *inode, struct vm_fault *vmf,
1680 struct address_space *mapping, pgoff_t index,
1681 unsigned long orders)
1682 {
1683 struct vm_area_struct *vma = vmf->vma;
1684 pgoff_t aligned_index;
1685 unsigned long pages;
1686 int order;
1687
1688 orders = thp_vma_suitable_orders(vma, vmf->address, orders);
1689 if (!orders)
1690 return 0;
1691
1692 /* Find the highest order that can add into the page cache */
1693 order = highest_order(orders);
1694 while (orders) {
1695 pages = 1UL << order;
1696 aligned_index = round_down(index, pages);
1697 if (!xa_find(&mapping->i_pages, &aligned_index,
1698 aligned_index + pages - 1, XA_PRESENT))
1699 break;
1700 order = next_order(&orders, order);
1701 }
1702
1703 return orders;
1704 }
1705 #else
1706 static unsigned long shmem_suitable_orders(struct inode *inode, struct vm_fault *vmf,
1707 struct address_space *mapping, pgoff_t index,
1708 unsigned long orders)
1709 {
1710 return 0;
1711 }
1712 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
1713
1714 static struct folio *shmem_alloc_folio(gfp_t gfp, int order,
1715 struct shmem_inode_info *info, pgoff_t index)
1716 {
1717 struct mempolicy *mpol;
1718 pgoff_t ilx;
1719 struct folio *folio;
1720
1721 mpol = shmem_get_pgoff_policy(info, index, order, &ilx);
1722 folio = folio_alloc_mpol(gfp, order, mpol, ilx, numa_node_id());
1723 mpol_cond_put(mpol);
1724
1725 return folio;
1726 }
1727
1728 static struct folio *shmem_alloc_and_add_folio(struct vm_fault *vmf,
1729 gfp_t gfp, struct inode *inode, pgoff_t index,
1730 struct mm_struct *fault_mm, unsigned long orders)
1731 {
1732 struct address_space *mapping = inode->i_mapping;
1733 struct shmem_inode_info *info = SHMEM_I(inode);
1734 struct vm_area_struct *vma = vmf ? vmf->vma : NULL;
1735 unsigned long suitable_orders = 0;
1736 struct folio *folio = NULL;
1737 long pages;
1738 int error, order;
1739
1740 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
1741 orders = 0;
1742
1743 if (orders > 0) {
1744 if (vma && vma_is_anon_shmem(vma)) {
1745 suitable_orders = shmem_suitable_orders(inode, vmf,
1746 mapping, index, orders);
1747 } else if (orders & BIT(HPAGE_PMD_ORDER)) {
1748 pages = HPAGE_PMD_NR;
1749 suitable_orders = BIT(HPAGE_PMD_ORDER);
1750 index = round_down(index, HPAGE_PMD_NR);
1751
1752 /*
1753 * Check for conflict before waiting on a huge allocation.
1754 * Conflict might be that a huge page has just been allocated
1755 * and added to page cache by a racing thread, or that there
1756 * is already at least one small page in the huge extent.
1757 * Be careful to retry when appropriate, but not forever!
1758 * Elsewhere -EEXIST would be the right code, but not here.
1759 */
1760 if (xa_find(&mapping->i_pages, &index,
1761 index + HPAGE_PMD_NR - 1, XA_PRESENT))
1762 return ERR_PTR(-E2BIG);
1763 }
1764
1765 order = highest_order(suitable_orders);
1766 while (suitable_orders) {
1767 pages = 1UL << order;
1768 index = round_down(index, pages);
1769 folio = shmem_alloc_folio(gfp, order, info, index);
1770 if (folio)
1771 goto allocated;
1772
1773 if (pages == HPAGE_PMD_NR)
1774 count_vm_event(THP_FILE_FALLBACK);
1775 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1776 count_mthp_stat(order, MTHP_STAT_SHMEM_FALLBACK);
1777 #endif
1778 order = next_order(&suitable_orders, order);
1779 }
1780 } else {
1781 pages = 1;
1782 folio = shmem_alloc_folio(gfp, 0, info, index);
1783 }
1784 if (!folio)
1785 return ERR_PTR(-ENOMEM);
1786
1787 allocated:
1788 __folio_set_locked(folio);
1789 __folio_set_swapbacked(folio);
1790
1791 gfp &= GFP_RECLAIM_MASK;
1792 error = mem_cgroup_charge(folio, fault_mm, gfp);
1793 if (error) {
1794 if (xa_find(&mapping->i_pages, &index,
1795 index + pages - 1, XA_PRESENT)) {
1796 error = -EEXIST;
1797 } else if (pages > 1) {
1798 if (pages == HPAGE_PMD_NR) {
1799 count_vm_event(THP_FILE_FALLBACK);
1800 count_vm_event(THP_FILE_FALLBACK_CHARGE);
1801 }
1802 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1803 count_mthp_stat(folio_order(folio), MTHP_STAT_SHMEM_FALLBACK);
1804 count_mthp_stat(folio_order(folio), MTHP_STAT_SHMEM_FALLBACK_CHARGE);
1805 #endif
1806 }
1807 goto unlock;
1808 }
1809
1810 error = shmem_add_to_page_cache(folio, mapping, index, NULL, gfp);
1811 if (error)
1812 goto unlock;
1813
1814 error = shmem_inode_acct_blocks(inode, pages);
1815 if (error) {
1816 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
1817 long freed;
1818 /*
1819 * Try to reclaim some space by splitting a few
1820 * large folios beyond i_size on the filesystem.
1821 */
1822 shmem_unused_huge_shrink(sbinfo, NULL, 2);
1823 /*
1824 * And do a shmem_recalc_inode() to account for freed pages:
1825 * except our folio is there in cache, so not quite balanced.
1826 */
1827 spin_lock(&info->lock);
1828 freed = pages + info->alloced - info->swapped -
1829 READ_ONCE(mapping->nrpages);
1830 if (freed > 0)
1831 info->alloced -= freed;
1832 spin_unlock(&info->lock);
1833 if (freed > 0)
1834 shmem_inode_unacct_blocks(inode, freed);
1835 error = shmem_inode_acct_blocks(inode, pages);
1836 if (error) {
1837 filemap_remove_folio(folio);
1838 goto unlock;
1839 }
1840 }
1841
1842 shmem_recalc_inode(inode, pages, 0);
1843 folio_add_lru(folio);
1844 return folio;
1845
1846 unlock:
1847 folio_unlock(folio);
1848 folio_put(folio);
1849 return ERR_PTR(error);
1850 }
1851
1852 /*
1853 * When a page is moved from swapcache to shmem filecache (either by the
1854 * usual swapin of shmem_get_folio_gfp(), or by the less common swapoff of
1855 * shmem_unuse_inode()), it may have been read in earlier from swap, in
1856 * ignorance of the mapping it belongs to. If that mapping has special
1857 * constraints (like the gma500 GEM driver, which requires RAM below 4GB),
1858 * we may need to copy to a suitable page before moving to filecache.
1859 *
1860 * In a future release, this may well be extended to respect cpuset and
1861 * NUMA mempolicy, and applied also to anonymous pages in do_swap_page();
1862 * but for now it is a simple matter of zone.
1863 */
1864 static bool shmem_should_replace_folio(struct folio *folio, gfp_t gfp)
1865 {
1866 return folio_zonenum(folio) > gfp_zone(gfp);
1867 }
1868
1869 static int shmem_replace_folio(struct folio **foliop, gfp_t gfp,
1870 struct shmem_inode_info *info, pgoff_t index)
1871 {
1872 struct folio *old, *new;
1873 struct address_space *swap_mapping;
1874 swp_entry_t entry;
1875 pgoff_t swap_index;
1876 int error;
1877
1878 old = *foliop;
1879 entry = old->swap;
1880 swap_index = swap_cache_index(entry);
1881 swap_mapping = swap_address_space(entry);
1882
1883 /*
1884 * We have arrived here because our zones are constrained, so don't
1885 * limit chance of success by further cpuset and node constraints.
1886 */
1887 gfp &= ~GFP_CONSTRAINT_MASK;
1888 VM_BUG_ON_FOLIO(folio_test_large(old), old);
1889 new = shmem_alloc_folio(gfp, 0, info, index);
1890 if (!new)
1891 return -ENOMEM;
1892
1893 folio_get(new);
1894 folio_copy(new, old);
1895 flush_dcache_folio(new);
1896
1897 __folio_set_locked(new);
1898 __folio_set_swapbacked(new);
1899 folio_mark_uptodate(new);
1900 new->swap = entry;
1901 folio_set_swapcache(new);
1902
1903 /*
1904 * Our caller will very soon move newpage out of swapcache, but it's
1905 * a nice clean interface for us to replace oldpage by newpage there.
1906 */
1907 xa_lock_irq(&swap_mapping->i_pages);
1908 error = shmem_replace_entry(swap_mapping, swap_index, old, new);
1909 if (!error) {
1910 mem_cgroup_replace_folio(old, new);
1911 __lruvec_stat_mod_folio(new, NR_FILE_PAGES, 1);
1912 __lruvec_stat_mod_folio(new, NR_SHMEM, 1);
1913 __lruvec_stat_mod_folio(old, NR_FILE_PAGES, -1);
1914 __lruvec_stat_mod_folio(old, NR_SHMEM, -1);
1915 }
1916 xa_unlock_irq(&swap_mapping->i_pages);
1917
1918 if (unlikely(error)) {
1919 /*
1920 * Is this possible? I think not, now that our callers check
1921 * both PageSwapCache and page_private after getting page lock;
1922 * but be defensive. Reverse old to newpage for clear and free.
1923 */
1924 old = new;
1925 } else {
1926 folio_add_lru(new);
1927 *foliop = new;
1928 }
1929
1930 folio_clear_swapcache(old);
1931 old->private = NULL;
1932
1933 folio_unlock(old);
1934 folio_put_refs(old, 2);
1935 return error;
1936 }
1937
1938 static void shmem_set_folio_swapin_error(struct inode *inode, pgoff_t index,
1939 struct folio *folio, swp_entry_t swap)
1940 {
1941 struct address_space *mapping = inode->i_mapping;
1942 swp_entry_t swapin_error;
1943 void *old;
1944
1945 swapin_error = make_poisoned_swp_entry();
1946 old = xa_cmpxchg_irq(&mapping->i_pages, index,
1947 swp_to_radix_entry(swap),
1948 swp_to_radix_entry(swapin_error), 0);
1949 if (old != swp_to_radix_entry(swap))
1950 return;
1951
1952 folio_wait_writeback(folio);
1953 delete_from_swap_cache(folio);
1954 /*
1955 * Don't treat swapin error folio as alloced. Otherwise inode->i_blocks
1956 * won't be 0 when inode is released and thus trigger WARN_ON(i_blocks)
1957 * in shmem_evict_inode().
1958 */
1959 shmem_recalc_inode(inode, -1, -1);
1960 swap_free(swap);
1961 }
1962
1963 /*
1964 * Swap in the folio pointed to by *foliop.
1965 * Caller has to make sure that *foliop contains a valid swapped folio.
1966 * Returns 0 and the folio in foliop if success. On failure, returns the
1967 * error code and NULL in *foliop.
1968 */
1969 static int shmem_swapin_folio(struct inode *inode, pgoff_t index,
1970 struct folio **foliop, enum sgp_type sgp,
1971 gfp_t gfp, struct mm_struct *fault_mm,
1972 vm_fault_t *fault_type)
1973 {
1974 struct address_space *mapping = inode->i_mapping;
1975 struct shmem_inode_info *info = SHMEM_I(inode);
1976 struct swap_info_struct *si;
1977 struct folio *folio = NULL;
1978 swp_entry_t swap;
1979 int error;
1980
1981 VM_BUG_ON(!*foliop || !xa_is_value(*foliop));
1982 swap = radix_to_swp_entry(*foliop);
1983 *foliop = NULL;
1984
1985 if (is_poisoned_swp_entry(swap))
1986 return -EIO;
1987
1988 si = get_swap_device(swap);
1989 if (!si) {
1990 if (!shmem_confirm_swap(mapping, index, swap))
1991 return -EEXIST;
1992 else
1993 return -EINVAL;
1994 }
1995
1996 /* Look it up and read it in.. */
1997 folio = swap_cache_get_folio(swap, NULL, 0);
1998 if (!folio) {
1999 /* Or update major stats only when swapin succeeds?? */
2000 if (fault_type) {
2001 *fault_type |= VM_FAULT_MAJOR;
2002 count_vm_event(PGMAJFAULT);
2003 count_memcg_event_mm(fault_mm, PGMAJFAULT);
2004 }
2005 /* Here we actually start the io */
2006 folio = shmem_swapin_cluster(swap, gfp, info, index);
2007 if (!folio) {
2008 error = -ENOMEM;
2009 goto failed;
2010 }
2011 }
2012
2013 /* We have to do this with folio locked to prevent races */
2014 folio_lock(folio);
2015 if (!folio_test_swapcache(folio) ||
2016 folio->swap.val != swap.val ||
2017 !shmem_confirm_swap(mapping, index, swap)) {
2018 error = -EEXIST;
2019 goto unlock;
2020 }
2021 if (!folio_test_uptodate(folio)) {
2022 error = -EIO;
2023 goto failed;
2024 }
2025 folio_wait_writeback(folio);
2026
2027 /*
2028 * Some architectures may have to restore extra metadata to the
2029 * folio after reading from swap.
2030 */
2031 arch_swap_restore(folio_swap(swap, folio), folio);
2032
2033 if (shmem_should_replace_folio(folio, gfp)) {
2034 error = shmem_replace_folio(&folio, gfp, info, index);
2035 if (error)
2036 goto failed;
2037 }
2038
2039 error = shmem_add_to_page_cache(folio, mapping, index,
2040 swp_to_radix_entry(swap), gfp);
2041 if (error)
2042 goto failed;
2043
2044 shmem_recalc_inode(inode, 0, -1);
2045
2046 if (sgp == SGP_WRITE)
2047 folio_mark_accessed(folio);
2048
2049 delete_from_swap_cache(folio);
2050 folio_mark_dirty(folio);
2051 swap_free(swap);
2052 put_swap_device(si);
2053
2054 *foliop = folio;
2055 return 0;
2056 failed:
2057 if (!shmem_confirm_swap(mapping, index, swap))
2058 error = -EEXIST;
2059 if (error == -EIO)
2060 shmem_set_folio_swapin_error(inode, index, folio, swap);
2061 unlock:
2062 if (folio) {
2063 folio_unlock(folio);
2064 folio_put(folio);
2065 }
2066 put_swap_device(si);
2067
2068 return error;
2069 }
2070
2071 /*
2072 * shmem_get_folio_gfp - find page in cache, or get from swap, or allocate
2073 *
2074 * If we allocate a new one we do not mark it dirty. That's up to the
2075 * vm. If we swap it in we mark it dirty since we also free the swap
2076 * entry since a page cannot live in both the swap and page cache.
2077 *
2078 * vmf and fault_type are only supplied by shmem_fault: otherwise they are NULL.
2079 */
2080 static int shmem_get_folio_gfp(struct inode *inode, pgoff_t index,
2081 struct folio **foliop, enum sgp_type sgp, gfp_t gfp,
2082 struct vm_fault *vmf, vm_fault_t *fault_type)
2083 {
2084 struct vm_area_struct *vma = vmf ? vmf->vma : NULL;
2085 struct mm_struct *fault_mm;
2086 struct folio *folio;
2087 int error;
2088 bool alloced, huge;
2089 unsigned long orders = 0;
2090
2091 if (WARN_ON_ONCE(!shmem_mapping(inode->i_mapping)))
2092 return -EINVAL;
2093
2094 if (index > (MAX_LFS_FILESIZE >> PAGE_SHIFT))
2095 return -EFBIG;
2096 repeat:
2097 if (sgp <= SGP_CACHE &&
2098 ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode))
2099 return -EINVAL;
2100
2101 alloced = false;
2102 fault_mm = vma ? vma->vm_mm : NULL;
2103
2104 folio = filemap_get_entry(inode->i_mapping, index);
2105 if (folio && vma && userfaultfd_minor(vma)) {
2106 if (!xa_is_value(folio))
2107 folio_put(folio);
2108 *fault_type = handle_userfault(vmf, VM_UFFD_MINOR);
2109 return 0;
2110 }
2111
2112 if (xa_is_value(folio)) {
2113 error = shmem_swapin_folio(inode, index, &folio,
2114 sgp, gfp, fault_mm, fault_type);
2115 if (error == -EEXIST)
2116 goto repeat;
2117
2118 *foliop = folio;
2119 return error;
2120 }
2121
2122 if (folio) {
2123 folio_lock(folio);
2124
2125 /* Has the folio been truncated or swapped out? */
2126 if (unlikely(folio->mapping != inode->i_mapping)) {
2127 folio_unlock(folio);
2128 folio_put(folio);
2129 goto repeat;
2130 }
2131 if (sgp == SGP_WRITE)
2132 folio_mark_accessed(folio);
2133 if (folio_test_uptodate(folio))
2134 goto out;
2135 /* fallocated folio */
2136 if (sgp != SGP_READ)
2137 goto clear;
2138 folio_unlock(folio);
2139 folio_put(folio);
2140 }
2141
2142 /*
2143 * SGP_READ: succeed on hole, with NULL folio, letting caller zero.
2144 * SGP_NOALLOC: fail on hole, with NULL folio, letting caller fail.
2145 */
2146 *foliop = NULL;
2147 if (sgp == SGP_READ)
2148 return 0;
2149 if (sgp == SGP_NOALLOC)
2150 return -ENOENT;
2151
2152 /*
2153 * Fast cache lookup and swap lookup did not find it: allocate.
2154 */
2155
2156 if (vma && userfaultfd_missing(vma)) {
2157 *fault_type = handle_userfault(vmf, VM_UFFD_MISSING);
2158 return 0;
2159 }
2160
2161 huge = shmem_is_huge(inode, index, false, fault_mm,
2162 vma ? vma->vm_flags : 0);
2163 /* Find hugepage orders that are allowed for anonymous shmem. */
2164 if (vma && vma_is_anon_shmem(vma))
2165 orders = shmem_allowable_huge_orders(inode, vma, index, huge);
2166 else if (huge)
2167 orders = BIT(HPAGE_PMD_ORDER);
2168
2169 if (orders > 0) {
2170 gfp_t huge_gfp;
2171
2172 huge_gfp = vma_thp_gfp_mask(vma);
2173 huge_gfp = limit_gfp_mask(huge_gfp, gfp);
2174 folio = shmem_alloc_and_add_folio(vmf, huge_gfp,
2175 inode, index, fault_mm, orders);
2176 if (!IS_ERR(folio)) {
2177 if (folio_test_pmd_mappable(folio))
2178 count_vm_event(THP_FILE_ALLOC);
2179 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
2180 count_mthp_stat(folio_order(folio), MTHP_STAT_SHMEM_ALLOC);
2181 #endif
2182 goto alloced;
2183 }
2184 if (PTR_ERR(folio) == -EEXIST)
2185 goto repeat;
2186 }
2187
2188 folio = shmem_alloc_and_add_folio(vmf, gfp, inode, index, fault_mm, 0);
2189 if (IS_ERR(folio)) {
2190 error = PTR_ERR(folio);
2191 if (error == -EEXIST)
2192 goto repeat;
2193 folio = NULL;
2194 goto unlock;
2195 }
2196
2197 alloced:
2198 alloced = true;
2199 if (folio_test_large(folio) &&
2200 DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE) <
2201 folio_next_index(folio) - 1) {
2202 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
2203 struct shmem_inode_info *info = SHMEM_I(inode);
2204 /*
2205 * Part of the large folio is beyond i_size: subject
2206 * to shrink under memory pressure.
2207 */
2208 spin_lock(&sbinfo->shrinklist_lock);
2209 /*
2210 * _careful to defend against unlocked access to
2211 * ->shrink_list in shmem_unused_huge_shrink()
2212 */
2213 if (list_empty_careful(&info->shrinklist)) {
2214 list_add_tail(&info->shrinklist,
2215 &sbinfo->shrinklist);
2216 sbinfo->shrinklist_len++;
2217 }
2218 spin_unlock(&sbinfo->shrinklist_lock);
2219 }
2220
2221 if (sgp == SGP_WRITE)
2222 folio_set_referenced(folio);
2223 /*
2224 * Let SGP_FALLOC use the SGP_WRITE optimization on a new folio.
2225 */
2226 if (sgp == SGP_FALLOC)
2227 sgp = SGP_WRITE;
2228 clear:
2229 /*
2230 * Let SGP_WRITE caller clear ends if write does not fill folio;
2231 * but SGP_FALLOC on a folio fallocated earlier must initialize
2232 * it now, lest undo on failure cancel our earlier guarantee.
2233 */
2234 if (sgp != SGP_WRITE && !folio_test_uptodate(folio)) {
2235 long i, n = folio_nr_pages(folio);
2236
2237 for (i = 0; i < n; i++)
2238 clear_highpage(folio_page(folio, i));
2239 flush_dcache_folio(folio);
2240 folio_mark_uptodate(folio);
2241 }
2242
2243 /* Perhaps the file has been truncated since we checked */
2244 if (sgp <= SGP_CACHE &&
2245 ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) {
2246 error = -EINVAL;
2247 goto unlock;
2248 }
2249 out:
2250 *foliop = folio;
2251 return 0;
2252
2253 /*
2254 * Error recovery.
2255 */
2256 unlock:
2257 if (alloced)
2258 filemap_remove_folio(folio);
2259 shmem_recalc_inode(inode, 0, 0);
2260 if (folio) {
2261 folio_unlock(folio);
2262 folio_put(folio);
2263 }
2264 return error;
2265 }
2266
2267 /**
2268 * shmem_get_folio - find, and lock a shmem folio.
2269 * @inode: inode to search
2270 * @index: the page index.
2271 * @foliop: pointer to the folio if found
2272 * @sgp: SGP_* flags to control behavior
2273 *
2274 * Looks up the page cache entry at @inode & @index. If a folio is
2275 * present, it is returned locked with an increased refcount.
2276 *
2277 * If the caller modifies data in the folio, it must call folio_mark_dirty()
2278 * before unlocking the folio to ensure that the folio is not reclaimed.
2279 * There is no need to reserve space before calling folio_mark_dirty().
2280 *
2281 * When no folio is found, the behavior depends on @sgp:
2282 * - for SGP_READ, *@foliop is %NULL and 0 is returned
2283 * - for SGP_NOALLOC, *@foliop is %NULL and -ENOENT is returned
2284 * - for all other flags a new folio is allocated, inserted into the
2285 * page cache and returned locked in @foliop.
2286 *
2287 * Context: May sleep.
2288 * Return: 0 if successful, else a negative error code.
2289 */
2290 int shmem_get_folio(struct inode *inode, pgoff_t index, struct folio **foliop,
2291 enum sgp_type sgp)
2292 {
2293 return shmem_get_folio_gfp(inode, index, foliop, sgp,
2294 mapping_gfp_mask(inode->i_mapping), NULL, NULL);
2295 }
2296 EXPORT_SYMBOL_GPL(shmem_get_folio);
2297
2298 /*
2299 * This is like autoremove_wake_function, but it removes the wait queue
2300 * entry unconditionally - even if something else had already woken the
2301 * target.
2302 */
2303 static int synchronous_wake_function(wait_queue_entry_t *wait,
2304 unsigned int mode, int sync, void *key)
2305 {
2306 int ret = default_wake_function(wait, mode, sync, key);
2307 list_del_init(&wait->entry);
2308 return ret;
2309 }
2310
2311 /*
2312 * Trinity finds that probing a hole which tmpfs is punching can
2313 * prevent the hole-punch from ever completing: which in turn
2314 * locks writers out with its hold on i_rwsem. So refrain from
2315 * faulting pages into the hole while it's being punched. Although
2316 * shmem_undo_range() does remove the additions, it may be unable to
2317 * keep up, as each new page needs its own unmap_mapping_range() call,
2318 * and the i_mmap tree grows ever slower to scan if new vmas are added.
2319 *
2320 * It does not matter if we sometimes reach this check just before the
2321 * hole-punch begins, so that one fault then races with the punch:
2322 * we just need to make racing faults a rare case.
2323 *
2324 * The implementation below would be much simpler if we just used a
2325 * standard mutex or completion: but we cannot take i_rwsem in fault,
2326 * and bloating every shmem inode for this unlikely case would be sad.
2327 */
2328 static vm_fault_t shmem_falloc_wait(struct vm_fault *vmf, struct inode *inode)
2329 {
2330 struct shmem_falloc *shmem_falloc;
2331 struct file *fpin = NULL;
2332 vm_fault_t ret = 0;
2333
2334 spin_lock(&inode->i_lock);
2335 shmem_falloc = inode->i_private;
2336 if (shmem_falloc &&
2337 shmem_falloc->waitq &&
2338 vmf->pgoff >= shmem_falloc->start &&
2339 vmf->pgoff < shmem_falloc->next) {
2340 wait_queue_head_t *shmem_falloc_waitq;
2341 DEFINE_WAIT_FUNC(shmem_fault_wait, synchronous_wake_function);
2342
2343 ret = VM_FAULT_NOPAGE;
2344 fpin = maybe_unlock_mmap_for_io(vmf, NULL);
2345 shmem_falloc_waitq = shmem_falloc->waitq;
2346 prepare_to_wait(shmem_falloc_waitq, &shmem_fault_wait,
2347 TASK_UNINTERRUPTIBLE);
2348 spin_unlock(&inode->i_lock);
2349 schedule();
2350
2351 /*
2352 * shmem_falloc_waitq points into the shmem_fallocate()
2353 * stack of the hole-punching task: shmem_falloc_waitq
2354 * is usually invalid by the time we reach here, but
2355 * finish_wait() does not dereference it in that case;
2356 * though i_lock needed lest racing with wake_up_all().
2357 */
2358 spin_lock(&inode->i_lock);
2359 finish_wait(shmem_falloc_waitq, &shmem_fault_wait);
2360 }
2361 spin_unlock(&inode->i_lock);
2362 if (fpin) {
2363 fput(fpin);
2364 ret = VM_FAULT_RETRY;
2365 }
2366 return ret;
2367 }
2368
2369 static vm_fault_t shmem_fault(struct vm_fault *vmf)
2370 {
2371 struct inode *inode = file_inode(vmf->vma->vm_file);
2372 gfp_t gfp = mapping_gfp_mask(inode->i_mapping);
2373 struct folio *folio = NULL;
2374 vm_fault_t ret = 0;
2375 int err;
2376
2377 /*
2378 * Trinity finds that probing a hole which tmpfs is punching can
2379 * prevent the hole-punch from ever completing: noted in i_private.
2380 */
2381 if (unlikely(inode->i_private)) {
2382 ret = shmem_falloc_wait(vmf, inode);
2383 if (ret)
2384 return ret;
2385 }
2386
2387 WARN_ON_ONCE(vmf->page != NULL);
2388 err = shmem_get_folio_gfp(inode, vmf->pgoff, &folio, SGP_CACHE,
2389 gfp, vmf, &ret);
2390 if (err)
2391 return vmf_error(err);
2392 if (folio) {
2393 vmf->page = folio_file_page(folio, vmf->pgoff);
2394 ret |= VM_FAULT_LOCKED;
2395 }
2396 return ret;
2397 }
2398
2399 unsigned long shmem_get_unmapped_area(struct file *file,
2400 unsigned long uaddr, unsigned long len,
2401 unsigned long pgoff, unsigned long flags)
2402 {
2403 unsigned long addr;
2404 unsigned long offset;
2405 unsigned long inflated_len;
2406 unsigned long inflated_addr;
2407 unsigned long inflated_offset;
2408 unsigned long hpage_size;
2409
2410 if (len > TASK_SIZE)
2411 return -ENOMEM;
2412
2413 addr = mm_get_unmapped_area(current->mm, file, uaddr, len, pgoff,
2414 flags);
2415
2416 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
2417 return addr;
2418 if (IS_ERR_VALUE(addr))
2419 return addr;
2420 if (addr & ~PAGE_MASK)
2421 return addr;
2422 if (addr > TASK_SIZE - len)
2423 return addr;
2424
2425 if (shmem_huge == SHMEM_HUGE_DENY)
2426 return addr;
2427 if (flags & MAP_FIXED)
2428 return addr;
2429 /*
2430 * Our priority is to support MAP_SHARED mapped hugely;
2431 * and support MAP_PRIVATE mapped hugely too, until it is COWed.
2432 * But if caller specified an address hint and we allocated area there
2433 * successfully, respect that as before.
2434 */
2435 if (uaddr == addr)
2436 return addr;
2437
2438 hpage_size = HPAGE_PMD_SIZE;
2439 if (shmem_huge != SHMEM_HUGE_FORCE) {
2440 struct super_block *sb;
2441 unsigned long __maybe_unused hpage_orders;
2442 int order = 0;
2443
2444 if (file) {
2445 VM_BUG_ON(file->f_op != &shmem_file_operations);
2446 sb = file_inode(file)->i_sb;
2447 } else {
2448 /*
2449 * Called directly from mm/mmap.c, or drivers/char/mem.c
2450 * for "/dev/zero", to create a shared anonymous object.
2451 */
2452 if (IS_ERR(shm_mnt))
2453 return addr;
2454 sb = shm_mnt->mnt_sb;
2455
2456 /*
2457 * Find the highest mTHP order used for anonymous shmem to
2458 * provide a suitable alignment address.
2459 */
2460 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
2461 hpage_orders = READ_ONCE(huge_shmem_orders_always);
2462 hpage_orders |= READ_ONCE(huge_shmem_orders_within_size);
2463 hpage_orders |= READ_ONCE(huge_shmem_orders_madvise);
2464 if (SHMEM_SB(sb)->huge != SHMEM_HUGE_NEVER)
2465 hpage_orders |= READ_ONCE(huge_shmem_orders_inherit);
2466
2467 if (hpage_orders > 0) {
2468 order = highest_order(hpage_orders);
2469 hpage_size = PAGE_SIZE << order;
2470 }
2471 #endif
2472 }
2473 if (SHMEM_SB(sb)->huge == SHMEM_HUGE_NEVER && !order)
2474 return addr;
2475 }
2476
2477 if (len < hpage_size)
2478 return addr;
2479
2480 offset = (pgoff << PAGE_SHIFT) & (hpage_size - 1);
2481 if (offset && offset + len < 2 * hpage_size)
2482 return addr;
2483 if ((addr & (hpage_size - 1)) == offset)
2484 return addr;
2485
2486 inflated_len = len + hpage_size - PAGE_SIZE;
2487 if (inflated_len > TASK_SIZE)
2488 return addr;
2489 if (inflated_len < len)
2490 return addr;
2491
2492 inflated_addr = mm_get_unmapped_area(current->mm, NULL, uaddr,
2493 inflated_len, 0, flags);
2494 if (IS_ERR_VALUE(inflated_addr))
2495 return addr;
2496 if (inflated_addr & ~PAGE_MASK)
2497 return addr;
2498
2499 inflated_offset = inflated_addr & (hpage_size - 1);
2500 inflated_addr += offset - inflated_offset;
2501 if (inflated_offset > offset)
2502 inflated_addr += hpage_size;
2503
2504 if (inflated_addr > TASK_SIZE - len)
2505 return addr;
2506 return inflated_addr;
2507 }
2508
2509 #ifdef CONFIG_NUMA
2510 static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *mpol)
2511 {
2512 struct inode *inode = file_inode(vma->vm_file);
2513 return mpol_set_shared_policy(&SHMEM_I(inode)->policy, vma, mpol);
2514 }
2515
2516 static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma,
2517 unsigned long addr, pgoff_t *ilx)
2518 {
2519 struct inode *inode = file_inode(vma->vm_file);
2520 pgoff_t index;
2521
2522 /*
2523 * Bias interleave by inode number to distribute better across nodes;
2524 * but this interface is independent of which page order is used, so
2525 * supplies only that bias, letting caller apply the offset (adjusted
2526 * by page order, as in shmem_get_pgoff_policy() and get_vma_policy()).
2527 */
2528 *ilx = inode->i_ino;
2529 index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
2530 return mpol_shared_policy_lookup(&SHMEM_I(inode)->policy, index);
2531 }
2532
2533 static struct mempolicy *shmem_get_pgoff_policy(struct shmem_inode_info *info,
2534 pgoff_t index, unsigned int order, pgoff_t *ilx)
2535 {
2536 struct mempolicy *mpol;
2537
2538 /* Bias interleave by inode number to distribute better across nodes */
2539 *ilx = info->vfs_inode.i_ino + (index >> order);
2540
2541 mpol = mpol_shared_policy_lookup(&info->policy, index);
2542 return mpol ? mpol : get_task_policy(current);
2543 }
2544 #else
2545 static struct mempolicy *shmem_get_pgoff_policy(struct shmem_inode_info *info,
2546 pgoff_t index, unsigned int order, pgoff_t *ilx)
2547 {
2548 *ilx = 0;
2549 return NULL;
2550 }
2551 #endif /* CONFIG_NUMA */
2552
2553 int shmem_lock(struct file *file, int lock, struct ucounts *ucounts)
2554 {
2555 struct inode *inode = file_inode(file);
2556 struct shmem_inode_info *info = SHMEM_I(inode);
2557 int retval = -ENOMEM;
2558
2559 /*
2560 * What serializes the accesses to info->flags?
2561 * ipc_lock_object() when called from shmctl_do_lock(),
2562 * no serialization needed when called from shm_destroy().
2563 */
2564 if (lock && !(info->flags & VM_LOCKED)) {
2565 if (!user_shm_lock(inode->i_size, ucounts))
2566 goto out_nomem;
2567 info->flags |= VM_LOCKED;
2568 mapping_set_unevictable(file->f_mapping);
2569 }
2570 if (!lock && (info->flags & VM_LOCKED) && ucounts) {
2571 user_shm_unlock(inode->i_size, ucounts);
2572 info->flags &= ~VM_LOCKED;
2573 mapping_clear_unevictable(file->f_mapping);
2574 }
2575 retval = 0;
2576
2577 out_nomem:
2578 return retval;
2579 }
2580
2581 static int shmem_mmap(struct file *file, struct vm_area_struct *vma)
2582 {
2583 struct inode *inode = file_inode(file);
2584 struct shmem_inode_info *info = SHMEM_I(inode);
2585 int ret;
2586
2587 ret = seal_check_write(info->seals, vma);
2588 if (ret)
2589 return ret;
2590
2591 /* arm64 - allow memory tagging on RAM-based files */
2592 vm_flags_set(vma, VM_MTE_ALLOWED);
2593
2594 file_accessed(file);
2595 /* This is anonymous shared memory if it is unlinked at the time of mmap */
2596 if (inode->i_nlink)
2597 vma->vm_ops = &shmem_vm_ops;
2598 else
2599 vma->vm_ops = &shmem_anon_vm_ops;
2600 return 0;
2601 }
2602
2603 static int shmem_file_open(struct inode *inode, struct file *file)
2604 {
2605 file->f_mode |= FMODE_CAN_ODIRECT;
2606 return generic_file_open(inode, file);
2607 }
2608
2609 #ifdef CONFIG_TMPFS_XATTR
2610 static int shmem_initxattrs(struct inode *, const struct xattr *, void *);
2611
2612 /*
2613 * chattr's fsflags are unrelated to extended attributes,
2614 * but tmpfs has chosen to enable them under the same config option.
2615 */
2616 static void shmem_set_inode_flags(struct inode *inode, unsigned int fsflags)
2617 {
2618 unsigned int i_flags = 0;
2619
2620 if (fsflags & FS_NOATIME_FL)
2621 i_flags |= S_NOATIME;
2622 if (fsflags & FS_APPEND_FL)
2623 i_flags |= S_APPEND;
2624 if (fsflags & FS_IMMUTABLE_FL)
2625 i_flags |= S_IMMUTABLE;
2626 /*
2627 * But FS_NODUMP_FL does not require any action in i_flags.
2628 */
2629 inode_set_flags(inode, i_flags, S_NOATIME | S_APPEND | S_IMMUTABLE);
2630 }
2631 #else
2632 static void shmem_set_inode_flags(struct inode *inode, unsigned int fsflags)
2633 {
2634 }
2635 #define shmem_initxattrs NULL
2636 #endif
2637
2638 static struct offset_ctx *shmem_get_offset_ctx(struct inode *inode)
2639 {
2640 return &SHMEM_I(inode)->dir_offsets;
2641 }
2642
2643 static struct inode *__shmem_get_inode(struct mnt_idmap *idmap,
2644 struct super_block *sb,
2645 struct inode *dir, umode_t mode,
2646 dev_t dev, unsigned long flags)
2647 {
2648 struct inode *inode;
2649 struct shmem_inode_info *info;
2650 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
2651 ino_t ino;
2652 int err;
2653
2654 err = shmem_reserve_inode(sb, &ino);
2655 if (err)
2656 return ERR_PTR(err);
2657
2658 inode = new_inode(sb);
2659 if (!inode) {
2660 shmem_free_inode(sb, 0);
2661 return ERR_PTR(-ENOSPC);
2662 }
2663
2664 inode->i_ino = ino;
2665 inode_init_owner(idmap, inode, dir, mode);
2666 inode->i_blocks = 0;
2667 simple_inode_init_ts(inode);
2668 inode->i_generation = get_random_u32();
2669 info = SHMEM_I(inode);
2670 memset(info, 0, (char *)inode - (char *)info);
2671 spin_lock_init(&info->lock);
2672 atomic_set(&info->stop_eviction, 0);
2673 info->seals = F_SEAL_SEAL;
2674 info->flags = flags & VM_NORESERVE;
2675 info->i_crtime = inode_get_mtime(inode);
2676 info->fsflags = (dir == NULL) ? 0 :
2677 SHMEM_I(dir)->fsflags & SHMEM_FL_INHERITED;
2678 if (info->fsflags)
2679 shmem_set_inode_flags(inode, info->fsflags);
2680 INIT_LIST_HEAD(&info->shrinklist);
2681 INIT_LIST_HEAD(&info->swaplist);
2682 simple_xattrs_init(&info->xattrs);
2683 cache_no_acl(inode);
2684 if (sbinfo->noswap)
2685 mapping_set_unevictable(inode->i_mapping);
2686 mapping_set_large_folios(inode->i_mapping);
2687
2688 switch (mode & S_IFMT) {
2689 default:
2690 inode->i_op = &shmem_special_inode_operations;
2691 init_special_inode(inode, mode, dev);
2692 break;
2693 case S_IFREG:
2694 inode->i_mapping->a_ops = &shmem_aops;
2695 inode->i_op = &shmem_inode_operations;
2696 inode->i_fop = &shmem_file_operations;
2697 mpol_shared_policy_init(&info->policy,
2698 shmem_get_sbmpol(sbinfo));
2699 break;
2700 case S_IFDIR:
2701 inc_nlink(inode);
2702 /* Some things misbehave if size == 0 on a directory */
2703 inode->i_size = 2 * BOGO_DIRENT_SIZE;
2704 inode->i_op = &shmem_dir_inode_operations;
2705 inode->i_fop = &simple_offset_dir_operations;
2706 simple_offset_init(shmem_get_offset_ctx(inode));
2707 break;
2708 case S_IFLNK:
2709 /*
2710 * Must not load anything in the rbtree,
2711 * mpol_free_shared_policy will not be called.
2712 */
2713 mpol_shared_policy_init(&info->policy, NULL);
2714 break;
2715 }
2716
2717 lockdep_annotate_inode_mutex_key(inode);
2718 return inode;
2719 }
2720
2721 #ifdef CONFIG_TMPFS_QUOTA
2722 static struct inode *shmem_get_inode(struct mnt_idmap *idmap,
2723 struct super_block *sb, struct inode *dir,
2724 umode_t mode, dev_t dev, unsigned long flags)
2725 {
2726 int err;
2727 struct inode *inode;
2728
2729 inode = __shmem_get_inode(idmap, sb, dir, mode, dev, flags);
2730 if (IS_ERR(inode))
2731 return inode;
2732
2733 err = dquot_initialize(inode);
2734 if (err)
2735 goto errout;
2736
2737 err = dquot_alloc_inode(inode);
2738 if (err) {
2739 dquot_drop(inode);
2740 goto errout;
2741 }
2742 return inode;
2743
2744 errout:
2745 inode->i_flags |= S_NOQUOTA;
2746 iput(inode);
2747 return ERR_PTR(err);
2748 }
2749 #else
2750 static inline struct inode *shmem_get_inode(struct mnt_idmap *idmap,
2751 struct super_block *sb, struct inode *dir,
2752 umode_t mode, dev_t dev, unsigned long flags)
2753 {
2754 return __shmem_get_inode(idmap, sb, dir, mode, dev, flags);
2755 }
2756 #endif /* CONFIG_TMPFS_QUOTA */
2757
2758 #ifdef CONFIG_USERFAULTFD
2759 int shmem_mfill_atomic_pte(pmd_t *dst_pmd,
2760 struct vm_area_struct *dst_vma,
2761 unsigned long dst_addr,
2762 unsigned long src_addr,
2763 uffd_flags_t flags,
2764 struct folio **foliop)
2765 {
2766 struct inode *inode = file_inode(dst_vma->vm_file);
2767 struct shmem_inode_info *info = SHMEM_I(inode);
2768 struct address_space *mapping = inode->i_mapping;
2769 gfp_t gfp = mapping_gfp_mask(mapping);
2770 pgoff_t pgoff = linear_page_index(dst_vma, dst_addr);
2771 void *page_kaddr;
2772 struct folio *folio;
2773 int ret;
2774 pgoff_t max_off;
2775
2776 if (shmem_inode_acct_blocks(inode, 1)) {
2777 /*
2778 * We may have got a page, returned -ENOENT triggering a retry,
2779 * and now we find ourselves with -ENOMEM. Release the page, to
2780 * avoid a BUG_ON in our caller.
2781 */
2782 if (unlikely(*foliop)) {
2783 folio_put(*foliop);
2784 *foliop = NULL;
2785 }
2786 return -ENOMEM;
2787 }
2788
2789 if (!*foliop) {
2790 ret = -ENOMEM;
2791 folio = shmem_alloc_folio(gfp, 0, info, pgoff);
2792 if (!folio)
2793 goto out_unacct_blocks;
2794
2795 if (uffd_flags_mode_is(flags, MFILL_ATOMIC_COPY)) {
2796 page_kaddr = kmap_local_folio(folio, 0);
2797 /*
2798 * The read mmap_lock is held here. Despite the
2799 * mmap_lock being read recursive a deadlock is still
2800 * possible if a writer has taken a lock. For example:
2801 *
2802 * process A thread 1 takes read lock on own mmap_lock
2803 * process A thread 2 calls mmap, blocks taking write lock
2804 * process B thread 1 takes page fault, read lock on own mmap lock
2805 * process B thread 2 calls mmap, blocks taking write lock
2806 * process A thread 1 blocks taking read lock on process B
2807 * process B thread 1 blocks taking read lock on process A
2808 *
2809 * Disable page faults to prevent potential deadlock
2810 * and retry the copy outside the mmap_lock.
2811 */
2812 pagefault_disable();
2813 ret = copy_from_user(page_kaddr,
2814 (const void __user *)src_addr,
2815 PAGE_SIZE);
2816 pagefault_enable();
2817 kunmap_local(page_kaddr);
2818
2819 /* fallback to copy_from_user outside mmap_lock */
2820 if (unlikely(ret)) {
2821 *foliop = folio;
2822 ret = -ENOENT;
2823 /* don't free the page */
2824 goto out_unacct_blocks;
2825 }
2826
2827 flush_dcache_folio(folio);
2828 } else { /* ZEROPAGE */
2829 clear_user_highpage(&folio->page, dst_addr);
2830 }
2831 } else {
2832 folio = *foliop;
2833 VM_BUG_ON_FOLIO(folio_test_large(folio), folio);
2834 *foliop = NULL;
2835 }
2836
2837 VM_BUG_ON(folio_test_locked(folio));
2838 VM_BUG_ON(folio_test_swapbacked(folio));
2839 __folio_set_locked(folio);
2840 __folio_set_swapbacked(folio);
2841 __folio_mark_uptodate(folio);
2842
2843 ret = -EFAULT;
2844 max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
2845 if (unlikely(pgoff >= max_off))
2846 goto out_release;
2847
2848 ret = mem_cgroup_charge(folio, dst_vma->vm_mm, gfp);
2849 if (ret)
2850 goto out_release;
2851 ret = shmem_add_to_page_cache(folio, mapping, pgoff, NULL, gfp);
2852 if (ret)
2853 goto out_release;
2854
2855 ret = mfill_atomic_install_pte(dst_pmd, dst_vma, dst_addr,
2856 &folio->page, true, flags);
2857 if (ret)
2858 goto out_delete_from_cache;
2859
2860 shmem_recalc_inode(inode, 1, 0);
2861 folio_unlock(folio);
2862 return 0;
2863 out_delete_from_cache:
2864 filemap_remove_folio(folio);
2865 out_release:
2866 folio_unlock(folio);
2867 folio_put(folio);
2868 out_unacct_blocks:
2869 shmem_inode_unacct_blocks(inode, 1);
2870 return ret;
2871 }
2872 #endif /* CONFIG_USERFAULTFD */
2873
2874 #ifdef CONFIG_TMPFS
2875 static const struct inode_operations shmem_symlink_inode_operations;
2876 static const struct inode_operations shmem_short_symlink_operations;
2877
2878 static int
2879 shmem_write_begin(struct file *file, struct address_space *mapping,
2880 loff_t pos, unsigned len,
2881 struct page **pagep, void **fsdata)
2882 {
2883 struct inode *inode = mapping->host;
2884 struct shmem_inode_info *info = SHMEM_I(inode);
2885 pgoff_t index = pos >> PAGE_SHIFT;
2886 struct folio *folio;
2887 int ret = 0;
2888
2889 /* i_rwsem is held by caller */
2890 if (unlikely(info->seals & (F_SEAL_GROW |
2891 F_SEAL_WRITE | F_SEAL_FUTURE_WRITE))) {
2892 if (info->seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE))
2893 return -EPERM;
2894 if ((info->seals & F_SEAL_GROW) && pos + len > inode->i_size)
2895 return -EPERM;
2896 }
2897
2898 ret = shmem_get_folio(inode, index, &folio, SGP_WRITE);
2899 if (ret)
2900 return ret;
2901
2902 *pagep = folio_file_page(folio, index);
2903 if (PageHWPoison(*pagep)) {
2904 folio_unlock(folio);
2905 folio_put(folio);
2906 *pagep = NULL;
2907 return -EIO;
2908 }
2909
2910 return 0;
2911 }
2912
2913 static int
2914 shmem_write_end(struct file *file, struct address_space *mapping,
2915 loff_t pos, unsigned len, unsigned copied,
2916 struct page *page, void *fsdata)
2917 {
2918 struct folio *folio = page_folio(page);
2919 struct inode *inode = mapping->host;
2920
2921 if (pos + copied > inode->i_size)
2922 i_size_write(inode, pos + copied);
2923
2924 if (!folio_test_uptodate(folio)) {
2925 if (copied < folio_size(folio)) {
2926 size_t from = offset_in_folio(folio, pos);
2927 folio_zero_segments(folio, 0, from,
2928 from + copied, folio_size(folio));
2929 }
2930 folio_mark_uptodate(folio);
2931 }
2932 folio_mark_dirty(folio);
2933 folio_unlock(folio);
2934 folio_put(folio);
2935
2936 return copied;
2937 }
2938
2939 static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
2940 {
2941 struct file *file = iocb->ki_filp;
2942 struct inode *inode = file_inode(file);
2943 struct address_space *mapping = inode->i_mapping;
2944 pgoff_t index;
2945 unsigned long offset;
2946 int error = 0;
2947 ssize_t retval = 0;
2948 loff_t *ppos = &iocb->ki_pos;
2949
2950 index = *ppos >> PAGE_SHIFT;
2951 offset = *ppos & ~PAGE_MASK;
2952
2953 for (;;) {
2954 struct folio *folio = NULL;
2955 struct page *page = NULL;
2956 pgoff_t end_index;
2957 unsigned long nr, ret;
2958 loff_t i_size = i_size_read(inode);
2959
2960 end_index = i_size >> PAGE_SHIFT;
2961 if (index > end_index)
2962 break;
2963 if (index == end_index) {
2964 nr = i_size & ~PAGE_MASK;
2965 if (nr <= offset)
2966 break;
2967 }
2968
2969 error = shmem_get_folio(inode, index, &folio, SGP_READ);
2970 if (error) {
2971 if (error == -EINVAL)
2972 error = 0;
2973 break;
2974 }
2975 if (folio) {
2976 folio_unlock(folio);
2977
2978 page = folio_file_page(folio, index);
2979 if (PageHWPoison(page)) {
2980 folio_put(folio);
2981 error = -EIO;
2982 break;
2983 }
2984 }
2985
2986 /*
2987 * We must evaluate after, since reads (unlike writes)
2988 * are called without i_rwsem protection against truncate
2989 */
2990 nr = PAGE_SIZE;
2991 i_size = i_size_read(inode);
2992 end_index = i_size >> PAGE_SHIFT;
2993 if (index == end_index) {
2994 nr = i_size & ~PAGE_MASK;
2995 if (nr <= offset) {
2996 if (folio)
2997 folio_put(folio);
2998 break;
2999 }
3000 }
3001 nr -= offset;
3002
3003 if (folio) {
3004 /*
3005 * If users can be writing to this page using arbitrary
3006 * virtual addresses, take care about potential aliasing
3007 * before reading the page on the kernel side.
3008 */
3009 if (mapping_writably_mapped(mapping))
3010 flush_dcache_page(page);
3011 /*
3012 * Mark the page accessed if we read the beginning.
3013 */
3014 if (!offset)
3015 folio_mark_accessed(folio);
3016 /*
3017 * Ok, we have the page, and it's up-to-date, so
3018 * now we can copy it to user space...
3019 */
3020 ret = copy_page_to_iter(page, offset, nr, to);
3021 folio_put(folio);
3022
3023 } else if (user_backed_iter(to)) {
3024 /*
3025 * Copy to user tends to be so well optimized, but
3026 * clear_user() not so much, that it is noticeably
3027 * faster to copy the zero page instead of clearing.
3028 */
3029 ret = copy_page_to_iter(ZERO_PAGE(0), offset, nr, to);
3030 } else {
3031 /*
3032 * But submitting the same page twice in a row to
3033 * splice() - or others? - can result in confusion:
3034 * so don't attempt that optimization on pipes etc.
3035 */
3036 ret = iov_iter_zero(nr, to);
3037 }
3038
3039 retval += ret;
3040 offset += ret;
3041 index += offset >> PAGE_SHIFT;
3042 offset &= ~PAGE_MASK;
3043
3044 if (!iov_iter_count(to))
3045 break;
3046 if (ret < nr) {
3047 error = -EFAULT;
3048 break;
3049 }
3050 cond_resched();
3051 }
3052
3053 *ppos = ((loff_t) index << PAGE_SHIFT) + offset;
3054 file_accessed(file);
3055 return retval ? retval : error;
3056 }
3057
3058 static ssize_t shmem_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
3059 {
3060 struct file *file = iocb->ki_filp;
3061 struct inode *inode = file->f_mapping->host;
3062 ssize_t ret;
3063
3064 inode_lock(inode);
3065 ret = generic_write_checks(iocb, from);
3066 if (ret <= 0)
3067 goto unlock;
3068 ret = file_remove_privs(file);
3069 if (ret)
3070 goto unlock;
3071 ret = file_update_time(file);
3072 if (ret)
3073 goto unlock;
3074 ret = generic_perform_write(iocb, from);
3075 unlock:
3076 inode_unlock(inode);
3077 return ret;
3078 }
3079
3080 static bool zero_pipe_buf_get(struct pipe_inode_info *pipe,
3081 struct pipe_buffer *buf)
3082 {
3083 return true;
3084 }
3085
3086 static void zero_pipe_buf_release(struct pipe_inode_info *pipe,
3087 struct pipe_buffer *buf)
3088 {
3089 }
3090
3091 static bool zero_pipe_buf_try_steal(struct pipe_inode_info *pipe,
3092 struct pipe_buffer *buf)
3093 {
3094 return false;
3095 }
3096
3097 static const struct pipe_buf_operations zero_pipe_buf_ops = {
3098 .release = zero_pipe_buf_release,
3099 .try_steal = zero_pipe_buf_try_steal,
3100 .get = zero_pipe_buf_get,
3101 };
3102
3103 static size_t splice_zeropage_into_pipe(struct pipe_inode_info *pipe,
3104 loff_t fpos, size_t size)
3105 {
3106 size_t offset = fpos & ~PAGE_MASK;
3107
3108 size = min_t(size_t, size, PAGE_SIZE - offset);
3109
3110 if (!pipe_full(pipe->head, pipe->tail, pipe->max_usage)) {
3111 struct pipe_buffer *buf = pipe_head_buf(pipe);
3112
3113 *buf = (struct pipe_buffer) {
3114 .ops = &zero_pipe_buf_ops,
3115 .page = ZERO_PAGE(0),
3116 .offset = offset,
3117 .len = size,
3118 };
3119 pipe->head++;
3120 }
3121
3122 return size;
3123 }
3124
3125 static ssize_t shmem_file_splice_read(struct file *in, loff_t *ppos,
3126 struct pipe_inode_info *pipe,
3127 size_t len, unsigned int flags)
3128 {
3129 struct inode *inode = file_inode(in);
3130 struct address_space *mapping = inode->i_mapping;
3131 struct folio *folio = NULL;
3132 size_t total_spliced = 0, used, npages, n, part;
3133 loff_t isize;
3134 int error = 0;
3135
3136 /* Work out how much data we can actually add into the pipe */
3137 used = pipe_occupancy(pipe->head, pipe->tail);
3138 npages = max_t(ssize_t, pipe->max_usage - used, 0);
3139 len = min_t(size_t, len, npages * PAGE_SIZE);
3140
3141 do {
3142 if (*ppos >= i_size_read(inode))
3143 break;
3144
3145 error = shmem_get_folio(inode, *ppos / PAGE_SIZE, &folio,
3146 SGP_READ);
3147 if (error) {
3148 if (error == -EINVAL)
3149 error = 0;
3150 break;
3151 }
3152 if (folio) {
3153 folio_unlock(folio);
3154
3155 if (folio_test_hwpoison(folio) ||
3156 (folio_test_large(folio) &&
3157 folio_test_has_hwpoisoned(folio))) {
3158 error = -EIO;
3159 break;
3160 }
3161 }
3162
3163 /*
3164 * i_size must be checked after we know the pages are Uptodate.
3165 *
3166 * Checking i_size after the check allows us to calculate
3167 * the correct value for "nr", which means the zero-filled
3168 * part of the page is not copied back to userspace (unless
3169 * another truncate extends the file - this is desired though).
3170 */
3171 isize = i_size_read(inode);
3172 if (unlikely(*ppos >= isize))
3173 break;
3174 part = min_t(loff_t, isize - *ppos, len);
3175
3176 if (folio) {
3177 /*
3178 * If users can be writing to this page using arbitrary
3179 * virtual addresses, take care about potential aliasing
3180 * before reading the page on the kernel side.
3181 */
3182 if (mapping_writably_mapped(mapping))
3183 flush_dcache_folio(folio);
3184 folio_mark_accessed(folio);
3185 /*
3186 * Ok, we have the page, and it's up-to-date, so we can
3187 * now splice it into the pipe.
3188 */
3189 n = splice_folio_into_pipe(pipe, folio, *ppos, part);
3190 folio_put(folio);
3191 folio = NULL;
3192 } else {
3193 n = splice_zeropage_into_pipe(pipe, *ppos, part);
3194 }
3195
3196 if (!n)
3197 break;
3198 len -= n;
3199 total_spliced += n;
3200 *ppos += n;
3201 in->f_ra.prev_pos = *ppos;
3202 if (pipe_full(pipe->head, pipe->tail, pipe->max_usage))
3203 break;
3204
3205 cond_resched();
3206 } while (len);
3207
3208 if (folio)
3209 folio_put(folio);
3210
3211 file_accessed(in);
3212 return total_spliced ? total_spliced : error;
3213 }
3214
3215 static loff_t shmem_file_llseek(struct file *file, loff_t offset, int whence)
3216 {
3217 struct address_space *mapping = file->f_mapping;
3218 struct inode *inode = mapping->host;
3219
3220 if (whence != SEEK_DATA && whence != SEEK_HOLE)
3221 return generic_file_llseek_size(file, offset, whence,
3222 MAX_LFS_FILESIZE, i_size_read(inode));
3223 if (offset < 0)
3224 return -ENXIO;
3225
3226 inode_lock(inode);
3227 /* We're holding i_rwsem so we can access i_size directly */
3228 offset = mapping_seek_hole_data(mapping, offset, inode->i_size, whence);
3229 if (offset >= 0)
3230 offset = vfs_setpos(file, offset, MAX_LFS_FILESIZE);
3231 inode_unlock(inode);
3232 return offset;
3233 }
3234
3235 static long shmem_fallocate(struct file *file, int mode, loff_t offset,
3236 loff_t len)
3237 {
3238 struct inode *inode = file_inode(file);
3239 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
3240 struct shmem_inode_info *info = SHMEM_I(inode);
3241 struct shmem_falloc shmem_falloc;
3242 pgoff_t start, index, end, undo_fallocend;
3243 int error;
3244
3245 if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE))
3246 return -EOPNOTSUPP;
3247
3248 inode_lock(inode);
3249
3250 if (mode & FALLOC_FL_PUNCH_HOLE) {
3251 struct address_space *mapping = file->f_mapping;
3252 loff_t unmap_start = round_up(offset, PAGE_SIZE);
3253 loff_t unmap_end = round_down(offset + len, PAGE_SIZE) - 1;
3254 DECLARE_WAIT_QUEUE_HEAD_ONSTACK(shmem_falloc_waitq);
3255
3256 /* protected by i_rwsem */
3257 if (info->seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE)) {
3258 error = -EPERM;
3259 goto out;
3260 }
3261
3262 shmem_falloc.waitq = &shmem_falloc_waitq;
3263 shmem_falloc.start = (u64)unmap_start >> PAGE_SHIFT;
3264 shmem_falloc.next = (unmap_end + 1) >> PAGE_SHIFT;
3265 spin_lock(&inode->i_lock);
3266 inode->i_private = &shmem_falloc;
3267 spin_unlock(&inode->i_lock);
3268
3269 if ((u64)unmap_end > (u64)unmap_start)
3270 unmap_mapping_range(mapping, unmap_start,
3271 1 + unmap_end - unmap_start, 0);
3272 shmem_truncate_range(inode, offset, offset + len - 1);
3273 /* No need to unmap again: hole-punching leaves COWed pages */
3274
3275 spin_lock(&inode->i_lock);
3276 inode->i_private = NULL;
3277 wake_up_all(&shmem_falloc_waitq);
3278 WARN_ON_ONCE(!list_empty(&shmem_falloc_waitq.head));
3279 spin_unlock(&inode->i_lock);
3280 error = 0;
3281 goto out;
3282 }
3283
3284 /* We need to check rlimit even when FALLOC_FL_KEEP_SIZE */
3285 error = inode_newsize_ok(inode, offset + len);
3286 if (error)
3287 goto out;
3288
3289 if ((info->seals & F_SEAL_GROW) && offset + len > inode->i_size) {
3290 error = -EPERM;
3291 goto out;
3292 }
3293
3294 start = offset >> PAGE_SHIFT;
3295 end = (offset + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
3296 /* Try to avoid a swapstorm if len is impossible to satisfy */
3297 if (sbinfo->max_blocks && end - start > sbinfo->max_blocks) {
3298 error = -ENOSPC;
3299 goto out;
3300 }
3301
3302 shmem_falloc.waitq = NULL;
3303 shmem_falloc.start = start;
3304 shmem_falloc.next = start;
3305 shmem_falloc.nr_falloced = 0;
3306 shmem_falloc.nr_unswapped = 0;
3307 spin_lock(&inode->i_lock);
3308 inode->i_private = &shmem_falloc;
3309 spin_unlock(&inode->i_lock);
3310
3311 /*
3312 * info->fallocend is only relevant when huge pages might be
3313 * involved: to prevent split_huge_page() freeing fallocated
3314 * pages when FALLOC_FL_KEEP_SIZE committed beyond i_size.
3315 */
3316 undo_fallocend = info->fallocend;
3317 if (info->fallocend < end)
3318 info->fallocend = end;
3319
3320 for (index = start; index < end; ) {
3321 struct folio *folio;
3322
3323 /*
3324 * Check for fatal signal so that we abort early in OOM
3325 * situations. We don't want to abort in case of non-fatal
3326 * signals as large fallocate can take noticeable time and
3327 * e.g. periodic timers may result in fallocate constantly
3328 * restarting.
3329 */
3330 if (fatal_signal_pending(current))
3331 error = -EINTR;
3332 else if (shmem_falloc.nr_unswapped > shmem_falloc.nr_falloced)
3333 error = -ENOMEM;
3334 else
3335 error = shmem_get_folio(inode, index, &folio,
3336 SGP_FALLOC);
3337 if (error) {
3338 info->fallocend = undo_fallocend;
3339 /* Remove the !uptodate folios we added */
3340 if (index > start) {
3341 shmem_undo_range(inode,
3342 (loff_t)start << PAGE_SHIFT,
3343 ((loff_t)index << PAGE_SHIFT) - 1, true);
3344 }
3345 goto undone;
3346 }
3347
3348 /*
3349 * Here is a more important optimization than it appears:
3350 * a second SGP_FALLOC on the same large folio will clear it,
3351 * making it uptodate and un-undoable if we fail later.
3352 */
3353 index = folio_next_index(folio);
3354 /* Beware 32-bit wraparound */
3355 if (!index)
3356 index--;
3357
3358 /*
3359 * Inform shmem_writepage() how far we have reached.
3360 * No need for lock or barrier: we have the page lock.
3361 */
3362 if (!folio_test_uptodate(folio))
3363 shmem_falloc.nr_falloced += index - shmem_falloc.next;
3364 shmem_falloc.next = index;
3365
3366 /*
3367 * If !uptodate, leave it that way so that freeable folios
3368 * can be recognized if we need to rollback on error later.
3369 * But mark it dirty so that memory pressure will swap rather
3370 * than free the folios we are allocating (and SGP_CACHE folios
3371 * might still be clean: we now need to mark those dirty too).
3372 */
3373 folio_mark_dirty(folio);
3374 folio_unlock(folio);
3375 folio_put(folio);
3376 cond_resched();
3377 }
3378
3379 if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size)
3380 i_size_write(inode, offset + len);
3381 undone:
3382 spin_lock(&inode->i_lock);
3383 inode->i_private = NULL;
3384 spin_unlock(&inode->i_lock);
3385 out:
3386 if (!error)
3387 file_modified(file);
3388 inode_unlock(inode);
3389 return error;
3390 }
3391
3392 static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf)
3393 {
3394 struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb);
3395
3396 buf->f_type = TMPFS_MAGIC;
3397 buf->f_bsize = PAGE_SIZE;
3398 buf->f_namelen = NAME_MAX;
3399 if (sbinfo->max_blocks) {
3400 buf->f_blocks = sbinfo->max_blocks;
3401 buf->f_bavail =
3402 buf->f_bfree = sbinfo->max_blocks -
3403 percpu_counter_sum(&sbinfo->used_blocks);
3404 }
3405 if (sbinfo->max_inodes) {
3406 buf->f_files = sbinfo->max_inodes;
3407 buf->f_ffree = sbinfo->free_ispace / BOGO_INODE_SIZE;
3408 }
3409 /* else leave those fields 0 like simple_statfs */
3410
3411 buf->f_fsid = uuid_to_fsid(dentry->d_sb->s_uuid.b);
3412
3413 return 0;
3414 }
3415
3416 /*
3417 * File creation. Allocate an inode, and we're done..
3418 */
3419 static int
3420 shmem_mknod(struct mnt_idmap *idmap, struct inode *dir,
3421 struct dentry *dentry, umode_t mode, dev_t dev)
3422 {
3423 struct inode *inode;
3424 int error;
3425
3426 inode = shmem_get_inode(idmap, dir->i_sb, dir, mode, dev, VM_NORESERVE);
3427 if (IS_ERR(inode))
3428 return PTR_ERR(inode);
3429
3430 error = simple_acl_create(dir, inode);
3431 if (error)
3432 goto out_iput;
3433 error = security_inode_init_security(inode, dir, &dentry->d_name,
3434 shmem_initxattrs, NULL);
3435 if (error && error != -EOPNOTSUPP)
3436 goto out_iput;
3437
3438 error = simple_offset_add(shmem_get_offset_ctx(dir), dentry);
3439 if (error)
3440 goto out_iput;
3441
3442 dir->i_size += BOGO_DIRENT_SIZE;
3443 inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir));
3444 inode_inc_iversion(dir);
3445 d_instantiate(dentry, inode);
3446 dget(dentry); /* Extra count - pin the dentry in core */
3447 return error;
3448
3449 out_iput:
3450 iput(inode);
3451 return error;
3452 }
3453
3454 static int
3455 shmem_tmpfile(struct mnt_idmap *idmap, struct inode *dir,
3456 struct file *file, umode_t mode)
3457 {
3458 struct inode *inode;
3459 int error;
3460
3461 inode = shmem_get_inode(idmap, dir->i_sb, dir, mode, 0, VM_NORESERVE);
3462 if (IS_ERR(inode)) {
3463 error = PTR_ERR(inode);
3464 goto err_out;
3465 }
3466 error = security_inode_init_security(inode, dir, NULL,
3467 shmem_initxattrs, NULL);
3468 if (error && error != -EOPNOTSUPP)
3469 goto out_iput;
3470 error = simple_acl_create(dir, inode);
3471 if (error)
3472 goto out_iput;
3473 d_tmpfile(file, inode);
3474
3475 err_out:
3476 return finish_open_simple(file, error);
3477 out_iput:
3478 iput(inode);
3479 return error;
3480 }
3481
3482 static int shmem_mkdir(struct mnt_idmap *idmap, struct inode *dir,
3483 struct dentry *dentry, umode_t mode)
3484 {
3485 int error;
3486
3487 error = shmem_mknod(idmap, dir, dentry, mode | S_IFDIR, 0);
3488 if (error)
3489 return error;
3490 inc_nlink(dir);
3491 return 0;
3492 }
3493
3494 static int shmem_create(struct mnt_idmap *idmap, struct inode *dir,
3495 struct dentry *dentry, umode_t mode, bool excl)
3496 {
3497 return shmem_mknod(idmap, dir, dentry, mode | S_IFREG, 0);
3498 }
3499
3500 /*
3501 * Link a file..
3502 */
3503 static int shmem_link(struct dentry *old_dentry, struct inode *dir,
3504 struct dentry *dentry)
3505 {
3506 struct inode *inode = d_inode(old_dentry);
3507 int ret = 0;
3508
3509 /*
3510 * No ordinary (disk based) filesystem counts links as inodes;
3511 * but each new link needs a new dentry, pinning lowmem, and
3512 * tmpfs dentries cannot be pruned until they are unlinked.
3513 * But if an O_TMPFILE file is linked into the tmpfs, the
3514 * first link must skip that, to get the accounting right.
3515 */
3516 if (inode->i_nlink) {
3517 ret = shmem_reserve_inode(inode->i_sb, NULL);
3518 if (ret)
3519 goto out;
3520 }
3521
3522 ret = simple_offset_add(shmem_get_offset_ctx(dir), dentry);
3523 if (ret) {
3524 if (inode->i_nlink)
3525 shmem_free_inode(inode->i_sb, 0);
3526 goto out;
3527 }
3528
3529 dir->i_size += BOGO_DIRENT_SIZE;
3530 inode_set_mtime_to_ts(dir,
3531 inode_set_ctime_to_ts(dir, inode_set_ctime_current(inode)));
3532 inode_inc_iversion(dir);
3533 inc_nlink(inode);
3534 ihold(inode); /* New dentry reference */
3535 dget(dentry); /* Extra pinning count for the created dentry */
3536 d_instantiate(dentry, inode);
3537 out:
3538 return ret;
3539 }
3540
3541 static int shmem_unlink(struct inode *dir, struct dentry *dentry)
3542 {
3543 struct inode *inode = d_inode(dentry);
3544
3545 if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode))
3546 shmem_free_inode(inode->i_sb, 0);
3547
3548 simple_offset_remove(shmem_get_offset_ctx(dir), dentry);
3549
3550 dir->i_size -= BOGO_DIRENT_SIZE;
3551 inode_set_mtime_to_ts(dir,
3552 inode_set_ctime_to_ts(dir, inode_set_ctime_current(inode)));
3553 inode_inc_iversion(dir);
3554 drop_nlink(inode);
3555 dput(dentry); /* Undo the count from "create" - does all the work */
3556 return 0;
3557 }
3558
3559 static int shmem_rmdir(struct inode *dir, struct dentry *dentry)
3560 {
3561 if (!simple_offset_empty(dentry))
3562 return -ENOTEMPTY;
3563
3564 drop_nlink(d_inode(dentry));
3565 drop_nlink(dir);
3566 return shmem_unlink(dir, dentry);
3567 }
3568
3569 static int shmem_whiteout(struct mnt_idmap *idmap,
3570 struct inode *old_dir, struct dentry *old_dentry)
3571 {
3572 struct dentry *whiteout;
3573 int error;
3574
3575 whiteout = d_alloc(old_dentry->d_parent, &old_dentry->d_name);
3576 if (!whiteout)
3577 return -ENOMEM;
3578
3579 error = shmem_mknod(idmap, old_dir, whiteout,
3580 S_IFCHR | WHITEOUT_MODE, WHITEOUT_DEV);
3581 dput(whiteout);
3582 if (error)
3583 return error;
3584
3585 /*
3586 * Cheat and hash the whiteout while the old dentry is still in
3587 * place, instead of playing games with FS_RENAME_DOES_D_MOVE.
3588 *
3589 * d_lookup() will consistently find one of them at this point,
3590 * not sure which one, but that isn't even important.
3591 */
3592 d_rehash(whiteout);
3593 return 0;
3594 }
3595
3596 /*
3597 * The VFS layer already does all the dentry stuff for rename,
3598 * we just have to decrement the usage count for the target if
3599 * it exists so that the VFS layer correctly free's it when it
3600 * gets overwritten.
3601 */
3602 static int shmem_rename2(struct mnt_idmap *idmap,
3603 struct inode *old_dir, struct dentry *old_dentry,
3604 struct inode *new_dir, struct dentry *new_dentry,
3605 unsigned int flags)
3606 {
3607 struct inode *inode = d_inode(old_dentry);
3608 int they_are_dirs = S_ISDIR(inode->i_mode);
3609 int error;
3610
3611 if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT))
3612 return -EINVAL;
3613
3614 if (flags & RENAME_EXCHANGE)
3615 return simple_offset_rename_exchange(old_dir, old_dentry,
3616 new_dir, new_dentry);
3617
3618 if (!simple_offset_empty(new_dentry))
3619 return -ENOTEMPTY;
3620
3621 if (flags & RENAME_WHITEOUT) {
3622 error = shmem_whiteout(idmap, old_dir, old_dentry);
3623 if (error)
3624 return error;
3625 }
3626
3627 error = simple_offset_rename(old_dir, old_dentry, new_dir, new_dentry);
3628 if (error)
3629 return error;
3630
3631 if (d_really_is_positive(new_dentry)) {
3632 (void) shmem_unlink(new_dir, new_dentry);
3633 if (they_are_dirs) {
3634 drop_nlink(d_inode(new_dentry));
3635 drop_nlink(old_dir);
3636 }
3637 } else if (they_are_dirs) {
3638 drop_nlink(old_dir);
3639 inc_nlink(new_dir);
3640 }
3641
3642 old_dir->i_size -= BOGO_DIRENT_SIZE;
3643 new_dir->i_size += BOGO_DIRENT_SIZE;
3644 simple_rename_timestamp(old_dir, old_dentry, new_dir, new_dentry);
3645 inode_inc_iversion(old_dir);
3646 inode_inc_iversion(new_dir);
3647 return 0;
3648 }
3649
3650 static int shmem_symlink(struct mnt_idmap *idmap, struct inode *dir,
3651 struct dentry *dentry, const char *symname)
3652 {
3653 int error;
3654 int len;
3655 struct inode *inode;
3656 struct folio *folio;
3657
3658 len = strlen(symname) + 1;
3659 if (len > PAGE_SIZE)
3660 return -ENAMETOOLONG;
3661
3662 inode = shmem_get_inode(idmap, dir->i_sb, dir, S_IFLNK | 0777, 0,
3663 VM_NORESERVE);
3664 if (IS_ERR(inode))
3665 return PTR_ERR(inode);
3666
3667 error = security_inode_init_security(inode, dir, &dentry->d_name,
3668 shmem_initxattrs, NULL);
3669 if (error && error != -EOPNOTSUPP)
3670 goto out_iput;
3671
3672 error = simple_offset_add(shmem_get_offset_ctx(dir), dentry);
3673 if (error)
3674 goto out_iput;
3675
3676 inode->i_size = len-1;
3677 if (len <= SHORT_SYMLINK_LEN) {
3678 inode->i_link = kmemdup(symname, len, GFP_KERNEL);
3679 if (!inode->i_link) {
3680 error = -ENOMEM;
3681 goto out_remove_offset;
3682 }
3683 inode->i_op = &shmem_short_symlink_operations;
3684 } else {
3685 inode_nohighmem(inode);
3686 inode->i_mapping->a_ops = &shmem_aops;
3687 error = shmem_get_folio(inode, 0, &folio, SGP_WRITE);
3688 if (error)
3689 goto out_remove_offset;
3690 inode->i_op = &shmem_symlink_inode_operations;
3691 memcpy(folio_address(folio), symname, len);
3692 folio_mark_uptodate(folio);
3693 folio_mark_dirty(folio);
3694 folio_unlock(folio);
3695 folio_put(folio);
3696 }
3697 dir->i_size += BOGO_DIRENT_SIZE;
3698 inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir));
3699 inode_inc_iversion(dir);
3700 d_instantiate(dentry, inode);
3701 dget(dentry);
3702 return 0;
3703
3704 out_remove_offset:
3705 simple_offset_remove(shmem_get_offset_ctx(dir), dentry);
3706 out_iput:
3707 iput(inode);
3708 return error;
3709 }
3710
3711 static void shmem_put_link(void *arg)
3712 {
3713 folio_mark_accessed(arg);
3714 folio_put(arg);
3715 }
3716
3717 static const char *shmem_get_link(struct dentry *dentry, struct inode *inode,
3718 struct delayed_call *done)
3719 {
3720 struct folio *folio = NULL;
3721 int error;
3722
3723 if (!dentry) {
3724 folio = filemap_get_folio(inode->i_mapping, 0);
3725 if (IS_ERR(folio))
3726 return ERR_PTR(-ECHILD);
3727 if (PageHWPoison(folio_page(folio, 0)) ||
3728 !folio_test_uptodate(folio)) {
3729 folio_put(folio);
3730 return ERR_PTR(-ECHILD);
3731 }
3732 } else {
3733 error = shmem_get_folio(inode, 0, &folio, SGP_READ);
3734 if (error)
3735 return ERR_PTR(error);
3736 if (!folio)
3737 return ERR_PTR(-ECHILD);
3738 if (PageHWPoison(folio_page(folio, 0))) {
3739 folio_unlock(folio);
3740 folio_put(folio);
3741 return ERR_PTR(-ECHILD);
3742 }
3743 folio_unlock(folio);
3744 }
3745 set_delayed_call(done, shmem_put_link, folio);
3746 return folio_address(folio);
3747 }
3748
3749 #ifdef CONFIG_TMPFS_XATTR
3750
3751 static int shmem_fileattr_get(struct dentry *dentry, struct fileattr *fa)
3752 {
3753 struct shmem_inode_info *info = SHMEM_I(d_inode(dentry));
3754
3755 fileattr_fill_flags(fa, info->fsflags & SHMEM_FL_USER_VISIBLE);
3756
3757 return 0;
3758 }
3759
3760 static int shmem_fileattr_set(struct mnt_idmap *idmap,
3761 struct dentry *dentry, struct fileattr *fa)
3762 {
3763 struct inode *inode = d_inode(dentry);
3764 struct shmem_inode_info *info = SHMEM_I(inode);
3765
3766 if (fileattr_has_fsx(fa))
3767 return -EOPNOTSUPP;
3768 if (fa->flags & ~SHMEM_FL_USER_MODIFIABLE)
3769 return -EOPNOTSUPP;
3770
3771 info->fsflags = (info->fsflags & ~SHMEM_FL_USER_MODIFIABLE) |
3772 (fa->flags & SHMEM_FL_USER_MODIFIABLE);
3773
3774 shmem_set_inode_flags(inode, info->fsflags);
3775 inode_set_ctime_current(inode);
3776 inode_inc_iversion(inode);
3777 return 0;
3778 }
3779
3780 /*
3781 * Superblocks without xattr inode operations may get some security.* xattr
3782 * support from the LSM "for free". As soon as we have any other xattrs
3783 * like ACLs, we also need to implement the security.* handlers at
3784 * filesystem level, though.
3785 */
3786
3787 /*
3788 * Callback for security_inode_init_security() for acquiring xattrs.
3789 */
3790 static int shmem_initxattrs(struct inode *inode,
3791 const struct xattr *xattr_array, void *fs_info)
3792 {
3793 struct shmem_inode_info *info = SHMEM_I(inode);
3794 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
3795 const struct xattr *xattr;
3796 struct simple_xattr *new_xattr;
3797 size_t ispace = 0;
3798 size_t len;
3799
3800 if (sbinfo->max_inodes) {
3801 for (xattr = xattr_array; xattr->name != NULL; xattr++) {
3802 ispace += simple_xattr_space(xattr->name,
3803 xattr->value_len + XATTR_SECURITY_PREFIX_LEN);
3804 }
3805 if (ispace) {
3806 raw_spin_lock(&sbinfo->stat_lock);
3807 if (sbinfo->free_ispace < ispace)
3808 ispace = 0;
3809 else
3810 sbinfo->free_ispace -= ispace;
3811 raw_spin_unlock(&sbinfo->stat_lock);
3812 if (!ispace)
3813 return -ENOSPC;
3814 }
3815 }
3816
3817 for (xattr = xattr_array; xattr->name != NULL; xattr++) {
3818 new_xattr = simple_xattr_alloc(xattr->value, xattr->value_len);
3819 if (!new_xattr)
3820 break;
3821
3822 len = strlen(xattr->name) + 1;
3823 new_xattr->name = kmalloc(XATTR_SECURITY_PREFIX_LEN + len,
3824 GFP_KERNEL_ACCOUNT);
3825 if (!new_xattr->name) {
3826 kvfree(new_xattr);
3827 break;
3828 }
3829
3830 memcpy(new_xattr->name, XATTR_SECURITY_PREFIX,
3831 XATTR_SECURITY_PREFIX_LEN);
3832 memcpy(new_xattr->name + XATTR_SECURITY_PREFIX_LEN,
3833 xattr->name, len);
3834
3835 simple_xattr_add(&info->xattrs, new_xattr);
3836 }
3837
3838 if (xattr->name != NULL) {
3839 if (ispace) {
3840 raw_spin_lock(&sbinfo->stat_lock);
3841 sbinfo->free_ispace += ispace;
3842 raw_spin_unlock(&sbinfo->stat_lock);
3843 }
3844 simple_xattrs_free(&info->xattrs, NULL);
3845 return -ENOMEM;
3846 }
3847
3848 return 0;
3849 }
3850
3851 static int shmem_xattr_handler_get(const struct xattr_handler *handler,
3852 struct dentry *unused, struct inode *inode,
3853 const char *name, void *buffer, size_t size)
3854 {
3855 struct shmem_inode_info *info = SHMEM_I(inode);
3856
3857 name = xattr_full_name(handler, name);
3858 return simple_xattr_get(&info->xattrs, name, buffer, size);
3859 }
3860
3861 static int shmem_xattr_handler_set(const struct xattr_handler *handler,
3862 struct mnt_idmap *idmap,
3863 struct dentry *unused, struct inode *inode,
3864 const char *name, const void *value,
3865 size_t size, int flags)
3866 {
3867 struct shmem_inode_info *info = SHMEM_I(inode);
3868 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
3869 struct simple_xattr *old_xattr;
3870 size_t ispace = 0;
3871
3872 name = xattr_full_name(handler, name);
3873 if (value && sbinfo->max_inodes) {
3874 ispace = simple_xattr_space(name, size);
3875 raw_spin_lock(&sbinfo->stat_lock);
3876 if (sbinfo->free_ispace < ispace)
3877 ispace = 0;
3878 else
3879 sbinfo->free_ispace -= ispace;
3880 raw_spin_unlock(&sbinfo->stat_lock);
3881 if (!ispace)
3882 return -ENOSPC;
3883 }
3884
3885 old_xattr = simple_xattr_set(&info->xattrs, name, value, size, flags);
3886 if (!IS_ERR(old_xattr)) {
3887 ispace = 0;
3888 if (old_xattr && sbinfo->max_inodes)
3889 ispace = simple_xattr_space(old_xattr->name,
3890 old_xattr->size);
3891 simple_xattr_free(old_xattr);
3892 old_xattr = NULL;
3893 inode_set_ctime_current(inode);
3894 inode_inc_iversion(inode);
3895 }
3896 if (ispace) {
3897 raw_spin_lock(&sbinfo->stat_lock);
3898 sbinfo->free_ispace += ispace;
3899 raw_spin_unlock(&sbinfo->stat_lock);
3900 }
3901 return PTR_ERR(old_xattr);
3902 }
3903
3904 static const struct xattr_handler shmem_security_xattr_handler = {
3905 .prefix = XATTR_SECURITY_PREFIX,
3906 .get = shmem_xattr_handler_get,
3907 .set = shmem_xattr_handler_set,
3908 };
3909
3910 static const struct xattr_handler shmem_trusted_xattr_handler = {
3911 .prefix = XATTR_TRUSTED_PREFIX,
3912 .get = shmem_xattr_handler_get,
3913 .set = shmem_xattr_handler_set,
3914 };
3915
3916 static const struct xattr_handler shmem_user_xattr_handler = {
3917 .prefix = XATTR_USER_PREFIX,
3918 .get = shmem_xattr_handler_get,
3919 .set = shmem_xattr_handler_set,
3920 };
3921
3922 static const struct xattr_handler * const shmem_xattr_handlers[] = {
3923 &shmem_security_xattr_handler,
3924 &shmem_trusted_xattr_handler,
3925 &shmem_user_xattr_handler,
3926 NULL
3927 };
3928
3929 static ssize_t shmem_listxattr(struct dentry *dentry, char *buffer, size_t size)
3930 {
3931 struct shmem_inode_info *info = SHMEM_I(d_inode(dentry));
3932 return simple_xattr_list(d_inode(dentry), &info->xattrs, buffer, size);
3933 }
3934 #endif /* CONFIG_TMPFS_XATTR */
3935
3936 static const struct inode_operations shmem_short_symlink_operations = {
3937 .getattr = shmem_getattr,
3938 .setattr = shmem_setattr,
3939 .get_link = simple_get_link,
3940 #ifdef CONFIG_TMPFS_XATTR
3941 .listxattr = shmem_listxattr,
3942 #endif
3943 };
3944
3945 static const struct inode_operations shmem_symlink_inode_operations = {
3946 .getattr = shmem_getattr,
3947 .setattr = shmem_setattr,
3948 .get_link = shmem_get_link,
3949 #ifdef CONFIG_TMPFS_XATTR
3950 .listxattr = shmem_listxattr,
3951 #endif
3952 };
3953
3954 static struct dentry *shmem_get_parent(struct dentry *child)
3955 {
3956 return ERR_PTR(-ESTALE);
3957 }
3958
3959 static int shmem_match(struct inode *ino, void *vfh)
3960 {
3961 __u32 *fh = vfh;
3962 __u64 inum = fh[2];
3963 inum = (inum << 32) | fh[1];
3964 return ino->i_ino == inum && fh[0] == ino->i_generation;
3965 }
3966
3967 /* Find any alias of inode, but prefer a hashed alias */
3968 static struct dentry *shmem_find_alias(struct inode *inode)
3969 {
3970 struct dentry *alias = d_find_alias(inode);
3971
3972 return alias ?: d_find_any_alias(inode);
3973 }
3974
3975 static struct dentry *shmem_fh_to_dentry(struct super_block *sb,
3976 struct fid *fid, int fh_len, int fh_type)
3977 {
3978 struct inode *inode;
3979 struct dentry *dentry = NULL;
3980 u64 inum;
3981
3982 if (fh_len < 3)
3983 return NULL;
3984
3985 inum = fid->raw[2];
3986 inum = (inum << 32) | fid->raw[1];
3987
3988 inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]),
3989 shmem_match, fid->raw);
3990 if (inode) {
3991 dentry = shmem_find_alias(inode);
3992 iput(inode);
3993 }
3994
3995 return dentry;
3996 }
3997
3998 static int shmem_encode_fh(struct inode *inode, __u32 *fh, int *len,
3999 struct inode *parent)
4000 {
4001 if (*len < 3) {
4002 *len = 3;
4003 return FILEID_INVALID;
4004 }
4005
4006 if (inode_unhashed(inode)) {
4007 /* Unfortunately insert_inode_hash is not idempotent,
4008 * so as we hash inodes here rather than at creation
4009 * time, we need a lock to ensure we only try
4010 * to do it once
4011 */
4012 static DEFINE_SPINLOCK(lock);
4013 spin_lock(&lock);
4014 if (inode_unhashed(inode))
4015 __insert_inode_hash(inode,
4016 inode->i_ino + inode->i_generation);
4017 spin_unlock(&lock);
4018 }
4019
4020 fh[0] = inode->i_generation;
4021 fh[1] = inode->i_ino;
4022 fh[2] = ((__u64)inode->i_ino) >> 32;
4023
4024 *len = 3;
4025 return 1;
4026 }
4027
4028 static const struct export_operations shmem_export_ops = {
4029 .get_parent = shmem_get_parent,
4030 .encode_fh = shmem_encode_fh,
4031 .fh_to_dentry = shmem_fh_to_dentry,
4032 };
4033
4034 enum shmem_param {
4035 Opt_gid,
4036 Opt_huge,
4037 Opt_mode,
4038 Opt_mpol,
4039 Opt_nr_blocks,
4040 Opt_nr_inodes,
4041 Opt_size,
4042 Opt_uid,
4043 Opt_inode32,
4044 Opt_inode64,
4045 Opt_noswap,
4046 Opt_quota,
4047 Opt_usrquota,
4048 Opt_grpquota,
4049 Opt_usrquota_block_hardlimit,
4050 Opt_usrquota_inode_hardlimit,
4051 Opt_grpquota_block_hardlimit,
4052 Opt_grpquota_inode_hardlimit,
4053 };
4054
4055 static const struct constant_table shmem_param_enums_huge[] = {
4056 {"never", SHMEM_HUGE_NEVER },
4057 {"always", SHMEM_HUGE_ALWAYS },
4058 {"within_size", SHMEM_HUGE_WITHIN_SIZE },
4059 {"advise", SHMEM_HUGE_ADVISE },
4060 {}
4061 };
4062
4063 const struct fs_parameter_spec shmem_fs_parameters[] = {
4064 fsparam_gid ("gid", Opt_gid),
4065 fsparam_enum ("huge", Opt_huge, shmem_param_enums_huge),
4066 fsparam_u32oct("mode", Opt_mode),
4067 fsparam_string("mpol", Opt_mpol),
4068 fsparam_string("nr_blocks", Opt_nr_blocks),
4069 fsparam_string("nr_inodes", Opt_nr_inodes),
4070 fsparam_string("size", Opt_size),
4071 fsparam_uid ("uid", Opt_uid),
4072 fsparam_flag ("inode32", Opt_inode32),
4073 fsparam_flag ("inode64", Opt_inode64),
4074 fsparam_flag ("noswap", Opt_noswap),
4075 #ifdef CONFIG_TMPFS_QUOTA
4076 fsparam_flag ("quota", Opt_quota),
4077 fsparam_flag ("usrquota", Opt_usrquota),
4078 fsparam_flag ("grpquota", Opt_grpquota),
4079 fsparam_string("usrquota_block_hardlimit", Opt_usrquota_block_hardlimit),
4080 fsparam_string("usrquota_inode_hardlimit", Opt_usrquota_inode_hardlimit),
4081 fsparam_string("grpquota_block_hardlimit", Opt_grpquota_block_hardlimit),
4082 fsparam_string("grpquota_inode_hardlimit", Opt_grpquota_inode_hardlimit),
4083 #endif
4084 {}
4085 };
4086
4087 static int shmem_parse_one(struct fs_context *fc, struct fs_parameter *param)
4088 {
4089 struct shmem_options *ctx = fc->fs_private;
4090 struct fs_parse_result result;
4091 unsigned long long size;
4092 char *rest;
4093 int opt;
4094 kuid_t kuid;
4095 kgid_t kgid;
4096
4097 opt = fs_parse(fc, shmem_fs_parameters, param, &result);
4098 if (opt < 0)
4099 return opt;
4100
4101 switch (opt) {
4102 case Opt_size:
4103 size = memparse(param->string, &rest);
4104 if (*rest == '%') {
4105 size <<= PAGE_SHIFT;
4106 size *= totalram_pages();
4107 do_div(size, 100);
4108 rest++;
4109 }
4110 if (*rest)
4111 goto bad_value;
4112 ctx->blocks = DIV_ROUND_UP(size, PAGE_SIZE);
4113 ctx->seen |= SHMEM_SEEN_BLOCKS;
4114 break;
4115 case Opt_nr_blocks:
4116 ctx->blocks = memparse(param->string, &rest);
4117 if (*rest || ctx->blocks > LONG_MAX)
4118 goto bad_value;
4119 ctx->seen |= SHMEM_SEEN_BLOCKS;
4120 break;
4121 case Opt_nr_inodes:
4122 ctx->inodes = memparse(param->string, &rest);
4123 if (*rest || ctx->inodes > ULONG_MAX / BOGO_INODE_SIZE)
4124 goto bad_value;
4125 ctx->seen |= SHMEM_SEEN_INODES;
4126 break;
4127 case Opt_mode:
4128 ctx->mode = result.uint_32 & 07777;
4129 break;
4130 case Opt_uid:
4131 kuid = result.uid;
4132
4133 /*
4134 * The requested uid must be representable in the
4135 * filesystem's idmapping.
4136 */
4137 if (!kuid_has_mapping(fc->user_ns, kuid))
4138 goto bad_value;
4139
4140 ctx->uid = kuid;
4141 break;
4142 case Opt_gid:
4143 kgid = result.gid;
4144
4145 /*
4146 * The requested gid must be representable in the
4147 * filesystem's idmapping.
4148 */
4149 if (!kgid_has_mapping(fc->user_ns, kgid))
4150 goto bad_value;
4151
4152 ctx->gid = kgid;
4153 break;
4154 case Opt_huge:
4155 ctx->huge = result.uint_32;
4156 if (ctx->huge != SHMEM_HUGE_NEVER &&
4157 !(IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&
4158 has_transparent_hugepage()))
4159 goto unsupported_parameter;
4160 ctx->seen |= SHMEM_SEEN_HUGE;
4161 break;
4162 case Opt_mpol:
4163 if (IS_ENABLED(CONFIG_NUMA)) {
4164 mpol_put(ctx->mpol);
4165 ctx->mpol = NULL;
4166 if (mpol_parse_str(param->string, &ctx->mpol))
4167 goto bad_value;
4168 break;
4169 }
4170 goto unsupported_parameter;
4171 case Opt_inode32:
4172 ctx->full_inums = false;
4173 ctx->seen |= SHMEM_SEEN_INUMS;
4174 break;
4175 case Opt_inode64:
4176 if (sizeof(ino_t) < 8) {
4177 return invalfc(fc,
4178 "Cannot use inode64 with <64bit inums in kernel\n");
4179 }
4180 ctx->full_inums = true;
4181 ctx->seen |= SHMEM_SEEN_INUMS;
4182 break;
4183 case Opt_noswap:
4184 if ((fc->user_ns != &init_user_ns) || !capable(CAP_SYS_ADMIN)) {
4185 return invalfc(fc,
4186 "Turning off swap in unprivileged tmpfs mounts unsupported");
4187 }
4188 ctx->noswap = true;
4189 ctx->seen |= SHMEM_SEEN_NOSWAP;
4190 break;
4191 case Opt_quota:
4192 if (fc->user_ns != &init_user_ns)
4193 return invalfc(fc, "Quotas in unprivileged tmpfs mounts are unsupported");
4194 ctx->seen |= SHMEM_SEEN_QUOTA;
4195 ctx->quota_types |= (QTYPE_MASK_USR | QTYPE_MASK_GRP);
4196 break;
4197 case Opt_usrquota:
4198 if (fc->user_ns != &init_user_ns)
4199 return invalfc(fc, "Quotas in unprivileged tmpfs mounts are unsupported");
4200 ctx->seen |= SHMEM_SEEN_QUOTA;
4201 ctx->quota_types |= QTYPE_MASK_USR;
4202 break;
4203 case Opt_grpquota:
4204 if (fc->user_ns != &init_user_ns)
4205 return invalfc(fc, "Quotas in unprivileged tmpfs mounts are unsupported");
4206 ctx->seen |= SHMEM_SEEN_QUOTA;
4207 ctx->quota_types |= QTYPE_MASK_GRP;
4208 break;
4209 case Opt_usrquota_block_hardlimit:
4210 size = memparse(param->string, &rest);
4211 if (*rest || !size)
4212 goto bad_value;
4213 if (size > SHMEM_QUOTA_MAX_SPC_LIMIT)
4214 return invalfc(fc,
4215 "User quota block hardlimit too large.");
4216 ctx->qlimits.usrquota_bhardlimit = size;
4217 break;
4218 case Opt_grpquota_block_hardlimit:
4219 size = memparse(param->string, &rest);
4220 if (*rest || !size)
4221 goto bad_value;
4222 if (size > SHMEM_QUOTA_MAX_SPC_LIMIT)
4223 return invalfc(fc,
4224 "Group quota block hardlimit too large.");
4225 ctx->qlimits.grpquota_bhardlimit = size;
4226 break;
4227 case Opt_usrquota_inode_hardlimit:
4228 size = memparse(param->string, &rest);
4229 if (*rest || !size)
4230 goto bad_value;
4231 if (size > SHMEM_QUOTA_MAX_INO_LIMIT)
4232 return invalfc(fc,
4233 "User quota inode hardlimit too large.");
4234 ctx->qlimits.usrquota_ihardlimit = size;
4235 break;
4236 case Opt_grpquota_inode_hardlimit:
4237 size = memparse(param->string, &rest);
4238 if (*rest || !size)
4239 goto bad_value;
4240 if (size > SHMEM_QUOTA_MAX_INO_LIMIT)
4241 return invalfc(fc,
4242 "Group quota inode hardlimit too large.");
4243 ctx->qlimits.grpquota_ihardlimit = size;
4244 break;
4245 }
4246 return 0;
4247
4248 unsupported_parameter:
4249 return invalfc(fc, "Unsupported parameter '%s'", param->key);
4250 bad_value:
4251 return invalfc(fc, "Bad value for '%s'", param->key);
4252 }
4253
4254 static int shmem_parse_options(struct fs_context *fc, void *data)
4255 {
4256 char *options = data;
4257
4258 if (options) {
4259 int err = security_sb_eat_lsm_opts(options, &fc->security);
4260 if (err)
4261 return err;
4262 }
4263
4264 while (options != NULL) {
4265 char *this_char = options;
4266 for (;;) {
4267 /*
4268 * NUL-terminate this option: unfortunately,
4269 * mount options form a comma-separated list,
4270 * but mpol's nodelist may also contain commas.
4271 */
4272 options = strchr(options, ',');
4273 if (options == NULL)
4274 break;
4275 options++;
4276 if (!isdigit(*options)) {
4277 options[-1] = '\0';
4278 break;
4279 }
4280 }
4281 if (*this_char) {
4282 char *value = strchr(this_char, '=');
4283 size_t len = 0;
4284 int err;
4285
4286 if (value) {
4287 *value++ = '\0';
4288 len = strlen(value);
4289 }
4290 err = vfs_parse_fs_string(fc, this_char, value, len);
4291 if (err < 0)
4292 return err;
4293 }
4294 }
4295 return 0;
4296 }
4297
4298 /*
4299 * Reconfigure a shmem filesystem.
4300 */
4301 static int shmem_reconfigure(struct fs_context *fc)
4302 {
4303 struct shmem_options *ctx = fc->fs_private;
4304 struct shmem_sb_info *sbinfo = SHMEM_SB(fc->root->d_sb);
4305 unsigned long used_isp;
4306 struct mempolicy *mpol = NULL;
4307 const char *err;
4308
4309 raw_spin_lock(&sbinfo->stat_lock);
4310 used_isp = sbinfo->max_inodes * BOGO_INODE_SIZE - sbinfo->free_ispace;
4311
4312 if ((ctx->seen & SHMEM_SEEN_BLOCKS) && ctx->blocks) {
4313 if (!sbinfo->max_blocks) {
4314 err = "Cannot retroactively limit size";
4315 goto out;
4316 }
4317 if (percpu_counter_compare(&sbinfo->used_blocks,
4318 ctx->blocks) > 0) {
4319 err = "Too small a size for current use";
4320 goto out;
4321 }
4322 }
4323 if ((ctx->seen & SHMEM_SEEN_INODES) && ctx->inodes) {
4324 if (!sbinfo->max_inodes) {
4325 err = "Cannot retroactively limit inodes";
4326 goto out;
4327 }
4328 if (ctx->inodes * BOGO_INODE_SIZE < used_isp) {
4329 err = "Too few inodes for current use";
4330 goto out;
4331 }
4332 }
4333
4334 if ((ctx->seen & SHMEM_SEEN_INUMS) && !ctx->full_inums &&
4335 sbinfo->next_ino > UINT_MAX) {
4336 err = "Current inum too high to switch to 32-bit inums";
4337 goto out;
4338 }
4339 if ((ctx->seen & SHMEM_SEEN_NOSWAP) && ctx->noswap && !sbinfo->noswap) {
4340 err = "Cannot disable swap on remount";
4341 goto out;
4342 }
4343 if (!(ctx->seen & SHMEM_SEEN_NOSWAP) && !ctx->noswap && sbinfo->noswap) {
4344 err = "Cannot enable swap on remount if it was disabled on first mount";
4345 goto out;
4346 }
4347
4348 if (ctx->seen & SHMEM_SEEN_QUOTA &&
4349 !sb_any_quota_loaded(fc->root->d_sb)) {
4350 err = "Cannot enable quota on remount";
4351 goto out;
4352 }
4353
4354 #ifdef CONFIG_TMPFS_QUOTA
4355 #define CHANGED_LIMIT(name) \
4356 (ctx->qlimits.name## hardlimit && \
4357 (ctx->qlimits.name## hardlimit != sbinfo->qlimits.name## hardlimit))
4358
4359 if (CHANGED_LIMIT(usrquota_b) || CHANGED_LIMIT(usrquota_i) ||
4360 CHANGED_LIMIT(grpquota_b) || CHANGED_LIMIT(grpquota_i)) {
4361 err = "Cannot change global quota limit on remount";
4362 goto out;
4363 }
4364 #endif /* CONFIG_TMPFS_QUOTA */
4365
4366 if (ctx->seen & SHMEM_SEEN_HUGE)
4367 sbinfo->huge = ctx->huge;
4368 if (ctx->seen & SHMEM_SEEN_INUMS)
4369 sbinfo->full_inums = ctx->full_inums;
4370 if (ctx->seen & SHMEM_SEEN_BLOCKS)
4371 sbinfo->max_blocks = ctx->blocks;
4372 if (ctx->seen & SHMEM_SEEN_INODES) {
4373 sbinfo->max_inodes = ctx->inodes;
4374 sbinfo->free_ispace = ctx->inodes * BOGO_INODE_SIZE - used_isp;
4375 }
4376
4377 /*
4378 * Preserve previous mempolicy unless mpol remount option was specified.
4379 */
4380 if (ctx->mpol) {
4381 mpol = sbinfo->mpol;
4382 sbinfo->mpol = ctx->mpol; /* transfers initial ref */
4383 ctx->mpol = NULL;
4384 }
4385
4386 if (ctx->noswap)
4387 sbinfo->noswap = true;
4388
4389 raw_spin_unlock(&sbinfo->stat_lock);
4390 mpol_put(mpol);
4391 return 0;
4392 out:
4393 raw_spin_unlock(&sbinfo->stat_lock);
4394 return invalfc(fc, "%s", err);
4395 }
4396
4397 static int shmem_show_options(struct seq_file *seq, struct dentry *root)
4398 {
4399 struct shmem_sb_info *sbinfo = SHMEM_SB(root->d_sb);
4400 struct mempolicy *mpol;
4401
4402 if (sbinfo->max_blocks != shmem_default_max_blocks())
4403 seq_printf(seq, ",size=%luk", K(sbinfo->max_blocks));
4404 if (sbinfo->max_inodes != shmem_default_max_inodes())
4405 seq_printf(seq, ",nr_inodes=%lu", sbinfo->max_inodes);
4406 if (sbinfo->mode != (0777 | S_ISVTX))
4407 seq_printf(seq, ",mode=%03ho", sbinfo->mode);
4408 if (!uid_eq(sbinfo->uid, GLOBAL_ROOT_UID))
4409 seq_printf(seq, ",uid=%u",
4410 from_kuid_munged(&init_user_ns, sbinfo->uid));
4411 if (!gid_eq(sbinfo->gid, GLOBAL_ROOT_GID))
4412 seq_printf(seq, ",gid=%u",
4413 from_kgid_munged(&init_user_ns, sbinfo->gid));
4414
4415 /*
4416 * Showing inode{64,32} might be useful even if it's the system default,
4417 * since then people don't have to resort to checking both here and
4418 * /proc/config.gz to confirm 64-bit inums were successfully applied
4419 * (which may not even exist if IKCONFIG_PROC isn't enabled).
4420 *
4421 * We hide it when inode64 isn't the default and we are using 32-bit
4422 * inodes, since that probably just means the feature isn't even under
4423 * consideration.
4424 *
4425 * As such:
4426 *
4427 * +-----------------+-----------------+
4428 * | TMPFS_INODE64=y | TMPFS_INODE64=n |
4429 * +------------------+-----------------+-----------------+
4430 * | full_inums=true | show | show |
4431 * | full_inums=false | show | hide |
4432 * +------------------+-----------------+-----------------+
4433 *
4434 */
4435 if (IS_ENABLED(CONFIG_TMPFS_INODE64) || sbinfo->full_inums)
4436 seq_printf(seq, ",inode%d", (sbinfo->full_inums ? 64 : 32));
4437 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
4438 /* Rightly or wrongly, show huge mount option unmasked by shmem_huge */
4439 if (sbinfo->huge)
4440 seq_printf(seq, ",huge=%s", shmem_format_huge(sbinfo->huge));
4441 #endif
4442 mpol = shmem_get_sbmpol(sbinfo);
4443 shmem_show_mpol(seq, mpol);
4444 mpol_put(mpol);
4445 if (sbinfo->noswap)
4446 seq_printf(seq, ",noswap");
4447 #ifdef CONFIG_TMPFS_QUOTA
4448 if (sb_has_quota_active(root->d_sb, USRQUOTA))
4449 seq_printf(seq, ",usrquota");
4450 if (sb_has_quota_active(root->d_sb, GRPQUOTA))
4451 seq_printf(seq, ",grpquota");
4452 if (sbinfo->qlimits.usrquota_bhardlimit)
4453 seq_printf(seq, ",usrquota_block_hardlimit=%lld",
4454 sbinfo->qlimits.usrquota_bhardlimit);
4455 if (sbinfo->qlimits.grpquota_bhardlimit)
4456 seq_printf(seq, ",grpquota_block_hardlimit=%lld",
4457 sbinfo->qlimits.grpquota_bhardlimit);
4458 if (sbinfo->qlimits.usrquota_ihardlimit)
4459 seq_printf(seq, ",usrquota_inode_hardlimit=%lld",
4460 sbinfo->qlimits.usrquota_ihardlimit);
4461 if (sbinfo->qlimits.grpquota_ihardlimit)
4462 seq_printf(seq, ",grpquota_inode_hardlimit=%lld",
4463 sbinfo->qlimits.grpquota_ihardlimit);
4464 #endif
4465 return 0;
4466 }
4467
4468 #endif /* CONFIG_TMPFS */
4469
4470 static void shmem_put_super(struct super_block *sb)
4471 {
4472 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
4473
4474 #ifdef CONFIG_TMPFS_QUOTA
4475 shmem_disable_quotas(sb);
4476 #endif
4477 free_percpu(sbinfo->ino_batch);
4478 percpu_counter_destroy(&sbinfo->used_blocks);
4479 mpol_put(sbinfo->mpol);
4480 kfree(sbinfo);
4481 sb->s_fs_info = NULL;
4482 }
4483
4484 static int shmem_fill_super(struct super_block *sb, struct fs_context *fc)
4485 {
4486 struct shmem_options *ctx = fc->fs_private;
4487 struct inode *inode;
4488 struct shmem_sb_info *sbinfo;
4489 int error = -ENOMEM;
4490
4491 /* Round up to L1_CACHE_BYTES to resist false sharing */
4492 sbinfo = kzalloc(max((int)sizeof(struct shmem_sb_info),
4493 L1_CACHE_BYTES), GFP_KERNEL);
4494 if (!sbinfo)
4495 return error;
4496
4497 sb->s_fs_info = sbinfo;
4498
4499 #ifdef CONFIG_TMPFS
4500 /*
4501 * Per default we only allow half of the physical ram per
4502 * tmpfs instance, limiting inodes to one per page of lowmem;
4503 * but the internal instance is left unlimited.
4504 */
4505 if (!(sb->s_flags & SB_KERNMOUNT)) {
4506 if (!(ctx->seen & SHMEM_SEEN_BLOCKS))
4507 ctx->blocks = shmem_default_max_blocks();
4508 if (!(ctx->seen & SHMEM_SEEN_INODES))
4509 ctx->inodes = shmem_default_max_inodes();
4510 if (!(ctx->seen & SHMEM_SEEN_INUMS))
4511 ctx->full_inums = IS_ENABLED(CONFIG_TMPFS_INODE64);
4512 sbinfo->noswap = ctx->noswap;
4513 } else {
4514 sb->s_flags |= SB_NOUSER;
4515 }
4516 sb->s_export_op = &shmem_export_ops;
4517 sb->s_flags |= SB_NOSEC | SB_I_VERSION;
4518 #else
4519 sb->s_flags |= SB_NOUSER;
4520 #endif
4521 sbinfo->max_blocks = ctx->blocks;
4522 sbinfo->max_inodes = ctx->inodes;
4523 sbinfo->free_ispace = sbinfo->max_inodes * BOGO_INODE_SIZE;
4524 if (sb->s_flags & SB_KERNMOUNT) {
4525 sbinfo->ino_batch = alloc_percpu(ino_t);
4526 if (!sbinfo->ino_batch)
4527 goto failed;
4528 }
4529 sbinfo->uid = ctx->uid;
4530 sbinfo->gid = ctx->gid;
4531 sbinfo->full_inums = ctx->full_inums;
4532 sbinfo->mode = ctx->mode;
4533 sbinfo->huge = ctx->huge;
4534 sbinfo->mpol = ctx->mpol;
4535 ctx->mpol = NULL;
4536
4537 raw_spin_lock_init(&sbinfo->stat_lock);
4538 if (percpu_counter_init(&sbinfo->used_blocks, 0, GFP_KERNEL))
4539 goto failed;
4540 spin_lock_init(&sbinfo->shrinklist_lock);
4541 INIT_LIST_HEAD(&sbinfo->shrinklist);
4542
4543 sb->s_maxbytes = MAX_LFS_FILESIZE;
4544 sb->s_blocksize = PAGE_SIZE;
4545 sb->s_blocksize_bits = PAGE_SHIFT;
4546 sb->s_magic = TMPFS_MAGIC;
4547 sb->s_op = &shmem_ops;
4548 sb->s_time_gran = 1;
4549 #ifdef CONFIG_TMPFS_XATTR
4550 sb->s_xattr = shmem_xattr_handlers;
4551 #endif
4552 #ifdef CONFIG_TMPFS_POSIX_ACL
4553 sb->s_flags |= SB_POSIXACL;
4554 #endif
4555 uuid_t uuid;
4556 uuid_gen(&uuid);
4557 super_set_uuid(sb, uuid.b, sizeof(uuid));
4558
4559 #ifdef CONFIG_TMPFS_QUOTA
4560 if (ctx->seen & SHMEM_SEEN_QUOTA) {
4561 sb->dq_op = &shmem_quota_operations;
4562 sb->s_qcop = &dquot_quotactl_sysfile_ops;
4563 sb->s_quota_types = QTYPE_MASK_USR | QTYPE_MASK_GRP;
4564
4565 /* Copy the default limits from ctx into sbinfo */
4566 memcpy(&sbinfo->qlimits, &ctx->qlimits,
4567 sizeof(struct shmem_quota_limits));
4568
4569 if (shmem_enable_quotas(sb, ctx->quota_types))
4570 goto failed;
4571 }
4572 #endif /* CONFIG_TMPFS_QUOTA */
4573
4574 inode = shmem_get_inode(&nop_mnt_idmap, sb, NULL,
4575 S_IFDIR | sbinfo->mode, 0, VM_NORESERVE);
4576 if (IS_ERR(inode)) {
4577 error = PTR_ERR(inode);
4578 goto failed;
4579 }
4580 inode->i_uid = sbinfo->uid;
4581 inode->i_gid = sbinfo->gid;
4582 sb->s_root = d_make_root(inode);
4583 if (!sb->s_root)
4584 goto failed;
4585 return 0;
4586
4587 failed:
4588 shmem_put_super(sb);
4589 return error;
4590 }
4591
4592 static int shmem_get_tree(struct fs_context *fc)
4593 {
4594 return get_tree_nodev(fc, shmem_fill_super);
4595 }
4596
4597 static void shmem_free_fc(struct fs_context *fc)
4598 {
4599 struct shmem_options *ctx = fc->fs_private;
4600
4601 if (ctx) {
4602 mpol_put(ctx->mpol);
4603 kfree(ctx);
4604 }
4605 }
4606
4607 static const struct fs_context_operations shmem_fs_context_ops = {
4608 .free = shmem_free_fc,
4609 .get_tree = shmem_get_tree,
4610 #ifdef CONFIG_TMPFS
4611 .parse_monolithic = shmem_parse_options,
4612 .parse_param = shmem_parse_one,
4613 .reconfigure = shmem_reconfigure,
4614 #endif
4615 };
4616
4617 static struct kmem_cache *shmem_inode_cachep __ro_after_init;
4618
4619 static struct inode *shmem_alloc_inode(struct super_block *sb)
4620 {
4621 struct shmem_inode_info *info;
4622 info = alloc_inode_sb(sb, shmem_inode_cachep, GFP_KERNEL);
4623 if (!info)
4624 return NULL;
4625 return &info->vfs_inode;
4626 }
4627
4628 static void shmem_free_in_core_inode(struct inode *inode)
4629 {
4630 if (S_ISLNK(inode->i_mode))
4631 kfree(inode->i_link);
4632 kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode));
4633 }
4634
4635 static void shmem_destroy_inode(struct inode *inode)
4636 {
4637 if (S_ISREG(inode->i_mode))
4638 mpol_free_shared_policy(&SHMEM_I(inode)->policy);
4639 if (S_ISDIR(inode->i_mode))
4640 simple_offset_destroy(shmem_get_offset_ctx(inode));
4641 }
4642
4643 static void shmem_init_inode(void *foo)
4644 {
4645 struct shmem_inode_info *info = foo;
4646 inode_init_once(&info->vfs_inode);
4647 }
4648
4649 static void __init shmem_init_inodecache(void)
4650 {
4651 shmem_inode_cachep = kmem_cache_create("shmem_inode_cache",
4652 sizeof(struct shmem_inode_info),
4653 0, SLAB_PANIC|SLAB_ACCOUNT, shmem_init_inode);
4654 }
4655
4656 static void __init shmem_destroy_inodecache(void)
4657 {
4658 kmem_cache_destroy(shmem_inode_cachep);
4659 }
4660
4661 /* Keep the page in page cache instead of truncating it */
4662 static int shmem_error_remove_folio(struct address_space *mapping,
4663 struct folio *folio)
4664 {
4665 return 0;
4666 }
4667
4668 static const struct address_space_operations shmem_aops = {
4669 .writepage = shmem_writepage,
4670 .dirty_folio = noop_dirty_folio,
4671 #ifdef CONFIG_TMPFS
4672 .write_begin = shmem_write_begin,
4673 .write_end = shmem_write_end,
4674 #endif
4675 #ifdef CONFIG_MIGRATION
4676 .migrate_folio = migrate_folio,
4677 #endif
4678 .error_remove_folio = shmem_error_remove_folio,
4679 };
4680
4681 static const struct file_operations shmem_file_operations = {
4682 .mmap = shmem_mmap,
4683 .open = shmem_file_open,
4684 .get_unmapped_area = shmem_get_unmapped_area,
4685 #ifdef CONFIG_TMPFS
4686 .llseek = shmem_file_llseek,
4687 .read_iter = shmem_file_read_iter,
4688 .write_iter = shmem_file_write_iter,
4689 .fsync = noop_fsync,
4690 .splice_read = shmem_file_splice_read,
4691 .splice_write = iter_file_splice_write,
4692 .fallocate = shmem_fallocate,
4693 #endif
4694 };
4695
4696 static const struct inode_operations shmem_inode_operations = {
4697 .getattr = shmem_getattr,
4698 .setattr = shmem_setattr,
4699 #ifdef CONFIG_TMPFS_XATTR
4700 .listxattr = shmem_listxattr,
4701 .set_acl = simple_set_acl,
4702 .fileattr_get = shmem_fileattr_get,
4703 .fileattr_set = shmem_fileattr_set,
4704 #endif
4705 };
4706
4707 static const struct inode_operations shmem_dir_inode_operations = {
4708 #ifdef CONFIG_TMPFS
4709 .getattr = shmem_getattr,
4710 .create = shmem_create,
4711 .lookup = simple_lookup,
4712 .link = shmem_link,
4713 .unlink = shmem_unlink,
4714 .symlink = shmem_symlink,
4715 .mkdir = shmem_mkdir,
4716 .rmdir = shmem_rmdir,
4717 .mknod = shmem_mknod,
4718 .rename = shmem_rename2,
4719 .tmpfile = shmem_tmpfile,
4720 .get_offset_ctx = shmem_get_offset_ctx,
4721 #endif
4722 #ifdef CONFIG_TMPFS_XATTR
4723 .listxattr = shmem_listxattr,
4724 .fileattr_get = shmem_fileattr_get,
4725 .fileattr_set = shmem_fileattr_set,
4726 #endif
4727 #ifdef CONFIG_TMPFS_POSIX_ACL
4728 .setattr = shmem_setattr,
4729 .set_acl = simple_set_acl,
4730 #endif
4731 };
4732
4733 static const struct inode_operations shmem_special_inode_operations = {
4734 .getattr = shmem_getattr,
4735 #ifdef CONFIG_TMPFS_XATTR
4736 .listxattr = shmem_listxattr,
4737 #endif
4738 #ifdef CONFIG_TMPFS_POSIX_ACL
4739 .setattr = shmem_setattr,
4740 .set_acl = simple_set_acl,
4741 #endif
4742 };
4743
4744 static const struct super_operations shmem_ops = {
4745 .alloc_inode = shmem_alloc_inode,
4746 .free_inode = shmem_free_in_core_inode,
4747 .destroy_inode = shmem_destroy_inode,
4748 #ifdef CONFIG_TMPFS
4749 .statfs = shmem_statfs,
4750 .show_options = shmem_show_options,
4751 #endif
4752 #ifdef CONFIG_TMPFS_QUOTA
4753 .get_dquots = shmem_get_dquots,
4754 #endif
4755 .evict_inode = shmem_evict_inode,
4756 .drop_inode = generic_delete_inode,
4757 .put_super = shmem_put_super,
4758 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
4759 .nr_cached_objects = shmem_unused_huge_count,
4760 .free_cached_objects = shmem_unused_huge_scan,
4761 #endif
4762 };
4763
4764 static const struct vm_operations_struct shmem_vm_ops = {
4765 .fault = shmem_fault,
4766 .map_pages = filemap_map_pages,
4767 #ifdef CONFIG_NUMA
4768 .set_policy = shmem_set_policy,
4769 .get_policy = shmem_get_policy,
4770 #endif
4771 };
4772
4773 static const struct vm_operations_struct shmem_anon_vm_ops = {
4774 .fault = shmem_fault,
4775 .map_pages = filemap_map_pages,
4776 #ifdef CONFIG_NUMA
4777 .set_policy = shmem_set_policy,
4778 .get_policy = shmem_get_policy,
4779 #endif
4780 };
4781
4782 int shmem_init_fs_context(struct fs_context *fc)
4783 {
4784 struct shmem_options *ctx;
4785
4786 ctx = kzalloc(sizeof(struct shmem_options), GFP_KERNEL);
4787 if (!ctx)
4788 return -ENOMEM;
4789
4790 ctx->mode = 0777 | S_ISVTX;
4791 ctx->uid = current_fsuid();
4792 ctx->gid = current_fsgid();
4793
4794 fc->fs_private = ctx;
4795 fc->ops = &shmem_fs_context_ops;
4796 return 0;
4797 }
4798
4799 static struct file_system_type shmem_fs_type = {
4800 .owner = THIS_MODULE,
4801 .name = "tmpfs",
4802 .init_fs_context = shmem_init_fs_context,
4803 #ifdef CONFIG_TMPFS
4804 .parameters = shmem_fs_parameters,
4805 #endif
4806 .kill_sb = kill_litter_super,
4807 .fs_flags = FS_USERNS_MOUNT | FS_ALLOW_IDMAP,
4808 };
4809
4810 void __init shmem_init(void)
4811 {
4812 int error;
4813
4814 shmem_init_inodecache();
4815
4816 #ifdef CONFIG_TMPFS_QUOTA
4817 error = register_quota_format(&shmem_quota_format);
4818 if (error < 0) {
4819 pr_err("Could not register quota format\n");
4820 goto out3;
4821 }
4822 #endif
4823
4824 error = register_filesystem(&shmem_fs_type);
4825 if (error) {
4826 pr_err("Could not register tmpfs\n");
4827 goto out2;
4828 }
4829
4830 shm_mnt = kern_mount(&shmem_fs_type);
4831 if (IS_ERR(shm_mnt)) {
4832 error = PTR_ERR(shm_mnt);
4833 pr_err("Could not kern_mount tmpfs\n");
4834 goto out1;
4835 }
4836
4837 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
4838 if (has_transparent_hugepage() && shmem_huge > SHMEM_HUGE_DENY)
4839 SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge;
4840 else
4841 shmem_huge = SHMEM_HUGE_NEVER; /* just in case it was patched */
4842
4843 /*
4844 * Default to setting PMD-sized THP to inherit the global setting and
4845 * disable all other multi-size THPs.
4846 */
4847 huge_shmem_orders_inherit = BIT(HPAGE_PMD_ORDER);
4848 #endif
4849 return;
4850
4851 out1:
4852 unregister_filesystem(&shmem_fs_type);
4853 out2:
4854 #ifdef CONFIG_TMPFS_QUOTA
4855 unregister_quota_format(&shmem_quota_format);
4856 out3:
4857 #endif
4858 shmem_destroy_inodecache();
4859 shm_mnt = ERR_PTR(error);
4860 }
4861
4862 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && defined(CONFIG_SYSFS)
4863 static ssize_t shmem_enabled_show(struct kobject *kobj,
4864 struct kobj_attribute *attr, char *buf)
4865 {
4866 static const int values[] = {
4867 SHMEM_HUGE_ALWAYS,
4868 SHMEM_HUGE_WITHIN_SIZE,
4869 SHMEM_HUGE_ADVISE,
4870 SHMEM_HUGE_NEVER,
4871 SHMEM_HUGE_DENY,
4872 SHMEM_HUGE_FORCE,
4873 };
4874 int len = 0;
4875 int i;
4876
4877 for (i = 0; i < ARRAY_SIZE(values); i++) {
4878 len += sysfs_emit_at(buf, len,
4879 shmem_huge == values[i] ? "%s[%s]" : "%s%s",
4880 i ? " " : "", shmem_format_huge(values[i]));
4881 }
4882 len += sysfs_emit_at(buf, len, "\n");
4883
4884 return len;
4885 }
4886
4887 static ssize_t shmem_enabled_store(struct kobject *kobj,
4888 struct kobj_attribute *attr, const char *buf, size_t count)
4889 {
4890 char tmp[16];
4891 int huge;
4892
4893 if (count + 1 > sizeof(tmp))
4894 return -EINVAL;
4895 memcpy(tmp, buf, count);
4896 tmp[count] = '\0';
4897 if (count && tmp[count - 1] == '\n')
4898 tmp[count - 1] = '\0';
4899
4900 huge = shmem_parse_huge(tmp);
4901 if (huge == -EINVAL)
4902 return -EINVAL;
4903 if (!has_transparent_hugepage() &&
4904 huge != SHMEM_HUGE_NEVER && huge != SHMEM_HUGE_DENY)
4905 return -EINVAL;
4906
4907 /* Do not override huge allocation policy with non-PMD sized mTHP */
4908 if (huge == SHMEM_HUGE_FORCE &&
4909 huge_shmem_orders_inherit != BIT(HPAGE_PMD_ORDER))
4910 return -EINVAL;
4911
4912 shmem_huge = huge;
4913 if (shmem_huge > SHMEM_HUGE_DENY)
4914 SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge;
4915 return count;
4916 }
4917
4918 struct kobj_attribute shmem_enabled_attr = __ATTR_RW(shmem_enabled);
4919 static DEFINE_SPINLOCK(huge_shmem_orders_lock);
4920
4921 static ssize_t thpsize_shmem_enabled_show(struct kobject *kobj,
4922 struct kobj_attribute *attr, char *buf)
4923 {
4924 int order = to_thpsize(kobj)->order;
4925 const char *output;
4926
4927 if (test_bit(order, &huge_shmem_orders_always))
4928 output = "[always] inherit within_size advise never";
4929 else if (test_bit(order, &huge_shmem_orders_inherit))
4930 output = "always [inherit] within_size advise never";
4931 else if (test_bit(order, &huge_shmem_orders_within_size))
4932 output = "always inherit [within_size] advise never";
4933 else if (test_bit(order, &huge_shmem_orders_madvise))
4934 output = "always inherit within_size [advise] never";
4935 else
4936 output = "always inherit within_size advise [never]";
4937
4938 return sysfs_emit(buf, "%s\n", output);
4939 }
4940
4941 static ssize_t thpsize_shmem_enabled_store(struct kobject *kobj,
4942 struct kobj_attribute *attr,
4943 const char *buf, size_t count)
4944 {
4945 int order = to_thpsize(kobj)->order;
4946 ssize_t ret = count;
4947
4948 if (sysfs_streq(buf, "always")) {
4949 spin_lock(&huge_shmem_orders_lock);
4950 clear_bit(order, &huge_shmem_orders_inherit);
4951 clear_bit(order, &huge_shmem_orders_madvise);
4952 clear_bit(order, &huge_shmem_orders_within_size);
4953 set_bit(order, &huge_shmem_orders_always);
4954 spin_unlock(&huge_shmem_orders_lock);
4955 } else if (sysfs_streq(buf, "inherit")) {
4956 /* Do not override huge allocation policy with non-PMD sized mTHP */
4957 if (shmem_huge == SHMEM_HUGE_FORCE &&
4958 order != HPAGE_PMD_ORDER)
4959 return -EINVAL;
4960
4961 spin_lock(&huge_shmem_orders_lock);
4962 clear_bit(order, &huge_shmem_orders_always);
4963 clear_bit(order, &huge_shmem_orders_madvise);
4964 clear_bit(order, &huge_shmem_orders_within_size);
4965 set_bit(order, &huge_shmem_orders_inherit);
4966 spin_unlock(&huge_shmem_orders_lock);
4967 } else if (sysfs_streq(buf, "within_size")) {
4968 spin_lock(&huge_shmem_orders_lock);
4969 clear_bit(order, &huge_shmem_orders_always);
4970 clear_bit(order, &huge_shmem_orders_inherit);
4971 clear_bit(order, &huge_shmem_orders_madvise);
4972 set_bit(order, &huge_shmem_orders_within_size);
4973 spin_unlock(&huge_shmem_orders_lock);
4974 } else if (sysfs_streq(buf, "advise")) {
4975 spin_lock(&huge_shmem_orders_lock);
4976 clear_bit(order, &huge_shmem_orders_always);
4977 clear_bit(order, &huge_shmem_orders_inherit);
4978 clear_bit(order, &huge_shmem_orders_within_size);
4979 set_bit(order, &huge_shmem_orders_madvise);
4980 spin_unlock(&huge_shmem_orders_lock);
4981 } else if (sysfs_streq(buf, "never")) {
4982 spin_lock(&huge_shmem_orders_lock);
4983 clear_bit(order, &huge_shmem_orders_always);
4984 clear_bit(order, &huge_shmem_orders_inherit);
4985 clear_bit(order, &huge_shmem_orders_within_size);
4986 clear_bit(order, &huge_shmem_orders_madvise);
4987 spin_unlock(&huge_shmem_orders_lock);
4988 } else {
4989 ret = -EINVAL;
4990 }
4991
4992 return ret;
4993 }
4994
4995 struct kobj_attribute thpsize_shmem_enabled_attr =
4996 __ATTR(shmem_enabled, 0644, thpsize_shmem_enabled_show, thpsize_shmem_enabled_store);
4997 #endif /* CONFIG_TRANSPARENT_HUGEPAGE && CONFIG_SYSFS */
4998
4999 #else /* !CONFIG_SHMEM */
5000
5001 /*
5002 * tiny-shmem: simple shmemfs and tmpfs using ramfs code
5003 *
5004 * This is intended for small system where the benefits of the full
5005 * shmem code (swap-backed and resource-limited) are outweighed by
5006 * their complexity. On systems without swap this code should be
5007 * effectively equivalent, but much lighter weight.
5008 */
5009
5010 static struct file_system_type shmem_fs_type = {
5011 .name = "tmpfs",
5012 .init_fs_context = ramfs_init_fs_context,
5013 .parameters = ramfs_fs_parameters,
5014 .kill_sb = ramfs_kill_sb,
5015 .fs_flags = FS_USERNS_MOUNT,
5016 };
5017
5018 void __init shmem_init(void)
5019 {
5020 BUG_ON(register_filesystem(&shmem_fs_type) != 0);
5021
5022 shm_mnt = kern_mount(&shmem_fs_type);
5023 BUG_ON(IS_ERR(shm_mnt));
5024 }
5025
5026 int shmem_unuse(unsigned int type)
5027 {
5028 return 0;
5029 }
5030
5031 int shmem_lock(struct file *file, int lock, struct ucounts *ucounts)
5032 {
5033 return 0;
5034 }
5035
5036 void shmem_unlock_mapping(struct address_space *mapping)
5037 {
5038 }
5039
5040 #ifdef CONFIG_MMU
5041 unsigned long shmem_get_unmapped_area(struct file *file,
5042 unsigned long addr, unsigned long len,
5043 unsigned long pgoff, unsigned long flags)
5044 {
5045 return mm_get_unmapped_area(current->mm, file, addr, len, pgoff, flags);
5046 }
5047 #endif
5048
5049 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
5050 {
5051 truncate_inode_pages_range(inode->i_mapping, lstart, lend);
5052 }
5053 EXPORT_SYMBOL_GPL(shmem_truncate_range);
5054
5055 #define shmem_vm_ops generic_file_vm_ops
5056 #define shmem_anon_vm_ops generic_file_vm_ops
5057 #define shmem_file_operations ramfs_file_operations
5058 #define shmem_acct_size(flags, size) 0
5059 #define shmem_unacct_size(flags, size) do {} while (0)
5060
5061 static inline struct inode *shmem_get_inode(struct mnt_idmap *idmap,
5062 struct super_block *sb, struct inode *dir,
5063 umode_t mode, dev_t dev, unsigned long flags)
5064 {
5065 struct inode *inode = ramfs_get_inode(sb, dir, mode, dev);
5066 return inode ? inode : ERR_PTR(-ENOSPC);
5067 }
5068
5069 #endif /* CONFIG_SHMEM */
5070
5071 /* common code */
5072
5073 static struct file *__shmem_file_setup(struct vfsmount *mnt, const char *name,
5074 loff_t size, unsigned long flags, unsigned int i_flags)
5075 {
5076 struct inode *inode;
5077 struct file *res;
5078
5079 if (IS_ERR(mnt))
5080 return ERR_CAST(mnt);
5081
5082 if (size < 0 || size > MAX_LFS_FILESIZE)
5083 return ERR_PTR(-EINVAL);
5084
5085 if (shmem_acct_size(flags, size))
5086 return ERR_PTR(-ENOMEM);
5087
5088 if (is_idmapped_mnt(mnt))
5089 return ERR_PTR(-EINVAL);
5090
5091 inode = shmem_get_inode(&nop_mnt_idmap, mnt->mnt_sb, NULL,
5092 S_IFREG | S_IRWXUGO, 0, flags);
5093 if (IS_ERR(inode)) {
5094 shmem_unacct_size(flags, size);
5095 return ERR_CAST(inode);
5096 }
5097 inode->i_flags |= i_flags;
5098 inode->i_size = size;
5099 clear_nlink(inode); /* It is unlinked */
5100 res = ERR_PTR(ramfs_nommu_expand_for_mapping(inode, size));
5101 if (!IS_ERR(res))
5102 res = alloc_file_pseudo(inode, mnt, name, O_RDWR,
5103 &shmem_file_operations);
5104 if (IS_ERR(res))
5105 iput(inode);
5106 return res;
5107 }
5108
5109 /**
5110 * shmem_kernel_file_setup - get an unlinked file living in tmpfs which must be
5111 * kernel internal. There will be NO LSM permission checks against the
5112 * underlying inode. So users of this interface must do LSM checks at a
5113 * higher layer. The users are the big_key and shm implementations. LSM
5114 * checks are provided at the key or shm level rather than the inode.
5115 * @name: name for dentry (to be seen in /proc/<pid>/maps
5116 * @size: size to be set for the file
5117 * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
5118 */
5119 struct file *shmem_kernel_file_setup(const char *name, loff_t size, unsigned long flags)
5120 {
5121 return __shmem_file_setup(shm_mnt, name, size, flags, S_PRIVATE);
5122 }
5123 EXPORT_SYMBOL_GPL(shmem_kernel_file_setup);
5124
5125 /**
5126 * shmem_file_setup - get an unlinked file living in tmpfs
5127 * @name: name for dentry (to be seen in /proc/<pid>/maps
5128 * @size: size to be set for the file
5129 * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
5130 */
5131 struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags)
5132 {
5133 return __shmem_file_setup(shm_mnt, name, size, flags, 0);
5134 }
5135 EXPORT_SYMBOL_GPL(shmem_file_setup);
5136
5137 /**
5138 * shmem_file_setup_with_mnt - get an unlinked file living in tmpfs
5139 * @mnt: the tmpfs mount where the file will be created
5140 * @name: name for dentry (to be seen in /proc/<pid>/maps
5141 * @size: size to be set for the file
5142 * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
5143 */
5144 struct file *shmem_file_setup_with_mnt(struct vfsmount *mnt, const char *name,
5145 loff_t size, unsigned long flags)
5146 {
5147 return __shmem_file_setup(mnt, name, size, flags, 0);
5148 }
5149 EXPORT_SYMBOL_GPL(shmem_file_setup_with_mnt);
5150
5151 /**
5152 * shmem_zero_setup - setup a shared anonymous mapping
5153 * @vma: the vma to be mmapped is prepared by do_mmap
5154 */
5155 int shmem_zero_setup(struct vm_area_struct *vma)
5156 {
5157 struct file *file;
5158 loff_t size = vma->vm_end - vma->vm_start;
5159
5160 /*
5161 * Cloning a new file under mmap_lock leads to a lock ordering conflict
5162 * between XFS directory reading and selinux: since this file is only
5163 * accessible to the user through its mapping, use S_PRIVATE flag to
5164 * bypass file security, in the same way as shmem_kernel_file_setup().
5165 */
5166 file = shmem_kernel_file_setup("dev/zero", size, vma->vm_flags);
5167 if (IS_ERR(file))
5168 return PTR_ERR(file);
5169
5170 if (vma->vm_file)
5171 fput(vma->vm_file);
5172 vma->vm_file = file;
5173 vma->vm_ops = &shmem_anon_vm_ops;
5174
5175 return 0;
5176 }
5177
5178 /**
5179 * shmem_read_folio_gfp - read into page cache, using specified page allocation flags.
5180 * @mapping: the folio's address_space
5181 * @index: the folio index
5182 * @gfp: the page allocator flags to use if allocating
5183 *
5184 * This behaves as a tmpfs "read_cache_page_gfp(mapping, index, gfp)",
5185 * with any new page allocations done using the specified allocation flags.
5186 * But read_cache_page_gfp() uses the ->read_folio() method: which does not
5187 * suit tmpfs, since it may have pages in swapcache, and needs to find those
5188 * for itself; although drivers/gpu/drm i915 and ttm rely upon this support.
5189 *
5190 * i915_gem_object_get_pages_gtt() mixes __GFP_NORETRY | __GFP_NOWARN in
5191 * with the mapping_gfp_mask(), to avoid OOMing the machine unnecessarily.
5192 */
5193 struct folio *shmem_read_folio_gfp(struct address_space *mapping,
5194 pgoff_t index, gfp_t gfp)
5195 {
5196 #ifdef CONFIG_SHMEM
5197 struct inode *inode = mapping->host;
5198 struct folio *folio;
5199 int error;
5200
5201 error = shmem_get_folio_gfp(inode, index, &folio, SGP_CACHE,
5202 gfp, NULL, NULL);
5203 if (error)
5204 return ERR_PTR(error);
5205
5206 folio_unlock(folio);
5207 return folio;
5208 #else
5209 /*
5210 * The tiny !SHMEM case uses ramfs without swap
5211 */
5212 return mapping_read_folio_gfp(mapping, index, gfp);
5213 #endif
5214 }
5215 EXPORT_SYMBOL_GPL(shmem_read_folio_gfp);
5216
5217 struct page *shmem_read_mapping_page_gfp(struct address_space *mapping,
5218 pgoff_t index, gfp_t gfp)
5219 {
5220 struct folio *folio = shmem_read_folio_gfp(mapping, index, gfp);
5221 struct page *page;
5222
5223 if (IS_ERR(folio))
5224 return &folio->page;
5225
5226 page = folio_file_page(folio, index);
5227 if (PageHWPoison(page)) {
5228 folio_put(folio);
5229 return ERR_PTR(-EIO);
5230 }
5231
5232 return page;
5233 }
5234 EXPORT_SYMBOL_GPL(shmem_read_mapping_page_gfp);
5235
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.