1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * linux/fs/file.c
4 *
5 * Copyright (C) 1998-1999, Stephen Tweedie and Bill Hawes
6 *
7 * Manage the dynamic fd arrays in the process files_struct.
8 */
9
10 #include <linux/syscalls.h>
11 #include <linux/export.h>
12 #include <linux/fs.h>
13 #include <linux/kernel.h>
14 #include <linux/mm.h>
15 #include <linux/sched/signal.h>
16 #include <linux/slab.h>
17 #include <linux/file.h>
18 #include <linux/fdtable.h>
19 #include <linux/bitops.h>
20 #include <linux/spinlock.h>
21 #include <linux/rcupdate.h>
22 #include <linux/close_range.h>
23 #include <net/sock.h>
24
25 #include "internal.h"
26
27 unsigned int sysctl_nr_open __read_mostly = 1024*1024;
28 unsigned int sysctl_nr_open_min = BITS_PER_LONG;
29 /* our min() is unusable in constant expressions ;-/ */
30 #define __const_min(x, y) ((x) < (y) ? (x) : (y))
31 unsigned int sysctl_nr_open_max =
32 __const_min(INT_MAX, ~(size_t)0/sizeof(void *)) & -BITS_PER_LONG;
33
34 static void __free_fdtable(struct fdtable *fdt)
35 {
36 kvfree(fdt->fd);
37 kvfree(fdt->open_fds);
38 kfree(fdt);
39 }
40
41 static void free_fdtable_rcu(struct rcu_head *rcu)
42 {
43 __free_fdtable(container_of(rcu, struct fdtable, rcu));
44 }
45
46 #define BITBIT_NR(nr) BITS_TO_LONGS(BITS_TO_LONGS(nr))
47 #define BITBIT_SIZE(nr) (BITBIT_NR(nr) * sizeof(long))
48
49 /*
50 * Copy 'count' fd bits from the old table to the new table and clear the extra
51 * space if any. This does not copy the file pointers. Called with the files
52 * spinlock held for write.
53 */
54 static void copy_fd_bitmaps(struct fdtable *nfdt, struct fdtable *ofdt,
55 unsigned int count)
56 {
57 unsigned int cpy, set;
58
59 cpy = count / BITS_PER_BYTE;
60 set = (nfdt->max_fds - count) / BITS_PER_BYTE;
61 memcpy(nfdt->open_fds, ofdt->open_fds, cpy);
62 memset((char *)nfdt->open_fds + cpy, 0, set);
63 memcpy(nfdt->close_on_exec, ofdt->close_on_exec, cpy);
64 memset((char *)nfdt->close_on_exec + cpy, 0, set);
65
66 cpy = BITBIT_SIZE(count);
67 set = BITBIT_SIZE(nfdt->max_fds) - cpy;
68 memcpy(nfdt->full_fds_bits, ofdt->full_fds_bits, cpy);
69 memset((char *)nfdt->full_fds_bits + cpy, 0, set);
70 }
71
72 /*
73 * Copy all file descriptors from the old table to the new, expanded table and
74 * clear the extra space. Called with the files spinlock held for write.
75 */
76 static void copy_fdtable(struct fdtable *nfdt, struct fdtable *ofdt)
77 {
78 size_t cpy, set;
79
80 BUG_ON(nfdt->max_fds < ofdt->max_fds);
81
82 cpy = ofdt->max_fds * sizeof(struct file *);
83 set = (nfdt->max_fds - ofdt->max_fds) * sizeof(struct file *);
84 memcpy(nfdt->fd, ofdt->fd, cpy);
85 memset((char *)nfdt->fd + cpy, 0, set);
86
87 copy_fd_bitmaps(nfdt, ofdt, ofdt->max_fds);
88 }
89
90 /*
91 * Note how the fdtable bitmap allocations very much have to be a multiple of
92 * BITS_PER_LONG. This is not only because we walk those things in chunks of
93 * 'unsigned long' in some places, but simply because that is how the Linux
94 * kernel bitmaps are defined to work: they are not "bits in an array of bytes",
95 * they are very much "bits in an array of unsigned long".
96 *
97 * The ALIGN(nr, BITS_PER_LONG) here is for clarity: since we just multiplied
98 * by that "1024/sizeof(ptr)" before, we already know there are sufficient
99 * clear low bits. Clang seems to realize that, gcc ends up being confused.
100 *
101 * On a 128-bit machine, the ALIGN() would actually matter. In the meantime,
102 * let's consider it documentation (and maybe a test-case for gcc to improve
103 * its code generation ;)
104 */
105 static struct fdtable * alloc_fdtable(unsigned int nr)
106 {
107 struct fdtable *fdt;
108 void *data;
109
110 /*
111 * Figure out how many fds we actually want to support in this fdtable.
112 * Allocation steps are keyed to the size of the fdarray, since it
113 * grows far faster than any of the other dynamic data. We try to fit
114 * the fdarray into comfortable page-tuned chunks: starting at 1024B
115 * and growing in powers of two from there on.
116 */
117 nr /= (1024 / sizeof(struct file *));
118 nr = roundup_pow_of_two(nr + 1);
119 nr *= (1024 / sizeof(struct file *));
120 nr = ALIGN(nr, BITS_PER_LONG);
121 /*
122 * Note that this can drive nr *below* what we had passed if sysctl_nr_open
123 * had been set lower between the check in expand_files() and here. Deal
124 * with that in caller, it's cheaper that way.
125 *
126 * We make sure that nr remains a multiple of BITS_PER_LONG - otherwise
127 * bitmaps handling below becomes unpleasant, to put it mildly...
128 */
129 if (unlikely(nr > sysctl_nr_open))
130 nr = ((sysctl_nr_open - 1) | (BITS_PER_LONG - 1)) + 1;
131
132 fdt = kmalloc(sizeof(struct fdtable), GFP_KERNEL_ACCOUNT);
133 if (!fdt)
134 goto out;
135 fdt->max_fds = nr;
136 data = kvmalloc_array(nr, sizeof(struct file *), GFP_KERNEL_ACCOUNT);
137 if (!data)
138 goto out_fdt;
139 fdt->fd = data;
140
141 data = kvmalloc(max_t(size_t,
142 2 * nr / BITS_PER_BYTE + BITBIT_SIZE(nr), L1_CACHE_BYTES),
143 GFP_KERNEL_ACCOUNT);
144 if (!data)
145 goto out_arr;
146 fdt->open_fds = data;
147 data += nr / BITS_PER_BYTE;
148 fdt->close_on_exec = data;
149 data += nr / BITS_PER_BYTE;
150 fdt->full_fds_bits = data;
151
152 return fdt;
153
154 out_arr:
155 kvfree(fdt->fd);
156 out_fdt:
157 kfree(fdt);
158 out:
159 return NULL;
160 }
161
162 /*
163 * Expand the file descriptor table.
164 * This function will allocate a new fdtable and both fd array and fdset, of
165 * the given size.
166 * Return <0 error code on error; 1 on successful completion.
167 * The files->file_lock should be held on entry, and will be held on exit.
168 */
169 static int expand_fdtable(struct files_struct *files, unsigned int nr)
170 __releases(files->file_lock)
171 __acquires(files->file_lock)
172 {
173 struct fdtable *new_fdt, *cur_fdt;
174
175 spin_unlock(&files->file_lock);
176 new_fdt = alloc_fdtable(nr);
177
178 /* make sure all fd_install() have seen resize_in_progress
179 * or have finished their rcu_read_lock_sched() section.
180 */
181 if (atomic_read(&files->count) > 1)
182 synchronize_rcu();
183
184 spin_lock(&files->file_lock);
185 if (!new_fdt)
186 return -ENOMEM;
187 /*
188 * extremely unlikely race - sysctl_nr_open decreased between the check in
189 * caller and alloc_fdtable(). Cheaper to catch it here...
190 */
191 if (unlikely(new_fdt->max_fds <= nr)) {
192 __free_fdtable(new_fdt);
193 return -EMFILE;
194 }
195 cur_fdt = files_fdtable(files);
196 BUG_ON(nr < cur_fdt->max_fds);
197 copy_fdtable(new_fdt, cur_fdt);
198 rcu_assign_pointer(files->fdt, new_fdt);
199 if (cur_fdt != &files->fdtab)
200 call_rcu(&cur_fdt->rcu, free_fdtable_rcu);
201 /* coupled with smp_rmb() in fd_install() */
202 smp_wmb();
203 return 1;
204 }
205
206 /*
207 * Expand files.
208 * This function will expand the file structures, if the requested size exceeds
209 * the current capacity and there is room for expansion.
210 * Return <0 error code on error; 0 when nothing done; 1 when files were
211 * expanded and execution may have blocked.
212 * The files->file_lock should be held on entry, and will be held on exit.
213 */
214 static int expand_files(struct files_struct *files, unsigned int nr)
215 __releases(files->file_lock)
216 __acquires(files->file_lock)
217 {
218 struct fdtable *fdt;
219 int expanded = 0;
220
221 repeat:
222 fdt = files_fdtable(files);
223
224 /* Do we need to expand? */
225 if (nr < fdt->max_fds)
226 return expanded;
227
228 /* Can we expand? */
229 if (nr >= sysctl_nr_open)
230 return -EMFILE;
231
232 if (unlikely(files->resize_in_progress)) {
233 spin_unlock(&files->file_lock);
234 expanded = 1;
235 wait_event(files->resize_wait, !files->resize_in_progress);
236 spin_lock(&files->file_lock);
237 goto repeat;
238 }
239
240 /* All good, so we try */
241 files->resize_in_progress = true;
242 expanded = expand_fdtable(files, nr);
243 files->resize_in_progress = false;
244
245 wake_up_all(&files->resize_wait);
246 return expanded;
247 }
248
249 static inline void __set_close_on_exec(unsigned int fd, struct fdtable *fdt)
250 {
251 __set_bit(fd, fdt->close_on_exec);
252 }
253
254 static inline void __clear_close_on_exec(unsigned int fd, struct fdtable *fdt)
255 {
256 if (test_bit(fd, fdt->close_on_exec))
257 __clear_bit(fd, fdt->close_on_exec);
258 }
259
260 static inline void __set_open_fd(unsigned int fd, struct fdtable *fdt)
261 {
262 __set_bit(fd, fdt->open_fds);
263 fd /= BITS_PER_LONG;
264 if (!~fdt->open_fds[fd])
265 __set_bit(fd, fdt->full_fds_bits);
266 }
267
268 static inline void __clear_open_fd(unsigned int fd, struct fdtable *fdt)
269 {
270 __clear_bit(fd, fdt->open_fds);
271 __clear_bit(fd / BITS_PER_LONG, fdt->full_fds_bits);
272 }
273
274 static inline bool fd_is_open(unsigned int fd, const struct fdtable *fdt)
275 {
276 return test_bit(fd, fdt->open_fds);
277 }
278
279 static unsigned int count_open_files(struct fdtable *fdt)
280 {
281 unsigned int size = fdt->max_fds;
282 unsigned int i;
283
284 /* Find the last open fd */
285 for (i = size / BITS_PER_LONG; i > 0; ) {
286 if (fdt->open_fds[--i])
287 break;
288 }
289 i = (i + 1) * BITS_PER_LONG;
290 return i;
291 }
292
293 /*
294 * Note that a sane fdtable size always has to be a multiple of
295 * BITS_PER_LONG, since we have bitmaps that are sized by this.
296 *
297 * 'max_fds' will normally already be properly aligned, but it
298 * turns out that in the close_range() -> __close_range() ->
299 * unshare_fd() -> dup_fd() -> sane_fdtable_size() we can end
300 * up having a 'max_fds' value that isn't already aligned.
301 *
302 * Rather than make close_range() have to worry about this,
303 * just make that BITS_PER_LONG alignment be part of a sane
304 * fdtable size. Becuase that's really what it is.
305 */
306 static unsigned int sane_fdtable_size(struct fdtable *fdt, unsigned int max_fds)
307 {
308 unsigned int count;
309
310 count = count_open_files(fdt);
311 if (max_fds < NR_OPEN_DEFAULT)
312 max_fds = NR_OPEN_DEFAULT;
313 return ALIGN(min(count, max_fds), BITS_PER_LONG);
314 }
315
316 /*
317 * Allocate a new files structure and copy contents from the
318 * passed in files structure.
319 * errorp will be valid only when the returned files_struct is NULL.
320 */
321 struct files_struct *dup_fd(struct files_struct *oldf, unsigned int max_fds, int *errorp)
322 {
323 struct files_struct *newf;
324 struct file **old_fds, **new_fds;
325 unsigned int open_files, i;
326 struct fdtable *old_fdt, *new_fdt;
327
328 *errorp = -ENOMEM;
329 newf = kmem_cache_alloc(files_cachep, GFP_KERNEL);
330 if (!newf)
331 goto out;
332
333 atomic_set(&newf->count, 1);
334
335 spin_lock_init(&newf->file_lock);
336 newf->resize_in_progress = false;
337 init_waitqueue_head(&newf->resize_wait);
338 newf->next_fd = 0;
339 new_fdt = &newf->fdtab;
340 new_fdt->max_fds = NR_OPEN_DEFAULT;
341 new_fdt->close_on_exec = newf->close_on_exec_init;
342 new_fdt->open_fds = newf->open_fds_init;
343 new_fdt->full_fds_bits = newf->full_fds_bits_init;
344 new_fdt->fd = &newf->fd_array[0];
345
346 spin_lock(&oldf->file_lock);
347 old_fdt = files_fdtable(oldf);
348 open_files = sane_fdtable_size(old_fdt, max_fds);
349
350 /*
351 * Check whether we need to allocate a larger fd array and fd set.
352 */
353 while (unlikely(open_files > new_fdt->max_fds)) {
354 spin_unlock(&oldf->file_lock);
355
356 if (new_fdt != &newf->fdtab)
357 __free_fdtable(new_fdt);
358
359 new_fdt = alloc_fdtable(open_files - 1);
360 if (!new_fdt) {
361 *errorp = -ENOMEM;
362 goto out_release;
363 }
364
365 /* beyond sysctl_nr_open; nothing to do */
366 if (unlikely(new_fdt->max_fds < open_files)) {
367 __free_fdtable(new_fdt);
368 *errorp = -EMFILE;
369 goto out_release;
370 }
371
372 /*
373 * Reacquire the oldf lock and a pointer to its fd table
374 * who knows it may have a new bigger fd table. We need
375 * the latest pointer.
376 */
377 spin_lock(&oldf->file_lock);
378 old_fdt = files_fdtable(oldf);
379 open_files = sane_fdtable_size(old_fdt, max_fds);
380 }
381
382 copy_fd_bitmaps(new_fdt, old_fdt, open_files);
383
384 old_fds = old_fdt->fd;
385 new_fds = new_fdt->fd;
386
387 for (i = open_files; i != 0; i--) {
388 struct file *f = *old_fds++;
389 if (f) {
390 get_file(f);
391 } else {
392 /*
393 * The fd may be claimed in the fd bitmap but not yet
394 * instantiated in the files array if a sibling thread
395 * is partway through open(). So make sure that this
396 * fd is available to the new process.
397 */
398 __clear_open_fd(open_files - i, new_fdt);
399 }
400 rcu_assign_pointer(*new_fds++, f);
401 }
402 spin_unlock(&oldf->file_lock);
403
404 /* clear the remainder */
405 memset(new_fds, 0, (new_fdt->max_fds - open_files) * sizeof(struct file *));
406
407 rcu_assign_pointer(newf->fdt, new_fdt);
408
409 return newf;
410
411 out_release:
412 kmem_cache_free(files_cachep, newf);
413 out:
414 return NULL;
415 }
416
417 static struct fdtable *close_files(struct files_struct * files)
418 {
419 /*
420 * It is safe to dereference the fd table without RCU or
421 * ->file_lock because this is the last reference to the
422 * files structure.
423 */
424 struct fdtable *fdt = rcu_dereference_raw(files->fdt);
425 unsigned int i, j = 0;
426
427 for (;;) {
428 unsigned long set;
429 i = j * BITS_PER_LONG;
430 if (i >= fdt->max_fds)
431 break;
432 set = fdt->open_fds[j++];
433 while (set) {
434 if (set & 1) {
435 struct file * file = xchg(&fdt->fd[i], NULL);
436 if (file) {
437 filp_close(file, files);
438 cond_resched();
439 }
440 }
441 i++;
442 set >>= 1;
443 }
444 }
445
446 return fdt;
447 }
448
449 void put_files_struct(struct files_struct *files)
450 {
451 if (atomic_dec_and_test(&files->count)) {
452 struct fdtable *fdt = close_files(files);
453
454 /* free the arrays if they are not embedded */
455 if (fdt != &files->fdtab)
456 __free_fdtable(fdt);
457 kmem_cache_free(files_cachep, files);
458 }
459 }
460
461 void exit_files(struct task_struct *tsk)
462 {
463 struct files_struct * files = tsk->files;
464
465 if (files) {
466 task_lock(tsk);
467 tsk->files = NULL;
468 task_unlock(tsk);
469 put_files_struct(files);
470 }
471 }
472
473 struct files_struct init_files = {
474 .count = ATOMIC_INIT(1),
475 .fdt = &init_files.fdtab,
476 .fdtab = {
477 .max_fds = NR_OPEN_DEFAULT,
478 .fd = &init_files.fd_array[0],
479 .close_on_exec = init_files.close_on_exec_init,
480 .open_fds = init_files.open_fds_init,
481 .full_fds_bits = init_files.full_fds_bits_init,
482 },
483 .file_lock = __SPIN_LOCK_UNLOCKED(init_files.file_lock),
484 .resize_wait = __WAIT_QUEUE_HEAD_INITIALIZER(init_files.resize_wait),
485 };
486
487 static unsigned int find_next_fd(struct fdtable *fdt, unsigned int start)
488 {
489 unsigned int maxfd = fdt->max_fds; /* always multiple of BITS_PER_LONG */
490 unsigned int maxbit = maxfd / BITS_PER_LONG;
491 unsigned int bitbit = start / BITS_PER_LONG;
492
493 bitbit = find_next_zero_bit(fdt->full_fds_bits, maxbit, bitbit) * BITS_PER_LONG;
494 if (bitbit >= maxfd)
495 return maxfd;
496 if (bitbit > start)
497 start = bitbit;
498 return find_next_zero_bit(fdt->open_fds, maxfd, start);
499 }
500
501 /*
502 * allocate a file descriptor, mark it busy.
503 */
504 static int alloc_fd(unsigned start, unsigned end, unsigned flags)
505 {
506 struct files_struct *files = current->files;
507 unsigned int fd;
508 int error;
509 struct fdtable *fdt;
510
511 spin_lock(&files->file_lock);
512 repeat:
513 fdt = files_fdtable(files);
514 fd = start;
515 if (fd < files->next_fd)
516 fd = files->next_fd;
517
518 if (fd < fdt->max_fds)
519 fd = find_next_fd(fdt, fd);
520
521 /*
522 * N.B. For clone tasks sharing a files structure, this test
523 * will limit the total number of files that can be opened.
524 */
525 error = -EMFILE;
526 if (fd >= end)
527 goto out;
528
529 error = expand_files(files, fd);
530 if (error < 0)
531 goto out;
532
533 /*
534 * If we needed to expand the fs array we
535 * might have blocked - try again.
536 */
537 if (error)
538 goto repeat;
539
540 if (start <= files->next_fd)
541 files->next_fd = fd + 1;
542
543 __set_open_fd(fd, fdt);
544 if (flags & O_CLOEXEC)
545 __set_close_on_exec(fd, fdt);
546 else
547 __clear_close_on_exec(fd, fdt);
548 error = fd;
549 #if 1
550 /* Sanity check */
551 if (rcu_access_pointer(fdt->fd[fd]) != NULL) {
552 printk(KERN_WARNING "alloc_fd: slot %d not NULL!\n", fd);
553 rcu_assign_pointer(fdt->fd[fd], NULL);
554 }
555 #endif
556
557 out:
558 spin_unlock(&files->file_lock);
559 return error;
560 }
561
562 int __get_unused_fd_flags(unsigned flags, unsigned long nofile)
563 {
564 return alloc_fd(0, nofile, flags);
565 }
566
567 int get_unused_fd_flags(unsigned flags)
568 {
569 return __get_unused_fd_flags(flags, rlimit(RLIMIT_NOFILE));
570 }
571 EXPORT_SYMBOL(get_unused_fd_flags);
572
573 static void __put_unused_fd(struct files_struct *files, unsigned int fd)
574 {
575 struct fdtable *fdt = files_fdtable(files);
576 __clear_open_fd(fd, fdt);
577 if (fd < files->next_fd)
578 files->next_fd = fd;
579 }
580
581 void put_unused_fd(unsigned int fd)
582 {
583 struct files_struct *files = current->files;
584 spin_lock(&files->file_lock);
585 __put_unused_fd(files, fd);
586 spin_unlock(&files->file_lock);
587 }
588
589 EXPORT_SYMBOL(put_unused_fd);
590
591 /*
592 * Install a file pointer in the fd array.
593 *
594 * The VFS is full of places where we drop the files lock between
595 * setting the open_fds bitmap and installing the file in the file
596 * array. At any such point, we are vulnerable to a dup2() race
597 * installing a file in the array before us. We need to detect this and
598 * fput() the struct file we are about to overwrite in this case.
599 *
600 * It should never happen - if we allow dup2() do it, _really_ bad things
601 * will follow.
602 *
603 * This consumes the "file" refcount, so callers should treat it
604 * as if they had called fput(file).
605 */
606
607 void fd_install(unsigned int fd, struct file *file)
608 {
609 struct files_struct *files = current->files;
610 struct fdtable *fdt;
611
612 if (WARN_ON_ONCE(unlikely(file->f_mode & FMODE_BACKING)))
613 return;
614
615 rcu_read_lock_sched();
616
617 if (unlikely(files->resize_in_progress)) {
618 rcu_read_unlock_sched();
619 spin_lock(&files->file_lock);
620 fdt = files_fdtable(files);
621 BUG_ON(fdt->fd[fd] != NULL);
622 rcu_assign_pointer(fdt->fd[fd], file);
623 spin_unlock(&files->file_lock);
624 return;
625 }
626 /* coupled with smp_wmb() in expand_fdtable() */
627 smp_rmb();
628 fdt = rcu_dereference_sched(files->fdt);
629 BUG_ON(fdt->fd[fd] != NULL);
630 rcu_assign_pointer(fdt->fd[fd], file);
631 rcu_read_unlock_sched();
632 }
633
634 EXPORT_SYMBOL(fd_install);
635
636 /**
637 * file_close_fd_locked - return file associated with fd
638 * @files: file struct to retrieve file from
639 * @fd: file descriptor to retrieve file for
640 *
641 * Doesn't take a separate reference count.
642 *
643 * Context: files_lock must be held.
644 *
645 * Returns: The file associated with @fd (NULL if @fd is not open)
646 */
647 struct file *file_close_fd_locked(struct files_struct *files, unsigned fd)
648 {
649 struct fdtable *fdt = files_fdtable(files);
650 struct file *file;
651
652 lockdep_assert_held(&files->file_lock);
653
654 if (fd >= fdt->max_fds)
655 return NULL;
656
657 fd = array_index_nospec(fd, fdt->max_fds);
658 file = fdt->fd[fd];
659 if (file) {
660 rcu_assign_pointer(fdt->fd[fd], NULL);
661 __put_unused_fd(files, fd);
662 }
663 return file;
664 }
665
666 int close_fd(unsigned fd)
667 {
668 struct files_struct *files = current->files;
669 struct file *file;
670
671 spin_lock(&files->file_lock);
672 file = file_close_fd_locked(files, fd);
673 spin_unlock(&files->file_lock);
674 if (!file)
675 return -EBADF;
676
677 return filp_close(file, files);
678 }
679 EXPORT_SYMBOL(close_fd); /* for ksys_close() */
680
681 /**
682 * last_fd - return last valid index into fd table
683 * @fdt: File descriptor table.
684 *
685 * Context: Either rcu read lock or files_lock must be held.
686 *
687 * Returns: Last valid index into fdtable.
688 */
689 static inline unsigned last_fd(struct fdtable *fdt)
690 {
691 return fdt->max_fds - 1;
692 }
693
694 static inline void __range_cloexec(struct files_struct *cur_fds,
695 unsigned int fd, unsigned int max_fd)
696 {
697 struct fdtable *fdt;
698
699 /* make sure we're using the correct maximum value */
700 spin_lock(&cur_fds->file_lock);
701 fdt = files_fdtable(cur_fds);
702 max_fd = min(last_fd(fdt), max_fd);
703 if (fd <= max_fd)
704 bitmap_set(fdt->close_on_exec, fd, max_fd - fd + 1);
705 spin_unlock(&cur_fds->file_lock);
706 }
707
708 static inline void __range_close(struct files_struct *files, unsigned int fd,
709 unsigned int max_fd)
710 {
711 struct file *file;
712 unsigned n;
713
714 spin_lock(&files->file_lock);
715 n = last_fd(files_fdtable(files));
716 max_fd = min(max_fd, n);
717
718 for (; fd <= max_fd; fd++) {
719 file = file_close_fd_locked(files, fd);
720 if (file) {
721 spin_unlock(&files->file_lock);
722 filp_close(file, files);
723 cond_resched();
724 spin_lock(&files->file_lock);
725 } else if (need_resched()) {
726 spin_unlock(&files->file_lock);
727 cond_resched();
728 spin_lock(&files->file_lock);
729 }
730 }
731 spin_unlock(&files->file_lock);
732 }
733
734 /**
735 * __close_range() - Close all file descriptors in a given range.
736 *
737 * @fd: starting file descriptor to close
738 * @max_fd: last file descriptor to close
739 * @flags: CLOSE_RANGE flags.
740 *
741 * This closes a range of file descriptors. All file descriptors
742 * from @fd up to and including @max_fd are closed.
743 */
744 int __close_range(unsigned fd, unsigned max_fd, unsigned int flags)
745 {
746 struct task_struct *me = current;
747 struct files_struct *cur_fds = me->files, *fds = NULL;
748
749 if (flags & ~(CLOSE_RANGE_UNSHARE | CLOSE_RANGE_CLOEXEC))
750 return -EINVAL;
751
752 if (fd > max_fd)
753 return -EINVAL;
754
755 if (flags & CLOSE_RANGE_UNSHARE) {
756 int ret;
757 unsigned int max_unshare_fds = NR_OPEN_MAX;
758
759 /*
760 * If the caller requested all fds to be made cloexec we always
761 * copy all of the file descriptors since they still want to
762 * use them.
763 */
764 if (!(flags & CLOSE_RANGE_CLOEXEC)) {
765 /*
766 * If the requested range is greater than the current
767 * maximum, we're closing everything so only copy all
768 * file descriptors beneath the lowest file descriptor.
769 */
770 rcu_read_lock();
771 if (max_fd >= last_fd(files_fdtable(cur_fds)))
772 max_unshare_fds = fd;
773 rcu_read_unlock();
774 }
775
776 ret = unshare_fd(CLONE_FILES, max_unshare_fds, &fds);
777 if (ret)
778 return ret;
779
780 /*
781 * We used to share our file descriptor table, and have now
782 * created a private one, make sure we're using it below.
783 */
784 if (fds)
785 swap(cur_fds, fds);
786 }
787
788 if (flags & CLOSE_RANGE_CLOEXEC)
789 __range_cloexec(cur_fds, fd, max_fd);
790 else
791 __range_close(cur_fds, fd, max_fd);
792
793 if (fds) {
794 /*
795 * We're done closing the files we were supposed to. Time to install
796 * the new file descriptor table and drop the old one.
797 */
798 task_lock(me);
799 me->files = cur_fds;
800 task_unlock(me);
801 put_files_struct(fds);
802 }
803
804 return 0;
805 }
806
807 /**
808 * file_close_fd - return file associated with fd
809 * @fd: file descriptor to retrieve file for
810 *
811 * Doesn't take a separate reference count.
812 *
813 * Returns: The file associated with @fd (NULL if @fd is not open)
814 */
815 struct file *file_close_fd(unsigned int fd)
816 {
817 struct files_struct *files = current->files;
818 struct file *file;
819
820 spin_lock(&files->file_lock);
821 file = file_close_fd_locked(files, fd);
822 spin_unlock(&files->file_lock);
823
824 return file;
825 }
826
827 void do_close_on_exec(struct files_struct *files)
828 {
829 unsigned i;
830 struct fdtable *fdt;
831
832 /* exec unshares first */
833 spin_lock(&files->file_lock);
834 for (i = 0; ; i++) {
835 unsigned long set;
836 unsigned fd = i * BITS_PER_LONG;
837 fdt = files_fdtable(files);
838 if (fd >= fdt->max_fds)
839 break;
840 set = fdt->close_on_exec[i];
841 if (!set)
842 continue;
843 fdt->close_on_exec[i] = 0;
844 for ( ; set ; fd++, set >>= 1) {
845 struct file *file;
846 if (!(set & 1))
847 continue;
848 file = fdt->fd[fd];
849 if (!file)
850 continue;
851 rcu_assign_pointer(fdt->fd[fd], NULL);
852 __put_unused_fd(files, fd);
853 spin_unlock(&files->file_lock);
854 filp_close(file, files);
855 cond_resched();
856 spin_lock(&files->file_lock);
857 }
858
859 }
860 spin_unlock(&files->file_lock);
861 }
862
863 static struct file *__get_file_rcu(struct file __rcu **f)
864 {
865 struct file __rcu *file;
866 struct file __rcu *file_reloaded;
867 struct file __rcu *file_reloaded_cmp;
868
869 file = rcu_dereference_raw(*f);
870 if (!file)
871 return NULL;
872
873 if (unlikely(!atomic_long_inc_not_zero(&file->f_count)))
874 return ERR_PTR(-EAGAIN);
875
876 file_reloaded = rcu_dereference_raw(*f);
877
878 /*
879 * Ensure that all accesses have a dependency on the load from
880 * rcu_dereference_raw() above so we get correct ordering
881 * between reuse/allocation and the pointer check below.
882 */
883 file_reloaded_cmp = file_reloaded;
884 OPTIMIZER_HIDE_VAR(file_reloaded_cmp);
885
886 /*
887 * atomic_long_inc_not_zero() above provided a full memory
888 * barrier when we acquired a reference.
889 *
890 * This is paired with the write barrier from assigning to the
891 * __rcu protected file pointer so that if that pointer still
892 * matches the current file, we know we have successfully
893 * acquired a reference to the right file.
894 *
895 * If the pointers don't match the file has been reallocated by
896 * SLAB_TYPESAFE_BY_RCU.
897 */
898 if (file == file_reloaded_cmp)
899 return file_reloaded;
900
901 fput(file);
902 return ERR_PTR(-EAGAIN);
903 }
904
905 /**
906 * get_file_rcu - try go get a reference to a file under rcu
907 * @f: the file to get a reference on
908 *
909 * This function tries to get a reference on @f carefully verifying that
910 * @f hasn't been reused.
911 *
912 * This function should rarely have to be used and only by users who
913 * understand the implications of SLAB_TYPESAFE_BY_RCU. Try to avoid it.
914 *
915 * Return: Returns @f with the reference count increased or NULL.
916 */
917 struct file *get_file_rcu(struct file __rcu **f)
918 {
919 for (;;) {
920 struct file __rcu *file;
921
922 file = __get_file_rcu(f);
923 if (!IS_ERR(file))
924 return file;
925 }
926 }
927 EXPORT_SYMBOL_GPL(get_file_rcu);
928
929 /**
930 * get_file_active - try go get a reference to a file
931 * @f: the file to get a reference on
932 *
933 * In contast to get_file_rcu() the pointer itself isn't part of the
934 * reference counting.
935 *
936 * This function should rarely have to be used and only by users who
937 * understand the implications of SLAB_TYPESAFE_BY_RCU. Try to avoid it.
938 *
939 * Return: Returns @f with the reference count increased or NULL.
940 */
941 struct file *get_file_active(struct file **f)
942 {
943 struct file __rcu *file;
944
945 rcu_read_lock();
946 file = __get_file_rcu(f);
947 rcu_read_unlock();
948 if (IS_ERR(file))
949 file = NULL;
950 return file;
951 }
952 EXPORT_SYMBOL_GPL(get_file_active);
953
954 static inline struct file *__fget_files_rcu(struct files_struct *files,
955 unsigned int fd, fmode_t mask)
956 {
957 for (;;) {
958 struct file *file;
959 struct fdtable *fdt = rcu_dereference_raw(files->fdt);
960 struct file __rcu **fdentry;
961 unsigned long nospec_mask;
962
963 /* Mask is a 0 for invalid fd's, ~0 for valid ones */
964 nospec_mask = array_index_mask_nospec(fd, fdt->max_fds);
965
966 /*
967 * fdentry points to the 'fd' offset, or fdt->fd[0].
968 * Loading from fdt->fd[0] is always safe, because the
969 * array always exists.
970 */
971 fdentry = fdt->fd + (fd & nospec_mask);
972
973 /* Do the load, then mask any invalid result */
974 file = rcu_dereference_raw(*fdentry);
975 file = (void *)(nospec_mask & (unsigned long)file);
976 if (unlikely(!file))
977 return NULL;
978
979 /*
980 * Ok, we have a file pointer that was valid at
981 * some point, but it might have become stale since.
982 *
983 * We need to confirm it by incrementing the refcount
984 * and then check the lookup again.
985 *
986 * atomic_long_inc_not_zero() gives us a full memory
987 * barrier. We only really need an 'acquire' one to
988 * protect the loads below, but we don't have that.
989 */
990 if (unlikely(!atomic_long_inc_not_zero(&file->f_count)))
991 continue;
992
993 /*
994 * Such a race can take two forms:
995 *
996 * (a) the file ref already went down to zero and the
997 * file hasn't been reused yet or the file count
998 * isn't zero but the file has already been reused.
999 *
1000 * (b) the file table entry has changed under us.
1001 * Note that we don't need to re-check the 'fdt->fd'
1002 * pointer having changed, because it always goes
1003 * hand-in-hand with 'fdt'.
1004 *
1005 * If so, we need to put our ref and try again.
1006 */
1007 if (unlikely(file != rcu_dereference_raw(*fdentry)) ||
1008 unlikely(rcu_dereference_raw(files->fdt) != fdt)) {
1009 fput(file);
1010 continue;
1011 }
1012
1013 /*
1014 * This isn't the file we're looking for or we're not
1015 * allowed to get a reference to it.
1016 */
1017 if (unlikely(file->f_mode & mask)) {
1018 fput(file);
1019 return NULL;
1020 }
1021
1022 /*
1023 * Ok, we have a ref to the file, and checked that it
1024 * still exists.
1025 */
1026 return file;
1027 }
1028 }
1029
1030 static struct file *__fget_files(struct files_struct *files, unsigned int fd,
1031 fmode_t mask)
1032 {
1033 struct file *file;
1034
1035 rcu_read_lock();
1036 file = __fget_files_rcu(files, fd, mask);
1037 rcu_read_unlock();
1038
1039 return file;
1040 }
1041
1042 static inline struct file *__fget(unsigned int fd, fmode_t mask)
1043 {
1044 return __fget_files(current->files, fd, mask);
1045 }
1046
1047 struct file *fget(unsigned int fd)
1048 {
1049 return __fget(fd, FMODE_PATH);
1050 }
1051 EXPORT_SYMBOL(fget);
1052
1053 struct file *fget_raw(unsigned int fd)
1054 {
1055 return __fget(fd, 0);
1056 }
1057 EXPORT_SYMBOL(fget_raw);
1058
1059 struct file *fget_task(struct task_struct *task, unsigned int fd)
1060 {
1061 struct file *file = NULL;
1062
1063 task_lock(task);
1064 if (task->files)
1065 file = __fget_files(task->files, fd, 0);
1066 task_unlock(task);
1067
1068 return file;
1069 }
1070
1071 struct file *lookup_fdget_rcu(unsigned int fd)
1072 {
1073 return __fget_files_rcu(current->files, fd, 0);
1074
1075 }
1076 EXPORT_SYMBOL_GPL(lookup_fdget_rcu);
1077
1078 struct file *task_lookup_fdget_rcu(struct task_struct *task, unsigned int fd)
1079 {
1080 /* Must be called with rcu_read_lock held */
1081 struct files_struct *files;
1082 struct file *file = NULL;
1083
1084 task_lock(task);
1085 files = task->files;
1086 if (files)
1087 file = __fget_files_rcu(files, fd, 0);
1088 task_unlock(task);
1089
1090 return file;
1091 }
1092
1093 struct file *task_lookup_next_fdget_rcu(struct task_struct *task, unsigned int *ret_fd)
1094 {
1095 /* Must be called with rcu_read_lock held */
1096 struct files_struct *files;
1097 unsigned int fd = *ret_fd;
1098 struct file *file = NULL;
1099
1100 task_lock(task);
1101 files = task->files;
1102 if (files) {
1103 for (; fd < files_fdtable(files)->max_fds; fd++) {
1104 file = __fget_files_rcu(files, fd, 0);
1105 if (file)
1106 break;
1107 }
1108 }
1109 task_unlock(task);
1110 *ret_fd = fd;
1111 return file;
1112 }
1113 EXPORT_SYMBOL(task_lookup_next_fdget_rcu);
1114
1115 /*
1116 * Lightweight file lookup - no refcnt increment if fd table isn't shared.
1117 *
1118 * You can use this instead of fget if you satisfy all of the following
1119 * conditions:
1120 * 1) You must call fput_light before exiting the syscall and returning control
1121 * to userspace (i.e. you cannot remember the returned struct file * after
1122 * returning to userspace).
1123 * 2) You must not call filp_close on the returned struct file * in between
1124 * calls to fget_light and fput_light.
1125 * 3) You must not clone the current task in between the calls to fget_light
1126 * and fput_light.
1127 *
1128 * The fput_needed flag returned by fget_light should be passed to the
1129 * corresponding fput_light.
1130 */
1131 static unsigned long __fget_light(unsigned int fd, fmode_t mask)
1132 {
1133 struct files_struct *files = current->files;
1134 struct file *file;
1135
1136 /*
1137 * If another thread is concurrently calling close_fd() followed
1138 * by put_files_struct(), we must not observe the old table
1139 * entry combined with the new refcount - otherwise we could
1140 * return a file that is concurrently being freed.
1141 *
1142 * atomic_read_acquire() pairs with atomic_dec_and_test() in
1143 * put_files_struct().
1144 */
1145 if (likely(atomic_read_acquire(&files->count) == 1)) {
1146 file = files_lookup_fd_raw(files, fd);
1147 if (!file || unlikely(file->f_mode & mask))
1148 return 0;
1149 return (unsigned long)file;
1150 } else {
1151 file = __fget_files(files, fd, mask);
1152 if (!file)
1153 return 0;
1154 return FDPUT_FPUT | (unsigned long)file;
1155 }
1156 }
1157 unsigned long __fdget(unsigned int fd)
1158 {
1159 return __fget_light(fd, FMODE_PATH);
1160 }
1161 EXPORT_SYMBOL(__fdget);
1162
1163 unsigned long __fdget_raw(unsigned int fd)
1164 {
1165 return __fget_light(fd, 0);
1166 }
1167
1168 /*
1169 * Try to avoid f_pos locking. We only need it if the
1170 * file is marked for FMODE_ATOMIC_POS, and it can be
1171 * accessed multiple ways.
1172 *
1173 * Always do it for directories, because pidfd_getfd()
1174 * can make a file accessible even if it otherwise would
1175 * not be, and for directories this is a correctness
1176 * issue, not a "POSIX requirement".
1177 */
1178 static inline bool file_needs_f_pos_lock(struct file *file)
1179 {
1180 return (file->f_mode & FMODE_ATOMIC_POS) &&
1181 (file_count(file) > 1 || file->f_op->iterate_shared);
1182 }
1183
1184 unsigned long __fdget_pos(unsigned int fd)
1185 {
1186 unsigned long v = __fdget(fd);
1187 struct file *file = (struct file *)(v & ~3);
1188
1189 if (file && file_needs_f_pos_lock(file)) {
1190 v |= FDPUT_POS_UNLOCK;
1191 mutex_lock(&file->f_pos_lock);
1192 }
1193 return v;
1194 }
1195
1196 void __f_unlock_pos(struct file *f)
1197 {
1198 mutex_unlock(&f->f_pos_lock);
1199 }
1200
1201 /*
1202 * We only lock f_pos if we have threads or if the file might be
1203 * shared with another process. In both cases we'll have an elevated
1204 * file count (done either by fdget() or by fork()).
1205 */
1206
1207 void set_close_on_exec(unsigned int fd, int flag)
1208 {
1209 struct files_struct *files = current->files;
1210 struct fdtable *fdt;
1211 spin_lock(&files->file_lock);
1212 fdt = files_fdtable(files);
1213 if (flag)
1214 __set_close_on_exec(fd, fdt);
1215 else
1216 __clear_close_on_exec(fd, fdt);
1217 spin_unlock(&files->file_lock);
1218 }
1219
1220 bool get_close_on_exec(unsigned int fd)
1221 {
1222 bool res;
1223 rcu_read_lock();
1224 res = close_on_exec(fd, current->files);
1225 rcu_read_unlock();
1226 return res;
1227 }
1228
1229 static int do_dup2(struct files_struct *files,
1230 struct file *file, unsigned fd, unsigned flags)
1231 __releases(&files->file_lock)
1232 {
1233 struct file *tofree;
1234 struct fdtable *fdt;
1235
1236 /*
1237 * We need to detect attempts to do dup2() over allocated but still
1238 * not finished descriptor. NB: OpenBSD avoids that at the price of
1239 * extra work in their equivalent of fget() - they insert struct
1240 * file immediately after grabbing descriptor, mark it larval if
1241 * more work (e.g. actual opening) is needed and make sure that
1242 * fget() treats larval files as absent. Potentially interesting,
1243 * but while extra work in fget() is trivial, locking implications
1244 * and amount of surgery on open()-related paths in VFS are not.
1245 * FreeBSD fails with -EBADF in the same situation, NetBSD "solution"
1246 * deadlocks in rather amusing ways, AFAICS. All of that is out of
1247 * scope of POSIX or SUS, since neither considers shared descriptor
1248 * tables and this condition does not arise without those.
1249 */
1250 fdt = files_fdtable(files);
1251 fd = array_index_nospec(fd, fdt->max_fds);
1252 tofree = fdt->fd[fd];
1253 if (!tofree && fd_is_open(fd, fdt))
1254 goto Ebusy;
1255 get_file(file);
1256 rcu_assign_pointer(fdt->fd[fd], file);
1257 __set_open_fd(fd, fdt);
1258 if (flags & O_CLOEXEC)
1259 __set_close_on_exec(fd, fdt);
1260 else
1261 __clear_close_on_exec(fd, fdt);
1262 spin_unlock(&files->file_lock);
1263
1264 if (tofree)
1265 filp_close(tofree, files);
1266
1267 return fd;
1268
1269 Ebusy:
1270 spin_unlock(&files->file_lock);
1271 return -EBUSY;
1272 }
1273
1274 int replace_fd(unsigned fd, struct file *file, unsigned flags)
1275 {
1276 int err;
1277 struct files_struct *files = current->files;
1278
1279 if (!file)
1280 return close_fd(fd);
1281
1282 if (fd >= rlimit(RLIMIT_NOFILE))
1283 return -EBADF;
1284
1285 spin_lock(&files->file_lock);
1286 err = expand_files(files, fd);
1287 if (unlikely(err < 0))
1288 goto out_unlock;
1289 return do_dup2(files, file, fd, flags);
1290
1291 out_unlock:
1292 spin_unlock(&files->file_lock);
1293 return err;
1294 }
1295
1296 /**
1297 * receive_fd() - Install received file into file descriptor table
1298 * @file: struct file that was received from another process
1299 * @ufd: __user pointer to write new fd number to
1300 * @o_flags: the O_* flags to apply to the new fd entry
1301 *
1302 * Installs a received file into the file descriptor table, with appropriate
1303 * checks and count updates. Optionally writes the fd number to userspace, if
1304 * @ufd is non-NULL.
1305 *
1306 * This helper handles its own reference counting of the incoming
1307 * struct file.
1308 *
1309 * Returns newly install fd or -ve on error.
1310 */
1311 int receive_fd(struct file *file, int __user *ufd, unsigned int o_flags)
1312 {
1313 int new_fd;
1314 int error;
1315
1316 error = security_file_receive(file);
1317 if (error)
1318 return error;
1319
1320 new_fd = get_unused_fd_flags(o_flags);
1321 if (new_fd < 0)
1322 return new_fd;
1323
1324 if (ufd) {
1325 error = put_user(new_fd, ufd);
1326 if (error) {
1327 put_unused_fd(new_fd);
1328 return error;
1329 }
1330 }
1331
1332 fd_install(new_fd, get_file(file));
1333 __receive_sock(file);
1334 return new_fd;
1335 }
1336 EXPORT_SYMBOL_GPL(receive_fd);
1337
1338 int receive_fd_replace(int new_fd, struct file *file, unsigned int o_flags)
1339 {
1340 int error;
1341
1342 error = security_file_receive(file);
1343 if (error)
1344 return error;
1345 error = replace_fd(new_fd, file, o_flags);
1346 if (error)
1347 return error;
1348 __receive_sock(file);
1349 return new_fd;
1350 }
1351
1352 static int ksys_dup3(unsigned int oldfd, unsigned int newfd, int flags)
1353 {
1354 int err = -EBADF;
1355 struct file *file;
1356 struct files_struct *files = current->files;
1357
1358 if ((flags & ~O_CLOEXEC) != 0)
1359 return -EINVAL;
1360
1361 if (unlikely(oldfd == newfd))
1362 return -EINVAL;
1363
1364 if (newfd >= rlimit(RLIMIT_NOFILE))
1365 return -EBADF;
1366
1367 spin_lock(&files->file_lock);
1368 err = expand_files(files, newfd);
1369 file = files_lookup_fd_locked(files, oldfd);
1370 if (unlikely(!file))
1371 goto Ebadf;
1372 if (unlikely(err < 0)) {
1373 if (err == -EMFILE)
1374 goto Ebadf;
1375 goto out_unlock;
1376 }
1377 return do_dup2(files, file, newfd, flags);
1378
1379 Ebadf:
1380 err = -EBADF;
1381 out_unlock:
1382 spin_unlock(&files->file_lock);
1383 return err;
1384 }
1385
1386 SYSCALL_DEFINE3(dup3, unsigned int, oldfd, unsigned int, newfd, int, flags)
1387 {
1388 return ksys_dup3(oldfd, newfd, flags);
1389 }
1390
1391 SYSCALL_DEFINE2(dup2, unsigned int, oldfd, unsigned int, newfd)
1392 {
1393 if (unlikely(newfd == oldfd)) { /* corner case */
1394 struct files_struct *files = current->files;
1395 struct file *f;
1396 int retval = oldfd;
1397
1398 rcu_read_lock();
1399 f = __fget_files_rcu(files, oldfd, 0);
1400 if (!f)
1401 retval = -EBADF;
1402 rcu_read_unlock();
1403 if (f)
1404 fput(f);
1405 return retval;
1406 }
1407 return ksys_dup3(oldfd, newfd, 0);
1408 }
1409
1410 SYSCALL_DEFINE1(dup, unsigned int, fildes)
1411 {
1412 int ret = -EBADF;
1413 struct file *file = fget_raw(fildes);
1414
1415 if (file) {
1416 ret = get_unused_fd_flags(0);
1417 if (ret >= 0)
1418 fd_install(ret, file);
1419 else
1420 fput(file);
1421 }
1422 return ret;
1423 }
1424
1425 int f_dupfd(unsigned int from, struct file *file, unsigned flags)
1426 {
1427 unsigned long nofile = rlimit(RLIMIT_NOFILE);
1428 int err;
1429 if (from >= nofile)
1430 return -EINVAL;
1431 err = alloc_fd(from, nofile, flags);
1432 if (err >= 0) {
1433 get_file(file);
1434 fd_install(err, file);
1435 }
1436 return err;
1437 }
1438
1439 int iterate_fd(struct files_struct *files, unsigned n,
1440 int (*f)(const void *, struct file *, unsigned),
1441 const void *p)
1442 {
1443 struct fdtable *fdt;
1444 int res = 0;
1445 if (!files)
1446 return 0;
1447 spin_lock(&files->file_lock);
1448 for (fdt = files_fdtable(files); n < fdt->max_fds; n++) {
1449 struct file *file;
1450 file = rcu_dereference_check_fdtable(files, fdt->fd[n]);
1451 if (!file)
1452 continue;
1453 res = f(p, file, n);
1454 if (res)
1455 break;
1456 }
1457 spin_unlock(&files->file_lock);
1458 return res;
1459 }
1460 EXPORT_SYMBOL(iterate_fd);
1461
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.