1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /* Copyright (c) 2021 Facebook
3 */
4
5 #ifndef __MMAP_UNLOCK_WORK_H__
6 #define __MMAP_UNLOCK_WORK_H__
7 #include <linux/irq_work.h>
8
9 /* irq_work to run mmap_read_unlock() in irq_work */
10 struct mmap_unlock_irq_work {
11 struct irq_work irq_work;
12 struct mm_struct *mm;
13 };
14
15 DECLARE_PER_CPU(struct mmap_unlock_irq_work, mmap_unlock_work);
16
17 /*
18 * We cannot do mmap_read_unlock() when the irq is disabled, because of
19 * risk to deadlock with rq_lock. To look up vma when the irqs are
20 * disabled, we need to run mmap_read_unlock() in irq_work. We use a
21 * percpu variable to do the irq_work. If the irq_work is already used
22 * by another lookup, we fall over.
23 */
24 static inline bool bpf_mmap_unlock_get_irq_work(struct mmap_unlock_irq_work **work_ptr)
25 {
26 struct mmap_unlock_irq_work *work = NULL;
27 bool irq_work_busy = false;
28
29 if (irqs_disabled()) {
30 if (!IS_ENABLED(CONFIG_PREEMPT_RT)) {
31 work = this_cpu_ptr(&mmap_unlock_work);
32 if (irq_work_is_busy(&work->irq_work)) {
33 /* cannot queue more up_read, fallback */
34 irq_work_busy = true;
35 }
36 } else {
37 /*
38 * PREEMPT_RT does not allow to trylock mmap sem in
39 * interrupt disabled context. Force the fallback code.
40 */
41 irq_work_busy = true;
42 }
43 }
44
45 *work_ptr = work;
46 return irq_work_busy;
47 }
48
49 static inline void bpf_mmap_unlock_mm(struct mmap_unlock_irq_work *work, struct mm_struct *mm)
50 {
51 if (!work) {
52 mmap_read_unlock(mm);
53 } else {
54 work->mm = mm;
55
56 /* The lock will be released once we're out of interrupt
57 * context. Tell lockdep that we've released it now so
58 * it doesn't complain that we forgot to release it.
59 */
60 rwsem_release(&mm->mmap_lock.dep_map, _RET_IP_);
61 irq_work_queue(&work->irq_work);
62 }
63 }
64
65 #endif /* __MMAP_UNLOCK_WORK_H__ */
66
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.