1 #ifndef _LINUX_MMAP_LOCK_H 1 #ifndef _LINUX_MMAP_LOCK_H
2 #define _LINUX_MMAP_LOCK_H 2 #define _LINUX_MMAP_LOCK_H
3 3
4 #include <linux/lockdep.h> <<
5 #include <linux/mm_types.h> <<
6 #include <linux/mmdebug.h> 4 #include <linux/mmdebug.h>
7 #include <linux/rwsem.h> <<
8 #include <linux/tracepoint-defs.h> <<
9 #include <linux/types.h> <<
10 5
11 #define MMAP_LOCK_INITIALIZER(name) \ 6 #define MMAP_LOCK_INITIALIZER(name) \
12 .mmap_lock = __RWSEM_INITIALIZER((name 7 .mmap_lock = __RWSEM_INITIALIZER((name).mmap_lock),
13 8
14 DECLARE_TRACEPOINT(mmap_lock_start_locking); <<
15 DECLARE_TRACEPOINT(mmap_lock_acquire_returned) <<
16 DECLARE_TRACEPOINT(mmap_lock_released); <<
17 <<
18 #ifdef CONFIG_TRACING <<
19 <<
20 void __mmap_lock_do_trace_start_locking(struct <<
21 void __mmap_lock_do_trace_acquire_returned(str <<
22 boo <<
23 void __mmap_lock_do_trace_released(struct mm_s <<
24 <<
25 static inline void __mmap_lock_trace_start_loc <<
26 <<
27 { <<
28 if (tracepoint_enabled(mmap_lock_start <<
29 __mmap_lock_do_trace_start_loc <<
30 } <<
31 <<
32 static inline void __mmap_lock_trace_acquire_r <<
33 <<
34 { <<
35 if (tracepoint_enabled(mmap_lock_acqui <<
36 __mmap_lock_do_trace_acquire_r <<
37 } <<
38 <<
39 static inline void __mmap_lock_trace_released( <<
40 { <<
41 if (tracepoint_enabled(mmap_lock_relea <<
42 __mmap_lock_do_trace_released( <<
43 } <<
44 <<
45 #else /* !CONFIG_TRACING */ <<
46 <<
47 static inline void __mmap_lock_trace_start_loc <<
48 <<
49 { <<
50 } <<
51 <<
52 static inline void __mmap_lock_trace_acquire_r <<
53 <<
54 { <<
55 } <<
56 <<
57 static inline void __mmap_lock_trace_released( <<
58 { <<
59 } <<
60 <<
61 #endif /* CONFIG_TRACING */ <<
62 <<
63 static inline void mmap_assert_locked(const st <<
64 { <<
65 rwsem_assert_held(&mm->mmap_lock); <<
66 } <<
67 <<
68 static inline void mmap_assert_write_locked(co <<
69 { <<
70 rwsem_assert_held_write(&mm->mmap_lock <<
71 } <<
72 <<
73 #ifdef CONFIG_PER_VMA_LOCK <<
74 /* <<
75 * Drop all currently-held per-VMA locks. <<
76 * This is called from the mmap_lock implement <<
77 * a write-locked mmap_lock (or downgrading it <<
78 * This should normally NOT be called manually <<
79 * If you want to call this manually anyway, k <<
80 * *all* VMA write locks, including ones from <<
81 */ <<
82 static inline void vma_end_write_all(struct mm <<
83 { <<
84 mmap_assert_write_locked(mm); <<
85 /* <<
86 * Nobody can concurrently modify mm-> <<
87 * mmap_lock being held. <<
88 * We need RELEASE semantics here to e <<
89 * the VMA take effect before we unloc <<
90 * Pairs with ACQUIRE semantics in vma <<
91 */ <<
92 smp_store_release(&mm->mm_lock_seq, mm <<
93 } <<
94 #else <<
95 static inline void vma_end_write_all(struct mm <<
96 #endif <<
97 <<
98 static inline void mmap_init_lock(struct mm_st 9 static inline void mmap_init_lock(struct mm_struct *mm)
99 { 10 {
100 init_rwsem(&mm->mmap_lock); 11 init_rwsem(&mm->mmap_lock);
101 } 12 }
102 13
103 static inline void mmap_write_lock(struct mm_s 14 static inline void mmap_write_lock(struct mm_struct *mm)
104 { 15 {
105 __mmap_lock_trace_start_locking(mm, tr <<
106 down_write(&mm->mmap_lock); 16 down_write(&mm->mmap_lock);
107 __mmap_lock_trace_acquire_returned(mm, <<
108 } 17 }
109 18
110 static inline void mmap_write_lock_nested(stru 19 static inline void mmap_write_lock_nested(struct mm_struct *mm, int subclass)
111 { 20 {
112 __mmap_lock_trace_start_locking(mm, tr <<
113 down_write_nested(&mm->mmap_lock, subc 21 down_write_nested(&mm->mmap_lock, subclass);
114 __mmap_lock_trace_acquire_returned(mm, <<
115 } 22 }
116 23
117 static inline int mmap_write_lock_killable(str 24 static inline int mmap_write_lock_killable(struct mm_struct *mm)
118 { 25 {
119 int ret; !! 26 return down_write_killable(&mm->mmap_lock);
>> 27 }
120 28
121 __mmap_lock_trace_start_locking(mm, tr !! 29 static inline bool mmap_write_trylock(struct mm_struct *mm)
122 ret = down_write_killable(&mm->mmap_lo !! 30 {
123 __mmap_lock_trace_acquire_returned(mm, !! 31 return down_write_trylock(&mm->mmap_lock) != 0;
124 return ret; <<
125 } 32 }
126 33
127 static inline void mmap_write_unlock(struct mm 34 static inline void mmap_write_unlock(struct mm_struct *mm)
128 { 35 {
129 __mmap_lock_trace_released(mm, true); <<
130 vma_end_write_all(mm); <<
131 up_write(&mm->mmap_lock); 36 up_write(&mm->mmap_lock);
132 } 37 }
133 38
134 static inline void mmap_write_downgrade(struct 39 static inline void mmap_write_downgrade(struct mm_struct *mm)
135 { 40 {
136 __mmap_lock_trace_acquire_returned(mm, <<
137 vma_end_write_all(mm); <<
138 downgrade_write(&mm->mmap_lock); 41 downgrade_write(&mm->mmap_lock);
139 } 42 }
140 43
141 static inline void mmap_read_lock(struct mm_st 44 static inline void mmap_read_lock(struct mm_struct *mm)
142 { 45 {
143 __mmap_lock_trace_start_locking(mm, fa <<
144 down_read(&mm->mmap_lock); 46 down_read(&mm->mmap_lock);
145 __mmap_lock_trace_acquire_returned(mm, <<
146 } 47 }
147 48
148 static inline int mmap_read_lock_killable(stru 49 static inline int mmap_read_lock_killable(struct mm_struct *mm)
149 { 50 {
150 int ret; !! 51 return down_read_killable(&mm->mmap_lock);
151 <<
152 __mmap_lock_trace_start_locking(mm, fa <<
153 ret = down_read_killable(&mm->mmap_loc <<
154 __mmap_lock_trace_acquire_returned(mm, <<
155 return ret; <<
156 } 52 }
157 53
158 static inline bool mmap_read_trylock(struct mm 54 static inline bool mmap_read_trylock(struct mm_struct *mm)
159 { 55 {
160 bool ret; !! 56 return down_read_trylock(&mm->mmap_lock) != 0;
161 <<
162 __mmap_lock_trace_start_locking(mm, fa <<
163 ret = down_read_trylock(&mm->mmap_lock <<
164 __mmap_lock_trace_acquire_returned(mm, <<
165 return ret; <<
166 } 57 }
167 58
168 static inline void mmap_read_unlock(struct mm_ 59 static inline void mmap_read_unlock(struct mm_struct *mm)
169 { 60 {
170 __mmap_lock_trace_released(mm, false); <<
171 up_read(&mm->mmap_lock); 61 up_read(&mm->mmap_lock);
172 } 62 }
173 63
>> 64 static inline bool mmap_read_trylock_non_owner(struct mm_struct *mm)
>> 65 {
>> 66 if (down_read_trylock(&mm->mmap_lock)) {
>> 67 rwsem_release(&mm->mmap_lock.dep_map, _RET_IP_);
>> 68 return true;
>> 69 }
>> 70 return false;
>> 71 }
>> 72
174 static inline void mmap_read_unlock_non_owner( 73 static inline void mmap_read_unlock_non_owner(struct mm_struct *mm)
175 { 74 {
176 __mmap_lock_trace_released(mm, false); <<
177 up_read_non_owner(&mm->mmap_lock); 75 up_read_non_owner(&mm->mmap_lock);
178 } 76 }
179 77
180 static inline int mmap_lock_is_contended(struc !! 78 static inline void mmap_assert_locked(struct mm_struct *mm)
>> 79 {
>> 80 lockdep_assert_held(&mm->mmap_lock);
>> 81 VM_BUG_ON_MM(!rwsem_is_locked(&mm->mmap_lock), mm);
>> 82 }
>> 83
>> 84 static inline void mmap_assert_write_locked(struct mm_struct *mm)
181 { 85 {
182 return rwsem_is_contended(&mm->mmap_lo !! 86 lockdep_assert_held_write(&mm->mmap_lock);
>> 87 VM_BUG_ON_MM(!rwsem_is_locked(&mm->mmap_lock), mm);
183 } 88 }
184 89
185 #endif /* _LINUX_MMAP_LOCK_H */ 90 #endif /* _LINUX_MMAP_LOCK_H */
186 91
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.