1 /* SPDX-License-Identifier: GPL-2.0 */ <<
2 #ifndef _LINUX_SWAIT_H 1 #ifndef _LINUX_SWAIT_H
3 #define _LINUX_SWAIT_H 2 #define _LINUX_SWAIT_H
4 3
5 #include <linux/list.h> 4 #include <linux/list.h>
6 #include <linux/stddef.h> 5 #include <linux/stddef.h>
7 #include <linux/spinlock.h> 6 #include <linux/spinlock.h>
8 #include <linux/wait.h> <<
9 #include <asm/current.h> 7 #include <asm/current.h>
10 8
11 /* 9 /*
12 * Simple waitqueues are semantically very dif !! 10 * Simple wait queues
13 * (wait.h). The most important difference is <<
14 * for deterministic behaviour -- IOW it has s <<
15 * times. <<
16 * 11 *
17 * Mainly, this is accomplished by two things. !! 12 * While these are very similar to the other/complex wait queues (wait.h) the
18 * from IRQ disabled, and dropping the lock up !! 13 * most important difference is that the simple waitqueue allows for
19 * priority task a chance to run. !! 14 * deterministic behaviour -- IOW it has strictly bounded IRQ and lock hold
>> 15 * times.
20 * 16 *
21 * Secondly, we had to drop a fair number of f !! 17 * In order to make this so, we had to drop a fair number of features of the
22 * code; notably: !! 18 * other waitqueue code; notably:
23 * 19 *
24 * - mixing INTERRUPTIBLE and UNINTERRUPTIBLE 20 * - mixing INTERRUPTIBLE and UNINTERRUPTIBLE sleeps on the same waitqueue;
25 * all wakeups are TASK_NORMAL in order to 21 * all wakeups are TASK_NORMAL in order to avoid O(n) lookups for the right
26 * sleeper state. 22 * sleeper state.
27 * 23 *
28 * - the !exclusive mode; because that leads !! 24 * - the exclusive mode; because this requires preserving the list order
29 * exclusive. As such swake_up_one will onl !! 25 * and this is hard.
>> 26 *
>> 27 * - custom wake functions; because you cannot give any guarantees about
>> 28 * random code.
30 * 29 *
31 * - custom wake callback functions; because !! 30 * As a side effect of this; the data structures are slimmer.
32 * about random code. This also allows swai <<
33 * raw spinlock can be used for the swait q <<
34 * 31 *
35 * As a side effect of these; the data structu !! 32 * One would recommend using this wait queue where possible.
36 * For all the above, note that simple wait qu <<
37 * very specific realtime constraints -- it is <<
38 * wait queues in most cases. <<
39 */ 33 */
40 34
41 struct task_struct; 35 struct task_struct;
42 36
43 struct swait_queue_head { 37 struct swait_queue_head {
44 raw_spinlock_t lock; 38 raw_spinlock_t lock;
45 struct list_head task_list; 39 struct list_head task_list;
46 }; 40 };
47 41
48 struct swait_queue { 42 struct swait_queue {
49 struct task_struct *task; 43 struct task_struct *task;
50 struct list_head task_list; 44 struct list_head task_list;
51 }; 45 };
52 46
53 #define __SWAITQUEUE_INITIALIZER(name) { 47 #define __SWAITQUEUE_INITIALIZER(name) { \
54 .task = current, 48 .task = current, \
55 .task_list = LIST_HEAD_INIT((name 49 .task_list = LIST_HEAD_INIT((name).task_list), \
56 } 50 }
57 51
58 #define DECLARE_SWAITQUEUE(name) 52 #define DECLARE_SWAITQUEUE(name) \
59 struct swait_queue name = __SWAITQUEUE 53 struct swait_queue name = __SWAITQUEUE_INITIALIZER(name)
60 54
61 #define __SWAIT_QUEUE_HEAD_INITIALIZER(name) { 55 #define __SWAIT_QUEUE_HEAD_INITIALIZER(name) { \
62 .lock = __RAW_SPIN_LOCK_UNLO 56 .lock = __RAW_SPIN_LOCK_UNLOCKED(name.lock), \
63 .task_list = LIST_HEAD_INIT((name 57 .task_list = LIST_HEAD_INIT((name).task_list), \
64 } 58 }
65 59
66 #define DECLARE_SWAIT_QUEUE_HEAD(name) 60 #define DECLARE_SWAIT_QUEUE_HEAD(name) \
67 struct swait_queue_head name = __SWAIT 61 struct swait_queue_head name = __SWAIT_QUEUE_HEAD_INITIALIZER(name)
68 62
69 extern void __init_swait_queue_head(struct swa 63 extern void __init_swait_queue_head(struct swait_queue_head *q, const char *name,
70 struct loc 64 struct lock_class_key *key);
71 65
72 #define init_swait_queue_head(q) 66 #define init_swait_queue_head(q) \
73 do { 67 do { \
74 static struct lock_class_key _ 68 static struct lock_class_key __key; \
75 __init_swait_queue_head((q), # 69 __init_swait_queue_head((q), #q, &__key); \
76 } while (0) 70 } while (0)
77 71
78 #ifdef CONFIG_LOCKDEP 72 #ifdef CONFIG_LOCKDEP
79 # define __SWAIT_QUEUE_HEAD_INIT_ONSTACK(name) 73 # define __SWAIT_QUEUE_HEAD_INIT_ONSTACK(name) \
80 ({ init_swait_queue_head(&name); name; 74 ({ init_swait_queue_head(&name); name; })
81 # define DECLARE_SWAIT_QUEUE_HEAD_ONSTACK(name 75 # define DECLARE_SWAIT_QUEUE_HEAD_ONSTACK(name) \
82 struct swait_queue_head name = __SWAIT 76 struct swait_queue_head name = __SWAIT_QUEUE_HEAD_INIT_ONSTACK(name)
83 #else 77 #else
84 # define DECLARE_SWAIT_QUEUE_HEAD_ONSTACK(name 78 # define DECLARE_SWAIT_QUEUE_HEAD_ONSTACK(name) \
85 DECLARE_SWAIT_QUEUE_HEAD(name) 79 DECLARE_SWAIT_QUEUE_HEAD(name)
86 #endif 80 #endif
87 81
88 /** !! 82 static inline int swait_active(struct swait_queue_head *q)
89 * swait_active -- locklessly test for waiters <<
90 * @wq: the waitqueue to test for waiters <<
91 * <<
92 * returns true if the wait list is not empty <<
93 * <<
94 * NOTE: this function is lockless and require <<
95 * lead to sporadic and non-obvious failure. <<
96 * <<
97 * NOTE2: this function has the same above imp <<
98 * <<
99 * Use either while holding swait_queue_head:: <<
100 * with an extra smp_mb() like: <<
101 * <<
102 * CPU0 - waker CPU1 - <<
103 * <<
104 * for (; <<
105 * @cond = true; prep <<
106 * smp_mb(); // s <<
107 * if (swait_active(wq_head)) if ( <<
108 * wake_up(wq_head); <<
109 * sche <<
110 * } <<
111 * finish <<
112 * <<
113 * Because without the explicit smp_mb() it's <<
114 * swait_active() load to get hoisted over the <<
115 * observe an empty wait list while the waiter <<
116 * This, in turn, can trigger missing wakeups. <<
117 * <<
118 * Also note that this 'optimization' trades a <<
119 * which (when the lock is uncontended) are of <<
120 */ <<
121 static inline int swait_active(struct swait_qu <<
122 { <<
123 return !list_empty(&wq->task_list); <<
124 } <<
125 <<
126 /** <<
127 * swq_has_sleeper - check if there are any wa <<
128 * @wq: the waitqueue to test for waiters <<
129 * <<
130 * Returns true if @wq has waiting processes <<
131 * <<
132 * Please refer to the comment for swait_activ <<
133 */ <<
134 static inline bool swq_has_sleeper(struct swai <<
135 { 83 {
136 /* !! 84 return !list_empty(&q->task_list);
137 * We need to be sure we are in sync w <<
138 * modifications to the wait queue (ta <<
139 * <<
140 * This memory barrier should be paire <<
141 * waiting side. <<
142 */ <<
143 smp_mb(); <<
144 return swait_active(wq); <<
145 } 85 }
146 86
147 extern void swake_up_one(struct swait_queue_he !! 87 extern void swake_up(struct swait_queue_head *q);
148 extern void swake_up_all(struct swait_queue_he 88 extern void swake_up_all(struct swait_queue_head *q);
149 extern void swake_up_locked(struct swait_queue !! 89 extern void swake_up_locked(struct swait_queue_head *q);
150 90
151 extern void prepare_to_swait_exclusive(struct !! 91 extern void __prepare_to_swait(struct swait_queue_head *q, struct swait_queue *wait);
>> 92 extern void prepare_to_swait(struct swait_queue_head *q, struct swait_queue *wait, int state);
152 extern long prepare_to_swait_event(struct swai 93 extern long prepare_to_swait_event(struct swait_queue_head *q, struct swait_queue *wait, int state);
153 94
154 extern void __finish_swait(struct swait_queue_ 95 extern void __finish_swait(struct swait_queue_head *q, struct swait_queue *wait);
155 extern void finish_swait(struct swait_queue_he 96 extern void finish_swait(struct swait_queue_head *q, struct swait_queue *wait);
156 97
157 /* as per ___wait_event() but for swait, there !! 98 /* as per ___wait_event() but for swait, therefore "exclusive == 0" */
158 #define ___swait_event(wq, condition, state, r 99 #define ___swait_event(wq, condition, state, ret, cmd) \
159 ({ 100 ({ \
160 __label__ __out; <<
161 struct swait_queue __wait; 101 struct swait_queue __wait; \
162 long __ret = ret; 102 long __ret = ret; \
163 103 \
164 INIT_LIST_HEAD(&__wait.task_list); 104 INIT_LIST_HEAD(&__wait.task_list); \
165 for (;;) { 105 for (;;) { \
166 long __int = prepare_to_swait_ 106 long __int = prepare_to_swait_event(&wq, &__wait, state);\
167 107 \
168 if (condition) 108 if (condition) \
169 break; 109 break; \
170 110 \
171 if (___wait_is_interruptible(s 111 if (___wait_is_interruptible(state) && __int) { \
172 __ret = __int; 112 __ret = __int; \
173 goto __out; !! 113 break; \
174 } 114 } \
175 115 \
176 cmd; 116 cmd; \
177 } 117 } \
178 finish_swait(&wq, &__wait); 118 finish_swait(&wq, &__wait); \
179 __out: __ret; !! 119 __ret; \
180 }) 120 })
181 121
182 #define __swait_event(wq, condition) 122 #define __swait_event(wq, condition) \
183 (void)___swait_event(wq, condition, TA 123 (void)___swait_event(wq, condition, TASK_UNINTERRUPTIBLE, 0, \
184 schedule()) 124 schedule())
185 125
186 #define swait_event_exclusive(wq, condition) !! 126 #define swait_event(wq, condition) \
187 do { 127 do { \
188 if (condition) 128 if (condition) \
189 break; 129 break; \
190 __swait_event(wq, condition); 130 __swait_event(wq, condition); \
191 } while (0) 131 } while (0)
192 132
193 #define __swait_event_timeout(wq, condition, t 133 #define __swait_event_timeout(wq, condition, timeout) \
194 ___swait_event(wq, ___wait_cond_timeou 134 ___swait_event(wq, ___wait_cond_timeout(condition), \
195 TASK_UNINTERRUPTIBLE, ti 135 TASK_UNINTERRUPTIBLE, timeout, \
196 __ret = schedule_timeout 136 __ret = schedule_timeout(__ret))
197 137
198 #define swait_event_timeout_exclusive(wq, cond !! 138 #define swait_event_timeout(wq, condition, timeout) \
199 ({ 139 ({ \
200 long __ret = timeout; 140 long __ret = timeout; \
201 if (!___wait_cond_timeout(condition)) 141 if (!___wait_cond_timeout(condition)) \
202 __ret = __swait_event_timeout( 142 __ret = __swait_event_timeout(wq, condition, timeout); \
203 __ret; 143 __ret; \
204 }) 144 })
205 145
206 #define __swait_event_interruptible(wq, condit 146 #define __swait_event_interruptible(wq, condition) \
207 ___swait_event(wq, condition, TASK_INT 147 ___swait_event(wq, condition, TASK_INTERRUPTIBLE, 0, \
208 schedule()) 148 schedule())
209 149
210 #define swait_event_interruptible_exclusive(wq !! 150 #define swait_event_interruptible(wq, condition) \
211 ({ 151 ({ \
212 int __ret = 0; 152 int __ret = 0; \
213 if (!(condition)) 153 if (!(condition)) \
214 __ret = __swait_event_interrup 154 __ret = __swait_event_interruptible(wq, condition); \
215 __ret; 155 __ret; \
216 }) 156 })
217 157
218 #define __swait_event_interruptible_timeout(wq 158 #define __swait_event_interruptible_timeout(wq, condition, timeout) \
219 ___swait_event(wq, ___wait_cond_timeou 159 ___swait_event(wq, ___wait_cond_timeout(condition), \
220 TASK_INTERRUPTIBLE, time 160 TASK_INTERRUPTIBLE, timeout, \
221 __ret = schedule_timeout 161 __ret = schedule_timeout(__ret))
222 162
223 #define swait_event_interruptible_timeout_excl !! 163 #define swait_event_interruptible_timeout(wq, condition, timeout) \
224 ({ 164 ({ \
225 long __ret = timeout; 165 long __ret = timeout; \
226 if (!___wait_cond_timeout(condition)) 166 if (!___wait_cond_timeout(condition)) \
227 __ret = __swait_event_interrup 167 __ret = __swait_event_interruptible_timeout(wq, \
228 168 condition, timeout); \
229 __ret; <<
230 }) <<
231 <<
232 #define __swait_event_idle(wq, condition) <<
233 (void)___swait_event(wq, condition, TA <<
234 <<
235 /** <<
236 * swait_event_idle_exclusive - wait without s <<
237 * @wq: the waitqueue to wait on <<
238 * @condition: a C expression for the event to <<
239 * <<
240 * The process is put to sleep (TASK_IDLE) unt <<
241 * true. The @condition is checked each time t <<
242 * <<
243 * This function is mostly used when a kthread <<
244 * condition and doesn't want to contribute to <<
245 * ignored. <<
246 */ <<
247 #define swait_event_idle_exclusive(wq, conditi <<
248 do { <<
249 if (condition) <<
250 break; <<
251 __swait_event_idle(wq, condition); <<
252 } while (0) <<
253 <<
254 #define __swait_event_idle_timeout(wq, conditi <<
255 ___swait_event(wq, ___wait_cond_timeou <<
256 TASK_IDLE, timeout, <<
257 __ret = schedule_timeou <<
258 <<
259 /** <<
260 * swait_event_idle_timeout_exclusive - wait u <<
261 * @wq: the waitqueue to wait on <<
262 * @condition: a C expression for the event to <<
263 * @timeout: timeout at which we'll give up in <<
264 * <<
265 * The process is put to sleep (TASK_IDLE) unt <<
266 * true. The @condition is checked each time t <<
267 * <<
268 * This function is mostly used when a kthread <<
269 * condition and doesn't want to contribute to <<
270 * ignored. <<
271 * <<
272 * Returns: <<
273 * 0 if the @condition evaluated to %false aft <<
274 * 1 if the @condition evaluated to %true afte <<
275 * or the remaining jiffies (at least 1) if th <<
276 * to %true before the @timeout elapsed. <<
277 */ <<
278 #define swait_event_idle_timeout_exclusive(wq, <<
279 ({ <<
280 long __ret = timeout; <<
281 if (!___wait_cond_timeout(condition)) <<
282 __ret = __swait_event_idle_tim <<
283 <<
284 __ret; 169 __ret; \
285 }) 170 })
286 171
287 #endif /* _LINUX_SWAIT_H */ 172 #endif /* _LINUX_SWAIT_H */
288 173
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.