1 // SPDX-License-Identifier: GPL-2.0 <<
2 <<
3 /* 1 /*
4 * Generic wait-for-completion handler; 2 * Generic wait-for-completion handler;
5 * 3 *
6 * It differs from semaphores in that their de 4 * It differs from semaphores in that their default case is the opposite,
7 * wait_for_completion default blocks whereas 5 * wait_for_completion default blocks whereas semaphore default non-block. The
8 * interface also makes it easy to 'complete' 6 * interface also makes it easy to 'complete' multiple waiting threads,
9 * something which isn't entirely natural for 7 * something which isn't entirely natural for semaphores.
10 * 8 *
11 * But more importantly, the primitive documen 9 * But more importantly, the primitive documents the usage. Semaphores would
12 * typically be used for exclusion which gives 10 * typically be used for exclusion which gives rise to priority inversion.
13 * Waiting for completion is a typically sync 11 * Waiting for completion is a typically sync point, but not an exclusion point.
14 */ 12 */
15 13
16 static void complete_with_flags(struct complet !! 14 #include <linux/sched/signal.h>
17 { !! 15 #include <linux/sched/debug.h>
18 unsigned long flags; !! 16 #include <linux/completion.h>
19 <<
20 raw_spin_lock_irqsave(&x->wait.lock, f <<
21 <<
22 if (x->done != UINT_MAX) <<
23 x->done++; <<
24 swake_up_locked(&x->wait, wake_flags); <<
25 raw_spin_unlock_irqrestore(&x->wait.lo <<
26 } <<
27 <<
28 void complete_on_current_cpu(struct completion <<
29 { <<
30 return complete_with_flags(x, WF_CURRE <<
31 } <<
32 17
33 /** 18 /**
34 * complete: - signals a single thread waiting 19 * complete: - signals a single thread waiting on this completion
35 * @x: holds the state of this particular com 20 * @x: holds the state of this particular completion
36 * 21 *
37 * This will wake up a single thread waiting o 22 * This will wake up a single thread waiting on this completion. Threads will be
38 * awakened in the same order in which they we 23 * awakened in the same order in which they were queued.
39 * 24 *
40 * See also complete_all(), wait_for_completio 25 * See also complete_all(), wait_for_completion() and related routines.
41 * 26 *
42 * If this function wakes up a task, it execut !! 27 * It may be assumed that this function implies a write memory barrier before
43 * accessing the task state. !! 28 * changing the task state if and only if any tasks are woken up.
44 */ 29 */
45 void complete(struct completion *x) 30 void complete(struct completion *x)
46 { 31 {
47 complete_with_flags(x, 0); !! 32 unsigned long flags;
>> 33
>> 34 spin_lock_irqsave(&x->wait.lock, flags);
>> 35 if (x->done != UINT_MAX)
>> 36 x->done++;
>> 37 __wake_up_locked(&x->wait, TASK_NORMAL, 1);
>> 38 spin_unlock_irqrestore(&x->wait.lock, flags);
48 } 39 }
49 EXPORT_SYMBOL(complete); 40 EXPORT_SYMBOL(complete);
50 41
51 /** 42 /**
52 * complete_all: - signals all threads waiting 43 * complete_all: - signals all threads waiting on this completion
53 * @x: holds the state of this particular com 44 * @x: holds the state of this particular completion
54 * 45 *
55 * This will wake up all threads waiting on th 46 * This will wake up all threads waiting on this particular completion event.
56 * 47 *
57 * If this function wakes up a task, it execut !! 48 * It may be assumed that this function implies a write memory barrier before
58 * accessing the task state. !! 49 * changing the task state if and only if any tasks are woken up.
59 * <<
60 * Since complete_all() sets the completion of <<
61 * to allow multiple waiters to finish, a call <<
62 * must be used on @x if @x is to be used agai <<
63 * sure that all waiters have woken and finish <<
64 * @x. Also note that the function completion_ <<
65 * to know if there are still waiters after co <<
66 */ 50 */
67 void complete_all(struct completion *x) 51 void complete_all(struct completion *x)
68 { 52 {
69 unsigned long flags; 53 unsigned long flags;
70 54
71 lockdep_assert_RT_in_threaded_ctx(); !! 55 spin_lock_irqsave(&x->wait.lock, flags);
72 <<
73 raw_spin_lock_irqsave(&x->wait.lock, f <<
74 x->done = UINT_MAX; 56 x->done = UINT_MAX;
75 swake_up_all_locked(&x->wait); !! 57 __wake_up_locked(&x->wait, TASK_NORMAL, 0);
76 raw_spin_unlock_irqrestore(&x->wait.lo !! 58 spin_unlock_irqrestore(&x->wait.lock, flags);
77 } 59 }
78 EXPORT_SYMBOL(complete_all); 60 EXPORT_SYMBOL(complete_all);
79 61
80 static inline long __sched 62 static inline long __sched
81 do_wait_for_common(struct completion *x, 63 do_wait_for_common(struct completion *x,
82 long (*action)(long), long 64 long (*action)(long), long timeout, int state)
83 { 65 {
84 if (!x->done) { 66 if (!x->done) {
85 DECLARE_SWAITQUEUE(wait); !! 67 DECLARE_WAITQUEUE(wait, current);
86 68
>> 69 __add_wait_queue_tail_exclusive(&x->wait, &wait);
87 do { 70 do {
88 if (signal_pending_sta 71 if (signal_pending_state(state, current)) {
89 timeout = -ERE 72 timeout = -ERESTARTSYS;
90 break; 73 break;
91 } 74 }
92 __prepare_to_swait(&x- <<
93 __set_current_state(st 75 __set_current_state(state);
94 raw_spin_unlock_irq(&x !! 76 spin_unlock_irq(&x->wait.lock);
95 timeout = action(timeo 77 timeout = action(timeout);
96 raw_spin_lock_irq(&x-> !! 78 spin_lock_irq(&x->wait.lock);
97 } while (!x->done && timeout); 79 } while (!x->done && timeout);
98 __finish_swait(&x->wait, &wait !! 80 __remove_wait_queue(&x->wait, &wait);
99 if (!x->done) 81 if (!x->done)
100 return timeout; 82 return timeout;
101 } 83 }
102 if (x->done != UINT_MAX) 84 if (x->done != UINT_MAX)
103 x->done--; 85 x->done--;
104 return timeout ?: 1; 86 return timeout ?: 1;
105 } 87 }
106 88
107 static inline long __sched 89 static inline long __sched
108 __wait_for_common(struct completion *x, 90 __wait_for_common(struct completion *x,
109 long (*action)(long), long t 91 long (*action)(long), long timeout, int state)
110 { 92 {
111 might_sleep(); 93 might_sleep();
112 94
113 complete_acquire(x); !! 95 spin_lock_irq(&x->wait.lock);
114 <<
115 raw_spin_lock_irq(&x->wait.lock); <<
116 timeout = do_wait_for_common(x, action 96 timeout = do_wait_for_common(x, action, timeout, state);
117 raw_spin_unlock_irq(&x->wait.lock); !! 97 spin_unlock_irq(&x->wait.lock);
118 <<
119 complete_release(x); <<
120 <<
121 return timeout; 98 return timeout;
122 } 99 }
123 100
124 static long __sched 101 static long __sched
125 wait_for_common(struct completion *x, long tim 102 wait_for_common(struct completion *x, long timeout, int state)
126 { 103 {
127 return __wait_for_common(x, schedule_t 104 return __wait_for_common(x, schedule_timeout, timeout, state);
128 } 105 }
129 106
130 static long __sched 107 static long __sched
131 wait_for_common_io(struct completion *x, long 108 wait_for_common_io(struct completion *x, long timeout, int state)
132 { 109 {
133 return __wait_for_common(x, io_schedul 110 return __wait_for_common(x, io_schedule_timeout, timeout, state);
134 } 111 }
135 112
136 /** 113 /**
137 * wait_for_completion: - waits for completion 114 * wait_for_completion: - waits for completion of a task
138 * @x: holds the state of this particular com 115 * @x: holds the state of this particular completion
139 * 116 *
140 * This waits to be signaled for completion of 117 * This waits to be signaled for completion of a specific task. It is NOT
141 * interruptible and there is no timeout. 118 * interruptible and there is no timeout.
142 * 119 *
143 * See also similar routines (i.e. wait_for_co 120 * See also similar routines (i.e. wait_for_completion_timeout()) with timeout
144 * and interrupt capability. Also see complete 121 * and interrupt capability. Also see complete().
145 */ 122 */
146 void __sched wait_for_completion(struct comple 123 void __sched wait_for_completion(struct completion *x)
147 { 124 {
148 wait_for_common(x, MAX_SCHEDULE_TIMEOU 125 wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
149 } 126 }
150 EXPORT_SYMBOL(wait_for_completion); 127 EXPORT_SYMBOL(wait_for_completion);
151 128
152 /** 129 /**
153 * wait_for_completion_timeout: - waits for co 130 * wait_for_completion_timeout: - waits for completion of a task (w/timeout)
154 * @x: holds the state of this particular com 131 * @x: holds the state of this particular completion
155 * @timeout: timeout value in jiffies 132 * @timeout: timeout value in jiffies
156 * 133 *
157 * This waits for either a completion of a spe 134 * This waits for either a completion of a specific task to be signaled or for a
158 * specified timeout to expire. The timeout is 135 * specified timeout to expire. The timeout is in jiffies. It is not
159 * interruptible. 136 * interruptible.
160 * 137 *
161 * Return: 0 if timed out, and positive (at le 138 * Return: 0 if timed out, and positive (at least 1, or number of jiffies left
162 * till timeout) if completed. 139 * till timeout) if completed.
163 */ 140 */
164 unsigned long __sched 141 unsigned long __sched
165 wait_for_completion_timeout(struct completion 142 wait_for_completion_timeout(struct completion *x, unsigned long timeout)
166 { 143 {
167 return wait_for_common(x, timeout, TAS 144 return wait_for_common(x, timeout, TASK_UNINTERRUPTIBLE);
168 } 145 }
169 EXPORT_SYMBOL(wait_for_completion_timeout); 146 EXPORT_SYMBOL(wait_for_completion_timeout);
170 147
171 /** 148 /**
172 * wait_for_completion_io: - waits for complet 149 * wait_for_completion_io: - waits for completion of a task
173 * @x: holds the state of this particular com 150 * @x: holds the state of this particular completion
174 * 151 *
175 * This waits to be signaled for completion of 152 * This waits to be signaled for completion of a specific task. It is NOT
176 * interruptible and there is no timeout. The 153 * interruptible and there is no timeout. The caller is accounted as waiting
177 * for IO (which traditionally means blkio onl 154 * for IO (which traditionally means blkio only).
178 */ 155 */
179 void __sched wait_for_completion_io(struct com 156 void __sched wait_for_completion_io(struct completion *x)
180 { 157 {
181 wait_for_common_io(x, MAX_SCHEDULE_TIM 158 wait_for_common_io(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
182 } 159 }
183 EXPORT_SYMBOL(wait_for_completion_io); 160 EXPORT_SYMBOL(wait_for_completion_io);
184 161
185 /** 162 /**
186 * wait_for_completion_io_timeout: - waits for 163 * wait_for_completion_io_timeout: - waits for completion of a task (w/timeout)
187 * @x: holds the state of this particular com 164 * @x: holds the state of this particular completion
188 * @timeout: timeout value in jiffies 165 * @timeout: timeout value in jiffies
189 * 166 *
190 * This waits for either a completion of a spe 167 * This waits for either a completion of a specific task to be signaled or for a
191 * specified timeout to expire. The timeout is 168 * specified timeout to expire. The timeout is in jiffies. It is not
192 * interruptible. The caller is accounted as w 169 * interruptible. The caller is accounted as waiting for IO (which traditionally
193 * means blkio only). 170 * means blkio only).
194 * 171 *
195 * Return: 0 if timed out, and positive (at le 172 * Return: 0 if timed out, and positive (at least 1, or number of jiffies left
196 * till timeout) if completed. 173 * till timeout) if completed.
197 */ 174 */
198 unsigned long __sched 175 unsigned long __sched
199 wait_for_completion_io_timeout(struct completi 176 wait_for_completion_io_timeout(struct completion *x, unsigned long timeout)
200 { 177 {
201 return wait_for_common_io(x, timeout, 178 return wait_for_common_io(x, timeout, TASK_UNINTERRUPTIBLE);
202 } 179 }
203 EXPORT_SYMBOL(wait_for_completion_io_timeout); 180 EXPORT_SYMBOL(wait_for_completion_io_timeout);
204 181
205 /** 182 /**
206 * wait_for_completion_interruptible: - waits 183 * wait_for_completion_interruptible: - waits for completion of a task (w/intr)
207 * @x: holds the state of this particular com 184 * @x: holds the state of this particular completion
208 * 185 *
209 * This waits for completion of a specific tas 186 * This waits for completion of a specific task to be signaled. It is
210 * interruptible. 187 * interruptible.
211 * 188 *
212 * Return: -ERESTARTSYS if interrupted, 0 if c 189 * Return: -ERESTARTSYS if interrupted, 0 if completed.
213 */ 190 */
214 int __sched wait_for_completion_interruptible( 191 int __sched wait_for_completion_interruptible(struct completion *x)
215 { 192 {
216 long t = wait_for_common(x, MAX_SCHEDU 193 long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE);
217 <<
218 if (t == -ERESTARTSYS) 194 if (t == -ERESTARTSYS)
219 return t; 195 return t;
220 return 0; 196 return 0;
221 } 197 }
222 EXPORT_SYMBOL(wait_for_completion_interruptibl 198 EXPORT_SYMBOL(wait_for_completion_interruptible);
223 199
224 /** 200 /**
225 * wait_for_completion_interruptible_timeout: 201 * wait_for_completion_interruptible_timeout: - waits for completion (w/(to,intr))
226 * @x: holds the state of this particular com 202 * @x: holds the state of this particular completion
227 * @timeout: timeout value in jiffies 203 * @timeout: timeout value in jiffies
228 * 204 *
229 * This waits for either a completion of a spe 205 * This waits for either a completion of a specific task to be signaled or for a
230 * specified timeout to expire. It is interrup 206 * specified timeout to expire. It is interruptible. The timeout is in jiffies.
231 * 207 *
232 * Return: -ERESTARTSYS if interrupted, 0 if t 208 * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1,
233 * or number of jiffies left till timeout) if 209 * or number of jiffies left till timeout) if completed.
234 */ 210 */
235 long __sched 211 long __sched
236 wait_for_completion_interruptible_timeout(stru 212 wait_for_completion_interruptible_timeout(struct completion *x,
237 unsi 213 unsigned long timeout)
238 { 214 {
239 return wait_for_common(x, timeout, TAS 215 return wait_for_common(x, timeout, TASK_INTERRUPTIBLE);
240 } 216 }
241 EXPORT_SYMBOL(wait_for_completion_interruptibl 217 EXPORT_SYMBOL(wait_for_completion_interruptible_timeout);
242 218
243 /** 219 /**
244 * wait_for_completion_killable: - waits for c 220 * wait_for_completion_killable: - waits for completion of a task (killable)
245 * @x: holds the state of this particular com 221 * @x: holds the state of this particular completion
246 * 222 *
247 * This waits to be signaled for completion of 223 * This waits to be signaled for completion of a specific task. It can be
248 * interrupted by a kill signal. 224 * interrupted by a kill signal.
249 * 225 *
250 * Return: -ERESTARTSYS if interrupted, 0 if c 226 * Return: -ERESTARTSYS if interrupted, 0 if completed.
251 */ 227 */
252 int __sched wait_for_completion_killable(struc 228 int __sched wait_for_completion_killable(struct completion *x)
253 { 229 {
254 long t = wait_for_common(x, MAX_SCHEDU 230 long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_KILLABLE);
255 <<
256 if (t == -ERESTARTSYS) 231 if (t == -ERESTARTSYS)
257 return t; 232 return t;
258 return 0; 233 return 0;
259 } 234 }
260 EXPORT_SYMBOL(wait_for_completion_killable); 235 EXPORT_SYMBOL(wait_for_completion_killable);
261 236
262 int __sched wait_for_completion_state(struct c <<
263 { <<
264 long t = wait_for_common(x, MAX_SCHEDU <<
265 <<
266 if (t == -ERESTARTSYS) <<
267 return t; <<
268 return 0; <<
269 } <<
270 EXPORT_SYMBOL(wait_for_completion_state); <<
271 <<
272 /** 237 /**
273 * wait_for_completion_killable_timeout: - wai 238 * wait_for_completion_killable_timeout: - waits for completion of a task (w/(to,killable))
274 * @x: holds the state of this particular com 239 * @x: holds the state of this particular completion
275 * @timeout: timeout value in jiffies 240 * @timeout: timeout value in jiffies
276 * 241 *
277 * This waits for either a completion of a spe 242 * This waits for either a completion of a specific task to be
278 * signaled or for a specified timeout to expi 243 * signaled or for a specified timeout to expire. It can be
279 * interrupted by a kill signal. The timeout i 244 * interrupted by a kill signal. The timeout is in jiffies.
280 * 245 *
281 * Return: -ERESTARTSYS if interrupted, 0 if t 246 * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1,
282 * or number of jiffies left till timeout) if 247 * or number of jiffies left till timeout) if completed.
283 */ 248 */
284 long __sched 249 long __sched
285 wait_for_completion_killable_timeout(struct co 250 wait_for_completion_killable_timeout(struct completion *x,
286 unsigned 251 unsigned long timeout)
287 { 252 {
288 return wait_for_common(x, timeout, TAS 253 return wait_for_common(x, timeout, TASK_KILLABLE);
289 } 254 }
290 EXPORT_SYMBOL(wait_for_completion_killable_tim 255 EXPORT_SYMBOL(wait_for_completion_killable_timeout);
291 256
292 /** 257 /**
293 * try_wait_for_completion - try to decre 258 * try_wait_for_completion - try to decrement a completion without blocking
294 * @x: completion structure 259 * @x: completion structure
295 * 260 *
296 * Return: 0 if a decrement cannot be don 261 * Return: 0 if a decrement cannot be done without blocking
297 * 1 if a decrement succeeded. 262 * 1 if a decrement succeeded.
298 * 263 *
299 * If a completion is being used as a cou 264 * If a completion is being used as a counting completion,
300 * attempt to decrement the counter witho 265 * attempt to decrement the counter without blocking. This
301 * enables us to avoid waiting if the res 266 * enables us to avoid waiting if the resource the completion
302 * is protecting is not available. 267 * is protecting is not available.
303 */ 268 */
304 bool try_wait_for_completion(struct completion 269 bool try_wait_for_completion(struct completion *x)
305 { 270 {
306 unsigned long flags; 271 unsigned long flags;
307 bool ret = true; !! 272 int ret = 1;
308 273
309 /* 274 /*
310 * Since x->done will need to be locke 275 * Since x->done will need to be locked only
311 * in the non-blocking case, we check 276 * in the non-blocking case, we check x->done
312 * first without taking the lock so we 277 * first without taking the lock so we can
313 * return early in the blocking case. 278 * return early in the blocking case.
314 */ 279 */
315 if (!READ_ONCE(x->done)) 280 if (!READ_ONCE(x->done))
316 return false; !! 281 return 0;
317 282
318 raw_spin_lock_irqsave(&x->wait.lock, f !! 283 spin_lock_irqsave(&x->wait.lock, flags);
319 if (!x->done) 284 if (!x->done)
320 ret = false; !! 285 ret = 0;
321 else if (x->done != UINT_MAX) 286 else if (x->done != UINT_MAX)
322 x->done--; 287 x->done--;
323 raw_spin_unlock_irqrestore(&x->wait.lo !! 288 spin_unlock_irqrestore(&x->wait.lock, flags);
324 return ret; 289 return ret;
325 } 290 }
326 EXPORT_SYMBOL(try_wait_for_completion); 291 EXPORT_SYMBOL(try_wait_for_completion);
327 292
328 /** 293 /**
329 * completion_done - Test to see if a com 294 * completion_done - Test to see if a completion has any waiters
330 * @x: completion structure 295 * @x: completion structure
331 * 296 *
332 * Return: 0 if there are waiters (wait_f 297 * Return: 0 if there are waiters (wait_for_completion() in progress)
333 * 1 if there are no waiters. 298 * 1 if there are no waiters.
334 * 299 *
335 * Note, this will always return true if <<
336 */ 300 */
337 bool completion_done(struct completion *x) 301 bool completion_done(struct completion *x)
338 { 302 {
339 unsigned long flags; <<
340 <<
341 if (!READ_ONCE(x->done)) 303 if (!READ_ONCE(x->done))
342 return false; 304 return false;
343 305
344 /* 306 /*
345 * If ->done, we need to wait for comp 307 * If ->done, we need to wait for complete() to release ->wait.lock
346 * otherwise we can end up freeing the 308 * otherwise we can end up freeing the completion before complete()
347 * is done referencing it. 309 * is done referencing it.
>> 310 *
>> 311 * The RMB pairs with complete()'s RELEASE of ->wait.lock and orders
>> 312 * the loads of ->done and ->wait.lock such that we cannot observe
>> 313 * the lock before complete() acquires it while observing the ->done
>> 314 * after it's acquired the lock.
348 */ 315 */
349 raw_spin_lock_irqsave(&x->wait.lock, f !! 316 smp_rmb();
350 raw_spin_unlock_irqrestore(&x->wait.lo !! 317 spin_unlock_wait(&x->wait.lock);
351 return true; 318 return true;
352 } 319 }
353 EXPORT_SYMBOL(completion_done); 320 EXPORT_SYMBOL(completion_done);
354 321
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.