1 // SPDX-License-Identifier: GPL-2.0 1 // SPDX-License-Identifier: GPL-2.0
2 2
3 /* 3 /*
4 * Generic wait-for-completion handler; 4 * Generic wait-for-completion handler;
5 * 5 *
6 * It differs from semaphores in that their de 6 * It differs from semaphores in that their default case is the opposite,
7 * wait_for_completion default blocks whereas 7 * wait_for_completion default blocks whereas semaphore default non-block. The
8 * interface also makes it easy to 'complete' 8 * interface also makes it easy to 'complete' multiple waiting threads,
9 * something which isn't entirely natural for 9 * something which isn't entirely natural for semaphores.
10 * 10 *
11 * But more importantly, the primitive documen 11 * But more importantly, the primitive documents the usage. Semaphores would
12 * typically be used for exclusion which gives 12 * typically be used for exclusion which gives rise to priority inversion.
13 * Waiting for completion is a typically sync 13 * Waiting for completion is a typically sync point, but not an exclusion point.
14 */ 14 */
15 15
16 static void complete_with_flags(struct complet 16 static void complete_with_flags(struct completion *x, int wake_flags)
17 { 17 {
18 unsigned long flags; 18 unsigned long flags;
19 19
20 raw_spin_lock_irqsave(&x->wait.lock, f 20 raw_spin_lock_irqsave(&x->wait.lock, flags);
21 21
22 if (x->done != UINT_MAX) 22 if (x->done != UINT_MAX)
23 x->done++; 23 x->done++;
24 swake_up_locked(&x->wait, wake_flags); 24 swake_up_locked(&x->wait, wake_flags);
25 raw_spin_unlock_irqrestore(&x->wait.lo 25 raw_spin_unlock_irqrestore(&x->wait.lock, flags);
26 } 26 }
27 27
28 void complete_on_current_cpu(struct completion 28 void complete_on_current_cpu(struct completion *x)
29 { 29 {
30 return complete_with_flags(x, WF_CURRE 30 return complete_with_flags(x, WF_CURRENT_CPU);
31 } 31 }
32 32
33 /** 33 /**
34 * complete: - signals a single thread waiting 34 * complete: - signals a single thread waiting on this completion
35 * @x: holds the state of this particular com 35 * @x: holds the state of this particular completion
36 * 36 *
37 * This will wake up a single thread waiting o 37 * This will wake up a single thread waiting on this completion. Threads will be
38 * awakened in the same order in which they we 38 * awakened in the same order in which they were queued.
39 * 39 *
40 * See also complete_all(), wait_for_completio 40 * See also complete_all(), wait_for_completion() and related routines.
41 * 41 *
42 * If this function wakes up a task, it execut 42 * If this function wakes up a task, it executes a full memory barrier before
43 * accessing the task state. 43 * accessing the task state.
44 */ 44 */
45 void complete(struct completion *x) 45 void complete(struct completion *x)
46 { 46 {
47 complete_with_flags(x, 0); 47 complete_with_flags(x, 0);
48 } 48 }
49 EXPORT_SYMBOL(complete); 49 EXPORT_SYMBOL(complete);
50 50
51 /** 51 /**
52 * complete_all: - signals all threads waiting 52 * complete_all: - signals all threads waiting on this completion
53 * @x: holds the state of this particular com 53 * @x: holds the state of this particular completion
54 * 54 *
55 * This will wake up all threads waiting on th 55 * This will wake up all threads waiting on this particular completion event.
56 * 56 *
57 * If this function wakes up a task, it execut 57 * If this function wakes up a task, it executes a full memory barrier before
58 * accessing the task state. 58 * accessing the task state.
59 * 59 *
60 * Since complete_all() sets the completion of 60 * Since complete_all() sets the completion of @x permanently to done
61 * to allow multiple waiters to finish, a call 61 * to allow multiple waiters to finish, a call to reinit_completion()
62 * must be used on @x if @x is to be used agai 62 * must be used on @x if @x is to be used again. The code must make
63 * sure that all waiters have woken and finish 63 * sure that all waiters have woken and finished before reinitializing
64 * @x. Also note that the function completion_ 64 * @x. Also note that the function completion_done() can not be used
65 * to know if there are still waiters after co 65 * to know if there are still waiters after complete_all() has been called.
66 */ 66 */
67 void complete_all(struct completion *x) 67 void complete_all(struct completion *x)
68 { 68 {
69 unsigned long flags; 69 unsigned long flags;
70 70
71 lockdep_assert_RT_in_threaded_ctx(); 71 lockdep_assert_RT_in_threaded_ctx();
72 72
73 raw_spin_lock_irqsave(&x->wait.lock, f 73 raw_spin_lock_irqsave(&x->wait.lock, flags);
74 x->done = UINT_MAX; 74 x->done = UINT_MAX;
75 swake_up_all_locked(&x->wait); 75 swake_up_all_locked(&x->wait);
76 raw_spin_unlock_irqrestore(&x->wait.lo 76 raw_spin_unlock_irqrestore(&x->wait.lock, flags);
77 } 77 }
78 EXPORT_SYMBOL(complete_all); 78 EXPORT_SYMBOL(complete_all);
79 79
80 static inline long __sched 80 static inline long __sched
81 do_wait_for_common(struct completion *x, 81 do_wait_for_common(struct completion *x,
82 long (*action)(long), long 82 long (*action)(long), long timeout, int state)
83 { 83 {
84 if (!x->done) { 84 if (!x->done) {
85 DECLARE_SWAITQUEUE(wait); 85 DECLARE_SWAITQUEUE(wait);
86 86
87 do { 87 do {
88 if (signal_pending_sta 88 if (signal_pending_state(state, current)) {
89 timeout = -ERE 89 timeout = -ERESTARTSYS;
90 break; 90 break;
91 } 91 }
92 __prepare_to_swait(&x- 92 __prepare_to_swait(&x->wait, &wait);
93 __set_current_state(st 93 __set_current_state(state);
94 raw_spin_unlock_irq(&x 94 raw_spin_unlock_irq(&x->wait.lock);
95 timeout = action(timeo 95 timeout = action(timeout);
96 raw_spin_lock_irq(&x-> 96 raw_spin_lock_irq(&x->wait.lock);
97 } while (!x->done && timeout); 97 } while (!x->done && timeout);
98 __finish_swait(&x->wait, &wait 98 __finish_swait(&x->wait, &wait);
99 if (!x->done) 99 if (!x->done)
100 return timeout; 100 return timeout;
101 } 101 }
102 if (x->done != UINT_MAX) 102 if (x->done != UINT_MAX)
103 x->done--; 103 x->done--;
104 return timeout ?: 1; 104 return timeout ?: 1;
105 } 105 }
106 106
107 static inline long __sched 107 static inline long __sched
108 __wait_for_common(struct completion *x, 108 __wait_for_common(struct completion *x,
109 long (*action)(long), long t 109 long (*action)(long), long timeout, int state)
110 { 110 {
111 might_sleep(); 111 might_sleep();
112 112
113 complete_acquire(x); 113 complete_acquire(x);
114 114
115 raw_spin_lock_irq(&x->wait.lock); 115 raw_spin_lock_irq(&x->wait.lock);
116 timeout = do_wait_for_common(x, action 116 timeout = do_wait_for_common(x, action, timeout, state);
117 raw_spin_unlock_irq(&x->wait.lock); 117 raw_spin_unlock_irq(&x->wait.lock);
118 118
119 complete_release(x); 119 complete_release(x);
120 120
121 return timeout; 121 return timeout;
122 } 122 }
123 123
124 static long __sched 124 static long __sched
125 wait_for_common(struct completion *x, long tim 125 wait_for_common(struct completion *x, long timeout, int state)
126 { 126 {
127 return __wait_for_common(x, schedule_t 127 return __wait_for_common(x, schedule_timeout, timeout, state);
128 } 128 }
129 129
130 static long __sched 130 static long __sched
131 wait_for_common_io(struct completion *x, long 131 wait_for_common_io(struct completion *x, long timeout, int state)
132 { 132 {
133 return __wait_for_common(x, io_schedul 133 return __wait_for_common(x, io_schedule_timeout, timeout, state);
134 } 134 }
135 135
136 /** 136 /**
137 * wait_for_completion: - waits for completion 137 * wait_for_completion: - waits for completion of a task
138 * @x: holds the state of this particular com 138 * @x: holds the state of this particular completion
139 * 139 *
140 * This waits to be signaled for completion of 140 * This waits to be signaled for completion of a specific task. It is NOT
141 * interruptible and there is no timeout. 141 * interruptible and there is no timeout.
142 * 142 *
143 * See also similar routines (i.e. wait_for_co 143 * See also similar routines (i.e. wait_for_completion_timeout()) with timeout
144 * and interrupt capability. Also see complete 144 * and interrupt capability. Also see complete().
145 */ 145 */
146 void __sched wait_for_completion(struct comple 146 void __sched wait_for_completion(struct completion *x)
147 { 147 {
148 wait_for_common(x, MAX_SCHEDULE_TIMEOU 148 wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
149 } 149 }
150 EXPORT_SYMBOL(wait_for_completion); 150 EXPORT_SYMBOL(wait_for_completion);
151 151
152 /** 152 /**
153 * wait_for_completion_timeout: - waits for co 153 * wait_for_completion_timeout: - waits for completion of a task (w/timeout)
154 * @x: holds the state of this particular com 154 * @x: holds the state of this particular completion
155 * @timeout: timeout value in jiffies 155 * @timeout: timeout value in jiffies
156 * 156 *
157 * This waits for either a completion of a spe 157 * This waits for either a completion of a specific task to be signaled or for a
158 * specified timeout to expire. The timeout is 158 * specified timeout to expire. The timeout is in jiffies. It is not
159 * interruptible. 159 * interruptible.
160 * 160 *
161 * Return: 0 if timed out, and positive (at le 161 * Return: 0 if timed out, and positive (at least 1, or number of jiffies left
162 * till timeout) if completed. 162 * till timeout) if completed.
163 */ 163 */
164 unsigned long __sched 164 unsigned long __sched
165 wait_for_completion_timeout(struct completion 165 wait_for_completion_timeout(struct completion *x, unsigned long timeout)
166 { 166 {
167 return wait_for_common(x, timeout, TAS 167 return wait_for_common(x, timeout, TASK_UNINTERRUPTIBLE);
168 } 168 }
169 EXPORT_SYMBOL(wait_for_completion_timeout); 169 EXPORT_SYMBOL(wait_for_completion_timeout);
170 170
171 /** 171 /**
172 * wait_for_completion_io: - waits for complet 172 * wait_for_completion_io: - waits for completion of a task
173 * @x: holds the state of this particular com 173 * @x: holds the state of this particular completion
174 * 174 *
175 * This waits to be signaled for completion of 175 * This waits to be signaled for completion of a specific task. It is NOT
176 * interruptible and there is no timeout. The 176 * interruptible and there is no timeout. The caller is accounted as waiting
177 * for IO (which traditionally means blkio onl 177 * for IO (which traditionally means blkio only).
178 */ 178 */
179 void __sched wait_for_completion_io(struct com 179 void __sched wait_for_completion_io(struct completion *x)
180 { 180 {
181 wait_for_common_io(x, MAX_SCHEDULE_TIM 181 wait_for_common_io(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
182 } 182 }
183 EXPORT_SYMBOL(wait_for_completion_io); 183 EXPORT_SYMBOL(wait_for_completion_io);
184 184
185 /** 185 /**
186 * wait_for_completion_io_timeout: - waits for 186 * wait_for_completion_io_timeout: - waits for completion of a task (w/timeout)
187 * @x: holds the state of this particular com 187 * @x: holds the state of this particular completion
188 * @timeout: timeout value in jiffies 188 * @timeout: timeout value in jiffies
189 * 189 *
190 * This waits for either a completion of a spe 190 * This waits for either a completion of a specific task to be signaled or for a
191 * specified timeout to expire. The timeout is 191 * specified timeout to expire. The timeout is in jiffies. It is not
192 * interruptible. The caller is accounted as w 192 * interruptible. The caller is accounted as waiting for IO (which traditionally
193 * means blkio only). 193 * means blkio only).
194 * 194 *
195 * Return: 0 if timed out, and positive (at le 195 * Return: 0 if timed out, and positive (at least 1, or number of jiffies left
196 * till timeout) if completed. 196 * till timeout) if completed.
197 */ 197 */
198 unsigned long __sched 198 unsigned long __sched
199 wait_for_completion_io_timeout(struct completi 199 wait_for_completion_io_timeout(struct completion *x, unsigned long timeout)
200 { 200 {
201 return wait_for_common_io(x, timeout, 201 return wait_for_common_io(x, timeout, TASK_UNINTERRUPTIBLE);
202 } 202 }
203 EXPORT_SYMBOL(wait_for_completion_io_timeout); 203 EXPORT_SYMBOL(wait_for_completion_io_timeout);
204 204
205 /** 205 /**
206 * wait_for_completion_interruptible: - waits 206 * wait_for_completion_interruptible: - waits for completion of a task (w/intr)
207 * @x: holds the state of this particular com 207 * @x: holds the state of this particular completion
208 * 208 *
209 * This waits for completion of a specific tas 209 * This waits for completion of a specific task to be signaled. It is
210 * interruptible. 210 * interruptible.
211 * 211 *
212 * Return: -ERESTARTSYS if interrupted, 0 if c 212 * Return: -ERESTARTSYS if interrupted, 0 if completed.
213 */ 213 */
214 int __sched wait_for_completion_interruptible( 214 int __sched wait_for_completion_interruptible(struct completion *x)
215 { 215 {
216 long t = wait_for_common(x, MAX_SCHEDU 216 long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE);
217 217
218 if (t == -ERESTARTSYS) 218 if (t == -ERESTARTSYS)
219 return t; 219 return t;
220 return 0; 220 return 0;
221 } 221 }
222 EXPORT_SYMBOL(wait_for_completion_interruptibl 222 EXPORT_SYMBOL(wait_for_completion_interruptible);
223 223
224 /** 224 /**
225 * wait_for_completion_interruptible_timeout: 225 * wait_for_completion_interruptible_timeout: - waits for completion (w/(to,intr))
226 * @x: holds the state of this particular com 226 * @x: holds the state of this particular completion
227 * @timeout: timeout value in jiffies 227 * @timeout: timeout value in jiffies
228 * 228 *
229 * This waits for either a completion of a spe 229 * This waits for either a completion of a specific task to be signaled or for a
230 * specified timeout to expire. It is interrup 230 * specified timeout to expire. It is interruptible. The timeout is in jiffies.
231 * 231 *
232 * Return: -ERESTARTSYS if interrupted, 0 if t 232 * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1,
233 * or number of jiffies left till timeout) if 233 * or number of jiffies left till timeout) if completed.
234 */ 234 */
235 long __sched 235 long __sched
236 wait_for_completion_interruptible_timeout(stru 236 wait_for_completion_interruptible_timeout(struct completion *x,
237 unsi 237 unsigned long timeout)
238 { 238 {
239 return wait_for_common(x, timeout, TAS 239 return wait_for_common(x, timeout, TASK_INTERRUPTIBLE);
240 } 240 }
241 EXPORT_SYMBOL(wait_for_completion_interruptibl 241 EXPORT_SYMBOL(wait_for_completion_interruptible_timeout);
242 242
243 /** 243 /**
244 * wait_for_completion_killable: - waits for c 244 * wait_for_completion_killable: - waits for completion of a task (killable)
245 * @x: holds the state of this particular com 245 * @x: holds the state of this particular completion
246 * 246 *
247 * This waits to be signaled for completion of 247 * This waits to be signaled for completion of a specific task. It can be
248 * interrupted by a kill signal. 248 * interrupted by a kill signal.
249 * 249 *
250 * Return: -ERESTARTSYS if interrupted, 0 if c 250 * Return: -ERESTARTSYS if interrupted, 0 if completed.
251 */ 251 */
252 int __sched wait_for_completion_killable(struc 252 int __sched wait_for_completion_killable(struct completion *x)
253 { 253 {
254 long t = wait_for_common(x, MAX_SCHEDU 254 long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_KILLABLE);
255 255
256 if (t == -ERESTARTSYS) 256 if (t == -ERESTARTSYS)
257 return t; 257 return t;
258 return 0; 258 return 0;
259 } 259 }
260 EXPORT_SYMBOL(wait_for_completion_killable); 260 EXPORT_SYMBOL(wait_for_completion_killable);
261 261
262 int __sched wait_for_completion_state(struct c 262 int __sched wait_for_completion_state(struct completion *x, unsigned int state)
263 { 263 {
264 long t = wait_for_common(x, MAX_SCHEDU 264 long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, state);
265 265
266 if (t == -ERESTARTSYS) 266 if (t == -ERESTARTSYS)
267 return t; 267 return t;
268 return 0; 268 return 0;
269 } 269 }
270 EXPORT_SYMBOL(wait_for_completion_state); 270 EXPORT_SYMBOL(wait_for_completion_state);
271 271
272 /** 272 /**
273 * wait_for_completion_killable_timeout: - wai 273 * wait_for_completion_killable_timeout: - waits for completion of a task (w/(to,killable))
274 * @x: holds the state of this particular com 274 * @x: holds the state of this particular completion
275 * @timeout: timeout value in jiffies 275 * @timeout: timeout value in jiffies
276 * 276 *
277 * This waits for either a completion of a spe 277 * This waits for either a completion of a specific task to be
278 * signaled or for a specified timeout to expi 278 * signaled or for a specified timeout to expire. It can be
279 * interrupted by a kill signal. The timeout i 279 * interrupted by a kill signal. The timeout is in jiffies.
280 * 280 *
281 * Return: -ERESTARTSYS if interrupted, 0 if t 281 * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1,
282 * or number of jiffies left till timeout) if 282 * or number of jiffies left till timeout) if completed.
283 */ 283 */
284 long __sched 284 long __sched
285 wait_for_completion_killable_timeout(struct co 285 wait_for_completion_killable_timeout(struct completion *x,
286 unsigned 286 unsigned long timeout)
287 { 287 {
288 return wait_for_common(x, timeout, TAS 288 return wait_for_common(x, timeout, TASK_KILLABLE);
289 } 289 }
290 EXPORT_SYMBOL(wait_for_completion_killable_tim 290 EXPORT_SYMBOL(wait_for_completion_killable_timeout);
291 291
292 /** 292 /**
293 * try_wait_for_completion - try to decre 293 * try_wait_for_completion - try to decrement a completion without blocking
294 * @x: completion structure 294 * @x: completion structure
295 * 295 *
296 * Return: 0 if a decrement cannot be don 296 * Return: 0 if a decrement cannot be done without blocking
297 * 1 if a decrement succeeded. 297 * 1 if a decrement succeeded.
298 * 298 *
299 * If a completion is being used as a cou 299 * If a completion is being used as a counting completion,
300 * attempt to decrement the counter witho 300 * attempt to decrement the counter without blocking. This
301 * enables us to avoid waiting if the res 301 * enables us to avoid waiting if the resource the completion
302 * is protecting is not available. 302 * is protecting is not available.
303 */ 303 */
304 bool try_wait_for_completion(struct completion 304 bool try_wait_for_completion(struct completion *x)
305 { 305 {
306 unsigned long flags; 306 unsigned long flags;
307 bool ret = true; 307 bool ret = true;
308 308
309 /* 309 /*
310 * Since x->done will need to be locke 310 * Since x->done will need to be locked only
311 * in the non-blocking case, we check 311 * in the non-blocking case, we check x->done
312 * first without taking the lock so we 312 * first without taking the lock so we can
313 * return early in the blocking case. 313 * return early in the blocking case.
314 */ 314 */
315 if (!READ_ONCE(x->done)) 315 if (!READ_ONCE(x->done))
316 return false; 316 return false;
317 317
318 raw_spin_lock_irqsave(&x->wait.lock, f 318 raw_spin_lock_irqsave(&x->wait.lock, flags);
319 if (!x->done) 319 if (!x->done)
320 ret = false; 320 ret = false;
321 else if (x->done != UINT_MAX) 321 else if (x->done != UINT_MAX)
322 x->done--; 322 x->done--;
323 raw_spin_unlock_irqrestore(&x->wait.lo 323 raw_spin_unlock_irqrestore(&x->wait.lock, flags);
324 return ret; 324 return ret;
325 } 325 }
326 EXPORT_SYMBOL(try_wait_for_completion); 326 EXPORT_SYMBOL(try_wait_for_completion);
327 327
328 /** 328 /**
329 * completion_done - Test to see if a com 329 * completion_done - Test to see if a completion has any waiters
330 * @x: completion structure 330 * @x: completion structure
331 * 331 *
332 * Return: 0 if there are waiters (wait_f 332 * Return: 0 if there are waiters (wait_for_completion() in progress)
333 * 1 if there are no waiters. 333 * 1 if there are no waiters.
334 * 334 *
335 * Note, this will always return true if 335 * Note, this will always return true if complete_all() was called on @X.
336 */ 336 */
337 bool completion_done(struct completion *x) 337 bool completion_done(struct completion *x)
338 { 338 {
339 unsigned long flags; 339 unsigned long flags;
340 340
341 if (!READ_ONCE(x->done)) 341 if (!READ_ONCE(x->done))
342 return false; 342 return false;
343 343
344 /* 344 /*
345 * If ->done, we need to wait for comp 345 * If ->done, we need to wait for complete() to release ->wait.lock
346 * otherwise we can end up freeing the 346 * otherwise we can end up freeing the completion before complete()
347 * is done referencing it. 347 * is done referencing it.
348 */ 348 */
349 raw_spin_lock_irqsave(&x->wait.lock, f 349 raw_spin_lock_irqsave(&x->wait.lock, flags);
350 raw_spin_unlock_irqrestore(&x->wait.lo 350 raw_spin_unlock_irqrestore(&x->wait.lock, flags);
351 return true; 351 return true;
352 } 352 }
353 EXPORT_SYMBOL(completion_done); 353 EXPORT_SYMBOL(completion_done);
354 354
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.