1 /* SPDX-License-Identifier: GPL-2.0+ */
2 /*
3 * Read-Copy Update mechanism for mutual exclusion (tree-based version)
4 * Internal non-public definitions that provide either classic
5 * or preemptible semantics.
6 *
7 * Copyright Red Hat, 2009
8 * Copyright IBM Corporation, 2009
9 * Copyright SUSE, 2021
10 *
11 * Author: Ingo Molnar <mingo@elte.hu>
12 * Paul E. McKenney <paulmck@linux.ibm.com>
13 * Frederic Weisbecker <frederic@kernel.org>
14 */
15
16 #ifdef CONFIG_RCU_NOCB_CPU
17 static cpumask_var_t rcu_nocb_mask; /* CPUs to have callbacks offloaded. */
18 static bool __read_mostly rcu_nocb_poll; /* Offload kthread are to poll. */
19 static inline int rcu_lockdep_is_held_nocb(struct rcu_data *rdp)
20 {
21 return lockdep_is_held(&rdp->nocb_lock);
22 }
23
24 static inline bool rcu_current_is_nocb_kthread(struct rcu_data *rdp)
25 {
26 /* Race on early boot between thread creation and assignment */
27 if (!rdp->nocb_cb_kthread || !rdp->nocb_gp_kthread)
28 return true;
29
30 if (current == rdp->nocb_cb_kthread || current == rdp->nocb_gp_kthread)
31 if (in_task())
32 return true;
33 return false;
34 }
35
36 /*
37 * Offload callback processing from the boot-time-specified set of CPUs
38 * specified by rcu_nocb_mask. For the CPUs in the set, there are kthreads
39 * created that pull the callbacks from the corresponding CPU, wait for
40 * a grace period to elapse, and invoke the callbacks. These kthreads
41 * are organized into GP kthreads, which manage incoming callbacks, wait for
42 * grace periods, and awaken CB kthreads, and the CB kthreads, which only
43 * invoke callbacks. Each GP kthread invokes its own CBs. The no-CBs CPUs
44 * do a wake_up() on their GP kthread when they insert a callback into any
45 * empty list, unless the rcu_nocb_poll boot parameter has been specified,
46 * in which case each kthread actively polls its CPU. (Which isn't so great
47 * for energy efficiency, but which does reduce RCU's overhead on that CPU.)
48 *
49 * This is intended to be used in conjunction with Frederic Weisbecker's
50 * adaptive-idle work, which would seriously reduce OS jitter on CPUs
51 * running CPU-bound user-mode computations.
52 *
53 * Offloading of callbacks can also be used as an energy-efficiency
54 * measure because CPUs with no RCU callbacks queued are more aggressive
55 * about entering dyntick-idle mode.
56 */
57
58
59 /*
60 * Parse the boot-time rcu_nocb_mask CPU list from the kernel parameters.
61 * If the list is invalid, a warning is emitted and all CPUs are offloaded.
62 */
63 static int __init rcu_nocb_setup(char *str)
64 {
65 alloc_bootmem_cpumask_var(&rcu_nocb_mask);
66 if (*str == '=') {
67 if (cpulist_parse(++str, rcu_nocb_mask)) {
68 pr_warn("rcu_nocbs= bad CPU range, all CPUs set\n");
69 cpumask_setall(rcu_nocb_mask);
70 }
71 }
72 rcu_state.nocb_is_setup = true;
73 return 1;
74 }
75 __setup("rcu_nocbs", rcu_nocb_setup);
76
77 static int __init parse_rcu_nocb_poll(char *arg)
78 {
79 rcu_nocb_poll = true;
80 return 1;
81 }
82 __setup("rcu_nocb_poll", parse_rcu_nocb_poll);
83
84 /*
85 * Don't bother bypassing ->cblist if the call_rcu() rate is low.
86 * After all, the main point of bypassing is to avoid lock contention
87 * on ->nocb_lock, which only can happen at high call_rcu() rates.
88 */
89 static int nocb_nobypass_lim_per_jiffy = 16 * 1000 / HZ;
90 module_param(nocb_nobypass_lim_per_jiffy, int, 0);
91
92 /*
93 * Acquire the specified rcu_data structure's ->nocb_bypass_lock. If the
94 * lock isn't immediately available, perform minimal sanity check.
95 */
96 static void rcu_nocb_bypass_lock(struct rcu_data *rdp)
97 __acquires(&rdp->nocb_bypass_lock)
98 {
99 lockdep_assert_irqs_disabled();
100 if (raw_spin_trylock(&rdp->nocb_bypass_lock))
101 return;
102 /*
103 * Contention expected only when local enqueue collide with
104 * remote flush from kthreads.
105 */
106 WARN_ON_ONCE(smp_processor_id() != rdp->cpu);
107 raw_spin_lock(&rdp->nocb_bypass_lock);
108 }
109
110 /*
111 * Conditionally acquire the specified rcu_data structure's
112 * ->nocb_bypass_lock.
113 */
114 static bool rcu_nocb_bypass_trylock(struct rcu_data *rdp)
115 {
116 lockdep_assert_irqs_disabled();
117 return raw_spin_trylock(&rdp->nocb_bypass_lock);
118 }
119
120 /*
121 * Release the specified rcu_data structure's ->nocb_bypass_lock.
122 */
123 static void rcu_nocb_bypass_unlock(struct rcu_data *rdp)
124 __releases(&rdp->nocb_bypass_lock)
125 {
126 lockdep_assert_irqs_disabled();
127 raw_spin_unlock(&rdp->nocb_bypass_lock);
128 }
129
130 /*
131 * Acquire the specified rcu_data structure's ->nocb_lock, but only
132 * if it corresponds to a no-CBs CPU.
133 */
134 static void rcu_nocb_lock(struct rcu_data *rdp)
135 {
136 lockdep_assert_irqs_disabled();
137 if (!rcu_rdp_is_offloaded(rdp))
138 return;
139 raw_spin_lock(&rdp->nocb_lock);
140 }
141
142 /*
143 * Release the specified rcu_data structure's ->nocb_lock, but only
144 * if it corresponds to a no-CBs CPU.
145 */
146 static void rcu_nocb_unlock(struct rcu_data *rdp)
147 {
148 if (rcu_rdp_is_offloaded(rdp)) {
149 lockdep_assert_irqs_disabled();
150 raw_spin_unlock(&rdp->nocb_lock);
151 }
152 }
153
154 /*
155 * Release the specified rcu_data structure's ->nocb_lock and restore
156 * interrupts, but only if it corresponds to a no-CBs CPU.
157 */
158 static void rcu_nocb_unlock_irqrestore(struct rcu_data *rdp,
159 unsigned long flags)
160 {
161 if (rcu_rdp_is_offloaded(rdp)) {
162 lockdep_assert_irqs_disabled();
163 raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags);
164 } else {
165 local_irq_restore(flags);
166 }
167 }
168
169 /* Lockdep check that ->cblist may be safely accessed. */
170 static void rcu_lockdep_assert_cblist_protected(struct rcu_data *rdp)
171 {
172 lockdep_assert_irqs_disabled();
173 if (rcu_rdp_is_offloaded(rdp))
174 lockdep_assert_held(&rdp->nocb_lock);
175 }
176
177 /*
178 * Wake up any no-CBs CPUs' kthreads that were waiting on the just-ended
179 * grace period.
180 */
181 static void rcu_nocb_gp_cleanup(struct swait_queue_head *sq)
182 {
183 swake_up_all(sq);
184 }
185
186 static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp)
187 {
188 return &rnp->nocb_gp_wq[rcu_seq_ctr(rnp->gp_seq) & 0x1];
189 }
190
191 static void rcu_init_one_nocb(struct rcu_node *rnp)
192 {
193 init_swait_queue_head(&rnp->nocb_gp_wq[0]);
194 init_swait_queue_head(&rnp->nocb_gp_wq[1]);
195 }
196
197 static bool __wake_nocb_gp(struct rcu_data *rdp_gp,
198 struct rcu_data *rdp,
199 bool force, unsigned long flags)
200 __releases(rdp_gp->nocb_gp_lock)
201 {
202 bool needwake = false;
203
204 if (!READ_ONCE(rdp_gp->nocb_gp_kthread)) {
205 raw_spin_unlock_irqrestore(&rdp_gp->nocb_gp_lock, flags);
206 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,
207 TPS("AlreadyAwake"));
208 return false;
209 }
210
211 if (rdp_gp->nocb_defer_wakeup > RCU_NOCB_WAKE_NOT) {
212 WRITE_ONCE(rdp_gp->nocb_defer_wakeup, RCU_NOCB_WAKE_NOT);
213 del_timer(&rdp_gp->nocb_timer);
214 }
215
216 if (force || READ_ONCE(rdp_gp->nocb_gp_sleep)) {
217 WRITE_ONCE(rdp_gp->nocb_gp_sleep, false);
218 needwake = true;
219 }
220 raw_spin_unlock_irqrestore(&rdp_gp->nocb_gp_lock, flags);
221 if (needwake) {
222 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("DoWake"));
223 if (cpu_is_offline(raw_smp_processor_id()))
224 swake_up_one_online(&rdp_gp->nocb_gp_wq);
225 else
226 wake_up_process(rdp_gp->nocb_gp_kthread);
227 }
228
229 return needwake;
230 }
231
232 /*
233 * Kick the GP kthread for this NOCB group.
234 */
235 static bool wake_nocb_gp(struct rcu_data *rdp, bool force)
236 {
237 unsigned long flags;
238 struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;
239
240 raw_spin_lock_irqsave(&rdp_gp->nocb_gp_lock, flags);
241 return __wake_nocb_gp(rdp_gp, rdp, force, flags);
242 }
243
244 #ifdef CONFIG_RCU_LAZY
245 /*
246 * LAZY_FLUSH_JIFFIES decides the maximum amount of time that
247 * can elapse before lazy callbacks are flushed. Lazy callbacks
248 * could be flushed much earlier for a number of other reasons
249 * however, LAZY_FLUSH_JIFFIES will ensure no lazy callbacks are
250 * left unsubmitted to RCU after those many jiffies.
251 */
252 #define LAZY_FLUSH_JIFFIES (10 * HZ)
253 static unsigned long jiffies_lazy_flush = LAZY_FLUSH_JIFFIES;
254
255 // To be called only from test code.
256 void rcu_set_jiffies_lazy_flush(unsigned long jif)
257 {
258 jiffies_lazy_flush = jif;
259 }
260 EXPORT_SYMBOL(rcu_set_jiffies_lazy_flush);
261
262 unsigned long rcu_get_jiffies_lazy_flush(void)
263 {
264 return jiffies_lazy_flush;
265 }
266 EXPORT_SYMBOL(rcu_get_jiffies_lazy_flush);
267 #endif
268
269 /*
270 * Arrange to wake the GP kthread for this NOCB group at some future
271 * time when it is safe to do so.
272 */
273 static void wake_nocb_gp_defer(struct rcu_data *rdp, int waketype,
274 const char *reason)
275 {
276 unsigned long flags;
277 struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;
278
279 raw_spin_lock_irqsave(&rdp_gp->nocb_gp_lock, flags);
280
281 /*
282 * Bypass wakeup overrides previous deferments. In case of
283 * callback storms, no need to wake up too early.
284 */
285 if (waketype == RCU_NOCB_WAKE_LAZY &&
286 rdp->nocb_defer_wakeup == RCU_NOCB_WAKE_NOT) {
287 mod_timer(&rdp_gp->nocb_timer, jiffies + rcu_get_jiffies_lazy_flush());
288 WRITE_ONCE(rdp_gp->nocb_defer_wakeup, waketype);
289 } else if (waketype == RCU_NOCB_WAKE_BYPASS) {
290 mod_timer(&rdp_gp->nocb_timer, jiffies + 2);
291 WRITE_ONCE(rdp_gp->nocb_defer_wakeup, waketype);
292 } else {
293 if (rdp_gp->nocb_defer_wakeup < RCU_NOCB_WAKE)
294 mod_timer(&rdp_gp->nocb_timer, jiffies + 1);
295 if (rdp_gp->nocb_defer_wakeup < waketype)
296 WRITE_ONCE(rdp_gp->nocb_defer_wakeup, waketype);
297 }
298
299 raw_spin_unlock_irqrestore(&rdp_gp->nocb_gp_lock, flags);
300
301 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, reason);
302 }
303
304 /*
305 * Flush the ->nocb_bypass queue into ->cblist, enqueuing rhp if non-NULL.
306 * However, if there is a callback to be enqueued and if ->nocb_bypass
307 * proves to be initially empty, just return false because the no-CB GP
308 * kthread may need to be awakened in this case.
309 *
310 * Return true if there was something to be flushed and it succeeded, otherwise
311 * false.
312 *
313 * Note that this function always returns true if rhp is NULL.
314 */
315 static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp_in,
316 unsigned long j, bool lazy)
317 {
318 struct rcu_cblist rcl;
319 struct rcu_head *rhp = rhp_in;
320
321 WARN_ON_ONCE(!rcu_rdp_is_offloaded(rdp));
322 rcu_lockdep_assert_cblist_protected(rdp);
323 lockdep_assert_held(&rdp->nocb_bypass_lock);
324 if (rhp && !rcu_cblist_n_cbs(&rdp->nocb_bypass)) {
325 raw_spin_unlock(&rdp->nocb_bypass_lock);
326 return false;
327 }
328 /* Note: ->cblist.len already accounts for ->nocb_bypass contents. */
329 if (rhp)
330 rcu_segcblist_inc_len(&rdp->cblist); /* Must precede enqueue. */
331
332 /*
333 * If the new CB requested was a lazy one, queue it onto the main
334 * ->cblist so that we can take advantage of the grace-period that will
335 * happen regardless. But queue it onto the bypass list first so that
336 * the lazy CB is ordered with the existing CBs in the bypass list.
337 */
338 if (lazy && rhp) {
339 rcu_cblist_enqueue(&rdp->nocb_bypass, rhp);
340 rhp = NULL;
341 }
342 rcu_cblist_flush_enqueue(&rcl, &rdp->nocb_bypass, rhp);
343 WRITE_ONCE(rdp->lazy_len, 0);
344
345 rcu_segcblist_insert_pend_cbs(&rdp->cblist, &rcl);
346 WRITE_ONCE(rdp->nocb_bypass_first, j);
347 rcu_nocb_bypass_unlock(rdp);
348 return true;
349 }
350
351 /*
352 * Flush the ->nocb_bypass queue into ->cblist, enqueuing rhp if non-NULL.
353 * However, if there is a callback to be enqueued and if ->nocb_bypass
354 * proves to be initially empty, just return false because the no-CB GP
355 * kthread may need to be awakened in this case.
356 *
357 * Note that this function always returns true if rhp is NULL.
358 */
359 static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp,
360 unsigned long j, bool lazy)
361 {
362 if (!rcu_rdp_is_offloaded(rdp))
363 return true;
364 rcu_lockdep_assert_cblist_protected(rdp);
365 rcu_nocb_bypass_lock(rdp);
366 return rcu_nocb_do_flush_bypass(rdp, rhp, j, lazy);
367 }
368
369 /*
370 * If the ->nocb_bypass_lock is immediately available, flush the
371 * ->nocb_bypass queue into ->cblist.
372 */
373 static void rcu_nocb_try_flush_bypass(struct rcu_data *rdp, unsigned long j)
374 {
375 rcu_lockdep_assert_cblist_protected(rdp);
376 if (!rcu_rdp_is_offloaded(rdp) ||
377 !rcu_nocb_bypass_trylock(rdp))
378 return;
379 WARN_ON_ONCE(!rcu_nocb_do_flush_bypass(rdp, NULL, j, false));
380 }
381
382 /*
383 * See whether it is appropriate to use the ->nocb_bypass list in order
384 * to control contention on ->nocb_lock. A limited number of direct
385 * enqueues are permitted into ->cblist per jiffy. If ->nocb_bypass
386 * is non-empty, further callbacks must be placed into ->nocb_bypass,
387 * otherwise rcu_barrier() breaks. Use rcu_nocb_flush_bypass() to switch
388 * back to direct use of ->cblist. However, ->nocb_bypass should not be
389 * used if ->cblist is empty, because otherwise callbacks can be stranded
390 * on ->nocb_bypass because we cannot count on the current CPU ever again
391 * invoking call_rcu(). The general rule is that if ->nocb_bypass is
392 * non-empty, the corresponding no-CBs grace-period kthread must not be
393 * in an indefinite sleep state.
394 *
395 * Finally, it is not permitted to use the bypass during early boot,
396 * as doing so would confuse the auto-initialization code. Besides
397 * which, there is no point in worrying about lock contention while
398 * there is only one CPU in operation.
399 */
400 static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp,
401 bool *was_alldone, unsigned long flags,
402 bool lazy)
403 {
404 unsigned long c;
405 unsigned long cur_gp_seq;
406 unsigned long j = jiffies;
407 long ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass);
408 bool bypass_is_lazy = (ncbs == READ_ONCE(rdp->lazy_len));
409
410 lockdep_assert_irqs_disabled();
411
412 // Pure softirq/rcuc based processing: no bypassing, no
413 // locking.
414 if (!rcu_rdp_is_offloaded(rdp)) {
415 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist);
416 return false;
417 }
418
419 // In the process of (de-)offloading: no bypassing, but
420 // locking.
421 if (!rcu_segcblist_completely_offloaded(&rdp->cblist)) {
422 rcu_nocb_lock(rdp);
423 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist);
424 return false; /* Not offloaded, no bypassing. */
425 }
426
427 // Don't use ->nocb_bypass during early boot.
428 if (rcu_scheduler_active != RCU_SCHEDULER_RUNNING) {
429 rcu_nocb_lock(rdp);
430 WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass));
431 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist);
432 return false;
433 }
434
435 // If we have advanced to a new jiffy, reset counts to allow
436 // moving back from ->nocb_bypass to ->cblist.
437 if (j == rdp->nocb_nobypass_last) {
438 c = rdp->nocb_nobypass_count + 1;
439 } else {
440 WRITE_ONCE(rdp->nocb_nobypass_last, j);
441 c = rdp->nocb_nobypass_count - nocb_nobypass_lim_per_jiffy;
442 if (ULONG_CMP_LT(rdp->nocb_nobypass_count,
443 nocb_nobypass_lim_per_jiffy))
444 c = 0;
445 else if (c > nocb_nobypass_lim_per_jiffy)
446 c = nocb_nobypass_lim_per_jiffy;
447 }
448 WRITE_ONCE(rdp->nocb_nobypass_count, c);
449
450 // If there hasn't yet been all that many ->cblist enqueues
451 // this jiffy, tell the caller to enqueue onto ->cblist. But flush
452 // ->nocb_bypass first.
453 // Lazy CBs throttle this back and do immediate bypass queuing.
454 if (rdp->nocb_nobypass_count < nocb_nobypass_lim_per_jiffy && !lazy) {
455 rcu_nocb_lock(rdp);
456 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist);
457 if (*was_alldone)
458 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,
459 TPS("FirstQ"));
460
461 WARN_ON_ONCE(!rcu_nocb_flush_bypass(rdp, NULL, j, false));
462 WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass));
463 return false; // Caller must enqueue the callback.
464 }
465
466 // If ->nocb_bypass has been used too long or is too full,
467 // flush ->nocb_bypass to ->cblist.
468 if ((ncbs && !bypass_is_lazy && j != READ_ONCE(rdp->nocb_bypass_first)) ||
469 (ncbs && bypass_is_lazy &&
470 (time_after(j, READ_ONCE(rdp->nocb_bypass_first) + rcu_get_jiffies_lazy_flush()))) ||
471 ncbs >= qhimark) {
472 rcu_nocb_lock(rdp);
473 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist);
474
475 if (!rcu_nocb_flush_bypass(rdp, rhp, j, lazy)) {
476 if (*was_alldone)
477 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,
478 TPS("FirstQ"));
479 WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass));
480 return false; // Caller must enqueue the callback.
481 }
482 if (j != rdp->nocb_gp_adv_time &&
483 rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) &&
484 rcu_seq_done(&rdp->mynode->gp_seq, cur_gp_seq)) {
485 rcu_advance_cbs_nowake(rdp->mynode, rdp);
486 rdp->nocb_gp_adv_time = j;
487 }
488
489 // The flush succeeded and we moved CBs into the regular list.
490 // Don't wait for the wake up timer as it may be too far ahead.
491 // Wake up the GP thread now instead, if the cblist was empty.
492 __call_rcu_nocb_wake(rdp, *was_alldone, flags);
493
494 return true; // Callback already enqueued.
495 }
496
497 // We need to use the bypass.
498 rcu_nocb_bypass_lock(rdp);
499 ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass);
500 rcu_segcblist_inc_len(&rdp->cblist); /* Must precede enqueue. */
501 rcu_cblist_enqueue(&rdp->nocb_bypass, rhp);
502
503 if (lazy)
504 WRITE_ONCE(rdp->lazy_len, rdp->lazy_len + 1);
505
506 if (!ncbs) {
507 WRITE_ONCE(rdp->nocb_bypass_first, j);
508 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("FirstBQ"));
509 }
510 rcu_nocb_bypass_unlock(rdp);
511 smp_mb(); /* Order enqueue before wake. */
512 // A wake up of the grace period kthread or timer adjustment
513 // needs to be done only if:
514 // 1. Bypass list was fully empty before (this is the first
515 // bypass list entry), or:
516 // 2. Both of these conditions are met:
517 // a. The bypass list previously had only lazy CBs, and:
518 // b. The new CB is non-lazy.
519 if (!ncbs || (bypass_is_lazy && !lazy)) {
520 // No-CBs GP kthread might be indefinitely asleep, if so, wake.
521 rcu_nocb_lock(rdp); // Rare during call_rcu() flood.
522 if (!rcu_segcblist_pend_cbs(&rdp->cblist)) {
523 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,
524 TPS("FirstBQwake"));
525 __call_rcu_nocb_wake(rdp, true, flags);
526 } else {
527 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,
528 TPS("FirstBQnoWake"));
529 rcu_nocb_unlock(rdp);
530 }
531 }
532 return true; // Callback already enqueued.
533 }
534
535 /*
536 * Awaken the no-CBs grace-period kthread if needed, either due to it
537 * legitimately being asleep or due to overload conditions.
538 *
539 * If warranted, also wake up the kthread servicing this CPUs queues.
540 */
541 static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone,
542 unsigned long flags)
543 __releases(rdp->nocb_lock)
544 {
545 long bypass_len;
546 unsigned long cur_gp_seq;
547 unsigned long j;
548 long lazy_len;
549 long len;
550 struct task_struct *t;
551 struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;
552
553 // If we are being polled or there is no kthread, just leave.
554 t = READ_ONCE(rdp->nocb_gp_kthread);
555 if (rcu_nocb_poll || !t) {
556 rcu_nocb_unlock(rdp);
557 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,
558 TPS("WakeNotPoll"));
559 return;
560 }
561 // Need to actually to a wakeup.
562 len = rcu_segcblist_n_cbs(&rdp->cblist);
563 bypass_len = rcu_cblist_n_cbs(&rdp->nocb_bypass);
564 lazy_len = READ_ONCE(rdp->lazy_len);
565 if (was_alldone) {
566 rdp->qlen_last_fqs_check = len;
567 // Only lazy CBs in bypass list
568 if (lazy_len && bypass_len == lazy_len) {
569 rcu_nocb_unlock(rdp);
570 wake_nocb_gp_defer(rdp, RCU_NOCB_WAKE_LAZY,
571 TPS("WakeLazy"));
572 } else if (!irqs_disabled_flags(flags) && cpu_online(rdp->cpu)) {
573 /* ... if queue was empty ... */
574 rcu_nocb_unlock(rdp);
575 wake_nocb_gp(rdp, false);
576 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,
577 TPS("WakeEmpty"));
578 } else {
579 /*
580 * Don't do the wake-up upfront on fragile paths.
581 * Also offline CPUs can't call swake_up_one_online() from
582 * (soft-)IRQs. Rely on the final deferred wake-up from
583 * rcutree_report_cpu_dead()
584 */
585 rcu_nocb_unlock(rdp);
586 wake_nocb_gp_defer(rdp, RCU_NOCB_WAKE,
587 TPS("WakeEmptyIsDeferred"));
588 }
589 } else if (len > rdp->qlen_last_fqs_check + qhimark) {
590 /* ... or if many callbacks queued. */
591 rdp->qlen_last_fqs_check = len;
592 j = jiffies;
593 if (j != rdp->nocb_gp_adv_time &&
594 rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) &&
595 rcu_seq_done(&rdp->mynode->gp_seq, cur_gp_seq)) {
596 rcu_advance_cbs_nowake(rdp->mynode, rdp);
597 rdp->nocb_gp_adv_time = j;
598 }
599 smp_mb(); /* Enqueue before timer_pending(). */
600 if ((rdp->nocb_cb_sleep ||
601 !rcu_segcblist_ready_cbs(&rdp->cblist)) &&
602 !timer_pending(&rdp_gp->nocb_timer)) {
603 rcu_nocb_unlock(rdp);
604 wake_nocb_gp_defer(rdp, RCU_NOCB_WAKE_FORCE,
605 TPS("WakeOvfIsDeferred"));
606 } else {
607 rcu_nocb_unlock(rdp);
608 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WakeNot"));
609 }
610 } else {
611 rcu_nocb_unlock(rdp);
612 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WakeNot"));
613 }
614 }
615
616 static void call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *head,
617 rcu_callback_t func, unsigned long flags, bool lazy)
618 {
619 bool was_alldone;
620
621 if (!rcu_nocb_try_bypass(rdp, head, &was_alldone, flags, lazy)) {
622 /* Not enqueued on bypass but locked, do regular enqueue */
623 rcutree_enqueue(rdp, head, func);
624 __call_rcu_nocb_wake(rdp, was_alldone, flags); /* unlocks */
625 }
626 }
627
628 static int nocb_gp_toggle_rdp(struct rcu_data *rdp)
629 {
630 struct rcu_segcblist *cblist = &rdp->cblist;
631 unsigned long flags;
632 int ret;
633
634 rcu_nocb_lock_irqsave(rdp, flags);
635 if (rcu_segcblist_test_flags(cblist, SEGCBLIST_OFFLOADED) &&
636 !rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_GP)) {
637 /*
638 * Offloading. Set our flag and notify the offload worker.
639 * We will handle this rdp until it ever gets de-offloaded.
640 */
641 rcu_segcblist_set_flags(cblist, SEGCBLIST_KTHREAD_GP);
642 ret = 1;
643 } else if (!rcu_segcblist_test_flags(cblist, SEGCBLIST_OFFLOADED) &&
644 rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_GP)) {
645 /*
646 * De-offloading. Clear our flag and notify the de-offload worker.
647 * We will ignore this rdp until it ever gets re-offloaded.
648 */
649 rcu_segcblist_clear_flags(cblist, SEGCBLIST_KTHREAD_GP);
650 ret = 0;
651 } else {
652 WARN_ON_ONCE(1);
653 ret = -1;
654 }
655
656 rcu_nocb_unlock_irqrestore(rdp, flags);
657
658 return ret;
659 }
660
661 static void nocb_gp_sleep(struct rcu_data *my_rdp, int cpu)
662 {
663 trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("Sleep"));
664 swait_event_interruptible_exclusive(my_rdp->nocb_gp_wq,
665 !READ_ONCE(my_rdp->nocb_gp_sleep));
666 trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("EndSleep"));
667 }
668
669 /*
670 * No-CBs GP kthreads come here to wait for additional callbacks to show up
671 * or for grace periods to end.
672 */
673 static void nocb_gp_wait(struct rcu_data *my_rdp)
674 {
675 bool bypass = false;
676 int __maybe_unused cpu = my_rdp->cpu;
677 unsigned long cur_gp_seq;
678 unsigned long flags;
679 bool gotcbs = false;
680 unsigned long j = jiffies;
681 bool lazy = false;
682 bool needwait_gp = false; // This prevents actual uninitialized use.
683 bool needwake;
684 bool needwake_gp;
685 struct rcu_data *rdp, *rdp_toggling = NULL;
686 struct rcu_node *rnp;
687 unsigned long wait_gp_seq = 0; // Suppress "use uninitialized" warning.
688 bool wasempty = false;
689
690 /*
691 * Each pass through the following loop checks for CBs and for the
692 * nearest grace period (if any) to wait for next. The CB kthreads
693 * and the global grace-period kthread are awakened if needed.
694 */
695 WARN_ON_ONCE(my_rdp->nocb_gp_rdp != my_rdp);
696 /*
697 * An rcu_data structure is removed from the list after its
698 * CPU is de-offloaded and added to the list before that CPU is
699 * (re-)offloaded. If the following loop happens to be referencing
700 * that rcu_data structure during the time that the corresponding
701 * CPU is de-offloaded and then immediately re-offloaded, this
702 * loop's rdp pointer will be carried to the end of the list by
703 * the resulting pair of list operations. This can cause the loop
704 * to skip over some of the rcu_data structures that were supposed
705 * to have been scanned. Fortunately a new iteration through the
706 * entire loop is forced after a given CPU's rcu_data structure
707 * is added to the list, so the skipped-over rcu_data structures
708 * won't be ignored for long.
709 */
710 list_for_each_entry(rdp, &my_rdp->nocb_head_rdp, nocb_entry_rdp) {
711 long bypass_ncbs;
712 bool flush_bypass = false;
713 long lazy_ncbs;
714
715 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("Check"));
716 rcu_nocb_lock_irqsave(rdp, flags);
717 lockdep_assert_held(&rdp->nocb_lock);
718 bypass_ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass);
719 lazy_ncbs = READ_ONCE(rdp->lazy_len);
720
721 if (bypass_ncbs && (lazy_ncbs == bypass_ncbs) &&
722 (time_after(j, READ_ONCE(rdp->nocb_bypass_first) + rcu_get_jiffies_lazy_flush()) ||
723 bypass_ncbs > 2 * qhimark)) {
724 flush_bypass = true;
725 } else if (bypass_ncbs && (lazy_ncbs != bypass_ncbs) &&
726 (time_after(j, READ_ONCE(rdp->nocb_bypass_first) + 1) ||
727 bypass_ncbs > 2 * qhimark)) {
728 flush_bypass = true;
729 } else if (!bypass_ncbs && rcu_segcblist_empty(&rdp->cblist)) {
730 rcu_nocb_unlock_irqrestore(rdp, flags);
731 continue; /* No callbacks here, try next. */
732 }
733
734 if (flush_bypass) {
735 // Bypass full or old, so flush it.
736 (void)rcu_nocb_try_flush_bypass(rdp, j);
737 bypass_ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass);
738 lazy_ncbs = READ_ONCE(rdp->lazy_len);
739 }
740
741 if (bypass_ncbs) {
742 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,
743 bypass_ncbs == lazy_ncbs ? TPS("Lazy") : TPS("Bypass"));
744 if (bypass_ncbs == lazy_ncbs)
745 lazy = true;
746 else
747 bypass = true;
748 }
749 rnp = rdp->mynode;
750
751 // Advance callbacks if helpful and low contention.
752 needwake_gp = false;
753 if (!rcu_segcblist_restempty(&rdp->cblist,
754 RCU_NEXT_READY_TAIL) ||
755 (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) &&
756 rcu_seq_done(&rnp->gp_seq, cur_gp_seq))) {
757 raw_spin_lock_rcu_node(rnp); /* irqs disabled. */
758 needwake_gp = rcu_advance_cbs(rnp, rdp);
759 wasempty = rcu_segcblist_restempty(&rdp->cblist,
760 RCU_NEXT_READY_TAIL);
761 raw_spin_unlock_rcu_node(rnp); /* irqs disabled. */
762 }
763 // Need to wait on some grace period?
764 WARN_ON_ONCE(wasempty &&
765 !rcu_segcblist_restempty(&rdp->cblist,
766 RCU_NEXT_READY_TAIL));
767 if (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq)) {
768 if (!needwait_gp ||
769 ULONG_CMP_LT(cur_gp_seq, wait_gp_seq))
770 wait_gp_seq = cur_gp_seq;
771 needwait_gp = true;
772 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,
773 TPS("NeedWaitGP"));
774 }
775 if (rcu_segcblist_ready_cbs(&rdp->cblist)) {
776 needwake = rdp->nocb_cb_sleep;
777 WRITE_ONCE(rdp->nocb_cb_sleep, false);
778 } else {
779 needwake = false;
780 }
781 rcu_nocb_unlock_irqrestore(rdp, flags);
782 if (needwake) {
783 swake_up_one(&rdp->nocb_cb_wq);
784 gotcbs = true;
785 }
786 if (needwake_gp)
787 rcu_gp_kthread_wake();
788 }
789
790 my_rdp->nocb_gp_bypass = bypass;
791 my_rdp->nocb_gp_gp = needwait_gp;
792 my_rdp->nocb_gp_seq = needwait_gp ? wait_gp_seq : 0;
793
794 // At least one child with non-empty ->nocb_bypass, so set
795 // timer in order to avoid stranding its callbacks.
796 if (!rcu_nocb_poll) {
797 // If bypass list only has lazy CBs. Add a deferred lazy wake up.
798 if (lazy && !bypass) {
799 wake_nocb_gp_defer(my_rdp, RCU_NOCB_WAKE_LAZY,
800 TPS("WakeLazyIsDeferred"));
801 // Otherwise add a deferred bypass wake up.
802 } else if (bypass) {
803 wake_nocb_gp_defer(my_rdp, RCU_NOCB_WAKE_BYPASS,
804 TPS("WakeBypassIsDeferred"));
805 }
806 }
807
808 if (rcu_nocb_poll) {
809 /* Polling, so trace if first poll in the series. */
810 if (gotcbs)
811 trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("Poll"));
812 if (list_empty(&my_rdp->nocb_head_rdp)) {
813 raw_spin_lock_irqsave(&my_rdp->nocb_gp_lock, flags);
814 if (!my_rdp->nocb_toggling_rdp)
815 WRITE_ONCE(my_rdp->nocb_gp_sleep, true);
816 raw_spin_unlock_irqrestore(&my_rdp->nocb_gp_lock, flags);
817 /* Wait for any offloading rdp */
818 nocb_gp_sleep(my_rdp, cpu);
819 } else {
820 schedule_timeout_idle(1);
821 }
822 } else if (!needwait_gp) {
823 /* Wait for callbacks to appear. */
824 nocb_gp_sleep(my_rdp, cpu);
825 } else {
826 rnp = my_rdp->mynode;
827 trace_rcu_this_gp(rnp, my_rdp, wait_gp_seq, TPS("StartWait"));
828 swait_event_interruptible_exclusive(
829 rnp->nocb_gp_wq[rcu_seq_ctr(wait_gp_seq) & 0x1],
830 rcu_seq_done(&rnp->gp_seq, wait_gp_seq) ||
831 !READ_ONCE(my_rdp->nocb_gp_sleep));
832 trace_rcu_this_gp(rnp, my_rdp, wait_gp_seq, TPS("EndWait"));
833 }
834
835 if (!rcu_nocb_poll) {
836 raw_spin_lock_irqsave(&my_rdp->nocb_gp_lock, flags);
837 // (De-)queue an rdp to/from the group if its nocb state is changing
838 rdp_toggling = my_rdp->nocb_toggling_rdp;
839 if (rdp_toggling)
840 my_rdp->nocb_toggling_rdp = NULL;
841
842 if (my_rdp->nocb_defer_wakeup > RCU_NOCB_WAKE_NOT) {
843 WRITE_ONCE(my_rdp->nocb_defer_wakeup, RCU_NOCB_WAKE_NOT);
844 del_timer(&my_rdp->nocb_timer);
845 }
846 WRITE_ONCE(my_rdp->nocb_gp_sleep, true);
847 raw_spin_unlock_irqrestore(&my_rdp->nocb_gp_lock, flags);
848 } else {
849 rdp_toggling = READ_ONCE(my_rdp->nocb_toggling_rdp);
850 if (rdp_toggling) {
851 /*
852 * Paranoid locking to make sure nocb_toggling_rdp is well
853 * reset *before* we (re)set SEGCBLIST_KTHREAD_GP or we could
854 * race with another round of nocb toggling for this rdp.
855 * Nocb locking should prevent from that already but we stick
856 * to paranoia, especially in rare path.
857 */
858 raw_spin_lock_irqsave(&my_rdp->nocb_gp_lock, flags);
859 my_rdp->nocb_toggling_rdp = NULL;
860 raw_spin_unlock_irqrestore(&my_rdp->nocb_gp_lock, flags);
861 }
862 }
863
864 if (rdp_toggling) {
865 int ret;
866
867 ret = nocb_gp_toggle_rdp(rdp_toggling);
868 if (ret == 1)
869 list_add_tail(&rdp_toggling->nocb_entry_rdp, &my_rdp->nocb_head_rdp);
870 else if (ret == 0)
871 list_del(&rdp_toggling->nocb_entry_rdp);
872
873 swake_up_one(&rdp_toggling->nocb_state_wq);
874 }
875
876 my_rdp->nocb_gp_seq = -1;
877 WARN_ON(signal_pending(current));
878 }
879
880 /*
881 * No-CBs grace-period-wait kthread. There is one of these per group
882 * of CPUs, but only once at least one CPU in that group has come online
883 * at least once since boot. This kthread checks for newly posted
884 * callbacks from any of the CPUs it is responsible for, waits for a
885 * grace period, then awakens all of the rcu_nocb_cb_kthread() instances
886 * that then have callback-invocation work to do.
887 */
888 static int rcu_nocb_gp_kthread(void *arg)
889 {
890 struct rcu_data *rdp = arg;
891
892 for (;;) {
893 WRITE_ONCE(rdp->nocb_gp_loops, rdp->nocb_gp_loops + 1);
894 nocb_gp_wait(rdp);
895 cond_resched_tasks_rcu_qs();
896 }
897 return 0;
898 }
899
900 static inline bool nocb_cb_wait_cond(struct rcu_data *rdp)
901 {
902 return !READ_ONCE(rdp->nocb_cb_sleep) || kthread_should_park();
903 }
904
905 /*
906 * Invoke any ready callbacks from the corresponding no-CBs CPU,
907 * then, if there are no more, wait for more to appear.
908 */
909 static void nocb_cb_wait(struct rcu_data *rdp)
910 {
911 struct rcu_segcblist *cblist = &rdp->cblist;
912 unsigned long cur_gp_seq;
913 unsigned long flags;
914 bool needwake_gp = false;
915 struct rcu_node *rnp = rdp->mynode;
916
917 swait_event_interruptible_exclusive(rdp->nocb_cb_wq,
918 nocb_cb_wait_cond(rdp));
919 if (kthread_should_park()) {
920 kthread_parkme();
921 } else if (READ_ONCE(rdp->nocb_cb_sleep)) {
922 WARN_ON(signal_pending(current));
923 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WokeEmpty"));
924 }
925
926 WARN_ON_ONCE(!rcu_rdp_is_offloaded(rdp));
927
928 local_irq_save(flags);
929 rcu_momentary_dyntick_idle();
930 local_irq_restore(flags);
931 /*
932 * Disable BH to provide the expected environment. Also, when
933 * transitioning to/from NOCB mode, a self-requeuing callback might
934 * be invoked from softirq. A short grace period could cause both
935 * instances of this callback would execute concurrently.
936 */
937 local_bh_disable();
938 rcu_do_batch(rdp);
939 local_bh_enable();
940 lockdep_assert_irqs_enabled();
941 rcu_nocb_lock_irqsave(rdp, flags);
942 if (rcu_segcblist_nextgp(cblist, &cur_gp_seq) &&
943 rcu_seq_done(&rnp->gp_seq, cur_gp_seq) &&
944 raw_spin_trylock_rcu_node(rnp)) { /* irqs already disabled. */
945 needwake_gp = rcu_advance_cbs(rdp->mynode, rdp);
946 raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */
947 }
948
949 if (!rcu_segcblist_ready_cbs(cblist)) {
950 WRITE_ONCE(rdp->nocb_cb_sleep, true);
951 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("CBSleep"));
952 } else {
953 WRITE_ONCE(rdp->nocb_cb_sleep, false);
954 }
955
956 rcu_nocb_unlock_irqrestore(rdp, flags);
957 if (needwake_gp)
958 rcu_gp_kthread_wake();
959 }
960
961 /*
962 * Per-rcu_data kthread, but only for no-CBs CPUs. Repeatedly invoke
963 * nocb_cb_wait() to do the dirty work.
964 */
965 static int rcu_nocb_cb_kthread(void *arg)
966 {
967 struct rcu_data *rdp = arg;
968
969 // Each pass through this loop does one callback batch, and,
970 // if there are no more ready callbacks, waits for them.
971 for (;;) {
972 nocb_cb_wait(rdp);
973 cond_resched_tasks_rcu_qs();
974 }
975 return 0;
976 }
977
978 /* Is a deferred wakeup of rcu_nocb_kthread() required? */
979 static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp, int level)
980 {
981 return READ_ONCE(rdp->nocb_defer_wakeup) >= level;
982 }
983
984 /* Do a deferred wakeup of rcu_nocb_kthread(). */
985 static bool do_nocb_deferred_wakeup_common(struct rcu_data *rdp_gp,
986 struct rcu_data *rdp, int level,
987 unsigned long flags)
988 __releases(rdp_gp->nocb_gp_lock)
989 {
990 int ndw;
991 int ret;
992
993 if (!rcu_nocb_need_deferred_wakeup(rdp_gp, level)) {
994 raw_spin_unlock_irqrestore(&rdp_gp->nocb_gp_lock, flags);
995 return false;
996 }
997
998 ndw = rdp_gp->nocb_defer_wakeup;
999 ret = __wake_nocb_gp(rdp_gp, rdp, ndw == RCU_NOCB_WAKE_FORCE, flags);
1000 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("DeferredWake"));
1001
1002 return ret;
1003 }
1004
1005 /* Do a deferred wakeup of rcu_nocb_kthread() from a timer handler. */
1006 static void do_nocb_deferred_wakeup_timer(struct timer_list *t)
1007 {
1008 unsigned long flags;
1009 struct rcu_data *rdp = from_timer(rdp, t, nocb_timer);
1010
1011 WARN_ON_ONCE(rdp->nocb_gp_rdp != rdp);
1012 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("Timer"));
1013
1014 raw_spin_lock_irqsave(&rdp->nocb_gp_lock, flags);
1015 smp_mb__after_spinlock(); /* Timer expire before wakeup. */
1016 do_nocb_deferred_wakeup_common(rdp, rdp, RCU_NOCB_WAKE_BYPASS, flags);
1017 }
1018
1019 /*
1020 * Do a deferred wakeup of rcu_nocb_kthread() from fastpath.
1021 * This means we do an inexact common-case check. Note that if
1022 * we miss, ->nocb_timer will eventually clean things up.
1023 */
1024 static bool do_nocb_deferred_wakeup(struct rcu_data *rdp)
1025 {
1026 unsigned long flags;
1027 struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;
1028
1029 if (!rdp_gp || !rcu_nocb_need_deferred_wakeup(rdp_gp, RCU_NOCB_WAKE))
1030 return false;
1031
1032 raw_spin_lock_irqsave(&rdp_gp->nocb_gp_lock, flags);
1033 return do_nocb_deferred_wakeup_common(rdp_gp, rdp, RCU_NOCB_WAKE, flags);
1034 }
1035
1036 void rcu_nocb_flush_deferred_wakeup(void)
1037 {
1038 do_nocb_deferred_wakeup(this_cpu_ptr(&rcu_data));
1039 }
1040 EXPORT_SYMBOL_GPL(rcu_nocb_flush_deferred_wakeup);
1041
1042 static int rdp_offload_toggle(struct rcu_data *rdp,
1043 bool offload, unsigned long flags)
1044 __releases(rdp->nocb_lock)
1045 {
1046 struct rcu_segcblist *cblist = &rdp->cblist;
1047 struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;
1048 bool wake_gp = false;
1049
1050 rcu_segcblist_offload(cblist, offload);
1051 rcu_nocb_unlock_irqrestore(rdp, flags);
1052
1053 raw_spin_lock_irqsave(&rdp_gp->nocb_gp_lock, flags);
1054 // Queue this rdp for add/del to/from the list to iterate on rcuog
1055 WRITE_ONCE(rdp_gp->nocb_toggling_rdp, rdp);
1056 if (rdp_gp->nocb_gp_sleep) {
1057 rdp_gp->nocb_gp_sleep = false;
1058 wake_gp = true;
1059 }
1060 raw_spin_unlock_irqrestore(&rdp_gp->nocb_gp_lock, flags);
1061
1062 return wake_gp;
1063 }
1064
1065 static long rcu_nocb_rdp_deoffload(void *arg)
1066 {
1067 struct rcu_data *rdp = arg;
1068 struct rcu_segcblist *cblist = &rdp->cblist;
1069 unsigned long flags;
1070 int wake_gp;
1071 struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;
1072
1073 /*
1074 * rcu_nocb_rdp_deoffload() may be called directly if
1075 * rcuog/o[p] spawn failed, because at this time the rdp->cpu
1076 * is not online yet.
1077 */
1078 WARN_ON_ONCE((rdp->cpu != raw_smp_processor_id()) && cpu_online(rdp->cpu));
1079
1080 pr_info("De-offloading %d\n", rdp->cpu);
1081
1082 rcu_nocb_lock_irqsave(rdp, flags);
1083 /*
1084 * Flush once and for all now. This suffices because we are
1085 * running on the target CPU holding ->nocb_lock (thus having
1086 * interrupts disabled), and because rdp_offload_toggle()
1087 * invokes rcu_segcblist_offload(), which clears SEGCBLIST_OFFLOADED.
1088 * Thus future calls to rcu_segcblist_completely_offloaded() will
1089 * return false, which means that future calls to rcu_nocb_try_bypass()
1090 * will refuse to put anything into the bypass.
1091 */
1092 WARN_ON_ONCE(!rcu_nocb_flush_bypass(rdp, NULL, jiffies, false));
1093 /*
1094 * Start with invoking rcu_core() early. This way if the current thread
1095 * happens to preempt an ongoing call to rcu_core() in the middle,
1096 * leaving some work dismissed because rcu_core() still thinks the rdp is
1097 * completely offloaded, we are guaranteed a nearby future instance of
1098 * rcu_core() to catch up.
1099 */
1100 rcu_segcblist_set_flags(cblist, SEGCBLIST_RCU_CORE);
1101 invoke_rcu_core();
1102 wake_gp = rdp_offload_toggle(rdp, false, flags);
1103
1104 mutex_lock(&rdp_gp->nocb_gp_kthread_mutex);
1105 if (rdp_gp->nocb_gp_kthread) {
1106 if (wake_gp)
1107 wake_up_process(rdp_gp->nocb_gp_kthread);
1108
1109 swait_event_exclusive(rdp->nocb_state_wq,
1110 !rcu_segcblist_test_flags(cblist,
1111 SEGCBLIST_KTHREAD_GP));
1112 if (rdp->nocb_cb_kthread)
1113 kthread_park(rdp->nocb_cb_kthread);
1114 } else {
1115 /*
1116 * No kthread to clear the flags for us or remove the rdp from the nocb list
1117 * to iterate. Do it here instead. Locking doesn't look stricly necessary
1118 * but we stick to paranoia in this rare path.
1119 */
1120 rcu_nocb_lock_irqsave(rdp, flags);
1121 rcu_segcblist_clear_flags(&rdp->cblist, SEGCBLIST_KTHREAD_GP);
1122 rcu_nocb_unlock_irqrestore(rdp, flags);
1123
1124 list_del(&rdp->nocb_entry_rdp);
1125 }
1126 mutex_unlock(&rdp_gp->nocb_gp_kthread_mutex);
1127
1128 /*
1129 * Lock one last time to acquire latest callback updates from kthreads
1130 * so we can later handle callbacks locally without locking.
1131 */
1132 rcu_nocb_lock_irqsave(rdp, flags);
1133 /*
1134 * Theoretically we could clear SEGCBLIST_LOCKING after the nocb
1135 * lock is released but how about being paranoid for once?
1136 */
1137 rcu_segcblist_clear_flags(cblist, SEGCBLIST_LOCKING);
1138 /*
1139 * Without SEGCBLIST_LOCKING, we can't use
1140 * rcu_nocb_unlock_irqrestore() anymore.
1141 */
1142 raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags);
1143
1144 /* Sanity check */
1145 WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass));
1146
1147
1148 return 0;
1149 }
1150
1151 int rcu_nocb_cpu_deoffload(int cpu)
1152 {
1153 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
1154 int ret = 0;
1155
1156 cpus_read_lock();
1157 mutex_lock(&rcu_state.barrier_mutex);
1158 if (rcu_rdp_is_offloaded(rdp)) {
1159 if (cpu_online(cpu)) {
1160 ret = work_on_cpu(cpu, rcu_nocb_rdp_deoffload, rdp);
1161 if (!ret)
1162 cpumask_clear_cpu(cpu, rcu_nocb_mask);
1163 } else {
1164 pr_info("NOCB: Cannot CB-deoffload offline CPU %d\n", rdp->cpu);
1165 ret = -EINVAL;
1166 }
1167 }
1168 mutex_unlock(&rcu_state.barrier_mutex);
1169 cpus_read_unlock();
1170
1171 return ret;
1172 }
1173 EXPORT_SYMBOL_GPL(rcu_nocb_cpu_deoffload);
1174
1175 static long rcu_nocb_rdp_offload(void *arg)
1176 {
1177 struct rcu_data *rdp = arg;
1178 struct rcu_segcblist *cblist = &rdp->cblist;
1179 unsigned long flags;
1180 int wake_gp;
1181 struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;
1182
1183 WARN_ON_ONCE(rdp->cpu != raw_smp_processor_id());
1184 /*
1185 * For now we only support re-offload, ie: the rdp must have been
1186 * offloaded on boot first.
1187 */
1188 if (!rdp->nocb_gp_rdp)
1189 return -EINVAL;
1190
1191 if (WARN_ON_ONCE(!rdp_gp->nocb_gp_kthread))
1192 return -EINVAL;
1193
1194 pr_info("Offloading %d\n", rdp->cpu);
1195
1196 /*
1197 * Can't use rcu_nocb_lock_irqsave() before SEGCBLIST_LOCKING
1198 * is set.
1199 */
1200 raw_spin_lock_irqsave(&rdp->nocb_lock, flags);
1201
1202 /*
1203 * We didn't take the nocb lock while working on the
1204 * rdp->cblist with SEGCBLIST_LOCKING cleared (pure softirq/rcuc mode).
1205 * Every modifications that have been done previously on
1206 * rdp->cblist must be visible remotely by the nocb kthreads
1207 * upon wake up after reading the cblist flags.
1208 *
1209 * The layout against nocb_lock enforces that ordering:
1210 *
1211 * __rcu_nocb_rdp_offload() nocb_cb_wait()/nocb_gp_wait()
1212 * ------------------------- ----------------------------
1213 * WRITE callbacks rcu_nocb_lock()
1214 * rcu_nocb_lock() READ flags
1215 * WRITE flags READ callbacks
1216 * rcu_nocb_unlock() rcu_nocb_unlock()
1217 */
1218 wake_gp = rdp_offload_toggle(rdp, true, flags);
1219 if (wake_gp)
1220 wake_up_process(rdp_gp->nocb_gp_kthread);
1221
1222 kthread_unpark(rdp->nocb_cb_kthread);
1223
1224 swait_event_exclusive(rdp->nocb_state_wq,
1225 rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_GP));
1226
1227 /*
1228 * All kthreads are ready to work, we can finally relieve rcu_core() and
1229 * enable nocb bypass.
1230 */
1231 rcu_nocb_lock_irqsave(rdp, flags);
1232 rcu_segcblist_clear_flags(cblist, SEGCBLIST_RCU_CORE);
1233 rcu_nocb_unlock_irqrestore(rdp, flags);
1234
1235 return 0;
1236 }
1237
1238 int rcu_nocb_cpu_offload(int cpu)
1239 {
1240 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
1241 int ret = 0;
1242
1243 cpus_read_lock();
1244 mutex_lock(&rcu_state.barrier_mutex);
1245 if (!rcu_rdp_is_offloaded(rdp)) {
1246 if (cpu_online(cpu)) {
1247 ret = work_on_cpu(cpu, rcu_nocb_rdp_offload, rdp);
1248 if (!ret)
1249 cpumask_set_cpu(cpu, rcu_nocb_mask);
1250 } else {
1251 pr_info("NOCB: Cannot CB-offload offline CPU %d\n", rdp->cpu);
1252 ret = -EINVAL;
1253 }
1254 }
1255 mutex_unlock(&rcu_state.barrier_mutex);
1256 cpus_read_unlock();
1257
1258 return ret;
1259 }
1260 EXPORT_SYMBOL_GPL(rcu_nocb_cpu_offload);
1261
1262 #ifdef CONFIG_RCU_LAZY
1263 static unsigned long
1264 lazy_rcu_shrink_count(struct shrinker *shrink, struct shrink_control *sc)
1265 {
1266 int cpu;
1267 unsigned long count = 0;
1268
1269 if (WARN_ON_ONCE(!cpumask_available(rcu_nocb_mask)))
1270 return 0;
1271
1272 /* Protect rcu_nocb_mask against concurrent (de-)offloading. */
1273 if (!mutex_trylock(&rcu_state.barrier_mutex))
1274 return 0;
1275
1276 /* Snapshot count of all CPUs */
1277 for_each_cpu(cpu, rcu_nocb_mask) {
1278 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
1279
1280 count += READ_ONCE(rdp->lazy_len);
1281 }
1282
1283 mutex_unlock(&rcu_state.barrier_mutex);
1284
1285 return count ? count : SHRINK_EMPTY;
1286 }
1287
1288 static unsigned long
1289 lazy_rcu_shrink_scan(struct shrinker *shrink, struct shrink_control *sc)
1290 {
1291 int cpu;
1292 unsigned long flags;
1293 unsigned long count = 0;
1294
1295 if (WARN_ON_ONCE(!cpumask_available(rcu_nocb_mask)))
1296 return 0;
1297 /*
1298 * Protect against concurrent (de-)offloading. Otherwise nocb locking
1299 * may be ignored or imbalanced.
1300 */
1301 if (!mutex_trylock(&rcu_state.barrier_mutex)) {
1302 /*
1303 * But really don't insist if barrier_mutex is contended since we
1304 * can't guarantee that it will never engage in a dependency
1305 * chain involving memory allocation. The lock is seldom contended
1306 * anyway.
1307 */
1308 return 0;
1309 }
1310
1311 /* Snapshot count of all CPUs */
1312 for_each_cpu(cpu, rcu_nocb_mask) {
1313 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
1314 int _count;
1315
1316 if (WARN_ON_ONCE(!rcu_rdp_is_offloaded(rdp)))
1317 continue;
1318
1319 if (!READ_ONCE(rdp->lazy_len))
1320 continue;
1321
1322 rcu_nocb_lock_irqsave(rdp, flags);
1323 /*
1324 * Recheck under the nocb lock. Since we are not holding the bypass
1325 * lock we may still race with increments from the enqueuer but still
1326 * we know for sure if there is at least one lazy callback.
1327 */
1328 _count = READ_ONCE(rdp->lazy_len);
1329 if (!_count) {
1330 rcu_nocb_unlock_irqrestore(rdp, flags);
1331 continue;
1332 }
1333 rcu_nocb_try_flush_bypass(rdp, jiffies);
1334 rcu_nocb_unlock_irqrestore(rdp, flags);
1335 wake_nocb_gp(rdp, false);
1336 sc->nr_to_scan -= _count;
1337 count += _count;
1338 if (sc->nr_to_scan <= 0)
1339 break;
1340 }
1341
1342 mutex_unlock(&rcu_state.barrier_mutex);
1343
1344 return count ? count : SHRINK_STOP;
1345 }
1346 #endif // #ifdef CONFIG_RCU_LAZY
1347
1348 void __init rcu_init_nohz(void)
1349 {
1350 int cpu;
1351 struct rcu_data *rdp;
1352 const struct cpumask *cpumask = NULL;
1353 struct shrinker * __maybe_unused lazy_rcu_shrinker;
1354
1355 #if defined(CONFIG_NO_HZ_FULL)
1356 if (tick_nohz_full_running && !cpumask_empty(tick_nohz_full_mask))
1357 cpumask = tick_nohz_full_mask;
1358 #endif
1359
1360 if (IS_ENABLED(CONFIG_RCU_NOCB_CPU_DEFAULT_ALL) &&
1361 !rcu_state.nocb_is_setup && !cpumask)
1362 cpumask = cpu_possible_mask;
1363
1364 if (cpumask) {
1365 if (!cpumask_available(rcu_nocb_mask)) {
1366 if (!zalloc_cpumask_var(&rcu_nocb_mask, GFP_KERNEL)) {
1367 pr_info("rcu_nocb_mask allocation failed, callback offloading disabled.\n");
1368 return;
1369 }
1370 }
1371
1372 cpumask_or(rcu_nocb_mask, rcu_nocb_mask, cpumask);
1373 rcu_state.nocb_is_setup = true;
1374 }
1375
1376 if (!rcu_state.nocb_is_setup)
1377 return;
1378
1379 #ifdef CONFIG_RCU_LAZY
1380 lazy_rcu_shrinker = shrinker_alloc(0, "rcu-lazy");
1381 if (!lazy_rcu_shrinker) {
1382 pr_err("Failed to allocate lazy_rcu shrinker!\n");
1383 } else {
1384 lazy_rcu_shrinker->count_objects = lazy_rcu_shrink_count;
1385 lazy_rcu_shrinker->scan_objects = lazy_rcu_shrink_scan;
1386
1387 shrinker_register(lazy_rcu_shrinker);
1388 }
1389 #endif // #ifdef CONFIG_RCU_LAZY
1390
1391 if (!cpumask_subset(rcu_nocb_mask, cpu_possible_mask)) {
1392 pr_info("\tNote: kernel parameter 'rcu_nocbs=', 'nohz_full', or 'isolcpus=' contains nonexistent CPUs.\n");
1393 cpumask_and(rcu_nocb_mask, cpu_possible_mask,
1394 rcu_nocb_mask);
1395 }
1396 if (cpumask_empty(rcu_nocb_mask))
1397 pr_info("\tOffload RCU callbacks from CPUs: (none).\n");
1398 else
1399 pr_info("\tOffload RCU callbacks from CPUs: %*pbl.\n",
1400 cpumask_pr_args(rcu_nocb_mask));
1401 if (rcu_nocb_poll)
1402 pr_info("\tPoll for callbacks from no-CBs CPUs.\n");
1403
1404 for_each_cpu(cpu, rcu_nocb_mask) {
1405 rdp = per_cpu_ptr(&rcu_data, cpu);
1406 if (rcu_segcblist_empty(&rdp->cblist))
1407 rcu_segcblist_init(&rdp->cblist);
1408 rcu_segcblist_offload(&rdp->cblist, true);
1409 rcu_segcblist_set_flags(&rdp->cblist, SEGCBLIST_KTHREAD_GP);
1410 rcu_segcblist_clear_flags(&rdp->cblist, SEGCBLIST_RCU_CORE);
1411 }
1412 rcu_organize_nocb_kthreads();
1413 }
1414
1415 /* Initialize per-rcu_data variables for no-CBs CPUs. */
1416 static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)
1417 {
1418 init_swait_queue_head(&rdp->nocb_cb_wq);
1419 init_swait_queue_head(&rdp->nocb_gp_wq);
1420 init_swait_queue_head(&rdp->nocb_state_wq);
1421 raw_spin_lock_init(&rdp->nocb_lock);
1422 raw_spin_lock_init(&rdp->nocb_bypass_lock);
1423 raw_spin_lock_init(&rdp->nocb_gp_lock);
1424 timer_setup(&rdp->nocb_timer, do_nocb_deferred_wakeup_timer, 0);
1425 rcu_cblist_init(&rdp->nocb_bypass);
1426 WRITE_ONCE(rdp->lazy_len, 0);
1427 mutex_init(&rdp->nocb_gp_kthread_mutex);
1428 }
1429
1430 /*
1431 * If the specified CPU is a no-CBs CPU that does not already have its
1432 * rcuo CB kthread, spawn it. Additionally, if the rcuo GP kthread
1433 * for this CPU's group has not yet been created, spawn it as well.
1434 */
1435 static void rcu_spawn_cpu_nocb_kthread(int cpu)
1436 {
1437 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
1438 struct rcu_data *rdp_gp;
1439 struct task_struct *t;
1440 struct sched_param sp;
1441
1442 if (!rcu_scheduler_fully_active || !rcu_state.nocb_is_setup)
1443 return;
1444
1445 /* If there already is an rcuo kthread, then nothing to do. */
1446 if (rdp->nocb_cb_kthread)
1447 return;
1448
1449 /* If we didn't spawn the GP kthread first, reorganize! */
1450 sp.sched_priority = kthread_prio;
1451 rdp_gp = rdp->nocb_gp_rdp;
1452 mutex_lock(&rdp_gp->nocb_gp_kthread_mutex);
1453 if (!rdp_gp->nocb_gp_kthread) {
1454 t = kthread_run(rcu_nocb_gp_kthread, rdp_gp,
1455 "rcuog/%d", rdp_gp->cpu);
1456 if (WARN_ONCE(IS_ERR(t), "%s: Could not start rcuo GP kthread, OOM is now expected behavior\n", __func__)) {
1457 mutex_unlock(&rdp_gp->nocb_gp_kthread_mutex);
1458 goto end;
1459 }
1460 WRITE_ONCE(rdp_gp->nocb_gp_kthread, t);
1461 if (kthread_prio)
1462 sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
1463 }
1464 mutex_unlock(&rdp_gp->nocb_gp_kthread_mutex);
1465
1466 /* Spawn the kthread for this CPU. */
1467 t = kthread_create(rcu_nocb_cb_kthread, rdp,
1468 "rcuo%c/%d", rcu_state.abbr, cpu);
1469 if (WARN_ONCE(IS_ERR(t), "%s: Could not start rcuo CB kthread, OOM is now expected behavior\n", __func__))
1470 goto end;
1471
1472 if (rcu_rdp_is_offloaded(rdp))
1473 wake_up_process(t);
1474 else
1475 kthread_park(t);
1476
1477 if (IS_ENABLED(CONFIG_RCU_NOCB_CPU_CB_BOOST) && kthread_prio)
1478 sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
1479
1480 WRITE_ONCE(rdp->nocb_cb_kthread, t);
1481 WRITE_ONCE(rdp->nocb_gp_kthread, rdp_gp->nocb_gp_kthread);
1482 return;
1483 end:
1484 mutex_lock(&rcu_state.barrier_mutex);
1485 if (rcu_rdp_is_offloaded(rdp)) {
1486 rcu_nocb_rdp_deoffload(rdp);
1487 cpumask_clear_cpu(cpu, rcu_nocb_mask);
1488 }
1489 mutex_unlock(&rcu_state.barrier_mutex);
1490 }
1491
1492 /* How many CB CPU IDs per GP kthread? Default of -1 for sqrt(nr_cpu_ids). */
1493 static int rcu_nocb_gp_stride = -1;
1494 module_param(rcu_nocb_gp_stride, int, 0444);
1495
1496 /*
1497 * Initialize GP-CB relationships for all no-CBs CPU.
1498 */
1499 static void __init rcu_organize_nocb_kthreads(void)
1500 {
1501 int cpu;
1502 bool firsttime = true;
1503 bool gotnocbs = false;
1504 bool gotnocbscbs = true;
1505 int ls = rcu_nocb_gp_stride;
1506 int nl = 0; /* Next GP kthread. */
1507 struct rcu_data *rdp;
1508 struct rcu_data *rdp_gp = NULL; /* Suppress misguided gcc warn. */
1509
1510 if (!cpumask_available(rcu_nocb_mask))
1511 return;
1512 if (ls == -1) {
1513 ls = nr_cpu_ids / int_sqrt(nr_cpu_ids);
1514 rcu_nocb_gp_stride = ls;
1515 }
1516
1517 /*
1518 * Each pass through this loop sets up one rcu_data structure.
1519 * Should the corresponding CPU come online in the future, then
1520 * we will spawn the needed set of rcu_nocb_kthread() kthreads.
1521 */
1522 for_each_possible_cpu(cpu) {
1523 rdp = per_cpu_ptr(&rcu_data, cpu);
1524 if (rdp->cpu >= nl) {
1525 /* New GP kthread, set up for CBs & next GP. */
1526 gotnocbs = true;
1527 nl = DIV_ROUND_UP(rdp->cpu + 1, ls) * ls;
1528 rdp_gp = rdp;
1529 INIT_LIST_HEAD(&rdp->nocb_head_rdp);
1530 if (dump_tree) {
1531 if (!firsttime)
1532 pr_cont("%s\n", gotnocbscbs
1533 ? "" : " (self only)");
1534 gotnocbscbs = false;
1535 firsttime = false;
1536 pr_alert("%s: No-CB GP kthread CPU %d:",
1537 __func__, cpu);
1538 }
1539 } else {
1540 /* Another CB kthread, link to previous GP kthread. */
1541 gotnocbscbs = true;
1542 if (dump_tree)
1543 pr_cont(" %d", cpu);
1544 }
1545 rdp->nocb_gp_rdp = rdp_gp;
1546 if (cpumask_test_cpu(cpu, rcu_nocb_mask))
1547 list_add_tail(&rdp->nocb_entry_rdp, &rdp_gp->nocb_head_rdp);
1548 }
1549 if (gotnocbs && dump_tree)
1550 pr_cont("%s\n", gotnocbscbs ? "" : " (self only)");
1551 }
1552
1553 /*
1554 * Bind the current task to the offloaded CPUs. If there are no offloaded
1555 * CPUs, leave the task unbound. Splat if the bind attempt fails.
1556 */
1557 void rcu_bind_current_to_nocb(void)
1558 {
1559 if (cpumask_available(rcu_nocb_mask) && !cpumask_empty(rcu_nocb_mask))
1560 WARN_ON(sched_setaffinity(current->pid, rcu_nocb_mask));
1561 }
1562 EXPORT_SYMBOL_GPL(rcu_bind_current_to_nocb);
1563
1564 // The ->on_cpu field is available only in CONFIG_SMP=y, so...
1565 #ifdef CONFIG_SMP
1566 static char *show_rcu_should_be_on_cpu(struct task_struct *tsp)
1567 {
1568 return tsp && task_is_running(tsp) && !tsp->on_cpu ? "!" : "";
1569 }
1570 #else // #ifdef CONFIG_SMP
1571 static char *show_rcu_should_be_on_cpu(struct task_struct *tsp)
1572 {
1573 return "";
1574 }
1575 #endif // #else #ifdef CONFIG_SMP
1576
1577 /*
1578 * Dump out nocb grace-period kthread state for the specified rcu_data
1579 * structure.
1580 */
1581 static void show_rcu_nocb_gp_state(struct rcu_data *rdp)
1582 {
1583 struct rcu_node *rnp = rdp->mynode;
1584
1585 pr_info("nocb GP %d %c%c%c%c%c %c[%c%c] %c%c:%ld rnp %d:%d %lu %c CPU %d%s\n",
1586 rdp->cpu,
1587 "kK"[!!rdp->nocb_gp_kthread],
1588 "lL"[raw_spin_is_locked(&rdp->nocb_gp_lock)],
1589 "dD"[!!rdp->nocb_defer_wakeup],
1590 "tT"[timer_pending(&rdp->nocb_timer)],
1591 "sS"[!!rdp->nocb_gp_sleep],
1592 ".W"[swait_active(&rdp->nocb_gp_wq)],
1593 ".W"[swait_active(&rnp->nocb_gp_wq[0])],
1594 ".W"[swait_active(&rnp->nocb_gp_wq[1])],
1595 ".B"[!!rdp->nocb_gp_bypass],
1596 ".G"[!!rdp->nocb_gp_gp],
1597 (long)rdp->nocb_gp_seq,
1598 rnp->grplo, rnp->grphi, READ_ONCE(rdp->nocb_gp_loops),
1599 rdp->nocb_gp_kthread ? task_state_to_char(rdp->nocb_gp_kthread) : '.',
1600 rdp->nocb_gp_kthread ? (int)task_cpu(rdp->nocb_gp_kthread) : -1,
1601 show_rcu_should_be_on_cpu(rdp->nocb_gp_kthread));
1602 }
1603
1604 /* Dump out nocb kthread state for the specified rcu_data structure. */
1605 static void show_rcu_nocb_state(struct rcu_data *rdp)
1606 {
1607 char bufw[20];
1608 char bufr[20];
1609 struct rcu_data *nocb_next_rdp;
1610 struct rcu_segcblist *rsclp = &rdp->cblist;
1611 bool waslocked;
1612 bool wassleep;
1613
1614 if (rdp->nocb_gp_rdp == rdp)
1615 show_rcu_nocb_gp_state(rdp);
1616
1617 nocb_next_rdp = list_next_or_null_rcu(&rdp->nocb_gp_rdp->nocb_head_rdp,
1618 &rdp->nocb_entry_rdp,
1619 typeof(*rdp),
1620 nocb_entry_rdp);
1621
1622 sprintf(bufw, "%ld", rsclp->gp_seq[RCU_WAIT_TAIL]);
1623 sprintf(bufr, "%ld", rsclp->gp_seq[RCU_NEXT_READY_TAIL]);
1624 pr_info(" CB %d^%d->%d %c%c%c%c%c F%ld L%ld C%d %c%c%s%c%s%c%c q%ld %c CPU %d%s\n",
1625 rdp->cpu, rdp->nocb_gp_rdp->cpu,
1626 nocb_next_rdp ? nocb_next_rdp->cpu : -1,
1627 "kK"[!!rdp->nocb_cb_kthread],
1628 "bB"[raw_spin_is_locked(&rdp->nocb_bypass_lock)],
1629 "lL"[raw_spin_is_locked(&rdp->nocb_lock)],
1630 "sS"[!!rdp->nocb_cb_sleep],
1631 ".W"[swait_active(&rdp->nocb_cb_wq)],
1632 jiffies - rdp->nocb_bypass_first,
1633 jiffies - rdp->nocb_nobypass_last,
1634 rdp->nocb_nobypass_count,
1635 ".D"[rcu_segcblist_ready_cbs(rsclp)],
1636 ".W"[!rcu_segcblist_segempty(rsclp, RCU_WAIT_TAIL)],
1637 rcu_segcblist_segempty(rsclp, RCU_WAIT_TAIL) ? "" : bufw,
1638 ".R"[!rcu_segcblist_segempty(rsclp, RCU_NEXT_READY_TAIL)],
1639 rcu_segcblist_segempty(rsclp, RCU_NEXT_READY_TAIL) ? "" : bufr,
1640 ".N"[!rcu_segcblist_segempty(rsclp, RCU_NEXT_TAIL)],
1641 ".B"[!!rcu_cblist_n_cbs(&rdp->nocb_bypass)],
1642 rcu_segcblist_n_cbs(&rdp->cblist),
1643 rdp->nocb_cb_kthread ? task_state_to_char(rdp->nocb_cb_kthread) : '.',
1644 rdp->nocb_cb_kthread ? (int)task_cpu(rdp->nocb_cb_kthread) : -1,
1645 show_rcu_should_be_on_cpu(rdp->nocb_cb_kthread));
1646
1647 /* It is OK for GP kthreads to have GP state. */
1648 if (rdp->nocb_gp_rdp == rdp)
1649 return;
1650
1651 waslocked = raw_spin_is_locked(&rdp->nocb_gp_lock);
1652 wassleep = swait_active(&rdp->nocb_gp_wq);
1653 if (!rdp->nocb_gp_sleep && !waslocked && !wassleep)
1654 return; /* Nothing untoward. */
1655
1656 pr_info(" nocb GP activity on CB-only CPU!!! %c%c%c %c\n",
1657 "lL"[waslocked],
1658 "dD"[!!rdp->nocb_defer_wakeup],
1659 "sS"[!!rdp->nocb_gp_sleep],
1660 ".W"[wassleep]);
1661 }
1662
1663 #else /* #ifdef CONFIG_RCU_NOCB_CPU */
1664
1665 static inline int rcu_lockdep_is_held_nocb(struct rcu_data *rdp)
1666 {
1667 return 0;
1668 }
1669
1670 static inline bool rcu_current_is_nocb_kthread(struct rcu_data *rdp)
1671 {
1672 return false;
1673 }
1674
1675 /* No ->nocb_lock to acquire. */
1676 static void rcu_nocb_lock(struct rcu_data *rdp)
1677 {
1678 }
1679
1680 /* No ->nocb_lock to release. */
1681 static void rcu_nocb_unlock(struct rcu_data *rdp)
1682 {
1683 }
1684
1685 /* No ->nocb_lock to release. */
1686 static void rcu_nocb_unlock_irqrestore(struct rcu_data *rdp,
1687 unsigned long flags)
1688 {
1689 local_irq_restore(flags);
1690 }
1691
1692 /* Lockdep check that ->cblist may be safely accessed. */
1693 static void rcu_lockdep_assert_cblist_protected(struct rcu_data *rdp)
1694 {
1695 lockdep_assert_irqs_disabled();
1696 }
1697
1698 static void rcu_nocb_gp_cleanup(struct swait_queue_head *sq)
1699 {
1700 }
1701
1702 static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp)
1703 {
1704 return NULL;
1705 }
1706
1707 static void rcu_init_one_nocb(struct rcu_node *rnp)
1708 {
1709 }
1710
1711 static bool wake_nocb_gp(struct rcu_data *rdp, bool force)
1712 {
1713 return false;
1714 }
1715
1716 static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp,
1717 unsigned long j, bool lazy)
1718 {
1719 return true;
1720 }
1721
1722 static void call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *head,
1723 rcu_callback_t func, unsigned long flags, bool lazy)
1724 {
1725 WARN_ON_ONCE(1); /* Should be dead code! */
1726 }
1727
1728 static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_empty,
1729 unsigned long flags)
1730 {
1731 WARN_ON_ONCE(1); /* Should be dead code! */
1732 }
1733
1734 static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)
1735 {
1736 }
1737
1738 static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp, int level)
1739 {
1740 return false;
1741 }
1742
1743 static bool do_nocb_deferred_wakeup(struct rcu_data *rdp)
1744 {
1745 return false;
1746 }
1747
1748 static void rcu_spawn_cpu_nocb_kthread(int cpu)
1749 {
1750 }
1751
1752 static void show_rcu_nocb_state(struct rcu_data *rdp)
1753 {
1754 }
1755
1756 #endif /* #else #ifdef CONFIG_RCU_NOCB_CPU */
1757
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.