1 // SPDX-License-Identifier: GPL-2.0+ 1 // SPDX-License-Identifier: GPL-2.0+
2 /* 2 /*
3 * Read-Copy Update mechanism for mutual exclu 3 * Read-Copy Update mechanism for mutual exclusion
4 * 4 *
5 * Copyright IBM Corporation, 2001 5 * Copyright IBM Corporation, 2001
6 * 6 *
7 * Authors: Dipankar Sarma <dipankar@in.ibm.co 7 * Authors: Dipankar Sarma <dipankar@in.ibm.com>
8 * Manfred Spraul <manfred@colorfulli 8 * Manfred Spraul <manfred@colorfullife.com>
9 * 9 *
10 * Based on the original work by Paul McKenney 10 * Based on the original work by Paul McKenney <paulmck@linux.ibm.com>
11 * and inputs from Rusty Russell, Andrea Arcan 11 * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen.
12 * Papers: 12 * Papers:
13 * http://www.rdrop.com/users/paulmck/paper/rc 13 * http://www.rdrop.com/users/paulmck/paper/rclockpdcsproof.pdf
14 * http://lse.sourceforge.net/locking/rclock_O 14 * http://lse.sourceforge.net/locking/rclock_OLS.2001.05.01c.sc.pdf (OLS2001)
15 * 15 *
16 * For detailed explanation of Read-Copy Updat 16 * For detailed explanation of Read-Copy Update mechanism see -
17 * http://lse.sourceforge.net/loc 17 * http://lse.sourceforge.net/locking/rcupdate.html
18 * 18 *
19 */ 19 */
20 #include <linux/types.h> 20 #include <linux/types.h>
21 #include <linux/kernel.h> 21 #include <linux/kernel.h>
22 #include <linux/init.h> 22 #include <linux/init.h>
23 #include <linux/spinlock.h> 23 #include <linux/spinlock.h>
24 #include <linux/smp.h> 24 #include <linux/smp.h>
25 #include <linux/interrupt.h> 25 #include <linux/interrupt.h>
26 #include <linux/sched/signal.h> 26 #include <linux/sched/signal.h>
27 #include <linux/sched/debug.h> 27 #include <linux/sched/debug.h>
28 #include <linux/torture.h> <<
29 #include <linux/atomic.h> 28 #include <linux/atomic.h>
30 #include <linux/bitops.h> 29 #include <linux/bitops.h>
31 #include <linux/percpu.h> 30 #include <linux/percpu.h>
32 #include <linux/notifier.h> 31 #include <linux/notifier.h>
33 #include <linux/cpu.h> 32 #include <linux/cpu.h>
34 #include <linux/mutex.h> 33 #include <linux/mutex.h>
35 #include <linux/export.h> 34 #include <linux/export.h>
36 #include <linux/hardirq.h> 35 #include <linux/hardirq.h>
37 #include <linux/delay.h> 36 #include <linux/delay.h>
38 #include <linux/moduleparam.h> 37 #include <linux/moduleparam.h>
39 #include <linux/kthread.h> 38 #include <linux/kthread.h>
40 #include <linux/tick.h> 39 #include <linux/tick.h>
41 #include <linux/rcupdate_wait.h> 40 #include <linux/rcupdate_wait.h>
42 #include <linux/sched/isolation.h> 41 #include <linux/sched/isolation.h>
43 #include <linux/kprobes.h> 42 #include <linux/kprobes.h>
44 #include <linux/slab.h> 43 #include <linux/slab.h>
45 #include <linux/irq_work.h> 44 #include <linux/irq_work.h>
46 #include <linux/rcupdate_trace.h> 45 #include <linux/rcupdate_trace.h>
47 46
48 #define CREATE_TRACE_POINTS 47 #define CREATE_TRACE_POINTS
49 48
50 #include "rcu.h" 49 #include "rcu.h"
51 50
52 #ifdef MODULE_PARAM_PREFIX 51 #ifdef MODULE_PARAM_PREFIX
53 #undef MODULE_PARAM_PREFIX 52 #undef MODULE_PARAM_PREFIX
54 #endif 53 #endif
55 #define MODULE_PARAM_PREFIX "rcupdate." 54 #define MODULE_PARAM_PREFIX "rcupdate."
56 55
57 #ifndef CONFIG_TINY_RCU 56 #ifndef CONFIG_TINY_RCU
58 module_param(rcu_expedited, int, 0444); !! 57 module_param(rcu_expedited, int, 0);
59 module_param(rcu_normal, int, 0444); !! 58 module_param(rcu_normal, int, 0);
60 static int rcu_normal_after_boot = IS_ENABLED( 59 static int rcu_normal_after_boot = IS_ENABLED(CONFIG_PREEMPT_RT);
61 #if !defined(CONFIG_PREEMPT_RT) || defined(CON !! 60 #ifndef CONFIG_PREEMPT_RT
62 module_param(rcu_normal_after_boot, int, 0444) !! 61 module_param(rcu_normal_after_boot, int, 0);
63 #endif 62 #endif
64 #endif /* #ifndef CONFIG_TINY_RCU */ 63 #endif /* #ifndef CONFIG_TINY_RCU */
65 64
66 #ifdef CONFIG_DEBUG_LOCK_ALLOC 65 #ifdef CONFIG_DEBUG_LOCK_ALLOC
67 /** 66 /**
68 * rcu_read_lock_held_common() - might we be i 67 * rcu_read_lock_held_common() - might we be in RCU-sched read-side critical section?
69 * @ret: Best guess answer if lockdep c 68 * @ret: Best guess answer if lockdep cannot be relied on
70 * 69 *
71 * Returns true if lockdep must be ignored, in 70 * Returns true if lockdep must be ignored, in which case ``*ret`` contains
72 * the best guess described below. Otherwise 71 * the best guess described below. Otherwise returns false, in which
73 * case ``*ret`` tells the caller nothing and 72 * case ``*ret`` tells the caller nothing and the caller should instead
74 * consult lockdep. 73 * consult lockdep.
75 * 74 *
76 * If CONFIG_DEBUG_LOCK_ALLOC is selected, set 75 * If CONFIG_DEBUG_LOCK_ALLOC is selected, set ``*ret`` to nonzero iff in an
77 * RCU-sched read-side critical section. In a 76 * RCU-sched read-side critical section. In absence of
78 * CONFIG_DEBUG_LOCK_ALLOC, this assumes we ar 77 * CONFIG_DEBUG_LOCK_ALLOC, this assumes we are in an RCU-sched read-side
79 * critical section unless it can prove otherw 78 * critical section unless it can prove otherwise. Note that disabling
80 * of preemption (including disabling irqs) co 79 * of preemption (including disabling irqs) counts as an RCU-sched
81 * read-side critical section. This is useful 80 * read-side critical section. This is useful for debug checks in functions
82 * that required that they be called within an 81 * that required that they be called within an RCU-sched read-side
83 * critical section. 82 * critical section.
84 * 83 *
85 * Check debug_lockdep_rcu_enabled() to preven 84 * Check debug_lockdep_rcu_enabled() to prevent false positives during boot
86 * and while lockdep is disabled. 85 * and while lockdep is disabled.
87 * 86 *
88 * Note that if the CPU is in the idle loop fr 87 * Note that if the CPU is in the idle loop from an RCU point of view (ie:
89 * that we are in the section between ct_idle_ !! 88 * that we are in the section between rcu_idle_enter() and rcu_idle_exit())
90 * then rcu_read_lock_held() sets ``*ret`` to 89 * then rcu_read_lock_held() sets ``*ret`` to false even if the CPU did an
91 * rcu_read_lock(). The reason for this is th 90 * rcu_read_lock(). The reason for this is that RCU ignores CPUs that are
92 * in such a section, considering these as in 91 * in such a section, considering these as in extended quiescent state,
93 * so such a CPU is effectively never in an RC 92 * so such a CPU is effectively never in an RCU read-side critical section
94 * regardless of what RCU primitives it invoke 93 * regardless of what RCU primitives it invokes. This state of affairs is
95 * required --- we need to keep an RCU-free wi 94 * required --- we need to keep an RCU-free window in idle where the CPU may
96 * possibly enter into low power mode. This wa 95 * possibly enter into low power mode. This way we can notice an extended
97 * quiescent state to other CPUs that started 96 * quiescent state to other CPUs that started a grace period. Otherwise
98 * we would delay any grace period as long as 97 * we would delay any grace period as long as we run in the idle task.
99 * 98 *
100 * Similarly, we avoid claiming an RCU read lo 99 * Similarly, we avoid claiming an RCU read lock held if the current
101 * CPU is offline. 100 * CPU is offline.
102 */ 101 */
103 static bool rcu_read_lock_held_common(bool *re 102 static bool rcu_read_lock_held_common(bool *ret)
104 { 103 {
105 if (!debug_lockdep_rcu_enabled()) { 104 if (!debug_lockdep_rcu_enabled()) {
106 *ret = true; 105 *ret = true;
107 return true; 106 return true;
108 } 107 }
109 if (!rcu_is_watching()) { 108 if (!rcu_is_watching()) {
110 *ret = false; 109 *ret = false;
111 return true; 110 return true;
112 } 111 }
113 if (!rcu_lockdep_current_cpu_online()) 112 if (!rcu_lockdep_current_cpu_online()) {
114 *ret = false; 113 *ret = false;
115 return true; 114 return true;
116 } 115 }
117 return false; 116 return false;
118 } 117 }
119 118
120 int rcu_read_lock_sched_held(void) 119 int rcu_read_lock_sched_held(void)
121 { 120 {
122 bool ret; 121 bool ret;
123 122
124 if (rcu_read_lock_held_common(&ret)) 123 if (rcu_read_lock_held_common(&ret))
125 return ret; 124 return ret;
126 return lock_is_held(&rcu_sched_lock_ma 125 return lock_is_held(&rcu_sched_lock_map) || !preemptible();
127 } 126 }
128 EXPORT_SYMBOL(rcu_read_lock_sched_held); 127 EXPORT_SYMBOL(rcu_read_lock_sched_held);
129 #endif 128 #endif
130 129
131 #ifndef CONFIG_TINY_RCU 130 #ifndef CONFIG_TINY_RCU
132 131
133 /* 132 /*
134 * Should expedited grace-period primitives al 133 * Should expedited grace-period primitives always fall back to their
135 * non-expedited counterparts? Intended for u 134 * non-expedited counterparts? Intended for use within RCU. Note
136 * that if the user specifies both rcu_expedit 135 * that if the user specifies both rcu_expedited and rcu_normal, then
137 * rcu_normal wins. (Except during the time p 136 * rcu_normal wins. (Except during the time period during boot from
138 * when the first task is spawned until the rc 137 * when the first task is spawned until the rcu_set_runtime_mode()
139 * core_initcall() is invoked, at which point 138 * core_initcall() is invoked, at which point everything is expedited.)
140 */ 139 */
141 bool rcu_gp_is_normal(void) 140 bool rcu_gp_is_normal(void)
142 { 141 {
143 return READ_ONCE(rcu_normal) && 142 return READ_ONCE(rcu_normal) &&
144 rcu_scheduler_active != RCU_SCH 143 rcu_scheduler_active != RCU_SCHEDULER_INIT;
145 } 144 }
146 EXPORT_SYMBOL_GPL(rcu_gp_is_normal); 145 EXPORT_SYMBOL_GPL(rcu_gp_is_normal);
147 146
148 static atomic_t rcu_async_hurry_nesting = ATOM <<
149 /* <<
150 * Should call_rcu() callbacks be processed wi <<
151 * they OK being executed with arbitrary delay <<
152 */ <<
153 bool rcu_async_should_hurry(void) <<
154 { <<
155 return !IS_ENABLED(CONFIG_RCU_LAZY) || <<
156 atomic_read(&rcu_async_hurry_ne <<
157 } <<
158 EXPORT_SYMBOL_GPL(rcu_async_should_hurry); <<
159 <<
160 /** <<
161 * rcu_async_hurry - Make future async RCU cal <<
162 * <<
163 * After a call to this function, future calls <<
164 * will be processed in a timely fashion. <<
165 */ <<
166 void rcu_async_hurry(void) <<
167 { <<
168 if (IS_ENABLED(CONFIG_RCU_LAZY)) <<
169 atomic_inc(&rcu_async_hurry_ne <<
170 } <<
171 EXPORT_SYMBOL_GPL(rcu_async_hurry); <<
172 <<
173 /** <<
174 * rcu_async_relax - Make future async RCU cal <<
175 * <<
176 * After a call to this function, future calls <<
177 * will be processed in a lazy fashion. <<
178 */ <<
179 void rcu_async_relax(void) <<
180 { <<
181 if (IS_ENABLED(CONFIG_RCU_LAZY)) <<
182 atomic_dec(&rcu_async_hurry_ne <<
183 } <<
184 EXPORT_SYMBOL_GPL(rcu_async_relax); <<
185 <<
186 static atomic_t rcu_expedited_nesting = ATOMIC 147 static atomic_t rcu_expedited_nesting = ATOMIC_INIT(1);
>> 148
187 /* 149 /*
188 * Should normal grace-period primitives be ex 150 * Should normal grace-period primitives be expedited? Intended for
189 * use within RCU. Note that this function ta 151 * use within RCU. Note that this function takes the rcu_expedited
190 * sysfs/boot variable and rcu_scheduler_activ 152 * sysfs/boot variable and rcu_scheduler_active into account as well
191 * as the rcu_expedite_gp() nesting. So loopi 153 * as the rcu_expedite_gp() nesting. So looping on rcu_unexpedite_gp()
192 * until rcu_gp_is_expedited() returns false i 154 * until rcu_gp_is_expedited() returns false is a -really- bad idea.
193 */ 155 */
194 bool rcu_gp_is_expedited(void) 156 bool rcu_gp_is_expedited(void)
195 { 157 {
196 return rcu_expedited || atomic_read(&r 158 return rcu_expedited || atomic_read(&rcu_expedited_nesting);
197 } 159 }
198 EXPORT_SYMBOL_GPL(rcu_gp_is_expedited); 160 EXPORT_SYMBOL_GPL(rcu_gp_is_expedited);
199 161
200 /** 162 /**
201 * rcu_expedite_gp - Expedite future RCU grace 163 * rcu_expedite_gp - Expedite future RCU grace periods
202 * 164 *
203 * After a call to this function, future calls 165 * After a call to this function, future calls to synchronize_rcu() and
204 * friends act as the corresponding synchroniz 166 * friends act as the corresponding synchronize_rcu_expedited() function
205 * had instead been called. 167 * had instead been called.
206 */ 168 */
207 void rcu_expedite_gp(void) 169 void rcu_expedite_gp(void)
208 { 170 {
209 atomic_inc(&rcu_expedited_nesting); 171 atomic_inc(&rcu_expedited_nesting);
210 } 172 }
211 EXPORT_SYMBOL_GPL(rcu_expedite_gp); 173 EXPORT_SYMBOL_GPL(rcu_expedite_gp);
212 174
213 /** 175 /**
214 * rcu_unexpedite_gp - Cancel prior rcu_expedi 176 * rcu_unexpedite_gp - Cancel prior rcu_expedite_gp() invocation
215 * 177 *
216 * Undo a prior call to rcu_expedite_gp(). If 178 * Undo a prior call to rcu_expedite_gp(). If all prior calls to
217 * rcu_expedite_gp() are undone by a subsequen 179 * rcu_expedite_gp() are undone by a subsequent call to rcu_unexpedite_gp(),
218 * and if the rcu_expedited sysfs/boot paramet 180 * and if the rcu_expedited sysfs/boot parameter is not set, then all
219 * subsequent calls to synchronize_rcu() and f 181 * subsequent calls to synchronize_rcu() and friends will return to
220 * their normal non-expedited behavior. 182 * their normal non-expedited behavior.
221 */ 183 */
222 void rcu_unexpedite_gp(void) 184 void rcu_unexpedite_gp(void)
223 { 185 {
224 atomic_dec(&rcu_expedited_nesting); 186 atomic_dec(&rcu_expedited_nesting);
225 } 187 }
226 EXPORT_SYMBOL_GPL(rcu_unexpedite_gp); 188 EXPORT_SYMBOL_GPL(rcu_unexpedite_gp);
227 189
228 static bool rcu_boot_ended __read_mostly; 190 static bool rcu_boot_ended __read_mostly;
229 191
230 /* 192 /*
231 * Inform RCU of the end of the in-kernel boot 193 * Inform RCU of the end of the in-kernel boot sequence.
232 */ 194 */
233 void rcu_end_inkernel_boot(void) 195 void rcu_end_inkernel_boot(void)
234 { 196 {
235 rcu_unexpedite_gp(); 197 rcu_unexpedite_gp();
236 rcu_async_relax(); <<
237 if (rcu_normal_after_boot) 198 if (rcu_normal_after_boot)
238 WRITE_ONCE(rcu_normal, 1); 199 WRITE_ONCE(rcu_normal, 1);
239 rcu_boot_ended = true; 200 rcu_boot_ended = true;
240 } 201 }
241 202
242 /* 203 /*
243 * Let rcutorture know when it is OK to turn i 204 * Let rcutorture know when it is OK to turn it up to eleven.
244 */ 205 */
245 bool rcu_inkernel_boot_has_ended(void) 206 bool rcu_inkernel_boot_has_ended(void)
246 { 207 {
247 return rcu_boot_ended; 208 return rcu_boot_ended;
248 } 209 }
249 EXPORT_SYMBOL_GPL(rcu_inkernel_boot_has_ended) 210 EXPORT_SYMBOL_GPL(rcu_inkernel_boot_has_ended);
250 211
251 #endif /* #ifndef CONFIG_TINY_RCU */ 212 #endif /* #ifndef CONFIG_TINY_RCU */
252 213
253 /* 214 /*
254 * Test each non-SRCU synchronous grace-period 215 * Test each non-SRCU synchronous grace-period wait API. This is
255 * useful just after a change in mode for thes 216 * useful just after a change in mode for these primitives, and
256 * during early boot. 217 * during early boot.
257 */ 218 */
258 void rcu_test_sync_prims(void) 219 void rcu_test_sync_prims(void)
259 { 220 {
260 if (!IS_ENABLED(CONFIG_PROVE_RCU)) 221 if (!IS_ENABLED(CONFIG_PROVE_RCU))
261 return; 222 return;
262 pr_info("Running RCU synchronous self <<
263 synchronize_rcu(); 223 synchronize_rcu();
264 synchronize_rcu_expedited(); 224 synchronize_rcu_expedited();
265 } 225 }
266 226
267 #if !defined(CONFIG_TINY_RCU) !! 227 #if !defined(CONFIG_TINY_RCU) || defined(CONFIG_SRCU)
268 228
269 /* 229 /*
270 * Switch to run-time mode once RCU has fully 230 * Switch to run-time mode once RCU has fully initialized.
271 */ 231 */
272 static int __init rcu_set_runtime_mode(void) 232 static int __init rcu_set_runtime_mode(void)
273 { 233 {
274 rcu_test_sync_prims(); 234 rcu_test_sync_prims();
275 rcu_scheduler_active = RCU_SCHEDULER_R 235 rcu_scheduler_active = RCU_SCHEDULER_RUNNING;
276 kfree_rcu_scheduler_running(); 236 kfree_rcu_scheduler_running();
277 rcu_test_sync_prims(); 237 rcu_test_sync_prims();
278 return 0; 238 return 0;
279 } 239 }
280 core_initcall(rcu_set_runtime_mode); 240 core_initcall(rcu_set_runtime_mode);
281 241
282 #endif /* #if !defined(CONFIG_TINY_RCU) */ !! 242 #endif /* #if !defined(CONFIG_TINY_RCU) || defined(CONFIG_SRCU) */
283 243
284 #ifdef CONFIG_DEBUG_LOCK_ALLOC 244 #ifdef CONFIG_DEBUG_LOCK_ALLOC
285 static struct lock_class_key rcu_lock_key; 245 static struct lock_class_key rcu_lock_key;
286 struct lockdep_map rcu_lock_map = { 246 struct lockdep_map rcu_lock_map = {
287 .name = "rcu_read_lock", 247 .name = "rcu_read_lock",
288 .key = &rcu_lock_key, 248 .key = &rcu_lock_key,
289 .wait_type_outer = LD_WAIT_FREE, 249 .wait_type_outer = LD_WAIT_FREE,
290 .wait_type_inner = LD_WAIT_CONFIG, /* !! 250 .wait_type_inner = LD_WAIT_CONFIG, /* XXX PREEMPT_RCU ? */
291 }; 251 };
292 EXPORT_SYMBOL_GPL(rcu_lock_map); 252 EXPORT_SYMBOL_GPL(rcu_lock_map);
293 253
294 static struct lock_class_key rcu_bh_lock_key; 254 static struct lock_class_key rcu_bh_lock_key;
295 struct lockdep_map rcu_bh_lock_map = { 255 struct lockdep_map rcu_bh_lock_map = {
296 .name = "rcu_read_lock_bh", 256 .name = "rcu_read_lock_bh",
297 .key = &rcu_bh_lock_key, 257 .key = &rcu_bh_lock_key,
298 .wait_type_outer = LD_WAIT_FREE, 258 .wait_type_outer = LD_WAIT_FREE,
299 .wait_type_inner = LD_WAIT_CONFIG, /* !! 259 .wait_type_inner = LD_WAIT_CONFIG, /* PREEMPT_LOCK also makes BH preemptible */
300 }; 260 };
301 EXPORT_SYMBOL_GPL(rcu_bh_lock_map); 261 EXPORT_SYMBOL_GPL(rcu_bh_lock_map);
302 262
303 static struct lock_class_key rcu_sched_lock_ke 263 static struct lock_class_key rcu_sched_lock_key;
304 struct lockdep_map rcu_sched_lock_map = { 264 struct lockdep_map rcu_sched_lock_map = {
305 .name = "rcu_read_lock_sched", 265 .name = "rcu_read_lock_sched",
306 .key = &rcu_sched_lock_key, 266 .key = &rcu_sched_lock_key,
307 .wait_type_outer = LD_WAIT_FREE, 267 .wait_type_outer = LD_WAIT_FREE,
308 .wait_type_inner = LD_WAIT_SPIN, 268 .wait_type_inner = LD_WAIT_SPIN,
309 }; 269 };
310 EXPORT_SYMBOL_GPL(rcu_sched_lock_map); 270 EXPORT_SYMBOL_GPL(rcu_sched_lock_map);
311 271
312 // Tell lockdep when RCU callbacks are being i 272 // Tell lockdep when RCU callbacks are being invoked.
313 static struct lock_class_key rcu_callback_key; 273 static struct lock_class_key rcu_callback_key;
314 struct lockdep_map rcu_callback_map = 274 struct lockdep_map rcu_callback_map =
315 STATIC_LOCKDEP_MAP_INIT("rcu_callback" 275 STATIC_LOCKDEP_MAP_INIT("rcu_callback", &rcu_callback_key);
316 EXPORT_SYMBOL_GPL(rcu_callback_map); 276 EXPORT_SYMBOL_GPL(rcu_callback_map);
317 277
318 noinstr int notrace debug_lockdep_rcu_enabled( 278 noinstr int notrace debug_lockdep_rcu_enabled(void)
319 { 279 {
320 return rcu_scheduler_active != RCU_SCH 280 return rcu_scheduler_active != RCU_SCHEDULER_INACTIVE && READ_ONCE(debug_locks) &&
321 current->lockdep_recursion == 0 281 current->lockdep_recursion == 0;
322 } 282 }
323 EXPORT_SYMBOL_GPL(debug_lockdep_rcu_enabled); 283 EXPORT_SYMBOL_GPL(debug_lockdep_rcu_enabled);
324 284
325 /** 285 /**
326 * rcu_read_lock_held() - might we be in RCU r 286 * rcu_read_lock_held() - might we be in RCU read-side critical section?
327 * 287 *
328 * If CONFIG_DEBUG_LOCK_ALLOC is selected, ret 288 * If CONFIG_DEBUG_LOCK_ALLOC is selected, returns nonzero iff in an RCU
329 * read-side critical section. In absence of 289 * read-side critical section. In absence of CONFIG_DEBUG_LOCK_ALLOC,
330 * this assumes we are in an RCU read-side cri 290 * this assumes we are in an RCU read-side critical section unless it can
331 * prove otherwise. This is useful for debug 291 * prove otherwise. This is useful for debug checks in functions that
332 * require that they be called within an RCU r 292 * require that they be called within an RCU read-side critical section.
333 * 293 *
334 * Checks debug_lockdep_rcu_enabled() to preve 294 * Checks debug_lockdep_rcu_enabled() to prevent false positives during boot
335 * and while lockdep is disabled. 295 * and while lockdep is disabled.
336 * 296 *
337 * Note that rcu_read_lock() and the matching 297 * Note that rcu_read_lock() and the matching rcu_read_unlock() must
338 * occur in the same context, for example, it 298 * occur in the same context, for example, it is illegal to invoke
339 * rcu_read_unlock() in process context if the 299 * rcu_read_unlock() in process context if the matching rcu_read_lock()
340 * was invoked from within an irq handler. 300 * was invoked from within an irq handler.
341 * 301 *
342 * Note that rcu_read_lock() is disallowed if 302 * Note that rcu_read_lock() is disallowed if the CPU is either idle or
343 * offline from an RCU perspective, so check f 303 * offline from an RCU perspective, so check for those as well.
344 */ 304 */
345 int rcu_read_lock_held(void) 305 int rcu_read_lock_held(void)
346 { 306 {
347 bool ret; 307 bool ret;
348 308
349 if (rcu_read_lock_held_common(&ret)) 309 if (rcu_read_lock_held_common(&ret))
350 return ret; 310 return ret;
351 return lock_is_held(&rcu_lock_map); 311 return lock_is_held(&rcu_lock_map);
352 } 312 }
353 EXPORT_SYMBOL_GPL(rcu_read_lock_held); 313 EXPORT_SYMBOL_GPL(rcu_read_lock_held);
354 314
355 /** 315 /**
356 * rcu_read_lock_bh_held() - might we be in RC 316 * rcu_read_lock_bh_held() - might we be in RCU-bh read-side critical section?
357 * 317 *
358 * Check for bottom half being disabled, which 318 * Check for bottom half being disabled, which covers both the
359 * CONFIG_PROVE_RCU and not cases. Note that 319 * CONFIG_PROVE_RCU and not cases. Note that if someone uses
360 * rcu_read_lock_bh(), but then later enables 320 * rcu_read_lock_bh(), but then later enables BH, lockdep (if enabled)
361 * will show the situation. This is useful fo 321 * will show the situation. This is useful for debug checks in functions
362 * that require that they be called within an 322 * that require that they be called within an RCU read-side critical
363 * section. 323 * section.
364 * 324 *
365 * Check debug_lockdep_rcu_enabled() to preven 325 * Check debug_lockdep_rcu_enabled() to prevent false positives during boot.
366 * 326 *
367 * Note that rcu_read_lock_bh() is disallowed 327 * Note that rcu_read_lock_bh() is disallowed if the CPU is either idle or
368 * offline from an RCU perspective, so check f 328 * offline from an RCU perspective, so check for those as well.
369 */ 329 */
370 int rcu_read_lock_bh_held(void) 330 int rcu_read_lock_bh_held(void)
371 { 331 {
372 bool ret; 332 bool ret;
373 333
374 if (rcu_read_lock_held_common(&ret)) 334 if (rcu_read_lock_held_common(&ret))
375 return ret; 335 return ret;
376 return in_softirq() || irqs_disabled() 336 return in_softirq() || irqs_disabled();
377 } 337 }
378 EXPORT_SYMBOL_GPL(rcu_read_lock_bh_held); 338 EXPORT_SYMBOL_GPL(rcu_read_lock_bh_held);
379 339
380 int rcu_read_lock_any_held(void) 340 int rcu_read_lock_any_held(void)
381 { 341 {
382 bool ret; 342 bool ret;
383 343
384 if (rcu_read_lock_held_common(&ret)) 344 if (rcu_read_lock_held_common(&ret))
385 return ret; 345 return ret;
386 if (lock_is_held(&rcu_lock_map) || 346 if (lock_is_held(&rcu_lock_map) ||
387 lock_is_held(&rcu_bh_lock_map) || 347 lock_is_held(&rcu_bh_lock_map) ||
388 lock_is_held(&rcu_sched_lock_map)) 348 lock_is_held(&rcu_sched_lock_map))
389 return 1; 349 return 1;
390 return !preemptible(); 350 return !preemptible();
391 } 351 }
392 EXPORT_SYMBOL_GPL(rcu_read_lock_any_held); 352 EXPORT_SYMBOL_GPL(rcu_read_lock_any_held);
393 353
394 #endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ 354 #endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
395 355
396 /** 356 /**
397 * wakeme_after_rcu() - Callback function to a 357 * wakeme_after_rcu() - Callback function to awaken a task after grace period
398 * @head: Pointer to rcu_head member within rc 358 * @head: Pointer to rcu_head member within rcu_synchronize structure
399 * 359 *
400 * Awaken the corresponding task now that a gr 360 * Awaken the corresponding task now that a grace period has elapsed.
401 */ 361 */
402 void wakeme_after_rcu(struct rcu_head *head) 362 void wakeme_after_rcu(struct rcu_head *head)
403 { 363 {
404 struct rcu_synchronize *rcu; 364 struct rcu_synchronize *rcu;
405 365
406 rcu = container_of(head, struct rcu_sy 366 rcu = container_of(head, struct rcu_synchronize, head);
407 complete(&rcu->completion); 367 complete(&rcu->completion);
408 } 368 }
409 EXPORT_SYMBOL_GPL(wakeme_after_rcu); 369 EXPORT_SYMBOL_GPL(wakeme_after_rcu);
410 370
411 void __wait_rcu_gp(bool checktiny, unsigned in !! 371 void __wait_rcu_gp(bool checktiny, int n, call_rcu_func_t *crcu_array,
412 struct rcu_synchronize *rs_ 372 struct rcu_synchronize *rs_array)
413 { 373 {
414 int i; 374 int i;
415 int j; 375 int j;
416 376
417 /* Initialize and register callbacks f 377 /* Initialize and register callbacks for each crcu_array element. */
418 for (i = 0; i < n; i++) { 378 for (i = 0; i < n; i++) {
419 if (checktiny && 379 if (checktiny &&
420 (crcu_array[i] == call_rcu 380 (crcu_array[i] == call_rcu)) {
421 might_sleep(); 381 might_sleep();
422 continue; 382 continue;
423 } 383 }
424 for (j = 0; j < i; j++) 384 for (j = 0; j < i; j++)
425 if (crcu_array[j] == c 385 if (crcu_array[j] == crcu_array[i])
426 break; 386 break;
427 if (j == i) { 387 if (j == i) {
428 init_rcu_head_on_stack 388 init_rcu_head_on_stack(&rs_array[i].head);
429 init_completion(&rs_ar 389 init_completion(&rs_array[i].completion);
430 (crcu_array[i])(&rs_ar 390 (crcu_array[i])(&rs_array[i].head, wakeme_after_rcu);
431 } 391 }
432 } 392 }
433 393
434 /* Wait for all callbacks to be invoke 394 /* Wait for all callbacks to be invoked. */
435 for (i = 0; i < n; i++) { 395 for (i = 0; i < n; i++) {
436 if (checktiny && 396 if (checktiny &&
437 (crcu_array[i] == call_rcu 397 (crcu_array[i] == call_rcu))
438 continue; 398 continue;
439 for (j = 0; j < i; j++) 399 for (j = 0; j < i; j++)
440 if (crcu_array[j] == c 400 if (crcu_array[j] == crcu_array[i])
441 break; 401 break;
442 if (j == i) { 402 if (j == i) {
443 wait_for_completion_st !! 403 wait_for_completion(&rs_array[i].completion);
444 destroy_rcu_head_on_st 404 destroy_rcu_head_on_stack(&rs_array[i].head);
445 } 405 }
446 } 406 }
447 } 407 }
448 EXPORT_SYMBOL_GPL(__wait_rcu_gp); 408 EXPORT_SYMBOL_GPL(__wait_rcu_gp);
449 409
450 void finish_rcuwait(struct rcuwait *w) <<
451 { <<
452 rcu_assign_pointer(w->task, NULL); <<
453 __set_current_state(TASK_RUNNING); <<
454 } <<
455 EXPORT_SYMBOL_GPL(finish_rcuwait); <<
456 <<
457 #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD 410 #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
458 void init_rcu_head(struct rcu_head *head) 411 void init_rcu_head(struct rcu_head *head)
459 { 412 {
460 debug_object_init(head, &rcuhead_debug 413 debug_object_init(head, &rcuhead_debug_descr);
461 } 414 }
462 EXPORT_SYMBOL_GPL(init_rcu_head); 415 EXPORT_SYMBOL_GPL(init_rcu_head);
463 416
464 void destroy_rcu_head(struct rcu_head *head) 417 void destroy_rcu_head(struct rcu_head *head)
465 { 418 {
466 debug_object_free(head, &rcuhead_debug 419 debug_object_free(head, &rcuhead_debug_descr);
467 } 420 }
468 EXPORT_SYMBOL_GPL(destroy_rcu_head); 421 EXPORT_SYMBOL_GPL(destroy_rcu_head);
469 422
470 static bool rcuhead_is_static_object(void *add 423 static bool rcuhead_is_static_object(void *addr)
471 { 424 {
472 return true; 425 return true;
473 } 426 }
474 427
475 /** 428 /**
476 * init_rcu_head_on_stack() - initialize on-st 429 * init_rcu_head_on_stack() - initialize on-stack rcu_head for debugobjects
477 * @head: pointer to rcu_head structure to be 430 * @head: pointer to rcu_head structure to be initialized
478 * 431 *
479 * This function informs debugobjects of a new 432 * This function informs debugobjects of a new rcu_head structure that
480 * has been allocated as an auto variable on t 433 * has been allocated as an auto variable on the stack. This function
481 * is not required for rcu_head structures tha 434 * is not required for rcu_head structures that are statically defined or
482 * that are dynamically allocated on the heap. 435 * that are dynamically allocated on the heap. This function has no
483 * effect for !CONFIG_DEBUG_OBJECTS_RCU_HEAD k 436 * effect for !CONFIG_DEBUG_OBJECTS_RCU_HEAD kernel builds.
484 */ 437 */
485 void init_rcu_head_on_stack(struct rcu_head *h 438 void init_rcu_head_on_stack(struct rcu_head *head)
486 { 439 {
487 debug_object_init_on_stack(head, &rcuh 440 debug_object_init_on_stack(head, &rcuhead_debug_descr);
488 } 441 }
489 EXPORT_SYMBOL_GPL(init_rcu_head_on_stack); 442 EXPORT_SYMBOL_GPL(init_rcu_head_on_stack);
490 443
491 /** 444 /**
492 * destroy_rcu_head_on_stack() - destroy on-st 445 * destroy_rcu_head_on_stack() - destroy on-stack rcu_head for debugobjects
493 * @head: pointer to rcu_head structure to be 446 * @head: pointer to rcu_head structure to be initialized
494 * 447 *
495 * This function informs debugobjects that an 448 * This function informs debugobjects that an on-stack rcu_head structure
496 * is about to go out of scope. As with init_ 449 * is about to go out of scope. As with init_rcu_head_on_stack(), this
497 * function is not required for rcu_head struc 450 * function is not required for rcu_head structures that are statically
498 * defined or that are dynamically allocated o 451 * defined or that are dynamically allocated on the heap. Also as with
499 * init_rcu_head_on_stack(), this function has 452 * init_rcu_head_on_stack(), this function has no effect for
500 * !CONFIG_DEBUG_OBJECTS_RCU_HEAD kernel build 453 * !CONFIG_DEBUG_OBJECTS_RCU_HEAD kernel builds.
501 */ 454 */
502 void destroy_rcu_head_on_stack(struct rcu_head 455 void destroy_rcu_head_on_stack(struct rcu_head *head)
503 { 456 {
504 debug_object_free(head, &rcuhead_debug 457 debug_object_free(head, &rcuhead_debug_descr);
505 } 458 }
506 EXPORT_SYMBOL_GPL(destroy_rcu_head_on_stack); 459 EXPORT_SYMBOL_GPL(destroy_rcu_head_on_stack);
507 460
508 const struct debug_obj_descr rcuhead_debug_des 461 const struct debug_obj_descr rcuhead_debug_descr = {
509 .name = "rcu_head", 462 .name = "rcu_head",
510 .is_static_object = rcuhead_is_static_ 463 .is_static_object = rcuhead_is_static_object,
511 }; 464 };
512 EXPORT_SYMBOL_GPL(rcuhead_debug_descr); 465 EXPORT_SYMBOL_GPL(rcuhead_debug_descr);
513 #endif /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD 466 #endif /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */
514 467
515 #if defined(CONFIG_TREE_RCU) || defined(CONFIG 468 #if defined(CONFIG_TREE_RCU) || defined(CONFIG_RCU_TRACE)
516 void do_trace_rcu_torture_read(const char *rcu 469 void do_trace_rcu_torture_read(const char *rcutorturename, struct rcu_head *rhp,
517 unsigned long s 470 unsigned long secs,
518 unsigned long c 471 unsigned long c_old, unsigned long c)
519 { 472 {
520 trace_rcu_torture_read(rcutorturename, 473 trace_rcu_torture_read(rcutorturename, rhp, secs, c_old, c);
521 } 474 }
522 EXPORT_SYMBOL_GPL(do_trace_rcu_torture_read); 475 EXPORT_SYMBOL_GPL(do_trace_rcu_torture_read);
523 #else 476 #else
524 #define do_trace_rcu_torture_read(rcutorturena 477 #define do_trace_rcu_torture_read(rcutorturename, rhp, secs, c_old, c) \
525 do { } while (0) 478 do { } while (0)
526 #endif 479 #endif
527 480
528 #if IS_ENABLED(CONFIG_RCU_TORTURE_TEST) || IS_ !! 481 #if IS_ENABLED(CONFIG_RCU_TORTURE_TEST) || IS_MODULE(CONFIG_RCU_TORTURE_TEST)
529 /* Get rcutorture access to sched_setaffinity( 482 /* Get rcutorture access to sched_setaffinity(). */
530 long torture_sched_setaffinity(pid_t pid, cons !! 483 long rcutorture_sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
531 { 484 {
532 int ret; 485 int ret;
533 486
534 ret = sched_setaffinity(pid, in_mask); 487 ret = sched_setaffinity(pid, in_mask);
535 WARN_ONCE(ret, "%s: sched_setaffinity( !! 488 WARN_ONCE(ret, "%s: sched_setaffinity() returned %d\n", __func__, ret);
536 return ret; 489 return ret;
537 } 490 }
538 EXPORT_SYMBOL_GPL(torture_sched_setaffinity); !! 491 EXPORT_SYMBOL_GPL(rcutorture_sched_setaffinity);
539 #endif 492 #endif
540 493
541 int rcu_cpu_stall_notifiers __read_mostly; // <<
542 EXPORT_SYMBOL_GPL(rcu_cpu_stall_notifiers); <<
543 <<
544 #ifdef CONFIG_RCU_STALL_COMMON 494 #ifdef CONFIG_RCU_STALL_COMMON
545 int rcu_cpu_stall_ftrace_dump __read_mostly; 495 int rcu_cpu_stall_ftrace_dump __read_mostly;
546 module_param(rcu_cpu_stall_ftrace_dump, int, 0 496 module_param(rcu_cpu_stall_ftrace_dump, int, 0644);
547 #ifdef CONFIG_RCU_CPU_STALL_NOTIFIER <<
548 module_param(rcu_cpu_stall_notifiers, int, 044 <<
549 #endif // #ifdef CONFIG_RCU_CPU_STALL_NOTIFIER <<
550 int rcu_cpu_stall_suppress __read_mostly; // ! 497 int rcu_cpu_stall_suppress __read_mostly; // !0 = suppress stall warnings.
551 EXPORT_SYMBOL_GPL(rcu_cpu_stall_suppress); 498 EXPORT_SYMBOL_GPL(rcu_cpu_stall_suppress);
552 module_param(rcu_cpu_stall_suppress, int, 0644 499 module_param(rcu_cpu_stall_suppress, int, 0644);
553 int rcu_cpu_stall_timeout __read_mostly = CONF 500 int rcu_cpu_stall_timeout __read_mostly = CONFIG_RCU_CPU_STALL_TIMEOUT;
554 module_param(rcu_cpu_stall_timeout, int, 0644) 501 module_param(rcu_cpu_stall_timeout, int, 0644);
555 int rcu_exp_cpu_stall_timeout __read_mostly = <<
556 module_param(rcu_exp_cpu_stall_timeout, int, 0 <<
557 int rcu_cpu_stall_cputime __read_mostly = IS_E <<
558 module_param(rcu_cpu_stall_cputime, int, 0644) <<
559 bool rcu_exp_stall_task_details __read_mostly; <<
560 module_param(rcu_exp_stall_task_details, bool, <<
561 #endif /* #ifdef CONFIG_RCU_STALL_COMMON */ 502 #endif /* #ifdef CONFIG_RCU_STALL_COMMON */
562 503
563 // Suppress boot-time RCU CPU stall warnings a 504 // Suppress boot-time RCU CPU stall warnings and rcutorture writer stall
564 // warnings. Also used by rcutorture even if 505 // warnings. Also used by rcutorture even if stall warnings are excluded.
565 int rcu_cpu_stall_suppress_at_boot __read_most 506 int rcu_cpu_stall_suppress_at_boot __read_mostly; // !0 = suppress boot stalls.
566 EXPORT_SYMBOL_GPL(rcu_cpu_stall_suppress_at_bo 507 EXPORT_SYMBOL_GPL(rcu_cpu_stall_suppress_at_boot);
567 module_param(rcu_cpu_stall_suppress_at_boot, i 508 module_param(rcu_cpu_stall_suppress_at_boot, int, 0444);
568 509
569 /** <<
570 * get_completed_synchronize_rcu - Return a pr <<
571 * <<
572 * Returns a value that will always be treated <<
573 * poll_state_synchronize_rcu() as a cookie wh <<
574 * completed. <<
575 */ <<
576 unsigned long get_completed_synchronize_rcu(vo <<
577 { <<
578 return RCU_GET_STATE_COMPLETED; <<
579 } <<
580 EXPORT_SYMBOL_GPL(get_completed_synchronize_rc <<
581 <<
582 #ifdef CONFIG_PROVE_RCU 510 #ifdef CONFIG_PROVE_RCU
583 511
584 /* 512 /*
585 * Early boot self test parameters. 513 * Early boot self test parameters.
586 */ 514 */
587 static bool rcu_self_test; 515 static bool rcu_self_test;
588 module_param(rcu_self_test, bool, 0444); 516 module_param(rcu_self_test, bool, 0444);
589 517
590 static int rcu_self_test_counter; 518 static int rcu_self_test_counter;
591 519
592 static void test_callback(struct rcu_head *r) 520 static void test_callback(struct rcu_head *r)
593 { 521 {
594 rcu_self_test_counter++; 522 rcu_self_test_counter++;
595 pr_info("RCU test callback executed %d 523 pr_info("RCU test callback executed %d\n", rcu_self_test_counter);
596 } 524 }
597 525
598 DEFINE_STATIC_SRCU(early_srcu); 526 DEFINE_STATIC_SRCU(early_srcu);
599 static unsigned long early_srcu_cookie; 527 static unsigned long early_srcu_cookie;
600 528
601 struct early_boot_kfree_rcu { 529 struct early_boot_kfree_rcu {
602 struct rcu_head rh; 530 struct rcu_head rh;
603 }; 531 };
604 532
605 static void early_boot_test_call_rcu(void) 533 static void early_boot_test_call_rcu(void)
606 { 534 {
607 static struct rcu_head head; 535 static struct rcu_head head;
608 int idx; <<
609 static struct rcu_head shead; 536 static struct rcu_head shead;
610 struct early_boot_kfree_rcu *rhp; 537 struct early_boot_kfree_rcu *rhp;
611 538
612 idx = srcu_down_read(&early_srcu); <<
613 srcu_up_read(&early_srcu, idx); <<
614 call_rcu(&head, test_callback); 539 call_rcu(&head, test_callback);
615 early_srcu_cookie = start_poll_synchro !! 540 if (IS_ENABLED(CONFIG_SRCU)) {
616 call_srcu(&early_srcu, &shead, test_ca !! 541 early_srcu_cookie = start_poll_synchronize_srcu(&early_srcu);
>> 542 call_srcu(&early_srcu, &shead, test_callback);
>> 543 }
617 rhp = kmalloc(sizeof(*rhp), GFP_KERNEL 544 rhp = kmalloc(sizeof(*rhp), GFP_KERNEL);
618 if (!WARN_ON_ONCE(!rhp)) 545 if (!WARN_ON_ONCE(!rhp))
619 kfree_rcu(rhp, rh); 546 kfree_rcu(rhp, rh);
620 } 547 }
621 548
622 void rcu_early_boot_tests(void) 549 void rcu_early_boot_tests(void)
623 { 550 {
624 pr_info("Running RCU self tests\n"); 551 pr_info("Running RCU self tests\n");
625 552
626 if (rcu_self_test) 553 if (rcu_self_test)
627 early_boot_test_call_rcu(); 554 early_boot_test_call_rcu();
628 rcu_test_sync_prims(); 555 rcu_test_sync_prims();
629 } 556 }
630 557
631 static int rcu_verify_early_boot_tests(void) 558 static int rcu_verify_early_boot_tests(void)
632 { 559 {
633 int ret = 0; 560 int ret = 0;
634 int early_boot_test_counter = 0; 561 int early_boot_test_counter = 0;
635 562
636 if (rcu_self_test) { 563 if (rcu_self_test) {
637 early_boot_test_counter++; 564 early_boot_test_counter++;
638 rcu_barrier(); 565 rcu_barrier();
639 early_boot_test_counter++; !! 566 if (IS_ENABLED(CONFIG_SRCU)) {
640 srcu_barrier(&early_srcu); !! 567 early_boot_test_counter++;
641 WARN_ON_ONCE(!poll_state_synch !! 568 srcu_barrier(&early_srcu);
642 cleanup_srcu_struct(&early_src !! 569 WARN_ON_ONCE(!poll_state_synchronize_srcu(&early_srcu, early_srcu_cookie));
>> 570 }
643 } 571 }
644 if (rcu_self_test_counter != early_boo 572 if (rcu_self_test_counter != early_boot_test_counter) {
645 WARN_ON(1); 573 WARN_ON(1);
646 ret = -1; 574 ret = -1;
647 } 575 }
648 576
649 return ret; 577 return ret;
650 } 578 }
651 late_initcall(rcu_verify_early_boot_tests); 579 late_initcall(rcu_verify_early_boot_tests);
652 #else 580 #else
653 void rcu_early_boot_tests(void) {} 581 void rcu_early_boot_tests(void) {}
654 #endif /* CONFIG_PROVE_RCU */ 582 #endif /* CONFIG_PROVE_RCU */
655 583
656 #include "tasks.h" 584 #include "tasks.h"
657 585
658 #ifndef CONFIG_TINY_RCU 586 #ifndef CONFIG_TINY_RCU
659 587
660 /* 588 /*
661 * Print any significant non-default boot-time 589 * Print any significant non-default boot-time settings.
662 */ 590 */
663 void __init rcupdate_announce_bootup_oddness(v 591 void __init rcupdate_announce_bootup_oddness(void)
664 { 592 {
665 if (rcu_normal) 593 if (rcu_normal)
666 pr_info("\tNo expedited grace 594 pr_info("\tNo expedited grace period (rcu_normal).\n");
667 else if (rcu_normal_after_boot) 595 else if (rcu_normal_after_boot)
668 pr_info("\tNo expedited grace 596 pr_info("\tNo expedited grace period (rcu_normal_after_boot).\n");
669 else if (rcu_expedited) 597 else if (rcu_expedited)
670 pr_info("\tAll grace periods a 598 pr_info("\tAll grace periods are expedited (rcu_expedited).\n");
671 if (rcu_cpu_stall_suppress) 599 if (rcu_cpu_stall_suppress)
672 pr_info("\tRCU CPU stall warni 600 pr_info("\tRCU CPU stall warnings suppressed (rcu_cpu_stall_suppress).\n");
673 if (rcu_cpu_stall_timeout != CONFIG_RC 601 if (rcu_cpu_stall_timeout != CONFIG_RCU_CPU_STALL_TIMEOUT)
674 pr_info("\tRCU CPU stall warni 602 pr_info("\tRCU CPU stall warnings timeout set to %d (rcu_cpu_stall_timeout).\n", rcu_cpu_stall_timeout);
675 rcu_tasks_bootup_oddness(); 603 rcu_tasks_bootup_oddness();
676 } 604 }
677 605
678 #endif /* #ifndef CONFIG_TINY_RCU */ 606 #endif /* #ifndef CONFIG_TINY_RCU */
679 607
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.