1 // SPDX-License-Identifier: GPL-2.0+ 1 // SPDX-License-Identifier: GPL-2.0+
2 /* 2 /*
3 * Read-Copy Update mechanism for mutual exclu 3 * Read-Copy Update mechanism for mutual exclusion
4 * 4 *
5 * Copyright IBM Corporation, 2001 5 * Copyright IBM Corporation, 2001
6 * 6 *
7 * Authors: Dipankar Sarma <dipankar@in.ibm.co 7 * Authors: Dipankar Sarma <dipankar@in.ibm.com>
8 * Manfred Spraul <manfred@colorfulli 8 * Manfred Spraul <manfred@colorfullife.com>
9 * 9 *
10 * Based on the original work by Paul McKenney 10 * Based on the original work by Paul McKenney <paulmck@linux.ibm.com>
11 * and inputs from Rusty Russell, Andrea Arcan 11 * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen.
12 * Papers: 12 * Papers:
13 * http://www.rdrop.com/users/paulmck/paper/rc 13 * http://www.rdrop.com/users/paulmck/paper/rclockpdcsproof.pdf
14 * http://lse.sourceforge.net/locking/rclock_O 14 * http://lse.sourceforge.net/locking/rclock_OLS.2001.05.01c.sc.pdf (OLS2001)
15 * 15 *
16 * For detailed explanation of Read-Copy Updat 16 * For detailed explanation of Read-Copy Update mechanism see -
17 * http://lse.sourceforge.net/loc 17 * http://lse.sourceforge.net/locking/rcupdate.html
18 * 18 *
19 */ 19 */
20 #include <linux/types.h> 20 #include <linux/types.h>
21 #include <linux/kernel.h> 21 #include <linux/kernel.h>
22 #include <linux/init.h> 22 #include <linux/init.h>
23 #include <linux/spinlock.h> 23 #include <linux/spinlock.h>
24 #include <linux/smp.h> 24 #include <linux/smp.h>
25 #include <linux/interrupt.h> 25 #include <linux/interrupt.h>
26 #include <linux/sched/signal.h> 26 #include <linux/sched/signal.h>
27 #include <linux/sched/debug.h> 27 #include <linux/sched/debug.h>
28 #include <linux/torture.h> <<
29 #include <linux/atomic.h> 28 #include <linux/atomic.h>
30 #include <linux/bitops.h> 29 #include <linux/bitops.h>
31 #include <linux/percpu.h> 30 #include <linux/percpu.h>
32 #include <linux/notifier.h> 31 #include <linux/notifier.h>
33 #include <linux/cpu.h> 32 #include <linux/cpu.h>
34 #include <linux/mutex.h> 33 #include <linux/mutex.h>
35 #include <linux/export.h> 34 #include <linux/export.h>
36 #include <linux/hardirq.h> 35 #include <linux/hardirq.h>
37 #include <linux/delay.h> 36 #include <linux/delay.h>
38 #include <linux/moduleparam.h> 37 #include <linux/moduleparam.h>
39 #include <linux/kthread.h> 38 #include <linux/kthread.h>
40 #include <linux/tick.h> 39 #include <linux/tick.h>
41 #include <linux/rcupdate_wait.h> 40 #include <linux/rcupdate_wait.h>
42 #include <linux/sched/isolation.h> 41 #include <linux/sched/isolation.h>
43 #include <linux/kprobes.h> 42 #include <linux/kprobes.h>
44 #include <linux/slab.h> 43 #include <linux/slab.h>
45 #include <linux/irq_work.h> 44 #include <linux/irq_work.h>
46 #include <linux/rcupdate_trace.h> 45 #include <linux/rcupdate_trace.h>
47 46
48 #define CREATE_TRACE_POINTS 47 #define CREATE_TRACE_POINTS
49 48
50 #include "rcu.h" 49 #include "rcu.h"
51 50
52 #ifdef MODULE_PARAM_PREFIX 51 #ifdef MODULE_PARAM_PREFIX
53 #undef MODULE_PARAM_PREFIX 52 #undef MODULE_PARAM_PREFIX
54 #endif 53 #endif
55 #define MODULE_PARAM_PREFIX "rcupdate." 54 #define MODULE_PARAM_PREFIX "rcupdate."
56 55
57 #ifndef CONFIG_TINY_RCU !! 56 #ifndef data_race
58 module_param(rcu_expedited, int, 0444); !! 57 #define data_race(expr) \
59 module_param(rcu_normal, int, 0444); !! 58 ({ \
60 static int rcu_normal_after_boot = IS_ENABLED( !! 59 expr; \
61 #if !defined(CONFIG_PREEMPT_RT) || defined(CON !! 60 })
62 module_param(rcu_normal_after_boot, int, 0444) !! 61 #endif
>> 62 #ifndef ASSERT_EXCLUSIVE_WRITER
>> 63 #define ASSERT_EXCLUSIVE_WRITER(var) do { } while (0)
>> 64 #endif
>> 65 #ifndef ASSERT_EXCLUSIVE_ACCESS
>> 66 #define ASSERT_EXCLUSIVE_ACCESS(var) do { } while (0)
63 #endif 67 #endif
>> 68
>> 69 #ifndef CONFIG_TINY_RCU
>> 70 module_param(rcu_expedited, int, 0);
>> 71 module_param(rcu_normal, int, 0);
>> 72 static int rcu_normal_after_boot;
>> 73 module_param(rcu_normal_after_boot, int, 0);
64 #endif /* #ifndef CONFIG_TINY_RCU */ 74 #endif /* #ifndef CONFIG_TINY_RCU */
65 75
66 #ifdef CONFIG_DEBUG_LOCK_ALLOC 76 #ifdef CONFIG_DEBUG_LOCK_ALLOC
67 /** 77 /**
68 * rcu_read_lock_held_common() - might we be i 78 * rcu_read_lock_held_common() - might we be in RCU-sched read-side critical section?
69 * @ret: Best guess answer if lockdep c 79 * @ret: Best guess answer if lockdep cannot be relied on
70 * 80 *
71 * Returns true if lockdep must be ignored, in 81 * Returns true if lockdep must be ignored, in which case ``*ret`` contains
72 * the best guess described below. Otherwise 82 * the best guess described below. Otherwise returns false, in which
73 * case ``*ret`` tells the caller nothing and 83 * case ``*ret`` tells the caller nothing and the caller should instead
74 * consult lockdep. 84 * consult lockdep.
75 * 85 *
76 * If CONFIG_DEBUG_LOCK_ALLOC is selected, set 86 * If CONFIG_DEBUG_LOCK_ALLOC is selected, set ``*ret`` to nonzero iff in an
77 * RCU-sched read-side critical section. In a 87 * RCU-sched read-side critical section. In absence of
78 * CONFIG_DEBUG_LOCK_ALLOC, this assumes we ar 88 * CONFIG_DEBUG_LOCK_ALLOC, this assumes we are in an RCU-sched read-side
79 * critical section unless it can prove otherw 89 * critical section unless it can prove otherwise. Note that disabling
80 * of preemption (including disabling irqs) co 90 * of preemption (including disabling irqs) counts as an RCU-sched
81 * read-side critical section. This is useful 91 * read-side critical section. This is useful for debug checks in functions
82 * that required that they be called within an 92 * that required that they be called within an RCU-sched read-side
83 * critical section. 93 * critical section.
84 * 94 *
85 * Check debug_lockdep_rcu_enabled() to preven 95 * Check debug_lockdep_rcu_enabled() to prevent false positives during boot
86 * and while lockdep is disabled. 96 * and while lockdep is disabled.
87 * 97 *
88 * Note that if the CPU is in the idle loop fr 98 * Note that if the CPU is in the idle loop from an RCU point of view (ie:
89 * that we are in the section between ct_idle_ !! 99 * that we are in the section between rcu_idle_enter() and rcu_idle_exit())
90 * then rcu_read_lock_held() sets ``*ret`` to 100 * then rcu_read_lock_held() sets ``*ret`` to false even if the CPU did an
91 * rcu_read_lock(). The reason for this is th 101 * rcu_read_lock(). The reason for this is that RCU ignores CPUs that are
92 * in such a section, considering these as in 102 * in such a section, considering these as in extended quiescent state,
93 * so such a CPU is effectively never in an RC 103 * so such a CPU is effectively never in an RCU read-side critical section
94 * regardless of what RCU primitives it invoke 104 * regardless of what RCU primitives it invokes. This state of affairs is
95 * required --- we need to keep an RCU-free wi 105 * required --- we need to keep an RCU-free window in idle where the CPU may
96 * possibly enter into low power mode. This wa 106 * possibly enter into low power mode. This way we can notice an extended
97 * quiescent state to other CPUs that started 107 * quiescent state to other CPUs that started a grace period. Otherwise
98 * we would delay any grace period as long as 108 * we would delay any grace period as long as we run in the idle task.
99 * 109 *
100 * Similarly, we avoid claiming an RCU read lo 110 * Similarly, we avoid claiming an RCU read lock held if the current
101 * CPU is offline. 111 * CPU is offline.
102 */ 112 */
103 static bool rcu_read_lock_held_common(bool *re 113 static bool rcu_read_lock_held_common(bool *ret)
104 { 114 {
105 if (!debug_lockdep_rcu_enabled()) { 115 if (!debug_lockdep_rcu_enabled()) {
106 *ret = true; 116 *ret = true;
107 return true; 117 return true;
108 } 118 }
109 if (!rcu_is_watching()) { 119 if (!rcu_is_watching()) {
110 *ret = false; 120 *ret = false;
111 return true; 121 return true;
112 } 122 }
113 if (!rcu_lockdep_current_cpu_online()) 123 if (!rcu_lockdep_current_cpu_online()) {
114 *ret = false; 124 *ret = false;
115 return true; 125 return true;
116 } 126 }
117 return false; 127 return false;
118 } 128 }
119 129
120 int rcu_read_lock_sched_held(void) 130 int rcu_read_lock_sched_held(void)
121 { 131 {
122 bool ret; 132 bool ret;
123 133
124 if (rcu_read_lock_held_common(&ret)) 134 if (rcu_read_lock_held_common(&ret))
125 return ret; 135 return ret;
126 return lock_is_held(&rcu_sched_lock_ma 136 return lock_is_held(&rcu_sched_lock_map) || !preemptible();
127 } 137 }
128 EXPORT_SYMBOL(rcu_read_lock_sched_held); 138 EXPORT_SYMBOL(rcu_read_lock_sched_held);
129 #endif 139 #endif
130 140
131 #ifndef CONFIG_TINY_RCU 141 #ifndef CONFIG_TINY_RCU
132 142
133 /* 143 /*
134 * Should expedited grace-period primitives al 144 * Should expedited grace-period primitives always fall back to their
135 * non-expedited counterparts? Intended for u 145 * non-expedited counterparts? Intended for use within RCU. Note
136 * that if the user specifies both rcu_expedit 146 * that if the user specifies both rcu_expedited and rcu_normal, then
137 * rcu_normal wins. (Except during the time p 147 * rcu_normal wins. (Except during the time period during boot from
138 * when the first task is spawned until the rc 148 * when the first task is spawned until the rcu_set_runtime_mode()
139 * core_initcall() is invoked, at which point 149 * core_initcall() is invoked, at which point everything is expedited.)
140 */ 150 */
141 bool rcu_gp_is_normal(void) 151 bool rcu_gp_is_normal(void)
142 { 152 {
143 return READ_ONCE(rcu_normal) && 153 return READ_ONCE(rcu_normal) &&
144 rcu_scheduler_active != RCU_SCH 154 rcu_scheduler_active != RCU_SCHEDULER_INIT;
145 } 155 }
146 EXPORT_SYMBOL_GPL(rcu_gp_is_normal); 156 EXPORT_SYMBOL_GPL(rcu_gp_is_normal);
147 157
148 static atomic_t rcu_async_hurry_nesting = ATOM <<
149 /* <<
150 * Should call_rcu() callbacks be processed wi <<
151 * they OK being executed with arbitrary delay <<
152 */ <<
153 bool rcu_async_should_hurry(void) <<
154 { <<
155 return !IS_ENABLED(CONFIG_RCU_LAZY) || <<
156 atomic_read(&rcu_async_hurry_ne <<
157 } <<
158 EXPORT_SYMBOL_GPL(rcu_async_should_hurry); <<
159 <<
160 /** <<
161 * rcu_async_hurry - Make future async RCU cal <<
162 * <<
163 * After a call to this function, future calls <<
164 * will be processed in a timely fashion. <<
165 */ <<
166 void rcu_async_hurry(void) <<
167 { <<
168 if (IS_ENABLED(CONFIG_RCU_LAZY)) <<
169 atomic_inc(&rcu_async_hurry_ne <<
170 } <<
171 EXPORT_SYMBOL_GPL(rcu_async_hurry); <<
172 <<
173 /** <<
174 * rcu_async_relax - Make future async RCU cal <<
175 * <<
176 * After a call to this function, future calls <<
177 * will be processed in a lazy fashion. <<
178 */ <<
179 void rcu_async_relax(void) <<
180 { <<
181 if (IS_ENABLED(CONFIG_RCU_LAZY)) <<
182 atomic_dec(&rcu_async_hurry_ne <<
183 } <<
184 EXPORT_SYMBOL_GPL(rcu_async_relax); <<
185 <<
186 static atomic_t rcu_expedited_nesting = ATOMIC 158 static atomic_t rcu_expedited_nesting = ATOMIC_INIT(1);
>> 159
187 /* 160 /*
188 * Should normal grace-period primitives be ex 161 * Should normal grace-period primitives be expedited? Intended for
189 * use within RCU. Note that this function ta 162 * use within RCU. Note that this function takes the rcu_expedited
190 * sysfs/boot variable and rcu_scheduler_activ 163 * sysfs/boot variable and rcu_scheduler_active into account as well
191 * as the rcu_expedite_gp() nesting. So loopi 164 * as the rcu_expedite_gp() nesting. So looping on rcu_unexpedite_gp()
192 * until rcu_gp_is_expedited() returns false i 165 * until rcu_gp_is_expedited() returns false is a -really- bad idea.
193 */ 166 */
194 bool rcu_gp_is_expedited(void) 167 bool rcu_gp_is_expedited(void)
195 { 168 {
196 return rcu_expedited || atomic_read(&r 169 return rcu_expedited || atomic_read(&rcu_expedited_nesting);
197 } 170 }
198 EXPORT_SYMBOL_GPL(rcu_gp_is_expedited); 171 EXPORT_SYMBOL_GPL(rcu_gp_is_expedited);
199 172
200 /** 173 /**
201 * rcu_expedite_gp - Expedite future RCU grace 174 * rcu_expedite_gp - Expedite future RCU grace periods
202 * 175 *
203 * After a call to this function, future calls 176 * After a call to this function, future calls to synchronize_rcu() and
204 * friends act as the corresponding synchroniz 177 * friends act as the corresponding synchronize_rcu_expedited() function
205 * had instead been called. 178 * had instead been called.
206 */ 179 */
207 void rcu_expedite_gp(void) 180 void rcu_expedite_gp(void)
208 { 181 {
209 atomic_inc(&rcu_expedited_nesting); 182 atomic_inc(&rcu_expedited_nesting);
210 } 183 }
211 EXPORT_SYMBOL_GPL(rcu_expedite_gp); 184 EXPORT_SYMBOL_GPL(rcu_expedite_gp);
212 185
213 /** 186 /**
214 * rcu_unexpedite_gp - Cancel prior rcu_expedi 187 * rcu_unexpedite_gp - Cancel prior rcu_expedite_gp() invocation
215 * 188 *
216 * Undo a prior call to rcu_expedite_gp(). If 189 * Undo a prior call to rcu_expedite_gp(). If all prior calls to
217 * rcu_expedite_gp() are undone by a subsequen 190 * rcu_expedite_gp() are undone by a subsequent call to rcu_unexpedite_gp(),
218 * and if the rcu_expedited sysfs/boot paramet 191 * and if the rcu_expedited sysfs/boot parameter is not set, then all
219 * subsequent calls to synchronize_rcu() and f 192 * subsequent calls to synchronize_rcu() and friends will return to
220 * their normal non-expedited behavior. 193 * their normal non-expedited behavior.
221 */ 194 */
222 void rcu_unexpedite_gp(void) 195 void rcu_unexpedite_gp(void)
223 { 196 {
224 atomic_dec(&rcu_expedited_nesting); 197 atomic_dec(&rcu_expedited_nesting);
225 } 198 }
226 EXPORT_SYMBOL_GPL(rcu_unexpedite_gp); 199 EXPORT_SYMBOL_GPL(rcu_unexpedite_gp);
227 200
228 static bool rcu_boot_ended __read_mostly; 201 static bool rcu_boot_ended __read_mostly;
229 202
230 /* 203 /*
231 * Inform RCU of the end of the in-kernel boot 204 * Inform RCU of the end of the in-kernel boot sequence.
232 */ 205 */
233 void rcu_end_inkernel_boot(void) 206 void rcu_end_inkernel_boot(void)
234 { 207 {
235 rcu_unexpedite_gp(); 208 rcu_unexpedite_gp();
236 rcu_async_relax(); <<
237 if (rcu_normal_after_boot) 209 if (rcu_normal_after_boot)
238 WRITE_ONCE(rcu_normal, 1); 210 WRITE_ONCE(rcu_normal, 1);
239 rcu_boot_ended = true; 211 rcu_boot_ended = true;
240 } 212 }
241 213
242 /* 214 /*
243 * Let rcutorture know when it is OK to turn i 215 * Let rcutorture know when it is OK to turn it up to eleven.
244 */ 216 */
245 bool rcu_inkernel_boot_has_ended(void) 217 bool rcu_inkernel_boot_has_ended(void)
246 { 218 {
247 return rcu_boot_ended; 219 return rcu_boot_ended;
248 } 220 }
249 EXPORT_SYMBOL_GPL(rcu_inkernel_boot_has_ended) 221 EXPORT_SYMBOL_GPL(rcu_inkernel_boot_has_ended);
250 222
251 #endif /* #ifndef CONFIG_TINY_RCU */ 223 #endif /* #ifndef CONFIG_TINY_RCU */
252 224
253 /* 225 /*
254 * Test each non-SRCU synchronous grace-period 226 * Test each non-SRCU synchronous grace-period wait API. This is
255 * useful just after a change in mode for thes 227 * useful just after a change in mode for these primitives, and
256 * during early boot. 228 * during early boot.
257 */ 229 */
258 void rcu_test_sync_prims(void) 230 void rcu_test_sync_prims(void)
259 { 231 {
260 if (!IS_ENABLED(CONFIG_PROVE_RCU)) 232 if (!IS_ENABLED(CONFIG_PROVE_RCU))
261 return; 233 return;
262 pr_info("Running RCU synchronous self <<
263 synchronize_rcu(); 234 synchronize_rcu();
264 synchronize_rcu_expedited(); 235 synchronize_rcu_expedited();
265 } 236 }
266 237
267 #if !defined(CONFIG_TINY_RCU) !! 238 #if !defined(CONFIG_TINY_RCU) || defined(CONFIG_SRCU)
268 239
269 /* 240 /*
270 * Switch to run-time mode once RCU has fully 241 * Switch to run-time mode once RCU has fully initialized.
271 */ 242 */
272 static int __init rcu_set_runtime_mode(void) 243 static int __init rcu_set_runtime_mode(void)
273 { 244 {
274 rcu_test_sync_prims(); 245 rcu_test_sync_prims();
275 rcu_scheduler_active = RCU_SCHEDULER_R 246 rcu_scheduler_active = RCU_SCHEDULER_RUNNING;
276 kfree_rcu_scheduler_running(); 247 kfree_rcu_scheduler_running();
277 rcu_test_sync_prims(); 248 rcu_test_sync_prims();
278 return 0; 249 return 0;
279 } 250 }
280 core_initcall(rcu_set_runtime_mode); 251 core_initcall(rcu_set_runtime_mode);
281 252
282 #endif /* #if !defined(CONFIG_TINY_RCU) */ !! 253 #endif /* #if !defined(CONFIG_TINY_RCU) || defined(CONFIG_SRCU) */
283 254
284 #ifdef CONFIG_DEBUG_LOCK_ALLOC 255 #ifdef CONFIG_DEBUG_LOCK_ALLOC
285 static struct lock_class_key rcu_lock_key; 256 static struct lock_class_key rcu_lock_key;
286 struct lockdep_map rcu_lock_map = { 257 struct lockdep_map rcu_lock_map = {
287 .name = "rcu_read_lock", 258 .name = "rcu_read_lock",
288 .key = &rcu_lock_key, 259 .key = &rcu_lock_key,
289 .wait_type_outer = LD_WAIT_FREE, 260 .wait_type_outer = LD_WAIT_FREE,
290 .wait_type_inner = LD_WAIT_CONFIG, /* !! 261 .wait_type_inner = LD_WAIT_CONFIG, /* XXX PREEMPT_RCU ? */
291 }; 262 };
292 EXPORT_SYMBOL_GPL(rcu_lock_map); 263 EXPORT_SYMBOL_GPL(rcu_lock_map);
293 264
294 static struct lock_class_key rcu_bh_lock_key; 265 static struct lock_class_key rcu_bh_lock_key;
295 struct lockdep_map rcu_bh_lock_map = { 266 struct lockdep_map rcu_bh_lock_map = {
296 .name = "rcu_read_lock_bh", 267 .name = "rcu_read_lock_bh",
297 .key = &rcu_bh_lock_key, 268 .key = &rcu_bh_lock_key,
298 .wait_type_outer = LD_WAIT_FREE, 269 .wait_type_outer = LD_WAIT_FREE,
299 .wait_type_inner = LD_WAIT_CONFIG, /* !! 270 .wait_type_inner = LD_WAIT_CONFIG, /* PREEMPT_LOCK also makes BH preemptible */
300 }; 271 };
301 EXPORT_SYMBOL_GPL(rcu_bh_lock_map); 272 EXPORT_SYMBOL_GPL(rcu_bh_lock_map);
302 273
303 static struct lock_class_key rcu_sched_lock_ke 274 static struct lock_class_key rcu_sched_lock_key;
304 struct lockdep_map rcu_sched_lock_map = { 275 struct lockdep_map rcu_sched_lock_map = {
305 .name = "rcu_read_lock_sched", 276 .name = "rcu_read_lock_sched",
306 .key = &rcu_sched_lock_key, 277 .key = &rcu_sched_lock_key,
307 .wait_type_outer = LD_WAIT_FREE, 278 .wait_type_outer = LD_WAIT_FREE,
308 .wait_type_inner = LD_WAIT_SPIN, 279 .wait_type_inner = LD_WAIT_SPIN,
309 }; 280 };
310 EXPORT_SYMBOL_GPL(rcu_sched_lock_map); 281 EXPORT_SYMBOL_GPL(rcu_sched_lock_map);
311 282
312 // Tell lockdep when RCU callbacks are being i 283 // Tell lockdep when RCU callbacks are being invoked.
313 static struct lock_class_key rcu_callback_key; 284 static struct lock_class_key rcu_callback_key;
314 struct lockdep_map rcu_callback_map = 285 struct lockdep_map rcu_callback_map =
315 STATIC_LOCKDEP_MAP_INIT("rcu_callback" 286 STATIC_LOCKDEP_MAP_INIT("rcu_callback", &rcu_callback_key);
316 EXPORT_SYMBOL_GPL(rcu_callback_map); 287 EXPORT_SYMBOL_GPL(rcu_callback_map);
317 288
318 noinstr int notrace debug_lockdep_rcu_enabled( 289 noinstr int notrace debug_lockdep_rcu_enabled(void)
319 { 290 {
320 return rcu_scheduler_active != RCU_SCH !! 291 return rcu_scheduler_active != RCU_SCHEDULER_INACTIVE && debug_locks &&
321 current->lockdep_recursion == 0 292 current->lockdep_recursion == 0;
322 } 293 }
323 EXPORT_SYMBOL_GPL(debug_lockdep_rcu_enabled); 294 EXPORT_SYMBOL_GPL(debug_lockdep_rcu_enabled);
324 295
325 /** 296 /**
326 * rcu_read_lock_held() - might we be in RCU r 297 * rcu_read_lock_held() - might we be in RCU read-side critical section?
327 * 298 *
328 * If CONFIG_DEBUG_LOCK_ALLOC is selected, ret 299 * If CONFIG_DEBUG_LOCK_ALLOC is selected, returns nonzero iff in an RCU
329 * read-side critical section. In absence of 300 * read-side critical section. In absence of CONFIG_DEBUG_LOCK_ALLOC,
330 * this assumes we are in an RCU read-side cri 301 * this assumes we are in an RCU read-side critical section unless it can
331 * prove otherwise. This is useful for debug 302 * prove otherwise. This is useful for debug checks in functions that
332 * require that they be called within an RCU r 303 * require that they be called within an RCU read-side critical section.
333 * 304 *
334 * Checks debug_lockdep_rcu_enabled() to preve 305 * Checks debug_lockdep_rcu_enabled() to prevent false positives during boot
335 * and while lockdep is disabled. 306 * and while lockdep is disabled.
336 * 307 *
337 * Note that rcu_read_lock() and the matching 308 * Note that rcu_read_lock() and the matching rcu_read_unlock() must
338 * occur in the same context, for example, it 309 * occur in the same context, for example, it is illegal to invoke
339 * rcu_read_unlock() in process context if the 310 * rcu_read_unlock() in process context if the matching rcu_read_lock()
340 * was invoked from within an irq handler. 311 * was invoked from within an irq handler.
341 * 312 *
342 * Note that rcu_read_lock() is disallowed if 313 * Note that rcu_read_lock() is disallowed if the CPU is either idle or
343 * offline from an RCU perspective, so check f 314 * offline from an RCU perspective, so check for those as well.
344 */ 315 */
345 int rcu_read_lock_held(void) 316 int rcu_read_lock_held(void)
346 { 317 {
347 bool ret; 318 bool ret;
348 319
349 if (rcu_read_lock_held_common(&ret)) 320 if (rcu_read_lock_held_common(&ret))
350 return ret; 321 return ret;
351 return lock_is_held(&rcu_lock_map); 322 return lock_is_held(&rcu_lock_map);
352 } 323 }
353 EXPORT_SYMBOL_GPL(rcu_read_lock_held); 324 EXPORT_SYMBOL_GPL(rcu_read_lock_held);
354 325
355 /** 326 /**
356 * rcu_read_lock_bh_held() - might we be in RC 327 * rcu_read_lock_bh_held() - might we be in RCU-bh read-side critical section?
357 * 328 *
358 * Check for bottom half being disabled, which 329 * Check for bottom half being disabled, which covers both the
359 * CONFIG_PROVE_RCU and not cases. Note that 330 * CONFIG_PROVE_RCU and not cases. Note that if someone uses
360 * rcu_read_lock_bh(), but then later enables 331 * rcu_read_lock_bh(), but then later enables BH, lockdep (if enabled)
361 * will show the situation. This is useful fo 332 * will show the situation. This is useful for debug checks in functions
362 * that require that they be called within an 333 * that require that they be called within an RCU read-side critical
363 * section. 334 * section.
364 * 335 *
365 * Check debug_lockdep_rcu_enabled() to preven 336 * Check debug_lockdep_rcu_enabled() to prevent false positives during boot.
366 * 337 *
367 * Note that rcu_read_lock_bh() is disallowed 338 * Note that rcu_read_lock_bh() is disallowed if the CPU is either idle or
368 * offline from an RCU perspective, so check f 339 * offline from an RCU perspective, so check for those as well.
369 */ 340 */
370 int rcu_read_lock_bh_held(void) 341 int rcu_read_lock_bh_held(void)
371 { 342 {
372 bool ret; 343 bool ret;
373 344
374 if (rcu_read_lock_held_common(&ret)) 345 if (rcu_read_lock_held_common(&ret))
375 return ret; 346 return ret;
376 return in_softirq() || irqs_disabled() 347 return in_softirq() || irqs_disabled();
377 } 348 }
378 EXPORT_SYMBOL_GPL(rcu_read_lock_bh_held); 349 EXPORT_SYMBOL_GPL(rcu_read_lock_bh_held);
379 350
380 int rcu_read_lock_any_held(void) 351 int rcu_read_lock_any_held(void)
381 { 352 {
382 bool ret; 353 bool ret;
383 354
384 if (rcu_read_lock_held_common(&ret)) 355 if (rcu_read_lock_held_common(&ret))
385 return ret; 356 return ret;
386 if (lock_is_held(&rcu_lock_map) || 357 if (lock_is_held(&rcu_lock_map) ||
387 lock_is_held(&rcu_bh_lock_map) || 358 lock_is_held(&rcu_bh_lock_map) ||
388 lock_is_held(&rcu_sched_lock_map)) 359 lock_is_held(&rcu_sched_lock_map))
389 return 1; 360 return 1;
390 return !preemptible(); 361 return !preemptible();
391 } 362 }
392 EXPORT_SYMBOL_GPL(rcu_read_lock_any_held); 363 EXPORT_SYMBOL_GPL(rcu_read_lock_any_held);
393 364
394 #endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ 365 #endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
395 366
396 /** 367 /**
397 * wakeme_after_rcu() - Callback function to a 368 * wakeme_after_rcu() - Callback function to awaken a task after grace period
398 * @head: Pointer to rcu_head member within rc 369 * @head: Pointer to rcu_head member within rcu_synchronize structure
399 * 370 *
400 * Awaken the corresponding task now that a gr 371 * Awaken the corresponding task now that a grace period has elapsed.
401 */ 372 */
402 void wakeme_after_rcu(struct rcu_head *head) 373 void wakeme_after_rcu(struct rcu_head *head)
403 { 374 {
404 struct rcu_synchronize *rcu; 375 struct rcu_synchronize *rcu;
405 376
406 rcu = container_of(head, struct rcu_sy 377 rcu = container_of(head, struct rcu_synchronize, head);
407 complete(&rcu->completion); 378 complete(&rcu->completion);
408 } 379 }
409 EXPORT_SYMBOL_GPL(wakeme_after_rcu); 380 EXPORT_SYMBOL_GPL(wakeme_after_rcu);
410 381
411 void __wait_rcu_gp(bool checktiny, unsigned in !! 382 void __wait_rcu_gp(bool checktiny, int n, call_rcu_func_t *crcu_array,
412 struct rcu_synchronize *rs_ 383 struct rcu_synchronize *rs_array)
413 { 384 {
414 int i; 385 int i;
415 int j; 386 int j;
416 387
417 /* Initialize and register callbacks f 388 /* Initialize and register callbacks for each crcu_array element. */
418 for (i = 0; i < n; i++) { 389 for (i = 0; i < n; i++) {
419 if (checktiny && 390 if (checktiny &&
420 (crcu_array[i] == call_rcu 391 (crcu_array[i] == call_rcu)) {
421 might_sleep(); 392 might_sleep();
422 continue; 393 continue;
423 } 394 }
424 for (j = 0; j < i; j++) 395 for (j = 0; j < i; j++)
425 if (crcu_array[j] == c 396 if (crcu_array[j] == crcu_array[i])
426 break; 397 break;
427 if (j == i) { 398 if (j == i) {
428 init_rcu_head_on_stack 399 init_rcu_head_on_stack(&rs_array[i].head);
429 init_completion(&rs_ar 400 init_completion(&rs_array[i].completion);
430 (crcu_array[i])(&rs_ar 401 (crcu_array[i])(&rs_array[i].head, wakeme_after_rcu);
431 } 402 }
432 } 403 }
433 404
434 /* Wait for all callbacks to be invoke 405 /* Wait for all callbacks to be invoked. */
435 for (i = 0; i < n; i++) { 406 for (i = 0; i < n; i++) {
436 if (checktiny && 407 if (checktiny &&
437 (crcu_array[i] == call_rcu 408 (crcu_array[i] == call_rcu))
438 continue; 409 continue;
439 for (j = 0; j < i; j++) 410 for (j = 0; j < i; j++)
440 if (crcu_array[j] == c 411 if (crcu_array[j] == crcu_array[i])
441 break; 412 break;
442 if (j == i) { 413 if (j == i) {
443 wait_for_completion_st !! 414 wait_for_completion(&rs_array[i].completion);
444 destroy_rcu_head_on_st 415 destroy_rcu_head_on_stack(&rs_array[i].head);
445 } 416 }
446 } 417 }
447 } 418 }
448 EXPORT_SYMBOL_GPL(__wait_rcu_gp); 419 EXPORT_SYMBOL_GPL(__wait_rcu_gp);
449 420
450 void finish_rcuwait(struct rcuwait *w) <<
451 { <<
452 rcu_assign_pointer(w->task, NULL); <<
453 __set_current_state(TASK_RUNNING); <<
454 } <<
455 EXPORT_SYMBOL_GPL(finish_rcuwait); <<
456 <<
457 #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD 421 #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
458 void init_rcu_head(struct rcu_head *head) 422 void init_rcu_head(struct rcu_head *head)
459 { 423 {
460 debug_object_init(head, &rcuhead_debug 424 debug_object_init(head, &rcuhead_debug_descr);
461 } 425 }
462 EXPORT_SYMBOL_GPL(init_rcu_head); 426 EXPORT_SYMBOL_GPL(init_rcu_head);
463 427
464 void destroy_rcu_head(struct rcu_head *head) 428 void destroy_rcu_head(struct rcu_head *head)
465 { 429 {
466 debug_object_free(head, &rcuhead_debug 430 debug_object_free(head, &rcuhead_debug_descr);
467 } 431 }
468 EXPORT_SYMBOL_GPL(destroy_rcu_head); 432 EXPORT_SYMBOL_GPL(destroy_rcu_head);
469 433
470 static bool rcuhead_is_static_object(void *add 434 static bool rcuhead_is_static_object(void *addr)
471 { 435 {
472 return true; 436 return true;
473 } 437 }
474 438
475 /** 439 /**
476 * init_rcu_head_on_stack() - initialize on-st 440 * init_rcu_head_on_stack() - initialize on-stack rcu_head for debugobjects
477 * @head: pointer to rcu_head structure to be 441 * @head: pointer to rcu_head structure to be initialized
478 * 442 *
479 * This function informs debugobjects of a new 443 * This function informs debugobjects of a new rcu_head structure that
480 * has been allocated as an auto variable on t 444 * has been allocated as an auto variable on the stack. This function
481 * is not required for rcu_head structures tha 445 * is not required for rcu_head structures that are statically defined or
482 * that are dynamically allocated on the heap. 446 * that are dynamically allocated on the heap. This function has no
483 * effect for !CONFIG_DEBUG_OBJECTS_RCU_HEAD k 447 * effect for !CONFIG_DEBUG_OBJECTS_RCU_HEAD kernel builds.
484 */ 448 */
485 void init_rcu_head_on_stack(struct rcu_head *h 449 void init_rcu_head_on_stack(struct rcu_head *head)
486 { 450 {
487 debug_object_init_on_stack(head, &rcuh 451 debug_object_init_on_stack(head, &rcuhead_debug_descr);
488 } 452 }
489 EXPORT_SYMBOL_GPL(init_rcu_head_on_stack); 453 EXPORT_SYMBOL_GPL(init_rcu_head_on_stack);
490 454
491 /** 455 /**
492 * destroy_rcu_head_on_stack() - destroy on-st 456 * destroy_rcu_head_on_stack() - destroy on-stack rcu_head for debugobjects
493 * @head: pointer to rcu_head structure to be 457 * @head: pointer to rcu_head structure to be initialized
494 * 458 *
495 * This function informs debugobjects that an 459 * This function informs debugobjects that an on-stack rcu_head structure
496 * is about to go out of scope. As with init_ 460 * is about to go out of scope. As with init_rcu_head_on_stack(), this
497 * function is not required for rcu_head struc 461 * function is not required for rcu_head structures that are statically
498 * defined or that are dynamically allocated o 462 * defined or that are dynamically allocated on the heap. Also as with
499 * init_rcu_head_on_stack(), this function has 463 * init_rcu_head_on_stack(), this function has no effect for
500 * !CONFIG_DEBUG_OBJECTS_RCU_HEAD kernel build 464 * !CONFIG_DEBUG_OBJECTS_RCU_HEAD kernel builds.
501 */ 465 */
502 void destroy_rcu_head_on_stack(struct rcu_head 466 void destroy_rcu_head_on_stack(struct rcu_head *head)
503 { 467 {
504 debug_object_free(head, &rcuhead_debug 468 debug_object_free(head, &rcuhead_debug_descr);
505 } 469 }
506 EXPORT_SYMBOL_GPL(destroy_rcu_head_on_stack); 470 EXPORT_SYMBOL_GPL(destroy_rcu_head_on_stack);
507 471
508 const struct debug_obj_descr rcuhead_debug_des !! 472 struct debug_obj_descr rcuhead_debug_descr = {
509 .name = "rcu_head", 473 .name = "rcu_head",
510 .is_static_object = rcuhead_is_static_ 474 .is_static_object = rcuhead_is_static_object,
511 }; 475 };
512 EXPORT_SYMBOL_GPL(rcuhead_debug_descr); 476 EXPORT_SYMBOL_GPL(rcuhead_debug_descr);
513 #endif /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD 477 #endif /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */
514 478
515 #if defined(CONFIG_TREE_RCU) || defined(CONFIG 479 #if defined(CONFIG_TREE_RCU) || defined(CONFIG_RCU_TRACE)
516 void do_trace_rcu_torture_read(const char *rcu 480 void do_trace_rcu_torture_read(const char *rcutorturename, struct rcu_head *rhp,
517 unsigned long s 481 unsigned long secs,
518 unsigned long c 482 unsigned long c_old, unsigned long c)
519 { 483 {
520 trace_rcu_torture_read(rcutorturename, 484 trace_rcu_torture_read(rcutorturename, rhp, secs, c_old, c);
521 } 485 }
522 EXPORT_SYMBOL_GPL(do_trace_rcu_torture_read); 486 EXPORT_SYMBOL_GPL(do_trace_rcu_torture_read);
523 #else 487 #else
524 #define do_trace_rcu_torture_read(rcutorturena 488 #define do_trace_rcu_torture_read(rcutorturename, rhp, secs, c_old, c) \
525 do { } while (0) 489 do { } while (0)
526 #endif 490 #endif
527 491
528 #if IS_ENABLED(CONFIG_RCU_TORTURE_TEST) || IS_ !! 492 #if IS_ENABLED(CONFIG_RCU_TORTURE_TEST) || IS_MODULE(CONFIG_RCU_TORTURE_TEST)
529 /* Get rcutorture access to sched_setaffinity( 493 /* Get rcutorture access to sched_setaffinity(). */
530 long torture_sched_setaffinity(pid_t pid, cons !! 494 long rcutorture_sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
531 { 495 {
532 int ret; 496 int ret;
533 497
534 ret = sched_setaffinity(pid, in_mask); 498 ret = sched_setaffinity(pid, in_mask);
535 WARN_ONCE(ret, "%s: sched_setaffinity( !! 499 WARN_ONCE(ret, "%s: sched_setaffinity() returned %d\n", __func__, ret);
536 return ret; 500 return ret;
537 } 501 }
538 EXPORT_SYMBOL_GPL(torture_sched_setaffinity); !! 502 EXPORT_SYMBOL_GPL(rcutorture_sched_setaffinity);
539 #endif 503 #endif
540 504
541 int rcu_cpu_stall_notifiers __read_mostly; // <<
542 EXPORT_SYMBOL_GPL(rcu_cpu_stall_notifiers); <<
543 <<
544 #ifdef CONFIG_RCU_STALL_COMMON 505 #ifdef CONFIG_RCU_STALL_COMMON
545 int rcu_cpu_stall_ftrace_dump __read_mostly; 506 int rcu_cpu_stall_ftrace_dump __read_mostly;
546 module_param(rcu_cpu_stall_ftrace_dump, int, 0 507 module_param(rcu_cpu_stall_ftrace_dump, int, 0644);
547 #ifdef CONFIG_RCU_CPU_STALL_NOTIFIER <<
548 module_param(rcu_cpu_stall_notifiers, int, 044 <<
549 #endif // #ifdef CONFIG_RCU_CPU_STALL_NOTIFIER <<
550 int rcu_cpu_stall_suppress __read_mostly; // ! 508 int rcu_cpu_stall_suppress __read_mostly; // !0 = suppress stall warnings.
551 EXPORT_SYMBOL_GPL(rcu_cpu_stall_suppress); 509 EXPORT_SYMBOL_GPL(rcu_cpu_stall_suppress);
552 module_param(rcu_cpu_stall_suppress, int, 0644 510 module_param(rcu_cpu_stall_suppress, int, 0644);
553 int rcu_cpu_stall_timeout __read_mostly = CONF 511 int rcu_cpu_stall_timeout __read_mostly = CONFIG_RCU_CPU_STALL_TIMEOUT;
554 module_param(rcu_cpu_stall_timeout, int, 0644) 512 module_param(rcu_cpu_stall_timeout, int, 0644);
555 int rcu_exp_cpu_stall_timeout __read_mostly = <<
556 module_param(rcu_exp_cpu_stall_timeout, int, 0 <<
557 int rcu_cpu_stall_cputime __read_mostly = IS_E <<
558 module_param(rcu_cpu_stall_cputime, int, 0644) <<
559 bool rcu_exp_stall_task_details __read_mostly; <<
560 module_param(rcu_exp_stall_task_details, bool, <<
561 #endif /* #ifdef CONFIG_RCU_STALL_COMMON */ 513 #endif /* #ifdef CONFIG_RCU_STALL_COMMON */
562 514
563 // Suppress boot-time RCU CPU stall warnings a 515 // Suppress boot-time RCU CPU stall warnings and rcutorture writer stall
564 // warnings. Also used by rcutorture even if 516 // warnings. Also used by rcutorture even if stall warnings are excluded.
565 int rcu_cpu_stall_suppress_at_boot __read_most 517 int rcu_cpu_stall_suppress_at_boot __read_mostly; // !0 = suppress boot stalls.
566 EXPORT_SYMBOL_GPL(rcu_cpu_stall_suppress_at_bo 518 EXPORT_SYMBOL_GPL(rcu_cpu_stall_suppress_at_boot);
567 module_param(rcu_cpu_stall_suppress_at_boot, i 519 module_param(rcu_cpu_stall_suppress_at_boot, int, 0444);
568 520
569 /** <<
570 * get_completed_synchronize_rcu - Return a pr <<
571 * <<
572 * Returns a value that will always be treated <<
573 * poll_state_synchronize_rcu() as a cookie wh <<
574 * completed. <<
575 */ <<
576 unsigned long get_completed_synchronize_rcu(vo <<
577 { <<
578 return RCU_GET_STATE_COMPLETED; <<
579 } <<
580 EXPORT_SYMBOL_GPL(get_completed_synchronize_rc <<
581 <<
582 #ifdef CONFIG_PROVE_RCU 521 #ifdef CONFIG_PROVE_RCU
583 522
584 /* 523 /*
585 * Early boot self test parameters. 524 * Early boot self test parameters.
586 */ 525 */
587 static bool rcu_self_test; 526 static bool rcu_self_test;
588 module_param(rcu_self_test, bool, 0444); 527 module_param(rcu_self_test, bool, 0444);
589 528
590 static int rcu_self_test_counter; 529 static int rcu_self_test_counter;
591 530
592 static void test_callback(struct rcu_head *r) 531 static void test_callback(struct rcu_head *r)
593 { 532 {
594 rcu_self_test_counter++; 533 rcu_self_test_counter++;
595 pr_info("RCU test callback executed %d 534 pr_info("RCU test callback executed %d\n", rcu_self_test_counter);
596 } 535 }
597 536
598 DEFINE_STATIC_SRCU(early_srcu); 537 DEFINE_STATIC_SRCU(early_srcu);
599 static unsigned long early_srcu_cookie; <<
600 538
601 struct early_boot_kfree_rcu { 539 struct early_boot_kfree_rcu {
602 struct rcu_head rh; 540 struct rcu_head rh;
603 }; 541 };
604 542
605 static void early_boot_test_call_rcu(void) 543 static void early_boot_test_call_rcu(void)
606 { 544 {
607 static struct rcu_head head; 545 static struct rcu_head head;
608 int idx; <<
609 static struct rcu_head shead; 546 static struct rcu_head shead;
610 struct early_boot_kfree_rcu *rhp; 547 struct early_boot_kfree_rcu *rhp;
611 548
612 idx = srcu_down_read(&early_srcu); <<
613 srcu_up_read(&early_srcu, idx); <<
614 call_rcu(&head, test_callback); 549 call_rcu(&head, test_callback);
615 early_srcu_cookie = start_poll_synchro !! 550 if (IS_ENABLED(CONFIG_SRCU))
616 call_srcu(&early_srcu, &shead, test_ca !! 551 call_srcu(&early_srcu, &shead, test_callback);
617 rhp = kmalloc(sizeof(*rhp), GFP_KERNEL 552 rhp = kmalloc(sizeof(*rhp), GFP_KERNEL);
618 if (!WARN_ON_ONCE(!rhp)) 553 if (!WARN_ON_ONCE(!rhp))
619 kfree_rcu(rhp, rh); 554 kfree_rcu(rhp, rh);
620 } 555 }
621 556
622 void rcu_early_boot_tests(void) 557 void rcu_early_boot_tests(void)
623 { 558 {
624 pr_info("Running RCU self tests\n"); 559 pr_info("Running RCU self tests\n");
625 560
626 if (rcu_self_test) 561 if (rcu_self_test)
627 early_boot_test_call_rcu(); 562 early_boot_test_call_rcu();
628 rcu_test_sync_prims(); 563 rcu_test_sync_prims();
629 } 564 }
630 565
631 static int rcu_verify_early_boot_tests(void) 566 static int rcu_verify_early_boot_tests(void)
632 { 567 {
633 int ret = 0; 568 int ret = 0;
634 int early_boot_test_counter = 0; 569 int early_boot_test_counter = 0;
635 570
636 if (rcu_self_test) { 571 if (rcu_self_test) {
637 early_boot_test_counter++; 572 early_boot_test_counter++;
638 rcu_barrier(); 573 rcu_barrier();
639 early_boot_test_counter++; !! 574 if (IS_ENABLED(CONFIG_SRCU)) {
640 srcu_barrier(&early_srcu); !! 575 early_boot_test_counter++;
641 WARN_ON_ONCE(!poll_state_synch !! 576 srcu_barrier(&early_srcu);
642 cleanup_srcu_struct(&early_src !! 577 }
643 } 578 }
644 if (rcu_self_test_counter != early_boo 579 if (rcu_self_test_counter != early_boot_test_counter) {
645 WARN_ON(1); 580 WARN_ON(1);
646 ret = -1; 581 ret = -1;
647 } 582 }
648 583
649 return ret; 584 return ret;
650 } 585 }
651 late_initcall(rcu_verify_early_boot_tests); 586 late_initcall(rcu_verify_early_boot_tests);
652 #else 587 #else
653 void rcu_early_boot_tests(void) {} 588 void rcu_early_boot_tests(void) {}
654 #endif /* CONFIG_PROVE_RCU */ 589 #endif /* CONFIG_PROVE_RCU */
655 590
656 #include "tasks.h" 591 #include "tasks.h"
657 592
658 #ifndef CONFIG_TINY_RCU 593 #ifndef CONFIG_TINY_RCU
659 594
660 /* 595 /*
661 * Print any significant non-default boot-time 596 * Print any significant non-default boot-time settings.
662 */ 597 */
663 void __init rcupdate_announce_bootup_oddness(v 598 void __init rcupdate_announce_bootup_oddness(void)
664 { 599 {
665 if (rcu_normal) 600 if (rcu_normal)
666 pr_info("\tNo expedited grace 601 pr_info("\tNo expedited grace period (rcu_normal).\n");
667 else if (rcu_normal_after_boot) 602 else if (rcu_normal_after_boot)
668 pr_info("\tNo expedited grace 603 pr_info("\tNo expedited grace period (rcu_normal_after_boot).\n");
669 else if (rcu_expedited) 604 else if (rcu_expedited)
670 pr_info("\tAll grace periods a 605 pr_info("\tAll grace periods are expedited (rcu_expedited).\n");
671 if (rcu_cpu_stall_suppress) 606 if (rcu_cpu_stall_suppress)
672 pr_info("\tRCU CPU stall warni 607 pr_info("\tRCU CPU stall warnings suppressed (rcu_cpu_stall_suppress).\n");
673 if (rcu_cpu_stall_timeout != CONFIG_RC 608 if (rcu_cpu_stall_timeout != CONFIG_RCU_CPU_STALL_TIMEOUT)
674 pr_info("\tRCU CPU stall warni 609 pr_info("\tRCU CPU stall warnings timeout set to %d (rcu_cpu_stall_timeout).\n", rcu_cpu_stall_timeout);
675 rcu_tasks_bootup_oddness(); 610 rcu_tasks_bootup_oddness();
676 } 611 }
677 612
678 #endif /* #ifndef CONFIG_TINY_RCU */ 613 #endif /* #ifndef CONFIG_TINY_RCU */
679 614
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.