1 // SPDX-License-Identifier: GPL-2.0+ 1 // SPDX-License-Identifier: GPL-2.0+
2 // 2 //
3 // Scalability test comparing RCU vs other mec 3 // Scalability test comparing RCU vs other mechanisms
4 // for acquiring references on objects. 4 // for acquiring references on objects.
5 // 5 //
6 // Copyright (C) Google, 2020. 6 // Copyright (C) Google, 2020.
7 // 7 //
8 // Author: Joel Fernandes <joel@joelfernandes. 8 // Author: Joel Fernandes <joel@joelfernandes.org>
9 9
10 #define pr_fmt(fmt) fmt 10 #define pr_fmt(fmt) fmt
11 11
12 #include <linux/atomic.h> 12 #include <linux/atomic.h>
13 #include <linux/bitops.h> 13 #include <linux/bitops.h>
14 #include <linux/completion.h> 14 #include <linux/completion.h>
15 #include <linux/cpu.h> 15 #include <linux/cpu.h>
16 #include <linux/delay.h> 16 #include <linux/delay.h>
17 #include <linux/err.h> 17 #include <linux/err.h>
18 #include <linux/init.h> 18 #include <linux/init.h>
19 #include <linux/interrupt.h> 19 #include <linux/interrupt.h>
20 #include <linux/kthread.h> 20 #include <linux/kthread.h>
21 #include <linux/kernel.h> 21 #include <linux/kernel.h>
22 #include <linux/mm.h> 22 #include <linux/mm.h>
23 #include <linux/module.h> 23 #include <linux/module.h>
24 #include <linux/moduleparam.h> 24 #include <linux/moduleparam.h>
25 #include <linux/notifier.h> 25 #include <linux/notifier.h>
26 #include <linux/percpu.h> 26 #include <linux/percpu.h>
27 #include <linux/rcupdate.h> 27 #include <linux/rcupdate.h>
28 #include <linux/rcupdate_trace.h> 28 #include <linux/rcupdate_trace.h>
29 #include <linux/reboot.h> 29 #include <linux/reboot.h>
30 #include <linux/sched.h> 30 #include <linux/sched.h>
31 #include <linux/seq_buf.h> 31 #include <linux/seq_buf.h>
32 #include <linux/spinlock.h> 32 #include <linux/spinlock.h>
33 #include <linux/smp.h> 33 #include <linux/smp.h>
34 #include <linux/stat.h> 34 #include <linux/stat.h>
35 #include <linux/srcu.h> 35 #include <linux/srcu.h>
36 #include <linux/slab.h> 36 #include <linux/slab.h>
37 #include <linux/torture.h> 37 #include <linux/torture.h>
38 #include <linux/types.h> 38 #include <linux/types.h>
39 39
40 #include "rcu.h" 40 #include "rcu.h"
41 41
42 #define SCALE_FLAG "-ref-scale: " 42 #define SCALE_FLAG "-ref-scale: "
43 43
44 #define SCALEOUT(s, x...) \ 44 #define SCALEOUT(s, x...) \
45 pr_alert("%s" SCALE_FLAG s, scale_type 45 pr_alert("%s" SCALE_FLAG s, scale_type, ## x)
46 46
47 #define VERBOSE_SCALEOUT(s, x...) \ 47 #define VERBOSE_SCALEOUT(s, x...) \
48 do { \ 48 do { \
49 if (verbose) \ 49 if (verbose) \
50 pr_alert("%s" SCALE_FL 50 pr_alert("%s" SCALE_FLAG s "\n", scale_type, ## x); \
51 } while (0) 51 } while (0)
52 52
53 static atomic_t verbose_batch_ctr; 53 static atomic_t verbose_batch_ctr;
54 54
55 #define VERBOSE_SCALEOUT_BATCH(s, x...) 55 #define VERBOSE_SCALEOUT_BATCH(s, x...) \
56 do { 56 do { \
57 if (verbose && 57 if (verbose && \
58 (verbose_batched <= 0 || 58 (verbose_batched <= 0 || \
59 !(atomic_inc_return(&verbose_batc 59 !(atomic_inc_return(&verbose_batch_ctr) % verbose_batched))) { \
60 schedule_timeout_uninterruptib 60 schedule_timeout_uninterruptible(1); \
61 pr_alert("%s" SCALE_FLAG s "\n 61 pr_alert("%s" SCALE_FLAG s "\n", scale_type, ## x); \
62 } 62 } \
63 } while (0) 63 } while (0)
64 64
65 #define SCALEOUT_ERRSTRING(s, x...) pr_alert(" 65 #define SCALEOUT_ERRSTRING(s, x...) pr_alert("%s" SCALE_FLAG "!!! " s "\n", scale_type, ## x)
66 66
67 MODULE_DESCRIPTION("Scalability test for objec 67 MODULE_DESCRIPTION("Scalability test for object reference mechanisms");
68 MODULE_LICENSE("GPL"); 68 MODULE_LICENSE("GPL");
69 MODULE_AUTHOR("Joel Fernandes (Google) <joel@j 69 MODULE_AUTHOR("Joel Fernandes (Google) <joel@joelfernandes.org>");
70 70
71 static char *scale_type = "rcu"; 71 static char *scale_type = "rcu";
72 module_param(scale_type, charp, 0444); 72 module_param(scale_type, charp, 0444);
73 MODULE_PARM_DESC(scale_type, "Type of test (rc 73 MODULE_PARM_DESC(scale_type, "Type of test (rcu, srcu, refcnt, rwsem, rwlock.");
74 74
75 torture_param(int, verbose, 0, "Enable verbose 75 torture_param(int, verbose, 0, "Enable verbose debugging printk()s");
76 torture_param(int, verbose_batched, 0, "Batch 76 torture_param(int, verbose_batched, 0, "Batch verbose debugging printk()s");
77 77
78 // Wait until there are multiple CPUs before s 78 // Wait until there are multiple CPUs before starting test.
79 torture_param(int, holdoff, IS_BUILTIN(CONFIG_ 79 torture_param(int, holdoff, IS_BUILTIN(CONFIG_RCU_REF_SCALE_TEST) ? 10 : 0,
80 "Holdoff time before test start 80 "Holdoff time before test start (s)");
81 // Number of typesafe_lookup structures, that 81 // Number of typesafe_lookup structures, that is, the degree of concurrency.
82 torture_param(long, lookup_instances, 0, "Numb 82 torture_param(long, lookup_instances, 0, "Number of typesafe_lookup structures.");
83 // Number of loops per experiment, all readers 83 // Number of loops per experiment, all readers execute operations concurrently.
84 torture_param(long, loops, 10000, "Number of l 84 torture_param(long, loops, 10000, "Number of loops per experiment.");
85 // Number of readers, with -1 defaulting to ab 85 // Number of readers, with -1 defaulting to about 75% of the CPUs.
86 torture_param(int, nreaders, -1, "Number of re 86 torture_param(int, nreaders, -1, "Number of readers, -1 for 75% of CPUs.");
87 // Number of runs. 87 // Number of runs.
88 torture_param(int, nruns, 30, "Number of exper 88 torture_param(int, nruns, 30, "Number of experiments to run.");
89 // Reader delay in nanoseconds, 0 for no delay 89 // Reader delay in nanoseconds, 0 for no delay.
90 torture_param(int, readdelay, 0, "Read-side de 90 torture_param(int, readdelay, 0, "Read-side delay in nanoseconds.");
91 91
92 #ifdef MODULE 92 #ifdef MODULE
93 # define REFSCALE_SHUTDOWN 0 93 # define REFSCALE_SHUTDOWN 0
94 #else 94 #else
95 # define REFSCALE_SHUTDOWN 1 95 # define REFSCALE_SHUTDOWN 1
96 #endif 96 #endif
97 97
98 torture_param(bool, shutdown, REFSCALE_SHUTDOW 98 torture_param(bool, shutdown, REFSCALE_SHUTDOWN,
99 "Shutdown at end of scalability 99 "Shutdown at end of scalability tests.");
100 100
101 struct reader_task { 101 struct reader_task {
102 struct task_struct *task; 102 struct task_struct *task;
103 int start_reader; 103 int start_reader;
104 wait_queue_head_t wq; 104 wait_queue_head_t wq;
105 u64 last_duration_ns; 105 u64 last_duration_ns;
106 }; 106 };
107 107
108 static struct task_struct *shutdown_task; 108 static struct task_struct *shutdown_task;
109 static wait_queue_head_t shutdown_wq; 109 static wait_queue_head_t shutdown_wq;
110 110
111 static struct task_struct *main_task; 111 static struct task_struct *main_task;
112 static wait_queue_head_t main_wq; 112 static wait_queue_head_t main_wq;
113 static int shutdown_start; 113 static int shutdown_start;
114 114
115 static struct reader_task *reader_tasks; 115 static struct reader_task *reader_tasks;
116 116
117 // Number of readers that are part of the curr 117 // Number of readers that are part of the current experiment.
118 static atomic_t nreaders_exp; 118 static atomic_t nreaders_exp;
119 119
120 // Use to wait for all threads to start. 120 // Use to wait for all threads to start.
121 static atomic_t n_init; 121 static atomic_t n_init;
122 static atomic_t n_started; 122 static atomic_t n_started;
123 static atomic_t n_warmedup; 123 static atomic_t n_warmedup;
124 static atomic_t n_cooleddown; 124 static atomic_t n_cooleddown;
125 125
126 // Track which experiment is currently running 126 // Track which experiment is currently running.
127 static int exp_idx; 127 static int exp_idx;
128 128
129 // Operations vector for selecting different t 129 // Operations vector for selecting different types of tests.
130 struct ref_scale_ops { 130 struct ref_scale_ops {
131 bool (*init)(void); 131 bool (*init)(void);
132 void (*cleanup)(void); 132 void (*cleanup)(void);
133 void (*readsection)(const int nloops); 133 void (*readsection)(const int nloops);
134 void (*delaysection)(const int nloops, 134 void (*delaysection)(const int nloops, const int udl, const int ndl);
135 const char *name; 135 const char *name;
136 }; 136 };
137 137
138 static const struct ref_scale_ops *cur_ops; 138 static const struct ref_scale_ops *cur_ops;
139 139
140 static void un_delay(const int udl, const int 140 static void un_delay(const int udl, const int ndl)
141 { 141 {
142 if (udl) 142 if (udl)
143 udelay(udl); 143 udelay(udl);
144 if (ndl) 144 if (ndl)
145 ndelay(ndl); 145 ndelay(ndl);
146 } 146 }
147 147
148 static void ref_rcu_read_section(const int nlo 148 static void ref_rcu_read_section(const int nloops)
149 { 149 {
150 int i; 150 int i;
151 151
152 for (i = nloops; i >= 0; i--) { 152 for (i = nloops; i >= 0; i--) {
153 rcu_read_lock(); 153 rcu_read_lock();
154 rcu_read_unlock(); 154 rcu_read_unlock();
155 } 155 }
156 } 156 }
157 157
158 static void ref_rcu_delay_section(const int nl 158 static void ref_rcu_delay_section(const int nloops, const int udl, const int ndl)
159 { 159 {
160 int i; 160 int i;
161 161
162 for (i = nloops; i >= 0; i--) { 162 for (i = nloops; i >= 0; i--) {
163 rcu_read_lock(); 163 rcu_read_lock();
164 un_delay(udl, ndl); 164 un_delay(udl, ndl);
165 rcu_read_unlock(); 165 rcu_read_unlock();
166 } 166 }
167 } 167 }
168 168
169 static bool rcu_sync_scale_init(void) 169 static bool rcu_sync_scale_init(void)
170 { 170 {
171 return true; 171 return true;
172 } 172 }
173 173
174 static const struct ref_scale_ops rcu_ops = { 174 static const struct ref_scale_ops rcu_ops = {
175 .init = rcu_sync_scale_init, 175 .init = rcu_sync_scale_init,
176 .readsection = ref_rcu_read_section 176 .readsection = ref_rcu_read_section,
177 .delaysection = ref_rcu_delay_sectio 177 .delaysection = ref_rcu_delay_section,
178 .name = "rcu" 178 .name = "rcu"
179 }; 179 };
180 180
181 // Definitions for SRCU ref scale testing. 181 // Definitions for SRCU ref scale testing.
182 DEFINE_STATIC_SRCU(srcu_refctl_scale); 182 DEFINE_STATIC_SRCU(srcu_refctl_scale);
183 static struct srcu_struct *srcu_ctlp = &srcu_r 183 static struct srcu_struct *srcu_ctlp = &srcu_refctl_scale;
184 184
185 static void srcu_ref_scale_read_section(const 185 static void srcu_ref_scale_read_section(const int nloops)
186 { 186 {
187 int i; 187 int i;
188 int idx; 188 int idx;
189 189
190 for (i = nloops; i >= 0; i--) { 190 for (i = nloops; i >= 0; i--) {
191 idx = srcu_read_lock(srcu_ctlp 191 idx = srcu_read_lock(srcu_ctlp);
192 srcu_read_unlock(srcu_ctlp, id 192 srcu_read_unlock(srcu_ctlp, idx);
193 } 193 }
194 } 194 }
195 195
196 static void srcu_ref_scale_delay_section(const 196 static void srcu_ref_scale_delay_section(const int nloops, const int udl, const int ndl)
197 { 197 {
198 int i; 198 int i;
199 int idx; 199 int idx;
200 200
201 for (i = nloops; i >= 0; i--) { 201 for (i = nloops; i >= 0; i--) {
202 idx = srcu_read_lock(srcu_ctlp 202 idx = srcu_read_lock(srcu_ctlp);
203 un_delay(udl, ndl); 203 un_delay(udl, ndl);
204 srcu_read_unlock(srcu_ctlp, id 204 srcu_read_unlock(srcu_ctlp, idx);
205 } 205 }
206 } 206 }
207 207
208 static const struct ref_scale_ops srcu_ops = { 208 static const struct ref_scale_ops srcu_ops = {
209 .init = rcu_sync_scale_init, 209 .init = rcu_sync_scale_init,
210 .readsection = srcu_ref_scale_read_ 210 .readsection = srcu_ref_scale_read_section,
211 .delaysection = srcu_ref_scale_delay 211 .delaysection = srcu_ref_scale_delay_section,
212 .name = "srcu" 212 .name = "srcu"
213 }; 213 };
214 214
215 #ifdef CONFIG_TASKS_RCU 215 #ifdef CONFIG_TASKS_RCU
216 216
217 // Definitions for RCU Tasks ref scale testing 217 // Definitions for RCU Tasks ref scale testing: Empty read markers.
218 // These definitions also work for RCU Rude re 218 // These definitions also work for RCU Rude readers.
219 static void rcu_tasks_ref_scale_read_section(c 219 static void rcu_tasks_ref_scale_read_section(const int nloops)
220 { 220 {
221 int i; 221 int i;
222 222
223 for (i = nloops; i >= 0; i--) 223 for (i = nloops; i >= 0; i--)
224 continue; 224 continue;
225 } 225 }
226 226
227 static void rcu_tasks_ref_scale_delay_section( 227 static void rcu_tasks_ref_scale_delay_section(const int nloops, const int udl, const int ndl)
228 { 228 {
229 int i; 229 int i;
230 230
231 for (i = nloops; i >= 0; i--) 231 for (i = nloops; i >= 0; i--)
232 un_delay(udl, ndl); 232 un_delay(udl, ndl);
233 } 233 }
234 234
235 static const struct ref_scale_ops rcu_tasks_op 235 static const struct ref_scale_ops rcu_tasks_ops = {
236 .init = rcu_sync_scale_init, 236 .init = rcu_sync_scale_init,
237 .readsection = rcu_tasks_ref_scale_ 237 .readsection = rcu_tasks_ref_scale_read_section,
238 .delaysection = rcu_tasks_ref_scale_ 238 .delaysection = rcu_tasks_ref_scale_delay_section,
239 .name = "rcu-tasks" 239 .name = "rcu-tasks"
240 }; 240 };
241 241
242 #define RCU_TASKS_OPS &rcu_tasks_ops, 242 #define RCU_TASKS_OPS &rcu_tasks_ops,
243 243
244 #else // #ifdef CONFIG_TASKS_RCU 244 #else // #ifdef CONFIG_TASKS_RCU
245 245
246 #define RCU_TASKS_OPS 246 #define RCU_TASKS_OPS
247 247
248 #endif // #else // #ifdef CONFIG_TASKS_RCU 248 #endif // #else // #ifdef CONFIG_TASKS_RCU
249 249
250 #ifdef CONFIG_TASKS_TRACE_RCU 250 #ifdef CONFIG_TASKS_TRACE_RCU
251 251
252 // Definitions for RCU Tasks Trace ref scale t 252 // Definitions for RCU Tasks Trace ref scale testing.
253 static void rcu_trace_ref_scale_read_section(c 253 static void rcu_trace_ref_scale_read_section(const int nloops)
254 { 254 {
255 int i; 255 int i;
256 256
257 for (i = nloops; i >= 0; i--) { 257 for (i = nloops; i >= 0; i--) {
258 rcu_read_lock_trace(); 258 rcu_read_lock_trace();
259 rcu_read_unlock_trace(); 259 rcu_read_unlock_trace();
260 } 260 }
261 } 261 }
262 262
263 static void rcu_trace_ref_scale_delay_section( 263 static void rcu_trace_ref_scale_delay_section(const int nloops, const int udl, const int ndl)
264 { 264 {
265 int i; 265 int i;
266 266
267 for (i = nloops; i >= 0; i--) { 267 for (i = nloops; i >= 0; i--) {
268 rcu_read_lock_trace(); 268 rcu_read_lock_trace();
269 un_delay(udl, ndl); 269 un_delay(udl, ndl);
270 rcu_read_unlock_trace(); 270 rcu_read_unlock_trace();
271 } 271 }
272 } 272 }
273 273
274 static const struct ref_scale_ops rcu_trace_op 274 static const struct ref_scale_ops rcu_trace_ops = {
275 .init = rcu_sync_scale_init, 275 .init = rcu_sync_scale_init,
276 .readsection = rcu_trace_ref_scale_ 276 .readsection = rcu_trace_ref_scale_read_section,
277 .delaysection = rcu_trace_ref_scale_ 277 .delaysection = rcu_trace_ref_scale_delay_section,
278 .name = "rcu-trace" 278 .name = "rcu-trace"
279 }; 279 };
280 280
281 #define RCU_TRACE_OPS &rcu_trace_ops, 281 #define RCU_TRACE_OPS &rcu_trace_ops,
282 282
283 #else // #ifdef CONFIG_TASKS_TRACE_RCU 283 #else // #ifdef CONFIG_TASKS_TRACE_RCU
284 284
285 #define RCU_TRACE_OPS 285 #define RCU_TRACE_OPS
286 286
287 #endif // #else // #ifdef CONFIG_TASKS_TRACE_R 287 #endif // #else // #ifdef CONFIG_TASKS_TRACE_RCU
288 288
289 // Definitions for reference count 289 // Definitions for reference count
290 static atomic_t refcnt; 290 static atomic_t refcnt;
291 291
292 static void ref_refcnt_section(const int nloop 292 static void ref_refcnt_section(const int nloops)
293 { 293 {
294 int i; 294 int i;
295 295
296 for (i = nloops; i >= 0; i--) { 296 for (i = nloops; i >= 0; i--) {
297 atomic_inc(&refcnt); 297 atomic_inc(&refcnt);
298 atomic_dec(&refcnt); 298 atomic_dec(&refcnt);
299 } 299 }
300 } 300 }
301 301
302 static void ref_refcnt_delay_section(const int 302 static void ref_refcnt_delay_section(const int nloops, const int udl, const int ndl)
303 { 303 {
304 int i; 304 int i;
305 305
306 for (i = nloops; i >= 0; i--) { 306 for (i = nloops; i >= 0; i--) {
307 atomic_inc(&refcnt); 307 atomic_inc(&refcnt);
308 un_delay(udl, ndl); 308 un_delay(udl, ndl);
309 atomic_dec(&refcnt); 309 atomic_dec(&refcnt);
310 } 310 }
311 } 311 }
312 312
313 static const struct ref_scale_ops refcnt_ops = 313 static const struct ref_scale_ops refcnt_ops = {
314 .init = rcu_sync_scale_init, 314 .init = rcu_sync_scale_init,
315 .readsection = ref_refcnt_section, 315 .readsection = ref_refcnt_section,
316 .delaysection = ref_refcnt_delay_sec 316 .delaysection = ref_refcnt_delay_section,
317 .name = "refcnt" 317 .name = "refcnt"
318 }; 318 };
319 319
320 // Definitions for rwlock 320 // Definitions for rwlock
321 static rwlock_t test_rwlock; 321 static rwlock_t test_rwlock;
322 322
323 static bool ref_rwlock_init(void) 323 static bool ref_rwlock_init(void)
324 { 324 {
325 rwlock_init(&test_rwlock); 325 rwlock_init(&test_rwlock);
326 return true; 326 return true;
327 } 327 }
328 328
329 static void ref_rwlock_section(const int nloop 329 static void ref_rwlock_section(const int nloops)
330 { 330 {
331 int i; 331 int i;
332 332
333 for (i = nloops; i >= 0; i--) { 333 for (i = nloops; i >= 0; i--) {
334 read_lock(&test_rwlock); 334 read_lock(&test_rwlock);
335 read_unlock(&test_rwlock); 335 read_unlock(&test_rwlock);
336 } 336 }
337 } 337 }
338 338
339 static void ref_rwlock_delay_section(const int 339 static void ref_rwlock_delay_section(const int nloops, const int udl, const int ndl)
340 { 340 {
341 int i; 341 int i;
342 342
343 for (i = nloops; i >= 0; i--) { 343 for (i = nloops; i >= 0; i--) {
344 read_lock(&test_rwlock); 344 read_lock(&test_rwlock);
345 un_delay(udl, ndl); 345 un_delay(udl, ndl);
346 read_unlock(&test_rwlock); 346 read_unlock(&test_rwlock);
347 } 347 }
348 } 348 }
349 349
350 static const struct ref_scale_ops rwlock_ops = 350 static const struct ref_scale_ops rwlock_ops = {
351 .init = ref_rwlock_init, 351 .init = ref_rwlock_init,
352 .readsection = ref_rwlock_section, 352 .readsection = ref_rwlock_section,
353 .delaysection = ref_rwlock_delay_sec 353 .delaysection = ref_rwlock_delay_section,
354 .name = "rwlock" 354 .name = "rwlock"
355 }; 355 };
356 356
357 // Definitions for rwsem 357 // Definitions for rwsem
358 static struct rw_semaphore test_rwsem; 358 static struct rw_semaphore test_rwsem;
359 359
360 static bool ref_rwsem_init(void) 360 static bool ref_rwsem_init(void)
361 { 361 {
362 init_rwsem(&test_rwsem); 362 init_rwsem(&test_rwsem);
363 return true; 363 return true;
364 } 364 }
365 365
366 static void ref_rwsem_section(const int nloops 366 static void ref_rwsem_section(const int nloops)
367 { 367 {
368 int i; 368 int i;
369 369
370 for (i = nloops; i >= 0; i--) { 370 for (i = nloops; i >= 0; i--) {
371 down_read(&test_rwsem); 371 down_read(&test_rwsem);
372 up_read(&test_rwsem); 372 up_read(&test_rwsem);
373 } 373 }
374 } 374 }
375 375
376 static void ref_rwsem_delay_section(const int 376 static void ref_rwsem_delay_section(const int nloops, const int udl, const int ndl)
377 { 377 {
378 int i; 378 int i;
379 379
380 for (i = nloops; i >= 0; i--) { 380 for (i = nloops; i >= 0; i--) {
381 down_read(&test_rwsem); 381 down_read(&test_rwsem);
382 un_delay(udl, ndl); 382 un_delay(udl, ndl);
383 up_read(&test_rwsem); 383 up_read(&test_rwsem);
384 } 384 }
385 } 385 }
386 386
387 static const struct ref_scale_ops rwsem_ops = 387 static const struct ref_scale_ops rwsem_ops = {
388 .init = ref_rwsem_init, 388 .init = ref_rwsem_init,
389 .readsection = ref_rwsem_section, 389 .readsection = ref_rwsem_section,
390 .delaysection = ref_rwsem_delay_sect 390 .delaysection = ref_rwsem_delay_section,
391 .name = "rwsem" 391 .name = "rwsem"
392 }; 392 };
393 393
394 // Definitions for global spinlock 394 // Definitions for global spinlock
395 static DEFINE_RAW_SPINLOCK(test_lock); 395 static DEFINE_RAW_SPINLOCK(test_lock);
396 396
397 static void ref_lock_section(const int nloops) 397 static void ref_lock_section(const int nloops)
398 { 398 {
399 int i; 399 int i;
400 400
401 preempt_disable(); 401 preempt_disable();
402 for (i = nloops; i >= 0; i--) { 402 for (i = nloops; i >= 0; i--) {
403 raw_spin_lock(&test_lock); 403 raw_spin_lock(&test_lock);
404 raw_spin_unlock(&test_lock); 404 raw_spin_unlock(&test_lock);
405 } 405 }
406 preempt_enable(); 406 preempt_enable();
407 } 407 }
408 408
409 static void ref_lock_delay_section(const int n 409 static void ref_lock_delay_section(const int nloops, const int udl, const int ndl)
410 { 410 {
411 int i; 411 int i;
412 412
413 preempt_disable(); 413 preempt_disable();
414 for (i = nloops; i >= 0; i--) { 414 for (i = nloops; i >= 0; i--) {
415 raw_spin_lock(&test_lock); 415 raw_spin_lock(&test_lock);
416 un_delay(udl, ndl); 416 un_delay(udl, ndl);
417 raw_spin_unlock(&test_lock); 417 raw_spin_unlock(&test_lock);
418 } 418 }
419 preempt_enable(); 419 preempt_enable();
420 } 420 }
421 421
422 static const struct ref_scale_ops lock_ops = { 422 static const struct ref_scale_ops lock_ops = {
423 .readsection = ref_lock_section, 423 .readsection = ref_lock_section,
424 .delaysection = ref_lock_delay_secti 424 .delaysection = ref_lock_delay_section,
425 .name = "lock" 425 .name = "lock"
426 }; 426 };
427 427
428 // Definitions for global irq-save spinlock 428 // Definitions for global irq-save spinlock
429 429
430 static void ref_lock_irq_section(const int nlo 430 static void ref_lock_irq_section(const int nloops)
431 { 431 {
432 unsigned long flags; 432 unsigned long flags;
433 int i; 433 int i;
434 434
435 preempt_disable(); 435 preempt_disable();
436 for (i = nloops; i >= 0; i--) { 436 for (i = nloops; i >= 0; i--) {
437 raw_spin_lock_irqsave(&test_lo 437 raw_spin_lock_irqsave(&test_lock, flags);
438 raw_spin_unlock_irqrestore(&te 438 raw_spin_unlock_irqrestore(&test_lock, flags);
439 } 439 }
440 preempt_enable(); 440 preempt_enable();
441 } 441 }
442 442
443 static void ref_lock_irq_delay_section(const i 443 static void ref_lock_irq_delay_section(const int nloops, const int udl, const int ndl)
444 { 444 {
445 unsigned long flags; 445 unsigned long flags;
446 int i; 446 int i;
447 447
448 preempt_disable(); 448 preempt_disable();
449 for (i = nloops; i >= 0; i--) { 449 for (i = nloops; i >= 0; i--) {
450 raw_spin_lock_irqsave(&test_lo 450 raw_spin_lock_irqsave(&test_lock, flags);
451 un_delay(udl, ndl); 451 un_delay(udl, ndl);
452 raw_spin_unlock_irqrestore(&te 452 raw_spin_unlock_irqrestore(&test_lock, flags);
453 } 453 }
454 preempt_enable(); 454 preempt_enable();
455 } 455 }
456 456
457 static const struct ref_scale_ops lock_irq_ops 457 static const struct ref_scale_ops lock_irq_ops = {
458 .readsection = ref_lock_irq_section 458 .readsection = ref_lock_irq_section,
459 .delaysection = ref_lock_irq_delay_s 459 .delaysection = ref_lock_irq_delay_section,
460 .name = "lock-irq" 460 .name = "lock-irq"
461 }; 461 };
462 462
463 // Definitions acquire-release. 463 // Definitions acquire-release.
464 static DEFINE_PER_CPU(unsigned long, test_acqr 464 static DEFINE_PER_CPU(unsigned long, test_acqrel);
465 465
466 static void ref_acqrel_section(const int nloop 466 static void ref_acqrel_section(const int nloops)
467 { 467 {
468 unsigned long x; 468 unsigned long x;
469 int i; 469 int i;
470 470
471 preempt_disable(); 471 preempt_disable();
472 for (i = nloops; i >= 0; i--) { 472 for (i = nloops; i >= 0; i--) {
473 x = smp_load_acquire(this_cpu_ 473 x = smp_load_acquire(this_cpu_ptr(&test_acqrel));
474 smp_store_release(this_cpu_ptr 474 smp_store_release(this_cpu_ptr(&test_acqrel), x + 1);
475 } 475 }
476 preempt_enable(); 476 preempt_enable();
477 } 477 }
478 478
479 static void ref_acqrel_delay_section(const int 479 static void ref_acqrel_delay_section(const int nloops, const int udl, const int ndl)
480 { 480 {
481 unsigned long x; 481 unsigned long x;
482 int i; 482 int i;
483 483
484 preempt_disable(); 484 preempt_disable();
485 for (i = nloops; i >= 0; i--) { 485 for (i = nloops; i >= 0; i--) {
486 x = smp_load_acquire(this_cpu_ 486 x = smp_load_acquire(this_cpu_ptr(&test_acqrel));
487 un_delay(udl, ndl); 487 un_delay(udl, ndl);
488 smp_store_release(this_cpu_ptr 488 smp_store_release(this_cpu_ptr(&test_acqrel), x + 1);
489 } 489 }
490 preempt_enable(); 490 preempt_enable();
491 } 491 }
492 492
493 static const struct ref_scale_ops acqrel_ops = 493 static const struct ref_scale_ops acqrel_ops = {
494 .readsection = ref_acqrel_section, 494 .readsection = ref_acqrel_section,
495 .delaysection = ref_acqrel_delay_sec 495 .delaysection = ref_acqrel_delay_section,
496 .name = "acqrel" 496 .name = "acqrel"
497 }; 497 };
498 498
499 static volatile u64 stopopts; 499 static volatile u64 stopopts;
500 500
501 static void ref_clock_section(const int nloops 501 static void ref_clock_section(const int nloops)
502 { 502 {
503 u64 x = 0; 503 u64 x = 0;
504 int i; 504 int i;
505 505
506 preempt_disable(); 506 preempt_disable();
507 for (i = nloops; i >= 0; i--) 507 for (i = nloops; i >= 0; i--)
508 x += ktime_get_real_fast_ns(); 508 x += ktime_get_real_fast_ns();
509 preempt_enable(); 509 preempt_enable();
510 stopopts = x; 510 stopopts = x;
511 } 511 }
512 512
513 static void ref_clock_delay_section(const int 513 static void ref_clock_delay_section(const int nloops, const int udl, const int ndl)
514 { 514 {
515 u64 x = 0; 515 u64 x = 0;
516 int i; 516 int i;
517 517
518 preempt_disable(); 518 preempt_disable();
519 for (i = nloops; i >= 0; i--) { 519 for (i = nloops; i >= 0; i--) {
520 x += ktime_get_real_fast_ns(); 520 x += ktime_get_real_fast_ns();
521 un_delay(udl, ndl); 521 un_delay(udl, ndl);
522 } 522 }
523 preempt_enable(); 523 preempt_enable();
524 stopopts = x; 524 stopopts = x;
525 } 525 }
526 526
527 static const struct ref_scale_ops clock_ops = 527 static const struct ref_scale_ops clock_ops = {
528 .readsection = ref_clock_section, 528 .readsection = ref_clock_section,
529 .delaysection = ref_clock_delay_sect 529 .delaysection = ref_clock_delay_section,
530 .name = "clock" 530 .name = "clock"
531 }; 531 };
532 532
533 static void ref_jiffies_section(const int nloo 533 static void ref_jiffies_section(const int nloops)
534 { 534 {
535 u64 x = 0; 535 u64 x = 0;
536 int i; 536 int i;
537 537
538 preempt_disable(); 538 preempt_disable();
539 for (i = nloops; i >= 0; i--) 539 for (i = nloops; i >= 0; i--)
540 x += jiffies; 540 x += jiffies;
541 preempt_enable(); 541 preempt_enable();
542 stopopts = x; 542 stopopts = x;
543 } 543 }
544 544
545 static void ref_jiffies_delay_section(const in 545 static void ref_jiffies_delay_section(const int nloops, const int udl, const int ndl)
546 { 546 {
547 u64 x = 0; 547 u64 x = 0;
548 int i; 548 int i;
549 549
550 preempt_disable(); 550 preempt_disable();
551 for (i = nloops; i >= 0; i--) { 551 for (i = nloops; i >= 0; i--) {
552 x += jiffies; 552 x += jiffies;
553 un_delay(udl, ndl); 553 un_delay(udl, ndl);
554 } 554 }
555 preempt_enable(); 555 preempt_enable();
556 stopopts = x; 556 stopopts = x;
557 } 557 }
558 558
559 static const struct ref_scale_ops jiffies_ops 559 static const struct ref_scale_ops jiffies_ops = {
560 .readsection = ref_jiffies_section, 560 .readsection = ref_jiffies_section,
561 .delaysection = ref_jiffies_delay_se 561 .delaysection = ref_jiffies_delay_section,
562 .name = "jiffies" 562 .name = "jiffies"
563 }; 563 };
564 564
565 ////////////////////////////////////////////// 565 ////////////////////////////////////////////////////////////////////////
566 // 566 //
567 // Methods leveraging SLAB_TYPESAFE_BY_RCU. 567 // Methods leveraging SLAB_TYPESAFE_BY_RCU.
568 // 568 //
569 569
570 // Item to look up in a typesafe manner. Arra 570 // Item to look up in a typesafe manner. Array of pointers to these.
571 struct refscale_typesafe { 571 struct refscale_typesafe {
572 atomic_t rts_refctr; // Used by all f 572 atomic_t rts_refctr; // Used by all flavors
573 spinlock_t rts_lock; 573 spinlock_t rts_lock;
574 seqlock_t rts_seqlock; 574 seqlock_t rts_seqlock;
575 unsigned int a; 575 unsigned int a;
576 unsigned int b; 576 unsigned int b;
577 }; 577 };
578 578
579 static struct kmem_cache *typesafe_kmem_cachep 579 static struct kmem_cache *typesafe_kmem_cachep;
580 static struct refscale_typesafe **rtsarray; 580 static struct refscale_typesafe **rtsarray;
581 static long rtsarray_size; 581 static long rtsarray_size;
582 static DEFINE_TORTURE_RANDOM_PERCPU(refscale_r 582 static DEFINE_TORTURE_RANDOM_PERCPU(refscale_rand);
583 static bool (*rts_acquire)(struct refscale_typ 583 static bool (*rts_acquire)(struct refscale_typesafe *rtsp, unsigned int *start);
584 static bool (*rts_release)(struct refscale_typ 584 static bool (*rts_release)(struct refscale_typesafe *rtsp, unsigned int start);
585 585
586 // Conditionally acquire an explicit in-struct 586 // Conditionally acquire an explicit in-structure reference count.
587 static bool typesafe_ref_acquire(struct refsca 587 static bool typesafe_ref_acquire(struct refscale_typesafe *rtsp, unsigned int *start)
588 { 588 {
589 return atomic_inc_not_zero(&rtsp->rts_ 589 return atomic_inc_not_zero(&rtsp->rts_refctr);
590 } 590 }
591 591
592 // Unconditionally release an explicit in-stru 592 // Unconditionally release an explicit in-structure reference count.
593 static bool typesafe_ref_release(struct refsca 593 static bool typesafe_ref_release(struct refscale_typesafe *rtsp, unsigned int start)
594 { 594 {
595 if (!atomic_dec_return(&rtsp->rts_refc 595 if (!atomic_dec_return(&rtsp->rts_refctr)) {
596 WRITE_ONCE(rtsp->a, rtsp->a + 596 WRITE_ONCE(rtsp->a, rtsp->a + 1);
597 kmem_cache_free(typesafe_kmem_ 597 kmem_cache_free(typesafe_kmem_cachep, rtsp);
598 } 598 }
599 return true; 599 return true;
600 } 600 }
601 601
602 // Unconditionally acquire an explicit in-stru 602 // Unconditionally acquire an explicit in-structure spinlock.
603 static bool typesafe_lock_acquire(struct refsc 603 static bool typesafe_lock_acquire(struct refscale_typesafe *rtsp, unsigned int *start)
604 { 604 {
605 spin_lock(&rtsp->rts_lock); 605 spin_lock(&rtsp->rts_lock);
606 return true; 606 return true;
607 } 607 }
608 608
609 // Unconditionally release an explicit in-stru 609 // Unconditionally release an explicit in-structure spinlock.
610 static bool typesafe_lock_release(struct refsc 610 static bool typesafe_lock_release(struct refscale_typesafe *rtsp, unsigned int start)
611 { 611 {
612 spin_unlock(&rtsp->rts_lock); 612 spin_unlock(&rtsp->rts_lock);
613 return true; 613 return true;
614 } 614 }
615 615
616 // Unconditionally acquire an explicit in-stru 616 // Unconditionally acquire an explicit in-structure sequence lock.
617 static bool typesafe_seqlock_acquire(struct re 617 static bool typesafe_seqlock_acquire(struct refscale_typesafe *rtsp, unsigned int *start)
618 { 618 {
619 *start = read_seqbegin(&rtsp->rts_seql 619 *start = read_seqbegin(&rtsp->rts_seqlock);
620 return true; 620 return true;
621 } 621 }
622 622
623 // Conditionally release an explicit in-struct 623 // Conditionally release an explicit in-structure sequence lock. Return
624 // true if this release was successful, that i 624 // true if this release was successful, that is, if no retry is required.
625 static bool typesafe_seqlock_release(struct re 625 static bool typesafe_seqlock_release(struct refscale_typesafe *rtsp, unsigned int start)
626 { 626 {
627 return !read_seqretry(&rtsp->rts_seqlo 627 return !read_seqretry(&rtsp->rts_seqlock, start);
628 } 628 }
629 629
630 // Do a read-side critical section with the sp 630 // Do a read-side critical section with the specified delay in
631 // microseconds and nanoseconds inserted so as 631 // microseconds and nanoseconds inserted so as to increase probability
632 // of failure. 632 // of failure.
633 static void typesafe_delay_section(const int n 633 static void typesafe_delay_section(const int nloops, const int udl, const int ndl)
634 { 634 {
635 unsigned int a; 635 unsigned int a;
636 unsigned int b; 636 unsigned int b;
637 int i; 637 int i;
638 long idx; 638 long idx;
639 struct refscale_typesafe *rtsp; 639 struct refscale_typesafe *rtsp;
640 unsigned int start; 640 unsigned int start;
641 641
642 for (i = nloops; i >= 0; i--) { 642 for (i = nloops; i >= 0; i--) {
643 preempt_disable(); 643 preempt_disable();
644 idx = torture_random(this_cpu_ 644 idx = torture_random(this_cpu_ptr(&refscale_rand)) % rtsarray_size;
645 preempt_enable(); 645 preempt_enable();
646 retry: 646 retry:
647 rcu_read_lock(); 647 rcu_read_lock();
648 rtsp = rcu_dereference(rtsarra 648 rtsp = rcu_dereference(rtsarray[idx]);
649 a = READ_ONCE(rtsp->a); 649 a = READ_ONCE(rtsp->a);
650 if (!rts_acquire(rtsp, &start) 650 if (!rts_acquire(rtsp, &start)) {
651 rcu_read_unlock(); 651 rcu_read_unlock();
652 goto retry; 652 goto retry;
653 } 653 }
654 if (a != READ_ONCE(rtsp->a)) { 654 if (a != READ_ONCE(rtsp->a)) {
655 (void)rts_release(rtsp 655 (void)rts_release(rtsp, start);
656 rcu_read_unlock(); 656 rcu_read_unlock();
657 goto retry; 657 goto retry;
658 } 658 }
659 un_delay(udl, ndl); 659 un_delay(udl, ndl);
660 b = READ_ONCE(rtsp->a); 660 b = READ_ONCE(rtsp->a);
661 // Remember, seqlock read-side 661 // Remember, seqlock read-side release can fail.
662 if (!rts_release(rtsp, start)) 662 if (!rts_release(rtsp, start)) {
663 rcu_read_unlock(); 663 rcu_read_unlock();
664 goto retry; 664 goto retry;
665 } 665 }
666 WARN_ONCE(a != b, "Re-read of 666 WARN_ONCE(a != b, "Re-read of ->a changed from %u to %u.\n", a, b);
667 b = rtsp->b; 667 b = rtsp->b;
668 rcu_read_unlock(); 668 rcu_read_unlock();
669 WARN_ON_ONCE(a * a != b); 669 WARN_ON_ONCE(a * a != b);
670 } 670 }
671 } 671 }
672 672
673 // Because the acquisition and release methods 673 // Because the acquisition and release methods are expensive, there
674 // is no point in optimizing away the un_delay 674 // is no point in optimizing away the un_delay() function's two checks.
675 // Thus simply define typesafe_read_section() 675 // Thus simply define typesafe_read_section() as a simple wrapper around
676 // typesafe_delay_section(). 676 // typesafe_delay_section().
677 static void typesafe_read_section(const int nl 677 static void typesafe_read_section(const int nloops)
678 { 678 {
679 typesafe_delay_section(nloops, 0, 0); 679 typesafe_delay_section(nloops, 0, 0);
680 } 680 }
681 681
682 // Allocate and initialize one refscale_typesa 682 // Allocate and initialize one refscale_typesafe structure.
683 static struct refscale_typesafe *typesafe_allo 683 static struct refscale_typesafe *typesafe_alloc_one(void)
684 { 684 {
685 struct refscale_typesafe *rtsp; 685 struct refscale_typesafe *rtsp;
686 686
687 rtsp = kmem_cache_alloc(typesafe_kmem_ 687 rtsp = kmem_cache_alloc(typesafe_kmem_cachep, GFP_KERNEL);
688 if (!rtsp) 688 if (!rtsp)
689 return NULL; 689 return NULL;
690 atomic_set(&rtsp->rts_refctr, 1); 690 atomic_set(&rtsp->rts_refctr, 1);
691 WRITE_ONCE(rtsp->a, rtsp->a + 1); 691 WRITE_ONCE(rtsp->a, rtsp->a + 1);
692 WRITE_ONCE(rtsp->b, rtsp->a * rtsp->a) 692 WRITE_ONCE(rtsp->b, rtsp->a * rtsp->a);
693 return rtsp; 693 return rtsp;
694 } 694 }
695 695
696 // Slab-allocator constructor for refscale_typ 696 // Slab-allocator constructor for refscale_typesafe structures created
697 // out of a new slab of system memory. 697 // out of a new slab of system memory.
698 static void refscale_typesafe_ctor(void *rtsp_ 698 static void refscale_typesafe_ctor(void *rtsp_in)
699 { 699 {
700 struct refscale_typesafe *rtsp = rtsp_ 700 struct refscale_typesafe *rtsp = rtsp_in;
701 701
702 spin_lock_init(&rtsp->rts_lock); 702 spin_lock_init(&rtsp->rts_lock);
703 seqlock_init(&rtsp->rts_seqlock); 703 seqlock_init(&rtsp->rts_seqlock);
704 preempt_disable(); 704 preempt_disable();
705 rtsp->a = torture_random(this_cpu_ptr( 705 rtsp->a = torture_random(this_cpu_ptr(&refscale_rand));
706 preempt_enable(); 706 preempt_enable();
707 } 707 }
708 708
709 static const struct ref_scale_ops typesafe_ref 709 static const struct ref_scale_ops typesafe_ref_ops;
710 static const struct ref_scale_ops typesafe_loc 710 static const struct ref_scale_ops typesafe_lock_ops;
711 static const struct ref_scale_ops typesafe_seq 711 static const struct ref_scale_ops typesafe_seqlock_ops;
712 712
713 // Initialize for a typesafe test. 713 // Initialize for a typesafe test.
714 static bool typesafe_init(void) 714 static bool typesafe_init(void)
715 { 715 {
716 long idx; 716 long idx;
717 long si = lookup_instances; 717 long si = lookup_instances;
718 718
719 typesafe_kmem_cachep = kmem_cache_crea 719 typesafe_kmem_cachep = kmem_cache_create("refscale_typesafe",
720 720 sizeof(struct refscale_typesafe), sizeof(void *),
721 721 SLAB_TYPESAFE_BY_RCU, refscale_typesafe_ctor);
722 if (!typesafe_kmem_cachep) 722 if (!typesafe_kmem_cachep)
723 return false; 723 return false;
724 if (si < 0) 724 if (si < 0)
725 si = -si * nr_cpu_ids; 725 si = -si * nr_cpu_ids;
726 else if (si == 0) 726 else if (si == 0)
727 si = nr_cpu_ids; 727 si = nr_cpu_ids;
728 rtsarray_size = si; 728 rtsarray_size = si;
729 rtsarray = kcalloc(si, sizeof(*rtsarra 729 rtsarray = kcalloc(si, sizeof(*rtsarray), GFP_KERNEL);
730 if (!rtsarray) 730 if (!rtsarray)
731 return false; 731 return false;
732 for (idx = 0; idx < rtsarray_size; idx 732 for (idx = 0; idx < rtsarray_size; idx++) {
733 rtsarray[idx] = typesafe_alloc 733 rtsarray[idx] = typesafe_alloc_one();
734 if (!rtsarray[idx]) 734 if (!rtsarray[idx])
735 return false; 735 return false;
736 } 736 }
737 if (cur_ops == &typesafe_ref_ops) { 737 if (cur_ops == &typesafe_ref_ops) {
738 rts_acquire = typesafe_ref_acq 738 rts_acquire = typesafe_ref_acquire;
739 rts_release = typesafe_ref_rel 739 rts_release = typesafe_ref_release;
740 } else if (cur_ops == &typesafe_lock_o 740 } else if (cur_ops == &typesafe_lock_ops) {
741 rts_acquire = typesafe_lock_ac 741 rts_acquire = typesafe_lock_acquire;
742 rts_release = typesafe_lock_re 742 rts_release = typesafe_lock_release;
743 } else if (cur_ops == &typesafe_seqloc 743 } else if (cur_ops == &typesafe_seqlock_ops) {
744 rts_acquire = typesafe_seqlock 744 rts_acquire = typesafe_seqlock_acquire;
745 rts_release = typesafe_seqlock 745 rts_release = typesafe_seqlock_release;
746 } else { 746 } else {
747 WARN_ON_ONCE(1); 747 WARN_ON_ONCE(1);
748 return false; 748 return false;
749 } 749 }
750 return true; 750 return true;
751 } 751 }
752 752
753 // Clean up after a typesafe test. 753 // Clean up after a typesafe test.
754 static void typesafe_cleanup(void) 754 static void typesafe_cleanup(void)
755 { 755 {
756 long idx; 756 long idx;
757 757
758 if (rtsarray) { 758 if (rtsarray) {
759 for (idx = 0; idx < rtsarray_s 759 for (idx = 0; idx < rtsarray_size; idx++)
760 kmem_cache_free(typesa 760 kmem_cache_free(typesafe_kmem_cachep, rtsarray[idx]);
761 kfree(rtsarray); 761 kfree(rtsarray);
762 rtsarray = NULL; 762 rtsarray = NULL;
763 rtsarray_size = 0; 763 rtsarray_size = 0;
764 } 764 }
765 kmem_cache_destroy(typesafe_kmem_cache 765 kmem_cache_destroy(typesafe_kmem_cachep);
766 typesafe_kmem_cachep = NULL; 766 typesafe_kmem_cachep = NULL;
767 rts_acquire = NULL; 767 rts_acquire = NULL;
768 rts_release = NULL; 768 rts_release = NULL;
769 } 769 }
770 770
771 // The typesafe_init() function distinguishes 771 // The typesafe_init() function distinguishes these structures by address.
772 static const struct ref_scale_ops typesafe_ref 772 static const struct ref_scale_ops typesafe_ref_ops = {
773 .init = typesafe_init, 773 .init = typesafe_init,
774 .cleanup = typesafe_cleanup, 774 .cleanup = typesafe_cleanup,
775 .readsection = typesafe_read_sectio 775 .readsection = typesafe_read_section,
776 .delaysection = typesafe_delay_secti 776 .delaysection = typesafe_delay_section,
777 .name = "typesafe_ref" 777 .name = "typesafe_ref"
778 }; 778 };
779 779
780 static const struct ref_scale_ops typesafe_loc 780 static const struct ref_scale_ops typesafe_lock_ops = {
781 .init = typesafe_init, 781 .init = typesafe_init,
782 .cleanup = typesafe_cleanup, 782 .cleanup = typesafe_cleanup,
783 .readsection = typesafe_read_sectio 783 .readsection = typesafe_read_section,
784 .delaysection = typesafe_delay_secti 784 .delaysection = typesafe_delay_section,
785 .name = "typesafe_lock" 785 .name = "typesafe_lock"
786 }; 786 };
787 787
788 static const struct ref_scale_ops typesafe_seq 788 static const struct ref_scale_ops typesafe_seqlock_ops = {
789 .init = typesafe_init, 789 .init = typesafe_init,
790 .cleanup = typesafe_cleanup, 790 .cleanup = typesafe_cleanup,
791 .readsection = typesafe_read_sectio 791 .readsection = typesafe_read_section,
792 .delaysection = typesafe_delay_secti 792 .delaysection = typesafe_delay_section,
793 .name = "typesafe_seqlock" 793 .name = "typesafe_seqlock"
794 }; 794 };
795 795
796 static void rcu_scale_one_reader(void) 796 static void rcu_scale_one_reader(void)
797 { 797 {
798 if (readdelay <= 0) 798 if (readdelay <= 0)
799 cur_ops->readsection(loops); 799 cur_ops->readsection(loops);
800 else 800 else
801 cur_ops->delaysection(loops, r 801 cur_ops->delaysection(loops, readdelay / 1000, readdelay % 1000);
802 } 802 }
803 803
804 // Reader kthread. Repeatedly does empty RCU 804 // Reader kthread. Repeatedly does empty RCU read-side
805 // critical section, minimizing update-side in 805 // critical section, minimizing update-side interference.
806 static int 806 static int
807 ref_scale_reader(void *arg) 807 ref_scale_reader(void *arg)
808 { 808 {
809 unsigned long flags; 809 unsigned long flags;
810 long me = (long)arg; 810 long me = (long)arg;
811 struct reader_task *rt = &(reader_task 811 struct reader_task *rt = &(reader_tasks[me]);
812 u64 start; 812 u64 start;
813 s64 duration; 813 s64 duration;
814 814
815 VERBOSE_SCALEOUT_BATCH("ref_scale_read 815 VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: task started", me);
816 WARN_ON_ONCE(set_cpus_allowed_ptr(curr 816 WARN_ON_ONCE(set_cpus_allowed_ptr(current, cpumask_of(me % nr_cpu_ids)));
817 set_user_nice(current, MAX_NICE); 817 set_user_nice(current, MAX_NICE);
818 atomic_inc(&n_init); 818 atomic_inc(&n_init);
819 if (holdoff) 819 if (holdoff)
820 schedule_timeout_interruptible 820 schedule_timeout_interruptible(holdoff * HZ);
821 repeat: 821 repeat:
822 VERBOSE_SCALEOUT_BATCH("ref_scale_read 822 VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: waiting to start next experiment on cpu %d", me, raw_smp_processor_id());
823 823
824 // Wait for signal that this reader ca 824 // Wait for signal that this reader can start.
825 wait_event(rt->wq, (atomic_read(&nread 825 wait_event(rt->wq, (atomic_read(&nreaders_exp) && smp_load_acquire(&rt->start_reader)) ||
826 torture_must_stop() 826 torture_must_stop());
827 827
828 if (torture_must_stop()) 828 if (torture_must_stop())
829 goto end; 829 goto end;
830 830
831 // Make sure that the CPU is affinitiz 831 // Make sure that the CPU is affinitized appropriately during testing.
832 WARN_ON_ONCE(raw_smp_processor_id() != 832 WARN_ON_ONCE(raw_smp_processor_id() != me);
833 833
834 WRITE_ONCE(rt->start_reader, 0); 834 WRITE_ONCE(rt->start_reader, 0);
835 if (!atomic_dec_return(&n_started)) 835 if (!atomic_dec_return(&n_started))
836 while (atomic_read_acquire(&n_ 836 while (atomic_read_acquire(&n_started))
837 cpu_relax(); 837 cpu_relax();
838 838
839 VERBOSE_SCALEOUT_BATCH("ref_scale_read 839 VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: experiment %d started", me, exp_idx);
840 840
841 841
842 // To reduce noise, do an initial cach 842 // To reduce noise, do an initial cache-warming invocation, check
843 // in, and then keep warming until eve 843 // in, and then keep warming until everyone has checked in.
844 rcu_scale_one_reader(); 844 rcu_scale_one_reader();
845 if (!atomic_dec_return(&n_warmedup)) 845 if (!atomic_dec_return(&n_warmedup))
846 while (atomic_read_acquire(&n_ 846 while (atomic_read_acquire(&n_warmedup))
847 rcu_scale_one_reader() 847 rcu_scale_one_reader();
848 // Also keep interrupts disabled. Thi 848 // Also keep interrupts disabled. This also has the effect
849 // of preventing entries into slow pat 849 // of preventing entries into slow path for rcu_read_unlock().
850 local_irq_save(flags); 850 local_irq_save(flags);
851 start = ktime_get_mono_fast_ns(); 851 start = ktime_get_mono_fast_ns();
852 852
853 rcu_scale_one_reader(); 853 rcu_scale_one_reader();
854 854
855 duration = ktime_get_mono_fast_ns() - 855 duration = ktime_get_mono_fast_ns() - start;
856 local_irq_restore(flags); 856 local_irq_restore(flags);
857 857
858 rt->last_duration_ns = WARN_ON_ONCE(du 858 rt->last_duration_ns = WARN_ON_ONCE(duration < 0) ? 0 : duration;
859 // To reduce runtime-skew noise, do ma 859 // To reduce runtime-skew noise, do maintain-load invocations until
860 // everyone is done. 860 // everyone is done.
861 if (!atomic_dec_return(&n_cooleddown)) 861 if (!atomic_dec_return(&n_cooleddown))
862 while (atomic_read_acquire(&n_ 862 while (atomic_read_acquire(&n_cooleddown))
863 rcu_scale_one_reader() 863 rcu_scale_one_reader();
864 864
865 if (atomic_dec_and_test(&nreaders_exp) 865 if (atomic_dec_and_test(&nreaders_exp))
866 wake_up(&main_wq); 866 wake_up(&main_wq);
867 867
868 VERBOSE_SCALEOUT_BATCH("ref_scale_read 868 VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: experiment %d ended, (readers remaining=%d)",
869 me, exp_idx, a 869 me, exp_idx, atomic_read(&nreaders_exp));
870 870
871 if (!torture_must_stop()) 871 if (!torture_must_stop())
872 goto repeat; 872 goto repeat;
873 end: 873 end:
874 torture_kthread_stopping("ref_scale_re 874 torture_kthread_stopping("ref_scale_reader");
875 return 0; 875 return 0;
876 } 876 }
877 877
878 static void reset_readers(void) 878 static void reset_readers(void)
879 { 879 {
880 int i; 880 int i;
881 struct reader_task *rt; 881 struct reader_task *rt;
882 882
883 for (i = 0; i < nreaders; i++) { 883 for (i = 0; i < nreaders; i++) {
884 rt = &(reader_tasks[i]); 884 rt = &(reader_tasks[i]);
885 885
886 rt->last_duration_ns = 0; 886 rt->last_duration_ns = 0;
887 } 887 }
888 } 888 }
889 889
890 // Print the results of each reader and return 890 // Print the results of each reader and return the sum of all their durations.
891 static u64 process_durations(int n) 891 static u64 process_durations(int n)
892 { 892 {
893 int i; 893 int i;
894 struct reader_task *rt; 894 struct reader_task *rt;
895 struct seq_buf s; 895 struct seq_buf s;
896 char *buf; 896 char *buf;
897 u64 sum = 0; 897 u64 sum = 0;
898 898
899 buf = kmalloc(800 + 64, GFP_KERNEL); 899 buf = kmalloc(800 + 64, GFP_KERNEL);
900 if (!buf) 900 if (!buf)
901 return 0; 901 return 0;
902 seq_buf_init(&s, buf, 800 + 64); 902 seq_buf_init(&s, buf, 800 + 64);
903 903
904 seq_buf_printf(&s, "Experiment #%d (Fo 904 seq_buf_printf(&s, "Experiment #%d (Format: <THREAD-NUM>:<Total loop time in ns>)",
905 exp_idx); 905 exp_idx);
906 906
907 for (i = 0; i < n && !torture_must_sto 907 for (i = 0; i < n && !torture_must_stop(); i++) {
908 rt = &(reader_tasks[i]); 908 rt = &(reader_tasks[i]);
909 909
910 if (i % 5 == 0) 910 if (i % 5 == 0)
911 seq_buf_putc(&s, '\n') 911 seq_buf_putc(&s, '\n');
912 912
913 if (seq_buf_used(&s) >= 800) { 913 if (seq_buf_used(&s) >= 800) {
914 pr_alert("%s", seq_buf 914 pr_alert("%s", seq_buf_str(&s));
915 seq_buf_clear(&s); 915 seq_buf_clear(&s);
916 } 916 }
917 917
918 seq_buf_printf(&s, "%d: %llu\t 918 seq_buf_printf(&s, "%d: %llu\t", i, rt->last_duration_ns);
919 919
920 sum += rt->last_duration_ns; 920 sum += rt->last_duration_ns;
921 } 921 }
922 pr_alert("%s\n", seq_buf_str(&s)); 922 pr_alert("%s\n", seq_buf_str(&s));
923 923
924 kfree(buf); 924 kfree(buf);
925 return sum; 925 return sum;
926 } 926 }
927 927
928 // The main_func is the main orchestrator, it 928 // The main_func is the main orchestrator, it performs a bunch of
929 // experiments. For every experiment, it orde 929 // experiments. For every experiment, it orders all the readers
930 // involved to start and waits for them to fin 930 // involved to start and waits for them to finish the experiment. It
931 // then reads their timestamps and starts the 931 // then reads their timestamps and starts the next experiment. Each
932 // experiment progresses from 1 concurrent rea 932 // experiment progresses from 1 concurrent reader to N of them at which
933 // point all the timestamps are printed. 933 // point all the timestamps are printed.
934 static int main_func(void *arg) 934 static int main_func(void *arg)
935 { 935 {
936 int exp, r; 936 int exp, r;
937 char buf1[64]; 937 char buf1[64];
938 char *buf; 938 char *buf;
939 u64 *result_avg; 939 u64 *result_avg;
940 940
941 set_cpus_allowed_ptr(current, cpumask_ 941 set_cpus_allowed_ptr(current, cpumask_of(nreaders % nr_cpu_ids));
942 set_user_nice(current, MAX_NICE); 942 set_user_nice(current, MAX_NICE);
943 943
944 VERBOSE_SCALEOUT("main_func task start 944 VERBOSE_SCALEOUT("main_func task started");
945 result_avg = kzalloc(nruns * sizeof(*r 945 result_avg = kzalloc(nruns * sizeof(*result_avg), GFP_KERNEL);
946 buf = kzalloc(800 + 64, GFP_KERNEL); 946 buf = kzalloc(800 + 64, GFP_KERNEL);
947 if (!result_avg || !buf) { 947 if (!result_avg || !buf) {
948 SCALEOUT_ERRSTRING("out of mem 948 SCALEOUT_ERRSTRING("out of memory");
949 goto oom_exit; 949 goto oom_exit;
950 } 950 }
951 if (holdoff) 951 if (holdoff)
952 schedule_timeout_interruptible 952 schedule_timeout_interruptible(holdoff * HZ);
953 953
954 // Wait for all threads to start. 954 // Wait for all threads to start.
955 atomic_inc(&n_init); 955 atomic_inc(&n_init);
956 while (atomic_read(&n_init) < nreaders 956 while (atomic_read(&n_init) < nreaders + 1)
957 schedule_timeout_uninterruptib 957 schedule_timeout_uninterruptible(1);
958 958
959 // Start exp readers up per experiment 959 // Start exp readers up per experiment
960 for (exp = 0; exp < nruns && !torture_ 960 for (exp = 0; exp < nruns && !torture_must_stop(); exp++) {
961 if (torture_must_stop()) 961 if (torture_must_stop())
962 goto end; 962 goto end;
963 963
964 reset_readers(); 964 reset_readers();
965 atomic_set(&nreaders_exp, nrea 965 atomic_set(&nreaders_exp, nreaders);
966 atomic_set(&n_started, nreader 966 atomic_set(&n_started, nreaders);
967 atomic_set(&n_warmedup, nreade 967 atomic_set(&n_warmedup, nreaders);
968 atomic_set(&n_cooleddown, nrea 968 atomic_set(&n_cooleddown, nreaders);
969 969
970 exp_idx = exp; 970 exp_idx = exp;
971 971
972 for (r = 0; r < nreaders; r++) 972 for (r = 0; r < nreaders; r++) {
973 smp_store_release(&rea 973 smp_store_release(&reader_tasks[r].start_reader, 1);
974 wake_up(&reader_tasks[ 974 wake_up(&reader_tasks[r].wq);
975 } 975 }
976 976
977 VERBOSE_SCALEOUT("main_func: e 977 VERBOSE_SCALEOUT("main_func: experiment started, waiting for %d readers",
978 nreaders); 978 nreaders);
979 979
980 wait_event(main_wq, 980 wait_event(main_wq,
981 !atomic_read(&nread 981 !atomic_read(&nreaders_exp) || torture_must_stop());
982 982
983 VERBOSE_SCALEOUT("main_func: e 983 VERBOSE_SCALEOUT("main_func: experiment ended");
984 984
985 if (torture_must_stop()) 985 if (torture_must_stop())
986 goto end; 986 goto end;
987 987
988 result_avg[exp] = div_u64(1000 988 result_avg[exp] = div_u64(1000 * process_durations(nreaders), nreaders * loops);
989 } 989 }
990 990
991 // Print the average of all experiment 991 // Print the average of all experiments
992 SCALEOUT("END OF TEST. Calculating ave 992 SCALEOUT("END OF TEST. Calculating average duration per loop (nanoseconds)...\n");
993 993
994 pr_alert("Runs\tTime(ns)\n"); 994 pr_alert("Runs\tTime(ns)\n");
995 for (exp = 0; exp < nruns; exp++) { 995 for (exp = 0; exp < nruns; exp++) {
996 u64 avg; 996 u64 avg;
997 u32 rem; 997 u32 rem;
998 998
999 avg = div_u64_rem(result_avg[e 999 avg = div_u64_rem(result_avg[exp], 1000, &rem);
1000 sprintf(buf1, "%d\t%llu.%03u\ 1000 sprintf(buf1, "%d\t%llu.%03u\n", exp + 1, avg, rem);
1001 strcat(buf, buf1); 1001 strcat(buf, buf1);
1002 if (strlen(buf) >= 800) { 1002 if (strlen(buf) >= 800) {
1003 pr_alert("%s", buf); 1003 pr_alert("%s", buf);
1004 buf[0] = 0; 1004 buf[0] = 0;
1005 } 1005 }
1006 } 1006 }
1007 1007
1008 pr_alert("%s", buf); 1008 pr_alert("%s", buf);
1009 1009
1010 oom_exit: 1010 oom_exit:
1011 // This will shutdown everything incl 1011 // This will shutdown everything including us.
1012 if (shutdown) { 1012 if (shutdown) {
1013 shutdown_start = 1; 1013 shutdown_start = 1;
1014 wake_up(&shutdown_wq); 1014 wake_up(&shutdown_wq);
1015 } 1015 }
1016 1016
1017 // Wait for torture to stop us 1017 // Wait for torture to stop us
1018 while (!torture_must_stop()) 1018 while (!torture_must_stop())
1019 schedule_timeout_uninterrupti 1019 schedule_timeout_uninterruptible(1);
1020 1020
1021 end: 1021 end:
1022 torture_kthread_stopping("main_func") 1022 torture_kthread_stopping("main_func");
1023 kfree(result_avg); 1023 kfree(result_avg);
1024 kfree(buf); 1024 kfree(buf);
1025 return 0; 1025 return 0;
1026 } 1026 }
1027 1027
1028 static void 1028 static void
1029 ref_scale_print_module_parms(const struct ref 1029 ref_scale_print_module_parms(const struct ref_scale_ops *cur_ops, const char *tag)
1030 { 1030 {
1031 pr_alert("%s" SCALE_FLAG 1031 pr_alert("%s" SCALE_FLAG
1032 "--- %s: verbose=%d verbose 1032 "--- %s: verbose=%d verbose_batched=%d shutdown=%d holdoff=%d lookup_instances=%ld loops=%ld nreaders=%d nruns=%d readdelay=%d\n", scale_type, tag,
1033 verbose, verbose_batched, sh 1033 verbose, verbose_batched, shutdown, holdoff, lookup_instances, loops, nreaders, nruns, readdelay);
1034 } 1034 }
1035 1035
1036 static void 1036 static void
1037 ref_scale_cleanup(void) 1037 ref_scale_cleanup(void)
1038 { 1038 {
1039 int i; 1039 int i;
1040 1040
1041 if (torture_cleanup_begin()) 1041 if (torture_cleanup_begin())
1042 return; 1042 return;
1043 1043
1044 if (!cur_ops) { 1044 if (!cur_ops) {
1045 torture_cleanup_end(); 1045 torture_cleanup_end();
1046 return; 1046 return;
1047 } 1047 }
1048 1048
1049 if (reader_tasks) { 1049 if (reader_tasks) {
1050 for (i = 0; i < nreaders; i++ 1050 for (i = 0; i < nreaders; i++)
1051 torture_stop_kthread( 1051 torture_stop_kthread("ref_scale_reader",
1052 1052 reader_tasks[i].task);
1053 } 1053 }
1054 kfree(reader_tasks); 1054 kfree(reader_tasks);
1055 1055
1056 torture_stop_kthread("main_task", mai 1056 torture_stop_kthread("main_task", main_task);
1057 kfree(main_task); 1057 kfree(main_task);
1058 1058
1059 // Do scale-type-specific cleanup ope 1059 // Do scale-type-specific cleanup operations.
1060 if (cur_ops->cleanup != NULL) 1060 if (cur_ops->cleanup != NULL)
1061 cur_ops->cleanup(); 1061 cur_ops->cleanup();
1062 1062
1063 torture_cleanup_end(); 1063 torture_cleanup_end();
1064 } 1064 }
1065 1065
1066 // Shutdown kthread. Just waits to be awaken 1066 // Shutdown kthread. Just waits to be awakened, then shuts down system.
1067 static int 1067 static int
1068 ref_scale_shutdown(void *arg) 1068 ref_scale_shutdown(void *arg)
1069 { 1069 {
1070 wait_event_idle(shutdown_wq, shutdown 1070 wait_event_idle(shutdown_wq, shutdown_start);
1071 1071
1072 smp_mb(); // Wake before output. 1072 smp_mb(); // Wake before output.
1073 ref_scale_cleanup(); 1073 ref_scale_cleanup();
1074 kernel_power_off(); 1074 kernel_power_off();
1075 1075
1076 return -EINVAL; 1076 return -EINVAL;
1077 } 1077 }
1078 1078
1079 static int __init 1079 static int __init
1080 ref_scale_init(void) 1080 ref_scale_init(void)
1081 { 1081 {
1082 long i; 1082 long i;
1083 int firsterr = 0; 1083 int firsterr = 0;
1084 static const struct ref_scale_ops *sc 1084 static const struct ref_scale_ops *scale_ops[] = {
1085 &rcu_ops, &srcu_ops, RCU_TRAC 1085 &rcu_ops, &srcu_ops, RCU_TRACE_OPS RCU_TASKS_OPS &refcnt_ops, &rwlock_ops,
1086 &rwsem_ops, &lock_ops, &lock_ 1086 &rwsem_ops, &lock_ops, &lock_irq_ops, &acqrel_ops, &clock_ops, &jiffies_ops,
1087 &typesafe_ref_ops, &typesafe_ 1087 &typesafe_ref_ops, &typesafe_lock_ops, &typesafe_seqlock_ops,
1088 }; 1088 };
1089 1089
1090 if (!torture_init_begin(scale_type, v 1090 if (!torture_init_begin(scale_type, verbose))
1091 return -EBUSY; 1091 return -EBUSY;
1092 1092
1093 for (i = 0; i < ARRAY_SIZE(scale_ops) 1093 for (i = 0; i < ARRAY_SIZE(scale_ops); i++) {
1094 cur_ops = scale_ops[i]; 1094 cur_ops = scale_ops[i];
1095 if (strcmp(scale_type, cur_op 1095 if (strcmp(scale_type, cur_ops->name) == 0)
1096 break; 1096 break;
1097 } 1097 }
1098 if (i == ARRAY_SIZE(scale_ops)) { 1098 if (i == ARRAY_SIZE(scale_ops)) {
1099 pr_alert("rcu-scale: invalid 1099 pr_alert("rcu-scale: invalid scale type: \"%s\"\n", scale_type);
1100 pr_alert("rcu-scale types:"); 1100 pr_alert("rcu-scale types:");
1101 for (i = 0; i < ARRAY_SIZE(sc 1101 for (i = 0; i < ARRAY_SIZE(scale_ops); i++)
1102 pr_cont(" %s", scale_ 1102 pr_cont(" %s", scale_ops[i]->name);
1103 pr_cont("\n"); 1103 pr_cont("\n");
1104 firsterr = -EINVAL; 1104 firsterr = -EINVAL;
1105 cur_ops = NULL; 1105 cur_ops = NULL;
1106 goto unwind; 1106 goto unwind;
1107 } 1107 }
1108 if (cur_ops->init) 1108 if (cur_ops->init)
1109 if (!cur_ops->init()) { 1109 if (!cur_ops->init()) {
1110 firsterr = -EUCLEAN; 1110 firsterr = -EUCLEAN;
1111 goto unwind; 1111 goto unwind;
1112 } 1112 }
1113 1113
1114 ref_scale_print_module_parms(cur_ops, 1114 ref_scale_print_module_parms(cur_ops, "Start of test");
1115 1115
1116 // Shutdown task 1116 // Shutdown task
1117 if (shutdown) { 1117 if (shutdown) {
1118 init_waitqueue_head(&shutdown 1118 init_waitqueue_head(&shutdown_wq);
1119 firsterr = torture_create_kth 1119 firsterr = torture_create_kthread(ref_scale_shutdown, NULL,
1120 1120 shutdown_task);
1121 if (torture_init_error(firste 1121 if (torture_init_error(firsterr))
1122 goto unwind; 1122 goto unwind;
1123 schedule_timeout_uninterrupti 1123 schedule_timeout_uninterruptible(1);
1124 } 1124 }
1125 1125
1126 // Reader tasks (default to ~75% of o 1126 // Reader tasks (default to ~75% of online CPUs).
1127 if (nreaders < 0) 1127 if (nreaders < 0)
1128 nreaders = (num_online_cpus() 1128 nreaders = (num_online_cpus() >> 1) + (num_online_cpus() >> 2);
1129 if (WARN_ONCE(loops <= 0, "%s: loops 1129 if (WARN_ONCE(loops <= 0, "%s: loops = %ld, adjusted to 1\n", __func__, loops))
1130 loops = 1; 1130 loops = 1;
1131 if (WARN_ONCE(nreaders <= 0, "%s: nre 1131 if (WARN_ONCE(nreaders <= 0, "%s: nreaders = %d, adjusted to 1\n", __func__, nreaders))
1132 nreaders = 1; 1132 nreaders = 1;
1133 if (WARN_ONCE(nruns <= 0, "%s: nruns 1133 if (WARN_ONCE(nruns <= 0, "%s: nruns = %d, adjusted to 1\n", __func__, nruns))
1134 nruns = 1; 1134 nruns = 1;
1135 reader_tasks = kcalloc(nreaders, size 1135 reader_tasks = kcalloc(nreaders, sizeof(reader_tasks[0]),
1136 GFP_KERNEL); 1136 GFP_KERNEL);
1137 if (!reader_tasks) { 1137 if (!reader_tasks) {
1138 SCALEOUT_ERRSTRING("out of me 1138 SCALEOUT_ERRSTRING("out of memory");
1139 firsterr = -ENOMEM; 1139 firsterr = -ENOMEM;
1140 goto unwind; 1140 goto unwind;
1141 } 1141 }
1142 1142
1143 VERBOSE_SCALEOUT("Starting %d reader 1143 VERBOSE_SCALEOUT("Starting %d reader threads", nreaders);
1144 1144
1145 for (i = 0; i < nreaders; i++) { 1145 for (i = 0; i < nreaders; i++) {
1146 init_waitqueue_head(&reader_t 1146 init_waitqueue_head(&reader_tasks[i].wq);
1147 firsterr = torture_create_kth 1147 firsterr = torture_create_kthread(ref_scale_reader, (void *)i,
1148 1148 reader_tasks[i].task);
1149 if (torture_init_error(firste 1149 if (torture_init_error(firsterr))
1150 goto unwind; 1150 goto unwind;
1151 } 1151 }
1152 1152
1153 // Main Task 1153 // Main Task
1154 init_waitqueue_head(&main_wq); 1154 init_waitqueue_head(&main_wq);
1155 firsterr = torture_create_kthread(mai 1155 firsterr = torture_create_kthread(main_func, NULL, main_task);
1156 if (torture_init_error(firsterr)) 1156 if (torture_init_error(firsterr))
1157 goto unwind; 1157 goto unwind;
1158 1158
1159 torture_init_end(); 1159 torture_init_end();
1160 return 0; 1160 return 0;
1161 1161
1162 unwind: 1162 unwind:
1163 torture_init_end(); 1163 torture_init_end();
1164 ref_scale_cleanup(); 1164 ref_scale_cleanup();
1165 if (shutdown) { 1165 if (shutdown) {
1166 WARN_ON(!IS_MODULE(CONFIG_RCU 1166 WARN_ON(!IS_MODULE(CONFIG_RCU_REF_SCALE_TEST));
1167 kernel_power_off(); 1167 kernel_power_off();
1168 } 1168 }
1169 return firsterr; 1169 return firsterr;
1170 } 1170 }
1171 1171
1172 module_init(ref_scale_init); 1172 module_init(ref_scale_init);
1173 module_exit(ref_scale_cleanup); 1173 module_exit(ref_scale_cleanup);
1174 1174
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.