1 // SPDX-License-Identifier: GPL-2.0+ 1 // SPDX-License-Identifier: GPL-2.0+
2 // 2 //
3 // Scalability test comparing RCU vs other mec 3 // Scalability test comparing RCU vs other mechanisms
4 // for acquiring references on objects. 4 // for acquiring references on objects.
5 // 5 //
6 // Copyright (C) Google, 2020. 6 // Copyright (C) Google, 2020.
7 // 7 //
8 // Author: Joel Fernandes <joel@joelfernandes. 8 // Author: Joel Fernandes <joel@joelfernandes.org>
9 9
10 #define pr_fmt(fmt) fmt 10 #define pr_fmt(fmt) fmt
11 11
12 #include <linux/atomic.h> 12 #include <linux/atomic.h>
13 #include <linux/bitops.h> 13 #include <linux/bitops.h>
14 #include <linux/completion.h> 14 #include <linux/completion.h>
15 #include <linux/cpu.h> 15 #include <linux/cpu.h>
16 #include <linux/delay.h> 16 #include <linux/delay.h>
17 #include <linux/err.h> 17 #include <linux/err.h>
18 #include <linux/init.h> 18 #include <linux/init.h>
19 #include <linux/interrupt.h> 19 #include <linux/interrupt.h>
20 #include <linux/kthread.h> 20 #include <linux/kthread.h>
21 #include <linux/kernel.h> 21 #include <linux/kernel.h>
22 #include <linux/mm.h> 22 #include <linux/mm.h>
23 #include <linux/module.h> 23 #include <linux/module.h>
24 #include <linux/moduleparam.h> 24 #include <linux/moduleparam.h>
25 #include <linux/notifier.h> 25 #include <linux/notifier.h>
26 #include <linux/percpu.h> 26 #include <linux/percpu.h>
27 #include <linux/rcupdate.h> 27 #include <linux/rcupdate.h>
28 #include <linux/rcupdate_trace.h> 28 #include <linux/rcupdate_trace.h>
29 #include <linux/reboot.h> 29 #include <linux/reboot.h>
30 #include <linux/sched.h> 30 #include <linux/sched.h>
31 #include <linux/seq_buf.h> <<
32 #include <linux/spinlock.h> 31 #include <linux/spinlock.h>
33 #include <linux/smp.h> 32 #include <linux/smp.h>
34 #include <linux/stat.h> 33 #include <linux/stat.h>
35 #include <linux/srcu.h> 34 #include <linux/srcu.h>
36 #include <linux/slab.h> 35 #include <linux/slab.h>
37 #include <linux/torture.h> 36 #include <linux/torture.h>
38 #include <linux/types.h> 37 #include <linux/types.h>
39 38
40 #include "rcu.h" 39 #include "rcu.h"
41 40
42 #define SCALE_FLAG "-ref-scale: " 41 #define SCALE_FLAG "-ref-scale: "
43 42
44 #define SCALEOUT(s, x...) \ 43 #define SCALEOUT(s, x...) \
45 pr_alert("%s" SCALE_FLAG s, scale_type 44 pr_alert("%s" SCALE_FLAG s, scale_type, ## x)
46 45
47 #define VERBOSE_SCALEOUT(s, x...) \ 46 #define VERBOSE_SCALEOUT(s, x...) \
48 do { \ !! 47 do { if (verbose) pr_alert("%s" SCALE_FLAG s, scale_type, ## x); } while (0)
49 if (verbose) \ <<
50 pr_alert("%s" SCALE_FL <<
51 } while (0) <<
52 48
53 static atomic_t verbose_batch_ctr; 49 static atomic_t verbose_batch_ctr;
54 50
55 #define VERBOSE_SCALEOUT_BATCH(s, x...) 51 #define VERBOSE_SCALEOUT_BATCH(s, x...) \
56 do { 52 do { \
57 if (verbose && 53 if (verbose && \
58 (verbose_batched <= 0 || 54 (verbose_batched <= 0 || \
59 !(atomic_inc_return(&verbose_batc 55 !(atomic_inc_return(&verbose_batch_ctr) % verbose_batched))) { \
60 schedule_timeout_uninterruptib 56 schedule_timeout_uninterruptible(1); \
61 pr_alert("%s" SCALE_FLAG s "\n !! 57 pr_alert("%s" SCALE_FLAG s, scale_type, ## x); \
62 } 58 } \
63 } while (0) 59 } while (0)
64 60
65 #define SCALEOUT_ERRSTRING(s, x...) pr_alert(" !! 61 #define VERBOSE_SCALEOUT_ERRSTRING(s, x...) \
>> 62 do { if (verbose) pr_alert("%s" SCALE_FLAG "!!! " s, scale_type, ## x); } while (0)
66 63
67 MODULE_DESCRIPTION("Scalability test for objec <<
68 MODULE_LICENSE("GPL"); 64 MODULE_LICENSE("GPL");
69 MODULE_AUTHOR("Joel Fernandes (Google) <joel@j 65 MODULE_AUTHOR("Joel Fernandes (Google) <joel@joelfernandes.org>");
70 66
71 static char *scale_type = "rcu"; 67 static char *scale_type = "rcu";
72 module_param(scale_type, charp, 0444); 68 module_param(scale_type, charp, 0444);
73 MODULE_PARM_DESC(scale_type, "Type of test (rc 69 MODULE_PARM_DESC(scale_type, "Type of test (rcu, srcu, refcnt, rwsem, rwlock.");
74 70
75 torture_param(int, verbose, 0, "Enable verbose 71 torture_param(int, verbose, 0, "Enable verbose debugging printk()s");
76 torture_param(int, verbose_batched, 0, "Batch 72 torture_param(int, verbose_batched, 0, "Batch verbose debugging printk()s");
77 73
78 // Wait until there are multiple CPUs before s 74 // Wait until there are multiple CPUs before starting test.
79 torture_param(int, holdoff, IS_BUILTIN(CONFIG_ 75 torture_param(int, holdoff, IS_BUILTIN(CONFIG_RCU_REF_SCALE_TEST) ? 10 : 0,
80 "Holdoff time before test start 76 "Holdoff time before test start (s)");
81 // Number of typesafe_lookup structures, that <<
82 torture_param(long, lookup_instances, 0, "Numb <<
83 // Number of loops per experiment, all readers 77 // Number of loops per experiment, all readers execute operations concurrently.
84 torture_param(long, loops, 10000, "Number of l 78 torture_param(long, loops, 10000, "Number of loops per experiment.");
85 // Number of readers, with -1 defaulting to ab 79 // Number of readers, with -1 defaulting to about 75% of the CPUs.
86 torture_param(int, nreaders, -1, "Number of re 80 torture_param(int, nreaders, -1, "Number of readers, -1 for 75% of CPUs.");
87 // Number of runs. 81 // Number of runs.
88 torture_param(int, nruns, 30, "Number of exper 82 torture_param(int, nruns, 30, "Number of experiments to run.");
89 // Reader delay in nanoseconds, 0 for no delay 83 // Reader delay in nanoseconds, 0 for no delay.
90 torture_param(int, readdelay, 0, "Read-side de 84 torture_param(int, readdelay, 0, "Read-side delay in nanoseconds.");
91 85
92 #ifdef MODULE 86 #ifdef MODULE
93 # define REFSCALE_SHUTDOWN 0 87 # define REFSCALE_SHUTDOWN 0
94 #else 88 #else
95 # define REFSCALE_SHUTDOWN 1 89 # define REFSCALE_SHUTDOWN 1
96 #endif 90 #endif
97 91
98 torture_param(bool, shutdown, REFSCALE_SHUTDOW 92 torture_param(bool, shutdown, REFSCALE_SHUTDOWN,
99 "Shutdown at end of scalability 93 "Shutdown at end of scalability tests.");
100 94
101 struct reader_task { 95 struct reader_task {
102 struct task_struct *task; 96 struct task_struct *task;
103 int start_reader; 97 int start_reader;
104 wait_queue_head_t wq; 98 wait_queue_head_t wq;
105 u64 last_duration_ns; 99 u64 last_duration_ns;
106 }; 100 };
107 101
108 static struct task_struct *shutdown_task; 102 static struct task_struct *shutdown_task;
109 static wait_queue_head_t shutdown_wq; 103 static wait_queue_head_t shutdown_wq;
110 104
111 static struct task_struct *main_task; 105 static struct task_struct *main_task;
112 static wait_queue_head_t main_wq; 106 static wait_queue_head_t main_wq;
113 static int shutdown_start; 107 static int shutdown_start;
114 108
115 static struct reader_task *reader_tasks; 109 static struct reader_task *reader_tasks;
116 110
117 // Number of readers that are part of the curr 111 // Number of readers that are part of the current experiment.
118 static atomic_t nreaders_exp; 112 static atomic_t nreaders_exp;
119 113
120 // Use to wait for all threads to start. 114 // Use to wait for all threads to start.
121 static atomic_t n_init; 115 static atomic_t n_init;
122 static atomic_t n_started; 116 static atomic_t n_started;
123 static atomic_t n_warmedup; 117 static atomic_t n_warmedup;
124 static atomic_t n_cooleddown; 118 static atomic_t n_cooleddown;
125 119
126 // Track which experiment is currently running 120 // Track which experiment is currently running.
127 static int exp_idx; 121 static int exp_idx;
128 122
129 // Operations vector for selecting different t 123 // Operations vector for selecting different types of tests.
130 struct ref_scale_ops { 124 struct ref_scale_ops {
131 bool (*init)(void); !! 125 void (*init)(void);
132 void (*cleanup)(void); 126 void (*cleanup)(void);
133 void (*readsection)(const int nloops); 127 void (*readsection)(const int nloops);
134 void (*delaysection)(const int nloops, 128 void (*delaysection)(const int nloops, const int udl, const int ndl);
135 const char *name; 129 const char *name;
136 }; 130 };
137 131
138 static const struct ref_scale_ops *cur_ops; !! 132 static struct ref_scale_ops *cur_ops;
139 133
140 static void un_delay(const int udl, const int 134 static void un_delay(const int udl, const int ndl)
141 { 135 {
142 if (udl) 136 if (udl)
143 udelay(udl); 137 udelay(udl);
144 if (ndl) 138 if (ndl)
145 ndelay(ndl); 139 ndelay(ndl);
146 } 140 }
147 141
148 static void ref_rcu_read_section(const int nlo 142 static void ref_rcu_read_section(const int nloops)
149 { 143 {
150 int i; 144 int i;
151 145
152 for (i = nloops; i >= 0; i--) { 146 for (i = nloops; i >= 0; i--) {
153 rcu_read_lock(); 147 rcu_read_lock();
154 rcu_read_unlock(); 148 rcu_read_unlock();
155 } 149 }
156 } 150 }
157 151
158 static void ref_rcu_delay_section(const int nl 152 static void ref_rcu_delay_section(const int nloops, const int udl, const int ndl)
159 { 153 {
160 int i; 154 int i;
161 155
162 for (i = nloops; i >= 0; i--) { 156 for (i = nloops; i >= 0; i--) {
163 rcu_read_lock(); 157 rcu_read_lock();
164 un_delay(udl, ndl); 158 un_delay(udl, ndl);
165 rcu_read_unlock(); 159 rcu_read_unlock();
166 } 160 }
167 } 161 }
168 162
169 static bool rcu_sync_scale_init(void) !! 163 static void rcu_sync_scale_init(void)
170 { 164 {
171 return true; <<
172 } 165 }
173 166
174 static const struct ref_scale_ops rcu_ops = { !! 167 static struct ref_scale_ops rcu_ops = {
175 .init = rcu_sync_scale_init, 168 .init = rcu_sync_scale_init,
176 .readsection = ref_rcu_read_section 169 .readsection = ref_rcu_read_section,
177 .delaysection = ref_rcu_delay_sectio 170 .delaysection = ref_rcu_delay_section,
178 .name = "rcu" 171 .name = "rcu"
179 }; 172 };
180 173
181 // Definitions for SRCU ref scale testing. 174 // Definitions for SRCU ref scale testing.
182 DEFINE_STATIC_SRCU(srcu_refctl_scale); 175 DEFINE_STATIC_SRCU(srcu_refctl_scale);
183 static struct srcu_struct *srcu_ctlp = &srcu_r 176 static struct srcu_struct *srcu_ctlp = &srcu_refctl_scale;
184 177
185 static void srcu_ref_scale_read_section(const 178 static void srcu_ref_scale_read_section(const int nloops)
186 { 179 {
187 int i; 180 int i;
188 int idx; 181 int idx;
189 182
190 for (i = nloops; i >= 0; i--) { 183 for (i = nloops; i >= 0; i--) {
191 idx = srcu_read_lock(srcu_ctlp 184 idx = srcu_read_lock(srcu_ctlp);
192 srcu_read_unlock(srcu_ctlp, id 185 srcu_read_unlock(srcu_ctlp, idx);
193 } 186 }
194 } 187 }
195 188
196 static void srcu_ref_scale_delay_section(const 189 static void srcu_ref_scale_delay_section(const int nloops, const int udl, const int ndl)
197 { 190 {
198 int i; 191 int i;
199 int idx; 192 int idx;
200 193
201 for (i = nloops; i >= 0; i--) { 194 for (i = nloops; i >= 0; i--) {
202 idx = srcu_read_lock(srcu_ctlp 195 idx = srcu_read_lock(srcu_ctlp);
203 un_delay(udl, ndl); 196 un_delay(udl, ndl);
204 srcu_read_unlock(srcu_ctlp, id 197 srcu_read_unlock(srcu_ctlp, idx);
205 } 198 }
206 } 199 }
207 200
208 static const struct ref_scale_ops srcu_ops = { !! 201 static struct ref_scale_ops srcu_ops = {
209 .init = rcu_sync_scale_init, 202 .init = rcu_sync_scale_init,
210 .readsection = srcu_ref_scale_read_ 203 .readsection = srcu_ref_scale_read_section,
211 .delaysection = srcu_ref_scale_delay 204 .delaysection = srcu_ref_scale_delay_section,
212 .name = "srcu" 205 .name = "srcu"
213 }; 206 };
214 207
215 #ifdef CONFIG_TASKS_RCU <<
216 <<
217 // Definitions for RCU Tasks ref scale testing 208 // Definitions for RCU Tasks ref scale testing: Empty read markers.
218 // These definitions also work for RCU Rude re 209 // These definitions also work for RCU Rude readers.
219 static void rcu_tasks_ref_scale_read_section(c 210 static void rcu_tasks_ref_scale_read_section(const int nloops)
220 { 211 {
221 int i; 212 int i;
222 213
223 for (i = nloops; i >= 0; i--) 214 for (i = nloops; i >= 0; i--)
224 continue; 215 continue;
225 } 216 }
226 217
227 static void rcu_tasks_ref_scale_delay_section( 218 static void rcu_tasks_ref_scale_delay_section(const int nloops, const int udl, const int ndl)
228 { 219 {
229 int i; 220 int i;
230 221
231 for (i = nloops; i >= 0; i--) 222 for (i = nloops; i >= 0; i--)
232 un_delay(udl, ndl); 223 un_delay(udl, ndl);
233 } 224 }
234 225
235 static const struct ref_scale_ops rcu_tasks_op !! 226 static struct ref_scale_ops rcu_tasks_ops = {
236 .init = rcu_sync_scale_init, 227 .init = rcu_sync_scale_init,
237 .readsection = rcu_tasks_ref_scale_ 228 .readsection = rcu_tasks_ref_scale_read_section,
238 .delaysection = rcu_tasks_ref_scale_ 229 .delaysection = rcu_tasks_ref_scale_delay_section,
239 .name = "rcu-tasks" 230 .name = "rcu-tasks"
240 }; 231 };
241 232
242 #define RCU_TASKS_OPS &rcu_tasks_ops, <<
243 <<
244 #else // #ifdef CONFIG_TASKS_RCU <<
245 <<
246 #define RCU_TASKS_OPS <<
247 <<
248 #endif // #else // #ifdef CONFIG_TASKS_RCU <<
249 <<
250 #ifdef CONFIG_TASKS_TRACE_RCU <<
251 <<
252 // Definitions for RCU Tasks Trace ref scale t 233 // Definitions for RCU Tasks Trace ref scale testing.
253 static void rcu_trace_ref_scale_read_section(c 234 static void rcu_trace_ref_scale_read_section(const int nloops)
254 { 235 {
255 int i; 236 int i;
256 237
257 for (i = nloops; i >= 0; i--) { 238 for (i = nloops; i >= 0; i--) {
258 rcu_read_lock_trace(); 239 rcu_read_lock_trace();
259 rcu_read_unlock_trace(); 240 rcu_read_unlock_trace();
260 } 241 }
261 } 242 }
262 243
263 static void rcu_trace_ref_scale_delay_section( 244 static void rcu_trace_ref_scale_delay_section(const int nloops, const int udl, const int ndl)
264 { 245 {
265 int i; 246 int i;
266 247
267 for (i = nloops; i >= 0; i--) { 248 for (i = nloops; i >= 0; i--) {
268 rcu_read_lock_trace(); 249 rcu_read_lock_trace();
269 un_delay(udl, ndl); 250 un_delay(udl, ndl);
270 rcu_read_unlock_trace(); 251 rcu_read_unlock_trace();
271 } 252 }
272 } 253 }
273 254
274 static const struct ref_scale_ops rcu_trace_op !! 255 static struct ref_scale_ops rcu_trace_ops = {
275 .init = rcu_sync_scale_init, 256 .init = rcu_sync_scale_init,
276 .readsection = rcu_trace_ref_scale_ 257 .readsection = rcu_trace_ref_scale_read_section,
277 .delaysection = rcu_trace_ref_scale_ 258 .delaysection = rcu_trace_ref_scale_delay_section,
278 .name = "rcu-trace" 259 .name = "rcu-trace"
279 }; 260 };
280 261
281 #define RCU_TRACE_OPS &rcu_trace_ops, <<
282 <<
283 #else // #ifdef CONFIG_TASKS_TRACE_RCU <<
284 <<
285 #define RCU_TRACE_OPS <<
286 <<
287 #endif // #else // #ifdef CONFIG_TASKS_TRACE_R <<
288 <<
289 // Definitions for reference count 262 // Definitions for reference count
290 static atomic_t refcnt; 263 static atomic_t refcnt;
291 264
292 static void ref_refcnt_section(const int nloop 265 static void ref_refcnt_section(const int nloops)
293 { 266 {
294 int i; 267 int i;
295 268
296 for (i = nloops; i >= 0; i--) { 269 for (i = nloops; i >= 0; i--) {
297 atomic_inc(&refcnt); 270 atomic_inc(&refcnt);
298 atomic_dec(&refcnt); 271 atomic_dec(&refcnt);
299 } 272 }
300 } 273 }
301 274
302 static void ref_refcnt_delay_section(const int 275 static void ref_refcnt_delay_section(const int nloops, const int udl, const int ndl)
303 { 276 {
304 int i; 277 int i;
305 278
306 for (i = nloops; i >= 0; i--) { 279 for (i = nloops; i >= 0; i--) {
307 atomic_inc(&refcnt); 280 atomic_inc(&refcnt);
308 un_delay(udl, ndl); 281 un_delay(udl, ndl);
309 atomic_dec(&refcnt); 282 atomic_dec(&refcnt);
310 } 283 }
311 } 284 }
312 285
313 static const struct ref_scale_ops refcnt_ops = !! 286 static struct ref_scale_ops refcnt_ops = {
314 .init = rcu_sync_scale_init, 287 .init = rcu_sync_scale_init,
315 .readsection = ref_refcnt_section, 288 .readsection = ref_refcnt_section,
316 .delaysection = ref_refcnt_delay_sec 289 .delaysection = ref_refcnt_delay_section,
317 .name = "refcnt" 290 .name = "refcnt"
318 }; 291 };
319 292
320 // Definitions for rwlock 293 // Definitions for rwlock
321 static rwlock_t test_rwlock; 294 static rwlock_t test_rwlock;
322 295
323 static bool ref_rwlock_init(void) !! 296 static void ref_rwlock_init(void)
324 { 297 {
325 rwlock_init(&test_rwlock); 298 rwlock_init(&test_rwlock);
326 return true; <<
327 } 299 }
328 300
329 static void ref_rwlock_section(const int nloop 301 static void ref_rwlock_section(const int nloops)
330 { 302 {
331 int i; 303 int i;
332 304
333 for (i = nloops; i >= 0; i--) { 305 for (i = nloops; i >= 0; i--) {
334 read_lock(&test_rwlock); 306 read_lock(&test_rwlock);
335 read_unlock(&test_rwlock); 307 read_unlock(&test_rwlock);
336 } 308 }
337 } 309 }
338 310
339 static void ref_rwlock_delay_section(const int 311 static void ref_rwlock_delay_section(const int nloops, const int udl, const int ndl)
340 { 312 {
341 int i; 313 int i;
342 314
343 for (i = nloops; i >= 0; i--) { 315 for (i = nloops; i >= 0; i--) {
344 read_lock(&test_rwlock); 316 read_lock(&test_rwlock);
345 un_delay(udl, ndl); 317 un_delay(udl, ndl);
346 read_unlock(&test_rwlock); 318 read_unlock(&test_rwlock);
347 } 319 }
348 } 320 }
349 321
350 static const struct ref_scale_ops rwlock_ops = !! 322 static struct ref_scale_ops rwlock_ops = {
351 .init = ref_rwlock_init, 323 .init = ref_rwlock_init,
352 .readsection = ref_rwlock_section, 324 .readsection = ref_rwlock_section,
353 .delaysection = ref_rwlock_delay_sec 325 .delaysection = ref_rwlock_delay_section,
354 .name = "rwlock" 326 .name = "rwlock"
355 }; 327 };
356 328
357 // Definitions for rwsem 329 // Definitions for rwsem
358 static struct rw_semaphore test_rwsem; 330 static struct rw_semaphore test_rwsem;
359 331
360 static bool ref_rwsem_init(void) !! 332 static void ref_rwsem_init(void)
361 { 333 {
362 init_rwsem(&test_rwsem); 334 init_rwsem(&test_rwsem);
363 return true; <<
364 } 335 }
365 336
366 static void ref_rwsem_section(const int nloops 337 static void ref_rwsem_section(const int nloops)
367 { 338 {
368 int i; 339 int i;
369 340
370 for (i = nloops; i >= 0; i--) { 341 for (i = nloops; i >= 0; i--) {
371 down_read(&test_rwsem); 342 down_read(&test_rwsem);
372 up_read(&test_rwsem); 343 up_read(&test_rwsem);
373 } 344 }
374 } 345 }
375 346
376 static void ref_rwsem_delay_section(const int 347 static void ref_rwsem_delay_section(const int nloops, const int udl, const int ndl)
377 { 348 {
378 int i; 349 int i;
379 350
380 for (i = nloops; i >= 0; i--) { 351 for (i = nloops; i >= 0; i--) {
381 down_read(&test_rwsem); 352 down_read(&test_rwsem);
382 un_delay(udl, ndl); 353 un_delay(udl, ndl);
383 up_read(&test_rwsem); 354 up_read(&test_rwsem);
384 } 355 }
385 } 356 }
386 357
387 static const struct ref_scale_ops rwsem_ops = !! 358 static struct ref_scale_ops rwsem_ops = {
388 .init = ref_rwsem_init, 359 .init = ref_rwsem_init,
389 .readsection = ref_rwsem_section, 360 .readsection = ref_rwsem_section,
390 .delaysection = ref_rwsem_delay_sect 361 .delaysection = ref_rwsem_delay_section,
391 .name = "rwsem" 362 .name = "rwsem"
392 }; 363 };
393 364
394 // Definitions for global spinlock <<
395 static DEFINE_RAW_SPINLOCK(test_lock); <<
396 <<
397 static void ref_lock_section(const int nloops) <<
398 { <<
399 int i; <<
400 <<
401 preempt_disable(); <<
402 for (i = nloops; i >= 0; i--) { <<
403 raw_spin_lock(&test_lock); <<
404 raw_spin_unlock(&test_lock); <<
405 } <<
406 preempt_enable(); <<
407 } <<
408 <<
409 static void ref_lock_delay_section(const int n <<
410 { <<
411 int i; <<
412 <<
413 preempt_disable(); <<
414 for (i = nloops; i >= 0; i--) { <<
415 raw_spin_lock(&test_lock); <<
416 un_delay(udl, ndl); <<
417 raw_spin_unlock(&test_lock); <<
418 } <<
419 preempt_enable(); <<
420 } <<
421 <<
422 static const struct ref_scale_ops lock_ops = { <<
423 .readsection = ref_lock_section, <<
424 .delaysection = ref_lock_delay_secti <<
425 .name = "lock" <<
426 }; <<
427 <<
428 // Definitions for global irq-save spinlock <<
429 <<
430 static void ref_lock_irq_section(const int nlo <<
431 { <<
432 unsigned long flags; <<
433 int i; <<
434 <<
435 preempt_disable(); <<
436 for (i = nloops; i >= 0; i--) { <<
437 raw_spin_lock_irqsave(&test_lo <<
438 raw_spin_unlock_irqrestore(&te <<
439 } <<
440 preempt_enable(); <<
441 } <<
442 <<
443 static void ref_lock_irq_delay_section(const i <<
444 { <<
445 unsigned long flags; <<
446 int i; <<
447 <<
448 preempt_disable(); <<
449 for (i = nloops; i >= 0; i--) { <<
450 raw_spin_lock_irqsave(&test_lo <<
451 un_delay(udl, ndl); <<
452 raw_spin_unlock_irqrestore(&te <<
453 } <<
454 preempt_enable(); <<
455 } <<
456 <<
457 static const struct ref_scale_ops lock_irq_ops <<
458 .readsection = ref_lock_irq_section <<
459 .delaysection = ref_lock_irq_delay_s <<
460 .name = "lock-irq" <<
461 }; <<
462 <<
463 // Definitions acquire-release. <<
464 static DEFINE_PER_CPU(unsigned long, test_acqr <<
465 <<
466 static void ref_acqrel_section(const int nloop <<
467 { <<
468 unsigned long x; <<
469 int i; <<
470 <<
471 preempt_disable(); <<
472 for (i = nloops; i >= 0; i--) { <<
473 x = smp_load_acquire(this_cpu_ <<
474 smp_store_release(this_cpu_ptr <<
475 } <<
476 preempt_enable(); <<
477 } <<
478 <<
479 static void ref_acqrel_delay_section(const int <<
480 { <<
481 unsigned long x; <<
482 int i; <<
483 <<
484 preempt_disable(); <<
485 for (i = nloops; i >= 0; i--) { <<
486 x = smp_load_acquire(this_cpu_ <<
487 un_delay(udl, ndl); <<
488 smp_store_release(this_cpu_ptr <<
489 } <<
490 preempt_enable(); <<
491 } <<
492 <<
493 static const struct ref_scale_ops acqrel_ops = <<
494 .readsection = ref_acqrel_section, <<
495 .delaysection = ref_acqrel_delay_sec <<
496 .name = "acqrel" <<
497 }; <<
498 <<
499 static volatile u64 stopopts; <<
500 <<
501 static void ref_clock_section(const int nloops <<
502 { <<
503 u64 x = 0; <<
504 int i; <<
505 <<
506 preempt_disable(); <<
507 for (i = nloops; i >= 0; i--) <<
508 x += ktime_get_real_fast_ns(); <<
509 preempt_enable(); <<
510 stopopts = x; <<
511 } <<
512 <<
513 static void ref_clock_delay_section(const int <<
514 { <<
515 u64 x = 0; <<
516 int i; <<
517 <<
518 preempt_disable(); <<
519 for (i = nloops; i >= 0; i--) { <<
520 x += ktime_get_real_fast_ns(); <<
521 un_delay(udl, ndl); <<
522 } <<
523 preempt_enable(); <<
524 stopopts = x; <<
525 } <<
526 <<
527 static const struct ref_scale_ops clock_ops = <<
528 .readsection = ref_clock_section, <<
529 .delaysection = ref_clock_delay_sect <<
530 .name = "clock" <<
531 }; <<
532 <<
533 static void ref_jiffies_section(const int nloo <<
534 { <<
535 u64 x = 0; <<
536 int i; <<
537 <<
538 preempt_disable(); <<
539 for (i = nloops; i >= 0; i--) <<
540 x += jiffies; <<
541 preempt_enable(); <<
542 stopopts = x; <<
543 } <<
544 <<
545 static void ref_jiffies_delay_section(const in <<
546 { <<
547 u64 x = 0; <<
548 int i; <<
549 <<
550 preempt_disable(); <<
551 for (i = nloops; i >= 0; i--) { <<
552 x += jiffies; <<
553 un_delay(udl, ndl); <<
554 } <<
555 preempt_enable(); <<
556 stopopts = x; <<
557 } <<
558 <<
559 static const struct ref_scale_ops jiffies_ops <<
560 .readsection = ref_jiffies_section, <<
561 .delaysection = ref_jiffies_delay_se <<
562 .name = "jiffies" <<
563 }; <<
564 <<
565 ////////////////////////////////////////////// <<
566 // <<
567 // Methods leveraging SLAB_TYPESAFE_BY_RCU. <<
568 // <<
569 <<
570 // Item to look up in a typesafe manner. Arra <<
571 struct refscale_typesafe { <<
572 atomic_t rts_refctr; // Used by all f <<
573 spinlock_t rts_lock; <<
574 seqlock_t rts_seqlock; <<
575 unsigned int a; <<
576 unsigned int b; <<
577 }; <<
578 <<
579 static struct kmem_cache *typesafe_kmem_cachep <<
580 static struct refscale_typesafe **rtsarray; <<
581 static long rtsarray_size; <<
582 static DEFINE_TORTURE_RANDOM_PERCPU(refscale_r <<
583 static bool (*rts_acquire)(struct refscale_typ <<
584 static bool (*rts_release)(struct refscale_typ <<
585 <<
586 // Conditionally acquire an explicit in-struct <<
587 static bool typesafe_ref_acquire(struct refsca <<
588 { <<
589 return atomic_inc_not_zero(&rtsp->rts_ <<
590 } <<
591 <<
592 // Unconditionally release an explicit in-stru <<
593 static bool typesafe_ref_release(struct refsca <<
594 { <<
595 if (!atomic_dec_return(&rtsp->rts_refc <<
596 WRITE_ONCE(rtsp->a, rtsp->a + <<
597 kmem_cache_free(typesafe_kmem_ <<
598 } <<
599 return true; <<
600 } <<
601 <<
602 // Unconditionally acquire an explicit in-stru <<
603 static bool typesafe_lock_acquire(struct refsc <<
604 { <<
605 spin_lock(&rtsp->rts_lock); <<
606 return true; <<
607 } <<
608 <<
609 // Unconditionally release an explicit in-stru <<
610 static bool typesafe_lock_release(struct refsc <<
611 { <<
612 spin_unlock(&rtsp->rts_lock); <<
613 return true; <<
614 } <<
615 <<
616 // Unconditionally acquire an explicit in-stru <<
617 static bool typesafe_seqlock_acquire(struct re <<
618 { <<
619 *start = read_seqbegin(&rtsp->rts_seql <<
620 return true; <<
621 } <<
622 <<
623 // Conditionally release an explicit in-struct <<
624 // true if this release was successful, that i <<
625 static bool typesafe_seqlock_release(struct re <<
626 { <<
627 return !read_seqretry(&rtsp->rts_seqlo <<
628 } <<
629 <<
630 // Do a read-side critical section with the sp <<
631 // microseconds and nanoseconds inserted so as <<
632 // of failure. <<
633 static void typesafe_delay_section(const int n <<
634 { <<
635 unsigned int a; <<
636 unsigned int b; <<
637 int i; <<
638 long idx; <<
639 struct refscale_typesafe *rtsp; <<
640 unsigned int start; <<
641 <<
642 for (i = nloops; i >= 0; i--) { <<
643 preempt_disable(); <<
644 idx = torture_random(this_cpu_ <<
645 preempt_enable(); <<
646 retry: <<
647 rcu_read_lock(); <<
648 rtsp = rcu_dereference(rtsarra <<
649 a = READ_ONCE(rtsp->a); <<
650 if (!rts_acquire(rtsp, &start) <<
651 rcu_read_unlock(); <<
652 goto retry; <<
653 } <<
654 if (a != READ_ONCE(rtsp->a)) { <<
655 (void)rts_release(rtsp <<
656 rcu_read_unlock(); <<
657 goto retry; <<
658 } <<
659 un_delay(udl, ndl); <<
660 b = READ_ONCE(rtsp->a); <<
661 // Remember, seqlock read-side <<
662 if (!rts_release(rtsp, start)) <<
663 rcu_read_unlock(); <<
664 goto retry; <<
665 } <<
666 WARN_ONCE(a != b, "Re-read of <<
667 b = rtsp->b; <<
668 rcu_read_unlock(); <<
669 WARN_ON_ONCE(a * a != b); <<
670 } <<
671 } <<
672 <<
673 // Because the acquisition and release methods <<
674 // is no point in optimizing away the un_delay <<
675 // Thus simply define typesafe_read_section() <<
676 // typesafe_delay_section(). <<
677 static void typesafe_read_section(const int nl <<
678 { <<
679 typesafe_delay_section(nloops, 0, 0); <<
680 } <<
681 <<
682 // Allocate and initialize one refscale_typesa <<
683 static struct refscale_typesafe *typesafe_allo <<
684 { <<
685 struct refscale_typesafe *rtsp; <<
686 <<
687 rtsp = kmem_cache_alloc(typesafe_kmem_ <<
688 if (!rtsp) <<
689 return NULL; <<
690 atomic_set(&rtsp->rts_refctr, 1); <<
691 WRITE_ONCE(rtsp->a, rtsp->a + 1); <<
692 WRITE_ONCE(rtsp->b, rtsp->a * rtsp->a) <<
693 return rtsp; <<
694 } <<
695 <<
696 // Slab-allocator constructor for refscale_typ <<
697 // out of a new slab of system memory. <<
698 static void refscale_typesafe_ctor(void *rtsp_ <<
699 { <<
700 struct refscale_typesafe *rtsp = rtsp_ <<
701 <<
702 spin_lock_init(&rtsp->rts_lock); <<
703 seqlock_init(&rtsp->rts_seqlock); <<
704 preempt_disable(); <<
705 rtsp->a = torture_random(this_cpu_ptr( <<
706 preempt_enable(); <<
707 } <<
708 <<
709 static const struct ref_scale_ops typesafe_ref <<
710 static const struct ref_scale_ops typesafe_loc <<
711 static const struct ref_scale_ops typesafe_seq <<
712 <<
713 // Initialize for a typesafe test. <<
714 static bool typesafe_init(void) <<
715 { <<
716 long idx; <<
717 long si = lookup_instances; <<
718 <<
719 typesafe_kmem_cachep = kmem_cache_crea <<
720 <<
721 <<
722 if (!typesafe_kmem_cachep) <<
723 return false; <<
724 if (si < 0) <<
725 si = -si * nr_cpu_ids; <<
726 else if (si == 0) <<
727 si = nr_cpu_ids; <<
728 rtsarray_size = si; <<
729 rtsarray = kcalloc(si, sizeof(*rtsarra <<
730 if (!rtsarray) <<
731 return false; <<
732 for (idx = 0; idx < rtsarray_size; idx <<
733 rtsarray[idx] = typesafe_alloc <<
734 if (!rtsarray[idx]) <<
735 return false; <<
736 } <<
737 if (cur_ops == &typesafe_ref_ops) { <<
738 rts_acquire = typesafe_ref_acq <<
739 rts_release = typesafe_ref_rel <<
740 } else if (cur_ops == &typesafe_lock_o <<
741 rts_acquire = typesafe_lock_ac <<
742 rts_release = typesafe_lock_re <<
743 } else if (cur_ops == &typesafe_seqloc <<
744 rts_acquire = typesafe_seqlock <<
745 rts_release = typesafe_seqlock <<
746 } else { <<
747 WARN_ON_ONCE(1); <<
748 return false; <<
749 } <<
750 return true; <<
751 } <<
752 <<
753 // Clean up after a typesafe test. <<
754 static void typesafe_cleanup(void) <<
755 { <<
756 long idx; <<
757 <<
758 if (rtsarray) { <<
759 for (idx = 0; idx < rtsarray_s <<
760 kmem_cache_free(typesa <<
761 kfree(rtsarray); <<
762 rtsarray = NULL; <<
763 rtsarray_size = 0; <<
764 } <<
765 kmem_cache_destroy(typesafe_kmem_cache <<
766 typesafe_kmem_cachep = NULL; <<
767 rts_acquire = NULL; <<
768 rts_release = NULL; <<
769 } <<
770 <<
771 // The typesafe_init() function distinguishes <<
772 static const struct ref_scale_ops typesafe_ref <<
773 .init = typesafe_init, <<
774 .cleanup = typesafe_cleanup, <<
775 .readsection = typesafe_read_sectio <<
776 .delaysection = typesafe_delay_secti <<
777 .name = "typesafe_ref" <<
778 }; <<
779 <<
780 static const struct ref_scale_ops typesafe_loc <<
781 .init = typesafe_init, <<
782 .cleanup = typesafe_cleanup, <<
783 .readsection = typesafe_read_sectio <<
784 .delaysection = typesafe_delay_secti <<
785 .name = "typesafe_lock" <<
786 }; <<
787 <<
788 static const struct ref_scale_ops typesafe_seq <<
789 .init = typesafe_init, <<
790 .cleanup = typesafe_cleanup, <<
791 .readsection = typesafe_read_sectio <<
792 .delaysection = typesafe_delay_secti <<
793 .name = "typesafe_seqlock" <<
794 }; <<
795 <<
796 static void rcu_scale_one_reader(void) 365 static void rcu_scale_one_reader(void)
797 { 366 {
798 if (readdelay <= 0) 367 if (readdelay <= 0)
799 cur_ops->readsection(loops); 368 cur_ops->readsection(loops);
800 else 369 else
801 cur_ops->delaysection(loops, r 370 cur_ops->delaysection(loops, readdelay / 1000, readdelay % 1000);
802 } 371 }
803 372
804 // Reader kthread. Repeatedly does empty RCU 373 // Reader kthread. Repeatedly does empty RCU read-side
805 // critical section, minimizing update-side in 374 // critical section, minimizing update-side interference.
806 static int 375 static int
807 ref_scale_reader(void *arg) 376 ref_scale_reader(void *arg)
808 { 377 {
809 unsigned long flags; 378 unsigned long flags;
810 long me = (long)arg; 379 long me = (long)arg;
811 struct reader_task *rt = &(reader_task 380 struct reader_task *rt = &(reader_tasks[me]);
812 u64 start; 381 u64 start;
813 s64 duration; 382 s64 duration;
814 383
815 VERBOSE_SCALEOUT_BATCH("ref_scale_read 384 VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: task started", me);
816 WARN_ON_ONCE(set_cpus_allowed_ptr(curr !! 385 set_cpus_allowed_ptr(current, cpumask_of(me % nr_cpu_ids));
817 set_user_nice(current, MAX_NICE); 386 set_user_nice(current, MAX_NICE);
818 atomic_inc(&n_init); 387 atomic_inc(&n_init);
819 if (holdoff) 388 if (holdoff)
820 schedule_timeout_interruptible 389 schedule_timeout_interruptible(holdoff * HZ);
821 repeat: 390 repeat:
822 VERBOSE_SCALEOUT_BATCH("ref_scale_read !! 391 VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: waiting to start next experiment on cpu %d", me, smp_processor_id());
823 392
824 // Wait for signal that this reader ca 393 // Wait for signal that this reader can start.
825 wait_event(rt->wq, (atomic_read(&nread 394 wait_event(rt->wq, (atomic_read(&nreaders_exp) && smp_load_acquire(&rt->start_reader)) ||
826 torture_must_stop() 395 torture_must_stop());
827 396
828 if (torture_must_stop()) 397 if (torture_must_stop())
829 goto end; 398 goto end;
830 399
831 // Make sure that the CPU is affinitiz 400 // Make sure that the CPU is affinitized appropriately during testing.
832 WARN_ON_ONCE(raw_smp_processor_id() != !! 401 WARN_ON_ONCE(smp_processor_id() != me);
833 402
834 WRITE_ONCE(rt->start_reader, 0); 403 WRITE_ONCE(rt->start_reader, 0);
835 if (!atomic_dec_return(&n_started)) 404 if (!atomic_dec_return(&n_started))
836 while (atomic_read_acquire(&n_ 405 while (atomic_read_acquire(&n_started))
837 cpu_relax(); 406 cpu_relax();
838 407
839 VERBOSE_SCALEOUT_BATCH("ref_scale_read 408 VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: experiment %d started", me, exp_idx);
840 409
841 410
842 // To reduce noise, do an initial cach 411 // To reduce noise, do an initial cache-warming invocation, check
843 // in, and then keep warming until eve 412 // in, and then keep warming until everyone has checked in.
844 rcu_scale_one_reader(); 413 rcu_scale_one_reader();
845 if (!atomic_dec_return(&n_warmedup)) 414 if (!atomic_dec_return(&n_warmedup))
846 while (atomic_read_acquire(&n_ 415 while (atomic_read_acquire(&n_warmedup))
847 rcu_scale_one_reader() 416 rcu_scale_one_reader();
848 // Also keep interrupts disabled. Thi 417 // Also keep interrupts disabled. This also has the effect
849 // of preventing entries into slow pat 418 // of preventing entries into slow path for rcu_read_unlock().
850 local_irq_save(flags); 419 local_irq_save(flags);
851 start = ktime_get_mono_fast_ns(); 420 start = ktime_get_mono_fast_ns();
852 421
853 rcu_scale_one_reader(); 422 rcu_scale_one_reader();
854 423
855 duration = ktime_get_mono_fast_ns() - 424 duration = ktime_get_mono_fast_ns() - start;
856 local_irq_restore(flags); 425 local_irq_restore(flags);
857 426
858 rt->last_duration_ns = WARN_ON_ONCE(du 427 rt->last_duration_ns = WARN_ON_ONCE(duration < 0) ? 0 : duration;
859 // To reduce runtime-skew noise, do ma 428 // To reduce runtime-skew noise, do maintain-load invocations until
860 // everyone is done. 429 // everyone is done.
861 if (!atomic_dec_return(&n_cooleddown)) 430 if (!atomic_dec_return(&n_cooleddown))
862 while (atomic_read_acquire(&n_ 431 while (atomic_read_acquire(&n_cooleddown))
863 rcu_scale_one_reader() 432 rcu_scale_one_reader();
864 433
865 if (atomic_dec_and_test(&nreaders_exp) 434 if (atomic_dec_and_test(&nreaders_exp))
866 wake_up(&main_wq); 435 wake_up(&main_wq);
867 436
868 VERBOSE_SCALEOUT_BATCH("ref_scale_read 437 VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: experiment %d ended, (readers remaining=%d)",
869 me, exp_idx, a 438 me, exp_idx, atomic_read(&nreaders_exp));
870 439
871 if (!torture_must_stop()) 440 if (!torture_must_stop())
872 goto repeat; 441 goto repeat;
873 end: 442 end:
874 torture_kthread_stopping("ref_scale_re 443 torture_kthread_stopping("ref_scale_reader");
875 return 0; 444 return 0;
876 } 445 }
877 446
878 static void reset_readers(void) 447 static void reset_readers(void)
879 { 448 {
880 int i; 449 int i;
881 struct reader_task *rt; 450 struct reader_task *rt;
882 451
883 for (i = 0; i < nreaders; i++) { 452 for (i = 0; i < nreaders; i++) {
884 rt = &(reader_tasks[i]); 453 rt = &(reader_tasks[i]);
885 454
886 rt->last_duration_ns = 0; 455 rt->last_duration_ns = 0;
887 } 456 }
888 } 457 }
889 458
890 // Print the results of each reader and return 459 // Print the results of each reader and return the sum of all their durations.
891 static u64 process_durations(int n) 460 static u64 process_durations(int n)
892 { 461 {
893 int i; 462 int i;
894 struct reader_task *rt; 463 struct reader_task *rt;
895 struct seq_buf s; !! 464 char buf1[64];
896 char *buf; 465 char *buf;
897 u64 sum = 0; 466 u64 sum = 0;
898 467
899 buf = kmalloc(800 + 64, GFP_KERNEL); !! 468 buf = kmalloc(128 + nreaders * 32, GFP_KERNEL);
900 if (!buf) 469 if (!buf)
901 return 0; 470 return 0;
902 seq_buf_init(&s, buf, 800 + 64); !! 471 buf[0] = 0;
903 !! 472 sprintf(buf, "Experiment #%d (Format: <THREAD-NUM>:<Total loop time in ns>)",
904 seq_buf_printf(&s, "Experiment #%d (Fo !! 473 exp_idx);
905 exp_idx); <<
906 474
907 for (i = 0; i < n && !torture_must_sto 475 for (i = 0; i < n && !torture_must_stop(); i++) {
908 rt = &(reader_tasks[i]); 476 rt = &(reader_tasks[i]);
>> 477 sprintf(buf1, "%d: %llu\t", i, rt->last_duration_ns);
909 478
910 if (i % 5 == 0) 479 if (i % 5 == 0)
911 seq_buf_putc(&s, '\n') !! 480 strcat(buf, "\n");
912 !! 481 strcat(buf, buf1);
913 if (seq_buf_used(&s) >= 800) { <<
914 pr_alert("%s", seq_buf <<
915 seq_buf_clear(&s); <<
916 } <<
917 <<
918 seq_buf_printf(&s, "%d: %llu\t <<
919 482
920 sum += rt->last_duration_ns; 483 sum += rt->last_duration_ns;
921 } 484 }
922 pr_alert("%s\n", seq_buf_str(&s)); !! 485 strcat(buf, "\n");
>> 486
>> 487 SCALEOUT("%s\n", buf);
923 488
924 kfree(buf); 489 kfree(buf);
925 return sum; 490 return sum;
926 } 491 }
927 492
928 // The main_func is the main orchestrator, it 493 // The main_func is the main orchestrator, it performs a bunch of
929 // experiments. For every experiment, it orde 494 // experiments. For every experiment, it orders all the readers
930 // involved to start and waits for them to fin 495 // involved to start and waits for them to finish the experiment. It
931 // then reads their timestamps and starts the 496 // then reads their timestamps and starts the next experiment. Each
932 // experiment progresses from 1 concurrent rea 497 // experiment progresses from 1 concurrent reader to N of them at which
933 // point all the timestamps are printed. 498 // point all the timestamps are printed.
934 static int main_func(void *arg) 499 static int main_func(void *arg)
935 { 500 {
>> 501 bool errexit = false;
936 int exp, r; 502 int exp, r;
937 char buf1[64]; 503 char buf1[64];
938 char *buf; 504 char *buf;
939 u64 *result_avg; 505 u64 *result_avg;
940 506
941 set_cpus_allowed_ptr(current, cpumask_ 507 set_cpus_allowed_ptr(current, cpumask_of(nreaders % nr_cpu_ids));
942 set_user_nice(current, MAX_NICE); 508 set_user_nice(current, MAX_NICE);
943 509
944 VERBOSE_SCALEOUT("main_func task start 510 VERBOSE_SCALEOUT("main_func task started");
945 result_avg = kzalloc(nruns * sizeof(*r 511 result_avg = kzalloc(nruns * sizeof(*result_avg), GFP_KERNEL);
946 buf = kzalloc(800 + 64, GFP_KERNEL); !! 512 buf = kzalloc(64 + nruns * 32, GFP_KERNEL);
947 if (!result_avg || !buf) { 513 if (!result_avg || !buf) {
948 SCALEOUT_ERRSTRING("out of mem !! 514 VERBOSE_SCALEOUT_ERRSTRING("out of memory");
949 goto oom_exit; !! 515 errexit = true;
950 } 516 }
951 if (holdoff) 517 if (holdoff)
952 schedule_timeout_interruptible 518 schedule_timeout_interruptible(holdoff * HZ);
953 519
954 // Wait for all threads to start. 520 // Wait for all threads to start.
955 atomic_inc(&n_init); 521 atomic_inc(&n_init);
956 while (atomic_read(&n_init) < nreaders 522 while (atomic_read(&n_init) < nreaders + 1)
957 schedule_timeout_uninterruptib 523 schedule_timeout_uninterruptible(1);
958 524
959 // Start exp readers up per experiment 525 // Start exp readers up per experiment
960 for (exp = 0; exp < nruns && !torture_ 526 for (exp = 0; exp < nruns && !torture_must_stop(); exp++) {
>> 527 if (errexit)
>> 528 break;
961 if (torture_must_stop()) 529 if (torture_must_stop())
962 goto end; 530 goto end;
963 531
964 reset_readers(); 532 reset_readers();
965 atomic_set(&nreaders_exp, nrea 533 atomic_set(&nreaders_exp, nreaders);
966 atomic_set(&n_started, nreader 534 atomic_set(&n_started, nreaders);
967 atomic_set(&n_warmedup, nreade 535 atomic_set(&n_warmedup, nreaders);
968 atomic_set(&n_cooleddown, nrea 536 atomic_set(&n_cooleddown, nreaders);
969 537
970 exp_idx = exp; 538 exp_idx = exp;
971 539
972 for (r = 0; r < nreaders; r++) 540 for (r = 0; r < nreaders; r++) {
973 smp_store_release(&rea 541 smp_store_release(&reader_tasks[r].start_reader, 1);
974 wake_up(&reader_tasks[ 542 wake_up(&reader_tasks[r].wq);
975 } 543 }
976 544
977 VERBOSE_SCALEOUT("main_func: e 545 VERBOSE_SCALEOUT("main_func: experiment started, waiting for %d readers",
978 nreaders); 546 nreaders);
979 547
980 wait_event(main_wq, 548 wait_event(main_wq,
981 !atomic_read(&nread 549 !atomic_read(&nreaders_exp) || torture_must_stop());
982 550
983 VERBOSE_SCALEOUT("main_func: e 551 VERBOSE_SCALEOUT("main_func: experiment ended");
984 552
985 if (torture_must_stop()) 553 if (torture_must_stop())
986 goto end; 554 goto end;
987 555
988 result_avg[exp] = div_u64(1000 556 result_avg[exp] = div_u64(1000 * process_durations(nreaders), nreaders * loops);
989 } 557 }
990 558
991 // Print the average of all experiment 559 // Print the average of all experiments
992 SCALEOUT("END OF TEST. Calculating ave 560 SCALEOUT("END OF TEST. Calculating average duration per loop (nanoseconds)...\n");
993 561
994 pr_alert("Runs\tTime(ns)\n"); !! 562 if (!errexit) {
>> 563 buf[0] = 0;
>> 564 strcat(buf, "\n");
>> 565 strcat(buf, "Runs\tTime(ns)\n");
>> 566 }
>> 567
995 for (exp = 0; exp < nruns; exp++) { 568 for (exp = 0; exp < nruns; exp++) {
996 u64 avg; 569 u64 avg;
997 u32 rem; 570 u32 rem;
998 571
>> 572 if (errexit)
>> 573 break;
999 avg = div_u64_rem(result_avg[e 574 avg = div_u64_rem(result_avg[exp], 1000, &rem);
1000 sprintf(buf1, "%d\t%llu.%03u\ 575 sprintf(buf1, "%d\t%llu.%03u\n", exp + 1, avg, rem);
1001 strcat(buf, buf1); 576 strcat(buf, buf1);
1002 if (strlen(buf) >= 800) { <<
1003 pr_alert("%s", buf); <<
1004 buf[0] = 0; <<
1005 } <<
1006 } 577 }
1007 578
1008 pr_alert("%s", buf); !! 579 if (!errexit)
>> 580 SCALEOUT("%s", buf);
1009 581
1010 oom_exit: <<
1011 // This will shutdown everything incl 582 // This will shutdown everything including us.
1012 if (shutdown) { 583 if (shutdown) {
1013 shutdown_start = 1; 584 shutdown_start = 1;
1014 wake_up(&shutdown_wq); 585 wake_up(&shutdown_wq);
1015 } 586 }
1016 587
1017 // Wait for torture to stop us 588 // Wait for torture to stop us
1018 while (!torture_must_stop()) 589 while (!torture_must_stop())
1019 schedule_timeout_uninterrupti 590 schedule_timeout_uninterruptible(1);
1020 591
1021 end: 592 end:
1022 torture_kthread_stopping("main_func") 593 torture_kthread_stopping("main_func");
1023 kfree(result_avg); 594 kfree(result_avg);
1024 kfree(buf); 595 kfree(buf);
1025 return 0; 596 return 0;
1026 } 597 }
1027 598
1028 static void 599 static void
1029 ref_scale_print_module_parms(const struct ref !! 600 ref_scale_print_module_parms(struct ref_scale_ops *cur_ops, const char *tag)
1030 { 601 {
1031 pr_alert("%s" SCALE_FLAG 602 pr_alert("%s" SCALE_FLAG
1032 "--- %s: verbose=%d verbose !! 603 "--- %s: verbose=%d shutdown=%d holdoff=%d loops=%ld nreaders=%d nruns=%d readdelay=%d\n", scale_type, tag,
1033 verbose, verbose_batched, sh !! 604 verbose, shutdown, holdoff, loops, nreaders, nruns, readdelay);
1034 } 605 }
1035 606
1036 static void 607 static void
1037 ref_scale_cleanup(void) 608 ref_scale_cleanup(void)
1038 { 609 {
1039 int i; 610 int i;
1040 611
1041 if (torture_cleanup_begin()) 612 if (torture_cleanup_begin())
1042 return; 613 return;
1043 614
1044 if (!cur_ops) { 615 if (!cur_ops) {
1045 torture_cleanup_end(); 616 torture_cleanup_end();
1046 return; 617 return;
1047 } 618 }
1048 619
1049 if (reader_tasks) { 620 if (reader_tasks) {
1050 for (i = 0; i < nreaders; i++ 621 for (i = 0; i < nreaders; i++)
1051 torture_stop_kthread( 622 torture_stop_kthread("ref_scale_reader",
1052 623 reader_tasks[i].task);
1053 } 624 }
1054 kfree(reader_tasks); 625 kfree(reader_tasks);
1055 626
1056 torture_stop_kthread("main_task", mai 627 torture_stop_kthread("main_task", main_task);
1057 kfree(main_task); 628 kfree(main_task);
1058 629
1059 // Do scale-type-specific cleanup ope 630 // Do scale-type-specific cleanup operations.
1060 if (cur_ops->cleanup != NULL) 631 if (cur_ops->cleanup != NULL)
1061 cur_ops->cleanup(); 632 cur_ops->cleanup();
1062 633
1063 torture_cleanup_end(); 634 torture_cleanup_end();
1064 } 635 }
1065 636
1066 // Shutdown kthread. Just waits to be awaken 637 // Shutdown kthread. Just waits to be awakened, then shuts down system.
1067 static int 638 static int
1068 ref_scale_shutdown(void *arg) 639 ref_scale_shutdown(void *arg)
1069 { 640 {
1070 wait_event_idle(shutdown_wq, shutdown !! 641 wait_event(shutdown_wq, shutdown_start);
1071 642
1072 smp_mb(); // Wake before output. 643 smp_mb(); // Wake before output.
1073 ref_scale_cleanup(); 644 ref_scale_cleanup();
1074 kernel_power_off(); 645 kernel_power_off();
1075 646
1076 return -EINVAL; 647 return -EINVAL;
1077 } 648 }
1078 649
1079 static int __init 650 static int __init
1080 ref_scale_init(void) 651 ref_scale_init(void)
1081 { 652 {
1082 long i; 653 long i;
1083 int firsterr = 0; 654 int firsterr = 0;
1084 static const struct ref_scale_ops *sc !! 655 static struct ref_scale_ops *scale_ops[] = {
1085 &rcu_ops, &srcu_ops, RCU_TRAC !! 656 &rcu_ops, &srcu_ops, &rcu_trace_ops, &rcu_tasks_ops,
1086 &rwsem_ops, &lock_ops, &lock_ !! 657 &refcnt_ops, &rwlock_ops, &rwsem_ops,
1087 &typesafe_ref_ops, &typesafe_ <<
1088 }; 658 };
1089 659
1090 if (!torture_init_begin(scale_type, v 660 if (!torture_init_begin(scale_type, verbose))
1091 return -EBUSY; 661 return -EBUSY;
1092 662
1093 for (i = 0; i < ARRAY_SIZE(scale_ops) 663 for (i = 0; i < ARRAY_SIZE(scale_ops); i++) {
1094 cur_ops = scale_ops[i]; 664 cur_ops = scale_ops[i];
1095 if (strcmp(scale_type, cur_op 665 if (strcmp(scale_type, cur_ops->name) == 0)
1096 break; 666 break;
1097 } 667 }
1098 if (i == ARRAY_SIZE(scale_ops)) { 668 if (i == ARRAY_SIZE(scale_ops)) {
1099 pr_alert("rcu-scale: invalid 669 pr_alert("rcu-scale: invalid scale type: \"%s\"\n", scale_type);
1100 pr_alert("rcu-scale types:"); 670 pr_alert("rcu-scale types:");
1101 for (i = 0; i < ARRAY_SIZE(sc 671 for (i = 0; i < ARRAY_SIZE(scale_ops); i++)
1102 pr_cont(" %s", scale_ 672 pr_cont(" %s", scale_ops[i]->name);
1103 pr_cont("\n"); 673 pr_cont("\n");
1104 firsterr = -EINVAL; 674 firsterr = -EINVAL;
1105 cur_ops = NULL; 675 cur_ops = NULL;
1106 goto unwind; 676 goto unwind;
1107 } 677 }
1108 if (cur_ops->init) 678 if (cur_ops->init)
1109 if (!cur_ops->init()) { !! 679 cur_ops->init();
1110 firsterr = -EUCLEAN; <<
1111 goto unwind; <<
1112 } <<
1113 680
1114 ref_scale_print_module_parms(cur_ops, 681 ref_scale_print_module_parms(cur_ops, "Start of test");
1115 682
1116 // Shutdown task 683 // Shutdown task
1117 if (shutdown) { 684 if (shutdown) {
1118 init_waitqueue_head(&shutdown 685 init_waitqueue_head(&shutdown_wq);
1119 firsterr = torture_create_kth 686 firsterr = torture_create_kthread(ref_scale_shutdown, NULL,
1120 687 shutdown_task);
1121 if (torture_init_error(firste !! 688 if (firsterr)
1122 goto unwind; 689 goto unwind;
1123 schedule_timeout_uninterrupti 690 schedule_timeout_uninterruptible(1);
1124 } 691 }
1125 692
1126 // Reader tasks (default to ~75% of o 693 // Reader tasks (default to ~75% of online CPUs).
1127 if (nreaders < 0) 694 if (nreaders < 0)
1128 nreaders = (num_online_cpus() 695 nreaders = (num_online_cpus() >> 1) + (num_online_cpus() >> 2);
1129 if (WARN_ONCE(loops <= 0, "%s: loops 696 if (WARN_ONCE(loops <= 0, "%s: loops = %ld, adjusted to 1\n", __func__, loops))
1130 loops = 1; 697 loops = 1;
1131 if (WARN_ONCE(nreaders <= 0, "%s: nre 698 if (WARN_ONCE(nreaders <= 0, "%s: nreaders = %d, adjusted to 1\n", __func__, nreaders))
1132 nreaders = 1; 699 nreaders = 1;
1133 if (WARN_ONCE(nruns <= 0, "%s: nruns 700 if (WARN_ONCE(nruns <= 0, "%s: nruns = %d, adjusted to 1\n", __func__, nruns))
1134 nruns = 1; 701 nruns = 1;
1135 reader_tasks = kcalloc(nreaders, size 702 reader_tasks = kcalloc(nreaders, sizeof(reader_tasks[0]),
1136 GFP_KERNEL); 703 GFP_KERNEL);
1137 if (!reader_tasks) { 704 if (!reader_tasks) {
1138 SCALEOUT_ERRSTRING("out of me !! 705 VERBOSE_SCALEOUT_ERRSTRING("out of memory");
1139 firsterr = -ENOMEM; 706 firsterr = -ENOMEM;
1140 goto unwind; 707 goto unwind;
1141 } 708 }
1142 709
1143 VERBOSE_SCALEOUT("Starting %d reader !! 710 VERBOSE_SCALEOUT("Starting %d reader threads\n", nreaders);
1144 711
1145 for (i = 0; i < nreaders; i++) { 712 for (i = 0; i < nreaders; i++) {
1146 init_waitqueue_head(&reader_t <<
1147 firsterr = torture_create_kth 713 firsterr = torture_create_kthread(ref_scale_reader, (void *)i,
1148 714 reader_tasks[i].task);
1149 if (torture_init_error(firste !! 715 if (firsterr)
1150 goto unwind; 716 goto unwind;
>> 717
>> 718 init_waitqueue_head(&(reader_tasks[i].wq));
1151 } 719 }
1152 720
1153 // Main Task 721 // Main Task
1154 init_waitqueue_head(&main_wq); 722 init_waitqueue_head(&main_wq);
1155 firsterr = torture_create_kthread(mai 723 firsterr = torture_create_kthread(main_func, NULL, main_task);
1156 if (torture_init_error(firsterr)) !! 724 if (firsterr)
1157 goto unwind; 725 goto unwind;
1158 726
1159 torture_init_end(); 727 torture_init_end();
1160 return 0; 728 return 0;
1161 729
1162 unwind: 730 unwind:
1163 torture_init_end(); 731 torture_init_end();
1164 ref_scale_cleanup(); 732 ref_scale_cleanup();
1165 if (shutdown) { 733 if (shutdown) {
1166 WARN_ON(!IS_MODULE(CONFIG_RCU 734 WARN_ON(!IS_MODULE(CONFIG_RCU_REF_SCALE_TEST));
1167 kernel_power_off(); 735 kernel_power_off();
1168 } 736 }
1169 return firsterr; 737 return firsterr;
1170 } 738 }
1171 739
1172 module_init(ref_scale_init); 740 module_init(ref_scale_init);
1173 module_exit(ref_scale_cleanup); 741 module_exit(ref_scale_cleanup);
1174 742
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.