1 // SPDX-License-Identifier: GPL-2.0+ 1
2 //
3 // Scalability test comparing RCU vs other mec
4 // for acquiring references on objects.
5 //
6 // Copyright (C) Google, 2020.
7 //
8 // Author: Joel Fernandes <joel@joelfernandes.
9
10 #define pr_fmt(fmt) fmt
11
12 #include <linux/atomic.h>
13 #include <linux/bitops.h>
14 #include <linux/completion.h>
15 #include <linux/cpu.h>
16 #include <linux/delay.h>
17 #include <linux/err.h>
18 #include <linux/init.h>
19 #include <linux/interrupt.h>
20 #include <linux/kthread.h>
21 #include <linux/kernel.h>
22 #include <linux/mm.h>
23 #include <linux/module.h>
24 #include <linux/moduleparam.h>
25 #include <linux/notifier.h>
26 #include <linux/percpu.h>
27 #include <linux/rcupdate.h>
28 #include <linux/rcupdate_trace.h>
29 #include <linux/reboot.h>
30 #include <linux/sched.h>
31 #include <linux/seq_buf.h>
32 #include <linux/spinlock.h>
33 #include <linux/smp.h>
34 #include <linux/stat.h>
35 #include <linux/srcu.h>
36 #include <linux/slab.h>
37 #include <linux/torture.h>
38 #include <linux/types.h>
39
40 #include "rcu.h"
41
42 #define SCALE_FLAG "-ref-scale: "
43
44 #define SCALEOUT(s, x...) \
45 pr_alert("%s" SCALE_FLAG s, scale_type
46
47 #define VERBOSE_SCALEOUT(s, x...) \
48 do { \
49 if (verbose) \
50 pr_alert("%s" SCALE_FL
51 } while (0)
52
53 static atomic_t verbose_batch_ctr;
54
55 #define VERBOSE_SCALEOUT_BATCH(s, x...)
56 do {
57 if (verbose &&
58 (verbose_batched <= 0 ||
59 !(atomic_inc_return(&verbose_batc
60 schedule_timeout_uninterruptib
61 pr_alert("%s" SCALE_FLAG s "\n
62 }
63 } while (0)
64
65 #define SCALEOUT_ERRSTRING(s, x...) pr_alert("
66
67 MODULE_DESCRIPTION("Scalability test for objec
68 MODULE_LICENSE("GPL");
69 MODULE_AUTHOR("Joel Fernandes (Google) <joel@j
70
71 static char *scale_type = "rcu";
72 module_param(scale_type, charp, 0444);
73 MODULE_PARM_DESC(scale_type, "Type of test (rc
74
75 torture_param(int, verbose, 0, "Enable verbose
76 torture_param(int, verbose_batched, 0, "Batch
77
78 // Wait until there are multiple CPUs before s
79 torture_param(int, holdoff, IS_BUILTIN(CONFIG_
80 "Holdoff time before test start
81 // Number of typesafe_lookup structures, that
82 torture_param(long, lookup_instances, 0, "Numb
83 // Number of loops per experiment, all readers
84 torture_param(long, loops, 10000, "Number of l
85 // Number of readers, with -1 defaulting to ab
86 torture_param(int, nreaders, -1, "Number of re
87 // Number of runs.
88 torture_param(int, nruns, 30, "Number of exper
89 // Reader delay in nanoseconds, 0 for no delay
90 torture_param(int, readdelay, 0, "Read-side de
91
92 #ifdef MODULE
93 # define REFSCALE_SHUTDOWN 0
94 #else
95 # define REFSCALE_SHUTDOWN 1
96 #endif
97
98 torture_param(bool, shutdown, REFSCALE_SHUTDOW
99 "Shutdown at end of scalability
100
101 struct reader_task {
102 struct task_struct *task;
103 int start_reader;
104 wait_queue_head_t wq;
105 u64 last_duration_ns;
106 };
107
108 static struct task_struct *shutdown_task;
109 static wait_queue_head_t shutdown_wq;
110
111 static struct task_struct *main_task;
112 static wait_queue_head_t main_wq;
113 static int shutdown_start;
114
115 static struct reader_task *reader_tasks;
116
117 // Number of readers that are part of the curr
118 static atomic_t nreaders_exp;
119
120 // Use to wait for all threads to start.
121 static atomic_t n_init;
122 static atomic_t n_started;
123 static atomic_t n_warmedup;
124 static atomic_t n_cooleddown;
125
126 // Track which experiment is currently running
127 static int exp_idx;
128
129 // Operations vector for selecting different t
130 struct ref_scale_ops {
131 bool (*init)(void);
132 void (*cleanup)(void);
133 void (*readsection)(const int nloops);
134 void (*delaysection)(const int nloops,
135 const char *name;
136 };
137
138 static const struct ref_scale_ops *cur_ops;
139
140 static void un_delay(const int udl, const int
141 {
142 if (udl)
143 udelay(udl);
144 if (ndl)
145 ndelay(ndl);
146 }
147
148 static void ref_rcu_read_section(const int nlo
149 {
150 int i;
151
152 for (i = nloops; i >= 0; i--) {
153 rcu_read_lock();
154 rcu_read_unlock();
155 }
156 }
157
158 static void ref_rcu_delay_section(const int nl
159 {
160 int i;
161
162 for (i = nloops; i >= 0; i--) {
163 rcu_read_lock();
164 un_delay(udl, ndl);
165 rcu_read_unlock();
166 }
167 }
168
169 static bool rcu_sync_scale_init(void)
170 {
171 return true;
172 }
173
174 static const struct ref_scale_ops rcu_ops = {
175 .init = rcu_sync_scale_init,
176 .readsection = ref_rcu_read_section
177 .delaysection = ref_rcu_delay_sectio
178 .name = "rcu"
179 };
180
181 // Definitions for SRCU ref scale testing.
182 DEFINE_STATIC_SRCU(srcu_refctl_scale);
183 static struct srcu_struct *srcu_ctlp = &srcu_r
184
185 static void srcu_ref_scale_read_section(const
186 {
187 int i;
188 int idx;
189
190 for (i = nloops; i >= 0; i--) {
191 idx = srcu_read_lock(srcu_ctlp
192 srcu_read_unlock(srcu_ctlp, id
193 }
194 }
195
196 static void srcu_ref_scale_delay_section(const
197 {
198 int i;
199 int idx;
200
201 for (i = nloops; i >= 0; i--) {
202 idx = srcu_read_lock(srcu_ctlp
203 un_delay(udl, ndl);
204 srcu_read_unlock(srcu_ctlp, id
205 }
206 }
207
208 static const struct ref_scale_ops srcu_ops = {
209 .init = rcu_sync_scale_init,
210 .readsection = srcu_ref_scale_read_
211 .delaysection = srcu_ref_scale_delay
212 .name = "srcu"
213 };
214
215 #ifdef CONFIG_TASKS_RCU
216
217 // Definitions for RCU Tasks ref scale testing
218 // These definitions also work for RCU Rude re
219 static void rcu_tasks_ref_scale_read_section(c
220 {
221 int i;
222
223 for (i = nloops; i >= 0; i--)
224 continue;
225 }
226
227 static void rcu_tasks_ref_scale_delay_section(
228 {
229 int i;
230
231 for (i = nloops; i >= 0; i--)
232 un_delay(udl, ndl);
233 }
234
235 static const struct ref_scale_ops rcu_tasks_op
236 .init = rcu_sync_scale_init,
237 .readsection = rcu_tasks_ref_scale_
238 .delaysection = rcu_tasks_ref_scale_
239 .name = "rcu-tasks"
240 };
241
242 #define RCU_TASKS_OPS &rcu_tasks_ops,
243
244 #else // #ifdef CONFIG_TASKS_RCU
245
246 #define RCU_TASKS_OPS
247
248 #endif // #else // #ifdef CONFIG_TASKS_RCU
249
250 #ifdef CONFIG_TASKS_TRACE_RCU
251
252 // Definitions for RCU Tasks Trace ref scale t
253 static void rcu_trace_ref_scale_read_section(c
254 {
255 int i;
256
257 for (i = nloops; i >= 0; i--) {
258 rcu_read_lock_trace();
259 rcu_read_unlock_trace();
260 }
261 }
262
263 static void rcu_trace_ref_scale_delay_section(
264 {
265 int i;
266
267 for (i = nloops; i >= 0; i--) {
268 rcu_read_lock_trace();
269 un_delay(udl, ndl);
270 rcu_read_unlock_trace();
271 }
272 }
273
274 static const struct ref_scale_ops rcu_trace_op
275 .init = rcu_sync_scale_init,
276 .readsection = rcu_trace_ref_scale_
277 .delaysection = rcu_trace_ref_scale_
278 .name = "rcu-trace"
279 };
280
281 #define RCU_TRACE_OPS &rcu_trace_ops,
282
283 #else // #ifdef CONFIG_TASKS_TRACE_RCU
284
285 #define RCU_TRACE_OPS
286
287 #endif // #else // #ifdef CONFIG_TASKS_TRACE_R
288
289 // Definitions for reference count
290 static atomic_t refcnt;
291
292 static void ref_refcnt_section(const int nloop
293 {
294 int i;
295
296 for (i = nloops; i >= 0; i--) {
297 atomic_inc(&refcnt);
298 atomic_dec(&refcnt);
299 }
300 }
301
302 static void ref_refcnt_delay_section(const int
303 {
304 int i;
305
306 for (i = nloops; i >= 0; i--) {
307 atomic_inc(&refcnt);
308 un_delay(udl, ndl);
309 atomic_dec(&refcnt);
310 }
311 }
312
313 static const struct ref_scale_ops refcnt_ops =
314 .init = rcu_sync_scale_init,
315 .readsection = ref_refcnt_section,
316 .delaysection = ref_refcnt_delay_sec
317 .name = "refcnt"
318 };
319
320 // Definitions for rwlock
321 static rwlock_t test_rwlock;
322
323 static bool ref_rwlock_init(void)
324 {
325 rwlock_init(&test_rwlock);
326 return true;
327 }
328
329 static void ref_rwlock_section(const int nloop
330 {
331 int i;
332
333 for (i = nloops; i >= 0; i--) {
334 read_lock(&test_rwlock);
335 read_unlock(&test_rwlock);
336 }
337 }
338
339 static void ref_rwlock_delay_section(const int
340 {
341 int i;
342
343 for (i = nloops; i >= 0; i--) {
344 read_lock(&test_rwlock);
345 un_delay(udl, ndl);
346 read_unlock(&test_rwlock);
347 }
348 }
349
350 static const struct ref_scale_ops rwlock_ops =
351 .init = ref_rwlock_init,
352 .readsection = ref_rwlock_section,
353 .delaysection = ref_rwlock_delay_sec
354 .name = "rwlock"
355 };
356
357 // Definitions for rwsem
358 static struct rw_semaphore test_rwsem;
359
360 static bool ref_rwsem_init(void)
361 {
362 init_rwsem(&test_rwsem);
363 return true;
364 }
365
366 static void ref_rwsem_section(const int nloops
367 {
368 int i;
369
370 for (i = nloops; i >= 0; i--) {
371 down_read(&test_rwsem);
372 up_read(&test_rwsem);
373 }
374 }
375
376 static void ref_rwsem_delay_section(const int
377 {
378 int i;
379
380 for (i = nloops; i >= 0; i--) {
381 down_read(&test_rwsem);
382 un_delay(udl, ndl);
383 up_read(&test_rwsem);
384 }
385 }
386
387 static const struct ref_scale_ops rwsem_ops =
388 .init = ref_rwsem_init,
389 .readsection = ref_rwsem_section,
390 .delaysection = ref_rwsem_delay_sect
391 .name = "rwsem"
392 };
393
394 // Definitions for global spinlock
395 static DEFINE_RAW_SPINLOCK(test_lock);
396
397 static void ref_lock_section(const int nloops)
398 {
399 int i;
400
401 preempt_disable();
402 for (i = nloops; i >= 0; i--) {
403 raw_spin_lock(&test_lock);
404 raw_spin_unlock(&test_lock);
405 }
406 preempt_enable();
407 }
408
409 static void ref_lock_delay_section(const int n
410 {
411 int i;
412
413 preempt_disable();
414 for (i = nloops; i >= 0; i--) {
415 raw_spin_lock(&test_lock);
416 un_delay(udl, ndl);
417 raw_spin_unlock(&test_lock);
418 }
419 preempt_enable();
420 }
421
422 static const struct ref_scale_ops lock_ops = {
423 .readsection = ref_lock_section,
424 .delaysection = ref_lock_delay_secti
425 .name = "lock"
426 };
427
428 // Definitions for global irq-save spinlock
429
430 static void ref_lock_irq_section(const int nlo
431 {
432 unsigned long flags;
433 int i;
434
435 preempt_disable();
436 for (i = nloops; i >= 0; i--) {
437 raw_spin_lock_irqsave(&test_lo
438 raw_spin_unlock_irqrestore(&te
439 }
440 preempt_enable();
441 }
442
443 static void ref_lock_irq_delay_section(const i
444 {
445 unsigned long flags;
446 int i;
447
448 preempt_disable();
449 for (i = nloops; i >= 0; i--) {
450 raw_spin_lock_irqsave(&test_lo
451 un_delay(udl, ndl);
452 raw_spin_unlock_irqrestore(&te
453 }
454 preempt_enable();
455 }
456
457 static const struct ref_scale_ops lock_irq_ops
458 .readsection = ref_lock_irq_section
459 .delaysection = ref_lock_irq_delay_s
460 .name = "lock-irq"
461 };
462
463 // Definitions acquire-release.
464 static DEFINE_PER_CPU(unsigned long, test_acqr
465
466 static void ref_acqrel_section(const int nloop
467 {
468 unsigned long x;
469 int i;
470
471 preempt_disable();
472 for (i = nloops; i >= 0; i--) {
473 x = smp_load_acquire(this_cpu_
474 smp_store_release(this_cpu_ptr
475 }
476 preempt_enable();
477 }
478
479 static void ref_acqrel_delay_section(const int
480 {
481 unsigned long x;
482 int i;
483
484 preempt_disable();
485 for (i = nloops; i >= 0; i--) {
486 x = smp_load_acquire(this_cpu_
487 un_delay(udl, ndl);
488 smp_store_release(this_cpu_ptr
489 }
490 preempt_enable();
491 }
492
493 static const struct ref_scale_ops acqrel_ops =
494 .readsection = ref_acqrel_section,
495 .delaysection = ref_acqrel_delay_sec
496 .name = "acqrel"
497 };
498
499 static volatile u64 stopopts;
500
501 static void ref_clock_section(const int nloops
502 {
503 u64 x = 0;
504 int i;
505
506 preempt_disable();
507 for (i = nloops; i >= 0; i--)
508 x += ktime_get_real_fast_ns();
509 preempt_enable();
510 stopopts = x;
511 }
512
513 static void ref_clock_delay_section(const int
514 {
515 u64 x = 0;
516 int i;
517
518 preempt_disable();
519 for (i = nloops; i >= 0; i--) {
520 x += ktime_get_real_fast_ns();
521 un_delay(udl, ndl);
522 }
523 preempt_enable();
524 stopopts = x;
525 }
526
527 static const struct ref_scale_ops clock_ops =
528 .readsection = ref_clock_section,
529 .delaysection = ref_clock_delay_sect
530 .name = "clock"
531 };
532
533 static void ref_jiffies_section(const int nloo
534 {
535 u64 x = 0;
536 int i;
537
538 preempt_disable();
539 for (i = nloops; i >= 0; i--)
540 x += jiffies;
541 preempt_enable();
542 stopopts = x;
543 }
544
545 static void ref_jiffies_delay_section(const in
546 {
547 u64 x = 0;
548 int i;
549
550 preempt_disable();
551 for (i = nloops; i >= 0; i--) {
552 x += jiffies;
553 un_delay(udl, ndl);
554 }
555 preempt_enable();
556 stopopts = x;
557 }
558
559 static const struct ref_scale_ops jiffies_ops
560 .readsection = ref_jiffies_section,
561 .delaysection = ref_jiffies_delay_se
562 .name = "jiffies"
563 };
564
565 //////////////////////////////////////////////
566 //
567 // Methods leveraging SLAB_TYPESAFE_BY_RCU.
568 //
569
570 // Item to look up in a typesafe manner. Arra
571 struct refscale_typesafe {
572 atomic_t rts_refctr; // Used by all f
573 spinlock_t rts_lock;
574 seqlock_t rts_seqlock;
575 unsigned int a;
576 unsigned int b;
577 };
578
579 static struct kmem_cache *typesafe_kmem_cachep
580 static struct refscale_typesafe **rtsarray;
581 static long rtsarray_size;
582 static DEFINE_TORTURE_RANDOM_PERCPU(refscale_r
583 static bool (*rts_acquire)(struct refscale_typ
584 static bool (*rts_release)(struct refscale_typ
585
586 // Conditionally acquire an explicit in-struct
587 static bool typesafe_ref_acquire(struct refsca
588 {
589 return atomic_inc_not_zero(&rtsp->rts_
590 }
591
592 // Unconditionally release an explicit in-stru
593 static bool typesafe_ref_release(struct refsca
594 {
595 if (!atomic_dec_return(&rtsp->rts_refc
596 WRITE_ONCE(rtsp->a, rtsp->a +
597 kmem_cache_free(typesafe_kmem_
598 }
599 return true;
600 }
601
602 // Unconditionally acquire an explicit in-stru
603 static bool typesafe_lock_acquire(struct refsc
604 {
605 spin_lock(&rtsp->rts_lock);
606 return true;
607 }
608
609 // Unconditionally release an explicit in-stru
610 static bool typesafe_lock_release(struct refsc
611 {
612 spin_unlock(&rtsp->rts_lock);
613 return true;
614 }
615
616 // Unconditionally acquire an explicit in-stru
617 static bool typesafe_seqlock_acquire(struct re
618 {
619 *start = read_seqbegin(&rtsp->rts_seql
620 return true;
621 }
622
623 // Conditionally release an explicit in-struct
624 // true if this release was successful, that i
625 static bool typesafe_seqlock_release(struct re
626 {
627 return !read_seqretry(&rtsp->rts_seqlo
628 }
629
630 // Do a read-side critical section with the sp
631 // microseconds and nanoseconds inserted so as
632 // of failure.
633 static void typesafe_delay_section(const int n
634 {
635 unsigned int a;
636 unsigned int b;
637 int i;
638 long idx;
639 struct refscale_typesafe *rtsp;
640 unsigned int start;
641
642 for (i = nloops; i >= 0; i--) {
643 preempt_disable();
644 idx = torture_random(this_cpu_
645 preempt_enable();
646 retry:
647 rcu_read_lock();
648 rtsp = rcu_dereference(rtsarra
649 a = READ_ONCE(rtsp->a);
650 if (!rts_acquire(rtsp, &start)
651 rcu_read_unlock();
652 goto retry;
653 }
654 if (a != READ_ONCE(rtsp->a)) {
655 (void)rts_release(rtsp
656 rcu_read_unlock();
657 goto retry;
658 }
659 un_delay(udl, ndl);
660 b = READ_ONCE(rtsp->a);
661 // Remember, seqlock read-side
662 if (!rts_release(rtsp, start))
663 rcu_read_unlock();
664 goto retry;
665 }
666 WARN_ONCE(a != b, "Re-read of
667 b = rtsp->b;
668 rcu_read_unlock();
669 WARN_ON_ONCE(a * a != b);
670 }
671 }
672
673 // Because the acquisition and release methods
674 // is no point in optimizing away the un_delay
675 // Thus simply define typesafe_read_section()
676 // typesafe_delay_section().
677 static void typesafe_read_section(const int nl
678 {
679 typesafe_delay_section(nloops, 0, 0);
680 }
681
682 // Allocate and initialize one refscale_typesa
683 static struct refscale_typesafe *typesafe_allo
684 {
685 struct refscale_typesafe *rtsp;
686
687 rtsp = kmem_cache_alloc(typesafe_kmem_
688 if (!rtsp)
689 return NULL;
690 atomic_set(&rtsp->rts_refctr, 1);
691 WRITE_ONCE(rtsp->a, rtsp->a + 1);
692 WRITE_ONCE(rtsp->b, rtsp->a * rtsp->a)
693 return rtsp;
694 }
695
696 // Slab-allocator constructor for refscale_typ
697 // out of a new slab of system memory.
698 static void refscale_typesafe_ctor(void *rtsp_
699 {
700 struct refscale_typesafe *rtsp = rtsp_
701
702 spin_lock_init(&rtsp->rts_lock);
703 seqlock_init(&rtsp->rts_seqlock);
704 preempt_disable();
705 rtsp->a = torture_random(this_cpu_ptr(
706 preempt_enable();
707 }
708
709 static const struct ref_scale_ops typesafe_ref
710 static const struct ref_scale_ops typesafe_loc
711 static const struct ref_scale_ops typesafe_seq
712
713 // Initialize for a typesafe test.
714 static bool typesafe_init(void)
715 {
716 long idx;
717 long si = lookup_instances;
718
719 typesafe_kmem_cachep = kmem_cache_crea
720
721
722 if (!typesafe_kmem_cachep)
723 return false;
724 if (si < 0)
725 si = -si * nr_cpu_ids;
726 else if (si == 0)
727 si = nr_cpu_ids;
728 rtsarray_size = si;
729 rtsarray = kcalloc(si, sizeof(*rtsarra
730 if (!rtsarray)
731 return false;
732 for (idx = 0; idx < rtsarray_size; idx
733 rtsarray[idx] = typesafe_alloc
734 if (!rtsarray[idx])
735 return false;
736 }
737 if (cur_ops == &typesafe_ref_ops) {
738 rts_acquire = typesafe_ref_acq
739 rts_release = typesafe_ref_rel
740 } else if (cur_ops == &typesafe_lock_o
741 rts_acquire = typesafe_lock_ac
742 rts_release = typesafe_lock_re
743 } else if (cur_ops == &typesafe_seqloc
744 rts_acquire = typesafe_seqlock
745 rts_release = typesafe_seqlock
746 } else {
747 WARN_ON_ONCE(1);
748 return false;
749 }
750 return true;
751 }
752
753 // Clean up after a typesafe test.
754 static void typesafe_cleanup(void)
755 {
756 long idx;
757
758 if (rtsarray) {
759 for (idx = 0; idx < rtsarray_s
760 kmem_cache_free(typesa
761 kfree(rtsarray);
762 rtsarray = NULL;
763 rtsarray_size = 0;
764 }
765 kmem_cache_destroy(typesafe_kmem_cache
766 typesafe_kmem_cachep = NULL;
767 rts_acquire = NULL;
768 rts_release = NULL;
769 }
770
771 // The typesafe_init() function distinguishes
772 static const struct ref_scale_ops typesafe_ref
773 .init = typesafe_init,
774 .cleanup = typesafe_cleanup,
775 .readsection = typesafe_read_sectio
776 .delaysection = typesafe_delay_secti
777 .name = "typesafe_ref"
778 };
779
780 static const struct ref_scale_ops typesafe_loc
781 .init = typesafe_init,
782 .cleanup = typesafe_cleanup,
783 .readsection = typesafe_read_sectio
784 .delaysection = typesafe_delay_secti
785 .name = "typesafe_lock"
786 };
787
788 static const struct ref_scale_ops typesafe_seq
789 .init = typesafe_init,
790 .cleanup = typesafe_cleanup,
791 .readsection = typesafe_read_sectio
792 .delaysection = typesafe_delay_secti
793 .name = "typesafe_seqlock"
794 };
795
796 static void rcu_scale_one_reader(void)
797 {
798 if (readdelay <= 0)
799 cur_ops->readsection(loops);
800 else
801 cur_ops->delaysection(loops, r
802 }
803
804 // Reader kthread. Repeatedly does empty RCU
805 // critical section, minimizing update-side in
806 static int
807 ref_scale_reader(void *arg)
808 {
809 unsigned long flags;
810 long me = (long)arg;
811 struct reader_task *rt = &(reader_task
812 u64 start;
813 s64 duration;
814
815 VERBOSE_SCALEOUT_BATCH("ref_scale_read
816 WARN_ON_ONCE(set_cpus_allowed_ptr(curr
817 set_user_nice(current, MAX_NICE);
818 atomic_inc(&n_init);
819 if (holdoff)
820 schedule_timeout_interruptible
821 repeat:
822 VERBOSE_SCALEOUT_BATCH("ref_scale_read
823
824 // Wait for signal that this reader ca
825 wait_event(rt->wq, (atomic_read(&nread
826 torture_must_stop()
827
828 if (torture_must_stop())
829 goto end;
830
831 // Make sure that the CPU is affinitiz
832 WARN_ON_ONCE(raw_smp_processor_id() !=
833
834 WRITE_ONCE(rt->start_reader, 0);
835 if (!atomic_dec_return(&n_started))
836 while (atomic_read_acquire(&n_
837 cpu_relax();
838
839 VERBOSE_SCALEOUT_BATCH("ref_scale_read
840
841
842 // To reduce noise, do an initial cach
843 // in, and then keep warming until eve
844 rcu_scale_one_reader();
845 if (!atomic_dec_return(&n_warmedup))
846 while (atomic_read_acquire(&n_
847 rcu_scale_one_reader()
848 // Also keep interrupts disabled. Thi
849 // of preventing entries into slow pat
850 local_irq_save(flags);
851 start = ktime_get_mono_fast_ns();
852
853 rcu_scale_one_reader();
854
855 duration = ktime_get_mono_fast_ns() -
856 local_irq_restore(flags);
857
858 rt->last_duration_ns = WARN_ON_ONCE(du
859 // To reduce runtime-skew noise, do ma
860 // everyone is done.
861 if (!atomic_dec_return(&n_cooleddown))
862 while (atomic_read_acquire(&n_
863 rcu_scale_one_reader()
864
865 if (atomic_dec_and_test(&nreaders_exp)
866 wake_up(&main_wq);
867
868 VERBOSE_SCALEOUT_BATCH("ref_scale_read
869 me, exp_idx, a
870
871 if (!torture_must_stop())
872 goto repeat;
873 end:
874 torture_kthread_stopping("ref_scale_re
875 return 0;
876 }
877
878 static void reset_readers(void)
879 {
880 int i;
881 struct reader_task *rt;
882
883 for (i = 0; i < nreaders; i++) {
884 rt = &(reader_tasks[i]);
885
886 rt->last_duration_ns = 0;
887 }
888 }
889
890 // Print the results of each reader and return
891 static u64 process_durations(int n)
892 {
893 int i;
894 struct reader_task *rt;
895 struct seq_buf s;
896 char *buf;
897 u64 sum = 0;
898
899 buf = kmalloc(800 + 64, GFP_KERNEL);
900 if (!buf)
901 return 0;
902 seq_buf_init(&s, buf, 800 + 64);
903
904 seq_buf_printf(&s, "Experiment #%d (Fo
905 exp_idx);
906
907 for (i = 0; i < n && !torture_must_sto
908 rt = &(reader_tasks[i]);
909
910 if (i % 5 == 0)
911 seq_buf_putc(&s, '\n')
912
913 if (seq_buf_used(&s) >= 800) {
914 pr_alert("%s", seq_buf
915 seq_buf_clear(&s);
916 }
917
918 seq_buf_printf(&s, "%d: %llu\t
919
920 sum += rt->last_duration_ns;
921 }
922 pr_alert("%s\n", seq_buf_str(&s));
923
924 kfree(buf);
925 return sum;
926 }
927
928 // The main_func is the main orchestrator, it
929 // experiments. For every experiment, it orde
930 // involved to start and waits for them to fin
931 // then reads their timestamps and starts the
932 // experiment progresses from 1 concurrent rea
933 // point all the timestamps are printed.
934 static int main_func(void *arg)
935 {
936 int exp, r;
937 char buf1[64];
938 char *buf;
939 u64 *result_avg;
940
941 set_cpus_allowed_ptr(current, cpumask_
942 set_user_nice(current, MAX_NICE);
943
944 VERBOSE_SCALEOUT("main_func task start
945 result_avg = kzalloc(nruns * sizeof(*r
946 buf = kzalloc(800 + 64, GFP_KERNEL);
947 if (!result_avg || !buf) {
948 SCALEOUT_ERRSTRING("out of mem
949 goto oom_exit;
950 }
951 if (holdoff)
952 schedule_timeout_interruptible
953
954 // Wait for all threads to start.
955 atomic_inc(&n_init);
956 while (atomic_read(&n_init) < nreaders
957 schedule_timeout_uninterruptib
958
959 // Start exp readers up per experiment
960 for (exp = 0; exp < nruns && !torture_
961 if (torture_must_stop())
962 goto end;
963
964 reset_readers();
965 atomic_set(&nreaders_exp, nrea
966 atomic_set(&n_started, nreader
967 atomic_set(&n_warmedup, nreade
968 atomic_set(&n_cooleddown, nrea
969
970 exp_idx = exp;
971
972 for (r = 0; r < nreaders; r++)
973 smp_store_release(&rea
974 wake_up(&reader_tasks[
975 }
976
977 VERBOSE_SCALEOUT("main_func: e
978 nreaders);
979
980 wait_event(main_wq,
981 !atomic_read(&nread
982
983 VERBOSE_SCALEOUT("main_func: e
984
985 if (torture_must_stop())
986 goto end;
987
988 result_avg[exp] = div_u64(1000
989 }
990
991 // Print the average of all experiment
992 SCALEOUT("END OF TEST. Calculating ave
993
994 pr_alert("Runs\tTime(ns)\n");
995 for (exp = 0; exp < nruns; exp++) {
996 u64 avg;
997 u32 rem;
998
999 avg = div_u64_rem(result_avg[e
1000 sprintf(buf1, "%d\t%llu.%03u\
1001 strcat(buf, buf1);
1002 if (strlen(buf) >= 800) {
1003 pr_alert("%s", buf);
1004 buf[0] = 0;
1005 }
1006 }
1007
1008 pr_alert("%s", buf);
1009
1010 oom_exit:
1011 // This will shutdown everything incl
1012 if (shutdown) {
1013 shutdown_start = 1;
1014 wake_up(&shutdown_wq);
1015 }
1016
1017 // Wait for torture to stop us
1018 while (!torture_must_stop())
1019 schedule_timeout_uninterrupti
1020
1021 end:
1022 torture_kthread_stopping("main_func")
1023 kfree(result_avg);
1024 kfree(buf);
1025 return 0;
1026 }
1027
1028 static void
1029 ref_scale_print_module_parms(const struct ref
1030 {
1031 pr_alert("%s" SCALE_FLAG
1032 "--- %s: verbose=%d verbose
1033 verbose, verbose_batched, sh
1034 }
1035
1036 static void
1037 ref_scale_cleanup(void)
1038 {
1039 int i;
1040
1041 if (torture_cleanup_begin())
1042 return;
1043
1044 if (!cur_ops) {
1045 torture_cleanup_end();
1046 return;
1047 }
1048
1049 if (reader_tasks) {
1050 for (i = 0; i < nreaders; i++
1051 torture_stop_kthread(
1052
1053 }
1054 kfree(reader_tasks);
1055
1056 torture_stop_kthread("main_task", mai
1057 kfree(main_task);
1058
1059 // Do scale-type-specific cleanup ope
1060 if (cur_ops->cleanup != NULL)
1061 cur_ops->cleanup();
1062
1063 torture_cleanup_end();
1064 }
1065
1066 // Shutdown kthread. Just waits to be awaken
1067 static int
1068 ref_scale_shutdown(void *arg)
1069 {
1070 wait_event_idle(shutdown_wq, shutdown
1071
1072 smp_mb(); // Wake before output.
1073 ref_scale_cleanup();
1074 kernel_power_off();
1075
1076 return -EINVAL;
1077 }
1078
1079 static int __init
1080 ref_scale_init(void)
1081 {
1082 long i;
1083 int firsterr = 0;
1084 static const struct ref_scale_ops *sc
1085 &rcu_ops, &srcu_ops, RCU_TRAC
1086 &rwsem_ops, &lock_ops, &lock_
1087 &typesafe_ref_ops, &typesafe_
1088 };
1089
1090 if (!torture_init_begin(scale_type, v
1091 return -EBUSY;
1092
1093 for (i = 0; i < ARRAY_SIZE(scale_ops)
1094 cur_ops = scale_ops[i];
1095 if (strcmp(scale_type, cur_op
1096 break;
1097 }
1098 if (i == ARRAY_SIZE(scale_ops)) {
1099 pr_alert("rcu-scale: invalid
1100 pr_alert("rcu-scale types:");
1101 for (i = 0; i < ARRAY_SIZE(sc
1102 pr_cont(" %s", scale_
1103 pr_cont("\n");
1104 firsterr = -EINVAL;
1105 cur_ops = NULL;
1106 goto unwind;
1107 }
1108 if (cur_ops->init)
1109 if (!cur_ops->init()) {
1110 firsterr = -EUCLEAN;
1111 goto unwind;
1112 }
1113
1114 ref_scale_print_module_parms(cur_ops,
1115
1116 // Shutdown task
1117 if (shutdown) {
1118 init_waitqueue_head(&shutdown
1119 firsterr = torture_create_kth
1120
1121 if (torture_init_error(firste
1122 goto unwind;
1123 schedule_timeout_uninterrupti
1124 }
1125
1126 // Reader tasks (default to ~75% of o
1127 if (nreaders < 0)
1128 nreaders = (num_online_cpus()
1129 if (WARN_ONCE(loops <= 0, "%s: loops
1130 loops = 1;
1131 if (WARN_ONCE(nreaders <= 0, "%s: nre
1132 nreaders = 1;
1133 if (WARN_ONCE(nruns <= 0, "%s: nruns
1134 nruns = 1;
1135 reader_tasks = kcalloc(nreaders, size
1136 GFP_KERNEL);
1137 if (!reader_tasks) {
1138 SCALEOUT_ERRSTRING("out of me
1139 firsterr = -ENOMEM;
1140 goto unwind;
1141 }
1142
1143 VERBOSE_SCALEOUT("Starting %d reader
1144
1145 for (i = 0; i < nreaders; i++) {
1146 init_waitqueue_head(&reader_t
1147 firsterr = torture_create_kth
1148
1149 if (torture_init_error(firste
1150 goto unwind;
1151 }
1152
1153 // Main Task
1154 init_waitqueue_head(&main_wq);
1155 firsterr = torture_create_kthread(mai
1156 if (torture_init_error(firsterr))
1157 goto unwind;
1158
1159 torture_init_end();
1160 return 0;
1161
1162 unwind:
1163 torture_init_end();
1164 ref_scale_cleanup();
1165 if (shutdown) {
1166 WARN_ON(!IS_MODULE(CONFIG_RCU
1167 kernel_power_off();
1168 }
1169 return firsterr;
1170 }
1171
1172 module_init(ref_scale_init);
1173 module_exit(ref_scale_cleanup);
1174
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.