1 /* SPDX-License-Identifier: GPL-2.0+ */ 1
2 /*
3 * Sleepable Read-Copy Update mechanism for mu
4 * tree variant.
5 *
6 * Copyright (C) IBM Corporation, 2017
7 *
8 * Author: Paul McKenney <paulmck@linux.ibm.co
9 */
10
11 #ifndef _LINUX_SRCU_TREE_H
12 #define _LINUX_SRCU_TREE_H
13
14 #include <linux/rcu_node_tree.h>
15 #include <linux/completion.h>
16
17 struct srcu_node;
18 struct srcu_struct;
19
20 /*
21 * Per-CPU structure feeding into leaf srcu_no
22 * to rcu_node.
23 */
24 struct srcu_data {
25 /* Read-side state. */
26 atomic_long_t srcu_lock_count[2];
27 atomic_long_t srcu_unlock_count[2];
28 int srcu_nmi_safety;
29
30 /* Update-side state. */
31 spinlock_t __private lock ____cachelin
32 struct rcu_segcblist srcu_cblist;
33 unsigned long srcu_gp_seq_needed;
34 unsigned long srcu_gp_seq_needed_exp;
35 bool srcu_cblist_invoking;
36 struct timer_list delay_work;
37 struct work_struct work;
38 struct rcu_head srcu_barrier_head;
39 struct srcu_node *mynode;
40 unsigned long grpmask;
41
42 int cpu;
43 struct srcu_struct *ssp;
44 };
45
46 /*
47 * Node in SRCU combining tree, similar in fun
48 */
49 struct srcu_node {
50 spinlock_t __private lock;
51 unsigned long srcu_have_cbs[4];
52
53 unsigned long srcu_data_have_cbs[4];
54 unsigned long srcu_gp_seq_needed_exp;
55 struct srcu_node *srcu_parent;
56 int grplo;
57 int grphi;
58 };
59
60 /*
61 * Per-SRCU-domain structure, update-side data
62 */
63 struct srcu_usage {
64 struct srcu_node *node;
65 struct srcu_node *level[RCU_NUM_LVLS +
66
67 int srcu_size_state;
68 struct mutex srcu_cb_mutex;
69 spinlock_t __private lock;
70 struct mutex srcu_gp_mutex;
71 unsigned long srcu_gp_seq;
72 unsigned long srcu_gp_seq_needed;
73 unsigned long srcu_gp_seq_needed_exp;
74 unsigned long srcu_gp_start;
75 unsigned long srcu_last_gp_end;
76 unsigned long srcu_size_jiffies;
77 unsigned long srcu_n_lock_retries;
78 unsigned long srcu_n_exp_nodelay;
79 bool sda_is_static;
80 unsigned long srcu_barrier_seq;
81 struct mutex srcu_barrier_mutex;
82 struct completion srcu_barrier_complet
83
84 atomic_t srcu_barrier_cpu_cnt;
85
86
87 unsigned long reschedule_jiffies;
88 unsigned long reschedule_count;
89 struct delayed_work work;
90 struct srcu_struct *srcu_ssp;
91 };
92
93 /*
94 * Per-SRCU-domain structure, similar in funct
95 */
96 struct srcu_struct {
97 unsigned int srcu_idx;
98 struct srcu_data __percpu *sda;
99 struct lockdep_map dep_map;
100 struct srcu_usage *srcu_sup;
101 };
102
103 // Values for size state variable (->srcu_size
104 // has been set to SRCU_SIZE_ALLOC, the grace-
105 // this state machine one step per grace perio
106 // is reached. Otherwise, the state machine r
107 // state indefinitely.
108 #define SRCU_SIZE_SMALL 0 // No
109 #define SRCU_SIZE_ALLOC 1 // An
110 // an
111 #define SRCU_SIZE_WAIT_BARRIER 2 // The
112 // ex
113 // By
114 // ar
115 #define SRCU_SIZE_WAIT_CALL 3 // The
116 // By
117 // in
118 // an
119 // co
120 #define SRCU_SIZE_WAIT_CBS1 4 // Don
121 #define SRCU_SIZE_WAIT_CBS2 5 // se
122 #define SRCU_SIZE_WAIT_CBS3 6 // CP
123 #define SRCU_SIZE_WAIT_CBS4 7 // ea
124 #define SRCU_SIZE_BIG 8 // The
125 // an
126
127 /* Values for state variable (bottom bits of -
128 #define SRCU_STATE_IDLE 0
129 #define SRCU_STATE_SCAN1 1
130 #define SRCU_STATE_SCAN2 2
131
132 /*
133 * Values for initializing gp sequence fields.
134 * occur earlier.
135 * The second value with state is useful in th
136 * srcu_usage where srcu_gp_seq_needed is expe
137 * lower bits (or else it will appear to be al
138 * the call check_init_srcu_struct()).
139 */
140 #define SRCU_GP_SEQ_INITIAL_VAL ((0UL - 100UL)
141 #define SRCU_GP_SEQ_INITIAL_VAL_WITH_STATE (SR
142
143 #define __SRCU_USAGE_INIT(name)
144 {
145 .lock = __SPIN_LOCK_UNLOCKED(name.lock
146 .srcu_gp_seq = SRCU_GP_SEQ_INITIAL_VAL
147 .srcu_gp_seq_needed = SRCU_GP_SEQ_INIT
148 .srcu_gp_seq_needed_exp = SRCU_GP_SEQ_
149 .work = __DELAYED_WORK_INITIALIZER(nam
150 }
151
152 #define __SRCU_STRUCT_INIT_COMMON(name, usage_
153 .srcu_sup = &usage_name,
154 __SRCU_DEP_MAP_INIT(name)
155
156 #define __SRCU_STRUCT_INIT_MODULE(name, usage_
157 {
158 __SRCU_STRUCT_INIT_COMMON(name, usage_
159 }
160
161 #define __SRCU_STRUCT_INIT(name, usage_name, p
162 {
163 .sda = &pcpu_name,
164 __SRCU_STRUCT_INIT_COMMON(name, usage_
165 }
166
167 /*
168 * Define and initialize a srcu struct at buil
169 * Do -not- call init_srcu_struct() nor cleanu
170 *
171 * Note that although DEFINE_STATIC_SRCU() hid
172 * files, the per-CPU variable rules neverthel
173 * chosen name be globally unique. These rule
174 * DEFINE_STATIC_SRCU() within a function. If
175 * restrictive, declare the srcu_struct manual
176 * each file:
177 *
178 * static struct srcu_struct my_srcu;
179 *
180 * Then, before the first use of each my_srcu,
181 *
182 * init_srcu_struct(&my_srcu);
183 *
184 * See include/linux/percpu-defs.h for the rul
185 */
186 #ifdef MODULE
187 # define __DEFINE_SRCU(name, is_static)
188 static struct srcu_usage name##_srcu_u
189 is_static struct srcu_struct name = __
190 extern struct srcu_struct * const __sr
191 struct srcu_struct * const __srcu_stru
192 __section("___srcu_struct_ptrs
193 #else
194 # define __DEFINE_SRCU(name, is_static)
195 static DEFINE_PER_CPU(struct srcu_data
196 static struct srcu_usage name##_srcu_u
197 is_static struct srcu_struct name =
198 __SRCU_STRUCT_INIT(name, name#
199 #endif
200 #define DEFINE_SRCU(name) __DEFI
201 #define DEFINE_STATIC_SRCU(name) __DEFI
202
203 void synchronize_srcu_expedited(struct srcu_st
204 void srcu_barrier(struct srcu_struct *ssp);
205 void srcu_torture_stats_print(struct srcu_stru
206
207 #endif
208
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.