1 // SPDX-License-Identifier: GPL-2.0-or-later 1
2 /*
3 * shadow.c - Shadow Variables
4 *
5 * Copyright (C) 2014 Josh Poimboeuf <jpoimboe
6 * Copyright (C) 2014 Seth Jennings <sjenning@
7 * Copyright (C) 2017 Joe Lawrence <joe.lawren
8 */
9
10 /**
11 * DOC: Shadow variable API concurrency notes:
12 *
13 * The shadow variable API provides a simple r
14 * <obj, id> pair and a pointer value. It is
15 * caller to provide any mutual exclusion requ
16 *
17 * Once a shadow variable is attached to its p
18 * klp_shadow_*alloc() API calls, it is consid
19 * call to klp_shadow_get() may then return th
20 * pointer. Callers of klp_shadow_*alloc() sh
21 * accordingly.
22 *
23 * The klp_shadow_*alloc() API calls may alloc
24 * variable structures. Their implementation
25 * inside any spinlocks, but API callers shoul
26 * to their specific needs.
27 *
28 * The klp_shadow_hash is an RCU-enabled hasht
29 * concurrent klp_shadow_free() and klp_shadow
30 */
31
32 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
33
34 #include <linux/hashtable.h>
35 #include <linux/slab.h>
36 #include <linux/livepatch.h>
37
38 static DEFINE_HASHTABLE(klp_shadow_hash, 12);
39
40 /*
41 * klp_shadow_lock provides exclusive access t
42 * the shadow variables it references.
43 */
44 static DEFINE_SPINLOCK(klp_shadow_lock);
45
46 /**
47 * struct klp_shadow - shadow variable structu
48 * @node: klp_shadow_hash hash table nod
49 * @rcu_head: RCU is used to safely free thi
50 * @obj: pointer to parent object
51 * @id: data identifier
52 * @data: data area
53 */
54 struct klp_shadow {
55 struct hlist_node node;
56 struct rcu_head rcu_head;
57 void *obj;
58 unsigned long id;
59 char data[];
60 };
61
62 /**
63 * klp_shadow_match() - verify a shadow variab
64 * @shadow: shadow variable to match
65 * @obj: pointer to parent object
66 * @id: data identifier
67 *
68 * Return: true if the shadow variable matches
69 */
70 static inline bool klp_shadow_match(struct klp
71 unsigned long
72 {
73 return shadow->obj == obj && shadow->i
74 }
75
76 /**
77 * klp_shadow_get() - retrieve a shadow variab
78 * @obj: pointer to parent object
79 * @id: data identifier
80 *
81 * Return: the shadow variable data element, N
82 */
83 void *klp_shadow_get(void *obj, unsigned long
84 {
85 struct klp_shadow *shadow;
86
87 rcu_read_lock();
88
89 hash_for_each_possible_rcu(klp_shadow_
90 (unsigned l
91
92 if (klp_shadow_match(shadow, o
93 rcu_read_unlock();
94 return shadow->data;
95 }
96 }
97
98 rcu_read_unlock();
99
100 return NULL;
101 }
102 EXPORT_SYMBOL_GPL(klp_shadow_get);
103
104 static void *__klp_shadow_get_or_alloc(void *o
105 size_t
106 klp_sha
107 bool wa
108 {
109 struct klp_shadow *new_shadow;
110 void *shadow_data;
111 unsigned long flags;
112
113 /* Check if the shadow variable alread
114 shadow_data = klp_shadow_get(obj, id);
115 if (shadow_data)
116 goto exists;
117
118 /*
119 * Allocate a new shadow variable. Fi
120 * More complex setting can be done by
121 * called only when the buffer is real
122 */
123 new_shadow = kzalloc(size + sizeof(*ne
124 if (!new_shadow)
125 return NULL;
126
127 /* Look for <obj, id> again under the
128 spin_lock_irqsave(&klp_shadow_lock, fl
129 shadow_data = klp_shadow_get(obj, id);
130 if (unlikely(shadow_data)) {
131 /*
132 * Shadow variable was found,
133 * allocation.
134 */
135 spin_unlock_irqrestore(&klp_sh
136 kfree(new_shadow);
137 goto exists;
138 }
139
140 new_shadow->obj = obj;
141 new_shadow->id = id;
142
143 if (ctor) {
144 int err;
145
146 err = ctor(obj, new_shadow->da
147 if (err) {
148 spin_unlock_irqrestore
149 kfree(new_shadow);
150 pr_err("Failed to cons
151 obj, id, err);
152 return NULL;
153 }
154 }
155
156 /* No <obj, id> found, so attach the n
157 hash_add_rcu(klp_shadow_hash, &new_sha
158 (unsigned long)new_shadow
159 spin_unlock_irqrestore(&klp_shadow_loc
160
161 return new_shadow->data;
162
163 exists:
164 if (warn_on_exist) {
165 WARN(1, "Duplicate shadow vari
166 return NULL;
167 }
168
169 return shadow_data;
170 }
171
172 /**
173 * klp_shadow_alloc() - allocate and add a new
174 * @obj: pointer to parent object
175 * @id: data identifier
176 * @size: size of attached data
177 * @gfp_flags: GFP mask for allocation
178 * @ctor: custom constructor to initiali
179 * @ctor_data: pointer to any data needed by
180 *
181 * Allocates @size bytes for new shadow variab
182 * The data are zeroed by default. They are f
183 * function if it is not NULL. The new shadow
184 * to the global hashtable.
185 *
186 * If an existing <obj, id> shadow variable ca
187 * issue a WARN, exit early and return NULL.
188 *
189 * This function guarantees that the construct
190 * the variable did not exist before. The cos
191 * in atomic context under a spin lock.
192 *
193 * Return: the shadow variable data element, N
194 * failure.
195 */
196 void *klp_shadow_alloc(void *obj, unsigned lon
197 size_t size, gfp_t gfp_
198 klp_shadow_ctor_t ctor,
199 {
200 return __klp_shadow_get_or_alloc(obj,
201 ctor,
202 }
203 EXPORT_SYMBOL_GPL(klp_shadow_alloc);
204
205 /**
206 * klp_shadow_get_or_alloc() - get existing or
207 * @obj: pointer to parent object
208 * @id: data identifier
209 * @size: size of attached data
210 * @gfp_flags: GFP mask for allocation
211 * @ctor: custom constructor to initiali
212 * @ctor_data: pointer to any data needed by
213 *
214 * Returns a pointer to existing shadow data i
215 * variable is already present. Otherwise, it
216 * variable like klp_shadow_alloc().
217 *
218 * This function guarantees that only one shad
219 * @id for the given @obj. It also guarantees
220 * will be called only when the variable did n
221 * that @ctor is called in atomic context unde
222 *
223 * Return: the shadow variable data element, N
224 */
225 void *klp_shadow_get_or_alloc(void *obj, unsig
226 size_t size, gfp
227 klp_shadow_ctor_
228 {
229 return __klp_shadow_get_or_alloc(obj,
230 ctor,
231 }
232 EXPORT_SYMBOL_GPL(klp_shadow_get_or_alloc);
233
234 static void klp_shadow_free_struct(struct klp_
235 klp_shadow_
236 {
237 hash_del_rcu(&shadow->node);
238 if (dtor)
239 dtor(shadow->obj, shadow->data
240 kfree_rcu(shadow, rcu_head);
241 }
242
243 /**
244 * klp_shadow_free() - detach and free a <obj,
245 * @obj: pointer to parent object
246 * @id: data identifier
247 * @dtor: custom callback that can be us
248 * and/or free data that the shad
249 *
250 * This function releases the memory for this
251 * instance, callers should stop referencing i
252 */
253 void klp_shadow_free(void *obj, unsigned long
254 {
255 struct klp_shadow *shadow;
256 unsigned long flags;
257
258 spin_lock_irqsave(&klp_shadow_lock, fl
259
260 /* Delete <obj, id> from hash */
261 hash_for_each_possible(klp_shadow_hash
262 (unsigned long)
263
264 if (klp_shadow_match(shadow, o
265 klp_shadow_free_struct
266 break;
267 }
268 }
269
270 spin_unlock_irqrestore(&klp_shadow_loc
271 }
272 EXPORT_SYMBOL_GPL(klp_shadow_free);
273
274 /**
275 * klp_shadow_free_all() - detach and free all
276 * @id: data identifier
277 * @dtor: custom callback that can be us
278 * and/or free data that the shad
279 *
280 * This function releases the memory for all <
281 * instances, callers should stop referencing
282 */
283 void klp_shadow_free_all(unsigned long id, klp
284 {
285 struct klp_shadow *shadow;
286 unsigned long flags;
287 int i;
288
289 spin_lock_irqsave(&klp_shadow_lock, fl
290
291 /* Delete all <_, id> from hash */
292 hash_for_each(klp_shadow_hash, i, shad
293 if (klp_shadow_match(shadow, s
294 klp_shadow_free_struct
295 }
296
297 spin_unlock_irqrestore(&klp_shadow_loc
298 }
299 EXPORT_SYMBOL_GPL(klp_shadow_free_all);
300
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.