1 // SPDX-License-Identifier: GPL-2.0-or-later 1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /* 2 /*
3 * shadow.c - Shadow Variables 3 * shadow.c - Shadow Variables
4 * 4 *
5 * Copyright (C) 2014 Josh Poimboeuf <jpoimboe 5 * Copyright (C) 2014 Josh Poimboeuf <jpoimboe@redhat.com>
6 * Copyright (C) 2014 Seth Jennings <sjenning@ 6 * Copyright (C) 2014 Seth Jennings <sjenning@redhat.com>
7 * Copyright (C) 2017 Joe Lawrence <joe.lawren 7 * Copyright (C) 2017 Joe Lawrence <joe.lawrence@redhat.com>
8 */ 8 */
9 9
10 /** 10 /**
11 * DOC: Shadow variable API concurrency notes: 11 * DOC: Shadow variable API concurrency notes:
12 * 12 *
13 * The shadow variable API provides a simple r 13 * The shadow variable API provides a simple relationship between an
14 * <obj, id> pair and a pointer value. It is 14 * <obj, id> pair and a pointer value. It is the responsibility of the
15 * caller to provide any mutual exclusion requ 15 * caller to provide any mutual exclusion required of the shadow data.
16 * 16 *
17 * Once a shadow variable is attached to its p 17 * Once a shadow variable is attached to its parent object via the
18 * klp_shadow_*alloc() API calls, it is consid 18 * klp_shadow_*alloc() API calls, it is considered live: any subsequent
19 * call to klp_shadow_get() may then return th 19 * call to klp_shadow_get() may then return the shadow variable's data
20 * pointer. Callers of klp_shadow_*alloc() sh 20 * pointer. Callers of klp_shadow_*alloc() should prepare shadow data
21 * accordingly. 21 * accordingly.
22 * 22 *
23 * The klp_shadow_*alloc() API calls may alloc 23 * The klp_shadow_*alloc() API calls may allocate memory for new shadow
24 * variable structures. Their implementation 24 * variable structures. Their implementation does not call kmalloc
25 * inside any spinlocks, but API callers shoul 25 * inside any spinlocks, but API callers should pass GFP flags according
26 * to their specific needs. 26 * to their specific needs.
27 * 27 *
28 * The klp_shadow_hash is an RCU-enabled hasht 28 * The klp_shadow_hash is an RCU-enabled hashtable and is safe against
29 * concurrent klp_shadow_free() and klp_shadow 29 * concurrent klp_shadow_free() and klp_shadow_get() operations.
30 */ 30 */
31 31
32 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 32 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
33 33
34 #include <linux/hashtable.h> 34 #include <linux/hashtable.h>
35 #include <linux/slab.h> 35 #include <linux/slab.h>
36 #include <linux/livepatch.h> 36 #include <linux/livepatch.h>
37 37
38 static DEFINE_HASHTABLE(klp_shadow_hash, 12); 38 static DEFINE_HASHTABLE(klp_shadow_hash, 12);
39 39
40 /* 40 /*
41 * klp_shadow_lock provides exclusive access t 41 * klp_shadow_lock provides exclusive access to the klp_shadow_hash and
42 * the shadow variables it references. 42 * the shadow variables it references.
43 */ 43 */
44 static DEFINE_SPINLOCK(klp_shadow_lock); 44 static DEFINE_SPINLOCK(klp_shadow_lock);
45 45
46 /** 46 /**
47 * struct klp_shadow - shadow variable structu 47 * struct klp_shadow - shadow variable structure
48 * @node: klp_shadow_hash hash table nod 48 * @node: klp_shadow_hash hash table node
49 * @rcu_head: RCU is used to safely free thi 49 * @rcu_head: RCU is used to safely free this structure
50 * @obj: pointer to parent object 50 * @obj: pointer to parent object
51 * @id: data identifier 51 * @id: data identifier
52 * @data: data area 52 * @data: data area
53 */ 53 */
54 struct klp_shadow { 54 struct klp_shadow {
55 struct hlist_node node; 55 struct hlist_node node;
56 struct rcu_head rcu_head; 56 struct rcu_head rcu_head;
57 void *obj; 57 void *obj;
58 unsigned long id; 58 unsigned long id;
59 char data[]; 59 char data[];
60 }; 60 };
61 61
62 /** 62 /**
63 * klp_shadow_match() - verify a shadow variab 63 * klp_shadow_match() - verify a shadow variable matches given <obj, id>
64 * @shadow: shadow variable to match 64 * @shadow: shadow variable to match
65 * @obj: pointer to parent object 65 * @obj: pointer to parent object
66 * @id: data identifier 66 * @id: data identifier
67 * 67 *
68 * Return: true if the shadow variable matches 68 * Return: true if the shadow variable matches.
69 */ 69 */
70 static inline bool klp_shadow_match(struct klp 70 static inline bool klp_shadow_match(struct klp_shadow *shadow, void *obj,
71 unsigned long 71 unsigned long id)
72 { 72 {
73 return shadow->obj == obj && shadow->i 73 return shadow->obj == obj && shadow->id == id;
74 } 74 }
75 75
76 /** 76 /**
77 * klp_shadow_get() - retrieve a shadow variab 77 * klp_shadow_get() - retrieve a shadow variable data pointer
78 * @obj: pointer to parent object 78 * @obj: pointer to parent object
79 * @id: data identifier 79 * @id: data identifier
80 * 80 *
81 * Return: the shadow variable data element, N 81 * Return: the shadow variable data element, NULL on failure.
82 */ 82 */
83 void *klp_shadow_get(void *obj, unsigned long 83 void *klp_shadow_get(void *obj, unsigned long id)
84 { 84 {
85 struct klp_shadow *shadow; 85 struct klp_shadow *shadow;
86 86
87 rcu_read_lock(); 87 rcu_read_lock();
88 88
89 hash_for_each_possible_rcu(klp_shadow_ 89 hash_for_each_possible_rcu(klp_shadow_hash, shadow, node,
90 (unsigned l 90 (unsigned long)obj) {
91 91
92 if (klp_shadow_match(shadow, o 92 if (klp_shadow_match(shadow, obj, id)) {
93 rcu_read_unlock(); 93 rcu_read_unlock();
94 return shadow->data; 94 return shadow->data;
95 } 95 }
96 } 96 }
97 97
98 rcu_read_unlock(); 98 rcu_read_unlock();
99 99
100 return NULL; 100 return NULL;
101 } 101 }
102 EXPORT_SYMBOL_GPL(klp_shadow_get); 102 EXPORT_SYMBOL_GPL(klp_shadow_get);
103 103
104 static void *__klp_shadow_get_or_alloc(void *o 104 static void *__klp_shadow_get_or_alloc(void *obj, unsigned long id,
105 size_t 105 size_t size, gfp_t gfp_flags,
106 klp_sha 106 klp_shadow_ctor_t ctor, void *ctor_data,
107 bool wa 107 bool warn_on_exist)
108 { 108 {
109 struct klp_shadow *new_shadow; 109 struct klp_shadow *new_shadow;
110 void *shadow_data; 110 void *shadow_data;
111 unsigned long flags; 111 unsigned long flags;
112 112
113 /* Check if the shadow variable alread 113 /* Check if the shadow variable already exists */
114 shadow_data = klp_shadow_get(obj, id); 114 shadow_data = klp_shadow_get(obj, id);
115 if (shadow_data) 115 if (shadow_data)
116 goto exists; 116 goto exists;
117 117
118 /* 118 /*
119 * Allocate a new shadow variable. Fi 119 * Allocate a new shadow variable. Fill it with zeroes by default.
120 * More complex setting can be done by 120 * More complex setting can be done by @ctor function. But it is
121 * called only when the buffer is real 121 * called only when the buffer is really used (under klp_shadow_lock).
122 */ 122 */
123 new_shadow = kzalloc(size + sizeof(*ne 123 new_shadow = kzalloc(size + sizeof(*new_shadow), gfp_flags);
124 if (!new_shadow) 124 if (!new_shadow)
125 return NULL; 125 return NULL;
126 126
127 /* Look for <obj, id> again under the 127 /* Look for <obj, id> again under the lock */
128 spin_lock_irqsave(&klp_shadow_lock, fl 128 spin_lock_irqsave(&klp_shadow_lock, flags);
129 shadow_data = klp_shadow_get(obj, id); 129 shadow_data = klp_shadow_get(obj, id);
130 if (unlikely(shadow_data)) { 130 if (unlikely(shadow_data)) {
131 /* 131 /*
132 * Shadow variable was found, 132 * Shadow variable was found, throw away speculative
133 * allocation. 133 * allocation.
134 */ 134 */
135 spin_unlock_irqrestore(&klp_sh 135 spin_unlock_irqrestore(&klp_shadow_lock, flags);
136 kfree(new_shadow); 136 kfree(new_shadow);
137 goto exists; 137 goto exists;
138 } 138 }
139 139
140 new_shadow->obj = obj; 140 new_shadow->obj = obj;
141 new_shadow->id = id; 141 new_shadow->id = id;
142 142
143 if (ctor) { 143 if (ctor) {
144 int err; 144 int err;
145 145
146 err = ctor(obj, new_shadow->da 146 err = ctor(obj, new_shadow->data, ctor_data);
147 if (err) { 147 if (err) {
148 spin_unlock_irqrestore 148 spin_unlock_irqrestore(&klp_shadow_lock, flags);
149 kfree(new_shadow); 149 kfree(new_shadow);
150 pr_err("Failed to cons 150 pr_err("Failed to construct shadow variable <%p, %lx> (%d)\n",
151 obj, id, err); 151 obj, id, err);
152 return NULL; 152 return NULL;
153 } 153 }
154 } 154 }
155 155
156 /* No <obj, id> found, so attach the n 156 /* No <obj, id> found, so attach the newly allocated one */
157 hash_add_rcu(klp_shadow_hash, &new_sha 157 hash_add_rcu(klp_shadow_hash, &new_shadow->node,
158 (unsigned long)new_shadow 158 (unsigned long)new_shadow->obj);
159 spin_unlock_irqrestore(&klp_shadow_loc 159 spin_unlock_irqrestore(&klp_shadow_lock, flags);
160 160
161 return new_shadow->data; 161 return new_shadow->data;
162 162
163 exists: 163 exists:
164 if (warn_on_exist) { 164 if (warn_on_exist) {
165 WARN(1, "Duplicate shadow vari 165 WARN(1, "Duplicate shadow variable <%p, %lx>\n", obj, id);
166 return NULL; 166 return NULL;
167 } 167 }
168 168
169 return shadow_data; 169 return shadow_data;
170 } 170 }
171 171
172 /** 172 /**
173 * klp_shadow_alloc() - allocate and add a new 173 * klp_shadow_alloc() - allocate and add a new shadow variable
174 * @obj: pointer to parent object 174 * @obj: pointer to parent object
175 * @id: data identifier 175 * @id: data identifier
176 * @size: size of attached data 176 * @size: size of attached data
177 * @gfp_flags: GFP mask for allocation 177 * @gfp_flags: GFP mask for allocation
178 * @ctor: custom constructor to initiali 178 * @ctor: custom constructor to initialize the shadow data (optional)
179 * @ctor_data: pointer to any data needed by 179 * @ctor_data: pointer to any data needed by @ctor (optional)
180 * 180 *
181 * Allocates @size bytes for new shadow variab 181 * Allocates @size bytes for new shadow variable data using @gfp_flags.
182 * The data are zeroed by default. They are f 182 * The data are zeroed by default. They are further initialized by @ctor
183 * function if it is not NULL. The new shadow 183 * function if it is not NULL. The new shadow variable is then added
184 * to the global hashtable. 184 * to the global hashtable.
185 * 185 *
186 * If an existing <obj, id> shadow variable ca 186 * If an existing <obj, id> shadow variable can be found, this routine will
187 * issue a WARN, exit early and return NULL. 187 * issue a WARN, exit early and return NULL.
188 * 188 *
189 * This function guarantees that the construct 189 * This function guarantees that the constructor function is called only when
190 * the variable did not exist before. The cos 190 * the variable did not exist before. The cost is that @ctor is called
191 * in atomic context under a spin lock. 191 * in atomic context under a spin lock.
192 * 192 *
193 * Return: the shadow variable data element, N 193 * Return: the shadow variable data element, NULL on duplicate or
194 * failure. 194 * failure.
195 */ 195 */
196 void *klp_shadow_alloc(void *obj, unsigned lon 196 void *klp_shadow_alloc(void *obj, unsigned long id,
197 size_t size, gfp_t gfp_ 197 size_t size, gfp_t gfp_flags,
198 klp_shadow_ctor_t ctor, 198 klp_shadow_ctor_t ctor, void *ctor_data)
199 { 199 {
200 return __klp_shadow_get_or_alloc(obj, 200 return __klp_shadow_get_or_alloc(obj, id, size, gfp_flags,
201 ctor, 201 ctor, ctor_data, true);
202 } 202 }
203 EXPORT_SYMBOL_GPL(klp_shadow_alloc); 203 EXPORT_SYMBOL_GPL(klp_shadow_alloc);
204 204
205 /** 205 /**
206 * klp_shadow_get_or_alloc() - get existing or 206 * klp_shadow_get_or_alloc() - get existing or allocate a new shadow variable
207 * @obj: pointer to parent object 207 * @obj: pointer to parent object
208 * @id: data identifier 208 * @id: data identifier
209 * @size: size of attached data 209 * @size: size of attached data
210 * @gfp_flags: GFP mask for allocation 210 * @gfp_flags: GFP mask for allocation
211 * @ctor: custom constructor to initiali 211 * @ctor: custom constructor to initialize the shadow data (optional)
212 * @ctor_data: pointer to any data needed by 212 * @ctor_data: pointer to any data needed by @ctor (optional)
213 * 213 *
214 * Returns a pointer to existing shadow data i 214 * Returns a pointer to existing shadow data if an <obj, id> shadow
215 * variable is already present. Otherwise, it 215 * variable is already present. Otherwise, it creates a new shadow
216 * variable like klp_shadow_alloc(). 216 * variable like klp_shadow_alloc().
217 * 217 *
218 * This function guarantees that only one shad 218 * This function guarantees that only one shadow variable exists with the given
219 * @id for the given @obj. It also guarantees 219 * @id for the given @obj. It also guarantees that the constructor function
220 * will be called only when the variable did n 220 * will be called only when the variable did not exist before. The cost is
221 * that @ctor is called in atomic context unde 221 * that @ctor is called in atomic context under a spin lock.
222 * 222 *
223 * Return: the shadow variable data element, N 223 * Return: the shadow variable data element, NULL on failure.
224 */ 224 */
225 void *klp_shadow_get_or_alloc(void *obj, unsig 225 void *klp_shadow_get_or_alloc(void *obj, unsigned long id,
226 size_t size, gfp 226 size_t size, gfp_t gfp_flags,
227 klp_shadow_ctor_ 227 klp_shadow_ctor_t ctor, void *ctor_data)
228 { 228 {
229 return __klp_shadow_get_or_alloc(obj, 229 return __klp_shadow_get_or_alloc(obj, id, size, gfp_flags,
230 ctor, 230 ctor, ctor_data, false);
231 } 231 }
232 EXPORT_SYMBOL_GPL(klp_shadow_get_or_alloc); 232 EXPORT_SYMBOL_GPL(klp_shadow_get_or_alloc);
233 233
234 static void klp_shadow_free_struct(struct klp_ 234 static void klp_shadow_free_struct(struct klp_shadow *shadow,
235 klp_shadow_ 235 klp_shadow_dtor_t dtor)
236 { 236 {
237 hash_del_rcu(&shadow->node); 237 hash_del_rcu(&shadow->node);
238 if (dtor) 238 if (dtor)
239 dtor(shadow->obj, shadow->data 239 dtor(shadow->obj, shadow->data);
240 kfree_rcu(shadow, rcu_head); 240 kfree_rcu(shadow, rcu_head);
241 } 241 }
242 242
243 /** 243 /**
244 * klp_shadow_free() - detach and free a <obj, 244 * klp_shadow_free() - detach and free a <obj, id> shadow variable
245 * @obj: pointer to parent object 245 * @obj: pointer to parent object
246 * @id: data identifier 246 * @id: data identifier
247 * @dtor: custom callback that can be us 247 * @dtor: custom callback that can be used to unregister the variable
248 * and/or free data that the shad 248 * and/or free data that the shadow variable points to (optional)
249 * 249 *
250 * This function releases the memory for this 250 * This function releases the memory for this <obj, id> shadow variable
251 * instance, callers should stop referencing i 251 * instance, callers should stop referencing it accordingly.
252 */ 252 */
253 void klp_shadow_free(void *obj, unsigned long 253 void klp_shadow_free(void *obj, unsigned long id, klp_shadow_dtor_t dtor)
254 { 254 {
255 struct klp_shadow *shadow; 255 struct klp_shadow *shadow;
256 unsigned long flags; 256 unsigned long flags;
257 257
258 spin_lock_irqsave(&klp_shadow_lock, fl 258 spin_lock_irqsave(&klp_shadow_lock, flags);
259 259
260 /* Delete <obj, id> from hash */ 260 /* Delete <obj, id> from hash */
261 hash_for_each_possible(klp_shadow_hash 261 hash_for_each_possible(klp_shadow_hash, shadow, node,
262 (unsigned long) 262 (unsigned long)obj) {
263 263
264 if (klp_shadow_match(shadow, o 264 if (klp_shadow_match(shadow, obj, id)) {
265 klp_shadow_free_struct 265 klp_shadow_free_struct(shadow, dtor);
266 break; 266 break;
267 } 267 }
268 } 268 }
269 269
270 spin_unlock_irqrestore(&klp_shadow_loc 270 spin_unlock_irqrestore(&klp_shadow_lock, flags);
271 } 271 }
272 EXPORT_SYMBOL_GPL(klp_shadow_free); 272 EXPORT_SYMBOL_GPL(klp_shadow_free);
273 273
274 /** 274 /**
275 * klp_shadow_free_all() - detach and free all !! 275 * klp_shadow_free_all() - detach and free all <*, id> shadow variables
276 * @id: data identifier 276 * @id: data identifier
277 * @dtor: custom callback that can be us 277 * @dtor: custom callback that can be used to unregister the variable
278 * and/or free data that the shad 278 * and/or free data that the shadow variable points to (optional)
279 * 279 *
280 * This function releases the memory for all < !! 280 * This function releases the memory for all <*, id> shadow variable
281 * instances, callers should stop referencing 281 * instances, callers should stop referencing them accordingly.
282 */ 282 */
283 void klp_shadow_free_all(unsigned long id, klp 283 void klp_shadow_free_all(unsigned long id, klp_shadow_dtor_t dtor)
284 { 284 {
285 struct klp_shadow *shadow; 285 struct klp_shadow *shadow;
286 unsigned long flags; 286 unsigned long flags;
287 int i; 287 int i;
288 288
289 spin_lock_irqsave(&klp_shadow_lock, fl 289 spin_lock_irqsave(&klp_shadow_lock, flags);
290 290
291 /* Delete all <_, id> from hash */ !! 291 /* Delete all <*, id> from hash */
292 hash_for_each(klp_shadow_hash, i, shad 292 hash_for_each(klp_shadow_hash, i, shadow, node) {
293 if (klp_shadow_match(shadow, s 293 if (klp_shadow_match(shadow, shadow->obj, id))
294 klp_shadow_free_struct 294 klp_shadow_free_struct(shadow, dtor);
295 } 295 }
296 296
297 spin_unlock_irqrestore(&klp_shadow_loc 297 spin_unlock_irqrestore(&klp_shadow_lock, flags);
298 } 298 }
299 EXPORT_SYMBOL_GPL(klp_shadow_free_all); 299 EXPORT_SYMBOL_GPL(klp_shadow_free_all);
300 300
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.