1 // SPDX-License-Identifier: GPL-2.0-or-later <<
2 /* 1 /*
3 * shadow.c - Shadow Variables 2 * shadow.c - Shadow Variables
4 * 3 *
5 * Copyright (C) 2014 Josh Poimboeuf <jpoimboe 4 * Copyright (C) 2014 Josh Poimboeuf <jpoimboe@redhat.com>
6 * Copyright (C) 2014 Seth Jennings <sjenning@ 5 * Copyright (C) 2014 Seth Jennings <sjenning@redhat.com>
7 * Copyright (C) 2017 Joe Lawrence <joe.lawren 6 * Copyright (C) 2017 Joe Lawrence <joe.lawrence@redhat.com>
>> 7 *
>> 8 * This program is free software; you can redistribute it and/or
>> 9 * modify it under the terms of the GNU General Public License
>> 10 * as published by the Free Software Foundation; either version 2
>> 11 * of the License, or (at your option) any later version.
>> 12 *
>> 13 * This program is distributed in the hope that it will be useful,
>> 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
>> 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
>> 16 * GNU General Public License for more details.
>> 17 *
>> 18 * You should have received a copy of the GNU General Public License
>> 19 * along with this program; if not, see <http://www.gnu.org/licenses/>.
8 */ 20 */
9 21
10 /** 22 /**
11 * DOC: Shadow variable API concurrency notes: 23 * DOC: Shadow variable API concurrency notes:
12 * 24 *
13 * The shadow variable API provides a simple r 25 * The shadow variable API provides a simple relationship between an
14 * <obj, id> pair and a pointer value. It is 26 * <obj, id> pair and a pointer value. It is the responsibility of the
15 * caller to provide any mutual exclusion requ 27 * caller to provide any mutual exclusion required of the shadow data.
16 * 28 *
17 * Once a shadow variable is attached to its p 29 * Once a shadow variable is attached to its parent object via the
18 * klp_shadow_*alloc() API calls, it is consid 30 * klp_shadow_*alloc() API calls, it is considered live: any subsequent
19 * call to klp_shadow_get() may then return th 31 * call to klp_shadow_get() may then return the shadow variable's data
20 * pointer. Callers of klp_shadow_*alloc() sh 32 * pointer. Callers of klp_shadow_*alloc() should prepare shadow data
21 * accordingly. 33 * accordingly.
22 * 34 *
23 * The klp_shadow_*alloc() API calls may alloc 35 * The klp_shadow_*alloc() API calls may allocate memory for new shadow
24 * variable structures. Their implementation 36 * variable structures. Their implementation does not call kmalloc
25 * inside any spinlocks, but API callers shoul 37 * inside any spinlocks, but API callers should pass GFP flags according
26 * to their specific needs. 38 * to their specific needs.
27 * 39 *
28 * The klp_shadow_hash is an RCU-enabled hasht 40 * The klp_shadow_hash is an RCU-enabled hashtable and is safe against
29 * concurrent klp_shadow_free() and klp_shadow 41 * concurrent klp_shadow_free() and klp_shadow_get() operations.
30 */ 42 */
31 43
32 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 44 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
33 45
34 #include <linux/hashtable.h> 46 #include <linux/hashtable.h>
35 #include <linux/slab.h> 47 #include <linux/slab.h>
36 #include <linux/livepatch.h> 48 #include <linux/livepatch.h>
37 49
38 static DEFINE_HASHTABLE(klp_shadow_hash, 12); 50 static DEFINE_HASHTABLE(klp_shadow_hash, 12);
39 51
40 /* 52 /*
41 * klp_shadow_lock provides exclusive access t 53 * klp_shadow_lock provides exclusive access to the klp_shadow_hash and
42 * the shadow variables it references. 54 * the shadow variables it references.
43 */ 55 */
44 static DEFINE_SPINLOCK(klp_shadow_lock); 56 static DEFINE_SPINLOCK(klp_shadow_lock);
45 57
46 /** 58 /**
47 * struct klp_shadow - shadow variable structu 59 * struct klp_shadow - shadow variable structure
48 * @node: klp_shadow_hash hash table nod 60 * @node: klp_shadow_hash hash table node
49 * @rcu_head: RCU is used to safely free thi 61 * @rcu_head: RCU is used to safely free this structure
50 * @obj: pointer to parent object 62 * @obj: pointer to parent object
51 * @id: data identifier 63 * @id: data identifier
52 * @data: data area 64 * @data: data area
53 */ 65 */
54 struct klp_shadow { 66 struct klp_shadow {
55 struct hlist_node node; 67 struct hlist_node node;
56 struct rcu_head rcu_head; 68 struct rcu_head rcu_head;
57 void *obj; 69 void *obj;
58 unsigned long id; 70 unsigned long id;
59 char data[]; 71 char data[];
60 }; 72 };
61 73
62 /** 74 /**
63 * klp_shadow_match() - verify a shadow variab 75 * klp_shadow_match() - verify a shadow variable matches given <obj, id>
64 * @shadow: shadow variable to match 76 * @shadow: shadow variable to match
65 * @obj: pointer to parent object 77 * @obj: pointer to parent object
66 * @id: data identifier 78 * @id: data identifier
67 * 79 *
68 * Return: true if the shadow variable matches 80 * Return: true if the shadow variable matches.
69 */ 81 */
70 static inline bool klp_shadow_match(struct klp 82 static inline bool klp_shadow_match(struct klp_shadow *shadow, void *obj,
71 unsigned long 83 unsigned long id)
72 { 84 {
73 return shadow->obj == obj && shadow->i 85 return shadow->obj == obj && shadow->id == id;
74 } 86 }
75 87
76 /** 88 /**
77 * klp_shadow_get() - retrieve a shadow variab 89 * klp_shadow_get() - retrieve a shadow variable data pointer
78 * @obj: pointer to parent object 90 * @obj: pointer to parent object
79 * @id: data identifier 91 * @id: data identifier
80 * 92 *
81 * Return: the shadow variable data element, N 93 * Return: the shadow variable data element, NULL on failure.
82 */ 94 */
83 void *klp_shadow_get(void *obj, unsigned long 95 void *klp_shadow_get(void *obj, unsigned long id)
84 { 96 {
85 struct klp_shadow *shadow; 97 struct klp_shadow *shadow;
86 98
87 rcu_read_lock(); 99 rcu_read_lock();
88 100
89 hash_for_each_possible_rcu(klp_shadow_ 101 hash_for_each_possible_rcu(klp_shadow_hash, shadow, node,
90 (unsigned l 102 (unsigned long)obj) {
91 103
92 if (klp_shadow_match(shadow, o 104 if (klp_shadow_match(shadow, obj, id)) {
93 rcu_read_unlock(); 105 rcu_read_unlock();
94 return shadow->data; 106 return shadow->data;
95 } 107 }
96 } 108 }
97 109
98 rcu_read_unlock(); 110 rcu_read_unlock();
99 111
100 return NULL; 112 return NULL;
101 } 113 }
102 EXPORT_SYMBOL_GPL(klp_shadow_get); 114 EXPORT_SYMBOL_GPL(klp_shadow_get);
103 115
104 static void *__klp_shadow_get_or_alloc(void *o 116 static void *__klp_shadow_get_or_alloc(void *obj, unsigned long id,
105 size_t 117 size_t size, gfp_t gfp_flags,
106 klp_sha 118 klp_shadow_ctor_t ctor, void *ctor_data,
107 bool wa 119 bool warn_on_exist)
108 { 120 {
109 struct klp_shadow *new_shadow; 121 struct klp_shadow *new_shadow;
110 void *shadow_data; 122 void *shadow_data;
111 unsigned long flags; 123 unsigned long flags;
112 124
113 /* Check if the shadow variable alread 125 /* Check if the shadow variable already exists */
114 shadow_data = klp_shadow_get(obj, id); 126 shadow_data = klp_shadow_get(obj, id);
115 if (shadow_data) 127 if (shadow_data)
116 goto exists; 128 goto exists;
117 129
118 /* 130 /*
119 * Allocate a new shadow variable. Fi 131 * Allocate a new shadow variable. Fill it with zeroes by default.
120 * More complex setting can be done by 132 * More complex setting can be done by @ctor function. But it is
121 * called only when the buffer is real 133 * called only when the buffer is really used (under klp_shadow_lock).
122 */ 134 */
123 new_shadow = kzalloc(size + sizeof(*ne 135 new_shadow = kzalloc(size + sizeof(*new_shadow), gfp_flags);
124 if (!new_shadow) 136 if (!new_shadow)
125 return NULL; 137 return NULL;
126 138
127 /* Look for <obj, id> again under the 139 /* Look for <obj, id> again under the lock */
128 spin_lock_irqsave(&klp_shadow_lock, fl 140 spin_lock_irqsave(&klp_shadow_lock, flags);
129 shadow_data = klp_shadow_get(obj, id); 141 shadow_data = klp_shadow_get(obj, id);
130 if (unlikely(shadow_data)) { 142 if (unlikely(shadow_data)) {
131 /* 143 /*
132 * Shadow variable was found, 144 * Shadow variable was found, throw away speculative
133 * allocation. 145 * allocation.
134 */ 146 */
135 spin_unlock_irqrestore(&klp_sh 147 spin_unlock_irqrestore(&klp_shadow_lock, flags);
136 kfree(new_shadow); 148 kfree(new_shadow);
137 goto exists; 149 goto exists;
138 } 150 }
139 151
140 new_shadow->obj = obj; 152 new_shadow->obj = obj;
141 new_shadow->id = id; 153 new_shadow->id = id;
142 154
143 if (ctor) { 155 if (ctor) {
144 int err; 156 int err;
145 157
146 err = ctor(obj, new_shadow->da 158 err = ctor(obj, new_shadow->data, ctor_data);
147 if (err) { 159 if (err) {
148 spin_unlock_irqrestore 160 spin_unlock_irqrestore(&klp_shadow_lock, flags);
149 kfree(new_shadow); 161 kfree(new_shadow);
150 pr_err("Failed to cons 162 pr_err("Failed to construct shadow variable <%p, %lx> (%d)\n",
151 obj, id, err); 163 obj, id, err);
152 return NULL; 164 return NULL;
153 } 165 }
154 } 166 }
155 167
156 /* No <obj, id> found, so attach the n 168 /* No <obj, id> found, so attach the newly allocated one */
157 hash_add_rcu(klp_shadow_hash, &new_sha 169 hash_add_rcu(klp_shadow_hash, &new_shadow->node,
158 (unsigned long)new_shadow 170 (unsigned long)new_shadow->obj);
159 spin_unlock_irqrestore(&klp_shadow_loc 171 spin_unlock_irqrestore(&klp_shadow_lock, flags);
160 172
161 return new_shadow->data; 173 return new_shadow->data;
162 174
163 exists: 175 exists:
164 if (warn_on_exist) { 176 if (warn_on_exist) {
165 WARN(1, "Duplicate shadow vari 177 WARN(1, "Duplicate shadow variable <%p, %lx>\n", obj, id);
166 return NULL; 178 return NULL;
167 } 179 }
168 180
169 return shadow_data; 181 return shadow_data;
170 } 182 }
171 183
172 /** 184 /**
173 * klp_shadow_alloc() - allocate and add a new 185 * klp_shadow_alloc() - allocate and add a new shadow variable
174 * @obj: pointer to parent object 186 * @obj: pointer to parent object
175 * @id: data identifier 187 * @id: data identifier
176 * @size: size of attached data 188 * @size: size of attached data
177 * @gfp_flags: GFP mask for allocation 189 * @gfp_flags: GFP mask for allocation
178 * @ctor: custom constructor to initiali 190 * @ctor: custom constructor to initialize the shadow data (optional)
179 * @ctor_data: pointer to any data needed by 191 * @ctor_data: pointer to any data needed by @ctor (optional)
180 * 192 *
181 * Allocates @size bytes for new shadow variab 193 * Allocates @size bytes for new shadow variable data using @gfp_flags.
182 * The data are zeroed by default. They are f 194 * The data are zeroed by default. They are further initialized by @ctor
183 * function if it is not NULL. The new shadow 195 * function if it is not NULL. The new shadow variable is then added
184 * to the global hashtable. 196 * to the global hashtable.
185 * 197 *
186 * If an existing <obj, id> shadow variable ca 198 * If an existing <obj, id> shadow variable can be found, this routine will
187 * issue a WARN, exit early and return NULL. 199 * issue a WARN, exit early and return NULL.
188 * 200 *
189 * This function guarantees that the construct 201 * This function guarantees that the constructor function is called only when
190 * the variable did not exist before. The cos 202 * the variable did not exist before. The cost is that @ctor is called
191 * in atomic context under a spin lock. 203 * in atomic context under a spin lock.
192 * 204 *
193 * Return: the shadow variable data element, N 205 * Return: the shadow variable data element, NULL on duplicate or
194 * failure. 206 * failure.
195 */ 207 */
196 void *klp_shadow_alloc(void *obj, unsigned lon 208 void *klp_shadow_alloc(void *obj, unsigned long id,
197 size_t size, gfp_t gfp_ 209 size_t size, gfp_t gfp_flags,
198 klp_shadow_ctor_t ctor, 210 klp_shadow_ctor_t ctor, void *ctor_data)
199 { 211 {
200 return __klp_shadow_get_or_alloc(obj, 212 return __klp_shadow_get_or_alloc(obj, id, size, gfp_flags,
201 ctor, 213 ctor, ctor_data, true);
202 } 214 }
203 EXPORT_SYMBOL_GPL(klp_shadow_alloc); 215 EXPORT_SYMBOL_GPL(klp_shadow_alloc);
204 216
205 /** 217 /**
206 * klp_shadow_get_or_alloc() - get existing or 218 * klp_shadow_get_or_alloc() - get existing or allocate a new shadow variable
207 * @obj: pointer to parent object 219 * @obj: pointer to parent object
208 * @id: data identifier 220 * @id: data identifier
209 * @size: size of attached data 221 * @size: size of attached data
210 * @gfp_flags: GFP mask for allocation 222 * @gfp_flags: GFP mask for allocation
211 * @ctor: custom constructor to initiali 223 * @ctor: custom constructor to initialize the shadow data (optional)
212 * @ctor_data: pointer to any data needed by 224 * @ctor_data: pointer to any data needed by @ctor (optional)
213 * 225 *
214 * Returns a pointer to existing shadow data i 226 * Returns a pointer to existing shadow data if an <obj, id> shadow
215 * variable is already present. Otherwise, it 227 * variable is already present. Otherwise, it creates a new shadow
216 * variable like klp_shadow_alloc(). 228 * variable like klp_shadow_alloc().
217 * 229 *
218 * This function guarantees that only one shad 230 * This function guarantees that only one shadow variable exists with the given
219 * @id for the given @obj. It also guarantees 231 * @id for the given @obj. It also guarantees that the constructor function
220 * will be called only when the variable did n 232 * will be called only when the variable did not exist before. The cost is
221 * that @ctor is called in atomic context unde 233 * that @ctor is called in atomic context under a spin lock.
222 * 234 *
223 * Return: the shadow variable data element, N 235 * Return: the shadow variable data element, NULL on failure.
224 */ 236 */
225 void *klp_shadow_get_or_alloc(void *obj, unsig 237 void *klp_shadow_get_or_alloc(void *obj, unsigned long id,
226 size_t size, gfp 238 size_t size, gfp_t gfp_flags,
227 klp_shadow_ctor_ 239 klp_shadow_ctor_t ctor, void *ctor_data)
228 { 240 {
229 return __klp_shadow_get_or_alloc(obj, 241 return __klp_shadow_get_or_alloc(obj, id, size, gfp_flags,
230 ctor, 242 ctor, ctor_data, false);
231 } 243 }
232 EXPORT_SYMBOL_GPL(klp_shadow_get_or_alloc); 244 EXPORT_SYMBOL_GPL(klp_shadow_get_or_alloc);
233 245
234 static void klp_shadow_free_struct(struct klp_ 246 static void klp_shadow_free_struct(struct klp_shadow *shadow,
235 klp_shadow_ 247 klp_shadow_dtor_t dtor)
236 { 248 {
237 hash_del_rcu(&shadow->node); 249 hash_del_rcu(&shadow->node);
238 if (dtor) 250 if (dtor)
239 dtor(shadow->obj, shadow->data 251 dtor(shadow->obj, shadow->data);
240 kfree_rcu(shadow, rcu_head); 252 kfree_rcu(shadow, rcu_head);
241 } 253 }
242 254
243 /** 255 /**
244 * klp_shadow_free() - detach and free a <obj, 256 * klp_shadow_free() - detach and free a <obj, id> shadow variable
245 * @obj: pointer to parent object 257 * @obj: pointer to parent object
246 * @id: data identifier 258 * @id: data identifier
247 * @dtor: custom callback that can be us 259 * @dtor: custom callback that can be used to unregister the variable
248 * and/or free data that the shad 260 * and/or free data that the shadow variable points to (optional)
249 * 261 *
250 * This function releases the memory for this 262 * This function releases the memory for this <obj, id> shadow variable
251 * instance, callers should stop referencing i 263 * instance, callers should stop referencing it accordingly.
252 */ 264 */
253 void klp_shadow_free(void *obj, unsigned long 265 void klp_shadow_free(void *obj, unsigned long id, klp_shadow_dtor_t dtor)
254 { 266 {
255 struct klp_shadow *shadow; 267 struct klp_shadow *shadow;
256 unsigned long flags; 268 unsigned long flags;
257 269
258 spin_lock_irqsave(&klp_shadow_lock, fl 270 spin_lock_irqsave(&klp_shadow_lock, flags);
259 271
260 /* Delete <obj, id> from hash */ 272 /* Delete <obj, id> from hash */
261 hash_for_each_possible(klp_shadow_hash 273 hash_for_each_possible(klp_shadow_hash, shadow, node,
262 (unsigned long) 274 (unsigned long)obj) {
263 275
264 if (klp_shadow_match(shadow, o 276 if (klp_shadow_match(shadow, obj, id)) {
265 klp_shadow_free_struct 277 klp_shadow_free_struct(shadow, dtor);
266 break; 278 break;
267 } 279 }
268 } 280 }
269 281
270 spin_unlock_irqrestore(&klp_shadow_loc 282 spin_unlock_irqrestore(&klp_shadow_lock, flags);
271 } 283 }
272 EXPORT_SYMBOL_GPL(klp_shadow_free); 284 EXPORT_SYMBOL_GPL(klp_shadow_free);
273 285
274 /** 286 /**
275 * klp_shadow_free_all() - detach and free all !! 287 * klp_shadow_free_all() - detach and free all <*, id> shadow variables
276 * @id: data identifier 288 * @id: data identifier
277 * @dtor: custom callback that can be us 289 * @dtor: custom callback that can be used to unregister the variable
278 * and/or free data that the shad 290 * and/or free data that the shadow variable points to (optional)
279 * 291 *
280 * This function releases the memory for all < !! 292 * This function releases the memory for all <*, id> shadow variable
281 * instances, callers should stop referencing 293 * instances, callers should stop referencing them accordingly.
282 */ 294 */
283 void klp_shadow_free_all(unsigned long id, klp 295 void klp_shadow_free_all(unsigned long id, klp_shadow_dtor_t dtor)
284 { 296 {
285 struct klp_shadow *shadow; 297 struct klp_shadow *shadow;
286 unsigned long flags; 298 unsigned long flags;
287 int i; 299 int i;
288 300
289 spin_lock_irqsave(&klp_shadow_lock, fl 301 spin_lock_irqsave(&klp_shadow_lock, flags);
290 302
291 /* Delete all <_, id> from hash */ !! 303 /* Delete all <*, id> from hash */
292 hash_for_each(klp_shadow_hash, i, shad 304 hash_for_each(klp_shadow_hash, i, shadow, node) {
293 if (klp_shadow_match(shadow, s 305 if (klp_shadow_match(shadow, shadow->obj, id))
294 klp_shadow_free_struct 306 klp_shadow_free_struct(shadow, dtor);
295 } 307 }
296 308
297 spin_unlock_irqrestore(&klp_shadow_loc 309 spin_unlock_irqrestore(&klp_shadow_lock, flags);
298 } 310 }
299 EXPORT_SYMBOL_GPL(klp_shadow_free_all); 311 EXPORT_SYMBOL_GPL(klp_shadow_free_all);
300 312
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.