1 // SPDX-License-Identifier: GPL-2.0-or-later <<
2 /* 1 /*
3 * shadow.c - Shadow Variables 2 * shadow.c - Shadow Variables
4 * 3 *
5 * Copyright (C) 2014 Josh Poimboeuf <jpoimboe 4 * Copyright (C) 2014 Josh Poimboeuf <jpoimboe@redhat.com>
6 * Copyright (C) 2014 Seth Jennings <sjenning@ 5 * Copyright (C) 2014 Seth Jennings <sjenning@redhat.com>
7 * Copyright (C) 2017 Joe Lawrence <joe.lawren 6 * Copyright (C) 2017 Joe Lawrence <joe.lawrence@redhat.com>
>> 7 *
>> 8 * This program is free software; you can redistribute it and/or
>> 9 * modify it under the terms of the GNU General Public License
>> 10 * as published by the Free Software Foundation; either version 2
>> 11 * of the License, or (at your option) any later version.
>> 12 *
>> 13 * This program is distributed in the hope that it will be useful,
>> 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
>> 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
>> 16 * GNU General Public License for more details.
>> 17 *
>> 18 * You should have received a copy of the GNU General Public License
>> 19 * along with this program; if not, see <http://www.gnu.org/licenses/>.
8 */ 20 */
9 21
10 /** 22 /**
11 * DOC: Shadow variable API concurrency notes: 23 * DOC: Shadow variable API concurrency notes:
12 * 24 *
13 * The shadow variable API provides a simple r 25 * The shadow variable API provides a simple relationship between an
14 * <obj, id> pair and a pointer value. It is 26 * <obj, id> pair and a pointer value. It is the responsibility of the
15 * caller to provide any mutual exclusion requ 27 * caller to provide any mutual exclusion required of the shadow data.
16 * 28 *
17 * Once a shadow variable is attached to its p 29 * Once a shadow variable is attached to its parent object via the
18 * klp_shadow_*alloc() API calls, it is consid 30 * klp_shadow_*alloc() API calls, it is considered live: any subsequent
19 * call to klp_shadow_get() may then return th 31 * call to klp_shadow_get() may then return the shadow variable's data
20 * pointer. Callers of klp_shadow_*alloc() sh 32 * pointer. Callers of klp_shadow_*alloc() should prepare shadow data
21 * accordingly. 33 * accordingly.
22 * 34 *
23 * The klp_shadow_*alloc() API calls may alloc 35 * The klp_shadow_*alloc() API calls may allocate memory for new shadow
24 * variable structures. Their implementation 36 * variable structures. Their implementation does not call kmalloc
25 * inside any spinlocks, but API callers shoul 37 * inside any spinlocks, but API callers should pass GFP flags according
26 * to their specific needs. 38 * to their specific needs.
27 * 39 *
28 * The klp_shadow_hash is an RCU-enabled hasht 40 * The klp_shadow_hash is an RCU-enabled hashtable and is safe against
29 * concurrent klp_shadow_free() and klp_shadow 41 * concurrent klp_shadow_free() and klp_shadow_get() operations.
30 */ 42 */
31 43
32 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 44 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
33 45
34 #include <linux/hashtable.h> 46 #include <linux/hashtable.h>
35 #include <linux/slab.h> 47 #include <linux/slab.h>
36 #include <linux/livepatch.h> 48 #include <linux/livepatch.h>
37 49
38 static DEFINE_HASHTABLE(klp_shadow_hash, 12); 50 static DEFINE_HASHTABLE(klp_shadow_hash, 12);
39 51
40 /* 52 /*
41 * klp_shadow_lock provides exclusive access t 53 * klp_shadow_lock provides exclusive access to the klp_shadow_hash and
42 * the shadow variables it references. 54 * the shadow variables it references.
43 */ 55 */
44 static DEFINE_SPINLOCK(klp_shadow_lock); 56 static DEFINE_SPINLOCK(klp_shadow_lock);
45 57
46 /** 58 /**
47 * struct klp_shadow - shadow variable structu 59 * struct klp_shadow - shadow variable structure
48 * @node: klp_shadow_hash hash table nod 60 * @node: klp_shadow_hash hash table node
49 * @rcu_head: RCU is used to safely free thi 61 * @rcu_head: RCU is used to safely free this structure
50 * @obj: pointer to parent object 62 * @obj: pointer to parent object
51 * @id: data identifier 63 * @id: data identifier
52 * @data: data area 64 * @data: data area
53 */ 65 */
54 struct klp_shadow { 66 struct klp_shadow {
55 struct hlist_node node; 67 struct hlist_node node;
56 struct rcu_head rcu_head; 68 struct rcu_head rcu_head;
57 void *obj; 69 void *obj;
58 unsigned long id; 70 unsigned long id;
59 char data[]; 71 char data[];
60 }; 72 };
61 73
62 /** 74 /**
63 * klp_shadow_match() - verify a shadow variab 75 * klp_shadow_match() - verify a shadow variable matches given <obj, id>
64 * @shadow: shadow variable to match 76 * @shadow: shadow variable to match
65 * @obj: pointer to parent object 77 * @obj: pointer to parent object
66 * @id: data identifier 78 * @id: data identifier
67 * 79 *
68 * Return: true if the shadow variable matches 80 * Return: true if the shadow variable matches.
69 */ 81 */
70 static inline bool klp_shadow_match(struct klp 82 static inline bool klp_shadow_match(struct klp_shadow *shadow, void *obj,
71 unsigned long 83 unsigned long id)
72 { 84 {
73 return shadow->obj == obj && shadow->i 85 return shadow->obj == obj && shadow->id == id;
74 } 86 }
75 87
76 /** 88 /**
77 * klp_shadow_get() - retrieve a shadow variab 89 * klp_shadow_get() - retrieve a shadow variable data pointer
78 * @obj: pointer to parent object 90 * @obj: pointer to parent object
79 * @id: data identifier 91 * @id: data identifier
80 * 92 *
81 * Return: the shadow variable data element, N 93 * Return: the shadow variable data element, NULL on failure.
82 */ 94 */
83 void *klp_shadow_get(void *obj, unsigned long 95 void *klp_shadow_get(void *obj, unsigned long id)
84 { 96 {
85 struct klp_shadow *shadow; 97 struct klp_shadow *shadow;
86 98
87 rcu_read_lock(); 99 rcu_read_lock();
88 100
89 hash_for_each_possible_rcu(klp_shadow_ 101 hash_for_each_possible_rcu(klp_shadow_hash, shadow, node,
90 (unsigned l 102 (unsigned long)obj) {
91 103
92 if (klp_shadow_match(shadow, o 104 if (klp_shadow_match(shadow, obj, id)) {
93 rcu_read_unlock(); 105 rcu_read_unlock();
94 return shadow->data; 106 return shadow->data;
95 } 107 }
96 } 108 }
97 109
98 rcu_read_unlock(); 110 rcu_read_unlock();
99 111
100 return NULL; 112 return NULL;
101 } 113 }
102 EXPORT_SYMBOL_GPL(klp_shadow_get); 114 EXPORT_SYMBOL_GPL(klp_shadow_get);
103 115
104 static void *__klp_shadow_get_or_alloc(void *o !! 116 static void *__klp_shadow_get_or_alloc(void *obj, unsigned long id, void *data,
105 size_t !! 117 size_t size, gfp_t gfp_flags, bool warn_on_exist)
106 klp_sha <<
107 bool wa <<
108 { 118 {
109 struct klp_shadow *new_shadow; 119 struct klp_shadow *new_shadow;
110 void *shadow_data; 120 void *shadow_data;
111 unsigned long flags; 121 unsigned long flags;
112 122
113 /* Check if the shadow variable alread 123 /* Check if the shadow variable already exists */
114 shadow_data = klp_shadow_get(obj, id); 124 shadow_data = klp_shadow_get(obj, id);
115 if (shadow_data) 125 if (shadow_data)
116 goto exists; 126 goto exists;
117 127
118 /* !! 128 /* Allocate a new shadow variable for use inside the lock below */
119 * Allocate a new shadow variable. Fi <<
120 * More complex setting can be done by <<
121 * called only when the buffer is real <<
122 */ <<
123 new_shadow = kzalloc(size + sizeof(*ne 129 new_shadow = kzalloc(size + sizeof(*new_shadow), gfp_flags);
124 if (!new_shadow) 130 if (!new_shadow)
125 return NULL; 131 return NULL;
126 132
>> 133 new_shadow->obj = obj;
>> 134 new_shadow->id = id;
>> 135
>> 136 /* Initialize the shadow variable if data provided */
>> 137 if (data)
>> 138 memcpy(new_shadow->data, data, size);
>> 139
127 /* Look for <obj, id> again under the 140 /* Look for <obj, id> again under the lock */
128 spin_lock_irqsave(&klp_shadow_lock, fl 141 spin_lock_irqsave(&klp_shadow_lock, flags);
129 shadow_data = klp_shadow_get(obj, id); 142 shadow_data = klp_shadow_get(obj, id);
130 if (unlikely(shadow_data)) { 143 if (unlikely(shadow_data)) {
131 /* 144 /*
132 * Shadow variable was found, 145 * Shadow variable was found, throw away speculative
133 * allocation. 146 * allocation.
134 */ 147 */
135 spin_unlock_irqrestore(&klp_sh 148 spin_unlock_irqrestore(&klp_shadow_lock, flags);
136 kfree(new_shadow); 149 kfree(new_shadow);
137 goto exists; 150 goto exists;
138 } 151 }
139 152
140 new_shadow->obj = obj; <<
141 new_shadow->id = id; <<
142 <<
143 if (ctor) { <<
144 int err; <<
145 <<
146 err = ctor(obj, new_shadow->da <<
147 if (err) { <<
148 spin_unlock_irqrestore <<
149 kfree(new_shadow); <<
150 pr_err("Failed to cons <<
151 obj, id, err); <<
152 return NULL; <<
153 } <<
154 } <<
155 <<
156 /* No <obj, id> found, so attach the n 153 /* No <obj, id> found, so attach the newly allocated one */
157 hash_add_rcu(klp_shadow_hash, &new_sha 154 hash_add_rcu(klp_shadow_hash, &new_shadow->node,
158 (unsigned long)new_shadow 155 (unsigned long)new_shadow->obj);
159 spin_unlock_irqrestore(&klp_shadow_loc 156 spin_unlock_irqrestore(&klp_shadow_lock, flags);
160 157
161 return new_shadow->data; 158 return new_shadow->data;
162 159
163 exists: 160 exists:
164 if (warn_on_exist) { 161 if (warn_on_exist) {
165 WARN(1, "Duplicate shadow vari 162 WARN(1, "Duplicate shadow variable <%p, %lx>\n", obj, id);
166 return NULL; 163 return NULL;
167 } 164 }
168 165
169 return shadow_data; 166 return shadow_data;
170 } 167 }
171 168
172 /** 169 /**
173 * klp_shadow_alloc() - allocate and add a new 170 * klp_shadow_alloc() - allocate and add a new shadow variable
174 * @obj: pointer to parent object 171 * @obj: pointer to parent object
175 * @id: data identifier 172 * @id: data identifier
>> 173 * @data: pointer to data to attach to parent
176 * @size: size of attached data 174 * @size: size of attached data
177 * @gfp_flags: GFP mask for allocation 175 * @gfp_flags: GFP mask for allocation
178 * @ctor: custom constructor to initiali <<
179 * @ctor_data: pointer to any data needed by <<
180 * 176 *
181 * Allocates @size bytes for new shadow variab !! 177 * Allocates @size bytes for new shadow variable data using @gfp_flags
182 * The data are zeroed by default. They are f !! 178 * and copies @size bytes from @data into the new shadow variable's own
183 * function if it is not NULL. The new shadow !! 179 * data space. If @data is NULL, @size bytes are still allocated, but
184 * to the global hashtable. !! 180 * no copy is performed. The new shadow variable is then added to the
185 * !! 181 * global hashtable.
186 * If an existing <obj, id> shadow variable ca !! 182 *
187 * issue a WARN, exit early and return NULL. !! 183 * If an existing <obj, id> shadow variable can be found, this routine
188 * !! 184 * will issue a WARN, exit early and return NULL.
189 * This function guarantees that the construct <<
190 * the variable did not exist before. The cos <<
191 * in atomic context under a spin lock. <<
192 * 185 *
193 * Return: the shadow variable data element, N 186 * Return: the shadow variable data element, NULL on duplicate or
194 * failure. 187 * failure.
195 */ 188 */
196 void *klp_shadow_alloc(void *obj, unsigned lon !! 189 void *klp_shadow_alloc(void *obj, unsigned long id, void *data,
197 size_t size, gfp_t gfp_ !! 190 size_t size, gfp_t gfp_flags)
198 klp_shadow_ctor_t ctor, <<
199 { 191 {
200 return __klp_shadow_get_or_alloc(obj, !! 192 return __klp_shadow_get_or_alloc(obj, id, data, size, gfp_flags, true);
201 ctor, <<
202 } 193 }
203 EXPORT_SYMBOL_GPL(klp_shadow_alloc); 194 EXPORT_SYMBOL_GPL(klp_shadow_alloc);
204 195
205 /** 196 /**
206 * klp_shadow_get_or_alloc() - get existing or 197 * klp_shadow_get_or_alloc() - get existing or allocate a new shadow variable
207 * @obj: pointer to parent object 198 * @obj: pointer to parent object
208 * @id: data identifier 199 * @id: data identifier
>> 200 * @data: pointer to data to attach to parent
209 * @size: size of attached data 201 * @size: size of attached data
210 * @gfp_flags: GFP mask for allocation 202 * @gfp_flags: GFP mask for allocation
211 * @ctor: custom constructor to initiali <<
212 * @ctor_data: pointer to any data needed by <<
213 * 203 *
214 * Returns a pointer to existing shadow data i 204 * Returns a pointer to existing shadow data if an <obj, id> shadow
215 * variable is already present. Otherwise, it 205 * variable is already present. Otherwise, it creates a new shadow
216 * variable like klp_shadow_alloc(). 206 * variable like klp_shadow_alloc().
217 * 207 *
218 * This function guarantees that only one shad !! 208 * This function guarantees that only one shadow variable exists with
219 * @id for the given @obj. It also guarantees !! 209 * the given @id for the given @obj. It also guarantees that the shadow
220 * will be called only when the variable did n !! 210 * variable will be initialized by the given @data only when it did not
221 * that @ctor is called in atomic context unde !! 211 * exist before.
222 * 212 *
223 * Return: the shadow variable data element, N 213 * Return: the shadow variable data element, NULL on failure.
224 */ 214 */
225 void *klp_shadow_get_or_alloc(void *obj, unsig !! 215 void *klp_shadow_get_or_alloc(void *obj, unsigned long id, void *data,
226 size_t size, gfp !! 216 size_t size, gfp_t gfp_flags)
227 klp_shadow_ctor_ <<
228 { 217 {
229 return __klp_shadow_get_or_alloc(obj, !! 218 return __klp_shadow_get_or_alloc(obj, id, data, size, gfp_flags, false);
230 ctor, <<
231 } 219 }
232 EXPORT_SYMBOL_GPL(klp_shadow_get_or_alloc); 220 EXPORT_SYMBOL_GPL(klp_shadow_get_or_alloc);
233 221
234 static void klp_shadow_free_struct(struct klp_ <<
235 klp_shadow_ <<
236 { <<
237 hash_del_rcu(&shadow->node); <<
238 if (dtor) <<
239 dtor(shadow->obj, shadow->data <<
240 kfree_rcu(shadow, rcu_head); <<
241 } <<
242 <<
243 /** 222 /**
244 * klp_shadow_free() - detach and free a <obj, 223 * klp_shadow_free() - detach and free a <obj, id> shadow variable
245 * @obj: pointer to parent object 224 * @obj: pointer to parent object
246 * @id: data identifier 225 * @id: data identifier
247 * @dtor: custom callback that can be us <<
248 * and/or free data that the shad <<
249 * 226 *
250 * This function releases the memory for this 227 * This function releases the memory for this <obj, id> shadow variable
251 * instance, callers should stop referencing i 228 * instance, callers should stop referencing it accordingly.
252 */ 229 */
253 void klp_shadow_free(void *obj, unsigned long !! 230 void klp_shadow_free(void *obj, unsigned long id)
254 { 231 {
255 struct klp_shadow *shadow; 232 struct klp_shadow *shadow;
256 unsigned long flags; 233 unsigned long flags;
257 234
258 spin_lock_irqsave(&klp_shadow_lock, fl 235 spin_lock_irqsave(&klp_shadow_lock, flags);
259 236
260 /* Delete <obj, id> from hash */ 237 /* Delete <obj, id> from hash */
261 hash_for_each_possible(klp_shadow_hash 238 hash_for_each_possible(klp_shadow_hash, shadow, node,
262 (unsigned long) 239 (unsigned long)obj) {
263 240
264 if (klp_shadow_match(shadow, o 241 if (klp_shadow_match(shadow, obj, id)) {
265 klp_shadow_free_struct !! 242 hash_del_rcu(&shadow->node);
>> 243 kfree_rcu(shadow, rcu_head);
266 break; 244 break;
267 } 245 }
268 } 246 }
269 247
270 spin_unlock_irqrestore(&klp_shadow_loc 248 spin_unlock_irqrestore(&klp_shadow_lock, flags);
271 } 249 }
272 EXPORT_SYMBOL_GPL(klp_shadow_free); 250 EXPORT_SYMBOL_GPL(klp_shadow_free);
273 251
274 /** 252 /**
275 * klp_shadow_free_all() - detach and free all !! 253 * klp_shadow_free_all() - detach and free all <*, id> shadow variables
276 * @id: data identifier 254 * @id: data identifier
277 * @dtor: custom callback that can be us <<
278 * and/or free data that the shad <<
279 * 255 *
280 * This function releases the memory for all < !! 256 * This function releases the memory for all <*, id> shadow variable
281 * instances, callers should stop referencing 257 * instances, callers should stop referencing them accordingly.
282 */ 258 */
283 void klp_shadow_free_all(unsigned long id, klp !! 259 void klp_shadow_free_all(unsigned long id)
284 { 260 {
285 struct klp_shadow *shadow; 261 struct klp_shadow *shadow;
286 unsigned long flags; 262 unsigned long flags;
287 int i; 263 int i;
288 264
289 spin_lock_irqsave(&klp_shadow_lock, fl 265 spin_lock_irqsave(&klp_shadow_lock, flags);
290 266
291 /* Delete all <_, id> from hash */ !! 267 /* Delete all <*, id> from hash */
292 hash_for_each(klp_shadow_hash, i, shad 268 hash_for_each(klp_shadow_hash, i, shadow, node) {
293 if (klp_shadow_match(shadow, s !! 269 if (klp_shadow_match(shadow, shadow->obj, id)) {
294 klp_shadow_free_struct !! 270 hash_del_rcu(&shadow->node);
>> 271 kfree_rcu(shadow, rcu_head);
>> 272 }
295 } 273 }
296 274
297 spin_unlock_irqrestore(&klp_shadow_loc 275 spin_unlock_irqrestore(&klp_shadow_lock, flags);
298 } 276 }
299 EXPORT_SYMBOL_GPL(klp_shadow_free_all); 277 EXPORT_SYMBOL_GPL(klp_shadow_free_all);
300 278
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.