1 // SPDX-License-Identifier: GPL-2.0-or-later 1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /* 2 /*
3 * Cryptographic API for algorithms (i.e., low 3 * Cryptographic API for algorithms (i.e., low-level API).
4 * 4 *
5 * Copyright (c) 2006 Herbert Xu <herbert@gond 5 * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
6 */ 6 */
7 7
8 #include <crypto/algapi.h> 8 #include <crypto/algapi.h>
9 #include <crypto/internal/simd.h> 9 #include <crypto/internal/simd.h>
10 #include <linux/err.h> 10 #include <linux/err.h>
11 #include <linux/errno.h> 11 #include <linux/errno.h>
12 #include <linux/fips.h> 12 #include <linux/fips.h>
13 #include <linux/init.h> 13 #include <linux/init.h>
14 #include <linux/kernel.h> 14 #include <linux/kernel.h>
15 #include <linux/list.h> 15 #include <linux/list.h>
16 #include <linux/module.h> 16 #include <linux/module.h>
17 #include <linux/rtnetlink.h> 17 #include <linux/rtnetlink.h>
18 #include <linux/slab.h> 18 #include <linux/slab.h>
19 #include <linux/string.h> 19 #include <linux/string.h>
20 #include <linux/workqueue.h> <<
21 20
22 #include "internal.h" 21 #include "internal.h"
23 22
24 static LIST_HEAD(crypto_template_list); 23 static LIST_HEAD(crypto_template_list);
25 24
26 #ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS 25 #ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
27 DEFINE_PER_CPU(bool, crypto_simd_disabled_for_ 26 DEFINE_PER_CPU(bool, crypto_simd_disabled_for_test);
28 EXPORT_PER_CPU_SYMBOL_GPL(crypto_simd_disabled 27 EXPORT_PER_CPU_SYMBOL_GPL(crypto_simd_disabled_for_test);
29 #endif 28 #endif
30 29
31 static inline void crypto_check_module_sig(str 30 static inline void crypto_check_module_sig(struct module *mod)
32 { 31 {
33 if (fips_enabled && mod && !module_sig 32 if (fips_enabled && mod && !module_sig_ok(mod))
34 panic("Module %s signature ver 33 panic("Module %s signature verification failed in FIPS mode\n",
35 module_name(mod)); 34 module_name(mod));
36 } 35 }
37 36
38 static int crypto_check_alg(struct crypto_alg 37 static int crypto_check_alg(struct crypto_alg *alg)
39 { 38 {
40 crypto_check_module_sig(alg->cra_modul 39 crypto_check_module_sig(alg->cra_module);
41 40
42 if (!alg->cra_name[0] || !alg->cra_dri 41 if (!alg->cra_name[0] || !alg->cra_driver_name[0])
43 return -EINVAL; 42 return -EINVAL;
44 43
45 if (alg->cra_alignmask & (alg->cra_ali 44 if (alg->cra_alignmask & (alg->cra_alignmask + 1))
46 return -EINVAL; 45 return -EINVAL;
47 46
48 /* General maximums for all algs. */ 47 /* General maximums for all algs. */
49 if (alg->cra_alignmask > MAX_ALGAPI_AL 48 if (alg->cra_alignmask > MAX_ALGAPI_ALIGNMASK)
50 return -EINVAL; 49 return -EINVAL;
51 50
52 if (alg->cra_blocksize > MAX_ALGAPI_BL 51 if (alg->cra_blocksize > MAX_ALGAPI_BLOCKSIZE)
53 return -EINVAL; 52 return -EINVAL;
54 53
55 /* Lower maximums for specific alg typ 54 /* Lower maximums for specific alg types. */
56 if (!alg->cra_type && (alg->cra_flags 55 if (!alg->cra_type && (alg->cra_flags & CRYPTO_ALG_TYPE_MASK) ==
57 CRYPTO_ALG_TYPE 56 CRYPTO_ALG_TYPE_CIPHER) {
58 if (alg->cra_alignmask > MAX_C 57 if (alg->cra_alignmask > MAX_CIPHER_ALIGNMASK)
59 return -EINVAL; 58 return -EINVAL;
60 59
61 if (alg->cra_blocksize > MAX_C 60 if (alg->cra_blocksize > MAX_CIPHER_BLOCKSIZE)
62 return -EINVAL; 61 return -EINVAL;
63 } 62 }
64 63
65 if (alg->cra_priority < 0) 64 if (alg->cra_priority < 0)
66 return -EINVAL; 65 return -EINVAL;
67 66
68 refcount_set(&alg->cra_refcnt, 1); 67 refcount_set(&alg->cra_refcnt, 1);
69 68
70 return 0; 69 return 0;
71 } 70 }
72 71
73 static void crypto_free_instance(struct crypto 72 static void crypto_free_instance(struct crypto_instance *inst)
74 { 73 {
75 inst->alg.cra_type->free(inst); 74 inst->alg.cra_type->free(inst);
76 } 75 }
77 76
78 static void crypto_destroy_instance_workfn(str !! 77 static void crypto_destroy_instance(struct crypto_alg *alg)
79 { 78 {
80 struct crypto_instance *inst = contain !! 79 struct crypto_instance *inst = (void *)alg;
81 <<
82 struct crypto_template *tmpl = inst->t 80 struct crypto_template *tmpl = inst->tmpl;
83 81
84 crypto_free_instance(inst); 82 crypto_free_instance(inst);
85 crypto_tmpl_put(tmpl); 83 crypto_tmpl_put(tmpl);
86 } 84 }
87 85
88 static void crypto_destroy_instance(struct cry <<
89 { <<
90 struct crypto_instance *inst = contain <<
91 <<
92 <<
93 <<
94 INIT_WORK(&inst->free_work, crypto_des <<
95 schedule_work(&inst->free_work); <<
96 } <<
97 <<
98 /* 86 /*
99 * This function adds a spawn to the list seco 87 * This function adds a spawn to the list secondary_spawns which
100 * will be used at the end of crypto_remove_sp 88 * will be used at the end of crypto_remove_spawns to unregister
101 * instances, unless the spawn happens to be o 89 * instances, unless the spawn happens to be one that is depended
102 * on by the new algorithm (nalg in crypto_rem 90 * on by the new algorithm (nalg in crypto_remove_spawns).
103 * 91 *
104 * This function is also responsible for resur 92 * This function is also responsible for resurrecting any algorithms
105 * in the dependency chain of nalg by unsettin 93 * in the dependency chain of nalg by unsetting n->dead.
106 */ 94 */
107 static struct list_head *crypto_more_spawns(st 95 static struct list_head *crypto_more_spawns(struct crypto_alg *alg,
108 st 96 struct list_head *stack,
109 st 97 struct list_head *top,
110 st 98 struct list_head *secondary_spawns)
111 { 99 {
112 struct crypto_spawn *spawn, *n; 100 struct crypto_spawn *spawn, *n;
113 101
114 spawn = list_first_entry_or_null(stack 102 spawn = list_first_entry_or_null(stack, struct crypto_spawn, list);
115 if (!spawn) 103 if (!spawn)
116 return NULL; 104 return NULL;
117 105
118 n = list_prev_entry(spawn, list); 106 n = list_prev_entry(spawn, list);
119 list_move(&spawn->list, secondary_spaw 107 list_move(&spawn->list, secondary_spawns);
120 108
121 if (list_is_last(&n->list, stack)) 109 if (list_is_last(&n->list, stack))
122 return top; 110 return top;
123 111
124 n = list_next_entry(n, list); 112 n = list_next_entry(n, list);
125 if (!spawn->dead) 113 if (!spawn->dead)
126 n->dead = false; 114 n->dead = false;
127 115
128 return &n->inst->alg.cra_users; 116 return &n->inst->alg.cra_users;
129 } 117 }
130 118
131 static void crypto_remove_instance(struct cryp 119 static void crypto_remove_instance(struct crypto_instance *inst,
132 struct list 120 struct list_head *list)
133 { 121 {
134 struct crypto_template *tmpl = inst->t 122 struct crypto_template *tmpl = inst->tmpl;
135 123
136 if (crypto_is_dead(&inst->alg)) 124 if (crypto_is_dead(&inst->alg))
137 return; 125 return;
138 126
139 inst->alg.cra_flags |= CRYPTO_ALG_DEAD 127 inst->alg.cra_flags |= CRYPTO_ALG_DEAD;
140 128
141 if (!tmpl || !crypto_tmpl_get(tmpl)) 129 if (!tmpl || !crypto_tmpl_get(tmpl))
142 return; 130 return;
143 131
144 list_move(&inst->alg.cra_list, list); 132 list_move(&inst->alg.cra_list, list);
145 hlist_del(&inst->list); 133 hlist_del(&inst->list);
146 inst->alg.cra_destroy = crypto_destroy 134 inst->alg.cra_destroy = crypto_destroy_instance;
147 135
148 BUG_ON(!list_empty(&inst->alg.cra_user 136 BUG_ON(!list_empty(&inst->alg.cra_users));
149 } 137 }
150 138
151 /* 139 /*
152 * Given an algorithm alg, remove all algorith 140 * Given an algorithm alg, remove all algorithms that depend on it
153 * through spawns. If nalg is not null, then 141 * through spawns. If nalg is not null, then exempt any algorithms
154 * that is depended on by nalg. This is usefu 142 * that is depended on by nalg. This is useful when nalg itself
155 * depends on alg. 143 * depends on alg.
156 */ 144 */
157 void crypto_remove_spawns(struct crypto_alg *a 145 void crypto_remove_spawns(struct crypto_alg *alg, struct list_head *list,
158 struct crypto_alg *n 146 struct crypto_alg *nalg)
159 { 147 {
160 u32 new_type = (nalg ?: alg)->cra_flag 148 u32 new_type = (nalg ?: alg)->cra_flags;
161 struct crypto_spawn *spawn, *n; 149 struct crypto_spawn *spawn, *n;
162 LIST_HEAD(secondary_spawns); 150 LIST_HEAD(secondary_spawns);
163 struct list_head *spawns; 151 struct list_head *spawns;
164 LIST_HEAD(stack); 152 LIST_HEAD(stack);
165 LIST_HEAD(top); 153 LIST_HEAD(top);
166 154
167 spawns = &alg->cra_users; 155 spawns = &alg->cra_users;
168 list_for_each_entry_safe(spawn, n, spa 156 list_for_each_entry_safe(spawn, n, spawns, list) {
169 if ((spawn->alg->cra_flags ^ n 157 if ((spawn->alg->cra_flags ^ new_type) & spawn->mask)
170 continue; 158 continue;
171 159
172 list_move(&spawn->list, &top); 160 list_move(&spawn->list, &top);
173 } 161 }
174 162
175 /* 163 /*
176 * Perform a depth-first walk starting 164 * Perform a depth-first walk starting from alg through
177 * the cra_users tree. The list stack 165 * the cra_users tree. The list stack records the path
178 * from alg to the current spawn. 166 * from alg to the current spawn.
179 */ 167 */
180 spawns = ⊤ 168 spawns = ⊤
181 do { 169 do {
182 while (!list_empty(spawns)) { 170 while (!list_empty(spawns)) {
183 struct crypto_instance 171 struct crypto_instance *inst;
184 172
185 spawn = list_first_ent 173 spawn = list_first_entry(spawns, struct crypto_spawn,
186 174 list);
187 inst = spawn->inst; 175 inst = spawn->inst;
188 176
189 list_move(&spawn->list 177 list_move(&spawn->list, &stack);
190 spawn->dead = !spawn-> 178 spawn->dead = !spawn->registered || &inst->alg != nalg;
191 179
192 if (!spawn->registered 180 if (!spawn->registered)
193 break; 181 break;
194 182
195 BUG_ON(&inst->alg == a 183 BUG_ON(&inst->alg == alg);
196 184
197 if (&inst->alg == nalg 185 if (&inst->alg == nalg)
198 break; 186 break;
199 187
200 spawns = &inst->alg.cr 188 spawns = &inst->alg.cra_users;
201 189
202 /* 190 /*
203 * Even if spawn->regi 191 * Even if spawn->registered is true, the
204 * instance itself may 192 * instance itself may still be unregistered.
205 * This is because it 193 * This is because it may have failed during
206 * registration. Ther 194 * registration. Therefore we still need to
207 * make the following 195 * make the following test.
208 * 196 *
209 * We may encounter an 197 * We may encounter an unregistered instance here, since
210 * an instance's spawn 198 * an instance's spawns are set up prior to the instance
211 * being registered. 199 * being registered. An unregistered instance will have
212 * NULL ->cra_users.ne 200 * NULL ->cra_users.next, since ->cra_users isn't
213 * properly initialize 201 * properly initialized until registration. But an
214 * unregistered instan 202 * unregistered instance cannot have any users, so treat
215 * it the same as ->cr 203 * it the same as ->cra_users being empty.
216 */ 204 */
217 if (spawns->next == NU 205 if (spawns->next == NULL)
218 break; 206 break;
219 } 207 }
220 } while ((spawns = crypto_more_spawns( 208 } while ((spawns = crypto_more_spawns(alg, &stack, &top,
221 209 &secondary_spawns)));
222 210
223 /* 211 /*
224 * Remove all instances that are marke 212 * Remove all instances that are marked as dead. Also
225 * complete the resurrection of the ot 213 * complete the resurrection of the others by moving them
226 * back to the cra_users list. 214 * back to the cra_users list.
227 */ 215 */
228 list_for_each_entry_safe(spawn, n, &se 216 list_for_each_entry_safe(spawn, n, &secondary_spawns, list) {
229 if (!spawn->dead) 217 if (!spawn->dead)
230 list_move(&spawn->list 218 list_move(&spawn->list, &spawn->alg->cra_users);
231 else if (spawn->registered) 219 else if (spawn->registered)
232 crypto_remove_instance 220 crypto_remove_instance(spawn->inst, list);
233 } 221 }
234 } 222 }
235 EXPORT_SYMBOL_GPL(crypto_remove_spawns); 223 EXPORT_SYMBOL_GPL(crypto_remove_spawns);
236 224
237 static void crypto_alg_finish_registration(str <<
238 boo <<
239 str <<
240 { <<
241 struct crypto_alg *q; <<
242 <<
243 list_for_each_entry(q, &crypto_alg_lis <<
244 if (q == alg) <<
245 continue; <<
246 <<
247 if (crypto_is_moribund(q)) <<
248 continue; <<
249 <<
250 if (crypto_is_larval(q)) { <<
251 struct crypto_larval * <<
252 <<
253 /* <<
254 * Check to see if eit <<
255 * specific name can s <<
256 * by the larval entry <<
257 */ <<
258 if (strcmp(alg->cra_na <<
259 strcmp(alg->cra_dr <<
260 continue; <<
261 <<
262 if (larval->adult) <<
263 continue; <<
264 if ((q->cra_flags ^ al <<
265 continue; <<
266 <<
267 if (fulfill_requests & <<
268 larval->adult <<
269 else <<
270 larval->adult <<
271 <<
272 continue; <<
273 } <<
274 <<
275 if (strcmp(alg->cra_name, q->c <<
276 continue; <<
277 <<
278 if (strcmp(alg->cra_driver_nam <<
279 q->cra_priority > alg->cra <<
280 continue; <<
281 <<
282 crypto_remove_spawns(q, algs_t <<
283 } <<
284 <<
285 crypto_notify(CRYPTO_MSG_ALG_LOADED, a <<
286 } <<
287 <<
288 static struct crypto_larval *crypto_alloc_test 225 static struct crypto_larval *crypto_alloc_test_larval(struct crypto_alg *alg)
289 { 226 {
290 struct crypto_larval *larval; 227 struct crypto_larval *larval;
291 228
292 if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER) !! 229 if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER))
293 IS_ENABLED(CONFIG_CRYPTO_MANAGER_D !! 230 return NULL;
294 (alg->cra_flags & CRYPTO_ALG_INTER <<
295 return NULL; /* No self-test n <<
296 231
297 larval = crypto_larval_alloc(alg->cra_ 232 larval = crypto_larval_alloc(alg->cra_name,
298 alg->cra_ 233 alg->cra_flags | CRYPTO_ALG_TESTED, 0);
299 if (IS_ERR(larval)) 234 if (IS_ERR(larval))
300 return larval; 235 return larval;
301 236
302 larval->adult = crypto_mod_get(alg); 237 larval->adult = crypto_mod_get(alg);
303 if (!larval->adult) { 238 if (!larval->adult) {
304 kfree(larval); 239 kfree(larval);
305 return ERR_PTR(-ENOENT); 240 return ERR_PTR(-ENOENT);
306 } 241 }
307 242
308 refcount_set(&larval->alg.cra_refcnt, 243 refcount_set(&larval->alg.cra_refcnt, 1);
309 memcpy(larval->alg.cra_driver_name, al 244 memcpy(larval->alg.cra_driver_name, alg->cra_driver_name,
310 CRYPTO_MAX_ALG_NAME); 245 CRYPTO_MAX_ALG_NAME);
311 larval->alg.cra_priority = alg->cra_pr 246 larval->alg.cra_priority = alg->cra_priority;
312 247
313 return larval; 248 return larval;
314 } 249 }
315 250
316 static struct crypto_larval * !! 251 static struct crypto_larval *__crypto_register_alg(struct crypto_alg *alg)
317 __crypto_register_alg(struct crypto_alg *alg, <<
318 { 252 {
319 struct crypto_alg *q; 253 struct crypto_alg *q;
320 struct crypto_larval *larval; 254 struct crypto_larval *larval;
321 int ret = -EAGAIN; 255 int ret = -EAGAIN;
322 256
323 if (crypto_is_dead(alg)) 257 if (crypto_is_dead(alg))
324 goto err; 258 goto err;
325 259
326 INIT_LIST_HEAD(&alg->cra_users); 260 INIT_LIST_HEAD(&alg->cra_users);
327 261
>> 262 /* No cheating! */
>> 263 alg->cra_flags &= ~CRYPTO_ALG_TESTED;
>> 264
328 ret = -EEXIST; 265 ret = -EEXIST;
329 266
330 list_for_each_entry(q, &crypto_alg_lis 267 list_for_each_entry(q, &crypto_alg_list, cra_list) {
331 if (q == alg) 268 if (q == alg)
332 goto err; 269 goto err;
333 270
334 if (crypto_is_moribund(q)) 271 if (crypto_is_moribund(q))
335 continue; 272 continue;
336 273
337 if (crypto_is_larval(q)) { 274 if (crypto_is_larval(q)) {
338 if (!strcmp(alg->cra_d 275 if (!strcmp(alg->cra_driver_name, q->cra_driver_name))
339 goto err; 276 goto err;
340 continue; 277 continue;
341 } 278 }
342 279
343 if (!strcmp(q->cra_driver_name 280 if (!strcmp(q->cra_driver_name, alg->cra_name) ||
344 !strcmp(q->cra_driver_name <<
345 !strcmp(q->cra_name, alg-> 281 !strcmp(q->cra_name, alg->cra_driver_name))
346 goto err; 282 goto err;
347 } 283 }
348 284
349 larval = crypto_alloc_test_larval(alg) 285 larval = crypto_alloc_test_larval(alg);
350 if (IS_ERR(larval)) 286 if (IS_ERR(larval))
351 goto out; 287 goto out;
352 288
353 list_add(&alg->cra_list, &crypto_alg_l 289 list_add(&alg->cra_list, &crypto_alg_list);
354 290
355 if (larval) { !! 291 if (larval)
356 /* No cheating! */ <<
357 alg->cra_flags &= ~CRYPTO_ALG_ <<
358 <<
359 list_add(&larval->alg.cra_list 292 list_add(&larval->alg.cra_list, &crypto_alg_list);
360 } else { !! 293 else
361 alg->cra_flags |= CRYPTO_ALG_T 294 alg->cra_flags |= CRYPTO_ALG_TESTED;
362 crypto_alg_finish_registration !! 295
363 } !! 296 crypto_stats_init(alg);
364 297
365 out: 298 out:
366 return larval; 299 return larval;
367 300
368 err: 301 err:
369 larval = ERR_PTR(ret); 302 larval = ERR_PTR(ret);
370 goto out; 303 goto out;
371 } 304 }
372 305
373 void crypto_alg_tested(const char *name, int e 306 void crypto_alg_tested(const char *name, int err)
374 { 307 {
375 struct crypto_larval *test; 308 struct crypto_larval *test;
376 struct crypto_alg *alg; 309 struct crypto_alg *alg;
377 struct crypto_alg *q; 310 struct crypto_alg *q;
378 LIST_HEAD(list); 311 LIST_HEAD(list);
379 bool best; 312 bool best;
380 313
381 down_write(&crypto_alg_sem); 314 down_write(&crypto_alg_sem);
382 list_for_each_entry(q, &crypto_alg_lis 315 list_for_each_entry(q, &crypto_alg_list, cra_list) {
383 if (crypto_is_moribund(q) || ! 316 if (crypto_is_moribund(q) || !crypto_is_larval(q))
384 continue; 317 continue;
385 318
386 test = (struct crypto_larval * 319 test = (struct crypto_larval *)q;
387 320
388 if (!strcmp(q->cra_driver_name 321 if (!strcmp(q->cra_driver_name, name))
389 goto found; 322 goto found;
390 } 323 }
391 324
392 pr_err("alg: Unexpected test result fo 325 pr_err("alg: Unexpected test result for %s: %d\n", name, err);
393 goto unlock; 326 goto unlock;
394 327
395 found: 328 found:
396 q->cra_flags |= CRYPTO_ALG_DEAD; 329 q->cra_flags |= CRYPTO_ALG_DEAD;
397 alg = test->adult; 330 alg = test->adult;
398 331
399 if (list_empty(&alg->cra_list)) 332 if (list_empty(&alg->cra_list))
400 goto complete; 333 goto complete;
401 334
402 if (err == -ECANCELED) 335 if (err == -ECANCELED)
403 alg->cra_flags |= CRYPTO_ALG_F 336 alg->cra_flags |= CRYPTO_ALG_FIPS_INTERNAL;
404 else if (err) 337 else if (err)
405 goto complete; 338 goto complete;
406 else 339 else
407 alg->cra_flags &= ~CRYPTO_ALG_ 340 alg->cra_flags &= ~CRYPTO_ALG_FIPS_INTERNAL;
408 341
409 alg->cra_flags |= CRYPTO_ALG_TESTED; 342 alg->cra_flags |= CRYPTO_ALG_TESTED;
410 343
411 /* !! 344 /* Only satisfy larval waiters if we are the best. */
412 * If a higher-priority implementation <<
413 * currently being tested, then don't <<
414 */ <<
415 best = true; 345 best = true;
416 list_for_each_entry(q, &crypto_alg_lis 346 list_for_each_entry(q, &crypto_alg_list, cra_list) {
417 if (crypto_is_moribund(q) || ! 347 if (crypto_is_moribund(q) || !crypto_is_larval(q))
418 continue; 348 continue;
419 349
420 if (strcmp(alg->cra_name, q->c 350 if (strcmp(alg->cra_name, q->cra_name))
421 continue; 351 continue;
422 352
423 if (q->cra_priority > alg->cra 353 if (q->cra_priority > alg->cra_priority) {
424 best = false; 354 best = false;
425 break; 355 break;
426 } 356 }
427 } 357 }
428 358
429 crypto_alg_finish_registration(alg, be !! 359 list_for_each_entry(q, &crypto_alg_list, cra_list) {
>> 360 if (q == alg)
>> 361 continue;
>> 362
>> 363 if (crypto_is_moribund(q))
>> 364 continue;
>> 365
>> 366 if (crypto_is_larval(q)) {
>> 367 struct crypto_larval *larval = (void *)q;
>> 368
>> 369 /*
>> 370 * Check to see if either our generic name or
>> 371 * specific name can satisfy the name requested
>> 372 * by the larval entry q.
>> 373 */
>> 374 if (strcmp(alg->cra_name, q->cra_name) &&
>> 375 strcmp(alg->cra_driver_name, q->cra_name))
>> 376 continue;
>> 377
>> 378 if (larval->adult)
>> 379 continue;
>> 380 if ((q->cra_flags ^ alg->cra_flags) & larval->mask)
>> 381 continue;
>> 382
>> 383 if (best && crypto_mod_get(alg))
>> 384 larval->adult = alg;
>> 385 else
>> 386 larval->adult = ERR_PTR(-EAGAIN);
>> 387
>> 388 continue;
>> 389 }
>> 390
>> 391 if (strcmp(alg->cra_name, q->cra_name))
>> 392 continue;
>> 393
>> 394 if (strcmp(alg->cra_driver_name, q->cra_driver_name) &&
>> 395 q->cra_priority > alg->cra_priority)
>> 396 continue;
>> 397
>> 398 crypto_remove_spawns(q, &list, alg);
>> 399 }
430 400
431 complete: 401 complete:
432 complete_all(&test->completion); 402 complete_all(&test->completion);
433 403
434 unlock: 404 unlock:
435 up_write(&crypto_alg_sem); 405 up_write(&crypto_alg_sem);
436 406
437 crypto_remove_final(&list); 407 crypto_remove_final(&list);
438 } 408 }
439 EXPORT_SYMBOL_GPL(crypto_alg_tested); 409 EXPORT_SYMBOL_GPL(crypto_alg_tested);
440 410
441 void crypto_remove_final(struct list_head *lis 411 void crypto_remove_final(struct list_head *list)
442 { 412 {
443 struct crypto_alg *alg; 413 struct crypto_alg *alg;
444 struct crypto_alg *n; 414 struct crypto_alg *n;
445 415
446 list_for_each_entry_safe(alg, n, list, 416 list_for_each_entry_safe(alg, n, list, cra_list) {
447 list_del_init(&alg->cra_list); 417 list_del_init(&alg->cra_list);
448 crypto_alg_put(alg); 418 crypto_alg_put(alg);
449 } 419 }
450 } 420 }
451 EXPORT_SYMBOL_GPL(crypto_remove_final); 421 EXPORT_SYMBOL_GPL(crypto_remove_final);
452 422
453 int crypto_register_alg(struct crypto_alg *alg 423 int crypto_register_alg(struct crypto_alg *alg)
454 { 424 {
455 struct crypto_larval *larval; 425 struct crypto_larval *larval;
456 LIST_HEAD(algs_to_put); !! 426 bool test_started;
457 bool test_started = false; <<
458 int err; 427 int err;
459 428
460 alg->cra_flags &= ~CRYPTO_ALG_DEAD; 429 alg->cra_flags &= ~CRYPTO_ALG_DEAD;
461 err = crypto_check_alg(alg); 430 err = crypto_check_alg(alg);
462 if (err) 431 if (err)
463 return err; 432 return err;
464 433
465 down_write(&crypto_alg_sem); 434 down_write(&crypto_alg_sem);
466 larval = __crypto_register_alg(alg, &a !! 435 larval = __crypto_register_alg(alg);
467 if (!IS_ERR_OR_NULL(larval)) { !! 436 test_started = static_key_enabled(&crypto_boot_test_finished);
468 test_started = crypto_boot_tes !! 437 if (!IS_ERR_OR_NULL(larval))
469 larval->test_started = test_st 438 larval->test_started = test_started;
470 } <<
471 up_write(&crypto_alg_sem); 439 up_write(&crypto_alg_sem);
472 440
473 if (IS_ERR(larval)) !! 441 if (IS_ERR_OR_NULL(larval))
474 return PTR_ERR(larval); 442 return PTR_ERR(larval);
>> 443
475 if (test_started) 444 if (test_started)
476 crypto_wait_for_test(larval); 445 crypto_wait_for_test(larval);
477 crypto_remove_final(&algs_to_put); <<
478 return 0; 446 return 0;
479 } 447 }
480 EXPORT_SYMBOL_GPL(crypto_register_alg); 448 EXPORT_SYMBOL_GPL(crypto_register_alg);
481 449
482 static int crypto_remove_alg(struct crypto_alg 450 static int crypto_remove_alg(struct crypto_alg *alg, struct list_head *list)
483 { 451 {
484 if (unlikely(list_empty(&alg->cra_list 452 if (unlikely(list_empty(&alg->cra_list)))
485 return -ENOENT; 453 return -ENOENT;
486 454
487 alg->cra_flags |= CRYPTO_ALG_DEAD; 455 alg->cra_flags |= CRYPTO_ALG_DEAD;
488 456
489 list_del_init(&alg->cra_list); 457 list_del_init(&alg->cra_list);
490 crypto_remove_spawns(alg, list, NULL); 458 crypto_remove_spawns(alg, list, NULL);
491 459
492 return 0; 460 return 0;
493 } 461 }
494 462
495 void crypto_unregister_alg(struct crypto_alg * 463 void crypto_unregister_alg(struct crypto_alg *alg)
496 { 464 {
497 int ret; 465 int ret;
498 LIST_HEAD(list); 466 LIST_HEAD(list);
499 467
500 down_write(&crypto_alg_sem); 468 down_write(&crypto_alg_sem);
501 ret = crypto_remove_alg(alg, &list); 469 ret = crypto_remove_alg(alg, &list);
502 up_write(&crypto_alg_sem); 470 up_write(&crypto_alg_sem);
503 471
504 if (WARN(ret, "Algorithm %s is not reg 472 if (WARN(ret, "Algorithm %s is not registered", alg->cra_driver_name))
505 return; 473 return;
506 474
507 if (WARN_ON(refcount_read(&alg->cra_re !! 475 BUG_ON(refcount_read(&alg->cra_refcnt) != 1);
508 return; <<
509 <<
510 if (alg->cra_destroy) 476 if (alg->cra_destroy)
511 alg->cra_destroy(alg); 477 alg->cra_destroy(alg);
512 478
513 crypto_remove_final(&list); 479 crypto_remove_final(&list);
514 } 480 }
515 EXPORT_SYMBOL_GPL(crypto_unregister_alg); 481 EXPORT_SYMBOL_GPL(crypto_unregister_alg);
516 482
517 int crypto_register_algs(struct crypto_alg *al 483 int crypto_register_algs(struct crypto_alg *algs, int count)
518 { 484 {
519 int i, ret; 485 int i, ret;
520 486
521 for (i = 0; i < count; i++) { 487 for (i = 0; i < count; i++) {
522 ret = crypto_register_alg(&alg 488 ret = crypto_register_alg(&algs[i]);
523 if (ret) 489 if (ret)
524 goto err; 490 goto err;
525 } 491 }
526 492
527 return 0; 493 return 0;
528 494
529 err: 495 err:
530 for (--i; i >= 0; --i) 496 for (--i; i >= 0; --i)
531 crypto_unregister_alg(&algs[i] 497 crypto_unregister_alg(&algs[i]);
532 498
533 return ret; 499 return ret;
534 } 500 }
535 EXPORT_SYMBOL_GPL(crypto_register_algs); 501 EXPORT_SYMBOL_GPL(crypto_register_algs);
536 502
537 void crypto_unregister_algs(struct crypto_alg 503 void crypto_unregister_algs(struct crypto_alg *algs, int count)
538 { 504 {
539 int i; 505 int i;
540 506
541 for (i = 0; i < count; i++) 507 for (i = 0; i < count; i++)
542 crypto_unregister_alg(&algs[i] 508 crypto_unregister_alg(&algs[i]);
543 } 509 }
544 EXPORT_SYMBOL_GPL(crypto_unregister_algs); 510 EXPORT_SYMBOL_GPL(crypto_unregister_algs);
545 511
546 int crypto_register_template(struct crypto_tem 512 int crypto_register_template(struct crypto_template *tmpl)
547 { 513 {
548 struct crypto_template *q; 514 struct crypto_template *q;
549 int err = -EEXIST; 515 int err = -EEXIST;
550 516
551 down_write(&crypto_alg_sem); 517 down_write(&crypto_alg_sem);
552 518
553 crypto_check_module_sig(tmpl->module); 519 crypto_check_module_sig(tmpl->module);
554 520
555 list_for_each_entry(q, &crypto_templat 521 list_for_each_entry(q, &crypto_template_list, list) {
556 if (q == tmpl) 522 if (q == tmpl)
557 goto out; 523 goto out;
558 } 524 }
559 525
560 list_add(&tmpl->list, &crypto_template 526 list_add(&tmpl->list, &crypto_template_list);
561 err = 0; 527 err = 0;
562 out: 528 out:
563 up_write(&crypto_alg_sem); 529 up_write(&crypto_alg_sem);
564 return err; 530 return err;
565 } 531 }
566 EXPORT_SYMBOL_GPL(crypto_register_template); 532 EXPORT_SYMBOL_GPL(crypto_register_template);
567 533
568 int crypto_register_templates(struct crypto_te 534 int crypto_register_templates(struct crypto_template *tmpls, int count)
569 { 535 {
570 int i, err; 536 int i, err;
571 537
572 for (i = 0; i < count; i++) { 538 for (i = 0; i < count; i++) {
573 err = crypto_register_template 539 err = crypto_register_template(&tmpls[i]);
574 if (err) 540 if (err)
575 goto out; 541 goto out;
576 } 542 }
577 return 0; 543 return 0;
578 544
579 out: 545 out:
580 for (--i; i >= 0; --i) 546 for (--i; i >= 0; --i)
581 crypto_unregister_template(&tm 547 crypto_unregister_template(&tmpls[i]);
582 return err; 548 return err;
583 } 549 }
584 EXPORT_SYMBOL_GPL(crypto_register_templates); 550 EXPORT_SYMBOL_GPL(crypto_register_templates);
585 551
586 void crypto_unregister_template(struct crypto_ 552 void crypto_unregister_template(struct crypto_template *tmpl)
587 { 553 {
588 struct crypto_instance *inst; 554 struct crypto_instance *inst;
589 struct hlist_node *n; 555 struct hlist_node *n;
590 struct hlist_head *list; 556 struct hlist_head *list;
591 LIST_HEAD(users); 557 LIST_HEAD(users);
592 558
593 down_write(&crypto_alg_sem); 559 down_write(&crypto_alg_sem);
594 560
595 BUG_ON(list_empty(&tmpl->list)); 561 BUG_ON(list_empty(&tmpl->list));
596 list_del_init(&tmpl->list); 562 list_del_init(&tmpl->list);
597 563
598 list = &tmpl->instances; 564 list = &tmpl->instances;
599 hlist_for_each_entry(inst, list, list) 565 hlist_for_each_entry(inst, list, list) {
600 int err = crypto_remove_alg(&i 566 int err = crypto_remove_alg(&inst->alg, &users);
601 567
602 BUG_ON(err); 568 BUG_ON(err);
603 } 569 }
604 570
605 up_write(&crypto_alg_sem); 571 up_write(&crypto_alg_sem);
606 572
607 hlist_for_each_entry_safe(inst, n, lis 573 hlist_for_each_entry_safe(inst, n, list, list) {
608 BUG_ON(refcount_read(&inst->al 574 BUG_ON(refcount_read(&inst->alg.cra_refcnt) != 1);
609 crypto_free_instance(inst); 575 crypto_free_instance(inst);
610 } 576 }
611 crypto_remove_final(&users); 577 crypto_remove_final(&users);
612 } 578 }
613 EXPORT_SYMBOL_GPL(crypto_unregister_template); 579 EXPORT_SYMBOL_GPL(crypto_unregister_template);
614 580
615 void crypto_unregister_templates(struct crypto 581 void crypto_unregister_templates(struct crypto_template *tmpls, int count)
616 { 582 {
617 int i; 583 int i;
618 584
619 for (i = count - 1; i >= 0; --i) 585 for (i = count - 1; i >= 0; --i)
620 crypto_unregister_template(&tm 586 crypto_unregister_template(&tmpls[i]);
621 } 587 }
622 EXPORT_SYMBOL_GPL(crypto_unregister_templates) 588 EXPORT_SYMBOL_GPL(crypto_unregister_templates);
623 589
624 static struct crypto_template *__crypto_lookup 590 static struct crypto_template *__crypto_lookup_template(const char *name)
625 { 591 {
626 struct crypto_template *q, *tmpl = NUL 592 struct crypto_template *q, *tmpl = NULL;
627 593
628 down_read(&crypto_alg_sem); 594 down_read(&crypto_alg_sem);
629 list_for_each_entry(q, &crypto_templat 595 list_for_each_entry(q, &crypto_template_list, list) {
630 if (strcmp(q->name, name)) 596 if (strcmp(q->name, name))
631 continue; 597 continue;
632 if (unlikely(!crypto_tmpl_get( 598 if (unlikely(!crypto_tmpl_get(q)))
633 continue; 599 continue;
634 600
635 tmpl = q; 601 tmpl = q;
636 break; 602 break;
637 } 603 }
638 up_read(&crypto_alg_sem); 604 up_read(&crypto_alg_sem);
639 605
640 return tmpl; 606 return tmpl;
641 } 607 }
642 608
643 struct crypto_template *crypto_lookup_template 609 struct crypto_template *crypto_lookup_template(const char *name)
644 { 610 {
645 return try_then_request_module(__crypt 611 return try_then_request_module(__crypto_lookup_template(name),
646 "crypto 612 "crypto-%s", name);
647 } 613 }
648 EXPORT_SYMBOL_GPL(crypto_lookup_template); 614 EXPORT_SYMBOL_GPL(crypto_lookup_template);
649 615
650 int crypto_register_instance(struct crypto_tem 616 int crypto_register_instance(struct crypto_template *tmpl,
651 struct crypto_ins 617 struct crypto_instance *inst)
652 { 618 {
653 struct crypto_larval *larval; 619 struct crypto_larval *larval;
654 struct crypto_spawn *spawn; 620 struct crypto_spawn *spawn;
655 u32 fips_internal = 0; 621 u32 fips_internal = 0;
656 LIST_HEAD(algs_to_put); <<
657 int err; 622 int err;
658 623
659 err = crypto_check_alg(&inst->alg); 624 err = crypto_check_alg(&inst->alg);
660 if (err) 625 if (err)
661 return err; 626 return err;
662 627
663 inst->alg.cra_module = tmpl->module; 628 inst->alg.cra_module = tmpl->module;
664 inst->alg.cra_flags |= CRYPTO_ALG_INST 629 inst->alg.cra_flags |= CRYPTO_ALG_INSTANCE;
665 630
666 down_write(&crypto_alg_sem); 631 down_write(&crypto_alg_sem);
667 632
668 larval = ERR_PTR(-EAGAIN); 633 larval = ERR_PTR(-EAGAIN);
669 for (spawn = inst->spawns; spawn;) { 634 for (spawn = inst->spawns; spawn;) {
670 struct crypto_spawn *next; 635 struct crypto_spawn *next;
671 636
672 if (spawn->dead) 637 if (spawn->dead)
673 goto unlock; 638 goto unlock;
674 639
675 next = spawn->next; 640 next = spawn->next;
676 spawn->inst = inst; 641 spawn->inst = inst;
677 spawn->registered = true; 642 spawn->registered = true;
678 643
679 fips_internal |= spawn->alg->c 644 fips_internal |= spawn->alg->cra_flags;
680 645
681 crypto_mod_put(spawn->alg); 646 crypto_mod_put(spawn->alg);
682 647
683 spawn = next; 648 spawn = next;
684 } 649 }
685 650
686 inst->alg.cra_flags |= (fips_internal 651 inst->alg.cra_flags |= (fips_internal & CRYPTO_ALG_FIPS_INTERNAL);
687 652
688 larval = __crypto_register_alg(&inst-> !! 653 larval = __crypto_register_alg(&inst->alg);
689 if (IS_ERR(larval)) 654 if (IS_ERR(larval))
690 goto unlock; 655 goto unlock;
691 else if (larval) 656 else if (larval)
692 larval->test_started = true; 657 larval->test_started = true;
693 658
694 hlist_add_head(&inst->list, &tmpl->ins 659 hlist_add_head(&inst->list, &tmpl->instances);
695 inst->tmpl = tmpl; 660 inst->tmpl = tmpl;
696 661
697 unlock: 662 unlock:
698 up_write(&crypto_alg_sem); 663 up_write(&crypto_alg_sem);
699 664
700 if (IS_ERR(larval)) !! 665 err = PTR_ERR(larval);
701 return PTR_ERR(larval); !! 666 if (IS_ERR_OR_NULL(larval))
702 if (larval) !! 667 goto err;
703 crypto_wait_for_test(larval); !! 668
704 crypto_remove_final(&algs_to_put); !! 669 crypto_wait_for_test(larval);
705 return 0; !! 670 err = 0;
>> 671
>> 672 err:
>> 673 return err;
706 } 674 }
707 EXPORT_SYMBOL_GPL(crypto_register_instance); 675 EXPORT_SYMBOL_GPL(crypto_register_instance);
708 676
709 void crypto_unregister_instance(struct crypto_ 677 void crypto_unregister_instance(struct crypto_instance *inst)
710 { 678 {
711 LIST_HEAD(list); 679 LIST_HEAD(list);
712 680
713 down_write(&crypto_alg_sem); 681 down_write(&crypto_alg_sem);
714 682
715 crypto_remove_spawns(&inst->alg, &list 683 crypto_remove_spawns(&inst->alg, &list, NULL);
716 crypto_remove_instance(inst, &list); 684 crypto_remove_instance(inst, &list);
717 685
718 up_write(&crypto_alg_sem); 686 up_write(&crypto_alg_sem);
719 687
720 crypto_remove_final(&list); 688 crypto_remove_final(&list);
721 } 689 }
722 EXPORT_SYMBOL_GPL(crypto_unregister_instance); 690 EXPORT_SYMBOL_GPL(crypto_unregister_instance);
723 691
724 int crypto_grab_spawn(struct crypto_spawn *spa 692 int crypto_grab_spawn(struct crypto_spawn *spawn, struct crypto_instance *inst,
725 const char *name, u32 ty 693 const char *name, u32 type, u32 mask)
726 { 694 {
727 struct crypto_alg *alg; 695 struct crypto_alg *alg;
728 int err = -EAGAIN; 696 int err = -EAGAIN;
729 697
730 if (WARN_ON_ONCE(inst == NULL)) 698 if (WARN_ON_ONCE(inst == NULL))
731 return -EINVAL; 699 return -EINVAL;
732 700
733 /* Allow the result of crypto_attr_alg 701 /* Allow the result of crypto_attr_alg_name() to be passed directly */
734 if (IS_ERR(name)) 702 if (IS_ERR(name))
735 return PTR_ERR(name); 703 return PTR_ERR(name);
736 704
737 alg = crypto_find_alg(name, spawn->fro 705 alg = crypto_find_alg(name, spawn->frontend,
738 type | CRYPTO_AL 706 type | CRYPTO_ALG_FIPS_INTERNAL, mask);
739 if (IS_ERR(alg)) 707 if (IS_ERR(alg))
740 return PTR_ERR(alg); 708 return PTR_ERR(alg);
741 709
742 down_write(&crypto_alg_sem); 710 down_write(&crypto_alg_sem);
743 if (!crypto_is_moribund(alg)) { 711 if (!crypto_is_moribund(alg)) {
744 list_add(&spawn->list, &alg->c 712 list_add(&spawn->list, &alg->cra_users);
745 spawn->alg = alg; 713 spawn->alg = alg;
746 spawn->mask = mask; 714 spawn->mask = mask;
747 spawn->next = inst->spawns; 715 spawn->next = inst->spawns;
748 inst->spawns = spawn; 716 inst->spawns = spawn;
749 inst->alg.cra_flags |= 717 inst->alg.cra_flags |=
750 (alg->cra_flags & CRYP 718 (alg->cra_flags & CRYPTO_ALG_INHERITED_FLAGS);
751 err = 0; 719 err = 0;
752 } 720 }
753 up_write(&crypto_alg_sem); 721 up_write(&crypto_alg_sem);
754 if (err) 722 if (err)
755 crypto_mod_put(alg); 723 crypto_mod_put(alg);
756 return err; 724 return err;
757 } 725 }
758 EXPORT_SYMBOL_GPL(crypto_grab_spawn); 726 EXPORT_SYMBOL_GPL(crypto_grab_spawn);
759 727
760 void crypto_drop_spawn(struct crypto_spawn *sp 728 void crypto_drop_spawn(struct crypto_spawn *spawn)
761 { 729 {
762 if (!spawn->alg) /* not yet initialize 730 if (!spawn->alg) /* not yet initialized? */
763 return; 731 return;
764 732
765 down_write(&crypto_alg_sem); 733 down_write(&crypto_alg_sem);
766 if (!spawn->dead) 734 if (!spawn->dead)
767 list_del(&spawn->list); 735 list_del(&spawn->list);
768 up_write(&crypto_alg_sem); 736 up_write(&crypto_alg_sem);
769 737
770 if (!spawn->registered) 738 if (!spawn->registered)
771 crypto_mod_put(spawn->alg); 739 crypto_mod_put(spawn->alg);
772 } 740 }
773 EXPORT_SYMBOL_GPL(crypto_drop_spawn); 741 EXPORT_SYMBOL_GPL(crypto_drop_spawn);
774 742
775 static struct crypto_alg *crypto_spawn_alg(str 743 static struct crypto_alg *crypto_spawn_alg(struct crypto_spawn *spawn)
776 { 744 {
777 struct crypto_alg *alg = ERR_PTR(-EAGA 745 struct crypto_alg *alg = ERR_PTR(-EAGAIN);
778 struct crypto_alg *target; 746 struct crypto_alg *target;
779 bool shoot = false; 747 bool shoot = false;
780 748
781 down_read(&crypto_alg_sem); 749 down_read(&crypto_alg_sem);
782 if (!spawn->dead) { 750 if (!spawn->dead) {
783 alg = spawn->alg; 751 alg = spawn->alg;
784 if (!crypto_mod_get(alg)) { 752 if (!crypto_mod_get(alg)) {
785 target = crypto_alg_ge 753 target = crypto_alg_get(alg);
786 shoot = true; 754 shoot = true;
787 alg = ERR_PTR(-EAGAIN) 755 alg = ERR_PTR(-EAGAIN);
788 } 756 }
789 } 757 }
790 up_read(&crypto_alg_sem); 758 up_read(&crypto_alg_sem);
791 759
792 if (shoot) { 760 if (shoot) {
793 crypto_shoot_alg(target); 761 crypto_shoot_alg(target);
794 crypto_alg_put(target); 762 crypto_alg_put(target);
795 } 763 }
796 764
797 return alg; 765 return alg;
798 } 766 }
799 767
800 struct crypto_tfm *crypto_spawn_tfm(struct cry 768 struct crypto_tfm *crypto_spawn_tfm(struct crypto_spawn *spawn, u32 type,
801 u32 mask) 769 u32 mask)
802 { 770 {
803 struct crypto_alg *alg; 771 struct crypto_alg *alg;
804 struct crypto_tfm *tfm; 772 struct crypto_tfm *tfm;
805 773
806 alg = crypto_spawn_alg(spawn); 774 alg = crypto_spawn_alg(spawn);
807 if (IS_ERR(alg)) 775 if (IS_ERR(alg))
808 return ERR_CAST(alg); 776 return ERR_CAST(alg);
809 777
810 tfm = ERR_PTR(-EINVAL); 778 tfm = ERR_PTR(-EINVAL);
811 if (unlikely((alg->cra_flags ^ type) & 779 if (unlikely((alg->cra_flags ^ type) & mask))
812 goto out_put_alg; 780 goto out_put_alg;
813 781
814 tfm = __crypto_alloc_tfm(alg, type, ma 782 tfm = __crypto_alloc_tfm(alg, type, mask);
815 if (IS_ERR(tfm)) 783 if (IS_ERR(tfm))
816 goto out_put_alg; 784 goto out_put_alg;
817 785
818 return tfm; 786 return tfm;
819 787
820 out_put_alg: 788 out_put_alg:
821 crypto_mod_put(alg); 789 crypto_mod_put(alg);
822 return tfm; 790 return tfm;
823 } 791 }
824 EXPORT_SYMBOL_GPL(crypto_spawn_tfm); 792 EXPORT_SYMBOL_GPL(crypto_spawn_tfm);
825 793
826 void *crypto_spawn_tfm2(struct crypto_spawn *s 794 void *crypto_spawn_tfm2(struct crypto_spawn *spawn)
827 { 795 {
828 struct crypto_alg *alg; 796 struct crypto_alg *alg;
829 struct crypto_tfm *tfm; 797 struct crypto_tfm *tfm;
830 798
831 alg = crypto_spawn_alg(spawn); 799 alg = crypto_spawn_alg(spawn);
832 if (IS_ERR(alg)) 800 if (IS_ERR(alg))
833 return ERR_CAST(alg); 801 return ERR_CAST(alg);
834 802
835 tfm = crypto_create_tfm(alg, spawn->fr 803 tfm = crypto_create_tfm(alg, spawn->frontend);
836 if (IS_ERR(tfm)) 804 if (IS_ERR(tfm))
837 goto out_put_alg; 805 goto out_put_alg;
838 806
839 return tfm; 807 return tfm;
840 808
841 out_put_alg: 809 out_put_alg:
842 crypto_mod_put(alg); 810 crypto_mod_put(alg);
843 return tfm; 811 return tfm;
844 } 812 }
845 EXPORT_SYMBOL_GPL(crypto_spawn_tfm2); 813 EXPORT_SYMBOL_GPL(crypto_spawn_tfm2);
846 814
847 int crypto_register_notifier(struct notifier_b 815 int crypto_register_notifier(struct notifier_block *nb)
848 { 816 {
849 return blocking_notifier_chain_registe 817 return blocking_notifier_chain_register(&crypto_chain, nb);
850 } 818 }
851 EXPORT_SYMBOL_GPL(crypto_register_notifier); 819 EXPORT_SYMBOL_GPL(crypto_register_notifier);
852 820
853 int crypto_unregister_notifier(struct notifier 821 int crypto_unregister_notifier(struct notifier_block *nb)
854 { 822 {
855 return blocking_notifier_chain_unregis 823 return blocking_notifier_chain_unregister(&crypto_chain, nb);
856 } 824 }
857 EXPORT_SYMBOL_GPL(crypto_unregister_notifier); 825 EXPORT_SYMBOL_GPL(crypto_unregister_notifier);
858 826
859 struct crypto_attr_type *crypto_get_attr_type( 827 struct crypto_attr_type *crypto_get_attr_type(struct rtattr **tb)
860 { 828 {
861 struct rtattr *rta = tb[0]; 829 struct rtattr *rta = tb[0];
862 struct crypto_attr_type *algt; 830 struct crypto_attr_type *algt;
863 831
864 if (!rta) 832 if (!rta)
865 return ERR_PTR(-ENOENT); 833 return ERR_PTR(-ENOENT);
866 if (RTA_PAYLOAD(rta) < sizeof(*algt)) 834 if (RTA_PAYLOAD(rta) < sizeof(*algt))
867 return ERR_PTR(-EINVAL); 835 return ERR_PTR(-EINVAL);
868 if (rta->rta_type != CRYPTOA_TYPE) 836 if (rta->rta_type != CRYPTOA_TYPE)
869 return ERR_PTR(-EINVAL); 837 return ERR_PTR(-EINVAL);
870 838
871 algt = RTA_DATA(rta); 839 algt = RTA_DATA(rta);
872 840
873 return algt; 841 return algt;
874 } 842 }
875 EXPORT_SYMBOL_GPL(crypto_get_attr_type); 843 EXPORT_SYMBOL_GPL(crypto_get_attr_type);
876 844
877 /** 845 /**
878 * crypto_check_attr_type() - check algorithm 846 * crypto_check_attr_type() - check algorithm type and compute inherited mask
879 * @tb: the template parameters 847 * @tb: the template parameters
880 * @type: the algorithm type the template woul 848 * @type: the algorithm type the template would be instantiated as
881 * @mask_ret: (output) the mask that should be 849 * @mask_ret: (output) the mask that should be passed to crypto_grab_*()
882 * to restrict the flags of any inn 850 * to restrict the flags of any inner algorithms
883 * 851 *
884 * Validate that the algorithm type the user r 852 * Validate that the algorithm type the user requested is compatible with the
885 * one the template would actually be instanti 853 * one the template would actually be instantiated as. E.g., if the user is
886 * doing crypto_alloc_shash("cbc(aes)", ...), 854 * doing crypto_alloc_shash("cbc(aes)", ...), this would return an error because
887 * the "cbc" template creates an "skcipher" al 855 * the "cbc" template creates an "skcipher" algorithm, not an "shash" algorithm.
888 * 856 *
889 * Also compute the mask to use to restrict th 857 * Also compute the mask to use to restrict the flags of any inner algorithms.
890 * 858 *
891 * Return: 0 on success; -errno on failure 859 * Return: 0 on success; -errno on failure
892 */ 860 */
893 int crypto_check_attr_type(struct rtattr **tb, 861 int crypto_check_attr_type(struct rtattr **tb, u32 type, u32 *mask_ret)
894 { 862 {
895 struct crypto_attr_type *algt; 863 struct crypto_attr_type *algt;
896 864
897 algt = crypto_get_attr_type(tb); 865 algt = crypto_get_attr_type(tb);
898 if (IS_ERR(algt)) 866 if (IS_ERR(algt))
899 return PTR_ERR(algt); 867 return PTR_ERR(algt);
900 868
901 if ((algt->type ^ type) & algt->mask) 869 if ((algt->type ^ type) & algt->mask)
902 return -EINVAL; 870 return -EINVAL;
903 871
904 *mask_ret = crypto_algt_inherited_mask 872 *mask_ret = crypto_algt_inherited_mask(algt);
905 return 0; 873 return 0;
906 } 874 }
907 EXPORT_SYMBOL_GPL(crypto_check_attr_type); 875 EXPORT_SYMBOL_GPL(crypto_check_attr_type);
908 876
909 const char *crypto_attr_alg_name(struct rtattr 877 const char *crypto_attr_alg_name(struct rtattr *rta)
910 { 878 {
911 struct crypto_attr_alg *alga; 879 struct crypto_attr_alg *alga;
912 880
913 if (!rta) 881 if (!rta)
914 return ERR_PTR(-ENOENT); 882 return ERR_PTR(-ENOENT);
915 if (RTA_PAYLOAD(rta) < sizeof(*alga)) 883 if (RTA_PAYLOAD(rta) < sizeof(*alga))
916 return ERR_PTR(-EINVAL); 884 return ERR_PTR(-EINVAL);
917 if (rta->rta_type != CRYPTOA_ALG) 885 if (rta->rta_type != CRYPTOA_ALG)
918 return ERR_PTR(-EINVAL); 886 return ERR_PTR(-EINVAL);
919 887
920 alga = RTA_DATA(rta); 888 alga = RTA_DATA(rta);
921 alga->name[CRYPTO_MAX_ALG_NAME - 1] = 889 alga->name[CRYPTO_MAX_ALG_NAME - 1] = 0;
922 890
923 return alga->name; 891 return alga->name;
924 } 892 }
925 EXPORT_SYMBOL_GPL(crypto_attr_alg_name); 893 EXPORT_SYMBOL_GPL(crypto_attr_alg_name);
926 894
927 int crypto_inst_setname(struct crypto_instance 895 int crypto_inst_setname(struct crypto_instance *inst, const char *name,
928 struct crypto_alg *alg 896 struct crypto_alg *alg)
929 { 897 {
930 if (snprintf(inst->alg.cra_name, CRYPT 898 if (snprintf(inst->alg.cra_name, CRYPTO_MAX_ALG_NAME, "%s(%s)", name,
931 alg->cra_name) >= CRYPTO_ 899 alg->cra_name) >= CRYPTO_MAX_ALG_NAME)
932 return -ENAMETOOLONG; 900 return -ENAMETOOLONG;
933 901
934 if (snprintf(inst->alg.cra_driver_name 902 if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s(%s)",
935 name, alg->cra_driver_nam 903 name, alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
936 return -ENAMETOOLONG; 904 return -ENAMETOOLONG;
937 905
938 return 0; 906 return 0;
939 } 907 }
940 EXPORT_SYMBOL_GPL(crypto_inst_setname); 908 EXPORT_SYMBOL_GPL(crypto_inst_setname);
941 909
942 void crypto_init_queue(struct crypto_queue *qu 910 void crypto_init_queue(struct crypto_queue *queue, unsigned int max_qlen)
943 { 911 {
944 INIT_LIST_HEAD(&queue->list); 912 INIT_LIST_HEAD(&queue->list);
945 queue->backlog = &queue->list; 913 queue->backlog = &queue->list;
946 queue->qlen = 0; 914 queue->qlen = 0;
947 queue->max_qlen = max_qlen; 915 queue->max_qlen = max_qlen;
948 } 916 }
949 EXPORT_SYMBOL_GPL(crypto_init_queue); 917 EXPORT_SYMBOL_GPL(crypto_init_queue);
950 918
951 int crypto_enqueue_request(struct crypto_queue 919 int crypto_enqueue_request(struct crypto_queue *queue,
952 struct crypto_async 920 struct crypto_async_request *request)
953 { 921 {
954 int err = -EINPROGRESS; 922 int err = -EINPROGRESS;
955 923
956 if (unlikely(queue->qlen >= queue->max 924 if (unlikely(queue->qlen >= queue->max_qlen)) {
957 if (!(request->flags & CRYPTO_ 925 if (!(request->flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
958 err = -ENOSPC; 926 err = -ENOSPC;
959 goto out; 927 goto out;
960 } 928 }
961 err = -EBUSY; 929 err = -EBUSY;
962 if (queue->backlog == &queue-> 930 if (queue->backlog == &queue->list)
963 queue->backlog = &requ 931 queue->backlog = &request->list;
964 } 932 }
965 933
966 queue->qlen++; 934 queue->qlen++;
967 list_add_tail(&request->list, &queue-> 935 list_add_tail(&request->list, &queue->list);
968 936
969 out: 937 out:
970 return err; 938 return err;
971 } 939 }
972 EXPORT_SYMBOL_GPL(crypto_enqueue_request); 940 EXPORT_SYMBOL_GPL(crypto_enqueue_request);
973 941
974 void crypto_enqueue_request_head(struct crypto 942 void crypto_enqueue_request_head(struct crypto_queue *queue,
975 struct crypto 943 struct crypto_async_request *request)
976 { 944 {
977 if (unlikely(queue->qlen >= queue->max <<
978 queue->backlog = queue->backlo <<
979 <<
980 queue->qlen++; 945 queue->qlen++;
981 list_add(&request->list, &queue->list) 946 list_add(&request->list, &queue->list);
982 } 947 }
983 EXPORT_SYMBOL_GPL(crypto_enqueue_request_head) 948 EXPORT_SYMBOL_GPL(crypto_enqueue_request_head);
984 949
985 struct crypto_async_request *crypto_dequeue_re 950 struct crypto_async_request *crypto_dequeue_request(struct crypto_queue *queue)
986 { 951 {
987 struct list_head *request; 952 struct list_head *request;
988 953
989 if (unlikely(!queue->qlen)) 954 if (unlikely(!queue->qlen))
990 return NULL; 955 return NULL;
991 956
992 queue->qlen--; 957 queue->qlen--;
993 958
994 if (queue->backlog != &queue->list) 959 if (queue->backlog != &queue->list)
995 queue->backlog = queue->backlo 960 queue->backlog = queue->backlog->next;
996 961
997 request = queue->list.next; 962 request = queue->list.next;
998 list_del(request); 963 list_del(request);
999 964
1000 return list_entry(request, struct cry 965 return list_entry(request, struct crypto_async_request, list);
1001 } 966 }
1002 EXPORT_SYMBOL_GPL(crypto_dequeue_request); 967 EXPORT_SYMBOL_GPL(crypto_dequeue_request);
1003 968
1004 static inline void crypto_inc_byte(u8 *a, uns 969 static inline void crypto_inc_byte(u8 *a, unsigned int size)
1005 { 970 {
1006 u8 *b = (a + size); 971 u8 *b = (a + size);
1007 u8 c; 972 u8 c;
1008 973
1009 for (; size; size--) { 974 for (; size; size--) {
1010 c = *--b + 1; 975 c = *--b + 1;
1011 *b = c; 976 *b = c;
1012 if (c) 977 if (c)
1013 break; 978 break;
1014 } 979 }
1015 } 980 }
1016 981
1017 void crypto_inc(u8 *a, unsigned int size) 982 void crypto_inc(u8 *a, unsigned int size)
1018 { 983 {
1019 __be32 *b = (__be32 *)(a + size); 984 __be32 *b = (__be32 *)(a + size);
1020 u32 c; 985 u32 c;
1021 986
1022 if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_ 987 if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) ||
1023 IS_ALIGNED((unsigned long)b, __al 988 IS_ALIGNED((unsigned long)b, __alignof__(*b)))
1024 for (; size >= 4; size -= 4) 989 for (; size >= 4; size -= 4) {
1025 c = be32_to_cpu(*--b) 990 c = be32_to_cpu(*--b) + 1;
1026 *b = cpu_to_be32(c); 991 *b = cpu_to_be32(c);
1027 if (likely(c)) 992 if (likely(c))
1028 return; 993 return;
1029 } 994 }
1030 995
1031 crypto_inc_byte(a, size); 996 crypto_inc_byte(a, size);
1032 } 997 }
1033 EXPORT_SYMBOL_GPL(crypto_inc); 998 EXPORT_SYMBOL_GPL(crypto_inc);
1034 999
>> 1000 void __crypto_xor(u8 *dst, const u8 *src1, const u8 *src2, unsigned int len)
>> 1001 {
>> 1002 int relalign = 0;
>> 1003
>> 1004 if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) {
>> 1005 int size = sizeof(unsigned long);
>> 1006 int d = (((unsigned long)dst ^ (unsigned long)src1) |
>> 1007 ((unsigned long)dst ^ (unsigned long)src2)) &
>> 1008 (size - 1);
>> 1009
>> 1010 relalign = d ? 1 << __ffs(d) : size;
>> 1011
>> 1012 /*
>> 1013 * If we care about alignment, process as many bytes as
>> 1014 * needed to advance dst and src to values whose alignments
>> 1015 * equal their relative alignment. This will allow us to
>> 1016 * process the remainder of the input using optimal strides.
>> 1017 */
>> 1018 while (((unsigned long)dst & (relalign - 1)) && len > 0) {
>> 1019 *dst++ = *src1++ ^ *src2++;
>> 1020 len--;
>> 1021 }
>> 1022 }
>> 1023
>> 1024 while (IS_ENABLED(CONFIG_64BIT) && len >= 8 && !(relalign & 7)) {
>> 1025 if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) {
>> 1026 u64 l = get_unaligned((u64 *)src1) ^
>> 1027 get_unaligned((u64 *)src2);
>> 1028 put_unaligned(l, (u64 *)dst);
>> 1029 } else {
>> 1030 *(u64 *)dst = *(u64 *)src1 ^ *(u64 *)src2;
>> 1031 }
>> 1032 dst += 8;
>> 1033 src1 += 8;
>> 1034 src2 += 8;
>> 1035 len -= 8;
>> 1036 }
>> 1037
>> 1038 while (len >= 4 && !(relalign & 3)) {
>> 1039 if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) {
>> 1040 u32 l = get_unaligned((u32 *)src1) ^
>> 1041 get_unaligned((u32 *)src2);
>> 1042 put_unaligned(l, (u32 *)dst);
>> 1043 } else {
>> 1044 *(u32 *)dst = *(u32 *)src1 ^ *(u32 *)src2;
>> 1045 }
>> 1046 dst += 4;
>> 1047 src1 += 4;
>> 1048 src2 += 4;
>> 1049 len -= 4;
>> 1050 }
>> 1051
>> 1052 while (len >= 2 && !(relalign & 1)) {
>> 1053 if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) {
>> 1054 u16 l = get_unaligned((u16 *)src1) ^
>> 1055 get_unaligned((u16 *)src2);
>> 1056 put_unaligned(l, (u16 *)dst);
>> 1057 } else {
>> 1058 *(u16 *)dst = *(u16 *)src1 ^ *(u16 *)src2;
>> 1059 }
>> 1060 dst += 2;
>> 1061 src1 += 2;
>> 1062 src2 += 2;
>> 1063 len -= 2;
>> 1064 }
>> 1065
>> 1066 while (len--)
>> 1067 *dst++ = *src1++ ^ *src2++;
>> 1068 }
>> 1069 EXPORT_SYMBOL_GPL(__crypto_xor);
>> 1070
1035 unsigned int crypto_alg_extsize(struct crypto 1071 unsigned int crypto_alg_extsize(struct crypto_alg *alg)
1036 { 1072 {
1037 return alg->cra_ctxsize + 1073 return alg->cra_ctxsize +
1038 (alg->cra_alignmask & ~(crypto 1074 (alg->cra_alignmask & ~(crypto_tfm_ctx_alignment() - 1));
1039 } 1075 }
1040 EXPORT_SYMBOL_GPL(crypto_alg_extsize); 1076 EXPORT_SYMBOL_GPL(crypto_alg_extsize);
1041 1077
1042 int crypto_type_has_alg(const char *name, con 1078 int crypto_type_has_alg(const char *name, const struct crypto_type *frontend,
1043 u32 type, u32 mask) 1079 u32 type, u32 mask)
1044 { 1080 {
1045 int ret = 0; 1081 int ret = 0;
1046 struct crypto_alg *alg = crypto_find_ 1082 struct crypto_alg *alg = crypto_find_alg(name, frontend, type, mask);
1047 1083
1048 if (!IS_ERR(alg)) { 1084 if (!IS_ERR(alg)) {
1049 crypto_mod_put(alg); 1085 crypto_mod_put(alg);
1050 ret = 1; 1086 ret = 1;
1051 } 1087 }
1052 1088
1053 return ret; 1089 return ret;
1054 } 1090 }
1055 EXPORT_SYMBOL_GPL(crypto_type_has_alg); 1091 EXPORT_SYMBOL_GPL(crypto_type_has_alg);
1056 1092
1057 static void __init crypto_start_tests(void) !! 1093 #ifdef CONFIG_CRYPTO_STATS
>> 1094 void crypto_stats_init(struct crypto_alg *alg)
1058 { 1095 {
1059 if (!IS_BUILTIN(CONFIG_CRYPTO_ALGAPI) !! 1096 memset(&alg->stats, 0, sizeof(alg->stats));
1060 return; !! 1097 }
>> 1098 EXPORT_SYMBOL_GPL(crypto_stats_init);
1061 1099
1062 if (IS_ENABLED(CONFIG_CRYPTO_MANAGER_ !! 1100 void crypto_stats_get(struct crypto_alg *alg)
1063 return; !! 1101 {
>> 1102 crypto_alg_get(alg);
>> 1103 }
>> 1104 EXPORT_SYMBOL_GPL(crypto_stats_get);
>> 1105
>> 1106 void crypto_stats_aead_encrypt(unsigned int cryptlen, struct crypto_alg *alg,
>> 1107 int ret)
>> 1108 {
>> 1109 if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
>> 1110 atomic64_inc(&alg->stats.aead.err_cnt);
>> 1111 } else {
>> 1112 atomic64_inc(&alg->stats.aead.encrypt_cnt);
>> 1113 atomic64_add(cryptlen, &alg->stats.aead.encrypt_tlen);
>> 1114 }
>> 1115 crypto_alg_put(alg);
>> 1116 }
>> 1117 EXPORT_SYMBOL_GPL(crypto_stats_aead_encrypt);
>> 1118
>> 1119 void crypto_stats_aead_decrypt(unsigned int cryptlen, struct crypto_alg *alg,
>> 1120 int ret)
>> 1121 {
>> 1122 if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
>> 1123 atomic64_inc(&alg->stats.aead.err_cnt);
>> 1124 } else {
>> 1125 atomic64_inc(&alg->stats.aead.decrypt_cnt);
>> 1126 atomic64_add(cryptlen, &alg->stats.aead.decrypt_tlen);
>> 1127 }
>> 1128 crypto_alg_put(alg);
>> 1129 }
>> 1130 EXPORT_SYMBOL_GPL(crypto_stats_aead_decrypt);
>> 1131
>> 1132 void crypto_stats_akcipher_encrypt(unsigned int src_len, int ret,
>> 1133 struct crypto_alg *alg)
>> 1134 {
>> 1135 if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
>> 1136 atomic64_inc(&alg->stats.akcipher.err_cnt);
>> 1137 } else {
>> 1138 atomic64_inc(&alg->stats.akcipher.encrypt_cnt);
>> 1139 atomic64_add(src_len, &alg->stats.akcipher.encrypt_tlen);
>> 1140 }
>> 1141 crypto_alg_put(alg);
>> 1142 }
>> 1143 EXPORT_SYMBOL_GPL(crypto_stats_akcipher_encrypt);
>> 1144
>> 1145 void crypto_stats_akcipher_decrypt(unsigned int src_len, int ret,
>> 1146 struct crypto_alg *alg)
>> 1147 {
>> 1148 if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
>> 1149 atomic64_inc(&alg->stats.akcipher.err_cnt);
>> 1150 } else {
>> 1151 atomic64_inc(&alg->stats.akcipher.decrypt_cnt);
>> 1152 atomic64_add(src_len, &alg->stats.akcipher.decrypt_tlen);
>> 1153 }
>> 1154 crypto_alg_put(alg);
>> 1155 }
>> 1156 EXPORT_SYMBOL_GPL(crypto_stats_akcipher_decrypt);
>> 1157
>> 1158 void crypto_stats_akcipher_sign(int ret, struct crypto_alg *alg)
>> 1159 {
>> 1160 if (ret && ret != -EINPROGRESS && ret != -EBUSY)
>> 1161 atomic64_inc(&alg->stats.akcipher.err_cnt);
>> 1162 else
>> 1163 atomic64_inc(&alg->stats.akcipher.sign_cnt);
>> 1164 crypto_alg_put(alg);
>> 1165 }
>> 1166 EXPORT_SYMBOL_GPL(crypto_stats_akcipher_sign);
>> 1167
>> 1168 void crypto_stats_akcipher_verify(int ret, struct crypto_alg *alg)
>> 1169 {
>> 1170 if (ret && ret != -EINPROGRESS && ret != -EBUSY)
>> 1171 atomic64_inc(&alg->stats.akcipher.err_cnt);
>> 1172 else
>> 1173 atomic64_inc(&alg->stats.akcipher.verify_cnt);
>> 1174 crypto_alg_put(alg);
>> 1175 }
>> 1176 EXPORT_SYMBOL_GPL(crypto_stats_akcipher_verify);
>> 1177
>> 1178 void crypto_stats_compress(unsigned int slen, int ret, struct crypto_alg *alg)
>> 1179 {
>> 1180 if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
>> 1181 atomic64_inc(&alg->stats.compress.err_cnt);
>> 1182 } else {
>> 1183 atomic64_inc(&alg->stats.compress.compress_cnt);
>> 1184 atomic64_add(slen, &alg->stats.compress.compress_tlen);
>> 1185 }
>> 1186 crypto_alg_put(alg);
>> 1187 }
>> 1188 EXPORT_SYMBOL_GPL(crypto_stats_compress);
>> 1189
>> 1190 void crypto_stats_decompress(unsigned int slen, int ret, struct crypto_alg *alg)
>> 1191 {
>> 1192 if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
>> 1193 atomic64_inc(&alg->stats.compress.err_cnt);
>> 1194 } else {
>> 1195 atomic64_inc(&alg->stats.compress.decompress_cnt);
>> 1196 atomic64_add(slen, &alg->stats.compress.decompress_tlen);
>> 1197 }
>> 1198 crypto_alg_put(alg);
>> 1199 }
>> 1200 EXPORT_SYMBOL_GPL(crypto_stats_decompress);
>> 1201
>> 1202 void crypto_stats_ahash_update(unsigned int nbytes, int ret,
>> 1203 struct crypto_alg *alg)
>> 1204 {
>> 1205 if (ret && ret != -EINPROGRESS && ret != -EBUSY)
>> 1206 atomic64_inc(&alg->stats.hash.err_cnt);
>> 1207 else
>> 1208 atomic64_add(nbytes, &alg->stats.hash.hash_tlen);
>> 1209 crypto_alg_put(alg);
>> 1210 }
>> 1211 EXPORT_SYMBOL_GPL(crypto_stats_ahash_update);
>> 1212
>> 1213 void crypto_stats_ahash_final(unsigned int nbytes, int ret,
>> 1214 struct crypto_alg *alg)
>> 1215 {
>> 1216 if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
>> 1217 atomic64_inc(&alg->stats.hash.err_cnt);
>> 1218 } else {
>> 1219 atomic64_inc(&alg->stats.hash.hash_cnt);
>> 1220 atomic64_add(nbytes, &alg->stats.hash.hash_tlen);
>> 1221 }
>> 1222 crypto_alg_put(alg);
>> 1223 }
>> 1224 EXPORT_SYMBOL_GPL(crypto_stats_ahash_final);
1064 1225
>> 1226 void crypto_stats_kpp_set_secret(struct crypto_alg *alg, int ret)
>> 1227 {
>> 1228 if (ret)
>> 1229 atomic64_inc(&alg->stats.kpp.err_cnt);
>> 1230 else
>> 1231 atomic64_inc(&alg->stats.kpp.setsecret_cnt);
>> 1232 crypto_alg_put(alg);
>> 1233 }
>> 1234 EXPORT_SYMBOL_GPL(crypto_stats_kpp_set_secret);
>> 1235
>> 1236 void crypto_stats_kpp_generate_public_key(struct crypto_alg *alg, int ret)
>> 1237 {
>> 1238 if (ret)
>> 1239 atomic64_inc(&alg->stats.kpp.err_cnt);
>> 1240 else
>> 1241 atomic64_inc(&alg->stats.kpp.generate_public_key_cnt);
>> 1242 crypto_alg_put(alg);
>> 1243 }
>> 1244 EXPORT_SYMBOL_GPL(crypto_stats_kpp_generate_public_key);
>> 1245
>> 1246 void crypto_stats_kpp_compute_shared_secret(struct crypto_alg *alg, int ret)
>> 1247 {
>> 1248 if (ret)
>> 1249 atomic64_inc(&alg->stats.kpp.err_cnt);
>> 1250 else
>> 1251 atomic64_inc(&alg->stats.kpp.compute_shared_secret_cnt);
>> 1252 crypto_alg_put(alg);
>> 1253 }
>> 1254 EXPORT_SYMBOL_GPL(crypto_stats_kpp_compute_shared_secret);
>> 1255
>> 1256 void crypto_stats_rng_seed(struct crypto_alg *alg, int ret)
>> 1257 {
>> 1258 if (ret && ret != -EINPROGRESS && ret != -EBUSY)
>> 1259 atomic64_inc(&alg->stats.rng.err_cnt);
>> 1260 else
>> 1261 atomic64_inc(&alg->stats.rng.seed_cnt);
>> 1262 crypto_alg_put(alg);
>> 1263 }
>> 1264 EXPORT_SYMBOL_GPL(crypto_stats_rng_seed);
>> 1265
>> 1266 void crypto_stats_rng_generate(struct crypto_alg *alg, unsigned int dlen,
>> 1267 int ret)
>> 1268 {
>> 1269 if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
>> 1270 atomic64_inc(&alg->stats.rng.err_cnt);
>> 1271 } else {
>> 1272 atomic64_inc(&alg->stats.rng.generate_cnt);
>> 1273 atomic64_add(dlen, &alg->stats.rng.generate_tlen);
>> 1274 }
>> 1275 crypto_alg_put(alg);
>> 1276 }
>> 1277 EXPORT_SYMBOL_GPL(crypto_stats_rng_generate);
>> 1278
>> 1279 void crypto_stats_skcipher_encrypt(unsigned int cryptlen, int ret,
>> 1280 struct crypto_alg *alg)
>> 1281 {
>> 1282 if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
>> 1283 atomic64_inc(&alg->stats.cipher.err_cnt);
>> 1284 } else {
>> 1285 atomic64_inc(&alg->stats.cipher.encrypt_cnt);
>> 1286 atomic64_add(cryptlen, &alg->stats.cipher.encrypt_tlen);
>> 1287 }
>> 1288 crypto_alg_put(alg);
>> 1289 }
>> 1290 EXPORT_SYMBOL_GPL(crypto_stats_skcipher_encrypt);
>> 1291
>> 1292 void crypto_stats_skcipher_decrypt(unsigned int cryptlen, int ret,
>> 1293 struct crypto_alg *alg)
>> 1294 {
>> 1295 if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
>> 1296 atomic64_inc(&alg->stats.cipher.err_cnt);
>> 1297 } else {
>> 1298 atomic64_inc(&alg->stats.cipher.decrypt_cnt);
>> 1299 atomic64_add(cryptlen, &alg->stats.cipher.decrypt_tlen);
>> 1300 }
>> 1301 crypto_alg_put(alg);
>> 1302 }
>> 1303 EXPORT_SYMBOL_GPL(crypto_stats_skcipher_decrypt);
>> 1304 #endif
>> 1305
>> 1306 static void __init crypto_start_tests(void)
>> 1307 {
1065 for (;;) { 1308 for (;;) {
1066 struct crypto_larval *larval 1309 struct crypto_larval *larval = NULL;
1067 struct crypto_alg *q; 1310 struct crypto_alg *q;
1068 1311
1069 down_write(&crypto_alg_sem); 1312 down_write(&crypto_alg_sem);
1070 1313
1071 list_for_each_entry(q, &crypt 1314 list_for_each_entry(q, &crypto_alg_list, cra_list) {
1072 struct crypto_larval 1315 struct crypto_larval *l;
1073 1316
1074 if (!crypto_is_larval 1317 if (!crypto_is_larval(q))
1075 continue; 1318 continue;
1076 1319
1077 l = (void *)q; 1320 l = (void *)q;
1078 1321
1079 if (!crypto_is_test_l 1322 if (!crypto_is_test_larval(l))
1080 continue; 1323 continue;
1081 1324
1082 if (l->test_started) 1325 if (l->test_started)
1083 continue; 1326 continue;
1084 1327
1085 l->test_started = tru 1328 l->test_started = true;
1086 larval = l; 1329 larval = l;
1087 break; 1330 break;
1088 } 1331 }
1089 1332
1090 up_write(&crypto_alg_sem); 1333 up_write(&crypto_alg_sem);
1091 1334
1092 if (!larval) 1335 if (!larval)
1093 break; 1336 break;
1094 1337
1095 crypto_wait_for_test(larval); 1338 crypto_wait_for_test(larval);
1096 } 1339 }
1097 1340
1098 set_crypto_boot_test_finished(); !! 1341 static_branch_enable(&crypto_boot_test_finished);
1099 } 1342 }
1100 1343
1101 static int __init crypto_algapi_init(void) 1344 static int __init crypto_algapi_init(void)
1102 { 1345 {
1103 crypto_init_proc(); 1346 crypto_init_proc();
1104 crypto_start_tests(); 1347 crypto_start_tests();
1105 return 0; 1348 return 0;
1106 } 1349 }
1107 1350
1108 static void __exit crypto_algapi_exit(void) 1351 static void __exit crypto_algapi_exit(void)
1109 { 1352 {
1110 crypto_exit_proc(); 1353 crypto_exit_proc();
1111 } 1354 }
1112 1355
1113 /* 1356 /*
1114 * We run this at late_initcall so that all t 1357 * We run this at late_initcall so that all the built-in algorithms
1115 * have had a chance to register themselves f 1358 * have had a chance to register themselves first.
1116 */ 1359 */
1117 late_initcall(crypto_algapi_init); 1360 late_initcall(crypto_algapi_init);
1118 module_exit(crypto_algapi_exit); 1361 module_exit(crypto_algapi_exit);
1119 1362
1120 MODULE_LICENSE("GPL"); 1363 MODULE_LICENSE("GPL");
1121 MODULE_DESCRIPTION("Cryptographic algorithms 1364 MODULE_DESCRIPTION("Cryptographic algorithms API");
1122 MODULE_SOFTDEP("pre: cryptomgr"); 1365 MODULE_SOFTDEP("pre: cryptomgr");
1123 1366
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.