1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Cryptographic API.
4 *
5 * s390 implementation of the AES Cipher Algorithm with protected keys.
6 *
7 * s390 Version:
8 * Copyright IBM Corp. 2017, 2023
9 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
10 * Harald Freudenberger <freude@de.ibm.com>
11 */
12
13 #define KMSG_COMPONENT "paes_s390"
14 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
15
16 #include <crypto/aes.h>
17 #include <crypto/algapi.h>
18 #include <linux/bug.h>
19 #include <linux/err.h>
20 #include <linux/module.h>
21 #include <linux/cpufeature.h>
22 #include <linux/init.h>
23 #include <linux/mutex.h>
24 #include <linux/spinlock.h>
25 #include <linux/delay.h>
26 #include <crypto/internal/skcipher.h>
27 #include <crypto/xts.h>
28 #include <asm/cpacf.h>
29 #include <asm/pkey.h>
30
31 /*
32 * Key blobs smaller/bigger than these defines are rejected
33 * by the common code even before the individual setkey function
34 * is called. As paes can handle different kinds of key blobs
35 * and padding is also possible, the limits need to be generous.
36 */
37 #define PAES_MIN_KEYSIZE 16
38 #define PAES_MAX_KEYSIZE MAXEP11AESKEYBLOBSIZE
39
40 static u8 *ctrblk;
41 static DEFINE_MUTEX(ctrblk_lock);
42
43 static cpacf_mask_t km_functions, kmc_functions, kmctr_functions;
44
45 struct key_blob {
46 /*
47 * Small keys will be stored in the keybuf. Larger keys are
48 * stored in extra allocated memory. In both cases does
49 * key point to the memory where the key is stored.
50 * The code distinguishes by checking keylen against
51 * sizeof(keybuf). See the two following helper functions.
52 */
53 u8 *key;
54 u8 keybuf[128];
55 unsigned int keylen;
56 };
57
58 static inline int _key_to_kb(struct key_blob *kb,
59 const u8 *key,
60 unsigned int keylen)
61 {
62 struct clearkey_header {
63 u8 type;
64 u8 res0[3];
65 u8 version;
66 u8 res1[3];
67 u32 keytype;
68 u32 len;
69 } __packed * h;
70
71 switch (keylen) {
72 case 16:
73 case 24:
74 case 32:
75 /* clear key value, prepare pkey clear key token in keybuf */
76 memset(kb->keybuf, 0, sizeof(kb->keybuf));
77 h = (struct clearkey_header *) kb->keybuf;
78 h->version = 0x02; /* TOKVER_CLEAR_KEY */
79 h->keytype = (keylen - 8) >> 3;
80 h->len = keylen;
81 memcpy(kb->keybuf + sizeof(*h), key, keylen);
82 kb->keylen = sizeof(*h) + keylen;
83 kb->key = kb->keybuf;
84 break;
85 default:
86 /* other key material, let pkey handle this */
87 if (keylen <= sizeof(kb->keybuf))
88 kb->key = kb->keybuf;
89 else {
90 kb->key = kmalloc(keylen, GFP_KERNEL);
91 if (!kb->key)
92 return -ENOMEM;
93 }
94 memcpy(kb->key, key, keylen);
95 kb->keylen = keylen;
96 break;
97 }
98
99 return 0;
100 }
101
102 static inline void _free_kb_keybuf(struct key_blob *kb)
103 {
104 if (kb->key && kb->key != kb->keybuf
105 && kb->keylen > sizeof(kb->keybuf)) {
106 kfree_sensitive(kb->key);
107 kb->key = NULL;
108 }
109 }
110
111 struct s390_paes_ctx {
112 struct key_blob kb;
113 struct pkey_protkey pk;
114 spinlock_t pk_lock;
115 unsigned long fc;
116 };
117
118 struct s390_pxts_ctx {
119 struct key_blob kb[2];
120 struct pkey_protkey pk[2];
121 spinlock_t pk_lock;
122 unsigned long fc;
123 };
124
125 static inline int __paes_keyblob2pkey(struct key_blob *kb,
126 struct pkey_protkey *pk)
127 {
128 int i, ret = -EIO;
129
130 /* try three times in case of busy card */
131 for (i = 0; ret && i < 3; i++) {
132 if (ret == -EBUSY && in_task()) {
133 if (msleep_interruptible(1000))
134 return -EINTR;
135 }
136 ret = pkey_keyblob2pkey(kb->key, kb->keylen,
137 pk->protkey, &pk->len, &pk->type);
138 }
139
140 return ret;
141 }
142
143 static inline int __paes_convert_key(struct s390_paes_ctx *ctx)
144 {
145 int ret;
146 struct pkey_protkey pkey;
147
148 pkey.len = sizeof(pkey.protkey);
149 ret = __paes_keyblob2pkey(&ctx->kb, &pkey);
150 if (ret)
151 return ret;
152
153 spin_lock_bh(&ctx->pk_lock);
154 memcpy(&ctx->pk, &pkey, sizeof(pkey));
155 spin_unlock_bh(&ctx->pk_lock);
156
157 return 0;
158 }
159
160 static int ecb_paes_init(struct crypto_skcipher *tfm)
161 {
162 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm);
163
164 ctx->kb.key = NULL;
165 spin_lock_init(&ctx->pk_lock);
166
167 return 0;
168 }
169
170 static void ecb_paes_exit(struct crypto_skcipher *tfm)
171 {
172 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm);
173
174 _free_kb_keybuf(&ctx->kb);
175 }
176
177 static inline int __ecb_paes_set_key(struct s390_paes_ctx *ctx)
178 {
179 int rc;
180 unsigned long fc;
181
182 rc = __paes_convert_key(ctx);
183 if (rc)
184 return rc;
185
186 /* Pick the correct function code based on the protected key type */
187 fc = (ctx->pk.type == PKEY_KEYTYPE_AES_128) ? CPACF_KM_PAES_128 :
188 (ctx->pk.type == PKEY_KEYTYPE_AES_192) ? CPACF_KM_PAES_192 :
189 (ctx->pk.type == PKEY_KEYTYPE_AES_256) ? CPACF_KM_PAES_256 : 0;
190
191 /* Check if the function code is available */
192 ctx->fc = (fc && cpacf_test_func(&km_functions, fc)) ? fc : 0;
193
194 return ctx->fc ? 0 : -EINVAL;
195 }
196
197 static int ecb_paes_set_key(struct crypto_skcipher *tfm, const u8 *in_key,
198 unsigned int key_len)
199 {
200 int rc;
201 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm);
202
203 _free_kb_keybuf(&ctx->kb);
204 rc = _key_to_kb(&ctx->kb, in_key, key_len);
205 if (rc)
206 return rc;
207
208 return __ecb_paes_set_key(ctx);
209 }
210
211 static int ecb_paes_crypt(struct skcipher_request *req, unsigned long modifier)
212 {
213 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
214 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm);
215 struct skcipher_walk walk;
216 unsigned int nbytes, n, k;
217 int ret;
218 struct {
219 u8 key[MAXPROTKEYSIZE];
220 } param;
221
222 ret = skcipher_walk_virt(&walk, req, false);
223 if (ret)
224 return ret;
225
226 spin_lock_bh(&ctx->pk_lock);
227 memcpy(param.key, ctx->pk.protkey, MAXPROTKEYSIZE);
228 spin_unlock_bh(&ctx->pk_lock);
229
230 while ((nbytes = walk.nbytes) != 0) {
231 /* only use complete blocks */
232 n = nbytes & ~(AES_BLOCK_SIZE - 1);
233 k = cpacf_km(ctx->fc | modifier, ¶m,
234 walk.dst.virt.addr, walk.src.virt.addr, n);
235 if (k)
236 ret = skcipher_walk_done(&walk, nbytes - k);
237 if (k < n) {
238 if (__paes_convert_key(ctx))
239 return skcipher_walk_done(&walk, -EIO);
240 spin_lock_bh(&ctx->pk_lock);
241 memcpy(param.key, ctx->pk.protkey, MAXPROTKEYSIZE);
242 spin_unlock_bh(&ctx->pk_lock);
243 }
244 }
245 return ret;
246 }
247
248 static int ecb_paes_encrypt(struct skcipher_request *req)
249 {
250 return ecb_paes_crypt(req, 0);
251 }
252
253 static int ecb_paes_decrypt(struct skcipher_request *req)
254 {
255 return ecb_paes_crypt(req, CPACF_DECRYPT);
256 }
257
258 static struct skcipher_alg ecb_paes_alg = {
259 .base.cra_name = "ecb(paes)",
260 .base.cra_driver_name = "ecb-paes-s390",
261 .base.cra_priority = 401, /* combo: aes + ecb + 1 */
262 .base.cra_blocksize = AES_BLOCK_SIZE,
263 .base.cra_ctxsize = sizeof(struct s390_paes_ctx),
264 .base.cra_module = THIS_MODULE,
265 .base.cra_list = LIST_HEAD_INIT(ecb_paes_alg.base.cra_list),
266 .init = ecb_paes_init,
267 .exit = ecb_paes_exit,
268 .min_keysize = PAES_MIN_KEYSIZE,
269 .max_keysize = PAES_MAX_KEYSIZE,
270 .setkey = ecb_paes_set_key,
271 .encrypt = ecb_paes_encrypt,
272 .decrypt = ecb_paes_decrypt,
273 };
274
275 static int cbc_paes_init(struct crypto_skcipher *tfm)
276 {
277 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm);
278
279 ctx->kb.key = NULL;
280 spin_lock_init(&ctx->pk_lock);
281
282 return 0;
283 }
284
285 static void cbc_paes_exit(struct crypto_skcipher *tfm)
286 {
287 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm);
288
289 _free_kb_keybuf(&ctx->kb);
290 }
291
292 static inline int __cbc_paes_set_key(struct s390_paes_ctx *ctx)
293 {
294 int rc;
295 unsigned long fc;
296
297 rc = __paes_convert_key(ctx);
298 if (rc)
299 return rc;
300
301 /* Pick the correct function code based on the protected key type */
302 fc = (ctx->pk.type == PKEY_KEYTYPE_AES_128) ? CPACF_KMC_PAES_128 :
303 (ctx->pk.type == PKEY_KEYTYPE_AES_192) ? CPACF_KMC_PAES_192 :
304 (ctx->pk.type == PKEY_KEYTYPE_AES_256) ? CPACF_KMC_PAES_256 : 0;
305
306 /* Check if the function code is available */
307 ctx->fc = (fc && cpacf_test_func(&kmc_functions, fc)) ? fc : 0;
308
309 return ctx->fc ? 0 : -EINVAL;
310 }
311
312 static int cbc_paes_set_key(struct crypto_skcipher *tfm, const u8 *in_key,
313 unsigned int key_len)
314 {
315 int rc;
316 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm);
317
318 _free_kb_keybuf(&ctx->kb);
319 rc = _key_to_kb(&ctx->kb, in_key, key_len);
320 if (rc)
321 return rc;
322
323 return __cbc_paes_set_key(ctx);
324 }
325
326 static int cbc_paes_crypt(struct skcipher_request *req, unsigned long modifier)
327 {
328 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
329 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm);
330 struct skcipher_walk walk;
331 unsigned int nbytes, n, k;
332 int ret;
333 struct {
334 u8 iv[AES_BLOCK_SIZE];
335 u8 key[MAXPROTKEYSIZE];
336 } param;
337
338 ret = skcipher_walk_virt(&walk, req, false);
339 if (ret)
340 return ret;
341
342 memcpy(param.iv, walk.iv, AES_BLOCK_SIZE);
343 spin_lock_bh(&ctx->pk_lock);
344 memcpy(param.key, ctx->pk.protkey, MAXPROTKEYSIZE);
345 spin_unlock_bh(&ctx->pk_lock);
346
347 while ((nbytes = walk.nbytes) != 0) {
348 /* only use complete blocks */
349 n = nbytes & ~(AES_BLOCK_SIZE - 1);
350 k = cpacf_kmc(ctx->fc | modifier, ¶m,
351 walk.dst.virt.addr, walk.src.virt.addr, n);
352 if (k) {
353 memcpy(walk.iv, param.iv, AES_BLOCK_SIZE);
354 ret = skcipher_walk_done(&walk, nbytes - k);
355 }
356 if (k < n) {
357 if (__paes_convert_key(ctx))
358 return skcipher_walk_done(&walk, -EIO);
359 spin_lock_bh(&ctx->pk_lock);
360 memcpy(param.key, ctx->pk.protkey, MAXPROTKEYSIZE);
361 spin_unlock_bh(&ctx->pk_lock);
362 }
363 }
364 return ret;
365 }
366
367 static int cbc_paes_encrypt(struct skcipher_request *req)
368 {
369 return cbc_paes_crypt(req, 0);
370 }
371
372 static int cbc_paes_decrypt(struct skcipher_request *req)
373 {
374 return cbc_paes_crypt(req, CPACF_DECRYPT);
375 }
376
377 static struct skcipher_alg cbc_paes_alg = {
378 .base.cra_name = "cbc(paes)",
379 .base.cra_driver_name = "cbc-paes-s390",
380 .base.cra_priority = 402, /* ecb-paes-s390 + 1 */
381 .base.cra_blocksize = AES_BLOCK_SIZE,
382 .base.cra_ctxsize = sizeof(struct s390_paes_ctx),
383 .base.cra_module = THIS_MODULE,
384 .base.cra_list = LIST_HEAD_INIT(cbc_paes_alg.base.cra_list),
385 .init = cbc_paes_init,
386 .exit = cbc_paes_exit,
387 .min_keysize = PAES_MIN_KEYSIZE,
388 .max_keysize = PAES_MAX_KEYSIZE,
389 .ivsize = AES_BLOCK_SIZE,
390 .setkey = cbc_paes_set_key,
391 .encrypt = cbc_paes_encrypt,
392 .decrypt = cbc_paes_decrypt,
393 };
394
395 static int xts_paes_init(struct crypto_skcipher *tfm)
396 {
397 struct s390_pxts_ctx *ctx = crypto_skcipher_ctx(tfm);
398
399 ctx->kb[0].key = NULL;
400 ctx->kb[1].key = NULL;
401 spin_lock_init(&ctx->pk_lock);
402
403 return 0;
404 }
405
406 static void xts_paes_exit(struct crypto_skcipher *tfm)
407 {
408 struct s390_pxts_ctx *ctx = crypto_skcipher_ctx(tfm);
409
410 _free_kb_keybuf(&ctx->kb[0]);
411 _free_kb_keybuf(&ctx->kb[1]);
412 }
413
414 static inline int __xts_paes_convert_key(struct s390_pxts_ctx *ctx)
415 {
416 struct pkey_protkey pkey0, pkey1;
417
418 pkey0.len = sizeof(pkey0.protkey);
419 pkey1.len = sizeof(pkey1.protkey);
420
421 if (__paes_keyblob2pkey(&ctx->kb[0], &pkey0) ||
422 __paes_keyblob2pkey(&ctx->kb[1], &pkey1))
423 return -EINVAL;
424
425 spin_lock_bh(&ctx->pk_lock);
426 memcpy(&ctx->pk[0], &pkey0, sizeof(pkey0));
427 memcpy(&ctx->pk[1], &pkey1, sizeof(pkey1));
428 spin_unlock_bh(&ctx->pk_lock);
429
430 return 0;
431 }
432
433 static inline int __xts_paes_set_key(struct s390_pxts_ctx *ctx)
434 {
435 unsigned long fc;
436
437 if (__xts_paes_convert_key(ctx))
438 return -EINVAL;
439
440 if (ctx->pk[0].type != ctx->pk[1].type)
441 return -EINVAL;
442
443 /* Pick the correct function code based on the protected key type */
444 fc = (ctx->pk[0].type == PKEY_KEYTYPE_AES_128) ? CPACF_KM_PXTS_128 :
445 (ctx->pk[0].type == PKEY_KEYTYPE_AES_256) ?
446 CPACF_KM_PXTS_256 : 0;
447
448 /* Check if the function code is available */
449 ctx->fc = (fc && cpacf_test_func(&km_functions, fc)) ? fc : 0;
450
451 return ctx->fc ? 0 : -EINVAL;
452 }
453
454 static int xts_paes_set_key(struct crypto_skcipher *tfm, const u8 *in_key,
455 unsigned int xts_key_len)
456 {
457 int rc;
458 struct s390_pxts_ctx *ctx = crypto_skcipher_ctx(tfm);
459 u8 ckey[2 * AES_MAX_KEY_SIZE];
460 unsigned int ckey_len, key_len;
461
462 if (xts_key_len % 2)
463 return -EINVAL;
464
465 key_len = xts_key_len / 2;
466
467 _free_kb_keybuf(&ctx->kb[0]);
468 _free_kb_keybuf(&ctx->kb[1]);
469 rc = _key_to_kb(&ctx->kb[0], in_key, key_len);
470 if (rc)
471 return rc;
472 rc = _key_to_kb(&ctx->kb[1], in_key + key_len, key_len);
473 if (rc)
474 return rc;
475
476 rc = __xts_paes_set_key(ctx);
477 if (rc)
478 return rc;
479
480 /*
481 * xts_verify_key verifies the key length is not odd and makes
482 * sure that the two keys are not the same. This can be done
483 * on the two protected keys as well
484 */
485 ckey_len = (ctx->pk[0].type == PKEY_KEYTYPE_AES_128) ?
486 AES_KEYSIZE_128 : AES_KEYSIZE_256;
487 memcpy(ckey, ctx->pk[0].protkey, ckey_len);
488 memcpy(ckey + ckey_len, ctx->pk[1].protkey, ckey_len);
489 return xts_verify_key(tfm, ckey, 2*ckey_len);
490 }
491
492 static int xts_paes_crypt(struct skcipher_request *req, unsigned long modifier)
493 {
494 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
495 struct s390_pxts_ctx *ctx = crypto_skcipher_ctx(tfm);
496 struct skcipher_walk walk;
497 unsigned int keylen, offset, nbytes, n, k;
498 int ret;
499 struct {
500 u8 key[MAXPROTKEYSIZE]; /* key + verification pattern */
501 u8 tweak[16];
502 u8 block[16];
503 u8 bit[16];
504 u8 xts[16];
505 } pcc_param;
506 struct {
507 u8 key[MAXPROTKEYSIZE]; /* key + verification pattern */
508 u8 init[16];
509 } xts_param;
510
511 ret = skcipher_walk_virt(&walk, req, false);
512 if (ret)
513 return ret;
514
515 keylen = (ctx->pk[0].type == PKEY_KEYTYPE_AES_128) ? 48 : 64;
516 offset = (ctx->pk[0].type == PKEY_KEYTYPE_AES_128) ? 16 : 0;
517
518 memset(&pcc_param, 0, sizeof(pcc_param));
519 memcpy(pcc_param.tweak, walk.iv, sizeof(pcc_param.tweak));
520 spin_lock_bh(&ctx->pk_lock);
521 memcpy(pcc_param.key + offset, ctx->pk[1].protkey, keylen);
522 memcpy(xts_param.key + offset, ctx->pk[0].protkey, keylen);
523 spin_unlock_bh(&ctx->pk_lock);
524 cpacf_pcc(ctx->fc, pcc_param.key + offset);
525 memcpy(xts_param.init, pcc_param.xts, 16);
526
527 while ((nbytes = walk.nbytes) != 0) {
528 /* only use complete blocks */
529 n = nbytes & ~(AES_BLOCK_SIZE - 1);
530 k = cpacf_km(ctx->fc | modifier, xts_param.key + offset,
531 walk.dst.virt.addr, walk.src.virt.addr, n);
532 if (k)
533 ret = skcipher_walk_done(&walk, nbytes - k);
534 if (k < n) {
535 if (__xts_paes_convert_key(ctx))
536 return skcipher_walk_done(&walk, -EIO);
537 spin_lock_bh(&ctx->pk_lock);
538 memcpy(xts_param.key + offset,
539 ctx->pk[0].protkey, keylen);
540 spin_unlock_bh(&ctx->pk_lock);
541 }
542 }
543
544 return ret;
545 }
546
547 static int xts_paes_encrypt(struct skcipher_request *req)
548 {
549 return xts_paes_crypt(req, 0);
550 }
551
552 static int xts_paes_decrypt(struct skcipher_request *req)
553 {
554 return xts_paes_crypt(req, CPACF_DECRYPT);
555 }
556
557 static struct skcipher_alg xts_paes_alg = {
558 .base.cra_name = "xts(paes)",
559 .base.cra_driver_name = "xts-paes-s390",
560 .base.cra_priority = 402, /* ecb-paes-s390 + 1 */
561 .base.cra_blocksize = AES_BLOCK_SIZE,
562 .base.cra_ctxsize = sizeof(struct s390_pxts_ctx),
563 .base.cra_module = THIS_MODULE,
564 .base.cra_list = LIST_HEAD_INIT(xts_paes_alg.base.cra_list),
565 .init = xts_paes_init,
566 .exit = xts_paes_exit,
567 .min_keysize = 2 * PAES_MIN_KEYSIZE,
568 .max_keysize = 2 * PAES_MAX_KEYSIZE,
569 .ivsize = AES_BLOCK_SIZE,
570 .setkey = xts_paes_set_key,
571 .encrypt = xts_paes_encrypt,
572 .decrypt = xts_paes_decrypt,
573 };
574
575 static int ctr_paes_init(struct crypto_skcipher *tfm)
576 {
577 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm);
578
579 ctx->kb.key = NULL;
580 spin_lock_init(&ctx->pk_lock);
581
582 return 0;
583 }
584
585 static void ctr_paes_exit(struct crypto_skcipher *tfm)
586 {
587 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm);
588
589 _free_kb_keybuf(&ctx->kb);
590 }
591
592 static inline int __ctr_paes_set_key(struct s390_paes_ctx *ctx)
593 {
594 int rc;
595 unsigned long fc;
596
597 rc = __paes_convert_key(ctx);
598 if (rc)
599 return rc;
600
601 /* Pick the correct function code based on the protected key type */
602 fc = (ctx->pk.type == PKEY_KEYTYPE_AES_128) ? CPACF_KMCTR_PAES_128 :
603 (ctx->pk.type == PKEY_KEYTYPE_AES_192) ? CPACF_KMCTR_PAES_192 :
604 (ctx->pk.type == PKEY_KEYTYPE_AES_256) ?
605 CPACF_KMCTR_PAES_256 : 0;
606
607 /* Check if the function code is available */
608 ctx->fc = (fc && cpacf_test_func(&kmctr_functions, fc)) ? fc : 0;
609
610 return ctx->fc ? 0 : -EINVAL;
611 }
612
613 static int ctr_paes_set_key(struct crypto_skcipher *tfm, const u8 *in_key,
614 unsigned int key_len)
615 {
616 int rc;
617 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm);
618
619 _free_kb_keybuf(&ctx->kb);
620 rc = _key_to_kb(&ctx->kb, in_key, key_len);
621 if (rc)
622 return rc;
623
624 return __ctr_paes_set_key(ctx);
625 }
626
627 static unsigned int __ctrblk_init(u8 *ctrptr, u8 *iv, unsigned int nbytes)
628 {
629 unsigned int i, n;
630
631 /* only use complete blocks, max. PAGE_SIZE */
632 memcpy(ctrptr, iv, AES_BLOCK_SIZE);
633 n = (nbytes > PAGE_SIZE) ? PAGE_SIZE : nbytes & ~(AES_BLOCK_SIZE - 1);
634 for (i = (n / AES_BLOCK_SIZE) - 1; i > 0; i--) {
635 memcpy(ctrptr + AES_BLOCK_SIZE, ctrptr, AES_BLOCK_SIZE);
636 crypto_inc(ctrptr + AES_BLOCK_SIZE, AES_BLOCK_SIZE);
637 ctrptr += AES_BLOCK_SIZE;
638 }
639 return n;
640 }
641
642 static int ctr_paes_crypt(struct skcipher_request *req)
643 {
644 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
645 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm);
646 u8 buf[AES_BLOCK_SIZE], *ctrptr;
647 struct skcipher_walk walk;
648 unsigned int nbytes, n, k;
649 int ret, locked;
650 struct {
651 u8 key[MAXPROTKEYSIZE];
652 } param;
653
654 ret = skcipher_walk_virt(&walk, req, false);
655 if (ret)
656 return ret;
657
658 spin_lock_bh(&ctx->pk_lock);
659 memcpy(param.key, ctx->pk.protkey, MAXPROTKEYSIZE);
660 spin_unlock_bh(&ctx->pk_lock);
661
662 locked = mutex_trylock(&ctrblk_lock);
663
664 while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
665 n = AES_BLOCK_SIZE;
666 if (nbytes >= 2*AES_BLOCK_SIZE && locked)
667 n = __ctrblk_init(ctrblk, walk.iv, nbytes);
668 ctrptr = (n > AES_BLOCK_SIZE) ? ctrblk : walk.iv;
669 k = cpacf_kmctr(ctx->fc, ¶m, walk.dst.virt.addr,
670 walk.src.virt.addr, n, ctrptr);
671 if (k) {
672 if (ctrptr == ctrblk)
673 memcpy(walk.iv, ctrptr + k - AES_BLOCK_SIZE,
674 AES_BLOCK_SIZE);
675 crypto_inc(walk.iv, AES_BLOCK_SIZE);
676 ret = skcipher_walk_done(&walk, nbytes - k);
677 }
678 if (k < n) {
679 if (__paes_convert_key(ctx)) {
680 if (locked)
681 mutex_unlock(&ctrblk_lock);
682 return skcipher_walk_done(&walk, -EIO);
683 }
684 spin_lock_bh(&ctx->pk_lock);
685 memcpy(param.key, ctx->pk.protkey, MAXPROTKEYSIZE);
686 spin_unlock_bh(&ctx->pk_lock);
687 }
688 }
689 if (locked)
690 mutex_unlock(&ctrblk_lock);
691 /*
692 * final block may be < AES_BLOCK_SIZE, copy only nbytes
693 */
694 if (nbytes) {
695 memset(buf, 0, AES_BLOCK_SIZE);
696 memcpy(buf, walk.src.virt.addr, nbytes);
697 while (1) {
698 if (cpacf_kmctr(ctx->fc, ¶m, buf,
699 buf, AES_BLOCK_SIZE,
700 walk.iv) == AES_BLOCK_SIZE)
701 break;
702 if (__paes_convert_key(ctx))
703 return skcipher_walk_done(&walk, -EIO);
704 spin_lock_bh(&ctx->pk_lock);
705 memcpy(param.key, ctx->pk.protkey, MAXPROTKEYSIZE);
706 spin_unlock_bh(&ctx->pk_lock);
707 }
708 memcpy(walk.dst.virt.addr, buf, nbytes);
709 crypto_inc(walk.iv, AES_BLOCK_SIZE);
710 ret = skcipher_walk_done(&walk, nbytes);
711 }
712
713 return ret;
714 }
715
716 static struct skcipher_alg ctr_paes_alg = {
717 .base.cra_name = "ctr(paes)",
718 .base.cra_driver_name = "ctr-paes-s390",
719 .base.cra_priority = 402, /* ecb-paes-s390 + 1 */
720 .base.cra_blocksize = 1,
721 .base.cra_ctxsize = sizeof(struct s390_paes_ctx),
722 .base.cra_module = THIS_MODULE,
723 .base.cra_list = LIST_HEAD_INIT(ctr_paes_alg.base.cra_list),
724 .init = ctr_paes_init,
725 .exit = ctr_paes_exit,
726 .min_keysize = PAES_MIN_KEYSIZE,
727 .max_keysize = PAES_MAX_KEYSIZE,
728 .ivsize = AES_BLOCK_SIZE,
729 .setkey = ctr_paes_set_key,
730 .encrypt = ctr_paes_crypt,
731 .decrypt = ctr_paes_crypt,
732 .chunksize = AES_BLOCK_SIZE,
733 };
734
735 static inline void __crypto_unregister_skcipher(struct skcipher_alg *alg)
736 {
737 if (!list_empty(&alg->base.cra_list))
738 crypto_unregister_skcipher(alg);
739 }
740
741 static void paes_s390_fini(void)
742 {
743 __crypto_unregister_skcipher(&ctr_paes_alg);
744 __crypto_unregister_skcipher(&xts_paes_alg);
745 __crypto_unregister_skcipher(&cbc_paes_alg);
746 __crypto_unregister_skcipher(&ecb_paes_alg);
747 if (ctrblk)
748 free_page((unsigned long) ctrblk);
749 }
750
751 static int __init paes_s390_init(void)
752 {
753 int ret;
754
755 /* Query available functions for KM, KMC and KMCTR */
756 cpacf_query(CPACF_KM, &km_functions);
757 cpacf_query(CPACF_KMC, &kmc_functions);
758 cpacf_query(CPACF_KMCTR, &kmctr_functions);
759
760 if (cpacf_test_func(&km_functions, CPACF_KM_PAES_128) ||
761 cpacf_test_func(&km_functions, CPACF_KM_PAES_192) ||
762 cpacf_test_func(&km_functions, CPACF_KM_PAES_256)) {
763 ret = crypto_register_skcipher(&ecb_paes_alg);
764 if (ret)
765 goto out_err;
766 }
767
768 if (cpacf_test_func(&kmc_functions, CPACF_KMC_PAES_128) ||
769 cpacf_test_func(&kmc_functions, CPACF_KMC_PAES_192) ||
770 cpacf_test_func(&kmc_functions, CPACF_KMC_PAES_256)) {
771 ret = crypto_register_skcipher(&cbc_paes_alg);
772 if (ret)
773 goto out_err;
774 }
775
776 if (cpacf_test_func(&km_functions, CPACF_KM_PXTS_128) ||
777 cpacf_test_func(&km_functions, CPACF_KM_PXTS_256)) {
778 ret = crypto_register_skcipher(&xts_paes_alg);
779 if (ret)
780 goto out_err;
781 }
782
783 if (cpacf_test_func(&kmctr_functions, CPACF_KMCTR_PAES_128) ||
784 cpacf_test_func(&kmctr_functions, CPACF_KMCTR_PAES_192) ||
785 cpacf_test_func(&kmctr_functions, CPACF_KMCTR_PAES_256)) {
786 ctrblk = (u8 *) __get_free_page(GFP_KERNEL);
787 if (!ctrblk) {
788 ret = -ENOMEM;
789 goto out_err;
790 }
791 ret = crypto_register_skcipher(&ctr_paes_alg);
792 if (ret)
793 goto out_err;
794 }
795
796 return 0;
797 out_err:
798 paes_s390_fini();
799 return ret;
800 }
801
802 module_init(paes_s390_init);
803 module_exit(paes_s390_fini);
804
805 MODULE_ALIAS_CRYPTO("paes");
806
807 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm with protected keys");
808 MODULE_LICENSE("GPL");
809
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.