1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * AES using the RISC-V vector crypto extensions. Includes the bare block
4 * cipher and the ECB, CBC, CBC-CTS, CTR, and XTS modes.
5 *
6 * Copyright (C) 2023 VRULL GmbH
7 * Author: Heiko Stuebner <heiko.stuebner@vrull.eu>
8 *
9 * Copyright (C) 2023 SiFive, Inc.
10 * Author: Jerry Shih <jerry.shih@sifive.com>
11 *
12 * Copyright 2024 Google LLC
13 */
14
15 #include <asm/simd.h>
16 #include <asm/vector.h>
17 #include <crypto/aes.h>
18 #include <crypto/internal/cipher.h>
19 #include <crypto/internal/simd.h>
20 #include <crypto/internal/skcipher.h>
21 #include <crypto/scatterwalk.h>
22 #include <crypto/xts.h>
23 #include <linux/linkage.h>
24 #include <linux/module.h>
25
26 asmlinkage void aes_encrypt_zvkned(const struct crypto_aes_ctx *key,
27 const u8 in[AES_BLOCK_SIZE],
28 u8 out[AES_BLOCK_SIZE]);
29 asmlinkage void aes_decrypt_zvkned(const struct crypto_aes_ctx *key,
30 const u8 in[AES_BLOCK_SIZE],
31 u8 out[AES_BLOCK_SIZE]);
32
33 asmlinkage void aes_ecb_encrypt_zvkned(const struct crypto_aes_ctx *key,
34 const u8 *in, u8 *out, size_t len);
35 asmlinkage void aes_ecb_decrypt_zvkned(const struct crypto_aes_ctx *key,
36 const u8 *in, u8 *out, size_t len);
37
38 asmlinkage void aes_cbc_encrypt_zvkned(const struct crypto_aes_ctx *key,
39 const u8 *in, u8 *out, size_t len,
40 u8 iv[AES_BLOCK_SIZE]);
41 asmlinkage void aes_cbc_decrypt_zvkned(const struct crypto_aes_ctx *key,
42 const u8 *in, u8 *out, size_t len,
43 u8 iv[AES_BLOCK_SIZE]);
44
45 asmlinkage void aes_cbc_cts_crypt_zvkned(const struct crypto_aes_ctx *key,
46 const u8 *in, u8 *out, size_t len,
47 const u8 iv[AES_BLOCK_SIZE], bool enc);
48
49 asmlinkage void aes_ctr32_crypt_zvkned_zvkb(const struct crypto_aes_ctx *key,
50 const u8 *in, u8 *out, size_t len,
51 u8 iv[AES_BLOCK_SIZE]);
52
53 asmlinkage void aes_xts_encrypt_zvkned_zvbb_zvkg(
54 const struct crypto_aes_ctx *key,
55 const u8 *in, u8 *out, size_t len,
56 u8 tweak[AES_BLOCK_SIZE]);
57
58 asmlinkage void aes_xts_decrypt_zvkned_zvbb_zvkg(
59 const struct crypto_aes_ctx *key,
60 const u8 *in, u8 *out, size_t len,
61 u8 tweak[AES_BLOCK_SIZE]);
62
63 static int riscv64_aes_setkey(struct crypto_aes_ctx *ctx,
64 const u8 *key, unsigned int keylen)
65 {
66 /*
67 * For now we just use the generic key expansion, for these reasons:
68 *
69 * - zvkned's key expansion instructions don't support AES-192.
70 * So, non-zvkned fallback code would be needed anyway.
71 *
72 * - Users of AES in Linux usually don't change keys frequently.
73 * So, key expansion isn't performance-critical.
74 *
75 * - For single-block AES exposed as a "cipher" algorithm, it's
76 * necessary to use struct crypto_aes_ctx and initialize its 'key_dec'
77 * field with the round keys for the Equivalent Inverse Cipher. This
78 * is because with "cipher", decryption can be requested from a
79 * context where the vector unit isn't usable, necessitating a
80 * fallback to aes_decrypt(). But, zvkned can only generate and use
81 * the normal round keys. Of course, it's preferable to not have
82 * special code just for "cipher", as e.g. XTS also uses a
83 * single-block AES encryption. It's simplest to just use
84 * struct crypto_aes_ctx and aes_expandkey() everywhere.
85 */
86 return aes_expandkey(ctx, key, keylen);
87 }
88
89 static int riscv64_aes_setkey_cipher(struct crypto_tfm *tfm,
90 const u8 *key, unsigned int keylen)
91 {
92 struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
93
94 return riscv64_aes_setkey(ctx, key, keylen);
95 }
96
97 static int riscv64_aes_setkey_skcipher(struct crypto_skcipher *tfm,
98 const u8 *key, unsigned int keylen)
99 {
100 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
101
102 return riscv64_aes_setkey(ctx, key, keylen);
103 }
104
105 /* Bare AES, without a mode of operation */
106
107 static void riscv64_aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
108 {
109 const struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
110
111 if (crypto_simd_usable()) {
112 kernel_vector_begin();
113 aes_encrypt_zvkned(ctx, src, dst);
114 kernel_vector_end();
115 } else {
116 aes_encrypt(ctx, dst, src);
117 }
118 }
119
120 static void riscv64_aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
121 {
122 const struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
123
124 if (crypto_simd_usable()) {
125 kernel_vector_begin();
126 aes_decrypt_zvkned(ctx, src, dst);
127 kernel_vector_end();
128 } else {
129 aes_decrypt(ctx, dst, src);
130 }
131 }
132
133 /* AES-ECB */
134
135 static inline int riscv64_aes_ecb_crypt(struct skcipher_request *req, bool enc)
136 {
137 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
138 const struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
139 struct skcipher_walk walk;
140 unsigned int nbytes;
141 int err;
142
143 err = skcipher_walk_virt(&walk, req, false);
144 while ((nbytes = walk.nbytes) != 0) {
145 kernel_vector_begin();
146 if (enc)
147 aes_ecb_encrypt_zvkned(ctx, walk.src.virt.addr,
148 walk.dst.virt.addr,
149 nbytes & ~(AES_BLOCK_SIZE - 1));
150 else
151 aes_ecb_decrypt_zvkned(ctx, walk.src.virt.addr,
152 walk.dst.virt.addr,
153 nbytes & ~(AES_BLOCK_SIZE - 1));
154 kernel_vector_end();
155 err = skcipher_walk_done(&walk, nbytes & (AES_BLOCK_SIZE - 1));
156 }
157
158 return err;
159 }
160
161 static int riscv64_aes_ecb_encrypt(struct skcipher_request *req)
162 {
163 return riscv64_aes_ecb_crypt(req, true);
164 }
165
166 static int riscv64_aes_ecb_decrypt(struct skcipher_request *req)
167 {
168 return riscv64_aes_ecb_crypt(req, false);
169 }
170
171 /* AES-CBC */
172
173 static int riscv64_aes_cbc_crypt(struct skcipher_request *req, bool enc)
174 {
175 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
176 const struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
177 struct skcipher_walk walk;
178 unsigned int nbytes;
179 int err;
180
181 err = skcipher_walk_virt(&walk, req, false);
182 while ((nbytes = walk.nbytes) != 0) {
183 kernel_vector_begin();
184 if (enc)
185 aes_cbc_encrypt_zvkned(ctx, walk.src.virt.addr,
186 walk.dst.virt.addr,
187 nbytes & ~(AES_BLOCK_SIZE - 1),
188 walk.iv);
189 else
190 aes_cbc_decrypt_zvkned(ctx, walk.src.virt.addr,
191 walk.dst.virt.addr,
192 nbytes & ~(AES_BLOCK_SIZE - 1),
193 walk.iv);
194 kernel_vector_end();
195 err = skcipher_walk_done(&walk, nbytes & (AES_BLOCK_SIZE - 1));
196 }
197
198 return err;
199 }
200
201 static int riscv64_aes_cbc_encrypt(struct skcipher_request *req)
202 {
203 return riscv64_aes_cbc_crypt(req, true);
204 }
205
206 static int riscv64_aes_cbc_decrypt(struct skcipher_request *req)
207 {
208 return riscv64_aes_cbc_crypt(req, false);
209 }
210
211 /* AES-CBC-CTS */
212
213 static int riscv64_aes_cbc_cts_crypt(struct skcipher_request *req, bool enc)
214 {
215 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
216 const struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
217 struct scatterlist sg_src[2], sg_dst[2];
218 struct skcipher_request subreq;
219 struct scatterlist *src, *dst;
220 struct skcipher_walk walk;
221 unsigned int cbc_len;
222 int err;
223
224 if (req->cryptlen < AES_BLOCK_SIZE)
225 return -EINVAL;
226
227 err = skcipher_walk_virt(&walk, req, false);
228 if (err)
229 return err;
230 /*
231 * If the full message is available in one step, decrypt it in one call
232 * to the CBC-CTS assembly function. This reduces overhead, especially
233 * on short messages. Otherwise, fall back to doing CBC up to the last
234 * two blocks, then invoke CTS just for the ciphertext stealing.
235 */
236 if (unlikely(walk.nbytes != req->cryptlen)) {
237 cbc_len = round_down(req->cryptlen - AES_BLOCK_SIZE - 1,
238 AES_BLOCK_SIZE);
239 skcipher_walk_abort(&walk);
240 skcipher_request_set_tfm(&subreq, tfm);
241 skcipher_request_set_callback(&subreq,
242 skcipher_request_flags(req),
243 NULL, NULL);
244 skcipher_request_set_crypt(&subreq, req->src, req->dst,
245 cbc_len, req->iv);
246 err = riscv64_aes_cbc_crypt(&subreq, enc);
247 if (err)
248 return err;
249 dst = src = scatterwalk_ffwd(sg_src, req->src, cbc_len);
250 if (req->dst != req->src)
251 dst = scatterwalk_ffwd(sg_dst, req->dst, cbc_len);
252 skcipher_request_set_crypt(&subreq, src, dst,
253 req->cryptlen - cbc_len, req->iv);
254 err = skcipher_walk_virt(&walk, &subreq, false);
255 if (err)
256 return err;
257 }
258 kernel_vector_begin();
259 aes_cbc_cts_crypt_zvkned(ctx, walk.src.virt.addr, walk.dst.virt.addr,
260 walk.nbytes, req->iv, enc);
261 kernel_vector_end();
262 return skcipher_walk_done(&walk, 0);
263 }
264
265 static int riscv64_aes_cbc_cts_encrypt(struct skcipher_request *req)
266 {
267 return riscv64_aes_cbc_cts_crypt(req, true);
268 }
269
270 static int riscv64_aes_cbc_cts_decrypt(struct skcipher_request *req)
271 {
272 return riscv64_aes_cbc_cts_crypt(req, false);
273 }
274
275 /* AES-CTR */
276
277 static int riscv64_aes_ctr_crypt(struct skcipher_request *req)
278 {
279 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
280 const struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
281 unsigned int nbytes, p1_nbytes;
282 struct skcipher_walk walk;
283 u32 ctr32, nblocks;
284 int err;
285
286 /* Get the low 32-bit word of the 128-bit big endian counter. */
287 ctr32 = get_unaligned_be32(req->iv + 12);
288
289 err = skcipher_walk_virt(&walk, req, false);
290 while ((nbytes = walk.nbytes) != 0) {
291 if (nbytes < walk.total) {
292 /* Not the end yet, so keep the length block-aligned. */
293 nbytes = round_down(nbytes, AES_BLOCK_SIZE);
294 nblocks = nbytes / AES_BLOCK_SIZE;
295 } else {
296 /* It's the end, so include any final partial block. */
297 nblocks = DIV_ROUND_UP(nbytes, AES_BLOCK_SIZE);
298 }
299 ctr32 += nblocks;
300
301 kernel_vector_begin();
302 if (ctr32 >= nblocks) {
303 /* The low 32-bit word of the counter won't overflow. */
304 aes_ctr32_crypt_zvkned_zvkb(ctx, walk.src.virt.addr,
305 walk.dst.virt.addr, nbytes,
306 req->iv);
307 } else {
308 /*
309 * The low 32-bit word of the counter will overflow.
310 * The assembly doesn't handle this case, so split the
311 * operation into two at the point where the overflow
312 * will occur. After the first part, add the carry bit.
313 */
314 p1_nbytes = min_t(unsigned int, nbytes,
315 (nblocks - ctr32) * AES_BLOCK_SIZE);
316 aes_ctr32_crypt_zvkned_zvkb(ctx, walk.src.virt.addr,
317 walk.dst.virt.addr,
318 p1_nbytes, req->iv);
319 crypto_inc(req->iv, 12);
320
321 if (ctr32) {
322 aes_ctr32_crypt_zvkned_zvkb(
323 ctx,
324 walk.src.virt.addr + p1_nbytes,
325 walk.dst.virt.addr + p1_nbytes,
326 nbytes - p1_nbytes, req->iv);
327 }
328 }
329 kernel_vector_end();
330
331 err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
332 }
333
334 return err;
335 }
336
337 /* AES-XTS */
338
339 struct riscv64_aes_xts_ctx {
340 struct crypto_aes_ctx ctx1;
341 struct crypto_aes_ctx ctx2;
342 };
343
344 static int riscv64_aes_xts_setkey(struct crypto_skcipher *tfm, const u8 *key,
345 unsigned int keylen)
346 {
347 struct riscv64_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
348
349 return xts_verify_key(tfm, key, keylen) ?:
350 riscv64_aes_setkey(&ctx->ctx1, key, keylen / 2) ?:
351 riscv64_aes_setkey(&ctx->ctx2, key + keylen / 2, keylen / 2);
352 }
353
354 static int riscv64_aes_xts_crypt(struct skcipher_request *req, bool enc)
355 {
356 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
357 const struct riscv64_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
358 int tail = req->cryptlen % AES_BLOCK_SIZE;
359 struct scatterlist sg_src[2], sg_dst[2];
360 struct skcipher_request subreq;
361 struct scatterlist *src, *dst;
362 struct skcipher_walk walk;
363 int err;
364
365 if (req->cryptlen < AES_BLOCK_SIZE)
366 return -EINVAL;
367
368 /* Encrypt the IV with the tweak key to get the first tweak. */
369 kernel_vector_begin();
370 aes_encrypt_zvkned(&ctx->ctx2, req->iv, req->iv);
371 kernel_vector_end();
372
373 err = skcipher_walk_virt(&walk, req, false);
374
375 /*
376 * If the message length isn't divisible by the AES block size and the
377 * full message isn't available in one step of the scatterlist walk,
378 * then separate off the last full block and the partial block. This
379 * ensures that they are processed in the same call to the assembly
380 * function, which is required for ciphertext stealing.
381 */
382 if (unlikely(tail > 0 && walk.nbytes < walk.total)) {
383 skcipher_walk_abort(&walk);
384
385 skcipher_request_set_tfm(&subreq, tfm);
386 skcipher_request_set_callback(&subreq,
387 skcipher_request_flags(req),
388 NULL, NULL);
389 skcipher_request_set_crypt(&subreq, req->src, req->dst,
390 req->cryptlen - tail - AES_BLOCK_SIZE,
391 req->iv);
392 req = &subreq;
393 err = skcipher_walk_virt(&walk, req, false);
394 } else {
395 tail = 0;
396 }
397
398 while (walk.nbytes) {
399 unsigned int nbytes = walk.nbytes;
400
401 if (nbytes < walk.total)
402 nbytes = round_down(nbytes, AES_BLOCK_SIZE);
403
404 kernel_vector_begin();
405 if (enc)
406 aes_xts_encrypt_zvkned_zvbb_zvkg(
407 &ctx->ctx1, walk.src.virt.addr,
408 walk.dst.virt.addr, nbytes, req->iv);
409 else
410 aes_xts_decrypt_zvkned_zvbb_zvkg(
411 &ctx->ctx1, walk.src.virt.addr,
412 walk.dst.virt.addr, nbytes, req->iv);
413 kernel_vector_end();
414 err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
415 }
416
417 if (err || likely(!tail))
418 return err;
419
420 /* Do ciphertext stealing with the last full block and partial block. */
421
422 dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen);
423 if (req->dst != req->src)
424 dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen);
425
426 skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail,
427 req->iv);
428
429 err = skcipher_walk_virt(&walk, req, false);
430 if (err)
431 return err;
432
433 kernel_vector_begin();
434 if (enc)
435 aes_xts_encrypt_zvkned_zvbb_zvkg(
436 &ctx->ctx1, walk.src.virt.addr,
437 walk.dst.virt.addr, walk.nbytes, req->iv);
438 else
439 aes_xts_decrypt_zvkned_zvbb_zvkg(
440 &ctx->ctx1, walk.src.virt.addr,
441 walk.dst.virt.addr, walk.nbytes, req->iv);
442 kernel_vector_end();
443
444 return skcipher_walk_done(&walk, 0);
445 }
446
447 static int riscv64_aes_xts_encrypt(struct skcipher_request *req)
448 {
449 return riscv64_aes_xts_crypt(req, true);
450 }
451
452 static int riscv64_aes_xts_decrypt(struct skcipher_request *req)
453 {
454 return riscv64_aes_xts_crypt(req, false);
455 }
456
457 /* Algorithm definitions */
458
459 static struct crypto_alg riscv64_zvkned_aes_cipher_alg = {
460 .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
461 .cra_blocksize = AES_BLOCK_SIZE,
462 .cra_ctxsize = sizeof(struct crypto_aes_ctx),
463 .cra_priority = 300,
464 .cra_name = "aes",
465 .cra_driver_name = "aes-riscv64-zvkned",
466 .cra_cipher = {
467 .cia_min_keysize = AES_MIN_KEY_SIZE,
468 .cia_max_keysize = AES_MAX_KEY_SIZE,
469 .cia_setkey = riscv64_aes_setkey_cipher,
470 .cia_encrypt = riscv64_aes_encrypt,
471 .cia_decrypt = riscv64_aes_decrypt,
472 },
473 .cra_module = THIS_MODULE,
474 };
475
476 static struct skcipher_alg riscv64_zvkned_aes_skcipher_algs[] = {
477 {
478 .setkey = riscv64_aes_setkey_skcipher,
479 .encrypt = riscv64_aes_ecb_encrypt,
480 .decrypt = riscv64_aes_ecb_decrypt,
481 .min_keysize = AES_MIN_KEY_SIZE,
482 .max_keysize = AES_MAX_KEY_SIZE,
483 .walksize = 8 * AES_BLOCK_SIZE, /* matches LMUL=8 */
484 .base = {
485 .cra_blocksize = AES_BLOCK_SIZE,
486 .cra_ctxsize = sizeof(struct crypto_aes_ctx),
487 .cra_priority = 300,
488 .cra_name = "ecb(aes)",
489 .cra_driver_name = "ecb-aes-riscv64-zvkned",
490 .cra_module = THIS_MODULE,
491 },
492 }, {
493 .setkey = riscv64_aes_setkey_skcipher,
494 .encrypt = riscv64_aes_cbc_encrypt,
495 .decrypt = riscv64_aes_cbc_decrypt,
496 .min_keysize = AES_MIN_KEY_SIZE,
497 .max_keysize = AES_MAX_KEY_SIZE,
498 .ivsize = AES_BLOCK_SIZE,
499 .base = {
500 .cra_blocksize = AES_BLOCK_SIZE,
501 .cra_ctxsize = sizeof(struct crypto_aes_ctx),
502 .cra_priority = 300,
503 .cra_name = "cbc(aes)",
504 .cra_driver_name = "cbc-aes-riscv64-zvkned",
505 .cra_module = THIS_MODULE,
506 },
507 }, {
508 .setkey = riscv64_aes_setkey_skcipher,
509 .encrypt = riscv64_aes_cbc_cts_encrypt,
510 .decrypt = riscv64_aes_cbc_cts_decrypt,
511 .min_keysize = AES_MIN_KEY_SIZE,
512 .max_keysize = AES_MAX_KEY_SIZE,
513 .ivsize = AES_BLOCK_SIZE,
514 .walksize = 4 * AES_BLOCK_SIZE, /* matches LMUL=4 */
515 .base = {
516 .cra_blocksize = AES_BLOCK_SIZE,
517 .cra_ctxsize = sizeof(struct crypto_aes_ctx),
518 .cra_priority = 300,
519 .cra_name = "cts(cbc(aes))",
520 .cra_driver_name = "cts-cbc-aes-riscv64-zvkned",
521 .cra_module = THIS_MODULE,
522 },
523 }
524 };
525
526 static struct skcipher_alg riscv64_zvkned_zvkb_aes_skcipher_alg = {
527 .setkey = riscv64_aes_setkey_skcipher,
528 .encrypt = riscv64_aes_ctr_crypt,
529 .decrypt = riscv64_aes_ctr_crypt,
530 .min_keysize = AES_MIN_KEY_SIZE,
531 .max_keysize = AES_MAX_KEY_SIZE,
532 .ivsize = AES_BLOCK_SIZE,
533 .chunksize = AES_BLOCK_SIZE,
534 .walksize = 4 * AES_BLOCK_SIZE, /* matches LMUL=4 */
535 .base = {
536 .cra_blocksize = 1,
537 .cra_ctxsize = sizeof(struct crypto_aes_ctx),
538 .cra_priority = 300,
539 .cra_name = "ctr(aes)",
540 .cra_driver_name = "ctr-aes-riscv64-zvkned-zvkb",
541 .cra_module = THIS_MODULE,
542 },
543 };
544
545 static struct skcipher_alg riscv64_zvkned_zvbb_zvkg_aes_skcipher_alg = {
546 .setkey = riscv64_aes_xts_setkey,
547 .encrypt = riscv64_aes_xts_encrypt,
548 .decrypt = riscv64_aes_xts_decrypt,
549 .min_keysize = 2 * AES_MIN_KEY_SIZE,
550 .max_keysize = 2 * AES_MAX_KEY_SIZE,
551 .ivsize = AES_BLOCK_SIZE,
552 .chunksize = AES_BLOCK_SIZE,
553 .walksize = 4 * AES_BLOCK_SIZE, /* matches LMUL=4 */
554 .base = {
555 .cra_blocksize = AES_BLOCK_SIZE,
556 .cra_ctxsize = sizeof(struct riscv64_aes_xts_ctx),
557 .cra_priority = 300,
558 .cra_name = "xts(aes)",
559 .cra_driver_name = "xts-aes-riscv64-zvkned-zvbb-zvkg",
560 .cra_module = THIS_MODULE,
561 },
562 };
563
564 static inline bool riscv64_aes_xts_supported(void)
565 {
566 return riscv_isa_extension_available(NULL, ZVBB) &&
567 riscv_isa_extension_available(NULL, ZVKG) &&
568 riscv_vector_vlen() < 2048 /* Implementation limitation */;
569 }
570
571 static int __init riscv64_aes_mod_init(void)
572 {
573 int err = -ENODEV;
574
575 if (riscv_isa_extension_available(NULL, ZVKNED) &&
576 riscv_vector_vlen() >= 128) {
577 err = crypto_register_alg(&riscv64_zvkned_aes_cipher_alg);
578 if (err)
579 return err;
580
581 err = crypto_register_skciphers(
582 riscv64_zvkned_aes_skcipher_algs,
583 ARRAY_SIZE(riscv64_zvkned_aes_skcipher_algs));
584 if (err)
585 goto unregister_zvkned_cipher_alg;
586
587 if (riscv_isa_extension_available(NULL, ZVKB)) {
588 err = crypto_register_skcipher(
589 &riscv64_zvkned_zvkb_aes_skcipher_alg);
590 if (err)
591 goto unregister_zvkned_skcipher_algs;
592 }
593
594 if (riscv64_aes_xts_supported()) {
595 err = crypto_register_skcipher(
596 &riscv64_zvkned_zvbb_zvkg_aes_skcipher_alg);
597 if (err)
598 goto unregister_zvkned_zvkb_skcipher_alg;
599 }
600 }
601
602 return err;
603
604 unregister_zvkned_zvkb_skcipher_alg:
605 if (riscv_isa_extension_available(NULL, ZVKB))
606 crypto_unregister_skcipher(&riscv64_zvkned_zvkb_aes_skcipher_alg);
607 unregister_zvkned_skcipher_algs:
608 crypto_unregister_skciphers(riscv64_zvkned_aes_skcipher_algs,
609 ARRAY_SIZE(riscv64_zvkned_aes_skcipher_algs));
610 unregister_zvkned_cipher_alg:
611 crypto_unregister_alg(&riscv64_zvkned_aes_cipher_alg);
612 return err;
613 }
614
615 static void __exit riscv64_aes_mod_exit(void)
616 {
617 if (riscv64_aes_xts_supported())
618 crypto_unregister_skcipher(&riscv64_zvkned_zvbb_zvkg_aes_skcipher_alg);
619 if (riscv_isa_extension_available(NULL, ZVKB))
620 crypto_unregister_skcipher(&riscv64_zvkned_zvkb_aes_skcipher_alg);
621 crypto_unregister_skciphers(riscv64_zvkned_aes_skcipher_algs,
622 ARRAY_SIZE(riscv64_zvkned_aes_skcipher_algs));
623 crypto_unregister_alg(&riscv64_zvkned_aes_cipher_alg);
624 }
625
626 module_init(riscv64_aes_mod_init);
627 module_exit(riscv64_aes_mod_exit);
628
629 MODULE_DESCRIPTION("AES-ECB/CBC/CTS/CTR/XTS (RISC-V accelerated)");
630 MODULE_AUTHOR("Jerry Shih <jerry.shih@sifive.com>");
631 MODULE_LICENSE("GPL");
632 MODULE_ALIAS_CRYPTO("aes");
633 MODULE_ALIAS_CRYPTO("ecb(aes)");
634 MODULE_ALIAS_CRYPTO("cbc(aes)");
635 MODULE_ALIAS_CRYPTO("cts(cbc(aes))");
636 MODULE_ALIAS_CRYPTO("ctr(aes)");
637 MODULE_ALIAS_CRYPTO("xts(aes)");
638
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.