1 // SPDX-License-Identifier: GPL-2.0-only 1
2 /*
3 * GHASH using the RISC-V vector crypto extens
4 *
5 * Copyright (C) 2023 VRULL GmbH
6 * Author: Heiko Stuebner <heiko.stuebner@vrul
7 *
8 * Copyright (C) 2023 SiFive, Inc.
9 * Author: Jerry Shih <jerry.shih@sifive.com>
10 */
11
12 #include <asm/simd.h>
13 #include <asm/vector.h>
14 #include <crypto/ghash.h>
15 #include <crypto/internal/hash.h>
16 #include <crypto/internal/simd.h>
17 #include <linux/linkage.h>
18 #include <linux/module.h>
19
20 asmlinkage void ghash_zvkg(be128 *accumulator,
21 size_t len);
22
23 struct riscv64_ghash_tfm_ctx {
24 be128 key;
25 };
26
27 struct riscv64_ghash_desc_ctx {
28 be128 accumulator;
29 u8 buffer[GHASH_BLOCK_SIZE];
30 u32 bytes;
31 };
32
33 static int riscv64_ghash_setkey(struct crypto_
34 unsigned int k
35 {
36 struct riscv64_ghash_tfm_ctx *tctx = c
37
38 if (keylen != GHASH_BLOCK_SIZE)
39 return -EINVAL;
40
41 memcpy(&tctx->key, key, GHASH_BLOCK_SI
42
43 return 0;
44 }
45
46 static int riscv64_ghash_init(struct shash_des
47 {
48 struct riscv64_ghash_desc_ctx *dctx =
49
50 *dctx = (struct riscv64_ghash_desc_ctx
51
52 return 0;
53 }
54
55 static inline void
56 riscv64_ghash_blocks(const struct riscv64_ghas
57 struct riscv64_ghash_desc
58 const u8 *src, size_t src
59 {
60 /* The srclen is nonzero and a multipl
61 if (crypto_simd_usable()) {
62 kernel_vector_begin();
63 ghash_zvkg(&dctx->accumulator,
64 kernel_vector_end();
65 } else {
66 do {
67 crypto_xor((u8 *)&dctx
68 GHASH_BLOCK
69 gf128mul_lle(&dctx->ac
70 src += GHASH_BLOCK_SIZ
71 srclen -= GHASH_BLOCK_
72 } while (srclen);
73 }
74 }
75
76 static int riscv64_ghash_update(struct shash_d
77 unsigned int s
78 {
79 const struct riscv64_ghash_tfm_ctx *tc
80 struct riscv64_ghash_desc_ctx *dctx =
81 unsigned int len;
82
83 if (dctx->bytes) {
84 if (dctx->bytes + srclen < GHA
85 memcpy(dctx->buffer +
86 dctx->bytes += srclen;
87 return 0;
88 }
89 memcpy(dctx->buffer + dctx->by
90 GHASH_BLOCK_SIZE - dctx
91 riscv64_ghash_blocks(tctx, dct
92 GHASH_BLO
93 src += GHASH_BLOCK_SIZE - dctx
94 srclen -= GHASH_BLOCK_SIZE - d
95 dctx->bytes = 0;
96 }
97
98 len = round_down(srclen, GHASH_BLOCK_S
99 if (len) {
100 riscv64_ghash_blocks(tctx, dct
101 src += len;
102 srclen -= len;
103 }
104
105 if (srclen) {
106 memcpy(dctx->buffer, src, srcl
107 dctx->bytes = srclen;
108 }
109
110 return 0;
111 }
112
113 static int riscv64_ghash_final(struct shash_de
114 {
115 const struct riscv64_ghash_tfm_ctx *tc
116 struct riscv64_ghash_desc_ctx *dctx =
117 int i;
118
119 if (dctx->bytes) {
120 for (i = dctx->bytes; i < GHAS
121 dctx->buffer[i] = 0;
122
123 riscv64_ghash_blocks(tctx, dct
124 GHASH_BLO
125 }
126
127 memcpy(out, &dctx->accumulator, GHASH_
128 return 0;
129 }
130
131 static struct shash_alg riscv64_ghash_alg = {
132 .init = riscv64_ghash_init,
133 .update = riscv64_ghash_update,
134 .final = riscv64_ghash_final,
135 .setkey = riscv64_ghash_setkey,
136 .descsize = sizeof(struct riscv64_ghas
137 .digestsize = GHASH_DIGEST_SIZE,
138 .base = {
139 .cra_blocksize = GHASH_BLOCK_S
140 .cra_ctxsize = sizeof(struct r
141 .cra_priority = 300,
142 .cra_name = "ghash",
143 .cra_driver_name = "ghash-risc
144 .cra_module = THIS_MODULE,
145 },
146 };
147
148 static int __init riscv64_ghash_mod_init(void)
149 {
150 if (riscv_isa_extension_available(NULL
151 riscv_vector_vlen() >= 128)
152 return crypto_register_shash(&
153
154 return -ENODEV;
155 }
156
157 static void __exit riscv64_ghash_mod_exit(void
158 {
159 crypto_unregister_shash(&riscv64_ghash
160 }
161
162 module_init(riscv64_ghash_mod_init);
163 module_exit(riscv64_ghash_mod_exit);
164
165 MODULE_DESCRIPTION("GHASH (RISC-V accelerated)
166 MODULE_AUTHOR("Heiko Stuebner <heiko.stuebner@
167 MODULE_LICENSE("GPL");
168 MODULE_ALIAS_CRYPTO("ghash");
169
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.