1 // SPDX-License-Identifier: GPL-2.0 1
2 /*
3 * OpenSSL/Cryptogams accelerated Poly1305 tra
4 *
5 * Copyright (C) 2019 Linaro Ltd. <ard.biesheu
6 */
7
8 #include <asm/hwcap.h>
9 #include <asm/neon.h>
10 #include <asm/simd.h>
11 #include <linux/unaligned.h>
12 #include <crypto/algapi.h>
13 #include <crypto/internal/hash.h>
14 #include <crypto/internal/poly1305.h>
15 #include <crypto/internal/simd.h>
16 #include <linux/cpufeature.h>
17 #include <linux/crypto.h>
18 #include <linux/jump_label.h>
19 #include <linux/module.h>
20
21 void poly1305_init_arm(void *state, const u8 *
22 void poly1305_blocks_arm(void *state, const u8
23 void poly1305_blocks_neon(void *state, const u
24 void poly1305_emit_arm(void *state, u8 *digest
25
26 void __weak poly1305_blocks_neon(void *state,
27 {
28 }
29
30 static __ro_after_init DEFINE_STATIC_KEY_FALSE
31
32 void poly1305_init_arch(struct poly1305_desc_c
33 {
34 poly1305_init_arm(&dctx->h, key);
35 dctx->s[0] = get_unaligned_le32(key +
36 dctx->s[1] = get_unaligned_le32(key +
37 dctx->s[2] = get_unaligned_le32(key +
38 dctx->s[3] = get_unaligned_le32(key +
39 dctx->buflen = 0;
40 }
41 EXPORT_SYMBOL(poly1305_init_arch);
42
43 static int arm_poly1305_init(struct shash_desc
44 {
45 struct poly1305_desc_ctx *dctx = shash
46
47 dctx->buflen = 0;
48 dctx->rset = 0;
49 dctx->sset = false;
50
51 return 0;
52 }
53
54 static void arm_poly1305_blocks(struct poly130
55 u32 len, u32
56 {
57 if (unlikely(!dctx->sset)) {
58 if (!dctx->rset) {
59 poly1305_init_arm(&dct
60 src += POLY1305_BLOCK_
61 len -= POLY1305_BLOCK_
62 dctx->rset = 1;
63 }
64 if (len >= POLY1305_BLOCK_SIZE
65 dctx->s[0] = get_unali
66 dctx->s[1] = get_unali
67 dctx->s[2] = get_unali
68 dctx->s[3] = get_unali
69 src += POLY1305_BLOCK_
70 len -= POLY1305_BLOCK_
71 dctx->sset = true;
72 }
73 if (len < POLY1305_BLOCK_SIZE)
74 return;
75 }
76
77 len &= ~(POLY1305_BLOCK_SIZE - 1);
78
79 if (static_branch_likely(&have_neon) &
80 poly1305_blocks_neon(&dctx->h,
81 else
82 poly1305_blocks_arm(&dctx->h,
83 }
84
85 static void arm_poly1305_do_update(struct poly
86 const u8 *
87 {
88 if (unlikely(dctx->buflen)) {
89 u32 bytes = min(len, POLY1305_
90
91 memcpy(dctx->buf + dctx->bufle
92 src += bytes;
93 len -= bytes;
94 dctx->buflen += bytes;
95
96 if (dctx->buflen == POLY1305_B
97 arm_poly1305_blocks(dc
98 PO
99 dctx->buflen = 0;
100 }
101 }
102
103 if (likely(len >= POLY1305_BLOCK_SIZE)
104 arm_poly1305_blocks(dctx, src,
105 src += round_down(len, POLY130
106 len %= POLY1305_BLOCK_SIZE;
107 }
108
109 if (unlikely(len)) {
110 dctx->buflen = len;
111 memcpy(dctx->buf, src, len);
112 }
113 }
114
115 static int arm_poly1305_update(struct shash_de
116 const u8 *src,
117 {
118 struct poly1305_desc_ctx *dctx = shash
119
120 arm_poly1305_do_update(dctx, src, srcl
121 return 0;
122 }
123
124 static int __maybe_unused arm_poly1305_update_
125
126
127 {
128 struct poly1305_desc_ctx *dctx = shash
129 bool do_neon = crypto_simd_usable() &&
130
131 if (static_branch_likely(&have_neon) &
132 kernel_neon_begin();
133 arm_poly1305_do_update(dctx, src, srcl
134 if (static_branch_likely(&have_neon) &
135 kernel_neon_end();
136 return 0;
137 }
138
139 void poly1305_update_arch(struct poly1305_desc
140 unsigned int nbytes)
141 {
142 bool do_neon = IS_ENABLED(CONFIG_KERNE
143 crypto_simd_usable();
144
145 if (unlikely(dctx->buflen)) {
146 u32 bytes = min(nbytes, POLY13
147
148 memcpy(dctx->buf + dctx->bufle
149 src += bytes;
150 nbytes -= bytes;
151 dctx->buflen += bytes;
152
153 if (dctx->buflen == POLY1305_B
154 poly1305_blocks_arm(&d
155 PO
156 dctx->buflen = 0;
157 }
158 }
159
160 if (likely(nbytes >= POLY1305_BLOCK_SI
161 unsigned int len = round_down(
162
163 if (static_branch_likely(&have
164 do {
165 unsigned int t
166
167 kernel_neon_be
168 poly1305_block
169 kernel_neon_en
170
171 len -= todo;
172 src += todo;
173 } while (len);
174 } else {
175 poly1305_blocks_arm(&d
176 src += len;
177 }
178 nbytes %= POLY1305_BLOCK_SIZE;
179 }
180
181 if (unlikely(nbytes)) {
182 dctx->buflen = nbytes;
183 memcpy(dctx->buf, src, nbytes)
184 }
185 }
186 EXPORT_SYMBOL(poly1305_update_arch);
187
188 void poly1305_final_arch(struct poly1305_desc_
189 {
190 if (unlikely(dctx->buflen)) {
191 dctx->buf[dctx->buflen++] = 1;
192 memset(dctx->buf + dctx->bufle
193 POLY1305_BLOCK_SIZE - d
194 poly1305_blocks_arm(&dctx->h,
195 }
196
197 poly1305_emit_arm(&dctx->h, dst, dctx-
198 *dctx = (struct poly1305_desc_ctx){};
199 }
200 EXPORT_SYMBOL(poly1305_final_arch);
201
202 static int arm_poly1305_final(struct shash_des
203 {
204 struct poly1305_desc_ctx *dctx = shash
205
206 if (unlikely(!dctx->sset))
207 return -ENOKEY;
208
209 poly1305_final_arch(dctx, dst);
210 return 0;
211 }
212
213 static struct shash_alg arm_poly1305_algs[] =
214 .init = arm_poly1305
215 .update = arm_poly1305
216 .final = arm_poly1305
217 .digestsize = POLY1305_DIG
218 .descsize = sizeof(struc
219
220 .base.cra_name = "poly1305",
221 .base.cra_driver_name = "poly1305-ar
222 .base.cra_priority = 150,
223 .base.cra_blocksize = POLY1305_BLO
224 .base.cra_module = THIS_MODULE,
225 #ifdef CONFIG_KERNEL_MODE_NEON
226 }, {
227 .init = arm_poly1305
228 .update = arm_poly1305
229 .final = arm_poly1305
230 .digestsize = POLY1305_DIG
231 .descsize = sizeof(struc
232
233 .base.cra_name = "poly1305",
234 .base.cra_driver_name = "poly1305-ne
235 .base.cra_priority = 200,
236 .base.cra_blocksize = POLY1305_BLO
237 .base.cra_module = THIS_MODULE,
238 #endif
239 }};
240
241 static int __init arm_poly1305_mod_init(void)
242 {
243 if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON
244 (elf_hwcap & HWCAP_NEON))
245 static_branch_enable(&have_neo
246 else if (IS_REACHABLE(CONFIG_CRYPTO_HA
247 /* register only the first ent
248 return crypto_register_shash(&
249
250 return IS_REACHABLE(CONFIG_CRYPTO_HASH
251 crypto_register_shashes(arm_po
252 ARRAY_
253 }
254
255 static void __exit arm_poly1305_mod_exit(void)
256 {
257 if (!IS_REACHABLE(CONFIG_CRYPTO_HASH))
258 return;
259 if (!static_branch_likely(&have_neon))
260 crypto_unregister_shash(&arm_p
261 return;
262 }
263 crypto_unregister_shashes(arm_poly1305
264 ARRAY_SIZE(a
265 }
266
267 module_init(arm_poly1305_mod_init);
268 module_exit(arm_poly1305_mod_exit);
269
270 MODULE_DESCRIPTION("Accelerated Poly1305 trans
271 MODULE_LICENSE("GPL v2");
272 MODULE_ALIAS_CRYPTO("poly1305");
273 MODULE_ALIAS_CRYPTO("poly1305-arm");
274 MODULE_ALIAS_CRYPTO("poly1305-neon");
275
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.