1 // SPDX-License-Identifier: GPL-2.0 1
2 /*
3 * OpenSSL/Cryptogams accelerated Poly1305 tra
4 *
5 * Copyright (C) 2019 Linaro Ltd. <ard.biesheu
6 */
7
8 #include <asm/hwcap.h>
9 #include <asm/neon.h>
10 #include <asm/simd.h>
11 #include <linux/unaligned.h>
12 #include <crypto/algapi.h>
13 #include <crypto/internal/hash.h>
14 #include <crypto/internal/poly1305.h>
15 #include <crypto/internal/simd.h>
16 #include <linux/cpufeature.h>
17 #include <linux/crypto.h>
18 #include <linux/jump_label.h>
19 #include <linux/module.h>
20
21 asmlinkage void poly1305_init_arm64(void *stat
22 asmlinkage void poly1305_blocks(void *state, c
23 asmlinkage void poly1305_blocks_neon(void *sta
24 asmlinkage void poly1305_emit(void *state, u8
25
26 static __ro_after_init DEFINE_STATIC_KEY_FALSE
27
28 void poly1305_init_arch(struct poly1305_desc_c
29 {
30 poly1305_init_arm64(&dctx->h, key);
31 dctx->s[0] = get_unaligned_le32(key +
32 dctx->s[1] = get_unaligned_le32(key +
33 dctx->s[2] = get_unaligned_le32(key +
34 dctx->s[3] = get_unaligned_le32(key +
35 dctx->buflen = 0;
36 }
37 EXPORT_SYMBOL(poly1305_init_arch);
38
39 static int neon_poly1305_init(struct shash_des
40 {
41 struct poly1305_desc_ctx *dctx = shash
42
43 dctx->buflen = 0;
44 dctx->rset = 0;
45 dctx->sset = false;
46
47 return 0;
48 }
49
50 static void neon_poly1305_blocks(struct poly13
51 u32 len, u32
52 {
53 if (unlikely(!dctx->sset)) {
54 if (!dctx->rset) {
55 poly1305_init_arm64(&d
56 src += POLY1305_BLOCK_
57 len -= POLY1305_BLOCK_
58 dctx->rset = 1;
59 }
60 if (len >= POLY1305_BLOCK_SIZE
61 dctx->s[0] = get_unali
62 dctx->s[1] = get_unali
63 dctx->s[2] = get_unali
64 dctx->s[3] = get_unali
65 src += POLY1305_BLOCK_
66 len -= POLY1305_BLOCK_
67 dctx->sset = true;
68 }
69 if (len < POLY1305_BLOCK_SIZE)
70 return;
71 }
72
73 len &= ~(POLY1305_BLOCK_SIZE - 1);
74
75 if (static_branch_likely(&have_neon) &
76 poly1305_blocks_neon(&dctx->h,
77 else
78 poly1305_blocks(&dctx->h, src,
79 }
80
81 static void neon_poly1305_do_update(struct pol
82 const u8 *
83 {
84 if (unlikely(dctx->buflen)) {
85 u32 bytes = min(len, POLY1305_
86
87 memcpy(dctx->buf + dctx->bufle
88 src += bytes;
89 len -= bytes;
90 dctx->buflen += bytes;
91
92 if (dctx->buflen == POLY1305_B
93 neon_poly1305_blocks(d
94 P
95 dctx->buflen = 0;
96 }
97 }
98
99 if (likely(len >= POLY1305_BLOCK_SIZE)
100 neon_poly1305_blocks(dctx, src
101 src += round_down(len, POLY130
102 len %= POLY1305_BLOCK_SIZE;
103 }
104
105 if (unlikely(len)) {
106 dctx->buflen = len;
107 memcpy(dctx->buf, src, len);
108 }
109 }
110
111 static int neon_poly1305_update(struct shash_d
112 const u8 *src,
113 {
114 bool do_neon = crypto_simd_usable() &&
115 struct poly1305_desc_ctx *dctx = shash
116
117 if (static_branch_likely(&have_neon) &
118 kernel_neon_begin();
119 neon_poly1305_do_update(dctx, src, src
120 if (static_branch_likely(&have_neon) &
121 kernel_neon_end();
122 return 0;
123 }
124
125 void poly1305_update_arch(struct poly1305_desc
126 unsigned int nbytes)
127 {
128 if (unlikely(dctx->buflen)) {
129 u32 bytes = min(nbytes, POLY13
130
131 memcpy(dctx->buf + dctx->bufle
132 src += bytes;
133 nbytes -= bytes;
134 dctx->buflen += bytes;
135
136 if (dctx->buflen == POLY1305_B
137 poly1305_blocks(&dctx-
138 dctx->buflen = 0;
139 }
140 }
141
142 if (likely(nbytes >= POLY1305_BLOCK_SI
143 unsigned int len = round_down(
144
145 if (static_branch_likely(&have
146 do {
147 unsigned int t
148
149 kernel_neon_be
150 poly1305_block
151 kernel_neon_en
152
153 len -= todo;
154 src += todo;
155 } while (len);
156 } else {
157 poly1305_blocks(&dctx-
158 src += len;
159 }
160 nbytes %= POLY1305_BLOCK_SIZE;
161 }
162
163 if (unlikely(nbytes)) {
164 dctx->buflen = nbytes;
165 memcpy(dctx->buf, src, nbytes)
166 }
167 }
168 EXPORT_SYMBOL(poly1305_update_arch);
169
170 void poly1305_final_arch(struct poly1305_desc_
171 {
172 if (unlikely(dctx->buflen)) {
173 dctx->buf[dctx->buflen++] = 1;
174 memset(dctx->buf + dctx->bufle
175 POLY1305_BLOCK_SIZE - d
176 poly1305_blocks(&dctx->h, dctx
177 }
178
179 poly1305_emit(&dctx->h, dst, dctx->s);
180 memzero_explicit(dctx, sizeof(*dctx));
181 }
182 EXPORT_SYMBOL(poly1305_final_arch);
183
184 static int neon_poly1305_final(struct shash_de
185 {
186 struct poly1305_desc_ctx *dctx = shash
187
188 if (unlikely(!dctx->sset))
189 return -ENOKEY;
190
191 poly1305_final_arch(dctx, dst);
192 return 0;
193 }
194
195 static struct shash_alg neon_poly1305_alg = {
196 .init = neon_poly130
197 .update = neon_poly130
198 .final = neon_poly130
199 .digestsize = POLY1305_DIG
200 .descsize = sizeof(struc
201
202 .base.cra_name = "poly1305",
203 .base.cra_driver_name = "poly1305-ne
204 .base.cra_priority = 200,
205 .base.cra_blocksize = POLY1305_BLO
206 .base.cra_module = THIS_MODULE,
207 };
208
209 static int __init neon_poly1305_mod_init(void)
210 {
211 if (!cpu_have_named_feature(ASIMD))
212 return 0;
213
214 static_branch_enable(&have_neon);
215
216 return IS_REACHABLE(CONFIG_CRYPTO_HASH
217 crypto_register_shash(&neon_po
218 }
219
220 static void __exit neon_poly1305_mod_exit(void
221 {
222 if (IS_REACHABLE(CONFIG_CRYPTO_HASH) &
223 crypto_unregister_shash(&neon_
224 }
225
226 module_init(neon_poly1305_mod_init);
227 module_exit(neon_poly1305_mod_exit);
228
229 MODULE_DESCRIPTION("Poly1305 transform using N
230 MODULE_LICENSE("GPL v2");
231 MODULE_ALIAS_CRYPTO("poly1305");
232 MODULE_ALIAS_CRYPTO("poly1305-neon");
233
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.