1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Accelerated CRC32(C) using ARM CRC, NEON and Crypto Extensions instructions
4 *
5 * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org>
6 */
7
8 #include <linux/cpufeature.h>
9 #include <linux/crc32.h>
10 #include <linux/init.h>
11 #include <linux/kernel.h>
12 #include <linux/module.h>
13 #include <linux/string.h>
14
15 #include <crypto/internal/hash.h>
16 #include <crypto/internal/simd.h>
17
18 #include <asm/hwcap.h>
19 #include <asm/neon.h>
20 #include <asm/simd.h>
21 #include <asm/unaligned.h>
22
23 #define PMULL_MIN_LEN 64L /* minimum size of buffer
24 * for crc32_pmull_le_16 */
25 #define SCALE_F 16L /* size of NEON register */
26
27 asmlinkage u32 crc32_pmull_le(const u8 buf[], u32 len, u32 init_crc);
28 asmlinkage u32 crc32_armv8_le(u32 init_crc, const u8 buf[], u32 len);
29
30 asmlinkage u32 crc32c_pmull_le(const u8 buf[], u32 len, u32 init_crc);
31 asmlinkage u32 crc32c_armv8_le(u32 init_crc, const u8 buf[], u32 len);
32
33 static u32 (*fallback_crc32)(u32 init_crc, const u8 buf[], u32 len);
34 static u32 (*fallback_crc32c)(u32 init_crc, const u8 buf[], u32 len);
35
36 static int crc32_cra_init(struct crypto_tfm *tfm)
37 {
38 u32 *key = crypto_tfm_ctx(tfm);
39
40 *key = 0;
41 return 0;
42 }
43
44 static int crc32c_cra_init(struct crypto_tfm *tfm)
45 {
46 u32 *key = crypto_tfm_ctx(tfm);
47
48 *key = ~0;
49 return 0;
50 }
51
52 static int crc32_setkey(struct crypto_shash *hash, const u8 *key,
53 unsigned int keylen)
54 {
55 u32 *mctx = crypto_shash_ctx(hash);
56
57 if (keylen != sizeof(u32))
58 return -EINVAL;
59 *mctx = le32_to_cpup((__le32 *)key);
60 return 0;
61 }
62
63 static int crc32_init(struct shash_desc *desc)
64 {
65 u32 *mctx = crypto_shash_ctx(desc->tfm);
66 u32 *crc = shash_desc_ctx(desc);
67
68 *crc = *mctx;
69 return 0;
70 }
71
72 static int crc32_update(struct shash_desc *desc, const u8 *data,
73 unsigned int length)
74 {
75 u32 *crc = shash_desc_ctx(desc);
76
77 *crc = crc32_armv8_le(*crc, data, length);
78 return 0;
79 }
80
81 static int crc32c_update(struct shash_desc *desc, const u8 *data,
82 unsigned int length)
83 {
84 u32 *crc = shash_desc_ctx(desc);
85
86 *crc = crc32c_armv8_le(*crc, data, length);
87 return 0;
88 }
89
90 static int crc32_final(struct shash_desc *desc, u8 *out)
91 {
92 u32 *crc = shash_desc_ctx(desc);
93
94 put_unaligned_le32(*crc, out);
95 return 0;
96 }
97
98 static int crc32c_final(struct shash_desc *desc, u8 *out)
99 {
100 u32 *crc = shash_desc_ctx(desc);
101
102 put_unaligned_le32(~*crc, out);
103 return 0;
104 }
105
106 static int crc32_pmull_update(struct shash_desc *desc, const u8 *data,
107 unsigned int length)
108 {
109 u32 *crc = shash_desc_ctx(desc);
110 unsigned int l;
111
112 if (crypto_simd_usable()) {
113 if ((u32)data % SCALE_F) {
114 l = min_t(u32, length, SCALE_F - ((u32)data % SCALE_F));
115
116 *crc = fallback_crc32(*crc, data, l);
117
118 data += l;
119 length -= l;
120 }
121
122 if (length >= PMULL_MIN_LEN) {
123 l = round_down(length, SCALE_F);
124
125 kernel_neon_begin();
126 *crc = crc32_pmull_le(data, l, *crc);
127 kernel_neon_end();
128
129 data += l;
130 length -= l;
131 }
132 }
133
134 if (length > 0)
135 *crc = fallback_crc32(*crc, data, length);
136
137 return 0;
138 }
139
140 static int crc32c_pmull_update(struct shash_desc *desc, const u8 *data,
141 unsigned int length)
142 {
143 u32 *crc = shash_desc_ctx(desc);
144 unsigned int l;
145
146 if (crypto_simd_usable()) {
147 if ((u32)data % SCALE_F) {
148 l = min_t(u32, length, SCALE_F - ((u32)data % SCALE_F));
149
150 *crc = fallback_crc32c(*crc, data, l);
151
152 data += l;
153 length -= l;
154 }
155
156 if (length >= PMULL_MIN_LEN) {
157 l = round_down(length, SCALE_F);
158
159 kernel_neon_begin();
160 *crc = crc32c_pmull_le(data, l, *crc);
161 kernel_neon_end();
162
163 data += l;
164 length -= l;
165 }
166 }
167
168 if (length > 0)
169 *crc = fallback_crc32c(*crc, data, length);
170
171 return 0;
172 }
173
174 static struct shash_alg crc32_pmull_algs[] = { {
175 .setkey = crc32_setkey,
176 .init = crc32_init,
177 .update = crc32_update,
178 .final = crc32_final,
179 .descsize = sizeof(u32),
180 .digestsize = sizeof(u32),
181
182 .base.cra_ctxsize = sizeof(u32),
183 .base.cra_init = crc32_cra_init,
184 .base.cra_name = "crc32",
185 .base.cra_driver_name = "crc32-arm-ce",
186 .base.cra_priority = 200,
187 .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
188 .base.cra_blocksize = 1,
189 .base.cra_module = THIS_MODULE,
190 }, {
191 .setkey = crc32_setkey,
192 .init = crc32_init,
193 .update = crc32c_update,
194 .final = crc32c_final,
195 .descsize = sizeof(u32),
196 .digestsize = sizeof(u32),
197
198 .base.cra_ctxsize = sizeof(u32),
199 .base.cra_init = crc32c_cra_init,
200 .base.cra_name = "crc32c",
201 .base.cra_driver_name = "crc32c-arm-ce",
202 .base.cra_priority = 200,
203 .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
204 .base.cra_blocksize = 1,
205 .base.cra_module = THIS_MODULE,
206 } };
207
208 static int __init crc32_pmull_mod_init(void)
209 {
210 if (elf_hwcap2 & HWCAP2_PMULL) {
211 crc32_pmull_algs[0].update = crc32_pmull_update;
212 crc32_pmull_algs[1].update = crc32c_pmull_update;
213
214 if (elf_hwcap2 & HWCAP2_CRC32) {
215 fallback_crc32 = crc32_armv8_le;
216 fallback_crc32c = crc32c_armv8_le;
217 } else {
218 fallback_crc32 = crc32_le;
219 fallback_crc32c = __crc32c_le;
220 }
221 } else if (!(elf_hwcap2 & HWCAP2_CRC32)) {
222 return -ENODEV;
223 }
224
225 return crypto_register_shashes(crc32_pmull_algs,
226 ARRAY_SIZE(crc32_pmull_algs));
227 }
228
229 static void __exit crc32_pmull_mod_exit(void)
230 {
231 crypto_unregister_shashes(crc32_pmull_algs,
232 ARRAY_SIZE(crc32_pmull_algs));
233 }
234
235 static const struct cpu_feature __maybe_unused crc32_cpu_feature[] = {
236 { cpu_feature(CRC32) }, { cpu_feature(PMULL) }, { }
237 };
238 MODULE_DEVICE_TABLE(cpu, crc32_cpu_feature);
239
240 module_init(crc32_pmull_mod_init);
241 module_exit(crc32_pmull_mod_exit);
242
243 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
244 MODULE_DESCRIPTION("Accelerated CRC32(C) using ARM CRC, NEON and Crypto Extensions");
245 MODULE_LICENSE("GPL v2");
246 MODULE_ALIAS_CRYPTO("crc32");
247 MODULE_ALIAS_CRYPTO("crc32c");
248
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.