1 // SPDX-License-Identifier: GPL-2.0-or-later 1
2 /*
3 * Linux/arm64 port of the OpenSSL SHA256 impl
4 *
5 * Copyright (c) 2016 Linaro Ltd. <ard.biesheu
6 */
7
8 #include <asm/hwcap.h>
9 #include <asm/neon.h>
10 #include <asm/simd.h>
11 #include <crypto/internal/hash.h>
12 #include <crypto/internal/simd.h>
13 #include <crypto/sha2.h>
14 #include <crypto/sha256_base.h>
15 #include <linux/module.h>
16 #include <linux/string.h>
17 #include <linux/types.h>
18
19 MODULE_DESCRIPTION("SHA-224/SHA-256 secure has
20 MODULE_AUTHOR("Andy Polyakov <appro@openssl.or
21 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@
22 MODULE_LICENSE("GPL v2");
23 MODULE_ALIAS_CRYPTO("sha224");
24 MODULE_ALIAS_CRYPTO("sha256");
25
26 asmlinkage void sha256_block_data_order(u32 *d
27 unsign
28 EXPORT_SYMBOL(sha256_block_data_order);
29
30 static void sha256_arm64_transform(struct sha2
31 int blocks)
32 {
33 sha256_block_data_order(sst->state, sr
34 }
35
36 asmlinkage void sha256_block_neon(u32 *digest,
37 unsigned int
38
39 static void sha256_neon_transform(struct sha25
40 int blocks)
41 {
42 sha256_block_neon(sst->state, src, blo
43 }
44
45 static int crypto_sha256_arm64_update(struct s
46 unsigned
47 {
48 return sha256_base_do_update(desc, dat
49 }
50
51 static int crypto_sha256_arm64_finup(struct sh
52 unsigned
53 {
54 if (len)
55 sha256_base_do_update(desc, da
56 sha256_base_do_finalize(desc, sha256_a
57
58 return sha256_base_finish(desc, out);
59 }
60
61 static int crypto_sha256_arm64_final(struct sh
62 {
63 return crypto_sha256_arm64_finup(desc,
64 }
65
66 static struct shash_alg algs[] = { {
67 .digestsize = SHA256_DIGES
68 .init = sha256_base_
69 .update = crypto_sha25
70 .final = crypto_sha25
71 .finup = crypto_sha25
72 .descsize = sizeof(struc
73 .base.cra_name = "sha256",
74 .base.cra_driver_name = "sha256-arm6
75 .base.cra_priority = 125,
76 .base.cra_blocksize = SHA256_BLOCK
77 .base.cra_module = THIS_MODULE,
78 }, {
79 .digestsize = SHA224_DIGES
80 .init = sha224_base_
81 .update = crypto_sha25
82 .final = crypto_sha25
83 .finup = crypto_sha25
84 .descsize = sizeof(struc
85 .base.cra_name = "sha224",
86 .base.cra_driver_name = "sha224-arm6
87 .base.cra_priority = 125,
88 .base.cra_blocksize = SHA224_BLOCK
89 .base.cra_module = THIS_MODULE,
90 } };
91
92 static int sha256_update_neon(struct shash_des
93 unsigned int len
94 {
95 struct sha256_state *sctx = shash_desc
96
97 if (!crypto_simd_usable())
98 return sha256_base_do_update(d
99 sha256_arm64_t
100
101 while (len > 0) {
102 unsigned int chunk = len;
103
104 /*
105 * Don't hog the CPU for the e
106 * input when running on a pre
107 * data block by block instead
108 */
109 if (IS_ENABLED(CONFIG_PREEMPTI
110 chunk + sctx->count % SHA2
111 chunk = SHA256_BLOCK_S
112 sctx->count %
113
114 kernel_neon_begin();
115 sha256_base_do_update(desc, da
116 kernel_neon_end();
117 data += chunk;
118 len -= chunk;
119 }
120 return 0;
121 }
122
123 static int sha256_finup_neon(struct shash_desc
124 unsigned int len,
125 {
126 if (!crypto_simd_usable()) {
127 if (len)
128 sha256_base_do_update(
129 sha256_arm64_t
130 sha256_base_do_finalize(desc,
131 } else {
132 if (len)
133 sha256_update_neon(des
134 kernel_neon_begin();
135 sha256_base_do_finalize(desc,
136 kernel_neon_end();
137 }
138 return sha256_base_finish(desc, out);
139 }
140
141 static int sha256_final_neon(struct shash_desc
142 {
143 return sha256_finup_neon(desc, NULL, 0
144 }
145
146 static struct shash_alg neon_algs[] = { {
147 .digestsize = SHA256_DIGES
148 .init = sha256_base_
149 .update = sha256_updat
150 .final = sha256_final
151 .finup = sha256_finup
152 .descsize = sizeof(struc
153 .base.cra_name = "sha256",
154 .base.cra_driver_name = "sha256-arm6
155 .base.cra_priority = 150,
156 .base.cra_blocksize = SHA256_BLOCK
157 .base.cra_module = THIS_MODULE,
158 }, {
159 .digestsize = SHA224_DIGES
160 .init = sha224_base_
161 .update = sha256_updat
162 .final = sha256_final
163 .finup = sha256_finup
164 .descsize = sizeof(struc
165 .base.cra_name = "sha224",
166 .base.cra_driver_name = "sha224-arm6
167 .base.cra_priority = 150,
168 .base.cra_blocksize = SHA224_BLOCK
169 .base.cra_module = THIS_MODULE,
170 } };
171
172 static int __init sha256_mod_init(void)
173 {
174 int ret = crypto_register_shashes(algs
175 if (ret)
176 return ret;
177
178 if (cpu_have_named_feature(ASIMD)) {
179 ret = crypto_register_shashes(
180 if (ret)
181 crypto_unregister_shas
182 }
183 return ret;
184 }
185
186 static void __exit sha256_mod_fini(void)
187 {
188 if (cpu_have_named_feature(ASIMD))
189 crypto_unregister_shashes(neon
190 crypto_unregister_shashes(algs, ARRAY_
191 }
192
193 module_init(sha256_mod_init);
194 module_exit(sha256_mod_fini);
195
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.