1 // SPDX-License-Identifier: GPL-2.0-or-later 1
2 /*
3 * PowerPC P10 (ppc64le) accelerated ChaCha an
4 * including ChaCha20 (RFC7539)
5 *
6 * Copyright 2023- IBM Corp. All rights reserv
7 */
8
9 #include <crypto/algapi.h>
10 #include <crypto/internal/chacha.h>
11 #include <crypto/internal/simd.h>
12 #include <crypto/internal/skcipher.h>
13 #include <linux/kernel.h>
14 #include <linux/module.h>
15 #include <linux/cpufeature.h>
16 #include <linux/sizes.h>
17 #include <asm/simd.h>
18 #include <asm/switch_to.h>
19
20 asmlinkage void chacha_p10le_8x(u32 *state, u8
21 unsigned int l
22
23 static __ro_after_init DEFINE_STATIC_KEY_FALSE
24
25 static void vsx_begin(void)
26 {
27 preempt_disable();
28 enable_kernel_vsx();
29 }
30
31 static void vsx_end(void)
32 {
33 disable_kernel_vsx();
34 preempt_enable();
35 }
36
37 static void chacha_p10_do_8x(u32 *state, u8 *d
38 unsigned int byte
39 {
40 unsigned int l = bytes & ~0x0FF;
41
42 if (l > 0) {
43 chacha_p10le_8x(state, dst, sr
44 bytes -= l;
45 src += l;
46 dst += l;
47 state[12] += l / CHACHA_BLOCK_
48 }
49
50 if (bytes > 0)
51 chacha_crypt_generic(state, ds
52 }
53
54 void hchacha_block_arch(const u32 *state, u32
55 {
56 hchacha_block_generic(state, stream, n
57 }
58 EXPORT_SYMBOL(hchacha_block_arch);
59
60 void chacha_init_arch(u32 *state, const u32 *k
61 {
62 chacha_init_generic(state, key, iv);
63 }
64 EXPORT_SYMBOL(chacha_init_arch);
65
66 void chacha_crypt_arch(u32 *state, u8 *dst, co
67 int nrounds)
68 {
69 if (!static_branch_likely(&have_p10) |
70 !crypto_simd_usable())
71 return chacha_crypt_generic(st
72
73 do {
74 unsigned int todo = min_t(unsi
75
76 vsx_begin();
77 chacha_p10_do_8x(state, dst, s
78 vsx_end();
79
80 bytes -= todo;
81 src += todo;
82 dst += todo;
83 } while (bytes);
84 }
85 EXPORT_SYMBOL(chacha_crypt_arch);
86
87 static int chacha_p10_stream_xor(struct skciph
88 const struct
89 {
90 struct skcipher_walk walk;
91 u32 state[16];
92 int err;
93
94 err = skcipher_walk_virt(&walk, req, f
95 if (err)
96 return err;
97
98 chacha_init_generic(state, ctx->key, i
99
100 while (walk.nbytes > 0) {
101 unsigned int nbytes = walk.nby
102
103 if (nbytes < walk.total)
104 nbytes = rounddown(nby
105
106 if (!crypto_simd_usable()) {
107 chacha_crypt_generic(s
108 w
109 c
110 } else {
111 vsx_begin();
112 chacha_p10_do_8x(state
113 walk.src
114 vsx_end();
115 }
116 err = skcipher_walk_done(&walk
117 if (err)
118 break;
119 }
120
121 return err;
122 }
123
124 static int chacha_p10(struct skcipher_request
125 {
126 struct crypto_skcipher *tfm = crypto_s
127 struct chacha_ctx *ctx = crypto_skciph
128
129 return chacha_p10_stream_xor(req, ctx,
130 }
131
132 static int xchacha_p10(struct skcipher_request
133 {
134 struct crypto_skcipher *tfm = crypto_s
135 struct chacha_ctx *ctx = crypto_skciph
136 struct chacha_ctx subctx;
137 u32 state[16];
138 u8 real_iv[16];
139
140 chacha_init_generic(state, ctx->key, r
141 hchacha_block_arch(state, subctx.key,
142 subctx.nrounds = ctx->nrounds;
143
144 memcpy(&real_iv[0], req->iv + 24, 8);
145 memcpy(&real_iv[8], req->iv + 16, 8);
146 return chacha_p10_stream_xor(req, &sub
147 }
148
149 static struct skcipher_alg algs[] = {
150 {
151 .base.cra_name = "cha
152 .base.cra_driver_name = "cha
153 .base.cra_priority = 300,
154 .base.cra_blocksize = 1,
155 .base.cra_ctxsize = size
156 .base.cra_module = THIS
157
158 .min_keysize = CHAC
159 .max_keysize = CHAC
160 .ivsize = CHAC
161 .chunksize = CHAC
162 .setkey = chac
163 .encrypt = chac
164 .decrypt = chac
165 }, {
166 .base.cra_name = "xch
167 .base.cra_driver_name = "xch
168 .base.cra_priority = 300,
169 .base.cra_blocksize = 1,
170 .base.cra_ctxsize = size
171 .base.cra_module = THIS
172
173 .min_keysize = CHAC
174 .max_keysize = CHAC
175 .ivsize = XCHA
176 .chunksize = CHAC
177 .setkey = chac
178 .encrypt = xcha
179 .decrypt = xcha
180 }, {
181 .base.cra_name = "xch
182 .base.cra_driver_name = "xch
183 .base.cra_priority = 300,
184 .base.cra_blocksize = 1,
185 .base.cra_ctxsize = size
186 .base.cra_module = THIS
187
188 .min_keysize = CHAC
189 .max_keysize = CHAC
190 .ivsize = XCHA
191 .chunksize = CHAC
192 .setkey = chac
193 .encrypt = xcha
194 .decrypt = xcha
195 }
196 };
197
198 static int __init chacha_p10_init(void)
199 {
200 if (!cpu_has_feature(CPU_FTR_ARCH_31))
201 return 0;
202
203 static_branch_enable(&have_p10);
204
205 return crypto_register_skciphers(algs,
206 }
207
208 static void __exit chacha_p10_exit(void)
209 {
210 if (!static_branch_likely(&have_p10))
211 return;
212
213 crypto_unregister_skciphers(algs, ARRA
214 }
215
216 module_init(chacha_p10_init);
217 module_exit(chacha_p10_exit);
218
219 MODULE_DESCRIPTION("ChaCha and XChaCha stream
220 MODULE_AUTHOR("Danny Tsen <dtsen@linux.ibm.com
221 MODULE_LICENSE("GPL v2");
222 MODULE_ALIAS_CRYPTO("chacha20");
223 MODULE_ALIAS_CRYPTO("chacha20-p10");
224 MODULE_ALIAS_CRYPTO("xchacha20");
225 MODULE_ALIAS_CRYPTO("xchacha20-p10");
226 MODULE_ALIAS_CRYPTO("xchacha12");
227 MODULE_ALIAS_CRYPTO("xchacha12-p10");
228
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.