1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Copyright 2024- IBM Corp.
4 *
5 * X25519 scalar multiplication with 51 bits limbs for PPC64le.
6 * Based on RFC7748 and AArch64 optimized implementation for X25519
7 * - Algorithm 1 Scalar multiplication of a variable point
8 */
9
10 #include <crypto/curve25519.h>
11 #include <crypto/internal/kpp.h>
12
13 #include <linux/types.h>
14 #include <linux/jump_label.h>
15 #include <linux/kernel.h>
16 #include <linux/module.h>
17 #include <linux/scatterlist.h>
18
19 #include <linux/cpufeature.h>
20 #include <linux/processor.h>
21
22 typedef uint64_t fe51[5];
23
24 asmlinkage void x25519_fe51_mul(fe51 h, const fe51 f, const fe51 g);
25 asmlinkage void x25519_fe51_sqr(fe51 h, const fe51 f);
26 asmlinkage void x25519_fe51_mul121666(fe51 h, fe51 f);
27 asmlinkage void x25519_fe51_sqr_times(fe51 h, const fe51 f, int n);
28 asmlinkage void x25519_fe51_frombytes(fe51 h, const uint8_t *s);
29 asmlinkage void x25519_fe51_tobytes(uint8_t *s, const fe51 h);
30 asmlinkage void x25519_cswap(fe51 p, fe51 q, unsigned int bit);
31
32 #define fmul x25519_fe51_mul
33 #define fsqr x25519_fe51_sqr
34 #define fmul121666 x25519_fe51_mul121666
35 #define fe51_tobytes x25519_fe51_tobytes
36
37 static void fadd(fe51 h, const fe51 f, const fe51 g)
38 {
39 h[0] = f[0] + g[0];
40 h[1] = f[1] + g[1];
41 h[2] = f[2] + g[2];
42 h[3] = f[3] + g[3];
43 h[4] = f[4] + g[4];
44 }
45
46 /*
47 * Prime = 2 ** 255 - 19, 255 bits
48 * (0x7fffffff ffffffff ffffffff ffffffff ffffffff ffffffff ffffffff ffffffed)
49 *
50 * Prime in 5 51-bit limbs
51 */
52 static fe51 prime51 = { 0x7ffffffffffed, 0x7ffffffffffff, 0x7ffffffffffff, 0x7ffffffffffff, 0x7ffffffffffff};
53
54 static void fsub(fe51 h, const fe51 f, const fe51 g)
55 {
56 h[0] = (f[0] + ((prime51[0] * 2))) - g[0];
57 h[1] = (f[1] + ((prime51[1] * 2))) - g[1];
58 h[2] = (f[2] + ((prime51[2] * 2))) - g[2];
59 h[3] = (f[3] + ((prime51[3] * 2))) - g[3];
60 h[4] = (f[4] + ((prime51[4] * 2))) - g[4];
61 }
62
63 static void fe51_frombytes(fe51 h, const uint8_t *s)
64 {
65 /*
66 * Make sure 64-bit aligned.
67 */
68 unsigned char sbuf[32+8];
69 unsigned char *sb = PTR_ALIGN((void *)sbuf, 8);
70
71 memcpy(sb, s, 32);
72 x25519_fe51_frombytes(h, sb);
73 }
74
75 static void finv(fe51 o, const fe51 i)
76 {
77 fe51 a0, b, c, t00;
78
79 fsqr(a0, i);
80 x25519_fe51_sqr_times(t00, a0, 2);
81
82 fmul(b, t00, i);
83 fmul(a0, b, a0);
84
85 fsqr(t00, a0);
86
87 fmul(b, t00, b);
88 x25519_fe51_sqr_times(t00, b, 5);
89
90 fmul(b, t00, b);
91 x25519_fe51_sqr_times(t00, b, 10);
92
93 fmul(c, t00, b);
94 x25519_fe51_sqr_times(t00, c, 20);
95
96 fmul(t00, t00, c);
97 x25519_fe51_sqr_times(t00, t00, 10);
98
99 fmul(b, t00, b);
100 x25519_fe51_sqr_times(t00, b, 50);
101
102 fmul(c, t00, b);
103 x25519_fe51_sqr_times(t00, c, 100);
104
105 fmul(t00, t00, c);
106 x25519_fe51_sqr_times(t00, t00, 50);
107
108 fmul(t00, t00, b);
109 x25519_fe51_sqr_times(t00, t00, 5);
110
111 fmul(o, t00, a0);
112 }
113
114 static void curve25519_fe51(uint8_t out[32], const uint8_t scalar[32],
115 const uint8_t point[32])
116 {
117 fe51 x1, x2, z2, x3, z3;
118 uint8_t s[32];
119 unsigned int swap = 0;
120 int i;
121
122 memcpy(s, scalar, 32);
123 s[0] &= 0xf8;
124 s[31] &= 0x7f;
125 s[31] |= 0x40;
126 fe51_frombytes(x1, point);
127
128 z2[0] = z2[1] = z2[2] = z2[3] = z2[4] = 0;
129 x3[0] = x1[0];
130 x3[1] = x1[1];
131 x3[2] = x1[2];
132 x3[3] = x1[3];
133 x3[4] = x1[4];
134
135 x2[0] = z3[0] = 1;
136 x2[1] = z3[1] = 0;
137 x2[2] = z3[2] = 0;
138 x2[3] = z3[3] = 0;
139 x2[4] = z3[4] = 0;
140
141 for (i = 254; i >= 0; --i) {
142 unsigned int k_t = 1 & (s[i / 8] >> (i & 7));
143 fe51 a, b, c, d, e;
144 fe51 da, cb, aa, bb;
145 fe51 dacb_p, dacb_m;
146
147 swap ^= k_t;
148 x25519_cswap(x2, x3, swap);
149 x25519_cswap(z2, z3, swap);
150 swap = k_t;
151
152 fsub(b, x2, z2); // B = x_2 - z_2
153 fadd(a, x2, z2); // A = x_2 + z_2
154 fsub(d, x3, z3); // D = x_3 - z_3
155 fadd(c, x3, z3); // C = x_3 + z_3
156
157 fsqr(bb, b); // BB = B^2
158 fsqr(aa, a); // AA = A^2
159 fmul(da, d, a); // DA = D * A
160 fmul(cb, c, b); // CB = C * B
161
162 fsub(e, aa, bb); // E = AA - BB
163 fmul(x2, aa, bb); // x2 = AA * BB
164 fadd(dacb_p, da, cb); // DA + CB
165 fsub(dacb_m, da, cb); // DA - CB
166
167 fmul121666(z3, e); // 121666 * E
168 fsqr(z2, dacb_m); // (DA - CB)^2
169 fsqr(x3, dacb_p); // x3 = (DA + CB)^2
170 fadd(b, bb, z3); // BB + 121666 * E
171 fmul(z3, x1, z2); // z3 = x1 * (DA - CB)^2
172 fmul(z2, e, b); // z2 = e * (BB + (DA + CB)^2)
173 }
174
175 finv(z2, z2);
176 fmul(x2, x2, z2);
177 fe51_tobytes(out, x2);
178 }
179
180 void curve25519_arch(u8 mypublic[CURVE25519_KEY_SIZE],
181 const u8 secret[CURVE25519_KEY_SIZE],
182 const u8 basepoint[CURVE25519_KEY_SIZE])
183 {
184 curve25519_fe51(mypublic, secret, basepoint);
185 }
186 EXPORT_SYMBOL(curve25519_arch);
187
188 void curve25519_base_arch(u8 pub[CURVE25519_KEY_SIZE],
189 const u8 secret[CURVE25519_KEY_SIZE])
190 {
191 curve25519_fe51(pub, secret, curve25519_base_point);
192 }
193 EXPORT_SYMBOL(curve25519_base_arch);
194
195 static int curve25519_set_secret(struct crypto_kpp *tfm, const void *buf,
196 unsigned int len)
197 {
198 u8 *secret = kpp_tfm_ctx(tfm);
199
200 if (!len)
201 curve25519_generate_secret(secret);
202 else if (len == CURVE25519_KEY_SIZE &&
203 crypto_memneq(buf, curve25519_null_point, CURVE25519_KEY_SIZE))
204 memcpy(secret, buf, CURVE25519_KEY_SIZE);
205 else
206 return -EINVAL;
207 return 0;
208 }
209
210 static int curve25519_generate_public_key(struct kpp_request *req)
211 {
212 struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
213 const u8 *secret = kpp_tfm_ctx(tfm);
214 u8 buf[CURVE25519_KEY_SIZE];
215 int copied, nbytes;
216
217 if (req->src)
218 return -EINVAL;
219
220 curve25519_base_arch(buf, secret);
221
222 /* might want less than we've got */
223 nbytes = min_t(size_t, CURVE25519_KEY_SIZE, req->dst_len);
224 copied = sg_copy_from_buffer(req->dst, sg_nents_for_len(req->dst,
225 nbytes),
226 buf, nbytes);
227 if (copied != nbytes)
228 return -EINVAL;
229 return 0;
230 }
231
232 static int curve25519_compute_shared_secret(struct kpp_request *req)
233 {
234 struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
235 const u8 *secret = kpp_tfm_ctx(tfm);
236 u8 public_key[CURVE25519_KEY_SIZE];
237 u8 buf[CURVE25519_KEY_SIZE];
238 int copied, nbytes;
239
240 if (!req->src)
241 return -EINVAL;
242
243 copied = sg_copy_to_buffer(req->src,
244 sg_nents_for_len(req->src,
245 CURVE25519_KEY_SIZE),
246 public_key, CURVE25519_KEY_SIZE);
247 if (copied != CURVE25519_KEY_SIZE)
248 return -EINVAL;
249
250 curve25519_arch(buf, secret, public_key);
251
252 /* might want less than we've got */
253 nbytes = min_t(size_t, CURVE25519_KEY_SIZE, req->dst_len);
254 copied = sg_copy_from_buffer(req->dst, sg_nents_for_len(req->dst,
255 nbytes),
256 buf, nbytes);
257 if (copied != nbytes)
258 return -EINVAL;
259 return 0;
260 }
261
262 static unsigned int curve25519_max_size(struct crypto_kpp *tfm)
263 {
264 return CURVE25519_KEY_SIZE;
265 }
266
267 static struct kpp_alg curve25519_alg = {
268 .base.cra_name = "curve25519",
269 .base.cra_driver_name = "curve25519-ppc64le",
270 .base.cra_priority = 200,
271 .base.cra_module = THIS_MODULE,
272 .base.cra_ctxsize = CURVE25519_KEY_SIZE,
273
274 .set_secret = curve25519_set_secret,
275 .generate_public_key = curve25519_generate_public_key,
276 .compute_shared_secret = curve25519_compute_shared_secret,
277 .max_size = curve25519_max_size,
278 };
279
280
281 static int __init curve25519_mod_init(void)
282 {
283 return IS_REACHABLE(CONFIG_CRYPTO_KPP) ?
284 crypto_register_kpp(&curve25519_alg) : 0;
285 }
286
287 static void __exit curve25519_mod_exit(void)
288 {
289 if (IS_REACHABLE(CONFIG_CRYPTO_KPP))
290 crypto_unregister_kpp(&curve25519_alg);
291 }
292
293 module_init(curve25519_mod_init);
294 module_exit(curve25519_mod_exit);
295
296 MODULE_ALIAS_CRYPTO("curve25519");
297 MODULE_ALIAS_CRYPTO("curve25519-ppc64le");
298 MODULE_LICENSE("GPL v2");
299 MODULE_AUTHOR("Danny Tsen <dtsen@us.ibm.com>");
300
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.