1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * Accelerated GHASH implementation with NEON/ARMv8 vmull.p8/64 instructions.
4 *
5 * Copyright (C) 2015 - 2017 Linaro Ltd.
6 * Copyright (C) 2023 Google LLC. <ardb@google.com>
7 */
8
9 #include <linux/linkage.h>
10 #include <asm/assembler.h>
11
12 .arch armv8-a
13 .fpu crypto-neon-fp-armv8
14
15 SHASH .req q0
16 T1 .req q1
17 XL .req q2
18 XM .req q3
19 XH .req q4
20 IN1 .req q4
21
22 SHASH_L .req d0
23 SHASH_H .req d1
24 T1_L .req d2
25 T1_H .req d3
26 XL_L .req d4
27 XL_H .req d5
28 XM_L .req d6
29 XM_H .req d7
30 XH_L .req d8
31
32 t0l .req d10
33 t0h .req d11
34 t1l .req d12
35 t1h .req d13
36 t2l .req d14
37 t2h .req d15
38 t3l .req d16
39 t3h .req d17
40 t4l .req d18
41 t4h .req d19
42
43 t0q .req q5
44 t1q .req q6
45 t2q .req q7
46 t3q .req q8
47 t4q .req q9
48 XH2 .req q9
49
50 s1l .req d20
51 s1h .req d21
52 s2l .req d22
53 s2h .req d23
54 s3l .req d24
55 s3h .req d25
56 s4l .req d26
57 s4h .req d27
58
59 MASK .req d28
60 SHASH2_p8 .req d28
61
62 k16 .req d29
63 k32 .req d30
64 k48 .req d31
65 SHASH2_p64 .req d31
66
67 HH .req q10
68 HH3 .req q11
69 HH4 .req q12
70 HH34 .req q13
71
72 HH_L .req d20
73 HH_H .req d21
74 HH3_L .req d22
75 HH3_H .req d23
76 HH4_L .req d24
77 HH4_H .req d25
78 HH34_L .req d26
79 HH34_H .req d27
80 SHASH2_H .req d29
81
82 XL2 .req q5
83 XM2 .req q6
84 T2 .req q7
85 T3 .req q8
86
87 XL2_L .req d10
88 XL2_H .req d11
89 XM2_L .req d12
90 XM2_H .req d13
91 T3_L .req d16
92 T3_H .req d17
93
94 .text
95
96 .macro __pmull_p64, rd, rn, rm, b1, b2, b3, b4
97 vmull.p64 \rd, \rn, \rm
98 .endm
99
100 /*
101 * This implementation of 64x64 -> 128 bit polynomial multiplication
102 * using vmull.p8 instructions (8x8 -> 16) is taken from the paper
103 * "Fast Software Polynomial Multiplication on ARM Processors Using
104 * the NEON Engine" by Danilo Camara, Conrado Gouvea, Julio Lopez and
105 * Ricardo Dahab (https://hal.inria.fr/hal-01506572)
106 *
107 * It has been slightly tweaked for in-order performance, and to allow
108 * 'rq' to overlap with 'ad' or 'bd'.
109 */
110 .macro __pmull_p8, rq, ad, bd, b1=t4l, b2=t3l, b3=t4l, b4=t3l
111 vext.8 t0l, \ad, \ad, #1 @ A1
112 .ifc \b1, t4l
113 vext.8 t4l, \bd, \bd, #1 @ B1
114 .endif
115 vmull.p8 t0q, t0l, \bd @ F = A1*B
116 vext.8 t1l, \ad, \ad, #2 @ A2
117 vmull.p8 t4q, \ad, \b1 @ E = A*B1
118 .ifc \b2, t3l
119 vext.8 t3l, \bd, \bd, #2 @ B2
120 .endif
121 vmull.p8 t1q, t1l, \bd @ H = A2*B
122 vext.8 t2l, \ad, \ad, #3 @ A3
123 vmull.p8 t3q, \ad, \b2 @ G = A*B2
124 veor t0q, t0q, t4q @ L = E + F
125 .ifc \b3, t4l
126 vext.8 t4l, \bd, \bd, #3 @ B3
127 .endif
128 vmull.p8 t2q, t2l, \bd @ J = A3*B
129 veor t0l, t0l, t0h @ t0 = (L) (P0 + P1) << 8
130 veor t1q, t1q, t3q @ M = G + H
131 .ifc \b4, t3l
132 vext.8 t3l, \bd, \bd, #4 @ B4
133 .endif
134 vmull.p8 t4q, \ad, \b3 @ I = A*B3
135 veor t1l, t1l, t1h @ t1 = (M) (P2 + P3) << 16
136 vmull.p8 t3q, \ad, \b4 @ K = A*B4
137 vand t0h, t0h, k48
138 vand t1h, t1h, k32
139 veor t2q, t2q, t4q @ N = I + J
140 veor t0l, t0l, t0h
141 veor t1l, t1l, t1h
142 veor t2l, t2l, t2h @ t2 = (N) (P4 + P5) << 24
143 vand t2h, t2h, k16
144 veor t3l, t3l, t3h @ t3 = (K) (P6 + P7) << 32
145 vmov.i64 t3h, #0
146 vext.8 t0q, t0q, t0q, #15
147 veor t2l, t2l, t2h
148 vext.8 t1q, t1q, t1q, #14
149 vmull.p8 \rq, \ad, \bd @ D = A*B
150 vext.8 t2q, t2q, t2q, #13
151 vext.8 t3q, t3q, t3q, #12
152 veor t0q, t0q, t1q
153 veor t2q, t2q, t3q
154 veor \rq, \rq, t0q
155 veor \rq, \rq, t2q
156 .endm
157
158 //
159 // PMULL (64x64->128) based reduction for CPUs that can do
160 // it in a single instruction.
161 //
162 .macro __pmull_reduce_p64
163 vmull.p64 T1, XL_L, MASK
164
165 veor XH_L, XH_L, XM_H
166 vext.8 T1, T1, T1, #8
167 veor XL_H, XL_H, XM_L
168 veor T1, T1, XL
169
170 vmull.p64 XL, T1_H, MASK
171 .endm
172
173 //
174 // Alternative reduction for CPUs that lack support for the
175 // 64x64->128 PMULL instruction
176 //
177 .macro __pmull_reduce_p8
178 veor XL_H, XL_H, XM_L
179 veor XH_L, XH_L, XM_H
180
181 vshl.i64 T1, XL, #57
182 vshl.i64 T2, XL, #62
183 veor T1, T1, T2
184 vshl.i64 T2, XL, #63
185 veor T1, T1, T2
186 veor XL_H, XL_H, T1_L
187 veor XH_L, XH_L, T1_H
188
189 vshr.u64 T1, XL, #1
190 veor XH, XH, XL
191 veor XL, XL, T1
192 vshr.u64 T1, T1, #6
193 vshr.u64 XL, XL, #1
194 .endm
195
196 .macro ghash_update, pn, enc, aggregate=1, head=1
197 vld1.64 {XL}, [r1]
198
199 .if \head
200 /* do the head block first, if supplied */
201 ldr ip, [sp]
202 teq ip, #0
203 beq 0f
204 vld1.64 {T1}, [ip]
205 teq r0, #0
206 b 3f
207 .endif
208
209 0: .ifc \pn, p64
210 .if \aggregate
211 tst r0, #3 // skip until #blocks is a
212 bne 2f // round multiple of 4
213
214 vld1.8 {XL2-XM2}, [r2]!
215 1: vld1.8 {T2-T3}, [r2]!
216
217 .ifnb \enc
218 \enc\()_4x XL2, XM2, T2, T3
219
220 add ip, r3, #16
221 vld1.64 {HH}, [ip, :128]!
222 vld1.64 {HH3-HH4}, [ip, :128]
223
224 veor SHASH2_p64, SHASH_L, SHASH_H
225 veor SHASH2_H, HH_L, HH_H
226 veor HH34_L, HH3_L, HH3_H
227 veor HH34_H, HH4_L, HH4_H
228
229 vmov.i8 MASK, #0xe1
230 vshl.u64 MASK, MASK, #57
231 .endif
232
233 vrev64.8 XL2, XL2
234 vrev64.8 XM2, XM2
235
236 subs r0, r0, #4
237
238 vext.8 T1, XL2, XL2, #8
239 veor XL2_H, XL2_H, XL_L
240 veor XL, XL, T1
241
242 vrev64.8 T1, T3
243 vrev64.8 T3, T2
244
245 vmull.p64 XH, HH4_H, XL_H // a1 * b1
246 veor XL2_H, XL2_H, XL_H
247 vmull.p64 XL, HH4_L, XL_L // a0 * b0
248 vmull.p64 XM, HH34_H, XL2_H // (a1 + a0)(b1 + b0)
249
250 vmull.p64 XH2, HH3_H, XM2_L // a1 * b1
251 veor XM2_L, XM2_L, XM2_H
252 vmull.p64 XL2, HH3_L, XM2_H // a0 * b0
253 vmull.p64 XM2, HH34_L, XM2_L // (a1 + a0)(b1 + b0)
254
255 veor XH, XH, XH2
256 veor XL, XL, XL2
257 veor XM, XM, XM2
258
259 vmull.p64 XH2, HH_H, T3_L // a1 * b1
260 veor T3_L, T3_L, T3_H
261 vmull.p64 XL2, HH_L, T3_H // a0 * b0
262 vmull.p64 XM2, SHASH2_H, T3_L // (a1 + a0)(b1 + b0)
263
264 veor XH, XH, XH2
265 veor XL, XL, XL2
266 veor XM, XM, XM2
267
268 vmull.p64 XH2, SHASH_H, T1_L // a1 * b1
269 veor T1_L, T1_L, T1_H
270 vmull.p64 XL2, SHASH_L, T1_H // a0 * b0
271 vmull.p64 XM2, SHASH2_p64, T1_L // (a1 + a0)(b1 + b0)
272
273 veor XH, XH, XH2
274 veor XL, XL, XL2
275 veor XM, XM, XM2
276
277 beq 4f
278
279 vld1.8 {XL2-XM2}, [r2]!
280
281 veor T1, XL, XH
282 veor XM, XM, T1
283
284 __pmull_reduce_p64
285
286 veor T1, T1, XH
287 veor XL, XL, T1
288
289 b 1b
290 .endif
291 .endif
292
293 2: vld1.8 {T1}, [r2]!
294
295 .ifnb \enc
296 \enc\()_1x T1
297 veor SHASH2_p64, SHASH_L, SHASH_H
298 vmov.i8 MASK, #0xe1
299 vshl.u64 MASK, MASK, #57
300 .endif
301
302 subs r0, r0, #1
303
304 3: /* multiply XL by SHASH in GF(2^128) */
305 vrev64.8 T1, T1
306
307 vext.8 IN1, T1, T1, #8
308 veor T1_L, T1_L, XL_H
309 veor XL, XL, IN1
310
311 __pmull_\pn XH, XL_H, SHASH_H, s1h, s2h, s3h, s4h @ a1 * b1
312 veor T1, T1, XL
313 __pmull_\pn XL, XL_L, SHASH_L, s1l, s2l, s3l, s4l @ a0 * b0
314 __pmull_\pn XM, T1_L, SHASH2_\pn @ (a1+a0)(b1+b0)
315
316 4: veor T1, XL, XH
317 veor XM, XM, T1
318
319 __pmull_reduce_\pn
320
321 veor T1, T1, XH
322 veor XL, XL, T1
323
324 bne 0b
325 .endm
326
327 /*
328 * void pmull_ghash_update(int blocks, u64 dg[], const char *src,
329 * struct ghash_key const *k, const char *head)
330 */
331 ENTRY(pmull_ghash_update_p64)
332 vld1.64 {SHASH}, [r3]!
333 vld1.64 {HH}, [r3]!
334 vld1.64 {HH3-HH4}, [r3]
335
336 veor SHASH2_p64, SHASH_L, SHASH_H
337 veor SHASH2_H, HH_L, HH_H
338 veor HH34_L, HH3_L, HH3_H
339 veor HH34_H, HH4_L, HH4_H
340
341 vmov.i8 MASK, #0xe1
342 vshl.u64 MASK, MASK, #57
343
344 ghash_update p64
345 vst1.64 {XL}, [r1]
346
347 bx lr
348 ENDPROC(pmull_ghash_update_p64)
349
350 ENTRY(pmull_ghash_update_p8)
351 vld1.64 {SHASH}, [r3]
352 veor SHASH2_p8, SHASH_L, SHASH_H
353
354 vext.8 s1l, SHASH_L, SHASH_L, #1
355 vext.8 s2l, SHASH_L, SHASH_L, #2
356 vext.8 s3l, SHASH_L, SHASH_L, #3
357 vext.8 s4l, SHASH_L, SHASH_L, #4
358 vext.8 s1h, SHASH_H, SHASH_H, #1
359 vext.8 s2h, SHASH_H, SHASH_H, #2
360 vext.8 s3h, SHASH_H, SHASH_H, #3
361 vext.8 s4h, SHASH_H, SHASH_H, #4
362
363 vmov.i64 k16, #0xffff
364 vmov.i64 k32, #0xffffffff
365 vmov.i64 k48, #0xffffffffffff
366
367 ghash_update p8
368 vst1.64 {XL}, [r1]
369
370 bx lr
371 ENDPROC(pmull_ghash_update_p8)
372
373 e0 .req q9
374 e1 .req q10
375 e2 .req q11
376 e3 .req q12
377 e0l .req d18
378 e0h .req d19
379 e2l .req d22
380 e2h .req d23
381 e3l .req d24
382 e3h .req d25
383 ctr .req q13
384 ctr0 .req d26
385 ctr1 .req d27
386
387 ek0 .req q14
388 ek1 .req q15
389
390 .macro round, rk:req, regs:vararg
391 .irp r, \regs
392 aese.8 \r, \rk
393 aesmc.8 \r, \r
394 .endr
395 .endm
396
397 .macro aes_encrypt, rkp, rounds, regs:vararg
398 vld1.8 {ek0-ek1}, [\rkp, :128]!
399 cmp \rounds, #12
400 blt .L\@ // AES-128
401
402 round ek0, \regs
403 vld1.8 {ek0}, [\rkp, :128]!
404 round ek1, \regs
405 vld1.8 {ek1}, [\rkp, :128]!
406
407 beq .L\@ // AES-192
408
409 round ek0, \regs
410 vld1.8 {ek0}, [\rkp, :128]!
411 round ek1, \regs
412 vld1.8 {ek1}, [\rkp, :128]!
413
414 .L\@: .rept 4
415 round ek0, \regs
416 vld1.8 {ek0}, [\rkp, :128]!
417 round ek1, \regs
418 vld1.8 {ek1}, [\rkp, :128]!
419 .endr
420
421 round ek0, \regs
422 vld1.8 {ek0}, [\rkp, :128]
423
424 .irp r, \regs
425 aese.8 \r, ek1
426 .endr
427 .irp r, \regs
428 veor \r, \r, ek0
429 .endr
430 .endm
431
432 pmull_aes_encrypt:
433 add ip, r5, #4
434 vld1.8 {ctr0}, [r5] // load 12 byte IV
435 vld1.8 {ctr1}, [ip]
436 rev r8, r7
437 vext.8 ctr1, ctr1, ctr1, #4
438 add r7, r7, #1
439 vmov.32 ctr1[1], r8
440 vmov e0, ctr
441
442 add ip, r3, #64
443 aes_encrypt ip, r6, e0
444 bx lr
445 ENDPROC(pmull_aes_encrypt)
446
447 pmull_aes_encrypt_4x:
448 add ip, r5, #4
449 vld1.8 {ctr0}, [r5]
450 vld1.8 {ctr1}, [ip]
451 rev r8, r7
452 vext.8 ctr1, ctr1, ctr1, #4
453 add r7, r7, #1
454 vmov.32 ctr1[1], r8
455 rev ip, r7
456 vmov e0, ctr
457 add r7, r7, #1
458 vmov.32 ctr1[1], ip
459 rev r8, r7
460 vmov e1, ctr
461 add r7, r7, #1
462 vmov.32 ctr1[1], r8
463 rev ip, r7
464 vmov e2, ctr
465 add r7, r7, #1
466 vmov.32 ctr1[1], ip
467 vmov e3, ctr
468
469 add ip, r3, #64
470 aes_encrypt ip, r6, e0, e1, e2, e3
471 bx lr
472 ENDPROC(pmull_aes_encrypt_4x)
473
474 pmull_aes_encrypt_final:
475 add ip, r5, #4
476 vld1.8 {ctr0}, [r5]
477 vld1.8 {ctr1}, [ip]
478 rev r8, r7
479 vext.8 ctr1, ctr1, ctr1, #4
480 mov r7, #1 << 24 // BE #1 for the tag
481 vmov.32 ctr1[1], r8
482 vmov e0, ctr
483 vmov.32 ctr1[1], r7
484 vmov e1, ctr
485
486 add ip, r3, #64
487 aes_encrypt ip, r6, e0, e1
488 bx lr
489 ENDPROC(pmull_aes_encrypt_final)
490
491 .macro enc_1x, in0
492 bl pmull_aes_encrypt
493 veor \in0, \in0, e0
494 vst1.8 {\in0}, [r4]!
495 .endm
496
497 .macro dec_1x, in0
498 bl pmull_aes_encrypt
499 veor e0, e0, \in0
500 vst1.8 {e0}, [r4]!
501 .endm
502
503 .macro enc_4x, in0, in1, in2, in3
504 bl pmull_aes_encrypt_4x
505
506 veor \in0, \in0, e0
507 veor \in1, \in1, e1
508 veor \in2, \in2, e2
509 veor \in3, \in3, e3
510
511 vst1.8 {\in0-\in1}, [r4]!
512 vst1.8 {\in2-\in3}, [r4]!
513 .endm
514
515 .macro dec_4x, in0, in1, in2, in3
516 bl pmull_aes_encrypt_4x
517
518 veor e0, e0, \in0
519 veor e1, e1, \in1
520 veor e2, e2, \in2
521 veor e3, e3, \in3
522
523 vst1.8 {e0-e1}, [r4]!
524 vst1.8 {e2-e3}, [r4]!
525 .endm
526
527 /*
528 * void pmull_gcm_encrypt(int blocks, u64 dg[], const char *src,
529 * struct gcm_key const *k, char *dst,
530 * char *iv, int rounds, u32 counter)
531 */
532 ENTRY(pmull_gcm_encrypt)
533 push {r4-r8, lr}
534 ldrd r4, r5, [sp, #24]
535 ldrd r6, r7, [sp, #32]
536
537 vld1.64 {SHASH}, [r3]
538
539 ghash_update p64, enc, head=0
540 vst1.64 {XL}, [r1]
541
542 pop {r4-r8, pc}
543 ENDPROC(pmull_gcm_encrypt)
544
545 /*
546 * void pmull_gcm_decrypt(int blocks, u64 dg[], const char *src,
547 * struct gcm_key const *k, char *dst,
548 * char *iv, int rounds, u32 counter)
549 */
550 ENTRY(pmull_gcm_decrypt)
551 push {r4-r8, lr}
552 ldrd r4, r5, [sp, #24]
553 ldrd r6, r7, [sp, #32]
554
555 vld1.64 {SHASH}, [r3]
556
557 ghash_update p64, dec, head=0
558 vst1.64 {XL}, [r1]
559
560 pop {r4-r8, pc}
561 ENDPROC(pmull_gcm_decrypt)
562
563 /*
564 * void pmull_gcm_enc_final(int bytes, u64 dg[], char *tag,
565 * struct gcm_key const *k, char *head,
566 * char *iv, int rounds, u32 counter)
567 */
568 ENTRY(pmull_gcm_enc_final)
569 push {r4-r8, lr}
570 ldrd r4, r5, [sp, #24]
571 ldrd r6, r7, [sp, #32]
572
573 bl pmull_aes_encrypt_final
574
575 cmp r0, #0
576 beq .Lenc_final
577
578 mov_l ip, .Lpermute
579 sub r4, r4, #16
580 add r8, ip, r0
581 add ip, ip, #32
582 add r4, r4, r0
583 sub ip, ip, r0
584
585 vld1.8 {e3}, [r8] // permute vector for key stream
586 vld1.8 {e2}, [ip] // permute vector for ghash input
587
588 vtbl.8 e3l, {e0}, e3l
589 vtbl.8 e3h, {e0}, e3h
590
591 vld1.8 {e0}, [r4] // encrypt tail block
592 veor e0, e0, e3
593 vst1.8 {e0}, [r4]
594
595 vtbl.8 T1_L, {e0}, e2l
596 vtbl.8 T1_H, {e0}, e2h
597
598 vld1.64 {XL}, [r1]
599 .Lenc_final:
600 vld1.64 {SHASH}, [r3, :128]
601 vmov.i8 MASK, #0xe1
602 veor SHASH2_p64, SHASH_L, SHASH_H
603 vshl.u64 MASK, MASK, #57
604 mov r0, #1
605 bne 3f // process head block first
606 ghash_update p64, aggregate=0, head=0
607
608 vrev64.8 XL, XL
609 vext.8 XL, XL, XL, #8
610 veor XL, XL, e1
611
612 sub r2, r2, #16 // rewind src pointer
613 vst1.8 {XL}, [r2] // store tag
614
615 pop {r4-r8, pc}
616 ENDPROC(pmull_gcm_enc_final)
617
618 /*
619 * int pmull_gcm_dec_final(int bytes, u64 dg[], char *tag,
620 * struct gcm_key const *k, char *head,
621 * char *iv, int rounds, u32 counter,
622 * const char *otag, int authsize)
623 */
624 ENTRY(pmull_gcm_dec_final)
625 push {r4-r8, lr}
626 ldrd r4, r5, [sp, #24]
627 ldrd r6, r7, [sp, #32]
628
629 bl pmull_aes_encrypt_final
630
631 cmp r0, #0
632 beq .Ldec_final
633
634 mov_l ip, .Lpermute
635 sub r4, r4, #16
636 add r8, ip, r0
637 add ip, ip, #32
638 add r4, r4, r0
639 sub ip, ip, r0
640
641 vld1.8 {e3}, [r8] // permute vector for key stream
642 vld1.8 {e2}, [ip] // permute vector for ghash input
643
644 vtbl.8 e3l, {e0}, e3l
645 vtbl.8 e3h, {e0}, e3h
646
647 vld1.8 {e0}, [r4]
648
649 vtbl.8 T1_L, {e0}, e2l
650 vtbl.8 T1_H, {e0}, e2h
651
652 veor e0, e0, e3
653 vst1.8 {e0}, [r4]
654
655 vld1.64 {XL}, [r1]
656 .Ldec_final:
657 vld1.64 {SHASH}, [r3]
658 vmov.i8 MASK, #0xe1
659 veor SHASH2_p64, SHASH_L, SHASH_H
660 vshl.u64 MASK, MASK, #57
661 mov r0, #1
662 bne 3f // process head block first
663 ghash_update p64, aggregate=0, head=0
664
665 vrev64.8 XL, XL
666 vext.8 XL, XL, XL, #8
667 veor XL, XL, e1
668
669 mov_l ip, .Lpermute
670 ldrd r2, r3, [sp, #40] // otag and authsize
671 vld1.8 {T1}, [r2]
672 add ip, ip, r3
673 vceq.i8 T1, T1, XL // compare tags
674 vmvn T1, T1 // 0 for eq, -1 for ne
675
676 vld1.8 {e0}, [ip]
677 vtbl.8 XL_L, {T1}, e0l // keep authsize bytes only
678 vtbl.8 XL_H, {T1}, e0h
679
680 vpmin.s8 XL_L, XL_L, XL_H // take the minimum s8 across the vector
681 vpmin.s8 XL_L, XL_L, XL_L
682 vmov.32 r0, XL_L[0] // fail if != 0x0
683
684 pop {r4-r8, pc}
685 ENDPROC(pmull_gcm_dec_final)
686
687 .section ".rodata", "a", %progbits
688 .align 5
689 .Lpermute:
690 .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
691 .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
692 .byte 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07
693 .byte 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f
694 .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
695 .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.