1 /* SPDX-License-Identifier: GPL-2.0-or-later *
2 /*
3 * Cast5 Cipher 16-way parallel algorithm (AVX
4 *
5 * Copyright (C) 2012 Johannes Goetzfried
6 * <Johannes.Goetzfried@informatik.stud.uni
7 *
8 * Copyright © 2012 Jussi Kivilinna <jussi.kiv
9 */
10
11 #include <linux/linkage.h>
12 #include <asm/frame.h>
13
14 .file "cast5-avx-x86_64-asm_64.S"
15
16 .extern cast_s1
17 .extern cast_s2
18 .extern cast_s3
19 .extern cast_s4
20
21 /* structure of crypto context */
22 #define km 0
23 #define kr (16*4)
24 #define rr ((16*4)+16)
25
26 /* s-boxes */
27 #define s1 cast_s1
28 #define s2 cast_s2
29 #define s3 cast_s3
30 #define s4 cast_s4
31
32 /*********************************************
33 16-way AVX cast5
34 *********************************************
35 #define CTX %r15
36
37 #define RL1 %xmm0
38 #define RR1 %xmm1
39 #define RL2 %xmm2
40 #define RR2 %xmm3
41 #define RL3 %xmm4
42 #define RR3 %xmm5
43 #define RL4 %xmm6
44 #define RR4 %xmm7
45
46 #define RX %xmm8
47
48 #define RKM %xmm9
49 #define RKR %xmm10
50 #define RKRF %xmm11
51 #define RKRR %xmm12
52
53 #define R32 %xmm13
54 #define R1ST %xmm14
55
56 #define RTMP %xmm15
57
58 #define RID1 %rdi
59 #define RID1d %edi
60 #define RID2 %rsi
61 #define RID2d %esi
62
63 #define RGI1 %rdx
64 #define RGI1bl %dl
65 #define RGI1bh %dh
66 #define RGI2 %rcx
67 #define RGI2bl %cl
68 #define RGI2bh %ch
69
70 #define RGI3 %rax
71 #define RGI3bl %al
72 #define RGI3bh %ah
73 #define RGI4 %rbx
74 #define RGI4bl %bl
75 #define RGI4bh %bh
76
77 #define RFS1 %r8
78 #define RFS1d %r8d
79 #define RFS2 %r9
80 #define RFS2d %r9d
81 #define RFS3 %r10
82 #define RFS3d %r10d
83
84
85 #define lookup_32bit(src, dst, op1, op2, op3,
86 movzbl src ## bh, RID1d;
87 leaq s1(%rip), RID2;
88 movl (RID2,RID1,4), dst ##
89 movzbl src ## bl, RID2d;
90 leaq s2(%rip), RID1;
91 op1 (RID1,RID2,4), dst ##
92 shrq $16, src;
93 movzbl src ## bh, RID1d;
94 leaq s3(%rip), RID2;
95 op2 (RID2,RID1,4), dst ##
96 movzbl src ## bl, RID2d;
97 interleave_op(il_reg);
98 leaq s4(%rip), RID1;
99 op3 (RID1,RID2,4), dst ##
100
101 #define dummy(d) /* do nothing */
102
103 #define shr_next(reg) \
104 shrq $16, reg;
105
106 #define F_head(a, x, gi1, gi2, op0) \
107 op0 a, RKM, x;
108 vpslld RKRF, x, RTMP;
109 vpsrld RKRR, x, x;
110 vpor RTMP, x, x;
111 \
112 vmovq x, gi1;
113 vpextrq $1, x, gi2;
114
115 #define F_tail(a, x, gi1, gi2, op1, op2, op3)
116 lookup_32bit(##gi1, RFS1, op1, op2, op
117 lookup_32bit(##gi2, RFS3, op1, op2, op
118 \
119 lookup_32bit(##gi1, RFS2, op1, op2, op
120 shlq $32, RFS2;
121 orq RFS1, RFS2;
122 lookup_32bit(##gi2, RFS1, op1, op2, op
123 shlq $32, RFS1;
124 orq RFS1, RFS3;
125 \
126 vmovq RFS2, x;
127 vpinsrq $1, RFS3, x, x;
128
129 #define F_2(a1, b1, a2, b2, op0, op1, op2, op3
130 F_head(b1, RX, RGI1, RGI2, op0);
131 F_head(b2, RX, RGI3, RGI4, op0);
132 \
133 F_tail(b1, RX, RGI1, RGI2, op1, op2, o
134 F_tail(b2, RTMP, RGI3, RGI4, op1, op2,
135 \
136 vpxor a1, RX, a1;
137 vpxor a2, RTMP, a2;
138
139 #define F1_2(a1, b1, a2, b2) \
140 F_2(a1, b1, a2, b2, vpaddd, xorl, subl
141 #define F2_2(a1, b1, a2, b2) \
142 F_2(a1, b1, a2, b2, vpxor, subl, addl,
143 #define F3_2(a1, b1, a2, b2) \
144 F_2(a1, b1, a2, b2, vpsubd, addl, xorl
145
146 #define subround(a1, b1, a2, b2, f) \
147 F ## f ## _2(a1, b1, a2, b2);
148
149 #define round(l, r, n, f) \
150 vbroadcastss (km+(4*n))(CTX), RKM;
151 vpand R1ST, RKR,
152 vpsubq RKRF, R32,
153 vpsrldq $1, RKR, RKR;
154 subround(l ## 1, r ## 1, l ## 2, r ##
155 subround(l ## 3, r ## 3, l ## 4, r ##
156
157 #define enc_preload_rkr() \
158 vbroadcastss .L16_mask(%rip),
159 /* add 16-bit rotation to key rotation
160 vpxor kr(CTX),
161
162 #define dec_preload_rkr() \
163 vbroadcastss .L16_mask(%rip),
164 /* add 16-bit rotation to key rotation
165 vpxor kr(CTX),
166 vpshufb .Lbswap128_mask(%rip),
167
168 #define transpose_2x4(x0, x1, t0, t1) \
169 vpunpckldq x1, x0, t0; \
170 vpunpckhdq x1, x0, t1; \
171 \
172 vpunpcklqdq t1, t0, x0; \
173 vpunpckhqdq t1, t0, x1;
174
175 #define inpack_blocks(x0, x1, t0, t1, rmask) \
176 vpshufb rmask, x0, x0; \
177 vpshufb rmask, x1, x1; \
178 \
179 transpose_2x4(x0, x1, t0, t1)
180
181 #define outunpack_blocks(x0, x1, t0, t1, rmask
182 transpose_2x4(x0, x1, t0, t1) \
183 \
184 vpshufb rmask, x0, x0; \
185 vpshufb rmask, x1, x1;
186
187 .section .rodata.cst16.bswap_mask, "aM"
188 .align 16
189 .Lbswap_mask:
190 .byte 3, 2, 1, 0, 7, 6, 5, 4, 11, 10,
191 .section .rodata.cst16.bswap128_mask, "
192 .align 16
193 .Lbswap128_mask:
194 .byte 15, 14, 13, 12, 11, 10, 9, 8, 7,
195 .section .rodata.cst16.bswap_iv_mask, "
196 .align 16
197 .Lbswap_iv_mask:
198 .byte 7, 6, 5, 4, 3, 2, 1, 0, 7, 6, 5,
199
200 .section .rodata.cst4.16_mask, "aM", @p
201 .align 4
202 .L16_mask:
203 .byte 16, 16, 16, 16
204 .section .rodata.cst4.32_mask, "aM", @p
205 .align 4
206 .L32_mask:
207 .byte 32, 0, 0, 0
208 .section .rodata.cst4.first_mask, "aM",
209 .align 4
210 .Lfirst_mask:
211 .byte 0x1f, 0, 0, 0
212
213 .text
214
215 SYM_FUNC_START_LOCAL(__cast5_enc_blk16)
216 /* input:
217 * %rdi: ctx
218 * RL1: blocks 1 and 2
219 * RR1: blocks 3 and 4
220 * RL2: blocks 5 and 6
221 * RR2: blocks 7 and 8
222 * RL3: blocks 9 and 10
223 * RR3: blocks 11 and 12
224 * RL4: blocks 13 and 14
225 * RR4: blocks 15 and 16
226 * output:
227 * RL1: encrypted blocks 1 and 2
228 * RR1: encrypted blocks 3 and 4
229 * RL2: encrypted blocks 5 and 6
230 * RR2: encrypted blocks 7 and 8
231 * RL3: encrypted blocks 9 and 10
232 * RR3: encrypted blocks 11 and 1
233 * RL4: encrypted blocks 13 and 1
234 * RR4: encrypted blocks 15 and 1
235 */
236
237 pushq %r15;
238 pushq %rbx;
239
240 movq %rdi, CTX;
241
242 vmovdqa .Lbswap_mask(%rip), RKM;
243 vmovd .Lfirst_mask(%rip), R1ST;
244 vmovd .L32_mask(%rip), R32;
245 enc_preload_rkr();
246
247 inpack_blocks(RL1, RR1, RTMP, RX, RKM)
248 inpack_blocks(RL2, RR2, RTMP, RX, RKM)
249 inpack_blocks(RL3, RR3, RTMP, RX, RKM)
250 inpack_blocks(RL4, RR4, RTMP, RX, RKM)
251
252 round(RL, RR, 0, 1);
253 round(RR, RL, 1, 2);
254 round(RL, RR, 2, 3);
255 round(RR, RL, 3, 1);
256 round(RL, RR, 4, 2);
257 round(RR, RL, 5, 3);
258 round(RL, RR, 6, 1);
259 round(RR, RL, 7, 2);
260 round(RL, RR, 8, 3);
261 round(RR, RL, 9, 1);
262 round(RL, RR, 10, 2);
263 round(RR, RL, 11, 3);
264
265 movzbl rr(CTX), %eax;
266 testl %eax, %eax;
267 jnz .L__skip_enc;
268
269 round(RL, RR, 12, 1);
270 round(RR, RL, 13, 2);
271 round(RL, RR, 14, 3);
272 round(RR, RL, 15, 1);
273
274 .L__skip_enc:
275 popq %rbx;
276 popq %r15;
277
278 vmovdqa .Lbswap_mask(%rip), RKM;
279
280 outunpack_blocks(RR1, RL1, RTMP, RX, R
281 outunpack_blocks(RR2, RL2, RTMP, RX, R
282 outunpack_blocks(RR3, RL3, RTMP, RX, R
283 outunpack_blocks(RR4, RL4, RTMP, RX, R
284
285 RET;
286 SYM_FUNC_END(__cast5_enc_blk16)
287
288 SYM_FUNC_START_LOCAL(__cast5_dec_blk16)
289 /* input:
290 * %rdi: ctx
291 * RL1: encrypted blocks 1 and 2
292 * RR1: encrypted blocks 3 and 4
293 * RL2: encrypted blocks 5 and 6
294 * RR2: encrypted blocks 7 and 8
295 * RL3: encrypted blocks 9 and 10
296 * RR3: encrypted blocks 11 and 1
297 * RL4: encrypted blocks 13 and 1
298 * RR4: encrypted blocks 15 and 1
299 * output:
300 * RL1: decrypted blocks 1 and 2
301 * RR1: decrypted blocks 3 and 4
302 * RL2: decrypted blocks 5 and 6
303 * RR2: decrypted blocks 7 and 8
304 * RL3: decrypted blocks 9 and 10
305 * RR3: decrypted blocks 11 and 1
306 * RL4: decrypted blocks 13 and 1
307 * RR4: decrypted blocks 15 and 1
308 */
309
310 pushq %r15;
311 pushq %rbx;
312
313 movq %rdi, CTX;
314
315 vmovdqa .Lbswap_mask(%rip), RKM;
316 vmovd .Lfirst_mask(%rip), R1ST;
317 vmovd .L32_mask(%rip), R32;
318 dec_preload_rkr();
319
320 inpack_blocks(RL1, RR1, RTMP, RX, RKM)
321 inpack_blocks(RL2, RR2, RTMP, RX, RKM)
322 inpack_blocks(RL3, RR3, RTMP, RX, RKM)
323 inpack_blocks(RL4, RR4, RTMP, RX, RKM)
324
325 movzbl rr(CTX), %eax;
326 testl %eax, %eax;
327 jnz .L__skip_dec;
328
329 round(RL, RR, 15, 1);
330 round(RR, RL, 14, 3);
331 round(RL, RR, 13, 2);
332 round(RR, RL, 12, 1);
333
334 .L__dec_tail:
335 round(RL, RR, 11, 3);
336 round(RR, RL, 10, 2);
337 round(RL, RR, 9, 1);
338 round(RR, RL, 8, 3);
339 round(RL, RR, 7, 2);
340 round(RR, RL, 6, 1);
341 round(RL, RR, 5, 3);
342 round(RR, RL, 4, 2);
343 round(RL, RR, 3, 1);
344 round(RR, RL, 2, 3);
345 round(RL, RR, 1, 2);
346 round(RR, RL, 0, 1);
347
348 vmovdqa .Lbswap_mask(%rip), RKM;
349 popq %rbx;
350 popq %r15;
351
352 outunpack_blocks(RR1, RL1, RTMP, RX, R
353 outunpack_blocks(RR2, RL2, RTMP, RX, R
354 outunpack_blocks(RR3, RL3, RTMP, RX, R
355 outunpack_blocks(RR4, RL4, RTMP, RX, R
356
357 RET;
358
359 .L__skip_dec:
360 vpsrldq $4, RKR, RKR;
361 jmp .L__dec_tail;
362 SYM_FUNC_END(__cast5_dec_blk16)
363
364 SYM_FUNC_START(cast5_ecb_enc_16way)
365 /* input:
366 * %rdi: ctx
367 * %rsi: dst
368 * %rdx: src
369 */
370 FRAME_BEGIN
371 pushq %r15;
372
373 movq %rdi, CTX;
374 movq %rsi, %r11;
375
376 vmovdqu (0*4*4)(%rdx), RL1;
377 vmovdqu (1*4*4)(%rdx), RR1;
378 vmovdqu (2*4*4)(%rdx), RL2;
379 vmovdqu (3*4*4)(%rdx), RR2;
380 vmovdqu (4*4*4)(%rdx), RL3;
381 vmovdqu (5*4*4)(%rdx), RR3;
382 vmovdqu (6*4*4)(%rdx), RL4;
383 vmovdqu (7*4*4)(%rdx), RR4;
384
385 call __cast5_enc_blk16;
386
387 vmovdqu RR1, (0*4*4)(%r11);
388 vmovdqu RL1, (1*4*4)(%r11);
389 vmovdqu RR2, (2*4*4)(%r11);
390 vmovdqu RL2, (3*4*4)(%r11);
391 vmovdqu RR3, (4*4*4)(%r11);
392 vmovdqu RL3, (5*4*4)(%r11);
393 vmovdqu RR4, (6*4*4)(%r11);
394 vmovdqu RL4, (7*4*4)(%r11);
395
396 popq %r15;
397 FRAME_END
398 RET;
399 SYM_FUNC_END(cast5_ecb_enc_16way)
400
401 SYM_FUNC_START(cast5_ecb_dec_16way)
402 /* input:
403 * %rdi: ctx
404 * %rsi: dst
405 * %rdx: src
406 */
407
408 FRAME_BEGIN
409 pushq %r15;
410
411 movq %rdi, CTX;
412 movq %rsi, %r11;
413
414 vmovdqu (0*4*4)(%rdx), RL1;
415 vmovdqu (1*4*4)(%rdx), RR1;
416 vmovdqu (2*4*4)(%rdx), RL2;
417 vmovdqu (3*4*4)(%rdx), RR2;
418 vmovdqu (4*4*4)(%rdx), RL3;
419 vmovdqu (5*4*4)(%rdx), RR3;
420 vmovdqu (6*4*4)(%rdx), RL4;
421 vmovdqu (7*4*4)(%rdx), RR4;
422
423 call __cast5_dec_blk16;
424
425 vmovdqu RR1, (0*4*4)(%r11);
426 vmovdqu RL1, (1*4*4)(%r11);
427 vmovdqu RR2, (2*4*4)(%r11);
428 vmovdqu RL2, (3*4*4)(%r11);
429 vmovdqu RR3, (4*4*4)(%r11);
430 vmovdqu RL3, (5*4*4)(%r11);
431 vmovdqu RR4, (6*4*4)(%r11);
432 vmovdqu RL4, (7*4*4)(%r11);
433
434 popq %r15;
435 FRAME_END
436 RET;
437 SYM_FUNC_END(cast5_ecb_dec_16way)
438
439 SYM_FUNC_START(cast5_cbc_dec_16way)
440 /* input:
441 * %rdi: ctx
442 * %rsi: dst
443 * %rdx: src
444 */
445 FRAME_BEGIN
446 pushq %r12;
447 pushq %r15;
448
449 movq %rdi, CTX;
450 movq %rsi, %r11;
451 movq %rdx, %r12;
452
453 vmovdqu (0*16)(%rdx), RL1;
454 vmovdqu (1*16)(%rdx), RR1;
455 vmovdqu (2*16)(%rdx), RL2;
456 vmovdqu (3*16)(%rdx), RR2;
457 vmovdqu (4*16)(%rdx), RL3;
458 vmovdqu (5*16)(%rdx), RR3;
459 vmovdqu (6*16)(%rdx), RL4;
460 vmovdqu (7*16)(%rdx), RR4;
461
462 call __cast5_dec_blk16;
463
464 /* xor with src */
465 vmovq (%r12), RX;
466 vpshufd $0x4f, RX, RX;
467 vpxor RX, RR1, RR1;
468 vpxor 0*16+8(%r12), RL1, RL1;
469 vpxor 1*16+8(%r12), RR2, RR2;
470 vpxor 2*16+8(%r12), RL2, RL2;
471 vpxor 3*16+8(%r12), RR3, RR3;
472 vpxor 4*16+8(%r12), RL3, RL3;
473 vpxor 5*16+8(%r12), RR4, RR4;
474 vpxor 6*16+8(%r12), RL4, RL4;
475
476 vmovdqu RR1, (0*16)(%r11);
477 vmovdqu RL1, (1*16)(%r11);
478 vmovdqu RR2, (2*16)(%r11);
479 vmovdqu RL2, (3*16)(%r11);
480 vmovdqu RR3, (4*16)(%r11);
481 vmovdqu RL3, (5*16)(%r11);
482 vmovdqu RR4, (6*16)(%r11);
483 vmovdqu RL4, (7*16)(%r11);
484
485 popq %r15;
486 popq %r12;
487 FRAME_END
488 RET;
489 SYM_FUNC_END(cast5_cbc_dec_16way)
490
491 SYM_FUNC_START(cast5_ctr_16way)
492 /* input:
493 * %rdi: ctx
494 * %rsi: dst
495 * %rdx: src
496 * %rcx: iv (big endian, 64bit)
497 */
498 FRAME_BEGIN
499 pushq %r12;
500 pushq %r15;
501
502 movq %rdi, CTX;
503 movq %rsi, %r11;
504 movq %rdx, %r12;
505
506 vpcmpeqd RTMP, RTMP, RTMP;
507 vpsrldq $8, RTMP, RTMP; /* low: -1, hi
508
509 vpcmpeqd RKR, RKR, RKR;
510 vpaddq RKR, RKR, RKR; /* low: -2, high
511 vmovdqa .Lbswap_iv_mask(%rip), R1ST;
512 vmovdqa .Lbswap128_mask(%rip), RKM;
513
514 /* load IV and byteswap */
515 vmovq (%rcx), RX;
516 vpshufb R1ST, RX, RX;
517
518 /* construct IVs */
519 vpsubq RTMP, RX, RX; /* le: IV1, IV0
520 vpshufb RKM, RX, RL1; /* be: IV0, IV1
521 vpsubq RKR, RX, RX;
522 vpshufb RKM, RX, RR1; /* be: IV2, IV3
523 vpsubq RKR, RX, RX;
524 vpshufb RKM, RX, RL2; /* be: IV4, IV5
525 vpsubq RKR, RX, RX;
526 vpshufb RKM, RX, RR2; /* be: IV6, IV7
527 vpsubq RKR, RX, RX;
528 vpshufb RKM, RX, RL3; /* be: IV8, IV9
529 vpsubq RKR, RX, RX;
530 vpshufb RKM, RX, RR3; /* be: IV10, IV1
531 vpsubq RKR, RX, RX;
532 vpshufb RKM, RX, RL4; /* be: IV12, IV1
533 vpsubq RKR, RX, RX;
534 vpshufb RKM, RX, RR4; /* be: IV14, IV1
535
536 /* store last IV */
537 vpsubq RTMP, RX, RX; /* le: IV16, IV14
538 vpshufb R1ST, RX, RX; /* be: IV16, IV1
539 vmovq RX, (%rcx);
540
541 call __cast5_enc_blk16;
542
543 /* dst = src ^ iv */
544 vpxor (0*16)(%r12), RR1, RR1;
545 vpxor (1*16)(%r12), RL1, RL1;
546 vpxor (2*16)(%r12), RR2, RR2;
547 vpxor (3*16)(%r12), RL2, RL2;
548 vpxor (4*16)(%r12), RR3, RR3;
549 vpxor (5*16)(%r12), RL3, RL3;
550 vpxor (6*16)(%r12), RR4, RR4;
551 vpxor (7*16)(%r12), RL4, RL4;
552 vmovdqu RR1, (0*16)(%r11);
553 vmovdqu RL1, (1*16)(%r11);
554 vmovdqu RR2, (2*16)(%r11);
555 vmovdqu RL2, (3*16)(%r11);
556 vmovdqu RR3, (4*16)(%r11);
557 vmovdqu RL3, (5*16)(%r11);
558 vmovdqu RR4, (6*16)(%r11);
559 vmovdqu RL4, (7*16)(%r11);
560
561 popq %r15;
562 popq %r12;
563 FRAME_END
564 RET;
565 SYM_FUNC_END(cast5_ctr_16way)
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.