1 /* SPDX-License-Identifier: GPL-2.0-or-later *
2 /*
3 * Twofish Cipher 3-way parallel algorithm (x8
4 *
5 * Copyright (C) 2011 Jussi Kivilinna <jussi.ki
6 */
7
8 #include <linux/linkage.h>
9
10 .file "twofish-x86_64-asm-3way.S"
11 .text
12
13 /* structure of crypto context */
14 #define s0 0
15 #define s1 1024
16 #define s2 2048
17 #define s3 3072
18 #define w 4096
19 #define k 4128
20
21 /*********************************************
22 3-way twofish
23 *********************************************
24 #define CTX %rdi
25 #define RIO %rdx
26
27 #define RAB0 %rax
28 #define RAB1 %rbx
29 #define RAB2 %rcx
30
31 #define RAB0d %eax
32 #define RAB1d %ebx
33 #define RAB2d %ecx
34
35 #define RAB0bh %ah
36 #define RAB1bh %bh
37 #define RAB2bh %ch
38
39 #define RAB0bl %al
40 #define RAB1bl %bl
41 #define RAB2bl %cl
42
43 #define CD0 0x0(%rsp)
44 #define CD1 0x8(%rsp)
45 #define CD2 0x10(%rsp)
46
47 # used only before/after all rounds
48 #define RCD0 %r8
49 #define RCD1 %r9
50 #define RCD2 %r10
51
52 # used only during rounds
53 #define RX0 %r8
54 #define RX1 %r9
55 #define RX2 %r10
56
57 #define RX0d %r8d
58 #define RX1d %r9d
59 #define RX2d %r10d
60
61 #define RY0 %r11
62 #define RY1 %r12
63 #define RY2 %r13
64
65 #define RY0d %r11d
66 #define RY1d %r12d
67 #define RY2d %r13d
68
69 #define RT0 %rdx
70 #define RT1 %rsi
71
72 #define RT0d %edx
73 #define RT1d %esi
74
75 #define RT1bl %sil
76
77 #define do16bit_ror(rot, op1, op2, T0, T1, tmp
78 movzbl ab ## bl, tmp2 #
79 movzbl ab ## bh, tmp1 #
80 rorq $(rot), ab; \
81 op1##l T0(CTX, tmp2, 4), dst ##
82 op2##l T1(CTX, tmp1, 4), dst ##
83
84 #define swap_ab_with_cd(ab, cd, tmp) \
85 movq cd, tmp; \
86 movq ab, cd; \
87 movq tmp, ab;
88
89 /*
90 * Combined G1 & G2 function. Reordered with h
91 * at beginning.
92 */
93 #define g1g2_3(ab, cd, Tx0, Tx1, Tx2, Tx3, Ty0
94 /* G1,1 && G2,1 */ \
95 do16bit_ror(32, mov, xor, Tx0, Tx1, RT
96 do16bit_ror(48, mov, xor, Ty1, Ty2, RT
97 \
98 do16bit_ror(32, mov, xor, Tx0, Tx1, RT
99 do16bit_ror(48, mov, xor, Ty1, Ty2, RT
100 \
101 do16bit_ror(32, mov, xor, Tx0, Tx1, RT
102 do16bit_ror(48, mov, xor, Ty1, Ty2, RT
103 \
104 /* G1,2 && G2,2 */ \
105 do16bit_ror(32, xor, xor, Tx2, Tx3, RT
106 do16bit_ror(16, xor, xor, Ty3, Ty0, RT
107 swap_ab_with_cd(ab ## 0, cd ## 0, RT0)
108 \
109 do16bit_ror(32, xor, xor, Tx2, Tx3, RT
110 do16bit_ror(16, xor, xor, Ty3, Ty0, RT
111 swap_ab_with_cd(ab ## 1, cd ## 1, RT0)
112 \
113 do16bit_ror(32, xor, xor, Tx2, Tx3, RT
114 do16bit_ror(16, xor, xor, Ty3, Ty0, RT
115 swap_ab_with_cd(ab ## 2, cd ## 2, RT0)
116
117 #define enc_round_end(ab, x, y, n) \
118 addl y ## d, x ## d
119 addl x ## d, y ## d
120 addl k+4*(2*(n))(CTX), x ## d
121 xorl ab ## d, x ## d
122 addl k+4*(2*(n)+1)(CTX), y ## d
123 shrq $32, ab; \
124 roll $1, ab ##
125 xorl y ## d, ab ##
126 shlq $32, ab; \
127 rorl $1, x ## d
128 orq x, ab;
129
130 #define dec_round_end(ba, x, y, n) \
131 addl y ## d, x ## d
132 addl x ## d, y ## d
133 addl k+4*(2*(n))(CTX), x ## d
134 addl k+4*(2*(n)+1)(CTX), y ## d
135 xorl ba ## d, y ## d
136 shrq $32, ba; \
137 roll $1, ba ##
138 xorl x ## d, ba ##
139 shlq $32, ba; \
140 rorl $1, y ## d
141 orq y, ba;
142
143 #define encrypt_round3(ab, cd, n) \
144 g1g2_3(ab, cd, s0, s1, s2, s3, s0, s1,
145 \
146 enc_round_end(ab ## 0, RX0, RY0, n); \
147 enc_round_end(ab ## 1, RX1, RY1, n); \
148 enc_round_end(ab ## 2, RX2, RY2, n);
149
150 #define decrypt_round3(ba, dc, n) \
151 g1g2_3(ba, dc, s1, s2, s3, s0, s3, s0,
152 \
153 dec_round_end(ba ## 0, RX0, RY0, n); \
154 dec_round_end(ba ## 1, RX1, RY1, n); \
155 dec_round_end(ba ## 2, RX2, RY2, n);
156
157 #define encrypt_cycle3(ab, cd, n) \
158 encrypt_round3(ab, cd, n*2); \
159 encrypt_round3(ab, cd, (n*2)+1);
160
161 #define decrypt_cycle3(ba, dc, n) \
162 decrypt_round3(ba, dc, (n*2)+1); \
163 decrypt_round3(ba, dc, (n*2));
164
165 #define push_cd() \
166 pushq RCD2; \
167 pushq RCD1; \
168 pushq RCD0;
169
170 #define pop_cd() \
171 popq RCD0; \
172 popq RCD1; \
173 popq RCD2;
174
175 #define inpack3(in, n, xy, m) \
176 movq 4*(n)(in), xy ##
177 xorq w+4*m(CTX), xy ##
178 \
179 movq 4*(4+(n))(in), xy ##
180 xorq w+4*m(CTX), xy ##
181 \
182 movq 4*(8+(n))(in), xy ##
183 xorq w+4*m(CTX), xy ##
184
185 #define outunpack3(op, out, n, xy, m) \
186 xorq w+4*m(CTX), xy ##
187 op ## q xy ## 0, 4*(n)(
188 \
189 xorq w+4*m(CTX), xy ##
190 op ## q xy ## 1, 4*(4+(
191 \
192 xorq w+4*m(CTX), xy ##
193 op ## q xy ## 2, 4*(8+(
194
195 #define inpack_enc3() \
196 inpack3(RIO, 0, RAB, 0); \
197 inpack3(RIO, 2, RCD, 2);
198
199 #define outunpack_enc3(op) \
200 outunpack3(op, RIO, 2, RAB, 6); \
201 outunpack3(op, RIO, 0, RCD, 4);
202
203 #define inpack_dec3() \
204 inpack3(RIO, 0, RAB, 4); \
205 rorq $32, RAB0;
206 rorq $32, RAB1;
207 rorq $32, RAB2;
208 inpack3(RIO, 2, RCD, 6); \
209 rorq $32, RCD0;
210 rorq $32, RCD1;
211 rorq $32, RCD2;
212
213 #define outunpack_dec3() \
214 rorq $32, RCD0;
215 rorq $32, RCD1;
216 rorq $32, RCD2;
217 outunpack3(mov, RIO, 0, RCD, 0); \
218 rorq $32, RAB0;
219 rorq $32, RAB1;
220 rorq $32, RAB2;
221 outunpack3(mov, RIO, 2, RAB, 2);
222
223 SYM_FUNC_START(__twofish_enc_blk_3way)
224 /* input:
225 * %rdi: ctx, CTX
226 * %rsi: dst
227 * %rdx: src, RIO
228 * %rcx: bool, if true: xor outpu
229 */
230 pushq %r13;
231 pushq %r12;
232 pushq %rbx;
233
234 pushq %rcx; /* bool xor */
235 pushq %rsi; /* dst */
236
237 inpack_enc3();
238
239 push_cd();
240 encrypt_cycle3(RAB, CD, 0);
241 encrypt_cycle3(RAB, CD, 1);
242 encrypt_cycle3(RAB, CD, 2);
243 encrypt_cycle3(RAB, CD, 3);
244 encrypt_cycle3(RAB, CD, 4);
245 encrypt_cycle3(RAB, CD, 5);
246 encrypt_cycle3(RAB, CD, 6);
247 encrypt_cycle3(RAB, CD, 7);
248 pop_cd();
249
250 popq RIO; /* dst */
251 popq RT1; /* bool xor */
252
253 testb RT1bl, RT1bl;
254 jnz .L__enc_xor3;
255
256 outunpack_enc3(mov);
257
258 popq %rbx;
259 popq %r12;
260 popq %r13;
261 RET;
262
263 .L__enc_xor3:
264 outunpack_enc3(xor);
265
266 popq %rbx;
267 popq %r12;
268 popq %r13;
269 RET;
270 SYM_FUNC_END(__twofish_enc_blk_3way)
271
272 SYM_FUNC_START(twofish_dec_blk_3way)
273 /* input:
274 * %rdi: ctx, CTX
275 * %rsi: dst
276 * %rdx: src, RIO
277 */
278 pushq %r13;
279 pushq %r12;
280 pushq %rbx;
281
282 pushq %rsi; /* dst */
283
284 inpack_dec3();
285
286 push_cd();
287 decrypt_cycle3(RAB, CD, 7);
288 decrypt_cycle3(RAB, CD, 6);
289 decrypt_cycle3(RAB, CD, 5);
290 decrypt_cycle3(RAB, CD, 4);
291 decrypt_cycle3(RAB, CD, 3);
292 decrypt_cycle3(RAB, CD, 2);
293 decrypt_cycle3(RAB, CD, 1);
294 decrypt_cycle3(RAB, CD, 0);
295 pop_cd();
296
297 popq RIO; /* dst */
298
299 outunpack_dec3();
300
301 popq %rbx;
302 popq %r12;
303 popq %r13;
304 RET;
305 SYM_FUNC_END(twofish_dec_blk_3way)
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.