1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * Copyright (C) 2024 Christophe Leroy <christo
4 */
5
6 #include <linux/linkage.h>
7
8 #include <asm/ppc_asm.h>
9
10 #define dst_bytes r3
11 #define key r4
12 #define counter r5
13 #define nblocks r6
14
15 #define idx_r0 r0
16 #define val4 r4
17
18 #define const0 0x61707865
19 #define const1 0x3320646e
20 #define const2 0x79622d32
21 #define const3 0x6b206574
22
23 #define key0 r5
24 #define key1 r6
25 #define key2 r7
26 #define key3 r8
27 #define key4 r9
28 #define key5 r10
29 #define key6 r11
30 #define key7 r12
31
32 #define counter0 r14
33 #define counter1 r15
34
35 #define state0 r16
36 #define state1 r17
37 #define state2 r18
38 #define state3 r19
39 #define state4 r20
40 #define state5 r21
41 #define state6 r22
42 #define state7 r23
43 #define state8 r24
44 #define state9 r25
45 #define state10 r26
46 #define state11 r27
47 #define state12 r28
48 #define state13 r29
49 #define state14 r30
50 #define state15 r31
51
52 .macro quarterround4 a1 b1 c1 d1 a2 b2 c2 d2 a
53 add \a1, \a1, \b1
54 add \a2, \a2, \b2
55 add \a3, \a3, \b3
56 add \a4, \a4, \b4
57 xor \d1, \d1, \a1
58 xor \d2, \d2, \a2
59 xor \d3, \d3, \a3
60 xor \d4, \d4, \a4
61 rotlwi \d1, \d1, 16
62 rotlwi \d2, \d2, 16
63 rotlwi \d3, \d3, 16
64 rotlwi \d4, \d4, 16
65 add \c1, \c1, \d1
66 add \c2, \c2, \d2
67 add \c3, \c3, \d3
68 add \c4, \c4, \d4
69 xor \b1, \b1, \c1
70 xor \b2, \b2, \c2
71 xor \b3, \b3, \c3
72 xor \b4, \b4, \c4
73 rotlwi \b1, \b1, 12
74 rotlwi \b2, \b2, 12
75 rotlwi \b3, \b3, 12
76 rotlwi \b4, \b4, 12
77 add \a1, \a1, \b1
78 add \a2, \a2, \b2
79 add \a3, \a3, \b3
80 add \a4, \a4, \b4
81 xor \d1, \d1, \a1
82 xor \d2, \d2, \a2
83 xor \d3, \d3, \a3
84 xor \d4, \d4, \a4
85 rotlwi \d1, \d1, 8
86 rotlwi \d2, \d2, 8
87 rotlwi \d3, \d3, 8
88 rotlwi \d4, \d4, 8
89 add \c1, \c1, \d1
90 add \c2, \c2, \d2
91 add \c3, \c3, \d3
92 add \c4, \c4, \d4
93 xor \b1, \b1, \c1
94 xor \b2, \b2, \c2
95 xor \b3, \b3, \c3
96 xor \b4, \b4, \c4
97 rotlwi \b1, \b1, 7
98 rotlwi \b2, \b2, 7
99 rotlwi \b3, \b3, 7
100 rotlwi \b4, \b4, 7
101 .endm
102
103 #define QUARTERROUND4(a1,b1,c1,d1,a2,b2,c2,d2,
104 quarterround4 state##a1 state##b1 stat
105 state##a2 state##b2 stat
106 state##a3 state##b3 stat
107 state##a4 state##b4 stat
108
109 /*
110 * Very basic 32 bits implementation of ChaCha
111 * of blocks of output with a nonce of 0, taki
112 * counter. Importantly does not spill to the
113 *
114 * r3: output bytes
115 * r4: 32-byte key input
116 * r5: 8-byte counter input/output (saved
117 * r6: number of 64-byte blocks to write
118 *
119 * r0: counter of blocks (initialised wit
120 * r4: Value '4' after key has been read.
121 * r5-r12: key
122 * r14-r15: counter
123 * r16-r31: state
124 */
125 SYM_FUNC_START(__arch_chacha20_blocks_nostack)
126 #ifdef __powerpc64__
127 std counter, -216(r1)
128
129 std r14, -144(r1)
130 std r15, -136(r1)
131 std r16, -128(r1)
132 std r17, -120(r1)
133 std r18, -112(r1)
134 std r19, -104(r1)
135 std r20, -96(r1)
136 std r21, -88(r1)
137 std r22, -80(r1)
138 std r23, -72(r1)
139 std r24, -64(r1)
140 std r25, -56(r1)
141 std r26, -48(r1)
142 std r27, -40(r1)
143 std r28, -32(r1)
144 std r29, -24(r1)
145 std r30, -16(r1)
146 std r31, -8(r1)
147 #else
148 stwu r1, -96(r1)
149 stw counter, 20(r1)
150 #ifdef __BIG_ENDIAN__
151 stmw r14, 24(r1)
152 #else
153 stw r14, 24(r1)
154 stw r15, 28(r1)
155 stw r16, 32(r1)
156 stw r17, 36(r1)
157 stw r18, 40(r1)
158 stw r19, 44(r1)
159 stw r20, 48(r1)
160 stw r21, 52(r1)
161 stw r22, 56(r1)
162 stw r23, 60(r1)
163 stw r24, 64(r1)
164 stw r25, 68(r1)
165 stw r26, 72(r1)
166 stw r27, 76(r1)
167 stw r28, 80(r1)
168 stw r29, 84(r1)
169 stw r30, 88(r1)
170 stw r31, 92(r1)
171 #endif
172 #endif /* __powerpc64__ */
173
174 lwz counter0, 0(counter)
175 lwz counter1, 4(counter)
176 #ifdef __powerpc64__
177 rldimi counter0, counter1, 32, 0
178 #endif
179 mr idx_r0, nblocks
180 subi dst_bytes, dst_bytes, 4
181
182 lwz key0, 0(key)
183 lwz key1, 4(key)
184 lwz key2, 8(key)
185 lwz key3, 12(key)
186 lwz key4, 16(key)
187 lwz key5, 20(key)
188 lwz key6, 24(key)
189 lwz key7, 28(key)
190
191 li val4, 4
192 .Lblock:
193 li r31, 10
194
195 lis state0, const0@ha
196 lis state1, const1@ha
197 lis state2, const2@ha
198 lis state3, const3@ha
199 addi state0, state0, const0@l
200 addi state1, state1, const1@l
201 addi state2, state2, const2@l
202 addi state3, state3, const3@l
203
204 mtctr r31
205
206 mr state4, key0
207 mr state5, key1
208 mr state6, key2
209 mr state7, key3
210 mr state8, key4
211 mr state9, key5
212 mr state10, key6
213 mr state11, key7
214
215 mr state12, counter0
216 mr state13, counter1
217
218 li state14, 0
219 li state15, 0
220
221 .Lpermute:
222 QUARTERROUND4( 0, 4, 8,12, 1, 5, 9,13,
223 QUARTERROUND4( 0, 5,10,15, 1, 6,11,12,
224
225 bdnz .Lpermute
226
227 addis state0, state0, const0@ha
228 addis state1, state1, const1@ha
229 addis state2, state2, const2@ha
230 addis state3, state3, const3@ha
231 addi state0, state0, const0@l
232 addi state1, state1, const1@l
233 addi state2, state2, const2@l
234 addi state3, state3, const3@l
235
236 add state4, state4, key0
237 add state5, state5, key1
238 add state6, state6, key2
239 add state7, state7, key3
240 add state8, state8, key4
241 add state9, state9, key5
242 add state10, state10, key6
243 add state11, state11, key7
244
245 add state12, state12, counter0
246 add state13, state13, counter1
247
248 #ifdef __BIG_ENDIAN__
249 stwbrx state0, val4, dst_bytes
250 addi dst_bytes, dst_bytes, 8
251 stwbrx state1, 0, dst_bytes
252 stwbrx state2, val4, dst_bytes
253 addi dst_bytes, dst_bytes, 8
254 stwbrx state3, 0, dst_bytes
255 stwbrx state4, val4, dst_bytes
256 addi dst_bytes, dst_bytes, 8
257 stwbrx state5, 0, dst_bytes
258 stwbrx state6, val4, dst_bytes
259 addi dst_bytes, dst_bytes, 8
260 stwbrx state7, 0, dst_bytes
261 stwbrx state8, val4, dst_bytes
262 addi dst_bytes, dst_bytes, 8
263 stwbrx state9, 0, dst_bytes
264 stwbrx state10, val4, dst_bytes
265 addi dst_bytes, dst_bytes, 8
266 stwbrx state11, 0, dst_bytes
267 stwbrx state12, val4, dst_bytes
268 addi dst_bytes, dst_bytes, 8
269 stwbrx state13, 0, dst_bytes
270 stwbrx state14, val4, dst_bytes
271 addi dst_bytes, dst_bytes, 8
272 stwbrx state15, 0, dst_bytes
273 #else
274 stw state0, 4(dst_bytes)
275 stw state1, 8(dst_bytes)
276 stw state2, 12(dst_bytes)
277 stw state3, 16(dst_bytes)
278 stw state4, 20(dst_bytes)
279 stw state5, 24(dst_bytes)
280 stw state6, 28(dst_bytes)
281 stw state7, 32(dst_bytes)
282 stw state8, 36(dst_bytes)
283 stw state9, 40(dst_bytes)
284 stw state10, 44(dst_bytes)
285 stw state11, 48(dst_bytes)
286 stw state12, 52(dst_bytes)
287 stw state13, 56(dst_bytes)
288 stw state14, 60(dst_bytes)
289 stwu state15, 64(dst_bytes)
290 #endif
291
292 subic. idx_r0, idx_r0, 1 /* sub
293
294 #ifdef __powerpc64__
295 addi counter0, counter0, 1
296 srdi counter1, counter0, 32
297 #else
298 addic counter0, counter0, 1
299 addze counter1, counter1
300 #endif
301
302 bne .Lblock
303
304 #ifdef __powerpc64__
305 ld counter, -216(r1)
306 #else
307 lwz counter, 20(r1)
308 #endif
309 stw counter0, 0(counter)
310 stw counter1, 4(counter)
311
312 li r6, 0
313 li r7, 0
314 li r8, 0
315 li r9, 0
316 li r10, 0
317 li r11, 0
318 li r12, 0
319
320 #ifdef __powerpc64__
321 ld r14, -144(r1)
322 ld r15, -136(r1)
323 ld r16, -128(r1)
324 ld r17, -120(r1)
325 ld r18, -112(r1)
326 ld r19, -104(r1)
327 ld r20, -96(r1)
328 ld r21, -88(r1)
329 ld r22, -80(r1)
330 ld r23, -72(r1)
331 ld r24, -64(r1)
332 ld r25, -56(r1)
333 ld r26, -48(r1)
334 ld r27, -40(r1)
335 ld r28, -32(r1)
336 ld r29, -24(r1)
337 ld r30, -16(r1)
338 ld r31, -8(r1)
339 #else
340 #ifdef __BIG_ENDIAN__
341 lmw r14, 24(r1)
342 #else
343 lwz r14, 24(r1)
344 lwz r15, 28(r1)
345 lwz r16, 32(r1)
346 lwz r17, 36(r1)
347 lwz r18, 40(r1)
348 lwz r19, 44(r1)
349 lwz r20, 48(r1)
350 lwz r21, 52(r1)
351 lwz r22, 56(r1)
352 lwz r23, 60(r1)
353 lwz r24, 64(r1)
354 lwz r25, 68(r1)
355 lwz r26, 72(r1)
356 lwz r27, 76(r1)
357 lwz r28, 80(r1)
358 lwz r29, 84(r1)
359 lwz r30, 88(r1)
360 lwz r31, 92(r1)
361 #endif
362 addi r1, r1, 96
363 #endif /* __powerpc64__ */
364 blr
365 SYM_FUNC_END(__arch_chacha20_blocks_nostack)
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.