1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * sha3-ce-core.S - core SHA-3 transform using v8.2 Crypto Extensions
4 *
5 * Copyright (C) 2018 Linaro Ltd <ard.biesheuvel@linaro.org>
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 */
11
12 #include <linux/linkage.h>
13 #include <asm/assembler.h>
14
15 .irp b,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31
16 .set .Lv\b\().2d, \b
17 .set .Lv\b\().16b, \b
18 .endr
19
20 /*
21 * ARMv8.2 Crypto Extensions instructions
22 */
23 .macro eor3, rd, rn, rm, ra
24 .inst 0xce000000 | .L\rd | (.L\rn << 5) | (.L\ra << 10) | (.L\rm << 16)
25 .endm
26
27 .macro rax1, rd, rn, rm
28 .inst 0xce608c00 | .L\rd | (.L\rn << 5) | (.L\rm << 16)
29 .endm
30
31 .macro bcax, rd, rn, rm, ra
32 .inst 0xce200000 | .L\rd | (.L\rn << 5) | (.L\ra << 10) | (.L\rm << 16)
33 .endm
34
35 .macro xar, rd, rn, rm, imm6
36 .inst 0xce800000 | .L\rd | (.L\rn << 5) | ((\imm6) << 10) | (.L\rm << 16)
37 .endm
38
39 /*
40 * int sha3_ce_transform(u64 *st, const u8 *data, int blocks, int dg_size)
41 */
42 .text
43 SYM_FUNC_START(sha3_ce_transform)
44 /* load state */
45 add x8, x0, #32
46 ld1 { v0.1d- v3.1d}, [x0]
47 ld1 { v4.1d- v7.1d}, [x8], #32
48 ld1 { v8.1d-v11.1d}, [x8], #32
49 ld1 {v12.1d-v15.1d}, [x8], #32
50 ld1 {v16.1d-v19.1d}, [x8], #32
51 ld1 {v20.1d-v23.1d}, [x8], #32
52 ld1 {v24.1d}, [x8]
53
54 0: sub w2, w2, #1
55 mov w8, #24
56 adr_l x9, .Lsha3_rcon
57
58 /* load input */
59 ld1 {v25.8b-v28.8b}, [x1], #32
60 ld1 {v29.8b-v31.8b}, [x1], #24
61 eor v0.8b, v0.8b, v25.8b
62 eor v1.8b, v1.8b, v26.8b
63 eor v2.8b, v2.8b, v27.8b
64 eor v3.8b, v3.8b, v28.8b
65 eor v4.8b, v4.8b, v29.8b
66 eor v5.8b, v5.8b, v30.8b
67 eor v6.8b, v6.8b, v31.8b
68
69 tbnz x3, #6, 2f // SHA3-512
70
71 ld1 {v25.8b-v28.8b}, [x1], #32
72 ld1 {v29.8b-v30.8b}, [x1], #16
73 eor v7.8b, v7.8b, v25.8b
74 eor v8.8b, v8.8b, v26.8b
75 eor v9.8b, v9.8b, v27.8b
76 eor v10.8b, v10.8b, v28.8b
77 eor v11.8b, v11.8b, v29.8b
78 eor v12.8b, v12.8b, v30.8b
79
80 tbnz x3, #4, 1f // SHA3-384 or SHA3-224
81
82 // SHA3-256
83 ld1 {v25.8b-v28.8b}, [x1], #32
84 eor v13.8b, v13.8b, v25.8b
85 eor v14.8b, v14.8b, v26.8b
86 eor v15.8b, v15.8b, v27.8b
87 eor v16.8b, v16.8b, v28.8b
88 b 3f
89
90 1: tbz x3, #2, 3f // bit 2 cleared? SHA-384
91
92 // SHA3-224
93 ld1 {v25.8b-v28.8b}, [x1], #32
94 ld1 {v29.8b}, [x1], #8
95 eor v13.8b, v13.8b, v25.8b
96 eor v14.8b, v14.8b, v26.8b
97 eor v15.8b, v15.8b, v27.8b
98 eor v16.8b, v16.8b, v28.8b
99 eor v17.8b, v17.8b, v29.8b
100 b 3f
101
102 // SHA3-512
103 2: ld1 {v25.8b-v26.8b}, [x1], #16
104 eor v7.8b, v7.8b, v25.8b
105 eor v8.8b, v8.8b, v26.8b
106
107 3: sub w8, w8, #1
108
109 eor3 v29.16b, v4.16b, v9.16b, v14.16b
110 eor3 v26.16b, v1.16b, v6.16b, v11.16b
111 eor3 v28.16b, v3.16b, v8.16b, v13.16b
112 eor3 v25.16b, v0.16b, v5.16b, v10.16b
113 eor3 v27.16b, v2.16b, v7.16b, v12.16b
114 eor3 v29.16b, v29.16b, v19.16b, v24.16b
115 eor3 v26.16b, v26.16b, v16.16b, v21.16b
116 eor3 v28.16b, v28.16b, v18.16b, v23.16b
117 eor3 v25.16b, v25.16b, v15.16b, v20.16b
118 eor3 v27.16b, v27.16b, v17.16b, v22.16b
119
120 rax1 v30.2d, v29.2d, v26.2d // bc[0]
121 rax1 v26.2d, v26.2d, v28.2d // bc[2]
122 rax1 v28.2d, v28.2d, v25.2d // bc[4]
123 rax1 v25.2d, v25.2d, v27.2d // bc[1]
124 rax1 v27.2d, v27.2d, v29.2d // bc[3]
125
126 eor v0.16b, v0.16b, v30.16b
127 xar v29.2d, v1.2d, v25.2d, (64 - 1)
128 xar v1.2d, v6.2d, v25.2d, (64 - 44)
129 xar v6.2d, v9.2d, v28.2d, (64 - 20)
130 xar v9.2d, v22.2d, v26.2d, (64 - 61)
131 xar v22.2d, v14.2d, v28.2d, (64 - 39)
132 xar v14.2d, v20.2d, v30.2d, (64 - 18)
133 xar v31.2d, v2.2d, v26.2d, (64 - 62)
134 xar v2.2d, v12.2d, v26.2d, (64 - 43)
135 xar v12.2d, v13.2d, v27.2d, (64 - 25)
136 xar v13.2d, v19.2d, v28.2d, (64 - 8)
137 xar v19.2d, v23.2d, v27.2d, (64 - 56)
138 xar v23.2d, v15.2d, v30.2d, (64 - 41)
139 xar v15.2d, v4.2d, v28.2d, (64 - 27)
140 xar v28.2d, v24.2d, v28.2d, (64 - 14)
141 xar v24.2d, v21.2d, v25.2d, (64 - 2)
142 xar v8.2d, v8.2d, v27.2d, (64 - 55)
143 xar v4.2d, v16.2d, v25.2d, (64 - 45)
144 xar v16.2d, v5.2d, v30.2d, (64 - 36)
145 xar v5.2d, v3.2d, v27.2d, (64 - 28)
146 xar v27.2d, v18.2d, v27.2d, (64 - 21)
147 xar v3.2d, v17.2d, v26.2d, (64 - 15)
148 xar v25.2d, v11.2d, v25.2d, (64 - 10)
149 xar v26.2d, v7.2d, v26.2d, (64 - 6)
150 xar v30.2d, v10.2d, v30.2d, (64 - 3)
151
152 bcax v20.16b, v31.16b, v22.16b, v8.16b
153 bcax v21.16b, v8.16b, v23.16b, v22.16b
154 bcax v22.16b, v22.16b, v24.16b, v23.16b
155 bcax v23.16b, v23.16b, v31.16b, v24.16b
156 bcax v24.16b, v24.16b, v8.16b, v31.16b
157
158 ld1r {v31.2d}, [x9], #8
159
160 bcax v17.16b, v25.16b, v19.16b, v3.16b
161 bcax v18.16b, v3.16b, v15.16b, v19.16b
162 bcax v19.16b, v19.16b, v16.16b, v15.16b
163 bcax v15.16b, v15.16b, v25.16b, v16.16b
164 bcax v16.16b, v16.16b, v3.16b, v25.16b
165
166 bcax v10.16b, v29.16b, v12.16b, v26.16b
167 bcax v11.16b, v26.16b, v13.16b, v12.16b
168 bcax v12.16b, v12.16b, v14.16b, v13.16b
169 bcax v13.16b, v13.16b, v29.16b, v14.16b
170 bcax v14.16b, v14.16b, v26.16b, v29.16b
171
172 bcax v7.16b, v30.16b, v9.16b, v4.16b
173 bcax v8.16b, v4.16b, v5.16b, v9.16b
174 bcax v9.16b, v9.16b, v6.16b, v5.16b
175 bcax v5.16b, v5.16b, v30.16b, v6.16b
176 bcax v6.16b, v6.16b, v4.16b, v30.16b
177
178 bcax v3.16b, v27.16b, v0.16b, v28.16b
179 bcax v4.16b, v28.16b, v1.16b, v0.16b
180 bcax v0.16b, v0.16b, v2.16b, v1.16b
181 bcax v1.16b, v1.16b, v27.16b, v2.16b
182 bcax v2.16b, v2.16b, v28.16b, v27.16b
183
184 eor v0.16b, v0.16b, v31.16b
185
186 cbnz w8, 3b
187 cond_yield 4f, x8, x9
188 cbnz w2, 0b
189
190 /* save state */
191 4: st1 { v0.1d- v3.1d}, [x0], #32
192 st1 { v4.1d- v7.1d}, [x0], #32
193 st1 { v8.1d-v11.1d}, [x0], #32
194 st1 {v12.1d-v15.1d}, [x0], #32
195 st1 {v16.1d-v19.1d}, [x0], #32
196 st1 {v20.1d-v23.1d}, [x0], #32
197 st1 {v24.1d}, [x0]
198 mov w0, w2
199 ret
200 SYM_FUNC_END(sha3_ce_transform)
201
202 .section ".rodata", "a"
203 .align 8
204 .Lsha3_rcon:
205 .quad 0x0000000000000001, 0x0000000000008082, 0x800000000000808a
206 .quad 0x8000000080008000, 0x000000000000808b, 0x0000000080000001
207 .quad 0x8000000080008081, 0x8000000000008009, 0x000000000000008a
208 .quad 0x0000000000000088, 0x0000000080008009, 0x000000008000000a
209 .quad 0x000000008000808b, 0x800000000000008b, 0x8000000000008089
210 .quad 0x8000000000008003, 0x8000000000008002, 0x8000000000000080
211 .quad 0x000000000000800a, 0x800000008000000a, 0x8000000080008081
212 .quad 0x8000000000008080, 0x0000000080000001, 0x8000000080008008
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.