1 /* SPDX-License-Identifier: GPL-2.0 */
2
3 #include <linux/stringify.h>
4 #include <linux/linkage.h>
5 #include <asm/alternative.h>
6 #include <asm/dwarf.h>
7 #include <asm/fpu-insn.h>
8
9 #define STATE0 %v0
10 #define STATE1 %v1
11 #define STATE2 %v2
12 #define STATE3 %v3
13 #define COPY0 %v4
14 #define COPY1 %v5
15 #define COPY2 %v6
16 #define COPY3 %v7
17 #define BEPERM %v19
18 #define TMP0 %v20
19 #define TMP1 %v21
20 #define TMP2 %v22
21 #define TMP3 %v23
22
23 .section .rodata
24
25 .balign 32
26 SYM_DATA_START_LOCAL(chacha20_constants)
27 .long 0x61707865,0x3320646e,0x79622d
28 .long 0x03020100,0x07060504,0x0b0a09
29 SYM_DATA_END(chacha20_constants)
30
31 .text
32 /*
33 * s390 ChaCha20 implementation meant for vDSO
34 * number of blocks of output with nonce 0, ta
35 * counter. Does not spill to the stack.
36 *
37 * void __arch_chacha20_blocks_nostack(uint8_t
38 * const u
39 * uint32_
40 * size_t
41 */
42 SYM_FUNC_START(__arch_chacha20_blocks_nostack)
43 CFI_STARTPROC
44 larl %r1,chacha20_constants
45
46 /* COPY0 = "expand 32-byte k" */
47 VL COPY0,0,,%r1
48
49 /* BEPERM = byte selectors for VPERM *
50 ALTERNATIVE __stringify(VL BEPERM,16,,
51
52 /* COPY1,COPY2 = key */
53 VLM COPY1,COPY2,0,%r3
54
55 /* COPY3 = counter || zero nonce */
56 lg %r3,0(%r4)
57 VZERO COPY3
58 VLVGG COPY3,%r3,0
59
60 lghi %r1,0
61 .Lblock:
62 VLR STATE0,COPY0
63 VLR STATE1,COPY1
64 VLR STATE2,COPY2
65 VLR STATE3,COPY3
66
67 lghi %r0,10
68 .Ldoubleround:
69 /* STATE0 += STATE1, STATE3 = rotl32(S
70 VAF STATE0,STATE0,STATE1
71 VX STATE3,STATE3,STATE0
72 VERLLF STATE3,STATE3,16
73
74 /* STATE2 += STATE3, STATE1 = rotl32(S
75 VAF STATE2,STATE2,STATE3
76 VX STATE1,STATE1,STATE2
77 VERLLF STATE1,STATE1,12
78
79 /* STATE0 += STATE1, STATE3 = rotl32(S
80 VAF STATE0,STATE0,STATE1
81 VX STATE3,STATE3,STATE0
82 VERLLF STATE3,STATE3,8
83
84 /* STATE2 += STATE3, STATE1 = rotl32(S
85 VAF STATE2,STATE2,STATE3
86 VX STATE1,STATE1,STATE2
87 VERLLF STATE1,STATE1,7
88
89 /* STATE1[0,1,2,3] = STATE1[1,2,3,0] *
90 VSLDB STATE1,STATE1,STATE1,4
91 /* STATE2[0,1,2,3] = STATE2[2,3,0,1] *
92 VSLDB STATE2,STATE2,STATE2,8
93 /* STATE3[0,1,2,3] = STATE3[3,0,1,2] *
94 VSLDB STATE3,STATE3,STATE3,12
95
96 /* STATE0 += STATE1, STATE3 = rotl32(S
97 VAF STATE0,STATE0,STATE1
98 VX STATE3,STATE3,STATE0
99 VERLLF STATE3,STATE3,16
100
101 /* STATE2 += STATE3, STATE1 = rotl32(S
102 VAF STATE2,STATE2,STATE3
103 VX STATE1,STATE1,STATE2
104 VERLLF STATE1,STATE1,12
105
106 /* STATE0 += STATE1, STATE3 = rotl32(S
107 VAF STATE0,STATE0,STATE1
108 VX STATE3,STATE3,STATE0
109 VERLLF STATE3,STATE3,8
110
111 /* STATE2 += STATE3, STATE1 = rotl32(S
112 VAF STATE2,STATE2,STATE3
113 VX STATE1,STATE1,STATE2
114 VERLLF STATE1,STATE1,7
115
116 /* STATE1[0,1,2,3] = STATE1[3,0,1,2] *
117 VSLDB STATE1,STATE1,STATE1,12
118 /* STATE2[0,1,2,3] = STATE2[2,3,0,1] *
119 VSLDB STATE2,STATE2,STATE2,8
120 /* STATE3[0,1,2,3] = STATE3[1,2,3,0] *
121 VSLDB STATE3,STATE3,STATE3,4
122 brctg %r0,.Ldoubleround
123
124 /* OUTPUT0 = STATE0 + COPY0 */
125 VAF STATE0,STATE0,COPY0
126 /* OUTPUT1 = STATE1 + COPY1 */
127 VAF STATE1,STATE1,COPY1
128 /* OUTPUT2 = STATE2 + COPY2 */
129 VAF STATE2,STATE2,COPY2
130 /* OUTPUT3 = STATE3 + COPY3 */
131 VAF STATE3,STATE3,COPY3
132
133 ALTERNATIVE
134 __stringify(
135 /* Convert STATE to little end
136 VPERM TMP0,STATE0,STATE0,BEP
137 VPERM TMP1,STATE1,STATE1,BEP
138 VPERM TMP2,STATE2,STATE2,BEP
139 VPERM TMP3,STATE3,STATE3,BEP
140 VSTM TMP0,TMP3,0,%r2),
141 __stringify(
142 /* 32 bit wise little endian s
143 VSTBRF STATE0,0,,%r2;
144 VSTBRF STATE1,16,,%r2;
145 VSTBRF STATE2,32,,%r2;
146 VSTBRF STATE3,48,,%r2;
147 brcl 0,0),
148 ALT_FACILITY(148)
149
150 /* ++COPY3.COUNTER */
151 /* alsih %r3,1 */
152 .insn rilu,0xcc0a00000000,%r3,1
153 alcr %r3,%r1
154 VLVGG COPY3,%r3,0
155
156 /* OUTPUT += 64, --NBLOCKS */
157 aghi %r2,64
158 brctg %r5,.Lblock
159
160 /* COUNTER = COPY3.COUNTER */
161 stg %r3,0(%r4)
162
163 /* Zero out potentially sensitive regs
164 VZERO STATE0
165 VZERO STATE1
166 VZERO STATE2
167 VZERO STATE3
168 VZERO COPY1
169 VZERO COPY2
170
171 /* Early exit if TMP0-TMP3 have not be
172 ALTERNATIVE "nopr", "br %r14", ALT_FAC
173
174 VZERO TMP0
175 VZERO TMP1
176 VZERO TMP2
177 VZERO TMP3
178
179 br %r14
180 CFI_ENDPROC
181 SYM_FUNC_END(__arch_chacha20_blocks_nostack)
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.