1 // SPDX-License-Identifier: GPL-2.0-only 1 // SPDX-License-Identifier: GPL-2.0-only
2 // Copyright (C) 2021 ARM Limited. 2 // Copyright (C) 2021 ARM Limited.
3 // 3 //
4 // Assembly portion of the syscall ABI test 4 // Assembly portion of the syscall ABI test
5 5
6 // 6 //
7 // Load values from memory into registers, inv 7 // Load values from memory into registers, invoke a syscall and save the
8 // register values back to memory for later ch 8 // register values back to memory for later checking. The syscall to be
9 // invoked is configured in x8 of the input GP 9 // invoked is configured in x8 of the input GPR data.
10 // 10 //
11 // x0: SVE VL, 0 for FP only 11 // x0: SVE VL, 0 for FP only
12 // x1: SME VL 12 // x1: SME VL
13 // 13 //
14 // GPRs: gpr_in, gpr_out 14 // GPRs: gpr_in, gpr_out
15 // FPRs: fpr_in, fpr_out 15 // FPRs: fpr_in, fpr_out
16 // Zn: z_in, z_out 16 // Zn: z_in, z_out
17 // Pn: p_in, p_out 17 // Pn: p_in, p_out
18 // FFR: ffr_in, ffr_out 18 // FFR: ffr_in, ffr_out
19 // ZA: za_in, za_out 19 // ZA: za_in, za_out
20 // SVCR: svcr_in, svcr_out 20 // SVCR: svcr_in, svcr_out
21 21
22 #include "syscall-abi.h" 22 #include "syscall-abi.h"
23 23
24 .arch_extension sve 24 .arch_extension sve
25 25
26 #define ID_AA64SMFR0_EL1_SMEver_SHIFT <<
27 #define ID_AA64SMFR0_EL1_SMEver_WIDTH <<
28 <<
29 /* 26 /*
30 * LDR (vector to ZA array): 27 * LDR (vector to ZA array):
31 * LDR ZA[\nw, #\offset], [X\nxbase, #\of 28 * LDR ZA[\nw, #\offset], [X\nxbase, #\offset, MUL VL]
32 */ 29 */
33 .macro _ldr_za nw, nxbase, offset=0 30 .macro _ldr_za nw, nxbase, offset=0
34 .inst 0xe1000000 31 .inst 0xe1000000 \
35 | (((\nw) & 3) << 13) 32 | (((\nw) & 3) << 13) \
36 | ((\nxbase) << 5) 33 | ((\nxbase) << 5) \
37 | ((\offset) & 7) 34 | ((\offset) & 7)
38 .endm 35 .endm
39 36
40 /* 37 /*
41 * STR (vector from ZA array): 38 * STR (vector from ZA array):
42 * STR ZA[\nw, #\offset], [X\nxbase, #\of 39 * STR ZA[\nw, #\offset], [X\nxbase, #\offset, MUL VL]
43 */ 40 */
44 .macro _str_za nw, nxbase, offset=0 41 .macro _str_za nw, nxbase, offset=0
45 .inst 0xe1200000 42 .inst 0xe1200000 \
46 | (((\nw) & 3) << 13) 43 | (((\nw) & 3) << 13) \
47 | ((\nxbase) << 5) 44 | ((\nxbase) << 5) \
48 | ((\offset) & 7) 45 | ((\offset) & 7)
49 .endm 46 .endm
50 47
51 /* <<
52 * LDR (ZT0) <<
53 * <<
54 * LDR ZT0, nx <<
55 */ <<
56 .macro _ldr_zt nx <<
57 .inst 0xe11f8000 <<
58 | (((\nx) & 0x1f) << 5) <<
59 .endm <<
60 <<
61 /* <<
62 * STR (ZT0) <<
63 * <<
64 * STR ZT0, nx <<
65 */ <<
66 .macro _str_zt nx <<
67 .inst 0xe13f8000 <<
68 | (((\nx) & 0x1f) << 5) <<
69 .endm <<
70 <<
71 .globl do_syscall 48 .globl do_syscall
72 do_syscall: 49 do_syscall:
73 // Store callee saved registers x19-x2 50 // Store callee saved registers x19-x29 (80 bytes) plus x0 and x1
74 stp x29, x30, [sp, #-112]! 51 stp x29, x30, [sp, #-112]!
75 mov x29, sp 52 mov x29, sp
76 stp x0, x1, [sp, #16] 53 stp x0, x1, [sp, #16]
77 stp x19, x20, [sp, #32] 54 stp x19, x20, [sp, #32]
78 stp x21, x22, [sp, #48] 55 stp x21, x22, [sp, #48]
79 stp x23, x24, [sp, #64] 56 stp x23, x24, [sp, #64]
80 stp x25, x26, [sp, #80] 57 stp x25, x26, [sp, #80]
81 stp x27, x28, [sp, #96] 58 stp x27, x28, [sp, #96]
82 59
83 // Set SVCR if we're doing SME 60 // Set SVCR if we're doing SME
84 cbz x1, 1f 61 cbz x1, 1f
85 adrp x2, svcr_in 62 adrp x2, svcr_in
86 ldr x2, [x2, :lo12:svcr_in] 63 ldr x2, [x2, :lo12:svcr_in]
87 msr S3_3_C4_C2_2, x2 64 msr S3_3_C4_C2_2, x2
88 1: 65 1:
89 66
90 // Load ZA and ZT0 if enabled - uses x !! 67 // Load ZA if it's enabled - uses x12 as scratch due to SME LDR
91 tbz x2, #SVCR_ZA_SHIFT, 1f 68 tbz x2, #SVCR_ZA_SHIFT, 1f
92 mov w12, #0 69 mov w12, #0
93 ldr x2, =za_in 70 ldr x2, =za_in
94 2: _ldr_za 12, 2 71 2: _ldr_za 12, 2
95 add x2, x2, x1 72 add x2, x2, x1
96 add x12, x12, #1 73 add x12, x12, #1
97 cmp x1, x12 74 cmp x1, x12
98 bne 2b 75 bne 2b
99 <<
100 // ZT0 <<
101 mrs x2, S3_0_C0_C4_5 // ID_ <<
102 ubfx x2, x2, #ID_AA64SMFR0_EL1_SMEv <<
103 #ID_AA64SMFR0_EL1_SME <<
104 cbz x2, 1f <<
105 adrp x2, zt_in <<
106 add x2, x2, :lo12:zt_in <<
107 _ldr_zt 2 <<
108 1: 76 1:
109 77
110 // Load GPRs x8-x28, and save our SP/F 78 // Load GPRs x8-x28, and save our SP/FP for later comparison
111 ldr x2, =gpr_in 79 ldr x2, =gpr_in
112 add x2, x2, #64 80 add x2, x2, #64
113 ldp x8, x9, [x2], #16 81 ldp x8, x9, [x2], #16
114 ldp x10, x11, [x2], #16 82 ldp x10, x11, [x2], #16
115 ldp x12, x13, [x2], #16 83 ldp x12, x13, [x2], #16
116 ldp x14, x15, [x2], #16 84 ldp x14, x15, [x2], #16
117 ldp x16, x17, [x2], #16 85 ldp x16, x17, [x2], #16
118 ldp x18, x19, [x2], #16 86 ldp x18, x19, [x2], #16
119 ldp x20, x21, [x2], #16 87 ldp x20, x21, [x2], #16
120 ldp x22, x23, [x2], #16 88 ldp x22, x23, [x2], #16
121 ldp x24, x25, [x2], #16 89 ldp x24, x25, [x2], #16
122 ldp x26, x27, [x2], #16 90 ldp x26, x27, [x2], #16
123 ldr x28, [x2], #8 91 ldr x28, [x2], #8
124 str x29, [x2], #8 // FP 92 str x29, [x2], #8 // FP
125 str x30, [x2], #8 // LR 93 str x30, [x2], #8 // LR
126 94
127 // Load FPRs if we're not doing neithe !! 95 // Load FPRs if we're not doing SVE
128 cbnz x0, 1f 96 cbnz x0, 1f
129 ldr x2, =svcr_in <<
130 tbnz x2, #SVCR_SM_SHIFT, 1f <<
131 <<
132 ldr x2, =fpr_in 97 ldr x2, =fpr_in
133 ldp q0, q1, [x2] 98 ldp q0, q1, [x2]
134 ldp q2, q3, [x2, #16 * 2] 99 ldp q2, q3, [x2, #16 * 2]
135 ldp q4, q5, [x2, #16 * 4] 100 ldp q4, q5, [x2, #16 * 4]
136 ldp q6, q7, [x2, #16 * 6] 101 ldp q6, q7, [x2, #16 * 6]
137 ldp q8, q9, [x2, #16 * 8] 102 ldp q8, q9, [x2, #16 * 8]
138 ldp q10, q11, [x2, #16 * 10] 103 ldp q10, q11, [x2, #16 * 10]
139 ldp q12, q13, [x2, #16 * 12] 104 ldp q12, q13, [x2, #16 * 12]
140 ldp q14, q15, [x2, #16 * 14] 105 ldp q14, q15, [x2, #16 * 14]
141 ldp q16, q17, [x2, #16 * 16] 106 ldp q16, q17, [x2, #16 * 16]
142 ldp q18, q19, [x2, #16 * 18] 107 ldp q18, q19, [x2, #16 * 18]
143 ldp q20, q21, [x2, #16 * 20] 108 ldp q20, q21, [x2, #16 * 20]
144 ldp q22, q23, [x2, #16 * 22] 109 ldp q22, q23, [x2, #16 * 22]
145 ldp q24, q25, [x2, #16 * 24] 110 ldp q24, q25, [x2, #16 * 24]
146 ldp q26, q27, [x2, #16 * 26] 111 ldp q26, q27, [x2, #16 * 26]
147 ldp q28, q29, [x2, #16 * 28] 112 ldp q28, q29, [x2, #16 * 28]
148 ldp q30, q31, [x2, #16 * 30] 113 ldp q30, q31, [x2, #16 * 30]
149 <<
150 b 2f <<
151 1: 114 1:
152 115
153 // Load the SVE registers if we're doi 116 // Load the SVE registers if we're doing SVE/SME
>> 117 cbz x0, 1f
154 118
155 ldr x2, =z_in 119 ldr x2, =z_in
156 ldr z0, [x2, #0, MUL VL] 120 ldr z0, [x2, #0, MUL VL]
157 ldr z1, [x2, #1, MUL VL] 121 ldr z1, [x2, #1, MUL VL]
158 ldr z2, [x2, #2, MUL VL] 122 ldr z2, [x2, #2, MUL VL]
159 ldr z3, [x2, #3, MUL VL] 123 ldr z3, [x2, #3, MUL VL]
160 ldr z4, [x2, #4, MUL VL] 124 ldr z4, [x2, #4, MUL VL]
161 ldr z5, [x2, #5, MUL VL] 125 ldr z5, [x2, #5, MUL VL]
162 ldr z6, [x2, #6, MUL VL] 126 ldr z6, [x2, #6, MUL VL]
163 ldr z7, [x2, #7, MUL VL] 127 ldr z7, [x2, #7, MUL VL]
164 ldr z8, [x2, #8, MUL VL] 128 ldr z8, [x2, #8, MUL VL]
165 ldr z9, [x2, #9, MUL VL] 129 ldr z9, [x2, #9, MUL VL]
166 ldr z10, [x2, #10, MUL VL] 130 ldr z10, [x2, #10, MUL VL]
167 ldr z11, [x2, #11, MUL VL] 131 ldr z11, [x2, #11, MUL VL]
168 ldr z12, [x2, #12, MUL VL] 132 ldr z12, [x2, #12, MUL VL]
169 ldr z13, [x2, #13, MUL VL] 133 ldr z13, [x2, #13, MUL VL]
170 ldr z14, [x2, #14, MUL VL] 134 ldr z14, [x2, #14, MUL VL]
171 ldr z15, [x2, #15, MUL VL] 135 ldr z15, [x2, #15, MUL VL]
172 ldr z16, [x2, #16, MUL VL] 136 ldr z16, [x2, #16, MUL VL]
173 ldr z17, [x2, #17, MUL VL] 137 ldr z17, [x2, #17, MUL VL]
174 ldr z18, [x2, #18, MUL VL] 138 ldr z18, [x2, #18, MUL VL]
175 ldr z19, [x2, #19, MUL VL] 139 ldr z19, [x2, #19, MUL VL]
176 ldr z20, [x2, #20, MUL VL] 140 ldr z20, [x2, #20, MUL VL]
177 ldr z21, [x2, #21, MUL VL] 141 ldr z21, [x2, #21, MUL VL]
178 ldr z22, [x2, #22, MUL VL] 142 ldr z22, [x2, #22, MUL VL]
179 ldr z23, [x2, #23, MUL VL] 143 ldr z23, [x2, #23, MUL VL]
180 ldr z24, [x2, #24, MUL VL] 144 ldr z24, [x2, #24, MUL VL]
181 ldr z25, [x2, #25, MUL VL] 145 ldr z25, [x2, #25, MUL VL]
182 ldr z26, [x2, #26, MUL VL] 146 ldr z26, [x2, #26, MUL VL]
183 ldr z27, [x2, #27, MUL VL] 147 ldr z27, [x2, #27, MUL VL]
184 ldr z28, [x2, #28, MUL VL] 148 ldr z28, [x2, #28, MUL VL]
185 ldr z29, [x2, #29, MUL VL] 149 ldr z29, [x2, #29, MUL VL]
186 ldr z30, [x2, #30, MUL VL] 150 ldr z30, [x2, #30, MUL VL]
187 ldr z31, [x2, #31, MUL VL] 151 ldr z31, [x2, #31, MUL VL]
188 152
189 // Only set a non-zero FFR, test patte 153 // Only set a non-zero FFR, test patterns must be zero since the
190 // syscall should clear it - this lets 154 // syscall should clear it - this lets us handle FA64.
191 ldr x2, =ffr_in 155 ldr x2, =ffr_in
192 ldr p0, [x2] 156 ldr p0, [x2]
193 ldr x2, [x2, #0] 157 ldr x2, [x2, #0]
194 cbz x2, 1f !! 158 cbz x2, 2f
195 wrffr p0.b 159 wrffr p0.b
196 1: !! 160 2:
197 161
198 ldr x2, =p_in 162 ldr x2, =p_in
199 ldr p0, [x2, #0, MUL VL] 163 ldr p0, [x2, #0, MUL VL]
200 ldr p1, [x2, #1, MUL VL] 164 ldr p1, [x2, #1, MUL VL]
201 ldr p2, [x2, #2, MUL VL] 165 ldr p2, [x2, #2, MUL VL]
202 ldr p3, [x2, #3, MUL VL] 166 ldr p3, [x2, #3, MUL VL]
203 ldr p4, [x2, #4, MUL VL] 167 ldr p4, [x2, #4, MUL VL]
204 ldr p5, [x2, #5, MUL VL] 168 ldr p5, [x2, #5, MUL VL]
205 ldr p6, [x2, #6, MUL VL] 169 ldr p6, [x2, #6, MUL VL]
206 ldr p7, [x2, #7, MUL VL] 170 ldr p7, [x2, #7, MUL VL]
207 ldr p8, [x2, #8, MUL VL] 171 ldr p8, [x2, #8, MUL VL]
208 ldr p9, [x2, #9, MUL VL] 172 ldr p9, [x2, #9, MUL VL]
209 ldr p10, [x2, #10, MUL VL] 173 ldr p10, [x2, #10, MUL VL]
210 ldr p11, [x2, #11, MUL VL] 174 ldr p11, [x2, #11, MUL VL]
211 ldr p12, [x2, #12, MUL VL] 175 ldr p12, [x2, #12, MUL VL]
212 ldr p13, [x2, #13, MUL VL] 176 ldr p13, [x2, #13, MUL VL]
213 ldr p14, [x2, #14, MUL VL] 177 ldr p14, [x2, #14, MUL VL]
214 ldr p15, [x2, #15, MUL VL] 178 ldr p15, [x2, #15, MUL VL]
215 2: !! 179 1:
216 180
217 // Do the syscall 181 // Do the syscall
218 svc #0 182 svc #0
219 183
220 // Save GPRs x8-x30 184 // Save GPRs x8-x30
221 ldr x2, =gpr_out 185 ldr x2, =gpr_out
222 add x2, x2, #64 186 add x2, x2, #64
223 stp x8, x9, [x2], #16 187 stp x8, x9, [x2], #16
224 stp x10, x11, [x2], #16 188 stp x10, x11, [x2], #16
225 stp x12, x13, [x2], #16 189 stp x12, x13, [x2], #16
226 stp x14, x15, [x2], #16 190 stp x14, x15, [x2], #16
227 stp x16, x17, [x2], #16 191 stp x16, x17, [x2], #16
228 stp x18, x19, [x2], #16 192 stp x18, x19, [x2], #16
229 stp x20, x21, [x2], #16 193 stp x20, x21, [x2], #16
230 stp x22, x23, [x2], #16 194 stp x22, x23, [x2], #16
231 stp x24, x25, [x2], #16 195 stp x24, x25, [x2], #16
232 stp x26, x27, [x2], #16 196 stp x26, x27, [x2], #16
233 stp x28, x29, [x2], #16 197 stp x28, x29, [x2], #16
234 str x30, [x2] 198 str x30, [x2]
235 199
236 // Restore x0 and x1 for feature check 200 // Restore x0 and x1 for feature checks
237 ldp x0, x1, [sp, #16] 201 ldp x0, x1, [sp, #16]
238 202
239 // Save FPSIMD state 203 // Save FPSIMD state
240 ldr x2, =fpr_out 204 ldr x2, =fpr_out
241 stp q0, q1, [x2] 205 stp q0, q1, [x2]
242 stp q2, q3, [x2, #16 * 2] 206 stp q2, q3, [x2, #16 * 2]
243 stp q4, q5, [x2, #16 * 4] 207 stp q4, q5, [x2, #16 * 4]
244 stp q6, q7, [x2, #16 * 6] 208 stp q6, q7, [x2, #16 * 6]
245 stp q8, q9, [x2, #16 * 8] 209 stp q8, q9, [x2, #16 * 8]
246 stp q10, q11, [x2, #16 * 10] 210 stp q10, q11, [x2, #16 * 10]
247 stp q12, q13, [x2, #16 * 12] 211 stp q12, q13, [x2, #16 * 12]
248 stp q14, q15, [x2, #16 * 14] 212 stp q14, q15, [x2, #16 * 14]
249 stp q16, q17, [x2, #16 * 16] 213 stp q16, q17, [x2, #16 * 16]
250 stp q18, q19, [x2, #16 * 18] 214 stp q18, q19, [x2, #16 * 18]
251 stp q20, q21, [x2, #16 * 20] 215 stp q20, q21, [x2, #16 * 20]
252 stp q22, q23, [x2, #16 * 22] 216 stp q22, q23, [x2, #16 * 22]
253 stp q24, q25, [x2, #16 * 24] 217 stp q24, q25, [x2, #16 * 24]
254 stp q26, q27, [x2, #16 * 26] 218 stp q26, q27, [x2, #16 * 26]
255 stp q28, q29, [x2, #16 * 28] 219 stp q28, q29, [x2, #16 * 28]
256 stp q30, q31, [x2, #16 * 30] 220 stp q30, q31, [x2, #16 * 30]
257 221
258 // Save SVCR if we're doing SME 222 // Save SVCR if we're doing SME
259 cbz x1, 1f 223 cbz x1, 1f
260 mrs x2, S3_3_C4_C2_2 224 mrs x2, S3_3_C4_C2_2
261 adrp x3, svcr_out 225 adrp x3, svcr_out
262 str x2, [x3, :lo12:svcr_out] 226 str x2, [x3, :lo12:svcr_out]
263 1: 227 1:
264 228
265 // Save ZA if it's enabled - uses x12 229 // Save ZA if it's enabled - uses x12 as scratch due to SME STR
266 tbz x2, #SVCR_ZA_SHIFT, 1f 230 tbz x2, #SVCR_ZA_SHIFT, 1f
267 mov w12, #0 231 mov w12, #0
268 ldr x2, =za_out 232 ldr x2, =za_out
269 2: _str_za 12, 2 233 2: _str_za 12, 2
270 add x2, x2, x1 234 add x2, x2, x1
271 add x12, x12, #1 235 add x12, x12, #1
272 cmp x1, x12 236 cmp x1, x12
273 bne 2b 237 bne 2b
274 <<
275 // ZT0 <<
276 mrs x2, S3_0_C0_C4_5 // ID_ <<
277 ubfx x2, x2, #ID_AA64SMFR0_EL1_SMEv <<
278 #ID_AA64SMFR0_EL1_SMEv <<
279 cbz x2, 1f <<
280 adrp x2, zt_out <<
281 add x2, x2, :lo12:zt_out <<
282 _str_zt 2 <<
283 1: 238 1:
284 239
285 // Save the SVE state if we have some 240 // Save the SVE state if we have some
286 cbz x0, 1f 241 cbz x0, 1f
287 242
288 ldr x2, =z_out 243 ldr x2, =z_out
289 str z0, [x2, #0, MUL VL] 244 str z0, [x2, #0, MUL VL]
290 str z1, [x2, #1, MUL VL] 245 str z1, [x2, #1, MUL VL]
291 str z2, [x2, #2, MUL VL] 246 str z2, [x2, #2, MUL VL]
292 str z3, [x2, #3, MUL VL] 247 str z3, [x2, #3, MUL VL]
293 str z4, [x2, #4, MUL VL] 248 str z4, [x2, #4, MUL VL]
294 str z5, [x2, #5, MUL VL] 249 str z5, [x2, #5, MUL VL]
295 str z6, [x2, #6, MUL VL] 250 str z6, [x2, #6, MUL VL]
296 str z7, [x2, #7, MUL VL] 251 str z7, [x2, #7, MUL VL]
297 str z8, [x2, #8, MUL VL] 252 str z8, [x2, #8, MUL VL]
298 str z9, [x2, #9, MUL VL] 253 str z9, [x2, #9, MUL VL]
299 str z10, [x2, #10, MUL VL] 254 str z10, [x2, #10, MUL VL]
300 str z11, [x2, #11, MUL VL] 255 str z11, [x2, #11, MUL VL]
301 str z12, [x2, #12, MUL VL] 256 str z12, [x2, #12, MUL VL]
302 str z13, [x2, #13, MUL VL] 257 str z13, [x2, #13, MUL VL]
303 str z14, [x2, #14, MUL VL] 258 str z14, [x2, #14, MUL VL]
304 str z15, [x2, #15, MUL VL] 259 str z15, [x2, #15, MUL VL]
305 str z16, [x2, #16, MUL VL] 260 str z16, [x2, #16, MUL VL]
306 str z17, [x2, #17, MUL VL] 261 str z17, [x2, #17, MUL VL]
307 str z18, [x2, #18, MUL VL] 262 str z18, [x2, #18, MUL VL]
308 str z19, [x2, #19, MUL VL] 263 str z19, [x2, #19, MUL VL]
309 str z20, [x2, #20, MUL VL] 264 str z20, [x2, #20, MUL VL]
310 str z21, [x2, #21, MUL VL] 265 str z21, [x2, #21, MUL VL]
311 str z22, [x2, #22, MUL VL] 266 str z22, [x2, #22, MUL VL]
312 str z23, [x2, #23, MUL VL] 267 str z23, [x2, #23, MUL VL]
313 str z24, [x2, #24, MUL VL] 268 str z24, [x2, #24, MUL VL]
314 str z25, [x2, #25, MUL VL] 269 str z25, [x2, #25, MUL VL]
315 str z26, [x2, #26, MUL VL] 270 str z26, [x2, #26, MUL VL]
316 str z27, [x2, #27, MUL VL] 271 str z27, [x2, #27, MUL VL]
317 str z28, [x2, #28, MUL VL] 272 str z28, [x2, #28, MUL VL]
318 str z29, [x2, #29, MUL VL] 273 str z29, [x2, #29, MUL VL]
319 str z30, [x2, #30, MUL VL] 274 str z30, [x2, #30, MUL VL]
320 str z31, [x2, #31, MUL VL] 275 str z31, [x2, #31, MUL VL]
321 276
322 ldr x2, =p_out 277 ldr x2, =p_out
323 str p0, [x2, #0, MUL VL] 278 str p0, [x2, #0, MUL VL]
324 str p1, [x2, #1, MUL VL] 279 str p1, [x2, #1, MUL VL]
325 str p2, [x2, #2, MUL VL] 280 str p2, [x2, #2, MUL VL]
326 str p3, [x2, #3, MUL VL] 281 str p3, [x2, #3, MUL VL]
327 str p4, [x2, #4, MUL VL] 282 str p4, [x2, #4, MUL VL]
328 str p5, [x2, #5, MUL VL] 283 str p5, [x2, #5, MUL VL]
329 str p6, [x2, #6, MUL VL] 284 str p6, [x2, #6, MUL VL]
330 str p7, [x2, #7, MUL VL] 285 str p7, [x2, #7, MUL VL]
331 str p8, [x2, #8, MUL VL] 286 str p8, [x2, #8, MUL VL]
332 str p9, [x2, #9, MUL VL] 287 str p9, [x2, #9, MUL VL]
333 str p10, [x2, #10, MUL VL] 288 str p10, [x2, #10, MUL VL]
334 str p11, [x2, #11, MUL VL] 289 str p11, [x2, #11, MUL VL]
335 str p12, [x2, #12, MUL VL] 290 str p12, [x2, #12, MUL VL]
336 str p13, [x2, #13, MUL VL] 291 str p13, [x2, #13, MUL VL]
337 str p14, [x2, #14, MUL VL] 292 str p14, [x2, #14, MUL VL]
338 str p15, [x2, #15, MUL VL] 293 str p15, [x2, #15, MUL VL]
339 294
340 // Only save FFR if we wrote a value f 295 // Only save FFR if we wrote a value for SME
341 ldr x2, =ffr_in 296 ldr x2, =ffr_in
342 ldr x2, [x2, #0] 297 ldr x2, [x2, #0]
343 cbz x2, 1f 298 cbz x2, 1f
344 ldr x2, =ffr_out 299 ldr x2, =ffr_out
345 rdffr p0.b 300 rdffr p0.b
346 str p0, [x2] 301 str p0, [x2]
347 1: 302 1:
348 303
349 // Restore callee saved registers x19- 304 // Restore callee saved registers x19-x30
350 ldp x19, x20, [sp, #32] 305 ldp x19, x20, [sp, #32]
351 ldp x21, x22, [sp, #48] 306 ldp x21, x22, [sp, #48]
352 ldp x23, x24, [sp, #64] 307 ldp x23, x24, [sp, #64]
353 ldp x25, x26, [sp, #80] 308 ldp x25, x26, [sp, #80]
354 ldp x27, x28, [sp, #96] 309 ldp x27, x28, [sp, #96]
355 ldp x29, x30, [sp], #112 310 ldp x29, x30, [sp], #112
356 311
357 // Clear SVCR if we were doing SME so 312 // Clear SVCR if we were doing SME so future tests don't have ZA
358 cbz x1, 1f 313 cbz x1, 1f
359 msr S3_3_C4_C2_2, xzr 314 msr S3_3_C4_C2_2, xzr
360 1: 315 1:
361 316
362 ret 317 ret
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.