1 // SPDX-License-Identifier: GPL-2.0-only
2 // Copyright (C) 2015-2019 ARM Limited.
3 // Original author: Dave Martin <Dave.Martin@ar
4 //
5 // Simple Scalable Vector Extension context sw
6 // Repeatedly writes unique test patterns into
7 // and reads them back to verify integrity.
8 //
9 // for x in `seq 1 NR_CPUS`; do sve-test & pid
10 // (leave it running for as long as you want..
11 // kill $pids
12
13 #include <asm/unistd.h>
14 #include "assembler.h"
15 #include "asm-offsets.h"
16 #include "sme-inst.h"
17
18 #define NZR 32
19 #define NPR 16
20 #define MAXVL_B (2048 / 8)
21
22 .arch_extension sve
23
24 .macro _sve_ldr_v zt, xn
25 ldr z\zt, [x\xn]
26 .endm
27
28 .macro _sve_str_v zt, xn
29 str z\zt, [x\xn]
30 .endm
31
32 .macro _sve_ldr_p pt, xn
33 ldr p\pt, [x\xn]
34 .endm
35
36 .macro _sve_str_p pt, xn
37 str p\pt, [x\xn]
38 .endm
39
40 // Generate accessor functions to read/write p
41 // SVE registers.
42 // x0 is the register index to access
43 // x1 is the memory address to read from (getz
44 // All clobber x0-x2
45 define_accessor setz, NZR, _sve_ldr_v
46 define_accessor getz, NZR, _sve_str_v
47 define_accessor setp, NPR, _sve_ldr_p
48 define_accessor getp, NPR, _sve_str_p
49
50 // Declare some storate space to shadow the SV
51 .pushsection .text
52 .data
53 .align 4
54 zref:
55 .space MAXVL_B * NZR
56 pref:
57 .space MAXVL_B / 8 * NPR
58 ffrref:
59 .space MAXVL_B / 8
60 scratch:
61 .space MAXVL_B
62 .popsection
63
64 // Generate a test pattern for storage in SVE
65 // x0: pid (16 bits)
66 // x1: register number (6 bits)
67 // x2: generation (4 bits)
68
69 // These values are used to constuct a 32-bit
70 // scratch buffer as many times as will fit:
71 // bits 31:28 generation number (increments
72 // bits 27:22 32-bit lane index
73 // bits 21:16 register number
74 // bits 15: 0 pid
75
76 function pattern
77 orr w1, w0, w1, lsl #16
78 orr w2, w1, w2, lsl #28
79
80 ldr x0, =scratch
81 mov w1, #MAXVL_B / 4
82
83 0: str w2, [x0], #4
84 add w2, w2, #(1 << 22)
85 subs w1, w1, #1
86 bne 0b
87
88 ret
89 endfunction
90
91 // Get the address of shadow data for SVE Z-re
92 .macro _adrz xd, xn, nrtmp
93 ldr \xd, =zref
94 rdvl x\nrtmp, #1
95 madd \xd, x\nrtmp, \xn, \xd
96 .endm
97
98 // Get the address of shadow data for SVE P-re
99 .macro _adrp xd, xn, nrtmp
100 ldr \xd, =pref
101 rdvl x\nrtmp, #1
102 lsr x\nrtmp, x\nrtmp, #3
103 sub \xn, \xn, #NZR
104 madd \xd, x\nrtmp, \xn, \xd
105 .endm
106
107 // Set up test pattern in a SVE Z-register
108 // x0: pid
109 // x1: register number
110 // x2: generation
111 function setup_zreg
112 mov x4, x30
113
114 mov x6, x1
115 bl pattern
116 _adrz x0, x6, 2
117 mov x5, x0
118 ldr x1, =scratch
119 bl memcpy
120
121 mov x0, x6
122 mov x1, x5
123 bl setz
124
125 ret x4
126 endfunction
127
128 // Set up test pattern in a SVE P-register
129 // x0: pid
130 // x1: register number
131 // x2: generation
132 function setup_preg
133 mov x4, x30
134
135 mov x6, x1
136 bl pattern
137 _adrp x0, x6, 2
138 mov x5, x0
139 ldr x1, =scratch
140 bl memcpy
141
142 mov x0, x6
143 mov x1, x5
144 bl setp
145
146 ret x4
147 endfunction
148
149 // Set up test pattern in the FFR
150 // x0: pid
151 // x2: generation
152 //
153 // We need to generate a canonical FFR value,
154 // low "1" bits, followed by a number of zeros
155 // per 16 bits of FFR, so we create a 4 bit si
156 // generation, and use that as the initial num
157 // We fill the upper lanes of FFR with zeros.
158 // Beware: corrupts P0.
159 function setup_ffr
160 #ifndef SSVE
161 mov x4, x30
162
163 and w0, w0, #0x3
164 bfi w0, w2, #2, #2
165 mov w1, #1
166 lsl w1, w1, w0
167 sub w1, w1, #1
168
169 ldr x0, =ffrref
170 strh w1, [x0], 2
171 rdvl x1, #1
172 lsr x1, x1, #3
173 sub x1, x1, #2
174 bl memclr
175
176 mov x0, #0
177 ldr x1, =ffrref
178 bl setp
179
180 wrffr p0.b
181
182 ret x4
183 #else
184 ret
185 #endif
186 endfunction
187
188 // Trivial memory compare: compare x2 bytes st
189 // bytes starting at address x1.
190 // Returns only if all bytes match; otherwise,
191 // Clobbers x0-x5.
192 function memcmp
193 cbz x2, 2f
194
195 stp x0, x1, [sp, #-0x20]!
196 str x2, [sp, #0x10]
197
198 mov x5, #0
199 0: ldrb w3, [x0, x5]
200 ldrb w4, [x1, x5]
201 add x5, x5, #1
202 cmp w3, w4
203 b.ne 1f
204 subs x2, x2, #1
205 b.ne 0b
206
207 1: ldr x2, [sp, #0x10]
208 ldp x0, x1, [sp], #0x20
209 b.ne barf
210
211 2: ret
212 endfunction
213
214 // Verify that a SVE Z-register matches its sh
215 // x0: reg number
216 // Clobbers x0-x7.
217 function check_zreg
218 mov x3, x30
219
220 _adrz x5, x0, 6
221 mov x4, x0
222 ldr x7, =scratch
223
224 mov x0, x7
225 mov x1, x6
226 bl memfill_ae
227
228 mov x0, x4
229 mov x1, x7
230 bl getz
231
232 mov x0, x5
233 mov x1, x7
234 mov x2, x6
235 mov x30, x3
236 b memcmp
237 endfunction
238
239 // Verify that a SVE P-register matches its sh
240 // x0: reg number
241 // Clobbers x0-x7.
242 function check_preg
243 mov x3, x30
244
245 _adrp x5, x0, 6
246 mov x4, x0
247 ldr x7, =scratch
248
249 mov x0, x7
250 mov x1, x6
251 bl memfill_ae
252
253 mov x0, x4
254 mov x1, x7
255 bl getp
256
257 mov x0, x5
258 mov x1, x7
259 mov x2, x6
260 mov x30, x3
261 b memcmp
262 endfunction
263
264 // Verify that the FFR matches its shadow in m
265 // Beware -- corrupts P0.
266 // Clobbers x0-x5.
267 function check_ffr
268 #ifndef SSVE
269 mov x3, x30
270
271 ldr x4, =scratch
272 rdvl x5, #1
273 lsr x5, x5, #3
274
275 mov x0, x4
276 mov x1, x5
277 bl memfill_ae
278
279 rdffr p0.b
280 mov x0, #0
281 mov x1, x4
282 bl getp
283
284 ldr x0, =ffrref
285 mov x1, x4
286 mov x2, x5
287 mov x30, x3
288 b memcmp
289 #else
290 ret
291 #endif
292 endfunction
293
294 // Any SVE register modified here can cause co
295 // thread -- but *only* the registers modified
296 function irritator_handler
297 // Increment the irritation signal cou
298 ldr x0, [x2, #ucontext_regs + 8 *
299 add x0, x0, #1
300 str x0, [x2, #ucontext_regs + 8 *
301
302 // Corrupt some random Z-regs
303 adr x0, .text + (irritator_handler
304 movi v0.8b, #1
305 movi v9.16b, #2
306 movi v31.8b, #3
307 #ifndef SSVE
308 // And P0
309 rdffr p0.b
310 // And FFR
311 wrffr p15.b
312 #endif
313
314 ret
315 endfunction
316
317 function tickle_handler
318 // Increment the signal count (x23):
319 ldr x0, [x2, #ucontext_regs + 8 *
320 add x0, x0, #1
321 str x0, [x2, #ucontext_regs + 8 *
322
323 ret
324 endfunction
325
326 function terminate_handler
327 mov w21, w0
328 mov x20, x2
329
330 puts "Terminated by signal "
331 mov w0, w21
332 bl putdec
333 puts ", no error, iterations="
334 ldr x0, [x20, #ucontext_regs + 8 *
335 bl putdec
336 puts ", signals="
337 ldr x0, [x20, #ucontext_regs + 8 *
338 bl putdecn
339
340 mov x0, #0
341 mov x8, #__NR_exit
342 svc #0
343 endfunction
344
345 // w0: signal number
346 // x1: sa_action
347 // w2: sa_flags
348 // Clobbers x0-x6,x8
349 function setsignal
350 str x30, [sp, #-((sa_sz + 15) / 16
351
352 mov w4, w0
353 mov x5, x1
354 mov w6, w2
355
356 add x0, sp, #16
357 mov x1, #sa_sz
358 bl memclr
359
360 mov w0, w4
361 add x1, sp, #16
362 str w6, [x1, #sa_flags]
363 str x5, [x1, #sa_handler]
364 mov x2, #0
365 mov x3, #sa_mask_sz
366 mov x8, #__NR_rt_sigaction
367 svc #0
368
369 cbz w0, 1f
370
371 puts "sigaction failure\n"
372 b .Labort
373
374 1: ldr x30, [sp], #((sa_sz + 15) / 16
375 ret
376 endfunction
377
378 // Main program entry point
379 .globl _start
380 function _start
381 mov x23, #0 // Irritation
382
383 mov w0, #SIGINT
384 adr x1, terminate_handler
385 mov w2, #SA_SIGINFO
386 bl setsignal
387
388 mov w0, #SIGTERM
389 adr x1, terminate_handler
390 mov w2, #SA_SIGINFO
391 bl setsignal
392
393 mov w0, #SIGUSR1
394 adr x1, irritator_handler
395 mov w2, #SA_SIGINFO
396 orr w2, w2, #SA_NODEFER
397 bl setsignal
398
399 mov w0, #SIGUSR2
400 adr x1, tickle_handler
401 mov w2, #SA_SIGINFO
402 orr w2, w2, #SA_NODEFER
403 bl setsignal
404
405 #ifdef SSVE
406 puts "Streaming mode "
407 smstart_sm
408 #endif
409
410 // Sanity-check and report the vector
411
412 rdvl x19, #8
413 cmp x19, #128
414 b.lo 1f
415 cmp x19, #2048
416 b.hi 1f
417 tst x19, #(8 - 1)
418 b.eq 2f
419
420 1: puts "Bad vector length: "
421 mov x0, x19
422 bl putdecn
423 b .Labort
424
425 2: puts "Vector length:\t"
426 mov x0, x19
427 bl putdec
428 puts " bits\n"
429
430 // Obtain our PID, to ensure test patt
431
432 mov x8, #__NR_getpid
433 svc #0
434 mov x20, x0
435
436 puts "PID:\t"
437 mov x0, x20
438 bl putdecn
439
440 #ifdef SSVE
441 smstart_sm // syscalls wi
442 #endif
443
444 mov x22, #0 // generation
445 .Ltest_loop:
446 rdvl x0, #8
447 cmp x0, x19
448 b.ne vl_barf
449
450 mov x21, #0 // Set up Z-re
451 0: mov x0, x20
452 mov x1, x21
453 and x2, x22, #0xf
454 bl setup_zreg
455 add x21, x21, #1
456 cmp x21, #NZR
457 b.lo 0b
458
459 mov x0, x20 // Set up FFR
460 mov x1, #NZR + NPR
461 and x2, x22, #0xf
462 bl setup_ffr
463
464 0: mov x0, x20 // Set up P-re
465 mov x1, x21
466 and x2, x22, #0xf
467 bl setup_preg
468 add x21, x21, #1
469 cmp x21, #NZR + NPR
470 b.lo 0b
471
472 // Can't do this when SVE state is volatile ac
473 // mov x8, #__NR_sched_yield // Enc
474 // svc #0
475
476 #ifdef SSVE
477 mrs x0, S3_3_C4_C2_2 // SVC
478 and x1, x0, #3
479 cmp x1, #1
480 b.ne svcr_barf
481 #endif
482
483 mov x21, #0
484 0: mov x0, x21
485 bl check_zreg
486 add x21, x21, #1
487 cmp x21, #NZR
488 b.lo 0b
489
490 0: mov x0, x21
491 bl check_preg
492 add x21, x21, #1
493 cmp x21, #NZR + NPR
494 b.lo 0b
495
496 bl check_ffr
497
498 add x22, x22, #1
499 b .Ltest_loop
500
501 .Labort:
502 mov x0, #0
503 mov x1, #SIGABRT
504 mov x8, #__NR_kill
505 svc #0
506 endfunction
507
508 function barf
509 // fpsimd.c acitivty log dump hack
510 // ldr w0, =0xdeadc0de
511 // mov w8, #__NR_exit
512 // svc #0
513 // end hack
514 mov x10, x0 // expected data
515 mov x11, x1 // actual data
516 mov x12, x2 // data size
517
518 #ifdef SSVE
519 mrs x13, S3_3_C4_C2_2
520 #endif
521
522 puts "Mismatch: PID="
523 mov x0, x20
524 bl putdec
525 puts ", iteration="
526 mov x0, x22
527 bl putdec
528 puts ", reg="
529 mov x0, x21
530 bl putdecn
531 puts "\tExpected ["
532 mov x0, x10
533 mov x1, x12
534 bl dumphex
535 puts "]\n\tGot ["
536 mov x0, x11
537 mov x1, x12
538 bl dumphex
539 puts "]\n"
540
541 #ifdef SSVE
542 puts "\tSVCR: "
543 mov x0, x13
544 bl putdecn
545 #endif
546
547 mov x8, #__NR_getpid
548 svc #0
549 // fpsimd.c acitivty log dump hack
550 // ldr w0, =0xdeadc0de
551 // mov w8, #__NR_exit
552 // svc #0
553 // ^ end of hack
554 mov x1, #SIGABRT
555 mov x8, #__NR_kill
556 svc #0
557 // mov x8, #__NR_exit
558 // mov x1, #1
559 // svc #0
560 endfunction
561
562 function vl_barf
563 mov x10, x0
564
565 puts "Bad active VL: "
566 mov x0, x10
567 bl putdecn
568
569 mov x8, #__NR_exit
570 mov x1, #1
571 svc #0
572 endfunction
573
574 function svcr_barf
575 mov x10, x0
576
577 puts "Bad SVCR: "
578 mov x0, x10
579 bl putdecn
580
581 mov x8, #__NR_exit
582 mov x1, #1
583 svc #0
584 endfunction
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.