1 #!/bin/bash
2 # SPDX-License-Identifier: GPL-2.0-or-later
3 # Copyright (c) 2016 Microsemi. All Rights Reserved.
4 #
5 # Author: Logan Gunthorpe <logang@deltatee.com>
6
7 REMOTE_HOST=
8 LIST_DEVS=FALSE
9
10 DEBUGFS=${DEBUGFS-/sys/kernel/debug}
11
12 PERF_RUN_ORDER=32
13 MAX_MW_SIZE=0
14 RUN_DMA_TESTS=
15 DONT_CLEANUP=
16 MW_SIZE=65536
17
18 function show_help()
19 {
20 echo "Usage: $0 [OPTIONS] LOCAL_DEV REMOTE_DEV"
21 echo "Run tests on a pair of NTB endpoints."
22 echo
23 echo "If the NTB device loops back to the same host then,"
24 echo "just specifying the two PCI ids on the command line is"
25 echo "sufficient. Otherwise, if the NTB link spans two hosts"
26 echo "use the -r option to specify the hostname for the remote"
27 echo "device. SSH will then be used to test the remote side."
28 echo "An SSH key between the root users of the host would then"
29 echo "be highly recommended."
30 echo
31 echo "Options:"
32 echo " -C don't cleanup ntb modules on exit"
33 echo " -h show this help message"
34 echo " -l list available local and remote PCI ids"
35 echo " -r REMOTE_HOST specify the remote's hostname to connect"
36 echo " to for the test (using ssh)"
37 echo " -m MW_SIZE memory window size for ntb_tool"
38 echo " (default: $MW_SIZE)"
39 echo " -d run dma tests for ntb_perf"
40 echo " -p ORDER total data order for ntb_perf"
41 echo " (default: $PERF_RUN_ORDER)"
42 echo " -w MAX_MW_SIZE maxmium memory window size for ntb_perf"
43 echo
44 }
45
46 function parse_args()
47 {
48 OPTIND=0
49 while getopts "b:Cdhlm:r:p:w:" opt; do
50 case "$opt" in
51 C) DONT_CLEANUP=1 ;;
52 d) RUN_DMA_TESTS=1 ;;
53 h) show_help; exit 0 ;;
54 l) LIST_DEVS=TRUE ;;
55 m) MW_SIZE=${OPTARG} ;;
56 r) REMOTE_HOST=${OPTARG} ;;
57 p) PERF_RUN_ORDER=${OPTARG} ;;
58 w) MAX_MW_SIZE=${OPTARG} ;;
59 \?)
60 echo "Invalid option: -$OPTARG" >&2
61 exit 1
62 ;;
63 esac
64 done
65 }
66
67 parse_args "$@"
68 shift $((OPTIND-1))
69 LOCAL_DEV=$1
70 shift
71 parse_args "$@"
72 shift $((OPTIND-1))
73 REMOTE_DEV=$1
74 shift
75 parse_args "$@"
76
77 set -e
78
79 function _modprobe()
80 {
81 modprobe "$@" || return 1
82
83 if [[ "$REMOTE_HOST" != "" ]]; then
84 ssh "$REMOTE_HOST" modprobe "$@" || return 1
85 fi
86 }
87
88 function split_remote()
89 {
90 VPATH=$1
91 REMOTE=
92
93 if [[ "$VPATH" == *":/"* ]]; then
94 REMOTE=${VPATH%%:*}
95 VPATH=${VPATH#*:}
96 fi
97 }
98
99 function read_file()
100 {
101 split_remote $1
102 if [[ "$REMOTE" != "" ]]; then
103 ssh "$REMOTE" cat "$VPATH"
104 else
105 cat "$VPATH"
106 fi
107 }
108
109 function write_file()
110 {
111 split_remote $2
112 VALUE=$1
113
114 if [[ "$REMOTE" != "" ]]; then
115 ssh "$REMOTE" "echo \"$VALUE\" > \"$VPATH\""
116 else
117 echo "$VALUE" > "$VPATH"
118 fi
119 }
120
121 function check_file()
122 {
123 split_remote $1
124
125 if [[ "$REMOTE" != "" ]]; then
126 ssh "$REMOTE" "[[ -e ${VPATH} ]]"
127 else
128 [[ -e ${VPATH} ]]
129 fi
130 }
131
132 function subdirname()
133 {
134 echo $(basename $(dirname $1)) 2> /dev/null
135 }
136
137 function find_pidx()
138 {
139 PORT=$1
140 PPATH=$2
141
142 for ((i = 0; i < 64; i++)); do
143 PEER_DIR="$PPATH/peer$i"
144
145 check_file ${PEER_DIR} || break
146
147 PEER_PORT=$(read_file "${PEER_DIR}/port")
148 if [[ ${PORT} -eq $PEER_PORT ]]; then
149 echo $i
150 return 0
151 fi
152 done
153
154 return 1
155 }
156
157 function port_test()
158 {
159 LOC=$1
160 REM=$2
161
162 echo "Running port tests on: $(basename $LOC) / $(basename $REM)"
163
164 LOCAL_PORT=$(read_file "$LOC/port")
165 REMOTE_PORT=$(read_file "$REM/port")
166
167 LOCAL_PIDX=$(find_pidx ${REMOTE_PORT} "$LOC")
168 REMOTE_PIDX=$(find_pidx ${LOCAL_PORT} "$REM")
169
170 echo "Local port ${LOCAL_PORT} with index ${REMOTE_PIDX} on remote host"
171 echo "Peer port ${REMOTE_PORT} with index ${LOCAL_PIDX} on local host"
172
173 echo " Passed"
174 }
175
176 function link_test()
177 {
178 LOC=$1
179 REM=$2
180 EXP=0
181
182 echo "Running link tests on: $(subdirname $LOC) / $(subdirname $REM)"
183
184 if ! write_file "N" "$LOC/../link" 2> /dev/null; then
185 echo " Unsupported"
186 return
187 fi
188
189 write_file "N" "$LOC/link_event"
190
191 if [[ $(read_file "$REM/link") != "N" ]]; then
192 echo "Expected link to be down in $REM/link" >&2
193 exit -1
194 fi
195
196 write_file "Y" "$LOC/../link"
197
198 echo " Passed"
199 }
200
201 function doorbell_test()
202 {
203 LOC=$1
204 REM=$2
205 EXP=0
206
207 echo "Running db tests on: $(basename $LOC) / $(basename $REM)"
208
209 DB_VALID_MASK=$(read_file "$LOC/db_valid_mask")
210
211 write_file "c $DB_VALID_MASK" "$REM/db"
212
213 for ((i = 0; i < 64; i++)); do
214 DB=$(read_file "$REM/db")
215 if [[ "$DB" -ne "$EXP" ]]; then
216 echo "Doorbell doesn't match expected value $EXP " \
217 "in $REM/db" >&2
218 exit -1
219 fi
220
221 let "MASK = (1 << $i) & $DB_VALID_MASK" || true
222 let "EXP = $EXP | $MASK" || true
223
224 write_file "s $MASK" "$LOC/peer_db"
225 done
226
227 write_file "c $DB_VALID_MASK" "$REM/db_mask"
228 write_file $DB_VALID_MASK "$REM/db_event"
229 write_file "s $DB_VALID_MASK" "$REM/db_mask"
230
231 write_file "c $DB_VALID_MASK" "$REM/db"
232
233 echo " Passed"
234 }
235
236 function get_files_count()
237 {
238 NAME=$1
239 LOC=$2
240
241 split_remote $LOC
242
243 if [[ "$REMOTE" == "" ]]; then
244 echo $(ls -1 "$VPATH"/${NAME}* 2>/dev/null | wc -l)
245 else
246 echo $(ssh "$REMOTE" "ls -1 \"$VPATH\"/${NAME}* | \
247 wc -l" 2> /dev/null)
248 fi
249 }
250
251 function scratchpad_test()
252 {
253 LOC=$1
254 REM=$2
255
256 echo "Running spad tests on: $(subdirname $LOC) / $(subdirname $REM)"
257
258 CNT=$(get_files_count "spad" "$LOC")
259
260 if [[ $CNT -eq 0 ]]; then
261 echo " Unsupported"
262 return
263 fi
264
265 for ((i = 0; i < $CNT; i++)); do
266 VAL=$RANDOM
267 write_file "$VAL" "$LOC/spad$i"
268 RVAL=$(read_file "$REM/../spad$i")
269
270 if [[ "$VAL" -ne "$RVAL" ]]; then
271 echo "Scratchpad $i value $RVAL doesn't match $VAL" >&2
272 exit -1
273 fi
274 done
275
276 echo " Passed"
277 }
278
279 function message_test()
280 {
281 LOC=$1
282 REM=$2
283
284 echo "Running msg tests on: $(subdirname $LOC) / $(subdirname $REM)"
285
286 CNT=$(get_files_count "msg" "$LOC")
287
288 if [[ $CNT -eq 0 ]]; then
289 echo " Unsupported"
290 return
291 fi
292
293 MSG_OUTBITS_MASK=$(read_file "$LOC/../msg_inbits")
294 MSG_INBITS_MASK=$(read_file "$REM/../msg_inbits")
295
296 write_file "c $MSG_OUTBITS_MASK" "$LOC/../msg_sts"
297 write_file "c $MSG_INBITS_MASK" "$REM/../msg_sts"
298
299 for ((i = 0; i < $CNT; i++)); do
300 VAL=$RANDOM
301 write_file "$VAL" "$LOC/msg$i"
302 RVAL=$(read_file "$REM/../msg$i")
303
304 if [[ "$VAL" -ne "${RVAL%%<-*}" ]]; then
305 echo "Message $i value $RVAL doesn't match $VAL" >&2
306 exit -1
307 fi
308 done
309
310 echo " Passed"
311 }
312
313 function get_number()
314 {
315 KEY=$1
316
317 sed -n "s/^\(${KEY}\)[ \t]*\(0x[0-9a-fA-F]*\)\(\[p\]\)\?$/\2/p"
318 }
319
320 function mw_alloc()
321 {
322 IDX=$1
323 LOC=$2
324 REM=$3
325
326 write_file $MW_SIZE "$LOC/mw_trans$IDX"
327
328 INB_MW=$(read_file "$LOC/mw_trans$IDX")
329 MW_ALIGNED_SIZE=$(echo "$INB_MW" | get_number "Window Size")
330 MW_DMA_ADDR=$(echo "$INB_MW" | get_number "DMA Address")
331
332 write_file "$MW_DMA_ADDR:$(($MW_ALIGNED_SIZE))" "$REM/peer_mw_trans$IDX"
333
334 if [[ $MW_SIZE -ne $MW_ALIGNED_SIZE ]]; then
335 echo "MW $IDX size aligned to $MW_ALIGNED_SIZE"
336 fi
337 }
338
339 function write_mw()
340 {
341 split_remote $2
342
343 if [[ "$REMOTE" != "" ]]; then
344 ssh "$REMOTE" \
345 dd if=/dev/urandom "of=$VPATH" 2> /dev/null || true
346 else
347 dd if=/dev/urandom "of=$VPATH" 2> /dev/null || true
348 fi
349 }
350
351 function mw_check()
352 {
353 IDX=$1
354 LOC=$2
355 REM=$3
356
357 write_mw "$LOC/mw$IDX"
358
359 split_remote "$LOC/mw$IDX"
360 if [[ "$REMOTE" == "" ]]; then
361 A=$VPATH
362 else
363 A=/tmp/ntb_test.$$.A
364 ssh "$REMOTE" cat "$VPATH" > "$A"
365 fi
366
367 split_remote "$REM/peer_mw$IDX"
368 if [[ "$REMOTE" == "" ]]; then
369 B=$VPATH
370 else
371 B=/tmp/ntb_test.$$.B
372 ssh "$REMOTE" cat "$VPATH" > "$B"
373 fi
374
375 cmp -n $MW_ALIGNED_SIZE "$A" "$B"
376 if [[ $? != 0 ]]; then
377 echo "Memory window $MW did not match!" >&2
378 fi
379
380 if [[ "$A" == "/tmp/*" ]]; then
381 rm "$A"
382 fi
383
384 if [[ "$B" == "/tmp/*" ]]; then
385 rm "$B"
386 fi
387 }
388
389 function mw_free()
390 {
391 IDX=$1
392 LOC=$2
393 REM=$3
394
395 write_file "$MW_DMA_ADDR:0" "$REM/peer_mw_trans$IDX"
396
397 write_file 0 "$LOC/mw_trans$IDX"
398 }
399
400 function mw_test()
401 {
402 LOC=$1
403 REM=$2
404
405 CNT=$(get_files_count "mw_trans" "$LOC")
406
407 for ((i = 0; i < $CNT; i++)); do
408 echo "Running mw$i tests on: $(subdirname $LOC) / " \
409 "$(subdirname $REM)"
410
411 mw_alloc $i $LOC $REM
412
413 mw_check $i $LOC $REM
414
415 mw_free $i $LOC $REM
416
417 echo " Passed"
418 done
419
420 }
421
422 function pingpong_test()
423 {
424 LOC=$1
425 REM=$2
426
427 echo "Running ping pong tests on: $(basename $LOC) / $(basename $REM)"
428
429 LOC_START=$(read_file "$LOC/count")
430 REM_START=$(read_file "$REM/count")
431
432 sleep 7
433
434 LOC_END=$(read_file "$LOC/count")
435 REM_END=$(read_file "$REM/count")
436
437 if [[ $LOC_START == $LOC_END ]] || [[ $REM_START == $REM_END ]]; then
438 echo "Ping pong counter not incrementing!" >&2
439 exit 1
440 fi
441
442 echo " Passed"
443 }
444
445 function msi_test()
446 {
447 LOC=$1
448 REM=$2
449
450 write_file 1 $LOC/ready
451
452 echo "Running MSI interrupt tests on: $(subdirname $LOC) / $(subdirname $REM)"
453
454 CNT=$(read_file "$LOC/count")
455 for ((i = 0; i < $CNT; i++)); do
456 START=$(read_file $REM/../irq${i}_occurrences)
457 write_file $i $LOC/trigger
458 END=$(read_file $REM/../irq${i}_occurrences)
459
460 if [[ $(($END - $START)) != 1 ]]; then
461 echo "MSI did not trigger the interrupt on the remote side!" >&2
462 exit 1
463 fi
464 done
465
466 echo " Passed"
467 }
468
469 function perf_test()
470 {
471 USE_DMA=$1
472
473 if [[ $USE_DMA == "1" ]]; then
474 WITH="with"
475 else
476 WITH="without"
477 fi
478
479 _modprobe ntb_perf total_order=$PERF_RUN_ORDER \
480 max_mw_size=$MAX_MW_SIZE use_dma=$USE_DMA
481
482 echo "Running local perf test $WITH DMA"
483 write_file "$LOCAL_PIDX" "$LOCAL_PERF/run"
484 echo -n " "
485 read_file "$LOCAL_PERF/run"
486 echo " Passed"
487
488 echo "Running remote perf test $WITH DMA"
489 write_file "$REMOTE_PIDX" "$REMOTE_PERF/run"
490 echo -n " "
491 read_file "$REMOTE_PERF/run"
492 echo " Passed"
493
494 _modprobe -r ntb_perf
495 }
496
497 function ntb_tool_tests()
498 {
499 LOCAL_TOOL="$DEBUGFS/ntb_tool/$LOCAL_DEV"
500 REMOTE_TOOL="$REMOTE_HOST:$DEBUGFS/ntb_tool/$REMOTE_DEV"
501
502 echo "Starting ntb_tool tests..."
503
504 _modprobe ntb_tool
505
506 port_test "$LOCAL_TOOL" "$REMOTE_TOOL"
507
508 LOCAL_PEER_TOOL="$LOCAL_TOOL/peer$LOCAL_PIDX"
509 REMOTE_PEER_TOOL="$REMOTE_TOOL/peer$REMOTE_PIDX"
510
511 link_test "$LOCAL_PEER_TOOL" "$REMOTE_PEER_TOOL"
512 link_test "$REMOTE_PEER_TOOL" "$LOCAL_PEER_TOOL"
513
514 #Ensure the link is up on both sides before continuing
515 write_file "Y" "$LOCAL_PEER_TOOL/link_event"
516 write_file "Y" "$REMOTE_PEER_TOOL/link_event"
517
518 doorbell_test "$LOCAL_TOOL" "$REMOTE_TOOL"
519 doorbell_test "$REMOTE_TOOL" "$LOCAL_TOOL"
520
521 scratchpad_test "$LOCAL_PEER_TOOL" "$REMOTE_PEER_TOOL"
522 scratchpad_test "$REMOTE_PEER_TOOL" "$LOCAL_PEER_TOOL"
523
524 message_test "$LOCAL_PEER_TOOL" "$REMOTE_PEER_TOOL"
525 message_test "$REMOTE_PEER_TOOL" "$LOCAL_PEER_TOOL"
526
527 mw_test "$LOCAL_PEER_TOOL" "$REMOTE_PEER_TOOL"
528 mw_test "$REMOTE_PEER_TOOL" "$LOCAL_PEER_TOOL"
529
530 _modprobe -r ntb_tool
531 }
532
533 function ntb_pingpong_tests()
534 {
535 LOCAL_PP="$DEBUGFS/ntb_pingpong/$LOCAL_DEV"
536 REMOTE_PP="$REMOTE_HOST:$DEBUGFS/ntb_pingpong/$REMOTE_DEV"
537
538 echo "Starting ntb_pingpong tests..."
539
540 _modprobe ntb_pingpong
541
542 pingpong_test $LOCAL_PP $REMOTE_PP
543
544 _modprobe -r ntb_pingpong
545 }
546
547 function ntb_msi_tests()
548 {
549 LOCAL_MSI="$DEBUGFS/ntb_msi_test/$LOCAL_DEV"
550 REMOTE_MSI="$REMOTE_HOST:$DEBUGFS/ntb_msi_test/$REMOTE_DEV"
551
552 echo "Starting ntb_msi_test tests..."
553
554 if ! _modprobe ntb_msi_test 2> /dev/null; then
555 echo " Not doing MSI tests seeing the module is not available."
556 return
557 fi
558
559 port_test $LOCAL_MSI $REMOTE_MSI
560
561 LOCAL_PEER="$LOCAL_MSI/peer$LOCAL_PIDX"
562 REMOTE_PEER="$REMOTE_MSI/peer$REMOTE_PIDX"
563
564 msi_test $LOCAL_PEER $REMOTE_PEER
565 msi_test $REMOTE_PEER $LOCAL_PEER
566
567 _modprobe -r ntb_msi_test
568 }
569
570 function ntb_perf_tests()
571 {
572 LOCAL_PERF="$DEBUGFS/ntb_perf/$LOCAL_DEV"
573 REMOTE_PERF="$REMOTE_HOST:$DEBUGFS/ntb_perf/$REMOTE_DEV"
574
575 echo "Starting ntb_perf tests..."
576
577 perf_test 0
578
579 if [[ $RUN_DMA_TESTS ]]; then
580 perf_test 1
581 fi
582 }
583
584 function cleanup()
585 {
586 set +e
587 _modprobe -r ntb_tool 2> /dev/null
588 _modprobe -r ntb_perf 2> /dev/null
589 _modprobe -r ntb_pingpong 2> /dev/null
590 _modprobe -r ntb_transport 2> /dev/null
591 _modprobe -r ntb_msi_test 2> /dev/null
592 set -e
593 }
594
595 cleanup
596
597 if ! [[ $$DONT_CLEANUP ]]; then
598 trap cleanup EXIT
599 fi
600
601 if [ "$(id -u)" != "0" ]; then
602 echo "This script must be run as root" 1>&2
603 exit 1
604 fi
605
606 if [[ "$LIST_DEVS" == TRUE ]]; then
607 echo "Local Devices:"
608 ls -1 /sys/bus/ntb/devices
609 echo
610
611 if [[ "$REMOTE_HOST" != "" ]]; then
612 echo "Remote Devices:"
613 ssh $REMOTE_HOST ls -1 /sys/bus/ntb/devices
614 fi
615
616 exit 0
617 fi
618
619 if [[ "$LOCAL_DEV" == $"" ]] || [[ "$REMOTE_DEV" == $"" ]]; then
620 show_help
621 exit 1
622 fi
623
624 ntb_tool_tests
625 echo
626 ntb_pingpong_tests
627 echo
628 ntb_msi_tests
629 echo
630 ntb_perf_tests
631 echo
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.