1 #!/bin/bash 1 #!/bin/bash
2 # SPDX-License-Identifier: GPL-2.0+ 2 # SPDX-License-Identifier: GPL-2.0+
3 # 3 #
4 # Run a series of tests on remote systems unde 4 # Run a series of tests on remote systems under KVM.
5 # 5 #
6 # Usage: kvm-remote.sh "systems" [ <kvm.sh arg 6 # Usage: kvm-remote.sh "systems" [ <kvm.sh args> ]
7 # kvm-remote.sh "systems" /path/to/old/ 7 # kvm-remote.sh "systems" /path/to/old/run [ <kvm-again.sh args> ]
8 # 8 #
9 # Copyright (C) 2021 Facebook, Inc. 9 # Copyright (C) 2021 Facebook, Inc.
10 # 10 #
11 # Authors: Paul E. McKenney <paulmck@kernel.org 11 # Authors: Paul E. McKenney <paulmck@kernel.org>
12 12
13 scriptname=$0 13 scriptname=$0
14 args="$*" 14 args="$*"
15 15
16 if ! test -d tools/testing/selftests/rcutortur 16 if ! test -d tools/testing/selftests/rcutorture/bin
17 then 17 then
18 echo $scriptname must be run from top- 18 echo $scriptname must be run from top-level directory of kernel source tree.
19 exit 1 19 exit 1
20 fi 20 fi
21 21
22 RCUTORTURE="`pwd`/tools/testing/selftests/rcut 22 RCUTORTURE="`pwd`/tools/testing/selftests/rcutorture"; export RCUTORTURE
23 PATH=${RCUTORTURE}/bin:$PATH; export PATH 23 PATH=${RCUTORTURE}/bin:$PATH; export PATH
24 . functions.sh 24 . functions.sh
25 25
26 starttime="`get_starttime`" 26 starttime="`get_starttime`"
27 27
28 systems="$1" 28 systems="$1"
29 if test -z "$systems" 29 if test -z "$systems"
30 then 30 then
31 echo $scriptname: Empty list of system 31 echo $scriptname: Empty list of systems will go nowhere good, giving up.
32 exit 1 32 exit 1
33 fi 33 fi
34 shift 34 shift
35 35
36 # Pathnames: 36 # Pathnames:
37 # T: /tmp/kvm-remote.sh.NNNNNN where "NNN 37 # T: /tmp/kvm-remote.sh.NNNNNN where "NNNNNN" is set by mktemp
38 # resdir: /tmp/kvm-remote.sh.NNNNNN/res 38 # resdir: /tmp/kvm-remote.sh.NNNNNN/res
39 # rundir: /tmp/kvm-remote.sh.NNNNNN/res/$ds (" 39 # rundir: /tmp/kvm-remote.sh.NNNNNN/res/$ds ("-remote" suffix)
40 # oldrun: `pwd`/tools/testing/.../res/$otherds 40 # oldrun: `pwd`/tools/testing/.../res/$otherds
41 # 41 #
42 # Pathname segments: 42 # Pathname segments:
43 # TD: kvm-remote.sh.NNNNNN 43 # TD: kvm-remote.sh.NNNNNN
44 # ds: yyyy.mm.dd-hh.mm.ss-remote 44 # ds: yyyy.mm.dd-hh.mm.ss-remote
45 45
46 T="`mktemp -d ${TMPDIR-/tmp}/kvm-remote.sh.XXX 46 T="`mktemp -d ${TMPDIR-/tmp}/kvm-remote.sh.XXXXXX`"
47 trap 'rm -rf $T' 0 47 trap 'rm -rf $T' 0
48 TD="`basename "$T"`" 48 TD="`basename "$T"`"
49 49
50 resdir="$T/res" 50 resdir="$T/res"
51 ds=`date +%Y.%m.%d-%H.%M.%S`-remote 51 ds=`date +%Y.%m.%d-%H.%M.%S`-remote
52 rundir=$resdir/$ds 52 rundir=$resdir/$ds
53 echo Results directory: $rundir 53 echo Results directory: $rundir
54 echo $scriptname $args 54 echo $scriptname $args
55 if echo $1 | grep -q '^--' 55 if echo $1 | grep -q '^--'
56 then 56 then
57 # Fresh build. Create a datestamp unl 57 # Fresh build. Create a datestamp unless the caller supplied one.
58 datestamp="`echo "$@" | awk -v ds="$ds 58 datestamp="`echo "$@" | awk -v ds="$ds" '{
59 for (i = 1; i < NF; i++) { 59 for (i = 1; i < NF; i++) {
60 if ($i == "--datestamp 60 if ($i == "--datestamp") {
61 ds = ""; 61 ds = "";
62 break; 62 break;
63 } 63 }
64 } 64 }
65 if (ds != "") 65 if (ds != "")
66 print "--datestamp " d 66 print "--datestamp " ds;
67 }'`" 67 }'`"
68 kvm.sh --remote "$@" $datestamp --buil 68 kvm.sh --remote "$@" $datestamp --buildonly > $T/kvm.sh.out 2>&1
69 ret=$? 69 ret=$?
70 if test "$ret" -ne 0 70 if test "$ret" -ne 0
71 then 71 then
72 echo $scriptname: kvm.sh faile 72 echo $scriptname: kvm.sh failed exit code $?
73 cat $T/kvm.sh.out 73 cat $T/kvm.sh.out
74 exit 2 74 exit 2
75 fi 75 fi
76 oldrun="`grep -m 1 "^Results directory 76 oldrun="`grep -m 1 "^Results directory: " $T/kvm.sh.out | awk '{ print $3 }'`"
77 touch "$oldrun/remote-log" 77 touch "$oldrun/remote-log"
78 echo $scriptname $args >> "$oldrun/rem 78 echo $scriptname $args >> "$oldrun/remote-log"
79 echo | tee -a "$oldrun/remote-log" 79 echo | tee -a "$oldrun/remote-log"
80 echo " ----" kvm.sh output: "(`date`)" 80 echo " ----" kvm.sh output: "(`date`)" | tee -a "$oldrun/remote-log"
81 cat $T/kvm.sh.out | tee -a "$oldrun/re 81 cat $T/kvm.sh.out | tee -a "$oldrun/remote-log"
82 # We are going to run this, so remove 82 # We are going to run this, so remove the buildonly files.
83 rm -f "$oldrun"/*/buildonly 83 rm -f "$oldrun"/*/buildonly
84 kvm-again.sh $oldrun --dryrun --remote 84 kvm-again.sh $oldrun --dryrun --remote --rundir "$rundir" > $T/kvm-again.sh.out 2>&1
85 ret=$? 85 ret=$?
86 if test "$ret" -ne 0 86 if test "$ret" -ne 0
87 then 87 then
88 echo $scriptname: kvm-again.sh 88 echo $scriptname: kvm-again.sh failed exit code $? | tee -a "$oldrun/remote-log"
89 cat $T/kvm-again.sh.out | tee 89 cat $T/kvm-again.sh.out | tee -a "$oldrun/remote-log"
90 exit 2 90 exit 2
91 fi 91 fi
92 else 92 else
93 # Re-use old run. 93 # Re-use old run.
94 oldrun="$1" 94 oldrun="$1"
95 if ! echo $oldrun | grep -q '^/' 95 if ! echo $oldrun | grep -q '^/'
96 then 96 then
97 oldrun="`pwd`/$oldrun" 97 oldrun="`pwd`/$oldrun"
98 fi 98 fi
99 shift 99 shift
100 touch "$oldrun/remote-log" 100 touch "$oldrun/remote-log"
101 echo $scriptname $args >> "$oldrun/rem 101 echo $scriptname $args >> "$oldrun/remote-log"
102 kvm-again.sh "$oldrun" "$@" --dryrun - 102 kvm-again.sh "$oldrun" "$@" --dryrun --remote --rundir "$rundir" > $T/kvm-again.sh.out 2>&1
103 ret=$? 103 ret=$?
104 if test "$ret" -ne 0 104 if test "$ret" -ne 0
105 then 105 then
106 echo $scriptname: kvm-again.sh 106 echo $scriptname: kvm-again.sh failed exit code $? | tee -a "$oldrun/remote-log"
107 cat $T/kvm-again.sh.out | tee 107 cat $T/kvm-again.sh.out | tee -a "$oldrun/remote-log"
108 exit 2 108 exit 2
109 fi 109 fi
110 cp -a "$rundir" "$RCUTORTURE/res/" 110 cp -a "$rundir" "$RCUTORTURE/res/"
111 oldrun="$RCUTORTURE/res/$ds" 111 oldrun="$RCUTORTURE/res/$ds"
112 fi 112 fi
113 echo | tee -a "$oldrun/remote-log" 113 echo | tee -a "$oldrun/remote-log"
114 echo " ----" kvm-again.sh output: "(`date`)" | 114 echo " ----" kvm-again.sh output: "(`date`)" | tee -a "$oldrun/remote-log"
115 cat $T/kvm-again.sh.out 115 cat $T/kvm-again.sh.out
116 echo | tee -a "$oldrun/remote-log" 116 echo | tee -a "$oldrun/remote-log"
117 echo Remote run directory: $rundir | tee -a "$ 117 echo Remote run directory: $rundir | tee -a "$oldrun/remote-log"
118 echo Local build-side run directory: $oldrun | 118 echo Local build-side run directory: $oldrun | tee -a "$oldrun/remote-log"
119 119
120 # Create the kvm-remote-N.sh scripts in the bi 120 # Create the kvm-remote-N.sh scripts in the bin directory.
121 awk < "$rundir"/scenarios -v dest="$T/bin" -v 121 awk < "$rundir"/scenarios -v dest="$T/bin" -v rundir="$rundir" '
122 { 122 {
123 n = $1; 123 n = $1;
124 sub(/\./, "", n); 124 sub(/\./, "", n);
125 fn = dest "/kvm-remote-" n ".sh" 125 fn = dest "/kvm-remote-" n ".sh"
126 print "kvm-remote-noreap.sh " rundir " 126 print "kvm-remote-noreap.sh " rundir " &" > fn;
127 scenarios = ""; 127 scenarios = "";
128 for (i = 2; i <= NF; i++) 128 for (i = 2; i <= NF; i++)
129 scenarios = scenarios " " $i; 129 scenarios = scenarios " " $i;
130 print "kvm-test-1-run-batch.sh" scenar 130 print "kvm-test-1-run-batch.sh" scenarios >> fn;
131 print "sync" >> fn; 131 print "sync" >> fn;
132 print "rm " rundir "/remote.run" >> fn 132 print "rm " rundir "/remote.run" >> fn;
133 }' 133 }'
134 chmod +x $T/bin/kvm-remote-*.sh 134 chmod +x $T/bin/kvm-remote-*.sh
135 ( cd "`dirname $T`"; tar -chzf $T/binres.tgz " 135 ( cd "`dirname $T`"; tar -chzf $T/binres.tgz "$TD/bin" "$TD/res" )
136 136
137 # Check first to avoid the need for cleanup fo 137 # Check first to avoid the need for cleanup for system-name typos
138 for i in $systems 138 for i in $systems
139 do 139 do
140 ssh -o BatchMode=yes $i getconf _NPROC 140 ssh -o BatchMode=yes $i getconf _NPROCESSORS_ONLN > $T/ssh.stdout 2> $T/ssh.stderr
141 ret=$? 141 ret=$?
142 if test "$ret" -ne 0 142 if test "$ret" -ne 0
143 then 143 then
144 echo "System $i unreachable ($ 144 echo "System $i unreachable ($ret), giving up." | tee -a "$oldrun/remote-log"
145 echo ' --- ssh stdout: vvv' | 145 echo ' --- ssh stdout: vvv' | tee -a "$oldrun/remote-log"
146 cat $T/ssh.stdout | tee -a "$o 146 cat $T/ssh.stdout | tee -a "$oldrun/remote-log"
147 echo ' --- ssh stdout: ^^^' | 147 echo ' --- ssh stdout: ^^^' | tee -a "$oldrun/remote-log"
148 echo ' --- ssh stderr: vvv' | 148 echo ' --- ssh stderr: vvv' | tee -a "$oldrun/remote-log"
149 cat $T/ssh.stderr | tee -a "$o 149 cat $T/ssh.stderr | tee -a "$oldrun/remote-log"
150 echo ' --- ssh stderr: ^^^' | 150 echo ' --- ssh stderr: ^^^' | tee -a "$oldrun/remote-log"
151 exit 4 151 exit 4
152 fi 152 fi
153 echo $i: `cat $T/ssh.stdout` CPUs " " 153 echo $i: `cat $T/ssh.stdout` CPUs " " `date` | tee -a "$oldrun/remote-log"
154 done 154 done
155 155
156 # Download and expand the tarball on all syste 156 # Download and expand the tarball on all systems.
157 echo Build-products tarball: `du -h $T/binres. 157 echo Build-products tarball: `du -h $T/binres.tgz` | tee -a "$oldrun/remote-log"
158 for i in $systems 158 for i in $systems
159 do 159 do
160 echo Downloading tarball to $i `date` 160 echo Downloading tarball to $i `date` | tee -a "$oldrun/remote-log"
161 cat $T/binres.tgz | ssh -o BatchMode=y 161 cat $T/binres.tgz | ssh -o BatchMode=yes $i "cd /tmp; tar -xzf -"
162 ret=$? 162 ret=$?
163 tries=0 163 tries=0
164 while test "$ret" -ne 0 164 while test "$ret" -ne 0
165 do 165 do
166 echo Unable to download $T/bin 166 echo Unable to download $T/binres.tgz to system $i, waiting and then retrying. $tries prior retries. | tee -a "$oldrun/remote-log"
167 sleep 60 167 sleep 60
168 cat $T/binres.tgz | ssh -o Bat 168 cat $T/binres.tgz | ssh -o BatchMode=yes $i "cd /tmp; tar -xzf -"
169 ret=$? 169 ret=$?
170 if test "$ret" -ne 0 170 if test "$ret" -ne 0
171 then 171 then
172 if test "$tries" > 5 172 if test "$tries" > 5
173 then 173 then
174 echo Unable to 174 echo Unable to download $T/binres.tgz to system $i, giving up. | tee -a "$oldrun/remote-log"
175 exit 10 175 exit 10
176 fi 176 fi
177 fi 177 fi
178 tries=$((tries+1)) 178 tries=$((tries+1))
179 done 179 done
180 done 180 done
181 181
182 # Function to check for presence of a file on 182 # Function to check for presence of a file on the specified system.
183 # Complain if the system cannot be reached, an 183 # Complain if the system cannot be reached, and retry after a wait.
184 # Currently just waits forever if a machine di 184 # Currently just waits forever if a machine disappears.
185 # 185 #
186 # Usage: checkremotefile system pathname 186 # Usage: checkremotefile system pathname
187 checkremotefile () { 187 checkremotefile () {
188 local ret 188 local ret
189 local sleeptime=60 189 local sleeptime=60
190 190
191 while : 191 while :
192 do 192 do
193 ssh -o BatchMode=yes $1 "test 193 ssh -o BatchMode=yes $1 "test -f \"$2\""
194 ret=$? 194 ret=$?
195 if test "$ret" -eq 255 195 if test "$ret" -eq 255
196 then 196 then
197 echo " ---" ssh failur 197 echo " ---" ssh failure to $1 checking for file $2, retry after $sleeptime seconds. `date` | tee -a "$oldrun/remote-log"
198 elif test "$ret" -eq 0 198 elif test "$ret" -eq 0
199 then 199 then
200 return 0 200 return 0
201 elif test "$ret" -eq 1 201 elif test "$ret" -eq 1
202 then 202 then
203 echo " ---" File \"$2\ 203 echo " ---" File \"$2\" not found: ssh $1 test -f \"$2\" | tee -a "$oldrun/remote-log"
204 return 1 204 return 1
205 else 205 else
206 echo " ---" Exit code 206 echo " ---" Exit code $ret: ssh $1 test -f \"$2\", retry after $sleeptime seconds. `date` | tee -a "$oldrun/remote-log"
207 return $ret 207 return $ret
208 fi 208 fi
209 sleep $sleeptime 209 sleep $sleeptime
210 done 210 done
211 } 211 }
212 212
213 # Function to start batches on idle remote $sy 213 # Function to start batches on idle remote $systems
214 # 214 #
215 # Usage: startbatches curbatch nbatches 215 # Usage: startbatches curbatch nbatches
216 # 216 #
217 # Batches are numbered starting at 1. Returns 217 # Batches are numbered starting at 1. Returns the next batch to start.
218 # Be careful to redirect all debug output to F 218 # Be careful to redirect all debug output to FD 2 (stderr).
219 startbatches () { 219 startbatches () {
220 local curbatch="$1" 220 local curbatch="$1"
221 local nbatches="$2" 221 local nbatches="$2"
222 local ret 222 local ret
223 223
224 # Each pass through the following loop 224 # Each pass through the following loop examines one system.
225 for i in $systems 225 for i in $systems
226 do 226 do
227 if test "$curbatch" -gt "$nbat 227 if test "$curbatch" -gt "$nbatches"
228 then 228 then
229 echo $((nbatches + 1)) 229 echo $((nbatches + 1))
230 return 0 230 return 0
231 fi 231 fi
232 if checkremotefile "$i" "$resd 232 if checkremotefile "$i" "$resdir/$ds/remote.run" 1>&2
233 then 233 then
234 continue # System stil 234 continue # System still running last test, skip.
235 fi 235 fi
236 ssh -o BatchMode=yes "$i" "cd 236 ssh -o BatchMode=yes "$i" "cd \"$resdir/$ds\"; touch remote.run; PATH=\"$T/bin:$PATH\" nohup kvm-remote-$curbatch.sh > kvm-remote-$curbatch.sh.out 2>&1 &" 1>&2
237 ret=$? 237 ret=$?
238 if test "$ret" -ne 0 238 if test "$ret" -ne 0
239 then 239 then
240 echo ssh $i failed: ex 240 echo ssh $i failed: exitcode $ret 1>&2
241 exit 11 241 exit 11
242 fi 242 fi
243 echo " ----" System $i Batch ` 243 echo " ----" System $i Batch `head -n $curbatch < "$rundir"/scenarios | tail -1` `date` 1>&2
244 curbatch=$((curbatch + 1)) 244 curbatch=$((curbatch + 1))
245 done 245 done
246 echo $curbatch 246 echo $curbatch
247 } 247 }
248 248
249 # Launch all the scenarios. 249 # Launch all the scenarios.
250 nbatches="`wc -l "$rundir"/scenarios | awk '{ 250 nbatches="`wc -l "$rundir"/scenarios | awk '{ print $1 }'`"
251 curbatch=1 251 curbatch=1
252 while test "$curbatch" -le "$nbatches" 252 while test "$curbatch" -le "$nbatches"
253 do 253 do
254 startbatches $curbatch $nbatches > $T/ 254 startbatches $curbatch $nbatches > $T/curbatch 2> $T/startbatches.stderr
255 curbatch="`cat $T/curbatch`" 255 curbatch="`cat $T/curbatch`"
256 if test -s "$T/startbatches.stderr" 256 if test -s "$T/startbatches.stderr"
257 then 257 then
258 cat "$T/startbatches.stderr" | 258 cat "$T/startbatches.stderr" | tee -a "$oldrun/remote-log"
259 fi 259 fi
260 if test "$curbatch" -le "$nbatches" 260 if test "$curbatch" -le "$nbatches"
261 then 261 then
262 sleep 30 262 sleep 30
263 fi 263 fi
264 done 264 done
265 echo All batches started. `date` | tee -a "$ol 265 echo All batches started. `date` | tee -a "$oldrun/remote-log"
266 266
267 # Wait for all remaining scenarios to complete 267 # Wait for all remaining scenarios to complete and collect results.
268 for i in $systems 268 for i in $systems
269 do 269 do
270 echo " ---" Waiting for $i `date` | te 270 echo " ---" Waiting for $i `date` | tee -a "$oldrun/remote-log"
271 while checkremotefile "$i" "$resdir/$d 271 while checkremotefile "$i" "$resdir/$ds/remote.run"
272 do 272 do
273 sleep 30 273 sleep 30
274 done 274 done
275 echo " ---" Collecting results from $i 275 echo " ---" Collecting results from $i `date` | tee -a "$oldrun/remote-log"
276 ( cd "$oldrun"; ssh -o BatchMode=yes $ 276 ( cd "$oldrun"; ssh -o BatchMode=yes $i "cd $rundir; tar -czf - kvm-remote-*.sh.out */console.log */kvm-test-1-run*.sh.out */qemu[_-]pid */qemu-retval */qemu-affinity; rm -rf $T > /dev/null 2>&1" | tar -xzf - )
277 done 277 done
278 278
279 ( kvm-end-run-stats.sh "$oldrun" "$starttime"; 279 ( kvm-end-run-stats.sh "$oldrun" "$starttime"; echo $? > $T/exitcode ) | tee -a "$oldrun/remote-log"
280 exit "`cat $T/exitcode`" 280 exit "`cat $T/exitcode`"
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.