1 #!/bin/bash
2 # SPDX-License-Identifier: GPL-2.0
3 #
4 # Test for cpuset v2 partition root state (PRS)
5 #
6 # The sched verbose flag can be optionally set so that the console log
7 # can be examined for the correct setting of scheduling domain.
8 #
9
10 skip_test() {
11 echo "$1"
12 echo "Test SKIPPED"
13 exit 4 # ksft_skip
14 }
15
16 [[ $(id -u) -eq 0 ]] || skip_test "Test must be run as root!"
17
18
19 # Get wait_inotify location
20 WAIT_INOTIFY=$(cd $(dirname $0); pwd)/wait_inotify
21
22 # Find cgroup v2 mount point
23 CGROUP2=$(mount -t cgroup2 | head -1 | awk -e '{print $3}')
24 [[ -n "$CGROUP2" ]] || skip_test "Cgroup v2 mount point not found!"
25 SUBPARTS_CPUS=$CGROUP2/.__DEBUG__.cpuset.cpus.subpartitions
26 CPULIST=$(cat $CGROUP2/cpuset.cpus.effective)
27
28 NR_CPUS=$(lscpu | grep "^CPU(s):" | sed -e "s/.*:[[:space:]]*//")
29 [[ $NR_CPUS -lt 8 ]] && skip_test "Test needs at least 8 cpus available!"
30
31 # Check to see if /dev/console exists and is writable
32 if [[ -c /dev/console && -w /dev/console ]]
33 then
34 CONSOLE=/dev/console
35 else
36 CONSOLE=/dev/null
37 fi
38
39 # Set verbose flag and delay factor
40 PROG=$1
41 VERBOSE=0
42 DELAY_FACTOR=1
43 SCHED_DEBUG=
44 while [[ "$1" = -* ]]
45 do
46 case "$1" in
47 -v) ((VERBOSE++))
48 # Enable sched/verbose can slow thing down
49 [[ $DELAY_FACTOR -eq 1 ]] &&
50 DELAY_FACTOR=2
51 ;;
52 -d) DELAY_FACTOR=$2
53 shift
54 ;;
55 *) echo "Usage: $PROG [-v] [-d <delay-factor>"
56 exit
57 ;;
58 esac
59 shift
60 done
61
62 # Set sched verbose flag if available when "-v" option is specified
63 if [[ $VERBOSE -gt 0 && -d /sys/kernel/debug/sched ]]
64 then
65 # Used to restore the original setting during cleanup
66 SCHED_DEBUG=$(cat /sys/kernel/debug/sched/verbose)
67 echo Y > /sys/kernel/debug/sched/verbose
68 fi
69
70 cd $CGROUP2
71 echo +cpuset > cgroup.subtree_control
72
73 #
74 # If cpuset has been set up and used in child cgroups, we may not be able to
75 # create partition under root cgroup because of the CPU exclusivity rule.
76 # So we are going to skip the test if this is the case.
77 #
78 [[ -d test ]] || mkdir test
79 echo 0-6 > test/cpuset.cpus
80 echo root > test/cpuset.cpus.partition
81 cat test/cpuset.cpus.partition | grep -q invalid
82 RESULT=$?
83 echo member > test/cpuset.cpus.partition
84 echo "" > test/cpuset.cpus
85 [[ $RESULT -eq 0 ]] && skip_test "Child cgroups are using cpuset!"
86
87 #
88 # If isolated CPUs have been reserved at boot time (as shown in
89 # cpuset.cpus.isolated), these isolated CPUs should be outside of CPUs 0-7
90 # that will be used by this script for testing purpose. If not, some of
91 # the tests may fail incorrectly. These isolated CPUs will also be removed
92 # before being compared with the expected results.
93 #
94 BOOT_ISOLCPUS=$(cat $CGROUP2/cpuset.cpus.isolated)
95 if [[ -n "$BOOT_ISOLCPUS" ]]
96 then
97 [[ $(echo $BOOT_ISOLCPUS | sed -e "s/[,-].*//") -le 7 ]] &&
98 skip_test "Pre-isolated CPUs ($BOOT_ISOLCPUS) overlap CPUs to be tested"
99 echo "Pre-isolated CPUs: $BOOT_ISOLCPUS"
100 fi
101 cleanup()
102 {
103 online_cpus
104 cd $CGROUP2
105 rmdir A1/A2/A3 A1/A2 A1 B1 > /dev/null 2>&1
106 rmdir test > /dev/null 2>&1
107 [[ -n "$SCHED_DEBUG" ]] &&
108 echo "$SCHED_DEBUG" > /sys/kernel/debug/sched/verbose
109 }
110
111 # Pause in ms
112 pause()
113 {
114 DELAY=$1
115 LOOP=0
116 while [[ $LOOP -lt $DELAY_FACTOR ]]
117 do
118 sleep $DELAY
119 ((LOOP++))
120 done
121 return 0
122 }
123
124 console_msg()
125 {
126 MSG=$1
127 echo "$MSG"
128 echo "" > $CONSOLE
129 echo "$MSG" > $CONSOLE
130 pause 0.01
131 }
132
133 test_partition()
134 {
135 EXPECTED_VAL=$1
136 echo $EXPECTED_VAL > cpuset.cpus.partition
137 [[ $? -eq 0 ]] || exit 1
138 ACTUAL_VAL=$(cat cpuset.cpus.partition)
139 [[ $ACTUAL_VAL != $EXPECTED_VAL ]] && {
140 echo "cpuset.cpus.partition: expect $EXPECTED_VAL, found $ACTUAL_VAL"
141 echo "Test FAILED"
142 exit 1
143 }
144 }
145
146 test_effective_cpus()
147 {
148 EXPECTED_VAL=$1
149 ACTUAL_VAL=$(cat cpuset.cpus.effective)
150 [[ "$ACTUAL_VAL" != "$EXPECTED_VAL" ]] && {
151 echo "cpuset.cpus.effective: expect '$EXPECTED_VAL', found '$ACTUAL_VAL'"
152 echo "Test FAILED"
153 exit 1
154 }
155 }
156
157 # Adding current process to cgroup.procs as a test
158 test_add_proc()
159 {
160 OUTSTR="$1"
161 ERRMSG=$((echo $$ > cgroup.procs) |& cat)
162 echo $ERRMSG | grep -q "$OUTSTR"
163 [[ $? -ne 0 ]] && {
164 echo "cgroup.procs: expect '$OUTSTR', got '$ERRMSG'"
165 echo "Test FAILED"
166 exit 1
167 }
168 echo $$ > $CGROUP2/cgroup.procs # Move out the task
169 }
170
171 #
172 # Cpuset controller state transition test matrix.
173 #
174 # Cgroup test hierarchy
175 #
176 # root -- A1 -- A2 -- A3
177 # +- B1
178 #
179 # P<v> = set cpus.partition (0:member, 1:root, 2:isolated)
180 # C<l> = add cpu-list to cpuset.cpus
181 # X<l> = add cpu-list to cpuset.cpus.exclusive
182 # S<p> = use prefix in subtree_control
183 # T = put a task into cgroup
184 # O<c>=<v> = Write <v> to CPU online file of <c>
185 #
186 # ECPUs - effective CPUs of cpusets
187 # Pstate - partition root state
188 # ISOLCPUS - isolated CPUs (<icpus>[,<icpus2>])
189 #
190 # Note that if there are 2 fields in ISOLCPUS, the first one is for
191 # sched-debug matching which includes offline CPUs and single-CPU partitions
192 # while the second one is for matching cpuset.cpus.isolated.
193 #
194 SETUP_A123_PARTITIONS="C1-3:P1:S+ C2-3:P1:S+ C3:P1"
195 TEST_MATRIX=(
196 # old-A1 old-A2 old-A3 old-B1 new-A1 new-A2 new-A3 new-B1 fail ECPUs Pstate ISOLCPUS
197 # ------ ------ ------ ------ ------ ------ ------ ------ ---- ----- ------ --------
198 " C0-1 . . C2-3 S+ C4-5 . . 0 A2:0-1"
199 " C0-1 . . C2-3 P1 . . . 0 "
200 " C0-1 . . C2-3 P1:S+ C0-1:P1 . . 0 "
201 " C0-1 . . C2-3 P1:S+ C1:P1 . . 0 "
202 " C0-1:S+ . . C2-3 . . . P1 0 "
203 " C0-1:P1 . . C2-3 S+ C1 . . 0 "
204 " C0-1:P1 . . C2-3 S+ C1:P1 . . 0 "
205 " C0-1:P1 . . C2-3 S+ C1:P1 . P1 0 "
206 " C0-1:P1 . . C2-3 C4-5 . . . 0 A1:4-5"
207 " C0-1:P1 . . C2-3 S+:C4-5 . . . 0 A1:4-5"
208 " C0-1 . . C2-3:P1 . . . C2 0 "
209 " C0-1 . . C2-3:P1 . . . C4-5 0 B1:4-5"
210 "C0-3:P1:S+ C2-3:P1 . . . . . . 0 A1:0-1,A2:2-3"
211 "C0-3:P1:S+ C2-3:P1 . . C1-3 . . . 0 A1:1,A2:2-3"
212 "C2-3:P1:S+ C3:P1 . . C3 . . . 0 A1:,A2:3 A1:P1,A2:P1"
213 "C2-3:P1:S+ C3:P1 . . C3 P0 . . 0 A1:3,A2:3 A1:P1,A2:P0"
214 "C2-3:P1:S+ C2:P1 . . C2-4 . . . 0 A1:3-4,A2:2"
215 "C2-3:P1:S+ C3:P1 . . C3 . . C0-2 0 A1:,B1:0-2 A1:P1,A2:P1"
216 "$SETUP_A123_PARTITIONS . C2-3 . . . 0 A1:,A2:2,A3:3 A1:P1,A2:P1,A3:P1"
217
218 # CPU offlining cases:
219 " C0-1 . . C2-3 S+ C4-5 . O2=0 0 A1:0-1,B1:3"
220 "C0-3:P1:S+ C2-3:P1 . . O2=0 . . . 0 A1:0-1,A2:3"
221 "C0-3:P1:S+ C2-3:P1 . . O2=0 O2=1 . . 0 A1:0-1,A2:2-3"
222 "C0-3:P1:S+ C2-3:P1 . . O1=0 . . . 0 A1:0,A2:2-3"
223 "C0-3:P1:S+ C2-3:P1 . . O1=0 O1=1 . . 0 A1:0-1,A2:2-3"
224 "C2-3:P1:S+ C3:P1 . . O3=0 O3=1 . . 0 A1:2,A2:3 A1:P1,A2:P1"
225 "C2-3:P1:S+ C3:P2 . . O3=0 O3=1 . . 0 A1:2,A2:3 A1:P1,A2:P2"
226 "C2-3:P1:S+ C3:P1 . . O2=0 O2=1 . . 0 A1:2,A2:3 A1:P1,A2:P1"
227 "C2-3:P1:S+ C3:P2 . . O2=0 O2=1 . . 0 A1:2,A2:3 A1:P1,A2:P2"
228 "C2-3:P1:S+ C3:P1 . . O2=0 . . . 0 A1:,A2:3 A1:P1,A2:P1"
229 "C2-3:P1:S+ C3:P1 . . O3=0 . . . 0 A1:2,A2: A1:P1,A2:P1"
230 "C2-3:P1:S+ C3:P1 . . T:O2=0 . . . 0 A1:3,A2:3 A1:P1,A2:P-1"
231 "C2-3:P1:S+ C3:P1 . . . T:O3=0 . . 0 A1:2,A2:2 A1:P1,A2:P-1"
232 "$SETUP_A123_PARTITIONS . O1=0 . . . 0 A1:,A2:2,A3:3 A1:P1,A2:P1,A3:P1"
233 "$SETUP_A123_PARTITIONS . O2=0 . . . 0 A1:1,A2:,A3:3 A1:P1,A2:P1,A3:P1"
234 "$SETUP_A123_PARTITIONS . O3=0 . . . 0 A1:1,A2:2,A3: A1:P1,A2:P1,A3:P1"
235 "$SETUP_A123_PARTITIONS . T:O1=0 . . . 0 A1:2-3,A2:2-3,A3:3 A1:P1,A2:P-1,A3:P-1"
236 "$SETUP_A123_PARTITIONS . . T:O2=0 . . 0 A1:1,A2:3,A3:3 A1:P1,A2:P1,A3:P-1"
237 "$SETUP_A123_PARTITIONS . . . T:O3=0 . 0 A1:1,A2:2,A3:2 A1:P1,A2:P1,A3:P-1"
238 "$SETUP_A123_PARTITIONS . T:O1=0 O1=1 . . 0 A1:1,A2:2,A3:3 A1:P1,A2:P1,A3:P1"
239 "$SETUP_A123_PARTITIONS . . T:O2=0 O2=1 . 0 A1:1,A2:2,A3:3 A1:P1,A2:P1,A3:P1"
240 "$SETUP_A123_PARTITIONS . . . T:O3=0 O3=1 0 A1:1,A2:2,A3:3 A1:P1,A2:P1,A3:P1"
241 "$SETUP_A123_PARTITIONS . T:O1=0 O2=0 O1=1 . 0 A1:1,A2:,A3:3 A1:P1,A2:P1,A3:P1"
242 "$SETUP_A123_PARTITIONS . T:O1=0 O2=0 O2=1 . 0 A1:2-3,A2:2-3,A3:3 A1:P1,A2:P-1,A3:P-1"
243
244 # old-A1 old-A2 old-A3 old-B1 new-A1 new-A2 new-A3 new-B1 fail ECPUs Pstate ISOLCPUS
245 # ------ ------ ------ ------ ------ ------ ------ ------ ---- ----- ------ --------
246 #
247 # Remote partition and cpuset.cpus.exclusive tests
248 #
249 " C0-3:S+ C1-3:S+ C2-3 . X2-3 . . . 0 A1:0-3,A2:1-3,A3:2-3,XA1:2-3"
250 " C0-3:S+ C1-3:S+ C2-3 . X2-3 X2-3:P2 . . 0 A1:0-1,A2:2-3,A3:2-3 A1:P0,A2:P2 2-3"
251 " C0-3:S+ C1-3:S+ C2-3 . X2-3 X3:P2 . . 0 A1:0-2,A2:3,A3:3 A1:P0,A2:P2 3"
252 " C0-3:S+ C1-3:S+ C2-3 . X2-3 X2-3 X2-3:P2 . 0 A1:0-1,A2:1,A3:2-3 A1:P0,A3:P2 2-3"
253 " C0-3:S+ C1-3:S+ C2-3 . X2-3 X2-3 X2-3:P2:C3 . 0 A1:0-1,A2:1,A3:2-3 A1:P0,A3:P2 2-3"
254 " C0-3:S+ C1-3:S+ C2-3 C2-3 . . . P2 0 A1:0-3,A2:1-3,A3:2-3,B1:2-3 A1:P0,A3:P0,B1:P-2"
255 " C0-3:S+ C1-3:S+ C2-3 C4-5 . . . P2 0 B1:4-5 B1:P2 4-5"
256 " C0-3:S+ C1-3:S+ C2-3 C4 X2-3 X2-3 X2-3:P2 P2 0 A3:2-3,B1:4 A3:P2,B1:P2 2-4"
257 " C0-3:S+ C1-3:S+ C2-3 C4 X2-3 X2-3 X2-3:P2:C1-3 P2 0 A3:2-3,B1:4 A3:P2,B1:P2 2-4"
258 " C0-3:S+ C1-3:S+ C2-3 C4 X1-3 X1-3:P2 P2 . 0 A2:1,A3:2-3 A2:P2,A3:P2 1-3"
259 " C0-3:S+ C1-3:S+ C2-3 C4 X2-3 X2-3 X2-3:P2 P2:C4-5 0 A3:2-3,B1:4-5 A3:P2,B1:P2 2-5"
260 " C4:X0-3:S+ X1-3:S+ X2-3 . . P2 . . 0 A1:4,A2:1-3,A3:1-3 A2:P2 1-3"
261 " C4:X0-3:S+ X1-3:S+ X2-3 . . . P2 . 0 A1:4,A2:4,A3:2-3 A3:P2 2-3"
262
263 # Nested remote/local partition tests
264 " C0-3:S+ C1-3:S+ C2-3 C4-5 X2-3 X2-3:P1 P2 P1 0 A1:0-1,A2:,A3:2-3,B1:4-5 \
265 A1:P0,A2:P1,A3:P2,B1:P1 2-3"
266 " C0-3:S+ C1-3:S+ C2-3 C4 X2-3 X2-3:P1 P2 P1 0 A1:0-1,A2:,A3:2-3,B1:4 \
267 A1:P0,A2:P1,A3:P2,B1:P1 2-4,2-3"
268 " C0-3:S+ C1-3:S+ C2-3 C4 X2-3 X2-3:P1 . P1 0 A1:0-1,A2:2-3,A3:2-3,B1:4 \
269 A1:P0,A2:P1,A3:P0,B1:P1"
270 " C0-3:S+ C1-3:S+ C3 C4 X2-3 X2-3:P1 P2 P1 0 A1:0-1,A2:2,A3:3,B1:4 \
271 A1:P0,A2:P1,A3:P2,B1:P1 2-4,3"
272 " C0-4:S+ C1-4:S+ C2-4 . X2-4 X2-4:P2 X4:P1 . 0 A1:0-1,A2:2-3,A3:4 \
273 A1:P0,A2:P2,A3:P1 2-4,2-3"
274 " C0-4:S+ C1-4:S+ C2-4 . X2-4 X2-4:P2 X3-4:P1 . 0 A1:0-1,A2:2,A3:3-4 \
275 A1:P0,A2:P2,A3:P1 2"
276 " C0-4:X2-4:S+ C1-4:X2-4:S+:P2 C2-4:X4:P1 \
277 . . X5 . . 0 A1:0-4,A2:1-4,A3:2-4 \
278 A1:P0,A2:P-2,A3:P-1"
279 " C0-4:X2-4:S+ C1-4:X2-4:S+:P2 C2-4:X4:P1 \
280 . . . X1 . 0 A1:0-1,A2:2-4,A3:2-4 \
281 A1:P0,A2:P2,A3:P-1 2-4"
282
283 # Remote partition offline tests
284 " C0-3:S+ C1-3:S+ C2-3 . X2-3 X2-3 X2-3:P2:O2=0 . 0 A1:0-1,A2:1,A3:3 A1:P0,A3:P2 2-3"
285 " C0-3:S+ C1-3:S+ C2-3 . X2-3 X2-3 X2-3:P2:O2=0 O2=1 0 A1:0-1,A2:1,A3:2-3 A1:P0,A3:P2 2-3"
286 " C0-3:S+ C1-3:S+ C3 . X2-3 X2-3 P2:O3=0 . 0 A1:0-2,A2:1-2,A3: A1:P0,A3:P2 3"
287 " C0-3:S+ C1-3:S+ C3 . X2-3 X2-3 T:P2:O3=0 . 0 A1:0-2,A2:1-2,A3:1-2 A1:P0,A3:P-2 3,"
288
289 # An invalidated remote partition cannot self-recover from hotplug
290 " C0-3:S+ C1-3:S+ C2 . X2-3 X2-3 T:P2:O2=0 O2=1 0 A1:0-3,A2:1-3,A3:2 A1:P0,A3:P-2"
291
292 # cpus.exclusive.effective clearing test
293 " C0-3:S+ C1-3:S+ C2 . X2-3:X . . . 0 A1:0-3,A2:1-3,A3:2,XA1:"
294
295 # Invalid to valid remote partition transition test
296 " C0-3:S+ C1-3 . . . X3:P2 . . 0 A1:0-3,A2:1-3,XA2: A2:P-2"
297 " C0-3:S+ C1-3:X3:P2
298 . . X2-3 P2 . . 0 A1:0-2,A2:3,XA2:3 A2:P2 3"
299
300 # Invalid to valid local partition direct transition tests
301 " C1-3:S+:P2 X4:P2 . . . . . . 0 A1:1-3,XA1:1-3,A2:1-3:XA2: A1:P2,A2:P-2 1-3"
302 " C1-3:S+:P2 X4:P2 . . . X3:P2 . . 0 A1:1-2,XA1:1-3,A2:3:XA2:3 A1:P2,A2:P2 1-3"
303 " C0-3:P2 . . C4-6 C0-4 . . . 0 A1:0-4,B1:4-6 A1:P-2,B1:P0"
304 " C0-3:P2 . . C4-6 C0-4:C0-3 . . . 0 A1:0-3,B1:4-6 A1:P2,B1:P0 0-3"
305 " C0-3:P2 . . C3-5:C4-5 . . . . 0 A1:0-3,B1:4-5 A1:P2,B1:P0 0-3"
306
307 # Local partition invalidation tests
308 " C0-3:X1-3:S+:P2 C1-3:X2-3:S+:P2 C2-3:X3:P2 \
309 . . . . . 0 A1:1,A2:2,A3:3 A1:P2,A2:P2,A3:P2 1-3"
310 " C0-3:X1-3:S+:P2 C1-3:X2-3:S+:P2 C2-3:X3:P2 \
311 . . X4 . . 0 A1:1-3,A2:1-3,A3:2-3,XA2:,XA3: A1:P2,A2:P-2,A3:P-2 1-3"
312 " C0-3:X1-3:S+:P2 C1-3:X2-3:S+:P2 C2-3:X3:P2 \
313 . . C4:X . . 0 A1:1-3,A2:1-3,A3:2-3,XA2:,XA3: A1:P2,A2:P-2,A3:P-2 1-3"
314 # Local partition CPU change tests
315 " C0-5:S+:P2 C4-5:S+:P1 . . . C3-5 . . 0 A1:0-2,A2:3-5 A1:P2,A2:P1 0-2"
316 " C0-5:S+:P2 C4-5:S+:P1 . . C1-5 . . . 0 A1:1-3,A2:4-5 A1:P2,A2:P1 1-3"
317
318 # cpus_allowed/exclusive_cpus update tests
319 " C0-3:X2-3:S+ C1-3:X2-3:S+ C2-3:X2-3 \
320 . X:C4 . P2 . 0 A1:4,A2:4,XA2:,XA3:,A3:4 \
321 A1:P0,A3:P-2"
322 " C0-3:X2-3:S+ C1-3:X2-3:S+ C2-3:X2-3 \
323 . X1 . P2 . 0 A1:0-3,A2:1-3,XA1:1,XA2:,XA3:,A3:2-3 \
324 A1:P0,A3:P-2"
325 " C0-3:X2-3:S+ C1-3:X2-3:S+ C2-3:X2-3 \
326 . . X3 P2 . 0 A1:0-2,A2:1-2,XA2:3,XA3:3,A3:3 \
327 A1:P0,A3:P2 3"
328 " C0-3:X2-3:S+ C1-3:X2-3:S+ C2-3:X2-3:P2 \
329 . . X3 . . 0 A1:0-3,A2:1-3,XA2:3,XA3:3,A3:2-3 \
330 A1:P0,A3:P-2"
331 " C0-3:X2-3:S+ C1-3:X2-3:S+ C2-3:X2-3:P2 \
332 . X4 . . . 0 A1:0-3,A2:1-3,A3:2-3,XA1:4,XA2:,XA3 \
333 A1:P0,A3:P-2"
334
335 # old-A1 old-A2 old-A3 old-B1 new-A1 new-A2 new-A3 new-B1 fail ECPUs Pstate ISOLCPUS
336 # ------ ------ ------ ------ ------ ------ ------ ------ ---- ----- ------ --------
337 #
338 # Incorrect change to cpuset.cpus[.exclusive] invalidates partition root
339 #
340 # Adding CPUs to partition root that are not in parent's
341 # cpuset.cpus is allowed, but those extra CPUs are ignored.
342 "C2-3:P1:S+ C3:P1 . . . C2-4 . . 0 A1:,A2:2-3 A1:P1,A2:P1"
343
344 # Taking away all CPUs from parent or itself if there are tasks
345 # will make the partition invalid.
346 "C2-3:P1:S+ C3:P1 . . T C2-3 . . 0 A1:2-3,A2:2-3 A1:P1,A2:P-1"
347 " C3:P1:S+ C3 . . T P1 . . 0 A1:3,A2:3 A1:P1,A2:P-1"
348 "$SETUP_A123_PARTITIONS . T:C2-3 . . . 0 A1:2-3,A2:2-3,A3:3 A1:P1,A2:P-1,A3:P-1"
349 "$SETUP_A123_PARTITIONS . T:C2-3:C1-3 . . . 0 A1:1,A2:2,A3:3 A1:P1,A2:P1,A3:P1"
350
351 # Changing a partition root to member makes child partitions invalid
352 "C2-3:P1:S+ C3:P1 . . P0 . . . 0 A1:2-3,A2:3 A1:P0,A2:P-1"
353 "$SETUP_A123_PARTITIONS . C2-3 P0 . . 0 A1:2-3,A2:2-3,A3:3 A1:P1,A2:P0,A3:P-1"
354
355 # cpuset.cpus can contains cpus not in parent's cpuset.cpus as long
356 # as they overlap.
357 "C2-3:P1:S+ . . . . C3-4:P1 . . 0 A1:2,A2:3 A1:P1,A2:P1"
358
359 # Deletion of CPUs distributed to child cgroup is allowed.
360 "C0-1:P1:S+ C1 . C2-3 C4-5 . . . 0 A1:4-5,A2:4-5"
361
362 # To become a valid partition root, cpuset.cpus must overlap parent's
363 # cpuset.cpus.
364 " C0-1:P1 . . C2-3 S+ C4-5:P1 . . 0 A1:0-1,A2:0-1 A1:P1,A2:P-1"
365
366 # Enabling partition with child cpusets is allowed
367 " C0-1:S+ C1 . C2-3 P1 . . . 0 A1:0-1,A2:1 A1:P1"
368
369 # A partition root with non-partition root parent is invalid, but it
370 # can be made valid if its parent becomes a partition root too.
371 " C0-1:S+ C1 . C2-3 . P2 . . 0 A1:0-1,A2:1 A1:P0,A2:P-2"
372 " C0-1:S+ C1:P2 . C2-3 P1 . . . 0 A1:0,A2:1 A1:P1,A2:P2"
373
374 # A non-exclusive cpuset.cpus change will invalidate partition and its siblings
375 " C0-1:P1 . . C2-3 C0-2 . . . 0 A1:0-2,B1:2-3 A1:P-1,B1:P0"
376 " C0-1:P1 . . P1:C2-3 C0-2 . . . 0 A1:0-2,B1:2-3 A1:P-1,B1:P-1"
377 " C0-1 . . P1:C2-3 C0-2 . . . 0 A1:0-2,B1:2-3 A1:P0,B1:P-1"
378
379 # cpuset.cpus can overlap with sibling cpuset.cpus.exclusive but not subsumed by it
380 " C0-3 . . C4-5 X5 . . . 0 A1:0-3,B1:4-5"
381
382 # Child partition root that try to take all CPUs from parent partition
383 # with tasks will remain invalid.
384 " C1-4:P1:S+ P1 . . . . . . 0 A1:1-4,A2:1-4 A1:P1,A2:P-1"
385 " C1-4:P1:S+ P1 . . . C1-4 . . 0 A1,A2:1-4 A1:P1,A2:P1"
386 " C1-4:P1:S+ P1 . . T C1-4 . . 0 A1:1-4,A2:1-4 A1:P1,A2:P-1"
387
388 # Clearing of cpuset.cpus with a preset cpuset.cpus.exclusive shouldn't
389 # affect cpuset.cpus.exclusive.effective.
390 " C1-4:X3:S+ C1:X3 . . . C . . 0 A2:1-4,XA2:3"
391
392 # old-A1 old-A2 old-A3 old-B1 new-A1 new-A2 new-A3 new-B1 fail ECPUs Pstate ISOLCPUS
393 # ------ ------ ------ ------ ------ ------ ------ ------ ---- ----- ------ --------
394 # Failure cases:
395
396 # A task cannot be added to a partition with no cpu
397 "C2-3:P1:S+ C3:P1 . . O2=0:T . . . 1 A1:,A2:3 A1:P1,A2:P1"
398
399 # Changes to cpuset.cpus.exclusive that violate exclusivity rule is rejected
400 " C0-3 . . C4-5 X0-3 . . X3-5 1 A1:0-3,B1:4-5"
401
402 # cpuset.cpus cannot be a subset of sibling cpuset.cpus.exclusive
403 " C0-3 . . C4-5 X3-5 . . . 1 A1:0-3,B1:4-5"
404 )
405
406 #
407 # Write to the cpu online file
408 # $1 - <c>=<v> where <c> = cpu number, <v> value to be written
409 #
410 write_cpu_online()
411 {
412 CPU=${1%=*}
413 VAL=${1#*=}
414 CPUFILE=//sys/devices/system/cpu/cpu${CPU}/online
415 if [[ $VAL -eq 0 ]]
416 then
417 OFFLINE_CPUS="$OFFLINE_CPUS $CPU"
418 else
419 [[ -n "$OFFLINE_CPUS" ]] && {
420 OFFLINE_CPUS=$(echo $CPU $CPU $OFFLINE_CPUS | fmt -1 |\
421 sort | uniq -u)
422 }
423 fi
424 echo $VAL > $CPUFILE
425 pause 0.05
426 }
427
428 #
429 # Set controller state
430 # $1 - cgroup directory
431 # $2 - state
432 # $3 - showerr
433 #
434 # The presence of ":" in state means transition from one to the next.
435 #
436 set_ctrl_state()
437 {
438 TMPMSG=/tmp/.msg_$$
439 CGRP=$1
440 STATE=$2
441 SHOWERR=${3}
442 CTRL=${CTRL:=$CONTROLLER}
443 HASERR=0
444 REDIRECT="2> $TMPMSG"
445 [[ -z "$STATE" || "$STATE" = '.' ]] && return 0
446 [[ $VERBOSE -gt 0 ]] && SHOWERR=1
447
448 rm -f $TMPMSG
449 for CMD in $(echo $STATE | sed -e "s/:/ /g")
450 do
451 TFILE=$CGRP/cgroup.procs
452 SFILE=$CGRP/cgroup.subtree_control
453 PFILE=$CGRP/cpuset.cpus.partition
454 CFILE=$CGRP/cpuset.cpus
455 XFILE=$CGRP/cpuset.cpus.exclusive
456 S=$(expr substr $CMD 1 1)
457 if [[ $S = S ]]
458 then
459 PREFIX=${CMD#?}
460 COMM="echo ${PREFIX}${CTRL} > $SFILE"
461 eval $COMM $REDIRECT
462 elif [[ $S = X ]]
463 then
464 CPUS=${CMD#?}
465 COMM="echo $CPUS > $XFILE"
466 eval $COMM $REDIRECT
467 elif [[ $S = C ]]
468 then
469 CPUS=${CMD#?}
470 COMM="echo $CPUS > $CFILE"
471 eval $COMM $REDIRECT
472 elif [[ $S = P ]]
473 then
474 VAL=${CMD#?}
475 case $VAL in
476 0) VAL=member
477 ;;
478 1) VAL=root
479 ;;
480 2) VAL=isolated
481 ;;
482 *)
483 echo "Invalid partition state - $VAL"
484 exit 1
485 ;;
486 esac
487 COMM="echo $VAL > $PFILE"
488 eval $COMM $REDIRECT
489 elif [[ $S = O ]]
490 then
491 VAL=${CMD#?}
492 write_cpu_online $VAL
493 elif [[ $S = T ]]
494 then
495 COMM="echo 0 > $TFILE"
496 eval $COMM $REDIRECT
497 fi
498 RET=$?
499 [[ $RET -ne 0 ]] && {
500 [[ -n "$SHOWERR" ]] && {
501 echo "$COMM"
502 cat $TMPMSG
503 }
504 HASERR=1
505 }
506 pause 0.01
507 rm -f $TMPMSG
508 done
509 return $HASERR
510 }
511
512 set_ctrl_state_noerr()
513 {
514 CGRP=$1
515 STATE=$2
516 [[ -d $CGRP ]] || mkdir $CGRP
517 set_ctrl_state $CGRP $STATE 1
518 [[ $? -ne 0 ]] && {
519 echo "ERROR: Failed to set $2 to cgroup $1!"
520 exit 1
521 }
522 }
523
524 online_cpus()
525 {
526 [[ -n "OFFLINE_CPUS" ]] && {
527 for C in $OFFLINE_CPUS
528 do
529 write_cpu_online ${C}=1
530 done
531 }
532 }
533
534 #
535 # Return 1 if the list of effective cpus isn't the same as the initial list.
536 #
537 reset_cgroup_states()
538 {
539 echo 0 > $CGROUP2/cgroup.procs
540 online_cpus
541 rmdir A1/A2/A3 A1/A2 A1 B1 > /dev/null 2>&1
542 pause 0.02
543 set_ctrl_state . R-
544 pause 0.01
545 }
546
547 dump_states()
548 {
549 for DIR in . A1 A1/A2 A1/A2/A3 B1
550 do
551 CPUS=$DIR/cpuset.cpus
552 ECPUS=$DIR/cpuset.cpus.effective
553 XCPUS=$DIR/cpuset.cpus.exclusive
554 XECPUS=$DIR/cpuset.cpus.exclusive.effective
555 PRS=$DIR/cpuset.cpus.partition
556 PCPUS=$DIR/.__DEBUG__.cpuset.cpus.subpartitions
557 ISCPUS=$DIR/cpuset.cpus.isolated
558 [[ -e $CPUS ]] && echo "$CPUS: $(cat $CPUS)"
559 [[ -e $XCPUS ]] && echo "$XCPUS: $(cat $XCPUS)"
560 [[ -e $ECPUS ]] && echo "$ECPUS: $(cat $ECPUS)"
561 [[ -e $XECPUS ]] && echo "$XECPUS: $(cat $XECPUS)"
562 [[ -e $PRS ]] && echo "$PRS: $(cat $PRS)"
563 [[ -e $PCPUS ]] && echo "$PCPUS: $(cat $PCPUS)"
564 [[ -e $ISCPUS ]] && echo "$ISCPUS: $(cat $ISCPUS)"
565 done
566 }
567
568 #
569 # Check effective cpus
570 # $1 - check string, format: <cgroup>:<cpu-list>[,<cgroup>:<cpu-list>]*
571 #
572 check_effective_cpus()
573 {
574 CHK_STR=$1
575 for CHK in $(echo $CHK_STR | sed -e "s/,/ /g")
576 do
577 set -- $(echo $CHK | sed -e "s/:/ /g")
578 CGRP=$1
579 CPUS=$2
580 if [[ $CGRP = X* ]]
581 then
582 CGRP=${CGRP#X}
583 FILE=cpuset.cpus.exclusive.effective
584 else
585 FILE=cpuset.cpus.effective
586 fi
587 [[ $CGRP = A2 ]] && CGRP=A1/A2
588 [[ $CGRP = A3 ]] && CGRP=A1/A2/A3
589 [[ -e $CGRP/$FILE ]] || return 1
590 [[ $CPUS = $(cat $CGRP/$FILE) ]] || return 1
591 done
592 }
593
594 #
595 # Check cgroup states
596 # $1 - check string, format: <cgroup>:<state>[,<cgroup>:<state>]*
597 #
598 check_cgroup_states()
599 {
600 CHK_STR=$1
601 for CHK in $(echo $CHK_STR | sed -e "s/,/ /g")
602 do
603 set -- $(echo $CHK | sed -e "s/:/ /g")
604 CGRP=$1
605 CGRP_DIR=$CGRP
606 STATE=$2
607 FILE=
608 EVAL=$(expr substr $STATE 2 2)
609 [[ $CGRP = A2 ]] && CGRP_DIR=A1/A2
610 [[ $CGRP = A3 ]] && CGRP_DIR=A1/A2/A3
611
612 case $STATE in
613 P*) FILE=$CGRP_DIR/cpuset.cpus.partition
614 ;;
615 *) echo "Unknown state: $STATE!"
616 exit 1
617 ;;
618 esac
619 VAL=$(cat $FILE)
620
621 case "$VAL" in
622 member) VAL=0
623 ;;
624 root) VAL=1
625 ;;
626 isolated)
627 VAL=2
628 ;;
629 "root invalid"*)
630 VAL=-1
631 ;;
632 "isolated invalid"*)
633 VAL=-2
634 ;;
635 esac
636 [[ $EVAL != $VAL ]] && return 1
637
638 #
639 # For root partition, dump sched-domains info to console if
640 # verbose mode set for manual comparison with sched debug info.
641 #
642 [[ $VAL -eq 1 && $VERBOSE -gt 0 ]] && {
643 DOMS=$(cat $CGRP_DIR/cpuset.cpus.effective)
644 [[ -n "$DOMS" ]] &&
645 echo " [$CGRP] sched-domain: $DOMS" > $CONSOLE
646 }
647 done
648 return 0
649 }
650
651 #
652 # Get isolated (including offline) CPUs by looking at
653 # /sys/kernel/debug/sched/domains and cpuset.cpus.isolated control file,
654 # if available, and compare that with the expected value.
655 #
656 # Note that isolated CPUs from the sched/domains context include offline
657 # CPUs as well as CPUs in non-isolated 1-CPU partition. Those CPUs may
658 # not be included in the cpuset.cpus.isolated control file which contains
659 # only CPUs in isolated partitions as well as those that are isolated at
660 # boot time.
661 #
662 # $1 - expected isolated cpu list(s) <isolcpus1>{,<isolcpus2>}
663 # <isolcpus1> - expected sched/domains value
664 # <isolcpus2> - cpuset.cpus.isolated value = <isolcpus1> if not defined
665 #
666 check_isolcpus()
667 {
668 EXPECT_VAL=$1
669 ISOLCPUS=
670 LASTISOLCPU=
671 SCHED_DOMAINS=/sys/kernel/debug/sched/domains
672 ISCPUS=${CGROUP2}/cpuset.cpus.isolated
673 if [[ $EXPECT_VAL = . ]]
674 then
675 EXPECT_VAL=
676 EXPECT_VAL2=
677 elif [[ $(expr $EXPECT_VAL : ".*,.*") > 0 ]]
678 then
679 set -- $(echo $EXPECT_VAL | sed -e "s/,/ /g")
680 EXPECT_VAL=$1
681 EXPECT_VAL2=$2
682 else
683 EXPECT_VAL2=$EXPECT_VAL
684 fi
685
686 #
687 # Check cpuset.cpus.isolated cpumask
688 #
689 if [[ -z "$BOOT_ISOLCPUS" ]]
690 then
691 ISOLCPUS=$(cat $ISCPUS)
692 else
693 ISOLCPUS=$(cat $ISCPUS | sed -e "s/,*$BOOT_ISOLCPUS//")
694 fi
695 [[ "$EXPECT_VAL2" != "$ISOLCPUS" ]] && {
696 # Take a 50ms pause and try again
697 pause 0.05
698 ISOLCPUS=$(cat $ISCPUS)
699 }
700 [[ "$EXPECT_VAL2" != "$ISOLCPUS" ]] && return 1
701 ISOLCPUS=
702
703 #
704 # Use the sched domain in debugfs to check isolated CPUs, if available
705 #
706 [[ -d $SCHED_DOMAINS ]] || return 0
707
708 for ((CPU=0; CPU < $NR_CPUS; CPU++))
709 do
710 [[ -n "$(ls ${SCHED_DOMAINS}/cpu$CPU)" ]] && continue
711
712 if [[ -z "$LASTISOLCPU" ]]
713 then
714 ISOLCPUS=$CPU
715 LASTISOLCPU=$CPU
716 elif [[ "$LASTISOLCPU" -eq $((CPU - 1)) ]]
717 then
718 echo $ISOLCPUS | grep -q "\<$LASTISOLCPU\$"
719 if [[ $? -eq 0 ]]
720 then
721 ISOLCPUS=${ISOLCPUS}-
722 fi
723 LASTISOLCPU=$CPU
724 else
725 if [[ $ISOLCPUS = *- ]]
726 then
727 ISOLCPUS=${ISOLCPUS}$LASTISOLCPU
728 fi
729 ISOLCPUS=${ISOLCPUS},$CPU
730 LASTISOLCPU=$CPU
731 fi
732 done
733 [[ "$ISOLCPUS" = *- ]] && ISOLCPUS=${ISOLCPUS}$LASTISOLCPU
734 [[ -n "BOOT_ISOLCPUS" ]] &&
735 ISOLCPUS=$(echo $ISOLCPUS | sed -e "s/,*$BOOT_ISOLCPUS//")
736
737 [[ "$EXPECT_VAL" = "$ISOLCPUS" ]]
738 }
739
740 test_fail()
741 {
742 TESTNUM=$1
743 TESTTYPE=$2
744 ADDINFO=$3
745 echo "Test $TEST[$TESTNUM] failed $TESTTYPE check!"
746 [[ -n "$ADDINFO" ]] && echo "*** $ADDINFO ***"
747 eval echo \${$TEST[$I]}
748 echo
749 dump_states
750 exit 1
751 }
752
753 #
754 # Check to see if there are unexpected isolated CPUs left beyond the boot
755 # time isolated ones.
756 #
757 null_isolcpus_check()
758 {
759 [[ $VERBOSE -gt 0 ]] || return 0
760 # Retry a few times before printing error
761 RETRY=0
762 while [[ $RETRY -lt 8 ]]
763 do
764 pause 0.02
765 check_isolcpus "."
766 [[ $? -eq 0 ]] && return 0
767 ((RETRY++))
768 done
769 echo "Unexpected isolated CPUs: $ISOLCPUS"
770 dump_states
771 exit 1
772 }
773
774 #
775 # Run cpuset state transition test
776 # $1 - test matrix name
777 #
778 # This test is somewhat fragile as delays (sleep x) are added in various
779 # places to make sure state changes are fully propagated before the next
780 # action. These delays may need to be adjusted if running in a slower machine.
781 #
782 run_state_test()
783 {
784 TEST=$1
785 CONTROLLER=cpuset
786 I=0
787 eval CNT="\${#$TEST[@]}"
788
789 reset_cgroup_states
790 console_msg "Running state transition test ..."
791
792 while [[ $I -lt $CNT ]]
793 do
794 echo "Running test $I ..." > $CONSOLE
795 [[ $VERBOSE -gt 1 ]] && {
796 echo ""
797 eval echo \${$TEST[$I]}
798 }
799 eval set -- "\${$TEST[$I]}"
800 OLD_A1=$1
801 OLD_A2=$2
802 OLD_A3=$3
803 OLD_B1=$4
804 NEW_A1=$5
805 NEW_A2=$6
806 NEW_A3=$7
807 NEW_B1=$8
808 RESULT=$9
809 ECPUS=${10}
810 STATES=${11}
811 ICPUS=${12}
812
813 set_ctrl_state_noerr B1 $OLD_B1
814 set_ctrl_state_noerr A1 $OLD_A1
815 set_ctrl_state_noerr A1/A2 $OLD_A2
816 set_ctrl_state_noerr A1/A2/A3 $OLD_A3
817 RETVAL=0
818 set_ctrl_state A1 $NEW_A1; ((RETVAL += $?))
819 set_ctrl_state A1/A2 $NEW_A2; ((RETVAL += $?))
820 set_ctrl_state A1/A2/A3 $NEW_A3; ((RETVAL += $?))
821 set_ctrl_state B1 $NEW_B1; ((RETVAL += $?))
822
823 [[ $RETVAL -ne $RESULT ]] && test_fail $I result
824
825 [[ -n "$ECPUS" && "$ECPUS" != . ]] && {
826 check_effective_cpus $ECPUS
827 [[ $? -ne 0 ]] && test_fail $I "effective CPU"
828 }
829
830 [[ -n "$STATES" && "$STATES" != . ]] && {
831 check_cgroup_states $STATES
832 [[ $? -ne 0 ]] && test_fail $I states
833 }
834
835 # Compare the expected isolated CPUs with the actual ones,
836 # if available
837 [[ -n "$ICPUS" ]] && {
838 check_isolcpus $ICPUS
839 [[ $? -ne 0 ]] && test_fail $I "isolated CPU" \
840 "Expect $ICPUS, get $ISOLCPUS instead"
841 }
842 reset_cgroup_states
843 #
844 # Check to see if effective cpu list changes
845 #
846 NEWLIST=$(cat cpuset.cpus.effective)
847 RETRY=0
848 while [[ $NEWLIST != $CPULIST && $RETRY -lt 8 ]]
849 do
850 # Wait a bit longer & recheck a few times
851 pause 0.02
852 ((RETRY++))
853 NEWLIST=$(cat cpuset.cpus.effective)
854 done
855 [[ $NEWLIST != $CPULIST ]] && {
856 echo "Effective cpus changed to $NEWLIST after test $I!"
857 exit 1
858 }
859 null_isolcpus_check
860 [[ $VERBOSE -gt 0 ]] && echo "Test $I done."
861 ((I++))
862 done
863 echo "All $I tests of $TEST PASSED."
864 }
865
866 #
867 # Testing the new "isolated" partition root type
868 #
869 test_isolated()
870 {
871 cd $CGROUP2/test
872 echo 2-3 > cpuset.cpus
873 TYPE=$(cat cpuset.cpus.partition)
874 [[ $TYPE = member ]] || echo member > cpuset.cpus.partition
875
876 console_msg "Change from member to root"
877 test_partition root
878
879 console_msg "Change from root to isolated"
880 test_partition isolated
881
882 console_msg "Change from isolated to member"
883 test_partition member
884
885 console_msg "Change from member to isolated"
886 test_partition isolated
887
888 console_msg "Change from isolated to root"
889 test_partition root
890
891 console_msg "Change from root to member"
892 test_partition member
893
894 #
895 # Testing partition root with no cpu
896 #
897 console_msg "Distribute all cpus to child partition"
898 echo +cpuset > cgroup.subtree_control
899 test_partition root
900
901 mkdir A1
902 cd A1
903 echo 2-3 > cpuset.cpus
904 test_partition root
905 test_effective_cpus 2-3
906 cd ..
907 test_effective_cpus ""
908
909 console_msg "Moving task to partition test"
910 test_add_proc "No space left"
911 cd A1
912 test_add_proc ""
913 cd ..
914
915 console_msg "Shrink and expand child partition"
916 cd A1
917 echo 2 > cpuset.cpus
918 cd ..
919 test_effective_cpus 3
920 cd A1
921 echo 2-3 > cpuset.cpus
922 cd ..
923 test_effective_cpus ""
924
925 # Cleaning up
926 console_msg "Cleaning up"
927 echo $$ > $CGROUP2/cgroup.procs
928 [[ -d A1 ]] && rmdir A1
929 null_isolcpus_check
930 }
931
932 #
933 # Wait for inotify event for the given file and read it
934 # $1: cgroup file to wait for
935 # $2: file to store the read result
936 #
937 wait_inotify()
938 {
939 CGROUP_FILE=$1
940 OUTPUT_FILE=$2
941
942 $WAIT_INOTIFY $CGROUP_FILE
943 cat $CGROUP_FILE > $OUTPUT_FILE
944 }
945
946 #
947 # Test if inotify events are properly generated when going into and out of
948 # invalid partition state.
949 #
950 test_inotify()
951 {
952 ERR=0
953 PRS=/tmp/.prs_$$
954 cd $CGROUP2/test
955 [[ -f $WAIT_INOTIFY ]] || {
956 echo "wait_inotify not found, inotify test SKIPPED."
957 return
958 }
959
960 pause 0.01
961 echo 1 > cpuset.cpus
962 echo 0 > cgroup.procs
963 echo root > cpuset.cpus.partition
964 pause 0.01
965 rm -f $PRS
966 wait_inotify $PWD/cpuset.cpus.partition $PRS &
967 pause 0.01
968 set_ctrl_state . "O1=0"
969 pause 0.01
970 check_cgroup_states ".:P-1"
971 if [[ $? -ne 0 ]]
972 then
973 echo "FAILED: Inotify test - partition not invalid"
974 ERR=1
975 elif [[ ! -f $PRS ]]
976 then
977 echo "FAILED: Inotify test - event not generated"
978 ERR=1
979 kill %1
980 elif [[ $(cat $PRS) != "root invalid"* ]]
981 then
982 echo "FAILED: Inotify test - incorrect state"
983 cat $PRS
984 ERR=1
985 fi
986 online_cpus
987 echo member > cpuset.cpus.partition
988 echo 0 > ../cgroup.procs
989 if [[ $ERR -ne 0 ]]
990 then
991 exit 1
992 else
993 echo "Inotify test PASSED"
994 fi
995 }
996
997 trap cleanup 0 2 3 6
998 run_state_test TEST_MATRIX
999 test_isolated
1000 test_inotify
1001 echo "All tests PASSED."
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.