1 // SPDX-License-Identifier: GPL-2.0-or-later <<
2 /* 1 /*
3 * kernel/stop_machine.c 2 * kernel/stop_machine.c
4 * 3 *
5 * Copyright (C) 2008, 2005 IBM Corporatio 4 * Copyright (C) 2008, 2005 IBM Corporation.
6 * Copyright (C) 2008, 2005 Rusty Russell 5 * Copyright (C) 2008, 2005 Rusty Russell rusty@rustcorp.com.au
7 * Copyright (C) 2010 SUSE Linux Pro 6 * Copyright (C) 2010 SUSE Linux Products GmbH
8 * Copyright (C) 2010 Tejun Heo <tj@ 7 * Copyright (C) 2010 Tejun Heo <tj@kernel.org>
>> 8 *
>> 9 * This file is released under the GPLv2 and any later version.
9 */ 10 */
10 #include <linux/compiler.h> <<
11 #include <linux/completion.h> 11 #include <linux/completion.h>
12 #include <linux/cpu.h> 12 #include <linux/cpu.h>
13 #include <linux/init.h> 13 #include <linux/init.h>
14 #include <linux/kthread.h> 14 #include <linux/kthread.h>
15 #include <linux/export.h> 15 #include <linux/export.h>
16 #include <linux/percpu.h> 16 #include <linux/percpu.h>
17 #include <linux/sched.h> 17 #include <linux/sched.h>
18 #include <linux/stop_machine.h> 18 #include <linux/stop_machine.h>
19 #include <linux/interrupt.h> 19 #include <linux/interrupt.h>
20 #include <linux/kallsyms.h> 20 #include <linux/kallsyms.h>
21 #include <linux/smpboot.h> 21 #include <linux/smpboot.h>
22 #include <linux/atomic.h> 22 #include <linux/atomic.h>
23 #include <linux/nmi.h> <<
24 #include <linux/sched/wake_q.h> <<
25 23
26 /* 24 /*
27 * Structure to determine completion condition 25 * Structure to determine completion condition and record errors. May
28 * be shared by works on different cpus. 26 * be shared by works on different cpus.
29 */ 27 */
30 struct cpu_stop_done { 28 struct cpu_stop_done {
31 atomic_t nr_todo; 29 atomic_t nr_todo; /* nr left to execute */
>> 30 bool executed; /* actually executed? */
32 int ret; 31 int ret; /* collected return value */
33 struct completion completion; 32 struct completion completion; /* fired if nr_todo reaches 0 */
34 }; 33 };
35 34
36 /* the actual stopper, one per every possible 35 /* the actual stopper, one per every possible cpu, enabled on online cpus */
37 struct cpu_stopper { 36 struct cpu_stopper {
38 struct task_struct *thread; !! 37 spinlock_t lock;
39 <<
40 raw_spinlock_t lock; <<
41 bool enabled; 38 bool enabled; /* is this stopper enabled? */
42 struct list_head works; 39 struct list_head works; /* list of pending works */
43 <<
44 struct cpu_stop_work stop_work; <<
45 unsigned long caller; <<
46 cpu_stop_fn_t fn; <<
47 }; 40 };
48 41
49 static DEFINE_PER_CPU(struct cpu_stopper, cpu_ 42 static DEFINE_PER_CPU(struct cpu_stopper, cpu_stopper);
>> 43 static DEFINE_PER_CPU(struct task_struct *, cpu_stopper_task);
50 static bool stop_machine_initialized = false; 44 static bool stop_machine_initialized = false;
51 45
52 void print_stop_info(const char *log_lvl, stru <<
53 { <<
54 /* <<
55 * If @task is a stopper task, it cann <<
56 * stable. <<
57 */ <<
58 struct cpu_stopper *stopper = per_cpu_ <<
59 <<
60 if (task != stopper->thread) <<
61 return; <<
62 <<
63 printk("%sStopper: %pS <- %pS\n", log_ <<
64 } <<
65 <<
66 /* static data for stop_cpus */ <<
67 static DEFINE_MUTEX(stop_cpus_mutex); <<
68 static bool stop_cpus_in_progress; <<
69 <<
70 static void cpu_stop_init_done(struct cpu_stop 46 static void cpu_stop_init_done(struct cpu_stop_done *done, unsigned int nr_todo)
71 { 47 {
72 memset(done, 0, sizeof(*done)); 48 memset(done, 0, sizeof(*done));
73 atomic_set(&done->nr_todo, nr_todo); 49 atomic_set(&done->nr_todo, nr_todo);
74 init_completion(&done->completion); 50 init_completion(&done->completion);
75 } 51 }
76 52
77 /* signal completion unless @done is NULL */ 53 /* signal completion unless @done is NULL */
78 static void cpu_stop_signal_done(struct cpu_st !! 54 static void cpu_stop_signal_done(struct cpu_stop_done *done, bool executed)
79 { <<
80 if (atomic_dec_and_test(&done->nr_todo <<
81 complete(&done->completion); <<
82 } <<
83 <<
84 static void __cpu_stop_queue_work(struct cpu_s <<
85 struct <<
86 struct <<
87 { 55 {
88 list_add_tail(&work->list, &stopper->w !! 56 if (done) {
89 wake_q_add(wakeq, stopper->thread); !! 57 if (executed)
>> 58 done->executed = true;
>> 59 if (atomic_dec_and_test(&done->nr_todo))
>> 60 complete(&done->completion);
>> 61 }
90 } 62 }
91 63
92 /* queue @work to @stopper. if offline, @work 64 /* queue @work to @stopper. if offline, @work is completed immediately */
93 static bool cpu_stop_queue_work(unsigned int c !! 65 static void cpu_stop_queue_work(unsigned int cpu, struct cpu_stop_work *work)
94 { 66 {
95 struct cpu_stopper *stopper = &per_cpu 67 struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
96 DEFINE_WAKE_Q(wakeq); !! 68 struct task_struct *p = per_cpu(cpu_stopper_task, cpu);
>> 69
97 unsigned long flags; 70 unsigned long flags;
98 bool enabled; <<
99 71
100 preempt_disable(); !! 72 spin_lock_irqsave(&stopper->lock, flags);
101 raw_spin_lock_irqsave(&stopper->lock, <<
102 enabled = stopper->enabled; <<
103 if (enabled) <<
104 __cpu_stop_queue_work(stopper, <<
105 else if (work->done) <<
106 cpu_stop_signal_done(work->don <<
107 raw_spin_unlock_irqrestore(&stopper->l <<
108 73
109 wake_up_q(&wakeq); !! 74 if (stopper->enabled) {
110 preempt_enable(); !! 75 list_add_tail(&work->list, &stopper->works);
>> 76 wake_up_process(p);
>> 77 } else
>> 78 cpu_stop_signal_done(work->done, false);
111 79
112 return enabled; !! 80 spin_unlock_irqrestore(&stopper->lock, flags);
113 } 81 }
114 82
115 /** 83 /**
116 * stop_one_cpu - stop a cpu 84 * stop_one_cpu - stop a cpu
117 * @cpu: cpu to stop 85 * @cpu: cpu to stop
118 * @fn: function to execute 86 * @fn: function to execute
119 * @arg: argument to @fn 87 * @arg: argument to @fn
120 * 88 *
121 * Execute @fn(@arg) on @cpu. @fn is run in a 89 * Execute @fn(@arg) on @cpu. @fn is run in a process context with
122 * the highest priority preempting any task on 90 * the highest priority preempting any task on the cpu and
123 * monopolizing it. This function returns aft 91 * monopolizing it. This function returns after the execution is
124 * complete. 92 * complete.
125 * 93 *
126 * This function doesn't guarantee @cpu stays 94 * This function doesn't guarantee @cpu stays online till @fn
127 * completes. If @cpu goes down in the middle 95 * completes. If @cpu goes down in the middle, execution may happen
128 * partially or fully on different cpus. @fn 96 * partially or fully on different cpus. @fn should either be ready
129 * for that or the caller should ensure that @ 97 * for that or the caller should ensure that @cpu stays online until
130 * this function completes. 98 * this function completes.
131 * 99 *
132 * CONTEXT: 100 * CONTEXT:
133 * Might sleep. 101 * Might sleep.
134 * 102 *
135 * RETURNS: 103 * RETURNS:
136 * -ENOENT if @fn(@arg) was not executed becau 104 * -ENOENT if @fn(@arg) was not executed because @cpu was offline;
137 * otherwise, the return value of @fn. 105 * otherwise, the return value of @fn.
138 */ 106 */
139 int stop_one_cpu(unsigned int cpu, cpu_stop_fn 107 int stop_one_cpu(unsigned int cpu, cpu_stop_fn_t fn, void *arg)
140 { 108 {
141 struct cpu_stop_done done; 109 struct cpu_stop_done done;
142 struct cpu_stop_work work = { .fn = fn !! 110 struct cpu_stop_work work = { .fn = fn, .arg = arg, .done = &done };
143 111
144 cpu_stop_init_done(&done, 1); 112 cpu_stop_init_done(&done, 1);
145 if (!cpu_stop_queue_work(cpu, &work)) !! 113 cpu_stop_queue_work(cpu, &work);
146 return -ENOENT; <<
147 /* <<
148 * In case @cpu == smp_proccessor_id() <<
149 * cycle by doing a preemption: <<
150 */ <<
151 cond_resched(); <<
152 wait_for_completion(&done.completion); 114 wait_for_completion(&done.completion);
153 return done.ret; !! 115 return done.executed ? done.ret : -ENOENT;
154 } <<
155 <<
156 /* This controls the threads on each CPU. */ <<
157 enum multi_stop_state { <<
158 /* Dummy starting state for thread. */ <<
159 MULTI_STOP_NONE, <<
160 /* Awaiting everyone to be scheduled. <<
161 MULTI_STOP_PREPARE, <<
162 /* Disable interrupts. */ <<
163 MULTI_STOP_DISABLE_IRQ, <<
164 /* Run the function */ <<
165 MULTI_STOP_RUN, <<
166 /* Exit */ <<
167 MULTI_STOP_EXIT, <<
168 }; <<
169 <<
170 struct multi_stop_data { <<
171 cpu_stop_fn_t fn; <<
172 void *data; <<
173 /* Like num_online_cpus(), but hotplug <<
174 unsigned int num_threads; <<
175 const struct cpumask *active_cpus; <<
176 <<
177 enum multi_stop_state state; <<
178 atomic_t thread_ack; <<
179 }; <<
180 <<
181 static void set_state(struct multi_stop_data * <<
182 enum multi_stop_state ne <<
183 { <<
184 /* Reset ack counter. */ <<
185 atomic_set(&msdata->thread_ack, msdata <<
186 smp_wmb(); <<
187 WRITE_ONCE(msdata->state, newstate); <<
188 } <<
189 <<
190 /* Last one to ack a state moves to the next s <<
191 static void ack_state(struct multi_stop_data * <<
192 { <<
193 if (atomic_dec_and_test(&msdata->threa <<
194 set_state(msdata, msdata->stat <<
195 } <<
196 <<
197 notrace void __weak stop_machine_yield(const s <<
198 { <<
199 cpu_relax(); <<
200 } <<
201 <<
202 /* This is the cpu_stop function which stops t <<
203 static int multi_cpu_stop(void *data) <<
204 { <<
205 struct multi_stop_data *msdata = data; <<
206 enum multi_stop_state newstate, cursta <<
207 int cpu = smp_processor_id(), err = 0; <<
208 const struct cpumask *cpumask; <<
209 unsigned long flags; <<
210 bool is_active; <<
211 <<
212 /* <<
213 * When called from stop_machine_from_ <<
214 * already be disabled. Save the stat <<
215 */ <<
216 local_save_flags(flags); <<
217 <<
218 if (!msdata->active_cpus) { <<
219 cpumask = cpu_online_mask; <<
220 is_active = cpu == cpumask_fir <<
221 } else { <<
222 cpumask = msdata->active_cpus; <<
223 is_active = cpumask_test_cpu(c <<
224 } <<
225 <<
226 /* Simple state machine */ <<
227 do { <<
228 /* Chill out and ensure we re- <<
229 stop_machine_yield(cpumask); <<
230 newstate = READ_ONCE(msdata->s <<
231 if (newstate != curstate) { <<
232 curstate = newstate; <<
233 switch (curstate) { <<
234 case MULTI_STOP_DISABL <<
235 local_irq_disa <<
236 hard_irq_disab <<
237 break; <<
238 case MULTI_STOP_RUN: <<
239 if (is_active) <<
240 err = <<
241 break; <<
242 default: <<
243 break; <<
244 } <<
245 ack_state(msdata); <<
246 } else if (curstate > MULTI_ST <<
247 /* <<
248 * At this stage all o <<
249 * in the same loop. A <<
250 * be detected and rep <<
251 */ <<
252 touch_nmi_watchdog(); <<
253 } <<
254 rcu_momentary_dyntick_idle(); <<
255 } while (curstate != MULTI_STOP_EXIT); <<
256 <<
257 local_irq_restore(flags); <<
258 return err; <<
259 } <<
260 <<
261 static int cpu_stop_queue_two_works(int cpu1, <<
262 int cpu2, <<
263 { <<
264 struct cpu_stopper *stopper1 = per_cpu <<
265 struct cpu_stopper *stopper2 = per_cpu <<
266 DEFINE_WAKE_Q(wakeq); <<
267 int err; <<
268 <<
269 retry: <<
270 /* <<
271 * The waking up of stopper threads ha <<
272 * scheduling context as the queueing. <<
273 * possibility of one of the above sto <<
274 * CPU, and preempting us. This will c <<
275 * stopper forever. <<
276 */ <<
277 preempt_disable(); <<
278 raw_spin_lock_irq(&stopper1->lock); <<
279 raw_spin_lock_nested(&stopper2->lock, <<
280 <<
281 if (!stopper1->enabled || !stopper2->e <<
282 err = -ENOENT; <<
283 goto unlock; <<
284 } <<
285 <<
286 /* <<
287 * Ensure that if we race with __stop_ <<
288 * queued up in reverse order leading <<
289 * <<
290 * We can't miss stop_cpus_in_progress <<
291 * queued a work on cpu1 but not on cp <<
292 * <<
293 * It can be falsely true but it is sa <<
294 * queue_stop_cpus_work() does everyth <<
295 */ <<
296 if (unlikely(stop_cpus_in_progress)) { <<
297 err = -EDEADLK; <<
298 goto unlock; <<
299 } <<
300 <<
301 err = 0; <<
302 __cpu_stop_queue_work(stopper1, work1, <<
303 __cpu_stop_queue_work(stopper2, work2, <<
304 <<
305 unlock: <<
306 raw_spin_unlock(&stopper2->lock); <<
307 raw_spin_unlock_irq(&stopper1->lock); <<
308 <<
309 if (unlikely(err == -EDEADLK)) { <<
310 preempt_enable(); <<
311 <<
312 while (stop_cpus_in_progress) <<
313 cpu_relax(); <<
314 <<
315 goto retry; <<
316 } <<
317 <<
318 wake_up_q(&wakeq); <<
319 preempt_enable(); <<
320 <<
321 return err; <<
322 } <<
323 /** <<
324 * stop_two_cpus - stops two cpus <<
325 * @cpu1: the cpu to stop <<
326 * @cpu2: the other cpu to stop <<
327 * @fn: function to execute <<
328 * @arg: argument to @fn <<
329 * <<
330 * Stops both the current and specified CPU an <<
331 * <<
332 * returns when both are completed. <<
333 */ <<
334 int stop_two_cpus(unsigned int cpu1, unsigned <<
335 { <<
336 struct cpu_stop_done done; <<
337 struct cpu_stop_work work1, work2; <<
338 struct multi_stop_data msdata; <<
339 <<
340 msdata = (struct multi_stop_data){ <<
341 .fn = fn, <<
342 .data = arg, <<
343 .num_threads = 2, <<
344 .active_cpus = cpumask_of(cpu1 <<
345 }; <<
346 <<
347 work1 = work2 = (struct cpu_stop_work) <<
348 .fn = multi_cpu_stop, <<
349 .arg = &msdata, <<
350 .done = &done, <<
351 .caller = _RET_IP_, <<
352 }; <<
353 <<
354 cpu_stop_init_done(&done, 2); <<
355 set_state(&msdata, MULTI_STOP_PREPARE) <<
356 <<
357 if (cpu1 > cpu2) <<
358 swap(cpu1, cpu2); <<
359 if (cpu_stop_queue_two_works(cpu1, &wo <<
360 return -ENOENT; <<
361 <<
362 wait_for_completion(&done.completion); <<
363 return done.ret; <<
364 } 116 }
365 117
366 /** 118 /**
367 * stop_one_cpu_nowait - stop a cpu but don't 119 * stop_one_cpu_nowait - stop a cpu but don't wait for completion
368 * @cpu: cpu to stop 120 * @cpu: cpu to stop
369 * @fn: function to execute 121 * @fn: function to execute
370 * @arg: argument to @fn 122 * @arg: argument to @fn
371 * @work_buf: pointer to cpu_stop_work structu <<
372 * 123 *
373 * Similar to stop_one_cpu() but doesn't wait 124 * Similar to stop_one_cpu() but doesn't wait for completion. The
374 * caller is responsible for ensuring @work_bu 125 * caller is responsible for ensuring @work_buf is currently unused
375 * and will remain untouched until stopper sta 126 * and will remain untouched until stopper starts executing @fn.
376 * 127 *
377 * CONTEXT: 128 * CONTEXT:
378 * Don't care. 129 * Don't care.
379 * <<
380 * RETURNS: <<
381 * true if cpu_stop_work was queued successful <<
382 * false otherwise. <<
383 */ 130 */
384 bool stop_one_cpu_nowait(unsigned int cpu, cpu !! 131 void stop_one_cpu_nowait(unsigned int cpu, cpu_stop_fn_t fn, void *arg,
385 struct cpu_stop_work * 132 struct cpu_stop_work *work_buf)
386 { 133 {
387 *work_buf = (struct cpu_stop_work){ .f !! 134 *work_buf = (struct cpu_stop_work){ .fn = fn, .arg = arg, };
388 return cpu_stop_queue_work(cpu, work_b !! 135 cpu_stop_queue_work(cpu, work_buf);
389 } 136 }
390 137
391 static bool queue_stop_cpus_work(const struct !! 138 /* static data for stop_cpus */
>> 139 static DEFINE_MUTEX(stop_cpus_mutex);
>> 140 static DEFINE_PER_CPU(struct cpu_stop_work, stop_cpus_work);
>> 141
>> 142 static void queue_stop_cpus_work(const struct cpumask *cpumask,
392 cpu_stop_fn_t 143 cpu_stop_fn_t fn, void *arg,
393 struct cpu_st 144 struct cpu_stop_done *done)
394 { 145 {
395 struct cpu_stop_work *work; 146 struct cpu_stop_work *work;
396 unsigned int cpu; 147 unsigned int cpu;
397 bool queued = false; !! 148
>> 149 /* initialize works and done */
>> 150 for_each_cpu(cpu, cpumask) {
>> 151 work = &per_cpu(stop_cpus_work, cpu);
>> 152 work->fn = fn;
>> 153 work->arg = arg;
>> 154 work->done = done;
>> 155 }
398 156
399 /* 157 /*
400 * Disable preemption while queueing t 158 * Disable preemption while queueing to avoid getting
401 * preempted by a stopper which might 159 * preempted by a stopper which might wait for other stoppers
402 * to enter @fn which can lead to dead 160 * to enter @fn which can lead to deadlock.
403 */ 161 */
404 preempt_disable(); 162 preempt_disable();
405 stop_cpus_in_progress = true; !! 163 for_each_cpu(cpu, cpumask)
406 barrier(); !! 164 cpu_stop_queue_work(cpu, &per_cpu(stop_cpus_work, cpu));
407 for_each_cpu(cpu, cpumask) { <<
408 work = &per_cpu(cpu_stopper.st <<
409 work->fn = fn; <<
410 work->arg = arg; <<
411 work->done = done; <<
412 work->caller = _RET_IP_; <<
413 if (cpu_stop_queue_work(cpu, w <<
414 queued = true; <<
415 } <<
416 barrier(); <<
417 stop_cpus_in_progress = false; <<
418 preempt_enable(); 165 preempt_enable();
419 <<
420 return queued; <<
421 } 166 }
422 167
423 static int __stop_cpus(const struct cpumask *c 168 static int __stop_cpus(const struct cpumask *cpumask,
424 cpu_stop_fn_t fn, void 169 cpu_stop_fn_t fn, void *arg)
425 { 170 {
426 struct cpu_stop_done done; 171 struct cpu_stop_done done;
427 172
428 cpu_stop_init_done(&done, cpumask_weig 173 cpu_stop_init_done(&done, cpumask_weight(cpumask));
429 if (!queue_stop_cpus_work(cpumask, fn, !! 174 queue_stop_cpus_work(cpumask, fn, arg, &done);
430 return -ENOENT; <<
431 wait_for_completion(&done.completion); 175 wait_for_completion(&done.completion);
432 return done.ret; !! 176 return done.executed ? done.ret : -ENOENT;
433 } 177 }
434 178
435 /** 179 /**
436 * stop_cpus - stop multiple cpus 180 * stop_cpus - stop multiple cpus
437 * @cpumask: cpus to stop 181 * @cpumask: cpus to stop
438 * @fn: function to execute 182 * @fn: function to execute
439 * @arg: argument to @fn 183 * @arg: argument to @fn
440 * 184 *
441 * Execute @fn(@arg) on online cpus in @cpumas 185 * Execute @fn(@arg) on online cpus in @cpumask. On each target cpu,
442 * @fn is run in a process context with the hi 186 * @fn is run in a process context with the highest priority
443 * preempting any task on the cpu and monopoli 187 * preempting any task on the cpu and monopolizing it. This function
444 * returns after all executions are complete. 188 * returns after all executions are complete.
445 * 189 *
446 * This function doesn't guarantee the cpus in 190 * This function doesn't guarantee the cpus in @cpumask stay online
447 * till @fn completes. If some cpus go down i 191 * till @fn completes. If some cpus go down in the middle, execution
448 * on the cpu may happen partially or fully on 192 * on the cpu may happen partially or fully on different cpus. @fn
449 * should either be ready for that or the call 193 * should either be ready for that or the caller should ensure that
450 * the cpus stay online until this function co 194 * the cpus stay online until this function completes.
451 * 195 *
452 * All stop_cpus() calls are serialized making 196 * All stop_cpus() calls are serialized making it safe for @fn to wait
453 * for all cpus to start executing it. 197 * for all cpus to start executing it.
454 * 198 *
455 * CONTEXT: 199 * CONTEXT:
456 * Might sleep. 200 * Might sleep.
457 * 201 *
458 * RETURNS: 202 * RETURNS:
459 * -ENOENT if @fn(@arg) was not executed at al 203 * -ENOENT if @fn(@arg) was not executed at all because all cpus in
460 * @cpumask were offline; otherwise, 0 if all 204 * @cpumask were offline; otherwise, 0 if all executions of @fn
461 * returned 0, any non zero return value if an 205 * returned 0, any non zero return value if any returned non zero.
462 */ 206 */
463 static int stop_cpus(const struct cpumask *cpu !! 207 int stop_cpus(const struct cpumask *cpumask, cpu_stop_fn_t fn, void *arg)
464 { 208 {
465 int ret; 209 int ret;
466 210
467 /* static works are used, process one 211 /* static works are used, process one request at a time */
468 mutex_lock(&stop_cpus_mutex); 212 mutex_lock(&stop_cpus_mutex);
469 ret = __stop_cpus(cpumask, fn, arg); 213 ret = __stop_cpus(cpumask, fn, arg);
470 mutex_unlock(&stop_cpus_mutex); 214 mutex_unlock(&stop_cpus_mutex);
471 return ret; 215 return ret;
472 } 216 }
473 217
>> 218 /**
>> 219 * try_stop_cpus - try to stop multiple cpus
>> 220 * @cpumask: cpus to stop
>> 221 * @fn: function to execute
>> 222 * @arg: argument to @fn
>> 223 *
>> 224 * Identical to stop_cpus() except that it fails with -EAGAIN if
>> 225 * someone else is already using the facility.
>> 226 *
>> 227 * CONTEXT:
>> 228 * Might sleep.
>> 229 *
>> 230 * RETURNS:
>> 231 * -EAGAIN if someone else is already stopping cpus, -ENOENT if
>> 232 * @fn(@arg) was not executed at all because all cpus in @cpumask were
>> 233 * offline; otherwise, 0 if all executions of @fn returned 0, any non
>> 234 * zero return value if any returned non zero.
>> 235 */
>> 236 int try_stop_cpus(const struct cpumask *cpumask, cpu_stop_fn_t fn, void *arg)
>> 237 {
>> 238 int ret;
>> 239
>> 240 /* static works are used, process one request at a time */
>> 241 if (!mutex_trylock(&stop_cpus_mutex))
>> 242 return -EAGAIN;
>> 243 ret = __stop_cpus(cpumask, fn, arg);
>> 244 mutex_unlock(&stop_cpus_mutex);
>> 245 return ret;
>> 246 }
>> 247
474 static int cpu_stop_should_run(unsigned int cp 248 static int cpu_stop_should_run(unsigned int cpu)
475 { 249 {
476 struct cpu_stopper *stopper = &per_cpu 250 struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
477 unsigned long flags; 251 unsigned long flags;
478 int run; 252 int run;
479 253
480 raw_spin_lock_irqsave(&stopper->lock, !! 254 spin_lock_irqsave(&stopper->lock, flags);
481 run = !list_empty(&stopper->works); 255 run = !list_empty(&stopper->works);
482 raw_spin_unlock_irqrestore(&stopper->l !! 256 spin_unlock_irqrestore(&stopper->lock, flags);
483 return run; 257 return run;
484 } 258 }
485 259
486 static void cpu_stopper_thread(unsigned int cp 260 static void cpu_stopper_thread(unsigned int cpu)
487 { 261 {
488 struct cpu_stopper *stopper = &per_cpu 262 struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
489 struct cpu_stop_work *work; 263 struct cpu_stop_work *work;
>> 264 int ret;
490 265
491 repeat: 266 repeat:
492 work = NULL; 267 work = NULL;
493 raw_spin_lock_irq(&stopper->lock); !! 268 spin_lock_irq(&stopper->lock);
494 if (!list_empty(&stopper->works)) { 269 if (!list_empty(&stopper->works)) {
495 work = list_first_entry(&stopp 270 work = list_first_entry(&stopper->works,
496 struct 271 struct cpu_stop_work, list);
497 list_del_init(&work->list); 272 list_del_init(&work->list);
498 } 273 }
499 raw_spin_unlock_irq(&stopper->lock); !! 274 spin_unlock_irq(&stopper->lock);
500 275
501 if (work) { 276 if (work) {
502 cpu_stop_fn_t fn = work->fn; 277 cpu_stop_fn_t fn = work->fn;
503 void *arg = work->arg; 278 void *arg = work->arg;
504 struct cpu_stop_done *done = w 279 struct cpu_stop_done *done = work->done;
505 int ret; !! 280 char ksym_buf[KSYM_NAME_LEN] __maybe_unused;
>> 281
>> 282 /* cpu stop callbacks are not allowed to sleep */
>> 283 preempt_disable();
506 284
507 /* cpu stop callbacks must not <<
508 stopper->caller = work->caller <<
509 stopper->fn = fn; <<
510 preempt_count_inc(); <<
511 ret = fn(arg); 285 ret = fn(arg);
512 if (done) { !! 286 if (ret)
513 if (ret) !! 287 done->ret = ret;
514 done->ret = re !! 288
515 cpu_stop_signal_done(d !! 289 /* restore preemption and check it's still balanced */
516 } !! 290 preempt_enable();
517 preempt_count_dec(); <<
518 stopper->fn = NULL; <<
519 stopper->caller = 0; <<
520 WARN_ONCE(preempt_count(), 291 WARN_ONCE(preempt_count(),
521 "cpu_stop: %ps(%p) l !! 292 "cpu_stop: %s(%p) leaked preempt count\n",
>> 293 kallsyms_lookup((unsigned long)fn, NULL, NULL, NULL,
>> 294 ksym_buf), arg);
>> 295
>> 296 cpu_stop_signal_done(done, true);
522 goto repeat; 297 goto repeat;
523 } 298 }
524 } 299 }
525 300
526 void stop_machine_park(int cpu) !! 301 extern void sched_set_stop_task(int cpu, struct task_struct *stop);
527 { <<
528 struct cpu_stopper *stopper = &per_cpu <<
529 /* <<
530 * Lockless. cpu_stopper_thread() will <<
531 * the pending works before it parks, <<
532 * the new works. <<
533 */ <<
534 stopper->enabled = false; <<
535 kthread_park(stopper->thread); <<
536 } <<
537 302
538 static void cpu_stop_create(unsigned int cpu) 303 static void cpu_stop_create(unsigned int cpu)
539 { 304 {
540 sched_set_stop_task(cpu, per_cpu(cpu_s !! 305 sched_set_stop_task(cpu, per_cpu(cpu_stopper_task, cpu));
541 } 306 }
542 307
543 static void cpu_stop_park(unsigned int cpu) 308 static void cpu_stop_park(unsigned int cpu)
544 { 309 {
545 struct cpu_stopper *stopper = &per_cpu 310 struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
>> 311 struct cpu_stop_work *work;
>> 312 unsigned long flags;
546 313
547 WARN_ON(!list_empty(&stopper->works)); !! 314 /* drain remaining works */
>> 315 spin_lock_irqsave(&stopper->lock, flags);
>> 316 list_for_each_entry(work, &stopper->works, list)
>> 317 cpu_stop_signal_done(work->done, false);
>> 318 stopper->enabled = false;
>> 319 spin_unlock_irqrestore(&stopper->lock, flags);
548 } 320 }
549 321
550 void stop_machine_unpark(int cpu) !! 322 static void cpu_stop_unpark(unsigned int cpu)
551 { 323 {
552 struct cpu_stopper *stopper = &per_cpu 324 struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
553 325
>> 326 spin_lock_irq(&stopper->lock);
554 stopper->enabled = true; 327 stopper->enabled = true;
555 kthread_unpark(stopper->thread); !! 328 spin_unlock_irq(&stopper->lock);
556 } 329 }
557 330
558 static struct smp_hotplug_thread cpu_stop_thre 331 static struct smp_hotplug_thread cpu_stop_threads = {
559 .store = &cpu_stopper !! 332 .store = &cpu_stopper_task,
560 .thread_should_run = cpu_stop_sho 333 .thread_should_run = cpu_stop_should_run,
561 .thread_fn = cpu_stopper_ 334 .thread_fn = cpu_stopper_thread,
562 .thread_comm = "migration/% 335 .thread_comm = "migration/%u",
563 .create = cpu_stop_cre 336 .create = cpu_stop_create,
>> 337 .setup = cpu_stop_unpark,
564 .park = cpu_stop_par 338 .park = cpu_stop_park,
>> 339 .pre_unpark = cpu_stop_unpark,
565 .selfparking = true, 340 .selfparking = true,
566 }; 341 };
567 342
568 static int __init cpu_stop_init(void) 343 static int __init cpu_stop_init(void)
569 { 344 {
570 unsigned int cpu; 345 unsigned int cpu;
571 346
572 for_each_possible_cpu(cpu) { 347 for_each_possible_cpu(cpu) {
573 struct cpu_stopper *stopper = 348 struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
574 349
575 raw_spin_lock_init(&stopper->l !! 350 spin_lock_init(&stopper->lock);
576 INIT_LIST_HEAD(&stopper->works 351 INIT_LIST_HEAD(&stopper->works);
577 } 352 }
578 353
579 BUG_ON(smpboot_register_percpu_thread( 354 BUG_ON(smpboot_register_percpu_thread(&cpu_stop_threads));
580 stop_machine_unpark(raw_smp_processor_ <<
581 stop_machine_initialized = true; 355 stop_machine_initialized = true;
582 return 0; 356 return 0;
583 } 357 }
584 early_initcall(cpu_stop_init); 358 early_initcall(cpu_stop_init);
585 359
586 int stop_machine_cpuslocked(cpu_stop_fn_t fn, !! 360 #ifdef CONFIG_STOP_MACHINE
587 const struct cpuma !! 361
>> 362 /* This controls the threads on each CPU. */
>> 363 enum stopmachine_state {
>> 364 /* Dummy starting state for thread. */
>> 365 STOPMACHINE_NONE,
>> 366 /* Awaiting everyone to be scheduled. */
>> 367 STOPMACHINE_PREPARE,
>> 368 /* Disable interrupts. */
>> 369 STOPMACHINE_DISABLE_IRQ,
>> 370 /* Run the function */
>> 371 STOPMACHINE_RUN,
>> 372 /* Exit */
>> 373 STOPMACHINE_EXIT,
>> 374 };
>> 375
>> 376 struct stop_machine_data {
>> 377 int (*fn)(void *);
>> 378 void *data;
>> 379 /* Like num_online_cpus(), but hotplug cpu uses us, so we need this. */
>> 380 unsigned int num_threads;
>> 381 const struct cpumask *active_cpus;
>> 382
>> 383 enum stopmachine_state state;
>> 384 atomic_t thread_ack;
>> 385 };
>> 386
>> 387 static void set_state(struct stop_machine_data *smdata,
>> 388 enum stopmachine_state newstate)
588 { 389 {
589 struct multi_stop_data msdata = { !! 390 /* Reset ack counter. */
590 .fn = fn, !! 391 atomic_set(&smdata->thread_ack, smdata->num_threads);
591 .data = data, !! 392 smp_wmb();
592 .num_threads = num_online_cpus !! 393 smdata->state = newstate;
593 .active_cpus = cpus, !! 394 }
594 }; !! 395
>> 396 /* Last one to ack a state moves to the next state. */
>> 397 static void ack_state(struct stop_machine_data *smdata)
>> 398 {
>> 399 if (atomic_dec_and_test(&smdata->thread_ack))
>> 400 set_state(smdata, smdata->state + 1);
>> 401 }
>> 402
>> 403 /* This is the cpu_stop function which stops the CPU. */
>> 404 static int stop_machine_cpu_stop(void *data)
>> 405 {
>> 406 struct stop_machine_data *smdata = data;
>> 407 enum stopmachine_state curstate = STOPMACHINE_NONE;
>> 408 int cpu = smp_processor_id(), err = 0;
>> 409 unsigned long flags;
>> 410 bool is_active;
>> 411
>> 412 /*
>> 413 * When called from stop_machine_from_inactive_cpu(), irq might
>> 414 * already be disabled. Save the state and restore it on exit.
>> 415 */
>> 416 local_save_flags(flags);
595 417
596 lockdep_assert_cpus_held(); !! 418 if (!smdata->active_cpus)
>> 419 is_active = cpu == cpumask_first(cpu_online_mask);
>> 420 else
>> 421 is_active = cpumask_test_cpu(cpu, smdata->active_cpus);
>> 422
>> 423 /* Simple state machine */
>> 424 do {
>> 425 /* Chill out and ensure we re-read stopmachine_state. */
>> 426 cpu_relax();
>> 427 if (smdata->state != curstate) {
>> 428 curstate = smdata->state;
>> 429 switch (curstate) {
>> 430 case STOPMACHINE_DISABLE_IRQ:
>> 431 local_irq_disable();
>> 432 hard_irq_disable();
>> 433 break;
>> 434 case STOPMACHINE_RUN:
>> 435 if (is_active)
>> 436 err = smdata->fn(smdata->data);
>> 437 break;
>> 438 default:
>> 439 break;
>> 440 }
>> 441 ack_state(smdata);
>> 442 }
>> 443 } while (curstate != STOPMACHINE_EXIT);
>> 444
>> 445 local_irq_restore(flags);
>> 446 return err;
>> 447 }
>> 448
>> 449 int __stop_machine(int (*fn)(void *), void *data, const struct cpumask *cpus)
>> 450 {
>> 451 struct stop_machine_data smdata = { .fn = fn, .data = data,
>> 452 .num_threads = num_online_cpus(),
>> 453 .active_cpus = cpus };
597 454
598 if (!stop_machine_initialized) { 455 if (!stop_machine_initialized) {
599 /* 456 /*
600 * Handle the case where stop_ 457 * Handle the case where stop_machine() is called
601 * early in boot before stop_m 458 * early in boot before stop_machine() has been
602 * initialized. 459 * initialized.
603 */ 460 */
604 unsigned long flags; 461 unsigned long flags;
605 int ret; 462 int ret;
606 463
607 WARN_ON_ONCE(msdata.num_thread !! 464 WARN_ON_ONCE(smdata.num_threads != 1);
608 465
609 local_irq_save(flags); 466 local_irq_save(flags);
610 hard_irq_disable(); 467 hard_irq_disable();
611 ret = (*fn)(data); 468 ret = (*fn)(data);
612 local_irq_restore(flags); 469 local_irq_restore(flags);
613 470
614 return ret; 471 return ret;
615 } 472 }
616 473
617 /* Set the initial state and stop all 474 /* Set the initial state and stop all online cpus. */
618 set_state(&msdata, MULTI_STOP_PREPARE) !! 475 set_state(&smdata, STOPMACHINE_PREPARE);
619 return stop_cpus(cpu_online_mask, mult !! 476 return stop_cpus(cpu_online_mask, stop_machine_cpu_stop, &smdata);
620 } 477 }
621 478
622 int stop_machine(cpu_stop_fn_t fn, void *data, !! 479 int stop_machine(int (*fn)(void *), void *data, const struct cpumask *cpus)
623 { 480 {
624 int ret; 481 int ret;
625 482
626 /* No CPUs can come up or down during 483 /* No CPUs can come up or down during this. */
627 cpus_read_lock(); !! 484 get_online_cpus();
628 ret = stop_machine_cpuslocked(fn, data !! 485 ret = __stop_machine(fn, data, cpus);
629 cpus_read_unlock(); !! 486 put_online_cpus();
630 return ret; 487 return ret;
631 } 488 }
632 EXPORT_SYMBOL_GPL(stop_machine); 489 EXPORT_SYMBOL_GPL(stop_machine);
633 490
634 #ifdef CONFIG_SCHED_SMT <<
635 int stop_core_cpuslocked(unsigned int cpu, cpu <<
636 { <<
637 const struct cpumask *smt_mask = cpu_s <<
638 <<
639 struct multi_stop_data msdata = { <<
640 .fn = fn, <<
641 .data = data, <<
642 .num_threads = cpumask_weight( <<
643 .active_cpus = smt_mask, <<
644 }; <<
645 <<
646 lockdep_assert_cpus_held(); <<
647 <<
648 /* Set the initial state and stop all <<
649 set_state(&msdata, MULTI_STOP_PREPARE) <<
650 return stop_cpus(smt_mask, multi_cpu_s <<
651 } <<
652 EXPORT_SYMBOL_GPL(stop_core_cpuslocked); <<
653 #endif <<
654 <<
655 /** 491 /**
656 * stop_machine_from_inactive_cpu - stop_machi 492 * stop_machine_from_inactive_cpu - stop_machine() from inactive CPU
657 * @fn: the function to run 493 * @fn: the function to run
658 * @data: the data ptr for the @fn() 494 * @data: the data ptr for the @fn()
659 * @cpus: the cpus to run the @fn() on (NULL = 495 * @cpus: the cpus to run the @fn() on (NULL = any online cpu)
660 * 496 *
661 * This is identical to stop_machine() but can 497 * This is identical to stop_machine() but can be called from a CPU which
662 * is not active. The local CPU is in the pro 498 * is not active. The local CPU is in the process of hotplug (so no other
663 * CPU hotplug can start) and not marked activ 499 * CPU hotplug can start) and not marked active and doesn't have enough
664 * context to sleep. 500 * context to sleep.
665 * 501 *
666 * This function provides stop_machine() funct 502 * This function provides stop_machine() functionality for such state by
667 * using busy-wait for synchronization and exe 503 * using busy-wait for synchronization and executing @fn directly for local
668 * CPU. 504 * CPU.
669 * 505 *
670 * CONTEXT: 506 * CONTEXT:
671 * Local CPU is inactive. Temporarily stops a 507 * Local CPU is inactive. Temporarily stops all active CPUs.
672 * 508 *
673 * RETURNS: 509 * RETURNS:
674 * 0 if all executions of @fn returned 0, any 510 * 0 if all executions of @fn returned 0, any non zero return value if any
675 * returned non zero. 511 * returned non zero.
676 */ 512 */
677 int stop_machine_from_inactive_cpu(cpu_stop_fn !! 513 int stop_machine_from_inactive_cpu(int (*fn)(void *), void *data,
678 const struct 514 const struct cpumask *cpus)
679 { 515 {
680 struct multi_stop_data msdata = { .fn !! 516 struct stop_machine_data smdata = { .fn = fn, .data = data,
681 .a 517 .active_cpus = cpus };
682 struct cpu_stop_done done; 518 struct cpu_stop_done done;
683 int ret; 519 int ret;
684 520
685 /* Local CPU must be inactive and CPU 521 /* Local CPU must be inactive and CPU hotplug in progress. */
686 BUG_ON(cpu_active(raw_smp_processor_id 522 BUG_ON(cpu_active(raw_smp_processor_id()));
687 msdata.num_threads = num_active_cpus() !! 523 smdata.num_threads = num_active_cpus() + 1; /* +1 for local */
688 524
689 /* No proper task established and can' 525 /* No proper task established and can't sleep - busy wait for lock. */
690 while (!mutex_trylock(&stop_cpus_mutex 526 while (!mutex_trylock(&stop_cpus_mutex))
691 cpu_relax(); 527 cpu_relax();
692 528
693 /* Schedule work on other CPUs and exe 529 /* Schedule work on other CPUs and execute directly for local CPU */
694 set_state(&msdata, MULTI_STOP_PREPARE) !! 530 set_state(&smdata, STOPMACHINE_PREPARE);
695 cpu_stop_init_done(&done, num_active_c 531 cpu_stop_init_done(&done, num_active_cpus());
696 queue_stop_cpus_work(cpu_active_mask, !! 532 queue_stop_cpus_work(cpu_active_mask, stop_machine_cpu_stop, &smdata,
697 &done); 533 &done);
698 ret = multi_cpu_stop(&msdata); !! 534 ret = stop_machine_cpu_stop(&smdata);
699 535
700 /* Busy wait for completion. */ 536 /* Busy wait for completion. */
701 while (!completion_done(&done.completi 537 while (!completion_done(&done.completion))
702 cpu_relax(); 538 cpu_relax();
703 539
704 mutex_unlock(&stop_cpus_mutex); 540 mutex_unlock(&stop_cpus_mutex);
705 return ret ?: done.ret; 541 return ret ?: done.ret;
706 } 542 }
>> 543
>> 544 #endif /* CONFIG_STOP_MACHINE */
707 545
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.