blob: b4f83f7bdf86cdfa21c3d52563a9ad45d37a1164 [file] [log] [blame]
Thomas Gleixner6ff3f912019-05-20 19:08:03 +02001// SPDX-License-Identifier: GPL-2.0-or-later
Tejun Heo1142d812010-05-06 18:49:20 +02002/*
3 * kernel/stop_machine.c
4 *
5 * Copyright (C) 2008, 2005 IBM Corporation.
6 * Copyright (C) 2008, 2005 Rusty Russell rusty@rustcorp.com.au
7 * Copyright (C) 2010 SUSE Linux Products GmbH
8 * Copyright (C) 2010 Tejun Heo <tj@kernel.org>
Rusty Russelle5582ca2006-09-29 02:01:35 -07009 */
Tejun Heo1142d812010-05-06 18:49:20 +020010#include <linux/completion.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070011#include <linux/cpu.h>
Tejun Heo1142d812010-05-06 18:49:20 +020012#include <linux/init.h>
Prarit Bhargavaee527cd2007-05-08 00:25:08 -070013#include <linux/kthread.h>
Paul Gortmaker9984de12011-05-23 14:51:41 -040014#include <linux/export.h>
Tejun Heo1142d812010-05-06 18:49:20 +020015#include <linux/percpu.h>
Prarit Bhargavaee527cd2007-05-08 00:25:08 -070016#include <linux/sched.h>
17#include <linux/stop_machine.h>
Benjamin Herrenschmidta12bb442007-05-10 22:22:47 -070018#include <linux/interrupt.h>
Tejun Heo1142d812010-05-06 18:49:20 +020019#include <linux/kallsyms.h>
Thomas Gleixner14e568e2013-01-31 12:11:14 +000020#include <linux/smpboot.h>
Arun Sharma600634972011-07-26 16:09:06 -070021#include <linux/atomic.h>
Oleg Nesterovce4f06d2016-07-26 20:57:36 +020022#include <linux/nmi.h>
Peter Zijlstra0b263512018-04-20 11:50:05 +020023#include <linux/sched/wake_q.h>
Tejun Heo1142d812010-05-06 18:49:20 +020024
25/*
26 * Structure to determine completion condition and record errors. May
27 * be shared by works on different cpus.
28 */
29struct cpu_stop_done {
30 atomic_t nr_todo; /* nr left to execute */
Tejun Heo1142d812010-05-06 18:49:20 +020031 int ret; /* collected return value */
32 struct completion completion; /* fired if nr_todo reaches 0 */
33};
34
35/* the actual stopper, one per every possible cpu, enabled on online cpus */
36struct cpu_stopper {
Oleg Nesterov02cb7aa2015-06-30 03:29:44 +020037 struct task_struct *thread;
38
Thomas Gleixnerde5b55c2018-04-23 21:16:35 +020039 raw_spinlock_t lock;
Richard Kennedy878ae122010-08-09 17:20:34 -070040 bool enabled; /* is this stopper enabled? */
Tejun Heo1142d812010-05-06 18:49:20 +020041 struct list_head works; /* list of pending works */
Oleg Nesterov02cb7aa2015-06-30 03:29:44 +020042
43 struct cpu_stop_work stop_work; /* for stop_cpus */
Tejun Heo1142d812010-05-06 18:49:20 +020044};
45
46static DEFINE_PER_CPU(struct cpu_stopper, cpu_stopper);
Jeremy Fitzhardingef4450272011-10-31 17:11:15 -070047static bool stop_machine_initialized = false;
Tejun Heo1142d812010-05-06 18:49:20 +020048
Oleg Nesterove6253972015-11-21 19:11:48 +010049/* static data for stop_cpus */
50static DEFINE_MUTEX(stop_cpus_mutex);
51static bool stop_cpus_in_progress;
Rik van Riel7053ea12013-11-01 10:41:46 -040052
Tejun Heo1142d812010-05-06 18:49:20 +020053static void cpu_stop_init_done(struct cpu_stop_done *done, unsigned int nr_todo)
54{
55 memset(done, 0, sizeof(*done));
56 atomic_set(&done->nr_todo, nr_todo);
57 init_completion(&done->completion);
58}
59
60/* signal completion unless @done is NULL */
Oleg Nesterov6fa3b822015-11-15 20:33:26 +010061static void cpu_stop_signal_done(struct cpu_stop_done *done)
Tejun Heo1142d812010-05-06 18:49:20 +020062{
Oleg Nesterovdd2e3122015-11-15 20:33:29 +010063 if (atomic_dec_and_test(&done->nr_todo))
64 complete(&done->completion);
Tejun Heo1142d812010-05-06 18:49:20 +020065}
66
Oleg Nesterov5caa1c02015-10-08 16:51:34 +020067static void __cpu_stop_queue_work(struct cpu_stopper *stopper,
Peter Zijlstra0b263512018-04-20 11:50:05 +020068 struct cpu_stop_work *work,
69 struct wake_q_head *wakeq)
Oleg Nesterov5caa1c02015-10-08 16:51:34 +020070{
71 list_add_tail(&work->list, &stopper->works);
Peter Zijlstra0b263512018-04-20 11:50:05 +020072 wake_q_add(wakeq, stopper->thread);
Oleg Nesterov5caa1c02015-10-08 16:51:34 +020073}
74
Tejun Heo1142d812010-05-06 18:49:20 +020075/* queue @work to @stopper. if offline, @work is completed immediately */
Oleg Nesterov1b034bd2015-11-17 18:05:23 +010076static bool cpu_stop_queue_work(unsigned int cpu, struct cpu_stop_work *work)
Tejun Heo1142d812010-05-06 18:49:20 +020077{
Thomas Gleixner860a0ff2013-01-31 12:11:13 +000078 struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
Peter Zijlstra0b263512018-04-20 11:50:05 +020079 DEFINE_WAKE_Q(wakeq);
Tejun Heo1142d812010-05-06 18:49:20 +020080 unsigned long flags;
Oleg Nesterov1b034bd2015-11-17 18:05:23 +010081 bool enabled;
Tejun Heo1142d812010-05-06 18:49:20 +020082
Prasad Sodagudicfd35512018-08-03 13:56:06 -070083 preempt_disable();
Thomas Gleixnerde5b55c2018-04-23 21:16:35 +020084 raw_spin_lock_irqsave(&stopper->lock, flags);
Oleg Nesterov1b034bd2015-11-17 18:05:23 +010085 enabled = stopper->enabled;
86 if (enabled)
Peter Zijlstra0b263512018-04-20 11:50:05 +020087 __cpu_stop_queue_work(stopper, work, &wakeq);
Oleg Nesterovdd2e3122015-11-15 20:33:29 +010088 else if (work->done)
Oleg Nesterov6fa3b822015-11-15 20:33:26 +010089 cpu_stop_signal_done(work->done);
Thomas Gleixnerde5b55c2018-04-23 21:16:35 +020090 raw_spin_unlock_irqrestore(&stopper->lock, flags);
Oleg Nesterov1b034bd2015-11-17 18:05:23 +010091
Peter Zijlstra0b263512018-04-20 11:50:05 +020092 wake_up_q(&wakeq);
Prasad Sodagudicfd35512018-08-03 13:56:06 -070093 preempt_enable();
Peter Zijlstra0b263512018-04-20 11:50:05 +020094
Oleg Nesterov1b034bd2015-11-17 18:05:23 +010095 return enabled;
Tejun Heo1142d812010-05-06 18:49:20 +020096}
97
98/**
99 * stop_one_cpu - stop a cpu
100 * @cpu: cpu to stop
101 * @fn: function to execute
102 * @arg: argument to @fn
103 *
104 * Execute @fn(@arg) on @cpu. @fn is run in a process context with
105 * the highest priority preempting any task on the cpu and
106 * monopolizing it. This function returns after the execution is
107 * complete.
108 *
109 * This function doesn't guarantee @cpu stays online till @fn
110 * completes. If @cpu goes down in the middle, execution may happen
111 * partially or fully on different cpus. @fn should either be ready
112 * for that or the caller should ensure that @cpu stays online until
113 * this function completes.
114 *
115 * CONTEXT:
116 * Might sleep.
117 *
118 * RETURNS:
119 * -ENOENT if @fn(@arg) was not executed because @cpu was offline;
120 * otherwise, the return value of @fn.
121 */
122int stop_one_cpu(unsigned int cpu, cpu_stop_fn_t fn, void *arg)
123{
124 struct cpu_stop_done done;
125 struct cpu_stop_work work = { .fn = fn, .arg = arg, .done = &done };
126
127 cpu_stop_init_done(&done, 1);
Oleg Nesterov958c5f842015-11-15 20:33:20 +0100128 if (!cpu_stop_queue_work(cpu, &work))
129 return -ENOENT;
Cheng Chaobf89a302016-09-14 10:01:50 +0800130 /*
131 * In case @cpu == smp_proccessor_id() we can avoid a sleep+wakeup
132 * cycle by doing a preemption:
133 */
134 cond_resched();
Tejun Heo1142d812010-05-06 18:49:20 +0200135 wait_for_completion(&done.completion);
Oleg Nesterov958c5f842015-11-15 20:33:20 +0100136 return done.ret;
Tejun Heo1142d812010-05-06 18:49:20 +0200137}
138
Peter Zijlstra1be0bd772013-10-07 11:29:15 +0100139/* This controls the threads on each CPU. */
140enum multi_stop_state {
141 /* Dummy starting state for thread. */
142 MULTI_STOP_NONE,
143 /* Awaiting everyone to be scheduled. */
144 MULTI_STOP_PREPARE,
145 /* Disable interrupts. */
146 MULTI_STOP_DISABLE_IRQ,
147 /* Run the function */
148 MULTI_STOP_RUN,
149 /* Exit */
150 MULTI_STOP_EXIT,
151};
152
153struct multi_stop_data {
Oleg Nesterov9a301f22015-06-30 03:29:55 +0200154 cpu_stop_fn_t fn;
Peter Zijlstra1be0bd772013-10-07 11:29:15 +0100155 void *data;
156 /* Like num_online_cpus(), but hotplug cpu uses us, so we need this. */
157 unsigned int num_threads;
158 const struct cpumask *active_cpus;
159
160 enum multi_stop_state state;
161 atomic_t thread_ack;
162};
163
164static void set_state(struct multi_stop_data *msdata,
165 enum multi_stop_state newstate)
166{
167 /* Reset ack counter. */
168 atomic_set(&msdata->thread_ack, msdata->num_threads);
169 smp_wmb();
170 msdata->state = newstate;
171}
172
173/* Last one to ack a state moves to the next state. */
174static void ack_state(struct multi_stop_data *msdata)
175{
176 if (atomic_dec_and_test(&msdata->thread_ack))
177 set_state(msdata, msdata->state + 1);
178}
179
Heiko Carstens4ecf0a42019-06-08 12:13:57 +0200180void __weak stop_machine_yield(const struct cpumask *cpumask)
181{
182 cpu_relax();
183}
184
Peter Zijlstra1be0bd772013-10-07 11:29:15 +0100185/* This is the cpu_stop function which stops the CPU. */
186static int multi_cpu_stop(void *data)
187{
188 struct multi_stop_data *msdata = data;
189 enum multi_stop_state curstate = MULTI_STOP_NONE;
190 int cpu = smp_processor_id(), err = 0;
Martin Schwidefsky38f2c6912019-05-17 12:50:42 +0200191 const struct cpumask *cpumask;
Peter Zijlstra1be0bd772013-10-07 11:29:15 +0100192 unsigned long flags;
193 bool is_active;
194
195 /*
196 * When called from stop_machine_from_inactive_cpu(), irq might
197 * already be disabled. Save the state and restore it on exit.
198 */
199 local_save_flags(flags);
200
Martin Schwidefsky38f2c6912019-05-17 12:50:42 +0200201 if (!msdata->active_cpus) {
202 cpumask = cpu_online_mask;
203 is_active = cpu == cpumask_first(cpumask);
204 } else {
205 cpumask = msdata->active_cpus;
206 is_active = cpumask_test_cpu(cpu, cpumask);
207 }
Peter Zijlstra1be0bd772013-10-07 11:29:15 +0100208
209 /* Simple state machine */
210 do {
211 /* Chill out and ensure we re-read multi_stop_state. */
Heiko Carstens4ecf0a42019-06-08 12:13:57 +0200212 stop_machine_yield(cpumask);
Peter Zijlstra1be0bd772013-10-07 11:29:15 +0100213 if (msdata->state != curstate) {
214 curstate = msdata->state;
215 switch (curstate) {
216 case MULTI_STOP_DISABLE_IRQ:
217 local_irq_disable();
218 hard_irq_disable();
219 break;
220 case MULTI_STOP_RUN:
221 if (is_active)
222 err = msdata->fn(msdata->data);
223 break;
224 default:
225 break;
226 }
227 ack_state(msdata);
Oleg Nesterovce4f06d2016-07-26 20:57:36 +0200228 } else if (curstate > MULTI_STOP_PREPARE) {
229 /*
230 * At this stage all other CPUs we depend on must spin
231 * in the same loop. Any reason for hard-lockup should
232 * be detected and reported on their side.
233 */
234 touch_nmi_watchdog();
Peter Zijlstra1be0bd772013-10-07 11:29:15 +0100235 }
236 } while (curstate != MULTI_STOP_EXIT);
237
238 local_irq_restore(flags);
239 return err;
240}
241
Oleg Nesterov5caa1c02015-10-08 16:51:34 +0200242static int cpu_stop_queue_two_works(int cpu1, struct cpu_stop_work *work1,
243 int cpu2, struct cpu_stop_work *work2)
244{
Oleg Nesterovd8bc8532015-10-08 19:01:41 +0200245 struct cpu_stopper *stopper1 = per_cpu_ptr(&cpu_stopper, cpu1);
246 struct cpu_stopper *stopper2 = per_cpu_ptr(&cpu_stopper, cpu2);
Peter Zijlstra0b263512018-04-20 11:50:05 +0200247 DEFINE_WAKE_Q(wakeq);
Oleg Nesterovd8bc8532015-10-08 19:01:41 +0200248 int err;
Peter Zijlstrab80a2bf2018-07-30 13:21:40 +0200249
Oleg Nesterove6253972015-11-21 19:11:48 +0100250retry:
Peter Zijlstrab80a2bf2018-07-30 13:21:40 +0200251 /*
252 * The waking up of stopper threads has to happen in the same
253 * scheduling context as the queueing. Otherwise, there is a
254 * possibility of one of the above stoppers being woken up by another
255 * CPU, and preempting us. This will cause us to not wake up the other
256 * stopper forever.
257 */
258 preempt_disable();
Thomas Gleixnerde5b55c2018-04-23 21:16:35 +0200259 raw_spin_lock_irq(&stopper1->lock);
260 raw_spin_lock_nested(&stopper2->lock, SINGLE_DEPTH_NESTING);
Oleg Nesterovd8bc8532015-10-08 19:01:41 +0200261
Peter Zijlstrab80a2bf2018-07-30 13:21:40 +0200262 if (!stopper1->enabled || !stopper2->enabled) {
263 err = -ENOENT;
Oleg Nesterovd8bc8532015-10-08 19:01:41 +0200264 goto unlock;
Peter Zijlstrab80a2bf2018-07-30 13:21:40 +0200265 }
266
Oleg Nesterove6253972015-11-21 19:11:48 +0100267 /*
268 * Ensure that if we race with __stop_cpus() the stoppers won't get
269 * queued up in reverse order leading to system deadlock.
270 *
271 * We can't miss stop_cpus_in_progress if queue_stop_cpus_work() has
272 * queued a work on cpu1 but not on cpu2, we hold both locks.
273 *
274 * It can be falsely true but it is safe to spin until it is cleared,
275 * queue_stop_cpus_work() does everything under preempt_disable().
276 */
Peter Zijlstrab80a2bf2018-07-30 13:21:40 +0200277 if (unlikely(stop_cpus_in_progress)) {
278 err = -EDEADLK;
279 goto unlock;
280 }
Oleg Nesterovd8bc8532015-10-08 19:01:41 +0200281
282 err = 0;
Peter Zijlstra0b263512018-04-20 11:50:05 +0200283 __cpu_stop_queue_work(stopper1, work1, &wakeq);
284 __cpu_stop_queue_work(stopper2, work2, &wakeq);
Peter Zijlstrab80a2bf2018-07-30 13:21:40 +0200285
Oleg Nesterovd8bc8532015-10-08 19:01:41 +0200286unlock:
Thomas Gleixnerde5b55c2018-04-23 21:16:35 +0200287 raw_spin_unlock(&stopper2->lock);
288 raw_spin_unlock_irq(&stopper1->lock);
Oleg Nesterov5caa1c02015-10-08 16:51:34 +0200289
Oleg Nesterove6253972015-11-21 19:11:48 +0100290 if (unlikely(err == -EDEADLK)) {
Peter Zijlstrab80a2bf2018-07-30 13:21:40 +0200291 preempt_enable();
292
Oleg Nesterove6253972015-11-21 19:11:48 +0100293 while (stop_cpus_in_progress)
294 cpu_relax();
Peter Zijlstrab80a2bf2018-07-30 13:21:40 +0200295
Oleg Nesterove6253972015-11-21 19:11:48 +0100296 goto retry;
297 }
Peter Zijlstra0b263512018-04-20 11:50:05 +0200298
Peter Zijlstrab80a2bf2018-07-30 13:21:40 +0200299 wake_up_q(&wakeq);
300 preempt_enable();
Peter Zijlstra0b263512018-04-20 11:50:05 +0200301
Oleg Nesterovd8bc8532015-10-08 19:01:41 +0200302 return err;
Oleg Nesterov5caa1c02015-10-08 16:51:34 +0200303}
Peter Zijlstra1be0bd772013-10-07 11:29:15 +0100304/**
305 * stop_two_cpus - stops two cpus
306 * @cpu1: the cpu to stop
307 * @cpu2: the other cpu to stop
308 * @fn: function to execute
309 * @arg: argument to @fn
310 *
311 * Stops both the current and specified CPU and runs @fn on one of them.
312 *
313 * returns when both are completed.
314 */
315int stop_two_cpus(unsigned int cpu1, unsigned int cpu2, cpu_stop_fn_t fn, void *arg)
316{
Peter Zijlstra1be0bd772013-10-07 11:29:15 +0100317 struct cpu_stop_done done;
318 struct cpu_stop_work work1, work2;
Peter Zijlstra6acce3e2013-10-11 14:38:20 +0200319 struct multi_stop_data msdata;
320
Peter Zijlstra6acce3e2013-10-11 14:38:20 +0200321 msdata = (struct multi_stop_data){
Peter Zijlstra1be0bd772013-10-07 11:29:15 +0100322 .fn = fn,
323 .data = arg,
324 .num_threads = 2,
325 .active_cpus = cpumask_of(cpu1),
326 };
327
328 work1 = work2 = (struct cpu_stop_work){
329 .fn = multi_cpu_stop,
330 .arg = &msdata,
331 .done = &done
332 };
333
Peter Zijlstra1be0bd772013-10-07 11:29:15 +0100334 cpu_stop_init_done(&done, 2);
335 set_state(&msdata, MULTI_STOP_PREPARE);
336
Oleg Nesterov5caa1c02015-10-08 16:51:34 +0200337 if (cpu1 > cpu2)
338 swap(cpu1, cpu2);
Oleg Nesterov6a190052015-11-15 20:33:14 +0100339 if (cpu_stop_queue_two_works(cpu1, &work1, cpu2, &work2))
Oleg Nesterov5caa1c02015-10-08 16:51:34 +0200340 return -ENOENT;
Peter Zijlstra1be0bd772013-10-07 11:29:15 +0100341
342 wait_for_completion(&done.completion);
Oleg Nesterov6a190052015-11-15 20:33:14 +0100343 return done.ret;
Peter Zijlstra1be0bd772013-10-07 11:29:15 +0100344}
345
Tejun Heo1142d812010-05-06 18:49:20 +0200346/**
347 * stop_one_cpu_nowait - stop a cpu but don't wait for completion
348 * @cpu: cpu to stop
349 * @fn: function to execute
350 * @arg: argument to @fn
Fabian Frederickcf250042014-06-04 16:11:22 -0700351 * @work_buf: pointer to cpu_stop_work structure
Tejun Heo1142d812010-05-06 18:49:20 +0200352 *
353 * Similar to stop_one_cpu() but doesn't wait for completion. The
354 * caller is responsible for ensuring @work_buf is currently unused
355 * and will remain untouched until stopper starts executing @fn.
356 *
357 * CONTEXT:
358 * Don't care.
Oleg Nesterov1b034bd2015-11-17 18:05:23 +0100359 *
360 * RETURNS:
361 * true if cpu_stop_work was queued successfully and @fn will be called,
362 * false otherwise.
Tejun Heo1142d812010-05-06 18:49:20 +0200363 */
Oleg Nesterov1b034bd2015-11-17 18:05:23 +0100364bool stop_one_cpu_nowait(unsigned int cpu, cpu_stop_fn_t fn, void *arg,
Tejun Heo1142d812010-05-06 18:49:20 +0200365 struct cpu_stop_work *work_buf)
366{
367 *work_buf = (struct cpu_stop_work){ .fn = fn, .arg = arg, };
Oleg Nesterov1b034bd2015-11-17 18:05:23 +0100368 return cpu_stop_queue_work(cpu, work_buf);
Tejun Heo1142d812010-05-06 18:49:20 +0200369}
370
Oleg Nesterov4aff1ca2015-11-15 20:33:23 +0100371static bool queue_stop_cpus_work(const struct cpumask *cpumask,
Tejun Heofd7355b2011-06-23 11:19:27 -0700372 cpu_stop_fn_t fn, void *arg,
373 struct cpu_stop_done *done)
Tejun Heo1142d812010-05-06 18:49:20 +0200374{
375 struct cpu_stop_work *work;
Tejun Heo1142d812010-05-06 18:49:20 +0200376 unsigned int cpu;
Oleg Nesterov4aff1ca2015-11-15 20:33:23 +0100377 bool queued = false;
Tejun Heo1142d812010-05-06 18:49:20 +0200378
Tejun Heo1142d812010-05-06 18:49:20 +0200379 /*
380 * Disable preemption while queueing to avoid getting
381 * preempted by a stopper which might wait for other stoppers
382 * to enter @fn which can lead to deadlock.
383 */
Oleg Nesterove6253972015-11-21 19:11:48 +0100384 preempt_disable();
385 stop_cpus_in_progress = true;
Oleg Nesterovb377c2a2015-06-30 03:29:48 +0200386 for_each_cpu(cpu, cpumask) {
387 work = &per_cpu(cpu_stopper.stop_work, cpu);
388 work->fn = fn;
389 work->arg = arg;
390 work->done = done;
Oleg Nesterov4aff1ca2015-11-15 20:33:23 +0100391 if (cpu_stop_queue_work(cpu, work))
392 queued = true;
Oleg Nesterovb377c2a2015-06-30 03:29:48 +0200393 }
Oleg Nesterove6253972015-11-21 19:11:48 +0100394 stop_cpus_in_progress = false;
395 preempt_enable();
Oleg Nesterov4aff1ca2015-11-15 20:33:23 +0100396
397 return queued;
Tejun Heofd7355b2011-06-23 11:19:27 -0700398}
Tejun Heo1142d812010-05-06 18:49:20 +0200399
Tejun Heofd7355b2011-06-23 11:19:27 -0700400static int __stop_cpus(const struct cpumask *cpumask,
401 cpu_stop_fn_t fn, void *arg)
402{
403 struct cpu_stop_done done;
404
405 cpu_stop_init_done(&done, cpumask_weight(cpumask));
Oleg Nesterov4aff1ca2015-11-15 20:33:23 +0100406 if (!queue_stop_cpus_work(cpumask, fn, arg, &done))
407 return -ENOENT;
Tejun Heo1142d812010-05-06 18:49:20 +0200408 wait_for_completion(&done.completion);
Oleg Nesterov4aff1ca2015-11-15 20:33:23 +0100409 return done.ret;
Tejun Heo1142d812010-05-06 18:49:20 +0200410}
411
412/**
413 * stop_cpus - stop multiple cpus
414 * @cpumask: cpus to stop
415 * @fn: function to execute
416 * @arg: argument to @fn
417 *
418 * Execute @fn(@arg) on online cpus in @cpumask. On each target cpu,
419 * @fn is run in a process context with the highest priority
420 * preempting any task on the cpu and monopolizing it. This function
421 * returns after all executions are complete.
422 *
423 * This function doesn't guarantee the cpus in @cpumask stay online
424 * till @fn completes. If some cpus go down in the middle, execution
425 * on the cpu may happen partially or fully on different cpus. @fn
426 * should either be ready for that or the caller should ensure that
427 * the cpus stay online until this function completes.
428 *
429 * All stop_cpus() calls are serialized making it safe for @fn to wait
430 * for all cpus to start executing it.
431 *
432 * CONTEXT:
433 * Might sleep.
434 *
435 * RETURNS:
436 * -ENOENT if @fn(@arg) was not executed at all because all cpus in
437 * @cpumask were offline; otherwise, 0 if all executions of @fn
438 * returned 0, any non zero return value if any returned non zero.
439 */
440int stop_cpus(const struct cpumask *cpumask, cpu_stop_fn_t fn, void *arg)
441{
442 int ret;
443
444 /* static works are used, process one request at a time */
445 mutex_lock(&stop_cpus_mutex);
446 ret = __stop_cpus(cpumask, fn, arg);
447 mutex_unlock(&stop_cpus_mutex);
448 return ret;
449}
450
451/**
452 * try_stop_cpus - try to stop multiple cpus
453 * @cpumask: cpus to stop
454 * @fn: function to execute
455 * @arg: argument to @fn
456 *
457 * Identical to stop_cpus() except that it fails with -EAGAIN if
458 * someone else is already using the facility.
459 *
460 * CONTEXT:
461 * Might sleep.
462 *
463 * RETURNS:
464 * -EAGAIN if someone else is already stopping cpus, -ENOENT if
465 * @fn(@arg) was not executed at all because all cpus in @cpumask were
466 * offline; otherwise, 0 if all executions of @fn returned 0, any non
467 * zero return value if any returned non zero.
468 */
469int try_stop_cpus(const struct cpumask *cpumask, cpu_stop_fn_t fn, void *arg)
470{
471 int ret;
472
473 /* static works are used, process one request at a time */
474 if (!mutex_trylock(&stop_cpus_mutex))
475 return -EAGAIN;
476 ret = __stop_cpus(cpumask, fn, arg);
477 mutex_unlock(&stop_cpus_mutex);
478 return ret;
479}
480
Thomas Gleixner14e568e2013-01-31 12:11:14 +0000481static int cpu_stop_should_run(unsigned int cpu)
Tejun Heo1142d812010-05-06 18:49:20 +0200482{
Thomas Gleixner14e568e2013-01-31 12:11:14 +0000483 struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
484 unsigned long flags;
485 int run;
486
Thomas Gleixnerde5b55c2018-04-23 21:16:35 +0200487 raw_spin_lock_irqsave(&stopper->lock, flags);
Thomas Gleixner14e568e2013-01-31 12:11:14 +0000488 run = !list_empty(&stopper->works);
Thomas Gleixnerde5b55c2018-04-23 21:16:35 +0200489 raw_spin_unlock_irqrestore(&stopper->lock, flags);
Thomas Gleixner14e568e2013-01-31 12:11:14 +0000490 return run;
491}
492
493static void cpu_stopper_thread(unsigned int cpu)
494{
495 struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
Tejun Heo1142d812010-05-06 18:49:20 +0200496 struct cpu_stop_work *work;
Tejun Heo1142d812010-05-06 18:49:20 +0200497
498repeat:
Tejun Heo1142d812010-05-06 18:49:20 +0200499 work = NULL;
Thomas Gleixnerde5b55c2018-04-23 21:16:35 +0200500 raw_spin_lock_irq(&stopper->lock);
Tejun Heo1142d812010-05-06 18:49:20 +0200501 if (!list_empty(&stopper->works)) {
502 work = list_first_entry(&stopper->works,
503 struct cpu_stop_work, list);
504 list_del_init(&work->list);
505 }
Thomas Gleixnerde5b55c2018-04-23 21:16:35 +0200506 raw_spin_unlock_irq(&stopper->lock);
Tejun Heo1142d812010-05-06 18:49:20 +0200507
508 if (work) {
509 cpu_stop_fn_t fn = work->fn;
510 void *arg = work->arg;
511 struct cpu_stop_done *done = work->done;
Oleg Nesterovaccaf6e2015-11-15 20:33:32 +0100512 int ret;
Tejun Heo1142d812010-05-06 18:49:20 +0200513
Oleg Nesterovaccaf6e2015-11-15 20:33:32 +0100514 /* cpu stop callbacks must not sleep, make in_atomic() == T */
515 preempt_count_inc();
Tejun Heo1142d812010-05-06 18:49:20 +0200516 ret = fn(arg);
Oleg Nesterovdd2e3122015-11-15 20:33:29 +0100517 if (done) {
518 if (ret)
519 done->ret = ret;
520 cpu_stop_signal_done(done);
521 }
Oleg Nesterovaccaf6e2015-11-15 20:33:32 +0100522 preempt_count_dec();
Tejun Heo1142d812010-05-06 18:49:20 +0200523 WARN_ONCE(preempt_count(),
Sakari Ailusd75f7732019-03-25 21:32:28 +0200524 "cpu_stop: %ps(%p) leaked preempt count\n", fn, arg);
Thomas Gleixner14e568e2013-01-31 12:11:14 +0000525 goto repeat;
526 }
Tejun Heo1142d812010-05-06 18:49:20 +0200527}
528
Oleg Nesterov233e7f22015-10-08 16:51:31 +0200529void stop_machine_park(int cpu)
530{
531 struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
532 /*
533 * Lockless. cpu_stopper_thread() will take stopper->lock and flush
534 * the pending works before it parks, until then it is fine to queue
535 * the new works.
536 */
537 stopper->enabled = false;
538 kthread_park(stopper->thread);
539}
540
Peter Zijlstra34f971f2010-09-22 13:53:15 +0200541extern void sched_set_stop_task(int cpu, struct task_struct *stop);
542
Thomas Gleixner14e568e2013-01-31 12:11:14 +0000543static void cpu_stop_create(unsigned int cpu)
Tejun Heo1142d812010-05-06 18:49:20 +0200544{
Oleg Nesterov02cb7aa2015-06-30 03:29:44 +0200545 sched_set_stop_task(cpu, per_cpu(cpu_stopper.thread, cpu));
Tejun Heo1142d812010-05-06 18:49:20 +0200546}
547
Thomas Gleixner14e568e2013-01-31 12:11:14 +0000548static void cpu_stop_park(unsigned int cpu)
549{
550 struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
Thomas Gleixner14e568e2013-01-31 12:11:14 +0000551
Oleg Nesterov233e7f22015-10-08 16:51:31 +0200552 WARN_ON(!list_empty(&stopper->works));
Thomas Gleixner14e568e2013-01-31 12:11:14 +0000553}
554
Oleg Nesterovc00166d2015-10-09 18:00:49 +0200555void stop_machine_unpark(int cpu)
556{
557 struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
558
Oleg Nesterovf0cf16c2015-10-09 18:00:51 +0200559 stopper->enabled = true;
Oleg Nesterovc00166d2015-10-09 18:00:49 +0200560 kthread_unpark(stopper->thread);
561}
562
Thomas Gleixner14e568e2013-01-31 12:11:14 +0000563static struct smp_hotplug_thread cpu_stop_threads = {
Oleg Nesterov02cb7aa2015-06-30 03:29:44 +0200564 .store = &cpu_stopper.thread,
Thomas Gleixner14e568e2013-01-31 12:11:14 +0000565 .thread_should_run = cpu_stop_should_run,
566 .thread_fn = cpu_stopper_thread,
567 .thread_comm = "migration/%u",
568 .create = cpu_stop_create,
Thomas Gleixner14e568e2013-01-31 12:11:14 +0000569 .park = cpu_stop_park,
Thomas Gleixner14e568e2013-01-31 12:11:14 +0000570 .selfparking = true,
Tejun Heo1142d812010-05-06 18:49:20 +0200571};
572
573static int __init cpu_stop_init(void)
574{
Tejun Heo1142d812010-05-06 18:49:20 +0200575 unsigned int cpu;
Tejun Heo1142d812010-05-06 18:49:20 +0200576
577 for_each_possible_cpu(cpu) {
578 struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
579
Thomas Gleixnerde5b55c2018-04-23 21:16:35 +0200580 raw_spin_lock_init(&stopper->lock);
Tejun Heo1142d812010-05-06 18:49:20 +0200581 INIT_LIST_HEAD(&stopper->works);
582 }
583
Thomas Gleixner14e568e2013-01-31 12:11:14 +0000584 BUG_ON(smpboot_register_percpu_thread(&cpu_stop_threads));
Oleg Nesterovc00166d2015-10-09 18:00:49 +0200585 stop_machine_unpark(raw_smp_processor_id());
Jeremy Fitzhardingef4450272011-10-31 17:11:15 -0700586 stop_machine_initialized = true;
Tejun Heo1142d812010-05-06 18:49:20 +0200587 return 0;
588}
589early_initcall(cpu_stop_init);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700590
Sebastian Andrzej Siewiorfe5595c2017-05-24 10:15:16 +0200591int stop_machine_cpuslocked(cpu_stop_fn_t fn, void *data,
592 const struct cpumask *cpus)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700593{
Peter Zijlstra1be0bd772013-10-07 11:29:15 +0100594 struct multi_stop_data msdata = {
595 .fn = fn,
596 .data = data,
597 .num_threads = num_online_cpus(),
598 .active_cpus = cpus,
599 };
Linus Torvalds1da177e2005-04-16 15:20:36 -0700600
Sebastian Andrzej Siewiorfe5595c2017-05-24 10:15:16 +0200601 lockdep_assert_cpus_held();
602
Jeremy Fitzhardingef4450272011-10-31 17:11:15 -0700603 if (!stop_machine_initialized) {
604 /*
605 * Handle the case where stop_machine() is called
606 * early in boot before stop_machine() has been
607 * initialized.
608 */
609 unsigned long flags;
610 int ret;
611
Peter Zijlstra1be0bd772013-10-07 11:29:15 +0100612 WARN_ON_ONCE(msdata.num_threads != 1);
Jeremy Fitzhardingef4450272011-10-31 17:11:15 -0700613
614 local_irq_save(flags);
615 hard_irq_disable();
616 ret = (*fn)(data);
617 local_irq_restore(flags);
618
619 return ret;
620 }
621
Tejun Heo3fc1f1e2010-05-06 18:49:20 +0200622 /* Set the initial state and stop all online cpus. */
Peter Zijlstra1be0bd772013-10-07 11:29:15 +0100623 set_state(&msdata, MULTI_STOP_PREPARE);
624 return stop_cpus(cpu_online_mask, multi_cpu_stop, &msdata);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700625}
626
Oleg Nesterov9a301f22015-06-30 03:29:55 +0200627int stop_machine(cpu_stop_fn_t fn, void *data, const struct cpumask *cpus)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700628{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700629 int ret;
630
631 /* No CPUs can come up or down during this. */
Sebastian Andrzej Siewiorfe5595c2017-05-24 10:15:16 +0200632 cpus_read_lock();
633 ret = stop_machine_cpuslocked(fn, data, cpus);
634 cpus_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700635 return ret;
636}
Rusty Russelleeec4fa2008-07-28 12:16:30 -0500637EXPORT_SYMBOL_GPL(stop_machine);
Tejun Heobbf1bb32010-05-08 16:20:53 +0200638
Tejun Heof740e6cd2011-06-23 11:19:28 -0700639/**
640 * stop_machine_from_inactive_cpu - stop_machine() from inactive CPU
641 * @fn: the function to run
642 * @data: the data ptr for the @fn()
643 * @cpus: the cpus to run the @fn() on (NULL = any online cpu)
644 *
645 * This is identical to stop_machine() but can be called from a CPU which
646 * is not active. The local CPU is in the process of hotplug (so no other
647 * CPU hotplug can start) and not marked active and doesn't have enough
648 * context to sleep.
649 *
650 * This function provides stop_machine() functionality for such state by
651 * using busy-wait for synchronization and executing @fn directly for local
652 * CPU.
653 *
654 * CONTEXT:
655 * Local CPU is inactive. Temporarily stops all active CPUs.
656 *
657 * RETURNS:
658 * 0 if all executions of @fn returned 0, any non zero return value if any
659 * returned non zero.
660 */
Oleg Nesterov9a301f22015-06-30 03:29:55 +0200661int stop_machine_from_inactive_cpu(cpu_stop_fn_t fn, void *data,
Tejun Heof740e6cd2011-06-23 11:19:28 -0700662 const struct cpumask *cpus)
663{
Peter Zijlstra1be0bd772013-10-07 11:29:15 +0100664 struct multi_stop_data msdata = { .fn = fn, .data = data,
Tejun Heof740e6cd2011-06-23 11:19:28 -0700665 .active_cpus = cpus };
666 struct cpu_stop_done done;
667 int ret;
668
669 /* Local CPU must be inactive and CPU hotplug in progress. */
670 BUG_ON(cpu_active(raw_smp_processor_id()));
Peter Zijlstra1be0bd772013-10-07 11:29:15 +0100671 msdata.num_threads = num_active_cpus() + 1; /* +1 for local */
Tejun Heof740e6cd2011-06-23 11:19:28 -0700672
673 /* No proper task established and can't sleep - busy wait for lock. */
674 while (!mutex_trylock(&stop_cpus_mutex))
675 cpu_relax();
676
677 /* Schedule work on other CPUs and execute directly for local CPU */
Peter Zijlstra1be0bd772013-10-07 11:29:15 +0100678 set_state(&msdata, MULTI_STOP_PREPARE);
Tejun Heof740e6cd2011-06-23 11:19:28 -0700679 cpu_stop_init_done(&done, num_active_cpus());
Peter Zijlstra1be0bd772013-10-07 11:29:15 +0100680 queue_stop_cpus_work(cpu_active_mask, multi_cpu_stop, &msdata,
Tejun Heof740e6cd2011-06-23 11:19:28 -0700681 &done);
Peter Zijlstra1be0bd772013-10-07 11:29:15 +0100682 ret = multi_cpu_stop(&msdata);
Tejun Heof740e6cd2011-06-23 11:19:28 -0700683
684 /* Busy wait for completion. */
685 while (!completion_done(&done.completion))
686 cpu_relax();
687
688 mutex_unlock(&stop_cpus_mutex);
689 return ret ?: done.ret;
690}