blob: 81aa1cdc6bc99b1f164aeb2dd1178764b914913b [file] [log] [blame]
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01001/*
2 * Read-Copy Update mechanism for mutual exclusion
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
Paul E. McKenney87de1cf2013-12-03 10:02:52 -080015 * along with this program; if not, you can access it online at
16 * http://www.gnu.org/licenses/gpl-2.0.html.
Paul E. McKenney64db4cf2008-12-18 21:55:32 +010017 *
18 * Copyright IBM Corporation, 2008
19 *
20 * Authors: Dipankar Sarma <dipankar@in.ibm.com>
21 * Manfred Spraul <manfred@colorfullife.com>
22 * Paul E. McKenney <paulmck@linux.vnet.ibm.com> Hierarchical version
23 *
24 * Based on the original work by Paul McKenney <paulmck@us.ibm.com>
25 * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen.
26 *
27 * For detailed explanation of Read-Copy Update mechanism see -
Paul E. McKenneya71fca52009-09-18 10:28:19 -070028 * Documentation/RCU
Paul E. McKenney64db4cf2008-12-18 21:55:32 +010029 */
30#include <linux/types.h>
31#include <linux/kernel.h>
32#include <linux/init.h>
33#include <linux/spinlock.h>
34#include <linux/smp.h>
35#include <linux/rcupdate.h>
36#include <linux/interrupt.h>
37#include <linux/sched.h>
Ingo Molnarc1dc0b92009-08-02 11:28:21 +020038#include <linux/nmi.h>
Paul E. McKenney8826f3b2011-05-11 05:41:41 -070039#include <linux/atomic.h>
Paul E. McKenney64db4cf2008-12-18 21:55:32 +010040#include <linux/bitops.h>
Paul Gortmaker9984de12011-05-23 14:51:41 -040041#include <linux/export.h>
Paul E. McKenney64db4cf2008-12-18 21:55:32 +010042#include <linux/completion.h>
43#include <linux/moduleparam.h>
Paul E. McKenney4102ada2013-10-08 20:23:47 -070044#include <linux/module.h>
Paul E. McKenney64db4cf2008-12-18 21:55:32 +010045#include <linux/percpu.h>
46#include <linux/notifier.h>
47#include <linux/cpu.h>
48#include <linux/mutex.h>
49#include <linux/time.h>
Paul E. McKenneybbad9372010-04-02 16:17:17 -070050#include <linux/kernel_stat.h>
Paul E. McKenneya26ac242011-01-12 14:10:23 -080051#include <linux/wait.h>
52#include <linux/kthread.h>
Linus Torvalds268bb0c2011-05-20 12:50:29 -070053#include <linux/prefetch.h>
Paul E. McKenney3d3b7db2012-01-23 17:05:46 -080054#include <linux/delay.h>
55#include <linux/stop_machine.h>
Paul E. McKenney661a85d2012-07-07 05:57:03 -070056#include <linux/random.h>
Steven Rostedt (Red Hat)af658dc2015-04-29 14:36:05 -040057#include <linux/trace_events.h>
Borislav Petkovd1d74d12013-04-22 00:12:42 +020058#include <linux/suspend.h>
Paul E. McKenney64db4cf2008-12-18 21:55:32 +010059
Paul E. McKenney4102ada2013-10-08 20:23:47 -070060#include "tree.h"
Paul E. McKenney29c00b42011-06-17 15:53:19 -070061#include "rcu.h"
Paul E. McKenney9f77da92009-08-22 13:56:45 -070062
Paul E. McKenney4102ada2013-10-08 20:23:47 -070063MODULE_ALIAS("rcutree");
64#ifdef MODULE_PARAM_PREFIX
65#undef MODULE_PARAM_PREFIX
66#endif
67#define MODULE_PARAM_PREFIX "rcutree."
68
Paul E. McKenney64db4cf2008-12-18 21:55:32 +010069/* Data structures. */
70
Paul E. McKenneyf885b7f2012-04-23 15:52:53 -070071static struct lock_class_key rcu_node_class[RCU_NUM_LVLS];
Paul E. McKenney394f2762012-06-26 17:00:35 -070072static struct lock_class_key rcu_fqs_class[RCU_NUM_LVLS];
Paul E. McKenney385b73c2015-06-24 14:20:08 -070073static struct lock_class_key rcu_exp_class[RCU_NUM_LVLS];
Peter Zijlstra88b91c72009-10-26 10:24:31 -070074
Steven Rostedt (Red Hat)f7f7bac2013-07-12 17:18:47 -040075/*
76 * In order to export the rcu_state name to the tracing tools, it
77 * needs to be added in the __tracepoint_string section.
78 * This requires defining a separate variable tp_<sname>_varname
79 * that points to the string being used, and this will allow
80 * the tracing userspace tools to be able to decipher the string
81 * address to the matching string.
82 */
Ard Biesheuvela8a29b32014-07-12 19:01:49 +020083#ifdef CONFIG_TRACING
84# define DEFINE_RCU_TPS(sname) \
Steven Rostedt (Red Hat)f7f7bac2013-07-12 17:18:47 -040085static char sname##_varname[] = #sname; \
Ard Biesheuvela8a29b32014-07-12 19:01:49 +020086static const char *tp_##sname##_varname __used __tracepoint_string = sname##_varname;
87# define RCU_STATE_NAME(sname) sname##_varname
88#else
89# define DEFINE_RCU_TPS(sname)
90# define RCU_STATE_NAME(sname) __stringify(sname)
91#endif
92
93#define RCU_STATE_INITIALIZER(sname, sabbr, cr) \
94DEFINE_RCU_TPS(sname) \
Nicolas Ioossc92fb052015-05-05 21:57:06 +080095static DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_data, sname##_data); \
Steven Rostedt (Red Hat)a41bfeb2013-07-12 17:00:28 -040096struct rcu_state sname##_state = { \
Paul E. McKenney6c90cc72012-05-28 19:41:41 -070097 .level = { &sname##_state.node[0] }, \
Paul E. McKenney27232492015-01-20 22:44:13 -080098 .rda = &sname##_data, \
Paul E. McKenney037b64e2012-05-28 23:26:01 -070099 .call = cr, \
Petr Mladek77f81fe2015-09-09 12:09:49 -0700100 .gp_state = RCU_GP_IDLE, \
Paul E. McKenney42c35332012-09-28 10:49:58 -0700101 .gpnum = 0UL - 300UL, \
102 .completed = 0UL - 300UL, \
Paul E. McKenney7b2e6012012-10-08 10:54:03 -0700103 .orphan_lock = __RAW_SPIN_LOCK_UNLOCKED(&sname##_state.orphan_lock), \
Paul E. McKenney6c90cc72012-05-28 19:41:41 -0700104 .orphan_nxttail = &sname##_state.orphan_nxtlist, \
105 .orphan_donetail = &sname##_state.orphan_donelist, \
Paul E. McKenney7be7f0b2012-05-29 05:18:53 -0700106 .barrier_mutex = __MUTEX_INITIALIZER(sname##_state.barrier_mutex), \
Ard Biesheuvela8a29b32014-07-12 19:01:49 +0200107 .name = RCU_STATE_NAME(sname), \
Paul E. McKenneya4889852012-12-03 08:16:28 -0800108 .abbr = sabbr, \
Paul E. McKenney27232492015-01-20 22:44:13 -0800109}
Paul E. McKenney64db4cf2008-12-18 21:55:32 +0100110
Steven Rostedt (Red Hat)a41bfeb2013-07-12 17:00:28 -0400111RCU_STATE_INITIALIZER(rcu_sched, 's', call_rcu_sched);
112RCU_STATE_INITIALIZER(rcu_bh, 'b', call_rcu_bh);
Ingo Molnarb1f77b02009-03-13 03:20:49 +0100113
Paul E. McKenneyb28a7c02015-03-04 07:39:27 -0800114static struct rcu_state *const rcu_state_p;
Paul E. McKenney2927a682015-03-04 07:53:04 -0800115static struct rcu_data __percpu *const rcu_data_p;
Paul E. McKenney6ce75a22012-06-12 11:01:13 -0700116LIST_HEAD(rcu_struct_flavors);
Paul E. McKenney27f4d282011-02-07 12:47:15 -0800117
Paul E. McKenneya3dc2942015-04-20 11:40:50 -0700118/* Dump rcu_node combining tree at boot to verify correct setup. */
119static bool dump_tree;
120module_param(dump_tree, bool, 0444);
Paul E. McKenney7fa27002015-04-20 10:27:15 -0700121/* Control rcu_node-tree auto-balancing at boot time. */
122static bool rcu_fanout_exact;
123module_param(rcu_fanout_exact, bool, 0444);
Paul E. McKenney47d631a2015-04-21 09:12:13 -0700124/* Increase (but not decrease) the RCU_FANOUT_LEAF at boot time. */
125static int rcu_fanout_leaf = RCU_FANOUT_LEAF;
Paul E. McKenney7e5c2df2012-07-01 15:42:33 -0700126module_param(rcu_fanout_leaf, int, 0444);
Paul E. McKenneyf885b7f2012-04-23 15:52:53 -0700127int rcu_num_lvls __read_mostly = RCU_NUM_LVLS;
Alexander Gordeevcb007102015-06-03 08:18:30 +0200128/* Number of rcu_nodes at specified level. */
129static int num_rcu_lvl[] = NUM_RCU_LVL_INIT;
Paul E. McKenneyf885b7f2012-04-23 15:52:53 -0700130int rcu_num_nodes __read_mostly = NUM_RCU_NODES; /* Total # rcu_nodes in use. */
131
Paul E. McKenneyb0d30412011-07-10 15:57:35 -0700132/*
133 * The rcu_scheduler_active variable transitions from zero to one just
134 * before the first task is spawned. So when this variable is zero, RCU
135 * can assume that there is but one task, allowing RCU to (for example)
Cody P Schaferb44f6652013-01-04 12:59:40 -0500136 * optimize synchronize_sched() to a simple barrier(). When this variable
Paul E. McKenneyb0d30412011-07-10 15:57:35 -0700137 * is one, RCU must actually do all the hard work required to detect real
138 * grace periods. This variable is also used to suppress boot-time false
139 * positives from lockdep-RCU error checking.
140 */
Paul E. McKenneybbad9372010-04-02 16:17:17 -0700141int rcu_scheduler_active __read_mostly;
142EXPORT_SYMBOL_GPL(rcu_scheduler_active);
143
Paul E. McKenneyb0d30412011-07-10 15:57:35 -0700144/*
145 * The rcu_scheduler_fully_active variable transitions from zero to one
146 * during the early_initcall() processing, which is after the scheduler
147 * is capable of creating new tasks. So RCU processing (for example,
148 * creating tasks for RCU priority boosting) must be delayed until after
149 * rcu_scheduler_fully_active transitions from zero to one. We also
150 * currently delay invocation of any RCU callbacks until after this point.
151 *
152 * It might later prove better for people registering RCU callbacks during
153 * early boot to take responsibility for these callbacks, but one step at
154 * a time.
155 */
156static int rcu_scheduler_fully_active __read_mostly;
157
Paul E. McKenney0aa04b02015-01-23 21:52:37 -0800158static void rcu_init_new_rnp(struct rcu_node *rnp_leaf);
159static void rcu_cleanup_dead_rnp(struct rcu_node *rnp_leaf);
Thomas Gleixner5d01bbd2012-07-16 10:42:35 +0000160static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu);
Paul E. McKenneya46e0892011-06-15 15:47:09 -0700161static void invoke_rcu_core(void);
162static void invoke_rcu_callbacks(struct rcu_state *rsp, struct rcu_data *rdp);
Paul E. McKenney6587a232015-08-06 16:50:39 -0700163static void rcu_report_exp_rdp(struct rcu_state *rsp,
164 struct rcu_data *rdp, bool wake);
Paul E. McKenneya26ac242011-01-12 14:10:23 -0800165
Paul E. McKenneya94844b2014-12-12 07:37:48 -0800166/* rcuc/rcub kthread realtime priority */
Paul E. McKenney26730f52015-04-21 09:22:14 -0700167#ifdef CONFIG_RCU_KTHREAD_PRIO
Paul E. McKenneya94844b2014-12-12 07:37:48 -0800168static int kthread_prio = CONFIG_RCU_KTHREAD_PRIO;
Paul E. McKenney26730f52015-04-21 09:22:14 -0700169#else /* #ifdef CONFIG_RCU_KTHREAD_PRIO */
170static int kthread_prio = IS_ENABLED(CONFIG_RCU_BOOST) ? 1 : 0;
171#endif /* #else #ifdef CONFIG_RCU_KTHREAD_PRIO */
Paul E. McKenneya94844b2014-12-12 07:37:48 -0800172module_param(kthread_prio, int, 0644);
173
Paul E. McKenney8d7dc922015-04-14 19:33:59 -0700174/* Delay in jiffies for grace-period initialization delays, debug only. */
Paul E. McKenney0f41c0d2015-03-10 18:33:20 -0700175
176#ifdef CONFIG_RCU_TORTURE_TEST_SLOW_PREINIT
177static int gp_preinit_delay = CONFIG_RCU_TORTURE_TEST_SLOW_PREINIT_DELAY;
178module_param(gp_preinit_delay, int, 0644);
179#else /* #ifdef CONFIG_RCU_TORTURE_TEST_SLOW_PREINIT */
180static const int gp_preinit_delay;
181#endif /* #else #ifdef CONFIG_RCU_TORTURE_TEST_SLOW_PREINIT */
182
Paul E. McKenney8d7dc922015-04-14 19:33:59 -0700183#ifdef CONFIG_RCU_TORTURE_TEST_SLOW_INIT
184static int gp_init_delay = CONFIG_RCU_TORTURE_TEST_SLOW_INIT_DELAY;
Paul E. McKenney37745d22015-01-22 18:24:08 -0800185module_param(gp_init_delay, int, 0644);
Paul E. McKenney8d7dc922015-04-14 19:33:59 -0700186#else /* #ifdef CONFIG_RCU_TORTURE_TEST_SLOW_INIT */
187static const int gp_init_delay;
188#endif /* #else #ifdef CONFIG_RCU_TORTURE_TEST_SLOW_INIT */
Paul E. McKenneyeab128e2015-04-15 12:08:22 -0700189
Paul E. McKenney0f41c0d2015-03-10 18:33:20 -0700190#ifdef CONFIG_RCU_TORTURE_TEST_SLOW_CLEANUP
191static int gp_cleanup_delay = CONFIG_RCU_TORTURE_TEST_SLOW_CLEANUP_DELAY;
192module_param(gp_cleanup_delay, int, 0644);
193#else /* #ifdef CONFIG_RCU_TORTURE_TEST_SLOW_CLEANUP */
194static const int gp_cleanup_delay;
195#endif /* #else #ifdef CONFIG_RCU_TORTURE_TEST_SLOW_CLEANUP */
196
Paul E. McKenneyeab128e2015-04-15 12:08:22 -0700197/*
198 * Number of grace periods between delays, normalized by the duration of
199 * the delay. The longer the the delay, the more the grace periods between
200 * each delay. The reason for this normalization is that it means that,
201 * for non-zero delays, the overall slowdown of grace periods is constant
202 * regardless of the duration of the delay. This arrangement balances
203 * the need for long delays to increase some race probabilities with the
204 * need for fast grace periods to increase other race probabilities.
205 */
206#define PER_RCU_NODE_PERIOD 3 /* Number of grace periods between delays. */
Paul E. McKenney37745d22015-01-22 18:24:08 -0800207
Paul E. McKenneya26ac242011-01-12 14:10:23 -0800208/*
Paul E. McKenney4a298652011-04-03 21:33:51 -0700209 * Track the rcutorture test sequence number and the update version
210 * number within a given test. The rcutorture_testseq is incremented
211 * on every rcutorture module load and unload, so has an odd value
212 * when a test is running. The rcutorture_vernum is set to zero
213 * when rcutorture starts and is incremented on each rcutorture update.
214 * These variables enable correlating rcutorture output with the
215 * RCU tracing information.
216 */
217unsigned long rcutorture_testseq;
218unsigned long rcutorture_vernum;
219
220/*
Paul E. McKenney0aa04b02015-01-23 21:52:37 -0800221 * Compute the mask of online CPUs for the specified rcu_node structure.
222 * This will not be stable unless the rcu_node structure's ->lock is
223 * held, but the bit corresponding to the current CPU will be stable
224 * in most contexts.
225 */
226unsigned long rcu_rnp_online_cpus(struct rcu_node *rnp)
227{
Paul E. McKenney7d0ae802015-03-03 14:57:58 -0800228 return READ_ONCE(rnp->qsmaskinitnext);
Paul E. McKenney0aa04b02015-01-23 21:52:37 -0800229}
230
231/*
Paul E. McKenney7d0ae802015-03-03 14:57:58 -0800232 * Return true if an RCU grace period is in progress. The READ_ONCE()s
Paul E. McKenneyfc2219d42009-09-23 09:50:41 -0700233 * permit this function to be invoked without holding the root rcu_node
234 * structure's ->lock, but of course results can be subject to change.
235 */
236static int rcu_gp_in_progress(struct rcu_state *rsp)
237{
Paul E. McKenney7d0ae802015-03-03 14:57:58 -0800238 return READ_ONCE(rsp->completed) != READ_ONCE(rsp->gpnum);
Paul E. McKenneyfc2219d42009-09-23 09:50:41 -0700239}
240
241/*
Paul E. McKenneyd6714c22009-08-22 13:56:46 -0700242 * Note a quiescent state. Because we do not need to know
Ingo Molnarb1f77b02009-03-13 03:20:49 +0100243 * how many quiescent states passed, just if there was at least
Paul E. McKenneyd6714c22009-08-22 13:56:46 -0700244 * one since the start of the grace period, this just sets a flag.
Paul E. McKenneye4cc1f22011-06-27 00:17:43 -0700245 * The caller must have disabled preemption.
Ingo Molnarb1f77b02009-03-13 03:20:49 +0100246 */
Paul E. McKenney284a8c92014-08-14 16:38:46 -0700247void rcu_sched_qs(void)
Ingo Molnarb1f77b02009-03-13 03:20:49 +0100248{
Paul E. McKenney338b0f72015-09-03 00:45:02 -0700249 unsigned long flags;
250
Paul E. McKenney5b74c452015-08-06 15:16:57 -0700251 if (__this_cpu_read(rcu_sched_data.cpu_no_qs.s)) {
Paul E. McKenney284a8c92014-08-14 16:38:46 -0700252 trace_rcu_grace_period(TPS("rcu_sched"),
253 __this_cpu_read(rcu_sched_data.gpnum),
254 TPS("cpuqs"));
Paul E. McKenney5b74c452015-08-06 15:16:57 -0700255 __this_cpu_write(rcu_sched_data.cpu_no_qs.b.norm, false);
Paul E. McKenney338b0f72015-09-03 00:45:02 -0700256 if (!__this_cpu_read(rcu_sched_data.cpu_no_qs.b.exp))
257 return;
258 local_irq_save(flags);
Paul E. McKenney6587a232015-08-06 16:50:39 -0700259 if (__this_cpu_read(rcu_sched_data.cpu_no_qs.b.exp)) {
260 __this_cpu_write(rcu_sched_data.cpu_no_qs.b.exp, false);
261 rcu_report_exp_rdp(&rcu_sched_state,
262 this_cpu_ptr(&rcu_sched_data),
263 true);
264 }
Paul E. McKenney338b0f72015-09-03 00:45:02 -0700265 local_irq_restore(flags);
Paul E. McKenney284a8c92014-08-14 16:38:46 -0700266 }
Ingo Molnarb1f77b02009-03-13 03:20:49 +0100267}
268
Paul E. McKenney284a8c92014-08-14 16:38:46 -0700269void rcu_bh_qs(void)
Ingo Molnarb1f77b02009-03-13 03:20:49 +0100270{
Paul E. McKenney5b74c452015-08-06 15:16:57 -0700271 if (__this_cpu_read(rcu_bh_data.cpu_no_qs.s)) {
Paul E. McKenney284a8c92014-08-14 16:38:46 -0700272 trace_rcu_grace_period(TPS("rcu_bh"),
273 __this_cpu_read(rcu_bh_data.gpnum),
274 TPS("cpuqs"));
Paul E. McKenney5b74c452015-08-06 15:16:57 -0700275 __this_cpu_write(rcu_bh_data.cpu_no_qs.b.norm, false);
Paul E. McKenney284a8c92014-08-14 16:38:46 -0700276 }
Ingo Molnarb1f77b02009-03-13 03:20:49 +0100277}
Paul E. McKenney64db4cf2008-12-18 21:55:32 +0100278
Paul E. McKenney4a81e832014-06-20 16:49:01 -0700279static DEFINE_PER_CPU(int, rcu_sched_qs_mask);
280
281static DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = {
282 .dynticks_nesting = DYNTICK_TASK_EXIT_IDLE,
283 .dynticks = ATOMIC_INIT(1),
284#ifdef CONFIG_NO_HZ_FULL_SYSIDLE
285 .dynticks_idle_nesting = DYNTICK_TASK_NEST_VALUE,
286 .dynticks_idle = ATOMIC_INIT(1),
287#endif /* #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */
288};
289
Paul E. McKenney5cd37192014-12-13 20:32:04 -0800290DEFINE_PER_CPU_SHARED_ALIGNED(unsigned long, rcu_qs_ctr);
291EXPORT_PER_CPU_SYMBOL_GPL(rcu_qs_ctr);
292
Paul E. McKenney4a81e832014-06-20 16:49:01 -0700293/*
294 * Let the RCU core know that this CPU has gone through the scheduler,
295 * which is a quiescent state. This is called when the need for a
296 * quiescent state is urgent, so we burn an atomic operation and full
297 * memory barriers to let the RCU core know about it, regardless of what
298 * this CPU might (or might not) do in the near future.
299 *
300 * We inform the RCU core by emulating a zero-duration dyntick-idle
301 * period, which we in turn do by incrementing the ->dynticks counter
302 * by two.
303 */
304static void rcu_momentary_dyntick_idle(void)
305{
306 unsigned long flags;
307 struct rcu_data *rdp;
308 struct rcu_dynticks *rdtp;
309 int resched_mask;
310 struct rcu_state *rsp;
311
312 local_irq_save(flags);
313
314 /*
315 * Yes, we can lose flag-setting operations. This is OK, because
316 * the flag will be set again after some delay.
317 */
318 resched_mask = raw_cpu_read(rcu_sched_qs_mask);
319 raw_cpu_write(rcu_sched_qs_mask, 0);
320
321 /* Find the flavor that needs a quiescent state. */
322 for_each_rcu_flavor(rsp) {
323 rdp = raw_cpu_ptr(rsp->rda);
324 if (!(resched_mask & rsp->flavor_mask))
325 continue;
326 smp_mb(); /* rcu_sched_qs_mask before cond_resched_completed. */
Paul E. McKenney7d0ae802015-03-03 14:57:58 -0800327 if (READ_ONCE(rdp->mynode->completed) !=
328 READ_ONCE(rdp->cond_resched_completed))
Paul E. McKenney4a81e832014-06-20 16:49:01 -0700329 continue;
330
331 /*
332 * Pretend to be momentarily idle for the quiescent state.
333 * This allows the grace-period kthread to record the
334 * quiescent state, with no need for this CPU to do anything
335 * further.
336 */
337 rdtp = this_cpu_ptr(&rcu_dynticks);
338 smp_mb__before_atomic(); /* Earlier stuff before QS. */
339 atomic_add(2, &rdtp->dynticks); /* QS. */
340 smp_mb__after_atomic(); /* Later stuff after QS. */
341 break;
342 }
343 local_irq_restore(flags);
344}
345
Paul E. McKenney25502a62010-04-01 17:37:01 -0700346/*
347 * Note a context switch. This is a quiescent state for RCU-sched,
348 * and requires special handling for preemptible RCU.
Paul E. McKenneye4cc1f22011-06-27 00:17:43 -0700349 * The caller must have disabled preemption.
Paul E. McKenney25502a62010-04-01 17:37:01 -0700350 */
Paul E. McKenney38200cf2014-10-21 12:50:04 -0700351void rcu_note_context_switch(void)
Paul E. McKenney25502a62010-04-01 17:37:01 -0700352{
Boqun Fengbb73c522015-07-30 16:55:38 -0700353 barrier(); /* Avoid RCU read-side critical sections leaking down. */
Steven Rostedt (Red Hat)f7f7bac2013-07-12 17:18:47 -0400354 trace_rcu_utilization(TPS("Start context switch"));
Paul E. McKenney284a8c92014-08-14 16:38:46 -0700355 rcu_sched_qs();
Paul E. McKenney38200cf2014-10-21 12:50:04 -0700356 rcu_preempt_note_context_switch();
Paul E. McKenney4a81e832014-06-20 16:49:01 -0700357 if (unlikely(raw_cpu_read(rcu_sched_qs_mask)))
358 rcu_momentary_dyntick_idle();
Steven Rostedt (Red Hat)f7f7bac2013-07-12 17:18:47 -0400359 trace_rcu_utilization(TPS("End context switch"));
Boqun Fengbb73c522015-07-30 16:55:38 -0700360 barrier(); /* Avoid RCU read-side critical sections leaking up. */
Paul E. McKenney25502a62010-04-01 17:37:01 -0700361}
Gleb Natapov29ce8312011-05-04 16:31:03 +0300362EXPORT_SYMBOL_GPL(rcu_note_context_switch);
Paul E. McKenney25502a62010-04-01 17:37:01 -0700363
Paul E. McKenney5cd37192014-12-13 20:32:04 -0800364/*
Paul E. McKenney1925d192015-01-18 17:45:05 -0800365 * Register a quiescent state for all RCU flavors. If there is an
Paul E. McKenney5cd37192014-12-13 20:32:04 -0800366 * emergency, invoke rcu_momentary_dyntick_idle() to do a heavy-weight
367 * dyntick-idle quiescent state visible to other CPUs (but only for those
Paul E. McKenney1925d192015-01-18 17:45:05 -0800368 * RCU flavors in desperate need of a quiescent state, which will normally
Paul E. McKenney5cd37192014-12-13 20:32:04 -0800369 * be none of them). Either way, do a lightweight quiescent state for
370 * all RCU flavors.
Boqun Fengbb73c522015-07-30 16:55:38 -0700371 *
372 * The barrier() calls are redundant in the common case when this is
373 * called externally, but just in case this is called from within this
374 * file.
375 *
Paul E. McKenney5cd37192014-12-13 20:32:04 -0800376 */
377void rcu_all_qs(void)
378{
Boqun Fengbb73c522015-07-30 16:55:38 -0700379 barrier(); /* Avoid RCU read-side critical sections leaking down. */
Paul E. McKenney5cd37192014-12-13 20:32:04 -0800380 if (unlikely(raw_cpu_read(rcu_sched_qs_mask)))
381 rcu_momentary_dyntick_idle();
382 this_cpu_inc(rcu_qs_ctr);
Boqun Fengbb73c522015-07-30 16:55:38 -0700383 barrier(); /* Avoid RCU read-side critical sections leaking up. */
Paul E. McKenney5cd37192014-12-13 20:32:04 -0800384}
385EXPORT_SYMBOL_GPL(rcu_all_qs);
386
Eric Dumazet878d7432012-10-18 04:55:36 -0700387static long blimit = 10; /* Maximum callbacks per rcu_do_batch. */
388static long qhimark = 10000; /* If this many pending, ignore blimit. */
389static long qlowmark = 100; /* Once only this many pending, use blimit. */
Paul E. McKenney64db4cf2008-12-18 21:55:32 +0100390
Eric Dumazet878d7432012-10-18 04:55:36 -0700391module_param(blimit, long, 0444);
392module_param(qhimark, long, 0444);
393module_param(qlowmark, long, 0444);
Paul E. McKenney3d76c082009-09-28 07:46:32 -0700394
Paul E. McKenney026ad282013-04-03 22:14:11 -0700395static ulong jiffies_till_first_fqs = ULONG_MAX;
396static ulong jiffies_till_next_fqs = ULONG_MAX;
Paul E. McKenneyd40011f2012-06-26 20:45:57 -0700397
398module_param(jiffies_till_first_fqs, ulong, 0644);
399module_param(jiffies_till_next_fqs, ulong, 0644);
400
Paul E. McKenney4a81e832014-06-20 16:49:01 -0700401/*
402 * How long the grace period must be before we start recruiting
403 * quiescent-state help from rcu_note_context_switch().
404 */
405static ulong jiffies_till_sched_qs = HZ / 20;
406module_param(jiffies_till_sched_qs, ulong, 0644);
407
Paul E. McKenney48a76392014-03-11 13:02:16 -0700408static bool rcu_start_gp_advanced(struct rcu_state *rsp, struct rcu_node *rnp,
Paul E. McKenney910ee452012-12-31 02:24:21 -0800409 struct rcu_data *rdp);
Paul E. McKenney217af2a2013-06-21 15:39:06 -0700410static void force_qs_rnp(struct rcu_state *rsp,
411 int (*f)(struct rcu_data *rsp, bool *isidle,
412 unsigned long *maxj),
413 bool *isidle, unsigned long *maxj);
Paul E. McKenney4cdfc1752012-06-22 17:06:26 -0700414static void force_quiescent_state(struct rcu_state *rsp);
Paul E. McKenneye3950ec2014-10-21 08:03:57 -0700415static int rcu_pending(void);
Paul E. McKenney64db4cf2008-12-18 21:55:32 +0100416
417/*
Paul E. McKenney917963d2014-11-21 17:10:16 -0800418 * Return the number of RCU batches started thus far for debug & stats.
Paul E. McKenney64db4cf2008-12-18 21:55:32 +0100419 */
Paul E. McKenney917963d2014-11-21 17:10:16 -0800420unsigned long rcu_batches_started(void)
421{
422 return rcu_state_p->gpnum;
423}
424EXPORT_SYMBOL_GPL(rcu_batches_started);
425
426/*
427 * Return the number of RCU-sched batches started thus far for debug & stats.
428 */
429unsigned long rcu_batches_started_sched(void)
430{
431 return rcu_sched_state.gpnum;
432}
433EXPORT_SYMBOL_GPL(rcu_batches_started_sched);
434
435/*
436 * Return the number of RCU BH batches started thus far for debug & stats.
437 */
438unsigned long rcu_batches_started_bh(void)
439{
440 return rcu_bh_state.gpnum;
441}
442EXPORT_SYMBOL_GPL(rcu_batches_started_bh);
443
444/*
445 * Return the number of RCU batches completed thus far for debug & stats.
446 */
447unsigned long rcu_batches_completed(void)
448{
449 return rcu_state_p->completed;
450}
451EXPORT_SYMBOL_GPL(rcu_batches_completed);
452
453/*
454 * Return the number of RCU-sched batches completed thus far for debug & stats.
Paul E. McKenney64db4cf2008-12-18 21:55:32 +0100455 */
Paul E. McKenney9733e4f2014-11-21 12:49:13 -0800456unsigned long rcu_batches_completed_sched(void)
Paul E. McKenney64db4cf2008-12-18 21:55:32 +0100457{
Paul E. McKenneyd6714c22009-08-22 13:56:46 -0700458 return rcu_sched_state.completed;
Paul E. McKenney64db4cf2008-12-18 21:55:32 +0100459}
Paul E. McKenneyd6714c22009-08-22 13:56:46 -0700460EXPORT_SYMBOL_GPL(rcu_batches_completed_sched);
Paul E. McKenney64db4cf2008-12-18 21:55:32 +0100461
462/*
Paul E. McKenney917963d2014-11-21 17:10:16 -0800463 * Return the number of RCU BH batches completed thus far for debug & stats.
Paul E. McKenney64db4cf2008-12-18 21:55:32 +0100464 */
Paul E. McKenney9733e4f2014-11-21 12:49:13 -0800465unsigned long rcu_batches_completed_bh(void)
Paul E. McKenney64db4cf2008-12-18 21:55:32 +0100466{
467 return rcu_bh_state.completed;
468}
469EXPORT_SYMBOL_GPL(rcu_batches_completed_bh);
470
471/*
Andreea-Cristina Bernata381d752014-03-19 11:18:31 +0200472 * Force a quiescent state.
473 */
474void rcu_force_quiescent_state(void)
475{
Uma Sharmae5341652014-03-23 22:32:09 -0700476 force_quiescent_state(rcu_state_p);
Andreea-Cristina Bernata381d752014-03-19 11:18:31 +0200477}
478EXPORT_SYMBOL_GPL(rcu_force_quiescent_state);
479
480/*
Paul E. McKenneybf66f182010-01-04 15:09:10 -0800481 * Force a quiescent state for RCU BH.
482 */
483void rcu_bh_force_quiescent_state(void)
484{
Paul E. McKenney4cdfc1752012-06-22 17:06:26 -0700485 force_quiescent_state(&rcu_bh_state);
Paul E. McKenneybf66f182010-01-04 15:09:10 -0800486}
487EXPORT_SYMBOL_GPL(rcu_bh_force_quiescent_state);
488
489/*
Paul E. McKenneye7580f32015-02-24 14:23:39 -0800490 * Force a quiescent state for RCU-sched.
491 */
492void rcu_sched_force_quiescent_state(void)
493{
494 force_quiescent_state(&rcu_sched_state);
495}
496EXPORT_SYMBOL_GPL(rcu_sched_force_quiescent_state);
497
498/*
Paul E. McKenneyafea2272014-03-12 07:10:41 -0700499 * Show the state of the grace-period kthreads.
500 */
501void show_rcu_gp_kthreads(void)
502{
503 struct rcu_state *rsp;
504
505 for_each_rcu_flavor(rsp) {
506 pr_info("%s: wait state: %d ->state: %#lx\n",
507 rsp->name, rsp->gp_state, rsp->gp_kthread->state);
508 /* sched_show_task(rsp->gp_kthread); */
509 }
510}
511EXPORT_SYMBOL_GPL(show_rcu_gp_kthreads);
512
513/*
Paul E. McKenney4a298652011-04-03 21:33:51 -0700514 * Record the number of times rcutorture tests have been initiated and
515 * terminated. This information allows the debugfs tracing stats to be
516 * correlated to the rcutorture messages, even when the rcutorture module
517 * is being repeatedly loaded and unloaded. In other words, we cannot
518 * store this state in rcutorture itself.
519 */
520void rcutorture_record_test_transition(void)
521{
522 rcutorture_testseq++;
523 rcutorture_vernum = 0;
524}
525EXPORT_SYMBOL_GPL(rcutorture_record_test_transition);
526
527/*
Paul E. McKenneyad0dc7f2014-02-19 10:51:42 -0800528 * Send along grace-period-related data for rcutorture diagnostics.
529 */
530void rcutorture_get_gp_data(enum rcutorture_type test_type, int *flags,
531 unsigned long *gpnum, unsigned long *completed)
532{
533 struct rcu_state *rsp = NULL;
534
535 switch (test_type) {
536 case RCU_FLAVOR:
Uma Sharmae5341652014-03-23 22:32:09 -0700537 rsp = rcu_state_p;
Paul E. McKenneyad0dc7f2014-02-19 10:51:42 -0800538 break;
539 case RCU_BH_FLAVOR:
540 rsp = &rcu_bh_state;
541 break;
542 case RCU_SCHED_FLAVOR:
543 rsp = &rcu_sched_state;
544 break;
545 default:
546 break;
547 }
548 if (rsp != NULL) {
Paul E. McKenney7d0ae802015-03-03 14:57:58 -0800549 *flags = READ_ONCE(rsp->gp_flags);
550 *gpnum = READ_ONCE(rsp->gpnum);
551 *completed = READ_ONCE(rsp->completed);
Paul E. McKenneyad0dc7f2014-02-19 10:51:42 -0800552 return;
553 }
554 *flags = 0;
555 *gpnum = 0;
556 *completed = 0;
557}
558EXPORT_SYMBOL_GPL(rcutorture_get_gp_data);
559
560/*
Paul E. McKenney4a298652011-04-03 21:33:51 -0700561 * Record the number of writer passes through the current rcutorture test.
562 * This is also used to correlate debugfs tracing stats with the rcutorture
563 * messages.
564 */
565void rcutorture_record_progress(unsigned long vernum)
566{
567 rcutorture_vernum++;
568}
569EXPORT_SYMBOL_GPL(rcutorture_record_progress);
570
571/*
Paul E. McKenney64db4cf2008-12-18 21:55:32 +0100572 * Does the CPU have callbacks ready to be invoked?
573 */
574static int
575cpu_has_callbacks_ready_to_invoke(struct rcu_data *rdp)
576{
Paul E. McKenney3fbfbf72012-08-19 21:35:53 -0700577 return &rdp->nxtlist != rdp->nxttail[RCU_DONE_TAIL] &&
578 rdp->nxttail[RCU_DONE_TAIL] != NULL;
Paul E. McKenney64db4cf2008-12-18 21:55:32 +0100579}
580
581/*
Paul E. McKenney365187f2014-03-10 10:55:52 -0700582 * Return the root node of the specified rcu_state structure.
583 */
584static struct rcu_node *rcu_get_root(struct rcu_state *rsp)
585{
586 return &rsp->node[0];
587}
588
589/*
590 * Is there any need for future grace periods?
591 * Interrupts must be disabled. If the caller does not hold the root
592 * rnp_node structure's ->lock, the results are advisory only.
593 */
594static int rcu_future_needs_gp(struct rcu_state *rsp)
595{
596 struct rcu_node *rnp = rcu_get_root(rsp);
Paul E. McKenney7d0ae802015-03-03 14:57:58 -0800597 int idx = (READ_ONCE(rnp->completed) + 1) & 0x1;
Paul E. McKenney365187f2014-03-10 10:55:52 -0700598 int *fp = &rnp->need_future_gp[idx];
599
Paul E. McKenney7d0ae802015-03-03 14:57:58 -0800600 return READ_ONCE(*fp);
Paul E. McKenney365187f2014-03-10 10:55:52 -0700601}
602
603/*
Paul E. McKenneydc35c892012-12-03 13:52:00 -0800604 * Does the current CPU require a not-yet-started grace period?
605 * The caller must have disabled interrupts to prevent races with
606 * normal callback registry.
Paul E. McKenney64db4cf2008-12-18 21:55:32 +0100607 */
608static int
609cpu_needs_another_gp(struct rcu_state *rsp, struct rcu_data *rdp)
610{
Paul E. McKenneydc35c892012-12-03 13:52:00 -0800611 int i;
Paul E. McKenney3fbfbf72012-08-19 21:35:53 -0700612
Paul E. McKenneydc35c892012-12-03 13:52:00 -0800613 if (rcu_gp_in_progress(rsp))
614 return 0; /* No, a grace period is already in progress. */
Paul E. McKenney365187f2014-03-10 10:55:52 -0700615 if (rcu_future_needs_gp(rsp))
Paul E. McKenney34ed62462013-01-07 13:37:42 -0800616 return 1; /* Yes, a no-CBs CPU needs one. */
Paul E. McKenneydc35c892012-12-03 13:52:00 -0800617 if (!rdp->nxttail[RCU_NEXT_TAIL])
618 return 0; /* No, this is a no-CBs (or offline) CPU. */
619 if (*rdp->nxttail[RCU_NEXT_READY_TAIL])
620 return 1; /* Yes, this CPU has newly registered callbacks. */
621 for (i = RCU_WAIT_TAIL; i < RCU_NEXT_TAIL; i++)
622 if (rdp->nxttail[i - 1] != rdp->nxttail[i] &&
Paul E. McKenney7d0ae802015-03-03 14:57:58 -0800623 ULONG_CMP_LT(READ_ONCE(rsp->completed),
Paul E. McKenneydc35c892012-12-03 13:52:00 -0800624 rdp->nxtcompleted[i]))
625 return 1; /* Yes, CBs for future grace period. */
626 return 0; /* No grace period needed. */
Paul E. McKenney64db4cf2008-12-18 21:55:32 +0100627}
628
629/*
Frederic Weisbeckeradf50912012-06-28 11:20:21 -0700630 * rcu_eqs_enter_common - current CPU is moving towards extended quiescent state
Paul E. McKenney64db4cf2008-12-18 21:55:32 +0100631 *
Paul E. McKenney9b2e4f12011-09-30 12:10:22 -0700632 * If the new value of the ->dynticks_nesting counter now is zero,
633 * we really have entered idle, and must do the appropriate accounting.
634 * The caller must have disabled interrupts.
Paul E. McKenney64db4cf2008-12-18 21:55:32 +0100635 */
Christoph Lameter28ced792014-09-02 14:13:44 -0700636static void rcu_eqs_enter_common(long long oldval, bool user)
Paul E. McKenney64db4cf2008-12-18 21:55:32 +0100637{
Paul E. McKenney96d3fd02013-10-04 14:33:34 -0700638 struct rcu_state *rsp;
639 struct rcu_data *rdp;
Christoph Lameter28ced792014-09-02 14:13:44 -0700640 struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
Paul E. McKenney96d3fd02013-10-04 14:33:34 -0700641
Steven Rostedt (Red Hat)f7f7bac2013-07-12 17:18:47 -0400642 trace_rcu_dyntick(TPS("Start"), oldval, rdtp->dynticks_nesting);
Paul E. McKenney1ce46ee2015-05-05 23:04:22 -0700643 if (IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
644 !user && !is_idle_task(current)) {
Paul E. McKenney289828e2013-08-31 19:23:29 -0700645 struct task_struct *idle __maybe_unused =
646 idle_task(smp_processor_id());
Paul E. McKenney0989cb42011-11-01 08:57:21 -0700647
Steven Rostedt (Red Hat)f7f7bac2013-07-12 17:18:47 -0400648 trace_rcu_dyntick(TPS("Error on entry: not idle task"), oldval, 0);
Paul E. McKenneybf1304e2012-05-09 15:55:39 -0700649 ftrace_dump(DUMP_ORIG);
Paul E. McKenney0989cb42011-11-01 08:57:21 -0700650 WARN_ONCE(1, "Current pid: %d comm: %s / Idle pid: %d comm: %s",
651 current->pid, current->comm,
652 idle->pid, idle->comm); /* must be idle task! */
Paul E. McKenney9b2e4f12011-09-30 12:10:22 -0700653 }
Paul E. McKenney96d3fd02013-10-04 14:33:34 -0700654 for_each_rcu_flavor(rsp) {
655 rdp = this_cpu_ptr(rsp->rda);
656 do_nocb_deferred_wakeup(rdp);
657 }
Paul E. McKenney198bbf82014-10-22 15:03:43 -0700658 rcu_prepare_for_idle();
Paul E. McKenney23b5c8f2010-09-07 10:38:22 -0700659 /* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */
Peter Zijlstra4e857c52014-03-17 18:06:10 +0100660 smp_mb__before_atomic(); /* See above. */
Paul E. McKenney23b5c8f2010-09-07 10:38:22 -0700661 atomic_inc(&rdtp->dynticks);
Peter Zijlstra4e857c52014-03-17 18:06:10 +0100662 smp_mb__after_atomic(); /* Force ordering with next sojourn. */
Paul E. McKenney1ce46ee2015-05-05 23:04:22 -0700663 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
664 atomic_read(&rdtp->dynticks) & 0x1);
Paul E. McKenney176f8f72014-08-04 17:43:50 -0700665 rcu_dynticks_task_enter();
Paul E. McKenneyc44e2cd2012-01-12 13:08:18 -0800666
667 /*
Frederic Weisbeckeradf50912012-06-28 11:20:21 -0700668 * It is illegal to enter an extended quiescent state while
Paul E. McKenneyc44e2cd2012-01-12 13:08:18 -0800669 * in an RCU read-side critical section.
670 */
Paul E. McKenneyf78f5b92015-06-18 15:50:02 -0700671 RCU_LOCKDEP_WARN(lock_is_held(&rcu_lock_map),
672 "Illegal idle entry in RCU read-side critical section.");
673 RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map),
674 "Illegal idle entry in RCU-bh read-side critical section.");
675 RCU_LOCKDEP_WARN(lock_is_held(&rcu_sched_lock_map),
676 "Illegal idle entry in RCU-sched read-side critical section.");
Paul E. McKenney64db4cf2008-12-18 21:55:32 +0100677}
678
Frederic Weisbeckeradf50912012-06-28 11:20:21 -0700679/*
680 * Enter an RCU extended quiescent state, which can be either the
681 * idle loop or adaptive-tickless usermode execution.
682 */
683static void rcu_eqs_enter(bool user)
684{
Frederic Weisbeckeradf50912012-06-28 11:20:21 -0700685 long long oldval;
686 struct rcu_dynticks *rdtp;
687
Christoph Lameterc9d4b0a2013-08-31 13:34:10 -0700688 rdtp = this_cpu_ptr(&rcu_dynticks);
Frederic Weisbeckeradf50912012-06-28 11:20:21 -0700689 oldval = rdtp->dynticks_nesting;
Paul E. McKenney1ce46ee2015-05-05 23:04:22 -0700690 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
691 (oldval & DYNTICK_TASK_NEST_MASK) == 0);
Paul E. McKenney3a5924052013-10-04 18:48:55 -0700692 if ((oldval & DYNTICK_TASK_NEST_MASK) == DYNTICK_TASK_NEST_VALUE) {
Frederic Weisbeckeradf50912012-06-28 11:20:21 -0700693 rdtp->dynticks_nesting = 0;
Christoph Lameter28ced792014-09-02 14:13:44 -0700694 rcu_eqs_enter_common(oldval, user);
Paul E. McKenney3a5924052013-10-04 18:48:55 -0700695 } else {
Frederic Weisbeckeradf50912012-06-28 11:20:21 -0700696 rdtp->dynticks_nesting -= DYNTICK_TASK_NEST_VALUE;
Paul E. McKenney3a5924052013-10-04 18:48:55 -0700697 }
Frederic Weisbeckeradf50912012-06-28 11:20:21 -0700698}
699
Paul E. McKenney9b2e4f12011-09-30 12:10:22 -0700700/**
701 * rcu_idle_enter - inform RCU that current CPU is entering idle
Paul E. McKenney64db4cf2008-12-18 21:55:32 +0100702 *
Paul E. McKenney9b2e4f12011-09-30 12:10:22 -0700703 * Enter idle mode, in other words, -leave- the mode in which RCU
704 * read-side critical sections can occur. (Though RCU read-side
705 * critical sections can occur in irq handlers in idle, a possibility
706 * handled by irq_enter() and irq_exit().)
707 *
708 * We crowbar the ->dynticks_nesting field to zero to allow for
709 * the possibility of usermode upcalls having messed up our count
710 * of interrupt nesting level during the prior busy period.
Paul E. McKenney64db4cf2008-12-18 21:55:32 +0100711 */
Paul E. McKenney9b2e4f12011-09-30 12:10:22 -0700712void rcu_idle_enter(void)
Paul E. McKenney64db4cf2008-12-18 21:55:32 +0100713{
Frederic Weisbeckerc5d900b2012-07-11 20:26:31 +0200714 unsigned long flags;
715
716 local_irq_save(flags);
Paul E. McKenneycb349ca2012-09-04 17:35:31 -0700717 rcu_eqs_enter(false);
Christoph Lameter28ced792014-09-02 14:13:44 -0700718 rcu_sysidle_enter(0);
Frederic Weisbeckerc5d900b2012-07-11 20:26:31 +0200719 local_irq_restore(flags);
Paul E. McKenney9b2e4f12011-09-30 12:10:22 -0700720}
Paul E. McKenney8a2ecf42012-02-02 15:42:04 -0800721EXPORT_SYMBOL_GPL(rcu_idle_enter);
Paul E. McKenney9b2e4f12011-09-30 12:10:22 -0700722
Paul E. McKenneyd1ec4c32015-05-13 10:41:58 -0700723#ifdef CONFIG_NO_HZ_FULL
Paul E. McKenney9b2e4f12011-09-30 12:10:22 -0700724/**
Frederic Weisbeckeradf50912012-06-28 11:20:21 -0700725 * rcu_user_enter - inform RCU that we are resuming userspace.
726 *
727 * Enter RCU idle mode right before resuming userspace. No use of RCU
728 * is permitted between this call and rcu_user_exit(). This way the
729 * CPU doesn't need to maintain the tick for RCU maintenance purposes
730 * when the CPU runs in userspace.
731 */
732void rcu_user_enter(void)
733{
Frederic Weisbecker91d1aa432012-11-27 19:33:25 +0100734 rcu_eqs_enter(1);
Frederic Weisbeckeradf50912012-06-28 11:20:21 -0700735}
Paul E. McKenneyd1ec4c32015-05-13 10:41:58 -0700736#endif /* CONFIG_NO_HZ_FULL */
Frederic Weisbecker19dd15912012-06-04 16:42:35 -0700737
738/**
Paul E. McKenney9b2e4f12011-09-30 12:10:22 -0700739 * rcu_irq_exit - inform RCU that current CPU is exiting irq towards idle
740 *
741 * Exit from an interrupt handler, which might possibly result in entering
742 * idle mode, in other words, leaving the mode in which read-side critical
743 * sections can occur.
744 *
745 * This code assumes that the idle loop never does anything that might
746 * result in unbalanced calls to irq_enter() and irq_exit(). If your
747 * architecture violates this assumption, RCU will give you what you
748 * deserve, good and hard. But very infrequently and irreproducibly.
749 *
750 * Use things like work queues to work around this limitation.
751 *
752 * You have been warned.
753 */
754void rcu_irq_exit(void)
755{
756 unsigned long flags;
Paul E. McKenney4145fa72011-10-31 15:01:54 -0700757 long long oldval;
Paul E. McKenney9b2e4f12011-09-30 12:10:22 -0700758 struct rcu_dynticks *rdtp;
759
760 local_irq_save(flags);
Christoph Lameterc9d4b0a2013-08-31 13:34:10 -0700761 rdtp = this_cpu_ptr(&rcu_dynticks);
Paul E. McKenney4145fa72011-10-31 15:01:54 -0700762 oldval = rdtp->dynticks_nesting;
Paul E. McKenney9b2e4f12011-09-30 12:10:22 -0700763 rdtp->dynticks_nesting--;
Paul E. McKenney1ce46ee2015-05-05 23:04:22 -0700764 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
765 rdtp->dynticks_nesting < 0);
Frederic Weisbeckerb6fc6022011-11-28 16:18:56 -0800766 if (rdtp->dynticks_nesting)
Steven Rostedt (Red Hat)f7f7bac2013-07-12 17:18:47 -0400767 trace_rcu_dyntick(TPS("--="), oldval, rdtp->dynticks_nesting);
Frederic Weisbeckerb6fc6022011-11-28 16:18:56 -0800768 else
Christoph Lameter28ced792014-09-02 14:13:44 -0700769 rcu_eqs_enter_common(oldval, true);
770 rcu_sysidle_enter(1);
Paul E. McKenney9b2e4f12011-09-30 12:10:22 -0700771 local_irq_restore(flags);
772}
773
774/*
Frederic Weisbeckeradf50912012-06-28 11:20:21 -0700775 * rcu_eqs_exit_common - current CPU moving away from extended quiescent state
Paul E. McKenney9b2e4f12011-09-30 12:10:22 -0700776 *
777 * If the new value of the ->dynticks_nesting counter was previously zero,
778 * we really have exited idle, and must do the appropriate accounting.
779 * The caller must have disabled interrupts.
780 */
Christoph Lameter28ced792014-09-02 14:13:44 -0700781static void rcu_eqs_exit_common(long long oldval, int user)
Paul E. McKenney9b2e4f12011-09-30 12:10:22 -0700782{
Christoph Lameter28ced792014-09-02 14:13:44 -0700783 struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
784
Paul E. McKenney176f8f72014-08-04 17:43:50 -0700785 rcu_dynticks_task_exit();
Peter Zijlstra4e857c52014-03-17 18:06:10 +0100786 smp_mb__before_atomic(); /* Force ordering w/previous sojourn. */
Paul E. McKenney23b5c8f2010-09-07 10:38:22 -0700787 atomic_inc(&rdtp->dynticks);
788 /* CPUs seeing atomic_inc() must see later RCU read-side crit sects */
Peter Zijlstra4e857c52014-03-17 18:06:10 +0100789 smp_mb__after_atomic(); /* See above. */
Paul E. McKenney1ce46ee2015-05-05 23:04:22 -0700790 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
791 !(atomic_read(&rdtp->dynticks) & 0x1));
Paul E. McKenney8fa78452014-10-22 15:07:37 -0700792 rcu_cleanup_after_idle();
Steven Rostedt (Red Hat)f7f7bac2013-07-12 17:18:47 -0400793 trace_rcu_dyntick(TPS("End"), oldval, rdtp->dynticks_nesting);
Paul E. McKenney1ce46ee2015-05-05 23:04:22 -0700794 if (IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
795 !user && !is_idle_task(current)) {
Paul E. McKenney289828e2013-08-31 19:23:29 -0700796 struct task_struct *idle __maybe_unused =
797 idle_task(smp_processor_id());
Paul E. McKenney0989cb42011-11-01 08:57:21 -0700798
Steven Rostedt (Red Hat)f7f7bac2013-07-12 17:18:47 -0400799 trace_rcu_dyntick(TPS("Error on exit: not idle task"),
Paul E. McKenney4145fa72011-10-31 15:01:54 -0700800 oldval, rdtp->dynticks_nesting);
Paul E. McKenneybf1304e2012-05-09 15:55:39 -0700801 ftrace_dump(DUMP_ORIG);
Paul E. McKenney0989cb42011-11-01 08:57:21 -0700802 WARN_ONCE(1, "Current pid: %d comm: %s / Idle pid: %d comm: %s",
803 current->pid, current->comm,
804 idle->pid, idle->comm); /* must be idle task! */
Paul E. McKenney9b2e4f12011-09-30 12:10:22 -0700805 }
806}
807
Frederic Weisbeckeradf50912012-06-28 11:20:21 -0700808/*
809 * Exit an RCU extended quiescent state, which can be either the
810 * idle loop or adaptive-tickless usermode execution.
811 */
812static void rcu_eqs_exit(bool user)
813{
Frederic Weisbeckeradf50912012-06-28 11:20:21 -0700814 struct rcu_dynticks *rdtp;
815 long long oldval;
816
Christoph Lameterc9d4b0a2013-08-31 13:34:10 -0700817 rdtp = this_cpu_ptr(&rcu_dynticks);
Frederic Weisbeckeradf50912012-06-28 11:20:21 -0700818 oldval = rdtp->dynticks_nesting;
Paul E. McKenney1ce46ee2015-05-05 23:04:22 -0700819 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && oldval < 0);
Paul E. McKenney3a5924052013-10-04 18:48:55 -0700820 if (oldval & DYNTICK_TASK_NEST_MASK) {
Frederic Weisbeckeradf50912012-06-28 11:20:21 -0700821 rdtp->dynticks_nesting += DYNTICK_TASK_NEST_VALUE;
Paul E. McKenney3a5924052013-10-04 18:48:55 -0700822 } else {
Frederic Weisbeckeradf50912012-06-28 11:20:21 -0700823 rdtp->dynticks_nesting = DYNTICK_TASK_EXIT_IDLE;
Christoph Lameter28ced792014-09-02 14:13:44 -0700824 rcu_eqs_exit_common(oldval, user);
Paul E. McKenney3a5924052013-10-04 18:48:55 -0700825 }
Frederic Weisbeckeradf50912012-06-28 11:20:21 -0700826}
827
Paul E. McKenney9b2e4f12011-09-30 12:10:22 -0700828/**
829 * rcu_idle_exit - inform RCU that current CPU is leaving idle
830 *
831 * Exit idle mode, in other words, -enter- the mode in which RCU
832 * read-side critical sections can occur.
833 *
Paul E. McKenney29e37d82011-11-17 16:55:56 -0800834 * We crowbar the ->dynticks_nesting field to DYNTICK_TASK_NEST to
Paul E. McKenney4145fa72011-10-31 15:01:54 -0700835 * allow for the possibility of usermode upcalls messing up our count
Paul E. McKenney9b2e4f12011-09-30 12:10:22 -0700836 * of interrupt nesting level during the busy period that is just
837 * now starting.
838 */
839void rcu_idle_exit(void)
840{
Frederic Weisbeckerc5d900b2012-07-11 20:26:31 +0200841 unsigned long flags;
842
843 local_irq_save(flags);
Paul E. McKenneycb349ca2012-09-04 17:35:31 -0700844 rcu_eqs_exit(false);
Christoph Lameter28ced792014-09-02 14:13:44 -0700845 rcu_sysidle_exit(0);
Frederic Weisbeckerc5d900b2012-07-11 20:26:31 +0200846 local_irq_restore(flags);
Paul E. McKenney9b2e4f12011-09-30 12:10:22 -0700847}
Paul E. McKenney8a2ecf42012-02-02 15:42:04 -0800848EXPORT_SYMBOL_GPL(rcu_idle_exit);
Paul E. McKenney9b2e4f12011-09-30 12:10:22 -0700849
Paul E. McKenneyd1ec4c32015-05-13 10:41:58 -0700850#ifdef CONFIG_NO_HZ_FULL
Paul E. McKenney9b2e4f12011-09-30 12:10:22 -0700851/**
Frederic Weisbeckeradf50912012-06-28 11:20:21 -0700852 * rcu_user_exit - inform RCU that we are exiting userspace.
853 *
854 * Exit RCU idle mode while entering the kernel because it can
855 * run a RCU read side critical section anytime.
856 */
857void rcu_user_exit(void)
858{
Frederic Weisbecker91d1aa432012-11-27 19:33:25 +0100859 rcu_eqs_exit(1);
Frederic Weisbeckeradf50912012-06-28 11:20:21 -0700860}
Paul E. McKenneyd1ec4c32015-05-13 10:41:58 -0700861#endif /* CONFIG_NO_HZ_FULL */
Frederic Weisbecker19dd15912012-06-04 16:42:35 -0700862
863/**
Paul E. McKenney9b2e4f12011-09-30 12:10:22 -0700864 * rcu_irq_enter - inform RCU that current CPU is entering irq away from idle
865 *
866 * Enter an interrupt handler, which might possibly result in exiting
867 * idle mode, in other words, entering the mode in which read-side critical
868 * sections can occur.
869 *
870 * Note that the Linux kernel is fully capable of entering an interrupt
871 * handler that it never exits, for example when doing upcalls to
872 * user mode! This code assumes that the idle loop never does upcalls to
873 * user mode. If your architecture does do upcalls from the idle loop (or
874 * does anything else that results in unbalanced calls to the irq_enter()
875 * and irq_exit() functions), RCU will give you what you deserve, good
876 * and hard. But very infrequently and irreproducibly.
877 *
878 * Use things like work queues to work around this limitation.
879 *
880 * You have been warned.
881 */
882void rcu_irq_enter(void)
883{
884 unsigned long flags;
885 struct rcu_dynticks *rdtp;
886 long long oldval;
887
888 local_irq_save(flags);
Christoph Lameterc9d4b0a2013-08-31 13:34:10 -0700889 rdtp = this_cpu_ptr(&rcu_dynticks);
Paul E. McKenney9b2e4f12011-09-30 12:10:22 -0700890 oldval = rdtp->dynticks_nesting;
891 rdtp->dynticks_nesting++;
Paul E. McKenney1ce46ee2015-05-05 23:04:22 -0700892 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
893 rdtp->dynticks_nesting == 0);
Frederic Weisbeckerb6fc6022011-11-28 16:18:56 -0800894 if (oldval)
Steven Rostedt (Red Hat)f7f7bac2013-07-12 17:18:47 -0400895 trace_rcu_dyntick(TPS("++="), oldval, rdtp->dynticks_nesting);
Frederic Weisbeckerb6fc6022011-11-28 16:18:56 -0800896 else
Christoph Lameter28ced792014-09-02 14:13:44 -0700897 rcu_eqs_exit_common(oldval, true);
898 rcu_sysidle_exit(1);
Paul E. McKenney64db4cf2008-12-18 21:55:32 +0100899 local_irq_restore(flags);
Paul E. McKenney64db4cf2008-12-18 21:55:32 +0100900}
901
902/**
903 * rcu_nmi_enter - inform RCU of entry to NMI context
904 *
Paul E. McKenney734d1682014-11-21 14:45:12 -0800905 * If the CPU was idle from RCU's viewpoint, update rdtp->dynticks and
906 * rdtp->dynticks_nmi_nesting to let the RCU grace-period handling know
907 * that the CPU is active. This implementation permits nested NMIs, as
908 * long as the nesting level does not overflow an int. (You will probably
909 * run out of stack space first.)
Paul E. McKenney64db4cf2008-12-18 21:55:32 +0100910 */
911void rcu_nmi_enter(void)
912{
Christoph Lameterc9d4b0a2013-08-31 13:34:10 -0700913 struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
Paul E. McKenney734d1682014-11-21 14:45:12 -0800914 int incby = 2;
Paul E. McKenney64db4cf2008-12-18 21:55:32 +0100915
Paul E. McKenney734d1682014-11-21 14:45:12 -0800916 /* Complain about underflow. */
917 WARN_ON_ONCE(rdtp->dynticks_nmi_nesting < 0);
918
919 /*
920 * If idle from RCU viewpoint, atomically increment ->dynticks
921 * to mark non-idle and increment ->dynticks_nmi_nesting by one.
922 * Otherwise, increment ->dynticks_nmi_nesting by two. This means
923 * if ->dynticks_nmi_nesting is equal to one, we are guaranteed
924 * to be in the outermost NMI handler that interrupted an RCU-idle
925 * period (observation due to Andy Lutomirski).
926 */
927 if (!(atomic_read(&rdtp->dynticks) & 0x1)) {
928 smp_mb__before_atomic(); /* Force delay from prior write. */
929 atomic_inc(&rdtp->dynticks);
930 /* atomic_inc() before later RCU read-side crit sects */
931 smp_mb__after_atomic(); /* See above. */
932 WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1));
933 incby = 1;
934 }
935 rdtp->dynticks_nmi_nesting += incby;
936 barrier();
Paul E. McKenney64db4cf2008-12-18 21:55:32 +0100937}
938
939/**
940 * rcu_nmi_exit - inform RCU of exit from NMI context
941 *
Paul E. McKenney734d1682014-11-21 14:45:12 -0800942 * If we are returning from the outermost NMI handler that interrupted an
943 * RCU-idle period, update rdtp->dynticks and rdtp->dynticks_nmi_nesting
944 * to let the RCU grace-period handling know that the CPU is back to
945 * being RCU-idle.
Paul E. McKenney64db4cf2008-12-18 21:55:32 +0100946 */
947void rcu_nmi_exit(void)
948{
Christoph Lameterc9d4b0a2013-08-31 13:34:10 -0700949 struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
Paul E. McKenney64db4cf2008-12-18 21:55:32 +0100950
Paul E. McKenney734d1682014-11-21 14:45:12 -0800951 /*
952 * Check for ->dynticks_nmi_nesting underflow and bad ->dynticks.
953 * (We are exiting an NMI handler, so RCU better be paying attention
954 * to us!)
955 */
956 WARN_ON_ONCE(rdtp->dynticks_nmi_nesting <= 0);
957 WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1));
958
959 /*
960 * If the nesting level is not 1, the CPU wasn't RCU-idle, so
961 * leave it in non-RCU-idle state.
962 */
963 if (rdtp->dynticks_nmi_nesting != 1) {
964 rdtp->dynticks_nmi_nesting -= 2;
Paul E. McKenney64db4cf2008-12-18 21:55:32 +0100965 return;
Paul E. McKenney734d1682014-11-21 14:45:12 -0800966 }
967
968 /* This NMI interrupted an RCU-idle CPU, restore RCU-idleness. */
969 rdtp->dynticks_nmi_nesting = 0;
Paul E. McKenney23b5c8f2010-09-07 10:38:22 -0700970 /* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */
Peter Zijlstra4e857c52014-03-17 18:06:10 +0100971 smp_mb__before_atomic(); /* See above. */
Paul E. McKenney23b5c8f2010-09-07 10:38:22 -0700972 atomic_inc(&rdtp->dynticks);
Peter Zijlstra4e857c52014-03-17 18:06:10 +0100973 smp_mb__after_atomic(); /* Force delay to next write. */
Paul E. McKenney23b5c8f2010-09-07 10:38:22 -0700974 WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1);
Paul E. McKenney64db4cf2008-12-18 21:55:32 +0100975}
976
Paul E. McKenney64db4cf2008-12-18 21:55:32 +0100977/**
Paul E. McKenneycc6783f2013-09-06 17:39:49 -0700978 * __rcu_is_watching - are RCU read-side critical sections safe?
979 *
980 * Return true if RCU is watching the running CPU, which means that
981 * this CPU can safely enter RCU read-side critical sections. Unlike
Paul E. McKenney5c173eb2013-09-13 17:20:11 -0700982 * rcu_is_watching(), the caller of __rcu_is_watching() must have at
Paul E. McKenneycc6783f2013-09-06 17:39:49 -0700983 * least disabled preemption.
984 */
Linus Torvaldsb29c8302013-11-16 12:23:18 -0800985bool notrace __rcu_is_watching(void)
Paul E. McKenneycc6783f2013-09-06 17:39:49 -0700986{
Paul E. McKenney5c173eb2013-09-13 17:20:11 -0700987 return atomic_read(this_cpu_ptr(&rcu_dynticks.dynticks)) & 0x1;
Paul E. McKenneycc6783f2013-09-06 17:39:49 -0700988}
989
Paul E. McKenney5c173eb2013-09-13 17:20:11 -0700990/**
991 * rcu_is_watching - see if RCU thinks that the current CPU is idle
Paul E. McKenney64db4cf2008-12-18 21:55:32 +0100992 *
Paul E. McKenney9b2e4f12011-09-30 12:10:22 -0700993 * If the current CPU is in its idle loop and is neither in an interrupt
Paul E. McKenney34240692011-10-03 11:38:52 -0700994 * or NMI handler, return true.
Paul E. McKenney64db4cf2008-12-18 21:55:32 +0100995 */
Linus Torvaldsb29c8302013-11-16 12:23:18 -0800996bool notrace rcu_is_watching(void)
Paul E. McKenney64db4cf2008-12-18 21:55:32 +0100997{
Pranith Kumarf534ed12014-07-08 18:26:11 -0400998 bool ret;
Paul E. McKenney34240692011-10-03 11:38:52 -0700999
Alexei Starovoitov46f00d12015-06-16 10:35:18 -07001000 preempt_disable_notrace();
Paul E. McKenney5c173eb2013-09-13 17:20:11 -07001001 ret = __rcu_is_watching();
Alexei Starovoitov46f00d12015-06-16 10:35:18 -07001002 preempt_enable_notrace();
Paul E. McKenney34240692011-10-03 11:38:52 -07001003 return ret;
Paul E. McKenney9b2e4f12011-09-30 12:10:22 -07001004}
Paul E. McKenney5c173eb2013-09-13 17:20:11 -07001005EXPORT_SYMBOL_GPL(rcu_is_watching);
Paul E. McKenney9b2e4f12011-09-30 12:10:22 -07001006
Paul E. McKenney62fde6e2012-05-22 22:10:24 -07001007#if defined(CONFIG_PROVE_RCU) && defined(CONFIG_HOTPLUG_CPU)
Paul E. McKenneyc0d6d012012-01-23 12:41:26 -08001008
1009/*
1010 * Is the current CPU online? Disable preemption to avoid false positives
1011 * that could otherwise happen due to the current CPU number being sampled,
1012 * this task being preempted, its old CPU being taken offline, resuming
1013 * on some other CPU, then determining that its old CPU is now offline.
1014 * It is OK to use RCU on an offline processor during initial boot, hence
Paul E. McKenney2036d942012-01-30 17:02:47 -08001015 * the check for rcu_scheduler_fully_active. Note also that it is OK
1016 * for a CPU coming online to use RCU for one jiffy prior to marking itself
1017 * online in the cpu_online_mask. Similarly, it is OK for a CPU going
1018 * offline to continue to use RCU for one jiffy after marking itself
1019 * offline in the cpu_online_mask. This leniency is necessary given the
1020 * non-atomic nature of the online and offline processing, for example,
1021 * the fact that a CPU enters the scheduler after completing the CPU_DYING
1022 * notifiers.
1023 *
1024 * This is also why RCU internally marks CPUs online during the
1025 * CPU_UP_PREPARE phase and offline during the CPU_DEAD phase.
Paul E. McKenneyc0d6d012012-01-23 12:41:26 -08001026 *
1027 * Disable checking if in an NMI handler because we cannot safely report
1028 * errors from NMI handlers anyway.
1029 */
1030bool rcu_lockdep_current_cpu_online(void)
1031{
Paul E. McKenney2036d942012-01-30 17:02:47 -08001032 struct rcu_data *rdp;
1033 struct rcu_node *rnp;
Paul E. McKenneyc0d6d012012-01-23 12:41:26 -08001034 bool ret;
1035
1036 if (in_nmi())
Fengguang Wuf6f7ee92013-10-10 11:08:33 -07001037 return true;
Paul E. McKenneyc0d6d012012-01-23 12:41:26 -08001038 preempt_disable();
Christoph Lameterc9d4b0a2013-08-31 13:34:10 -07001039 rdp = this_cpu_ptr(&rcu_sched_data);
Paul E. McKenney2036d942012-01-30 17:02:47 -08001040 rnp = rdp->mynode;
Paul E. McKenney0aa04b02015-01-23 21:52:37 -08001041 ret = (rdp->grpmask & rcu_rnp_online_cpus(rnp)) ||
Paul E. McKenneyc0d6d012012-01-23 12:41:26 -08001042 !rcu_scheduler_fully_active;
1043 preempt_enable();
1044 return ret;
1045}
1046EXPORT_SYMBOL_GPL(rcu_lockdep_current_cpu_online);
1047
Paul E. McKenney62fde6e2012-05-22 22:10:24 -07001048#endif /* #if defined(CONFIG_PROVE_RCU) && defined(CONFIG_HOTPLUG_CPU) */
Paul E. McKenney9b2e4f12011-09-30 12:10:22 -07001049
1050/**
1051 * rcu_is_cpu_rrupt_from_idle - see if idle or immediately interrupted from idle
1052 *
1053 * If the current CPU is idle or running at a first-level (not nested)
1054 * interrupt from idle, return true. The caller must have at least
1055 * disabled preemption.
1056 */
Josh Triplett62e3cb12012-11-20 09:55:26 -08001057static int rcu_is_cpu_rrupt_from_idle(void)
Paul E. McKenney9b2e4f12011-09-30 12:10:22 -07001058{
Christoph Lameterc9d4b0a2013-08-31 13:34:10 -07001059 return __this_cpu_read(rcu_dynticks.dynticks_nesting) <= 1;
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01001060}
1061
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01001062/*
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01001063 * Snapshot the specified CPU's dynticks counter so that we can later
1064 * credit them with an implicit quiescent state. Return 1 if this CPU
Paul E. McKenney1eba8f82009-09-23 09:50:42 -07001065 * is in dynticks idle mode, which is an extended quiescent state.
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01001066 */
Paul E. McKenney217af2a2013-06-21 15:39:06 -07001067static int dyntick_save_progress_counter(struct rcu_data *rdp,
1068 bool *isidle, unsigned long *maxj)
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01001069{
Paul E. McKenney23b5c8f2010-09-07 10:38:22 -07001070 rdp->dynticks_snap = atomic_add_return(0, &rdp->dynticks->dynticks);
Paul E. McKenney0edd1b12013-06-21 16:37:22 -07001071 rcu_sysidle_check_cpu(rdp, isidle, maxj);
Andreea-Cristina Bernat7941dbd2014-03-17 18:33:28 +02001072 if ((rdp->dynticks_snap & 0x1) == 0) {
1073 trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("dti"));
1074 return 1;
1075 } else {
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08001076 if (ULONG_CMP_LT(READ_ONCE(rdp->gpnum) + ULONG_MAX / 4,
Paul E. McKenneye3663b12014-12-08 20:26:55 -08001077 rdp->mynode->gpnum))
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08001078 WRITE_ONCE(rdp->gpwrap, true);
Andreea-Cristina Bernat7941dbd2014-03-17 18:33:28 +02001079 return 0;
1080 }
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01001081}
1082
1083/*
1084 * Return true if the specified CPU has passed through a quiescent
1085 * state by virtue of being in or having passed through an dynticks
1086 * idle state since the last call to dyntick_save_progress_counter()
Paul E. McKenneya82dcc72012-08-01 14:29:20 -07001087 * for this same CPU, or by virtue of having been offline.
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01001088 */
Paul E. McKenney217af2a2013-06-21 15:39:06 -07001089static int rcu_implicit_dynticks_qs(struct rcu_data *rdp,
1090 bool *isidle, unsigned long *maxj)
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01001091{
Paul E. McKenney7eb4f452011-07-30 07:32:48 -07001092 unsigned int curr;
Paul E. McKenney4a81e832014-06-20 16:49:01 -07001093 int *rcrmp;
Paul E. McKenney7eb4f452011-07-30 07:32:48 -07001094 unsigned int snap;
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01001095
Paul E. McKenney7eb4f452011-07-30 07:32:48 -07001096 curr = (unsigned int)atomic_add_return(0, &rdp->dynticks->dynticks);
1097 snap = (unsigned int)rdp->dynticks_snap;
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01001098
1099 /*
1100 * If the CPU passed through or entered a dynticks idle phase with
1101 * no active irq/NMI handlers, then we can safely pretend that the CPU
1102 * already acknowledged the request to pass through a quiescent
1103 * state. Either way, that CPU cannot possibly be in an RCU
1104 * read-side critical section that started before the beginning
1105 * of the current RCU grace period.
1106 */
Paul E. McKenney7eb4f452011-07-30 07:32:48 -07001107 if ((curr & 0x1) == 0 || UINT_CMP_GE(curr, snap + 2)) {
Steven Rostedt (Red Hat)f7f7bac2013-07-12 17:18:47 -04001108 trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("dti"));
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01001109 rdp->dynticks_fqs++;
1110 return 1;
1111 }
1112
Paul E. McKenneya82dcc72012-08-01 14:29:20 -07001113 /*
1114 * Check for the CPU being offline, but only if the grace period
1115 * is old enough. We don't need to worry about the CPU changing
1116 * state: If we see it offline even once, it has been through a
1117 * quiescent state.
1118 *
1119 * The reason for insisting that the grace period be at least
1120 * one jiffy old is that CPUs that are not quite online and that
1121 * have just gone offline can still execute RCU read-side critical
1122 * sections.
1123 */
1124 if (ULONG_CMP_GE(rdp->rsp->gp_start + 2, jiffies))
1125 return 0; /* Grace period is not old enough. */
1126 barrier();
1127 if (cpu_is_offline(rdp->cpu)) {
Steven Rostedt (Red Hat)f7f7bac2013-07-12 17:18:47 -04001128 trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("ofl"));
Paul E. McKenneya82dcc72012-08-01 14:29:20 -07001129 rdp->offline_fqs++;
1130 return 1;
1131 }
Paul E. McKenney65d798f2013-04-12 16:19:10 -07001132
1133 /*
Paul E. McKenney4a81e832014-06-20 16:49:01 -07001134 * A CPU running for an extended time within the kernel can
1135 * delay RCU grace periods. When the CPU is in NO_HZ_FULL mode,
1136 * even context-switching back and forth between a pair of
1137 * in-kernel CPU-bound tasks cannot advance grace periods.
1138 * So if the grace period is old enough, make the CPU pay attention.
1139 * Note that the unsynchronized assignments to the per-CPU
1140 * rcu_sched_qs_mask variable are safe. Yes, setting of
1141 * bits can be lost, but they will be set again on the next
1142 * force-quiescent-state pass. So lost bit sets do not result
1143 * in incorrect behavior, merely in a grace period lasting
1144 * a few jiffies longer than it might otherwise. Because
1145 * there are at most four threads involved, and because the
1146 * updates are only once every few jiffies, the probability of
1147 * lossage (and thus of slight grace-period extension) is
1148 * quite low.
1149 *
1150 * Note that if the jiffies_till_sched_qs boot/sysfs parameter
1151 * is set too high, we override with half of the RCU CPU stall
1152 * warning delay.
Paul E. McKenney65d798f2013-04-12 16:19:10 -07001153 */
Paul E. McKenney4a81e832014-06-20 16:49:01 -07001154 rcrmp = &per_cpu(rcu_sched_qs_mask, rdp->cpu);
1155 if (ULONG_CMP_GE(jiffies,
1156 rdp->rsp->gp_start + jiffies_till_sched_qs) ||
Paul E. McKenneycb1e78c2013-12-04 18:42:03 -08001157 ULONG_CMP_GE(jiffies, rdp->rsp->jiffies_resched)) {
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08001158 if (!(READ_ONCE(*rcrmp) & rdp->rsp->flavor_mask)) {
1159 WRITE_ONCE(rdp->cond_resched_completed,
1160 READ_ONCE(rdp->mynode->completed));
Paul E. McKenney4a81e832014-06-20 16:49:01 -07001161 smp_mb(); /* ->cond_resched_completed before *rcrmp. */
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08001162 WRITE_ONCE(*rcrmp,
1163 READ_ONCE(*rcrmp) + rdp->rsp->flavor_mask);
Paul E. McKenney4a81e832014-06-20 16:49:01 -07001164 resched_cpu(rdp->cpu); /* Force CPU into scheduler. */
1165 rdp->rsp->jiffies_resched += 5; /* Enable beating. */
1166 } else if (ULONG_CMP_GE(jiffies, rdp->rsp->jiffies_resched)) {
1167 /* Time to beat on that CPU again! */
1168 resched_cpu(rdp->cpu); /* Force CPU into scheduler. */
1169 rdp->rsp->jiffies_resched += 5; /* Re-enable beating. */
1170 }
Paul E. McKenney6193c762013-09-23 13:57:18 -07001171 }
1172
Paul E. McKenneya82dcc72012-08-01 14:29:20 -07001173 return 0;
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01001174}
1175
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01001176static void record_gp_stall_check_time(struct rcu_state *rsp)
1177{
Paul E. McKenneycb1e78c2013-12-04 18:42:03 -08001178 unsigned long j = jiffies;
Paul E. McKenney6193c762013-09-23 13:57:18 -07001179 unsigned long j1;
Paul E. McKenney26cdfed2013-09-04 10:51:13 -07001180
1181 rsp->gp_start = j;
1182 smp_wmb(); /* Record start time before stall time. */
Paul E. McKenney6193c762013-09-23 13:57:18 -07001183 j1 = rcu_jiffies_till_stall_check();
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08001184 WRITE_ONCE(rsp->jiffies_stall, j + j1);
Paul E. McKenney6193c762013-09-23 13:57:18 -07001185 rsp->jiffies_resched = j + j1 / 2;
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08001186 rsp->n_force_qs_gpstart = READ_ONCE(rsp->n_force_qs);
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01001187}
1188
Paul E. McKenneyb637a322012-09-19 16:58:38 -07001189/*
Paul E. McKenneyfb81a442014-12-17 08:35:02 -08001190 * Complain about starvation of grace-period kthread.
1191 */
1192static void rcu_check_gp_kthread_starvation(struct rcu_state *rsp)
1193{
1194 unsigned long gpa;
1195 unsigned long j;
1196
1197 j = jiffies;
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08001198 gpa = READ_ONCE(rsp->gp_activity);
Paul E. McKenneyfb81a442014-12-17 08:35:02 -08001199 if (j - gpa > 2 * HZ)
Paul E. McKenney319362c2015-05-19 14:16:52 -07001200 pr_err("%s kthread starved for %ld jiffies! g%lu c%lu f%#x s%d ->state=%#lx\n",
Paul E. McKenney81e701e432015-04-16 11:02:25 -07001201 rsp->name, j - gpa,
Paul E. McKenney319362c2015-05-19 14:16:52 -07001202 rsp->gpnum, rsp->completed,
1203 rsp->gp_flags, rsp->gp_state,
1204 rsp->gp_kthread ? rsp->gp_kthread->state : 0);
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01001205}
1206
Paul E. McKenneyb637a322012-09-19 16:58:38 -07001207/*
Paul E. McKenneybc1dce52014-06-18 09:18:31 -07001208 * Dump stacks of all tasks running on stalled CPUs.
Paul E. McKenneyb637a322012-09-19 16:58:38 -07001209 */
1210static void rcu_dump_cpu_stacks(struct rcu_state *rsp)
1211{
1212 int cpu;
1213 unsigned long flags;
1214 struct rcu_node *rnp;
1215
1216 rcu_for_each_leaf_node(rsp, rnp) {
Paul E. McKenney6cf10082015-10-08 15:36:54 -07001217 raw_spin_lock_irqsave_rcu_node(rnp, flags);
Paul E. McKenneyb637a322012-09-19 16:58:38 -07001218 if (rnp->qsmask != 0) {
1219 for (cpu = 0; cpu <= rnp->grphi - rnp->grplo; cpu++)
1220 if (rnp->qsmask & (1UL << cpu))
1221 dump_cpu_task(rnp->grplo + cpu);
1222 }
1223 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1224 }
1225}
1226
Paul E. McKenney6ccd2ec2014-12-11 10:20:59 -08001227static void print_other_cpu_stall(struct rcu_state *rsp, unsigned long gpnum)
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01001228{
1229 int cpu;
1230 long delta;
1231 unsigned long flags;
Paul E. McKenney6ccd2ec2014-12-11 10:20:59 -08001232 unsigned long gpa;
1233 unsigned long j;
Paul E. McKenney285fe292012-05-09 08:45:12 -07001234 int ndetected = 0;
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01001235 struct rcu_node *rnp = rcu_get_root(rsp);
Paul E. McKenney53bb8572012-09-21 16:35:25 -07001236 long totqlen = 0;
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01001237
1238 /* Only let one CPU complain about others per time interval. */
1239
Paul E. McKenney6cf10082015-10-08 15:36:54 -07001240 raw_spin_lock_irqsave_rcu_node(rnp, flags);
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08001241 delta = jiffies - READ_ONCE(rsp->jiffies_stall);
Paul E. McKenneyfc2219d42009-09-23 09:50:41 -07001242 if (delta < RCU_STALL_RAT_DELAY || !rcu_gp_in_progress(rsp)) {
Paul E. McKenney1304afb2010-02-22 17:05:02 -08001243 raw_spin_unlock_irqrestore(&rnp->lock, flags);
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01001244 return;
1245 }
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08001246 WRITE_ONCE(rsp->jiffies_stall,
1247 jiffies + 3 * rcu_jiffies_till_stall_check() + 3);
Paul E. McKenney1304afb2010-02-22 17:05:02 -08001248 raw_spin_unlock_irqrestore(&rnp->lock, flags);
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01001249
Paul E. McKenney8cdd32a2010-08-09 14:23:03 -07001250 /*
1251 * OK, time to rat on our buddy...
1252 * See Documentation/RCU/stallwarn.txt for info on how to debug
1253 * RCU CPU stall warnings.
1254 */
Paul E. McKenneyd7f3e202013-03-18 16:19:52 -07001255 pr_err("INFO: %s detected stalls on CPUs/tasks:",
Paul E. McKenney4300aa62010-04-13 16:18:22 -07001256 rsp->name);
Paul E. McKenneya858af22012-01-16 13:29:10 -08001257 print_cpu_stall_info_begin();
Paul E. McKenneya0b6c9a2009-09-28 07:46:33 -07001258 rcu_for_each_leaf_node(rsp, rnp) {
Paul E. McKenney6cf10082015-10-08 15:36:54 -07001259 raw_spin_lock_irqsave_rcu_node(rnp, flags);
Paul E. McKenney9bc8b552011-08-13 13:31:47 -07001260 ndetected += rcu_print_task_stall(rnp);
Paul E. McKenneyc8020a62012-08-10 16:55:59 -07001261 if (rnp->qsmask != 0) {
1262 for (cpu = 0; cpu <= rnp->grphi - rnp->grplo; cpu++)
1263 if (rnp->qsmask & (1UL << cpu)) {
1264 print_cpu_stall_info(rsp,
1265 rnp->grplo + cpu);
1266 ndetected++;
1267 }
1268 }
Paul E. McKenney3acd9eb2010-02-22 17:05:03 -08001269 raw_spin_unlock_irqrestore(&rnp->lock, flags);
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01001270 }
Paul E. McKenneya858af22012-01-16 13:29:10 -08001271
Paul E. McKenneya858af22012-01-16 13:29:10 -08001272 print_cpu_stall_info_end();
Paul E. McKenney53bb8572012-09-21 16:35:25 -07001273 for_each_possible_cpu(cpu)
1274 totqlen += per_cpu_ptr(rsp->rda, cpu)->qlen;
Paul E. McKenney83ebe632014-03-06 11:09:10 -08001275 pr_cont("(detected by %d, t=%ld jiffies, g=%ld, c=%ld, q=%lu)\n",
Paul E. McKenneyeee05882012-09-21 14:15:05 -07001276 smp_processor_id(), (long)(jiffies - rsp->gp_start),
Paul E. McKenney83ebe632014-03-06 11:09:10 -08001277 (long)rsp->gpnum, (long)rsp->completed, totqlen);
Paul E. McKenney6ccd2ec2014-12-11 10:20:59 -08001278 if (ndetected) {
Paul E. McKenneyb637a322012-09-19 16:58:38 -07001279 rcu_dump_cpu_stacks(rsp);
Paul E. McKenney6ccd2ec2014-12-11 10:20:59 -08001280 } else {
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08001281 if (READ_ONCE(rsp->gpnum) != gpnum ||
1282 READ_ONCE(rsp->completed) == gpnum) {
Paul E. McKenney6ccd2ec2014-12-11 10:20:59 -08001283 pr_err("INFO: Stall ended before state dump start\n");
1284 } else {
1285 j = jiffies;
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08001286 gpa = READ_ONCE(rsp->gp_activity);
Paul E. McKenney237a0f22015-01-22 14:32:06 -08001287 pr_err("All QSes seen, last %s kthread activity %ld (%ld-%ld), jiffies_till_next_fqs=%ld, root ->qsmask %#lx\n",
Paul E. McKenney6ccd2ec2014-12-11 10:20:59 -08001288 rsp->name, j - gpa, j, gpa,
Paul E. McKenney237a0f22015-01-22 14:32:06 -08001289 jiffies_till_next_fqs,
1290 rcu_get_root(rsp)->qsmask);
Paul E. McKenney6ccd2ec2014-12-11 10:20:59 -08001291 /* In this case, the current CPU might be at fault. */
1292 sched_show_task(current);
1293 }
1294 }
Ingo Molnarc1dc0b92009-08-02 11:28:21 +02001295
Paul E. McKenney4cdfc1752012-06-22 17:06:26 -07001296 /* Complain about tasks blocking the grace period. */
Paul E. McKenney1ed509a2010-02-22 17:05:05 -08001297 rcu_print_detail_task_stall(rsp);
1298
Paul E. McKenneyfb81a442014-12-17 08:35:02 -08001299 rcu_check_gp_kthread_starvation(rsp);
1300
Paul E. McKenney4cdfc1752012-06-22 17:06:26 -07001301 force_quiescent_state(rsp); /* Kick them all. */
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01001302}
1303
1304static void print_cpu_stall(struct rcu_state *rsp)
1305{
Paul E. McKenney53bb8572012-09-21 16:35:25 -07001306 int cpu;
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01001307 unsigned long flags;
1308 struct rcu_node *rnp = rcu_get_root(rsp);
Paul E. McKenney53bb8572012-09-21 16:35:25 -07001309 long totqlen = 0;
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01001310
Paul E. McKenney8cdd32a2010-08-09 14:23:03 -07001311 /*
1312 * OK, time to rat on ourselves...
1313 * See Documentation/RCU/stallwarn.txt for info on how to debug
1314 * RCU CPU stall warnings.
1315 */
Paul E. McKenneyd7f3e202013-03-18 16:19:52 -07001316 pr_err("INFO: %s self-detected stall on CPU", rsp->name);
Paul E. McKenneya858af22012-01-16 13:29:10 -08001317 print_cpu_stall_info_begin();
1318 print_cpu_stall_info(rsp, smp_processor_id());
1319 print_cpu_stall_info_end();
Paul E. McKenney53bb8572012-09-21 16:35:25 -07001320 for_each_possible_cpu(cpu)
1321 totqlen += per_cpu_ptr(rsp->rda, cpu)->qlen;
Paul E. McKenney83ebe632014-03-06 11:09:10 -08001322 pr_cont(" (t=%lu jiffies g=%ld c=%ld q=%lu)\n",
1323 jiffies - rsp->gp_start,
1324 (long)rsp->gpnum, (long)rsp->completed, totqlen);
Paul E. McKenneyfb81a442014-12-17 08:35:02 -08001325
1326 rcu_check_gp_kthread_starvation(rsp);
1327
Paul E. McKenneybc1dce52014-06-18 09:18:31 -07001328 rcu_dump_cpu_stacks(rsp);
Ingo Molnarc1dc0b92009-08-02 11:28:21 +02001329
Paul E. McKenney6cf10082015-10-08 15:36:54 -07001330 raw_spin_lock_irqsave_rcu_node(rnp, flags);
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08001331 if (ULONG_CMP_GE(jiffies, READ_ONCE(rsp->jiffies_stall)))
1332 WRITE_ONCE(rsp->jiffies_stall,
1333 jiffies + 3 * rcu_jiffies_till_stall_check() + 3);
Paul E. McKenney1304afb2010-02-22 17:05:02 -08001334 raw_spin_unlock_irqrestore(&rnp->lock, flags);
Ingo Molnarc1dc0b92009-08-02 11:28:21 +02001335
Peter Zijlstrab021fe32013-09-17 09:30:55 +02001336 /*
1337 * Attempt to revive the RCU machinery by forcing a context switch.
1338 *
1339 * A context switch would normally allow the RCU state machine to make
1340 * progress and it could be we're stuck in kernel space without context
1341 * switches for an entirely unreasonable amount of time.
1342 */
1343 resched_cpu(smp_processor_id());
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01001344}
1345
1346static void check_cpu_stall(struct rcu_state *rsp, struct rcu_data *rdp)
1347{
Paul E. McKenney26cdfed2013-09-04 10:51:13 -07001348 unsigned long completed;
1349 unsigned long gpnum;
1350 unsigned long gps;
Paul E. McKenneybad6e132011-05-02 23:40:04 -07001351 unsigned long j;
1352 unsigned long js;
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01001353 struct rcu_node *rnp;
1354
Paul E. McKenney26cdfed2013-09-04 10:51:13 -07001355 if (rcu_cpu_stall_suppress || !rcu_gp_in_progress(rsp))
Paul E. McKenneyc68de202010-04-15 10:12:40 -07001356 return;
Paul E. McKenneycb1e78c2013-12-04 18:42:03 -08001357 j = jiffies;
Paul E. McKenney26cdfed2013-09-04 10:51:13 -07001358
1359 /*
1360 * Lots of memory barriers to reject false positives.
1361 *
1362 * The idea is to pick up rsp->gpnum, then rsp->jiffies_stall,
1363 * then rsp->gp_start, and finally rsp->completed. These values
1364 * are updated in the opposite order with memory barriers (or
1365 * equivalent) during grace-period initialization and cleanup.
1366 * Now, a false positive can occur if we get an new value of
1367 * rsp->gp_start and a old value of rsp->jiffies_stall. But given
1368 * the memory barriers, the only way that this can happen is if one
1369 * grace period ends and another starts between these two fetches.
1370 * Detect this by comparing rsp->completed with the previous fetch
1371 * from rsp->gpnum.
1372 *
1373 * Given this check, comparisons of jiffies, rsp->jiffies_stall,
1374 * and rsp->gp_start suffice to forestall false positives.
1375 */
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08001376 gpnum = READ_ONCE(rsp->gpnum);
Paul E. McKenney26cdfed2013-09-04 10:51:13 -07001377 smp_rmb(); /* Pick up ->gpnum first... */
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08001378 js = READ_ONCE(rsp->jiffies_stall);
Paul E. McKenney26cdfed2013-09-04 10:51:13 -07001379 smp_rmb(); /* ...then ->jiffies_stall before the rest... */
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08001380 gps = READ_ONCE(rsp->gp_start);
Paul E. McKenney26cdfed2013-09-04 10:51:13 -07001381 smp_rmb(); /* ...and finally ->gp_start before ->completed. */
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08001382 completed = READ_ONCE(rsp->completed);
Paul E. McKenney26cdfed2013-09-04 10:51:13 -07001383 if (ULONG_CMP_GE(completed, gpnum) ||
1384 ULONG_CMP_LT(j, js) ||
1385 ULONG_CMP_GE(gps, js))
1386 return; /* No stall or GP completed since entering function. */
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01001387 rnp = rdp->mynode;
Paul E. McKenneyc96ea7c2012-08-13 11:17:06 -07001388 if (rcu_gp_in_progress(rsp) &&
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08001389 (READ_ONCE(rnp->qsmask) & rdp->grpmask)) {
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01001390
1391 /* We haven't checked in, so go dump stack. */
1392 print_cpu_stall(rsp);
1393
Paul E. McKenneybad6e132011-05-02 23:40:04 -07001394 } else if (rcu_gp_in_progress(rsp) &&
1395 ULONG_CMP_GE(j, js + RCU_STALL_RAT_DELAY)) {
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01001396
Paul E. McKenneybad6e132011-05-02 23:40:04 -07001397 /* They had a few time units to dump stack, so complain. */
Paul E. McKenney6ccd2ec2014-12-11 10:20:59 -08001398 print_other_cpu_stall(rsp, gpnum);
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01001399 }
1400}
1401
Paul E. McKenney53d84e02010-08-10 14:28:53 -07001402/**
1403 * rcu_cpu_stall_reset - prevent further stall warnings in current grace period
1404 *
1405 * Set the stall-warning timeout way off into the future, thus preventing
1406 * any RCU CPU stall-warning messages from appearing in the current set of
1407 * RCU grace periods.
1408 *
1409 * The caller must disable hard irqs.
1410 */
1411void rcu_cpu_stall_reset(void)
1412{
Paul E. McKenney6ce75a22012-06-12 11:01:13 -07001413 struct rcu_state *rsp;
1414
1415 for_each_rcu_flavor(rsp)
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08001416 WRITE_ONCE(rsp->jiffies_stall, jiffies + ULONG_MAX / 2);
Paul E. McKenney53d84e02010-08-10 14:28:53 -07001417}
1418
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01001419/*
Paul E. McKenneyd3f3f3f2015-01-18 18:21:09 -08001420 * Initialize the specified rcu_data structure's default callback list
1421 * to empty. The default callback list is the one that is not used by
1422 * no-callbacks CPUs.
1423 */
1424static void init_default_callback_list(struct rcu_data *rdp)
1425{
1426 int i;
1427
1428 rdp->nxtlist = NULL;
1429 for (i = 0; i < RCU_NEXT_SIZE; i++)
1430 rdp->nxttail[i] = &rdp->nxtlist;
1431}
1432
1433/*
Paul E. McKenney3f5d3ea2012-05-09 15:39:56 -07001434 * Initialize the specified rcu_data structure's callback list to empty.
1435 */
1436static void init_callback_list(struct rcu_data *rdp)
1437{
Paul E. McKenney34ed62462013-01-07 13:37:42 -08001438 if (init_nocb_callback_list(rdp))
1439 return;
Paul E. McKenneyd3f3f3f2015-01-18 18:21:09 -08001440 init_default_callback_list(rdp);
Paul E. McKenney3f5d3ea2012-05-09 15:39:56 -07001441}
1442
1443/*
Paul E. McKenneydc35c892012-12-03 13:52:00 -08001444 * Determine the value that ->completed will have at the end of the
1445 * next subsequent grace period. This is used to tag callbacks so that
1446 * a CPU can invoke callbacks in a timely fashion even if that CPU has
1447 * been dyntick-idle for an extended period with callbacks under the
1448 * influence of RCU_FAST_NO_HZ.
1449 *
1450 * The caller must hold rnp->lock with interrupts disabled.
1451 */
1452static unsigned long rcu_cbs_completed(struct rcu_state *rsp,
1453 struct rcu_node *rnp)
1454{
1455 /*
1456 * If RCU is idle, we just wait for the next grace period.
1457 * But we can only be sure that RCU is idle if we are looking
1458 * at the root rcu_node structure -- otherwise, a new grace
1459 * period might have started, but just not yet gotten around
1460 * to initializing the current non-root rcu_node structure.
1461 */
1462 if (rcu_get_root(rsp) == rnp && rnp->gpnum == rnp->completed)
1463 return rnp->completed + 1;
1464
1465 /*
1466 * Otherwise, wait for a possible partial grace period and
1467 * then the subsequent full grace period.
1468 */
1469 return rnp->completed + 2;
1470}
1471
1472/*
Paul E. McKenney0446be42012-12-30 15:21:01 -08001473 * Trace-event helper function for rcu_start_future_gp() and
1474 * rcu_nocb_wait_gp().
1475 */
1476static void trace_rcu_future_gp(struct rcu_node *rnp, struct rcu_data *rdp,
Steven Rostedt (Red Hat)e66c33d2013-07-12 16:50:28 -04001477 unsigned long c, const char *s)
Paul E. McKenney0446be42012-12-30 15:21:01 -08001478{
1479 trace_rcu_future_grace_period(rdp->rsp->name, rnp->gpnum,
1480 rnp->completed, c, rnp->level,
1481 rnp->grplo, rnp->grphi, s);
1482}
1483
1484/*
1485 * Start some future grace period, as needed to handle newly arrived
1486 * callbacks. The required future grace periods are recorded in each
Paul E. McKenney48a76392014-03-11 13:02:16 -07001487 * rcu_node structure's ->need_future_gp field. Returns true if there
1488 * is reason to awaken the grace-period kthread.
Paul E. McKenney0446be42012-12-30 15:21:01 -08001489 *
1490 * The caller must hold the specified rcu_node structure's ->lock.
1491 */
Paul E. McKenney48a76392014-03-11 13:02:16 -07001492static bool __maybe_unused
1493rcu_start_future_gp(struct rcu_node *rnp, struct rcu_data *rdp,
1494 unsigned long *c_out)
Paul E. McKenney0446be42012-12-30 15:21:01 -08001495{
1496 unsigned long c;
1497 int i;
Paul E. McKenney48a76392014-03-11 13:02:16 -07001498 bool ret = false;
Paul E. McKenney0446be42012-12-30 15:21:01 -08001499 struct rcu_node *rnp_root = rcu_get_root(rdp->rsp);
1500
1501 /*
1502 * Pick up grace-period number for new callbacks. If this
1503 * grace period is already marked as needed, return to the caller.
1504 */
1505 c = rcu_cbs_completed(rdp->rsp, rnp);
Steven Rostedt (Red Hat)f7f7bac2013-07-12 17:18:47 -04001506 trace_rcu_future_gp(rnp, rdp, c, TPS("Startleaf"));
Paul E. McKenney0446be42012-12-30 15:21:01 -08001507 if (rnp->need_future_gp[c & 0x1]) {
Steven Rostedt (Red Hat)f7f7bac2013-07-12 17:18:47 -04001508 trace_rcu_future_gp(rnp, rdp, c, TPS("Prestartleaf"));
Paul E. McKenney48a76392014-03-11 13:02:16 -07001509 goto out;
Paul E. McKenney0446be42012-12-30 15:21:01 -08001510 }
1511
1512 /*
1513 * If either this rcu_node structure or the root rcu_node structure
1514 * believe that a grace period is in progress, then we must wait
1515 * for the one following, which is in "c". Because our request
1516 * will be noticed at the end of the current grace period, we don't
Pranith Kumar48bd8e92014-06-11 10:32:47 -07001517 * need to explicitly start one. We only do the lockless check
1518 * of rnp_root's fields if the current rcu_node structure thinks
1519 * there is no grace period in flight, and because we hold rnp->lock,
1520 * the only possible change is when rnp_root's two fields are
1521 * equal, in which case rnp_root->gpnum might be concurrently
1522 * incremented. But that is OK, as it will just result in our
1523 * doing some extra useless work.
Paul E. McKenney0446be42012-12-30 15:21:01 -08001524 */
1525 if (rnp->gpnum != rnp->completed ||
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08001526 READ_ONCE(rnp_root->gpnum) != READ_ONCE(rnp_root->completed)) {
Paul E. McKenney0446be42012-12-30 15:21:01 -08001527 rnp->need_future_gp[c & 0x1]++;
Steven Rostedt (Red Hat)f7f7bac2013-07-12 17:18:47 -04001528 trace_rcu_future_gp(rnp, rdp, c, TPS("Startedleaf"));
Paul E. McKenney48a76392014-03-11 13:02:16 -07001529 goto out;
Paul E. McKenney0446be42012-12-30 15:21:01 -08001530 }
1531
1532 /*
1533 * There might be no grace period in progress. If we don't already
1534 * hold it, acquire the root rcu_node structure's lock in order to
1535 * start one (if needed).
1536 */
Peter Zijlstra2a67e742015-10-08 12:24:23 +02001537 if (rnp != rnp_root)
1538 raw_spin_lock_rcu_node(rnp_root);
Paul E. McKenney0446be42012-12-30 15:21:01 -08001539
1540 /*
1541 * Get a new grace-period number. If there really is no grace
1542 * period in progress, it will be smaller than the one we obtained
1543 * earlier. Adjust callbacks as needed. Note that even no-CBs
1544 * CPUs have a ->nxtcompleted[] array, so no no-CBs checks needed.
1545 */
1546 c = rcu_cbs_completed(rdp->rsp, rnp_root);
1547 for (i = RCU_DONE_TAIL; i < RCU_NEXT_TAIL; i++)
1548 if (ULONG_CMP_LT(c, rdp->nxtcompleted[i]))
1549 rdp->nxtcompleted[i] = c;
1550
1551 /*
1552 * If the needed for the required grace period is already
1553 * recorded, trace and leave.
1554 */
1555 if (rnp_root->need_future_gp[c & 0x1]) {
Steven Rostedt (Red Hat)f7f7bac2013-07-12 17:18:47 -04001556 trace_rcu_future_gp(rnp, rdp, c, TPS("Prestartedroot"));
Paul E. McKenney0446be42012-12-30 15:21:01 -08001557 goto unlock_out;
1558 }
1559
1560 /* Record the need for the future grace period. */
1561 rnp_root->need_future_gp[c & 0x1]++;
1562
1563 /* If a grace period is not already in progress, start one. */
1564 if (rnp_root->gpnum != rnp_root->completed) {
Steven Rostedt (Red Hat)f7f7bac2013-07-12 17:18:47 -04001565 trace_rcu_future_gp(rnp, rdp, c, TPS("Startedleafroot"));
Paul E. McKenney0446be42012-12-30 15:21:01 -08001566 } else {
Steven Rostedt (Red Hat)f7f7bac2013-07-12 17:18:47 -04001567 trace_rcu_future_gp(rnp, rdp, c, TPS("Startedroot"));
Paul E. McKenney48a76392014-03-11 13:02:16 -07001568 ret = rcu_start_gp_advanced(rdp->rsp, rnp_root, rdp);
Paul E. McKenney0446be42012-12-30 15:21:01 -08001569 }
1570unlock_out:
1571 if (rnp != rnp_root)
1572 raw_spin_unlock(&rnp_root->lock);
Paul E. McKenney48a76392014-03-11 13:02:16 -07001573out:
1574 if (c_out != NULL)
1575 *c_out = c;
1576 return ret;
Paul E. McKenney0446be42012-12-30 15:21:01 -08001577}
1578
1579/*
1580 * Clean up any old requests for the just-ended grace period. Also return
1581 * whether any additional grace periods have been requested. Also invoke
1582 * rcu_nocb_gp_cleanup() in order to wake up any no-callbacks kthreads
1583 * waiting for this grace period to complete.
1584 */
1585static int rcu_future_gp_cleanup(struct rcu_state *rsp, struct rcu_node *rnp)
1586{
1587 int c = rnp->completed;
1588 int needmore;
1589 struct rcu_data *rdp = this_cpu_ptr(rsp->rda);
1590
1591 rcu_nocb_gp_cleanup(rsp, rnp);
1592 rnp->need_future_gp[c & 0x1] = 0;
1593 needmore = rnp->need_future_gp[(c + 1) & 0x1];
Steven Rostedt (Red Hat)f7f7bac2013-07-12 17:18:47 -04001594 trace_rcu_future_gp(rnp, rdp, c,
1595 needmore ? TPS("CleanupMore") : TPS("Cleanup"));
Paul E. McKenney0446be42012-12-30 15:21:01 -08001596 return needmore;
1597}
1598
1599/*
Paul E. McKenney48a76392014-03-11 13:02:16 -07001600 * Awaken the grace-period kthread for the specified flavor of RCU.
1601 * Don't do a self-awaken, and don't bother awakening when there is
1602 * nothing for the grace-period kthread to do (as in several CPUs
1603 * raced to awaken, and we lost), and finally don't try to awaken
1604 * a kthread that has not yet been created.
1605 */
1606static void rcu_gp_kthread_wake(struct rcu_state *rsp)
1607{
1608 if (current == rsp->gp_kthread ||
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08001609 !READ_ONCE(rsp->gp_flags) ||
Paul E. McKenney48a76392014-03-11 13:02:16 -07001610 !rsp->gp_kthread)
1611 return;
1612 wake_up(&rsp->gp_wq);
1613}
1614
1615/*
Paul E. McKenneydc35c892012-12-03 13:52:00 -08001616 * If there is room, assign a ->completed number to any callbacks on
1617 * this CPU that have not already been assigned. Also accelerate any
1618 * callbacks that were previously assigned a ->completed number that has
1619 * since proven to be too conservative, which can happen if callbacks get
1620 * assigned a ->completed number while RCU is idle, but with reference to
1621 * a non-root rcu_node structure. This function is idempotent, so it does
Paul E. McKenney48a76392014-03-11 13:02:16 -07001622 * not hurt to call it repeatedly. Returns an flag saying that we should
1623 * awaken the RCU grace-period kthread.
Paul E. McKenneydc35c892012-12-03 13:52:00 -08001624 *
1625 * The caller must hold rnp->lock with interrupts disabled.
1626 */
Paul E. McKenney48a76392014-03-11 13:02:16 -07001627static bool rcu_accelerate_cbs(struct rcu_state *rsp, struct rcu_node *rnp,
Paul E. McKenneydc35c892012-12-03 13:52:00 -08001628 struct rcu_data *rdp)
1629{
1630 unsigned long c;
1631 int i;
Paul E. McKenney48a76392014-03-11 13:02:16 -07001632 bool ret;
Paul E. McKenneydc35c892012-12-03 13:52:00 -08001633
1634 /* If the CPU has no callbacks, nothing to do. */
1635 if (!rdp->nxttail[RCU_NEXT_TAIL] || !*rdp->nxttail[RCU_DONE_TAIL])
Paul E. McKenney48a76392014-03-11 13:02:16 -07001636 return false;
Paul E. McKenneydc35c892012-12-03 13:52:00 -08001637
1638 /*
1639 * Starting from the sublist containing the callbacks most
1640 * recently assigned a ->completed number and working down, find the
1641 * first sublist that is not assignable to an upcoming grace period.
1642 * Such a sublist has something in it (first two tests) and has
1643 * a ->completed number assigned that will complete sooner than
1644 * the ->completed number for newly arrived callbacks (last test).
1645 *
1646 * The key point is that any later sublist can be assigned the
1647 * same ->completed number as the newly arrived callbacks, which
1648 * means that the callbacks in any of these later sublist can be
1649 * grouped into a single sublist, whether or not they have already
1650 * been assigned a ->completed number.
1651 */
1652 c = rcu_cbs_completed(rsp, rnp);
1653 for (i = RCU_NEXT_TAIL - 1; i > RCU_DONE_TAIL; i--)
1654 if (rdp->nxttail[i] != rdp->nxttail[i - 1] &&
1655 !ULONG_CMP_GE(rdp->nxtcompleted[i], c))
1656 break;
1657
1658 /*
1659 * If there are no sublist for unassigned callbacks, leave.
1660 * At the same time, advance "i" one sublist, so that "i" will
1661 * index into the sublist where all the remaining callbacks should
1662 * be grouped into.
1663 */
1664 if (++i >= RCU_NEXT_TAIL)
Paul E. McKenney48a76392014-03-11 13:02:16 -07001665 return false;
Paul E. McKenneydc35c892012-12-03 13:52:00 -08001666
1667 /*
1668 * Assign all subsequent callbacks' ->completed number to the next
1669 * full grace period and group them all in the sublist initially
1670 * indexed by "i".
1671 */
1672 for (; i <= RCU_NEXT_TAIL; i++) {
1673 rdp->nxttail[i] = rdp->nxttail[RCU_NEXT_TAIL];
1674 rdp->nxtcompleted[i] = c;
1675 }
Paul E. McKenney910ee452012-12-31 02:24:21 -08001676 /* Record any needed additional grace periods. */
Paul E. McKenney48a76392014-03-11 13:02:16 -07001677 ret = rcu_start_future_gp(rnp, rdp, NULL);
Paul E. McKenney6d4b4182012-11-27 16:55:44 -08001678
1679 /* Trace depending on how much we were able to accelerate. */
1680 if (!*rdp->nxttail[RCU_WAIT_TAIL])
Steven Rostedt (Red Hat)f7f7bac2013-07-12 17:18:47 -04001681 trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("AccWaitCB"));
Paul E. McKenney6d4b4182012-11-27 16:55:44 -08001682 else
Steven Rostedt (Red Hat)f7f7bac2013-07-12 17:18:47 -04001683 trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("AccReadyCB"));
Paul E. McKenney48a76392014-03-11 13:02:16 -07001684 return ret;
Paul E. McKenneydc35c892012-12-03 13:52:00 -08001685}
1686
1687/*
1688 * Move any callbacks whose grace period has completed to the
1689 * RCU_DONE_TAIL sublist, then compact the remaining sublists and
1690 * assign ->completed numbers to any callbacks in the RCU_NEXT_TAIL
1691 * sublist. This function is idempotent, so it does not hurt to
1692 * invoke it repeatedly. As long as it is not invoked -too- often...
Paul E. McKenney48a76392014-03-11 13:02:16 -07001693 * Returns true if the RCU grace-period kthread needs to be awakened.
Paul E. McKenneydc35c892012-12-03 13:52:00 -08001694 *
1695 * The caller must hold rnp->lock with interrupts disabled.
1696 */
Paul E. McKenney48a76392014-03-11 13:02:16 -07001697static bool rcu_advance_cbs(struct rcu_state *rsp, struct rcu_node *rnp,
Paul E. McKenneydc35c892012-12-03 13:52:00 -08001698 struct rcu_data *rdp)
1699{
1700 int i, j;
1701
1702 /* If the CPU has no callbacks, nothing to do. */
1703 if (!rdp->nxttail[RCU_NEXT_TAIL] || !*rdp->nxttail[RCU_DONE_TAIL])
Paul E. McKenney48a76392014-03-11 13:02:16 -07001704 return false;
Paul E. McKenneydc35c892012-12-03 13:52:00 -08001705
1706 /*
1707 * Find all callbacks whose ->completed numbers indicate that they
1708 * are ready to invoke, and put them into the RCU_DONE_TAIL sublist.
1709 */
1710 for (i = RCU_WAIT_TAIL; i < RCU_NEXT_TAIL; i++) {
1711 if (ULONG_CMP_LT(rnp->completed, rdp->nxtcompleted[i]))
1712 break;
1713 rdp->nxttail[RCU_DONE_TAIL] = rdp->nxttail[i];
1714 }
1715 /* Clean up any sublist tail pointers that were misordered above. */
1716 for (j = RCU_WAIT_TAIL; j < i; j++)
1717 rdp->nxttail[j] = rdp->nxttail[RCU_DONE_TAIL];
1718
1719 /* Copy down callbacks to fill in empty sublists. */
1720 for (j = RCU_WAIT_TAIL; i < RCU_NEXT_TAIL; i++, j++) {
1721 if (rdp->nxttail[j] == rdp->nxttail[RCU_NEXT_TAIL])
1722 break;
1723 rdp->nxttail[j] = rdp->nxttail[i];
1724 rdp->nxtcompleted[j] = rdp->nxtcompleted[i];
1725 }
1726
1727 /* Classify any remaining callbacks. */
Paul E. McKenney48a76392014-03-11 13:02:16 -07001728 return rcu_accelerate_cbs(rsp, rnp, rdp);
Paul E. McKenneydc35c892012-12-03 13:52:00 -08001729}
1730
1731/*
Paul E. McKenneyba9fbe92013-03-19 11:53:31 -07001732 * Update CPU-local rcu_data state to record the beginnings and ends of
1733 * grace periods. The caller must hold the ->lock of the leaf rcu_node
1734 * structure corresponding to the current CPU, and must have irqs disabled.
Paul E. McKenney48a76392014-03-11 13:02:16 -07001735 * Returns true if the grace-period kthread needs to be awakened.
Paul E. McKenneyd09b62d2009-11-02 13:52:28 -08001736 */
Paul E. McKenney48a76392014-03-11 13:02:16 -07001737static bool __note_gp_changes(struct rcu_state *rsp, struct rcu_node *rnp,
1738 struct rcu_data *rdp)
Paul E. McKenneyd09b62d2009-11-02 13:52:28 -08001739{
Paul E. McKenney48a76392014-03-11 13:02:16 -07001740 bool ret;
1741
Paul E. McKenneyba9fbe92013-03-19 11:53:31 -07001742 /* Handle the ends of any preceding grace periods first. */
Paul E. McKenneye3663b12014-12-08 20:26:55 -08001743 if (rdp->completed == rnp->completed &&
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08001744 !unlikely(READ_ONCE(rdp->gpwrap))) {
Paul E. McKenneyd09b62d2009-11-02 13:52:28 -08001745
Paul E. McKenneyba9fbe92013-03-19 11:53:31 -07001746 /* No grace period end, so just accelerate recent callbacks. */
Paul E. McKenney48a76392014-03-11 13:02:16 -07001747 ret = rcu_accelerate_cbs(rsp, rnp, rdp);
Paul E. McKenneydc35c892012-12-03 13:52:00 -08001748
1749 } else {
1750
1751 /* Advance callbacks. */
Paul E. McKenney48a76392014-03-11 13:02:16 -07001752 ret = rcu_advance_cbs(rsp, rnp, rdp);
Paul E. McKenneyd09b62d2009-11-02 13:52:28 -08001753
1754 /* Remember that we saw this grace-period completion. */
1755 rdp->completed = rnp->completed;
Steven Rostedt (Red Hat)f7f7bac2013-07-12 17:18:47 -04001756 trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpuend"));
Paul E. McKenneyd09b62d2009-11-02 13:52:28 -08001757 }
Paul E. McKenney398ebe62013-03-19 10:53:14 -07001758
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08001759 if (rdp->gpnum != rnp->gpnum || unlikely(READ_ONCE(rdp->gpwrap))) {
Paul E. McKenney6eaef632013-03-19 10:08:37 -07001760 /*
1761 * If the current grace period is waiting for this CPU,
1762 * set up to detect a quiescent state, otherwise don't
1763 * go looking for one.
1764 */
1765 rdp->gpnum = rnp->gpnum;
Steven Rostedt (Red Hat)f7f7bac2013-07-12 17:18:47 -04001766 trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpustart"));
Paul E. McKenney5b74c452015-08-06 15:16:57 -07001767 rdp->cpu_no_qs.b.norm = true;
Paul E. McKenney5cd37192014-12-13 20:32:04 -08001768 rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_qs_ctr);
Paul E. McKenney97c668b2015-08-06 11:31:51 -07001769 rdp->core_needs_qs = !!(rnp->qsmask & rdp->grpmask);
Paul E. McKenney6eaef632013-03-19 10:08:37 -07001770 zero_cpu_stall_ticks(rdp);
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08001771 WRITE_ONCE(rdp->gpwrap, false);
Paul E. McKenney6eaef632013-03-19 10:08:37 -07001772 }
Paul E. McKenney48a76392014-03-11 13:02:16 -07001773 return ret;
Paul E. McKenney6eaef632013-03-19 10:08:37 -07001774}
1775
Paul E. McKenneyd34ea322013-03-19 11:10:43 -07001776static void note_gp_changes(struct rcu_state *rsp, struct rcu_data *rdp)
Paul E. McKenney6eaef632013-03-19 10:08:37 -07001777{
1778 unsigned long flags;
Paul E. McKenney48a76392014-03-11 13:02:16 -07001779 bool needwake;
Paul E. McKenney6eaef632013-03-19 10:08:37 -07001780 struct rcu_node *rnp;
1781
1782 local_irq_save(flags);
1783 rnp = rdp->mynode;
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08001784 if ((rdp->gpnum == READ_ONCE(rnp->gpnum) &&
1785 rdp->completed == READ_ONCE(rnp->completed) &&
1786 !unlikely(READ_ONCE(rdp->gpwrap))) || /* w/out lock. */
Peter Zijlstra2a67e742015-10-08 12:24:23 +02001787 !raw_spin_trylock_rcu_node(rnp)) { /* irqs already off, so later. */
Paul E. McKenney6eaef632013-03-19 10:08:37 -07001788 local_irq_restore(flags);
1789 return;
1790 }
Paul E. McKenney48a76392014-03-11 13:02:16 -07001791 needwake = __note_gp_changes(rsp, rnp, rdp);
Paul E. McKenney6eaef632013-03-19 10:08:37 -07001792 raw_spin_unlock_irqrestore(&rnp->lock, flags);
Paul E. McKenney48a76392014-03-11 13:02:16 -07001793 if (needwake)
1794 rcu_gp_kthread_wake(rsp);
Paul E. McKenney6eaef632013-03-19 10:08:37 -07001795}
1796
Paul E. McKenney0f41c0d2015-03-10 18:33:20 -07001797static void rcu_gp_slow(struct rcu_state *rsp, int delay)
1798{
1799 if (delay > 0 &&
1800 !(rsp->gpnum % (rcu_num_nodes * PER_RCU_NODE_PERIOD * delay)))
1801 schedule_timeout_uninterruptible(delay);
1802}
1803
Paul E. McKenney6eaef632013-03-19 10:08:37 -07001804/*
Paul E. McKenneyf7be8202013-08-08 18:27:52 -07001805 * Initialize a new grace period. Return 0 if no grace period required.
Paul E. McKenney7fdefc12012-06-22 11:08:41 -07001806 */
1807static int rcu_gp_init(struct rcu_state *rsp)
1808{
Paul E. McKenney0aa04b02015-01-23 21:52:37 -08001809 unsigned long oldmask;
Paul E. McKenney7fdefc12012-06-22 11:08:41 -07001810 struct rcu_data *rdp;
1811 struct rcu_node *rnp = rcu_get_root(rsp);
1812
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08001813 WRITE_ONCE(rsp->gp_activity, jiffies);
Peter Zijlstra2a67e742015-10-08 12:24:23 +02001814 raw_spin_lock_irq_rcu_node(rnp);
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08001815 if (!READ_ONCE(rsp->gp_flags)) {
Paul E. McKenneyf7be8202013-08-08 18:27:52 -07001816 /* Spurious wakeup, tell caller to go back to sleep. */
1817 raw_spin_unlock_irq(&rnp->lock);
1818 return 0;
1819 }
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08001820 WRITE_ONCE(rsp->gp_flags, 0); /* Clear all flags: New grace period. */
Paul E. McKenney7fdefc12012-06-22 11:08:41 -07001821
Paul E. McKenneyf7be8202013-08-08 18:27:52 -07001822 if (WARN_ON_ONCE(rcu_gp_in_progress(rsp))) {
1823 /*
1824 * Grace period already in progress, don't start another.
1825 * Not supposed to be able to happen.
1826 */
Paul E. McKenney7fdefc12012-06-22 11:08:41 -07001827 raw_spin_unlock_irq(&rnp->lock);
1828 return 0;
1829 }
1830
Paul E. McKenney7fdefc12012-06-22 11:08:41 -07001831 /* Advance to a new grace period and initialize state. */
Paul E. McKenney26cdfed2013-09-04 10:51:13 -07001832 record_gp_stall_check_time(rsp);
Paul E. McKenney765a3f42014-03-14 16:37:08 -07001833 /* Record GP times before starting GP, hence smp_store_release(). */
1834 smp_store_release(&rsp->gpnum, rsp->gpnum + 1);
Steven Rostedt (Red Hat)f7f7bac2013-07-12 17:18:47 -04001835 trace_rcu_grace_period(rsp->name, rsp->gpnum, TPS("start"));
Paul E. McKenney7fdefc12012-06-22 11:08:41 -07001836 raw_spin_unlock_irq(&rnp->lock);
1837
Paul E. McKenney7fdefc12012-06-22 11:08:41 -07001838 /*
Paul E. McKenney0aa04b02015-01-23 21:52:37 -08001839 * Apply per-leaf buffered online and offline operations to the
1840 * rcu_node tree. Note that this new grace period need not wait
1841 * for subsequent online CPUs, and that quiescent-state forcing
1842 * will handle subsequent offline CPUs.
1843 */
1844 rcu_for_each_leaf_node(rsp, rnp) {
Paul E. McKenney0f41c0d2015-03-10 18:33:20 -07001845 rcu_gp_slow(rsp, gp_preinit_delay);
Peter Zijlstra2a67e742015-10-08 12:24:23 +02001846 raw_spin_lock_irq_rcu_node(rnp);
Paul E. McKenney0aa04b02015-01-23 21:52:37 -08001847 if (rnp->qsmaskinit == rnp->qsmaskinitnext &&
1848 !rnp->wait_blkd_tasks) {
1849 /* Nothing to do on this leaf rcu_node structure. */
1850 raw_spin_unlock_irq(&rnp->lock);
1851 continue;
1852 }
1853
1854 /* Record old state, apply changes to ->qsmaskinit field. */
1855 oldmask = rnp->qsmaskinit;
1856 rnp->qsmaskinit = rnp->qsmaskinitnext;
1857
1858 /* If zero-ness of ->qsmaskinit changed, propagate up tree. */
1859 if (!oldmask != !rnp->qsmaskinit) {
1860 if (!oldmask) /* First online CPU for this rcu_node. */
1861 rcu_init_new_rnp(rnp);
1862 else if (rcu_preempt_has_tasks(rnp)) /* blocked tasks */
1863 rnp->wait_blkd_tasks = true;
1864 else /* Last offline CPU and can propagate. */
1865 rcu_cleanup_dead_rnp(rnp);
1866 }
1867
1868 /*
1869 * If all waited-on tasks from prior grace period are
1870 * done, and if all this rcu_node structure's CPUs are
1871 * still offline, propagate up the rcu_node tree and
1872 * clear ->wait_blkd_tasks. Otherwise, if one of this
1873 * rcu_node structure's CPUs has since come back online,
1874 * simply clear ->wait_blkd_tasks (but rcu_cleanup_dead_rnp()
1875 * checks for this, so just call it unconditionally).
1876 */
1877 if (rnp->wait_blkd_tasks &&
1878 (!rcu_preempt_has_tasks(rnp) ||
1879 rnp->qsmaskinit)) {
1880 rnp->wait_blkd_tasks = false;
1881 rcu_cleanup_dead_rnp(rnp);
1882 }
1883
1884 raw_spin_unlock_irq(&rnp->lock);
1885 }
Paul E. McKenney7fdefc12012-06-22 11:08:41 -07001886
1887 /*
1888 * Set the quiescent-state-needed bits in all the rcu_node
1889 * structures for all currently online CPUs in breadth-first order,
1890 * starting from the root rcu_node structure, relying on the layout
1891 * of the tree within the rsp->node[] array. Note that other CPUs
1892 * will access only the leaves of the hierarchy, thus seeing that no
1893 * grace period is in progress, at least until the corresponding
1894 * leaf node has been initialized. In addition, we have excluded
1895 * CPU-hotplug operations.
1896 *
1897 * The grace period cannot complete until the initialization
1898 * process finishes, because this kthread handles both.
1899 */
1900 rcu_for_each_node_breadth_first(rsp, rnp) {
Paul E. McKenney0f41c0d2015-03-10 18:33:20 -07001901 rcu_gp_slow(rsp, gp_init_delay);
Peter Zijlstra2a67e742015-10-08 12:24:23 +02001902 raw_spin_lock_irq_rcu_node(rnp);
Paul E. McKenney7fdefc12012-06-22 11:08:41 -07001903 rdp = this_cpu_ptr(rsp->rda);
1904 rcu_preempt_check_blocked_tasks(rnp);
1905 rnp->qsmask = rnp->qsmaskinit;
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08001906 WRITE_ONCE(rnp->gpnum, rsp->gpnum);
Lai Jiangshan3f47da02015-01-13 15:30:34 +08001907 if (WARN_ON_ONCE(rnp->completed != rsp->completed))
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08001908 WRITE_ONCE(rnp->completed, rsp->completed);
Paul E. McKenney7fdefc12012-06-22 11:08:41 -07001909 if (rnp == rdp->mynode)
Paul E. McKenney48a76392014-03-11 13:02:16 -07001910 (void)__note_gp_changes(rsp, rnp, rdp);
Paul E. McKenney7fdefc12012-06-22 11:08:41 -07001911 rcu_preempt_boost_start_gp(rnp);
1912 trace_rcu_grace_period_init(rsp->name, rnp->gpnum,
1913 rnp->level, rnp->grplo,
1914 rnp->grphi, rnp->qsmask);
1915 raw_spin_unlock_irq(&rnp->lock);
Paul E. McKenneybde6c3a2014-07-01 11:26:57 -07001916 cond_resched_rcu_qs();
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08001917 WRITE_ONCE(rsp->gp_activity, jiffies);
Paul E. McKenney7fdefc12012-06-22 11:08:41 -07001918 }
1919
Paul E. McKenney7fdefc12012-06-22 11:08:41 -07001920 return 1;
1921}
1922
1923/*
Paul E. McKenneyb9a425c2015-07-01 13:50:28 -07001924 * Helper function for wait_event_interruptible_timeout() wakeup
1925 * at force-quiescent-state time.
1926 */
1927static bool rcu_gp_fqs_check_wake(struct rcu_state *rsp, int *gfp)
1928{
1929 struct rcu_node *rnp = rcu_get_root(rsp);
1930
1931 /* Someone like call_rcu() requested a force-quiescent-state scan. */
1932 *gfp = READ_ONCE(rsp->gp_flags);
1933 if (*gfp & RCU_GP_FLAG_FQS)
1934 return true;
1935
1936 /* The current grace period has completed. */
1937 if (!READ_ONCE(rnp->qsmask) && !rcu_preempt_blocked_readers_cgp(rnp))
1938 return true;
1939
1940 return false;
1941}
1942
1943/*
Paul E. McKenney4cdfc1752012-06-22 17:06:26 -07001944 * Do one round of quiescent-state forcing.
1945 */
Petr Mladek77f81fe2015-09-09 12:09:49 -07001946static void rcu_gp_fqs(struct rcu_state *rsp, bool first_time)
Paul E. McKenney4cdfc1752012-06-22 17:06:26 -07001947{
Paul E. McKenney217af2a2013-06-21 15:39:06 -07001948 bool isidle = false;
1949 unsigned long maxj;
Paul E. McKenney4cdfc1752012-06-22 17:06:26 -07001950 struct rcu_node *rnp = rcu_get_root(rsp);
1951
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08001952 WRITE_ONCE(rsp->gp_activity, jiffies);
Paul E. McKenney4cdfc1752012-06-22 17:06:26 -07001953 rsp->n_force_qs++;
Petr Mladek77f81fe2015-09-09 12:09:49 -07001954 if (first_time) {
Paul E. McKenney4cdfc1752012-06-22 17:06:26 -07001955 /* Collect dyntick-idle snapshots. */
Paul E. McKenney0edd1b12013-06-21 16:37:22 -07001956 if (is_sysidle_rcu_state(rsp)) {
Pranith Kumare02b2ed2014-07-09 00:08:17 -04001957 isidle = true;
Paul E. McKenney0edd1b12013-06-21 16:37:22 -07001958 maxj = jiffies - ULONG_MAX / 4;
1959 }
Paul E. McKenney217af2a2013-06-21 15:39:06 -07001960 force_qs_rnp(rsp, dyntick_save_progress_counter,
1961 &isidle, &maxj);
Paul E. McKenney0edd1b12013-06-21 16:37:22 -07001962 rcu_sysidle_report_gp(rsp, isidle, maxj);
Paul E. McKenney4cdfc1752012-06-22 17:06:26 -07001963 } else {
1964 /* Handle dyntick-idle and offline CPUs. */
Paul E. McKenney675da672015-02-23 15:57:07 -08001965 isidle = true;
Paul E. McKenney217af2a2013-06-21 15:39:06 -07001966 force_qs_rnp(rsp, rcu_implicit_dynticks_qs, &isidle, &maxj);
Paul E. McKenney4cdfc1752012-06-22 17:06:26 -07001967 }
1968 /* Clear flag to prevent immediate re-entry. */
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08001969 if (READ_ONCE(rsp->gp_flags) & RCU_GP_FLAG_FQS) {
Peter Zijlstra2a67e742015-10-08 12:24:23 +02001970 raw_spin_lock_irq_rcu_node(rnp);
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08001971 WRITE_ONCE(rsp->gp_flags,
1972 READ_ONCE(rsp->gp_flags) & ~RCU_GP_FLAG_FQS);
Paul E. McKenney4cdfc1752012-06-22 17:06:26 -07001973 raw_spin_unlock_irq(&rnp->lock);
1974 }
Paul E. McKenney4cdfc1752012-06-22 17:06:26 -07001975}
1976
1977/*
Paul E. McKenney7fdefc12012-06-22 11:08:41 -07001978 * Clean up after the old grace period.
1979 */
Paul E. McKenney4cdfc1752012-06-22 17:06:26 -07001980static void rcu_gp_cleanup(struct rcu_state *rsp)
Paul E. McKenney7fdefc12012-06-22 11:08:41 -07001981{
1982 unsigned long gp_duration;
Paul E. McKenney48a76392014-03-11 13:02:16 -07001983 bool needgp = false;
Paul E. McKenneydae6e642013-02-10 20:48:58 -08001984 int nocb = 0;
Paul E. McKenney7fdefc12012-06-22 11:08:41 -07001985 struct rcu_data *rdp;
1986 struct rcu_node *rnp = rcu_get_root(rsp);
1987
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08001988 WRITE_ONCE(rsp->gp_activity, jiffies);
Peter Zijlstra2a67e742015-10-08 12:24:23 +02001989 raw_spin_lock_irq_rcu_node(rnp);
Paul E. McKenney7fdefc12012-06-22 11:08:41 -07001990 gp_duration = jiffies - rsp->gp_start;
1991 if (gp_duration > rsp->gp_max)
1992 rsp->gp_max = gp_duration;
1993
1994 /*
1995 * We know the grace period is complete, but to everyone else
1996 * it appears to still be ongoing. But it is also the case
1997 * that to everyone else it looks like there is nothing that
1998 * they can do to advance the grace period. It is therefore
1999 * safe for us to drop the lock in order to mark the grace
2000 * period as completed in all of the rcu_node structures.
Paul E. McKenney7fdefc12012-06-22 11:08:41 -07002001 */
Paul E. McKenney5d4b8652012-07-07 07:56:57 -07002002 raw_spin_unlock_irq(&rnp->lock);
Paul E. McKenney7fdefc12012-06-22 11:08:41 -07002003
Paul E. McKenney5d4b8652012-07-07 07:56:57 -07002004 /*
2005 * Propagate new ->completed value to rcu_node structures so
2006 * that other CPUs don't have to wait until the start of the next
2007 * grace period to process their callbacks. This also avoids
2008 * some nasty RCU grace-period initialization races by forcing
2009 * the end of the current grace period to be completely recorded in
2010 * all of the rcu_node structures before the beginning of the next
2011 * grace period is recorded in any of the rcu_node structures.
2012 */
2013 rcu_for_each_node_breadth_first(rsp, rnp) {
Peter Zijlstra2a67e742015-10-08 12:24:23 +02002014 raw_spin_lock_irq_rcu_node(rnp);
Paul E. McKenney5c60d252015-02-09 05:37:47 -08002015 WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp));
2016 WARN_ON_ONCE(rnp->qsmask);
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08002017 WRITE_ONCE(rnp->completed, rsp->gpnum);
Paul E. McKenneyb11cc572012-12-17 14:21:14 -08002018 rdp = this_cpu_ptr(rsp->rda);
2019 if (rnp == rdp->mynode)
Paul E. McKenney48a76392014-03-11 13:02:16 -07002020 needgp = __note_gp_changes(rsp, rnp, rdp) || needgp;
Paul E. McKenney78e4bc32013-09-24 15:04:06 -07002021 /* smp_mb() provided by prior unlock-lock pair. */
Paul E. McKenney0446be42012-12-30 15:21:01 -08002022 nocb += rcu_future_gp_cleanup(rsp, rnp);
Paul E. McKenney5d4b8652012-07-07 07:56:57 -07002023 raw_spin_unlock_irq(&rnp->lock);
Paul E. McKenneybde6c3a2014-07-01 11:26:57 -07002024 cond_resched_rcu_qs();
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08002025 WRITE_ONCE(rsp->gp_activity, jiffies);
Paul E. McKenney0f41c0d2015-03-10 18:33:20 -07002026 rcu_gp_slow(rsp, gp_cleanup_delay);
Paul E. McKenney7fdefc12012-06-22 11:08:41 -07002027 }
Paul E. McKenney5d4b8652012-07-07 07:56:57 -07002028 rnp = rcu_get_root(rsp);
Peter Zijlstra2a67e742015-10-08 12:24:23 +02002029 raw_spin_lock_irq_rcu_node(rnp); /* Order GP before ->completed update. */
Paul E. McKenneydae6e642013-02-10 20:48:58 -08002030 rcu_nocb_gp_set(rnp, nocb);
Paul E. McKenney7fdefc12012-06-22 11:08:41 -07002031
Paul E. McKenney765a3f42014-03-14 16:37:08 -07002032 /* Declare grace period done. */
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08002033 WRITE_ONCE(rsp->completed, rsp->gpnum);
Steven Rostedt (Red Hat)f7f7bac2013-07-12 17:18:47 -04002034 trace_rcu_grace_period(rsp->name, rsp->completed, TPS("end"));
Petr Mladek77f81fe2015-09-09 12:09:49 -07002035 rsp->gp_state = RCU_GP_IDLE;
Paul E. McKenney5d4b8652012-07-07 07:56:57 -07002036 rdp = this_cpu_ptr(rsp->rda);
Paul E. McKenney48a76392014-03-11 13:02:16 -07002037 /* Advance CBs to reduce false positives below. */
2038 needgp = rcu_advance_cbs(rsp, rnp, rdp) || needgp;
2039 if (needgp || cpu_needs_another_gp(rsp, rdp)) {
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08002040 WRITE_ONCE(rsp->gp_flags, RCU_GP_FLAG_INIT);
Paul E. McKenneybb311ec2013-08-09 16:02:09 -07002041 trace_rcu_grace_period(rsp->name,
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08002042 READ_ONCE(rsp->gpnum),
Paul E. McKenneybb311ec2013-08-09 16:02:09 -07002043 TPS("newreq"));
2044 }
Paul E. McKenney7fdefc12012-06-22 11:08:41 -07002045 raw_spin_unlock_irq(&rnp->lock);
Paul E. McKenney7fdefc12012-06-22 11:08:41 -07002046}
2047
2048/*
Paul E. McKenneyb3dbec72012-06-18 18:36:08 -07002049 * Body of kthread that handles grace periods.
2050 */
Paul E. McKenney755609a2012-06-19 17:18:20 -07002051static int __noreturn rcu_gp_kthread(void *arg)
Paul E. McKenneyb3dbec72012-06-18 18:36:08 -07002052{
Petr Mladek77f81fe2015-09-09 12:09:49 -07002053 bool first_gp_fqs;
Paul E. McKenney88d6df62013-08-08 21:44:31 -07002054 int gf;
Paul E. McKenneyd40011f2012-06-26 20:45:57 -07002055 unsigned long j;
Paul E. McKenney4cdfc1752012-06-22 17:06:26 -07002056 int ret;
Paul E. McKenneyb3dbec72012-06-18 18:36:08 -07002057 struct rcu_state *rsp = arg;
Paul E. McKenney7fdefc12012-06-22 11:08:41 -07002058 struct rcu_node *rnp = rcu_get_root(rsp);
Paul E. McKenneyb3dbec72012-06-18 18:36:08 -07002059
Paul E. McKenney58719682015-02-24 11:05:36 -08002060 rcu_bind_gp_kthread();
Paul E. McKenneyb3dbec72012-06-18 18:36:08 -07002061 for (;;) {
2062
2063 /* Handle grace-period start. */
Paul E. McKenneyb3dbec72012-06-18 18:36:08 -07002064 for (;;) {
Paul E. McKenney63c4db72013-08-09 12:19:29 -07002065 trace_rcu_grace_period(rsp->name,
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08002066 READ_ONCE(rsp->gpnum),
Paul E. McKenney63c4db72013-08-09 12:19:29 -07002067 TPS("reqwait"));
Paul E. McKenneyafea2272014-03-12 07:10:41 -07002068 rsp->gp_state = RCU_GP_WAIT_GPS;
Paul E. McKenney4cdfc1752012-06-22 17:06:26 -07002069 wait_event_interruptible(rsp->gp_wq,
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08002070 READ_ONCE(rsp->gp_flags) &
Paul E. McKenney4cdfc1752012-06-22 17:06:26 -07002071 RCU_GP_FLAG_INIT);
Paul E. McKenney319362c2015-05-19 14:16:52 -07002072 rsp->gp_state = RCU_GP_DONE_GPS;
Paul E. McKenney78e4bc32013-09-24 15:04:06 -07002073 /* Locking provides needed memory barrier. */
Paul E. McKenneyf7be8202013-08-08 18:27:52 -07002074 if (rcu_gp_init(rsp))
Paul E. McKenneyb3dbec72012-06-18 18:36:08 -07002075 break;
Paul E. McKenneybde6c3a2014-07-01 11:26:57 -07002076 cond_resched_rcu_qs();
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08002077 WRITE_ONCE(rsp->gp_activity, jiffies);
Paul E. McKenney73a860c2014-08-14 10:28:23 -07002078 WARN_ON(signal_pending(current));
Paul E. McKenney63c4db72013-08-09 12:19:29 -07002079 trace_rcu_grace_period(rsp->name,
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08002080 READ_ONCE(rsp->gpnum),
Paul E. McKenney63c4db72013-08-09 12:19:29 -07002081 TPS("reqwaitsig"));
Paul E. McKenneyb3dbec72012-06-18 18:36:08 -07002082 }
Paul E. McKenneycabc49c2012-06-20 17:07:14 -07002083
Paul E. McKenney4cdfc1752012-06-22 17:06:26 -07002084 /* Handle quiescent-state forcing. */
Petr Mladek77f81fe2015-09-09 12:09:49 -07002085 first_gp_fqs = true;
Paul E. McKenneyd40011f2012-06-26 20:45:57 -07002086 j = jiffies_till_first_fqs;
2087 if (j > HZ) {
2088 j = HZ;
2089 jiffies_till_first_fqs = HZ;
2090 }
Paul E. McKenney88d6df62013-08-08 21:44:31 -07002091 ret = 0;
Paul E. McKenneycabc49c2012-06-20 17:07:14 -07002092 for (;;) {
Paul E. McKenney88d6df62013-08-08 21:44:31 -07002093 if (!ret)
2094 rsp->jiffies_force_qs = jiffies + j;
Paul E. McKenney63c4db72013-08-09 12:19:29 -07002095 trace_rcu_grace_period(rsp->name,
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08002096 READ_ONCE(rsp->gpnum),
Paul E. McKenney63c4db72013-08-09 12:19:29 -07002097 TPS("fqswait"));
Paul E. McKenneyafea2272014-03-12 07:10:41 -07002098 rsp->gp_state = RCU_GP_WAIT_FQS;
Paul E. McKenney4cdfc1752012-06-22 17:06:26 -07002099 ret = wait_event_interruptible_timeout(rsp->gp_wq,
Paul E. McKenneyb9a425c2015-07-01 13:50:28 -07002100 rcu_gp_fqs_check_wake(rsp, &gf), j);
Paul E. McKenney32bb1c72015-07-02 12:27:31 -07002101 rsp->gp_state = RCU_GP_DOING_FQS;
Paul E. McKenney78e4bc32013-09-24 15:04:06 -07002102 /* Locking provides needed memory barriers. */
Paul E. McKenney4cdfc1752012-06-22 17:06:26 -07002103 /* If grace period done, leave loop. */
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08002104 if (!READ_ONCE(rnp->qsmask) &&
Paul E. McKenney4cdfc1752012-06-22 17:06:26 -07002105 !rcu_preempt_blocked_readers_cgp(rnp))
Paul E. McKenneycabc49c2012-06-20 17:07:14 -07002106 break;
Paul E. McKenney4cdfc1752012-06-22 17:06:26 -07002107 /* If time for quiescent-state forcing, do it. */
Paul E. McKenney88d6df62013-08-08 21:44:31 -07002108 if (ULONG_CMP_GE(jiffies, rsp->jiffies_force_qs) ||
2109 (gf & RCU_GP_FLAG_FQS)) {
Paul E. McKenney63c4db72013-08-09 12:19:29 -07002110 trace_rcu_grace_period(rsp->name,
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08002111 READ_ONCE(rsp->gpnum),
Paul E. McKenney63c4db72013-08-09 12:19:29 -07002112 TPS("fqsstart"));
Petr Mladek77f81fe2015-09-09 12:09:49 -07002113 rcu_gp_fqs(rsp, first_gp_fqs);
2114 first_gp_fqs = false;
Paul E. McKenney63c4db72013-08-09 12:19:29 -07002115 trace_rcu_grace_period(rsp->name,
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08002116 READ_ONCE(rsp->gpnum),
Paul E. McKenney63c4db72013-08-09 12:19:29 -07002117 TPS("fqsend"));
Paul E. McKenneybde6c3a2014-07-01 11:26:57 -07002118 cond_resched_rcu_qs();
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08002119 WRITE_ONCE(rsp->gp_activity, jiffies);
Paul E. McKenney4cdfc1752012-06-22 17:06:26 -07002120 } else {
2121 /* Deal with stray signal. */
Paul E. McKenneybde6c3a2014-07-01 11:26:57 -07002122 cond_resched_rcu_qs();
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08002123 WRITE_ONCE(rsp->gp_activity, jiffies);
Paul E. McKenney73a860c2014-08-14 10:28:23 -07002124 WARN_ON(signal_pending(current));
Paul E. McKenney63c4db72013-08-09 12:19:29 -07002125 trace_rcu_grace_period(rsp->name,
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08002126 READ_ONCE(rsp->gpnum),
Paul E. McKenney63c4db72013-08-09 12:19:29 -07002127 TPS("fqswaitsig"));
Paul E. McKenney4cdfc1752012-06-22 17:06:26 -07002128 }
Paul E. McKenneyd40011f2012-06-26 20:45:57 -07002129 j = jiffies_till_next_fqs;
2130 if (j > HZ) {
2131 j = HZ;
2132 jiffies_till_next_fqs = HZ;
2133 } else if (j < 1) {
2134 j = 1;
2135 jiffies_till_next_fqs = 1;
2136 }
Paul E. McKenneycabc49c2012-06-20 17:07:14 -07002137 }
Paul E. McKenney4cdfc1752012-06-22 17:06:26 -07002138
2139 /* Handle grace-period end. */
Paul E. McKenney319362c2015-05-19 14:16:52 -07002140 rsp->gp_state = RCU_GP_CLEANUP;
Paul E. McKenney4cdfc1752012-06-22 17:06:26 -07002141 rcu_gp_cleanup(rsp);
Paul E. McKenney319362c2015-05-19 14:16:52 -07002142 rsp->gp_state = RCU_GP_CLEANED;
Paul E. McKenneyb3dbec72012-06-18 18:36:08 -07002143 }
Paul E. McKenneyb3dbec72012-06-18 18:36:08 -07002144}
2145
2146/*
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002147 * Start a new RCU grace period if warranted, re-initializing the hierarchy
2148 * in preparation for detecting the next grace period. The caller must hold
Paul E. McKenneyb8462082012-12-29 22:04:18 -08002149 * the root node's ->lock and hard irqs must be disabled.
Paul E. McKenneye5601402012-01-07 11:03:57 -08002150 *
2151 * Note that it is legal for a dying CPU (which is marked as offline) to
2152 * invoke this function. This can happen when the dying CPU reports its
2153 * quiescent state.
Paul E. McKenney48a76392014-03-11 13:02:16 -07002154 *
2155 * Returns true if the grace-period kthread must be awakened.
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002156 */
Paul E. McKenney48a76392014-03-11 13:02:16 -07002157static bool
Paul E. McKenney910ee452012-12-31 02:24:21 -08002158rcu_start_gp_advanced(struct rcu_state *rsp, struct rcu_node *rnp,
2159 struct rcu_data *rdp)
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002160{
Paul E. McKenneyb8462082012-12-29 22:04:18 -08002161 if (!rsp->gp_kthread || !cpu_needs_another_gp(rsp, rdp)) {
Paul E. McKenneyb32e9eb2009-11-12 22:35:03 -08002162 /*
Paul E. McKenneyb3dbec72012-06-18 18:36:08 -07002163 * Either we have not yet spawned the grace-period
Paul E. McKenney62da1922012-09-20 16:02:49 -07002164 * task, this CPU does not need another grace period,
2165 * or a grace period is already in progress.
Paul E. McKenneyb3dbec72012-06-18 18:36:08 -07002166 * Either way, don't start a new grace period.
Paul E. McKenneyb32e9eb2009-11-12 22:35:03 -08002167 */
Paul E. McKenney48a76392014-03-11 13:02:16 -07002168 return false;
Paul E. McKenneyafe24b12011-08-24 16:52:09 -07002169 }
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08002170 WRITE_ONCE(rsp->gp_flags, RCU_GP_FLAG_INIT);
2171 trace_rcu_grace_period(rsp->name, READ_ONCE(rsp->gpnum),
Paul E. McKenneybb311ec2013-08-09 16:02:09 -07002172 TPS("newreq"));
Paul E. McKenney62da1922012-09-20 16:02:49 -07002173
Steven Rostedt016a8d52013-05-28 17:32:53 -04002174 /*
2175 * We can't do wakeups while holding the rnp->lock, as that
Paul E. McKenney1eafd312013-06-20 13:50:40 -07002176 * could cause possible deadlocks with the rq->lock. Defer
Paul E. McKenney48a76392014-03-11 13:02:16 -07002177 * the wakeup to our caller.
Steven Rostedt016a8d52013-05-28 17:32:53 -04002178 */
Paul E. McKenney48a76392014-03-11 13:02:16 -07002179 return true;
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002180}
2181
2182/*
Paul E. McKenney910ee452012-12-31 02:24:21 -08002183 * Similar to rcu_start_gp_advanced(), but also advance the calling CPU's
2184 * callbacks. Note that rcu_start_gp_advanced() cannot do this because it
2185 * is invoked indirectly from rcu_advance_cbs(), which would result in
2186 * endless recursion -- or would do so if it wasn't for the self-deadlock
2187 * that is encountered beforehand.
Paul E. McKenney48a76392014-03-11 13:02:16 -07002188 *
2189 * Returns true if the grace-period kthread needs to be awakened.
Paul E. McKenney910ee452012-12-31 02:24:21 -08002190 */
Paul E. McKenney48a76392014-03-11 13:02:16 -07002191static bool rcu_start_gp(struct rcu_state *rsp)
Paul E. McKenney910ee452012-12-31 02:24:21 -08002192{
2193 struct rcu_data *rdp = this_cpu_ptr(rsp->rda);
2194 struct rcu_node *rnp = rcu_get_root(rsp);
Paul E. McKenney48a76392014-03-11 13:02:16 -07002195 bool ret = false;
Paul E. McKenney910ee452012-12-31 02:24:21 -08002196
2197 /*
2198 * If there is no grace period in progress right now, any
2199 * callbacks we have up to this point will be satisfied by the
2200 * next grace period. Also, advancing the callbacks reduces the
2201 * probability of false positives from cpu_needs_another_gp()
2202 * resulting in pointless grace periods. So, advance callbacks
2203 * then start the grace period!
2204 */
Paul E. McKenney48a76392014-03-11 13:02:16 -07002205 ret = rcu_advance_cbs(rsp, rnp, rdp) || ret;
2206 ret = rcu_start_gp_advanced(rsp, rnp, rdp) || ret;
2207 return ret;
Paul E. McKenney910ee452012-12-31 02:24:21 -08002208}
2209
2210/*
Paul E. McKenneyd3f6bad2009-12-02 12:10:13 -08002211 * Report a full set of quiescent states to the specified rcu_state
2212 * data structure. This involves cleaning up after the prior grace
2213 * period and letting rcu_start_gp() start up the next grace period
Paul E. McKenneyb8462082012-12-29 22:04:18 -08002214 * if one is needed. Note that the caller must hold rnp->lock, which
2215 * is released before return.
Paul E. McKenneyf41d9112009-08-22 13:56:52 -07002216 */
Paul E. McKenneyd3f6bad2009-12-02 12:10:13 -08002217static void rcu_report_qs_rsp(struct rcu_state *rsp, unsigned long flags)
Paul E. McKenneyfc2219d42009-09-23 09:50:41 -07002218 __releases(rcu_get_root(rsp)->lock)
Paul E. McKenneyf41d9112009-08-22 13:56:52 -07002219{
Paul E. McKenneyfc2219d42009-09-23 09:50:41 -07002220 WARN_ON_ONCE(!rcu_gp_in_progress(rsp));
Paul E. McKenneycd73ca22015-03-16 11:53:52 -07002221 WRITE_ONCE(rsp->gp_flags, READ_ONCE(rsp->gp_flags) | RCU_GP_FLAG_FQS);
Paul E. McKenneycabc49c2012-06-20 17:07:14 -07002222 raw_spin_unlock_irqrestore(&rcu_get_root(rsp)->lock, flags);
Pranith Kumar2aa792e2014-08-12 13:07:47 -04002223 rcu_gp_kthread_wake(rsp);
Paul E. McKenneyf41d9112009-08-22 13:56:52 -07002224}
2225
2226/*
Paul E. McKenneyd3f6bad2009-12-02 12:10:13 -08002227 * Similar to rcu_report_qs_rdp(), for which it is a helper function.
2228 * Allows quiescent states for a group of CPUs to be reported at one go
2229 * to the specified rcu_node structure, though all the CPUs in the group
Paul E. McKenney654e9532015-03-15 09:19:35 -07002230 * must be represented by the same rcu_node structure (which need not be a
2231 * leaf rcu_node structure, though it often will be). The gps parameter
2232 * is the grace-period snapshot, which means that the quiescent states
2233 * are valid only if rnp->gpnum is equal to gps. That structure's lock
2234 * must be held upon entry, and it is released before return.
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002235 */
2236static void
Paul E. McKenneyd3f6bad2009-12-02 12:10:13 -08002237rcu_report_qs_rnp(unsigned long mask, struct rcu_state *rsp,
Paul E. McKenney654e9532015-03-15 09:19:35 -07002238 struct rcu_node *rnp, unsigned long gps, unsigned long flags)
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002239 __releases(rnp->lock)
2240{
Paul E. McKenney654e9532015-03-15 09:19:35 -07002241 unsigned long oldmask = 0;
Paul E. McKenney28ecd582009-09-18 09:50:17 -07002242 struct rcu_node *rnp_c;
2243
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002244 /* Walk up the rcu_node hierarchy. */
2245 for (;;) {
Paul E. McKenney654e9532015-03-15 09:19:35 -07002246 if (!(rnp->qsmask & mask) || rnp->gpnum != gps) {
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002247
Paul E. McKenney654e9532015-03-15 09:19:35 -07002248 /*
2249 * Our bit has already been cleared, or the
2250 * relevant grace period is already over, so done.
2251 */
Paul E. McKenney1304afb2010-02-22 17:05:02 -08002252 raw_spin_unlock_irqrestore(&rnp->lock, flags);
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002253 return;
2254 }
Paul E. McKenney654e9532015-03-15 09:19:35 -07002255 WARN_ON_ONCE(oldmask); /* Any child must be all zeroed! */
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002256 rnp->qsmask &= ~mask;
Paul E. McKenneyd4c08f22011-06-25 06:36:56 -07002257 trace_rcu_quiescent_state_report(rsp->name, rnp->gpnum,
2258 mask, rnp->qsmask, rnp->level,
2259 rnp->grplo, rnp->grphi,
2260 !!rnp->gp_tasks);
Paul E. McKenney27f4d282011-02-07 12:47:15 -08002261 if (rnp->qsmask != 0 || rcu_preempt_blocked_readers_cgp(rnp)) {
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002262
2263 /* Other bits still set at this level, so done. */
Paul E. McKenney1304afb2010-02-22 17:05:02 -08002264 raw_spin_unlock_irqrestore(&rnp->lock, flags);
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002265 return;
2266 }
2267 mask = rnp->grpmask;
2268 if (rnp->parent == NULL) {
2269
2270 /* No more levels. Exit loop holding root lock. */
2271
2272 break;
2273 }
Paul E. McKenney1304afb2010-02-22 17:05:02 -08002274 raw_spin_unlock_irqrestore(&rnp->lock, flags);
Paul E. McKenney28ecd582009-09-18 09:50:17 -07002275 rnp_c = rnp;
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002276 rnp = rnp->parent;
Peter Zijlstra2a67e742015-10-08 12:24:23 +02002277 raw_spin_lock_irqsave_rcu_node(rnp, flags);
Paul E. McKenney654e9532015-03-15 09:19:35 -07002278 oldmask = rnp_c->qsmask;
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002279 }
2280
2281 /*
2282 * Get here if we are the last CPU to pass through a quiescent
Paul E. McKenneyd3f6bad2009-12-02 12:10:13 -08002283 * state for this grace period. Invoke rcu_report_qs_rsp()
Paul E. McKenneyf41d9112009-08-22 13:56:52 -07002284 * to clean up and start the next grace period if one is needed.
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002285 */
Paul E. McKenneyd3f6bad2009-12-02 12:10:13 -08002286 rcu_report_qs_rsp(rsp, flags); /* releases rnp->lock. */
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002287}
2288
2289/*
Paul E. McKenneycc99a312015-02-23 08:59:29 -08002290 * Record a quiescent state for all tasks that were previously queued
2291 * on the specified rcu_node structure and that were blocking the current
2292 * RCU grace period. The caller must hold the specified rnp->lock with
2293 * irqs disabled, and this lock is released upon return, but irqs remain
2294 * disabled.
2295 */
Paul E. McKenney0aa04b02015-01-23 21:52:37 -08002296static void rcu_report_unblock_qs_rnp(struct rcu_state *rsp,
Paul E. McKenneycc99a312015-02-23 08:59:29 -08002297 struct rcu_node *rnp, unsigned long flags)
2298 __releases(rnp->lock)
2299{
Paul E. McKenney654e9532015-03-15 09:19:35 -07002300 unsigned long gps;
Paul E. McKenneycc99a312015-02-23 08:59:29 -08002301 unsigned long mask;
2302 struct rcu_node *rnp_p;
2303
Paul E. McKenneya77da142015-03-08 14:52:27 -07002304 if (rcu_state_p == &rcu_sched_state || rsp != rcu_state_p ||
2305 rnp->qsmask != 0 || rcu_preempt_blocked_readers_cgp(rnp)) {
Paul E. McKenneycc99a312015-02-23 08:59:29 -08002306 raw_spin_unlock_irqrestore(&rnp->lock, flags);
2307 return; /* Still need more quiescent states! */
2308 }
2309
2310 rnp_p = rnp->parent;
2311 if (rnp_p == NULL) {
2312 /*
Paul E. McKenneya77da142015-03-08 14:52:27 -07002313 * Only one rcu_node structure in the tree, so don't
2314 * try to report up to its nonexistent parent!
Paul E. McKenneycc99a312015-02-23 08:59:29 -08002315 */
2316 rcu_report_qs_rsp(rsp, flags);
2317 return;
2318 }
2319
Paul E. McKenney654e9532015-03-15 09:19:35 -07002320 /* Report up the rest of the hierarchy, tracking current ->gpnum. */
2321 gps = rnp->gpnum;
Paul E. McKenneycc99a312015-02-23 08:59:29 -08002322 mask = rnp->grpmask;
2323 raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */
Peter Zijlstra2a67e742015-10-08 12:24:23 +02002324 raw_spin_lock_rcu_node(rnp_p); /* irqs already disabled. */
Paul E. McKenney654e9532015-03-15 09:19:35 -07002325 rcu_report_qs_rnp(mask, rsp, rnp_p, gps, flags);
Paul E. McKenneycc99a312015-02-23 08:59:29 -08002326}
2327
2328/*
Paul E. McKenneyd3f6bad2009-12-02 12:10:13 -08002329 * Record a quiescent state for the specified CPU to that CPU's rcu_data
2330 * structure. This must be either called from the specified CPU, or
2331 * called when the specified CPU is known to be offline (and when it is
2332 * also known that no other CPU is concurrently trying to help the offline
2333 * CPU). The lastcomp argument is used to make sure we are still in the
2334 * grace period of interest. We don't want to end the current grace period
2335 * based on quiescent states detected in an earlier grace period!
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002336 */
2337static void
Paul E. McKenneyd7d6a112012-08-21 15:00:05 -07002338rcu_report_qs_rdp(int cpu, struct rcu_state *rsp, struct rcu_data *rdp)
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002339{
2340 unsigned long flags;
2341 unsigned long mask;
Paul E. McKenney48a76392014-03-11 13:02:16 -07002342 bool needwake;
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002343 struct rcu_node *rnp;
2344
2345 rnp = rdp->mynode;
Peter Zijlstra2a67e742015-10-08 12:24:23 +02002346 raw_spin_lock_irqsave_rcu_node(rnp, flags);
Paul E. McKenney5b74c452015-08-06 15:16:57 -07002347 if ((rdp->cpu_no_qs.b.norm &&
Paul E. McKenney5cd37192014-12-13 20:32:04 -08002348 rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr)) ||
2349 rdp->gpnum != rnp->gpnum || rnp->completed == rnp->gpnum ||
2350 rdp->gpwrap) {
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002351
2352 /*
Paul E. McKenneye4cc1f22011-06-27 00:17:43 -07002353 * The grace period in which this quiescent state was
2354 * recorded has ended, so don't report it upwards.
2355 * We will instead need a new quiescent state that lies
2356 * within the current grace period.
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002357 */
Paul E. McKenney5b74c452015-08-06 15:16:57 -07002358 rdp->cpu_no_qs.b.norm = true; /* need qs for new gp. */
Paul E. McKenney5cd37192014-12-13 20:32:04 -08002359 rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_qs_ctr);
Paul E. McKenney1304afb2010-02-22 17:05:02 -08002360 raw_spin_unlock_irqrestore(&rnp->lock, flags);
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002361 return;
2362 }
2363 mask = rdp->grpmask;
2364 if ((rnp->qsmask & mask) == 0) {
Paul E. McKenney1304afb2010-02-22 17:05:02 -08002365 raw_spin_unlock_irqrestore(&rnp->lock, flags);
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002366 } else {
Paul E. McKenney97c668b2015-08-06 11:31:51 -07002367 rdp->core_needs_qs = 0;
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002368
2369 /*
2370 * This GP can't end until cpu checks in, so all of our
2371 * callbacks can be processed during the next GP.
2372 */
Paul E. McKenney48a76392014-03-11 13:02:16 -07002373 needwake = rcu_accelerate_cbs(rsp, rnp, rdp);
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002374
Paul E. McKenney654e9532015-03-15 09:19:35 -07002375 rcu_report_qs_rnp(mask, rsp, rnp, rnp->gpnum, flags);
2376 /* ^^^ Released rnp->lock */
Paul E. McKenney48a76392014-03-11 13:02:16 -07002377 if (needwake)
2378 rcu_gp_kthread_wake(rsp);
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002379 }
2380}
2381
2382/*
2383 * Check to see if there is a new grace period of which this CPU
2384 * is not yet aware, and if so, set up local rcu_data state for it.
2385 * Otherwise, see if this CPU has just passed through its first
2386 * quiescent state for this grace period, and record that fact if so.
2387 */
2388static void
2389rcu_check_quiescent_state(struct rcu_state *rsp, struct rcu_data *rdp)
2390{
Paul E. McKenney05eb5522013-03-19 12:38:24 -07002391 /* Check for grace-period ends and beginnings. */
2392 note_gp_changes(rsp, rdp);
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002393
2394 /*
2395 * Does this CPU still need to do its part for current grace period?
2396 * If no, return and let the other CPUs do their part as well.
2397 */
Paul E. McKenney97c668b2015-08-06 11:31:51 -07002398 if (!rdp->core_needs_qs)
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002399 return;
2400
2401 /*
2402 * Was there a quiescent state since the beginning of the grace
2403 * period? If no, then exit and wait for the next call.
2404 */
Paul E. McKenney5b74c452015-08-06 15:16:57 -07002405 if (rdp->cpu_no_qs.b.norm &&
Paul E. McKenney5cd37192014-12-13 20:32:04 -08002406 rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr))
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002407 return;
2408
Paul E. McKenneyd3f6bad2009-12-02 12:10:13 -08002409 /*
2410 * Tell RCU we are done (but rcu_report_qs_rdp() will be the
2411 * judge of that).
2412 */
Paul E. McKenneyd7d6a112012-08-21 15:00:05 -07002413 rcu_report_qs_rdp(rdp->cpu, rsp, rdp);
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002414}
2415
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002416/*
Paul E. McKenneyb1420f12012-03-01 13:18:08 -08002417 * Send the specified CPU's RCU callbacks to the orphanage. The
2418 * specified CPU must be offline, and the caller must hold the
Paul E. McKenney7b2e6012012-10-08 10:54:03 -07002419 * ->orphan_lock.
Paul E. McKenneye74f4c42009-10-06 21:48:17 -07002420 */
Paul E. McKenneyb1420f12012-03-01 13:18:08 -08002421static void
2422rcu_send_cbs_to_orphanage(int cpu, struct rcu_state *rsp,
2423 struct rcu_node *rnp, struct rcu_data *rdp)
Paul E. McKenneye74f4c42009-10-06 21:48:17 -07002424{
Paul E. McKenney3fbfbf72012-08-19 21:35:53 -07002425 /* No-CBs CPUs do not have orphanable callbacks. */
Paul E. McKenneyea463512015-03-03 14:05:26 -08002426 if (!IS_ENABLED(CONFIG_HOTPLUG_CPU) || rcu_is_nocb_cpu(rdp->cpu))
Paul E. McKenney3fbfbf72012-08-19 21:35:53 -07002427 return;
2428
Paul E. McKenneyb1420f12012-03-01 13:18:08 -08002429 /*
2430 * Orphan the callbacks. First adjust the counts. This is safe
Paul E. McKenneyabfd6e52012-09-20 16:59:47 -07002431 * because _rcu_barrier() excludes CPU-hotplug operations, so it
2432 * cannot be running now. Thus no memory barrier is required.
Paul E. McKenneyb1420f12012-03-01 13:18:08 -08002433 */
Paul E. McKenneya50c3af2012-01-10 17:52:31 -08002434 if (rdp->nxtlist != NULL) {
Paul E. McKenneyb1420f12012-03-01 13:18:08 -08002435 rsp->qlen_lazy += rdp->qlen_lazy;
2436 rsp->qlen += rdp->qlen;
2437 rdp->n_cbs_orphaned += rdp->qlen;
Paul E. McKenneya50c3af2012-01-10 17:52:31 -08002438 rdp->qlen_lazy = 0;
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08002439 WRITE_ONCE(rdp->qlen, 0);
Paul E. McKenneya50c3af2012-01-10 17:52:31 -08002440 }
2441
2442 /*
Paul E. McKenneyb1420f12012-03-01 13:18:08 -08002443 * Next, move those callbacks still needing a grace period to
2444 * the orphanage, where some other CPU will pick them up.
2445 * Some of the callbacks might have gone partway through a grace
2446 * period, but that is too bad. They get to start over because we
2447 * cannot assume that grace periods are synchronized across CPUs.
2448 * We don't bother updating the ->nxttail[] array yet, instead
2449 * we just reset the whole thing later on.
Paul E. McKenneya50c3af2012-01-10 17:52:31 -08002450 */
Paul E. McKenneyb1420f12012-03-01 13:18:08 -08002451 if (*rdp->nxttail[RCU_DONE_TAIL] != NULL) {
2452 *rsp->orphan_nxttail = *rdp->nxttail[RCU_DONE_TAIL];
2453 rsp->orphan_nxttail = rdp->nxttail[RCU_NEXT_TAIL];
2454 *rdp->nxttail[RCU_DONE_TAIL] = NULL;
Paul E. McKenneya50c3af2012-01-10 17:52:31 -08002455 }
2456
2457 /*
Paul E. McKenneyb1420f12012-03-01 13:18:08 -08002458 * Then move the ready-to-invoke callbacks to the orphanage,
2459 * where some other CPU will pick them up. These will not be
2460 * required to pass though another grace period: They are done.
Paul E. McKenneya50c3af2012-01-10 17:52:31 -08002461 */
Paul E. McKenneye5601402012-01-07 11:03:57 -08002462 if (rdp->nxtlist != NULL) {
Paul E. McKenneyb1420f12012-03-01 13:18:08 -08002463 *rsp->orphan_donetail = rdp->nxtlist;
2464 rsp->orphan_donetail = rdp->nxttail[RCU_DONE_TAIL];
Paul E. McKenneye5601402012-01-07 11:03:57 -08002465 }
Lai Jiangshan29494be2010-10-20 14:13:06 +08002466
Paul E. McKenneyb33078b2015-01-16 14:01:21 -08002467 /*
2468 * Finally, initialize the rcu_data structure's list to empty and
2469 * disallow further callbacks on this CPU.
2470 */
Paul E. McKenney3f5d3ea2012-05-09 15:39:56 -07002471 init_callback_list(rdp);
Paul E. McKenneyb33078b2015-01-16 14:01:21 -08002472 rdp->nxttail[RCU_NEXT_TAIL] = NULL;
Paul E. McKenneyb1420f12012-03-01 13:18:08 -08002473}
2474
2475/*
2476 * Adopt the RCU callbacks from the specified rcu_state structure's
Paul E. McKenney7b2e6012012-10-08 10:54:03 -07002477 * orphanage. The caller must hold the ->orphan_lock.
Paul E. McKenneyb1420f12012-03-01 13:18:08 -08002478 */
Paul E. McKenney96d3fd02013-10-04 14:33:34 -07002479static void rcu_adopt_orphan_cbs(struct rcu_state *rsp, unsigned long flags)
Paul E. McKenneyb1420f12012-03-01 13:18:08 -08002480{
2481 int i;
Christoph Lameterfa07a582014-04-15 12:20:12 -07002482 struct rcu_data *rdp = raw_cpu_ptr(rsp->rda);
Paul E. McKenneyb1420f12012-03-01 13:18:08 -08002483
Paul E. McKenney3fbfbf72012-08-19 21:35:53 -07002484 /* No-CBs CPUs are handled specially. */
Paul E. McKenneyea463512015-03-03 14:05:26 -08002485 if (!IS_ENABLED(CONFIG_HOTPLUG_CPU) ||
2486 rcu_nocb_adopt_orphan_cbs(rsp, rdp, flags))
Paul E. McKenney3fbfbf72012-08-19 21:35:53 -07002487 return;
2488
Paul E. McKenneyb1420f12012-03-01 13:18:08 -08002489 /* Do the accounting first. */
2490 rdp->qlen_lazy += rsp->qlen_lazy;
2491 rdp->qlen += rsp->qlen;
2492 rdp->n_cbs_adopted += rsp->qlen;
Paul E. McKenney8f5af6f2012-05-04 08:31:53 -07002493 if (rsp->qlen_lazy != rsp->qlen)
2494 rcu_idle_count_callbacks_posted();
Paul E. McKenneyb1420f12012-03-01 13:18:08 -08002495 rsp->qlen_lazy = 0;
2496 rsp->qlen = 0;
2497
2498 /*
2499 * We do not need a memory barrier here because the only way we
2500 * can get here if there is an rcu_barrier() in flight is if
2501 * we are the task doing the rcu_barrier().
2502 */
2503
2504 /* First adopt the ready-to-invoke callbacks. */
2505 if (rsp->orphan_donelist != NULL) {
2506 *rsp->orphan_donetail = *rdp->nxttail[RCU_DONE_TAIL];
2507 *rdp->nxttail[RCU_DONE_TAIL] = rsp->orphan_donelist;
2508 for (i = RCU_NEXT_SIZE - 1; i >= RCU_DONE_TAIL; i--)
2509 if (rdp->nxttail[i] == rdp->nxttail[RCU_DONE_TAIL])
2510 rdp->nxttail[i] = rsp->orphan_donetail;
2511 rsp->orphan_donelist = NULL;
2512 rsp->orphan_donetail = &rsp->orphan_donelist;
2513 }
2514
2515 /* And then adopt the callbacks that still need a grace period. */
2516 if (rsp->orphan_nxtlist != NULL) {
2517 *rdp->nxttail[RCU_NEXT_TAIL] = rsp->orphan_nxtlist;
2518 rdp->nxttail[RCU_NEXT_TAIL] = rsp->orphan_nxttail;
2519 rsp->orphan_nxtlist = NULL;
2520 rsp->orphan_nxttail = &rsp->orphan_nxtlist;
2521 }
2522}
2523
2524/*
2525 * Trace the fact that this CPU is going offline.
2526 */
2527static void rcu_cleanup_dying_cpu(struct rcu_state *rsp)
2528{
2529 RCU_TRACE(unsigned long mask);
2530 RCU_TRACE(struct rcu_data *rdp = this_cpu_ptr(rsp->rda));
2531 RCU_TRACE(struct rcu_node *rnp = rdp->mynode);
2532
Paul E. McKenneyea463512015-03-03 14:05:26 -08002533 if (!IS_ENABLED(CONFIG_HOTPLUG_CPU))
2534 return;
2535
Paul E. McKenneyb1420f12012-03-01 13:18:08 -08002536 RCU_TRACE(mask = rdp->grpmask);
Paul E. McKenneye5601402012-01-07 11:03:57 -08002537 trace_rcu_grace_period(rsp->name,
2538 rnp->gpnum + 1 - !!(rnp->qsmask & mask),
Steven Rostedt (Red Hat)f7f7bac2013-07-12 17:18:47 -04002539 TPS("cpuofl"));
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002540}
2541
2542/*
Paul E. McKenney8af3a5e2014-10-31 11:22:37 -07002543 * All CPUs for the specified rcu_node structure have gone offline,
2544 * and all tasks that were preempted within an RCU read-side critical
2545 * section while running on one of those CPUs have since exited their RCU
2546 * read-side critical section. Some other CPU is reporting this fact with
2547 * the specified rcu_node structure's ->lock held and interrupts disabled.
2548 * This function therefore goes up the tree of rcu_node structures,
2549 * clearing the corresponding bits in the ->qsmaskinit fields. Note that
2550 * the leaf rcu_node structure's ->qsmaskinit field has already been
2551 * updated
2552 *
2553 * This function does check that the specified rcu_node structure has
2554 * all CPUs offline and no blocked tasks, so it is OK to invoke it
2555 * prematurely. That said, invoking it after the fact will cost you
2556 * a needless lock acquisition. So once it has done its work, don't
2557 * invoke it again.
2558 */
2559static void rcu_cleanup_dead_rnp(struct rcu_node *rnp_leaf)
2560{
2561 long mask;
2562 struct rcu_node *rnp = rnp_leaf;
2563
Paul E. McKenneyea463512015-03-03 14:05:26 -08002564 if (!IS_ENABLED(CONFIG_HOTPLUG_CPU) ||
2565 rnp->qsmaskinit || rcu_preempt_has_tasks(rnp))
Paul E. McKenney8af3a5e2014-10-31 11:22:37 -07002566 return;
2567 for (;;) {
2568 mask = rnp->grpmask;
2569 rnp = rnp->parent;
2570 if (!rnp)
2571 break;
Peter Zijlstra2a67e742015-10-08 12:24:23 +02002572 raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */
Paul E. McKenney8af3a5e2014-10-31 11:22:37 -07002573 rnp->qsmaskinit &= ~mask;
Paul E. McKenney0aa04b02015-01-23 21:52:37 -08002574 rnp->qsmask &= ~mask;
Paul E. McKenney8af3a5e2014-10-31 11:22:37 -07002575 if (rnp->qsmaskinit) {
2576 raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */
2577 return;
2578 }
2579 raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */
2580 }
2581}
2582
2583/*
Paul E. McKenney88428cc52015-01-28 14:42:09 -08002584 * The CPU is exiting the idle loop into the arch_cpu_idle_dead()
2585 * function. We now remove it from the rcu_node tree's ->qsmaskinit
2586 * bit masks.
2587 */
2588static void rcu_cleanup_dying_idle_cpu(int cpu, struct rcu_state *rsp)
2589{
2590 unsigned long flags;
2591 unsigned long mask;
2592 struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
2593 struct rcu_node *rnp = rdp->mynode; /* Outgoing CPU's rdp & rnp. */
2594
Paul E. McKenneyea463512015-03-03 14:05:26 -08002595 if (!IS_ENABLED(CONFIG_HOTPLUG_CPU))
2596 return;
2597
Paul E. McKenney88428cc52015-01-28 14:42:09 -08002598 /* Remove outgoing CPU from mask in the leaf rcu_node structure. */
2599 mask = rdp->grpmask;
Peter Zijlstra2a67e742015-10-08 12:24:23 +02002600 raw_spin_lock_irqsave_rcu_node(rnp, flags); /* Enforce GP memory-order guarantee. */
Paul E. McKenney88428cc52015-01-28 14:42:09 -08002601 rnp->qsmaskinitnext &= ~mask;
2602 raw_spin_unlock_irqrestore(&rnp->lock, flags);
2603}
2604
2605/*
Paul E. McKenneye5601402012-01-07 11:03:57 -08002606 * The CPU has been completely removed, and some other CPU is reporting
Paul E. McKenneyb1420f12012-03-01 13:18:08 -08002607 * this fact from process context. Do the remainder of the cleanup,
2608 * including orphaning the outgoing CPU's RCU callbacks, and also
Paul E. McKenney1331e7a2012-08-02 17:43:50 -07002609 * adopting them. There can only be one CPU hotplug operation at a time,
2610 * so no other CPU can be attempting to update rcu_cpu_kthread_task.
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002611 */
Paul E. McKenneye5601402012-01-07 11:03:57 -08002612static void rcu_cleanup_dead_cpu(int cpu, struct rcu_state *rsp)
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002613{
Paul E. McKenney2036d942012-01-30 17:02:47 -08002614 unsigned long flags;
Paul E. McKenneye5601402012-01-07 11:03:57 -08002615 struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
Paul E. McKenneyb1420f12012-03-01 13:18:08 -08002616 struct rcu_node *rnp = rdp->mynode; /* Outgoing CPU's rdp & rnp. */
Paul E. McKenneye5601402012-01-07 11:03:57 -08002617
Paul E. McKenneyea463512015-03-03 14:05:26 -08002618 if (!IS_ENABLED(CONFIG_HOTPLUG_CPU))
2619 return;
2620
Paul E. McKenney2036d942012-01-30 17:02:47 -08002621 /* Adjust any no-longer-needed kthreads. */
Thomas Gleixner5d01bbd2012-07-16 10:42:35 +00002622 rcu_boost_kthread_setaffinity(rnp, -1);
Paul E. McKenney2036d942012-01-30 17:02:47 -08002623
Paul E. McKenneyb1420f12012-03-01 13:18:08 -08002624 /* Orphan the dead CPU's callbacks, and adopt them if appropriate. */
Paul E. McKenney78043c42015-01-18 17:46:24 -08002625 raw_spin_lock_irqsave(&rsp->orphan_lock, flags);
Paul E. McKenneyb1420f12012-03-01 13:18:08 -08002626 rcu_send_cbs_to_orphanage(cpu, rsp, rnp, rdp);
Paul E. McKenney96d3fd02013-10-04 14:33:34 -07002627 rcu_adopt_orphan_cbs(rsp, flags);
Paul E. McKenneya8f4cba2014-10-31 13:48:41 -07002628 raw_spin_unlock_irqrestore(&rsp->orphan_lock, flags);
Paul E. McKenneyb1420f12012-03-01 13:18:08 -08002629
Paul E. McKenneycf015372012-06-21 11:26:42 -07002630 WARN_ONCE(rdp->qlen != 0 || rdp->nxtlist != NULL,
2631 "rcu_cleanup_dead_cpu: Callbacks on offline CPU %d: qlen=%lu, nxtlist=%p\n",
2632 cpu, rdp->qlen, rdp->nxtlist);
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002633}
2634
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002635/*
2636 * Invoke any RCU callbacks that have made it to the end of their grace
2637 * period. Thottle as specified by rdp->blimit.
2638 */
Paul E. McKenney37c72e52009-10-14 10:15:55 -07002639static void rcu_do_batch(struct rcu_state *rsp, struct rcu_data *rdp)
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002640{
2641 unsigned long flags;
2642 struct rcu_head *next, *list, **tail;
Eric Dumazet878d7432012-10-18 04:55:36 -07002643 long bl, count, count_lazy;
2644 int i;
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002645
Paul E. McKenneydc35c892012-12-03 13:52:00 -08002646 /* If no callbacks are ready, just return. */
Paul E. McKenney29c00b42011-06-17 15:53:19 -07002647 if (!cpu_has_callbacks_ready_to_invoke(rdp)) {
Paul E. McKenney486e2592012-01-06 14:11:30 -08002648 trace_rcu_batch_start(rsp->name, rdp->qlen_lazy, rdp->qlen, 0);
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08002649 trace_rcu_batch_end(rsp->name, 0, !!READ_ONCE(rdp->nxtlist),
Paul E. McKenney4968c302011-12-07 16:32:40 -08002650 need_resched(), is_idle_task(current),
2651 rcu_is_callbacks_kthread());
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002652 return;
Paul E. McKenney29c00b42011-06-17 15:53:19 -07002653 }
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002654
2655 /*
2656 * Extract the list of ready callbacks, disabling to prevent
2657 * races with call_rcu() from interrupt handlers.
2658 */
2659 local_irq_save(flags);
Paul E. McKenney8146c4e22012-01-10 14:23:29 -08002660 WARN_ON_ONCE(cpu_is_offline(smp_processor_id()));
Paul E. McKenney29c00b42011-06-17 15:53:19 -07002661 bl = rdp->blimit;
Paul E. McKenney486e2592012-01-06 14:11:30 -08002662 trace_rcu_batch_start(rsp->name, rdp->qlen_lazy, rdp->qlen, bl);
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002663 list = rdp->nxtlist;
2664 rdp->nxtlist = *rdp->nxttail[RCU_DONE_TAIL];
2665 *rdp->nxttail[RCU_DONE_TAIL] = NULL;
2666 tail = rdp->nxttail[RCU_DONE_TAIL];
Paul E. McKenneyb41772a2012-06-21 20:50:42 -07002667 for (i = RCU_NEXT_SIZE - 1; i >= 0; i--)
2668 if (rdp->nxttail[i] == rdp->nxttail[RCU_DONE_TAIL])
2669 rdp->nxttail[i] = &rdp->nxtlist;
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002670 local_irq_restore(flags);
2671
2672 /* Invoke callbacks. */
Paul E. McKenney486e2592012-01-06 14:11:30 -08002673 count = count_lazy = 0;
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002674 while (list) {
2675 next = list->next;
2676 prefetch(next);
Mathieu Desnoyers551d55a2010-04-17 08:48:42 -04002677 debug_rcu_head_unqueue(list);
Paul E. McKenney486e2592012-01-06 14:11:30 -08002678 if (__rcu_reclaim(rsp->name, list))
2679 count_lazy++;
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002680 list = next;
Paul E. McKenneydff16722011-11-29 15:57:13 -08002681 /* Stop only if limit reached and CPU has something to do. */
2682 if (++count >= bl &&
2683 (need_resched() ||
2684 (!is_idle_task(current) && !rcu_is_callbacks_kthread())))
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002685 break;
2686 }
2687
2688 local_irq_save(flags);
Paul E. McKenney4968c302011-12-07 16:32:40 -08002689 trace_rcu_batch_end(rsp->name, count, !!list, need_resched(),
2690 is_idle_task(current),
2691 rcu_is_callbacks_kthread());
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002692
2693 /* Update count, and requeue any remaining callbacks. */
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002694 if (list != NULL) {
2695 *tail = rdp->nxtlist;
2696 rdp->nxtlist = list;
Paul E. McKenneyb41772a2012-06-21 20:50:42 -07002697 for (i = 0; i < RCU_NEXT_SIZE; i++)
2698 if (&rdp->nxtlist == rdp->nxttail[i])
2699 rdp->nxttail[i] = tail;
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002700 else
2701 break;
2702 }
Paul E. McKenneyb1420f12012-03-01 13:18:08 -08002703 smp_mb(); /* List handling before counting for rcu_barrier(). */
2704 rdp->qlen_lazy -= count_lazy;
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08002705 WRITE_ONCE(rdp->qlen, rdp->qlen - count);
Paul E. McKenneyb1420f12012-03-01 13:18:08 -08002706 rdp->n_cbs_invoked += count;
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002707
2708 /* Reinstate batch limit if we have worked down the excess. */
2709 if (rdp->blimit == LONG_MAX && rdp->qlen <= qlowmark)
2710 rdp->blimit = blimit;
2711
Paul E. McKenney37c72e52009-10-14 10:15:55 -07002712 /* Reset ->qlen_last_fqs_check trigger if enough CBs have drained. */
2713 if (rdp->qlen == 0 && rdp->qlen_last_fqs_check != 0) {
2714 rdp->qlen_last_fqs_check = 0;
2715 rdp->n_force_qs_snap = rsp->n_force_qs;
2716 } else if (rdp->qlen < rdp->qlen_last_fqs_check - qhimark)
2717 rdp->qlen_last_fqs_check = rdp->qlen;
Paul E. McKenneycfca9272012-06-25 12:54:17 -07002718 WARN_ON_ONCE((rdp->nxtlist == NULL) != (rdp->qlen == 0));
Paul E. McKenney37c72e52009-10-14 10:15:55 -07002719
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002720 local_irq_restore(flags);
2721
Paul E. McKenneye0f23062011-06-21 01:29:39 -07002722 /* Re-invoke RCU core processing if there are callbacks remaining. */
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002723 if (cpu_has_callbacks_ready_to_invoke(rdp))
Paul E. McKenneya46e0892011-06-15 15:47:09 -07002724 invoke_rcu_core();
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002725}
2726
2727/*
2728 * Check to see if this CPU is in a non-context-switch quiescent state
2729 * (user mode or idle loop for rcu, non-softirq execution for rcu_bh).
Paul E. McKenneye0f23062011-06-21 01:29:39 -07002730 * Also schedule RCU core processing.
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002731 *
Paul E. McKenney9b2e4f12011-09-30 12:10:22 -07002732 * This function must be called from hardirq context. It is normally
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002733 * invoked from the scheduling-clock interrupt. If rcu_pending returns
2734 * false, there is no point in invoking rcu_check_callbacks().
2735 */
Paul E. McKenneyc3377c2d2014-10-21 07:53:02 -07002736void rcu_check_callbacks(int user)
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002737{
Steven Rostedt (Red Hat)f7f7bac2013-07-12 17:18:47 -04002738 trace_rcu_utilization(TPS("Start scheduler-tick"));
Paul E. McKenneya858af22012-01-16 13:29:10 -08002739 increment_cpu_stall_ticks();
Paul E. McKenney9b2e4f12011-09-30 12:10:22 -07002740 if (user || rcu_is_cpu_rrupt_from_idle()) {
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002741
2742 /*
2743 * Get here if this CPU took its interrupt from user
2744 * mode or from the idle loop, and if this is not a
2745 * nested interrupt. In this case, the CPU is in
Paul E. McKenneyd6714c22009-08-22 13:56:46 -07002746 * a quiescent state, so note it.
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002747 *
2748 * No memory barrier is required here because both
Paul E. McKenneyd6714c22009-08-22 13:56:46 -07002749 * rcu_sched_qs() and rcu_bh_qs() reference only CPU-local
2750 * variables that other CPUs neither access nor modify,
2751 * at least not while the corresponding CPU is online.
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002752 */
2753
Paul E. McKenney284a8c92014-08-14 16:38:46 -07002754 rcu_sched_qs();
2755 rcu_bh_qs();
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002756
2757 } else if (!in_softirq()) {
2758
2759 /*
2760 * Get here if this CPU did not take its interrupt from
2761 * softirq, in other words, if it is not interrupting
2762 * a rcu_bh read-side critical section. This is an _bh
Paul E. McKenneyd6714c22009-08-22 13:56:46 -07002763 * critical section, so note it.
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002764 */
2765
Paul E. McKenney284a8c92014-08-14 16:38:46 -07002766 rcu_bh_qs();
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002767 }
Paul E. McKenney86aea0e2014-10-21 08:12:00 -07002768 rcu_preempt_check_callbacks();
Paul E. McKenneye3950ec2014-10-21 08:03:57 -07002769 if (rcu_pending())
Paul E. McKenneya46e0892011-06-15 15:47:09 -07002770 invoke_rcu_core();
Paul E. McKenney8315f422014-06-27 13:42:20 -07002771 if (user)
2772 rcu_note_voluntary_context_switch(current);
Steven Rostedt (Red Hat)f7f7bac2013-07-12 17:18:47 -04002773 trace_rcu_utilization(TPS("End scheduler-tick"));
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002774}
2775
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002776/*
2777 * Scan the leaf rcu_node structures, processing dyntick state for any that
2778 * have not yet encountered a quiescent state, using the function specified.
Paul E. McKenney27f4d282011-02-07 12:47:15 -08002779 * Also initiate boosting for any threads blocked on the root rcu_node.
2780 *
Paul E. McKenneyee47eb92010-01-04 15:09:07 -08002781 * The caller must have suppressed start of new grace periods.
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002782 */
Paul E. McKenney217af2a2013-06-21 15:39:06 -07002783static void force_qs_rnp(struct rcu_state *rsp,
2784 int (*f)(struct rcu_data *rsp, bool *isidle,
2785 unsigned long *maxj),
2786 bool *isidle, unsigned long *maxj)
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002787{
2788 unsigned long bit;
2789 int cpu;
2790 unsigned long flags;
2791 unsigned long mask;
Paul E. McKenneya0b6c9a2009-09-28 07:46:33 -07002792 struct rcu_node *rnp;
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002793
Paul E. McKenneya0b6c9a2009-09-28 07:46:33 -07002794 rcu_for_each_leaf_node(rsp, rnp) {
Paul E. McKenneybde6c3a2014-07-01 11:26:57 -07002795 cond_resched_rcu_qs();
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002796 mask = 0;
Peter Zijlstra2a67e742015-10-08 12:24:23 +02002797 raw_spin_lock_irqsave_rcu_node(rnp, flags);
Paul E. McKenneya0b6c9a2009-09-28 07:46:33 -07002798 if (rnp->qsmask == 0) {
Paul E. McKenneya77da142015-03-08 14:52:27 -07002799 if (rcu_state_p == &rcu_sched_state ||
2800 rsp != rcu_state_p ||
2801 rcu_preempt_blocked_readers_cgp(rnp)) {
2802 /*
2803 * No point in scanning bits because they
2804 * are all zero. But we might need to
2805 * priority-boost blocked readers.
2806 */
2807 rcu_initiate_boost(rnp, flags);
2808 /* rcu_initiate_boost() releases rnp->lock */
2809 continue;
2810 }
2811 if (rnp->parent &&
2812 (rnp->parent->qsmask & rnp->grpmask)) {
2813 /*
2814 * Race between grace-period
2815 * initialization and task exiting RCU
2816 * read-side critical section: Report.
2817 */
2818 rcu_report_unblock_qs_rnp(rsp, rnp, flags);
2819 /* rcu_report_unblock_qs_rnp() rlses ->lock */
2820 continue;
2821 }
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002822 }
Paul E. McKenneya0b6c9a2009-09-28 07:46:33 -07002823 cpu = rnp->grplo;
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002824 bit = 1;
Paul E. McKenneya0b6c9a2009-09-28 07:46:33 -07002825 for (; cpu <= rnp->grphi; cpu++, bit <<= 1) {
Paul E. McKenney0edd1b12013-06-21 16:37:22 -07002826 if ((rnp->qsmask & bit) != 0) {
Paul E. McKenney0edd1b12013-06-21 16:37:22 -07002827 if (f(per_cpu_ptr(rsp->rda, cpu), isidle, maxj))
2828 mask |= bit;
2829 }
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002830 }
Paul E. McKenney45f014c52e2010-01-04 15:09:08 -08002831 if (mask != 0) {
Paul E. McKenney654e9532015-03-15 09:19:35 -07002832 /* Idle/offline CPUs, report (releases rnp->lock. */
2833 rcu_report_qs_rnp(mask, rsp, rnp, rnp->gpnum, flags);
Paul E. McKenney0aa04b02015-01-23 21:52:37 -08002834 } else {
2835 /* Nothing to do here, so just drop the lock. */
2836 raw_spin_unlock_irqrestore(&rnp->lock, flags);
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002837 }
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002838 }
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002839}
2840
2841/*
2842 * Force quiescent states on reluctant CPUs, and also detect which
2843 * CPUs are in dyntick-idle mode.
2844 */
Paul E. McKenney4cdfc1752012-06-22 17:06:26 -07002845static void force_quiescent_state(struct rcu_state *rsp)
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002846{
2847 unsigned long flags;
Paul E. McKenney394f2762012-06-26 17:00:35 -07002848 bool ret;
2849 struct rcu_node *rnp;
2850 struct rcu_node *rnp_old = NULL;
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002851
Paul E. McKenney394f2762012-06-26 17:00:35 -07002852 /* Funnel through hierarchy to reduce memory contention. */
Shan Weid860d402014-06-19 14:12:44 -07002853 rnp = __this_cpu_read(rsp->rda->mynode);
Paul E. McKenney394f2762012-06-26 17:00:35 -07002854 for (; rnp != NULL; rnp = rnp->parent) {
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08002855 ret = (READ_ONCE(rsp->gp_flags) & RCU_GP_FLAG_FQS) ||
Paul E. McKenney394f2762012-06-26 17:00:35 -07002856 !raw_spin_trylock(&rnp->fqslock);
2857 if (rnp_old != NULL)
2858 raw_spin_unlock(&rnp_old->fqslock);
2859 if (ret) {
Paul E. McKenneya7925632014-06-02 14:54:34 -07002860 rsp->n_force_qs_lh++;
Paul E. McKenney394f2762012-06-26 17:00:35 -07002861 return;
2862 }
2863 rnp_old = rnp;
2864 }
2865 /* rnp_old == rcu_get_root(rsp), rnp == NULL. */
2866
2867 /* Reached the root of the rcu_node tree, acquire lock. */
Peter Zijlstra2a67e742015-10-08 12:24:23 +02002868 raw_spin_lock_irqsave_rcu_node(rnp_old, flags);
Paul E. McKenney394f2762012-06-26 17:00:35 -07002869 raw_spin_unlock(&rnp_old->fqslock);
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08002870 if (READ_ONCE(rsp->gp_flags) & RCU_GP_FLAG_FQS) {
Paul E. McKenneya7925632014-06-02 14:54:34 -07002871 rsp->n_force_qs_lh++;
Paul E. McKenney394f2762012-06-26 17:00:35 -07002872 raw_spin_unlock_irqrestore(&rnp_old->lock, flags);
Paul E. McKenney4cdfc1752012-06-22 17:06:26 -07002873 return; /* Someone beat us to it. */
Paul E. McKenney46a1e342010-01-04 15:09:09 -08002874 }
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08002875 WRITE_ONCE(rsp->gp_flags, READ_ONCE(rsp->gp_flags) | RCU_GP_FLAG_FQS);
Paul E. McKenney394f2762012-06-26 17:00:35 -07002876 raw_spin_unlock_irqrestore(&rnp_old->lock, flags);
Pranith Kumar2aa792e2014-08-12 13:07:47 -04002877 rcu_gp_kthread_wake(rsp);
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002878}
2879
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002880/*
Paul E. McKenneye0f23062011-06-21 01:29:39 -07002881 * This does the RCU core processing work for the specified rcu_state
2882 * and rcu_data structures. This may be called only from the CPU to
2883 * whom the rdp belongs.
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002884 */
2885static void
Paul E. McKenney1bca8cf2012-06-12 09:40:38 -07002886__rcu_process_callbacks(struct rcu_state *rsp)
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002887{
2888 unsigned long flags;
Paul E. McKenney48a76392014-03-11 13:02:16 -07002889 bool needwake;
Christoph Lameterfa07a582014-04-15 12:20:12 -07002890 struct rcu_data *rdp = raw_cpu_ptr(rsp->rda);
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002891
Paul E. McKenney2e597552009-08-15 09:53:48 -07002892 WARN_ON_ONCE(rdp->beenonline == 0);
2893
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002894 /* Update RCU state based on any recent quiescent states. */
2895 rcu_check_quiescent_state(rsp, rdp);
2896
2897 /* Does this CPU require a not-yet-started grace period? */
Paul E. McKenneydc35c892012-12-03 13:52:00 -08002898 local_irq_save(flags);
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002899 if (cpu_needs_another_gp(rsp, rdp)) {
Paul E. McKenney6cf10082015-10-08 15:36:54 -07002900 raw_spin_lock_rcu_node(rcu_get_root(rsp)); /* irqs disabled. */
Paul E. McKenney48a76392014-03-11 13:02:16 -07002901 needwake = rcu_start_gp(rsp);
Paul E. McKenneyb8462082012-12-29 22:04:18 -08002902 raw_spin_unlock_irqrestore(&rcu_get_root(rsp)->lock, flags);
Paul E. McKenney48a76392014-03-11 13:02:16 -07002903 if (needwake)
2904 rcu_gp_kthread_wake(rsp);
Paul E. McKenneydc35c892012-12-03 13:52:00 -08002905 } else {
2906 local_irq_restore(flags);
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002907 }
2908
2909 /* If there are callbacks ready, invoke them. */
Shaohua Li09223372011-06-14 13:26:25 +08002910 if (cpu_has_callbacks_ready_to_invoke(rdp))
Paul E. McKenneya46e0892011-06-15 15:47:09 -07002911 invoke_rcu_callbacks(rsp, rdp);
Paul E. McKenney96d3fd02013-10-04 14:33:34 -07002912
2913 /* Do any needed deferred wakeups of rcuo kthreads. */
2914 do_nocb_deferred_wakeup(rdp);
Shaohua Li09223372011-06-14 13:26:25 +08002915}
2916
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002917/*
Paul E. McKenneye0f23062011-06-21 01:29:39 -07002918 * Do RCU core processing for the current CPU.
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002919 */
Shaohua Li09223372011-06-14 13:26:25 +08002920static void rcu_process_callbacks(struct softirq_action *unused)
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002921{
Paul E. McKenney6ce75a22012-06-12 11:01:13 -07002922 struct rcu_state *rsp;
2923
Paul E. McKenneybfa00b42012-06-21 09:54:10 -07002924 if (cpu_is_offline(smp_processor_id()))
2925 return;
Steven Rostedt (Red Hat)f7f7bac2013-07-12 17:18:47 -04002926 trace_rcu_utilization(TPS("Start RCU core"));
Paul E. McKenney6ce75a22012-06-12 11:01:13 -07002927 for_each_rcu_flavor(rsp)
2928 __rcu_process_callbacks(rsp);
Steven Rostedt (Red Hat)f7f7bac2013-07-12 17:18:47 -04002929 trace_rcu_utilization(TPS("End RCU core"));
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002930}
2931
Paul E. McKenneya26ac242011-01-12 14:10:23 -08002932/*
Paul E. McKenneye0f23062011-06-21 01:29:39 -07002933 * Schedule RCU callback invocation. If the specified type of RCU
2934 * does not support RCU priority boosting, just do a direct call,
2935 * otherwise wake up the per-CPU kernel kthread. Note that because we
Paul E. McKenney924df8a2014-10-29 15:37:58 -07002936 * are running on the current CPU with softirqs disabled, the
Paul E. McKenneye0f23062011-06-21 01:29:39 -07002937 * rcu_cpu_kthread_task cannot disappear out from under us.
Paul E. McKenneya26ac242011-01-12 14:10:23 -08002938 */
Paul E. McKenneya46e0892011-06-15 15:47:09 -07002939static void invoke_rcu_callbacks(struct rcu_state *rsp, struct rcu_data *rdp)
Paul E. McKenneya26ac242011-01-12 14:10:23 -08002940{
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08002941 if (unlikely(!READ_ONCE(rcu_scheduler_fully_active)))
Paul E. McKenneyb0d30412011-07-10 15:57:35 -07002942 return;
Paul E. McKenneya46e0892011-06-15 15:47:09 -07002943 if (likely(!rsp->boost)) {
2944 rcu_do_batch(rsp, rdp);
Paul E. McKenneya26ac242011-01-12 14:10:23 -08002945 return;
2946 }
Paul E. McKenneya46e0892011-06-15 15:47:09 -07002947 invoke_rcu_callbacks_kthread();
Paul E. McKenneya26ac242011-01-12 14:10:23 -08002948}
2949
Paul E. McKenneya46e0892011-06-15 15:47:09 -07002950static void invoke_rcu_core(void)
Shaohua Li09223372011-06-14 13:26:25 +08002951{
Paul E. McKenneyb0f74032013-02-04 12:14:24 -08002952 if (cpu_online(smp_processor_id()))
2953 raise_softirq(RCU_SOFTIRQ);
Shaohua Li09223372011-06-14 13:26:25 +08002954}
2955
Paul E. McKenney29154c52012-05-30 03:21:48 -07002956/*
2957 * Handle any core-RCU processing required by a call_rcu() invocation.
2958 */
2959static void __call_rcu_core(struct rcu_state *rsp, struct rcu_data *rdp,
2960 struct rcu_head *head, unsigned long flags)
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002961{
Paul E. McKenney48a76392014-03-11 13:02:16 -07002962 bool needwake;
2963
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002964 /*
Paul E. McKenney29154c52012-05-30 03:21:48 -07002965 * If called from an extended quiescent state, invoke the RCU
2966 * core in order to force a re-evaluation of RCU's idleness.
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002967 */
Yao Dongdong9910aff2015-02-25 17:09:46 +08002968 if (!rcu_is_watching())
Paul E. McKenney29154c52012-05-30 03:21:48 -07002969 invoke_rcu_core();
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002970
Paul E. McKenney29154c52012-05-30 03:21:48 -07002971 /* If interrupts were disabled or CPU offline, don't invoke RCU core. */
2972 if (irqs_disabled_flags(flags) || cpu_is_offline(smp_processor_id()))
Paul E. McKenney2655d572011-04-07 22:47:23 -07002973 return;
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01002974
Paul E. McKenney37c72e52009-10-14 10:15:55 -07002975 /*
2976 * Force the grace period if too many callbacks or too long waiting.
2977 * Enforce hysteresis, and don't invoke force_quiescent_state()
2978 * if some other CPU has recently done so. Also, don't bother
2979 * invoking force_quiescent_state() if the newly enqueued callback
2980 * is the only one waiting for a grace period to complete.
2981 */
Paul E. McKenney2655d572011-04-07 22:47:23 -07002982 if (unlikely(rdp->qlen > rdp->qlen_last_fqs_check + qhimark)) {
Paul E. McKenneyb52573d2010-12-14 17:36:02 -08002983
2984 /* Are we ignoring a completed grace period? */
Paul E. McKenney470716f2013-03-19 11:32:11 -07002985 note_gp_changes(rsp, rdp);
Paul E. McKenneyb52573d2010-12-14 17:36:02 -08002986
2987 /* Start a new grace period if one not already started. */
2988 if (!rcu_gp_in_progress(rsp)) {
Paul E. McKenneyb52573d2010-12-14 17:36:02 -08002989 struct rcu_node *rnp_root = rcu_get_root(rsp);
2990
Peter Zijlstra2a67e742015-10-08 12:24:23 +02002991 raw_spin_lock_rcu_node(rnp_root);
Paul E. McKenney48a76392014-03-11 13:02:16 -07002992 needwake = rcu_start_gp(rsp);
Paul E. McKenneyb8462082012-12-29 22:04:18 -08002993 raw_spin_unlock(&rnp_root->lock);
Paul E. McKenney48a76392014-03-11 13:02:16 -07002994 if (needwake)
2995 rcu_gp_kthread_wake(rsp);
Paul E. McKenneyb52573d2010-12-14 17:36:02 -08002996 } else {
2997 /* Give the grace period a kick. */
2998 rdp->blimit = LONG_MAX;
2999 if (rsp->n_force_qs == rdp->n_force_qs_snap &&
3000 *rdp->nxttail[RCU_DONE_TAIL] != head)
Paul E. McKenney4cdfc1752012-06-22 17:06:26 -07003001 force_quiescent_state(rsp);
Paul E. McKenneyb52573d2010-12-14 17:36:02 -08003002 rdp->n_force_qs_snap = rsp->n_force_qs;
3003 rdp->qlen_last_fqs_check = rdp->qlen;
3004 }
Paul E. McKenney4cdfc1752012-06-22 17:06:26 -07003005 }
Paul E. McKenney29154c52012-05-30 03:21:48 -07003006}
3007
Paul E. McKenney3fbfbf72012-08-19 21:35:53 -07003008/*
Paul E. McKenneyae150182013-04-23 13:20:57 -07003009 * RCU callback function to leak a callback.
3010 */
3011static void rcu_leak_callback(struct rcu_head *rhp)
3012{
3013}
3014
3015/*
Paul E. McKenney3fbfbf72012-08-19 21:35:53 -07003016 * Helper function for call_rcu() and friends. The cpu argument will
3017 * normally be -1, indicating "currently running CPU". It may specify
3018 * a CPU only if that CPU is a no-CBs CPU. Currently, only _rcu_barrier()
3019 * is expected to specify a CPU.
3020 */
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01003021static void
Boqun Fengb6a4ae72015-07-29 13:29:38 +08003022__call_rcu(struct rcu_head *head, rcu_callback_t func,
Paul E. McKenney3fbfbf72012-08-19 21:35:53 -07003023 struct rcu_state *rsp, int cpu, bool lazy)
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01003024{
3025 unsigned long flags;
3026 struct rcu_data *rdp;
3027
Paul E. McKenney1146edc2014-06-09 08:24:17 -07003028 WARN_ON_ONCE((unsigned long)head & 0x1); /* Misaligned rcu_head! */
Paul E. McKenneyae150182013-04-23 13:20:57 -07003029 if (debug_rcu_head_queue(head)) {
3030 /* Probable double call_rcu(), so leak the callback. */
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08003031 WRITE_ONCE(head->func, rcu_leak_callback);
Paul E. McKenneyae150182013-04-23 13:20:57 -07003032 WARN_ONCE(1, "__call_rcu(): Leaked duplicate callback\n");
3033 return;
3034 }
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01003035 head->func = func;
3036 head->next = NULL;
3037
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01003038 /*
3039 * Opportunistically note grace-period endings and beginnings.
3040 * Note that we might see a beginning right after we see an
3041 * end, but never vice versa, since this CPU has to pass through
3042 * a quiescent state betweentimes.
3043 */
3044 local_irq_save(flags);
3045 rdp = this_cpu_ptr(rsp->rda);
3046
3047 /* Add the callback to our list. */
Paul E. McKenney3fbfbf72012-08-19 21:35:53 -07003048 if (unlikely(rdp->nxttail[RCU_NEXT_TAIL] == NULL) || cpu != -1) {
3049 int offline;
3050
3051 if (cpu != -1)
3052 rdp = per_cpu_ptr(rsp->rda, cpu);
Paul E. McKenney143da9c2015-01-19 19:57:32 -08003053 if (likely(rdp->mynode)) {
3054 /* Post-boot, so this should be for a no-CBs CPU. */
3055 offline = !__call_rcu_nocb(rdp, head, lazy, flags);
3056 WARN_ON_ONCE(offline);
3057 /* Offline CPU, _call_rcu() illegal, leak callback. */
3058 local_irq_restore(flags);
3059 return;
3060 }
3061 /*
3062 * Very early boot, before rcu_init(). Initialize if needed
3063 * and then drop through to queue the callback.
3064 */
3065 BUG_ON(cpu != -1);
Paul E. McKenney34404ca2015-01-19 20:39:20 -08003066 WARN_ON_ONCE(!rcu_is_watching());
Paul E. McKenney143da9c2015-01-19 19:57:32 -08003067 if (!likely(rdp->nxtlist))
3068 init_default_callback_list(rdp);
Paul E. McKenney0d8ee372012-08-03 13:16:15 -07003069 }
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08003070 WRITE_ONCE(rdp->qlen, rdp->qlen + 1);
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01003071 if (lazy)
3072 rdp->qlen_lazy++;
3073 else
3074 rcu_idle_count_callbacks_posted();
3075 smp_mb(); /* Count before adding callback for rcu_barrier(). */
3076 *rdp->nxttail[RCU_NEXT_TAIL] = head;
3077 rdp->nxttail[RCU_NEXT_TAIL] = &head->next;
3078
3079 if (__is_kfree_rcu_offset((unsigned long)func))
3080 trace_rcu_kfree_callback(rsp->name, head, (unsigned long)func,
3081 rdp->qlen_lazy, rdp->qlen);
3082 else
3083 trace_rcu_callback(rsp->name, head, rdp->qlen_lazy, rdp->qlen);
3084
Paul E. McKenney29154c52012-05-30 03:21:48 -07003085 /* Go handle any RCU core processing required. */
3086 __call_rcu_core(rsp, rdp, head, flags);
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01003087 local_irq_restore(flags);
3088}
3089
3090/*
Paul E. McKenneyd6714c22009-08-22 13:56:46 -07003091 * Queue an RCU-sched callback for invocation after a grace period.
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01003092 */
Boqun Fengb6a4ae72015-07-29 13:29:38 +08003093void call_rcu_sched(struct rcu_head *head, rcu_callback_t func)
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01003094{
Paul E. McKenney3fbfbf72012-08-19 21:35:53 -07003095 __call_rcu(head, func, &rcu_sched_state, -1, 0);
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01003096}
Paul E. McKenneyd6714c22009-08-22 13:56:46 -07003097EXPORT_SYMBOL_GPL(call_rcu_sched);
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01003098
3099/*
Paul E. McKenney486e2592012-01-06 14:11:30 -08003100 * Queue an RCU callback for invocation after a quicker grace period.
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01003101 */
Boqun Fengb6a4ae72015-07-29 13:29:38 +08003102void call_rcu_bh(struct rcu_head *head, rcu_callback_t func)
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01003103{
Paul E. McKenney3fbfbf72012-08-19 21:35:53 -07003104 __call_rcu(head, func, &rcu_bh_state, -1, 0);
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01003105}
3106EXPORT_SYMBOL_GPL(call_rcu_bh);
3107
Paul E. McKenney6d813392012-02-23 13:30:16 -08003108/*
Andreea-Cristina Bernat495aa962014-03-18 20:48:48 +02003109 * Queue an RCU callback for lazy invocation after a grace period.
3110 * This will likely be later named something like "call_rcu_lazy()",
3111 * but this change will require some way of tagging the lazy RCU
3112 * callbacks in the list of pending callbacks. Until then, this
3113 * function may only be called from __kfree_rcu().
3114 */
3115void kfree_call_rcu(struct rcu_head *head,
Boqun Fengb6a4ae72015-07-29 13:29:38 +08003116 rcu_callback_t func)
Andreea-Cristina Bernat495aa962014-03-18 20:48:48 +02003117{
Uma Sharmae5341652014-03-23 22:32:09 -07003118 __call_rcu(head, func, rcu_state_p, -1, 1);
Andreea-Cristina Bernat495aa962014-03-18 20:48:48 +02003119}
3120EXPORT_SYMBOL_GPL(kfree_call_rcu);
3121
3122/*
Paul E. McKenney6d813392012-02-23 13:30:16 -08003123 * Because a context switch is a grace period for RCU-sched and RCU-bh,
3124 * any blocking grace-period wait automatically implies a grace period
3125 * if there is only one CPU online at any point time during execution
3126 * of either synchronize_sched() or synchronize_rcu_bh(). It is OK to
3127 * occasionally incorrectly indicate that there are multiple CPUs online
3128 * when there was in fact only one the whole time, as this just adds
3129 * some overhead: RCU still operates correctly.
Paul E. McKenney6d813392012-02-23 13:30:16 -08003130 */
3131static inline int rcu_blocking_is_gp(void)
3132{
Paul E. McKenney95f0c1d2012-06-19 11:58:27 -07003133 int ret;
3134
Paul E. McKenney6d813392012-02-23 13:30:16 -08003135 might_sleep(); /* Check for RCU read-side critical section. */
Paul E. McKenney95f0c1d2012-06-19 11:58:27 -07003136 preempt_disable();
3137 ret = num_online_cpus() <= 1;
3138 preempt_enable();
3139 return ret;
Paul E. McKenney6d813392012-02-23 13:30:16 -08003140}
3141
Paul E. McKenney6ebb2372009-11-22 08:53:50 -08003142/**
3143 * synchronize_sched - wait until an rcu-sched grace period has elapsed.
3144 *
3145 * Control will return to the caller some time after a full rcu-sched
3146 * grace period has elapsed, in other words after all currently executing
3147 * rcu-sched read-side critical sections have completed. These read-side
3148 * critical sections are delimited by rcu_read_lock_sched() and
3149 * rcu_read_unlock_sched(), and may be nested. Note that preempt_disable(),
3150 * local_irq_disable(), and so on may be used in place of
3151 * rcu_read_lock_sched().
3152 *
3153 * This means that all preempt_disable code sequences, including NMI and
Paul E. McKenneyf0a0e6f2012-10-23 13:47:01 -07003154 * non-threaded hardware-interrupt handlers, in progress on entry will
3155 * have completed before this primitive returns. However, this does not
3156 * guarantee that softirq handlers will have completed, since in some
3157 * kernels, these handlers can run in process context, and can block.
3158 *
3159 * Note that this guarantee implies further memory-ordering guarantees.
3160 * On systems with more than one CPU, when synchronize_sched() returns,
3161 * each CPU is guaranteed to have executed a full memory barrier since the
3162 * end of its last RCU-sched read-side critical section whose beginning
3163 * preceded the call to synchronize_sched(). In addition, each CPU having
3164 * an RCU read-side critical section that extends beyond the return from
3165 * synchronize_sched() is guaranteed to have executed a full memory barrier
3166 * after the beginning of synchronize_sched() and before the beginning of
3167 * that RCU read-side critical section. Note that these guarantees include
3168 * CPUs that are offline, idle, or executing in user mode, as well as CPUs
3169 * that are executing in the kernel.
3170 *
3171 * Furthermore, if CPU A invoked synchronize_sched(), which returned
3172 * to its caller on CPU B, then both CPU A and CPU B are guaranteed
3173 * to have executed a full memory barrier during the execution of
3174 * synchronize_sched() -- even if CPU A and CPU B are the same CPU (but
3175 * again only if the system has more than one CPU).
Paul E. McKenney6ebb2372009-11-22 08:53:50 -08003176 *
3177 * This primitive provides the guarantees made by the (now removed)
3178 * synchronize_kernel() API. In contrast, synchronize_rcu() only
3179 * guarantees that rcu_read_lock() sections will have completed.
3180 * In "classic RCU", these two guarantees happen to be one and
3181 * the same, but can differ in realtime RCU implementations.
3182 */
3183void synchronize_sched(void)
3184{
Paul E. McKenneyf78f5b92015-06-18 15:50:02 -07003185 RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map) ||
3186 lock_is_held(&rcu_lock_map) ||
3187 lock_is_held(&rcu_sched_lock_map),
3188 "Illegal synchronize_sched() in RCU-sched read-side critical section");
Paul E. McKenney6ebb2372009-11-22 08:53:50 -08003189 if (rcu_blocking_is_gp())
3190 return;
Paul E. McKenney5afff482015-02-18 16:39:09 -08003191 if (rcu_gp_is_expedited())
Antti P Miettinen3705b882012-10-05 09:59:15 +03003192 synchronize_sched_expedited();
3193 else
3194 wait_rcu_gp(call_rcu_sched);
Paul E. McKenney6ebb2372009-11-22 08:53:50 -08003195}
3196EXPORT_SYMBOL_GPL(synchronize_sched);
3197
3198/**
3199 * synchronize_rcu_bh - wait until an rcu_bh grace period has elapsed.
3200 *
3201 * Control will return to the caller some time after a full rcu_bh grace
3202 * period has elapsed, in other words after all currently executing rcu_bh
3203 * read-side critical sections have completed. RCU read-side critical
3204 * sections are delimited by rcu_read_lock_bh() and rcu_read_unlock_bh(),
3205 * and may be nested.
Paul E. McKenneyf0a0e6f2012-10-23 13:47:01 -07003206 *
3207 * See the description of synchronize_sched() for more detailed information
3208 * on memory ordering guarantees.
Paul E. McKenney6ebb2372009-11-22 08:53:50 -08003209 */
3210void synchronize_rcu_bh(void)
3211{
Paul E. McKenneyf78f5b92015-06-18 15:50:02 -07003212 RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map) ||
3213 lock_is_held(&rcu_lock_map) ||
3214 lock_is_held(&rcu_sched_lock_map),
3215 "Illegal synchronize_rcu_bh() in RCU-bh read-side critical section");
Paul E. McKenney6ebb2372009-11-22 08:53:50 -08003216 if (rcu_blocking_is_gp())
3217 return;
Paul E. McKenney5afff482015-02-18 16:39:09 -08003218 if (rcu_gp_is_expedited())
Antti P Miettinen3705b882012-10-05 09:59:15 +03003219 synchronize_rcu_bh_expedited();
3220 else
3221 wait_rcu_gp(call_rcu_bh);
Paul E. McKenney6ebb2372009-11-22 08:53:50 -08003222}
3223EXPORT_SYMBOL_GPL(synchronize_rcu_bh);
3224
Paul E. McKenney765a3f42014-03-14 16:37:08 -07003225/**
3226 * get_state_synchronize_rcu - Snapshot current RCU state
3227 *
3228 * Returns a cookie that is used by a later call to cond_synchronize_rcu()
3229 * to determine whether or not a full grace period has elapsed in the
3230 * meantime.
3231 */
3232unsigned long get_state_synchronize_rcu(void)
3233{
3234 /*
3235 * Any prior manipulation of RCU-protected data must happen
3236 * before the load from ->gpnum.
3237 */
3238 smp_mb(); /* ^^^ */
3239
3240 /*
3241 * Make sure this load happens before the purportedly
3242 * time-consuming work between get_state_synchronize_rcu()
3243 * and cond_synchronize_rcu().
3244 */
Uma Sharmae5341652014-03-23 22:32:09 -07003245 return smp_load_acquire(&rcu_state_p->gpnum);
Paul E. McKenney765a3f42014-03-14 16:37:08 -07003246}
3247EXPORT_SYMBOL_GPL(get_state_synchronize_rcu);
3248
3249/**
3250 * cond_synchronize_rcu - Conditionally wait for an RCU grace period
3251 *
3252 * @oldstate: return value from earlier call to get_state_synchronize_rcu()
3253 *
3254 * If a full RCU grace period has elapsed since the earlier call to
3255 * get_state_synchronize_rcu(), just return. Otherwise, invoke
3256 * synchronize_rcu() to wait for a full grace period.
3257 *
3258 * Yes, this function does not take counter wrap into account. But
3259 * counter wrap is harmless. If the counter wraps, we have waited for
3260 * more than 2 billion grace periods (and way more on a 64-bit system!),
3261 * so waiting for one additional grace period should be just fine.
3262 */
3263void cond_synchronize_rcu(unsigned long oldstate)
3264{
3265 unsigned long newstate;
3266
3267 /*
3268 * Ensure that this load happens before any RCU-destructive
3269 * actions the caller might carry out after we return.
3270 */
Uma Sharmae5341652014-03-23 22:32:09 -07003271 newstate = smp_load_acquire(&rcu_state_p->completed);
Paul E. McKenney765a3f42014-03-14 16:37:08 -07003272 if (ULONG_CMP_GE(oldstate, newstate))
3273 synchronize_rcu();
3274}
3275EXPORT_SYMBOL_GPL(cond_synchronize_rcu);
3276
Paul E. McKenney24560052015-05-30 10:11:24 -07003277/**
3278 * get_state_synchronize_sched - Snapshot current RCU-sched state
3279 *
3280 * Returns a cookie that is used by a later call to cond_synchronize_sched()
3281 * to determine whether or not a full grace period has elapsed in the
3282 * meantime.
3283 */
3284unsigned long get_state_synchronize_sched(void)
3285{
3286 /*
3287 * Any prior manipulation of RCU-protected data must happen
3288 * before the load from ->gpnum.
3289 */
3290 smp_mb(); /* ^^^ */
3291
3292 /*
3293 * Make sure this load happens before the purportedly
3294 * time-consuming work between get_state_synchronize_sched()
3295 * and cond_synchronize_sched().
3296 */
3297 return smp_load_acquire(&rcu_sched_state.gpnum);
3298}
3299EXPORT_SYMBOL_GPL(get_state_synchronize_sched);
3300
3301/**
3302 * cond_synchronize_sched - Conditionally wait for an RCU-sched grace period
3303 *
3304 * @oldstate: return value from earlier call to get_state_synchronize_sched()
3305 *
3306 * If a full RCU-sched grace period has elapsed since the earlier call to
3307 * get_state_synchronize_sched(), just return. Otherwise, invoke
3308 * synchronize_sched() to wait for a full grace period.
3309 *
3310 * Yes, this function does not take counter wrap into account. But
3311 * counter wrap is harmless. If the counter wraps, we have waited for
3312 * more than 2 billion grace periods (and way more on a 64-bit system!),
3313 * so waiting for one additional grace period should be just fine.
3314 */
3315void cond_synchronize_sched(unsigned long oldstate)
3316{
3317 unsigned long newstate;
3318
3319 /*
3320 * Ensure that this load happens before any RCU-destructive
3321 * actions the caller might carry out after we return.
3322 */
3323 newstate = smp_load_acquire(&rcu_sched_state.completed);
3324 if (ULONG_CMP_GE(oldstate, newstate))
3325 synchronize_sched();
3326}
3327EXPORT_SYMBOL_GPL(cond_synchronize_sched);
3328
Paul E. McKenney28f00762015-06-25 15:00:58 -07003329/* Adjust sequence number for start of update-side operation. */
3330static void rcu_seq_start(unsigned long *sp)
3331{
3332 WRITE_ONCE(*sp, *sp + 1);
3333 smp_mb(); /* Ensure update-side operation after counter increment. */
3334 WARN_ON_ONCE(!(*sp & 0x1));
3335}
3336
3337/* Adjust sequence number for end of update-side operation. */
3338static void rcu_seq_end(unsigned long *sp)
3339{
3340 smp_mb(); /* Ensure update-side operation before counter increment. */
3341 WRITE_ONCE(*sp, *sp + 1);
3342 WARN_ON_ONCE(*sp & 0x1);
3343}
3344
3345/* Take a snapshot of the update side's sequence number. */
3346static unsigned long rcu_seq_snap(unsigned long *sp)
3347{
3348 unsigned long s;
3349
3350 smp_mb(); /* Caller's modifications seen first by other CPUs. */
3351 s = (READ_ONCE(*sp) + 3) & ~0x1;
3352 smp_mb(); /* Above access must not bleed into critical section. */
3353 return s;
3354}
3355
3356/*
3357 * Given a snapshot from rcu_seq_snap(), determine whether or not a
3358 * full update-side operation has occurred.
3359 */
3360static bool rcu_seq_done(unsigned long *sp, unsigned long s)
3361{
3362 return ULONG_CMP_GE(READ_ONCE(*sp), s);
3363}
3364
3365/* Wrapper functions for expedited grace periods. */
3366static void rcu_exp_gp_seq_start(struct rcu_state *rsp)
3367{
3368 rcu_seq_start(&rsp->expedited_sequence);
3369}
3370static void rcu_exp_gp_seq_end(struct rcu_state *rsp)
3371{
3372 rcu_seq_end(&rsp->expedited_sequence);
Paul E. McKenney704dd432015-06-27 09:36:29 -07003373 smp_mb(); /* Ensure that consecutive grace periods serialize. */
Paul E. McKenney28f00762015-06-25 15:00:58 -07003374}
3375static unsigned long rcu_exp_gp_seq_snap(struct rcu_state *rsp)
3376{
3377 return rcu_seq_snap(&rsp->expedited_sequence);
3378}
3379static bool rcu_exp_gp_seq_done(struct rcu_state *rsp, unsigned long s)
3380{
3381 return rcu_seq_done(&rsp->expedited_sequence, s);
3382}
3383
Paul E. McKenney7922cd02015-07-31 13:34:32 -07003384/*
Paul E. McKenneyb9585e92015-07-31 16:04:45 -07003385 * Reset the ->expmaskinit values in the rcu_node tree to reflect any
3386 * recent CPU-online activity. Note that these masks are not cleared
3387 * when CPUs go offline, so they reflect the union of all CPUs that have
3388 * ever been online. This means that this function normally takes its
3389 * no-work-to-do fastpath.
3390 */
3391static void sync_exp_reset_tree_hotplug(struct rcu_state *rsp)
3392{
3393 bool done;
3394 unsigned long flags;
3395 unsigned long mask;
3396 unsigned long oldmask;
3397 int ncpus = READ_ONCE(rsp->ncpus);
3398 struct rcu_node *rnp;
3399 struct rcu_node *rnp_up;
3400
3401 /* If no new CPUs onlined since last time, nothing to do. */
3402 if (likely(ncpus == rsp->ncpus_snap))
3403 return;
3404 rsp->ncpus_snap = ncpus;
3405
3406 /*
3407 * Each pass through the following loop propagates newly onlined
3408 * CPUs for the current rcu_node structure up the rcu_node tree.
3409 */
3410 rcu_for_each_leaf_node(rsp, rnp) {
Peter Zijlstra2a67e742015-10-08 12:24:23 +02003411 raw_spin_lock_irqsave_rcu_node(rnp, flags);
Paul E. McKenneyb9585e92015-07-31 16:04:45 -07003412 if (rnp->expmaskinit == rnp->expmaskinitnext) {
3413 raw_spin_unlock_irqrestore(&rnp->lock, flags);
3414 continue; /* No new CPUs, nothing to do. */
3415 }
3416
3417 /* Update this node's mask, track old value for propagation. */
3418 oldmask = rnp->expmaskinit;
3419 rnp->expmaskinit = rnp->expmaskinitnext;
3420 raw_spin_unlock_irqrestore(&rnp->lock, flags);
3421
3422 /* If was already nonzero, nothing to propagate. */
3423 if (oldmask)
3424 continue;
3425
3426 /* Propagate the new CPU up the tree. */
3427 mask = rnp->grpmask;
3428 rnp_up = rnp->parent;
3429 done = false;
3430 while (rnp_up) {
Peter Zijlstra2a67e742015-10-08 12:24:23 +02003431 raw_spin_lock_irqsave_rcu_node(rnp_up, flags);
Paul E. McKenneyb9585e92015-07-31 16:04:45 -07003432 if (rnp_up->expmaskinit)
3433 done = true;
3434 rnp_up->expmaskinit |= mask;
3435 raw_spin_unlock_irqrestore(&rnp_up->lock, flags);
3436 if (done)
3437 break;
3438 mask = rnp_up->grpmask;
3439 rnp_up = rnp_up->parent;
3440 }
3441 }
3442}
3443
3444/*
3445 * Reset the ->expmask values in the rcu_node tree in preparation for
3446 * a new expedited grace period.
3447 */
3448static void __maybe_unused sync_exp_reset_tree(struct rcu_state *rsp)
3449{
3450 unsigned long flags;
3451 struct rcu_node *rnp;
3452
3453 sync_exp_reset_tree_hotplug(rsp);
3454 rcu_for_each_node_breadth_first(rsp, rnp) {
Peter Zijlstra2a67e742015-10-08 12:24:23 +02003455 raw_spin_lock_irqsave_rcu_node(rnp, flags);
Paul E. McKenneyb9585e92015-07-31 16:04:45 -07003456 WARN_ON_ONCE(rnp->expmask);
3457 rnp->expmask = rnp->expmaskinit;
3458 raw_spin_unlock_irqrestore(&rnp->lock, flags);
3459 }
3460}
3461
3462/*
Paul E. McKenney8203d6d2015-08-02 13:53:17 -07003463 * Return non-zero if there is no RCU expedited grace period in progress
Paul E. McKenney7922cd02015-07-31 13:34:32 -07003464 * for the specified rcu_node structure, in other words, if all CPUs and
3465 * tasks covered by the specified rcu_node structure have done their bit
3466 * for the current expedited grace period. Works only for preemptible
3467 * RCU -- other RCU implementation use other means.
3468 *
3469 * Caller must hold the root rcu_node's exp_funnel_mutex.
3470 */
3471static int sync_rcu_preempt_exp_done(struct rcu_node *rnp)
3472{
Paul E. McKenney8203d6d2015-08-02 13:53:17 -07003473 return rnp->exp_tasks == NULL &&
Paul E. McKenney7922cd02015-07-31 13:34:32 -07003474 READ_ONCE(rnp->expmask) == 0;
3475}
3476
3477/*
3478 * Report the exit from RCU read-side critical section for the last task
3479 * that queued itself during or before the current expedited preemptible-RCU
3480 * grace period. This event is reported either to the rcu_node structure on
3481 * which the task was queued or to one of that rcu_node structure's ancestors,
3482 * recursively up the tree. (Calm down, calm down, we do the recursion
3483 * iteratively!)
3484 *
Paul E. McKenney8203d6d2015-08-02 13:53:17 -07003485 * Caller must hold the root rcu_node's exp_funnel_mutex and the
3486 * specified rcu_node structure's ->lock.
Paul E. McKenney7922cd02015-07-31 13:34:32 -07003487 */
Paul E. McKenney8203d6d2015-08-02 13:53:17 -07003488static void __rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp,
3489 bool wake, unsigned long flags)
3490 __releases(rnp->lock)
Paul E. McKenney7922cd02015-07-31 13:34:32 -07003491{
Paul E. McKenney7922cd02015-07-31 13:34:32 -07003492 unsigned long mask;
3493
Paul E. McKenney7922cd02015-07-31 13:34:32 -07003494 for (;;) {
3495 if (!sync_rcu_preempt_exp_done(rnp)) {
Paul E. McKenney8203d6d2015-08-02 13:53:17 -07003496 if (!rnp->expmask)
3497 rcu_initiate_boost(rnp, flags);
3498 else
3499 raw_spin_unlock_irqrestore(&rnp->lock, flags);
Paul E. McKenney7922cd02015-07-31 13:34:32 -07003500 break;
3501 }
3502 if (rnp->parent == NULL) {
3503 raw_spin_unlock_irqrestore(&rnp->lock, flags);
3504 if (wake) {
3505 smp_mb(); /* EGP done before wake_up(). */
3506 wake_up(&rsp->expedited_wq);
3507 }
3508 break;
3509 }
3510 mask = rnp->grpmask;
3511 raw_spin_unlock(&rnp->lock); /* irqs remain disabled */
3512 rnp = rnp->parent;
Peter Zijlstra2a67e742015-10-08 12:24:23 +02003513 raw_spin_lock_rcu_node(rnp); /* irqs already disabled */
Paul E. McKenney8203d6d2015-08-02 13:53:17 -07003514 WARN_ON_ONCE(!(rnp->expmask & mask));
Paul E. McKenney7922cd02015-07-31 13:34:32 -07003515 rnp->expmask &= ~mask;
3516 }
3517}
3518
Paul E. McKenney8203d6d2015-08-02 13:53:17 -07003519/*
3520 * Report expedited quiescent state for specified node. This is a
3521 * lock-acquisition wrapper function for __rcu_report_exp_rnp().
3522 *
3523 * Caller must hold the root rcu_node's exp_funnel_mutex.
3524 */
3525static void __maybe_unused rcu_report_exp_rnp(struct rcu_state *rsp,
3526 struct rcu_node *rnp, bool wake)
3527{
3528 unsigned long flags;
3529
Peter Zijlstra2a67e742015-10-08 12:24:23 +02003530 raw_spin_lock_irqsave_rcu_node(rnp, flags);
Paul E. McKenney8203d6d2015-08-02 13:53:17 -07003531 __rcu_report_exp_rnp(rsp, rnp, wake, flags);
3532}
3533
3534/*
3535 * Report expedited quiescent state for multiple CPUs, all covered by the
3536 * specified leaf rcu_node structure. Caller must hold the root
3537 * rcu_node's exp_funnel_mutex.
3538 */
3539static void rcu_report_exp_cpu_mult(struct rcu_state *rsp, struct rcu_node *rnp,
3540 unsigned long mask, bool wake)
3541{
3542 unsigned long flags;
3543
Peter Zijlstra2a67e742015-10-08 12:24:23 +02003544 raw_spin_lock_irqsave_rcu_node(rnp, flags);
Paul E. McKenney338b0f72015-09-03 00:45:02 -07003545 if (!(rnp->expmask & mask)) {
3546 raw_spin_unlock_irqrestore(&rnp->lock, flags);
3547 return;
3548 }
Paul E. McKenney8203d6d2015-08-02 13:53:17 -07003549 rnp->expmask &= ~mask;
3550 __rcu_report_exp_rnp(rsp, rnp, wake, flags); /* Releases rnp->lock. */
3551}
3552
3553/*
3554 * Report expedited quiescent state for specified rcu_data (CPU).
3555 * Caller must hold the root rcu_node's exp_funnel_mutex.
3556 */
Paul E. McKenney6587a232015-08-06 16:50:39 -07003557static void rcu_report_exp_rdp(struct rcu_state *rsp, struct rcu_data *rdp,
3558 bool wake)
Paul E. McKenney8203d6d2015-08-02 13:53:17 -07003559{
3560 rcu_report_exp_cpu_mult(rsp, rdp->mynode, rdp->grpmask, wake);
3561}
3562
Paul E. McKenney29fd9302015-06-25 19:03:16 -07003563/* Common code for synchronize_{rcu,sched}_expedited() work-done checking. */
3564static bool sync_exp_work_done(struct rcu_state *rsp, struct rcu_node *rnp,
Paul E. McKenney2cd6ffa2015-06-29 17:06:39 -07003565 struct rcu_data *rdp,
Paul E. McKenney29fd9302015-06-25 19:03:16 -07003566 atomic_long_t *stat, unsigned long s)
Paul E. McKenney385b73c2015-06-24 14:20:08 -07003567{
Paul E. McKenney28f00762015-06-25 15:00:58 -07003568 if (rcu_exp_gp_seq_done(rsp, s)) {
Paul E. McKenney385b73c2015-06-24 14:20:08 -07003569 if (rnp)
3570 mutex_unlock(&rnp->exp_funnel_mutex);
Paul E. McKenney2cd6ffa2015-06-29 17:06:39 -07003571 else if (rdp)
3572 mutex_unlock(&rdp->exp_funnel_mutex);
Paul E. McKenney385b73c2015-06-24 14:20:08 -07003573 /* Ensure test happens before caller kfree(). */
3574 smp_mb__before_atomic(); /* ^^^ */
3575 atomic_long_inc(stat);
Paul E. McKenney385b73c2015-06-24 14:20:08 -07003576 return true;
3577 }
3578 return false;
3579}
3580
Paul E. McKenneyb09e5f82015-06-25 16:30:54 -07003581/*
3582 * Funnel-lock acquisition for expedited grace periods. Returns a
3583 * pointer to the root rcu_node structure, or NULL if some other
3584 * task did the expedited grace period for us.
3585 */
3586static struct rcu_node *exp_funnel_lock(struct rcu_state *rsp, unsigned long s)
3587{
Paul E. McKenney2cd6ffa2015-06-29 17:06:39 -07003588 struct rcu_data *rdp;
Paul E. McKenneyb09e5f82015-06-25 16:30:54 -07003589 struct rcu_node *rnp0;
3590 struct rcu_node *rnp1 = NULL;
3591
3592 /*
Paul E. McKenneycdacbe12015-07-11 16:24:45 -07003593 * First try directly acquiring the root lock in order to reduce
3594 * latency in the common case where expedited grace periods are
3595 * rare. We check mutex_is_locked() to avoid pathological levels of
3596 * memory contention on ->exp_funnel_mutex in the heavy-load case.
3597 */
3598 rnp0 = rcu_get_root(rsp);
3599 if (!mutex_is_locked(&rnp0->exp_funnel_mutex)) {
3600 if (mutex_trylock(&rnp0->exp_funnel_mutex)) {
3601 if (sync_exp_work_done(rsp, rnp0, NULL,
3602 &rsp->expedited_workdone0, s))
3603 return NULL;
3604 return rnp0;
3605 }
3606 }
3607
3608 /*
Paul E. McKenneyb09e5f82015-06-25 16:30:54 -07003609 * Each pass through the following loop works its way
3610 * up the rcu_node tree, returning if others have done the
3611 * work or otherwise falls through holding the root rnp's
3612 * ->exp_funnel_mutex. The mapping from CPU to rcu_node structure
3613 * can be inexact, as it is just promoting locality and is not
3614 * strictly needed for correctness.
3615 */
Paul E. McKenney2cd6ffa2015-06-29 17:06:39 -07003616 rdp = per_cpu_ptr(rsp->rda, raw_smp_processor_id());
3617 if (sync_exp_work_done(rsp, NULL, NULL, &rsp->expedited_workdone1, s))
3618 return NULL;
3619 mutex_lock(&rdp->exp_funnel_mutex);
3620 rnp0 = rdp->mynode;
Paul E. McKenneyb09e5f82015-06-25 16:30:54 -07003621 for (; rnp0 != NULL; rnp0 = rnp0->parent) {
Paul E. McKenney2cd6ffa2015-06-29 17:06:39 -07003622 if (sync_exp_work_done(rsp, rnp1, rdp,
3623 &rsp->expedited_workdone2, s))
Paul E. McKenneyb09e5f82015-06-25 16:30:54 -07003624 return NULL;
3625 mutex_lock(&rnp0->exp_funnel_mutex);
3626 if (rnp1)
3627 mutex_unlock(&rnp1->exp_funnel_mutex);
Paul E. McKenney2cd6ffa2015-06-29 17:06:39 -07003628 else
3629 mutex_unlock(&rdp->exp_funnel_mutex);
Paul E. McKenneyb09e5f82015-06-25 16:30:54 -07003630 rnp1 = rnp0;
3631 }
Paul E. McKenney2cd6ffa2015-06-29 17:06:39 -07003632 if (sync_exp_work_done(rsp, rnp1, rdp,
3633 &rsp->expedited_workdone3, s))
Paul E. McKenneyb09e5f82015-06-25 16:30:54 -07003634 return NULL;
3635 return rnp1;
3636}
3637
Paul E. McKenneycf3620a2015-06-30 11:14:32 -07003638/* Invoked on each online non-idle CPU for expedited quiescent state. */
Paul E. McKenney338b0f72015-09-03 00:45:02 -07003639static void sync_sched_exp_handler(void *data)
Paul E. McKenney3d3b7db2012-01-23 17:05:46 -08003640{
Paul E. McKenney338b0f72015-09-03 00:45:02 -07003641 struct rcu_data *rdp;
3642 struct rcu_node *rnp;
3643 struct rcu_state *rsp = data;
Paul E. McKenneyb09e5f82015-06-25 16:30:54 -07003644
Paul E. McKenney338b0f72015-09-03 00:45:02 -07003645 rdp = this_cpu_ptr(rsp->rda);
3646 rnp = rdp->mynode;
3647 if (!(READ_ONCE(rnp->expmask) & rdp->grpmask) ||
3648 __this_cpu_read(rcu_sched_data.cpu_no_qs.b.exp))
3649 return;
Paul E. McKenney6587a232015-08-06 16:50:39 -07003650 __this_cpu_write(rcu_sched_data.cpu_no_qs.b.exp, true);
3651 resched_cpu(smp_processor_id());
Paul E. McKenney3d3b7db2012-01-23 17:05:46 -08003652}
3653
Paul E. McKenney338b0f72015-09-03 00:45:02 -07003654/* Send IPI for expedited cleanup if needed at end of CPU-hotplug operation. */
3655static void sync_sched_exp_online_cleanup(int cpu)
3656{
3657 struct rcu_data *rdp;
3658 int ret;
3659 struct rcu_node *rnp;
3660 struct rcu_state *rsp = &rcu_sched_state;
3661
3662 rdp = per_cpu_ptr(rsp->rda, cpu);
3663 rnp = rdp->mynode;
3664 if (!(READ_ONCE(rnp->expmask) & rdp->grpmask))
3665 return;
3666 ret = smp_call_function_single(cpu, sync_sched_exp_handler, rsp, 0);
3667 WARN_ON_ONCE(ret);
3668}
3669
Paul E. McKenneybce5fa12015-08-05 16:03:54 -07003670/*
3671 * Select the nodes that the upcoming expedited grace period needs
3672 * to wait for.
3673 */
Paul E. McKenneydcdb8802015-08-15 19:00:31 -07003674static void sync_rcu_exp_select_cpus(struct rcu_state *rsp,
3675 smp_call_func_t func)
Paul E. McKenneybce5fa12015-08-05 16:03:54 -07003676{
3677 int cpu;
3678 unsigned long flags;
3679 unsigned long mask;
3680 unsigned long mask_ofl_test;
3681 unsigned long mask_ofl_ipi;
Paul E. McKenney6587a232015-08-06 16:50:39 -07003682 int ret;
Paul E. McKenneybce5fa12015-08-05 16:03:54 -07003683 struct rcu_node *rnp;
3684
3685 sync_exp_reset_tree(rsp);
3686 rcu_for_each_leaf_node(rsp, rnp) {
Peter Zijlstra2a67e742015-10-08 12:24:23 +02003687 raw_spin_lock_irqsave_rcu_node(rnp, flags);
Paul E. McKenneybce5fa12015-08-05 16:03:54 -07003688
3689 /* Each pass checks a CPU for identity, offline, and idle. */
3690 mask_ofl_test = 0;
3691 for (cpu = rnp->grplo; cpu <= rnp->grphi; cpu++) {
3692 struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
3693 struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
3694
3695 if (raw_smp_processor_id() == cpu ||
Paul E. McKenneybce5fa12015-08-05 16:03:54 -07003696 !(atomic_add_return(0, &rdtp->dynticks) & 0x1))
3697 mask_ofl_test |= rdp->grpmask;
3698 }
3699 mask_ofl_ipi = rnp->expmask & ~mask_ofl_test;
3700
3701 /*
3702 * Need to wait for any blocked tasks as well. Note that
3703 * additional blocking tasks will also block the expedited
3704 * GP until such time as the ->expmask bits are cleared.
3705 */
3706 if (rcu_preempt_has_tasks(rnp))
3707 rnp->exp_tasks = rnp->blkd_tasks.next;
3708 raw_spin_unlock_irqrestore(&rnp->lock, flags);
3709
3710 /* IPI the remaining CPUs for expedited quiescent state. */
3711 mask = 1;
3712 for (cpu = rnp->grplo; cpu <= rnp->grphi; cpu++, mask <<= 1) {
3713 if (!(mask_ofl_ipi & mask))
3714 continue;
Paul E. McKenney338b0f72015-09-03 00:45:02 -07003715retry_ipi:
Paul E. McKenneydcdb8802015-08-15 19:00:31 -07003716 ret = smp_call_function_single(cpu, func, rsp, 0);
Paul E. McKenney338b0f72015-09-03 00:45:02 -07003717 if (!ret) {
Paul E. McKenney6587a232015-08-06 16:50:39 -07003718 mask_ofl_ipi &= ~mask;
Paul E. McKenney338b0f72015-09-03 00:45:02 -07003719 } else {
3720 /* Failed, raced with offline. */
Paul E. McKenney6cf10082015-10-08 15:36:54 -07003721 raw_spin_lock_irqsave_rcu_node(rnp, flags);
Paul E. McKenney338b0f72015-09-03 00:45:02 -07003722 if (cpu_online(cpu) &&
3723 (rnp->expmask & mask)) {
3724 raw_spin_unlock_irqrestore(&rnp->lock,
3725 flags);
3726 schedule_timeout_uninterruptible(1);
3727 if (cpu_online(cpu) &&
3728 (rnp->expmask & mask))
3729 goto retry_ipi;
Paul E. McKenney6cf10082015-10-08 15:36:54 -07003730 raw_spin_lock_irqsave_rcu_node(rnp,
3731 flags);
Paul E. McKenney338b0f72015-09-03 00:45:02 -07003732 }
3733 if (!(rnp->expmask & mask))
3734 mask_ofl_ipi &= ~mask;
3735 raw_spin_unlock_irqrestore(&rnp->lock, flags);
3736 }
Paul E. McKenneybce5fa12015-08-05 16:03:54 -07003737 }
3738 /* Report quiescent states for those that went offline. */
3739 mask_ofl_test |= mask_ofl_ipi;
3740 if (mask_ofl_test)
3741 rcu_report_exp_cpu_mult(rsp, rnp, mask_ofl_test, false);
3742 }
Paul E. McKenney3d3b7db2012-01-23 17:05:46 -08003743}
3744
Paul E. McKenneycf3620a2015-06-30 11:14:32 -07003745static void synchronize_sched_expedited_wait(struct rcu_state *rsp)
3746{
3747 int cpu;
3748 unsigned long jiffies_stall;
3749 unsigned long jiffies_start;
Paul E. McKenneybce5fa12015-08-05 16:03:54 -07003750 unsigned long mask;
3751 struct rcu_node *rnp;
3752 struct rcu_node *rnp_root = rcu_get_root(rsp);
Paul E. McKenneycf3620a2015-06-30 11:14:32 -07003753 int ret;
3754
3755 jiffies_stall = rcu_jiffies_till_stall_check();
3756 jiffies_start = jiffies;
3757
3758 for (;;) {
3759 ret = wait_event_interruptible_timeout(
3760 rsp->expedited_wq,
Paul E. McKenneybce5fa12015-08-05 16:03:54 -07003761 sync_rcu_preempt_exp_done(rnp_root),
Paul E. McKenneycf3620a2015-06-30 11:14:32 -07003762 jiffies_stall);
3763 if (ret > 0)
3764 return;
3765 if (ret < 0) {
3766 /* Hit a signal, disable CPU stall warnings. */
3767 wait_event(rsp->expedited_wq,
Paul E. McKenneybce5fa12015-08-05 16:03:54 -07003768 sync_rcu_preempt_exp_done(rnp_root));
Paul E. McKenneycf3620a2015-06-30 11:14:32 -07003769 return;
3770 }
Paul E. McKenneyc5865632015-08-18 11:25:48 -07003771 pr_err("INFO: %s detected expedited stalls on CPUs/tasks: {",
Paul E. McKenneycf3620a2015-06-30 11:14:32 -07003772 rsp->name);
Paul E. McKenneybce5fa12015-08-05 16:03:54 -07003773 rcu_for_each_leaf_node(rsp, rnp) {
Paul E. McKenney74611ec2015-08-18 10:20:43 -07003774 (void)rcu_print_task_exp_stall(rnp);
Paul E. McKenneybce5fa12015-08-05 16:03:54 -07003775 mask = 1;
3776 for (cpu = rnp->grplo; cpu <= rnp->grphi; cpu++, mask <<= 1) {
Paul E. McKenney74611ec2015-08-18 10:20:43 -07003777 struct rcu_data *rdp;
Paul E. McKenneycf3620a2015-06-30 11:14:32 -07003778
Paul E. McKenneybce5fa12015-08-05 16:03:54 -07003779 if (!(rnp->expmask & mask))
3780 continue;
Paul E. McKenney74611ec2015-08-18 10:20:43 -07003781 rdp = per_cpu_ptr(rsp->rda, cpu);
3782 pr_cont(" %d-%c%c%c", cpu,
3783 "O."[cpu_online(cpu)],
3784 "o."[!!(rdp->grpmask & rnp->expmaskinit)],
3785 "N."[!!(rdp->grpmask & rnp->expmaskinitnext)]);
Paul E. McKenneybce5fa12015-08-05 16:03:54 -07003786 }
3787 mask <<= 1;
Paul E. McKenneycf3620a2015-06-30 11:14:32 -07003788 }
3789 pr_cont(" } %lu jiffies s: %lu\n",
3790 jiffies - jiffies_start, rsp->expedited_sequence);
Paul E. McKenneybce5fa12015-08-05 16:03:54 -07003791 rcu_for_each_leaf_node(rsp, rnp) {
3792 mask = 1;
3793 for (cpu = rnp->grplo; cpu <= rnp->grphi; cpu++, mask <<= 1) {
3794 if (!(rnp->expmask & mask))
3795 continue;
3796 dump_cpu_task(cpu);
3797 }
Paul E. McKenneycf3620a2015-06-30 11:14:32 -07003798 }
3799 jiffies_stall = 3 * rcu_jiffies_till_stall_check() + 3;
3800 }
3801}
3802
Paul E. McKenney236fefa2012-01-31 14:00:41 -08003803/**
3804 * synchronize_sched_expedited - Brute-force RCU-sched grace period
Paul E. McKenney3d3b7db2012-01-23 17:05:46 -08003805 *
Paul E. McKenney236fefa2012-01-31 14:00:41 -08003806 * Wait for an RCU-sched grace period to elapse, but use a "big hammer"
3807 * approach to force the grace period to end quickly. This consumes
3808 * significant time on all CPUs and is unfriendly to real-time workloads,
3809 * so is thus not recommended for any sort of common-case code. In fact,
3810 * if you are using synchronize_sched_expedited() in a loop, please
3811 * restructure your code to batch your updates, and then use a single
3812 * synchronize_sched() instead.
3813 *
Paul E. McKenneyd6ada2c2015-06-24 10:46:30 -07003814 * This implementation can be thought of as an application of sequence
3815 * locking to expedited grace periods, but using the sequence counter to
3816 * determine when someone else has already done the work instead of for
Paul E. McKenney385b73c2015-06-24 14:20:08 -07003817 * retrying readers.
Paul E. McKenney3d3b7db2012-01-23 17:05:46 -08003818 */
3819void synchronize_sched_expedited(void)
3820{
Paul E. McKenney7fd0ddc2015-06-25 16:35:03 -07003821 unsigned long s;
Paul E. McKenneyb09e5f82015-06-25 16:30:54 -07003822 struct rcu_node *rnp;
Paul E. McKenney40694d62012-10-11 15:24:03 -07003823 struct rcu_state *rsp = &rcu_sched_state;
Paul E. McKenney3d3b7db2012-01-23 17:05:46 -08003824
Paul E. McKenneyd6ada2c2015-06-24 10:46:30 -07003825 /* Take a snapshot of the sequence number. */
Paul E. McKenney28f00762015-06-25 15:00:58 -07003826 s = rcu_exp_gp_seq_snap(rsp);
Paul E. McKenney1924bcb2012-10-11 12:30:37 -07003827
Paul E. McKenneyb09e5f82015-06-25 16:30:54 -07003828 rnp = exp_funnel_lock(rsp, s);
Paul E. McKenney807226e2015-08-07 12:03:45 -07003829 if (rnp == NULL)
Paul E. McKenneyb09e5f82015-06-25 16:30:54 -07003830 return; /* Someone else did our work for us. */
Paul E. McKenney3d3b7db2012-01-23 17:05:46 -08003831
Paul E. McKenney28f00762015-06-25 15:00:58 -07003832 rcu_exp_gp_seq_start(rsp);
Paul E. McKenney338b0f72015-09-03 00:45:02 -07003833 sync_rcu_exp_select_cpus(rsp, sync_sched_exp_handler);
Paul E. McKenneybce5fa12015-08-05 16:03:54 -07003834 synchronize_sched_expedited_wait(rsp);
Peter Zijlstra3a6d7c62015-06-25 11:27:10 -07003835
Paul E. McKenney28f00762015-06-25 15:00:58 -07003836 rcu_exp_gp_seq_end(rsp);
Paul E. McKenneyb09e5f82015-06-25 16:30:54 -07003837 mutex_unlock(&rnp->exp_funnel_mutex);
Paul E. McKenney3d3b7db2012-01-23 17:05:46 -08003838}
3839EXPORT_SYMBOL_GPL(synchronize_sched_expedited);
3840
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01003841/*
3842 * Check to see if there is any immediate RCU-related work to be done
3843 * by the current CPU, for the specified type of RCU, returning 1 if so.
3844 * The checks are in order of increasing expense: checks that can be
3845 * carried out against CPU-local state are performed first. However,
3846 * we must check for CPU stalls first, else we might not get a chance.
3847 */
3848static int __rcu_pending(struct rcu_state *rsp, struct rcu_data *rdp)
3849{
Paul E. McKenney2f51f982009-11-13 19:51:39 -08003850 struct rcu_node *rnp = rdp->mynode;
3851
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01003852 rdp->n_rcu_pending++;
3853
3854 /* Check for CPU stalls, if enabled. */
3855 check_cpu_stall(rsp, rdp);
3856
Paul E. McKenneya0969322013-11-08 09:03:10 -08003857 /* Is this CPU a NO_HZ_FULL CPU that should ignore RCU? */
3858 if (rcu_nohz_full_cpu(rsp))
3859 return 0;
3860
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01003861 /* Is the RCU core waiting for a quiescent state from this CPU? */
Paul E. McKenney5c51dd72011-08-04 06:59:03 -07003862 if (rcu_scheduler_fully_active &&
Paul E. McKenney5b74c452015-08-06 15:16:57 -07003863 rdp->core_needs_qs && rdp->cpu_no_qs.b.norm &&
Paul E. McKenney5cd37192014-12-13 20:32:04 -08003864 rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr)) {
Paul E. McKenney97c668b2015-08-06 11:31:51 -07003865 rdp->n_rp_core_needs_qs++;
3866 } else if (rdp->core_needs_qs &&
Paul E. McKenney5b74c452015-08-06 15:16:57 -07003867 (!rdp->cpu_no_qs.b.norm ||
Paul E. McKenney5cd37192014-12-13 20:32:04 -08003868 rdp->rcu_qs_ctr_snap != __this_cpu_read(rcu_qs_ctr))) {
Paul E. McKenneyd21670a2010-04-14 17:39:26 -07003869 rdp->n_rp_report_qs++;
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01003870 return 1;
Paul E. McKenney7ba5c842009-04-13 21:31:17 -07003871 }
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01003872
3873 /* Does this CPU have callbacks ready to invoke? */
Paul E. McKenney7ba5c842009-04-13 21:31:17 -07003874 if (cpu_has_callbacks_ready_to_invoke(rdp)) {
3875 rdp->n_rp_cb_ready++;
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01003876 return 1;
Paul E. McKenney7ba5c842009-04-13 21:31:17 -07003877 }
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01003878
3879 /* Has RCU gone idle with this CPU needing another grace period? */
Paul E. McKenney7ba5c842009-04-13 21:31:17 -07003880 if (cpu_needs_another_gp(rsp, rdp)) {
3881 rdp->n_rp_cpu_needs_gp++;
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01003882 return 1;
Paul E. McKenney7ba5c842009-04-13 21:31:17 -07003883 }
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01003884
3885 /* Has another RCU grace period completed? */
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08003886 if (READ_ONCE(rnp->completed) != rdp->completed) { /* outside lock */
Paul E. McKenney7ba5c842009-04-13 21:31:17 -07003887 rdp->n_rp_gp_completed++;
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01003888 return 1;
Paul E. McKenney7ba5c842009-04-13 21:31:17 -07003889 }
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01003890
3891 /* Has a new RCU grace period started? */
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08003892 if (READ_ONCE(rnp->gpnum) != rdp->gpnum ||
3893 unlikely(READ_ONCE(rdp->gpwrap))) { /* outside lock */
Paul E. McKenney7ba5c842009-04-13 21:31:17 -07003894 rdp->n_rp_gp_started++;
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01003895 return 1;
Paul E. McKenney7ba5c842009-04-13 21:31:17 -07003896 }
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01003897
Paul E. McKenney96d3fd02013-10-04 14:33:34 -07003898 /* Does this CPU need a deferred NOCB wakeup? */
3899 if (rcu_nocb_need_deferred_wakeup(rdp)) {
3900 rdp->n_rp_nocb_defer_wakeup++;
3901 return 1;
3902 }
3903
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01003904 /* nothing to do */
Paul E. McKenney7ba5c842009-04-13 21:31:17 -07003905 rdp->n_rp_need_nothing++;
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01003906 return 0;
3907}
3908
3909/*
3910 * Check to see if there is any immediate RCU-related work to be done
3911 * by the current CPU, returning 1 if so. This function is part of the
3912 * RCU implementation; it is -not- an exported member of the RCU API.
3913 */
Paul E. McKenneye3950ec2014-10-21 08:03:57 -07003914static int rcu_pending(void)
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01003915{
Paul E. McKenney6ce75a22012-06-12 11:01:13 -07003916 struct rcu_state *rsp;
3917
3918 for_each_rcu_flavor(rsp)
Paul E. McKenneye3950ec2014-10-21 08:03:57 -07003919 if (__rcu_pending(rsp, this_cpu_ptr(rsp->rda)))
Paul E. McKenney6ce75a22012-06-12 11:01:13 -07003920 return 1;
3921 return 0;
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01003922}
3923
3924/*
Paul E. McKenneyc0f4dfd2012-12-28 11:30:36 -08003925 * Return true if the specified CPU has any callback. If all_lazy is
3926 * non-NULL, store an indication of whether all callbacks are lazy.
3927 * (If there are no callbacks, all of them are deemed to be lazy.)
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01003928 */
Nicholas Mc Guire82072c42015-05-11 18:12:27 +02003929static bool __maybe_unused rcu_cpu_has_callbacks(bool *all_lazy)
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01003930{
Paul E. McKenneyc0f4dfd2012-12-28 11:30:36 -08003931 bool al = true;
3932 bool hc = false;
3933 struct rcu_data *rdp;
Paul E. McKenney6ce75a22012-06-12 11:01:13 -07003934 struct rcu_state *rsp;
3935
Paul E. McKenneyc0f4dfd2012-12-28 11:30:36 -08003936 for_each_rcu_flavor(rsp) {
Paul E. McKenneyaa6da512014-10-21 13:23:08 -07003937 rdp = this_cpu_ptr(rsp->rda);
Paul E. McKenney69c8d282013-09-03 09:52:20 -07003938 if (!rdp->nxtlist)
3939 continue;
3940 hc = true;
3941 if (rdp->qlen != rdp->qlen_lazy || !all_lazy) {
Paul E. McKenneyc0f4dfd2012-12-28 11:30:36 -08003942 al = false;
Paul E. McKenney69c8d282013-09-03 09:52:20 -07003943 break;
3944 }
Paul E. McKenneyc0f4dfd2012-12-28 11:30:36 -08003945 }
3946 if (all_lazy)
3947 *all_lazy = al;
3948 return hc;
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01003949}
3950
Paul E. McKenneyb1420f12012-03-01 13:18:08 -08003951/*
Paul E. McKenneya83eff02012-05-23 18:47:05 -07003952 * Helper function for _rcu_barrier() tracing. If tracing is disabled,
3953 * the compiler is expected to optimize this away.
3954 */
Steven Rostedt (Red Hat)e66c33d2013-07-12 16:50:28 -04003955static void _rcu_barrier_trace(struct rcu_state *rsp, const char *s,
Paul E. McKenneya83eff02012-05-23 18:47:05 -07003956 int cpu, unsigned long done)
3957{
3958 trace_rcu_barrier(rsp->name, s, cpu,
3959 atomic_read(&rsp->barrier_cpu_count), done);
3960}
3961
3962/*
Paul E. McKenneyb1420f12012-03-01 13:18:08 -08003963 * RCU callback function for _rcu_barrier(). If we are last, wake
3964 * up the task executing _rcu_barrier().
3965 */
Paul E. McKenney24ebbca2012-05-29 00:34:56 -07003966static void rcu_barrier_callback(struct rcu_head *rhp)
Paul E. McKenneyd0ec7742009-10-06 21:48:16 -07003967{
Paul E. McKenney24ebbca2012-05-29 00:34:56 -07003968 struct rcu_data *rdp = container_of(rhp, struct rcu_data, barrier_head);
3969 struct rcu_state *rsp = rdp->rsp;
3970
Paul E. McKenneya83eff02012-05-23 18:47:05 -07003971 if (atomic_dec_and_test(&rsp->barrier_cpu_count)) {
Paul E. McKenney4f525a52015-06-26 11:20:00 -07003972 _rcu_barrier_trace(rsp, "LastCB", -1, rsp->barrier_sequence);
Paul E. McKenney7db74df2012-05-29 03:03:37 -07003973 complete(&rsp->barrier_completion);
Paul E. McKenneya83eff02012-05-23 18:47:05 -07003974 } else {
Paul E. McKenney4f525a52015-06-26 11:20:00 -07003975 _rcu_barrier_trace(rsp, "CB", -1, rsp->barrier_sequence);
Paul E. McKenneya83eff02012-05-23 18:47:05 -07003976 }
Paul E. McKenneyd0ec7742009-10-06 21:48:16 -07003977}
3978
3979/*
3980 * Called with preemption disabled, and from cross-cpu IRQ context.
3981 */
3982static void rcu_barrier_func(void *type)
3983{
Paul E. McKenney037b64e2012-05-28 23:26:01 -07003984 struct rcu_state *rsp = type;
Christoph Lameterfa07a582014-04-15 12:20:12 -07003985 struct rcu_data *rdp = raw_cpu_ptr(rsp->rda);
Paul E. McKenneyd0ec7742009-10-06 21:48:16 -07003986
Paul E. McKenney4f525a52015-06-26 11:20:00 -07003987 _rcu_barrier_trace(rsp, "IRQ", -1, rsp->barrier_sequence);
Paul E. McKenney24ebbca2012-05-29 00:34:56 -07003988 atomic_inc(&rsp->barrier_cpu_count);
Paul E. McKenney06668ef2012-05-28 23:57:46 -07003989 rsp->call(&rdp->barrier_head, rcu_barrier_callback);
Paul E. McKenneyd0ec7742009-10-06 21:48:16 -07003990}
3991
Paul E. McKenneyd0ec7742009-10-06 21:48:16 -07003992/*
3993 * Orchestrate the specified type of RCU barrier, waiting for all
3994 * RCU callbacks of the specified type to complete.
3995 */
Paul E. McKenney037b64e2012-05-28 23:26:01 -07003996static void _rcu_barrier(struct rcu_state *rsp)
Paul E. McKenneyd0ec7742009-10-06 21:48:16 -07003997{
Paul E. McKenneyb1420f12012-03-01 13:18:08 -08003998 int cpu;
Paul E. McKenneyb1420f12012-03-01 13:18:08 -08003999 struct rcu_data *rdp;
Paul E. McKenney4f525a52015-06-26 11:20:00 -07004000 unsigned long s = rcu_seq_snap(&rsp->barrier_sequence);
Paul E. McKenneyb1420f12012-03-01 13:18:08 -08004001
Paul E. McKenney4f525a52015-06-26 11:20:00 -07004002 _rcu_barrier_trace(rsp, "Begin", -1, s);
Paul E. McKenneyb1420f12012-03-01 13:18:08 -08004003
Paul E. McKenneye74f4c42009-10-06 21:48:17 -07004004 /* Take mutex to serialize concurrent rcu_barrier() requests. */
Paul E. McKenney7be7f0b2012-05-29 05:18:53 -07004005 mutex_lock(&rsp->barrier_mutex);
Paul E. McKenneyb1420f12012-03-01 13:18:08 -08004006
Paul E. McKenney4f525a52015-06-26 11:20:00 -07004007 /* Did someone else do our work for us? */
4008 if (rcu_seq_done(&rsp->barrier_sequence, s)) {
4009 _rcu_barrier_trace(rsp, "EarlyExit", -1, rsp->barrier_sequence);
Paul E. McKenneycf3a9c42012-05-29 14:56:46 -07004010 smp_mb(); /* caller's subsequent code after above check. */
4011 mutex_unlock(&rsp->barrier_mutex);
4012 return;
4013 }
4014
Paul E. McKenney4f525a52015-06-26 11:20:00 -07004015 /* Mark the start of the barrier operation. */
4016 rcu_seq_start(&rsp->barrier_sequence);
4017 _rcu_barrier_trace(rsp, "Inc1", -1, rsp->barrier_sequence);
Paul E. McKenneyb1420f12012-03-01 13:18:08 -08004018
Paul E. McKenneyd0ec7742009-10-06 21:48:16 -07004019 /*
Paul E. McKenneyb1420f12012-03-01 13:18:08 -08004020 * Initialize the count to one rather than to zero in order to
4021 * avoid a too-soon return to zero in case of a short grace period
Paul E. McKenney1331e7a2012-08-02 17:43:50 -07004022 * (or preemption of this task). Exclude CPU-hotplug operations
4023 * to ensure that no offline CPU has callbacks queued.
Paul E. McKenneyd0ec7742009-10-06 21:48:16 -07004024 */
Paul E. McKenney7db74df2012-05-29 03:03:37 -07004025 init_completion(&rsp->barrier_completion);
Paul E. McKenney24ebbca2012-05-29 00:34:56 -07004026 atomic_set(&rsp->barrier_cpu_count, 1);
Paul E. McKenney1331e7a2012-08-02 17:43:50 -07004027 get_online_cpus();
Paul E. McKenneyb1420f12012-03-01 13:18:08 -08004028
4029 /*
Paul E. McKenney1331e7a2012-08-02 17:43:50 -07004030 * Force each CPU with callbacks to register a new callback.
4031 * When that callback is invoked, we will know that all of the
4032 * corresponding CPU's preceding callbacks have been invoked.
Paul E. McKenneyb1420f12012-03-01 13:18:08 -08004033 */
Paul E. McKenney3fbfbf72012-08-19 21:35:53 -07004034 for_each_possible_cpu(cpu) {
Frederic Weisbeckerd1e43fa2013-03-26 23:47:24 +01004035 if (!cpu_online(cpu) && !rcu_is_nocb_cpu(cpu))
Paul E. McKenney3fbfbf72012-08-19 21:35:53 -07004036 continue;
Paul E. McKenneyb1420f12012-03-01 13:18:08 -08004037 rdp = per_cpu_ptr(rsp->rda, cpu);
Frederic Weisbeckerd1e43fa2013-03-26 23:47:24 +01004038 if (rcu_is_nocb_cpu(cpu)) {
Paul E. McKenneyd7e29932014-10-27 09:15:54 -07004039 if (!rcu_nocb_cpu_needs_barrier(rsp, cpu)) {
4040 _rcu_barrier_trace(rsp, "OfflineNoCB", cpu,
Paul E. McKenney4f525a52015-06-26 11:20:00 -07004041 rsp->barrier_sequence);
Paul E. McKenneyd7e29932014-10-27 09:15:54 -07004042 } else {
4043 _rcu_barrier_trace(rsp, "OnlineNoCB", cpu,
Paul E. McKenney4f525a52015-06-26 11:20:00 -07004044 rsp->barrier_sequence);
Paul E. McKenney41050a02014-12-18 12:31:27 -08004045 smp_mb__before_atomic();
Paul E. McKenneyd7e29932014-10-27 09:15:54 -07004046 atomic_inc(&rsp->barrier_cpu_count);
4047 __call_rcu(&rdp->barrier_head,
4048 rcu_barrier_callback, rsp, cpu, 0);
4049 }
Paul E. McKenney7d0ae802015-03-03 14:57:58 -08004050 } else if (READ_ONCE(rdp->qlen)) {
Paul E. McKenneya83eff02012-05-23 18:47:05 -07004051 _rcu_barrier_trace(rsp, "OnlineQ", cpu,
Paul E. McKenney4f525a52015-06-26 11:20:00 -07004052 rsp->barrier_sequence);
Paul E. McKenney037b64e2012-05-28 23:26:01 -07004053 smp_call_function_single(cpu, rcu_barrier_func, rsp, 1);
Paul E. McKenneyb1420f12012-03-01 13:18:08 -08004054 } else {
Paul E. McKenneya83eff02012-05-23 18:47:05 -07004055 _rcu_barrier_trace(rsp, "OnlineNQ", cpu,
Paul E. McKenney4f525a52015-06-26 11:20:00 -07004056 rsp->barrier_sequence);
Paul E. McKenneyb1420f12012-03-01 13:18:08 -08004057 }
4058 }
Paul E. McKenney1331e7a2012-08-02 17:43:50 -07004059 put_online_cpus();
Paul E. McKenneyb1420f12012-03-01 13:18:08 -08004060
4061 /*
4062 * Now that we have an rcu_barrier_callback() callback on each
4063 * CPU, and thus each counted, remove the initial count.
4064 */
Paul E. McKenney24ebbca2012-05-29 00:34:56 -07004065 if (atomic_dec_and_test(&rsp->barrier_cpu_count))
Paul E. McKenney7db74df2012-05-29 03:03:37 -07004066 complete(&rsp->barrier_completion);
Paul E. McKenneyb1420f12012-03-01 13:18:08 -08004067
4068 /* Wait for all rcu_barrier_callback() callbacks to be invoked. */
Paul E. McKenney7db74df2012-05-29 03:03:37 -07004069 wait_for_completion(&rsp->barrier_completion);
Paul E. McKenneyb1420f12012-03-01 13:18:08 -08004070
Paul E. McKenney4f525a52015-06-26 11:20:00 -07004071 /* Mark the end of the barrier operation. */
4072 _rcu_barrier_trace(rsp, "Inc2", -1, rsp->barrier_sequence);
4073 rcu_seq_end(&rsp->barrier_sequence);
4074
Paul E. McKenneyb1420f12012-03-01 13:18:08 -08004075 /* Other rcu_barrier() invocations can now safely proceed. */
Paul E. McKenney7be7f0b2012-05-29 05:18:53 -07004076 mutex_unlock(&rsp->barrier_mutex);
Paul E. McKenneyd0ec7742009-10-06 21:48:16 -07004077}
4078
4079/**
Paul E. McKenneyd0ec7742009-10-06 21:48:16 -07004080 * rcu_barrier_bh - Wait until all in-flight call_rcu_bh() callbacks complete.
4081 */
4082void rcu_barrier_bh(void)
4083{
Paul E. McKenney037b64e2012-05-28 23:26:01 -07004084 _rcu_barrier(&rcu_bh_state);
Paul E. McKenneyd0ec7742009-10-06 21:48:16 -07004085}
4086EXPORT_SYMBOL_GPL(rcu_barrier_bh);
4087
4088/**
4089 * rcu_barrier_sched - Wait for in-flight call_rcu_sched() callbacks.
4090 */
4091void rcu_barrier_sched(void)
4092{
Paul E. McKenney037b64e2012-05-28 23:26:01 -07004093 _rcu_barrier(&rcu_sched_state);
Paul E. McKenneyd0ec7742009-10-06 21:48:16 -07004094}
4095EXPORT_SYMBOL_GPL(rcu_barrier_sched);
4096
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01004097/*
Paul E. McKenney0aa04b02015-01-23 21:52:37 -08004098 * Propagate ->qsinitmask bits up the rcu_node tree to account for the
4099 * first CPU in a given leaf rcu_node structure coming online. The caller
4100 * must hold the corresponding leaf rcu_node ->lock with interrrupts
4101 * disabled.
4102 */
4103static void rcu_init_new_rnp(struct rcu_node *rnp_leaf)
4104{
4105 long mask;
4106 struct rcu_node *rnp = rnp_leaf;
4107
4108 for (;;) {
4109 mask = rnp->grpmask;
4110 rnp = rnp->parent;
4111 if (rnp == NULL)
4112 return;
Paul E. McKenney6cf10082015-10-08 15:36:54 -07004113 raw_spin_lock_rcu_node(rnp); /* Interrupts already disabled. */
Paul E. McKenney0aa04b02015-01-23 21:52:37 -08004114 rnp->qsmaskinit |= mask;
4115 raw_spin_unlock(&rnp->lock); /* Interrupts remain disabled. */
4116 }
4117}
4118
4119/*
Paul E. McKenney27569622009-08-15 09:53:46 -07004120 * Do boot-time initialization of a CPU's per-CPU RCU data.
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01004121 */
Paul E. McKenney27569622009-08-15 09:53:46 -07004122static void __init
4123rcu_boot_init_percpu_data(int cpu, struct rcu_state *rsp)
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01004124{
4125 unsigned long flags;
Lai Jiangshan394f99a2010-06-28 16:25:04 +08004126 struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
Paul E. McKenney27569622009-08-15 09:53:46 -07004127 struct rcu_node *rnp = rcu_get_root(rsp);
4128
4129 /* Set up local state, ensuring consistent view of global state. */
Paul E. McKenney6cf10082015-10-08 15:36:54 -07004130 raw_spin_lock_irqsave_rcu_node(rnp, flags);
Paul E. McKenney27569622009-08-15 09:53:46 -07004131 rdp->grpmask = 1UL << (cpu - rdp->mynode->grplo);
Paul E. McKenney27569622009-08-15 09:53:46 -07004132 rdp->dynticks = &per_cpu(rcu_dynticks, cpu);
Paul E. McKenney29e37d82011-11-17 16:55:56 -08004133 WARN_ON_ONCE(rdp->dynticks->dynticks_nesting != DYNTICK_TASK_EXIT_IDLE);
Paul E. McKenney9b2e4f12011-09-30 12:10:22 -07004134 WARN_ON_ONCE(atomic_read(&rdp->dynticks->dynticks) != 1);
Paul E. McKenney27569622009-08-15 09:53:46 -07004135 rdp->cpu = cpu;
Paul E. McKenneyd4c08f22011-06-25 06:36:56 -07004136 rdp->rsp = rsp;
Paul E. McKenney2cd6ffa2015-06-29 17:06:39 -07004137 mutex_init(&rdp->exp_funnel_mutex);
Paul E. McKenney3fbfbf72012-08-19 21:35:53 -07004138 rcu_boot_init_nocb_percpu_data(rdp);
Paul E. McKenney1304afb2010-02-22 17:05:02 -08004139 raw_spin_unlock_irqrestore(&rnp->lock, flags);
Paul E. McKenney27569622009-08-15 09:53:46 -07004140}
4141
4142/*
4143 * Initialize a CPU's per-CPU RCU data. Note that only one online or
4144 * offline event can be happening at a given time. Note also that we
4145 * can accept some slop in the rsp->completed access due to the fact
4146 * that this CPU cannot possibly have any RCU callbacks in flight yet.
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01004147 */
Paul Gortmaker49fb4c62013-06-19 14:52:21 -04004148static void
Iulia Manda9b671222014-03-11 13:18:22 +02004149rcu_init_percpu_data(int cpu, struct rcu_state *rsp)
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01004150{
4151 unsigned long flags;
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01004152 unsigned long mask;
Lai Jiangshan394f99a2010-06-28 16:25:04 +08004153 struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01004154 struct rcu_node *rnp = rcu_get_root(rsp);
4155
4156 /* Set up local state, ensuring consistent view of global state. */
Paul E. McKenney6cf10082015-10-08 15:36:54 -07004157 raw_spin_lock_irqsave_rcu_node(rnp, flags);
Paul E. McKenney37c72e52009-10-14 10:15:55 -07004158 rdp->qlen_last_fqs_check = 0;
4159 rdp->n_force_qs_snap = rsp->n_force_qs;
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01004160 rdp->blimit = blimit;
Paul E. McKenney39c8d312015-01-20 23:42:38 -08004161 if (!rdp->nxtlist)
4162 init_callback_list(rdp); /* Re-enable callbacks on this CPU. */
Paul E. McKenney29e37d82011-11-17 16:55:56 -08004163 rdp->dynticks->dynticks_nesting = DYNTICK_TASK_EXIT_IDLE;
Paul E. McKenney23332102013-06-21 12:34:33 -07004164 rcu_sysidle_init_percpu_data(rdp->dynticks);
Paul E. McKenneyc92b1312011-11-23 13:38:58 -08004165 atomic_set(&rdp->dynticks->dynticks,
4166 (atomic_read(&rdp->dynticks->dynticks) & ~0x1) + 1);
Paul E. McKenney1304afb2010-02-22 17:05:02 -08004167 raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01004168
Paul E. McKenney0aa04b02015-01-23 21:52:37 -08004169 /*
4170 * Add CPU to leaf rcu_node pending-online bitmask. Any needed
4171 * propagation up the rcu_node tree will happen at the beginning
4172 * of the next grace period.
4173 */
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01004174 rnp = rdp->mynode;
4175 mask = rdp->grpmask;
Peter Zijlstra2a67e742015-10-08 12:24:23 +02004176 raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */
Paul E. McKenney0aa04b02015-01-23 21:52:37 -08004177 rnp->qsmaskinitnext |= mask;
Paul E. McKenneyb9585e92015-07-31 16:04:45 -07004178 rnp->expmaskinitnext |= mask;
4179 if (!rdp->beenonline)
4180 WRITE_ONCE(rsp->ncpus, READ_ONCE(rsp->ncpus) + 1);
4181 rdp->beenonline = true; /* We have now been online. */
Paul E. McKenney0aa04b02015-01-23 21:52:37 -08004182 rdp->gpnum = rnp->completed; /* Make CPU later note any new GP. */
4183 rdp->completed = rnp->completed;
Paul E. McKenney5b74c452015-08-06 15:16:57 -07004184 rdp->cpu_no_qs.b.norm = true;
Paul E. McKenneya738eec2015-03-10 14:53:29 -07004185 rdp->rcu_qs_ctr_snap = per_cpu(rcu_qs_ctr, cpu);
Paul E. McKenney97c668b2015-08-06 11:31:51 -07004186 rdp->core_needs_qs = false;
Paul E. McKenney0aa04b02015-01-23 21:52:37 -08004187 trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpuonl"));
4188 raw_spin_unlock_irqrestore(&rnp->lock, flags);
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01004189}
4190
Paul Gortmaker49fb4c62013-06-19 14:52:21 -04004191static void rcu_prepare_cpu(int cpu)
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01004192{
Paul E. McKenney6ce75a22012-06-12 11:01:13 -07004193 struct rcu_state *rsp;
4194
4195 for_each_rcu_flavor(rsp)
Iulia Manda9b671222014-03-11 13:18:22 +02004196 rcu_init_percpu_data(cpu, rsp);
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01004197}
4198
4199/*
Paul E. McKenneyf41d9112009-08-22 13:56:52 -07004200 * Handle CPU online/offline notification events.
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01004201 */
Paul E. McKenney88428cc52015-01-28 14:42:09 -08004202int rcu_cpu_notify(struct notifier_block *self,
4203 unsigned long action, void *hcpu)
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01004204{
4205 long cpu = (long)hcpu;
Uma Sharmae5341652014-03-23 22:32:09 -07004206 struct rcu_data *rdp = per_cpu_ptr(rcu_state_p->rda, cpu);
Paul E. McKenneya26ac242011-01-12 14:10:23 -08004207 struct rcu_node *rnp = rdp->mynode;
Paul E. McKenney6ce75a22012-06-12 11:01:13 -07004208 struct rcu_state *rsp;
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01004209
4210 switch (action) {
4211 case CPU_UP_PREPARE:
4212 case CPU_UP_PREPARE_FROZEN:
Peter Zijlstrad72bce02011-05-30 13:34:51 +02004213 rcu_prepare_cpu(cpu);
4214 rcu_prepare_kthreads(cpu);
Paul E. McKenney35ce7f22014-07-11 11:30:24 -07004215 rcu_spawn_all_nocb_kthreads(cpu);
Paul E. McKenneya26ac242011-01-12 14:10:23 -08004216 break;
4217 case CPU_ONLINE:
Paul E. McKenney0f962a52011-04-14 12:13:53 -07004218 case CPU_DOWN_FAILED:
Paul E. McKenney338b0f72015-09-03 00:45:02 -07004219 sync_sched_exp_online_cleanup(cpu);
Thomas Gleixner5d01bbd2012-07-16 10:42:35 +00004220 rcu_boost_kthread_setaffinity(rnp, -1);
Paul E. McKenney0f962a52011-04-14 12:13:53 -07004221 break;
4222 case CPU_DOWN_PREPARE:
Paul E. McKenney34ed62462013-01-07 13:37:42 -08004223 rcu_boost_kthread_setaffinity(rnp, cpu);
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01004224 break;
Paul E. McKenneyd0ec7742009-10-06 21:48:16 -07004225 case CPU_DYING:
4226 case CPU_DYING_FROZEN:
Paul E. McKenney6ce75a22012-06-12 11:01:13 -07004227 for_each_rcu_flavor(rsp)
4228 rcu_cleanup_dying_cpu(rsp);
Paul E. McKenneyd0ec7742009-10-06 21:48:16 -07004229 break;
Paul E. McKenney88428cc52015-01-28 14:42:09 -08004230 case CPU_DYING_IDLE:
Paul E. McKenney6587a232015-08-06 16:50:39 -07004231 /* QS for any half-done expedited RCU-sched GP. */
Paul E. McKenney338b0f72015-09-03 00:45:02 -07004232 preempt_disable();
4233 rcu_report_exp_rdp(&rcu_sched_state,
4234 this_cpu_ptr(rcu_sched_state.rda), true);
4235 preempt_enable();
Paul E. McKenney6587a232015-08-06 16:50:39 -07004236
Paul E. McKenney88428cc52015-01-28 14:42:09 -08004237 for_each_rcu_flavor(rsp) {
4238 rcu_cleanup_dying_idle_cpu(cpu, rsp);
4239 }
4240 break;
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01004241 case CPU_DEAD:
4242 case CPU_DEAD_FROZEN:
4243 case CPU_UP_CANCELED:
4244 case CPU_UP_CANCELED_FROZEN:
Paul E. McKenney776d6802014-10-23 10:50:41 -07004245 for_each_rcu_flavor(rsp) {
Paul E. McKenney6ce75a22012-06-12 11:01:13 -07004246 rcu_cleanup_dead_cpu(cpu, rsp);
Paul E. McKenney776d6802014-10-23 10:50:41 -07004247 do_nocb_deferred_wakeup(per_cpu_ptr(rsp->rda, cpu));
4248 }
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01004249 break;
4250 default:
4251 break;
4252 }
Paul E. McKenney34ed62462013-01-07 13:37:42 -08004253 return NOTIFY_OK;
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01004254}
4255
Borislav Petkovd1d74d12013-04-22 00:12:42 +02004256static int rcu_pm_notify(struct notifier_block *self,
4257 unsigned long action, void *hcpu)
4258{
4259 switch (action) {
4260 case PM_HIBERNATION_PREPARE:
4261 case PM_SUSPEND_PREPARE:
4262 if (nr_cpu_ids <= 256) /* Expediting bad for large systems. */
Paul E. McKenney5afff482015-02-18 16:39:09 -08004263 rcu_expedite_gp();
Borislav Petkovd1d74d12013-04-22 00:12:42 +02004264 break;
4265 case PM_POST_HIBERNATION:
4266 case PM_POST_SUSPEND:
Paul E. McKenney5afff482015-02-18 16:39:09 -08004267 if (nr_cpu_ids <= 256) /* Expediting bad for large systems. */
4268 rcu_unexpedite_gp();
Borislav Petkovd1d74d12013-04-22 00:12:42 +02004269 break;
4270 default:
4271 break;
4272 }
4273 return NOTIFY_OK;
4274}
4275
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01004276/*
Paul E. McKenney9386c0b2014-07-13 12:00:53 -07004277 * Spawn the kthreads that handle each RCU flavor's grace periods.
Paul E. McKenneyb3dbec72012-06-18 18:36:08 -07004278 */
4279static int __init rcu_spawn_gp_kthread(void)
4280{
4281 unsigned long flags;
Paul E. McKenneya94844b2014-12-12 07:37:48 -08004282 int kthread_prio_in = kthread_prio;
Paul E. McKenneyb3dbec72012-06-18 18:36:08 -07004283 struct rcu_node *rnp;
4284 struct rcu_state *rsp;
Paul E. McKenneya94844b2014-12-12 07:37:48 -08004285 struct sched_param sp;
Paul E. McKenneyb3dbec72012-06-18 18:36:08 -07004286 struct task_struct *t;
4287
Paul E. McKenneya94844b2014-12-12 07:37:48 -08004288 /* Force priority into range. */
4289 if (IS_ENABLED(CONFIG_RCU_BOOST) && kthread_prio < 1)
4290 kthread_prio = 1;
4291 else if (kthread_prio < 0)
4292 kthread_prio = 0;
4293 else if (kthread_prio > 99)
4294 kthread_prio = 99;
4295 if (kthread_prio != kthread_prio_in)
4296 pr_alert("rcu_spawn_gp_kthread(): Limited prio to %d from %d\n",
4297 kthread_prio, kthread_prio_in);
4298
Paul E. McKenney9386c0b2014-07-13 12:00:53 -07004299 rcu_scheduler_fully_active = 1;
Paul E. McKenneyb3dbec72012-06-18 18:36:08 -07004300 for_each_rcu_flavor(rsp) {
Paul E. McKenneya94844b2014-12-12 07:37:48 -08004301 t = kthread_create(rcu_gp_kthread, rsp, "%s", rsp->name);
Paul E. McKenneyb3dbec72012-06-18 18:36:08 -07004302 BUG_ON(IS_ERR(t));
4303 rnp = rcu_get_root(rsp);
Paul E. McKenney6cf10082015-10-08 15:36:54 -07004304 raw_spin_lock_irqsave_rcu_node(rnp, flags);
Paul E. McKenneyb3dbec72012-06-18 18:36:08 -07004305 rsp->gp_kthread = t;
Paul E. McKenneya94844b2014-12-12 07:37:48 -08004306 if (kthread_prio) {
4307 sp.sched_priority = kthread_prio;
4308 sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
4309 }
4310 wake_up_process(t);
Paul E. McKenneyb3dbec72012-06-18 18:36:08 -07004311 raw_spin_unlock_irqrestore(&rnp->lock, flags);
4312 }
Paul E. McKenney35ce7f22014-07-11 11:30:24 -07004313 rcu_spawn_nocb_kthreads();
Paul E. McKenney9386c0b2014-07-13 12:00:53 -07004314 rcu_spawn_boost_kthreads();
Paul E. McKenneyb3dbec72012-06-18 18:36:08 -07004315 return 0;
4316}
4317early_initcall(rcu_spawn_gp_kthread);
4318
4319/*
Paul E. McKenneybbad9372010-04-02 16:17:17 -07004320 * This function is invoked towards the end of the scheduler's initialization
4321 * process. Before this is called, the idle task might contain
4322 * RCU read-side critical sections (during which time, this idle
4323 * task is booting the system). After this function is called, the
4324 * idle tasks are prohibited from containing RCU read-side critical
4325 * sections. This function also enables RCU lockdep checking.
4326 */
4327void rcu_scheduler_starting(void)
4328{
4329 WARN_ON(num_online_cpus() != 1);
4330 WARN_ON(nr_context_switches() > 0);
4331 rcu_scheduler_active = 1;
4332}
4333
4334/*
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01004335 * Compute the per-level fanout, either using the exact fanout specified
Paul E. McKenney7fa27002015-04-20 10:27:15 -07004336 * or balancing the tree, depending on the rcu_fanout_exact boot parameter.
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01004337 */
Alexander Gordeev199977b2015-06-03 08:18:29 +02004338static void __init rcu_init_levelspread(int *levelspread, const int *levelcnt)
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01004339{
4340 int i;
4341
Paul E. McKenney7fa27002015-04-20 10:27:15 -07004342 if (rcu_fanout_exact) {
Alexander Gordeev199977b2015-06-03 08:18:29 +02004343 levelspread[rcu_num_lvls - 1] = rcu_fanout_leaf;
Paul E. McKenney66292402015-01-19 19:16:38 -08004344 for (i = rcu_num_lvls - 2; i >= 0; i--)
Alexander Gordeev199977b2015-06-03 08:18:29 +02004345 levelspread[i] = RCU_FANOUT;
Paul E. McKenney66292402015-01-19 19:16:38 -08004346 } else {
4347 int ccur;
4348 int cprv;
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01004349
Paul E. McKenney66292402015-01-19 19:16:38 -08004350 cprv = nr_cpu_ids;
4351 for (i = rcu_num_lvls - 1; i >= 0; i--) {
Alexander Gordeev199977b2015-06-03 08:18:29 +02004352 ccur = levelcnt[i];
4353 levelspread[i] = (cprv + ccur - 1) / ccur;
Paul E. McKenney66292402015-01-19 19:16:38 -08004354 cprv = ccur;
4355 }
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01004356 }
4357}
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01004358
4359/*
4360 * Helper function for rcu_init() that initializes one rcu_state structure.
4361 */
Lai Jiangshan394f99a2010-06-28 16:25:04 +08004362static void __init rcu_init_one(struct rcu_state *rsp,
4363 struct rcu_data __percpu *rda)
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01004364{
Alexander Gordeevcb007102015-06-03 08:18:30 +02004365 static const char * const buf[] = RCU_NODE_NAME_INIT;
4366 static const char * const fqs[] = RCU_FQS_NAME_INIT;
Paul E. McKenney385b73c2015-06-24 14:20:08 -07004367 static const char * const exp[] = RCU_EXP_NAME_INIT;
Paul E. McKenney4a81e832014-06-20 16:49:01 -07004368 static u8 fl_mask = 0x1;
Alexander Gordeev199977b2015-06-03 08:18:29 +02004369
4370 int levelcnt[RCU_NUM_LVLS]; /* # nodes in each level. */
4371 int levelspread[RCU_NUM_LVLS]; /* kids/node in each level. */
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01004372 int cpustride = 1;
4373 int i;
4374 int j;
4375 struct rcu_node *rnp;
4376
Alexander Gordeev05b84ae2015-06-03 08:18:28 +02004377 BUILD_BUG_ON(RCU_NUM_LVLS > ARRAY_SIZE(buf)); /* Fix buf[] init! */
Paul E. McKenneyb6407e82010-01-04 16:04:02 -08004378
Paul E. McKenney3eaaaf6c2015-03-09 16:51:17 -07004379 /* Silence gcc 4.8 false positive about array index out of range. */
4380 if (rcu_num_lvls <= 0 || rcu_num_lvls > RCU_NUM_LVLS)
4381 panic("rcu_init_one: rcu_num_lvls out of range");
Paul E. McKenney49305212012-11-29 13:49:00 -08004382
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01004383 /* Initialize the level-tracking arrays. */
4384
Paul E. McKenneyf885b7f2012-04-23 15:52:53 -07004385 for (i = 0; i < rcu_num_lvls; i++)
Alexander Gordeev199977b2015-06-03 08:18:29 +02004386 levelcnt[i] = num_rcu_lvl[i];
Paul E. McKenneyf885b7f2012-04-23 15:52:53 -07004387 for (i = 1; i < rcu_num_lvls; i++)
Alexander Gordeev199977b2015-06-03 08:18:29 +02004388 rsp->level[i] = rsp->level[i - 1] + levelcnt[i - 1];
4389 rcu_init_levelspread(levelspread, levelcnt);
Paul E. McKenney4a81e832014-06-20 16:49:01 -07004390 rsp->flavor_mask = fl_mask;
4391 fl_mask <<= 1;
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01004392
4393 /* Initialize the elements themselves, starting from the leaves. */
4394
Paul E. McKenneyf885b7f2012-04-23 15:52:53 -07004395 for (i = rcu_num_lvls - 1; i >= 0; i--) {
Alexander Gordeev199977b2015-06-03 08:18:29 +02004396 cpustride *= levelspread[i];
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01004397 rnp = rsp->level[i];
Alexander Gordeev199977b2015-06-03 08:18:29 +02004398 for (j = 0; j < levelcnt[i]; j++, rnp++) {
Paul E. McKenney1304afb2010-02-22 17:05:02 -08004399 raw_spin_lock_init(&rnp->lock);
Paul E. McKenneyb6407e82010-01-04 16:04:02 -08004400 lockdep_set_class_and_name(&rnp->lock,
4401 &rcu_node_class[i], buf[i]);
Paul E. McKenney394f2762012-06-26 17:00:35 -07004402 raw_spin_lock_init(&rnp->fqslock);
4403 lockdep_set_class_and_name(&rnp->fqslock,
4404 &rcu_fqs_class[i], fqs[i]);
Paul E. McKenney25d30cf2012-07-11 05:23:18 -07004405 rnp->gpnum = rsp->gpnum;
4406 rnp->completed = rsp->completed;
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01004407 rnp->qsmask = 0;
4408 rnp->qsmaskinit = 0;
4409 rnp->grplo = j * cpustride;
4410 rnp->grphi = (j + 1) * cpustride - 1;
Himangi Saraogi595f3902014-03-18 22:52:26 +05304411 if (rnp->grphi >= nr_cpu_ids)
4412 rnp->grphi = nr_cpu_ids - 1;
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01004413 if (i == 0) {
4414 rnp->grpnum = 0;
4415 rnp->grpmask = 0;
4416 rnp->parent = NULL;
4417 } else {
Alexander Gordeev199977b2015-06-03 08:18:29 +02004418 rnp->grpnum = j % levelspread[i - 1];
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01004419 rnp->grpmask = 1UL << rnp->grpnum;
4420 rnp->parent = rsp->level[i - 1] +
Alexander Gordeev199977b2015-06-03 08:18:29 +02004421 j / levelspread[i - 1];
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01004422 }
4423 rnp->level = i;
Paul E. McKenney12f5f522010-11-29 21:56:39 -08004424 INIT_LIST_HEAD(&rnp->blkd_tasks);
Paul E. McKenneydae6e642013-02-10 20:48:58 -08004425 rcu_init_one_nocb(rnp);
Paul E. McKenney385b73c2015-06-24 14:20:08 -07004426 mutex_init(&rnp->exp_funnel_mutex);
Paul E. McKenney83c2c732015-08-06 20:43:02 -07004427 lockdep_set_class_and_name(&rnp->exp_funnel_mutex,
4428 &rcu_exp_class[i], exp[i]);
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01004429 }
4430 }
Lai Jiangshan0c340292010-03-28 11:12:30 +08004431
Paul E. McKenneyb3dbec72012-06-18 18:36:08 -07004432 init_waitqueue_head(&rsp->gp_wq);
Paul E. McKenneyf4ecea32015-07-29 17:28:11 -07004433 init_waitqueue_head(&rsp->expedited_wq);
Paul E. McKenneyf885b7f2012-04-23 15:52:53 -07004434 rnp = rsp->level[rcu_num_lvls - 1];
Lai Jiangshan0c340292010-03-28 11:12:30 +08004435 for_each_possible_cpu(i) {
Paul E. McKenney4a90a062010-04-14 16:48:11 -07004436 while (i > rnp->grphi)
Lai Jiangshan0c340292010-03-28 11:12:30 +08004437 rnp++;
Lai Jiangshan394f99a2010-06-28 16:25:04 +08004438 per_cpu_ptr(rsp->rda, i)->mynode = rnp;
Lai Jiangshan0c340292010-03-28 11:12:30 +08004439 rcu_boot_init_percpu_data(i, rsp);
4440 }
Paul E. McKenney6ce75a22012-06-12 11:01:13 -07004441 list_add(&rsp->flavors, &rcu_struct_flavors);
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01004442}
4443
Paul E. McKenneyf885b7f2012-04-23 15:52:53 -07004444/*
4445 * Compute the rcu_node tree geometry from kernel parameters. This cannot
Paul E. McKenney4102ada2013-10-08 20:23:47 -07004446 * replace the definitions in tree.h because those are needed to size
Paul E. McKenneyf885b7f2012-04-23 15:52:53 -07004447 * the ->node array in the rcu_state structure.
4448 */
4449static void __init rcu_init_geometry(void)
4450{
Paul E. McKenney026ad282013-04-03 22:14:11 -07004451 ulong d;
Paul E. McKenneyf885b7f2012-04-23 15:52:53 -07004452 int i;
Alexander Gordeev05b84ae2015-06-03 08:18:28 +02004453 int rcu_capacity[RCU_NUM_LVLS];
Paul E. McKenneyf885b7f2012-04-23 15:52:53 -07004454
Paul E. McKenney026ad282013-04-03 22:14:11 -07004455 /*
4456 * Initialize any unspecified boot parameters.
4457 * The default values of jiffies_till_first_fqs and
4458 * jiffies_till_next_fqs are set to the RCU_JIFFIES_TILL_FORCE_QS
4459 * value, which is a function of HZ, then adding one for each
4460 * RCU_JIFFIES_FQS_DIV CPUs that might be on the system.
4461 */
4462 d = RCU_JIFFIES_TILL_FORCE_QS + nr_cpu_ids / RCU_JIFFIES_FQS_DIV;
4463 if (jiffies_till_first_fqs == ULONG_MAX)
4464 jiffies_till_first_fqs = d;
4465 if (jiffies_till_next_fqs == ULONG_MAX)
4466 jiffies_till_next_fqs = d;
4467
Paul E. McKenneyf885b7f2012-04-23 15:52:53 -07004468 /* If the compile-time values are accurate, just leave. */
Paul E. McKenney47d631a2015-04-21 09:12:13 -07004469 if (rcu_fanout_leaf == RCU_FANOUT_LEAF &&
Paul E. McKenneyb17c7032012-09-06 15:38:02 -07004470 nr_cpu_ids == NR_CPUS)
Paul E. McKenneyf885b7f2012-04-23 15:52:53 -07004471 return;
Paul E. McKenney39479092013-10-09 15:20:33 -07004472 pr_info("RCU: Adjusting geometry for rcu_fanout_leaf=%d, nr_cpu_ids=%d\n",
4473 rcu_fanout_leaf, nr_cpu_ids);
Paul E. McKenneyf885b7f2012-04-23 15:52:53 -07004474
4475 /*
Paul E. McKenneyee968ac2015-07-31 08:28:35 -07004476 * The boot-time rcu_fanout_leaf parameter must be at least two
4477 * and cannot exceed the number of bits in the rcu_node masks.
4478 * Complain and fall back to the compile-time values if this
4479 * limit is exceeded.
Paul E. McKenneyf885b7f2012-04-23 15:52:53 -07004480 */
Paul E. McKenneyee968ac2015-07-31 08:28:35 -07004481 if (rcu_fanout_leaf < 2 ||
Alexander Gordeev75cf15a2015-06-03 08:18:23 +02004482 rcu_fanout_leaf > sizeof(unsigned long) * 8) {
Paul E. McKenney13bd64942015-06-04 10:06:01 -07004483 rcu_fanout_leaf = RCU_FANOUT_LEAF;
Paul E. McKenneyf885b7f2012-04-23 15:52:53 -07004484 WARN_ON(1);
4485 return;
4486 }
4487
Alexander Gordeev75cf15a2015-06-03 08:18:23 +02004488 /*
Paul E. McKenneyf885b7f2012-04-23 15:52:53 -07004489 * Compute number of nodes that can be handled an rcu_node tree
Alexander Gordeev96181382015-06-03 08:18:26 +02004490 * with the given number of levels.
Paul E. McKenneyf885b7f2012-04-23 15:52:53 -07004491 */
Alexander Gordeev96181382015-06-03 08:18:26 +02004492 rcu_capacity[0] = rcu_fanout_leaf;
Alexander Gordeev05b84ae2015-06-03 08:18:28 +02004493 for (i = 1; i < RCU_NUM_LVLS; i++)
Paul E. McKenneyf885b7f2012-04-23 15:52:53 -07004494 rcu_capacity[i] = rcu_capacity[i - 1] * RCU_FANOUT;
4495
4496 /*
Alexander Gordeev75cf15a2015-06-03 08:18:23 +02004497 * The tree must be able to accommodate the configured number of CPUs.
Paul E. McKenneyee968ac2015-07-31 08:28:35 -07004498 * If this limit is exceeded, fall back to the compile-time values.
Paul E. McKenneyf885b7f2012-04-23 15:52:53 -07004499 */
Paul E. McKenneyee968ac2015-07-31 08:28:35 -07004500 if (nr_cpu_ids > rcu_capacity[RCU_NUM_LVLS - 1]) {
4501 rcu_fanout_leaf = RCU_FANOUT_LEAF;
4502 WARN_ON(1);
4503 return;
4504 }
Paul E. McKenneyf885b7f2012-04-23 15:52:53 -07004505
Alexander Gordeev679f9852015-06-03 08:18:25 +02004506 /* Calculate the number of levels in the tree. */
Alexander Gordeev96181382015-06-03 08:18:26 +02004507 for (i = 0; nr_cpu_ids > rcu_capacity[i]; i++) {
Alexander Gordeev679f9852015-06-03 08:18:25 +02004508 }
Alexander Gordeev96181382015-06-03 08:18:26 +02004509 rcu_num_lvls = i + 1;
Alexander Gordeev679f9852015-06-03 08:18:25 +02004510
Paul E. McKenneyf885b7f2012-04-23 15:52:53 -07004511 /* Calculate the number of rcu_nodes at each level of the tree. */
Alexander Gordeev679f9852015-06-03 08:18:25 +02004512 for (i = 0; i < rcu_num_lvls; i++) {
Alexander Gordeev96181382015-06-03 08:18:26 +02004513 int cap = rcu_capacity[(rcu_num_lvls - 1) - i];
Alexander Gordeev679f9852015-06-03 08:18:25 +02004514 num_rcu_lvl[i] = DIV_ROUND_UP(nr_cpu_ids, cap);
4515 }
Paul E. McKenneyf885b7f2012-04-23 15:52:53 -07004516
4517 /* Calculate the total number of rcu_node structures. */
4518 rcu_num_nodes = 0;
Alexander Gordeev679f9852015-06-03 08:18:25 +02004519 for (i = 0; i < rcu_num_lvls; i++)
Paul E. McKenneyf885b7f2012-04-23 15:52:53 -07004520 rcu_num_nodes += num_rcu_lvl[i];
Paul E. McKenneyf885b7f2012-04-23 15:52:53 -07004521}
4522
Paul E. McKenneya3dc2942015-04-20 11:40:50 -07004523/*
4524 * Dump out the structure of the rcu_node combining tree associated
4525 * with the rcu_state structure referenced by rsp.
4526 */
4527static void __init rcu_dump_rcu_node_tree(struct rcu_state *rsp)
4528{
4529 int level = 0;
4530 struct rcu_node *rnp;
4531
4532 pr_info("rcu_node tree layout dump\n");
4533 pr_info(" ");
4534 rcu_for_each_node_breadth_first(rsp, rnp) {
4535 if (rnp->level != level) {
4536 pr_cont("\n");
4537 pr_info(" ");
4538 level = rnp->level;
4539 }
4540 pr_cont("%d:%d ^%d ", rnp->grplo, rnp->grphi, rnp->grpnum);
4541 }
4542 pr_cont("\n");
4543}
4544
Paul E. McKenney9f680ab2009-11-22 08:53:49 -08004545void __init rcu_init(void)
Paul E. McKenneyf41d9112009-08-22 13:56:52 -07004546{
Paul E. McKenney017c4262010-01-14 16:10:58 -08004547 int cpu;
Paul E. McKenney9f680ab2009-11-22 08:53:49 -08004548
Paul E. McKenney47627672015-01-19 21:10:21 -08004549 rcu_early_boot_tests();
4550
Paul E. McKenneyf41d9112009-08-22 13:56:52 -07004551 rcu_bootup_announce();
Paul E. McKenneyf885b7f2012-04-23 15:52:53 -07004552 rcu_init_geometry();
Lai Jiangshan394f99a2010-06-28 16:25:04 +08004553 rcu_init_one(&rcu_bh_state, &rcu_bh_data);
Paul E. McKenney69c8d282013-09-03 09:52:20 -07004554 rcu_init_one(&rcu_sched_state, &rcu_sched_data);
Paul E. McKenneya3dc2942015-04-20 11:40:50 -07004555 if (dump_tree)
4556 rcu_dump_rcu_node_tree(&rcu_sched_state);
Paul E. McKenneyf41d9112009-08-22 13:56:52 -07004557 __rcu_init_preempt();
Jiang Fangb5b39362013-02-02 14:13:42 -08004558 open_softirq(RCU_SOFTIRQ, rcu_process_callbacks);
Paul E. McKenney9f680ab2009-11-22 08:53:49 -08004559
4560 /*
4561 * We don't need protection against CPU-hotplug here because
4562 * this is called early in boot, before either interrupts
4563 * or the scheduler are operational.
4564 */
4565 cpu_notifier(rcu_cpu_notify, 0);
Borislav Petkovd1d74d12013-04-22 00:12:42 +02004566 pm_notifier(rcu_pm_notify, 0);
Paul E. McKenney017c4262010-01-14 16:10:58 -08004567 for_each_online_cpu(cpu)
4568 rcu_cpu_notify(NULL, CPU_UP_PREPARE, (void *)(long)cpu);
Paul E. McKenney64db4cf2008-12-18 21:55:32 +01004569}
4570
Paul E. McKenney4102ada2013-10-08 20:23:47 -07004571#include "tree_plugin.h"