Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 1 | /* |
| 2 | * Read-Copy Update mechanism for mutual exclusion |
| 3 | * |
| 4 | * This program is free software; you can redistribute it and/or modify |
| 5 | * it under the terms of the GNU General Public License as published by |
| 6 | * the Free Software Foundation; either version 2 of the License, or |
| 7 | * (at your option) any later version. |
| 8 | * |
| 9 | * This program is distributed in the hope that it will be useful, |
| 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 12 | * GNU General Public License for more details. |
| 13 | * |
| 14 | * You should have received a copy of the GNU General Public License |
Paul E. McKenney | 87de1cf | 2013-12-03 10:02:52 -0800 | [diff] [blame] | 15 | * along with this program; if not, you can access it online at |
| 16 | * http://www.gnu.org/licenses/gpl-2.0.html. |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 17 | * |
| 18 | * Copyright IBM Corporation, 2008 |
| 19 | * |
| 20 | * Authors: Dipankar Sarma <dipankar@in.ibm.com> |
| 21 | * Manfred Spraul <manfred@colorfullife.com> |
| 22 | * Paul E. McKenney <paulmck@linux.vnet.ibm.com> Hierarchical version |
| 23 | * |
| 24 | * Based on the original work by Paul McKenney <paulmck@us.ibm.com> |
| 25 | * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen. |
| 26 | * |
| 27 | * For detailed explanation of Read-Copy Update mechanism see - |
Paul E. McKenney | a71fca5 | 2009-09-18 10:28:19 -0700 | [diff] [blame] | 28 | * Documentation/RCU |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 29 | */ |
| 30 | #include <linux/types.h> |
| 31 | #include <linux/kernel.h> |
| 32 | #include <linux/init.h> |
| 33 | #include <linux/spinlock.h> |
| 34 | #include <linux/smp.h> |
Ingo Molnar | f9411eb | 2017-02-06 09:50:49 +0100 | [diff] [blame] | 35 | #include <linux/rcupdate_wait.h> |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 36 | #include <linux/interrupt.h> |
| 37 | #include <linux/sched.h> |
Ingo Molnar | b17b015 | 2017-02-08 18:51:35 +0100 | [diff] [blame] | 38 | #include <linux/sched/debug.h> |
Ingo Molnar | c1dc0b9 | 2009-08-02 11:28:21 +0200 | [diff] [blame] | 39 | #include <linux/nmi.h> |
Paul E. McKenney | 8826f3b | 2011-05-11 05:41:41 -0700 | [diff] [blame] | 40 | #include <linux/atomic.h> |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 41 | #include <linux/bitops.h> |
Paul Gortmaker | 9984de1 | 2011-05-23 14:51:41 -0400 | [diff] [blame] | 42 | #include <linux/export.h> |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 43 | #include <linux/completion.h> |
| 44 | #include <linux/moduleparam.h> |
| 45 | #include <linux/percpu.h> |
| 46 | #include <linux/notifier.h> |
| 47 | #include <linux/cpu.h> |
| 48 | #include <linux/mutex.h> |
| 49 | #include <linux/time.h> |
Paul E. McKenney | bbad937 | 2010-04-02 16:17:17 -0700 | [diff] [blame] | 50 | #include <linux/kernel_stat.h> |
Paul E. McKenney | a26ac24 | 2011-01-12 14:10:23 -0800 | [diff] [blame] | 51 | #include <linux/wait.h> |
| 52 | #include <linux/kthread.h> |
Ingo Molnar | ae7e81c | 2017-02-01 18:07:51 +0100 | [diff] [blame] | 53 | #include <uapi/linux/sched/types.h> |
Linus Torvalds | 268bb0c | 2011-05-20 12:50:29 -0700 | [diff] [blame] | 54 | #include <linux/prefetch.h> |
Paul E. McKenney | 3d3b7db | 2012-01-23 17:05:46 -0800 | [diff] [blame] | 55 | #include <linux/delay.h> |
| 56 | #include <linux/stop_machine.h> |
Paul E. McKenney | 661a85d | 2012-07-07 05:57:03 -0700 | [diff] [blame] | 57 | #include <linux/random.h> |
Steven Rostedt (Red Hat) | af658dc | 2015-04-29 14:36:05 -0400 | [diff] [blame] | 58 | #include <linux/trace_events.h> |
Borislav Petkov | d1d74d1 | 2013-04-22 00:12:42 +0200 | [diff] [blame] | 59 | #include <linux/suspend.h> |
Paul E. McKenney | a278d47 | 2017-04-05 09:05:18 -0700 | [diff] [blame] | 60 | #include <linux/ftrace.h> |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 61 | |
Paul E. McKenney | 4102ada | 2013-10-08 20:23:47 -0700 | [diff] [blame] | 62 | #include "tree.h" |
Paul E. McKenney | 29c00b4 | 2011-06-17 15:53:19 -0700 | [diff] [blame] | 63 | #include "rcu.h" |
Paul E. McKenney | 9f77da9 | 2009-08-22 13:56:45 -0700 | [diff] [blame] | 64 | |
Paul E. McKenney | 4102ada | 2013-10-08 20:23:47 -0700 | [diff] [blame] | 65 | #ifdef MODULE_PARAM_PREFIX |
| 66 | #undef MODULE_PARAM_PREFIX |
| 67 | #endif |
| 68 | #define MODULE_PARAM_PREFIX "rcutree." |
| 69 | |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 70 | /* Data structures. */ |
| 71 | |
Steven Rostedt (Red Hat) | f7f7bac | 2013-07-12 17:18:47 -0400 | [diff] [blame] | 72 | /* |
| 73 | * In order to export the rcu_state name to the tracing tools, it |
| 74 | * needs to be added in the __tracepoint_string section. |
| 75 | * This requires defining a separate variable tp_<sname>_varname |
| 76 | * that points to the string being used, and this will allow |
| 77 | * the tracing userspace tools to be able to decipher the string |
| 78 | * address to the matching string. |
| 79 | */ |
Ard Biesheuvel | a8a29b3 | 2014-07-12 19:01:49 +0200 | [diff] [blame] | 80 | #ifdef CONFIG_TRACING |
| 81 | # define DEFINE_RCU_TPS(sname) \ |
Steven Rostedt (Red Hat) | f7f7bac | 2013-07-12 17:18:47 -0400 | [diff] [blame] | 82 | static char sname##_varname[] = #sname; \ |
Ard Biesheuvel | a8a29b3 | 2014-07-12 19:01:49 +0200 | [diff] [blame] | 83 | static const char *tp_##sname##_varname __used __tracepoint_string = sname##_varname; |
| 84 | # define RCU_STATE_NAME(sname) sname##_varname |
| 85 | #else |
| 86 | # define DEFINE_RCU_TPS(sname) |
| 87 | # define RCU_STATE_NAME(sname) __stringify(sname) |
| 88 | #endif |
| 89 | |
| 90 | #define RCU_STATE_INITIALIZER(sname, sabbr, cr) \ |
| 91 | DEFINE_RCU_TPS(sname) \ |
Nicolas Iooss | c92fb05 | 2015-05-05 21:57:06 +0800 | [diff] [blame] | 92 | static DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_data, sname##_data); \ |
Steven Rostedt (Red Hat) | a41bfeb | 2013-07-12 17:00:28 -0400 | [diff] [blame] | 93 | struct rcu_state sname##_state = { \ |
Paul E. McKenney | 6c90cc7 | 2012-05-28 19:41:41 -0700 | [diff] [blame] | 94 | .level = { &sname##_state.node[0] }, \ |
Paul E. McKenney | 2723249 | 2015-01-20 22:44:13 -0800 | [diff] [blame] | 95 | .rda = &sname##_data, \ |
Paul E. McKenney | 037b64e | 2012-05-28 23:26:01 -0700 | [diff] [blame] | 96 | .call = cr, \ |
Petr Mladek | 77f81fe | 2015-09-09 12:09:49 -0700 | [diff] [blame] | 97 | .gp_state = RCU_GP_IDLE, \ |
Paul E. McKenney | 42c3533 | 2012-09-28 10:49:58 -0700 | [diff] [blame] | 98 | .gpnum = 0UL - 300UL, \ |
| 99 | .completed = 0UL - 300UL, \ |
Paul E. McKenney | de30ad5 | 2018-04-26 11:52:09 -0700 | [diff] [blame] | 100 | .gp_seq = (0UL - 300UL) << RCU_SEQ_CTR_SHIFT, \ |
Paul E. McKenney | 7be7f0b | 2012-05-29 05:18:53 -0700 | [diff] [blame] | 101 | .barrier_mutex = __MUTEX_INITIALIZER(sname##_state.barrier_mutex), \ |
Ard Biesheuvel | a8a29b3 | 2014-07-12 19:01:49 +0200 | [diff] [blame] | 102 | .name = RCU_STATE_NAME(sname), \ |
Paul E. McKenney | a488985 | 2012-12-03 08:16:28 -0800 | [diff] [blame] | 103 | .abbr = sabbr, \ |
Paul E. McKenney | f6a12f3 | 2016-01-30 17:57:35 -0800 | [diff] [blame] | 104 | .exp_mutex = __MUTEX_INITIALIZER(sname##_state.exp_mutex), \ |
Paul E. McKenney | 3b5f668 | 2016-03-16 16:47:55 -0700 | [diff] [blame] | 105 | .exp_wake_mutex = __MUTEX_INITIALIZER(sname##_state.exp_wake_mutex), \ |
Paul E. McKenney | 2723249 | 2015-01-20 22:44:13 -0800 | [diff] [blame] | 106 | } |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 107 | |
Steven Rostedt (Red Hat) | a41bfeb | 2013-07-12 17:00:28 -0400 | [diff] [blame] | 108 | RCU_STATE_INITIALIZER(rcu_sched, 's', call_rcu_sched); |
| 109 | RCU_STATE_INITIALIZER(rcu_bh, 'b', call_rcu_bh); |
Ingo Molnar | b1f77b0 | 2009-03-13 03:20:49 +0100 | [diff] [blame] | 110 | |
Paul E. McKenney | b28a7c0 | 2015-03-04 07:39:27 -0800 | [diff] [blame] | 111 | static struct rcu_state *const rcu_state_p; |
Paul E. McKenney | 6ce75a2 | 2012-06-12 11:01:13 -0700 | [diff] [blame] | 112 | LIST_HEAD(rcu_struct_flavors); |
Paul E. McKenney | 27f4d28 | 2011-02-07 12:47:15 -0800 | [diff] [blame] | 113 | |
Paul E. McKenney | a3dc294 | 2015-04-20 11:40:50 -0700 | [diff] [blame] | 114 | /* Dump rcu_node combining tree at boot to verify correct setup. */ |
| 115 | static bool dump_tree; |
| 116 | module_param(dump_tree, bool, 0444); |
Paul E. McKenney | 7fa2700 | 2015-04-20 10:27:15 -0700 | [diff] [blame] | 117 | /* Control rcu_node-tree auto-balancing at boot time. */ |
| 118 | static bool rcu_fanout_exact; |
| 119 | module_param(rcu_fanout_exact, bool, 0444); |
Paul E. McKenney | 47d631a | 2015-04-21 09:12:13 -0700 | [diff] [blame] | 120 | /* Increase (but not decrease) the RCU_FANOUT_LEAF at boot time. */ |
| 121 | static int rcu_fanout_leaf = RCU_FANOUT_LEAF; |
Paul E. McKenney | 7e5c2df | 2012-07-01 15:42:33 -0700 | [diff] [blame] | 122 | module_param(rcu_fanout_leaf, int, 0444); |
Paul E. McKenney | f885b7f | 2012-04-23 15:52:53 -0700 | [diff] [blame] | 123 | int rcu_num_lvls __read_mostly = RCU_NUM_LVLS; |
Alexander Gordeev | cb00710 | 2015-06-03 08:18:30 +0200 | [diff] [blame] | 124 | /* Number of rcu_nodes at specified level. */ |
Paul E. McKenney | e95d68d | 2017-03-15 13:11:11 -0700 | [diff] [blame] | 125 | int num_rcu_lvl[] = NUM_RCU_LVL_INIT; |
Paul E. McKenney | f885b7f | 2012-04-23 15:52:53 -0700 | [diff] [blame] | 126 | int rcu_num_nodes __read_mostly = NUM_RCU_NODES; /* Total # rcu_nodes in use. */ |
Daniel Bristot de Oliveira | 088e9d2 | 2016-06-02 13:51:41 -0300 | [diff] [blame] | 127 | /* panic() on RCU Stall sysctl. */ |
| 128 | int sysctl_panic_on_rcu_stall __read_mostly; |
Paul E. McKenney | f885b7f | 2012-04-23 15:52:53 -0700 | [diff] [blame] | 129 | |
Paul E. McKenney | b0d3041 | 2011-07-10 15:57:35 -0700 | [diff] [blame] | 130 | /* |
Paul E. McKenney | 52d7e48 | 2017-01-10 02:28:26 -0800 | [diff] [blame] | 131 | * The rcu_scheduler_active variable is initialized to the value |
| 132 | * RCU_SCHEDULER_INACTIVE and transitions RCU_SCHEDULER_INIT just before the |
| 133 | * first task is spawned. So when this variable is RCU_SCHEDULER_INACTIVE, |
| 134 | * RCU can assume that there is but one task, allowing RCU to (for example) |
Paul E. McKenney | 0d95092 | 2016-04-08 05:00:03 -0700 | [diff] [blame] | 135 | * optimize synchronize_rcu() to a simple barrier(). When this variable |
Paul E. McKenney | 52d7e48 | 2017-01-10 02:28:26 -0800 | [diff] [blame] | 136 | * is RCU_SCHEDULER_INIT, RCU must actually do all the hard work required |
| 137 | * to detect real grace periods. This variable is also used to suppress |
| 138 | * boot-time false positives from lockdep-RCU error checking. Finally, it |
| 139 | * transitions from RCU_SCHEDULER_INIT to RCU_SCHEDULER_RUNNING after RCU |
| 140 | * is fully initialized, including all of its kthreads having been spawned. |
Paul E. McKenney | b0d3041 | 2011-07-10 15:57:35 -0700 | [diff] [blame] | 141 | */ |
Paul E. McKenney | bbad937 | 2010-04-02 16:17:17 -0700 | [diff] [blame] | 142 | int rcu_scheduler_active __read_mostly; |
| 143 | EXPORT_SYMBOL_GPL(rcu_scheduler_active); |
| 144 | |
Paul E. McKenney | b0d3041 | 2011-07-10 15:57:35 -0700 | [diff] [blame] | 145 | /* |
| 146 | * The rcu_scheduler_fully_active variable transitions from zero to one |
| 147 | * during the early_initcall() processing, which is after the scheduler |
| 148 | * is capable of creating new tasks. So RCU processing (for example, |
| 149 | * creating tasks for RCU priority boosting) must be delayed until after |
| 150 | * rcu_scheduler_fully_active transitions from zero to one. We also |
| 151 | * currently delay invocation of any RCU callbacks until after this point. |
| 152 | * |
| 153 | * It might later prove better for people registering RCU callbacks during |
| 154 | * early boot to take responsibility for these callbacks, but one step at |
| 155 | * a time. |
| 156 | */ |
| 157 | static int rcu_scheduler_fully_active __read_mostly; |
| 158 | |
Paul E. McKenney | 0aa04b0 | 2015-01-23 21:52:37 -0800 | [diff] [blame] | 159 | static void rcu_init_new_rnp(struct rcu_node *rnp_leaf); |
| 160 | static void rcu_cleanup_dead_rnp(struct rcu_node *rnp_leaf); |
Thomas Gleixner | 5d01bbd | 2012-07-16 10:42:35 +0000 | [diff] [blame] | 161 | static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu); |
Paul E. McKenney | a46e089 | 2011-06-15 15:47:09 -0700 | [diff] [blame] | 162 | static void invoke_rcu_core(void); |
| 163 | static void invoke_rcu_callbacks(struct rcu_state *rsp, struct rcu_data *rdp); |
Paul E. McKenney | 6587a23 | 2015-08-06 16:50:39 -0700 | [diff] [blame] | 164 | static void rcu_report_exp_rdp(struct rcu_state *rsp, |
| 165 | struct rcu_data *rdp, bool wake); |
Paul E. McKenney | 3549c2b | 2016-04-15 16:35:29 -0700 | [diff] [blame] | 166 | static void sync_sched_exp_online_cleanup(int cpu); |
Paul E. McKenney | a26ac24 | 2011-01-12 14:10:23 -0800 | [diff] [blame] | 167 | |
Paul E. McKenney | a94844b | 2014-12-12 07:37:48 -0800 | [diff] [blame] | 168 | /* rcuc/rcub kthread realtime priority */ |
Paul E. McKenney | 26730f5 | 2015-04-21 09:22:14 -0700 | [diff] [blame] | 169 | static int kthread_prio = IS_ENABLED(CONFIG_RCU_BOOST) ? 1 : 0; |
Paul E. McKenney | a94844b | 2014-12-12 07:37:48 -0800 | [diff] [blame] | 170 | module_param(kthread_prio, int, 0644); |
| 171 | |
Paul E. McKenney | 8d7dc92 | 2015-04-14 19:33:59 -0700 | [diff] [blame] | 172 | /* Delay in jiffies for grace-period initialization delays, debug only. */ |
Paul E. McKenney | 0f41c0d | 2015-03-10 18:33:20 -0700 | [diff] [blame] | 173 | |
Paul E. McKenney | 90040c9 | 2017-05-10 14:36:55 -0700 | [diff] [blame] | 174 | static int gp_preinit_delay; |
| 175 | module_param(gp_preinit_delay, int, 0444); |
| 176 | static int gp_init_delay; |
| 177 | module_param(gp_init_delay, int, 0444); |
| 178 | static int gp_cleanup_delay; |
| 179 | module_param(gp_cleanup_delay, int, 0444); |
Paul E. McKenney | 0f41c0d | 2015-03-10 18:33:20 -0700 | [diff] [blame] | 180 | |
Paul E. McKenney | eab128e | 2015-04-15 12:08:22 -0700 | [diff] [blame] | 181 | /* |
| 182 | * Number of grace periods between delays, normalized by the duration of |
Paul E. McKenney | bfd090b | 2017-02-08 14:49:27 -0800 | [diff] [blame] | 183 | * the delay. The longer the delay, the more the grace periods between |
Paul E. McKenney | eab128e | 2015-04-15 12:08:22 -0700 | [diff] [blame] | 184 | * each delay. The reason for this normalization is that it means that, |
| 185 | * for non-zero delays, the overall slowdown of grace periods is constant |
| 186 | * regardless of the duration of the delay. This arrangement balances |
| 187 | * the need for long delays to increase some race probabilities with the |
| 188 | * need for fast grace periods to increase other race probabilities. |
| 189 | */ |
| 190 | #define PER_RCU_NODE_PERIOD 3 /* Number of grace periods between delays. */ |
Paul E. McKenney | 37745d2 | 2015-01-22 18:24:08 -0800 | [diff] [blame] | 191 | |
Paul E. McKenney | a26ac24 | 2011-01-12 14:10:23 -0800 | [diff] [blame] | 192 | /* |
Paul E. McKenney | 4a29865 | 2011-04-03 21:33:51 -0700 | [diff] [blame] | 193 | * Track the rcutorture test sequence number and the update version |
| 194 | * number within a given test. The rcutorture_testseq is incremented |
| 195 | * on every rcutorture module load and unload, so has an odd value |
| 196 | * when a test is running. The rcutorture_vernum is set to zero |
| 197 | * when rcutorture starts and is incremented on each rcutorture update. |
| 198 | * These variables enable correlating rcutorture output with the |
| 199 | * RCU tracing information. |
| 200 | */ |
| 201 | unsigned long rcutorture_testseq; |
| 202 | unsigned long rcutorture_vernum; |
| 203 | |
| 204 | /* |
Paul E. McKenney | 0aa04b0 | 2015-01-23 21:52:37 -0800 | [diff] [blame] | 205 | * Compute the mask of online CPUs for the specified rcu_node structure. |
| 206 | * This will not be stable unless the rcu_node structure's ->lock is |
| 207 | * held, but the bit corresponding to the current CPU will be stable |
| 208 | * in most contexts. |
| 209 | */ |
| 210 | unsigned long rcu_rnp_online_cpus(struct rcu_node *rnp) |
| 211 | { |
Paul E. McKenney | 7d0ae80 | 2015-03-03 14:57:58 -0800 | [diff] [blame] | 212 | return READ_ONCE(rnp->qsmaskinitnext); |
Paul E. McKenney | 0aa04b0 | 2015-01-23 21:52:37 -0800 | [diff] [blame] | 213 | } |
| 214 | |
| 215 | /* |
Paul E. McKenney | 7d0ae80 | 2015-03-03 14:57:58 -0800 | [diff] [blame] | 216 | * Return true if an RCU grace period is in progress. The READ_ONCE()s |
Paul E. McKenney | fc2219d4 | 2009-09-23 09:50:41 -0700 | [diff] [blame] | 217 | * permit this function to be invoked without holding the root rcu_node |
| 218 | * structure's ->lock, but of course results can be subject to change. |
| 219 | */ |
| 220 | static int rcu_gp_in_progress(struct rcu_state *rsp) |
| 221 | { |
Paul E. McKenney | 7d0ae80 | 2015-03-03 14:57:58 -0800 | [diff] [blame] | 222 | return READ_ONCE(rsp->completed) != READ_ONCE(rsp->gpnum); |
Paul E. McKenney | fc2219d4 | 2009-09-23 09:50:41 -0700 | [diff] [blame] | 223 | } |
| 224 | |
| 225 | /* |
Paul E. McKenney | d6714c2 | 2009-08-22 13:56:46 -0700 | [diff] [blame] | 226 | * Note a quiescent state. Because we do not need to know |
Ingo Molnar | b1f77b0 | 2009-03-13 03:20:49 +0100 | [diff] [blame] | 227 | * how many quiescent states passed, just if there was at least |
Paul E. McKenney | d6714c2 | 2009-08-22 13:56:46 -0700 | [diff] [blame] | 228 | * one since the start of the grace period, this just sets a flag. |
Paul E. McKenney | e4cc1f2 | 2011-06-27 00:17:43 -0700 | [diff] [blame] | 229 | * The caller must have disabled preemption. |
Ingo Molnar | b1f77b0 | 2009-03-13 03:20:49 +0100 | [diff] [blame] | 230 | */ |
Paul E. McKenney | 284a8c9 | 2014-08-14 16:38:46 -0700 | [diff] [blame] | 231 | void rcu_sched_qs(void) |
Ingo Molnar | b1f77b0 | 2009-03-13 03:20:49 +0100 | [diff] [blame] | 232 | { |
Paul E. McKenney | f4687d2 | 2017-04-27 16:13:53 -0700 | [diff] [blame] | 233 | RCU_LOCKDEP_WARN(preemptible(), "rcu_sched_qs() invoked with preemption enabled!!!"); |
Paul E. McKenney | fecbf6f | 2015-09-28 18:19:24 -0700 | [diff] [blame] | 234 | if (!__this_cpu_read(rcu_sched_data.cpu_no_qs.s)) |
| 235 | return; |
| 236 | trace_rcu_grace_period(TPS("rcu_sched"), |
| 237 | __this_cpu_read(rcu_sched_data.gpnum), |
| 238 | TPS("cpuqs")); |
| 239 | __this_cpu_write(rcu_sched_data.cpu_no_qs.b.norm, false); |
| 240 | if (!__this_cpu_read(rcu_sched_data.cpu_no_qs.b.exp)) |
| 241 | return; |
Paul E. McKenney | 46a5d16 | 2015-10-07 09:10:48 -0700 | [diff] [blame] | 242 | __this_cpu_write(rcu_sched_data.cpu_no_qs.b.exp, false); |
| 243 | rcu_report_exp_rdp(&rcu_sched_state, |
| 244 | this_cpu_ptr(&rcu_sched_data), true); |
Ingo Molnar | b1f77b0 | 2009-03-13 03:20:49 +0100 | [diff] [blame] | 245 | } |
| 246 | |
Paul E. McKenney | 284a8c9 | 2014-08-14 16:38:46 -0700 | [diff] [blame] | 247 | void rcu_bh_qs(void) |
Ingo Molnar | b1f77b0 | 2009-03-13 03:20:49 +0100 | [diff] [blame] | 248 | { |
Paul E. McKenney | f4687d2 | 2017-04-27 16:13:53 -0700 | [diff] [blame] | 249 | RCU_LOCKDEP_WARN(preemptible(), "rcu_bh_qs() invoked with preemption enabled!!!"); |
Paul E. McKenney | 5b74c45 | 2015-08-06 15:16:57 -0700 | [diff] [blame] | 250 | if (__this_cpu_read(rcu_bh_data.cpu_no_qs.s)) { |
Paul E. McKenney | 284a8c9 | 2014-08-14 16:38:46 -0700 | [diff] [blame] | 251 | trace_rcu_grace_period(TPS("rcu_bh"), |
| 252 | __this_cpu_read(rcu_bh_data.gpnum), |
| 253 | TPS("cpuqs")); |
Paul E. McKenney | 5b74c45 | 2015-08-06 15:16:57 -0700 | [diff] [blame] | 254 | __this_cpu_write(rcu_bh_data.cpu_no_qs.b.norm, false); |
Paul E. McKenney | 284a8c9 | 2014-08-14 16:38:46 -0700 | [diff] [blame] | 255 | } |
Ingo Molnar | b1f77b0 | 2009-03-13 03:20:49 +0100 | [diff] [blame] | 256 | } |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 257 | |
Paul E. McKenney | b8c17e6 | 2016-11-08 14:25:21 -0800 | [diff] [blame] | 258 | /* |
| 259 | * Steal a bit from the bottom of ->dynticks for idle entry/exit |
| 260 | * control. Initially this is for TLB flushing. |
| 261 | */ |
| 262 | #define RCU_DYNTICK_CTRL_MASK 0x1 |
| 263 | #define RCU_DYNTICK_CTRL_CTR (RCU_DYNTICK_CTRL_MASK + 1) |
| 264 | #ifndef rcu_eqs_special_exit |
| 265 | #define rcu_eqs_special_exit() do { } while (0) |
| 266 | #endif |
Paul E. McKenney | 4a81e83 | 2014-06-20 16:49:01 -0700 | [diff] [blame] | 267 | |
| 268 | static DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = { |
Paul E. McKenney | 51a1fd3 | 2017-10-03 14:43:40 -0700 | [diff] [blame] | 269 | .dynticks_nesting = 1, |
Paul E. McKenney | 58721f5 | 2017-10-03 10:42:22 -0700 | [diff] [blame] | 270 | .dynticks_nmi_nesting = DYNTICK_IRQ_NONIDLE, |
Paul E. McKenney | b8c17e6 | 2016-11-08 14:25:21 -0800 | [diff] [blame] | 271 | .dynticks = ATOMIC_INIT(RCU_DYNTICK_CTRL_CTR), |
Paul E. McKenney | 4a81e83 | 2014-06-20 16:49:01 -0700 | [diff] [blame] | 272 | }; |
| 273 | |
Paul E. McKenney | 6563de9 | 2016-11-02 13:33:57 -0700 | [diff] [blame] | 274 | /* |
Paul E. McKenney | 2625d46 | 2016-11-02 14:23:30 -0700 | [diff] [blame] | 275 | * Record entry into an extended quiescent state. This is only to be |
| 276 | * called when not already in an extended quiescent state. |
| 277 | */ |
| 278 | static void rcu_dynticks_eqs_enter(void) |
| 279 | { |
| 280 | struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks); |
Paul E. McKenney | b8c17e6 | 2016-11-08 14:25:21 -0800 | [diff] [blame] | 281 | int seq; |
Paul E. McKenney | 2625d46 | 2016-11-02 14:23:30 -0700 | [diff] [blame] | 282 | |
| 283 | /* |
Paul E. McKenney | b8c17e6 | 2016-11-08 14:25:21 -0800 | [diff] [blame] | 284 | * CPUs seeing atomic_add_return() must see prior RCU read-side |
Paul E. McKenney | 2625d46 | 2016-11-02 14:23:30 -0700 | [diff] [blame] | 285 | * critical sections, and we also must force ordering with the |
| 286 | * next idle sojourn. |
| 287 | */ |
Paul E. McKenney | b8c17e6 | 2016-11-08 14:25:21 -0800 | [diff] [blame] | 288 | seq = atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdtp->dynticks); |
| 289 | /* Better be in an extended quiescent state! */ |
| 290 | WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && |
| 291 | (seq & RCU_DYNTICK_CTRL_CTR)); |
| 292 | /* Better not have special action (TLB flush) pending! */ |
| 293 | WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && |
| 294 | (seq & RCU_DYNTICK_CTRL_MASK)); |
Paul E. McKenney | 2625d46 | 2016-11-02 14:23:30 -0700 | [diff] [blame] | 295 | } |
| 296 | |
| 297 | /* |
| 298 | * Record exit from an extended quiescent state. This is only to be |
| 299 | * called from an extended quiescent state. |
| 300 | */ |
| 301 | static void rcu_dynticks_eqs_exit(void) |
| 302 | { |
| 303 | struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks); |
Paul E. McKenney | b8c17e6 | 2016-11-08 14:25:21 -0800 | [diff] [blame] | 304 | int seq; |
Paul E. McKenney | 2625d46 | 2016-11-02 14:23:30 -0700 | [diff] [blame] | 305 | |
| 306 | /* |
Paul E. McKenney | b8c17e6 | 2016-11-08 14:25:21 -0800 | [diff] [blame] | 307 | * CPUs seeing atomic_add_return() must see prior idle sojourns, |
Paul E. McKenney | 2625d46 | 2016-11-02 14:23:30 -0700 | [diff] [blame] | 308 | * and we also must force ordering with the next RCU read-side |
| 309 | * critical section. |
| 310 | */ |
Paul E. McKenney | b8c17e6 | 2016-11-08 14:25:21 -0800 | [diff] [blame] | 311 | seq = atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdtp->dynticks); |
| 312 | WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && |
| 313 | !(seq & RCU_DYNTICK_CTRL_CTR)); |
| 314 | if (seq & RCU_DYNTICK_CTRL_MASK) { |
| 315 | atomic_andnot(RCU_DYNTICK_CTRL_MASK, &rdtp->dynticks); |
| 316 | smp_mb__after_atomic(); /* _exit after clearing mask. */ |
| 317 | /* Prefer duplicate flushes to losing a flush. */ |
| 318 | rcu_eqs_special_exit(); |
| 319 | } |
Paul E. McKenney | 2625d46 | 2016-11-02 14:23:30 -0700 | [diff] [blame] | 320 | } |
| 321 | |
| 322 | /* |
| 323 | * Reset the current CPU's ->dynticks counter to indicate that the |
| 324 | * newly onlined CPU is no longer in an extended quiescent state. |
| 325 | * This will either leave the counter unchanged, or increment it |
| 326 | * to the next non-quiescent value. |
| 327 | * |
| 328 | * The non-atomic test/increment sequence works because the upper bits |
| 329 | * of the ->dynticks counter are manipulated only by the corresponding CPU, |
| 330 | * or when the corresponding CPU is offline. |
| 331 | */ |
| 332 | static void rcu_dynticks_eqs_online(void) |
| 333 | { |
| 334 | struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks); |
| 335 | |
Paul E. McKenney | b8c17e6 | 2016-11-08 14:25:21 -0800 | [diff] [blame] | 336 | if (atomic_read(&rdtp->dynticks) & RCU_DYNTICK_CTRL_CTR) |
Paul E. McKenney | 2625d46 | 2016-11-02 14:23:30 -0700 | [diff] [blame] | 337 | return; |
Paul E. McKenney | b8c17e6 | 2016-11-08 14:25:21 -0800 | [diff] [blame] | 338 | atomic_add(RCU_DYNTICK_CTRL_CTR, &rdtp->dynticks); |
Paul E. McKenney | 2625d46 | 2016-11-02 14:23:30 -0700 | [diff] [blame] | 339 | } |
| 340 | |
| 341 | /* |
Paul E. McKenney | 02a5c550 | 2016-11-02 17:25:06 -0700 | [diff] [blame] | 342 | * Is the current CPU in an extended quiescent state? |
| 343 | * |
| 344 | * No ordering, as we are sampling CPU-local information. |
| 345 | */ |
| 346 | bool rcu_dynticks_curr_cpu_in_eqs(void) |
| 347 | { |
| 348 | struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks); |
| 349 | |
Paul E. McKenney | b8c17e6 | 2016-11-08 14:25:21 -0800 | [diff] [blame] | 350 | return !(atomic_read(&rdtp->dynticks) & RCU_DYNTICK_CTRL_CTR); |
Paul E. McKenney | 02a5c550 | 2016-11-02 17:25:06 -0700 | [diff] [blame] | 351 | } |
| 352 | |
| 353 | /* |
Paul E. McKenney | 8b2f63a | 2016-11-02 14:12:05 -0700 | [diff] [blame] | 354 | * Snapshot the ->dynticks counter with full ordering so as to allow |
| 355 | * stable comparison of this counter with past and future snapshots. |
| 356 | */ |
Paul E. McKenney | 02a5c550 | 2016-11-02 17:25:06 -0700 | [diff] [blame] | 357 | int rcu_dynticks_snap(struct rcu_dynticks *rdtp) |
Paul E. McKenney | 8b2f63a | 2016-11-02 14:12:05 -0700 | [diff] [blame] | 358 | { |
| 359 | int snap = atomic_add_return(0, &rdtp->dynticks); |
| 360 | |
Paul E. McKenney | b8c17e6 | 2016-11-08 14:25:21 -0800 | [diff] [blame] | 361 | return snap & ~RCU_DYNTICK_CTRL_MASK; |
Paul E. McKenney | 8b2f63a | 2016-11-02 14:12:05 -0700 | [diff] [blame] | 362 | } |
| 363 | |
| 364 | /* |
Paul E. McKenney | 02a5c550 | 2016-11-02 17:25:06 -0700 | [diff] [blame] | 365 | * Return true if the snapshot returned from rcu_dynticks_snap() |
| 366 | * indicates that RCU is in an extended quiescent state. |
| 367 | */ |
| 368 | static bool rcu_dynticks_in_eqs(int snap) |
| 369 | { |
Paul E. McKenney | b8c17e6 | 2016-11-08 14:25:21 -0800 | [diff] [blame] | 370 | return !(snap & RCU_DYNTICK_CTRL_CTR); |
Paul E. McKenney | 02a5c550 | 2016-11-02 17:25:06 -0700 | [diff] [blame] | 371 | } |
| 372 | |
| 373 | /* |
| 374 | * Return true if the CPU corresponding to the specified rcu_dynticks |
| 375 | * structure has spent some time in an extended quiescent state since |
| 376 | * rcu_dynticks_snap() returned the specified snapshot. |
| 377 | */ |
| 378 | static bool rcu_dynticks_in_eqs_since(struct rcu_dynticks *rdtp, int snap) |
| 379 | { |
| 380 | return snap != rcu_dynticks_snap(rdtp); |
| 381 | } |
| 382 | |
| 383 | /* |
Paul E. McKenney | 6563de9 | 2016-11-02 13:33:57 -0700 | [diff] [blame] | 384 | * Do a double-increment of the ->dynticks counter to emulate a |
| 385 | * momentary idle-CPU quiescent state. |
| 386 | */ |
| 387 | static void rcu_dynticks_momentary_idle(void) |
| 388 | { |
| 389 | struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks); |
Paul E. McKenney | b8c17e6 | 2016-11-08 14:25:21 -0800 | [diff] [blame] | 390 | int special = atomic_add_return(2 * RCU_DYNTICK_CTRL_CTR, |
| 391 | &rdtp->dynticks); |
Paul E. McKenney | 6563de9 | 2016-11-02 13:33:57 -0700 | [diff] [blame] | 392 | |
| 393 | /* It is illegal to call this from idle state. */ |
Paul E. McKenney | b8c17e6 | 2016-11-08 14:25:21 -0800 | [diff] [blame] | 394 | WARN_ON_ONCE(!(special & RCU_DYNTICK_CTRL_CTR)); |
Paul E. McKenney | 6563de9 | 2016-11-02 13:33:57 -0700 | [diff] [blame] | 395 | } |
| 396 | |
Paul E. McKenney | b8c17e6 | 2016-11-08 14:25:21 -0800 | [diff] [blame] | 397 | /* |
| 398 | * Set the special (bottom) bit of the specified CPU so that it |
| 399 | * will take special action (such as flushing its TLB) on the |
| 400 | * next exit from an extended quiescent state. Returns true if |
| 401 | * the bit was successfully set, or false if the CPU was not in |
| 402 | * an extended quiescent state. |
| 403 | */ |
| 404 | bool rcu_eqs_special_set(int cpu) |
| 405 | { |
| 406 | int old; |
| 407 | int new; |
| 408 | struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu); |
| 409 | |
| 410 | do { |
| 411 | old = atomic_read(&rdtp->dynticks); |
| 412 | if (old & RCU_DYNTICK_CTRL_CTR) |
| 413 | return false; |
| 414 | new = old | RCU_DYNTICK_CTRL_MASK; |
| 415 | } while (atomic_cmpxchg(&rdtp->dynticks, old, new) != old); |
| 416 | return true; |
Paul E. McKenney | 4a81e83 | 2014-06-20 16:49:01 -0700 | [diff] [blame] | 417 | } |
Paul E. McKenney | 5cd3719 | 2014-12-13 20:32:04 -0800 | [diff] [blame] | 418 | |
Paul E. McKenney | 4a81e83 | 2014-06-20 16:49:01 -0700 | [diff] [blame] | 419 | /* |
| 420 | * Let the RCU core know that this CPU has gone through the scheduler, |
| 421 | * which is a quiescent state. This is called when the need for a |
| 422 | * quiescent state is urgent, so we burn an atomic operation and full |
| 423 | * memory barriers to let the RCU core know about it, regardless of what |
| 424 | * this CPU might (or might not) do in the near future. |
| 425 | * |
Paul E. McKenney | 0f9be8c | 2017-01-27 13:17:02 -0800 | [diff] [blame] | 426 | * We inform the RCU core by emulating a zero-duration dyntick-idle period. |
Paul E. McKenney | 46a5d16 | 2015-10-07 09:10:48 -0700 | [diff] [blame] | 427 | * |
| 428 | * The caller must have disabled interrupts. |
Paul E. McKenney | 4a81e83 | 2014-06-20 16:49:01 -0700 | [diff] [blame] | 429 | */ |
| 430 | static void rcu_momentary_dyntick_idle(void) |
| 431 | { |
Paul E. McKenney | 0f9be8c | 2017-01-27 13:17:02 -0800 | [diff] [blame] | 432 | raw_cpu_write(rcu_dynticks.rcu_need_heavy_qs, false); |
| 433 | rcu_dynticks_momentary_idle(); |
Paul E. McKenney | 4a81e83 | 2014-06-20 16:49:01 -0700 | [diff] [blame] | 434 | } |
| 435 | |
Paul E. McKenney | 25502a6 | 2010-04-01 17:37:01 -0700 | [diff] [blame] | 436 | /* |
| 437 | * Note a context switch. This is a quiescent state for RCU-sched, |
| 438 | * and requires special handling for preemptible RCU. |
Paul E. McKenney | 46a5d16 | 2015-10-07 09:10:48 -0700 | [diff] [blame] | 439 | * The caller must have disabled interrupts. |
Paul E. McKenney | 25502a6 | 2010-04-01 17:37:01 -0700 | [diff] [blame] | 440 | */ |
Paul E. McKenney | bcbfdd0 | 2017-04-11 15:50:41 -0700 | [diff] [blame] | 441 | void rcu_note_context_switch(bool preempt) |
Paul E. McKenney | 25502a6 | 2010-04-01 17:37:01 -0700 | [diff] [blame] | 442 | { |
Boqun Feng | bb73c52 | 2015-07-30 16:55:38 -0700 | [diff] [blame] | 443 | barrier(); /* Avoid RCU read-side critical sections leaking down. */ |
Steven Rostedt (Red Hat) | f7f7bac | 2013-07-12 17:18:47 -0400 | [diff] [blame] | 444 | trace_rcu_utilization(TPS("Start context switch")); |
Paul E. McKenney | 284a8c9 | 2014-08-14 16:38:46 -0700 | [diff] [blame] | 445 | rcu_sched_qs(); |
Paul E. McKenney | 5b72f96 | 2017-04-12 15:29:14 -0700 | [diff] [blame] | 446 | rcu_preempt_note_context_switch(preempt); |
Paul E. McKenney | 9226b10 | 2017-01-27 14:17:50 -0800 | [diff] [blame] | 447 | /* Load rcu_urgent_qs before other flags. */ |
| 448 | if (!smp_load_acquire(this_cpu_ptr(&rcu_dynticks.rcu_urgent_qs))) |
| 449 | goto out; |
| 450 | this_cpu_write(rcu_dynticks.rcu_urgent_qs, false); |
Paul E. McKenney | 0f9be8c | 2017-01-27 13:17:02 -0800 | [diff] [blame] | 451 | if (unlikely(raw_cpu_read(rcu_dynticks.rcu_need_heavy_qs))) |
Paul E. McKenney | 4a81e83 | 2014-06-20 16:49:01 -0700 | [diff] [blame] | 452 | rcu_momentary_dyntick_idle(); |
Paul E. McKenney | 9226b10 | 2017-01-27 14:17:50 -0800 | [diff] [blame] | 453 | this_cpu_inc(rcu_dynticks.rcu_qs_ctr); |
Paul E. McKenney | bcbfdd0 | 2017-04-11 15:50:41 -0700 | [diff] [blame] | 454 | if (!preempt) |
| 455 | rcu_note_voluntary_context_switch_lite(current); |
Paul E. McKenney | 9226b10 | 2017-01-27 14:17:50 -0800 | [diff] [blame] | 456 | out: |
Steven Rostedt (Red Hat) | f7f7bac | 2013-07-12 17:18:47 -0400 | [diff] [blame] | 457 | trace_rcu_utilization(TPS("End context switch")); |
Boqun Feng | bb73c52 | 2015-07-30 16:55:38 -0700 | [diff] [blame] | 458 | barrier(); /* Avoid RCU read-side critical sections leaking up. */ |
Paul E. McKenney | 25502a6 | 2010-04-01 17:37:01 -0700 | [diff] [blame] | 459 | } |
Gleb Natapov | 29ce831 | 2011-05-04 16:31:03 +0300 | [diff] [blame] | 460 | EXPORT_SYMBOL_GPL(rcu_note_context_switch); |
Paul E. McKenney | 25502a6 | 2010-04-01 17:37:01 -0700 | [diff] [blame] | 461 | |
Paul E. McKenney | 5cd3719 | 2014-12-13 20:32:04 -0800 | [diff] [blame] | 462 | /* |
Paul E. McKenney | 1925d19 | 2015-01-18 17:45:05 -0800 | [diff] [blame] | 463 | * Register a quiescent state for all RCU flavors. If there is an |
Paul E. McKenney | 5cd3719 | 2014-12-13 20:32:04 -0800 | [diff] [blame] | 464 | * emergency, invoke rcu_momentary_dyntick_idle() to do a heavy-weight |
| 465 | * dyntick-idle quiescent state visible to other CPUs (but only for those |
Paul E. McKenney | 1925d19 | 2015-01-18 17:45:05 -0800 | [diff] [blame] | 466 | * RCU flavors in desperate need of a quiescent state, which will normally |
Paul E. McKenney | 5cd3719 | 2014-12-13 20:32:04 -0800 | [diff] [blame] | 467 | * be none of them). Either way, do a lightweight quiescent state for |
| 468 | * all RCU flavors. |
Boqun Feng | bb73c52 | 2015-07-30 16:55:38 -0700 | [diff] [blame] | 469 | * |
| 470 | * The barrier() calls are redundant in the common case when this is |
| 471 | * called externally, but just in case this is called from within this |
| 472 | * file. |
| 473 | * |
Paul E. McKenney | 5cd3719 | 2014-12-13 20:32:04 -0800 | [diff] [blame] | 474 | */ |
| 475 | void rcu_all_qs(void) |
| 476 | { |
Paul E. McKenney | 46a5d16 | 2015-10-07 09:10:48 -0700 | [diff] [blame] | 477 | unsigned long flags; |
| 478 | |
Paul E. McKenney | 9226b10 | 2017-01-27 14:17:50 -0800 | [diff] [blame] | 479 | if (!raw_cpu_read(rcu_dynticks.rcu_urgent_qs)) |
| 480 | return; |
| 481 | preempt_disable(); |
| 482 | /* Load rcu_urgent_qs before other flags. */ |
| 483 | if (!smp_load_acquire(this_cpu_ptr(&rcu_dynticks.rcu_urgent_qs))) { |
| 484 | preempt_enable(); |
| 485 | return; |
| 486 | } |
| 487 | this_cpu_write(rcu_dynticks.rcu_urgent_qs, false); |
Boqun Feng | bb73c52 | 2015-07-30 16:55:38 -0700 | [diff] [blame] | 488 | barrier(); /* Avoid RCU read-side critical sections leaking down. */ |
Paul E. McKenney | 0f9be8c | 2017-01-27 13:17:02 -0800 | [diff] [blame] | 489 | if (unlikely(raw_cpu_read(rcu_dynticks.rcu_need_heavy_qs))) { |
Paul E. McKenney | 46a5d16 | 2015-10-07 09:10:48 -0700 | [diff] [blame] | 490 | local_irq_save(flags); |
Paul E. McKenney | 5cd3719 | 2014-12-13 20:32:04 -0800 | [diff] [blame] | 491 | rcu_momentary_dyntick_idle(); |
Paul E. McKenney | 46a5d16 | 2015-10-07 09:10:48 -0700 | [diff] [blame] | 492 | local_irq_restore(flags); |
| 493 | } |
Paul E. McKenney | 9226b10 | 2017-01-27 14:17:50 -0800 | [diff] [blame] | 494 | if (unlikely(raw_cpu_read(rcu_sched_data.cpu_no_qs.b.exp))) |
Paul E. McKenney | a1e1224 | 2016-01-13 13:57:54 -0800 | [diff] [blame] | 495 | rcu_sched_qs(); |
Paul E. McKenney | 9577df9 | 2017-01-26 16:18:07 -0800 | [diff] [blame] | 496 | this_cpu_inc(rcu_dynticks.rcu_qs_ctr); |
Boqun Feng | bb73c52 | 2015-07-30 16:55:38 -0700 | [diff] [blame] | 497 | barrier(); /* Avoid RCU read-side critical sections leaking up. */ |
Paul E. McKenney | 9226b10 | 2017-01-27 14:17:50 -0800 | [diff] [blame] | 498 | preempt_enable(); |
Paul E. McKenney | 5cd3719 | 2014-12-13 20:32:04 -0800 | [diff] [blame] | 499 | } |
| 500 | EXPORT_SYMBOL_GPL(rcu_all_qs); |
| 501 | |
Paul E. McKenney | 17c7798 | 2017-04-28 11:12:34 -0700 | [diff] [blame] | 502 | #define DEFAULT_RCU_BLIMIT 10 /* Maximum callbacks per rcu_do_batch. */ |
| 503 | static long blimit = DEFAULT_RCU_BLIMIT; |
| 504 | #define DEFAULT_RCU_QHIMARK 10000 /* If this many pending, ignore blimit. */ |
| 505 | static long qhimark = DEFAULT_RCU_QHIMARK; |
| 506 | #define DEFAULT_RCU_QLOMARK 100 /* Once only this many pending, use blimit. */ |
| 507 | static long qlowmark = DEFAULT_RCU_QLOMARK; |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 508 | |
Eric Dumazet | 878d743 | 2012-10-18 04:55:36 -0700 | [diff] [blame] | 509 | module_param(blimit, long, 0444); |
| 510 | module_param(qhimark, long, 0444); |
| 511 | module_param(qlowmark, long, 0444); |
Paul E. McKenney | 3d76c08 | 2009-09-28 07:46:32 -0700 | [diff] [blame] | 512 | |
Paul E. McKenney | 026ad28 | 2013-04-03 22:14:11 -0700 | [diff] [blame] | 513 | static ulong jiffies_till_first_fqs = ULONG_MAX; |
| 514 | static ulong jiffies_till_next_fqs = ULONG_MAX; |
Paul E. McKenney | 8c7c482 | 2016-01-03 20:29:57 -0800 | [diff] [blame] | 515 | static bool rcu_kick_kthreads; |
Paul E. McKenney | d40011f | 2012-06-26 20:45:57 -0700 | [diff] [blame] | 516 | |
| 517 | module_param(jiffies_till_first_fqs, ulong, 0644); |
| 518 | module_param(jiffies_till_next_fqs, ulong, 0644); |
Paul E. McKenney | 8c7c482 | 2016-01-03 20:29:57 -0800 | [diff] [blame] | 519 | module_param(rcu_kick_kthreads, bool, 0644); |
Paul E. McKenney | d40011f | 2012-06-26 20:45:57 -0700 | [diff] [blame] | 520 | |
Paul E. McKenney | 4a81e83 | 2014-06-20 16:49:01 -0700 | [diff] [blame] | 521 | /* |
| 522 | * How long the grace period must be before we start recruiting |
| 523 | * quiescent-state help from rcu_note_context_switch(). |
| 524 | */ |
Paul E. McKenney | f79c3ad | 2016-11-30 06:24:30 -0800 | [diff] [blame] | 525 | static ulong jiffies_till_sched_qs = HZ / 10; |
| 526 | module_param(jiffies_till_sched_qs, ulong, 0444); |
Paul E. McKenney | 4a81e83 | 2014-06-20 16:49:01 -0700 | [diff] [blame] | 527 | |
Paul E. McKenney | fe5ac72 | 2017-05-11 11:26:22 -0700 | [diff] [blame] | 528 | static void force_qs_rnp(struct rcu_state *rsp, int (*f)(struct rcu_data *rsp)); |
Paul E. McKenney | 4cdfc175 | 2012-06-22 17:06:26 -0700 | [diff] [blame] | 529 | static void force_quiescent_state(struct rcu_state *rsp); |
Paul E. McKenney | e3950ec | 2014-10-21 08:03:57 -0700 | [diff] [blame] | 530 | static int rcu_pending(void); |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 531 | |
| 532 | /* |
Paul E. McKenney | 17ef2fe | 2018-04-27 11:39:34 -0700 | [diff] [blame] | 533 | * Return the number of RCU GPs completed thus far for debug & stats. |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 534 | */ |
Paul E. McKenney | 17ef2fe | 2018-04-27 11:39:34 -0700 | [diff] [blame] | 535 | unsigned long rcu_get_gp_seq(void) |
Paul E. McKenney | 917963d | 2014-11-21 17:10:16 -0800 | [diff] [blame] | 536 | { |
Paul E. McKenney | 17ef2fe | 2018-04-27 11:39:34 -0700 | [diff] [blame] | 537 | return rcu_seq_ctr(READ_ONCE(rcu_state_p->gp_seq)); |
Paul E. McKenney | 917963d | 2014-11-21 17:10:16 -0800 | [diff] [blame] | 538 | } |
Paul E. McKenney | 17ef2fe | 2018-04-27 11:39:34 -0700 | [diff] [blame] | 539 | EXPORT_SYMBOL_GPL(rcu_get_gp_seq); |
Paul E. McKenney | 917963d | 2014-11-21 17:10:16 -0800 | [diff] [blame] | 540 | |
| 541 | /* |
Paul E. McKenney | 17ef2fe | 2018-04-27 11:39:34 -0700 | [diff] [blame] | 542 | * Return the number of RCU-sched GPs completed thus far for debug & stats. |
Paul E. McKenney | 917963d | 2014-11-21 17:10:16 -0800 | [diff] [blame] | 543 | */ |
Paul E. McKenney | 17ef2fe | 2018-04-27 11:39:34 -0700 | [diff] [blame] | 544 | unsigned long rcu_sched_get_gp_seq(void) |
Paul E. McKenney | 917963d | 2014-11-21 17:10:16 -0800 | [diff] [blame] | 545 | { |
Paul E. McKenney | 17ef2fe | 2018-04-27 11:39:34 -0700 | [diff] [blame] | 546 | return rcu_seq_ctr(READ_ONCE(rcu_sched_state.gp_seq)); |
Paul E. McKenney | 917963d | 2014-11-21 17:10:16 -0800 | [diff] [blame] | 547 | } |
Paul E. McKenney | 17ef2fe | 2018-04-27 11:39:34 -0700 | [diff] [blame] | 548 | EXPORT_SYMBOL_GPL(rcu_sched_get_gp_seq); |
Paul E. McKenney | 917963d | 2014-11-21 17:10:16 -0800 | [diff] [blame] | 549 | |
| 550 | /* |
Paul E. McKenney | 17ef2fe | 2018-04-27 11:39:34 -0700 | [diff] [blame] | 551 | * Return the number of RCU-bh GPs completed thus far for debug & stats. |
Paul E. McKenney | 917963d | 2014-11-21 17:10:16 -0800 | [diff] [blame] | 552 | */ |
Paul E. McKenney | 17ef2fe | 2018-04-27 11:39:34 -0700 | [diff] [blame] | 553 | unsigned long rcu_bh_get_gp_seq(void) |
Paul E. McKenney | 917963d | 2014-11-21 17:10:16 -0800 | [diff] [blame] | 554 | { |
Paul E. McKenney | 17ef2fe | 2018-04-27 11:39:34 -0700 | [diff] [blame] | 555 | return rcu_seq_ctr(READ_ONCE(rcu_bh_state.gp_seq)); |
Paul E. McKenney | 917963d | 2014-11-21 17:10:16 -0800 | [diff] [blame] | 556 | } |
Paul E. McKenney | 17ef2fe | 2018-04-27 11:39:34 -0700 | [diff] [blame] | 557 | EXPORT_SYMBOL_GPL(rcu_bh_get_gp_seq); |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 558 | |
| 559 | /* |
Paul E. McKenney | 291783b | 2016-01-12 13:43:30 -0800 | [diff] [blame] | 560 | * Return the number of RCU expedited batches completed thus far for |
| 561 | * debug & stats. Odd numbers mean that a batch is in progress, even |
| 562 | * numbers mean idle. The value returned will thus be roughly double |
| 563 | * the cumulative batches since boot. |
| 564 | */ |
| 565 | unsigned long rcu_exp_batches_completed(void) |
| 566 | { |
| 567 | return rcu_state_p->expedited_sequence; |
| 568 | } |
| 569 | EXPORT_SYMBOL_GPL(rcu_exp_batches_completed); |
| 570 | |
| 571 | /* |
| 572 | * Return the number of RCU-sched expedited batches completed thus far |
| 573 | * for debug & stats. Similar to rcu_exp_batches_completed(). |
| 574 | */ |
| 575 | unsigned long rcu_exp_batches_completed_sched(void) |
| 576 | { |
| 577 | return rcu_sched_state.expedited_sequence; |
| 578 | } |
| 579 | EXPORT_SYMBOL_GPL(rcu_exp_batches_completed_sched); |
| 580 | |
| 581 | /* |
Andreea-Cristina Bernat | a381d75 | 2014-03-19 11:18:31 +0200 | [diff] [blame] | 582 | * Force a quiescent state. |
| 583 | */ |
| 584 | void rcu_force_quiescent_state(void) |
| 585 | { |
Uma Sharma | e534165 | 2014-03-23 22:32:09 -0700 | [diff] [blame] | 586 | force_quiescent_state(rcu_state_p); |
Andreea-Cristina Bernat | a381d75 | 2014-03-19 11:18:31 +0200 | [diff] [blame] | 587 | } |
| 588 | EXPORT_SYMBOL_GPL(rcu_force_quiescent_state); |
| 589 | |
| 590 | /* |
Paul E. McKenney | bf66f18 | 2010-01-04 15:09:10 -0800 | [diff] [blame] | 591 | * Force a quiescent state for RCU BH. |
| 592 | */ |
| 593 | void rcu_bh_force_quiescent_state(void) |
| 594 | { |
Paul E. McKenney | 4cdfc175 | 2012-06-22 17:06:26 -0700 | [diff] [blame] | 595 | force_quiescent_state(&rcu_bh_state); |
Paul E. McKenney | bf66f18 | 2010-01-04 15:09:10 -0800 | [diff] [blame] | 596 | } |
| 597 | EXPORT_SYMBOL_GPL(rcu_bh_force_quiescent_state); |
| 598 | |
| 599 | /* |
Paul E. McKenney | e7580f3 | 2015-02-24 14:23:39 -0800 | [diff] [blame] | 600 | * Force a quiescent state for RCU-sched. |
| 601 | */ |
| 602 | void rcu_sched_force_quiescent_state(void) |
| 603 | { |
| 604 | force_quiescent_state(&rcu_sched_state); |
| 605 | } |
| 606 | EXPORT_SYMBOL_GPL(rcu_sched_force_quiescent_state); |
| 607 | |
| 608 | /* |
Paul E. McKenney | afea227 | 2014-03-12 07:10:41 -0700 | [diff] [blame] | 609 | * Show the state of the grace-period kthreads. |
| 610 | */ |
| 611 | void show_rcu_gp_kthreads(void) |
| 612 | { |
| 613 | struct rcu_state *rsp; |
| 614 | |
| 615 | for_each_rcu_flavor(rsp) { |
| 616 | pr_info("%s: wait state: %d ->state: %#lx\n", |
| 617 | rsp->name, rsp->gp_state, rsp->gp_kthread->state); |
| 618 | /* sched_show_task(rsp->gp_kthread); */ |
| 619 | } |
| 620 | } |
| 621 | EXPORT_SYMBOL_GPL(show_rcu_gp_kthreads); |
| 622 | |
| 623 | /* |
Paul E. McKenney | 4a29865 | 2011-04-03 21:33:51 -0700 | [diff] [blame] | 624 | * Record the number of times rcutorture tests have been initiated and |
| 625 | * terminated. This information allows the debugfs tracing stats to be |
| 626 | * correlated to the rcutorture messages, even when the rcutorture module |
| 627 | * is being repeatedly loaded and unloaded. In other words, we cannot |
| 628 | * store this state in rcutorture itself. |
| 629 | */ |
| 630 | void rcutorture_record_test_transition(void) |
| 631 | { |
| 632 | rcutorture_testseq++; |
| 633 | rcutorture_vernum = 0; |
| 634 | } |
| 635 | EXPORT_SYMBOL_GPL(rcutorture_record_test_transition); |
| 636 | |
| 637 | /* |
Paul E. McKenney | ad0dc7f | 2014-02-19 10:51:42 -0800 | [diff] [blame] | 638 | * Send along grace-period-related data for rcutorture diagnostics. |
| 639 | */ |
| 640 | void rcutorture_get_gp_data(enum rcutorture_type test_type, int *flags, |
| 641 | unsigned long *gpnum, unsigned long *completed) |
| 642 | { |
| 643 | struct rcu_state *rsp = NULL; |
| 644 | |
| 645 | switch (test_type) { |
| 646 | case RCU_FLAVOR: |
Uma Sharma | e534165 | 2014-03-23 22:32:09 -0700 | [diff] [blame] | 647 | rsp = rcu_state_p; |
Paul E. McKenney | ad0dc7f | 2014-02-19 10:51:42 -0800 | [diff] [blame] | 648 | break; |
| 649 | case RCU_BH_FLAVOR: |
| 650 | rsp = &rcu_bh_state; |
| 651 | break; |
| 652 | case RCU_SCHED_FLAVOR: |
| 653 | rsp = &rcu_sched_state; |
| 654 | break; |
| 655 | default: |
| 656 | break; |
| 657 | } |
Paul E. McKenney | 7f6733c | 2017-04-18 17:17:35 -0700 | [diff] [blame] | 658 | if (rsp == NULL) |
Paul E. McKenney | ad0dc7f | 2014-02-19 10:51:42 -0800 | [diff] [blame] | 659 | return; |
Paul E. McKenney | 7f6733c | 2017-04-18 17:17:35 -0700 | [diff] [blame] | 660 | *flags = READ_ONCE(rsp->gp_flags); |
| 661 | *gpnum = READ_ONCE(rsp->gpnum); |
| 662 | *completed = READ_ONCE(rsp->completed); |
Paul E. McKenney | ad0dc7f | 2014-02-19 10:51:42 -0800 | [diff] [blame] | 663 | } |
| 664 | EXPORT_SYMBOL_GPL(rcutorture_get_gp_data); |
| 665 | |
| 666 | /* |
Paul E. McKenney | 4a29865 | 2011-04-03 21:33:51 -0700 | [diff] [blame] | 667 | * Record the number of writer passes through the current rcutorture test. |
| 668 | * This is also used to correlate debugfs tracing stats with the rcutorture |
| 669 | * messages. |
| 670 | */ |
| 671 | void rcutorture_record_progress(unsigned long vernum) |
| 672 | { |
| 673 | rcutorture_vernum++; |
| 674 | } |
| 675 | EXPORT_SYMBOL_GPL(rcutorture_record_progress); |
| 676 | |
| 677 | /* |
Paul E. McKenney | 365187f | 2014-03-10 10:55:52 -0700 | [diff] [blame] | 678 | * Return the root node of the specified rcu_state structure. |
| 679 | */ |
| 680 | static struct rcu_node *rcu_get_root(struct rcu_state *rsp) |
| 681 | { |
| 682 | return &rsp->node[0]; |
| 683 | } |
| 684 | |
| 685 | /* |
Paul E. McKenney | 215bba9 | 2017-10-05 16:37:03 -0700 | [diff] [blame] | 686 | * Enter an RCU extended quiescent state, which can be either the |
| 687 | * idle loop or adaptive-tickless usermode execution. |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 688 | * |
Paul E. McKenney | 215bba9 | 2017-10-05 16:37:03 -0700 | [diff] [blame] | 689 | * We crowbar the ->dynticks_nmi_nesting field to zero to allow for |
| 690 | * the possibility of usermode upcalls having messed up our count |
| 691 | * of interrupt nesting level during the prior busy period. |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 692 | */ |
Paul E. McKenney | 215bba9 | 2017-10-05 16:37:03 -0700 | [diff] [blame] | 693 | static void rcu_eqs_enter(bool user) |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 694 | { |
Paul E. McKenney | 96d3fd0 | 2013-10-04 14:33:34 -0700 | [diff] [blame] | 695 | struct rcu_state *rsp; |
| 696 | struct rcu_data *rdp; |
Paul E. McKenney | 215bba9 | 2017-10-05 16:37:03 -0700 | [diff] [blame] | 697 | struct rcu_dynticks *rdtp; |
| 698 | |
| 699 | rdtp = this_cpu_ptr(&rcu_dynticks); |
| 700 | WRITE_ONCE(rdtp->dynticks_nmi_nesting, 0); |
| 701 | WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && |
| 702 | rdtp->dynticks_nesting == 0); |
| 703 | if (rdtp->dynticks_nesting != 1) { |
| 704 | rdtp->dynticks_nesting--; |
| 705 | return; |
| 706 | } |
Paul E. McKenney | 96d3fd0 | 2013-10-04 14:33:34 -0700 | [diff] [blame] | 707 | |
Frederic Weisbecker | b04db8e | 2017-11-06 16:01:30 +0100 | [diff] [blame] | 708 | lockdep_assert_irqs_disabled(); |
Paul E. McKenney | dec9890 | 2017-10-04 16:24:29 -0700 | [diff] [blame] | 709 | trace_rcu_dyntick(TPS("Start"), rdtp->dynticks_nesting, 0, rdtp->dynticks); |
Paul E. McKenney | e68bbb2 | 2017-10-05 19:55:31 -0700 | [diff] [blame] | 710 | WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current)); |
Paul E. McKenney | 96d3fd0 | 2013-10-04 14:33:34 -0700 | [diff] [blame] | 711 | for_each_rcu_flavor(rsp) { |
| 712 | rdp = this_cpu_ptr(rsp->rda); |
| 713 | do_nocb_deferred_wakeup(rdp); |
| 714 | } |
Paul E. McKenney | 198bbf8 | 2014-10-22 15:03:43 -0700 | [diff] [blame] | 715 | rcu_prepare_for_idle(); |
Paul E. McKenney | 2342172 | 2017-10-05 15:03:10 -0700 | [diff] [blame] | 716 | WRITE_ONCE(rdtp->dynticks_nesting, 0); /* Avoid irq-access tearing. */ |
Paul E. McKenney | 844ccdd | 2017-10-03 16:51:47 -0700 | [diff] [blame] | 717 | rcu_dynticks_eqs_enter(); |
Paul E. McKenney | 176f8f7 | 2014-08-04 17:43:50 -0700 | [diff] [blame] | 718 | rcu_dynticks_task_enter(); |
Frederic Weisbecker | adf5091 | 2012-06-28 11:20:21 -0700 | [diff] [blame] | 719 | } |
| 720 | |
Paul E. McKenney | 9b2e4f1 | 2011-09-30 12:10:22 -0700 | [diff] [blame] | 721 | /** |
| 722 | * rcu_idle_enter - inform RCU that current CPU is entering idle |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 723 | * |
Paul E. McKenney | 9b2e4f1 | 2011-09-30 12:10:22 -0700 | [diff] [blame] | 724 | * Enter idle mode, in other words, -leave- the mode in which RCU |
| 725 | * read-side critical sections can occur. (Though RCU read-side |
| 726 | * critical sections can occur in irq handlers in idle, a possibility |
| 727 | * handled by irq_enter() and irq_exit().) |
| 728 | * |
Paul E. McKenney | c0da313 | 2017-09-22 09:58:47 -0700 | [diff] [blame] | 729 | * If you add or remove a call to rcu_idle_enter(), be sure to test with |
| 730 | * CONFIG_RCU_EQS_DEBUG=y. |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 731 | */ |
Paul E. McKenney | 9b2e4f1 | 2011-09-30 12:10:22 -0700 | [diff] [blame] | 732 | void rcu_idle_enter(void) |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 733 | { |
Frederic Weisbecker | b04db8e | 2017-11-06 16:01:30 +0100 | [diff] [blame] | 734 | lockdep_assert_irqs_disabled(); |
Paul E. McKenney | cb349ca | 2012-09-04 17:35:31 -0700 | [diff] [blame] | 735 | rcu_eqs_enter(false); |
Paul E. McKenney | 9b2e4f1 | 2011-09-30 12:10:22 -0700 | [diff] [blame] | 736 | } |
| 737 | |
Paul E. McKenney | d1ec4c3 | 2015-05-13 10:41:58 -0700 | [diff] [blame] | 738 | #ifdef CONFIG_NO_HZ_FULL |
Paul E. McKenney | 9b2e4f1 | 2011-09-30 12:10:22 -0700 | [diff] [blame] | 739 | /** |
Frederic Weisbecker | adf5091 | 2012-06-28 11:20:21 -0700 | [diff] [blame] | 740 | * rcu_user_enter - inform RCU that we are resuming userspace. |
| 741 | * |
| 742 | * Enter RCU idle mode right before resuming userspace. No use of RCU |
| 743 | * is permitted between this call and rcu_user_exit(). This way the |
| 744 | * CPU doesn't need to maintain the tick for RCU maintenance purposes |
| 745 | * when the CPU runs in userspace. |
Paul E. McKenney | c0da313 | 2017-09-22 09:58:47 -0700 | [diff] [blame] | 746 | * |
| 747 | * If you add or remove a call to rcu_user_enter(), be sure to test with |
| 748 | * CONFIG_RCU_EQS_DEBUG=y. |
Frederic Weisbecker | adf5091 | 2012-06-28 11:20:21 -0700 | [diff] [blame] | 749 | */ |
| 750 | void rcu_user_enter(void) |
| 751 | { |
Frederic Weisbecker | b04db8e | 2017-11-06 16:01:30 +0100 | [diff] [blame] | 752 | lockdep_assert_irqs_disabled(); |
Paul E. McKenney | d4db30a | 2017-07-12 09:03:35 -0700 | [diff] [blame] | 753 | rcu_eqs_enter(true); |
Frederic Weisbecker | adf5091 | 2012-06-28 11:20:21 -0700 | [diff] [blame] | 754 | } |
Paul E. McKenney | d1ec4c3 | 2015-05-13 10:41:58 -0700 | [diff] [blame] | 755 | #endif /* CONFIG_NO_HZ_FULL */ |
Frederic Weisbecker | 19dd1591 | 2012-06-04 16:42:35 -0700 | [diff] [blame] | 756 | |
| 757 | /** |
Paul E. McKenney | fd581a9 | 2017-10-02 21:56:20 -0700 | [diff] [blame] | 758 | * rcu_nmi_exit - inform RCU of exit from NMI context |
| 759 | * |
| 760 | * If we are returning from the outermost NMI handler that interrupted an |
| 761 | * RCU-idle period, update rdtp->dynticks and rdtp->dynticks_nmi_nesting |
| 762 | * to let the RCU grace-period handling know that the CPU is back to |
| 763 | * being RCU-idle. |
| 764 | * |
| 765 | * If you add or remove a call to rcu_nmi_exit(), be sure to test |
| 766 | * with CONFIG_RCU_EQS_DEBUG=y. |
| 767 | */ |
| 768 | void rcu_nmi_exit(void) |
| 769 | { |
| 770 | struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks); |
| 771 | |
| 772 | /* |
| 773 | * Check for ->dynticks_nmi_nesting underflow and bad ->dynticks. |
| 774 | * (We are exiting an NMI handler, so RCU better be paying attention |
| 775 | * to us!) |
| 776 | */ |
| 777 | WARN_ON_ONCE(rdtp->dynticks_nmi_nesting <= 0); |
| 778 | WARN_ON_ONCE(rcu_dynticks_curr_cpu_in_eqs()); |
| 779 | |
| 780 | /* |
| 781 | * If the nesting level is not 1, the CPU wasn't RCU-idle, so |
| 782 | * leave it in non-RCU-idle state. |
| 783 | */ |
| 784 | if (rdtp->dynticks_nmi_nesting != 1) { |
Paul E. McKenney | dec9890 | 2017-10-04 16:24:29 -0700 | [diff] [blame] | 785 | trace_rcu_dyntick(TPS("--="), rdtp->dynticks_nmi_nesting, rdtp->dynticks_nmi_nesting - 2, rdtp->dynticks); |
Paul E. McKenney | fd581a9 | 2017-10-02 21:56:20 -0700 | [diff] [blame] | 786 | WRITE_ONCE(rdtp->dynticks_nmi_nesting, /* No store tearing. */ |
| 787 | rdtp->dynticks_nmi_nesting - 2); |
| 788 | return; |
| 789 | } |
| 790 | |
| 791 | /* This NMI interrupted an RCU-idle CPU, restore RCU-idleness. */ |
Paul E. McKenney | dec9890 | 2017-10-04 16:24:29 -0700 | [diff] [blame] | 792 | trace_rcu_dyntick(TPS("Startirq"), rdtp->dynticks_nmi_nesting, 0, rdtp->dynticks); |
Paul E. McKenney | fd581a9 | 2017-10-02 21:56:20 -0700 | [diff] [blame] | 793 | WRITE_ONCE(rdtp->dynticks_nmi_nesting, 0); /* Avoid store tearing. */ |
| 794 | rcu_dynticks_eqs_enter(); |
| 795 | } |
| 796 | |
| 797 | /** |
Paul E. McKenney | 9b2e4f1 | 2011-09-30 12:10:22 -0700 | [diff] [blame] | 798 | * rcu_irq_exit - inform RCU that current CPU is exiting irq towards idle |
| 799 | * |
| 800 | * Exit from an interrupt handler, which might possibly result in entering |
| 801 | * idle mode, in other words, leaving the mode in which read-side critical |
Paul E. McKenney | 7c9906c | 2015-10-31 00:59:01 -0700 | [diff] [blame] | 802 | * sections can occur. The caller must have disabled interrupts. |
Paul E. McKenney | 9b2e4f1 | 2011-09-30 12:10:22 -0700 | [diff] [blame] | 803 | * |
| 804 | * This code assumes that the idle loop never does anything that might |
| 805 | * result in unbalanced calls to irq_enter() and irq_exit(). If your |
Paul E. McKenney | 58721f5 | 2017-10-03 10:42:22 -0700 | [diff] [blame] | 806 | * architecture's idle loop violates this assumption, RCU will give you what |
| 807 | * you deserve, good and hard. But very infrequently and irreproducibly. |
Paul E. McKenney | 9b2e4f1 | 2011-09-30 12:10:22 -0700 | [diff] [blame] | 808 | * |
| 809 | * Use things like work queues to work around this limitation. |
| 810 | * |
| 811 | * You have been warned. |
Paul E. McKenney | c0da313 | 2017-09-22 09:58:47 -0700 | [diff] [blame] | 812 | * |
| 813 | * If you add or remove a call to rcu_irq_exit(), be sure to test with |
| 814 | * CONFIG_RCU_EQS_DEBUG=y. |
Paul E. McKenney | 9b2e4f1 | 2011-09-30 12:10:22 -0700 | [diff] [blame] | 815 | */ |
| 816 | void rcu_irq_exit(void) |
| 817 | { |
Paul E. McKenney | 58721f5 | 2017-10-03 10:42:22 -0700 | [diff] [blame] | 818 | struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks); |
Paul E. McKenney | 9b2e4f1 | 2011-09-30 12:10:22 -0700 | [diff] [blame] | 819 | |
Frederic Weisbecker | b04db8e | 2017-11-06 16:01:30 +0100 | [diff] [blame] | 820 | lockdep_assert_irqs_disabled(); |
Paul E. McKenney | 58721f5 | 2017-10-03 10:42:22 -0700 | [diff] [blame] | 821 | if (rdtp->dynticks_nmi_nesting == 1) |
| 822 | rcu_prepare_for_idle(); |
| 823 | rcu_nmi_exit(); |
| 824 | if (rdtp->dynticks_nmi_nesting == 0) |
| 825 | rcu_dynticks_task_enter(); |
Paul E. McKenney | 7c9906c | 2015-10-31 00:59:01 -0700 | [diff] [blame] | 826 | } |
| 827 | |
| 828 | /* |
| 829 | * Wrapper for rcu_irq_exit() where interrupts are enabled. |
Paul E. McKenney | c0da313 | 2017-09-22 09:58:47 -0700 | [diff] [blame] | 830 | * |
| 831 | * If you add or remove a call to rcu_irq_exit_irqson(), be sure to test |
| 832 | * with CONFIG_RCU_EQS_DEBUG=y. |
Paul E. McKenney | 7c9906c | 2015-10-31 00:59:01 -0700 | [diff] [blame] | 833 | */ |
| 834 | void rcu_irq_exit_irqson(void) |
| 835 | { |
| 836 | unsigned long flags; |
| 837 | |
| 838 | local_irq_save(flags); |
| 839 | rcu_irq_exit(); |
Paul E. McKenney | 9b2e4f1 | 2011-09-30 12:10:22 -0700 | [diff] [blame] | 840 | local_irq_restore(flags); |
| 841 | } |
| 842 | |
| 843 | /* |
Frederic Weisbecker | adf5091 | 2012-06-28 11:20:21 -0700 | [diff] [blame] | 844 | * Exit an RCU extended quiescent state, which can be either the |
| 845 | * idle loop or adaptive-tickless usermode execution. |
Paul E. McKenney | 51a1fd3 | 2017-10-03 14:43:40 -0700 | [diff] [blame] | 846 | * |
| 847 | * We crowbar the ->dynticks_nmi_nesting field to DYNTICK_IRQ_NONIDLE to |
| 848 | * allow for the possibility of usermode upcalls messing up our count of |
| 849 | * interrupt nesting level during the busy period that is just now starting. |
Frederic Weisbecker | adf5091 | 2012-06-28 11:20:21 -0700 | [diff] [blame] | 850 | */ |
| 851 | static void rcu_eqs_exit(bool user) |
| 852 | { |
Frederic Weisbecker | adf5091 | 2012-06-28 11:20:21 -0700 | [diff] [blame] | 853 | struct rcu_dynticks *rdtp; |
Paul E. McKenney | 84585aa | 2017-10-04 15:55:16 -0700 | [diff] [blame] | 854 | long oldval; |
Frederic Weisbecker | adf5091 | 2012-06-28 11:20:21 -0700 | [diff] [blame] | 855 | |
Frederic Weisbecker | b04db8e | 2017-11-06 16:01:30 +0100 | [diff] [blame] | 856 | lockdep_assert_irqs_disabled(); |
Christoph Lameter | c9d4b0a | 2013-08-31 13:34:10 -0700 | [diff] [blame] | 857 | rdtp = this_cpu_ptr(&rcu_dynticks); |
Frederic Weisbecker | adf5091 | 2012-06-28 11:20:21 -0700 | [diff] [blame] | 858 | oldval = rdtp->dynticks_nesting; |
Paul E. McKenney | 1ce46ee | 2015-05-05 23:04:22 -0700 | [diff] [blame] | 859 | WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && oldval < 0); |
Paul E. McKenney | 51a1fd3 | 2017-10-03 14:43:40 -0700 | [diff] [blame] | 860 | if (oldval) { |
| 861 | rdtp->dynticks_nesting++; |
Paul E. McKenney | 9dd238e | 2017-10-05 16:56:26 -0700 | [diff] [blame] | 862 | return; |
Paul E. McKenney | 3a592405 | 2013-10-04 18:48:55 -0700 | [diff] [blame] | 863 | } |
Paul E. McKenney | 9dd238e | 2017-10-05 16:56:26 -0700 | [diff] [blame] | 864 | rcu_dynticks_task_exit(); |
| 865 | rcu_dynticks_eqs_exit(); |
| 866 | rcu_cleanup_after_idle(); |
| 867 | trace_rcu_dyntick(TPS("End"), rdtp->dynticks_nesting, 1, rdtp->dynticks); |
Paul E. McKenney | e68bbb2 | 2017-10-05 19:55:31 -0700 | [diff] [blame] | 868 | WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current)); |
Paul E. McKenney | 9dd238e | 2017-10-05 16:56:26 -0700 | [diff] [blame] | 869 | WRITE_ONCE(rdtp->dynticks_nesting, 1); |
| 870 | WRITE_ONCE(rdtp->dynticks_nmi_nesting, DYNTICK_IRQ_NONIDLE); |
Frederic Weisbecker | adf5091 | 2012-06-28 11:20:21 -0700 | [diff] [blame] | 871 | } |
| 872 | |
Paul E. McKenney | 9b2e4f1 | 2011-09-30 12:10:22 -0700 | [diff] [blame] | 873 | /** |
| 874 | * rcu_idle_exit - inform RCU that current CPU is leaving idle |
| 875 | * |
| 876 | * Exit idle mode, in other words, -enter- the mode in which RCU |
| 877 | * read-side critical sections can occur. |
| 878 | * |
Paul E. McKenney | c0da313 | 2017-09-22 09:58:47 -0700 | [diff] [blame] | 879 | * If you add or remove a call to rcu_idle_exit(), be sure to test with |
| 880 | * CONFIG_RCU_EQS_DEBUG=y. |
Paul E. McKenney | 9b2e4f1 | 2011-09-30 12:10:22 -0700 | [diff] [blame] | 881 | */ |
| 882 | void rcu_idle_exit(void) |
| 883 | { |
Frederic Weisbecker | c5d900b | 2012-07-11 20:26:31 +0200 | [diff] [blame] | 884 | unsigned long flags; |
| 885 | |
| 886 | local_irq_save(flags); |
Paul E. McKenney | cb349ca | 2012-09-04 17:35:31 -0700 | [diff] [blame] | 887 | rcu_eqs_exit(false); |
Frederic Weisbecker | c5d900b | 2012-07-11 20:26:31 +0200 | [diff] [blame] | 888 | local_irq_restore(flags); |
Paul E. McKenney | 9b2e4f1 | 2011-09-30 12:10:22 -0700 | [diff] [blame] | 889 | } |
| 890 | |
Paul E. McKenney | d1ec4c3 | 2015-05-13 10:41:58 -0700 | [diff] [blame] | 891 | #ifdef CONFIG_NO_HZ_FULL |
Paul E. McKenney | 9b2e4f1 | 2011-09-30 12:10:22 -0700 | [diff] [blame] | 892 | /** |
Frederic Weisbecker | adf5091 | 2012-06-28 11:20:21 -0700 | [diff] [blame] | 893 | * rcu_user_exit - inform RCU that we are exiting userspace. |
| 894 | * |
| 895 | * Exit RCU idle mode while entering the kernel because it can |
| 896 | * run a RCU read side critical section anytime. |
Paul E. McKenney | c0da313 | 2017-09-22 09:58:47 -0700 | [diff] [blame] | 897 | * |
| 898 | * If you add or remove a call to rcu_user_exit(), be sure to test with |
| 899 | * CONFIG_RCU_EQS_DEBUG=y. |
Frederic Weisbecker | adf5091 | 2012-06-28 11:20:21 -0700 | [diff] [blame] | 900 | */ |
| 901 | void rcu_user_exit(void) |
| 902 | { |
Frederic Weisbecker | 91d1aa43 | 2012-11-27 19:33:25 +0100 | [diff] [blame] | 903 | rcu_eqs_exit(1); |
Frederic Weisbecker | adf5091 | 2012-06-28 11:20:21 -0700 | [diff] [blame] | 904 | } |
Paul E. McKenney | d1ec4c3 | 2015-05-13 10:41:58 -0700 | [diff] [blame] | 905 | #endif /* CONFIG_NO_HZ_FULL */ |
Frederic Weisbecker | 19dd1591 | 2012-06-04 16:42:35 -0700 | [diff] [blame] | 906 | |
| 907 | /** |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 908 | * rcu_nmi_enter - inform RCU of entry to NMI context |
| 909 | * |
Paul E. McKenney | 734d168 | 2014-11-21 14:45:12 -0800 | [diff] [blame] | 910 | * If the CPU was idle from RCU's viewpoint, update rdtp->dynticks and |
| 911 | * rdtp->dynticks_nmi_nesting to let the RCU grace-period handling know |
| 912 | * that the CPU is active. This implementation permits nested NMIs, as |
| 913 | * long as the nesting level does not overflow an int. (You will probably |
| 914 | * run out of stack space first.) |
Paul E. McKenney | c0da313 | 2017-09-22 09:58:47 -0700 | [diff] [blame] | 915 | * |
| 916 | * If you add or remove a call to rcu_nmi_enter(), be sure to test |
| 917 | * with CONFIG_RCU_EQS_DEBUG=y. |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 918 | */ |
| 919 | void rcu_nmi_enter(void) |
| 920 | { |
Christoph Lameter | c9d4b0a | 2013-08-31 13:34:10 -0700 | [diff] [blame] | 921 | struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks); |
Paul E. McKenney | 84585aa | 2017-10-04 15:55:16 -0700 | [diff] [blame] | 922 | long incby = 2; |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 923 | |
Paul E. McKenney | 734d168 | 2014-11-21 14:45:12 -0800 | [diff] [blame] | 924 | /* Complain about underflow. */ |
| 925 | WARN_ON_ONCE(rdtp->dynticks_nmi_nesting < 0); |
| 926 | |
| 927 | /* |
| 928 | * If idle from RCU viewpoint, atomically increment ->dynticks |
| 929 | * to mark non-idle and increment ->dynticks_nmi_nesting by one. |
| 930 | * Otherwise, increment ->dynticks_nmi_nesting by two. This means |
| 931 | * if ->dynticks_nmi_nesting is equal to one, we are guaranteed |
| 932 | * to be in the outermost NMI handler that interrupted an RCU-idle |
| 933 | * period (observation due to Andy Lutomirski). |
| 934 | */ |
Paul E. McKenney | 02a5c550 | 2016-11-02 17:25:06 -0700 | [diff] [blame] | 935 | if (rcu_dynticks_curr_cpu_in_eqs()) { |
Paul E. McKenney | 2625d46 | 2016-11-02 14:23:30 -0700 | [diff] [blame] | 936 | rcu_dynticks_eqs_exit(); |
Paul E. McKenney | 734d168 | 2014-11-21 14:45:12 -0800 | [diff] [blame] | 937 | incby = 1; |
| 938 | } |
Paul E. McKenney | bd2b879 | 2017-10-04 12:29:01 -0700 | [diff] [blame] | 939 | trace_rcu_dyntick(incby == 1 ? TPS("Endirq") : TPS("++="), |
| 940 | rdtp->dynticks_nmi_nesting, |
Paul E. McKenney | dec9890 | 2017-10-04 16:24:29 -0700 | [diff] [blame] | 941 | rdtp->dynticks_nmi_nesting + incby, rdtp->dynticks); |
Paul E. McKenney | fd581a9 | 2017-10-02 21:56:20 -0700 | [diff] [blame] | 942 | WRITE_ONCE(rdtp->dynticks_nmi_nesting, /* Prevent store tearing. */ |
| 943 | rdtp->dynticks_nmi_nesting + incby); |
Paul E. McKenney | 734d168 | 2014-11-21 14:45:12 -0800 | [diff] [blame] | 944 | barrier(); |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 945 | } |
| 946 | |
| 947 | /** |
Paul E. McKenney | 3424069 | 2011-10-03 11:38:52 -0700 | [diff] [blame] | 948 | * rcu_irq_enter - inform RCU that current CPU is entering irq away from idle |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 949 | * |
Paul E. McKenney | 3424069 | 2011-10-03 11:38:52 -0700 | [diff] [blame] | 950 | * Enter an interrupt handler, which might possibly result in exiting |
| 951 | * idle mode, in other words, entering the mode in which read-side critical |
| 952 | * sections can occur. The caller must have disabled interrupts. |
Paul E. McKenney | c0da313 | 2017-09-22 09:58:47 -0700 | [diff] [blame] | 953 | * |
Paul E. McKenney | 9b2e4f1 | 2011-09-30 12:10:22 -0700 | [diff] [blame] | 954 | * Note that the Linux kernel is fully capable of entering an interrupt |
Paul E. McKenney | 58721f5 | 2017-10-03 10:42:22 -0700 | [diff] [blame] | 955 | * handler that it never exits, for example when doing upcalls to user mode! |
| 956 | * This code assumes that the idle loop never does upcalls to user mode. |
| 957 | * If your architecture's idle loop does do upcalls to user mode (or does |
| 958 | * anything else that results in unbalanced calls to the irq_enter() and |
| 959 | * irq_exit() functions), RCU will give you what you deserve, good and hard. |
| 960 | * But very infrequently and irreproducibly. |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 961 | * |
| 962 | * Use things like work queues to work around this limitation. |
| 963 | * |
| 964 | * You have been warned. |
| 965 | * |
| 966 | * If you add or remove a call to rcu_irq_enter(), be sure to test with |
| 967 | * CONFIG_RCU_EQS_DEBUG=y. |
Paul E. McKenney | 23b5c8f | 2010-09-07 10:38:22 -0700 | [diff] [blame] | 968 | */ |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 969 | void rcu_irq_enter(void) |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 970 | { |
Christoph Lameter | c9d4b0a | 2013-08-31 13:34:10 -0700 | [diff] [blame] | 971 | struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks); |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 972 | |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 973 | lockdep_assert_irqs_disabled(); |
Paul E. McKenney | 58721f5 | 2017-10-03 10:42:22 -0700 | [diff] [blame] | 974 | if (rdtp->dynticks_nmi_nesting == 0) |
| 975 | rcu_dynticks_task_exit(); |
| 976 | rcu_nmi_enter(); |
| 977 | if (rdtp->dynticks_nmi_nesting == 1) |
| 978 | rcu_cleanup_after_idle(); |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 979 | } |
Paul E. McKenney | 734d168 | 2014-11-21 14:45:12 -0800 | [diff] [blame] | 980 | |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 981 | /* |
| 982 | * Wrapper for rcu_irq_enter() where interrupts are enabled. |
| 983 | * |
| 984 | * If you add or remove a call to rcu_irq_enter_irqson(), be sure to test |
| 985 | * with CONFIG_RCU_EQS_DEBUG=y. |
| 986 | */ |
| 987 | void rcu_irq_enter_irqson(void) |
| 988 | { |
| 989 | unsigned long flags; |
Paul E. McKenney | 734d168 | 2014-11-21 14:45:12 -0800 | [diff] [blame] | 990 | |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 991 | local_irq_save(flags); |
| 992 | rcu_irq_enter(); |
| 993 | local_irq_restore(flags); |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 994 | } |
| 995 | |
| 996 | /** |
Paul E. McKenney | 5c173eb | 2013-09-13 17:20:11 -0700 | [diff] [blame] | 997 | * rcu_is_watching - see if RCU thinks that the current CPU is idle |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 998 | * |
Paul E. McKenney | 791875d | 2017-05-03 11:06:05 -0700 | [diff] [blame] | 999 | * Return true if RCU is watching the running CPU, which means that this |
| 1000 | * CPU can safely enter RCU read-side critical sections. In other words, |
| 1001 | * if the current CPU is in its idle loop and is neither in an interrupt |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 1002 | * or NMI handler, return true. |
| 1003 | */ |
Linus Torvalds | b29c830 | 2013-11-16 12:23:18 -0800 | [diff] [blame] | 1004 | bool notrace rcu_is_watching(void) |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 1005 | { |
Pranith Kumar | f534ed1 | 2014-07-08 18:26:11 -0400 | [diff] [blame] | 1006 | bool ret; |
Paul E. McKenney | 9b2e4f1 | 2011-09-30 12:10:22 -0700 | [diff] [blame] | 1007 | |
Alexei Starovoitov | 46f00d1 | 2015-06-16 10:35:18 -0700 | [diff] [blame] | 1008 | preempt_disable_notrace(); |
Paul E. McKenney | 791875d | 2017-05-03 11:06:05 -0700 | [diff] [blame] | 1009 | ret = !rcu_dynticks_curr_cpu_in_eqs(); |
Alexei Starovoitov | 46f00d1 | 2015-06-16 10:35:18 -0700 | [diff] [blame] | 1010 | preempt_enable_notrace(); |
Paul E. McKenney | 9b2e4f1 | 2011-09-30 12:10:22 -0700 | [diff] [blame] | 1011 | return ret; |
| 1012 | } |
Paul E. McKenney | 5c173eb | 2013-09-13 17:20:11 -0700 | [diff] [blame] | 1013 | EXPORT_SYMBOL_GPL(rcu_is_watching); |
Paul E. McKenney | 9b2e4f1 | 2011-09-30 12:10:22 -0700 | [diff] [blame] | 1014 | |
Paul E. McKenney | bcbfdd0 | 2017-04-11 15:50:41 -0700 | [diff] [blame] | 1015 | /* |
| 1016 | * If a holdout task is actually running, request an urgent quiescent |
| 1017 | * state from its CPU. This is unsynchronized, so migrations can cause |
| 1018 | * the request to go to the wrong CPU. Which is OK, all that will happen |
| 1019 | * is that the CPU's next context switch will be a bit slower and next |
| 1020 | * time around this task will generate another request. |
| 1021 | */ |
| 1022 | void rcu_request_urgent_qs_task(struct task_struct *t) |
| 1023 | { |
| 1024 | int cpu; |
| 1025 | |
| 1026 | barrier(); |
| 1027 | cpu = task_cpu(t); |
| 1028 | if (!task_curr(t)) |
| 1029 | return; /* This task is not running on that CPU. */ |
| 1030 | smp_store_release(per_cpu_ptr(&rcu_dynticks.rcu_urgent_qs, cpu), true); |
| 1031 | } |
| 1032 | |
Paul E. McKenney | 62fde6e | 2012-05-22 22:10:24 -0700 | [diff] [blame] | 1033 | #if defined(CONFIG_PROVE_RCU) && defined(CONFIG_HOTPLUG_CPU) |
Paul E. McKenney | c0d6d01 | 2012-01-23 12:41:26 -0800 | [diff] [blame] | 1034 | |
| 1035 | /* |
| 1036 | * Is the current CPU online? Disable preemption to avoid false positives |
| 1037 | * that could otherwise happen due to the current CPU number being sampled, |
| 1038 | * this task being preempted, its old CPU being taken offline, resuming |
| 1039 | * on some other CPU, then determining that its old CPU is now offline. |
| 1040 | * It is OK to use RCU on an offline processor during initial boot, hence |
Paul E. McKenney | 2036d94 | 2012-01-30 17:02:47 -0800 | [diff] [blame] | 1041 | * the check for rcu_scheduler_fully_active. Note also that it is OK |
| 1042 | * for a CPU coming online to use RCU for one jiffy prior to marking itself |
| 1043 | * online in the cpu_online_mask. Similarly, it is OK for a CPU going |
| 1044 | * offline to continue to use RCU for one jiffy after marking itself |
| 1045 | * offline in the cpu_online_mask. This leniency is necessary given the |
| 1046 | * non-atomic nature of the online and offline processing, for example, |
Thomas Gleixner | 4df8374 | 2016-07-13 17:17:03 +0000 | [diff] [blame] | 1047 | * the fact that a CPU enters the scheduler after completing the teardown |
| 1048 | * of the CPU. |
Paul E. McKenney | 2036d94 | 2012-01-30 17:02:47 -0800 | [diff] [blame] | 1049 | * |
Thomas Gleixner | 4df8374 | 2016-07-13 17:17:03 +0000 | [diff] [blame] | 1050 | * This is also why RCU internally marks CPUs online during in the |
| 1051 | * preparation phase and offline after the CPU has been taken down. |
Paul E. McKenney | c0d6d01 | 2012-01-23 12:41:26 -0800 | [diff] [blame] | 1052 | * |
| 1053 | * Disable checking if in an NMI handler because we cannot safely report |
| 1054 | * errors from NMI handlers anyway. |
| 1055 | */ |
| 1056 | bool rcu_lockdep_current_cpu_online(void) |
| 1057 | { |
Paul E. McKenney | 2036d94 | 2012-01-30 17:02:47 -0800 | [diff] [blame] | 1058 | struct rcu_data *rdp; |
| 1059 | struct rcu_node *rnp; |
Paul E. McKenney | c0d6d01 | 2012-01-23 12:41:26 -0800 | [diff] [blame] | 1060 | bool ret; |
| 1061 | |
| 1062 | if (in_nmi()) |
Fengguang Wu | f6f7ee9 | 2013-10-10 11:08:33 -0700 | [diff] [blame] | 1063 | return true; |
Paul E. McKenney | c0d6d01 | 2012-01-23 12:41:26 -0800 | [diff] [blame] | 1064 | preempt_disable(); |
Christoph Lameter | c9d4b0a | 2013-08-31 13:34:10 -0700 | [diff] [blame] | 1065 | rdp = this_cpu_ptr(&rcu_sched_data); |
Paul E. McKenney | 2036d94 | 2012-01-30 17:02:47 -0800 | [diff] [blame] | 1066 | rnp = rdp->mynode; |
Paul E. McKenney | 0aa04b0 | 2015-01-23 21:52:37 -0800 | [diff] [blame] | 1067 | ret = (rdp->grpmask & rcu_rnp_online_cpus(rnp)) || |
Paul E. McKenney | c0d6d01 | 2012-01-23 12:41:26 -0800 | [diff] [blame] | 1068 | !rcu_scheduler_fully_active; |
| 1069 | preempt_enable(); |
| 1070 | return ret; |
| 1071 | } |
| 1072 | EXPORT_SYMBOL_GPL(rcu_lockdep_current_cpu_online); |
| 1073 | |
Paul E. McKenney | 62fde6e | 2012-05-22 22:10:24 -0700 | [diff] [blame] | 1074 | #endif /* #if defined(CONFIG_PROVE_RCU) && defined(CONFIG_HOTPLUG_CPU) */ |
Paul E. McKenney | 9b2e4f1 | 2011-09-30 12:10:22 -0700 | [diff] [blame] | 1075 | |
| 1076 | /** |
| 1077 | * rcu_is_cpu_rrupt_from_idle - see if idle or immediately interrupted from idle |
| 1078 | * |
| 1079 | * If the current CPU is idle or running at a first-level (not nested) |
| 1080 | * interrupt from idle, return true. The caller must have at least |
| 1081 | * disabled preemption. |
| 1082 | */ |
Josh Triplett | 62e3cb1 | 2012-11-20 09:55:26 -0800 | [diff] [blame] | 1083 | static int rcu_is_cpu_rrupt_from_idle(void) |
Paul E. McKenney | 9b2e4f1 | 2011-09-30 12:10:22 -0700 | [diff] [blame] | 1084 | { |
Paul E. McKenney | 51a1fd3 | 2017-10-03 14:43:40 -0700 | [diff] [blame] | 1085 | return __this_cpu_read(rcu_dynticks.dynticks_nesting) <= 0 && |
| 1086 | __this_cpu_read(rcu_dynticks.dynticks_nmi_nesting) <= 1; |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 1087 | } |
| 1088 | |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 1089 | /* |
Paul E. McKenney | 9b9500d | 2017-08-17 17:05:59 -0700 | [diff] [blame] | 1090 | * We are reporting a quiescent state on behalf of some other CPU, so |
| 1091 | * it is our responsibility to check for and handle potential overflow |
| 1092 | * of the rcu_node ->gpnum counter with respect to the rcu_data counters. |
| 1093 | * After all, the CPU might be in deep idle state, and thus executing no |
| 1094 | * code whatsoever. |
| 1095 | */ |
| 1096 | static void rcu_gpnum_ovf(struct rcu_node *rnp, struct rcu_data *rdp) |
| 1097 | { |
Matthew Wilcox | a32e01e | 2018-01-17 06:24:30 -0800 | [diff] [blame] | 1098 | raw_lockdep_assert_held_rcu_node(rnp); |
Paul E. McKenney | 9b9500d | 2017-08-17 17:05:59 -0700 | [diff] [blame] | 1099 | if (ULONG_CMP_LT(READ_ONCE(rdp->gpnum) + ULONG_MAX / 4, rnp->gpnum)) |
| 1100 | WRITE_ONCE(rdp->gpwrap, true); |
| 1101 | if (ULONG_CMP_LT(rdp->rcu_iw_gpnum + ULONG_MAX / 4, rnp->gpnum)) |
| 1102 | rdp->rcu_iw_gpnum = rnp->gpnum + ULONG_MAX / 4; |
| 1103 | } |
| 1104 | |
| 1105 | /* |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 1106 | * Snapshot the specified CPU's dynticks counter so that we can later |
| 1107 | * credit them with an implicit quiescent state. Return 1 if this CPU |
Paul E. McKenney | 1eba8f8 | 2009-09-23 09:50:42 -0700 | [diff] [blame] | 1108 | * is in dynticks idle mode, which is an extended quiescent state. |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 1109 | */ |
Paul E. McKenney | fe5ac72 | 2017-05-11 11:26:22 -0700 | [diff] [blame] | 1110 | static int dyntick_save_progress_counter(struct rcu_data *rdp) |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 1111 | { |
Paul E. McKenney | 8b2f63a | 2016-11-02 14:12:05 -0700 | [diff] [blame] | 1112 | rdp->dynticks_snap = rcu_dynticks_snap(rdp->dynticks); |
Paul E. McKenney | 02a5c550 | 2016-11-02 17:25:06 -0700 | [diff] [blame] | 1113 | if (rcu_dynticks_in_eqs(rdp->dynticks_snap)) { |
Andreea-Cristina Bernat | 7941dbd | 2014-03-17 18:33:28 +0200 | [diff] [blame] | 1114 | trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("dti")); |
Paul E. McKenney | 9b9500d | 2017-08-17 17:05:59 -0700 | [diff] [blame] | 1115 | rcu_gpnum_ovf(rdp->mynode, rdp); |
Paul E. McKenney | 23a9bac | 2015-12-13 08:57:10 -0800 | [diff] [blame] | 1116 | return 1; |
Andreea-Cristina Bernat | 7941dbd | 2014-03-17 18:33:28 +0200 | [diff] [blame] | 1117 | } |
Paul E. McKenney | 23a9bac | 2015-12-13 08:57:10 -0800 | [diff] [blame] | 1118 | return 0; |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 1119 | } |
| 1120 | |
| 1121 | /* |
Paul E. McKenney | 9b9500d | 2017-08-17 17:05:59 -0700 | [diff] [blame] | 1122 | * Handler for the irq_work request posted when a grace period has |
| 1123 | * gone on for too long, but not yet long enough for an RCU CPU |
| 1124 | * stall warning. Set state appropriately, but just complain if |
| 1125 | * there is unexpected state on entry. |
| 1126 | */ |
| 1127 | static void rcu_iw_handler(struct irq_work *iwp) |
| 1128 | { |
| 1129 | struct rcu_data *rdp; |
| 1130 | struct rcu_node *rnp; |
| 1131 | |
| 1132 | rdp = container_of(iwp, struct rcu_data, rcu_iw); |
| 1133 | rnp = rdp->mynode; |
| 1134 | raw_spin_lock_rcu_node(rnp); |
| 1135 | if (!WARN_ON_ONCE(!rdp->rcu_iw_pending)) { |
| 1136 | rdp->rcu_iw_gpnum = rnp->gpnum; |
| 1137 | rdp->rcu_iw_pending = false; |
| 1138 | } |
| 1139 | raw_spin_unlock_rcu_node(rnp); |
| 1140 | } |
| 1141 | |
| 1142 | /* |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 1143 | * Return true if the specified CPU has passed through a quiescent |
| 1144 | * state by virtue of being in or having passed through an dynticks |
| 1145 | * idle state since the last call to dyntick_save_progress_counter() |
Paul E. McKenney | a82dcc7 | 2012-08-01 14:29:20 -0700 | [diff] [blame] | 1146 | * for this same CPU, or by virtue of having been offline. |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 1147 | */ |
Paul E. McKenney | fe5ac72 | 2017-05-11 11:26:22 -0700 | [diff] [blame] | 1148 | static int rcu_implicit_dynticks_qs(struct rcu_data *rdp) |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 1149 | { |
Paul E. McKenney | 3a19b46 | 2016-11-30 11:21:21 -0800 | [diff] [blame] | 1150 | unsigned long jtsq; |
Paul E. McKenney | 0f9be8c | 2017-01-27 13:17:02 -0800 | [diff] [blame] | 1151 | bool *rnhqp; |
Paul E. McKenney | 9226b10 | 2017-01-27 14:17:50 -0800 | [diff] [blame] | 1152 | bool *ruqp; |
Paul E. McKenney | 9b9500d | 2017-08-17 17:05:59 -0700 | [diff] [blame] | 1153 | struct rcu_node *rnp = rdp->mynode; |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 1154 | |
| 1155 | /* |
| 1156 | * If the CPU passed through or entered a dynticks idle phase with |
| 1157 | * no active irq/NMI handlers, then we can safely pretend that the CPU |
| 1158 | * already acknowledged the request to pass through a quiescent |
| 1159 | * state. Either way, that CPU cannot possibly be in an RCU |
| 1160 | * read-side critical section that started before the beginning |
| 1161 | * of the current RCU grace period. |
| 1162 | */ |
Paul E. McKenney | 02a5c550 | 2016-11-02 17:25:06 -0700 | [diff] [blame] | 1163 | if (rcu_dynticks_in_eqs_since(rdp->dynticks, rdp->dynticks_snap)) { |
Steven Rostedt (Red Hat) | f7f7bac | 2013-07-12 17:18:47 -0400 | [diff] [blame] | 1164 | trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("dti")); |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 1165 | rdp->dynticks_fqs++; |
Paul E. McKenney | 9b9500d | 2017-08-17 17:05:59 -0700 | [diff] [blame] | 1166 | rcu_gpnum_ovf(rnp, rdp); |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 1167 | return 1; |
| 1168 | } |
| 1169 | |
Paul E. McKenney | a82dcc7 | 2012-08-01 14:29:20 -0700 | [diff] [blame] | 1170 | /* |
Paul E. McKenney | cee4393 | 2018-03-02 16:35:27 -0800 | [diff] [blame] | 1171 | * Has this CPU encountered a cond_resched() since the beginning |
| 1172 | * of the grace period? For this to be the case, the CPU has to |
| 1173 | * have noticed the current grace period. This might not be the |
| 1174 | * case for nohz_full CPUs looping in the kernel. |
Paul E. McKenney | a82dcc7 | 2012-08-01 14:29:20 -0700 | [diff] [blame] | 1175 | */ |
Paul E. McKenney | f79c3ad | 2016-11-30 06:24:30 -0800 | [diff] [blame] | 1176 | jtsq = jiffies_till_sched_qs; |
Paul E. McKenney | 9226b10 | 2017-01-27 14:17:50 -0800 | [diff] [blame] | 1177 | ruqp = per_cpu_ptr(&rcu_dynticks.rcu_urgent_qs, rdp->cpu); |
Paul E. McKenney | 3a19b46 | 2016-11-30 11:21:21 -0800 | [diff] [blame] | 1178 | if (time_after(jiffies, rdp->rsp->gp_start + jtsq) && |
Paul E. McKenney | 9577df9 | 2017-01-26 16:18:07 -0800 | [diff] [blame] | 1179 | READ_ONCE(rdp->rcu_qs_ctr_snap) != per_cpu(rcu_dynticks.rcu_qs_ctr, rdp->cpu) && |
Paul E. McKenney | 3a19b46 | 2016-11-30 11:21:21 -0800 | [diff] [blame] | 1180 | READ_ONCE(rdp->gpnum) == rnp->gpnum && !rdp->gpwrap) { |
| 1181 | trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("rqc")); |
Paul E. McKenney | 9b9500d | 2017-08-17 17:05:59 -0700 | [diff] [blame] | 1182 | rcu_gpnum_ovf(rnp, rdp); |
Paul E. McKenney | 3a19b46 | 2016-11-30 11:21:21 -0800 | [diff] [blame] | 1183 | return 1; |
Paul E. McKenney | f79c3ad | 2016-11-30 06:24:30 -0800 | [diff] [blame] | 1184 | } else if (time_after(jiffies, rdp->rsp->gp_start + jtsq)) { |
Paul E. McKenney | 9226b10 | 2017-01-27 14:17:50 -0800 | [diff] [blame] | 1185 | /* Load rcu_qs_ctr before store to rcu_urgent_qs. */ |
| 1186 | smp_store_release(ruqp, true); |
Paul E. McKenney | 3a19b46 | 2016-11-30 11:21:21 -0800 | [diff] [blame] | 1187 | } |
| 1188 | |
Paul E. McKenney | 38d30b3 | 2016-12-01 09:55:27 -0800 | [diff] [blame] | 1189 | /* Check for the CPU being offline. */ |
| 1190 | if (!(rdp->grpmask & rcu_rnp_online_cpus(rnp))) { |
Steven Rostedt (Red Hat) | f7f7bac | 2013-07-12 17:18:47 -0400 | [diff] [blame] | 1191 | trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("ofl")); |
Paul E. McKenney | a82dcc7 | 2012-08-01 14:29:20 -0700 | [diff] [blame] | 1192 | rdp->offline_fqs++; |
Paul E. McKenney | 9b9500d | 2017-08-17 17:05:59 -0700 | [diff] [blame] | 1193 | rcu_gpnum_ovf(rnp, rdp); |
Paul E. McKenney | a82dcc7 | 2012-08-01 14:29:20 -0700 | [diff] [blame] | 1194 | return 1; |
| 1195 | } |
Paul E. McKenney | 65d798f | 2013-04-12 16:19:10 -0700 | [diff] [blame] | 1196 | |
| 1197 | /* |
Paul E. McKenney | 4a81e83 | 2014-06-20 16:49:01 -0700 | [diff] [blame] | 1198 | * A CPU running for an extended time within the kernel can |
| 1199 | * delay RCU grace periods. When the CPU is in NO_HZ_FULL mode, |
| 1200 | * even context-switching back and forth between a pair of |
| 1201 | * in-kernel CPU-bound tasks cannot advance grace periods. |
| 1202 | * So if the grace period is old enough, make the CPU pay attention. |
| 1203 | * Note that the unsynchronized assignments to the per-CPU |
Paul E. McKenney | 0f9be8c | 2017-01-27 13:17:02 -0800 | [diff] [blame] | 1204 | * rcu_need_heavy_qs variable are safe. Yes, setting of |
Paul E. McKenney | 4a81e83 | 2014-06-20 16:49:01 -0700 | [diff] [blame] | 1205 | * bits can be lost, but they will be set again on the next |
| 1206 | * force-quiescent-state pass. So lost bit sets do not result |
| 1207 | * in incorrect behavior, merely in a grace period lasting |
| 1208 | * a few jiffies longer than it might otherwise. Because |
| 1209 | * there are at most four threads involved, and because the |
| 1210 | * updates are only once every few jiffies, the probability of |
| 1211 | * lossage (and thus of slight grace-period extension) is |
| 1212 | * quite low. |
Paul E. McKenney | 65d798f | 2013-04-12 16:19:10 -0700 | [diff] [blame] | 1213 | */ |
Paul E. McKenney | 0f9be8c | 2017-01-27 13:17:02 -0800 | [diff] [blame] | 1214 | rnhqp = &per_cpu(rcu_dynticks.rcu_need_heavy_qs, rdp->cpu); |
| 1215 | if (!READ_ONCE(*rnhqp) && |
| 1216 | (time_after(jiffies, rdp->rsp->gp_start + jtsq) || |
| 1217 | time_after(jiffies, rdp->rsp->jiffies_resched))) { |
| 1218 | WRITE_ONCE(*rnhqp, true); |
Paul E. McKenney | 9226b10 | 2017-01-27 14:17:50 -0800 | [diff] [blame] | 1219 | /* Store rcu_need_heavy_qs before rcu_urgent_qs. */ |
| 1220 | smp_store_release(ruqp, true); |
Paul E. McKenney | f79c3ad | 2016-11-30 06:24:30 -0800 | [diff] [blame] | 1221 | rdp->rsp->jiffies_resched += jtsq; /* Re-enable beating. */ |
Paul E. McKenney | 6193c76 | 2013-09-23 13:57:18 -0700 | [diff] [blame] | 1222 | } |
| 1223 | |
Paul E. McKenney | 28053bc | 2016-12-01 11:31:31 -0800 | [diff] [blame] | 1224 | /* |
Paul E. McKenney | 9b9500d | 2017-08-17 17:05:59 -0700 | [diff] [blame] | 1225 | * If more than halfway to RCU CPU stall-warning time, do a |
| 1226 | * resched_cpu() to try to loosen things up a bit. Also check to |
| 1227 | * see if the CPU is getting hammered with interrupts, but only |
| 1228 | * once per grace period, just to keep the IPIs down to a dull roar. |
Paul E. McKenney | 28053bc | 2016-12-01 11:31:31 -0800 | [diff] [blame] | 1229 | */ |
Paul E. McKenney | 9b9500d | 2017-08-17 17:05:59 -0700 | [diff] [blame] | 1230 | if (jiffies - rdp->rsp->gp_start > rcu_jiffies_till_stall_check() / 2) { |
Paul E. McKenney | 28053bc | 2016-12-01 11:31:31 -0800 | [diff] [blame] | 1231 | resched_cpu(rdp->cpu); |
Paul E. McKenney | 9b9500d | 2017-08-17 17:05:59 -0700 | [diff] [blame] | 1232 | if (IS_ENABLED(CONFIG_IRQ_WORK) && |
| 1233 | !rdp->rcu_iw_pending && rdp->rcu_iw_gpnum != rnp->gpnum && |
| 1234 | (rnp->ffmask & rdp->grpmask)) { |
| 1235 | init_irq_work(&rdp->rcu_iw, rcu_iw_handler); |
| 1236 | rdp->rcu_iw_pending = true; |
| 1237 | rdp->rcu_iw_gpnum = rnp->gpnum; |
| 1238 | irq_work_queue_on(&rdp->rcu_iw, rdp->cpu); |
| 1239 | } |
| 1240 | } |
Paul E. McKenney | 4914950 | 2015-12-11 13:48:43 -0800 | [diff] [blame] | 1241 | |
Paul E. McKenney | a82dcc7 | 2012-08-01 14:29:20 -0700 | [diff] [blame] | 1242 | return 0; |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 1243 | } |
| 1244 | |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 1245 | static void record_gp_stall_check_time(struct rcu_state *rsp) |
| 1246 | { |
Paul E. McKenney | cb1e78c | 2013-12-04 18:42:03 -0800 | [diff] [blame] | 1247 | unsigned long j = jiffies; |
Paul E. McKenney | 6193c76 | 2013-09-23 13:57:18 -0700 | [diff] [blame] | 1248 | unsigned long j1; |
Paul E. McKenney | 26cdfed | 2013-09-04 10:51:13 -0700 | [diff] [blame] | 1249 | |
| 1250 | rsp->gp_start = j; |
| 1251 | smp_wmb(); /* Record start time before stall time. */ |
Paul E. McKenney | 6193c76 | 2013-09-23 13:57:18 -0700 | [diff] [blame] | 1252 | j1 = rcu_jiffies_till_stall_check(); |
Paul E. McKenney | 7d0ae80 | 2015-03-03 14:57:58 -0800 | [diff] [blame] | 1253 | WRITE_ONCE(rsp->jiffies_stall, j + j1); |
Paul E. McKenney | 6193c76 | 2013-09-23 13:57:18 -0700 | [diff] [blame] | 1254 | rsp->jiffies_resched = j + j1 / 2; |
Paul E. McKenney | 7d0ae80 | 2015-03-03 14:57:58 -0800 | [diff] [blame] | 1255 | rsp->n_force_qs_gpstart = READ_ONCE(rsp->n_force_qs); |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 1256 | } |
| 1257 | |
Paul E. McKenney | b637a32 | 2012-09-19 16:58:38 -0700 | [diff] [blame] | 1258 | /* |
Paul E. McKenney | 6b50e11 | 2015-11-17 14:39:26 -0800 | [diff] [blame] | 1259 | * Convert a ->gp_state value to a character string. |
| 1260 | */ |
| 1261 | static const char *gp_state_getname(short gs) |
| 1262 | { |
| 1263 | if (gs < 0 || gs >= ARRAY_SIZE(gp_state_names)) |
| 1264 | return "???"; |
| 1265 | return gp_state_names[gs]; |
| 1266 | } |
| 1267 | |
| 1268 | /* |
Paul E. McKenney | fb81a44 | 2014-12-17 08:35:02 -0800 | [diff] [blame] | 1269 | * Complain about starvation of grace-period kthread. |
| 1270 | */ |
| 1271 | static void rcu_check_gp_kthread_starvation(struct rcu_state *rsp) |
| 1272 | { |
| 1273 | unsigned long gpa; |
| 1274 | unsigned long j; |
| 1275 | |
| 1276 | j = jiffies; |
Paul E. McKenney | 7d0ae80 | 2015-03-03 14:57:58 -0800 | [diff] [blame] | 1277 | gpa = READ_ONCE(rsp->gp_activity); |
Paul E. McKenney | b1adb3e | 2015-10-01 10:38:16 -0700 | [diff] [blame] | 1278 | if (j - gpa > 2 * HZ) { |
Paul E. McKenney | 78c5a67 | 2018-04-27 13:32:28 -0700 | [diff] [blame] | 1279 | pr_err("%s kthread starved for %ld jiffies! g%ld f%#x %s(%d) ->state=%#lx ->cpu=%d\n", |
Paul E. McKenney | 81e701e43 | 2015-04-16 11:02:25 -0700 | [diff] [blame] | 1280 | rsp->name, j - gpa, |
Paul E. McKenney | 78c5a67 | 2018-04-27 13:32:28 -0700 | [diff] [blame] | 1281 | (long)rcu_seq_current(&rsp->gp_seq), |
Paul E. McKenney | 6b50e11 | 2015-11-17 14:39:26 -0800 | [diff] [blame] | 1282 | rsp->gp_flags, |
| 1283 | gp_state_getname(rsp->gp_state), rsp->gp_state, |
Paul E. McKenney | 96036c4 | 2017-07-18 13:52:18 -0700 | [diff] [blame] | 1284 | rsp->gp_kthread ? rsp->gp_kthread->state : ~0, |
| 1285 | rsp->gp_kthread ? task_cpu(rsp->gp_kthread) : -1); |
Paul E. McKenney | 86057b8 | 2015-12-31 08:48:36 -0800 | [diff] [blame] | 1286 | if (rsp->gp_kthread) { |
Paul E. McKenney | d07aee2 | 2018-01-11 12:08:20 -0800 | [diff] [blame] | 1287 | pr_err("RCU grace-period kthread stack dump:\n"); |
Paul E. McKenney | b1adb3e | 2015-10-01 10:38:16 -0700 | [diff] [blame] | 1288 | sched_show_task(rsp->gp_kthread); |
Paul E. McKenney | 86057b8 | 2015-12-31 08:48:36 -0800 | [diff] [blame] | 1289 | wake_up_process(rsp->gp_kthread); |
| 1290 | } |
Paul E. McKenney | b1adb3e | 2015-10-01 10:38:16 -0700 | [diff] [blame] | 1291 | } |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 1292 | } |
| 1293 | |
Paul E. McKenney | b637a32 | 2012-09-19 16:58:38 -0700 | [diff] [blame] | 1294 | /* |
Paul E. McKenney | 7aa9223 | 2016-11-29 05:49:06 -0800 | [diff] [blame] | 1295 | * Dump stacks of all tasks running on stalled CPUs. First try using |
| 1296 | * NMIs, but fall back to manual remote stack tracing on architectures |
| 1297 | * that don't support NMI-based stack dumps. The NMI-triggered stack |
| 1298 | * traces are more accurate because they are printed by the target CPU. |
Paul E. McKenney | b637a32 | 2012-09-19 16:58:38 -0700 | [diff] [blame] | 1299 | */ |
| 1300 | static void rcu_dump_cpu_stacks(struct rcu_state *rsp) |
| 1301 | { |
| 1302 | int cpu; |
| 1303 | unsigned long flags; |
| 1304 | struct rcu_node *rnp; |
| 1305 | |
| 1306 | rcu_for_each_leaf_node(rsp, rnp) { |
Paul E. McKenney | 6cf1008 | 2015-10-08 15:36:54 -0700 | [diff] [blame] | 1307 | raw_spin_lock_irqsave_rcu_node(rnp, flags); |
Paul E. McKenney | 7aa9223 | 2016-11-29 05:49:06 -0800 | [diff] [blame] | 1308 | for_each_leaf_node_possible_cpu(rnp, cpu) |
| 1309 | if (rnp->qsmask & leaf_node_cpu_bit(rnp, cpu)) |
| 1310 | if (!trigger_single_cpu_backtrace(cpu)) |
Mark Rutland | bc75e99 | 2016-06-03 15:20:04 +0100 | [diff] [blame] | 1311 | dump_cpu_task(cpu); |
Boqun Feng | 67c583a7 | 2015-12-29 12:18:47 +0800 | [diff] [blame] | 1312 | raw_spin_unlock_irqrestore_rcu_node(rnp, flags); |
Paul E. McKenney | b637a32 | 2012-09-19 16:58:38 -0700 | [diff] [blame] | 1313 | } |
| 1314 | } |
| 1315 | |
Paul E. McKenney | 8c7c482 | 2016-01-03 20:29:57 -0800 | [diff] [blame] | 1316 | /* |
| 1317 | * If too much time has passed in the current grace period, and if |
| 1318 | * so configured, go kick the relevant kthreads. |
| 1319 | */ |
| 1320 | static void rcu_stall_kick_kthreads(struct rcu_state *rsp) |
| 1321 | { |
| 1322 | unsigned long j; |
| 1323 | |
| 1324 | if (!rcu_kick_kthreads) |
| 1325 | return; |
| 1326 | j = READ_ONCE(rsp->jiffies_kick_kthreads); |
Paul E. McKenney | aa3e0bf | 2016-03-23 10:43:23 -0700 | [diff] [blame] | 1327 | if (time_after(jiffies, j) && rsp->gp_kthread && |
| 1328 | (rcu_gp_in_progress(rsp) || READ_ONCE(rsp->gp_flags))) { |
Paul E. McKenney | 8c7c482 | 2016-01-03 20:29:57 -0800 | [diff] [blame] | 1329 | WARN_ONCE(1, "Kicking %s grace-period kthread\n", rsp->name); |
Paul E. McKenney | 5dffed1 | 2016-02-17 11:54:28 -0800 | [diff] [blame] | 1330 | rcu_ftrace_dump(DUMP_ALL); |
Paul E. McKenney | 8c7c482 | 2016-01-03 20:29:57 -0800 | [diff] [blame] | 1331 | wake_up_process(rsp->gp_kthread); |
| 1332 | WRITE_ONCE(rsp->jiffies_kick_kthreads, j + HZ); |
| 1333 | } |
| 1334 | } |
| 1335 | |
Daniel Bristot de Oliveira | 088e9d2 | 2016-06-02 13:51:41 -0300 | [diff] [blame] | 1336 | static inline void panic_on_rcu_stall(void) |
| 1337 | { |
| 1338 | if (sysctl_panic_on_rcu_stall) |
| 1339 | panic("RCU Stall\n"); |
| 1340 | } |
| 1341 | |
Paul E. McKenney | 6ccd2ec | 2014-12-11 10:20:59 -0800 | [diff] [blame] | 1342 | static void print_other_cpu_stall(struct rcu_state *rsp, unsigned long gpnum) |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 1343 | { |
| 1344 | int cpu; |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 1345 | unsigned long flags; |
Paul E. McKenney | 6ccd2ec | 2014-12-11 10:20:59 -0800 | [diff] [blame] | 1346 | unsigned long gpa; |
| 1347 | unsigned long j; |
Paul E. McKenney | 285fe29 | 2012-05-09 08:45:12 -0700 | [diff] [blame] | 1348 | int ndetected = 0; |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 1349 | struct rcu_node *rnp = rcu_get_root(rsp); |
Paul E. McKenney | 53bb857 | 2012-09-21 16:35:25 -0700 | [diff] [blame] | 1350 | long totqlen = 0; |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 1351 | |
Paul E. McKenney | 8c7c482 | 2016-01-03 20:29:57 -0800 | [diff] [blame] | 1352 | /* Kick and suppress, if so configured. */ |
| 1353 | rcu_stall_kick_kthreads(rsp); |
| 1354 | if (rcu_cpu_stall_suppress) |
| 1355 | return; |
| 1356 | |
Paul E. McKenney | 8cdd32a | 2010-08-09 14:23:03 -0700 | [diff] [blame] | 1357 | /* |
| 1358 | * OK, time to rat on our buddy... |
| 1359 | * See Documentation/RCU/stallwarn.txt for info on how to debug |
| 1360 | * RCU CPU stall warnings. |
| 1361 | */ |
Paul E. McKenney | d7f3e20 | 2013-03-18 16:19:52 -0700 | [diff] [blame] | 1362 | pr_err("INFO: %s detected stalls on CPUs/tasks:", |
Paul E. McKenney | 4300aa6 | 2010-04-13 16:18:22 -0700 | [diff] [blame] | 1363 | rsp->name); |
Paul E. McKenney | a858af2 | 2012-01-16 13:29:10 -0800 | [diff] [blame] | 1364 | print_cpu_stall_info_begin(); |
Paul E. McKenney | a0b6c9a | 2009-09-28 07:46:33 -0700 | [diff] [blame] | 1365 | rcu_for_each_leaf_node(rsp, rnp) { |
Paul E. McKenney | 6cf1008 | 2015-10-08 15:36:54 -0700 | [diff] [blame] | 1366 | raw_spin_lock_irqsave_rcu_node(rnp, flags); |
Paul E. McKenney | 9bc8b55 | 2011-08-13 13:31:47 -0700 | [diff] [blame] | 1367 | ndetected += rcu_print_task_stall(rnp); |
Paul E. McKenney | c8020a6 | 2012-08-10 16:55:59 -0700 | [diff] [blame] | 1368 | if (rnp->qsmask != 0) { |
Mark Rutland | bc75e99 | 2016-06-03 15:20:04 +0100 | [diff] [blame] | 1369 | for_each_leaf_node_possible_cpu(rnp, cpu) |
| 1370 | if (rnp->qsmask & leaf_node_cpu_bit(rnp, cpu)) { |
| 1371 | print_cpu_stall_info(rsp, cpu); |
Paul E. McKenney | c8020a6 | 2012-08-10 16:55:59 -0700 | [diff] [blame] | 1372 | ndetected++; |
| 1373 | } |
| 1374 | } |
Boqun Feng | 67c583a7 | 2015-12-29 12:18:47 +0800 | [diff] [blame] | 1375 | raw_spin_unlock_irqrestore_rcu_node(rnp, flags); |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 1376 | } |
Paul E. McKenney | a858af2 | 2012-01-16 13:29:10 -0800 | [diff] [blame] | 1377 | |
Paul E. McKenney | a858af2 | 2012-01-16 13:29:10 -0800 | [diff] [blame] | 1378 | print_cpu_stall_info_end(); |
Paul E. McKenney | 53bb857 | 2012-09-21 16:35:25 -0700 | [diff] [blame] | 1379 | for_each_possible_cpu(cpu) |
Paul E. McKenney | 15fecf8 | 2017-02-08 12:36:42 -0800 | [diff] [blame] | 1380 | totqlen += rcu_segcblist_n_cbs(&per_cpu_ptr(rsp->rda, |
| 1381 | cpu)->cblist); |
Paul E. McKenney | 83ebe63 | 2014-03-06 11:09:10 -0800 | [diff] [blame] | 1382 | pr_cont("(detected by %d, t=%ld jiffies, g=%ld, c=%ld, q=%lu)\n", |
Paul E. McKenney | eee0588 | 2012-09-21 14:15:05 -0700 | [diff] [blame] | 1383 | smp_processor_id(), (long)(jiffies - rsp->gp_start), |
Paul E. McKenney | 83ebe63 | 2014-03-06 11:09:10 -0800 | [diff] [blame] | 1384 | (long)rsp->gpnum, (long)rsp->completed, totqlen); |
Paul E. McKenney | 6ccd2ec | 2014-12-11 10:20:59 -0800 | [diff] [blame] | 1385 | if (ndetected) { |
Paul E. McKenney | b637a32 | 2012-09-19 16:58:38 -0700 | [diff] [blame] | 1386 | rcu_dump_cpu_stacks(rsp); |
Byungchul Park | c4402b2 | 2016-11-09 17:57:13 +0900 | [diff] [blame] | 1387 | |
| 1388 | /* Complain about tasks blocking the grace period. */ |
| 1389 | rcu_print_detail_task_stall(rsp); |
Paul E. McKenney | 6ccd2ec | 2014-12-11 10:20:59 -0800 | [diff] [blame] | 1390 | } else { |
Paul E. McKenney | 7d0ae80 | 2015-03-03 14:57:58 -0800 | [diff] [blame] | 1391 | if (READ_ONCE(rsp->gpnum) != gpnum || |
| 1392 | READ_ONCE(rsp->completed) == gpnum) { |
Paul E. McKenney | 6ccd2ec | 2014-12-11 10:20:59 -0800 | [diff] [blame] | 1393 | pr_err("INFO: Stall ended before state dump start\n"); |
| 1394 | } else { |
| 1395 | j = jiffies; |
Paul E. McKenney | 7d0ae80 | 2015-03-03 14:57:58 -0800 | [diff] [blame] | 1396 | gpa = READ_ONCE(rsp->gp_activity); |
Paul E. McKenney | 237a0f2 | 2015-01-22 14:32:06 -0800 | [diff] [blame] | 1397 | pr_err("All QSes seen, last %s kthread activity %ld (%ld-%ld), jiffies_till_next_fqs=%ld, root ->qsmask %#lx\n", |
Paul E. McKenney | 6ccd2ec | 2014-12-11 10:20:59 -0800 | [diff] [blame] | 1398 | rsp->name, j - gpa, j, gpa, |
Paul E. McKenney | 237a0f2 | 2015-01-22 14:32:06 -0800 | [diff] [blame] | 1399 | jiffies_till_next_fqs, |
| 1400 | rcu_get_root(rsp)->qsmask); |
Paul E. McKenney | 6ccd2ec | 2014-12-11 10:20:59 -0800 | [diff] [blame] | 1401 | /* In this case, the current CPU might be at fault. */ |
| 1402 | sched_show_task(current); |
| 1403 | } |
| 1404 | } |
Paul E. McKenney | 8c42b1f | 2018-04-09 11:04:46 -0700 | [diff] [blame] | 1405 | /* Rewrite if needed in case of slow consoles. */ |
| 1406 | if (ULONG_CMP_GE(jiffies, READ_ONCE(rsp->jiffies_stall))) |
| 1407 | WRITE_ONCE(rsp->jiffies_stall, |
| 1408 | jiffies + 3 * rcu_jiffies_till_stall_check() + 3); |
Ingo Molnar | c1dc0b9 | 2009-08-02 11:28:21 +0200 | [diff] [blame] | 1409 | |
Paul E. McKenney | fb81a44 | 2014-12-17 08:35:02 -0800 | [diff] [blame] | 1410 | rcu_check_gp_kthread_starvation(rsp); |
| 1411 | |
Daniel Bristot de Oliveira | 088e9d2 | 2016-06-02 13:51:41 -0300 | [diff] [blame] | 1412 | panic_on_rcu_stall(); |
| 1413 | |
Paul E. McKenney | 4cdfc175 | 2012-06-22 17:06:26 -0700 | [diff] [blame] | 1414 | force_quiescent_state(rsp); /* Kick them all. */ |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 1415 | } |
| 1416 | |
| 1417 | static void print_cpu_stall(struct rcu_state *rsp) |
| 1418 | { |
Paul E. McKenney | 53bb857 | 2012-09-21 16:35:25 -0700 | [diff] [blame] | 1419 | int cpu; |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 1420 | unsigned long flags; |
Paul E. McKenney | 9b9500d | 2017-08-17 17:05:59 -0700 | [diff] [blame] | 1421 | struct rcu_data *rdp = this_cpu_ptr(rsp->rda); |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 1422 | struct rcu_node *rnp = rcu_get_root(rsp); |
Paul E. McKenney | 53bb857 | 2012-09-21 16:35:25 -0700 | [diff] [blame] | 1423 | long totqlen = 0; |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 1424 | |
Paul E. McKenney | 8c7c482 | 2016-01-03 20:29:57 -0800 | [diff] [blame] | 1425 | /* Kick and suppress, if so configured. */ |
| 1426 | rcu_stall_kick_kthreads(rsp); |
| 1427 | if (rcu_cpu_stall_suppress) |
| 1428 | return; |
| 1429 | |
Paul E. McKenney | 8cdd32a | 2010-08-09 14:23:03 -0700 | [diff] [blame] | 1430 | /* |
| 1431 | * OK, time to rat on ourselves... |
| 1432 | * See Documentation/RCU/stallwarn.txt for info on how to debug |
| 1433 | * RCU CPU stall warnings. |
| 1434 | */ |
Paul E. McKenney | d7f3e20 | 2013-03-18 16:19:52 -0700 | [diff] [blame] | 1435 | pr_err("INFO: %s self-detected stall on CPU", rsp->name); |
Paul E. McKenney | a858af2 | 2012-01-16 13:29:10 -0800 | [diff] [blame] | 1436 | print_cpu_stall_info_begin(); |
Paul E. McKenney | 9b9500d | 2017-08-17 17:05:59 -0700 | [diff] [blame] | 1437 | raw_spin_lock_irqsave_rcu_node(rdp->mynode, flags); |
Paul E. McKenney | a858af2 | 2012-01-16 13:29:10 -0800 | [diff] [blame] | 1438 | print_cpu_stall_info(rsp, smp_processor_id()); |
Paul E. McKenney | 9b9500d | 2017-08-17 17:05:59 -0700 | [diff] [blame] | 1439 | raw_spin_unlock_irqrestore_rcu_node(rdp->mynode, flags); |
Paul E. McKenney | a858af2 | 2012-01-16 13:29:10 -0800 | [diff] [blame] | 1440 | print_cpu_stall_info_end(); |
Paul E. McKenney | 53bb857 | 2012-09-21 16:35:25 -0700 | [diff] [blame] | 1441 | for_each_possible_cpu(cpu) |
Paul E. McKenney | 15fecf8 | 2017-02-08 12:36:42 -0800 | [diff] [blame] | 1442 | totqlen += rcu_segcblist_n_cbs(&per_cpu_ptr(rsp->rda, |
| 1443 | cpu)->cblist); |
Paul E. McKenney | 83ebe63 | 2014-03-06 11:09:10 -0800 | [diff] [blame] | 1444 | pr_cont(" (t=%lu jiffies g=%ld c=%ld q=%lu)\n", |
| 1445 | jiffies - rsp->gp_start, |
| 1446 | (long)rsp->gpnum, (long)rsp->completed, totqlen); |
Paul E. McKenney | fb81a44 | 2014-12-17 08:35:02 -0800 | [diff] [blame] | 1447 | |
| 1448 | rcu_check_gp_kthread_starvation(rsp); |
| 1449 | |
Paul E. McKenney | bc1dce5 | 2014-06-18 09:18:31 -0700 | [diff] [blame] | 1450 | rcu_dump_cpu_stacks(rsp); |
Ingo Molnar | c1dc0b9 | 2009-08-02 11:28:21 +0200 | [diff] [blame] | 1451 | |
Paul E. McKenney | 6cf1008 | 2015-10-08 15:36:54 -0700 | [diff] [blame] | 1452 | raw_spin_lock_irqsave_rcu_node(rnp, flags); |
Paul E. McKenney | 8c42b1f | 2018-04-09 11:04:46 -0700 | [diff] [blame] | 1453 | /* Rewrite if needed in case of slow consoles. */ |
Paul E. McKenney | 7d0ae80 | 2015-03-03 14:57:58 -0800 | [diff] [blame] | 1454 | if (ULONG_CMP_GE(jiffies, READ_ONCE(rsp->jiffies_stall))) |
| 1455 | WRITE_ONCE(rsp->jiffies_stall, |
| 1456 | jiffies + 3 * rcu_jiffies_till_stall_check() + 3); |
Boqun Feng | 67c583a7 | 2015-12-29 12:18:47 +0800 | [diff] [blame] | 1457 | raw_spin_unlock_irqrestore_rcu_node(rnp, flags); |
Ingo Molnar | c1dc0b9 | 2009-08-02 11:28:21 +0200 | [diff] [blame] | 1458 | |
Daniel Bristot de Oliveira | 088e9d2 | 2016-06-02 13:51:41 -0300 | [diff] [blame] | 1459 | panic_on_rcu_stall(); |
| 1460 | |
Peter Zijlstra | b021fe3 | 2013-09-17 09:30:55 +0200 | [diff] [blame] | 1461 | /* |
| 1462 | * Attempt to revive the RCU machinery by forcing a context switch. |
| 1463 | * |
| 1464 | * A context switch would normally allow the RCU state machine to make |
| 1465 | * progress and it could be we're stuck in kernel space without context |
| 1466 | * switches for an entirely unreasonable amount of time. |
| 1467 | */ |
| 1468 | resched_cpu(smp_processor_id()); |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 1469 | } |
| 1470 | |
| 1471 | static void check_cpu_stall(struct rcu_state *rsp, struct rcu_data *rdp) |
| 1472 | { |
Paul E. McKenney | 26cdfed | 2013-09-04 10:51:13 -0700 | [diff] [blame] | 1473 | unsigned long completed; |
| 1474 | unsigned long gpnum; |
| 1475 | unsigned long gps; |
Paul E. McKenney | bad6e13 | 2011-05-02 23:40:04 -0700 | [diff] [blame] | 1476 | unsigned long j; |
Paul E. McKenney | 8c42b1f | 2018-04-09 11:04:46 -0700 | [diff] [blame] | 1477 | unsigned long jn; |
Paul E. McKenney | bad6e13 | 2011-05-02 23:40:04 -0700 | [diff] [blame] | 1478 | unsigned long js; |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 1479 | struct rcu_node *rnp; |
| 1480 | |
Paul E. McKenney | 8c7c482 | 2016-01-03 20:29:57 -0800 | [diff] [blame] | 1481 | if ((rcu_cpu_stall_suppress && !rcu_kick_kthreads) || |
| 1482 | !rcu_gp_in_progress(rsp)) |
Paul E. McKenney | c68de20 | 2010-04-15 10:12:40 -0700 | [diff] [blame] | 1483 | return; |
Paul E. McKenney | 8c7c482 | 2016-01-03 20:29:57 -0800 | [diff] [blame] | 1484 | rcu_stall_kick_kthreads(rsp); |
Paul E. McKenney | cb1e78c | 2013-12-04 18:42:03 -0800 | [diff] [blame] | 1485 | j = jiffies; |
Paul E. McKenney | 26cdfed | 2013-09-04 10:51:13 -0700 | [diff] [blame] | 1486 | |
| 1487 | /* |
| 1488 | * Lots of memory barriers to reject false positives. |
| 1489 | * |
| 1490 | * The idea is to pick up rsp->gpnum, then rsp->jiffies_stall, |
| 1491 | * then rsp->gp_start, and finally rsp->completed. These values |
| 1492 | * are updated in the opposite order with memory barriers (or |
| 1493 | * equivalent) during grace-period initialization and cleanup. |
| 1494 | * Now, a false positive can occur if we get an new value of |
| 1495 | * rsp->gp_start and a old value of rsp->jiffies_stall. But given |
| 1496 | * the memory barriers, the only way that this can happen is if one |
| 1497 | * grace period ends and another starts between these two fetches. |
| 1498 | * Detect this by comparing rsp->completed with the previous fetch |
| 1499 | * from rsp->gpnum. |
| 1500 | * |
| 1501 | * Given this check, comparisons of jiffies, rsp->jiffies_stall, |
| 1502 | * and rsp->gp_start suffice to forestall false positives. |
| 1503 | */ |
Paul E. McKenney | 7d0ae80 | 2015-03-03 14:57:58 -0800 | [diff] [blame] | 1504 | gpnum = READ_ONCE(rsp->gpnum); |
Paul E. McKenney | 26cdfed | 2013-09-04 10:51:13 -0700 | [diff] [blame] | 1505 | smp_rmb(); /* Pick up ->gpnum first... */ |
Paul E. McKenney | 7d0ae80 | 2015-03-03 14:57:58 -0800 | [diff] [blame] | 1506 | js = READ_ONCE(rsp->jiffies_stall); |
Paul E. McKenney | 26cdfed | 2013-09-04 10:51:13 -0700 | [diff] [blame] | 1507 | smp_rmb(); /* ...then ->jiffies_stall before the rest... */ |
Paul E. McKenney | 7d0ae80 | 2015-03-03 14:57:58 -0800 | [diff] [blame] | 1508 | gps = READ_ONCE(rsp->gp_start); |
Paul E. McKenney | 26cdfed | 2013-09-04 10:51:13 -0700 | [diff] [blame] | 1509 | smp_rmb(); /* ...and finally ->gp_start before ->completed. */ |
Paul E. McKenney | 7d0ae80 | 2015-03-03 14:57:58 -0800 | [diff] [blame] | 1510 | completed = READ_ONCE(rsp->completed); |
Paul E. McKenney | 26cdfed | 2013-09-04 10:51:13 -0700 | [diff] [blame] | 1511 | if (ULONG_CMP_GE(completed, gpnum) || |
| 1512 | ULONG_CMP_LT(j, js) || |
| 1513 | ULONG_CMP_GE(gps, js)) |
| 1514 | return; /* No stall or GP completed since entering function. */ |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 1515 | rnp = rdp->mynode; |
Paul E. McKenney | 8c42b1f | 2018-04-09 11:04:46 -0700 | [diff] [blame] | 1516 | jn = jiffies + 3 * rcu_jiffies_till_stall_check() + 3; |
Paul E. McKenney | c96ea7c | 2012-08-13 11:17:06 -0700 | [diff] [blame] | 1517 | if (rcu_gp_in_progress(rsp) && |
Paul E. McKenney | 8c42b1f | 2018-04-09 11:04:46 -0700 | [diff] [blame] | 1518 | (READ_ONCE(rnp->qsmask) & rdp->grpmask) && |
| 1519 | cmpxchg(&rsp->jiffies_stall, js, jn) == js) { |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 1520 | |
| 1521 | /* We haven't checked in, so go dump stack. */ |
| 1522 | print_cpu_stall(rsp); |
| 1523 | |
Paul E. McKenney | bad6e13 | 2011-05-02 23:40:04 -0700 | [diff] [blame] | 1524 | } else if (rcu_gp_in_progress(rsp) && |
Paul E. McKenney | 8c42b1f | 2018-04-09 11:04:46 -0700 | [diff] [blame] | 1525 | ULONG_CMP_GE(j, js + RCU_STALL_RAT_DELAY) && |
| 1526 | cmpxchg(&rsp->jiffies_stall, js, jn) == js) { |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 1527 | |
Paul E. McKenney | bad6e13 | 2011-05-02 23:40:04 -0700 | [diff] [blame] | 1528 | /* They had a few time units to dump stack, so complain. */ |
Paul E. McKenney | 6ccd2ec | 2014-12-11 10:20:59 -0800 | [diff] [blame] | 1529 | print_other_cpu_stall(rsp, gpnum); |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 1530 | } |
| 1531 | } |
| 1532 | |
Paul E. McKenney | 53d84e0 | 2010-08-10 14:28:53 -0700 | [diff] [blame] | 1533 | /** |
| 1534 | * rcu_cpu_stall_reset - prevent further stall warnings in current grace period |
| 1535 | * |
| 1536 | * Set the stall-warning timeout way off into the future, thus preventing |
| 1537 | * any RCU CPU stall-warning messages from appearing in the current set of |
| 1538 | * RCU grace periods. |
| 1539 | * |
| 1540 | * The caller must disable hard irqs. |
| 1541 | */ |
| 1542 | void rcu_cpu_stall_reset(void) |
| 1543 | { |
Paul E. McKenney | 6ce75a2 | 2012-06-12 11:01:13 -0700 | [diff] [blame] | 1544 | struct rcu_state *rsp; |
| 1545 | |
| 1546 | for_each_rcu_flavor(rsp) |
Paul E. McKenney | 7d0ae80 | 2015-03-03 14:57:58 -0800 | [diff] [blame] | 1547 | WRITE_ONCE(rsp->jiffies_stall, jiffies + ULONG_MAX / 2); |
Paul E. McKenney | 53d84e0 | 2010-08-10 14:28:53 -0700 | [diff] [blame] | 1548 | } |
| 1549 | |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 1550 | /* |
Paul E. McKenney | dc35c89 | 2012-12-03 13:52:00 -0800 | [diff] [blame] | 1551 | * Determine the value that ->completed will have at the end of the |
| 1552 | * next subsequent grace period. This is used to tag callbacks so that |
| 1553 | * a CPU can invoke callbacks in a timely fashion even if that CPU has |
| 1554 | * been dyntick-idle for an extended period with callbacks under the |
| 1555 | * influence of RCU_FAST_NO_HZ. |
| 1556 | * |
| 1557 | * The caller must hold rnp->lock with interrupts disabled. |
| 1558 | */ |
| 1559 | static unsigned long rcu_cbs_completed(struct rcu_state *rsp, |
| 1560 | struct rcu_node *rnp) |
| 1561 | { |
Matthew Wilcox | a32e01e | 2018-01-17 06:24:30 -0800 | [diff] [blame] | 1562 | raw_lockdep_assert_held_rcu_node(rnp); |
Paul E. McKenney | c0b334c | 2017-04-28 12:32:15 -0700 | [diff] [blame] | 1563 | |
Paul E. McKenney | dc35c89 | 2012-12-03 13:52:00 -0800 | [diff] [blame] | 1564 | /* |
| 1565 | * If RCU is idle, we just wait for the next grace period. |
| 1566 | * But we can only be sure that RCU is idle if we are looking |
| 1567 | * at the root rcu_node structure -- otherwise, a new grace |
| 1568 | * period might have started, but just not yet gotten around |
| 1569 | * to initializing the current non-root rcu_node structure. |
| 1570 | */ |
| 1571 | if (rcu_get_root(rsp) == rnp && rnp->gpnum == rnp->completed) |
| 1572 | return rnp->completed + 1; |
| 1573 | |
| 1574 | /* |
Paul E. McKenney | 9036c2f | 2018-04-10 21:17:56 -0700 | [diff] [blame] | 1575 | * If the current rcu_node structure believes that RCU is |
| 1576 | * idle, and if the rcu_state structure does not yet reflect |
| 1577 | * the start of a new grace period, then the next grace period |
| 1578 | * will suffice. The memory barrier is needed to accurately |
| 1579 | * sample the rsp->gpnum, and pairs with the second lock |
| 1580 | * acquisition in rcu_gp_init(), which is augmented with |
| 1581 | * smp_mb__after_unlock_lock() for this purpose. |
| 1582 | */ |
| 1583 | if (rnp->gpnum == rnp->completed) { |
| 1584 | smp_mb(); /* See above block comment. */ |
| 1585 | if (READ_ONCE(rsp->gpnum) == rnp->completed) |
| 1586 | return rnp->completed + 1; |
| 1587 | } |
| 1588 | |
| 1589 | /* |
Paul E. McKenney | dc35c89 | 2012-12-03 13:52:00 -0800 | [diff] [blame] | 1590 | * Otherwise, wait for a possible partial grace period and |
| 1591 | * then the subsequent full grace period. |
| 1592 | */ |
| 1593 | return rnp->completed + 2; |
| 1594 | } |
| 1595 | |
Paul E. McKenney | 41e8059 | 2018-04-12 11:24:09 -0700 | [diff] [blame] | 1596 | /* Trace-event wrapper function for trace_rcu_future_grace_period. */ |
| 1597 | static void trace_rcu_this_gp(struct rcu_node *rnp, struct rcu_data *rdp, |
| 1598 | unsigned long c, const char *s) |
Paul E. McKenney | 0446be4 | 2012-12-30 15:21:01 -0800 | [diff] [blame] | 1599 | { |
| 1600 | trace_rcu_future_grace_period(rdp->rsp->name, rnp->gpnum, |
| 1601 | rnp->completed, c, rnp->level, |
| 1602 | rnp->grplo, rnp->grphi, s); |
| 1603 | } |
| 1604 | |
| 1605 | /* |
Paul E. McKenney | 41e8059 | 2018-04-12 11:24:09 -0700 | [diff] [blame] | 1606 | * Start the specified grace period, as needed to handle newly arrived |
Paul E. McKenney | 0446be4 | 2012-12-30 15:21:01 -0800 | [diff] [blame] | 1607 | * callbacks. The required future grace periods are recorded in each |
Paul E. McKenney | 41e8059 | 2018-04-12 11:24:09 -0700 | [diff] [blame] | 1608 | * rcu_node structure's ->need_future_gp[] field. Returns true if there |
Paul E. McKenney | 48a7639 | 2014-03-11 13:02:16 -0700 | [diff] [blame] | 1609 | * is reason to awaken the grace-period kthread. |
Paul E. McKenney | 0446be4 | 2012-12-30 15:21:01 -0800 | [diff] [blame] | 1610 | * |
Paul E. McKenney | d5cd968 | 2018-04-12 10:45:06 -0700 | [diff] [blame] | 1611 | * The caller must hold the specified rcu_node structure's ->lock, which |
| 1612 | * is why the caller is responsible for waking the grace-period kthread. |
Paul E. McKenney | 0446be4 | 2012-12-30 15:21:01 -0800 | [diff] [blame] | 1613 | */ |
Paul E. McKenney | 41e8059 | 2018-04-12 11:24:09 -0700 | [diff] [blame] | 1614 | static bool rcu_start_this_gp(struct rcu_node *rnp, struct rcu_data *rdp, |
| 1615 | unsigned long c) |
Paul E. McKenney | 0446be4 | 2012-12-30 15:21:01 -0800 | [diff] [blame] | 1616 | { |
Paul E. McKenney | 48a7639 | 2014-03-11 13:02:16 -0700 | [diff] [blame] | 1617 | bool ret = false; |
Paul E. McKenney | d5cd968 | 2018-04-12 10:45:06 -0700 | [diff] [blame] | 1618 | struct rcu_state *rsp = rdp->rsp; |
Paul E. McKenney | 360e0da | 2018-04-12 11:50:41 -0700 | [diff] [blame] | 1619 | struct rcu_node *rnp_root; |
Paul E. McKenney | 0446be4 | 2012-12-30 15:21:01 -0800 | [diff] [blame] | 1620 | |
Paul E. McKenney | 360e0da | 2018-04-12 11:50:41 -0700 | [diff] [blame] | 1621 | /* |
| 1622 | * Use funnel locking to either acquire the root rcu_node |
| 1623 | * structure's lock or bail out if the need for this grace period |
| 1624 | * has already been recorded -- or has already started. If there |
| 1625 | * is already a grace period in progress in a non-leaf node, no |
| 1626 | * recording is needed because the end of the grace period will |
| 1627 | * scan the leaf rcu_node structures. Note that rnp->lock must |
| 1628 | * not be released. |
| 1629 | */ |
Matthew Wilcox | a32e01e | 2018-01-17 06:24:30 -0800 | [diff] [blame] | 1630 | raw_lockdep_assert_held_rcu_node(rnp); |
Paul E. McKenney | 41e8059 | 2018-04-12 11:24:09 -0700 | [diff] [blame] | 1631 | trace_rcu_this_gp(rnp, rdp, c, TPS("Startleaf")); |
Paul E. McKenney | 360e0da | 2018-04-12 11:50:41 -0700 | [diff] [blame] | 1632 | for (rnp_root = rnp; 1; rnp_root = rnp_root->parent) { |
| 1633 | if (rnp_root != rnp) |
| 1634 | raw_spin_lock_rcu_node(rnp_root); |
Paul E. McKenney | 665f08f | 2018-04-19 12:23:34 -0700 | [diff] [blame] | 1635 | WARN_ON_ONCE(ULONG_CMP_LT(rnp_root->gpnum + |
| 1636 | need_future_gp_mask(), c)); |
Paul E. McKenney | 360e0da | 2018-04-12 11:50:41 -0700 | [diff] [blame] | 1637 | if (need_future_gp_element(rnp_root, c) || |
| 1638 | ULONG_CMP_GE(rnp_root->gpnum, c) || |
| 1639 | (rnp != rnp_root && |
| 1640 | rnp_root->gpnum != rnp_root->completed)) { |
| 1641 | trace_rcu_this_gp(rnp_root, rdp, c, TPS("Prestarted")); |
| 1642 | goto unlock_out; |
| 1643 | } |
| 1644 | need_future_gp_element(rnp_root, c) = true; |
| 1645 | if (rnp_root != rnp && rnp_root->parent != NULL) |
| 1646 | raw_spin_unlock_rcu_node(rnp_root); |
| 1647 | if (!rnp_root->parent) |
| 1648 | break; /* At root, and perhaps also leaf. */ |
Paul E. McKenney | 0446be4 | 2012-12-30 15:21:01 -0800 | [diff] [blame] | 1649 | } |
| 1650 | |
Paul E. McKenney | 360e0da | 2018-04-12 11:50:41 -0700 | [diff] [blame] | 1651 | /* If GP already in progress, just leave, otherwise start one. */ |
| 1652 | if (rnp_root->gpnum != rnp_root->completed) { |
| 1653 | trace_rcu_this_gp(rnp_root, rdp, c, TPS("Startedleafroot")); |
Paul E. McKenney | 0446be4 | 2012-12-30 15:21:01 -0800 | [diff] [blame] | 1654 | goto unlock_out; |
| 1655 | } |
Paul E. McKenney | 360e0da | 2018-04-12 11:50:41 -0700 | [diff] [blame] | 1656 | trace_rcu_this_gp(rnp_root, rdp, c, TPS("Startedroot")); |
| 1657 | WRITE_ONCE(rsp->gp_flags, rsp->gp_flags | RCU_GP_FLAG_INIT); |
Paul E. McKenney | 26d950a | 2018-04-21 20:44:11 -0700 | [diff] [blame] | 1658 | rsp->gp_req_activity = jiffies; |
Paul E. McKenney | 360e0da | 2018-04-12 11:50:41 -0700 | [diff] [blame] | 1659 | if (!rsp->gp_kthread) { |
| 1660 | trace_rcu_this_gp(rnp_root, rdp, c, TPS("NoGPkthread")); |
| 1661 | goto unlock_out; |
Paul E. McKenney | 0446be4 | 2012-12-30 15:21:01 -0800 | [diff] [blame] | 1662 | } |
Paul E. McKenney | 360e0da | 2018-04-12 11:50:41 -0700 | [diff] [blame] | 1663 | trace_rcu_grace_period(rsp->name, READ_ONCE(rsp->gpnum), TPS("newreq")); |
| 1664 | ret = true; /* Caller must wake GP kthread. */ |
Paul E. McKenney | 0446be4 | 2012-12-30 15:21:01 -0800 | [diff] [blame] | 1665 | unlock_out: |
| 1666 | if (rnp != rnp_root) |
Boqun Feng | 67c583a7 | 2015-12-29 12:18:47 +0800 | [diff] [blame] | 1667 | raw_spin_unlock_rcu_node(rnp_root); |
Paul E. McKenney | 48a7639 | 2014-03-11 13:02:16 -0700 | [diff] [blame] | 1668 | return ret; |
Paul E. McKenney | 0446be4 | 2012-12-30 15:21:01 -0800 | [diff] [blame] | 1669 | } |
| 1670 | |
| 1671 | /* |
| 1672 | * Clean up any old requests for the just-ended grace period. Also return |
Paul E. McKenney | d1e4f01 | 2017-02-08 14:58:41 -0800 | [diff] [blame] | 1673 | * whether any additional grace periods have been requested. |
Paul E. McKenney | 0446be4 | 2012-12-30 15:21:01 -0800 | [diff] [blame] | 1674 | */ |
Paul E. McKenney | fb31340 | 2018-04-12 07:20:30 -0700 | [diff] [blame] | 1675 | static bool rcu_future_gp_cleanup(struct rcu_state *rsp, struct rcu_node *rnp) |
Paul E. McKenney | 0446be4 | 2012-12-30 15:21:01 -0800 | [diff] [blame] | 1676 | { |
Paul E. McKenney | a508aa5 | 2018-04-19 09:05:50 -0700 | [diff] [blame] | 1677 | unsigned long c = rnp->completed; |
Paul E. McKenney | fb31340 | 2018-04-12 07:20:30 -0700 | [diff] [blame] | 1678 | bool needmore; |
Paul E. McKenney | 0446be4 | 2012-12-30 15:21:01 -0800 | [diff] [blame] | 1679 | struct rcu_data *rdp = this_cpu_ptr(rsp->rda); |
| 1680 | |
Paul E. McKenney | 6f576e2 | 2018-04-18 16:50:31 -0700 | [diff] [blame] | 1681 | need_future_gp_element(rnp, c) = false; |
Paul E. McKenney | fb31340 | 2018-04-12 07:20:30 -0700 | [diff] [blame] | 1682 | needmore = need_any_future_gp(rnp); |
Paul E. McKenney | 41e8059 | 2018-04-12 11:24:09 -0700 | [diff] [blame] | 1683 | trace_rcu_this_gp(rnp, rdp, c, |
| 1684 | needmore ? TPS("CleanupMore") : TPS("Cleanup")); |
Paul E. McKenney | 0446be4 | 2012-12-30 15:21:01 -0800 | [diff] [blame] | 1685 | return needmore; |
| 1686 | } |
| 1687 | |
| 1688 | /* |
Paul E. McKenney | 48a7639 | 2014-03-11 13:02:16 -0700 | [diff] [blame] | 1689 | * Awaken the grace-period kthread for the specified flavor of RCU. |
| 1690 | * Don't do a self-awaken, and don't bother awakening when there is |
| 1691 | * nothing for the grace-period kthread to do (as in several CPUs |
| 1692 | * raced to awaken, and we lost), and finally don't try to awaken |
| 1693 | * a kthread that has not yet been created. |
| 1694 | */ |
| 1695 | static void rcu_gp_kthread_wake(struct rcu_state *rsp) |
| 1696 | { |
| 1697 | if (current == rsp->gp_kthread || |
Paul E. McKenney | 7d0ae80 | 2015-03-03 14:57:58 -0800 | [diff] [blame] | 1698 | !READ_ONCE(rsp->gp_flags) || |
Paul E. McKenney | 48a7639 | 2014-03-11 13:02:16 -0700 | [diff] [blame] | 1699 | !rsp->gp_kthread) |
| 1700 | return; |
Paul Gortmaker | abedf8e | 2016-02-19 09:46:41 +0100 | [diff] [blame] | 1701 | swake_up(&rsp->gp_wq); |
Paul E. McKenney | 48a7639 | 2014-03-11 13:02:16 -0700 | [diff] [blame] | 1702 | } |
| 1703 | |
| 1704 | /* |
Paul E. McKenney | dc35c89 | 2012-12-03 13:52:00 -0800 | [diff] [blame] | 1705 | * If there is room, assign a ->completed number to any callbacks on |
| 1706 | * this CPU that have not already been assigned. Also accelerate any |
| 1707 | * callbacks that were previously assigned a ->completed number that has |
| 1708 | * since proven to be too conservative, which can happen if callbacks get |
| 1709 | * assigned a ->completed number while RCU is idle, but with reference to |
| 1710 | * a non-root rcu_node structure. This function is idempotent, so it does |
Paul E. McKenney | 48a7639 | 2014-03-11 13:02:16 -0700 | [diff] [blame] | 1711 | * not hurt to call it repeatedly. Returns an flag saying that we should |
| 1712 | * awaken the RCU grace-period kthread. |
Paul E. McKenney | dc35c89 | 2012-12-03 13:52:00 -0800 | [diff] [blame] | 1713 | * |
| 1714 | * The caller must hold rnp->lock with interrupts disabled. |
| 1715 | */ |
Paul E. McKenney | 48a7639 | 2014-03-11 13:02:16 -0700 | [diff] [blame] | 1716 | static bool rcu_accelerate_cbs(struct rcu_state *rsp, struct rcu_node *rnp, |
Paul E. McKenney | dc35c89 | 2012-12-03 13:52:00 -0800 | [diff] [blame] | 1717 | struct rcu_data *rdp) |
| 1718 | { |
Paul E. McKenney | 41e8059 | 2018-04-12 11:24:09 -0700 | [diff] [blame] | 1719 | unsigned long c; |
Paul E. McKenney | 15fecf8 | 2017-02-08 12:36:42 -0800 | [diff] [blame] | 1720 | bool ret = false; |
Paul E. McKenney | dc35c89 | 2012-12-03 13:52:00 -0800 | [diff] [blame] | 1721 | |
Matthew Wilcox | a32e01e | 2018-01-17 06:24:30 -0800 | [diff] [blame] | 1722 | raw_lockdep_assert_held_rcu_node(rnp); |
Paul E. McKenney | c0b334c | 2017-04-28 12:32:15 -0700 | [diff] [blame] | 1723 | |
Paul E. McKenney | 15fecf8 | 2017-02-08 12:36:42 -0800 | [diff] [blame] | 1724 | /* If no pending (not yet ready to invoke) callbacks, nothing to do. */ |
| 1725 | if (!rcu_segcblist_pend_cbs(&rdp->cblist)) |
Paul E. McKenney | 48a7639 | 2014-03-11 13:02:16 -0700 | [diff] [blame] | 1726 | return false; |
Paul E. McKenney | dc35c89 | 2012-12-03 13:52:00 -0800 | [diff] [blame] | 1727 | |
| 1728 | /* |
Paul E. McKenney | 15fecf8 | 2017-02-08 12:36:42 -0800 | [diff] [blame] | 1729 | * Callbacks are often registered with incomplete grace-period |
| 1730 | * information. Something about the fact that getting exact |
| 1731 | * information requires acquiring a global lock... RCU therefore |
| 1732 | * makes a conservative estimate of the grace period number at which |
| 1733 | * a given callback will become ready to invoke. The following |
| 1734 | * code checks this estimate and improves it when possible, thus |
| 1735 | * accelerating callback invocation to an earlier grace-period |
| 1736 | * number. |
Paul E. McKenney | dc35c89 | 2012-12-03 13:52:00 -0800 | [diff] [blame] | 1737 | */ |
Paul E. McKenney | 41e8059 | 2018-04-12 11:24:09 -0700 | [diff] [blame] | 1738 | c = rcu_cbs_completed(rsp, rnp); |
| 1739 | if (rcu_segcblist_accelerate(&rdp->cblist, c)) |
| 1740 | ret = rcu_start_this_gp(rnp, rdp, c); |
Paul E. McKenney | 6d4b418 | 2012-11-27 16:55:44 -0800 | [diff] [blame] | 1741 | |
| 1742 | /* Trace depending on how much we were able to accelerate. */ |
Paul E. McKenney | 15fecf8 | 2017-02-08 12:36:42 -0800 | [diff] [blame] | 1743 | if (rcu_segcblist_restempty(&rdp->cblist, RCU_WAIT_TAIL)) |
Steven Rostedt (Red Hat) | f7f7bac | 2013-07-12 17:18:47 -0400 | [diff] [blame] | 1744 | trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("AccWaitCB")); |
Paul E. McKenney | 6d4b418 | 2012-11-27 16:55:44 -0800 | [diff] [blame] | 1745 | else |
Steven Rostedt (Red Hat) | f7f7bac | 2013-07-12 17:18:47 -0400 | [diff] [blame] | 1746 | trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("AccReadyCB")); |
Paul E. McKenney | 48a7639 | 2014-03-11 13:02:16 -0700 | [diff] [blame] | 1747 | return ret; |
Paul E. McKenney | dc35c89 | 2012-12-03 13:52:00 -0800 | [diff] [blame] | 1748 | } |
| 1749 | |
| 1750 | /* |
| 1751 | * Move any callbacks whose grace period has completed to the |
| 1752 | * RCU_DONE_TAIL sublist, then compact the remaining sublists and |
| 1753 | * assign ->completed numbers to any callbacks in the RCU_NEXT_TAIL |
| 1754 | * sublist. This function is idempotent, so it does not hurt to |
| 1755 | * invoke it repeatedly. As long as it is not invoked -too- often... |
Paul E. McKenney | 48a7639 | 2014-03-11 13:02:16 -0700 | [diff] [blame] | 1756 | * Returns true if the RCU grace-period kthread needs to be awakened. |
Paul E. McKenney | dc35c89 | 2012-12-03 13:52:00 -0800 | [diff] [blame] | 1757 | * |
| 1758 | * The caller must hold rnp->lock with interrupts disabled. |
| 1759 | */ |
Paul E. McKenney | 48a7639 | 2014-03-11 13:02:16 -0700 | [diff] [blame] | 1760 | static bool rcu_advance_cbs(struct rcu_state *rsp, struct rcu_node *rnp, |
Paul E. McKenney | dc35c89 | 2012-12-03 13:52:00 -0800 | [diff] [blame] | 1761 | struct rcu_data *rdp) |
| 1762 | { |
Matthew Wilcox | a32e01e | 2018-01-17 06:24:30 -0800 | [diff] [blame] | 1763 | raw_lockdep_assert_held_rcu_node(rnp); |
Paul E. McKenney | c0b334c | 2017-04-28 12:32:15 -0700 | [diff] [blame] | 1764 | |
Paul E. McKenney | 15fecf8 | 2017-02-08 12:36:42 -0800 | [diff] [blame] | 1765 | /* If no pending (not yet ready to invoke) callbacks, nothing to do. */ |
| 1766 | if (!rcu_segcblist_pend_cbs(&rdp->cblist)) |
Paul E. McKenney | 48a7639 | 2014-03-11 13:02:16 -0700 | [diff] [blame] | 1767 | return false; |
Paul E. McKenney | dc35c89 | 2012-12-03 13:52:00 -0800 | [diff] [blame] | 1768 | |
| 1769 | /* |
| 1770 | * Find all callbacks whose ->completed numbers indicate that they |
| 1771 | * are ready to invoke, and put them into the RCU_DONE_TAIL sublist. |
| 1772 | */ |
Paul E. McKenney | 15fecf8 | 2017-02-08 12:36:42 -0800 | [diff] [blame] | 1773 | rcu_segcblist_advance(&rdp->cblist, rnp->completed); |
Paul E. McKenney | dc35c89 | 2012-12-03 13:52:00 -0800 | [diff] [blame] | 1774 | |
| 1775 | /* Classify any remaining callbacks. */ |
Paul E. McKenney | 48a7639 | 2014-03-11 13:02:16 -0700 | [diff] [blame] | 1776 | return rcu_accelerate_cbs(rsp, rnp, rdp); |
Paul E. McKenney | dc35c89 | 2012-12-03 13:52:00 -0800 | [diff] [blame] | 1777 | } |
| 1778 | |
| 1779 | /* |
Paul E. McKenney | ba9fbe9 | 2013-03-19 11:53:31 -0700 | [diff] [blame] | 1780 | * Update CPU-local rcu_data state to record the beginnings and ends of |
| 1781 | * grace periods. The caller must hold the ->lock of the leaf rcu_node |
| 1782 | * structure corresponding to the current CPU, and must have irqs disabled. |
Paul E. McKenney | 48a7639 | 2014-03-11 13:02:16 -0700 | [diff] [blame] | 1783 | * Returns true if the grace-period kthread needs to be awakened. |
Paul E. McKenney | d09b62d | 2009-11-02 13:52:28 -0800 | [diff] [blame] | 1784 | */ |
Paul E. McKenney | 48a7639 | 2014-03-11 13:02:16 -0700 | [diff] [blame] | 1785 | static bool __note_gp_changes(struct rcu_state *rsp, struct rcu_node *rnp, |
| 1786 | struct rcu_data *rdp) |
Paul E. McKenney | d09b62d | 2009-11-02 13:52:28 -0800 | [diff] [blame] | 1787 | { |
Paul E. McKenney | 48a7639 | 2014-03-11 13:02:16 -0700 | [diff] [blame] | 1788 | bool ret; |
Paul E. McKenney | 3563a43 | 2016-07-28 09:39:11 -0700 | [diff] [blame] | 1789 | bool need_gp; |
Paul E. McKenney | 48a7639 | 2014-03-11 13:02:16 -0700 | [diff] [blame] | 1790 | |
Matthew Wilcox | a32e01e | 2018-01-17 06:24:30 -0800 | [diff] [blame] | 1791 | raw_lockdep_assert_held_rcu_node(rnp); |
Paul E. McKenney | c0b334c | 2017-04-28 12:32:15 -0700 | [diff] [blame] | 1792 | |
Paul E. McKenney | ba9fbe9 | 2013-03-19 11:53:31 -0700 | [diff] [blame] | 1793 | /* Handle the ends of any preceding grace periods first. */ |
Paul E. McKenney | e3663b1 | 2014-12-08 20:26:55 -0800 | [diff] [blame] | 1794 | if (rdp->completed == rnp->completed && |
Paul E. McKenney | 7d0ae80 | 2015-03-03 14:57:58 -0800 | [diff] [blame] | 1795 | !unlikely(READ_ONCE(rdp->gpwrap))) { |
Paul E. McKenney | d09b62d | 2009-11-02 13:52:28 -0800 | [diff] [blame] | 1796 | |
Paul E. McKenney | ba9fbe9 | 2013-03-19 11:53:31 -0700 | [diff] [blame] | 1797 | /* No grace period end, so just accelerate recent callbacks. */ |
Paul E. McKenney | 48a7639 | 2014-03-11 13:02:16 -0700 | [diff] [blame] | 1798 | ret = rcu_accelerate_cbs(rsp, rnp, rdp); |
Paul E. McKenney | dc35c89 | 2012-12-03 13:52:00 -0800 | [diff] [blame] | 1799 | |
| 1800 | } else { |
| 1801 | |
| 1802 | /* Advance callbacks. */ |
Paul E. McKenney | 48a7639 | 2014-03-11 13:02:16 -0700 | [diff] [blame] | 1803 | ret = rcu_advance_cbs(rsp, rnp, rdp); |
Paul E. McKenney | d09b62d | 2009-11-02 13:52:28 -0800 | [diff] [blame] | 1804 | |
| 1805 | /* Remember that we saw this grace-period completion. */ |
| 1806 | rdp->completed = rnp->completed; |
Steven Rostedt (Red Hat) | f7f7bac | 2013-07-12 17:18:47 -0400 | [diff] [blame] | 1807 | trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpuend")); |
Paul E. McKenney | d09b62d | 2009-11-02 13:52:28 -0800 | [diff] [blame] | 1808 | } |
Paul E. McKenney | 398ebe6 | 2013-03-19 10:53:14 -0700 | [diff] [blame] | 1809 | |
Paul E. McKenney | 7d0ae80 | 2015-03-03 14:57:58 -0800 | [diff] [blame] | 1810 | if (rdp->gpnum != rnp->gpnum || unlikely(READ_ONCE(rdp->gpwrap))) { |
Paul E. McKenney | 6eaef63 | 2013-03-19 10:08:37 -0700 | [diff] [blame] | 1811 | /* |
| 1812 | * If the current grace period is waiting for this CPU, |
| 1813 | * set up to detect a quiescent state, otherwise don't |
| 1814 | * go looking for one. |
| 1815 | */ |
| 1816 | rdp->gpnum = rnp->gpnum; |
Steven Rostedt (Red Hat) | f7f7bac | 2013-07-12 17:18:47 -0400 | [diff] [blame] | 1817 | trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpustart")); |
Paul E. McKenney | 3563a43 | 2016-07-28 09:39:11 -0700 | [diff] [blame] | 1818 | need_gp = !!(rnp->qsmask & rdp->grpmask); |
| 1819 | rdp->cpu_no_qs.b.norm = need_gp; |
Paul E. McKenney | 9577df9 | 2017-01-26 16:18:07 -0800 | [diff] [blame] | 1820 | rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_dynticks.rcu_qs_ctr); |
Paul E. McKenney | 3563a43 | 2016-07-28 09:39:11 -0700 | [diff] [blame] | 1821 | rdp->core_needs_qs = need_gp; |
Paul E. McKenney | 6eaef63 | 2013-03-19 10:08:37 -0700 | [diff] [blame] | 1822 | zero_cpu_stall_ticks(rdp); |
Paul E. McKenney | 7d0ae80 | 2015-03-03 14:57:58 -0800 | [diff] [blame] | 1823 | WRITE_ONCE(rdp->gpwrap, false); |
Paul E. McKenney | 9b9500d | 2017-08-17 17:05:59 -0700 | [diff] [blame] | 1824 | rcu_gpnum_ovf(rnp, rdp); |
Paul E. McKenney | 6eaef63 | 2013-03-19 10:08:37 -0700 | [diff] [blame] | 1825 | } |
Paul E. McKenney | de30ad5 | 2018-04-26 11:52:09 -0700 | [diff] [blame] | 1826 | if (rdp->gp_seq != rnp->gp_seq) |
| 1827 | rdp->gp_seq = rnp->gp_seq; |
Paul E. McKenney | 48a7639 | 2014-03-11 13:02:16 -0700 | [diff] [blame] | 1828 | return ret; |
Paul E. McKenney | 6eaef63 | 2013-03-19 10:08:37 -0700 | [diff] [blame] | 1829 | } |
| 1830 | |
Paul E. McKenney | d34ea32 | 2013-03-19 11:10:43 -0700 | [diff] [blame] | 1831 | static void note_gp_changes(struct rcu_state *rsp, struct rcu_data *rdp) |
Paul E. McKenney | 6eaef63 | 2013-03-19 10:08:37 -0700 | [diff] [blame] | 1832 | { |
| 1833 | unsigned long flags; |
Paul E. McKenney | 48a7639 | 2014-03-11 13:02:16 -0700 | [diff] [blame] | 1834 | bool needwake; |
Paul E. McKenney | 6eaef63 | 2013-03-19 10:08:37 -0700 | [diff] [blame] | 1835 | struct rcu_node *rnp; |
| 1836 | |
| 1837 | local_irq_save(flags); |
| 1838 | rnp = rdp->mynode; |
Paul E. McKenney | 7d0ae80 | 2015-03-03 14:57:58 -0800 | [diff] [blame] | 1839 | if ((rdp->gpnum == READ_ONCE(rnp->gpnum) && |
| 1840 | rdp->completed == READ_ONCE(rnp->completed) && |
| 1841 | !unlikely(READ_ONCE(rdp->gpwrap))) || /* w/out lock. */ |
Peter Zijlstra | 2a67e74 | 2015-10-08 12:24:23 +0200 | [diff] [blame] | 1842 | !raw_spin_trylock_rcu_node(rnp)) { /* irqs already off, so later. */ |
Paul E. McKenney | 6eaef63 | 2013-03-19 10:08:37 -0700 | [diff] [blame] | 1843 | local_irq_restore(flags); |
| 1844 | return; |
| 1845 | } |
Paul E. McKenney | 48a7639 | 2014-03-11 13:02:16 -0700 | [diff] [blame] | 1846 | needwake = __note_gp_changes(rsp, rnp, rdp); |
Boqun Feng | 67c583a7 | 2015-12-29 12:18:47 +0800 | [diff] [blame] | 1847 | raw_spin_unlock_irqrestore_rcu_node(rnp, flags); |
Paul E. McKenney | 48a7639 | 2014-03-11 13:02:16 -0700 | [diff] [blame] | 1848 | if (needwake) |
| 1849 | rcu_gp_kthread_wake(rsp); |
Paul E. McKenney | 6eaef63 | 2013-03-19 10:08:37 -0700 | [diff] [blame] | 1850 | } |
| 1851 | |
Paul E. McKenney | 0f41c0d | 2015-03-10 18:33:20 -0700 | [diff] [blame] | 1852 | static void rcu_gp_slow(struct rcu_state *rsp, int delay) |
| 1853 | { |
| 1854 | if (delay > 0 && |
Paul E. McKenney | dee4f42 | 2018-04-26 15:30:28 -0700 | [diff] [blame] | 1855 | !(rcu_seq_ctr(rsp->gp_seq) % |
| 1856 | (rcu_num_nodes * PER_RCU_NODE_PERIOD * delay))) |
Paul E. McKenney | 0f41c0d | 2015-03-10 18:33:20 -0700 | [diff] [blame] | 1857 | schedule_timeout_uninterruptible(delay); |
| 1858 | } |
| 1859 | |
Paul E. McKenney | 6eaef63 | 2013-03-19 10:08:37 -0700 | [diff] [blame] | 1860 | /* |
Paul E. McKenney | 45fed3e | 2015-11-07 23:35:00 -0800 | [diff] [blame] | 1861 | * Initialize a new grace period. Return false if no grace period required. |
Paul E. McKenney | 7fdefc1 | 2012-06-22 11:08:41 -0700 | [diff] [blame] | 1862 | */ |
Paul E. McKenney | 45fed3e | 2015-11-07 23:35:00 -0800 | [diff] [blame] | 1863 | static bool rcu_gp_init(struct rcu_state *rsp) |
Paul E. McKenney | 7fdefc1 | 2012-06-22 11:08:41 -0700 | [diff] [blame] | 1864 | { |
Paul E. McKenney | 0aa04b0 | 2015-01-23 21:52:37 -0800 | [diff] [blame] | 1865 | unsigned long oldmask; |
Paul E. McKenney | 7fdefc1 | 2012-06-22 11:08:41 -0700 | [diff] [blame] | 1866 | struct rcu_data *rdp; |
| 1867 | struct rcu_node *rnp = rcu_get_root(rsp); |
| 1868 | |
Paul E. McKenney | 7d0ae80 | 2015-03-03 14:57:58 -0800 | [diff] [blame] | 1869 | WRITE_ONCE(rsp->gp_activity, jiffies); |
Peter Zijlstra | 2a67e74 | 2015-10-08 12:24:23 +0200 | [diff] [blame] | 1870 | raw_spin_lock_irq_rcu_node(rnp); |
Paul E. McKenney | 7d0ae80 | 2015-03-03 14:57:58 -0800 | [diff] [blame] | 1871 | if (!READ_ONCE(rsp->gp_flags)) { |
Paul E. McKenney | f7be820 | 2013-08-08 18:27:52 -0700 | [diff] [blame] | 1872 | /* Spurious wakeup, tell caller to go back to sleep. */ |
Boqun Feng | 67c583a7 | 2015-12-29 12:18:47 +0800 | [diff] [blame] | 1873 | raw_spin_unlock_irq_rcu_node(rnp); |
Paul E. McKenney | 45fed3e | 2015-11-07 23:35:00 -0800 | [diff] [blame] | 1874 | return false; |
Paul E. McKenney | f7be820 | 2013-08-08 18:27:52 -0700 | [diff] [blame] | 1875 | } |
Paul E. McKenney | 7d0ae80 | 2015-03-03 14:57:58 -0800 | [diff] [blame] | 1876 | WRITE_ONCE(rsp->gp_flags, 0); /* Clear all flags: New grace period. */ |
Paul E. McKenney | 7fdefc1 | 2012-06-22 11:08:41 -0700 | [diff] [blame] | 1877 | |
Paul E. McKenney | f7be820 | 2013-08-08 18:27:52 -0700 | [diff] [blame] | 1878 | if (WARN_ON_ONCE(rcu_gp_in_progress(rsp))) { |
| 1879 | /* |
| 1880 | * Grace period already in progress, don't start another. |
| 1881 | * Not supposed to be able to happen. |
| 1882 | */ |
Boqun Feng | 67c583a7 | 2015-12-29 12:18:47 +0800 | [diff] [blame] | 1883 | raw_spin_unlock_irq_rcu_node(rnp); |
Paul E. McKenney | 45fed3e | 2015-11-07 23:35:00 -0800 | [diff] [blame] | 1884 | return false; |
Paul E. McKenney | 7fdefc1 | 2012-06-22 11:08:41 -0700 | [diff] [blame] | 1885 | } |
| 1886 | |
Paul E. McKenney | 7fdefc1 | 2012-06-22 11:08:41 -0700 | [diff] [blame] | 1887 | /* Advance to a new grace period and initialize state. */ |
Paul E. McKenney | 26cdfed | 2013-09-04 10:51:13 -0700 | [diff] [blame] | 1888 | record_gp_stall_check_time(rsp); |
Paul E. McKenney | 765a3f4 | 2014-03-14 16:37:08 -0700 | [diff] [blame] | 1889 | /* Record GP times before starting GP, hence smp_store_release(). */ |
Paul E. McKenney | de30ad5 | 2018-04-26 11:52:09 -0700 | [diff] [blame] | 1890 | WARN_ON_ONCE(rsp->gpnum << RCU_SEQ_CTR_SHIFT != rsp->gp_seq); |
Paul E. McKenney | 765a3f4 | 2014-03-14 16:37:08 -0700 | [diff] [blame] | 1891 | smp_store_release(&rsp->gpnum, rsp->gpnum + 1); |
Paul E. McKenney | de30ad5 | 2018-04-26 11:52:09 -0700 | [diff] [blame] | 1892 | smp_mb(); /* Pairs with barriers in stall-warning code. */ |
| 1893 | rcu_seq_start(&rsp->gp_seq); |
Steven Rostedt (Red Hat) | f7f7bac | 2013-07-12 17:18:47 -0400 | [diff] [blame] | 1894 | trace_rcu_grace_period(rsp->name, rsp->gpnum, TPS("start")); |
Boqun Feng | 67c583a7 | 2015-12-29 12:18:47 +0800 | [diff] [blame] | 1895 | raw_spin_unlock_irq_rcu_node(rnp); |
Paul E. McKenney | 7fdefc1 | 2012-06-22 11:08:41 -0700 | [diff] [blame] | 1896 | |
Paul E. McKenney | 7fdefc1 | 2012-06-22 11:08:41 -0700 | [diff] [blame] | 1897 | /* |
Paul E. McKenney | 0aa04b0 | 2015-01-23 21:52:37 -0800 | [diff] [blame] | 1898 | * Apply per-leaf buffered online and offline operations to the |
| 1899 | * rcu_node tree. Note that this new grace period need not wait |
| 1900 | * for subsequent online CPUs, and that quiescent-state forcing |
| 1901 | * will handle subsequent offline CPUs. |
| 1902 | */ |
| 1903 | rcu_for_each_leaf_node(rsp, rnp) { |
Paul E. McKenney | 0f41c0d | 2015-03-10 18:33:20 -0700 | [diff] [blame] | 1904 | rcu_gp_slow(rsp, gp_preinit_delay); |
Peter Zijlstra | 2a67e74 | 2015-10-08 12:24:23 +0200 | [diff] [blame] | 1905 | raw_spin_lock_irq_rcu_node(rnp); |
Paul E. McKenney | 0aa04b0 | 2015-01-23 21:52:37 -0800 | [diff] [blame] | 1906 | if (rnp->qsmaskinit == rnp->qsmaskinitnext && |
| 1907 | !rnp->wait_blkd_tasks) { |
| 1908 | /* Nothing to do on this leaf rcu_node structure. */ |
Boqun Feng | 67c583a7 | 2015-12-29 12:18:47 +0800 | [diff] [blame] | 1909 | raw_spin_unlock_irq_rcu_node(rnp); |
Paul E. McKenney | 0aa04b0 | 2015-01-23 21:52:37 -0800 | [diff] [blame] | 1910 | continue; |
| 1911 | } |
| 1912 | |
| 1913 | /* Record old state, apply changes to ->qsmaskinit field. */ |
| 1914 | oldmask = rnp->qsmaskinit; |
| 1915 | rnp->qsmaskinit = rnp->qsmaskinitnext; |
| 1916 | |
| 1917 | /* If zero-ness of ->qsmaskinit changed, propagate up tree. */ |
| 1918 | if (!oldmask != !rnp->qsmaskinit) { |
| 1919 | if (!oldmask) /* First online CPU for this rcu_node. */ |
| 1920 | rcu_init_new_rnp(rnp); |
| 1921 | else if (rcu_preempt_has_tasks(rnp)) /* blocked tasks */ |
| 1922 | rnp->wait_blkd_tasks = true; |
| 1923 | else /* Last offline CPU and can propagate. */ |
| 1924 | rcu_cleanup_dead_rnp(rnp); |
| 1925 | } |
| 1926 | |
| 1927 | /* |
| 1928 | * If all waited-on tasks from prior grace period are |
| 1929 | * done, and if all this rcu_node structure's CPUs are |
| 1930 | * still offline, propagate up the rcu_node tree and |
| 1931 | * clear ->wait_blkd_tasks. Otherwise, if one of this |
| 1932 | * rcu_node structure's CPUs has since come back online, |
| 1933 | * simply clear ->wait_blkd_tasks (but rcu_cleanup_dead_rnp() |
| 1934 | * checks for this, so just call it unconditionally). |
| 1935 | */ |
| 1936 | if (rnp->wait_blkd_tasks && |
| 1937 | (!rcu_preempt_has_tasks(rnp) || |
| 1938 | rnp->qsmaskinit)) { |
| 1939 | rnp->wait_blkd_tasks = false; |
| 1940 | rcu_cleanup_dead_rnp(rnp); |
| 1941 | } |
| 1942 | |
Boqun Feng | 67c583a7 | 2015-12-29 12:18:47 +0800 | [diff] [blame] | 1943 | raw_spin_unlock_irq_rcu_node(rnp); |
Paul E. McKenney | 0aa04b0 | 2015-01-23 21:52:37 -0800 | [diff] [blame] | 1944 | } |
Paul E. McKenney | 7fdefc1 | 2012-06-22 11:08:41 -0700 | [diff] [blame] | 1945 | |
| 1946 | /* |
| 1947 | * Set the quiescent-state-needed bits in all the rcu_node |
| 1948 | * structures for all currently online CPUs in breadth-first order, |
| 1949 | * starting from the root rcu_node structure, relying on the layout |
| 1950 | * of the tree within the rsp->node[] array. Note that other CPUs |
| 1951 | * will access only the leaves of the hierarchy, thus seeing that no |
| 1952 | * grace period is in progress, at least until the corresponding |
Paul E. McKenney | 590d175 | 2016-04-10 08:23:24 -0700 | [diff] [blame] | 1953 | * leaf node has been initialized. |
Paul E. McKenney | 7fdefc1 | 2012-06-22 11:08:41 -0700 | [diff] [blame] | 1954 | * |
| 1955 | * The grace period cannot complete until the initialization |
| 1956 | * process finishes, because this kthread handles both. |
| 1957 | */ |
| 1958 | rcu_for_each_node_breadth_first(rsp, rnp) { |
Paul E. McKenney | 0f41c0d | 2015-03-10 18:33:20 -0700 | [diff] [blame] | 1959 | rcu_gp_slow(rsp, gp_init_delay); |
Peter Zijlstra | 2a67e74 | 2015-10-08 12:24:23 +0200 | [diff] [blame] | 1960 | raw_spin_lock_irq_rcu_node(rnp); |
Paul E. McKenney | 7fdefc1 | 2012-06-22 11:08:41 -0700 | [diff] [blame] | 1961 | rdp = this_cpu_ptr(rsp->rda); |
| 1962 | rcu_preempt_check_blocked_tasks(rnp); |
| 1963 | rnp->qsmask = rnp->qsmaskinit; |
Paul E. McKenney | 7d0ae80 | 2015-03-03 14:57:58 -0800 | [diff] [blame] | 1964 | WRITE_ONCE(rnp->gpnum, rsp->gpnum); |
Lai Jiangshan | 3f47da0 | 2015-01-13 15:30:34 +0800 | [diff] [blame] | 1965 | if (WARN_ON_ONCE(rnp->completed != rsp->completed)) |
Paul E. McKenney | 7d0ae80 | 2015-03-03 14:57:58 -0800 | [diff] [blame] | 1966 | WRITE_ONCE(rnp->completed, rsp->completed); |
Paul E. McKenney | de30ad5 | 2018-04-26 11:52:09 -0700 | [diff] [blame] | 1967 | WRITE_ONCE(rnp->gp_seq, rsp->gp_seq); |
Paul E. McKenney | 7fdefc1 | 2012-06-22 11:08:41 -0700 | [diff] [blame] | 1968 | if (rnp == rdp->mynode) |
Paul E. McKenney | 48a7639 | 2014-03-11 13:02:16 -0700 | [diff] [blame] | 1969 | (void)__note_gp_changes(rsp, rnp, rdp); |
Paul E. McKenney | 7fdefc1 | 2012-06-22 11:08:41 -0700 | [diff] [blame] | 1970 | rcu_preempt_boost_start_gp(rnp); |
| 1971 | trace_rcu_grace_period_init(rsp->name, rnp->gpnum, |
| 1972 | rnp->level, rnp->grplo, |
| 1973 | rnp->grphi, rnp->qsmask); |
Boqun Feng | 67c583a7 | 2015-12-29 12:18:47 +0800 | [diff] [blame] | 1974 | raw_spin_unlock_irq_rcu_node(rnp); |
Paul E. McKenney | cee4393 | 2018-03-02 16:35:27 -0800 | [diff] [blame] | 1975 | cond_resched_tasks_rcu_qs(); |
Paul E. McKenney | 7d0ae80 | 2015-03-03 14:57:58 -0800 | [diff] [blame] | 1976 | WRITE_ONCE(rsp->gp_activity, jiffies); |
Paul E. McKenney | 7fdefc1 | 2012-06-22 11:08:41 -0700 | [diff] [blame] | 1977 | } |
| 1978 | |
Paul E. McKenney | 45fed3e | 2015-11-07 23:35:00 -0800 | [diff] [blame] | 1979 | return true; |
Paul E. McKenney | 7fdefc1 | 2012-06-22 11:08:41 -0700 | [diff] [blame] | 1980 | } |
| 1981 | |
| 1982 | /* |
Luis R. Rodriguez | d537422 | 2017-06-20 14:45:47 -0700 | [diff] [blame] | 1983 | * Helper function for swait_event_idle() wakeup at force-quiescent-state |
| 1984 | * time. |
Paul E. McKenney | b9a425c | 2015-07-01 13:50:28 -0700 | [diff] [blame] | 1985 | */ |
| 1986 | static bool rcu_gp_fqs_check_wake(struct rcu_state *rsp, int *gfp) |
| 1987 | { |
| 1988 | struct rcu_node *rnp = rcu_get_root(rsp); |
| 1989 | |
| 1990 | /* Someone like call_rcu() requested a force-quiescent-state scan. */ |
| 1991 | *gfp = READ_ONCE(rsp->gp_flags); |
| 1992 | if (*gfp & RCU_GP_FLAG_FQS) |
| 1993 | return true; |
| 1994 | |
| 1995 | /* The current grace period has completed. */ |
| 1996 | if (!READ_ONCE(rnp->qsmask) && !rcu_preempt_blocked_readers_cgp(rnp)) |
| 1997 | return true; |
| 1998 | |
| 1999 | return false; |
| 2000 | } |
| 2001 | |
| 2002 | /* |
Paul E. McKenney | 4cdfc175 | 2012-06-22 17:06:26 -0700 | [diff] [blame] | 2003 | * Do one round of quiescent-state forcing. |
| 2004 | */ |
Petr Mladek | 77f81fe | 2015-09-09 12:09:49 -0700 | [diff] [blame] | 2005 | static void rcu_gp_fqs(struct rcu_state *rsp, bool first_time) |
Paul E. McKenney | 4cdfc175 | 2012-06-22 17:06:26 -0700 | [diff] [blame] | 2006 | { |
Paul E. McKenney | 4cdfc175 | 2012-06-22 17:06:26 -0700 | [diff] [blame] | 2007 | struct rcu_node *rnp = rcu_get_root(rsp); |
| 2008 | |
Paul E. McKenney | 7d0ae80 | 2015-03-03 14:57:58 -0800 | [diff] [blame] | 2009 | WRITE_ONCE(rsp->gp_activity, jiffies); |
Paul E. McKenney | 4cdfc175 | 2012-06-22 17:06:26 -0700 | [diff] [blame] | 2010 | rsp->n_force_qs++; |
Petr Mladek | 77f81fe | 2015-09-09 12:09:49 -0700 | [diff] [blame] | 2011 | if (first_time) { |
Paul E. McKenney | 4cdfc175 | 2012-06-22 17:06:26 -0700 | [diff] [blame] | 2012 | /* Collect dyntick-idle snapshots. */ |
Paul E. McKenney | fe5ac72 | 2017-05-11 11:26:22 -0700 | [diff] [blame] | 2013 | force_qs_rnp(rsp, dyntick_save_progress_counter); |
Paul E. McKenney | 4cdfc175 | 2012-06-22 17:06:26 -0700 | [diff] [blame] | 2014 | } else { |
| 2015 | /* Handle dyntick-idle and offline CPUs. */ |
Paul E. McKenney | fe5ac72 | 2017-05-11 11:26:22 -0700 | [diff] [blame] | 2016 | force_qs_rnp(rsp, rcu_implicit_dynticks_qs); |
Paul E. McKenney | 4cdfc175 | 2012-06-22 17:06:26 -0700 | [diff] [blame] | 2017 | } |
| 2018 | /* Clear flag to prevent immediate re-entry. */ |
Paul E. McKenney | 7d0ae80 | 2015-03-03 14:57:58 -0800 | [diff] [blame] | 2019 | if (READ_ONCE(rsp->gp_flags) & RCU_GP_FLAG_FQS) { |
Peter Zijlstra | 2a67e74 | 2015-10-08 12:24:23 +0200 | [diff] [blame] | 2020 | raw_spin_lock_irq_rcu_node(rnp); |
Paul E. McKenney | 7d0ae80 | 2015-03-03 14:57:58 -0800 | [diff] [blame] | 2021 | WRITE_ONCE(rsp->gp_flags, |
| 2022 | READ_ONCE(rsp->gp_flags) & ~RCU_GP_FLAG_FQS); |
Boqun Feng | 67c583a7 | 2015-12-29 12:18:47 +0800 | [diff] [blame] | 2023 | raw_spin_unlock_irq_rcu_node(rnp); |
Paul E. McKenney | 4cdfc175 | 2012-06-22 17:06:26 -0700 | [diff] [blame] | 2024 | } |
Paul E. McKenney | 4cdfc175 | 2012-06-22 17:06:26 -0700 | [diff] [blame] | 2025 | } |
| 2026 | |
| 2027 | /* |
Paul E. McKenney | 7fdefc1 | 2012-06-22 11:08:41 -0700 | [diff] [blame] | 2028 | * Clean up after the old grace period. |
| 2029 | */ |
Paul E. McKenney | 4cdfc175 | 2012-06-22 17:06:26 -0700 | [diff] [blame] | 2030 | static void rcu_gp_cleanup(struct rcu_state *rsp) |
Paul E. McKenney | 7fdefc1 | 2012-06-22 11:08:41 -0700 | [diff] [blame] | 2031 | { |
| 2032 | unsigned long gp_duration; |
Paul E. McKenney | 48a7639 | 2014-03-11 13:02:16 -0700 | [diff] [blame] | 2033 | bool needgp = false; |
Paul E. McKenney | de30ad5 | 2018-04-26 11:52:09 -0700 | [diff] [blame] | 2034 | unsigned long new_gp_seq; |
Paul E. McKenney | 7fdefc1 | 2012-06-22 11:08:41 -0700 | [diff] [blame] | 2035 | struct rcu_data *rdp; |
| 2036 | struct rcu_node *rnp = rcu_get_root(rsp); |
Paul Gortmaker | abedf8e | 2016-02-19 09:46:41 +0100 | [diff] [blame] | 2037 | struct swait_queue_head *sq; |
Paul E. McKenney | 7fdefc1 | 2012-06-22 11:08:41 -0700 | [diff] [blame] | 2038 | |
Paul E. McKenney | 7d0ae80 | 2015-03-03 14:57:58 -0800 | [diff] [blame] | 2039 | WRITE_ONCE(rsp->gp_activity, jiffies); |
Peter Zijlstra | 2a67e74 | 2015-10-08 12:24:23 +0200 | [diff] [blame] | 2040 | raw_spin_lock_irq_rcu_node(rnp); |
Paul E. McKenney | 7fdefc1 | 2012-06-22 11:08:41 -0700 | [diff] [blame] | 2041 | gp_duration = jiffies - rsp->gp_start; |
| 2042 | if (gp_duration > rsp->gp_max) |
| 2043 | rsp->gp_max = gp_duration; |
| 2044 | |
| 2045 | /* |
| 2046 | * We know the grace period is complete, but to everyone else |
| 2047 | * it appears to still be ongoing. But it is also the case |
| 2048 | * that to everyone else it looks like there is nothing that |
| 2049 | * they can do to advance the grace period. It is therefore |
| 2050 | * safe for us to drop the lock in order to mark the grace |
| 2051 | * period as completed in all of the rcu_node structures. |
Paul E. McKenney | 7fdefc1 | 2012-06-22 11:08:41 -0700 | [diff] [blame] | 2052 | */ |
Boqun Feng | 67c583a7 | 2015-12-29 12:18:47 +0800 | [diff] [blame] | 2053 | raw_spin_unlock_irq_rcu_node(rnp); |
Paul E. McKenney | 7fdefc1 | 2012-06-22 11:08:41 -0700 | [diff] [blame] | 2054 | |
Paul E. McKenney | 5d4b865 | 2012-07-07 07:56:57 -0700 | [diff] [blame] | 2055 | /* |
| 2056 | * Propagate new ->completed value to rcu_node structures so |
| 2057 | * that other CPUs don't have to wait until the start of the next |
| 2058 | * grace period to process their callbacks. This also avoids |
| 2059 | * some nasty RCU grace-period initialization races by forcing |
| 2060 | * the end of the current grace period to be completely recorded in |
| 2061 | * all of the rcu_node structures before the beginning of the next |
| 2062 | * grace period is recorded in any of the rcu_node structures. |
| 2063 | */ |
Paul E. McKenney | de30ad5 | 2018-04-26 11:52:09 -0700 | [diff] [blame] | 2064 | new_gp_seq = rsp->gp_seq; |
| 2065 | rcu_seq_end(&new_gp_seq); |
Paul E. McKenney | 5d4b865 | 2012-07-07 07:56:57 -0700 | [diff] [blame] | 2066 | rcu_for_each_node_breadth_first(rsp, rnp) { |
Peter Zijlstra | 2a67e74 | 2015-10-08 12:24:23 +0200 | [diff] [blame] | 2067 | raw_spin_lock_irq_rcu_node(rnp); |
Paul E. McKenney | 4bc8d55 | 2017-11-27 15:13:56 -0800 | [diff] [blame] | 2068 | if (WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp))) |
| 2069 | dump_blkd_tasks(rnp, 10); |
Paul E. McKenney | 5c60d25 | 2015-02-09 05:37:47 -0800 | [diff] [blame] | 2070 | WARN_ON_ONCE(rnp->qsmask); |
Paul E. McKenney | 7d0ae80 | 2015-03-03 14:57:58 -0800 | [diff] [blame] | 2071 | WRITE_ONCE(rnp->completed, rsp->gpnum); |
Paul E. McKenney | de30ad5 | 2018-04-26 11:52:09 -0700 | [diff] [blame] | 2072 | WRITE_ONCE(rnp->gp_seq, new_gp_seq); |
Paul E. McKenney | b11cc57 | 2012-12-17 14:21:14 -0800 | [diff] [blame] | 2073 | rdp = this_cpu_ptr(rsp->rda); |
| 2074 | if (rnp == rdp->mynode) |
Paul E. McKenney | 48a7639 | 2014-03-11 13:02:16 -0700 | [diff] [blame] | 2075 | needgp = __note_gp_changes(rsp, rnp, rdp) || needgp; |
Paul E. McKenney | 78e4bc3 | 2013-09-24 15:04:06 -0700 | [diff] [blame] | 2076 | /* smp_mb() provided by prior unlock-lock pair. */ |
Paul E. McKenney | fb31340 | 2018-04-12 07:20:30 -0700 | [diff] [blame] | 2077 | needgp = rcu_future_gp_cleanup(rsp, rnp) || needgp; |
Daniel Wagner | 065bb78 | 2016-02-19 09:46:40 +0100 | [diff] [blame] | 2078 | sq = rcu_nocb_gp_get(rnp); |
Boqun Feng | 67c583a7 | 2015-12-29 12:18:47 +0800 | [diff] [blame] | 2079 | raw_spin_unlock_irq_rcu_node(rnp); |
Daniel Wagner | 065bb78 | 2016-02-19 09:46:40 +0100 | [diff] [blame] | 2080 | rcu_nocb_gp_cleanup(sq); |
Paul E. McKenney | cee4393 | 2018-03-02 16:35:27 -0800 | [diff] [blame] | 2081 | cond_resched_tasks_rcu_qs(); |
Paul E. McKenney | 7d0ae80 | 2015-03-03 14:57:58 -0800 | [diff] [blame] | 2082 | WRITE_ONCE(rsp->gp_activity, jiffies); |
Paul E. McKenney | 0f41c0d | 2015-03-10 18:33:20 -0700 | [diff] [blame] | 2083 | rcu_gp_slow(rsp, gp_cleanup_delay); |
Paul E. McKenney | 7fdefc1 | 2012-06-22 11:08:41 -0700 | [diff] [blame] | 2084 | } |
Paul E. McKenney | 5d4b865 | 2012-07-07 07:56:57 -0700 | [diff] [blame] | 2085 | rnp = rcu_get_root(rsp); |
Paul E. McKenney | de30ad5 | 2018-04-26 11:52:09 -0700 | [diff] [blame] | 2086 | raw_spin_lock_irq_rcu_node(rnp); /* GP before rsp->gp_seq update. */ |
Paul E. McKenney | 7fdefc1 | 2012-06-22 11:08:41 -0700 | [diff] [blame] | 2087 | |
Paul E. McKenney | 765a3f4 | 2014-03-14 16:37:08 -0700 | [diff] [blame] | 2088 | /* Declare grace period done. */ |
Paul E. McKenney | 7d0ae80 | 2015-03-03 14:57:58 -0800 | [diff] [blame] | 2089 | WRITE_ONCE(rsp->completed, rsp->gpnum); |
Paul E. McKenney | de30ad5 | 2018-04-26 11:52:09 -0700 | [diff] [blame] | 2090 | rcu_seq_end(&rsp->gp_seq); |
Steven Rostedt (Red Hat) | f7f7bac | 2013-07-12 17:18:47 -0400 | [diff] [blame] | 2091 | trace_rcu_grace_period(rsp->name, rsp->completed, TPS("end")); |
Petr Mladek | 77f81fe | 2015-09-09 12:09:49 -0700 | [diff] [blame] | 2092 | rsp->gp_state = RCU_GP_IDLE; |
Paul E. McKenney | fb31340 | 2018-04-12 07:20:30 -0700 | [diff] [blame] | 2093 | /* Check for GP requests since above loop. */ |
Paul E. McKenney | 5d4b865 | 2012-07-07 07:56:57 -0700 | [diff] [blame] | 2094 | rdp = this_cpu_ptr(rsp->rda); |
Paul E. McKenney | fb31340 | 2018-04-12 07:20:30 -0700 | [diff] [blame] | 2095 | if (need_any_future_gp(rnp)) { |
Paul E. McKenney | 41e8059 | 2018-04-12 11:24:09 -0700 | [diff] [blame] | 2096 | trace_rcu_this_gp(rnp, rdp, rsp->completed - 1, |
| 2097 | TPS("CleanupMore")); |
Paul E. McKenney | fb31340 | 2018-04-12 07:20:30 -0700 | [diff] [blame] | 2098 | needgp = true; |
| 2099 | } |
Paul E. McKenney | 48a7639 | 2014-03-11 13:02:16 -0700 | [diff] [blame] | 2100 | /* Advance CBs to reduce false positives below. */ |
Paul E. McKenney | 384f77f | 2018-04-12 16:16:45 -0700 | [diff] [blame] | 2101 | if (!rcu_accelerate_cbs(rsp, rnp, rdp) && needgp) { |
Paul E. McKenney | 7d0ae80 | 2015-03-03 14:57:58 -0800 | [diff] [blame] | 2102 | WRITE_ONCE(rsp->gp_flags, RCU_GP_FLAG_INIT); |
Paul E. McKenney | 26d950a | 2018-04-21 20:44:11 -0700 | [diff] [blame] | 2103 | rsp->gp_req_activity = jiffies; |
Paul E. McKenney | 384f77f | 2018-04-12 16:16:45 -0700 | [diff] [blame] | 2104 | trace_rcu_grace_period(rsp->name, READ_ONCE(rsp->gpnum), |
Paul E. McKenney | bb311ec | 2013-08-09 16:02:09 -0700 | [diff] [blame] | 2105 | TPS("newreq")); |
Paul E. McKenney | 18390ae | 2018-04-22 15:06:05 -0700 | [diff] [blame] | 2106 | } else { |
| 2107 | WRITE_ONCE(rsp->gp_flags, rsp->gp_flags & RCU_GP_FLAG_INIT); |
Paul E. McKenney | bb311ec | 2013-08-09 16:02:09 -0700 | [diff] [blame] | 2108 | } |
Boqun Feng | 67c583a7 | 2015-12-29 12:18:47 +0800 | [diff] [blame] | 2109 | raw_spin_unlock_irq_rcu_node(rnp); |
Paul E. McKenney | 7fdefc1 | 2012-06-22 11:08:41 -0700 | [diff] [blame] | 2110 | } |
| 2111 | |
| 2112 | /* |
Paul E. McKenney | b3dbec7 | 2012-06-18 18:36:08 -0700 | [diff] [blame] | 2113 | * Body of kthread that handles grace periods. |
| 2114 | */ |
Paul E. McKenney | 755609a | 2012-06-19 17:18:20 -0700 | [diff] [blame] | 2115 | static int __noreturn rcu_gp_kthread(void *arg) |
Paul E. McKenney | b3dbec7 | 2012-06-18 18:36:08 -0700 | [diff] [blame] | 2116 | { |
Petr Mladek | 77f81fe | 2015-09-09 12:09:49 -0700 | [diff] [blame] | 2117 | bool first_gp_fqs; |
Paul E. McKenney | 88d6df6 | 2013-08-08 21:44:31 -0700 | [diff] [blame] | 2118 | int gf; |
Paul E. McKenney | d40011f | 2012-06-26 20:45:57 -0700 | [diff] [blame] | 2119 | unsigned long j; |
Paul E. McKenney | 4cdfc175 | 2012-06-22 17:06:26 -0700 | [diff] [blame] | 2120 | int ret; |
Paul E. McKenney | b3dbec7 | 2012-06-18 18:36:08 -0700 | [diff] [blame] | 2121 | struct rcu_state *rsp = arg; |
Paul E. McKenney | 7fdefc1 | 2012-06-22 11:08:41 -0700 | [diff] [blame] | 2122 | struct rcu_node *rnp = rcu_get_root(rsp); |
Paul E. McKenney | b3dbec7 | 2012-06-18 18:36:08 -0700 | [diff] [blame] | 2123 | |
Paul E. McKenney | 5871968 | 2015-02-24 11:05:36 -0800 | [diff] [blame] | 2124 | rcu_bind_gp_kthread(); |
Paul E. McKenney | b3dbec7 | 2012-06-18 18:36:08 -0700 | [diff] [blame] | 2125 | for (;;) { |
| 2126 | |
| 2127 | /* Handle grace-period start. */ |
Paul E. McKenney | b3dbec7 | 2012-06-18 18:36:08 -0700 | [diff] [blame] | 2128 | for (;;) { |
Paul E. McKenney | 63c4db7 | 2013-08-09 12:19:29 -0700 | [diff] [blame] | 2129 | trace_rcu_grace_period(rsp->name, |
Paul E. McKenney | 7d0ae80 | 2015-03-03 14:57:58 -0800 | [diff] [blame] | 2130 | READ_ONCE(rsp->gpnum), |
Paul E. McKenney | 63c4db7 | 2013-08-09 12:19:29 -0700 | [diff] [blame] | 2131 | TPS("reqwait")); |
Paul E. McKenney | afea227 | 2014-03-12 07:10:41 -0700 | [diff] [blame] | 2132 | rsp->gp_state = RCU_GP_WAIT_GPS; |
Luis R. Rodriguez | d537422 | 2017-06-20 14:45:47 -0700 | [diff] [blame] | 2133 | swait_event_idle(rsp->gp_wq, READ_ONCE(rsp->gp_flags) & |
| 2134 | RCU_GP_FLAG_INIT); |
Paul E. McKenney | 319362c | 2015-05-19 14:16:52 -0700 | [diff] [blame] | 2135 | rsp->gp_state = RCU_GP_DONE_GPS; |
Paul E. McKenney | 78e4bc3 | 2013-09-24 15:04:06 -0700 | [diff] [blame] | 2136 | /* Locking provides needed memory barrier. */ |
Paul E. McKenney | f7be820 | 2013-08-08 18:27:52 -0700 | [diff] [blame] | 2137 | if (rcu_gp_init(rsp)) |
Paul E. McKenney | b3dbec7 | 2012-06-18 18:36:08 -0700 | [diff] [blame] | 2138 | break; |
Paul E. McKenney | cee4393 | 2018-03-02 16:35:27 -0800 | [diff] [blame] | 2139 | cond_resched_tasks_rcu_qs(); |
Paul E. McKenney | 7d0ae80 | 2015-03-03 14:57:58 -0800 | [diff] [blame] | 2140 | WRITE_ONCE(rsp->gp_activity, jiffies); |
Paul E. McKenney | 73a860c | 2014-08-14 10:28:23 -0700 | [diff] [blame] | 2141 | WARN_ON(signal_pending(current)); |
Paul E. McKenney | 63c4db7 | 2013-08-09 12:19:29 -0700 | [diff] [blame] | 2142 | trace_rcu_grace_period(rsp->name, |
Paul E. McKenney | 7d0ae80 | 2015-03-03 14:57:58 -0800 | [diff] [blame] | 2143 | READ_ONCE(rsp->gpnum), |
Paul E. McKenney | 63c4db7 | 2013-08-09 12:19:29 -0700 | [diff] [blame] | 2144 | TPS("reqwaitsig")); |
Paul E. McKenney | b3dbec7 | 2012-06-18 18:36:08 -0700 | [diff] [blame] | 2145 | } |
Paul E. McKenney | cabc49c | 2012-06-20 17:07:14 -0700 | [diff] [blame] | 2146 | |
Paul E. McKenney | 4cdfc175 | 2012-06-22 17:06:26 -0700 | [diff] [blame] | 2147 | /* Handle quiescent-state forcing. */ |
Petr Mladek | 77f81fe | 2015-09-09 12:09:49 -0700 | [diff] [blame] | 2148 | first_gp_fqs = true; |
Paul E. McKenney | d40011f | 2012-06-26 20:45:57 -0700 | [diff] [blame] | 2149 | j = jiffies_till_first_fqs; |
| 2150 | if (j > HZ) { |
| 2151 | j = HZ; |
| 2152 | jiffies_till_first_fqs = HZ; |
| 2153 | } |
Paul E. McKenney | 88d6df6 | 2013-08-08 21:44:31 -0700 | [diff] [blame] | 2154 | ret = 0; |
Paul E. McKenney | cabc49c | 2012-06-20 17:07:14 -0700 | [diff] [blame] | 2155 | for (;;) { |
Paul E. McKenney | 8c7c482 | 2016-01-03 20:29:57 -0800 | [diff] [blame] | 2156 | if (!ret) { |
Paul E. McKenney | 88d6df6 | 2013-08-08 21:44:31 -0700 | [diff] [blame] | 2157 | rsp->jiffies_force_qs = jiffies + j; |
Paul E. McKenney | 8c7c482 | 2016-01-03 20:29:57 -0800 | [diff] [blame] | 2158 | WRITE_ONCE(rsp->jiffies_kick_kthreads, |
| 2159 | jiffies + 3 * j); |
| 2160 | } |
Paul E. McKenney | 63c4db7 | 2013-08-09 12:19:29 -0700 | [diff] [blame] | 2161 | trace_rcu_grace_period(rsp->name, |
Paul E. McKenney | 7d0ae80 | 2015-03-03 14:57:58 -0800 | [diff] [blame] | 2162 | READ_ONCE(rsp->gpnum), |
Paul E. McKenney | 63c4db7 | 2013-08-09 12:19:29 -0700 | [diff] [blame] | 2163 | TPS("fqswait")); |
Paul E. McKenney | afea227 | 2014-03-12 07:10:41 -0700 | [diff] [blame] | 2164 | rsp->gp_state = RCU_GP_WAIT_FQS; |
Luis R. Rodriguez | d537422 | 2017-06-20 14:45:47 -0700 | [diff] [blame] | 2165 | ret = swait_event_idle_timeout(rsp->gp_wq, |
Paul E. McKenney | b9a425c | 2015-07-01 13:50:28 -0700 | [diff] [blame] | 2166 | rcu_gp_fqs_check_wake(rsp, &gf), j); |
Paul E. McKenney | 32bb1c7 | 2015-07-02 12:27:31 -0700 | [diff] [blame] | 2167 | rsp->gp_state = RCU_GP_DOING_FQS; |
Paul E. McKenney | 78e4bc3 | 2013-09-24 15:04:06 -0700 | [diff] [blame] | 2168 | /* Locking provides needed memory barriers. */ |
Paul E. McKenney | 4cdfc175 | 2012-06-22 17:06:26 -0700 | [diff] [blame] | 2169 | /* If grace period done, leave loop. */ |
Paul E. McKenney | 7d0ae80 | 2015-03-03 14:57:58 -0800 | [diff] [blame] | 2170 | if (!READ_ONCE(rnp->qsmask) && |
Paul E. McKenney | 4cdfc175 | 2012-06-22 17:06:26 -0700 | [diff] [blame] | 2171 | !rcu_preempt_blocked_readers_cgp(rnp)) |
Paul E. McKenney | cabc49c | 2012-06-20 17:07:14 -0700 | [diff] [blame] | 2172 | break; |
Paul E. McKenney | 4cdfc175 | 2012-06-22 17:06:26 -0700 | [diff] [blame] | 2173 | /* If time for quiescent-state forcing, do it. */ |
Paul E. McKenney | 88d6df6 | 2013-08-08 21:44:31 -0700 | [diff] [blame] | 2174 | if (ULONG_CMP_GE(jiffies, rsp->jiffies_force_qs) || |
| 2175 | (gf & RCU_GP_FLAG_FQS)) { |
Paul E. McKenney | 63c4db7 | 2013-08-09 12:19:29 -0700 | [diff] [blame] | 2176 | trace_rcu_grace_period(rsp->name, |
Paul E. McKenney | 7d0ae80 | 2015-03-03 14:57:58 -0800 | [diff] [blame] | 2177 | READ_ONCE(rsp->gpnum), |
Paul E. McKenney | 63c4db7 | 2013-08-09 12:19:29 -0700 | [diff] [blame] | 2178 | TPS("fqsstart")); |
Petr Mladek | 77f81fe | 2015-09-09 12:09:49 -0700 | [diff] [blame] | 2179 | rcu_gp_fqs(rsp, first_gp_fqs); |
| 2180 | first_gp_fqs = false; |
Paul E. McKenney | 63c4db7 | 2013-08-09 12:19:29 -0700 | [diff] [blame] | 2181 | trace_rcu_grace_period(rsp->name, |
Paul E. McKenney | 7d0ae80 | 2015-03-03 14:57:58 -0800 | [diff] [blame] | 2182 | READ_ONCE(rsp->gpnum), |
Paul E. McKenney | 63c4db7 | 2013-08-09 12:19:29 -0700 | [diff] [blame] | 2183 | TPS("fqsend")); |
Paul E. McKenney | cee4393 | 2018-03-02 16:35:27 -0800 | [diff] [blame] | 2184 | cond_resched_tasks_rcu_qs(); |
Paul E. McKenney | 7d0ae80 | 2015-03-03 14:57:58 -0800 | [diff] [blame] | 2185 | WRITE_ONCE(rsp->gp_activity, jiffies); |
Paul E. McKenney | fcfd0a2 | 2016-01-03 16:42:18 -0800 | [diff] [blame] | 2186 | ret = 0; /* Force full wait till next FQS. */ |
| 2187 | j = jiffies_till_next_fqs; |
| 2188 | if (j > HZ) { |
| 2189 | j = HZ; |
| 2190 | jiffies_till_next_fqs = HZ; |
| 2191 | } else if (j < 1) { |
| 2192 | j = 1; |
| 2193 | jiffies_till_next_fqs = 1; |
| 2194 | } |
Paul E. McKenney | 4cdfc175 | 2012-06-22 17:06:26 -0700 | [diff] [blame] | 2195 | } else { |
| 2196 | /* Deal with stray signal. */ |
Paul E. McKenney | cee4393 | 2018-03-02 16:35:27 -0800 | [diff] [blame] | 2197 | cond_resched_tasks_rcu_qs(); |
Paul E. McKenney | 7d0ae80 | 2015-03-03 14:57:58 -0800 | [diff] [blame] | 2198 | WRITE_ONCE(rsp->gp_activity, jiffies); |
Paul E. McKenney | 73a860c | 2014-08-14 10:28:23 -0700 | [diff] [blame] | 2199 | WARN_ON(signal_pending(current)); |
Paul E. McKenney | 63c4db7 | 2013-08-09 12:19:29 -0700 | [diff] [blame] | 2200 | trace_rcu_grace_period(rsp->name, |
Paul E. McKenney | 7d0ae80 | 2015-03-03 14:57:58 -0800 | [diff] [blame] | 2201 | READ_ONCE(rsp->gpnum), |
Paul E. McKenney | 63c4db7 | 2013-08-09 12:19:29 -0700 | [diff] [blame] | 2202 | TPS("fqswaitsig")); |
Paul E. McKenney | fcfd0a2 | 2016-01-03 16:42:18 -0800 | [diff] [blame] | 2203 | ret = 1; /* Keep old FQS timing. */ |
| 2204 | j = jiffies; |
| 2205 | if (time_after(jiffies, rsp->jiffies_force_qs)) |
| 2206 | j = 1; |
| 2207 | else |
| 2208 | j = rsp->jiffies_force_qs - j; |
Paul E. McKenney | d40011f | 2012-06-26 20:45:57 -0700 | [diff] [blame] | 2209 | } |
Paul E. McKenney | cabc49c | 2012-06-20 17:07:14 -0700 | [diff] [blame] | 2210 | } |
Paul E. McKenney | 4cdfc175 | 2012-06-22 17:06:26 -0700 | [diff] [blame] | 2211 | |
| 2212 | /* Handle grace-period end. */ |
Paul E. McKenney | 319362c | 2015-05-19 14:16:52 -0700 | [diff] [blame] | 2213 | rsp->gp_state = RCU_GP_CLEANUP; |
Paul E. McKenney | 4cdfc175 | 2012-06-22 17:06:26 -0700 | [diff] [blame] | 2214 | rcu_gp_cleanup(rsp); |
Paul E. McKenney | 319362c | 2015-05-19 14:16:52 -0700 | [diff] [blame] | 2215 | rsp->gp_state = RCU_GP_CLEANED; |
Paul E. McKenney | b3dbec7 | 2012-06-18 18:36:08 -0700 | [diff] [blame] | 2216 | } |
Paul E. McKenney | b3dbec7 | 2012-06-18 18:36:08 -0700 | [diff] [blame] | 2217 | } |
| 2218 | |
| 2219 | /* |
Paul E. McKenney | 8994515 | 2015-12-10 09:59:43 -0800 | [diff] [blame] | 2220 | * Report a full set of quiescent states to the specified rcu_state data |
| 2221 | * structure. Invoke rcu_gp_kthread_wake() to awaken the grace-period |
| 2222 | * kthread if another grace period is required. Whether we wake |
| 2223 | * the grace-period kthread or it awakens itself for the next round |
| 2224 | * of quiescent-state forcing, that kthread will clean up after the |
| 2225 | * just-completed grace period. Note that the caller must hold rnp->lock, |
| 2226 | * which is released before return. |
Paul E. McKenney | f41d911 | 2009-08-22 13:56:52 -0700 | [diff] [blame] | 2227 | */ |
Paul E. McKenney | d3f6bad | 2009-12-02 12:10:13 -0800 | [diff] [blame] | 2228 | static void rcu_report_qs_rsp(struct rcu_state *rsp, unsigned long flags) |
Paul E. McKenney | fc2219d4 | 2009-09-23 09:50:41 -0700 | [diff] [blame] | 2229 | __releases(rcu_get_root(rsp)->lock) |
Paul E. McKenney | f41d911 | 2009-08-22 13:56:52 -0700 | [diff] [blame] | 2230 | { |
Matthew Wilcox | a32e01e | 2018-01-17 06:24:30 -0800 | [diff] [blame] | 2231 | raw_lockdep_assert_held_rcu_node(rcu_get_root(rsp)); |
Paul E. McKenney | fc2219d4 | 2009-09-23 09:50:41 -0700 | [diff] [blame] | 2232 | WARN_ON_ONCE(!rcu_gp_in_progress(rsp)); |
Paul E. McKenney | cd73ca2 | 2015-03-16 11:53:52 -0700 | [diff] [blame] | 2233 | WRITE_ONCE(rsp->gp_flags, READ_ONCE(rsp->gp_flags) | RCU_GP_FLAG_FQS); |
Boqun Feng | 67c583a7 | 2015-12-29 12:18:47 +0800 | [diff] [blame] | 2234 | raw_spin_unlock_irqrestore_rcu_node(rcu_get_root(rsp), flags); |
Jisheng Zhang | 94d4477 | 2016-06-22 17:19:27 +0800 | [diff] [blame] | 2235 | rcu_gp_kthread_wake(rsp); |
Paul E. McKenney | f41d911 | 2009-08-22 13:56:52 -0700 | [diff] [blame] | 2236 | } |
| 2237 | |
| 2238 | /* |
Paul E. McKenney | d3f6bad | 2009-12-02 12:10:13 -0800 | [diff] [blame] | 2239 | * Similar to rcu_report_qs_rdp(), for which it is a helper function. |
| 2240 | * Allows quiescent states for a group of CPUs to be reported at one go |
| 2241 | * to the specified rcu_node structure, though all the CPUs in the group |
Paul E. McKenney | 654e953 | 2015-03-15 09:19:35 -0700 | [diff] [blame] | 2242 | * must be represented by the same rcu_node structure (which need not be a |
| 2243 | * leaf rcu_node structure, though it often will be). The gps parameter |
| 2244 | * is the grace-period snapshot, which means that the quiescent states |
Paul E. McKenney | c9a24e2 | 2018-04-27 14:54:46 -0700 | [diff] [blame^] | 2245 | * are valid only if rnp->gp_seq is equal to gps. That structure's lock |
Paul E. McKenney | 654e953 | 2015-03-15 09:19:35 -0700 | [diff] [blame] | 2246 | * must be held upon entry, and it is released before return. |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2247 | */ |
| 2248 | static void |
Paul E. McKenney | d3f6bad | 2009-12-02 12:10:13 -0800 | [diff] [blame] | 2249 | rcu_report_qs_rnp(unsigned long mask, struct rcu_state *rsp, |
Paul E. McKenney | 654e953 | 2015-03-15 09:19:35 -0700 | [diff] [blame] | 2250 | struct rcu_node *rnp, unsigned long gps, unsigned long flags) |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2251 | __releases(rnp->lock) |
| 2252 | { |
Paul E. McKenney | 654e953 | 2015-03-15 09:19:35 -0700 | [diff] [blame] | 2253 | unsigned long oldmask = 0; |
Paul E. McKenney | 28ecd58 | 2009-09-18 09:50:17 -0700 | [diff] [blame] | 2254 | struct rcu_node *rnp_c; |
| 2255 | |
Matthew Wilcox | a32e01e | 2018-01-17 06:24:30 -0800 | [diff] [blame] | 2256 | raw_lockdep_assert_held_rcu_node(rnp); |
Paul E. McKenney | c0b334c | 2017-04-28 12:32:15 -0700 | [diff] [blame] | 2257 | |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2258 | /* Walk up the rcu_node hierarchy. */ |
| 2259 | for (;;) { |
Paul E. McKenney | c9a24e2 | 2018-04-27 14:54:46 -0700 | [diff] [blame^] | 2260 | if (!(rnp->qsmask & mask) || rnp->gp_seq != gps) { |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2261 | |
Paul E. McKenney | 654e953 | 2015-03-15 09:19:35 -0700 | [diff] [blame] | 2262 | /* |
| 2263 | * Our bit has already been cleared, or the |
| 2264 | * relevant grace period is already over, so done. |
| 2265 | */ |
Boqun Feng | 67c583a7 | 2015-12-29 12:18:47 +0800 | [diff] [blame] | 2266 | raw_spin_unlock_irqrestore_rcu_node(rnp, flags); |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2267 | return; |
| 2268 | } |
Paul E. McKenney | 654e953 | 2015-03-15 09:19:35 -0700 | [diff] [blame] | 2269 | WARN_ON_ONCE(oldmask); /* Any child must be all zeroed! */ |
Paul E. McKenney | 5b4c11d | 2018-04-13 17:11:44 -0700 | [diff] [blame] | 2270 | WARN_ON_ONCE(!rcu_is_leaf_node(rnp) && |
Paul E. McKenney | 2dee940 | 2017-07-11 21:52:31 -0700 | [diff] [blame] | 2271 | rcu_preempt_blocked_readers_cgp(rnp)); |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2272 | rnp->qsmask &= ~mask; |
Paul E. McKenney | d4c08f2 | 2011-06-25 06:36:56 -0700 | [diff] [blame] | 2273 | trace_rcu_quiescent_state_report(rsp->name, rnp->gpnum, |
| 2274 | mask, rnp->qsmask, rnp->level, |
| 2275 | rnp->grplo, rnp->grphi, |
| 2276 | !!rnp->gp_tasks); |
Paul E. McKenney | 27f4d28 | 2011-02-07 12:47:15 -0800 | [diff] [blame] | 2277 | if (rnp->qsmask != 0 || rcu_preempt_blocked_readers_cgp(rnp)) { |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2278 | |
| 2279 | /* Other bits still set at this level, so done. */ |
Boqun Feng | 67c583a7 | 2015-12-29 12:18:47 +0800 | [diff] [blame] | 2280 | raw_spin_unlock_irqrestore_rcu_node(rnp, flags); |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2281 | return; |
| 2282 | } |
Paul E. McKenney | 4bc8d55 | 2017-11-27 15:13:56 -0800 | [diff] [blame] | 2283 | rnp->completedqs = rnp->gpnum; |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2284 | mask = rnp->grpmask; |
| 2285 | if (rnp->parent == NULL) { |
| 2286 | |
| 2287 | /* No more levels. Exit loop holding root lock. */ |
| 2288 | |
| 2289 | break; |
| 2290 | } |
Boqun Feng | 67c583a7 | 2015-12-29 12:18:47 +0800 | [diff] [blame] | 2291 | raw_spin_unlock_irqrestore_rcu_node(rnp, flags); |
Paul E. McKenney | 28ecd58 | 2009-09-18 09:50:17 -0700 | [diff] [blame] | 2292 | rnp_c = rnp; |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2293 | rnp = rnp->parent; |
Peter Zijlstra | 2a67e74 | 2015-10-08 12:24:23 +0200 | [diff] [blame] | 2294 | raw_spin_lock_irqsave_rcu_node(rnp, flags); |
Paul E. McKenney | 654e953 | 2015-03-15 09:19:35 -0700 | [diff] [blame] | 2295 | oldmask = rnp_c->qsmask; |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2296 | } |
| 2297 | |
| 2298 | /* |
| 2299 | * Get here if we are the last CPU to pass through a quiescent |
Paul E. McKenney | d3f6bad | 2009-12-02 12:10:13 -0800 | [diff] [blame] | 2300 | * state for this grace period. Invoke rcu_report_qs_rsp() |
Paul E. McKenney | f41d911 | 2009-08-22 13:56:52 -0700 | [diff] [blame] | 2301 | * to clean up and start the next grace period if one is needed. |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2302 | */ |
Paul E. McKenney | d3f6bad | 2009-12-02 12:10:13 -0800 | [diff] [blame] | 2303 | rcu_report_qs_rsp(rsp, flags); /* releases rnp->lock. */ |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2304 | } |
| 2305 | |
| 2306 | /* |
Paul E. McKenney | cc99a31 | 2015-02-23 08:59:29 -0800 | [diff] [blame] | 2307 | * Record a quiescent state for all tasks that were previously queued |
| 2308 | * on the specified rcu_node structure and that were blocking the current |
| 2309 | * RCU grace period. The caller must hold the specified rnp->lock with |
| 2310 | * irqs disabled, and this lock is released upon return, but irqs remain |
| 2311 | * disabled. |
| 2312 | */ |
Paul E. McKenney | 0aa04b0 | 2015-01-23 21:52:37 -0800 | [diff] [blame] | 2313 | static void rcu_report_unblock_qs_rnp(struct rcu_state *rsp, |
Paul E. McKenney | cc99a31 | 2015-02-23 08:59:29 -0800 | [diff] [blame] | 2314 | struct rcu_node *rnp, unsigned long flags) |
| 2315 | __releases(rnp->lock) |
| 2316 | { |
Paul E. McKenney | 654e953 | 2015-03-15 09:19:35 -0700 | [diff] [blame] | 2317 | unsigned long gps; |
Paul E. McKenney | cc99a31 | 2015-02-23 08:59:29 -0800 | [diff] [blame] | 2318 | unsigned long mask; |
| 2319 | struct rcu_node *rnp_p; |
| 2320 | |
Matthew Wilcox | a32e01e | 2018-01-17 06:24:30 -0800 | [diff] [blame] | 2321 | raw_lockdep_assert_held_rcu_node(rnp); |
Paul E. McKenney | a77da14 | 2015-03-08 14:52:27 -0700 | [diff] [blame] | 2322 | if (rcu_state_p == &rcu_sched_state || rsp != rcu_state_p || |
| 2323 | rnp->qsmask != 0 || rcu_preempt_blocked_readers_cgp(rnp)) { |
Boqun Feng | 67c583a7 | 2015-12-29 12:18:47 +0800 | [diff] [blame] | 2324 | raw_spin_unlock_irqrestore_rcu_node(rnp, flags); |
Paul E. McKenney | cc99a31 | 2015-02-23 08:59:29 -0800 | [diff] [blame] | 2325 | return; /* Still need more quiescent states! */ |
| 2326 | } |
| 2327 | |
| 2328 | rnp_p = rnp->parent; |
| 2329 | if (rnp_p == NULL) { |
| 2330 | /* |
Paul E. McKenney | a77da14 | 2015-03-08 14:52:27 -0700 | [diff] [blame] | 2331 | * Only one rcu_node structure in the tree, so don't |
| 2332 | * try to report up to its nonexistent parent! |
Paul E. McKenney | cc99a31 | 2015-02-23 08:59:29 -0800 | [diff] [blame] | 2333 | */ |
| 2334 | rcu_report_qs_rsp(rsp, flags); |
| 2335 | return; |
| 2336 | } |
| 2337 | |
Paul E. McKenney | c9a24e2 | 2018-04-27 14:54:46 -0700 | [diff] [blame^] | 2338 | /* Report up the rest of the hierarchy, tracking current ->gp_seq. */ |
| 2339 | gps = rnp->gp_seq; |
Paul E. McKenney | cc99a31 | 2015-02-23 08:59:29 -0800 | [diff] [blame] | 2340 | mask = rnp->grpmask; |
Boqun Feng | 67c583a7 | 2015-12-29 12:18:47 +0800 | [diff] [blame] | 2341 | raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */ |
Peter Zijlstra | 2a67e74 | 2015-10-08 12:24:23 +0200 | [diff] [blame] | 2342 | raw_spin_lock_rcu_node(rnp_p); /* irqs already disabled. */ |
Paul E. McKenney | 654e953 | 2015-03-15 09:19:35 -0700 | [diff] [blame] | 2343 | rcu_report_qs_rnp(mask, rsp, rnp_p, gps, flags); |
Paul E. McKenney | cc99a31 | 2015-02-23 08:59:29 -0800 | [diff] [blame] | 2344 | } |
| 2345 | |
| 2346 | /* |
Paul E. McKenney | d3f6bad | 2009-12-02 12:10:13 -0800 | [diff] [blame] | 2347 | * Record a quiescent state for the specified CPU to that CPU's rcu_data |
Paul E. McKenney | 4b455dc | 2016-01-27 22:44:45 -0800 | [diff] [blame] | 2348 | * structure. This must be called from the specified CPU. |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2349 | */ |
| 2350 | static void |
Paul E. McKenney | d7d6a11 | 2012-08-21 15:00:05 -0700 | [diff] [blame] | 2351 | rcu_report_qs_rdp(int cpu, struct rcu_state *rsp, struct rcu_data *rdp) |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2352 | { |
| 2353 | unsigned long flags; |
| 2354 | unsigned long mask; |
Paul E. McKenney | 48a7639 | 2014-03-11 13:02:16 -0700 | [diff] [blame] | 2355 | bool needwake; |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2356 | struct rcu_node *rnp; |
| 2357 | |
| 2358 | rnp = rdp->mynode; |
Peter Zijlstra | 2a67e74 | 2015-10-08 12:24:23 +0200 | [diff] [blame] | 2359 | raw_spin_lock_irqsave_rcu_node(rnp, flags); |
Paul E. McKenney | c9a24e2 | 2018-04-27 14:54:46 -0700 | [diff] [blame^] | 2360 | if (rdp->cpu_no_qs.b.norm || rdp->gp_seq != rnp->gp_seq || |
| 2361 | rdp->gpwrap) { |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2362 | |
| 2363 | /* |
Paul E. McKenney | e4cc1f2 | 2011-06-27 00:17:43 -0700 | [diff] [blame] | 2364 | * The grace period in which this quiescent state was |
| 2365 | * recorded has ended, so don't report it upwards. |
| 2366 | * We will instead need a new quiescent state that lies |
| 2367 | * within the current grace period. |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2368 | */ |
Paul E. McKenney | 5b74c45 | 2015-08-06 15:16:57 -0700 | [diff] [blame] | 2369 | rdp->cpu_no_qs.b.norm = true; /* need qs for new gp. */ |
Paul E. McKenney | 9577df9 | 2017-01-26 16:18:07 -0800 | [diff] [blame] | 2370 | rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_dynticks.rcu_qs_ctr); |
Boqun Feng | 67c583a7 | 2015-12-29 12:18:47 +0800 | [diff] [blame] | 2371 | raw_spin_unlock_irqrestore_rcu_node(rnp, flags); |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2372 | return; |
| 2373 | } |
| 2374 | mask = rdp->grpmask; |
| 2375 | if ((rnp->qsmask & mask) == 0) { |
Boqun Feng | 67c583a7 | 2015-12-29 12:18:47 +0800 | [diff] [blame] | 2376 | raw_spin_unlock_irqrestore_rcu_node(rnp, flags); |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2377 | } else { |
Paul E. McKenney | bb53e41 | 2015-12-10 09:30:12 -0800 | [diff] [blame] | 2378 | rdp->core_needs_qs = false; |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2379 | |
| 2380 | /* |
| 2381 | * This GP can't end until cpu checks in, so all of our |
| 2382 | * callbacks can be processed during the next GP. |
| 2383 | */ |
Paul E. McKenney | 48a7639 | 2014-03-11 13:02:16 -0700 | [diff] [blame] | 2384 | needwake = rcu_accelerate_cbs(rsp, rnp, rdp); |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2385 | |
Paul E. McKenney | c9a24e2 | 2018-04-27 14:54:46 -0700 | [diff] [blame^] | 2386 | rcu_report_qs_rnp(mask, rsp, rnp, rnp->gp_seq, flags); |
Paul E. McKenney | 654e953 | 2015-03-15 09:19:35 -0700 | [diff] [blame] | 2387 | /* ^^^ Released rnp->lock */ |
Paul E. McKenney | 48a7639 | 2014-03-11 13:02:16 -0700 | [diff] [blame] | 2388 | if (needwake) |
| 2389 | rcu_gp_kthread_wake(rsp); |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2390 | } |
| 2391 | } |
| 2392 | |
| 2393 | /* |
| 2394 | * Check to see if there is a new grace period of which this CPU |
| 2395 | * is not yet aware, and if so, set up local rcu_data state for it. |
| 2396 | * Otherwise, see if this CPU has just passed through its first |
| 2397 | * quiescent state for this grace period, and record that fact if so. |
| 2398 | */ |
| 2399 | static void |
| 2400 | rcu_check_quiescent_state(struct rcu_state *rsp, struct rcu_data *rdp) |
| 2401 | { |
Paul E. McKenney | 05eb552 | 2013-03-19 12:38:24 -0700 | [diff] [blame] | 2402 | /* Check for grace-period ends and beginnings. */ |
| 2403 | note_gp_changes(rsp, rdp); |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2404 | |
| 2405 | /* |
| 2406 | * Does this CPU still need to do its part for current grace period? |
| 2407 | * If no, return and let the other CPUs do their part as well. |
| 2408 | */ |
Paul E. McKenney | 97c668b | 2015-08-06 11:31:51 -0700 | [diff] [blame] | 2409 | if (!rdp->core_needs_qs) |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2410 | return; |
| 2411 | |
| 2412 | /* |
| 2413 | * Was there a quiescent state since the beginning of the grace |
| 2414 | * period? If no, then exit and wait for the next call. |
| 2415 | */ |
Paul E. McKenney | 3a19b46 | 2016-11-30 11:21:21 -0800 | [diff] [blame] | 2416 | if (rdp->cpu_no_qs.b.norm) |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2417 | return; |
| 2418 | |
Paul E. McKenney | d3f6bad | 2009-12-02 12:10:13 -0800 | [diff] [blame] | 2419 | /* |
| 2420 | * Tell RCU we are done (but rcu_report_qs_rdp() will be the |
| 2421 | * judge of that). |
| 2422 | */ |
Paul E. McKenney | d7d6a11 | 2012-08-21 15:00:05 -0700 | [diff] [blame] | 2423 | rcu_report_qs_rdp(rdp->cpu, rsp, rdp); |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2424 | } |
| 2425 | |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2426 | /* |
Paul E. McKenney | b1420f1 | 2012-03-01 13:18:08 -0800 | [diff] [blame] | 2427 | * Trace the fact that this CPU is going offline. |
| 2428 | */ |
| 2429 | static void rcu_cleanup_dying_cpu(struct rcu_state *rsp) |
| 2430 | { |
Paul E. McKenney | 88a4976 | 2017-01-23 12:04:46 -0800 | [diff] [blame] | 2431 | RCU_TRACE(unsigned long mask;) |
| 2432 | RCU_TRACE(struct rcu_data *rdp = this_cpu_ptr(rsp->rda);) |
| 2433 | RCU_TRACE(struct rcu_node *rnp = rdp->mynode;) |
Paul E. McKenney | b1420f1 | 2012-03-01 13:18:08 -0800 | [diff] [blame] | 2434 | |
Paul E. McKenney | ea46351 | 2015-03-03 14:05:26 -0800 | [diff] [blame] | 2435 | if (!IS_ENABLED(CONFIG_HOTPLUG_CPU)) |
| 2436 | return; |
| 2437 | |
Paul E. McKenney | 88a4976 | 2017-01-23 12:04:46 -0800 | [diff] [blame] | 2438 | RCU_TRACE(mask = rdp->grpmask;) |
Paul E. McKenney | e560140 | 2012-01-07 11:03:57 -0800 | [diff] [blame] | 2439 | trace_rcu_grace_period(rsp->name, |
| 2440 | rnp->gpnum + 1 - !!(rnp->qsmask & mask), |
Steven Rostedt (Red Hat) | f7f7bac | 2013-07-12 17:18:47 -0400 | [diff] [blame] | 2441 | TPS("cpuofl")); |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2442 | } |
| 2443 | |
| 2444 | /* |
Paul E. McKenney | 8af3a5e | 2014-10-31 11:22:37 -0700 | [diff] [blame] | 2445 | * All CPUs for the specified rcu_node structure have gone offline, |
| 2446 | * and all tasks that were preempted within an RCU read-side critical |
| 2447 | * section while running on one of those CPUs have since exited their RCU |
| 2448 | * read-side critical section. Some other CPU is reporting this fact with |
| 2449 | * the specified rcu_node structure's ->lock held and interrupts disabled. |
| 2450 | * This function therefore goes up the tree of rcu_node structures, |
| 2451 | * clearing the corresponding bits in the ->qsmaskinit fields. Note that |
| 2452 | * the leaf rcu_node structure's ->qsmaskinit field has already been |
| 2453 | * updated |
| 2454 | * |
| 2455 | * This function does check that the specified rcu_node structure has |
| 2456 | * all CPUs offline and no blocked tasks, so it is OK to invoke it |
| 2457 | * prematurely. That said, invoking it after the fact will cost you |
| 2458 | * a needless lock acquisition. So once it has done its work, don't |
| 2459 | * invoke it again. |
| 2460 | */ |
| 2461 | static void rcu_cleanup_dead_rnp(struct rcu_node *rnp_leaf) |
| 2462 | { |
| 2463 | long mask; |
| 2464 | struct rcu_node *rnp = rnp_leaf; |
| 2465 | |
Matthew Wilcox | a32e01e | 2018-01-17 06:24:30 -0800 | [diff] [blame] | 2466 | raw_lockdep_assert_held_rcu_node(rnp); |
Paul E. McKenney | ea46351 | 2015-03-03 14:05:26 -0800 | [diff] [blame] | 2467 | if (!IS_ENABLED(CONFIG_HOTPLUG_CPU) || |
| 2468 | rnp->qsmaskinit || rcu_preempt_has_tasks(rnp)) |
Paul E. McKenney | 8af3a5e | 2014-10-31 11:22:37 -0700 | [diff] [blame] | 2469 | return; |
| 2470 | for (;;) { |
| 2471 | mask = rnp->grpmask; |
| 2472 | rnp = rnp->parent; |
| 2473 | if (!rnp) |
| 2474 | break; |
Peter Zijlstra | 2a67e74 | 2015-10-08 12:24:23 +0200 | [diff] [blame] | 2475 | raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */ |
Paul E. McKenney | 8af3a5e | 2014-10-31 11:22:37 -0700 | [diff] [blame] | 2476 | rnp->qsmaskinit &= ~mask; |
Paul E. McKenney | 0aa04b0 | 2015-01-23 21:52:37 -0800 | [diff] [blame] | 2477 | rnp->qsmask &= ~mask; |
Paul E. McKenney | 8af3a5e | 2014-10-31 11:22:37 -0700 | [diff] [blame] | 2478 | if (rnp->qsmaskinit) { |
Boqun Feng | 67c583a7 | 2015-12-29 12:18:47 +0800 | [diff] [blame] | 2479 | raw_spin_unlock_rcu_node(rnp); |
| 2480 | /* irqs remain disabled. */ |
Paul E. McKenney | 8af3a5e | 2014-10-31 11:22:37 -0700 | [diff] [blame] | 2481 | return; |
| 2482 | } |
Boqun Feng | 67c583a7 | 2015-12-29 12:18:47 +0800 | [diff] [blame] | 2483 | raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */ |
Paul E. McKenney | 8af3a5e | 2014-10-31 11:22:37 -0700 | [diff] [blame] | 2484 | } |
| 2485 | } |
| 2486 | |
| 2487 | /* |
Paul E. McKenney | e560140 | 2012-01-07 11:03:57 -0800 | [diff] [blame] | 2488 | * The CPU has been completely removed, and some other CPU is reporting |
Paul E. McKenney | a58163d | 2017-06-20 12:11:34 -0700 | [diff] [blame] | 2489 | * this fact from process context. Do the remainder of the cleanup. |
| 2490 | * There can only be one CPU hotplug operation at a time, so no need for |
| 2491 | * explicit locking. |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2492 | */ |
Paul E. McKenney | e560140 | 2012-01-07 11:03:57 -0800 | [diff] [blame] | 2493 | static void rcu_cleanup_dead_cpu(int cpu, struct rcu_state *rsp) |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2494 | { |
Paul E. McKenney | e560140 | 2012-01-07 11:03:57 -0800 | [diff] [blame] | 2495 | struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu); |
Paul E. McKenney | b1420f1 | 2012-03-01 13:18:08 -0800 | [diff] [blame] | 2496 | struct rcu_node *rnp = rdp->mynode; /* Outgoing CPU's rdp & rnp. */ |
Paul E. McKenney | e560140 | 2012-01-07 11:03:57 -0800 | [diff] [blame] | 2497 | |
Paul E. McKenney | ea46351 | 2015-03-03 14:05:26 -0800 | [diff] [blame] | 2498 | if (!IS_ENABLED(CONFIG_HOTPLUG_CPU)) |
| 2499 | return; |
| 2500 | |
Paul E. McKenney | 2036d94 | 2012-01-30 17:02:47 -0800 | [diff] [blame] | 2501 | /* Adjust any no-longer-needed kthreads. */ |
Thomas Gleixner | 5d01bbd | 2012-07-16 10:42:35 +0000 | [diff] [blame] | 2502 | rcu_boost_kthread_setaffinity(rnp, -1); |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2503 | } |
| 2504 | |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2505 | /* |
| 2506 | * Invoke any RCU callbacks that have made it to the end of their grace |
| 2507 | * period. Thottle as specified by rdp->blimit. |
| 2508 | */ |
Paul E. McKenney | 37c72e5 | 2009-10-14 10:15:55 -0700 | [diff] [blame] | 2509 | static void rcu_do_batch(struct rcu_state *rsp, struct rcu_data *rdp) |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2510 | { |
| 2511 | unsigned long flags; |
Paul E. McKenney | 15fecf8 | 2017-02-08 12:36:42 -0800 | [diff] [blame] | 2512 | struct rcu_head *rhp; |
| 2513 | struct rcu_cblist rcl = RCU_CBLIST_INITIALIZER(rcl); |
| 2514 | long bl, count; |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2515 | |
Paul E. McKenney | dc35c89 | 2012-12-03 13:52:00 -0800 | [diff] [blame] | 2516 | /* If no callbacks are ready, just return. */ |
Paul E. McKenney | 15fecf8 | 2017-02-08 12:36:42 -0800 | [diff] [blame] | 2517 | if (!rcu_segcblist_ready_cbs(&rdp->cblist)) { |
| 2518 | trace_rcu_batch_start(rsp->name, |
| 2519 | rcu_segcblist_n_lazy_cbs(&rdp->cblist), |
| 2520 | rcu_segcblist_n_cbs(&rdp->cblist), 0); |
| 2521 | trace_rcu_batch_end(rsp->name, 0, |
| 2522 | !rcu_segcblist_empty(&rdp->cblist), |
Paul E. McKenney | 4968c30 | 2011-12-07 16:32:40 -0800 | [diff] [blame] | 2523 | need_resched(), is_idle_task(current), |
| 2524 | rcu_is_callbacks_kthread()); |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2525 | return; |
Paul E. McKenney | 29c00b4 | 2011-06-17 15:53:19 -0700 | [diff] [blame] | 2526 | } |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2527 | |
| 2528 | /* |
| 2529 | * Extract the list of ready callbacks, disabling to prevent |
Paul E. McKenney | 15fecf8 | 2017-02-08 12:36:42 -0800 | [diff] [blame] | 2530 | * races with call_rcu() from interrupt handlers. Leave the |
| 2531 | * callback counts, as rcu_barrier() needs to be conservative. |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2532 | */ |
| 2533 | local_irq_save(flags); |
Paul E. McKenney | 8146c4e2 | 2012-01-10 14:23:29 -0800 | [diff] [blame] | 2534 | WARN_ON_ONCE(cpu_is_offline(smp_processor_id())); |
Paul E. McKenney | 29c00b4 | 2011-06-17 15:53:19 -0700 | [diff] [blame] | 2535 | bl = rdp->blimit; |
Paul E. McKenney | 15fecf8 | 2017-02-08 12:36:42 -0800 | [diff] [blame] | 2536 | trace_rcu_batch_start(rsp->name, rcu_segcblist_n_lazy_cbs(&rdp->cblist), |
| 2537 | rcu_segcblist_n_cbs(&rdp->cblist), bl); |
| 2538 | rcu_segcblist_extract_done_cbs(&rdp->cblist, &rcl); |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2539 | local_irq_restore(flags); |
| 2540 | |
| 2541 | /* Invoke callbacks. */ |
Paul E. McKenney | 15fecf8 | 2017-02-08 12:36:42 -0800 | [diff] [blame] | 2542 | rhp = rcu_cblist_dequeue(&rcl); |
| 2543 | for (; rhp; rhp = rcu_cblist_dequeue(&rcl)) { |
| 2544 | debug_rcu_head_unqueue(rhp); |
| 2545 | if (__rcu_reclaim(rsp->name, rhp)) |
| 2546 | rcu_cblist_dequeued_lazy(&rcl); |
| 2547 | /* |
| 2548 | * Stop only if limit reached and CPU has something to do. |
| 2549 | * Note: The rcl structure counts down from zero. |
| 2550 | */ |
Paul E. McKenney | 4b27f20 | 2017-05-02 08:45:25 -0700 | [diff] [blame] | 2551 | if (-rcl.len >= bl && |
Paul E. McKenney | dff1672 | 2011-11-29 15:57:13 -0800 | [diff] [blame] | 2552 | (need_resched() || |
| 2553 | (!is_idle_task(current) && !rcu_is_callbacks_kthread()))) |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2554 | break; |
| 2555 | } |
| 2556 | |
| 2557 | local_irq_save(flags); |
Paul E. McKenney | 4b27f20 | 2017-05-02 08:45:25 -0700 | [diff] [blame] | 2558 | count = -rcl.len; |
Paul E. McKenney | 8ef0f37 | 2017-05-02 08:18:40 -0700 | [diff] [blame] | 2559 | trace_rcu_batch_end(rsp->name, count, !!rcl.head, need_resched(), |
| 2560 | is_idle_task(current), rcu_is_callbacks_kthread()); |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2561 | |
Paul E. McKenney | 15fecf8 | 2017-02-08 12:36:42 -0800 | [diff] [blame] | 2562 | /* Update counts and requeue any remaining callbacks. */ |
| 2563 | rcu_segcblist_insert_done_cbs(&rdp->cblist, &rcl); |
Paul E. McKenney | b1420f1 | 2012-03-01 13:18:08 -0800 | [diff] [blame] | 2564 | smp_mb(); /* List handling before counting for rcu_barrier(). */ |
Paul E. McKenney | 15fecf8 | 2017-02-08 12:36:42 -0800 | [diff] [blame] | 2565 | rcu_segcblist_insert_count(&rdp->cblist, &rcl); |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2566 | |
| 2567 | /* Reinstate batch limit if we have worked down the excess. */ |
Paul E. McKenney | 15fecf8 | 2017-02-08 12:36:42 -0800 | [diff] [blame] | 2568 | count = rcu_segcblist_n_cbs(&rdp->cblist); |
| 2569 | if (rdp->blimit == LONG_MAX && count <= qlowmark) |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2570 | rdp->blimit = blimit; |
| 2571 | |
Paul E. McKenney | 37c72e5 | 2009-10-14 10:15:55 -0700 | [diff] [blame] | 2572 | /* Reset ->qlen_last_fqs_check trigger if enough CBs have drained. */ |
Paul E. McKenney | 15fecf8 | 2017-02-08 12:36:42 -0800 | [diff] [blame] | 2573 | if (count == 0 && rdp->qlen_last_fqs_check != 0) { |
Paul E. McKenney | 37c72e5 | 2009-10-14 10:15:55 -0700 | [diff] [blame] | 2574 | rdp->qlen_last_fqs_check = 0; |
| 2575 | rdp->n_force_qs_snap = rsp->n_force_qs; |
Paul E. McKenney | 15fecf8 | 2017-02-08 12:36:42 -0800 | [diff] [blame] | 2576 | } else if (count < rdp->qlen_last_fqs_check - qhimark) |
| 2577 | rdp->qlen_last_fqs_check = count; |
Paul E. McKenney | efd88b0 | 2017-10-19 14:52:41 -0700 | [diff] [blame] | 2578 | |
| 2579 | /* |
| 2580 | * The following usually indicates a double call_rcu(). To track |
| 2581 | * this down, try building with CONFIG_DEBUG_OBJECTS_RCU_HEAD=y. |
| 2582 | */ |
Paul E. McKenney | 15fecf8 | 2017-02-08 12:36:42 -0800 | [diff] [blame] | 2583 | WARN_ON_ONCE(rcu_segcblist_empty(&rdp->cblist) != (count == 0)); |
Paul E. McKenney | 37c72e5 | 2009-10-14 10:15:55 -0700 | [diff] [blame] | 2584 | |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2585 | local_irq_restore(flags); |
| 2586 | |
Paul E. McKenney | e0f2306 | 2011-06-21 01:29:39 -0700 | [diff] [blame] | 2587 | /* Re-invoke RCU core processing if there are callbacks remaining. */ |
Paul E. McKenney | 15fecf8 | 2017-02-08 12:36:42 -0800 | [diff] [blame] | 2588 | if (rcu_segcblist_ready_cbs(&rdp->cblist)) |
Paul E. McKenney | a46e089 | 2011-06-15 15:47:09 -0700 | [diff] [blame] | 2589 | invoke_rcu_core(); |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2590 | } |
| 2591 | |
| 2592 | /* |
| 2593 | * Check to see if this CPU is in a non-context-switch quiescent state |
| 2594 | * (user mode or idle loop for rcu, non-softirq execution for rcu_bh). |
Paul E. McKenney | e0f2306 | 2011-06-21 01:29:39 -0700 | [diff] [blame] | 2595 | * Also schedule RCU core processing. |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2596 | * |
Paul E. McKenney | 9b2e4f1 | 2011-09-30 12:10:22 -0700 | [diff] [blame] | 2597 | * This function must be called from hardirq context. It is normally |
Paul E. McKenney | 5403d36 | 2016-09-01 05:04:24 -0700 | [diff] [blame] | 2598 | * invoked from the scheduling-clock interrupt. |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2599 | */ |
Paul E. McKenney | c3377c2d | 2014-10-21 07:53:02 -0700 | [diff] [blame] | 2600 | void rcu_check_callbacks(int user) |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2601 | { |
Steven Rostedt (Red Hat) | f7f7bac | 2013-07-12 17:18:47 -0400 | [diff] [blame] | 2602 | trace_rcu_utilization(TPS("Start scheduler-tick")); |
Paul E. McKenney | a858af2 | 2012-01-16 13:29:10 -0800 | [diff] [blame] | 2603 | increment_cpu_stall_ticks(); |
Paul E. McKenney | 9b2e4f1 | 2011-09-30 12:10:22 -0700 | [diff] [blame] | 2604 | if (user || rcu_is_cpu_rrupt_from_idle()) { |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2605 | |
| 2606 | /* |
| 2607 | * Get here if this CPU took its interrupt from user |
| 2608 | * mode or from the idle loop, and if this is not a |
| 2609 | * nested interrupt. In this case, the CPU is in |
Paul E. McKenney | d6714c2 | 2009-08-22 13:56:46 -0700 | [diff] [blame] | 2610 | * a quiescent state, so note it. |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2611 | * |
| 2612 | * No memory barrier is required here because both |
Paul E. McKenney | d6714c2 | 2009-08-22 13:56:46 -0700 | [diff] [blame] | 2613 | * rcu_sched_qs() and rcu_bh_qs() reference only CPU-local |
| 2614 | * variables that other CPUs neither access nor modify, |
| 2615 | * at least not while the corresponding CPU is online. |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2616 | */ |
| 2617 | |
Paul E. McKenney | 284a8c9 | 2014-08-14 16:38:46 -0700 | [diff] [blame] | 2618 | rcu_sched_qs(); |
| 2619 | rcu_bh_qs(); |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2620 | |
| 2621 | } else if (!in_softirq()) { |
| 2622 | |
| 2623 | /* |
| 2624 | * Get here if this CPU did not take its interrupt from |
| 2625 | * softirq, in other words, if it is not interrupting |
| 2626 | * a rcu_bh read-side critical section. This is an _bh |
Paul E. McKenney | d6714c2 | 2009-08-22 13:56:46 -0700 | [diff] [blame] | 2627 | * critical section, so note it. |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2628 | */ |
| 2629 | |
Paul E. McKenney | 284a8c9 | 2014-08-14 16:38:46 -0700 | [diff] [blame] | 2630 | rcu_bh_qs(); |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2631 | } |
Paul E. McKenney | 86aea0e | 2014-10-21 08:12:00 -0700 | [diff] [blame] | 2632 | rcu_preempt_check_callbacks(); |
Paul E. McKenney | e3950ec | 2014-10-21 08:03:57 -0700 | [diff] [blame] | 2633 | if (rcu_pending()) |
Paul E. McKenney | a46e089 | 2011-06-15 15:47:09 -0700 | [diff] [blame] | 2634 | invoke_rcu_core(); |
Paul E. McKenney | 8315f42 | 2014-06-27 13:42:20 -0700 | [diff] [blame] | 2635 | if (user) |
| 2636 | rcu_note_voluntary_context_switch(current); |
Steven Rostedt (Red Hat) | f7f7bac | 2013-07-12 17:18:47 -0400 | [diff] [blame] | 2637 | trace_rcu_utilization(TPS("End scheduler-tick")); |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2638 | } |
| 2639 | |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2640 | /* |
| 2641 | * Scan the leaf rcu_node structures, processing dyntick state for any that |
| 2642 | * have not yet encountered a quiescent state, using the function specified. |
Paul E. McKenney | 27f4d28 | 2011-02-07 12:47:15 -0800 | [diff] [blame] | 2643 | * Also initiate boosting for any threads blocked on the root rcu_node. |
| 2644 | * |
Paul E. McKenney | ee47eb9 | 2010-01-04 15:09:07 -0800 | [diff] [blame] | 2645 | * The caller must have suppressed start of new grace periods. |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2646 | */ |
Paul E. McKenney | fe5ac72 | 2017-05-11 11:26:22 -0700 | [diff] [blame] | 2647 | static void force_qs_rnp(struct rcu_state *rsp, int (*f)(struct rcu_data *rsp)) |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2648 | { |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2649 | int cpu; |
| 2650 | unsigned long flags; |
| 2651 | unsigned long mask; |
Paul E. McKenney | a0b6c9a | 2009-09-28 07:46:33 -0700 | [diff] [blame] | 2652 | struct rcu_node *rnp; |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2653 | |
Paul E. McKenney | a0b6c9a | 2009-09-28 07:46:33 -0700 | [diff] [blame] | 2654 | rcu_for_each_leaf_node(rsp, rnp) { |
Paul E. McKenney | cee4393 | 2018-03-02 16:35:27 -0800 | [diff] [blame] | 2655 | cond_resched_tasks_rcu_qs(); |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2656 | mask = 0; |
Peter Zijlstra | 2a67e74 | 2015-10-08 12:24:23 +0200 | [diff] [blame] | 2657 | raw_spin_lock_irqsave_rcu_node(rnp, flags); |
Paul E. McKenney | a0b6c9a | 2009-09-28 07:46:33 -0700 | [diff] [blame] | 2658 | if (rnp->qsmask == 0) { |
Paul E. McKenney | a77da14 | 2015-03-08 14:52:27 -0700 | [diff] [blame] | 2659 | if (rcu_state_p == &rcu_sched_state || |
| 2660 | rsp != rcu_state_p || |
| 2661 | rcu_preempt_blocked_readers_cgp(rnp)) { |
| 2662 | /* |
| 2663 | * No point in scanning bits because they |
| 2664 | * are all zero. But we might need to |
| 2665 | * priority-boost blocked readers. |
| 2666 | */ |
| 2667 | rcu_initiate_boost(rnp, flags); |
| 2668 | /* rcu_initiate_boost() releases rnp->lock */ |
| 2669 | continue; |
| 2670 | } |
| 2671 | if (rnp->parent && |
| 2672 | (rnp->parent->qsmask & rnp->grpmask)) { |
| 2673 | /* |
| 2674 | * Race between grace-period |
| 2675 | * initialization and task exiting RCU |
| 2676 | * read-side critical section: Report. |
| 2677 | */ |
| 2678 | rcu_report_unblock_qs_rnp(rsp, rnp, flags); |
| 2679 | /* rcu_report_unblock_qs_rnp() rlses ->lock */ |
| 2680 | continue; |
| 2681 | } |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2682 | } |
Mark Rutland | bc75e99 | 2016-06-03 15:20:04 +0100 | [diff] [blame] | 2683 | for_each_leaf_node_possible_cpu(rnp, cpu) { |
| 2684 | unsigned long bit = leaf_node_cpu_bit(rnp, cpu); |
Paul E. McKenney | 0edd1b1 | 2013-06-21 16:37:22 -0700 | [diff] [blame] | 2685 | if ((rnp->qsmask & bit) != 0) { |
Paul E. McKenney | fe5ac72 | 2017-05-11 11:26:22 -0700 | [diff] [blame] | 2686 | if (f(per_cpu_ptr(rsp->rda, cpu))) |
Paul E. McKenney | 0edd1b1 | 2013-06-21 16:37:22 -0700 | [diff] [blame] | 2687 | mask |= bit; |
| 2688 | } |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2689 | } |
Paul E. McKenney | 45f014c52e | 2010-01-04 15:09:08 -0800 | [diff] [blame] | 2690 | if (mask != 0) { |
Paul E. McKenney | c9a24e2 | 2018-04-27 14:54:46 -0700 | [diff] [blame^] | 2691 | /* Idle/offline CPUs, report (releases rnp->lock). */ |
| 2692 | rcu_report_qs_rnp(mask, rsp, rnp, rnp->gp_seq, flags); |
Paul E. McKenney | 0aa04b0 | 2015-01-23 21:52:37 -0800 | [diff] [blame] | 2693 | } else { |
| 2694 | /* Nothing to do here, so just drop the lock. */ |
Boqun Feng | 67c583a7 | 2015-12-29 12:18:47 +0800 | [diff] [blame] | 2695 | raw_spin_unlock_irqrestore_rcu_node(rnp, flags); |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2696 | } |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2697 | } |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2698 | } |
| 2699 | |
| 2700 | /* |
| 2701 | * Force quiescent states on reluctant CPUs, and also detect which |
| 2702 | * CPUs are in dyntick-idle mode. |
| 2703 | */ |
Paul E. McKenney | 4cdfc175 | 2012-06-22 17:06:26 -0700 | [diff] [blame] | 2704 | static void force_quiescent_state(struct rcu_state *rsp) |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2705 | { |
| 2706 | unsigned long flags; |
Paul E. McKenney | 394f276 | 2012-06-26 17:00:35 -0700 | [diff] [blame] | 2707 | bool ret; |
| 2708 | struct rcu_node *rnp; |
| 2709 | struct rcu_node *rnp_old = NULL; |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2710 | |
Paul E. McKenney | 394f276 | 2012-06-26 17:00:35 -0700 | [diff] [blame] | 2711 | /* Funnel through hierarchy to reduce memory contention. */ |
Shan Wei | d860d40 | 2014-06-19 14:12:44 -0700 | [diff] [blame] | 2712 | rnp = __this_cpu_read(rsp->rda->mynode); |
Paul E. McKenney | 394f276 | 2012-06-26 17:00:35 -0700 | [diff] [blame] | 2713 | for (; rnp != NULL; rnp = rnp->parent) { |
Paul E. McKenney | 7d0ae80 | 2015-03-03 14:57:58 -0800 | [diff] [blame] | 2714 | ret = (READ_ONCE(rsp->gp_flags) & RCU_GP_FLAG_FQS) || |
Paul E. McKenney | 394f276 | 2012-06-26 17:00:35 -0700 | [diff] [blame] | 2715 | !raw_spin_trylock(&rnp->fqslock); |
| 2716 | if (rnp_old != NULL) |
| 2717 | raw_spin_unlock(&rnp_old->fqslock); |
Paul E. McKenney | d62df57 | 2018-01-10 13:10:49 -0800 | [diff] [blame] | 2718 | if (ret) |
Paul E. McKenney | 394f276 | 2012-06-26 17:00:35 -0700 | [diff] [blame] | 2719 | return; |
Paul E. McKenney | 394f276 | 2012-06-26 17:00:35 -0700 | [diff] [blame] | 2720 | rnp_old = rnp; |
| 2721 | } |
| 2722 | /* rnp_old == rcu_get_root(rsp), rnp == NULL. */ |
| 2723 | |
| 2724 | /* Reached the root of the rcu_node tree, acquire lock. */ |
Peter Zijlstra | 2a67e74 | 2015-10-08 12:24:23 +0200 | [diff] [blame] | 2725 | raw_spin_lock_irqsave_rcu_node(rnp_old, flags); |
Paul E. McKenney | 394f276 | 2012-06-26 17:00:35 -0700 | [diff] [blame] | 2726 | raw_spin_unlock(&rnp_old->fqslock); |
Paul E. McKenney | 7d0ae80 | 2015-03-03 14:57:58 -0800 | [diff] [blame] | 2727 | if (READ_ONCE(rsp->gp_flags) & RCU_GP_FLAG_FQS) { |
Boqun Feng | 67c583a7 | 2015-12-29 12:18:47 +0800 | [diff] [blame] | 2728 | raw_spin_unlock_irqrestore_rcu_node(rnp_old, flags); |
Paul E. McKenney | 4cdfc175 | 2012-06-22 17:06:26 -0700 | [diff] [blame] | 2729 | return; /* Someone beat us to it. */ |
Paul E. McKenney | 46a1e34 | 2010-01-04 15:09:09 -0800 | [diff] [blame] | 2730 | } |
Paul E. McKenney | 7d0ae80 | 2015-03-03 14:57:58 -0800 | [diff] [blame] | 2731 | WRITE_ONCE(rsp->gp_flags, READ_ONCE(rsp->gp_flags) | RCU_GP_FLAG_FQS); |
Boqun Feng | 67c583a7 | 2015-12-29 12:18:47 +0800 | [diff] [blame] | 2732 | raw_spin_unlock_irqrestore_rcu_node(rnp_old, flags); |
Jisheng Zhang | 94d4477 | 2016-06-22 17:19:27 +0800 | [diff] [blame] | 2733 | rcu_gp_kthread_wake(rsp); |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2734 | } |
| 2735 | |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2736 | /* |
Paul E. McKenney | 26d950a | 2018-04-21 20:44:11 -0700 | [diff] [blame] | 2737 | * This function checks for grace-period requests that fail to motivate |
| 2738 | * RCU to come out of its idle mode. |
| 2739 | */ |
| 2740 | static void |
| 2741 | rcu_check_gp_start_stall(struct rcu_state *rsp, struct rcu_node *rnp, |
| 2742 | struct rcu_data *rdp) |
| 2743 | { |
| 2744 | unsigned long flags; |
| 2745 | unsigned long j; |
| 2746 | struct rcu_node *rnp_root = rcu_get_root(rsp); |
| 2747 | static atomic_t warned = ATOMIC_INIT(0); |
| 2748 | |
| 2749 | if (!IS_ENABLED(CONFIG_PROVE_RCU) || |
| 2750 | rcu_gp_in_progress(rsp) || !need_any_future_gp(rcu_get_root(rsp))) |
| 2751 | return; |
| 2752 | j = jiffies; /* Expensive access, and in common case don't get here. */ |
| 2753 | if (time_before(j, READ_ONCE(rsp->gp_req_activity) + HZ) || |
| 2754 | time_before(j, READ_ONCE(rsp->gp_activity) + HZ) || |
| 2755 | atomic_read(&warned)) |
| 2756 | return; |
| 2757 | |
| 2758 | raw_spin_lock_irqsave_rcu_node(rnp, flags); |
| 2759 | j = jiffies; |
| 2760 | if (rcu_gp_in_progress(rsp) || !need_any_future_gp(rcu_get_root(rsp)) || |
| 2761 | time_before(j, READ_ONCE(rsp->gp_req_activity) + HZ) || |
| 2762 | time_before(j, READ_ONCE(rsp->gp_activity) + HZ) || |
| 2763 | atomic_read(&warned)) { |
| 2764 | raw_spin_unlock_irqrestore_rcu_node(rnp, flags); |
| 2765 | return; |
| 2766 | } |
| 2767 | /* Hold onto the leaf lock to make others see warned==1. */ |
| 2768 | |
| 2769 | if (rnp_root != rnp) |
| 2770 | raw_spin_lock_rcu_node(rnp_root); /* irqs already disabled. */ |
| 2771 | j = jiffies; |
| 2772 | if (rcu_gp_in_progress(rsp) || !need_any_future_gp(rcu_get_root(rsp)) || |
| 2773 | time_before(j, rsp->gp_req_activity + HZ) || |
| 2774 | time_before(j, rsp->gp_activity + HZ) || |
| 2775 | atomic_xchg(&warned, 1)) { |
| 2776 | raw_spin_unlock_rcu_node(rnp_root); /* irqs remain disabled. */ |
| 2777 | raw_spin_unlock_irqrestore_rcu_node(rnp, flags); |
| 2778 | return; |
| 2779 | } |
| 2780 | pr_alert("%s: g%lu %d%d%d%d gar:%lu ga:%lu f%#x %s->state:%#lx\n", |
| 2781 | __func__, READ_ONCE(rsp->gpnum), |
| 2782 | need_future_gp_element(rcu_get_root(rsp), 0), |
| 2783 | need_future_gp_element(rcu_get_root(rsp), 1), |
| 2784 | need_future_gp_element(rcu_get_root(rsp), 2), |
| 2785 | need_future_gp_element(rcu_get_root(rsp), 3), |
| 2786 | j - rsp->gp_req_activity, j - rsp->gp_activity, |
| 2787 | rsp->gp_flags, rsp->name, |
| 2788 | rsp->gp_kthread ? rsp->gp_kthread->state : 0x1ffffL); |
| 2789 | WARN_ON(1); |
| 2790 | if (rnp_root != rnp) |
| 2791 | raw_spin_unlock_rcu_node(rnp_root); |
| 2792 | raw_spin_unlock_irqrestore_rcu_node(rnp, flags); |
| 2793 | } |
| 2794 | |
| 2795 | /* |
Paul E. McKenney | e0f2306 | 2011-06-21 01:29:39 -0700 | [diff] [blame] | 2796 | * This does the RCU core processing work for the specified rcu_state |
| 2797 | * and rcu_data structures. This may be called only from the CPU to |
| 2798 | * whom the rdp belongs. |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2799 | */ |
| 2800 | static void |
Paul E. McKenney | 1bca8cf | 2012-06-12 09:40:38 -0700 | [diff] [blame] | 2801 | __rcu_process_callbacks(struct rcu_state *rsp) |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2802 | { |
| 2803 | unsigned long flags; |
Paul E. McKenney | 48a7639 | 2014-03-11 13:02:16 -0700 | [diff] [blame] | 2804 | bool needwake; |
Christoph Lameter | fa07a58 | 2014-04-15 12:20:12 -0700 | [diff] [blame] | 2805 | struct rcu_data *rdp = raw_cpu_ptr(rsp->rda); |
Paul E. McKenney | 26d950a | 2018-04-21 20:44:11 -0700 | [diff] [blame] | 2806 | struct rcu_node *rnp = rdp->mynode; |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2807 | |
Nicholas Mc Guire | 50dc7de | 2017-03-25 20:46:01 +0100 | [diff] [blame] | 2808 | WARN_ON_ONCE(!rdp->beenonline); |
Paul E. McKenney | 2e59755 | 2009-08-15 09:53:48 -0700 | [diff] [blame] | 2809 | |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2810 | /* Update RCU state based on any recent quiescent states. */ |
| 2811 | rcu_check_quiescent_state(rsp, rdp); |
| 2812 | |
Paul E. McKenney | bd7af84 | 2018-04-11 09:51:20 -0700 | [diff] [blame] | 2813 | /* No grace period and unregistered callbacks? */ |
| 2814 | if (!rcu_gp_in_progress(rsp) && |
| 2815 | rcu_segcblist_is_enabled(&rdp->cblist)) { |
| 2816 | local_irq_save(flags); |
| 2817 | if (rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL)) { |
| 2818 | local_irq_restore(flags); |
| 2819 | } else { |
Paul E. McKenney | bd7af84 | 2018-04-11 09:51:20 -0700 | [diff] [blame] | 2820 | raw_spin_lock_rcu_node(rnp); /* irqs disabled. */ |
| 2821 | needwake = rcu_accelerate_cbs(rsp, rnp, rdp); |
| 2822 | raw_spin_unlock_irqrestore_rcu_node(rnp, flags); |
| 2823 | if (needwake) |
| 2824 | rcu_gp_kthread_wake(rsp); |
| 2825 | } |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2826 | } |
| 2827 | |
Paul E. McKenney | 26d950a | 2018-04-21 20:44:11 -0700 | [diff] [blame] | 2828 | rcu_check_gp_start_stall(rsp, rnp, rdp); |
| 2829 | |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2830 | /* If there are callbacks ready, invoke them. */ |
Paul E. McKenney | 15fecf8 | 2017-02-08 12:36:42 -0800 | [diff] [blame] | 2831 | if (rcu_segcblist_ready_cbs(&rdp->cblist)) |
Paul E. McKenney | a46e089 | 2011-06-15 15:47:09 -0700 | [diff] [blame] | 2832 | invoke_rcu_callbacks(rsp, rdp); |
Paul E. McKenney | 96d3fd0 | 2013-10-04 14:33:34 -0700 | [diff] [blame] | 2833 | |
| 2834 | /* Do any needed deferred wakeups of rcuo kthreads. */ |
| 2835 | do_nocb_deferred_wakeup(rdp); |
Shaohua Li | 0922337 | 2011-06-14 13:26:25 +0800 | [diff] [blame] | 2836 | } |
| 2837 | |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2838 | /* |
Paul E. McKenney | e0f2306 | 2011-06-21 01:29:39 -0700 | [diff] [blame] | 2839 | * Do RCU core processing for the current CPU. |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2840 | */ |
Emese Revfy | 0766f78 | 2016-06-20 20:42:34 +0200 | [diff] [blame] | 2841 | static __latent_entropy void rcu_process_callbacks(struct softirq_action *unused) |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2842 | { |
Paul E. McKenney | 6ce75a2 | 2012-06-12 11:01:13 -0700 | [diff] [blame] | 2843 | struct rcu_state *rsp; |
| 2844 | |
Paul E. McKenney | bfa00b4 | 2012-06-21 09:54:10 -0700 | [diff] [blame] | 2845 | if (cpu_is_offline(smp_processor_id())) |
| 2846 | return; |
Steven Rostedt (Red Hat) | f7f7bac | 2013-07-12 17:18:47 -0400 | [diff] [blame] | 2847 | trace_rcu_utilization(TPS("Start RCU core")); |
Paul E. McKenney | 6ce75a2 | 2012-06-12 11:01:13 -0700 | [diff] [blame] | 2848 | for_each_rcu_flavor(rsp) |
| 2849 | __rcu_process_callbacks(rsp); |
Steven Rostedt (Red Hat) | f7f7bac | 2013-07-12 17:18:47 -0400 | [diff] [blame] | 2850 | trace_rcu_utilization(TPS("End RCU core")); |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2851 | } |
| 2852 | |
Paul E. McKenney | a26ac24 | 2011-01-12 14:10:23 -0800 | [diff] [blame] | 2853 | /* |
Paul E. McKenney | e0f2306 | 2011-06-21 01:29:39 -0700 | [diff] [blame] | 2854 | * Schedule RCU callback invocation. If the specified type of RCU |
| 2855 | * does not support RCU priority boosting, just do a direct call, |
| 2856 | * otherwise wake up the per-CPU kernel kthread. Note that because we |
Paul E. McKenney | 924df8a | 2014-10-29 15:37:58 -0700 | [diff] [blame] | 2857 | * are running on the current CPU with softirqs disabled, the |
Paul E. McKenney | e0f2306 | 2011-06-21 01:29:39 -0700 | [diff] [blame] | 2858 | * rcu_cpu_kthread_task cannot disappear out from under us. |
Paul E. McKenney | a26ac24 | 2011-01-12 14:10:23 -0800 | [diff] [blame] | 2859 | */ |
Paul E. McKenney | a46e089 | 2011-06-15 15:47:09 -0700 | [diff] [blame] | 2860 | static void invoke_rcu_callbacks(struct rcu_state *rsp, struct rcu_data *rdp) |
Paul E. McKenney | a26ac24 | 2011-01-12 14:10:23 -0800 | [diff] [blame] | 2861 | { |
Paul E. McKenney | 7d0ae80 | 2015-03-03 14:57:58 -0800 | [diff] [blame] | 2862 | if (unlikely(!READ_ONCE(rcu_scheduler_fully_active))) |
Paul E. McKenney | b0d3041 | 2011-07-10 15:57:35 -0700 | [diff] [blame] | 2863 | return; |
Paul E. McKenney | a46e089 | 2011-06-15 15:47:09 -0700 | [diff] [blame] | 2864 | if (likely(!rsp->boost)) { |
| 2865 | rcu_do_batch(rsp, rdp); |
Paul E. McKenney | a26ac24 | 2011-01-12 14:10:23 -0800 | [diff] [blame] | 2866 | return; |
| 2867 | } |
Paul E. McKenney | a46e089 | 2011-06-15 15:47:09 -0700 | [diff] [blame] | 2868 | invoke_rcu_callbacks_kthread(); |
Paul E. McKenney | a26ac24 | 2011-01-12 14:10:23 -0800 | [diff] [blame] | 2869 | } |
| 2870 | |
Paul E. McKenney | a46e089 | 2011-06-15 15:47:09 -0700 | [diff] [blame] | 2871 | static void invoke_rcu_core(void) |
Shaohua Li | 0922337 | 2011-06-14 13:26:25 +0800 | [diff] [blame] | 2872 | { |
Paul E. McKenney | b0f7403 | 2013-02-04 12:14:24 -0800 | [diff] [blame] | 2873 | if (cpu_online(smp_processor_id())) |
| 2874 | raise_softirq(RCU_SOFTIRQ); |
Shaohua Li | 0922337 | 2011-06-14 13:26:25 +0800 | [diff] [blame] | 2875 | } |
| 2876 | |
Paul E. McKenney | 29154c5 | 2012-05-30 03:21:48 -0700 | [diff] [blame] | 2877 | /* |
| 2878 | * Handle any core-RCU processing required by a call_rcu() invocation. |
| 2879 | */ |
| 2880 | static void __call_rcu_core(struct rcu_state *rsp, struct rcu_data *rdp, |
| 2881 | struct rcu_head *head, unsigned long flags) |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2882 | { |
Paul E. McKenney | 48a7639 | 2014-03-11 13:02:16 -0700 | [diff] [blame] | 2883 | bool needwake; |
| 2884 | |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2885 | /* |
Paul E. McKenney | 29154c5 | 2012-05-30 03:21:48 -0700 | [diff] [blame] | 2886 | * If called from an extended quiescent state, invoke the RCU |
| 2887 | * core in order to force a re-evaluation of RCU's idleness. |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2888 | */ |
Yao Dongdong | 9910aff | 2015-02-25 17:09:46 +0800 | [diff] [blame] | 2889 | if (!rcu_is_watching()) |
Paul E. McKenney | 29154c5 | 2012-05-30 03:21:48 -0700 | [diff] [blame] | 2890 | invoke_rcu_core(); |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2891 | |
Paul E. McKenney | 29154c5 | 2012-05-30 03:21:48 -0700 | [diff] [blame] | 2892 | /* If interrupts were disabled or CPU offline, don't invoke RCU core. */ |
| 2893 | if (irqs_disabled_flags(flags) || cpu_is_offline(smp_processor_id())) |
Paul E. McKenney | 2655d57 | 2011-04-07 22:47:23 -0700 | [diff] [blame] | 2894 | return; |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2895 | |
Paul E. McKenney | 37c72e5 | 2009-10-14 10:15:55 -0700 | [diff] [blame] | 2896 | /* |
| 2897 | * Force the grace period if too many callbacks or too long waiting. |
| 2898 | * Enforce hysteresis, and don't invoke force_quiescent_state() |
| 2899 | * if some other CPU has recently done so. Also, don't bother |
| 2900 | * invoking force_quiescent_state() if the newly enqueued callback |
| 2901 | * is the only one waiting for a grace period to complete. |
| 2902 | */ |
Paul E. McKenney | 15fecf8 | 2017-02-08 12:36:42 -0800 | [diff] [blame] | 2903 | if (unlikely(rcu_segcblist_n_cbs(&rdp->cblist) > |
| 2904 | rdp->qlen_last_fqs_check + qhimark)) { |
Paul E. McKenney | b52573d | 2010-12-14 17:36:02 -0800 | [diff] [blame] | 2905 | |
| 2906 | /* Are we ignoring a completed grace period? */ |
Paul E. McKenney | 470716f | 2013-03-19 11:32:11 -0700 | [diff] [blame] | 2907 | note_gp_changes(rsp, rdp); |
Paul E. McKenney | b52573d | 2010-12-14 17:36:02 -0800 | [diff] [blame] | 2908 | |
| 2909 | /* Start a new grace period if one not already started. */ |
| 2910 | if (!rcu_gp_in_progress(rsp)) { |
Paul E. McKenney | a6058d8 | 2018-04-11 14:33:18 -0700 | [diff] [blame] | 2911 | struct rcu_node *rnp = rdp->mynode; |
Paul E. McKenney | b52573d | 2010-12-14 17:36:02 -0800 | [diff] [blame] | 2912 | |
Paul E. McKenney | a6058d8 | 2018-04-11 14:33:18 -0700 | [diff] [blame] | 2913 | raw_spin_lock_rcu_node(rnp); |
| 2914 | needwake = rcu_accelerate_cbs(rsp, rnp, rdp); |
| 2915 | raw_spin_unlock_rcu_node(rnp); |
Paul E. McKenney | 48a7639 | 2014-03-11 13:02:16 -0700 | [diff] [blame] | 2916 | if (needwake) |
| 2917 | rcu_gp_kthread_wake(rsp); |
Paul E. McKenney | b52573d | 2010-12-14 17:36:02 -0800 | [diff] [blame] | 2918 | } else { |
| 2919 | /* Give the grace period a kick. */ |
| 2920 | rdp->blimit = LONG_MAX; |
| 2921 | if (rsp->n_force_qs == rdp->n_force_qs_snap && |
Paul E. McKenney | 15fecf8 | 2017-02-08 12:36:42 -0800 | [diff] [blame] | 2922 | rcu_segcblist_first_pend_cb(&rdp->cblist) != head) |
Paul E. McKenney | 4cdfc175 | 2012-06-22 17:06:26 -0700 | [diff] [blame] | 2923 | force_quiescent_state(rsp); |
Paul E. McKenney | b52573d | 2010-12-14 17:36:02 -0800 | [diff] [blame] | 2924 | rdp->n_force_qs_snap = rsp->n_force_qs; |
Paul E. McKenney | 15fecf8 | 2017-02-08 12:36:42 -0800 | [diff] [blame] | 2925 | rdp->qlen_last_fqs_check = rcu_segcblist_n_cbs(&rdp->cblist); |
Paul E. McKenney | b52573d | 2010-12-14 17:36:02 -0800 | [diff] [blame] | 2926 | } |
Paul E. McKenney | 4cdfc175 | 2012-06-22 17:06:26 -0700 | [diff] [blame] | 2927 | } |
Paul E. McKenney | 29154c5 | 2012-05-30 03:21:48 -0700 | [diff] [blame] | 2928 | } |
| 2929 | |
Paul E. McKenney | 3fbfbf7 | 2012-08-19 21:35:53 -0700 | [diff] [blame] | 2930 | /* |
Paul E. McKenney | ae15018 | 2013-04-23 13:20:57 -0700 | [diff] [blame] | 2931 | * RCU callback function to leak a callback. |
| 2932 | */ |
| 2933 | static void rcu_leak_callback(struct rcu_head *rhp) |
| 2934 | { |
| 2935 | } |
| 2936 | |
| 2937 | /* |
Paul E. McKenney | 3fbfbf7 | 2012-08-19 21:35:53 -0700 | [diff] [blame] | 2938 | * Helper function for call_rcu() and friends. The cpu argument will |
| 2939 | * normally be -1, indicating "currently running CPU". It may specify |
| 2940 | * a CPU only if that CPU is a no-CBs CPU. Currently, only _rcu_barrier() |
| 2941 | * is expected to specify a CPU. |
| 2942 | */ |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2943 | static void |
Boqun Feng | b6a4ae7 | 2015-07-29 13:29:38 +0800 | [diff] [blame] | 2944 | __call_rcu(struct rcu_head *head, rcu_callback_t func, |
Paul E. McKenney | 3fbfbf7 | 2012-08-19 21:35:53 -0700 | [diff] [blame] | 2945 | struct rcu_state *rsp, int cpu, bool lazy) |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2946 | { |
| 2947 | unsigned long flags; |
| 2948 | struct rcu_data *rdp; |
| 2949 | |
Paul E. McKenney | b8f2ed5 | 2016-08-23 06:51:47 -0700 | [diff] [blame] | 2950 | /* Misaligned rcu_head! */ |
| 2951 | WARN_ON_ONCE((unsigned long)head & (sizeof(void *) - 1)); |
| 2952 | |
Paul E. McKenney | ae15018 | 2013-04-23 13:20:57 -0700 | [diff] [blame] | 2953 | if (debug_rcu_head_queue(head)) { |
Paul E. McKenney | fa3c664 | 2017-05-03 11:38:55 -0700 | [diff] [blame] | 2954 | /* |
| 2955 | * Probable double call_rcu(), so leak the callback. |
| 2956 | * Use rcu:rcu_callback trace event to find the previous |
| 2957 | * time callback was passed to __call_rcu(). |
| 2958 | */ |
| 2959 | WARN_ONCE(1, "__call_rcu(): Double-freed CB %p->%pF()!!!\n", |
| 2960 | head, head->func); |
Paul E. McKenney | 7d0ae80 | 2015-03-03 14:57:58 -0800 | [diff] [blame] | 2961 | WRITE_ONCE(head->func, rcu_leak_callback); |
Paul E. McKenney | ae15018 | 2013-04-23 13:20:57 -0700 | [diff] [blame] | 2962 | return; |
| 2963 | } |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2964 | head->func = func; |
| 2965 | head->next = NULL; |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2966 | local_irq_save(flags); |
| 2967 | rdp = this_cpu_ptr(rsp->rda); |
| 2968 | |
| 2969 | /* Add the callback to our list. */ |
Paul E. McKenney | 15fecf8 | 2017-02-08 12:36:42 -0800 | [diff] [blame] | 2970 | if (unlikely(!rcu_segcblist_is_enabled(&rdp->cblist)) || cpu != -1) { |
Paul E. McKenney | 3fbfbf7 | 2012-08-19 21:35:53 -0700 | [diff] [blame] | 2971 | int offline; |
| 2972 | |
| 2973 | if (cpu != -1) |
| 2974 | rdp = per_cpu_ptr(rsp->rda, cpu); |
Paul E. McKenney | 143da9c | 2015-01-19 19:57:32 -0800 | [diff] [blame] | 2975 | if (likely(rdp->mynode)) { |
| 2976 | /* Post-boot, so this should be for a no-CBs CPU. */ |
| 2977 | offline = !__call_rcu_nocb(rdp, head, lazy, flags); |
| 2978 | WARN_ON_ONCE(offline); |
| 2979 | /* Offline CPU, _call_rcu() illegal, leak callback. */ |
| 2980 | local_irq_restore(flags); |
| 2981 | return; |
| 2982 | } |
| 2983 | /* |
| 2984 | * Very early boot, before rcu_init(). Initialize if needed |
| 2985 | * and then drop through to queue the callback. |
| 2986 | */ |
| 2987 | BUG_ON(cpu != -1); |
Paul E. McKenney | 34404ca | 2015-01-19 20:39:20 -0800 | [diff] [blame] | 2988 | WARN_ON_ONCE(!rcu_is_watching()); |
Paul E. McKenney | 15fecf8 | 2017-02-08 12:36:42 -0800 | [diff] [blame] | 2989 | if (rcu_segcblist_empty(&rdp->cblist)) |
| 2990 | rcu_segcblist_init(&rdp->cblist); |
Paul E. McKenney | 0d8ee37 | 2012-08-03 13:16:15 -0700 | [diff] [blame] | 2991 | } |
Paul E. McKenney | 15fecf8 | 2017-02-08 12:36:42 -0800 | [diff] [blame] | 2992 | rcu_segcblist_enqueue(&rdp->cblist, head, lazy); |
| 2993 | if (!lazy) |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2994 | rcu_idle_count_callbacks_posted(); |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 2995 | |
| 2996 | if (__is_kfree_rcu_offset((unsigned long)func)) |
| 2997 | trace_rcu_kfree_callback(rsp->name, head, (unsigned long)func, |
Paul E. McKenney | 15fecf8 | 2017-02-08 12:36:42 -0800 | [diff] [blame] | 2998 | rcu_segcblist_n_lazy_cbs(&rdp->cblist), |
| 2999 | rcu_segcblist_n_cbs(&rdp->cblist)); |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 3000 | else |
Paul E. McKenney | 15fecf8 | 2017-02-08 12:36:42 -0800 | [diff] [blame] | 3001 | trace_rcu_callback(rsp->name, head, |
| 3002 | rcu_segcblist_n_lazy_cbs(&rdp->cblist), |
| 3003 | rcu_segcblist_n_cbs(&rdp->cblist)); |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 3004 | |
Paul E. McKenney | 29154c5 | 2012-05-30 03:21:48 -0700 | [diff] [blame] | 3005 | /* Go handle any RCU core processing required. */ |
| 3006 | __call_rcu_core(rsp, rdp, head, flags); |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 3007 | local_irq_restore(flags); |
| 3008 | } |
| 3009 | |
Paul E. McKenney | a68a2bb | 2017-05-03 08:34:57 -0700 | [diff] [blame] | 3010 | /** |
| 3011 | * call_rcu_sched() - Queue an RCU for invocation after sched grace period. |
| 3012 | * @head: structure to be used for queueing the RCU updates. |
| 3013 | * @func: actual callback function to be invoked after the grace period |
| 3014 | * |
| 3015 | * The callback function will be invoked some time after a full grace |
| 3016 | * period elapses, in other words after all currently executing RCU |
| 3017 | * read-side critical sections have completed. call_rcu_sched() assumes |
| 3018 | * that the read-side critical sections end on enabling of preemption |
| 3019 | * or on voluntary preemption. |
Paul E. McKenney | 27fdb35 | 2017-10-19 14:26:21 -0700 | [diff] [blame] | 3020 | * RCU read-side critical sections are delimited by: |
| 3021 | * |
| 3022 | * - rcu_read_lock_sched() and rcu_read_unlock_sched(), OR |
| 3023 | * - anything that disables preemption. |
Paul E. McKenney | a68a2bb | 2017-05-03 08:34:57 -0700 | [diff] [blame] | 3024 | * |
| 3025 | * These may be nested. |
| 3026 | * |
| 3027 | * See the description of call_rcu() for more detailed information on |
| 3028 | * memory ordering guarantees. |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 3029 | */ |
Boqun Feng | b6a4ae7 | 2015-07-29 13:29:38 +0800 | [diff] [blame] | 3030 | void call_rcu_sched(struct rcu_head *head, rcu_callback_t func) |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 3031 | { |
Paul E. McKenney | 3fbfbf7 | 2012-08-19 21:35:53 -0700 | [diff] [blame] | 3032 | __call_rcu(head, func, &rcu_sched_state, -1, 0); |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 3033 | } |
Paul E. McKenney | d6714c2 | 2009-08-22 13:56:46 -0700 | [diff] [blame] | 3034 | EXPORT_SYMBOL_GPL(call_rcu_sched); |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 3035 | |
Paul E. McKenney | a68a2bb | 2017-05-03 08:34:57 -0700 | [diff] [blame] | 3036 | /** |
| 3037 | * call_rcu_bh() - Queue an RCU for invocation after a quicker grace period. |
| 3038 | * @head: structure to be used for queueing the RCU updates. |
| 3039 | * @func: actual callback function to be invoked after the grace period |
| 3040 | * |
| 3041 | * The callback function will be invoked some time after a full grace |
| 3042 | * period elapses, in other words after all currently executing RCU |
| 3043 | * read-side critical sections have completed. call_rcu_bh() assumes |
| 3044 | * that the read-side critical sections end on completion of a softirq |
| 3045 | * handler. This means that read-side critical sections in process |
| 3046 | * context must not be interrupted by softirqs. This interface is to be |
| 3047 | * used when most of the read-side critical sections are in softirq context. |
Paul E. McKenney | 27fdb35 | 2017-10-19 14:26:21 -0700 | [diff] [blame] | 3048 | * RCU read-side critical sections are delimited by: |
| 3049 | * |
| 3050 | * - rcu_read_lock() and rcu_read_unlock(), if in interrupt context, OR |
| 3051 | * - rcu_read_lock_bh() and rcu_read_unlock_bh(), if in process context. |
| 3052 | * |
| 3053 | * These may be nested. |
Paul E. McKenney | a68a2bb | 2017-05-03 08:34:57 -0700 | [diff] [blame] | 3054 | * |
| 3055 | * See the description of call_rcu() for more detailed information on |
| 3056 | * memory ordering guarantees. |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 3057 | */ |
Boqun Feng | b6a4ae7 | 2015-07-29 13:29:38 +0800 | [diff] [blame] | 3058 | void call_rcu_bh(struct rcu_head *head, rcu_callback_t func) |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 3059 | { |
Paul E. McKenney | 3fbfbf7 | 2012-08-19 21:35:53 -0700 | [diff] [blame] | 3060 | __call_rcu(head, func, &rcu_bh_state, -1, 0); |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 3061 | } |
| 3062 | EXPORT_SYMBOL_GPL(call_rcu_bh); |
| 3063 | |
Paul E. McKenney | 6d81339 | 2012-02-23 13:30:16 -0800 | [diff] [blame] | 3064 | /* |
Andreea-Cristina Bernat | 495aa96 | 2014-03-18 20:48:48 +0200 | [diff] [blame] | 3065 | * Queue an RCU callback for lazy invocation after a grace period. |
| 3066 | * This will likely be later named something like "call_rcu_lazy()", |
| 3067 | * but this change will require some way of tagging the lazy RCU |
| 3068 | * callbacks in the list of pending callbacks. Until then, this |
| 3069 | * function may only be called from __kfree_rcu(). |
| 3070 | */ |
| 3071 | void kfree_call_rcu(struct rcu_head *head, |
Boqun Feng | b6a4ae7 | 2015-07-29 13:29:38 +0800 | [diff] [blame] | 3072 | rcu_callback_t func) |
Andreea-Cristina Bernat | 495aa96 | 2014-03-18 20:48:48 +0200 | [diff] [blame] | 3073 | { |
Uma Sharma | e534165 | 2014-03-23 22:32:09 -0700 | [diff] [blame] | 3074 | __call_rcu(head, func, rcu_state_p, -1, 1); |
Andreea-Cristina Bernat | 495aa96 | 2014-03-18 20:48:48 +0200 | [diff] [blame] | 3075 | } |
| 3076 | EXPORT_SYMBOL_GPL(kfree_call_rcu); |
| 3077 | |
| 3078 | /* |
Paul E. McKenney | 6d81339 | 2012-02-23 13:30:16 -0800 | [diff] [blame] | 3079 | * Because a context switch is a grace period for RCU-sched and RCU-bh, |
| 3080 | * any blocking grace-period wait automatically implies a grace period |
| 3081 | * if there is only one CPU online at any point time during execution |
| 3082 | * of either synchronize_sched() or synchronize_rcu_bh(). It is OK to |
| 3083 | * occasionally incorrectly indicate that there are multiple CPUs online |
| 3084 | * when there was in fact only one the whole time, as this just adds |
| 3085 | * some overhead: RCU still operates correctly. |
Paul E. McKenney | 6d81339 | 2012-02-23 13:30:16 -0800 | [diff] [blame] | 3086 | */ |
| 3087 | static inline int rcu_blocking_is_gp(void) |
| 3088 | { |
Paul E. McKenney | 95f0c1d | 2012-06-19 11:58:27 -0700 | [diff] [blame] | 3089 | int ret; |
| 3090 | |
Paul E. McKenney | 6d81339 | 2012-02-23 13:30:16 -0800 | [diff] [blame] | 3091 | might_sleep(); /* Check for RCU read-side critical section. */ |
Paul E. McKenney | 95f0c1d | 2012-06-19 11:58:27 -0700 | [diff] [blame] | 3092 | preempt_disable(); |
| 3093 | ret = num_online_cpus() <= 1; |
| 3094 | preempt_enable(); |
| 3095 | return ret; |
Paul E. McKenney | 6d81339 | 2012-02-23 13:30:16 -0800 | [diff] [blame] | 3096 | } |
| 3097 | |
Paul E. McKenney | 6ebb237 | 2009-11-22 08:53:50 -0800 | [diff] [blame] | 3098 | /** |
| 3099 | * synchronize_sched - wait until an rcu-sched grace period has elapsed. |
| 3100 | * |
| 3101 | * Control will return to the caller some time after a full rcu-sched |
| 3102 | * grace period has elapsed, in other words after all currently executing |
| 3103 | * rcu-sched read-side critical sections have completed. These read-side |
| 3104 | * critical sections are delimited by rcu_read_lock_sched() and |
| 3105 | * rcu_read_unlock_sched(), and may be nested. Note that preempt_disable(), |
| 3106 | * local_irq_disable(), and so on may be used in place of |
| 3107 | * rcu_read_lock_sched(). |
| 3108 | * |
| 3109 | * This means that all preempt_disable code sequences, including NMI and |
Paul E. McKenney | f0a0e6f | 2012-10-23 13:47:01 -0700 | [diff] [blame] | 3110 | * non-threaded hardware-interrupt handlers, in progress on entry will |
| 3111 | * have completed before this primitive returns. However, this does not |
| 3112 | * guarantee that softirq handlers will have completed, since in some |
| 3113 | * kernels, these handlers can run in process context, and can block. |
| 3114 | * |
| 3115 | * Note that this guarantee implies further memory-ordering guarantees. |
| 3116 | * On systems with more than one CPU, when synchronize_sched() returns, |
| 3117 | * each CPU is guaranteed to have executed a full memory barrier since the |
| 3118 | * end of its last RCU-sched read-side critical section whose beginning |
| 3119 | * preceded the call to synchronize_sched(). In addition, each CPU having |
| 3120 | * an RCU read-side critical section that extends beyond the return from |
| 3121 | * synchronize_sched() is guaranteed to have executed a full memory barrier |
| 3122 | * after the beginning of synchronize_sched() and before the beginning of |
| 3123 | * that RCU read-side critical section. Note that these guarantees include |
| 3124 | * CPUs that are offline, idle, or executing in user mode, as well as CPUs |
| 3125 | * that are executing in the kernel. |
| 3126 | * |
| 3127 | * Furthermore, if CPU A invoked synchronize_sched(), which returned |
| 3128 | * to its caller on CPU B, then both CPU A and CPU B are guaranteed |
| 3129 | * to have executed a full memory barrier during the execution of |
| 3130 | * synchronize_sched() -- even if CPU A and CPU B are the same CPU (but |
| 3131 | * again only if the system has more than one CPU). |
Paul E. McKenney | 6ebb237 | 2009-11-22 08:53:50 -0800 | [diff] [blame] | 3132 | */ |
| 3133 | void synchronize_sched(void) |
| 3134 | { |
Paul E. McKenney | f78f5b9 | 2015-06-18 15:50:02 -0700 | [diff] [blame] | 3135 | RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map) || |
| 3136 | lock_is_held(&rcu_lock_map) || |
| 3137 | lock_is_held(&rcu_sched_lock_map), |
| 3138 | "Illegal synchronize_sched() in RCU-sched read-side critical section"); |
Paul E. McKenney | 6ebb237 | 2009-11-22 08:53:50 -0800 | [diff] [blame] | 3139 | if (rcu_blocking_is_gp()) |
| 3140 | return; |
Paul E. McKenney | 5afff48 | 2015-02-18 16:39:09 -0800 | [diff] [blame] | 3141 | if (rcu_gp_is_expedited()) |
Antti P Miettinen | 3705b88 | 2012-10-05 09:59:15 +0300 | [diff] [blame] | 3142 | synchronize_sched_expedited(); |
| 3143 | else |
| 3144 | wait_rcu_gp(call_rcu_sched); |
Paul E. McKenney | 6ebb237 | 2009-11-22 08:53:50 -0800 | [diff] [blame] | 3145 | } |
| 3146 | EXPORT_SYMBOL_GPL(synchronize_sched); |
| 3147 | |
| 3148 | /** |
| 3149 | * synchronize_rcu_bh - wait until an rcu_bh grace period has elapsed. |
| 3150 | * |
| 3151 | * Control will return to the caller some time after a full rcu_bh grace |
| 3152 | * period has elapsed, in other words after all currently executing rcu_bh |
| 3153 | * read-side critical sections have completed. RCU read-side critical |
| 3154 | * sections are delimited by rcu_read_lock_bh() and rcu_read_unlock_bh(), |
| 3155 | * and may be nested. |
Paul E. McKenney | f0a0e6f | 2012-10-23 13:47:01 -0700 | [diff] [blame] | 3156 | * |
| 3157 | * See the description of synchronize_sched() for more detailed information |
| 3158 | * on memory ordering guarantees. |
Paul E. McKenney | 6ebb237 | 2009-11-22 08:53:50 -0800 | [diff] [blame] | 3159 | */ |
| 3160 | void synchronize_rcu_bh(void) |
| 3161 | { |
Paul E. McKenney | f78f5b9 | 2015-06-18 15:50:02 -0700 | [diff] [blame] | 3162 | RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map) || |
| 3163 | lock_is_held(&rcu_lock_map) || |
| 3164 | lock_is_held(&rcu_sched_lock_map), |
| 3165 | "Illegal synchronize_rcu_bh() in RCU-bh read-side critical section"); |
Paul E. McKenney | 6ebb237 | 2009-11-22 08:53:50 -0800 | [diff] [blame] | 3166 | if (rcu_blocking_is_gp()) |
| 3167 | return; |
Paul E. McKenney | 5afff48 | 2015-02-18 16:39:09 -0800 | [diff] [blame] | 3168 | if (rcu_gp_is_expedited()) |
Antti P Miettinen | 3705b88 | 2012-10-05 09:59:15 +0300 | [diff] [blame] | 3169 | synchronize_rcu_bh_expedited(); |
| 3170 | else |
| 3171 | wait_rcu_gp(call_rcu_bh); |
Paul E. McKenney | 6ebb237 | 2009-11-22 08:53:50 -0800 | [diff] [blame] | 3172 | } |
| 3173 | EXPORT_SYMBOL_GPL(synchronize_rcu_bh); |
| 3174 | |
Paul E. McKenney | 765a3f4 | 2014-03-14 16:37:08 -0700 | [diff] [blame] | 3175 | /** |
| 3176 | * get_state_synchronize_rcu - Snapshot current RCU state |
| 3177 | * |
| 3178 | * Returns a cookie that is used by a later call to cond_synchronize_rcu() |
| 3179 | * to determine whether or not a full grace period has elapsed in the |
| 3180 | * meantime. |
| 3181 | */ |
| 3182 | unsigned long get_state_synchronize_rcu(void) |
| 3183 | { |
| 3184 | /* |
| 3185 | * Any prior manipulation of RCU-protected data must happen |
| 3186 | * before the load from ->gpnum. |
| 3187 | */ |
| 3188 | smp_mb(); /* ^^^ */ |
| 3189 | |
| 3190 | /* |
| 3191 | * Make sure this load happens before the purportedly |
| 3192 | * time-consuming work between get_state_synchronize_rcu() |
| 3193 | * and cond_synchronize_rcu(). |
| 3194 | */ |
Uma Sharma | e534165 | 2014-03-23 22:32:09 -0700 | [diff] [blame] | 3195 | return smp_load_acquire(&rcu_state_p->gpnum); |
Paul E. McKenney | 765a3f4 | 2014-03-14 16:37:08 -0700 | [diff] [blame] | 3196 | } |
| 3197 | EXPORT_SYMBOL_GPL(get_state_synchronize_rcu); |
| 3198 | |
| 3199 | /** |
| 3200 | * cond_synchronize_rcu - Conditionally wait for an RCU grace period |
| 3201 | * |
| 3202 | * @oldstate: return value from earlier call to get_state_synchronize_rcu() |
| 3203 | * |
| 3204 | * If a full RCU grace period has elapsed since the earlier call to |
| 3205 | * get_state_synchronize_rcu(), just return. Otherwise, invoke |
| 3206 | * synchronize_rcu() to wait for a full grace period. |
| 3207 | * |
| 3208 | * Yes, this function does not take counter wrap into account. But |
| 3209 | * counter wrap is harmless. If the counter wraps, we have waited for |
| 3210 | * more than 2 billion grace periods (and way more on a 64-bit system!), |
| 3211 | * so waiting for one additional grace period should be just fine. |
| 3212 | */ |
| 3213 | void cond_synchronize_rcu(unsigned long oldstate) |
| 3214 | { |
| 3215 | unsigned long newstate; |
| 3216 | |
| 3217 | /* |
| 3218 | * Ensure that this load happens before any RCU-destructive |
| 3219 | * actions the caller might carry out after we return. |
| 3220 | */ |
Uma Sharma | e534165 | 2014-03-23 22:32:09 -0700 | [diff] [blame] | 3221 | newstate = smp_load_acquire(&rcu_state_p->completed); |
Paul E. McKenney | 765a3f4 | 2014-03-14 16:37:08 -0700 | [diff] [blame] | 3222 | if (ULONG_CMP_GE(oldstate, newstate)) |
| 3223 | synchronize_rcu(); |
| 3224 | } |
| 3225 | EXPORT_SYMBOL_GPL(cond_synchronize_rcu); |
| 3226 | |
Paul E. McKenney | 2456005 | 2015-05-30 10:11:24 -0700 | [diff] [blame] | 3227 | /** |
| 3228 | * get_state_synchronize_sched - Snapshot current RCU-sched state |
| 3229 | * |
| 3230 | * Returns a cookie that is used by a later call to cond_synchronize_sched() |
| 3231 | * to determine whether or not a full grace period has elapsed in the |
| 3232 | * meantime. |
| 3233 | */ |
| 3234 | unsigned long get_state_synchronize_sched(void) |
| 3235 | { |
| 3236 | /* |
| 3237 | * Any prior manipulation of RCU-protected data must happen |
| 3238 | * before the load from ->gpnum. |
| 3239 | */ |
| 3240 | smp_mb(); /* ^^^ */ |
| 3241 | |
| 3242 | /* |
| 3243 | * Make sure this load happens before the purportedly |
| 3244 | * time-consuming work between get_state_synchronize_sched() |
| 3245 | * and cond_synchronize_sched(). |
| 3246 | */ |
| 3247 | return smp_load_acquire(&rcu_sched_state.gpnum); |
| 3248 | } |
| 3249 | EXPORT_SYMBOL_GPL(get_state_synchronize_sched); |
| 3250 | |
| 3251 | /** |
| 3252 | * cond_synchronize_sched - Conditionally wait for an RCU-sched grace period |
| 3253 | * |
| 3254 | * @oldstate: return value from earlier call to get_state_synchronize_sched() |
| 3255 | * |
| 3256 | * If a full RCU-sched grace period has elapsed since the earlier call to |
| 3257 | * get_state_synchronize_sched(), just return. Otherwise, invoke |
| 3258 | * synchronize_sched() to wait for a full grace period. |
| 3259 | * |
| 3260 | * Yes, this function does not take counter wrap into account. But |
| 3261 | * counter wrap is harmless. If the counter wraps, we have waited for |
| 3262 | * more than 2 billion grace periods (and way more on a 64-bit system!), |
| 3263 | * so waiting for one additional grace period should be just fine. |
| 3264 | */ |
| 3265 | void cond_synchronize_sched(unsigned long oldstate) |
| 3266 | { |
| 3267 | unsigned long newstate; |
| 3268 | |
| 3269 | /* |
| 3270 | * Ensure that this load happens before any RCU-destructive |
| 3271 | * actions the caller might carry out after we return. |
| 3272 | */ |
| 3273 | newstate = smp_load_acquire(&rcu_sched_state.completed); |
| 3274 | if (ULONG_CMP_GE(oldstate, newstate)) |
| 3275 | synchronize_sched(); |
| 3276 | } |
| 3277 | EXPORT_SYMBOL_GPL(cond_synchronize_sched); |
| 3278 | |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 3279 | /* |
| 3280 | * Check to see if there is any immediate RCU-related work to be done |
| 3281 | * by the current CPU, for the specified type of RCU, returning 1 if so. |
| 3282 | * The checks are in order of increasing expense: checks that can be |
| 3283 | * carried out against CPU-local state are performed first. However, |
| 3284 | * we must check for CPU stalls first, else we might not get a chance. |
| 3285 | */ |
| 3286 | static int __rcu_pending(struct rcu_state *rsp, struct rcu_data *rdp) |
| 3287 | { |
Paul E. McKenney | 2f51f98 | 2009-11-13 19:51:39 -0800 | [diff] [blame] | 3288 | struct rcu_node *rnp = rdp->mynode; |
| 3289 | |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 3290 | /* Check for CPU stalls, if enabled. */ |
| 3291 | check_cpu_stall(rsp, rdp); |
| 3292 | |
Paul E. McKenney | a096932 | 2013-11-08 09:03:10 -0800 | [diff] [blame] | 3293 | /* Is this CPU a NO_HZ_FULL CPU that should ignore RCU? */ |
| 3294 | if (rcu_nohz_full_cpu(rsp)) |
| 3295 | return 0; |
| 3296 | |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 3297 | /* Is the RCU core waiting for a quiescent state from this CPU? */ |
Paul E. McKenney | 01c495f | 2018-01-10 12:36:00 -0800 | [diff] [blame] | 3298 | if (rdp->core_needs_qs && !rdp->cpu_no_qs.b.norm) |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 3299 | return 1; |
| 3300 | |
| 3301 | /* Does this CPU have callbacks ready to invoke? */ |
Paul E. McKenney | 01c495f | 2018-01-10 12:36:00 -0800 | [diff] [blame] | 3302 | if (rcu_segcblist_ready_cbs(&rdp->cblist)) |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 3303 | return 1; |
| 3304 | |
| 3305 | /* Has RCU gone idle with this CPU needing another grace period? */ |
Paul E. McKenney | c193520 | 2018-04-12 16:29:13 -0700 | [diff] [blame] | 3306 | if (!rcu_gp_in_progress(rsp) && |
| 3307 | rcu_segcblist_is_enabled(&rdp->cblist) && |
| 3308 | !rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL)) |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 3309 | return 1; |
| 3310 | |
| 3311 | /* Has another RCU grace period completed? */ |
Paul E. McKenney | 01c495f | 2018-01-10 12:36:00 -0800 | [diff] [blame] | 3312 | if (READ_ONCE(rnp->completed) != rdp->completed) /* outside lock */ |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 3313 | return 1; |
| 3314 | |
| 3315 | /* Has a new RCU grace period started? */ |
Paul E. McKenney | 7d0ae80 | 2015-03-03 14:57:58 -0800 | [diff] [blame] | 3316 | if (READ_ONCE(rnp->gpnum) != rdp->gpnum || |
Paul E. McKenney | 01c495f | 2018-01-10 12:36:00 -0800 | [diff] [blame] | 3317 | unlikely(READ_ONCE(rdp->gpwrap))) /* outside lock */ |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 3318 | return 1; |
| 3319 | |
Paul E. McKenney | 96d3fd0 | 2013-10-04 14:33:34 -0700 | [diff] [blame] | 3320 | /* Does this CPU need a deferred NOCB wakeup? */ |
Paul E. McKenney | 01c495f | 2018-01-10 12:36:00 -0800 | [diff] [blame] | 3321 | if (rcu_nocb_need_deferred_wakeup(rdp)) |
Paul E. McKenney | 96d3fd0 | 2013-10-04 14:33:34 -0700 | [diff] [blame] | 3322 | return 1; |
Paul E. McKenney | 96d3fd0 | 2013-10-04 14:33:34 -0700 | [diff] [blame] | 3323 | |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 3324 | /* nothing to do */ |
| 3325 | return 0; |
| 3326 | } |
| 3327 | |
| 3328 | /* |
| 3329 | * Check to see if there is any immediate RCU-related work to be done |
| 3330 | * by the current CPU, returning 1 if so. This function is part of the |
| 3331 | * RCU implementation; it is -not- an exported member of the RCU API. |
| 3332 | */ |
Paul E. McKenney | e3950ec | 2014-10-21 08:03:57 -0700 | [diff] [blame] | 3333 | static int rcu_pending(void) |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 3334 | { |
Paul E. McKenney | 6ce75a2 | 2012-06-12 11:01:13 -0700 | [diff] [blame] | 3335 | struct rcu_state *rsp; |
| 3336 | |
| 3337 | for_each_rcu_flavor(rsp) |
Paul E. McKenney | e3950ec | 2014-10-21 08:03:57 -0700 | [diff] [blame] | 3338 | if (__rcu_pending(rsp, this_cpu_ptr(rsp->rda))) |
Paul E. McKenney | 6ce75a2 | 2012-06-12 11:01:13 -0700 | [diff] [blame] | 3339 | return 1; |
| 3340 | return 0; |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 3341 | } |
| 3342 | |
| 3343 | /* |
Paul E. McKenney | c0f4dfd | 2012-12-28 11:30:36 -0800 | [diff] [blame] | 3344 | * Return true if the specified CPU has any callback. If all_lazy is |
| 3345 | * non-NULL, store an indication of whether all callbacks are lazy. |
| 3346 | * (If there are no callbacks, all of them are deemed to be lazy.) |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 3347 | */ |
Nicholas Mc Guire | 82072c4 | 2015-05-11 18:12:27 +0200 | [diff] [blame] | 3348 | static bool __maybe_unused rcu_cpu_has_callbacks(bool *all_lazy) |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 3349 | { |
Paul E. McKenney | c0f4dfd | 2012-12-28 11:30:36 -0800 | [diff] [blame] | 3350 | bool al = true; |
| 3351 | bool hc = false; |
| 3352 | struct rcu_data *rdp; |
Paul E. McKenney | 6ce75a2 | 2012-06-12 11:01:13 -0700 | [diff] [blame] | 3353 | struct rcu_state *rsp; |
| 3354 | |
Paul E. McKenney | c0f4dfd | 2012-12-28 11:30:36 -0800 | [diff] [blame] | 3355 | for_each_rcu_flavor(rsp) { |
Paul E. McKenney | aa6da51 | 2014-10-21 13:23:08 -0700 | [diff] [blame] | 3356 | rdp = this_cpu_ptr(rsp->rda); |
Paul E. McKenney | 15fecf8 | 2017-02-08 12:36:42 -0800 | [diff] [blame] | 3357 | if (rcu_segcblist_empty(&rdp->cblist)) |
Paul E. McKenney | 69c8d28 | 2013-09-03 09:52:20 -0700 | [diff] [blame] | 3358 | continue; |
| 3359 | hc = true; |
Paul E. McKenney | 15fecf8 | 2017-02-08 12:36:42 -0800 | [diff] [blame] | 3360 | if (rcu_segcblist_n_nonlazy_cbs(&rdp->cblist) || !all_lazy) { |
Paul E. McKenney | c0f4dfd | 2012-12-28 11:30:36 -0800 | [diff] [blame] | 3361 | al = false; |
Paul E. McKenney | 69c8d28 | 2013-09-03 09:52:20 -0700 | [diff] [blame] | 3362 | break; |
| 3363 | } |
Paul E. McKenney | c0f4dfd | 2012-12-28 11:30:36 -0800 | [diff] [blame] | 3364 | } |
| 3365 | if (all_lazy) |
| 3366 | *all_lazy = al; |
| 3367 | return hc; |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 3368 | } |
| 3369 | |
Paul E. McKenney | b1420f1 | 2012-03-01 13:18:08 -0800 | [diff] [blame] | 3370 | /* |
Paul E. McKenney | a83eff0 | 2012-05-23 18:47:05 -0700 | [diff] [blame] | 3371 | * Helper function for _rcu_barrier() tracing. If tracing is disabled, |
| 3372 | * the compiler is expected to optimize this away. |
| 3373 | */ |
Steven Rostedt (Red Hat) | e66c33d | 2013-07-12 16:50:28 -0400 | [diff] [blame] | 3374 | static void _rcu_barrier_trace(struct rcu_state *rsp, const char *s, |
Paul E. McKenney | a83eff0 | 2012-05-23 18:47:05 -0700 | [diff] [blame] | 3375 | int cpu, unsigned long done) |
| 3376 | { |
| 3377 | trace_rcu_barrier(rsp->name, s, cpu, |
| 3378 | atomic_read(&rsp->barrier_cpu_count), done); |
| 3379 | } |
| 3380 | |
| 3381 | /* |
Paul E. McKenney | b1420f1 | 2012-03-01 13:18:08 -0800 | [diff] [blame] | 3382 | * RCU callback function for _rcu_barrier(). If we are last, wake |
| 3383 | * up the task executing _rcu_barrier(). |
| 3384 | */ |
Paul E. McKenney | 24ebbca | 2012-05-29 00:34:56 -0700 | [diff] [blame] | 3385 | static void rcu_barrier_callback(struct rcu_head *rhp) |
Paul E. McKenney | d0ec774 | 2009-10-06 21:48:16 -0700 | [diff] [blame] | 3386 | { |
Paul E. McKenney | 24ebbca | 2012-05-29 00:34:56 -0700 | [diff] [blame] | 3387 | struct rcu_data *rdp = container_of(rhp, struct rcu_data, barrier_head); |
| 3388 | struct rcu_state *rsp = rdp->rsp; |
| 3389 | |
Paul E. McKenney | a83eff0 | 2012-05-23 18:47:05 -0700 | [diff] [blame] | 3390 | if (atomic_dec_and_test(&rsp->barrier_cpu_count)) { |
Paul E. McKenney | d8db2e8 | 2017-06-27 13:22:17 -0700 | [diff] [blame] | 3391 | _rcu_barrier_trace(rsp, TPS("LastCB"), -1, |
| 3392 | rsp->barrier_sequence); |
Paul E. McKenney | 7db74df | 2012-05-29 03:03:37 -0700 | [diff] [blame] | 3393 | complete(&rsp->barrier_completion); |
Paul E. McKenney | a83eff0 | 2012-05-23 18:47:05 -0700 | [diff] [blame] | 3394 | } else { |
Paul E. McKenney | d8db2e8 | 2017-06-27 13:22:17 -0700 | [diff] [blame] | 3395 | _rcu_barrier_trace(rsp, TPS("CB"), -1, rsp->barrier_sequence); |
Paul E. McKenney | a83eff0 | 2012-05-23 18:47:05 -0700 | [diff] [blame] | 3396 | } |
Paul E. McKenney | d0ec774 | 2009-10-06 21:48:16 -0700 | [diff] [blame] | 3397 | } |
| 3398 | |
| 3399 | /* |
| 3400 | * Called with preemption disabled, and from cross-cpu IRQ context. |
| 3401 | */ |
| 3402 | static void rcu_barrier_func(void *type) |
| 3403 | { |
Paul E. McKenney | 037b64e | 2012-05-28 23:26:01 -0700 | [diff] [blame] | 3404 | struct rcu_state *rsp = type; |
Christoph Lameter | fa07a58 | 2014-04-15 12:20:12 -0700 | [diff] [blame] | 3405 | struct rcu_data *rdp = raw_cpu_ptr(rsp->rda); |
Paul E. McKenney | d0ec774 | 2009-10-06 21:48:16 -0700 | [diff] [blame] | 3406 | |
Paul E. McKenney | d8db2e8 | 2017-06-27 13:22:17 -0700 | [diff] [blame] | 3407 | _rcu_barrier_trace(rsp, TPS("IRQ"), -1, rsp->barrier_sequence); |
Paul E. McKenney | f92c734 | 2017-04-10 15:40:35 -0700 | [diff] [blame] | 3408 | rdp->barrier_head.func = rcu_barrier_callback; |
| 3409 | debug_rcu_head_queue(&rdp->barrier_head); |
| 3410 | if (rcu_segcblist_entrain(&rdp->cblist, &rdp->barrier_head, 0)) { |
| 3411 | atomic_inc(&rsp->barrier_cpu_count); |
| 3412 | } else { |
| 3413 | debug_rcu_head_unqueue(&rdp->barrier_head); |
Paul E. McKenney | d8db2e8 | 2017-06-27 13:22:17 -0700 | [diff] [blame] | 3414 | _rcu_barrier_trace(rsp, TPS("IRQNQ"), -1, |
| 3415 | rsp->barrier_sequence); |
Paul E. McKenney | f92c734 | 2017-04-10 15:40:35 -0700 | [diff] [blame] | 3416 | } |
Paul E. McKenney | d0ec774 | 2009-10-06 21:48:16 -0700 | [diff] [blame] | 3417 | } |
| 3418 | |
Paul E. McKenney | d0ec774 | 2009-10-06 21:48:16 -0700 | [diff] [blame] | 3419 | /* |
| 3420 | * Orchestrate the specified type of RCU barrier, waiting for all |
| 3421 | * RCU callbacks of the specified type to complete. |
| 3422 | */ |
Paul E. McKenney | 037b64e | 2012-05-28 23:26:01 -0700 | [diff] [blame] | 3423 | static void _rcu_barrier(struct rcu_state *rsp) |
Paul E. McKenney | d0ec774 | 2009-10-06 21:48:16 -0700 | [diff] [blame] | 3424 | { |
Paul E. McKenney | b1420f1 | 2012-03-01 13:18:08 -0800 | [diff] [blame] | 3425 | int cpu; |
Paul E. McKenney | b1420f1 | 2012-03-01 13:18:08 -0800 | [diff] [blame] | 3426 | struct rcu_data *rdp; |
Paul E. McKenney | 4f525a5 | 2015-06-26 11:20:00 -0700 | [diff] [blame] | 3427 | unsigned long s = rcu_seq_snap(&rsp->barrier_sequence); |
Paul E. McKenney | b1420f1 | 2012-03-01 13:18:08 -0800 | [diff] [blame] | 3428 | |
Paul E. McKenney | d8db2e8 | 2017-06-27 13:22:17 -0700 | [diff] [blame] | 3429 | _rcu_barrier_trace(rsp, TPS("Begin"), -1, s); |
Paul E. McKenney | b1420f1 | 2012-03-01 13:18:08 -0800 | [diff] [blame] | 3430 | |
Paul E. McKenney | e74f4c4 | 2009-10-06 21:48:17 -0700 | [diff] [blame] | 3431 | /* Take mutex to serialize concurrent rcu_barrier() requests. */ |
Paul E. McKenney | 7be7f0b | 2012-05-29 05:18:53 -0700 | [diff] [blame] | 3432 | mutex_lock(&rsp->barrier_mutex); |
Paul E. McKenney | b1420f1 | 2012-03-01 13:18:08 -0800 | [diff] [blame] | 3433 | |
Paul E. McKenney | 4f525a5 | 2015-06-26 11:20:00 -0700 | [diff] [blame] | 3434 | /* Did someone else do our work for us? */ |
| 3435 | if (rcu_seq_done(&rsp->barrier_sequence, s)) { |
Paul E. McKenney | d8db2e8 | 2017-06-27 13:22:17 -0700 | [diff] [blame] | 3436 | _rcu_barrier_trace(rsp, TPS("EarlyExit"), -1, |
| 3437 | rsp->barrier_sequence); |
Paul E. McKenney | cf3a9c4 | 2012-05-29 14:56:46 -0700 | [diff] [blame] | 3438 | smp_mb(); /* caller's subsequent code after above check. */ |
| 3439 | mutex_unlock(&rsp->barrier_mutex); |
| 3440 | return; |
| 3441 | } |
| 3442 | |
Paul E. McKenney | 4f525a5 | 2015-06-26 11:20:00 -0700 | [diff] [blame] | 3443 | /* Mark the start of the barrier operation. */ |
| 3444 | rcu_seq_start(&rsp->barrier_sequence); |
Paul E. McKenney | d8db2e8 | 2017-06-27 13:22:17 -0700 | [diff] [blame] | 3445 | _rcu_barrier_trace(rsp, TPS("Inc1"), -1, rsp->barrier_sequence); |
Paul E. McKenney | b1420f1 | 2012-03-01 13:18:08 -0800 | [diff] [blame] | 3446 | |
Paul E. McKenney | d0ec774 | 2009-10-06 21:48:16 -0700 | [diff] [blame] | 3447 | /* |
Paul E. McKenney | b1420f1 | 2012-03-01 13:18:08 -0800 | [diff] [blame] | 3448 | * Initialize the count to one rather than to zero in order to |
| 3449 | * avoid a too-soon return to zero in case of a short grace period |
Paul E. McKenney | 1331e7a | 2012-08-02 17:43:50 -0700 | [diff] [blame] | 3450 | * (or preemption of this task). Exclude CPU-hotplug operations |
| 3451 | * to ensure that no offline CPU has callbacks queued. |
Paul E. McKenney | d0ec774 | 2009-10-06 21:48:16 -0700 | [diff] [blame] | 3452 | */ |
Paul E. McKenney | 7db74df | 2012-05-29 03:03:37 -0700 | [diff] [blame] | 3453 | init_completion(&rsp->barrier_completion); |
Paul E. McKenney | 24ebbca | 2012-05-29 00:34:56 -0700 | [diff] [blame] | 3454 | atomic_set(&rsp->barrier_cpu_count, 1); |
Paul E. McKenney | 1331e7a | 2012-08-02 17:43:50 -0700 | [diff] [blame] | 3455 | get_online_cpus(); |
Paul E. McKenney | b1420f1 | 2012-03-01 13:18:08 -0800 | [diff] [blame] | 3456 | |
| 3457 | /* |
Paul E. McKenney | 1331e7a | 2012-08-02 17:43:50 -0700 | [diff] [blame] | 3458 | * Force each CPU with callbacks to register a new callback. |
| 3459 | * When that callback is invoked, we will know that all of the |
| 3460 | * corresponding CPU's preceding callbacks have been invoked. |
Paul E. McKenney | b1420f1 | 2012-03-01 13:18:08 -0800 | [diff] [blame] | 3461 | */ |
Paul E. McKenney | 3fbfbf7 | 2012-08-19 21:35:53 -0700 | [diff] [blame] | 3462 | for_each_possible_cpu(cpu) { |
Frederic Weisbecker | d1e43fa | 2013-03-26 23:47:24 +0100 | [diff] [blame] | 3463 | if (!cpu_online(cpu) && !rcu_is_nocb_cpu(cpu)) |
Paul E. McKenney | 3fbfbf7 | 2012-08-19 21:35:53 -0700 | [diff] [blame] | 3464 | continue; |
Paul E. McKenney | b1420f1 | 2012-03-01 13:18:08 -0800 | [diff] [blame] | 3465 | rdp = per_cpu_ptr(rsp->rda, cpu); |
Frederic Weisbecker | d1e43fa | 2013-03-26 23:47:24 +0100 | [diff] [blame] | 3466 | if (rcu_is_nocb_cpu(cpu)) { |
Paul E. McKenney | d7e2993 | 2014-10-27 09:15:54 -0700 | [diff] [blame] | 3467 | if (!rcu_nocb_cpu_needs_barrier(rsp, cpu)) { |
Paul E. McKenney | d8db2e8 | 2017-06-27 13:22:17 -0700 | [diff] [blame] | 3468 | _rcu_barrier_trace(rsp, TPS("OfflineNoCB"), cpu, |
Paul E. McKenney | 4f525a5 | 2015-06-26 11:20:00 -0700 | [diff] [blame] | 3469 | rsp->barrier_sequence); |
Paul E. McKenney | d7e2993 | 2014-10-27 09:15:54 -0700 | [diff] [blame] | 3470 | } else { |
Paul E. McKenney | d8db2e8 | 2017-06-27 13:22:17 -0700 | [diff] [blame] | 3471 | _rcu_barrier_trace(rsp, TPS("OnlineNoCB"), cpu, |
Paul E. McKenney | 4f525a5 | 2015-06-26 11:20:00 -0700 | [diff] [blame] | 3472 | rsp->barrier_sequence); |
Paul E. McKenney | 41050a0 | 2014-12-18 12:31:27 -0800 | [diff] [blame] | 3473 | smp_mb__before_atomic(); |
Paul E. McKenney | d7e2993 | 2014-10-27 09:15:54 -0700 | [diff] [blame] | 3474 | atomic_inc(&rsp->barrier_cpu_count); |
| 3475 | __call_rcu(&rdp->barrier_head, |
| 3476 | rcu_barrier_callback, rsp, cpu, 0); |
| 3477 | } |
Paul E. McKenney | 15fecf8 | 2017-02-08 12:36:42 -0800 | [diff] [blame] | 3478 | } else if (rcu_segcblist_n_cbs(&rdp->cblist)) { |
Paul E. McKenney | d8db2e8 | 2017-06-27 13:22:17 -0700 | [diff] [blame] | 3479 | _rcu_barrier_trace(rsp, TPS("OnlineQ"), cpu, |
Paul E. McKenney | 4f525a5 | 2015-06-26 11:20:00 -0700 | [diff] [blame] | 3480 | rsp->barrier_sequence); |
Paul E. McKenney | 037b64e | 2012-05-28 23:26:01 -0700 | [diff] [blame] | 3481 | smp_call_function_single(cpu, rcu_barrier_func, rsp, 1); |
Paul E. McKenney | b1420f1 | 2012-03-01 13:18:08 -0800 | [diff] [blame] | 3482 | } else { |
Paul E. McKenney | d8db2e8 | 2017-06-27 13:22:17 -0700 | [diff] [blame] | 3483 | _rcu_barrier_trace(rsp, TPS("OnlineNQ"), cpu, |
Paul E. McKenney | 4f525a5 | 2015-06-26 11:20:00 -0700 | [diff] [blame] | 3484 | rsp->barrier_sequence); |
Paul E. McKenney | b1420f1 | 2012-03-01 13:18:08 -0800 | [diff] [blame] | 3485 | } |
| 3486 | } |
Paul E. McKenney | 1331e7a | 2012-08-02 17:43:50 -0700 | [diff] [blame] | 3487 | put_online_cpus(); |
Paul E. McKenney | b1420f1 | 2012-03-01 13:18:08 -0800 | [diff] [blame] | 3488 | |
| 3489 | /* |
| 3490 | * Now that we have an rcu_barrier_callback() callback on each |
| 3491 | * CPU, and thus each counted, remove the initial count. |
| 3492 | */ |
Paul E. McKenney | 24ebbca | 2012-05-29 00:34:56 -0700 | [diff] [blame] | 3493 | if (atomic_dec_and_test(&rsp->barrier_cpu_count)) |
Paul E. McKenney | 7db74df | 2012-05-29 03:03:37 -0700 | [diff] [blame] | 3494 | complete(&rsp->barrier_completion); |
Paul E. McKenney | b1420f1 | 2012-03-01 13:18:08 -0800 | [diff] [blame] | 3495 | |
| 3496 | /* Wait for all rcu_barrier_callback() callbacks to be invoked. */ |
Paul E. McKenney | 7db74df | 2012-05-29 03:03:37 -0700 | [diff] [blame] | 3497 | wait_for_completion(&rsp->barrier_completion); |
Paul E. McKenney | b1420f1 | 2012-03-01 13:18:08 -0800 | [diff] [blame] | 3498 | |
Paul E. McKenney | 4f525a5 | 2015-06-26 11:20:00 -0700 | [diff] [blame] | 3499 | /* Mark the end of the barrier operation. */ |
Paul E. McKenney | d8db2e8 | 2017-06-27 13:22:17 -0700 | [diff] [blame] | 3500 | _rcu_barrier_trace(rsp, TPS("Inc2"), -1, rsp->barrier_sequence); |
Paul E. McKenney | 4f525a5 | 2015-06-26 11:20:00 -0700 | [diff] [blame] | 3501 | rcu_seq_end(&rsp->barrier_sequence); |
| 3502 | |
Paul E. McKenney | b1420f1 | 2012-03-01 13:18:08 -0800 | [diff] [blame] | 3503 | /* Other rcu_barrier() invocations can now safely proceed. */ |
Paul E. McKenney | 7be7f0b | 2012-05-29 05:18:53 -0700 | [diff] [blame] | 3504 | mutex_unlock(&rsp->barrier_mutex); |
Paul E. McKenney | d0ec774 | 2009-10-06 21:48:16 -0700 | [diff] [blame] | 3505 | } |
| 3506 | |
| 3507 | /** |
Paul E. McKenney | d0ec774 | 2009-10-06 21:48:16 -0700 | [diff] [blame] | 3508 | * rcu_barrier_bh - Wait until all in-flight call_rcu_bh() callbacks complete. |
| 3509 | */ |
| 3510 | void rcu_barrier_bh(void) |
| 3511 | { |
Paul E. McKenney | 037b64e | 2012-05-28 23:26:01 -0700 | [diff] [blame] | 3512 | _rcu_barrier(&rcu_bh_state); |
Paul E. McKenney | d0ec774 | 2009-10-06 21:48:16 -0700 | [diff] [blame] | 3513 | } |
| 3514 | EXPORT_SYMBOL_GPL(rcu_barrier_bh); |
| 3515 | |
| 3516 | /** |
| 3517 | * rcu_barrier_sched - Wait for in-flight call_rcu_sched() callbacks. |
| 3518 | */ |
| 3519 | void rcu_barrier_sched(void) |
| 3520 | { |
Paul E. McKenney | 037b64e | 2012-05-28 23:26:01 -0700 | [diff] [blame] | 3521 | _rcu_barrier(&rcu_sched_state); |
Paul E. McKenney | d0ec774 | 2009-10-06 21:48:16 -0700 | [diff] [blame] | 3522 | } |
| 3523 | EXPORT_SYMBOL_GPL(rcu_barrier_sched); |
| 3524 | |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 3525 | /* |
Paul E. McKenney | 0aa04b0 | 2015-01-23 21:52:37 -0800 | [diff] [blame] | 3526 | * Propagate ->qsinitmask bits up the rcu_node tree to account for the |
| 3527 | * first CPU in a given leaf rcu_node structure coming online. The caller |
| 3528 | * must hold the corresponding leaf rcu_node ->lock with interrrupts |
| 3529 | * disabled. |
| 3530 | */ |
| 3531 | static void rcu_init_new_rnp(struct rcu_node *rnp_leaf) |
| 3532 | { |
| 3533 | long mask; |
| 3534 | struct rcu_node *rnp = rnp_leaf; |
| 3535 | |
Matthew Wilcox | a32e01e | 2018-01-17 06:24:30 -0800 | [diff] [blame] | 3536 | raw_lockdep_assert_held_rcu_node(rnp); |
Paul E. McKenney | 0aa04b0 | 2015-01-23 21:52:37 -0800 | [diff] [blame] | 3537 | for (;;) { |
| 3538 | mask = rnp->grpmask; |
| 3539 | rnp = rnp->parent; |
| 3540 | if (rnp == NULL) |
| 3541 | return; |
Paul E. McKenney | 6cf1008 | 2015-10-08 15:36:54 -0700 | [diff] [blame] | 3542 | raw_spin_lock_rcu_node(rnp); /* Interrupts already disabled. */ |
Paul E. McKenney | 0aa04b0 | 2015-01-23 21:52:37 -0800 | [diff] [blame] | 3543 | rnp->qsmaskinit |= mask; |
Boqun Feng | 67c583a7 | 2015-12-29 12:18:47 +0800 | [diff] [blame] | 3544 | raw_spin_unlock_rcu_node(rnp); /* Interrupts remain disabled. */ |
Paul E. McKenney | 0aa04b0 | 2015-01-23 21:52:37 -0800 | [diff] [blame] | 3545 | } |
| 3546 | } |
| 3547 | |
| 3548 | /* |
Paul E. McKenney | 2756962 | 2009-08-15 09:53:46 -0700 | [diff] [blame] | 3549 | * Do boot-time initialization of a CPU's per-CPU RCU data. |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 3550 | */ |
Paul E. McKenney | 2756962 | 2009-08-15 09:53:46 -0700 | [diff] [blame] | 3551 | static void __init |
| 3552 | rcu_boot_init_percpu_data(int cpu, struct rcu_state *rsp) |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 3553 | { |
Lai Jiangshan | 394f99a | 2010-06-28 16:25:04 +0800 | [diff] [blame] | 3554 | struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu); |
Paul E. McKenney | 2756962 | 2009-08-15 09:53:46 -0700 | [diff] [blame] | 3555 | |
| 3556 | /* Set up local state, ensuring consistent view of global state. */ |
Mark Rutland | bc75e99 | 2016-06-03 15:20:04 +0100 | [diff] [blame] | 3557 | rdp->grpmask = leaf_node_cpu_bit(rdp->mynode, cpu); |
Paul E. McKenney | 2756962 | 2009-08-15 09:53:46 -0700 | [diff] [blame] | 3558 | rdp->dynticks = &per_cpu(rcu_dynticks, cpu); |
Paul E. McKenney | 51a1fd3 | 2017-10-03 14:43:40 -0700 | [diff] [blame] | 3559 | WARN_ON_ONCE(rdp->dynticks->dynticks_nesting != 1); |
Paul E. McKenney | 02a5c550 | 2016-11-02 17:25:06 -0700 | [diff] [blame] | 3560 | WARN_ON_ONCE(rcu_dynticks_in_eqs(rcu_dynticks_snap(rdp->dynticks))); |
Paul E. McKenney | 2756962 | 2009-08-15 09:53:46 -0700 | [diff] [blame] | 3561 | rdp->cpu = cpu; |
Paul E. McKenney | d4c08f2 | 2011-06-25 06:36:56 -0700 | [diff] [blame] | 3562 | rdp->rsp = rsp; |
Paul E. McKenney | 3fbfbf7 | 2012-08-19 21:35:53 -0700 | [diff] [blame] | 3563 | rcu_boot_init_nocb_percpu_data(rdp); |
Paul E. McKenney | 2756962 | 2009-08-15 09:53:46 -0700 | [diff] [blame] | 3564 | } |
| 3565 | |
| 3566 | /* |
| 3567 | * Initialize a CPU's per-CPU RCU data. Note that only one online or |
| 3568 | * offline event can be happening at a given time. Note also that we |
| 3569 | * can accept some slop in the rsp->completed access due to the fact |
| 3570 | * that this CPU cannot possibly have any RCU callbacks in flight yet. |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 3571 | */ |
Paul Gortmaker | 49fb4c6 | 2013-06-19 14:52:21 -0400 | [diff] [blame] | 3572 | static void |
Iulia Manda | 9b67122 | 2014-03-11 13:18:22 +0200 | [diff] [blame] | 3573 | rcu_init_percpu_data(int cpu, struct rcu_state *rsp) |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 3574 | { |
| 3575 | unsigned long flags; |
Lai Jiangshan | 394f99a | 2010-06-28 16:25:04 +0800 | [diff] [blame] | 3576 | struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu); |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 3577 | struct rcu_node *rnp = rcu_get_root(rsp); |
| 3578 | |
| 3579 | /* Set up local state, ensuring consistent view of global state. */ |
Paul E. McKenney | 6cf1008 | 2015-10-08 15:36:54 -0700 | [diff] [blame] | 3580 | raw_spin_lock_irqsave_rcu_node(rnp, flags); |
Paul E. McKenney | 37c72e5 | 2009-10-14 10:15:55 -0700 | [diff] [blame] | 3581 | rdp->qlen_last_fqs_check = 0; |
| 3582 | rdp->n_force_qs_snap = rsp->n_force_qs; |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 3583 | rdp->blimit = blimit; |
Paul E. McKenney | 15fecf8 | 2017-02-08 12:36:42 -0800 | [diff] [blame] | 3584 | if (rcu_segcblist_empty(&rdp->cblist) && /* No early-boot CBs? */ |
| 3585 | !init_nocb_callback_list(rdp)) |
| 3586 | rcu_segcblist_init(&rdp->cblist); /* Re-enable callbacks. */ |
Paul E. McKenney | 2342172 | 2017-10-05 15:03:10 -0700 | [diff] [blame] | 3587 | rdp->dynticks->dynticks_nesting = 1; /* CPU not up, no tearing. */ |
Paul E. McKenney | 2625d46 | 2016-11-02 14:23:30 -0700 | [diff] [blame] | 3588 | rcu_dynticks_eqs_online(); |
Boqun Feng | 67c583a7 | 2015-12-29 12:18:47 +0800 | [diff] [blame] | 3589 | raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */ |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 3590 | |
Paul E. McKenney | 0aa04b0 | 2015-01-23 21:52:37 -0800 | [diff] [blame] | 3591 | /* |
| 3592 | * Add CPU to leaf rcu_node pending-online bitmask. Any needed |
| 3593 | * propagation up the rcu_node tree will happen at the beginning |
| 3594 | * of the next grace period. |
| 3595 | */ |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 3596 | rnp = rdp->mynode; |
Peter Zijlstra | 2a67e74 | 2015-10-08 12:24:23 +0200 | [diff] [blame] | 3597 | raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */ |
Paul E. McKenney | b9585e9 | 2015-07-31 16:04:45 -0700 | [diff] [blame] | 3598 | rdp->beenonline = true; /* We have now been online. */ |
Paul E. McKenney | 0aa04b0 | 2015-01-23 21:52:37 -0800 | [diff] [blame] | 3599 | rdp->gpnum = rnp->completed; /* Make CPU later note any new GP. */ |
| 3600 | rdp->completed = rnp->completed; |
Paul E. McKenney | de30ad5 | 2018-04-26 11:52:09 -0700 | [diff] [blame] | 3601 | rdp->gp_seq = rnp->gp_seq; |
Paul E. McKenney | 5b74c45 | 2015-08-06 15:16:57 -0700 | [diff] [blame] | 3602 | rdp->cpu_no_qs.b.norm = true; |
Paul E. McKenney | 9577df9 | 2017-01-26 16:18:07 -0800 | [diff] [blame] | 3603 | rdp->rcu_qs_ctr_snap = per_cpu(rcu_dynticks.rcu_qs_ctr, cpu); |
Paul E. McKenney | 97c668b | 2015-08-06 11:31:51 -0700 | [diff] [blame] | 3604 | rdp->core_needs_qs = false; |
Paul E. McKenney | 9b9500d | 2017-08-17 17:05:59 -0700 | [diff] [blame] | 3605 | rdp->rcu_iw_pending = false; |
| 3606 | rdp->rcu_iw_gpnum = rnp->gpnum - 1; |
Paul E. McKenney | 0aa04b0 | 2015-01-23 21:52:37 -0800 | [diff] [blame] | 3607 | trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpuonl")); |
Boqun Feng | 67c583a7 | 2015-12-29 12:18:47 +0800 | [diff] [blame] | 3608 | raw_spin_unlock_irqrestore_rcu_node(rnp, flags); |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 3609 | } |
| 3610 | |
Paul E. McKenney | deb34f3 | 2017-03-23 13:21:30 -0700 | [diff] [blame] | 3611 | /* |
| 3612 | * Invoked early in the CPU-online process, when pretty much all |
| 3613 | * services are available. The incoming CPU is not present. |
| 3614 | */ |
Thomas Gleixner | 4df8374 | 2016-07-13 17:17:03 +0000 | [diff] [blame] | 3615 | int rcutree_prepare_cpu(unsigned int cpu) |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 3616 | { |
Paul E. McKenney | 6ce75a2 | 2012-06-12 11:01:13 -0700 | [diff] [blame] | 3617 | struct rcu_state *rsp; |
| 3618 | |
| 3619 | for_each_rcu_flavor(rsp) |
Iulia Manda | 9b67122 | 2014-03-11 13:18:22 +0200 | [diff] [blame] | 3620 | rcu_init_percpu_data(cpu, rsp); |
Thomas Gleixner | 4df8374 | 2016-07-13 17:17:03 +0000 | [diff] [blame] | 3621 | |
| 3622 | rcu_prepare_kthreads(cpu); |
| 3623 | rcu_spawn_all_nocb_kthreads(cpu); |
| 3624 | |
| 3625 | return 0; |
| 3626 | } |
| 3627 | |
Paul E. McKenney | deb34f3 | 2017-03-23 13:21:30 -0700 | [diff] [blame] | 3628 | /* |
| 3629 | * Update RCU priority boot kthread affinity for CPU-hotplug changes. |
| 3630 | */ |
Thomas Gleixner | 4df8374 | 2016-07-13 17:17:03 +0000 | [diff] [blame] | 3631 | static void rcutree_affinity_setting(unsigned int cpu, int outgoing) |
| 3632 | { |
| 3633 | struct rcu_data *rdp = per_cpu_ptr(rcu_state_p->rda, cpu); |
| 3634 | |
| 3635 | rcu_boost_kthread_setaffinity(rdp->mynode, outgoing); |
| 3636 | } |
| 3637 | |
Paul E. McKenney | deb34f3 | 2017-03-23 13:21:30 -0700 | [diff] [blame] | 3638 | /* |
| 3639 | * Near the end of the CPU-online process. Pretty much all services |
| 3640 | * enabled, and the CPU is now very much alive. |
| 3641 | */ |
Thomas Gleixner | 4df8374 | 2016-07-13 17:17:03 +0000 | [diff] [blame] | 3642 | int rcutree_online_cpu(unsigned int cpu) |
| 3643 | { |
Paul E. McKenney | 9b9500d | 2017-08-17 17:05:59 -0700 | [diff] [blame] | 3644 | unsigned long flags; |
| 3645 | struct rcu_data *rdp; |
| 3646 | struct rcu_node *rnp; |
| 3647 | struct rcu_state *rsp; |
| 3648 | |
| 3649 | for_each_rcu_flavor(rsp) { |
| 3650 | rdp = per_cpu_ptr(rsp->rda, cpu); |
| 3651 | rnp = rdp->mynode; |
| 3652 | raw_spin_lock_irqsave_rcu_node(rnp, flags); |
| 3653 | rnp->ffmask |= rdp->grpmask; |
| 3654 | raw_spin_unlock_irqrestore_rcu_node(rnp, flags); |
| 3655 | } |
Paul E. McKenney | da915ad | 2017-04-05 09:01:53 -0700 | [diff] [blame] | 3656 | if (IS_ENABLED(CONFIG_TREE_SRCU)) |
| 3657 | srcu_online_cpu(cpu); |
Paul E. McKenney | 9b9500d | 2017-08-17 17:05:59 -0700 | [diff] [blame] | 3658 | if (rcu_scheduler_active == RCU_SCHEDULER_INACTIVE) |
| 3659 | return 0; /* Too early in boot for scheduler work. */ |
| 3660 | sync_sched_exp_online_cleanup(cpu); |
| 3661 | rcutree_affinity_setting(cpu, -1); |
Thomas Gleixner | 4df8374 | 2016-07-13 17:17:03 +0000 | [diff] [blame] | 3662 | return 0; |
| 3663 | } |
| 3664 | |
Paul E. McKenney | deb34f3 | 2017-03-23 13:21:30 -0700 | [diff] [blame] | 3665 | /* |
| 3666 | * Near the beginning of the process. The CPU is still very much alive |
| 3667 | * with pretty much all services enabled. |
| 3668 | */ |
Thomas Gleixner | 4df8374 | 2016-07-13 17:17:03 +0000 | [diff] [blame] | 3669 | int rcutree_offline_cpu(unsigned int cpu) |
| 3670 | { |
Paul E. McKenney | 9b9500d | 2017-08-17 17:05:59 -0700 | [diff] [blame] | 3671 | unsigned long flags; |
| 3672 | struct rcu_data *rdp; |
| 3673 | struct rcu_node *rnp; |
| 3674 | struct rcu_state *rsp; |
| 3675 | |
| 3676 | for_each_rcu_flavor(rsp) { |
| 3677 | rdp = per_cpu_ptr(rsp->rda, cpu); |
| 3678 | rnp = rdp->mynode; |
| 3679 | raw_spin_lock_irqsave_rcu_node(rnp, flags); |
| 3680 | rnp->ffmask &= ~rdp->grpmask; |
| 3681 | raw_spin_unlock_irqrestore_rcu_node(rnp, flags); |
| 3682 | } |
| 3683 | |
Thomas Gleixner | 4df8374 | 2016-07-13 17:17:03 +0000 | [diff] [blame] | 3684 | rcutree_affinity_setting(cpu, cpu); |
Paul E. McKenney | da915ad | 2017-04-05 09:01:53 -0700 | [diff] [blame] | 3685 | if (IS_ENABLED(CONFIG_TREE_SRCU)) |
| 3686 | srcu_offline_cpu(cpu); |
Thomas Gleixner | 4df8374 | 2016-07-13 17:17:03 +0000 | [diff] [blame] | 3687 | return 0; |
| 3688 | } |
| 3689 | |
Paul E. McKenney | deb34f3 | 2017-03-23 13:21:30 -0700 | [diff] [blame] | 3690 | /* |
| 3691 | * Near the end of the offline process. We do only tracing here. |
| 3692 | */ |
Thomas Gleixner | 4df8374 | 2016-07-13 17:17:03 +0000 | [diff] [blame] | 3693 | int rcutree_dying_cpu(unsigned int cpu) |
| 3694 | { |
| 3695 | struct rcu_state *rsp; |
| 3696 | |
| 3697 | for_each_rcu_flavor(rsp) |
| 3698 | rcu_cleanup_dying_cpu(rsp); |
| 3699 | return 0; |
| 3700 | } |
| 3701 | |
Paul E. McKenney | deb34f3 | 2017-03-23 13:21:30 -0700 | [diff] [blame] | 3702 | /* |
| 3703 | * The outgoing CPU is gone and we are running elsewhere. |
| 3704 | */ |
Thomas Gleixner | 4df8374 | 2016-07-13 17:17:03 +0000 | [diff] [blame] | 3705 | int rcutree_dead_cpu(unsigned int cpu) |
| 3706 | { |
| 3707 | struct rcu_state *rsp; |
| 3708 | |
| 3709 | for_each_rcu_flavor(rsp) { |
| 3710 | rcu_cleanup_dead_cpu(cpu, rsp); |
| 3711 | do_nocb_deferred_wakeup(per_cpu_ptr(rsp->rda, cpu)); |
| 3712 | } |
| 3713 | return 0; |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 3714 | } |
| 3715 | |
Peter Zijlstra | f64c601 | 2018-05-22 09:50:53 -0700 | [diff] [blame] | 3716 | static DEFINE_PER_CPU(int, rcu_cpu_started); |
| 3717 | |
Paul E. McKenney | 7ec99de | 2016-06-30 13:58:26 -0700 | [diff] [blame] | 3718 | /* |
| 3719 | * Mark the specified CPU as being online so that subsequent grace periods |
| 3720 | * (both expedited and normal) will wait on it. Note that this means that |
| 3721 | * incoming CPUs are not allowed to use RCU read-side critical sections |
| 3722 | * until this function is called. Failing to observe this restriction |
| 3723 | * will result in lockdep splats. |
Paul E. McKenney | deb34f3 | 2017-03-23 13:21:30 -0700 | [diff] [blame] | 3724 | * |
| 3725 | * Note that this function is special in that it is invoked directly |
| 3726 | * from the incoming CPU rather than from the cpuhp_step mechanism. |
| 3727 | * This is because this function must be invoked at a precise location. |
Paul E. McKenney | 7ec99de | 2016-06-30 13:58:26 -0700 | [diff] [blame] | 3728 | */ |
| 3729 | void rcu_cpu_starting(unsigned int cpu) |
| 3730 | { |
| 3731 | unsigned long flags; |
| 3732 | unsigned long mask; |
Paul E. McKenney | 313517fc | 2017-06-08 16:55:40 -0700 | [diff] [blame] | 3733 | int nbits; |
| 3734 | unsigned long oldmask; |
Paul E. McKenney | 7ec99de | 2016-06-30 13:58:26 -0700 | [diff] [blame] | 3735 | struct rcu_data *rdp; |
| 3736 | struct rcu_node *rnp; |
| 3737 | struct rcu_state *rsp; |
| 3738 | |
Peter Zijlstra | f64c601 | 2018-05-22 09:50:53 -0700 | [diff] [blame] | 3739 | if (per_cpu(rcu_cpu_started, cpu)) |
| 3740 | return; |
| 3741 | |
| 3742 | per_cpu(rcu_cpu_started, cpu) = 1; |
| 3743 | |
Paul E. McKenney | 7ec99de | 2016-06-30 13:58:26 -0700 | [diff] [blame] | 3744 | for_each_rcu_flavor(rsp) { |
Paul E. McKenney | fdbb9b3 | 2016-12-20 07:17:58 -0800 | [diff] [blame] | 3745 | rdp = per_cpu_ptr(rsp->rda, cpu); |
Paul E. McKenney | 7ec99de | 2016-06-30 13:58:26 -0700 | [diff] [blame] | 3746 | rnp = rdp->mynode; |
| 3747 | mask = rdp->grpmask; |
| 3748 | raw_spin_lock_irqsave_rcu_node(rnp, flags); |
| 3749 | rnp->qsmaskinitnext |= mask; |
Paul E. McKenney | 313517fc | 2017-06-08 16:55:40 -0700 | [diff] [blame] | 3750 | oldmask = rnp->expmaskinitnext; |
Paul E. McKenney | 7ec99de | 2016-06-30 13:58:26 -0700 | [diff] [blame] | 3751 | rnp->expmaskinitnext |= mask; |
Paul E. McKenney | 313517fc | 2017-06-08 16:55:40 -0700 | [diff] [blame] | 3752 | oldmask ^= rnp->expmaskinitnext; |
| 3753 | nbits = bitmap_weight(&oldmask, BITS_PER_LONG); |
| 3754 | /* Allow lockless access for expedited grace periods. */ |
| 3755 | smp_store_release(&rsp->ncpus, rsp->ncpus + nbits); /* ^^^ */ |
Paul E. McKenney | 7ec99de | 2016-06-30 13:58:26 -0700 | [diff] [blame] | 3756 | raw_spin_unlock_irqrestore_rcu_node(rnp, flags); |
| 3757 | } |
Paul E. McKenney | 313517fc | 2017-06-08 16:55:40 -0700 | [diff] [blame] | 3758 | smp_mb(); /* Ensure RCU read-side usage follows above initialization. */ |
Paul E. McKenney | 7ec99de | 2016-06-30 13:58:26 -0700 | [diff] [blame] | 3759 | } |
| 3760 | |
Thomas Gleixner | 27d50c7 | 2016-02-26 18:43:44 +0000 | [diff] [blame] | 3761 | #ifdef CONFIG_HOTPLUG_CPU |
| 3762 | /* |
| 3763 | * The CPU is exiting the idle loop into the arch_cpu_idle_dead() |
| 3764 | * function. We now remove it from the rcu_node tree's ->qsmaskinit |
| 3765 | * bit masks. |
| 3766 | */ |
| 3767 | static void rcu_cleanup_dying_idle_cpu(int cpu, struct rcu_state *rsp) |
| 3768 | { |
| 3769 | unsigned long flags; |
| 3770 | unsigned long mask; |
| 3771 | struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu); |
| 3772 | struct rcu_node *rnp = rdp->mynode; /* Outgoing CPU's rdp & rnp. */ |
| 3773 | |
Thomas Gleixner | 27d50c7 | 2016-02-26 18:43:44 +0000 | [diff] [blame] | 3774 | /* Remove outgoing CPU from mask in the leaf rcu_node structure. */ |
| 3775 | mask = rdp->grpmask; |
| 3776 | raw_spin_lock_irqsave_rcu_node(rnp, flags); /* Enforce GP memory-order guarantee. */ |
| 3777 | rnp->qsmaskinitnext &= ~mask; |
Linus Torvalds | 710d60c | 2016-03-15 13:50:29 -0700 | [diff] [blame] | 3778 | raw_spin_unlock_irqrestore_rcu_node(rnp, flags); |
Thomas Gleixner | 27d50c7 | 2016-02-26 18:43:44 +0000 | [diff] [blame] | 3779 | } |
| 3780 | |
Paul E. McKenney | deb34f3 | 2017-03-23 13:21:30 -0700 | [diff] [blame] | 3781 | /* |
| 3782 | * The outgoing function has no further need of RCU, so remove it from |
| 3783 | * the list of CPUs that RCU must track. |
| 3784 | * |
| 3785 | * Note that this function is special in that it is invoked directly |
| 3786 | * from the outgoing CPU rather than from the cpuhp_step mechanism. |
| 3787 | * This is because this function must be invoked at a precise location. |
| 3788 | */ |
Thomas Gleixner | 27d50c7 | 2016-02-26 18:43:44 +0000 | [diff] [blame] | 3789 | void rcu_report_dead(unsigned int cpu) |
| 3790 | { |
| 3791 | struct rcu_state *rsp; |
| 3792 | |
| 3793 | /* QS for any half-done expedited RCU-sched GP. */ |
| 3794 | preempt_disable(); |
| 3795 | rcu_report_exp_rdp(&rcu_sched_state, |
| 3796 | this_cpu_ptr(rcu_sched_state.rda), true); |
| 3797 | preempt_enable(); |
| 3798 | for_each_rcu_flavor(rsp) |
| 3799 | rcu_cleanup_dying_idle_cpu(cpu, rsp); |
Peter Zijlstra | f64c601 | 2018-05-22 09:50:53 -0700 | [diff] [blame] | 3800 | |
| 3801 | per_cpu(rcu_cpu_started, cpu) = 0; |
Thomas Gleixner | 27d50c7 | 2016-02-26 18:43:44 +0000 | [diff] [blame] | 3802 | } |
Paul E. McKenney | a58163d | 2017-06-20 12:11:34 -0700 | [diff] [blame] | 3803 | |
Paul E. McKenney | f2dbe4a | 2017-06-27 07:44:06 -0700 | [diff] [blame] | 3804 | /* Migrate the dead CPU's callbacks to the current CPU. */ |
Paul E. McKenney | a58163d | 2017-06-20 12:11:34 -0700 | [diff] [blame] | 3805 | static void rcu_migrate_callbacks(int cpu, struct rcu_state *rsp) |
| 3806 | { |
| 3807 | unsigned long flags; |
Paul E. McKenney | b1a2d79 | 2017-06-26 12:23:46 -0700 | [diff] [blame] | 3808 | struct rcu_data *my_rdp; |
Paul E. McKenney | a58163d | 2017-06-20 12:11:34 -0700 | [diff] [blame] | 3809 | struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu); |
Paul E. McKenney | 9fa46fb | 2017-06-26 15:43:27 -0700 | [diff] [blame] | 3810 | struct rcu_node *rnp_root = rcu_get_root(rdp->rsp); |
Paul E. McKenney | ec4eacc | 2018-04-22 08:49:24 -0700 | [diff] [blame] | 3811 | bool needwake; |
Paul E. McKenney | a58163d | 2017-06-20 12:11:34 -0700 | [diff] [blame] | 3812 | |
Paul E. McKenney | 95335c0 | 2017-06-26 10:49:50 -0700 | [diff] [blame] | 3813 | if (rcu_is_nocb_cpu(cpu) || rcu_segcblist_empty(&rdp->cblist)) |
| 3814 | return; /* No callbacks to migrate. */ |
| 3815 | |
Paul E. McKenney | b1a2d79 | 2017-06-26 12:23:46 -0700 | [diff] [blame] | 3816 | local_irq_save(flags); |
| 3817 | my_rdp = this_cpu_ptr(rsp->rda); |
| 3818 | if (rcu_nocb_adopt_orphan_cbs(my_rdp, rdp, flags)) { |
| 3819 | local_irq_restore(flags); |
| 3820 | return; |
| 3821 | } |
Paul E. McKenney | 9fa46fb | 2017-06-26 15:43:27 -0700 | [diff] [blame] | 3822 | raw_spin_lock_rcu_node(rnp_root); /* irqs already disabled. */ |
Paul E. McKenney | ec4eacc | 2018-04-22 08:49:24 -0700 | [diff] [blame] | 3823 | /* Leverage recent GPs and set GP for new callbacks. */ |
| 3824 | needwake = rcu_advance_cbs(rsp, rnp_root, rdp) || |
| 3825 | rcu_advance_cbs(rsp, rnp_root, my_rdp); |
Paul E. McKenney | f2dbe4a | 2017-06-27 07:44:06 -0700 | [diff] [blame] | 3826 | rcu_segcblist_merge(&my_rdp->cblist, &rdp->cblist); |
Paul E. McKenney | 09efeee | 2017-07-19 10:56:46 -0700 | [diff] [blame] | 3827 | WARN_ON_ONCE(rcu_segcblist_empty(&my_rdp->cblist) != |
| 3828 | !rcu_segcblist_n_cbs(&my_rdp->cblist)); |
Paul E. McKenney | 537b85c | 2017-06-26 17:59:02 -0700 | [diff] [blame] | 3829 | raw_spin_unlock_irqrestore_rcu_node(rnp_root, flags); |
Paul E. McKenney | ec4eacc | 2018-04-22 08:49:24 -0700 | [diff] [blame] | 3830 | if (needwake) |
| 3831 | rcu_gp_kthread_wake(rsp); |
Paul E. McKenney | a58163d | 2017-06-20 12:11:34 -0700 | [diff] [blame] | 3832 | WARN_ONCE(rcu_segcblist_n_cbs(&rdp->cblist) != 0 || |
| 3833 | !rcu_segcblist_empty(&rdp->cblist), |
| 3834 | "rcu_cleanup_dead_cpu: Callbacks on offline CPU %d: qlen=%lu, 1stCB=%p\n", |
| 3835 | cpu, rcu_segcblist_n_cbs(&rdp->cblist), |
| 3836 | rcu_segcblist_first_cb(&rdp->cblist)); |
| 3837 | } |
| 3838 | |
| 3839 | /* |
| 3840 | * The outgoing CPU has just passed through the dying-idle state, |
| 3841 | * and we are being invoked from the CPU that was IPIed to continue the |
| 3842 | * offline operation. We need to migrate the outgoing CPU's callbacks. |
| 3843 | */ |
| 3844 | void rcutree_migrate_callbacks(int cpu) |
| 3845 | { |
| 3846 | struct rcu_state *rsp; |
| 3847 | |
| 3848 | for_each_rcu_flavor(rsp) |
| 3849 | rcu_migrate_callbacks(cpu, rsp); |
| 3850 | } |
Thomas Gleixner | 27d50c7 | 2016-02-26 18:43:44 +0000 | [diff] [blame] | 3851 | #endif |
| 3852 | |
Paul E. McKenney | deb34f3 | 2017-03-23 13:21:30 -0700 | [diff] [blame] | 3853 | /* |
| 3854 | * On non-huge systems, use expedited RCU grace periods to make suspend |
| 3855 | * and hibernation run faster. |
| 3856 | */ |
Borislav Petkov | d1d74d1 | 2013-04-22 00:12:42 +0200 | [diff] [blame] | 3857 | static int rcu_pm_notify(struct notifier_block *self, |
| 3858 | unsigned long action, void *hcpu) |
| 3859 | { |
| 3860 | switch (action) { |
| 3861 | case PM_HIBERNATION_PREPARE: |
| 3862 | case PM_SUSPEND_PREPARE: |
| 3863 | if (nr_cpu_ids <= 256) /* Expediting bad for large systems. */ |
Paul E. McKenney | 5afff48 | 2015-02-18 16:39:09 -0800 | [diff] [blame] | 3864 | rcu_expedite_gp(); |
Borislav Petkov | d1d74d1 | 2013-04-22 00:12:42 +0200 | [diff] [blame] | 3865 | break; |
| 3866 | case PM_POST_HIBERNATION: |
| 3867 | case PM_POST_SUSPEND: |
Paul E. McKenney | 5afff48 | 2015-02-18 16:39:09 -0800 | [diff] [blame] | 3868 | if (nr_cpu_ids <= 256) /* Expediting bad for large systems. */ |
| 3869 | rcu_unexpedite_gp(); |
Borislav Petkov | d1d74d1 | 2013-04-22 00:12:42 +0200 | [diff] [blame] | 3870 | break; |
| 3871 | default: |
| 3872 | break; |
| 3873 | } |
| 3874 | return NOTIFY_OK; |
| 3875 | } |
| 3876 | |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 3877 | /* |
Paul E. McKenney | 9386c0b | 2014-07-13 12:00:53 -0700 | [diff] [blame] | 3878 | * Spawn the kthreads that handle each RCU flavor's grace periods. |
Paul E. McKenney | b3dbec7 | 2012-06-18 18:36:08 -0700 | [diff] [blame] | 3879 | */ |
| 3880 | static int __init rcu_spawn_gp_kthread(void) |
| 3881 | { |
| 3882 | unsigned long flags; |
Paul E. McKenney | a94844b | 2014-12-12 07:37:48 -0800 | [diff] [blame] | 3883 | int kthread_prio_in = kthread_prio; |
Paul E. McKenney | b3dbec7 | 2012-06-18 18:36:08 -0700 | [diff] [blame] | 3884 | struct rcu_node *rnp; |
| 3885 | struct rcu_state *rsp; |
Paul E. McKenney | a94844b | 2014-12-12 07:37:48 -0800 | [diff] [blame] | 3886 | struct sched_param sp; |
Paul E. McKenney | b3dbec7 | 2012-06-18 18:36:08 -0700 | [diff] [blame] | 3887 | struct task_struct *t; |
| 3888 | |
Paul E. McKenney | a94844b | 2014-12-12 07:37:48 -0800 | [diff] [blame] | 3889 | /* Force priority into range. */ |
| 3890 | if (IS_ENABLED(CONFIG_RCU_BOOST) && kthread_prio < 1) |
| 3891 | kthread_prio = 1; |
| 3892 | else if (kthread_prio < 0) |
| 3893 | kthread_prio = 0; |
| 3894 | else if (kthread_prio > 99) |
| 3895 | kthread_prio = 99; |
| 3896 | if (kthread_prio != kthread_prio_in) |
| 3897 | pr_alert("rcu_spawn_gp_kthread(): Limited prio to %d from %d\n", |
| 3898 | kthread_prio, kthread_prio_in); |
| 3899 | |
Paul E. McKenney | 9386c0b | 2014-07-13 12:00:53 -0700 | [diff] [blame] | 3900 | rcu_scheduler_fully_active = 1; |
Paul E. McKenney | b3dbec7 | 2012-06-18 18:36:08 -0700 | [diff] [blame] | 3901 | for_each_rcu_flavor(rsp) { |
Paul E. McKenney | a94844b | 2014-12-12 07:37:48 -0800 | [diff] [blame] | 3902 | t = kthread_create(rcu_gp_kthread, rsp, "%s", rsp->name); |
Paul E. McKenney | b3dbec7 | 2012-06-18 18:36:08 -0700 | [diff] [blame] | 3903 | BUG_ON(IS_ERR(t)); |
| 3904 | rnp = rcu_get_root(rsp); |
Paul E. McKenney | 6cf1008 | 2015-10-08 15:36:54 -0700 | [diff] [blame] | 3905 | raw_spin_lock_irqsave_rcu_node(rnp, flags); |
Paul E. McKenney | b3dbec7 | 2012-06-18 18:36:08 -0700 | [diff] [blame] | 3906 | rsp->gp_kthread = t; |
Paul E. McKenney | a94844b | 2014-12-12 07:37:48 -0800 | [diff] [blame] | 3907 | if (kthread_prio) { |
| 3908 | sp.sched_priority = kthread_prio; |
| 3909 | sched_setscheduler_nocheck(t, SCHED_FIFO, &sp); |
| 3910 | } |
Boqun Feng | 67c583a7 | 2015-12-29 12:18:47 +0800 | [diff] [blame] | 3911 | raw_spin_unlock_irqrestore_rcu_node(rnp, flags); |
Peter Zijlstra | e11f133 | 2015-11-04 08:22:05 -0800 | [diff] [blame] | 3912 | wake_up_process(t); |
Paul E. McKenney | b3dbec7 | 2012-06-18 18:36:08 -0700 | [diff] [blame] | 3913 | } |
Paul E. McKenney | 35ce7f2 | 2014-07-11 11:30:24 -0700 | [diff] [blame] | 3914 | rcu_spawn_nocb_kthreads(); |
Paul E. McKenney | 9386c0b | 2014-07-13 12:00:53 -0700 | [diff] [blame] | 3915 | rcu_spawn_boost_kthreads(); |
Paul E. McKenney | b3dbec7 | 2012-06-18 18:36:08 -0700 | [diff] [blame] | 3916 | return 0; |
| 3917 | } |
| 3918 | early_initcall(rcu_spawn_gp_kthread); |
| 3919 | |
| 3920 | /* |
Paul E. McKenney | 52d7e48 | 2017-01-10 02:28:26 -0800 | [diff] [blame] | 3921 | * This function is invoked towards the end of the scheduler's |
| 3922 | * initialization process. Before this is called, the idle task might |
| 3923 | * contain synchronous grace-period primitives (during which time, this idle |
| 3924 | * task is booting the system, and such primitives are no-ops). After this |
| 3925 | * function is called, any synchronous grace-period primitives are run as |
| 3926 | * expedited, with the requesting task driving the grace period forward. |
Paul E. McKenney | 900b102 | 2017-02-10 14:32:54 -0800 | [diff] [blame] | 3927 | * A later core_initcall() rcu_set_runtime_mode() will switch to full |
Paul E. McKenney | 52d7e48 | 2017-01-10 02:28:26 -0800 | [diff] [blame] | 3928 | * runtime RCU functionality. |
Paul E. McKenney | bbad937 | 2010-04-02 16:17:17 -0700 | [diff] [blame] | 3929 | */ |
| 3930 | void rcu_scheduler_starting(void) |
| 3931 | { |
| 3932 | WARN_ON(num_online_cpus() != 1); |
| 3933 | WARN_ON(nr_context_switches() > 0); |
Paul E. McKenney | 52d7e48 | 2017-01-10 02:28:26 -0800 | [diff] [blame] | 3934 | rcu_test_sync_prims(); |
| 3935 | rcu_scheduler_active = RCU_SCHEDULER_INIT; |
| 3936 | rcu_test_sync_prims(); |
Paul E. McKenney | bbad937 | 2010-04-02 16:17:17 -0700 | [diff] [blame] | 3937 | } |
| 3938 | |
| 3939 | /* |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 3940 | * Helper function for rcu_init() that initializes one rcu_state structure. |
| 3941 | */ |
Paul E. McKenney | a87f203 | 2015-10-20 12:38:49 -0700 | [diff] [blame] | 3942 | static void __init rcu_init_one(struct rcu_state *rsp) |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 3943 | { |
Alexander Gordeev | cb00710 | 2015-06-03 08:18:30 +0200 | [diff] [blame] | 3944 | static const char * const buf[] = RCU_NODE_NAME_INIT; |
| 3945 | static const char * const fqs[] = RCU_FQS_NAME_INIT; |
Paul E. McKenney | 3dc5dbe | 2015-09-26 14:51:24 -0700 | [diff] [blame] | 3946 | static struct lock_class_key rcu_node_class[RCU_NUM_LVLS]; |
| 3947 | static struct lock_class_key rcu_fqs_class[RCU_NUM_LVLS]; |
Alexander Gordeev | 199977b | 2015-06-03 08:18:29 +0200 | [diff] [blame] | 3948 | |
Alexander Gordeev | 199977b | 2015-06-03 08:18:29 +0200 | [diff] [blame] | 3949 | int levelspread[RCU_NUM_LVLS]; /* kids/node in each level. */ |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 3950 | int cpustride = 1; |
| 3951 | int i; |
| 3952 | int j; |
| 3953 | struct rcu_node *rnp; |
| 3954 | |
Alexander Gordeev | 05b84ae | 2015-06-03 08:18:28 +0200 | [diff] [blame] | 3955 | BUILD_BUG_ON(RCU_NUM_LVLS > ARRAY_SIZE(buf)); /* Fix buf[] init! */ |
Paul E. McKenney | b6407e8 | 2010-01-04 16:04:02 -0800 | [diff] [blame] | 3956 | |
Paul E. McKenney | 3eaaaf6c | 2015-03-09 16:51:17 -0700 | [diff] [blame] | 3957 | /* Silence gcc 4.8 false positive about array index out of range. */ |
| 3958 | if (rcu_num_lvls <= 0 || rcu_num_lvls > RCU_NUM_LVLS) |
| 3959 | panic("rcu_init_one: rcu_num_lvls out of range"); |
Paul E. McKenney | 4930521 | 2012-11-29 13:49:00 -0800 | [diff] [blame] | 3960 | |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 3961 | /* Initialize the level-tracking arrays. */ |
| 3962 | |
Paul E. McKenney | f885b7f | 2012-04-23 15:52:53 -0700 | [diff] [blame] | 3963 | for (i = 1; i < rcu_num_lvls; i++) |
Paul E. McKenney | 41f5c63 | 2017-03-15 12:59:17 -0700 | [diff] [blame] | 3964 | rsp->level[i] = rsp->level[i - 1] + num_rcu_lvl[i - 1]; |
| 3965 | rcu_init_levelspread(levelspread, num_rcu_lvl); |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 3966 | |
| 3967 | /* Initialize the elements themselves, starting from the leaves. */ |
| 3968 | |
Paul E. McKenney | f885b7f | 2012-04-23 15:52:53 -0700 | [diff] [blame] | 3969 | for (i = rcu_num_lvls - 1; i >= 0; i--) { |
Alexander Gordeev | 199977b | 2015-06-03 08:18:29 +0200 | [diff] [blame] | 3970 | cpustride *= levelspread[i]; |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 3971 | rnp = rsp->level[i]; |
Paul E. McKenney | 41f5c63 | 2017-03-15 12:59:17 -0700 | [diff] [blame] | 3972 | for (j = 0; j < num_rcu_lvl[i]; j++, rnp++) { |
Boqun Feng | 67c583a7 | 2015-12-29 12:18:47 +0800 | [diff] [blame] | 3973 | raw_spin_lock_init(&ACCESS_PRIVATE(rnp, lock)); |
| 3974 | lockdep_set_class_and_name(&ACCESS_PRIVATE(rnp, lock), |
Paul E. McKenney | b6407e8 | 2010-01-04 16:04:02 -0800 | [diff] [blame] | 3975 | &rcu_node_class[i], buf[i]); |
Paul E. McKenney | 394f276 | 2012-06-26 17:00:35 -0700 | [diff] [blame] | 3976 | raw_spin_lock_init(&rnp->fqslock); |
| 3977 | lockdep_set_class_and_name(&rnp->fqslock, |
| 3978 | &rcu_fqs_class[i], fqs[i]); |
Paul E. McKenney | 25d30cf | 2012-07-11 05:23:18 -0700 | [diff] [blame] | 3979 | rnp->gpnum = rsp->gpnum; |
| 3980 | rnp->completed = rsp->completed; |
Paul E. McKenney | de30ad5 | 2018-04-26 11:52:09 -0700 | [diff] [blame] | 3981 | rnp->gp_seq = rsp->gp_seq; |
Paul E. McKenney | 4bc8d55 | 2017-11-27 15:13:56 -0800 | [diff] [blame] | 3982 | rnp->completedqs = rsp->completed; |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 3983 | rnp->qsmask = 0; |
| 3984 | rnp->qsmaskinit = 0; |
| 3985 | rnp->grplo = j * cpustride; |
| 3986 | rnp->grphi = (j + 1) * cpustride - 1; |
Himangi Saraogi | 595f390 | 2014-03-18 22:52:26 +0530 | [diff] [blame] | 3987 | if (rnp->grphi >= nr_cpu_ids) |
| 3988 | rnp->grphi = nr_cpu_ids - 1; |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 3989 | if (i == 0) { |
| 3990 | rnp->grpnum = 0; |
| 3991 | rnp->grpmask = 0; |
| 3992 | rnp->parent = NULL; |
| 3993 | } else { |
Alexander Gordeev | 199977b | 2015-06-03 08:18:29 +0200 | [diff] [blame] | 3994 | rnp->grpnum = j % levelspread[i - 1]; |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 3995 | rnp->grpmask = 1UL << rnp->grpnum; |
| 3996 | rnp->parent = rsp->level[i - 1] + |
Alexander Gordeev | 199977b | 2015-06-03 08:18:29 +0200 | [diff] [blame] | 3997 | j / levelspread[i - 1]; |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 3998 | } |
| 3999 | rnp->level = i; |
Paul E. McKenney | 12f5f52 | 2010-11-29 21:56:39 -0800 | [diff] [blame] | 4000 | INIT_LIST_HEAD(&rnp->blkd_tasks); |
Paul E. McKenney | dae6e64 | 2013-02-10 20:48:58 -0800 | [diff] [blame] | 4001 | rcu_init_one_nocb(rnp); |
Paul E. McKenney | f6a12f3 | 2016-01-30 17:57:35 -0800 | [diff] [blame] | 4002 | init_waitqueue_head(&rnp->exp_wq[0]); |
| 4003 | init_waitqueue_head(&rnp->exp_wq[1]); |
Paul E. McKenney | 3b5f668 | 2016-03-16 16:47:55 -0700 | [diff] [blame] | 4004 | init_waitqueue_head(&rnp->exp_wq[2]); |
| 4005 | init_waitqueue_head(&rnp->exp_wq[3]); |
Paul E. McKenney | f6a12f3 | 2016-01-30 17:57:35 -0800 | [diff] [blame] | 4006 | spin_lock_init(&rnp->exp_lock); |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 4007 | } |
| 4008 | } |
Lai Jiangshan | 0c34029 | 2010-03-28 11:12:30 +0800 | [diff] [blame] | 4009 | |
Paul Gortmaker | abedf8e | 2016-02-19 09:46:41 +0100 | [diff] [blame] | 4010 | init_swait_queue_head(&rsp->gp_wq); |
| 4011 | init_swait_queue_head(&rsp->expedited_wq); |
Paul E. McKenney | 5b4c11d | 2018-04-13 17:11:44 -0700 | [diff] [blame] | 4012 | rnp = rcu_first_leaf_node(rsp); |
Lai Jiangshan | 0c34029 | 2010-03-28 11:12:30 +0800 | [diff] [blame] | 4013 | for_each_possible_cpu(i) { |
Paul E. McKenney | 4a90a06 | 2010-04-14 16:48:11 -0700 | [diff] [blame] | 4014 | while (i > rnp->grphi) |
Lai Jiangshan | 0c34029 | 2010-03-28 11:12:30 +0800 | [diff] [blame] | 4015 | rnp++; |
Lai Jiangshan | 394f99a | 2010-06-28 16:25:04 +0800 | [diff] [blame] | 4016 | per_cpu_ptr(rsp->rda, i)->mynode = rnp; |
Lai Jiangshan | 0c34029 | 2010-03-28 11:12:30 +0800 | [diff] [blame] | 4017 | rcu_boot_init_percpu_data(i, rsp); |
| 4018 | } |
Paul E. McKenney | 6ce75a2 | 2012-06-12 11:01:13 -0700 | [diff] [blame] | 4019 | list_add(&rsp->flavors, &rcu_struct_flavors); |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 4020 | } |
| 4021 | |
Paul E. McKenney | f885b7f | 2012-04-23 15:52:53 -0700 | [diff] [blame] | 4022 | /* |
| 4023 | * Compute the rcu_node tree geometry from kernel parameters. This cannot |
Paul E. McKenney | 4102ada | 2013-10-08 20:23:47 -0700 | [diff] [blame] | 4024 | * replace the definitions in tree.h because those are needed to size |
Paul E. McKenney | f885b7f | 2012-04-23 15:52:53 -0700 | [diff] [blame] | 4025 | * the ->node array in the rcu_state structure. |
| 4026 | */ |
| 4027 | static void __init rcu_init_geometry(void) |
| 4028 | { |
Paul E. McKenney | 026ad28 | 2013-04-03 22:14:11 -0700 | [diff] [blame] | 4029 | ulong d; |
Paul E. McKenney | f885b7f | 2012-04-23 15:52:53 -0700 | [diff] [blame] | 4030 | int i; |
Alexander Gordeev | 05b84ae | 2015-06-03 08:18:28 +0200 | [diff] [blame] | 4031 | int rcu_capacity[RCU_NUM_LVLS]; |
Paul E. McKenney | f885b7f | 2012-04-23 15:52:53 -0700 | [diff] [blame] | 4032 | |
Paul E. McKenney | 026ad28 | 2013-04-03 22:14:11 -0700 | [diff] [blame] | 4033 | /* |
| 4034 | * Initialize any unspecified boot parameters. |
| 4035 | * The default values of jiffies_till_first_fqs and |
| 4036 | * jiffies_till_next_fqs are set to the RCU_JIFFIES_TILL_FORCE_QS |
| 4037 | * value, which is a function of HZ, then adding one for each |
| 4038 | * RCU_JIFFIES_FQS_DIV CPUs that might be on the system. |
| 4039 | */ |
| 4040 | d = RCU_JIFFIES_TILL_FORCE_QS + nr_cpu_ids / RCU_JIFFIES_FQS_DIV; |
| 4041 | if (jiffies_till_first_fqs == ULONG_MAX) |
| 4042 | jiffies_till_first_fqs = d; |
| 4043 | if (jiffies_till_next_fqs == ULONG_MAX) |
| 4044 | jiffies_till_next_fqs = d; |
| 4045 | |
Paul E. McKenney | f885b7f | 2012-04-23 15:52:53 -0700 | [diff] [blame] | 4046 | /* If the compile-time values are accurate, just leave. */ |
Paul E. McKenney | 47d631a | 2015-04-21 09:12:13 -0700 | [diff] [blame] | 4047 | if (rcu_fanout_leaf == RCU_FANOUT_LEAF && |
Paul E. McKenney | b17c703 | 2012-09-06 15:38:02 -0700 | [diff] [blame] | 4048 | nr_cpu_ids == NR_CPUS) |
Paul E. McKenney | f885b7f | 2012-04-23 15:52:53 -0700 | [diff] [blame] | 4049 | return; |
Alexey Dobriyan | 9b130ad | 2017-09-08 16:14:18 -0700 | [diff] [blame] | 4050 | pr_info("RCU: Adjusting geometry for rcu_fanout_leaf=%d, nr_cpu_ids=%u\n", |
Paul E. McKenney | 3947909 | 2013-10-09 15:20:33 -0700 | [diff] [blame] | 4051 | rcu_fanout_leaf, nr_cpu_ids); |
Paul E. McKenney | f885b7f | 2012-04-23 15:52:53 -0700 | [diff] [blame] | 4052 | |
| 4053 | /* |
Paul E. McKenney | ee968ac | 2015-07-31 08:28:35 -0700 | [diff] [blame] | 4054 | * The boot-time rcu_fanout_leaf parameter must be at least two |
| 4055 | * and cannot exceed the number of bits in the rcu_node masks. |
| 4056 | * Complain and fall back to the compile-time values if this |
| 4057 | * limit is exceeded. |
Paul E. McKenney | f885b7f | 2012-04-23 15:52:53 -0700 | [diff] [blame] | 4058 | */ |
Paul E. McKenney | ee968ac | 2015-07-31 08:28:35 -0700 | [diff] [blame] | 4059 | if (rcu_fanout_leaf < 2 || |
Alexander Gordeev | 75cf15a | 2015-06-03 08:18:23 +0200 | [diff] [blame] | 4060 | rcu_fanout_leaf > sizeof(unsigned long) * 8) { |
Paul E. McKenney | 13bd6494 | 2015-06-04 10:06:01 -0700 | [diff] [blame] | 4061 | rcu_fanout_leaf = RCU_FANOUT_LEAF; |
Paul E. McKenney | f885b7f | 2012-04-23 15:52:53 -0700 | [diff] [blame] | 4062 | WARN_ON(1); |
| 4063 | return; |
| 4064 | } |
| 4065 | |
Alexander Gordeev | 75cf15a | 2015-06-03 08:18:23 +0200 | [diff] [blame] | 4066 | /* |
Paul E. McKenney | f885b7f | 2012-04-23 15:52:53 -0700 | [diff] [blame] | 4067 | * Compute number of nodes that can be handled an rcu_node tree |
Alexander Gordeev | 9618138 | 2015-06-03 08:18:26 +0200 | [diff] [blame] | 4068 | * with the given number of levels. |
Paul E. McKenney | f885b7f | 2012-04-23 15:52:53 -0700 | [diff] [blame] | 4069 | */ |
Alexander Gordeev | 9618138 | 2015-06-03 08:18:26 +0200 | [diff] [blame] | 4070 | rcu_capacity[0] = rcu_fanout_leaf; |
Alexander Gordeev | 05b84ae | 2015-06-03 08:18:28 +0200 | [diff] [blame] | 4071 | for (i = 1; i < RCU_NUM_LVLS; i++) |
Paul E. McKenney | f885b7f | 2012-04-23 15:52:53 -0700 | [diff] [blame] | 4072 | rcu_capacity[i] = rcu_capacity[i - 1] * RCU_FANOUT; |
| 4073 | |
| 4074 | /* |
Alexander Gordeev | 75cf15a | 2015-06-03 08:18:23 +0200 | [diff] [blame] | 4075 | * The tree must be able to accommodate the configured number of CPUs. |
Paul E. McKenney | ee968ac | 2015-07-31 08:28:35 -0700 | [diff] [blame] | 4076 | * If this limit is exceeded, fall back to the compile-time values. |
Paul E. McKenney | f885b7f | 2012-04-23 15:52:53 -0700 | [diff] [blame] | 4077 | */ |
Paul E. McKenney | ee968ac | 2015-07-31 08:28:35 -0700 | [diff] [blame] | 4078 | if (nr_cpu_ids > rcu_capacity[RCU_NUM_LVLS - 1]) { |
| 4079 | rcu_fanout_leaf = RCU_FANOUT_LEAF; |
| 4080 | WARN_ON(1); |
| 4081 | return; |
| 4082 | } |
Paul E. McKenney | f885b7f | 2012-04-23 15:52:53 -0700 | [diff] [blame] | 4083 | |
Alexander Gordeev | 679f985 | 2015-06-03 08:18:25 +0200 | [diff] [blame] | 4084 | /* Calculate the number of levels in the tree. */ |
Alexander Gordeev | 9618138 | 2015-06-03 08:18:26 +0200 | [diff] [blame] | 4085 | for (i = 0; nr_cpu_ids > rcu_capacity[i]; i++) { |
Alexander Gordeev | 679f985 | 2015-06-03 08:18:25 +0200 | [diff] [blame] | 4086 | } |
Alexander Gordeev | 9618138 | 2015-06-03 08:18:26 +0200 | [diff] [blame] | 4087 | rcu_num_lvls = i + 1; |
Alexander Gordeev | 679f985 | 2015-06-03 08:18:25 +0200 | [diff] [blame] | 4088 | |
Paul E. McKenney | f885b7f | 2012-04-23 15:52:53 -0700 | [diff] [blame] | 4089 | /* Calculate the number of rcu_nodes at each level of the tree. */ |
Alexander Gordeev | 679f985 | 2015-06-03 08:18:25 +0200 | [diff] [blame] | 4090 | for (i = 0; i < rcu_num_lvls; i++) { |
Alexander Gordeev | 9618138 | 2015-06-03 08:18:26 +0200 | [diff] [blame] | 4091 | int cap = rcu_capacity[(rcu_num_lvls - 1) - i]; |
Alexander Gordeev | 679f985 | 2015-06-03 08:18:25 +0200 | [diff] [blame] | 4092 | num_rcu_lvl[i] = DIV_ROUND_UP(nr_cpu_ids, cap); |
| 4093 | } |
Paul E. McKenney | f885b7f | 2012-04-23 15:52:53 -0700 | [diff] [blame] | 4094 | |
| 4095 | /* Calculate the total number of rcu_node structures. */ |
| 4096 | rcu_num_nodes = 0; |
Alexander Gordeev | 679f985 | 2015-06-03 08:18:25 +0200 | [diff] [blame] | 4097 | for (i = 0; i < rcu_num_lvls; i++) |
Paul E. McKenney | f885b7f | 2012-04-23 15:52:53 -0700 | [diff] [blame] | 4098 | rcu_num_nodes += num_rcu_lvl[i]; |
Paul E. McKenney | f885b7f | 2012-04-23 15:52:53 -0700 | [diff] [blame] | 4099 | } |
| 4100 | |
Paul E. McKenney | a3dc294 | 2015-04-20 11:40:50 -0700 | [diff] [blame] | 4101 | /* |
| 4102 | * Dump out the structure of the rcu_node combining tree associated |
| 4103 | * with the rcu_state structure referenced by rsp. |
| 4104 | */ |
| 4105 | static void __init rcu_dump_rcu_node_tree(struct rcu_state *rsp) |
| 4106 | { |
| 4107 | int level = 0; |
| 4108 | struct rcu_node *rnp; |
| 4109 | |
| 4110 | pr_info("rcu_node tree layout dump\n"); |
| 4111 | pr_info(" "); |
| 4112 | rcu_for_each_node_breadth_first(rsp, rnp) { |
| 4113 | if (rnp->level != level) { |
| 4114 | pr_cont("\n"); |
| 4115 | pr_info(" "); |
| 4116 | level = rnp->level; |
| 4117 | } |
| 4118 | pr_cont("%d:%d ^%d ", rnp->grplo, rnp->grphi, rnp->grpnum); |
| 4119 | } |
| 4120 | pr_cont("\n"); |
| 4121 | } |
| 4122 | |
Paul E. McKenney | ad7c946 | 2018-01-08 14:35:52 -0800 | [diff] [blame] | 4123 | struct workqueue_struct *rcu_gp_wq; |
Paul E. McKenney | 25f3d7e | 2018-02-01 22:05:38 -0800 | [diff] [blame] | 4124 | struct workqueue_struct *rcu_par_gp_wq; |
Paul E. McKenney | ad7c946 | 2018-01-08 14:35:52 -0800 | [diff] [blame] | 4125 | |
Paul E. McKenney | 9f680ab | 2009-11-22 08:53:49 -0800 | [diff] [blame] | 4126 | void __init rcu_init(void) |
Paul E. McKenney | f41d911 | 2009-08-22 13:56:52 -0700 | [diff] [blame] | 4127 | { |
Paul E. McKenney | 017c426 | 2010-01-14 16:10:58 -0800 | [diff] [blame] | 4128 | int cpu; |
Paul E. McKenney | 9f680ab | 2009-11-22 08:53:49 -0800 | [diff] [blame] | 4129 | |
Paul E. McKenney | 4762767 | 2015-01-19 21:10:21 -0800 | [diff] [blame] | 4130 | rcu_early_boot_tests(); |
| 4131 | |
Paul E. McKenney | f41d911 | 2009-08-22 13:56:52 -0700 | [diff] [blame] | 4132 | rcu_bootup_announce(); |
Paul E. McKenney | f885b7f | 2012-04-23 15:52:53 -0700 | [diff] [blame] | 4133 | rcu_init_geometry(); |
Paul E. McKenney | a87f203 | 2015-10-20 12:38:49 -0700 | [diff] [blame] | 4134 | rcu_init_one(&rcu_bh_state); |
| 4135 | rcu_init_one(&rcu_sched_state); |
Paul E. McKenney | a3dc294 | 2015-04-20 11:40:50 -0700 | [diff] [blame] | 4136 | if (dump_tree) |
| 4137 | rcu_dump_rcu_node_tree(&rcu_sched_state); |
Paul E. McKenney | f41d911 | 2009-08-22 13:56:52 -0700 | [diff] [blame] | 4138 | __rcu_init_preempt(); |
Jiang Fang | b5b3936 | 2013-02-02 14:13:42 -0800 | [diff] [blame] | 4139 | open_softirq(RCU_SOFTIRQ, rcu_process_callbacks); |
Paul E. McKenney | 9f680ab | 2009-11-22 08:53:49 -0800 | [diff] [blame] | 4140 | |
| 4141 | /* |
| 4142 | * We don't need protection against CPU-hotplug here because |
| 4143 | * this is called early in boot, before either interrupts |
| 4144 | * or the scheduler are operational. |
| 4145 | */ |
Borislav Petkov | d1d74d1 | 2013-04-22 00:12:42 +0200 | [diff] [blame] | 4146 | pm_notifier(rcu_pm_notify, 0); |
Paul E. McKenney | 7ec99de | 2016-06-30 13:58:26 -0700 | [diff] [blame] | 4147 | for_each_online_cpu(cpu) { |
Thomas Gleixner | 4df8374 | 2016-07-13 17:17:03 +0000 | [diff] [blame] | 4148 | rcutree_prepare_cpu(cpu); |
Paul E. McKenney | 7ec99de | 2016-06-30 13:58:26 -0700 | [diff] [blame] | 4149 | rcu_cpu_starting(cpu); |
Paul E. McKenney | 9b9500d | 2017-08-17 17:05:59 -0700 | [diff] [blame] | 4150 | rcutree_online_cpu(cpu); |
Paul E. McKenney | 7ec99de | 2016-06-30 13:58:26 -0700 | [diff] [blame] | 4151 | } |
Paul E. McKenney | ad7c946 | 2018-01-08 14:35:52 -0800 | [diff] [blame] | 4152 | |
| 4153 | /* Create workqueue for expedited GPs and for Tree SRCU. */ |
| 4154 | rcu_gp_wq = alloc_workqueue("rcu_gp", WQ_MEM_RECLAIM, 0); |
| 4155 | WARN_ON(!rcu_gp_wq); |
Paul E. McKenney | 25f3d7e | 2018-02-01 22:05:38 -0800 | [diff] [blame] | 4156 | rcu_par_gp_wq = alloc_workqueue("rcu_par_gp", WQ_MEM_RECLAIM, 0); |
| 4157 | WARN_ON(!rcu_par_gp_wq); |
Paul E. McKenney | 64db4cf | 2008-12-18 21:55:32 +0100 | [diff] [blame] | 4158 | } |
| 4159 | |
Paul E. McKenney | 3549c2b | 2016-04-15 16:35:29 -0700 | [diff] [blame] | 4160 | #include "tree_exp.h" |
Paul E. McKenney | 4102ada | 2013-10-08 20:23:47 -0700 | [diff] [blame] | 4161 | #include "tree_plugin.h" |