Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0+ |
| 2 | // |
| 3 | // Torture test for smp_call_function() and friends. |
| 4 | // |
| 5 | // Copyright (C) Facebook, 2020. |
| 6 | // |
| 7 | // Author: Paul E. McKenney <paulmck@kernel.org> |
| 8 | |
| 9 | #define pr_fmt(fmt) fmt |
| 10 | |
| 11 | #include <linux/atomic.h> |
| 12 | #include <linux/bitops.h> |
| 13 | #include <linux/completion.h> |
| 14 | #include <linux/cpu.h> |
| 15 | #include <linux/delay.h> |
| 16 | #include <linux/err.h> |
| 17 | #include <linux/init.h> |
| 18 | #include <linux/interrupt.h> |
| 19 | #include <linux/kthread.h> |
| 20 | #include <linux/kernel.h> |
| 21 | #include <linux/mm.h> |
| 22 | #include <linux/module.h> |
| 23 | #include <linux/moduleparam.h> |
| 24 | #include <linux/notifier.h> |
| 25 | #include <linux/percpu.h> |
| 26 | #include <linux/rcupdate.h> |
| 27 | #include <linux/rcupdate_trace.h> |
| 28 | #include <linux/reboot.h> |
| 29 | #include <linux/sched.h> |
| 30 | #include <linux/spinlock.h> |
| 31 | #include <linux/smp.h> |
| 32 | #include <linux/stat.h> |
| 33 | #include <linux/srcu.h> |
| 34 | #include <linux/slab.h> |
| 35 | #include <linux/torture.h> |
| 36 | #include <linux/types.h> |
| 37 | |
| 38 | #define SCFTORT_STRING "scftorture" |
| 39 | #define SCFTORT_FLAG SCFTORT_STRING ": " |
| 40 | |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 41 | #define VERBOSE_SCFTORTOUT(s, x...) \ |
Li Zhijian | 71f6ea2a | 2021-10-29 17:40:25 +0800 | [diff] [blame] | 42 | do { if (verbose) pr_alert(SCFTORT_FLAG s "\n", ## x); } while (0) |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 43 | |
| 44 | #define VERBOSE_SCFTORTOUT_ERRSTRING(s, x...) \ |
Li Zhijian | 71f6ea2a | 2021-10-29 17:40:25 +0800 | [diff] [blame] | 45 | do { if (verbose) pr_alert(SCFTORT_FLAG "!!! " s "\n", ## x); } while (0) |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 46 | |
| 47 | MODULE_LICENSE("GPL"); |
| 48 | MODULE_AUTHOR("Paul E. McKenney <paulmck@kernel.org>"); |
| 49 | |
| 50 | // Wait until there are multiple CPUs before starting test. |
| 51 | torture_param(int, holdoff, IS_BUILTIN(CONFIG_SCF_TORTURE_TEST) ? 10 : 0, |
| 52 | "Holdoff time before test start (s)"); |
| 53 | torture_param(int, longwait, 0, "Include ridiculously long waits? (seconds)"); |
| 54 | torture_param(int, nthreads, -1, "# threads, defaults to -1 for all CPUs."); |
| 55 | torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)"); |
| 56 | torture_param(int, onoff_interval, 0, "Time between CPU hotplugs (s), 0=disable"); |
| 57 | torture_param(int, shutdown_secs, 0, "Shutdown time (ms), <= zero to disable."); |
| 58 | torture_param(int, stat_interval, 60, "Number of seconds between stats printk()s."); |
Paul E. McKenney | 8555818 | 2020-09-24 12:11:57 -0700 | [diff] [blame] | 59 | torture_param(int, stutter, 5, "Number of jiffies to run/halt test, 0=disable"); |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 60 | torture_param(bool, use_cpus_read_lock, 0, "Use cpus_read_lock() to exclude CPU hotplug."); |
| 61 | torture_param(int, verbose, 0, "Enable verbose debugging printk()s"); |
Paul E. McKenney | 1ac78b4 | 2020-09-03 13:09:47 -0700 | [diff] [blame] | 62 | torture_param(int, weight_resched, -1, "Testing weight for resched_cpu() operations."); |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 63 | torture_param(int, weight_single, -1, "Testing weight for single-CPU no-wait operations."); |
Paul E. McKenney | 9b9a806 | 2021-06-24 17:53:43 -0700 | [diff] [blame] | 64 | torture_param(int, weight_single_rpc, -1, "Testing weight for single-CPU RPC operations."); |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 65 | torture_param(int, weight_single_wait, -1, "Testing weight for single-CPU operations."); |
Paul E. McKenney | 5022b8a | 2020-06-25 17:05:58 -0700 | [diff] [blame] | 66 | torture_param(int, weight_many, -1, "Testing weight for multi-CPU no-wait operations."); |
| 67 | torture_param(int, weight_many_wait, -1, "Testing weight for multi-CPU operations."); |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 68 | torture_param(int, weight_all, -1, "Testing weight for all-CPU no-wait operations."); |
| 69 | torture_param(int, weight_all_wait, -1, "Testing weight for all-CPU operations."); |
| 70 | |
| 71 | char *torture_type = ""; |
| 72 | |
| 73 | #ifdef MODULE |
| 74 | # define SCFTORT_SHUTDOWN 0 |
| 75 | #else |
| 76 | # define SCFTORT_SHUTDOWN 1 |
| 77 | #endif |
| 78 | |
| 79 | torture_param(bool, shutdown, SCFTORT_SHUTDOWN, "Shutdown at end of torture test."); |
| 80 | |
| 81 | struct scf_statistics { |
| 82 | struct task_struct *task; |
| 83 | int cpu; |
Paul E. McKenney | 1ac78b4 | 2020-09-03 13:09:47 -0700 | [diff] [blame] | 84 | long long n_resched; |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 85 | long long n_single; |
Paul E. McKenney | 5022b8a | 2020-06-25 17:05:58 -0700 | [diff] [blame] | 86 | long long n_single_ofl; |
Paul E. McKenney | 9b9a806 | 2021-06-24 17:53:43 -0700 | [diff] [blame] | 87 | long long n_single_rpc; |
| 88 | long long n_single_rpc_ofl; |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 89 | long long n_single_wait; |
Paul E. McKenney | 5022b8a | 2020-06-25 17:05:58 -0700 | [diff] [blame] | 90 | long long n_single_wait_ofl; |
| 91 | long long n_many; |
| 92 | long long n_many_wait; |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 93 | long long n_all; |
| 94 | long long n_all_wait; |
| 95 | }; |
| 96 | |
| 97 | static struct scf_statistics *scf_stats_p; |
| 98 | static struct task_struct *scf_torture_stats_task; |
| 99 | static DEFINE_PER_CPU(long long, scf_invoked_count); |
| 100 | |
Paul E. McKenney | 5022b8a | 2020-06-25 17:05:58 -0700 | [diff] [blame] | 101 | // Data for random primitive selection |
Paul E. McKenney | 1ac78b4 | 2020-09-03 13:09:47 -0700 | [diff] [blame] | 102 | #define SCF_PRIM_RESCHED 0 |
| 103 | #define SCF_PRIM_SINGLE 1 |
Paul E. McKenney | 9b9a806 | 2021-06-24 17:53:43 -0700 | [diff] [blame] | 104 | #define SCF_PRIM_SINGLE_RPC 2 |
| 105 | #define SCF_PRIM_MANY 3 |
| 106 | #define SCF_PRIM_ALL 4 |
| 107 | #define SCF_NPRIMS 8 // Need wait and no-wait versions of each, |
| 108 | // except for SCF_PRIM_RESCHED and |
| 109 | // SCF_PRIM_SINGLE_RPC. |
Paul E. McKenney | 5022b8a | 2020-06-25 17:05:58 -0700 | [diff] [blame] | 110 | |
| 111 | static char *scf_prim_name[] = { |
Paul E. McKenney | 1ac78b4 | 2020-09-03 13:09:47 -0700 | [diff] [blame] | 112 | "resched_cpu", |
Paul E. McKenney | 5022b8a | 2020-06-25 17:05:58 -0700 | [diff] [blame] | 113 | "smp_call_function_single", |
Paul E. McKenney | 9b9a806 | 2021-06-24 17:53:43 -0700 | [diff] [blame] | 114 | "smp_call_function_single_rpc", |
Paul E. McKenney | 5022b8a | 2020-06-25 17:05:58 -0700 | [diff] [blame] | 115 | "smp_call_function_many", |
| 116 | "smp_call_function", |
| 117 | }; |
| 118 | |
| 119 | struct scf_selector { |
| 120 | unsigned long scfs_weight; |
| 121 | int scfs_prim; |
| 122 | bool scfs_wait; |
| 123 | }; |
| 124 | static struct scf_selector scf_sel_array[SCF_NPRIMS]; |
| 125 | static int scf_sel_array_len; |
| 126 | static unsigned long scf_sel_totweight; |
| 127 | |
Paul E. McKenney | b93e21a | 2020-06-30 20:49:50 -0700 | [diff] [blame] | 128 | // Communicate between caller and handler. |
| 129 | struct scf_check { |
| 130 | bool scfc_in; |
| 131 | bool scfc_out; |
| 132 | int scfc_cpu; // -1 for not _single(). |
| 133 | bool scfc_wait; |
Paul E. McKenney | 9b9a806 | 2021-06-24 17:53:43 -0700 | [diff] [blame] | 134 | bool scfc_rpc; |
| 135 | struct completion scfc_completion; |
Paul E. McKenney | b93e21a | 2020-06-30 20:49:50 -0700 | [diff] [blame] | 136 | }; |
| 137 | |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 138 | // Use to wait for all threads to start. |
| 139 | static atomic_t n_started; |
| 140 | static atomic_t n_errs; |
Paul E. McKenney | b93e21a | 2020-06-30 20:49:50 -0700 | [diff] [blame] | 141 | static atomic_t n_mb_in_errs; |
| 142 | static atomic_t n_mb_out_errs; |
| 143 | static atomic_t n_alloc_errs; |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 144 | static bool scfdone; |
Paul E. McKenney | dbf83b6 | 2020-07-01 16:06:22 -0700 | [diff] [blame] | 145 | static char *bangstr = ""; |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 146 | |
Wei Yongjun | 9a52a574 | 2020-07-02 09:56:50 -0700 | [diff] [blame] | 147 | static DEFINE_TORTURE_RANDOM_PERCPU(scf_torture_rand); |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 148 | |
Paul E. McKenney | 1ac78b4 | 2020-09-03 13:09:47 -0700 | [diff] [blame] | 149 | extern void resched_cpu(int cpu); // An alternative IPI vector. |
| 150 | |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 151 | // Print torture statistics. Caller must ensure serialization. |
| 152 | static void scf_torture_stats_print(void) |
| 153 | { |
| 154 | int cpu; |
Paul E. McKenney | dba3142 | 2020-06-30 16:13:37 -0700 | [diff] [blame] | 155 | int i; |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 156 | long long invoked_count = 0; |
| 157 | bool isdone = READ_ONCE(scfdone); |
Paul E. McKenney | dba3142 | 2020-06-30 16:13:37 -0700 | [diff] [blame] | 158 | struct scf_statistics scfs = {}; |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 159 | |
| 160 | for_each_possible_cpu(cpu) |
| 161 | invoked_count += data_race(per_cpu(scf_invoked_count, cpu)); |
Paul E. McKenney | dba3142 | 2020-06-30 16:13:37 -0700 | [diff] [blame] | 162 | for (i = 0; i < nthreads; i++) { |
Paul E. McKenney | 1ac78b4 | 2020-09-03 13:09:47 -0700 | [diff] [blame] | 163 | scfs.n_resched += scf_stats_p[i].n_resched; |
Paul E. McKenney | dba3142 | 2020-06-30 16:13:37 -0700 | [diff] [blame] | 164 | scfs.n_single += scf_stats_p[i].n_single; |
| 165 | scfs.n_single_ofl += scf_stats_p[i].n_single_ofl; |
Paul E. McKenney | 9b9a806 | 2021-06-24 17:53:43 -0700 | [diff] [blame] | 166 | scfs.n_single_rpc += scf_stats_p[i].n_single_rpc; |
Paul E. McKenney | dba3142 | 2020-06-30 16:13:37 -0700 | [diff] [blame] | 167 | scfs.n_single_wait += scf_stats_p[i].n_single_wait; |
| 168 | scfs.n_single_wait_ofl += scf_stats_p[i].n_single_wait_ofl; |
| 169 | scfs.n_many += scf_stats_p[i].n_many; |
| 170 | scfs.n_many_wait += scf_stats_p[i].n_many_wait; |
| 171 | scfs.n_all += scf_stats_p[i].n_all; |
| 172 | scfs.n_all_wait += scf_stats_p[i].n_all_wait; |
| 173 | } |
Paul E. McKenney | dbf83b6 | 2020-07-01 16:06:22 -0700 | [diff] [blame] | 174 | if (atomic_read(&n_errs) || atomic_read(&n_mb_in_errs) || |
| 175 | atomic_read(&n_mb_out_errs) || atomic_read(&n_alloc_errs)) |
| 176 | bangstr = "!!! "; |
Paul E. McKenney | 9b9a806 | 2021-06-24 17:53:43 -0700 | [diff] [blame] | 177 | pr_alert("%s %sscf_invoked_count %s: %lld resched: %lld single: %lld/%lld single_ofl: %lld/%lld single_rpc: %lld single_rpc_ofl: %lld many: %lld/%lld all: %lld/%lld ", |
Paul E. McKenney | 1ac78b4 | 2020-09-03 13:09:47 -0700 | [diff] [blame] | 178 | SCFTORT_FLAG, bangstr, isdone ? "VER" : "ver", invoked_count, scfs.n_resched, |
Paul E. McKenney | dba3142 | 2020-06-30 16:13:37 -0700 | [diff] [blame] | 179 | scfs.n_single, scfs.n_single_wait, scfs.n_single_ofl, scfs.n_single_wait_ofl, |
Paul E. McKenney | 9b9a806 | 2021-06-24 17:53:43 -0700 | [diff] [blame] | 180 | scfs.n_single_rpc, scfs.n_single_rpc_ofl, |
Paul E. McKenney | dba3142 | 2020-06-30 16:13:37 -0700 | [diff] [blame] | 181 | scfs.n_many, scfs.n_many_wait, scfs.n_all, scfs.n_all_wait); |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 182 | torture_onoff_stats(); |
Paul E. McKenney | dbf83b6 | 2020-07-01 16:06:22 -0700 | [diff] [blame] | 183 | pr_cont("ste: %d stnmie: %d stnmoe: %d staf: %d\n", atomic_read(&n_errs), |
| 184 | atomic_read(&n_mb_in_errs), atomic_read(&n_mb_out_errs), |
| 185 | atomic_read(&n_alloc_errs)); |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 186 | } |
| 187 | |
| 188 | // Periodically prints torture statistics, if periodic statistics printing |
| 189 | // was specified via the stat_interval module parameter. |
| 190 | static int |
| 191 | scf_torture_stats(void *arg) |
| 192 | { |
| 193 | VERBOSE_TOROUT_STRING("scf_torture_stats task started"); |
| 194 | do { |
| 195 | schedule_timeout_interruptible(stat_interval * HZ); |
| 196 | scf_torture_stats_print(); |
| 197 | torture_shutdown_absorb("scf_torture_stats"); |
| 198 | } while (!torture_must_stop()); |
| 199 | torture_kthread_stopping("scf_torture_stats"); |
| 200 | return 0; |
| 201 | } |
| 202 | |
Paul E. McKenney | 5022b8a | 2020-06-25 17:05:58 -0700 | [diff] [blame] | 203 | // Add a primitive to the scf_sel_array[]. |
| 204 | static void scf_sel_add(unsigned long weight, int prim, bool wait) |
| 205 | { |
| 206 | struct scf_selector *scfsp = &scf_sel_array[scf_sel_array_len]; |
| 207 | |
| 208 | // If no weight, if array would overflow, if computing three-place |
| 209 | // percentages would overflow, or if the scf_prim_name[] array would |
| 210 | // overflow, don't bother. In the last three two cases, complain. |
| 211 | if (!weight || |
| 212 | WARN_ON_ONCE(scf_sel_array_len >= ARRAY_SIZE(scf_sel_array)) || |
| 213 | WARN_ON_ONCE(0 - 100000 * weight <= 100000 * scf_sel_totweight) || |
| 214 | WARN_ON_ONCE(prim >= ARRAY_SIZE(scf_prim_name))) |
| 215 | return; |
| 216 | scf_sel_totweight += weight; |
| 217 | scfsp->scfs_weight = scf_sel_totweight; |
| 218 | scfsp->scfs_prim = prim; |
| 219 | scfsp->scfs_wait = wait; |
| 220 | scf_sel_array_len++; |
| 221 | } |
| 222 | |
| 223 | // Dump out weighting percentages for scf_prim_name[] array. |
| 224 | static void scf_sel_dump(void) |
| 225 | { |
| 226 | int i; |
| 227 | unsigned long oldw = 0; |
| 228 | struct scf_selector *scfsp; |
| 229 | unsigned long w; |
| 230 | |
| 231 | for (i = 0; i < scf_sel_array_len; i++) { |
| 232 | scfsp = &scf_sel_array[i]; |
| 233 | w = (scfsp->scfs_weight - oldw) * 100000 / scf_sel_totweight; |
| 234 | pr_info("%s: %3lu.%03lu %s(%s)\n", __func__, w / 1000, w % 1000, |
| 235 | scf_prim_name[scfsp->scfs_prim], |
| 236 | scfsp->scfs_wait ? "wait" : "nowait"); |
| 237 | oldw = scfsp->scfs_weight; |
| 238 | } |
| 239 | } |
| 240 | |
| 241 | // Randomly pick a primitive and wait/nowait, based on weightings. |
| 242 | static struct scf_selector *scf_sel_rand(struct torture_random_state *trsp) |
| 243 | { |
| 244 | int i; |
| 245 | unsigned long w = torture_random(trsp) % (scf_sel_totweight + 1); |
| 246 | |
| 247 | for (i = 0; i < scf_sel_array_len; i++) |
| 248 | if (scf_sel_array[i].scfs_weight >= w) |
| 249 | return &scf_sel_array[i]; |
| 250 | WARN_ON_ONCE(1); |
| 251 | return &scf_sel_array[0]; |
| 252 | } |
| 253 | |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 254 | // Update statistics and occasionally burn up mass quantities of CPU time, |
| 255 | // if told to do so via scftorture.longwait. Otherwise, occasionally burn |
| 256 | // a little bit. |
Paul E. McKenney | b93e21a | 2020-06-30 20:49:50 -0700 | [diff] [blame] | 257 | static void scf_handler(void *scfc_in) |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 258 | { |
| 259 | int i; |
| 260 | int j; |
| 261 | unsigned long r = torture_random(this_cpu_ptr(&scf_torture_rand)); |
Paul E. McKenney | b93e21a | 2020-06-30 20:49:50 -0700 | [diff] [blame] | 262 | struct scf_check *scfcp = scfc_in; |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 263 | |
Paul E. McKenney | 980205e | 2020-07-01 12:30:02 -0700 | [diff] [blame] | 264 | if (likely(scfcp)) { |
| 265 | WRITE_ONCE(scfcp->scfc_out, false); // For multiple receivers. |
| 266 | if (WARN_ON_ONCE(unlikely(!READ_ONCE(scfcp->scfc_in)))) |
| 267 | atomic_inc(&n_mb_in_errs); |
| 268 | } |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 269 | this_cpu_inc(scf_invoked_count); |
| 270 | if (longwait <= 0) { |
| 271 | if (!(r & 0xffc0)) |
| 272 | udelay(r & 0x3f); |
Paul E. McKenney | b93e21a | 2020-06-30 20:49:50 -0700 | [diff] [blame] | 273 | goto out; |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 274 | } |
| 275 | if (r & 0xfff) |
Paul E. McKenney | b93e21a | 2020-06-30 20:49:50 -0700 | [diff] [blame] | 276 | goto out; |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 277 | r = (r >> 12); |
| 278 | if (longwait <= 0) { |
| 279 | udelay((r & 0xff) + 1); |
Paul E. McKenney | b93e21a | 2020-06-30 20:49:50 -0700 | [diff] [blame] | 280 | goto out; |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 281 | } |
| 282 | r = r % longwait + 1; |
| 283 | for (i = 0; i < r; i++) { |
| 284 | for (j = 0; j < 1000; j++) { |
| 285 | udelay(1000); |
| 286 | cpu_relax(); |
| 287 | } |
| 288 | } |
Paul E. McKenney | b93e21a | 2020-06-30 20:49:50 -0700 | [diff] [blame] | 289 | out: |
| 290 | if (unlikely(!scfcp)) |
| 291 | return; |
Paul E. McKenney | 9b9a806 | 2021-06-24 17:53:43 -0700 | [diff] [blame] | 292 | if (scfcp->scfc_wait) { |
Paul E. McKenney | b93e21a | 2020-06-30 20:49:50 -0700 | [diff] [blame] | 293 | WRITE_ONCE(scfcp->scfc_out, true); |
Paul E. McKenney | 9b9a806 | 2021-06-24 17:53:43 -0700 | [diff] [blame] | 294 | if (scfcp->scfc_rpc) |
| 295 | complete(&scfcp->scfc_completion); |
| 296 | } else { |
Paul E. McKenney | b93e21a | 2020-06-30 20:49:50 -0700 | [diff] [blame] | 297 | kfree(scfcp); |
Paul E. McKenney | 9b9a806 | 2021-06-24 17:53:43 -0700 | [diff] [blame] | 298 | } |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 299 | } |
| 300 | |
Paul E. McKenney | 5022b8a | 2020-06-25 17:05:58 -0700 | [diff] [blame] | 301 | // As above, but check for correct CPU. |
Paul E. McKenney | b93e21a | 2020-06-30 20:49:50 -0700 | [diff] [blame] | 302 | static void scf_handler_1(void *scfc_in) |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 303 | { |
Paul E. McKenney | b93e21a | 2020-06-30 20:49:50 -0700 | [diff] [blame] | 304 | struct scf_check *scfcp = scfc_in; |
| 305 | |
| 306 | if (likely(scfcp) && WARN_ONCE(smp_processor_id() != scfcp->scfc_cpu, "%s: Wanted CPU %d got CPU %d\n", __func__, scfcp->scfc_cpu, smp_processor_id())) { |
Paul E. McKenney | 5022b8a | 2020-06-25 17:05:58 -0700 | [diff] [blame] | 307 | atomic_inc(&n_errs); |
Paul E. McKenney | b93e21a | 2020-06-30 20:49:50 -0700 | [diff] [blame] | 308 | } |
| 309 | scf_handler(scfcp); |
Paul E. McKenney | 5022b8a | 2020-06-25 17:05:58 -0700 | [diff] [blame] | 310 | } |
| 311 | |
| 312 | // Randomly do an smp_call_function*() invocation. |
| 313 | static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_random_state *trsp) |
| 314 | { |
| 315 | uintptr_t cpu; |
Paul E. McKenney | 676e546 | 2020-07-01 14:13:02 -0700 | [diff] [blame] | 316 | int ret = 0; |
Paul E. McKenney | b93e21a | 2020-06-30 20:49:50 -0700 | [diff] [blame] | 317 | struct scf_check *scfcp = NULL; |
Paul E. McKenney | 5022b8a | 2020-06-25 17:05:58 -0700 | [diff] [blame] | 318 | struct scf_selector *scfsp = scf_sel_rand(trsp); |
| 319 | |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 320 | if (use_cpus_read_lock) |
| 321 | cpus_read_lock(); |
| 322 | else |
| 323 | preempt_disable(); |
Paul E. McKenney | 34e8c48 | 2020-07-01 13:49:06 -0700 | [diff] [blame] | 324 | if (scfsp->scfs_prim == SCF_PRIM_SINGLE || scfsp->scfs_wait) { |
Paul E. McKenney | b93e21a | 2020-06-30 20:49:50 -0700 | [diff] [blame] | 325 | scfcp = kmalloc(sizeof(*scfcp), GFP_ATOMIC); |
Paul E. McKenney | 4df55bd | 2020-07-09 13:58:32 -0700 | [diff] [blame] | 326 | if (WARN_ON_ONCE(!scfcp)) { |
Paul E. McKenney | b93e21a | 2020-06-30 20:49:50 -0700 | [diff] [blame] | 327 | atomic_inc(&n_alloc_errs); |
Paul E. McKenney | 4df55bd | 2020-07-09 13:58:32 -0700 | [diff] [blame] | 328 | } else { |
| 329 | scfcp->scfc_cpu = -1; |
| 330 | scfcp->scfc_wait = scfsp->scfs_wait; |
| 331 | scfcp->scfc_out = false; |
Paul E. McKenney | 9b9a806 | 2021-06-24 17:53:43 -0700 | [diff] [blame] | 332 | scfcp->scfc_rpc = false; |
Paul E. McKenney | 4df55bd | 2020-07-09 13:58:32 -0700 | [diff] [blame] | 333 | } |
Paul E. McKenney | 34e8c48 | 2020-07-01 13:49:06 -0700 | [diff] [blame] | 334 | } |
| 335 | switch (scfsp->scfs_prim) { |
Paul E. McKenney | 1ac78b4 | 2020-09-03 13:09:47 -0700 | [diff] [blame] | 336 | case SCF_PRIM_RESCHED: |
| 337 | if (IS_BUILTIN(CONFIG_SCF_TORTURE_TEST)) { |
| 338 | cpu = torture_random(trsp) % nr_cpu_ids; |
| 339 | scfp->n_resched++; |
| 340 | resched_cpu(cpu); |
Paul E. McKenney | c3d0258 | 2021-07-14 06:53:51 -0700 | [diff] [blame] | 341 | this_cpu_inc(scf_invoked_count); |
Paul E. McKenney | 1ac78b4 | 2020-09-03 13:09:47 -0700 | [diff] [blame] | 342 | } |
| 343 | break; |
Paul E. McKenney | 34e8c48 | 2020-07-01 13:49:06 -0700 | [diff] [blame] | 344 | case SCF_PRIM_SINGLE: |
Paul E. McKenney | 5022b8a | 2020-06-25 17:05:58 -0700 | [diff] [blame] | 345 | cpu = torture_random(trsp) % nr_cpu_ids; |
| 346 | if (scfsp->scfs_wait) |
| 347 | scfp->n_single_wait++; |
| 348 | else |
| 349 | scfp->n_single++; |
Paul E. McKenney | b93e21a | 2020-06-30 20:49:50 -0700 | [diff] [blame] | 350 | if (scfcp) { |
| 351 | scfcp->scfc_cpu = cpu; |
Paul E. McKenney | ee7035d | 2020-07-01 16:38:16 -0700 | [diff] [blame] | 352 | barrier(); // Prevent race-reduction compiler optimizations. |
Paul E. McKenney | b93e21a | 2020-06-30 20:49:50 -0700 | [diff] [blame] | 353 | scfcp->scfc_in = true; |
| 354 | } |
| 355 | ret = smp_call_function_single(cpu, scf_handler_1, (void *)scfcp, scfsp->scfs_wait); |
Paul E. McKenney | 5022b8a | 2020-06-25 17:05:58 -0700 | [diff] [blame] | 356 | if (ret) { |
| 357 | if (scfsp->scfs_wait) |
| 358 | scfp->n_single_wait_ofl++; |
| 359 | else |
| 360 | scfp->n_single_ofl++; |
Paul E. McKenney | b93e21a | 2020-06-30 20:49:50 -0700 | [diff] [blame] | 361 | kfree(scfcp); |
Paul E. McKenney | 676e546 | 2020-07-01 14:13:02 -0700 | [diff] [blame] | 362 | scfcp = NULL; |
Paul E. McKenney | 5022b8a | 2020-06-25 17:05:58 -0700 | [diff] [blame] | 363 | } |
| 364 | break; |
Paul E. McKenney | 9b9a806 | 2021-06-24 17:53:43 -0700 | [diff] [blame] | 365 | case SCF_PRIM_SINGLE_RPC: |
| 366 | if (!scfcp) |
| 367 | break; |
| 368 | cpu = torture_random(trsp) % nr_cpu_ids; |
| 369 | scfp->n_single_rpc++; |
| 370 | scfcp->scfc_cpu = cpu; |
| 371 | scfcp->scfc_wait = true; |
| 372 | init_completion(&scfcp->scfc_completion); |
| 373 | scfcp->scfc_rpc = true; |
| 374 | barrier(); // Prevent race-reduction compiler optimizations. |
| 375 | scfcp->scfc_in = true; |
| 376 | ret = smp_call_function_single(cpu, scf_handler_1, (void *)scfcp, 0); |
| 377 | if (!ret) { |
| 378 | if (use_cpus_read_lock) |
| 379 | cpus_read_unlock(); |
| 380 | else |
| 381 | preempt_enable(); |
| 382 | wait_for_completion(&scfcp->scfc_completion); |
| 383 | if (use_cpus_read_lock) |
| 384 | cpus_read_lock(); |
| 385 | else |
| 386 | preempt_disable(); |
| 387 | } else { |
| 388 | scfp->n_single_rpc_ofl++; |
| 389 | kfree(scfcp); |
| 390 | scfcp = NULL; |
| 391 | } |
| 392 | break; |
Paul E. McKenney | 5022b8a | 2020-06-25 17:05:58 -0700 | [diff] [blame] | 393 | case SCF_PRIM_MANY: |
| 394 | if (scfsp->scfs_wait) |
| 395 | scfp->n_many_wait++; |
| 396 | else |
| 397 | scfp->n_many++; |
Paul E. McKenney | ee7035d | 2020-07-01 16:38:16 -0700 | [diff] [blame] | 398 | if (scfcp) { |
| 399 | barrier(); // Prevent race-reduction compiler optimizations. |
Paul E. McKenney | 980205e | 2020-07-01 12:30:02 -0700 | [diff] [blame] | 400 | scfcp->scfc_in = true; |
Paul E. McKenney | ee7035d | 2020-07-01 16:38:16 -0700 | [diff] [blame] | 401 | } |
Paul E. McKenney | 980205e | 2020-07-01 12:30:02 -0700 | [diff] [blame] | 402 | smp_call_function_many(cpu_online_mask, scf_handler, scfcp, scfsp->scfs_wait); |
Paul E. McKenney | 5022b8a | 2020-06-25 17:05:58 -0700 | [diff] [blame] | 403 | break; |
| 404 | case SCF_PRIM_ALL: |
| 405 | if (scfsp->scfs_wait) |
| 406 | scfp->n_all_wait++; |
| 407 | else |
| 408 | scfp->n_all++; |
Paul E. McKenney | ee7035d | 2020-07-01 16:38:16 -0700 | [diff] [blame] | 409 | if (scfcp) { |
| 410 | barrier(); // Prevent race-reduction compiler optimizations. |
Paul E. McKenney | 34e8c48 | 2020-07-01 13:49:06 -0700 | [diff] [blame] | 411 | scfcp->scfc_in = true; |
Paul E. McKenney | ee7035d | 2020-07-01 16:38:16 -0700 | [diff] [blame] | 412 | } |
Paul E. McKenney | 34e8c48 | 2020-07-01 13:49:06 -0700 | [diff] [blame] | 413 | smp_call_function(scf_handler, scfcp, scfsp->scfs_wait); |
Paul E. McKenney | 5022b8a | 2020-06-25 17:05:58 -0700 | [diff] [blame] | 414 | break; |
Paul E. McKenney | de77d4d | 2020-07-02 12:15:37 -0700 | [diff] [blame] | 415 | default: |
| 416 | WARN_ON_ONCE(1); |
| 417 | if (scfcp) |
| 418 | scfcp->scfc_out = true; |
Paul E. McKenney | 5022b8a | 2020-06-25 17:05:58 -0700 | [diff] [blame] | 419 | } |
Paul E. McKenney | 676e546 | 2020-07-01 14:13:02 -0700 | [diff] [blame] | 420 | if (scfcp && scfsp->scfs_wait) { |
Paul E. McKenney | 9e66bf0 | 2020-07-03 15:23:19 -0700 | [diff] [blame] | 421 | if (WARN_ON_ONCE((num_online_cpus() > 1 || scfsp->scfs_prim == SCF_PRIM_SINGLE) && |
Paul E. McKenney | 9b9a806 | 2021-06-24 17:53:43 -0700 | [diff] [blame] | 422 | !scfcp->scfc_out)) { |
| 423 | pr_warn("%s: Memory-ordering failure, scfs_prim: %d.\n", __func__, scfsp->scfs_prim); |
Paul E. McKenney | 676e546 | 2020-07-01 14:13:02 -0700 | [diff] [blame] | 424 | atomic_inc(&n_mb_out_errs); // Leak rather than trash! |
Paul E. McKenney | 9b9a806 | 2021-06-24 17:53:43 -0700 | [diff] [blame] | 425 | } else { |
Paul E. McKenney | 676e546 | 2020-07-01 14:13:02 -0700 | [diff] [blame] | 426 | kfree(scfcp); |
Paul E. McKenney | 9b9a806 | 2021-06-24 17:53:43 -0700 | [diff] [blame] | 427 | } |
Paul E. McKenney | ee7035d | 2020-07-01 16:38:16 -0700 | [diff] [blame] | 428 | barrier(); // Prevent race-reduction compiler optimizations. |
Paul E. McKenney | 676e546 | 2020-07-01 14:13:02 -0700 | [diff] [blame] | 429 | } |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 430 | if (use_cpus_read_lock) |
| 431 | cpus_read_unlock(); |
| 432 | else |
| 433 | preempt_enable(); |
| 434 | if (!(torture_random(trsp) & 0xfff)) |
| 435 | schedule_timeout_uninterruptible(1); |
| 436 | } |
| 437 | |
| 438 | // SCF test kthread. Repeatedly does calls to members of the |
| 439 | // smp_call_function() family of functions. |
| 440 | static int scftorture_invoker(void *arg) |
| 441 | { |
Paul E. McKenney | a7c072ef | 2020-07-02 14:15:33 -0700 | [diff] [blame] | 442 | int cpu; |
Paul E. McKenney | f3ea978 | 2020-11-11 10:12:05 -0800 | [diff] [blame] | 443 | int curcpu; |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 444 | DEFINE_TORTURE_RANDOM(rand); |
| 445 | struct scf_statistics *scfp = (struct scf_statistics *)arg; |
Paul E. McKenney | a7c072ef | 2020-07-02 14:15:33 -0700 | [diff] [blame] | 446 | bool was_offline = false; |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 447 | |
| 448 | VERBOSE_SCFTORTOUT("scftorture_invoker %d: task started", scfp->cpu); |
Paul E. McKenney | a7c072ef | 2020-07-02 14:15:33 -0700 | [diff] [blame] | 449 | cpu = scfp->cpu % nr_cpu_ids; |
Paul E. McKenney | 22b6d14 | 2021-06-04 12:37:44 -0700 | [diff] [blame] | 450 | WARN_ON_ONCE(set_cpus_allowed_ptr(current, cpumask_of(cpu))); |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 451 | set_user_nice(current, MAX_NICE); |
| 452 | if (holdoff) |
| 453 | schedule_timeout_interruptible(holdoff * HZ); |
| 454 | |
Paul E. McKenney | 22b6d14 | 2021-06-04 12:37:44 -0700 | [diff] [blame] | 455 | VERBOSE_SCFTORTOUT("scftorture_invoker %d: Waiting for all SCF torturers from cpu %d", scfp->cpu, raw_smp_processor_id()); |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 456 | |
| 457 | // Make sure that the CPU is affinitized appropriately during testing. |
Paul E. McKenney | 22b6d14 | 2021-06-04 12:37:44 -0700 | [diff] [blame] | 458 | curcpu = raw_smp_processor_id(); |
Paul E. McKenney | f3ea978 | 2020-11-11 10:12:05 -0800 | [diff] [blame] | 459 | WARN_ONCE(curcpu != scfp->cpu % nr_cpu_ids, |
| 460 | "%s: Wanted CPU %d, running on %d, nr_cpu_ids = %d\n", |
| 461 | __func__, scfp->cpu, curcpu, nr_cpu_ids); |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 462 | |
| 463 | if (!atomic_dec_return(&n_started)) |
| 464 | while (atomic_read_acquire(&n_started)) { |
| 465 | if (torture_must_stop()) { |
| 466 | VERBOSE_SCFTORTOUT("scftorture_invoker %d ended before starting", scfp->cpu); |
| 467 | goto end; |
| 468 | } |
| 469 | schedule_timeout_uninterruptible(1); |
| 470 | } |
| 471 | |
| 472 | VERBOSE_SCFTORTOUT("scftorture_invoker %d started", scfp->cpu); |
| 473 | |
| 474 | do { |
| 475 | scftorture_invoke_one(scfp, &rand); |
Paul E. McKenney | a7c072ef | 2020-07-02 14:15:33 -0700 | [diff] [blame] | 476 | while (cpu_is_offline(cpu) && !torture_must_stop()) { |
| 477 | schedule_timeout_interruptible(HZ / 5); |
| 478 | was_offline = true; |
| 479 | } |
| 480 | if (was_offline) { |
| 481 | set_cpus_allowed_ptr(current, cpumask_of(cpu)); |
| 482 | was_offline = false; |
| 483 | } |
Paul E. McKenney | 65bd77f | 2020-07-23 15:53:02 -0700 | [diff] [blame] | 484 | cond_resched(); |
Paul E. McKenney | 8555818 | 2020-09-24 12:11:57 -0700 | [diff] [blame] | 485 | stutter_wait("scftorture_invoker"); |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 486 | } while (!torture_must_stop()); |
| 487 | |
| 488 | VERBOSE_SCFTORTOUT("scftorture_invoker %d ended", scfp->cpu); |
| 489 | end: |
| 490 | torture_kthread_stopping("scftorture_invoker"); |
| 491 | return 0; |
| 492 | } |
| 493 | |
| 494 | static void |
| 495 | scftorture_print_module_parms(const char *tag) |
| 496 | { |
| 497 | pr_alert(SCFTORT_FLAG |
Paul E. McKenney | 9b9a806 | 2021-06-24 17:53:43 -0700 | [diff] [blame] | 498 | "--- %s: verbose=%d holdoff=%d longwait=%d nthreads=%d onoff_holdoff=%d onoff_interval=%d shutdown_secs=%d stat_interval=%d stutter=%d use_cpus_read_lock=%d, weight_resched=%d, weight_single=%d, weight_single_rpc=%d, weight_single_wait=%d, weight_many=%d, weight_many_wait=%d, weight_all=%d, weight_all_wait=%d\n", tag, |
| 499 | verbose, holdoff, longwait, nthreads, onoff_holdoff, onoff_interval, shutdown, stat_interval, stutter, use_cpus_read_lock, weight_resched, weight_single, weight_single_rpc, weight_single_wait, weight_many, weight_many_wait, weight_all, weight_all_wait); |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 500 | } |
| 501 | |
| 502 | static void scf_cleanup_handler(void *unused) |
| 503 | { |
| 504 | } |
| 505 | |
| 506 | static void scf_torture_cleanup(void) |
| 507 | { |
| 508 | int i; |
| 509 | |
| 510 | if (torture_cleanup_begin()) |
| 511 | return; |
| 512 | |
| 513 | WRITE_ONCE(scfdone, true); |
Paul E. McKenney | 586e4d4 | 2021-07-09 17:53:27 -0700 | [diff] [blame] | 514 | if (nthreads && scf_stats_p) |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 515 | for (i = 0; i < nthreads; i++) |
| 516 | torture_stop_kthread("scftorture_invoker", scf_stats_p[i].task); |
| 517 | else |
| 518 | goto end; |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 519 | smp_call_function(scf_cleanup_handler, NULL, 0); |
| 520 | torture_stop_kthread(scf_torture_stats, scf_torture_stats_task); |
| 521 | scf_torture_stats_print(); // -After- the stats thread is stopped! |
Paul E. McKenney | dba3142 | 2020-06-30 16:13:37 -0700 | [diff] [blame] | 522 | kfree(scf_stats_p); // -After- the last stats print has completed! |
| 523 | scf_stats_p = NULL; |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 524 | |
Paul E. McKenney | dbf83b6 | 2020-07-01 16:06:22 -0700 | [diff] [blame] | 525 | if (atomic_read(&n_errs) || atomic_read(&n_mb_in_errs) || atomic_read(&n_mb_out_errs)) |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 526 | scftorture_print_module_parms("End of test: FAILURE"); |
| 527 | else if (torture_onoff_failures()) |
| 528 | scftorture_print_module_parms("End of test: LOCK_HOTPLUG"); |
| 529 | else |
| 530 | scftorture_print_module_parms("End of test: SUCCESS"); |
| 531 | |
| 532 | end: |
| 533 | torture_cleanup_end(); |
| 534 | } |
| 535 | |
| 536 | static int __init scf_torture_init(void) |
| 537 | { |
| 538 | long i; |
| 539 | int firsterr = 0; |
Paul E. McKenney | 1ac78b4 | 2020-09-03 13:09:47 -0700 | [diff] [blame] | 540 | unsigned long weight_resched1 = weight_resched; |
Paul E. McKenney | 5022b8a | 2020-06-25 17:05:58 -0700 | [diff] [blame] | 541 | unsigned long weight_single1 = weight_single; |
Paul E. McKenney | 9b9a806 | 2021-06-24 17:53:43 -0700 | [diff] [blame] | 542 | unsigned long weight_single_rpc1 = weight_single_rpc; |
Paul E. McKenney | 5022b8a | 2020-06-25 17:05:58 -0700 | [diff] [blame] | 543 | unsigned long weight_single_wait1 = weight_single_wait; |
| 544 | unsigned long weight_many1 = weight_many; |
| 545 | unsigned long weight_many_wait1 = weight_many_wait; |
| 546 | unsigned long weight_all1 = weight_all; |
| 547 | unsigned long weight_all_wait1 = weight_all_wait; |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 548 | |
| 549 | if (!torture_init_begin(SCFTORT_STRING, verbose)) |
| 550 | return -EBUSY; |
| 551 | |
| 552 | scftorture_print_module_parms("Start of test"); |
| 553 | |
Paul E. McKenney | 2f611d0 | 2021-07-13 14:12:54 -0700 | [diff] [blame] | 554 | if (weight_resched <= 0 && |
| 555 | weight_single <= 0 && weight_single_rpc <= 0 && weight_single_wait <= 0 && |
| 556 | weight_many <= 0 && weight_many_wait <= 0 && |
| 557 | weight_all <= 0 && weight_all_wait <= 0) { |
| 558 | weight_resched1 = weight_resched == 0 ? 0 : 2 * nr_cpu_ids; |
| 559 | weight_single1 = weight_single == 0 ? 0 : 2 * nr_cpu_ids; |
| 560 | weight_single_rpc1 = weight_single_rpc == 0 ? 0 : 2 * nr_cpu_ids; |
| 561 | weight_single_wait1 = weight_single_wait == 0 ? 0 : 2 * nr_cpu_ids; |
| 562 | weight_many1 = weight_many == 0 ? 0 : 2; |
| 563 | weight_many_wait1 = weight_many_wait == 0 ? 0 : 2; |
| 564 | weight_all1 = weight_all == 0 ? 0 : 1; |
| 565 | weight_all_wait1 = weight_all_wait == 0 ? 0 : 1; |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 566 | } else { |
Paul E. McKenney | 1ac78b4 | 2020-09-03 13:09:47 -0700 | [diff] [blame] | 567 | if (weight_resched == -1) |
| 568 | weight_resched1 = 0; |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 569 | if (weight_single == -1) |
Paul E. McKenney | 5022b8a | 2020-06-25 17:05:58 -0700 | [diff] [blame] | 570 | weight_single1 = 0; |
Paul E. McKenney | 9b9a806 | 2021-06-24 17:53:43 -0700 | [diff] [blame] | 571 | if (weight_single_rpc == -1) |
| 572 | weight_single_rpc1 = 0; |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 573 | if (weight_single_wait == -1) |
Paul E. McKenney | 5022b8a | 2020-06-25 17:05:58 -0700 | [diff] [blame] | 574 | weight_single_wait1 = 0; |
| 575 | if (weight_many == -1) |
| 576 | weight_many1 = 0; |
| 577 | if (weight_many_wait == -1) |
| 578 | weight_many_wait1 = 0; |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 579 | if (weight_all == -1) |
Paul E. McKenney | 5022b8a | 2020-06-25 17:05:58 -0700 | [diff] [blame] | 580 | weight_all1 = 0; |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 581 | if (weight_all_wait == -1) |
Paul E. McKenney | 5022b8a | 2020-06-25 17:05:58 -0700 | [diff] [blame] | 582 | weight_all_wait1 = 0; |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 583 | } |
Paul E. McKenney | da9366c | 2021-07-13 15:13:56 -0700 | [diff] [blame] | 584 | if (weight_resched1 == 0 && weight_single1 == 0 && weight_single_rpc1 == 0 && |
| 585 | weight_single_wait1 == 0 && weight_many1 == 0 && weight_many_wait1 == 0 && |
Paul E. McKenney | 5022b8a | 2020-06-25 17:05:58 -0700 | [diff] [blame] | 586 | weight_all1 == 0 && weight_all_wait1 == 0) { |
| 587 | VERBOSE_SCFTORTOUT_ERRSTRING("all zero weights makes no sense"); |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 588 | firsterr = -EINVAL; |
| 589 | goto unwind; |
| 590 | } |
Paul E. McKenney | 1ac78b4 | 2020-09-03 13:09:47 -0700 | [diff] [blame] | 591 | if (IS_BUILTIN(CONFIG_SCF_TORTURE_TEST)) |
| 592 | scf_sel_add(weight_resched1, SCF_PRIM_RESCHED, false); |
| 593 | else if (weight_resched1) |
| 594 | VERBOSE_SCFTORTOUT_ERRSTRING("built as module, weight_resched ignored"); |
Paul E. McKenney | 5022b8a | 2020-06-25 17:05:58 -0700 | [diff] [blame] | 595 | scf_sel_add(weight_single1, SCF_PRIM_SINGLE, false); |
Paul E. McKenney | 9b9a806 | 2021-06-24 17:53:43 -0700 | [diff] [blame] | 596 | scf_sel_add(weight_single_rpc1, SCF_PRIM_SINGLE_RPC, true); |
Paul E. McKenney | 5022b8a | 2020-06-25 17:05:58 -0700 | [diff] [blame] | 597 | scf_sel_add(weight_single_wait1, SCF_PRIM_SINGLE, true); |
| 598 | scf_sel_add(weight_many1, SCF_PRIM_MANY, false); |
| 599 | scf_sel_add(weight_many_wait1, SCF_PRIM_MANY, true); |
| 600 | scf_sel_add(weight_all1, SCF_PRIM_ALL, false); |
| 601 | scf_sel_add(weight_all_wait1, SCF_PRIM_ALL, true); |
| 602 | scf_sel_dump(); |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 603 | |
| 604 | if (onoff_interval > 0) { |
| 605 | firsterr = torture_onoff_init(onoff_holdoff * HZ, onoff_interval, NULL); |
Paul E. McKenney | f2bdf7d | 2021-08-05 16:01:51 -0700 | [diff] [blame] | 606 | if (torture_init_error(firsterr)) |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 607 | goto unwind; |
| 608 | } |
| 609 | if (shutdown_secs > 0) { |
| 610 | firsterr = torture_shutdown_init(shutdown_secs, scf_torture_cleanup); |
Paul E. McKenney | f2bdf7d | 2021-08-05 16:01:51 -0700 | [diff] [blame] | 611 | if (torture_init_error(firsterr)) |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 612 | goto unwind; |
| 613 | } |
Paul E. McKenney | 8555818 | 2020-09-24 12:11:57 -0700 | [diff] [blame] | 614 | if (stutter > 0) { |
| 615 | firsterr = torture_stutter_init(stutter, stutter); |
Paul E. McKenney | f2bdf7d | 2021-08-05 16:01:51 -0700 | [diff] [blame] | 616 | if (torture_init_error(firsterr)) |
Paul E. McKenney | 8555818 | 2020-09-24 12:11:57 -0700 | [diff] [blame] | 617 | goto unwind; |
| 618 | } |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 619 | |
| 620 | // Worker tasks invoking smp_call_function(). |
| 621 | if (nthreads < 0) |
| 622 | nthreads = num_online_cpus(); |
| 623 | scf_stats_p = kcalloc(nthreads, sizeof(scf_stats_p[0]), GFP_KERNEL); |
| 624 | if (!scf_stats_p) { |
| 625 | VERBOSE_SCFTORTOUT_ERRSTRING("out of memory"); |
| 626 | firsterr = -ENOMEM; |
| 627 | goto unwind; |
| 628 | } |
| 629 | |
Li Zhijian | 71f6ea2a | 2021-10-29 17:40:25 +0800 | [diff] [blame] | 630 | VERBOSE_SCFTORTOUT("Starting %d smp_call_function() threads", nthreads); |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 631 | |
| 632 | atomic_set(&n_started, nthreads); |
| 633 | for (i = 0; i < nthreads; i++) { |
| 634 | scf_stats_p[i].cpu = i; |
| 635 | firsterr = torture_create_kthread(scftorture_invoker, (void *)&scf_stats_p[i], |
| 636 | scf_stats_p[i].task); |
Paul E. McKenney | f2bdf7d | 2021-08-05 16:01:51 -0700 | [diff] [blame] | 637 | if (torture_init_error(firsterr)) |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 638 | goto unwind; |
| 639 | } |
| 640 | if (stat_interval > 0) { |
| 641 | firsterr = torture_create_kthread(scf_torture_stats, NULL, scf_torture_stats_task); |
Paul E. McKenney | f2bdf7d | 2021-08-05 16:01:51 -0700 | [diff] [blame] | 642 | if (torture_init_error(firsterr)) |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 643 | goto unwind; |
| 644 | } |
| 645 | |
| 646 | torture_init_end(); |
| 647 | return 0; |
| 648 | |
| 649 | unwind: |
| 650 | torture_init_end(); |
| 651 | scf_torture_cleanup(); |
Paul E. McKenney | 2b1388f | 2021-07-13 14:20:35 -0700 | [diff] [blame] | 652 | if (shutdown_secs) { |
| 653 | WARN_ON(!IS_MODULE(CONFIG_SCF_TORTURE_TEST)); |
| 654 | kernel_power_off(); |
| 655 | } |
Paul E. McKenney | e9d338a | 2020-06-24 15:59:59 -0700 | [diff] [blame] | 656 | return firsterr; |
| 657 | } |
| 658 | |
| 659 | module_init(scf_torture_init); |
| 660 | module_exit(scf_torture_cleanup); |