Thomas Gleixner | c942fdd | 2019-05-27 08:55:06 +0200 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
Mathieu Desnoyers | 22e4ebb | 2017-07-28 16:40:40 -0400 | [diff] [blame] | 2 | /* |
| 3 | * Copyright (C) 2010-2017 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> |
| 4 | * |
| 5 | * membarrier system call |
Mathieu Desnoyers | 22e4ebb | 2017-07-28 16:40:40 -0400 | [diff] [blame] | 6 | */ |
Ingo Molnar | 325ea10 | 2018-03-03 12:20:47 +0100 | [diff] [blame] | 7 | #include "sched.h" |
Mathieu Desnoyers | 22e4ebb | 2017-07-28 16:40:40 -0400 | [diff] [blame] | 8 | |
| 9 | /* |
Mathieu Desnoyers | 25595eb | 2020-10-20 09:47:15 -0400 | [diff] [blame^] | 10 | * For documentation purposes, here are some membarrier ordering |
| 11 | * scenarios to keep in mind: |
| 12 | * |
| 13 | * A) Userspace thread execution after IPI vs membarrier's memory |
| 14 | * barrier before sending the IPI |
| 15 | * |
| 16 | * Userspace variables: |
| 17 | * |
| 18 | * int x = 0, y = 0; |
| 19 | * |
| 20 | * The memory barrier at the start of membarrier() on CPU0 is necessary in |
| 21 | * order to enforce the guarantee that any writes occurring on CPU0 before |
| 22 | * the membarrier() is executed will be visible to any code executing on |
| 23 | * CPU1 after the IPI-induced memory barrier: |
| 24 | * |
| 25 | * CPU0 CPU1 |
| 26 | * |
| 27 | * x = 1 |
| 28 | * membarrier(): |
| 29 | * a: smp_mb() |
| 30 | * b: send IPI IPI-induced mb |
| 31 | * c: smp_mb() |
| 32 | * r2 = y |
| 33 | * y = 1 |
| 34 | * barrier() |
| 35 | * r1 = x |
| 36 | * |
| 37 | * BUG_ON(r1 == 0 && r2 == 0) |
| 38 | * |
| 39 | * The write to y and load from x by CPU1 are unordered by the hardware, |
| 40 | * so it's possible to have "r1 = x" reordered before "y = 1" at any |
| 41 | * point after (b). If the memory barrier at (a) is omitted, then "x = 1" |
| 42 | * can be reordered after (a) (although not after (c)), so we get r1 == 0 |
| 43 | * and r2 == 0. This violates the guarantee that membarrier() is |
| 44 | * supposed by provide. |
| 45 | * |
| 46 | * The timing of the memory barrier at (a) has to ensure that it executes |
| 47 | * before the IPI-induced memory barrier on CPU1. |
| 48 | * |
| 49 | * B) Userspace thread execution before IPI vs membarrier's memory |
| 50 | * barrier after completing the IPI |
| 51 | * |
| 52 | * Userspace variables: |
| 53 | * |
| 54 | * int x = 0, y = 0; |
| 55 | * |
| 56 | * The memory barrier at the end of membarrier() on CPU0 is necessary in |
| 57 | * order to enforce the guarantee that any writes occurring on CPU1 before |
| 58 | * the membarrier() is executed will be visible to any code executing on |
| 59 | * CPU0 after the membarrier(): |
| 60 | * |
| 61 | * CPU0 CPU1 |
| 62 | * |
| 63 | * x = 1 |
| 64 | * barrier() |
| 65 | * y = 1 |
| 66 | * r2 = y |
| 67 | * membarrier(): |
| 68 | * a: smp_mb() |
| 69 | * b: send IPI IPI-induced mb |
| 70 | * c: smp_mb() |
| 71 | * r1 = x |
| 72 | * BUG_ON(r1 == 0 && r2 == 1) |
| 73 | * |
| 74 | * The writes to x and y are unordered by the hardware, so it's possible to |
| 75 | * have "r2 = 1" even though the write to x doesn't execute until (b). If |
| 76 | * the memory barrier at (c) is omitted then "r1 = x" can be reordered |
| 77 | * before (b) (although not before (a)), so we get "r1 = 0". This violates |
| 78 | * the guarantee that membarrier() is supposed to provide. |
| 79 | * |
| 80 | * The timing of the memory barrier at (c) has to ensure that it executes |
| 81 | * after the IPI-induced memory barrier on CPU1. |
| 82 | * |
| 83 | * C) Scheduling userspace thread -> kthread -> userspace thread vs membarrier |
| 84 | * |
| 85 | * CPU0 CPU1 |
| 86 | * |
| 87 | * membarrier(): |
| 88 | * a: smp_mb() |
| 89 | * d: switch to kthread (includes mb) |
| 90 | * b: read rq->curr->mm == NULL |
| 91 | * e: switch to user (includes mb) |
| 92 | * c: smp_mb() |
| 93 | * |
| 94 | * Using the scenario from (A), we can show that (a) needs to be paired |
| 95 | * with (e). Using the scenario from (B), we can show that (c) needs to |
| 96 | * be paired with (d). |
| 97 | * |
| 98 | * D) exit_mm vs membarrier |
| 99 | * |
| 100 | * Two thread groups are created, A and B. Thread group B is created by |
| 101 | * issuing clone from group A with flag CLONE_VM set, but not CLONE_THREAD. |
| 102 | * Let's assume we have a single thread within each thread group (Thread A |
| 103 | * and Thread B). Thread A runs on CPU0, Thread B runs on CPU1. |
| 104 | * |
| 105 | * CPU0 CPU1 |
| 106 | * |
| 107 | * membarrier(): |
| 108 | * a: smp_mb() |
| 109 | * exit_mm(): |
| 110 | * d: smp_mb() |
| 111 | * e: current->mm = NULL |
| 112 | * b: read rq->curr->mm == NULL |
| 113 | * c: smp_mb() |
| 114 | * |
| 115 | * Using scenario (B), we can show that (c) needs to be paired with (d). |
| 116 | * |
| 117 | * E) kthread_{use,unuse}_mm vs membarrier |
| 118 | * |
| 119 | * CPU0 CPU1 |
| 120 | * |
| 121 | * membarrier(): |
| 122 | * a: smp_mb() |
| 123 | * kthread_unuse_mm() |
| 124 | * d: smp_mb() |
| 125 | * e: current->mm = NULL |
| 126 | * b: read rq->curr->mm == NULL |
| 127 | * kthread_use_mm() |
| 128 | * f: current->mm = mm |
| 129 | * g: smp_mb() |
| 130 | * c: smp_mb() |
| 131 | * |
| 132 | * Using the scenario from (A), we can show that (a) needs to be paired |
| 133 | * with (g). Using the scenario from (B), we can show that (c) needs to |
| 134 | * be paired with (d). |
| 135 | */ |
| 136 | |
| 137 | /* |
Mathieu Desnoyers | 22e4ebb | 2017-07-28 16:40:40 -0400 | [diff] [blame] | 138 | * Bitmask made from a "or" of all commands within enum membarrier_cmd, |
| 139 | * except MEMBARRIER_CMD_QUERY. |
| 140 | */ |
Mathieu Desnoyers | 70216e1 | 2018-01-29 15:20:17 -0500 | [diff] [blame] | 141 | #ifdef CONFIG_ARCH_HAS_MEMBARRIER_SYNC_CORE |
Ingo Molnar | 97fb7a0 | 2018-03-03 14:01:12 +0100 | [diff] [blame] | 142 | #define MEMBARRIER_PRIVATE_EXPEDITED_SYNC_CORE_BITMASK \ |
| 143 | (MEMBARRIER_CMD_PRIVATE_EXPEDITED_SYNC_CORE \ |
Mathieu Desnoyers | 70216e1 | 2018-01-29 15:20:17 -0500 | [diff] [blame] | 144 | | MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_SYNC_CORE) |
| 145 | #else |
| 146 | #define MEMBARRIER_PRIVATE_EXPEDITED_SYNC_CORE_BITMASK 0 |
| 147 | #endif |
| 148 | |
Peter Oskolkov | 2a36ab717 | 2020-09-23 16:36:16 -0700 | [diff] [blame] | 149 | #ifdef CONFIG_RSEQ |
| 150 | #define MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ_BITMASK \ |
| 151 | (MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ \ |
| 152 | | MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_RSEQ_BITMASK) |
| 153 | #else |
| 154 | #define MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ_BITMASK 0 |
| 155 | #endif |
| 156 | |
Ingo Molnar | 97fb7a0 | 2018-03-03 14:01:12 +0100 | [diff] [blame] | 157 | #define MEMBARRIER_CMD_BITMASK \ |
| 158 | (MEMBARRIER_CMD_GLOBAL | MEMBARRIER_CMD_GLOBAL_EXPEDITED \ |
| 159 | | MEMBARRIER_CMD_REGISTER_GLOBAL_EXPEDITED \ |
| 160 | | MEMBARRIER_CMD_PRIVATE_EXPEDITED \ |
| 161 | | MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED \ |
Mathieu Desnoyers | 70216e1 | 2018-01-29 15:20:17 -0500 | [diff] [blame] | 162 | | MEMBARRIER_PRIVATE_EXPEDITED_SYNC_CORE_BITMASK) |
Mathieu Desnoyers | 22e4ebb | 2017-07-28 16:40:40 -0400 | [diff] [blame] | 163 | |
| 164 | static void ipi_mb(void *info) |
| 165 | { |
| 166 | smp_mb(); /* IPIs should be serializing but paranoid. */ |
| 167 | } |
| 168 | |
Peter Oskolkov | 2a36ab717 | 2020-09-23 16:36:16 -0700 | [diff] [blame] | 169 | static void ipi_rseq(void *info) |
| 170 | { |
| 171 | rseq_preempt(current); |
| 172 | } |
| 173 | |
Mathieu Desnoyers | 227a4aa | 2019-09-19 13:37:02 -0400 | [diff] [blame] | 174 | static void ipi_sync_rq_state(void *info) |
| 175 | { |
| 176 | struct mm_struct *mm = (struct mm_struct *) info; |
| 177 | |
| 178 | if (current->mm != mm) |
| 179 | return; |
| 180 | this_cpu_write(runqueues.membarrier_state, |
| 181 | atomic_read(&mm->membarrier_state)); |
| 182 | /* |
| 183 | * Issue a memory barrier after setting |
| 184 | * MEMBARRIER_STATE_GLOBAL_EXPEDITED in the current runqueue to |
| 185 | * guarantee that no memory access following registration is reordered |
| 186 | * before registration. |
| 187 | */ |
| 188 | smp_mb(); |
| 189 | } |
| 190 | |
| 191 | void membarrier_exec_mmap(struct mm_struct *mm) |
| 192 | { |
| 193 | /* |
| 194 | * Issue a memory barrier before clearing membarrier_state to |
| 195 | * guarantee that no memory access prior to exec is reordered after |
| 196 | * clearing this state. |
| 197 | */ |
| 198 | smp_mb(); |
| 199 | atomic_set(&mm->membarrier_state, 0); |
| 200 | /* |
| 201 | * Keep the runqueue membarrier_state in sync with this mm |
| 202 | * membarrier_state. |
| 203 | */ |
| 204 | this_cpu_write(runqueues.membarrier_state, 0); |
| 205 | } |
| 206 | |
Mathieu Desnoyers | 5bc7850 | 2020-10-20 09:47:13 -0400 | [diff] [blame] | 207 | void membarrier_update_current_mm(struct mm_struct *next_mm) |
| 208 | { |
| 209 | struct rq *rq = this_rq(); |
| 210 | int membarrier_state = 0; |
| 211 | |
| 212 | if (next_mm) |
| 213 | membarrier_state = atomic_read(&next_mm->membarrier_state); |
| 214 | if (READ_ONCE(rq->membarrier_state) == membarrier_state) |
| 215 | return; |
| 216 | WRITE_ONCE(rq->membarrier_state, membarrier_state); |
| 217 | } |
| 218 | |
Mathieu Desnoyers | c5f58bd | 2018-01-29 15:20:13 -0500 | [diff] [blame] | 219 | static int membarrier_global_expedited(void) |
| 220 | { |
| 221 | int cpu; |
Mathieu Desnoyers | c5f58bd | 2018-01-29 15:20:13 -0500 | [diff] [blame] | 222 | cpumask_var_t tmpmask; |
| 223 | |
| 224 | if (num_online_cpus() == 1) |
| 225 | return 0; |
| 226 | |
| 227 | /* |
| 228 | * Matches memory barriers around rq->curr modification in |
| 229 | * scheduler. |
| 230 | */ |
| 231 | smp_mb(); /* system call entry is not a mb. */ |
| 232 | |
Mathieu Desnoyers | c172e0a | 2019-09-19 13:37:05 -0400 | [diff] [blame] | 233 | if (!zalloc_cpumask_var(&tmpmask, GFP_KERNEL)) |
| 234 | return -ENOMEM; |
Mathieu Desnoyers | c5f58bd | 2018-01-29 15:20:13 -0500 | [diff] [blame] | 235 | |
| 236 | cpus_read_lock(); |
Mathieu Desnoyers | 227a4aa | 2019-09-19 13:37:02 -0400 | [diff] [blame] | 237 | rcu_read_lock(); |
Mathieu Desnoyers | c5f58bd | 2018-01-29 15:20:13 -0500 | [diff] [blame] | 238 | for_each_online_cpu(cpu) { |
| 239 | struct task_struct *p; |
| 240 | |
| 241 | /* |
| 242 | * Skipping the current CPU is OK even through we can be |
| 243 | * migrated at any point. The current CPU, at the point |
| 244 | * where we read raw_smp_processor_id(), is ensured to |
| 245 | * be in program order with respect to the caller |
| 246 | * thread. Therefore, we can skip this CPU from the |
| 247 | * iteration. |
| 248 | */ |
| 249 | if (cpu == raw_smp_processor_id()) |
| 250 | continue; |
Ingo Molnar | 97fb7a0 | 2018-03-03 14:01:12 +0100 | [diff] [blame] | 251 | |
Mathieu Desnoyers | 227a4aa | 2019-09-19 13:37:02 -0400 | [diff] [blame] | 252 | if (!(READ_ONCE(cpu_rq(cpu)->membarrier_state) & |
| 253 | MEMBARRIER_STATE_GLOBAL_EXPEDITED)) |
| 254 | continue; |
| 255 | |
| 256 | /* |
Mathieu Desnoyers | 618758e | 2020-10-20 09:47:14 -0400 | [diff] [blame] | 257 | * Skip the CPU if it runs a kernel thread which is not using |
| 258 | * a task mm. |
Mathieu Desnoyers | 227a4aa | 2019-09-19 13:37:02 -0400 | [diff] [blame] | 259 | */ |
Eric W. Biederman | 154abaf | 2019-09-14 07:34:30 -0500 | [diff] [blame] | 260 | p = rcu_dereference(cpu_rq(cpu)->curr); |
Mathieu Desnoyers | 618758e | 2020-10-20 09:47:14 -0400 | [diff] [blame] | 261 | if (!p->mm) |
Mathieu Desnoyers | 227a4aa | 2019-09-19 13:37:02 -0400 | [diff] [blame] | 262 | continue; |
| 263 | |
Mathieu Desnoyers | c172e0a | 2019-09-19 13:37:05 -0400 | [diff] [blame] | 264 | __cpumask_set_cpu(cpu, tmpmask); |
Mathieu Desnoyers | c5f58bd | 2018-01-29 15:20:13 -0500 | [diff] [blame] | 265 | } |
Mathieu Desnoyers | 227a4aa | 2019-09-19 13:37:02 -0400 | [diff] [blame] | 266 | rcu_read_unlock(); |
Mathieu Desnoyers | c172e0a | 2019-09-19 13:37:05 -0400 | [diff] [blame] | 267 | |
| 268 | preempt_disable(); |
| 269 | smp_call_function_many(tmpmask, ipi_mb, NULL, 1); |
| 270 | preempt_enable(); |
| 271 | |
| 272 | free_cpumask_var(tmpmask); |
Mathieu Desnoyers | c5f58bd | 2018-01-29 15:20:13 -0500 | [diff] [blame] | 273 | cpus_read_unlock(); |
| 274 | |
| 275 | /* |
| 276 | * Memory barrier on the caller thread _after_ we finished |
| 277 | * waiting for the last IPI. Matches memory barriers around |
| 278 | * rq->curr modification in scheduler. |
| 279 | */ |
| 280 | smp_mb(); /* exit from system call is not a mb */ |
| 281 | return 0; |
| 282 | } |
| 283 | |
Peter Oskolkov | 2a36ab717 | 2020-09-23 16:36:16 -0700 | [diff] [blame] | 284 | static int membarrier_private_expedited(int flags, int cpu_id) |
Mathieu Desnoyers | 22e4ebb | 2017-07-28 16:40:40 -0400 | [diff] [blame] | 285 | { |
Mathieu Desnoyers | 22e4ebb | 2017-07-28 16:40:40 -0400 | [diff] [blame] | 286 | cpumask_var_t tmpmask; |
Mathieu Desnoyers | c6d68c1 | 2019-09-19 13:37:04 -0400 | [diff] [blame] | 287 | struct mm_struct *mm = current->mm; |
Peter Oskolkov | 2a36ab717 | 2020-09-23 16:36:16 -0700 | [diff] [blame] | 288 | smp_call_func_t ipi_func = ipi_mb; |
Mathieu Desnoyers | 22e4ebb | 2017-07-28 16:40:40 -0400 | [diff] [blame] | 289 | |
Peter Oskolkov | 2a36ab717 | 2020-09-23 16:36:16 -0700 | [diff] [blame] | 290 | if (flags == MEMBARRIER_FLAG_SYNC_CORE) { |
Mathieu Desnoyers | 70216e1 | 2018-01-29 15:20:17 -0500 | [diff] [blame] | 291 | if (!IS_ENABLED(CONFIG_ARCH_HAS_MEMBARRIER_SYNC_CORE)) |
| 292 | return -EINVAL; |
Mathieu Desnoyers | c6d68c1 | 2019-09-19 13:37:04 -0400 | [diff] [blame] | 293 | if (!(atomic_read(&mm->membarrier_state) & |
Mathieu Desnoyers | 70216e1 | 2018-01-29 15:20:17 -0500 | [diff] [blame] | 294 | MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE_READY)) |
| 295 | return -EPERM; |
Peter Oskolkov | 2a36ab717 | 2020-09-23 16:36:16 -0700 | [diff] [blame] | 296 | } else if (flags == MEMBARRIER_FLAG_RSEQ) { |
| 297 | if (!IS_ENABLED(CONFIG_RSEQ)) |
| 298 | return -EINVAL; |
| 299 | if (!(atomic_read(&mm->membarrier_state) & |
| 300 | MEMBARRIER_STATE_PRIVATE_EXPEDITED_RSEQ_READY)) |
| 301 | return -EPERM; |
| 302 | ipi_func = ipi_rseq; |
Mathieu Desnoyers | 70216e1 | 2018-01-29 15:20:17 -0500 | [diff] [blame] | 303 | } else { |
Peter Oskolkov | 2a36ab717 | 2020-09-23 16:36:16 -0700 | [diff] [blame] | 304 | WARN_ON_ONCE(flags); |
Mathieu Desnoyers | c6d68c1 | 2019-09-19 13:37:04 -0400 | [diff] [blame] | 305 | if (!(atomic_read(&mm->membarrier_state) & |
Mathieu Desnoyers | 70216e1 | 2018-01-29 15:20:17 -0500 | [diff] [blame] | 306 | MEMBARRIER_STATE_PRIVATE_EXPEDITED_READY)) |
| 307 | return -EPERM; |
| 308 | } |
Mathieu Desnoyers | a961e40 | 2017-10-19 13:30:15 -0400 | [diff] [blame] | 309 | |
Mathieu Desnoyers | c6d68c1 | 2019-09-19 13:37:04 -0400 | [diff] [blame] | 310 | if (atomic_read(&mm->mm_users) == 1 || num_online_cpus() == 1) |
Mathieu Desnoyers | a961e40 | 2017-10-19 13:30:15 -0400 | [diff] [blame] | 311 | return 0; |
Mathieu Desnoyers | 22e4ebb | 2017-07-28 16:40:40 -0400 | [diff] [blame] | 312 | |
| 313 | /* |
| 314 | * Matches memory barriers around rq->curr modification in |
| 315 | * scheduler. |
| 316 | */ |
| 317 | smp_mb(); /* system call entry is not a mb. */ |
| 318 | |
Peter Oskolkov | 2a36ab717 | 2020-09-23 16:36:16 -0700 | [diff] [blame] | 319 | if (cpu_id < 0 && !zalloc_cpumask_var(&tmpmask, GFP_KERNEL)) |
Mathieu Desnoyers | c172e0a | 2019-09-19 13:37:05 -0400 | [diff] [blame] | 320 | return -ENOMEM; |
Mathieu Desnoyers | 22e4ebb | 2017-07-28 16:40:40 -0400 | [diff] [blame] | 321 | |
| 322 | cpus_read_lock(); |
Peter Oskolkov | 2a36ab717 | 2020-09-23 16:36:16 -0700 | [diff] [blame] | 323 | |
| 324 | if (cpu_id >= 0) { |
Mathieu Desnoyers | 22e4ebb | 2017-07-28 16:40:40 -0400 | [diff] [blame] | 325 | struct task_struct *p; |
| 326 | |
Peter Oskolkov | 2a36ab717 | 2020-09-23 16:36:16 -0700 | [diff] [blame] | 327 | if (cpu_id >= nr_cpu_ids || !cpu_online(cpu_id)) |
| 328 | goto out; |
| 329 | if (cpu_id == raw_smp_processor_id()) |
| 330 | goto out; |
| 331 | rcu_read_lock(); |
| 332 | p = rcu_dereference(cpu_rq(cpu_id)->curr); |
| 333 | if (!p || p->mm != mm) { |
| 334 | rcu_read_unlock(); |
| 335 | goto out; |
| 336 | } |
| 337 | rcu_read_unlock(); |
| 338 | } else { |
| 339 | int cpu; |
| 340 | |
| 341 | rcu_read_lock(); |
| 342 | for_each_online_cpu(cpu) { |
| 343 | struct task_struct *p; |
| 344 | |
| 345 | /* |
| 346 | * Skipping the current CPU is OK even through we can be |
| 347 | * migrated at any point. The current CPU, at the point |
| 348 | * where we read raw_smp_processor_id(), is ensured to |
| 349 | * be in program order with respect to the caller |
| 350 | * thread. Therefore, we can skip this CPU from the |
| 351 | * iteration. |
| 352 | */ |
| 353 | if (cpu == raw_smp_processor_id()) |
| 354 | continue; |
| 355 | p = rcu_dereference(cpu_rq(cpu)->curr); |
| 356 | if (p && p->mm == mm) |
| 357 | __cpumask_set_cpu(cpu, tmpmask); |
| 358 | } |
| 359 | rcu_read_unlock(); |
Mathieu Desnoyers | 22e4ebb | 2017-07-28 16:40:40 -0400 | [diff] [blame] | 360 | } |
Mathieu Desnoyers | c172e0a | 2019-09-19 13:37:05 -0400 | [diff] [blame] | 361 | |
| 362 | preempt_disable(); |
Peter Oskolkov | 2a36ab717 | 2020-09-23 16:36:16 -0700 | [diff] [blame] | 363 | if (cpu_id >= 0) |
| 364 | smp_call_function_single(cpu_id, ipi_func, NULL, 1); |
| 365 | else |
| 366 | smp_call_function_many(tmpmask, ipi_func, NULL, 1); |
Mathieu Desnoyers | c172e0a | 2019-09-19 13:37:05 -0400 | [diff] [blame] | 367 | preempt_enable(); |
| 368 | |
Peter Oskolkov | 2a36ab717 | 2020-09-23 16:36:16 -0700 | [diff] [blame] | 369 | out: |
| 370 | if (cpu_id < 0) |
| 371 | free_cpumask_var(tmpmask); |
Mathieu Desnoyers | 22e4ebb | 2017-07-28 16:40:40 -0400 | [diff] [blame] | 372 | cpus_read_unlock(); |
| 373 | |
| 374 | /* |
| 375 | * Memory barrier on the caller thread _after_ we finished |
| 376 | * waiting for the last IPI. Matches memory barriers around |
| 377 | * rq->curr modification in scheduler. |
| 378 | */ |
| 379 | smp_mb(); /* exit from system call is not a mb */ |
Ingo Molnar | 97fb7a0 | 2018-03-03 14:01:12 +0100 | [diff] [blame] | 380 | |
Mathieu Desnoyers | a961e40 | 2017-10-19 13:30:15 -0400 | [diff] [blame] | 381 | return 0; |
| 382 | } |
| 383 | |
Mathieu Desnoyers | 227a4aa | 2019-09-19 13:37:02 -0400 | [diff] [blame] | 384 | static int sync_runqueues_membarrier_state(struct mm_struct *mm) |
| 385 | { |
| 386 | int membarrier_state = atomic_read(&mm->membarrier_state); |
| 387 | cpumask_var_t tmpmask; |
| 388 | int cpu; |
| 389 | |
| 390 | if (atomic_read(&mm->mm_users) == 1 || num_online_cpus() == 1) { |
| 391 | this_cpu_write(runqueues.membarrier_state, membarrier_state); |
| 392 | |
| 393 | /* |
| 394 | * For single mm user, we can simply issue a memory barrier |
| 395 | * after setting MEMBARRIER_STATE_GLOBAL_EXPEDITED in the |
| 396 | * mm and in the current runqueue to guarantee that no memory |
| 397 | * access following registration is reordered before |
| 398 | * registration. |
| 399 | */ |
| 400 | smp_mb(); |
| 401 | return 0; |
| 402 | } |
| 403 | |
| 404 | if (!zalloc_cpumask_var(&tmpmask, GFP_KERNEL)) |
| 405 | return -ENOMEM; |
| 406 | |
| 407 | /* |
| 408 | * For mm with multiple users, we need to ensure all future |
| 409 | * scheduler executions will observe @mm's new membarrier |
| 410 | * state. |
| 411 | */ |
| 412 | synchronize_rcu(); |
| 413 | |
| 414 | /* |
| 415 | * For each cpu runqueue, if the task's mm match @mm, ensure that all |
| 416 | * @mm's membarrier state set bits are also set in in the runqueue's |
| 417 | * membarrier state. This ensures that a runqueue scheduling |
| 418 | * between threads which are users of @mm has its membarrier state |
| 419 | * updated. |
| 420 | */ |
| 421 | cpus_read_lock(); |
| 422 | rcu_read_lock(); |
| 423 | for_each_online_cpu(cpu) { |
| 424 | struct rq *rq = cpu_rq(cpu); |
| 425 | struct task_struct *p; |
| 426 | |
Mathieu Desnoyers | c172e0a | 2019-09-19 13:37:05 -0400 | [diff] [blame] | 427 | p = rcu_dereference(rq->curr); |
Mathieu Desnoyers | 227a4aa | 2019-09-19 13:37:02 -0400 | [diff] [blame] | 428 | if (p && p->mm == mm) |
| 429 | __cpumask_set_cpu(cpu, tmpmask); |
| 430 | } |
| 431 | rcu_read_unlock(); |
| 432 | |
| 433 | preempt_disable(); |
| 434 | smp_call_function_many(tmpmask, ipi_sync_rq_state, mm, 1); |
| 435 | preempt_enable(); |
| 436 | |
| 437 | free_cpumask_var(tmpmask); |
| 438 | cpus_read_unlock(); |
| 439 | |
| 440 | return 0; |
| 441 | } |
| 442 | |
Mathieu Desnoyers | c5f58bd | 2018-01-29 15:20:13 -0500 | [diff] [blame] | 443 | static int membarrier_register_global_expedited(void) |
| 444 | { |
| 445 | struct task_struct *p = current; |
| 446 | struct mm_struct *mm = p->mm; |
Mathieu Desnoyers | 227a4aa | 2019-09-19 13:37:02 -0400 | [diff] [blame] | 447 | int ret; |
Mathieu Desnoyers | c5f58bd | 2018-01-29 15:20:13 -0500 | [diff] [blame] | 448 | |
| 449 | if (atomic_read(&mm->membarrier_state) & |
| 450 | MEMBARRIER_STATE_GLOBAL_EXPEDITED_READY) |
| 451 | return 0; |
| 452 | atomic_or(MEMBARRIER_STATE_GLOBAL_EXPEDITED, &mm->membarrier_state); |
Mathieu Desnoyers | 227a4aa | 2019-09-19 13:37:02 -0400 | [diff] [blame] | 453 | ret = sync_runqueues_membarrier_state(mm); |
| 454 | if (ret) |
| 455 | return ret; |
Mathieu Desnoyers | c5f58bd | 2018-01-29 15:20:13 -0500 | [diff] [blame] | 456 | atomic_or(MEMBARRIER_STATE_GLOBAL_EXPEDITED_READY, |
| 457 | &mm->membarrier_state); |
Ingo Molnar | 97fb7a0 | 2018-03-03 14:01:12 +0100 | [diff] [blame] | 458 | |
Mathieu Desnoyers | c5f58bd | 2018-01-29 15:20:13 -0500 | [diff] [blame] | 459 | return 0; |
| 460 | } |
| 461 | |
Mathieu Desnoyers | 70216e1 | 2018-01-29 15:20:17 -0500 | [diff] [blame] | 462 | static int membarrier_register_private_expedited(int flags) |
Mathieu Desnoyers | a961e40 | 2017-10-19 13:30:15 -0400 | [diff] [blame] | 463 | { |
| 464 | struct task_struct *p = current; |
| 465 | struct mm_struct *mm = p->mm; |
Mathieu Desnoyers | 227a4aa | 2019-09-19 13:37:02 -0400 | [diff] [blame] | 466 | int ready_state = MEMBARRIER_STATE_PRIVATE_EXPEDITED_READY, |
| 467 | set_state = MEMBARRIER_STATE_PRIVATE_EXPEDITED, |
| 468 | ret; |
Mathieu Desnoyers | 70216e1 | 2018-01-29 15:20:17 -0500 | [diff] [blame] | 469 | |
Peter Oskolkov | 2a36ab717 | 2020-09-23 16:36:16 -0700 | [diff] [blame] | 470 | if (flags == MEMBARRIER_FLAG_SYNC_CORE) { |
Mathieu Desnoyers | 70216e1 | 2018-01-29 15:20:17 -0500 | [diff] [blame] | 471 | if (!IS_ENABLED(CONFIG_ARCH_HAS_MEMBARRIER_SYNC_CORE)) |
| 472 | return -EINVAL; |
Mathieu Desnoyers | 227a4aa | 2019-09-19 13:37:02 -0400 | [diff] [blame] | 473 | ready_state = |
| 474 | MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE_READY; |
Peter Oskolkov | 2a36ab717 | 2020-09-23 16:36:16 -0700 | [diff] [blame] | 475 | } else if (flags == MEMBARRIER_FLAG_RSEQ) { |
| 476 | if (!IS_ENABLED(CONFIG_RSEQ)) |
| 477 | return -EINVAL; |
| 478 | ready_state = |
| 479 | MEMBARRIER_STATE_PRIVATE_EXPEDITED_RSEQ_READY; |
| 480 | } else { |
| 481 | WARN_ON_ONCE(flags); |
Mathieu Desnoyers | 70216e1 | 2018-01-29 15:20:17 -0500 | [diff] [blame] | 482 | } |
Mathieu Desnoyers | a961e40 | 2017-10-19 13:30:15 -0400 | [diff] [blame] | 483 | |
| 484 | /* |
| 485 | * We need to consider threads belonging to different thread |
| 486 | * groups, which use the same mm. (CLONE_VM but not |
| 487 | * CLONE_THREAD). |
| 488 | */ |
Mathieu Desnoyers | 227a4aa | 2019-09-19 13:37:02 -0400 | [diff] [blame] | 489 | if ((atomic_read(&mm->membarrier_state) & ready_state) == ready_state) |
Mathieu Desnoyers | c5f58bd | 2018-01-29 15:20:13 -0500 | [diff] [blame] | 490 | return 0; |
Mathieu Desnoyers | 70216e1 | 2018-01-29 15:20:17 -0500 | [diff] [blame] | 491 | if (flags & MEMBARRIER_FLAG_SYNC_CORE) |
Mathieu Desnoyers | 227a4aa | 2019-09-19 13:37:02 -0400 | [diff] [blame] | 492 | set_state |= MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE; |
Peter Oskolkov | 2a36ab717 | 2020-09-23 16:36:16 -0700 | [diff] [blame] | 493 | if (flags & MEMBARRIER_FLAG_RSEQ) |
| 494 | set_state |= MEMBARRIER_STATE_PRIVATE_EXPEDITED_RSEQ; |
Mathieu Desnoyers | 227a4aa | 2019-09-19 13:37:02 -0400 | [diff] [blame] | 495 | atomic_or(set_state, &mm->membarrier_state); |
| 496 | ret = sync_runqueues_membarrier_state(mm); |
| 497 | if (ret) |
| 498 | return ret; |
| 499 | atomic_or(ready_state, &mm->membarrier_state); |
Ingo Molnar | 97fb7a0 | 2018-03-03 14:01:12 +0100 | [diff] [blame] | 500 | |
Mathieu Desnoyers | c5f58bd | 2018-01-29 15:20:13 -0500 | [diff] [blame] | 501 | return 0; |
Mathieu Desnoyers | 22e4ebb | 2017-07-28 16:40:40 -0400 | [diff] [blame] | 502 | } |
| 503 | |
| 504 | /** |
| 505 | * sys_membarrier - issue memory barriers on a set of threads |
Peter Oskolkov | 2a36ab717 | 2020-09-23 16:36:16 -0700 | [diff] [blame] | 506 | * @cmd: Takes command values defined in enum membarrier_cmd. |
| 507 | * @flags: Currently needs to be 0 for all commands other than |
| 508 | * MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ: in the latter |
| 509 | * case it can be MEMBARRIER_CMD_FLAG_CPU, indicating that @cpu_id |
| 510 | * contains the CPU on which to interrupt (= restart) |
| 511 | * the RSEQ critical section. |
| 512 | * @cpu_id: if @flags == MEMBARRIER_CMD_FLAG_CPU, indicates the cpu on which |
| 513 | * RSEQ CS should be interrupted (@cmd must be |
| 514 | * MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ). |
Mathieu Desnoyers | 22e4ebb | 2017-07-28 16:40:40 -0400 | [diff] [blame] | 515 | * |
| 516 | * If this system call is not implemented, -ENOSYS is returned. If the |
| 517 | * command specified does not exist, not available on the running |
| 518 | * kernel, or if the command argument is invalid, this system call |
| 519 | * returns -EINVAL. For a given command, with flags argument set to 0, |
Mathieu Desnoyers | 227a4aa | 2019-09-19 13:37:02 -0400 | [diff] [blame] | 520 | * if this system call returns -ENOSYS or -EINVAL, it is guaranteed to |
| 521 | * always return the same value until reboot. In addition, it can return |
| 522 | * -ENOMEM if there is not enough memory available to perform the system |
| 523 | * call. |
Mathieu Desnoyers | 22e4ebb | 2017-07-28 16:40:40 -0400 | [diff] [blame] | 524 | * |
| 525 | * All memory accesses performed in program order from each targeted thread |
| 526 | * is guaranteed to be ordered with respect to sys_membarrier(). If we use |
| 527 | * the semantic "barrier()" to represent a compiler barrier forcing memory |
| 528 | * accesses to be performed in program order across the barrier, and |
| 529 | * smp_mb() to represent explicit memory barriers forcing full memory |
| 530 | * ordering across the barrier, we have the following ordering table for |
| 531 | * each pair of barrier(), sys_membarrier() and smp_mb(): |
| 532 | * |
| 533 | * The pair ordering is detailed as (O: ordered, X: not ordered): |
| 534 | * |
| 535 | * barrier() smp_mb() sys_membarrier() |
| 536 | * barrier() X X O |
| 537 | * smp_mb() X O O |
| 538 | * sys_membarrier() O O O |
| 539 | */ |
Peter Oskolkov | 2a36ab717 | 2020-09-23 16:36:16 -0700 | [diff] [blame] | 540 | SYSCALL_DEFINE3(membarrier, int, cmd, unsigned int, flags, int, cpu_id) |
Mathieu Desnoyers | 22e4ebb | 2017-07-28 16:40:40 -0400 | [diff] [blame] | 541 | { |
Peter Oskolkov | 2a36ab717 | 2020-09-23 16:36:16 -0700 | [diff] [blame] | 542 | switch (cmd) { |
| 543 | case MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ: |
| 544 | if (unlikely(flags && flags != MEMBARRIER_CMD_FLAG_CPU)) |
| 545 | return -EINVAL; |
| 546 | break; |
| 547 | default: |
| 548 | if (unlikely(flags)) |
| 549 | return -EINVAL; |
| 550 | } |
| 551 | |
| 552 | if (!(flags & MEMBARRIER_CMD_FLAG_CPU)) |
| 553 | cpu_id = -1; |
| 554 | |
Mathieu Desnoyers | 22e4ebb | 2017-07-28 16:40:40 -0400 | [diff] [blame] | 555 | switch (cmd) { |
| 556 | case MEMBARRIER_CMD_QUERY: |
| 557 | { |
| 558 | int cmd_mask = MEMBARRIER_CMD_BITMASK; |
| 559 | |
| 560 | if (tick_nohz_full_enabled()) |
Mathieu Desnoyers | c5f58bd | 2018-01-29 15:20:13 -0500 | [diff] [blame] | 561 | cmd_mask &= ~MEMBARRIER_CMD_GLOBAL; |
Mathieu Desnoyers | 22e4ebb | 2017-07-28 16:40:40 -0400 | [diff] [blame] | 562 | return cmd_mask; |
| 563 | } |
Mathieu Desnoyers | c5f58bd | 2018-01-29 15:20:13 -0500 | [diff] [blame] | 564 | case MEMBARRIER_CMD_GLOBAL: |
| 565 | /* MEMBARRIER_CMD_GLOBAL is not compatible with nohz_full. */ |
Mathieu Desnoyers | 22e4ebb | 2017-07-28 16:40:40 -0400 | [diff] [blame] | 566 | if (tick_nohz_full_enabled()) |
| 567 | return -EINVAL; |
| 568 | if (num_online_cpus() > 1) |
Paul E. McKenney | 78d125d | 2018-07-11 15:36:43 -0700 | [diff] [blame] | 569 | synchronize_rcu(); |
Mathieu Desnoyers | 22e4ebb | 2017-07-28 16:40:40 -0400 | [diff] [blame] | 570 | return 0; |
Mathieu Desnoyers | c5f58bd | 2018-01-29 15:20:13 -0500 | [diff] [blame] | 571 | case MEMBARRIER_CMD_GLOBAL_EXPEDITED: |
| 572 | return membarrier_global_expedited(); |
| 573 | case MEMBARRIER_CMD_REGISTER_GLOBAL_EXPEDITED: |
| 574 | return membarrier_register_global_expedited(); |
Mathieu Desnoyers | 22e4ebb | 2017-07-28 16:40:40 -0400 | [diff] [blame] | 575 | case MEMBARRIER_CMD_PRIVATE_EXPEDITED: |
Peter Oskolkov | 2a36ab717 | 2020-09-23 16:36:16 -0700 | [diff] [blame] | 576 | return membarrier_private_expedited(0, cpu_id); |
Mathieu Desnoyers | a961e40 | 2017-10-19 13:30:15 -0400 | [diff] [blame] | 577 | case MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED: |
Mathieu Desnoyers | 70216e1 | 2018-01-29 15:20:17 -0500 | [diff] [blame] | 578 | return membarrier_register_private_expedited(0); |
| 579 | case MEMBARRIER_CMD_PRIVATE_EXPEDITED_SYNC_CORE: |
Peter Oskolkov | 2a36ab717 | 2020-09-23 16:36:16 -0700 | [diff] [blame] | 580 | return membarrier_private_expedited(MEMBARRIER_FLAG_SYNC_CORE, cpu_id); |
Mathieu Desnoyers | 70216e1 | 2018-01-29 15:20:17 -0500 | [diff] [blame] | 581 | case MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_SYNC_CORE: |
| 582 | return membarrier_register_private_expedited(MEMBARRIER_FLAG_SYNC_CORE); |
Peter Oskolkov | 2a36ab717 | 2020-09-23 16:36:16 -0700 | [diff] [blame] | 583 | case MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ: |
| 584 | return membarrier_private_expedited(MEMBARRIER_FLAG_RSEQ, cpu_id); |
| 585 | case MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_RSEQ: |
| 586 | return membarrier_register_private_expedited(MEMBARRIER_FLAG_RSEQ); |
Mathieu Desnoyers | 22e4ebb | 2017-07-28 16:40:40 -0400 | [diff] [blame] | 587 | default: |
| 588 | return -EINVAL; |
| 589 | } |
| 590 | } |