blob: 5a40b3828ff28c6031bb689d0caf7bcf1bcb0a27 [file] [log] [blame]
Thomas Gleixnerc942fdd2019-05-27 08:55:06 +02001// SPDX-License-Identifier: GPL-2.0-or-later
Mathieu Desnoyers22e4ebb2017-07-28 16:40:40 -04002/*
3 * Copyright (C) 2010-2017 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
4 *
5 * membarrier system call
Mathieu Desnoyers22e4ebb2017-07-28 16:40:40 -04006 */
Ingo Molnar325ea102018-03-03 12:20:47 +01007#include "sched.h"
Mathieu Desnoyers22e4ebb2017-07-28 16:40:40 -04008
9/*
Mathieu Desnoyers25595eb2020-10-20 09:47:15 -040010 * For documentation purposes, here are some membarrier ordering
11 * scenarios to keep in mind:
12 *
13 * A) Userspace thread execution after IPI vs membarrier's memory
14 * barrier before sending the IPI
15 *
16 * Userspace variables:
17 *
18 * int x = 0, y = 0;
19 *
20 * The memory barrier at the start of membarrier() on CPU0 is necessary in
21 * order to enforce the guarantee that any writes occurring on CPU0 before
22 * the membarrier() is executed will be visible to any code executing on
23 * CPU1 after the IPI-induced memory barrier:
24 *
25 * CPU0 CPU1
26 *
27 * x = 1
28 * membarrier():
29 * a: smp_mb()
30 * b: send IPI IPI-induced mb
31 * c: smp_mb()
32 * r2 = y
33 * y = 1
34 * barrier()
35 * r1 = x
36 *
37 * BUG_ON(r1 == 0 && r2 == 0)
38 *
39 * The write to y and load from x by CPU1 are unordered by the hardware,
40 * so it's possible to have "r1 = x" reordered before "y = 1" at any
41 * point after (b). If the memory barrier at (a) is omitted, then "x = 1"
42 * can be reordered after (a) (although not after (c)), so we get r1 == 0
43 * and r2 == 0. This violates the guarantee that membarrier() is
44 * supposed by provide.
45 *
46 * The timing of the memory barrier at (a) has to ensure that it executes
47 * before the IPI-induced memory barrier on CPU1.
48 *
49 * B) Userspace thread execution before IPI vs membarrier's memory
50 * barrier after completing the IPI
51 *
52 * Userspace variables:
53 *
54 * int x = 0, y = 0;
55 *
56 * The memory barrier at the end of membarrier() on CPU0 is necessary in
57 * order to enforce the guarantee that any writes occurring on CPU1 before
58 * the membarrier() is executed will be visible to any code executing on
59 * CPU0 after the membarrier():
60 *
61 * CPU0 CPU1
62 *
63 * x = 1
64 * barrier()
65 * y = 1
66 * r2 = y
67 * membarrier():
68 * a: smp_mb()
69 * b: send IPI IPI-induced mb
70 * c: smp_mb()
71 * r1 = x
72 * BUG_ON(r1 == 0 && r2 == 1)
73 *
74 * The writes to x and y are unordered by the hardware, so it's possible to
75 * have "r2 = 1" even though the write to x doesn't execute until (b). If
76 * the memory barrier at (c) is omitted then "r1 = x" can be reordered
77 * before (b) (although not before (a)), so we get "r1 = 0". This violates
78 * the guarantee that membarrier() is supposed to provide.
79 *
80 * The timing of the memory barrier at (c) has to ensure that it executes
81 * after the IPI-induced memory barrier on CPU1.
82 *
83 * C) Scheduling userspace thread -> kthread -> userspace thread vs membarrier
84 *
85 * CPU0 CPU1
86 *
87 * membarrier():
88 * a: smp_mb()
89 * d: switch to kthread (includes mb)
90 * b: read rq->curr->mm == NULL
91 * e: switch to user (includes mb)
92 * c: smp_mb()
93 *
94 * Using the scenario from (A), we can show that (a) needs to be paired
95 * with (e). Using the scenario from (B), we can show that (c) needs to
96 * be paired with (d).
97 *
98 * D) exit_mm vs membarrier
99 *
100 * Two thread groups are created, A and B. Thread group B is created by
101 * issuing clone from group A with flag CLONE_VM set, but not CLONE_THREAD.
102 * Let's assume we have a single thread within each thread group (Thread A
103 * and Thread B). Thread A runs on CPU0, Thread B runs on CPU1.
104 *
105 * CPU0 CPU1
106 *
107 * membarrier():
108 * a: smp_mb()
109 * exit_mm():
110 * d: smp_mb()
111 * e: current->mm = NULL
112 * b: read rq->curr->mm == NULL
113 * c: smp_mb()
114 *
115 * Using scenario (B), we can show that (c) needs to be paired with (d).
116 *
117 * E) kthread_{use,unuse}_mm vs membarrier
118 *
119 * CPU0 CPU1
120 *
121 * membarrier():
122 * a: smp_mb()
123 * kthread_unuse_mm()
124 * d: smp_mb()
125 * e: current->mm = NULL
126 * b: read rq->curr->mm == NULL
127 * kthread_use_mm()
128 * f: current->mm = mm
129 * g: smp_mb()
130 * c: smp_mb()
131 *
132 * Using the scenario from (A), we can show that (a) needs to be paired
133 * with (g). Using the scenario from (B), we can show that (c) needs to
134 * be paired with (d).
135 */
136
137/*
Mathieu Desnoyers22e4ebb2017-07-28 16:40:40 -0400138 * Bitmask made from a "or" of all commands within enum membarrier_cmd,
139 * except MEMBARRIER_CMD_QUERY.
140 */
Mathieu Desnoyers70216e12018-01-29 15:20:17 -0500141#ifdef CONFIG_ARCH_HAS_MEMBARRIER_SYNC_CORE
Ingo Molnar97fb7a02018-03-03 14:01:12 +0100142#define MEMBARRIER_PRIVATE_EXPEDITED_SYNC_CORE_BITMASK \
143 (MEMBARRIER_CMD_PRIVATE_EXPEDITED_SYNC_CORE \
Mathieu Desnoyers70216e12018-01-29 15:20:17 -0500144 | MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_SYNC_CORE)
145#else
146#define MEMBARRIER_PRIVATE_EXPEDITED_SYNC_CORE_BITMASK 0
147#endif
148
Peter Oskolkov2a36ab7172020-09-23 16:36:16 -0700149#ifdef CONFIG_RSEQ
150#define MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ_BITMASK \
151 (MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ \
152 | MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_RSEQ_BITMASK)
153#else
154#define MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ_BITMASK 0
155#endif
156
Ingo Molnar97fb7a02018-03-03 14:01:12 +0100157#define MEMBARRIER_CMD_BITMASK \
158 (MEMBARRIER_CMD_GLOBAL | MEMBARRIER_CMD_GLOBAL_EXPEDITED \
159 | MEMBARRIER_CMD_REGISTER_GLOBAL_EXPEDITED \
160 | MEMBARRIER_CMD_PRIVATE_EXPEDITED \
161 | MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED \
Mathieu Desnoyers70216e12018-01-29 15:20:17 -0500162 | MEMBARRIER_PRIVATE_EXPEDITED_SYNC_CORE_BITMASK)
Mathieu Desnoyers22e4ebb2017-07-28 16:40:40 -0400163
164static void ipi_mb(void *info)
165{
166 smp_mb(); /* IPIs should be serializing but paranoid. */
167}
168
Peter Oskolkov2a36ab7172020-09-23 16:36:16 -0700169static void ipi_rseq(void *info)
170{
171 rseq_preempt(current);
172}
173
Mathieu Desnoyers227a4aa2019-09-19 13:37:02 -0400174static void ipi_sync_rq_state(void *info)
175{
176 struct mm_struct *mm = (struct mm_struct *) info;
177
178 if (current->mm != mm)
179 return;
180 this_cpu_write(runqueues.membarrier_state,
181 atomic_read(&mm->membarrier_state));
182 /*
183 * Issue a memory barrier after setting
184 * MEMBARRIER_STATE_GLOBAL_EXPEDITED in the current runqueue to
185 * guarantee that no memory access following registration is reordered
186 * before registration.
187 */
188 smp_mb();
189}
190
191void membarrier_exec_mmap(struct mm_struct *mm)
192{
193 /*
194 * Issue a memory barrier before clearing membarrier_state to
195 * guarantee that no memory access prior to exec is reordered after
196 * clearing this state.
197 */
198 smp_mb();
199 atomic_set(&mm->membarrier_state, 0);
200 /*
201 * Keep the runqueue membarrier_state in sync with this mm
202 * membarrier_state.
203 */
204 this_cpu_write(runqueues.membarrier_state, 0);
205}
206
Mathieu Desnoyers5bc78502020-10-20 09:47:13 -0400207void membarrier_update_current_mm(struct mm_struct *next_mm)
208{
209 struct rq *rq = this_rq();
210 int membarrier_state = 0;
211
212 if (next_mm)
213 membarrier_state = atomic_read(&next_mm->membarrier_state);
214 if (READ_ONCE(rq->membarrier_state) == membarrier_state)
215 return;
216 WRITE_ONCE(rq->membarrier_state, membarrier_state);
217}
218
Mathieu Desnoyersc5f58bd2018-01-29 15:20:13 -0500219static int membarrier_global_expedited(void)
220{
221 int cpu;
Mathieu Desnoyersc5f58bd2018-01-29 15:20:13 -0500222 cpumask_var_t tmpmask;
223
224 if (num_online_cpus() == 1)
225 return 0;
226
227 /*
228 * Matches memory barriers around rq->curr modification in
229 * scheduler.
230 */
231 smp_mb(); /* system call entry is not a mb. */
232
Mathieu Desnoyersc172e0a2019-09-19 13:37:05 -0400233 if (!zalloc_cpumask_var(&tmpmask, GFP_KERNEL))
234 return -ENOMEM;
Mathieu Desnoyersc5f58bd2018-01-29 15:20:13 -0500235
236 cpus_read_lock();
Mathieu Desnoyers227a4aa2019-09-19 13:37:02 -0400237 rcu_read_lock();
Mathieu Desnoyersc5f58bd2018-01-29 15:20:13 -0500238 for_each_online_cpu(cpu) {
239 struct task_struct *p;
240
241 /*
242 * Skipping the current CPU is OK even through we can be
243 * migrated at any point. The current CPU, at the point
244 * where we read raw_smp_processor_id(), is ensured to
245 * be in program order with respect to the caller
246 * thread. Therefore, we can skip this CPU from the
247 * iteration.
248 */
249 if (cpu == raw_smp_processor_id())
250 continue;
Ingo Molnar97fb7a02018-03-03 14:01:12 +0100251
Mathieu Desnoyers227a4aa2019-09-19 13:37:02 -0400252 if (!(READ_ONCE(cpu_rq(cpu)->membarrier_state) &
253 MEMBARRIER_STATE_GLOBAL_EXPEDITED))
254 continue;
255
256 /*
Mathieu Desnoyers618758e2020-10-20 09:47:14 -0400257 * Skip the CPU if it runs a kernel thread which is not using
258 * a task mm.
Mathieu Desnoyers227a4aa2019-09-19 13:37:02 -0400259 */
Eric W. Biederman154abaf2019-09-14 07:34:30 -0500260 p = rcu_dereference(cpu_rq(cpu)->curr);
Mathieu Desnoyers618758e2020-10-20 09:47:14 -0400261 if (!p->mm)
Mathieu Desnoyers227a4aa2019-09-19 13:37:02 -0400262 continue;
263
Mathieu Desnoyersc172e0a2019-09-19 13:37:05 -0400264 __cpumask_set_cpu(cpu, tmpmask);
Mathieu Desnoyersc5f58bd2018-01-29 15:20:13 -0500265 }
Mathieu Desnoyers227a4aa2019-09-19 13:37:02 -0400266 rcu_read_unlock();
Mathieu Desnoyersc172e0a2019-09-19 13:37:05 -0400267
268 preempt_disable();
269 smp_call_function_many(tmpmask, ipi_mb, NULL, 1);
270 preempt_enable();
271
272 free_cpumask_var(tmpmask);
Mathieu Desnoyersc5f58bd2018-01-29 15:20:13 -0500273 cpus_read_unlock();
274
275 /*
276 * Memory barrier on the caller thread _after_ we finished
277 * waiting for the last IPI. Matches memory barriers around
278 * rq->curr modification in scheduler.
279 */
280 smp_mb(); /* exit from system call is not a mb */
281 return 0;
282}
283
Peter Oskolkov2a36ab7172020-09-23 16:36:16 -0700284static int membarrier_private_expedited(int flags, int cpu_id)
Mathieu Desnoyers22e4ebb2017-07-28 16:40:40 -0400285{
Mathieu Desnoyers22e4ebb2017-07-28 16:40:40 -0400286 cpumask_var_t tmpmask;
Mathieu Desnoyersc6d68c12019-09-19 13:37:04 -0400287 struct mm_struct *mm = current->mm;
Peter Oskolkov2a36ab7172020-09-23 16:36:16 -0700288 smp_call_func_t ipi_func = ipi_mb;
Mathieu Desnoyers22e4ebb2017-07-28 16:40:40 -0400289
Peter Oskolkov2a36ab7172020-09-23 16:36:16 -0700290 if (flags == MEMBARRIER_FLAG_SYNC_CORE) {
Mathieu Desnoyers70216e12018-01-29 15:20:17 -0500291 if (!IS_ENABLED(CONFIG_ARCH_HAS_MEMBARRIER_SYNC_CORE))
292 return -EINVAL;
Mathieu Desnoyersc6d68c12019-09-19 13:37:04 -0400293 if (!(atomic_read(&mm->membarrier_state) &
Mathieu Desnoyers70216e12018-01-29 15:20:17 -0500294 MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE_READY))
295 return -EPERM;
Peter Oskolkov2a36ab7172020-09-23 16:36:16 -0700296 } else if (flags == MEMBARRIER_FLAG_RSEQ) {
297 if (!IS_ENABLED(CONFIG_RSEQ))
298 return -EINVAL;
299 if (!(atomic_read(&mm->membarrier_state) &
300 MEMBARRIER_STATE_PRIVATE_EXPEDITED_RSEQ_READY))
301 return -EPERM;
302 ipi_func = ipi_rseq;
Mathieu Desnoyers70216e12018-01-29 15:20:17 -0500303 } else {
Peter Oskolkov2a36ab7172020-09-23 16:36:16 -0700304 WARN_ON_ONCE(flags);
Mathieu Desnoyersc6d68c12019-09-19 13:37:04 -0400305 if (!(atomic_read(&mm->membarrier_state) &
Mathieu Desnoyers70216e12018-01-29 15:20:17 -0500306 MEMBARRIER_STATE_PRIVATE_EXPEDITED_READY))
307 return -EPERM;
308 }
Mathieu Desnoyersa961e402017-10-19 13:30:15 -0400309
Mathieu Desnoyersc6d68c12019-09-19 13:37:04 -0400310 if (atomic_read(&mm->mm_users) == 1 || num_online_cpus() == 1)
Mathieu Desnoyersa961e402017-10-19 13:30:15 -0400311 return 0;
Mathieu Desnoyers22e4ebb2017-07-28 16:40:40 -0400312
313 /*
314 * Matches memory barriers around rq->curr modification in
315 * scheduler.
316 */
317 smp_mb(); /* system call entry is not a mb. */
318
Peter Oskolkov2a36ab7172020-09-23 16:36:16 -0700319 if (cpu_id < 0 && !zalloc_cpumask_var(&tmpmask, GFP_KERNEL))
Mathieu Desnoyersc172e0a2019-09-19 13:37:05 -0400320 return -ENOMEM;
Mathieu Desnoyers22e4ebb2017-07-28 16:40:40 -0400321
322 cpus_read_lock();
Peter Oskolkov2a36ab7172020-09-23 16:36:16 -0700323
324 if (cpu_id >= 0) {
Mathieu Desnoyers22e4ebb2017-07-28 16:40:40 -0400325 struct task_struct *p;
326
Peter Oskolkov2a36ab7172020-09-23 16:36:16 -0700327 if (cpu_id >= nr_cpu_ids || !cpu_online(cpu_id))
328 goto out;
329 if (cpu_id == raw_smp_processor_id())
330 goto out;
331 rcu_read_lock();
332 p = rcu_dereference(cpu_rq(cpu_id)->curr);
333 if (!p || p->mm != mm) {
334 rcu_read_unlock();
335 goto out;
336 }
337 rcu_read_unlock();
338 } else {
339 int cpu;
340
341 rcu_read_lock();
342 for_each_online_cpu(cpu) {
343 struct task_struct *p;
344
345 /*
346 * Skipping the current CPU is OK even through we can be
347 * migrated at any point. The current CPU, at the point
348 * where we read raw_smp_processor_id(), is ensured to
349 * be in program order with respect to the caller
350 * thread. Therefore, we can skip this CPU from the
351 * iteration.
352 */
353 if (cpu == raw_smp_processor_id())
354 continue;
355 p = rcu_dereference(cpu_rq(cpu)->curr);
356 if (p && p->mm == mm)
357 __cpumask_set_cpu(cpu, tmpmask);
358 }
359 rcu_read_unlock();
Mathieu Desnoyers22e4ebb2017-07-28 16:40:40 -0400360 }
Mathieu Desnoyersc172e0a2019-09-19 13:37:05 -0400361
362 preempt_disable();
Peter Oskolkov2a36ab7172020-09-23 16:36:16 -0700363 if (cpu_id >= 0)
364 smp_call_function_single(cpu_id, ipi_func, NULL, 1);
365 else
366 smp_call_function_many(tmpmask, ipi_func, NULL, 1);
Mathieu Desnoyersc172e0a2019-09-19 13:37:05 -0400367 preempt_enable();
368
Peter Oskolkov2a36ab7172020-09-23 16:36:16 -0700369out:
370 if (cpu_id < 0)
371 free_cpumask_var(tmpmask);
Mathieu Desnoyers22e4ebb2017-07-28 16:40:40 -0400372 cpus_read_unlock();
373
374 /*
375 * Memory barrier on the caller thread _after_ we finished
376 * waiting for the last IPI. Matches memory barriers around
377 * rq->curr modification in scheduler.
378 */
379 smp_mb(); /* exit from system call is not a mb */
Ingo Molnar97fb7a02018-03-03 14:01:12 +0100380
Mathieu Desnoyersa961e402017-10-19 13:30:15 -0400381 return 0;
382}
383
Mathieu Desnoyers227a4aa2019-09-19 13:37:02 -0400384static int sync_runqueues_membarrier_state(struct mm_struct *mm)
385{
386 int membarrier_state = atomic_read(&mm->membarrier_state);
387 cpumask_var_t tmpmask;
388 int cpu;
389
390 if (atomic_read(&mm->mm_users) == 1 || num_online_cpus() == 1) {
391 this_cpu_write(runqueues.membarrier_state, membarrier_state);
392
393 /*
394 * For single mm user, we can simply issue a memory barrier
395 * after setting MEMBARRIER_STATE_GLOBAL_EXPEDITED in the
396 * mm and in the current runqueue to guarantee that no memory
397 * access following registration is reordered before
398 * registration.
399 */
400 smp_mb();
401 return 0;
402 }
403
404 if (!zalloc_cpumask_var(&tmpmask, GFP_KERNEL))
405 return -ENOMEM;
406
407 /*
408 * For mm with multiple users, we need to ensure all future
409 * scheduler executions will observe @mm's new membarrier
410 * state.
411 */
412 synchronize_rcu();
413
414 /*
415 * For each cpu runqueue, if the task's mm match @mm, ensure that all
416 * @mm's membarrier state set bits are also set in in the runqueue's
417 * membarrier state. This ensures that a runqueue scheduling
418 * between threads which are users of @mm has its membarrier state
419 * updated.
420 */
421 cpus_read_lock();
422 rcu_read_lock();
423 for_each_online_cpu(cpu) {
424 struct rq *rq = cpu_rq(cpu);
425 struct task_struct *p;
426
Mathieu Desnoyersc172e0a2019-09-19 13:37:05 -0400427 p = rcu_dereference(rq->curr);
Mathieu Desnoyers227a4aa2019-09-19 13:37:02 -0400428 if (p && p->mm == mm)
429 __cpumask_set_cpu(cpu, tmpmask);
430 }
431 rcu_read_unlock();
432
433 preempt_disable();
434 smp_call_function_many(tmpmask, ipi_sync_rq_state, mm, 1);
435 preempt_enable();
436
437 free_cpumask_var(tmpmask);
438 cpus_read_unlock();
439
440 return 0;
441}
442
Mathieu Desnoyersc5f58bd2018-01-29 15:20:13 -0500443static int membarrier_register_global_expedited(void)
444{
445 struct task_struct *p = current;
446 struct mm_struct *mm = p->mm;
Mathieu Desnoyers227a4aa2019-09-19 13:37:02 -0400447 int ret;
Mathieu Desnoyersc5f58bd2018-01-29 15:20:13 -0500448
449 if (atomic_read(&mm->membarrier_state) &
450 MEMBARRIER_STATE_GLOBAL_EXPEDITED_READY)
451 return 0;
452 atomic_or(MEMBARRIER_STATE_GLOBAL_EXPEDITED, &mm->membarrier_state);
Mathieu Desnoyers227a4aa2019-09-19 13:37:02 -0400453 ret = sync_runqueues_membarrier_state(mm);
454 if (ret)
455 return ret;
Mathieu Desnoyersc5f58bd2018-01-29 15:20:13 -0500456 atomic_or(MEMBARRIER_STATE_GLOBAL_EXPEDITED_READY,
457 &mm->membarrier_state);
Ingo Molnar97fb7a02018-03-03 14:01:12 +0100458
Mathieu Desnoyersc5f58bd2018-01-29 15:20:13 -0500459 return 0;
460}
461
Mathieu Desnoyers70216e12018-01-29 15:20:17 -0500462static int membarrier_register_private_expedited(int flags)
Mathieu Desnoyersa961e402017-10-19 13:30:15 -0400463{
464 struct task_struct *p = current;
465 struct mm_struct *mm = p->mm;
Mathieu Desnoyers227a4aa2019-09-19 13:37:02 -0400466 int ready_state = MEMBARRIER_STATE_PRIVATE_EXPEDITED_READY,
467 set_state = MEMBARRIER_STATE_PRIVATE_EXPEDITED,
468 ret;
Mathieu Desnoyers70216e12018-01-29 15:20:17 -0500469
Peter Oskolkov2a36ab7172020-09-23 16:36:16 -0700470 if (flags == MEMBARRIER_FLAG_SYNC_CORE) {
Mathieu Desnoyers70216e12018-01-29 15:20:17 -0500471 if (!IS_ENABLED(CONFIG_ARCH_HAS_MEMBARRIER_SYNC_CORE))
472 return -EINVAL;
Mathieu Desnoyers227a4aa2019-09-19 13:37:02 -0400473 ready_state =
474 MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE_READY;
Peter Oskolkov2a36ab7172020-09-23 16:36:16 -0700475 } else if (flags == MEMBARRIER_FLAG_RSEQ) {
476 if (!IS_ENABLED(CONFIG_RSEQ))
477 return -EINVAL;
478 ready_state =
479 MEMBARRIER_STATE_PRIVATE_EXPEDITED_RSEQ_READY;
480 } else {
481 WARN_ON_ONCE(flags);
Mathieu Desnoyers70216e12018-01-29 15:20:17 -0500482 }
Mathieu Desnoyersa961e402017-10-19 13:30:15 -0400483
484 /*
485 * We need to consider threads belonging to different thread
486 * groups, which use the same mm. (CLONE_VM but not
487 * CLONE_THREAD).
488 */
Mathieu Desnoyers227a4aa2019-09-19 13:37:02 -0400489 if ((atomic_read(&mm->membarrier_state) & ready_state) == ready_state)
Mathieu Desnoyersc5f58bd2018-01-29 15:20:13 -0500490 return 0;
Mathieu Desnoyers70216e12018-01-29 15:20:17 -0500491 if (flags & MEMBARRIER_FLAG_SYNC_CORE)
Mathieu Desnoyers227a4aa2019-09-19 13:37:02 -0400492 set_state |= MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE;
Peter Oskolkov2a36ab7172020-09-23 16:36:16 -0700493 if (flags & MEMBARRIER_FLAG_RSEQ)
494 set_state |= MEMBARRIER_STATE_PRIVATE_EXPEDITED_RSEQ;
Mathieu Desnoyers227a4aa2019-09-19 13:37:02 -0400495 atomic_or(set_state, &mm->membarrier_state);
496 ret = sync_runqueues_membarrier_state(mm);
497 if (ret)
498 return ret;
499 atomic_or(ready_state, &mm->membarrier_state);
Ingo Molnar97fb7a02018-03-03 14:01:12 +0100500
Mathieu Desnoyersc5f58bd2018-01-29 15:20:13 -0500501 return 0;
Mathieu Desnoyers22e4ebb2017-07-28 16:40:40 -0400502}
503
504/**
505 * sys_membarrier - issue memory barriers on a set of threads
Peter Oskolkov2a36ab7172020-09-23 16:36:16 -0700506 * @cmd: Takes command values defined in enum membarrier_cmd.
507 * @flags: Currently needs to be 0 for all commands other than
508 * MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ: in the latter
509 * case it can be MEMBARRIER_CMD_FLAG_CPU, indicating that @cpu_id
510 * contains the CPU on which to interrupt (= restart)
511 * the RSEQ critical section.
512 * @cpu_id: if @flags == MEMBARRIER_CMD_FLAG_CPU, indicates the cpu on which
513 * RSEQ CS should be interrupted (@cmd must be
514 * MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ).
Mathieu Desnoyers22e4ebb2017-07-28 16:40:40 -0400515 *
516 * If this system call is not implemented, -ENOSYS is returned. If the
517 * command specified does not exist, not available on the running
518 * kernel, or if the command argument is invalid, this system call
519 * returns -EINVAL. For a given command, with flags argument set to 0,
Mathieu Desnoyers227a4aa2019-09-19 13:37:02 -0400520 * if this system call returns -ENOSYS or -EINVAL, it is guaranteed to
521 * always return the same value until reboot. In addition, it can return
522 * -ENOMEM if there is not enough memory available to perform the system
523 * call.
Mathieu Desnoyers22e4ebb2017-07-28 16:40:40 -0400524 *
525 * All memory accesses performed in program order from each targeted thread
526 * is guaranteed to be ordered with respect to sys_membarrier(). If we use
527 * the semantic "barrier()" to represent a compiler barrier forcing memory
528 * accesses to be performed in program order across the barrier, and
529 * smp_mb() to represent explicit memory barriers forcing full memory
530 * ordering across the barrier, we have the following ordering table for
531 * each pair of barrier(), sys_membarrier() and smp_mb():
532 *
533 * The pair ordering is detailed as (O: ordered, X: not ordered):
534 *
535 * barrier() smp_mb() sys_membarrier()
536 * barrier() X X O
537 * smp_mb() X O O
538 * sys_membarrier() O O O
539 */
Peter Oskolkov2a36ab7172020-09-23 16:36:16 -0700540SYSCALL_DEFINE3(membarrier, int, cmd, unsigned int, flags, int, cpu_id)
Mathieu Desnoyers22e4ebb2017-07-28 16:40:40 -0400541{
Peter Oskolkov2a36ab7172020-09-23 16:36:16 -0700542 switch (cmd) {
543 case MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ:
544 if (unlikely(flags && flags != MEMBARRIER_CMD_FLAG_CPU))
545 return -EINVAL;
546 break;
547 default:
548 if (unlikely(flags))
549 return -EINVAL;
550 }
551
552 if (!(flags & MEMBARRIER_CMD_FLAG_CPU))
553 cpu_id = -1;
554
Mathieu Desnoyers22e4ebb2017-07-28 16:40:40 -0400555 switch (cmd) {
556 case MEMBARRIER_CMD_QUERY:
557 {
558 int cmd_mask = MEMBARRIER_CMD_BITMASK;
559
560 if (tick_nohz_full_enabled())
Mathieu Desnoyersc5f58bd2018-01-29 15:20:13 -0500561 cmd_mask &= ~MEMBARRIER_CMD_GLOBAL;
Mathieu Desnoyers22e4ebb2017-07-28 16:40:40 -0400562 return cmd_mask;
563 }
Mathieu Desnoyersc5f58bd2018-01-29 15:20:13 -0500564 case MEMBARRIER_CMD_GLOBAL:
565 /* MEMBARRIER_CMD_GLOBAL is not compatible with nohz_full. */
Mathieu Desnoyers22e4ebb2017-07-28 16:40:40 -0400566 if (tick_nohz_full_enabled())
567 return -EINVAL;
568 if (num_online_cpus() > 1)
Paul E. McKenney78d125d2018-07-11 15:36:43 -0700569 synchronize_rcu();
Mathieu Desnoyers22e4ebb2017-07-28 16:40:40 -0400570 return 0;
Mathieu Desnoyersc5f58bd2018-01-29 15:20:13 -0500571 case MEMBARRIER_CMD_GLOBAL_EXPEDITED:
572 return membarrier_global_expedited();
573 case MEMBARRIER_CMD_REGISTER_GLOBAL_EXPEDITED:
574 return membarrier_register_global_expedited();
Mathieu Desnoyers22e4ebb2017-07-28 16:40:40 -0400575 case MEMBARRIER_CMD_PRIVATE_EXPEDITED:
Peter Oskolkov2a36ab7172020-09-23 16:36:16 -0700576 return membarrier_private_expedited(0, cpu_id);
Mathieu Desnoyersa961e402017-10-19 13:30:15 -0400577 case MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED:
Mathieu Desnoyers70216e12018-01-29 15:20:17 -0500578 return membarrier_register_private_expedited(0);
579 case MEMBARRIER_CMD_PRIVATE_EXPEDITED_SYNC_CORE:
Peter Oskolkov2a36ab7172020-09-23 16:36:16 -0700580 return membarrier_private_expedited(MEMBARRIER_FLAG_SYNC_CORE, cpu_id);
Mathieu Desnoyers70216e12018-01-29 15:20:17 -0500581 case MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_SYNC_CORE:
582 return membarrier_register_private_expedited(MEMBARRIER_FLAG_SYNC_CORE);
Peter Oskolkov2a36ab7172020-09-23 16:36:16 -0700583 case MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ:
584 return membarrier_private_expedited(MEMBARRIER_FLAG_RSEQ, cpu_id);
585 case MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_RSEQ:
586 return membarrier_register_private_expedited(MEMBARRIER_FLAG_RSEQ);
Mathieu Desnoyers22e4ebb2017-07-28 16:40:40 -0400587 default:
588 return -EINVAL;
589 }
590}