blob: ccbdbd62f0d8ecf16a19f2f191d5809e317da9d5 [file] [log] [blame]
Thomas Gleixner20c8ccb2019-06-04 10:11:32 +02001// SPDX-License-Identifier: GPL-2.0-only
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07002/*
3 * fs/userfaultfd.c
4 *
5 * Copyright (C) 2007 Davide Libenzi <davidel@xmailserver.org>
6 * Copyright (C) 2008-2009 Red Hat, Inc.
7 * Copyright (C) 2015 Red Hat, Inc.
8 *
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07009 * Some part derived from fs/eventfd.c (anon inode setup) and
10 * mm/ksm.c (mm hashing).
11 */
12
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -080013#include <linux/list.h>
Andrea Arcangeli86039bd2015-09-04 15:46:31 -070014#include <linux/hashtable.h>
Ingo Molnar174cd4b2017-02-02 19:15:33 +010015#include <linux/sched/signal.h>
Ingo Molnar6e84f312017-02-08 18:51:29 +010016#include <linux/sched/mm.h>
Andrea Arcangeli86039bd2015-09-04 15:46:31 -070017#include <linux/mm.h>
18#include <linux/poll.h>
19#include <linux/slab.h>
20#include <linux/seq_file.h>
21#include <linux/file.h>
22#include <linux/bug.h>
23#include <linux/anon_inodes.h>
24#include <linux/syscalls.h>
25#include <linux/userfaultfd_k.h>
26#include <linux/mempolicy.h>
27#include <linux/ioctl.h>
28#include <linux/security.h>
Mike Kravetzcab350a2017-02-22 15:43:04 -080029#include <linux/hugetlb.h>
Andrea Arcangeli86039bd2015-09-04 15:46:31 -070030
Peter Xucefdca02019-05-13 17:16:41 -070031int sysctl_unprivileged_userfaultfd __read_mostly = 1;
32
Andrea Arcangeli3004ec92015-09-04 15:46:48 -070033static struct kmem_cache *userfaultfd_ctx_cachep __read_mostly;
34
Andrea Arcangeli86039bd2015-09-04 15:46:31 -070035enum userfaultfd_state {
36 UFFD_STATE_WAIT_API,
37 UFFD_STATE_RUNNING,
38};
39
Andrea Arcangeli3004ec92015-09-04 15:46:48 -070040/*
41 * Start with fault_pending_wqh and fault_wqh so they're more likely
42 * to be in the same cacheline.
Eric Biggerscbcfa132019-07-04 15:14:39 -070043 *
44 * Locking order:
45 * fd_wqh.lock
46 * fault_pending_wqh.lock
47 * fault_wqh.lock
48 * event_wqh.lock
49 *
50 * To avoid deadlocks, IRQs must be disabled when taking any of the above locks,
51 * since fd_wqh.lock is taken by aio_poll() while it's holding a lock that's
52 * also taken in IRQ context.
Andrea Arcangeli3004ec92015-09-04 15:46:48 -070053 */
Andrea Arcangeli86039bd2015-09-04 15:46:31 -070054struct userfaultfd_ctx {
Andrea Arcangeli15b726e2015-09-04 15:46:44 -070055 /* waitqueue head for the pending (i.e. not read) userfaults */
56 wait_queue_head_t fault_pending_wqh;
57 /* waitqueue head for the userfaults */
Andrea Arcangeli86039bd2015-09-04 15:46:31 -070058 wait_queue_head_t fault_wqh;
59 /* waitqueue head for the pseudo fd to wakeup poll/read */
60 wait_queue_head_t fd_wqh;
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -080061 /* waitqueue head for events */
62 wait_queue_head_t event_wqh;
Andrea Arcangeli2c5b7e12015-09-04 15:47:23 -070063 /* a refile sequence protected by fault_pending_wqh lock */
64 struct seqcount refile_seq;
Andrea Arcangeli3004ec92015-09-04 15:46:48 -070065 /* pseudo fd refcounting */
Eric Biggersca880422018-12-28 00:34:43 -080066 refcount_t refcount;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -070067 /* userfaultfd syscall flags */
68 unsigned int flags;
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -080069 /* features requested from the userspace */
70 unsigned int features;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -070071 /* state machine */
72 enum userfaultfd_state state;
73 /* released */
74 bool released;
Mike Rapoportdf2cc962018-06-07 17:09:25 -070075 /* memory mappings are changing because of non-cooperative event */
76 bool mmap_changing;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -070077 /* mm with one ore more vmas attached to this userfaultfd_ctx */
78 struct mm_struct *mm;
79};
80
Pavel Emelyanov893e26e2017-02-22 15:42:27 -080081struct userfaultfd_fork_ctx {
82 struct userfaultfd_ctx *orig;
83 struct userfaultfd_ctx *new;
84 struct list_head list;
85};
86
Mike Rapoport897ab3e2017-02-24 14:58:22 -080087struct userfaultfd_unmap_ctx {
88 struct userfaultfd_ctx *ctx;
89 unsigned long start;
90 unsigned long end;
91 struct list_head list;
92};
93
Andrea Arcangeli86039bd2015-09-04 15:46:31 -070094struct userfaultfd_wait_queue {
Andrea Arcangelia9b85f92015-09-04 15:46:37 -070095 struct uffd_msg msg;
Ingo Molnarac6424b2017-06-20 12:06:13 +020096 wait_queue_entry_t wq;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -070097 struct userfaultfd_ctx *ctx;
Andrea Arcangeli15a77c62017-01-24 15:17:59 -080098 bool waken;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -070099};
100
101struct userfaultfd_wake_range {
102 unsigned long start;
103 unsigned long len;
104};
105
Ingo Molnarac6424b2017-06-20 12:06:13 +0200106static int userfaultfd_wake_function(wait_queue_entry_t *wq, unsigned mode,
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700107 int wake_flags, void *key)
108{
109 struct userfaultfd_wake_range *range = key;
110 int ret;
111 struct userfaultfd_wait_queue *uwq;
112 unsigned long start, len;
113
114 uwq = container_of(wq, struct userfaultfd_wait_queue, wq);
115 ret = 0;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700116 /* len == 0 means wake all */
117 start = range->start;
118 len = range->len;
Andrea Arcangelia9b85f92015-09-04 15:46:37 -0700119 if (len && (start > uwq->msg.arg.pagefault.address ||
120 start + len <= uwq->msg.arg.pagefault.address))
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700121 goto out;
Andrea Arcangeli15a77c62017-01-24 15:17:59 -0800122 WRITE_ONCE(uwq->waken, true);
123 /*
Peter Zijlstraa9668cd2017-06-07 17:51:27 +0200124 * The Program-Order guarantees provided by the scheduler
125 * ensure uwq->waken is visible before the task is woken.
Andrea Arcangeli15a77c62017-01-24 15:17:59 -0800126 */
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700127 ret = wake_up_state(wq->private, mode);
Peter Zijlstraa9668cd2017-06-07 17:51:27 +0200128 if (ret) {
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700129 /*
130 * Wake only once, autoremove behavior.
131 *
Peter Zijlstraa9668cd2017-06-07 17:51:27 +0200132 * After the effect of list_del_init is visible to the other
133 * CPUs, the waitqueue may disappear from under us, see the
134 * !list_empty_careful() in handle_userfault().
135 *
136 * try_to_wake_up() has an implicit smp_mb(), and the
137 * wq->private is read before calling the extern function
138 * "wake_up_state" (which in turns calls try_to_wake_up).
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700139 */
Ingo Molnar2055da92017-06-20 12:06:46 +0200140 list_del_init(&wq->entry);
Peter Zijlstraa9668cd2017-06-07 17:51:27 +0200141 }
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700142out:
143 return ret;
144}
145
146/**
147 * userfaultfd_ctx_get - Acquires a reference to the internal userfaultfd
148 * context.
149 * @ctx: [in] Pointer to the userfaultfd context.
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700150 */
151static void userfaultfd_ctx_get(struct userfaultfd_ctx *ctx)
152{
Eric Biggersca880422018-12-28 00:34:43 -0800153 refcount_inc(&ctx->refcount);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700154}
155
156/**
157 * userfaultfd_ctx_put - Releases a reference to the internal userfaultfd
158 * context.
159 * @ctx: [in] Pointer to userfaultfd context.
160 *
161 * The userfaultfd context reference must have been previously acquired either
162 * with userfaultfd_ctx_get() or userfaultfd_ctx_fdget().
163 */
164static void userfaultfd_ctx_put(struct userfaultfd_ctx *ctx)
165{
Eric Biggersca880422018-12-28 00:34:43 -0800166 if (refcount_dec_and_test(&ctx->refcount)) {
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700167 VM_BUG_ON(spin_is_locked(&ctx->fault_pending_wqh.lock));
168 VM_BUG_ON(waitqueue_active(&ctx->fault_pending_wqh));
169 VM_BUG_ON(spin_is_locked(&ctx->fault_wqh.lock));
170 VM_BUG_ON(waitqueue_active(&ctx->fault_wqh));
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -0800171 VM_BUG_ON(spin_is_locked(&ctx->event_wqh.lock));
172 VM_BUG_ON(waitqueue_active(&ctx->event_wqh));
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700173 VM_BUG_ON(spin_is_locked(&ctx->fd_wqh.lock));
174 VM_BUG_ON(waitqueue_active(&ctx->fd_wqh));
Oleg Nesterovd2005e32016-05-20 16:58:36 -0700175 mmdrop(ctx->mm);
Andrea Arcangeli3004ec92015-09-04 15:46:48 -0700176 kmem_cache_free(userfaultfd_ctx_cachep, ctx);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700177 }
178}
179
Andrea Arcangelia9b85f92015-09-04 15:46:37 -0700180static inline void msg_init(struct uffd_msg *msg)
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700181{
Andrea Arcangelia9b85f92015-09-04 15:46:37 -0700182 BUILD_BUG_ON(sizeof(struct uffd_msg) != 32);
183 /*
184 * Must use memset to zero out the paddings or kernel data is
185 * leaked to userland.
186 */
187 memset(msg, 0, sizeof(struct uffd_msg));
188}
189
190static inline struct uffd_msg userfault_msg(unsigned long address,
191 unsigned int flags,
Alexey Perevalov9d4ac932017-09-06 16:23:56 -0700192 unsigned long reason,
193 unsigned int features)
Andrea Arcangelia9b85f92015-09-04 15:46:37 -0700194{
195 struct uffd_msg msg;
196 msg_init(&msg);
197 msg.event = UFFD_EVENT_PAGEFAULT;
198 msg.arg.pagefault.address = address;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700199 if (flags & FAULT_FLAG_WRITE)
200 /*
Andrea Arcangelia4605a62017-02-22 15:42:09 -0800201 * If UFFD_FEATURE_PAGEFAULT_FLAG_WP was set in the
Andrea Arcangelia9b85f92015-09-04 15:46:37 -0700202 * uffdio_api.features and UFFD_PAGEFAULT_FLAG_WRITE
203 * was not set in a UFFD_EVENT_PAGEFAULT, it means it
204 * was a read fault, otherwise if set it means it's
205 * a write fault.
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700206 */
Andrea Arcangelia9b85f92015-09-04 15:46:37 -0700207 msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_WRITE;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700208 if (reason & VM_UFFD_WP)
209 /*
Andrea Arcangelia9b85f92015-09-04 15:46:37 -0700210 * If UFFD_FEATURE_PAGEFAULT_FLAG_WP was set in the
211 * uffdio_api.features and UFFD_PAGEFAULT_FLAG_WP was
212 * not set in a UFFD_EVENT_PAGEFAULT, it means it was
213 * a missing fault, otherwise if set it means it's a
214 * write protect fault.
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700215 */
Andrea Arcangelia9b85f92015-09-04 15:46:37 -0700216 msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_WP;
Alexey Perevalov9d4ac932017-09-06 16:23:56 -0700217 if (features & UFFD_FEATURE_THREAD_ID)
Andrea Arcangelia36985d2017-09-06 16:23:59 -0700218 msg.arg.pagefault.feat.ptid = task_pid_vnr(current);
Andrea Arcangelia9b85f92015-09-04 15:46:37 -0700219 return msg;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700220}
221
Mike Kravetz369cd2122017-02-22 15:43:10 -0800222#ifdef CONFIG_HUGETLB_PAGE
223/*
224 * Same functionality as userfaultfd_must_wait below with modifications for
225 * hugepmd ranges.
226 */
227static inline bool userfaultfd_huge_must_wait(struct userfaultfd_ctx *ctx,
Punit Agrawal7868a202017-07-06 15:39:42 -0700228 struct vm_area_struct *vma,
Mike Kravetz369cd2122017-02-22 15:43:10 -0800229 unsigned long address,
230 unsigned long flags,
231 unsigned long reason)
232{
233 struct mm_struct *mm = ctx->mm;
Janosch Frank1e2c0432018-07-03 17:02:39 -0700234 pte_t *ptep, pte;
Mike Kravetz369cd2122017-02-22 15:43:10 -0800235 bool ret = true;
236
237 VM_BUG_ON(!rwsem_is_locked(&mm->mmap_sem));
238
Janosch Frank1e2c0432018-07-03 17:02:39 -0700239 ptep = huge_pte_offset(mm, address, vma_mmu_pagesize(vma));
240
241 if (!ptep)
Mike Kravetz369cd2122017-02-22 15:43:10 -0800242 goto out;
243
244 ret = false;
Janosch Frank1e2c0432018-07-03 17:02:39 -0700245 pte = huge_ptep_get(ptep);
Mike Kravetz369cd2122017-02-22 15:43:10 -0800246
247 /*
248 * Lockless access: we're in a wait_event so it's ok if it
249 * changes under us.
250 */
Janosch Frank1e2c0432018-07-03 17:02:39 -0700251 if (huge_pte_none(pte))
Mike Kravetz369cd2122017-02-22 15:43:10 -0800252 ret = true;
Janosch Frank1e2c0432018-07-03 17:02:39 -0700253 if (!huge_pte_write(pte) && (reason & VM_UFFD_WP))
Mike Kravetz369cd2122017-02-22 15:43:10 -0800254 ret = true;
255out:
256 return ret;
257}
258#else
259static inline bool userfaultfd_huge_must_wait(struct userfaultfd_ctx *ctx,
Punit Agrawal7868a202017-07-06 15:39:42 -0700260 struct vm_area_struct *vma,
Mike Kravetz369cd2122017-02-22 15:43:10 -0800261 unsigned long address,
262 unsigned long flags,
263 unsigned long reason)
264{
265 return false; /* should never get here */
266}
267#endif /* CONFIG_HUGETLB_PAGE */
268
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700269/*
Andrea Arcangeli8d2afd92015-09-04 15:46:51 -0700270 * Verify the pagetables are still not ok after having reigstered into
271 * the fault_pending_wqh to avoid userland having to UFFDIO_WAKE any
272 * userfault that has already been resolved, if userfaultfd_read and
273 * UFFDIO_COPY|ZEROPAGE are being run simultaneously on two different
274 * threads.
275 */
276static inline bool userfaultfd_must_wait(struct userfaultfd_ctx *ctx,
277 unsigned long address,
278 unsigned long flags,
279 unsigned long reason)
280{
281 struct mm_struct *mm = ctx->mm;
282 pgd_t *pgd;
Kirill A. Shutemovc2febaf2017-03-09 17:24:07 +0300283 p4d_t *p4d;
Andrea Arcangeli8d2afd92015-09-04 15:46:51 -0700284 pud_t *pud;
285 pmd_t *pmd, _pmd;
286 pte_t *pte;
287 bool ret = true;
288
289 VM_BUG_ON(!rwsem_is_locked(&mm->mmap_sem));
290
291 pgd = pgd_offset(mm, address);
292 if (!pgd_present(*pgd))
293 goto out;
Kirill A. Shutemovc2febaf2017-03-09 17:24:07 +0300294 p4d = p4d_offset(pgd, address);
295 if (!p4d_present(*p4d))
296 goto out;
297 pud = pud_offset(p4d, address);
Andrea Arcangeli8d2afd92015-09-04 15:46:51 -0700298 if (!pud_present(*pud))
299 goto out;
300 pmd = pmd_offset(pud, address);
301 /*
302 * READ_ONCE must function as a barrier with narrower scope
303 * and it must be equivalent to:
304 * _pmd = *pmd; barrier();
305 *
306 * This is to deal with the instability (as in
307 * pmd_trans_unstable) of the pmd.
308 */
309 _pmd = READ_ONCE(*pmd);
Huang Yinga365ac02018-01-31 16:17:32 -0800310 if (pmd_none(_pmd))
Andrea Arcangeli8d2afd92015-09-04 15:46:51 -0700311 goto out;
312
313 ret = false;
Huang Yinga365ac02018-01-31 16:17:32 -0800314 if (!pmd_present(_pmd))
315 goto out;
316
Andrea Arcangeli8d2afd92015-09-04 15:46:51 -0700317 if (pmd_trans_huge(_pmd))
318 goto out;
319
320 /*
321 * the pmd is stable (as in !pmd_trans_unstable) so we can re-read it
322 * and use the standard pte_offset_map() instead of parsing _pmd.
323 */
324 pte = pte_offset_map(pmd, address);
325 /*
326 * Lockless access: we're in a wait_event so it's ok if it
327 * changes under us.
328 */
329 if (pte_none(*pte))
330 ret = true;
331 pte_unmap(pte);
332
333out:
334 return ret;
335}
336
337/*
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700338 * The locking rules involved in returning VM_FAULT_RETRY depending on
339 * FAULT_FLAG_ALLOW_RETRY, FAULT_FLAG_RETRY_NOWAIT and
340 * FAULT_FLAG_KILLABLE are not straightforward. The "Caution"
341 * recommendation in __lock_page_or_retry is not an understatement.
342 *
343 * If FAULT_FLAG_ALLOW_RETRY is set, the mmap_sem must be released
344 * before returning VM_FAULT_RETRY only if FAULT_FLAG_RETRY_NOWAIT is
345 * not set.
346 *
347 * If FAULT_FLAG_ALLOW_RETRY is set but FAULT_FLAG_KILLABLE is not
348 * set, VM_FAULT_RETRY can still be returned if and only if there are
349 * fatal_signal_pending()s, and the mmap_sem must be released before
350 * returning it.
351 */
Souptick Joarder2b740302018-08-23 17:01:36 -0700352vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason)
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700353{
Jan Kara82b0f8c2016-12-14 15:06:58 -0800354 struct mm_struct *mm = vmf->vma->vm_mm;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700355 struct userfaultfd_ctx *ctx;
356 struct userfaultfd_wait_queue uwq;
Souptick Joarder2b740302018-08-23 17:01:36 -0700357 vm_fault_t ret = VM_FAULT_SIGBUS;
Andrea Arcangelidfa37dc2015-09-04 15:47:18 -0700358 bool must_wait, return_to_userland;
Andrea Arcangeli15a77c62017-01-24 15:17:59 -0800359 long blocking_state;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700360
Andrea Arcangeli64c2b202017-06-16 14:02:37 -0700361 /*
362 * We don't do userfault handling for the final child pid update.
363 *
364 * We also don't do userfault handling during
365 * coredumping. hugetlbfs has the special
366 * follow_hugetlb_page() to skip missing pages in the
367 * FOLL_DUMP case, anon memory also checks for FOLL_DUMP with
368 * the no_page_table() helper in follow_page_mask(), but the
369 * shmem_vm_ops->fault method is invoked even during
370 * coredumping without mmap_sem and it ends up here.
371 */
372 if (current->flags & (PF_EXITING|PF_DUMPCORE))
373 goto out;
374
375 /*
376 * Coredumping runs without mmap_sem so we can only check that
377 * the mmap_sem is held, if PF_DUMPCORE was not set.
378 */
379 WARN_ON_ONCE(!rwsem_is_locked(&mm->mmap_sem));
380
Jan Kara82b0f8c2016-12-14 15:06:58 -0800381 ctx = vmf->vma->vm_userfaultfd_ctx.ctx;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700382 if (!ctx)
Andrea Arcangeliba85c702015-09-04 15:46:41 -0700383 goto out;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700384
385 BUG_ON(ctx->mm != mm);
386
387 VM_BUG_ON(reason & ~(VM_UFFD_MISSING|VM_UFFD_WP));
388 VM_BUG_ON(!(reason & VM_UFFD_MISSING) ^ !!(reason & VM_UFFD_WP));
389
Prakash Sangappa2d6d6f52017-09-06 16:23:39 -0700390 if (ctx->features & UFFD_FEATURE_SIGBUS)
391 goto out;
392
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700393 /*
394 * If it's already released don't get it. This avoids to loop
395 * in __get_user_pages if userfaultfd_release waits on the
396 * caller of handle_userfault to release the mmap_sem.
397 */
Mark Rutland6aa7de02017-10-23 14:07:29 -0700398 if (unlikely(READ_ONCE(ctx->released))) {
Andrea Arcangeli656710a2017-09-08 16:12:42 -0700399 /*
400 * Don't return VM_FAULT_SIGBUS in this case, so a non
401 * cooperative manager can close the uffd after the
402 * last UFFDIO_COPY, without risking to trigger an
403 * involuntary SIGBUS if the process was starting the
404 * userfaultfd while the userfaultfd was still armed
405 * (but after the last UFFDIO_COPY). If the uffd
406 * wasn't already closed when the userfault reached
407 * this point, that would normally be solved by
408 * userfaultfd_must_wait returning 'false'.
409 *
410 * If we were to return VM_FAULT_SIGBUS here, the non
411 * cooperative manager would be instead forced to
412 * always call UFFDIO_UNREGISTER before it can safely
413 * close the uffd.
414 */
415 ret = VM_FAULT_NOPAGE;
Andrea Arcangeliba85c702015-09-04 15:46:41 -0700416 goto out;
Andrea Arcangeli656710a2017-09-08 16:12:42 -0700417 }
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700418
419 /*
420 * Check that we can return VM_FAULT_RETRY.
421 *
422 * NOTE: it should become possible to return VM_FAULT_RETRY
423 * even if FAULT_FLAG_TRIED is set without leading to gup()
424 * -EBUSY failures, if the userfaultfd is to be extended for
425 * VM_UFFD_WP tracking and we intend to arm the userfault
426 * without first stopping userland access to the memory. For
427 * VM_UFFD_MISSING userfaults this is enough for now.
428 */
Jan Kara82b0f8c2016-12-14 15:06:58 -0800429 if (unlikely(!(vmf->flags & FAULT_FLAG_ALLOW_RETRY))) {
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700430 /*
431 * Validate the invariant that nowait must allow retry
432 * to be sure not to return SIGBUS erroneously on
433 * nowait invocations.
434 */
Jan Kara82b0f8c2016-12-14 15:06:58 -0800435 BUG_ON(vmf->flags & FAULT_FLAG_RETRY_NOWAIT);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700436#ifdef CONFIG_DEBUG_VM
437 if (printk_ratelimit()) {
438 printk(KERN_WARNING
Jan Kara82b0f8c2016-12-14 15:06:58 -0800439 "FAULT_FLAG_ALLOW_RETRY missing %x\n",
440 vmf->flags);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700441 dump_stack();
442 }
443#endif
Andrea Arcangeliba85c702015-09-04 15:46:41 -0700444 goto out;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700445 }
446
447 /*
448 * Handle nowait, not much to do other than tell it to retry
449 * and wait.
450 */
Andrea Arcangeliba85c702015-09-04 15:46:41 -0700451 ret = VM_FAULT_RETRY;
Jan Kara82b0f8c2016-12-14 15:06:58 -0800452 if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT)
Andrea Arcangeliba85c702015-09-04 15:46:41 -0700453 goto out;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700454
455 /* take the reference before dropping the mmap_sem */
456 userfaultfd_ctx_get(ctx);
457
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700458 init_waitqueue_func_entry(&uwq.wq, userfaultfd_wake_function);
459 uwq.wq.private = current;
Alexey Perevalov9d4ac932017-09-06 16:23:56 -0700460 uwq.msg = userfault_msg(vmf->address, vmf->flags, reason,
461 ctx->features);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700462 uwq.ctx = ctx;
Andrea Arcangeli15a77c62017-01-24 15:17:59 -0800463 uwq.waken = false;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700464
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -0700465 return_to_userland =
Jan Kara82b0f8c2016-12-14 15:06:58 -0800466 (vmf->flags & (FAULT_FLAG_USER|FAULT_FLAG_KILLABLE)) ==
Andrea Arcangelidfa37dc2015-09-04 15:47:18 -0700467 (FAULT_FLAG_USER|FAULT_FLAG_KILLABLE);
Andrea Arcangeli15a77c62017-01-24 15:17:59 -0800468 blocking_state = return_to_userland ? TASK_INTERRUPTIBLE :
469 TASK_KILLABLE;
Andrea Arcangelidfa37dc2015-09-04 15:47:18 -0700470
Eric Biggerscbcfa132019-07-04 15:14:39 -0700471 spin_lock_irq(&ctx->fault_pending_wqh.lock);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700472 /*
473 * After the __add_wait_queue the uwq is visible to userland
474 * through poll/read().
475 */
Andrea Arcangeli15b726e2015-09-04 15:46:44 -0700476 __add_wait_queue(&ctx->fault_pending_wqh, &uwq.wq);
477 /*
478 * The smp_mb() after __set_current_state prevents the reads
479 * following the spin_unlock to happen before the list_add in
480 * __add_wait_queue.
481 */
Andrea Arcangeli15a77c62017-01-24 15:17:59 -0800482 set_current_state(blocking_state);
Eric Biggerscbcfa132019-07-04 15:14:39 -0700483 spin_unlock_irq(&ctx->fault_pending_wqh.lock);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700484
Mike Kravetz369cd2122017-02-22 15:43:10 -0800485 if (!is_vm_hugetlb_page(vmf->vma))
486 must_wait = userfaultfd_must_wait(ctx, vmf->address, vmf->flags,
487 reason);
488 else
Punit Agrawal7868a202017-07-06 15:39:42 -0700489 must_wait = userfaultfd_huge_must_wait(ctx, vmf->vma,
490 vmf->address,
Mike Kravetz369cd2122017-02-22 15:43:10 -0800491 vmf->flags, reason);
Andrea Arcangeli8d2afd92015-09-04 15:46:51 -0700492 up_read(&mm->mmap_sem);
493
Mark Rutland6aa7de02017-10-23 14:07:29 -0700494 if (likely(must_wait && !READ_ONCE(ctx->released) &&
Andrea Arcangelidfa37dc2015-09-04 15:47:18 -0700495 (return_to_userland ? !signal_pending(current) :
496 !fatal_signal_pending(current)))) {
Linus Torvaldsa9a08842018-02-11 14:34:03 -0800497 wake_up_poll(&ctx->fd_wqh, EPOLLIN);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700498 schedule();
Andrea Arcangeliba85c702015-09-04 15:46:41 -0700499 ret |= VM_FAULT_MAJOR;
Andrea Arcangeli15a77c62017-01-24 15:17:59 -0800500
501 /*
502 * False wakeups can orginate even from rwsem before
503 * up_read() however userfaults will wait either for a
504 * targeted wakeup on the specific uwq waitqueue from
505 * wake_userfault() or for signals or for uffd
506 * release.
507 */
508 while (!READ_ONCE(uwq.waken)) {
509 /*
510 * This needs the full smp_store_mb()
511 * guarantee as the state write must be
512 * visible to other CPUs before reading
513 * uwq.waken from other CPUs.
514 */
515 set_current_state(blocking_state);
516 if (READ_ONCE(uwq.waken) ||
517 READ_ONCE(ctx->released) ||
518 (return_to_userland ? signal_pending(current) :
519 fatal_signal_pending(current)))
520 break;
521 schedule();
522 }
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700523 }
Andrea Arcangeliba85c702015-09-04 15:46:41 -0700524
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700525 __set_current_state(TASK_RUNNING);
Andrea Arcangeli15b726e2015-09-04 15:46:44 -0700526
Andrea Arcangelidfa37dc2015-09-04 15:47:18 -0700527 if (return_to_userland) {
528 if (signal_pending(current) &&
529 !fatal_signal_pending(current)) {
530 /*
531 * If we got a SIGSTOP or SIGCONT and this is
532 * a normal userland page fault, just let
533 * userland return so the signal will be
534 * handled and gdb debugging works. The page
535 * fault code immediately after we return from
536 * this function is going to release the
537 * mmap_sem and it's not depending on it
538 * (unlike gup would if we were not to return
539 * VM_FAULT_RETRY).
540 *
541 * If a fatal signal is pending we still take
542 * the streamlined VM_FAULT_RETRY failure path
543 * and there's no need to retake the mmap_sem
544 * in such case.
545 */
546 down_read(&mm->mmap_sem);
Andrea Arcangeli6bbc4a42017-03-09 16:16:28 -0800547 ret = VM_FAULT_NOPAGE;
Andrea Arcangelidfa37dc2015-09-04 15:47:18 -0700548 }
549 }
550
Andrea Arcangeli15b726e2015-09-04 15:46:44 -0700551 /*
552 * Here we race with the list_del; list_add in
553 * userfaultfd_ctx_read(), however because we don't ever run
554 * list_del_init() to refile across the two lists, the prev
555 * and next pointers will never point to self. list_add also
556 * would never let any of the two pointers to point to
557 * self. So list_empty_careful won't risk to see both pointers
558 * pointing to self at any time during the list refile. The
559 * only case where list_del_init() is called is the full
560 * removal in the wake function and there we don't re-list_add
561 * and it's fine not to block on the spinlock. The uwq on this
562 * kernel stack can be released after the list_del_init.
563 */
Ingo Molnar2055da92017-06-20 12:06:46 +0200564 if (!list_empty_careful(&uwq.wq.entry)) {
Eric Biggerscbcfa132019-07-04 15:14:39 -0700565 spin_lock_irq(&ctx->fault_pending_wqh.lock);
Andrea Arcangeli15b726e2015-09-04 15:46:44 -0700566 /*
567 * No need of list_del_init(), the uwq on the stack
568 * will be freed shortly anyway.
569 */
Ingo Molnar2055da92017-06-20 12:06:46 +0200570 list_del(&uwq.wq.entry);
Eric Biggerscbcfa132019-07-04 15:14:39 -0700571 spin_unlock_irq(&ctx->fault_pending_wqh.lock);
Andrea Arcangeliba85c702015-09-04 15:46:41 -0700572 }
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700573
574 /*
575 * ctx may go away after this if the userfault pseudo fd is
576 * already released.
577 */
578 userfaultfd_ctx_put(ctx);
579
Andrea Arcangeliba85c702015-09-04 15:46:41 -0700580out:
581 return ret;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700582}
583
Andrea Arcangeli8c9e7bb2017-03-09 16:16:54 -0800584static void userfaultfd_event_wait_completion(struct userfaultfd_ctx *ctx,
585 struct userfaultfd_wait_queue *ewq)
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -0800586{
Andrea Arcangeli0cbb4b42018-01-04 16:18:09 -0800587 struct userfaultfd_ctx *release_new_ctx;
588
Andrea Arcangeli9a69a822017-03-09 16:16:52 -0800589 if (WARN_ON_ONCE(current->flags & PF_EXITING))
590 goto out;
591
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -0800592 ewq->ctx = ctx;
593 init_waitqueue_entry(&ewq->wq, current);
Andrea Arcangeli0cbb4b42018-01-04 16:18:09 -0800594 release_new_ctx = NULL;
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -0800595
Eric Biggerscbcfa132019-07-04 15:14:39 -0700596 spin_lock_irq(&ctx->event_wqh.lock);
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -0800597 /*
598 * After the __add_wait_queue the uwq is visible to userland
599 * through poll/read().
600 */
601 __add_wait_queue(&ctx->event_wqh, &ewq->wq);
602 for (;;) {
603 set_current_state(TASK_KILLABLE);
604 if (ewq->msg.event == 0)
605 break;
Mark Rutland6aa7de02017-10-23 14:07:29 -0700606 if (READ_ONCE(ctx->released) ||
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -0800607 fatal_signal_pending(current)) {
Andrea Arcangeli384632e2017-10-03 16:15:38 -0700608 /*
609 * &ewq->wq may be queued in fork_event, but
610 * __remove_wait_queue ignores the head
611 * parameter. It would be a problem if it
612 * didn't.
613 */
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -0800614 __remove_wait_queue(&ctx->event_wqh, &ewq->wq);
Mike Rapoport7eb76d42017-03-09 16:17:09 -0800615 if (ewq->msg.event == UFFD_EVENT_FORK) {
616 struct userfaultfd_ctx *new;
617
618 new = (struct userfaultfd_ctx *)
619 (unsigned long)
620 ewq->msg.arg.reserved.reserved1;
Andrea Arcangeli0cbb4b42018-01-04 16:18:09 -0800621 release_new_ctx = new;
Mike Rapoport7eb76d42017-03-09 16:17:09 -0800622 }
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -0800623 break;
624 }
625
Eric Biggerscbcfa132019-07-04 15:14:39 -0700626 spin_unlock_irq(&ctx->event_wqh.lock);
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -0800627
Linus Torvaldsa9a08842018-02-11 14:34:03 -0800628 wake_up_poll(&ctx->fd_wqh, EPOLLIN);
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -0800629 schedule();
630
Eric Biggerscbcfa132019-07-04 15:14:39 -0700631 spin_lock_irq(&ctx->event_wqh.lock);
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -0800632 }
633 __set_current_state(TASK_RUNNING);
Eric Biggerscbcfa132019-07-04 15:14:39 -0700634 spin_unlock_irq(&ctx->event_wqh.lock);
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -0800635
Andrea Arcangeli0cbb4b42018-01-04 16:18:09 -0800636 if (release_new_ctx) {
637 struct vm_area_struct *vma;
638 struct mm_struct *mm = release_new_ctx->mm;
639
640 /* the various vma->vm_userfaultfd_ctx still points to it */
641 down_write(&mm->mmap_sem);
Andrea Arcangeli04f58662019-04-18 17:50:52 -0700642 /* no task can run (and in turn coredump) yet */
643 VM_WARN_ON(!mmget_still_valid(mm));
Andrea Arcangeli0cbb4b42018-01-04 16:18:09 -0800644 for (vma = mm->mmap; vma; vma = vma->vm_next)
Mike Rapoport31e810a2018-08-02 15:36:09 -0700645 if (vma->vm_userfaultfd_ctx.ctx == release_new_ctx) {
Andrea Arcangeli0cbb4b42018-01-04 16:18:09 -0800646 vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
Mike Rapoport31e810a2018-08-02 15:36:09 -0700647 vma->vm_flags &= ~(VM_UFFD_WP | VM_UFFD_MISSING);
648 }
Andrea Arcangeli0cbb4b42018-01-04 16:18:09 -0800649 up_write(&mm->mmap_sem);
650
651 userfaultfd_ctx_put(release_new_ctx);
652 }
653
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -0800654 /*
655 * ctx may go away after this if the userfault pseudo fd is
656 * already released.
657 */
Andrea Arcangeli9a69a822017-03-09 16:16:52 -0800658out:
Mike Rapoportdf2cc962018-06-07 17:09:25 -0700659 WRITE_ONCE(ctx->mmap_changing, false);
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -0800660 userfaultfd_ctx_put(ctx);
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -0800661}
662
663static void userfaultfd_event_complete(struct userfaultfd_ctx *ctx,
664 struct userfaultfd_wait_queue *ewq)
665{
666 ewq->msg.event = 0;
667 wake_up_locked(&ctx->event_wqh);
668 __remove_wait_queue(&ctx->event_wqh, &ewq->wq);
669}
670
Pavel Emelyanov893e26e2017-02-22 15:42:27 -0800671int dup_userfaultfd(struct vm_area_struct *vma, struct list_head *fcs)
672{
673 struct userfaultfd_ctx *ctx = NULL, *octx;
674 struct userfaultfd_fork_ctx *fctx;
675
676 octx = vma->vm_userfaultfd_ctx.ctx;
677 if (!octx || !(octx->features & UFFD_FEATURE_EVENT_FORK)) {
678 vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
679 vma->vm_flags &= ~(VM_UFFD_WP | VM_UFFD_MISSING);
680 return 0;
681 }
682
683 list_for_each_entry(fctx, fcs, list)
684 if (fctx->orig == octx) {
685 ctx = fctx->new;
686 break;
687 }
688
689 if (!ctx) {
690 fctx = kmalloc(sizeof(*fctx), GFP_KERNEL);
691 if (!fctx)
692 return -ENOMEM;
693
694 ctx = kmem_cache_alloc(userfaultfd_ctx_cachep, GFP_KERNEL);
695 if (!ctx) {
696 kfree(fctx);
697 return -ENOMEM;
698 }
699
Eric Biggersca880422018-12-28 00:34:43 -0800700 refcount_set(&ctx->refcount, 1);
Pavel Emelyanov893e26e2017-02-22 15:42:27 -0800701 ctx->flags = octx->flags;
702 ctx->state = UFFD_STATE_RUNNING;
703 ctx->features = octx->features;
704 ctx->released = false;
Mike Rapoportdf2cc962018-06-07 17:09:25 -0700705 ctx->mmap_changing = false;
Pavel Emelyanov893e26e2017-02-22 15:42:27 -0800706 ctx->mm = vma->vm_mm;
Mike Rapoport00bb31f2017-11-15 17:36:56 -0800707 mmgrab(ctx->mm);
Pavel Emelyanov893e26e2017-02-22 15:42:27 -0800708
709 userfaultfd_ctx_get(octx);
Mike Rapoportdf2cc962018-06-07 17:09:25 -0700710 WRITE_ONCE(octx->mmap_changing, true);
Pavel Emelyanov893e26e2017-02-22 15:42:27 -0800711 fctx->orig = octx;
712 fctx->new = ctx;
713 list_add_tail(&fctx->list, fcs);
714 }
715
716 vma->vm_userfaultfd_ctx.ctx = ctx;
717 return 0;
718}
719
Andrea Arcangeli8c9e7bb2017-03-09 16:16:54 -0800720static void dup_fctx(struct userfaultfd_fork_ctx *fctx)
Pavel Emelyanov893e26e2017-02-22 15:42:27 -0800721{
722 struct userfaultfd_ctx *ctx = fctx->orig;
723 struct userfaultfd_wait_queue ewq;
724
725 msg_init(&ewq.msg);
726
727 ewq.msg.event = UFFD_EVENT_FORK;
728 ewq.msg.arg.reserved.reserved1 = (unsigned long)fctx->new;
729
Andrea Arcangeli8c9e7bb2017-03-09 16:16:54 -0800730 userfaultfd_event_wait_completion(ctx, &ewq);
Pavel Emelyanov893e26e2017-02-22 15:42:27 -0800731}
732
733void dup_userfaultfd_complete(struct list_head *fcs)
734{
Pavel Emelyanov893e26e2017-02-22 15:42:27 -0800735 struct userfaultfd_fork_ctx *fctx, *n;
736
737 list_for_each_entry_safe(fctx, n, fcs, list) {
Andrea Arcangeli8c9e7bb2017-03-09 16:16:54 -0800738 dup_fctx(fctx);
Pavel Emelyanov893e26e2017-02-22 15:42:27 -0800739 list_del(&fctx->list);
740 kfree(fctx);
741 }
742}
743
Pavel Emelyanov72f87652017-02-22 15:42:34 -0800744void mremap_userfaultfd_prep(struct vm_area_struct *vma,
745 struct vm_userfaultfd_ctx *vm_ctx)
746{
747 struct userfaultfd_ctx *ctx;
748
749 ctx = vma->vm_userfaultfd_ctx.ctx;
Peter Xu3cfd22b2018-12-28 00:38:47 -0800750
751 if (!ctx)
752 return;
753
754 if (ctx->features & UFFD_FEATURE_EVENT_REMAP) {
Pavel Emelyanov72f87652017-02-22 15:42:34 -0800755 vm_ctx->ctx = ctx;
756 userfaultfd_ctx_get(ctx);
Mike Rapoportdf2cc962018-06-07 17:09:25 -0700757 WRITE_ONCE(ctx->mmap_changing, true);
Peter Xu3cfd22b2018-12-28 00:38:47 -0800758 } else {
759 /* Drop uffd context if remap feature not enabled */
760 vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
761 vma->vm_flags &= ~(VM_UFFD_WP | VM_UFFD_MISSING);
Pavel Emelyanov72f87652017-02-22 15:42:34 -0800762 }
763}
764
Andrea Arcangeli90794bf2017-02-22 15:42:37 -0800765void mremap_userfaultfd_complete(struct vm_userfaultfd_ctx *vm_ctx,
Pavel Emelyanov72f87652017-02-22 15:42:34 -0800766 unsigned long from, unsigned long to,
767 unsigned long len)
768{
Andrea Arcangeli90794bf2017-02-22 15:42:37 -0800769 struct userfaultfd_ctx *ctx = vm_ctx->ctx;
Pavel Emelyanov72f87652017-02-22 15:42:34 -0800770 struct userfaultfd_wait_queue ewq;
771
772 if (!ctx)
773 return;
774
775 if (to & ~PAGE_MASK) {
776 userfaultfd_ctx_put(ctx);
777 return;
778 }
779
780 msg_init(&ewq.msg);
781
782 ewq.msg.event = UFFD_EVENT_REMAP;
783 ewq.msg.arg.remap.from = from;
784 ewq.msg.arg.remap.to = to;
785 ewq.msg.arg.remap.len = len;
786
787 userfaultfd_event_wait_completion(ctx, &ewq);
788}
789
Andrea Arcangeli70ccb922017-03-09 16:17:11 -0800790bool userfaultfd_remove(struct vm_area_struct *vma,
Mike Rapoportd8119142017-02-24 14:56:02 -0800791 unsigned long start, unsigned long end)
Pavel Emelyanov05ce7722017-02-22 15:42:40 -0800792{
793 struct mm_struct *mm = vma->vm_mm;
794 struct userfaultfd_ctx *ctx;
795 struct userfaultfd_wait_queue ewq;
796
797 ctx = vma->vm_userfaultfd_ctx.ctx;
Mike Rapoportd8119142017-02-24 14:56:02 -0800798 if (!ctx || !(ctx->features & UFFD_FEATURE_EVENT_REMOVE))
Andrea Arcangeli70ccb922017-03-09 16:17:11 -0800799 return true;
Pavel Emelyanov05ce7722017-02-22 15:42:40 -0800800
801 userfaultfd_ctx_get(ctx);
Mike Rapoportdf2cc962018-06-07 17:09:25 -0700802 WRITE_ONCE(ctx->mmap_changing, true);
Pavel Emelyanov05ce7722017-02-22 15:42:40 -0800803 up_read(&mm->mmap_sem);
804
Pavel Emelyanov05ce7722017-02-22 15:42:40 -0800805 msg_init(&ewq.msg);
806
Mike Rapoportd8119142017-02-24 14:56:02 -0800807 ewq.msg.event = UFFD_EVENT_REMOVE;
808 ewq.msg.arg.remove.start = start;
809 ewq.msg.arg.remove.end = end;
Pavel Emelyanov05ce7722017-02-22 15:42:40 -0800810
811 userfaultfd_event_wait_completion(ctx, &ewq);
812
Andrea Arcangeli70ccb922017-03-09 16:17:11 -0800813 return false;
Pavel Emelyanov05ce7722017-02-22 15:42:40 -0800814}
815
Mike Rapoport897ab3e2017-02-24 14:58:22 -0800816static bool has_unmap_ctx(struct userfaultfd_ctx *ctx, struct list_head *unmaps,
817 unsigned long start, unsigned long end)
818{
819 struct userfaultfd_unmap_ctx *unmap_ctx;
820
821 list_for_each_entry(unmap_ctx, unmaps, list)
822 if (unmap_ctx->ctx == ctx && unmap_ctx->start == start &&
823 unmap_ctx->end == end)
824 return true;
825
826 return false;
827}
828
829int userfaultfd_unmap_prep(struct vm_area_struct *vma,
830 unsigned long start, unsigned long end,
831 struct list_head *unmaps)
832{
833 for ( ; vma && vma->vm_start < end; vma = vma->vm_next) {
834 struct userfaultfd_unmap_ctx *unmap_ctx;
835 struct userfaultfd_ctx *ctx = vma->vm_userfaultfd_ctx.ctx;
836
837 if (!ctx || !(ctx->features & UFFD_FEATURE_EVENT_UNMAP) ||
838 has_unmap_ctx(ctx, unmaps, start, end))
839 continue;
840
841 unmap_ctx = kzalloc(sizeof(*unmap_ctx), GFP_KERNEL);
842 if (!unmap_ctx)
843 return -ENOMEM;
844
845 userfaultfd_ctx_get(ctx);
Mike Rapoportdf2cc962018-06-07 17:09:25 -0700846 WRITE_ONCE(ctx->mmap_changing, true);
Mike Rapoport897ab3e2017-02-24 14:58:22 -0800847 unmap_ctx->ctx = ctx;
848 unmap_ctx->start = start;
849 unmap_ctx->end = end;
850 list_add_tail(&unmap_ctx->list, unmaps);
851 }
852
853 return 0;
854}
855
856void userfaultfd_unmap_complete(struct mm_struct *mm, struct list_head *uf)
857{
858 struct userfaultfd_unmap_ctx *ctx, *n;
859 struct userfaultfd_wait_queue ewq;
860
861 list_for_each_entry_safe(ctx, n, uf, list) {
862 msg_init(&ewq.msg);
863
864 ewq.msg.event = UFFD_EVENT_UNMAP;
865 ewq.msg.arg.remove.start = ctx->start;
866 ewq.msg.arg.remove.end = ctx->end;
867
868 userfaultfd_event_wait_completion(ctx->ctx, &ewq);
869
870 list_del(&ctx->list);
871 kfree(ctx);
872 }
873}
874
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700875static int userfaultfd_release(struct inode *inode, struct file *file)
876{
877 struct userfaultfd_ctx *ctx = file->private_data;
878 struct mm_struct *mm = ctx->mm;
879 struct vm_area_struct *vma, *prev;
880 /* len == 0 means wake all */
881 struct userfaultfd_wake_range range = { .len = 0, };
882 unsigned long new_flags;
883
Mark Rutland6aa7de02017-10-23 14:07:29 -0700884 WRITE_ONCE(ctx->released, true);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700885
Oleg Nesterovd2005e32016-05-20 16:58:36 -0700886 if (!mmget_not_zero(mm))
887 goto wakeup;
888
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700889 /*
890 * Flush page faults out of all CPUs. NOTE: all page faults
891 * must be retried without returning VM_FAULT_SIGBUS if
892 * userfaultfd_ctx_get() succeeds but vma->vma_userfault_ctx
893 * changes while handle_userfault released the mmap_sem. So
894 * it's critical that released is set to true (above), before
895 * taking the mmap_sem for writing.
896 */
897 down_write(&mm->mmap_sem);
Andrea Arcangeli04f58662019-04-18 17:50:52 -0700898 if (!mmget_still_valid(mm))
899 goto skip_mm;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700900 prev = NULL;
901 for (vma = mm->mmap; vma; vma = vma->vm_next) {
902 cond_resched();
903 BUG_ON(!!vma->vm_userfaultfd_ctx.ctx ^
904 !!(vma->vm_flags & (VM_UFFD_MISSING | VM_UFFD_WP)));
905 if (vma->vm_userfaultfd_ctx.ctx != ctx) {
906 prev = vma;
907 continue;
908 }
909 new_flags = vma->vm_flags & ~(VM_UFFD_MISSING | VM_UFFD_WP);
910 prev = vma_merge(mm, prev, vma->vm_start, vma->vm_end,
911 new_flags, vma->anon_vma,
912 vma->vm_file, vma->vm_pgoff,
913 vma_policy(vma),
914 NULL_VM_UFFD_CTX);
915 if (prev)
916 vma = prev;
917 else
918 prev = vma;
919 vma->vm_flags = new_flags;
920 vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
921 }
Andrea Arcangeli04f58662019-04-18 17:50:52 -0700922skip_mm:
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700923 up_write(&mm->mmap_sem);
Oleg Nesterovd2005e32016-05-20 16:58:36 -0700924 mmput(mm);
925wakeup:
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700926 /*
Andrea Arcangeli15b726e2015-09-04 15:46:44 -0700927 * After no new page faults can wait on this fault_*wqh, flush
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700928 * the last page faults that may have been already waiting on
Andrea Arcangeli15b726e2015-09-04 15:46:44 -0700929 * the fault_*wqh.
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700930 */
Eric Biggerscbcfa132019-07-04 15:14:39 -0700931 spin_lock_irq(&ctx->fault_pending_wqh.lock);
Andrea Arcangeliac5be6b2015-09-22 14:58:49 -0700932 __wake_up_locked_key(&ctx->fault_pending_wqh, TASK_NORMAL, &range);
Matthew Wilcoxc430d1e2018-08-21 21:56:30 -0700933 __wake_up(&ctx->fault_wqh, TASK_NORMAL, 1, &range);
Eric Biggerscbcfa132019-07-04 15:14:39 -0700934 spin_unlock_irq(&ctx->fault_pending_wqh.lock);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700935
Mike Rapoport5a18b642017-08-02 13:32:24 -0700936 /* Flush pending events that may still wait on event_wqh */
937 wake_up_all(&ctx->event_wqh);
938
Linus Torvaldsa9a08842018-02-11 14:34:03 -0800939 wake_up_poll(&ctx->fd_wqh, EPOLLHUP);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700940 userfaultfd_ctx_put(ctx);
941 return 0;
942}
943
Andrea Arcangeli15b726e2015-09-04 15:46:44 -0700944/* fault_pending_wqh.lock must be hold by the caller */
Pavel Emelyanov6dcc27f2017-02-22 15:42:18 -0800945static inline struct userfaultfd_wait_queue *find_userfault_in(
946 wait_queue_head_t *wqh)
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700947{
Ingo Molnarac6424b2017-06-20 12:06:13 +0200948 wait_queue_entry_t *wq;
Andrea Arcangeli15b726e2015-09-04 15:46:44 -0700949 struct userfaultfd_wait_queue *uwq;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700950
Lance Roy456a7372018-10-04 23:45:44 -0700951 lockdep_assert_held(&wqh->lock);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700952
Andrea Arcangeli15b726e2015-09-04 15:46:44 -0700953 uwq = NULL;
Pavel Emelyanov6dcc27f2017-02-22 15:42:18 -0800954 if (!waitqueue_active(wqh))
Andrea Arcangeli15b726e2015-09-04 15:46:44 -0700955 goto out;
956 /* walk in reverse to provide FIFO behavior to read userfaults */
Ingo Molnar2055da92017-06-20 12:06:46 +0200957 wq = list_last_entry(&wqh->head, typeof(*wq), entry);
Andrea Arcangeli15b726e2015-09-04 15:46:44 -0700958 uwq = container_of(wq, struct userfaultfd_wait_queue, wq);
959out:
960 return uwq;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700961}
962
Pavel Emelyanov6dcc27f2017-02-22 15:42:18 -0800963static inline struct userfaultfd_wait_queue *find_userfault(
964 struct userfaultfd_ctx *ctx)
965{
966 return find_userfault_in(&ctx->fault_pending_wqh);
967}
968
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -0800969static inline struct userfaultfd_wait_queue *find_userfault_evt(
970 struct userfaultfd_ctx *ctx)
971{
972 return find_userfault_in(&ctx->event_wqh);
973}
974
Al Viro076ccb72017-07-03 01:02:18 -0400975static __poll_t userfaultfd_poll(struct file *file, poll_table *wait)
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700976{
977 struct userfaultfd_ctx *ctx = file->private_data;
Al Viro076ccb72017-07-03 01:02:18 -0400978 __poll_t ret;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700979
980 poll_wait(file, &ctx->fd_wqh, wait);
981
982 switch (ctx->state) {
983 case UFFD_STATE_WAIT_API:
Linus Torvaldsa9a08842018-02-11 14:34:03 -0800984 return EPOLLERR;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700985 case UFFD_STATE_RUNNING:
Andrea Arcangeliba85c702015-09-04 15:46:41 -0700986 /*
987 * poll() never guarantees that read won't block.
988 * userfaults can be waken before they're read().
989 */
990 if (unlikely(!(file->f_flags & O_NONBLOCK)))
Linus Torvaldsa9a08842018-02-11 14:34:03 -0800991 return EPOLLERR;
Andrea Arcangeli15b726e2015-09-04 15:46:44 -0700992 /*
993 * lockless access to see if there are pending faults
994 * __pollwait last action is the add_wait_queue but
995 * the spin_unlock would allow the waitqueue_active to
996 * pass above the actual list_add inside
997 * add_wait_queue critical section. So use a full
998 * memory barrier to serialize the list_add write of
999 * add_wait_queue() with the waitqueue_active read
1000 * below.
1001 */
1002 ret = 0;
1003 smp_mb();
1004 if (waitqueue_active(&ctx->fault_pending_wqh))
Linus Torvaldsa9a08842018-02-11 14:34:03 -08001005 ret = EPOLLIN;
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -08001006 else if (waitqueue_active(&ctx->event_wqh))
Linus Torvaldsa9a08842018-02-11 14:34:03 -08001007 ret = EPOLLIN;
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -08001008
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001009 return ret;
1010 default:
Andrea Arcangeli84749012017-02-22 15:42:12 -08001011 WARN_ON_ONCE(1);
Linus Torvaldsa9a08842018-02-11 14:34:03 -08001012 return EPOLLERR;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001013 }
1014}
1015
Pavel Emelyanov893e26e2017-02-22 15:42:27 -08001016static const struct file_operations userfaultfd_fops;
1017
1018static int resolve_userfault_fork(struct userfaultfd_ctx *ctx,
1019 struct userfaultfd_ctx *new,
1020 struct uffd_msg *msg)
1021{
1022 int fd;
Pavel Emelyanov893e26e2017-02-22 15:42:27 -08001023
Eric Biggers284cd242018-01-31 16:19:48 -08001024 fd = anon_inode_getfd("[userfaultfd]", &userfaultfd_fops, new,
1025 O_RDWR | (new->flags & UFFD_SHARED_FCNTL_FLAGS));
Pavel Emelyanov893e26e2017-02-22 15:42:27 -08001026 if (fd < 0)
1027 return fd;
1028
Pavel Emelyanov893e26e2017-02-22 15:42:27 -08001029 msg->arg.reserved.reserved1 = 0;
1030 msg->arg.fork.ufd = fd;
Pavel Emelyanov893e26e2017-02-22 15:42:27 -08001031 return 0;
1032}
1033
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001034static ssize_t userfaultfd_ctx_read(struct userfaultfd_ctx *ctx, int no_wait,
Andrea Arcangelia9b85f92015-09-04 15:46:37 -07001035 struct uffd_msg *msg)
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001036{
1037 ssize_t ret;
1038 DECLARE_WAITQUEUE(wait, current);
Andrea Arcangeli15b726e2015-09-04 15:46:44 -07001039 struct userfaultfd_wait_queue *uwq;
Pavel Emelyanov893e26e2017-02-22 15:42:27 -08001040 /*
1041 * Handling fork event requires sleeping operations, so
1042 * we drop the event_wqh lock, then do these ops, then
1043 * lock it back and wake up the waiter. While the lock is
1044 * dropped the ewq may go away so we keep track of it
1045 * carefully.
1046 */
1047 LIST_HEAD(fork_event);
1048 struct userfaultfd_ctx *fork_nctx = NULL;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001049
Andrea Arcangeli15b726e2015-09-04 15:46:44 -07001050 /* always take the fd_wqh lock before the fault_pending_wqh lock */
Christoph Hellwigae62c162018-10-26 15:02:19 -07001051 spin_lock_irq(&ctx->fd_wqh.lock);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001052 __add_wait_queue(&ctx->fd_wqh, &wait);
1053 for (;;) {
1054 set_current_state(TASK_INTERRUPTIBLE);
Andrea Arcangeli15b726e2015-09-04 15:46:44 -07001055 spin_lock(&ctx->fault_pending_wqh.lock);
1056 uwq = find_userfault(ctx);
1057 if (uwq) {
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001058 /*
Andrea Arcangeli2c5b7e12015-09-04 15:47:23 -07001059 * Use a seqcount to repeat the lockless check
1060 * in wake_userfault() to avoid missing
1061 * wakeups because during the refile both
1062 * waitqueue could become empty if this is the
1063 * only userfault.
1064 */
1065 write_seqcount_begin(&ctx->refile_seq);
1066
1067 /*
Andrea Arcangeli15b726e2015-09-04 15:46:44 -07001068 * The fault_pending_wqh.lock prevents the uwq
1069 * to disappear from under us.
1070 *
1071 * Refile this userfault from
1072 * fault_pending_wqh to fault_wqh, it's not
1073 * pending anymore after we read it.
1074 *
1075 * Use list_del() by hand (as
1076 * userfaultfd_wake_function also uses
1077 * list_del_init() by hand) to be sure nobody
1078 * changes __remove_wait_queue() to use
1079 * list_del_init() in turn breaking the
1080 * !list_empty_careful() check in
Ingo Molnar2055da92017-06-20 12:06:46 +02001081 * handle_userfault(). The uwq->wq.head list
Andrea Arcangeli15b726e2015-09-04 15:46:44 -07001082 * must never be empty at any time during the
1083 * refile, or the waitqueue could disappear
1084 * from under us. The "wait_queue_head_t"
1085 * parameter of __remove_wait_queue() is unused
1086 * anyway.
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001087 */
Ingo Molnar2055da92017-06-20 12:06:46 +02001088 list_del(&uwq->wq.entry);
Matthew Wilcoxc430d1e2018-08-21 21:56:30 -07001089 add_wait_queue(&ctx->fault_wqh, &uwq->wq);
Andrea Arcangeli15b726e2015-09-04 15:46:44 -07001090
Andrea Arcangeli2c5b7e12015-09-04 15:47:23 -07001091 write_seqcount_end(&ctx->refile_seq);
1092
Andrea Arcangelia9b85f92015-09-04 15:46:37 -07001093 /* careful to always initialize msg if ret == 0 */
1094 *msg = uwq->msg;
Andrea Arcangeli15b726e2015-09-04 15:46:44 -07001095 spin_unlock(&ctx->fault_pending_wqh.lock);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001096 ret = 0;
1097 break;
1098 }
Andrea Arcangeli15b726e2015-09-04 15:46:44 -07001099 spin_unlock(&ctx->fault_pending_wqh.lock);
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -08001100
1101 spin_lock(&ctx->event_wqh.lock);
1102 uwq = find_userfault_evt(ctx);
1103 if (uwq) {
1104 *msg = uwq->msg;
1105
Pavel Emelyanov893e26e2017-02-22 15:42:27 -08001106 if (uwq->msg.event == UFFD_EVENT_FORK) {
1107 fork_nctx = (struct userfaultfd_ctx *)
1108 (unsigned long)
1109 uwq->msg.arg.reserved.reserved1;
Ingo Molnar2055da92017-06-20 12:06:46 +02001110 list_move(&uwq->wq.entry, &fork_event);
Andrea Arcangeli384632e2017-10-03 16:15:38 -07001111 /*
1112 * fork_nctx can be freed as soon as
1113 * we drop the lock, unless we take a
1114 * reference on it.
1115 */
1116 userfaultfd_ctx_get(fork_nctx);
Pavel Emelyanov893e26e2017-02-22 15:42:27 -08001117 spin_unlock(&ctx->event_wqh.lock);
1118 ret = 0;
1119 break;
1120 }
1121
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -08001122 userfaultfd_event_complete(ctx, uwq);
1123 spin_unlock(&ctx->event_wqh.lock);
1124 ret = 0;
1125 break;
1126 }
1127 spin_unlock(&ctx->event_wqh.lock);
1128
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001129 if (signal_pending(current)) {
1130 ret = -ERESTARTSYS;
1131 break;
1132 }
1133 if (no_wait) {
1134 ret = -EAGAIN;
1135 break;
1136 }
Christoph Hellwigae62c162018-10-26 15:02:19 -07001137 spin_unlock_irq(&ctx->fd_wqh.lock);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001138 schedule();
Christoph Hellwigae62c162018-10-26 15:02:19 -07001139 spin_lock_irq(&ctx->fd_wqh.lock);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001140 }
1141 __remove_wait_queue(&ctx->fd_wqh, &wait);
1142 __set_current_state(TASK_RUNNING);
Christoph Hellwigae62c162018-10-26 15:02:19 -07001143 spin_unlock_irq(&ctx->fd_wqh.lock);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001144
Pavel Emelyanov893e26e2017-02-22 15:42:27 -08001145 if (!ret && msg->event == UFFD_EVENT_FORK) {
1146 ret = resolve_userfault_fork(ctx, fork_nctx, msg);
Eric Biggerscbcfa132019-07-04 15:14:39 -07001147 spin_lock_irq(&ctx->event_wqh.lock);
Andrea Arcangeli384632e2017-10-03 16:15:38 -07001148 if (!list_empty(&fork_event)) {
1149 /*
1150 * The fork thread didn't abort, so we can
1151 * drop the temporary refcount.
1152 */
1153 userfaultfd_ctx_put(fork_nctx);
Pavel Emelyanov893e26e2017-02-22 15:42:27 -08001154
Andrea Arcangeli384632e2017-10-03 16:15:38 -07001155 uwq = list_first_entry(&fork_event,
1156 typeof(*uwq),
1157 wq.entry);
1158 /*
1159 * If fork_event list wasn't empty and in turn
1160 * the event wasn't already released by fork
1161 * (the event is allocated on fork kernel
1162 * stack), put the event back to its place in
1163 * the event_wq. fork_event head will be freed
1164 * as soon as we return so the event cannot
1165 * stay queued there no matter the current
1166 * "ret" value.
1167 */
1168 list_del(&uwq->wq.entry);
1169 __add_wait_queue(&ctx->event_wqh, &uwq->wq);
1170
1171 /*
1172 * Leave the event in the waitqueue and report
1173 * error to userland if we failed to resolve
1174 * the userfault fork.
1175 */
1176 if (likely(!ret))
Pavel Emelyanov893e26e2017-02-22 15:42:27 -08001177 userfaultfd_event_complete(ctx, uwq);
Andrea Arcangeli384632e2017-10-03 16:15:38 -07001178 } else {
1179 /*
1180 * Here the fork thread aborted and the
1181 * refcount from the fork thread on fork_nctx
1182 * has already been released. We still hold
1183 * the reference we took before releasing the
1184 * lock above. If resolve_userfault_fork
1185 * failed we've to drop it because the
1186 * fork_nctx has to be freed in such case. If
1187 * it succeeded we'll hold it because the new
1188 * uffd references it.
1189 */
1190 if (ret)
1191 userfaultfd_ctx_put(fork_nctx);
Pavel Emelyanov893e26e2017-02-22 15:42:27 -08001192 }
Eric Biggerscbcfa132019-07-04 15:14:39 -07001193 spin_unlock_irq(&ctx->event_wqh.lock);
Pavel Emelyanov893e26e2017-02-22 15:42:27 -08001194 }
1195
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001196 return ret;
1197}
1198
1199static ssize_t userfaultfd_read(struct file *file, char __user *buf,
1200 size_t count, loff_t *ppos)
1201{
1202 struct userfaultfd_ctx *ctx = file->private_data;
1203 ssize_t _ret, ret = 0;
Andrea Arcangelia9b85f92015-09-04 15:46:37 -07001204 struct uffd_msg msg;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001205 int no_wait = file->f_flags & O_NONBLOCK;
1206
1207 if (ctx->state == UFFD_STATE_WAIT_API)
1208 return -EINVAL;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001209
1210 for (;;) {
Andrea Arcangelia9b85f92015-09-04 15:46:37 -07001211 if (count < sizeof(msg))
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001212 return ret ? ret : -EINVAL;
Andrea Arcangelia9b85f92015-09-04 15:46:37 -07001213 _ret = userfaultfd_ctx_read(ctx, no_wait, &msg);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001214 if (_ret < 0)
1215 return ret ? ret : _ret;
Andrea Arcangelia9b85f92015-09-04 15:46:37 -07001216 if (copy_to_user((__u64 __user *) buf, &msg, sizeof(msg)))
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001217 return ret ? ret : -EFAULT;
Andrea Arcangelia9b85f92015-09-04 15:46:37 -07001218 ret += sizeof(msg);
1219 buf += sizeof(msg);
1220 count -= sizeof(msg);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001221 /*
1222 * Allow to read more than one fault at time but only
1223 * block if waiting for the very first one.
1224 */
1225 no_wait = O_NONBLOCK;
1226 }
1227}
1228
1229static void __wake_userfault(struct userfaultfd_ctx *ctx,
1230 struct userfaultfd_wake_range *range)
1231{
Eric Biggerscbcfa132019-07-04 15:14:39 -07001232 spin_lock_irq(&ctx->fault_pending_wqh.lock);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001233 /* wake all in the range and autoremove */
Andrea Arcangeli15b726e2015-09-04 15:46:44 -07001234 if (waitqueue_active(&ctx->fault_pending_wqh))
Andrea Arcangeliac5be6b2015-09-22 14:58:49 -07001235 __wake_up_locked_key(&ctx->fault_pending_wqh, TASK_NORMAL,
Andrea Arcangeli15b726e2015-09-04 15:46:44 -07001236 range);
1237 if (waitqueue_active(&ctx->fault_wqh))
Matthew Wilcoxc430d1e2018-08-21 21:56:30 -07001238 __wake_up(&ctx->fault_wqh, TASK_NORMAL, 1, range);
Eric Biggerscbcfa132019-07-04 15:14:39 -07001239 spin_unlock_irq(&ctx->fault_pending_wqh.lock);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001240}
1241
1242static __always_inline void wake_userfault(struct userfaultfd_ctx *ctx,
1243 struct userfaultfd_wake_range *range)
1244{
Andrea Arcangeli2c5b7e12015-09-04 15:47:23 -07001245 unsigned seq;
1246 bool need_wakeup;
1247
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001248 /*
1249 * To be sure waitqueue_active() is not reordered by the CPU
1250 * before the pagetable update, use an explicit SMP memory
1251 * barrier here. PT lock release or up_read(mmap_sem) still
1252 * have release semantics that can allow the
1253 * waitqueue_active() to be reordered before the pte update.
1254 */
1255 smp_mb();
1256
1257 /*
1258 * Use waitqueue_active because it's very frequent to
1259 * change the address space atomically even if there are no
1260 * userfaults yet. So we take the spinlock only when we're
1261 * sure we've userfaults to wake.
1262 */
Andrea Arcangeli2c5b7e12015-09-04 15:47:23 -07001263 do {
1264 seq = read_seqcount_begin(&ctx->refile_seq);
1265 need_wakeup = waitqueue_active(&ctx->fault_pending_wqh) ||
1266 waitqueue_active(&ctx->fault_wqh);
1267 cond_resched();
1268 } while (read_seqcount_retry(&ctx->refile_seq, seq));
1269 if (need_wakeup)
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001270 __wake_userfault(ctx, range);
1271}
1272
1273static __always_inline int validate_range(struct mm_struct *mm,
1274 __u64 start, __u64 len)
1275{
1276 __u64 task_size = mm->task_size;
1277
1278 if (start & ~PAGE_MASK)
1279 return -EINVAL;
1280 if (len & ~PAGE_MASK)
1281 return -EINVAL;
1282 if (!len)
1283 return -EINVAL;
1284 if (start < mmap_min_addr)
1285 return -EINVAL;
1286 if (start >= task_size)
1287 return -EINVAL;
1288 if (len > task_size - start)
1289 return -EINVAL;
1290 return 0;
1291}
1292
Mike Rapoportba6907d2017-02-22 15:43:22 -08001293static inline bool vma_can_userfault(struct vm_area_struct *vma)
1294{
Mike Rapoportcac67322017-02-22 15:43:40 -08001295 return vma_is_anonymous(vma) || is_vm_hugetlb_page(vma) ||
1296 vma_is_shmem(vma);
Mike Rapoportba6907d2017-02-22 15:43:22 -08001297}
1298
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001299static int userfaultfd_register(struct userfaultfd_ctx *ctx,
1300 unsigned long arg)
1301{
1302 struct mm_struct *mm = ctx->mm;
1303 struct vm_area_struct *vma, *prev, *cur;
1304 int ret;
1305 struct uffdio_register uffdio_register;
1306 struct uffdio_register __user *user_uffdio_register;
1307 unsigned long vm_flags, new_flags;
1308 bool found;
Mike Rapoportce53e8e2017-09-06 16:23:12 -07001309 bool basic_ioctls;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001310 unsigned long start, end, vma_end;
1311
1312 user_uffdio_register = (struct uffdio_register __user *) arg;
1313
1314 ret = -EFAULT;
1315 if (copy_from_user(&uffdio_register, user_uffdio_register,
1316 sizeof(uffdio_register)-sizeof(__u64)))
1317 goto out;
1318
1319 ret = -EINVAL;
1320 if (!uffdio_register.mode)
1321 goto out;
1322 if (uffdio_register.mode & ~(UFFDIO_REGISTER_MODE_MISSING|
1323 UFFDIO_REGISTER_MODE_WP))
1324 goto out;
1325 vm_flags = 0;
1326 if (uffdio_register.mode & UFFDIO_REGISTER_MODE_MISSING)
1327 vm_flags |= VM_UFFD_MISSING;
1328 if (uffdio_register.mode & UFFDIO_REGISTER_MODE_WP) {
1329 vm_flags |= VM_UFFD_WP;
1330 /*
1331 * FIXME: remove the below error constraint by
1332 * implementing the wprotect tracking mode.
1333 */
1334 ret = -EINVAL;
1335 goto out;
1336 }
1337
1338 ret = validate_range(mm, uffdio_register.range.start,
1339 uffdio_register.range.len);
1340 if (ret)
1341 goto out;
1342
1343 start = uffdio_register.range.start;
1344 end = start + uffdio_register.range.len;
1345
Oleg Nesterovd2005e32016-05-20 16:58:36 -07001346 ret = -ENOMEM;
1347 if (!mmget_not_zero(mm))
1348 goto out;
1349
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001350 down_write(&mm->mmap_sem);
Andrea Arcangeli04f58662019-04-18 17:50:52 -07001351 if (!mmget_still_valid(mm))
1352 goto out_unlock;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001353 vma = find_vma_prev(mm, start, &prev);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001354 if (!vma)
1355 goto out_unlock;
1356
1357 /* check that there's at least one vma in the range */
1358 ret = -EINVAL;
1359 if (vma->vm_start >= end)
1360 goto out_unlock;
1361
1362 /*
Mike Kravetzcab350a2017-02-22 15:43:04 -08001363 * If the first vma contains huge pages, make sure start address
1364 * is aligned to huge page size.
1365 */
1366 if (is_vm_hugetlb_page(vma)) {
1367 unsigned long vma_hpagesize = vma_kernel_pagesize(vma);
1368
1369 if (start & (vma_hpagesize - 1))
1370 goto out_unlock;
1371 }
1372
1373 /*
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001374 * Search for not compatible vmas.
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001375 */
1376 found = false;
Mike Rapoportce53e8e2017-09-06 16:23:12 -07001377 basic_ioctls = false;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001378 for (cur = vma; cur && cur->vm_start < end; cur = cur->vm_next) {
1379 cond_resched();
1380
1381 BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^
1382 !!(cur->vm_flags & (VM_UFFD_MISSING | VM_UFFD_WP)));
1383
1384 /* check not compatible vmas */
1385 ret = -EINVAL;
Mike Rapoportba6907d2017-02-22 15:43:22 -08001386 if (!vma_can_userfault(cur))
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001387 goto out_unlock;
Andrea Arcangeli29ec90662018-11-30 14:09:32 -08001388
1389 /*
1390 * UFFDIO_COPY will fill file holes even without
1391 * PROT_WRITE. This check enforces that if this is a
1392 * MAP_SHARED, the process has write permission to the backing
1393 * file. If VM_MAYWRITE is set it also enforces that on a
1394 * MAP_SHARED vma: there is no F_WRITE_SEAL and no further
1395 * F_WRITE_SEAL can be taken until the vma is destroyed.
1396 */
1397 ret = -EPERM;
1398 if (unlikely(!(cur->vm_flags & VM_MAYWRITE)))
1399 goto out_unlock;
1400
Mike Kravetzcab350a2017-02-22 15:43:04 -08001401 /*
1402 * If this vma contains ending address, and huge pages
1403 * check alignment.
1404 */
1405 if (is_vm_hugetlb_page(cur) && end <= cur->vm_end &&
1406 end > cur->vm_start) {
1407 unsigned long vma_hpagesize = vma_kernel_pagesize(cur);
1408
1409 ret = -EINVAL;
1410
1411 if (end & (vma_hpagesize - 1))
1412 goto out_unlock;
1413 }
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001414
1415 /*
1416 * Check that this vma isn't already owned by a
1417 * different userfaultfd. We can't allow more than one
1418 * userfaultfd to own a single vma simultaneously or we
1419 * wouldn't know which one to deliver the userfaults to.
1420 */
1421 ret = -EBUSY;
1422 if (cur->vm_userfaultfd_ctx.ctx &&
1423 cur->vm_userfaultfd_ctx.ctx != ctx)
1424 goto out_unlock;
1425
Mike Kravetzcab350a2017-02-22 15:43:04 -08001426 /*
1427 * Note vmas containing huge pages
1428 */
Mike Rapoportce53e8e2017-09-06 16:23:12 -07001429 if (is_vm_hugetlb_page(cur))
1430 basic_ioctls = true;
Mike Kravetzcab350a2017-02-22 15:43:04 -08001431
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001432 found = true;
1433 }
1434 BUG_ON(!found);
1435
1436 if (vma->vm_start < start)
1437 prev = vma;
1438
1439 ret = 0;
1440 do {
1441 cond_resched();
1442
Mike Rapoportba6907d2017-02-22 15:43:22 -08001443 BUG_ON(!vma_can_userfault(vma));
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001444 BUG_ON(vma->vm_userfaultfd_ctx.ctx &&
1445 vma->vm_userfaultfd_ctx.ctx != ctx);
Andrea Arcangeli29ec90662018-11-30 14:09:32 -08001446 WARN_ON(!(vma->vm_flags & VM_MAYWRITE));
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001447
1448 /*
1449 * Nothing to do: this vma is already registered into this
1450 * userfaultfd and with the right tracking mode too.
1451 */
1452 if (vma->vm_userfaultfd_ctx.ctx == ctx &&
1453 (vma->vm_flags & vm_flags) == vm_flags)
1454 goto skip;
1455
1456 if (vma->vm_start > start)
1457 start = vma->vm_start;
1458 vma_end = min(end, vma->vm_end);
1459
1460 new_flags = (vma->vm_flags & ~vm_flags) | vm_flags;
1461 prev = vma_merge(mm, prev, start, vma_end, new_flags,
1462 vma->anon_vma, vma->vm_file, vma->vm_pgoff,
1463 vma_policy(vma),
1464 ((struct vm_userfaultfd_ctx){ ctx }));
1465 if (prev) {
1466 vma = prev;
1467 goto next;
1468 }
1469 if (vma->vm_start < start) {
1470 ret = split_vma(mm, vma, start, 1);
1471 if (ret)
1472 break;
1473 }
1474 if (vma->vm_end > end) {
1475 ret = split_vma(mm, vma, end, 0);
1476 if (ret)
1477 break;
1478 }
1479 next:
1480 /*
1481 * In the vma_merge() successful mprotect-like case 8:
1482 * the next vma was merged into the current one and
1483 * the current one has not been updated yet.
1484 */
1485 vma->vm_flags = new_flags;
1486 vma->vm_userfaultfd_ctx.ctx = ctx;
1487
1488 skip:
1489 prev = vma;
1490 start = vma->vm_end;
1491 vma = vma->vm_next;
1492 } while (vma && vma->vm_start < end);
1493out_unlock:
1494 up_write(&mm->mmap_sem);
Oleg Nesterovd2005e32016-05-20 16:58:36 -07001495 mmput(mm);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001496 if (!ret) {
1497 /*
1498 * Now that we scanned all vmas we can already tell
1499 * userland which ioctls methods are guaranteed to
1500 * succeed on this range.
1501 */
Mike Rapoportce53e8e2017-09-06 16:23:12 -07001502 if (put_user(basic_ioctls ? UFFD_API_RANGE_IOCTLS_BASIC :
Mike Kravetzcab350a2017-02-22 15:43:04 -08001503 UFFD_API_RANGE_IOCTLS,
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001504 &user_uffdio_register->ioctls))
1505 ret = -EFAULT;
1506 }
1507out:
1508 return ret;
1509}
1510
1511static int userfaultfd_unregister(struct userfaultfd_ctx *ctx,
1512 unsigned long arg)
1513{
1514 struct mm_struct *mm = ctx->mm;
1515 struct vm_area_struct *vma, *prev, *cur;
1516 int ret;
1517 struct uffdio_range uffdio_unregister;
1518 unsigned long new_flags;
1519 bool found;
1520 unsigned long start, end, vma_end;
1521 const void __user *buf = (void __user *)arg;
1522
1523 ret = -EFAULT;
1524 if (copy_from_user(&uffdio_unregister, buf, sizeof(uffdio_unregister)))
1525 goto out;
1526
1527 ret = validate_range(mm, uffdio_unregister.start,
1528 uffdio_unregister.len);
1529 if (ret)
1530 goto out;
1531
1532 start = uffdio_unregister.start;
1533 end = start + uffdio_unregister.len;
1534
Oleg Nesterovd2005e32016-05-20 16:58:36 -07001535 ret = -ENOMEM;
1536 if (!mmget_not_zero(mm))
1537 goto out;
1538
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001539 down_write(&mm->mmap_sem);
Andrea Arcangeli04f58662019-04-18 17:50:52 -07001540 if (!mmget_still_valid(mm))
1541 goto out_unlock;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001542 vma = find_vma_prev(mm, start, &prev);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001543 if (!vma)
1544 goto out_unlock;
1545
1546 /* check that there's at least one vma in the range */
1547 ret = -EINVAL;
1548 if (vma->vm_start >= end)
1549 goto out_unlock;
1550
1551 /*
Mike Kravetzcab350a2017-02-22 15:43:04 -08001552 * If the first vma contains huge pages, make sure start address
1553 * is aligned to huge page size.
1554 */
1555 if (is_vm_hugetlb_page(vma)) {
1556 unsigned long vma_hpagesize = vma_kernel_pagesize(vma);
1557
1558 if (start & (vma_hpagesize - 1))
1559 goto out_unlock;
1560 }
1561
1562 /*
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001563 * Search for not compatible vmas.
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001564 */
1565 found = false;
1566 ret = -EINVAL;
1567 for (cur = vma; cur && cur->vm_start < end; cur = cur->vm_next) {
1568 cond_resched();
1569
1570 BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^
1571 !!(cur->vm_flags & (VM_UFFD_MISSING | VM_UFFD_WP)));
1572
1573 /*
1574 * Check not compatible vmas, not strictly required
1575 * here as not compatible vmas cannot have an
1576 * userfaultfd_ctx registered on them, but this
1577 * provides for more strict behavior to notice
1578 * unregistration errors.
1579 */
Mike Rapoportba6907d2017-02-22 15:43:22 -08001580 if (!vma_can_userfault(cur))
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001581 goto out_unlock;
1582
1583 found = true;
1584 }
1585 BUG_ON(!found);
1586
1587 if (vma->vm_start < start)
1588 prev = vma;
1589
1590 ret = 0;
1591 do {
1592 cond_resched();
1593
Mike Rapoportba6907d2017-02-22 15:43:22 -08001594 BUG_ON(!vma_can_userfault(vma));
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001595
1596 /*
1597 * Nothing to do: this vma is already registered into this
1598 * userfaultfd and with the right tracking mode too.
1599 */
1600 if (!vma->vm_userfaultfd_ctx.ctx)
1601 goto skip;
1602
Andrea Arcangeli01e881f2018-12-14 14:17:17 -08001603 WARN_ON(!(vma->vm_flags & VM_MAYWRITE));
1604
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001605 if (vma->vm_start > start)
1606 start = vma->vm_start;
1607 vma_end = min(end, vma->vm_end);
1608
Andrea Arcangeli09fa5292017-02-22 15:42:46 -08001609 if (userfaultfd_missing(vma)) {
1610 /*
1611 * Wake any concurrent pending userfault while
1612 * we unregister, so they will not hang
1613 * permanently and it avoids userland to call
1614 * UFFDIO_WAKE explicitly.
1615 */
1616 struct userfaultfd_wake_range range;
1617 range.start = start;
1618 range.len = vma_end - start;
1619 wake_userfault(vma->vm_userfaultfd_ctx.ctx, &range);
1620 }
1621
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001622 new_flags = vma->vm_flags & ~(VM_UFFD_MISSING | VM_UFFD_WP);
1623 prev = vma_merge(mm, prev, start, vma_end, new_flags,
1624 vma->anon_vma, vma->vm_file, vma->vm_pgoff,
1625 vma_policy(vma),
1626 NULL_VM_UFFD_CTX);
1627 if (prev) {
1628 vma = prev;
1629 goto next;
1630 }
1631 if (vma->vm_start < start) {
1632 ret = split_vma(mm, vma, start, 1);
1633 if (ret)
1634 break;
1635 }
1636 if (vma->vm_end > end) {
1637 ret = split_vma(mm, vma, end, 0);
1638 if (ret)
1639 break;
1640 }
1641 next:
1642 /*
1643 * In the vma_merge() successful mprotect-like case 8:
1644 * the next vma was merged into the current one and
1645 * the current one has not been updated yet.
1646 */
1647 vma->vm_flags = new_flags;
1648 vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
1649
1650 skip:
1651 prev = vma;
1652 start = vma->vm_end;
1653 vma = vma->vm_next;
1654 } while (vma && vma->vm_start < end);
1655out_unlock:
1656 up_write(&mm->mmap_sem);
Oleg Nesterovd2005e32016-05-20 16:58:36 -07001657 mmput(mm);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001658out:
1659 return ret;
1660}
1661
1662/*
Andrea Arcangeliba85c702015-09-04 15:46:41 -07001663 * userfaultfd_wake may be used in combination with the
1664 * UFFDIO_*_MODE_DONTWAKE to wakeup userfaults in batches.
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001665 */
1666static int userfaultfd_wake(struct userfaultfd_ctx *ctx,
1667 unsigned long arg)
1668{
1669 int ret;
1670 struct uffdio_range uffdio_wake;
1671 struct userfaultfd_wake_range range;
1672 const void __user *buf = (void __user *)arg;
1673
1674 ret = -EFAULT;
1675 if (copy_from_user(&uffdio_wake, buf, sizeof(uffdio_wake)))
1676 goto out;
1677
1678 ret = validate_range(ctx->mm, uffdio_wake.start, uffdio_wake.len);
1679 if (ret)
1680 goto out;
1681
1682 range.start = uffdio_wake.start;
1683 range.len = uffdio_wake.len;
1684
1685 /*
1686 * len == 0 means wake all and we don't want to wake all here,
1687 * so check it again to be sure.
1688 */
1689 VM_BUG_ON(!range.len);
1690
1691 wake_userfault(ctx, &range);
1692 ret = 0;
1693
1694out:
1695 return ret;
1696}
1697
Andrea Arcangeliad465cae2015-09-04 15:47:11 -07001698static int userfaultfd_copy(struct userfaultfd_ctx *ctx,
1699 unsigned long arg)
1700{
1701 __s64 ret;
1702 struct uffdio_copy uffdio_copy;
1703 struct uffdio_copy __user *user_uffdio_copy;
1704 struct userfaultfd_wake_range range;
1705
1706 user_uffdio_copy = (struct uffdio_copy __user *) arg;
1707
Mike Rapoportdf2cc962018-06-07 17:09:25 -07001708 ret = -EAGAIN;
1709 if (READ_ONCE(ctx->mmap_changing))
1710 goto out;
1711
Andrea Arcangeliad465cae2015-09-04 15:47:11 -07001712 ret = -EFAULT;
1713 if (copy_from_user(&uffdio_copy, user_uffdio_copy,
1714 /* don't copy "copy" last field */
1715 sizeof(uffdio_copy)-sizeof(__s64)))
1716 goto out;
1717
1718 ret = validate_range(ctx->mm, uffdio_copy.dst, uffdio_copy.len);
1719 if (ret)
1720 goto out;
1721 /*
1722 * double check for wraparound just in case. copy_from_user()
1723 * will later check uffdio_copy.src + uffdio_copy.len to fit
1724 * in the userland range.
1725 */
1726 ret = -EINVAL;
1727 if (uffdio_copy.src + uffdio_copy.len <= uffdio_copy.src)
1728 goto out;
1729 if (uffdio_copy.mode & ~UFFDIO_COPY_MODE_DONTWAKE)
1730 goto out;
Oleg Nesterovd2005e32016-05-20 16:58:36 -07001731 if (mmget_not_zero(ctx->mm)) {
1732 ret = mcopy_atomic(ctx->mm, uffdio_copy.dst, uffdio_copy.src,
Mike Rapoportdf2cc962018-06-07 17:09:25 -07001733 uffdio_copy.len, &ctx->mmap_changing);
Oleg Nesterovd2005e32016-05-20 16:58:36 -07001734 mmput(ctx->mm);
Mike Rapoport96333182017-02-24 14:58:31 -08001735 } else {
Mike Rapoporte86b2982017-08-10 15:24:32 -07001736 return -ESRCH;
Oleg Nesterovd2005e32016-05-20 16:58:36 -07001737 }
Andrea Arcangeliad465cae2015-09-04 15:47:11 -07001738 if (unlikely(put_user(ret, &user_uffdio_copy->copy)))
1739 return -EFAULT;
1740 if (ret < 0)
1741 goto out;
1742 BUG_ON(!ret);
1743 /* len == 0 would wake all */
1744 range.len = ret;
1745 if (!(uffdio_copy.mode & UFFDIO_COPY_MODE_DONTWAKE)) {
1746 range.start = uffdio_copy.dst;
1747 wake_userfault(ctx, &range);
1748 }
1749 ret = range.len == uffdio_copy.len ? 0 : -EAGAIN;
1750out:
1751 return ret;
1752}
1753
1754static int userfaultfd_zeropage(struct userfaultfd_ctx *ctx,
1755 unsigned long arg)
1756{
1757 __s64 ret;
1758 struct uffdio_zeropage uffdio_zeropage;
1759 struct uffdio_zeropage __user *user_uffdio_zeropage;
1760 struct userfaultfd_wake_range range;
1761
1762 user_uffdio_zeropage = (struct uffdio_zeropage __user *) arg;
1763
Mike Rapoportdf2cc962018-06-07 17:09:25 -07001764 ret = -EAGAIN;
1765 if (READ_ONCE(ctx->mmap_changing))
1766 goto out;
1767
Andrea Arcangeliad465cae2015-09-04 15:47:11 -07001768 ret = -EFAULT;
1769 if (copy_from_user(&uffdio_zeropage, user_uffdio_zeropage,
1770 /* don't copy "zeropage" last field */
1771 sizeof(uffdio_zeropage)-sizeof(__s64)))
1772 goto out;
1773
1774 ret = validate_range(ctx->mm, uffdio_zeropage.range.start,
1775 uffdio_zeropage.range.len);
1776 if (ret)
1777 goto out;
1778 ret = -EINVAL;
1779 if (uffdio_zeropage.mode & ~UFFDIO_ZEROPAGE_MODE_DONTWAKE)
1780 goto out;
1781
Oleg Nesterovd2005e32016-05-20 16:58:36 -07001782 if (mmget_not_zero(ctx->mm)) {
1783 ret = mfill_zeropage(ctx->mm, uffdio_zeropage.range.start,
Mike Rapoportdf2cc962018-06-07 17:09:25 -07001784 uffdio_zeropage.range.len,
1785 &ctx->mmap_changing);
Oleg Nesterovd2005e32016-05-20 16:58:36 -07001786 mmput(ctx->mm);
Mike Rapoport9d95aa42017-08-02 13:32:15 -07001787 } else {
Mike Rapoporte86b2982017-08-10 15:24:32 -07001788 return -ESRCH;
Oleg Nesterovd2005e32016-05-20 16:58:36 -07001789 }
Andrea Arcangeliad465cae2015-09-04 15:47:11 -07001790 if (unlikely(put_user(ret, &user_uffdio_zeropage->zeropage)))
1791 return -EFAULT;
1792 if (ret < 0)
1793 goto out;
1794 /* len == 0 would wake all */
1795 BUG_ON(!ret);
1796 range.len = ret;
1797 if (!(uffdio_zeropage.mode & UFFDIO_ZEROPAGE_MODE_DONTWAKE)) {
1798 range.start = uffdio_zeropage.range.start;
1799 wake_userfault(ctx, &range);
1800 }
1801 ret = range.len == uffdio_zeropage.range.len ? 0 : -EAGAIN;
1802out:
1803 return ret;
1804}
1805
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -08001806static inline unsigned int uffd_ctx_features(__u64 user_features)
1807{
1808 /*
1809 * For the current set of features the bits just coincide
1810 */
1811 return (unsigned int)user_features;
1812}
1813
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001814/*
1815 * userland asks for a certain API version and we return which bits
1816 * and ioctl commands are implemented in this kernel for such API
1817 * version or -EINVAL if unknown.
1818 */
1819static int userfaultfd_api(struct userfaultfd_ctx *ctx,
1820 unsigned long arg)
1821{
1822 struct uffdio_api uffdio_api;
1823 void __user *buf = (void __user *)arg;
1824 int ret;
Andrea Arcangeli65603142017-02-22 15:42:24 -08001825 __u64 features;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001826
1827 ret = -EINVAL;
1828 if (ctx->state != UFFD_STATE_WAIT_API)
1829 goto out;
1830 ret = -EFAULT;
Andrea Arcangelia9b85f92015-09-04 15:46:37 -07001831 if (copy_from_user(&uffdio_api, buf, sizeof(uffdio_api)))
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001832 goto out;
Andrea Arcangeli65603142017-02-22 15:42:24 -08001833 features = uffdio_api.features;
1834 if (uffdio_api.api != UFFD_API || (features & ~UFFD_API_FEATURES)) {
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001835 memset(&uffdio_api, 0, sizeof(uffdio_api));
1836 if (copy_to_user(buf, &uffdio_api, sizeof(uffdio_api)))
1837 goto out;
1838 ret = -EINVAL;
1839 goto out;
1840 }
Andrea Arcangeli65603142017-02-22 15:42:24 -08001841 /* report all available features and ioctls to userland */
1842 uffdio_api.features = UFFD_API_FEATURES;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001843 uffdio_api.ioctls = UFFD_API_IOCTLS;
1844 ret = -EFAULT;
1845 if (copy_to_user(buf, &uffdio_api, sizeof(uffdio_api)))
1846 goto out;
1847 ctx->state = UFFD_STATE_RUNNING;
Andrea Arcangeli65603142017-02-22 15:42:24 -08001848 /* only enable the requested features for this uffd context */
1849 ctx->features = uffd_ctx_features(features);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001850 ret = 0;
1851out:
1852 return ret;
1853}
1854
1855static long userfaultfd_ioctl(struct file *file, unsigned cmd,
1856 unsigned long arg)
1857{
1858 int ret = -EINVAL;
1859 struct userfaultfd_ctx *ctx = file->private_data;
1860
Andrea Arcangelie6485a42015-09-04 15:47:15 -07001861 if (cmd != UFFDIO_API && ctx->state == UFFD_STATE_WAIT_API)
1862 return -EINVAL;
1863
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001864 switch(cmd) {
1865 case UFFDIO_API:
1866 ret = userfaultfd_api(ctx, arg);
1867 break;
1868 case UFFDIO_REGISTER:
1869 ret = userfaultfd_register(ctx, arg);
1870 break;
1871 case UFFDIO_UNREGISTER:
1872 ret = userfaultfd_unregister(ctx, arg);
1873 break;
1874 case UFFDIO_WAKE:
1875 ret = userfaultfd_wake(ctx, arg);
1876 break;
Andrea Arcangeliad465cae2015-09-04 15:47:11 -07001877 case UFFDIO_COPY:
1878 ret = userfaultfd_copy(ctx, arg);
1879 break;
1880 case UFFDIO_ZEROPAGE:
1881 ret = userfaultfd_zeropage(ctx, arg);
1882 break;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001883 }
1884 return ret;
1885}
1886
1887#ifdef CONFIG_PROC_FS
1888static void userfaultfd_show_fdinfo(struct seq_file *m, struct file *f)
1889{
1890 struct userfaultfd_ctx *ctx = f->private_data;
Ingo Molnarac6424b2017-06-20 12:06:13 +02001891 wait_queue_entry_t *wq;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001892 unsigned long pending = 0, total = 0;
1893
Eric Biggerscbcfa132019-07-04 15:14:39 -07001894 spin_lock_irq(&ctx->fault_pending_wqh.lock);
Ingo Molnar2055da92017-06-20 12:06:46 +02001895 list_for_each_entry(wq, &ctx->fault_pending_wqh.head, entry) {
Andrea Arcangeli15b726e2015-09-04 15:46:44 -07001896 pending++;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001897 total++;
1898 }
Ingo Molnar2055da92017-06-20 12:06:46 +02001899 list_for_each_entry(wq, &ctx->fault_wqh.head, entry) {
Andrea Arcangeli15b726e2015-09-04 15:46:44 -07001900 total++;
1901 }
Eric Biggerscbcfa132019-07-04 15:14:39 -07001902 spin_unlock_irq(&ctx->fault_pending_wqh.lock);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001903
1904 /*
1905 * If more protocols will be added, there will be all shown
1906 * separated by a space. Like this:
1907 * protocols: aa:... bb:...
1908 */
1909 seq_printf(m, "pending:\t%lu\ntotal:\t%lu\nAPI:\t%Lx:%x:%Lx\n",
Mike Rapoport045098e2017-04-07 16:04:42 -07001910 pending, total, UFFD_API, ctx->features,
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001911 UFFD_API_IOCTLS|UFFD_API_RANGE_IOCTLS);
1912}
1913#endif
1914
1915static const struct file_operations userfaultfd_fops = {
1916#ifdef CONFIG_PROC_FS
1917 .show_fdinfo = userfaultfd_show_fdinfo,
1918#endif
1919 .release = userfaultfd_release,
1920 .poll = userfaultfd_poll,
1921 .read = userfaultfd_read,
1922 .unlocked_ioctl = userfaultfd_ioctl,
1923 .compat_ioctl = userfaultfd_ioctl,
1924 .llseek = noop_llseek,
1925};
1926
Andrea Arcangeli3004ec92015-09-04 15:46:48 -07001927static void init_once_userfaultfd_ctx(void *mem)
1928{
1929 struct userfaultfd_ctx *ctx = (struct userfaultfd_ctx *) mem;
1930
1931 init_waitqueue_head(&ctx->fault_pending_wqh);
1932 init_waitqueue_head(&ctx->fault_wqh);
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -08001933 init_waitqueue_head(&ctx->event_wqh);
Andrea Arcangeli3004ec92015-09-04 15:46:48 -07001934 init_waitqueue_head(&ctx->fd_wqh);
Andrea Arcangeli2c5b7e12015-09-04 15:47:23 -07001935 seqcount_init(&ctx->refile_seq);
Andrea Arcangeli3004ec92015-09-04 15:46:48 -07001936}
1937
Eric Biggers284cd242018-01-31 16:19:48 -08001938SYSCALL_DEFINE1(userfaultfd, int, flags)
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001939{
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001940 struct userfaultfd_ctx *ctx;
Eric Biggers284cd242018-01-31 16:19:48 -08001941 int fd;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001942
Peter Xucefdca02019-05-13 17:16:41 -07001943 if (!sysctl_unprivileged_userfaultfd && !capable(CAP_SYS_PTRACE))
1944 return -EPERM;
1945
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001946 BUG_ON(!current->mm);
1947
1948 /* Check the UFFD_* constants for consistency. */
1949 BUILD_BUG_ON(UFFD_CLOEXEC != O_CLOEXEC);
1950 BUILD_BUG_ON(UFFD_NONBLOCK != O_NONBLOCK);
1951
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001952 if (flags & ~UFFD_SHARED_FCNTL_FLAGS)
Eric Biggers284cd242018-01-31 16:19:48 -08001953 return -EINVAL;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001954
Andrea Arcangeli3004ec92015-09-04 15:46:48 -07001955 ctx = kmem_cache_alloc(userfaultfd_ctx_cachep, GFP_KERNEL);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001956 if (!ctx)
Eric Biggers284cd242018-01-31 16:19:48 -08001957 return -ENOMEM;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001958
Eric Biggersca880422018-12-28 00:34:43 -08001959 refcount_set(&ctx->refcount, 1);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001960 ctx->flags = flags;
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -08001961 ctx->features = 0;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001962 ctx->state = UFFD_STATE_WAIT_API;
1963 ctx->released = false;
Mike Rapoportdf2cc962018-06-07 17:09:25 -07001964 ctx->mmap_changing = false;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001965 ctx->mm = current->mm;
1966 /* prevent the mm struct to be freed */
Vegard Nossumf1f10072017-02-27 14:30:07 -08001967 mmgrab(ctx->mm);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001968
Eric Biggers284cd242018-01-31 16:19:48 -08001969 fd = anon_inode_getfd("[userfaultfd]", &userfaultfd_fops, ctx,
1970 O_RDWR | (flags & UFFD_SHARED_FCNTL_FLAGS));
1971 if (fd < 0) {
Oleg Nesterovd2005e32016-05-20 16:58:36 -07001972 mmdrop(ctx->mm);
Andrea Arcangeli3004ec92015-09-04 15:46:48 -07001973 kmem_cache_free(userfaultfd_ctx_cachep, ctx);
Eric Biggersc03e9462015-09-17 16:01:54 -07001974 }
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001975 return fd;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001976}
Andrea Arcangeli3004ec92015-09-04 15:46:48 -07001977
1978static int __init userfaultfd_init(void)
1979{
1980 userfaultfd_ctx_cachep = kmem_cache_create("userfaultfd_ctx_cache",
1981 sizeof(struct userfaultfd_ctx),
1982 0,
1983 SLAB_HWCACHE_ALIGN|SLAB_PANIC,
1984 init_once_userfaultfd_ctx);
1985 return 0;
1986}
1987__initcall(userfaultfd_init);