blob: 5ad652fa24b891bcf428af02ad84255a5dec19ca [file] [log] [blame]
Jens Axboe2b188cc2019-01-07 10:46:33 -07001// SPDX-License-Identifier: GPL-2.0
2/*
3 * Shared application/kernel submission and completion ring pairs, for
4 * supporting fast/efficient IO.
5 *
6 * A note on the read/write ordering memory barriers that are matched between
Stefan Bühler1e84b972019-04-24 23:54:16 +02007 * the application and kernel side.
8 *
9 * After the application reads the CQ ring tail, it must use an
10 * appropriate smp_rmb() to pair with the smp_wmb() the kernel uses
11 * before writing the tail (using smp_load_acquire to read the tail will
12 * do). It also needs a smp_mb() before updating CQ head (ordering the
13 * entry load(s) with the head store), pairing with an implicit barrier
14 * through a control-dependency in io_get_cqring (smp_store_release to
15 * store head will do). Failure to do so could lead to reading invalid
16 * CQ entries.
17 *
18 * Likewise, the application must use an appropriate smp_wmb() before
19 * writing the SQ tail (ordering SQ entry stores with the tail store),
20 * which pairs with smp_load_acquire in io_get_sqring (smp_store_release
21 * to store the tail will do). And it needs a barrier ordering the SQ
22 * head load before writing new SQ entries (smp_load_acquire to read
23 * head will do).
24 *
25 * When using the SQ poll thread (IORING_SETUP_SQPOLL), the application
26 * needs to check the SQ flags for IORING_SQ_NEED_WAKEUP *after*
27 * updating the SQ tail; a full memory barrier smp_mb() is needed
28 * between.
Jens Axboe2b188cc2019-01-07 10:46:33 -070029 *
30 * Also see the examples in the liburing library:
31 *
32 * git://git.kernel.dk/liburing
33 *
34 * io_uring also uses READ/WRITE_ONCE() for _any_ store or load that happens
35 * from data shared between the kernel and application. This is done both
36 * for ordering purposes, but also to ensure that once a value is loaded from
37 * data that the application could potentially modify, it remains stable.
38 *
39 * Copyright (C) 2018-2019 Jens Axboe
Christoph Hellwigc992fe22019-01-11 09:43:02 -070040 * Copyright (c) 2018-2019 Christoph Hellwig
Jens Axboe2b188cc2019-01-07 10:46:33 -070041 */
42#include <linux/kernel.h>
43#include <linux/init.h>
44#include <linux/errno.h>
45#include <linux/syscalls.h>
46#include <linux/compat.h>
47#include <linux/refcount.h>
48#include <linux/uio.h>
49
50#include <linux/sched/signal.h>
51#include <linux/fs.h>
52#include <linux/file.h>
53#include <linux/fdtable.h>
54#include <linux/mm.h>
55#include <linux/mman.h>
56#include <linux/mmu_context.h>
57#include <linux/percpu.h>
58#include <linux/slab.h>
Jens Axboe6c271ce2019-01-10 11:22:30 -070059#include <linux/kthread.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070060#include <linux/blkdev.h>
Jens Axboeedafcce2019-01-09 09:16:05 -070061#include <linux/bvec.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070062#include <linux/net.h>
63#include <net/sock.h>
64#include <net/af_unix.h>
Jens Axboe6b063142019-01-10 22:13:58 -070065#include <net/scm.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070066#include <linux/anon_inodes.h>
67#include <linux/sched/mm.h>
68#include <linux/uaccess.h>
69#include <linux/nospec.h>
Jens Axboeedafcce2019-01-09 09:16:05 -070070#include <linux/sizes.h>
71#include <linux/hugetlb.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070072
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +020073#define CREATE_TRACE_POINTS
74#include <trace/events/io_uring.h>
75
Jens Axboe2b188cc2019-01-07 10:46:33 -070076#include <uapi/linux/io_uring.h>
77
78#include "internal.h"
Jens Axboe561fb042019-10-24 07:25:42 -060079#include "io-wq.h"
Jens Axboe2b188cc2019-01-07 10:46:33 -070080
Daniel Xu5277dea2019-09-14 14:23:45 -070081#define IORING_MAX_ENTRIES 32768
Jens Axboe33a107f2019-10-04 12:10:03 -060082#define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES)
Jens Axboe65e19f52019-10-26 07:20:21 -060083
84/*
85 * Shift of 9 is 512 entries, or exactly one page on 64-bit archs
86 */
87#define IORING_FILE_TABLE_SHIFT 9
88#define IORING_MAX_FILES_TABLE (1U << IORING_FILE_TABLE_SHIFT)
89#define IORING_FILE_TABLE_MASK (IORING_MAX_FILES_TABLE - 1)
90#define IORING_MAX_FIXED_FILES (64 * IORING_MAX_FILES_TABLE)
Jens Axboe2b188cc2019-01-07 10:46:33 -070091
92struct io_uring {
93 u32 head ____cacheline_aligned_in_smp;
94 u32 tail ____cacheline_aligned_in_smp;
95};
96
Stefan Bühler1e84b972019-04-24 23:54:16 +020097/*
Hristo Venev75b28af2019-08-26 17:23:46 +000098 * This data is shared with the application through the mmap at offsets
99 * IORING_OFF_SQ_RING and IORING_OFF_CQ_RING.
Stefan Bühler1e84b972019-04-24 23:54:16 +0200100 *
101 * The offsets to the member fields are published through struct
102 * io_sqring_offsets when calling io_uring_setup.
103 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000104struct io_rings {
Stefan Bühler1e84b972019-04-24 23:54:16 +0200105 /*
106 * Head and tail offsets into the ring; the offsets need to be
107 * masked to get valid indices.
108 *
Hristo Venev75b28af2019-08-26 17:23:46 +0000109 * The kernel controls head of the sq ring and the tail of the cq ring,
110 * and the application controls tail of the sq ring and the head of the
111 * cq ring.
Stefan Bühler1e84b972019-04-24 23:54:16 +0200112 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000113 struct io_uring sq, cq;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200114 /*
Hristo Venev75b28af2019-08-26 17:23:46 +0000115 * Bitmasks to apply to head and tail offsets (constant, equals
Stefan Bühler1e84b972019-04-24 23:54:16 +0200116 * ring_entries - 1)
117 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000118 u32 sq_ring_mask, cq_ring_mask;
119 /* Ring sizes (constant, power of 2) */
120 u32 sq_ring_entries, cq_ring_entries;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200121 /*
122 * Number of invalid entries dropped by the kernel due to
123 * invalid index stored in array
124 *
125 * Written by the kernel, shouldn't be modified by the
126 * application (i.e. get number of "new events" by comparing to
127 * cached value).
128 *
129 * After a new SQ head value was read by the application this
130 * counter includes all submissions that were dropped reaching
131 * the new SQ head (and possibly more).
132 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000133 u32 sq_dropped;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200134 /*
135 * Runtime flags
136 *
137 * Written by the kernel, shouldn't be modified by the
138 * application.
139 *
140 * The application needs a full memory barrier before checking
141 * for IORING_SQ_NEED_WAKEUP after updating the sq tail.
142 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000143 u32 sq_flags;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200144 /*
145 * Number of completion events lost because the queue was full;
146 * this should be avoided by the application by making sure
147 * there are not more requests pending thatn there is space in
148 * the completion queue.
149 *
150 * Written by the kernel, shouldn't be modified by the
151 * application (i.e. get number of "new events" by comparing to
152 * cached value).
153 *
154 * As completion events come in out of order this counter is not
155 * ordered with any other data.
156 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000157 u32 cq_overflow;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200158 /*
159 * Ring buffer of completion events.
160 *
161 * The kernel writes completion events fresh every time they are
162 * produced, so the application is allowed to modify pending
163 * entries.
164 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000165 struct io_uring_cqe cqes[] ____cacheline_aligned_in_smp;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700166};
167
Jens Axboeedafcce2019-01-09 09:16:05 -0700168struct io_mapped_ubuf {
169 u64 ubuf;
170 size_t len;
171 struct bio_vec *bvec;
172 unsigned int nr_bvecs;
173};
174
Jens Axboe65e19f52019-10-26 07:20:21 -0600175struct fixed_file_table {
176 struct file **files;
177};
178
Jens Axboe2b188cc2019-01-07 10:46:33 -0700179struct io_ring_ctx {
180 struct {
181 struct percpu_ref refs;
182 } ____cacheline_aligned_in_smp;
183
184 struct {
185 unsigned int flags;
186 bool compat;
187 bool account_mem;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -0700188 bool cq_overflow_flushed;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700189
Hristo Venev75b28af2019-08-26 17:23:46 +0000190 /*
191 * Ring buffer of indices into array of io_uring_sqe, which is
192 * mmapped by the application using the IORING_OFF_SQES offset.
193 *
194 * This indirection could e.g. be used to assign fixed
195 * io_uring_sqe entries to operations and only submit them to
196 * the queue when needed.
197 *
198 * The kernel modifies neither the indices array nor the entries
199 * array.
200 */
201 u32 *sq_array;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700202 unsigned cached_sq_head;
203 unsigned sq_entries;
204 unsigned sq_mask;
Jens Axboe6c271ce2019-01-10 11:22:30 -0700205 unsigned sq_thread_idle;
Jens Axboe498ccd92019-10-25 10:04:25 -0600206 unsigned cached_sq_dropped;
Jens Axboe206aefd2019-11-07 18:27:42 -0700207 atomic_t cached_cq_overflow;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700208 struct io_uring_sqe *sq_sqes;
Jens Axboede0617e2019-04-06 21:51:27 -0600209
210 struct list_head defer_list;
Jens Axboe5262f562019-09-17 12:26:57 -0600211 struct list_head timeout_list;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -0700212 struct list_head cq_overflow_list;
Jens Axboefcb323c2019-10-24 12:39:47 -0600213
214 wait_queue_head_t inflight_wait;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700215 } ____cacheline_aligned_in_smp;
216
Jens Axboe206aefd2019-11-07 18:27:42 -0700217 struct io_rings *rings;
218
Jens Axboe2b188cc2019-01-07 10:46:33 -0700219 /* IO offload */
Jens Axboe561fb042019-10-24 07:25:42 -0600220 struct io_wq *io_wq;
Jens Axboe6c271ce2019-01-10 11:22:30 -0700221 struct task_struct *sqo_thread; /* if using sq thread polling */
Jens Axboe2b188cc2019-01-07 10:46:33 -0700222 struct mm_struct *sqo_mm;
Jens Axboe6c271ce2019-01-10 11:22:30 -0700223 wait_queue_head_t sqo_wait;
Hristo Venev75b28af2019-08-26 17:23:46 +0000224
Jens Axboe6b063142019-01-10 22:13:58 -0700225 /*
226 * If used, fixed file set. Writers must ensure that ->refs is dead,
227 * readers must ensure that ->refs is alive as long as the file* is
228 * used. Only updated through io_uring_register(2).
229 */
Jens Axboe65e19f52019-10-26 07:20:21 -0600230 struct fixed_file_table *file_table;
Jens Axboe6b063142019-01-10 22:13:58 -0700231 unsigned nr_user_files;
232
Jens Axboeedafcce2019-01-09 09:16:05 -0700233 /* if used, fixed mapped user buffers */
234 unsigned nr_user_bufs;
235 struct io_mapped_ubuf *user_bufs;
236
Jens Axboe2b188cc2019-01-07 10:46:33 -0700237 struct user_struct *user;
238
Jens Axboe206aefd2019-11-07 18:27:42 -0700239 /* 0 is for ctx quiesce/reinit/free, 1 is for sqo_thread started */
240 struct completion *completions;
241
Jens Axboe0ddf92e2019-11-08 08:52:53 -0700242 /* if all else fails... */
243 struct io_kiocb *fallback_req;
244
Jens Axboe206aefd2019-11-07 18:27:42 -0700245#if defined(CONFIG_UNIX)
246 struct socket *ring_sock;
247#endif
248
249 struct {
250 unsigned cached_cq_tail;
251 unsigned cq_entries;
252 unsigned cq_mask;
253 atomic_t cq_timeouts;
254 struct wait_queue_head cq_wait;
255 struct fasync_struct *cq_fasync;
256 struct eventfd_ctx *cq_ev_fd;
257 } ____cacheline_aligned_in_smp;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700258
259 struct {
260 struct mutex uring_lock;
261 wait_queue_head_t wait;
262 } ____cacheline_aligned_in_smp;
263
264 struct {
265 spinlock_t completion_lock;
Jens Axboedef596e2019-01-09 08:59:42 -0700266 bool poll_multi_file;
267 /*
268 * ->poll_list is protected by the ctx->uring_lock for
269 * io_uring instances that don't use IORING_SETUP_SQPOLL.
270 * For SQPOLL, only the single threaded io_sq_thread() will
271 * manipulate the list, hence no extra locking is needed there.
272 */
273 struct list_head poll_list;
Jens Axboeeac406c2019-11-14 12:09:58 -0700274 struct rb_root cancel_tree;
Jens Axboefcb323c2019-10-24 12:39:47 -0600275
276 spinlock_t inflight_lock;
277 struct list_head inflight_list;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700278 } ____cacheline_aligned_in_smp;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700279};
280
281struct sqe_submit {
282 const struct io_uring_sqe *sqe;
Jens Axboefcb323c2019-10-24 12:39:47 -0600283 struct file *ring_file;
284 int ring_fd;
Jackie Liu8776f3f2019-09-09 20:50:39 +0800285 u32 sequence;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700286 bool has_user;
Jackie Liuba5290c2019-10-09 09:19:59 +0800287 bool in_async;
Jens Axboe6c271ce2019-01-10 11:22:30 -0700288 bool needs_fixed_file;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700289};
290
Jens Axboe09bb8392019-03-13 12:39:28 -0600291/*
292 * First field must be the file pointer in all the
293 * iocb unions! See also 'struct kiocb' in <linux/fs.h>
294 */
Jens Axboe221c5eb2019-01-17 09:41:58 -0700295struct io_poll_iocb {
296 struct file *file;
297 struct wait_queue_head *head;
298 __poll_t events;
Jens Axboe8c838782019-03-12 15:48:16 -0600299 bool done;
Jens Axboe221c5eb2019-01-17 09:41:58 -0700300 bool canceled;
301 struct wait_queue_entry wait;
302};
303
Jens Axboe5262f562019-09-17 12:26:57 -0600304struct io_timeout {
305 struct file *file;
306 struct hrtimer timer;
307};
308
Jens Axboe09bb8392019-03-13 12:39:28 -0600309/*
310 * NOTE! Each of the iocb union members has the file pointer
311 * as the first entry in their struct definition. So you can
312 * access the file pointer through any of the sub-structs,
313 * or directly as just 'ki_filp' in this struct.
314 */
Jens Axboe2b188cc2019-01-07 10:46:33 -0700315struct io_kiocb {
Jens Axboe221c5eb2019-01-17 09:41:58 -0700316 union {
Jens Axboe09bb8392019-03-13 12:39:28 -0600317 struct file *file;
Jens Axboe221c5eb2019-01-17 09:41:58 -0700318 struct kiocb rw;
319 struct io_poll_iocb poll;
Jens Axboe5262f562019-09-17 12:26:57 -0600320 struct io_timeout timeout;
Jens Axboe221c5eb2019-01-17 09:41:58 -0700321 };
Jens Axboe2b188cc2019-01-07 10:46:33 -0700322
323 struct sqe_submit submit;
324
325 struct io_ring_ctx *ctx;
Jens Axboeeac406c2019-11-14 12:09:58 -0700326 union {
327 struct list_head list;
328 struct rb_node rb_node;
329 };
Jens Axboe9e645e112019-05-10 16:07:28 -0600330 struct list_head link_list;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700331 unsigned int flags;
Jens Axboec16361c2019-01-17 08:39:48 -0700332 refcount_t refs;
Stefan Bühler8449eed2019-04-27 20:34:19 +0200333#define REQ_F_NOWAIT 1 /* must not punt to workers */
Jens Axboedef596e2019-01-09 08:59:42 -0700334#define REQ_F_IOPOLL_COMPLETED 2 /* polled IO has completed */
Jens Axboe6b063142019-01-10 22:13:58 -0700335#define REQ_F_FIXED_FILE 4 /* ctx owns file */
Jens Axboe31b51512019-01-18 22:56:34 -0700336#define REQ_F_SEQ_PREV 8 /* sequential with previous */
Stefan Bühlere2033e32019-05-11 19:08:01 +0200337#define REQ_F_IO_DRAIN 16 /* drain existing IO first */
338#define REQ_F_IO_DRAINED 32 /* drain done */
Jens Axboe9e645e112019-05-10 16:07:28 -0600339#define REQ_F_LINK 64 /* linked sqes */
Jens Axboe2665abf2019-11-05 12:40:47 -0700340#define REQ_F_LINK_TIMEOUT 128 /* has linked timeout */
Zhengyuan Liuf7b76ac2019-07-16 23:26:14 +0800341#define REQ_F_FAIL_LINK 256 /* fail rest of links */
Jackie Liu4fe2c962019-09-09 20:50:40 +0800342#define REQ_F_SHADOW_DRAIN 512 /* link-drain shadow req */
Jens Axboe5262f562019-09-17 12:26:57 -0600343#define REQ_F_TIMEOUT 1024 /* timeout request */
Jens Axboe491381ce2019-10-17 09:20:46 -0600344#define REQ_F_ISREG 2048 /* regular file */
345#define REQ_F_MUST_PUNT 4096 /* must be punted even for NONBLOCK */
Jens Axboefcb323c2019-10-24 12:39:47 -0600346#define REQ_F_INFLIGHT 8192 /* on inflight list */
Jens Axboe7c9e7f02019-11-12 08:15:53 -0700347#define REQ_F_COMP_LOCKED 16384 /* completion under lock */
Jens Axboe2b188cc2019-01-07 10:46:33 -0700348 u64 user_data;
Jens Axboe9e645e112019-05-10 16:07:28 -0600349 u32 result;
Jens Axboede0617e2019-04-06 21:51:27 -0600350 u32 sequence;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700351
Jens Axboefcb323c2019-10-24 12:39:47 -0600352 struct list_head inflight_entry;
353
Jens Axboe561fb042019-10-24 07:25:42 -0600354 struct io_wq_work work;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700355};
356
357#define IO_PLUG_THRESHOLD 2
Jens Axboedef596e2019-01-09 08:59:42 -0700358#define IO_IOPOLL_BATCH 8
Jens Axboe2b188cc2019-01-07 10:46:33 -0700359
Jens Axboe9a56a232019-01-09 09:06:50 -0700360struct io_submit_state {
361 struct blk_plug plug;
362
363 /*
Jens Axboe2579f912019-01-09 09:10:43 -0700364 * io_kiocb alloc cache
365 */
366 void *reqs[IO_IOPOLL_BATCH];
367 unsigned int free_reqs;
368 unsigned int cur_req;
369
370 /*
Jens Axboe9a56a232019-01-09 09:06:50 -0700371 * File reference cache
372 */
373 struct file *file;
374 unsigned int fd;
375 unsigned int has_refs;
376 unsigned int used_refs;
377 unsigned int ios_left;
378};
379
Jens Axboe561fb042019-10-24 07:25:42 -0600380static void io_wq_submit_work(struct io_wq_work **workptr);
Jens Axboe78e19bb2019-11-06 15:21:34 -0700381static void io_cqring_fill_event(struct io_kiocb *req, long res);
Jackie Liu4fe2c962019-09-09 20:50:40 +0800382static void __io_free_req(struct io_kiocb *req);
Jackie Liuec9c02a2019-11-08 23:50:36 +0800383static void io_put_req(struct io_kiocb *req);
Jens Axboe78e19bb2019-11-06 15:21:34 -0700384static void io_double_put_req(struct io_kiocb *req);
Jens Axboede0617e2019-04-06 21:51:27 -0600385
Jens Axboe2b188cc2019-01-07 10:46:33 -0700386static struct kmem_cache *req_cachep;
387
388static const struct file_operations io_uring_fops;
389
390struct sock *io_uring_get_socket(struct file *file)
391{
392#if defined(CONFIG_UNIX)
393 if (file->f_op == &io_uring_fops) {
394 struct io_ring_ctx *ctx = file->private_data;
395
396 return ctx->ring_sock->sk;
397 }
398#endif
399 return NULL;
400}
401EXPORT_SYMBOL(io_uring_get_socket);
402
403static void io_ring_ctx_ref_free(struct percpu_ref *ref)
404{
405 struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs);
406
Jens Axboe206aefd2019-11-07 18:27:42 -0700407 complete(&ctx->completions[0]);
Jens Axboe2b188cc2019-01-07 10:46:33 -0700408}
409
410static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
411{
412 struct io_ring_ctx *ctx;
413
414 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
415 if (!ctx)
416 return NULL;
417
Jens Axboe0ddf92e2019-11-08 08:52:53 -0700418 ctx->fallback_req = kmem_cache_alloc(req_cachep, GFP_KERNEL);
419 if (!ctx->fallback_req)
420 goto err;
421
Jens Axboe206aefd2019-11-07 18:27:42 -0700422 ctx->completions = kmalloc(2 * sizeof(struct completion), GFP_KERNEL);
423 if (!ctx->completions)
424 goto err;
425
Roman Gushchin21482892019-05-07 10:01:48 -0700426 if (percpu_ref_init(&ctx->refs, io_ring_ctx_ref_free,
Jens Axboe206aefd2019-11-07 18:27:42 -0700427 PERCPU_REF_ALLOW_REINIT, GFP_KERNEL))
428 goto err;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700429
430 ctx->flags = p->flags;
431 init_waitqueue_head(&ctx->cq_wait);
Jens Axboe1d7bb1d2019-11-06 11:31:17 -0700432 INIT_LIST_HEAD(&ctx->cq_overflow_list);
Jens Axboe206aefd2019-11-07 18:27:42 -0700433 init_completion(&ctx->completions[0]);
434 init_completion(&ctx->completions[1]);
Jens Axboe2b188cc2019-01-07 10:46:33 -0700435 mutex_init(&ctx->uring_lock);
436 init_waitqueue_head(&ctx->wait);
437 spin_lock_init(&ctx->completion_lock);
Jens Axboedef596e2019-01-09 08:59:42 -0700438 INIT_LIST_HEAD(&ctx->poll_list);
Jens Axboeeac406c2019-11-14 12:09:58 -0700439 ctx->cancel_tree = RB_ROOT;
Jens Axboede0617e2019-04-06 21:51:27 -0600440 INIT_LIST_HEAD(&ctx->defer_list);
Jens Axboe5262f562019-09-17 12:26:57 -0600441 INIT_LIST_HEAD(&ctx->timeout_list);
Jens Axboefcb323c2019-10-24 12:39:47 -0600442 init_waitqueue_head(&ctx->inflight_wait);
443 spin_lock_init(&ctx->inflight_lock);
444 INIT_LIST_HEAD(&ctx->inflight_list);
Jens Axboe2b188cc2019-01-07 10:46:33 -0700445 return ctx;
Jens Axboe206aefd2019-11-07 18:27:42 -0700446err:
Jens Axboe0ddf92e2019-11-08 08:52:53 -0700447 if (ctx->fallback_req)
448 kmem_cache_free(req_cachep, ctx->fallback_req);
Jens Axboe206aefd2019-11-07 18:27:42 -0700449 kfree(ctx->completions);
450 kfree(ctx);
451 return NULL;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700452}
453
Bob Liu9d858b22019-11-13 18:06:25 +0800454static inline bool __req_need_defer(struct io_kiocb *req)
Jens Axboede0617e2019-04-06 21:51:27 -0600455{
Jackie Liua197f662019-11-08 08:09:12 -0700456 struct io_ring_ctx *ctx = req->ctx;
457
Jens Axboe498ccd92019-10-25 10:04:25 -0600458 return req->sequence != ctx->cached_cq_tail + ctx->cached_sq_dropped
459 + atomic_read(&ctx->cached_cq_overflow);
Jens Axboede0617e2019-04-06 21:51:27 -0600460}
461
Bob Liu9d858b22019-11-13 18:06:25 +0800462static inline bool req_need_defer(struct io_kiocb *req)
Jens Axboe7adf4ea2019-10-10 21:42:58 -0600463{
Bob Liu9d858b22019-11-13 18:06:25 +0800464 if ((req->flags & (REQ_F_IO_DRAIN|REQ_F_IO_DRAINED)) == REQ_F_IO_DRAIN)
465 return __req_need_defer(req);
Jens Axboe7adf4ea2019-10-10 21:42:58 -0600466
Bob Liu9d858b22019-11-13 18:06:25 +0800467 return false;
Jens Axboe7adf4ea2019-10-10 21:42:58 -0600468}
469
470static struct io_kiocb *io_get_deferred_req(struct io_ring_ctx *ctx)
Jens Axboede0617e2019-04-06 21:51:27 -0600471{
472 struct io_kiocb *req;
473
Jens Axboe7adf4ea2019-10-10 21:42:58 -0600474 req = list_first_entry_or_null(&ctx->defer_list, struct io_kiocb, list);
Bob Liu9d858b22019-11-13 18:06:25 +0800475 if (req && !req_need_defer(req)) {
Jens Axboede0617e2019-04-06 21:51:27 -0600476 list_del_init(&req->list);
477 return req;
478 }
479
480 return NULL;
481}
482
Jens Axboe5262f562019-09-17 12:26:57 -0600483static struct io_kiocb *io_get_timeout_req(struct io_ring_ctx *ctx)
484{
Jens Axboe7adf4ea2019-10-10 21:42:58 -0600485 struct io_kiocb *req;
486
487 req = list_first_entry_or_null(&ctx->timeout_list, struct io_kiocb, list);
Bob Liu9d858b22019-11-13 18:06:25 +0800488 if (req && !__req_need_defer(req)) {
Jens Axboe7adf4ea2019-10-10 21:42:58 -0600489 list_del_init(&req->list);
490 return req;
491 }
492
493 return NULL;
Jens Axboe5262f562019-09-17 12:26:57 -0600494}
495
Jens Axboede0617e2019-04-06 21:51:27 -0600496static void __io_commit_cqring(struct io_ring_ctx *ctx)
Jens Axboe2b188cc2019-01-07 10:46:33 -0700497{
Hristo Venev75b28af2019-08-26 17:23:46 +0000498 struct io_rings *rings = ctx->rings;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700499
Hristo Venev75b28af2019-08-26 17:23:46 +0000500 if (ctx->cached_cq_tail != READ_ONCE(rings->cq.tail)) {
Jens Axboe2b188cc2019-01-07 10:46:33 -0700501 /* order cqe stores with ring update */
Hristo Venev75b28af2019-08-26 17:23:46 +0000502 smp_store_release(&rings->cq.tail, ctx->cached_cq_tail);
Jens Axboe2b188cc2019-01-07 10:46:33 -0700503
Jens Axboe2b188cc2019-01-07 10:46:33 -0700504 if (wq_has_sleeper(&ctx->cq_wait)) {
505 wake_up_interruptible(&ctx->cq_wait);
506 kill_fasync(&ctx->cq_fasync, SIGIO, POLL_IN);
507 }
508 }
509}
510
Jens Axboe561fb042019-10-24 07:25:42 -0600511static inline bool io_sqe_needs_user(const struct io_uring_sqe *sqe)
Jens Axboe18d9be12019-09-10 09:13:05 -0600512{
Jens Axboe561fb042019-10-24 07:25:42 -0600513 u8 opcode = READ_ONCE(sqe->opcode);
514
515 return !(opcode == IORING_OP_READ_FIXED ||
516 opcode == IORING_OP_WRITE_FIXED);
517}
518
519static inline bool io_prep_async_work(struct io_kiocb *req)
520{
521 bool do_hashed = false;
Jens Axboe54a91f32019-09-10 09:15:04 -0600522
Jens Axboe6cc47d12019-09-18 11:18:23 -0600523 if (req->submit.sqe) {
524 switch (req->submit.sqe->opcode) {
525 case IORING_OP_WRITEV:
526 case IORING_OP_WRITE_FIXED:
Jens Axboe561fb042019-10-24 07:25:42 -0600527 do_hashed = true;
Jens Axboe5f8fd2d2019-11-07 10:57:36 -0700528 /* fall-through */
529 case IORING_OP_READV:
530 case IORING_OP_READ_FIXED:
531 case IORING_OP_SENDMSG:
532 case IORING_OP_RECVMSG:
533 case IORING_OP_ACCEPT:
534 case IORING_OP_POLL_ADD:
535 /*
536 * We know REQ_F_ISREG is not set on some of these
537 * opcodes, but this enables us to keep the check in
538 * just one place.
539 */
540 if (!(req->flags & REQ_F_ISREG))
541 req->work.flags |= IO_WQ_WORK_UNBOUND;
Jens Axboe6cc47d12019-09-18 11:18:23 -0600542 break;
543 }
Jens Axboe561fb042019-10-24 07:25:42 -0600544 if (io_sqe_needs_user(req->submit.sqe))
545 req->work.flags |= IO_WQ_WORK_NEEDS_USER;
Jens Axboe54a91f32019-09-10 09:15:04 -0600546 }
547
Jens Axboe561fb042019-10-24 07:25:42 -0600548 return do_hashed;
549}
550
Jackie Liua197f662019-11-08 08:09:12 -0700551static inline void io_queue_async_work(struct io_kiocb *req)
Jens Axboe561fb042019-10-24 07:25:42 -0600552{
553 bool do_hashed = io_prep_async_work(req);
Jackie Liua197f662019-11-08 08:09:12 -0700554 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe561fb042019-10-24 07:25:42 -0600555
556 trace_io_uring_queue_async_work(ctx, do_hashed, req, &req->work,
557 req->flags);
558 if (!do_hashed) {
559 io_wq_enqueue(ctx->io_wq, &req->work);
560 } else {
561 io_wq_enqueue_hashed(ctx->io_wq, &req->work,
562 file_inode(req->file));
563 }
Jens Axboe18d9be12019-09-10 09:13:05 -0600564}
565
Jens Axboe5262f562019-09-17 12:26:57 -0600566static void io_kill_timeout(struct io_kiocb *req)
567{
568 int ret;
569
570 ret = hrtimer_try_to_cancel(&req->timeout.timer);
571 if (ret != -1) {
572 atomic_inc(&req->ctx->cq_timeouts);
Jens Axboe842f9612019-10-29 12:34:10 -0600573 list_del_init(&req->list);
Jens Axboe78e19bb2019-11-06 15:21:34 -0700574 io_cqring_fill_event(req, 0);
Jackie Liuec9c02a2019-11-08 23:50:36 +0800575 io_put_req(req);
Jens Axboe5262f562019-09-17 12:26:57 -0600576 }
577}
578
579static void io_kill_timeouts(struct io_ring_ctx *ctx)
580{
581 struct io_kiocb *req, *tmp;
582
583 spin_lock_irq(&ctx->completion_lock);
584 list_for_each_entry_safe(req, tmp, &ctx->timeout_list, list)
585 io_kill_timeout(req);
586 spin_unlock_irq(&ctx->completion_lock);
587}
588
Jens Axboede0617e2019-04-06 21:51:27 -0600589static void io_commit_cqring(struct io_ring_ctx *ctx)
590{
591 struct io_kiocb *req;
592
Jens Axboe5262f562019-09-17 12:26:57 -0600593 while ((req = io_get_timeout_req(ctx)) != NULL)
594 io_kill_timeout(req);
595
Jens Axboede0617e2019-04-06 21:51:27 -0600596 __io_commit_cqring(ctx);
597
598 while ((req = io_get_deferred_req(ctx)) != NULL) {
Jackie Liu4fe2c962019-09-09 20:50:40 +0800599 if (req->flags & REQ_F_SHADOW_DRAIN) {
600 /* Just for drain, free it. */
601 __io_free_req(req);
602 continue;
603 }
Jens Axboede0617e2019-04-06 21:51:27 -0600604 req->flags |= REQ_F_IO_DRAINED;
Jackie Liua197f662019-11-08 08:09:12 -0700605 io_queue_async_work(req);
Jens Axboede0617e2019-04-06 21:51:27 -0600606 }
607}
608
Jens Axboe2b188cc2019-01-07 10:46:33 -0700609static struct io_uring_cqe *io_get_cqring(struct io_ring_ctx *ctx)
610{
Hristo Venev75b28af2019-08-26 17:23:46 +0000611 struct io_rings *rings = ctx->rings;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700612 unsigned tail;
613
614 tail = ctx->cached_cq_tail;
Stefan Bühler115e12e2019-04-24 23:54:18 +0200615 /*
616 * writes to the cq entry need to come after reading head; the
617 * control dependency is enough as we're using WRITE_ONCE to
618 * fill the cq entry
619 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000620 if (tail - READ_ONCE(rings->cq.head) == rings->cq_ring_entries)
Jens Axboe2b188cc2019-01-07 10:46:33 -0700621 return NULL;
622
623 ctx->cached_cq_tail++;
Hristo Venev75b28af2019-08-26 17:23:46 +0000624 return &rings->cqes[tail & ctx->cq_mask];
Jens Axboe2b188cc2019-01-07 10:46:33 -0700625}
626
Jens Axboe1d7bb1d2019-11-06 11:31:17 -0700627static void io_cqring_ev_posted(struct io_ring_ctx *ctx)
628{
629 if (waitqueue_active(&ctx->wait))
630 wake_up(&ctx->wait);
631 if (waitqueue_active(&ctx->sqo_wait))
632 wake_up(&ctx->sqo_wait);
633 if (ctx->cq_ev_fd)
634 eventfd_signal(ctx->cq_ev_fd, 1);
635}
636
637static void io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force)
638{
639 struct io_rings *rings = ctx->rings;
640 struct io_uring_cqe *cqe;
641 struct io_kiocb *req;
642 unsigned long flags;
643 LIST_HEAD(list);
644
645 if (!force) {
646 if (list_empty_careful(&ctx->cq_overflow_list))
647 return;
648 if ((ctx->cached_cq_tail - READ_ONCE(rings->cq.head) ==
649 rings->cq_ring_entries))
650 return;
651 }
652
653 spin_lock_irqsave(&ctx->completion_lock, flags);
654
655 /* if force is set, the ring is going away. always drop after that */
656 if (force)
657 ctx->cq_overflow_flushed = true;
658
659 while (!list_empty(&ctx->cq_overflow_list)) {
660 cqe = io_get_cqring(ctx);
661 if (!cqe && !force)
662 break;
663
664 req = list_first_entry(&ctx->cq_overflow_list, struct io_kiocb,
665 list);
666 list_move(&req->list, &list);
667 if (cqe) {
668 WRITE_ONCE(cqe->user_data, req->user_data);
669 WRITE_ONCE(cqe->res, req->result);
670 WRITE_ONCE(cqe->flags, 0);
671 } else {
672 WRITE_ONCE(ctx->rings->cq_overflow,
673 atomic_inc_return(&ctx->cached_cq_overflow));
674 }
675 }
676
677 io_commit_cqring(ctx);
678 spin_unlock_irqrestore(&ctx->completion_lock, flags);
679 io_cqring_ev_posted(ctx);
680
681 while (!list_empty(&list)) {
682 req = list_first_entry(&list, struct io_kiocb, list);
683 list_del(&req->list);
Jackie Liuec9c02a2019-11-08 23:50:36 +0800684 io_put_req(req);
Jens Axboe1d7bb1d2019-11-06 11:31:17 -0700685 }
686}
687
Jens Axboe78e19bb2019-11-06 15:21:34 -0700688static void io_cqring_fill_event(struct io_kiocb *req, long res)
Jens Axboe2b188cc2019-01-07 10:46:33 -0700689{
Jens Axboe78e19bb2019-11-06 15:21:34 -0700690 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700691 struct io_uring_cqe *cqe;
692
Jens Axboe78e19bb2019-11-06 15:21:34 -0700693 trace_io_uring_complete(ctx, req->user_data, res);
Jens Axboe51c3ff62019-11-03 06:52:50 -0700694
Jens Axboe2b188cc2019-01-07 10:46:33 -0700695 /*
696 * If we can't get a cq entry, userspace overflowed the
697 * submission (by quite a lot). Increment the overflow count in
698 * the ring.
699 */
700 cqe = io_get_cqring(ctx);
Jens Axboe1d7bb1d2019-11-06 11:31:17 -0700701 if (likely(cqe)) {
Jens Axboe78e19bb2019-11-06 15:21:34 -0700702 WRITE_ONCE(cqe->user_data, req->user_data);
Jens Axboe2b188cc2019-01-07 10:46:33 -0700703 WRITE_ONCE(cqe->res, res);
Jens Axboec71ffb62019-05-13 20:58:29 -0600704 WRITE_ONCE(cqe->flags, 0);
Jens Axboe1d7bb1d2019-11-06 11:31:17 -0700705 } else if (ctx->cq_overflow_flushed) {
Jens Axboe498ccd92019-10-25 10:04:25 -0600706 WRITE_ONCE(ctx->rings->cq_overflow,
707 atomic_inc_return(&ctx->cached_cq_overflow));
Jens Axboe1d7bb1d2019-11-06 11:31:17 -0700708 } else {
709 refcount_inc(&req->refs);
710 req->result = res;
711 list_add_tail(&req->list, &ctx->cq_overflow_list);
Jens Axboe2b188cc2019-01-07 10:46:33 -0700712 }
713}
714
Jens Axboe78e19bb2019-11-06 15:21:34 -0700715static void io_cqring_add_event(struct io_kiocb *req, long res)
Jens Axboe2b188cc2019-01-07 10:46:33 -0700716{
Jens Axboe78e19bb2019-11-06 15:21:34 -0700717 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700718 unsigned long flags;
719
720 spin_lock_irqsave(&ctx->completion_lock, flags);
Jens Axboe78e19bb2019-11-06 15:21:34 -0700721 io_cqring_fill_event(req, res);
Jens Axboe2b188cc2019-01-07 10:46:33 -0700722 io_commit_cqring(ctx);
723 spin_unlock_irqrestore(&ctx->completion_lock, flags);
724
Jens Axboe8c838782019-03-12 15:48:16 -0600725 io_cqring_ev_posted(ctx);
Jens Axboe2b188cc2019-01-07 10:46:33 -0700726}
727
Jens Axboe0ddf92e2019-11-08 08:52:53 -0700728static inline bool io_is_fallback_req(struct io_kiocb *req)
729{
730 return req == (struct io_kiocb *)
731 ((unsigned long) req->ctx->fallback_req & ~1UL);
732}
733
734static struct io_kiocb *io_get_fallback_req(struct io_ring_ctx *ctx)
735{
736 struct io_kiocb *req;
737
738 req = ctx->fallback_req;
739 if (!test_and_set_bit_lock(0, (unsigned long *) ctx->fallback_req))
740 return req;
741
742 return NULL;
743}
744
Jens Axboe2579f912019-01-09 09:10:43 -0700745static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx,
746 struct io_submit_state *state)
Jens Axboe2b188cc2019-01-07 10:46:33 -0700747{
Jens Axboefd6fab22019-03-14 16:30:06 -0600748 gfp_t gfp = GFP_KERNEL | __GFP_NOWARN;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700749 struct io_kiocb *req;
750
751 if (!percpu_ref_tryget(&ctx->refs))
752 return NULL;
753
Jens Axboe2579f912019-01-09 09:10:43 -0700754 if (!state) {
Jens Axboefd6fab22019-03-14 16:30:06 -0600755 req = kmem_cache_alloc(req_cachep, gfp);
Jens Axboe2579f912019-01-09 09:10:43 -0700756 if (unlikely(!req))
Jens Axboe0ddf92e2019-11-08 08:52:53 -0700757 goto fallback;
Jens Axboe2579f912019-01-09 09:10:43 -0700758 } else if (!state->free_reqs) {
759 size_t sz;
760 int ret;
761
762 sz = min_t(size_t, state->ios_left, ARRAY_SIZE(state->reqs));
Jens Axboefd6fab22019-03-14 16:30:06 -0600763 ret = kmem_cache_alloc_bulk(req_cachep, gfp, sz, state->reqs);
764
765 /*
766 * Bulk alloc is all-or-nothing. If we fail to get a batch,
767 * retry single alloc to be on the safe side.
768 */
769 if (unlikely(ret <= 0)) {
770 state->reqs[0] = kmem_cache_alloc(req_cachep, gfp);
771 if (!state->reqs[0])
Jens Axboe0ddf92e2019-11-08 08:52:53 -0700772 goto fallback;
Jens Axboefd6fab22019-03-14 16:30:06 -0600773 ret = 1;
774 }
Jens Axboe2579f912019-01-09 09:10:43 -0700775 state->free_reqs = ret - 1;
776 state->cur_req = 1;
777 req = state->reqs[0];
778 } else {
779 req = state->reqs[state->cur_req];
780 state->free_reqs--;
781 state->cur_req++;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700782 }
783
Jens Axboe0ddf92e2019-11-08 08:52:53 -0700784got_it:
Jens Axboe60c112b2019-06-21 10:20:18 -0600785 req->file = NULL;
Jens Axboe2579f912019-01-09 09:10:43 -0700786 req->ctx = ctx;
787 req->flags = 0;
Jens Axboee65ef562019-03-12 10:16:44 -0600788 /* one is dropped after submission, the other at completion */
789 refcount_set(&req->refs, 2);
Jens Axboe9e645e112019-05-10 16:07:28 -0600790 req->result = 0;
Jens Axboe561fb042019-10-24 07:25:42 -0600791 INIT_IO_WORK(&req->work, io_wq_submit_work);
Jens Axboe2579f912019-01-09 09:10:43 -0700792 return req;
Jens Axboe0ddf92e2019-11-08 08:52:53 -0700793fallback:
794 req = io_get_fallback_req(ctx);
795 if (req)
796 goto got_it;
Pavel Begunkov6805b322019-10-08 02:18:42 +0300797 percpu_ref_put(&ctx->refs);
Jens Axboe2b188cc2019-01-07 10:46:33 -0700798 return NULL;
799}
800
Jens Axboedef596e2019-01-09 08:59:42 -0700801static void io_free_req_many(struct io_ring_ctx *ctx, void **reqs, int *nr)
802{
803 if (*nr) {
804 kmem_cache_free_bulk(req_cachep, *nr, reqs);
Pavel Begunkov6805b322019-10-08 02:18:42 +0300805 percpu_ref_put_many(&ctx->refs, *nr);
Jens Axboedef596e2019-01-09 08:59:42 -0700806 *nr = 0;
807 }
808}
809
Jens Axboe9e645e112019-05-10 16:07:28 -0600810static void __io_free_req(struct io_kiocb *req)
Jens Axboe2b188cc2019-01-07 10:46:33 -0700811{
Jens Axboefcb323c2019-10-24 12:39:47 -0600812 struct io_ring_ctx *ctx = req->ctx;
813
Jens Axboe09bb8392019-03-13 12:39:28 -0600814 if (req->file && !(req->flags & REQ_F_FIXED_FILE))
815 fput(req->file);
Jens Axboefcb323c2019-10-24 12:39:47 -0600816 if (req->flags & REQ_F_INFLIGHT) {
817 unsigned long flags;
818
819 spin_lock_irqsave(&ctx->inflight_lock, flags);
820 list_del(&req->inflight_entry);
821 if (waitqueue_active(&ctx->inflight_wait))
822 wake_up(&ctx->inflight_wait);
823 spin_unlock_irqrestore(&ctx->inflight_lock, flags);
824 }
825 percpu_ref_put(&ctx->refs);
Jens Axboe0ddf92e2019-11-08 08:52:53 -0700826 if (likely(!io_is_fallback_req(req)))
827 kmem_cache_free(req_cachep, req);
828 else
829 clear_bit_unlock(0, (unsigned long *) ctx->fallback_req);
Jens Axboee65ef562019-03-12 10:16:44 -0600830}
831
Jackie Liua197f662019-11-08 08:09:12 -0700832static bool io_link_cancel_timeout(struct io_kiocb *req)
Jens Axboe2665abf2019-11-05 12:40:47 -0700833{
Jackie Liua197f662019-11-08 08:09:12 -0700834 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe2665abf2019-11-05 12:40:47 -0700835 int ret;
836
837 ret = hrtimer_try_to_cancel(&req->timeout.timer);
838 if (ret != -1) {
Jens Axboe78e19bb2019-11-06 15:21:34 -0700839 io_cqring_fill_event(req, -ECANCELED);
Jens Axboe2665abf2019-11-05 12:40:47 -0700840 io_commit_cqring(ctx);
841 req->flags &= ~REQ_F_LINK;
Jackie Liuec9c02a2019-11-08 23:50:36 +0800842 io_put_req(req);
Jens Axboe2665abf2019-11-05 12:40:47 -0700843 return true;
844 }
845
846 return false;
847}
848
Jens Axboeba816ad2019-09-28 11:36:45 -0600849static void io_req_link_next(struct io_kiocb *req, struct io_kiocb **nxtptr)
Jens Axboe9e645e112019-05-10 16:07:28 -0600850{
Jens Axboe2665abf2019-11-05 12:40:47 -0700851 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe9e645e112019-05-10 16:07:28 -0600852 struct io_kiocb *nxt;
Jens Axboe2665abf2019-11-05 12:40:47 -0700853 bool wake_ev = false;
Jens Axboe9e645e112019-05-10 16:07:28 -0600854
855 /*
856 * The list should never be empty when we are called here. But could
857 * potentially happen if the chain is messed up, check to be on the
858 * safe side.
859 */
860 nxt = list_first_entry_or_null(&req->link_list, struct io_kiocb, list);
Jens Axboe2665abf2019-11-05 12:40:47 -0700861 while (nxt) {
Jens Axboe76a46e02019-11-10 23:34:16 -0700862 list_del_init(&nxt->list);
Jens Axboe9e645e112019-05-10 16:07:28 -0600863 if (!list_empty(&req->link_list)) {
864 INIT_LIST_HEAD(&nxt->link_list);
865 list_splice(&req->link_list, &nxt->link_list);
866 nxt->flags |= REQ_F_LINK;
867 }
868
Jens Axboeba816ad2019-09-28 11:36:45 -0600869 /*
870 * If we're in async work, we can continue processing the chain
871 * in this context instead of having to queue up new async work.
872 */
Jens Axboe2665abf2019-11-05 12:40:47 -0700873 if (req->flags & REQ_F_LINK_TIMEOUT) {
Jackie Liua197f662019-11-08 08:09:12 -0700874 wake_ev = io_link_cancel_timeout(nxt);
Jens Axboe2665abf2019-11-05 12:40:47 -0700875
876 /* we dropped this link, get next */
877 nxt = list_first_entry_or_null(&req->link_list,
878 struct io_kiocb, list);
Jens Axboe960e4322019-11-12 07:56:39 -0700879 } else if (nxtptr && io_wq_current_is_worker()) {
Jens Axboeba816ad2019-09-28 11:36:45 -0600880 *nxtptr = nxt;
Jens Axboe2665abf2019-11-05 12:40:47 -0700881 break;
882 } else {
Jackie Liua197f662019-11-08 08:09:12 -0700883 io_queue_async_work(nxt);
Jens Axboe2665abf2019-11-05 12:40:47 -0700884 break;
885 }
Jens Axboe9e645e112019-05-10 16:07:28 -0600886 }
Jens Axboe2665abf2019-11-05 12:40:47 -0700887
888 if (wake_ev)
889 io_cqring_ev_posted(ctx);
Jens Axboe9e645e112019-05-10 16:07:28 -0600890}
891
892/*
893 * Called if REQ_F_LINK is set, and we fail the head request
894 */
895static void io_fail_links(struct io_kiocb *req)
896{
Jens Axboe2665abf2019-11-05 12:40:47 -0700897 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe9e645e112019-05-10 16:07:28 -0600898 struct io_kiocb *link;
Jens Axboe2665abf2019-11-05 12:40:47 -0700899 unsigned long flags;
900
901 spin_lock_irqsave(&ctx->completion_lock, flags);
Jens Axboe9e645e112019-05-10 16:07:28 -0600902
903 while (!list_empty(&req->link_list)) {
904 link = list_first_entry(&req->link_list, struct io_kiocb, list);
Jens Axboe2665abf2019-11-05 12:40:47 -0700905 list_del_init(&link->list);
Jens Axboe9e645e112019-05-10 16:07:28 -0600906
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +0200907 trace_io_uring_fail_link(req, link);
Jens Axboe2665abf2019-11-05 12:40:47 -0700908
909 if ((req->flags & REQ_F_LINK_TIMEOUT) &&
910 link->submit.sqe->opcode == IORING_OP_LINK_TIMEOUT) {
Jackie Liua197f662019-11-08 08:09:12 -0700911 io_link_cancel_timeout(link);
Jens Axboe2665abf2019-11-05 12:40:47 -0700912 } else {
Jens Axboe78e19bb2019-11-06 15:21:34 -0700913 io_cqring_fill_event(link, -ECANCELED);
914 io_double_put_req(link);
Jens Axboe2665abf2019-11-05 12:40:47 -0700915 }
Jens Axboe9e645e112019-05-10 16:07:28 -0600916 }
Jens Axboe2665abf2019-11-05 12:40:47 -0700917
918 io_commit_cqring(ctx);
919 spin_unlock_irqrestore(&ctx->completion_lock, flags);
920 io_cqring_ev_posted(ctx);
Jens Axboe9e645e112019-05-10 16:07:28 -0600921}
922
Jackie Liuc69f8db2019-11-09 11:00:08 +0800923static void io_free_req_find_next(struct io_kiocb *req, struct io_kiocb **nxt)
Jens Axboe9e645e112019-05-10 16:07:28 -0600924{
Jens Axboe2665abf2019-11-05 12:40:47 -0700925 if (likely(!(req->flags & REQ_F_LINK))) {
926 __io_free_req(req);
927 return;
928 }
929
Jens Axboe9e645e112019-05-10 16:07:28 -0600930 /*
931 * If LINK is set, we have dependent requests in this chain. If we
932 * didn't fail this request, queue the first one up, moving any other
933 * dependencies to the next request. In case of failure, fail the rest
934 * of the chain.
935 */
Jens Axboe2665abf2019-11-05 12:40:47 -0700936 if (req->flags & REQ_F_FAIL_LINK) {
937 io_fail_links(req);
Jens Axboe7c9e7f02019-11-12 08:15:53 -0700938 } else if ((req->flags & (REQ_F_LINK_TIMEOUT | REQ_F_COMP_LOCKED)) ==
939 REQ_F_LINK_TIMEOUT) {
Jens Axboe2665abf2019-11-05 12:40:47 -0700940 struct io_ring_ctx *ctx = req->ctx;
941 unsigned long flags;
942
943 /*
944 * If this is a timeout link, we could be racing with the
945 * timeout timer. Grab the completion lock for this case to
Jens Axboe7c9e7f02019-11-12 08:15:53 -0700946 * protect against that.
Jens Axboe2665abf2019-11-05 12:40:47 -0700947 */
948 spin_lock_irqsave(&ctx->completion_lock, flags);
949 io_req_link_next(req, nxt);
950 spin_unlock_irqrestore(&ctx->completion_lock, flags);
951 } else {
952 io_req_link_next(req, nxt);
Jens Axboe9e645e112019-05-10 16:07:28 -0600953 }
954
955 __io_free_req(req);
956}
957
Jackie Liuc69f8db2019-11-09 11:00:08 +0800958static void io_free_req(struct io_kiocb *req)
959{
960 io_free_req_find_next(req, NULL);
961}
962
Jens Axboeba816ad2019-09-28 11:36:45 -0600963/*
964 * Drop reference to request, return next in chain (if there is one) if this
965 * was the last reference to this request.
966 */
Jackie Liuec9c02a2019-11-08 23:50:36 +0800967static void io_put_req_find_next(struct io_kiocb *req, struct io_kiocb **nxtptr)
Jens Axboee65ef562019-03-12 10:16:44 -0600968{
Jens Axboeba816ad2019-09-28 11:36:45 -0600969 struct io_kiocb *nxt = NULL;
970
Jens Axboee65ef562019-03-12 10:16:44 -0600971 if (refcount_dec_and_test(&req->refs))
Jackie Liuc69f8db2019-11-09 11:00:08 +0800972 io_free_req_find_next(req, &nxt);
Jens Axboeba816ad2019-09-28 11:36:45 -0600973
Jens Axboeba816ad2019-09-28 11:36:45 -0600974 if (nxt) {
Jens Axboe561fb042019-10-24 07:25:42 -0600975 if (nxtptr)
Jens Axboeba816ad2019-09-28 11:36:45 -0600976 *nxtptr = nxt;
Jens Axboe561fb042019-10-24 07:25:42 -0600977 else
Jackie Liua197f662019-11-08 08:09:12 -0700978 io_queue_async_work(nxt);
Jens Axboeba816ad2019-09-28 11:36:45 -0600979 }
Jens Axboe2b188cc2019-01-07 10:46:33 -0700980}
981
Jackie Liuec9c02a2019-11-08 23:50:36 +0800982static void io_put_req(struct io_kiocb *req)
983{
984 if (refcount_dec_and_test(&req->refs))
Jackie Liuc69f8db2019-11-09 11:00:08 +0800985 io_free_req(req);
Jackie Liuec9c02a2019-11-08 23:50:36 +0800986}
987
Jens Axboe78e19bb2019-11-06 15:21:34 -0700988static void io_double_put_req(struct io_kiocb *req)
989{
990 /* drop both submit and complete references */
991 if (refcount_sub_and_test(2, &req->refs))
992 __io_free_req(req);
993}
994
Jens Axboe1d7bb1d2019-11-06 11:31:17 -0700995static unsigned io_cqring_events(struct io_ring_ctx *ctx, bool noflush)
Jens Axboea3a0e432019-08-20 11:03:11 -0600996{
Jens Axboe84f97dc2019-11-06 11:27:53 -0700997 struct io_rings *rings = ctx->rings;
998
Jens Axboe1d7bb1d2019-11-06 11:31:17 -0700999 /*
1000 * noflush == true is from the waitqueue handler, just ensure we wake
1001 * up the task, and the next invocation will flush the entries. We
1002 * cannot safely to it from here.
1003 */
1004 if (noflush && !list_empty(&ctx->cq_overflow_list))
1005 return -1U;
1006
1007 io_cqring_overflow_flush(ctx, false);
1008
Jens Axboea3a0e432019-08-20 11:03:11 -06001009 /* See comment at the top of this file */
1010 smp_rmb();
Hristo Venev75b28af2019-08-26 17:23:46 +00001011 return READ_ONCE(rings->cq.tail) - READ_ONCE(rings->cq.head);
Jens Axboea3a0e432019-08-20 11:03:11 -06001012}
1013
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03001014static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx)
1015{
1016 struct io_rings *rings = ctx->rings;
1017
1018 /* make sure SQ entry isn't read before tail */
1019 return smp_load_acquire(&rings->sq.tail) - ctx->cached_sq_head;
1020}
1021
Jens Axboedef596e2019-01-09 08:59:42 -07001022/*
1023 * Find and free completed poll iocbs
1024 */
1025static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events,
1026 struct list_head *done)
1027{
1028 void *reqs[IO_IOPOLL_BATCH];
1029 struct io_kiocb *req;
Jens Axboe09bb8392019-03-13 12:39:28 -06001030 int to_free;
Jens Axboedef596e2019-01-09 08:59:42 -07001031
Jens Axboe09bb8392019-03-13 12:39:28 -06001032 to_free = 0;
Jens Axboedef596e2019-01-09 08:59:42 -07001033 while (!list_empty(done)) {
1034 req = list_first_entry(done, struct io_kiocb, list);
1035 list_del(&req->list);
1036
Jens Axboe78e19bb2019-11-06 15:21:34 -07001037 io_cqring_fill_event(req, req->result);
Jens Axboedef596e2019-01-09 08:59:42 -07001038 (*nr_events)++;
1039
Jens Axboe09bb8392019-03-13 12:39:28 -06001040 if (refcount_dec_and_test(&req->refs)) {
1041 /* If we're not using fixed files, we have to pair the
1042 * completion part with the file put. Use regular
1043 * completions for those, only batch free for fixed
Jens Axboe9e645e112019-05-10 16:07:28 -06001044 * file and non-linked commands.
Jens Axboe09bb8392019-03-13 12:39:28 -06001045 */
Jens Axboe0ddf92e2019-11-08 08:52:53 -07001046 if (((req->flags & (REQ_F_FIXED_FILE|REQ_F_LINK)) ==
1047 REQ_F_FIXED_FILE) && !io_is_fallback_req(req)) {
Jens Axboe09bb8392019-03-13 12:39:28 -06001048 reqs[to_free++] = req;
1049 if (to_free == ARRAY_SIZE(reqs))
1050 io_free_req_many(ctx, reqs, &to_free);
Jens Axboe6b063142019-01-10 22:13:58 -07001051 } else {
Jackie Liuc69f8db2019-11-09 11:00:08 +08001052 io_free_req(req);
Jens Axboe6b063142019-01-10 22:13:58 -07001053 }
Jens Axboe9a56a232019-01-09 09:06:50 -07001054 }
Jens Axboedef596e2019-01-09 08:59:42 -07001055 }
Jens Axboedef596e2019-01-09 08:59:42 -07001056
Jens Axboe09bb8392019-03-13 12:39:28 -06001057 io_commit_cqring(ctx);
Jens Axboedef596e2019-01-09 08:59:42 -07001058 io_free_req_many(ctx, reqs, &to_free);
1059}
1060
1061static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events,
1062 long min)
1063{
1064 struct io_kiocb *req, *tmp;
1065 LIST_HEAD(done);
1066 bool spin;
1067 int ret;
1068
1069 /*
1070 * Only spin for completions if we don't have multiple devices hanging
1071 * off our complete list, and we're under the requested amount.
1072 */
1073 spin = !ctx->poll_multi_file && *nr_events < min;
1074
1075 ret = 0;
1076 list_for_each_entry_safe(req, tmp, &ctx->poll_list, list) {
1077 struct kiocb *kiocb = &req->rw;
1078
1079 /*
1080 * Move completed entries to our local list. If we find a
1081 * request that requires polling, break out and complete
1082 * the done list first, if we have entries there.
1083 */
1084 if (req->flags & REQ_F_IOPOLL_COMPLETED) {
1085 list_move_tail(&req->list, &done);
1086 continue;
1087 }
1088 if (!list_empty(&done))
1089 break;
1090
1091 ret = kiocb->ki_filp->f_op->iopoll(kiocb, spin);
1092 if (ret < 0)
1093 break;
1094
1095 if (ret && spin)
1096 spin = false;
1097 ret = 0;
1098 }
1099
1100 if (!list_empty(&done))
1101 io_iopoll_complete(ctx, nr_events, &done);
1102
1103 return ret;
1104}
1105
1106/*
1107 * Poll for a mininum of 'min' events. Note that if min == 0 we consider that a
1108 * non-spinning poll check - we'll still enter the driver poll loop, but only
1109 * as a non-spinning completion check.
1110 */
1111static int io_iopoll_getevents(struct io_ring_ctx *ctx, unsigned int *nr_events,
1112 long min)
1113{
Jens Axboe08f54392019-08-21 22:19:11 -06001114 while (!list_empty(&ctx->poll_list) && !need_resched()) {
Jens Axboedef596e2019-01-09 08:59:42 -07001115 int ret;
1116
1117 ret = io_do_iopoll(ctx, nr_events, min);
1118 if (ret < 0)
1119 return ret;
1120 if (!min || *nr_events >= min)
1121 return 0;
1122 }
1123
1124 return 1;
1125}
1126
1127/*
1128 * We can't just wait for polled events to come to us, we have to actively
1129 * find and complete them.
1130 */
1131static void io_iopoll_reap_events(struct io_ring_ctx *ctx)
1132{
1133 if (!(ctx->flags & IORING_SETUP_IOPOLL))
1134 return;
1135
1136 mutex_lock(&ctx->uring_lock);
1137 while (!list_empty(&ctx->poll_list)) {
1138 unsigned int nr_events = 0;
1139
1140 io_iopoll_getevents(ctx, &nr_events, 1);
Jens Axboe08f54392019-08-21 22:19:11 -06001141
1142 /*
1143 * Ensure we allow local-to-the-cpu processing to take place,
1144 * in this case we need to ensure that we reap all events.
1145 */
1146 cond_resched();
Jens Axboedef596e2019-01-09 08:59:42 -07001147 }
1148 mutex_unlock(&ctx->uring_lock);
1149}
1150
Jens Axboe2b2ed972019-10-25 10:06:15 -06001151static int __io_iopoll_check(struct io_ring_ctx *ctx, unsigned *nr_events,
1152 long min)
Jens Axboedef596e2019-01-09 08:59:42 -07001153{
Jens Axboe2b2ed972019-10-25 10:06:15 -06001154 int iters = 0, ret = 0;
Jens Axboedef596e2019-01-09 08:59:42 -07001155
1156 do {
1157 int tmin = 0;
1158
Jens Axboe500f9fb2019-08-19 12:15:59 -06001159 /*
Jens Axboea3a0e432019-08-20 11:03:11 -06001160 * Don't enter poll loop if we already have events pending.
1161 * If we do, we can potentially be spinning for commands that
1162 * already triggered a CQE (eg in error).
1163 */
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001164 if (io_cqring_events(ctx, false))
Jens Axboea3a0e432019-08-20 11:03:11 -06001165 break;
1166
1167 /*
Jens Axboe500f9fb2019-08-19 12:15:59 -06001168 * If a submit got punted to a workqueue, we can have the
1169 * application entering polling for a command before it gets
1170 * issued. That app will hold the uring_lock for the duration
1171 * of the poll right here, so we need to take a breather every
1172 * now and then to ensure that the issue has a chance to add
1173 * the poll to the issued list. Otherwise we can spin here
1174 * forever, while the workqueue is stuck trying to acquire the
1175 * very same mutex.
1176 */
1177 if (!(++iters & 7)) {
1178 mutex_unlock(&ctx->uring_lock);
1179 mutex_lock(&ctx->uring_lock);
1180 }
1181
Jens Axboedef596e2019-01-09 08:59:42 -07001182 if (*nr_events < min)
1183 tmin = min - *nr_events;
1184
1185 ret = io_iopoll_getevents(ctx, nr_events, tmin);
1186 if (ret <= 0)
1187 break;
1188 ret = 0;
1189 } while (min && !*nr_events && !need_resched());
1190
Jens Axboe2b2ed972019-10-25 10:06:15 -06001191 return ret;
1192}
1193
1194static int io_iopoll_check(struct io_ring_ctx *ctx, unsigned *nr_events,
1195 long min)
1196{
1197 int ret;
1198
1199 /*
1200 * We disallow the app entering submit/complete with polling, but we
1201 * still need to lock the ring to prevent racing with polled issue
1202 * that got punted to a workqueue.
1203 */
1204 mutex_lock(&ctx->uring_lock);
1205 ret = __io_iopoll_check(ctx, nr_events, min);
Jens Axboe500f9fb2019-08-19 12:15:59 -06001206 mutex_unlock(&ctx->uring_lock);
Jens Axboedef596e2019-01-09 08:59:42 -07001207 return ret;
1208}
1209
Jens Axboe491381ce2019-10-17 09:20:46 -06001210static void kiocb_end_write(struct io_kiocb *req)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001211{
Jens Axboe491381ce2019-10-17 09:20:46 -06001212 /*
1213 * Tell lockdep we inherited freeze protection from submission
1214 * thread.
1215 */
1216 if (req->flags & REQ_F_ISREG) {
1217 struct inode *inode = file_inode(req->file);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001218
Jens Axboe491381ce2019-10-17 09:20:46 -06001219 __sb_writers_acquired(inode->i_sb, SB_FREEZE_WRITE);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001220 }
Jens Axboe491381ce2019-10-17 09:20:46 -06001221 file_end_write(req->file);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001222}
1223
Jens Axboeba816ad2019-09-28 11:36:45 -06001224static void io_complete_rw_common(struct kiocb *kiocb, long res)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001225{
1226 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw);
1227
Jens Axboe491381ce2019-10-17 09:20:46 -06001228 if (kiocb->ki_flags & IOCB_WRITE)
1229 kiocb_end_write(req);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001230
Jens Axboe9e645e112019-05-10 16:07:28 -06001231 if ((req->flags & REQ_F_LINK) && res != req->result)
1232 req->flags |= REQ_F_FAIL_LINK;
Jens Axboe78e19bb2019-11-06 15:21:34 -07001233 io_cqring_add_event(req, res);
Jens Axboeba816ad2019-09-28 11:36:45 -06001234}
1235
1236static void io_complete_rw(struct kiocb *kiocb, long res, long res2)
1237{
1238 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw);
1239
1240 io_complete_rw_common(kiocb, res);
Jackie Liuec9c02a2019-11-08 23:50:36 +08001241 io_put_req(req);
Jens Axboeba816ad2019-09-28 11:36:45 -06001242}
1243
1244static struct io_kiocb *__io_complete_rw(struct kiocb *kiocb, long res)
1245{
1246 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw);
Jackie Liuec9c02a2019-11-08 23:50:36 +08001247 struct io_kiocb *nxt = NULL;
Jens Axboeba816ad2019-09-28 11:36:45 -06001248
1249 io_complete_rw_common(kiocb, res);
Jackie Liuec9c02a2019-11-08 23:50:36 +08001250 io_put_req_find_next(req, &nxt);
1251
1252 return nxt;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001253}
1254
Jens Axboedef596e2019-01-09 08:59:42 -07001255static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2)
1256{
1257 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw);
1258
Jens Axboe491381ce2019-10-17 09:20:46 -06001259 if (kiocb->ki_flags & IOCB_WRITE)
1260 kiocb_end_write(req);
Jens Axboedef596e2019-01-09 08:59:42 -07001261
Jens Axboe9e645e112019-05-10 16:07:28 -06001262 if ((req->flags & REQ_F_LINK) && res != req->result)
1263 req->flags |= REQ_F_FAIL_LINK;
1264 req->result = res;
Jens Axboedef596e2019-01-09 08:59:42 -07001265 if (res != -EAGAIN)
1266 req->flags |= REQ_F_IOPOLL_COMPLETED;
1267}
1268
1269/*
1270 * After the iocb has been issued, it's safe to be found on the poll list.
1271 * Adding the kiocb to the list AFTER submission ensures that we don't
1272 * find it from a io_iopoll_getevents() thread before the issuer is done
1273 * accessing the kiocb cookie.
1274 */
1275static void io_iopoll_req_issued(struct io_kiocb *req)
1276{
1277 struct io_ring_ctx *ctx = req->ctx;
1278
1279 /*
1280 * Track whether we have multiple files in our lists. This will impact
1281 * how we do polling eventually, not spinning if we're on potentially
1282 * different devices.
1283 */
1284 if (list_empty(&ctx->poll_list)) {
1285 ctx->poll_multi_file = false;
1286 } else if (!ctx->poll_multi_file) {
1287 struct io_kiocb *list_req;
1288
1289 list_req = list_first_entry(&ctx->poll_list, struct io_kiocb,
1290 list);
1291 if (list_req->rw.ki_filp != req->rw.ki_filp)
1292 ctx->poll_multi_file = true;
1293 }
1294
1295 /*
1296 * For fast devices, IO may have already completed. If it has, add
1297 * it to the front so we find it first.
1298 */
1299 if (req->flags & REQ_F_IOPOLL_COMPLETED)
1300 list_add(&req->list, &ctx->poll_list);
1301 else
1302 list_add_tail(&req->list, &ctx->poll_list);
1303}
1304
Jens Axboe3d6770f2019-04-13 11:50:54 -06001305static void io_file_put(struct io_submit_state *state)
Jens Axboe9a56a232019-01-09 09:06:50 -07001306{
Jens Axboe3d6770f2019-04-13 11:50:54 -06001307 if (state->file) {
Jens Axboe9a56a232019-01-09 09:06:50 -07001308 int diff = state->has_refs - state->used_refs;
1309
1310 if (diff)
1311 fput_many(state->file, diff);
1312 state->file = NULL;
1313 }
1314}
1315
1316/*
1317 * Get as many references to a file as we have IOs left in this submission,
1318 * assuming most submissions are for one file, or at least that each file
1319 * has more than one submission.
1320 */
1321static struct file *io_file_get(struct io_submit_state *state, int fd)
1322{
1323 if (!state)
1324 return fget(fd);
1325
1326 if (state->file) {
1327 if (state->fd == fd) {
1328 state->used_refs++;
1329 state->ios_left--;
1330 return state->file;
1331 }
Jens Axboe3d6770f2019-04-13 11:50:54 -06001332 io_file_put(state);
Jens Axboe9a56a232019-01-09 09:06:50 -07001333 }
1334 state->file = fget_many(fd, state->ios_left);
1335 if (!state->file)
1336 return NULL;
1337
1338 state->fd = fd;
1339 state->has_refs = state->ios_left;
1340 state->used_refs = 1;
1341 state->ios_left--;
1342 return state->file;
1343}
1344
Jens Axboe2b188cc2019-01-07 10:46:33 -07001345/*
1346 * If we tracked the file through the SCM inflight mechanism, we could support
1347 * any file. For now, just ensure that anything potentially problematic is done
1348 * inline.
1349 */
1350static bool io_file_supports_async(struct file *file)
1351{
1352 umode_t mode = file_inode(file)->i_mode;
1353
1354 if (S_ISBLK(mode) || S_ISCHR(mode))
1355 return true;
1356 if (S_ISREG(mode) && file->f_op != &io_uring_fops)
1357 return true;
1358
1359 return false;
1360}
1361
Pavel Begunkov267bc902019-11-07 01:41:08 +03001362static int io_prep_rw(struct io_kiocb *req, bool force_nonblock)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001363{
Pavel Begunkov267bc902019-11-07 01:41:08 +03001364 const struct io_uring_sqe *sqe = req->submit.sqe;
Jens Axboedef596e2019-01-09 08:59:42 -07001365 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001366 struct kiocb *kiocb = &req->rw;
Jens Axboe09bb8392019-03-13 12:39:28 -06001367 unsigned ioprio;
1368 int ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001369
Jens Axboe09bb8392019-03-13 12:39:28 -06001370 if (!req->file)
1371 return -EBADF;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001372
Jens Axboe491381ce2019-10-17 09:20:46 -06001373 if (S_ISREG(file_inode(req->file)->i_mode))
1374 req->flags |= REQ_F_ISREG;
1375
1376 /*
1377 * If the file doesn't support async, mark it as REQ_F_MUST_PUNT so
1378 * we know to async punt it even if it was opened O_NONBLOCK
1379 */
1380 if (force_nonblock && !io_file_supports_async(req->file)) {
1381 req->flags |= REQ_F_MUST_PUNT;
1382 return -EAGAIN;
1383 }
Jens Axboe6b063142019-01-10 22:13:58 -07001384
Jens Axboe2b188cc2019-01-07 10:46:33 -07001385 kiocb->ki_pos = READ_ONCE(sqe->off);
1386 kiocb->ki_flags = iocb_flags(kiocb->ki_filp);
1387 kiocb->ki_hint = ki_hint_validate(file_write_hint(kiocb->ki_filp));
1388
1389 ioprio = READ_ONCE(sqe->ioprio);
1390 if (ioprio) {
1391 ret = ioprio_check_cap(ioprio);
1392 if (ret)
Jens Axboe09bb8392019-03-13 12:39:28 -06001393 return ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001394
1395 kiocb->ki_ioprio = ioprio;
1396 } else
1397 kiocb->ki_ioprio = get_current_ioprio();
1398
1399 ret = kiocb_set_rw_flags(kiocb, READ_ONCE(sqe->rw_flags));
1400 if (unlikely(ret))
Jens Axboe09bb8392019-03-13 12:39:28 -06001401 return ret;
Stefan Bühler8449eed2019-04-27 20:34:19 +02001402
1403 /* don't allow async punt if RWF_NOWAIT was requested */
Jens Axboe491381ce2019-10-17 09:20:46 -06001404 if ((kiocb->ki_flags & IOCB_NOWAIT) ||
1405 (req->file->f_flags & O_NONBLOCK))
Stefan Bühler8449eed2019-04-27 20:34:19 +02001406 req->flags |= REQ_F_NOWAIT;
1407
1408 if (force_nonblock)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001409 kiocb->ki_flags |= IOCB_NOWAIT;
Stefan Bühler8449eed2019-04-27 20:34:19 +02001410
Jens Axboedef596e2019-01-09 08:59:42 -07001411 if (ctx->flags & IORING_SETUP_IOPOLL) {
Jens Axboedef596e2019-01-09 08:59:42 -07001412 if (!(kiocb->ki_flags & IOCB_DIRECT) ||
1413 !kiocb->ki_filp->f_op->iopoll)
Jens Axboe09bb8392019-03-13 12:39:28 -06001414 return -EOPNOTSUPP;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001415
Jens Axboedef596e2019-01-09 08:59:42 -07001416 kiocb->ki_flags |= IOCB_HIPRI;
1417 kiocb->ki_complete = io_complete_rw_iopoll;
1418 } else {
Jens Axboe09bb8392019-03-13 12:39:28 -06001419 if (kiocb->ki_flags & IOCB_HIPRI)
1420 return -EINVAL;
Jens Axboedef596e2019-01-09 08:59:42 -07001421 kiocb->ki_complete = io_complete_rw;
1422 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07001423 return 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001424}
1425
1426static inline void io_rw_done(struct kiocb *kiocb, ssize_t ret)
1427{
1428 switch (ret) {
1429 case -EIOCBQUEUED:
1430 break;
1431 case -ERESTARTSYS:
1432 case -ERESTARTNOINTR:
1433 case -ERESTARTNOHAND:
1434 case -ERESTART_RESTARTBLOCK:
1435 /*
1436 * We can't just restart the syscall, since previously
1437 * submitted sqes may already be in progress. Just fail this
1438 * IO with EINTR.
1439 */
1440 ret = -EINTR;
1441 /* fall through */
1442 default:
1443 kiocb->ki_complete(kiocb, ret, 0);
1444 }
1445}
1446
Jens Axboeba816ad2019-09-28 11:36:45 -06001447static void kiocb_done(struct kiocb *kiocb, ssize_t ret, struct io_kiocb **nxt,
1448 bool in_async)
1449{
1450 if (in_async && ret >= 0 && nxt && kiocb->ki_complete == io_complete_rw)
1451 *nxt = __io_complete_rw(kiocb, ret);
1452 else
1453 io_rw_done(kiocb, ret);
1454}
1455
Jens Axboeedafcce2019-01-09 09:16:05 -07001456static int io_import_fixed(struct io_ring_ctx *ctx, int rw,
1457 const struct io_uring_sqe *sqe,
1458 struct iov_iter *iter)
1459{
1460 size_t len = READ_ONCE(sqe->len);
1461 struct io_mapped_ubuf *imu;
1462 unsigned index, buf_index;
1463 size_t offset;
1464 u64 buf_addr;
1465
1466 /* attempt to use fixed buffers without having provided iovecs */
1467 if (unlikely(!ctx->user_bufs))
1468 return -EFAULT;
1469
1470 buf_index = READ_ONCE(sqe->buf_index);
1471 if (unlikely(buf_index >= ctx->nr_user_bufs))
1472 return -EFAULT;
1473
1474 index = array_index_nospec(buf_index, ctx->nr_user_bufs);
1475 imu = &ctx->user_bufs[index];
1476 buf_addr = READ_ONCE(sqe->addr);
1477
1478 /* overflow */
1479 if (buf_addr + len < buf_addr)
1480 return -EFAULT;
1481 /* not inside the mapped region */
1482 if (buf_addr < imu->ubuf || buf_addr + len > imu->ubuf + imu->len)
1483 return -EFAULT;
1484
1485 /*
1486 * May not be a start of buffer, set size appropriately
1487 * and advance us to the beginning.
1488 */
1489 offset = buf_addr - imu->ubuf;
1490 iov_iter_bvec(iter, rw, imu->bvec, imu->nr_bvecs, offset + len);
Jens Axboebd11b3a2019-07-20 08:37:31 -06001491
1492 if (offset) {
1493 /*
1494 * Don't use iov_iter_advance() here, as it's really slow for
1495 * using the latter parts of a big fixed buffer - it iterates
1496 * over each segment manually. We can cheat a bit here, because
1497 * we know that:
1498 *
1499 * 1) it's a BVEC iter, we set it up
1500 * 2) all bvecs are PAGE_SIZE in size, except potentially the
1501 * first and last bvec
1502 *
1503 * So just find our index, and adjust the iterator afterwards.
1504 * If the offset is within the first bvec (or the whole first
1505 * bvec, just use iov_iter_advance(). This makes it easier
1506 * since we can just skip the first segment, which may not
1507 * be PAGE_SIZE aligned.
1508 */
1509 const struct bio_vec *bvec = imu->bvec;
1510
1511 if (offset <= bvec->bv_len) {
1512 iov_iter_advance(iter, offset);
1513 } else {
1514 unsigned long seg_skip;
1515
1516 /* skip first vec */
1517 offset -= bvec->bv_len;
1518 seg_skip = 1 + (offset >> PAGE_SHIFT);
1519
1520 iter->bvec = bvec + seg_skip;
1521 iter->nr_segs -= seg_skip;
Aleix Roca Nonell99c79f62019-08-15 14:03:22 +02001522 iter->count -= bvec->bv_len + offset;
Jens Axboebd11b3a2019-07-20 08:37:31 -06001523 iter->iov_offset = offset & ~PAGE_MASK;
Jens Axboebd11b3a2019-07-20 08:37:31 -06001524 }
1525 }
1526
Jens Axboeedafcce2019-01-09 09:16:05 -07001527 return 0;
1528}
1529
Jens Axboe87e5e6d2019-05-14 16:02:22 -06001530static ssize_t io_import_iovec(struct io_ring_ctx *ctx, int rw,
1531 const struct sqe_submit *s, struct iovec **iovec,
1532 struct iov_iter *iter)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001533{
1534 const struct io_uring_sqe *sqe = s->sqe;
1535 void __user *buf = u64_to_user_ptr(READ_ONCE(sqe->addr));
1536 size_t sqe_len = READ_ONCE(sqe->len);
Jens Axboeedafcce2019-01-09 09:16:05 -07001537 u8 opcode;
1538
1539 /*
1540 * We're reading ->opcode for the second time, but the first read
1541 * doesn't care whether it's _FIXED or not, so it doesn't matter
1542 * whether ->opcode changes concurrently. The first read does care
1543 * about whether it is a READ or a WRITE, so we don't trust this read
1544 * for that purpose and instead let the caller pass in the read/write
1545 * flag.
1546 */
1547 opcode = READ_ONCE(sqe->opcode);
1548 if (opcode == IORING_OP_READ_FIXED ||
1549 opcode == IORING_OP_WRITE_FIXED) {
Jens Axboe87e5e6d2019-05-14 16:02:22 -06001550 ssize_t ret = io_import_fixed(ctx, rw, sqe, iter);
Jens Axboeedafcce2019-01-09 09:16:05 -07001551 *iovec = NULL;
1552 return ret;
1553 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07001554
1555 if (!s->has_user)
1556 return -EFAULT;
1557
1558#ifdef CONFIG_COMPAT
1559 if (ctx->compat)
1560 return compat_import_iovec(rw, buf, sqe_len, UIO_FASTIOV,
1561 iovec, iter);
1562#endif
1563
1564 return import_iovec(rw, buf, sqe_len, UIO_FASTIOV, iovec, iter);
1565}
1566
Jens Axboe32960612019-09-23 11:05:34 -06001567/*
1568 * For files that don't have ->read_iter() and ->write_iter(), handle them
1569 * by looping over ->read() or ->write() manually.
1570 */
1571static ssize_t loop_rw_iter(int rw, struct file *file, struct kiocb *kiocb,
1572 struct iov_iter *iter)
1573{
1574 ssize_t ret = 0;
1575
1576 /*
1577 * Don't support polled IO through this interface, and we can't
1578 * support non-blocking either. For the latter, this just causes
1579 * the kiocb to be handled from an async context.
1580 */
1581 if (kiocb->ki_flags & IOCB_HIPRI)
1582 return -EOPNOTSUPP;
1583 if (kiocb->ki_flags & IOCB_NOWAIT)
1584 return -EAGAIN;
1585
1586 while (iov_iter_count(iter)) {
1587 struct iovec iovec = iov_iter_iovec(iter);
1588 ssize_t nr;
1589
1590 if (rw == READ) {
1591 nr = file->f_op->read(file, iovec.iov_base,
1592 iovec.iov_len, &kiocb->ki_pos);
1593 } else {
1594 nr = file->f_op->write(file, iovec.iov_base,
1595 iovec.iov_len, &kiocb->ki_pos);
1596 }
1597
1598 if (nr < 0) {
1599 if (!ret)
1600 ret = nr;
1601 break;
1602 }
1603 ret += nr;
1604 if (nr != iovec.iov_len)
1605 break;
1606 iov_iter_advance(iter, nr);
1607 }
1608
1609 return ret;
1610}
1611
Pavel Begunkov267bc902019-11-07 01:41:08 +03001612static int io_read(struct io_kiocb *req, struct io_kiocb **nxt,
1613 bool force_nonblock)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001614{
1615 struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
1616 struct kiocb *kiocb = &req->rw;
1617 struct iov_iter iter;
1618 struct file *file;
Jens Axboe31b51512019-01-18 22:56:34 -07001619 size_t iov_count;
Jens Axboe9d93a3f2019-05-15 13:53:07 -06001620 ssize_t read_size, ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001621
Pavel Begunkov267bc902019-11-07 01:41:08 +03001622 ret = io_prep_rw(req, force_nonblock);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001623 if (ret)
1624 return ret;
1625 file = kiocb->ki_filp;
1626
Jens Axboe2b188cc2019-01-07 10:46:33 -07001627 if (unlikely(!(file->f_mode & FMODE_READ)))
Jens Axboe09bb8392019-03-13 12:39:28 -06001628 return -EBADF;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001629
Pavel Begunkov267bc902019-11-07 01:41:08 +03001630 ret = io_import_iovec(req->ctx, READ, &req->submit, &iovec, &iter);
Jens Axboe87e5e6d2019-05-14 16:02:22 -06001631 if (ret < 0)
Jens Axboe09bb8392019-03-13 12:39:28 -06001632 return ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001633
Jens Axboe9d93a3f2019-05-15 13:53:07 -06001634 read_size = ret;
Jens Axboe9e645e112019-05-10 16:07:28 -06001635 if (req->flags & REQ_F_LINK)
1636 req->result = read_size;
1637
Jens Axboe31b51512019-01-18 22:56:34 -07001638 iov_count = iov_iter_count(&iter);
1639 ret = rw_verify_area(READ, file, &kiocb->ki_pos, iov_count);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001640 if (!ret) {
1641 ssize_t ret2;
1642
Jens Axboe32960612019-09-23 11:05:34 -06001643 if (file->f_op->read_iter)
1644 ret2 = call_read_iter(file, kiocb, &iter);
1645 else
1646 ret2 = loop_rw_iter(READ, file, kiocb, &iter);
1647
Jens Axboe9d93a3f2019-05-15 13:53:07 -06001648 /*
1649 * In case of a short read, punt to async. This can happen
1650 * if we have data partially cached. Alternatively we can
1651 * return the short read, in which case the application will
1652 * need to issue another SQE and wait for it. That SQE will
1653 * need async punt anyway, so it's more efficient to do it
1654 * here.
1655 */
Jens Axboe491381ce2019-10-17 09:20:46 -06001656 if (force_nonblock && !(req->flags & REQ_F_NOWAIT) &&
1657 (req->flags & REQ_F_ISREG) &&
1658 ret2 > 0 && ret2 < read_size)
Jens Axboe9d93a3f2019-05-15 13:53:07 -06001659 ret2 = -EAGAIN;
1660 /* Catch -EAGAIN return for forced non-blocking submission */
Jens Axboe561fb042019-10-24 07:25:42 -06001661 if (!force_nonblock || ret2 != -EAGAIN)
Pavel Begunkov267bc902019-11-07 01:41:08 +03001662 kiocb_done(kiocb, ret2, nxt, req->submit.in_async);
Jens Axboe561fb042019-10-24 07:25:42 -06001663 else
Jens Axboe2b188cc2019-01-07 10:46:33 -07001664 ret = -EAGAIN;
1665 }
1666 kfree(iovec);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001667 return ret;
1668}
1669
Pavel Begunkov267bc902019-11-07 01:41:08 +03001670static int io_write(struct io_kiocb *req, struct io_kiocb **nxt,
1671 bool force_nonblock)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001672{
1673 struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
1674 struct kiocb *kiocb = &req->rw;
1675 struct iov_iter iter;
1676 struct file *file;
Jens Axboe31b51512019-01-18 22:56:34 -07001677 size_t iov_count;
Jens Axboe87e5e6d2019-05-14 16:02:22 -06001678 ssize_t ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001679
Pavel Begunkov267bc902019-11-07 01:41:08 +03001680 ret = io_prep_rw(req, force_nonblock);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001681 if (ret)
1682 return ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001683
Jens Axboe2b188cc2019-01-07 10:46:33 -07001684 file = kiocb->ki_filp;
1685 if (unlikely(!(file->f_mode & FMODE_WRITE)))
Jens Axboe09bb8392019-03-13 12:39:28 -06001686 return -EBADF;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001687
Pavel Begunkov267bc902019-11-07 01:41:08 +03001688 ret = io_import_iovec(req->ctx, WRITE, &req->submit, &iovec, &iter);
Jens Axboe87e5e6d2019-05-14 16:02:22 -06001689 if (ret < 0)
Jens Axboe09bb8392019-03-13 12:39:28 -06001690 return ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001691
Jens Axboe9e645e112019-05-10 16:07:28 -06001692 if (req->flags & REQ_F_LINK)
1693 req->result = ret;
1694
Jens Axboe31b51512019-01-18 22:56:34 -07001695 iov_count = iov_iter_count(&iter);
1696
1697 ret = -EAGAIN;
Jens Axboe561fb042019-10-24 07:25:42 -06001698 if (force_nonblock && !(kiocb->ki_flags & IOCB_DIRECT))
Jens Axboe31b51512019-01-18 22:56:34 -07001699 goto out_free;
Jens Axboe31b51512019-01-18 22:56:34 -07001700
1701 ret = rw_verify_area(WRITE, file, &kiocb->ki_pos, iov_count);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001702 if (!ret) {
Roman Penyaev9bf79332019-03-25 20:09:24 +01001703 ssize_t ret2;
1704
Jens Axboe2b188cc2019-01-07 10:46:33 -07001705 /*
1706 * Open-code file_start_write here to grab freeze protection,
1707 * which will be released by another thread in
1708 * io_complete_rw(). Fool lockdep by telling it the lock got
1709 * released so that it doesn't complain about the held lock when
1710 * we return to userspace.
1711 */
Jens Axboe491381ce2019-10-17 09:20:46 -06001712 if (req->flags & REQ_F_ISREG) {
Jens Axboe2b188cc2019-01-07 10:46:33 -07001713 __sb_start_write(file_inode(file)->i_sb,
1714 SB_FREEZE_WRITE, true);
1715 __sb_writers_release(file_inode(file)->i_sb,
1716 SB_FREEZE_WRITE);
1717 }
1718 kiocb->ki_flags |= IOCB_WRITE;
Roman Penyaev9bf79332019-03-25 20:09:24 +01001719
Jens Axboe32960612019-09-23 11:05:34 -06001720 if (file->f_op->write_iter)
1721 ret2 = call_write_iter(file, kiocb, &iter);
1722 else
1723 ret2 = loop_rw_iter(WRITE, file, kiocb, &iter);
Jens Axboe561fb042019-10-24 07:25:42 -06001724 if (!force_nonblock || ret2 != -EAGAIN)
Pavel Begunkov267bc902019-11-07 01:41:08 +03001725 kiocb_done(kiocb, ret2, nxt, req->submit.in_async);
Jens Axboe561fb042019-10-24 07:25:42 -06001726 else
Roman Penyaev9bf79332019-03-25 20:09:24 +01001727 ret = -EAGAIN;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001728 }
Jens Axboe31b51512019-01-18 22:56:34 -07001729out_free:
Jens Axboe2b188cc2019-01-07 10:46:33 -07001730 kfree(iovec);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001731 return ret;
1732}
1733
1734/*
1735 * IORING_OP_NOP just posts a completion event, nothing else.
1736 */
Jens Axboe78e19bb2019-11-06 15:21:34 -07001737static int io_nop(struct io_kiocb *req)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001738{
1739 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001740
Jens Axboedef596e2019-01-09 08:59:42 -07001741 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
1742 return -EINVAL;
1743
Jens Axboe78e19bb2019-11-06 15:21:34 -07001744 io_cqring_add_event(req, 0);
Jackie Liuec9c02a2019-11-08 23:50:36 +08001745 io_put_req(req);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001746 return 0;
1747}
1748
Christoph Hellwigc992fe22019-01-11 09:43:02 -07001749static int io_prep_fsync(struct io_kiocb *req, const struct io_uring_sqe *sqe)
1750{
Jens Axboe6b063142019-01-10 22:13:58 -07001751 struct io_ring_ctx *ctx = req->ctx;
Christoph Hellwigc992fe22019-01-11 09:43:02 -07001752
Jens Axboe09bb8392019-03-13 12:39:28 -06001753 if (!req->file)
1754 return -EBADF;
Christoph Hellwigc992fe22019-01-11 09:43:02 -07001755
Jens Axboe6b063142019-01-10 22:13:58 -07001756 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
Jens Axboedef596e2019-01-09 08:59:42 -07001757 return -EINVAL;
Jens Axboeedafcce2019-01-09 09:16:05 -07001758 if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index))
Christoph Hellwigc992fe22019-01-11 09:43:02 -07001759 return -EINVAL;
1760
Christoph Hellwigc992fe22019-01-11 09:43:02 -07001761 return 0;
1762}
1763
1764static int io_fsync(struct io_kiocb *req, const struct io_uring_sqe *sqe,
Jens Axboeba816ad2019-09-28 11:36:45 -06001765 struct io_kiocb **nxt, bool force_nonblock)
Christoph Hellwigc992fe22019-01-11 09:43:02 -07001766{
1767 loff_t sqe_off = READ_ONCE(sqe->off);
1768 loff_t sqe_len = READ_ONCE(sqe->len);
1769 loff_t end = sqe_off + sqe_len;
1770 unsigned fsync_flags;
1771 int ret;
1772
1773 fsync_flags = READ_ONCE(sqe->fsync_flags);
1774 if (unlikely(fsync_flags & ~IORING_FSYNC_DATASYNC))
1775 return -EINVAL;
1776
1777 ret = io_prep_fsync(req, sqe);
1778 if (ret)
1779 return ret;
1780
1781 /* fsync always requires a blocking context */
1782 if (force_nonblock)
1783 return -EAGAIN;
1784
1785 ret = vfs_fsync_range(req->rw.ki_filp, sqe_off,
1786 end > 0 ? end : LLONG_MAX,
1787 fsync_flags & IORING_FSYNC_DATASYNC);
1788
Jens Axboe9e645e112019-05-10 16:07:28 -06001789 if (ret < 0 && (req->flags & REQ_F_LINK))
1790 req->flags |= REQ_F_FAIL_LINK;
Jens Axboe78e19bb2019-11-06 15:21:34 -07001791 io_cqring_add_event(req, ret);
Jackie Liuec9c02a2019-11-08 23:50:36 +08001792 io_put_req_find_next(req, nxt);
Christoph Hellwigc992fe22019-01-11 09:43:02 -07001793 return 0;
1794}
1795
Jens Axboe5d17b4a2019-04-09 14:56:44 -06001796static int io_prep_sfr(struct io_kiocb *req, const struct io_uring_sqe *sqe)
1797{
1798 struct io_ring_ctx *ctx = req->ctx;
1799 int ret = 0;
1800
1801 if (!req->file)
1802 return -EBADF;
Jens Axboe5d17b4a2019-04-09 14:56:44 -06001803
1804 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
1805 return -EINVAL;
1806 if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index))
1807 return -EINVAL;
1808
Jens Axboe5d17b4a2019-04-09 14:56:44 -06001809 return ret;
1810}
1811
1812static int io_sync_file_range(struct io_kiocb *req,
1813 const struct io_uring_sqe *sqe,
Jens Axboeba816ad2019-09-28 11:36:45 -06001814 struct io_kiocb **nxt,
Jens Axboe5d17b4a2019-04-09 14:56:44 -06001815 bool force_nonblock)
1816{
1817 loff_t sqe_off;
1818 loff_t sqe_len;
1819 unsigned flags;
1820 int ret;
1821
1822 ret = io_prep_sfr(req, sqe);
1823 if (ret)
1824 return ret;
1825
1826 /* sync_file_range always requires a blocking context */
1827 if (force_nonblock)
1828 return -EAGAIN;
1829
1830 sqe_off = READ_ONCE(sqe->off);
1831 sqe_len = READ_ONCE(sqe->len);
1832 flags = READ_ONCE(sqe->sync_range_flags);
1833
1834 ret = sync_file_range(req->rw.ki_filp, sqe_off, sqe_len, flags);
1835
Jens Axboe9e645e112019-05-10 16:07:28 -06001836 if (ret < 0 && (req->flags & REQ_F_LINK))
1837 req->flags |= REQ_F_FAIL_LINK;
Jens Axboe78e19bb2019-11-06 15:21:34 -07001838 io_cqring_add_event(req, ret);
Jackie Liuec9c02a2019-11-08 23:50:36 +08001839 io_put_req_find_next(req, nxt);
Jens Axboe5d17b4a2019-04-09 14:56:44 -06001840 return 0;
1841}
1842
Jens Axboe0fa03c62019-04-19 13:34:07 -06001843#if defined(CONFIG_NET)
Jens Axboeaa1fa282019-04-19 13:38:09 -06001844static int io_send_recvmsg(struct io_kiocb *req, const struct io_uring_sqe *sqe,
Jens Axboeba816ad2019-09-28 11:36:45 -06001845 struct io_kiocb **nxt, bool force_nonblock,
Jens Axboeaa1fa282019-04-19 13:38:09 -06001846 long (*fn)(struct socket *, struct user_msghdr __user *,
1847 unsigned int))
1848{
Jens Axboe0fa03c62019-04-19 13:34:07 -06001849 struct socket *sock;
1850 int ret;
1851
1852 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
1853 return -EINVAL;
1854
1855 sock = sock_from_file(req->file, &ret);
1856 if (sock) {
1857 struct user_msghdr __user *msg;
1858 unsigned flags;
1859
1860 flags = READ_ONCE(sqe->msg_flags);
1861 if (flags & MSG_DONTWAIT)
1862 req->flags |= REQ_F_NOWAIT;
1863 else if (force_nonblock)
1864 flags |= MSG_DONTWAIT;
1865
1866 msg = (struct user_msghdr __user *) (unsigned long)
1867 READ_ONCE(sqe->addr);
1868
Jens Axboeaa1fa282019-04-19 13:38:09 -06001869 ret = fn(sock, msg, flags);
Jens Axboe0fa03c62019-04-19 13:34:07 -06001870 if (force_nonblock && ret == -EAGAIN)
1871 return ret;
1872 }
1873
Jens Axboe78e19bb2019-11-06 15:21:34 -07001874 io_cqring_add_event(req, ret);
Jens Axboef1f40852019-11-05 20:33:16 -07001875 if (ret < 0 && (req->flags & REQ_F_LINK))
1876 req->flags |= REQ_F_FAIL_LINK;
Jackie Liuec9c02a2019-11-08 23:50:36 +08001877 io_put_req_find_next(req, nxt);
Jens Axboe0fa03c62019-04-19 13:34:07 -06001878 return 0;
Jens Axboeaa1fa282019-04-19 13:38:09 -06001879}
1880#endif
1881
1882static int io_sendmsg(struct io_kiocb *req, const struct io_uring_sqe *sqe,
Jens Axboeba816ad2019-09-28 11:36:45 -06001883 struct io_kiocb **nxt, bool force_nonblock)
Jens Axboeaa1fa282019-04-19 13:38:09 -06001884{
1885#if defined(CONFIG_NET)
Jens Axboeba816ad2019-09-28 11:36:45 -06001886 return io_send_recvmsg(req, sqe, nxt, force_nonblock,
1887 __sys_sendmsg_sock);
Jens Axboeaa1fa282019-04-19 13:38:09 -06001888#else
1889 return -EOPNOTSUPP;
1890#endif
1891}
1892
1893static int io_recvmsg(struct io_kiocb *req, const struct io_uring_sqe *sqe,
Jens Axboeba816ad2019-09-28 11:36:45 -06001894 struct io_kiocb **nxt, bool force_nonblock)
Jens Axboeaa1fa282019-04-19 13:38:09 -06001895{
1896#if defined(CONFIG_NET)
Jens Axboeba816ad2019-09-28 11:36:45 -06001897 return io_send_recvmsg(req, sqe, nxt, force_nonblock,
1898 __sys_recvmsg_sock);
Jens Axboe0fa03c62019-04-19 13:34:07 -06001899#else
1900 return -EOPNOTSUPP;
1901#endif
1902}
1903
Jens Axboe17f2fe32019-10-17 14:42:58 -06001904static int io_accept(struct io_kiocb *req, const struct io_uring_sqe *sqe,
1905 struct io_kiocb **nxt, bool force_nonblock)
1906{
1907#if defined(CONFIG_NET)
1908 struct sockaddr __user *addr;
1909 int __user *addr_len;
1910 unsigned file_flags;
1911 int flags, ret;
1912
1913 if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_SQPOLL)))
1914 return -EINVAL;
1915 if (sqe->ioprio || sqe->off || sqe->len || sqe->buf_index)
1916 return -EINVAL;
1917
1918 addr = (struct sockaddr __user *) (unsigned long) READ_ONCE(sqe->addr);
1919 addr_len = (int __user *) (unsigned long) READ_ONCE(sqe->addr2);
1920 flags = READ_ONCE(sqe->accept_flags);
1921 file_flags = force_nonblock ? O_NONBLOCK : 0;
1922
1923 ret = __sys_accept4_file(req->file, file_flags, addr, addr_len, flags);
1924 if (ret == -EAGAIN && force_nonblock) {
1925 req->work.flags |= IO_WQ_WORK_NEEDS_FILES;
1926 return -EAGAIN;
1927 }
Jens Axboe8e3cca12019-11-09 19:52:33 -07001928 if (ret == -ERESTARTSYS)
1929 ret = -EINTR;
Jens Axboe17f2fe32019-10-17 14:42:58 -06001930 if (ret < 0 && (req->flags & REQ_F_LINK))
1931 req->flags |= REQ_F_FAIL_LINK;
Jens Axboe78e19bb2019-11-06 15:21:34 -07001932 io_cqring_add_event(req, ret);
Jackie Liuec9c02a2019-11-08 23:50:36 +08001933 io_put_req_find_next(req, nxt);
Jens Axboe17f2fe32019-10-17 14:42:58 -06001934 return 0;
1935#else
1936 return -EOPNOTSUPP;
1937#endif
1938}
1939
Jens Axboeeac406c2019-11-14 12:09:58 -07001940static inline void io_poll_remove_req(struct io_kiocb *req)
1941{
1942 if (!RB_EMPTY_NODE(&req->rb_node)) {
1943 rb_erase(&req->rb_node, &req->ctx->cancel_tree);
1944 RB_CLEAR_NODE(&req->rb_node);
1945 }
1946}
1947
Jens Axboe221c5eb2019-01-17 09:41:58 -07001948static void io_poll_remove_one(struct io_kiocb *req)
1949{
1950 struct io_poll_iocb *poll = &req->poll;
1951
1952 spin_lock(&poll->head->lock);
1953 WRITE_ONCE(poll->canceled, true);
1954 if (!list_empty(&poll->wait.entry)) {
1955 list_del_init(&poll->wait.entry);
Jackie Liua197f662019-11-08 08:09:12 -07001956 io_queue_async_work(req);
Jens Axboe221c5eb2019-01-17 09:41:58 -07001957 }
1958 spin_unlock(&poll->head->lock);
Jens Axboeeac406c2019-11-14 12:09:58 -07001959 io_poll_remove_req(req);
Jens Axboe221c5eb2019-01-17 09:41:58 -07001960}
1961
1962static void io_poll_remove_all(struct io_ring_ctx *ctx)
1963{
Jens Axboeeac406c2019-11-14 12:09:58 -07001964 struct rb_node *node;
Jens Axboe221c5eb2019-01-17 09:41:58 -07001965 struct io_kiocb *req;
1966
1967 spin_lock_irq(&ctx->completion_lock);
Jens Axboeeac406c2019-11-14 12:09:58 -07001968 while ((node = rb_first(&ctx->cancel_tree)) != NULL) {
1969 req = rb_entry(node, struct io_kiocb, rb_node);
Jens Axboe221c5eb2019-01-17 09:41:58 -07001970 io_poll_remove_one(req);
1971 }
1972 spin_unlock_irq(&ctx->completion_lock);
1973}
1974
Jens Axboe47f46762019-11-09 17:43:02 -07001975static int io_poll_cancel(struct io_ring_ctx *ctx, __u64 sqe_addr)
1976{
Jens Axboeeac406c2019-11-14 12:09:58 -07001977 struct rb_node *p, *parent = NULL;
Jens Axboe47f46762019-11-09 17:43:02 -07001978 struct io_kiocb *req;
1979
Jens Axboeeac406c2019-11-14 12:09:58 -07001980 p = ctx->cancel_tree.rb_node;
1981 while (p) {
1982 parent = p;
1983 req = rb_entry(parent, struct io_kiocb, rb_node);
1984 if (sqe_addr < req->user_data) {
1985 p = p->rb_left;
1986 } else if (sqe_addr > req->user_data) {
1987 p = p->rb_right;
1988 } else {
1989 io_poll_remove_one(req);
1990 return 0;
1991 }
Jens Axboe47f46762019-11-09 17:43:02 -07001992 }
1993
1994 return -ENOENT;
1995}
1996
Jens Axboe221c5eb2019-01-17 09:41:58 -07001997/*
1998 * Find a running poll command that matches one specified in sqe->addr,
1999 * and remove it if found.
2000 */
2001static int io_poll_remove(struct io_kiocb *req, const struct io_uring_sqe *sqe)
2002{
2003 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe47f46762019-11-09 17:43:02 -07002004 int ret;
Jens Axboe221c5eb2019-01-17 09:41:58 -07002005
2006 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
2007 return -EINVAL;
2008 if (sqe->ioprio || sqe->off || sqe->len || sqe->buf_index ||
2009 sqe->poll_events)
2010 return -EINVAL;
2011
2012 spin_lock_irq(&ctx->completion_lock);
Jens Axboe47f46762019-11-09 17:43:02 -07002013 ret = io_poll_cancel(ctx, READ_ONCE(sqe->addr));
Jens Axboe221c5eb2019-01-17 09:41:58 -07002014 spin_unlock_irq(&ctx->completion_lock);
2015
Jens Axboe78e19bb2019-11-06 15:21:34 -07002016 io_cqring_add_event(req, ret);
Jens Axboef1f40852019-11-05 20:33:16 -07002017 if (ret < 0 && (req->flags & REQ_F_LINK))
2018 req->flags |= REQ_F_FAIL_LINK;
Jackie Liuec9c02a2019-11-08 23:50:36 +08002019 io_put_req(req);
Jens Axboe221c5eb2019-01-17 09:41:58 -07002020 return 0;
2021}
2022
Jackie Liua197f662019-11-08 08:09:12 -07002023static void io_poll_complete(struct io_kiocb *req, __poll_t mask)
Jens Axboe221c5eb2019-01-17 09:41:58 -07002024{
Jackie Liua197f662019-11-08 08:09:12 -07002025 struct io_ring_ctx *ctx = req->ctx;
2026
Jens Axboe8c838782019-03-12 15:48:16 -06002027 req->poll.done = true;
Jens Axboe78e19bb2019-11-06 15:21:34 -07002028 io_cqring_fill_event(req, mangle_poll(mask));
Jens Axboe8c838782019-03-12 15:48:16 -06002029 io_commit_cqring(ctx);
Jens Axboe221c5eb2019-01-17 09:41:58 -07002030}
2031
Jens Axboe561fb042019-10-24 07:25:42 -06002032static void io_poll_complete_work(struct io_wq_work **workptr)
Jens Axboe221c5eb2019-01-17 09:41:58 -07002033{
Jens Axboe561fb042019-10-24 07:25:42 -06002034 struct io_wq_work *work = *workptr;
Jens Axboe221c5eb2019-01-17 09:41:58 -07002035 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
2036 struct io_poll_iocb *poll = &req->poll;
2037 struct poll_table_struct pt = { ._key = poll->events };
2038 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe89723d02019-11-05 15:32:58 -07002039 struct io_kiocb *nxt = NULL;
Jens Axboe221c5eb2019-01-17 09:41:58 -07002040 __poll_t mask = 0;
2041
Jens Axboe561fb042019-10-24 07:25:42 -06002042 if (work->flags & IO_WQ_WORK_CANCEL)
2043 WRITE_ONCE(poll->canceled, true);
2044
Jens Axboe221c5eb2019-01-17 09:41:58 -07002045 if (!READ_ONCE(poll->canceled))
2046 mask = vfs_poll(poll->file, &pt) & poll->events;
2047
2048 /*
2049 * Note that ->ki_cancel callers also delete iocb from active_reqs after
2050 * calling ->ki_cancel. We need the ctx_lock roundtrip here to
2051 * synchronize with them. In the cancellation case the list_del_init
2052 * itself is not actually needed, but harmless so we keep it in to
2053 * avoid further branches in the fast path.
2054 */
2055 spin_lock_irq(&ctx->completion_lock);
2056 if (!mask && !READ_ONCE(poll->canceled)) {
2057 add_wait_queue(poll->head, &poll->wait);
2058 spin_unlock_irq(&ctx->completion_lock);
2059 return;
2060 }
Jens Axboeeac406c2019-11-14 12:09:58 -07002061 io_poll_remove_req(req);
Jackie Liua197f662019-11-08 08:09:12 -07002062 io_poll_complete(req, mask);
Jens Axboe221c5eb2019-01-17 09:41:58 -07002063 spin_unlock_irq(&ctx->completion_lock);
2064
Jens Axboe8c838782019-03-12 15:48:16 -06002065 io_cqring_ev_posted(ctx);
Jens Axboe89723d02019-11-05 15:32:58 -07002066
Jackie Liuec9c02a2019-11-08 23:50:36 +08002067 io_put_req_find_next(req, &nxt);
Jens Axboe89723d02019-11-05 15:32:58 -07002068 if (nxt)
2069 *workptr = &nxt->work;
Jens Axboe221c5eb2019-01-17 09:41:58 -07002070}
2071
2072static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
2073 void *key)
2074{
2075 struct io_poll_iocb *poll = container_of(wait, struct io_poll_iocb,
2076 wait);
2077 struct io_kiocb *req = container_of(poll, struct io_kiocb, poll);
2078 struct io_ring_ctx *ctx = req->ctx;
2079 __poll_t mask = key_to_poll(key);
Jens Axboe8c838782019-03-12 15:48:16 -06002080 unsigned long flags;
Jens Axboe221c5eb2019-01-17 09:41:58 -07002081
2082 /* for instances that support it check for an event match first: */
Jens Axboe8c838782019-03-12 15:48:16 -06002083 if (mask && !(mask & poll->events))
2084 return 0;
Jens Axboe221c5eb2019-01-17 09:41:58 -07002085
2086 list_del_init(&poll->wait.entry);
Jens Axboe8c838782019-03-12 15:48:16 -06002087
Jens Axboe7c9e7f02019-11-12 08:15:53 -07002088 /*
2089 * Run completion inline if we can. We're using trylock here because
2090 * we are violating the completion_lock -> poll wq lock ordering.
2091 * If we have a link timeout we're going to need the completion_lock
2092 * for finalizing the request, mark us as having grabbed that already.
2093 */
Jens Axboe8c838782019-03-12 15:48:16 -06002094 if (mask && spin_trylock_irqsave(&ctx->completion_lock, flags)) {
Jens Axboeeac406c2019-11-14 12:09:58 -07002095 io_poll_remove_req(req);
Jackie Liua197f662019-11-08 08:09:12 -07002096 io_poll_complete(req, mask);
Jens Axboe7c9e7f02019-11-12 08:15:53 -07002097 req->flags |= REQ_F_COMP_LOCKED;
2098 io_put_req(req);
Jens Axboe8c838782019-03-12 15:48:16 -06002099 spin_unlock_irqrestore(&ctx->completion_lock, flags);
2100
2101 io_cqring_ev_posted(ctx);
Jens Axboe8c838782019-03-12 15:48:16 -06002102 } else {
Jackie Liua197f662019-11-08 08:09:12 -07002103 io_queue_async_work(req);
Jens Axboe8c838782019-03-12 15:48:16 -06002104 }
2105
Jens Axboe221c5eb2019-01-17 09:41:58 -07002106 return 1;
2107}
2108
2109struct io_poll_table {
2110 struct poll_table_struct pt;
2111 struct io_kiocb *req;
2112 int error;
2113};
2114
2115static void io_poll_queue_proc(struct file *file, struct wait_queue_head *head,
2116 struct poll_table_struct *p)
2117{
2118 struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);
2119
2120 if (unlikely(pt->req->poll.head)) {
2121 pt->error = -EINVAL;
2122 return;
2123 }
2124
2125 pt->error = 0;
2126 pt->req->poll.head = head;
2127 add_wait_queue(head, &pt->req->poll.wait);
2128}
2129
Jens Axboeeac406c2019-11-14 12:09:58 -07002130static void io_poll_req_insert(struct io_kiocb *req)
2131{
2132 struct io_ring_ctx *ctx = req->ctx;
2133 struct rb_node **p = &ctx->cancel_tree.rb_node;
2134 struct rb_node *parent = NULL;
2135 struct io_kiocb *tmp;
2136
2137 while (*p) {
2138 parent = *p;
2139 tmp = rb_entry(parent, struct io_kiocb, rb_node);
2140 if (req->user_data < tmp->user_data)
2141 p = &(*p)->rb_left;
2142 else
2143 p = &(*p)->rb_right;
2144 }
2145 rb_link_node(&req->rb_node, parent, p);
2146 rb_insert_color(&req->rb_node, &ctx->cancel_tree);
2147}
2148
Jens Axboe89723d02019-11-05 15:32:58 -07002149static int io_poll_add(struct io_kiocb *req, const struct io_uring_sqe *sqe,
2150 struct io_kiocb **nxt)
Jens Axboe221c5eb2019-01-17 09:41:58 -07002151{
2152 struct io_poll_iocb *poll = &req->poll;
2153 struct io_ring_ctx *ctx = req->ctx;
2154 struct io_poll_table ipt;
Jens Axboe8c838782019-03-12 15:48:16 -06002155 bool cancel = false;
Jens Axboe221c5eb2019-01-17 09:41:58 -07002156 __poll_t mask;
2157 u16 events;
Jens Axboe221c5eb2019-01-17 09:41:58 -07002158
2159 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
2160 return -EINVAL;
2161 if (sqe->addr || sqe->ioprio || sqe->off || sqe->len || sqe->buf_index)
2162 return -EINVAL;
Jens Axboe09bb8392019-03-13 12:39:28 -06002163 if (!poll->file)
2164 return -EBADF;
Jens Axboe221c5eb2019-01-17 09:41:58 -07002165
Jens Axboe6cc47d12019-09-18 11:18:23 -06002166 req->submit.sqe = NULL;
Jens Axboe561fb042019-10-24 07:25:42 -06002167 INIT_IO_WORK(&req->work, io_poll_complete_work);
Jens Axboe221c5eb2019-01-17 09:41:58 -07002168 events = READ_ONCE(sqe->poll_events);
2169 poll->events = demangle_poll(events) | EPOLLERR | EPOLLHUP;
Jens Axboeeac406c2019-11-14 12:09:58 -07002170 RB_CLEAR_NODE(&req->rb_node);
Jens Axboe221c5eb2019-01-17 09:41:58 -07002171
Jens Axboe221c5eb2019-01-17 09:41:58 -07002172 poll->head = NULL;
Jens Axboe8c838782019-03-12 15:48:16 -06002173 poll->done = false;
Jens Axboe221c5eb2019-01-17 09:41:58 -07002174 poll->canceled = false;
2175
2176 ipt.pt._qproc = io_poll_queue_proc;
2177 ipt.pt._key = poll->events;
2178 ipt.req = req;
2179 ipt.error = -EINVAL; /* same as no support for IOCB_CMD_POLL */
2180
2181 /* initialized the list so that we can do list_empty checks */
2182 INIT_LIST_HEAD(&poll->wait.entry);
2183 init_waitqueue_func_entry(&poll->wait, io_poll_wake);
2184
Jens Axboe36703242019-07-25 10:20:18 -06002185 INIT_LIST_HEAD(&req->list);
2186
Jens Axboe221c5eb2019-01-17 09:41:58 -07002187 mask = vfs_poll(poll->file, &ipt.pt) & poll->events;
Jens Axboe221c5eb2019-01-17 09:41:58 -07002188
2189 spin_lock_irq(&ctx->completion_lock);
Jens Axboe8c838782019-03-12 15:48:16 -06002190 if (likely(poll->head)) {
2191 spin_lock(&poll->head->lock);
2192 if (unlikely(list_empty(&poll->wait.entry))) {
2193 if (ipt.error)
2194 cancel = true;
2195 ipt.error = 0;
2196 mask = 0;
2197 }
2198 if (mask || ipt.error)
2199 list_del_init(&poll->wait.entry);
2200 else if (cancel)
2201 WRITE_ONCE(poll->canceled, true);
2202 else if (!poll->done) /* actually waiting for an event */
Jens Axboeeac406c2019-11-14 12:09:58 -07002203 io_poll_req_insert(req);
Jens Axboe8c838782019-03-12 15:48:16 -06002204 spin_unlock(&poll->head->lock);
Jens Axboe221c5eb2019-01-17 09:41:58 -07002205 }
Jens Axboe8c838782019-03-12 15:48:16 -06002206 if (mask) { /* no async, we'd stolen it */
Jens Axboe8c838782019-03-12 15:48:16 -06002207 ipt.error = 0;
Jackie Liua197f662019-11-08 08:09:12 -07002208 io_poll_complete(req, mask);
Jens Axboe8c838782019-03-12 15:48:16 -06002209 }
Jens Axboe221c5eb2019-01-17 09:41:58 -07002210 spin_unlock_irq(&ctx->completion_lock);
2211
Jens Axboe8c838782019-03-12 15:48:16 -06002212 if (mask) {
2213 io_cqring_ev_posted(ctx);
Jackie Liuec9c02a2019-11-08 23:50:36 +08002214 io_put_req_find_next(req, nxt);
Jens Axboe221c5eb2019-01-17 09:41:58 -07002215 }
Jens Axboe8c838782019-03-12 15:48:16 -06002216 return ipt.error;
Jens Axboe221c5eb2019-01-17 09:41:58 -07002217}
2218
Jens Axboe5262f562019-09-17 12:26:57 -06002219static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer)
2220{
2221 struct io_ring_ctx *ctx;
Jens Axboe11365042019-10-16 09:08:32 -06002222 struct io_kiocb *req;
Jens Axboe5262f562019-09-17 12:26:57 -06002223 unsigned long flags;
2224
2225 req = container_of(timer, struct io_kiocb, timeout.timer);
2226 ctx = req->ctx;
2227 atomic_inc(&ctx->cq_timeouts);
2228
2229 spin_lock_irqsave(&ctx->completion_lock, flags);
zhangyi (F)ef036812019-10-23 15:10:08 +08002230 /*
Jens Axboe11365042019-10-16 09:08:32 -06002231 * We could be racing with timeout deletion. If the list is empty,
2232 * then timeout lookup already found it and will be handling it.
zhangyi (F)ef036812019-10-23 15:10:08 +08002233 */
Jens Axboe842f9612019-10-29 12:34:10 -06002234 if (!list_empty(&req->list)) {
Jens Axboe11365042019-10-16 09:08:32 -06002235 struct io_kiocb *prev;
Jens Axboe5262f562019-09-17 12:26:57 -06002236
Jens Axboe11365042019-10-16 09:08:32 -06002237 /*
2238 * Adjust the reqs sequence before the current one because it
2239 * will consume a slot in the cq_ring and the the cq_tail
2240 * pointer will be increased, otherwise other timeout reqs may
2241 * return in advance without waiting for enough wait_nr.
2242 */
2243 prev = req;
2244 list_for_each_entry_continue_reverse(prev, &ctx->timeout_list, list)
2245 prev->sequence++;
Jens Axboe11365042019-10-16 09:08:32 -06002246 list_del_init(&req->list);
Jens Axboe11365042019-10-16 09:08:32 -06002247 }
Jens Axboe842f9612019-10-29 12:34:10 -06002248
Jens Axboe78e19bb2019-11-06 15:21:34 -07002249 io_cqring_fill_event(req, -ETIME);
Jens Axboe842f9612019-10-29 12:34:10 -06002250 io_commit_cqring(ctx);
Jens Axboe5262f562019-09-17 12:26:57 -06002251 spin_unlock_irqrestore(&ctx->completion_lock, flags);
2252
Jens Axboe842f9612019-10-29 12:34:10 -06002253 io_cqring_ev_posted(ctx);
Jens Axboef1f40852019-11-05 20:33:16 -07002254 if (req->flags & REQ_F_LINK)
2255 req->flags |= REQ_F_FAIL_LINK;
Jackie Liuec9c02a2019-11-08 23:50:36 +08002256 io_put_req(req);
Jens Axboe11365042019-10-16 09:08:32 -06002257 return HRTIMER_NORESTART;
2258}
2259
Jens Axboe47f46762019-11-09 17:43:02 -07002260static int io_timeout_cancel(struct io_ring_ctx *ctx, __u64 user_data)
2261{
2262 struct io_kiocb *req;
2263 int ret = -ENOENT;
2264
2265 list_for_each_entry(req, &ctx->timeout_list, list) {
2266 if (user_data == req->user_data) {
2267 list_del_init(&req->list);
2268 ret = 0;
2269 break;
2270 }
2271 }
2272
2273 if (ret == -ENOENT)
2274 return ret;
2275
2276 ret = hrtimer_try_to_cancel(&req->timeout.timer);
2277 if (ret == -1)
2278 return -EALREADY;
2279
2280 io_cqring_fill_event(req, -ECANCELED);
2281 io_put_req(req);
2282 return 0;
2283}
2284
Jens Axboe11365042019-10-16 09:08:32 -06002285/*
2286 * Remove or update an existing timeout command
2287 */
2288static int io_timeout_remove(struct io_kiocb *req,
2289 const struct io_uring_sqe *sqe)
2290{
2291 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe11365042019-10-16 09:08:32 -06002292 unsigned flags;
Jens Axboe47f46762019-11-09 17:43:02 -07002293 int ret;
Jens Axboe11365042019-10-16 09:08:32 -06002294
2295 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
2296 return -EINVAL;
2297 if (sqe->flags || sqe->ioprio || sqe->buf_index || sqe->len)
2298 return -EINVAL;
2299 flags = READ_ONCE(sqe->timeout_flags);
2300 if (flags)
2301 return -EINVAL;
2302
Jens Axboe11365042019-10-16 09:08:32 -06002303 spin_lock_irq(&ctx->completion_lock);
Jens Axboe47f46762019-11-09 17:43:02 -07002304 ret = io_timeout_cancel(ctx, READ_ONCE(sqe->addr));
Jens Axboe11365042019-10-16 09:08:32 -06002305
Jens Axboe47f46762019-11-09 17:43:02 -07002306 io_cqring_fill_event(req, ret);
Jens Axboe11365042019-10-16 09:08:32 -06002307 io_commit_cqring(ctx);
2308 spin_unlock_irq(&ctx->completion_lock);
Jens Axboe5262f562019-09-17 12:26:57 -06002309 io_cqring_ev_posted(ctx);
Jens Axboe47f46762019-11-09 17:43:02 -07002310 if (ret < 0 && req->flags & REQ_F_LINK)
2311 req->flags |= REQ_F_FAIL_LINK;
Jackie Liuec9c02a2019-11-08 23:50:36 +08002312 io_put_req(req);
Jens Axboe11365042019-10-16 09:08:32 -06002313 return 0;
Jens Axboe5262f562019-09-17 12:26:57 -06002314}
2315
2316static int io_timeout(struct io_kiocb *req, const struct io_uring_sqe *sqe)
2317{
yangerkun5da0fb12019-10-15 21:59:29 +08002318 unsigned count;
Jens Axboe5262f562019-09-17 12:26:57 -06002319 struct io_ring_ctx *ctx = req->ctx;
2320 struct list_head *entry;
Jens Axboea41525a2019-10-15 16:48:15 -06002321 enum hrtimer_mode mode;
Arnd Bergmannbdf20072019-10-01 09:53:29 -06002322 struct timespec64 ts;
zhangyi (F)a1f58ba2019-10-23 15:10:09 +08002323 unsigned span = 0;
Jens Axboea41525a2019-10-15 16:48:15 -06002324 unsigned flags;
Jens Axboe5262f562019-09-17 12:26:57 -06002325
2326 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
2327 return -EINVAL;
Jens Axboea41525a2019-10-15 16:48:15 -06002328 if (sqe->flags || sqe->ioprio || sqe->buf_index || sqe->len != 1)
2329 return -EINVAL;
2330 flags = READ_ONCE(sqe->timeout_flags);
2331 if (flags & ~IORING_TIMEOUT_ABS)
Jens Axboe5262f562019-09-17 12:26:57 -06002332 return -EINVAL;
Arnd Bergmannbdf20072019-10-01 09:53:29 -06002333
2334 if (get_timespec64(&ts, u64_to_user_ptr(sqe->addr)))
Jens Axboe5262f562019-09-17 12:26:57 -06002335 return -EFAULT;
2336
Jens Axboe11365042019-10-16 09:08:32 -06002337 if (flags & IORING_TIMEOUT_ABS)
2338 mode = HRTIMER_MODE_ABS;
2339 else
2340 mode = HRTIMER_MODE_REL;
2341
2342 hrtimer_init(&req->timeout.timer, CLOCK_MONOTONIC, mode);
2343
Jens Axboe5262f562019-09-17 12:26:57 -06002344 /*
2345 * sqe->off holds how many events that need to occur for this
2346 * timeout event to be satisfied.
2347 */
2348 count = READ_ONCE(sqe->off);
2349 if (!count)
2350 count = 1;
2351
2352 req->sequence = ctx->cached_sq_head + count - 1;
yangerkun5da0fb12019-10-15 21:59:29 +08002353 /* reuse it to store the count */
2354 req->submit.sequence = count;
Jens Axboe5262f562019-09-17 12:26:57 -06002355 req->flags |= REQ_F_TIMEOUT;
2356
2357 /*
2358 * Insertion sort, ensuring the first entry in the list is always
2359 * the one we need first.
2360 */
Jens Axboe5262f562019-09-17 12:26:57 -06002361 spin_lock_irq(&ctx->completion_lock);
2362 list_for_each_prev(entry, &ctx->timeout_list) {
2363 struct io_kiocb *nxt = list_entry(entry, struct io_kiocb, list);
yangerkun5da0fb12019-10-15 21:59:29 +08002364 unsigned nxt_sq_head;
2365 long long tmp, tmp_nxt;
Jens Axboe5262f562019-09-17 12:26:57 -06002366
yangerkun5da0fb12019-10-15 21:59:29 +08002367 /*
2368 * Since cached_sq_head + count - 1 can overflow, use type long
2369 * long to store it.
2370 */
2371 tmp = (long long)ctx->cached_sq_head + count - 1;
2372 nxt_sq_head = nxt->sequence - nxt->submit.sequence + 1;
2373 tmp_nxt = (long long)nxt_sq_head + nxt->submit.sequence - 1;
2374
2375 /*
2376 * cached_sq_head may overflow, and it will never overflow twice
2377 * once there is some timeout req still be valid.
2378 */
2379 if (ctx->cached_sq_head < nxt_sq_head)
yangerkun8b07a652019-10-17 12:12:35 +08002380 tmp += UINT_MAX;
yangerkun5da0fb12019-10-15 21:59:29 +08002381
zhangyi (F)a1f58ba2019-10-23 15:10:09 +08002382 if (tmp > tmp_nxt)
Jens Axboe5262f562019-09-17 12:26:57 -06002383 break;
zhangyi (F)a1f58ba2019-10-23 15:10:09 +08002384
2385 /*
2386 * Sequence of reqs after the insert one and itself should
2387 * be adjusted because each timeout req consumes a slot.
2388 */
2389 span++;
2390 nxt->sequence++;
Jens Axboe5262f562019-09-17 12:26:57 -06002391 }
zhangyi (F)a1f58ba2019-10-23 15:10:09 +08002392 req->sequence -= span;
Jens Axboe5262f562019-09-17 12:26:57 -06002393 list_add(&req->list, entry);
Jens Axboe5262f562019-09-17 12:26:57 -06002394 req->timeout.timer.function = io_timeout_fn;
Jens Axboea41525a2019-10-15 16:48:15 -06002395 hrtimer_start(&req->timeout.timer, timespec64_to_ktime(ts), mode);
Jens Axboe842f9612019-10-29 12:34:10 -06002396 spin_unlock_irq(&ctx->completion_lock);
Jens Axboe5262f562019-09-17 12:26:57 -06002397 return 0;
2398}
2399
Jens Axboe62755e32019-10-28 21:49:21 -06002400static bool io_cancel_cb(struct io_wq_work *work, void *data)
2401{
2402 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
2403
2404 return req->user_data == (unsigned long) data;
2405}
2406
Jens Axboee977d6d2019-11-05 12:39:45 -07002407static int io_async_cancel_one(struct io_ring_ctx *ctx, void *sqe_addr)
Jens Axboe62755e32019-10-28 21:49:21 -06002408{
Jens Axboe62755e32019-10-28 21:49:21 -06002409 enum io_wq_cancel cancel_ret;
Jens Axboe62755e32019-10-28 21:49:21 -06002410 int ret = 0;
2411
Jens Axboe62755e32019-10-28 21:49:21 -06002412 cancel_ret = io_wq_cancel_cb(ctx->io_wq, io_cancel_cb, sqe_addr);
2413 switch (cancel_ret) {
2414 case IO_WQ_CANCEL_OK:
2415 ret = 0;
2416 break;
2417 case IO_WQ_CANCEL_RUNNING:
2418 ret = -EALREADY;
2419 break;
2420 case IO_WQ_CANCEL_NOTFOUND:
2421 ret = -ENOENT;
2422 break;
2423 }
2424
Jens Axboee977d6d2019-11-05 12:39:45 -07002425 return ret;
2426}
2427
Jens Axboe47f46762019-11-09 17:43:02 -07002428static void io_async_find_and_cancel(struct io_ring_ctx *ctx,
2429 struct io_kiocb *req, __u64 sqe_addr,
2430 struct io_kiocb **nxt)
2431{
2432 unsigned long flags;
2433 int ret;
2434
2435 ret = io_async_cancel_one(ctx, (void *) (unsigned long) sqe_addr);
2436 if (ret != -ENOENT) {
2437 spin_lock_irqsave(&ctx->completion_lock, flags);
2438 goto done;
2439 }
2440
2441 spin_lock_irqsave(&ctx->completion_lock, flags);
2442 ret = io_timeout_cancel(ctx, sqe_addr);
2443 if (ret != -ENOENT)
2444 goto done;
2445 ret = io_poll_cancel(ctx, sqe_addr);
2446done:
2447 io_cqring_fill_event(req, ret);
2448 io_commit_cqring(ctx);
2449 spin_unlock_irqrestore(&ctx->completion_lock, flags);
2450 io_cqring_ev_posted(ctx);
2451
2452 if (ret < 0 && (req->flags & REQ_F_LINK))
2453 req->flags |= REQ_F_FAIL_LINK;
2454 io_put_req_find_next(req, nxt);
2455}
2456
Jens Axboee977d6d2019-11-05 12:39:45 -07002457static int io_async_cancel(struct io_kiocb *req, const struct io_uring_sqe *sqe,
2458 struct io_kiocb **nxt)
2459{
2460 struct io_ring_ctx *ctx = req->ctx;
Jens Axboee977d6d2019-11-05 12:39:45 -07002461
2462 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
2463 return -EINVAL;
2464 if (sqe->flags || sqe->ioprio || sqe->off || sqe->len ||
2465 sqe->cancel_flags)
2466 return -EINVAL;
2467
Jens Axboe47f46762019-11-09 17:43:02 -07002468 io_async_find_and_cancel(ctx, req, READ_ONCE(sqe->addr), NULL);
Jens Axboe62755e32019-10-28 21:49:21 -06002469 return 0;
2470}
2471
Jackie Liua197f662019-11-08 08:09:12 -07002472static int io_req_defer(struct io_kiocb *req)
Jens Axboede0617e2019-04-06 21:51:27 -06002473{
Pavel Begunkov267bc902019-11-07 01:41:08 +03002474 const struct io_uring_sqe *sqe = req->submit.sqe;
Jens Axboede0617e2019-04-06 21:51:27 -06002475 struct io_uring_sqe *sqe_copy;
Jackie Liua197f662019-11-08 08:09:12 -07002476 struct io_ring_ctx *ctx = req->ctx;
Jens Axboede0617e2019-04-06 21:51:27 -06002477
Bob Liu9d858b22019-11-13 18:06:25 +08002478 /* Still need defer if there is pending req in defer list. */
2479 if (!req_need_defer(req) && list_empty(&ctx->defer_list))
Jens Axboede0617e2019-04-06 21:51:27 -06002480 return 0;
2481
2482 sqe_copy = kmalloc(sizeof(*sqe_copy), GFP_KERNEL);
2483 if (!sqe_copy)
2484 return -EAGAIN;
2485
2486 spin_lock_irq(&ctx->completion_lock);
Bob Liu9d858b22019-11-13 18:06:25 +08002487 if (!req_need_defer(req) && list_empty(&ctx->defer_list)) {
Jens Axboede0617e2019-04-06 21:51:27 -06002488 spin_unlock_irq(&ctx->completion_lock);
2489 kfree(sqe_copy);
2490 return 0;
2491 }
2492
2493 memcpy(sqe_copy, sqe, sizeof(*sqe_copy));
2494 req->submit.sqe = sqe_copy;
2495
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +02002496 trace_io_uring_defer(ctx, req, false);
Jens Axboede0617e2019-04-06 21:51:27 -06002497 list_add_tail(&req->list, &ctx->defer_list);
2498 spin_unlock_irq(&ctx->completion_lock);
2499 return -EIOCBQUEUED;
2500}
2501
Jackie Liua197f662019-11-08 08:09:12 -07002502static int __io_submit_sqe(struct io_kiocb *req, struct io_kiocb **nxt,
2503 bool force_nonblock)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002504{
Jens Axboee0c5c572019-03-12 10:18:47 -06002505 int ret, opcode;
Pavel Begunkov267bc902019-11-07 01:41:08 +03002506 struct sqe_submit *s = &req->submit;
Jackie Liua197f662019-11-08 08:09:12 -07002507 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002508
Jens Axboe2b188cc2019-01-07 10:46:33 -07002509 opcode = READ_ONCE(s->sqe->opcode);
2510 switch (opcode) {
2511 case IORING_OP_NOP:
Jens Axboe78e19bb2019-11-06 15:21:34 -07002512 ret = io_nop(req);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002513 break;
2514 case IORING_OP_READV:
Jens Axboeedafcce2019-01-09 09:16:05 -07002515 if (unlikely(s->sqe->buf_index))
2516 return -EINVAL;
Pavel Begunkov267bc902019-11-07 01:41:08 +03002517 ret = io_read(req, nxt, force_nonblock);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002518 break;
2519 case IORING_OP_WRITEV:
Jens Axboeedafcce2019-01-09 09:16:05 -07002520 if (unlikely(s->sqe->buf_index))
2521 return -EINVAL;
Pavel Begunkov267bc902019-11-07 01:41:08 +03002522 ret = io_write(req, nxt, force_nonblock);
Jens Axboeedafcce2019-01-09 09:16:05 -07002523 break;
2524 case IORING_OP_READ_FIXED:
Pavel Begunkov267bc902019-11-07 01:41:08 +03002525 ret = io_read(req, nxt, force_nonblock);
Jens Axboeedafcce2019-01-09 09:16:05 -07002526 break;
2527 case IORING_OP_WRITE_FIXED:
Pavel Begunkov267bc902019-11-07 01:41:08 +03002528 ret = io_write(req, nxt, force_nonblock);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002529 break;
Christoph Hellwigc992fe22019-01-11 09:43:02 -07002530 case IORING_OP_FSYNC:
Jens Axboeba816ad2019-09-28 11:36:45 -06002531 ret = io_fsync(req, s->sqe, nxt, force_nonblock);
Christoph Hellwigc992fe22019-01-11 09:43:02 -07002532 break;
Jens Axboe221c5eb2019-01-17 09:41:58 -07002533 case IORING_OP_POLL_ADD:
Jens Axboe89723d02019-11-05 15:32:58 -07002534 ret = io_poll_add(req, s->sqe, nxt);
Jens Axboe221c5eb2019-01-17 09:41:58 -07002535 break;
2536 case IORING_OP_POLL_REMOVE:
2537 ret = io_poll_remove(req, s->sqe);
2538 break;
Jens Axboe5d17b4a2019-04-09 14:56:44 -06002539 case IORING_OP_SYNC_FILE_RANGE:
Jens Axboeba816ad2019-09-28 11:36:45 -06002540 ret = io_sync_file_range(req, s->sqe, nxt, force_nonblock);
Jens Axboe5d17b4a2019-04-09 14:56:44 -06002541 break;
Jens Axboe0fa03c62019-04-19 13:34:07 -06002542 case IORING_OP_SENDMSG:
Jens Axboeba816ad2019-09-28 11:36:45 -06002543 ret = io_sendmsg(req, s->sqe, nxt, force_nonblock);
Jens Axboe0fa03c62019-04-19 13:34:07 -06002544 break;
Jens Axboeaa1fa282019-04-19 13:38:09 -06002545 case IORING_OP_RECVMSG:
Jens Axboeba816ad2019-09-28 11:36:45 -06002546 ret = io_recvmsg(req, s->sqe, nxt, force_nonblock);
Jens Axboeaa1fa282019-04-19 13:38:09 -06002547 break;
Jens Axboe5262f562019-09-17 12:26:57 -06002548 case IORING_OP_TIMEOUT:
2549 ret = io_timeout(req, s->sqe);
2550 break;
Jens Axboe11365042019-10-16 09:08:32 -06002551 case IORING_OP_TIMEOUT_REMOVE:
2552 ret = io_timeout_remove(req, s->sqe);
2553 break;
Jens Axboe17f2fe32019-10-17 14:42:58 -06002554 case IORING_OP_ACCEPT:
2555 ret = io_accept(req, s->sqe, nxt, force_nonblock);
2556 break;
Jens Axboe62755e32019-10-28 21:49:21 -06002557 case IORING_OP_ASYNC_CANCEL:
2558 ret = io_async_cancel(req, s->sqe, nxt);
2559 break;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002560 default:
2561 ret = -EINVAL;
2562 break;
2563 }
2564
Jens Axboedef596e2019-01-09 08:59:42 -07002565 if (ret)
2566 return ret;
2567
2568 if (ctx->flags & IORING_SETUP_IOPOLL) {
Jens Axboe9e645e112019-05-10 16:07:28 -06002569 if (req->result == -EAGAIN)
Jens Axboedef596e2019-01-09 08:59:42 -07002570 return -EAGAIN;
2571
2572 /* workqueue context doesn't hold uring_lock, grab it now */
Jackie Liuba5290c2019-10-09 09:19:59 +08002573 if (s->in_async)
Jens Axboedef596e2019-01-09 08:59:42 -07002574 mutex_lock(&ctx->uring_lock);
2575 io_iopoll_req_issued(req);
Jackie Liuba5290c2019-10-09 09:19:59 +08002576 if (s->in_async)
Jens Axboedef596e2019-01-09 08:59:42 -07002577 mutex_unlock(&ctx->uring_lock);
2578 }
2579
2580 return 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002581}
2582
Jens Axboe561fb042019-10-24 07:25:42 -06002583static void io_wq_submit_work(struct io_wq_work **workptr)
Jens Axboe31b51512019-01-18 22:56:34 -07002584{
Jens Axboe561fb042019-10-24 07:25:42 -06002585 struct io_wq_work *work = *workptr;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002586 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
Jens Axboe561fb042019-10-24 07:25:42 -06002587 struct sqe_submit *s = &req->submit;
2588 const struct io_uring_sqe *sqe = s->sqe;
2589 struct io_kiocb *nxt = NULL;
2590 int ret = 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002591
Jens Axboe561fb042019-10-24 07:25:42 -06002592 /* Ensure we clear previously set non-block flag */
2593 req->rw.ki_flags &= ~IOCB_NOWAIT;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002594
Jens Axboe561fb042019-10-24 07:25:42 -06002595 if (work->flags & IO_WQ_WORK_CANCEL)
2596 ret = -ECANCELED;
Jens Axboe31b51512019-01-18 22:56:34 -07002597
Jens Axboe561fb042019-10-24 07:25:42 -06002598 if (!ret) {
2599 s->has_user = (work->flags & IO_WQ_WORK_HAS_MM) != 0;
2600 s->in_async = true;
2601 do {
Jackie Liua197f662019-11-08 08:09:12 -07002602 ret = __io_submit_sqe(req, &nxt, false);
Jens Axboe561fb042019-10-24 07:25:42 -06002603 /*
2604 * We can get EAGAIN for polled IO even though we're
2605 * forcing a sync submission from here, since we can't
2606 * wait for request slots on the block side.
2607 */
2608 if (ret != -EAGAIN)
2609 break;
2610 cond_resched();
2611 } while (1);
2612 }
Jens Axboe31b51512019-01-18 22:56:34 -07002613
Jens Axboe561fb042019-10-24 07:25:42 -06002614 /* drop submission reference */
Jackie Liuec9c02a2019-11-08 23:50:36 +08002615 io_put_req(req);
Jens Axboe817869d2019-04-30 14:44:05 -06002616
Jens Axboe561fb042019-10-24 07:25:42 -06002617 if (ret) {
Jens Axboef1f40852019-11-05 20:33:16 -07002618 if (req->flags & REQ_F_LINK)
2619 req->flags |= REQ_F_FAIL_LINK;
Jens Axboe78e19bb2019-11-06 15:21:34 -07002620 io_cqring_add_event(req, ret);
Jackie Liuec9c02a2019-11-08 23:50:36 +08002621 io_put_req(req);
Jens Axboeedafcce2019-01-09 09:16:05 -07002622 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07002623
Jens Axboe561fb042019-10-24 07:25:42 -06002624 /* async context always use a copy of the sqe */
2625 kfree(sqe);
2626
2627 /* if a dependent link is ready, pass it back */
2628 if (!ret && nxt) {
2629 io_prep_async_work(nxt);
2630 *workptr = &nxt->work;
Jens Axboeedafcce2019-01-09 09:16:05 -07002631 }
Jens Axboe31b51512019-01-18 22:56:34 -07002632}
Jens Axboe2b188cc2019-01-07 10:46:33 -07002633
Jens Axboe09bb8392019-03-13 12:39:28 -06002634static bool io_op_needs_file(const struct io_uring_sqe *sqe)
2635{
2636 int op = READ_ONCE(sqe->opcode);
2637
2638 switch (op) {
2639 case IORING_OP_NOP:
2640 case IORING_OP_POLL_REMOVE:
Pavel Begunkova320e9f2019-11-14 00:11:01 +03002641 case IORING_OP_TIMEOUT:
2642 case IORING_OP_TIMEOUT_REMOVE:
2643 case IORING_OP_ASYNC_CANCEL:
2644 case IORING_OP_LINK_TIMEOUT:
Jens Axboe09bb8392019-03-13 12:39:28 -06002645 return false;
2646 default:
2647 return true;
2648 }
2649}
2650
Jens Axboe65e19f52019-10-26 07:20:21 -06002651static inline struct file *io_file_from_index(struct io_ring_ctx *ctx,
2652 int index)
2653{
2654 struct fixed_file_table *table;
2655
2656 table = &ctx->file_table[index >> IORING_FILE_TABLE_SHIFT];
2657 return table->files[index & IORING_FILE_TABLE_MASK];
2658}
2659
Jackie Liua197f662019-11-08 08:09:12 -07002660static int io_req_set_file(struct io_submit_state *state, struct io_kiocb *req)
Jens Axboe09bb8392019-03-13 12:39:28 -06002661{
Pavel Begunkov267bc902019-11-07 01:41:08 +03002662 struct sqe_submit *s = &req->submit;
Jackie Liua197f662019-11-08 08:09:12 -07002663 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe09bb8392019-03-13 12:39:28 -06002664 unsigned flags;
2665 int fd;
2666
2667 flags = READ_ONCE(s->sqe->flags);
2668 fd = READ_ONCE(s->sqe->fd);
2669
Jackie Liu4fe2c962019-09-09 20:50:40 +08002670 if (flags & IOSQE_IO_DRAIN)
Jens Axboede0617e2019-04-06 21:51:27 -06002671 req->flags |= REQ_F_IO_DRAIN;
Jackie Liu4fe2c962019-09-09 20:50:40 +08002672 /*
2673 * All io need record the previous position, if LINK vs DARIN,
2674 * it can be used to mark the position of the first IO in the
2675 * link list.
2676 */
2677 req->sequence = s->sequence;
Jens Axboede0617e2019-04-06 21:51:27 -06002678
Jens Axboe60c112b2019-06-21 10:20:18 -06002679 if (!io_op_needs_file(s->sqe))
Jens Axboe09bb8392019-03-13 12:39:28 -06002680 return 0;
Jens Axboe09bb8392019-03-13 12:39:28 -06002681
2682 if (flags & IOSQE_FIXED_FILE) {
Jens Axboe65e19f52019-10-26 07:20:21 -06002683 if (unlikely(!ctx->file_table ||
Jens Axboe09bb8392019-03-13 12:39:28 -06002684 (unsigned) fd >= ctx->nr_user_files))
2685 return -EBADF;
Jens Axboeb7620122019-10-26 07:22:55 -06002686 fd = array_index_nospec(fd, ctx->nr_user_files);
Jens Axboe65e19f52019-10-26 07:20:21 -06002687 req->file = io_file_from_index(ctx, fd);
2688 if (!req->file)
Jens Axboe08a45172019-10-03 08:11:03 -06002689 return -EBADF;
Jens Axboe09bb8392019-03-13 12:39:28 -06002690 req->flags |= REQ_F_FIXED_FILE;
2691 } else {
2692 if (s->needs_fixed_file)
2693 return -EBADF;
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +02002694 trace_io_uring_file_get(ctx, fd);
Jens Axboe09bb8392019-03-13 12:39:28 -06002695 req->file = io_file_get(state, fd);
2696 if (unlikely(!req->file))
2697 return -EBADF;
2698 }
2699
2700 return 0;
2701}
2702
Jackie Liua197f662019-11-08 08:09:12 -07002703static int io_grab_files(struct io_kiocb *req)
Jens Axboefcb323c2019-10-24 12:39:47 -06002704{
2705 int ret = -EBADF;
Jackie Liua197f662019-11-08 08:09:12 -07002706 struct io_ring_ctx *ctx = req->ctx;
Jens Axboefcb323c2019-10-24 12:39:47 -06002707
2708 rcu_read_lock();
2709 spin_lock_irq(&ctx->inflight_lock);
2710 /*
2711 * We use the f_ops->flush() handler to ensure that we can flush
2712 * out work accessing these files if the fd is closed. Check if
2713 * the fd has changed since we started down this path, and disallow
2714 * this operation if it has.
2715 */
2716 if (fcheck(req->submit.ring_fd) == req->submit.ring_file) {
2717 list_add(&req->inflight_entry, &ctx->inflight_list);
2718 req->flags |= REQ_F_INFLIGHT;
2719 req->work.files = current->files;
2720 ret = 0;
2721 }
2722 spin_unlock_irq(&ctx->inflight_lock);
2723 rcu_read_unlock();
2724
2725 return ret;
2726}
2727
Jens Axboe2665abf2019-11-05 12:40:47 -07002728static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer)
2729{
2730 struct io_kiocb *req = container_of(timer, struct io_kiocb,
2731 timeout.timer);
2732 struct io_ring_ctx *ctx = req->ctx;
2733 struct io_kiocb *prev = NULL;
2734 unsigned long flags;
Jens Axboe2665abf2019-11-05 12:40:47 -07002735
2736 spin_lock_irqsave(&ctx->completion_lock, flags);
2737
2738 /*
2739 * We don't expect the list to be empty, that will only happen if we
2740 * race with the completion of the linked work.
2741 */
2742 if (!list_empty(&req->list)) {
2743 prev = list_entry(req->list.prev, struct io_kiocb, link_list);
Jens Axboe76a46e02019-11-10 23:34:16 -07002744 if (refcount_inc_not_zero(&prev->refs))
2745 list_del_init(&req->list);
2746 else
2747 prev = NULL;
Jens Axboe2665abf2019-11-05 12:40:47 -07002748 }
2749
2750 spin_unlock_irqrestore(&ctx->completion_lock, flags);
2751
2752 if (prev) {
Jens Axboe47f46762019-11-09 17:43:02 -07002753 io_async_find_and_cancel(ctx, req, prev->user_data, NULL);
Jens Axboe76a46e02019-11-10 23:34:16 -07002754 io_put_req(prev);
Jens Axboe47f46762019-11-09 17:43:02 -07002755 } else {
2756 io_cqring_add_event(req, -ETIME);
2757 io_put_req(req);
Jens Axboe2665abf2019-11-05 12:40:47 -07002758 }
Jens Axboe2665abf2019-11-05 12:40:47 -07002759 return HRTIMER_NORESTART;
2760}
2761
Jens Axboe76a46e02019-11-10 23:34:16 -07002762static void io_queue_linked_timeout(struct io_kiocb *req, struct timespec64 *ts,
2763 enum hrtimer_mode *mode)
Jens Axboe2665abf2019-11-05 12:40:47 -07002764{
Jens Axboe76a46e02019-11-10 23:34:16 -07002765 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe2665abf2019-11-05 12:40:47 -07002766
Jens Axboe76a46e02019-11-10 23:34:16 -07002767 /*
2768 * If the list is now empty, then our linked request finished before
2769 * we got a chance to setup the timer
2770 */
2771 spin_lock_irq(&ctx->completion_lock);
2772 if (!list_empty(&req->list)) {
2773 req->timeout.timer.function = io_link_timeout_fn;
2774 hrtimer_start(&req->timeout.timer, timespec64_to_ktime(*ts),
2775 *mode);
Jens Axboe2665abf2019-11-05 12:40:47 -07002776 }
Jens Axboe76a46e02019-11-10 23:34:16 -07002777 spin_unlock_irq(&ctx->completion_lock);
Jens Axboe2665abf2019-11-05 12:40:47 -07002778
Jens Axboe2665abf2019-11-05 12:40:47 -07002779 /* drop submission reference */
Jens Axboe76a46e02019-11-10 23:34:16 -07002780 io_put_req(req);
Jens Axboe2665abf2019-11-05 12:40:47 -07002781}
2782
Jens Axboe76a46e02019-11-10 23:34:16 -07002783static int io_validate_link_timeout(const struct io_uring_sqe *sqe,
2784 struct timespec64 *ts)
2785{
2786 if (sqe->ioprio || sqe->buf_index || sqe->len != 1 || sqe->off)
2787 return -EINVAL;
2788 if (sqe->timeout_flags & ~IORING_TIMEOUT_ABS)
2789 return -EINVAL;
2790 if (get_timespec64(ts, u64_to_user_ptr(sqe->addr)))
2791 return -EFAULT;
2792
2793 return 0;
2794}
2795
2796static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req,
2797 struct timespec64 *ts,
2798 enum hrtimer_mode *mode)
Jens Axboe2665abf2019-11-05 12:40:47 -07002799{
2800 struct io_kiocb *nxt;
Jens Axboe76a46e02019-11-10 23:34:16 -07002801 int ret;
Jens Axboe2665abf2019-11-05 12:40:47 -07002802
2803 if (!(req->flags & REQ_F_LINK))
2804 return NULL;
2805
2806 nxt = list_first_entry_or_null(&req->link_list, struct io_kiocb, list);
Jens Axboe76a46e02019-11-10 23:34:16 -07002807 if (!nxt || nxt->submit.sqe->opcode != IORING_OP_LINK_TIMEOUT)
2808 return NULL;
Jens Axboe2665abf2019-11-05 12:40:47 -07002809
Jens Axboe76a46e02019-11-10 23:34:16 -07002810 ret = io_validate_link_timeout(nxt->submit.sqe, ts);
2811 if (ret) {
2812 list_del_init(&nxt->list);
2813 io_cqring_add_event(nxt, ret);
2814 io_double_put_req(nxt);
2815 return ERR_PTR(-ECANCELED);
2816 }
2817
2818 if (nxt->submit.sqe->timeout_flags & IORING_TIMEOUT_ABS)
2819 *mode = HRTIMER_MODE_ABS;
2820 else
2821 *mode = HRTIMER_MODE_REL;
2822
2823 req->flags |= REQ_F_LINK_TIMEOUT;
2824 hrtimer_init(&nxt->timeout.timer, CLOCK_MONOTONIC, *mode);
2825 return nxt;
Jens Axboe2665abf2019-11-05 12:40:47 -07002826}
2827
Jackie Liua197f662019-11-08 08:09:12 -07002828static int __io_queue_sqe(struct io_kiocb *req)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002829{
Jens Axboe76a46e02019-11-10 23:34:16 -07002830 enum hrtimer_mode mode;
Jens Axboe2665abf2019-11-05 12:40:47 -07002831 struct io_kiocb *nxt;
Jens Axboe76a46e02019-11-10 23:34:16 -07002832 struct timespec64 ts;
Jens Axboee0c5c572019-03-12 10:18:47 -06002833 int ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002834
Jens Axboe76a46e02019-11-10 23:34:16 -07002835 nxt = io_prep_linked_timeout(req, &ts, &mode);
2836 if (IS_ERR(nxt)) {
2837 ret = PTR_ERR(nxt);
2838 nxt = NULL;
2839 goto err;
Jens Axboe2665abf2019-11-05 12:40:47 -07002840 }
2841
Jackie Liua197f662019-11-08 08:09:12 -07002842 ret = __io_submit_sqe(req, NULL, true);
Jens Axboe491381ce2019-10-17 09:20:46 -06002843
2844 /*
2845 * We async punt it if the file wasn't marked NOWAIT, or if the file
2846 * doesn't support non-blocking read/write attempts
2847 */
2848 if (ret == -EAGAIN && (!(req->flags & REQ_F_NOWAIT) ||
2849 (req->flags & REQ_F_MUST_PUNT))) {
Pavel Begunkov267bc902019-11-07 01:41:08 +03002850 struct sqe_submit *s = &req->submit;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002851 struct io_uring_sqe *sqe_copy;
2852
Jackie Liu954dab12019-09-18 10:37:52 +08002853 sqe_copy = kmemdup(s->sqe, sizeof(*sqe_copy), GFP_KERNEL);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002854 if (sqe_copy) {
Jens Axboe2b188cc2019-01-07 10:46:33 -07002855 s->sqe = sqe_copy;
Jens Axboefcb323c2019-10-24 12:39:47 -06002856 if (req->work.flags & IO_WQ_WORK_NEEDS_FILES) {
Jackie Liua197f662019-11-08 08:09:12 -07002857 ret = io_grab_files(req);
Jens Axboefcb323c2019-10-24 12:39:47 -06002858 if (ret) {
2859 kfree(sqe_copy);
2860 goto err;
2861 }
2862 }
Jens Axboee65ef562019-03-12 10:16:44 -06002863
2864 /*
2865 * Queued up for async execution, worker will release
Jens Axboe9e645e112019-05-10 16:07:28 -06002866 * submit reference when the iocb is actually submitted.
Jens Axboee65ef562019-03-12 10:16:44 -06002867 */
Jackie Liua197f662019-11-08 08:09:12 -07002868 io_queue_async_work(req);
Jens Axboe76a46e02019-11-10 23:34:16 -07002869
2870 if (nxt)
2871 io_queue_linked_timeout(nxt, &ts, &mode);
2872
Jens Axboee65ef562019-03-12 10:16:44 -06002873 return 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002874 }
2875 }
Jens Axboee65ef562019-03-12 10:16:44 -06002876
Jens Axboefcb323c2019-10-24 12:39:47 -06002877err:
Jens Axboe76a46e02019-11-10 23:34:16 -07002878 /* drop submission reference */
Jackie Liuec9c02a2019-11-08 23:50:36 +08002879 io_put_req(req);
Jens Axboee65ef562019-03-12 10:16:44 -06002880
Jens Axboe76a46e02019-11-10 23:34:16 -07002881 if (nxt) {
2882 if (!ret)
2883 io_queue_linked_timeout(nxt, &ts, &mode);
2884 else
2885 io_put_req(nxt);
2886 }
2887
Jens Axboee65ef562019-03-12 10:16:44 -06002888 /* and drop final reference, if we failed */
Jens Axboe9e645e112019-05-10 16:07:28 -06002889 if (ret) {
Jens Axboe78e19bb2019-11-06 15:21:34 -07002890 io_cqring_add_event(req, ret);
Jens Axboe9e645e112019-05-10 16:07:28 -06002891 if (req->flags & REQ_F_LINK)
2892 req->flags |= REQ_F_FAIL_LINK;
Jackie Liuec9c02a2019-11-08 23:50:36 +08002893 io_put_req(req);
Jens Axboe9e645e112019-05-10 16:07:28 -06002894 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07002895
2896 return ret;
2897}
2898
Jackie Liua197f662019-11-08 08:09:12 -07002899static int io_queue_sqe(struct io_kiocb *req)
Jackie Liu4fe2c962019-09-09 20:50:40 +08002900{
2901 int ret;
2902
Jackie Liua197f662019-11-08 08:09:12 -07002903 ret = io_req_defer(req);
Jackie Liu4fe2c962019-09-09 20:50:40 +08002904 if (ret) {
2905 if (ret != -EIOCBQUEUED) {
Jens Axboe78e19bb2019-11-06 15:21:34 -07002906 io_cqring_add_event(req, ret);
2907 io_double_put_req(req);
Jackie Liu4fe2c962019-09-09 20:50:40 +08002908 }
2909 return 0;
2910 }
2911
Jackie Liua197f662019-11-08 08:09:12 -07002912 return __io_queue_sqe(req);
Jackie Liu4fe2c962019-09-09 20:50:40 +08002913}
2914
Jackie Liua197f662019-11-08 08:09:12 -07002915static int io_queue_link_head(struct io_kiocb *req, struct io_kiocb *shadow)
Jackie Liu4fe2c962019-09-09 20:50:40 +08002916{
2917 int ret;
2918 int need_submit = false;
Jackie Liua197f662019-11-08 08:09:12 -07002919 struct io_ring_ctx *ctx = req->ctx;
Jackie Liu4fe2c962019-09-09 20:50:40 +08002920
2921 if (!shadow)
Jackie Liua197f662019-11-08 08:09:12 -07002922 return io_queue_sqe(req);
Jackie Liu4fe2c962019-09-09 20:50:40 +08002923
2924 /*
2925 * Mark the first IO in link list as DRAIN, let all the following
2926 * IOs enter the defer list. all IO needs to be completed before link
2927 * list.
2928 */
2929 req->flags |= REQ_F_IO_DRAIN;
Jackie Liua197f662019-11-08 08:09:12 -07002930 ret = io_req_defer(req);
Jackie Liu4fe2c962019-09-09 20:50:40 +08002931 if (ret) {
2932 if (ret != -EIOCBQUEUED) {
Jens Axboe78e19bb2019-11-06 15:21:34 -07002933 io_cqring_add_event(req, ret);
2934 io_double_put_req(req);
Pavel Begunkov7b202382019-10-27 22:10:36 +03002935 __io_free_req(shadow);
Jackie Liu4fe2c962019-09-09 20:50:40 +08002936 return 0;
2937 }
2938 } else {
2939 /*
2940 * If ret == 0 means that all IOs in front of link io are
2941 * running done. let's queue link head.
2942 */
2943 need_submit = true;
2944 }
2945
2946 /* Insert shadow req to defer_list, blocking next IOs */
2947 spin_lock_irq(&ctx->completion_lock);
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +02002948 trace_io_uring_defer(ctx, shadow, true);
Jackie Liu4fe2c962019-09-09 20:50:40 +08002949 list_add_tail(&shadow->list, &ctx->defer_list);
2950 spin_unlock_irq(&ctx->completion_lock);
2951
2952 if (need_submit)
Jackie Liua197f662019-11-08 08:09:12 -07002953 return __io_queue_sqe(req);
Jackie Liu4fe2c962019-09-09 20:50:40 +08002954
2955 return 0;
2956}
2957
Jens Axboe9e645e112019-05-10 16:07:28 -06002958#define SQE_VALID_FLAGS (IOSQE_FIXED_FILE|IOSQE_IO_DRAIN|IOSQE_IO_LINK)
2959
Jackie Liua197f662019-11-08 08:09:12 -07002960static void io_submit_sqe(struct io_kiocb *req, struct io_submit_state *state,
2961 struct io_kiocb **link)
Jens Axboe9e645e112019-05-10 16:07:28 -06002962{
2963 struct io_uring_sqe *sqe_copy;
Pavel Begunkov267bc902019-11-07 01:41:08 +03002964 struct sqe_submit *s = &req->submit;
Jackie Liua197f662019-11-08 08:09:12 -07002965 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe9e645e112019-05-10 16:07:28 -06002966 int ret;
2967
Jens Axboe78e19bb2019-11-06 15:21:34 -07002968 req->user_data = s->sqe->user_data;
2969
Jens Axboe9e645e112019-05-10 16:07:28 -06002970 /* enforce forwards compatibility on users */
2971 if (unlikely(s->sqe->flags & ~SQE_VALID_FLAGS)) {
2972 ret = -EINVAL;
Pavel Begunkov196be952019-11-07 01:41:06 +03002973 goto err_req;
Jens Axboe9e645e112019-05-10 16:07:28 -06002974 }
2975
Jackie Liua197f662019-11-08 08:09:12 -07002976 ret = io_req_set_file(state, req);
Jens Axboe9e645e112019-05-10 16:07:28 -06002977 if (unlikely(ret)) {
2978err_req:
Jens Axboe78e19bb2019-11-06 15:21:34 -07002979 io_cqring_add_event(req, ret);
2980 io_double_put_req(req);
Jens Axboe9e645e112019-05-10 16:07:28 -06002981 return;
2982 }
2983
Jens Axboe9e645e112019-05-10 16:07:28 -06002984 /*
2985 * If we already have a head request, queue this one for async
2986 * submittal once the head completes. If we don't have a head but
2987 * IOSQE_IO_LINK is set in the sqe, start a new head. This one will be
2988 * submitted sync once the chain is complete. If none of those
2989 * conditions are true (normal request), then just queue it.
2990 */
2991 if (*link) {
2992 struct io_kiocb *prev = *link;
2993
2994 sqe_copy = kmemdup(s->sqe, sizeof(*sqe_copy), GFP_KERNEL);
2995 if (!sqe_copy) {
2996 ret = -EAGAIN;
2997 goto err_req;
2998 }
2999
3000 s->sqe = sqe_copy;
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +02003001 trace_io_uring_link(ctx, req, prev);
Jens Axboe9e645e112019-05-10 16:07:28 -06003002 list_add_tail(&req->list, &prev->link_list);
3003 } else if (s->sqe->flags & IOSQE_IO_LINK) {
3004 req->flags |= REQ_F_LINK;
3005
Jens Axboe9e645e112019-05-10 16:07:28 -06003006 INIT_LIST_HEAD(&req->link_list);
3007 *link = req;
Jens Axboe2665abf2019-11-05 12:40:47 -07003008 } else if (READ_ONCE(s->sqe->opcode) == IORING_OP_LINK_TIMEOUT) {
3009 /* Only valid as a linked SQE */
3010 ret = -EINVAL;
3011 goto err_req;
Jens Axboe9e645e112019-05-10 16:07:28 -06003012 } else {
Jackie Liua197f662019-11-08 08:09:12 -07003013 io_queue_sqe(req);
Jens Axboe9e645e112019-05-10 16:07:28 -06003014 }
3015}
3016
Jens Axboe9a56a232019-01-09 09:06:50 -07003017/*
3018 * Batched submission is done, ensure local IO is flushed out.
3019 */
3020static void io_submit_state_end(struct io_submit_state *state)
3021{
3022 blk_finish_plug(&state->plug);
Jens Axboe3d6770f2019-04-13 11:50:54 -06003023 io_file_put(state);
Jens Axboe2579f912019-01-09 09:10:43 -07003024 if (state->free_reqs)
3025 kmem_cache_free_bulk(req_cachep, state->free_reqs,
3026 &state->reqs[state->cur_req]);
Jens Axboe9a56a232019-01-09 09:06:50 -07003027}
3028
3029/*
3030 * Start submission side cache.
3031 */
3032static void io_submit_state_start(struct io_submit_state *state,
3033 struct io_ring_ctx *ctx, unsigned max_ios)
3034{
3035 blk_start_plug(&state->plug);
Jens Axboe2579f912019-01-09 09:10:43 -07003036 state->free_reqs = 0;
Jens Axboe9a56a232019-01-09 09:06:50 -07003037 state->file = NULL;
3038 state->ios_left = max_ios;
3039}
3040
Jens Axboe2b188cc2019-01-07 10:46:33 -07003041static void io_commit_sqring(struct io_ring_ctx *ctx)
3042{
Hristo Venev75b28af2019-08-26 17:23:46 +00003043 struct io_rings *rings = ctx->rings;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003044
Hristo Venev75b28af2019-08-26 17:23:46 +00003045 if (ctx->cached_sq_head != READ_ONCE(rings->sq.head)) {
Jens Axboe2b188cc2019-01-07 10:46:33 -07003046 /*
3047 * Ensure any loads from the SQEs are done at this point,
3048 * since once we write the new head, the application could
3049 * write new data to them.
3050 */
Hristo Venev75b28af2019-08-26 17:23:46 +00003051 smp_store_release(&rings->sq.head, ctx->cached_sq_head);
Jens Axboe2b188cc2019-01-07 10:46:33 -07003052 }
3053}
3054
3055/*
Jens Axboe2b188cc2019-01-07 10:46:33 -07003056 * Fetch an sqe, if one is available. Note that s->sqe will point to memory
3057 * that is mapped by userspace. This means that care needs to be taken to
3058 * ensure that reads are stable, as we cannot rely on userspace always
3059 * being a good citizen. If members of the sqe are validated and then later
3060 * used, it's important that those reads are done through READ_ONCE() to
3061 * prevent a re-load down the line.
3062 */
3063static bool io_get_sqring(struct io_ring_ctx *ctx, struct sqe_submit *s)
3064{
Hristo Venev75b28af2019-08-26 17:23:46 +00003065 struct io_rings *rings = ctx->rings;
3066 u32 *sq_array = ctx->sq_array;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003067 unsigned head;
3068
3069 /*
3070 * The cached sq head (or cq tail) serves two purposes:
3071 *
3072 * 1) allows us to batch the cost of updating the user visible
3073 * head updates.
3074 * 2) allows the kernel side to track the head on its own, even
3075 * though the application is the one updating it.
3076 */
3077 head = ctx->cached_sq_head;
Stefan Bühlere523a292019-04-19 11:57:44 +02003078 /* make sure SQ entry isn't read before tail */
Hristo Venev75b28af2019-08-26 17:23:46 +00003079 if (head == smp_load_acquire(&rings->sq.tail))
Jens Axboe2b188cc2019-01-07 10:46:33 -07003080 return false;
3081
Hristo Venev75b28af2019-08-26 17:23:46 +00003082 head = READ_ONCE(sq_array[head & ctx->sq_mask]);
Jens Axboe2b188cc2019-01-07 10:46:33 -07003083 if (head < ctx->sq_entries) {
Jens Axboefcb323c2019-10-24 12:39:47 -06003084 s->ring_file = NULL;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003085 s->sqe = &ctx->sq_sqes[head];
Jackie Liu8776f3f2019-09-09 20:50:39 +08003086 s->sequence = ctx->cached_sq_head;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003087 ctx->cached_sq_head++;
3088 return true;
3089 }
3090
3091 /* drop invalid entries */
3092 ctx->cached_sq_head++;
Jens Axboe498ccd92019-10-25 10:04:25 -06003093 ctx->cached_sq_dropped++;
3094 WRITE_ONCE(rings->sq_dropped, ctx->cached_sq_dropped);
Jens Axboe2b188cc2019-01-07 10:46:33 -07003095 return false;
3096}
3097
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03003098static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
Pavel Begunkovae9428c2019-11-06 00:22:14 +03003099 struct file *ring_file, int ring_fd,
3100 struct mm_struct **mm, bool async)
Jens Axboe6c271ce2019-01-10 11:22:30 -07003101{
3102 struct io_submit_state state, *statep = NULL;
Jens Axboe9e645e112019-05-10 16:07:28 -06003103 struct io_kiocb *link = NULL;
Jackie Liu4fe2c962019-09-09 20:50:40 +08003104 struct io_kiocb *shadow_req = NULL;
Jens Axboe9e645e112019-05-10 16:07:28 -06003105 int i, submitted = 0;
Pavel Begunkov95a1b3ff2019-10-27 23:15:41 +03003106 bool mm_fault = false;
Jens Axboe6c271ce2019-01-10 11:22:30 -07003107
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07003108 if (!list_empty(&ctx->cq_overflow_list)) {
3109 io_cqring_overflow_flush(ctx, false);
3110 return -EBUSY;
3111 }
3112
Jens Axboe6c271ce2019-01-10 11:22:30 -07003113 if (nr > IO_PLUG_THRESHOLD) {
3114 io_submit_state_start(&state, ctx, nr);
3115 statep = &state;
3116 }
3117
3118 for (i = 0; i < nr; i++) {
Pavel Begunkov196be952019-11-07 01:41:06 +03003119 struct io_kiocb *req;
Pavel Begunkov50585b92019-11-07 01:41:07 +03003120 unsigned int sqe_flags;
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03003121
Pavel Begunkov196be952019-11-07 01:41:06 +03003122 req = io_get_req(ctx, statep);
3123 if (unlikely(!req)) {
3124 if (!submitted)
3125 submitted = -EAGAIN;
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03003126 break;
Pavel Begunkov196be952019-11-07 01:41:06 +03003127 }
Pavel Begunkov50585b92019-11-07 01:41:07 +03003128 if (!io_get_sqring(ctx, &req->submit)) {
Pavel Begunkov196be952019-11-07 01:41:06 +03003129 __io_free_req(req);
3130 break;
3131 }
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03003132
Pavel Begunkov50585b92019-11-07 01:41:07 +03003133 if (io_sqe_needs_user(req->submit.sqe) && !*mm) {
Pavel Begunkov95a1b3ff2019-10-27 23:15:41 +03003134 mm_fault = mm_fault || !mmget_not_zero(ctx->sqo_mm);
3135 if (!mm_fault) {
3136 use_mm(ctx->sqo_mm);
3137 *mm = ctx->sqo_mm;
3138 }
3139 }
3140
Pavel Begunkov50585b92019-11-07 01:41:07 +03003141 sqe_flags = req->submit.sqe->flags;
3142
3143 if (link && (sqe_flags & IOSQE_IO_DRAIN)) {
Jackie Liu4fe2c962019-09-09 20:50:40 +08003144 if (!shadow_req) {
3145 shadow_req = io_get_req(ctx, NULL);
Jackie Liua1041c22019-09-18 17:25:52 +08003146 if (unlikely(!shadow_req))
3147 goto out;
Jackie Liu4fe2c962019-09-09 20:50:40 +08003148 shadow_req->flags |= (REQ_F_IO_DRAIN | REQ_F_SHADOW_DRAIN);
3149 refcount_dec(&shadow_req->refs);
3150 }
Pavel Begunkov50585b92019-11-07 01:41:07 +03003151 shadow_req->sequence = req->submit.sequence;
Jackie Liu4fe2c962019-09-09 20:50:40 +08003152 }
3153
Jackie Liua1041c22019-09-18 17:25:52 +08003154out:
Pavel Begunkov50585b92019-11-07 01:41:07 +03003155 req->submit.ring_file = ring_file;
3156 req->submit.ring_fd = ring_fd;
3157 req->submit.has_user = *mm != NULL;
3158 req->submit.in_async = async;
3159 req->submit.needs_fixed_file = async;
3160 trace_io_uring_submit_sqe(ctx, req->submit.sqe->user_data,
3161 true, async);
Jackie Liua197f662019-11-08 08:09:12 -07003162 io_submit_sqe(req, statep, &link);
Pavel Begunkov95a1b3ff2019-10-27 23:15:41 +03003163 submitted++;
Pavel Begunkove5eb6362019-11-06 00:22:15 +03003164
3165 /*
3166 * If previous wasn't linked and we have a linked command,
3167 * that's the end of the chain. Submit the previous link.
3168 */
Pavel Begunkov50585b92019-11-07 01:41:07 +03003169 if (!(sqe_flags & IOSQE_IO_LINK) && link) {
Jackie Liua197f662019-11-08 08:09:12 -07003170 io_queue_link_head(link, shadow_req);
Pavel Begunkove5eb6362019-11-06 00:22:15 +03003171 link = NULL;
3172 shadow_req = NULL;
3173 }
Jens Axboe6c271ce2019-01-10 11:22:30 -07003174 }
3175
Jens Axboe9e645e112019-05-10 16:07:28 -06003176 if (link)
Jackie Liua197f662019-11-08 08:09:12 -07003177 io_queue_link_head(link, shadow_req);
Jens Axboe6c271ce2019-01-10 11:22:30 -07003178 if (statep)
3179 io_submit_state_end(&state);
3180
Pavel Begunkovae9428c2019-11-06 00:22:14 +03003181 /* Commit SQ ring head once we've consumed and submitted all SQEs */
3182 io_commit_sqring(ctx);
3183
Jens Axboe6c271ce2019-01-10 11:22:30 -07003184 return submitted;
3185}
3186
3187static int io_sq_thread(void *data)
3188{
Jens Axboe6c271ce2019-01-10 11:22:30 -07003189 struct io_ring_ctx *ctx = data;
3190 struct mm_struct *cur_mm = NULL;
3191 mm_segment_t old_fs;
3192 DEFINE_WAIT(wait);
3193 unsigned inflight;
3194 unsigned long timeout;
Jens Axboec1edbf52019-11-10 16:56:04 -07003195 int ret;
Jens Axboe6c271ce2019-01-10 11:22:30 -07003196
Jens Axboe206aefd2019-11-07 18:27:42 -07003197 complete(&ctx->completions[1]);
Jackie Liua4c0b3d2019-07-08 13:41:12 +08003198
Jens Axboe6c271ce2019-01-10 11:22:30 -07003199 old_fs = get_fs();
3200 set_fs(USER_DS);
3201
Jens Axboec1edbf52019-11-10 16:56:04 -07003202 ret = timeout = inflight = 0;
Roman Penyaev2bbcd6d2019-05-16 10:53:57 +02003203 while (!kthread_should_park()) {
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03003204 unsigned int to_submit;
Jens Axboe6c271ce2019-01-10 11:22:30 -07003205
3206 if (inflight) {
3207 unsigned nr_events = 0;
3208
3209 if (ctx->flags & IORING_SETUP_IOPOLL) {
Jens Axboe2b2ed972019-10-25 10:06:15 -06003210 /*
3211 * inflight is the count of the maximum possible
3212 * entries we submitted, but it can be smaller
3213 * if we dropped some of them. If we don't have
3214 * poll entries available, then we know that we
3215 * have nothing left to poll for. Reset the
3216 * inflight count to zero in that case.
3217 */
3218 mutex_lock(&ctx->uring_lock);
3219 if (!list_empty(&ctx->poll_list))
3220 __io_iopoll_check(ctx, &nr_events, 0);
3221 else
3222 inflight = 0;
3223 mutex_unlock(&ctx->uring_lock);
Jens Axboe6c271ce2019-01-10 11:22:30 -07003224 } else {
3225 /*
3226 * Normal IO, just pretend everything completed.
3227 * We don't have to poll completions for that.
3228 */
3229 nr_events = inflight;
3230 }
3231
3232 inflight -= nr_events;
3233 if (!inflight)
3234 timeout = jiffies + ctx->sq_thread_idle;
3235 }
3236
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03003237 to_submit = io_sqring_entries(ctx);
Jens Axboec1edbf52019-11-10 16:56:04 -07003238
3239 /*
3240 * If submit got -EBUSY, flag us as needing the application
3241 * to enter the kernel to reap and flush events.
3242 */
3243 if (!to_submit || ret == -EBUSY) {
Jens Axboe6c271ce2019-01-10 11:22:30 -07003244 /*
3245 * We're polling. If we're within the defined idle
3246 * period, then let us spin without work before going
Jens Axboec1edbf52019-11-10 16:56:04 -07003247 * to sleep. The exception is if we got EBUSY doing
3248 * more IO, we should wait for the application to
3249 * reap events and wake us up.
Jens Axboe6c271ce2019-01-10 11:22:30 -07003250 */
Jens Axboec1edbf52019-11-10 16:56:04 -07003251 if (inflight ||
3252 (!time_after(jiffies, timeout) && ret != -EBUSY)) {
Jens Axboe9831a902019-09-19 09:48:55 -06003253 cond_resched();
Jens Axboe6c271ce2019-01-10 11:22:30 -07003254 continue;
3255 }
3256
3257 /*
3258 * Drop cur_mm before scheduling, we can't hold it for
3259 * long periods (or over schedule()). Do this before
3260 * adding ourselves to the waitqueue, as the unuse/drop
3261 * may sleep.
3262 */
3263 if (cur_mm) {
3264 unuse_mm(cur_mm);
3265 mmput(cur_mm);
3266 cur_mm = NULL;
3267 }
3268
3269 prepare_to_wait(&ctx->sqo_wait, &wait,
3270 TASK_INTERRUPTIBLE);
3271
3272 /* Tell userspace we may need a wakeup call */
Hristo Venev75b28af2019-08-26 17:23:46 +00003273 ctx->rings->sq_flags |= IORING_SQ_NEED_WAKEUP;
Stefan Bühler0d7bae62019-04-19 11:57:45 +02003274 /* make sure to read SQ tail after writing flags */
3275 smp_mb();
Jens Axboe6c271ce2019-01-10 11:22:30 -07003276
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03003277 to_submit = io_sqring_entries(ctx);
Jens Axboec1edbf52019-11-10 16:56:04 -07003278 if (!to_submit || ret == -EBUSY) {
Roman Penyaev2bbcd6d2019-05-16 10:53:57 +02003279 if (kthread_should_park()) {
Jens Axboe6c271ce2019-01-10 11:22:30 -07003280 finish_wait(&ctx->sqo_wait, &wait);
3281 break;
3282 }
3283 if (signal_pending(current))
3284 flush_signals(current);
3285 schedule();
3286 finish_wait(&ctx->sqo_wait, &wait);
3287
Hristo Venev75b28af2019-08-26 17:23:46 +00003288 ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP;
Jens Axboe6c271ce2019-01-10 11:22:30 -07003289 continue;
3290 }
3291 finish_wait(&ctx->sqo_wait, &wait);
3292
Hristo Venev75b28af2019-08-26 17:23:46 +00003293 ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP;
Jens Axboe6c271ce2019-01-10 11:22:30 -07003294 }
3295
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03003296 to_submit = min(to_submit, ctx->sq_entries);
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07003297 ret = io_submit_sqes(ctx, to_submit, NULL, -1, &cur_mm, true);
3298 if (ret > 0)
3299 inflight += ret;
Jens Axboe6c271ce2019-01-10 11:22:30 -07003300 }
3301
3302 set_fs(old_fs);
3303 if (cur_mm) {
3304 unuse_mm(cur_mm);
3305 mmput(cur_mm);
3306 }
Jens Axboe06058632019-04-13 09:26:03 -06003307
Roman Penyaev2bbcd6d2019-05-16 10:53:57 +02003308 kthread_parkme();
Jens Axboe06058632019-04-13 09:26:03 -06003309
Jens Axboe6c271ce2019-01-10 11:22:30 -07003310 return 0;
3311}
3312
Jens Axboebda52162019-09-24 13:47:15 -06003313struct io_wait_queue {
3314 struct wait_queue_entry wq;
3315 struct io_ring_ctx *ctx;
3316 unsigned to_wait;
3317 unsigned nr_timeouts;
3318};
3319
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07003320static inline bool io_should_wake(struct io_wait_queue *iowq, bool noflush)
Jens Axboebda52162019-09-24 13:47:15 -06003321{
3322 struct io_ring_ctx *ctx = iowq->ctx;
3323
3324 /*
3325 * Wake up if we have enough events, or if a timeout occured since we
3326 * started waiting. For timeouts, we always want to return to userspace,
3327 * regardless of event count.
3328 */
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07003329 return io_cqring_events(ctx, noflush) >= iowq->to_wait ||
Jens Axboebda52162019-09-24 13:47:15 -06003330 atomic_read(&ctx->cq_timeouts) != iowq->nr_timeouts;
3331}
3332
3333static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode,
3334 int wake_flags, void *key)
3335{
3336 struct io_wait_queue *iowq = container_of(curr, struct io_wait_queue,
3337 wq);
3338
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07003339 /* use noflush == true, as we can't safely rely on locking context */
3340 if (!io_should_wake(iowq, true))
Jens Axboebda52162019-09-24 13:47:15 -06003341 return -1;
3342
3343 return autoremove_wake_function(curr, mode, wake_flags, key);
3344}
3345
Jens Axboe2b188cc2019-01-07 10:46:33 -07003346/*
3347 * Wait until events become available, if we don't already have some. The
3348 * application must reap them itself, as they reside on the shared cq ring.
3349 */
3350static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,
3351 const sigset_t __user *sig, size_t sigsz)
3352{
Jens Axboebda52162019-09-24 13:47:15 -06003353 struct io_wait_queue iowq = {
3354 .wq = {
3355 .private = current,
3356 .func = io_wake_function,
3357 .entry = LIST_HEAD_INIT(iowq.wq.entry),
3358 },
3359 .ctx = ctx,
3360 .to_wait = min_events,
3361 };
Hristo Venev75b28af2019-08-26 17:23:46 +00003362 struct io_rings *rings = ctx->rings;
Jackie Liue9ffa5c2019-10-29 11:16:42 +08003363 int ret = 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003364
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07003365 if (io_cqring_events(ctx, false) >= min_events)
Jens Axboe2b188cc2019-01-07 10:46:33 -07003366 return 0;
3367
3368 if (sig) {
Arnd Bergmann9e75ad52019-03-25 15:34:53 +01003369#ifdef CONFIG_COMPAT
3370 if (in_compat_syscall())
3371 ret = set_compat_user_sigmask((const compat_sigset_t __user *)sig,
Oleg Nesterovb7724342019-07-16 16:29:53 -07003372 sigsz);
Arnd Bergmann9e75ad52019-03-25 15:34:53 +01003373 else
3374#endif
Oleg Nesterovb7724342019-07-16 16:29:53 -07003375 ret = set_user_sigmask(sig, sigsz);
Arnd Bergmann9e75ad52019-03-25 15:34:53 +01003376
Jens Axboe2b188cc2019-01-07 10:46:33 -07003377 if (ret)
3378 return ret;
3379 }
3380
Jens Axboebda52162019-09-24 13:47:15 -06003381 iowq.nr_timeouts = atomic_read(&ctx->cq_timeouts);
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +02003382 trace_io_uring_cqring_wait(ctx, min_events);
Jens Axboebda52162019-09-24 13:47:15 -06003383 do {
3384 prepare_to_wait_exclusive(&ctx->wait, &iowq.wq,
3385 TASK_INTERRUPTIBLE);
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07003386 if (io_should_wake(&iowq, false))
Jens Axboebda52162019-09-24 13:47:15 -06003387 break;
3388 schedule();
3389 if (signal_pending(current)) {
Jackie Liue9ffa5c2019-10-29 11:16:42 +08003390 ret = -EINTR;
Jens Axboebda52162019-09-24 13:47:15 -06003391 break;
3392 }
3393 } while (1);
3394 finish_wait(&ctx->wait, &iowq.wq);
3395
Jackie Liue9ffa5c2019-10-29 11:16:42 +08003396 restore_saved_sigmask_unless(ret == -EINTR);
Jens Axboe2b188cc2019-01-07 10:46:33 -07003397
Hristo Venev75b28af2019-08-26 17:23:46 +00003398 return READ_ONCE(rings->cq.head) == READ_ONCE(rings->cq.tail) ? ret : 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003399}
3400
Jens Axboe6b063142019-01-10 22:13:58 -07003401static void __io_sqe_files_unregister(struct io_ring_ctx *ctx)
3402{
3403#if defined(CONFIG_UNIX)
3404 if (ctx->ring_sock) {
3405 struct sock *sock = ctx->ring_sock->sk;
3406 struct sk_buff *skb;
3407
3408 while ((skb = skb_dequeue(&sock->sk_receive_queue)) != NULL)
3409 kfree_skb(skb);
3410 }
3411#else
3412 int i;
3413
Jens Axboe65e19f52019-10-26 07:20:21 -06003414 for (i = 0; i < ctx->nr_user_files; i++) {
3415 struct file *file;
3416
3417 file = io_file_from_index(ctx, i);
3418 if (file)
3419 fput(file);
3420 }
Jens Axboe6b063142019-01-10 22:13:58 -07003421#endif
3422}
3423
3424static int io_sqe_files_unregister(struct io_ring_ctx *ctx)
3425{
Jens Axboe65e19f52019-10-26 07:20:21 -06003426 unsigned nr_tables, i;
3427
3428 if (!ctx->file_table)
Jens Axboe6b063142019-01-10 22:13:58 -07003429 return -ENXIO;
3430
3431 __io_sqe_files_unregister(ctx);
Jens Axboe65e19f52019-10-26 07:20:21 -06003432 nr_tables = DIV_ROUND_UP(ctx->nr_user_files, IORING_MAX_FILES_TABLE);
3433 for (i = 0; i < nr_tables; i++)
3434 kfree(ctx->file_table[i].files);
3435 kfree(ctx->file_table);
3436 ctx->file_table = NULL;
Jens Axboe6b063142019-01-10 22:13:58 -07003437 ctx->nr_user_files = 0;
3438 return 0;
3439}
3440
Jens Axboe6c271ce2019-01-10 11:22:30 -07003441static void io_sq_thread_stop(struct io_ring_ctx *ctx)
3442{
3443 if (ctx->sqo_thread) {
Jens Axboe206aefd2019-11-07 18:27:42 -07003444 wait_for_completion(&ctx->completions[1]);
Roman Penyaev2bbcd6d2019-05-16 10:53:57 +02003445 /*
3446 * The park is a bit of a work-around, without it we get
3447 * warning spews on shutdown with SQPOLL set and affinity
3448 * set to a single CPU.
3449 */
Jens Axboe06058632019-04-13 09:26:03 -06003450 kthread_park(ctx->sqo_thread);
Jens Axboe6c271ce2019-01-10 11:22:30 -07003451 kthread_stop(ctx->sqo_thread);
3452 ctx->sqo_thread = NULL;
3453 }
3454}
3455
Jens Axboe6b063142019-01-10 22:13:58 -07003456static void io_finish_async(struct io_ring_ctx *ctx)
3457{
Jens Axboe6c271ce2019-01-10 11:22:30 -07003458 io_sq_thread_stop(ctx);
3459
Jens Axboe561fb042019-10-24 07:25:42 -06003460 if (ctx->io_wq) {
3461 io_wq_destroy(ctx->io_wq);
3462 ctx->io_wq = NULL;
Jens Axboe6b063142019-01-10 22:13:58 -07003463 }
3464}
3465
3466#if defined(CONFIG_UNIX)
3467static void io_destruct_skb(struct sk_buff *skb)
3468{
3469 struct io_ring_ctx *ctx = skb->sk->sk_user_data;
3470
Jens Axboe561fb042019-10-24 07:25:42 -06003471 if (ctx->io_wq)
3472 io_wq_flush(ctx->io_wq);
Jens Axboe8a997342019-10-09 14:40:13 -06003473
Jens Axboe6b063142019-01-10 22:13:58 -07003474 unix_destruct_scm(skb);
3475}
3476
3477/*
3478 * Ensure the UNIX gc is aware of our file set, so we are certain that
3479 * the io_uring can be safely unregistered on process exit, even if we have
3480 * loops in the file referencing.
3481 */
3482static int __io_sqe_files_scm(struct io_ring_ctx *ctx, int nr, int offset)
3483{
3484 struct sock *sk = ctx->ring_sock->sk;
3485 struct scm_fp_list *fpl;
3486 struct sk_buff *skb;
Jens Axboe08a45172019-10-03 08:11:03 -06003487 int i, nr_files;
Jens Axboe6b063142019-01-10 22:13:58 -07003488
3489 if (!capable(CAP_SYS_RESOURCE) && !capable(CAP_SYS_ADMIN)) {
3490 unsigned long inflight = ctx->user->unix_inflight + nr;
3491
3492 if (inflight > task_rlimit(current, RLIMIT_NOFILE))
3493 return -EMFILE;
3494 }
3495
3496 fpl = kzalloc(sizeof(*fpl), GFP_KERNEL);
3497 if (!fpl)
3498 return -ENOMEM;
3499
3500 skb = alloc_skb(0, GFP_KERNEL);
3501 if (!skb) {
3502 kfree(fpl);
3503 return -ENOMEM;
3504 }
3505
3506 skb->sk = sk;
Jens Axboe6b063142019-01-10 22:13:58 -07003507
Jens Axboe08a45172019-10-03 08:11:03 -06003508 nr_files = 0;
Jens Axboe6b063142019-01-10 22:13:58 -07003509 fpl->user = get_uid(ctx->user);
3510 for (i = 0; i < nr; i++) {
Jens Axboe65e19f52019-10-26 07:20:21 -06003511 struct file *file = io_file_from_index(ctx, i + offset);
3512
3513 if (!file)
Jens Axboe08a45172019-10-03 08:11:03 -06003514 continue;
Jens Axboe65e19f52019-10-26 07:20:21 -06003515 fpl->fp[nr_files] = get_file(file);
Jens Axboe08a45172019-10-03 08:11:03 -06003516 unix_inflight(fpl->user, fpl->fp[nr_files]);
3517 nr_files++;
Jens Axboe6b063142019-01-10 22:13:58 -07003518 }
3519
Jens Axboe08a45172019-10-03 08:11:03 -06003520 if (nr_files) {
3521 fpl->max = SCM_MAX_FD;
3522 fpl->count = nr_files;
3523 UNIXCB(skb).fp = fpl;
3524 skb->destructor = io_destruct_skb;
3525 refcount_add(skb->truesize, &sk->sk_wmem_alloc);
3526 skb_queue_head(&sk->sk_receive_queue, skb);
Jens Axboe6b063142019-01-10 22:13:58 -07003527
Jens Axboe08a45172019-10-03 08:11:03 -06003528 for (i = 0; i < nr_files; i++)
3529 fput(fpl->fp[i]);
3530 } else {
3531 kfree_skb(skb);
3532 kfree(fpl);
3533 }
Jens Axboe6b063142019-01-10 22:13:58 -07003534
3535 return 0;
3536}
3537
3538/*
3539 * If UNIX sockets are enabled, fd passing can cause a reference cycle which
3540 * causes regular reference counting to break down. We rely on the UNIX
3541 * garbage collection to take care of this problem for us.
3542 */
3543static int io_sqe_files_scm(struct io_ring_ctx *ctx)
3544{
3545 unsigned left, total;
3546 int ret = 0;
3547
3548 total = 0;
3549 left = ctx->nr_user_files;
3550 while (left) {
3551 unsigned this_files = min_t(unsigned, left, SCM_MAX_FD);
Jens Axboe6b063142019-01-10 22:13:58 -07003552
3553 ret = __io_sqe_files_scm(ctx, this_files, total);
3554 if (ret)
3555 break;
3556 left -= this_files;
3557 total += this_files;
3558 }
3559
3560 if (!ret)
3561 return 0;
3562
3563 while (total < ctx->nr_user_files) {
Jens Axboe65e19f52019-10-26 07:20:21 -06003564 struct file *file = io_file_from_index(ctx, total);
3565
3566 if (file)
3567 fput(file);
Jens Axboe6b063142019-01-10 22:13:58 -07003568 total++;
3569 }
3570
3571 return ret;
3572}
3573#else
3574static int io_sqe_files_scm(struct io_ring_ctx *ctx)
3575{
3576 return 0;
3577}
3578#endif
3579
Jens Axboe65e19f52019-10-26 07:20:21 -06003580static int io_sqe_alloc_file_tables(struct io_ring_ctx *ctx, unsigned nr_tables,
3581 unsigned nr_files)
3582{
3583 int i;
3584
3585 for (i = 0; i < nr_tables; i++) {
3586 struct fixed_file_table *table = &ctx->file_table[i];
3587 unsigned this_files;
3588
3589 this_files = min(nr_files, IORING_MAX_FILES_TABLE);
3590 table->files = kcalloc(this_files, sizeof(struct file *),
3591 GFP_KERNEL);
3592 if (!table->files)
3593 break;
3594 nr_files -= this_files;
3595 }
3596
3597 if (i == nr_tables)
3598 return 0;
3599
3600 for (i = 0; i < nr_tables; i++) {
3601 struct fixed_file_table *table = &ctx->file_table[i];
3602 kfree(table->files);
3603 }
3604 return 1;
3605}
3606
Jens Axboe6b063142019-01-10 22:13:58 -07003607static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,
3608 unsigned nr_args)
3609{
3610 __s32 __user *fds = (__s32 __user *) arg;
Jens Axboe65e19f52019-10-26 07:20:21 -06003611 unsigned nr_tables;
Jens Axboe6b063142019-01-10 22:13:58 -07003612 int fd, ret = 0;
3613 unsigned i;
3614
Jens Axboe65e19f52019-10-26 07:20:21 -06003615 if (ctx->file_table)
Jens Axboe6b063142019-01-10 22:13:58 -07003616 return -EBUSY;
3617 if (!nr_args)
3618 return -EINVAL;
3619 if (nr_args > IORING_MAX_FIXED_FILES)
3620 return -EMFILE;
3621
Jens Axboe65e19f52019-10-26 07:20:21 -06003622 nr_tables = DIV_ROUND_UP(nr_args, IORING_MAX_FILES_TABLE);
3623 ctx->file_table = kcalloc(nr_tables, sizeof(struct fixed_file_table),
3624 GFP_KERNEL);
3625 if (!ctx->file_table)
Jens Axboe6b063142019-01-10 22:13:58 -07003626 return -ENOMEM;
3627
Jens Axboe65e19f52019-10-26 07:20:21 -06003628 if (io_sqe_alloc_file_tables(ctx, nr_tables, nr_args)) {
3629 kfree(ctx->file_table);
Jens Axboe46568e92019-11-10 08:40:53 -07003630 ctx->file_table = NULL;
Jens Axboe65e19f52019-10-26 07:20:21 -06003631 return -ENOMEM;
3632 }
3633
Jens Axboe08a45172019-10-03 08:11:03 -06003634 for (i = 0; i < nr_args; i++, ctx->nr_user_files++) {
Jens Axboe65e19f52019-10-26 07:20:21 -06003635 struct fixed_file_table *table;
3636 unsigned index;
3637
Jens Axboe6b063142019-01-10 22:13:58 -07003638 ret = -EFAULT;
3639 if (copy_from_user(&fd, &fds[i], sizeof(fd)))
3640 break;
Jens Axboe08a45172019-10-03 08:11:03 -06003641 /* allow sparse sets */
3642 if (fd == -1) {
3643 ret = 0;
3644 continue;
3645 }
Jens Axboe6b063142019-01-10 22:13:58 -07003646
Jens Axboe65e19f52019-10-26 07:20:21 -06003647 table = &ctx->file_table[i >> IORING_FILE_TABLE_SHIFT];
3648 index = i & IORING_FILE_TABLE_MASK;
3649 table->files[index] = fget(fd);
Jens Axboe6b063142019-01-10 22:13:58 -07003650
3651 ret = -EBADF;
Jens Axboe65e19f52019-10-26 07:20:21 -06003652 if (!table->files[index])
Jens Axboe6b063142019-01-10 22:13:58 -07003653 break;
3654 /*
3655 * Don't allow io_uring instances to be registered. If UNIX
3656 * isn't enabled, then this causes a reference cycle and this
3657 * instance can never get freed. If UNIX is enabled we'll
3658 * handle it just fine, but there's still no point in allowing
3659 * a ring fd as it doesn't support regular read/write anyway.
3660 */
Jens Axboe65e19f52019-10-26 07:20:21 -06003661 if (table->files[index]->f_op == &io_uring_fops) {
3662 fput(table->files[index]);
Jens Axboe6b063142019-01-10 22:13:58 -07003663 break;
3664 }
Jens Axboe6b063142019-01-10 22:13:58 -07003665 ret = 0;
3666 }
3667
3668 if (ret) {
Jens Axboe65e19f52019-10-26 07:20:21 -06003669 for (i = 0; i < ctx->nr_user_files; i++) {
3670 struct file *file;
Jens Axboe6b063142019-01-10 22:13:58 -07003671
Jens Axboe65e19f52019-10-26 07:20:21 -06003672 file = io_file_from_index(ctx, i);
3673 if (file)
3674 fput(file);
3675 }
3676 for (i = 0; i < nr_tables; i++)
3677 kfree(ctx->file_table[i].files);
3678
3679 kfree(ctx->file_table);
3680 ctx->file_table = NULL;
Jens Axboe6b063142019-01-10 22:13:58 -07003681 ctx->nr_user_files = 0;
3682 return ret;
3683 }
3684
3685 ret = io_sqe_files_scm(ctx);
3686 if (ret)
3687 io_sqe_files_unregister(ctx);
3688
3689 return ret;
3690}
3691
Jens Axboec3a31e62019-10-03 13:59:56 -06003692static void io_sqe_file_unregister(struct io_ring_ctx *ctx, int index)
3693{
3694#if defined(CONFIG_UNIX)
Jens Axboe65e19f52019-10-26 07:20:21 -06003695 struct file *file = io_file_from_index(ctx, index);
Jens Axboec3a31e62019-10-03 13:59:56 -06003696 struct sock *sock = ctx->ring_sock->sk;
3697 struct sk_buff_head list, *head = &sock->sk_receive_queue;
3698 struct sk_buff *skb;
3699 int i;
3700
3701 __skb_queue_head_init(&list);
3702
3703 /*
3704 * Find the skb that holds this file in its SCM_RIGHTS. When found,
3705 * remove this entry and rearrange the file array.
3706 */
3707 skb = skb_dequeue(head);
3708 while (skb) {
3709 struct scm_fp_list *fp;
3710
3711 fp = UNIXCB(skb).fp;
3712 for (i = 0; i < fp->count; i++) {
3713 int left;
3714
3715 if (fp->fp[i] != file)
3716 continue;
3717
3718 unix_notinflight(fp->user, fp->fp[i]);
3719 left = fp->count - 1 - i;
3720 if (left) {
3721 memmove(&fp->fp[i], &fp->fp[i + 1],
3722 left * sizeof(struct file *));
3723 }
3724 fp->count--;
3725 if (!fp->count) {
3726 kfree_skb(skb);
3727 skb = NULL;
3728 } else {
3729 __skb_queue_tail(&list, skb);
3730 }
3731 fput(file);
3732 file = NULL;
3733 break;
3734 }
3735
3736 if (!file)
3737 break;
3738
3739 __skb_queue_tail(&list, skb);
3740
3741 skb = skb_dequeue(head);
3742 }
3743
3744 if (skb_peek(&list)) {
3745 spin_lock_irq(&head->lock);
3746 while ((skb = __skb_dequeue(&list)) != NULL)
3747 __skb_queue_tail(head, skb);
3748 spin_unlock_irq(&head->lock);
3749 }
3750#else
Jens Axboe65e19f52019-10-26 07:20:21 -06003751 fput(io_file_from_index(ctx, index));
Jens Axboec3a31e62019-10-03 13:59:56 -06003752#endif
3753}
3754
3755static int io_sqe_file_register(struct io_ring_ctx *ctx, struct file *file,
3756 int index)
3757{
3758#if defined(CONFIG_UNIX)
3759 struct sock *sock = ctx->ring_sock->sk;
3760 struct sk_buff_head *head = &sock->sk_receive_queue;
3761 struct sk_buff *skb;
3762
3763 /*
3764 * See if we can merge this file into an existing skb SCM_RIGHTS
3765 * file set. If there's no room, fall back to allocating a new skb
3766 * and filling it in.
3767 */
3768 spin_lock_irq(&head->lock);
3769 skb = skb_peek(head);
3770 if (skb) {
3771 struct scm_fp_list *fpl = UNIXCB(skb).fp;
3772
3773 if (fpl->count < SCM_MAX_FD) {
3774 __skb_unlink(skb, head);
3775 spin_unlock_irq(&head->lock);
3776 fpl->fp[fpl->count] = get_file(file);
3777 unix_inflight(fpl->user, fpl->fp[fpl->count]);
3778 fpl->count++;
3779 spin_lock_irq(&head->lock);
3780 __skb_queue_head(head, skb);
3781 } else {
3782 skb = NULL;
3783 }
3784 }
3785 spin_unlock_irq(&head->lock);
3786
3787 if (skb) {
3788 fput(file);
3789 return 0;
3790 }
3791
3792 return __io_sqe_files_scm(ctx, 1, index);
3793#else
3794 return 0;
3795#endif
3796}
3797
3798static int io_sqe_files_update(struct io_ring_ctx *ctx, void __user *arg,
3799 unsigned nr_args)
3800{
3801 struct io_uring_files_update up;
3802 __s32 __user *fds;
3803 int fd, i, err;
3804 __u32 done;
3805
Jens Axboe65e19f52019-10-26 07:20:21 -06003806 if (!ctx->file_table)
Jens Axboec3a31e62019-10-03 13:59:56 -06003807 return -ENXIO;
3808 if (!nr_args)
3809 return -EINVAL;
3810 if (copy_from_user(&up, arg, sizeof(up)))
3811 return -EFAULT;
3812 if (check_add_overflow(up.offset, nr_args, &done))
3813 return -EOVERFLOW;
3814 if (done > ctx->nr_user_files)
3815 return -EINVAL;
3816
3817 done = 0;
3818 fds = (__s32 __user *) up.fds;
3819 while (nr_args) {
Jens Axboe65e19f52019-10-26 07:20:21 -06003820 struct fixed_file_table *table;
3821 unsigned index;
3822
Jens Axboec3a31e62019-10-03 13:59:56 -06003823 err = 0;
3824 if (copy_from_user(&fd, &fds[done], sizeof(fd))) {
3825 err = -EFAULT;
3826 break;
3827 }
3828 i = array_index_nospec(up.offset, ctx->nr_user_files);
Jens Axboe65e19f52019-10-26 07:20:21 -06003829 table = &ctx->file_table[i >> IORING_FILE_TABLE_SHIFT];
3830 index = i & IORING_FILE_TABLE_MASK;
3831 if (table->files[index]) {
Jens Axboec3a31e62019-10-03 13:59:56 -06003832 io_sqe_file_unregister(ctx, i);
Jens Axboe65e19f52019-10-26 07:20:21 -06003833 table->files[index] = NULL;
Jens Axboec3a31e62019-10-03 13:59:56 -06003834 }
3835 if (fd != -1) {
3836 struct file *file;
3837
3838 file = fget(fd);
3839 if (!file) {
3840 err = -EBADF;
3841 break;
3842 }
3843 /*
3844 * Don't allow io_uring instances to be registered. If
3845 * UNIX isn't enabled, then this causes a reference
3846 * cycle and this instance can never get freed. If UNIX
3847 * is enabled we'll handle it just fine, but there's
3848 * still no point in allowing a ring fd as it doesn't
3849 * support regular read/write anyway.
3850 */
3851 if (file->f_op == &io_uring_fops) {
3852 fput(file);
3853 err = -EBADF;
3854 break;
3855 }
Jens Axboe65e19f52019-10-26 07:20:21 -06003856 table->files[index] = file;
Jens Axboec3a31e62019-10-03 13:59:56 -06003857 err = io_sqe_file_register(ctx, file, i);
3858 if (err)
3859 break;
3860 }
3861 nr_args--;
3862 done++;
3863 up.offset++;
3864 }
3865
3866 return done ? done : err;
3867}
3868
Jens Axboe7d723062019-11-12 22:31:31 -07003869static void io_put_work(struct io_wq_work *work)
3870{
3871 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
3872
3873 io_put_req(req);
3874}
3875
3876static void io_get_work(struct io_wq_work *work)
3877{
3878 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
3879
3880 refcount_inc(&req->refs);
3881}
3882
Jens Axboe6c271ce2019-01-10 11:22:30 -07003883static int io_sq_offload_start(struct io_ring_ctx *ctx,
3884 struct io_uring_params *p)
Jens Axboe2b188cc2019-01-07 10:46:33 -07003885{
Jens Axboe561fb042019-10-24 07:25:42 -06003886 unsigned concurrency;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003887 int ret;
3888
Jens Axboe6c271ce2019-01-10 11:22:30 -07003889 init_waitqueue_head(&ctx->sqo_wait);
Jens Axboe2b188cc2019-01-07 10:46:33 -07003890 mmgrab(current->mm);
3891 ctx->sqo_mm = current->mm;
3892
Jens Axboe6c271ce2019-01-10 11:22:30 -07003893 if (ctx->flags & IORING_SETUP_SQPOLL) {
Jens Axboe3ec482d2019-04-08 10:51:01 -06003894 ret = -EPERM;
3895 if (!capable(CAP_SYS_ADMIN))
3896 goto err;
3897
Jens Axboe917257d2019-04-13 09:28:55 -06003898 ctx->sq_thread_idle = msecs_to_jiffies(p->sq_thread_idle);
3899 if (!ctx->sq_thread_idle)
3900 ctx->sq_thread_idle = HZ;
3901
Jens Axboe6c271ce2019-01-10 11:22:30 -07003902 if (p->flags & IORING_SETUP_SQ_AFF) {
Jens Axboe44a9bd12019-05-14 20:00:30 -06003903 int cpu = p->sq_thread_cpu;
Jens Axboe6c271ce2019-01-10 11:22:30 -07003904
Jens Axboe917257d2019-04-13 09:28:55 -06003905 ret = -EINVAL;
Jens Axboe44a9bd12019-05-14 20:00:30 -06003906 if (cpu >= nr_cpu_ids)
3907 goto err;
Shenghui Wang7889f442019-05-07 16:03:19 +08003908 if (!cpu_online(cpu))
Jens Axboe917257d2019-04-13 09:28:55 -06003909 goto err;
3910
Jens Axboe6c271ce2019-01-10 11:22:30 -07003911 ctx->sqo_thread = kthread_create_on_cpu(io_sq_thread,
3912 ctx, cpu,
3913 "io_uring-sq");
3914 } else {
3915 ctx->sqo_thread = kthread_create(io_sq_thread, ctx,
3916 "io_uring-sq");
3917 }
3918 if (IS_ERR(ctx->sqo_thread)) {
3919 ret = PTR_ERR(ctx->sqo_thread);
3920 ctx->sqo_thread = NULL;
3921 goto err;
3922 }
3923 wake_up_process(ctx->sqo_thread);
3924 } else if (p->flags & IORING_SETUP_SQ_AFF) {
3925 /* Can't have SQ_AFF without SQPOLL */
3926 ret = -EINVAL;
3927 goto err;
3928 }
3929
Jens Axboe561fb042019-10-24 07:25:42 -06003930 /* Do QD, or 4 * CPUS, whatever is smallest */
3931 concurrency = min(ctx->sq_entries, 4 * num_online_cpus());
Jens Axboe7d723062019-11-12 22:31:31 -07003932 ctx->io_wq = io_wq_create(concurrency, ctx->sqo_mm, ctx->user,
3933 io_get_work, io_put_work);
Jens Axboe975c99a52019-10-30 08:42:56 -06003934 if (IS_ERR(ctx->io_wq)) {
3935 ret = PTR_ERR(ctx->io_wq);
3936 ctx->io_wq = NULL;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003937 goto err;
3938 }
3939
3940 return 0;
3941err:
Jens Axboe54a91f32019-09-10 09:15:04 -06003942 io_finish_async(ctx);
Jens Axboe2b188cc2019-01-07 10:46:33 -07003943 mmdrop(ctx->sqo_mm);
3944 ctx->sqo_mm = NULL;
3945 return ret;
3946}
3947
3948static void io_unaccount_mem(struct user_struct *user, unsigned long nr_pages)
3949{
3950 atomic_long_sub(nr_pages, &user->locked_vm);
3951}
3952
3953static int io_account_mem(struct user_struct *user, unsigned long nr_pages)
3954{
3955 unsigned long page_limit, cur_pages, new_pages;
3956
3957 /* Don't allow more pages than we can safely lock */
3958 page_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
3959
3960 do {
3961 cur_pages = atomic_long_read(&user->locked_vm);
3962 new_pages = cur_pages + nr_pages;
3963 if (new_pages > page_limit)
3964 return -ENOMEM;
3965 } while (atomic_long_cmpxchg(&user->locked_vm, cur_pages,
3966 new_pages) != cur_pages);
3967
3968 return 0;
3969}
3970
3971static void io_mem_free(void *ptr)
3972{
Mark Rutland52e04ef2019-04-30 17:30:21 +01003973 struct page *page;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003974
Mark Rutland52e04ef2019-04-30 17:30:21 +01003975 if (!ptr)
3976 return;
3977
3978 page = virt_to_head_page(ptr);
Jens Axboe2b188cc2019-01-07 10:46:33 -07003979 if (put_page_testzero(page))
3980 free_compound_page(page);
3981}
3982
3983static void *io_mem_alloc(size_t size)
3984{
3985 gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN | __GFP_COMP |
3986 __GFP_NORETRY;
3987
3988 return (void *) __get_free_pages(gfp_flags, get_order(size));
3989}
3990
Hristo Venev75b28af2019-08-26 17:23:46 +00003991static unsigned long rings_size(unsigned sq_entries, unsigned cq_entries,
3992 size_t *sq_offset)
3993{
3994 struct io_rings *rings;
3995 size_t off, sq_array_size;
3996
3997 off = struct_size(rings, cqes, cq_entries);
3998 if (off == SIZE_MAX)
3999 return SIZE_MAX;
4000
4001#ifdef CONFIG_SMP
4002 off = ALIGN(off, SMP_CACHE_BYTES);
4003 if (off == 0)
4004 return SIZE_MAX;
4005#endif
4006
4007 sq_array_size = array_size(sizeof(u32), sq_entries);
4008 if (sq_array_size == SIZE_MAX)
4009 return SIZE_MAX;
4010
4011 if (check_add_overflow(off, sq_array_size, &off))
4012 return SIZE_MAX;
4013
4014 if (sq_offset)
4015 *sq_offset = off;
4016
4017 return off;
4018}
4019
Jens Axboe2b188cc2019-01-07 10:46:33 -07004020static unsigned long ring_pages(unsigned sq_entries, unsigned cq_entries)
4021{
Hristo Venev75b28af2019-08-26 17:23:46 +00004022 size_t pages;
Jens Axboe2b188cc2019-01-07 10:46:33 -07004023
Hristo Venev75b28af2019-08-26 17:23:46 +00004024 pages = (size_t)1 << get_order(
4025 rings_size(sq_entries, cq_entries, NULL));
4026 pages += (size_t)1 << get_order(
4027 array_size(sizeof(struct io_uring_sqe), sq_entries));
Jens Axboe2b188cc2019-01-07 10:46:33 -07004028
Hristo Venev75b28af2019-08-26 17:23:46 +00004029 return pages;
Jens Axboe2b188cc2019-01-07 10:46:33 -07004030}
4031
Jens Axboeedafcce2019-01-09 09:16:05 -07004032static int io_sqe_buffer_unregister(struct io_ring_ctx *ctx)
4033{
4034 int i, j;
4035
4036 if (!ctx->user_bufs)
4037 return -ENXIO;
4038
4039 for (i = 0; i < ctx->nr_user_bufs; i++) {
4040 struct io_mapped_ubuf *imu = &ctx->user_bufs[i];
4041
4042 for (j = 0; j < imu->nr_bvecs; j++)
John Hubbard27c4d3a2019-08-04 19:32:06 -07004043 put_user_page(imu->bvec[j].bv_page);
Jens Axboeedafcce2019-01-09 09:16:05 -07004044
4045 if (ctx->account_mem)
4046 io_unaccount_mem(ctx->user, imu->nr_bvecs);
Mark Rutlandd4ef6472019-05-01 16:59:16 +01004047 kvfree(imu->bvec);
Jens Axboeedafcce2019-01-09 09:16:05 -07004048 imu->nr_bvecs = 0;
4049 }
4050
4051 kfree(ctx->user_bufs);
4052 ctx->user_bufs = NULL;
4053 ctx->nr_user_bufs = 0;
4054 return 0;
4055}
4056
4057static int io_copy_iov(struct io_ring_ctx *ctx, struct iovec *dst,
4058 void __user *arg, unsigned index)
4059{
4060 struct iovec __user *src;
4061
4062#ifdef CONFIG_COMPAT
4063 if (ctx->compat) {
4064 struct compat_iovec __user *ciovs;
4065 struct compat_iovec ciov;
4066
4067 ciovs = (struct compat_iovec __user *) arg;
4068 if (copy_from_user(&ciov, &ciovs[index], sizeof(ciov)))
4069 return -EFAULT;
4070
4071 dst->iov_base = (void __user *) (unsigned long) ciov.iov_base;
4072 dst->iov_len = ciov.iov_len;
4073 return 0;
4074 }
4075#endif
4076 src = (struct iovec __user *) arg;
4077 if (copy_from_user(dst, &src[index], sizeof(*dst)))
4078 return -EFAULT;
4079 return 0;
4080}
4081
4082static int io_sqe_buffer_register(struct io_ring_ctx *ctx, void __user *arg,
4083 unsigned nr_args)
4084{
4085 struct vm_area_struct **vmas = NULL;
4086 struct page **pages = NULL;
4087 int i, j, got_pages = 0;
4088 int ret = -EINVAL;
4089
4090 if (ctx->user_bufs)
4091 return -EBUSY;
4092 if (!nr_args || nr_args > UIO_MAXIOV)
4093 return -EINVAL;
4094
4095 ctx->user_bufs = kcalloc(nr_args, sizeof(struct io_mapped_ubuf),
4096 GFP_KERNEL);
4097 if (!ctx->user_bufs)
4098 return -ENOMEM;
4099
4100 for (i = 0; i < nr_args; i++) {
4101 struct io_mapped_ubuf *imu = &ctx->user_bufs[i];
4102 unsigned long off, start, end, ubuf;
4103 int pret, nr_pages;
4104 struct iovec iov;
4105 size_t size;
4106
4107 ret = io_copy_iov(ctx, &iov, arg, i);
4108 if (ret)
Pavel Begunkova2786822019-05-26 12:35:47 +03004109 goto err;
Jens Axboeedafcce2019-01-09 09:16:05 -07004110
4111 /*
4112 * Don't impose further limits on the size and buffer
4113 * constraints here, we'll -EINVAL later when IO is
4114 * submitted if they are wrong.
4115 */
4116 ret = -EFAULT;
4117 if (!iov.iov_base || !iov.iov_len)
4118 goto err;
4119
4120 /* arbitrary limit, but we need something */
4121 if (iov.iov_len > SZ_1G)
4122 goto err;
4123
4124 ubuf = (unsigned long) iov.iov_base;
4125 end = (ubuf + iov.iov_len + PAGE_SIZE - 1) >> PAGE_SHIFT;
4126 start = ubuf >> PAGE_SHIFT;
4127 nr_pages = end - start;
4128
4129 if (ctx->account_mem) {
4130 ret = io_account_mem(ctx->user, nr_pages);
4131 if (ret)
4132 goto err;
4133 }
4134
4135 ret = 0;
4136 if (!pages || nr_pages > got_pages) {
4137 kfree(vmas);
4138 kfree(pages);
Mark Rutlandd4ef6472019-05-01 16:59:16 +01004139 pages = kvmalloc_array(nr_pages, sizeof(struct page *),
Jens Axboeedafcce2019-01-09 09:16:05 -07004140 GFP_KERNEL);
Mark Rutlandd4ef6472019-05-01 16:59:16 +01004141 vmas = kvmalloc_array(nr_pages,
Jens Axboeedafcce2019-01-09 09:16:05 -07004142 sizeof(struct vm_area_struct *),
4143 GFP_KERNEL);
4144 if (!pages || !vmas) {
4145 ret = -ENOMEM;
4146 if (ctx->account_mem)
4147 io_unaccount_mem(ctx->user, nr_pages);
4148 goto err;
4149 }
4150 got_pages = nr_pages;
4151 }
4152
Mark Rutlandd4ef6472019-05-01 16:59:16 +01004153 imu->bvec = kvmalloc_array(nr_pages, sizeof(struct bio_vec),
Jens Axboeedafcce2019-01-09 09:16:05 -07004154 GFP_KERNEL);
4155 ret = -ENOMEM;
4156 if (!imu->bvec) {
4157 if (ctx->account_mem)
4158 io_unaccount_mem(ctx->user, nr_pages);
4159 goto err;
4160 }
4161
4162 ret = 0;
4163 down_read(&current->mm->mmap_sem);
Ira Weiny932f4a62019-05-13 17:17:03 -07004164 pret = get_user_pages(ubuf, nr_pages,
4165 FOLL_WRITE | FOLL_LONGTERM,
4166 pages, vmas);
Jens Axboeedafcce2019-01-09 09:16:05 -07004167 if (pret == nr_pages) {
4168 /* don't support file backed memory */
4169 for (j = 0; j < nr_pages; j++) {
4170 struct vm_area_struct *vma = vmas[j];
4171
4172 if (vma->vm_file &&
4173 !is_file_hugepages(vma->vm_file)) {
4174 ret = -EOPNOTSUPP;
4175 break;
4176 }
4177 }
4178 } else {
4179 ret = pret < 0 ? pret : -EFAULT;
4180 }
4181 up_read(&current->mm->mmap_sem);
4182 if (ret) {
4183 /*
4184 * if we did partial map, or found file backed vmas,
4185 * release any pages we did get
4186 */
John Hubbard27c4d3a2019-08-04 19:32:06 -07004187 if (pret > 0)
4188 put_user_pages(pages, pret);
Jens Axboeedafcce2019-01-09 09:16:05 -07004189 if (ctx->account_mem)
4190 io_unaccount_mem(ctx->user, nr_pages);
Mark Rutlandd4ef6472019-05-01 16:59:16 +01004191 kvfree(imu->bvec);
Jens Axboeedafcce2019-01-09 09:16:05 -07004192 goto err;
4193 }
4194
4195 off = ubuf & ~PAGE_MASK;
4196 size = iov.iov_len;
4197 for (j = 0; j < nr_pages; j++) {
4198 size_t vec_len;
4199
4200 vec_len = min_t(size_t, size, PAGE_SIZE - off);
4201 imu->bvec[j].bv_page = pages[j];
4202 imu->bvec[j].bv_len = vec_len;
4203 imu->bvec[j].bv_offset = off;
4204 off = 0;
4205 size -= vec_len;
4206 }
4207 /* store original address for later verification */
4208 imu->ubuf = ubuf;
4209 imu->len = iov.iov_len;
4210 imu->nr_bvecs = nr_pages;
4211
4212 ctx->nr_user_bufs++;
4213 }
Mark Rutlandd4ef6472019-05-01 16:59:16 +01004214 kvfree(pages);
4215 kvfree(vmas);
Jens Axboeedafcce2019-01-09 09:16:05 -07004216 return 0;
4217err:
Mark Rutlandd4ef6472019-05-01 16:59:16 +01004218 kvfree(pages);
4219 kvfree(vmas);
Jens Axboeedafcce2019-01-09 09:16:05 -07004220 io_sqe_buffer_unregister(ctx);
4221 return ret;
4222}
4223
Jens Axboe9b402842019-04-11 11:45:41 -06004224static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg)
4225{
4226 __s32 __user *fds = arg;
4227 int fd;
4228
4229 if (ctx->cq_ev_fd)
4230 return -EBUSY;
4231
4232 if (copy_from_user(&fd, fds, sizeof(*fds)))
4233 return -EFAULT;
4234
4235 ctx->cq_ev_fd = eventfd_ctx_fdget(fd);
4236 if (IS_ERR(ctx->cq_ev_fd)) {
4237 int ret = PTR_ERR(ctx->cq_ev_fd);
4238 ctx->cq_ev_fd = NULL;
4239 return ret;
4240 }
4241
4242 return 0;
4243}
4244
4245static int io_eventfd_unregister(struct io_ring_ctx *ctx)
4246{
4247 if (ctx->cq_ev_fd) {
4248 eventfd_ctx_put(ctx->cq_ev_fd);
4249 ctx->cq_ev_fd = NULL;
4250 return 0;
4251 }
4252
4253 return -ENXIO;
4254}
4255
Jens Axboe2b188cc2019-01-07 10:46:33 -07004256static void io_ring_ctx_free(struct io_ring_ctx *ctx)
4257{
Jens Axboe6b063142019-01-10 22:13:58 -07004258 io_finish_async(ctx);
Jens Axboe2b188cc2019-01-07 10:46:33 -07004259 if (ctx->sqo_mm)
4260 mmdrop(ctx->sqo_mm);
Jens Axboedef596e2019-01-09 08:59:42 -07004261
4262 io_iopoll_reap_events(ctx);
Jens Axboeedafcce2019-01-09 09:16:05 -07004263 io_sqe_buffer_unregister(ctx);
Jens Axboe6b063142019-01-10 22:13:58 -07004264 io_sqe_files_unregister(ctx);
Jens Axboe9b402842019-04-11 11:45:41 -06004265 io_eventfd_unregister(ctx);
Jens Axboedef596e2019-01-09 08:59:42 -07004266
Jens Axboe2b188cc2019-01-07 10:46:33 -07004267#if defined(CONFIG_UNIX)
Eric Biggers355e8d22019-06-12 14:58:43 -07004268 if (ctx->ring_sock) {
4269 ctx->ring_sock->file = NULL; /* so that iput() is called */
Jens Axboe2b188cc2019-01-07 10:46:33 -07004270 sock_release(ctx->ring_sock);
Eric Biggers355e8d22019-06-12 14:58:43 -07004271 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07004272#endif
4273
Hristo Venev75b28af2019-08-26 17:23:46 +00004274 io_mem_free(ctx->rings);
Jens Axboe2b188cc2019-01-07 10:46:33 -07004275 io_mem_free(ctx->sq_sqes);
Jens Axboe2b188cc2019-01-07 10:46:33 -07004276
4277 percpu_ref_exit(&ctx->refs);
4278 if (ctx->account_mem)
4279 io_unaccount_mem(ctx->user,
4280 ring_pages(ctx->sq_entries, ctx->cq_entries));
4281 free_uid(ctx->user);
Jens Axboe206aefd2019-11-07 18:27:42 -07004282 kfree(ctx->completions);
Jens Axboe0ddf92e2019-11-08 08:52:53 -07004283 kmem_cache_free(req_cachep, ctx->fallback_req);
Jens Axboe2b188cc2019-01-07 10:46:33 -07004284 kfree(ctx);
4285}
4286
4287static __poll_t io_uring_poll(struct file *file, poll_table *wait)
4288{
4289 struct io_ring_ctx *ctx = file->private_data;
4290 __poll_t mask = 0;
4291
4292 poll_wait(file, &ctx->cq_wait, wait);
Stefan Bühler4f7067c2019-04-24 23:54:17 +02004293 /*
4294 * synchronizes with barrier from wq_has_sleeper call in
4295 * io_commit_cqring
4296 */
Jens Axboe2b188cc2019-01-07 10:46:33 -07004297 smp_rmb();
Hristo Venev75b28af2019-08-26 17:23:46 +00004298 if (READ_ONCE(ctx->rings->sq.tail) - ctx->cached_sq_head !=
4299 ctx->rings->sq_ring_entries)
Jens Axboe2b188cc2019-01-07 10:46:33 -07004300 mask |= EPOLLOUT | EPOLLWRNORM;
yangerkundaa5de52019-09-24 20:53:34 +08004301 if (READ_ONCE(ctx->rings->cq.head) != ctx->cached_cq_tail)
Jens Axboe2b188cc2019-01-07 10:46:33 -07004302 mask |= EPOLLIN | EPOLLRDNORM;
4303
4304 return mask;
4305}
4306
4307static int io_uring_fasync(int fd, struct file *file, int on)
4308{
4309 struct io_ring_ctx *ctx = file->private_data;
4310
4311 return fasync_helper(fd, file, on, &ctx->cq_fasync);
4312}
4313
4314static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)
4315{
4316 mutex_lock(&ctx->uring_lock);
4317 percpu_ref_kill(&ctx->refs);
4318 mutex_unlock(&ctx->uring_lock);
4319
Jens Axboe5262f562019-09-17 12:26:57 -06004320 io_kill_timeouts(ctx);
Jens Axboe221c5eb2019-01-17 09:41:58 -07004321 io_poll_remove_all(ctx);
Jens Axboe561fb042019-10-24 07:25:42 -06004322
4323 if (ctx->io_wq)
4324 io_wq_cancel_all(ctx->io_wq);
4325
Jens Axboedef596e2019-01-09 08:59:42 -07004326 io_iopoll_reap_events(ctx);
Jens Axboe15dff282019-11-13 09:09:23 -07004327 /* if we failed setting up the ctx, we might not have any rings */
4328 if (ctx->rings)
4329 io_cqring_overflow_flush(ctx, true);
Jens Axboe206aefd2019-11-07 18:27:42 -07004330 wait_for_completion(&ctx->completions[0]);
Jens Axboe2b188cc2019-01-07 10:46:33 -07004331 io_ring_ctx_free(ctx);
4332}
4333
4334static int io_uring_release(struct inode *inode, struct file *file)
4335{
4336 struct io_ring_ctx *ctx = file->private_data;
4337
4338 file->private_data = NULL;
4339 io_ring_ctx_wait_and_kill(ctx);
4340 return 0;
4341}
4342
Jens Axboefcb323c2019-10-24 12:39:47 -06004343static void io_uring_cancel_files(struct io_ring_ctx *ctx,
4344 struct files_struct *files)
4345{
4346 struct io_kiocb *req;
4347 DEFINE_WAIT(wait);
4348
4349 while (!list_empty_careful(&ctx->inflight_list)) {
Jens Axboe768134d2019-11-10 20:30:53 -07004350 struct io_kiocb *cancel_req = NULL;
Jens Axboefcb323c2019-10-24 12:39:47 -06004351
4352 spin_lock_irq(&ctx->inflight_lock);
4353 list_for_each_entry(req, &ctx->inflight_list, inflight_entry) {
Jens Axboe768134d2019-11-10 20:30:53 -07004354 if (req->work.files != files)
4355 continue;
4356 /* req is being completed, ignore */
4357 if (!refcount_inc_not_zero(&req->refs))
4358 continue;
4359 cancel_req = req;
4360 break;
Jens Axboefcb323c2019-10-24 12:39:47 -06004361 }
Jens Axboe768134d2019-11-10 20:30:53 -07004362 if (cancel_req)
Jens Axboefcb323c2019-10-24 12:39:47 -06004363 prepare_to_wait(&ctx->inflight_wait, &wait,
Jens Axboe768134d2019-11-10 20:30:53 -07004364 TASK_UNINTERRUPTIBLE);
Jens Axboefcb323c2019-10-24 12:39:47 -06004365 spin_unlock_irq(&ctx->inflight_lock);
4366
Jens Axboe768134d2019-11-10 20:30:53 -07004367 /* We need to keep going until we don't find a matching req */
4368 if (!cancel_req)
Jens Axboefcb323c2019-10-24 12:39:47 -06004369 break;
Bob Liu2f6d9b92019-11-13 18:06:24 +08004370
4371 io_wq_cancel_work(ctx->io_wq, &cancel_req->work);
4372 io_put_req(cancel_req);
Jens Axboefcb323c2019-10-24 12:39:47 -06004373 schedule();
4374 }
Jens Axboe768134d2019-11-10 20:30:53 -07004375 finish_wait(&ctx->inflight_wait, &wait);
Jens Axboefcb323c2019-10-24 12:39:47 -06004376}
4377
4378static int io_uring_flush(struct file *file, void *data)
4379{
4380 struct io_ring_ctx *ctx = file->private_data;
4381
4382 io_uring_cancel_files(ctx, data);
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07004383 if (fatal_signal_pending(current) || (current->flags & PF_EXITING)) {
4384 io_cqring_overflow_flush(ctx, true);
Jens Axboefcb323c2019-10-24 12:39:47 -06004385 io_wq_cancel_all(ctx->io_wq);
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07004386 }
Jens Axboefcb323c2019-10-24 12:39:47 -06004387 return 0;
4388}
4389
Jens Axboe2b188cc2019-01-07 10:46:33 -07004390static int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
4391{
4392 loff_t offset = (loff_t) vma->vm_pgoff << PAGE_SHIFT;
4393 unsigned long sz = vma->vm_end - vma->vm_start;
4394 struct io_ring_ctx *ctx = file->private_data;
4395 unsigned long pfn;
4396 struct page *page;
4397 void *ptr;
4398
4399 switch (offset) {
4400 case IORING_OFF_SQ_RING:
Hristo Venev75b28af2019-08-26 17:23:46 +00004401 case IORING_OFF_CQ_RING:
4402 ptr = ctx->rings;
Jens Axboe2b188cc2019-01-07 10:46:33 -07004403 break;
4404 case IORING_OFF_SQES:
4405 ptr = ctx->sq_sqes;
4406 break;
Jens Axboe2b188cc2019-01-07 10:46:33 -07004407 default:
4408 return -EINVAL;
4409 }
4410
4411 page = virt_to_head_page(ptr);
Matthew Wilcox (Oracle)a50b8542019-09-23 15:34:25 -07004412 if (sz > page_size(page))
Jens Axboe2b188cc2019-01-07 10:46:33 -07004413 return -EINVAL;
4414
4415 pfn = virt_to_phys(ptr) >> PAGE_SHIFT;
4416 return remap_pfn_range(vma, vma->vm_start, pfn, sz, vma->vm_page_prot);
4417}
4418
4419SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
4420 u32, min_complete, u32, flags, const sigset_t __user *, sig,
4421 size_t, sigsz)
4422{
4423 struct io_ring_ctx *ctx;
4424 long ret = -EBADF;
4425 int submitted = 0;
4426 struct fd f;
4427
Jens Axboe6c271ce2019-01-10 11:22:30 -07004428 if (flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP))
Jens Axboe2b188cc2019-01-07 10:46:33 -07004429 return -EINVAL;
4430
4431 f = fdget(fd);
4432 if (!f.file)
4433 return -EBADF;
4434
4435 ret = -EOPNOTSUPP;
4436 if (f.file->f_op != &io_uring_fops)
4437 goto out_fput;
4438
4439 ret = -ENXIO;
4440 ctx = f.file->private_data;
4441 if (!percpu_ref_tryget(&ctx->refs))
4442 goto out_fput;
4443
Jens Axboe6c271ce2019-01-10 11:22:30 -07004444 /*
4445 * For SQ polling, the thread will do all submissions and completions.
4446 * Just return the requested submit count, and wake the thread if
4447 * we were asked to.
4448 */
Jens Axboeb2a9ead2019-09-12 14:19:16 -06004449 ret = 0;
Jens Axboe6c271ce2019-01-10 11:22:30 -07004450 if (ctx->flags & IORING_SETUP_SQPOLL) {
Jens Axboec1edbf52019-11-10 16:56:04 -07004451 if (!list_empty_careful(&ctx->cq_overflow_list))
4452 io_cqring_overflow_flush(ctx, false);
Jens Axboe6c271ce2019-01-10 11:22:30 -07004453 if (flags & IORING_ENTER_SQ_WAKEUP)
4454 wake_up(&ctx->sqo_wait);
4455 submitted = to_submit;
Jens Axboeb2a9ead2019-09-12 14:19:16 -06004456 } else if (to_submit) {
Pavel Begunkovae9428c2019-11-06 00:22:14 +03004457 struct mm_struct *cur_mm;
Jens Axboe2b188cc2019-01-07 10:46:33 -07004458
Pavel Begunkovae9428c2019-11-06 00:22:14 +03004459 to_submit = min(to_submit, ctx->sq_entries);
Jens Axboe2b188cc2019-01-07 10:46:33 -07004460 mutex_lock(&ctx->uring_lock);
Pavel Begunkovae9428c2019-11-06 00:22:14 +03004461 /* already have mm, so io_submit_sqes() won't try to grab it */
4462 cur_mm = ctx->sqo_mm;
4463 submitted = io_submit_sqes(ctx, to_submit, f.file, fd,
4464 &cur_mm, false);
Jens Axboe2b188cc2019-01-07 10:46:33 -07004465 mutex_unlock(&ctx->uring_lock);
Jens Axboe2b188cc2019-01-07 10:46:33 -07004466 }
4467 if (flags & IORING_ENTER_GETEVENTS) {
Jens Axboedef596e2019-01-09 08:59:42 -07004468 unsigned nr_events = 0;
4469
Jens Axboe2b188cc2019-01-07 10:46:33 -07004470 min_complete = min(min_complete, ctx->cq_entries);
4471
Jens Axboedef596e2019-01-09 08:59:42 -07004472 if (ctx->flags & IORING_SETUP_IOPOLL) {
Jens Axboedef596e2019-01-09 08:59:42 -07004473 ret = io_iopoll_check(ctx, &nr_events, min_complete);
Jens Axboedef596e2019-01-09 08:59:42 -07004474 } else {
4475 ret = io_cqring_wait(ctx, min_complete, sig, sigsz);
4476 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07004477 }
4478
Pavel Begunkov6805b322019-10-08 02:18:42 +03004479 percpu_ref_put(&ctx->refs);
Jens Axboe2b188cc2019-01-07 10:46:33 -07004480out_fput:
4481 fdput(f);
4482 return submitted ? submitted : ret;
4483}
4484
4485static const struct file_operations io_uring_fops = {
4486 .release = io_uring_release,
Jens Axboefcb323c2019-10-24 12:39:47 -06004487 .flush = io_uring_flush,
Jens Axboe2b188cc2019-01-07 10:46:33 -07004488 .mmap = io_uring_mmap,
4489 .poll = io_uring_poll,
4490 .fasync = io_uring_fasync,
4491};
4492
4493static int io_allocate_scq_urings(struct io_ring_ctx *ctx,
4494 struct io_uring_params *p)
4495{
Hristo Venev75b28af2019-08-26 17:23:46 +00004496 struct io_rings *rings;
4497 size_t size, sq_array_offset;
Jens Axboe2b188cc2019-01-07 10:46:33 -07004498
Hristo Venev75b28af2019-08-26 17:23:46 +00004499 size = rings_size(p->sq_entries, p->cq_entries, &sq_array_offset);
4500 if (size == SIZE_MAX)
4501 return -EOVERFLOW;
4502
4503 rings = io_mem_alloc(size);
4504 if (!rings)
Jens Axboe2b188cc2019-01-07 10:46:33 -07004505 return -ENOMEM;
4506
Hristo Venev75b28af2019-08-26 17:23:46 +00004507 ctx->rings = rings;
4508 ctx->sq_array = (u32 *)((char *)rings + sq_array_offset);
4509 rings->sq_ring_mask = p->sq_entries - 1;
4510 rings->cq_ring_mask = p->cq_entries - 1;
4511 rings->sq_ring_entries = p->sq_entries;
4512 rings->cq_ring_entries = p->cq_entries;
4513 ctx->sq_mask = rings->sq_ring_mask;
4514 ctx->cq_mask = rings->cq_ring_mask;
4515 ctx->sq_entries = rings->sq_ring_entries;
4516 ctx->cq_entries = rings->cq_ring_entries;
Jens Axboe2b188cc2019-01-07 10:46:33 -07004517
4518 size = array_size(sizeof(struct io_uring_sqe), p->sq_entries);
4519 if (size == SIZE_MAX)
4520 return -EOVERFLOW;
4521
4522 ctx->sq_sqes = io_mem_alloc(size);
Mark Rutland52e04ef2019-04-30 17:30:21 +01004523 if (!ctx->sq_sqes)
Jens Axboe2b188cc2019-01-07 10:46:33 -07004524 return -ENOMEM;
Jens Axboe2b188cc2019-01-07 10:46:33 -07004525
Jens Axboe2b188cc2019-01-07 10:46:33 -07004526 return 0;
4527}
4528
4529/*
4530 * Allocate an anonymous fd, this is what constitutes the application
4531 * visible backing of an io_uring instance. The application mmaps this
4532 * fd to gain access to the SQ/CQ ring details. If UNIX sockets are enabled,
4533 * we have to tie this fd to a socket for file garbage collection purposes.
4534 */
4535static int io_uring_get_fd(struct io_ring_ctx *ctx)
4536{
4537 struct file *file;
4538 int ret;
4539
4540#if defined(CONFIG_UNIX)
4541 ret = sock_create_kern(&init_net, PF_UNIX, SOCK_RAW, IPPROTO_IP,
4542 &ctx->ring_sock);
4543 if (ret)
4544 return ret;
4545#endif
4546
4547 ret = get_unused_fd_flags(O_RDWR | O_CLOEXEC);
4548 if (ret < 0)
4549 goto err;
4550
4551 file = anon_inode_getfile("[io_uring]", &io_uring_fops, ctx,
4552 O_RDWR | O_CLOEXEC);
4553 if (IS_ERR(file)) {
4554 put_unused_fd(ret);
4555 ret = PTR_ERR(file);
4556 goto err;
4557 }
4558
4559#if defined(CONFIG_UNIX)
4560 ctx->ring_sock->file = file;
Jens Axboe6b063142019-01-10 22:13:58 -07004561 ctx->ring_sock->sk->sk_user_data = ctx;
Jens Axboe2b188cc2019-01-07 10:46:33 -07004562#endif
4563 fd_install(ret, file);
4564 return ret;
4565err:
4566#if defined(CONFIG_UNIX)
4567 sock_release(ctx->ring_sock);
4568 ctx->ring_sock = NULL;
4569#endif
4570 return ret;
4571}
4572
4573static int io_uring_create(unsigned entries, struct io_uring_params *p)
4574{
4575 struct user_struct *user = NULL;
4576 struct io_ring_ctx *ctx;
4577 bool account_mem;
4578 int ret;
4579
4580 if (!entries || entries > IORING_MAX_ENTRIES)
4581 return -EINVAL;
4582
4583 /*
4584 * Use twice as many entries for the CQ ring. It's possible for the
4585 * application to drive a higher depth than the size of the SQ ring,
4586 * since the sqes are only used at submission time. This allows for
Jens Axboe33a107f2019-10-04 12:10:03 -06004587 * some flexibility in overcommitting a bit. If the application has
4588 * set IORING_SETUP_CQSIZE, it will have passed in the desired number
4589 * of CQ ring entries manually.
Jens Axboe2b188cc2019-01-07 10:46:33 -07004590 */
4591 p->sq_entries = roundup_pow_of_two(entries);
Jens Axboe33a107f2019-10-04 12:10:03 -06004592 if (p->flags & IORING_SETUP_CQSIZE) {
4593 /*
4594 * If IORING_SETUP_CQSIZE is set, we do the same roundup
4595 * to a power-of-two, if it isn't already. We do NOT impose
4596 * any cq vs sq ring sizing.
4597 */
4598 if (p->cq_entries < p->sq_entries || p->cq_entries > IORING_MAX_CQ_ENTRIES)
4599 return -EINVAL;
4600 p->cq_entries = roundup_pow_of_two(p->cq_entries);
4601 } else {
4602 p->cq_entries = 2 * p->sq_entries;
4603 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07004604
4605 user = get_uid(current_user());
4606 account_mem = !capable(CAP_IPC_LOCK);
4607
4608 if (account_mem) {
4609 ret = io_account_mem(user,
4610 ring_pages(p->sq_entries, p->cq_entries));
4611 if (ret) {
4612 free_uid(user);
4613 return ret;
4614 }
4615 }
4616
4617 ctx = io_ring_ctx_alloc(p);
4618 if (!ctx) {
4619 if (account_mem)
4620 io_unaccount_mem(user, ring_pages(p->sq_entries,
4621 p->cq_entries));
4622 free_uid(user);
4623 return -ENOMEM;
4624 }
4625 ctx->compat = in_compat_syscall();
4626 ctx->account_mem = account_mem;
4627 ctx->user = user;
4628
4629 ret = io_allocate_scq_urings(ctx, p);
4630 if (ret)
4631 goto err;
4632
Jens Axboe6c271ce2019-01-10 11:22:30 -07004633 ret = io_sq_offload_start(ctx, p);
Jens Axboe2b188cc2019-01-07 10:46:33 -07004634 if (ret)
4635 goto err;
4636
Jens Axboe2b188cc2019-01-07 10:46:33 -07004637 memset(&p->sq_off, 0, sizeof(p->sq_off));
Hristo Venev75b28af2019-08-26 17:23:46 +00004638 p->sq_off.head = offsetof(struct io_rings, sq.head);
4639 p->sq_off.tail = offsetof(struct io_rings, sq.tail);
4640 p->sq_off.ring_mask = offsetof(struct io_rings, sq_ring_mask);
4641 p->sq_off.ring_entries = offsetof(struct io_rings, sq_ring_entries);
4642 p->sq_off.flags = offsetof(struct io_rings, sq_flags);
4643 p->sq_off.dropped = offsetof(struct io_rings, sq_dropped);
4644 p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings;
Jens Axboe2b188cc2019-01-07 10:46:33 -07004645
4646 memset(&p->cq_off, 0, sizeof(p->cq_off));
Hristo Venev75b28af2019-08-26 17:23:46 +00004647 p->cq_off.head = offsetof(struct io_rings, cq.head);
4648 p->cq_off.tail = offsetof(struct io_rings, cq.tail);
4649 p->cq_off.ring_mask = offsetof(struct io_rings, cq_ring_mask);
4650 p->cq_off.ring_entries = offsetof(struct io_rings, cq_ring_entries);
4651 p->cq_off.overflow = offsetof(struct io_rings, cq_overflow);
4652 p->cq_off.cqes = offsetof(struct io_rings, cqes);
Jens Axboeac90f242019-09-06 10:26:21 -06004653
Jens Axboe044c1ab2019-10-28 09:15:33 -06004654 /*
4655 * Install ring fd as the very last thing, so we don't risk someone
4656 * having closed it before we finish setup
4657 */
4658 ret = io_uring_get_fd(ctx);
4659 if (ret < 0)
4660 goto err;
4661
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07004662 p->features = IORING_FEAT_SINGLE_MMAP | IORING_FEAT_NODROP;
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +02004663 trace_io_uring_create(ret, ctx, p->sq_entries, p->cq_entries, p->flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07004664 return ret;
4665err:
4666 io_ring_ctx_wait_and_kill(ctx);
4667 return ret;
4668}
4669
4670/*
4671 * Sets up an aio uring context, and returns the fd. Applications asks for a
4672 * ring size, we return the actual sq/cq ring sizes (among other things) in the
4673 * params structure passed in.
4674 */
4675static long io_uring_setup(u32 entries, struct io_uring_params __user *params)
4676{
4677 struct io_uring_params p;
4678 long ret;
4679 int i;
4680
4681 if (copy_from_user(&p, params, sizeof(p)))
4682 return -EFAULT;
4683 for (i = 0; i < ARRAY_SIZE(p.resv); i++) {
4684 if (p.resv[i])
4685 return -EINVAL;
4686 }
4687
Jens Axboe6c271ce2019-01-10 11:22:30 -07004688 if (p.flags & ~(IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL |
Jens Axboe33a107f2019-10-04 12:10:03 -06004689 IORING_SETUP_SQ_AFF | IORING_SETUP_CQSIZE))
Jens Axboe2b188cc2019-01-07 10:46:33 -07004690 return -EINVAL;
4691
4692 ret = io_uring_create(entries, &p);
4693 if (ret < 0)
4694 return ret;
4695
4696 if (copy_to_user(params, &p, sizeof(p)))
4697 return -EFAULT;
4698
4699 return ret;
4700}
4701
4702SYSCALL_DEFINE2(io_uring_setup, u32, entries,
4703 struct io_uring_params __user *, params)
4704{
4705 return io_uring_setup(entries, params);
4706}
4707
Jens Axboeedafcce2019-01-09 09:16:05 -07004708static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,
4709 void __user *arg, unsigned nr_args)
Jens Axboeb19062a2019-04-15 10:49:38 -06004710 __releases(ctx->uring_lock)
4711 __acquires(ctx->uring_lock)
Jens Axboeedafcce2019-01-09 09:16:05 -07004712{
4713 int ret;
4714
Jens Axboe35fa71a2019-04-22 10:23:23 -06004715 /*
4716 * We're inside the ring mutex, if the ref is already dying, then
4717 * someone else killed the ctx or is already going through
4718 * io_uring_register().
4719 */
4720 if (percpu_ref_is_dying(&ctx->refs))
4721 return -ENXIO;
4722
Jens Axboeedafcce2019-01-09 09:16:05 -07004723 percpu_ref_kill(&ctx->refs);
Jens Axboeb19062a2019-04-15 10:49:38 -06004724
4725 /*
4726 * Drop uring mutex before waiting for references to exit. If another
4727 * thread is currently inside io_uring_enter() it might need to grab
4728 * the uring_lock to make progress. If we hold it here across the drain
4729 * wait, then we can deadlock. It's safe to drop the mutex here, since
4730 * no new references will come in after we've killed the percpu ref.
4731 */
4732 mutex_unlock(&ctx->uring_lock);
Jens Axboe206aefd2019-11-07 18:27:42 -07004733 wait_for_completion(&ctx->completions[0]);
Jens Axboeb19062a2019-04-15 10:49:38 -06004734 mutex_lock(&ctx->uring_lock);
Jens Axboeedafcce2019-01-09 09:16:05 -07004735
4736 switch (opcode) {
4737 case IORING_REGISTER_BUFFERS:
4738 ret = io_sqe_buffer_register(ctx, arg, nr_args);
4739 break;
4740 case IORING_UNREGISTER_BUFFERS:
4741 ret = -EINVAL;
4742 if (arg || nr_args)
4743 break;
4744 ret = io_sqe_buffer_unregister(ctx);
4745 break;
Jens Axboe6b063142019-01-10 22:13:58 -07004746 case IORING_REGISTER_FILES:
4747 ret = io_sqe_files_register(ctx, arg, nr_args);
4748 break;
4749 case IORING_UNREGISTER_FILES:
4750 ret = -EINVAL;
4751 if (arg || nr_args)
4752 break;
4753 ret = io_sqe_files_unregister(ctx);
4754 break;
Jens Axboec3a31e62019-10-03 13:59:56 -06004755 case IORING_REGISTER_FILES_UPDATE:
4756 ret = io_sqe_files_update(ctx, arg, nr_args);
4757 break;
Jens Axboe9b402842019-04-11 11:45:41 -06004758 case IORING_REGISTER_EVENTFD:
4759 ret = -EINVAL;
4760 if (nr_args != 1)
4761 break;
4762 ret = io_eventfd_register(ctx, arg);
4763 break;
4764 case IORING_UNREGISTER_EVENTFD:
4765 ret = -EINVAL;
4766 if (arg || nr_args)
4767 break;
4768 ret = io_eventfd_unregister(ctx);
4769 break;
Jens Axboeedafcce2019-01-09 09:16:05 -07004770 default:
4771 ret = -EINVAL;
4772 break;
4773 }
4774
4775 /* bring the ctx back to life */
Jens Axboe206aefd2019-11-07 18:27:42 -07004776 reinit_completion(&ctx->completions[0]);
Jens Axboeedafcce2019-01-09 09:16:05 -07004777 percpu_ref_reinit(&ctx->refs);
4778 return ret;
4779}
4780
4781SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode,
4782 void __user *, arg, unsigned int, nr_args)
4783{
4784 struct io_ring_ctx *ctx;
4785 long ret = -EBADF;
4786 struct fd f;
4787
4788 f = fdget(fd);
4789 if (!f.file)
4790 return -EBADF;
4791
4792 ret = -EOPNOTSUPP;
4793 if (f.file->f_op != &io_uring_fops)
4794 goto out_fput;
4795
4796 ctx = f.file->private_data;
4797
4798 mutex_lock(&ctx->uring_lock);
4799 ret = __io_uring_register(ctx, opcode, arg, nr_args);
4800 mutex_unlock(&ctx->uring_lock);
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +02004801 trace_io_uring_register(ctx, opcode, ctx->nr_user_files, ctx->nr_user_bufs,
4802 ctx->cq_ev_fd != NULL, ret);
Jens Axboeedafcce2019-01-09 09:16:05 -07004803out_fput:
4804 fdput(f);
4805 return ret;
4806}
4807
Jens Axboe2b188cc2019-01-07 10:46:33 -07004808static int __init io_uring_init(void)
4809{
4810 req_cachep = KMEM_CACHE(io_kiocb, SLAB_HWCACHE_ALIGN | SLAB_PANIC);
4811 return 0;
4812};
4813__initcall(io_uring_init);