blob: dc9301d31f12cde0df941cfd2f2a930668aa8332 [file] [log] [blame]
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00001/* Copyright (C) 2009 Red Hat, Inc.
2 * Copyright (C) 2006 Rusty Russell IBM Corporation
3 *
4 * Author: Michael S. Tsirkin <mst@redhat.com>
5 *
6 * Inspiration, some code, and most witty comments come from
Rob Landley61516582011-05-06 09:27:36 -07007 * Documentation/virtual/lguest/lguest.c, by Rusty Russell
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00008 *
9 * This work is licensed under the terms of the GNU GPL, version 2.
10 *
11 * Generic code for virtio server in host kernel.
12 */
13
14#include <linux/eventfd.h>
15#include <linux/vhost.h>
Asias He35596b22013-08-19 09:23:19 +080016#include <linux/uio.h>
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +000017#include <linux/mm.h>
Michael S. Tsirkin64e1c802010-10-06 15:34:45 +020018#include <linux/mmu_context.h>
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +000019#include <linux/miscdevice.h>
20#include <linux/mutex.h>
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +000021#include <linux/poll.h>
22#include <linux/file.h>
23#include <linux/highmem.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090024#include <linux/slab.h>
Igor Mammedov4de72552015-07-01 11:07:09 +020025#include <linux/vmalloc.h>
Tejun Heoc23f34452010-06-02 20:40:00 +020026#include <linux/kthread.h>
Michael S. Tsirkin9e3d1952010-07-27 22:56:50 +030027#include <linux/cgroup.h>
Asias He6ac1afb2013-05-06 16:38:21 +080028#include <linux/module.h>
Igor Mammedovbcfeaca2015-06-16 18:33:35 +020029#include <linux/sort.h>
Ingo Molnar6e84f312017-02-08 18:51:29 +010030#include <linux/sched/mm.h>
Ingo Molnar174cd4b2017-02-02 19:15:33 +010031#include <linux/sched/signal.h>
Jason Wanga9709d62016-06-23 02:04:31 -040032#include <linux/interval_tree_generic.h>
Jason Wangff002262018-10-30 14:10:49 +080033#include <linux/nospec.h>
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +000034
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +000035#include "vhost.h"
36
Igor Mammedovc9ce42f2015-07-02 15:08:11 +020037static ushort max_mem_regions = 64;
38module_param(max_mem_regions, ushort, 0444);
39MODULE_PARM_DESC(max_mem_regions,
40 "Maximum number of memory regions in memory map. (default: 64)");
Jason Wang6b1e6cc2016-06-23 02:04:32 -040041static int max_iotlb_entries = 2048;
42module_param(max_iotlb_entries, int, 0444);
43MODULE_PARM_DESC(max_iotlb_entries,
44 "Maximum number of iotlb entries. (default: 2048)");
Igor Mammedovc9ce42f2015-07-02 15:08:11 +020045
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +000046enum {
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +000047 VHOST_MEMORY_F_LOG = 0x1,
48};
49
Michael S. Tsirkin3b1bbe82014-10-24 14:04:47 +030050#define vhost_used_event(vq) ((__virtio16 __user *)&vq->avail->ring[vq->num])
51#define vhost_avail_event(vq) ((__virtio16 __user *)&vq->used->ring[vq->num])
Michael S. Tsirkin8ea8cf82011-05-20 02:10:54 +030052
Jason Wanga9709d62016-06-23 02:04:31 -040053INTERVAL_TREE_DEFINE(struct vhost_umem_node,
54 rb, __u64, __subtree_last,
Michael S. Tsirkin2f952c02016-12-06 05:57:54 +020055 START, LAST, static inline, vhost_umem_interval_tree);
Jason Wanga9709d62016-06-23 02:04:31 -040056
Greg Kurz2751c982015-04-24 14:27:24 +020057#ifdef CONFIG_VHOST_CROSS_ENDIAN_LEGACY
Greg Kurzc5072032016-02-16 15:59:34 +010058static void vhost_disable_cross_endian(struct vhost_virtqueue *vq)
Greg Kurz2751c982015-04-24 14:27:24 +020059{
60 vq->user_be = !virtio_legacy_is_little_endian();
61}
62
Greg Kurzc5072032016-02-16 15:59:34 +010063static void vhost_enable_cross_endian_big(struct vhost_virtqueue *vq)
64{
65 vq->user_be = true;
66}
67
68static void vhost_enable_cross_endian_little(struct vhost_virtqueue *vq)
69{
70 vq->user_be = false;
71}
72
Greg Kurz2751c982015-04-24 14:27:24 +020073static long vhost_set_vring_endian(struct vhost_virtqueue *vq, int __user *argp)
74{
75 struct vhost_vring_state s;
76
77 if (vq->private_data)
78 return -EBUSY;
79
80 if (copy_from_user(&s, argp, sizeof(s)))
81 return -EFAULT;
82
83 if (s.num != VHOST_VRING_LITTLE_ENDIAN &&
84 s.num != VHOST_VRING_BIG_ENDIAN)
85 return -EINVAL;
86
Greg Kurzc5072032016-02-16 15:59:34 +010087 if (s.num == VHOST_VRING_BIG_ENDIAN)
88 vhost_enable_cross_endian_big(vq);
89 else
90 vhost_enable_cross_endian_little(vq);
Greg Kurz2751c982015-04-24 14:27:24 +020091
92 return 0;
93}
94
95static long vhost_get_vring_endian(struct vhost_virtqueue *vq, u32 idx,
96 int __user *argp)
97{
98 struct vhost_vring_state s = {
99 .index = idx,
100 .num = vq->user_be
101 };
102
103 if (copy_to_user(argp, &s, sizeof(s)))
104 return -EFAULT;
105
106 return 0;
107}
108
109static void vhost_init_is_le(struct vhost_virtqueue *vq)
110{
111 /* Note for legacy virtio: user_be is initialized at reset time
112 * according to the host endianness. If userspace does not set an
113 * explicit endianness, the default behavior is native endian, as
114 * expected by legacy virtio.
115 */
116 vq->is_le = vhost_has_feature(vq, VIRTIO_F_VERSION_1) || !vq->user_be;
117}
118#else
Greg Kurzc5072032016-02-16 15:59:34 +0100119static void vhost_disable_cross_endian(struct vhost_virtqueue *vq)
Greg Kurz2751c982015-04-24 14:27:24 +0200120{
121}
122
123static long vhost_set_vring_endian(struct vhost_virtqueue *vq, int __user *argp)
124{
125 return -ENOIOCTLCMD;
126}
127
128static long vhost_get_vring_endian(struct vhost_virtqueue *vq, u32 idx,
129 int __user *argp)
130{
131 return -ENOIOCTLCMD;
132}
133
134static void vhost_init_is_le(struct vhost_virtqueue *vq)
135{
Halil Pasiccda8bba2017-01-30 11:09:36 +0100136 vq->is_le = vhost_has_feature(vq, VIRTIO_F_VERSION_1)
137 || virtio_legacy_is_little_endian();
Greg Kurz2751c982015-04-24 14:27:24 +0200138}
139#endif /* CONFIG_VHOST_CROSS_ENDIAN_LEGACY */
140
Greg Kurzc5072032016-02-16 15:59:34 +0100141static void vhost_reset_is_le(struct vhost_virtqueue *vq)
142{
Halil Pasiccda8bba2017-01-30 11:09:36 +0100143 vhost_init_is_le(vq);
Greg Kurzc5072032016-02-16 15:59:34 +0100144}
145
Jason Wang7235acd2016-04-25 22:14:32 -0400146struct vhost_flush_struct {
147 struct vhost_work work;
148 struct completion wait_event;
149};
150
151static void vhost_flush_work(struct vhost_work *work)
152{
153 struct vhost_flush_struct *s;
154
155 s = container_of(work, struct vhost_flush_struct, work);
156 complete(&s->wait_event);
157}
158
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000159static void vhost_poll_func(struct file *file, wait_queue_head_t *wqh,
160 poll_table *pt)
161{
162 struct vhost_poll *poll;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000163
Krishna Kumard47effe2011-03-01 17:06:37 +0530164 poll = container_of(pt, struct vhost_poll, table);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000165 poll->wqh = wqh;
166 add_wait_queue(wqh, &poll->wait);
167}
168
Ingo Molnarac6424b2017-06-20 12:06:13 +0200169static int vhost_poll_wakeup(wait_queue_entry_t *wait, unsigned mode, int sync,
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000170 void *key)
171{
Tejun Heoc23f34452010-06-02 20:40:00 +0200172 struct vhost_poll *poll = container_of(wait, struct vhost_poll, wait);
173
Al Viro3ad6f932017-07-03 20:14:56 -0400174 if (!(key_to_poll(key) & poll->mask))
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000175 return 0;
176
Tejun Heoc23f34452010-06-02 20:40:00 +0200177 vhost_poll_queue(poll);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000178 return 0;
179}
180
Stefan Hajnoczi163049a2012-07-21 06:55:37 +0000181void vhost_work_init(struct vhost_work *work, vhost_work_fn_t fn)
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000182{
Jason Wang04b96e52016-04-25 22:14:33 -0400183 clear_bit(VHOST_WORK_QUEUED, &work->flags);
Tejun Heoc23f34452010-06-02 20:40:00 +0200184 work->fn = fn;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000185}
Asias He6ac1afb2013-05-06 16:38:21 +0800186EXPORT_SYMBOL_GPL(vhost_work_init);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000187
Michael S. Tsirkin87d6a412010-09-02 14:05:30 +0300188/* Init poll structure */
189void vhost_poll_init(struct vhost_poll *poll, vhost_work_fn_t fn,
Al Viro58e3b602017-07-03 23:50:40 -0400190 __poll_t mask, struct vhost_dev *dev)
Michael S. Tsirkin87d6a412010-09-02 14:05:30 +0300191{
192 init_waitqueue_func_entry(&poll->wait, vhost_poll_wakeup);
193 init_poll_funcptr(&poll->table, vhost_poll_func);
194 poll->mask = mask;
195 poll->dev = dev;
Jason Wang2b8b3282013-01-28 01:05:18 +0000196 poll->wqh = NULL;
Michael S. Tsirkin87d6a412010-09-02 14:05:30 +0300197
198 vhost_work_init(&poll->work, fn);
199}
Asias He6ac1afb2013-05-06 16:38:21 +0800200EXPORT_SYMBOL_GPL(vhost_poll_init);
Michael S. Tsirkin87d6a412010-09-02 14:05:30 +0300201
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000202/* Start polling a file. We add ourselves to file's wait queue. The caller must
203 * keep a reference to a file until after vhost_poll_stop is called. */
Jason Wang2b8b3282013-01-28 01:05:18 +0000204int vhost_poll_start(struct vhost_poll *poll, struct file *file)
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000205{
Al Viroe6c8adc2017-07-03 22:25:56 -0400206 __poll_t mask;
Jason Wang2b8b3282013-01-28 01:05:18 +0000207 int ret = 0;
Krishna Kumard47effe2011-03-01 17:06:37 +0530208
Jason Wang70181d512013-04-10 20:50:48 +0000209 if (poll->wqh)
210 return 0;
211
Christoph Hellwig9965ed172018-03-05 07:26:05 -0800212 mask = vfs_poll(file, &poll->table);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000213 if (mask)
Al Viro3ad6f932017-07-03 20:14:56 -0400214 vhost_poll_wakeup(&poll->wait, 0, 0, poll_to_key(mask));
Linus Torvaldsa9a08842018-02-11 14:34:03 -0800215 if (mask & EPOLLERR) {
Jason Wangdc6455a2018-03-27 20:50:52 +0800216 vhost_poll_stop(poll);
Jason Wang2b8b3282013-01-28 01:05:18 +0000217 ret = -EINVAL;
218 }
219
220 return ret;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000221}
Asias He6ac1afb2013-05-06 16:38:21 +0800222EXPORT_SYMBOL_GPL(vhost_poll_start);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000223
224/* Stop polling a file. After this function returns, it becomes safe to drop the
225 * file reference. You must also flush afterwards. */
226void vhost_poll_stop(struct vhost_poll *poll)
227{
Jason Wang2b8b3282013-01-28 01:05:18 +0000228 if (poll->wqh) {
229 remove_wait_queue(poll->wqh, &poll->wait);
230 poll->wqh = NULL;
231 }
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000232}
Asias He6ac1afb2013-05-06 16:38:21 +0800233EXPORT_SYMBOL_GPL(vhost_poll_stop);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000234
Asias He6ac1afb2013-05-06 16:38:21 +0800235void vhost_work_flush(struct vhost_dev *dev, struct vhost_work *work)
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000236{
Jason Wang7235acd2016-04-25 22:14:32 -0400237 struct vhost_flush_struct flush;
Tejun Heoc23f34452010-06-02 20:40:00 +0200238
Jason Wang7235acd2016-04-25 22:14:32 -0400239 if (dev->worker) {
240 init_completion(&flush.wait_event);
241 vhost_work_init(&flush.work, vhost_flush_work);
242
243 vhost_work_queue(dev, &flush.work);
244 wait_for_completion(&flush.wait_event);
245 }
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000246}
Asias He6ac1afb2013-05-06 16:38:21 +0800247EXPORT_SYMBOL_GPL(vhost_work_flush);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000248
Michael S. Tsirkin87d6a412010-09-02 14:05:30 +0300249/* Flush any work that has been scheduled. When calling this, don't hold any
250 * locks that are also used by the callback. */
251void vhost_poll_flush(struct vhost_poll *poll)
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000252{
Michael S. Tsirkin87d6a412010-09-02 14:05:30 +0300253 vhost_work_flush(poll->dev, &poll->work);
254}
Asias He6ac1afb2013-05-06 16:38:21 +0800255EXPORT_SYMBOL_GPL(vhost_poll_flush);
Michael S. Tsirkin87d6a412010-09-02 14:05:30 +0300256
Stefan Hajnoczi163049a2012-07-21 06:55:37 +0000257void vhost_work_queue(struct vhost_dev *dev, struct vhost_work *work)
Michael S. Tsirkin87d6a412010-09-02 14:05:30 +0300258{
Jason Wang04b96e52016-04-25 22:14:33 -0400259 if (!dev->worker)
260 return;
Tejun Heoc23f34452010-06-02 20:40:00 +0200261
Jason Wang04b96e52016-04-25 22:14:33 -0400262 if (!test_and_set_bit(VHOST_WORK_QUEUED, &work->flags)) {
263 /* We can only add the work to the list after we're
264 * sure it was not in the list.
Peng Tao635abf02016-12-07 17:52:19 +0800265 * test_and_set_bit() implies a memory barrier.
Jason Wang04b96e52016-04-25 22:14:33 -0400266 */
Jason Wang04b96e52016-04-25 22:14:33 -0400267 llist_add(&work->node, &dev->work_list);
Tejun Heoc23f34452010-06-02 20:40:00 +0200268 wake_up_process(dev->worker);
269 }
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000270}
Asias He6ac1afb2013-05-06 16:38:21 +0800271EXPORT_SYMBOL_GPL(vhost_work_queue);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000272
Jason Wang526d3e72016-03-04 06:24:51 -0500273/* A lockless hint for busy polling code to exit the loop */
274bool vhost_has_work(struct vhost_dev *dev)
275{
Jason Wang04b96e52016-04-25 22:14:33 -0400276 return !llist_empty(&dev->work_list);
Jason Wang526d3e72016-03-04 06:24:51 -0500277}
278EXPORT_SYMBOL_GPL(vhost_has_work);
279
Michael S. Tsirkin87d6a412010-09-02 14:05:30 +0300280void vhost_poll_queue(struct vhost_poll *poll)
281{
282 vhost_work_queue(poll->dev, &poll->work);
283}
Asias He6ac1afb2013-05-06 16:38:21 +0800284EXPORT_SYMBOL_GPL(vhost_poll_queue);
Michael S. Tsirkin87d6a412010-09-02 14:05:30 +0300285
Jason Wangf8894912017-02-28 17:56:02 +0800286static void __vhost_vq_meta_reset(struct vhost_virtqueue *vq)
287{
288 int j;
289
290 for (j = 0; j < VHOST_NUM_ADDRS; j++)
291 vq->meta_iotlb[j] = NULL;
292}
293
294static void vhost_vq_meta_reset(struct vhost_dev *d)
295{
296 int i;
297
Jason Wang86a07da2018-12-13 10:53:39 +0800298 for (i = 0; i < d->nvqs; ++i)
Jason Wangf8894912017-02-28 17:56:02 +0800299 __vhost_vq_meta_reset(d->vqs[i]);
300}
301
Jason Wang7f466032019-05-24 04:12:18 -0400302#if VHOST_ARCH_CAN_ACCEL_UACCESS
303static void vhost_map_unprefetch(struct vhost_map *map)
304{
305 kfree(map->pages);
306 map->pages = NULL;
307 map->npages = 0;
308 map->addr = NULL;
309}
310
311static void vhost_uninit_vq_maps(struct vhost_virtqueue *vq)
312{
313 struct vhost_map *map[VHOST_NUM_ADDRS];
314 int i;
315
316 spin_lock(&vq->mmu_lock);
317 for (i = 0; i < VHOST_NUM_ADDRS; i++) {
318 map[i] = rcu_dereference_protected(vq->maps[i],
319 lockdep_is_held(&vq->mmu_lock));
320 if (map[i])
321 rcu_assign_pointer(vq->maps[i], NULL);
322 }
323 spin_unlock(&vq->mmu_lock);
324
325 synchronize_rcu();
326
327 for (i = 0; i < VHOST_NUM_ADDRS; i++)
328 if (map[i])
329 vhost_map_unprefetch(map[i]);
330
331}
332
333static void vhost_reset_vq_maps(struct vhost_virtqueue *vq)
334{
335 int i;
336
337 vhost_uninit_vq_maps(vq);
338 for (i = 0; i < VHOST_NUM_ADDRS; i++)
339 vq->uaddrs[i].size = 0;
340}
341
342static bool vhost_map_range_overlap(struct vhost_uaddr *uaddr,
343 unsigned long start,
344 unsigned long end)
345{
346 if (unlikely(!uaddr->size))
347 return false;
348
349 return !(end < uaddr->uaddr || start > uaddr->uaddr - 1 + uaddr->size);
350}
351
352static void vhost_invalidate_vq_start(struct vhost_virtqueue *vq,
353 int index,
354 unsigned long start,
355 unsigned long end)
356{
357 struct vhost_uaddr *uaddr = &vq->uaddrs[index];
358 struct vhost_map *map;
359 int i;
360
361 if (!vhost_map_range_overlap(uaddr, start, end))
362 return;
363
364 spin_lock(&vq->mmu_lock);
365 ++vq->invalidate_count;
366
367 map = rcu_dereference_protected(vq->maps[index],
368 lockdep_is_held(&vq->mmu_lock));
369 if (map) {
370 if (uaddr->write) {
371 for (i = 0; i < map->npages; i++)
372 set_page_dirty(map->pages[i]);
373 }
374 rcu_assign_pointer(vq->maps[index], NULL);
375 }
376 spin_unlock(&vq->mmu_lock);
377
378 if (map) {
379 synchronize_rcu();
380 vhost_map_unprefetch(map);
381 }
382}
383
384static void vhost_invalidate_vq_end(struct vhost_virtqueue *vq,
385 int index,
386 unsigned long start,
387 unsigned long end)
388{
389 if (!vhost_map_range_overlap(&vq->uaddrs[index], start, end))
390 return;
391
392 spin_lock(&vq->mmu_lock);
393 --vq->invalidate_count;
394 spin_unlock(&vq->mmu_lock);
395}
396
397static int vhost_invalidate_range_start(struct mmu_notifier *mn,
398 const struct mmu_notifier_range *range)
399{
400 struct vhost_dev *dev = container_of(mn, struct vhost_dev,
401 mmu_notifier);
402 int i, j;
403
404 if (!mmu_notifier_range_blockable(range))
405 return -EAGAIN;
406
407 for (i = 0; i < dev->nvqs; i++) {
408 struct vhost_virtqueue *vq = dev->vqs[i];
409
410 for (j = 0; j < VHOST_NUM_ADDRS; j++)
411 vhost_invalidate_vq_start(vq, j,
412 range->start,
413 range->end);
414 }
415
416 return 0;
417}
418
419static void vhost_invalidate_range_end(struct mmu_notifier *mn,
420 const struct mmu_notifier_range *range)
421{
422 struct vhost_dev *dev = container_of(mn, struct vhost_dev,
423 mmu_notifier);
424 int i, j;
425
426 for (i = 0; i < dev->nvqs; i++) {
427 struct vhost_virtqueue *vq = dev->vqs[i];
428
429 for (j = 0; j < VHOST_NUM_ADDRS; j++)
430 vhost_invalidate_vq_end(vq, j,
431 range->start,
432 range->end);
433 }
434}
435
436static const struct mmu_notifier_ops vhost_mmu_notifier_ops = {
437 .invalidate_range_start = vhost_invalidate_range_start,
438 .invalidate_range_end = vhost_invalidate_range_end,
439};
440
441static void vhost_init_maps(struct vhost_dev *dev)
442{
443 struct vhost_virtqueue *vq;
444 int i, j;
445
446 dev->mmu_notifier.ops = &vhost_mmu_notifier_ops;
447
448 for (i = 0; i < dev->nvqs; ++i) {
449 vq = dev->vqs[i];
450 for (j = 0; j < VHOST_NUM_ADDRS; j++)
451 RCU_INIT_POINTER(vq->maps[j], NULL);
452 }
453}
454#endif
455
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000456static void vhost_vq_reset(struct vhost_dev *dev,
457 struct vhost_virtqueue *vq)
458{
459 vq->num = 1;
460 vq->desc = NULL;
461 vq->avail = NULL;
462 vq->used = NULL;
463 vq->last_avail_idx = 0;
464 vq->avail_idx = 0;
465 vq->last_used_idx = 0;
Michael S. Tsirkin8ea8cf82011-05-20 02:10:54 +0300466 vq->signalled_used = 0;
467 vq->signalled_used_valid = false;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000468 vq->used_flags = 0;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000469 vq->log_used = false;
470 vq->log_addr = -1ull;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000471 vq->private_data = NULL;
Michael S. Tsirkinea16c512014-06-05 15:20:23 +0300472 vq->acked_features = 0;
Jason Wang429711a2018-08-06 11:17:47 +0800473 vq->acked_backend_features = 0;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000474 vq->log_base = NULL;
475 vq->error_ctx = NULL;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000476 vq->kick = NULL;
477 vq->call_ctx = NULL;
Michael S. Tsirkin73a99f02010-02-23 11:23:45 +0200478 vq->log_ctx = NULL;
Greg Kurzc5072032016-02-16 15:59:34 +0100479 vhost_reset_is_le(vq);
480 vhost_disable_cross_endian(vq);
Jason Wang03088132016-03-04 06:24:53 -0500481 vq->busyloop_timeout = 0;
Jason Wanga9709d62016-06-23 02:04:31 -0400482 vq->umem = NULL;
Jason Wang6b1e6cc2016-06-23 02:04:32 -0400483 vq->iotlb = NULL;
Jason Wang7f466032019-05-24 04:12:18 -0400484 vq->invalidate_count = 0;
Jason Wangf8894912017-02-28 17:56:02 +0800485 __vhost_vq_meta_reset(vq);
Jason Wang7f466032019-05-24 04:12:18 -0400486#if VHOST_ARCH_CAN_ACCEL_UACCESS
487 vhost_reset_vq_maps(vq);
488#endif
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000489}
490
Tejun Heoc23f34452010-06-02 20:40:00 +0200491static int vhost_worker(void *data)
492{
493 struct vhost_dev *dev = data;
Jason Wang04b96e52016-04-25 22:14:33 -0400494 struct vhost_work *work, *work_next;
495 struct llist_node *node;
Jens Freimannd7ffde32012-06-26 00:59:58 +0000496 mm_segment_t oldfs = get_fs();
Tejun Heoc23f34452010-06-02 20:40:00 +0200497
Jens Freimannd7ffde32012-06-26 00:59:58 +0000498 set_fs(USER_DS);
Michael S. Tsirkin64e1c802010-10-06 15:34:45 +0200499 use_mm(dev->mm);
500
Tejun Heoc23f34452010-06-02 20:40:00 +0200501 for (;;) {
502 /* mb paired w/ kthread_stop */
503 set_current_state(TASK_INTERRUPTIBLE);
504
Tejun Heoc23f34452010-06-02 20:40:00 +0200505 if (kthread_should_stop()) {
Tejun Heoc23f34452010-06-02 20:40:00 +0200506 __set_current_state(TASK_RUNNING);
Michael S. Tsirkin64e1c802010-10-06 15:34:45 +0200507 break;
Tejun Heoc23f34452010-06-02 20:40:00 +0200508 }
Tejun Heoc23f34452010-06-02 20:40:00 +0200509
Jason Wang04b96e52016-04-25 22:14:33 -0400510 node = llist_del_all(&dev->work_list);
511 if (!node)
512 schedule();
513
514 node = llist_reverse_order(node);
515 /* make sure flag is seen after deletion */
516 smp_wmb();
517 llist_for_each_entry_safe(work, work_next, node, node) {
518 clear_bit(VHOST_WORK_QUEUED, &work->flags);
Tejun Heoc23f34452010-06-02 20:40:00 +0200519 __set_current_state(TASK_RUNNING);
520 work->fn(work);
Nadav Har'Eld550dda2012-02-27 15:07:29 +0200521 if (need_resched())
522 schedule();
Jason Wang04b96e52016-04-25 22:14:33 -0400523 }
Tejun Heoc23f34452010-06-02 20:40:00 +0200524 }
Michael S. Tsirkin64e1c802010-10-06 15:34:45 +0200525 unuse_mm(dev->mm);
Jens Freimannd7ffde32012-06-26 00:59:58 +0000526 set_fs(oldfs);
Michael S. Tsirkin64e1c802010-10-06 15:34:45 +0200527 return 0;
Tejun Heoc23f34452010-06-02 20:40:00 +0200528}
529
Michael S. Tsirkinbab632d2011-07-18 03:48:46 +0000530static void vhost_vq_free_iovecs(struct vhost_virtqueue *vq)
531{
532 kfree(vq->indirect);
533 vq->indirect = NULL;
534 kfree(vq->log);
535 vq->log = NULL;
536 kfree(vq->heads);
537 vq->heads = NULL;
Michael S. Tsirkinbab632d2011-07-18 03:48:46 +0000538}
539
Jason Wange0e9b402010-09-14 23:53:05 +0800540/* Helper to allocate iovec buffers for all vqs. */
541static long vhost_dev_alloc_iovecs(struct vhost_dev *dev)
542{
Asias He6d5e6aa2013-05-06 16:38:23 +0800543 struct vhost_virtqueue *vq;
Jason Wange0e9b402010-09-14 23:53:05 +0800544 int i;
Krishna Kumard47effe2011-03-01 17:06:37 +0530545
Jason Wange0e9b402010-09-14 23:53:05 +0800546 for (i = 0; i < dev->nvqs; ++i) {
Asias He6d5e6aa2013-05-06 16:38:23 +0800547 vq = dev->vqs[i];
Kees Cook6da2ec52018-06-12 13:55:00 -0700548 vq->indirect = kmalloc_array(UIO_MAXIOV,
549 sizeof(*vq->indirect),
550 GFP_KERNEL);
Jason Wangb46a0bf2019-01-28 15:05:05 +0800551 vq->log = kmalloc_array(dev->iov_limit, sizeof(*vq->log),
Kees Cook6da2ec52018-06-12 13:55:00 -0700552 GFP_KERNEL);
Jason Wangb46a0bf2019-01-28 15:05:05 +0800553 vq->heads = kmalloc_array(dev->iov_limit, sizeof(*vq->heads),
Kees Cook6da2ec52018-06-12 13:55:00 -0700554 GFP_KERNEL);
Asias He6d5e6aa2013-05-06 16:38:23 +0800555 if (!vq->indirect || !vq->log || !vq->heads)
Jason Wange0e9b402010-09-14 23:53:05 +0800556 goto err_nomem;
557 }
558 return 0;
Krishna Kumard47effe2011-03-01 17:06:37 +0530559
Jason Wange0e9b402010-09-14 23:53:05 +0800560err_nomem:
Michael S. Tsirkinbab632d2011-07-18 03:48:46 +0000561 for (; i >= 0; --i)
Asias He3ab2e422013-04-27 11:16:48 +0800562 vhost_vq_free_iovecs(dev->vqs[i]);
Jason Wange0e9b402010-09-14 23:53:05 +0800563 return -ENOMEM;
564}
565
566static void vhost_dev_free_iovecs(struct vhost_dev *dev)
567{
568 int i;
Krishna Kumard47effe2011-03-01 17:06:37 +0530569
Michael S. Tsirkinbab632d2011-07-18 03:48:46 +0000570 for (i = 0; i < dev->nvqs; ++i)
Asias He3ab2e422013-04-27 11:16:48 +0800571 vhost_vq_free_iovecs(dev->vqs[i]);
Jason Wange0e9b402010-09-14 23:53:05 +0800572}
573
Jason Wange82b9b02019-05-17 00:29:49 -0400574bool vhost_exceeds_weight(struct vhost_virtqueue *vq,
575 int pkts, int total_len)
576{
577 struct vhost_dev *dev = vq->dev;
578
579 if ((dev->byte_weight && total_len >= dev->byte_weight) ||
580 pkts >= dev->weight) {
581 vhost_poll_queue(&vq->poll);
582 return true;
583 }
584
585 return false;
586}
587EXPORT_SYMBOL_GPL(vhost_exceeds_weight);
588
Jason Wang4942e822019-05-24 04:12:16 -0400589static size_t vhost_get_avail_size(struct vhost_virtqueue *vq,
590 unsigned int num)
591{
592 size_t event __maybe_unused =
593 vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
594
595 return sizeof(*vq->avail) +
596 sizeof(*vq->avail->ring) * num + event;
597}
598
599static size_t vhost_get_used_size(struct vhost_virtqueue *vq,
600 unsigned int num)
601{
602 size_t event __maybe_unused =
603 vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
604
605 return sizeof(*vq->used) +
606 sizeof(*vq->used->ring) * num + event;
607}
608
609static size_t vhost_get_desc_size(struct vhost_virtqueue *vq,
610 unsigned int num)
611{
612 return sizeof(*vq->desc) * num;
613}
614
Zhi Yong Wu59566b6e2013-12-07 04:13:03 +0800615void vhost_dev_init(struct vhost_dev *dev,
Jason Wange82b9b02019-05-17 00:29:49 -0400616 struct vhost_virtqueue **vqs, int nvqs,
617 int iov_limit, int weight, int byte_weight)
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000618{
Asias He6d5e6aa2013-05-06 16:38:23 +0800619 struct vhost_virtqueue *vq;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000620 int i;
Tejun Heoc23f34452010-06-02 20:40:00 +0200621
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000622 dev->vqs = vqs;
623 dev->nvqs = nvqs;
624 mutex_init(&dev->mutex);
625 dev->log_ctx = NULL;
Jason Wanga9709d62016-06-23 02:04:31 -0400626 dev->umem = NULL;
Jason Wang6b1e6cc2016-06-23 02:04:32 -0400627 dev->iotlb = NULL;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000628 dev->mm = NULL;
Tejun Heoc23f34452010-06-02 20:40:00 +0200629 dev->worker = NULL;
Jason Wangb46a0bf2019-01-28 15:05:05 +0800630 dev->iov_limit = iov_limit;
Jason Wange82b9b02019-05-17 00:29:49 -0400631 dev->weight = weight;
632 dev->byte_weight = byte_weight;
Jason Wang04b96e52016-04-25 22:14:33 -0400633 init_llist_head(&dev->work_list);
Jason Wang6b1e6cc2016-06-23 02:04:32 -0400634 init_waitqueue_head(&dev->wait);
635 INIT_LIST_HEAD(&dev->read_list);
636 INIT_LIST_HEAD(&dev->pending_list);
637 spin_lock_init(&dev->iotlb_lock);
Jason Wang7f466032019-05-24 04:12:18 -0400638#if VHOST_ARCH_CAN_ACCEL_UACCESS
639 vhost_init_maps(dev);
640#endif
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000641
642 for (i = 0; i < dev->nvqs; ++i) {
Asias He6d5e6aa2013-05-06 16:38:23 +0800643 vq = dev->vqs[i];
644 vq->log = NULL;
645 vq->indirect = NULL;
646 vq->heads = NULL;
647 vq->dev = dev;
648 mutex_init(&vq->mutex);
Jason Wang7f466032019-05-24 04:12:18 -0400649 spin_lock_init(&vq->mmu_lock);
Asias He6d5e6aa2013-05-06 16:38:23 +0800650 vhost_vq_reset(dev, vq);
651 if (vq->handle_kick)
652 vhost_poll_init(&vq->poll, vq->handle_kick,
Linus Torvaldsa9a08842018-02-11 14:34:03 -0800653 EPOLLIN, dev);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000654 }
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000655}
Asias He6ac1afb2013-05-06 16:38:21 +0800656EXPORT_SYMBOL_GPL(vhost_dev_init);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000657
658/* Caller should have device mutex */
659long vhost_dev_check_owner(struct vhost_dev *dev)
660{
661 /* Are you the owner? If not, I don't think you mean to do that */
662 return dev->mm == current->mm ? 0 : -EPERM;
663}
Asias He6ac1afb2013-05-06 16:38:21 +0800664EXPORT_SYMBOL_GPL(vhost_dev_check_owner);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000665
Michael S. Tsirkin87d6a412010-09-02 14:05:30 +0300666struct vhost_attach_cgroups_struct {
Krishna Kumard47effe2011-03-01 17:06:37 +0530667 struct vhost_work work;
668 struct task_struct *owner;
669 int ret;
Michael S. Tsirkin87d6a412010-09-02 14:05:30 +0300670};
671
672static void vhost_attach_cgroups_work(struct vhost_work *work)
673{
Krishna Kumard47effe2011-03-01 17:06:37 +0530674 struct vhost_attach_cgroups_struct *s;
675
676 s = container_of(work, struct vhost_attach_cgroups_struct, work);
677 s->ret = cgroup_attach_task_all(s->owner, current);
Michael S. Tsirkin87d6a412010-09-02 14:05:30 +0300678}
679
680static int vhost_attach_cgroups(struct vhost_dev *dev)
681{
Krishna Kumard47effe2011-03-01 17:06:37 +0530682 struct vhost_attach_cgroups_struct attach;
683
684 attach.owner = current;
685 vhost_work_init(&attach.work, vhost_attach_cgroups_work);
686 vhost_work_queue(dev, &attach.work);
687 vhost_work_flush(dev, &attach.work);
688 return attach.ret;
Michael S. Tsirkin87d6a412010-09-02 14:05:30 +0300689}
690
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000691/* Caller should have device mutex */
Michael S. Tsirkin05c05352013-06-06 15:20:39 +0300692bool vhost_dev_has_owner(struct vhost_dev *dev)
693{
694 return dev->mm;
695}
Asias He6ac1afb2013-05-06 16:38:21 +0800696EXPORT_SYMBOL_GPL(vhost_dev_has_owner);
Michael S. Tsirkin05c05352013-06-06 15:20:39 +0300697
698/* Caller should have device mutex */
Asias He54db63c2013-05-06 11:15:59 +0800699long vhost_dev_set_owner(struct vhost_dev *dev)
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000700{
Tejun Heoc23f34452010-06-02 20:40:00 +0200701 struct task_struct *worker;
702 int err;
Krishna Kumard47effe2011-03-01 17:06:37 +0530703
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000704 /* Is there an owner already? */
Michael S. Tsirkin05c05352013-06-06 15:20:39 +0300705 if (vhost_dev_has_owner(dev)) {
Tejun Heoc23f34452010-06-02 20:40:00 +0200706 err = -EBUSY;
707 goto err_mm;
708 }
Krishna Kumard47effe2011-03-01 17:06:37 +0530709
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000710 /* No owner, become one */
711 dev->mm = get_task_mm(current);
Tejun Heoc23f34452010-06-02 20:40:00 +0200712 worker = kthread_create(vhost_worker, dev, "vhost-%d", current->pid);
713 if (IS_ERR(worker)) {
714 err = PTR_ERR(worker);
715 goto err_worker;
716 }
717
718 dev->worker = worker;
Michael S. Tsirkin87d6a412010-09-02 14:05:30 +0300719 wake_up_process(worker); /* avoid contributing to loadavg */
720
721 err = vhost_attach_cgroups(dev);
Michael S. Tsirkin9e3d1952010-07-27 22:56:50 +0300722 if (err)
723 goto err_cgroup;
Tejun Heoc23f34452010-06-02 20:40:00 +0200724
Jason Wange0e9b402010-09-14 23:53:05 +0800725 err = vhost_dev_alloc_iovecs(dev);
726 if (err)
727 goto err_cgroup;
728
Jason Wang7f466032019-05-24 04:12:18 -0400729#if VHOST_ARCH_CAN_ACCEL_UACCESS
730 err = mmu_notifier_register(&dev->mmu_notifier, dev->mm);
731 if (err)
732 goto err_mmu_notifier;
733#endif
734
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000735 return 0;
Jason Wang7f466032019-05-24 04:12:18 -0400736
737#if VHOST_ARCH_CAN_ACCEL_UACCESS
738err_mmu_notifier:
739 vhost_dev_free_iovecs(dev);
740#endif
Michael S. Tsirkin9e3d1952010-07-27 22:56:50 +0300741err_cgroup:
742 kthread_stop(worker);
Michael S. Tsirkin615cc222010-09-02 14:16:36 +0300743 dev->worker = NULL;
Tejun Heoc23f34452010-06-02 20:40:00 +0200744err_worker:
745 if (dev->mm)
746 mmput(dev->mm);
747 dev->mm = NULL;
748err_mm:
749 return err;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000750}
Asias He6ac1afb2013-05-06 16:38:21 +0800751EXPORT_SYMBOL_GPL(vhost_dev_set_owner);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000752
Jason Wanga9709d62016-06-23 02:04:31 -0400753struct vhost_umem *vhost_dev_reset_owner_prepare(void)
754{
Michal Hocko6c5ab652017-05-08 15:57:15 -0700755 return kvzalloc(sizeof(struct vhost_umem), GFP_KERNEL);
Michael S. Tsirkin150b9e52013-04-28 17:12:08 +0300756}
Asias He6ac1afb2013-05-06 16:38:21 +0800757EXPORT_SYMBOL_GPL(vhost_dev_reset_owner_prepare);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000758
Michael S. Tsirkin150b9e52013-04-28 17:12:08 +0300759/* Caller should have device mutex */
Jason Wanga9709d62016-06-23 02:04:31 -0400760void vhost_dev_reset_owner(struct vhost_dev *dev, struct vhost_umem *umem)
Michael S. Tsirkin150b9e52013-04-28 17:12:08 +0300761{
Michael S. Tsirkin47283be2014-06-05 15:20:27 +0300762 int i;
763
夷则(Caspar)f6f93f72017-12-25 00:08:58 +0800764 vhost_dev_cleanup(dev);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000765
Michael S. Tsirkin150b9e52013-04-28 17:12:08 +0300766 /* Restore memory to default empty mapping. */
Jason Wanga9709d62016-06-23 02:04:31 -0400767 INIT_LIST_HEAD(&umem->umem_list);
768 dev->umem = umem;
Michael S. Tsirkin47283be2014-06-05 15:20:27 +0300769 /* We don't need VQ locks below since vhost_dev_cleanup makes sure
770 * VQs aren't running.
771 */
772 for (i = 0; i < dev->nvqs; ++i)
Jason Wanga9709d62016-06-23 02:04:31 -0400773 dev->vqs[i]->umem = umem;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000774}
Asias He6ac1afb2013-05-06 16:38:21 +0800775EXPORT_SYMBOL_GPL(vhost_dev_reset_owner);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000776
Michael S. Tsirkinb2116162012-11-01 09:16:46 +0000777void vhost_dev_stop(struct vhost_dev *dev)
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000778{
779 int i;
Krishna Kumard47effe2011-03-01 17:06:37 +0530780
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000781 for (i = 0; i < dev->nvqs; ++i) {
Asias He3ab2e422013-04-27 11:16:48 +0800782 if (dev->vqs[i]->kick && dev->vqs[i]->handle_kick) {
783 vhost_poll_stop(&dev->vqs[i]->poll);
784 vhost_poll_flush(&dev->vqs[i]->poll);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000785 }
Michael S. Tsirkinb2116162012-11-01 09:16:46 +0000786 }
787}
Asias He6ac1afb2013-05-06 16:38:21 +0800788EXPORT_SYMBOL_GPL(vhost_dev_stop);
Michael S. Tsirkinbab632d2011-07-18 03:48:46 +0000789
Jason Wang6b1e6cc2016-06-23 02:04:32 -0400790static void vhost_umem_free(struct vhost_umem *umem,
791 struct vhost_umem_node *node)
792{
793 vhost_umem_interval_tree_remove(node, &umem->umem_tree);
794 list_del(&node->link);
795 kfree(node);
796 umem->numem--;
797}
798
Jason Wanga9709d62016-06-23 02:04:31 -0400799static void vhost_umem_clean(struct vhost_umem *umem)
800{
801 struct vhost_umem_node *node, *tmp;
802
803 if (!umem)
804 return;
805
Jason Wang6b1e6cc2016-06-23 02:04:32 -0400806 list_for_each_entry_safe(node, tmp, &umem->umem_list, link)
807 vhost_umem_free(umem, node);
808
Jason Wanga9709d62016-06-23 02:04:31 -0400809 kvfree(umem);
810}
811
Jason Wang6b1e6cc2016-06-23 02:04:32 -0400812static void vhost_clear_msg(struct vhost_dev *dev)
813{
814 struct vhost_msg_node *node, *n;
815
816 spin_lock(&dev->iotlb_lock);
817
818 list_for_each_entry_safe(node, n, &dev->read_list, node) {
819 list_del(&node->node);
820 kfree(node);
821 }
822
823 list_for_each_entry_safe(node, n, &dev->pending_list, node) {
824 list_del(&node->node);
825 kfree(node);
826 }
827
828 spin_unlock(&dev->iotlb_lock);
829}
830
Jason Wang7f466032019-05-24 04:12:18 -0400831#if VHOST_ARCH_CAN_ACCEL_UACCESS
832static void vhost_setup_uaddr(struct vhost_virtqueue *vq,
833 int index, unsigned long uaddr,
834 size_t size, bool write)
835{
836 struct vhost_uaddr *addr = &vq->uaddrs[index];
837
838 addr->uaddr = uaddr;
839 addr->size = size;
840 addr->write = write;
841}
842
843static void vhost_setup_vq_uaddr(struct vhost_virtqueue *vq)
844{
845 vhost_setup_uaddr(vq, VHOST_ADDR_DESC,
846 (unsigned long)vq->desc,
847 vhost_get_desc_size(vq, vq->num),
848 false);
849 vhost_setup_uaddr(vq, VHOST_ADDR_AVAIL,
850 (unsigned long)vq->avail,
851 vhost_get_avail_size(vq, vq->num),
852 false);
853 vhost_setup_uaddr(vq, VHOST_ADDR_USED,
854 (unsigned long)vq->used,
855 vhost_get_used_size(vq, vq->num),
856 true);
857}
858
859static int vhost_map_prefetch(struct vhost_virtqueue *vq,
860 int index)
861{
862 struct vhost_map *map;
863 struct vhost_uaddr *uaddr = &vq->uaddrs[index];
864 struct page **pages;
865 int npages = DIV_ROUND_UP(uaddr->size, PAGE_SIZE);
866 int npinned;
867 void *vaddr, *v;
868 int err;
869 int i;
870
871 spin_lock(&vq->mmu_lock);
872
873 err = -EFAULT;
874 if (vq->invalidate_count)
875 goto err;
876
877 err = -ENOMEM;
878 map = kmalloc(sizeof(*map), GFP_ATOMIC);
879 if (!map)
880 goto err;
881
882 pages = kmalloc_array(npages, sizeof(struct page *), GFP_ATOMIC);
883 if (!pages)
884 goto err_pages;
885
886 err = EFAULT;
887 npinned = __get_user_pages_fast(uaddr->uaddr, npages,
888 uaddr->write, pages);
889 if (npinned > 0)
890 release_pages(pages, npinned);
891 if (npinned != npages)
892 goto err_gup;
893
894 for (i = 0; i < npinned; i++)
895 if (PageHighMem(pages[i]))
896 goto err_gup;
897
898 vaddr = v = page_address(pages[0]);
899
900 /* For simplicity, fallback to userspace address if VA is not
901 * contigious.
902 */
903 for (i = 1; i < npinned; i++) {
904 v += PAGE_SIZE;
905 if (v != page_address(pages[i]))
906 goto err_gup;
907 }
908
909 map->addr = vaddr + (uaddr->uaddr & (PAGE_SIZE - 1));
910 map->npages = npages;
911 map->pages = pages;
912
913 rcu_assign_pointer(vq->maps[index], map);
914 /* No need for a synchronize_rcu(). This function should be
915 * called by dev->worker so we are serialized with all
916 * readers.
917 */
918 spin_unlock(&vq->mmu_lock);
919
920 return 0;
921
922err_gup:
923 kfree(pages);
924err_pages:
925 kfree(map);
926err:
927 spin_unlock(&vq->mmu_lock);
928 return err;
929}
930#endif
931
夷则(Caspar)f6f93f72017-12-25 00:08:58 +0800932void vhost_dev_cleanup(struct vhost_dev *dev)
Michael S. Tsirkinb2116162012-11-01 09:16:46 +0000933{
934 int i;
Michael S. Tsirkinbab632d2011-07-18 03:48:46 +0000935
Michael S. Tsirkinb2116162012-11-01 09:16:46 +0000936 for (i = 0; i < dev->nvqs; ++i) {
Asias He3ab2e422013-04-27 11:16:48 +0800937 if (dev->vqs[i]->error_ctx)
938 eventfd_ctx_put(dev->vqs[i]->error_ctx);
Asias He3ab2e422013-04-27 11:16:48 +0800939 if (dev->vqs[i]->kick)
940 fput(dev->vqs[i]->kick);
941 if (dev->vqs[i]->call_ctx)
942 eventfd_ctx_put(dev->vqs[i]->call_ctx);
Asias He3ab2e422013-04-27 11:16:48 +0800943 vhost_vq_reset(dev, dev->vqs[i]);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000944 }
Jason Wange0e9b402010-09-14 23:53:05 +0800945 vhost_dev_free_iovecs(dev);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000946 if (dev->log_ctx)
947 eventfd_ctx_put(dev->log_ctx);
948 dev->log_ctx = NULL;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000949 /* No one will access memory at this point */
Jason Wanga9709d62016-06-23 02:04:31 -0400950 vhost_umem_clean(dev->umem);
951 dev->umem = NULL;
Jason Wang6b1e6cc2016-06-23 02:04:32 -0400952 vhost_umem_clean(dev->iotlb);
953 dev->iotlb = NULL;
954 vhost_clear_msg(dev);
Linus Torvaldsa9a08842018-02-11 14:34:03 -0800955 wake_up_interruptible_poll(&dev->wait, EPOLLIN | EPOLLRDNORM);
Jason Wang04b96e52016-04-25 22:14:33 -0400956 WARN_ON(!llist_empty(&dev->work_list));
Eric Dumazet78b620c2010-08-31 02:05:57 +0000957 if (dev->worker) {
958 kthread_stop(dev->worker);
959 dev->worker = NULL;
960 }
Jason Wang7f466032019-05-24 04:12:18 -0400961 if (dev->mm) {
962#if VHOST_ARCH_CAN_ACCEL_UACCESS
963 mmu_notifier_unregister(&dev->mmu_notifier, dev->mm);
964#endif
Michael S. Tsirkin533a19b2010-10-06 15:34:38 +0200965 mmput(dev->mm);
Jason Wang7f466032019-05-24 04:12:18 -0400966 }
967#if VHOST_ARCH_CAN_ACCEL_UACCESS
968 for (i = 0; i < dev->nvqs; i++)
969 vhost_uninit_vq_maps(dev->vqs[i]);
970#endif
Michael S. Tsirkin533a19b2010-10-06 15:34:38 +0200971 dev->mm = NULL;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000972}
Asias He6ac1afb2013-05-06 16:38:21 +0800973EXPORT_SYMBOL_GPL(vhost_dev_cleanup);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000974
Stefan Hajnocziddd3d402018-04-11 10:35:41 +0800975static bool log_access_ok(void __user *log_base, u64 addr, unsigned long sz)
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000976{
977 u64 a = addr / VHOST_PAGE_SIZE / 8;
Krishna Kumard47effe2011-03-01 17:06:37 +0530978
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000979 /* Make sure 64 bit math will not overflow. */
980 if (a > ULONG_MAX - (unsigned long)log_base ||
981 a + (unsigned long)log_base > ULONG_MAX)
Stefan Hajnocziddd3d402018-04-11 10:35:41 +0800982 return false;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000983
Linus Torvalds96d4f262019-01-03 18:57:57 -0800984 return access_ok(log_base + a,
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000985 (sz + VHOST_PAGE_SIZE * 8 - 1) / VHOST_PAGE_SIZE / 8);
986}
987
Michael S. Tsirkinec33d032016-08-01 23:20:53 +0300988static bool vhost_overflow(u64 uaddr, u64 size)
989{
990 /* Make sure 64 bit math will not overflow. */
991 return uaddr > ULONG_MAX || size > ULONG_MAX || uaddr > ULONG_MAX - size;
992}
993
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000994/* Caller should have vq mutex and device mutex. */
Stefan Hajnocziddd3d402018-04-11 10:35:41 +0800995static bool vq_memory_access_ok(void __user *log_base, struct vhost_umem *umem,
996 int log_all)
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000997{
Jason Wanga9709d62016-06-23 02:04:31 -0400998 struct vhost_umem_node *node;
Jeff Dike179b2842010-04-07 09:59:10 -0400999
Jason Wanga9709d62016-06-23 02:04:31 -04001000 if (!umem)
Stefan Hajnocziddd3d402018-04-11 10:35:41 +08001001 return false;
Jeff Dike179b2842010-04-07 09:59:10 -04001002
Jason Wanga9709d62016-06-23 02:04:31 -04001003 list_for_each_entry(node, &umem->umem_list, link) {
1004 unsigned long a = node->userspace_addr;
1005
Michael S. Tsirkinec33d032016-08-01 23:20:53 +03001006 if (vhost_overflow(node->userspace_addr, node->size))
Stefan Hajnocziddd3d402018-04-11 10:35:41 +08001007 return false;
Michael S. Tsirkinec33d032016-08-01 23:20:53 +03001008
1009
Linus Torvalds96d4f262019-01-03 18:57:57 -08001010 if (!access_ok((void __user *)a,
Jason Wanga9709d62016-06-23 02:04:31 -04001011 node->size))
Stefan Hajnocziddd3d402018-04-11 10:35:41 +08001012 return false;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00001013 else if (log_all && !log_access_ok(log_base,
Jason Wanga9709d62016-06-23 02:04:31 -04001014 node->start,
1015 node->size))
Stefan Hajnocziddd3d402018-04-11 10:35:41 +08001016 return false;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00001017 }
Stefan Hajnocziddd3d402018-04-11 10:35:41 +08001018 return true;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00001019}
1020
Jason Wangf8894912017-02-28 17:56:02 +08001021static inline void __user *vhost_vq_meta_fetch(struct vhost_virtqueue *vq,
1022 u64 addr, unsigned int size,
1023 int type)
1024{
1025 const struct vhost_umem_node *node = vq->meta_iotlb[type];
1026
1027 if (!node)
1028 return NULL;
1029
1030 return (void *)(uintptr_t)(node->userspace_addr + addr - node->start);
1031}
1032
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00001033/* Can we switch to this memory table? */
1034/* Caller should have device mutex but not vq mutex */
Stefan Hajnocziddd3d402018-04-11 10:35:41 +08001035static bool memory_access_ok(struct vhost_dev *d, struct vhost_umem *umem,
1036 int log_all)
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00001037{
1038 int i;
Krishna Kumard47effe2011-03-01 17:06:37 +05301039
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00001040 for (i = 0; i < d->nvqs; ++i) {
Stefan Hajnocziddd3d402018-04-11 10:35:41 +08001041 bool ok;
Michael S. Tsirkinea16c512014-06-05 15:20:23 +03001042 bool log;
1043
Asias He3ab2e422013-04-27 11:16:48 +08001044 mutex_lock(&d->vqs[i]->mutex);
Michael S. Tsirkinea16c512014-06-05 15:20:23 +03001045 log = log_all || vhost_has_feature(d->vqs[i], VHOST_F_LOG_ALL);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00001046 /* If ring is inactive, will check when it's enabled. */
Asias He3ab2e422013-04-27 11:16:48 +08001047 if (d->vqs[i]->private_data)
Jason Wanga9709d62016-06-23 02:04:31 -04001048 ok = vq_memory_access_ok(d->vqs[i]->log_base,
1049 umem, log);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00001050 else
Stefan Hajnocziddd3d402018-04-11 10:35:41 +08001051 ok = true;
Asias He3ab2e422013-04-27 11:16:48 +08001052 mutex_unlock(&d->vqs[i]->mutex);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00001053 if (!ok)
Stefan Hajnocziddd3d402018-04-11 10:35:41 +08001054 return false;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00001055 }
Stefan Hajnocziddd3d402018-04-11 10:35:41 +08001056 return true;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00001057}
1058
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001059static int translate_desc(struct vhost_virtqueue *vq, u64 addr, u32 len,
1060 struct iovec iov[], int iov_size, int access);
Jason Wangbfe2bc52016-06-23 02:04:30 -04001061
Michael S. Tsirkin72952cc2016-12-06 06:01:41 +02001062static int vhost_copy_to_user(struct vhost_virtqueue *vq, void __user *to,
Jason Wangbfe2bc52016-06-23 02:04:30 -04001063 const void *from, unsigned size)
1064{
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001065 int ret;
Jason Wangbfe2bc52016-06-23 02:04:30 -04001066
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001067 if (!vq->iotlb)
1068 return __copy_to_user(to, from, size);
1069 else {
1070 /* This function should be called after iotlb
1071 * prefetch, which means we're sure that all vq
1072 * could be access through iotlb. So -EAGAIN should
1073 * not happen in this case.
1074 */
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001075 struct iov_iter t;
Jason Wangf8894912017-02-28 17:56:02 +08001076 void __user *uaddr = vhost_vq_meta_fetch(vq,
1077 (u64)(uintptr_t)to, size,
Eric Auger7ced6c92018-04-11 15:30:38 +02001078 VHOST_ADDR_USED);
Jason Wangf8894912017-02-28 17:56:02 +08001079
1080 if (uaddr)
1081 return __copy_to_user(uaddr, from, size);
1082
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001083 ret = translate_desc(vq, (u64)(uintptr_t)to, size, vq->iotlb_iov,
1084 ARRAY_SIZE(vq->iotlb_iov),
1085 VHOST_ACCESS_WO);
1086 if (ret < 0)
1087 goto out;
1088 iov_iter_init(&t, WRITE, vq->iotlb_iov, ret, size);
1089 ret = copy_to_iter(from, size, &t);
1090 if (ret == size)
1091 ret = 0;
1092 }
1093out:
1094 return ret;
1095}
Jason Wangbfe2bc52016-06-23 02:04:30 -04001096
1097static int vhost_copy_from_user(struct vhost_virtqueue *vq, void *to,
Michael S. Tsirkin72952cc2016-12-06 06:01:41 +02001098 void __user *from, unsigned size)
Jason Wangbfe2bc52016-06-23 02:04:30 -04001099{
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001100 int ret;
1101
1102 if (!vq->iotlb)
1103 return __copy_from_user(to, from, size);
1104 else {
1105 /* This function should be called after iotlb
1106 * prefetch, which means we're sure that vq
1107 * could be access through iotlb. So -EAGAIN should
1108 * not happen in this case.
1109 */
Jason Wangf8894912017-02-28 17:56:02 +08001110 void __user *uaddr = vhost_vq_meta_fetch(vq,
1111 (u64)(uintptr_t)from, size,
1112 VHOST_ADDR_DESC);
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001113 struct iov_iter f;
Jason Wangf8894912017-02-28 17:56:02 +08001114
1115 if (uaddr)
1116 return __copy_from_user(to, uaddr, size);
1117
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001118 ret = translate_desc(vq, (u64)(uintptr_t)from, size, vq->iotlb_iov,
1119 ARRAY_SIZE(vq->iotlb_iov),
1120 VHOST_ACCESS_RO);
1121 if (ret < 0) {
1122 vq_err(vq, "IOTLB translation failure: uaddr "
1123 "%p size 0x%llx\n", from,
1124 (unsigned long long) size);
1125 goto out;
1126 }
1127 iov_iter_init(&f, READ, vq->iotlb_iov, ret, size);
1128 ret = copy_from_iter(to, size, &f);
1129 if (ret == size)
1130 ret = 0;
1131 }
1132
1133out:
1134 return ret;
1135}
1136
Jason Wangf8894912017-02-28 17:56:02 +08001137static void __user *__vhost_get_user_slow(struct vhost_virtqueue *vq,
1138 void __user *addr, unsigned int size,
1139 int type)
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001140{
1141 int ret;
1142
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001143 ret = translate_desc(vq, (u64)(uintptr_t)addr, size, vq->iotlb_iov,
1144 ARRAY_SIZE(vq->iotlb_iov),
1145 VHOST_ACCESS_RO);
1146 if (ret < 0) {
1147 vq_err(vq, "IOTLB translation failure: uaddr "
1148 "%p size 0x%llx\n", addr,
1149 (unsigned long long) size);
1150 return NULL;
1151 }
1152
1153 if (ret != 1 || vq->iotlb_iov[0].iov_len != size) {
1154 vq_err(vq, "Non atomic userspace memory access: uaddr "
1155 "%p size 0x%llx\n", addr,
1156 (unsigned long long) size);
1157 return NULL;
1158 }
1159
1160 return vq->iotlb_iov[0].iov_base;
1161}
1162
Jason Wangf8894912017-02-28 17:56:02 +08001163/* This function should be called after iotlb
1164 * prefetch, which means we're sure that vq
1165 * could be access through iotlb. So -EAGAIN should
1166 * not happen in this case.
1167 */
1168static inline void __user *__vhost_get_user(struct vhost_virtqueue *vq,
1169 void *addr, unsigned int size,
1170 int type)
1171{
1172 void __user *uaddr = vhost_vq_meta_fetch(vq,
1173 (u64)(uintptr_t)addr, size, type);
1174 if (uaddr)
1175 return uaddr;
1176
1177 return __vhost_get_user_slow(vq, addr, size, type);
1178}
1179
1180#define vhost_put_user(vq, x, ptr) \
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001181({ \
1182 int ret = -EFAULT; \
1183 if (!vq->iotlb) { \
1184 ret = __put_user(x, ptr); \
1185 } else { \
1186 __typeof__(ptr) to = \
Jason Wangf8894912017-02-28 17:56:02 +08001187 (__typeof__(ptr)) __vhost_get_user(vq, ptr, \
1188 sizeof(*ptr), VHOST_ADDR_USED); \
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001189 if (to != NULL) \
1190 ret = __put_user(x, to); \
1191 else \
1192 ret = -EFAULT; \
1193 } \
1194 ret; \
1195})
1196
Jason Wang7b5d7532019-05-24 04:12:14 -04001197static inline int vhost_put_avail_event(struct vhost_virtqueue *vq)
1198{
Jason Wang7f466032019-05-24 04:12:18 -04001199#if VHOST_ARCH_CAN_ACCEL_UACCESS
1200 struct vhost_map *map;
1201 struct vring_used *used;
1202
1203 if (!vq->iotlb) {
1204 rcu_read_lock();
1205
1206 map = rcu_dereference(vq->maps[VHOST_ADDR_USED]);
1207 if (likely(map)) {
1208 used = map->addr;
1209 *((__virtio16 *)&used->ring[vq->num]) =
1210 cpu_to_vhost16(vq, vq->avail_idx);
1211 rcu_read_unlock();
1212 return 0;
1213 }
1214
1215 rcu_read_unlock();
1216 }
1217#endif
1218
Jason Wang7b5d7532019-05-24 04:12:14 -04001219 return vhost_put_user(vq, cpu_to_vhost16(vq, vq->avail_idx),
1220 vhost_avail_event(vq));
1221}
1222
1223static inline int vhost_put_used(struct vhost_virtqueue *vq,
1224 struct vring_used_elem *head, int idx,
1225 int count)
1226{
Jason Wang7f466032019-05-24 04:12:18 -04001227#if VHOST_ARCH_CAN_ACCEL_UACCESS
1228 struct vhost_map *map;
1229 struct vring_used *used;
1230 size_t size;
1231
1232 if (!vq->iotlb) {
1233 rcu_read_lock();
1234
1235 map = rcu_dereference(vq->maps[VHOST_ADDR_USED]);
1236 if (likely(map)) {
1237 used = map->addr;
1238 size = count * sizeof(*head);
1239 memcpy(used->ring + idx, head, size);
1240 rcu_read_unlock();
1241 return 0;
1242 }
1243
1244 rcu_read_unlock();
1245 }
1246#endif
1247
Jason Wang7b5d7532019-05-24 04:12:14 -04001248 return vhost_copy_to_user(vq, vq->used->ring + idx, head,
1249 count * sizeof(*head));
1250}
1251
1252static inline int vhost_put_used_flags(struct vhost_virtqueue *vq)
1253
1254{
Jason Wang7f466032019-05-24 04:12:18 -04001255#if VHOST_ARCH_CAN_ACCEL_UACCESS
1256 struct vhost_map *map;
1257 struct vring_used *used;
1258
1259 if (!vq->iotlb) {
1260 rcu_read_lock();
1261
1262 map = rcu_dereference(vq->maps[VHOST_ADDR_USED]);
1263 if (likely(map)) {
1264 used = map->addr;
1265 used->flags = cpu_to_vhost16(vq, vq->used_flags);
1266 rcu_read_unlock();
1267 return 0;
1268 }
1269
1270 rcu_read_unlock();
1271 }
1272#endif
1273
Jason Wang7b5d7532019-05-24 04:12:14 -04001274 return vhost_put_user(vq, cpu_to_vhost16(vq, vq->used_flags),
1275 &vq->used->flags);
1276}
1277
1278static inline int vhost_put_used_idx(struct vhost_virtqueue *vq)
1279
1280{
Jason Wang7f466032019-05-24 04:12:18 -04001281#if VHOST_ARCH_CAN_ACCEL_UACCESS
1282 struct vhost_map *map;
1283 struct vring_used *used;
1284
1285 if (!vq->iotlb) {
1286 rcu_read_lock();
1287
1288 map = rcu_dereference(vq->maps[VHOST_ADDR_USED]);
1289 if (likely(map)) {
1290 used = map->addr;
1291 used->idx = cpu_to_vhost16(vq, vq->last_used_idx);
1292 rcu_read_unlock();
1293 return 0;
1294 }
1295
1296 rcu_read_unlock();
1297 }
1298#endif
1299
Jason Wang7b5d7532019-05-24 04:12:14 -04001300 return vhost_put_user(vq, cpu_to_vhost16(vq, vq->last_used_idx),
1301 &vq->used->idx);
1302}
1303
Jason Wangf8894912017-02-28 17:56:02 +08001304#define vhost_get_user(vq, x, ptr, type) \
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001305({ \
1306 int ret; \
1307 if (!vq->iotlb) { \
1308 ret = __get_user(x, ptr); \
1309 } else { \
1310 __typeof__(ptr) from = \
Jason Wangf8894912017-02-28 17:56:02 +08001311 (__typeof__(ptr)) __vhost_get_user(vq, ptr, \
1312 sizeof(*ptr), \
1313 type); \
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001314 if (from != NULL) \
1315 ret = __get_user(x, from); \
1316 else \
1317 ret = -EFAULT; \
1318 } \
1319 ret; \
1320})
1321
Jason Wangf8894912017-02-28 17:56:02 +08001322#define vhost_get_avail(vq, x, ptr) \
1323 vhost_get_user(vq, x, ptr, VHOST_ADDR_AVAIL)
1324
1325#define vhost_get_used(vq, x, ptr) \
1326 vhost_get_user(vq, x, ptr, VHOST_ADDR_USED)
1327
Jason Wang86a07da2018-12-13 10:53:39 +08001328static void vhost_dev_lock_vqs(struct vhost_dev *d)
1329{
1330 int i = 0;
1331 for (i = 0; i < d->nvqs; ++i)
1332 mutex_lock_nested(&d->vqs[i]->mutex, i);
1333}
1334
1335static void vhost_dev_unlock_vqs(struct vhost_dev *d)
1336{
1337 int i = 0;
1338 for (i = 0; i < d->nvqs; ++i)
1339 mutex_unlock(&d->vqs[i]->mutex);
1340}
1341
Jason Wang7b5d7532019-05-24 04:12:14 -04001342static inline int vhost_get_avail_idx(struct vhost_virtqueue *vq,
1343 __virtio16 *idx)
1344{
Jason Wang7f466032019-05-24 04:12:18 -04001345#if VHOST_ARCH_CAN_ACCEL_UACCESS
1346 struct vhost_map *map;
1347 struct vring_avail *avail;
1348
1349 if (!vq->iotlb) {
1350 rcu_read_lock();
1351
1352 map = rcu_dereference(vq->maps[VHOST_ADDR_AVAIL]);
1353 if (likely(map)) {
1354 avail = map->addr;
1355 *idx = avail->idx;
1356 rcu_read_unlock();
1357 return 0;
1358 }
1359
1360 rcu_read_unlock();
1361 }
1362#endif
1363
Jason Wang7b5d7532019-05-24 04:12:14 -04001364 return vhost_get_avail(vq, *idx, &vq->avail->idx);
1365}
1366
1367static inline int vhost_get_avail_head(struct vhost_virtqueue *vq,
1368 __virtio16 *head, int idx)
1369{
Jason Wang7f466032019-05-24 04:12:18 -04001370#if VHOST_ARCH_CAN_ACCEL_UACCESS
1371 struct vhost_map *map;
1372 struct vring_avail *avail;
1373
1374 if (!vq->iotlb) {
1375 rcu_read_lock();
1376
1377 map = rcu_dereference(vq->maps[VHOST_ADDR_AVAIL]);
1378 if (likely(map)) {
1379 avail = map->addr;
1380 *head = avail->ring[idx & (vq->num - 1)];
1381 rcu_read_unlock();
1382 return 0;
1383 }
1384
1385 rcu_read_unlock();
1386 }
1387#endif
1388
Jason Wang7b5d7532019-05-24 04:12:14 -04001389 return vhost_get_avail(vq, *head,
1390 &vq->avail->ring[idx & (vq->num - 1)]);
1391}
1392
1393static inline int vhost_get_avail_flags(struct vhost_virtqueue *vq,
1394 __virtio16 *flags)
1395{
Jason Wang7f466032019-05-24 04:12:18 -04001396#if VHOST_ARCH_CAN_ACCEL_UACCESS
1397 struct vhost_map *map;
1398 struct vring_avail *avail;
1399
1400 if (!vq->iotlb) {
1401 rcu_read_lock();
1402
1403 map = rcu_dereference(vq->maps[VHOST_ADDR_AVAIL]);
1404 if (likely(map)) {
1405 avail = map->addr;
1406 *flags = avail->flags;
1407 rcu_read_unlock();
1408 return 0;
1409 }
1410
1411 rcu_read_unlock();
1412 }
1413#endif
1414
Jason Wang7b5d7532019-05-24 04:12:14 -04001415 return vhost_get_avail(vq, *flags, &vq->avail->flags);
1416}
1417
1418static inline int vhost_get_used_event(struct vhost_virtqueue *vq,
1419 __virtio16 *event)
1420{
Jason Wang7f466032019-05-24 04:12:18 -04001421#if VHOST_ARCH_CAN_ACCEL_UACCESS
1422 struct vhost_map *map;
1423 struct vring_avail *avail;
1424
1425 if (!vq->iotlb) {
1426 rcu_read_lock();
1427 map = rcu_dereference(vq->maps[VHOST_ADDR_AVAIL]);
1428 if (likely(map)) {
1429 avail = map->addr;
1430 *event = (__virtio16)avail->ring[vq->num];
1431 rcu_read_unlock();
1432 return 0;
1433 }
1434 rcu_read_unlock();
1435 }
1436#endif
1437
Jason Wang7b5d7532019-05-24 04:12:14 -04001438 return vhost_get_avail(vq, *event, vhost_used_event(vq));
1439}
1440
1441static inline int vhost_get_used_idx(struct vhost_virtqueue *vq,
1442 __virtio16 *idx)
1443{
Jason Wang7f466032019-05-24 04:12:18 -04001444#if VHOST_ARCH_CAN_ACCEL_UACCESS
1445 struct vhost_map *map;
1446 struct vring_used *used;
1447
1448 if (!vq->iotlb) {
1449 rcu_read_lock();
1450
1451 map = rcu_dereference(vq->maps[VHOST_ADDR_USED]);
1452 if (likely(map)) {
1453 used = map->addr;
1454 *idx = used->idx;
1455 rcu_read_unlock();
1456 return 0;
1457 }
1458
1459 rcu_read_unlock();
1460 }
1461#endif
1462
Jason Wang7b5d7532019-05-24 04:12:14 -04001463 return vhost_get_used(vq, *idx, &vq->used->idx);
1464}
1465
1466static inline int vhost_get_desc(struct vhost_virtqueue *vq,
1467 struct vring_desc *desc, int idx)
1468{
Jason Wang7f466032019-05-24 04:12:18 -04001469#if VHOST_ARCH_CAN_ACCEL_UACCESS
1470 struct vhost_map *map;
1471 struct vring_desc *d;
1472
1473 if (!vq->iotlb) {
1474 rcu_read_lock();
1475
1476 map = rcu_dereference(vq->maps[VHOST_ADDR_DESC]);
1477 if (likely(map)) {
1478 d = map->addr;
1479 *desc = *(d + idx);
1480 rcu_read_unlock();
1481 return 0;
1482 }
1483
1484 rcu_read_unlock();
1485 }
1486#endif
1487
Jason Wang7b5d7532019-05-24 04:12:14 -04001488 return vhost_copy_from_user(vq, desc, vq->desc + idx, sizeof(*desc));
1489}
1490
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001491static int vhost_new_umem_range(struct vhost_umem *umem,
1492 u64 start, u64 size, u64 end,
1493 u64 userspace_addr, int perm)
1494{
Jason Wang813dbeb2019-04-09 12:10:25 +08001495 struct vhost_umem_node *tmp, *node;
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001496
Jason Wang813dbeb2019-04-09 12:10:25 +08001497 if (!size)
1498 return -EFAULT;
1499
1500 node = kmalloc(sizeof(*node), GFP_ATOMIC);
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001501 if (!node)
1502 return -ENOMEM;
1503
1504 if (umem->numem == max_iotlb_entries) {
1505 tmp = list_first_entry(&umem->umem_list, typeof(*tmp), link);
1506 vhost_umem_free(umem, tmp);
1507 }
1508
1509 node->start = start;
1510 node->size = size;
1511 node->last = end;
1512 node->userspace_addr = userspace_addr;
1513 node->perm = perm;
1514 INIT_LIST_HEAD(&node->link);
1515 list_add_tail(&node->link, &umem->umem_list);
1516 vhost_umem_interval_tree_insert(node, &umem->umem_tree);
1517 umem->numem++;
1518
1519 return 0;
1520}
1521
1522static void vhost_del_umem_range(struct vhost_umem *umem,
1523 u64 start, u64 end)
1524{
1525 struct vhost_umem_node *node;
1526
1527 while ((node = vhost_umem_interval_tree_iter_first(&umem->umem_tree,
1528 start, end)))
1529 vhost_umem_free(umem, node);
1530}
1531
1532static void vhost_iotlb_notify_vq(struct vhost_dev *d,
1533 struct vhost_iotlb_msg *msg)
1534{
1535 struct vhost_msg_node *node, *n;
1536
1537 spin_lock(&d->iotlb_lock);
1538
1539 list_for_each_entry_safe(node, n, &d->pending_list, node) {
1540 struct vhost_iotlb_msg *vq_msg = &node->msg.iotlb;
1541 if (msg->iova <= vq_msg->iova &&
Jason Wang2d66f992018-08-24 16:53:13 +08001542 msg->iova + msg->size - 1 >= vq_msg->iova &&
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001543 vq_msg->type == VHOST_IOTLB_MISS) {
1544 vhost_poll_queue(&node->vq->poll);
1545 list_del(&node->node);
1546 kfree(node);
1547 }
1548 }
1549
1550 spin_unlock(&d->iotlb_lock);
1551}
1552
Stefan Hajnocziddd3d402018-04-11 10:35:41 +08001553static bool umem_access_ok(u64 uaddr, u64 size, int access)
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001554{
1555 unsigned long a = uaddr;
1556
Michael S. Tsirkinec33d032016-08-01 23:20:53 +03001557 /* Make sure 64 bit math will not overflow. */
1558 if (vhost_overflow(uaddr, size))
Stefan Hajnocziddd3d402018-04-11 10:35:41 +08001559 return false;
Michael S. Tsirkinec33d032016-08-01 23:20:53 +03001560
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001561 if ((access & VHOST_ACCESS_RO) &&
Linus Torvalds96d4f262019-01-03 18:57:57 -08001562 !access_ok((void __user *)a, size))
Stefan Hajnocziddd3d402018-04-11 10:35:41 +08001563 return false;
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001564 if ((access & VHOST_ACCESS_WO) &&
Linus Torvalds96d4f262019-01-03 18:57:57 -08001565 !access_ok((void __user *)a, size))
Stefan Hajnocziddd3d402018-04-11 10:35:41 +08001566 return false;
1567 return true;
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001568}
1569
Michael S. Tsirkin72952cc2016-12-06 06:01:41 +02001570static int vhost_process_iotlb_msg(struct vhost_dev *dev,
1571 struct vhost_iotlb_msg *msg)
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001572{
1573 int ret = 0;
1574
Jason Wang1b15ad62018-05-22 19:58:57 +08001575 mutex_lock(&dev->mutex);
Jason Wang86a07da2018-12-13 10:53:39 +08001576 vhost_dev_lock_vqs(dev);
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001577 switch (msg->type) {
1578 case VHOST_IOTLB_UPDATE:
1579 if (!dev->iotlb) {
1580 ret = -EFAULT;
1581 break;
1582 }
Stefan Hajnocziddd3d402018-04-11 10:35:41 +08001583 if (!umem_access_ok(msg->uaddr, msg->size, msg->perm)) {
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001584 ret = -EFAULT;
1585 break;
1586 }
Jason Wangf8894912017-02-28 17:56:02 +08001587 vhost_vq_meta_reset(dev);
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001588 if (vhost_new_umem_range(dev->iotlb, msg->iova, msg->size,
1589 msg->iova + msg->size - 1,
1590 msg->uaddr, msg->perm)) {
1591 ret = -ENOMEM;
1592 break;
1593 }
1594 vhost_iotlb_notify_vq(dev, msg);
1595 break;
1596 case VHOST_IOTLB_INVALIDATE:
Jason Wang6f3180a2018-01-23 17:27:26 +08001597 if (!dev->iotlb) {
1598 ret = -EFAULT;
1599 break;
1600 }
Jason Wangf8894912017-02-28 17:56:02 +08001601 vhost_vq_meta_reset(dev);
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001602 vhost_del_umem_range(dev->iotlb, msg->iova,
1603 msg->iova + msg->size - 1);
1604 break;
1605 default:
1606 ret = -EINVAL;
1607 break;
1608 }
1609
Jason Wang86a07da2018-12-13 10:53:39 +08001610 vhost_dev_unlock_vqs(dev);
Jason Wang1b15ad62018-05-22 19:58:57 +08001611 mutex_unlock(&dev->mutex);
1612
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001613 return ret;
1614}
1615ssize_t vhost_chr_write_iter(struct vhost_dev *dev,
1616 struct iov_iter *from)
1617{
Jason Wang429711a2018-08-06 11:17:47 +08001618 struct vhost_iotlb_msg msg;
1619 size_t offset;
1620 int type, ret;
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001621
Jason Wang429711a2018-08-06 11:17:47 +08001622 ret = copy_from_iter(&type, sizeof(type), from);
Pavel Tikhomirov74ad7412018-12-13 17:53:50 +03001623 if (ret != sizeof(type)) {
1624 ret = -EINVAL;
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001625 goto done;
Pavel Tikhomirov74ad7412018-12-13 17:53:50 +03001626 }
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001627
Jason Wang429711a2018-08-06 11:17:47 +08001628 switch (type) {
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001629 case VHOST_IOTLB_MSG:
Jason Wang429711a2018-08-06 11:17:47 +08001630 /* There maybe a hole after type for V1 message type,
1631 * so skip it here.
1632 */
1633 offset = offsetof(struct vhost_msg, iotlb) - sizeof(int);
1634 break;
1635 case VHOST_IOTLB_MSG_V2:
1636 offset = sizeof(__u32);
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001637 break;
1638 default:
1639 ret = -EINVAL;
Jason Wang429711a2018-08-06 11:17:47 +08001640 goto done;
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001641 }
1642
Jason Wang429711a2018-08-06 11:17:47 +08001643 iov_iter_advance(from, offset);
1644 ret = copy_from_iter(&msg, sizeof(msg), from);
Pavel Tikhomirov74ad7412018-12-13 17:53:50 +03001645 if (ret != sizeof(msg)) {
1646 ret = -EINVAL;
Jason Wang429711a2018-08-06 11:17:47 +08001647 goto done;
Pavel Tikhomirov74ad7412018-12-13 17:53:50 +03001648 }
Jason Wang429711a2018-08-06 11:17:47 +08001649 if (vhost_process_iotlb_msg(dev, &msg)) {
1650 ret = -EFAULT;
1651 goto done;
1652 }
1653
1654 ret = (type == VHOST_IOTLB_MSG) ? sizeof(struct vhost_msg) :
1655 sizeof(struct vhost_msg_v2);
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001656done:
1657 return ret;
1658}
1659EXPORT_SYMBOL(vhost_chr_write_iter);
1660
Al Viroafc9a422017-07-03 06:39:46 -04001661__poll_t vhost_chr_poll(struct file *file, struct vhost_dev *dev,
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001662 poll_table *wait)
1663{
Al Viroafc9a422017-07-03 06:39:46 -04001664 __poll_t mask = 0;
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001665
1666 poll_wait(file, &dev->wait, wait);
1667
1668 if (!list_empty(&dev->read_list))
Linus Torvaldsa9a08842018-02-11 14:34:03 -08001669 mask |= EPOLLIN | EPOLLRDNORM;
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001670
1671 return mask;
1672}
1673EXPORT_SYMBOL(vhost_chr_poll);
1674
1675ssize_t vhost_chr_read_iter(struct vhost_dev *dev, struct iov_iter *to,
1676 int noblock)
1677{
1678 DEFINE_WAIT(wait);
1679 struct vhost_msg_node *node;
1680 ssize_t ret = 0;
1681 unsigned size = sizeof(struct vhost_msg);
1682
1683 if (iov_iter_count(to) < size)
1684 return 0;
1685
1686 while (1) {
1687 if (!noblock)
1688 prepare_to_wait(&dev->wait, &wait,
1689 TASK_INTERRUPTIBLE);
1690
1691 node = vhost_dequeue_msg(dev, &dev->read_list);
1692 if (node)
1693 break;
1694 if (noblock) {
1695 ret = -EAGAIN;
1696 break;
1697 }
1698 if (signal_pending(current)) {
1699 ret = -ERESTARTSYS;
1700 break;
1701 }
1702 if (!dev->iotlb) {
1703 ret = -EBADFD;
1704 break;
1705 }
1706
1707 schedule();
1708 }
1709
1710 if (!noblock)
1711 finish_wait(&dev->wait, &wait);
1712
1713 if (node) {
Jason Wang429711a2018-08-06 11:17:47 +08001714 struct vhost_iotlb_msg *msg;
1715 void *start = &node->msg;
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001716
Jason Wang429711a2018-08-06 11:17:47 +08001717 switch (node->msg.type) {
1718 case VHOST_IOTLB_MSG:
1719 size = sizeof(node->msg);
1720 msg = &node->msg.iotlb;
1721 break;
1722 case VHOST_IOTLB_MSG_V2:
1723 size = sizeof(node->msg_v2);
1724 msg = &node->msg_v2.iotlb;
1725 break;
1726 default:
1727 BUG();
1728 break;
1729 }
1730
1731 ret = copy_to_iter(start, size, to);
1732 if (ret != size || msg->type != VHOST_IOTLB_MISS) {
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001733 kfree(node);
1734 return ret;
1735 }
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001736 vhost_enqueue_msg(dev, &dev->pending_list, node);
1737 }
1738
1739 return ret;
1740}
1741EXPORT_SYMBOL_GPL(vhost_chr_read_iter);
1742
1743static int vhost_iotlb_miss(struct vhost_virtqueue *vq, u64 iova, int access)
1744{
1745 struct vhost_dev *dev = vq->dev;
1746 struct vhost_msg_node *node;
1747 struct vhost_iotlb_msg *msg;
Jason Wang429711a2018-08-06 11:17:47 +08001748 bool v2 = vhost_backend_has_feature(vq, VHOST_BACKEND_F_IOTLB_MSG_V2);
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001749
Jason Wang429711a2018-08-06 11:17:47 +08001750 node = vhost_new_msg(vq, v2 ? VHOST_IOTLB_MSG_V2 : VHOST_IOTLB_MSG);
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001751 if (!node)
1752 return -ENOMEM;
1753
Jason Wang429711a2018-08-06 11:17:47 +08001754 if (v2) {
1755 node->msg_v2.type = VHOST_IOTLB_MSG_V2;
1756 msg = &node->msg_v2.iotlb;
1757 } else {
1758 msg = &node->msg.iotlb;
1759 }
1760
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001761 msg->type = VHOST_IOTLB_MISS;
1762 msg->iova = iova;
1763 msg->perm = access;
1764
1765 vhost_enqueue_msg(dev, &dev->read_list, node);
1766
1767 return 0;
Jason Wangbfe2bc52016-06-23 02:04:30 -04001768}
1769
Stefan Hajnocziddd3d402018-04-11 10:35:41 +08001770static bool vq_access_ok(struct vhost_virtqueue *vq, unsigned int num,
1771 struct vring_desc __user *desc,
1772 struct vring_avail __user *avail,
1773 struct vring_used __user *used)
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001774
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00001775{
Jason Wang4942e822019-05-24 04:12:16 -04001776 return access_ok(desc, vhost_get_desc_size(vq, num)) &&
1777 access_ok(avail, vhost_get_avail_size(vq, num)) &&
1778 access_ok(used, vhost_get_used_size(vq, num));
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00001779}
1780
Jason Wangf8894912017-02-28 17:56:02 +08001781static void vhost_vq_meta_update(struct vhost_virtqueue *vq,
1782 const struct vhost_umem_node *node,
1783 int type)
1784{
1785 int access = (type == VHOST_ADDR_USED) ?
1786 VHOST_ACCESS_WO : VHOST_ACCESS_RO;
1787
1788 if (likely(node->perm & access))
1789 vq->meta_iotlb[type] = node;
1790}
1791
Stefan Hajnocziddd3d402018-04-11 10:35:41 +08001792static bool iotlb_access_ok(struct vhost_virtqueue *vq,
1793 int access, u64 addr, u64 len, int type)
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001794{
1795 const struct vhost_umem_node *node;
1796 struct vhost_umem *umem = vq->iotlb;
Michael S. Tsirkinca2c5b32017-08-21 22:33:33 +03001797 u64 s = 0, size, orig_addr = addr, last = addr + len - 1;
Jason Wangf8894912017-02-28 17:56:02 +08001798
1799 if (vhost_vq_meta_fetch(vq, addr, len, type))
1800 return true;
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001801
1802 while (len > s) {
1803 node = vhost_umem_interval_tree_iter_first(&umem->umem_tree,
1804 addr,
Michael S. Tsirkinca2c5b32017-08-21 22:33:33 +03001805 last);
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001806 if (node == NULL || node->start > addr) {
1807 vhost_iotlb_miss(vq, addr, access);
1808 return false;
1809 } else if (!(node->perm & access)) {
1810 /* Report the possible access violation by
1811 * request another translation from userspace.
1812 */
1813 return false;
1814 }
1815
1816 size = node->size - addr + node->start;
Jason Wangf8894912017-02-28 17:56:02 +08001817
1818 if (orig_addr == addr && size >= len)
1819 vhost_vq_meta_update(vq, node, type);
1820
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001821 s += size;
1822 addr += size;
1823 }
1824
1825 return true;
1826}
1827
Jason Wang7f466032019-05-24 04:12:18 -04001828#if VHOST_ARCH_CAN_ACCEL_UACCESS
1829static void vhost_vq_map_prefetch(struct vhost_virtqueue *vq)
1830{
1831 struct vhost_map __rcu *map;
1832 int i;
1833
1834 for (i = 0; i < VHOST_NUM_ADDRS; i++) {
1835 rcu_read_lock();
1836 map = rcu_dereference(vq->maps[i]);
1837 rcu_read_unlock();
1838 if (unlikely(!map))
1839 vhost_map_prefetch(vq, i);
1840 }
1841}
1842#endif
1843
Jason Wang9b5e8302019-05-24 04:12:15 -04001844int vq_meta_prefetch(struct vhost_virtqueue *vq)
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001845{
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001846 unsigned int num = vq->num;
1847
Jason Wang7f466032019-05-24 04:12:18 -04001848 if (!vq->iotlb) {
1849#if VHOST_ARCH_CAN_ACCEL_UACCESS
1850 vhost_vq_map_prefetch(vq);
1851#endif
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001852 return 1;
Jason Wang7f466032019-05-24 04:12:18 -04001853 }
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001854
1855 return iotlb_access_ok(vq, VHOST_ACCESS_RO, (u64)(uintptr_t)vq->desc,
Jason Wang4942e822019-05-24 04:12:16 -04001856 vhost_get_desc_size(vq, num), VHOST_ADDR_DESC) &&
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001857 iotlb_access_ok(vq, VHOST_ACCESS_RO, (u64)(uintptr_t)vq->avail,
Jason Wang4942e822019-05-24 04:12:16 -04001858 vhost_get_avail_size(vq, num),
Jason Wangf8894912017-02-28 17:56:02 +08001859 VHOST_ADDR_AVAIL) &&
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001860 iotlb_access_ok(vq, VHOST_ACCESS_WO, (u64)(uintptr_t)vq->used,
Jason Wang4942e822019-05-24 04:12:16 -04001861 vhost_get_used_size(vq, num), VHOST_ADDR_USED);
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001862}
Jason Wang9b5e8302019-05-24 04:12:15 -04001863EXPORT_SYMBOL_GPL(vq_meta_prefetch);
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001864
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00001865/* Can we log writes? */
1866/* Caller should have device mutex but not vq mutex */
Stefan Hajnocziddd3d402018-04-11 10:35:41 +08001867bool vhost_log_access_ok(struct vhost_dev *dev)
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00001868{
Jason Wanga9709d62016-06-23 02:04:31 -04001869 return memory_access_ok(dev, dev->umem, 1);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00001870}
Asias He6ac1afb2013-05-06 16:38:21 +08001871EXPORT_SYMBOL_GPL(vhost_log_access_ok);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00001872
1873/* Verify access for write logging. */
1874/* Caller should have vq mutex and device mutex */
Stefan Hajnocziddd3d402018-04-11 10:35:41 +08001875static bool vq_log_access_ok(struct vhost_virtqueue *vq,
1876 void __user *log_base)
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00001877{
Jason Wanga9709d62016-06-23 02:04:31 -04001878 return vq_memory_access_ok(log_base, vq->umem,
Michael S. Tsirkinea16c512014-06-05 15:20:23 +03001879 vhost_has_feature(vq, VHOST_F_LOG_ALL)) &&
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00001880 (!vq->log_used || log_access_ok(log_base, vq->log_addr,
Jason Wang4942e822019-05-24 04:12:16 -04001881 vhost_get_used_size(vq, vq->num)));
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00001882}
1883
1884/* Can we start vq? */
1885/* Caller should have vq mutex and device mutex */
Stefan Hajnocziddd3d402018-04-11 10:35:41 +08001886bool vhost_vq_access_ok(struct vhost_virtqueue *vq)
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00001887{
Stefan Hajnoczid14d2b72018-04-11 10:35:40 +08001888 if (!vq_log_access_ok(vq, vq->log_base))
Stefan Hajnocziddd3d402018-04-11 10:35:41 +08001889 return false;
Jason Wangd65026c2018-03-29 16:00:04 +08001890
Stefan Hajnoczid14d2b72018-04-11 10:35:40 +08001891 /* Access validation occurs at prefetch time with IOTLB */
1892 if (vq->iotlb)
Stefan Hajnocziddd3d402018-04-11 10:35:41 +08001893 return true;
Jason Wangd65026c2018-03-29 16:00:04 +08001894
1895 return vq_access_ok(vq, vq->num, vq->desc, vq->avail, vq->used);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00001896}
Asias He6ac1afb2013-05-06 16:38:21 +08001897EXPORT_SYMBOL_GPL(vhost_vq_access_ok);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00001898
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001899static struct vhost_umem *vhost_umem_alloc(void)
1900{
Michal Hocko6c5ab652017-05-08 15:57:15 -07001901 struct vhost_umem *umem = kvzalloc(sizeof(*umem), GFP_KERNEL);
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001902
1903 if (!umem)
1904 return NULL;
1905
Davidlohr Buesof808c132017-09-08 16:15:08 -07001906 umem->umem_tree = RB_ROOT_CACHED;
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001907 umem->numem = 0;
1908 INIT_LIST_HEAD(&umem->umem_list);
1909
1910 return umem;
1911}
1912
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00001913static long vhost_set_memory(struct vhost_dev *d, struct vhost_memory __user *m)
1914{
Jason Wanga9709d62016-06-23 02:04:31 -04001915 struct vhost_memory mem, *newmem;
1916 struct vhost_memory_region *region;
Jason Wanga9709d62016-06-23 02:04:31 -04001917 struct vhost_umem *newumem, *oldumem;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00001918 unsigned long size = offsetof(struct vhost_memory, regions);
Michael S. Tsirkin98f9ca02014-05-28 17:07:02 +03001919 int i;
Krishna Kumard47effe2011-03-01 17:06:37 +05301920
Takuya Yoshikawa7ad9c9d2010-05-27 18:58:03 +09001921 if (copy_from_user(&mem, m, size))
1922 return -EFAULT;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00001923 if (mem.padding)
1924 return -EOPNOTSUPP;
Igor Mammedovc9ce42f2015-07-02 15:08:11 +02001925 if (mem.nregions > max_mem_regions)
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00001926 return -E2BIG;
Matthew Wilcoxb2303d72018-06-07 07:57:18 -07001927 newmem = kvzalloc(struct_size(newmem, regions, mem.nregions),
1928 GFP_KERNEL);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00001929 if (!newmem)
1930 return -ENOMEM;
1931
1932 memcpy(newmem, &mem, size);
Takuya Yoshikawa7ad9c9d2010-05-27 18:58:03 +09001933 if (copy_from_user(newmem->regions, m->regions,
1934 mem.nregions * sizeof *m->regions)) {
Igor Mammedovbcfeaca2015-06-16 18:33:35 +02001935 kvfree(newmem);
Takuya Yoshikawa7ad9c9d2010-05-27 18:58:03 +09001936 return -EFAULT;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00001937 }
1938
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001939 newumem = vhost_umem_alloc();
Jason Wanga9709d62016-06-23 02:04:31 -04001940 if (!newumem) {
Igor Mammedov4de72552015-07-01 11:07:09 +02001941 kvfree(newmem);
Jason Wanga9709d62016-06-23 02:04:31 -04001942 return -ENOMEM;
Takuya Yoshikawaa02c3782010-05-27 19:03:56 +09001943 }
Jason Wanga9709d62016-06-23 02:04:31 -04001944
Jason Wanga9709d62016-06-23 02:04:31 -04001945 for (region = newmem->regions;
1946 region < newmem->regions + mem.nregions;
1947 region++) {
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001948 if (vhost_new_umem_range(newumem,
1949 region->guest_phys_addr,
1950 region->memory_size,
1951 region->guest_phys_addr +
1952 region->memory_size - 1,
1953 region->userspace_addr,
1954 VHOST_ACCESS_RW))
Jason Wanga9709d62016-06-23 02:04:31 -04001955 goto err;
Jason Wanga9709d62016-06-23 02:04:31 -04001956 }
1957
1958 if (!memory_access_ok(d, newumem, 0))
1959 goto err;
1960
1961 oldumem = d->umem;
1962 d->umem = newumem;
Michael S. Tsirkin98f9ca02014-05-28 17:07:02 +03001963
Michael S. Tsirkin47283be2014-06-05 15:20:27 +03001964 /* All memory accesses are done under some VQ mutex. */
Michael S. Tsirkin98f9ca02014-05-28 17:07:02 +03001965 for (i = 0; i < d->nvqs; ++i) {
1966 mutex_lock(&d->vqs[i]->mutex);
Jason Wanga9709d62016-06-23 02:04:31 -04001967 d->vqs[i]->umem = newumem;
Michael S. Tsirkin98f9ca02014-05-28 17:07:02 +03001968 mutex_unlock(&d->vqs[i]->mutex);
1969 }
Jason Wanga9709d62016-06-23 02:04:31 -04001970
1971 kvfree(newmem);
1972 vhost_umem_clean(oldumem);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00001973 return 0;
Jason Wanga9709d62016-06-23 02:04:31 -04001974
1975err:
1976 vhost_umem_clean(newumem);
1977 kvfree(newmem);
1978 return -EFAULT;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00001979}
1980
Jason Wangfeebcae2019-05-24 04:12:17 -04001981static long vhost_vring_set_num(struct vhost_dev *d,
1982 struct vhost_virtqueue *vq,
1983 void __user *argp)
1984{
1985 struct vhost_vring_state s;
1986
1987 /* Resizing ring with an active backend?
1988 * You don't want to do that. */
1989 if (vq->private_data)
1990 return -EBUSY;
1991
1992 if (copy_from_user(&s, argp, sizeof s))
1993 return -EFAULT;
1994
1995 if (!s.num || s.num > 0xffff || (s.num & (s.num - 1)))
1996 return -EINVAL;
1997 vq->num = s.num;
1998
1999 return 0;
2000}
2001
2002static long vhost_vring_set_addr(struct vhost_dev *d,
2003 struct vhost_virtqueue *vq,
2004 void __user *argp)
2005{
2006 struct vhost_vring_addr a;
2007
2008 if (copy_from_user(&a, argp, sizeof a))
2009 return -EFAULT;
2010 if (a.flags & ~(0x1 << VHOST_VRING_F_LOG))
2011 return -EOPNOTSUPP;
2012
2013 /* For 32bit, verify that the top 32bits of the user
2014 data are set to zero. */
2015 if ((u64)(unsigned long)a.desc_user_addr != a.desc_user_addr ||
2016 (u64)(unsigned long)a.used_user_addr != a.used_user_addr ||
2017 (u64)(unsigned long)a.avail_user_addr != a.avail_user_addr)
2018 return -EFAULT;
2019
2020 /* Make sure it's safe to cast pointers to vring types. */
2021 BUILD_BUG_ON(__alignof__ *vq->avail > VRING_AVAIL_ALIGN_SIZE);
2022 BUILD_BUG_ON(__alignof__ *vq->used > VRING_USED_ALIGN_SIZE);
2023 if ((a.avail_user_addr & (VRING_AVAIL_ALIGN_SIZE - 1)) ||
2024 (a.used_user_addr & (VRING_USED_ALIGN_SIZE - 1)) ||
2025 (a.log_guest_addr & (VRING_USED_ALIGN_SIZE - 1)))
2026 return -EINVAL;
2027
2028 /* We only verify access here if backend is configured.
2029 * If it is not, we don't as size might not have been setup.
2030 * We will verify when backend is configured. */
2031 if (vq->private_data) {
2032 if (!vq_access_ok(vq, vq->num,
2033 (void __user *)(unsigned long)a.desc_user_addr,
2034 (void __user *)(unsigned long)a.avail_user_addr,
2035 (void __user *)(unsigned long)a.used_user_addr))
2036 return -EINVAL;
2037
2038 /* Also validate log access for used ring if enabled. */
2039 if ((a.flags & (0x1 << VHOST_VRING_F_LOG)) &&
2040 !log_access_ok(vq->log_base, a.log_guest_addr,
2041 sizeof *vq->used +
2042 vq->num * sizeof *vq->used->ring))
2043 return -EINVAL;
2044 }
2045
2046 vq->log_used = !!(a.flags & (0x1 << VHOST_VRING_F_LOG));
2047 vq->desc = (void __user *)(unsigned long)a.desc_user_addr;
2048 vq->avail = (void __user *)(unsigned long)a.avail_user_addr;
2049 vq->log_addr = a.log_guest_addr;
2050 vq->used = (void __user *)(unsigned long)a.used_user_addr;
2051
2052 return 0;
2053}
2054
2055static long vhost_vring_set_num_addr(struct vhost_dev *d,
2056 struct vhost_virtqueue *vq,
2057 unsigned int ioctl,
2058 void __user *argp)
2059{
2060 long r;
2061
2062 mutex_lock(&vq->mutex);
2063
Jason Wang7f466032019-05-24 04:12:18 -04002064#if VHOST_ARCH_CAN_ACCEL_UACCESS
2065 /* Unregister MMU notifer to allow invalidation callback
2066 * can access vq->uaddrs[] without holding a lock.
2067 */
2068 if (d->mm)
2069 mmu_notifier_unregister(&d->mmu_notifier, d->mm);
2070
2071 vhost_uninit_vq_maps(vq);
2072#endif
2073
Jason Wangfeebcae2019-05-24 04:12:17 -04002074 switch (ioctl) {
2075 case VHOST_SET_VRING_NUM:
2076 r = vhost_vring_set_num(d, vq, argp);
2077 break;
2078 case VHOST_SET_VRING_ADDR:
2079 r = vhost_vring_set_addr(d, vq, argp);
2080 break;
2081 default:
2082 BUG();
2083 }
2084
Jason Wang7f466032019-05-24 04:12:18 -04002085#if VHOST_ARCH_CAN_ACCEL_UACCESS
2086 vhost_setup_vq_uaddr(vq);
2087
2088 if (d->mm)
2089 mmu_notifier_register(&d->mmu_notifier, d->mm);
2090#endif
2091
Jason Wangfeebcae2019-05-24 04:12:17 -04002092 mutex_unlock(&vq->mutex);
2093
2094 return r;
2095}
Sonny Rao26b36602018-03-14 10:05:06 -07002096long vhost_vring_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *argp)
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002097{
Al Virocecb46f2012-08-27 14:21:39 -04002098 struct file *eventfp, *filep = NULL;
2099 bool pollstart = false, pollstop = false;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002100 struct eventfd_ctx *ctx = NULL;
2101 u32 __user *idxp = argp;
2102 struct vhost_virtqueue *vq;
2103 struct vhost_vring_state s;
2104 struct vhost_vring_file f;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002105 u32 idx;
2106 long r;
2107
2108 r = get_user(idx, idxp);
2109 if (r < 0)
2110 return r;
Krishna Kumar0f3d9a12010-05-25 11:10:36 +05302111 if (idx >= d->nvqs)
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002112 return -ENOBUFS;
2113
Jason Wangff002262018-10-30 14:10:49 +08002114 idx = array_index_nospec(idx, d->nvqs);
Asias He3ab2e422013-04-27 11:16:48 +08002115 vq = d->vqs[idx];
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002116
Jason Wangfeebcae2019-05-24 04:12:17 -04002117 if (ioctl == VHOST_SET_VRING_NUM ||
2118 ioctl == VHOST_SET_VRING_ADDR) {
2119 return vhost_vring_set_num_addr(d, vq, ioctl, argp);
2120 }
2121
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002122 mutex_lock(&vq->mutex);
2123
2124 switch (ioctl) {
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002125 case VHOST_SET_VRING_BASE:
2126 /* Moving base with an active backend?
2127 * You don't want to do that. */
2128 if (vq->private_data) {
2129 r = -EBUSY;
2130 break;
2131 }
Takuya Yoshikawa7ad9c9d2010-05-27 18:58:03 +09002132 if (copy_from_user(&s, argp, sizeof s)) {
2133 r = -EFAULT;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002134 break;
Takuya Yoshikawa7ad9c9d2010-05-27 18:58:03 +09002135 }
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002136 if (s.num > 0xffff) {
2137 r = -EINVAL;
2138 break;
2139 }
Jason Wang8d658432017-07-27 11:22:05 +08002140 vq->last_avail_idx = s.num;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002141 /* Forget the cached index value. */
2142 vq->avail_idx = vq->last_avail_idx;
2143 break;
2144 case VHOST_GET_VRING_BASE:
2145 s.index = idx;
2146 s.num = vq->last_avail_idx;
Takuya Yoshikawa7ad9c9d2010-05-27 18:58:03 +09002147 if (copy_to_user(argp, &s, sizeof s))
2148 r = -EFAULT;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002149 break;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002150 case VHOST_SET_VRING_KICK:
Takuya Yoshikawa7ad9c9d2010-05-27 18:58:03 +09002151 if (copy_from_user(&f, argp, sizeof f)) {
2152 r = -EFAULT;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002153 break;
Takuya Yoshikawa7ad9c9d2010-05-27 18:58:03 +09002154 }
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002155 eventfp = f.fd == -1 ? NULL : eventfd_fget(f.fd);
Michael S. Tsirkin535297a2010-03-17 16:06:11 +02002156 if (IS_ERR(eventfp)) {
2157 r = PTR_ERR(eventfp);
2158 break;
2159 }
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002160 if (eventfp != vq->kick) {
Al Virocecb46f2012-08-27 14:21:39 -04002161 pollstop = (filep = vq->kick) != NULL;
2162 pollstart = (vq->kick = eventfp) != NULL;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002163 } else
2164 filep = eventfp;
2165 break;
2166 case VHOST_SET_VRING_CALL:
Takuya Yoshikawa7ad9c9d2010-05-27 18:58:03 +09002167 if (copy_from_user(&f, argp, sizeof f)) {
2168 r = -EFAULT;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002169 break;
Takuya Yoshikawa7ad9c9d2010-05-27 18:58:03 +09002170 }
Eric Biggerse050c7d2018-01-06 14:52:19 -08002171 ctx = f.fd == -1 ? NULL : eventfd_ctx_fdget(f.fd);
2172 if (IS_ERR(ctx)) {
2173 r = PTR_ERR(ctx);
Michael S. Tsirkin535297a2010-03-17 16:06:11 +02002174 break;
2175 }
Eric Biggerse050c7d2018-01-06 14:52:19 -08002176 swap(ctx, vq->call_ctx);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002177 break;
2178 case VHOST_SET_VRING_ERR:
Takuya Yoshikawa7ad9c9d2010-05-27 18:58:03 +09002179 if (copy_from_user(&f, argp, sizeof f)) {
2180 r = -EFAULT;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002181 break;
Takuya Yoshikawa7ad9c9d2010-05-27 18:58:03 +09002182 }
Eric Biggers09f332a2018-01-06 14:52:20 -08002183 ctx = f.fd == -1 ? NULL : eventfd_ctx_fdget(f.fd);
2184 if (IS_ERR(ctx)) {
2185 r = PTR_ERR(ctx);
Michael S. Tsirkin535297a2010-03-17 16:06:11 +02002186 break;
2187 }
Eric Biggers09f332a2018-01-06 14:52:20 -08002188 swap(ctx, vq->error_ctx);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002189 break;
Greg Kurz2751c982015-04-24 14:27:24 +02002190 case VHOST_SET_VRING_ENDIAN:
2191 r = vhost_set_vring_endian(vq, argp);
2192 break;
2193 case VHOST_GET_VRING_ENDIAN:
2194 r = vhost_get_vring_endian(vq, idx, argp);
2195 break;
Jason Wang03088132016-03-04 06:24:53 -05002196 case VHOST_SET_VRING_BUSYLOOP_TIMEOUT:
2197 if (copy_from_user(&s, argp, sizeof(s))) {
2198 r = -EFAULT;
2199 break;
2200 }
2201 vq->busyloop_timeout = s.num;
2202 break;
2203 case VHOST_GET_VRING_BUSYLOOP_TIMEOUT:
2204 s.index = idx;
2205 s.num = vq->busyloop_timeout;
2206 if (copy_to_user(argp, &s, sizeof(s)))
2207 r = -EFAULT;
2208 break;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002209 default:
2210 r = -ENOIOCTLCMD;
2211 }
2212
2213 if (pollstop && vq->handle_kick)
2214 vhost_poll_stop(&vq->poll);
2215
Eric Biggerse050c7d2018-01-06 14:52:19 -08002216 if (!IS_ERR_OR_NULL(ctx))
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002217 eventfd_ctx_put(ctx);
2218 if (filep)
2219 fput(filep);
2220
2221 if (pollstart && vq->handle_kick)
Jason Wang2b8b3282013-01-28 01:05:18 +00002222 r = vhost_poll_start(&vq->poll, vq->kick);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002223
2224 mutex_unlock(&vq->mutex);
2225
2226 if (pollstop && vq->handle_kick)
2227 vhost_poll_flush(&vq->poll);
2228 return r;
2229}
Asias He6ac1afb2013-05-06 16:38:21 +08002230EXPORT_SYMBOL_GPL(vhost_vring_ioctl);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002231
Jason Wang6b1e6cc2016-06-23 02:04:32 -04002232int vhost_init_device_iotlb(struct vhost_dev *d, bool enabled)
2233{
2234 struct vhost_umem *niotlb, *oiotlb;
2235 int i;
2236
2237 niotlb = vhost_umem_alloc();
2238 if (!niotlb)
2239 return -ENOMEM;
2240
2241 oiotlb = d->iotlb;
2242 d->iotlb = niotlb;
2243
2244 for (i = 0; i < d->nvqs; ++i) {
Jason Wangb13f9c62018-08-08 11:43:04 +08002245 struct vhost_virtqueue *vq = d->vqs[i];
2246
2247 mutex_lock(&vq->mutex);
2248 vq->iotlb = niotlb;
2249 __vhost_vq_meta_reset(vq);
2250 mutex_unlock(&vq->mutex);
Jason Wang6b1e6cc2016-06-23 02:04:32 -04002251 }
2252
2253 vhost_umem_clean(oiotlb);
2254
2255 return 0;
2256}
2257EXPORT_SYMBOL_GPL(vhost_init_device_iotlb);
2258
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002259/* Caller must have device mutex */
Michael S. Tsirkin935cdee2012-12-06 14:03:34 +02002260long vhost_dev_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *argp)
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002261{
Eric Biggersd25cc432018-01-06 14:52:21 -08002262 struct eventfd_ctx *ctx;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002263 u64 p;
2264 long r;
2265 int i, fd;
2266
2267 /* If you are not the owner, you can become one */
2268 if (ioctl == VHOST_SET_OWNER) {
2269 r = vhost_dev_set_owner(d);
2270 goto done;
2271 }
2272
2273 /* You must be the owner to do anything else */
2274 r = vhost_dev_check_owner(d);
2275 if (r)
2276 goto done;
2277
2278 switch (ioctl) {
2279 case VHOST_SET_MEM_TABLE:
2280 r = vhost_set_memory(d, argp);
2281 break;
2282 case VHOST_SET_LOG_BASE:
Takuya Yoshikawa7ad9c9d2010-05-27 18:58:03 +09002283 if (copy_from_user(&p, argp, sizeof p)) {
2284 r = -EFAULT;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002285 break;
Takuya Yoshikawa7ad9c9d2010-05-27 18:58:03 +09002286 }
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002287 if ((u64)(unsigned long)p != p) {
2288 r = -EFAULT;
2289 break;
2290 }
2291 for (i = 0; i < d->nvqs; ++i) {
2292 struct vhost_virtqueue *vq;
2293 void __user *base = (void __user *)(unsigned long)p;
Asias He3ab2e422013-04-27 11:16:48 +08002294 vq = d->vqs[i];
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002295 mutex_lock(&vq->mutex);
2296 /* If ring is inactive, will check when it's enabled. */
Michael S. Tsirkinea16c512014-06-05 15:20:23 +03002297 if (vq->private_data && !vq_log_access_ok(vq, base))
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002298 r = -EFAULT;
2299 else
2300 vq->log_base = base;
2301 mutex_unlock(&vq->mutex);
2302 }
2303 break;
2304 case VHOST_SET_LOG_FD:
2305 r = get_user(fd, (int __user *)argp);
2306 if (r < 0)
2307 break;
Eric Biggersd25cc432018-01-06 14:52:21 -08002308 ctx = fd == -1 ? NULL : eventfd_ctx_fdget(fd);
2309 if (IS_ERR(ctx)) {
2310 r = PTR_ERR(ctx);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002311 break;
2312 }
Eric Biggersd25cc432018-01-06 14:52:21 -08002313 swap(ctx, d->log_ctx);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002314 for (i = 0; i < d->nvqs; ++i) {
Asias He3ab2e422013-04-27 11:16:48 +08002315 mutex_lock(&d->vqs[i]->mutex);
2316 d->vqs[i]->log_ctx = d->log_ctx;
2317 mutex_unlock(&d->vqs[i]->mutex);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002318 }
2319 if (ctx)
2320 eventfd_ctx_put(ctx);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002321 break;
2322 default:
Michael S. Tsirkin935cdee2012-12-06 14:03:34 +02002323 r = -ENOIOCTLCMD;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002324 break;
2325 }
2326done:
2327 return r;
2328}
Asias He6ac1afb2013-05-06 16:38:21 +08002329EXPORT_SYMBOL_GPL(vhost_dev_ioctl);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002330
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002331/* TODO: This is really inefficient. We need something like get_user()
2332 * (instruction directly accesses the data, with an exception table entry
2333 * returning -EFAULT). See Documentation/x86/exception-tables.txt.
2334 */
2335static int set_bit_to_user(int nr, void __user *addr)
2336{
2337 unsigned long log = (unsigned long)addr;
2338 struct page *page;
2339 void *base;
2340 int bit = nr + (log % PAGE_SIZE) * 8;
2341 int r;
Krishna Kumard47effe2011-03-01 17:06:37 +05302342
Ira Weiny73b01402019-05-13 17:17:11 -07002343 r = get_user_pages_fast(log, 1, FOLL_WRITE, &page);
Michael S. Tsirkind6db3f52010-02-23 11:25:23 +02002344 if (r < 0)
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002345 return r;
Michael S. Tsirkind6db3f52010-02-23 11:25:23 +02002346 BUG_ON(r != 1);
Cong Wangc6daa7f2011-11-25 23:14:26 +08002347 base = kmap_atomic(page);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002348 set_bit(bit, base);
Cong Wangc6daa7f2011-11-25 23:14:26 +08002349 kunmap_atomic(base);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002350 set_page_dirty_lock(page);
2351 put_page(page);
2352 return 0;
2353}
2354
2355static int log_write(void __user *log_base,
2356 u64 write_address, u64 write_length)
2357{
Michael S. Tsirkin28831ee2010-11-29 10:22:10 +02002358 u64 write_page = write_address / VHOST_PAGE_SIZE;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002359 int r;
Krishna Kumard47effe2011-03-01 17:06:37 +05302360
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002361 if (!write_length)
2362 return 0;
Michael S. Tsirkin3bf9be42010-11-29 10:19:07 +02002363 write_length += write_address % VHOST_PAGE_SIZE;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002364 for (;;) {
2365 u64 base = (u64)(unsigned long)log_base;
Michael S. Tsirkin28831ee2010-11-29 10:22:10 +02002366 u64 log = base + write_page / 8;
2367 int bit = write_page % 8;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002368 if ((u64)(unsigned long)log != log)
2369 return -EFAULT;
2370 r = set_bit_to_user(bit, (void __user *)(unsigned long)log);
2371 if (r < 0)
2372 return r;
2373 if (write_length <= VHOST_PAGE_SIZE)
2374 break;
2375 write_length -= VHOST_PAGE_SIZE;
Michael S. Tsirkin28831ee2010-11-29 10:22:10 +02002376 write_page += 1;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002377 }
2378 return r;
2379}
2380
Jason Wangcc5e7102019-01-16 16:54:42 +08002381static int log_write_hva(struct vhost_virtqueue *vq, u64 hva, u64 len)
2382{
2383 struct vhost_umem *umem = vq->umem;
2384 struct vhost_umem_node *u;
2385 u64 start, end, l, min;
2386 int r;
2387 bool hit = false;
2388
2389 while (len) {
2390 min = len;
2391 /* More than one GPAs can be mapped into a single HVA. So
2392 * iterate all possible umems here to be safe.
2393 */
2394 list_for_each_entry(u, &umem->umem_list, link) {
2395 if (u->userspace_addr > hva - 1 + len ||
2396 u->userspace_addr - 1 + u->size < hva)
2397 continue;
2398 start = max(u->userspace_addr, hva);
2399 end = min(u->userspace_addr - 1 + u->size,
2400 hva - 1 + len);
2401 l = end - start + 1;
2402 r = log_write(vq->log_base,
2403 u->start + start - u->userspace_addr,
2404 l);
2405 if (r < 0)
2406 return r;
2407 hit = true;
2408 min = min(l, min);
2409 }
2410
2411 if (!hit)
2412 return -EFAULT;
2413
2414 len -= min;
2415 hva += min;
2416 }
2417
2418 return 0;
2419}
2420
2421static int log_used(struct vhost_virtqueue *vq, u64 used_offset, u64 len)
2422{
2423 struct iovec iov[64];
2424 int i, ret;
2425
2426 if (!vq->iotlb)
2427 return log_write(vq->log_base, vq->log_addr + used_offset, len);
2428
2429 ret = translate_desc(vq, (uintptr_t)vq->used + used_offset,
2430 len, iov, 64, VHOST_ACCESS_WO);
Jason Wang816db762019-02-19 14:53:44 +08002431 if (ret < 0)
Jason Wangcc5e7102019-01-16 16:54:42 +08002432 return ret;
2433
2434 for (i = 0; i < ret; i++) {
2435 ret = log_write_hva(vq, (uintptr_t)iov[i].iov_base,
2436 iov[i].iov_len);
2437 if (ret)
2438 return ret;
2439 }
2440
2441 return 0;
2442}
2443
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002444int vhost_log_write(struct vhost_virtqueue *vq, struct vhost_log *log,
Jason Wangcc5e7102019-01-16 16:54:42 +08002445 unsigned int log_num, u64 len, struct iovec *iov, int count)
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002446{
2447 int i, r;
2448
2449 /* Make sure data written is seen before log. */
Michael S. Tsirkin56593382010-02-01 07:21:02 +00002450 smp_wmb();
Jason Wangcc5e7102019-01-16 16:54:42 +08002451
2452 if (vq->iotlb) {
2453 for (i = 0; i < count; i++) {
2454 r = log_write_hva(vq, (uintptr_t)iov[i].iov_base,
2455 iov[i].iov_len);
2456 if (r < 0)
2457 return r;
2458 }
2459 return 0;
2460 }
2461
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002462 for (i = 0; i < log_num; ++i) {
2463 u64 l = min(log[i].len, len);
2464 r = log_write(vq->log_base, log[i].addr, l);
2465 if (r < 0)
2466 return r;
2467 len -= l;
Michael S. Tsirkin5786aee2010-09-22 12:31:53 +02002468 if (!len) {
2469 if (vq->log_ctx)
2470 eventfd_signal(vq->log_ctx, 1);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002471 return 0;
Michael S. Tsirkin5786aee2010-09-22 12:31:53 +02002472 }
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002473 }
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002474 /* Length written exceeds what we have stored. This is a bug. */
2475 BUG();
2476 return 0;
2477}
Asias He6ac1afb2013-05-06 16:38:21 +08002478EXPORT_SYMBOL_GPL(vhost_log_write);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002479
Jason Wang2723fea2011-06-21 18:04:38 +08002480static int vhost_update_used_flags(struct vhost_virtqueue *vq)
2481{
2482 void __user *used;
Jason Wang7b5d7532019-05-24 04:12:14 -04002483 if (vhost_put_used_flags(vq))
Jason Wang2723fea2011-06-21 18:04:38 +08002484 return -EFAULT;
2485 if (unlikely(vq->log_used)) {
2486 /* Make sure the flag is seen before log. */
2487 smp_wmb();
2488 /* Log used flag write. */
2489 used = &vq->used->flags;
Jason Wangcc5e7102019-01-16 16:54:42 +08002490 log_used(vq, (used - (void __user *)vq->used),
2491 sizeof vq->used->flags);
Jason Wang2723fea2011-06-21 18:04:38 +08002492 if (vq->log_ctx)
2493 eventfd_signal(vq->log_ctx, 1);
2494 }
2495 return 0;
2496}
2497
2498static int vhost_update_avail_event(struct vhost_virtqueue *vq, u16 avail_event)
2499{
Jason Wang7b5d7532019-05-24 04:12:14 -04002500 if (vhost_put_avail_event(vq))
Jason Wang2723fea2011-06-21 18:04:38 +08002501 return -EFAULT;
2502 if (unlikely(vq->log_used)) {
2503 void __user *used;
2504 /* Make sure the event is seen before log. */
2505 smp_wmb();
2506 /* Log avail event write */
2507 used = vhost_avail_event(vq);
Jason Wangcc5e7102019-01-16 16:54:42 +08002508 log_used(vq, (used - (void __user *)vq->used),
2509 sizeof *vhost_avail_event(vq));
Jason Wang2723fea2011-06-21 18:04:38 +08002510 if (vq->log_ctx)
2511 eventfd_signal(vq->log_ctx, 1);
2512 }
2513 return 0;
2514}
2515
Greg Kurz80f7d032016-02-16 15:59:44 +01002516int vhost_vq_init_access(struct vhost_virtqueue *vq)
Jason Wang2723fea2011-06-21 18:04:38 +08002517{
Michael S. Tsirkin3b1bbe82014-10-24 14:04:47 +03002518 __virtio16 last_used_idx;
Jason Wang2723fea2011-06-21 18:04:38 +08002519 int r;
Greg Kurze1f33be2016-02-16 15:54:28 +01002520 bool is_le = vq->is_le;
2521
Halil Pasiccda8bba2017-01-30 11:09:36 +01002522 if (!vq->private_data)
Jason Wang2723fea2011-06-21 18:04:38 +08002523 return 0;
Greg Kurz2751c982015-04-24 14:27:24 +02002524
2525 vhost_init_is_le(vq);
Jason Wang2723fea2011-06-21 18:04:38 +08002526
2527 r = vhost_update_used_flags(vq);
2528 if (r)
Greg Kurze1f33be2016-02-16 15:54:28 +01002529 goto err;
Jason Wang2723fea2011-06-21 18:04:38 +08002530 vq->signalled_used_valid = false;
Jason Wang6b1e6cc2016-06-23 02:04:32 -04002531 if (!vq->iotlb &&
Linus Torvalds96d4f262019-01-03 18:57:57 -08002532 !access_ok(&vq->used->idx, sizeof vq->used->idx)) {
Greg Kurze1f33be2016-02-16 15:54:28 +01002533 r = -EFAULT;
2534 goto err;
2535 }
Jason Wang7b5d7532019-05-24 04:12:14 -04002536 r = vhost_get_used_idx(vq, &last_used_idx);
Jason Wang6b1e6cc2016-06-23 02:04:32 -04002537 if (r) {
2538 vq_err(vq, "Can't access used idx at %p\n",
2539 &vq->used->idx);
Greg Kurze1f33be2016-02-16 15:54:28 +01002540 goto err;
Jason Wang6b1e6cc2016-06-23 02:04:32 -04002541 }
Michael S. Tsirkin3b1bbe82014-10-24 14:04:47 +03002542 vq->last_used_idx = vhost16_to_cpu(vq, last_used_idx);
Michael S. Tsirkin64f7f052014-12-01 17:39:39 +02002543 return 0;
Jason Wang6b1e6cc2016-06-23 02:04:32 -04002544
Greg Kurze1f33be2016-02-16 15:54:28 +01002545err:
2546 vq->is_le = is_le;
2547 return r;
Jason Wang2723fea2011-06-21 18:04:38 +08002548}
Greg Kurz80f7d032016-02-16 15:59:44 +01002549EXPORT_SYMBOL_GPL(vhost_vq_init_access);
Jason Wang2723fea2011-06-21 18:04:38 +08002550
Michael S. Tsirkin47283be2014-06-05 15:20:27 +03002551static int translate_desc(struct vhost_virtqueue *vq, u64 addr, u32 len,
Jason Wang6b1e6cc2016-06-23 02:04:32 -04002552 struct iovec iov[], int iov_size, int access)
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002553{
Jason Wanga9709d62016-06-23 02:04:31 -04002554 const struct vhost_umem_node *node;
Jason Wang6b1e6cc2016-06-23 02:04:32 -04002555 struct vhost_dev *dev = vq->dev;
2556 struct vhost_umem *umem = dev->iotlb ? dev->iotlb : dev->umem;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002557 struct iovec *_iov;
2558 u64 s = 0;
2559 int ret = 0;
2560
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002561 while ((u64)len > s) {
2562 u64 size;
Michael S. Tsirkin7b3384f2010-07-01 18:40:12 +03002563 if (unlikely(ret >= iov_size)) {
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002564 ret = -ENOBUFS;
2565 break;
2566 }
Jason Wang6b1e6cc2016-06-23 02:04:32 -04002567
Jason Wanga9709d62016-06-23 02:04:31 -04002568 node = vhost_umem_interval_tree_iter_first(&umem->umem_tree,
2569 addr, addr + len - 1);
2570 if (node == NULL || node->start > addr) {
Jason Wang6b1e6cc2016-06-23 02:04:32 -04002571 if (umem != dev->iotlb) {
2572 ret = -EFAULT;
2573 break;
2574 }
2575 ret = -EAGAIN;
2576 break;
2577 } else if (!(node->perm & access)) {
2578 ret = -EPERM;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002579 break;
2580 }
Jason Wang6b1e6cc2016-06-23 02:04:32 -04002581
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002582 _iov = iov + ret;
Jason Wanga9709d62016-06-23 02:04:31 -04002583 size = node->size - addr + node->start;
Michael S. Tsirkinbd971202012-11-26 05:57:27 +00002584 _iov->iov_len = min((u64)len - s, size);
Christoph Hellwiga8d37822010-04-13 14:11:25 -04002585 _iov->iov_base = (void __user *)(unsigned long)
Jason Wanga9709d62016-06-23 02:04:31 -04002586 (node->userspace_addr + addr - node->start);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002587 s += size;
2588 addr += size;
2589 ++ret;
2590 }
2591
Jason Wang6b1e6cc2016-06-23 02:04:32 -04002592 if (ret == -EAGAIN)
2593 vhost_iotlb_miss(vq, addr, access);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002594 return ret;
2595}
2596
2597/* Each buffer in the virtqueues is actually a chain of descriptors. This
2598 * function returns the next descriptor in the chain,
2599 * or -1U if we're at the end. */
Michael S. Tsirkin3b1bbe82014-10-24 14:04:47 +03002600static unsigned next_desc(struct vhost_virtqueue *vq, struct vring_desc *desc)
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002601{
2602 unsigned int next;
2603
2604 /* If this descriptor says it doesn't chain, we're done. */
Michael S. Tsirkin3b1bbe82014-10-24 14:04:47 +03002605 if (!(desc->flags & cpu_to_vhost16(vq, VRING_DESC_F_NEXT)))
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002606 return -1U;
2607
2608 /* Check they're not leading us off end of descriptors. */
Paul E. McKenney3a5db0b2017-11-27 09:45:10 -08002609 next = vhost16_to_cpu(vq, READ_ONCE(desc->next));
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002610 return next;
2611}
2612
Michael S. Tsirkin47283be2014-06-05 15:20:27 +03002613static int get_indirect(struct vhost_virtqueue *vq,
Michael S. Tsirkin7b3384f2010-07-01 18:40:12 +03002614 struct iovec iov[], unsigned int iov_size,
2615 unsigned int *out_num, unsigned int *in_num,
2616 struct vhost_log *log, unsigned int *log_num,
2617 struct vring_desc *indirect)
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002618{
2619 struct vring_desc desc;
2620 unsigned int i = 0, count, found = 0;
Michael S. Tsirkin3b1bbe82014-10-24 14:04:47 +03002621 u32 len = vhost32_to_cpu(vq, indirect->len);
Al Viroaad9a1c2014-12-10 14:49:01 -05002622 struct iov_iter from;
Jason Wang6b1e6cc2016-06-23 02:04:32 -04002623 int ret, access;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002624
2625 /* Sanity check */
Michael S. Tsirkin3b1bbe82014-10-24 14:04:47 +03002626 if (unlikely(len % sizeof desc)) {
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002627 vq_err(vq, "Invalid length in indirect descriptor: "
2628 "len 0x%llx not multiple of 0x%zx\n",
Michael S. Tsirkin3b1bbe82014-10-24 14:04:47 +03002629 (unsigned long long)len,
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002630 sizeof desc);
2631 return -EINVAL;
2632 }
2633
Michael S. Tsirkin3b1bbe82014-10-24 14:04:47 +03002634 ret = translate_desc(vq, vhost64_to_cpu(vq, indirect->addr), len, vq->indirect,
Jason Wang6b1e6cc2016-06-23 02:04:32 -04002635 UIO_MAXIOV, VHOST_ACCESS_RO);
Michael S. Tsirkin7b3384f2010-07-01 18:40:12 +03002636 if (unlikely(ret < 0)) {
Jason Wang6b1e6cc2016-06-23 02:04:32 -04002637 if (ret != -EAGAIN)
2638 vq_err(vq, "Translation failure %d in indirect.\n", ret);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002639 return ret;
2640 }
Al Viroaad9a1c2014-12-10 14:49:01 -05002641 iov_iter_init(&from, READ, vq->indirect, ret, len);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002642
2643 /* We will use the result as an address to read from, so most
2644 * architectures only need a compiler barrier here. */
2645 read_barrier_depends();
2646
Michael S. Tsirkin3b1bbe82014-10-24 14:04:47 +03002647 count = len / sizeof desc;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002648 /* Buffers are chained via a 16 bit next field, so
2649 * we can have at most 2^16 of these. */
Michael S. Tsirkin7b3384f2010-07-01 18:40:12 +03002650 if (unlikely(count > USHRT_MAX + 1)) {
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002651 vq_err(vq, "Indirect buffer length too big: %d\n",
2652 indirect->len);
2653 return -E2BIG;
2654 }
2655
2656 do {
2657 unsigned iov_count = *in_num + *out_num;
Michael S. Tsirkin7b3384f2010-07-01 18:40:12 +03002658 if (unlikely(++found > count)) {
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002659 vq_err(vq, "Loop detected: last one at %u "
2660 "indirect size %u\n",
2661 i, count);
2662 return -EINVAL;
2663 }
Al Virocbbd26b2016-11-01 22:09:04 -04002664 if (unlikely(!copy_from_iter_full(&desc, sizeof(desc), &from))) {
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002665 vq_err(vq, "Failed indirect descriptor: idx %d, %zx\n",
Michael S. Tsirkin3b1bbe82014-10-24 14:04:47 +03002666 i, (size_t)vhost64_to_cpu(vq, indirect->addr) + i * sizeof desc);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002667 return -EINVAL;
2668 }
Michael S. Tsirkin3b1bbe82014-10-24 14:04:47 +03002669 if (unlikely(desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_INDIRECT))) {
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002670 vq_err(vq, "Nested indirect descriptor: idx %d, %zx\n",
Michael S. Tsirkin3b1bbe82014-10-24 14:04:47 +03002671 i, (size_t)vhost64_to_cpu(vq, indirect->addr) + i * sizeof desc);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002672 return -EINVAL;
2673 }
2674
Jason Wang6b1e6cc2016-06-23 02:04:32 -04002675 if (desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_WRITE))
2676 access = VHOST_ACCESS_WO;
2677 else
2678 access = VHOST_ACCESS_RO;
2679
Michael S. Tsirkin3b1bbe82014-10-24 14:04:47 +03002680 ret = translate_desc(vq, vhost64_to_cpu(vq, desc.addr),
2681 vhost32_to_cpu(vq, desc.len), iov + iov_count,
Jason Wang6b1e6cc2016-06-23 02:04:32 -04002682 iov_size - iov_count, access);
Michael S. Tsirkin7b3384f2010-07-01 18:40:12 +03002683 if (unlikely(ret < 0)) {
Jason Wang6b1e6cc2016-06-23 02:04:32 -04002684 if (ret != -EAGAIN)
2685 vq_err(vq, "Translation failure %d indirect idx %d\n",
2686 ret, i);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002687 return ret;
2688 }
2689 /* If this is an input descriptor, increment that count. */
Jason Wang6b1e6cc2016-06-23 02:04:32 -04002690 if (access == VHOST_ACCESS_WO) {
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002691 *in_num += ret;
2692 if (unlikely(log)) {
Michael S. Tsirkin3b1bbe82014-10-24 14:04:47 +03002693 log[*log_num].addr = vhost64_to_cpu(vq, desc.addr);
2694 log[*log_num].len = vhost32_to_cpu(vq, desc.len);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002695 ++*log_num;
2696 }
2697 } else {
2698 /* If it's an output descriptor, they're all supposed
2699 * to come before any input descriptors. */
Michael S. Tsirkin7b3384f2010-07-01 18:40:12 +03002700 if (unlikely(*in_num)) {
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002701 vq_err(vq, "Indirect descriptor "
2702 "has out after in: idx %d\n", i);
2703 return -EINVAL;
2704 }
2705 *out_num += ret;
2706 }
Michael S. Tsirkin3b1bbe82014-10-24 14:04:47 +03002707 } while ((i = next_desc(vq, &desc)) != -1);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002708 return 0;
2709}
2710
2711/* This looks in the virtqueue and for the first available buffer, and converts
2712 * it to an iovec for convenient access. Since descriptors consist of some
2713 * number of output then some number of input descriptors, it's actually two
2714 * iovecs, but we pack them into one and note how many of each there were.
2715 *
Michael S. Tsirkind5675bd2010-06-24 16:59:59 +03002716 * This function returns the descriptor number found, or vq->num (which is
2717 * never a valid descriptor number) if none was found. A negative code is
2718 * returned on error. */
Michael S. Tsirkin47283be2014-06-05 15:20:27 +03002719int vhost_get_vq_desc(struct vhost_virtqueue *vq,
Michael S. Tsirkind5675bd2010-06-24 16:59:59 +03002720 struct iovec iov[], unsigned int iov_size,
2721 unsigned int *out_num, unsigned int *in_num,
2722 struct vhost_log *log, unsigned int *log_num)
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002723{
2724 struct vring_desc desc;
2725 unsigned int i, head, found = 0;
2726 u16 last_avail_idx;
Michael S. Tsirkin3b1bbe82014-10-24 14:04:47 +03002727 __virtio16 avail_idx;
2728 __virtio16 ring_head;
Jason Wang6b1e6cc2016-06-23 02:04:32 -04002729 int ret, access;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002730
2731 /* Check it isn't doing very strange things with descriptor numbers. */
2732 last_avail_idx = vq->last_avail_idx;
Jason Wange3b56cd2017-02-07 15:49:50 +08002733
2734 if (vq->avail_idx == vq->last_avail_idx) {
Jason Wang7b5d7532019-05-24 04:12:14 -04002735 if (unlikely(vhost_get_avail_idx(vq, &avail_idx))) {
Jason Wange3b56cd2017-02-07 15:49:50 +08002736 vq_err(vq, "Failed to access avail idx at %p\n",
2737 &vq->avail->idx);
2738 return -EFAULT;
2739 }
2740 vq->avail_idx = vhost16_to_cpu(vq, avail_idx);
2741
2742 if (unlikely((u16)(vq->avail_idx - last_avail_idx) > vq->num)) {
2743 vq_err(vq, "Guest moved used index from %u to %u",
2744 last_avail_idx, vq->avail_idx);
2745 return -EFAULT;
2746 }
2747
2748 /* If there's nothing new since last we looked, return
2749 * invalid.
2750 */
2751 if (vq->avail_idx == last_avail_idx)
2752 return vq->num;
2753
2754 /* Only get avail ring entries after they have been
2755 * exposed by guest.
2756 */
2757 smp_rmb();
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002758 }
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002759
2760 /* Grab the next descriptor number they're advertising, and increment
2761 * the index we've seen. */
Jason Wang7b5d7532019-05-24 04:12:14 -04002762 if (unlikely(vhost_get_avail_head(vq, &ring_head, last_avail_idx))) {
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002763 vq_err(vq, "Failed to read head: idx %d address %p\n",
2764 last_avail_idx,
2765 &vq->avail->ring[last_avail_idx % vq->num]);
Michael S. Tsirkind5675bd2010-06-24 16:59:59 +03002766 return -EFAULT;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002767 }
2768
Michael S. Tsirkin3b1bbe82014-10-24 14:04:47 +03002769 head = vhost16_to_cpu(vq, ring_head);
2770
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002771 /* If their number is silly, that's an error. */
Michael S. Tsirkin7b3384f2010-07-01 18:40:12 +03002772 if (unlikely(head >= vq->num)) {
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002773 vq_err(vq, "Guest says index %u > %u is available",
2774 head, vq->num);
Michael S. Tsirkind5675bd2010-06-24 16:59:59 +03002775 return -EINVAL;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002776 }
2777
2778 /* When we start there are none of either input nor output. */
2779 *out_num = *in_num = 0;
2780 if (unlikely(log))
2781 *log_num = 0;
2782
2783 i = head;
2784 do {
2785 unsigned iov_count = *in_num + *out_num;
Michael S. Tsirkin7b3384f2010-07-01 18:40:12 +03002786 if (unlikely(i >= vq->num)) {
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002787 vq_err(vq, "Desc index is %u > %u, head = %u",
2788 i, vq->num, head);
Michael S. Tsirkind5675bd2010-06-24 16:59:59 +03002789 return -EINVAL;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002790 }
Michael S. Tsirkin7b3384f2010-07-01 18:40:12 +03002791 if (unlikely(++found > vq->num)) {
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002792 vq_err(vq, "Loop detected: last one at %u "
2793 "vq size %u head %u\n",
2794 i, vq->num, head);
Michael S. Tsirkind5675bd2010-06-24 16:59:59 +03002795 return -EINVAL;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002796 }
Jason Wang7b5d7532019-05-24 04:12:14 -04002797 ret = vhost_get_desc(vq, &desc, i);
Michael S. Tsirkin7b3384f2010-07-01 18:40:12 +03002798 if (unlikely(ret)) {
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002799 vq_err(vq, "Failed to get descriptor: idx %d addr %p\n",
2800 i, vq->desc + i);
Michael S. Tsirkind5675bd2010-06-24 16:59:59 +03002801 return -EFAULT;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002802 }
Michael S. Tsirkin3b1bbe82014-10-24 14:04:47 +03002803 if (desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_INDIRECT)) {
Michael S. Tsirkin47283be2014-06-05 15:20:27 +03002804 ret = get_indirect(vq, iov, iov_size,
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002805 out_num, in_num,
2806 log, log_num, &desc);
Michael S. Tsirkin7b3384f2010-07-01 18:40:12 +03002807 if (unlikely(ret < 0)) {
Jason Wang6b1e6cc2016-06-23 02:04:32 -04002808 if (ret != -EAGAIN)
2809 vq_err(vq, "Failure detected "
2810 "in indirect descriptor at idx %d\n", i);
Michael S. Tsirkind5675bd2010-06-24 16:59:59 +03002811 return ret;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002812 }
2813 continue;
2814 }
2815
Jason Wang6b1e6cc2016-06-23 02:04:32 -04002816 if (desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_WRITE))
2817 access = VHOST_ACCESS_WO;
2818 else
2819 access = VHOST_ACCESS_RO;
Michael S. Tsirkin3b1bbe82014-10-24 14:04:47 +03002820 ret = translate_desc(vq, vhost64_to_cpu(vq, desc.addr),
2821 vhost32_to_cpu(vq, desc.len), iov + iov_count,
Jason Wang6b1e6cc2016-06-23 02:04:32 -04002822 iov_size - iov_count, access);
Michael S. Tsirkin7b3384f2010-07-01 18:40:12 +03002823 if (unlikely(ret < 0)) {
Jason Wang6b1e6cc2016-06-23 02:04:32 -04002824 if (ret != -EAGAIN)
2825 vq_err(vq, "Translation failure %d descriptor idx %d\n",
2826 ret, i);
Michael S. Tsirkind5675bd2010-06-24 16:59:59 +03002827 return ret;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002828 }
Jason Wang6b1e6cc2016-06-23 02:04:32 -04002829 if (access == VHOST_ACCESS_WO) {
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002830 /* If this is an input descriptor,
2831 * increment that count. */
2832 *in_num += ret;
2833 if (unlikely(log)) {
Michael S. Tsirkin3b1bbe82014-10-24 14:04:47 +03002834 log[*log_num].addr = vhost64_to_cpu(vq, desc.addr);
2835 log[*log_num].len = vhost32_to_cpu(vq, desc.len);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002836 ++*log_num;
2837 }
2838 } else {
2839 /* If it's an output descriptor, they're all supposed
2840 * to come before any input descriptors. */
Michael S. Tsirkin7b3384f2010-07-01 18:40:12 +03002841 if (unlikely(*in_num)) {
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002842 vq_err(vq, "Descriptor has out after in: "
2843 "idx %d\n", i);
Michael S. Tsirkind5675bd2010-06-24 16:59:59 +03002844 return -EINVAL;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002845 }
2846 *out_num += ret;
2847 }
Michael S. Tsirkin3b1bbe82014-10-24 14:04:47 +03002848 } while ((i = next_desc(vq, &desc)) != -1);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002849
2850 /* On success, increment avail index. */
2851 vq->last_avail_idx++;
Michael S. Tsirkin8ea8cf82011-05-20 02:10:54 +03002852
2853 /* Assume notifications from guest are disabled at this point,
2854 * if they aren't we would need to update avail_event index. */
2855 BUG_ON(!(vq->used_flags & VRING_USED_F_NO_NOTIFY));
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002856 return head;
2857}
Asias He6ac1afb2013-05-06 16:38:21 +08002858EXPORT_SYMBOL_GPL(vhost_get_vq_desc);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002859
2860/* Reverse the effect of vhost_get_vq_desc. Useful for error handling. */
David Stevens8dd014a2010-07-27 18:52:21 +03002861void vhost_discard_vq_desc(struct vhost_virtqueue *vq, int n)
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002862{
David Stevens8dd014a2010-07-27 18:52:21 +03002863 vq->last_avail_idx -= n;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002864}
Asias He6ac1afb2013-05-06 16:38:21 +08002865EXPORT_SYMBOL_GPL(vhost_discard_vq_desc);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002866
2867/* After we've used one of their buffers, we tell them about it. We'll then
2868 * want to notify the guest, using eventfd. */
2869int vhost_add_used(struct vhost_virtqueue *vq, unsigned int head, int len)
2870{
Michael S. Tsirkin3b1bbe82014-10-24 14:04:47 +03002871 struct vring_used_elem heads = {
2872 cpu_to_vhost32(vq, head),
2873 cpu_to_vhost32(vq, len)
2874 };
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002875
Jason Wangc49e4e52013-09-02 16:40:58 +08002876 return vhost_add_used_n(vq, &heads, 1);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002877}
Asias He6ac1afb2013-05-06 16:38:21 +08002878EXPORT_SYMBOL_GPL(vhost_add_used);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002879
David Stevens8dd014a2010-07-27 18:52:21 +03002880static int __vhost_add_used_n(struct vhost_virtqueue *vq,
2881 struct vring_used_elem *heads,
2882 unsigned count)
2883{
2884 struct vring_used_elem __user *used;
Michael S. Tsirkin8ea8cf82011-05-20 02:10:54 +03002885 u16 old, new;
David Stevens8dd014a2010-07-27 18:52:21 +03002886 int start;
2887
Michael S. Tsirkin5fba13b2015-11-29 13:34:44 +02002888 start = vq->last_used_idx & (vq->num - 1);
David Stevens8dd014a2010-07-27 18:52:21 +03002889 used = vq->used->ring + start;
Jason Wang7b5d7532019-05-24 04:12:14 -04002890 if (vhost_put_used(vq, heads, start, count)) {
David Stevens8dd014a2010-07-27 18:52:21 +03002891 vq_err(vq, "Failed to write used");
2892 return -EFAULT;
2893 }
2894 if (unlikely(vq->log_used)) {
2895 /* Make sure data is seen before log. */
2896 smp_wmb();
2897 /* Log used ring entry write. */
Jason Wangcc5e7102019-01-16 16:54:42 +08002898 log_used(vq, ((void __user *)used - (void __user *)vq->used),
2899 count * sizeof *used);
David Stevens8dd014a2010-07-27 18:52:21 +03002900 }
Michael S. Tsirkin8ea8cf82011-05-20 02:10:54 +03002901 old = vq->last_used_idx;
2902 new = (vq->last_used_idx += count);
2903 /* If the driver never bothers to signal in a very long while,
2904 * used index might wrap around. If that happens, invalidate
2905 * signalled_used index we stored. TODO: make sure driver
2906 * signals at least once in 2^16 and remove this. */
2907 if (unlikely((u16)(new - vq->signalled_used) < (u16)(new - old)))
2908 vq->signalled_used_valid = false;
David Stevens8dd014a2010-07-27 18:52:21 +03002909 return 0;
2910}
2911
2912/* After we've used one of their buffers, we tell them about it. We'll then
2913 * want to notify the guest, using eventfd. */
2914int vhost_add_used_n(struct vhost_virtqueue *vq, struct vring_used_elem *heads,
2915 unsigned count)
2916{
2917 int start, n, r;
2918
Michael S. Tsirkin5fba13b2015-11-29 13:34:44 +02002919 start = vq->last_used_idx & (vq->num - 1);
David Stevens8dd014a2010-07-27 18:52:21 +03002920 n = vq->num - start;
2921 if (n < count) {
2922 r = __vhost_add_used_n(vq, heads, n);
2923 if (r < 0)
2924 return r;
2925 heads += n;
2926 count -= n;
2927 }
2928 r = __vhost_add_used_n(vq, heads, count);
2929
2930 /* Make sure buffer is written before we update index. */
2931 smp_wmb();
Jason Wang7b5d7532019-05-24 04:12:14 -04002932 if (vhost_put_used_idx(vq)) {
David Stevens8dd014a2010-07-27 18:52:21 +03002933 vq_err(vq, "Failed to increment used idx");
2934 return -EFAULT;
2935 }
2936 if (unlikely(vq->log_used)) {
Jason Wang841df922018-12-13 10:53:37 +08002937 /* Make sure used idx is seen before log. */
2938 smp_wmb();
David Stevens8dd014a2010-07-27 18:52:21 +03002939 /* Log used index update. */
Jason Wangcc5e7102019-01-16 16:54:42 +08002940 log_used(vq, offsetof(struct vring_used, idx),
2941 sizeof vq->used->idx);
David Stevens8dd014a2010-07-27 18:52:21 +03002942 if (vq->log_ctx)
2943 eventfd_signal(vq->log_ctx, 1);
2944 }
2945 return r;
2946}
Asias He6ac1afb2013-05-06 16:38:21 +08002947EXPORT_SYMBOL_GPL(vhost_add_used_n);
David Stevens8dd014a2010-07-27 18:52:21 +03002948
Michael S. Tsirkin8ea8cf82011-05-20 02:10:54 +03002949static bool vhost_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002950{
Michael S. Tsirkin3b1bbe82014-10-24 14:04:47 +03002951 __u16 old, new;
2952 __virtio16 event;
Michael S. Tsirkin8ea8cf82011-05-20 02:10:54 +03002953 bool v;
Jason Wang8d658432017-07-27 11:22:05 +08002954 /* Flush out used index updates. This is paired
2955 * with the barrier that the Guest executes when enabling
2956 * interrupts. */
2957 smp_mb();
Michael S. Tsirkin0d499352010-05-11 19:44:17 +03002958
Michael S. Tsirkinea16c512014-06-05 15:20:23 +03002959 if (vhost_has_feature(vq, VIRTIO_F_NOTIFY_ON_EMPTY) &&
Michael S. Tsirkin8ea8cf82011-05-20 02:10:54 +03002960 unlikely(vq->avail_idx == vq->last_avail_idx))
2961 return true;
2962
Michael S. Tsirkinea16c512014-06-05 15:20:23 +03002963 if (!vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX)) {
Michael S. Tsirkin3b1bbe82014-10-24 14:04:47 +03002964 __virtio16 flags;
Jason Wang7b5d7532019-05-24 04:12:14 -04002965 if (vhost_get_avail_flags(vq, &flags)) {
Michael S. Tsirkin8ea8cf82011-05-20 02:10:54 +03002966 vq_err(vq, "Failed to get flags");
2967 return true;
2968 }
Michael S. Tsirkin3b1bbe82014-10-24 14:04:47 +03002969 return !(flags & cpu_to_vhost16(vq, VRING_AVAIL_F_NO_INTERRUPT));
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002970 }
Michael S. Tsirkin8ea8cf82011-05-20 02:10:54 +03002971 old = vq->signalled_used;
2972 v = vq->signalled_used_valid;
2973 new = vq->signalled_used = vq->last_used_idx;
2974 vq->signalled_used_valid = true;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002975
Michael S. Tsirkin8ea8cf82011-05-20 02:10:54 +03002976 if (unlikely(!v))
2977 return true;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002978
Jason Wang7b5d7532019-05-24 04:12:14 -04002979 if (vhost_get_used_event(vq, &event)) {
Michael S. Tsirkin8ea8cf82011-05-20 02:10:54 +03002980 vq_err(vq, "Failed to get used event idx");
2981 return true;
2982 }
Jason Wang8d658432017-07-27 11:22:05 +08002983 return vring_need_event(vhost16_to_cpu(vq, event), new, old);
Michael S. Tsirkin8ea8cf82011-05-20 02:10:54 +03002984}
2985
2986/* This actually signals the guest, using eventfd. */
2987void vhost_signal(struct vhost_dev *dev, struct vhost_virtqueue *vq)
2988{
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002989 /* Signal the Guest tell them we used something up. */
Michael S. Tsirkin8ea8cf82011-05-20 02:10:54 +03002990 if (vq->call_ctx && vhost_notify(dev, vq))
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002991 eventfd_signal(vq->call_ctx, 1);
2992}
Asias He6ac1afb2013-05-06 16:38:21 +08002993EXPORT_SYMBOL_GPL(vhost_signal);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00002994
2995/* And here's the combo meal deal. Supersize me! */
2996void vhost_add_used_and_signal(struct vhost_dev *dev,
2997 struct vhost_virtqueue *vq,
2998 unsigned int head, int len)
2999{
3000 vhost_add_used(vq, head, len);
3001 vhost_signal(dev, vq);
3002}
Asias He6ac1afb2013-05-06 16:38:21 +08003003EXPORT_SYMBOL_GPL(vhost_add_used_and_signal);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00003004
David Stevens8dd014a2010-07-27 18:52:21 +03003005/* multi-buffer version of vhost_add_used_and_signal */
3006void vhost_add_used_and_signal_n(struct vhost_dev *dev,
3007 struct vhost_virtqueue *vq,
3008 struct vring_used_elem *heads, unsigned count)
3009{
3010 vhost_add_used_n(vq, heads, count);
3011 vhost_signal(dev, vq);
3012}
Asias He6ac1afb2013-05-06 16:38:21 +08003013EXPORT_SYMBOL_GPL(vhost_add_used_and_signal_n);
David Stevens8dd014a2010-07-27 18:52:21 +03003014
Jason Wangd4a60602016-03-04 06:24:52 -05003015/* return true if we're sure that avaiable ring is empty */
3016bool vhost_vq_avail_empty(struct vhost_dev *dev, struct vhost_virtqueue *vq)
3017{
3018 __virtio16 avail_idx;
3019 int r;
3020
Jason Wang275bf962017-01-18 15:02:01 +08003021 if (vq->avail_idx != vq->last_avail_idx)
Jason Wangd4a60602016-03-04 06:24:52 -05003022 return false;
3023
Jason Wang7b5d7532019-05-24 04:12:14 -04003024 r = vhost_get_avail_idx(vq, &avail_idx);
Jason Wang275bf962017-01-18 15:02:01 +08003025 if (unlikely(r))
3026 return false;
3027 vq->avail_idx = vhost16_to_cpu(vq, avail_idx);
3028
3029 return vq->avail_idx == vq->last_avail_idx;
Jason Wangd4a60602016-03-04 06:24:52 -05003030}
3031EXPORT_SYMBOL_GPL(vhost_vq_avail_empty);
3032
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00003033/* OK, now we need to know about added descriptors. */
Michael S. Tsirkin8ea8cf82011-05-20 02:10:54 +03003034bool vhost_enable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00003035{
Michael S. Tsirkin3b1bbe82014-10-24 14:04:47 +03003036 __virtio16 avail_idx;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00003037 int r;
Krishna Kumard47effe2011-03-01 17:06:37 +05303038
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00003039 if (!(vq->used_flags & VRING_USED_F_NO_NOTIFY))
3040 return false;
3041 vq->used_flags &= ~VRING_USED_F_NO_NOTIFY;
Michael S. Tsirkinea16c512014-06-05 15:20:23 +03003042 if (!vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX)) {
Jason Wang2723fea2011-06-21 18:04:38 +08003043 r = vhost_update_used_flags(vq);
Michael S. Tsirkin8ea8cf82011-05-20 02:10:54 +03003044 if (r) {
3045 vq_err(vq, "Failed to enable notification at %p: %d\n",
3046 &vq->used->flags, r);
3047 return false;
3048 }
3049 } else {
Jason Wang2723fea2011-06-21 18:04:38 +08003050 r = vhost_update_avail_event(vq, vq->avail_idx);
Michael S. Tsirkin8ea8cf82011-05-20 02:10:54 +03003051 if (r) {
3052 vq_err(vq, "Failed to update avail event index at %p: %d\n",
3053 vhost_avail_event(vq), r);
3054 return false;
3055 }
3056 }
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00003057 /* They could have slipped one in as we were doing that: make
3058 * sure it's written, then check again. */
Michael S. Tsirkin56593382010-02-01 07:21:02 +00003059 smp_mb();
Jason Wang7b5d7532019-05-24 04:12:14 -04003060 r = vhost_get_avail_idx(vq, &avail_idx);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00003061 if (r) {
3062 vq_err(vq, "Failed to check avail idx at %p: %d\n",
3063 &vq->avail->idx, r);
3064 return false;
3065 }
3066
Michael S. Tsirkin3b1bbe82014-10-24 14:04:47 +03003067 return vhost16_to_cpu(vq, avail_idx) != vq->avail_idx;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00003068}
Asias He6ac1afb2013-05-06 16:38:21 +08003069EXPORT_SYMBOL_GPL(vhost_enable_notify);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00003070
3071/* We don't need to be notified again. */
Michael S. Tsirkin8ea8cf82011-05-20 02:10:54 +03003072void vhost_disable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00003073{
3074 int r;
Krishna Kumard47effe2011-03-01 17:06:37 +05303075
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00003076 if (vq->used_flags & VRING_USED_F_NO_NOTIFY)
3077 return;
3078 vq->used_flags |= VRING_USED_F_NO_NOTIFY;
Michael S. Tsirkinea16c512014-06-05 15:20:23 +03003079 if (!vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX)) {
Jason Wang2723fea2011-06-21 18:04:38 +08003080 r = vhost_update_used_flags(vq);
Michael S. Tsirkin8ea8cf82011-05-20 02:10:54 +03003081 if (r)
3082 vq_err(vq, "Failed to enable notification at %p: %d\n",
3083 &vq->used->flags, r);
3084 }
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00003085}
Asias He6ac1afb2013-05-06 16:38:21 +08003086EXPORT_SYMBOL_GPL(vhost_disable_notify);
3087
Jason Wang6b1e6cc2016-06-23 02:04:32 -04003088/* Create a new message. */
3089struct vhost_msg_node *vhost_new_msg(struct vhost_virtqueue *vq, int type)
3090{
3091 struct vhost_msg_node *node = kmalloc(sizeof *node, GFP_KERNEL);
3092 if (!node)
3093 return NULL;
Michael S. Tsirkin670ae9c2018-05-12 00:33:10 +03003094
3095 /* Make sure all padding within the structure is initialized. */
3096 memset(&node->msg, 0, sizeof node->msg);
Jason Wang6b1e6cc2016-06-23 02:04:32 -04003097 node->vq = vq;
3098 node->msg.type = type;
3099 return node;
3100}
3101EXPORT_SYMBOL_GPL(vhost_new_msg);
3102
3103void vhost_enqueue_msg(struct vhost_dev *dev, struct list_head *head,
3104 struct vhost_msg_node *node)
3105{
3106 spin_lock(&dev->iotlb_lock);
3107 list_add_tail(&node->node, head);
3108 spin_unlock(&dev->iotlb_lock);
3109
Linus Torvaldsa9a08842018-02-11 14:34:03 -08003110 wake_up_interruptible_poll(&dev->wait, EPOLLIN | EPOLLRDNORM);
Jason Wang6b1e6cc2016-06-23 02:04:32 -04003111}
3112EXPORT_SYMBOL_GPL(vhost_enqueue_msg);
3113
3114struct vhost_msg_node *vhost_dequeue_msg(struct vhost_dev *dev,
3115 struct list_head *head)
3116{
3117 struct vhost_msg_node *node = NULL;
3118
3119 spin_lock(&dev->iotlb_lock);
3120 if (!list_empty(head)) {
3121 node = list_first_entry(head, struct vhost_msg_node,
3122 node);
3123 list_del(&node->node);
3124 }
3125 spin_unlock(&dev->iotlb_lock);
3126
3127 return node;
3128}
3129EXPORT_SYMBOL_GPL(vhost_dequeue_msg);
3130
3131
Asias He6ac1afb2013-05-06 16:38:21 +08003132static int __init vhost_init(void)
3133{
3134 return 0;
3135}
3136
3137static void __exit vhost_exit(void)
3138{
3139}
3140
3141module_init(vhost_init);
3142module_exit(vhost_exit);
3143
3144MODULE_VERSION("0.0.1");
3145MODULE_LICENSE("GPL v2");
3146MODULE_AUTHOR("Michael S. Tsirkin");
3147MODULE_DESCRIPTION("Host kernel accelerator for virtio");