blob: 5087cf703ed17767f8aa31130c231367364480e9 [file] [log] [blame]
Avi Kivity6aa8b732006-12-10 02:21:36 -08001/*
2 * Kernel-based Virtual Machine driver for Linux
3 *
4 * This module enables machines with Intel VT-x extensions to run virtual
5 * machines without emulation or binary translation.
6 *
7 * Copyright (C) 2006 Qumranet, Inc.
Nicolas Kaiser9611c182010-10-06 14:23:22 +02008 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
Avi Kivity6aa8b732006-12-10 02:21:36 -08009 *
10 * Authors:
11 * Avi Kivity <avi@qumranet.com>
12 * Yaniv Kamay <yaniv@qumranet.com>
13 *
14 * This work is licensed under the terms of the GNU GPL, version 2. See
15 * the COPYING file in the top-level directory.
16 *
17 */
18
Andre Przywaraaf669ac2015-03-26 14:39:29 +000019#include <kvm/iodev.h>
Avi Kivity6aa8b732006-12-10 02:21:36 -080020
Avi Kivityedf88412007-12-16 11:02:48 +020021#include <linux/kvm_host.h>
Avi Kivity6aa8b732006-12-10 02:21:36 -080022#include <linux/kvm.h>
23#include <linux/module.h>
24#include <linux/errno.h>
Avi Kivity6aa8b732006-12-10 02:21:36 -080025#include <linux/percpu.h>
Avi Kivity6aa8b732006-12-10 02:21:36 -080026#include <linux/mm.h>
27#include <linux/miscdevice.h>
28#include <linux/vmalloc.h>
Avi Kivity6aa8b732006-12-10 02:21:36 -080029#include <linux/reboot.h>
Avi Kivity6aa8b732006-12-10 02:21:36 -080030#include <linux/debugfs.h>
31#include <linux/highmem.h>
32#include <linux/file.h>
Rafael J. Wysockifb3600c2011-03-23 22:16:23 +010033#include <linux/syscore_ops.h>
Avi Kivity774c47f2007-02-12 00:54:47 -080034#include <linux/cpu.h>
Ingo Molnar174cd4b2017-02-02 19:15:33 +010035#include <linux/sched/signal.h>
Ingo Molnar6e84f312017-02-08 18:51:29 +010036#include <linux/sched/mm.h>
Ingo Molnar03441a32017-02-08 18:51:35 +010037#include <linux/sched/stat.h>
Avi Kivityd9e368d2007-06-07 19:18:30 +030038#include <linux/cpumask.h>
39#include <linux/smp.h>
Avi Kivityd6d28162007-06-28 08:38:16 -040040#include <linux/anon_inodes.h>
Avi Kivity04d2cc72007-09-10 18:10:54 +030041#include <linux/profile.h>
Anthony Liguori7aa81cc2007-09-17 14:57:50 -050042#include <linux/kvm_para.h>
Izik Eidus6fc138d2007-10-09 19:20:39 +020043#include <linux/pagemap.h>
Anthony Liguori8d4e1282007-10-18 09:59:34 -050044#include <linux/mman.h>
Anthony Liguori35149e22008-04-02 14:46:56 -050045#include <linux/swap.h>
Sheng Yange56d5322009-03-12 21:45:39 +080046#include <linux/bitops.h>
Marcelo Tosatti547de292009-05-07 17:55:13 -030047#include <linux/spinlock.h>
Arnd Bergmann6ff58942009-10-22 14:19:27 +020048#include <linux/compat.h>
Marcelo Tosattibc6678a2009-12-23 14:35:21 -020049#include <linux/srcu.h>
Joerg Roedel8f0b1ab2010-01-28 12:37:56 +010050#include <linux/hugetlb.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090051#include <linux/slab.h>
Sasha Levin743eeb02011-07-27 16:00:48 +030052#include <linux/sort.h>
53#include <linux/bsearch.h>
Avi Kivity6aa8b732006-12-10 02:21:36 -080054
Avi Kivitye4956062007-06-28 14:15:57 -040055#include <asm/processor.h>
Avi Kivitye4956062007-06-28 14:15:57 -040056#include <asm/io.h>
David Matlack2ea75be2014-09-19 16:03:25 -070057#include <asm/ioctl.h>
Linus Torvalds7c0f6ba2016-12-24 11:46:01 -080058#include <linux/uaccess.h>
Izik Eidus3e021bf2007-11-19 11:16:57 +020059#include <asm/pgtable.h>
Avi Kivity6aa8b732006-12-10 02:21:36 -080060
Laurent Vivier5f94c172008-05-30 16:05:54 +020061#include "coalesced_mmio.h"
Gleb Natapovaf585b92010-10-14 11:22:46 +020062#include "async_pf.h"
Paolo Bonzini3c3c29f2014-09-24 13:02:46 +020063#include "vfio.h"
Laurent Vivier5f94c172008-05-30 16:05:54 +020064
Marcelo Tosatti229456f2009-06-17 09:22:14 -030065#define CREATE_TRACE_POINTS
66#include <trace/events/kvm.h>
67
Janosch Frank536a6f82016-05-18 13:26:23 +020068/* Worst case buffer size needed for holding an integer. */
69#define ITOA_MAX_LEN 12
70
Avi Kivity6aa8b732006-12-10 02:21:36 -080071MODULE_AUTHOR("Qumranet");
72MODULE_LICENSE("GPL");
73
David Hildenbrand920552b2015-09-18 12:34:53 +020074/* Architectures should define their poll value according to the halt latency */
Suraj Jitindar Singhec76d812016-10-14 11:53:19 +110075unsigned int halt_poll_ns = KVM_HALT_POLL_NS_DEFAULT;
Roman Storozhenko039c5d12017-06-27 12:51:18 +030076module_param(halt_poll_ns, uint, 0644);
Suraj Jitindar Singhec76d812016-10-14 11:53:19 +110077EXPORT_SYMBOL_GPL(halt_poll_ns);
Paolo Bonzinif7819512015-02-04 18:20:58 +010078
Wanpeng Liaca6ff22015-09-03 22:07:38 +080079/* Default doubles per-vcpu halt_poll_ns. */
Suraj Jitindar Singhec76d812016-10-14 11:53:19 +110080unsigned int halt_poll_ns_grow = 2;
Roman Storozhenko039c5d12017-06-27 12:51:18 +030081module_param(halt_poll_ns_grow, uint, 0644);
Suraj Jitindar Singhec76d812016-10-14 11:53:19 +110082EXPORT_SYMBOL_GPL(halt_poll_ns_grow);
Wanpeng Liaca6ff22015-09-03 22:07:38 +080083
Nir Weiner49113d32019-01-27 12:17:15 +020084/* The start value to grow halt_poll_ns from */
85unsigned int halt_poll_ns_grow_start = 10000; /* 10us */
86module_param(halt_poll_ns_grow_start, uint, 0644);
87EXPORT_SYMBOL_GPL(halt_poll_ns_grow_start);
88
Wanpeng Liaca6ff22015-09-03 22:07:38 +080089/* Default resets per-vcpu halt_poll_ns . */
Suraj Jitindar Singhec76d812016-10-14 11:53:19 +110090unsigned int halt_poll_ns_shrink;
Roman Storozhenko039c5d12017-06-27 12:51:18 +030091module_param(halt_poll_ns_shrink, uint, 0644);
Suraj Jitindar Singhec76d812016-10-14 11:53:19 +110092EXPORT_SYMBOL_GPL(halt_poll_ns_shrink);
Wanpeng Liaca6ff22015-09-03 22:07:38 +080093
Marcelo Tosattifa40a822009-06-04 15:08:24 -030094/*
95 * Ordering of locks:
96 *
Xiubo Lib7d409d2015-02-26 14:58:24 +080097 * kvm->lock --> kvm->slots_lock --> kvm->irq_lock
Marcelo Tosattifa40a822009-06-04 15:08:24 -030098 */
99
Paolo Bonzini2f303b72013-09-25 13:53:07 +0200100DEFINE_SPINLOCK(kvm_lock);
Paolo Bonzini4a937f92013-09-10 12:58:35 +0200101static DEFINE_RAW_SPINLOCK(kvm_count_lock);
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +0800102LIST_HEAD(vm_list);
Avi Kivity133de902007-02-12 00:54:44 -0800103
Rusty Russell7f59f492008-12-07 21:25:45 +1030104static cpumask_var_t cpus_hardware_enabled;
Xiubo Lif4fee932015-02-26 14:58:21 +0800105static int kvm_usage_count;
Alexander Graf10474ae2009-09-15 11:37:46 +0200106static atomic_t hardware_enable_failed;
Avi Kivity1b6c0162007-05-24 13:03:52 +0300107
Rusty Russellc16f8622007-07-30 21:12:19 +1000108struct kmem_cache *kvm_vcpu_cache;
109EXPORT_SYMBOL_GPL(kvm_vcpu_cache);
Avi Kivity1165f5f2007-04-19 17:27:43 +0300110
Avi Kivity15ad7142007-07-11 18:17:21 +0300111static __read_mostly struct preempt_ops kvm_preempt_ops;
112
Hollis Blanchard76f7c872008-04-15 16:05:42 -0500113struct dentry *kvm_debugfs_dir;
Paul Mackerrase23a8082015-03-28 14:21:01 +1100114EXPORT_SYMBOL_GPL(kvm_debugfs_dir);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800115
Janosch Frank536a6f82016-05-18 13:26:23 +0200116static int kvm_debugfs_num_entries;
117static const struct file_operations *stat_fops_per_vm[];
118
Avi Kivitybccf2152007-02-21 18:04:26 +0200119static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
120 unsigned long arg);
Christian Borntraegerde8e5d72015-02-03 09:35:15 +0100121#ifdef CONFIG_KVM_COMPAT
Alexander Graf1dda6062011-06-08 02:45:37 +0200122static long kvm_vcpu_compat_ioctl(struct file *file, unsigned int ioctl,
123 unsigned long arg);
Marc Zyngier7ddfd3e2018-06-17 10:16:21 +0100124#define KVM_COMPAT(c) .compat_ioctl = (c)
125#else
126static long kvm_no_compat_ioctl(struct file *file, unsigned int ioctl,
127 unsigned long arg) { return -EINVAL; }
128#define KVM_COMPAT(c) .compat_ioctl = kvm_no_compat_ioctl
Alexander Graf1dda6062011-06-08 02:45:37 +0200129#endif
Alexander Graf10474ae2009-09-15 11:37:46 +0200130static int hardware_enable_all(void);
131static void hardware_disable_all(void);
Avi Kivitybccf2152007-02-21 18:04:26 +0200132
Marcelo Tosattie93f8a02009-12-23 14:35:24 -0200133static void kvm_io_bus_destroy(struct kvm_io_bus *bus);
Stephen Hemminger79408762013-12-29 12:12:29 -0800134
Paolo Bonzinibc009e42015-05-26 12:43:41 +0200135static void mark_page_dirty_in_slot(struct kvm_memory_slot *memslot, gfn_t gfn);
Marcelo Tosattie93f8a02009-12-23 14:35:24 -0200136
Andi Kleen52480132014-02-08 08:51:57 +0100137__visible bool kvm_rebooting;
Avi Kivityb7c41452010-12-02 17:52:50 +0200138EXPORT_SYMBOL_GPL(kvm_rebooting);
Avi Kivity4ecac3f2008-05-13 13:23:38 +0300139
Marcelo Tosatti54dee992009-06-11 12:07:44 -0300140static bool largepages_enabled = true;
141
Claudio Imbrenda286de8f2017-07-12 17:56:44 +0200142#define KVM_EVENT_CREATE_VM 0
143#define KVM_EVENT_DESTROY_VM 1
144static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm);
145static unsigned long long kvm_createvm_count;
146static unsigned long long kvm_active_vms;
147
Michal Hocko93065ac2018-08-21 21:52:33 -0700148__weak int kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm,
149 unsigned long start, unsigned long end, bool blockable)
Radim Krčmářb1394e72017-11-30 19:05:45 +0100150{
Michal Hocko93065ac2018-08-21 21:52:33 -0700151 return 0;
Radim Krčmářb1394e72017-11-30 19:05:45 +0100152}
153
Dan Williamsba049e92016-01-15 16:56:11 -0800154bool kvm_is_reserved_pfn(kvm_pfn_t pfn)
Ben-Ami Yassourcbff90a2008-07-28 19:26:24 +0300155{
Andrea Arcangeli11feeb42013-07-25 03:04:38 +0200156 if (pfn_valid(pfn))
Ard Biesheuvelbf4bea82014-11-10 08:33:56 +0000157 return PageReserved(pfn_to_page(pfn));
Ben-Ami Yassourcbff90a2008-07-28 19:26:24 +0300158
159 return true;
160}
161
Avi Kivity6aa8b732006-12-10 02:21:36 -0800162/*
163 * Switches to specified vcpu, until a matching vcpu_put()
164 */
Christoffer Dallec7660c2017-12-04 21:35:23 +0100165void vcpu_load(struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800166{
Christoffer Dallec7660c2017-12-04 21:35:23 +0100167 int cpu = get_cpu();
Avi Kivity15ad7142007-07-11 18:17:21 +0300168 preempt_notifier_register(&vcpu->preempt_notifier);
Carsten Otte313a3dc2007-10-11 19:16:52 +0200169 kvm_arch_vcpu_load(vcpu, cpu);
Avi Kivity15ad7142007-07-11 18:17:21 +0300170 put_cpu();
Avi Kivitybccf2152007-02-21 18:04:26 +0200171}
Jim Mattson2f1fe812016-07-08 15:36:06 -0700172EXPORT_SYMBOL_GPL(vcpu_load);
Avi Kivitybccf2152007-02-21 18:04:26 +0200173
Carsten Otte313a3dc2007-10-11 19:16:52 +0200174void vcpu_put(struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800175{
Avi Kivity15ad7142007-07-11 18:17:21 +0300176 preempt_disable();
Carsten Otte313a3dc2007-10-11 19:16:52 +0200177 kvm_arch_vcpu_put(vcpu);
Avi Kivity15ad7142007-07-11 18:17:21 +0300178 preempt_notifier_unregister(&vcpu->preempt_notifier);
179 preempt_enable();
Avi Kivity6aa8b732006-12-10 02:21:36 -0800180}
Jim Mattson2f1fe812016-07-08 15:36:06 -0700181EXPORT_SYMBOL_GPL(vcpu_put);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800182
Paolo Bonzini7a97cec2017-04-27 14:33:43 +0200183/* TODO: merge with kvm_arch_vcpu_should_kick */
184static bool kvm_request_needs_ipi(struct kvm_vcpu *vcpu, unsigned req)
185{
186 int mode = kvm_vcpu_exiting_guest_mode(vcpu);
187
188 /*
189 * We need to wait for the VCPU to reenable interrupts and get out of
190 * READING_SHADOW_PAGE_TABLES mode.
191 */
192 if (req & KVM_REQUEST_WAIT)
193 return mode != OUTSIDE_GUEST_MODE;
194
195 /*
196 * Need to kick a running VCPU, but otherwise there is nothing to do.
197 */
198 return mode == IN_GUEST_MODE;
199}
200
Avi Kivityd9e368d2007-06-07 19:18:30 +0300201static void ack_flush(void *_completed)
202{
Avi Kivityd9e368d2007-06-07 19:18:30 +0300203}
204
Paolo Bonzinib49defe2017-06-30 13:25:45 +0200205static inline bool kvm_kick_many_cpus(const struct cpumask *cpus, bool wait)
206{
207 if (unlikely(!cpus))
208 cpus = cpu_online_mask;
209
210 if (cpumask_empty(cpus))
211 return false;
212
213 smp_call_function_many(cpus, ack_flush, NULL, wait);
214 return true;
215}
216
Vitaly Kuznetsov7053df42018-05-16 17:21:28 +0200217bool kvm_make_vcpus_request_mask(struct kvm *kvm, unsigned int req,
218 unsigned long *vcpu_bitmap, cpumask_var_t tmp)
Avi Kivityd9e368d2007-06-07 19:18:30 +0300219{
Avi Kivity597a5f52008-07-20 14:24:22 +0300220 int i, cpu, me;
Avi Kivityd9e368d2007-06-07 19:18:30 +0300221 struct kvm_vcpu *vcpu;
Vitaly Kuznetsov7053df42018-05-16 17:21:28 +0200222 bool called;
Rusty Russell6ef7a1b2008-12-08 20:28:04 +1030223
Xiao Guangrong3cba4132011-01-12 15:41:22 +0800224 me = get_cpu();
Vitaly Kuznetsov7053df42018-05-16 17:21:28 +0200225
Gleb Natapov988a2ca2009-06-09 15:56:29 +0300226 kvm_for_each_vcpu(i, vcpu, kvm) {
Vitaly Kuznetsova8122972018-08-22 12:18:29 +0200227 if (vcpu_bitmap && !test_bit(i, vcpu_bitmap))
Vitaly Kuznetsov7053df42018-05-16 17:21:28 +0200228 continue;
229
Xiao Guangrong3cba4132011-01-12 15:41:22 +0800230 kvm_make_request(req, vcpu);
Avi Kivityd9e368d2007-06-07 19:18:30 +0300231 cpu = vcpu->cpu;
Xiao Guangrong6b7e2d02011-01-12 15:40:31 +0800232
Radim Krčmář178f02f2017-04-26 22:32:26 +0200233 if (!(req & KVM_REQUEST_NO_WAKEUP) && kvm_vcpu_wake_up(vcpu))
234 continue;
Radim Krčmář6c6e8362017-04-26 22:32:23 +0200235
Vitaly Kuznetsov7053df42018-05-16 17:21:28 +0200236 if (tmp != NULL && cpu != -1 && cpu != me &&
Paolo Bonzini7a97cec2017-04-27 14:33:43 +0200237 kvm_request_needs_ipi(vcpu, req))
Vitaly Kuznetsov7053df42018-05-16 17:21:28 +0200238 __cpumask_set_cpu(cpu, tmp);
Avi Kivityd9e368d2007-06-07 19:18:30 +0300239 }
Vitaly Kuznetsov7053df42018-05-16 17:21:28 +0200240
241 called = kvm_kick_many_cpus(tmp, !!(req & KVM_REQUEST_WAIT));
Xiao Guangrong3cba4132011-01-12 15:41:22 +0800242 put_cpu();
Vitaly Kuznetsov7053df42018-05-16 17:21:28 +0200243
244 return called;
245}
246
247bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req)
248{
249 cpumask_var_t cpus;
250 bool called;
Vitaly Kuznetsov7053df42018-05-16 17:21:28 +0200251
252 zalloc_cpumask_var(&cpus, GFP_ATOMIC);
253
Vitaly Kuznetsova8122972018-08-22 12:18:29 +0200254 called = kvm_make_vcpus_request_mask(kvm, req, NULL, cpus);
Vitaly Kuznetsov7053df42018-05-16 17:21:28 +0200255
Rusty Russell6ef7a1b2008-12-08 20:28:04 +1030256 free_cpumask_var(cpus);
Rusty Russell49846892008-12-08 20:26:24 +1030257 return called;
258}
259
Mario Smarducha6d51012015-01-15 15:58:52 -0800260#ifndef CONFIG_HAVE_KVM_ARCH_TLB_FLUSH_ALL
Rusty Russell49846892008-12-08 20:26:24 +1030261void kvm_flush_remote_tlbs(struct kvm *kvm)
262{
Lan Tianyu4ae3cb32016-03-13 11:10:28 +0800263 /*
264 * Read tlbs_dirty before setting KVM_REQ_TLB_FLUSH in
265 * kvm_make_all_cpus_request.
266 */
267 long dirty_count = smp_load_acquire(&kvm->tlbs_dirty);
Xiao Guangronga086f6a2014-04-17 17:06:12 +0800268
Lan Tianyu4ae3cb32016-03-13 11:10:28 +0800269 /*
270 * We want to publish modifications to the page tables before reading
271 * mode. Pairs with a memory barrier in arch-specific code.
272 * - x86: smp_mb__after_srcu_read_unlock in vcpu_enter_guest
273 * and smp_mb in walk_shadow_page_lockless_begin/end.
274 * - powerpc: smp_mb in kvmppc_prepare_to_enter.
275 *
276 * There is already an smp_mb__after_atomic() before
277 * kvm_make_all_cpus_request() reads vcpu->mode. We reuse that
278 * barrier here.
279 */
Tianyu Lanb08660e2018-07-19 08:40:17 +0000280 if (!kvm_arch_flush_remote_tlb(kvm)
281 || kvm_make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH))
Rusty Russell49846892008-12-08 20:26:24 +1030282 ++kvm->stat.remote_tlb_flush;
Xiao Guangronga086f6a2014-04-17 17:06:12 +0800283 cmpxchg(&kvm->tlbs_dirty, dirty_count, 0);
Avi Kivityd9e368d2007-06-07 19:18:30 +0300284}
Aneesh Kumar K.V2ba9f0d2013-10-07 22:17:59 +0530285EXPORT_SYMBOL_GPL(kvm_flush_remote_tlbs);
Mario Smarducha6d51012015-01-15 15:58:52 -0800286#endif
Avi Kivityd9e368d2007-06-07 19:18:30 +0300287
Marcelo Tosatti2e53d632008-02-20 14:47:24 -0500288void kvm_reload_remote_mmus(struct kvm *kvm)
289{
Tang Chen445b8232014-09-24 15:57:55 +0800290 kvm_make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD);
Marcelo Tosatti2e53d632008-02-20 14:47:24 -0500291}
292
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000293int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
294{
295 struct page *page;
296 int r;
297
298 mutex_init(&vcpu->mutex);
299 vcpu->cpu = -1;
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000300 vcpu->kvm = kvm;
301 vcpu->vcpu_id = id;
Rik van Riel34bb10b2011-02-01 09:52:41 -0500302 vcpu->pid = NULL;
Marcelo Tosatti85773702016-02-19 09:46:39 +0100303 init_swait_queue_head(&vcpu->wq);
Gleb Natapovaf585b92010-10-14 11:22:46 +0200304 kvm_async_pf_vcpu_init(vcpu);
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000305
Feng Wubf9f6ac2015-09-18 22:29:55 +0800306 vcpu->pre_pcpu = -1;
307 INIT_LIST_HEAD(&vcpu->blocked_vcpu_list);
308
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000309 page = alloc_page(GFP_KERNEL | __GFP_ZERO);
310 if (!page) {
311 r = -ENOMEM;
312 goto fail;
313 }
314 vcpu->run = page_address(page);
315
Raghavendra K T4c088492012-07-18 19:07:46 +0530316 kvm_vcpu_set_in_spin_loop(vcpu, false);
317 kvm_vcpu_set_dy_eligible(vcpu, false);
Raghavendra K T3a08a8f2013-03-04 23:32:07 +0530318 vcpu->preempted = false;
Raghavendra K T4c088492012-07-18 19:07:46 +0530319
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +0800320 r = kvm_arch_vcpu_init(vcpu);
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000321 if (r < 0)
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +0800322 goto fail_free_run;
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000323 return 0;
324
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000325fail_free_run:
326 free_page((unsigned long)vcpu->run);
327fail:
Rusty Russell76fafa52007-10-08 10:50:48 +1000328 return r;
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000329}
330EXPORT_SYMBOL_GPL(kvm_vcpu_init);
331
332void kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
333{
Christian Borntraeger0e4524a2017-07-06 14:44:28 +0200334 /*
335 * no need for rcu_read_lock as VCPU_RUN is the only place that
336 * will change the vcpu->pid pointer and on uninit all file
337 * descriptors are already gone.
338 */
339 put_pid(rcu_dereference_protected(vcpu->pid, 1));
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +0800340 kvm_arch_vcpu_uninit(vcpu);
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000341 free_page((unsigned long)vcpu->run);
342}
343EXPORT_SYMBOL_GPL(kvm_vcpu_uninit);
344
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200345#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
346static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn)
347{
348 return container_of(mn, struct kvm, mmu_notifier);
349}
350
Izik Eidus3da0dd42009-09-23 21:47:18 +0300351static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn,
352 struct mm_struct *mm,
353 unsigned long address,
354 pte_t pte)
355{
356 struct kvm *kvm = mmu_notifier_to_kvm(mn);
Marcelo Tosattibc6678a2009-12-23 14:35:21 -0200357 int idx;
Izik Eidus3da0dd42009-09-23 21:47:18 +0300358
Marcelo Tosattibc6678a2009-12-23 14:35:21 -0200359 idx = srcu_read_lock(&kvm->srcu);
Izik Eidus3da0dd42009-09-23 21:47:18 +0300360 spin_lock(&kvm->mmu_lock);
361 kvm->mmu_notifier_seq++;
Lan Tianyu0cf853c2018-12-06 21:21:11 +0800362
363 if (kvm_set_spte_hva(kvm, address, pte))
364 kvm_flush_remote_tlbs(kvm);
365
Izik Eidus3da0dd42009-09-23 21:47:18 +0300366 spin_unlock(&kvm->mmu_lock);
Marcelo Tosattibc6678a2009-12-23 14:35:21 -0200367 srcu_read_unlock(&kvm->srcu, idx);
Izik Eidus3da0dd42009-09-23 21:47:18 +0300368}
369
Michal Hocko93065ac2018-08-21 21:52:33 -0700370static int kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
Jérôme Glisse5d6527a2018-12-28 00:38:05 -0800371 const struct mmu_notifier_range *range)
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200372{
373 struct kvm *kvm = mmu_notifier_to_kvm(mn);
Marcelo Tosattibc6678a2009-12-23 14:35:21 -0200374 int need_tlb_flush = 0, idx;
Michal Hocko93065ac2018-08-21 21:52:33 -0700375 int ret;
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200376
Marcelo Tosattibc6678a2009-12-23 14:35:21 -0200377 idx = srcu_read_lock(&kvm->srcu);
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200378 spin_lock(&kvm->mmu_lock);
379 /*
380 * The count increase must become visible at unlock time as no
381 * spte can be established without taking the mmu_lock and
382 * count is also read inside the mmu_lock critical section.
383 */
384 kvm->mmu_notifier_count++;
Jérôme Glisse5d6527a2018-12-28 00:38:05 -0800385 need_tlb_flush = kvm_unmap_hva_range(kvm, range->start, range->end);
Xiao Guangronga4ee1ca2010-11-23 11:13:00 +0800386 need_tlb_flush |= kvm->tlbs_dirty;
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200387 /* we've to flush the tlb before the pages can be freed */
388 if (need_tlb_flush)
389 kvm_flush_remote_tlbs(kvm);
Takuya Yoshikawa565f3be2012-02-10 15:28:31 +0900390
391 spin_unlock(&kvm->mmu_lock);
Radim Krčmářb1394e72017-11-30 19:05:45 +0100392
Jérôme Glisse5d6527a2018-12-28 00:38:05 -0800393 ret = kvm_arch_mmu_notifier_invalidate_range(kvm, range->start,
394 range->end, range->blockable);
Radim Krčmářb1394e72017-11-30 19:05:45 +0100395
Takuya Yoshikawa565f3be2012-02-10 15:28:31 +0900396 srcu_read_unlock(&kvm->srcu, idx);
Michal Hocko93065ac2018-08-21 21:52:33 -0700397
398 return ret;
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200399}
400
401static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
Jérôme Glisse5d6527a2018-12-28 00:38:05 -0800402 const struct mmu_notifier_range *range)
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200403{
404 struct kvm *kvm = mmu_notifier_to_kvm(mn);
405
406 spin_lock(&kvm->mmu_lock);
407 /*
408 * This sequence increase will notify the kvm page fault that
409 * the page that is going to be mapped in the spte could have
410 * been freed.
411 */
412 kvm->mmu_notifier_seq++;
Paul Mackerrasa355aa52011-12-12 12:37:21 +0000413 smp_wmb();
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200414 /*
415 * The above sequence increase must be visible before the
Paul Mackerrasa355aa52011-12-12 12:37:21 +0000416 * below count decrease, which is ensured by the smp_wmb above
417 * in conjunction with the smp_rmb in mmu_notifier_retry().
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200418 */
419 kvm->mmu_notifier_count--;
420 spin_unlock(&kvm->mmu_lock);
421
422 BUG_ON(kvm->mmu_notifier_count < 0);
423}
424
425static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn,
426 struct mm_struct *mm,
Andres Lagar-Cavilla57128462014-09-22 14:54:42 -0700427 unsigned long start,
428 unsigned long end)
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200429{
430 struct kvm *kvm = mmu_notifier_to_kvm(mn);
Marcelo Tosattibc6678a2009-12-23 14:35:21 -0200431 int young, idx;
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200432
Marcelo Tosattibc6678a2009-12-23 14:35:21 -0200433 idx = srcu_read_lock(&kvm->srcu);
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200434 spin_lock(&kvm->mmu_lock);
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200435
Andres Lagar-Cavilla57128462014-09-22 14:54:42 -0700436 young = kvm_age_hva(kvm, start, end);
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200437 if (young)
438 kvm_flush_remote_tlbs(kvm);
439
Takuya Yoshikawa565f3be2012-02-10 15:28:31 +0900440 spin_unlock(&kvm->mmu_lock);
441 srcu_read_unlock(&kvm->srcu, idx);
442
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200443 return young;
444}
445
Vladimir Davydov1d7715c2015-09-09 15:35:41 -0700446static int kvm_mmu_notifier_clear_young(struct mmu_notifier *mn,
447 struct mm_struct *mm,
448 unsigned long start,
449 unsigned long end)
450{
451 struct kvm *kvm = mmu_notifier_to_kvm(mn);
452 int young, idx;
453
454 idx = srcu_read_lock(&kvm->srcu);
455 spin_lock(&kvm->mmu_lock);
456 /*
457 * Even though we do not flush TLB, this will still adversely
458 * affect performance on pre-Haswell Intel EPT, where there is
459 * no EPT Access Bit to clear so that we have to tear down EPT
460 * tables instead. If we find this unacceptable, we can always
461 * add a parameter to kvm_age_hva so that it effectively doesn't
462 * do anything on clear_young.
463 *
464 * Also note that currently we never issue secondary TLB flushes
465 * from clear_young, leaving this job up to the regular system
466 * cadence. If we find this inaccurate, we might come up with a
467 * more sophisticated heuristic later.
468 */
469 young = kvm_age_hva(kvm, start, end);
470 spin_unlock(&kvm->mmu_lock);
471 srcu_read_unlock(&kvm->srcu, idx);
472
473 return young;
474}
475
Andrea Arcangeli8ee53822011-01-13 15:47:10 -0800476static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn,
477 struct mm_struct *mm,
478 unsigned long address)
479{
480 struct kvm *kvm = mmu_notifier_to_kvm(mn);
481 int young, idx;
482
483 idx = srcu_read_lock(&kvm->srcu);
484 spin_lock(&kvm->mmu_lock);
485 young = kvm_test_age_hva(kvm, address);
486 spin_unlock(&kvm->mmu_lock);
487 srcu_read_unlock(&kvm->srcu, idx);
488
489 return young;
490}
491
Marcelo Tosatti85db06e2008-12-10 21:23:26 +0100492static void kvm_mmu_notifier_release(struct mmu_notifier *mn,
493 struct mm_struct *mm)
494{
495 struct kvm *kvm = mmu_notifier_to_kvm(mn);
Lai Jiangshaneda2bed2010-04-20 14:29:29 +0800496 int idx;
497
498 idx = srcu_read_lock(&kvm->srcu);
Marcelo Tosatti2df72e92012-08-24 15:54:57 -0300499 kvm_arch_flush_shadow_all(kvm);
Lai Jiangshaneda2bed2010-04-20 14:29:29 +0800500 srcu_read_unlock(&kvm->srcu, idx);
Marcelo Tosatti85db06e2008-12-10 21:23:26 +0100501}
502
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200503static const struct mmu_notifier_ops kvm_mmu_notifier_ops = {
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200504 .invalidate_range_start = kvm_mmu_notifier_invalidate_range_start,
505 .invalidate_range_end = kvm_mmu_notifier_invalidate_range_end,
506 .clear_flush_young = kvm_mmu_notifier_clear_flush_young,
Vladimir Davydov1d7715c2015-09-09 15:35:41 -0700507 .clear_young = kvm_mmu_notifier_clear_young,
Andrea Arcangeli8ee53822011-01-13 15:47:10 -0800508 .test_young = kvm_mmu_notifier_test_young,
Izik Eidus3da0dd42009-09-23 21:47:18 +0300509 .change_pte = kvm_mmu_notifier_change_pte,
Marcelo Tosatti85db06e2008-12-10 21:23:26 +0100510 .release = kvm_mmu_notifier_release,
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200511};
Avi Kivity4c07b0a2009-12-20 14:54:04 +0200512
513static int kvm_init_mmu_notifier(struct kvm *kvm)
514{
515 kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops;
516 return mmu_notifier_register(&kvm->mmu_notifier, current->mm);
517}
518
519#else /* !(CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER) */
520
521static int kvm_init_mmu_notifier(struct kvm *kvm)
522{
523 return 0;
524}
525
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200526#endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */
527
Paolo Bonzinia47d2b02015-05-17 11:41:37 +0200528static struct kvm_memslots *kvm_alloc_memslots(void)
Xiao Guangrongbf3e05b2011-11-24 17:40:57 +0800529{
530 int i;
Paolo Bonzinia47d2b02015-05-17 11:41:37 +0200531 struct kvm_memslots *slots;
Xiao Guangrongbf3e05b2011-11-24 17:40:57 +0800532
Ben Gardonb12ce362019-02-11 11:02:49 -0800533 slots = kvzalloc(sizeof(struct kvm_memslots), GFP_KERNEL_ACCOUNT);
Paolo Bonzinia47d2b02015-05-17 11:41:37 +0200534 if (!slots)
535 return NULL;
536
Xiao Guangrongbf3e05b2011-11-24 17:40:57 +0800537 for (i = 0; i < KVM_MEM_SLOTS_NUM; i++)
Xiao Guangrongf85e2cb2011-11-24 17:41:54 +0800538 slots->id_to_index[i] = slots->memslots[i].id = i;
Paolo Bonzinia47d2b02015-05-17 11:41:37 +0200539
540 return slots;
541}
542
543static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot)
544{
545 if (!memslot->dirty_bitmap)
546 return;
547
548 kvfree(memslot->dirty_bitmap);
549 memslot->dirty_bitmap = NULL;
550}
551
552/*
553 * Free any memory in @free but not in @dont.
554 */
555static void kvm_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free,
556 struct kvm_memory_slot *dont)
557{
558 if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
559 kvm_destroy_dirty_bitmap(free);
560
561 kvm_arch_free_memslot(kvm, free, dont);
562
563 free->npages = 0;
564}
565
566static void kvm_free_memslots(struct kvm *kvm, struct kvm_memslots *slots)
567{
568 struct kvm_memory_slot *memslot;
569
570 if (!slots)
571 return;
572
573 kvm_for_each_memslot(memslot, slots)
574 kvm_free_memslot(kvm, memslot, NULL);
575
576 kvfree(slots);
Xiao Guangrongbf3e05b2011-11-24 17:40:57 +0800577}
578
Janosch Frank536a6f82016-05-18 13:26:23 +0200579static void kvm_destroy_vm_debugfs(struct kvm *kvm)
580{
581 int i;
582
583 if (!kvm->debugfs_dentry)
584 return;
585
586 debugfs_remove_recursive(kvm->debugfs_dentry);
587
Luiz Capitulino9d5a1dc2016-09-07 14:47:21 -0400588 if (kvm->debugfs_stat_data) {
589 for (i = 0; i < kvm_debugfs_num_entries; i++)
590 kfree(kvm->debugfs_stat_data[i]);
591 kfree(kvm->debugfs_stat_data);
592 }
Janosch Frank536a6f82016-05-18 13:26:23 +0200593}
594
595static int kvm_create_vm_debugfs(struct kvm *kvm, int fd)
596{
597 char dir_name[ITOA_MAX_LEN * 2];
598 struct kvm_stat_data *stat_data;
599 struct kvm_stats_debugfs_item *p;
600
601 if (!debugfs_initialized())
602 return 0;
603
604 snprintf(dir_name, sizeof(dir_name), "%d-%d", task_pid_nr(current), fd);
Greg Kroah-Hartman929f45e2018-05-29 18:22:04 +0200605 kvm->debugfs_dentry = debugfs_create_dir(dir_name, kvm_debugfs_dir);
Janosch Frank536a6f82016-05-18 13:26:23 +0200606
607 kvm->debugfs_stat_data = kcalloc(kvm_debugfs_num_entries,
608 sizeof(*kvm->debugfs_stat_data),
Ben Gardonb12ce362019-02-11 11:02:49 -0800609 GFP_KERNEL_ACCOUNT);
Janosch Frank536a6f82016-05-18 13:26:23 +0200610 if (!kvm->debugfs_stat_data)
611 return -ENOMEM;
612
613 for (p = debugfs_entries; p->name; p++) {
Ben Gardonb12ce362019-02-11 11:02:49 -0800614 stat_data = kzalloc(sizeof(*stat_data), GFP_KERNEL_ACCOUNT);
Janosch Frank536a6f82016-05-18 13:26:23 +0200615 if (!stat_data)
616 return -ENOMEM;
617
618 stat_data->kvm = kvm;
619 stat_data->offset = p->offset;
620 kvm->debugfs_stat_data[p - debugfs_entries] = stat_data;
Greg Kroah-Hartman929f45e2018-05-29 18:22:04 +0200621 debugfs_create_file(p->name, 0644, kvm->debugfs_dentry,
622 stat_data, stat_fops_per_vm[p->kind]);
Janosch Frank536a6f82016-05-18 13:26:23 +0200623 }
624 return 0;
625}
626
Carsten Ottee08b9632012-01-04 10:25:20 +0100627static struct kvm *kvm_create_vm(unsigned long type)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800628{
Jan Kiszkad89f5ef2010-11-09 17:02:49 +0100629 int r, i;
630 struct kvm *kvm = kvm_arch_alloc_vm();
Avi Kivity6aa8b732006-12-10 02:21:36 -0800631
Jan Kiszkad89f5ef2010-11-09 17:02:49 +0100632 if (!kvm)
633 return ERR_PTR(-ENOMEM);
634
Paolo Bonzinie9ad4ec2016-03-21 10:15:25 +0100635 spin_lock_init(&kvm->mmu_lock);
Vegard Nossumf1f10072017-02-27 14:30:07 -0800636 mmgrab(current->mm);
Paolo Bonzinie9ad4ec2016-03-21 10:15:25 +0100637 kvm->mm = current->mm;
638 kvm_eventfd_init(kvm);
639 mutex_init(&kvm->lock);
640 mutex_init(&kvm->irq_lock);
641 mutex_init(&kvm->slots_lock);
Elena Reshetovae3736c32017-02-20 13:06:21 +0200642 refcount_set(&kvm->users_count, 1);
Paolo Bonzinie9ad4ec2016-03-21 10:15:25 +0100643 INIT_LIST_HEAD(&kvm->devices);
644
Carsten Ottee08b9632012-01-04 10:25:20 +0100645 r = kvm_arch_init_vm(kvm, type);
Jan Kiszkad89f5ef2010-11-09 17:02:49 +0100646 if (r)
Christian Borntraeger719d93c2014-01-16 13:44:20 +0100647 goto out_err_no_disable;
Alexander Graf10474ae2009-09-15 11:37:46 +0200648
649 r = hardware_enable_all();
650 if (r)
Christian Borntraeger719d93c2014-01-16 13:44:20 +0100651 goto out_err_no_disable;
Alexander Graf10474ae2009-09-15 11:37:46 +0200652
Paolo Bonzinic77dcac2014-08-06 14:24:45 +0200653#ifdef CONFIG_HAVE_KVM_IRQFD
Gleb Natapov136bdfe2009-08-24 11:54:23 +0300654 INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list);
Avi Kivity75858a82009-01-04 17:10:50 +0200655#endif
Avi Kivity6aa8b732006-12-10 02:21:36 -0800656
Alex Williamson1e702d92012-12-10 10:33:32 -0700657 BUILD_BUG_ON(KVM_MEM_SLOTS_NUM > SHRT_MAX);
658
Marcelo Tosatti46a26bf2009-12-23 14:35:16 -0200659 r = -ENOMEM;
Paolo Bonzinif481b062015-05-17 17:30:37 +0200660 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
Paolo Bonzini4bd518f2017-02-03 20:44:51 -0800661 struct kvm_memslots *slots = kvm_alloc_memslots();
662 if (!slots)
Paolo Bonzinif481b062015-05-17 17:30:37 +0200663 goto out_err_no_srcu;
Sean Christopherson0e329582019-02-05 13:01:17 -0800664 /* Generations must be different for each address space. */
Sean Christopherson164bf7e2019-02-05 13:01:18 -0800665 slots->generation = i;
Paolo Bonzini4bd518f2017-02-03 20:44:51 -0800666 rcu_assign_pointer(kvm->memslots[i], slots);
Paolo Bonzinif481b062015-05-17 17:30:37 +0200667 }
Paolo Bonzini00f034a2014-08-20 14:29:21 +0200668
Marcelo Tosattibc6678a2009-12-23 14:35:21 -0200669 if (init_srcu_struct(&kvm->srcu))
Christian Borntraeger719d93c2014-01-16 13:44:20 +0100670 goto out_err_no_srcu;
671 if (init_srcu_struct(&kvm->irq_srcu))
672 goto out_err_no_irq_srcu;
Marcelo Tosattie93f8a02009-12-23 14:35:24 -0200673 for (i = 0; i < KVM_NR_BUSES; i++) {
Christian Borntraeger4a12f952017-07-07 10:51:38 +0200674 rcu_assign_pointer(kvm->buses[i],
Ben Gardonb12ce362019-02-11 11:02:49 -0800675 kzalloc(sizeof(struct kvm_io_bus), GFP_KERNEL_ACCOUNT));
Jan Kiszka57e7fbe2010-11-09 12:42:12 +0100676 if (!kvm->buses[i])
Marcelo Tosattie93f8a02009-12-23 14:35:24 -0200677 goto out_err;
Marcelo Tosattie93f8a02009-12-23 14:35:24 -0200678 }
Mike Waychison74b5c5b2011-06-03 13:04:53 -0700679
Mike Waychison74b5c5b2011-06-03 13:04:53 -0700680 r = kvm_init_mmu_notifier(kvm);
681 if (r)
682 goto out_err;
683
Paolo Bonzini2f303b72013-09-25 13:53:07 +0200684 spin_lock(&kvm_lock);
Rusty Russell5e58cfe2007-07-23 17:08:21 +1000685 list_add(&kvm->vm_list, &vm_list);
Paolo Bonzini2f303b72013-09-25 13:53:07 +0200686 spin_unlock(&kvm_lock);
Jan Kiszkad89f5ef2010-11-09 17:02:49 +0100687
Peter Zijlstra2ecd9d22015-07-03 18:53:58 +0200688 preempt_notifier_inc();
689
Avi Kivityf17abe92007-02-21 19:28:04 +0200690 return kvm;
Alexander Graf10474ae2009-09-15 11:37:46 +0200691
692out_err:
Christian Borntraeger719d93c2014-01-16 13:44:20 +0100693 cleanup_srcu_struct(&kvm->irq_srcu);
694out_err_no_irq_srcu:
Jan Kiszka57e7fbe2010-11-09 12:42:12 +0100695 cleanup_srcu_struct(&kvm->srcu);
Christian Borntraeger719d93c2014-01-16 13:44:20 +0100696out_err_no_srcu:
Alexander Graf10474ae2009-09-15 11:37:46 +0200697 hardware_disable_all();
Christian Borntraeger719d93c2014-01-16 13:44:20 +0100698out_err_no_disable:
Christian Borntraeger021086e2017-09-13 14:17:22 +0200699 refcount_set(&kvm->users_count, 0);
Marcelo Tosattie93f8a02009-12-23 14:35:24 -0200700 for (i = 0; i < KVM_NR_BUSES; i++)
Paolo Bonzini3898da92017-08-02 17:55:54 +0200701 kfree(kvm_get_bus(kvm, i));
Paolo Bonzinif481b062015-05-17 17:30:37 +0200702 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++)
Paolo Bonzini3898da92017-08-02 17:55:54 +0200703 kvm_free_memslots(kvm, __kvm_memslots(kvm, i));
Jan Kiszkad89f5ef2010-11-09 17:02:49 +0100704 kvm_arch_free_vm(kvm);
Paolo Bonzinie9ad4ec2016-03-21 10:15:25 +0100705 mmdrop(current->mm);
Alexander Graf10474ae2009-09-15 11:37:46 +0200706 return ERR_PTR(r);
Avi Kivityf17abe92007-02-21 19:28:04 +0200707}
708
Scott Wood07f0a7b2013-04-25 14:11:23 +0000709static void kvm_destroy_devices(struct kvm *kvm)
710{
Geliang Tange6e3b5a2016-01-01 19:47:12 +0800711 struct kvm_device *dev, *tmp;
Scott Wood07f0a7b2013-04-25 14:11:23 +0000712
Christoffer Dalla28ebea2016-08-09 19:13:01 +0200713 /*
714 * We do not need to take the kvm->lock here, because nobody else
715 * has a reference to the struct kvm at this point and therefore
716 * cannot access the devices list anyhow.
717 */
Geliang Tange6e3b5a2016-01-01 19:47:12 +0800718 list_for_each_entry_safe(dev, tmp, &kvm->devices, vm_node) {
719 list_del(&dev->vm_node);
Scott Wood07f0a7b2013-04-25 14:11:23 +0000720 dev->ops->destroy(dev);
721 }
722}
723
Avi Kivityf17abe92007-02-21 19:28:04 +0200724static void kvm_destroy_vm(struct kvm *kvm)
725{
Marcelo Tosattie93f8a02009-12-23 14:35:24 -0200726 int i;
Avi Kivity6d4e4c42007-11-21 16:41:05 +0200727 struct mm_struct *mm = kvm->mm;
728
Claudio Imbrenda286de8f2017-07-12 17:56:44 +0200729 kvm_uevent_notify_change(KVM_EVENT_DESTROY_VM, kvm);
Janosch Frank536a6f82016-05-18 13:26:23 +0200730 kvm_destroy_vm_debugfs(kvm);
Sheng Yangad8ba2c2009-01-06 10:03:02 +0800731 kvm_arch_sync_events(kvm);
Paolo Bonzini2f303b72013-09-25 13:53:07 +0200732 spin_lock(&kvm_lock);
Avi Kivity133de902007-02-12 00:54:44 -0800733 list_del(&kvm->vm_list);
Paolo Bonzini2f303b72013-09-25 13:53:07 +0200734 spin_unlock(&kvm_lock);
Avi Kivity399ec802008-11-19 13:58:46 +0200735 kvm_free_irq_routing(kvm);
Peter Xudf630b82017-03-15 16:01:17 +0800736 for (i = 0; i < KVM_NR_BUSES; i++) {
Paolo Bonzini3898da92017-08-02 17:55:54 +0200737 struct kvm_io_bus *bus = kvm_get_bus(kvm, i);
Christian Borntraeger4a12f952017-07-07 10:51:38 +0200738
Christian Borntraeger4a12f952017-07-07 10:51:38 +0200739 if (bus)
740 kvm_io_bus_destroy(bus);
Peter Xudf630b82017-03-15 16:01:17 +0800741 kvm->buses[i] = NULL;
742 }
Avi Kivity980da6c2009-12-20 15:13:43 +0200743 kvm_coalesced_mmio_free(kvm);
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200744#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
745 mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm);
Gleb Natapovf00be0c2009-03-19 12:20:36 +0200746#else
Marcelo Tosatti2df72e92012-08-24 15:54:57 -0300747 kvm_arch_flush_shadow_all(kvm);
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200748#endif
Zhang Xiantaod19a9cd2007-11-18 18:43:45 +0800749 kvm_arch_destroy_vm(kvm);
Scott Wood07f0a7b2013-04-25 14:11:23 +0000750 kvm_destroy_devices(kvm);
Paolo Bonzinif481b062015-05-17 17:30:37 +0200751 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++)
Paolo Bonzini3898da92017-08-02 17:55:54 +0200752 kvm_free_memslots(kvm, __kvm_memslots(kvm, i));
Paolo Bonzini820b3fc2014-06-03 13:44:17 +0200753 cleanup_srcu_struct(&kvm->irq_srcu);
Jan Kiszkad89f5ef2010-11-09 17:02:49 +0100754 cleanup_srcu_struct(&kvm->srcu);
755 kvm_arch_free_vm(kvm);
Peter Zijlstra2ecd9d22015-07-03 18:53:58 +0200756 preempt_notifier_dec();
Alexander Graf10474ae2009-09-15 11:37:46 +0200757 hardware_disable_all();
Avi Kivity6d4e4c42007-11-21 16:41:05 +0200758 mmdrop(mm);
Avi Kivityf17abe92007-02-21 19:28:04 +0200759}
760
Izik Eidusd39f13b2008-03-30 16:01:25 +0300761void kvm_get_kvm(struct kvm *kvm)
762{
Elena Reshetovae3736c32017-02-20 13:06:21 +0200763 refcount_inc(&kvm->users_count);
Izik Eidusd39f13b2008-03-30 16:01:25 +0300764}
765EXPORT_SYMBOL_GPL(kvm_get_kvm);
766
767void kvm_put_kvm(struct kvm *kvm)
768{
Elena Reshetovae3736c32017-02-20 13:06:21 +0200769 if (refcount_dec_and_test(&kvm->users_count))
Izik Eidusd39f13b2008-03-30 16:01:25 +0300770 kvm_destroy_vm(kvm);
771}
772EXPORT_SYMBOL_GPL(kvm_put_kvm);
773
774
Avi Kivityf17abe92007-02-21 19:28:04 +0200775static int kvm_vm_release(struct inode *inode, struct file *filp)
776{
777 struct kvm *kvm = filp->private_data;
778
Gregory Haskins721eecbf2009-05-20 10:30:49 -0400779 kvm_irqfd_release(kvm);
780
Izik Eidusd39f13b2008-03-30 16:01:25 +0300781 kvm_put_kvm(kvm);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800782 return 0;
783}
784
Takuya Yoshikawa515a0122010-10-27 18:23:54 +0900785/*
786 * Allocation size is twice as large as the actual dirty bitmap size.
Takuya Yoshikawa93474b22012-03-01 19:34:45 +0900787 * See x86's kvm_vm_ioctl_get_dirty_log() why this is needed.
Takuya Yoshikawa515a0122010-10-27 18:23:54 +0900788 */
Takuya Yoshikawaa36a57b12010-10-27 18:22:19 +0900789static int kvm_create_dirty_bitmap(struct kvm_memory_slot *memslot)
790{
Takuya Yoshikawa515a0122010-10-27 18:23:54 +0900791 unsigned long dirty_bytes = 2 * kvm_dirty_bitmap_bytes(memslot);
Takuya Yoshikawaa36a57b12010-10-27 18:22:19 +0900792
Ben Gardonb12ce362019-02-11 11:02:49 -0800793 memslot->dirty_bitmap = kvzalloc(dirty_bytes, GFP_KERNEL_ACCOUNT);
Takuya Yoshikawaa36a57b12010-10-27 18:22:19 +0900794 if (!memslot->dirty_bitmap)
795 return -ENOMEM;
796
Takuya Yoshikawaa36a57b12010-10-27 18:22:19 +0900797 return 0;
798}
799
Xiao Guangrongbf3e05b2011-11-24 17:40:57 +0800800/*
Igor Mammedov0e60b072014-12-01 17:29:26 +0000801 * Insert memslot and re-sort memslots based on their GFN,
802 * so binary search could be used to lookup GFN.
803 * Sorting algorithm takes advantage of having initially
804 * sorted array and known changed memslot position.
Xiao Guangrongbf3e05b2011-11-24 17:40:57 +0800805 */
Paolo Bonzini5cc15022014-11-14 10:55:31 +0100806static void update_memslots(struct kvm_memslots *slots,
Wei Yang31fc4f92018-08-22 21:57:11 +0800807 struct kvm_memory_slot *new,
808 enum kvm_mr_change change)
Xiao Guangrongbf3e05b2011-11-24 17:40:57 +0800809{
Paolo Bonzini85931762014-11-14 10:22:07 +0100810 int id = new->id;
811 int i = slots->id_to_index[id];
Igor Mammedov063584d2014-11-13 23:00:13 +0000812 struct kvm_memory_slot *mslots = slots->memslots;
Xiao Guangrongf85e2cb2011-11-24 17:41:54 +0800813
Paolo Bonzini85931762014-11-14 10:22:07 +0100814 WARN_ON(mslots[i].id != id);
Wei Yang31fc4f92018-08-22 21:57:11 +0800815 switch (change) {
816 case KVM_MR_CREATE:
817 slots->used_slots++;
818 WARN_ON(mslots[i].npages || !new->npages);
819 break;
820 case KVM_MR_DELETE:
821 slots->used_slots--;
822 WARN_ON(new->npages || !mslots[i].npages);
823 break;
824 default:
825 break;
Igor Mammedov9c1a5d382014-12-01 17:29:27 +0000826 }
Igor Mammedov0e60b072014-12-01 17:29:26 +0000827
Igor Mammedov7f379cf2014-12-01 17:29:24 +0000828 while (i < KVM_MEM_SLOTS_NUM - 1 &&
Igor Mammedov0e60b072014-12-01 17:29:26 +0000829 new->base_gfn <= mslots[i + 1].base_gfn) {
830 if (!mslots[i + 1].npages)
831 break;
Igor Mammedov7f379cf2014-12-01 17:29:24 +0000832 mslots[i] = mslots[i + 1];
833 slots->id_to_index[mslots[i].id] = i;
834 i++;
835 }
Paolo Bonziniefbeec72014-12-27 18:01:00 +0100836
837 /*
838 * The ">=" is needed when creating a slot with base_gfn == 0,
839 * so that it moves before all those with base_gfn == npages == 0.
840 *
841 * On the other hand, if new->npages is zero, the above loop has
842 * already left i pointing to the beginning of the empty part of
843 * mslots, and the ">=" would move the hole backwards in this
844 * case---which is wrong. So skip the loop when deleting a slot.
845 */
846 if (new->npages) {
847 while (i > 0 &&
848 new->base_gfn >= mslots[i - 1].base_gfn) {
849 mslots[i] = mslots[i - 1];
850 slots->id_to_index[mslots[i].id] = i;
851 i--;
852 }
Paolo Bonzinidbaff302014-12-27 21:08:16 +0100853 } else
854 WARN_ON_ONCE(i != slots->used_slots);
Xiao Guangrongf85e2cb2011-11-24 17:41:54 +0800855
Paolo Bonzini85931762014-11-14 10:22:07 +0100856 mslots[i] = *new;
857 slots->id_to_index[mslots[i].id] = i;
Xiao Guangrongbf3e05b2011-11-24 17:40:57 +0800858}
859
Paolo Bonzini09170a42015-05-18 13:59:39 +0200860static int check_memory_region_flags(const struct kvm_userspace_memory_region *mem)
Xiao Guangronga50d64d2012-08-21 10:58:13 +0800861{
Xiao Guangrong4d8b81a2012-08-21 11:02:51 +0800862 u32 valid_flags = KVM_MEM_LOG_DIRTY_PAGES;
863
Christoffer Dall0f8a4de2014-08-26 14:00:37 +0200864#ifdef __KVM_HAVE_READONLY_MEM
Xiao Guangrong4d8b81a2012-08-21 11:02:51 +0800865 valid_flags |= KVM_MEM_READONLY;
866#endif
867
868 if (mem->flags & ~valid_flags)
Xiao Guangronga50d64d2012-08-21 10:58:13 +0800869 return -EINVAL;
870
871 return 0;
872}
873
Gleb Natapov7ec4fb42012-12-24 17:49:30 +0200874static struct kvm_memslots *install_new_memslots(struct kvm *kvm,
Paolo Bonzinif481b062015-05-17 17:30:37 +0200875 int as_id, struct kvm_memslots *slots)
Gleb Natapov7ec4fb42012-12-24 17:49:30 +0200876{
Paolo Bonzinif481b062015-05-17 17:30:37 +0200877 struct kvm_memslots *old_memslots = __kvm_memslots(kvm, as_id);
Sean Christopherson361209e2019-02-05 13:01:14 -0800878 u64 gen = old_memslots->generation;
Gleb Natapov7ec4fb42012-12-24 17:49:30 +0200879
Sean Christopherson361209e2019-02-05 13:01:14 -0800880 WARN_ON(gen & KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS);
881 slots->generation = gen | KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS;
David Matlackee3d1572014-08-18 15:46:06 -0700882
Paolo Bonzinif481b062015-05-17 17:30:37 +0200883 rcu_assign_pointer(kvm->memslots[as_id], slots);
Gleb Natapov7ec4fb42012-12-24 17:49:30 +0200884 synchronize_srcu_expedited(&kvm->srcu);
Takuya Yoshikawae59dbe02013-07-04 13:40:29 +0900885
David Matlackee3d1572014-08-18 15:46:06 -0700886 /*
Sean Christopherson361209e2019-02-05 13:01:14 -0800887 * Increment the new memslot generation a second time, dropping the
888 * update in-progress flag and incrementing then generation based on
889 * the number of address spaces. This provides a unique and easily
890 * identifiable generation number while the memslots are in flux.
891 */
892 gen = slots->generation & ~KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS;
893
894 /*
Paolo Bonzini4bd518f2017-02-03 20:44:51 -0800895 * Generations must be unique even across address spaces. We do not need
896 * a global counter for that, instead the generation space is evenly split
897 * across address spaces. For example, with two address spaces, address
Sean Christopherson164bf7e2019-02-05 13:01:18 -0800898 * space 0 will use generations 0, 2, 4, ... while address space 1 will
899 * use generations 1, 3, 5, ...
David Matlackee3d1572014-08-18 15:46:06 -0700900 */
Sean Christopherson164bf7e2019-02-05 13:01:18 -0800901 gen += KVM_ADDRESS_SPACE_NUM;
David Matlackee3d1572014-08-18 15:46:06 -0700902
Sean Christopherson15248252019-02-05 12:54:17 -0800903 kvm_arch_memslots_updated(kvm, gen);
904
905 slots->generation = gen;
Takuya Yoshikawae59dbe02013-07-04 13:40:29 +0900906
907 return old_memslots;
Gleb Natapov7ec4fb42012-12-24 17:49:30 +0200908}
909
Avi Kivity6aa8b732006-12-10 02:21:36 -0800910/*
Avi Kivity6aa8b732006-12-10 02:21:36 -0800911 * Allocate some memory and give it an address in the guest physical address
912 * space.
913 *
914 * Discontiguous memory is allowed, mostly for framebuffers.
Sheng Yangf78e0e22007-10-29 09:40:42 +0800915 *
Dominik Dingel02d5d552014-10-27 16:22:56 +0100916 * Must be called holding kvm->slots_lock for write.
Avi Kivity6aa8b732006-12-10 02:21:36 -0800917 */
Sheng Yangf78e0e22007-10-29 09:40:42 +0800918int __kvm_set_memory_region(struct kvm *kvm,
Paolo Bonzini09170a42015-05-18 13:59:39 +0200919 const struct kvm_userspace_memory_region *mem)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800920{
Avi Kivity8234b222010-12-27 12:08:45 +0200921 int r;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800922 gfn_t base_gfn;
Heiko Carstens28bcb112009-09-03 17:35:35 +0200923 unsigned long npages;
Takuya Yoshikawaa843fac2013-01-11 18:27:43 +0900924 struct kvm_memory_slot *slot;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800925 struct kvm_memory_slot old, new;
Alex Williamsonb7f69c52012-12-10 10:33:03 -0700926 struct kvm_memslots *slots = NULL, *old_memslots;
Paolo Bonzinif481b062015-05-17 17:30:37 +0200927 int as_id, id;
Takuya Yoshikawaf64c0392013-01-29 11:00:07 +0900928 enum kvm_mr_change change;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800929
Xiao Guangronga50d64d2012-08-21 10:58:13 +0800930 r = check_memory_region_flags(mem);
931 if (r)
932 goto out;
933
Avi Kivity6aa8b732006-12-10 02:21:36 -0800934 r = -EINVAL;
Paolo Bonzinif481b062015-05-17 17:30:37 +0200935 as_id = mem->slot >> 16;
936 id = (u16)mem->slot;
937
Avi Kivity6aa8b732006-12-10 02:21:36 -0800938 /* General sanity checks */
939 if (mem->memory_size & (PAGE_SIZE - 1))
940 goto out;
941 if (mem->guest_phys_addr & (PAGE_SIZE - 1))
942 goto out;
Takuya Yoshikawafa3d3152011-05-07 16:35:38 +0900943 /* We can read the guest memory with __xxx_user() later on. */
Paolo Bonzinif481b062015-05-17 17:30:37 +0200944 if ((id < KVM_USER_MEM_SLOTS) &&
Takuya Yoshikawafa3d3152011-05-07 16:35:38 +0900945 ((mem->userspace_addr & (PAGE_SIZE - 1)) ||
Linus Torvalds96d4f262019-01-03 18:57:57 -0800946 !access_ok((void __user *)(unsigned long)mem->userspace_addr,
Heiko Carstens9e3bb6b2011-05-24 07:51:27 +0200947 mem->memory_size)))
Hollis Blanchard78749802008-11-07 13:32:12 -0600948 goto out;
Paolo Bonzinif481b062015-05-17 17:30:37 +0200949 if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_MEM_SLOTS_NUM)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800950 goto out;
951 if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
952 goto out;
953
Paolo Bonzinif481b062015-05-17 17:30:37 +0200954 slot = id_to_memslot(__kvm_memslots(kvm, as_id), id);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800955 base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
956 npages = mem->memory_size >> PAGE_SHIFT;
957
Takuya Yoshikawa660c22c2010-04-13 22:47:24 +0900958 if (npages > KVM_MEM_MAX_NR_PAGES)
959 goto out;
960
Takuya Yoshikawaa843fac2013-01-11 18:27:43 +0900961 new = old = *slot;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800962
Paolo Bonzinif481b062015-05-17 17:30:37 +0200963 new.id = id;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800964 new.base_gfn = base_gfn;
965 new.npages = npages;
966 new.flags = mem->flags;
967
Takuya Yoshikawaf64c0392013-01-29 11:00:07 +0900968 if (npages) {
969 if (!old.npages)
970 change = KVM_MR_CREATE;
971 else { /* Modify an existing slot. */
972 if ((mem->userspace_addr != old.userspace_addr) ||
Takuya Yoshikawa75d61fb2013-01-30 19:40:41 +0900973 (npages != old.npages) ||
974 ((new.flags ^ old.flags) & KVM_MEM_READONLY))
Takuya Yoshikawaf64c0392013-01-29 11:00:07 +0900975 goto out;
976
977 if (base_gfn != old.base_gfn)
978 change = KVM_MR_MOVE;
979 else if (new.flags != old.flags)
980 change = KVM_MR_FLAGS_ONLY;
981 else { /* Nothing to change. */
982 r = 0;
983 goto out;
984 }
985 }
Paolo Bonzini09170a42015-05-18 13:59:39 +0200986 } else {
987 if (!old.npages)
988 goto out;
989
Takuya Yoshikawaf64c0392013-01-29 11:00:07 +0900990 change = KVM_MR_DELETE;
Paolo Bonzini09170a42015-05-18 13:59:39 +0200991 new.base_gfn = 0;
992 new.flags = 0;
993 }
Avi Kivity6aa8b732006-12-10 02:21:36 -0800994
Takuya Yoshikawaf64c0392013-01-29 11:00:07 +0900995 if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) {
Takuya Yoshikawa0a706be2013-01-11 18:26:55 +0900996 /* Check for overlaps */
997 r = -EEXIST;
Paolo Bonzinif481b062015-05-17 17:30:37 +0200998 kvm_for_each_memslot(slot, __kvm_memslots(kvm, as_id)) {
Wanpeng Lib28676b2018-02-13 15:36:00 +0100999 if (slot->id == id)
Takuya Yoshikawa0a706be2013-01-11 18:26:55 +09001000 continue;
1001 if (!((base_gfn + npages <= slot->base_gfn) ||
1002 (base_gfn >= slot->base_gfn + slot->npages)))
1003 goto out;
1004 }
Avi Kivity6aa8b732006-12-10 02:21:36 -08001005 }
Avi Kivity6aa8b732006-12-10 02:21:36 -08001006
Avi Kivity6aa8b732006-12-10 02:21:36 -08001007 /* Free page dirty bitmap if unneeded */
1008 if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
Al Viro8b6d44c2007-02-09 16:38:40 +00001009 new.dirty_bitmap = NULL;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001010
1011 r = -ENOMEM;
Takuya Yoshikawaf64c0392013-01-29 11:00:07 +09001012 if (change == KVM_MR_CREATE) {
Takuya Yoshikawa189a2f72012-02-08 13:01:09 +09001013 new.userspace_addr = mem->userspace_addr;
Takuya Yoshikawad89cc612012-08-01 18:03:28 +09001014
Aneesh Kumar K.V55870272013-10-07 22:18:00 +05301015 if (kvm_arch_create_memslot(kvm, &new, npages))
Takuya Yoshikawadb3fe4e2012-02-08 13:02:18 +09001016 goto out_free;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001017 }
Joerg Roedelec04b262009-06-19 15:16:23 +02001018
Avi Kivity6aa8b732006-12-10 02:21:36 -08001019 /* Allocate page dirty bitmap if needed */
1020 if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
Takuya Yoshikawaa36a57b12010-10-27 18:22:19 +09001021 if (kvm_create_dirty_bitmap(&new) < 0)
Sheng Yangf78e0e22007-10-29 09:40:42 +08001022 goto out_free;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001023 }
1024
Ben Gardonb12ce362019-02-11 11:02:49 -08001025 slots = kvzalloc(sizeof(struct kvm_memslots), GFP_KERNEL_ACCOUNT);
Paolo Bonzinif2a81032014-11-14 10:46:45 +01001026 if (!slots)
1027 goto out_free;
Paolo Bonzinif481b062015-05-17 17:30:37 +02001028 memcpy(slots, __kvm_memslots(kvm, as_id), sizeof(struct kvm_memslots));
Paolo Bonzinif2a81032014-11-14 10:46:45 +01001029
Takuya Yoshikawaf64c0392013-01-29 11:00:07 +09001030 if ((change == KVM_MR_DELETE) || (change == KVM_MR_MOVE)) {
Paolo Bonzinif481b062015-05-17 17:30:37 +02001031 slot = id_to_memslot(slots, id);
Xiao Guangrong28a37542011-11-24 19:04:35 +08001032 slot->flags |= KVM_MEMSLOT_INVALID;
1033
Paolo Bonzinif481b062015-05-17 17:30:37 +02001034 old_memslots = install_new_memslots(kvm, as_id, slots);
Marcelo Tosattibc6678a2009-12-23 14:35:21 -02001035
Marcelo Tosatti12d6e752012-08-24 15:54:58 -03001036 /* From this point no new shadow pages pointing to a deleted,
1037 * or moved, memslot will be created.
Marcelo Tosattibc6678a2009-12-23 14:35:21 -02001038 *
1039 * validation of sp->gfn happens in:
Xiubo Lib7d409d2015-02-26 14:58:24 +08001040 * - gfn_to_hva (kvm_read_guest, gfn_to_pfn)
1041 * - kvm_is_visible_gfn (mmu_check_roots)
Marcelo Tosattibc6678a2009-12-23 14:35:21 -02001042 */
Marcelo Tosatti2df72e92012-08-24 15:54:57 -03001043 kvm_arch_flush_shadow_memslot(kvm, slot);
Paolo Bonzinif2a81032014-11-14 10:46:45 +01001044
1045 /*
1046 * We can re-use the old_memslots from above, the only difference
1047 * from the currently installed memslots is the invalid flag. This
1048 * will get overwritten by update_memslots anyway.
1049 */
Alex Williamsonb7f69c52012-12-10 10:33:03 -07001050 slots = old_memslots;
Marcelo Tosattibc6678a2009-12-23 14:35:21 -02001051 }
Marcelo Tosatti34d4cb82008-07-10 20:49:31 -03001052
Takuya Yoshikawa7b6195a2013-02-27 19:44:34 +09001053 r = kvm_arch_prepare_memory_region(kvm, &new, mem, change);
Marcelo Tosattif7784b82009-12-23 14:35:18 -02001054 if (r)
Alex Williamsonb7f69c52012-12-10 10:33:03 -07001055 goto out_slots;
Marcelo Tosattif7784b82009-12-23 14:35:18 -02001056
Paolo Bonzinia47d2b02015-05-17 11:41:37 +02001057 /* actual memory is freed via old in kvm_free_memslot below */
Takuya Yoshikawaf64c0392013-01-29 11:00:07 +09001058 if (change == KVM_MR_DELETE) {
Marcelo Tosattibc6678a2009-12-23 14:35:21 -02001059 new.dirty_bitmap = NULL;
Takuya Yoshikawadb3fe4e2012-02-08 13:02:18 +09001060 memset(&new.arch, 0, sizeof(new.arch));
Marcelo Tosattibc6678a2009-12-23 14:35:21 -02001061 }
1062
Wei Yang31fc4f92018-08-22 21:57:11 +08001063 update_memslots(slots, &new, change);
Paolo Bonzinif481b062015-05-17 17:30:37 +02001064 old_memslots = install_new_memslots(kvm, as_id, slots);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001065
Paolo Bonzinif36f3f22015-05-18 13:20:23 +02001066 kvm_arch_commit_memory_region(kvm, mem, &old, &new, change);
Zhang Xiantao3ad82a72007-11-20 13:11:38 +08001067
Paolo Bonzinia47d2b02015-05-17 11:41:37 +02001068 kvm_free_memslot(kvm, &old, &new);
Igor Mammedov74496132015-03-20 12:21:37 +00001069 kvfree(old_memslots);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001070 return 0;
1071
Alex Williamsone40f1932012-12-10 10:32:57 -07001072out_slots:
Igor Mammedov74496132015-03-20 12:21:37 +00001073 kvfree(slots);
Sheng Yangf78e0e22007-10-29 09:40:42 +08001074out_free:
Paolo Bonzinia47d2b02015-05-17 11:41:37 +02001075 kvm_free_memslot(kvm, &new, &old);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001076out:
1077 return r;
Izik Eidus210c7c42007-10-24 23:52:57 +02001078}
Sheng Yangf78e0e22007-10-29 09:40:42 +08001079EXPORT_SYMBOL_GPL(__kvm_set_memory_region);
1080
1081int kvm_set_memory_region(struct kvm *kvm,
Paolo Bonzini09170a42015-05-18 13:59:39 +02001082 const struct kvm_userspace_memory_region *mem)
Sheng Yangf78e0e22007-10-29 09:40:42 +08001083{
1084 int r;
1085
Marcelo Tosatti79fac952009-12-23 14:35:26 -02001086 mutex_lock(&kvm->slots_lock);
Takuya Yoshikawa47ae31e2013-02-27 19:43:00 +09001087 r = __kvm_set_memory_region(kvm, mem);
Marcelo Tosatti79fac952009-12-23 14:35:26 -02001088 mutex_unlock(&kvm->slots_lock);
Sheng Yangf78e0e22007-10-29 09:40:42 +08001089 return r;
1090}
Izik Eidus210c7c42007-10-24 23:52:57 +02001091EXPORT_SYMBOL_GPL(kvm_set_memory_region);
1092
Stephen Hemminger79408762013-12-29 12:12:29 -08001093static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
1094 struct kvm_userspace_memory_region *mem)
Izik Eidus210c7c42007-10-24 23:52:57 +02001095{
Paolo Bonzinif481b062015-05-17 17:30:37 +02001096 if ((u16)mem->slot >= KVM_USER_MEM_SLOTS)
Izik Eiduse0d62c72007-10-24 23:57:46 +02001097 return -EINVAL;
Paolo Bonzini09170a42015-05-18 13:59:39 +02001098
Takuya Yoshikawa47ae31e2013-02-27 19:43:00 +09001099 return kvm_set_memory_region(kvm, mem);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001100}
1101
Zhang Xiantao5bb064d2007-11-18 20:29:43 +08001102int kvm_get_dirty_log(struct kvm *kvm,
1103 struct kvm_dirty_log *log, int *is_dirty)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001104{
Paolo Bonzini9f6b8022015-05-17 16:20:07 +02001105 struct kvm_memslots *slots;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001106 struct kvm_memory_slot *memslot;
Markus Elfring843574a2017-01-22 17:41:07 +01001107 int i, as_id, id;
Takuya Yoshikawa87bf6e72010-04-12 19:35:35 +09001108 unsigned long n;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001109 unsigned long any = 0;
1110
Paolo Bonzinif481b062015-05-17 17:30:37 +02001111 as_id = log->slot >> 16;
1112 id = (u16)log->slot;
1113 if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS)
Markus Elfring843574a2017-01-22 17:41:07 +01001114 return -EINVAL;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001115
Paolo Bonzinif481b062015-05-17 17:30:37 +02001116 slots = __kvm_memslots(kvm, as_id);
1117 memslot = id_to_memslot(slots, id);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001118 if (!memslot->dirty_bitmap)
Markus Elfring843574a2017-01-22 17:41:07 +01001119 return -ENOENT;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001120
Takuya Yoshikawa87bf6e72010-04-12 19:35:35 +09001121 n = kvm_dirty_bitmap_bytes(memslot);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001122
Uri Lublincd1a4a92007-02-22 16:43:09 +02001123 for (i = 0; !any && i < n/sizeof(long); ++i)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001124 any = memslot->dirty_bitmap[i];
1125
Avi Kivity6aa8b732006-12-10 02:21:36 -08001126 if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
Markus Elfring843574a2017-01-22 17:41:07 +01001127 return -EFAULT;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001128
Zhang Xiantao5bb064d2007-11-18 20:29:43 +08001129 if (any)
1130 *is_dirty = 1;
Markus Elfring843574a2017-01-22 17:41:07 +01001131 return 0;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001132}
Aneesh Kumar K.V2ba9f0d2013-10-07 22:17:59 +05301133EXPORT_SYMBOL_GPL(kvm_get_dirty_log);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001134
Mario Smarduchba0513b2015-01-15 15:58:53 -08001135#ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
1136/**
1137 * kvm_get_dirty_log_protect - get a snapshot of dirty pages, and if any pages
Paolo Bonzini2a31b9d2018-10-23 02:36:47 +02001138 * and reenable dirty page tracking for the corresponding pages.
Mario Smarduchba0513b2015-01-15 15:58:53 -08001139 * @kvm: pointer to kvm instance
1140 * @log: slot id and address to which we copy the log
1141 * @is_dirty: flag set if any page is dirty
1142 *
1143 * We need to keep it in mind that VCPU threads can write to the bitmap
1144 * concurrently. So, to avoid losing track of dirty pages we keep the
1145 * following order:
1146 *
1147 * 1. Take a snapshot of the bit and clear it if needed.
1148 * 2. Write protect the corresponding page.
1149 * 3. Copy the snapshot to the userspace.
1150 * 4. Upon return caller flushes TLB's if needed.
1151 *
1152 * Between 2 and 4, the guest may write to the page using the remaining TLB
1153 * entry. This is not a problem because the page is reported dirty using
1154 * the snapshot taken before and step 4 ensures that writes done after
1155 * exiting to userspace will be logged for the next call.
1156 *
1157 */
1158int kvm_get_dirty_log_protect(struct kvm *kvm,
Paolo Bonzini8fe65a82018-10-23 02:18:42 +02001159 struct kvm_dirty_log *log, bool *flush)
Mario Smarduchba0513b2015-01-15 15:58:53 -08001160{
Paolo Bonzini9f6b8022015-05-17 16:20:07 +02001161 struct kvm_memslots *slots;
Mario Smarduchba0513b2015-01-15 15:58:53 -08001162 struct kvm_memory_slot *memslot;
Markus Elfring58d6db32017-01-22 17:30:16 +01001163 int i, as_id, id;
Mario Smarduchba0513b2015-01-15 15:58:53 -08001164 unsigned long n;
1165 unsigned long *dirty_bitmap;
1166 unsigned long *dirty_bitmap_buffer;
1167
Paolo Bonzinif481b062015-05-17 17:30:37 +02001168 as_id = log->slot >> 16;
1169 id = (u16)log->slot;
1170 if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS)
Markus Elfring58d6db32017-01-22 17:30:16 +01001171 return -EINVAL;
Mario Smarduchba0513b2015-01-15 15:58:53 -08001172
Paolo Bonzinif481b062015-05-17 17:30:37 +02001173 slots = __kvm_memslots(kvm, as_id);
1174 memslot = id_to_memslot(slots, id);
Mario Smarduchba0513b2015-01-15 15:58:53 -08001175
1176 dirty_bitmap = memslot->dirty_bitmap;
Mario Smarduchba0513b2015-01-15 15:58:53 -08001177 if (!dirty_bitmap)
Markus Elfring58d6db32017-01-22 17:30:16 +01001178 return -ENOENT;
Mario Smarduchba0513b2015-01-15 15:58:53 -08001179
1180 n = kvm_dirty_bitmap_bytes(memslot);
Paolo Bonzini8fe65a82018-10-23 02:18:42 +02001181 *flush = false;
Paolo Bonzini2a31b9d2018-10-23 02:36:47 +02001182 if (kvm->manual_dirty_log_protect) {
1183 /*
1184 * Unlike kvm_get_dirty_log, we always return false in *flush,
1185 * because no flush is needed until KVM_CLEAR_DIRTY_LOG. There
1186 * is some code duplication between this function and
1187 * kvm_get_dirty_log, but hopefully all architecture
1188 * transition to kvm_get_dirty_log_protect and kvm_get_dirty_log
1189 * can be eliminated.
1190 */
1191 dirty_bitmap_buffer = dirty_bitmap;
1192 } else {
1193 dirty_bitmap_buffer = kvm_second_dirty_bitmap(memslot);
1194 memset(dirty_bitmap_buffer, 0, n);
Mario Smarduchba0513b2015-01-15 15:58:53 -08001195
Paolo Bonzini2a31b9d2018-10-23 02:36:47 +02001196 spin_lock(&kvm->mmu_lock);
1197 for (i = 0; i < n / sizeof(long); i++) {
1198 unsigned long mask;
1199 gfn_t offset;
Mario Smarduchba0513b2015-01-15 15:58:53 -08001200
Paolo Bonzini2a31b9d2018-10-23 02:36:47 +02001201 if (!dirty_bitmap[i])
1202 continue;
Mario Smarduchba0513b2015-01-15 15:58:53 -08001203
Paolo Bonzini2a31b9d2018-10-23 02:36:47 +02001204 *flush = true;
1205 mask = xchg(&dirty_bitmap[i], 0);
1206 dirty_bitmap_buffer[i] = mask;
Mario Smarduchba0513b2015-01-15 15:58:53 -08001207
Paolo Bonzini2a31b9d2018-10-23 02:36:47 +02001208 if (mask) {
1209 offset = i * BITS_PER_LONG;
1210 kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot,
1211 offset, mask);
1212 }
Takuya Yoshikawa58d29302015-03-17 16:19:58 +09001213 }
Paolo Bonzini2a31b9d2018-10-23 02:36:47 +02001214 spin_unlock(&kvm->mmu_lock);
Mario Smarduchba0513b2015-01-15 15:58:53 -08001215 }
1216
Mario Smarduchba0513b2015-01-15 15:58:53 -08001217 if (copy_to_user(log->dirty_bitmap, dirty_bitmap_buffer, n))
Markus Elfring58d6db32017-01-22 17:30:16 +01001218 return -EFAULT;
1219 return 0;
Mario Smarduchba0513b2015-01-15 15:58:53 -08001220}
1221EXPORT_SYMBOL_GPL(kvm_get_dirty_log_protect);
Paolo Bonzini2a31b9d2018-10-23 02:36:47 +02001222
1223/**
1224 * kvm_clear_dirty_log_protect - clear dirty bits in the bitmap
1225 * and reenable dirty page tracking for the corresponding pages.
1226 * @kvm: pointer to kvm instance
1227 * @log: slot id and address from which to fetch the bitmap of dirty pages
1228 */
1229int kvm_clear_dirty_log_protect(struct kvm *kvm,
1230 struct kvm_clear_dirty_log *log, bool *flush)
1231{
1232 struct kvm_memslots *slots;
1233 struct kvm_memory_slot *memslot;
Tomas Bortoli98938aa2019-01-02 18:29:37 +01001234 int as_id, id;
Paolo Bonzini2a31b9d2018-10-23 02:36:47 +02001235 gfn_t offset;
Tomas Bortoli98938aa2019-01-02 18:29:37 +01001236 unsigned long i, n;
Paolo Bonzini2a31b9d2018-10-23 02:36:47 +02001237 unsigned long *dirty_bitmap;
1238 unsigned long *dirty_bitmap_buffer;
1239
1240 as_id = log->slot >> 16;
1241 id = (u16)log->slot;
1242 if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS)
1243 return -EINVAL;
1244
1245 if ((log->first_page & 63) || (log->num_pages & 63))
1246 return -EINVAL;
1247
1248 slots = __kvm_memslots(kvm, as_id);
1249 memslot = id_to_memslot(slots, id);
1250
1251 dirty_bitmap = memslot->dirty_bitmap;
1252 if (!dirty_bitmap)
1253 return -ENOENT;
1254
1255 n = kvm_dirty_bitmap_bytes(memslot);
Tomas Bortoli98938aa2019-01-02 18:29:37 +01001256
1257 if (log->first_page > memslot->npages ||
1258 log->num_pages > memslot->npages - log->first_page)
1259 return -EINVAL;
1260
Paolo Bonzini2a31b9d2018-10-23 02:36:47 +02001261 *flush = false;
1262 dirty_bitmap_buffer = kvm_second_dirty_bitmap(memslot);
1263 if (copy_from_user(dirty_bitmap_buffer, log->dirty_bitmap, n))
1264 return -EFAULT;
1265
1266 spin_lock(&kvm->mmu_lock);
1267 for (offset = log->first_page,
1268 i = offset / BITS_PER_LONG, n = log->num_pages / BITS_PER_LONG; n--;
1269 i++, offset += BITS_PER_LONG) {
1270 unsigned long mask = *dirty_bitmap_buffer++;
1271 atomic_long_t *p = (atomic_long_t *) &dirty_bitmap[i];
1272 if (!mask)
1273 continue;
1274
1275 mask &= atomic_long_fetch_andnot(mask, p);
1276
1277 /*
1278 * mask contains the bits that really have been cleared. This
1279 * never includes any bits beyond the length of the memslot (if
1280 * the length is not aligned to 64 pages), therefore it is not
1281 * a problem if userspace sets them in log->dirty_bitmap.
1282 */
1283 if (mask) {
1284 *flush = true;
1285 kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot,
1286 offset, mask);
1287 }
1288 }
1289 spin_unlock(&kvm->mmu_lock);
1290
1291 return 0;
1292}
1293EXPORT_SYMBOL_GPL(kvm_clear_dirty_log_protect);
Mario Smarduchba0513b2015-01-15 15:58:53 -08001294#endif
1295
Takuya Yoshikawadb3fe4e2012-02-08 13:02:18 +09001296bool kvm_largepages_enabled(void)
1297{
1298 return largepages_enabled;
1299}
1300
Marcelo Tosatti54dee992009-06-11 12:07:44 -03001301void kvm_disable_largepages(void)
1302{
1303 largepages_enabled = false;
1304}
1305EXPORT_SYMBOL_GPL(kvm_disable_largepages);
1306
Gleb Natapov49c77542010-10-18 15:22:23 +02001307struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
1308{
1309 return __gfn_to_memslot(kvm_memslots(kvm), gfn);
1310}
Avi Kivitya1f4d3952010-06-21 11:44:20 +03001311EXPORT_SYMBOL_GPL(gfn_to_memslot);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001312
Paolo Bonzini8e734852015-05-17 13:58:53 +02001313struct kvm_memory_slot *kvm_vcpu_gfn_to_memslot(struct kvm_vcpu *vcpu, gfn_t gfn)
1314{
1315 return __gfn_to_memslot(kvm_vcpu_memslots(vcpu), gfn);
1316}
1317
Yaowei Bai33e94152015-11-14 11:21:06 +08001318bool kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
Izik Eiduse0d62c72007-10-24 23:57:46 +02001319{
Xiao Guangrongbf3e05b2011-11-24 17:40:57 +08001320 struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn);
Izik Eiduse0d62c72007-10-24 23:57:46 +02001321
Alex Williamsonbbacc0c2012-12-10 10:33:09 -07001322 if (!memslot || memslot->id >= KVM_USER_MEM_SLOTS ||
Xiao Guangrongbf3e05b2011-11-24 17:40:57 +08001323 memslot->flags & KVM_MEMSLOT_INVALID)
Yaowei Bai33e94152015-11-14 11:21:06 +08001324 return false;
Izik Eiduse0d62c72007-10-24 23:57:46 +02001325
Yaowei Bai33e94152015-11-14 11:21:06 +08001326 return true;
Izik Eiduse0d62c72007-10-24 23:57:46 +02001327}
1328EXPORT_SYMBOL_GPL(kvm_is_visible_gfn);
1329
Joerg Roedel8f0b1ab2010-01-28 12:37:56 +01001330unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn)
1331{
1332 struct vm_area_struct *vma;
1333 unsigned long addr, size;
1334
1335 size = PAGE_SIZE;
1336
1337 addr = gfn_to_hva(kvm, gfn);
1338 if (kvm_is_error_hva(addr))
1339 return PAGE_SIZE;
1340
1341 down_read(&current->mm->mmap_sem);
1342 vma = find_vma(current->mm, addr);
1343 if (!vma)
1344 goto out;
1345
1346 size = vma_kernel_pagesize(vma);
1347
1348out:
1349 up_read(&current->mm->mmap_sem);
1350
1351 return size;
1352}
1353
Xiao Guangrong4d8b81a2012-08-21 11:02:51 +08001354static bool memslot_is_readonly(struct kvm_memory_slot *slot)
1355{
1356 return slot->flags & KVM_MEM_READONLY;
1357}
1358
Xiao Guangrong4d8b81a2012-08-21 11:02:51 +08001359static unsigned long __gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
1360 gfn_t *nr_pages, bool write)
Izik Eidus539cb662007-11-11 22:05:04 +02001361{
Marcelo Tosattibc6678a2009-12-23 14:35:21 -02001362 if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
Xiao Guangrongca3a4902012-08-21 11:01:50 +08001363 return KVM_HVA_ERR_BAD;
Xiao Guangrong48987782010-08-22 19:11:43 +08001364
Xiao Guangrong4d8b81a2012-08-21 11:02:51 +08001365 if (memslot_is_readonly(slot) && write)
1366 return KVM_HVA_ERR_RO_BAD;
Xiao Guangrong48987782010-08-22 19:11:43 +08001367
1368 if (nr_pages)
1369 *nr_pages = slot->npages - (gfn - slot->base_gfn);
1370
Xiao Guangrong4d8b81a2012-08-21 11:02:51 +08001371 return __gfn_to_hva_memslot(slot, gfn);
Izik Eidus539cb662007-11-11 22:05:04 +02001372}
Xiao Guangrong48987782010-08-22 19:11:43 +08001373
Xiao Guangrong4d8b81a2012-08-21 11:02:51 +08001374static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
1375 gfn_t *nr_pages)
1376{
1377 return __gfn_to_hva_many(slot, gfn, nr_pages, true);
1378}
1379
1380unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot,
Stephen Hemminger79408762013-12-29 12:12:29 -08001381 gfn_t gfn)
Xiao Guangrong4d8b81a2012-08-21 11:02:51 +08001382{
1383 return gfn_to_hva_many(slot, gfn, NULL);
1384}
1385EXPORT_SYMBOL_GPL(gfn_to_hva_memslot);
1386
Xiao Guangrong48987782010-08-22 19:11:43 +08001387unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)
1388{
Gleb Natapov49c77542010-10-18 15:22:23 +02001389 return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL);
Xiao Guangrong48987782010-08-22 19:11:43 +08001390}
Sheng Yang0d150292008-04-25 21:44:50 +08001391EXPORT_SYMBOL_GPL(gfn_to_hva);
Izik Eidus539cb662007-11-11 22:05:04 +02001392
Paolo Bonzini8e734852015-05-17 13:58:53 +02001393unsigned long kvm_vcpu_gfn_to_hva(struct kvm_vcpu *vcpu, gfn_t gfn)
1394{
1395 return gfn_to_hva_many(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn, NULL);
1396}
1397EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_hva);
1398
Xiao Guangrong86ab8cf2012-08-21 10:59:53 +08001399/*
Wei Yang970c0d42018-10-09 10:41:15 +08001400 * Return the hva of a @gfn and the R/W attribute if possible.
1401 *
1402 * @slot: the kvm_memory_slot which contains @gfn
1403 * @gfn: the gfn to be translated
1404 * @writable: used to return the read/write attribute of the @slot if the hva
1405 * is valid and @writable is not NULL
Xiao Guangrong86ab8cf2012-08-21 10:59:53 +08001406 */
Christoffer Dall64d83122014-08-19 12:15:00 +02001407unsigned long gfn_to_hva_memslot_prot(struct kvm_memory_slot *slot,
1408 gfn_t gfn, bool *writable)
Gleb Natapov80300892010-10-19 18:13:41 +02001409{
Gleb Natapova2ac07f2013-10-01 19:58:36 +03001410 unsigned long hva = __gfn_to_hva_many(slot, gfn, NULL, false);
1411
1412 if (!kvm_is_error_hva(hva) && writable)
Paolo Bonziniba6a3542013-09-09 13:52:33 +02001413 *writable = !memslot_is_readonly(slot);
1414
Gleb Natapova2ac07f2013-10-01 19:58:36 +03001415 return hva;
Xiao Guangrong86ab8cf2012-08-21 10:59:53 +08001416}
1417
Christoffer Dall64d83122014-08-19 12:15:00 +02001418unsigned long gfn_to_hva_prot(struct kvm *kvm, gfn_t gfn, bool *writable)
1419{
1420 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
1421
1422 return gfn_to_hva_memslot_prot(slot, gfn, writable);
1423}
1424
Paolo Bonzini8e734852015-05-17 13:58:53 +02001425unsigned long kvm_vcpu_gfn_to_hva_prot(struct kvm_vcpu *vcpu, gfn_t gfn, bool *writable)
1426{
1427 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
1428
1429 return gfn_to_hva_memslot_prot(slot, gfn, writable);
1430}
1431
Huang Yingfafc3db2011-01-30 11:15:49 +08001432static inline int check_user_page_hwpoison(unsigned long addr)
1433{
Lorenzo Stoakes0d731752016-10-24 10:57:25 +01001434 int rc, flags = FOLL_HWPOISON | FOLL_WRITE;
Huang Yingfafc3db2011-01-30 11:15:49 +08001435
Lorenzo Stoakes0d731752016-10-24 10:57:25 +01001436 rc = get_user_pages(addr, 1, flags, NULL, NULL);
Huang Yingfafc3db2011-01-30 11:15:49 +08001437 return rc == -EHWPOISON;
1438}
1439
Xiao Guangrong2fc84312012-08-21 11:00:22 +08001440/*
Paolo Bonzinib9b33da2018-07-27 17:44:41 +02001441 * The fast path to get the writable pfn which will be stored in @pfn,
1442 * true indicates success, otherwise false is returned. It's also the
1443 * only part that runs if we can are in atomic context.
Xiao Guangrong2fc84312012-08-21 11:00:22 +08001444 */
Paolo Bonzinib9b33da2018-07-27 17:44:41 +02001445static bool hva_to_pfn_fast(unsigned long addr, bool write_fault,
1446 bool *writable, kvm_pfn_t *pfn)
Xiao Guangrong2fc84312012-08-21 11:00:22 +08001447{
1448 struct page *page[1];
1449 int npages;
1450
Xiao Guangrong12ce13f2012-08-21 11:00:49 +08001451 /*
1452 * Fast pin a writable pfn only if it is a write fault request
1453 * or the caller allows to map a writable pfn for a read fault
1454 * request.
1455 */
1456 if (!(write_fault || writable))
1457 return false;
1458
Xiao Guangrong2fc84312012-08-21 11:00:22 +08001459 npages = __get_user_pages_fast(addr, 1, 1, page);
1460 if (npages == 1) {
1461 *pfn = page_to_pfn(page[0]);
1462
1463 if (writable)
1464 *writable = true;
1465 return true;
1466 }
1467
1468 return false;
1469}
1470
1471/*
1472 * The slow path to get the pfn of the specified host virtual address,
1473 * 1 indicates success, -errno is returned if error is detected.
1474 */
1475static int hva_to_pfn_slow(unsigned long addr, bool *async, bool write_fault,
Dan Williamsba049e92016-01-15 16:56:11 -08001476 bool *writable, kvm_pfn_t *pfn)
Avi Kivity954bbbc2007-03-30 14:02:32 +03001477{
Al Viroce530532017-11-19 17:47:33 -05001478 unsigned int flags = FOLL_HWPOISON;
1479 struct page *page;
Gleb Natapovaf585b92010-10-14 11:22:46 +02001480 int npages = 0;
Xiao Guangrong2fc84312012-08-21 11:00:22 +08001481
1482 might_sleep();
1483
1484 if (writable)
1485 *writable = write_fault;
1486
Al Viroce530532017-11-19 17:47:33 -05001487 if (write_fault)
1488 flags |= FOLL_WRITE;
1489 if (async)
1490 flags |= FOLL_NOWAIT;
Lorenzo Stoakesd4944b02016-10-13 01:20:12 +01001491
Al Viroce530532017-11-19 17:47:33 -05001492 npages = get_user_pages_unlocked(addr, 1, &page, flags);
Xiao Guangrong2fc84312012-08-21 11:00:22 +08001493 if (npages != 1)
1494 return npages;
1495
1496 /* map read fault as writable if possible */
Xiao Guangrong12ce13f2012-08-21 11:00:49 +08001497 if (unlikely(!write_fault) && writable) {
Al Viroce530532017-11-19 17:47:33 -05001498 struct page *wpage;
Xiao Guangrong2fc84312012-08-21 11:00:22 +08001499
Al Viroce530532017-11-19 17:47:33 -05001500 if (__get_user_pages_fast(addr, 1, 1, &wpage) == 1) {
Xiao Guangrong2fc84312012-08-21 11:00:22 +08001501 *writable = true;
Al Viroce530532017-11-19 17:47:33 -05001502 put_page(page);
1503 page = wpage;
Xiao Guangrong2fc84312012-08-21 11:00:22 +08001504 }
Xiao Guangrong2fc84312012-08-21 11:00:22 +08001505 }
Al Viroce530532017-11-19 17:47:33 -05001506 *pfn = page_to_pfn(page);
Xiao Guangrong2fc84312012-08-21 11:00:22 +08001507 return npages;
1508}
1509
Xiao Guangrong4d8b81a2012-08-21 11:02:51 +08001510static bool vma_is_valid(struct vm_area_struct *vma, bool write_fault)
1511{
1512 if (unlikely(!(vma->vm_flags & VM_READ)))
1513 return false;
1514
1515 if (write_fault && (unlikely(!(vma->vm_flags & VM_WRITE))))
1516 return false;
1517
1518 return true;
1519}
1520
Paolo Bonzini92176a82016-06-07 16:22:47 +02001521static int hva_to_pfn_remapped(struct vm_area_struct *vma,
1522 unsigned long addr, bool *async,
KarimAllah Ahmeda340b3e2018-01-17 19:18:56 +01001523 bool write_fault, bool *writable,
1524 kvm_pfn_t *p_pfn)
Paolo Bonzini92176a82016-06-07 16:22:47 +02001525{
Paolo Bonziniadd6a0c2016-06-07 17:51:18 +02001526 unsigned long pfn;
1527 int r;
1528
1529 r = follow_pfn(vma, addr, &pfn);
1530 if (r) {
1531 /*
1532 * get_user_pages fails for VM_IO and VM_PFNMAP vmas and does
1533 * not call the fault handler, so do it here.
1534 */
1535 bool unlocked = false;
1536 r = fixup_user_fault(current, current->mm, addr,
1537 (write_fault ? FAULT_FLAG_WRITE : 0),
1538 &unlocked);
1539 if (unlocked)
1540 return -EAGAIN;
1541 if (r)
1542 return r;
1543
1544 r = follow_pfn(vma, addr, &pfn);
1545 if (r)
1546 return r;
1547
1548 }
1549
KarimAllah Ahmeda340b3e2018-01-17 19:18:56 +01001550 if (writable)
1551 *writable = true;
Paolo Bonziniadd6a0c2016-06-07 17:51:18 +02001552
1553 /*
1554 * Get a reference here because callers of *hva_to_pfn* and
1555 * *gfn_to_pfn* ultimately call kvm_release_pfn_clean on the
1556 * returned pfn. This is only needed if the VMA has VM_MIXEDMAP
1557 * set, but the kvm_get_pfn/kvm_release_pfn_clean pair will
1558 * simply do nothing for reserved pfns.
1559 *
1560 * Whoever called remap_pfn_range is also going to call e.g.
1561 * unmap_mapping_range before the underlying pages are freed,
1562 * causing a call to our MMU notifier.
1563 */
1564 kvm_get_pfn(pfn);
1565
1566 *p_pfn = pfn;
Paolo Bonzini92176a82016-06-07 16:22:47 +02001567 return 0;
1568}
1569
Xiao Guangrong12ce13f2012-08-21 11:00:49 +08001570/*
1571 * Pin guest page in memory and return its pfn.
1572 * @addr: host virtual address which maps memory to the guest
1573 * @atomic: whether this function can sleep
1574 * @async: whether this function need to wait IO complete if the
1575 * host page is not in the memory
1576 * @write_fault: whether we should get a writable host page
1577 * @writable: whether it allows to map a writable host page for !@write_fault
1578 *
1579 * The function will map a writable host page for these two cases:
1580 * 1): @write_fault = true
1581 * 2): @write_fault = false && @writable, @writable will tell the caller
1582 * whether the mapping is writable.
1583 */
Dan Williamsba049e92016-01-15 16:56:11 -08001584static kvm_pfn_t hva_to_pfn(unsigned long addr, bool atomic, bool *async,
Xiao Guangrong2fc84312012-08-21 11:00:22 +08001585 bool write_fault, bool *writable)
1586{
1587 struct vm_area_struct *vma;
Dan Williamsba049e92016-01-15 16:56:11 -08001588 kvm_pfn_t pfn = 0;
Paolo Bonzini92176a82016-06-07 16:22:47 +02001589 int npages, r;
Avi Kivity954bbbc2007-03-30 14:02:32 +03001590
Gleb Natapovaf585b92010-10-14 11:22:46 +02001591 /* we can do it either atomically or asynchronously, not both */
1592 BUG_ON(atomic && async);
1593
Paolo Bonzinib9b33da2018-07-27 17:44:41 +02001594 if (hva_to_pfn_fast(addr, write_fault, writable, &pfn))
Xiao Guangrong2fc84312012-08-21 11:00:22 +08001595 return pfn;
Marcelo Tosatti612819c2010-10-22 14:18:18 -02001596
Xiao Guangrong2fc84312012-08-21 11:00:22 +08001597 if (atomic)
1598 return KVM_PFN_ERR_FAULT;
Marcelo Tosatti612819c2010-10-22 14:18:18 -02001599
Xiao Guangrong2fc84312012-08-21 11:00:22 +08001600 npages = hva_to_pfn_slow(addr, async, write_fault, writable, &pfn);
1601 if (npages == 1)
1602 return pfn;
Gleb Natapovaf585b92010-10-14 11:22:46 +02001603
Xiao Guangrong2fc84312012-08-21 11:00:22 +08001604 down_read(&current->mm->mmap_sem);
1605 if (npages == -EHWPOISON ||
1606 (!async && check_user_page_hwpoison(addr))) {
1607 pfn = KVM_PFN_ERR_HWPOISON;
1608 goto exit;
Xiao Guangrong887c08a2010-08-22 19:10:28 +08001609 }
Izik Eidus539cb662007-11-11 22:05:04 +02001610
Paolo Bonziniadd6a0c2016-06-07 17:51:18 +02001611retry:
Xiao Guangrong2fc84312012-08-21 11:00:22 +08001612 vma = find_vma_intersection(current->mm, addr, addr + 1);
Anthony Liguori8d4e1282007-10-18 09:59:34 -05001613
Xiao Guangrong2fc84312012-08-21 11:00:22 +08001614 if (vma == NULL)
1615 pfn = KVM_PFN_ERR_FAULT;
Paolo Bonzini92176a82016-06-07 16:22:47 +02001616 else if (vma->vm_flags & (VM_IO | VM_PFNMAP)) {
KarimAllah Ahmeda340b3e2018-01-17 19:18:56 +01001617 r = hva_to_pfn_remapped(vma, addr, async, write_fault, writable, &pfn);
Paolo Bonziniadd6a0c2016-06-07 17:51:18 +02001618 if (r == -EAGAIN)
1619 goto retry;
Paolo Bonzini92176a82016-06-07 16:22:47 +02001620 if (r < 0)
1621 pfn = KVM_PFN_ERR_FAULT;
Xiao Guangrong2fc84312012-08-21 11:00:22 +08001622 } else {
Xiao Guangrong4d8b81a2012-08-21 11:02:51 +08001623 if (async && vma_is_valid(vma, write_fault))
Xiao Guangrong2fc84312012-08-21 11:00:22 +08001624 *async = true;
1625 pfn = KVM_PFN_ERR_FAULT;
1626 }
1627exit:
1628 up_read(&current->mm->mmap_sem);
Anthony Liguori2e2e3732008-04-30 15:37:07 -05001629 return pfn;
Anthony Liguori35149e22008-04-02 14:46:56 -05001630}
1631
Dan Williamsba049e92016-01-15 16:56:11 -08001632kvm_pfn_t __gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn,
1633 bool atomic, bool *async, bool write_fault,
1634 bool *writable)
Xiao Guangrong887c08a2010-08-22 19:10:28 +08001635{
Xiao Guangrong4d8b81a2012-08-21 11:02:51 +08001636 unsigned long addr = __gfn_to_hva_many(slot, gfn, NULL, write_fault);
1637
Paolo Bonzinib2740d32016-02-23 15:36:01 +01001638 if (addr == KVM_HVA_ERR_RO_BAD) {
1639 if (writable)
1640 *writable = false;
Xiao Guangrong4d8b81a2012-08-21 11:02:51 +08001641 return KVM_PFN_ERR_RO_FAULT;
Paolo Bonzinib2740d32016-02-23 15:36:01 +01001642 }
Xiao Guangrong4d8b81a2012-08-21 11:02:51 +08001643
Paolo Bonzinib2740d32016-02-23 15:36:01 +01001644 if (kvm_is_error_hva(addr)) {
1645 if (writable)
1646 *writable = false;
Xiao Guangrong81c52c52012-10-16 20:10:59 +08001647 return KVM_PFN_NOSLOT;
Paolo Bonzinib2740d32016-02-23 15:36:01 +01001648 }
Xiao Guangrong4d8b81a2012-08-21 11:02:51 +08001649
1650 /* Do not map writable pfn in the readonly memslot. */
1651 if (writable && memslot_is_readonly(slot)) {
1652 *writable = false;
1653 writable = NULL;
1654 }
1655
1656 return hva_to_pfn(addr, atomic, async, write_fault,
1657 writable);
Xiao Guangrong887c08a2010-08-22 19:10:28 +08001658}
Paolo Bonzini35204692015-04-02 11:20:48 +02001659EXPORT_SYMBOL_GPL(__gfn_to_pfn_memslot);
Xiao Guangrong887c08a2010-08-22 19:10:28 +08001660
Dan Williamsba049e92016-01-15 16:56:11 -08001661kvm_pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault,
Marcelo Tosatti612819c2010-10-22 14:18:18 -02001662 bool *writable)
1663{
Paolo Bonzinie37afc62015-05-19 16:09:04 +02001664 return __gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn, false, NULL,
1665 write_fault, writable);
Marcelo Tosatti612819c2010-10-22 14:18:18 -02001666}
1667EXPORT_SYMBOL_GPL(gfn_to_pfn_prot);
1668
Dan Williamsba049e92016-01-15 16:56:11 -08001669kvm_pfn_t gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn)
Marcelo Tosatti506f0d62009-12-23 14:35:19 -02001670{
Xiao Guangrong4d8b81a2012-08-21 11:02:51 +08001671 return __gfn_to_pfn_memslot(slot, gfn, false, NULL, true, NULL);
Marcelo Tosatti506f0d62009-12-23 14:35:19 -02001672}
Paolo Bonzinie37afc62015-05-19 16:09:04 +02001673EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot);
Marcelo Tosatti506f0d62009-12-23 14:35:19 -02001674
Dan Williamsba049e92016-01-15 16:56:11 -08001675kvm_pfn_t gfn_to_pfn_memslot_atomic(struct kvm_memory_slot *slot, gfn_t gfn)
Xiao Guangrong037d92d2012-08-21 10:59:12 +08001676{
Xiao Guangrong4d8b81a2012-08-21 11:02:51 +08001677 return __gfn_to_pfn_memslot(slot, gfn, true, NULL, true, NULL);
Xiao Guangrong037d92d2012-08-21 10:59:12 +08001678}
1679EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot_atomic);
1680
Dan Williamsba049e92016-01-15 16:56:11 -08001681kvm_pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn)
Paolo Bonzinie37afc62015-05-19 16:09:04 +02001682{
1683 return gfn_to_pfn_memslot_atomic(gfn_to_memslot(kvm, gfn), gfn);
1684}
1685EXPORT_SYMBOL_GPL(gfn_to_pfn_atomic);
1686
Dan Williamsba049e92016-01-15 16:56:11 -08001687kvm_pfn_t kvm_vcpu_gfn_to_pfn_atomic(struct kvm_vcpu *vcpu, gfn_t gfn)
Paolo Bonzini8e734852015-05-17 13:58:53 +02001688{
1689 return gfn_to_pfn_memslot_atomic(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn);
1690}
1691EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_pfn_atomic);
1692
Dan Williamsba049e92016-01-15 16:56:11 -08001693kvm_pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn)
Paolo Bonzinie37afc62015-05-19 16:09:04 +02001694{
1695 return gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn);
1696}
1697EXPORT_SYMBOL_GPL(gfn_to_pfn);
1698
Dan Williamsba049e92016-01-15 16:56:11 -08001699kvm_pfn_t kvm_vcpu_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn)
Paolo Bonzini8e734852015-05-17 13:58:53 +02001700{
1701 return gfn_to_pfn_memslot(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn);
1702}
1703EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_pfn);
1704
Paolo Bonzinid9ef13c2015-05-19 16:01:50 +02001705int gfn_to_page_many_atomic(struct kvm_memory_slot *slot, gfn_t gfn,
1706 struct page **pages, int nr_pages)
Xiao Guangrong48987782010-08-22 19:11:43 +08001707{
1708 unsigned long addr;
Arnd Bergmann076b9252017-08-10 14:14:39 +02001709 gfn_t entry = 0;
Xiao Guangrong48987782010-08-22 19:11:43 +08001710
Paolo Bonzinid9ef13c2015-05-19 16:01:50 +02001711 addr = gfn_to_hva_many(slot, gfn, &entry);
Xiao Guangrong48987782010-08-22 19:11:43 +08001712 if (kvm_is_error_hva(addr))
1713 return -1;
1714
1715 if (entry < nr_pages)
1716 return 0;
1717
1718 return __get_user_pages_fast(addr, nr_pages, 1, pages);
1719}
1720EXPORT_SYMBOL_GPL(gfn_to_page_many_atomic);
1721
Dan Williamsba049e92016-01-15 16:56:11 -08001722static struct page *kvm_pfn_to_page(kvm_pfn_t pfn)
Xiao Guangronga2766322012-07-26 11:58:59 +08001723{
Xiao Guangrong81c52c52012-10-16 20:10:59 +08001724 if (is_error_noslot_pfn(pfn))
Xiao Guangrong6cede2e2012-08-03 15:41:22 +08001725 return KVM_ERR_PTR_BAD_PAGE;
Xiao Guangronga2766322012-07-26 11:58:59 +08001726
Ard Biesheuvelbf4bea82014-11-10 08:33:56 +00001727 if (kvm_is_reserved_pfn(pfn)) {
Xiao Guangrongcb9aaa32012-08-03 15:42:10 +08001728 WARN_ON(1);
1729 return KVM_ERR_PTR_BAD_PAGE;
1730 }
1731
Xiao Guangronga2766322012-07-26 11:58:59 +08001732 return pfn_to_page(pfn);
1733}
1734
Anthony Liguori35149e22008-04-02 14:46:56 -05001735struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
1736{
Dan Williamsba049e92016-01-15 16:56:11 -08001737 kvm_pfn_t pfn;
Anthony Liguori2e2e3732008-04-30 15:37:07 -05001738
1739 pfn = gfn_to_pfn(kvm, gfn);
Anthony Liguori2e2e3732008-04-30 15:37:07 -05001740
Xiao Guangronga2766322012-07-26 11:58:59 +08001741 return kvm_pfn_to_page(pfn);
Avi Kivity954bbbc2007-03-30 14:02:32 +03001742}
1743EXPORT_SYMBOL_GPL(gfn_to_page);
1744
Paolo Bonzini8e734852015-05-17 13:58:53 +02001745struct page *kvm_vcpu_gfn_to_page(struct kvm_vcpu *vcpu, gfn_t gfn)
1746{
Dan Williamsba049e92016-01-15 16:56:11 -08001747 kvm_pfn_t pfn;
Paolo Bonzini8e734852015-05-17 13:58:53 +02001748
1749 pfn = kvm_vcpu_gfn_to_pfn(vcpu, gfn);
1750
1751 return kvm_pfn_to_page(pfn);
1752}
1753EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_page);
1754
Izik Eidusb4231d62007-11-20 11:49:33 +02001755void kvm_release_page_clean(struct page *page)
1756{
Xiao Guangrong32cad842012-08-03 15:42:52 +08001757 WARN_ON(is_error_page(page));
1758
Anthony Liguori35149e22008-04-02 14:46:56 -05001759 kvm_release_pfn_clean(page_to_pfn(page));
Izik Eidusb4231d62007-11-20 11:49:33 +02001760}
1761EXPORT_SYMBOL_GPL(kvm_release_page_clean);
1762
Dan Williamsba049e92016-01-15 16:56:11 -08001763void kvm_release_pfn_clean(kvm_pfn_t pfn)
Anthony Liguori35149e22008-04-02 14:46:56 -05001764{
Ard Biesheuvelbf4bea82014-11-10 08:33:56 +00001765 if (!is_error_noslot_pfn(pfn) && !kvm_is_reserved_pfn(pfn))
Anthony Liguori2e2e3732008-04-30 15:37:07 -05001766 put_page(pfn_to_page(pfn));
Anthony Liguori35149e22008-04-02 14:46:56 -05001767}
1768EXPORT_SYMBOL_GPL(kvm_release_pfn_clean);
1769
Izik Eidusb4231d62007-11-20 11:49:33 +02001770void kvm_release_page_dirty(struct page *page)
Izik Eidus8a7ae052007-10-18 11:09:33 +02001771{
Xiao Guangronga2766322012-07-26 11:58:59 +08001772 WARN_ON(is_error_page(page));
1773
Anthony Liguori35149e22008-04-02 14:46:56 -05001774 kvm_release_pfn_dirty(page_to_pfn(page));
Izik Eidus8a7ae052007-10-18 11:09:33 +02001775}
Izik Eidusb4231d62007-11-20 11:49:33 +02001776EXPORT_SYMBOL_GPL(kvm_release_page_dirty);
Izik Eidus8a7ae052007-10-18 11:09:33 +02001777
David Hildenbrandf7a65092017-09-01 17:11:43 +02001778void kvm_release_pfn_dirty(kvm_pfn_t pfn)
Anthony Liguori35149e22008-04-02 14:46:56 -05001779{
1780 kvm_set_pfn_dirty(pfn);
1781 kvm_release_pfn_clean(pfn);
1782}
David Hildenbrandf7a65092017-09-01 17:11:43 +02001783EXPORT_SYMBOL_GPL(kvm_release_pfn_dirty);
Anthony Liguori35149e22008-04-02 14:46:56 -05001784
Dan Williamsba049e92016-01-15 16:56:11 -08001785void kvm_set_pfn_dirty(kvm_pfn_t pfn)
Anthony Liguori35149e22008-04-02 14:46:56 -05001786{
Ard Biesheuvelbf4bea82014-11-10 08:33:56 +00001787 if (!kvm_is_reserved_pfn(pfn)) {
Anthony Liguori2e2e3732008-04-30 15:37:07 -05001788 struct page *page = pfn_to_page(pfn);
Xiubo Lif95ef0c2015-02-26 14:58:23 +08001789
Anthony Liguori2e2e3732008-04-30 15:37:07 -05001790 if (!PageReserved(page))
1791 SetPageDirty(page);
1792 }
Anthony Liguori35149e22008-04-02 14:46:56 -05001793}
1794EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty);
1795
Dan Williamsba049e92016-01-15 16:56:11 -08001796void kvm_set_pfn_accessed(kvm_pfn_t pfn)
Anthony Liguori35149e22008-04-02 14:46:56 -05001797{
Ard Biesheuvelbf4bea82014-11-10 08:33:56 +00001798 if (!kvm_is_reserved_pfn(pfn))
Anthony Liguori2e2e3732008-04-30 15:37:07 -05001799 mark_page_accessed(pfn_to_page(pfn));
Anthony Liguori35149e22008-04-02 14:46:56 -05001800}
1801EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed);
1802
Dan Williamsba049e92016-01-15 16:56:11 -08001803void kvm_get_pfn(kvm_pfn_t pfn)
Anthony Liguori35149e22008-04-02 14:46:56 -05001804{
Ard Biesheuvelbf4bea82014-11-10 08:33:56 +00001805 if (!kvm_is_reserved_pfn(pfn))
Anthony Liguori2e2e3732008-04-30 15:37:07 -05001806 get_page(pfn_to_page(pfn));
Anthony Liguori35149e22008-04-02 14:46:56 -05001807}
1808EXPORT_SYMBOL_GPL(kvm_get_pfn);
1809
Izik Eidus195aefd2007-10-01 22:14:18 +02001810static int next_segment(unsigned long len, int offset)
1811{
1812 if (len > PAGE_SIZE - offset)
1813 return PAGE_SIZE - offset;
1814 else
1815 return len;
1816}
1817
Paolo Bonzini8e734852015-05-17 13:58:53 +02001818static int __kvm_read_guest_page(struct kvm_memory_slot *slot, gfn_t gfn,
1819 void *data, int offset, int len)
Izik Eidus195aefd2007-10-01 22:14:18 +02001820{
Izik Eiduse0506bc2007-11-11 22:10:22 +02001821 int r;
1822 unsigned long addr;
Izik Eidus195aefd2007-10-01 22:14:18 +02001823
Paolo Bonzini8e734852015-05-17 13:58:53 +02001824 addr = gfn_to_hva_memslot_prot(slot, gfn, NULL);
Izik Eiduse0506bc2007-11-11 22:10:22 +02001825 if (kvm_is_error_hva(addr))
Izik Eidus195aefd2007-10-01 22:14:18 +02001826 return -EFAULT;
Paolo Bonzini3180a7f2015-04-02 14:08:20 +02001827 r = __copy_from_user(data, (void __user *)addr + offset, len);
Izik Eiduse0506bc2007-11-11 22:10:22 +02001828 if (r)
1829 return -EFAULT;
Izik Eidus195aefd2007-10-01 22:14:18 +02001830 return 0;
1831}
Paolo Bonzini8e734852015-05-17 13:58:53 +02001832
1833int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
1834 int len)
1835{
1836 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
1837
1838 return __kvm_read_guest_page(slot, gfn, data, offset, len);
1839}
Izik Eidus195aefd2007-10-01 22:14:18 +02001840EXPORT_SYMBOL_GPL(kvm_read_guest_page);
1841
Paolo Bonzini8e734852015-05-17 13:58:53 +02001842int kvm_vcpu_read_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, void *data,
1843 int offset, int len)
1844{
1845 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
1846
1847 return __kvm_read_guest_page(slot, gfn, data, offset, len);
1848}
1849EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_page);
1850
Izik Eidus195aefd2007-10-01 22:14:18 +02001851int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
1852{
1853 gfn_t gfn = gpa >> PAGE_SHIFT;
1854 int seg;
1855 int offset = offset_in_page(gpa);
1856 int ret;
1857
1858 while ((seg = next_segment(len, offset)) != 0) {
1859 ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);
1860 if (ret < 0)
1861 return ret;
1862 offset = 0;
1863 len -= seg;
1864 data += seg;
1865 ++gfn;
1866 }
1867 return 0;
1868}
1869EXPORT_SYMBOL_GPL(kvm_read_guest);
1870
Paolo Bonzini8e734852015-05-17 13:58:53 +02001871int kvm_vcpu_read_guest(struct kvm_vcpu *vcpu, gpa_t gpa, void *data, unsigned long len)
1872{
1873 gfn_t gfn = gpa >> PAGE_SHIFT;
1874 int seg;
1875 int offset = offset_in_page(gpa);
1876 int ret;
1877
1878 while ((seg = next_segment(len, offset)) != 0) {
1879 ret = kvm_vcpu_read_guest_page(vcpu, gfn, data, offset, seg);
1880 if (ret < 0)
1881 return ret;
1882 offset = 0;
1883 len -= seg;
1884 data += seg;
1885 ++gfn;
1886 }
1887 return 0;
1888}
1889EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest);
1890
1891static int __kvm_read_guest_atomic(struct kvm_memory_slot *slot, gfn_t gfn,
1892 void *data, int offset, unsigned long len)
Marcelo Tosatti7ec54582007-12-20 19:18:23 -05001893{
1894 int r;
1895 unsigned long addr;
Marcelo Tosatti7ec54582007-12-20 19:18:23 -05001896
Paolo Bonzini8e734852015-05-17 13:58:53 +02001897 addr = gfn_to_hva_memslot_prot(slot, gfn, NULL);
Marcelo Tosatti7ec54582007-12-20 19:18:23 -05001898 if (kvm_is_error_hva(addr))
1899 return -EFAULT;
Andrea Arcangeli0aac03f2008-01-30 19:57:35 +01001900 pagefault_disable();
Paolo Bonzini3180a7f2015-04-02 14:08:20 +02001901 r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len);
Andrea Arcangeli0aac03f2008-01-30 19:57:35 +01001902 pagefault_enable();
Marcelo Tosatti7ec54582007-12-20 19:18:23 -05001903 if (r)
1904 return -EFAULT;
1905 return 0;
1906}
Marcelo Tosatti7ec54582007-12-20 19:18:23 -05001907
Paolo Bonzini8e734852015-05-17 13:58:53 +02001908int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
1909 unsigned long len)
1910{
1911 gfn_t gfn = gpa >> PAGE_SHIFT;
1912 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
1913 int offset = offset_in_page(gpa);
1914
1915 return __kvm_read_guest_atomic(slot, gfn, data, offset, len);
1916}
1917EXPORT_SYMBOL_GPL(kvm_read_guest_atomic);
1918
1919int kvm_vcpu_read_guest_atomic(struct kvm_vcpu *vcpu, gpa_t gpa,
1920 void *data, unsigned long len)
1921{
1922 gfn_t gfn = gpa >> PAGE_SHIFT;
1923 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
1924 int offset = offset_in_page(gpa);
1925
1926 return __kvm_read_guest_atomic(slot, gfn, data, offset, len);
1927}
1928EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_atomic);
1929
1930static int __kvm_write_guest_page(struct kvm_memory_slot *memslot, gfn_t gfn,
1931 const void *data, int offset, int len)
Izik Eidus195aefd2007-10-01 22:14:18 +02001932{
Izik Eiduse0506bc2007-11-11 22:10:22 +02001933 int r;
1934 unsigned long addr;
Izik Eidus195aefd2007-10-01 22:14:18 +02001935
Radim Krčmář251eb842015-04-10 21:47:27 +02001936 addr = gfn_to_hva_memslot(memslot, gfn);
Izik Eiduse0506bc2007-11-11 22:10:22 +02001937 if (kvm_is_error_hva(addr))
Izik Eidus195aefd2007-10-01 22:14:18 +02001938 return -EFAULT;
Xiao Guangrong8b0cedf2011-05-15 23:22:04 +08001939 r = __copy_to_user((void __user *)addr + offset, data, len);
Izik Eiduse0506bc2007-11-11 22:10:22 +02001940 if (r)
1941 return -EFAULT;
Paolo Bonzinibc009e42015-05-26 12:43:41 +02001942 mark_page_dirty_in_slot(memslot, gfn);
Izik Eidus195aefd2007-10-01 22:14:18 +02001943 return 0;
1944}
Paolo Bonzini8e734852015-05-17 13:58:53 +02001945
1946int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn,
1947 const void *data, int offset, int len)
1948{
1949 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
1950
1951 return __kvm_write_guest_page(slot, gfn, data, offset, len);
1952}
Izik Eidus195aefd2007-10-01 22:14:18 +02001953EXPORT_SYMBOL_GPL(kvm_write_guest_page);
1954
Paolo Bonzini8e734852015-05-17 13:58:53 +02001955int kvm_vcpu_write_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn,
1956 const void *data, int offset, int len)
1957{
1958 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
1959
1960 return __kvm_write_guest_page(slot, gfn, data, offset, len);
1961}
1962EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest_page);
1963
Izik Eidus195aefd2007-10-01 22:14:18 +02001964int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
1965 unsigned long len)
1966{
1967 gfn_t gfn = gpa >> PAGE_SHIFT;
1968 int seg;
1969 int offset = offset_in_page(gpa);
1970 int ret;
1971
1972 while ((seg = next_segment(len, offset)) != 0) {
1973 ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);
1974 if (ret < 0)
1975 return ret;
1976 offset = 0;
1977 len -= seg;
1978 data += seg;
1979 ++gfn;
1980 }
1981 return 0;
1982}
Wincy Vanff651cb2014-12-11 08:52:58 +03001983EXPORT_SYMBOL_GPL(kvm_write_guest);
Izik Eidus195aefd2007-10-01 22:14:18 +02001984
Paolo Bonzini8e734852015-05-17 13:58:53 +02001985int kvm_vcpu_write_guest(struct kvm_vcpu *vcpu, gpa_t gpa, const void *data,
1986 unsigned long len)
1987{
1988 gfn_t gfn = gpa >> PAGE_SHIFT;
1989 int seg;
1990 int offset = offset_in_page(gpa);
1991 int ret;
1992
1993 while ((seg = next_segment(len, offset)) != 0) {
1994 ret = kvm_vcpu_write_guest_page(vcpu, gfn, data, offset, seg);
1995 if (ret < 0)
1996 return ret;
1997 offset = 0;
1998 len -= seg;
1999 data += seg;
2000 ++gfn;
2001 }
2002 return 0;
2003}
2004EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest);
2005
Paolo Bonzini5a2d4362017-02-03 20:32:28 -08002006static int __kvm_gfn_to_hva_cache_init(struct kvm_memslots *slots,
2007 struct gfn_to_hva_cache *ghc,
2008 gpa_t gpa, unsigned long len)
Gleb Natapov49c77542010-10-18 15:22:23 +02002009{
Gleb Natapov49c77542010-10-18 15:22:23 +02002010 int offset = offset_in_page(gpa);
Andrew Honig8f964522013-03-29 09:35:21 -07002011 gfn_t start_gfn = gpa >> PAGE_SHIFT;
2012 gfn_t end_gfn = (gpa + len - 1) >> PAGE_SHIFT;
2013 gfn_t nr_pages_needed = end_gfn - start_gfn + 1;
2014 gfn_t nr_pages_avail;
Jim Mattsonf1b9dd52018-12-17 13:53:33 -08002015 int r = start_gfn <= end_gfn ? 0 : -EINVAL;
Gleb Natapov49c77542010-10-18 15:22:23 +02002016
2017 ghc->gpa = gpa;
2018 ghc->generation = slots->generation;
Andrew Honig8f964522013-03-29 09:35:21 -07002019 ghc->len = len;
Jim Mattsonf1b9dd52018-12-17 13:53:33 -08002020 ghc->hva = KVM_HVA_ERR_BAD;
2021
2022 /*
2023 * If the requested region crosses two memslots, we still
2024 * verify that the entire region is valid here.
2025 */
2026 while (!r && start_gfn <= end_gfn) {
2027 ghc->memslot = __gfn_to_memslot(slots, start_gfn);
2028 ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn,
2029 &nr_pages_avail);
2030 if (kvm_is_error_hva(ghc->hva))
2031 r = -EFAULT;
2032 start_gfn += nr_pages_avail;
Andrew Honig8f964522013-03-29 09:35:21 -07002033 }
Jim Mattsonf1b9dd52018-12-17 13:53:33 -08002034
2035 /* Use the slow path for cross page reads and writes. */
2036 if (!r && nr_pages_needed == 1)
2037 ghc->hva += offset;
2038 else
2039 ghc->memslot = NULL;
2040
2041 return r;
Gleb Natapov49c77542010-10-18 15:22:23 +02002042}
Paolo Bonzini5a2d4362017-02-03 20:32:28 -08002043
Paolo Bonzini4e335d92017-05-02 16:20:18 +02002044int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
Paolo Bonzini5a2d4362017-02-03 20:32:28 -08002045 gpa_t gpa, unsigned long len)
2046{
Paolo Bonzini4e335d92017-05-02 16:20:18 +02002047 struct kvm_memslots *slots = kvm_memslots(kvm);
Paolo Bonzini5a2d4362017-02-03 20:32:28 -08002048 return __kvm_gfn_to_hva_cache_init(slots, ghc, gpa, len);
2049}
Paolo Bonzini4e335d92017-05-02 16:20:18 +02002050EXPORT_SYMBOL_GPL(kvm_gfn_to_hva_cache_init);
Gleb Natapov49c77542010-10-18 15:22:23 +02002051
Paolo Bonzini4e335d92017-05-02 16:20:18 +02002052int kvm_write_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
Jim Mattson7a86dab2018-12-14 14:34:43 -08002053 void *data, unsigned int offset,
2054 unsigned long len)
Gleb Natapov49c77542010-10-18 15:22:23 +02002055{
Paolo Bonzini4e335d92017-05-02 16:20:18 +02002056 struct kvm_memslots *slots = kvm_memslots(kvm);
Gleb Natapov49c77542010-10-18 15:22:23 +02002057 int r;
Pan Xinhui4ec6e862016-11-02 05:08:34 -04002058 gpa_t gpa = ghc->gpa + offset;
Gleb Natapov49c77542010-10-18 15:22:23 +02002059
Pan Xinhui4ec6e862016-11-02 05:08:34 -04002060 BUG_ON(len + offset > ghc->len);
Andrew Honig8f964522013-03-29 09:35:21 -07002061
Gleb Natapov49c77542010-10-18 15:22:23 +02002062 if (slots->generation != ghc->generation)
Paolo Bonzini5a2d4362017-02-03 20:32:28 -08002063 __kvm_gfn_to_hva_cache_init(slots, ghc, ghc->gpa, ghc->len);
Andrew Honig8f964522013-03-29 09:35:21 -07002064
2065 if (unlikely(!ghc->memslot))
Paolo Bonzini4e335d92017-05-02 16:20:18 +02002066 return kvm_write_guest(kvm, gpa, data, len);
Gleb Natapov49c77542010-10-18 15:22:23 +02002067
2068 if (kvm_is_error_hva(ghc->hva))
2069 return -EFAULT;
2070
Pan Xinhui4ec6e862016-11-02 05:08:34 -04002071 r = __copy_to_user((void __user *)ghc->hva + offset, data, len);
Gleb Natapov49c77542010-10-18 15:22:23 +02002072 if (r)
2073 return -EFAULT;
Pan Xinhui4ec6e862016-11-02 05:08:34 -04002074 mark_page_dirty_in_slot(ghc->memslot, gpa >> PAGE_SHIFT);
Gleb Natapov49c77542010-10-18 15:22:23 +02002075
2076 return 0;
2077}
Paolo Bonzini4e335d92017-05-02 16:20:18 +02002078EXPORT_SYMBOL_GPL(kvm_write_guest_offset_cached);
Pan Xinhui4ec6e862016-11-02 05:08:34 -04002079
Paolo Bonzini4e335d92017-05-02 16:20:18 +02002080int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
2081 void *data, unsigned long len)
Pan Xinhui4ec6e862016-11-02 05:08:34 -04002082{
Paolo Bonzini4e335d92017-05-02 16:20:18 +02002083 return kvm_write_guest_offset_cached(kvm, ghc, data, 0, len);
Pan Xinhui4ec6e862016-11-02 05:08:34 -04002084}
Paolo Bonzini4e335d92017-05-02 16:20:18 +02002085EXPORT_SYMBOL_GPL(kvm_write_guest_cached);
Gleb Natapov49c77542010-10-18 15:22:23 +02002086
Paolo Bonzini4e335d92017-05-02 16:20:18 +02002087int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
2088 void *data, unsigned long len)
Gleb Natapove03b6442011-07-11 15:28:11 -04002089{
Paolo Bonzini4e335d92017-05-02 16:20:18 +02002090 struct kvm_memslots *slots = kvm_memslots(kvm);
Gleb Natapove03b6442011-07-11 15:28:11 -04002091 int r;
2092
Andrew Honig8f964522013-03-29 09:35:21 -07002093 BUG_ON(len > ghc->len);
2094
Gleb Natapove03b6442011-07-11 15:28:11 -04002095 if (slots->generation != ghc->generation)
Paolo Bonzini5a2d4362017-02-03 20:32:28 -08002096 __kvm_gfn_to_hva_cache_init(slots, ghc, ghc->gpa, ghc->len);
Andrew Honig8f964522013-03-29 09:35:21 -07002097
2098 if (unlikely(!ghc->memslot))
Paolo Bonzini4e335d92017-05-02 16:20:18 +02002099 return kvm_read_guest(kvm, ghc->gpa, data, len);
Gleb Natapove03b6442011-07-11 15:28:11 -04002100
2101 if (kvm_is_error_hva(ghc->hva))
2102 return -EFAULT;
2103
2104 r = __copy_from_user(data, (void __user *)ghc->hva, len);
2105 if (r)
2106 return -EFAULT;
2107
2108 return 0;
2109}
Paolo Bonzini4e335d92017-05-02 16:20:18 +02002110EXPORT_SYMBOL_GPL(kvm_read_guest_cached);
Gleb Natapove03b6442011-07-11 15:28:11 -04002111
Izik Eidus195aefd2007-10-01 22:14:18 +02002112int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len)
2113{
Heiko Carstens8a3caa62013-11-18 10:35:55 +01002114 const void *zero_page = (const void *) __va(page_to_phys(ZERO_PAGE(0)));
2115
2116 return kvm_write_guest_page(kvm, gfn, zero_page, offset, len);
Izik Eidus195aefd2007-10-01 22:14:18 +02002117}
2118EXPORT_SYMBOL_GPL(kvm_clear_guest_page);
2119
2120int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)
2121{
2122 gfn_t gfn = gpa >> PAGE_SHIFT;
2123 int seg;
2124 int offset = offset_in_page(gpa);
2125 int ret;
2126
Kevin Mulveybfda0e82015-02-20 08:21:36 -05002127 while ((seg = next_segment(len, offset)) != 0) {
Izik Eidus195aefd2007-10-01 22:14:18 +02002128 ret = kvm_clear_guest_page(kvm, gfn, offset, seg);
2129 if (ret < 0)
2130 return ret;
2131 offset = 0;
2132 len -= seg;
2133 ++gfn;
2134 }
2135 return 0;
2136}
2137EXPORT_SYMBOL_GPL(kvm_clear_guest);
2138
Paolo Bonzinibc009e42015-05-26 12:43:41 +02002139static void mark_page_dirty_in_slot(struct kvm_memory_slot *memslot,
Stephen Hemminger79408762013-12-29 12:12:29 -08002140 gfn_t gfn)
Avi Kivity6aa8b732006-12-10 02:21:36 -08002141{
Rusty Russell7e9d6192007-07-31 20:41:14 +10002142 if (memslot && memslot->dirty_bitmap) {
2143 unsigned long rel_gfn = gfn - memslot->base_gfn;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002144
Takuya Yoshikawab74ca3b2012-10-04 17:13:12 -07002145 set_bit_le(rel_gfn, memslot->dirty_bitmap);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002146 }
2147}
2148
Gleb Natapov49c77542010-10-18 15:22:23 +02002149void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
2150{
2151 struct kvm_memory_slot *memslot;
2152
2153 memslot = gfn_to_memslot(kvm, gfn);
Paolo Bonzinibc009e42015-05-26 12:43:41 +02002154 mark_page_dirty_in_slot(memslot, gfn);
Gleb Natapov49c77542010-10-18 15:22:23 +02002155}
Aneesh Kumar K.V2ba9f0d2013-10-07 22:17:59 +05302156EXPORT_SYMBOL_GPL(mark_page_dirty);
Gleb Natapov49c77542010-10-18 15:22:23 +02002157
Paolo Bonzini8e734852015-05-17 13:58:53 +02002158void kvm_vcpu_mark_page_dirty(struct kvm_vcpu *vcpu, gfn_t gfn)
2159{
2160 struct kvm_memory_slot *memslot;
2161
2162 memslot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
2163 mark_page_dirty_in_slot(memslot, gfn);
2164}
2165EXPORT_SYMBOL_GPL(kvm_vcpu_mark_page_dirty);
2166
Jan H. Schönherr20b70352017-11-24 22:39:01 +01002167void kvm_sigset_activate(struct kvm_vcpu *vcpu)
2168{
2169 if (!vcpu->sigset_active)
2170 return;
2171
2172 /*
2173 * This does a lockless modification of ->real_blocked, which is fine
2174 * because, only current can change ->real_blocked and all readers of
2175 * ->real_blocked don't care as long ->real_blocked is always a subset
2176 * of ->blocked.
2177 */
2178 sigprocmask(SIG_SETMASK, &vcpu->sigset, &current->real_blocked);
2179}
2180
2181void kvm_sigset_deactivate(struct kvm_vcpu *vcpu)
2182{
2183 if (!vcpu->sigset_active)
2184 return;
2185
2186 sigprocmask(SIG_SETMASK, &current->real_blocked, NULL);
2187 sigemptyset(&current->real_blocked);
2188}
2189
Wanpeng Liaca6ff22015-09-03 22:07:38 +08002190static void grow_halt_poll_ns(struct kvm_vcpu *vcpu)
2191{
Nir Weinerdee339b2019-01-27 12:17:16 +02002192 unsigned int old, val, grow, grow_start;
Wanpeng Liaca6ff22015-09-03 22:07:38 +08002193
Wanpeng Li2cbd7822015-09-03 22:07:39 +08002194 old = val = vcpu->halt_poll_ns;
Nir Weinerdee339b2019-01-27 12:17:16 +02002195 grow_start = READ_ONCE(halt_poll_ns_grow_start);
Christian Borntraeger6b6de682016-02-09 13:47:55 +01002196 grow = READ_ONCE(halt_poll_ns_grow);
Nir Weiner7fa08e72019-01-27 12:17:14 +02002197 if (!grow)
2198 goto out;
2199
Nir Weinerdee339b2019-01-27 12:17:16 +02002200 val *= grow;
2201 if (val < grow_start)
2202 val = grow_start;
Wanpeng Liaca6ff22015-09-03 22:07:38 +08002203
David Matlack313f6362016-03-08 16:19:44 -08002204 if (val > halt_poll_ns)
2205 val = halt_poll_ns;
2206
Wanpeng Liaca6ff22015-09-03 22:07:38 +08002207 vcpu->halt_poll_ns = val;
Nir Weiner7fa08e72019-01-27 12:17:14 +02002208out:
Wanpeng Li2cbd7822015-09-03 22:07:39 +08002209 trace_kvm_halt_poll_ns_grow(vcpu->vcpu_id, val, old);
Wanpeng Liaca6ff22015-09-03 22:07:38 +08002210}
2211
2212static void shrink_halt_poll_ns(struct kvm_vcpu *vcpu)
2213{
Christian Borntraeger6b6de682016-02-09 13:47:55 +01002214 unsigned int old, val, shrink;
Wanpeng Liaca6ff22015-09-03 22:07:38 +08002215
Wanpeng Li2cbd7822015-09-03 22:07:39 +08002216 old = val = vcpu->halt_poll_ns;
Christian Borntraeger6b6de682016-02-09 13:47:55 +01002217 shrink = READ_ONCE(halt_poll_ns_shrink);
2218 if (shrink == 0)
Wanpeng Liaca6ff22015-09-03 22:07:38 +08002219 val = 0;
2220 else
Christian Borntraeger6b6de682016-02-09 13:47:55 +01002221 val /= shrink;
Wanpeng Liaca6ff22015-09-03 22:07:38 +08002222
2223 vcpu->halt_poll_ns = val;
Wanpeng Li2cbd7822015-09-03 22:07:39 +08002224 trace_kvm_halt_poll_ns_shrink(vcpu->vcpu_id, val, old);
Wanpeng Liaca6ff22015-09-03 22:07:38 +08002225}
2226
Paolo Bonzinif7819512015-02-04 18:20:58 +01002227static int kvm_vcpu_check_block(struct kvm_vcpu *vcpu)
2228{
Junaid Shahid50c28f22018-06-27 14:59:11 -07002229 int ret = -EINTR;
2230 int idx = srcu_read_lock(&vcpu->kvm->srcu);
2231
Paolo Bonzinif7819512015-02-04 18:20:58 +01002232 if (kvm_arch_vcpu_runnable(vcpu)) {
2233 kvm_make_request(KVM_REQ_UNHALT, vcpu);
Junaid Shahid50c28f22018-06-27 14:59:11 -07002234 goto out;
Paolo Bonzinif7819512015-02-04 18:20:58 +01002235 }
2236 if (kvm_cpu_has_pending_timer(vcpu))
Junaid Shahid50c28f22018-06-27 14:59:11 -07002237 goto out;
Paolo Bonzinif7819512015-02-04 18:20:58 +01002238 if (signal_pending(current))
Junaid Shahid50c28f22018-06-27 14:59:11 -07002239 goto out;
Paolo Bonzinif7819512015-02-04 18:20:58 +01002240
Junaid Shahid50c28f22018-06-27 14:59:11 -07002241 ret = 0;
2242out:
2243 srcu_read_unlock(&vcpu->kvm->srcu, idx);
2244 return ret;
Paolo Bonzinif7819512015-02-04 18:20:58 +01002245}
2246
Eddie Dongb6958ce2007-07-18 12:15:21 +03002247/*
2248 * The vCPU has executed a HLT instruction with in-kernel mode enabled.
2249 */
Hollis Blanchard8776e512007-10-31 17:24:24 -05002250void kvm_vcpu_block(struct kvm_vcpu *vcpu)
Eddie Dongb6958ce2007-07-18 12:15:21 +03002251{
Paolo Bonzinif7819512015-02-04 18:20:58 +01002252 ktime_t start, cur;
Marcelo Tosatti85773702016-02-19 09:46:39 +01002253 DECLARE_SWAITQUEUE(wait);
Paolo Bonzinif7819512015-02-04 18:20:58 +01002254 bool waited = false;
Wanpeng Liaca6ff22015-09-03 22:07:38 +08002255 u64 block_ns;
Paolo Bonzinif7819512015-02-04 18:20:58 +01002256
2257 start = cur = ktime_get();
Wanpeng Li19020f82015-09-03 22:07:37 +08002258 if (vcpu->halt_poll_ns) {
2259 ktime_t stop = ktime_add_ns(ktime_get(), vcpu->halt_poll_ns);
Xiubo Lif95ef0c2015-02-26 14:58:23 +08002260
Paolo Bonzini62bea5b2015-09-15 18:27:57 +02002261 ++vcpu->stat.halt_attempted_poll;
Paolo Bonzinif7819512015-02-04 18:20:58 +01002262 do {
2263 /*
2264 * This sets KVM_REQ_UNHALT if an interrupt
2265 * arrives.
2266 */
2267 if (kvm_vcpu_check_block(vcpu) < 0) {
2268 ++vcpu->stat.halt_successful_poll;
Christian Borntraeger3491caf2016-05-13 12:16:35 +02002269 if (!vcpu_valid_wakeup(vcpu))
2270 ++vcpu->stat.halt_poll_invalid;
Paolo Bonzinif7819512015-02-04 18:20:58 +01002271 goto out;
2272 }
2273 cur = ktime_get();
2274 } while (single_task_running() && ktime_before(cur, stop));
2275 }
Eddie Dongb6958ce2007-07-18 12:15:21 +03002276
Christoffer Dall3217f7c2015-08-27 16:41:15 +02002277 kvm_arch_vcpu_blocking(vcpu);
2278
Marcelo Tosattie5c239c2008-05-08 19:47:01 -03002279 for (;;) {
Peter Zijlstrab3dae102018-06-12 10:34:52 +02002280 prepare_to_swait_exclusive(&vcpu->wq, &wait, TASK_INTERRUPTIBLE);
Eddie Dongb6958ce2007-07-18 12:15:21 +03002281
Paolo Bonzinif7819512015-02-04 18:20:58 +01002282 if (kvm_vcpu_check_block(vcpu) < 0)
Marcelo Tosattie5c239c2008-05-08 19:47:01 -03002283 break;
2284
Paolo Bonzinif7819512015-02-04 18:20:58 +01002285 waited = true;
Eddie Dongb6958ce2007-07-18 12:15:21 +03002286 schedule();
Eddie Dongb6958ce2007-07-18 12:15:21 +03002287 }
2288
Marcelo Tosatti85773702016-02-19 09:46:39 +01002289 finish_swait(&vcpu->wq, &wait);
Paolo Bonzinif7819512015-02-04 18:20:58 +01002290 cur = ktime_get();
2291
Christoffer Dall3217f7c2015-08-27 16:41:15 +02002292 kvm_arch_vcpu_unblocking(vcpu);
Paolo Bonzinif7819512015-02-04 18:20:58 +01002293out:
Wanpeng Liaca6ff22015-09-03 22:07:38 +08002294 block_ns = ktime_to_ns(cur) - ktime_to_ns(start);
2295
Christian Borntraeger2086d322016-05-17 10:49:22 +02002296 if (!vcpu_valid_wakeup(vcpu))
2297 shrink_halt_poll_ns(vcpu);
2298 else if (halt_poll_ns) {
Wanpeng Liaca6ff22015-09-03 22:07:38 +08002299 if (block_ns <= vcpu->halt_poll_ns)
2300 ;
2301 /* we had a long block, shrink polling */
Christian Borntraeger2086d322016-05-17 10:49:22 +02002302 else if (vcpu->halt_poll_ns && block_ns > halt_poll_ns)
Wanpeng Liaca6ff22015-09-03 22:07:38 +08002303 shrink_halt_poll_ns(vcpu);
2304 /* we had a short halt and our poll time is too small */
2305 else if (vcpu->halt_poll_ns < halt_poll_ns &&
2306 block_ns < halt_poll_ns)
2307 grow_halt_poll_ns(vcpu);
Wanpeng Liedb92722015-09-14 17:38:51 +08002308 } else
2309 vcpu->halt_poll_ns = 0;
Wanpeng Liaca6ff22015-09-03 22:07:38 +08002310
Christian Borntraeger3491caf2016-05-13 12:16:35 +02002311 trace_kvm_vcpu_wakeup(block_ns, waited, vcpu_valid_wakeup(vcpu));
2312 kvm_arch_vcpu_block_finish(vcpu);
Eddie Dongb6958ce2007-07-18 12:15:21 +03002313}
Aneesh Kumar K.V2ba9f0d2013-10-07 22:17:59 +05302314EXPORT_SYMBOL_GPL(kvm_vcpu_block);
Eddie Dongb6958ce2007-07-18 12:15:21 +03002315
Radim Krčmář178f02f2017-04-26 22:32:26 +02002316bool kvm_vcpu_wake_up(struct kvm_vcpu *vcpu)
Christoffer Dallb6d33832012-03-08 16:44:24 -05002317{
Marcelo Tosatti85773702016-02-19 09:46:39 +01002318 struct swait_queue_head *wqp;
Christoffer Dallb6d33832012-03-08 16:44:24 -05002319
2320 wqp = kvm_arch_vcpu_wq(vcpu);
Davidlohr Bueso5e0018b2017-09-13 13:08:22 -07002321 if (swq_has_sleeper(wqp)) {
Peter Zijlstrab3dae102018-06-12 10:34:52 +02002322 swake_up_one(wqp);
Christoffer Dallb6d33832012-03-08 16:44:24 -05002323 ++vcpu->stat.halt_wakeup;
Radim Krčmář178f02f2017-04-26 22:32:26 +02002324 return true;
Christoffer Dallb6d33832012-03-08 16:44:24 -05002325 }
2326
Radim Krčmář178f02f2017-04-26 22:32:26 +02002327 return false;
Radim Krčmářdd1a4cc2016-05-04 14:09:44 -05002328}
2329EXPORT_SYMBOL_GPL(kvm_vcpu_wake_up);
2330
Paolo Bonzini0266c892017-05-04 15:14:13 +02002331#ifndef CONFIG_S390
Radim Krčmářdd1a4cc2016-05-04 14:09:44 -05002332/*
2333 * Kick a sleeping VCPU, or a guest VCPU in guest mode, into host kernel mode.
2334 */
2335void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
2336{
2337 int me;
2338 int cpu = vcpu->cpu;
2339
Radim Krčmář178f02f2017-04-26 22:32:26 +02002340 if (kvm_vcpu_wake_up(vcpu))
2341 return;
2342
Christoffer Dallb6d33832012-03-08 16:44:24 -05002343 me = get_cpu();
2344 if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu))
2345 if (kvm_arch_vcpu_should_kick(vcpu))
2346 smp_send_reschedule(cpu);
2347 put_cpu();
2348}
Yang Zhanga20ed542013-04-11 19:25:15 +08002349EXPORT_SYMBOL_GPL(kvm_vcpu_kick);
Paolo Bonzini0266c892017-05-04 15:14:13 +02002350#endif /* !CONFIG_S390 */
Christoffer Dallb6d33832012-03-08 16:44:24 -05002351
Dan Carpenterfa933842014-05-23 13:20:42 +03002352int kvm_vcpu_yield_to(struct kvm_vcpu *target)
Konstantin Weitz41628d32012-04-25 15:30:38 +02002353{
2354 struct pid *pid;
2355 struct task_struct *task = NULL;
Dan Carpenterfa933842014-05-23 13:20:42 +03002356 int ret = 0;
Konstantin Weitz41628d32012-04-25 15:30:38 +02002357
2358 rcu_read_lock();
2359 pid = rcu_dereference(target->pid);
2360 if (pid)
Sam Bobroff27fbe64b2014-09-19 09:40:41 +10002361 task = get_pid_task(pid, PIDTYPE_PID);
Konstantin Weitz41628d32012-04-25 15:30:38 +02002362 rcu_read_unlock();
2363 if (!task)
Raghavendra K Tc45c5282013-01-22 13:09:24 +05302364 return ret;
Raghavendra K Tc45c5282013-01-22 13:09:24 +05302365 ret = yield_to(task, 1);
Konstantin Weitz41628d32012-04-25 15:30:38 +02002366 put_task_struct(task);
Raghavendra K Tc45c5282013-01-22 13:09:24 +05302367
2368 return ret;
Konstantin Weitz41628d32012-04-25 15:30:38 +02002369}
2370EXPORT_SYMBOL_GPL(kvm_vcpu_yield_to);
2371
Raghavendra K T06e48c52012-07-19 15:17:52 +05302372/*
2373 * Helper that checks whether a VCPU is eligible for directed yield.
2374 * Most eligible candidate to yield is decided by following heuristics:
2375 *
2376 * (a) VCPU which has not done pl-exit or cpu relax intercepted recently
2377 * (preempted lock holder), indicated by @in_spin_loop.
2378 * Set at the beiginning and cleared at the end of interception/PLE handler.
2379 *
2380 * (b) VCPU which has done pl-exit/ cpu relax intercepted but did not get
2381 * chance last time (mostly it has become eligible now since we have probably
2382 * yielded to lockholder in last iteration. This is done by toggling
2383 * @dy_eligible each time a VCPU checked for eligibility.)
2384 *
2385 * Yielding to a recently pl-exited/cpu relax intercepted VCPU before yielding
2386 * to preempted lock-holder could result in wrong VCPU selection and CPU
2387 * burning. Giving priority for a potential lock-holder increases lock
2388 * progress.
2389 *
2390 * Since algorithm is based on heuristics, accessing another VCPU data without
2391 * locking does not harm. It may result in trying to yield to same VCPU, fail
2392 * and continue with next VCPU and so on.
2393 */
Stephen Hemminger79408762013-12-29 12:12:29 -08002394static bool kvm_vcpu_eligible_for_directed_yield(struct kvm_vcpu *vcpu)
Raghavendra K T06e48c52012-07-19 15:17:52 +05302395{
Scott Wood4a55dd72014-01-09 18:43:16 -06002396#ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT
Raghavendra K T06e48c52012-07-19 15:17:52 +05302397 bool eligible;
2398
2399 eligible = !vcpu->spin_loop.in_spin_loop ||
Christian Borntraeger34656112014-09-04 21:13:31 +02002400 vcpu->spin_loop.dy_eligible;
Raghavendra K T06e48c52012-07-19 15:17:52 +05302401
2402 if (vcpu->spin_loop.in_spin_loop)
2403 kvm_vcpu_set_dy_eligible(vcpu, !vcpu->spin_loop.dy_eligible);
2404
2405 return eligible;
Scott Wood4a55dd72014-01-09 18:43:16 -06002406#else
2407 return true;
Raghavendra K T06e48c52012-07-19 15:17:52 +05302408#endif
Scott Wood4a55dd72014-01-09 18:43:16 -06002409}
Raghavendra K Tc45c5282013-01-22 13:09:24 +05302410
Longpeng(Mike)199b5762017-08-08 12:05:32 +08002411void kvm_vcpu_on_spin(struct kvm_vcpu *me, bool yield_to_kernel_mode)
Zhai, Edwind255f4f2009-10-09 18:03:20 +08002412{
Rik van Riel217ece62011-02-01 09:53:28 -05002413 struct kvm *kvm = me->kvm;
2414 struct kvm_vcpu *vcpu;
2415 int last_boosted_vcpu = me->kvm->last_boosted_vcpu;
2416 int yielded = 0;
Raghavendra K Tc45c5282013-01-22 13:09:24 +05302417 int try = 3;
Rik van Riel217ece62011-02-01 09:53:28 -05002418 int pass;
2419 int i;
Zhai, Edwind255f4f2009-10-09 18:03:20 +08002420
Raghavendra K T4c088492012-07-18 19:07:46 +05302421 kvm_vcpu_set_in_spin_loop(me, true);
Rik van Riel217ece62011-02-01 09:53:28 -05002422 /*
2423 * We boost the priority of a VCPU that is runnable but not
2424 * currently running, because it got preempted by something
2425 * else and called schedule in __vcpu_run. Hopefully that
2426 * VCPU is holding the lock that we need and will release it.
2427 * We approximate round-robin by starting at the last boosted VCPU.
2428 */
Raghavendra K Tc45c5282013-01-22 13:09:24 +05302429 for (pass = 0; pass < 2 && !yielded && try; pass++) {
Rik van Riel217ece62011-02-01 09:53:28 -05002430 kvm_for_each_vcpu(i, vcpu, kvm) {
Rik van Riel5cfc2aa2012-06-19 16:51:04 -04002431 if (!pass && i <= last_boosted_vcpu) {
Rik van Riel217ece62011-02-01 09:53:28 -05002432 i = last_boosted_vcpu;
2433 continue;
2434 } else if (pass && i > last_boosted_vcpu)
2435 break;
Mark Rutland6aa7de02017-10-23 14:07:29 -07002436 if (!READ_ONCE(vcpu->preempted))
Raghavendra K T7bc7ae22013-03-04 23:32:27 +05302437 continue;
Rik van Riel217ece62011-02-01 09:53:28 -05002438 if (vcpu == me)
2439 continue;
Marcelo Tosatti85773702016-02-19 09:46:39 +01002440 if (swait_active(&vcpu->wq) && !kvm_arch_vcpu_runnable(vcpu))
Rik van Riel217ece62011-02-01 09:53:28 -05002441 continue;
Longpeng(Mike)199b5762017-08-08 12:05:32 +08002442 if (yield_to_kernel_mode && !kvm_arch_vcpu_in_kernel(vcpu))
2443 continue;
Raghavendra K T06e48c52012-07-19 15:17:52 +05302444 if (!kvm_vcpu_eligible_for_directed_yield(vcpu))
2445 continue;
Raghavendra K Tc45c5282013-01-22 13:09:24 +05302446
2447 yielded = kvm_vcpu_yield_to(vcpu);
2448 if (yielded > 0) {
Rik van Riel217ece62011-02-01 09:53:28 -05002449 kvm->last_boosted_vcpu = i;
Rik van Riel217ece62011-02-01 09:53:28 -05002450 break;
Raghavendra K Tc45c5282013-01-22 13:09:24 +05302451 } else if (yielded < 0) {
2452 try--;
2453 if (!try)
2454 break;
Rik van Riel217ece62011-02-01 09:53:28 -05002455 }
Rik van Riel217ece62011-02-01 09:53:28 -05002456 }
2457 }
Raghavendra K T4c088492012-07-18 19:07:46 +05302458 kvm_vcpu_set_in_spin_loop(me, false);
Raghavendra K T06e48c52012-07-19 15:17:52 +05302459
2460 /* Ensure vcpu is not eligible during next spinloop */
2461 kvm_vcpu_set_dy_eligible(me, false);
Zhai, Edwind255f4f2009-10-09 18:03:20 +08002462}
2463EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin);
2464
Souptick Joarder1499fa82018-04-19 00:49:58 +05302465static vm_fault_t kvm_vcpu_fault(struct vm_fault *vmf)
Avi Kivity9a2bb7f2007-02-22 12:58:31 +02002466{
Dave Jiang11bac802017-02-24 14:56:41 -08002467 struct kvm_vcpu *vcpu = vmf->vma->vm_file->private_data;
Avi Kivity9a2bb7f2007-02-22 12:58:31 +02002468 struct page *page;
2469
npiggin@suse.dee4a533a2007-12-05 18:15:52 +11002470 if (vmf->pgoff == 0)
Avi Kivity039576c2007-03-20 12:46:50 +02002471 page = virt_to_page(vcpu->run);
Avi Kivity09566762008-01-23 18:14:23 +02002472#ifdef CONFIG_X86
npiggin@suse.dee4a533a2007-12-05 18:15:52 +11002473 else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET)
Zhang Xiantaoad312c72007-12-13 23:50:52 +08002474 page = virt_to_page(vcpu->arch.pio_data);
Avi Kivity09566762008-01-23 18:14:23 +02002475#endif
Paolo Bonzini4b4357e2017-03-31 13:53:23 +02002476#ifdef CONFIG_KVM_MMIO
Laurent Vivier5f94c172008-05-30 16:05:54 +02002477 else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET)
2478 page = virt_to_page(vcpu->kvm->coalesced_mmio_ring);
2479#endif
Avi Kivity039576c2007-03-20 12:46:50 +02002480 else
Carsten Otte5b1c1492012-01-04 10:25:23 +01002481 return kvm_arch_vcpu_fault(vcpu, vmf);
Avi Kivity9a2bb7f2007-02-22 12:58:31 +02002482 get_page(page);
npiggin@suse.dee4a533a2007-12-05 18:15:52 +11002483 vmf->page = page;
2484 return 0;
Avi Kivity9a2bb7f2007-02-22 12:58:31 +02002485}
2486
Alexey Dobriyanf0f37e2f2009-09-27 22:29:37 +04002487static const struct vm_operations_struct kvm_vcpu_vm_ops = {
npiggin@suse.dee4a533a2007-12-05 18:15:52 +11002488 .fault = kvm_vcpu_fault,
Avi Kivity9a2bb7f2007-02-22 12:58:31 +02002489};
2490
2491static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
2492{
2493 vma->vm_ops = &kvm_vcpu_vm_ops;
2494 return 0;
2495}
2496
Avi Kivitybccf2152007-02-21 18:04:26 +02002497static int kvm_vcpu_release(struct inode *inode, struct file *filp)
2498{
2499 struct kvm_vcpu *vcpu = filp->private_data;
2500
Luiz Capitulino45b59392016-09-16 10:27:35 -04002501 debugfs_remove_recursive(vcpu->debugfs_dentry);
Al Viro66c0b392008-04-19 20:33:56 +01002502 kvm_put_kvm(vcpu->kvm);
Avi Kivitybccf2152007-02-21 18:04:26 +02002503 return 0;
2504}
2505
Christian Borntraeger3d3aab12008-12-02 11:17:32 +01002506static struct file_operations kvm_vcpu_fops = {
Avi Kivitybccf2152007-02-21 18:04:26 +02002507 .release = kvm_vcpu_release,
2508 .unlocked_ioctl = kvm_vcpu_ioctl,
Avi Kivity9a2bb7f2007-02-22 12:58:31 +02002509 .mmap = kvm_vcpu_mmap,
Arnd Bergmann6038f372010-08-15 18:52:59 +02002510 .llseek = noop_llseek,
Marc Zyngier7ddfd3e2018-06-17 10:16:21 +01002511 KVM_COMPAT(kvm_vcpu_compat_ioctl),
Avi Kivitybccf2152007-02-21 18:04:26 +02002512};
2513
2514/*
2515 * Allocates an inode for the vcpu.
2516 */
2517static int create_vcpu_fd(struct kvm_vcpu *vcpu)
2518{
Masatake YAMATOe46b4692018-01-20 04:04:22 +09002519 char name[8 + 1 + ITOA_MAX_LEN + 1];
2520
2521 snprintf(name, sizeof(name), "kvm-vcpu:%d", vcpu->vcpu_id);
2522 return anon_inode_getfd(name, &kvm_vcpu_fops, vcpu, O_RDWR | O_CLOEXEC);
Avi Kivitybccf2152007-02-21 18:04:26 +02002523}
2524
Luiz Capitulino45b59392016-09-16 10:27:35 -04002525static int kvm_create_vcpu_debugfs(struct kvm_vcpu *vcpu)
2526{
2527 char dir_name[ITOA_MAX_LEN * 2];
2528 int ret;
2529
2530 if (!kvm_arch_has_vcpu_debugfs())
2531 return 0;
2532
2533 if (!debugfs_initialized())
2534 return 0;
2535
2536 snprintf(dir_name, sizeof(dir_name), "vcpu%d", vcpu->vcpu_id);
2537 vcpu->debugfs_dentry = debugfs_create_dir(dir_name,
2538 vcpu->kvm->debugfs_dentry);
2539 if (!vcpu->debugfs_dentry)
2540 return -ENOMEM;
2541
2542 ret = kvm_arch_create_vcpu_debugfs(vcpu);
2543 if (ret < 0) {
2544 debugfs_remove_recursive(vcpu->debugfs_dentry);
2545 return ret;
2546 }
2547
2548 return 0;
2549}
2550
Avi Kivityc5ea7662007-02-20 18:41:05 +02002551/*
2552 * Creates some virtual cpus. Good luck creating more than one.
2553 */
Gleb Natapov73880c82009-06-09 15:56:28 +03002554static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id)
Avi Kivityc5ea7662007-02-20 18:41:05 +02002555{
2556 int r;
David Hildenbrande09fefd2015-11-05 09:03:50 +01002557 struct kvm_vcpu *vcpu;
Avi Kivityc5ea7662007-02-20 18:41:05 +02002558
Greg Kurz0b1b1df2016-05-09 18:13:37 +02002559 if (id >= KVM_MAX_VCPU_ID)
Andy Honig338c7db2013-11-18 16:09:22 -08002560 return -EINVAL;
2561
Paolo Bonzini6c7caeb2016-06-13 14:48:25 +02002562 mutex_lock(&kvm->lock);
2563 if (kvm->created_vcpus == KVM_MAX_VCPUS) {
2564 mutex_unlock(&kvm->lock);
2565 return -EINVAL;
2566 }
2567
2568 kvm->created_vcpus++;
2569 mutex_unlock(&kvm->lock);
2570
Gleb Natapov73880c82009-06-09 15:56:28 +03002571 vcpu = kvm_arch_vcpu_create(kvm, id);
Paolo Bonzini6c7caeb2016-06-13 14:48:25 +02002572 if (IS_ERR(vcpu)) {
2573 r = PTR_ERR(vcpu);
2574 goto vcpu_decrement;
2575 }
Avi Kivityc5ea7662007-02-20 18:41:05 +02002576
Avi Kivity15ad7142007-07-11 18:17:21 +03002577 preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);
2578
Avi Kivity26e52152007-11-20 15:30:24 +02002579 r = kvm_arch_vcpu_setup(vcpu);
2580 if (r)
Jan Kiszkad7805922011-05-23 10:33:05 +02002581 goto vcpu_destroy;
Avi Kivity26e52152007-11-20 15:30:24 +02002582
Luiz Capitulino45b59392016-09-16 10:27:35 -04002583 r = kvm_create_vcpu_debugfs(vcpu);
2584 if (r)
2585 goto vcpu_destroy;
2586
Shaohua Li11ec2802007-07-23 14:51:37 +08002587 mutex_lock(&kvm->lock);
David Hildenbrande09fefd2015-11-05 09:03:50 +01002588 if (kvm_get_vcpu_by_id(kvm, id)) {
2589 r = -EEXIST;
2590 goto unlock_vcpu_destroy;
2591 }
Gleb Natapov73880c82009-06-09 15:56:28 +03002592
2593 BUG_ON(kvm->vcpus[atomic_read(&kvm->online_vcpus)]);
Rusty Russellfb3f0f52007-07-27 17:16:56 +10002594
2595 /* Now it's all set up, let userspace reach it */
Al Viro66c0b392008-04-19 20:33:56 +01002596 kvm_get_kvm(kvm);
Avi Kivitybccf2152007-02-21 18:04:26 +02002597 r = create_vcpu_fd(vcpu);
Gleb Natapov73880c82009-06-09 15:56:28 +03002598 if (r < 0) {
2599 kvm_put_kvm(kvm);
Jan Kiszkad7805922011-05-23 10:33:05 +02002600 goto unlock_vcpu_destroy;
Gleb Natapov73880c82009-06-09 15:56:28 +03002601 }
2602
2603 kvm->vcpus[atomic_read(&kvm->online_vcpus)] = vcpu;
Paolo Bonzinidd489242015-07-29 11:32:20 +02002604
2605 /*
2606 * Pairs with smp_rmb() in kvm_get_vcpu. Write kvm->vcpus
2607 * before kvm->online_vcpu's incremented value.
2608 */
Gleb Natapov73880c82009-06-09 15:56:28 +03002609 smp_wmb();
2610 atomic_inc(&kvm->online_vcpus);
2611
Gleb Natapov73880c82009-06-09 15:56:28 +03002612 mutex_unlock(&kvm->lock);
Marcelo Tosatti42897d82012-11-27 23:29:02 -02002613 kvm_arch_vcpu_postcreate(vcpu);
Avi Kivitybccf2152007-02-21 18:04:26 +02002614 return r;
Avi Kivityc5ea7662007-02-20 18:41:05 +02002615
Jan Kiszkad7805922011-05-23 10:33:05 +02002616unlock_vcpu_destroy:
Glauber Costa7d8fece2008-09-17 23:16:59 -03002617 mutex_unlock(&kvm->lock);
Luiz Capitulino45b59392016-09-16 10:27:35 -04002618 debugfs_remove_recursive(vcpu->debugfs_dentry);
Jan Kiszkad7805922011-05-23 10:33:05 +02002619vcpu_destroy:
Hollis Blanchardd40ccc62007-11-19 14:04:43 -06002620 kvm_arch_vcpu_destroy(vcpu);
Paolo Bonzini6c7caeb2016-06-13 14:48:25 +02002621vcpu_decrement:
2622 mutex_lock(&kvm->lock);
2623 kvm->created_vcpus--;
2624 mutex_unlock(&kvm->lock);
Avi Kivityc5ea7662007-02-20 18:41:05 +02002625 return r;
2626}
2627
Avi Kivity1961d272007-03-05 19:46:05 +02002628static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
2629{
2630 if (sigset) {
2631 sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
2632 vcpu->sigset_active = 1;
2633 vcpu->sigset = *sigset;
2634 } else
2635 vcpu->sigset_active = 0;
2636 return 0;
2637}
2638
Avi Kivitybccf2152007-02-21 18:04:26 +02002639static long kvm_vcpu_ioctl(struct file *filp,
2640 unsigned int ioctl, unsigned long arg)
Avi Kivity6aa8b732006-12-10 02:21:36 -08002641{
Avi Kivitybccf2152007-02-21 18:04:26 +02002642 struct kvm_vcpu *vcpu = filp->private_data;
Al Viro2f3669872007-02-09 16:38:35 +00002643 void __user *argp = (void __user *)arg;
Carsten Otte313a3dc2007-10-11 19:16:52 +02002644 int r;
Dave Hansenfa3795a2008-08-11 10:01:46 -07002645 struct kvm_fpu *fpu = NULL;
2646 struct kvm_sregs *kvm_sregs = NULL;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002647
Avi Kivity6d4e4c42007-11-21 16:41:05 +02002648 if (vcpu->kvm->mm != current->mm)
2649 return -EIO;
Avi Kivity2122ff52010-05-13 11:25:04 +03002650
David Matlack2ea75be2014-09-19 16:03:25 -07002651 if (unlikely(_IOC_TYPE(ioctl) != KVMIO))
2652 return -EINVAL;
2653
Avi Kivity2122ff52010-05-13 11:25:04 +03002654 /*
Paolo Bonzini5cb09442017-12-12 17:41:34 +01002655 * Some architectures have vcpu ioctls that are asynchronous to vcpu
2656 * execution; mutex_lock() would break them.
Avi Kivity2122ff52010-05-13 11:25:04 +03002657 */
Paolo Bonzini5cb09442017-12-12 17:41:34 +01002658 r = kvm_arch_vcpu_async_ioctl(filp, ioctl, arg);
2659 if (r != -ENOIOCTLCMD)
Michael S. Tsirkin9fc77442012-09-16 11:50:30 +03002660 return r;
Avi Kivity2122ff52010-05-13 11:25:04 +03002661
Christoffer Dallec7660c2017-12-04 21:35:23 +01002662 if (mutex_lock_killable(&vcpu->mutex))
2663 return -EINTR;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002664 switch (ioctl) {
Christian Borntraeger0e4524a2017-07-06 14:44:28 +02002665 case KVM_RUN: {
2666 struct pid *oldpid;
Avi Kivityf0fe5102007-03-07 13:11:17 +02002667 r = -EINVAL;
2668 if (arg)
2669 goto out;
Christian Borntraeger0e4524a2017-07-06 14:44:28 +02002670 oldpid = rcu_access_pointer(vcpu->pid);
Eric W. Biederman71dbc8a2017-07-16 21:39:32 -05002671 if (unlikely(oldpid != task_pid(current))) {
Christian Borntraeger7a72f7a2014-08-05 16:44:14 +02002672 /* The thread running this VCPU changed. */
Christoffer Dallbd2a6392018-02-23 17:23:57 +01002673 struct pid *newpid;
Xiubo Lif95ef0c2015-02-26 14:58:23 +08002674
Christoffer Dallbd2a6392018-02-23 17:23:57 +01002675 r = kvm_arch_vcpu_run_pid_change(vcpu);
2676 if (r)
2677 break;
2678
2679 newpid = get_task_pid(current, PIDTYPE_PID);
Christian Borntraeger7a72f7a2014-08-05 16:44:14 +02002680 rcu_assign_pointer(vcpu->pid, newpid);
2681 if (oldpid)
2682 synchronize_rcu();
2683 put_pid(oldpid);
2684 }
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -05002685 r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run);
Gleb Natapov64be5002010-10-24 16:49:08 +02002686 trace_kvm_userspace_exit(vcpu->run->exit_reason, r);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002687 break;
Christian Borntraeger0e4524a2017-07-06 14:44:28 +02002688 }
Avi Kivity6aa8b732006-12-10 02:21:36 -08002689 case KVM_GET_REGS: {
Xiantao Zhang3e4bb3a2008-02-25 18:52:20 +08002690 struct kvm_regs *kvm_regs;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002691
Xiantao Zhang3e4bb3a2008-02-25 18:52:20 +08002692 r = -ENOMEM;
Ben Gardonb12ce362019-02-11 11:02:49 -08002693 kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL_ACCOUNT);
Xiantao Zhang3e4bb3a2008-02-25 18:52:20 +08002694 if (!kvm_regs)
2695 goto out;
2696 r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002697 if (r)
Xiantao Zhang3e4bb3a2008-02-25 18:52:20 +08002698 goto out_free1;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002699 r = -EFAULT;
Xiantao Zhang3e4bb3a2008-02-25 18:52:20 +08002700 if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs)))
2701 goto out_free1;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002702 r = 0;
Xiantao Zhang3e4bb3a2008-02-25 18:52:20 +08002703out_free1:
2704 kfree(kvm_regs);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002705 break;
2706 }
2707 case KVM_SET_REGS: {
Xiantao Zhang3e4bb3a2008-02-25 18:52:20 +08002708 struct kvm_regs *kvm_regs;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002709
Xiantao Zhang3e4bb3a2008-02-25 18:52:20 +08002710 r = -ENOMEM;
Sasha Levinff5c2c02011-12-04 19:36:29 +02002711 kvm_regs = memdup_user(argp, sizeof(*kvm_regs));
2712 if (IS_ERR(kvm_regs)) {
2713 r = PTR_ERR(kvm_regs);
Xiantao Zhang3e4bb3a2008-02-25 18:52:20 +08002714 goto out;
Sasha Levinff5c2c02011-12-04 19:36:29 +02002715 }
Xiantao Zhang3e4bb3a2008-02-25 18:52:20 +08002716 r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs);
Xiantao Zhang3e4bb3a2008-02-25 18:52:20 +08002717 kfree(kvm_regs);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002718 break;
2719 }
2720 case KVM_GET_SREGS: {
Ben Gardonb12ce362019-02-11 11:02:49 -08002721 kvm_sregs = kzalloc(sizeof(struct kvm_sregs),
2722 GFP_KERNEL_ACCOUNT);
Dave Hansenfa3795a2008-08-11 10:01:46 -07002723 r = -ENOMEM;
2724 if (!kvm_sregs)
2725 goto out;
2726 r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002727 if (r)
2728 goto out;
2729 r = -EFAULT;
Dave Hansenfa3795a2008-08-11 10:01:46 -07002730 if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs)))
Avi Kivity6aa8b732006-12-10 02:21:36 -08002731 goto out;
2732 r = 0;
2733 break;
2734 }
2735 case KVM_SET_SREGS: {
Sasha Levinff5c2c02011-12-04 19:36:29 +02002736 kvm_sregs = memdup_user(argp, sizeof(*kvm_sregs));
2737 if (IS_ERR(kvm_sregs)) {
2738 r = PTR_ERR(kvm_sregs);
Guo Chao18595412012-11-02 18:33:21 +08002739 kvm_sregs = NULL;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002740 goto out;
Sasha Levinff5c2c02011-12-04 19:36:29 +02002741 }
Dave Hansenfa3795a2008-08-11 10:01:46 -07002742 r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002743 break;
2744 }
Marcelo Tosatti62d9f0d2008-04-11 13:24:45 -03002745 case KVM_GET_MP_STATE: {
2746 struct kvm_mp_state mp_state;
2747
2748 r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state);
2749 if (r)
2750 goto out;
2751 r = -EFAULT;
Xiubo Li893bdbf2015-02-26 14:58:19 +08002752 if (copy_to_user(argp, &mp_state, sizeof(mp_state)))
Marcelo Tosatti62d9f0d2008-04-11 13:24:45 -03002753 goto out;
2754 r = 0;
2755 break;
2756 }
2757 case KVM_SET_MP_STATE: {
2758 struct kvm_mp_state mp_state;
2759
2760 r = -EFAULT;
Xiubo Li893bdbf2015-02-26 14:58:19 +08002761 if (copy_from_user(&mp_state, argp, sizeof(mp_state)))
Marcelo Tosatti62d9f0d2008-04-11 13:24:45 -03002762 goto out;
2763 r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state);
Marcelo Tosatti62d9f0d2008-04-11 13:24:45 -03002764 break;
2765 }
Avi Kivity6aa8b732006-12-10 02:21:36 -08002766 case KVM_TRANSLATE: {
2767 struct kvm_translation tr;
2768
2769 r = -EFAULT;
Xiubo Li893bdbf2015-02-26 14:58:19 +08002770 if (copy_from_user(&tr, argp, sizeof(tr)))
Avi Kivity6aa8b732006-12-10 02:21:36 -08002771 goto out;
Zhang Xiantao8b006792007-11-16 13:05:55 +08002772 r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002773 if (r)
2774 goto out;
2775 r = -EFAULT;
Xiubo Li893bdbf2015-02-26 14:58:19 +08002776 if (copy_to_user(argp, &tr, sizeof(tr)))
Avi Kivity6aa8b732006-12-10 02:21:36 -08002777 goto out;
2778 r = 0;
2779 break;
2780 }
Jan Kiszkad0bfb942008-12-15 13:52:10 +01002781 case KVM_SET_GUEST_DEBUG: {
2782 struct kvm_guest_debug dbg;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002783
2784 r = -EFAULT;
Xiubo Li893bdbf2015-02-26 14:58:19 +08002785 if (copy_from_user(&dbg, argp, sizeof(dbg)))
Avi Kivity6aa8b732006-12-10 02:21:36 -08002786 goto out;
Jan Kiszkad0bfb942008-12-15 13:52:10 +01002787 r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002788 break;
2789 }
Avi Kivity1961d272007-03-05 19:46:05 +02002790 case KVM_SET_SIGNAL_MASK: {
2791 struct kvm_signal_mask __user *sigmask_arg = argp;
2792 struct kvm_signal_mask kvm_sigmask;
2793 sigset_t sigset, *p;
2794
2795 p = NULL;
2796 if (argp) {
2797 r = -EFAULT;
2798 if (copy_from_user(&kvm_sigmask, argp,
Xiubo Li893bdbf2015-02-26 14:58:19 +08002799 sizeof(kvm_sigmask)))
Avi Kivity1961d272007-03-05 19:46:05 +02002800 goto out;
2801 r = -EINVAL;
Xiubo Li893bdbf2015-02-26 14:58:19 +08002802 if (kvm_sigmask.len != sizeof(sigset))
Avi Kivity1961d272007-03-05 19:46:05 +02002803 goto out;
2804 r = -EFAULT;
2805 if (copy_from_user(&sigset, sigmask_arg->sigset,
Xiubo Li893bdbf2015-02-26 14:58:19 +08002806 sizeof(sigset)))
Avi Kivity1961d272007-03-05 19:46:05 +02002807 goto out;
2808 p = &sigset;
2809 }
Andi Kleen376d41f2010-06-10 13:10:47 +02002810 r = kvm_vcpu_ioctl_set_sigmask(vcpu, p);
Avi Kivity1961d272007-03-05 19:46:05 +02002811 break;
2812 }
Avi Kivityb8836732007-04-01 16:34:31 +03002813 case KVM_GET_FPU: {
Ben Gardonb12ce362019-02-11 11:02:49 -08002814 fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL_ACCOUNT);
Dave Hansenfa3795a2008-08-11 10:01:46 -07002815 r = -ENOMEM;
2816 if (!fpu)
2817 goto out;
2818 r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu);
Avi Kivityb8836732007-04-01 16:34:31 +03002819 if (r)
2820 goto out;
2821 r = -EFAULT;
Dave Hansenfa3795a2008-08-11 10:01:46 -07002822 if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu)))
Avi Kivityb8836732007-04-01 16:34:31 +03002823 goto out;
2824 r = 0;
2825 break;
2826 }
2827 case KVM_SET_FPU: {
Sasha Levinff5c2c02011-12-04 19:36:29 +02002828 fpu = memdup_user(argp, sizeof(*fpu));
2829 if (IS_ERR(fpu)) {
2830 r = PTR_ERR(fpu);
Guo Chao18595412012-11-02 18:33:21 +08002831 fpu = NULL;
Avi Kivityb8836732007-04-01 16:34:31 +03002832 goto out;
Sasha Levinff5c2c02011-12-04 19:36:29 +02002833 }
Dave Hansenfa3795a2008-08-11 10:01:46 -07002834 r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu);
Avi Kivityb8836732007-04-01 16:34:31 +03002835 break;
2836 }
Avi Kivitybccf2152007-02-21 18:04:26 +02002837 default:
Carsten Otte313a3dc2007-10-11 19:16:52 +02002838 r = kvm_arch_vcpu_ioctl(filp, ioctl, arg);
Avi Kivitybccf2152007-02-21 18:04:26 +02002839 }
2840out:
Christoffer Dallec7660c2017-12-04 21:35:23 +01002841 mutex_unlock(&vcpu->mutex);
Dave Hansenfa3795a2008-08-11 10:01:46 -07002842 kfree(fpu);
2843 kfree(kvm_sregs);
Avi Kivitybccf2152007-02-21 18:04:26 +02002844 return r;
2845}
2846
Christian Borntraegerde8e5d72015-02-03 09:35:15 +01002847#ifdef CONFIG_KVM_COMPAT
Alexander Graf1dda6062011-06-08 02:45:37 +02002848static long kvm_vcpu_compat_ioctl(struct file *filp,
2849 unsigned int ioctl, unsigned long arg)
2850{
2851 struct kvm_vcpu *vcpu = filp->private_data;
2852 void __user *argp = compat_ptr(arg);
2853 int r;
2854
2855 if (vcpu->kvm->mm != current->mm)
2856 return -EIO;
2857
2858 switch (ioctl) {
2859 case KVM_SET_SIGNAL_MASK: {
2860 struct kvm_signal_mask __user *sigmask_arg = argp;
2861 struct kvm_signal_mask kvm_sigmask;
Alexander Graf1dda6062011-06-08 02:45:37 +02002862 sigset_t sigset;
2863
2864 if (argp) {
2865 r = -EFAULT;
2866 if (copy_from_user(&kvm_sigmask, argp,
Xiubo Li893bdbf2015-02-26 14:58:19 +08002867 sizeof(kvm_sigmask)))
Alexander Graf1dda6062011-06-08 02:45:37 +02002868 goto out;
2869 r = -EINVAL;
Al Viro3968cf62017-09-03 21:45:17 -04002870 if (kvm_sigmask.len != sizeof(compat_sigset_t))
Alexander Graf1dda6062011-06-08 02:45:37 +02002871 goto out;
2872 r = -EFAULT;
Al Viro3968cf62017-09-03 21:45:17 -04002873 if (get_compat_sigset(&sigset, (void *)sigmask_arg->sigset))
Alexander Graf1dda6062011-06-08 02:45:37 +02002874 goto out;
Alan Cox760a9a32012-08-22 14:34:11 +01002875 r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
2876 } else
2877 r = kvm_vcpu_ioctl_set_sigmask(vcpu, NULL);
Alexander Graf1dda6062011-06-08 02:45:37 +02002878 break;
2879 }
2880 default:
2881 r = kvm_vcpu_ioctl(filp, ioctl, arg);
2882 }
2883
2884out:
2885 return r;
2886}
2887#endif
2888
Scott Wood852b6d52013-04-12 14:08:42 +00002889static int kvm_device_ioctl_attr(struct kvm_device *dev,
2890 int (*accessor)(struct kvm_device *dev,
2891 struct kvm_device_attr *attr),
2892 unsigned long arg)
2893{
2894 struct kvm_device_attr attr;
2895
2896 if (!accessor)
2897 return -EPERM;
2898
2899 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
2900 return -EFAULT;
2901
2902 return accessor(dev, &attr);
2903}
2904
2905static long kvm_device_ioctl(struct file *filp, unsigned int ioctl,
2906 unsigned long arg)
2907{
2908 struct kvm_device *dev = filp->private_data;
2909
2910 switch (ioctl) {
2911 case KVM_SET_DEVICE_ATTR:
2912 return kvm_device_ioctl_attr(dev, dev->ops->set_attr, arg);
2913 case KVM_GET_DEVICE_ATTR:
2914 return kvm_device_ioctl_attr(dev, dev->ops->get_attr, arg);
2915 case KVM_HAS_DEVICE_ATTR:
2916 return kvm_device_ioctl_attr(dev, dev->ops->has_attr, arg);
2917 default:
2918 if (dev->ops->ioctl)
2919 return dev->ops->ioctl(dev, ioctl, arg);
2920
2921 return -ENOTTY;
2922 }
2923}
2924
Scott Wood852b6d52013-04-12 14:08:42 +00002925static int kvm_device_release(struct inode *inode, struct file *filp)
2926{
2927 struct kvm_device *dev = filp->private_data;
2928 struct kvm *kvm = dev->kvm;
2929
Scott Wood852b6d52013-04-12 14:08:42 +00002930 kvm_put_kvm(kvm);
2931 return 0;
2932}
2933
2934static const struct file_operations kvm_device_fops = {
2935 .unlocked_ioctl = kvm_device_ioctl,
2936 .release = kvm_device_release,
Marc Zyngier7ddfd3e2018-06-17 10:16:21 +01002937 KVM_COMPAT(kvm_device_ioctl),
Scott Wood852b6d52013-04-12 14:08:42 +00002938};
2939
2940struct kvm_device *kvm_device_from_filp(struct file *filp)
2941{
2942 if (filp->f_op != &kvm_device_fops)
2943 return NULL;
2944
2945 return filp->private_data;
2946}
2947
Will Deacond60eacb2014-09-02 10:27:33 +01002948static struct kvm_device_ops *kvm_device_ops_table[KVM_DEV_TYPE_MAX] = {
2949#ifdef CONFIG_KVM_MPIC
2950 [KVM_DEV_TYPE_FSL_MPIC_20] = &kvm_mpic_ops,
2951 [KVM_DEV_TYPE_FSL_MPIC_42] = &kvm_mpic_ops,
2952#endif
Will Deacond60eacb2014-09-02 10:27:33 +01002953};
2954
2955int kvm_register_device_ops(struct kvm_device_ops *ops, u32 type)
2956{
2957 if (type >= ARRAY_SIZE(kvm_device_ops_table))
2958 return -ENOSPC;
2959
2960 if (kvm_device_ops_table[type] != NULL)
2961 return -EEXIST;
2962
2963 kvm_device_ops_table[type] = ops;
2964 return 0;
2965}
2966
Wanpeng Li571ee1b2014-10-09 18:30:08 +08002967void kvm_unregister_device_ops(u32 type)
2968{
2969 if (kvm_device_ops_table[type] != NULL)
2970 kvm_device_ops_table[type] = NULL;
2971}
2972
Scott Wood852b6d52013-04-12 14:08:42 +00002973static int kvm_ioctl_create_device(struct kvm *kvm,
2974 struct kvm_create_device *cd)
2975{
2976 struct kvm_device_ops *ops = NULL;
2977 struct kvm_device *dev;
2978 bool test = cd->flags & KVM_CREATE_DEVICE_TEST;
2979 int ret;
2980
Will Deacond60eacb2014-09-02 10:27:33 +01002981 if (cd->type >= ARRAY_SIZE(kvm_device_ops_table))
Scott Wood852b6d52013-04-12 14:08:42 +00002982 return -ENODEV;
Will Deacond60eacb2014-09-02 10:27:33 +01002983
2984 ops = kvm_device_ops_table[cd->type];
2985 if (ops == NULL)
2986 return -ENODEV;
Scott Wood852b6d52013-04-12 14:08:42 +00002987
2988 if (test)
2989 return 0;
2990
Ben Gardonb12ce362019-02-11 11:02:49 -08002991 dev = kzalloc(sizeof(*dev), GFP_KERNEL_ACCOUNT);
Scott Wood852b6d52013-04-12 14:08:42 +00002992 if (!dev)
2993 return -ENOMEM;
2994
2995 dev->ops = ops;
2996 dev->kvm = kvm;
Scott Wood852b6d52013-04-12 14:08:42 +00002997
Christoffer Dalla28ebea2016-08-09 19:13:01 +02002998 mutex_lock(&kvm->lock);
Scott Wood852b6d52013-04-12 14:08:42 +00002999 ret = ops->create(dev, cd->type);
3000 if (ret < 0) {
Christoffer Dalla28ebea2016-08-09 19:13:01 +02003001 mutex_unlock(&kvm->lock);
Scott Wood852b6d52013-04-12 14:08:42 +00003002 kfree(dev);
3003 return ret;
3004 }
Christoffer Dalla28ebea2016-08-09 19:13:01 +02003005 list_add(&dev->vm_node, &kvm->devices);
3006 mutex_unlock(&kvm->lock);
Scott Wood852b6d52013-04-12 14:08:42 +00003007
Christoffer Dall023e9fd2016-08-09 19:13:00 +02003008 if (ops->init)
3009 ops->init(dev);
3010
Jann Horncfa39382019-01-26 01:54:33 +01003011 kvm_get_kvm(kvm);
Yann Droneaud24009b02013-08-24 22:14:07 +02003012 ret = anon_inode_getfd(ops->name, &kvm_device_fops, dev, O_RDWR | O_CLOEXEC);
Scott Wood852b6d52013-04-12 14:08:42 +00003013 if (ret < 0) {
Jann Horncfa39382019-01-26 01:54:33 +01003014 kvm_put_kvm(kvm);
Christoffer Dalla28ebea2016-08-09 19:13:01 +02003015 mutex_lock(&kvm->lock);
3016 list_del(&dev->vm_node);
3017 mutex_unlock(&kvm->lock);
Dan Carpentera0f1d212016-11-30 22:21:05 +03003018 ops->destroy(dev);
Scott Wood852b6d52013-04-12 14:08:42 +00003019 return ret;
3020 }
3021
Scott Wood852b6d52013-04-12 14:08:42 +00003022 cd->fd = ret;
3023 return 0;
3024}
3025
Alexander Graf92b591a2014-07-14 18:33:08 +02003026static long kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg)
3027{
3028 switch (arg) {
3029 case KVM_CAP_USER_MEMORY:
3030 case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
3031 case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS:
Alexander Graf92b591a2014-07-14 18:33:08 +02003032 case KVM_CAP_INTERNAL_ERROR_DATA:
3033#ifdef CONFIG_HAVE_KVM_MSI
3034 case KVM_CAP_SIGNAL_MSI:
3035#endif
Paul Mackerras297e2102014-06-30 20:51:13 +10003036#ifdef CONFIG_HAVE_KVM_IRQFD
Paolo Bonzinidc9be0f2015-03-05 11:54:46 +01003037 case KVM_CAP_IRQFD:
Alexander Graf92b591a2014-07-14 18:33:08 +02003038 case KVM_CAP_IRQFD_RESAMPLE:
3039#endif
Jason Wange9ea5062015-09-15 14:41:59 +08003040 case KVM_CAP_IOEVENTFD_ANY_LENGTH:
Alexander Graf92b591a2014-07-14 18:33:08 +02003041 case KVM_CAP_CHECK_EXTENSION_VM:
Paolo Bonzinie5d83c72017-02-16 10:40:56 +01003042 case KVM_CAP_ENABLE_CAP_VM:
Paolo Bonzini2a31b9d2018-10-23 02:36:47 +02003043#ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
3044 case KVM_CAP_MANUAL_DIRTY_LOG_PROTECT:
3045#endif
Alexander Graf92b591a2014-07-14 18:33:08 +02003046 return 1;
Paolo Bonzini4b4357e2017-03-31 13:53:23 +02003047#ifdef CONFIG_KVM_MMIO
Paolo Bonzini30422552017-03-31 13:53:22 +02003048 case KVM_CAP_COALESCED_MMIO:
3049 return KVM_COALESCED_MMIO_PAGE_OFFSET;
Peng Hao0804c842018-10-14 07:09:55 +08003050 case KVM_CAP_COALESCED_PIO:
3051 return 1;
Paolo Bonzini30422552017-03-31 13:53:22 +02003052#endif
Alexander Graf92b591a2014-07-14 18:33:08 +02003053#ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
3054 case KVM_CAP_IRQ_ROUTING:
3055 return KVM_MAX_IRQ_ROUTES;
3056#endif
Paolo Bonzinif481b062015-05-17 17:30:37 +02003057#if KVM_ADDRESS_SPACE_NUM > 1
3058 case KVM_CAP_MULTI_ADDRESS_SPACE:
3059 return KVM_ADDRESS_SPACE_NUM;
3060#endif
Greg Kurz0b1b1df2016-05-09 18:13:37 +02003061 case KVM_CAP_MAX_VCPU_ID:
3062 return KVM_MAX_VCPU_ID;
Alexander Graf92b591a2014-07-14 18:33:08 +02003063 default:
3064 break;
3065 }
3066 return kvm_vm_ioctl_check_extension(kvm, arg);
3067}
3068
Paolo Bonzinie5d83c72017-02-16 10:40:56 +01003069int __attribute__((weak)) kvm_vm_ioctl_enable_cap(struct kvm *kvm,
3070 struct kvm_enable_cap *cap)
3071{
3072 return -EINVAL;
3073}
3074
3075static int kvm_vm_ioctl_enable_cap_generic(struct kvm *kvm,
3076 struct kvm_enable_cap *cap)
3077{
3078 switch (cap->cap) {
Paolo Bonzini2a31b9d2018-10-23 02:36:47 +02003079#ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
3080 case KVM_CAP_MANUAL_DIRTY_LOG_PROTECT:
3081 if (cap->flags || (cap->args[0] & ~1))
3082 return -EINVAL;
3083 kvm->manual_dirty_log_protect = cap->args[0];
3084 return 0;
3085#endif
Paolo Bonzinie5d83c72017-02-16 10:40:56 +01003086 default:
3087 return kvm_vm_ioctl_enable_cap(kvm, cap);
3088 }
3089}
3090
Avi Kivitybccf2152007-02-21 18:04:26 +02003091static long kvm_vm_ioctl(struct file *filp,
3092 unsigned int ioctl, unsigned long arg)
3093{
3094 struct kvm *kvm = filp->private_data;
3095 void __user *argp = (void __user *)arg;
Carsten Otte1fe779f2007-10-29 16:08:35 +01003096 int r;
Avi Kivitybccf2152007-02-21 18:04:26 +02003097
Avi Kivity6d4e4c42007-11-21 16:41:05 +02003098 if (kvm->mm != current->mm)
3099 return -EIO;
Avi Kivitybccf2152007-02-21 18:04:26 +02003100 switch (ioctl) {
3101 case KVM_CREATE_VCPU:
3102 r = kvm_vm_ioctl_create_vcpu(kvm, arg);
Avi Kivitybccf2152007-02-21 18:04:26 +02003103 break;
Paolo Bonzinie5d83c72017-02-16 10:40:56 +01003104 case KVM_ENABLE_CAP: {
3105 struct kvm_enable_cap cap;
3106
3107 r = -EFAULT;
3108 if (copy_from_user(&cap, argp, sizeof(cap)))
3109 goto out;
3110 r = kvm_vm_ioctl_enable_cap_generic(kvm, &cap);
3111 break;
3112 }
Izik Eidus6fc138d2007-10-09 19:20:39 +02003113 case KVM_SET_USER_MEMORY_REGION: {
3114 struct kvm_userspace_memory_region kvm_userspace_mem;
3115
3116 r = -EFAULT;
3117 if (copy_from_user(&kvm_userspace_mem, argp,
Xiubo Li893bdbf2015-02-26 14:58:19 +08003118 sizeof(kvm_userspace_mem)))
Izik Eidus6fc138d2007-10-09 19:20:39 +02003119 goto out;
3120
Takuya Yoshikawa47ae31e2013-02-27 19:43:00 +09003121 r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem);
Avi Kivity6aa8b732006-12-10 02:21:36 -08003122 break;
3123 }
3124 case KVM_GET_DIRTY_LOG: {
3125 struct kvm_dirty_log log;
3126
3127 r = -EFAULT;
Xiubo Li893bdbf2015-02-26 14:58:19 +08003128 if (copy_from_user(&log, argp, sizeof(log)))
Avi Kivity6aa8b732006-12-10 02:21:36 -08003129 goto out;
Avi Kivity2c6f5df2007-02-20 18:27:58 +02003130 r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
Avi Kivity6aa8b732006-12-10 02:21:36 -08003131 break;
3132 }
Paolo Bonzini2a31b9d2018-10-23 02:36:47 +02003133#ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
3134 case KVM_CLEAR_DIRTY_LOG: {
3135 struct kvm_clear_dirty_log log;
3136
3137 r = -EFAULT;
3138 if (copy_from_user(&log, argp, sizeof(log)))
3139 goto out;
3140 r = kvm_vm_ioctl_clear_dirty_log(kvm, &log);
3141 break;
3142 }
3143#endif
Paolo Bonzini4b4357e2017-03-31 13:53:23 +02003144#ifdef CONFIG_KVM_MMIO
Laurent Vivier5f94c172008-05-30 16:05:54 +02003145 case KVM_REGISTER_COALESCED_MMIO: {
3146 struct kvm_coalesced_mmio_zone zone;
Xiubo Lif95ef0c2015-02-26 14:58:23 +08003147
Laurent Vivier5f94c172008-05-30 16:05:54 +02003148 r = -EFAULT;
Xiubo Li893bdbf2015-02-26 14:58:19 +08003149 if (copy_from_user(&zone, argp, sizeof(zone)))
Laurent Vivier5f94c172008-05-30 16:05:54 +02003150 goto out;
Laurent Vivier5f94c172008-05-30 16:05:54 +02003151 r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone);
Laurent Vivier5f94c172008-05-30 16:05:54 +02003152 break;
3153 }
3154 case KVM_UNREGISTER_COALESCED_MMIO: {
3155 struct kvm_coalesced_mmio_zone zone;
Xiubo Lif95ef0c2015-02-26 14:58:23 +08003156
Laurent Vivier5f94c172008-05-30 16:05:54 +02003157 r = -EFAULT;
Xiubo Li893bdbf2015-02-26 14:58:19 +08003158 if (copy_from_user(&zone, argp, sizeof(zone)))
Laurent Vivier5f94c172008-05-30 16:05:54 +02003159 goto out;
Laurent Vivier5f94c172008-05-30 16:05:54 +02003160 r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone);
Laurent Vivier5f94c172008-05-30 16:05:54 +02003161 break;
3162 }
3163#endif
Gregory Haskins721eecbf2009-05-20 10:30:49 -04003164 case KVM_IRQFD: {
3165 struct kvm_irqfd data;
3166
3167 r = -EFAULT;
Xiubo Li893bdbf2015-02-26 14:58:19 +08003168 if (copy_from_user(&data, argp, sizeof(data)))
Gregory Haskins721eecbf2009-05-20 10:30:49 -04003169 goto out;
Alex Williamsond4db2932012-06-29 09:56:08 -06003170 r = kvm_irqfd(kvm, &data);
Gregory Haskins721eecbf2009-05-20 10:30:49 -04003171 break;
3172 }
Gregory Haskinsd34e6b12009-07-07 17:08:49 -04003173 case KVM_IOEVENTFD: {
3174 struct kvm_ioeventfd data;
3175
3176 r = -EFAULT;
Xiubo Li893bdbf2015-02-26 14:58:19 +08003177 if (copy_from_user(&data, argp, sizeof(data)))
Gregory Haskinsd34e6b12009-07-07 17:08:49 -04003178 goto out;
3179 r = kvm_ioeventfd(kvm, &data);
3180 break;
3181 }
Jan Kiszka07975ad2012-03-29 21:14:12 +02003182#ifdef CONFIG_HAVE_KVM_MSI
3183 case KVM_SIGNAL_MSI: {
3184 struct kvm_msi msi;
3185
3186 r = -EFAULT;
Xiubo Li893bdbf2015-02-26 14:58:19 +08003187 if (copy_from_user(&msi, argp, sizeof(msi)))
Jan Kiszka07975ad2012-03-29 21:14:12 +02003188 goto out;
3189 r = kvm_send_userspace_msi(kvm, &msi);
3190 break;
3191 }
3192#endif
Christoffer Dall23d43cf2012-07-24 08:51:20 -04003193#ifdef __KVM_HAVE_IRQ_LINE
3194 case KVM_IRQ_LINE_STATUS:
3195 case KVM_IRQ_LINE: {
3196 struct kvm_irq_level irq_event;
3197
3198 r = -EFAULT;
Xiubo Li893bdbf2015-02-26 14:58:19 +08003199 if (copy_from_user(&irq_event, argp, sizeof(irq_event)))
Christoffer Dall23d43cf2012-07-24 08:51:20 -04003200 goto out;
3201
Yang Zhangaa2fbe62013-04-11 19:21:40 +08003202 r = kvm_vm_ioctl_irq_line(kvm, &irq_event,
3203 ioctl == KVM_IRQ_LINE_STATUS);
Christoffer Dall23d43cf2012-07-24 08:51:20 -04003204 if (r)
3205 goto out;
3206
3207 r = -EFAULT;
3208 if (ioctl == KVM_IRQ_LINE_STATUS) {
Xiubo Li893bdbf2015-02-26 14:58:19 +08003209 if (copy_to_user(argp, &irq_event, sizeof(irq_event)))
Christoffer Dall23d43cf2012-07-24 08:51:20 -04003210 goto out;
3211 }
3212
3213 r = 0;
3214 break;
3215 }
3216#endif
Alexander Grafaa8d5942013-04-15 21:12:53 +02003217#ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
3218 case KVM_SET_GSI_ROUTING: {
3219 struct kvm_irq_routing routing;
3220 struct kvm_irq_routing __user *urouting;
Paolo Bonzinif8c1b852016-06-01 14:09:22 +02003221 struct kvm_irq_routing_entry *entries = NULL;
Alexander Grafaa8d5942013-04-15 21:12:53 +02003222
3223 r = -EFAULT;
3224 if (copy_from_user(&routing, argp, sizeof(routing)))
3225 goto out;
3226 r = -EINVAL;
David Hildenbrand5c0aea02017-04-28 17:06:20 +02003227 if (!kvm_arch_can_set_irq_routing(kvm))
3228 goto out;
Xiubo Licaf1ff22016-06-15 18:00:33 +08003229 if (routing.nr > KVM_MAX_IRQ_ROUTES)
Alexander Grafaa8d5942013-04-15 21:12:53 +02003230 goto out;
3231 if (routing.flags)
3232 goto out;
Paolo Bonzinif8c1b852016-06-01 14:09:22 +02003233 if (routing.nr) {
3234 r = -ENOMEM;
Kees Cook42bc47b2018-06-12 14:27:11 -07003235 entries = vmalloc(array_size(sizeof(*entries),
3236 routing.nr));
Paolo Bonzinif8c1b852016-06-01 14:09:22 +02003237 if (!entries)
3238 goto out;
3239 r = -EFAULT;
3240 urouting = argp;
3241 if (copy_from_user(entries, urouting->entries,
3242 routing.nr * sizeof(*entries)))
3243 goto out_free_irq_routing;
3244 }
Alexander Grafaa8d5942013-04-15 21:12:53 +02003245 r = kvm_set_irq_routing(kvm, entries, routing.nr,
3246 routing.flags);
Xiubo Lia642a172015-02-26 14:58:20 +08003247out_free_irq_routing:
Alexander Grafaa8d5942013-04-15 21:12:53 +02003248 vfree(entries);
3249 break;
3250 }
3251#endif /* CONFIG_HAVE_KVM_IRQ_ROUTING */
Scott Wood852b6d52013-04-12 14:08:42 +00003252 case KVM_CREATE_DEVICE: {
3253 struct kvm_create_device cd;
3254
3255 r = -EFAULT;
3256 if (copy_from_user(&cd, argp, sizeof(cd)))
3257 goto out;
3258
3259 r = kvm_ioctl_create_device(kvm, &cd);
3260 if (r)
3261 goto out;
3262
3263 r = -EFAULT;
3264 if (copy_to_user(argp, &cd, sizeof(cd)))
3265 goto out;
3266
3267 r = 0;
3268 break;
3269 }
Alexander Graf92b591a2014-07-14 18:33:08 +02003270 case KVM_CHECK_EXTENSION:
3271 r = kvm_vm_ioctl_check_extension_generic(kvm, arg);
3272 break;
Avi Kivityf17abe92007-02-21 19:28:04 +02003273 default:
Carsten Otte1fe779f2007-10-29 16:08:35 +01003274 r = kvm_arch_vm_ioctl(filp, ioctl, arg);
Avi Kivityf17abe92007-02-21 19:28:04 +02003275 }
3276out:
3277 return r;
3278}
3279
Christian Borntraegerde8e5d72015-02-03 09:35:15 +01003280#ifdef CONFIG_KVM_COMPAT
Arnd Bergmann6ff58942009-10-22 14:19:27 +02003281struct compat_kvm_dirty_log {
3282 __u32 slot;
3283 __u32 padding1;
3284 union {
3285 compat_uptr_t dirty_bitmap; /* one bit per page */
3286 __u64 padding2;
3287 };
3288};
3289
3290static long kvm_vm_compat_ioctl(struct file *filp,
3291 unsigned int ioctl, unsigned long arg)
3292{
3293 struct kvm *kvm = filp->private_data;
3294 int r;
3295
3296 if (kvm->mm != current->mm)
3297 return -EIO;
3298 switch (ioctl) {
3299 case KVM_GET_DIRTY_LOG: {
3300 struct compat_kvm_dirty_log compat_log;
3301 struct kvm_dirty_log log;
3302
Arnd Bergmann6ff58942009-10-22 14:19:27 +02003303 if (copy_from_user(&compat_log, (void __user *)arg,
3304 sizeof(compat_log)))
Markus Elfringf6a3b162017-01-22 11:30:21 +01003305 return -EFAULT;
Arnd Bergmann6ff58942009-10-22 14:19:27 +02003306 log.slot = compat_log.slot;
3307 log.padding1 = compat_log.padding1;
3308 log.padding2 = compat_log.padding2;
3309 log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap);
3310
3311 r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
Arnd Bergmann6ff58942009-10-22 14:19:27 +02003312 break;
3313 }
3314 default:
3315 r = kvm_vm_ioctl(filp, ioctl, arg);
3316 }
Arnd Bergmann6ff58942009-10-22 14:19:27 +02003317 return r;
3318}
3319#endif
3320
Christian Borntraeger3d3aab12008-12-02 11:17:32 +01003321static struct file_operations kvm_vm_fops = {
Avi Kivityf17abe92007-02-21 19:28:04 +02003322 .release = kvm_vm_release,
3323 .unlocked_ioctl = kvm_vm_ioctl,
Arnd Bergmann6038f372010-08-15 18:52:59 +02003324 .llseek = noop_llseek,
Marc Zyngier7ddfd3e2018-06-17 10:16:21 +01003325 KVM_COMPAT(kvm_vm_compat_ioctl),
Avi Kivityf17abe92007-02-21 19:28:04 +02003326};
3327
Carsten Ottee08b9632012-01-04 10:25:20 +01003328static int kvm_dev_ioctl_create_vm(unsigned long type)
Avi Kivityf17abe92007-02-21 19:28:04 +02003329{
Heiko Carstensaac87632010-10-27 17:22:10 +02003330 int r;
Avi Kivityf17abe92007-02-21 19:28:04 +02003331 struct kvm *kvm;
Al Viro506cfba2016-07-14 18:54:17 +02003332 struct file *file;
Avi Kivityf17abe92007-02-21 19:28:04 +02003333
Carsten Ottee08b9632012-01-04 10:25:20 +01003334 kvm = kvm_create_vm(type);
Avi Kivityd6d28162007-06-28 08:38:16 -04003335 if (IS_ERR(kvm))
3336 return PTR_ERR(kvm);
Paolo Bonzini4b4357e2017-03-31 13:53:23 +02003337#ifdef CONFIG_KVM_MMIO
Takuya Yoshikawa6ce5a092010-03-15 22:13:30 +09003338 r = kvm_coalesced_mmio_init(kvm);
Markus Elfring78588332017-11-21 13:40:17 +01003339 if (r < 0)
3340 goto put_kvm;
Takuya Yoshikawa6ce5a092010-03-15 22:13:30 +09003341#endif
Al Viro506cfba2016-07-14 18:54:17 +02003342 r = get_unused_fd_flags(O_CLOEXEC);
Markus Elfring78588332017-11-21 13:40:17 +01003343 if (r < 0)
3344 goto put_kvm;
3345
Al Viro506cfba2016-07-14 18:54:17 +02003346 file = anon_inode_getfile("kvm-vm", &kvm_vm_fops, kvm, O_RDWR);
3347 if (IS_ERR(file)) {
3348 put_unused_fd(r);
Markus Elfring78588332017-11-21 13:40:17 +01003349 r = PTR_ERR(file);
3350 goto put_kvm;
Al Viro506cfba2016-07-14 18:54:17 +02003351 }
Janosch Frank536a6f82016-05-18 13:26:23 +02003352
Paolo Bonzini525df862017-06-27 15:45:09 +02003353 /*
3354 * Don't call kvm_put_kvm anymore at this point; file->f_op is
3355 * already set, with ->release() being kvm_vm_release(). In error
3356 * cases it will be called by the final fput(file) and will take
3357 * care of doing kvm_put_kvm(kvm).
3358 */
Janosch Frank536a6f82016-05-18 13:26:23 +02003359 if (kvm_create_vm_debugfs(kvm, r) < 0) {
Al Viro506cfba2016-07-14 18:54:17 +02003360 put_unused_fd(r);
3361 fput(file);
Janosch Frank536a6f82016-05-18 13:26:23 +02003362 return -ENOMEM;
3363 }
Claudio Imbrenda286de8f2017-07-12 17:56:44 +02003364 kvm_uevent_notify_change(KVM_EVENT_CREATE_VM, kvm);
Avi Kivityf17abe92007-02-21 19:28:04 +02003365
Al Viro506cfba2016-07-14 18:54:17 +02003366 fd_install(r, file);
Heiko Carstensaac87632010-10-27 17:22:10 +02003367 return r;
Markus Elfring78588332017-11-21 13:40:17 +01003368
3369put_kvm:
3370 kvm_put_kvm(kvm);
3371 return r;
Avi Kivityf17abe92007-02-21 19:28:04 +02003372}
3373
3374static long kvm_dev_ioctl(struct file *filp,
3375 unsigned int ioctl, unsigned long arg)
3376{
Avi Kivity07c45a32007-03-07 13:05:38 +02003377 long r = -EINVAL;
Avi Kivityf17abe92007-02-21 19:28:04 +02003378
3379 switch (ioctl) {
3380 case KVM_GET_API_VERSION:
Avi Kivityf0fe5102007-03-07 13:11:17 +02003381 if (arg)
3382 goto out;
Avi Kivityf17abe92007-02-21 19:28:04 +02003383 r = KVM_API_VERSION;
3384 break;
3385 case KVM_CREATE_VM:
Carsten Ottee08b9632012-01-04 10:25:20 +01003386 r = kvm_dev_ioctl_create_vm(arg);
Avi Kivityf17abe92007-02-21 19:28:04 +02003387 break;
Zhang Xiantao018d00d2007-11-15 23:07:47 +08003388 case KVM_CHECK_EXTENSION:
Alexander Graf784aa3d2014-07-14 18:27:35 +02003389 r = kvm_vm_ioctl_check_extension_generic(NULL, arg);
Avi Kivity5d308f42007-03-01 17:56:20 +02003390 break;
Avi Kivity07c45a32007-03-07 13:05:38 +02003391 case KVM_GET_VCPU_MMAP_SIZE:
Avi Kivity07c45a32007-03-07 13:05:38 +02003392 if (arg)
3393 goto out;
Avi Kivityadb1ff42008-01-24 15:13:08 +02003394 r = PAGE_SIZE; /* struct kvm_run */
3395#ifdef CONFIG_X86
3396 r += PAGE_SIZE; /* pio data page */
3397#endif
Paolo Bonzini4b4357e2017-03-31 13:53:23 +02003398#ifdef CONFIG_KVM_MMIO
Laurent Vivier5f94c172008-05-30 16:05:54 +02003399 r += PAGE_SIZE; /* coalesced mmio ring page */
3400#endif
Avi Kivity07c45a32007-03-07 13:05:38 +02003401 break;
Feng(Eric) Liud4c9ff22008-04-10 08:47:53 -04003402 case KVM_TRACE_ENABLE:
3403 case KVM_TRACE_PAUSE:
3404 case KVM_TRACE_DISABLE:
Marcelo Tosatti2023a292009-06-18 11:47:28 -03003405 r = -EOPNOTSUPP;
Feng(Eric) Liud4c9ff22008-04-10 08:47:53 -04003406 break;
Avi Kivity6aa8b732006-12-10 02:21:36 -08003407 default:
Carsten Otte043405e2007-10-10 17:16:19 +02003408 return kvm_arch_dev_ioctl(filp, ioctl, arg);
Avi Kivity6aa8b732006-12-10 02:21:36 -08003409 }
3410out:
3411 return r;
3412}
3413
Avi Kivity6aa8b732006-12-10 02:21:36 -08003414static struct file_operations kvm_chardev_ops = {
Avi Kivity6aa8b732006-12-10 02:21:36 -08003415 .unlocked_ioctl = kvm_dev_ioctl,
Arnd Bergmann6038f372010-08-15 18:52:59 +02003416 .llseek = noop_llseek,
Marc Zyngier7ddfd3e2018-06-17 10:16:21 +01003417 KVM_COMPAT(kvm_dev_ioctl),
Avi Kivity6aa8b732006-12-10 02:21:36 -08003418};
3419
3420static struct miscdevice kvm_dev = {
Avi Kivitybbe44322007-03-04 13:27:36 +02003421 KVM_MINOR,
Avi Kivity6aa8b732006-12-10 02:21:36 -08003422 "kvm",
3423 &kvm_chardev_ops,
3424};
3425
Takuya Yoshikawa75b71272010-11-16 17:37:41 +09003426static void hardware_enable_nolock(void *junk)
Avi Kivity1b6c0162007-05-24 13:03:52 +03003427{
3428 int cpu = raw_smp_processor_id();
Alexander Graf10474ae2009-09-15 11:37:46 +02003429 int r;
Avi Kivity1b6c0162007-05-24 13:03:52 +03003430
Rusty Russell7f59f492008-12-07 21:25:45 +10303431 if (cpumask_test_cpu(cpu, cpus_hardware_enabled))
Avi Kivity1b6c0162007-05-24 13:03:52 +03003432 return;
Alexander Graf10474ae2009-09-15 11:37:46 +02003433
Rusty Russell7f59f492008-12-07 21:25:45 +10303434 cpumask_set_cpu(cpu, cpus_hardware_enabled);
Alexander Graf10474ae2009-09-15 11:37:46 +02003435
Radim Krčmář13a34e02014-08-28 15:13:03 +02003436 r = kvm_arch_hardware_enable();
Alexander Graf10474ae2009-09-15 11:37:46 +02003437
3438 if (r) {
3439 cpumask_clear_cpu(cpu, cpus_hardware_enabled);
3440 atomic_inc(&hardware_enable_failed);
Xiubo Li1170adc2015-02-26 14:58:26 +08003441 pr_info("kvm: enabling virtualization on CPU%d failed\n", cpu);
Alexander Graf10474ae2009-09-15 11:37:46 +02003442 }
Avi Kivity1b6c0162007-05-24 13:03:52 +03003443}
3444
Thomas Gleixner8c18b2d2016-07-13 17:16:37 +00003445static int kvm_starting_cpu(unsigned int cpu)
Takuya Yoshikawa75b71272010-11-16 17:37:41 +09003446{
Paolo Bonzini4a937f92013-09-10 12:58:35 +02003447 raw_spin_lock(&kvm_count_lock);
Paolo Bonzini4fa92fb2013-09-10 12:57:17 +02003448 if (kvm_usage_count)
3449 hardware_enable_nolock(NULL);
Paolo Bonzini4a937f92013-09-10 12:58:35 +02003450 raw_spin_unlock(&kvm_count_lock);
Thomas Gleixner8c18b2d2016-07-13 17:16:37 +00003451 return 0;
Takuya Yoshikawa75b71272010-11-16 17:37:41 +09003452}
3453
3454static void hardware_disable_nolock(void *junk)
Avi Kivity1b6c0162007-05-24 13:03:52 +03003455{
3456 int cpu = raw_smp_processor_id();
3457
Rusty Russell7f59f492008-12-07 21:25:45 +10303458 if (!cpumask_test_cpu(cpu, cpus_hardware_enabled))
Avi Kivity1b6c0162007-05-24 13:03:52 +03003459 return;
Rusty Russell7f59f492008-12-07 21:25:45 +10303460 cpumask_clear_cpu(cpu, cpus_hardware_enabled);
Radim Krčmář13a34e02014-08-28 15:13:03 +02003461 kvm_arch_hardware_disable();
Avi Kivity1b6c0162007-05-24 13:03:52 +03003462}
3463
Thomas Gleixner8c18b2d2016-07-13 17:16:37 +00003464static int kvm_dying_cpu(unsigned int cpu)
Takuya Yoshikawa75b71272010-11-16 17:37:41 +09003465{
Paolo Bonzini4a937f92013-09-10 12:58:35 +02003466 raw_spin_lock(&kvm_count_lock);
Paolo Bonzini4fa92fb2013-09-10 12:57:17 +02003467 if (kvm_usage_count)
3468 hardware_disable_nolock(NULL);
Paolo Bonzini4a937f92013-09-10 12:58:35 +02003469 raw_spin_unlock(&kvm_count_lock);
Thomas Gleixner8c18b2d2016-07-13 17:16:37 +00003470 return 0;
Takuya Yoshikawa75b71272010-11-16 17:37:41 +09003471}
3472
Alexander Graf10474ae2009-09-15 11:37:46 +02003473static void hardware_disable_all_nolock(void)
3474{
3475 BUG_ON(!kvm_usage_count);
3476
3477 kvm_usage_count--;
3478 if (!kvm_usage_count)
Takuya Yoshikawa75b71272010-11-16 17:37:41 +09003479 on_each_cpu(hardware_disable_nolock, NULL, 1);
Alexander Graf10474ae2009-09-15 11:37:46 +02003480}
3481
3482static void hardware_disable_all(void)
3483{
Paolo Bonzini4a937f92013-09-10 12:58:35 +02003484 raw_spin_lock(&kvm_count_lock);
Alexander Graf10474ae2009-09-15 11:37:46 +02003485 hardware_disable_all_nolock();
Paolo Bonzini4a937f92013-09-10 12:58:35 +02003486 raw_spin_unlock(&kvm_count_lock);
Alexander Graf10474ae2009-09-15 11:37:46 +02003487}
3488
3489static int hardware_enable_all(void)
3490{
3491 int r = 0;
3492
Paolo Bonzini4a937f92013-09-10 12:58:35 +02003493 raw_spin_lock(&kvm_count_lock);
Alexander Graf10474ae2009-09-15 11:37:46 +02003494
3495 kvm_usage_count++;
3496 if (kvm_usage_count == 1) {
3497 atomic_set(&hardware_enable_failed, 0);
Takuya Yoshikawa75b71272010-11-16 17:37:41 +09003498 on_each_cpu(hardware_enable_nolock, NULL, 1);
Alexander Graf10474ae2009-09-15 11:37:46 +02003499
3500 if (atomic_read(&hardware_enable_failed)) {
3501 hardware_disable_all_nolock();
3502 r = -EBUSY;
3503 }
3504 }
3505
Paolo Bonzini4a937f92013-09-10 12:58:35 +02003506 raw_spin_unlock(&kvm_count_lock);
Alexander Graf10474ae2009-09-15 11:37:46 +02003507
3508 return r;
3509}
3510
Rusty Russell9a2b85c2007-07-17 23:17:55 +10003511static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
Mike Dayd77c26f2007-10-08 09:02:08 -04003512 void *v)
Rusty Russell9a2b85c2007-07-17 23:17:55 +10003513{
Sheng Yang8e1c1812009-04-29 11:09:04 +08003514 /*
3515 * Some (well, at least mine) BIOSes hang on reboot if
3516 * in vmx root mode.
3517 *
3518 * And Intel TXT required VMX off for all cpu when system shutdown.
3519 */
Xiubo Li1170adc2015-02-26 14:58:26 +08003520 pr_info("kvm: exiting hardware virtualization\n");
Sheng Yang8e1c1812009-04-29 11:09:04 +08003521 kvm_rebooting = true;
Takuya Yoshikawa75b71272010-11-16 17:37:41 +09003522 on_each_cpu(hardware_disable_nolock, NULL, 1);
Rusty Russell9a2b85c2007-07-17 23:17:55 +10003523 return NOTIFY_OK;
3524}
3525
3526static struct notifier_block kvm_reboot_notifier = {
3527 .notifier_call = kvm_reboot,
3528 .priority = 0,
3529};
3530
Marcelo Tosattie93f8a02009-12-23 14:35:24 -02003531static void kvm_io_bus_destroy(struct kvm_io_bus *bus)
Gregory Haskins2eeb2e92007-05-31 14:08:53 -04003532{
3533 int i;
3534
3535 for (i = 0; i < bus->dev_count; i++) {
Sasha Levin743eeb02011-07-27 16:00:48 +03003536 struct kvm_io_device *pos = bus->range[i].dev;
Gregory Haskins2eeb2e92007-05-31 14:08:53 -04003537
3538 kvm_iodevice_destructor(pos);
3539 }
Marcelo Tosattie93f8a02009-12-23 14:35:24 -02003540 kfree(bus);
Gregory Haskins2eeb2e92007-05-31 14:08:53 -04003541}
3542
Paolo Bonzinic21fbff2013-08-27 15:41:41 +02003543static inline int kvm_io_bus_cmp(const struct kvm_io_range *r1,
Xiubo Li20e87b72015-02-26 14:58:25 +08003544 const struct kvm_io_range *r2)
Sasha Levin743eeb02011-07-27 16:00:48 +03003545{
Jason Wang8f4216c72015-09-15 14:41:57 +08003546 gpa_t addr1 = r1->addr;
3547 gpa_t addr2 = r2->addr;
3548
3549 if (addr1 < addr2)
Sasha Levin743eeb02011-07-27 16:00:48 +03003550 return -1;
Jason Wang8f4216c72015-09-15 14:41:57 +08003551
3552 /* If r2->len == 0, match the exact address. If r2->len != 0,
3553 * accept any overlapping write. Any order is acceptable for
3554 * overlapping ranges, because kvm_io_bus_get_first_dev ensures
3555 * we process all of them.
3556 */
3557 if (r2->len) {
3558 addr1 += r1->len;
3559 addr2 += r2->len;
3560 }
3561
3562 if (addr1 > addr2)
Sasha Levin743eeb02011-07-27 16:00:48 +03003563 return 1;
Jason Wang8f4216c72015-09-15 14:41:57 +08003564
Sasha Levin743eeb02011-07-27 16:00:48 +03003565 return 0;
3566}
3567
Paolo Bonzinia343c9b2013-07-16 13:03:29 +02003568static int kvm_io_bus_sort_cmp(const void *p1, const void *p2)
3569{
Paolo Bonzinic21fbff2013-08-27 15:41:41 +02003570 return kvm_io_bus_cmp(p1, p2);
Paolo Bonzinia343c9b2013-07-16 13:03:29 +02003571}
3572
Geoff Levand39369f72013-04-05 19:20:30 +00003573static int kvm_io_bus_get_first_dev(struct kvm_io_bus *bus,
Sasha Levin743eeb02011-07-27 16:00:48 +03003574 gpa_t addr, int len)
3575{
3576 struct kvm_io_range *range, key;
3577 int off;
3578
3579 key = (struct kvm_io_range) {
3580 .addr = addr,
3581 .len = len,
3582 };
3583
3584 range = bsearch(&key, bus->range, bus->dev_count,
3585 sizeof(struct kvm_io_range), kvm_io_bus_sort_cmp);
3586 if (range == NULL)
3587 return -ENOENT;
3588
3589 off = range - bus->range;
3590
Paolo Bonzinic21fbff2013-08-27 15:41:41 +02003591 while (off > 0 && kvm_io_bus_cmp(&key, &bus->range[off-1]) == 0)
Sasha Levin743eeb02011-07-27 16:00:48 +03003592 off--;
3593
3594 return off;
3595}
3596
Nikolay Nikolaeve32edf42015-03-26 14:39:28 +00003597static int __kvm_io_bus_write(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus,
Cornelia Huck126a5af2013-07-03 16:30:53 +02003598 struct kvm_io_range *range, const void *val)
3599{
3600 int idx;
3601
3602 idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len);
3603 if (idx < 0)
3604 return -EOPNOTSUPP;
3605
3606 while (idx < bus->dev_count &&
Paolo Bonzinic21fbff2013-08-27 15:41:41 +02003607 kvm_io_bus_cmp(range, &bus->range[idx]) == 0) {
Nikolay Nikolaeve32edf42015-03-26 14:39:28 +00003608 if (!kvm_iodevice_write(vcpu, bus->range[idx].dev, range->addr,
Cornelia Huck126a5af2013-07-03 16:30:53 +02003609 range->len, val))
3610 return idx;
3611 idx++;
3612 }
3613
3614 return -EOPNOTSUPP;
3615}
3616
Michael S. Tsirkinbda90202009-06-29 22:24:32 +03003617/* kvm_io_bus_write - called under kvm->slots_lock */
Nikolay Nikolaeve32edf42015-03-26 14:39:28 +00003618int kvm_io_bus_write(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
Michael S. Tsirkinbda90202009-06-29 22:24:32 +03003619 int len, const void *val)
Gregory Haskins2eeb2e92007-05-31 14:08:53 -04003620{
Cornelia Huck126a5af2013-07-03 16:30:53 +02003621 struct kvm_io_bus *bus;
3622 struct kvm_io_range range;
3623 int r;
3624
3625 range = (struct kvm_io_range) {
3626 .addr = addr,
3627 .len = len,
3628 };
3629
Nikolay Nikolaeve32edf42015-03-26 14:39:28 +00003630 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
David Hildenbrand90db1042017-03-23 18:24:19 +01003631 if (!bus)
3632 return -ENOMEM;
Nikolay Nikolaeve32edf42015-03-26 14:39:28 +00003633 r = __kvm_io_bus_write(vcpu, bus, &range, val);
Cornelia Huck126a5af2013-07-03 16:30:53 +02003634 return r < 0 ? r : 0;
3635}
3636
3637/* kvm_io_bus_write_cookie - called under kvm->slots_lock */
Nikolay Nikolaeve32edf42015-03-26 14:39:28 +00003638int kvm_io_bus_write_cookie(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx,
3639 gpa_t addr, int len, const void *val, long cookie)
Cornelia Huck126a5af2013-07-03 16:30:53 +02003640{
Lai Jiangshan90d83dc2010-04-19 17:41:23 +08003641 struct kvm_io_bus *bus;
Sasha Levin743eeb02011-07-27 16:00:48 +03003642 struct kvm_io_range range;
3643
3644 range = (struct kvm_io_range) {
3645 .addr = addr,
3646 .len = len,
3647 };
Lai Jiangshan90d83dc2010-04-19 17:41:23 +08003648
Nikolay Nikolaeve32edf42015-03-26 14:39:28 +00003649 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
David Hildenbrand90db1042017-03-23 18:24:19 +01003650 if (!bus)
3651 return -ENOMEM;
Cornelia Huck126a5af2013-07-03 16:30:53 +02003652
3653 /* First try the device referenced by cookie. */
3654 if ((cookie >= 0) && (cookie < bus->dev_count) &&
Paolo Bonzinic21fbff2013-08-27 15:41:41 +02003655 (kvm_io_bus_cmp(&range, &bus->range[cookie]) == 0))
Nikolay Nikolaeve32edf42015-03-26 14:39:28 +00003656 if (!kvm_iodevice_write(vcpu, bus->range[cookie].dev, addr, len,
Cornelia Huck126a5af2013-07-03 16:30:53 +02003657 val))
3658 return cookie;
3659
3660 /*
3661 * cookie contained garbage; fall back to search and return the
3662 * correct cookie value.
3663 */
Nikolay Nikolaeve32edf42015-03-26 14:39:28 +00003664 return __kvm_io_bus_write(vcpu, bus, &range, val);
Cornelia Huck126a5af2013-07-03 16:30:53 +02003665}
3666
Nikolay Nikolaeve32edf42015-03-26 14:39:28 +00003667static int __kvm_io_bus_read(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus,
3668 struct kvm_io_range *range, void *val)
Cornelia Huck126a5af2013-07-03 16:30:53 +02003669{
3670 int idx;
3671
3672 idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len);
Sasha Levin743eeb02011-07-27 16:00:48 +03003673 if (idx < 0)
3674 return -EOPNOTSUPP;
3675
3676 while (idx < bus->dev_count &&
Paolo Bonzinic21fbff2013-08-27 15:41:41 +02003677 kvm_io_bus_cmp(range, &bus->range[idx]) == 0) {
Nikolay Nikolaeve32edf42015-03-26 14:39:28 +00003678 if (!kvm_iodevice_read(vcpu, bus->range[idx].dev, range->addr,
Cornelia Huck126a5af2013-07-03 16:30:53 +02003679 range->len, val))
3680 return idx;
Sasha Levin743eeb02011-07-27 16:00:48 +03003681 idx++;
3682 }
3683
Michael S. Tsirkinbda90202009-06-29 22:24:32 +03003684 return -EOPNOTSUPP;
3685}
Michael S. Tsirkin68c3b4d2014-03-31 21:50:44 +03003686EXPORT_SYMBOL_GPL(kvm_io_bus_write);
Gregory Haskins2eeb2e92007-05-31 14:08:53 -04003687
Michael S. Tsirkinbda90202009-06-29 22:24:32 +03003688/* kvm_io_bus_read - called under kvm->slots_lock */
Nikolay Nikolaeve32edf42015-03-26 14:39:28 +00003689int kvm_io_bus_read(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
Marcelo Tosattie93f8a02009-12-23 14:35:24 -02003690 int len, void *val)
Michael S. Tsirkinbda90202009-06-29 22:24:32 +03003691{
Cornelia Huck126a5af2013-07-03 16:30:53 +02003692 struct kvm_io_bus *bus;
3693 struct kvm_io_range range;
3694 int r;
3695
3696 range = (struct kvm_io_range) {
3697 .addr = addr,
3698 .len = len,
3699 };
3700
Nikolay Nikolaeve32edf42015-03-26 14:39:28 +00003701 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
David Hildenbrand90db1042017-03-23 18:24:19 +01003702 if (!bus)
3703 return -ENOMEM;
Nikolay Nikolaeve32edf42015-03-26 14:39:28 +00003704 r = __kvm_io_bus_read(vcpu, bus, &range, val);
Cornelia Huck126a5af2013-07-03 16:30:53 +02003705 return r < 0 ? r : 0;
3706}
3707
Gregory Haskins2eeb2e92007-05-31 14:08:53 -04003708
Marcelo Tosatti79fac952009-12-23 14:35:26 -02003709/* Caller must hold slots_lock. */
Sasha Levin743eeb02011-07-27 16:00:48 +03003710int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
3711 int len, struct kvm_io_device *dev)
Michael S. Tsirkin6c474692009-06-29 22:24:26 +03003712{
Gal Hammerd4c67a72018-01-16 15:34:41 +02003713 int i;
Marcelo Tosattie93f8a02009-12-23 14:35:24 -02003714 struct kvm_io_bus *new_bus, *bus;
Gal Hammerd4c67a72018-01-16 15:34:41 +02003715 struct kvm_io_range range;
Gregory Haskins090b7af2009-07-07 17:08:44 -04003716
Christian Borntraeger4a12f952017-07-07 10:51:38 +02003717 bus = kvm_get_bus(kvm, bus_idx);
David Hildenbrand90db1042017-03-23 18:24:19 +01003718 if (!bus)
3719 return -ENOMEM;
3720
Amos Kong6ea34c92013-05-25 06:44:15 +08003721 /* exclude ioeventfd which is limited by maximum fd */
3722 if (bus->dev_count - bus->ioeventfd_count > NR_IOBUS_DEVS - 1)
Gregory Haskins090b7af2009-07-07 17:08:44 -04003723 return -ENOSPC;
3724
Gustavo A. R. Silva90952cd2019-01-30 17:07:47 +01003725 new_bus = kmalloc(struct_size(bus, range, bus->dev_count + 1),
Ben Gardonb12ce362019-02-11 11:02:49 -08003726 GFP_KERNEL_ACCOUNT);
Marcelo Tosattie93f8a02009-12-23 14:35:24 -02003727 if (!new_bus)
3728 return -ENOMEM;
Gal Hammerd4c67a72018-01-16 15:34:41 +02003729
3730 range = (struct kvm_io_range) {
3731 .addr = addr,
3732 .len = len,
3733 .dev = dev,
3734 };
3735
3736 for (i = 0; i < bus->dev_count; i++)
3737 if (kvm_io_bus_cmp(&bus->range[i], &range) > 0)
3738 break;
3739
3740 memcpy(new_bus, bus, sizeof(*bus) + i * sizeof(struct kvm_io_range));
3741 new_bus->dev_count++;
3742 new_bus->range[i] = range;
3743 memcpy(new_bus->range + i + 1, bus->range + i,
3744 (bus->dev_count - i) * sizeof(struct kvm_io_range));
Marcelo Tosattie93f8a02009-12-23 14:35:24 -02003745 rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
3746 synchronize_srcu_expedited(&kvm->srcu);
3747 kfree(bus);
Gregory Haskins090b7af2009-07-07 17:08:44 -04003748
3749 return 0;
3750}
3751
Marcelo Tosatti79fac952009-12-23 14:35:26 -02003752/* Caller must hold slots_lock. */
David Hildenbrand90db1042017-03-23 18:24:19 +01003753void kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
3754 struct kvm_io_device *dev)
Gregory Haskins090b7af2009-07-07 17:08:44 -04003755{
David Hildenbrand90db1042017-03-23 18:24:19 +01003756 int i;
Marcelo Tosattie93f8a02009-12-23 14:35:24 -02003757 struct kvm_io_bus *new_bus, *bus;
Michael S. Tsirkin6c474692009-06-29 22:24:26 +03003758
Christian Borntraeger4a12f952017-07-07 10:51:38 +02003759 bus = kvm_get_bus(kvm, bus_idx);
Peter Xudf630b82017-03-15 16:01:17 +08003760 if (!bus)
David Hildenbrand90db1042017-03-23 18:24:19 +01003761 return;
Peter Xudf630b82017-03-15 16:01:17 +08003762
Amos Konga13007162012-03-09 12:17:32 +08003763 for (i = 0; i < bus->dev_count; i++)
3764 if (bus->range[i].dev == dev) {
Gregory Haskins090b7af2009-07-07 17:08:44 -04003765 break;
3766 }
Marcelo Tosattie93f8a02009-12-23 14:35:24 -02003767
David Hildenbrand90db1042017-03-23 18:24:19 +01003768 if (i == bus->dev_count)
3769 return;
Amos Konga13007162012-03-09 12:17:32 +08003770
Gustavo A. R. Silva90952cd2019-01-30 17:07:47 +01003771 new_bus = kmalloc(struct_size(bus, range, bus->dev_count - 1),
Ben Gardonb12ce362019-02-11 11:02:49 -08003772 GFP_KERNEL_ACCOUNT);
David Hildenbrand90db1042017-03-23 18:24:19 +01003773 if (!new_bus) {
3774 pr_err("kvm: failed to shrink bus, removing it completely\n");
3775 goto broken;
3776 }
Amos Konga13007162012-03-09 12:17:32 +08003777
3778 memcpy(new_bus, bus, sizeof(*bus) + i * sizeof(struct kvm_io_range));
3779 new_bus->dev_count--;
3780 memcpy(new_bus->range + i, bus->range + i + 1,
3781 (new_bus->dev_count - i) * sizeof(struct kvm_io_range));
Marcelo Tosattie93f8a02009-12-23 14:35:24 -02003782
David Hildenbrand90db1042017-03-23 18:24:19 +01003783broken:
Marcelo Tosattie93f8a02009-12-23 14:35:24 -02003784 rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
3785 synchronize_srcu_expedited(&kvm->srcu);
3786 kfree(bus);
David Hildenbrand90db1042017-03-23 18:24:19 +01003787 return;
Gregory Haskins2eeb2e92007-05-31 14:08:53 -04003788}
3789
Andre Przywara8a39d002016-07-15 12:43:26 +01003790struct kvm_io_device *kvm_io_bus_get_dev(struct kvm *kvm, enum kvm_bus bus_idx,
3791 gpa_t addr)
3792{
3793 struct kvm_io_bus *bus;
3794 int dev_idx, srcu_idx;
3795 struct kvm_io_device *iodev = NULL;
3796
3797 srcu_idx = srcu_read_lock(&kvm->srcu);
3798
3799 bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
David Hildenbrand90db1042017-03-23 18:24:19 +01003800 if (!bus)
3801 goto out_unlock;
Andre Przywara8a39d002016-07-15 12:43:26 +01003802
3803 dev_idx = kvm_io_bus_get_first_dev(bus, addr, 1);
3804 if (dev_idx < 0)
3805 goto out_unlock;
3806
3807 iodev = bus->range[dev_idx].dev;
3808
3809out_unlock:
3810 srcu_read_unlock(&kvm->srcu, srcu_idx);
3811
3812 return iodev;
3813}
3814EXPORT_SYMBOL_GPL(kvm_io_bus_get_dev);
3815
Janosch Frank536a6f82016-05-18 13:26:23 +02003816static int kvm_debugfs_open(struct inode *inode, struct file *file,
3817 int (*get)(void *, u64 *), int (*set)(void *, u64),
3818 const char *fmt)
3819{
3820 struct kvm_stat_data *stat_data = (struct kvm_stat_data *)
3821 inode->i_private;
3822
3823 /* The debugfs files are a reference to the kvm struct which
3824 * is still valid when kvm_destroy_vm is called.
3825 * To avoid the race between open and the removal of the debugfs
3826 * directory we test against the users count.
3827 */
Elena Reshetovae3736c32017-02-20 13:06:21 +02003828 if (!refcount_inc_not_zero(&stat_data->kvm->users_count))
Janosch Frank536a6f82016-05-18 13:26:23 +02003829 return -ENOENT;
3830
3831 if (simple_attr_open(inode, file, get, set, fmt)) {
3832 kvm_put_kvm(stat_data->kvm);
3833 return -ENOMEM;
3834 }
3835
3836 return 0;
3837}
3838
3839static int kvm_debugfs_release(struct inode *inode, struct file *file)
3840{
3841 struct kvm_stat_data *stat_data = (struct kvm_stat_data *)
3842 inode->i_private;
3843
3844 simple_attr_release(inode, file);
3845 kvm_put_kvm(stat_data->kvm);
3846
3847 return 0;
3848}
3849
3850static int vm_stat_get_per_vm(void *data, u64 *val)
3851{
3852 struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data;
3853
Suraj Jitindar Singh8a7e75d2016-08-02 14:03:22 +10003854 *val = *(ulong *)((void *)stat_data->kvm + stat_data->offset);
Janosch Frank536a6f82016-05-18 13:26:23 +02003855
3856 return 0;
3857}
3858
Suraj Jitindar Singhce35ef22016-10-19 13:49:47 +11003859static int vm_stat_clear_per_vm(void *data, u64 val)
3860{
3861 struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data;
3862
3863 if (val)
3864 return -EINVAL;
3865
3866 *(ulong *)((void *)stat_data->kvm + stat_data->offset) = 0;
3867
3868 return 0;
3869}
3870
Janosch Frank536a6f82016-05-18 13:26:23 +02003871static int vm_stat_get_per_vm_open(struct inode *inode, struct file *file)
3872{
3873 __simple_attr_check_format("%llu\n", 0ull);
3874 return kvm_debugfs_open(inode, file, vm_stat_get_per_vm,
Suraj Jitindar Singhce35ef22016-10-19 13:49:47 +11003875 vm_stat_clear_per_vm, "%llu\n");
Janosch Frank536a6f82016-05-18 13:26:23 +02003876}
3877
3878static const struct file_operations vm_stat_get_per_vm_fops = {
3879 .owner = THIS_MODULE,
3880 .open = vm_stat_get_per_vm_open,
3881 .release = kvm_debugfs_release,
3882 .read = simple_attr_read,
3883 .write = simple_attr_write,
Geliang Tang3bed8882017-05-06 23:37:19 +08003884 .llseek = no_llseek,
Janosch Frank536a6f82016-05-18 13:26:23 +02003885};
3886
3887static int vcpu_stat_get_per_vm(void *data, u64 *val)
3888{
3889 int i;
3890 struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data;
3891 struct kvm_vcpu *vcpu;
3892
3893 *val = 0;
3894
3895 kvm_for_each_vcpu(i, vcpu, stat_data->kvm)
Suraj Jitindar Singh8a7e75d2016-08-02 14:03:22 +10003896 *val += *(u64 *)((void *)vcpu + stat_data->offset);
Janosch Frank536a6f82016-05-18 13:26:23 +02003897
3898 return 0;
3899}
3900
Suraj Jitindar Singhce35ef22016-10-19 13:49:47 +11003901static int vcpu_stat_clear_per_vm(void *data, u64 val)
3902{
3903 int i;
3904 struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data;
3905 struct kvm_vcpu *vcpu;
3906
3907 if (val)
3908 return -EINVAL;
3909
3910 kvm_for_each_vcpu(i, vcpu, stat_data->kvm)
3911 *(u64 *)((void *)vcpu + stat_data->offset) = 0;
3912
3913 return 0;
3914}
3915
Janosch Frank536a6f82016-05-18 13:26:23 +02003916static int vcpu_stat_get_per_vm_open(struct inode *inode, struct file *file)
3917{
3918 __simple_attr_check_format("%llu\n", 0ull);
3919 return kvm_debugfs_open(inode, file, vcpu_stat_get_per_vm,
Suraj Jitindar Singhce35ef22016-10-19 13:49:47 +11003920 vcpu_stat_clear_per_vm, "%llu\n");
Janosch Frank536a6f82016-05-18 13:26:23 +02003921}
3922
3923static const struct file_operations vcpu_stat_get_per_vm_fops = {
3924 .owner = THIS_MODULE,
3925 .open = vcpu_stat_get_per_vm_open,
3926 .release = kvm_debugfs_release,
3927 .read = simple_attr_read,
3928 .write = simple_attr_write,
Geliang Tang3bed8882017-05-06 23:37:19 +08003929 .llseek = no_llseek,
Janosch Frank536a6f82016-05-18 13:26:23 +02003930};
3931
3932static const struct file_operations *stat_fops_per_vm[] = {
3933 [KVM_STAT_VCPU] = &vcpu_stat_get_per_vm_fops,
3934 [KVM_STAT_VM] = &vm_stat_get_per_vm_fops,
3935};
3936
Christoph Hellwig8b88b092008-02-08 04:20:26 -08003937static int vm_stat_get(void *_offset, u64 *val)
Avi Kivityba1389b2007-11-18 16:24:12 +02003938{
3939 unsigned offset = (long)_offset;
Avi Kivityba1389b2007-11-18 16:24:12 +02003940 struct kvm *kvm;
Janosch Frank536a6f82016-05-18 13:26:23 +02003941 struct kvm_stat_data stat_tmp = {.offset = offset};
3942 u64 tmp_val;
Avi Kivityba1389b2007-11-18 16:24:12 +02003943
Christoph Hellwig8b88b092008-02-08 04:20:26 -08003944 *val = 0;
Paolo Bonzini2f303b72013-09-25 13:53:07 +02003945 spin_lock(&kvm_lock);
Janosch Frank536a6f82016-05-18 13:26:23 +02003946 list_for_each_entry(kvm, &vm_list, vm_list) {
3947 stat_tmp.kvm = kvm;
3948 vm_stat_get_per_vm((void *)&stat_tmp, &tmp_val);
3949 *val += tmp_val;
3950 }
Paolo Bonzini2f303b72013-09-25 13:53:07 +02003951 spin_unlock(&kvm_lock);
Christoph Hellwig8b88b092008-02-08 04:20:26 -08003952 return 0;
Avi Kivityba1389b2007-11-18 16:24:12 +02003953}
3954
Suraj Jitindar Singhce35ef22016-10-19 13:49:47 +11003955static int vm_stat_clear(void *_offset, u64 val)
3956{
3957 unsigned offset = (long)_offset;
3958 struct kvm *kvm;
3959 struct kvm_stat_data stat_tmp = {.offset = offset};
3960
3961 if (val)
3962 return -EINVAL;
3963
3964 spin_lock(&kvm_lock);
3965 list_for_each_entry(kvm, &vm_list, vm_list) {
3966 stat_tmp.kvm = kvm;
3967 vm_stat_clear_per_vm((void *)&stat_tmp, 0);
3968 }
3969 spin_unlock(&kvm_lock);
3970
3971 return 0;
3972}
3973
3974DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, vm_stat_clear, "%llu\n");
Avi Kivityba1389b2007-11-18 16:24:12 +02003975
Christoph Hellwig8b88b092008-02-08 04:20:26 -08003976static int vcpu_stat_get(void *_offset, u64 *val)
Avi Kivity1165f5f2007-04-19 17:27:43 +03003977{
3978 unsigned offset = (long)_offset;
Avi Kivity1165f5f2007-04-19 17:27:43 +03003979 struct kvm *kvm;
Janosch Frank536a6f82016-05-18 13:26:23 +02003980 struct kvm_stat_data stat_tmp = {.offset = offset};
3981 u64 tmp_val;
Avi Kivity1165f5f2007-04-19 17:27:43 +03003982
Christoph Hellwig8b88b092008-02-08 04:20:26 -08003983 *val = 0;
Paolo Bonzini2f303b72013-09-25 13:53:07 +02003984 spin_lock(&kvm_lock);
Janosch Frank536a6f82016-05-18 13:26:23 +02003985 list_for_each_entry(kvm, &vm_list, vm_list) {
3986 stat_tmp.kvm = kvm;
3987 vcpu_stat_get_per_vm((void *)&stat_tmp, &tmp_val);
3988 *val += tmp_val;
3989 }
Paolo Bonzini2f303b72013-09-25 13:53:07 +02003990 spin_unlock(&kvm_lock);
Christoph Hellwig8b88b092008-02-08 04:20:26 -08003991 return 0;
Avi Kivity1165f5f2007-04-19 17:27:43 +03003992}
3993
Suraj Jitindar Singhce35ef22016-10-19 13:49:47 +11003994static int vcpu_stat_clear(void *_offset, u64 val)
3995{
3996 unsigned offset = (long)_offset;
3997 struct kvm *kvm;
3998 struct kvm_stat_data stat_tmp = {.offset = offset};
3999
4000 if (val)
4001 return -EINVAL;
4002
4003 spin_lock(&kvm_lock);
4004 list_for_each_entry(kvm, &vm_list, vm_list) {
4005 stat_tmp.kvm = kvm;
4006 vcpu_stat_clear_per_vm((void *)&stat_tmp, 0);
4007 }
4008 spin_unlock(&kvm_lock);
4009
4010 return 0;
4011}
4012
4013DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, vcpu_stat_clear,
4014 "%llu\n");
Avi Kivityba1389b2007-11-18 16:24:12 +02004015
Alexey Dobriyan828c0952009-10-01 15:43:56 -07004016static const struct file_operations *stat_fops[] = {
Avi Kivityba1389b2007-11-18 16:24:12 +02004017 [KVM_STAT_VCPU] = &vcpu_stat_fops,
4018 [KVM_STAT_VM] = &vm_stat_fops,
4019};
Avi Kivity1165f5f2007-04-19 17:27:43 +03004020
Claudio Imbrenda286de8f2017-07-12 17:56:44 +02004021static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm)
4022{
4023 struct kobj_uevent_env *env;
Claudio Imbrenda286de8f2017-07-12 17:56:44 +02004024 unsigned long long created, active;
4025
4026 if (!kvm_dev.this_device || !kvm)
4027 return;
4028
4029 spin_lock(&kvm_lock);
4030 if (type == KVM_EVENT_CREATE_VM) {
4031 kvm_createvm_count++;
4032 kvm_active_vms++;
4033 } else if (type == KVM_EVENT_DESTROY_VM) {
4034 kvm_active_vms--;
4035 }
4036 created = kvm_createvm_count;
4037 active = kvm_active_vms;
4038 spin_unlock(&kvm_lock);
4039
Ben Gardonb12ce362019-02-11 11:02:49 -08004040 env = kzalloc(sizeof(*env), GFP_KERNEL_ACCOUNT);
Claudio Imbrenda286de8f2017-07-12 17:56:44 +02004041 if (!env)
4042 return;
4043
4044 add_uevent_var(env, "CREATED=%llu", created);
4045 add_uevent_var(env, "COUNT=%llu", active);
4046
Claudio Imbrendafdeaf7e2017-07-24 13:40:03 +02004047 if (type == KVM_EVENT_CREATE_VM) {
Claudio Imbrenda286de8f2017-07-12 17:56:44 +02004048 add_uevent_var(env, "EVENT=create");
Claudio Imbrendafdeaf7e2017-07-24 13:40:03 +02004049 kvm->userspace_pid = task_pid_nr(current);
4050 } else if (type == KVM_EVENT_DESTROY_VM) {
Claudio Imbrenda286de8f2017-07-12 17:56:44 +02004051 add_uevent_var(env, "EVENT=destroy");
Claudio Imbrendafdeaf7e2017-07-24 13:40:03 +02004052 }
4053 add_uevent_var(env, "PID=%d", kvm->userspace_pid);
Claudio Imbrenda286de8f2017-07-12 17:56:44 +02004054
4055 if (kvm->debugfs_dentry) {
Ben Gardonb12ce362019-02-11 11:02:49 -08004056 char *tmp, *p = kmalloc(PATH_MAX, GFP_KERNEL_ACCOUNT);
Claudio Imbrenda286de8f2017-07-12 17:56:44 +02004057
Claudio Imbrendafdeaf7e2017-07-24 13:40:03 +02004058 if (p) {
4059 tmp = dentry_path_raw(kvm->debugfs_dentry, p, PATH_MAX);
4060 if (!IS_ERR(tmp))
4061 add_uevent_var(env, "STATS_PATH=%s", tmp);
4062 kfree(p);
Claudio Imbrenda286de8f2017-07-12 17:56:44 +02004063 }
4064 }
4065 /* no need for checks, since we are adding at most only 5 keys */
4066 env->envp[env->envp_idx++] = NULL;
4067 kobject_uevent_env(&kvm_dev.this_device->kobj, KOBJ_CHANGE, env->envp);
4068 kfree(env);
Claudio Imbrenda286de8f2017-07-12 17:56:44 +02004069}
4070
Greg Kroah-Hartman929f45e2018-05-29 18:22:04 +02004071static void kvm_init_debug(void)
Avi Kivity6aa8b732006-12-10 02:21:36 -08004072{
4073 struct kvm_stats_debugfs_item *p;
4074
Hollis Blanchard76f7c872008-04-15 16:05:42 -05004075 kvm_debugfs_dir = debugfs_create_dir("kvm", NULL);
Hamo4f69b682011-12-15 14:23:16 +08004076
Janosch Frank536a6f82016-05-18 13:26:23 +02004077 kvm_debugfs_num_entries = 0;
4078 for (p = debugfs_entries; p->name; ++p, kvm_debugfs_num_entries++) {
Greg Kroah-Hartman929f45e2018-05-29 18:22:04 +02004079 debugfs_create_file(p->name, 0644, kvm_debugfs_dir,
4080 (void *)(long)p->offset,
4081 stat_fops[p->kind]);
Hamo4f69b682011-12-15 14:23:16 +08004082 }
Avi Kivity6aa8b732006-12-10 02:21:36 -08004083}
4084
Rafael J. Wysockifb3600c2011-03-23 22:16:23 +01004085static int kvm_suspend(void)
Avi Kivity59ae6c62007-02-12 00:54:48 -08004086{
Alexander Graf10474ae2009-09-15 11:37:46 +02004087 if (kvm_usage_count)
Takuya Yoshikawa75b71272010-11-16 17:37:41 +09004088 hardware_disable_nolock(NULL);
Avi Kivity59ae6c62007-02-12 00:54:48 -08004089 return 0;
4090}
4091
Rafael J. Wysockifb3600c2011-03-23 22:16:23 +01004092static void kvm_resume(void)
Avi Kivity59ae6c62007-02-12 00:54:48 -08004093{
Zachary Amsdenca84d1a2010-08-19 22:07:28 -10004094 if (kvm_usage_count) {
Paolo Bonzini4a937f92013-09-10 12:58:35 +02004095 WARN_ON(raw_spin_is_locked(&kvm_count_lock));
Takuya Yoshikawa75b71272010-11-16 17:37:41 +09004096 hardware_enable_nolock(NULL);
Zachary Amsdenca84d1a2010-08-19 22:07:28 -10004097 }
Avi Kivity59ae6c62007-02-12 00:54:48 -08004098}
4099
Rafael J. Wysockifb3600c2011-03-23 22:16:23 +01004100static struct syscore_ops kvm_syscore_ops = {
Avi Kivity59ae6c62007-02-12 00:54:48 -08004101 .suspend = kvm_suspend,
4102 .resume = kvm_resume,
4103};
4104
Avi Kivity15ad7142007-07-11 18:17:21 +03004105static inline
4106struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
4107{
4108 return container_of(pn, struct kvm_vcpu, preempt_notifier);
4109}
4110
4111static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
4112{
4113 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
Xiubo Lif95ef0c2015-02-26 14:58:23 +08004114
Raghavendra K T3a08a8f2013-03-04 23:32:07 +05304115 if (vcpu->preempted)
4116 vcpu->preempted = false;
Avi Kivity15ad7142007-07-11 18:17:21 +03004117
Radim Krčmáře790d9e2014-08-21 18:08:05 +02004118 kvm_arch_sched_in(vcpu, cpu);
4119
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +08004120 kvm_arch_vcpu_load(vcpu, cpu);
Avi Kivity15ad7142007-07-11 18:17:21 +03004121}
4122
4123static void kvm_sched_out(struct preempt_notifier *pn,
4124 struct task_struct *next)
4125{
4126 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
4127
Raghavendra K T3a08a8f2013-03-04 23:32:07 +05304128 if (current->state == TASK_RUNNING)
4129 vcpu->preempted = true;
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +08004130 kvm_arch_vcpu_put(vcpu);
Avi Kivity15ad7142007-07-11 18:17:21 +03004131}
4132
Avi Kivity0ee75be2010-04-28 15:39:01 +03004133int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
Rusty Russellc16f8622007-07-30 21:12:19 +10004134 struct module *module)
Avi Kivity6aa8b732006-12-10 02:21:36 -08004135{
4136 int r;
Yang, Sheng002c7f72007-07-31 14:23:01 +03004137 int cpu;
Avi Kivity6aa8b732006-12-10 02:21:36 -08004138
Zhang Xiantaof8c16bb2007-11-14 20:40:21 +08004139 r = kvm_arch_init(opaque);
4140 if (r)
Zhang Xiantaod23087842007-11-29 15:35:39 +08004141 goto out_fail;
Zhang Xiantaocb498ea2007-11-14 20:39:31 +08004142
Asias He7dac16c2013-05-08 10:57:29 +08004143 /*
4144 * kvm_arch_init makes sure there's at most one caller
4145 * for architectures that support multiple implementations,
4146 * like intel and amd on x86.
Paolo Bonzini36343f62016-10-26 13:35:56 +02004147 * kvm_arch_init must be called before kvm_irqfd_init to avoid creating
4148 * conflicts in case kvm is already setup for another implementation.
Asias He7dac16c2013-05-08 10:57:29 +08004149 */
Paolo Bonzini36343f62016-10-26 13:35:56 +02004150 r = kvm_irqfd_init();
4151 if (r)
4152 goto out_irqfd;
Asias He7dac16c2013-05-08 10:57:29 +08004153
Avi Kivity8437a6172009-06-06 14:52:35 -07004154 if (!zalloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) {
Rusty Russell7f59f492008-12-07 21:25:45 +10304155 r = -ENOMEM;
4156 goto out_free_0;
4157 }
4158
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +08004159 r = kvm_arch_hardware_setup();
Avi Kivity6aa8b732006-12-10 02:21:36 -08004160 if (r < 0)
Rusty Russell7f59f492008-12-07 21:25:45 +10304161 goto out_free_0a;
Avi Kivity6aa8b732006-12-10 02:21:36 -08004162
Yang, Sheng002c7f72007-07-31 14:23:01 +03004163 for_each_online_cpu(cpu) {
4164 smp_call_function_single(cpu,
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +08004165 kvm_arch_check_processor_compat,
Jens Axboe8691e5a2008-06-06 11:18:06 +02004166 &r, 1);
Yang, Sheng002c7f72007-07-31 14:23:01 +03004167 if (r < 0)
Zhang Xiantaod23087842007-11-29 15:35:39 +08004168 goto out_free_1;
Yang, Sheng002c7f72007-07-31 14:23:01 +03004169 }
4170
Thomas Gleixner73c1b412016-12-21 20:19:54 +01004171 r = cpuhp_setup_state_nocalls(CPUHP_AP_KVM_STARTING, "kvm/cpu:starting",
Thomas Gleixner8c18b2d2016-07-13 17:16:37 +00004172 kvm_starting_cpu, kvm_dying_cpu);
Avi Kivity774c47f2007-02-12 00:54:47 -08004173 if (r)
Zhang Xiantaod23087842007-11-29 15:35:39 +08004174 goto out_free_2;
Avi Kivity6aa8b732006-12-10 02:21:36 -08004175 register_reboot_notifier(&kvm_reboot_notifier);
4176
Rusty Russellc16f8622007-07-30 21:12:19 +10004177 /* A kmem cache lets us meet the alignment requirements of fx_save. */
Avi Kivity0ee75be2010-04-28 15:39:01 +03004178 if (!vcpu_align)
4179 vcpu_align = __alignof__(struct kvm_vcpu);
Paolo Bonzini46515732017-10-26 15:45:46 +02004180 kvm_vcpu_cache =
4181 kmem_cache_create_usercopy("kvm_vcpu", vcpu_size, vcpu_align,
4182 SLAB_ACCOUNT,
4183 offsetof(struct kvm_vcpu, arch),
4184 sizeof_field(struct kvm_vcpu, arch),
4185 NULL);
Rusty Russellc16f8622007-07-30 21:12:19 +10004186 if (!kvm_vcpu_cache) {
4187 r = -ENOMEM;
Rafael J. Wysockifb3600c2011-03-23 22:16:23 +01004188 goto out_free_3;
Rusty Russellc16f8622007-07-30 21:12:19 +10004189 }
4190
Gleb Natapovaf585b92010-10-14 11:22:46 +02004191 r = kvm_async_pf_init();
4192 if (r)
4193 goto out_free;
4194
Avi Kivity6aa8b732006-12-10 02:21:36 -08004195 kvm_chardev_ops.owner = module;
Christian Borntraeger3d3aab12008-12-02 11:17:32 +01004196 kvm_vm_fops.owner = module;
4197 kvm_vcpu_fops.owner = module;
Avi Kivity6aa8b732006-12-10 02:21:36 -08004198
4199 r = misc_register(&kvm_dev);
4200 if (r) {
Xiubo Li1170adc2015-02-26 14:58:26 +08004201 pr_err("kvm: misc device register failed\n");
Gleb Natapovaf585b92010-10-14 11:22:46 +02004202 goto out_unreg;
Avi Kivity6aa8b732006-12-10 02:21:36 -08004203 }
4204
Rafael J. Wysockifb3600c2011-03-23 22:16:23 +01004205 register_syscore_ops(&kvm_syscore_ops);
4206
Avi Kivity15ad7142007-07-11 18:17:21 +03004207 kvm_preempt_ops.sched_in = kvm_sched_in;
4208 kvm_preempt_ops.sched_out = kvm_sched_out;
4209
Greg Kroah-Hartman929f45e2018-05-29 18:22:04 +02004210 kvm_init_debug();
Darrick J. Wong0ea4ed82009-10-14 16:21:00 -07004211
Paolo Bonzini3c3c29f2014-09-24 13:02:46 +02004212 r = kvm_vfio_ops_init();
4213 WARN_ON(r);
4214
Avi Kivityc7addb92007-09-16 18:58:32 +02004215 return 0;
Avi Kivity6aa8b732006-12-10 02:21:36 -08004216
Gleb Natapovaf585b92010-10-14 11:22:46 +02004217out_unreg:
4218 kvm_async_pf_deinit();
Avi Kivity6aa8b732006-12-10 02:21:36 -08004219out_free:
Rusty Russellc16f8622007-07-30 21:12:19 +10004220 kmem_cache_destroy(kvm_vcpu_cache);
Zhang Xiantaod23087842007-11-29 15:35:39 +08004221out_free_3:
Avi Kivity6aa8b732006-12-10 02:21:36 -08004222 unregister_reboot_notifier(&kvm_reboot_notifier);
Thomas Gleixner8c18b2d2016-07-13 17:16:37 +00004223 cpuhp_remove_state_nocalls(CPUHP_AP_KVM_STARTING);
Zhang Xiantaod23087842007-11-29 15:35:39 +08004224out_free_2:
Zhang Xiantaod23087842007-11-29 15:35:39 +08004225out_free_1:
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +08004226 kvm_arch_hardware_unsetup();
Rusty Russell7f59f492008-12-07 21:25:45 +10304227out_free_0a:
4228 free_cpumask_var(cpus_hardware_enabled);
Zhang Xiantaod23087842007-11-29 15:35:39 +08004229out_free_0:
Cornelia Hucka0f155e2013-02-28 12:33:18 +01004230 kvm_irqfd_exit();
Paolo Bonzini36343f62016-10-26 13:35:56 +02004231out_irqfd:
Asias He7dac16c2013-05-08 10:57:29 +08004232 kvm_arch_exit();
4233out_fail:
Avi Kivity6aa8b732006-12-10 02:21:36 -08004234 return r;
4235}
Zhang Xiantaocb498ea2007-11-14 20:39:31 +08004236EXPORT_SYMBOL_GPL(kvm_init);
Avi Kivity6aa8b732006-12-10 02:21:36 -08004237
Zhang Xiantaocb498ea2007-11-14 20:39:31 +08004238void kvm_exit(void)
Avi Kivity6aa8b732006-12-10 02:21:36 -08004239{
Janosch Frank4bd33b52015-10-14 12:37:35 +02004240 debugfs_remove_recursive(kvm_debugfs_dir);
Avi Kivity6aa8b732006-12-10 02:21:36 -08004241 misc_deregister(&kvm_dev);
Rusty Russellc16f8622007-07-30 21:12:19 +10004242 kmem_cache_destroy(kvm_vcpu_cache);
Gleb Natapovaf585b92010-10-14 11:22:46 +02004243 kvm_async_pf_deinit();
Rafael J. Wysockifb3600c2011-03-23 22:16:23 +01004244 unregister_syscore_ops(&kvm_syscore_ops);
Avi Kivity6aa8b732006-12-10 02:21:36 -08004245 unregister_reboot_notifier(&kvm_reboot_notifier);
Thomas Gleixner8c18b2d2016-07-13 17:16:37 +00004246 cpuhp_remove_state_nocalls(CPUHP_AP_KVM_STARTING);
Takuya Yoshikawa75b71272010-11-16 17:37:41 +09004247 on_each_cpu(hardware_disable_nolock, NULL, 1);
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +08004248 kvm_arch_hardware_unsetup();
Zhang Xiantaof8c16bb2007-11-14 20:40:21 +08004249 kvm_arch_exit();
Cornelia Hucka0f155e2013-02-28 12:33:18 +01004250 kvm_irqfd_exit();
Rusty Russell7f59f492008-12-07 21:25:45 +10304251 free_cpumask_var(cpus_hardware_enabled);
Wanpeng Li571ee1b2014-10-09 18:30:08 +08004252 kvm_vfio_ops_exit();
Avi Kivity6aa8b732006-12-10 02:21:36 -08004253}
Zhang Xiantaocb498ea2007-11-14 20:39:31 +08004254EXPORT_SYMBOL_GPL(kvm_exit);