blob: 20f7c994afebf212d01fee9c9fbbfaaf41445ccf [file] [log] [blame]
Vegard Nossuma656c8e2008-07-22 21:27:11 +02001/*
Carsten Otte043405e2007-10-10 17:16:19 +02002 * Kernel-based Virtual Machine driver for Linux
3 *
4 * This header defines architecture specific interfaces, x86 version
5 *
6 * This work is licensed under the terms of the GNU GPL, version 2. See
7 * the COPYING file in the top-level directory.
8 *
9 */
10
H. Peter Anvin1965aae2008-10-22 22:26:29 -070011#ifndef _ASM_X86_KVM_HOST_H
12#define _ASM_X86_KVM_HOST_H
Carsten Otte043405e2007-10-10 17:16:19 +020013
Zhang Xiantao34c16ee2007-10-20 15:34:38 +080014#include <linux/types.h>
15#include <linux/mm.h>
Andrea Arcangelie930bff2008-07-25 16:24:52 +020016#include <linux/mmu_notifier.h>
Marcelo Tosatti229456f2009-06-17 09:22:14 -030017#include <linux/tracepoint.h>
Sheng Yangf5f48ee2010-06-30 12:25:15 +080018#include <linux/cpumask.h>
Gleb Natapovf5132b02011-11-10 14:57:22 +020019#include <linux/irq_work.h>
Nicolai Stange447ae312018-07-29 12:15:33 +020020#include <linux/irq.h>
Zhang Xiantao34c16ee2007-10-20 15:34:38 +080021
22#include <linux/kvm.h>
23#include <linux/kvm_para.h>
Avi Kivityedf88412007-12-16 11:02:48 +020024#include <linux/kvm_types.h>
Gleb Natapovf5132b02011-11-10 14:57:22 +020025#include <linux/perf_event.h>
Marcelo Tosattid8281992012-11-27 23:29:01 -020026#include <linux/pvclock_gtod.h>
27#include <linux/clocksource.h>
Feng Wu87276882015-09-18 22:29:40 +080028#include <linux/irqbypass.h>
Andrey Smetanin5c9194122015-11-10 15:36:34 +030029#include <linux/hyperv.h>
Zhang Xiantao34c16ee2007-10-20 15:34:38 +080030
Suravee Suthikulpanit7d669f52016-06-15 17:23:45 -050031#include <asm/apic.h>
Gerd Hoffmann50d0a0f2008-06-03 16:17:31 +020032#include <asm/pvclock-abi.h>
Hollis Blancharde01a1b52007-12-03 15:30:25 -060033#include <asm/desc.h>
Sheng Yang0bed3b52008-10-09 16:01:54 +080034#include <asm/mtrr.h>
Alexander Graf9962d032008-11-25 20:17:02 +010035#include <asm/msr-index.h>
H. Peter Anvin3ee89722012-04-20 13:41:59 -070036#include <asm/asm.h>
Xiao Guangrong21ebbed2016-02-24 17:51:09 +080037#include <asm/kvm_page_track.h>
Vitaly Kuznetsov5a485802018-03-20 15:02:05 +010038#include <asm/hyperv-tlfs.h>
Hollis Blancharde01a1b52007-12-03 15:30:25 -060039
Radim Krčmář682f7322016-07-12 22:09:29 +020040#define KVM_MAX_VCPUS 288
Radim Krčmář757883d2016-07-12 22:09:17 +020041#define KVM_SOFT_MAX_VCPUS 240
Radim Krčmářaf1bae52016-07-12 22:09:30 +020042#define KVM_MAX_VCPU_ID 1023
Igor Mammedov1d4e7e32014-11-06 15:52:47 +000043#define KVM_USER_MEM_SLOTS 509
Alex Williamson07432472012-12-10 10:33:15 -070044/* memory slots that are not exposed to userspace */
45#define KVM_PRIVATE_MEM_SLOTS 3
Alex Williamsonbbacc0c2012-12-10 10:33:09 -070046#define KVM_MEM_SLOTS_NUM (KVM_USER_MEM_SLOTS + KVM_PRIVATE_MEM_SLOTS)
Xiao Guangrong93a5cef2011-11-24 17:37:48 +080047
Paolo Bonzinib401ee02017-04-18 12:41:18 +020048#define KVM_HALT_POLL_NS_DEFAULT 200000
Avi Kivity69a9f692008-03-21 12:38:23 +020049
Alexander Graf8175e5b2013-04-15 10:42:33 +020050#define KVM_IRQCHIP_NUM_PINS KVM_IOAPIC_NUM_PINS
51
Paolo Bonzini2860c4b2016-01-07 15:05:10 +010052/* x86-specific vcpu->requests bit members */
Andrew Jones23871492017-06-04 14:43:51 +020053#define KVM_REQ_MIGRATE_TIMER KVM_ARCH_REQ(0)
54#define KVM_REQ_REPORT_TPR_ACCESS KVM_ARCH_REQ(1)
55#define KVM_REQ_TRIPLE_FAULT KVM_ARCH_REQ(2)
56#define KVM_REQ_MMU_SYNC KVM_ARCH_REQ(3)
57#define KVM_REQ_CLOCK_UPDATE KVM_ARCH_REQ(4)
Junaid Shahid6e427822018-06-27 14:59:08 -070058#define KVM_REQ_LOAD_CR3 KVM_ARCH_REQ(5)
Andrew Jones23871492017-06-04 14:43:51 +020059#define KVM_REQ_EVENT KVM_ARCH_REQ(6)
60#define KVM_REQ_APF_HALT KVM_ARCH_REQ(7)
61#define KVM_REQ_STEAL_UPDATE KVM_ARCH_REQ(8)
62#define KVM_REQ_NMI KVM_ARCH_REQ(9)
63#define KVM_REQ_PMU KVM_ARCH_REQ(10)
64#define KVM_REQ_PMI KVM_ARCH_REQ(11)
65#define KVM_REQ_SMI KVM_ARCH_REQ(12)
66#define KVM_REQ_MASTERCLOCK_UPDATE KVM_ARCH_REQ(13)
67#define KVM_REQ_MCLOCK_INPROGRESS \
68 KVM_ARCH_REQ_FLAGS(14, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
69#define KVM_REQ_SCAN_IOAPIC \
70 KVM_ARCH_REQ_FLAGS(15, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
71#define KVM_REQ_GLOBAL_CLOCK_UPDATE KVM_ARCH_REQ(16)
72#define KVM_REQ_APIC_PAGE_RELOAD \
73 KVM_ARCH_REQ_FLAGS(17, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
74#define KVM_REQ_HV_CRASH KVM_ARCH_REQ(18)
75#define KVM_REQ_IOAPIC_EOI_EXIT KVM_ARCH_REQ(19)
76#define KVM_REQ_HV_RESET KVM_ARCH_REQ(20)
77#define KVM_REQ_HV_EXIT KVM_ARCH_REQ(21)
78#define KVM_REQ_HV_STIMER KVM_ARCH_REQ(22)
Liran Alone40ff1d2018-03-21 02:50:31 +020079#define KVM_REQ_LOAD_EOI_EXITMAP KVM_ARCH_REQ(23)
Paolo Bonzini7f7f1ba2018-07-18 18:49:01 +020080#define KVM_REQ_GET_VMCS12_PAGES KVM_ARCH_REQ(24)
Paolo Bonzini2860c4b2016-01-07 15:05:10 +010081
Joerg Roedelcfec82c2011-04-04 12:39:28 +020082#define CR0_RESERVED_BITS \
83 (~(unsigned long)(X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS \
84 | X86_CR0_ET | X86_CR0_NE | X86_CR0_WP | X86_CR0_AM \
85 | X86_CR0_NW | X86_CR0_CD | X86_CR0_PG))
86
Joerg Roedelcfec82c2011-04-04 12:39:28 +020087#define CR4_RESERVED_BITS \
88 (~(unsigned long)(X86_CR4_VME | X86_CR4_PVI | X86_CR4_TSD | X86_CR4_DE\
89 | X86_CR4_PSE | X86_CR4_PAE | X86_CR4_MCE \
Mao, Junjiead756a12012-07-02 01:18:48 +000090 | X86_CR4_PGE | X86_CR4_PCE | X86_CR4_OSFXSR | X86_CR4_PCIDE \
H. Peter Anvinafcbf132013-04-27 16:37:47 -070091 | X86_CR4_OSXSAVE | X86_CR4_SMEP | X86_CR4_FSGSBASE \
Yu Zhangfd8cb432017-08-24 20:27:56 +080092 | X86_CR4_OSXMMEXCPT | X86_CR4_LA57 | X86_CR4_VMXE \
Paolo Bonziniae3e61e2016-07-12 10:36:41 +020093 | X86_CR4_SMAP | X86_CR4_PKE | X86_CR4_UMIP))
Joerg Roedelcfec82c2011-04-04 12:39:28 +020094
95#define CR8_RESERVED_BITS (~(unsigned long)X86_CR8_TPR)
96
97
Zhang Xiantaocd6e8f82007-11-19 14:33:37 +080098
Zhang Xiantaocd6e8f82007-11-19 14:33:37 +080099#define INVALID_PAGE (~(hpa_t)0)
Xiao Guangrongdd180b32010-07-03 16:02:42 +0800100#define VALID_PAGE(x) ((x) != INVALID_PAGE)
101
Zhang Xiantaocd6e8f82007-11-19 14:33:37 +0800102#define UNMAPPED_GVA (~(gpa_t)0)
103
Joerg Roedelec04b262009-06-19 15:16:23 +0200104/* KVM Hugepage definitions for x86 */
Wei Yang4fef0f42018-09-28 22:08:50 +0800105enum {
106 PT_PAGE_TABLE_LEVEL = 1,
107 PT_DIRECTORY_LEVEL = 2,
108 PT_PDPE_LEVEL = 3,
109 /* set max level to the biggest one */
110 PT_MAX_HUGEPAGE_LEVEL = PT_PDPE_LEVEL,
111};
112#define KVM_NR_PAGE_SIZES (PT_MAX_HUGEPAGE_LEVEL - \
113 PT_PAGE_TABLE_LEVEL + 1)
Joerg Roedel82855412010-07-01 16:00:11 +0200114#define KVM_HPAGE_GFN_SHIFT(x) (((x) - 1) * 9)
115#define KVM_HPAGE_SHIFT(x) (PAGE_SHIFT + KVM_HPAGE_GFN_SHIFT(x))
Joerg Roedelec04b262009-06-19 15:16:23 +0200116#define KVM_HPAGE_SIZE(x) (1UL << KVM_HPAGE_SHIFT(x))
117#define KVM_HPAGE_MASK(x) (~(KVM_HPAGE_SIZE(x) - 1))
118#define KVM_PAGES_PER_HPAGE(x) (KVM_HPAGE_SIZE(x) / PAGE_SIZE)
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300119
Christoffer Dall6d9d41e2013-10-02 14:22:28 -0700120static inline gfn_t gfn_to_index(gfn_t gfn, gfn_t base_gfn, int level)
121{
122 /* KVM_HPAGE_GFN_SHIFT(PT_PAGE_TABLE_LEVEL) must be 0. */
123 return (gfn >> KVM_HPAGE_GFN_SHIFT(level)) -
124 (base_gfn >> KVM_HPAGE_GFN_SHIFT(level));
125}
126
Zhang Xiantaod657a982007-12-14 09:41:22 +0800127#define KVM_PERMILLE_MMU_PAGES 20
128#define KVM_MIN_ALLOC_MMU_PAGES 64
David Matlack114df302016-12-19 13:58:25 -0800129#define KVM_MMU_HASH_SHIFT 12
Dong, Eddie1ae0a132008-01-07 13:20:25 +0200130#define KVM_NUM_MMU_PAGES (1 << KVM_MMU_HASH_SHIFT)
Zhang Xiantaod657a982007-12-14 09:41:22 +0800131#define KVM_MIN_FREE_MMU_PAGES 5
132#define KVM_REFILL_PAGES 25
Andre Przywara73c11602010-12-01 12:17:44 +0100133#define KVM_MAX_CPUID_ENTRIES 80
Sheng Yang0bed3b52008-10-09 16:01:54 +0800134#define KVM_NR_FIXED_MTRR_REGION 88
Paolo Bonzini0d234da2014-08-18 16:39:48 +0200135#define KVM_NR_VAR_MTRR 8
Zhang Xiantaod657a982007-12-14 09:41:22 +0800136
Gleb Natapovaf585b92010-10-14 11:22:46 +0200137#define ASYNC_PF_PER_VCPU 64
138
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -0300139enum kvm_reg {
Zhang Xiantao2b3ccfa2007-11-19 14:56:05 +0800140 VCPU_REGS_RAX = 0,
141 VCPU_REGS_RCX = 1,
142 VCPU_REGS_RDX = 2,
143 VCPU_REGS_RBX = 3,
144 VCPU_REGS_RSP = 4,
145 VCPU_REGS_RBP = 5,
146 VCPU_REGS_RSI = 6,
147 VCPU_REGS_RDI = 7,
148#ifdef CONFIG_X86_64
149 VCPU_REGS_R8 = 8,
150 VCPU_REGS_R9 = 9,
151 VCPU_REGS_R10 = 10,
152 VCPU_REGS_R11 = 11,
153 VCPU_REGS_R12 = 12,
154 VCPU_REGS_R13 = 13,
155 VCPU_REGS_R14 = 14,
156 VCPU_REGS_R15 = 15,
157#endif
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -0300158 VCPU_REGS_RIP,
Zhang Xiantao2b3ccfa2007-11-19 14:56:05 +0800159 NR_VCPU_REGS
160};
161
Avi Kivity6de4f3a2009-05-31 22:58:47 +0300162enum kvm_reg_ex {
163 VCPU_EXREG_PDPTR = NR_VCPU_REGS,
Avi Kivityaff48ba2010-12-05 18:56:11 +0200164 VCPU_EXREG_CR3,
Avi Kivity6de12732011-03-07 12:51:22 +0200165 VCPU_EXREG_RFLAGS,
Avi Kivity2fb92db2011-04-27 19:42:18 +0300166 VCPU_EXREG_SEGMENTS,
Avi Kivity6de4f3a2009-05-31 22:58:47 +0300167};
168
Zhang Xiantao2b3ccfa2007-11-19 14:56:05 +0800169enum {
Zhang Xiantao2b3ccfa2007-11-19 14:56:05 +0800170 VCPU_SREG_ES,
Avi Kivity81609e32008-05-27 16:26:01 +0300171 VCPU_SREG_CS,
172 VCPU_SREG_SS,
173 VCPU_SREG_DS,
Zhang Xiantao2b3ccfa2007-11-19 14:56:05 +0800174 VCPU_SREG_FS,
175 VCPU_SREG_GS,
Zhang Xiantao2b3ccfa2007-11-19 14:56:05 +0800176 VCPU_SREG_TR,
177 VCPU_SREG_LDTR,
178};
179
Avi Kivity56e82312009-08-12 15:04:37 +0300180#include <asm/kvm_emulate.h>
Zhang Xiantao2b3ccfa2007-11-19 14:56:05 +0800181
Zhang Xiantaod657a982007-12-14 09:41:22 +0800182#define KVM_NR_MEM_OBJS 40
183
Jan Kiszka42dbaa52008-12-15 13:52:10 +0100184#define KVM_NR_DB_REGS 4
185
186#define DR6_BD (1 << 13)
187#define DR6_BS (1 << 14)
Jim Mattsoncfb634f2018-09-21 10:36:17 -0700188#define DR6_BT (1 << 15)
Nadav Amit6f43ed02014-07-15 17:37:46 +0300189#define DR6_RTM (1 << 16)
190#define DR6_FIXED_1 0xfffe0ff0
191#define DR6_INIT 0xffff0ff0
192#define DR6_VOLATILE 0x0001e00f
Jan Kiszka42dbaa52008-12-15 13:52:10 +0100193
194#define DR7_BP_EN_MASK 0x000000ff
195#define DR7_GE (1 << 9)
196#define DR7_GD (1 << 13)
197#define DR7_FIXED_1 0x00000400
Nadav Amit6f43ed02014-07-15 17:37:46 +0300198#define DR7_VOLATILE 0xffff2bff
Jan Kiszka42dbaa52008-12-15 13:52:10 +0100199
Nadav Amitc205fb72014-12-25 02:52:16 +0200200#define PFERR_PRESENT_BIT 0
201#define PFERR_WRITE_BIT 1
202#define PFERR_USER_BIT 2
203#define PFERR_RSVD_BIT 3
204#define PFERR_FETCH_BIT 4
Huaitong Hanbe94f6b2016-03-22 16:51:20 +0800205#define PFERR_PK_BIT 5
Tom Lendacky14727752016-11-23 12:01:38 -0500206#define PFERR_GUEST_FINAL_BIT 32
207#define PFERR_GUEST_PAGE_BIT 33
Nadav Amitc205fb72014-12-25 02:52:16 +0200208
209#define PFERR_PRESENT_MASK (1U << PFERR_PRESENT_BIT)
210#define PFERR_WRITE_MASK (1U << PFERR_WRITE_BIT)
211#define PFERR_USER_MASK (1U << PFERR_USER_BIT)
212#define PFERR_RSVD_MASK (1U << PFERR_RSVD_BIT)
213#define PFERR_FETCH_MASK (1U << PFERR_FETCH_BIT)
Huaitong Hanbe94f6b2016-03-22 16:51:20 +0800214#define PFERR_PK_MASK (1U << PFERR_PK_BIT)
Tom Lendacky14727752016-11-23 12:01:38 -0500215#define PFERR_GUEST_FINAL_MASK (1ULL << PFERR_GUEST_FINAL_BIT)
216#define PFERR_GUEST_PAGE_MASK (1ULL << PFERR_GUEST_PAGE_BIT)
217
218#define PFERR_NESTED_GUEST_PAGE (PFERR_GUEST_PAGE_MASK | \
Tom Lendacky14727752016-11-23 12:01:38 -0500219 PFERR_WRITE_MASK | \
220 PFERR_PRESENT_MASK)
Nadav Amitc205fb72014-12-25 02:52:16 +0200221
Junaid Shahid37f0e8f2016-12-06 16:46:15 -0800222/*
223 * The mask used to denote special SPTEs, which can be either MMIO SPTEs or
224 * Access Tracking SPTEs. We use bit 62 instead of bit 63 to avoid conflicting
225 * with the SVE bit in EPT PTEs.
226 */
227#define SPTE_SPECIAL_MASK (1ULL << 62)
228
Gleb Natapov41383772012-04-19 14:06:29 +0300229/* apic attention bits */
230#define KVM_APIC_CHECK_VAPIC 0
Michael S. Tsirkinae7a2a32012-06-24 19:25:07 +0300231/*
232 * The following bit is set with PV-EOI, unset on EOI.
233 * We detect PV-EOI changes by guest by comparing
234 * this bit with PV-EOI in guest memory.
235 * See the implementation in apic_update_pv_eoi.
236 */
237#define KVM_APIC_PV_EOI_PENDING 1
Gleb Natapov41383772012-04-19 14:06:29 +0300238
Feng Wud84f1e02015-09-18 22:29:49 +0800239struct kvm_kernel_irq_routing_entry;
240
Zhang Xiantaod657a982007-12-14 09:41:22 +0800241/*
242 * We don't want allocation failures within the mmu code, so we preallocate
243 * enough memory for a single page fault in a cache.
244 */
245struct kvm_mmu_memory_cache {
246 int nobjs;
247 void *objects[KVM_NR_MEM_OBJS];
248};
249
Xiao Guangrong21ebbed2016-02-24 17:51:09 +0800250/*
251 * the pages used as guest page table on soft mmu are tracked by
252 * kvm_memory_slot.arch.gfn_track which is 16 bits, so the role bits used
253 * by indirect shadow page can not be more than 15 bits.
254 *
255 * Currently, we used 14 bits that are @level, @cr4_pae, @quadrant, @access,
256 * @nxe, @cr0_wp, @smep_andnot_wp and @smap_andnot_wp.
257 */
Zhang Xiantaod657a982007-12-14 09:41:22 +0800258union kvm_mmu_page_role {
Vitaly Kuznetsov36d9594d2018-10-08 21:28:10 +0200259 u32 word;
Zhang Xiantaod657a982007-12-14 09:41:22 +0800260 struct {
Joe Perches7d76b4d2008-03-23 01:02:34 -0700261 unsigned level:4;
Avi Kivity5b7e0102010-04-14 19:20:03 +0300262 unsigned cr4_pae:1;
Joe Perches7d76b4d2008-03-23 01:02:34 -0700263 unsigned quadrant:2;
Avi Kivityf6e2c02b2009-01-11 13:02:10 +0200264 unsigned direct:1;
Joe Perches7d76b4d2008-03-23 01:02:34 -0700265 unsigned access:3;
Marcelo Tosatti2e53d632008-02-20 14:47:24 -0500266 unsigned invalid:1;
Avi Kivity9645bb562009-03-31 11:31:54 +0300267 unsigned nxe:1;
Avi Kivity3dbe1412010-05-12 11:48:18 +0300268 unsigned cr0_wp:1;
Avi Kivity411c5882011-06-06 16:11:54 +0300269 unsigned smep_andnot_wp:1;
Xiao Guangrong0be02262015-05-11 22:55:21 +0800270 unsigned smap_andnot_wp:1;
Peter Feinerac8d57e2017-06-30 17:26:31 -0700271 unsigned ad_disabled:1;
Jim Mattson1313cc22018-05-09 17:02:04 -0400272 unsigned guest_mode:1;
273 unsigned :6;
Paolo Bonzini699023e2015-05-18 15:03:39 +0200274
275 /*
276 * This is left at the top of the word so that
277 * kvm_memslots_for_spte_role can extract it with a
278 * simple shift. While there is room, give it a whole
279 * byte so it is also faster to load it from memory.
280 */
281 unsigned smm:8;
Zhang Xiantaod657a982007-12-14 09:41:22 +0800282 };
283};
284
Vitaly Kuznetsov36d9594d2018-10-08 21:28:10 +0200285union kvm_mmu_extended_role {
Vitaly Kuznetsova3362822018-10-08 21:28:11 +0200286/*
287 * This structure complements kvm_mmu_page_role caching everything needed for
288 * MMU configuration. If nothing in both these structures changed, MMU
289 * re-configuration can be skipped. @valid bit is set on first usage so we don't
290 * treat all-zero structure as valid data.
291 */
Vitaly Kuznetsov36d9594d2018-10-08 21:28:10 +0200292 u32 word;
Vitaly Kuznetsova3362822018-10-08 21:28:11 +0200293 struct {
294 unsigned int valid:1;
295 unsigned int execonly:1;
Vitaly Kuznetsov7dcd5752018-10-08 21:28:12 +0200296 unsigned int cr0_pg:1;
Vitaly Kuznetsova3362822018-10-08 21:28:11 +0200297 unsigned int cr4_pse:1;
298 unsigned int cr4_pke:1;
299 unsigned int cr4_smap:1;
300 unsigned int cr4_smep:1;
Vitaly Kuznetsov7dcd5752018-10-08 21:28:12 +0200301 unsigned int cr4_la57:1;
Vitaly Kuznetsova3362822018-10-08 21:28:11 +0200302 };
Vitaly Kuznetsov36d9594d2018-10-08 21:28:10 +0200303};
304
305union kvm_mmu_role {
306 u64 as_u64;
307 struct {
308 union kvm_mmu_page_role base;
309 union kvm_mmu_extended_role ext;
310 };
311};
312
Takuya Yoshikawa018aabb52015-11-20 17:41:28 +0900313struct kvm_rmap_head {
314 unsigned long val;
315};
316
Zhang Xiantaod657a982007-12-14 09:41:22 +0800317struct kvm_mmu_page {
318 struct list_head link;
319 struct hlist_node hash_link;
Wei Yang3ff519f2018-09-06 05:58:16 +0800320 bool unsync;
Zhang Xiantaod657a982007-12-14 09:41:22 +0800321
322 /*
323 * The following two entries are used to key the shadow page in the
324 * hash table.
325 */
Zhang Xiantaod657a982007-12-14 09:41:22 +0800326 union kvm_mmu_page_role role;
Wei Yang3ff519f2018-09-06 05:58:16 +0800327 gfn_t gfn;
Zhang Xiantaod657a982007-12-14 09:41:22 +0800328
329 u64 *spt;
330 /* hold the gfn of each spte inside spt */
331 gfn_t *gfns;
Xiao Guangrong0571d362010-04-16 21:27:54 +0800332 int root_count; /* Currently serving as active root */
Marcelo Tosatti60c8aec2008-12-01 22:32:02 -0200333 unsigned int unsync_children;
Takuya Yoshikawa018aabb52015-11-20 17:41:28 +0900334 struct kvm_rmap_head parent_ptes; /* rmap pointers to parent sptes */
Xiao Guangrongf6f8ade2013-06-19 17:09:24 +0800335
336 /* The page is obsolete if mmu_valid_gen != kvm->arch.mmu_valid_gen. */
Xiao Guangrong5304b8d2013-05-31 08:36:22 +0800337 unsigned long mmu_valid_gen;
Xiao Guangrongf6f8ade2013-06-19 17:09:24 +0800338
Marcelo Tosatti0074ff62008-09-23 13:18:40 -0300339 DECLARE_BITMAP(unsync_child_bitmap, 512);
Xiao Guangrongc2a2ac22011-07-12 03:32:13 +0800340
341#ifdef CONFIG_X86_32
Xiao Guangrongaccaefe2013-06-19 17:09:20 +0800342 /*
343 * Used out of the mmu-lock to avoid reading spte values while an
344 * update is in progress; see the comments in __get_spte_lockless().
345 */
Xiao Guangrongc2a2ac22011-07-12 03:32:13 +0800346 int clear_spte_count;
347#endif
348
Xiao Guangrong0cbf8e42013-06-19 17:09:21 +0800349 /* Number of writes since the last time traversal visited this page. */
Xiao Guangronge5691a82016-02-24 17:51:12 +0800350 atomic_t write_flooding_count;
Zhang Xiantaod657a982007-12-14 09:41:22 +0800351};
352
Avi Kivity1c083642009-01-04 12:39:07 +0200353struct kvm_pio_request {
354 unsigned long count;
Avi Kivity1c083642009-01-04 12:39:07 +0200355 int in;
356 int port;
357 int size;
Avi Kivity1c083642009-01-04 12:39:07 +0200358};
359
Yu Zhang855feb62017-08-24 20:27:55 +0800360#define PT64_ROOT_MAX_LEVEL 5
Yu Zhang2a7266a2017-08-24 20:27:54 +0800361
Xiao Guangronga0a64f52015-08-05 12:04:21 +0800362struct rsvd_bits_validate {
Yu Zhang2a7266a2017-08-24 20:27:54 +0800363 u64 rsvd_bits_mask[2][PT64_ROOT_MAX_LEVEL];
Xiao Guangronga0a64f52015-08-05 12:04:21 +0800364 u64 bad_mt_xwr;
365};
366
Junaid Shahid7c390d32018-06-27 14:59:06 -0700367struct kvm_mmu_root_info {
368 gpa_t cr3;
369 hpa_t hpa;
370};
371
372#define KVM_MMU_ROOT_INFO_INVALID \
373 ((struct kvm_mmu_root_info) { .cr3 = INVALID_PAGE, .hpa = INVALID_PAGE })
374
Junaid Shahidb94742c2018-06-27 14:59:20 -0700375#define KVM_MMU_NUM_PREV_ROOTS 3
376
Zhang Xiantaod657a982007-12-14 09:41:22 +0800377/*
Yu Zhang855feb62017-08-24 20:27:55 +0800378 * x86 supports 4 paging modes (5-level 64-bit, 4-level 64-bit, 3-level 32-bit,
379 * and 2-level 32-bit). The kvm_mmu structure abstracts the details of the
380 * current mmu mode.
Zhang Xiantaod657a982007-12-14 09:41:22 +0800381 */
382struct kvm_mmu {
Joerg Roedelf43addd2010-09-10 17:30:40 +0200383 void (*set_cr3)(struct kvm_vcpu *vcpu, unsigned long root);
Joerg Roedel5777ed32010-09-10 17:30:42 +0200384 unsigned long (*get_cr3)(struct kvm_vcpu *vcpu);
Avi Kivitye4e517b2011-07-28 11:36:17 +0300385 u64 (*get_pdptr)(struct kvm_vcpu *vcpu, int index);
Xiao Guangrong78b2c542010-12-07 10:48:06 +0800386 int (*page_fault)(struct kvm_vcpu *vcpu, gva_t gva, u32 err,
387 bool prefault);
Avi Kivity6389ee92010-11-29 16:12:30 +0200388 void (*inject_page_fault)(struct kvm_vcpu *vcpu,
389 struct x86_exception *fault);
Gleb Natapov1871c602010-02-10 14:21:32 +0200390 gpa_t (*gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t gva, u32 access,
Avi Kivityab9ae312010-11-22 17:53:26 +0200391 struct x86_exception *exception);
Paolo Bonzini54987b72014-09-02 13:23:06 +0200392 gpa_t (*translate_gpa)(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access,
393 struct x86_exception *exception);
Marcelo Tosattie8bc2172008-09-23 13:18:33 -0300394 int (*sync_page)(struct kvm_vcpu *vcpu,
Xiao Guangronga4a8e6f2010-11-19 17:04:03 +0800395 struct kvm_mmu_page *sp);
Junaid Shahid7eb77e92018-06-27 14:59:16 -0700396 void (*invlpg)(struct kvm_vcpu *vcpu, gva_t gva, hpa_t root_hpa);
Xiao Guangrong0f53b5b2011-03-09 15:43:51 +0800397 void (*update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
Xiao Guangrong7c562522011-03-28 10:29:27 +0800398 u64 *spte, const void *pte);
Zhang Xiantaod657a982007-12-14 09:41:22 +0800399 hpa_t root_hpa;
Vitaly Kuznetsov36d9594d2018-10-08 21:28:10 +0200400 union kvm_mmu_role mmu_role;
Paolo Bonziniae1e2d12017-03-30 11:55:30 +0200401 u8 root_level;
402 u8 shadow_root_level;
403 u8 ept_ad;
Joerg Roedelc5a78f2b2010-09-10 17:30:39 +0200404 bool direct_map;
Junaid Shahidb94742c2018-06-27 14:59:20 -0700405 struct kvm_mmu_root_info prev_roots[KVM_MMU_NUM_PREV_ROOTS];
Zhang Xiantaod657a982007-12-14 09:41:22 +0800406
Avi Kivity97d64b72012-09-12 14:52:00 +0300407 /*
408 * Bitmap; bit set = permission fault
409 * Byte index: page fault error code [4:1]
410 * Bit index: pte permissions in ACC_* format
411 */
412 u8 permissions[16];
413
Huaitong Han2d344102016-03-22 16:51:19 +0800414 /*
415 * The pkru_mask indicates if protection key checks are needed. It
416 * consists of 16 domains indexed by page fault error code bits [4:1],
417 * with PFEC.RSVD replaced by ACC_USER_MASK from the page tables.
418 * Each domain has 2 bits which are ANDed with AD and WD from PKRU.
419 */
420 u32 pkru_mask;
421
Zhang Xiantaod657a982007-12-14 09:41:22 +0800422 u64 *pae_root;
Joerg Roedel81407ca2010-09-10 17:31:00 +0200423 u64 *lm_root;
Xiao Guangrongc258b622015-08-05 12:04:24 +0800424
425 /*
426 * check zero bits on shadow page table entries, these
427 * bits include not only hardware reserved bits but also
428 * the bits spte never used.
429 */
430 struct rsvd_bits_validate shadow_zero_check;
431
Xiao Guangronga0a64f52015-08-05 12:04:21 +0800432 struct rsvd_bits_validate guest_rsvd_check;
Joerg Roedelff03a072010-09-10 17:30:57 +0200433
Paolo Bonzini6bb69c92016-02-23 12:51:19 +0100434 /* Can have large pages at levels 2..last_nonleaf_level-1. */
435 u8 last_nonleaf_level;
Avi Kivity6fd01b72012-09-12 20:46:56 +0300436
Joerg Roedel2d48a982010-09-10 17:31:01 +0200437 bool nx;
438
Joerg Roedelff03a072010-09-10 17:30:57 +0200439 u64 pdptrs[4]; /* pae */
Zhang Xiantaod657a982007-12-14 09:41:22 +0800440};
441
Gleb Natapovf5132b02011-11-10 14:57:22 +0200442enum pmc_type {
443 KVM_PMC_GP = 0,
444 KVM_PMC_FIXED,
445};
446
447struct kvm_pmc {
448 enum pmc_type type;
449 u8 idx;
450 u64 counter;
451 u64 eventsel;
452 struct perf_event *perf_event;
453 struct kvm_vcpu *vcpu;
454};
455
456struct kvm_pmu {
457 unsigned nr_arch_gp_counters;
458 unsigned nr_arch_fixed_counters;
459 unsigned available_event_types;
460 u64 fixed_ctr_ctrl;
461 u64 global_ctrl;
462 u64 global_status;
463 u64 global_ovf_ctrl;
464 u64 counter_bitmask[2];
465 u64 global_ctrl_mask;
Andi Kleen103af0a2013-07-18 15:57:02 -0700466 u64 reserved_bits;
Gleb Natapovf5132b02011-11-10 14:57:22 +0200467 u8 version;
Robert Richter15c7ad52012-06-20 20:46:33 +0200468 struct kvm_pmc gp_counters[INTEL_PMC_MAX_GENERIC];
469 struct kvm_pmc fixed_counters[INTEL_PMC_MAX_FIXED];
Gleb Natapovf5132b02011-11-10 14:57:22 +0200470 struct irq_work irq_work;
471 u64 reprogram_pmi;
472};
473
Wei Huang25462f72015-06-19 15:45:05 +0200474struct kvm_pmu_ops;
475
Paolo Bonzini360b9482014-02-21 09:55:56 +0100476enum {
477 KVM_DEBUGREG_BP_ENABLED = 1,
Paolo Bonzinic77fb5f2014-02-21 10:17:24 +0100478 KVM_DEBUGREG_WONT_EXIT = 2,
Nadav Amitae561ed2015-04-02 03:10:37 +0300479 KVM_DEBUGREG_RELOAD = 4,
Paolo Bonzini360b9482014-02-21 09:55:56 +0100480};
481
Xiao Guangrong86fd5272015-06-15 16:55:27 +0800482struct kvm_mtrr_range {
483 u64 base;
484 u64 mask;
Xiao Guangrong19efffa2015-06-15 16:55:31 +0800485 struct list_head node;
Xiao Guangrong86fd5272015-06-15 16:55:27 +0800486};
487
Xiao Guangrong70109e72015-06-15 16:55:24 +0800488struct kvm_mtrr {
Xiao Guangrong86fd5272015-06-15 16:55:27 +0800489 struct kvm_mtrr_range var_ranges[KVM_NR_VAR_MTRR];
Xiao Guangrong70109e72015-06-15 16:55:24 +0800490 mtrr_type fixed_ranges[KVM_NR_FIXED_MTRR_REGION];
Xiao Guangrong10fac2d2015-06-15 16:55:26 +0800491 u64 deftype;
Xiao Guangrong19efffa2015-06-15 16:55:31 +0800492
493 struct list_head head;
Xiao Guangrong70109e72015-06-15 16:55:24 +0800494};
495
Andrey Smetanin1f4b34f2015-11-30 19:22:21 +0300496/* Hyper-V SynIC timer */
497struct kvm_vcpu_hv_stimer {
498 struct hrtimer timer;
499 int index;
500 u64 config;
501 u64 count;
502 u64 exp_time;
503 struct hv_message msg;
504 bool msg_pending;
505};
506
Andrey Smetanin5c9194122015-11-10 15:36:34 +0300507/* Hyper-V synthetic interrupt controller (SynIC)*/
508struct kvm_vcpu_hv_synic {
509 u64 version;
510 u64 control;
511 u64 msg_page;
512 u64 evt_page;
513 atomic64_t sint[HV_SYNIC_SINT_COUNT];
514 atomic_t sint_to_gsi[HV_SYNIC_SINT_COUNT];
515 DECLARE_BITMAP(auto_eoi_bitmap, 256);
516 DECLARE_BITMAP(vec_bitmap, 256);
517 bool active;
Roman Kaganefc479e2017-06-22 16:51:01 +0300518 bool dont_zero_synic_pages;
Andrey Smetanin5c9194122015-11-10 15:36:34 +0300519};
520
Andrey Smetanine83d5882015-07-03 15:01:34 +0300521/* Hyper-V per vcpu emulation context */
522struct kvm_vcpu_hv {
Roman Kagand3457c82017-07-14 17:13:20 +0300523 u32 vp_index;
Andrey Smetanine83d5882015-07-03 15:01:34 +0300524 u64 hv_vapic;
Andrey Smetanin9eec50b2015-09-16 12:29:50 +0300525 s64 runtime_offset;
Andrey Smetanin5c9194122015-11-10 15:36:34 +0300526 struct kvm_vcpu_hv_synic synic;
Andrey Smetanindb3975712015-11-10 15:36:35 +0300527 struct kvm_hyperv_exit exit;
Andrey Smetanin1f4b34f2015-11-30 19:22:21 +0300528 struct kvm_vcpu_hv_stimer stimer[HV_SYNIC_STIMER_COUNT];
529 DECLARE_BITMAP(stimer_pending_bitmap, HV_SYNIC_STIMER_COUNT);
Vitaly Kuznetsove6b6c482018-10-08 19:19:04 +0200530 cpumask_t tlb_flush;
Andrey Smetanine83d5882015-07-03 15:01:34 +0300531};
532
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800533struct kvm_vcpu_arch {
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -0300534 /*
535 * rip and regs accesses must go through
536 * kvm_{register,rip}_{read,write} functions.
537 */
538 unsigned long regs[NR_VCPU_REGS];
539 u32 regs_avail;
540 u32 regs_dirty;
Zhang Xiantao34c16ee2007-10-20 15:34:38 +0800541
542 unsigned long cr0;
Avi Kivitye8467fd2009-12-29 18:43:06 +0200543 unsigned long cr0_guest_owned_bits;
Zhang Xiantao34c16ee2007-10-20 15:34:38 +0800544 unsigned long cr2;
545 unsigned long cr3;
546 unsigned long cr4;
Avi Kivityfc78f512009-12-07 12:16:48 +0200547 unsigned long cr4_guest_owned_bits;
Zhang Xiantao34c16ee2007-10-20 15:34:38 +0800548 unsigned long cr8;
Paolo Bonzinib9dd21e2017-08-23 23:14:38 +0200549 u32 pkru;
Alexander Graf1371d902008-11-25 20:17:04 +0100550 u32 hflags;
Avi Kivityf6801df2010-01-21 15:31:50 +0200551 u64 efer;
Zhang Xiantao34c16ee2007-10-20 15:34:38 +0800552 u64 apic_base;
553 struct kvm_lapic *apic; /* kernel irqchip context */
Andrey Smetanind62caab2015-11-10 15:36:33 +0300554 bool apicv_active;
Liran Alone40ff1d2018-03-21 02:50:31 +0200555 bool load_eoi_exitmap_pending;
Andrey Smetanin63086302015-11-10 15:36:32 +0300556 DECLARE_BITMAP(ioapic_handled_vectors, 256);
Gleb Natapov41383772012-04-19 14:06:29 +0300557 unsigned long apic_attention;
Gleb Natapove1035712009-03-05 16:34:59 +0200558 int32_t apic_arb_prio;
Zhang Xiantao34c16ee2007-10-20 15:34:38 +0800559 int mp_state;
Zhang Xiantao34c16ee2007-10-20 15:34:38 +0800560 u64 ia32_misc_enable_msr;
Paolo Bonzini64d60672015-05-07 11:36:11 +0200561 u64 smbase;
Liran Alon52797bf2017-11-15 13:43:14 +0200562 u64 smi_count;
Avi Kivityb209749f2007-10-22 16:50:39 +0200563 bool tpr_access_reporting;
Wanpeng Li20300092014-12-02 19:14:59 +0800564 u64 ia32_xss;
Wanpeng Li518e7b92018-02-28 14:03:31 +0800565 u64 microcode_version;
Zhang Xiantao34c16ee2007-10-20 15:34:38 +0800566
Joerg Roedel14dfe852010-09-10 17:30:49 +0200567 /*
568 * Paging state of the vcpu
569 *
570 * If the vcpu runs in guest mode with two level paging this still saves
571 * the paging mode of the l1 guest. This context is always used to
572 * handle faults.
573 */
Vitaly Kuznetsov44dd3ff2018-10-08 21:28:05 +0200574 struct kvm_mmu *mmu;
575
576 /* Non-nested MMU for L1 */
577 struct kvm_mmu root_mmu;
Joerg Roedel8df25a32010-09-10 17:30:46 +0200578
Vitaly Kuznetsov14c07ad2018-10-08 21:28:08 +0200579 /* L1 MMU when running nested */
580 struct kvm_mmu guest_mmu;
581
Joerg Roedel8df25a32010-09-10 17:30:46 +0200582 /*
Joerg Roedel6539e732010-09-10 17:30:50 +0200583 * Paging state of an L2 guest (used for nested npt)
584 *
585 * This context will save all necessary information to walk page tables
586 * of the an L2 guest. This context is only initialized for page table
587 * walking and not for faulting since we never handle l2 page faults on
588 * the host.
589 */
590 struct kvm_mmu nested_mmu;
591
592 /*
Joerg Roedel14dfe852010-09-10 17:30:49 +0200593 * Pointer to the mmu context currently used for
594 * gva_to_gpa translations.
595 */
596 struct kvm_mmu *walk_mmu;
597
Xiao Guangrong53c07b12011-05-15 23:26:20 +0800598 struct kvm_mmu_memory_cache mmu_pte_list_desc_cache;
Zhang Xiantao34c16ee2007-10-20 15:34:38 +0800599 struct kvm_mmu_memory_cache mmu_page_cache;
600 struct kvm_mmu_memory_cache mmu_page_header_cache;
601
Rik van Rielf775b132017-11-14 16:54:23 -0500602 /*
603 * QEMU userspace and the guest each have their own FPU state.
604 * In vcpu_run, we switch between the user and guest FPU contexts.
605 * While running a VCPU, the VCPU thread will have the guest FPU
606 * context.
607 *
608 * Note that while the PKRU state lives inside the fpu registers,
609 * it is switched out separately at VMENTER and VMEXIT time. The
610 * "guest_fpu" state here contains the guest FPU context, with the
611 * host PRKU bits.
612 */
613 struct fpu user_fpu;
Sheng Yang98918832010-05-17 17:08:28 +0800614 struct fpu guest_fpu;
Rik van Rielf775b132017-11-14 16:54:23 -0500615
Dexuan Cui2acf9232010-06-10 11:27:12 +0800616 u64 xcr0;
Paolo Bonzinid7876f12013-10-02 16:06:15 +0200617 u64 guest_supported_xcr0;
Paolo Bonzini4344ee92013-10-02 16:06:16 +0200618 u32 guest_xstate_size;
Zhang Xiantao34c16ee2007-10-20 15:34:38 +0800619
Zhang Xiantao34c16ee2007-10-20 15:34:38 +0800620 struct kvm_pio_request pio;
621 void *pio_data;
622
Gleb Natapov66fd3f72009-05-11 13:35:50 +0300623 u8 event_exit_inst_len;
624
Avi Kivity298101d2007-11-25 13:41:11 +0200625 struct kvm_queued_exception {
626 bool pending;
Wanpeng Li664f8e22017-08-24 03:35:09 -0700627 bool injected;
Avi Kivity298101d2007-11-25 13:41:11 +0200628 bool has_error_code;
629 u8 nr;
630 u32 error_code;
Jim Mattsonc8514362018-10-16 14:29:19 -0700631 unsigned long payload;
632 bool has_payload;
Wanpeng Liadfe20f2017-07-13 18:30:41 -0700633 u8 nested_apf;
Avi Kivity298101d2007-11-25 13:41:11 +0200634 } exception;
635
Avi Kivity937a7ea2008-07-03 15:17:01 +0300636 struct kvm_queued_interrupt {
Liran Alon04140b42018-03-23 03:01:31 +0300637 bool injected;
Gleb Natapov66fd3f72009-05-11 13:35:50 +0300638 bool soft;
Avi Kivity937a7ea2008-07-03 15:17:01 +0300639 u8 nr;
640 } interrupt;
641
Zhang Xiantao34c16ee2007-10-20 15:34:38 +0800642 int halt_request; /* real mode on Intel only */
643
644 int cpuid_nent;
Dan Kenigsberg07716712007-11-21 17:10:04 +0200645 struct kvm_cpuid_entry2 cpuid_entries[KVM_MAX_CPUID_ENTRIES];
Eugene Korenevsky5a4f55c2015-03-29 23:56:12 +0300646
647 int maxphyaddr;
648
Zhang Xiantao34c16ee2007-10-20 15:34:38 +0800649 /* emulate context */
650
651 struct x86_emulate_ctxt emulate_ctxt;
Gleb Natapov7ae441e2011-03-31 12:06:41 +0200652 bool emulate_regs_need_sync_to_vcpu;
653 bool emulate_regs_need_sync_from_vcpu;
Gleb Natapov716d51a2012-09-03 15:24:26 +0300654 int (*complete_userspace_io)(struct kvm_vcpu *vcpu);
Glauber de Oliveira Costa18068522008-02-15 17:52:47 -0200655
656 gpa_t time;
Gerd Hoffmann50d0a0f2008-06-03 16:17:31 +0200657 struct pvclock_vcpu_time_info hv_clock;
Zachary Amsdene48672f2010-08-19 22:07:23 -1000658 unsigned int hw_tsc_khz;
Andy Honig0b794592013-02-20 14:48:10 -0800659 struct gfn_to_hva_cache pv_time;
660 bool pv_time_enabled;
Marcelo Tosatti51d59c62012-08-03 15:57:49 -0300661 /* set guest stopped flag in pvclock flags field */
662 bool pvclock_set_guest_stopped_request;
Glauber Costac9aaa892011-07-11 15:28:14 -0400663
664 struct {
665 u64 msr_val;
666 u64 last_steal;
Glauber Costac9aaa892011-07-11 15:28:14 -0400667 struct gfn_to_hva_cache stime;
668 struct kvm_steal_time steal;
669 } st;
670
Luiz Capitulinoa545ab62016-09-07 14:47:19 -0400671 u64 tsc_offset;
Zachary Amsden1d5f0662010-08-19 22:07:30 -1000672 u64 last_guest_tsc;
Zachary Amsden6f526ec2012-02-03 15:43:54 -0200673 u64 last_host_tsc;
Zachary Amsden0dd6a6e2012-02-03 15:43:56 -0200674 u64 tsc_offset_adjustment;
Zachary Amsdene26101b2012-02-03 15:43:57 -0200675 u64 this_tsc_nsec;
676 u64 this_tsc_write;
Tomasz Grabiec0d3da0d2014-06-24 09:42:43 +0200677 u64 this_tsc_generation;
Zachary Amsdenc2855452010-09-18 14:38:15 -1000678 bool tsc_catchup;
Zachary Amsdencc578282012-02-03 15:43:50 -0200679 bool tsc_always_catchup;
680 s8 virtual_tsc_shift;
681 u32 virtual_tsc_mult;
682 u32 virtual_tsc_khz;
Will Auldba904632012-11-29 12:42:50 -0800683 s64 ia32_tsc_adjust_msr;
Haozhong Zhangad7218832015-10-20 15:39:02 +0800684 u64 tsc_scaling_ratio;
Sheng Yang3419ffc2008-05-15 09:52:48 +0800685
Avi Kivity7460fb4a2011-09-20 13:43:14 +0300686 atomic_t nmi_queued; /* unprocessed asynchronous NMIs */
687 unsigned nmi_pending; /* NMI queued after currently running handler */
688 bool nmi_injected; /* Trying to inject an NMI this entry */
Paolo Bonzinif0778252015-04-01 15:06:40 +0200689 bool smi_pending; /* SMI queued after currently running handler */
Avi Kivity9ba075a2008-05-26 20:06:35 +0300690
Xiao Guangrong70109e72015-06-15 16:55:24 +0800691 struct kvm_mtrr mtrr_state;
Paolo Bonzini7cb060a2014-06-19 11:40:18 +0200692 u64 pat;
Jan Kiszka42dbaa52008-12-15 13:52:10 +0100693
Paolo Bonzini360b9482014-02-21 09:55:56 +0100694 unsigned switch_db_regs;
Jan Kiszka42dbaa52008-12-15 13:52:10 +0100695 unsigned long db[KVM_NR_DB_REGS];
696 unsigned long dr6;
697 unsigned long dr7;
698 unsigned long eff_db[KVM_NR_DB_REGS];
Jan Kiszkac8639012012-09-21 05:42:55 +0200699 unsigned long guest_debug_dr7;
Kyle Hueydb2336a2017-03-20 01:16:28 -0700700 u64 msr_platform_info;
701 u64 msr_misc_features_enables;
Huang Ying890ca9a2009-05-11 16:48:15 +0800702
703 u64 mcg_cap;
704 u64 mcg_status;
705 u64 mcg_ctl;
Ashok Rajc45dcc72016-06-22 14:59:56 +0800706 u64 mcg_ext_ctl;
Huang Ying890ca9a2009-05-11 16:48:15 +0800707 u64 *mce_banks;
Jan Kiszka94fe45d2009-10-18 13:24:44 +0200708
Xiao Guangrongbebb1062011-07-12 03:23:20 +0800709 /* Cache MMIO info */
710 u64 mmio_gva;
711 unsigned access;
712 gfn_t mmio_gfn;
David Matlack56f17dd2014-08-18 15:46:07 -0700713 u64 mmio_gen;
Xiao Guangrongbebb1062011-07-12 03:23:20 +0800714
Gleb Natapovf5132b02011-11-10 14:57:22 +0200715 struct kvm_pmu pmu;
716
Jan Kiszka94fe45d2009-10-18 13:24:44 +0200717 /* used for guest single stepping over the given code position */
Jan Kiszka94fe45d2009-10-18 13:24:44 +0200718 unsigned long singlestep_rip;
Jan Kiszkaf92653e2010-02-23 17:47:55 +0100719
Andrey Smetanine83d5882015-07-03 15:01:34 +0300720 struct kvm_vcpu_hv hyperv;
Sheng Yangf5f48ee2010-06-30 12:25:15 +0800721
722 cpumask_var_t wbinvd_dirty_mask;
Gleb Natapovaf585b92010-10-14 11:22:46 +0200723
Xiao Guangrong1cb3f3a2011-09-22 17:02:48 +0800724 unsigned long last_retry_eip;
725 unsigned long last_retry_addr;
726
Gleb Natapovaf585b92010-10-14 11:22:46 +0200727 struct {
728 bool halted;
729 gfn_t gfns[roundup_pow_of_two(ASYNC_PF_PER_VCPU)];
Gleb Natapov344d9582010-10-14 11:22:50 +0200730 struct gfn_to_hva_cache data;
731 u64 msr_val;
Gleb Natapov7c907052010-10-14 11:22:53 +0200732 u32 id;
Gleb Natapov6adba522010-10-14 11:22:55 +0200733 bool send_user_only;
Wanpeng Li1261bfa2017-07-13 18:30:40 -0700734 u32 host_apf_reason;
Wanpeng Liadfe20f2017-07-13 18:30:41 -0700735 unsigned long nested_apf_token;
Wanpeng Li52a5c152017-07-13 18:30:42 -0700736 bool delivery_as_pf_vmexit;
Gleb Natapovaf585b92010-10-14 11:22:46 +0200737 } apf;
Boris Ostrovsky2b036c62012-01-09 14:00:35 -0500738
739 /* OSVW MSRs (AMD only) */
740 struct {
741 u64 length;
742 u64 status;
743 } osvw;
Michael S. Tsirkinae7a2a32012-06-24 19:25:07 +0300744
745 struct {
746 u64 msr_val;
747 struct gfn_to_hva_cache data;
748 } pv_eoi;
Xiao Guangrong93c05d32013-01-13 23:49:07 +0800749
750 /*
751 * Indicate whether the access faults on its page table in guest
752 * which is set when fix page fault and used to detect unhandeable
753 * instruction.
754 */
755 bool write_fault_to_shadow_pgtable;
Yang Zhang25d92082013-08-06 12:00:32 +0300756
757 /* set at EPT violation at this point */
758 unsigned long exit_qualification;
Srivatsa Vaddagiri6aef2662013-08-26 14:18:34 +0530759
760 /* pv related host specific info */
761 struct {
762 bool pv_unhalted;
763 } pv;
Steve Rutherford7543a632015-07-29 23:21:41 -0700764
765 int pending_ioapic_eoi;
Steve Rutherford1c1a9ce2015-07-30 11:27:16 +0200766 int pending_external_vector;
Tom Lendacky0f89b202016-12-14 14:59:23 -0500767
Brijesh Singh618232e2017-08-17 18:36:57 +0200768 /* GPA available */
Tom Lendacky0f89b202016-12-14 14:59:23 -0500769 bool gpa_available;
Brijesh Singh618232e2017-08-17 18:36:57 +0200770 gpa_t gpa_val;
Longpeng(Mike)de63ad42017-08-08 12:05:33 +0800771
772 /* be preempted when it's in kernel-mode(cpl=0) */
773 bool preempted_in_kernel;
Paolo Bonzinic595cee2018-07-02 13:07:14 +0200774
775 /* Flush the L1 Data cache for L1TF mitigation on VMENTER */
776 bool l1tf_flush_l1d;
Zhang Xiantao34c16ee2007-10-20 15:34:38 +0800777};
778
Takuya Yoshikawadb3fe4e2012-02-08 13:02:18 +0900779struct kvm_lpage_info {
Xiao Guangrong92f94f12016-02-24 17:51:06 +0800780 int disallow_lpage;
Takuya Yoshikawadb3fe4e2012-02-08 13:02:18 +0900781};
782
783struct kvm_arch_memory_slot {
Takuya Yoshikawa018aabb52015-11-20 17:41:28 +0900784 struct kvm_rmap_head *rmap[KVM_NR_PAGE_SIZES];
Takuya Yoshikawadb3fe4e2012-02-08 13:02:18 +0900785 struct kvm_lpage_info *lpage_info[KVM_NR_PAGE_SIZES - 1];
Xiao Guangrong21ebbed2016-02-24 17:51:09 +0800786 unsigned short *gfn_track[KVM_PAGE_TRACK_MAX];
Takuya Yoshikawadb3fe4e2012-02-08 13:02:18 +0900787};
788
Radim Krčmář3548a252015-02-12 19:41:33 +0100789/*
790 * We use as the mode the number of bits allocated in the LDR for the
791 * logical processor ID. It happens that these are all powers of two.
792 * This makes it is very easy to detect cases where the APICs are
793 * configured for multiple modes; in that case, we cannot use the map and
794 * hence cannot use kvm_irq_delivery_to_apic_fast either.
795 */
796#define KVM_APIC_MODE_XAPIC_CLUSTER 4
797#define KVM_APIC_MODE_XAPIC_FLAT 8
798#define KVM_APIC_MODE_X2APIC 16
799
Gleb Natapov1e08ec42012-09-13 17:19:24 +0300800struct kvm_apic_map {
801 struct rcu_head rcu;
Radim Krčmář3548a252015-02-12 19:41:33 +0100802 u8 mode;
Radim Krčmář0ca52e72016-07-12 22:09:20 +0200803 u32 max_apic_id;
Radim Krčmáře45115b2016-07-12 22:09:19 +0200804 union {
805 struct kvm_lapic *xapic_flat_map[8];
806 struct kvm_lapic *xapic_cluster_map[16][4];
807 };
Radim Krčmář0ca52e72016-07-12 22:09:20 +0200808 struct kvm_lapic *phys_map[];
Gleb Natapov1e08ec42012-09-13 17:19:24 +0300809};
810
Andrey Smetanine83d5882015-07-03 15:01:34 +0300811/* Hyper-V emulation context */
812struct kvm_hv {
Paolo Bonzini3f5ad8b2016-12-12 10:12:53 +0100813 struct mutex hv_lock;
Andrey Smetanine83d5882015-07-03 15:01:34 +0300814 u64 hv_guest_os_id;
815 u64 hv_hypercall;
816 u64 hv_tsc_page;
Andrey Smetanine7d95132015-07-03 15:01:37 +0300817
818 /* Hyper-v based guest crash (NT kernel bugcheck) parameters */
819 u64 hv_crash_param[HV_X64_MSR_CRASH_PARAMS];
820 u64 hv_crash_ctl;
Paolo Bonzini095cf552016-02-08 12:54:12 +0100821
822 HV_REFERENCE_TSC_PAGE tsc_ref;
Roman Kaganfaeb7832018-02-01 16:48:32 +0300823
824 struct idr conn_to_evt;
Vitaly Kuznetsova2e164e2018-03-01 15:15:12 +0100825
826 u64 hv_reenlightenment_control;
827 u64 hv_tsc_emulation_control;
828 u64 hv_tsc_emulation_status;
Vitaly Kuznetsov87ee6132018-09-26 19:02:56 +0200829
830 /* How many vCPUs have VP index != vCPU index */
831 atomic_t num_mismatched_vp_indexes;
Andrey Smetanine83d5882015-07-03 15:01:34 +0300832};
833
Radim Krčmář49776fa2016-12-16 16:10:02 +0100834enum kvm_irqchip_mode {
835 KVM_IRQCHIP_NONE,
836 KVM_IRQCHIP_KERNEL, /* created with KVM_CREATE_IRQCHIP */
837 KVM_IRQCHIP_SPLIT, /* created with KVM_CAP_SPLIT_IRQCHIP */
838};
839
Marcelo Tosattifef9cce2009-12-23 14:35:17 -0200840struct kvm_arch {
Dave Hansen49d5ca22010-08-19 18:11:28 -0700841 unsigned int n_used_mmu_pages;
Zhang Xiantaof05e70a2007-12-14 10:01:48 +0800842 unsigned int n_requested_mmu_pages;
Dave Hansen39de71e2010-08-19 18:11:14 -0700843 unsigned int n_max_mmu_pages;
Xiao Guangrong332b2072011-05-15 23:20:27 +0800844 unsigned int indirect_shadow_pages;
Xiao Guangrong5304b8d2013-05-31 08:36:22 +0800845 unsigned long mmu_valid_gen;
Zhang Xiantaof05e70a2007-12-14 10:01:48 +0800846 struct hlist_head mmu_page_hash[KVM_NUM_MMU_PAGES];
847 /*
848 * Hash table of struct kvm_mmu_page.
849 */
850 struct list_head active_mmu_pages;
Xiao Guangrong365c8862013-05-31 08:36:29 +0800851 struct list_head zapped_obsolete_pages;
Xiao Guangrong13d268c2016-02-24 17:51:16 +0800852 struct kvm_page_track_notifier_node mmu_sp_tracker;
Xiao Guangrong0eb05bf2016-02-24 17:51:13 +0800853 struct kvm_page_track_notifier_head track_notifier_head;
Xiao Guangrong365c8862013-05-31 08:36:29 +0800854
Ben-Ami Yassour4d5c5d02008-07-28 19:26:26 +0300855 struct list_head assigned_dev_head;
Joerg Roedel19de40a2008-12-03 14:43:34 +0100856 struct iommu_domain *iommu_domain;
Alex Williamsond96eb2c2013-10-30 11:02:23 -0600857 bool iommu_noncoherent;
Alex Williamsone0f0bbc2013-10-30 11:02:30 -0600858#define __KVM_HAVE_ARCH_NONCOHERENT_DMA
859 atomic_t noncoherent_dma_count;
Paolo Bonzini5544eb92015-07-07 15:41:58 +0200860#define __KVM_HAVE_ARCH_ASSIGNED_DEVICE
861 atomic_t assigned_device_count;
Zhang Xiantaod7deeeb02007-12-14 10:17:34 +0800862 struct kvm_pic *vpic;
863 struct kvm_ioapic *vioapic;
Sheng Yang78376992008-01-28 05:10:22 +0800864 struct kvm_pit *vpit;
Radim Krčmář42720132015-07-01 15:31:49 +0200865 atomic_t vapics_in_nmi_mode;
Gleb Natapov1e08ec42012-09-13 17:19:24 +0300866 struct mutex apic_map_lock;
867 struct kvm_apic_map *apic_map;
Zhang Xiantaobfc6d222007-12-14 10:20:16 +0800868
Tang Chenc24ae0d2014-09-24 15:57:58 +0800869 bool apic_access_page_done;
Glauber de Oliveira Costa18068522008-02-15 17:52:47 -0200870
871 gpa_t wall_clock;
Sheng Yangb7ebfb02008-04-25 21:44:52 +0800872
Wanpeng Li4d5422c2018-03-12 04:53:02 -0700873 bool mwait_in_guest;
Wanpeng Licaa057a2018-03-12 04:53:03 -0700874 bool hlt_in_guest;
Wanpeng Lib31c1142018-03-12 04:53:04 -0700875 bool pause_in_guest;
Wanpeng Li4d5422c2018-03-12 04:53:02 -0700876
Sheng Yang5550af42008-10-15 20:15:06 +0800877 unsigned long irq_sources_bitmap;
Glauber Costaafbcf7a2009-10-16 15:28:36 -0400878 s64 kvmclock_offset;
Jan Kiszka038f8c12011-02-04 10:49:11 +0100879 raw_spinlock_t tsc_write_lock;
Zachary Amsdenf38e0982010-08-19 22:07:20 -1000880 u64 last_tsc_nsec;
Zachary Amsdenf38e0982010-08-19 22:07:20 -1000881 u64 last_tsc_write;
Zachary Amsden5d3cb0f62012-02-03 15:43:51 -0200882 u32 last_tsc_khz;
Zachary Amsdene26101b2012-02-03 15:43:57 -0200883 u64 cur_tsc_nsec;
884 u64 cur_tsc_write;
885 u64 cur_tsc_offset;
Tomasz Grabiec0d3da0d2014-06-24 09:42:43 +0200886 u64 cur_tsc_generation;
Marcelo Tosattib48aa972012-11-27 23:29:03 -0200887 int nr_vcpus_matched_tsc;
Ed Swierkffde22a2009-10-15 15:21:43 -0700888
Marcelo Tosattid8281992012-11-27 23:29:01 -0200889 spinlock_t pvclock_gtod_sync_lock;
890 bool use_master_clock;
891 u64 master_kernel_ns;
Thomas Gleixnera5a1d1c2016-12-21 20:32:01 +0100892 u64 master_cycle_now;
Andrew Jones7e44e442014-02-28 12:52:54 +0100893 struct delayed_work kvmclock_update_work;
Andrew Jones332967a2014-02-28 12:52:55 +0100894 struct delayed_work kvmclock_sync_work;
Marcelo Tosattid8281992012-11-27 23:29:01 -0200895
Ed Swierkffde22a2009-10-15 15:21:43 -0700896 struct kvm_xen_hvm_config xen_hvm_config;
Gleb Natapov55cd8e52010-01-17 15:51:22 +0200897
Paolo Bonzini6ef768f2014-11-20 13:45:31 +0100898 /* reads protected by irq_srcu, writes by irq_lock */
899 struct hlist_head mask_notifier_list;
900
Andrey Smetanine83d5882015-07-03 15:01:34 +0300901 struct kvm_hv hyperv;
Xiao Guangrongb034cf02010-12-23 16:08:35 +0800902
903 #ifdef CONFIG_KVM_MMU_AUDIT
904 int audit_point;
905 #endif
Marcelo Tosatti54750f22015-01-20 15:54:52 -0200906
Ladi Proseka826faf2017-06-26 09:56:43 +0200907 bool backwards_tsc_observed;
Marcelo Tosatti54750f22015-01-20 15:54:52 -0200908 bool boot_vcpu_runs_old_kvmclock;
Paolo Bonzinid71ba782015-07-29 11:56:48 +0200909 u32 bsp_vcpu_id;
Nadav Amit90de4a12015-04-13 01:53:41 +0300910
911 u64 disabled_quirks;
Steve Rutherford49df6392015-07-29 23:21:40 -0700912
Radim Krčmář49776fa2016-12-16 16:10:02 +0100913 enum kvm_irqchip_mode irqchip_mode;
Steve Rutherfordb053b2a2015-07-29 23:32:35 -0700914 u8 nr_reserved_ioapic_pins;
Feng Wu520040142016-01-25 16:53:33 +0800915
916 bool disabled_lapic_found;
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -0500917
Radim Krčmář371313132016-07-12 22:09:27 +0200918 bool x2apic_format;
Radim Krčmářc5192652016-07-12 22:09:28 +0200919 bool x2apic_broadcast_quirk_disabled;
Drew Schmitt6fbbde92018-08-20 10:32:15 -0700920
921 bool guest_can_read_msr_platform_info;
Zhang Xiantaod69fb812007-12-14 09:54:20 +0800922};
923
Zhang Xiantao07114562007-12-14 10:23:23 +0800924struct kvm_vm_stat {
Suraj Jitindar Singh8a7e75d2016-08-02 14:03:22 +1000925 ulong mmu_shadow_zapped;
926 ulong mmu_pte_write;
927 ulong mmu_pte_updated;
928 ulong mmu_pde_zapped;
929 ulong mmu_flooded;
930 ulong mmu_recycled;
931 ulong mmu_cache_miss;
932 ulong mmu_unsync;
933 ulong remote_tlb_flush;
934 ulong lpages;
David Matlackf3414bc2016-12-20 15:25:57 -0800935 ulong max_mmu_page_hash_collisions;
Zhang Xiantao07114562007-12-14 10:23:23 +0800936};
937
Zhang Xiantao77b4c252007-12-14 09:49:26 +0800938struct kvm_vcpu_stat {
Suraj Jitindar Singh8a7e75d2016-08-02 14:03:22 +1000939 u64 pf_fixed;
940 u64 pf_guest;
941 u64 tlb_flush;
942 u64 invlpg;
Zhang Xiantao77b4c252007-12-14 09:49:26 +0800943
Suraj Jitindar Singh8a7e75d2016-08-02 14:03:22 +1000944 u64 exits;
945 u64 io_exits;
946 u64 mmio_exits;
947 u64 signal_exits;
948 u64 irq_window_exits;
949 u64 nmi_window_exits;
Paolo Bonzinic595cee2018-07-02 13:07:14 +0200950 u64 l1d_flush;
Suraj Jitindar Singh8a7e75d2016-08-02 14:03:22 +1000951 u64 halt_exits;
952 u64 halt_successful_poll;
953 u64 halt_attempted_poll;
954 u64 halt_poll_invalid;
955 u64 halt_wakeup;
956 u64 request_irq_exits;
957 u64 irq_exits;
958 u64 host_state_reload;
Suraj Jitindar Singh8a7e75d2016-08-02 14:03:22 +1000959 u64 fpu_reload;
960 u64 insn_emulation;
961 u64 insn_emulation_fail;
962 u64 hypercalls;
963 u64 irq_injections;
964 u64 nmi_injections;
Paolo Bonzini0f1e2612016-12-17 16:05:19 +0100965 u64 req_event;
Zhang Xiantao77b4c252007-12-14 09:49:26 +0800966};
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800967
Joerg Roedel8a76d7f2011-04-04 12:39:27 +0200968struct x86_instruction_info;
969
Will Auld8fe8ab42012-11-29 12:42:12 -0800970struct msr_data {
971 bool host_initiated;
972 u32 index;
973 u64 data;
974};
975
Paolo Bonzinicb5281a2014-12-17 18:17:20 +0100976struct kvm_lapic_irq {
977 u32 vector;
Paolo Bonzinib7cb2232015-04-21 14:57:05 +0200978 u16 delivery_mode;
979 u16 dest_mode;
980 bool level;
981 u16 trig_mode;
Paolo Bonzinicb5281a2014-12-17 18:17:20 +0100982 u32 shorthand;
983 u32 dest_id;
James Sullivan93bbf0b2015-03-18 19:26:03 -0600984 bool msi_redir_hint;
Paolo Bonzinicb5281a2014-12-17 18:17:20 +0100985};
986
Zhang Xiantaoea4a5ff2007-11-19 14:40:47 +0800987struct kvm_x86_ops {
988 int (*cpu_has_kvm_support)(void); /* __init */
989 int (*disabled_by_bios)(void); /* __init */
Radim Krčmář13a34e02014-08-28 15:13:03 +0200990 int (*hardware_enable)(void);
991 void (*hardware_disable)(void);
Zhang Xiantaoea4a5ff2007-11-19 14:40:47 +0800992 void (*check_processor_compatibility)(void *rtn);
993 int (*hardware_setup)(void); /* __init */
994 void (*hardware_unsetup)(void); /* __exit */
Avi Kivity774ead32007-12-26 13:57:04 +0200995 bool (*cpu_has_accelerated_tpr)(void);
Tom Lendackybc226f02018-05-10 22:06:39 +0200996 bool (*has_emulated_msr)(int index);
Sheng Yang0e851882009-12-18 16:48:46 +0800997 void (*cpuid_update)(struct kvm_vcpu *vcpu);
Zhang Xiantaoea4a5ff2007-11-19 14:40:47 +0800998
Sean Christopherson434a1e92018-03-20 12:17:18 -0700999 struct kvm *(*vm_alloc)(void);
1000 void (*vm_free)(struct kvm *);
Suravee Suthikulpanit03543132016-05-04 14:09:42 -05001001 int (*vm_init)(struct kvm *kvm);
1002 void (*vm_destroy)(struct kvm *kvm);
1003
Zhang Xiantaoea4a5ff2007-11-19 14:40:47 +08001004 /* Create, but do not attach this VCPU */
1005 struct kvm_vcpu *(*vcpu_create)(struct kvm *kvm, unsigned id);
1006 void (*vcpu_free)(struct kvm_vcpu *vcpu);
Nadav Amitd28bc9d2015-04-13 14:34:08 +03001007 void (*vcpu_reset)(struct kvm_vcpu *vcpu, bool init_event);
Zhang Xiantaoea4a5ff2007-11-19 14:40:47 +08001008
1009 void (*prepare_guest_switch)(struct kvm_vcpu *vcpu);
1010 void (*vcpu_load)(struct kvm_vcpu *vcpu, int cpu);
1011 void (*vcpu_put)(struct kvm_vcpu *vcpu);
Zhang Xiantaoea4a5ff2007-11-19 14:40:47 +08001012
Paolo Bonzinia96036b2015-11-10 11:55:36 +01001013 void (*update_bp_intercept)(struct kvm_vcpu *vcpu);
Paolo Bonzini609e36d2015-04-08 15:30:38 +02001014 int (*get_msr)(struct kvm_vcpu *vcpu, struct msr_data *msr);
Will Auld8fe8ab42012-11-29 12:42:12 -08001015 int (*set_msr)(struct kvm_vcpu *vcpu, struct msr_data *msr);
Zhang Xiantaoea4a5ff2007-11-19 14:40:47 +08001016 u64 (*get_segment_base)(struct kvm_vcpu *vcpu, int seg);
1017 void (*get_segment)(struct kvm_vcpu *vcpu,
1018 struct kvm_segment *var, int seg);
Izik Eidus2e4d2652008-03-24 19:38:34 +02001019 int (*get_cpl)(struct kvm_vcpu *vcpu);
Zhang Xiantaoea4a5ff2007-11-19 14:40:47 +08001020 void (*set_segment)(struct kvm_vcpu *vcpu,
1021 struct kvm_segment *var, int seg);
1022 void (*get_cs_db_l_bits)(struct kvm_vcpu *vcpu, int *db, int *l);
Avi Kivitye8467fd2009-12-29 18:43:06 +02001023 void (*decache_cr0_guest_bits)(struct kvm_vcpu *vcpu);
Avi Kivityaff48ba2010-12-05 18:56:11 +02001024 void (*decache_cr3)(struct kvm_vcpu *vcpu);
Zhang Xiantaoea4a5ff2007-11-19 14:40:47 +08001025 void (*decache_cr4_guest_bits)(struct kvm_vcpu *vcpu);
1026 void (*set_cr0)(struct kvm_vcpu *vcpu, unsigned long cr0);
1027 void (*set_cr3)(struct kvm_vcpu *vcpu, unsigned long cr3);
Nadav Har'El5e1746d2011-05-25 23:03:24 +03001028 int (*set_cr4)(struct kvm_vcpu *vcpu, unsigned long cr4);
Zhang Xiantaoea4a5ff2007-11-19 14:40:47 +08001029 void (*set_efer)(struct kvm_vcpu *vcpu, u64 efer);
Gleb Natapov89a27f42010-02-16 10:51:48 +02001030 void (*get_idt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt);
1031 void (*set_idt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt);
1032 void (*get_gdt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt);
1033 void (*set_gdt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt);
Jan Kiszka73aaf249e2014-01-04 18:47:16 +01001034 u64 (*get_dr6)(struct kvm_vcpu *vcpu);
1035 void (*set_dr6)(struct kvm_vcpu *vcpu, unsigned long value);
Paolo Bonzinic77fb5f2014-02-21 10:17:24 +01001036 void (*sync_dirty_debug_regs)(struct kvm_vcpu *vcpu);
Gleb Natapov020df072010-04-13 10:05:23 +03001037 void (*set_dr7)(struct kvm_vcpu *vcpu, unsigned long value);
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -03001038 void (*cache_reg)(struct kvm_vcpu *vcpu, enum kvm_reg reg);
Zhang Xiantaoea4a5ff2007-11-19 14:40:47 +08001039 unsigned long (*get_rflags)(struct kvm_vcpu *vcpu);
1040 void (*set_rflags)(struct kvm_vcpu *vcpu, unsigned long rflags);
1041
Wanpeng Lic2ba05c2017-12-12 17:33:03 -08001042 void (*tlb_flush)(struct kvm_vcpu *vcpu, bool invalidate_gpa);
Tianyu Lanb08660e2018-07-19 08:40:17 +00001043 int (*tlb_remote_flush)(struct kvm *kvm);
Zhang Xiantaoea4a5ff2007-11-19 14:40:47 +08001044
Junaid Shahidfaff8752018-06-29 13:10:05 -07001045 /*
1046 * Flush any TLB entries associated with the given GVA.
1047 * Does not need to flush GPA->HPA mappings.
1048 * Can potentially get non-canonical addresses through INVLPGs, which
1049 * the implementation may choose to ignore if appropriate.
1050 */
1051 void (*tlb_flush_gva)(struct kvm_vcpu *vcpu, gva_t addr);
Zhang Xiantaoea4a5ff2007-11-19 14:40:47 +08001052
Avi Kivity851ba692009-08-24 11:10:17 +03001053 void (*run)(struct kvm_vcpu *vcpu);
1054 int (*handle_exit)(struct kvm_vcpu *vcpu);
Zhang Xiantaoea4a5ff2007-11-19 14:40:47 +08001055 void (*skip_emulated_instruction)(struct kvm_vcpu *vcpu);
Glauber Costa2809f5d2009-05-12 16:21:05 -04001056 void (*set_interrupt_shadow)(struct kvm_vcpu *vcpu, int mask);
Paolo Bonzini37ccdcb2014-05-20 14:29:47 +02001057 u32 (*get_interrupt_shadow)(struct kvm_vcpu *vcpu);
Zhang Xiantaoea4a5ff2007-11-19 14:40:47 +08001058 void (*patch_hypercall)(struct kvm_vcpu *vcpu,
1059 unsigned char *hypercall_addr);
Gleb Natapov66fd3f72009-05-11 13:35:50 +03001060 void (*set_irq)(struct kvm_vcpu *vcpu);
Gleb Natapov95ba8273132009-04-21 17:45:08 +03001061 void (*set_nmi)(struct kvm_vcpu *vcpu);
Wanpeng Licfcd20e2017-07-13 18:30:39 -07001062 void (*queue_exception)(struct kvm_vcpu *vcpu);
Avi Kivityb463a6f2010-07-20 15:06:17 +03001063 void (*cancel_injection)(struct kvm_vcpu *vcpu);
Gleb Natapov78646122009-03-23 12:12:11 +02001064 int (*interrupt_allowed)(struct kvm_vcpu *vcpu);
Gleb Natapov95ba8273132009-04-21 17:45:08 +03001065 int (*nmi_allowed)(struct kvm_vcpu *vcpu);
Jan Kiszka3cfc3092009-11-12 01:04:25 +01001066 bool (*get_nmi_mask)(struct kvm_vcpu *vcpu);
1067 void (*set_nmi_mask)(struct kvm_vcpu *vcpu, bool masked);
Jan Kiszkac9a79532014-03-07 20:03:15 +01001068 void (*enable_nmi_window)(struct kvm_vcpu *vcpu);
1069 void (*enable_irq_window)(struct kvm_vcpu *vcpu);
Gleb Natapov95ba8273132009-04-21 17:45:08 +03001070 void (*update_cr8_intercept)(struct kvm_vcpu *vcpu, int tpr, int irr);
Suravee Suthikulpanitb2a05fe2017-09-12 10:42:41 -05001071 bool (*get_enable_apicv)(struct kvm_vcpu *vcpu);
Andrey Smetanind62caab2015-11-10 15:36:33 +03001072 void (*refresh_apicv_exec_ctrl)(struct kvm_vcpu *vcpu);
Yang Zhangc7c9c562013-01-25 10:18:51 +08001073 void (*hwapic_irr_update)(struct kvm_vcpu *vcpu, int max_irr);
Paolo Bonzini67c9ddd2016-05-10 17:01:23 +02001074 void (*hwapic_isr_update)(struct kvm_vcpu *vcpu, int isr);
Liran Alone6c67d82018-09-04 10:56:52 +03001075 bool (*guest_apic_has_interrupt)(struct kvm_vcpu *vcpu);
Andrey Smetanin63086302015-11-10 15:36:32 +03001076 void (*load_eoi_exitmap)(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap);
Jim Mattson8d860bb2018-05-09 16:56:05 -04001077 void (*set_virtual_apic_mode)(struct kvm_vcpu *vcpu);
Tang Chen4256f432014-09-24 15:57:54 +08001078 void (*set_apic_access_page_addr)(struct kvm_vcpu *vcpu, hpa_t hpa);
Yang Zhanga20ed542013-04-11 19:25:15 +08001079 void (*deliver_posted_interrupt)(struct kvm_vcpu *vcpu, int vector);
Paolo Bonzini76dfafd52016-12-19 17:17:11 +01001080 int (*sync_pir_to_irr)(struct kvm_vcpu *vcpu);
Zhang Xiantaoea4a5ff2007-11-19 14:40:47 +08001081 int (*set_tss_addr)(struct kvm *kvm, unsigned int addr);
Sean Christopherson2ac52ab2018-03-20 12:17:19 -07001082 int (*set_identity_map_addr)(struct kvm *kvm, u64 ident_addr);
Yu Zhang855feb62017-08-24 20:27:55 +08001083 int (*get_tdp_level)(struct kvm_vcpu *vcpu);
Sheng Yang4b12f0d2009-04-27 20:35:42 +08001084 u64 (*get_mt_mask)(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio);
Sheng Yang17cc3932010-01-05 19:02:27 +08001085 int (*get_lpage_level)(void);
Sheng Yang4e47c7a2009-12-18 16:48:47 +08001086 bool (*rdtscp_supported)(void);
Mao, Junjiead756a12012-07-02 01:18:48 +00001087 bool (*invpcid_supported)(void);
Joerg Roedel344f4142009-07-27 16:30:48 +02001088
Joerg Roedel1c97f0a2010-09-10 17:30:41 +02001089 void (*set_tdp_cr3)(struct kvm_vcpu *vcpu, unsigned long cr3);
1090
Joerg Roedeld4330ef2010-04-22 12:33:11 +02001091 void (*set_supported_cpuid)(u32 func, struct kvm_cpuid_entry2 *entry);
1092
Sheng Yangf5f48ee2010-06-30 12:25:15 +08001093 bool (*has_wbinvd_exit)(void);
1094
KarimAllah Ahmede79f2452018-04-14 05:10:52 +02001095 u64 (*read_l1_tsc_offset)(struct kvm_vcpu *vcpu);
Zachary Amsden99e3e302010-08-19 22:07:17 -10001096 void (*write_tsc_offset)(struct kvm_vcpu *vcpu, u64 offset);
1097
Avi Kivity586f9602010-11-18 13:09:54 +02001098 void (*get_exit_info)(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2);
Joerg Roedel8a76d7f2011-04-04 12:39:27 +02001099
1100 int (*check_intercept)(struct kvm_vcpu *vcpu,
1101 struct x86_instruction_info *info,
1102 enum x86_intercept_stage stage);
Yang Zhanga547c6d2013-04-11 19:25:10 +08001103 void (*handle_external_intr)(struct kvm_vcpu *vcpu);
Liu, Jinsongda8999d2014-02-24 10:55:46 +00001104 bool (*mpx_supported)(void);
Wanpeng Li55412b22014-12-02 19:21:30 +08001105 bool (*xsaves_supported)(void);
Paolo Bonzini66336ca2016-07-12 10:36:41 +02001106 bool (*umip_emulated)(void);
Jan Kiszkab6b8a142014-03-07 20:03:12 +01001107
1108 int (*check_nested_events)(struct kvm_vcpu *vcpu, bool external_intr);
Sean Christophersond264ee02018-08-27 15:21:12 -07001109 void (*request_immediate_exit)(struct kvm_vcpu *vcpu);
Radim Krčmářae97a3b2014-08-21 18:08:06 +02001110
1111 void (*sched_in)(struct kvm_vcpu *kvm, int cpu);
Kai Huang88178fd2015-01-28 10:54:27 +08001112
1113 /*
1114 * Arch-specific dirty logging hooks. These hooks are only supposed to
1115 * be valid if the specific arch has hardware-accelerated dirty logging
1116 * mechanism. Currently only for PML on VMX.
1117 *
1118 * - slot_enable_log_dirty:
1119 * called when enabling log dirty mode for the slot.
1120 * - slot_disable_log_dirty:
1121 * called when disabling log dirty mode for the slot.
1122 * also called when slot is created with log dirty disabled.
1123 * - flush_log_dirty:
1124 * called before reporting dirty_bitmap to userspace.
1125 * - enable_log_dirty_pt_masked:
1126 * called when reenabling log dirty for the GFNs in the mask after
1127 * corresponding bits are cleared in slot->dirty_bitmap.
1128 */
1129 void (*slot_enable_log_dirty)(struct kvm *kvm,
1130 struct kvm_memory_slot *slot);
1131 void (*slot_disable_log_dirty)(struct kvm *kvm,
1132 struct kvm_memory_slot *slot);
1133 void (*flush_log_dirty)(struct kvm *kvm);
1134 void (*enable_log_dirty_pt_masked)(struct kvm *kvm,
1135 struct kvm_memory_slot *slot,
1136 gfn_t offset, unsigned long mask);
Bandan Dasbab41652017-05-05 15:25:13 -04001137 int (*write_log_dirty)(struct kvm_vcpu *vcpu);
1138
Wei Huang25462f72015-06-19 15:45:05 +02001139 /* pmu operations of sub-arch */
1140 const struct kvm_pmu_ops *pmu_ops;
Feng Wuefc64402015-09-18 22:29:51 +08001141
Feng Wubf9f6ac2015-09-18 22:29:55 +08001142 /*
1143 * Architecture specific hooks for vCPU blocking due to
1144 * HLT instruction.
1145 * Returns for .pre_block():
1146 * - 0 means continue to block the vCPU.
1147 * - 1 means we cannot block the vCPU since some event
1148 * happens during this period, such as, 'ON' bit in
1149 * posted-interrupts descriptor is set.
1150 */
1151 int (*pre_block)(struct kvm_vcpu *vcpu);
1152 void (*post_block)(struct kvm_vcpu *vcpu);
Suravee Suthikulpanitd1ed0922016-05-04 14:09:43 -05001153
1154 void (*vcpu_blocking)(struct kvm_vcpu *vcpu);
1155 void (*vcpu_unblocking)(struct kvm_vcpu *vcpu);
1156
Feng Wuefc64402015-09-18 22:29:51 +08001157 int (*update_pi_irte)(struct kvm *kvm, unsigned int host_irq,
1158 uint32_t guest_irq, bool set);
Suravee Suthikulpanitbe8ca172016-05-04 14:09:49 -05001159 void (*apicv_post_state_restore)(struct kvm_vcpu *vcpu);
Yunhong Jiangce7a0582016-06-13 14:20:01 -07001160
1161 int (*set_hv_timer)(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc);
1162 void (*cancel_hv_timer)(struct kvm_vcpu *vcpu);
Ashok Rajc45dcc72016-06-22 14:59:56 +08001163
1164 void (*setup_mce)(struct kvm_vcpu *vcpu);
Ladi Prosek0234bf82017-10-11 16:54:40 +02001165
Jim Mattson8fcc4b52018-07-10 11:27:20 +02001166 int (*get_nested_state)(struct kvm_vcpu *vcpu,
1167 struct kvm_nested_state __user *user_kvm_nested_state,
1168 unsigned user_data_size);
1169 int (*set_nested_state)(struct kvm_vcpu *vcpu,
1170 struct kvm_nested_state __user *user_kvm_nested_state,
1171 struct kvm_nested_state *kvm_state);
Paolo Bonzini7f7f1ba2018-07-18 18:49:01 +02001172 void (*get_vmcs12_pages)(struct kvm_vcpu *vcpu);
1173
Ladi Prosek72d7b372017-10-11 16:54:41 +02001174 int (*smi_allowed)(struct kvm_vcpu *vcpu);
Ladi Prosek0234bf82017-10-11 16:54:40 +02001175 int (*pre_enter_smm)(struct kvm_vcpu *vcpu, char *smstate);
1176 int (*pre_leave_smm)(struct kvm_vcpu *vcpu, u64 smbase);
Ladi Prosekcc3d9672017-10-17 16:02:39 +02001177 int (*enable_smi_window)(struct kvm_vcpu *vcpu);
Brijesh Singh5acc5c02017-12-04 10:57:26 -06001178
1179 int (*mem_enc_op)(struct kvm *kvm, void __user *argp);
Brijesh Singh69eaede2017-12-04 10:57:26 -06001180 int (*mem_enc_reg_region)(struct kvm *kvm, struct kvm_enc_region *argp);
1181 int (*mem_enc_unreg_region)(struct kvm *kvm, struct kvm_enc_region *argp);
Tom Lendacky801e4592018-02-21 13:39:51 -06001182
1183 int (*get_msr_feature)(struct kvm_msr_entry *entry);
Vitaly Kuznetsov57b119d2018-10-16 18:50:01 +02001184
1185 int (*nested_enable_evmcs)(struct kvm_vcpu *vcpu,
1186 uint16_t *vmcs_version);
Zhang Xiantaoea4a5ff2007-11-19 14:40:47 +08001187};
1188
Gleb Natapovaf585b92010-10-14 11:22:46 +02001189struct kvm_arch_async_pf {
Gleb Natapov7c907052010-10-14 11:22:53 +02001190 u32 token;
Gleb Natapovaf585b92010-10-14 11:22:46 +02001191 gfn_t gfn;
Xiao Guangrongfb67e142010-12-07 10:35:25 +08001192 unsigned long cr3;
Xiao Guangrongc4806ac2010-11-12 14:49:55 +08001193 bool direct_map;
Gleb Natapovaf585b92010-10-14 11:22:46 +02001194};
1195
Zhang Xiantao97896d02007-11-14 20:09:30 +08001196extern struct kvm_x86_ops *kvm_x86_ops;
1197
Sean Christopherson434a1e92018-03-20 12:17:18 -07001198#define __KVM_HAVE_ARCH_VM_ALLOC
1199static inline struct kvm *kvm_arch_alloc_vm(void)
1200{
1201 return kvm_x86_ops->vm_alloc();
1202}
1203
1204static inline void kvm_arch_free_vm(struct kvm *kvm)
1205{
1206 return kvm_x86_ops->vm_free(kvm);
1207}
1208
Tianyu Lanb08660e2018-07-19 08:40:17 +00001209#define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLB
1210static inline int kvm_arch_flush_remote_tlb(struct kvm *kvm)
1211{
1212 if (kvm_x86_ops->tlb_remote_flush &&
1213 !kvm_x86_ops->tlb_remote_flush(kvm))
1214 return 0;
1215 else
1216 return -ENOTSUPP;
1217}
1218
Zhang Xiantao54f15852007-11-19 15:24:28 +08001219int kvm_mmu_module_init(void);
1220void kvm_mmu_module_exit(void);
1221
1222void kvm_mmu_destroy(struct kvm_vcpu *vcpu);
1223int kvm_mmu_create(struct kvm_vcpu *vcpu);
Xiao Guangrong13d268c2016-02-24 17:51:16 +08001224void kvm_mmu_init_vm(struct kvm *kvm);
1225void kvm_mmu_uninit_vm(struct kvm *kvm);
Sheng Yang7b523452008-04-25 21:13:50 +08001226void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask,
Junaid Shahidf160c7b2016-12-06 16:46:16 -08001227 u64 dirty_mask, u64 nx_mask, u64 x_mask, u64 p_mask,
Tom Lendackyd0ec49d2017-07-17 16:10:27 -05001228 u64 acc_track_mask, u64 me_mask);
Zhang Xiantao54f15852007-11-19 15:24:28 +08001229
Paolo Bonzini8a3c1a332013-10-02 16:56:13 +02001230void kvm_mmu_reset_context(struct kvm_vcpu *vcpu);
Kai Huang1c91cad42015-01-28 10:54:26 +08001231void kvm_mmu_slot_remove_write_access(struct kvm *kvm,
1232 struct kvm_memory_slot *memslot);
Wanpeng Li3ea3b7f2015-04-03 15:40:25 +08001233void kvm_mmu_zap_collapsible_sptes(struct kvm *kvm,
Paolo Bonzinif36f3f22015-05-18 13:20:23 +02001234 const struct kvm_memory_slot *memslot);
Kai Huangf4b4b182015-01-28 10:54:24 +08001235void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm,
1236 struct kvm_memory_slot *memslot);
1237void kvm_mmu_slot_largepage_remove_write_access(struct kvm *kvm,
1238 struct kvm_memory_slot *memslot);
1239void kvm_mmu_slot_set_dirty(struct kvm *kvm,
1240 struct kvm_memory_slot *memslot);
1241void kvm_mmu_clear_dirty_pt_masked(struct kvm *kvm,
1242 struct kvm_memory_slot *slot,
1243 gfn_t gfn_offset, unsigned long mask);
Zhang Xiantao54f15852007-11-19 15:24:28 +08001244void kvm_mmu_zap_all(struct kvm *kvm);
Paolo Bonzini54bf36a2015-04-08 15:39:23 +02001245void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, struct kvm_memslots *slots);
Zhang Xiantao3ad82a72007-11-20 13:11:38 +08001246unsigned int kvm_mmu_calculate_mmu_pages(struct kvm *kvm);
Zhang Xiantao54f15852007-11-19 15:24:28 +08001247void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int kvm_nr_mmu_pages);
1248
Joerg Roedelff03a072010-09-10 17:30:57 +02001249int load_pdptrs(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, unsigned long cr3);
Ladi Prosek9ed38ffa2016-11-30 16:03:10 +01001250bool pdptrs_changed(struct kvm_vcpu *vcpu);
Joerg Roedelcc4b6872008-02-07 13:47:43 +01001251
Marcelo Tosatti3200f402008-03-29 20:17:59 -03001252int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
Avi Kivity9f811282008-03-02 14:06:05 +02001253 const void *val, int bytes);
Marcelo Tosatti2f333bc2008-02-22 12:21:37 -05001254
Paolo Bonzini6ef768f2014-11-20 13:45:31 +01001255struct kvm_irq_mask_notifier {
1256 void (*func)(struct kvm_irq_mask_notifier *kimn, bool masked);
1257 int irq;
1258 struct hlist_node link;
1259};
1260
1261void kvm_register_irq_mask_notifier(struct kvm *kvm, int irq,
1262 struct kvm_irq_mask_notifier *kimn);
1263void kvm_unregister_irq_mask_notifier(struct kvm *kvm, int irq,
1264 struct kvm_irq_mask_notifier *kimn);
1265void kvm_fire_mask_notifiers(struct kvm *kvm, unsigned irqchip, unsigned pin,
1266 bool mask);
1267
Marcelo Tosatti2f333bc2008-02-22 12:21:37 -05001268extern bool tdp_enabled;
Avi Kivity9f811282008-03-02 14:06:05 +02001269
Liu, Jinsonga3e06bb2011-09-22 16:55:52 +08001270u64 vcpu_tsc_khz(struct kvm_vcpu *vcpu);
1271
Joerg Roedel92a1f122011-03-25 09:44:51 +01001272/* control of guest tsc rate supported? */
1273extern bool kvm_has_tsc_control;
Joerg Roedel92a1f122011-03-25 09:44:51 +01001274/* maximum supported tsc_khz for guests */
1275extern u32 kvm_max_guest_tsc_khz;
Haozhong Zhangbc9b9612015-10-20 15:39:01 +08001276/* number of bits of the fractional part of the TSC scaling ratio */
1277extern u8 kvm_tsc_scaling_ratio_frac_bits;
1278/* maximum allowed value of TSC scaling ratio */
1279extern u64 kvm_max_tsc_scaling_ratio;
Yunhong Jiang64672c92016-06-13 14:19:59 -07001280/* 1ull << kvm_tsc_scaling_ratio_frac_bits */
1281extern u64 kvm_default_tsc_scaling_ratio;
Joerg Roedel92a1f122011-03-25 09:44:51 +01001282
Ashok Rajc45dcc72016-06-22 14:59:56 +08001283extern u64 kvm_mce_cap_supported;
Sheng Yang571008d2008-01-02 14:49:22 +08001284
1285enum emulation_result {
Paolo Bonziniac0a48c2013-06-25 18:24:41 +02001286 EMULATE_DONE, /* no further processing */
1287 EMULATE_USER_EXIT, /* kvm_run ready for userspace exit */
Zhang Xiantao54f15852007-11-19 15:24:28 +08001288 EMULATE_FAIL, /* can't emulate this instruction */
1289};
1290
1291#define EMULTYPE_NO_DECODE (1 << 0)
1292#define EMULTYPE_TRAP_UD (1 << 1)
Gleb Natapovba8afb62009-04-12 13:36:57 +03001293#define EMULTYPE_SKIP (1 << 2)
Sean Christopherson384bf222018-08-23 13:56:49 -07001294#define EMULTYPE_ALLOW_RETRY (1 << 3)
1295#define EMULTYPE_NO_UD_ON_FAIL (1 << 4)
1296#define EMULTYPE_VMWARE (1 << 5)
Sean Christophersonc60658d2018-08-23 13:56:53 -07001297int kvm_emulate_instruction(struct kvm_vcpu *vcpu, int emulation_type);
1298int kvm_emulate_instruction_from_buffer(struct kvm_vcpu *vcpu,
1299 void *insn, int insn_len);
Sean Christopherson35be0ad2018-08-23 13:56:47 -07001300
Joerg Roedelf2b4b7d2008-01-31 14:57:37 +01001301void kvm_enable_efer_bits(u64);
Jan Kiszka384bb782013-04-20 10:52:36 +02001302bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer);
Paolo Bonzini609e36d2015-04-08 15:30:38 +02001303int kvm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr);
Will Auld8fe8ab42012-11-29 12:42:12 -08001304int kvm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr);
Zhang Xiantao54f15852007-11-19 15:24:28 +08001305
1306struct x86_emulate_ctxt;
1307
Sean Christophersondca7f122018-03-08 08:57:27 -08001308int kvm_fast_pio(struct kvm_vcpu *vcpu, int size, unsigned short port, int in);
Kyle Huey6a908b62016-11-29 12:40:37 -08001309int kvm_emulate_cpuid(struct kvm_vcpu *vcpu);
Zhang Xiantao54f15852007-11-19 15:24:28 +08001310int kvm_emulate_halt(struct kvm_vcpu *vcpu);
Joel Schopp5cb56052015-03-02 13:43:31 -06001311int kvm_vcpu_halt(struct kvm_vcpu *vcpu);
Sheng Yangf5f48ee2010-06-30 12:25:15 +08001312int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu);
Zhang Xiantao54f15852007-11-19 15:24:28 +08001313
Guillaume Thouvenin3e6e0aa2008-05-27 10:18:46 +02001314void kvm_get_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg);
Gleb Natapovc6975182010-02-18 12:15:01 +02001315int kvm_load_segment_descriptor(struct kvm_vcpu *vcpu, u16 selector, int seg);
Paolo Bonzini2b4a2732014-11-24 14:35:24 +01001316void kvm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector);
Guillaume Thouvenin3e6e0aa2008-05-27 10:18:46 +02001317
Kevin Wolf7f3d35f2012-02-08 14:34:38 +01001318int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index,
1319 int reason, bool has_error_code, u32 error_code);
Izik Eidus37817f22008-03-24 23:14:53 +02001320
Avi Kivity49a9b072010-06-10 17:02:14 +03001321int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0);
Avi Kivity23902182010-06-10 17:02:16 +03001322int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3);
Avi Kivitya83b29c2010-06-10 17:02:15 +03001323int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4);
Andre Przywaraeea1cff2010-12-21 11:12:00 +01001324int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8);
Gleb Natapov020df072010-04-13 10:05:23 +03001325int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val);
1326int kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val);
Avi Kivity2d3ad1f2008-02-24 11:20:43 +02001327unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu);
1328void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw);
Zhang Xiantao54f15852007-11-19 15:24:28 +08001329void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l);
Dexuan Cui2acf9232010-06-10 11:27:12 +08001330int kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr);
Zhang Xiantao54f15852007-11-19 15:24:28 +08001331
Paolo Bonzini609e36d2015-04-08 15:30:38 +02001332int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr);
Will Auld8fe8ab42012-11-29 12:42:12 -08001333int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr);
Zhang Xiantao54f15852007-11-19 15:24:28 +08001334
Jan Kiszka91586a32009-10-05 13:07:21 +02001335unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu);
1336void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags);
Avi Kivity022cd0e2011-11-10 14:57:23 +02001337bool kvm_rdpmc(struct kvm_vcpu *vcpu);
Jan Kiszka91586a32009-10-05 13:07:21 +02001338
Avi Kivity298101d2007-11-25 13:41:11 +02001339void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr);
1340void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code);
Joerg Roedelce7ddec2010-04-22 12:33:13 +02001341void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr);
1342void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code);
Avi Kivity6389ee92010-11-29 16:12:30 +02001343void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault);
Joerg Roedelec92fe42010-09-10 17:30:51 +02001344int kvm_read_guest_page_mmu(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
1345 gfn_t gfn, void *data, int offset, int len,
1346 u32 access);
Avi Kivity0a79b002009-09-01 12:03:25 +03001347bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl);
Nadav Amit16f8a6f2014-10-03 01:10:05 +03001348bool kvm_require_dr(struct kvm_vcpu *vcpu, int dr);
Avi Kivity298101d2007-11-25 13:41:11 +02001349
Michael S. Tsirkin1a577b722012-07-19 13:45:20 +03001350static inline int __kvm_irq_line_state(unsigned long *irq_state,
1351 int irq_source_id, int level)
1352{
1353 /* Logical OR for level trig interrupt */
1354 if (level)
1355 __set_bit(irq_source_id, irq_state);
1356 else
1357 __clear_bit(irq_source_id, irq_state);
1358
1359 return !!(*irq_state);
1360}
1361
Junaid Shahidb94742c2018-06-27 14:59:20 -07001362#define KVM_MMU_ROOT_CURRENT BIT(0)
1363#define KVM_MMU_ROOT_PREVIOUS(i) BIT(1+i)
1364#define KVM_MMU_ROOTS_ALL (~0UL)
Junaid Shahid08fb59d2018-06-27 14:59:17 -07001365
Michael S. Tsirkin1a577b722012-07-19 13:45:20 +03001366int kvm_pic_set_irq(struct kvm_pic *pic, int irq, int irq_source_id, int level);
1367void kvm_pic_clear_all(struct kvm_pic *pic, int irq_source_id);
Xiantao Zhang3de42dc2008-10-06 13:48:45 +08001368
Sheng Yang3419ffc2008-05-15 09:52:48 +08001369void kvm_inject_nmi(struct kvm_vcpu *vcpu);
1370
Xiao Guangrong1cb3f3a2011-09-22 17:02:48 +08001371int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn);
Zhang Xiantao54f15852007-11-19 15:24:28 +08001372int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva);
1373void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu);
1374int kvm_mmu_load(struct kvm_vcpu *vcpu);
1375void kvm_mmu_unload(struct kvm_vcpu *vcpu);
Marcelo Tosatti0ba73cd2008-09-23 13:18:34 -03001376void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu);
Vitaly Kuznetsov6a82cd12018-10-08 21:28:07 +02001377void kvm_mmu_free_roots(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
1378 ulong roots_to_free);
Paolo Bonzini54987b72014-09-02 13:23:06 +02001379gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access,
1380 struct x86_exception *exception);
Avi Kivityab9ae312010-11-22 17:53:26 +02001381gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva,
1382 struct x86_exception *exception);
1383gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva,
1384 struct x86_exception *exception);
1385gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva,
1386 struct x86_exception *exception);
1387gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva,
1388 struct x86_exception *exception);
Zhang Xiantao54f15852007-11-19 15:24:28 +08001389
Andrey Smetanind62caab2015-11-10 15:36:33 +03001390void kvm_vcpu_deactivate_apicv(struct kvm_vcpu *vcpu);
1391
Zhang Xiantao54f15852007-11-19 15:24:28 +08001392int kvm_emulate_hypercall(struct kvm_vcpu *vcpu);
1393
Tom Lendacky14727752016-11-23 12:01:38 -05001394int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t gva, u64 error_code,
Andre Przywaradc25e892010-12-21 11:12:07 +01001395 void *insn, int insn_len);
Marcelo Tosattia7052892008-09-23 13:18:35 -03001396void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva);
Junaid Shahideb4b2482018-06-27 14:59:14 -07001397void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid);
Junaid Shahidade61e22018-06-27 14:59:15 -07001398void kvm_mmu_new_cr3(struct kvm_vcpu *vcpu, gpa_t new_cr3, bool skip_tlb_flush);
Zhang Xiantao34c16ee2007-10-20 15:34:38 +08001399
Joerg Roedel18552672008-02-07 13:47:41 +01001400void kvm_enable_tdp(void);
Joerg Roedel5f4cb662008-07-14 20:36:36 +02001401void kvm_disable_tdp(void);
Joerg Roedel18552672008-02-07 13:47:41 +01001402
Paolo Bonzini54987b72014-09-02 13:23:06 +02001403static inline gpa_t translate_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access,
1404 struct x86_exception *exception)
Xiao Guangronge459e322011-11-28 20:42:16 +08001405{
1406 return gpa;
1407}
1408
Zhang Xiantaoec6d2732007-11-19 15:08:31 +08001409static inline struct kvm_mmu_page *page_header(hpa_t shadow_page)
1410{
1411 struct page *page = pfn_to_page(shadow_page >> PAGE_SHIFT);
1412
1413 return (struct kvm_mmu_page *)page_private(page);
1414}
1415
Avi Kivityd6e88ae2008-07-10 16:53:33 +03001416static inline u16 kvm_read_ldt(void)
Zhang Xiantaoec6d2732007-11-19 15:08:31 +08001417{
1418 u16 ldt;
1419 asm("sldt %0" : "=g"(ldt));
1420 return ldt;
1421}
1422
Avi Kivityd6e88ae2008-07-10 16:53:33 +03001423static inline void kvm_load_ldt(u16 sel)
Zhang Xiantaoec6d2732007-11-19 15:08:31 +08001424{
1425 asm("lldt %0" : : "rm"(sel));
1426}
Zhang Xiantaoec6d2732007-11-19 15:08:31 +08001427
Zhang Xiantaoec6d2732007-11-19 15:08:31 +08001428#ifdef CONFIG_X86_64
1429static inline unsigned long read_msr(unsigned long msr)
1430{
1431 u64 value;
1432
1433 rdmsrl(msr, value);
1434 return value;
1435}
1436#endif
1437
Zhang Xiantaoec6d2732007-11-19 15:08:31 +08001438static inline u32 get_rdx_init_val(void)
1439{
1440 return 0x600; /* P6 family */
1441}
1442
Avi Kivityc1a5d4f2007-11-25 14:12:03 +02001443static inline void kvm_inject_gp(struct kvm_vcpu *vcpu, u32 error_code)
1444{
1445 kvm_queue_exception_e(vcpu, GP_VECTOR, error_code);
1446}
1447
Zhang Xiantaoec6d2732007-11-19 15:08:31 +08001448#define TSS_IOPB_BASE_OFFSET 0x66
1449#define TSS_BASE_SIZE 0x68
1450#define TSS_IOPB_SIZE (65536 / 8)
1451#define TSS_REDIRECTION_SIZE (256 / 8)
Joe Perches7d76b4d2008-03-23 01:02:34 -07001452#define RMODE_TSS_SIZE \
1453 (TSS_BASE_SIZE + TSS_REDIRECTION_SIZE + TSS_IOPB_SIZE + 1)
Hollis Blanchard53e0aa72007-12-03 16:15:26 -06001454
Izik Eidus37817f22008-03-24 23:14:53 +02001455enum {
1456 TASK_SWITCH_CALL = 0,
1457 TASK_SWITCH_IRET = 1,
1458 TASK_SWITCH_JMP = 2,
1459 TASK_SWITCH_GATE = 3,
1460};
1461
Alexander Graf1371d902008-11-25 20:17:04 +01001462#define HF_GIF_MASK (1 << 0)
Alexander Graf3d6368e2008-11-25 20:17:07 +01001463#define HF_HIF_MASK (1 << 1)
1464#define HF_VINTR_MASK (1 << 2)
Gleb Natapov95ba8273132009-04-21 17:45:08 +03001465#define HF_NMI_MASK (1 << 3)
Gleb Natapov44c11432009-05-11 13:35:52 +03001466#define HF_IRET_MASK (1 << 4)
Joerg Roedelec9e60b2010-11-29 17:51:47 +01001467#define HF_GUEST_MASK (1 << 5) /* VCPU is in guest-mode */
Paolo Bonzinif0778252015-04-01 15:06:40 +02001468#define HF_SMM_MASK (1 << 6)
1469#define HF_SMM_INSIDE_NMI_MASK (1 << 7)
Alexander Graf1371d902008-11-25 20:17:04 +01001470
Paolo Bonzini699023e2015-05-18 15:03:39 +02001471#define __KVM_VCPU_MULTIPLE_ADDRESS_SPACE
1472#define KVM_ADDRESS_SPACE_NUM 2
1473
1474#define kvm_arch_vcpu_memslots_id(vcpu) ((vcpu)->arch.hflags & HF_SMM_MASK ? 1 : 0)
1475#define kvm_memslots_for_spte_role(kvm, role) __kvm_memslots(kvm, (role).smm)
Avi Kivity4ecac3f2008-05-13 13:23:38 +03001476
1477/*
1478 * Hardware virtualization extension instructions may fault if a
1479 * reboot turns off virtualization while processes are running.
1480 * Trap the fault and ignore the instruction if that happens.
1481 */
Avi Kivityb7c41452010-12-02 17:52:50 +02001482asmlinkage void kvm_spurious_fault(void);
Avi Kivity4ecac3f2008-05-13 13:23:38 +03001483
Avi Kivity5e520e62011-05-15 10:13:12 -04001484#define ____kvm_handle_fault_on_reboot(insn, cleanup_insn) \
Avi Kivity4ecac3f2008-05-13 13:23:38 +03001485 "666: " insn "\n\t" \
Avi Kivityb7c41452010-12-02 17:52:50 +02001486 "668: \n\t" \
Eduardo Habkost18b13e52008-08-19 20:00:08 -03001487 ".pushsection .fixup, \"ax\" \n" \
Avi Kivity4ecac3f2008-05-13 13:23:38 +03001488 "667: \n\t" \
Avi Kivity5e520e62011-05-15 10:13:12 -04001489 cleanup_insn "\n\t" \
Avi Kivityb7c41452010-12-02 17:52:50 +02001490 "cmpb $0, kvm_rebooting \n\t" \
1491 "jne 668b \n\t" \
Avi Kivity8ceed3472008-08-14 21:25:47 +03001492 __ASM_SIZE(push) " $666b \n\t" \
Avi Kivityb7c41452010-12-02 17:52:50 +02001493 "call kvm_spurious_fault \n\t" \
Avi Kivity4ecac3f2008-05-13 13:23:38 +03001494 ".popsection \n\t" \
H. Peter Anvin3ee89722012-04-20 13:41:59 -07001495 _ASM_EXTABLE(666b, 667b)
Avi Kivity4ecac3f2008-05-13 13:23:38 +03001496
Avi Kivity5e520e62011-05-15 10:13:12 -04001497#define __kvm_handle_fault_on_reboot(insn) \
1498 ____kvm_handle_fault_on_reboot(insn, "")
1499
Andrea Arcangelie930bff2008-07-25 16:24:52 +02001500#define KVM_ARCH_WANT_MMU_NOTIFIER
Takuya Yoshikawab3ae2092012-07-02 17:56:33 +09001501int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end);
Andres Lagar-Cavilla57128462014-09-22 14:54:42 -07001502int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end);
Andrea Arcangeli8ee53822011-01-13 15:47:10 -08001503int kvm_test_age_hva(struct kvm *kvm, unsigned long hva);
Izik Eidus3da0dd42009-09-23 21:47:18 +03001504void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte);
Yang Zhangc7c9c562013-01-25 10:18:51 +08001505int kvm_cpu_has_injectable_intr(struct kvm_vcpu *v);
Gleb Natapova1b37102009-07-09 15:33:52 +03001506int kvm_cpu_has_interrupt(struct kvm_vcpu *vcpu);
1507int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu);
Gleb Natapov0b717852009-07-09 15:33:53 +03001508int kvm_cpu_get_interrupt(struct kvm_vcpu *v);
Nadav Amitd28bc9d2015-04-13 14:34:08 +03001509void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event);
Tang Chen4256f432014-09-24 15:57:54 +08001510void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu);
Andrea Arcangelie930bff2008-07-25 16:24:52 +02001511
Wanpeng Li4180bf12018-07-23 14:39:54 +08001512int kvm_pv_send_ipi(struct kvm *kvm, unsigned long ipi_bitmap_low,
Wanpeng Libdf7ffc2018-08-30 10:03:30 +08001513 unsigned long ipi_bitmap_high, u32 min,
Wanpeng Li4180bf12018-07-23 14:39:54 +08001514 unsigned long icr, int op_64_bit);
1515
Paolo Bonzini5b76a3c2018-08-05 16:07:47 +02001516u64 kvm_get_arch_capabilities(void);
Avi Kivity18863bd2009-09-07 11:12:18 +03001517void kvm_define_shared_msr(unsigned index, u32 msr);
Andy Honig8b3c3102014-08-27 11:16:44 -07001518int kvm_set_shared_msr(unsigned index, u64 val, u64 mask);
Avi Kivity18863bd2009-09-07 11:12:18 +03001519
Haozhong Zhang35181e82015-10-20 15:39:03 +08001520u64 kvm_scale_tsc(struct kvm_vcpu *vcpu, u64 tsc);
Haozhong Zhang4ba76532015-10-20 15:39:07 +08001521u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc);
Haozhong Zhang35181e82015-10-20 15:39:03 +08001522
Nadav Amit82b32772014-11-02 11:54:45 +02001523unsigned long kvm_get_linear_rip(struct kvm_vcpu *vcpu);
Jan Kiszkaf92653e2010-02-23 17:47:55 +01001524bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip);
1525
Paolo Bonzini2860c4b2016-01-07 15:05:10 +01001526void kvm_make_mclock_inprogress_request(struct kvm *kvm);
1527void kvm_make_scan_ioapic_request(struct kvm *kvm);
1528
Gleb Natapovaf585b92010-10-14 11:22:46 +02001529void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
1530 struct kvm_async_pf *work);
1531void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
1532 struct kvm_async_pf *work);
Gleb Natapov56028d02010-10-17 18:13:42 +02001533void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
1534 struct kvm_async_pf *work);
Gleb Natapov7c907052010-10-14 11:22:53 +02001535bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu);
Gleb Natapovaf585b92010-10-14 11:22:46 +02001536extern bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn);
1537
Kyle Huey6affcbe2016-11-29 12:40:40 -08001538int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu);
1539int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err);
Sean Christophersond264ee02018-08-27 15:21:12 -07001540void __kvm_request_immediate_exit(struct kvm_vcpu *vcpu);
Andre Przywaradb8fcef2010-12-21 11:12:01 +01001541
Gleb Natapovf5132b02011-11-10 14:57:22 +02001542int kvm_is_in_guest(void);
1543
Paolo Bonzini1d8007b2015-10-12 13:38:32 +02001544int __x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa, u32 size);
1545int x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa, u32 size);
Paolo Bonzinid71ba782015-07-29 11:56:48 +02001546bool kvm_vcpu_is_reset_bsp(struct kvm_vcpu *vcpu);
1547bool kvm_vcpu_is_bsp(struct kvm_vcpu *vcpu);
Gleb Natapovf5132b02011-11-10 14:57:22 +02001548
Feng Wu8feb4a02015-09-18 22:29:47 +08001549bool kvm_intr_is_single_vcpu(struct kvm *kvm, struct kvm_lapic_irq *irq,
1550 struct kvm_vcpu **dest_vcpu);
1551
Radim Krčmář371313132016-07-12 22:09:27 +02001552void kvm_set_msi_irq(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e,
Feng Wud84f1e02015-09-18 22:29:49 +08001553 struct kvm_lapic_irq *irq);
Paolo Bonzini197a4f42015-11-04 16:24:17 +01001554
Suravee Suthikulpanitd1ed0922016-05-04 14:09:43 -05001555static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu)
1556{
1557 if (kvm_x86_ops->vcpu_blocking)
1558 kvm_x86_ops->vcpu_blocking(vcpu);
1559}
1560
1561static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu)
1562{
1563 if (kvm_x86_ops->vcpu_unblocking)
1564 kvm_x86_ops->vcpu_unblocking(vcpu);
1565}
1566
Christian Borntraeger3491caf2016-05-13 12:16:35 +02001567static inline void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu) {}
Christoffer Dall3217f7c2015-08-27 16:41:15 +02001568
Suravee Suthikulpanit7d669f52016-06-15 17:23:45 -05001569static inline int kvm_cpu_get_apicid(int mps_cpu)
1570{
1571#ifdef CONFIG_X86_LOCAL_APIC
Thomas Gleixner64063502017-09-13 23:29:21 +02001572 return default_cpu_present_to_apicid(mps_cpu);
Suravee Suthikulpanit7d669f52016-06-15 17:23:45 -05001573#else
1574 WARN_ON_ONCE(1);
1575 return BAD_APICID;
1576#endif
1577}
1578
Ladi Prosek05cade72017-10-11 16:54:45 +02001579#define put_smstate(type, buf, offset, val) \
1580 *(type *)((buf) + (offset) - 0x7e00) = val
1581
H. Peter Anvin1965aae2008-10-22 22:26:29 -07001582#endif /* _ASM_X86_KVM_HOST_H */