blob: 31f4186698275cac9acb878acecacba10bbc5fae [file] [log] [blame]
Vegard Nossuma656c8e2008-07-22 21:27:11 +02001/*
Carsten Otte043405e2007-10-10 17:16:19 +02002 * Kernel-based Virtual Machine driver for Linux
3 *
4 * This header defines architecture specific interfaces, x86 version
5 *
6 * This work is licensed under the terms of the GNU GPL, version 2. See
7 * the COPYING file in the top-level directory.
8 *
9 */
10
H. Peter Anvin1965aae2008-10-22 22:26:29 -070011#ifndef _ASM_X86_KVM_HOST_H
12#define _ASM_X86_KVM_HOST_H
Carsten Otte043405e2007-10-10 17:16:19 +020013
Zhang Xiantao34c16ee2007-10-20 15:34:38 +080014#include <linux/types.h>
15#include <linux/mm.h>
Andrea Arcangelie930bff2008-07-25 16:24:52 +020016#include <linux/mmu_notifier.h>
Marcelo Tosatti229456f2009-06-17 09:22:14 -030017#include <linux/tracepoint.h>
Sheng Yangf5f48ee2010-06-30 12:25:15 +080018#include <linux/cpumask.h>
Gleb Natapovf5132b02011-11-10 14:57:22 +020019#include <linux/irq_work.h>
Zhang Xiantao34c16ee2007-10-20 15:34:38 +080020
21#include <linux/kvm.h>
22#include <linux/kvm_para.h>
Avi Kivityedf88412007-12-16 11:02:48 +020023#include <linux/kvm_types.h>
Gleb Natapovf5132b02011-11-10 14:57:22 +020024#include <linux/perf_event.h>
Marcelo Tosattid8281992012-11-27 23:29:01 -020025#include <linux/pvclock_gtod.h>
26#include <linux/clocksource.h>
Feng Wu87276882015-09-18 22:29:40 +080027#include <linux/irqbypass.h>
Andrey Smetanin5c9194122015-11-10 15:36:34 +030028#include <linux/hyperv.h>
Zhang Xiantao34c16ee2007-10-20 15:34:38 +080029
Suravee Suthikulpanit7d669f52016-06-15 17:23:45 -050030#include <asm/apic.h>
Gerd Hoffmann50d0a0f2008-06-03 16:17:31 +020031#include <asm/pvclock-abi.h>
Hollis Blancharde01a1b52007-12-03 15:30:25 -060032#include <asm/desc.h>
Sheng Yang0bed3b52008-10-09 16:01:54 +080033#include <asm/mtrr.h>
Alexander Graf9962d032008-11-25 20:17:02 +010034#include <asm/msr-index.h>
H. Peter Anvin3ee89722012-04-20 13:41:59 -070035#include <asm/asm.h>
Xiao Guangrong21ebbed2016-02-24 17:51:09 +080036#include <asm/kvm_page_track.h>
Hollis Blancharde01a1b52007-12-03 15:30:25 -060037
Radim Krčmář682f7322016-07-12 22:09:29 +020038#define KVM_MAX_VCPUS 288
Radim Krčmář757883d2016-07-12 22:09:17 +020039#define KVM_SOFT_MAX_VCPUS 240
Radim Krčmářaf1bae52016-07-12 22:09:30 +020040#define KVM_MAX_VCPU_ID 1023
Igor Mammedov1d4e7e32014-11-06 15:52:47 +000041#define KVM_USER_MEM_SLOTS 509
Alex Williamson07432472012-12-10 10:33:15 -070042/* memory slots that are not exposed to userspace */
43#define KVM_PRIVATE_MEM_SLOTS 3
Alex Williamsonbbacc0c2012-12-10 10:33:09 -070044#define KVM_MEM_SLOTS_NUM (KVM_USER_MEM_SLOTS + KVM_PRIVATE_MEM_SLOTS)
Xiao Guangrong93a5cef2011-11-24 17:37:48 +080045
Paolo Bonzinib401ee02017-04-18 12:41:18 +020046#define KVM_HALT_POLL_NS_DEFAULT 200000
Avi Kivity69a9f692008-03-21 12:38:23 +020047
Alexander Graf8175e5b2013-04-15 10:42:33 +020048#define KVM_IRQCHIP_NUM_PINS KVM_IOAPIC_NUM_PINS
49
Paolo Bonzini2860c4b2016-01-07 15:05:10 +010050/* x86-specific vcpu->requests bit members */
Andrew Jones23871492017-06-04 14:43:51 +020051#define KVM_REQ_MIGRATE_TIMER KVM_ARCH_REQ(0)
52#define KVM_REQ_REPORT_TPR_ACCESS KVM_ARCH_REQ(1)
53#define KVM_REQ_TRIPLE_FAULT KVM_ARCH_REQ(2)
54#define KVM_REQ_MMU_SYNC KVM_ARCH_REQ(3)
55#define KVM_REQ_CLOCK_UPDATE KVM_ARCH_REQ(4)
56#define KVM_REQ_EVENT KVM_ARCH_REQ(6)
57#define KVM_REQ_APF_HALT KVM_ARCH_REQ(7)
58#define KVM_REQ_STEAL_UPDATE KVM_ARCH_REQ(8)
59#define KVM_REQ_NMI KVM_ARCH_REQ(9)
60#define KVM_REQ_PMU KVM_ARCH_REQ(10)
61#define KVM_REQ_PMI KVM_ARCH_REQ(11)
62#define KVM_REQ_SMI KVM_ARCH_REQ(12)
63#define KVM_REQ_MASTERCLOCK_UPDATE KVM_ARCH_REQ(13)
64#define KVM_REQ_MCLOCK_INPROGRESS \
65 KVM_ARCH_REQ_FLAGS(14, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
66#define KVM_REQ_SCAN_IOAPIC \
67 KVM_ARCH_REQ_FLAGS(15, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
68#define KVM_REQ_GLOBAL_CLOCK_UPDATE KVM_ARCH_REQ(16)
69#define KVM_REQ_APIC_PAGE_RELOAD \
70 KVM_ARCH_REQ_FLAGS(17, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
71#define KVM_REQ_HV_CRASH KVM_ARCH_REQ(18)
72#define KVM_REQ_IOAPIC_EOI_EXIT KVM_ARCH_REQ(19)
73#define KVM_REQ_HV_RESET KVM_ARCH_REQ(20)
74#define KVM_REQ_HV_EXIT KVM_ARCH_REQ(21)
75#define KVM_REQ_HV_STIMER KVM_ARCH_REQ(22)
Paolo Bonzini2860c4b2016-01-07 15:05:10 +010076
Joerg Roedelcfec82c2011-04-04 12:39:28 +020077#define CR0_RESERVED_BITS \
78 (~(unsigned long)(X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS \
79 | X86_CR0_ET | X86_CR0_NE | X86_CR0_WP | X86_CR0_AM \
80 | X86_CR0_NW | X86_CR0_CD | X86_CR0_PG))
81
Borislav Petkovcfaa7902015-01-15 09:44:56 +010082#define CR3_PCID_INVD BIT_64(63)
Joerg Roedelcfec82c2011-04-04 12:39:28 +020083#define CR4_RESERVED_BITS \
84 (~(unsigned long)(X86_CR4_VME | X86_CR4_PVI | X86_CR4_TSD | X86_CR4_DE\
85 | X86_CR4_PSE | X86_CR4_PAE | X86_CR4_MCE \
Mao, Junjiead756a12012-07-02 01:18:48 +000086 | X86_CR4_PGE | X86_CR4_PCE | X86_CR4_OSFXSR | X86_CR4_PCIDE \
H. Peter Anvinafcbf132013-04-27 16:37:47 -070087 | X86_CR4_OSXSAVE | X86_CR4_SMEP | X86_CR4_FSGSBASE \
Yu Zhangfd8cb432017-08-24 20:27:56 +080088 | X86_CR4_OSXMMEXCPT | X86_CR4_LA57 | X86_CR4_VMXE \
Paolo Bonziniae3e61e2016-07-12 10:36:41 +020089 | X86_CR4_SMAP | X86_CR4_PKE | X86_CR4_UMIP))
Joerg Roedelcfec82c2011-04-04 12:39:28 +020090
91#define CR8_RESERVED_BITS (~(unsigned long)X86_CR8_TPR)
92
93
Zhang Xiantaocd6e8f82007-11-19 14:33:37 +080094
Zhang Xiantaocd6e8f82007-11-19 14:33:37 +080095#define INVALID_PAGE (~(hpa_t)0)
Xiao Guangrongdd180b32010-07-03 16:02:42 +080096#define VALID_PAGE(x) ((x) != INVALID_PAGE)
97
Zhang Xiantaocd6e8f82007-11-19 14:33:37 +080098#define UNMAPPED_GVA (~(gpa_t)0)
99
Joerg Roedelec04b262009-06-19 15:16:23 +0200100/* KVM Hugepage definitions for x86 */
Joerg Roedel04326ca2009-07-27 16:30:47 +0200101#define KVM_NR_PAGE_SIZES 3
Joerg Roedel82855412010-07-01 16:00:11 +0200102#define KVM_HPAGE_GFN_SHIFT(x) (((x) - 1) * 9)
103#define KVM_HPAGE_SHIFT(x) (PAGE_SHIFT + KVM_HPAGE_GFN_SHIFT(x))
Joerg Roedelec04b262009-06-19 15:16:23 +0200104#define KVM_HPAGE_SIZE(x) (1UL << KVM_HPAGE_SHIFT(x))
105#define KVM_HPAGE_MASK(x) (~(KVM_HPAGE_SIZE(x) - 1))
106#define KVM_PAGES_PER_HPAGE(x) (KVM_HPAGE_SIZE(x) / PAGE_SIZE)
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300107
Christoffer Dall6d9d41e2013-10-02 14:22:28 -0700108static inline gfn_t gfn_to_index(gfn_t gfn, gfn_t base_gfn, int level)
109{
110 /* KVM_HPAGE_GFN_SHIFT(PT_PAGE_TABLE_LEVEL) must be 0. */
111 return (gfn >> KVM_HPAGE_GFN_SHIFT(level)) -
112 (base_gfn >> KVM_HPAGE_GFN_SHIFT(level));
113}
114
Zhang Xiantaod657a982007-12-14 09:41:22 +0800115#define KVM_PERMILLE_MMU_PAGES 20
116#define KVM_MIN_ALLOC_MMU_PAGES 64
David Matlack114df302016-12-19 13:58:25 -0800117#define KVM_MMU_HASH_SHIFT 12
Dong, Eddie1ae0a132008-01-07 13:20:25 +0200118#define KVM_NUM_MMU_PAGES (1 << KVM_MMU_HASH_SHIFT)
Zhang Xiantaod657a982007-12-14 09:41:22 +0800119#define KVM_MIN_FREE_MMU_PAGES 5
120#define KVM_REFILL_PAGES 25
Andre Przywara73c11602010-12-01 12:17:44 +0100121#define KVM_MAX_CPUID_ENTRIES 80
Sheng Yang0bed3b52008-10-09 16:01:54 +0800122#define KVM_NR_FIXED_MTRR_REGION 88
Paolo Bonzini0d234da2014-08-18 16:39:48 +0200123#define KVM_NR_VAR_MTRR 8
Zhang Xiantaod657a982007-12-14 09:41:22 +0800124
Gleb Natapovaf585b92010-10-14 11:22:46 +0200125#define ASYNC_PF_PER_VCPU 64
126
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -0300127enum kvm_reg {
Zhang Xiantao2b3ccfa2007-11-19 14:56:05 +0800128 VCPU_REGS_RAX = 0,
129 VCPU_REGS_RCX = 1,
130 VCPU_REGS_RDX = 2,
131 VCPU_REGS_RBX = 3,
132 VCPU_REGS_RSP = 4,
133 VCPU_REGS_RBP = 5,
134 VCPU_REGS_RSI = 6,
135 VCPU_REGS_RDI = 7,
136#ifdef CONFIG_X86_64
137 VCPU_REGS_R8 = 8,
138 VCPU_REGS_R9 = 9,
139 VCPU_REGS_R10 = 10,
140 VCPU_REGS_R11 = 11,
141 VCPU_REGS_R12 = 12,
142 VCPU_REGS_R13 = 13,
143 VCPU_REGS_R14 = 14,
144 VCPU_REGS_R15 = 15,
145#endif
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -0300146 VCPU_REGS_RIP,
Zhang Xiantao2b3ccfa2007-11-19 14:56:05 +0800147 NR_VCPU_REGS
148};
149
Avi Kivity6de4f3a2009-05-31 22:58:47 +0300150enum kvm_reg_ex {
151 VCPU_EXREG_PDPTR = NR_VCPU_REGS,
Avi Kivityaff48ba2010-12-05 18:56:11 +0200152 VCPU_EXREG_CR3,
Avi Kivity6de12732011-03-07 12:51:22 +0200153 VCPU_EXREG_RFLAGS,
Avi Kivity2fb92db2011-04-27 19:42:18 +0300154 VCPU_EXREG_SEGMENTS,
Avi Kivity6de4f3a2009-05-31 22:58:47 +0300155};
156
Zhang Xiantao2b3ccfa2007-11-19 14:56:05 +0800157enum {
Zhang Xiantao2b3ccfa2007-11-19 14:56:05 +0800158 VCPU_SREG_ES,
Avi Kivity81609e32008-05-27 16:26:01 +0300159 VCPU_SREG_CS,
160 VCPU_SREG_SS,
161 VCPU_SREG_DS,
Zhang Xiantao2b3ccfa2007-11-19 14:56:05 +0800162 VCPU_SREG_FS,
163 VCPU_SREG_GS,
Zhang Xiantao2b3ccfa2007-11-19 14:56:05 +0800164 VCPU_SREG_TR,
165 VCPU_SREG_LDTR,
166};
167
Avi Kivity56e82312009-08-12 15:04:37 +0300168#include <asm/kvm_emulate.h>
Zhang Xiantao2b3ccfa2007-11-19 14:56:05 +0800169
Zhang Xiantaod657a982007-12-14 09:41:22 +0800170#define KVM_NR_MEM_OBJS 40
171
Jan Kiszka42dbaa52008-12-15 13:52:10 +0100172#define KVM_NR_DB_REGS 4
173
174#define DR6_BD (1 << 13)
175#define DR6_BS (1 << 14)
Nadav Amit6f43ed02014-07-15 17:37:46 +0300176#define DR6_RTM (1 << 16)
177#define DR6_FIXED_1 0xfffe0ff0
178#define DR6_INIT 0xffff0ff0
179#define DR6_VOLATILE 0x0001e00f
Jan Kiszka42dbaa52008-12-15 13:52:10 +0100180
181#define DR7_BP_EN_MASK 0x000000ff
182#define DR7_GE (1 << 9)
183#define DR7_GD (1 << 13)
184#define DR7_FIXED_1 0x00000400
Nadav Amit6f43ed02014-07-15 17:37:46 +0300185#define DR7_VOLATILE 0xffff2bff
Jan Kiszka42dbaa52008-12-15 13:52:10 +0100186
Nadav Amitc205fb72014-12-25 02:52:16 +0200187#define PFERR_PRESENT_BIT 0
188#define PFERR_WRITE_BIT 1
189#define PFERR_USER_BIT 2
190#define PFERR_RSVD_BIT 3
191#define PFERR_FETCH_BIT 4
Huaitong Hanbe94f6b2016-03-22 16:51:20 +0800192#define PFERR_PK_BIT 5
Tom Lendacky14727752016-11-23 12:01:38 -0500193#define PFERR_GUEST_FINAL_BIT 32
194#define PFERR_GUEST_PAGE_BIT 33
Nadav Amitc205fb72014-12-25 02:52:16 +0200195
196#define PFERR_PRESENT_MASK (1U << PFERR_PRESENT_BIT)
197#define PFERR_WRITE_MASK (1U << PFERR_WRITE_BIT)
198#define PFERR_USER_MASK (1U << PFERR_USER_BIT)
199#define PFERR_RSVD_MASK (1U << PFERR_RSVD_BIT)
200#define PFERR_FETCH_MASK (1U << PFERR_FETCH_BIT)
Huaitong Hanbe94f6b2016-03-22 16:51:20 +0800201#define PFERR_PK_MASK (1U << PFERR_PK_BIT)
Tom Lendacky14727752016-11-23 12:01:38 -0500202#define PFERR_GUEST_FINAL_MASK (1ULL << PFERR_GUEST_FINAL_BIT)
203#define PFERR_GUEST_PAGE_MASK (1ULL << PFERR_GUEST_PAGE_BIT)
204
205#define PFERR_NESTED_GUEST_PAGE (PFERR_GUEST_PAGE_MASK | \
Tom Lendacky14727752016-11-23 12:01:38 -0500206 PFERR_WRITE_MASK | \
207 PFERR_PRESENT_MASK)
Nadav Amitc205fb72014-12-25 02:52:16 +0200208
Junaid Shahid37f0e8f2016-12-06 16:46:15 -0800209/*
210 * The mask used to denote special SPTEs, which can be either MMIO SPTEs or
211 * Access Tracking SPTEs. We use bit 62 instead of bit 63 to avoid conflicting
212 * with the SVE bit in EPT PTEs.
213 */
214#define SPTE_SPECIAL_MASK (1ULL << 62)
215
Gleb Natapov41383772012-04-19 14:06:29 +0300216/* apic attention bits */
217#define KVM_APIC_CHECK_VAPIC 0
Michael S. Tsirkinae7a2a32012-06-24 19:25:07 +0300218/*
219 * The following bit is set with PV-EOI, unset on EOI.
220 * We detect PV-EOI changes by guest by comparing
221 * this bit with PV-EOI in guest memory.
222 * See the implementation in apic_update_pv_eoi.
223 */
224#define KVM_APIC_PV_EOI_PENDING 1
Gleb Natapov41383772012-04-19 14:06:29 +0300225
Feng Wud84f1e02015-09-18 22:29:49 +0800226struct kvm_kernel_irq_routing_entry;
227
Zhang Xiantaod657a982007-12-14 09:41:22 +0800228/*
229 * We don't want allocation failures within the mmu code, so we preallocate
230 * enough memory for a single page fault in a cache.
231 */
232struct kvm_mmu_memory_cache {
233 int nobjs;
234 void *objects[KVM_NR_MEM_OBJS];
235};
236
Xiao Guangrong21ebbed2016-02-24 17:51:09 +0800237/*
238 * the pages used as guest page table on soft mmu are tracked by
239 * kvm_memory_slot.arch.gfn_track which is 16 bits, so the role bits used
240 * by indirect shadow page can not be more than 15 bits.
241 *
242 * Currently, we used 14 bits that are @level, @cr4_pae, @quadrant, @access,
243 * @nxe, @cr0_wp, @smep_andnot_wp and @smap_andnot_wp.
244 */
Zhang Xiantaod657a982007-12-14 09:41:22 +0800245union kvm_mmu_page_role {
246 unsigned word;
247 struct {
Joe Perches7d76b4d2008-03-23 01:02:34 -0700248 unsigned level:4;
Avi Kivity5b7e0102010-04-14 19:20:03 +0300249 unsigned cr4_pae:1;
Joe Perches7d76b4d2008-03-23 01:02:34 -0700250 unsigned quadrant:2;
Avi Kivityf6e2c02b2009-01-11 13:02:10 +0200251 unsigned direct:1;
Joe Perches7d76b4d2008-03-23 01:02:34 -0700252 unsigned access:3;
Marcelo Tosatti2e53d632008-02-20 14:47:24 -0500253 unsigned invalid:1;
Avi Kivity9645bb562009-03-31 11:31:54 +0300254 unsigned nxe:1;
Avi Kivity3dbe1412010-05-12 11:48:18 +0300255 unsigned cr0_wp:1;
Avi Kivity411c5882011-06-06 16:11:54 +0300256 unsigned smep_andnot_wp:1;
Xiao Guangrong0be02262015-05-11 22:55:21 +0800257 unsigned smap_andnot_wp:1;
Peter Feinerac8d57e2017-06-30 17:26:31 -0700258 unsigned ad_disabled:1;
259 unsigned :7;
Paolo Bonzini699023e2015-05-18 15:03:39 +0200260
261 /*
262 * This is left at the top of the word so that
263 * kvm_memslots_for_spte_role can extract it with a
264 * simple shift. While there is room, give it a whole
265 * byte so it is also faster to load it from memory.
266 */
267 unsigned smm:8;
Zhang Xiantaod657a982007-12-14 09:41:22 +0800268 };
269};
270
Takuya Yoshikawa018aabb52015-11-20 17:41:28 +0900271struct kvm_rmap_head {
272 unsigned long val;
273};
274
Zhang Xiantaod657a982007-12-14 09:41:22 +0800275struct kvm_mmu_page {
276 struct list_head link;
277 struct hlist_node hash_link;
278
279 /*
280 * The following two entries are used to key the shadow page in the
281 * hash table.
282 */
283 gfn_t gfn;
284 union kvm_mmu_page_role role;
285
286 u64 *spt;
287 /* hold the gfn of each spte inside spt */
288 gfn_t *gfns;
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -0300289 bool unsync;
Xiao Guangrong0571d362010-04-16 21:27:54 +0800290 int root_count; /* Currently serving as active root */
Marcelo Tosatti60c8aec2008-12-01 22:32:02 -0200291 unsigned int unsync_children;
Takuya Yoshikawa018aabb52015-11-20 17:41:28 +0900292 struct kvm_rmap_head parent_ptes; /* rmap pointers to parent sptes */
Xiao Guangrongf6f8ade2013-06-19 17:09:24 +0800293
294 /* The page is obsolete if mmu_valid_gen != kvm->arch.mmu_valid_gen. */
Xiao Guangrong5304b8d2013-05-31 08:36:22 +0800295 unsigned long mmu_valid_gen;
Xiao Guangrongf6f8ade2013-06-19 17:09:24 +0800296
Marcelo Tosatti0074ff62008-09-23 13:18:40 -0300297 DECLARE_BITMAP(unsync_child_bitmap, 512);
Xiao Guangrongc2a2ac22011-07-12 03:32:13 +0800298
299#ifdef CONFIG_X86_32
Xiao Guangrongaccaefe2013-06-19 17:09:20 +0800300 /*
301 * Used out of the mmu-lock to avoid reading spte values while an
302 * update is in progress; see the comments in __get_spte_lockless().
303 */
Xiao Guangrongc2a2ac22011-07-12 03:32:13 +0800304 int clear_spte_count;
305#endif
306
Xiao Guangrong0cbf8e42013-06-19 17:09:21 +0800307 /* Number of writes since the last time traversal visited this page. */
Xiao Guangronge5691a82016-02-24 17:51:12 +0800308 atomic_t write_flooding_count;
Zhang Xiantaod657a982007-12-14 09:41:22 +0800309};
310
Avi Kivity1c083642009-01-04 12:39:07 +0200311struct kvm_pio_request {
312 unsigned long count;
Avi Kivity1c083642009-01-04 12:39:07 +0200313 int in;
314 int port;
315 int size;
Avi Kivity1c083642009-01-04 12:39:07 +0200316};
317
Yu Zhang855feb62017-08-24 20:27:55 +0800318#define PT64_ROOT_MAX_LEVEL 5
Yu Zhang2a7266a2017-08-24 20:27:54 +0800319
Xiao Guangronga0a64f52015-08-05 12:04:21 +0800320struct rsvd_bits_validate {
Yu Zhang2a7266a2017-08-24 20:27:54 +0800321 u64 rsvd_bits_mask[2][PT64_ROOT_MAX_LEVEL];
Xiao Guangronga0a64f52015-08-05 12:04:21 +0800322 u64 bad_mt_xwr;
323};
324
Zhang Xiantaod657a982007-12-14 09:41:22 +0800325/*
Yu Zhang855feb62017-08-24 20:27:55 +0800326 * x86 supports 4 paging modes (5-level 64-bit, 4-level 64-bit, 3-level 32-bit,
327 * and 2-level 32-bit). The kvm_mmu structure abstracts the details of the
328 * current mmu mode.
Zhang Xiantaod657a982007-12-14 09:41:22 +0800329 */
330struct kvm_mmu {
Joerg Roedelf43addd2010-09-10 17:30:40 +0200331 void (*set_cr3)(struct kvm_vcpu *vcpu, unsigned long root);
Joerg Roedel5777ed32010-09-10 17:30:42 +0200332 unsigned long (*get_cr3)(struct kvm_vcpu *vcpu);
Avi Kivitye4e517b2011-07-28 11:36:17 +0300333 u64 (*get_pdptr)(struct kvm_vcpu *vcpu, int index);
Xiao Guangrong78b2c542010-12-07 10:48:06 +0800334 int (*page_fault)(struct kvm_vcpu *vcpu, gva_t gva, u32 err,
335 bool prefault);
Avi Kivity6389ee92010-11-29 16:12:30 +0200336 void (*inject_page_fault)(struct kvm_vcpu *vcpu,
337 struct x86_exception *fault);
Gleb Natapov1871c602010-02-10 14:21:32 +0200338 gpa_t (*gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t gva, u32 access,
Avi Kivityab9ae312010-11-22 17:53:26 +0200339 struct x86_exception *exception);
Paolo Bonzini54987b72014-09-02 13:23:06 +0200340 gpa_t (*translate_gpa)(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access,
341 struct x86_exception *exception);
Marcelo Tosattie8bc2172008-09-23 13:18:33 -0300342 int (*sync_page)(struct kvm_vcpu *vcpu,
Xiao Guangronga4a8e6f2010-11-19 17:04:03 +0800343 struct kvm_mmu_page *sp);
Marcelo Tosattia7052892008-09-23 13:18:35 -0300344 void (*invlpg)(struct kvm_vcpu *vcpu, gva_t gva);
Xiao Guangrong0f53b5b2011-03-09 15:43:51 +0800345 void (*update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
Xiao Guangrong7c562522011-03-28 10:29:27 +0800346 u64 *spte, const void *pte);
Zhang Xiantaod657a982007-12-14 09:41:22 +0800347 hpa_t root_hpa;
Avi Kivitya770f6f2008-12-21 19:20:09 +0200348 union kvm_mmu_page_role base_role;
Paolo Bonziniae1e2d12017-03-30 11:55:30 +0200349 u8 root_level;
350 u8 shadow_root_level;
351 u8 ept_ad;
Joerg Roedelc5a78f2b2010-09-10 17:30:39 +0200352 bool direct_map;
Zhang Xiantaod657a982007-12-14 09:41:22 +0800353
Avi Kivity97d64b72012-09-12 14:52:00 +0300354 /*
355 * Bitmap; bit set = permission fault
356 * Byte index: page fault error code [4:1]
357 * Bit index: pte permissions in ACC_* format
358 */
359 u8 permissions[16];
360
Huaitong Han2d344102016-03-22 16:51:19 +0800361 /*
362 * The pkru_mask indicates if protection key checks are needed. It
363 * consists of 16 domains indexed by page fault error code bits [4:1],
364 * with PFEC.RSVD replaced by ACC_USER_MASK from the page tables.
365 * Each domain has 2 bits which are ANDed with AD and WD from PKRU.
366 */
367 u32 pkru_mask;
368
Zhang Xiantaod657a982007-12-14 09:41:22 +0800369 u64 *pae_root;
Joerg Roedel81407ca2010-09-10 17:31:00 +0200370 u64 *lm_root;
Xiao Guangrongc258b622015-08-05 12:04:24 +0800371
372 /*
373 * check zero bits on shadow page table entries, these
374 * bits include not only hardware reserved bits but also
375 * the bits spte never used.
376 */
377 struct rsvd_bits_validate shadow_zero_check;
378
Xiao Guangronga0a64f52015-08-05 12:04:21 +0800379 struct rsvd_bits_validate guest_rsvd_check;
Joerg Roedelff03a072010-09-10 17:30:57 +0200380
Paolo Bonzini6bb69c92016-02-23 12:51:19 +0100381 /* Can have large pages at levels 2..last_nonleaf_level-1. */
382 u8 last_nonleaf_level;
Avi Kivity6fd01b72012-09-12 20:46:56 +0300383
Joerg Roedel2d48a982010-09-10 17:31:01 +0200384 bool nx;
385
Joerg Roedelff03a072010-09-10 17:30:57 +0200386 u64 pdptrs[4]; /* pae */
Zhang Xiantaod657a982007-12-14 09:41:22 +0800387};
388
Gleb Natapovf5132b02011-11-10 14:57:22 +0200389enum pmc_type {
390 KVM_PMC_GP = 0,
391 KVM_PMC_FIXED,
392};
393
394struct kvm_pmc {
395 enum pmc_type type;
396 u8 idx;
397 u64 counter;
398 u64 eventsel;
399 struct perf_event *perf_event;
400 struct kvm_vcpu *vcpu;
401};
402
403struct kvm_pmu {
404 unsigned nr_arch_gp_counters;
405 unsigned nr_arch_fixed_counters;
406 unsigned available_event_types;
407 u64 fixed_ctr_ctrl;
408 u64 global_ctrl;
409 u64 global_status;
410 u64 global_ovf_ctrl;
411 u64 counter_bitmask[2];
412 u64 global_ctrl_mask;
Andi Kleen103af0a2013-07-18 15:57:02 -0700413 u64 reserved_bits;
Gleb Natapovf5132b02011-11-10 14:57:22 +0200414 u8 version;
Robert Richter15c7ad52012-06-20 20:46:33 +0200415 struct kvm_pmc gp_counters[INTEL_PMC_MAX_GENERIC];
416 struct kvm_pmc fixed_counters[INTEL_PMC_MAX_FIXED];
Gleb Natapovf5132b02011-11-10 14:57:22 +0200417 struct irq_work irq_work;
418 u64 reprogram_pmi;
419};
420
Wei Huang25462f72015-06-19 15:45:05 +0200421struct kvm_pmu_ops;
422
Paolo Bonzini360b9482014-02-21 09:55:56 +0100423enum {
424 KVM_DEBUGREG_BP_ENABLED = 1,
Paolo Bonzinic77fb5f2014-02-21 10:17:24 +0100425 KVM_DEBUGREG_WONT_EXIT = 2,
Nadav Amitae561ed2015-04-02 03:10:37 +0300426 KVM_DEBUGREG_RELOAD = 4,
Paolo Bonzini360b9482014-02-21 09:55:56 +0100427};
428
Xiao Guangrong86fd5272015-06-15 16:55:27 +0800429struct kvm_mtrr_range {
430 u64 base;
431 u64 mask;
Xiao Guangrong19efffa2015-06-15 16:55:31 +0800432 struct list_head node;
Xiao Guangrong86fd5272015-06-15 16:55:27 +0800433};
434
Xiao Guangrong70109e72015-06-15 16:55:24 +0800435struct kvm_mtrr {
Xiao Guangrong86fd5272015-06-15 16:55:27 +0800436 struct kvm_mtrr_range var_ranges[KVM_NR_VAR_MTRR];
Xiao Guangrong70109e72015-06-15 16:55:24 +0800437 mtrr_type fixed_ranges[KVM_NR_FIXED_MTRR_REGION];
Xiao Guangrong10fac2d2015-06-15 16:55:26 +0800438 u64 deftype;
Xiao Guangrong19efffa2015-06-15 16:55:31 +0800439
440 struct list_head head;
Xiao Guangrong70109e72015-06-15 16:55:24 +0800441};
442
Andrey Smetanin1f4b34f2015-11-30 19:22:21 +0300443/* Hyper-V SynIC timer */
444struct kvm_vcpu_hv_stimer {
445 struct hrtimer timer;
446 int index;
447 u64 config;
448 u64 count;
449 u64 exp_time;
450 struct hv_message msg;
451 bool msg_pending;
452};
453
Andrey Smetanin5c9194122015-11-10 15:36:34 +0300454/* Hyper-V synthetic interrupt controller (SynIC)*/
455struct kvm_vcpu_hv_synic {
456 u64 version;
457 u64 control;
458 u64 msg_page;
459 u64 evt_page;
460 atomic64_t sint[HV_SYNIC_SINT_COUNT];
461 atomic_t sint_to_gsi[HV_SYNIC_SINT_COUNT];
462 DECLARE_BITMAP(auto_eoi_bitmap, 256);
463 DECLARE_BITMAP(vec_bitmap, 256);
464 bool active;
Roman Kaganefc479e2017-06-22 16:51:01 +0300465 bool dont_zero_synic_pages;
Andrey Smetanin5c9194122015-11-10 15:36:34 +0300466};
467
Andrey Smetanine83d5882015-07-03 15:01:34 +0300468/* Hyper-V per vcpu emulation context */
469struct kvm_vcpu_hv {
Roman Kagand3457c82017-07-14 17:13:20 +0300470 u32 vp_index;
Andrey Smetanine83d5882015-07-03 15:01:34 +0300471 u64 hv_vapic;
Andrey Smetanin9eec50b2015-09-16 12:29:50 +0300472 s64 runtime_offset;
Andrey Smetanin5c9194122015-11-10 15:36:34 +0300473 struct kvm_vcpu_hv_synic synic;
Andrey Smetanindb3975712015-11-10 15:36:35 +0300474 struct kvm_hyperv_exit exit;
Andrey Smetanin1f4b34f2015-11-30 19:22:21 +0300475 struct kvm_vcpu_hv_stimer stimer[HV_SYNIC_STIMER_COUNT];
476 DECLARE_BITMAP(stimer_pending_bitmap, HV_SYNIC_STIMER_COUNT);
Andrey Smetanine83d5882015-07-03 15:01:34 +0300477};
478
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800479struct kvm_vcpu_arch {
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -0300480 /*
481 * rip and regs accesses must go through
482 * kvm_{register,rip}_{read,write} functions.
483 */
484 unsigned long regs[NR_VCPU_REGS];
485 u32 regs_avail;
486 u32 regs_dirty;
Zhang Xiantao34c16ee2007-10-20 15:34:38 +0800487
488 unsigned long cr0;
Avi Kivitye8467fd2009-12-29 18:43:06 +0200489 unsigned long cr0_guest_owned_bits;
Zhang Xiantao34c16ee2007-10-20 15:34:38 +0800490 unsigned long cr2;
491 unsigned long cr3;
492 unsigned long cr4;
Avi Kivityfc78f512009-12-07 12:16:48 +0200493 unsigned long cr4_guest_owned_bits;
Zhang Xiantao34c16ee2007-10-20 15:34:38 +0800494 unsigned long cr8;
Paolo Bonzinib9dd21e2017-08-23 23:14:38 +0200495 u32 pkru;
Alexander Graf1371d902008-11-25 20:17:04 +0100496 u32 hflags;
Avi Kivityf6801df2010-01-21 15:31:50 +0200497 u64 efer;
Zhang Xiantao34c16ee2007-10-20 15:34:38 +0800498 u64 apic_base;
499 struct kvm_lapic *apic; /* kernel irqchip context */
Andrey Smetanind62caab2015-11-10 15:36:33 +0300500 bool apicv_active;
Andrey Smetanin63086302015-11-10 15:36:32 +0300501 DECLARE_BITMAP(ioapic_handled_vectors, 256);
Gleb Natapov41383772012-04-19 14:06:29 +0300502 unsigned long apic_attention;
Gleb Natapove1035712009-03-05 16:34:59 +0200503 int32_t apic_arb_prio;
Zhang Xiantao34c16ee2007-10-20 15:34:38 +0800504 int mp_state;
Zhang Xiantao34c16ee2007-10-20 15:34:38 +0800505 u64 ia32_misc_enable_msr;
Paolo Bonzini64d60672015-05-07 11:36:11 +0200506 u64 smbase;
Liran Alon52797bf2017-11-15 13:43:14 +0200507 u64 smi_count;
Avi Kivityb209749f2007-10-22 16:50:39 +0200508 bool tpr_access_reporting;
Wanpeng Li20300092014-12-02 19:14:59 +0800509 u64 ia32_xss;
Wanpeng Li518e7b92018-02-28 14:03:31 +0800510 u64 microcode_version;
Zhang Xiantao34c16ee2007-10-20 15:34:38 +0800511
Joerg Roedel14dfe852010-09-10 17:30:49 +0200512 /*
513 * Paging state of the vcpu
514 *
515 * If the vcpu runs in guest mode with two level paging this still saves
516 * the paging mode of the l1 guest. This context is always used to
517 * handle faults.
518 */
Zhang Xiantao34c16ee2007-10-20 15:34:38 +0800519 struct kvm_mmu mmu;
Joerg Roedel8df25a32010-09-10 17:30:46 +0200520
521 /*
Joerg Roedel6539e732010-09-10 17:30:50 +0200522 * Paging state of an L2 guest (used for nested npt)
523 *
524 * This context will save all necessary information to walk page tables
525 * of the an L2 guest. This context is only initialized for page table
526 * walking and not for faulting since we never handle l2 page faults on
527 * the host.
528 */
529 struct kvm_mmu nested_mmu;
530
531 /*
Joerg Roedel14dfe852010-09-10 17:30:49 +0200532 * Pointer to the mmu context currently used for
533 * gva_to_gpa translations.
534 */
535 struct kvm_mmu *walk_mmu;
536
Xiao Guangrong53c07b12011-05-15 23:26:20 +0800537 struct kvm_mmu_memory_cache mmu_pte_list_desc_cache;
Zhang Xiantao34c16ee2007-10-20 15:34:38 +0800538 struct kvm_mmu_memory_cache mmu_page_cache;
539 struct kvm_mmu_memory_cache mmu_page_header_cache;
540
Rik van Rielf775b132017-11-14 16:54:23 -0500541 /*
542 * QEMU userspace and the guest each have their own FPU state.
543 * In vcpu_run, we switch between the user and guest FPU contexts.
544 * While running a VCPU, the VCPU thread will have the guest FPU
545 * context.
546 *
547 * Note that while the PKRU state lives inside the fpu registers,
548 * it is switched out separately at VMENTER and VMEXIT time. The
549 * "guest_fpu" state here contains the guest FPU context, with the
550 * host PRKU bits.
551 */
552 struct fpu user_fpu;
Sheng Yang98918832010-05-17 17:08:28 +0800553 struct fpu guest_fpu;
Rik van Rielf775b132017-11-14 16:54:23 -0500554
Dexuan Cui2acf9232010-06-10 11:27:12 +0800555 u64 xcr0;
Paolo Bonzinid7876f12013-10-02 16:06:15 +0200556 u64 guest_supported_xcr0;
Paolo Bonzini4344ee92013-10-02 16:06:16 +0200557 u32 guest_xstate_size;
Zhang Xiantao34c16ee2007-10-20 15:34:38 +0800558
Zhang Xiantao34c16ee2007-10-20 15:34:38 +0800559 struct kvm_pio_request pio;
560 void *pio_data;
561
Gleb Natapov66fd3f72009-05-11 13:35:50 +0300562 u8 event_exit_inst_len;
563
Avi Kivity298101d2007-11-25 13:41:11 +0200564 struct kvm_queued_exception {
565 bool pending;
Wanpeng Li664f8e22017-08-24 03:35:09 -0700566 bool injected;
Avi Kivity298101d2007-11-25 13:41:11 +0200567 bool has_error_code;
568 u8 nr;
569 u32 error_code;
Wanpeng Liadfe20f2017-07-13 18:30:41 -0700570 u8 nested_apf;
Avi Kivity298101d2007-11-25 13:41:11 +0200571 } exception;
572
Avi Kivity937a7ea2008-07-03 15:17:01 +0300573 struct kvm_queued_interrupt {
574 bool pending;
Gleb Natapov66fd3f72009-05-11 13:35:50 +0300575 bool soft;
Avi Kivity937a7ea2008-07-03 15:17:01 +0300576 u8 nr;
577 } interrupt;
578
Zhang Xiantao34c16ee2007-10-20 15:34:38 +0800579 int halt_request; /* real mode on Intel only */
580
581 int cpuid_nent;
Dan Kenigsberg07716712007-11-21 17:10:04 +0200582 struct kvm_cpuid_entry2 cpuid_entries[KVM_MAX_CPUID_ENTRIES];
Eugene Korenevsky5a4f55c2015-03-29 23:56:12 +0300583
584 int maxphyaddr;
585
Zhang Xiantao34c16ee2007-10-20 15:34:38 +0800586 /* emulate context */
587
588 struct x86_emulate_ctxt emulate_ctxt;
Gleb Natapov7ae441e2011-03-31 12:06:41 +0200589 bool emulate_regs_need_sync_to_vcpu;
590 bool emulate_regs_need_sync_from_vcpu;
Gleb Natapov716d51a2012-09-03 15:24:26 +0300591 int (*complete_userspace_io)(struct kvm_vcpu *vcpu);
Glauber de Oliveira Costa18068522008-02-15 17:52:47 -0200592
593 gpa_t time;
Gerd Hoffmann50d0a0f2008-06-03 16:17:31 +0200594 struct pvclock_vcpu_time_info hv_clock;
Zachary Amsdene48672f2010-08-19 22:07:23 -1000595 unsigned int hw_tsc_khz;
Andy Honig0b794592013-02-20 14:48:10 -0800596 struct gfn_to_hva_cache pv_time;
597 bool pv_time_enabled;
Marcelo Tosatti51d59c62012-08-03 15:57:49 -0300598 /* set guest stopped flag in pvclock flags field */
599 bool pvclock_set_guest_stopped_request;
Glauber Costac9aaa892011-07-11 15:28:14 -0400600
601 struct {
602 u64 msr_val;
603 u64 last_steal;
Glauber Costac9aaa892011-07-11 15:28:14 -0400604 struct gfn_to_hva_cache stime;
605 struct kvm_steal_time steal;
606 } st;
607
Luiz Capitulinoa545ab62016-09-07 14:47:19 -0400608 u64 tsc_offset;
Zachary Amsden1d5f0662010-08-19 22:07:30 -1000609 u64 last_guest_tsc;
Zachary Amsden6f526ec2012-02-03 15:43:54 -0200610 u64 last_host_tsc;
Zachary Amsden0dd6a6e2012-02-03 15:43:56 -0200611 u64 tsc_offset_adjustment;
Zachary Amsdene26101b2012-02-03 15:43:57 -0200612 u64 this_tsc_nsec;
613 u64 this_tsc_write;
Tomasz Grabiec0d3da0d2014-06-24 09:42:43 +0200614 u64 this_tsc_generation;
Zachary Amsdenc2855452010-09-18 14:38:15 -1000615 bool tsc_catchup;
Zachary Amsdencc578282012-02-03 15:43:50 -0200616 bool tsc_always_catchup;
617 s8 virtual_tsc_shift;
618 u32 virtual_tsc_mult;
619 u32 virtual_tsc_khz;
Will Auldba904632012-11-29 12:42:50 -0800620 s64 ia32_tsc_adjust_msr;
Haozhong Zhangad7218832015-10-20 15:39:02 +0800621 u64 tsc_scaling_ratio;
Sheng Yang3419ffc2008-05-15 09:52:48 +0800622
Avi Kivity7460fb4a2011-09-20 13:43:14 +0300623 atomic_t nmi_queued; /* unprocessed asynchronous NMIs */
624 unsigned nmi_pending; /* NMI queued after currently running handler */
625 bool nmi_injected; /* Trying to inject an NMI this entry */
Paolo Bonzinif0778252015-04-01 15:06:40 +0200626 bool smi_pending; /* SMI queued after currently running handler */
Avi Kivity9ba075a2008-05-26 20:06:35 +0300627
Xiao Guangrong70109e72015-06-15 16:55:24 +0800628 struct kvm_mtrr mtrr_state;
Paolo Bonzini7cb060a2014-06-19 11:40:18 +0200629 u64 pat;
Jan Kiszka42dbaa52008-12-15 13:52:10 +0100630
Paolo Bonzini360b9482014-02-21 09:55:56 +0100631 unsigned switch_db_regs;
Jan Kiszka42dbaa52008-12-15 13:52:10 +0100632 unsigned long db[KVM_NR_DB_REGS];
633 unsigned long dr6;
634 unsigned long dr7;
635 unsigned long eff_db[KVM_NR_DB_REGS];
Jan Kiszkac8639012012-09-21 05:42:55 +0200636 unsigned long guest_debug_dr7;
Kyle Hueydb2336a2017-03-20 01:16:28 -0700637 u64 msr_platform_info;
638 u64 msr_misc_features_enables;
Huang Ying890ca9a2009-05-11 16:48:15 +0800639
640 u64 mcg_cap;
641 u64 mcg_status;
642 u64 mcg_ctl;
Ashok Rajc45dcc72016-06-22 14:59:56 +0800643 u64 mcg_ext_ctl;
Huang Ying890ca9a2009-05-11 16:48:15 +0800644 u64 *mce_banks;
Jan Kiszka94fe45d2009-10-18 13:24:44 +0200645
Xiao Guangrongbebb1062011-07-12 03:23:20 +0800646 /* Cache MMIO info */
647 u64 mmio_gva;
648 unsigned access;
649 gfn_t mmio_gfn;
David Matlack56f17dd2014-08-18 15:46:07 -0700650 u64 mmio_gen;
Xiao Guangrongbebb1062011-07-12 03:23:20 +0800651
Gleb Natapovf5132b02011-11-10 14:57:22 +0200652 struct kvm_pmu pmu;
653
Jan Kiszka94fe45d2009-10-18 13:24:44 +0200654 /* used for guest single stepping over the given code position */
Jan Kiszka94fe45d2009-10-18 13:24:44 +0200655 unsigned long singlestep_rip;
Jan Kiszkaf92653e2010-02-23 17:47:55 +0100656
Andrey Smetanine83d5882015-07-03 15:01:34 +0300657 struct kvm_vcpu_hv hyperv;
Sheng Yangf5f48ee2010-06-30 12:25:15 +0800658
659 cpumask_var_t wbinvd_dirty_mask;
Gleb Natapovaf585b92010-10-14 11:22:46 +0200660
Xiao Guangrong1cb3f3a2011-09-22 17:02:48 +0800661 unsigned long last_retry_eip;
662 unsigned long last_retry_addr;
663
Gleb Natapovaf585b92010-10-14 11:22:46 +0200664 struct {
665 bool halted;
666 gfn_t gfns[roundup_pow_of_two(ASYNC_PF_PER_VCPU)];
Gleb Natapov344d9582010-10-14 11:22:50 +0200667 struct gfn_to_hva_cache data;
668 u64 msr_val;
Gleb Natapov7c907052010-10-14 11:22:53 +0200669 u32 id;
Gleb Natapov6adba522010-10-14 11:22:55 +0200670 bool send_user_only;
Wanpeng Li1261bfa2017-07-13 18:30:40 -0700671 u32 host_apf_reason;
Wanpeng Liadfe20f2017-07-13 18:30:41 -0700672 unsigned long nested_apf_token;
Wanpeng Li52a5c152017-07-13 18:30:42 -0700673 bool delivery_as_pf_vmexit;
Gleb Natapovaf585b92010-10-14 11:22:46 +0200674 } apf;
Boris Ostrovsky2b036c62012-01-09 14:00:35 -0500675
676 /* OSVW MSRs (AMD only) */
677 struct {
678 u64 length;
679 u64 status;
680 } osvw;
Michael S. Tsirkinae7a2a32012-06-24 19:25:07 +0300681
682 struct {
683 u64 msr_val;
684 struct gfn_to_hva_cache data;
685 } pv_eoi;
Xiao Guangrong93c05d32013-01-13 23:49:07 +0800686
687 /*
688 * Indicate whether the access faults on its page table in guest
689 * which is set when fix page fault and used to detect unhandeable
690 * instruction.
691 */
692 bool write_fault_to_shadow_pgtable;
Yang Zhang25d92082013-08-06 12:00:32 +0300693
694 /* set at EPT violation at this point */
695 unsigned long exit_qualification;
Srivatsa Vaddagiri6aef2662013-08-26 14:18:34 +0530696
697 /* pv related host specific info */
698 struct {
699 bool pv_unhalted;
700 } pv;
Steve Rutherford7543a632015-07-29 23:21:41 -0700701
702 int pending_ioapic_eoi;
Steve Rutherford1c1a9ce2015-07-30 11:27:16 +0200703 int pending_external_vector;
Tom Lendacky0f89b202016-12-14 14:59:23 -0500704
Brijesh Singh618232e2017-08-17 18:36:57 +0200705 /* GPA available */
Tom Lendacky0f89b202016-12-14 14:59:23 -0500706 bool gpa_available;
Brijesh Singh618232e2017-08-17 18:36:57 +0200707 gpa_t gpa_val;
Longpeng(Mike)de63ad42017-08-08 12:05:33 +0800708
709 /* be preempted when it's in kernel-mode(cpl=0) */
710 bool preempted_in_kernel;
Zhang Xiantao34c16ee2007-10-20 15:34:38 +0800711};
712
Takuya Yoshikawadb3fe4e2012-02-08 13:02:18 +0900713struct kvm_lpage_info {
Xiao Guangrong92f94f12016-02-24 17:51:06 +0800714 int disallow_lpage;
Takuya Yoshikawadb3fe4e2012-02-08 13:02:18 +0900715};
716
717struct kvm_arch_memory_slot {
Takuya Yoshikawa018aabb52015-11-20 17:41:28 +0900718 struct kvm_rmap_head *rmap[KVM_NR_PAGE_SIZES];
Takuya Yoshikawadb3fe4e2012-02-08 13:02:18 +0900719 struct kvm_lpage_info *lpage_info[KVM_NR_PAGE_SIZES - 1];
Xiao Guangrong21ebbed2016-02-24 17:51:09 +0800720 unsigned short *gfn_track[KVM_PAGE_TRACK_MAX];
Takuya Yoshikawadb3fe4e2012-02-08 13:02:18 +0900721};
722
Radim Krčmář3548a252015-02-12 19:41:33 +0100723/*
724 * We use as the mode the number of bits allocated in the LDR for the
725 * logical processor ID. It happens that these are all powers of two.
726 * This makes it is very easy to detect cases where the APICs are
727 * configured for multiple modes; in that case, we cannot use the map and
728 * hence cannot use kvm_irq_delivery_to_apic_fast either.
729 */
730#define KVM_APIC_MODE_XAPIC_CLUSTER 4
731#define KVM_APIC_MODE_XAPIC_FLAT 8
732#define KVM_APIC_MODE_X2APIC 16
733
Gleb Natapov1e08ec42012-09-13 17:19:24 +0300734struct kvm_apic_map {
735 struct rcu_head rcu;
Radim Krčmář3548a252015-02-12 19:41:33 +0100736 u8 mode;
Radim Krčmář0ca52e72016-07-12 22:09:20 +0200737 u32 max_apic_id;
Radim Krčmáře45115b2016-07-12 22:09:19 +0200738 union {
739 struct kvm_lapic *xapic_flat_map[8];
740 struct kvm_lapic *xapic_cluster_map[16][4];
741 };
Radim Krčmář0ca52e72016-07-12 22:09:20 +0200742 struct kvm_lapic *phys_map[];
Gleb Natapov1e08ec42012-09-13 17:19:24 +0300743};
744
Andrey Smetanine83d5882015-07-03 15:01:34 +0300745/* Hyper-V emulation context */
746struct kvm_hv {
Paolo Bonzini3f5ad8b2016-12-12 10:12:53 +0100747 struct mutex hv_lock;
Andrey Smetanine83d5882015-07-03 15:01:34 +0300748 u64 hv_guest_os_id;
749 u64 hv_hypercall;
750 u64 hv_tsc_page;
Andrey Smetanine7d95132015-07-03 15:01:37 +0300751
752 /* Hyper-v based guest crash (NT kernel bugcheck) parameters */
753 u64 hv_crash_param[HV_X64_MSR_CRASH_PARAMS];
754 u64 hv_crash_ctl;
Paolo Bonzini095cf552016-02-08 12:54:12 +0100755
756 HV_REFERENCE_TSC_PAGE tsc_ref;
Roman Kaganfaeb7832018-02-01 16:48:32 +0300757
758 struct idr conn_to_evt;
Vitaly Kuznetsova2e164e2018-03-01 15:15:12 +0100759
760 u64 hv_reenlightenment_control;
761 u64 hv_tsc_emulation_control;
762 u64 hv_tsc_emulation_status;
Andrey Smetanine83d5882015-07-03 15:01:34 +0300763};
764
Radim Krčmář49776fa2016-12-16 16:10:02 +0100765enum kvm_irqchip_mode {
766 KVM_IRQCHIP_NONE,
767 KVM_IRQCHIP_KERNEL, /* created with KVM_CREATE_IRQCHIP */
768 KVM_IRQCHIP_SPLIT, /* created with KVM_CAP_SPLIT_IRQCHIP */
769};
770
Brijesh Singh1654efc2017-12-04 10:57:34 -0600771struct kvm_sev_info {
772 bool active; /* SEV enabled guest */
773 unsigned int asid; /* ASID used for this guest */
Brijesh Singh59414c92017-12-04 10:57:35 -0600774 unsigned int handle; /* SEV firmware handle */
775 int fd; /* SEV device fd */
Brijesh Singh89c50582017-12-04 10:57:35 -0600776 unsigned long pages_locked; /* Number of pages locked */
Brijesh Singh1e80fdc2017-12-04 10:57:38 -0600777 struct list_head regions_list; /* List of registered regions */
Brijesh Singh1654efc2017-12-04 10:57:34 -0600778};
779
Marcelo Tosattifef9cce2009-12-23 14:35:17 -0200780struct kvm_arch {
Dave Hansen49d5ca22010-08-19 18:11:28 -0700781 unsigned int n_used_mmu_pages;
Zhang Xiantaof05e70a2007-12-14 10:01:48 +0800782 unsigned int n_requested_mmu_pages;
Dave Hansen39de71e2010-08-19 18:11:14 -0700783 unsigned int n_max_mmu_pages;
Xiao Guangrong332b2072011-05-15 23:20:27 +0800784 unsigned int indirect_shadow_pages;
Xiao Guangrong5304b8d2013-05-31 08:36:22 +0800785 unsigned long mmu_valid_gen;
Zhang Xiantaof05e70a2007-12-14 10:01:48 +0800786 struct hlist_head mmu_page_hash[KVM_NUM_MMU_PAGES];
787 /*
788 * Hash table of struct kvm_mmu_page.
789 */
790 struct list_head active_mmu_pages;
Xiao Guangrong365c8862013-05-31 08:36:29 +0800791 struct list_head zapped_obsolete_pages;
Xiao Guangrong13d268c2016-02-24 17:51:16 +0800792 struct kvm_page_track_notifier_node mmu_sp_tracker;
Xiao Guangrong0eb05bf2016-02-24 17:51:13 +0800793 struct kvm_page_track_notifier_head track_notifier_head;
Xiao Guangrong365c8862013-05-31 08:36:29 +0800794
Ben-Ami Yassour4d5c5d02008-07-28 19:26:26 +0300795 struct list_head assigned_dev_head;
Joerg Roedel19de40a2008-12-03 14:43:34 +0100796 struct iommu_domain *iommu_domain;
Alex Williamsond96eb2c2013-10-30 11:02:23 -0600797 bool iommu_noncoherent;
Alex Williamsone0f0bbc2013-10-30 11:02:30 -0600798#define __KVM_HAVE_ARCH_NONCOHERENT_DMA
799 atomic_t noncoherent_dma_count;
Paolo Bonzini5544eb92015-07-07 15:41:58 +0200800#define __KVM_HAVE_ARCH_ASSIGNED_DEVICE
801 atomic_t assigned_device_count;
Zhang Xiantaod7deeeb02007-12-14 10:17:34 +0800802 struct kvm_pic *vpic;
803 struct kvm_ioapic *vioapic;
Sheng Yang78376992008-01-28 05:10:22 +0800804 struct kvm_pit *vpit;
Radim Krčmář42720132015-07-01 15:31:49 +0200805 atomic_t vapics_in_nmi_mode;
Gleb Natapov1e08ec42012-09-13 17:19:24 +0300806 struct mutex apic_map_lock;
807 struct kvm_apic_map *apic_map;
Zhang Xiantaobfc6d222007-12-14 10:20:16 +0800808
Zhang Xiantaobfc6d222007-12-14 10:20:16 +0800809 unsigned int tss_addr;
Tang Chenc24ae0d2014-09-24 15:57:58 +0800810 bool apic_access_page_done;
Glauber de Oliveira Costa18068522008-02-15 17:52:47 -0200811
812 gpa_t wall_clock;
Sheng Yangb7ebfb02008-04-25 21:44:52 +0800813
Wanpeng Li4d5422c2018-03-12 04:53:02 -0700814 bool mwait_in_guest;
Wanpeng Licaa057a2018-03-12 04:53:03 -0700815 bool hlt_in_guest;
Wanpeng Li4d5422c2018-03-12 04:53:02 -0700816
Sheng Yangb7ebfb02008-04-25 21:44:52 +0800817 bool ept_identity_pagetable_done;
Sheng Yangb927a3c2009-07-21 10:42:48 +0800818 gpa_t ept_identity_map_addr;
Sheng Yang5550af42008-10-15 20:15:06 +0800819
820 unsigned long irq_sources_bitmap;
Glauber Costaafbcf7a2009-10-16 15:28:36 -0400821 s64 kvmclock_offset;
Jan Kiszka038f8c12011-02-04 10:49:11 +0100822 raw_spinlock_t tsc_write_lock;
Zachary Amsdenf38e0982010-08-19 22:07:20 -1000823 u64 last_tsc_nsec;
Zachary Amsdenf38e0982010-08-19 22:07:20 -1000824 u64 last_tsc_write;
Zachary Amsden5d3cb0f62012-02-03 15:43:51 -0200825 u32 last_tsc_khz;
Zachary Amsdene26101b2012-02-03 15:43:57 -0200826 u64 cur_tsc_nsec;
827 u64 cur_tsc_write;
828 u64 cur_tsc_offset;
Tomasz Grabiec0d3da0d2014-06-24 09:42:43 +0200829 u64 cur_tsc_generation;
Marcelo Tosattib48aa972012-11-27 23:29:03 -0200830 int nr_vcpus_matched_tsc;
Ed Swierkffde22a2009-10-15 15:21:43 -0700831
Marcelo Tosattid8281992012-11-27 23:29:01 -0200832 spinlock_t pvclock_gtod_sync_lock;
833 bool use_master_clock;
834 u64 master_kernel_ns;
Thomas Gleixnera5a1d1c2016-12-21 20:32:01 +0100835 u64 master_cycle_now;
Andrew Jones7e44e442014-02-28 12:52:54 +0100836 struct delayed_work kvmclock_update_work;
Andrew Jones332967a2014-02-28 12:52:55 +0100837 struct delayed_work kvmclock_sync_work;
Marcelo Tosattid8281992012-11-27 23:29:01 -0200838
Ed Swierkffde22a2009-10-15 15:21:43 -0700839 struct kvm_xen_hvm_config xen_hvm_config;
Gleb Natapov55cd8e52010-01-17 15:51:22 +0200840
Paolo Bonzini6ef768f2014-11-20 13:45:31 +0100841 /* reads protected by irq_srcu, writes by irq_lock */
842 struct hlist_head mask_notifier_list;
843
Andrey Smetanine83d5882015-07-03 15:01:34 +0300844 struct kvm_hv hyperv;
Xiao Guangrongb034cf02010-12-23 16:08:35 +0800845
846 #ifdef CONFIG_KVM_MMU_AUDIT
847 int audit_point;
848 #endif
Marcelo Tosatti54750f22015-01-20 15:54:52 -0200849
Ladi Proseka826faf2017-06-26 09:56:43 +0200850 bool backwards_tsc_observed;
Marcelo Tosatti54750f22015-01-20 15:54:52 -0200851 bool boot_vcpu_runs_old_kvmclock;
Paolo Bonzinid71ba782015-07-29 11:56:48 +0200852 u32 bsp_vcpu_id;
Nadav Amit90de4a12015-04-13 01:53:41 +0300853
854 u64 disabled_quirks;
Steve Rutherford49df6392015-07-29 23:21:40 -0700855
Radim Krčmář49776fa2016-12-16 16:10:02 +0100856 enum kvm_irqchip_mode irqchip_mode;
Steve Rutherfordb053b2a2015-07-29 23:32:35 -0700857 u8 nr_reserved_ioapic_pins;
Feng Wu520040142016-01-25 16:53:33 +0800858
859 bool disabled_lapic_found;
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -0500860
861 /* Struct members for AVIC */
Suravee Suthikulpanit5ea11f22016-08-23 13:52:41 -0500862 u32 avic_vm_id;
Suravee Suthikulpanit18f40c52016-05-04 14:09:48 -0500863 u32 ldr_mode;
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -0500864 struct page *avic_logical_id_table_page;
865 struct page *avic_physical_id_table_page;
Suravee Suthikulpanit5881f732016-08-23 13:52:42 -0500866 struct hlist_node hnode;
Radim Krčmář371313132016-07-12 22:09:27 +0200867
868 bool x2apic_format;
Radim Krčmářc5192652016-07-12 22:09:28 +0200869 bool x2apic_broadcast_quirk_disabled;
Brijesh Singh1654efc2017-12-04 10:57:34 -0600870
871 struct kvm_sev_info sev_info;
Zhang Xiantaod69fb812007-12-14 09:54:20 +0800872};
873
Zhang Xiantao07114562007-12-14 10:23:23 +0800874struct kvm_vm_stat {
Suraj Jitindar Singh8a7e75d2016-08-02 14:03:22 +1000875 ulong mmu_shadow_zapped;
876 ulong mmu_pte_write;
877 ulong mmu_pte_updated;
878 ulong mmu_pde_zapped;
879 ulong mmu_flooded;
880 ulong mmu_recycled;
881 ulong mmu_cache_miss;
882 ulong mmu_unsync;
883 ulong remote_tlb_flush;
884 ulong lpages;
David Matlackf3414bc2016-12-20 15:25:57 -0800885 ulong max_mmu_page_hash_collisions;
Zhang Xiantao07114562007-12-14 10:23:23 +0800886};
887
Zhang Xiantao77b4c252007-12-14 09:49:26 +0800888struct kvm_vcpu_stat {
Suraj Jitindar Singh8a7e75d2016-08-02 14:03:22 +1000889 u64 pf_fixed;
890 u64 pf_guest;
891 u64 tlb_flush;
892 u64 invlpg;
Zhang Xiantao77b4c252007-12-14 09:49:26 +0800893
Suraj Jitindar Singh8a7e75d2016-08-02 14:03:22 +1000894 u64 exits;
895 u64 io_exits;
896 u64 mmio_exits;
897 u64 signal_exits;
898 u64 irq_window_exits;
899 u64 nmi_window_exits;
900 u64 halt_exits;
901 u64 halt_successful_poll;
902 u64 halt_attempted_poll;
903 u64 halt_poll_invalid;
904 u64 halt_wakeup;
905 u64 request_irq_exits;
906 u64 irq_exits;
907 u64 host_state_reload;
Suraj Jitindar Singh8a7e75d2016-08-02 14:03:22 +1000908 u64 fpu_reload;
909 u64 insn_emulation;
910 u64 insn_emulation_fail;
911 u64 hypercalls;
912 u64 irq_injections;
913 u64 nmi_injections;
Paolo Bonzini0f1e2612016-12-17 16:05:19 +0100914 u64 req_event;
Zhang Xiantao77b4c252007-12-14 09:49:26 +0800915};
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800916
Joerg Roedel8a76d7f2011-04-04 12:39:27 +0200917struct x86_instruction_info;
918
Will Auld8fe8ab42012-11-29 12:42:12 -0800919struct msr_data {
920 bool host_initiated;
921 u32 index;
922 u64 data;
923};
924
Paolo Bonzinicb5281a2014-12-17 18:17:20 +0100925struct kvm_lapic_irq {
926 u32 vector;
Paolo Bonzinib7cb2232015-04-21 14:57:05 +0200927 u16 delivery_mode;
928 u16 dest_mode;
929 bool level;
930 u16 trig_mode;
Paolo Bonzinicb5281a2014-12-17 18:17:20 +0100931 u32 shorthand;
932 u32 dest_id;
James Sullivan93bbf0b2015-03-18 19:26:03 -0600933 bool msi_redir_hint;
Paolo Bonzinicb5281a2014-12-17 18:17:20 +0100934};
935
Zhang Xiantaoea4a5ff2007-11-19 14:40:47 +0800936struct kvm_x86_ops {
937 int (*cpu_has_kvm_support)(void); /* __init */
938 int (*disabled_by_bios)(void); /* __init */
Radim Krčmář13a34e02014-08-28 15:13:03 +0200939 int (*hardware_enable)(void);
940 void (*hardware_disable)(void);
Zhang Xiantaoea4a5ff2007-11-19 14:40:47 +0800941 void (*check_processor_compatibility)(void *rtn);
942 int (*hardware_setup)(void); /* __init */
943 void (*hardware_unsetup)(void); /* __exit */
Avi Kivity774ead32007-12-26 13:57:04 +0200944 bool (*cpu_has_accelerated_tpr)(void);
Paolo Bonzini6d396b52015-04-01 14:25:33 +0200945 bool (*cpu_has_high_real_mode_segbase)(void);
Sheng Yang0e851882009-12-18 16:48:46 +0800946 void (*cpuid_update)(struct kvm_vcpu *vcpu);
Zhang Xiantaoea4a5ff2007-11-19 14:40:47 +0800947
Suravee Suthikulpanit03543132016-05-04 14:09:42 -0500948 int (*vm_init)(struct kvm *kvm);
949 void (*vm_destroy)(struct kvm *kvm);
950
Zhang Xiantaoea4a5ff2007-11-19 14:40:47 +0800951 /* Create, but do not attach this VCPU */
952 struct kvm_vcpu *(*vcpu_create)(struct kvm *kvm, unsigned id);
953 void (*vcpu_free)(struct kvm_vcpu *vcpu);
Nadav Amitd28bc9d2015-04-13 14:34:08 +0300954 void (*vcpu_reset)(struct kvm_vcpu *vcpu, bool init_event);
Zhang Xiantaoea4a5ff2007-11-19 14:40:47 +0800955
956 void (*prepare_guest_switch)(struct kvm_vcpu *vcpu);
957 void (*vcpu_load)(struct kvm_vcpu *vcpu, int cpu);
958 void (*vcpu_put)(struct kvm_vcpu *vcpu);
Zhang Xiantaoea4a5ff2007-11-19 14:40:47 +0800959
Paolo Bonzinia96036b2015-11-10 11:55:36 +0100960 void (*update_bp_intercept)(struct kvm_vcpu *vcpu);
Paolo Bonzini609e36d2015-04-08 15:30:38 +0200961 int (*get_msr)(struct kvm_vcpu *vcpu, struct msr_data *msr);
Will Auld8fe8ab42012-11-29 12:42:12 -0800962 int (*set_msr)(struct kvm_vcpu *vcpu, struct msr_data *msr);
Zhang Xiantaoea4a5ff2007-11-19 14:40:47 +0800963 u64 (*get_segment_base)(struct kvm_vcpu *vcpu, int seg);
964 void (*get_segment)(struct kvm_vcpu *vcpu,
965 struct kvm_segment *var, int seg);
Izik Eidus2e4d2652008-03-24 19:38:34 +0200966 int (*get_cpl)(struct kvm_vcpu *vcpu);
Zhang Xiantaoea4a5ff2007-11-19 14:40:47 +0800967 void (*set_segment)(struct kvm_vcpu *vcpu,
968 struct kvm_segment *var, int seg);
969 void (*get_cs_db_l_bits)(struct kvm_vcpu *vcpu, int *db, int *l);
Avi Kivitye8467fd2009-12-29 18:43:06 +0200970 void (*decache_cr0_guest_bits)(struct kvm_vcpu *vcpu);
Avi Kivityaff48ba2010-12-05 18:56:11 +0200971 void (*decache_cr3)(struct kvm_vcpu *vcpu);
Zhang Xiantaoea4a5ff2007-11-19 14:40:47 +0800972 void (*decache_cr4_guest_bits)(struct kvm_vcpu *vcpu);
973 void (*set_cr0)(struct kvm_vcpu *vcpu, unsigned long cr0);
974 void (*set_cr3)(struct kvm_vcpu *vcpu, unsigned long cr3);
Nadav Har'El5e1746d2011-05-25 23:03:24 +0300975 int (*set_cr4)(struct kvm_vcpu *vcpu, unsigned long cr4);
Zhang Xiantaoea4a5ff2007-11-19 14:40:47 +0800976 void (*set_efer)(struct kvm_vcpu *vcpu, u64 efer);
Gleb Natapov89a27f42010-02-16 10:51:48 +0200977 void (*get_idt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt);
978 void (*set_idt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt);
979 void (*get_gdt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt);
980 void (*set_gdt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt);
Jan Kiszka73aaf249e2014-01-04 18:47:16 +0100981 u64 (*get_dr6)(struct kvm_vcpu *vcpu);
982 void (*set_dr6)(struct kvm_vcpu *vcpu, unsigned long value);
Paolo Bonzinic77fb5f2014-02-21 10:17:24 +0100983 void (*sync_dirty_debug_regs)(struct kvm_vcpu *vcpu);
Gleb Natapov020df072010-04-13 10:05:23 +0300984 void (*set_dr7)(struct kvm_vcpu *vcpu, unsigned long value);
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -0300985 void (*cache_reg)(struct kvm_vcpu *vcpu, enum kvm_reg reg);
Zhang Xiantaoea4a5ff2007-11-19 14:40:47 +0800986 unsigned long (*get_rflags)(struct kvm_vcpu *vcpu);
987 void (*set_rflags)(struct kvm_vcpu *vcpu, unsigned long rflags);
988
Wanpeng Lic2ba05c2017-12-12 17:33:03 -0800989 void (*tlb_flush)(struct kvm_vcpu *vcpu, bool invalidate_gpa);
Zhang Xiantaoea4a5ff2007-11-19 14:40:47 +0800990
Avi Kivity851ba692009-08-24 11:10:17 +0300991 void (*run)(struct kvm_vcpu *vcpu);
992 int (*handle_exit)(struct kvm_vcpu *vcpu);
Zhang Xiantaoea4a5ff2007-11-19 14:40:47 +0800993 void (*skip_emulated_instruction)(struct kvm_vcpu *vcpu);
Glauber Costa2809f5d2009-05-12 16:21:05 -0400994 void (*set_interrupt_shadow)(struct kvm_vcpu *vcpu, int mask);
Paolo Bonzini37ccdcb2014-05-20 14:29:47 +0200995 u32 (*get_interrupt_shadow)(struct kvm_vcpu *vcpu);
Zhang Xiantaoea4a5ff2007-11-19 14:40:47 +0800996 void (*patch_hypercall)(struct kvm_vcpu *vcpu,
997 unsigned char *hypercall_addr);
Gleb Natapov66fd3f72009-05-11 13:35:50 +0300998 void (*set_irq)(struct kvm_vcpu *vcpu);
Gleb Natapov95ba8273132009-04-21 17:45:08 +0300999 void (*set_nmi)(struct kvm_vcpu *vcpu);
Wanpeng Licfcd20e2017-07-13 18:30:39 -07001000 void (*queue_exception)(struct kvm_vcpu *vcpu);
Avi Kivityb463a6f2010-07-20 15:06:17 +03001001 void (*cancel_injection)(struct kvm_vcpu *vcpu);
Gleb Natapov78646122009-03-23 12:12:11 +02001002 int (*interrupt_allowed)(struct kvm_vcpu *vcpu);
Gleb Natapov95ba8273132009-04-21 17:45:08 +03001003 int (*nmi_allowed)(struct kvm_vcpu *vcpu);
Jan Kiszka3cfc3092009-11-12 01:04:25 +01001004 bool (*get_nmi_mask)(struct kvm_vcpu *vcpu);
1005 void (*set_nmi_mask)(struct kvm_vcpu *vcpu, bool masked);
Jan Kiszkac9a79532014-03-07 20:03:15 +01001006 void (*enable_nmi_window)(struct kvm_vcpu *vcpu);
1007 void (*enable_irq_window)(struct kvm_vcpu *vcpu);
Gleb Natapov95ba8273132009-04-21 17:45:08 +03001008 void (*update_cr8_intercept)(struct kvm_vcpu *vcpu, int tpr, int irr);
Suravee Suthikulpanitb2a05fe2017-09-12 10:42:41 -05001009 bool (*get_enable_apicv)(struct kvm_vcpu *vcpu);
Andrey Smetanind62caab2015-11-10 15:36:33 +03001010 void (*refresh_apicv_exec_ctrl)(struct kvm_vcpu *vcpu);
Yang Zhangc7c9c562013-01-25 10:18:51 +08001011 void (*hwapic_irr_update)(struct kvm_vcpu *vcpu, int max_irr);
Paolo Bonzini67c9ddd2016-05-10 17:01:23 +02001012 void (*hwapic_isr_update)(struct kvm_vcpu *vcpu, int isr);
Andrey Smetanin63086302015-11-10 15:36:32 +03001013 void (*load_eoi_exitmap)(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap);
Yang Zhang8d146952013-01-25 10:18:50 +08001014 void (*set_virtual_x2apic_mode)(struct kvm_vcpu *vcpu, bool set);
Tang Chen4256f432014-09-24 15:57:54 +08001015 void (*set_apic_access_page_addr)(struct kvm_vcpu *vcpu, hpa_t hpa);
Yang Zhanga20ed542013-04-11 19:25:15 +08001016 void (*deliver_posted_interrupt)(struct kvm_vcpu *vcpu, int vector);
Paolo Bonzini76dfafd52016-12-19 17:17:11 +01001017 int (*sync_pir_to_irr)(struct kvm_vcpu *vcpu);
Zhang Xiantaoea4a5ff2007-11-19 14:40:47 +08001018 int (*set_tss_addr)(struct kvm *kvm, unsigned int addr);
Yu Zhang855feb62017-08-24 20:27:55 +08001019 int (*get_tdp_level)(struct kvm_vcpu *vcpu);
Sheng Yang4b12f0d2009-04-27 20:35:42 +08001020 u64 (*get_mt_mask)(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio);
Sheng Yang17cc3932010-01-05 19:02:27 +08001021 int (*get_lpage_level)(void);
Sheng Yang4e47c7a2009-12-18 16:48:47 +08001022 bool (*rdtscp_supported)(void);
Mao, Junjiead756a12012-07-02 01:18:48 +00001023 bool (*invpcid_supported)(void);
Joerg Roedel344f4142009-07-27 16:30:48 +02001024
Joerg Roedel1c97f0a2010-09-10 17:30:41 +02001025 void (*set_tdp_cr3)(struct kvm_vcpu *vcpu, unsigned long cr3);
1026
Joerg Roedeld4330ef2010-04-22 12:33:11 +02001027 void (*set_supported_cpuid)(u32 func, struct kvm_cpuid_entry2 *entry);
1028
Sheng Yangf5f48ee2010-06-30 12:25:15 +08001029 bool (*has_wbinvd_exit)(void);
1030
Zachary Amsden99e3e302010-08-19 22:07:17 -10001031 void (*write_tsc_offset)(struct kvm_vcpu *vcpu, u64 offset);
1032
Avi Kivity586f9602010-11-18 13:09:54 +02001033 void (*get_exit_info)(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2);
Joerg Roedel8a76d7f2011-04-04 12:39:27 +02001034
1035 int (*check_intercept)(struct kvm_vcpu *vcpu,
1036 struct x86_instruction_info *info,
1037 enum x86_intercept_stage stage);
Yang Zhanga547c6d2013-04-11 19:25:10 +08001038 void (*handle_external_intr)(struct kvm_vcpu *vcpu);
Liu, Jinsongda8999d2014-02-24 10:55:46 +00001039 bool (*mpx_supported)(void);
Wanpeng Li55412b22014-12-02 19:21:30 +08001040 bool (*xsaves_supported)(void);
Paolo Bonzini66336ca2016-07-12 10:36:41 +02001041 bool (*umip_emulated)(void);
Jan Kiszkab6b8a142014-03-07 20:03:12 +01001042
1043 int (*check_nested_events)(struct kvm_vcpu *vcpu, bool external_intr);
Radim Krčmářae97a3b2014-08-21 18:08:06 +02001044
1045 void (*sched_in)(struct kvm_vcpu *kvm, int cpu);
Kai Huang88178fd2015-01-28 10:54:27 +08001046
1047 /*
1048 * Arch-specific dirty logging hooks. These hooks are only supposed to
1049 * be valid if the specific arch has hardware-accelerated dirty logging
1050 * mechanism. Currently only for PML on VMX.
1051 *
1052 * - slot_enable_log_dirty:
1053 * called when enabling log dirty mode for the slot.
1054 * - slot_disable_log_dirty:
1055 * called when disabling log dirty mode for the slot.
1056 * also called when slot is created with log dirty disabled.
1057 * - flush_log_dirty:
1058 * called before reporting dirty_bitmap to userspace.
1059 * - enable_log_dirty_pt_masked:
1060 * called when reenabling log dirty for the GFNs in the mask after
1061 * corresponding bits are cleared in slot->dirty_bitmap.
1062 */
1063 void (*slot_enable_log_dirty)(struct kvm *kvm,
1064 struct kvm_memory_slot *slot);
1065 void (*slot_disable_log_dirty)(struct kvm *kvm,
1066 struct kvm_memory_slot *slot);
1067 void (*flush_log_dirty)(struct kvm *kvm);
1068 void (*enable_log_dirty_pt_masked)(struct kvm *kvm,
1069 struct kvm_memory_slot *slot,
1070 gfn_t offset, unsigned long mask);
Bandan Dasbab41652017-05-05 15:25:13 -04001071 int (*write_log_dirty)(struct kvm_vcpu *vcpu);
1072
Wei Huang25462f72015-06-19 15:45:05 +02001073 /* pmu operations of sub-arch */
1074 const struct kvm_pmu_ops *pmu_ops;
Feng Wuefc64402015-09-18 22:29:51 +08001075
Feng Wubf9f6ac2015-09-18 22:29:55 +08001076 /*
1077 * Architecture specific hooks for vCPU blocking due to
1078 * HLT instruction.
1079 * Returns for .pre_block():
1080 * - 0 means continue to block the vCPU.
1081 * - 1 means we cannot block the vCPU since some event
1082 * happens during this period, such as, 'ON' bit in
1083 * posted-interrupts descriptor is set.
1084 */
1085 int (*pre_block)(struct kvm_vcpu *vcpu);
1086 void (*post_block)(struct kvm_vcpu *vcpu);
Suravee Suthikulpanitd1ed0922016-05-04 14:09:43 -05001087
1088 void (*vcpu_blocking)(struct kvm_vcpu *vcpu);
1089 void (*vcpu_unblocking)(struct kvm_vcpu *vcpu);
1090
Feng Wuefc64402015-09-18 22:29:51 +08001091 int (*update_pi_irte)(struct kvm *kvm, unsigned int host_irq,
1092 uint32_t guest_irq, bool set);
Suravee Suthikulpanitbe8ca172016-05-04 14:09:49 -05001093 void (*apicv_post_state_restore)(struct kvm_vcpu *vcpu);
Yunhong Jiangce7a0582016-06-13 14:20:01 -07001094
1095 int (*set_hv_timer)(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc);
1096 void (*cancel_hv_timer)(struct kvm_vcpu *vcpu);
Ashok Rajc45dcc72016-06-22 14:59:56 +08001097
1098 void (*setup_mce)(struct kvm_vcpu *vcpu);
Ladi Prosek0234bf82017-10-11 16:54:40 +02001099
Ladi Prosek72d7b372017-10-11 16:54:41 +02001100 int (*smi_allowed)(struct kvm_vcpu *vcpu);
Ladi Prosek0234bf82017-10-11 16:54:40 +02001101 int (*pre_enter_smm)(struct kvm_vcpu *vcpu, char *smstate);
1102 int (*pre_leave_smm)(struct kvm_vcpu *vcpu, u64 smbase);
Ladi Prosekcc3d9672017-10-17 16:02:39 +02001103 int (*enable_smi_window)(struct kvm_vcpu *vcpu);
Brijesh Singh5acc5c02017-12-04 10:57:26 -06001104
1105 int (*mem_enc_op)(struct kvm *kvm, void __user *argp);
Brijesh Singh69eaede2017-12-04 10:57:26 -06001106 int (*mem_enc_reg_region)(struct kvm *kvm, struct kvm_enc_region *argp);
1107 int (*mem_enc_unreg_region)(struct kvm *kvm, struct kvm_enc_region *argp);
Tom Lendacky801e4592018-02-21 13:39:51 -06001108
1109 int (*get_msr_feature)(struct kvm_msr_entry *entry);
Zhang Xiantaoea4a5ff2007-11-19 14:40:47 +08001110};
1111
Gleb Natapovaf585b92010-10-14 11:22:46 +02001112struct kvm_arch_async_pf {
Gleb Natapov7c907052010-10-14 11:22:53 +02001113 u32 token;
Gleb Natapovaf585b92010-10-14 11:22:46 +02001114 gfn_t gfn;
Xiao Guangrongfb67e142010-12-07 10:35:25 +08001115 unsigned long cr3;
Xiao Guangrongc4806ac2010-11-12 14:49:55 +08001116 bool direct_map;
Gleb Natapovaf585b92010-10-14 11:22:46 +02001117};
1118
Zhang Xiantao97896d02007-11-14 20:09:30 +08001119extern struct kvm_x86_ops *kvm_x86_ops;
1120
Zhang Xiantao54f15852007-11-19 15:24:28 +08001121int kvm_mmu_module_init(void);
1122void kvm_mmu_module_exit(void);
1123
1124void kvm_mmu_destroy(struct kvm_vcpu *vcpu);
1125int kvm_mmu_create(struct kvm_vcpu *vcpu);
Paolo Bonzini8a3c1a332013-10-02 16:56:13 +02001126void kvm_mmu_setup(struct kvm_vcpu *vcpu);
Xiao Guangrong13d268c2016-02-24 17:51:16 +08001127void kvm_mmu_init_vm(struct kvm *kvm);
1128void kvm_mmu_uninit_vm(struct kvm *kvm);
Sheng Yang7b523452008-04-25 21:13:50 +08001129void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask,
Junaid Shahidf160c7b2016-12-06 16:46:16 -08001130 u64 dirty_mask, u64 nx_mask, u64 x_mask, u64 p_mask,
Tom Lendackyd0ec49d2017-07-17 16:10:27 -05001131 u64 acc_track_mask, u64 me_mask);
Zhang Xiantao54f15852007-11-19 15:24:28 +08001132
Paolo Bonzini8a3c1a332013-10-02 16:56:13 +02001133void kvm_mmu_reset_context(struct kvm_vcpu *vcpu);
Kai Huang1c91cad42015-01-28 10:54:26 +08001134void kvm_mmu_slot_remove_write_access(struct kvm *kvm,
1135 struct kvm_memory_slot *memslot);
Wanpeng Li3ea3b7f2015-04-03 15:40:25 +08001136void kvm_mmu_zap_collapsible_sptes(struct kvm *kvm,
Paolo Bonzinif36f3f22015-05-18 13:20:23 +02001137 const struct kvm_memory_slot *memslot);
Kai Huangf4b4b182015-01-28 10:54:24 +08001138void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm,
1139 struct kvm_memory_slot *memslot);
1140void kvm_mmu_slot_largepage_remove_write_access(struct kvm *kvm,
1141 struct kvm_memory_slot *memslot);
1142void kvm_mmu_slot_set_dirty(struct kvm *kvm,
1143 struct kvm_memory_slot *memslot);
1144void kvm_mmu_clear_dirty_pt_masked(struct kvm *kvm,
1145 struct kvm_memory_slot *slot,
1146 gfn_t gfn_offset, unsigned long mask);
Zhang Xiantao54f15852007-11-19 15:24:28 +08001147void kvm_mmu_zap_all(struct kvm *kvm);
Paolo Bonzini54bf36a2015-04-08 15:39:23 +02001148void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, struct kvm_memslots *slots);
Zhang Xiantao3ad82a72007-11-20 13:11:38 +08001149unsigned int kvm_mmu_calculate_mmu_pages(struct kvm *kvm);
Zhang Xiantao54f15852007-11-19 15:24:28 +08001150void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int kvm_nr_mmu_pages);
1151
Joerg Roedelff03a072010-09-10 17:30:57 +02001152int load_pdptrs(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, unsigned long cr3);
Ladi Prosek9ed38ffa2016-11-30 16:03:10 +01001153bool pdptrs_changed(struct kvm_vcpu *vcpu);
Joerg Roedelcc4b6872008-02-07 13:47:43 +01001154
Marcelo Tosatti3200f402008-03-29 20:17:59 -03001155int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
Avi Kivity9f811282008-03-02 14:06:05 +02001156 const void *val, int bytes);
Marcelo Tosatti2f333bc2008-02-22 12:21:37 -05001157
Paolo Bonzini6ef768f2014-11-20 13:45:31 +01001158struct kvm_irq_mask_notifier {
1159 void (*func)(struct kvm_irq_mask_notifier *kimn, bool masked);
1160 int irq;
1161 struct hlist_node link;
1162};
1163
1164void kvm_register_irq_mask_notifier(struct kvm *kvm, int irq,
1165 struct kvm_irq_mask_notifier *kimn);
1166void kvm_unregister_irq_mask_notifier(struct kvm *kvm, int irq,
1167 struct kvm_irq_mask_notifier *kimn);
1168void kvm_fire_mask_notifiers(struct kvm *kvm, unsigned irqchip, unsigned pin,
1169 bool mask);
1170
Marcelo Tosatti2f333bc2008-02-22 12:21:37 -05001171extern bool tdp_enabled;
Avi Kivity9f811282008-03-02 14:06:05 +02001172
Liu, Jinsonga3e06bb2011-09-22 16:55:52 +08001173u64 vcpu_tsc_khz(struct kvm_vcpu *vcpu);
1174
Joerg Roedel92a1f122011-03-25 09:44:51 +01001175/* control of guest tsc rate supported? */
1176extern bool kvm_has_tsc_control;
Joerg Roedel92a1f122011-03-25 09:44:51 +01001177/* maximum supported tsc_khz for guests */
1178extern u32 kvm_max_guest_tsc_khz;
Haozhong Zhangbc9b9612015-10-20 15:39:01 +08001179/* number of bits of the fractional part of the TSC scaling ratio */
1180extern u8 kvm_tsc_scaling_ratio_frac_bits;
1181/* maximum allowed value of TSC scaling ratio */
1182extern u64 kvm_max_tsc_scaling_ratio;
Yunhong Jiang64672c92016-06-13 14:19:59 -07001183/* 1ull << kvm_tsc_scaling_ratio_frac_bits */
1184extern u64 kvm_default_tsc_scaling_ratio;
Joerg Roedel92a1f122011-03-25 09:44:51 +01001185
Ashok Rajc45dcc72016-06-22 14:59:56 +08001186extern u64 kvm_mce_cap_supported;
Sheng Yang571008d2008-01-02 14:49:22 +08001187
1188enum emulation_result {
Paolo Bonziniac0a48c2013-06-25 18:24:41 +02001189 EMULATE_DONE, /* no further processing */
1190 EMULATE_USER_EXIT, /* kvm_run ready for userspace exit */
Zhang Xiantao54f15852007-11-19 15:24:28 +08001191 EMULATE_FAIL, /* can't emulate this instruction */
1192};
1193
1194#define EMULTYPE_NO_DECODE (1 << 0)
1195#define EMULTYPE_TRAP_UD (1 << 1)
Gleb Natapovba8afb62009-04-12 13:36:57 +03001196#define EMULTYPE_SKIP (1 << 2)
Xiao Guangrong1cb3f3a2011-09-22 17:02:48 +08001197#define EMULTYPE_RETRY (1 << 3)
Gleb Natapov991eebf2013-04-11 12:10:51 +03001198#define EMULTYPE_NO_REEXECUTE (1 << 4)
Liran Alone2366172018-03-12 13:12:49 +02001199#define EMULTYPE_NO_UD_ON_FAIL (1 << 5)
Liran Alon04789b62018-03-12 13:12:50 +02001200#define EMULTYPE_VMWARE (1 << 6)
Andre Przywaradc25e892010-12-21 11:12:07 +01001201int x86_emulate_instruction(struct kvm_vcpu *vcpu, unsigned long cr2,
1202 int emulation_type, void *insn, int insn_len);
Andre Przywara51d8b662010-12-21 11:12:02 +01001203
1204static inline int emulate_instruction(struct kvm_vcpu *vcpu,
1205 int emulation_type)
1206{
Liran Alon9b8ae632017-11-05 16:56:34 +02001207 return x86_emulate_instruction(vcpu, 0,
1208 emulation_type | EMULTYPE_NO_REEXECUTE, NULL, 0);
Andre Przywara51d8b662010-12-21 11:12:02 +01001209}
1210
Joerg Roedelf2b4b7d2008-01-31 14:57:37 +01001211void kvm_enable_efer_bits(u64);
Jan Kiszka384bb782013-04-20 10:52:36 +02001212bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer);
Paolo Bonzini609e36d2015-04-08 15:30:38 +02001213int kvm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr);
Will Auld8fe8ab42012-11-29 12:42:12 -08001214int kvm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr);
Zhang Xiantao54f15852007-11-19 15:24:28 +08001215
1216struct x86_emulate_ctxt;
1217
Sean Christophersondca7f122018-03-08 08:57:27 -08001218int kvm_fast_pio(struct kvm_vcpu *vcpu, int size, unsigned short port, int in);
Kyle Huey6a908b62016-11-29 12:40:37 -08001219int kvm_emulate_cpuid(struct kvm_vcpu *vcpu);
Zhang Xiantao54f15852007-11-19 15:24:28 +08001220int kvm_emulate_halt(struct kvm_vcpu *vcpu);
Joel Schopp5cb56052015-03-02 13:43:31 -06001221int kvm_vcpu_halt(struct kvm_vcpu *vcpu);
Sheng Yangf5f48ee2010-06-30 12:25:15 +08001222int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu);
Zhang Xiantao54f15852007-11-19 15:24:28 +08001223
Guillaume Thouvenin3e6e0aa2008-05-27 10:18:46 +02001224void kvm_get_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg);
Gleb Natapovc6975182010-02-18 12:15:01 +02001225int kvm_load_segment_descriptor(struct kvm_vcpu *vcpu, u16 selector, int seg);
Paolo Bonzini2b4a2732014-11-24 14:35:24 +01001226void kvm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector);
Guillaume Thouvenin3e6e0aa2008-05-27 10:18:46 +02001227
Kevin Wolf7f3d35f2012-02-08 14:34:38 +01001228int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index,
1229 int reason, bool has_error_code, u32 error_code);
Izik Eidus37817f22008-03-24 23:14:53 +02001230
Avi Kivity49a9b072010-06-10 17:02:14 +03001231int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0);
Avi Kivity23902182010-06-10 17:02:16 +03001232int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3);
Avi Kivitya83b29c2010-06-10 17:02:15 +03001233int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4);
Andre Przywaraeea1cff2010-12-21 11:12:00 +01001234int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8);
Gleb Natapov020df072010-04-13 10:05:23 +03001235int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val);
1236int kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val);
Avi Kivity2d3ad1f2008-02-24 11:20:43 +02001237unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu);
1238void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw);
Zhang Xiantao54f15852007-11-19 15:24:28 +08001239void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l);
Dexuan Cui2acf9232010-06-10 11:27:12 +08001240int kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr);
Zhang Xiantao54f15852007-11-19 15:24:28 +08001241
Paolo Bonzini609e36d2015-04-08 15:30:38 +02001242int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr);
Will Auld8fe8ab42012-11-29 12:42:12 -08001243int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr);
Zhang Xiantao54f15852007-11-19 15:24:28 +08001244
Jan Kiszka91586a32009-10-05 13:07:21 +02001245unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu);
1246void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags);
Avi Kivity022cd0e2011-11-10 14:57:23 +02001247bool kvm_rdpmc(struct kvm_vcpu *vcpu);
Jan Kiszka91586a32009-10-05 13:07:21 +02001248
Avi Kivity298101d2007-11-25 13:41:11 +02001249void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr);
1250void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code);
Joerg Roedelce7ddec2010-04-22 12:33:13 +02001251void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr);
1252void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code);
Avi Kivity6389ee92010-11-29 16:12:30 +02001253void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault);
Joerg Roedelec92fe42010-09-10 17:30:51 +02001254int kvm_read_guest_page_mmu(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
1255 gfn_t gfn, void *data, int offset, int len,
1256 u32 access);
Avi Kivity0a79b002009-09-01 12:03:25 +03001257bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl);
Nadav Amit16f8a6f2014-10-03 01:10:05 +03001258bool kvm_require_dr(struct kvm_vcpu *vcpu, int dr);
Avi Kivity298101d2007-11-25 13:41:11 +02001259
Michael S. Tsirkin1a577b722012-07-19 13:45:20 +03001260static inline int __kvm_irq_line_state(unsigned long *irq_state,
1261 int irq_source_id, int level)
1262{
1263 /* Logical OR for level trig interrupt */
1264 if (level)
1265 __set_bit(irq_source_id, irq_state);
1266 else
1267 __clear_bit(irq_source_id, irq_state);
1268
1269 return !!(*irq_state);
1270}
1271
1272int kvm_pic_set_irq(struct kvm_pic *pic, int irq, int irq_source_id, int level);
1273void kvm_pic_clear_all(struct kvm_pic *pic, int irq_source_id);
Xiantao Zhang3de42dc2008-10-06 13:48:45 +08001274
Sheng Yang3419ffc2008-05-15 09:52:48 +08001275void kvm_inject_nmi(struct kvm_vcpu *vcpu);
1276
Xiao Guangrong1cb3f3a2011-09-22 17:02:48 +08001277int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn);
Zhang Xiantao54f15852007-11-19 15:24:28 +08001278int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva);
1279void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu);
1280int kvm_mmu_load(struct kvm_vcpu *vcpu);
1281void kvm_mmu_unload(struct kvm_vcpu *vcpu);
Marcelo Tosatti0ba73cd2008-09-23 13:18:34 -03001282void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu);
Paolo Bonzini54987b72014-09-02 13:23:06 +02001283gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access,
1284 struct x86_exception *exception);
Avi Kivityab9ae312010-11-22 17:53:26 +02001285gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva,
1286 struct x86_exception *exception);
1287gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva,
1288 struct x86_exception *exception);
1289gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva,
1290 struct x86_exception *exception);
1291gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva,
1292 struct x86_exception *exception);
Zhang Xiantao54f15852007-11-19 15:24:28 +08001293
Andrey Smetanind62caab2015-11-10 15:36:33 +03001294void kvm_vcpu_deactivate_apicv(struct kvm_vcpu *vcpu);
1295
Zhang Xiantao54f15852007-11-19 15:24:28 +08001296int kvm_emulate_hypercall(struct kvm_vcpu *vcpu);
1297
Tom Lendacky14727752016-11-23 12:01:38 -05001298int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t gva, u64 error_code,
Andre Przywaradc25e892010-12-21 11:12:07 +01001299 void *insn, int insn_len);
Marcelo Tosattia7052892008-09-23 13:18:35 -03001300void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva);
Paolo Bonzinid8d173d2013-10-02 16:56:11 +02001301void kvm_mmu_new_cr3(struct kvm_vcpu *vcpu);
Zhang Xiantao34c16ee2007-10-20 15:34:38 +08001302
Joerg Roedel18552672008-02-07 13:47:41 +01001303void kvm_enable_tdp(void);
Joerg Roedel5f4cb662008-07-14 20:36:36 +02001304void kvm_disable_tdp(void);
Joerg Roedel18552672008-02-07 13:47:41 +01001305
Paolo Bonzini54987b72014-09-02 13:23:06 +02001306static inline gpa_t translate_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access,
1307 struct x86_exception *exception)
Xiao Guangronge459e322011-11-28 20:42:16 +08001308{
1309 return gpa;
1310}
1311
Zhang Xiantaoec6d2732007-11-19 15:08:31 +08001312static inline struct kvm_mmu_page *page_header(hpa_t shadow_page)
1313{
1314 struct page *page = pfn_to_page(shadow_page >> PAGE_SHIFT);
1315
1316 return (struct kvm_mmu_page *)page_private(page);
1317}
1318
Avi Kivityd6e88ae2008-07-10 16:53:33 +03001319static inline u16 kvm_read_ldt(void)
Zhang Xiantaoec6d2732007-11-19 15:08:31 +08001320{
1321 u16 ldt;
1322 asm("sldt %0" : "=g"(ldt));
1323 return ldt;
1324}
1325
Avi Kivityd6e88ae2008-07-10 16:53:33 +03001326static inline void kvm_load_ldt(u16 sel)
Zhang Xiantaoec6d2732007-11-19 15:08:31 +08001327{
1328 asm("lldt %0" : : "rm"(sel));
1329}
Zhang Xiantaoec6d2732007-11-19 15:08:31 +08001330
Zhang Xiantaoec6d2732007-11-19 15:08:31 +08001331#ifdef CONFIG_X86_64
1332static inline unsigned long read_msr(unsigned long msr)
1333{
1334 u64 value;
1335
1336 rdmsrl(msr, value);
1337 return value;
1338}
1339#endif
1340
Zhang Xiantaoec6d2732007-11-19 15:08:31 +08001341static inline u32 get_rdx_init_val(void)
1342{
1343 return 0x600; /* P6 family */
1344}
1345
Avi Kivityc1a5d4f2007-11-25 14:12:03 +02001346static inline void kvm_inject_gp(struct kvm_vcpu *vcpu, u32 error_code)
1347{
1348 kvm_queue_exception_e(vcpu, GP_VECTOR, error_code);
1349}
1350
Zhang Xiantaoec6d2732007-11-19 15:08:31 +08001351#define TSS_IOPB_BASE_OFFSET 0x66
1352#define TSS_BASE_SIZE 0x68
1353#define TSS_IOPB_SIZE (65536 / 8)
1354#define TSS_REDIRECTION_SIZE (256 / 8)
Joe Perches7d76b4d2008-03-23 01:02:34 -07001355#define RMODE_TSS_SIZE \
1356 (TSS_BASE_SIZE + TSS_REDIRECTION_SIZE + TSS_IOPB_SIZE + 1)
Hollis Blanchard53e0aa72007-12-03 16:15:26 -06001357
Izik Eidus37817f22008-03-24 23:14:53 +02001358enum {
1359 TASK_SWITCH_CALL = 0,
1360 TASK_SWITCH_IRET = 1,
1361 TASK_SWITCH_JMP = 2,
1362 TASK_SWITCH_GATE = 3,
1363};
1364
Alexander Graf1371d902008-11-25 20:17:04 +01001365#define HF_GIF_MASK (1 << 0)
Alexander Graf3d6368e2008-11-25 20:17:07 +01001366#define HF_HIF_MASK (1 << 1)
1367#define HF_VINTR_MASK (1 << 2)
Gleb Natapov95ba8273132009-04-21 17:45:08 +03001368#define HF_NMI_MASK (1 << 3)
Gleb Natapov44c11432009-05-11 13:35:52 +03001369#define HF_IRET_MASK (1 << 4)
Joerg Roedelec9e60b2010-11-29 17:51:47 +01001370#define HF_GUEST_MASK (1 << 5) /* VCPU is in guest-mode */
Paolo Bonzinif0778252015-04-01 15:06:40 +02001371#define HF_SMM_MASK (1 << 6)
1372#define HF_SMM_INSIDE_NMI_MASK (1 << 7)
Alexander Graf1371d902008-11-25 20:17:04 +01001373
Paolo Bonzini699023e2015-05-18 15:03:39 +02001374#define __KVM_VCPU_MULTIPLE_ADDRESS_SPACE
1375#define KVM_ADDRESS_SPACE_NUM 2
1376
1377#define kvm_arch_vcpu_memslots_id(vcpu) ((vcpu)->arch.hflags & HF_SMM_MASK ? 1 : 0)
1378#define kvm_memslots_for_spte_role(kvm, role) __kvm_memslots(kvm, (role).smm)
Avi Kivity4ecac3f2008-05-13 13:23:38 +03001379
1380/*
1381 * Hardware virtualization extension instructions may fault if a
1382 * reboot turns off virtualization while processes are running.
1383 * Trap the fault and ignore the instruction if that happens.
1384 */
Avi Kivityb7c41452010-12-02 17:52:50 +02001385asmlinkage void kvm_spurious_fault(void);
Avi Kivity4ecac3f2008-05-13 13:23:38 +03001386
Avi Kivity5e520e62011-05-15 10:13:12 -04001387#define ____kvm_handle_fault_on_reboot(insn, cleanup_insn) \
Avi Kivity4ecac3f2008-05-13 13:23:38 +03001388 "666: " insn "\n\t" \
Avi Kivityb7c41452010-12-02 17:52:50 +02001389 "668: \n\t" \
Eduardo Habkost18b13e52008-08-19 20:00:08 -03001390 ".pushsection .fixup, \"ax\" \n" \
Avi Kivity4ecac3f2008-05-13 13:23:38 +03001391 "667: \n\t" \
Avi Kivity5e520e62011-05-15 10:13:12 -04001392 cleanup_insn "\n\t" \
Avi Kivityb7c41452010-12-02 17:52:50 +02001393 "cmpb $0, kvm_rebooting \n\t" \
1394 "jne 668b \n\t" \
Avi Kivity8ceed3472008-08-14 21:25:47 +03001395 __ASM_SIZE(push) " $666b \n\t" \
Avi Kivityb7c41452010-12-02 17:52:50 +02001396 "call kvm_spurious_fault \n\t" \
Avi Kivity4ecac3f2008-05-13 13:23:38 +03001397 ".popsection \n\t" \
H. Peter Anvin3ee89722012-04-20 13:41:59 -07001398 _ASM_EXTABLE(666b, 667b)
Avi Kivity4ecac3f2008-05-13 13:23:38 +03001399
Avi Kivity5e520e62011-05-15 10:13:12 -04001400#define __kvm_handle_fault_on_reboot(insn) \
1401 ____kvm_handle_fault_on_reboot(insn, "")
1402
Andrea Arcangelie930bff2008-07-25 16:24:52 +02001403#define KVM_ARCH_WANT_MMU_NOTIFIER
1404int kvm_unmap_hva(struct kvm *kvm, unsigned long hva);
Takuya Yoshikawab3ae2092012-07-02 17:56:33 +09001405int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end);
Andres Lagar-Cavilla57128462014-09-22 14:54:42 -07001406int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end);
Andrea Arcangeli8ee53822011-01-13 15:47:10 -08001407int kvm_test_age_hva(struct kvm *kvm, unsigned long hva);
Izik Eidus3da0dd42009-09-23 21:47:18 +03001408void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte);
Yang Zhangc7c9c562013-01-25 10:18:51 +08001409int kvm_cpu_has_injectable_intr(struct kvm_vcpu *v);
Gleb Natapova1b37102009-07-09 15:33:52 +03001410int kvm_cpu_has_interrupt(struct kvm_vcpu *vcpu);
1411int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu);
Gleb Natapov0b717852009-07-09 15:33:53 +03001412int kvm_cpu_get_interrupt(struct kvm_vcpu *v);
Nadav Amitd28bc9d2015-04-13 14:34:08 +03001413void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event);
Tang Chen4256f432014-09-24 15:57:54 +08001414void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu);
Andrea Arcangelie930bff2008-07-25 16:24:52 +02001415
Avi Kivity18863bd2009-09-07 11:12:18 +03001416void kvm_define_shared_msr(unsigned index, u32 msr);
Andy Honig8b3c3102014-08-27 11:16:44 -07001417int kvm_set_shared_msr(unsigned index, u64 val, u64 mask);
Avi Kivity18863bd2009-09-07 11:12:18 +03001418
Haozhong Zhang35181e82015-10-20 15:39:03 +08001419u64 kvm_scale_tsc(struct kvm_vcpu *vcpu, u64 tsc);
Haozhong Zhang4ba76532015-10-20 15:39:07 +08001420u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc);
Haozhong Zhang35181e82015-10-20 15:39:03 +08001421
Nadav Amit82b32772014-11-02 11:54:45 +02001422unsigned long kvm_get_linear_rip(struct kvm_vcpu *vcpu);
Jan Kiszkaf92653e2010-02-23 17:47:55 +01001423bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip);
1424
Paolo Bonzini2860c4b2016-01-07 15:05:10 +01001425void kvm_make_mclock_inprogress_request(struct kvm *kvm);
1426void kvm_make_scan_ioapic_request(struct kvm *kvm);
1427
Gleb Natapovaf585b92010-10-14 11:22:46 +02001428void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
1429 struct kvm_async_pf *work);
1430void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
1431 struct kvm_async_pf *work);
Gleb Natapov56028d02010-10-17 18:13:42 +02001432void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
1433 struct kvm_async_pf *work);
Gleb Natapov7c907052010-10-14 11:22:53 +02001434bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu);
Gleb Natapovaf585b92010-10-14 11:22:46 +02001435extern bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn);
1436
Kyle Huey6affcbe2016-11-29 12:40:40 -08001437int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu);
1438int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err);
Andre Przywaradb8fcef2010-12-21 11:12:01 +01001439
Gleb Natapovf5132b02011-11-10 14:57:22 +02001440int kvm_is_in_guest(void);
1441
Paolo Bonzini1d8007b2015-10-12 13:38:32 +02001442int __x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa, u32 size);
1443int x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa, u32 size);
Paolo Bonzinid71ba782015-07-29 11:56:48 +02001444bool kvm_vcpu_is_reset_bsp(struct kvm_vcpu *vcpu);
1445bool kvm_vcpu_is_bsp(struct kvm_vcpu *vcpu);
Gleb Natapovf5132b02011-11-10 14:57:22 +02001446
Feng Wu8feb4a02015-09-18 22:29:47 +08001447bool kvm_intr_is_single_vcpu(struct kvm *kvm, struct kvm_lapic_irq *irq,
1448 struct kvm_vcpu **dest_vcpu);
1449
Radim Krčmář371313132016-07-12 22:09:27 +02001450void kvm_set_msi_irq(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e,
Feng Wud84f1e02015-09-18 22:29:49 +08001451 struct kvm_lapic_irq *irq);
Paolo Bonzini197a4f42015-11-04 16:24:17 +01001452
Suravee Suthikulpanitd1ed0922016-05-04 14:09:43 -05001453static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu)
1454{
1455 if (kvm_x86_ops->vcpu_blocking)
1456 kvm_x86_ops->vcpu_blocking(vcpu);
1457}
1458
1459static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu)
1460{
1461 if (kvm_x86_ops->vcpu_unblocking)
1462 kvm_x86_ops->vcpu_unblocking(vcpu);
1463}
1464
Christian Borntraeger3491caf2016-05-13 12:16:35 +02001465static inline void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu) {}
Christoffer Dall3217f7c2015-08-27 16:41:15 +02001466
Suravee Suthikulpanit7d669f52016-06-15 17:23:45 -05001467static inline int kvm_cpu_get_apicid(int mps_cpu)
1468{
1469#ifdef CONFIG_X86_LOCAL_APIC
Thomas Gleixner64063502017-09-13 23:29:21 +02001470 return default_cpu_present_to_apicid(mps_cpu);
Suravee Suthikulpanit7d669f52016-06-15 17:23:45 -05001471#else
1472 WARN_ON_ONCE(1);
1473 return BAD_APICID;
1474#endif
1475}
1476
Ladi Prosek05cade72017-10-11 16:54:45 +02001477#define put_smstate(type, buf, offset, val) \
1478 *(type *)((buf) + (offset) - 0x7e00) = val
1479
H. Peter Anvin1965aae2008-10-22 22:26:29 -07001480#endif /* _ASM_X86_KVM_HOST_H */