Vegard Nossum | a656c8e | 2008-07-22 21:27:11 +0200 | [diff] [blame] | 1 | /* |
Carsten Otte | 043405e | 2007-10-10 17:16:19 +0200 | [diff] [blame] | 2 | * Kernel-based Virtual Machine driver for Linux |
| 3 | * |
| 4 | * This header defines architecture specific interfaces, x86 version |
| 5 | * |
| 6 | * This work is licensed under the terms of the GNU GPL, version 2. See |
| 7 | * the COPYING file in the top-level directory. |
| 8 | * |
| 9 | */ |
| 10 | |
H. Peter Anvin | 1965aae | 2008-10-22 22:26:29 -0700 | [diff] [blame] | 11 | #ifndef _ASM_X86_KVM_HOST_H |
| 12 | #define _ASM_X86_KVM_HOST_H |
Carsten Otte | 043405e | 2007-10-10 17:16:19 +0200 | [diff] [blame] | 13 | |
Zhang Xiantao | 34c16ee | 2007-10-20 15:34:38 +0800 | [diff] [blame] | 14 | #include <linux/types.h> |
| 15 | #include <linux/mm.h> |
Andrea Arcangeli | e930bff | 2008-07-25 16:24:52 +0200 | [diff] [blame] | 16 | #include <linux/mmu_notifier.h> |
Marcelo Tosatti | 229456f | 2009-06-17 09:22:14 -0300 | [diff] [blame] | 17 | #include <linux/tracepoint.h> |
Sheng Yang | f5f48ee | 2010-06-30 12:25:15 +0800 | [diff] [blame] | 18 | #include <linux/cpumask.h> |
Gleb Natapov | f5132b0 | 2011-11-10 14:57:22 +0200 | [diff] [blame] | 19 | #include <linux/irq_work.h> |
Nicolai Stange | 447ae31 | 2018-07-29 12:15:33 +0200 | [diff] [blame] | 20 | #include <linux/irq.h> |
Zhang Xiantao | 34c16ee | 2007-10-20 15:34:38 +0800 | [diff] [blame] | 21 | |
| 22 | #include <linux/kvm.h> |
| 23 | #include <linux/kvm_para.h> |
Avi Kivity | edf8841 | 2007-12-16 11:02:48 +0200 | [diff] [blame] | 24 | #include <linux/kvm_types.h> |
Gleb Natapov | f5132b0 | 2011-11-10 14:57:22 +0200 | [diff] [blame] | 25 | #include <linux/perf_event.h> |
Marcelo Tosatti | d828199 | 2012-11-27 23:29:01 -0200 | [diff] [blame] | 26 | #include <linux/pvclock_gtod.h> |
| 27 | #include <linux/clocksource.h> |
Feng Wu | 8727688 | 2015-09-18 22:29:40 +0800 | [diff] [blame] | 28 | #include <linux/irqbypass.h> |
Andrey Smetanin | 5c919412 | 2015-11-10 15:36:34 +0300 | [diff] [blame] | 29 | #include <linux/hyperv.h> |
Zhang Xiantao | 34c16ee | 2007-10-20 15:34:38 +0800 | [diff] [blame] | 30 | |
Suravee Suthikulpanit | 7d669f5 | 2016-06-15 17:23:45 -0500 | [diff] [blame] | 31 | #include <asm/apic.h> |
Gerd Hoffmann | 50d0a0f | 2008-06-03 16:17:31 +0200 | [diff] [blame] | 32 | #include <asm/pvclock-abi.h> |
Hollis Blanchard | e01a1b5 | 2007-12-03 15:30:25 -0600 | [diff] [blame] | 33 | #include <asm/desc.h> |
Sheng Yang | 0bed3b5 | 2008-10-09 16:01:54 +0800 | [diff] [blame] | 34 | #include <asm/mtrr.h> |
Alexander Graf | 9962d03 | 2008-11-25 20:17:02 +0100 | [diff] [blame] | 35 | #include <asm/msr-index.h> |
H. Peter Anvin | 3ee8972 | 2012-04-20 13:41:59 -0700 | [diff] [blame] | 36 | #include <asm/asm.h> |
Xiao Guangrong | 21ebbed | 2016-02-24 17:51:09 +0800 | [diff] [blame] | 37 | #include <asm/kvm_page_track.h> |
Vitaly Kuznetsov | 5a48580 | 2018-03-20 15:02:05 +0100 | [diff] [blame] | 38 | #include <asm/hyperv-tlfs.h> |
Hollis Blanchard | e01a1b5 | 2007-12-03 15:30:25 -0600 | [diff] [blame] | 39 | |
Radim Krčmář | 682f732 | 2016-07-12 22:09:29 +0200 | [diff] [blame] | 40 | #define KVM_MAX_VCPUS 288 |
Radim Krčmář | 757883d | 2016-07-12 22:09:17 +0200 | [diff] [blame] | 41 | #define KVM_SOFT_MAX_VCPUS 240 |
Radim Krčmář | af1bae5 | 2016-07-12 22:09:30 +0200 | [diff] [blame] | 42 | #define KVM_MAX_VCPU_ID 1023 |
Igor Mammedov | 1d4e7e3 | 2014-11-06 15:52:47 +0000 | [diff] [blame] | 43 | #define KVM_USER_MEM_SLOTS 509 |
Alex Williamson | 0743247 | 2012-12-10 10:33:15 -0700 | [diff] [blame] | 44 | /* memory slots that are not exposed to userspace */ |
| 45 | #define KVM_PRIVATE_MEM_SLOTS 3 |
Alex Williamson | bbacc0c | 2012-12-10 10:33:09 -0700 | [diff] [blame] | 46 | #define KVM_MEM_SLOTS_NUM (KVM_USER_MEM_SLOTS + KVM_PRIVATE_MEM_SLOTS) |
Xiao Guangrong | 93a5cef | 2011-11-24 17:37:48 +0800 | [diff] [blame] | 47 | |
Paolo Bonzini | b401ee0 | 2017-04-18 12:41:18 +0200 | [diff] [blame] | 48 | #define KVM_HALT_POLL_NS_DEFAULT 200000 |
Avi Kivity | 69a9f69 | 2008-03-21 12:38:23 +0200 | [diff] [blame] | 49 | |
Alexander Graf | 8175e5b | 2013-04-15 10:42:33 +0200 | [diff] [blame] | 50 | #define KVM_IRQCHIP_NUM_PINS KVM_IOAPIC_NUM_PINS |
| 51 | |
Paolo Bonzini | 2860c4b | 2016-01-07 15:05:10 +0100 | [diff] [blame] | 52 | /* x86-specific vcpu->requests bit members */ |
Andrew Jones | 2387149 | 2017-06-04 14:43:51 +0200 | [diff] [blame] | 53 | #define KVM_REQ_MIGRATE_TIMER KVM_ARCH_REQ(0) |
| 54 | #define KVM_REQ_REPORT_TPR_ACCESS KVM_ARCH_REQ(1) |
| 55 | #define KVM_REQ_TRIPLE_FAULT KVM_ARCH_REQ(2) |
| 56 | #define KVM_REQ_MMU_SYNC KVM_ARCH_REQ(3) |
| 57 | #define KVM_REQ_CLOCK_UPDATE KVM_ARCH_REQ(4) |
Junaid Shahid | 6e42782 | 2018-06-27 14:59:08 -0700 | [diff] [blame] | 58 | #define KVM_REQ_LOAD_CR3 KVM_ARCH_REQ(5) |
Andrew Jones | 2387149 | 2017-06-04 14:43:51 +0200 | [diff] [blame] | 59 | #define KVM_REQ_EVENT KVM_ARCH_REQ(6) |
| 60 | #define KVM_REQ_APF_HALT KVM_ARCH_REQ(7) |
| 61 | #define KVM_REQ_STEAL_UPDATE KVM_ARCH_REQ(8) |
| 62 | #define KVM_REQ_NMI KVM_ARCH_REQ(9) |
| 63 | #define KVM_REQ_PMU KVM_ARCH_REQ(10) |
| 64 | #define KVM_REQ_PMI KVM_ARCH_REQ(11) |
| 65 | #define KVM_REQ_SMI KVM_ARCH_REQ(12) |
| 66 | #define KVM_REQ_MASTERCLOCK_UPDATE KVM_ARCH_REQ(13) |
| 67 | #define KVM_REQ_MCLOCK_INPROGRESS \ |
| 68 | KVM_ARCH_REQ_FLAGS(14, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) |
| 69 | #define KVM_REQ_SCAN_IOAPIC \ |
| 70 | KVM_ARCH_REQ_FLAGS(15, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) |
| 71 | #define KVM_REQ_GLOBAL_CLOCK_UPDATE KVM_ARCH_REQ(16) |
| 72 | #define KVM_REQ_APIC_PAGE_RELOAD \ |
| 73 | KVM_ARCH_REQ_FLAGS(17, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) |
| 74 | #define KVM_REQ_HV_CRASH KVM_ARCH_REQ(18) |
| 75 | #define KVM_REQ_IOAPIC_EOI_EXIT KVM_ARCH_REQ(19) |
| 76 | #define KVM_REQ_HV_RESET KVM_ARCH_REQ(20) |
| 77 | #define KVM_REQ_HV_EXIT KVM_ARCH_REQ(21) |
| 78 | #define KVM_REQ_HV_STIMER KVM_ARCH_REQ(22) |
Liran Alon | e40ff1d | 2018-03-21 02:50:31 +0200 | [diff] [blame] | 79 | #define KVM_REQ_LOAD_EOI_EXITMAP KVM_ARCH_REQ(23) |
Paolo Bonzini | 7f7f1ba | 2018-07-18 18:49:01 +0200 | [diff] [blame] | 80 | #define KVM_REQ_GET_VMCS12_PAGES KVM_ARCH_REQ(24) |
Paolo Bonzini | 2860c4b | 2016-01-07 15:05:10 +0100 | [diff] [blame] | 81 | |
Joerg Roedel | cfec82c | 2011-04-04 12:39:28 +0200 | [diff] [blame] | 82 | #define CR0_RESERVED_BITS \ |
| 83 | (~(unsigned long)(X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS \ |
| 84 | | X86_CR0_ET | X86_CR0_NE | X86_CR0_WP | X86_CR0_AM \ |
| 85 | | X86_CR0_NW | X86_CR0_CD | X86_CR0_PG)) |
| 86 | |
Joerg Roedel | cfec82c | 2011-04-04 12:39:28 +0200 | [diff] [blame] | 87 | #define CR4_RESERVED_BITS \ |
| 88 | (~(unsigned long)(X86_CR4_VME | X86_CR4_PVI | X86_CR4_TSD | X86_CR4_DE\ |
| 89 | | X86_CR4_PSE | X86_CR4_PAE | X86_CR4_MCE \ |
Mao, Junjie | ad756a1 | 2012-07-02 01:18:48 +0000 | [diff] [blame] | 90 | | X86_CR4_PGE | X86_CR4_PCE | X86_CR4_OSFXSR | X86_CR4_PCIDE \ |
H. Peter Anvin | afcbf13 | 2013-04-27 16:37:47 -0700 | [diff] [blame] | 91 | | X86_CR4_OSXSAVE | X86_CR4_SMEP | X86_CR4_FSGSBASE \ |
Yu Zhang | fd8cb43 | 2017-08-24 20:27:56 +0800 | [diff] [blame] | 92 | | X86_CR4_OSXMMEXCPT | X86_CR4_LA57 | X86_CR4_VMXE \ |
Paolo Bonzini | ae3e61e | 2016-07-12 10:36:41 +0200 | [diff] [blame] | 93 | | X86_CR4_SMAP | X86_CR4_PKE | X86_CR4_UMIP)) |
Joerg Roedel | cfec82c | 2011-04-04 12:39:28 +0200 | [diff] [blame] | 94 | |
| 95 | #define CR8_RESERVED_BITS (~(unsigned long)X86_CR8_TPR) |
| 96 | |
| 97 | |
Zhang Xiantao | cd6e8f8 | 2007-11-19 14:33:37 +0800 | [diff] [blame] | 98 | |
Zhang Xiantao | cd6e8f8 | 2007-11-19 14:33:37 +0800 | [diff] [blame] | 99 | #define INVALID_PAGE (~(hpa_t)0) |
Xiao Guangrong | dd180b3 | 2010-07-03 16:02:42 +0800 | [diff] [blame] | 100 | #define VALID_PAGE(x) ((x) != INVALID_PAGE) |
| 101 | |
Zhang Xiantao | cd6e8f8 | 2007-11-19 14:33:37 +0800 | [diff] [blame] | 102 | #define UNMAPPED_GVA (~(gpa_t)0) |
| 103 | |
Joerg Roedel | ec04b26 | 2009-06-19 15:16:23 +0200 | [diff] [blame] | 104 | /* KVM Hugepage definitions for x86 */ |
Wei Yang | 4fef0f4 | 2018-09-28 22:08:50 +0800 | [diff] [blame] | 105 | enum { |
| 106 | PT_PAGE_TABLE_LEVEL = 1, |
| 107 | PT_DIRECTORY_LEVEL = 2, |
| 108 | PT_PDPE_LEVEL = 3, |
| 109 | /* set max level to the biggest one */ |
| 110 | PT_MAX_HUGEPAGE_LEVEL = PT_PDPE_LEVEL, |
| 111 | }; |
| 112 | #define KVM_NR_PAGE_SIZES (PT_MAX_HUGEPAGE_LEVEL - \ |
| 113 | PT_PAGE_TABLE_LEVEL + 1) |
Joerg Roedel | 8285541 | 2010-07-01 16:00:11 +0200 | [diff] [blame] | 114 | #define KVM_HPAGE_GFN_SHIFT(x) (((x) - 1) * 9) |
| 115 | #define KVM_HPAGE_SHIFT(x) (PAGE_SHIFT + KVM_HPAGE_GFN_SHIFT(x)) |
Joerg Roedel | ec04b26 | 2009-06-19 15:16:23 +0200 | [diff] [blame] | 116 | #define KVM_HPAGE_SIZE(x) (1UL << KVM_HPAGE_SHIFT(x)) |
| 117 | #define KVM_HPAGE_MASK(x) (~(KVM_HPAGE_SIZE(x) - 1)) |
| 118 | #define KVM_PAGES_PER_HPAGE(x) (KVM_HPAGE_SIZE(x) / PAGE_SIZE) |
Marcelo Tosatti | 05da455 | 2008-02-23 11:44:30 -0300 | [diff] [blame] | 119 | |
Christoffer Dall | 6d9d41e | 2013-10-02 14:22:28 -0700 | [diff] [blame] | 120 | static inline gfn_t gfn_to_index(gfn_t gfn, gfn_t base_gfn, int level) |
| 121 | { |
| 122 | /* KVM_HPAGE_GFN_SHIFT(PT_PAGE_TABLE_LEVEL) must be 0. */ |
| 123 | return (gfn >> KVM_HPAGE_GFN_SHIFT(level)) - |
| 124 | (base_gfn >> KVM_HPAGE_GFN_SHIFT(level)); |
| 125 | } |
| 126 | |
Zhang Xiantao | d657a98 | 2007-12-14 09:41:22 +0800 | [diff] [blame] | 127 | #define KVM_PERMILLE_MMU_PAGES 20 |
| 128 | #define KVM_MIN_ALLOC_MMU_PAGES 64 |
David Matlack | 114df30 | 2016-12-19 13:58:25 -0800 | [diff] [blame] | 129 | #define KVM_MMU_HASH_SHIFT 12 |
Dong, Eddie | 1ae0a13 | 2008-01-07 13:20:25 +0200 | [diff] [blame] | 130 | #define KVM_NUM_MMU_PAGES (1 << KVM_MMU_HASH_SHIFT) |
Zhang Xiantao | d657a98 | 2007-12-14 09:41:22 +0800 | [diff] [blame] | 131 | #define KVM_MIN_FREE_MMU_PAGES 5 |
| 132 | #define KVM_REFILL_PAGES 25 |
Andre Przywara | 73c1160 | 2010-12-01 12:17:44 +0100 | [diff] [blame] | 133 | #define KVM_MAX_CPUID_ENTRIES 80 |
Sheng Yang | 0bed3b5 | 2008-10-09 16:01:54 +0800 | [diff] [blame] | 134 | #define KVM_NR_FIXED_MTRR_REGION 88 |
Paolo Bonzini | 0d234da | 2014-08-18 16:39:48 +0200 | [diff] [blame] | 135 | #define KVM_NR_VAR_MTRR 8 |
Zhang Xiantao | d657a98 | 2007-12-14 09:41:22 +0800 | [diff] [blame] | 136 | |
Gleb Natapov | af585b9 | 2010-10-14 11:22:46 +0200 | [diff] [blame] | 137 | #define ASYNC_PF_PER_VCPU 64 |
| 138 | |
Marcelo Tosatti | 5fdbf97 | 2008-06-27 14:58:02 -0300 | [diff] [blame] | 139 | enum kvm_reg { |
Zhang Xiantao | 2b3ccfa | 2007-11-19 14:56:05 +0800 | [diff] [blame] | 140 | VCPU_REGS_RAX = 0, |
| 141 | VCPU_REGS_RCX = 1, |
| 142 | VCPU_REGS_RDX = 2, |
| 143 | VCPU_REGS_RBX = 3, |
| 144 | VCPU_REGS_RSP = 4, |
| 145 | VCPU_REGS_RBP = 5, |
| 146 | VCPU_REGS_RSI = 6, |
| 147 | VCPU_REGS_RDI = 7, |
| 148 | #ifdef CONFIG_X86_64 |
| 149 | VCPU_REGS_R8 = 8, |
| 150 | VCPU_REGS_R9 = 9, |
| 151 | VCPU_REGS_R10 = 10, |
| 152 | VCPU_REGS_R11 = 11, |
| 153 | VCPU_REGS_R12 = 12, |
| 154 | VCPU_REGS_R13 = 13, |
| 155 | VCPU_REGS_R14 = 14, |
| 156 | VCPU_REGS_R15 = 15, |
| 157 | #endif |
Marcelo Tosatti | 5fdbf97 | 2008-06-27 14:58:02 -0300 | [diff] [blame] | 158 | VCPU_REGS_RIP, |
Zhang Xiantao | 2b3ccfa | 2007-11-19 14:56:05 +0800 | [diff] [blame] | 159 | NR_VCPU_REGS |
| 160 | }; |
| 161 | |
Avi Kivity | 6de4f3a | 2009-05-31 22:58:47 +0300 | [diff] [blame] | 162 | enum kvm_reg_ex { |
| 163 | VCPU_EXREG_PDPTR = NR_VCPU_REGS, |
Avi Kivity | aff48ba | 2010-12-05 18:56:11 +0200 | [diff] [blame] | 164 | VCPU_EXREG_CR3, |
Avi Kivity | 6de1273 | 2011-03-07 12:51:22 +0200 | [diff] [blame] | 165 | VCPU_EXREG_RFLAGS, |
Avi Kivity | 2fb92db | 2011-04-27 19:42:18 +0300 | [diff] [blame] | 166 | VCPU_EXREG_SEGMENTS, |
Avi Kivity | 6de4f3a | 2009-05-31 22:58:47 +0300 | [diff] [blame] | 167 | }; |
| 168 | |
Zhang Xiantao | 2b3ccfa | 2007-11-19 14:56:05 +0800 | [diff] [blame] | 169 | enum { |
Zhang Xiantao | 2b3ccfa | 2007-11-19 14:56:05 +0800 | [diff] [blame] | 170 | VCPU_SREG_ES, |
Avi Kivity | 81609e3 | 2008-05-27 16:26:01 +0300 | [diff] [blame] | 171 | VCPU_SREG_CS, |
| 172 | VCPU_SREG_SS, |
| 173 | VCPU_SREG_DS, |
Zhang Xiantao | 2b3ccfa | 2007-11-19 14:56:05 +0800 | [diff] [blame] | 174 | VCPU_SREG_FS, |
| 175 | VCPU_SREG_GS, |
Zhang Xiantao | 2b3ccfa | 2007-11-19 14:56:05 +0800 | [diff] [blame] | 176 | VCPU_SREG_TR, |
| 177 | VCPU_SREG_LDTR, |
| 178 | }; |
| 179 | |
Avi Kivity | 56e8231 | 2009-08-12 15:04:37 +0300 | [diff] [blame] | 180 | #include <asm/kvm_emulate.h> |
Zhang Xiantao | 2b3ccfa | 2007-11-19 14:56:05 +0800 | [diff] [blame] | 181 | |
Zhang Xiantao | d657a98 | 2007-12-14 09:41:22 +0800 | [diff] [blame] | 182 | #define KVM_NR_MEM_OBJS 40 |
| 183 | |
Jan Kiszka | 42dbaa5 | 2008-12-15 13:52:10 +0100 | [diff] [blame] | 184 | #define KVM_NR_DB_REGS 4 |
| 185 | |
| 186 | #define DR6_BD (1 << 13) |
| 187 | #define DR6_BS (1 << 14) |
Jim Mattson | cfb634f | 2018-09-21 10:36:17 -0700 | [diff] [blame] | 188 | #define DR6_BT (1 << 15) |
Nadav Amit | 6f43ed0 | 2014-07-15 17:37:46 +0300 | [diff] [blame] | 189 | #define DR6_RTM (1 << 16) |
| 190 | #define DR6_FIXED_1 0xfffe0ff0 |
| 191 | #define DR6_INIT 0xffff0ff0 |
| 192 | #define DR6_VOLATILE 0x0001e00f |
Jan Kiszka | 42dbaa5 | 2008-12-15 13:52:10 +0100 | [diff] [blame] | 193 | |
| 194 | #define DR7_BP_EN_MASK 0x000000ff |
| 195 | #define DR7_GE (1 << 9) |
| 196 | #define DR7_GD (1 << 13) |
| 197 | #define DR7_FIXED_1 0x00000400 |
Nadav Amit | 6f43ed0 | 2014-07-15 17:37:46 +0300 | [diff] [blame] | 198 | #define DR7_VOLATILE 0xffff2bff |
Jan Kiszka | 42dbaa5 | 2008-12-15 13:52:10 +0100 | [diff] [blame] | 199 | |
Nadav Amit | c205fb7 | 2014-12-25 02:52:16 +0200 | [diff] [blame] | 200 | #define PFERR_PRESENT_BIT 0 |
| 201 | #define PFERR_WRITE_BIT 1 |
| 202 | #define PFERR_USER_BIT 2 |
| 203 | #define PFERR_RSVD_BIT 3 |
| 204 | #define PFERR_FETCH_BIT 4 |
Huaitong Han | be94f6b | 2016-03-22 16:51:20 +0800 | [diff] [blame] | 205 | #define PFERR_PK_BIT 5 |
Tom Lendacky | 1472775 | 2016-11-23 12:01:38 -0500 | [diff] [blame] | 206 | #define PFERR_GUEST_FINAL_BIT 32 |
| 207 | #define PFERR_GUEST_PAGE_BIT 33 |
Nadav Amit | c205fb7 | 2014-12-25 02:52:16 +0200 | [diff] [blame] | 208 | |
| 209 | #define PFERR_PRESENT_MASK (1U << PFERR_PRESENT_BIT) |
| 210 | #define PFERR_WRITE_MASK (1U << PFERR_WRITE_BIT) |
| 211 | #define PFERR_USER_MASK (1U << PFERR_USER_BIT) |
| 212 | #define PFERR_RSVD_MASK (1U << PFERR_RSVD_BIT) |
| 213 | #define PFERR_FETCH_MASK (1U << PFERR_FETCH_BIT) |
Huaitong Han | be94f6b | 2016-03-22 16:51:20 +0800 | [diff] [blame] | 214 | #define PFERR_PK_MASK (1U << PFERR_PK_BIT) |
Tom Lendacky | 1472775 | 2016-11-23 12:01:38 -0500 | [diff] [blame] | 215 | #define PFERR_GUEST_FINAL_MASK (1ULL << PFERR_GUEST_FINAL_BIT) |
| 216 | #define PFERR_GUEST_PAGE_MASK (1ULL << PFERR_GUEST_PAGE_BIT) |
| 217 | |
| 218 | #define PFERR_NESTED_GUEST_PAGE (PFERR_GUEST_PAGE_MASK | \ |
Tom Lendacky | 1472775 | 2016-11-23 12:01:38 -0500 | [diff] [blame] | 219 | PFERR_WRITE_MASK | \ |
| 220 | PFERR_PRESENT_MASK) |
Nadav Amit | c205fb7 | 2014-12-25 02:52:16 +0200 | [diff] [blame] | 221 | |
Junaid Shahid | 37f0e8f | 2016-12-06 16:46:15 -0800 | [diff] [blame] | 222 | /* |
| 223 | * The mask used to denote special SPTEs, which can be either MMIO SPTEs or |
| 224 | * Access Tracking SPTEs. We use bit 62 instead of bit 63 to avoid conflicting |
| 225 | * with the SVE bit in EPT PTEs. |
| 226 | */ |
| 227 | #define SPTE_SPECIAL_MASK (1ULL << 62) |
| 228 | |
Gleb Natapov | 4138377 | 2012-04-19 14:06:29 +0300 | [diff] [blame] | 229 | /* apic attention bits */ |
| 230 | #define KVM_APIC_CHECK_VAPIC 0 |
Michael S. Tsirkin | ae7a2a3 | 2012-06-24 19:25:07 +0300 | [diff] [blame] | 231 | /* |
| 232 | * The following bit is set with PV-EOI, unset on EOI. |
| 233 | * We detect PV-EOI changes by guest by comparing |
| 234 | * this bit with PV-EOI in guest memory. |
| 235 | * See the implementation in apic_update_pv_eoi. |
| 236 | */ |
| 237 | #define KVM_APIC_PV_EOI_PENDING 1 |
Gleb Natapov | 4138377 | 2012-04-19 14:06:29 +0300 | [diff] [blame] | 238 | |
Feng Wu | d84f1e0 | 2015-09-18 22:29:49 +0800 | [diff] [blame] | 239 | struct kvm_kernel_irq_routing_entry; |
| 240 | |
Zhang Xiantao | d657a98 | 2007-12-14 09:41:22 +0800 | [diff] [blame] | 241 | /* |
| 242 | * We don't want allocation failures within the mmu code, so we preallocate |
| 243 | * enough memory for a single page fault in a cache. |
| 244 | */ |
| 245 | struct kvm_mmu_memory_cache { |
| 246 | int nobjs; |
| 247 | void *objects[KVM_NR_MEM_OBJS]; |
| 248 | }; |
| 249 | |
Xiao Guangrong | 21ebbed | 2016-02-24 17:51:09 +0800 | [diff] [blame] | 250 | /* |
| 251 | * the pages used as guest page table on soft mmu are tracked by |
| 252 | * kvm_memory_slot.arch.gfn_track which is 16 bits, so the role bits used |
| 253 | * by indirect shadow page can not be more than 15 bits. |
| 254 | * |
| 255 | * Currently, we used 14 bits that are @level, @cr4_pae, @quadrant, @access, |
| 256 | * @nxe, @cr0_wp, @smep_andnot_wp and @smap_andnot_wp. |
| 257 | */ |
Zhang Xiantao | d657a98 | 2007-12-14 09:41:22 +0800 | [diff] [blame] | 258 | union kvm_mmu_page_role { |
Vitaly Kuznetsov | 36d9594d | 2018-10-08 21:28:10 +0200 | [diff] [blame] | 259 | u32 word; |
Zhang Xiantao | d657a98 | 2007-12-14 09:41:22 +0800 | [diff] [blame] | 260 | struct { |
Joe Perches | 7d76b4d | 2008-03-23 01:02:34 -0700 | [diff] [blame] | 261 | unsigned level:4; |
Avi Kivity | 5b7e010 | 2010-04-14 19:20:03 +0300 | [diff] [blame] | 262 | unsigned cr4_pae:1; |
Joe Perches | 7d76b4d | 2008-03-23 01:02:34 -0700 | [diff] [blame] | 263 | unsigned quadrant:2; |
Avi Kivity | f6e2c02b | 2009-01-11 13:02:10 +0200 | [diff] [blame] | 264 | unsigned direct:1; |
Joe Perches | 7d76b4d | 2008-03-23 01:02:34 -0700 | [diff] [blame] | 265 | unsigned access:3; |
Marcelo Tosatti | 2e53d63 | 2008-02-20 14:47:24 -0500 | [diff] [blame] | 266 | unsigned invalid:1; |
Avi Kivity | 9645bb56 | 2009-03-31 11:31:54 +0300 | [diff] [blame] | 267 | unsigned nxe:1; |
Avi Kivity | 3dbe141 | 2010-05-12 11:48:18 +0300 | [diff] [blame] | 268 | unsigned cr0_wp:1; |
Avi Kivity | 411c588 | 2011-06-06 16:11:54 +0300 | [diff] [blame] | 269 | unsigned smep_andnot_wp:1; |
Xiao Guangrong | 0be0226 | 2015-05-11 22:55:21 +0800 | [diff] [blame] | 270 | unsigned smap_andnot_wp:1; |
Peter Feiner | ac8d57e | 2017-06-30 17:26:31 -0700 | [diff] [blame] | 271 | unsigned ad_disabled:1; |
Jim Mattson | 1313cc2 | 2018-05-09 17:02:04 -0400 | [diff] [blame] | 272 | unsigned guest_mode:1; |
| 273 | unsigned :6; |
Paolo Bonzini | 699023e | 2015-05-18 15:03:39 +0200 | [diff] [blame] | 274 | |
| 275 | /* |
| 276 | * This is left at the top of the word so that |
| 277 | * kvm_memslots_for_spte_role can extract it with a |
| 278 | * simple shift. While there is room, give it a whole |
| 279 | * byte so it is also faster to load it from memory. |
| 280 | */ |
| 281 | unsigned smm:8; |
Zhang Xiantao | d657a98 | 2007-12-14 09:41:22 +0800 | [diff] [blame] | 282 | }; |
| 283 | }; |
| 284 | |
Vitaly Kuznetsov | 36d9594d | 2018-10-08 21:28:10 +0200 | [diff] [blame] | 285 | union kvm_mmu_extended_role { |
Vitaly Kuznetsov | a336282 | 2018-10-08 21:28:11 +0200 | [diff] [blame] | 286 | /* |
| 287 | * This structure complements kvm_mmu_page_role caching everything needed for |
| 288 | * MMU configuration. If nothing in both these structures changed, MMU |
| 289 | * re-configuration can be skipped. @valid bit is set on first usage so we don't |
| 290 | * treat all-zero structure as valid data. |
| 291 | */ |
Vitaly Kuznetsov | 36d9594d | 2018-10-08 21:28:10 +0200 | [diff] [blame] | 292 | u32 word; |
Vitaly Kuznetsov | a336282 | 2018-10-08 21:28:11 +0200 | [diff] [blame] | 293 | struct { |
| 294 | unsigned int valid:1; |
| 295 | unsigned int execonly:1; |
Vitaly Kuznetsov | 7dcd575 | 2018-10-08 21:28:12 +0200 | [diff] [blame] | 296 | unsigned int cr0_pg:1; |
Vitaly Kuznetsov | a336282 | 2018-10-08 21:28:11 +0200 | [diff] [blame] | 297 | unsigned int cr4_pse:1; |
| 298 | unsigned int cr4_pke:1; |
| 299 | unsigned int cr4_smap:1; |
| 300 | unsigned int cr4_smep:1; |
Vitaly Kuznetsov | 7dcd575 | 2018-10-08 21:28:12 +0200 | [diff] [blame] | 301 | unsigned int cr4_la57:1; |
Yu Zhang | de3ccd2 | 2019-02-01 00:09:23 +0800 | [diff] [blame^] | 302 | unsigned int maxphyaddr:6; |
Vitaly Kuznetsov | a336282 | 2018-10-08 21:28:11 +0200 | [diff] [blame] | 303 | }; |
Vitaly Kuznetsov | 36d9594d | 2018-10-08 21:28:10 +0200 | [diff] [blame] | 304 | }; |
| 305 | |
| 306 | union kvm_mmu_role { |
| 307 | u64 as_u64; |
| 308 | struct { |
| 309 | union kvm_mmu_page_role base; |
| 310 | union kvm_mmu_extended_role ext; |
| 311 | }; |
| 312 | }; |
| 313 | |
Takuya Yoshikawa | 018aabb5 | 2015-11-20 17:41:28 +0900 | [diff] [blame] | 314 | struct kvm_rmap_head { |
| 315 | unsigned long val; |
| 316 | }; |
| 317 | |
Zhang Xiantao | d657a98 | 2007-12-14 09:41:22 +0800 | [diff] [blame] | 318 | struct kvm_mmu_page { |
| 319 | struct list_head link; |
| 320 | struct hlist_node hash_link; |
Wei Yang | 3ff519f | 2018-09-06 05:58:16 +0800 | [diff] [blame] | 321 | bool unsync; |
Zhang Xiantao | d657a98 | 2007-12-14 09:41:22 +0800 | [diff] [blame] | 322 | |
| 323 | /* |
| 324 | * The following two entries are used to key the shadow page in the |
| 325 | * hash table. |
| 326 | */ |
Zhang Xiantao | d657a98 | 2007-12-14 09:41:22 +0800 | [diff] [blame] | 327 | union kvm_mmu_page_role role; |
Wei Yang | 3ff519f | 2018-09-06 05:58:16 +0800 | [diff] [blame] | 328 | gfn_t gfn; |
Zhang Xiantao | d657a98 | 2007-12-14 09:41:22 +0800 | [diff] [blame] | 329 | |
| 330 | u64 *spt; |
| 331 | /* hold the gfn of each spte inside spt */ |
| 332 | gfn_t *gfns; |
Xiao Guangrong | 0571d36 | 2010-04-16 21:27:54 +0800 | [diff] [blame] | 333 | int root_count; /* Currently serving as active root */ |
Marcelo Tosatti | 60c8aec | 2008-12-01 22:32:02 -0200 | [diff] [blame] | 334 | unsigned int unsync_children; |
Takuya Yoshikawa | 018aabb5 | 2015-11-20 17:41:28 +0900 | [diff] [blame] | 335 | struct kvm_rmap_head parent_ptes; /* rmap pointers to parent sptes */ |
Xiao Guangrong | f6f8ade | 2013-06-19 17:09:24 +0800 | [diff] [blame] | 336 | |
| 337 | /* The page is obsolete if mmu_valid_gen != kvm->arch.mmu_valid_gen. */ |
Xiao Guangrong | 5304b8d | 2013-05-31 08:36:22 +0800 | [diff] [blame] | 338 | unsigned long mmu_valid_gen; |
Xiao Guangrong | f6f8ade | 2013-06-19 17:09:24 +0800 | [diff] [blame] | 339 | |
Marcelo Tosatti | 0074ff6 | 2008-09-23 13:18:40 -0300 | [diff] [blame] | 340 | DECLARE_BITMAP(unsync_child_bitmap, 512); |
Xiao Guangrong | c2a2ac2 | 2011-07-12 03:32:13 +0800 | [diff] [blame] | 341 | |
| 342 | #ifdef CONFIG_X86_32 |
Xiao Guangrong | accaefe | 2013-06-19 17:09:20 +0800 | [diff] [blame] | 343 | /* |
| 344 | * Used out of the mmu-lock to avoid reading spte values while an |
| 345 | * update is in progress; see the comments in __get_spte_lockless(). |
| 346 | */ |
Xiao Guangrong | c2a2ac2 | 2011-07-12 03:32:13 +0800 | [diff] [blame] | 347 | int clear_spte_count; |
| 348 | #endif |
| 349 | |
Xiao Guangrong | 0cbf8e4 | 2013-06-19 17:09:21 +0800 | [diff] [blame] | 350 | /* Number of writes since the last time traversal visited this page. */ |
Xiao Guangrong | e5691a8 | 2016-02-24 17:51:12 +0800 | [diff] [blame] | 351 | atomic_t write_flooding_count; |
Zhang Xiantao | d657a98 | 2007-12-14 09:41:22 +0800 | [diff] [blame] | 352 | }; |
| 353 | |
Avi Kivity | 1c08364 | 2009-01-04 12:39:07 +0200 | [diff] [blame] | 354 | struct kvm_pio_request { |
| 355 | unsigned long count; |
Avi Kivity | 1c08364 | 2009-01-04 12:39:07 +0200 | [diff] [blame] | 356 | int in; |
| 357 | int port; |
| 358 | int size; |
Avi Kivity | 1c08364 | 2009-01-04 12:39:07 +0200 | [diff] [blame] | 359 | }; |
| 360 | |
Yu Zhang | 855feb6 | 2017-08-24 20:27:55 +0800 | [diff] [blame] | 361 | #define PT64_ROOT_MAX_LEVEL 5 |
Yu Zhang | 2a7266a | 2017-08-24 20:27:54 +0800 | [diff] [blame] | 362 | |
Xiao Guangrong | a0a64f5 | 2015-08-05 12:04:21 +0800 | [diff] [blame] | 363 | struct rsvd_bits_validate { |
Yu Zhang | 2a7266a | 2017-08-24 20:27:54 +0800 | [diff] [blame] | 364 | u64 rsvd_bits_mask[2][PT64_ROOT_MAX_LEVEL]; |
Xiao Guangrong | a0a64f5 | 2015-08-05 12:04:21 +0800 | [diff] [blame] | 365 | u64 bad_mt_xwr; |
| 366 | }; |
| 367 | |
Junaid Shahid | 7c390d3 | 2018-06-27 14:59:06 -0700 | [diff] [blame] | 368 | struct kvm_mmu_root_info { |
| 369 | gpa_t cr3; |
| 370 | hpa_t hpa; |
| 371 | }; |
| 372 | |
| 373 | #define KVM_MMU_ROOT_INFO_INVALID \ |
| 374 | ((struct kvm_mmu_root_info) { .cr3 = INVALID_PAGE, .hpa = INVALID_PAGE }) |
| 375 | |
Junaid Shahid | b94742c | 2018-06-27 14:59:20 -0700 | [diff] [blame] | 376 | #define KVM_MMU_NUM_PREV_ROOTS 3 |
| 377 | |
Zhang Xiantao | d657a98 | 2007-12-14 09:41:22 +0800 | [diff] [blame] | 378 | /* |
Yu Zhang | 855feb6 | 2017-08-24 20:27:55 +0800 | [diff] [blame] | 379 | * x86 supports 4 paging modes (5-level 64-bit, 4-level 64-bit, 3-level 32-bit, |
| 380 | * and 2-level 32-bit). The kvm_mmu structure abstracts the details of the |
| 381 | * current mmu mode. |
Zhang Xiantao | d657a98 | 2007-12-14 09:41:22 +0800 | [diff] [blame] | 382 | */ |
| 383 | struct kvm_mmu { |
Joerg Roedel | f43addd | 2010-09-10 17:30:40 +0200 | [diff] [blame] | 384 | void (*set_cr3)(struct kvm_vcpu *vcpu, unsigned long root); |
Joerg Roedel | 5777ed3 | 2010-09-10 17:30:42 +0200 | [diff] [blame] | 385 | unsigned long (*get_cr3)(struct kvm_vcpu *vcpu); |
Avi Kivity | e4e517b | 2011-07-28 11:36:17 +0300 | [diff] [blame] | 386 | u64 (*get_pdptr)(struct kvm_vcpu *vcpu, int index); |
Xiao Guangrong | 78b2c54 | 2010-12-07 10:48:06 +0800 | [diff] [blame] | 387 | int (*page_fault)(struct kvm_vcpu *vcpu, gva_t gva, u32 err, |
| 388 | bool prefault); |
Avi Kivity | 6389ee9 | 2010-11-29 16:12:30 +0200 | [diff] [blame] | 389 | void (*inject_page_fault)(struct kvm_vcpu *vcpu, |
| 390 | struct x86_exception *fault); |
Gleb Natapov | 1871c60 | 2010-02-10 14:21:32 +0200 | [diff] [blame] | 391 | gpa_t (*gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t gva, u32 access, |
Avi Kivity | ab9ae31 | 2010-11-22 17:53:26 +0200 | [diff] [blame] | 392 | struct x86_exception *exception); |
Paolo Bonzini | 54987b7 | 2014-09-02 13:23:06 +0200 | [diff] [blame] | 393 | gpa_t (*translate_gpa)(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access, |
| 394 | struct x86_exception *exception); |
Marcelo Tosatti | e8bc217 | 2008-09-23 13:18:33 -0300 | [diff] [blame] | 395 | int (*sync_page)(struct kvm_vcpu *vcpu, |
Xiao Guangrong | a4a8e6f | 2010-11-19 17:04:03 +0800 | [diff] [blame] | 396 | struct kvm_mmu_page *sp); |
Junaid Shahid | 7eb77e9 | 2018-06-27 14:59:16 -0700 | [diff] [blame] | 397 | void (*invlpg)(struct kvm_vcpu *vcpu, gva_t gva, hpa_t root_hpa); |
Xiao Guangrong | 0f53b5b | 2011-03-09 15:43:51 +0800 | [diff] [blame] | 398 | void (*update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, |
Xiao Guangrong | 7c56252 | 2011-03-28 10:29:27 +0800 | [diff] [blame] | 399 | u64 *spte, const void *pte); |
Zhang Xiantao | d657a98 | 2007-12-14 09:41:22 +0800 | [diff] [blame] | 400 | hpa_t root_hpa; |
Vitaly Kuznetsov | ad7dc69 | 2019-02-22 17:45:01 +0100 | [diff] [blame] | 401 | gpa_t root_cr3; |
Vitaly Kuznetsov | 36d9594d | 2018-10-08 21:28:10 +0200 | [diff] [blame] | 402 | union kvm_mmu_role mmu_role; |
Paolo Bonzini | ae1e2d1 | 2017-03-30 11:55:30 +0200 | [diff] [blame] | 403 | u8 root_level; |
| 404 | u8 shadow_root_level; |
| 405 | u8 ept_ad; |
Joerg Roedel | c5a78f2b | 2010-09-10 17:30:39 +0200 | [diff] [blame] | 406 | bool direct_map; |
Junaid Shahid | b94742c | 2018-06-27 14:59:20 -0700 | [diff] [blame] | 407 | struct kvm_mmu_root_info prev_roots[KVM_MMU_NUM_PREV_ROOTS]; |
Zhang Xiantao | d657a98 | 2007-12-14 09:41:22 +0800 | [diff] [blame] | 408 | |
Avi Kivity | 97d64b7 | 2012-09-12 14:52:00 +0300 | [diff] [blame] | 409 | /* |
| 410 | * Bitmap; bit set = permission fault |
| 411 | * Byte index: page fault error code [4:1] |
| 412 | * Bit index: pte permissions in ACC_* format |
| 413 | */ |
| 414 | u8 permissions[16]; |
| 415 | |
Huaitong Han | 2d34410 | 2016-03-22 16:51:19 +0800 | [diff] [blame] | 416 | /* |
| 417 | * The pkru_mask indicates if protection key checks are needed. It |
| 418 | * consists of 16 domains indexed by page fault error code bits [4:1], |
| 419 | * with PFEC.RSVD replaced by ACC_USER_MASK from the page tables. |
| 420 | * Each domain has 2 bits which are ANDed with AD and WD from PKRU. |
| 421 | */ |
| 422 | u32 pkru_mask; |
| 423 | |
Zhang Xiantao | d657a98 | 2007-12-14 09:41:22 +0800 | [diff] [blame] | 424 | u64 *pae_root; |
Joerg Roedel | 81407ca | 2010-09-10 17:31:00 +0200 | [diff] [blame] | 425 | u64 *lm_root; |
Xiao Guangrong | c258b62 | 2015-08-05 12:04:24 +0800 | [diff] [blame] | 426 | |
| 427 | /* |
| 428 | * check zero bits on shadow page table entries, these |
| 429 | * bits include not only hardware reserved bits but also |
| 430 | * the bits spte never used. |
| 431 | */ |
| 432 | struct rsvd_bits_validate shadow_zero_check; |
| 433 | |
Xiao Guangrong | a0a64f5 | 2015-08-05 12:04:21 +0800 | [diff] [blame] | 434 | struct rsvd_bits_validate guest_rsvd_check; |
Joerg Roedel | ff03a07 | 2010-09-10 17:30:57 +0200 | [diff] [blame] | 435 | |
Paolo Bonzini | 6bb69c9 | 2016-02-23 12:51:19 +0100 | [diff] [blame] | 436 | /* Can have large pages at levels 2..last_nonleaf_level-1. */ |
| 437 | u8 last_nonleaf_level; |
Avi Kivity | 6fd01b7 | 2012-09-12 20:46:56 +0300 | [diff] [blame] | 438 | |
Joerg Roedel | 2d48a98 | 2010-09-10 17:31:01 +0200 | [diff] [blame] | 439 | bool nx; |
| 440 | |
Joerg Roedel | ff03a07 | 2010-09-10 17:30:57 +0200 | [diff] [blame] | 441 | u64 pdptrs[4]; /* pae */ |
Zhang Xiantao | d657a98 | 2007-12-14 09:41:22 +0800 | [diff] [blame] | 442 | }; |
| 443 | |
Lan Tianyu | a49b963 | 2018-12-06 21:21:04 +0800 | [diff] [blame] | 444 | struct kvm_tlb_range { |
| 445 | u64 start_gfn; |
| 446 | u64 pages; |
| 447 | }; |
| 448 | |
Gleb Natapov | f5132b0 | 2011-11-10 14:57:22 +0200 | [diff] [blame] | 449 | enum pmc_type { |
| 450 | KVM_PMC_GP = 0, |
| 451 | KVM_PMC_FIXED, |
| 452 | }; |
| 453 | |
| 454 | struct kvm_pmc { |
| 455 | enum pmc_type type; |
| 456 | u8 idx; |
| 457 | u64 counter; |
| 458 | u64 eventsel; |
| 459 | struct perf_event *perf_event; |
| 460 | struct kvm_vcpu *vcpu; |
| 461 | }; |
| 462 | |
| 463 | struct kvm_pmu { |
| 464 | unsigned nr_arch_gp_counters; |
| 465 | unsigned nr_arch_fixed_counters; |
| 466 | unsigned available_event_types; |
| 467 | u64 fixed_ctr_ctrl; |
| 468 | u64 global_ctrl; |
| 469 | u64 global_status; |
| 470 | u64 global_ovf_ctrl; |
| 471 | u64 counter_bitmask[2]; |
| 472 | u64 global_ctrl_mask; |
Andi Kleen | 103af0a | 2013-07-18 15:57:02 -0700 | [diff] [blame] | 473 | u64 reserved_bits; |
Gleb Natapov | f5132b0 | 2011-11-10 14:57:22 +0200 | [diff] [blame] | 474 | u8 version; |
Robert Richter | 15c7ad5 | 2012-06-20 20:46:33 +0200 | [diff] [blame] | 475 | struct kvm_pmc gp_counters[INTEL_PMC_MAX_GENERIC]; |
| 476 | struct kvm_pmc fixed_counters[INTEL_PMC_MAX_FIXED]; |
Gleb Natapov | f5132b0 | 2011-11-10 14:57:22 +0200 | [diff] [blame] | 477 | struct irq_work irq_work; |
| 478 | u64 reprogram_pmi; |
| 479 | }; |
| 480 | |
Wei Huang | 25462f7 | 2015-06-19 15:45:05 +0200 | [diff] [blame] | 481 | struct kvm_pmu_ops; |
| 482 | |
Paolo Bonzini | 360b948 | 2014-02-21 09:55:56 +0100 | [diff] [blame] | 483 | enum { |
| 484 | KVM_DEBUGREG_BP_ENABLED = 1, |
Paolo Bonzini | c77fb5f | 2014-02-21 10:17:24 +0100 | [diff] [blame] | 485 | KVM_DEBUGREG_WONT_EXIT = 2, |
Nadav Amit | ae561ed | 2015-04-02 03:10:37 +0300 | [diff] [blame] | 486 | KVM_DEBUGREG_RELOAD = 4, |
Paolo Bonzini | 360b948 | 2014-02-21 09:55:56 +0100 | [diff] [blame] | 487 | }; |
| 488 | |
Xiao Guangrong | 86fd527 | 2015-06-15 16:55:27 +0800 | [diff] [blame] | 489 | struct kvm_mtrr_range { |
| 490 | u64 base; |
| 491 | u64 mask; |
Xiao Guangrong | 19efffa | 2015-06-15 16:55:31 +0800 | [diff] [blame] | 492 | struct list_head node; |
Xiao Guangrong | 86fd527 | 2015-06-15 16:55:27 +0800 | [diff] [blame] | 493 | }; |
| 494 | |
Xiao Guangrong | 70109e7 | 2015-06-15 16:55:24 +0800 | [diff] [blame] | 495 | struct kvm_mtrr { |
Xiao Guangrong | 86fd527 | 2015-06-15 16:55:27 +0800 | [diff] [blame] | 496 | struct kvm_mtrr_range var_ranges[KVM_NR_VAR_MTRR]; |
Xiao Guangrong | 70109e7 | 2015-06-15 16:55:24 +0800 | [diff] [blame] | 497 | mtrr_type fixed_ranges[KVM_NR_FIXED_MTRR_REGION]; |
Xiao Guangrong | 10fac2d | 2015-06-15 16:55:26 +0800 | [diff] [blame] | 498 | u64 deftype; |
Xiao Guangrong | 19efffa | 2015-06-15 16:55:31 +0800 | [diff] [blame] | 499 | |
| 500 | struct list_head head; |
Xiao Guangrong | 70109e7 | 2015-06-15 16:55:24 +0800 | [diff] [blame] | 501 | }; |
| 502 | |
Andrey Smetanin | 1f4b34f | 2015-11-30 19:22:21 +0300 | [diff] [blame] | 503 | /* Hyper-V SynIC timer */ |
| 504 | struct kvm_vcpu_hv_stimer { |
| 505 | struct hrtimer timer; |
| 506 | int index; |
Vitaly Kuznetsov | 6a058a1e | 2018-11-26 16:47:30 +0100 | [diff] [blame] | 507 | union hv_stimer_config config; |
Andrey Smetanin | 1f4b34f | 2015-11-30 19:22:21 +0300 | [diff] [blame] | 508 | u64 count; |
| 509 | u64 exp_time; |
| 510 | struct hv_message msg; |
| 511 | bool msg_pending; |
| 512 | }; |
| 513 | |
Andrey Smetanin | 5c919412 | 2015-11-10 15:36:34 +0300 | [diff] [blame] | 514 | /* Hyper-V synthetic interrupt controller (SynIC)*/ |
| 515 | struct kvm_vcpu_hv_synic { |
| 516 | u64 version; |
| 517 | u64 control; |
| 518 | u64 msg_page; |
| 519 | u64 evt_page; |
| 520 | atomic64_t sint[HV_SYNIC_SINT_COUNT]; |
| 521 | atomic_t sint_to_gsi[HV_SYNIC_SINT_COUNT]; |
| 522 | DECLARE_BITMAP(auto_eoi_bitmap, 256); |
| 523 | DECLARE_BITMAP(vec_bitmap, 256); |
| 524 | bool active; |
Roman Kagan | efc479e | 2017-06-22 16:51:01 +0300 | [diff] [blame] | 525 | bool dont_zero_synic_pages; |
Andrey Smetanin | 5c919412 | 2015-11-10 15:36:34 +0300 | [diff] [blame] | 526 | }; |
| 527 | |
Andrey Smetanin | e83d588 | 2015-07-03 15:01:34 +0300 | [diff] [blame] | 528 | /* Hyper-V per vcpu emulation context */ |
| 529 | struct kvm_vcpu_hv { |
Roman Kagan | d3457c8 | 2017-07-14 17:13:20 +0300 | [diff] [blame] | 530 | u32 vp_index; |
Andrey Smetanin | e83d588 | 2015-07-03 15:01:34 +0300 | [diff] [blame] | 531 | u64 hv_vapic; |
Andrey Smetanin | 9eec50b | 2015-09-16 12:29:50 +0300 | [diff] [blame] | 532 | s64 runtime_offset; |
Andrey Smetanin | 5c919412 | 2015-11-10 15:36:34 +0300 | [diff] [blame] | 533 | struct kvm_vcpu_hv_synic synic; |
Andrey Smetanin | db397571 | 2015-11-10 15:36:35 +0300 | [diff] [blame] | 534 | struct kvm_hyperv_exit exit; |
Andrey Smetanin | 1f4b34f | 2015-11-30 19:22:21 +0300 | [diff] [blame] | 535 | struct kvm_vcpu_hv_stimer stimer[HV_SYNIC_STIMER_COUNT]; |
| 536 | DECLARE_BITMAP(stimer_pending_bitmap, HV_SYNIC_STIMER_COUNT); |
Vitaly Kuznetsov | e6b6c48 | 2018-10-08 19:19:04 +0200 | [diff] [blame] | 537 | cpumask_t tlb_flush; |
Andrey Smetanin | e83d588 | 2015-07-03 15:01:34 +0300 | [diff] [blame] | 538 | }; |
| 539 | |
Zhang Xiantao | ad312c7 | 2007-12-13 23:50:52 +0800 | [diff] [blame] | 540 | struct kvm_vcpu_arch { |
Marcelo Tosatti | 5fdbf97 | 2008-06-27 14:58:02 -0300 | [diff] [blame] | 541 | /* |
| 542 | * rip and regs accesses must go through |
| 543 | * kvm_{register,rip}_{read,write} functions. |
| 544 | */ |
| 545 | unsigned long regs[NR_VCPU_REGS]; |
| 546 | u32 regs_avail; |
| 547 | u32 regs_dirty; |
Zhang Xiantao | 34c16ee | 2007-10-20 15:34:38 +0800 | [diff] [blame] | 548 | |
| 549 | unsigned long cr0; |
Avi Kivity | e8467fd | 2009-12-29 18:43:06 +0200 | [diff] [blame] | 550 | unsigned long cr0_guest_owned_bits; |
Zhang Xiantao | 34c16ee | 2007-10-20 15:34:38 +0800 | [diff] [blame] | 551 | unsigned long cr2; |
| 552 | unsigned long cr3; |
| 553 | unsigned long cr4; |
Avi Kivity | fc78f51 | 2009-12-07 12:16:48 +0200 | [diff] [blame] | 554 | unsigned long cr4_guest_owned_bits; |
Zhang Xiantao | 34c16ee | 2007-10-20 15:34:38 +0800 | [diff] [blame] | 555 | unsigned long cr8; |
Paolo Bonzini | b9dd21e | 2017-08-23 23:14:38 +0200 | [diff] [blame] | 556 | u32 pkru; |
Alexander Graf | 1371d90 | 2008-11-25 20:17:04 +0100 | [diff] [blame] | 557 | u32 hflags; |
Avi Kivity | f6801df | 2010-01-21 15:31:50 +0200 | [diff] [blame] | 558 | u64 efer; |
Zhang Xiantao | 34c16ee | 2007-10-20 15:34:38 +0800 | [diff] [blame] | 559 | u64 apic_base; |
| 560 | struct kvm_lapic *apic; /* kernel irqchip context */ |
Andrey Smetanin | d62caab | 2015-11-10 15:36:33 +0300 | [diff] [blame] | 561 | bool apicv_active; |
Liran Alon | e40ff1d | 2018-03-21 02:50:31 +0200 | [diff] [blame] | 562 | bool load_eoi_exitmap_pending; |
Andrey Smetanin | 6308630 | 2015-11-10 15:36:32 +0300 | [diff] [blame] | 563 | DECLARE_BITMAP(ioapic_handled_vectors, 256); |
Gleb Natapov | 4138377 | 2012-04-19 14:06:29 +0300 | [diff] [blame] | 564 | unsigned long apic_attention; |
Gleb Natapov | e103571 | 2009-03-05 16:34:59 +0200 | [diff] [blame] | 565 | int32_t apic_arb_prio; |
Zhang Xiantao | 34c16ee | 2007-10-20 15:34:38 +0800 | [diff] [blame] | 566 | int mp_state; |
Zhang Xiantao | 34c16ee | 2007-10-20 15:34:38 +0800 | [diff] [blame] | 567 | u64 ia32_misc_enable_msr; |
Paolo Bonzini | 64d6067 | 2015-05-07 11:36:11 +0200 | [diff] [blame] | 568 | u64 smbase; |
Liran Alon | 52797bf | 2017-11-15 13:43:14 +0200 | [diff] [blame] | 569 | u64 smi_count; |
Avi Kivity | b209749f | 2007-10-22 16:50:39 +0200 | [diff] [blame] | 570 | bool tpr_access_reporting; |
Wanpeng Li | 2030009 | 2014-12-02 19:14:59 +0800 | [diff] [blame] | 571 | u64 ia32_xss; |
Wanpeng Li | 518e7b9 | 2018-02-28 14:03:31 +0800 | [diff] [blame] | 572 | u64 microcode_version; |
Zhang Xiantao | 34c16ee | 2007-10-20 15:34:38 +0800 | [diff] [blame] | 573 | |
Joerg Roedel | 14dfe85 | 2010-09-10 17:30:49 +0200 | [diff] [blame] | 574 | /* |
| 575 | * Paging state of the vcpu |
| 576 | * |
| 577 | * If the vcpu runs in guest mode with two level paging this still saves |
| 578 | * the paging mode of the l1 guest. This context is always used to |
| 579 | * handle faults. |
| 580 | */ |
Vitaly Kuznetsov | 44dd3ff | 2018-10-08 21:28:05 +0200 | [diff] [blame] | 581 | struct kvm_mmu *mmu; |
| 582 | |
| 583 | /* Non-nested MMU for L1 */ |
| 584 | struct kvm_mmu root_mmu; |
Joerg Roedel | 8df25a3 | 2010-09-10 17:30:46 +0200 | [diff] [blame] | 585 | |
Vitaly Kuznetsov | 14c07ad | 2018-10-08 21:28:08 +0200 | [diff] [blame] | 586 | /* L1 MMU when running nested */ |
| 587 | struct kvm_mmu guest_mmu; |
| 588 | |
Joerg Roedel | 8df25a3 | 2010-09-10 17:30:46 +0200 | [diff] [blame] | 589 | /* |
Joerg Roedel | 6539e73 | 2010-09-10 17:30:50 +0200 | [diff] [blame] | 590 | * Paging state of an L2 guest (used for nested npt) |
| 591 | * |
| 592 | * This context will save all necessary information to walk page tables |
| 593 | * of the an L2 guest. This context is only initialized for page table |
| 594 | * walking and not for faulting since we never handle l2 page faults on |
| 595 | * the host. |
| 596 | */ |
| 597 | struct kvm_mmu nested_mmu; |
| 598 | |
| 599 | /* |
Joerg Roedel | 14dfe85 | 2010-09-10 17:30:49 +0200 | [diff] [blame] | 600 | * Pointer to the mmu context currently used for |
| 601 | * gva_to_gpa translations. |
| 602 | */ |
| 603 | struct kvm_mmu *walk_mmu; |
| 604 | |
Xiao Guangrong | 53c07b1 | 2011-05-15 23:26:20 +0800 | [diff] [blame] | 605 | struct kvm_mmu_memory_cache mmu_pte_list_desc_cache; |
Zhang Xiantao | 34c16ee | 2007-10-20 15:34:38 +0800 | [diff] [blame] | 606 | struct kvm_mmu_memory_cache mmu_page_cache; |
| 607 | struct kvm_mmu_memory_cache mmu_page_header_cache; |
| 608 | |
Rik van Riel | f775b13 | 2017-11-14 16:54:23 -0500 | [diff] [blame] | 609 | /* |
| 610 | * QEMU userspace and the guest each have their own FPU state. |
Marc Orr | 240c35a | 2018-11-06 14:53:55 -0800 | [diff] [blame] | 611 | * In vcpu_run, we switch between the user, maintained in the |
| 612 | * task_struct struct, and guest FPU contexts. While running a VCPU, |
| 613 | * the VCPU thread will have the guest FPU context. |
Rik van Riel | f775b13 | 2017-11-14 16:54:23 -0500 | [diff] [blame] | 614 | * |
| 615 | * Note that while the PKRU state lives inside the fpu registers, |
| 616 | * it is switched out separately at VMENTER and VMEXIT time. The |
| 617 | * "guest_fpu" state here contains the guest FPU context, with the |
| 618 | * host PRKU bits. |
| 619 | */ |
Marc Orr | b666a4b | 2018-11-06 14:53:56 -0800 | [diff] [blame] | 620 | struct fpu *guest_fpu; |
Rik van Riel | f775b13 | 2017-11-14 16:54:23 -0500 | [diff] [blame] | 621 | |
Dexuan Cui | 2acf923 | 2010-06-10 11:27:12 +0800 | [diff] [blame] | 622 | u64 xcr0; |
Paolo Bonzini | d7876f1 | 2013-10-02 16:06:15 +0200 | [diff] [blame] | 623 | u64 guest_supported_xcr0; |
Paolo Bonzini | 4344ee9 | 2013-10-02 16:06:16 +0200 | [diff] [blame] | 624 | u32 guest_xstate_size; |
Zhang Xiantao | 34c16ee | 2007-10-20 15:34:38 +0800 | [diff] [blame] | 625 | |
Zhang Xiantao | 34c16ee | 2007-10-20 15:34:38 +0800 | [diff] [blame] | 626 | struct kvm_pio_request pio; |
| 627 | void *pio_data; |
| 628 | |
Gleb Natapov | 66fd3f7 | 2009-05-11 13:35:50 +0300 | [diff] [blame] | 629 | u8 event_exit_inst_len; |
| 630 | |
Avi Kivity | 298101d | 2007-11-25 13:41:11 +0200 | [diff] [blame] | 631 | struct kvm_queued_exception { |
| 632 | bool pending; |
Wanpeng Li | 664f8e2 | 2017-08-24 03:35:09 -0700 | [diff] [blame] | 633 | bool injected; |
Avi Kivity | 298101d | 2007-11-25 13:41:11 +0200 | [diff] [blame] | 634 | bool has_error_code; |
| 635 | u8 nr; |
| 636 | u32 error_code; |
Jim Mattson | c851436 | 2018-10-16 14:29:19 -0700 | [diff] [blame] | 637 | unsigned long payload; |
| 638 | bool has_payload; |
Wanpeng Li | adfe20f | 2017-07-13 18:30:41 -0700 | [diff] [blame] | 639 | u8 nested_apf; |
Avi Kivity | 298101d | 2007-11-25 13:41:11 +0200 | [diff] [blame] | 640 | } exception; |
| 641 | |
Avi Kivity | 937a7ea | 2008-07-03 15:17:01 +0300 | [diff] [blame] | 642 | struct kvm_queued_interrupt { |
Liran Alon | 04140b4 | 2018-03-23 03:01:31 +0300 | [diff] [blame] | 643 | bool injected; |
Gleb Natapov | 66fd3f7 | 2009-05-11 13:35:50 +0300 | [diff] [blame] | 644 | bool soft; |
Avi Kivity | 937a7ea | 2008-07-03 15:17:01 +0300 | [diff] [blame] | 645 | u8 nr; |
| 646 | } interrupt; |
| 647 | |
Zhang Xiantao | 34c16ee | 2007-10-20 15:34:38 +0800 | [diff] [blame] | 648 | int halt_request; /* real mode on Intel only */ |
| 649 | |
| 650 | int cpuid_nent; |
Dan Kenigsberg | 0771671 | 2007-11-21 17:10:04 +0200 | [diff] [blame] | 651 | struct kvm_cpuid_entry2 cpuid_entries[KVM_MAX_CPUID_ENTRIES]; |
Eugene Korenevsky | 5a4f55c | 2015-03-29 23:56:12 +0300 | [diff] [blame] | 652 | |
| 653 | int maxphyaddr; |
| 654 | |
Zhang Xiantao | 34c16ee | 2007-10-20 15:34:38 +0800 | [diff] [blame] | 655 | /* emulate context */ |
| 656 | |
| 657 | struct x86_emulate_ctxt emulate_ctxt; |
Gleb Natapov | 7ae441e | 2011-03-31 12:06:41 +0200 | [diff] [blame] | 658 | bool emulate_regs_need_sync_to_vcpu; |
| 659 | bool emulate_regs_need_sync_from_vcpu; |
Gleb Natapov | 716d51a | 2012-09-03 15:24:26 +0300 | [diff] [blame] | 660 | int (*complete_userspace_io)(struct kvm_vcpu *vcpu); |
Glauber de Oliveira Costa | 1806852 | 2008-02-15 17:52:47 -0200 | [diff] [blame] | 661 | |
| 662 | gpa_t time; |
Gerd Hoffmann | 50d0a0f | 2008-06-03 16:17:31 +0200 | [diff] [blame] | 663 | struct pvclock_vcpu_time_info hv_clock; |
Zachary Amsden | e48672f | 2010-08-19 22:07:23 -1000 | [diff] [blame] | 664 | unsigned int hw_tsc_khz; |
Andy Honig | 0b79459 | 2013-02-20 14:48:10 -0800 | [diff] [blame] | 665 | struct gfn_to_hva_cache pv_time; |
| 666 | bool pv_time_enabled; |
Marcelo Tosatti | 51d59c6 | 2012-08-03 15:57:49 -0300 | [diff] [blame] | 667 | /* set guest stopped flag in pvclock flags field */ |
| 668 | bool pvclock_set_guest_stopped_request; |
Glauber Costa | c9aaa89 | 2011-07-11 15:28:14 -0400 | [diff] [blame] | 669 | |
| 670 | struct { |
| 671 | u64 msr_val; |
| 672 | u64 last_steal; |
Glauber Costa | c9aaa89 | 2011-07-11 15:28:14 -0400 | [diff] [blame] | 673 | struct gfn_to_hva_cache stime; |
| 674 | struct kvm_steal_time steal; |
| 675 | } st; |
| 676 | |
Luiz Capitulino | a545ab6 | 2016-09-07 14:47:19 -0400 | [diff] [blame] | 677 | u64 tsc_offset; |
Zachary Amsden | 1d5f066 | 2010-08-19 22:07:30 -1000 | [diff] [blame] | 678 | u64 last_guest_tsc; |
Zachary Amsden | 6f526ec | 2012-02-03 15:43:54 -0200 | [diff] [blame] | 679 | u64 last_host_tsc; |
Zachary Amsden | 0dd6a6e | 2012-02-03 15:43:56 -0200 | [diff] [blame] | 680 | u64 tsc_offset_adjustment; |
Zachary Amsden | e26101b | 2012-02-03 15:43:57 -0200 | [diff] [blame] | 681 | u64 this_tsc_nsec; |
| 682 | u64 this_tsc_write; |
Tomasz Grabiec | 0d3da0d | 2014-06-24 09:42:43 +0200 | [diff] [blame] | 683 | u64 this_tsc_generation; |
Zachary Amsden | c285545 | 2010-09-18 14:38:15 -1000 | [diff] [blame] | 684 | bool tsc_catchup; |
Zachary Amsden | cc57828 | 2012-02-03 15:43:50 -0200 | [diff] [blame] | 685 | bool tsc_always_catchup; |
| 686 | s8 virtual_tsc_shift; |
| 687 | u32 virtual_tsc_mult; |
| 688 | u32 virtual_tsc_khz; |
Will Auld | ba90463 | 2012-11-29 12:42:50 -0800 | [diff] [blame] | 689 | s64 ia32_tsc_adjust_msr; |
Haozhong Zhang | ad721883 | 2015-10-20 15:39:02 +0800 | [diff] [blame] | 690 | u64 tsc_scaling_ratio; |
Sheng Yang | 3419ffc | 2008-05-15 09:52:48 +0800 | [diff] [blame] | 691 | |
Avi Kivity | 7460fb4a | 2011-09-20 13:43:14 +0300 | [diff] [blame] | 692 | atomic_t nmi_queued; /* unprocessed asynchronous NMIs */ |
| 693 | unsigned nmi_pending; /* NMI queued after currently running handler */ |
| 694 | bool nmi_injected; /* Trying to inject an NMI this entry */ |
Paolo Bonzini | f077825 | 2015-04-01 15:06:40 +0200 | [diff] [blame] | 695 | bool smi_pending; /* SMI queued after currently running handler */ |
Avi Kivity | 9ba075a | 2008-05-26 20:06:35 +0300 | [diff] [blame] | 696 | |
Xiao Guangrong | 70109e7 | 2015-06-15 16:55:24 +0800 | [diff] [blame] | 697 | struct kvm_mtrr mtrr_state; |
Paolo Bonzini | 7cb060a | 2014-06-19 11:40:18 +0200 | [diff] [blame] | 698 | u64 pat; |
Jan Kiszka | 42dbaa5 | 2008-12-15 13:52:10 +0100 | [diff] [blame] | 699 | |
Paolo Bonzini | 360b948 | 2014-02-21 09:55:56 +0100 | [diff] [blame] | 700 | unsigned switch_db_regs; |
Jan Kiszka | 42dbaa5 | 2008-12-15 13:52:10 +0100 | [diff] [blame] | 701 | unsigned long db[KVM_NR_DB_REGS]; |
| 702 | unsigned long dr6; |
| 703 | unsigned long dr7; |
| 704 | unsigned long eff_db[KVM_NR_DB_REGS]; |
Jan Kiszka | c863901 | 2012-09-21 05:42:55 +0200 | [diff] [blame] | 705 | unsigned long guest_debug_dr7; |
Kyle Huey | db2336a | 2017-03-20 01:16:28 -0700 | [diff] [blame] | 706 | u64 msr_platform_info; |
| 707 | u64 msr_misc_features_enables; |
Huang Ying | 890ca9a | 2009-05-11 16:48:15 +0800 | [diff] [blame] | 708 | |
| 709 | u64 mcg_cap; |
| 710 | u64 mcg_status; |
| 711 | u64 mcg_ctl; |
Ashok Raj | c45dcc7 | 2016-06-22 14:59:56 +0800 | [diff] [blame] | 712 | u64 mcg_ext_ctl; |
Huang Ying | 890ca9a | 2009-05-11 16:48:15 +0800 | [diff] [blame] | 713 | u64 *mce_banks; |
Jan Kiszka | 94fe45d | 2009-10-18 13:24:44 +0200 | [diff] [blame] | 714 | |
Xiao Guangrong | bebb106 | 2011-07-12 03:23:20 +0800 | [diff] [blame] | 715 | /* Cache MMIO info */ |
| 716 | u64 mmio_gva; |
| 717 | unsigned access; |
| 718 | gfn_t mmio_gfn; |
David Matlack | 56f17dd | 2014-08-18 15:46:07 -0700 | [diff] [blame] | 719 | u64 mmio_gen; |
Xiao Guangrong | bebb106 | 2011-07-12 03:23:20 +0800 | [diff] [blame] | 720 | |
Gleb Natapov | f5132b0 | 2011-11-10 14:57:22 +0200 | [diff] [blame] | 721 | struct kvm_pmu pmu; |
| 722 | |
Jan Kiszka | 94fe45d | 2009-10-18 13:24:44 +0200 | [diff] [blame] | 723 | /* used for guest single stepping over the given code position */ |
Jan Kiszka | 94fe45d | 2009-10-18 13:24:44 +0200 | [diff] [blame] | 724 | unsigned long singlestep_rip; |
Jan Kiszka | f92653e | 2010-02-23 17:47:55 +0100 | [diff] [blame] | 725 | |
Andrey Smetanin | e83d588 | 2015-07-03 15:01:34 +0300 | [diff] [blame] | 726 | struct kvm_vcpu_hv hyperv; |
Sheng Yang | f5f48ee | 2010-06-30 12:25:15 +0800 | [diff] [blame] | 727 | |
| 728 | cpumask_var_t wbinvd_dirty_mask; |
Gleb Natapov | af585b9 | 2010-10-14 11:22:46 +0200 | [diff] [blame] | 729 | |
Xiao Guangrong | 1cb3f3a | 2011-09-22 17:02:48 +0800 | [diff] [blame] | 730 | unsigned long last_retry_eip; |
| 731 | unsigned long last_retry_addr; |
| 732 | |
Gleb Natapov | af585b9 | 2010-10-14 11:22:46 +0200 | [diff] [blame] | 733 | struct { |
| 734 | bool halted; |
| 735 | gfn_t gfns[roundup_pow_of_two(ASYNC_PF_PER_VCPU)]; |
Gleb Natapov | 344d958 | 2010-10-14 11:22:50 +0200 | [diff] [blame] | 736 | struct gfn_to_hva_cache data; |
| 737 | u64 msr_val; |
Gleb Natapov | 7c90705 | 2010-10-14 11:22:53 +0200 | [diff] [blame] | 738 | u32 id; |
Gleb Natapov | 6adba52 | 2010-10-14 11:22:55 +0200 | [diff] [blame] | 739 | bool send_user_only; |
Wanpeng Li | 1261bfa | 2017-07-13 18:30:40 -0700 | [diff] [blame] | 740 | u32 host_apf_reason; |
Wanpeng Li | adfe20f | 2017-07-13 18:30:41 -0700 | [diff] [blame] | 741 | unsigned long nested_apf_token; |
Wanpeng Li | 52a5c15 | 2017-07-13 18:30:42 -0700 | [diff] [blame] | 742 | bool delivery_as_pf_vmexit; |
Gleb Natapov | af585b9 | 2010-10-14 11:22:46 +0200 | [diff] [blame] | 743 | } apf; |
Boris Ostrovsky | 2b036c6 | 2012-01-09 14:00:35 -0500 | [diff] [blame] | 744 | |
| 745 | /* OSVW MSRs (AMD only) */ |
| 746 | struct { |
| 747 | u64 length; |
| 748 | u64 status; |
| 749 | } osvw; |
Michael S. Tsirkin | ae7a2a3 | 2012-06-24 19:25:07 +0300 | [diff] [blame] | 750 | |
| 751 | struct { |
| 752 | u64 msr_val; |
| 753 | struct gfn_to_hva_cache data; |
| 754 | } pv_eoi; |
Xiao Guangrong | 93c05d3 | 2013-01-13 23:49:07 +0800 | [diff] [blame] | 755 | |
| 756 | /* |
| 757 | * Indicate whether the access faults on its page table in guest |
| 758 | * which is set when fix page fault and used to detect unhandeable |
| 759 | * instruction. |
| 760 | */ |
| 761 | bool write_fault_to_shadow_pgtable; |
Yang Zhang | 25d9208 | 2013-08-06 12:00:32 +0300 | [diff] [blame] | 762 | |
| 763 | /* set at EPT violation at this point */ |
| 764 | unsigned long exit_qualification; |
Srivatsa Vaddagiri | 6aef266 | 2013-08-26 14:18:34 +0530 | [diff] [blame] | 765 | |
| 766 | /* pv related host specific info */ |
| 767 | struct { |
| 768 | bool pv_unhalted; |
| 769 | } pv; |
Steve Rutherford | 7543a63 | 2015-07-29 23:21:41 -0700 | [diff] [blame] | 770 | |
| 771 | int pending_ioapic_eoi; |
Steve Rutherford | 1c1a9ce | 2015-07-30 11:27:16 +0200 | [diff] [blame] | 772 | int pending_external_vector; |
Tom Lendacky | 0f89b20 | 2016-12-14 14:59:23 -0500 | [diff] [blame] | 773 | |
Brijesh Singh | 618232e | 2017-08-17 18:36:57 +0200 | [diff] [blame] | 774 | /* GPA available */ |
Tom Lendacky | 0f89b20 | 2016-12-14 14:59:23 -0500 | [diff] [blame] | 775 | bool gpa_available; |
Brijesh Singh | 618232e | 2017-08-17 18:36:57 +0200 | [diff] [blame] | 776 | gpa_t gpa_val; |
Longpeng(Mike) | de63ad4 | 2017-08-08 12:05:33 +0800 | [diff] [blame] | 777 | |
| 778 | /* be preempted when it's in kernel-mode(cpl=0) */ |
| 779 | bool preempted_in_kernel; |
Paolo Bonzini | c595cee | 2018-07-02 13:07:14 +0200 | [diff] [blame] | 780 | |
| 781 | /* Flush the L1 Data cache for L1TF mitigation on VMENTER */ |
| 782 | bool l1tf_flush_l1d; |
Zhang Xiantao | 34c16ee | 2007-10-20 15:34:38 +0800 | [diff] [blame] | 783 | }; |
| 784 | |
Takuya Yoshikawa | db3fe4e | 2012-02-08 13:02:18 +0900 | [diff] [blame] | 785 | struct kvm_lpage_info { |
Xiao Guangrong | 92f94f1 | 2016-02-24 17:51:06 +0800 | [diff] [blame] | 786 | int disallow_lpage; |
Takuya Yoshikawa | db3fe4e | 2012-02-08 13:02:18 +0900 | [diff] [blame] | 787 | }; |
| 788 | |
| 789 | struct kvm_arch_memory_slot { |
Takuya Yoshikawa | 018aabb5 | 2015-11-20 17:41:28 +0900 | [diff] [blame] | 790 | struct kvm_rmap_head *rmap[KVM_NR_PAGE_SIZES]; |
Takuya Yoshikawa | db3fe4e | 2012-02-08 13:02:18 +0900 | [diff] [blame] | 791 | struct kvm_lpage_info *lpage_info[KVM_NR_PAGE_SIZES - 1]; |
Xiao Guangrong | 21ebbed | 2016-02-24 17:51:09 +0800 | [diff] [blame] | 792 | unsigned short *gfn_track[KVM_PAGE_TRACK_MAX]; |
Takuya Yoshikawa | db3fe4e | 2012-02-08 13:02:18 +0900 | [diff] [blame] | 793 | }; |
| 794 | |
Radim Krčmář | 3548a25 | 2015-02-12 19:41:33 +0100 | [diff] [blame] | 795 | /* |
| 796 | * We use as the mode the number of bits allocated in the LDR for the |
| 797 | * logical processor ID. It happens that these are all powers of two. |
| 798 | * This makes it is very easy to detect cases where the APICs are |
| 799 | * configured for multiple modes; in that case, we cannot use the map and |
| 800 | * hence cannot use kvm_irq_delivery_to_apic_fast either. |
| 801 | */ |
| 802 | #define KVM_APIC_MODE_XAPIC_CLUSTER 4 |
| 803 | #define KVM_APIC_MODE_XAPIC_FLAT 8 |
| 804 | #define KVM_APIC_MODE_X2APIC 16 |
| 805 | |
Gleb Natapov | 1e08ec4 | 2012-09-13 17:19:24 +0300 | [diff] [blame] | 806 | struct kvm_apic_map { |
| 807 | struct rcu_head rcu; |
Radim Krčmář | 3548a25 | 2015-02-12 19:41:33 +0100 | [diff] [blame] | 808 | u8 mode; |
Radim Krčmář | 0ca52e7 | 2016-07-12 22:09:20 +0200 | [diff] [blame] | 809 | u32 max_apic_id; |
Radim Krčmář | e45115b | 2016-07-12 22:09:19 +0200 | [diff] [blame] | 810 | union { |
| 811 | struct kvm_lapic *xapic_flat_map[8]; |
| 812 | struct kvm_lapic *xapic_cluster_map[16][4]; |
| 813 | }; |
Radim Krčmář | 0ca52e7 | 2016-07-12 22:09:20 +0200 | [diff] [blame] | 814 | struct kvm_lapic *phys_map[]; |
Gleb Natapov | 1e08ec4 | 2012-09-13 17:19:24 +0300 | [diff] [blame] | 815 | }; |
| 816 | |
Andrey Smetanin | e83d588 | 2015-07-03 15:01:34 +0300 | [diff] [blame] | 817 | /* Hyper-V emulation context */ |
| 818 | struct kvm_hv { |
Paolo Bonzini | 3f5ad8b | 2016-12-12 10:12:53 +0100 | [diff] [blame] | 819 | struct mutex hv_lock; |
Andrey Smetanin | e83d588 | 2015-07-03 15:01:34 +0300 | [diff] [blame] | 820 | u64 hv_guest_os_id; |
| 821 | u64 hv_hypercall; |
| 822 | u64 hv_tsc_page; |
Andrey Smetanin | e7d9513 | 2015-07-03 15:01:37 +0300 | [diff] [blame] | 823 | |
| 824 | /* Hyper-v based guest crash (NT kernel bugcheck) parameters */ |
| 825 | u64 hv_crash_param[HV_X64_MSR_CRASH_PARAMS]; |
| 826 | u64 hv_crash_ctl; |
Paolo Bonzini | 095cf55 | 2016-02-08 12:54:12 +0100 | [diff] [blame] | 827 | |
| 828 | HV_REFERENCE_TSC_PAGE tsc_ref; |
Roman Kagan | faeb783 | 2018-02-01 16:48:32 +0300 | [diff] [blame] | 829 | |
| 830 | struct idr conn_to_evt; |
Vitaly Kuznetsov | a2e164e | 2018-03-01 15:15:12 +0100 | [diff] [blame] | 831 | |
| 832 | u64 hv_reenlightenment_control; |
| 833 | u64 hv_tsc_emulation_control; |
| 834 | u64 hv_tsc_emulation_status; |
Vitaly Kuznetsov | 87ee613 | 2018-09-26 19:02:56 +0200 | [diff] [blame] | 835 | |
| 836 | /* How many vCPUs have VP index != vCPU index */ |
| 837 | atomic_t num_mismatched_vp_indexes; |
Andrey Smetanin | e83d588 | 2015-07-03 15:01:34 +0300 | [diff] [blame] | 838 | }; |
| 839 | |
Radim Krčmář | 49776fa | 2016-12-16 16:10:02 +0100 | [diff] [blame] | 840 | enum kvm_irqchip_mode { |
| 841 | KVM_IRQCHIP_NONE, |
| 842 | KVM_IRQCHIP_KERNEL, /* created with KVM_CREATE_IRQCHIP */ |
| 843 | KVM_IRQCHIP_SPLIT, /* created with KVM_CAP_SPLIT_IRQCHIP */ |
| 844 | }; |
| 845 | |
Marcelo Tosatti | fef9cce | 2009-12-23 14:35:17 -0200 | [diff] [blame] | 846 | struct kvm_arch { |
Dave Hansen | 49d5ca2 | 2010-08-19 18:11:28 -0700 | [diff] [blame] | 847 | unsigned int n_used_mmu_pages; |
Zhang Xiantao | f05e70a | 2007-12-14 10:01:48 +0800 | [diff] [blame] | 848 | unsigned int n_requested_mmu_pages; |
Dave Hansen | 39de71e | 2010-08-19 18:11:14 -0700 | [diff] [blame] | 849 | unsigned int n_max_mmu_pages; |
Xiao Guangrong | 332b207 | 2011-05-15 23:20:27 +0800 | [diff] [blame] | 850 | unsigned int indirect_shadow_pages; |
Xiao Guangrong | 5304b8d | 2013-05-31 08:36:22 +0800 | [diff] [blame] | 851 | unsigned long mmu_valid_gen; |
Zhang Xiantao | f05e70a | 2007-12-14 10:01:48 +0800 | [diff] [blame] | 852 | struct hlist_head mmu_page_hash[KVM_NUM_MMU_PAGES]; |
| 853 | /* |
| 854 | * Hash table of struct kvm_mmu_page. |
| 855 | */ |
| 856 | struct list_head active_mmu_pages; |
Xiao Guangrong | 365c886 | 2013-05-31 08:36:29 +0800 | [diff] [blame] | 857 | struct list_head zapped_obsolete_pages; |
Xiao Guangrong | 13d268c | 2016-02-24 17:51:16 +0800 | [diff] [blame] | 858 | struct kvm_page_track_notifier_node mmu_sp_tracker; |
Xiao Guangrong | 0eb05bf | 2016-02-24 17:51:13 +0800 | [diff] [blame] | 859 | struct kvm_page_track_notifier_head track_notifier_head; |
Xiao Guangrong | 365c886 | 2013-05-31 08:36:29 +0800 | [diff] [blame] | 860 | |
Ben-Ami Yassour | 4d5c5d0 | 2008-07-28 19:26:26 +0300 | [diff] [blame] | 861 | struct list_head assigned_dev_head; |
Joerg Roedel | 19de40a | 2008-12-03 14:43:34 +0100 | [diff] [blame] | 862 | struct iommu_domain *iommu_domain; |
Alex Williamson | d96eb2c | 2013-10-30 11:02:23 -0600 | [diff] [blame] | 863 | bool iommu_noncoherent; |
Alex Williamson | e0f0bbc | 2013-10-30 11:02:30 -0600 | [diff] [blame] | 864 | #define __KVM_HAVE_ARCH_NONCOHERENT_DMA |
| 865 | atomic_t noncoherent_dma_count; |
Paolo Bonzini | 5544eb9 | 2015-07-07 15:41:58 +0200 | [diff] [blame] | 866 | #define __KVM_HAVE_ARCH_ASSIGNED_DEVICE |
| 867 | atomic_t assigned_device_count; |
Zhang Xiantao | d7deeeb0 | 2007-12-14 10:17:34 +0800 | [diff] [blame] | 868 | struct kvm_pic *vpic; |
| 869 | struct kvm_ioapic *vioapic; |
Sheng Yang | 7837699 | 2008-01-28 05:10:22 +0800 | [diff] [blame] | 870 | struct kvm_pit *vpit; |
Radim Krčmář | 4272013 | 2015-07-01 15:31:49 +0200 | [diff] [blame] | 871 | atomic_t vapics_in_nmi_mode; |
Gleb Natapov | 1e08ec4 | 2012-09-13 17:19:24 +0300 | [diff] [blame] | 872 | struct mutex apic_map_lock; |
| 873 | struct kvm_apic_map *apic_map; |
Zhang Xiantao | bfc6d22 | 2007-12-14 10:20:16 +0800 | [diff] [blame] | 874 | |
Tang Chen | c24ae0d | 2014-09-24 15:57:58 +0800 | [diff] [blame] | 875 | bool apic_access_page_done; |
Glauber de Oliveira Costa | 1806852 | 2008-02-15 17:52:47 -0200 | [diff] [blame] | 876 | |
| 877 | gpa_t wall_clock; |
Sheng Yang | b7ebfb0 | 2008-04-25 21:44:52 +0800 | [diff] [blame] | 878 | |
Wanpeng Li | 4d5422c | 2018-03-12 04:53:02 -0700 | [diff] [blame] | 879 | bool mwait_in_guest; |
Wanpeng Li | caa057a | 2018-03-12 04:53:03 -0700 | [diff] [blame] | 880 | bool hlt_in_guest; |
Wanpeng Li | b31c114 | 2018-03-12 04:53:04 -0700 | [diff] [blame] | 881 | bool pause_in_guest; |
Wanpeng Li | 4d5422c | 2018-03-12 04:53:02 -0700 | [diff] [blame] | 882 | |
Sheng Yang | 5550af4 | 2008-10-15 20:15:06 +0800 | [diff] [blame] | 883 | unsigned long irq_sources_bitmap; |
Glauber Costa | afbcf7a | 2009-10-16 15:28:36 -0400 | [diff] [blame] | 884 | s64 kvmclock_offset; |
Jan Kiszka | 038f8c1 | 2011-02-04 10:49:11 +0100 | [diff] [blame] | 885 | raw_spinlock_t tsc_write_lock; |
Zachary Amsden | f38e098 | 2010-08-19 22:07:20 -1000 | [diff] [blame] | 886 | u64 last_tsc_nsec; |
Zachary Amsden | f38e098 | 2010-08-19 22:07:20 -1000 | [diff] [blame] | 887 | u64 last_tsc_write; |
Zachary Amsden | 5d3cb0f6 | 2012-02-03 15:43:51 -0200 | [diff] [blame] | 888 | u32 last_tsc_khz; |
Zachary Amsden | e26101b | 2012-02-03 15:43:57 -0200 | [diff] [blame] | 889 | u64 cur_tsc_nsec; |
| 890 | u64 cur_tsc_write; |
| 891 | u64 cur_tsc_offset; |
Tomasz Grabiec | 0d3da0d | 2014-06-24 09:42:43 +0200 | [diff] [blame] | 892 | u64 cur_tsc_generation; |
Marcelo Tosatti | b48aa97 | 2012-11-27 23:29:03 -0200 | [diff] [blame] | 893 | int nr_vcpus_matched_tsc; |
Ed Swierk | ffde22a | 2009-10-15 15:21:43 -0700 | [diff] [blame] | 894 | |
Marcelo Tosatti | d828199 | 2012-11-27 23:29:01 -0200 | [diff] [blame] | 895 | spinlock_t pvclock_gtod_sync_lock; |
| 896 | bool use_master_clock; |
| 897 | u64 master_kernel_ns; |
Thomas Gleixner | a5a1d1c | 2016-12-21 20:32:01 +0100 | [diff] [blame] | 898 | u64 master_cycle_now; |
Andrew Jones | 7e44e44 | 2014-02-28 12:52:54 +0100 | [diff] [blame] | 899 | struct delayed_work kvmclock_update_work; |
Andrew Jones | 332967a | 2014-02-28 12:52:55 +0100 | [diff] [blame] | 900 | struct delayed_work kvmclock_sync_work; |
Marcelo Tosatti | d828199 | 2012-11-27 23:29:01 -0200 | [diff] [blame] | 901 | |
Ed Swierk | ffde22a | 2009-10-15 15:21:43 -0700 | [diff] [blame] | 902 | struct kvm_xen_hvm_config xen_hvm_config; |
Gleb Natapov | 55cd8e5 | 2010-01-17 15:51:22 +0200 | [diff] [blame] | 903 | |
Paolo Bonzini | 6ef768f | 2014-11-20 13:45:31 +0100 | [diff] [blame] | 904 | /* reads protected by irq_srcu, writes by irq_lock */ |
| 905 | struct hlist_head mask_notifier_list; |
| 906 | |
Andrey Smetanin | e83d588 | 2015-07-03 15:01:34 +0300 | [diff] [blame] | 907 | struct kvm_hv hyperv; |
Xiao Guangrong | b034cf0 | 2010-12-23 16:08:35 +0800 | [diff] [blame] | 908 | |
| 909 | #ifdef CONFIG_KVM_MMU_AUDIT |
| 910 | int audit_point; |
| 911 | #endif |
Marcelo Tosatti | 54750f2 | 2015-01-20 15:54:52 -0200 | [diff] [blame] | 912 | |
Ladi Prosek | a826faf | 2017-06-26 09:56:43 +0200 | [diff] [blame] | 913 | bool backwards_tsc_observed; |
Marcelo Tosatti | 54750f2 | 2015-01-20 15:54:52 -0200 | [diff] [blame] | 914 | bool boot_vcpu_runs_old_kvmclock; |
Paolo Bonzini | d71ba78 | 2015-07-29 11:56:48 +0200 | [diff] [blame] | 915 | u32 bsp_vcpu_id; |
Nadav Amit | 90de4a1 | 2015-04-13 01:53:41 +0300 | [diff] [blame] | 916 | |
| 917 | u64 disabled_quirks; |
Steve Rutherford | 49df639 | 2015-07-29 23:21:40 -0700 | [diff] [blame] | 918 | |
Radim Krčmář | 49776fa | 2016-12-16 16:10:02 +0100 | [diff] [blame] | 919 | enum kvm_irqchip_mode irqchip_mode; |
Steve Rutherford | b053b2a | 2015-07-29 23:32:35 -0700 | [diff] [blame] | 920 | u8 nr_reserved_ioapic_pins; |
Feng Wu | 52004014 | 2016-01-25 16:53:33 +0800 | [diff] [blame] | 921 | |
| 922 | bool disabled_lapic_found; |
Suravee Suthikulpanit | 44a95da | 2016-05-04 14:09:46 -0500 | [diff] [blame] | 923 | |
Radim Krčmář | 37131313 | 2016-07-12 22:09:27 +0200 | [diff] [blame] | 924 | bool x2apic_format; |
Radim Krčmář | c519265 | 2016-07-12 22:09:28 +0200 | [diff] [blame] | 925 | bool x2apic_broadcast_quirk_disabled; |
Drew Schmitt | 6fbbde9 | 2018-08-20 10:32:15 -0700 | [diff] [blame] | 926 | |
| 927 | bool guest_can_read_msr_platform_info; |
Jim Mattson | 59073aa | 2018-10-16 14:29:20 -0700 | [diff] [blame] | 928 | bool exception_payload_enabled; |
Zhang Xiantao | d69fb81 | 2007-12-14 09:54:20 +0800 | [diff] [blame] | 929 | }; |
| 930 | |
Zhang Xiantao | 0711456 | 2007-12-14 10:23:23 +0800 | [diff] [blame] | 931 | struct kvm_vm_stat { |
Suraj Jitindar Singh | 8a7e75d | 2016-08-02 14:03:22 +1000 | [diff] [blame] | 932 | ulong mmu_shadow_zapped; |
| 933 | ulong mmu_pte_write; |
| 934 | ulong mmu_pte_updated; |
| 935 | ulong mmu_pde_zapped; |
| 936 | ulong mmu_flooded; |
| 937 | ulong mmu_recycled; |
| 938 | ulong mmu_cache_miss; |
| 939 | ulong mmu_unsync; |
| 940 | ulong remote_tlb_flush; |
| 941 | ulong lpages; |
David Matlack | f3414bc | 2016-12-20 15:25:57 -0800 | [diff] [blame] | 942 | ulong max_mmu_page_hash_collisions; |
Zhang Xiantao | 0711456 | 2007-12-14 10:23:23 +0800 | [diff] [blame] | 943 | }; |
| 944 | |
Zhang Xiantao | 77b4c25 | 2007-12-14 09:49:26 +0800 | [diff] [blame] | 945 | struct kvm_vcpu_stat { |
Suraj Jitindar Singh | 8a7e75d | 2016-08-02 14:03:22 +1000 | [diff] [blame] | 946 | u64 pf_fixed; |
| 947 | u64 pf_guest; |
| 948 | u64 tlb_flush; |
| 949 | u64 invlpg; |
Zhang Xiantao | 77b4c25 | 2007-12-14 09:49:26 +0800 | [diff] [blame] | 950 | |
Suraj Jitindar Singh | 8a7e75d | 2016-08-02 14:03:22 +1000 | [diff] [blame] | 951 | u64 exits; |
| 952 | u64 io_exits; |
| 953 | u64 mmio_exits; |
| 954 | u64 signal_exits; |
| 955 | u64 irq_window_exits; |
| 956 | u64 nmi_window_exits; |
Paolo Bonzini | c595cee | 2018-07-02 13:07:14 +0200 | [diff] [blame] | 957 | u64 l1d_flush; |
Suraj Jitindar Singh | 8a7e75d | 2016-08-02 14:03:22 +1000 | [diff] [blame] | 958 | u64 halt_exits; |
| 959 | u64 halt_successful_poll; |
| 960 | u64 halt_attempted_poll; |
| 961 | u64 halt_poll_invalid; |
| 962 | u64 halt_wakeup; |
| 963 | u64 request_irq_exits; |
| 964 | u64 irq_exits; |
| 965 | u64 host_state_reload; |
Suraj Jitindar Singh | 8a7e75d | 2016-08-02 14:03:22 +1000 | [diff] [blame] | 966 | u64 fpu_reload; |
| 967 | u64 insn_emulation; |
| 968 | u64 insn_emulation_fail; |
| 969 | u64 hypercalls; |
| 970 | u64 irq_injections; |
| 971 | u64 nmi_injections; |
Paolo Bonzini | 0f1e261 | 2016-12-17 16:05:19 +0100 | [diff] [blame] | 972 | u64 req_event; |
Zhang Xiantao | 77b4c25 | 2007-12-14 09:49:26 +0800 | [diff] [blame] | 973 | }; |
Zhang Xiantao | ad312c7 | 2007-12-13 23:50:52 +0800 | [diff] [blame] | 974 | |
Joerg Roedel | 8a76d7f | 2011-04-04 12:39:27 +0200 | [diff] [blame] | 975 | struct x86_instruction_info; |
| 976 | |
Will Auld | 8fe8ab4 | 2012-11-29 12:42:12 -0800 | [diff] [blame] | 977 | struct msr_data { |
| 978 | bool host_initiated; |
| 979 | u32 index; |
| 980 | u64 data; |
| 981 | }; |
| 982 | |
Paolo Bonzini | cb5281a | 2014-12-17 18:17:20 +0100 | [diff] [blame] | 983 | struct kvm_lapic_irq { |
| 984 | u32 vector; |
Paolo Bonzini | b7cb223 | 2015-04-21 14:57:05 +0200 | [diff] [blame] | 985 | u16 delivery_mode; |
| 986 | u16 dest_mode; |
| 987 | bool level; |
| 988 | u16 trig_mode; |
Paolo Bonzini | cb5281a | 2014-12-17 18:17:20 +0100 | [diff] [blame] | 989 | u32 shorthand; |
| 990 | u32 dest_id; |
James Sullivan | 93bbf0b | 2015-03-18 19:26:03 -0600 | [diff] [blame] | 991 | bool msi_redir_hint; |
Paolo Bonzini | cb5281a | 2014-12-17 18:17:20 +0100 | [diff] [blame] | 992 | }; |
| 993 | |
Zhang Xiantao | ea4a5ff | 2007-11-19 14:40:47 +0800 | [diff] [blame] | 994 | struct kvm_x86_ops { |
| 995 | int (*cpu_has_kvm_support)(void); /* __init */ |
| 996 | int (*disabled_by_bios)(void); /* __init */ |
Radim Krčmář | 13a34e0 | 2014-08-28 15:13:03 +0200 | [diff] [blame] | 997 | int (*hardware_enable)(void); |
| 998 | void (*hardware_disable)(void); |
Zhang Xiantao | ea4a5ff | 2007-11-19 14:40:47 +0800 | [diff] [blame] | 999 | void (*check_processor_compatibility)(void *rtn); |
| 1000 | int (*hardware_setup)(void); /* __init */ |
| 1001 | void (*hardware_unsetup)(void); /* __exit */ |
Avi Kivity | 774ead3 | 2007-12-26 13:57:04 +0200 | [diff] [blame] | 1002 | bool (*cpu_has_accelerated_tpr)(void); |
Tom Lendacky | bc226f0 | 2018-05-10 22:06:39 +0200 | [diff] [blame] | 1003 | bool (*has_emulated_msr)(int index); |
Sheng Yang | 0e85188 | 2009-12-18 16:48:46 +0800 | [diff] [blame] | 1004 | void (*cpuid_update)(struct kvm_vcpu *vcpu); |
Zhang Xiantao | ea4a5ff | 2007-11-19 14:40:47 +0800 | [diff] [blame] | 1005 | |
Sean Christopherson | 434a1e9 | 2018-03-20 12:17:18 -0700 | [diff] [blame] | 1006 | struct kvm *(*vm_alloc)(void); |
| 1007 | void (*vm_free)(struct kvm *); |
Suravee Suthikulpanit | 0354313 | 2016-05-04 14:09:42 -0500 | [diff] [blame] | 1008 | int (*vm_init)(struct kvm *kvm); |
| 1009 | void (*vm_destroy)(struct kvm *kvm); |
| 1010 | |
Zhang Xiantao | ea4a5ff | 2007-11-19 14:40:47 +0800 | [diff] [blame] | 1011 | /* Create, but do not attach this VCPU */ |
| 1012 | struct kvm_vcpu *(*vcpu_create)(struct kvm *kvm, unsigned id); |
| 1013 | void (*vcpu_free)(struct kvm_vcpu *vcpu); |
Nadav Amit | d28bc9d | 2015-04-13 14:34:08 +0300 | [diff] [blame] | 1014 | void (*vcpu_reset)(struct kvm_vcpu *vcpu, bool init_event); |
Zhang Xiantao | ea4a5ff | 2007-11-19 14:40:47 +0800 | [diff] [blame] | 1015 | |
| 1016 | void (*prepare_guest_switch)(struct kvm_vcpu *vcpu); |
| 1017 | void (*vcpu_load)(struct kvm_vcpu *vcpu, int cpu); |
| 1018 | void (*vcpu_put)(struct kvm_vcpu *vcpu); |
Zhang Xiantao | ea4a5ff | 2007-11-19 14:40:47 +0800 | [diff] [blame] | 1019 | |
Paolo Bonzini | a96036b | 2015-11-10 11:55:36 +0100 | [diff] [blame] | 1020 | void (*update_bp_intercept)(struct kvm_vcpu *vcpu); |
Paolo Bonzini | 609e36d | 2015-04-08 15:30:38 +0200 | [diff] [blame] | 1021 | int (*get_msr)(struct kvm_vcpu *vcpu, struct msr_data *msr); |
Will Auld | 8fe8ab4 | 2012-11-29 12:42:12 -0800 | [diff] [blame] | 1022 | int (*set_msr)(struct kvm_vcpu *vcpu, struct msr_data *msr); |
Zhang Xiantao | ea4a5ff | 2007-11-19 14:40:47 +0800 | [diff] [blame] | 1023 | u64 (*get_segment_base)(struct kvm_vcpu *vcpu, int seg); |
| 1024 | void (*get_segment)(struct kvm_vcpu *vcpu, |
| 1025 | struct kvm_segment *var, int seg); |
Izik Eidus | 2e4d265 | 2008-03-24 19:38:34 +0200 | [diff] [blame] | 1026 | int (*get_cpl)(struct kvm_vcpu *vcpu); |
Zhang Xiantao | ea4a5ff | 2007-11-19 14:40:47 +0800 | [diff] [blame] | 1027 | void (*set_segment)(struct kvm_vcpu *vcpu, |
| 1028 | struct kvm_segment *var, int seg); |
| 1029 | void (*get_cs_db_l_bits)(struct kvm_vcpu *vcpu, int *db, int *l); |
Avi Kivity | e8467fd | 2009-12-29 18:43:06 +0200 | [diff] [blame] | 1030 | void (*decache_cr0_guest_bits)(struct kvm_vcpu *vcpu); |
Avi Kivity | aff48ba | 2010-12-05 18:56:11 +0200 | [diff] [blame] | 1031 | void (*decache_cr3)(struct kvm_vcpu *vcpu); |
Zhang Xiantao | ea4a5ff | 2007-11-19 14:40:47 +0800 | [diff] [blame] | 1032 | void (*decache_cr4_guest_bits)(struct kvm_vcpu *vcpu); |
| 1033 | void (*set_cr0)(struct kvm_vcpu *vcpu, unsigned long cr0); |
| 1034 | void (*set_cr3)(struct kvm_vcpu *vcpu, unsigned long cr3); |
Nadav Har'El | 5e1746d | 2011-05-25 23:03:24 +0300 | [diff] [blame] | 1035 | int (*set_cr4)(struct kvm_vcpu *vcpu, unsigned long cr4); |
Zhang Xiantao | ea4a5ff | 2007-11-19 14:40:47 +0800 | [diff] [blame] | 1036 | void (*set_efer)(struct kvm_vcpu *vcpu, u64 efer); |
Gleb Natapov | 89a27f4 | 2010-02-16 10:51:48 +0200 | [diff] [blame] | 1037 | void (*get_idt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt); |
| 1038 | void (*set_idt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt); |
| 1039 | void (*get_gdt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt); |
| 1040 | void (*set_gdt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt); |
Jan Kiszka | 73aaf249e | 2014-01-04 18:47:16 +0100 | [diff] [blame] | 1041 | u64 (*get_dr6)(struct kvm_vcpu *vcpu); |
| 1042 | void (*set_dr6)(struct kvm_vcpu *vcpu, unsigned long value); |
Paolo Bonzini | c77fb5f | 2014-02-21 10:17:24 +0100 | [diff] [blame] | 1043 | void (*sync_dirty_debug_regs)(struct kvm_vcpu *vcpu); |
Gleb Natapov | 020df07 | 2010-04-13 10:05:23 +0300 | [diff] [blame] | 1044 | void (*set_dr7)(struct kvm_vcpu *vcpu, unsigned long value); |
Marcelo Tosatti | 5fdbf97 | 2008-06-27 14:58:02 -0300 | [diff] [blame] | 1045 | void (*cache_reg)(struct kvm_vcpu *vcpu, enum kvm_reg reg); |
Zhang Xiantao | ea4a5ff | 2007-11-19 14:40:47 +0800 | [diff] [blame] | 1046 | unsigned long (*get_rflags)(struct kvm_vcpu *vcpu); |
| 1047 | void (*set_rflags)(struct kvm_vcpu *vcpu, unsigned long rflags); |
| 1048 | |
Wanpeng Li | c2ba05c | 2017-12-12 17:33:03 -0800 | [diff] [blame] | 1049 | void (*tlb_flush)(struct kvm_vcpu *vcpu, bool invalidate_gpa); |
Tianyu Lan | b08660e | 2018-07-19 08:40:17 +0000 | [diff] [blame] | 1050 | int (*tlb_remote_flush)(struct kvm *kvm); |
Lan Tianyu | a49b963 | 2018-12-06 21:21:04 +0800 | [diff] [blame] | 1051 | int (*tlb_remote_flush_with_range)(struct kvm *kvm, |
| 1052 | struct kvm_tlb_range *range); |
Zhang Xiantao | ea4a5ff | 2007-11-19 14:40:47 +0800 | [diff] [blame] | 1053 | |
Junaid Shahid | faff875 | 2018-06-29 13:10:05 -0700 | [diff] [blame] | 1054 | /* |
| 1055 | * Flush any TLB entries associated with the given GVA. |
| 1056 | * Does not need to flush GPA->HPA mappings. |
| 1057 | * Can potentially get non-canonical addresses through INVLPGs, which |
| 1058 | * the implementation may choose to ignore if appropriate. |
| 1059 | */ |
| 1060 | void (*tlb_flush_gva)(struct kvm_vcpu *vcpu, gva_t addr); |
Zhang Xiantao | ea4a5ff | 2007-11-19 14:40:47 +0800 | [diff] [blame] | 1061 | |
Avi Kivity | 851ba69 | 2009-08-24 11:10:17 +0300 | [diff] [blame] | 1062 | void (*run)(struct kvm_vcpu *vcpu); |
| 1063 | int (*handle_exit)(struct kvm_vcpu *vcpu); |
Zhang Xiantao | ea4a5ff | 2007-11-19 14:40:47 +0800 | [diff] [blame] | 1064 | void (*skip_emulated_instruction)(struct kvm_vcpu *vcpu); |
Glauber Costa | 2809f5d | 2009-05-12 16:21:05 -0400 | [diff] [blame] | 1065 | void (*set_interrupt_shadow)(struct kvm_vcpu *vcpu, int mask); |
Paolo Bonzini | 37ccdcb | 2014-05-20 14:29:47 +0200 | [diff] [blame] | 1066 | u32 (*get_interrupt_shadow)(struct kvm_vcpu *vcpu); |
Zhang Xiantao | ea4a5ff | 2007-11-19 14:40:47 +0800 | [diff] [blame] | 1067 | void (*patch_hypercall)(struct kvm_vcpu *vcpu, |
| 1068 | unsigned char *hypercall_addr); |
Gleb Natapov | 66fd3f7 | 2009-05-11 13:35:50 +0300 | [diff] [blame] | 1069 | void (*set_irq)(struct kvm_vcpu *vcpu); |
Gleb Natapov | 95ba827313 | 2009-04-21 17:45:08 +0300 | [diff] [blame] | 1070 | void (*set_nmi)(struct kvm_vcpu *vcpu); |
Wanpeng Li | cfcd20e | 2017-07-13 18:30:39 -0700 | [diff] [blame] | 1071 | void (*queue_exception)(struct kvm_vcpu *vcpu); |
Avi Kivity | b463a6f | 2010-07-20 15:06:17 +0300 | [diff] [blame] | 1072 | void (*cancel_injection)(struct kvm_vcpu *vcpu); |
Gleb Natapov | 7864612 | 2009-03-23 12:12:11 +0200 | [diff] [blame] | 1073 | int (*interrupt_allowed)(struct kvm_vcpu *vcpu); |
Gleb Natapov | 95ba827313 | 2009-04-21 17:45:08 +0300 | [diff] [blame] | 1074 | int (*nmi_allowed)(struct kvm_vcpu *vcpu); |
Jan Kiszka | 3cfc309 | 2009-11-12 01:04:25 +0100 | [diff] [blame] | 1075 | bool (*get_nmi_mask)(struct kvm_vcpu *vcpu); |
| 1076 | void (*set_nmi_mask)(struct kvm_vcpu *vcpu, bool masked); |
Jan Kiszka | c9a7953 | 2014-03-07 20:03:15 +0100 | [diff] [blame] | 1077 | void (*enable_nmi_window)(struct kvm_vcpu *vcpu); |
| 1078 | void (*enable_irq_window)(struct kvm_vcpu *vcpu); |
Gleb Natapov | 95ba827313 | 2009-04-21 17:45:08 +0300 | [diff] [blame] | 1079 | void (*update_cr8_intercept)(struct kvm_vcpu *vcpu, int tpr, int irr); |
Suravee Suthikulpanit | b2a05fe | 2017-09-12 10:42:41 -0500 | [diff] [blame] | 1080 | bool (*get_enable_apicv)(struct kvm_vcpu *vcpu); |
Andrey Smetanin | d62caab | 2015-11-10 15:36:33 +0300 | [diff] [blame] | 1081 | void (*refresh_apicv_exec_ctrl)(struct kvm_vcpu *vcpu); |
Yang Zhang | c7c9c56 | 2013-01-25 10:18:51 +0800 | [diff] [blame] | 1082 | void (*hwapic_irr_update)(struct kvm_vcpu *vcpu, int max_irr); |
Paolo Bonzini | 67c9ddd | 2016-05-10 17:01:23 +0200 | [diff] [blame] | 1083 | void (*hwapic_isr_update)(struct kvm_vcpu *vcpu, int isr); |
Liran Alon | e6c67d8 | 2018-09-04 10:56:52 +0300 | [diff] [blame] | 1084 | bool (*guest_apic_has_interrupt)(struct kvm_vcpu *vcpu); |
Andrey Smetanin | 6308630 | 2015-11-10 15:36:32 +0300 | [diff] [blame] | 1085 | void (*load_eoi_exitmap)(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap); |
Jim Mattson | 8d860bb | 2018-05-09 16:56:05 -0400 | [diff] [blame] | 1086 | void (*set_virtual_apic_mode)(struct kvm_vcpu *vcpu); |
Tang Chen | 4256f43 | 2014-09-24 15:57:54 +0800 | [diff] [blame] | 1087 | void (*set_apic_access_page_addr)(struct kvm_vcpu *vcpu, hpa_t hpa); |
Yang Zhang | a20ed54 | 2013-04-11 19:25:15 +0800 | [diff] [blame] | 1088 | void (*deliver_posted_interrupt)(struct kvm_vcpu *vcpu, int vector); |
Paolo Bonzini | 76dfafd5 | 2016-12-19 17:17:11 +0100 | [diff] [blame] | 1089 | int (*sync_pir_to_irr)(struct kvm_vcpu *vcpu); |
Zhang Xiantao | ea4a5ff | 2007-11-19 14:40:47 +0800 | [diff] [blame] | 1090 | int (*set_tss_addr)(struct kvm *kvm, unsigned int addr); |
Sean Christopherson | 2ac52ab | 2018-03-20 12:17:19 -0700 | [diff] [blame] | 1091 | int (*set_identity_map_addr)(struct kvm *kvm, u64 ident_addr); |
Yu Zhang | 855feb6 | 2017-08-24 20:27:55 +0800 | [diff] [blame] | 1092 | int (*get_tdp_level)(struct kvm_vcpu *vcpu); |
Sheng Yang | 4b12f0d | 2009-04-27 20:35:42 +0800 | [diff] [blame] | 1093 | u64 (*get_mt_mask)(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio); |
Sheng Yang | 17cc393 | 2010-01-05 19:02:27 +0800 | [diff] [blame] | 1094 | int (*get_lpage_level)(void); |
Sheng Yang | 4e47c7a | 2009-12-18 16:48:47 +0800 | [diff] [blame] | 1095 | bool (*rdtscp_supported)(void); |
Mao, Junjie | ad756a1 | 2012-07-02 01:18:48 +0000 | [diff] [blame] | 1096 | bool (*invpcid_supported)(void); |
Joerg Roedel | 344f414 | 2009-07-27 16:30:48 +0200 | [diff] [blame] | 1097 | |
Joerg Roedel | 1c97f0a | 2010-09-10 17:30:41 +0200 | [diff] [blame] | 1098 | void (*set_tdp_cr3)(struct kvm_vcpu *vcpu, unsigned long cr3); |
| 1099 | |
Joerg Roedel | d4330ef | 2010-04-22 12:33:11 +0200 | [diff] [blame] | 1100 | void (*set_supported_cpuid)(u32 func, struct kvm_cpuid_entry2 *entry); |
| 1101 | |
Sheng Yang | f5f48ee | 2010-06-30 12:25:15 +0800 | [diff] [blame] | 1102 | bool (*has_wbinvd_exit)(void); |
| 1103 | |
KarimAllah Ahmed | e79f245 | 2018-04-14 05:10:52 +0200 | [diff] [blame] | 1104 | u64 (*read_l1_tsc_offset)(struct kvm_vcpu *vcpu); |
Leonid Shatz | 326e742 | 2018-11-06 12:14:25 +0200 | [diff] [blame] | 1105 | /* Returns actual tsc_offset set in active VMCS */ |
| 1106 | u64 (*write_l1_tsc_offset)(struct kvm_vcpu *vcpu, u64 offset); |
Zachary Amsden | 99e3e30 | 2010-08-19 22:07:17 -1000 | [diff] [blame] | 1107 | |
Avi Kivity | 586f960 | 2010-11-18 13:09:54 +0200 | [diff] [blame] | 1108 | void (*get_exit_info)(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2); |
Joerg Roedel | 8a76d7f | 2011-04-04 12:39:27 +0200 | [diff] [blame] | 1109 | |
| 1110 | int (*check_intercept)(struct kvm_vcpu *vcpu, |
| 1111 | struct x86_instruction_info *info, |
| 1112 | enum x86_intercept_stage stage); |
Yang Zhang | a547c6d | 2013-04-11 19:25:10 +0800 | [diff] [blame] | 1113 | void (*handle_external_intr)(struct kvm_vcpu *vcpu); |
Liu, Jinsong | da8999d | 2014-02-24 10:55:46 +0000 | [diff] [blame] | 1114 | bool (*mpx_supported)(void); |
Wanpeng Li | 55412b2 | 2014-12-02 19:21:30 +0800 | [diff] [blame] | 1115 | bool (*xsaves_supported)(void); |
Paolo Bonzini | 66336ca | 2016-07-12 10:36:41 +0200 | [diff] [blame] | 1116 | bool (*umip_emulated)(void); |
Chao Peng | 86f5201 | 2018-10-24 16:05:11 +0800 | [diff] [blame] | 1117 | bool (*pt_supported)(void); |
Jan Kiszka | b6b8a14 | 2014-03-07 20:03:12 +0100 | [diff] [blame] | 1118 | |
| 1119 | int (*check_nested_events)(struct kvm_vcpu *vcpu, bool external_intr); |
Sean Christopherson | d264ee0 | 2018-08-27 15:21:12 -0700 | [diff] [blame] | 1120 | void (*request_immediate_exit)(struct kvm_vcpu *vcpu); |
Radim Krčmář | ae97a3b | 2014-08-21 18:08:06 +0200 | [diff] [blame] | 1121 | |
| 1122 | void (*sched_in)(struct kvm_vcpu *kvm, int cpu); |
Kai Huang | 88178fd | 2015-01-28 10:54:27 +0800 | [diff] [blame] | 1123 | |
| 1124 | /* |
| 1125 | * Arch-specific dirty logging hooks. These hooks are only supposed to |
| 1126 | * be valid if the specific arch has hardware-accelerated dirty logging |
| 1127 | * mechanism. Currently only for PML on VMX. |
| 1128 | * |
| 1129 | * - slot_enable_log_dirty: |
| 1130 | * called when enabling log dirty mode for the slot. |
| 1131 | * - slot_disable_log_dirty: |
| 1132 | * called when disabling log dirty mode for the slot. |
| 1133 | * also called when slot is created with log dirty disabled. |
| 1134 | * - flush_log_dirty: |
| 1135 | * called before reporting dirty_bitmap to userspace. |
| 1136 | * - enable_log_dirty_pt_masked: |
| 1137 | * called when reenabling log dirty for the GFNs in the mask after |
| 1138 | * corresponding bits are cleared in slot->dirty_bitmap. |
| 1139 | */ |
| 1140 | void (*slot_enable_log_dirty)(struct kvm *kvm, |
| 1141 | struct kvm_memory_slot *slot); |
| 1142 | void (*slot_disable_log_dirty)(struct kvm *kvm, |
| 1143 | struct kvm_memory_slot *slot); |
| 1144 | void (*flush_log_dirty)(struct kvm *kvm); |
| 1145 | void (*enable_log_dirty_pt_masked)(struct kvm *kvm, |
| 1146 | struct kvm_memory_slot *slot, |
| 1147 | gfn_t offset, unsigned long mask); |
Bandan Das | bab4165 | 2017-05-05 15:25:13 -0400 | [diff] [blame] | 1148 | int (*write_log_dirty)(struct kvm_vcpu *vcpu); |
| 1149 | |
Wei Huang | 25462f7 | 2015-06-19 15:45:05 +0200 | [diff] [blame] | 1150 | /* pmu operations of sub-arch */ |
| 1151 | const struct kvm_pmu_ops *pmu_ops; |
Feng Wu | efc6440 | 2015-09-18 22:29:51 +0800 | [diff] [blame] | 1152 | |
Feng Wu | bf9f6ac | 2015-09-18 22:29:55 +0800 | [diff] [blame] | 1153 | /* |
| 1154 | * Architecture specific hooks for vCPU blocking due to |
| 1155 | * HLT instruction. |
| 1156 | * Returns for .pre_block(): |
| 1157 | * - 0 means continue to block the vCPU. |
| 1158 | * - 1 means we cannot block the vCPU since some event |
| 1159 | * happens during this period, such as, 'ON' bit in |
| 1160 | * posted-interrupts descriptor is set. |
| 1161 | */ |
| 1162 | int (*pre_block)(struct kvm_vcpu *vcpu); |
| 1163 | void (*post_block)(struct kvm_vcpu *vcpu); |
Suravee Suthikulpanit | d1ed092 | 2016-05-04 14:09:43 -0500 | [diff] [blame] | 1164 | |
| 1165 | void (*vcpu_blocking)(struct kvm_vcpu *vcpu); |
| 1166 | void (*vcpu_unblocking)(struct kvm_vcpu *vcpu); |
| 1167 | |
Feng Wu | efc6440 | 2015-09-18 22:29:51 +0800 | [diff] [blame] | 1168 | int (*update_pi_irte)(struct kvm *kvm, unsigned int host_irq, |
| 1169 | uint32_t guest_irq, bool set); |
Suravee Suthikulpanit | be8ca17 | 2016-05-04 14:09:49 -0500 | [diff] [blame] | 1170 | void (*apicv_post_state_restore)(struct kvm_vcpu *vcpu); |
Yunhong Jiang | ce7a058 | 2016-06-13 14:20:01 -0700 | [diff] [blame] | 1171 | |
| 1172 | int (*set_hv_timer)(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc); |
| 1173 | void (*cancel_hv_timer)(struct kvm_vcpu *vcpu); |
Ashok Raj | c45dcc7 | 2016-06-22 14:59:56 +0800 | [diff] [blame] | 1174 | |
| 1175 | void (*setup_mce)(struct kvm_vcpu *vcpu); |
Ladi Prosek | 0234bf8 | 2017-10-11 16:54:40 +0200 | [diff] [blame] | 1176 | |
Jim Mattson | 8fcc4b5 | 2018-07-10 11:27:20 +0200 | [diff] [blame] | 1177 | int (*get_nested_state)(struct kvm_vcpu *vcpu, |
| 1178 | struct kvm_nested_state __user *user_kvm_nested_state, |
| 1179 | unsigned user_data_size); |
| 1180 | int (*set_nested_state)(struct kvm_vcpu *vcpu, |
| 1181 | struct kvm_nested_state __user *user_kvm_nested_state, |
| 1182 | struct kvm_nested_state *kvm_state); |
Paolo Bonzini | 7f7f1ba | 2018-07-18 18:49:01 +0200 | [diff] [blame] | 1183 | void (*get_vmcs12_pages)(struct kvm_vcpu *vcpu); |
| 1184 | |
Ladi Prosek | 72d7b37 | 2017-10-11 16:54:41 +0200 | [diff] [blame] | 1185 | int (*smi_allowed)(struct kvm_vcpu *vcpu); |
Ladi Prosek | 0234bf8 | 2017-10-11 16:54:40 +0200 | [diff] [blame] | 1186 | int (*pre_enter_smm)(struct kvm_vcpu *vcpu, char *smstate); |
| 1187 | int (*pre_leave_smm)(struct kvm_vcpu *vcpu, u64 smbase); |
Ladi Prosek | cc3d967 | 2017-10-17 16:02:39 +0200 | [diff] [blame] | 1188 | int (*enable_smi_window)(struct kvm_vcpu *vcpu); |
Brijesh Singh | 5acc5c0 | 2017-12-04 10:57:26 -0600 | [diff] [blame] | 1189 | |
| 1190 | int (*mem_enc_op)(struct kvm *kvm, void __user *argp); |
Brijesh Singh | 69eaede | 2017-12-04 10:57:26 -0600 | [diff] [blame] | 1191 | int (*mem_enc_reg_region)(struct kvm *kvm, struct kvm_enc_region *argp); |
| 1192 | int (*mem_enc_unreg_region)(struct kvm *kvm, struct kvm_enc_region *argp); |
Tom Lendacky | 801e459 | 2018-02-21 13:39:51 -0600 | [diff] [blame] | 1193 | |
| 1194 | int (*get_msr_feature)(struct kvm_msr_entry *entry); |
Vitaly Kuznetsov | 57b119d | 2018-10-16 18:50:01 +0200 | [diff] [blame] | 1195 | |
| 1196 | int (*nested_enable_evmcs)(struct kvm_vcpu *vcpu, |
| 1197 | uint16_t *vmcs_version); |
Vitaly Kuznetsov | e2e871a | 2018-12-10 18:21:55 +0100 | [diff] [blame] | 1198 | uint16_t (*nested_get_evmcs_version)(struct kvm_vcpu *vcpu); |
Zhang Xiantao | ea4a5ff | 2007-11-19 14:40:47 +0800 | [diff] [blame] | 1199 | }; |
| 1200 | |
Gleb Natapov | af585b9 | 2010-10-14 11:22:46 +0200 | [diff] [blame] | 1201 | struct kvm_arch_async_pf { |
Gleb Natapov | 7c90705 | 2010-10-14 11:22:53 +0200 | [diff] [blame] | 1202 | u32 token; |
Gleb Natapov | af585b9 | 2010-10-14 11:22:46 +0200 | [diff] [blame] | 1203 | gfn_t gfn; |
Xiao Guangrong | fb67e14 | 2010-12-07 10:35:25 +0800 | [diff] [blame] | 1204 | unsigned long cr3; |
Xiao Guangrong | c4806ac | 2010-11-12 14:49:55 +0800 | [diff] [blame] | 1205 | bool direct_map; |
Gleb Natapov | af585b9 | 2010-10-14 11:22:46 +0200 | [diff] [blame] | 1206 | }; |
| 1207 | |
Zhang Xiantao | 97896d0 | 2007-11-14 20:09:30 +0800 | [diff] [blame] | 1208 | extern struct kvm_x86_ops *kvm_x86_ops; |
Marc Orr | b666a4b | 2018-11-06 14:53:56 -0800 | [diff] [blame] | 1209 | extern struct kmem_cache *x86_fpu_cache; |
Zhang Xiantao | 97896d0 | 2007-11-14 20:09:30 +0800 | [diff] [blame] | 1210 | |
Sean Christopherson | 434a1e9 | 2018-03-20 12:17:18 -0700 | [diff] [blame] | 1211 | #define __KVM_HAVE_ARCH_VM_ALLOC |
| 1212 | static inline struct kvm *kvm_arch_alloc_vm(void) |
| 1213 | { |
| 1214 | return kvm_x86_ops->vm_alloc(); |
| 1215 | } |
| 1216 | |
| 1217 | static inline void kvm_arch_free_vm(struct kvm *kvm) |
| 1218 | { |
| 1219 | return kvm_x86_ops->vm_free(kvm); |
| 1220 | } |
| 1221 | |
Tianyu Lan | b08660e | 2018-07-19 08:40:17 +0000 | [diff] [blame] | 1222 | #define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLB |
| 1223 | static inline int kvm_arch_flush_remote_tlb(struct kvm *kvm) |
| 1224 | { |
| 1225 | if (kvm_x86_ops->tlb_remote_flush && |
| 1226 | !kvm_x86_ops->tlb_remote_flush(kvm)) |
| 1227 | return 0; |
| 1228 | else |
| 1229 | return -ENOTSUPP; |
| 1230 | } |
| 1231 | |
Zhang Xiantao | 54f1585 | 2007-11-19 15:24:28 +0800 | [diff] [blame] | 1232 | int kvm_mmu_module_init(void); |
| 1233 | void kvm_mmu_module_exit(void); |
| 1234 | |
| 1235 | void kvm_mmu_destroy(struct kvm_vcpu *vcpu); |
| 1236 | int kvm_mmu_create(struct kvm_vcpu *vcpu); |
Xiao Guangrong | 13d268c | 2016-02-24 17:51:16 +0800 | [diff] [blame] | 1237 | void kvm_mmu_init_vm(struct kvm *kvm); |
| 1238 | void kvm_mmu_uninit_vm(struct kvm *kvm); |
Sheng Yang | 7b52345 | 2008-04-25 21:13:50 +0800 | [diff] [blame] | 1239 | void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask, |
Junaid Shahid | f160c7b | 2016-12-06 16:46:16 -0800 | [diff] [blame] | 1240 | u64 dirty_mask, u64 nx_mask, u64 x_mask, u64 p_mask, |
Tom Lendacky | d0ec49d | 2017-07-17 16:10:27 -0500 | [diff] [blame] | 1241 | u64 acc_track_mask, u64 me_mask); |
Zhang Xiantao | 54f1585 | 2007-11-19 15:24:28 +0800 | [diff] [blame] | 1242 | |
Paolo Bonzini | 8a3c1a33 | 2013-10-02 16:56:13 +0200 | [diff] [blame] | 1243 | void kvm_mmu_reset_context(struct kvm_vcpu *vcpu); |
Kai Huang | 1c91cad4 | 2015-01-28 10:54:26 +0800 | [diff] [blame] | 1244 | void kvm_mmu_slot_remove_write_access(struct kvm *kvm, |
| 1245 | struct kvm_memory_slot *memslot); |
Wanpeng Li | 3ea3b7f | 2015-04-03 15:40:25 +0800 | [diff] [blame] | 1246 | void kvm_mmu_zap_collapsible_sptes(struct kvm *kvm, |
Paolo Bonzini | f36f3f2 | 2015-05-18 13:20:23 +0200 | [diff] [blame] | 1247 | const struct kvm_memory_slot *memslot); |
Kai Huang | f4b4b18 | 2015-01-28 10:54:24 +0800 | [diff] [blame] | 1248 | void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm, |
| 1249 | struct kvm_memory_slot *memslot); |
| 1250 | void kvm_mmu_slot_largepage_remove_write_access(struct kvm *kvm, |
| 1251 | struct kvm_memory_slot *memslot); |
| 1252 | void kvm_mmu_slot_set_dirty(struct kvm *kvm, |
| 1253 | struct kvm_memory_slot *memslot); |
| 1254 | void kvm_mmu_clear_dirty_pt_masked(struct kvm *kvm, |
| 1255 | struct kvm_memory_slot *slot, |
| 1256 | gfn_t gfn_offset, unsigned long mask); |
Zhang Xiantao | 54f1585 | 2007-11-19 15:24:28 +0800 | [diff] [blame] | 1257 | void kvm_mmu_zap_all(struct kvm *kvm); |
Paolo Bonzini | 54bf36a | 2015-04-08 15:39:23 +0200 | [diff] [blame] | 1258 | void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, struct kvm_memslots *slots); |
Zhang Xiantao | 3ad82a7 | 2007-11-20 13:11:38 +0800 | [diff] [blame] | 1259 | unsigned int kvm_mmu_calculate_mmu_pages(struct kvm *kvm); |
Zhang Xiantao | 54f1585 | 2007-11-19 15:24:28 +0800 | [diff] [blame] | 1260 | void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int kvm_nr_mmu_pages); |
| 1261 | |
Joerg Roedel | ff03a07 | 2010-09-10 17:30:57 +0200 | [diff] [blame] | 1262 | int load_pdptrs(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, unsigned long cr3); |
Ladi Prosek | 9ed38ffa | 2016-11-30 16:03:10 +0100 | [diff] [blame] | 1263 | bool pdptrs_changed(struct kvm_vcpu *vcpu); |
Joerg Roedel | cc4b687 | 2008-02-07 13:47:43 +0100 | [diff] [blame] | 1264 | |
Marcelo Tosatti | 3200f40 | 2008-03-29 20:17:59 -0300 | [diff] [blame] | 1265 | int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa, |
Avi Kivity | 9f81128 | 2008-03-02 14:06:05 +0200 | [diff] [blame] | 1266 | const void *val, int bytes); |
Marcelo Tosatti | 2f333bc | 2008-02-22 12:21:37 -0500 | [diff] [blame] | 1267 | |
Paolo Bonzini | 6ef768f | 2014-11-20 13:45:31 +0100 | [diff] [blame] | 1268 | struct kvm_irq_mask_notifier { |
| 1269 | void (*func)(struct kvm_irq_mask_notifier *kimn, bool masked); |
| 1270 | int irq; |
| 1271 | struct hlist_node link; |
| 1272 | }; |
| 1273 | |
| 1274 | void kvm_register_irq_mask_notifier(struct kvm *kvm, int irq, |
| 1275 | struct kvm_irq_mask_notifier *kimn); |
| 1276 | void kvm_unregister_irq_mask_notifier(struct kvm *kvm, int irq, |
| 1277 | struct kvm_irq_mask_notifier *kimn); |
| 1278 | void kvm_fire_mask_notifiers(struct kvm *kvm, unsigned irqchip, unsigned pin, |
| 1279 | bool mask); |
| 1280 | |
Marcelo Tosatti | 2f333bc | 2008-02-22 12:21:37 -0500 | [diff] [blame] | 1281 | extern bool tdp_enabled; |
Avi Kivity | 9f81128 | 2008-03-02 14:06:05 +0200 | [diff] [blame] | 1282 | |
Liu, Jinsong | a3e06bb | 2011-09-22 16:55:52 +0800 | [diff] [blame] | 1283 | u64 vcpu_tsc_khz(struct kvm_vcpu *vcpu); |
| 1284 | |
Joerg Roedel | 92a1f12 | 2011-03-25 09:44:51 +0100 | [diff] [blame] | 1285 | /* control of guest tsc rate supported? */ |
| 1286 | extern bool kvm_has_tsc_control; |
Joerg Roedel | 92a1f12 | 2011-03-25 09:44:51 +0100 | [diff] [blame] | 1287 | /* maximum supported tsc_khz for guests */ |
| 1288 | extern u32 kvm_max_guest_tsc_khz; |
Haozhong Zhang | bc9b961 | 2015-10-20 15:39:01 +0800 | [diff] [blame] | 1289 | /* number of bits of the fractional part of the TSC scaling ratio */ |
| 1290 | extern u8 kvm_tsc_scaling_ratio_frac_bits; |
| 1291 | /* maximum allowed value of TSC scaling ratio */ |
| 1292 | extern u64 kvm_max_tsc_scaling_ratio; |
Yunhong Jiang | 64672c9 | 2016-06-13 14:19:59 -0700 | [diff] [blame] | 1293 | /* 1ull << kvm_tsc_scaling_ratio_frac_bits */ |
| 1294 | extern u64 kvm_default_tsc_scaling_ratio; |
Joerg Roedel | 92a1f12 | 2011-03-25 09:44:51 +0100 | [diff] [blame] | 1295 | |
Ashok Raj | c45dcc7 | 2016-06-22 14:59:56 +0800 | [diff] [blame] | 1296 | extern u64 kvm_mce_cap_supported; |
Sheng Yang | 571008d | 2008-01-02 14:49:22 +0800 | [diff] [blame] | 1297 | |
| 1298 | enum emulation_result { |
Paolo Bonzini | ac0a48c | 2013-06-25 18:24:41 +0200 | [diff] [blame] | 1299 | EMULATE_DONE, /* no further processing */ |
| 1300 | EMULATE_USER_EXIT, /* kvm_run ready for userspace exit */ |
Zhang Xiantao | 54f1585 | 2007-11-19 15:24:28 +0800 | [diff] [blame] | 1301 | EMULATE_FAIL, /* can't emulate this instruction */ |
| 1302 | }; |
| 1303 | |
| 1304 | #define EMULTYPE_NO_DECODE (1 << 0) |
| 1305 | #define EMULTYPE_TRAP_UD (1 << 1) |
Gleb Natapov | ba8afb6 | 2009-04-12 13:36:57 +0300 | [diff] [blame] | 1306 | #define EMULTYPE_SKIP (1 << 2) |
Sean Christopherson | 384bf22 | 2018-08-23 13:56:49 -0700 | [diff] [blame] | 1307 | #define EMULTYPE_ALLOW_RETRY (1 << 3) |
| 1308 | #define EMULTYPE_NO_UD_ON_FAIL (1 << 4) |
| 1309 | #define EMULTYPE_VMWARE (1 << 5) |
Sean Christopherson | c60658d | 2018-08-23 13:56:53 -0700 | [diff] [blame] | 1310 | int kvm_emulate_instruction(struct kvm_vcpu *vcpu, int emulation_type); |
| 1311 | int kvm_emulate_instruction_from_buffer(struct kvm_vcpu *vcpu, |
| 1312 | void *insn, int insn_len); |
Sean Christopherson | 35be0ad | 2018-08-23 13:56:47 -0700 | [diff] [blame] | 1313 | |
Joerg Roedel | f2b4b7d | 2008-01-31 14:57:37 +0100 | [diff] [blame] | 1314 | void kvm_enable_efer_bits(u64); |
Jan Kiszka | 384bb78 | 2013-04-20 10:52:36 +0200 | [diff] [blame] | 1315 | bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer); |
Paolo Bonzini | 609e36d | 2015-04-08 15:30:38 +0200 | [diff] [blame] | 1316 | int kvm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr); |
Will Auld | 8fe8ab4 | 2012-11-29 12:42:12 -0800 | [diff] [blame] | 1317 | int kvm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr); |
Zhang Xiantao | 54f1585 | 2007-11-19 15:24:28 +0800 | [diff] [blame] | 1318 | |
| 1319 | struct x86_emulate_ctxt; |
| 1320 | |
Sean Christopherson | dca7f12 | 2018-03-08 08:57:27 -0800 | [diff] [blame] | 1321 | int kvm_fast_pio(struct kvm_vcpu *vcpu, int size, unsigned short port, int in); |
Kyle Huey | 6a908b6 | 2016-11-29 12:40:37 -0800 | [diff] [blame] | 1322 | int kvm_emulate_cpuid(struct kvm_vcpu *vcpu); |
Zhang Xiantao | 54f1585 | 2007-11-19 15:24:28 +0800 | [diff] [blame] | 1323 | int kvm_emulate_halt(struct kvm_vcpu *vcpu); |
Joel Schopp | 5cb5605 | 2015-03-02 13:43:31 -0600 | [diff] [blame] | 1324 | int kvm_vcpu_halt(struct kvm_vcpu *vcpu); |
Sheng Yang | f5f48ee | 2010-06-30 12:25:15 +0800 | [diff] [blame] | 1325 | int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu); |
Zhang Xiantao | 54f1585 | 2007-11-19 15:24:28 +0800 | [diff] [blame] | 1326 | |
Guillaume Thouvenin | 3e6e0aa | 2008-05-27 10:18:46 +0200 | [diff] [blame] | 1327 | void kvm_get_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg); |
Gleb Natapov | c697518 | 2010-02-18 12:15:01 +0200 | [diff] [blame] | 1328 | int kvm_load_segment_descriptor(struct kvm_vcpu *vcpu, u16 selector, int seg); |
Paolo Bonzini | 2b4a273 | 2014-11-24 14:35:24 +0100 | [diff] [blame] | 1329 | void kvm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector); |
Guillaume Thouvenin | 3e6e0aa | 2008-05-27 10:18:46 +0200 | [diff] [blame] | 1330 | |
Kevin Wolf | 7f3d35f | 2012-02-08 14:34:38 +0100 | [diff] [blame] | 1331 | int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index, |
| 1332 | int reason, bool has_error_code, u32 error_code); |
Izik Eidus | 37817f2 | 2008-03-24 23:14:53 +0200 | [diff] [blame] | 1333 | |
Avi Kivity | 49a9b07 | 2010-06-10 17:02:14 +0300 | [diff] [blame] | 1334 | int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0); |
Avi Kivity | 2390218 | 2010-06-10 17:02:16 +0300 | [diff] [blame] | 1335 | int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3); |
Avi Kivity | a83b29c | 2010-06-10 17:02:15 +0300 | [diff] [blame] | 1336 | int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4); |
Andre Przywara | eea1cff | 2010-12-21 11:12:00 +0100 | [diff] [blame] | 1337 | int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8); |
Gleb Natapov | 020df07 | 2010-04-13 10:05:23 +0300 | [diff] [blame] | 1338 | int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val); |
| 1339 | int kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val); |
Avi Kivity | 2d3ad1f | 2008-02-24 11:20:43 +0200 | [diff] [blame] | 1340 | unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu); |
| 1341 | void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw); |
Zhang Xiantao | 54f1585 | 2007-11-19 15:24:28 +0800 | [diff] [blame] | 1342 | void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l); |
Dexuan Cui | 2acf923 | 2010-06-10 11:27:12 +0800 | [diff] [blame] | 1343 | int kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr); |
Zhang Xiantao | 54f1585 | 2007-11-19 15:24:28 +0800 | [diff] [blame] | 1344 | |
Paolo Bonzini | 609e36d | 2015-04-08 15:30:38 +0200 | [diff] [blame] | 1345 | int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr); |
Will Auld | 8fe8ab4 | 2012-11-29 12:42:12 -0800 | [diff] [blame] | 1346 | int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr); |
Zhang Xiantao | 54f1585 | 2007-11-19 15:24:28 +0800 | [diff] [blame] | 1347 | |
Jan Kiszka | 91586a3 | 2009-10-05 13:07:21 +0200 | [diff] [blame] | 1348 | unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu); |
| 1349 | void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags); |
Avi Kivity | 022cd0e | 2011-11-10 14:57:23 +0200 | [diff] [blame] | 1350 | bool kvm_rdpmc(struct kvm_vcpu *vcpu); |
Jan Kiszka | 91586a3 | 2009-10-05 13:07:21 +0200 | [diff] [blame] | 1351 | |
Avi Kivity | 298101d | 2007-11-25 13:41:11 +0200 | [diff] [blame] | 1352 | void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr); |
| 1353 | void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code); |
Joerg Roedel | ce7ddec | 2010-04-22 12:33:13 +0200 | [diff] [blame] | 1354 | void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr); |
| 1355 | void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code); |
Avi Kivity | 6389ee9 | 2010-11-29 16:12:30 +0200 | [diff] [blame] | 1356 | void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault); |
Joerg Roedel | ec92fe4 | 2010-09-10 17:30:51 +0200 | [diff] [blame] | 1357 | int kvm_read_guest_page_mmu(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, |
| 1358 | gfn_t gfn, void *data, int offset, int len, |
| 1359 | u32 access); |
Avi Kivity | 0a79b00 | 2009-09-01 12:03:25 +0300 | [diff] [blame] | 1360 | bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl); |
Nadav Amit | 16f8a6f | 2014-10-03 01:10:05 +0300 | [diff] [blame] | 1361 | bool kvm_require_dr(struct kvm_vcpu *vcpu, int dr); |
Avi Kivity | 298101d | 2007-11-25 13:41:11 +0200 | [diff] [blame] | 1362 | |
Michael S. Tsirkin | 1a577b72 | 2012-07-19 13:45:20 +0300 | [diff] [blame] | 1363 | static inline int __kvm_irq_line_state(unsigned long *irq_state, |
| 1364 | int irq_source_id, int level) |
| 1365 | { |
| 1366 | /* Logical OR for level trig interrupt */ |
| 1367 | if (level) |
| 1368 | __set_bit(irq_source_id, irq_state); |
| 1369 | else |
| 1370 | __clear_bit(irq_source_id, irq_state); |
| 1371 | |
| 1372 | return !!(*irq_state); |
| 1373 | } |
| 1374 | |
Junaid Shahid | b94742c | 2018-06-27 14:59:20 -0700 | [diff] [blame] | 1375 | #define KVM_MMU_ROOT_CURRENT BIT(0) |
| 1376 | #define KVM_MMU_ROOT_PREVIOUS(i) BIT(1+i) |
| 1377 | #define KVM_MMU_ROOTS_ALL (~0UL) |
Junaid Shahid | 08fb59d | 2018-06-27 14:59:17 -0700 | [diff] [blame] | 1378 | |
Michael S. Tsirkin | 1a577b72 | 2012-07-19 13:45:20 +0300 | [diff] [blame] | 1379 | int kvm_pic_set_irq(struct kvm_pic *pic, int irq, int irq_source_id, int level); |
| 1380 | void kvm_pic_clear_all(struct kvm_pic *pic, int irq_source_id); |
Xiantao Zhang | 3de42dc | 2008-10-06 13:48:45 +0800 | [diff] [blame] | 1381 | |
Sheng Yang | 3419ffc | 2008-05-15 09:52:48 +0800 | [diff] [blame] | 1382 | void kvm_inject_nmi(struct kvm_vcpu *vcpu); |
| 1383 | |
Xiao Guangrong | 1cb3f3a | 2011-09-22 17:02:48 +0800 | [diff] [blame] | 1384 | int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn); |
Zhang Xiantao | 54f1585 | 2007-11-19 15:24:28 +0800 | [diff] [blame] | 1385 | int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva); |
| 1386 | void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu); |
| 1387 | int kvm_mmu_load(struct kvm_vcpu *vcpu); |
| 1388 | void kvm_mmu_unload(struct kvm_vcpu *vcpu); |
Marcelo Tosatti | 0ba73cd | 2008-09-23 13:18:34 -0300 | [diff] [blame] | 1389 | void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu); |
Vitaly Kuznetsov | 6a82cd1 | 2018-10-08 21:28:07 +0200 | [diff] [blame] | 1390 | void kvm_mmu_free_roots(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, |
| 1391 | ulong roots_to_free); |
Paolo Bonzini | 54987b7 | 2014-09-02 13:23:06 +0200 | [diff] [blame] | 1392 | gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access, |
| 1393 | struct x86_exception *exception); |
Avi Kivity | ab9ae31 | 2010-11-22 17:53:26 +0200 | [diff] [blame] | 1394 | gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva, |
| 1395 | struct x86_exception *exception); |
| 1396 | gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva, |
| 1397 | struct x86_exception *exception); |
| 1398 | gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva, |
| 1399 | struct x86_exception *exception); |
| 1400 | gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva, |
| 1401 | struct x86_exception *exception); |
Zhang Xiantao | 54f1585 | 2007-11-19 15:24:28 +0800 | [diff] [blame] | 1402 | |
Andrey Smetanin | d62caab | 2015-11-10 15:36:33 +0300 | [diff] [blame] | 1403 | void kvm_vcpu_deactivate_apicv(struct kvm_vcpu *vcpu); |
| 1404 | |
Zhang Xiantao | 54f1585 | 2007-11-19 15:24:28 +0800 | [diff] [blame] | 1405 | int kvm_emulate_hypercall(struct kvm_vcpu *vcpu); |
| 1406 | |
Tom Lendacky | 1472775 | 2016-11-23 12:01:38 -0500 | [diff] [blame] | 1407 | int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t gva, u64 error_code, |
Andre Przywara | dc25e89 | 2010-12-21 11:12:07 +0100 | [diff] [blame] | 1408 | void *insn, int insn_len); |
Marcelo Tosatti | a705289 | 2008-09-23 13:18:35 -0300 | [diff] [blame] | 1409 | void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva); |
Junaid Shahid | eb4b248 | 2018-06-27 14:59:14 -0700 | [diff] [blame] | 1410 | void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid); |
Junaid Shahid | ade61e2 | 2018-06-27 14:59:15 -0700 | [diff] [blame] | 1411 | void kvm_mmu_new_cr3(struct kvm_vcpu *vcpu, gpa_t new_cr3, bool skip_tlb_flush); |
Zhang Xiantao | 34c16ee | 2007-10-20 15:34:38 +0800 | [diff] [blame] | 1412 | |
Joerg Roedel | 1855267 | 2008-02-07 13:47:41 +0100 | [diff] [blame] | 1413 | void kvm_enable_tdp(void); |
Joerg Roedel | 5f4cb66 | 2008-07-14 20:36:36 +0200 | [diff] [blame] | 1414 | void kvm_disable_tdp(void); |
Joerg Roedel | 1855267 | 2008-02-07 13:47:41 +0100 | [diff] [blame] | 1415 | |
Paolo Bonzini | 54987b7 | 2014-09-02 13:23:06 +0200 | [diff] [blame] | 1416 | static inline gpa_t translate_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access, |
| 1417 | struct x86_exception *exception) |
Xiao Guangrong | e459e32 | 2011-11-28 20:42:16 +0800 | [diff] [blame] | 1418 | { |
| 1419 | return gpa; |
| 1420 | } |
| 1421 | |
Zhang Xiantao | ec6d273 | 2007-11-19 15:08:31 +0800 | [diff] [blame] | 1422 | static inline struct kvm_mmu_page *page_header(hpa_t shadow_page) |
| 1423 | { |
| 1424 | struct page *page = pfn_to_page(shadow_page >> PAGE_SHIFT); |
| 1425 | |
| 1426 | return (struct kvm_mmu_page *)page_private(page); |
| 1427 | } |
| 1428 | |
Avi Kivity | d6e88ae | 2008-07-10 16:53:33 +0300 | [diff] [blame] | 1429 | static inline u16 kvm_read_ldt(void) |
Zhang Xiantao | ec6d273 | 2007-11-19 15:08:31 +0800 | [diff] [blame] | 1430 | { |
| 1431 | u16 ldt; |
| 1432 | asm("sldt %0" : "=g"(ldt)); |
| 1433 | return ldt; |
| 1434 | } |
| 1435 | |
Avi Kivity | d6e88ae | 2008-07-10 16:53:33 +0300 | [diff] [blame] | 1436 | static inline void kvm_load_ldt(u16 sel) |
Zhang Xiantao | ec6d273 | 2007-11-19 15:08:31 +0800 | [diff] [blame] | 1437 | { |
| 1438 | asm("lldt %0" : : "rm"(sel)); |
| 1439 | } |
Zhang Xiantao | ec6d273 | 2007-11-19 15:08:31 +0800 | [diff] [blame] | 1440 | |
Zhang Xiantao | ec6d273 | 2007-11-19 15:08:31 +0800 | [diff] [blame] | 1441 | #ifdef CONFIG_X86_64 |
| 1442 | static inline unsigned long read_msr(unsigned long msr) |
| 1443 | { |
| 1444 | u64 value; |
| 1445 | |
| 1446 | rdmsrl(msr, value); |
| 1447 | return value; |
| 1448 | } |
| 1449 | #endif |
| 1450 | |
Zhang Xiantao | ec6d273 | 2007-11-19 15:08:31 +0800 | [diff] [blame] | 1451 | static inline u32 get_rdx_init_val(void) |
| 1452 | { |
| 1453 | return 0x600; /* P6 family */ |
| 1454 | } |
| 1455 | |
Avi Kivity | c1a5d4f | 2007-11-25 14:12:03 +0200 | [diff] [blame] | 1456 | static inline void kvm_inject_gp(struct kvm_vcpu *vcpu, u32 error_code) |
| 1457 | { |
| 1458 | kvm_queue_exception_e(vcpu, GP_VECTOR, error_code); |
| 1459 | } |
| 1460 | |
Zhang Xiantao | ec6d273 | 2007-11-19 15:08:31 +0800 | [diff] [blame] | 1461 | #define TSS_IOPB_BASE_OFFSET 0x66 |
| 1462 | #define TSS_BASE_SIZE 0x68 |
| 1463 | #define TSS_IOPB_SIZE (65536 / 8) |
| 1464 | #define TSS_REDIRECTION_SIZE (256 / 8) |
Joe Perches | 7d76b4d | 2008-03-23 01:02:34 -0700 | [diff] [blame] | 1465 | #define RMODE_TSS_SIZE \ |
| 1466 | (TSS_BASE_SIZE + TSS_REDIRECTION_SIZE + TSS_IOPB_SIZE + 1) |
Hollis Blanchard | 53e0aa7 | 2007-12-03 16:15:26 -0600 | [diff] [blame] | 1467 | |
Izik Eidus | 37817f2 | 2008-03-24 23:14:53 +0200 | [diff] [blame] | 1468 | enum { |
| 1469 | TASK_SWITCH_CALL = 0, |
| 1470 | TASK_SWITCH_IRET = 1, |
| 1471 | TASK_SWITCH_JMP = 2, |
| 1472 | TASK_SWITCH_GATE = 3, |
| 1473 | }; |
| 1474 | |
Alexander Graf | 1371d90 | 2008-11-25 20:17:04 +0100 | [diff] [blame] | 1475 | #define HF_GIF_MASK (1 << 0) |
Alexander Graf | 3d6368e | 2008-11-25 20:17:07 +0100 | [diff] [blame] | 1476 | #define HF_HIF_MASK (1 << 1) |
| 1477 | #define HF_VINTR_MASK (1 << 2) |
Gleb Natapov | 95ba827313 | 2009-04-21 17:45:08 +0300 | [diff] [blame] | 1478 | #define HF_NMI_MASK (1 << 3) |
Gleb Natapov | 44c1143 | 2009-05-11 13:35:52 +0300 | [diff] [blame] | 1479 | #define HF_IRET_MASK (1 << 4) |
Joerg Roedel | ec9e60b | 2010-11-29 17:51:47 +0100 | [diff] [blame] | 1480 | #define HF_GUEST_MASK (1 << 5) /* VCPU is in guest-mode */ |
Paolo Bonzini | f077825 | 2015-04-01 15:06:40 +0200 | [diff] [blame] | 1481 | #define HF_SMM_MASK (1 << 6) |
| 1482 | #define HF_SMM_INSIDE_NMI_MASK (1 << 7) |
Alexander Graf | 1371d90 | 2008-11-25 20:17:04 +0100 | [diff] [blame] | 1483 | |
Paolo Bonzini | 699023e | 2015-05-18 15:03:39 +0200 | [diff] [blame] | 1484 | #define __KVM_VCPU_MULTIPLE_ADDRESS_SPACE |
| 1485 | #define KVM_ADDRESS_SPACE_NUM 2 |
| 1486 | |
| 1487 | #define kvm_arch_vcpu_memslots_id(vcpu) ((vcpu)->arch.hflags & HF_SMM_MASK ? 1 : 0) |
| 1488 | #define kvm_memslots_for_spte_role(kvm, role) __kvm_memslots(kvm, (role).smm) |
Avi Kivity | 4ecac3f | 2008-05-13 13:23:38 +0300 | [diff] [blame] | 1489 | |
| 1490 | /* |
| 1491 | * Hardware virtualization extension instructions may fault if a |
| 1492 | * reboot turns off virtualization while processes are running. |
| 1493 | * Trap the fault and ignore the instruction if that happens. |
| 1494 | */ |
Avi Kivity | b7c4145 | 2010-12-02 17:52:50 +0200 | [diff] [blame] | 1495 | asmlinkage void kvm_spurious_fault(void); |
Avi Kivity | 4ecac3f | 2008-05-13 13:23:38 +0300 | [diff] [blame] | 1496 | |
Avi Kivity | 5e520e6 | 2011-05-15 10:13:12 -0400 | [diff] [blame] | 1497 | #define ____kvm_handle_fault_on_reboot(insn, cleanup_insn) \ |
Avi Kivity | 4ecac3f | 2008-05-13 13:23:38 +0300 | [diff] [blame] | 1498 | "666: " insn "\n\t" \ |
Avi Kivity | b7c4145 | 2010-12-02 17:52:50 +0200 | [diff] [blame] | 1499 | "668: \n\t" \ |
Eduardo Habkost | 18b13e5 | 2008-08-19 20:00:08 -0300 | [diff] [blame] | 1500 | ".pushsection .fixup, \"ax\" \n" \ |
Avi Kivity | 4ecac3f | 2008-05-13 13:23:38 +0300 | [diff] [blame] | 1501 | "667: \n\t" \ |
Avi Kivity | 5e520e6 | 2011-05-15 10:13:12 -0400 | [diff] [blame] | 1502 | cleanup_insn "\n\t" \ |
Avi Kivity | b7c4145 | 2010-12-02 17:52:50 +0200 | [diff] [blame] | 1503 | "cmpb $0, kvm_rebooting \n\t" \ |
| 1504 | "jne 668b \n\t" \ |
Avi Kivity | 8ceed347 | 2008-08-14 21:25:47 +0300 | [diff] [blame] | 1505 | __ASM_SIZE(push) " $666b \n\t" \ |
Sean Christopherson | e814349 | 2018-12-20 14:21:08 -0800 | [diff] [blame] | 1506 | "jmp kvm_spurious_fault \n\t" \ |
Avi Kivity | 4ecac3f | 2008-05-13 13:23:38 +0300 | [diff] [blame] | 1507 | ".popsection \n\t" \ |
H. Peter Anvin | 3ee8972 | 2012-04-20 13:41:59 -0700 | [diff] [blame] | 1508 | _ASM_EXTABLE(666b, 667b) |
Avi Kivity | 4ecac3f | 2008-05-13 13:23:38 +0300 | [diff] [blame] | 1509 | |
Avi Kivity | 5e520e6 | 2011-05-15 10:13:12 -0400 | [diff] [blame] | 1510 | #define __kvm_handle_fault_on_reboot(insn) \ |
| 1511 | ____kvm_handle_fault_on_reboot(insn, "") |
| 1512 | |
Andrea Arcangeli | e930bff | 2008-07-25 16:24:52 +0200 | [diff] [blame] | 1513 | #define KVM_ARCH_WANT_MMU_NOTIFIER |
Takuya Yoshikawa | b3ae209 | 2012-07-02 17:56:33 +0900 | [diff] [blame] | 1514 | int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end); |
Andres Lagar-Cavilla | 5712846 | 2014-09-22 14:54:42 -0700 | [diff] [blame] | 1515 | int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end); |
Andrea Arcangeli | 8ee5382 | 2011-01-13 15:47:10 -0800 | [diff] [blame] | 1516 | int kvm_test_age_hva(struct kvm *kvm, unsigned long hva); |
Lan Tianyu | 748c0e3 | 2018-12-06 21:21:10 +0800 | [diff] [blame] | 1517 | int kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte); |
Yang Zhang | c7c9c56 | 2013-01-25 10:18:51 +0800 | [diff] [blame] | 1518 | int kvm_cpu_has_injectable_intr(struct kvm_vcpu *v); |
Gleb Natapov | a1b3710 | 2009-07-09 15:33:52 +0300 | [diff] [blame] | 1519 | int kvm_cpu_has_interrupt(struct kvm_vcpu *vcpu); |
| 1520 | int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu); |
Gleb Natapov | 0b71785 | 2009-07-09 15:33:53 +0300 | [diff] [blame] | 1521 | int kvm_cpu_get_interrupt(struct kvm_vcpu *v); |
Nadav Amit | d28bc9d | 2015-04-13 14:34:08 +0300 | [diff] [blame] | 1522 | void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event); |
Tang Chen | 4256f43 | 2014-09-24 15:57:54 +0800 | [diff] [blame] | 1523 | void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu); |
Andrea Arcangeli | e930bff | 2008-07-25 16:24:52 +0200 | [diff] [blame] | 1524 | |
Wanpeng Li | 4180bf1 | 2018-07-23 14:39:54 +0800 | [diff] [blame] | 1525 | int kvm_pv_send_ipi(struct kvm *kvm, unsigned long ipi_bitmap_low, |
Wanpeng Li | bdf7ffc | 2018-08-30 10:03:30 +0800 | [diff] [blame] | 1526 | unsigned long ipi_bitmap_high, u32 min, |
Wanpeng Li | 4180bf1 | 2018-07-23 14:39:54 +0800 | [diff] [blame] | 1527 | unsigned long icr, int op_64_bit); |
| 1528 | |
Paolo Bonzini | 5b76a3c | 2018-08-05 16:07:47 +0200 | [diff] [blame] | 1529 | u64 kvm_get_arch_capabilities(void); |
Avi Kivity | 18863bd | 2009-09-07 11:12:18 +0300 | [diff] [blame] | 1530 | void kvm_define_shared_msr(unsigned index, u32 msr); |
Andy Honig | 8b3c310 | 2014-08-27 11:16:44 -0700 | [diff] [blame] | 1531 | int kvm_set_shared_msr(unsigned index, u64 val, u64 mask); |
Avi Kivity | 18863bd | 2009-09-07 11:12:18 +0300 | [diff] [blame] | 1532 | |
Haozhong Zhang | 35181e8 | 2015-10-20 15:39:03 +0800 | [diff] [blame] | 1533 | u64 kvm_scale_tsc(struct kvm_vcpu *vcpu, u64 tsc); |
Haozhong Zhang | 4ba7653 | 2015-10-20 15:39:07 +0800 | [diff] [blame] | 1534 | u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc); |
Haozhong Zhang | 35181e8 | 2015-10-20 15:39:03 +0800 | [diff] [blame] | 1535 | |
Nadav Amit | 82b3277 | 2014-11-02 11:54:45 +0200 | [diff] [blame] | 1536 | unsigned long kvm_get_linear_rip(struct kvm_vcpu *vcpu); |
Jan Kiszka | f92653e | 2010-02-23 17:47:55 +0100 | [diff] [blame] | 1537 | bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip); |
| 1538 | |
Paolo Bonzini | 2860c4b | 2016-01-07 15:05:10 +0100 | [diff] [blame] | 1539 | void kvm_make_mclock_inprogress_request(struct kvm *kvm); |
| 1540 | void kvm_make_scan_ioapic_request(struct kvm *kvm); |
| 1541 | |
Gleb Natapov | af585b9 | 2010-10-14 11:22:46 +0200 | [diff] [blame] | 1542 | void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu, |
| 1543 | struct kvm_async_pf *work); |
| 1544 | void kvm_arch_async_page_present(struct kvm_vcpu *vcpu, |
| 1545 | struct kvm_async_pf *work); |
Gleb Natapov | 56028d0 | 2010-10-17 18:13:42 +0200 | [diff] [blame] | 1546 | void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, |
| 1547 | struct kvm_async_pf *work); |
Gleb Natapov | 7c90705 | 2010-10-14 11:22:53 +0200 | [diff] [blame] | 1548 | bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu); |
Gleb Natapov | af585b9 | 2010-10-14 11:22:46 +0200 | [diff] [blame] | 1549 | extern bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn); |
| 1550 | |
Kyle Huey | 6affcbe | 2016-11-29 12:40:40 -0800 | [diff] [blame] | 1551 | int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu); |
| 1552 | int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err); |
Sean Christopherson | d264ee0 | 2018-08-27 15:21:12 -0700 | [diff] [blame] | 1553 | void __kvm_request_immediate_exit(struct kvm_vcpu *vcpu); |
Andre Przywara | db8fcef | 2010-12-21 11:12:01 +0100 | [diff] [blame] | 1554 | |
Gleb Natapov | f5132b0 | 2011-11-10 14:57:22 +0200 | [diff] [blame] | 1555 | int kvm_is_in_guest(void); |
| 1556 | |
Paolo Bonzini | 1d8007b | 2015-10-12 13:38:32 +0200 | [diff] [blame] | 1557 | int __x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa, u32 size); |
| 1558 | int x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa, u32 size); |
Paolo Bonzini | d71ba78 | 2015-07-29 11:56:48 +0200 | [diff] [blame] | 1559 | bool kvm_vcpu_is_reset_bsp(struct kvm_vcpu *vcpu); |
| 1560 | bool kvm_vcpu_is_bsp(struct kvm_vcpu *vcpu); |
Gleb Natapov | f5132b0 | 2011-11-10 14:57:22 +0200 | [diff] [blame] | 1561 | |
Feng Wu | 8feb4a0 | 2015-09-18 22:29:47 +0800 | [diff] [blame] | 1562 | bool kvm_intr_is_single_vcpu(struct kvm *kvm, struct kvm_lapic_irq *irq, |
| 1563 | struct kvm_vcpu **dest_vcpu); |
| 1564 | |
Radim Krčmář | 37131313 | 2016-07-12 22:09:27 +0200 | [diff] [blame] | 1565 | void kvm_set_msi_irq(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e, |
Feng Wu | d84f1e0 | 2015-09-18 22:29:49 +0800 | [diff] [blame] | 1566 | struct kvm_lapic_irq *irq); |
Paolo Bonzini | 197a4f4 | 2015-11-04 16:24:17 +0100 | [diff] [blame] | 1567 | |
Suravee Suthikulpanit | d1ed092 | 2016-05-04 14:09:43 -0500 | [diff] [blame] | 1568 | static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) |
| 1569 | { |
| 1570 | if (kvm_x86_ops->vcpu_blocking) |
| 1571 | kvm_x86_ops->vcpu_blocking(vcpu); |
| 1572 | } |
| 1573 | |
| 1574 | static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) |
| 1575 | { |
| 1576 | if (kvm_x86_ops->vcpu_unblocking) |
| 1577 | kvm_x86_ops->vcpu_unblocking(vcpu); |
| 1578 | } |
| 1579 | |
Christian Borntraeger | 3491caf | 2016-05-13 12:16:35 +0200 | [diff] [blame] | 1580 | static inline void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu) {} |
Christoffer Dall | 3217f7c | 2015-08-27 16:41:15 +0200 | [diff] [blame] | 1581 | |
Suravee Suthikulpanit | 7d669f5 | 2016-06-15 17:23:45 -0500 | [diff] [blame] | 1582 | static inline int kvm_cpu_get_apicid(int mps_cpu) |
| 1583 | { |
| 1584 | #ifdef CONFIG_X86_LOCAL_APIC |
Thomas Gleixner | 6406350 | 2017-09-13 23:29:21 +0200 | [diff] [blame] | 1585 | return default_cpu_present_to_apicid(mps_cpu); |
Suravee Suthikulpanit | 7d669f5 | 2016-06-15 17:23:45 -0500 | [diff] [blame] | 1586 | #else |
| 1587 | WARN_ON_ONCE(1); |
| 1588 | return BAD_APICID; |
| 1589 | #endif |
| 1590 | } |
| 1591 | |
Ladi Prosek | 05cade7 | 2017-10-11 16:54:45 +0200 | [diff] [blame] | 1592 | #define put_smstate(type, buf, offset, val) \ |
| 1593 | *(type *)((buf) + (offset) - 0x7e00) = val |
| 1594 | |
H. Peter Anvin | 1965aae | 2008-10-22 22:26:29 -0700 | [diff] [blame] | 1595 | #endif /* _ASM_X86_KVM_HOST_H */ |