Vegard Nossum | a656c8e | 2008-07-22 21:27:11 +0200 | [diff] [blame] | 1 | /* |
Carsten Otte | 043405e | 2007-10-10 17:16:19 +0200 | [diff] [blame] | 2 | * Kernel-based Virtual Machine driver for Linux |
| 3 | * |
| 4 | * This header defines architecture specific interfaces, x86 version |
| 5 | * |
| 6 | * This work is licensed under the terms of the GNU GPL, version 2. See |
| 7 | * the COPYING file in the top-level directory. |
| 8 | * |
| 9 | */ |
| 10 | |
H. Peter Anvin | 1965aae | 2008-10-22 22:26:29 -0700 | [diff] [blame] | 11 | #ifndef _ASM_X86_KVM_HOST_H |
| 12 | #define _ASM_X86_KVM_HOST_H |
Carsten Otte | 043405e | 2007-10-10 17:16:19 +0200 | [diff] [blame] | 13 | |
Zhang Xiantao | 34c16ee | 2007-10-20 15:34:38 +0800 | [diff] [blame] | 14 | #include <linux/types.h> |
| 15 | #include <linux/mm.h> |
Andrea Arcangeli | e930bff | 2008-07-25 16:24:52 +0200 | [diff] [blame] | 16 | #include <linux/mmu_notifier.h> |
Marcelo Tosatti | 229456f | 2009-06-17 09:22:14 -0300 | [diff] [blame] | 17 | #include <linux/tracepoint.h> |
Sheng Yang | f5f48ee | 2010-06-30 12:25:15 +0800 | [diff] [blame] | 18 | #include <linux/cpumask.h> |
Gleb Natapov | f5132b0 | 2011-11-10 14:57:22 +0200 | [diff] [blame] | 19 | #include <linux/irq_work.h> |
Nicolai Stange | 447ae31 | 2018-07-29 12:15:33 +0200 | [diff] [blame] | 20 | #include <linux/irq.h> |
Zhang Xiantao | 34c16ee | 2007-10-20 15:34:38 +0800 | [diff] [blame] | 21 | |
| 22 | #include <linux/kvm.h> |
| 23 | #include <linux/kvm_para.h> |
Avi Kivity | edf8841 | 2007-12-16 11:02:48 +0200 | [diff] [blame] | 24 | #include <linux/kvm_types.h> |
Gleb Natapov | f5132b0 | 2011-11-10 14:57:22 +0200 | [diff] [blame] | 25 | #include <linux/perf_event.h> |
Marcelo Tosatti | d828199 | 2012-11-27 23:29:01 -0200 | [diff] [blame] | 26 | #include <linux/pvclock_gtod.h> |
| 27 | #include <linux/clocksource.h> |
Feng Wu | 8727688 | 2015-09-18 22:29:40 +0800 | [diff] [blame] | 28 | #include <linux/irqbypass.h> |
Andrey Smetanin | 5c919412 | 2015-11-10 15:36:34 +0300 | [diff] [blame] | 29 | #include <linux/hyperv.h> |
Zhang Xiantao | 34c16ee | 2007-10-20 15:34:38 +0800 | [diff] [blame] | 30 | |
Suravee Suthikulpanit | 7d669f5 | 2016-06-15 17:23:45 -0500 | [diff] [blame] | 31 | #include <asm/apic.h> |
Gerd Hoffmann | 50d0a0f | 2008-06-03 16:17:31 +0200 | [diff] [blame] | 32 | #include <asm/pvclock-abi.h> |
Hollis Blanchard | e01a1b5 | 2007-12-03 15:30:25 -0600 | [diff] [blame] | 33 | #include <asm/desc.h> |
Sheng Yang | 0bed3b5 | 2008-10-09 16:01:54 +0800 | [diff] [blame] | 34 | #include <asm/mtrr.h> |
Alexander Graf | 9962d03 | 2008-11-25 20:17:02 +0100 | [diff] [blame] | 35 | #include <asm/msr-index.h> |
H. Peter Anvin | 3ee8972 | 2012-04-20 13:41:59 -0700 | [diff] [blame] | 36 | #include <asm/asm.h> |
Xiao Guangrong | 21ebbed | 2016-02-24 17:51:09 +0800 | [diff] [blame] | 37 | #include <asm/kvm_page_track.h> |
Vitaly Kuznetsov | 5a48580 | 2018-03-20 15:02:05 +0100 | [diff] [blame] | 38 | #include <asm/hyperv-tlfs.h> |
Hollis Blanchard | e01a1b5 | 2007-12-03 15:30:25 -0600 | [diff] [blame] | 39 | |
Radim Krčmář | 682f732 | 2016-07-12 22:09:29 +0200 | [diff] [blame] | 40 | #define KVM_MAX_VCPUS 288 |
Radim Krčmář | 757883d | 2016-07-12 22:09:17 +0200 | [diff] [blame] | 41 | #define KVM_SOFT_MAX_VCPUS 240 |
Radim Krčmář | af1bae5 | 2016-07-12 22:09:30 +0200 | [diff] [blame] | 42 | #define KVM_MAX_VCPU_ID 1023 |
Igor Mammedov | 1d4e7e3 | 2014-11-06 15:52:47 +0000 | [diff] [blame] | 43 | #define KVM_USER_MEM_SLOTS 509 |
Alex Williamson | 0743247 | 2012-12-10 10:33:15 -0700 | [diff] [blame] | 44 | /* memory slots that are not exposed to userspace */ |
| 45 | #define KVM_PRIVATE_MEM_SLOTS 3 |
Alex Williamson | bbacc0c | 2012-12-10 10:33:09 -0700 | [diff] [blame] | 46 | #define KVM_MEM_SLOTS_NUM (KVM_USER_MEM_SLOTS + KVM_PRIVATE_MEM_SLOTS) |
Xiao Guangrong | 93a5cef | 2011-11-24 17:37:48 +0800 | [diff] [blame] | 47 | |
Paolo Bonzini | b401ee0 | 2017-04-18 12:41:18 +0200 | [diff] [blame] | 48 | #define KVM_HALT_POLL_NS_DEFAULT 200000 |
Avi Kivity | 69a9f69 | 2008-03-21 12:38:23 +0200 | [diff] [blame] | 49 | |
Alexander Graf | 8175e5b | 2013-04-15 10:42:33 +0200 | [diff] [blame] | 50 | #define KVM_IRQCHIP_NUM_PINS KVM_IOAPIC_NUM_PINS |
| 51 | |
Paolo Bonzini | 2860c4b | 2016-01-07 15:05:10 +0100 | [diff] [blame] | 52 | /* x86-specific vcpu->requests bit members */ |
Andrew Jones | 2387149 | 2017-06-04 14:43:51 +0200 | [diff] [blame] | 53 | #define KVM_REQ_MIGRATE_TIMER KVM_ARCH_REQ(0) |
| 54 | #define KVM_REQ_REPORT_TPR_ACCESS KVM_ARCH_REQ(1) |
| 55 | #define KVM_REQ_TRIPLE_FAULT KVM_ARCH_REQ(2) |
| 56 | #define KVM_REQ_MMU_SYNC KVM_ARCH_REQ(3) |
| 57 | #define KVM_REQ_CLOCK_UPDATE KVM_ARCH_REQ(4) |
Junaid Shahid | 6e42782 | 2018-06-27 14:59:08 -0700 | [diff] [blame] | 58 | #define KVM_REQ_LOAD_CR3 KVM_ARCH_REQ(5) |
Andrew Jones | 2387149 | 2017-06-04 14:43:51 +0200 | [diff] [blame] | 59 | #define KVM_REQ_EVENT KVM_ARCH_REQ(6) |
| 60 | #define KVM_REQ_APF_HALT KVM_ARCH_REQ(7) |
| 61 | #define KVM_REQ_STEAL_UPDATE KVM_ARCH_REQ(8) |
| 62 | #define KVM_REQ_NMI KVM_ARCH_REQ(9) |
| 63 | #define KVM_REQ_PMU KVM_ARCH_REQ(10) |
| 64 | #define KVM_REQ_PMI KVM_ARCH_REQ(11) |
| 65 | #define KVM_REQ_SMI KVM_ARCH_REQ(12) |
| 66 | #define KVM_REQ_MASTERCLOCK_UPDATE KVM_ARCH_REQ(13) |
| 67 | #define KVM_REQ_MCLOCK_INPROGRESS \ |
| 68 | KVM_ARCH_REQ_FLAGS(14, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) |
| 69 | #define KVM_REQ_SCAN_IOAPIC \ |
| 70 | KVM_ARCH_REQ_FLAGS(15, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) |
| 71 | #define KVM_REQ_GLOBAL_CLOCK_UPDATE KVM_ARCH_REQ(16) |
| 72 | #define KVM_REQ_APIC_PAGE_RELOAD \ |
| 73 | KVM_ARCH_REQ_FLAGS(17, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) |
| 74 | #define KVM_REQ_HV_CRASH KVM_ARCH_REQ(18) |
| 75 | #define KVM_REQ_IOAPIC_EOI_EXIT KVM_ARCH_REQ(19) |
| 76 | #define KVM_REQ_HV_RESET KVM_ARCH_REQ(20) |
| 77 | #define KVM_REQ_HV_EXIT KVM_ARCH_REQ(21) |
| 78 | #define KVM_REQ_HV_STIMER KVM_ARCH_REQ(22) |
Liran Alon | e40ff1d | 2018-03-21 02:50:31 +0200 | [diff] [blame] | 79 | #define KVM_REQ_LOAD_EOI_EXITMAP KVM_ARCH_REQ(23) |
Paolo Bonzini | 7f7f1ba | 2018-07-18 18:49:01 +0200 | [diff] [blame] | 80 | #define KVM_REQ_GET_VMCS12_PAGES KVM_ARCH_REQ(24) |
Paolo Bonzini | 2860c4b | 2016-01-07 15:05:10 +0100 | [diff] [blame] | 81 | |
Joerg Roedel | cfec82c | 2011-04-04 12:39:28 +0200 | [diff] [blame] | 82 | #define CR0_RESERVED_BITS \ |
| 83 | (~(unsigned long)(X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS \ |
| 84 | | X86_CR0_ET | X86_CR0_NE | X86_CR0_WP | X86_CR0_AM \ |
| 85 | | X86_CR0_NW | X86_CR0_CD | X86_CR0_PG)) |
| 86 | |
Joerg Roedel | cfec82c | 2011-04-04 12:39:28 +0200 | [diff] [blame] | 87 | #define CR4_RESERVED_BITS \ |
| 88 | (~(unsigned long)(X86_CR4_VME | X86_CR4_PVI | X86_CR4_TSD | X86_CR4_DE\ |
| 89 | | X86_CR4_PSE | X86_CR4_PAE | X86_CR4_MCE \ |
Mao, Junjie | ad756a1 | 2012-07-02 01:18:48 +0000 | [diff] [blame] | 90 | | X86_CR4_PGE | X86_CR4_PCE | X86_CR4_OSFXSR | X86_CR4_PCIDE \ |
H. Peter Anvin | afcbf13 | 2013-04-27 16:37:47 -0700 | [diff] [blame] | 91 | | X86_CR4_OSXSAVE | X86_CR4_SMEP | X86_CR4_FSGSBASE \ |
Yu Zhang | fd8cb43 | 2017-08-24 20:27:56 +0800 | [diff] [blame] | 92 | | X86_CR4_OSXMMEXCPT | X86_CR4_LA57 | X86_CR4_VMXE \ |
Paolo Bonzini | ae3e61e | 2016-07-12 10:36:41 +0200 | [diff] [blame] | 93 | | X86_CR4_SMAP | X86_CR4_PKE | X86_CR4_UMIP)) |
Joerg Roedel | cfec82c | 2011-04-04 12:39:28 +0200 | [diff] [blame] | 94 | |
| 95 | #define CR8_RESERVED_BITS (~(unsigned long)X86_CR8_TPR) |
| 96 | |
| 97 | |
Zhang Xiantao | cd6e8f8 | 2007-11-19 14:33:37 +0800 | [diff] [blame] | 98 | |
Zhang Xiantao | cd6e8f8 | 2007-11-19 14:33:37 +0800 | [diff] [blame] | 99 | #define INVALID_PAGE (~(hpa_t)0) |
Xiao Guangrong | dd180b3 | 2010-07-03 16:02:42 +0800 | [diff] [blame] | 100 | #define VALID_PAGE(x) ((x) != INVALID_PAGE) |
| 101 | |
Zhang Xiantao | cd6e8f8 | 2007-11-19 14:33:37 +0800 | [diff] [blame] | 102 | #define UNMAPPED_GVA (~(gpa_t)0) |
| 103 | |
Joerg Roedel | ec04b26 | 2009-06-19 15:16:23 +0200 | [diff] [blame] | 104 | /* KVM Hugepage definitions for x86 */ |
Wei Yang | 4fef0f4 | 2018-09-28 22:08:50 +0800 | [diff] [blame] | 105 | enum { |
| 106 | PT_PAGE_TABLE_LEVEL = 1, |
| 107 | PT_DIRECTORY_LEVEL = 2, |
| 108 | PT_PDPE_LEVEL = 3, |
| 109 | /* set max level to the biggest one */ |
| 110 | PT_MAX_HUGEPAGE_LEVEL = PT_PDPE_LEVEL, |
| 111 | }; |
| 112 | #define KVM_NR_PAGE_SIZES (PT_MAX_HUGEPAGE_LEVEL - \ |
| 113 | PT_PAGE_TABLE_LEVEL + 1) |
Joerg Roedel | 8285541 | 2010-07-01 16:00:11 +0200 | [diff] [blame] | 114 | #define KVM_HPAGE_GFN_SHIFT(x) (((x) - 1) * 9) |
| 115 | #define KVM_HPAGE_SHIFT(x) (PAGE_SHIFT + KVM_HPAGE_GFN_SHIFT(x)) |
Joerg Roedel | ec04b26 | 2009-06-19 15:16:23 +0200 | [diff] [blame] | 116 | #define KVM_HPAGE_SIZE(x) (1UL << KVM_HPAGE_SHIFT(x)) |
| 117 | #define KVM_HPAGE_MASK(x) (~(KVM_HPAGE_SIZE(x) - 1)) |
| 118 | #define KVM_PAGES_PER_HPAGE(x) (KVM_HPAGE_SIZE(x) / PAGE_SIZE) |
Marcelo Tosatti | 05da455 | 2008-02-23 11:44:30 -0300 | [diff] [blame] | 119 | |
Christoffer Dall | 6d9d41e | 2013-10-02 14:22:28 -0700 | [diff] [blame] | 120 | static inline gfn_t gfn_to_index(gfn_t gfn, gfn_t base_gfn, int level) |
| 121 | { |
| 122 | /* KVM_HPAGE_GFN_SHIFT(PT_PAGE_TABLE_LEVEL) must be 0. */ |
| 123 | return (gfn >> KVM_HPAGE_GFN_SHIFT(level)) - |
| 124 | (base_gfn >> KVM_HPAGE_GFN_SHIFT(level)); |
| 125 | } |
| 126 | |
Zhang Xiantao | d657a98 | 2007-12-14 09:41:22 +0800 | [diff] [blame] | 127 | #define KVM_PERMILLE_MMU_PAGES 20 |
| 128 | #define KVM_MIN_ALLOC_MMU_PAGES 64 |
David Matlack | 114df30 | 2016-12-19 13:58:25 -0800 | [diff] [blame] | 129 | #define KVM_MMU_HASH_SHIFT 12 |
Dong, Eddie | 1ae0a13 | 2008-01-07 13:20:25 +0200 | [diff] [blame] | 130 | #define KVM_NUM_MMU_PAGES (1 << KVM_MMU_HASH_SHIFT) |
Zhang Xiantao | d657a98 | 2007-12-14 09:41:22 +0800 | [diff] [blame] | 131 | #define KVM_MIN_FREE_MMU_PAGES 5 |
| 132 | #define KVM_REFILL_PAGES 25 |
Andre Przywara | 73c1160 | 2010-12-01 12:17:44 +0100 | [diff] [blame] | 133 | #define KVM_MAX_CPUID_ENTRIES 80 |
Sheng Yang | 0bed3b5 | 2008-10-09 16:01:54 +0800 | [diff] [blame] | 134 | #define KVM_NR_FIXED_MTRR_REGION 88 |
Paolo Bonzini | 0d234da | 2014-08-18 16:39:48 +0200 | [diff] [blame] | 135 | #define KVM_NR_VAR_MTRR 8 |
Zhang Xiantao | d657a98 | 2007-12-14 09:41:22 +0800 | [diff] [blame] | 136 | |
Gleb Natapov | af585b9 | 2010-10-14 11:22:46 +0200 | [diff] [blame] | 137 | #define ASYNC_PF_PER_VCPU 64 |
| 138 | |
Marcelo Tosatti | 5fdbf97 | 2008-06-27 14:58:02 -0300 | [diff] [blame] | 139 | enum kvm_reg { |
Zhang Xiantao | 2b3ccfa | 2007-11-19 14:56:05 +0800 | [diff] [blame] | 140 | VCPU_REGS_RAX = 0, |
| 141 | VCPU_REGS_RCX = 1, |
| 142 | VCPU_REGS_RDX = 2, |
| 143 | VCPU_REGS_RBX = 3, |
| 144 | VCPU_REGS_RSP = 4, |
| 145 | VCPU_REGS_RBP = 5, |
| 146 | VCPU_REGS_RSI = 6, |
| 147 | VCPU_REGS_RDI = 7, |
| 148 | #ifdef CONFIG_X86_64 |
| 149 | VCPU_REGS_R8 = 8, |
| 150 | VCPU_REGS_R9 = 9, |
| 151 | VCPU_REGS_R10 = 10, |
| 152 | VCPU_REGS_R11 = 11, |
| 153 | VCPU_REGS_R12 = 12, |
| 154 | VCPU_REGS_R13 = 13, |
| 155 | VCPU_REGS_R14 = 14, |
| 156 | VCPU_REGS_R15 = 15, |
| 157 | #endif |
Marcelo Tosatti | 5fdbf97 | 2008-06-27 14:58:02 -0300 | [diff] [blame] | 158 | VCPU_REGS_RIP, |
Zhang Xiantao | 2b3ccfa | 2007-11-19 14:56:05 +0800 | [diff] [blame] | 159 | NR_VCPU_REGS |
| 160 | }; |
| 161 | |
Avi Kivity | 6de4f3a | 2009-05-31 22:58:47 +0300 | [diff] [blame] | 162 | enum kvm_reg_ex { |
| 163 | VCPU_EXREG_PDPTR = NR_VCPU_REGS, |
Avi Kivity | aff48ba | 2010-12-05 18:56:11 +0200 | [diff] [blame] | 164 | VCPU_EXREG_CR3, |
Avi Kivity | 6de1273 | 2011-03-07 12:51:22 +0200 | [diff] [blame] | 165 | VCPU_EXREG_RFLAGS, |
Avi Kivity | 2fb92db | 2011-04-27 19:42:18 +0300 | [diff] [blame] | 166 | VCPU_EXREG_SEGMENTS, |
Avi Kivity | 6de4f3a | 2009-05-31 22:58:47 +0300 | [diff] [blame] | 167 | }; |
| 168 | |
Zhang Xiantao | 2b3ccfa | 2007-11-19 14:56:05 +0800 | [diff] [blame] | 169 | enum { |
Zhang Xiantao | 2b3ccfa | 2007-11-19 14:56:05 +0800 | [diff] [blame] | 170 | VCPU_SREG_ES, |
Avi Kivity | 81609e3 | 2008-05-27 16:26:01 +0300 | [diff] [blame] | 171 | VCPU_SREG_CS, |
| 172 | VCPU_SREG_SS, |
| 173 | VCPU_SREG_DS, |
Zhang Xiantao | 2b3ccfa | 2007-11-19 14:56:05 +0800 | [diff] [blame] | 174 | VCPU_SREG_FS, |
| 175 | VCPU_SREG_GS, |
Zhang Xiantao | 2b3ccfa | 2007-11-19 14:56:05 +0800 | [diff] [blame] | 176 | VCPU_SREG_TR, |
| 177 | VCPU_SREG_LDTR, |
| 178 | }; |
| 179 | |
Avi Kivity | 56e8231 | 2009-08-12 15:04:37 +0300 | [diff] [blame] | 180 | #include <asm/kvm_emulate.h> |
Zhang Xiantao | 2b3ccfa | 2007-11-19 14:56:05 +0800 | [diff] [blame] | 181 | |
Zhang Xiantao | d657a98 | 2007-12-14 09:41:22 +0800 | [diff] [blame] | 182 | #define KVM_NR_MEM_OBJS 40 |
| 183 | |
Jan Kiszka | 42dbaa5 | 2008-12-15 13:52:10 +0100 | [diff] [blame] | 184 | #define KVM_NR_DB_REGS 4 |
| 185 | |
| 186 | #define DR6_BD (1 << 13) |
| 187 | #define DR6_BS (1 << 14) |
Jim Mattson | cfb634f | 2018-09-21 10:36:17 -0700 | [diff] [blame] | 188 | #define DR6_BT (1 << 15) |
Nadav Amit | 6f43ed0 | 2014-07-15 17:37:46 +0300 | [diff] [blame] | 189 | #define DR6_RTM (1 << 16) |
| 190 | #define DR6_FIXED_1 0xfffe0ff0 |
| 191 | #define DR6_INIT 0xffff0ff0 |
| 192 | #define DR6_VOLATILE 0x0001e00f |
Jan Kiszka | 42dbaa5 | 2008-12-15 13:52:10 +0100 | [diff] [blame] | 193 | |
| 194 | #define DR7_BP_EN_MASK 0x000000ff |
| 195 | #define DR7_GE (1 << 9) |
| 196 | #define DR7_GD (1 << 13) |
| 197 | #define DR7_FIXED_1 0x00000400 |
Nadav Amit | 6f43ed0 | 2014-07-15 17:37:46 +0300 | [diff] [blame] | 198 | #define DR7_VOLATILE 0xffff2bff |
Jan Kiszka | 42dbaa5 | 2008-12-15 13:52:10 +0100 | [diff] [blame] | 199 | |
Nadav Amit | c205fb7 | 2014-12-25 02:52:16 +0200 | [diff] [blame] | 200 | #define PFERR_PRESENT_BIT 0 |
| 201 | #define PFERR_WRITE_BIT 1 |
| 202 | #define PFERR_USER_BIT 2 |
| 203 | #define PFERR_RSVD_BIT 3 |
| 204 | #define PFERR_FETCH_BIT 4 |
Huaitong Han | be94f6b | 2016-03-22 16:51:20 +0800 | [diff] [blame] | 205 | #define PFERR_PK_BIT 5 |
Tom Lendacky | 1472775 | 2016-11-23 12:01:38 -0500 | [diff] [blame] | 206 | #define PFERR_GUEST_FINAL_BIT 32 |
| 207 | #define PFERR_GUEST_PAGE_BIT 33 |
Nadav Amit | c205fb7 | 2014-12-25 02:52:16 +0200 | [diff] [blame] | 208 | |
| 209 | #define PFERR_PRESENT_MASK (1U << PFERR_PRESENT_BIT) |
| 210 | #define PFERR_WRITE_MASK (1U << PFERR_WRITE_BIT) |
| 211 | #define PFERR_USER_MASK (1U << PFERR_USER_BIT) |
| 212 | #define PFERR_RSVD_MASK (1U << PFERR_RSVD_BIT) |
| 213 | #define PFERR_FETCH_MASK (1U << PFERR_FETCH_BIT) |
Huaitong Han | be94f6b | 2016-03-22 16:51:20 +0800 | [diff] [blame] | 214 | #define PFERR_PK_MASK (1U << PFERR_PK_BIT) |
Tom Lendacky | 1472775 | 2016-11-23 12:01:38 -0500 | [diff] [blame] | 215 | #define PFERR_GUEST_FINAL_MASK (1ULL << PFERR_GUEST_FINAL_BIT) |
| 216 | #define PFERR_GUEST_PAGE_MASK (1ULL << PFERR_GUEST_PAGE_BIT) |
| 217 | |
| 218 | #define PFERR_NESTED_GUEST_PAGE (PFERR_GUEST_PAGE_MASK | \ |
Tom Lendacky | 1472775 | 2016-11-23 12:01:38 -0500 | [diff] [blame] | 219 | PFERR_WRITE_MASK | \ |
| 220 | PFERR_PRESENT_MASK) |
Nadav Amit | c205fb7 | 2014-12-25 02:52:16 +0200 | [diff] [blame] | 221 | |
Junaid Shahid | 37f0e8f | 2016-12-06 16:46:15 -0800 | [diff] [blame] | 222 | /* |
| 223 | * The mask used to denote special SPTEs, which can be either MMIO SPTEs or |
| 224 | * Access Tracking SPTEs. We use bit 62 instead of bit 63 to avoid conflicting |
| 225 | * with the SVE bit in EPT PTEs. |
| 226 | */ |
| 227 | #define SPTE_SPECIAL_MASK (1ULL << 62) |
| 228 | |
Gleb Natapov | 4138377 | 2012-04-19 14:06:29 +0300 | [diff] [blame] | 229 | /* apic attention bits */ |
| 230 | #define KVM_APIC_CHECK_VAPIC 0 |
Michael S. Tsirkin | ae7a2a3 | 2012-06-24 19:25:07 +0300 | [diff] [blame] | 231 | /* |
| 232 | * The following bit is set with PV-EOI, unset on EOI. |
| 233 | * We detect PV-EOI changes by guest by comparing |
| 234 | * this bit with PV-EOI in guest memory. |
| 235 | * See the implementation in apic_update_pv_eoi. |
| 236 | */ |
| 237 | #define KVM_APIC_PV_EOI_PENDING 1 |
Gleb Natapov | 4138377 | 2012-04-19 14:06:29 +0300 | [diff] [blame] | 238 | |
Feng Wu | d84f1e0 | 2015-09-18 22:29:49 +0800 | [diff] [blame] | 239 | struct kvm_kernel_irq_routing_entry; |
| 240 | |
Zhang Xiantao | d657a98 | 2007-12-14 09:41:22 +0800 | [diff] [blame] | 241 | /* |
| 242 | * We don't want allocation failures within the mmu code, so we preallocate |
| 243 | * enough memory for a single page fault in a cache. |
| 244 | */ |
| 245 | struct kvm_mmu_memory_cache { |
| 246 | int nobjs; |
| 247 | void *objects[KVM_NR_MEM_OBJS]; |
| 248 | }; |
| 249 | |
Xiao Guangrong | 21ebbed | 2016-02-24 17:51:09 +0800 | [diff] [blame] | 250 | /* |
| 251 | * the pages used as guest page table on soft mmu are tracked by |
| 252 | * kvm_memory_slot.arch.gfn_track which is 16 bits, so the role bits used |
| 253 | * by indirect shadow page can not be more than 15 bits. |
| 254 | * |
| 255 | * Currently, we used 14 bits that are @level, @cr4_pae, @quadrant, @access, |
| 256 | * @nxe, @cr0_wp, @smep_andnot_wp and @smap_andnot_wp. |
| 257 | */ |
Zhang Xiantao | d657a98 | 2007-12-14 09:41:22 +0800 | [diff] [blame] | 258 | union kvm_mmu_page_role { |
Vitaly Kuznetsov | 36d9594d | 2018-10-08 21:28:10 +0200 | [diff] [blame] | 259 | u32 word; |
Zhang Xiantao | d657a98 | 2007-12-14 09:41:22 +0800 | [diff] [blame] | 260 | struct { |
Joe Perches | 7d76b4d | 2008-03-23 01:02:34 -0700 | [diff] [blame] | 261 | unsigned level:4; |
Avi Kivity | 5b7e010 | 2010-04-14 19:20:03 +0300 | [diff] [blame] | 262 | unsigned cr4_pae:1; |
Joe Perches | 7d76b4d | 2008-03-23 01:02:34 -0700 | [diff] [blame] | 263 | unsigned quadrant:2; |
Avi Kivity | f6e2c02b | 2009-01-11 13:02:10 +0200 | [diff] [blame] | 264 | unsigned direct:1; |
Joe Perches | 7d76b4d | 2008-03-23 01:02:34 -0700 | [diff] [blame] | 265 | unsigned access:3; |
Marcelo Tosatti | 2e53d63 | 2008-02-20 14:47:24 -0500 | [diff] [blame] | 266 | unsigned invalid:1; |
Avi Kivity | 9645bb56 | 2009-03-31 11:31:54 +0300 | [diff] [blame] | 267 | unsigned nxe:1; |
Avi Kivity | 3dbe141 | 2010-05-12 11:48:18 +0300 | [diff] [blame] | 268 | unsigned cr0_wp:1; |
Avi Kivity | 411c588 | 2011-06-06 16:11:54 +0300 | [diff] [blame] | 269 | unsigned smep_andnot_wp:1; |
Xiao Guangrong | 0be0226 | 2015-05-11 22:55:21 +0800 | [diff] [blame] | 270 | unsigned smap_andnot_wp:1; |
Peter Feiner | ac8d57e | 2017-06-30 17:26:31 -0700 | [diff] [blame] | 271 | unsigned ad_disabled:1; |
Jim Mattson | 1313cc2 | 2018-05-09 17:02:04 -0400 | [diff] [blame] | 272 | unsigned guest_mode:1; |
| 273 | unsigned :6; |
Paolo Bonzini | 699023e | 2015-05-18 15:03:39 +0200 | [diff] [blame] | 274 | |
| 275 | /* |
| 276 | * This is left at the top of the word so that |
| 277 | * kvm_memslots_for_spte_role can extract it with a |
| 278 | * simple shift. While there is room, give it a whole |
| 279 | * byte so it is also faster to load it from memory. |
| 280 | */ |
| 281 | unsigned smm:8; |
Zhang Xiantao | d657a98 | 2007-12-14 09:41:22 +0800 | [diff] [blame] | 282 | }; |
| 283 | }; |
| 284 | |
Vitaly Kuznetsov | 36d9594d | 2018-10-08 21:28:10 +0200 | [diff] [blame] | 285 | union kvm_mmu_extended_role { |
Vitaly Kuznetsov | a336282 | 2018-10-08 21:28:11 +0200 | [diff] [blame] | 286 | /* |
| 287 | * This structure complements kvm_mmu_page_role caching everything needed for |
| 288 | * MMU configuration. If nothing in both these structures changed, MMU |
| 289 | * re-configuration can be skipped. @valid bit is set on first usage so we don't |
| 290 | * treat all-zero structure as valid data. |
| 291 | */ |
Vitaly Kuznetsov | 36d9594d | 2018-10-08 21:28:10 +0200 | [diff] [blame] | 292 | u32 word; |
Vitaly Kuznetsov | a336282 | 2018-10-08 21:28:11 +0200 | [diff] [blame] | 293 | struct { |
| 294 | unsigned int valid:1; |
| 295 | unsigned int execonly:1; |
Vitaly Kuznetsov | 7dcd575 | 2018-10-08 21:28:12 +0200 | [diff] [blame] | 296 | unsigned int cr0_pg:1; |
Vitaly Kuznetsov | a336282 | 2018-10-08 21:28:11 +0200 | [diff] [blame] | 297 | unsigned int cr4_pse:1; |
| 298 | unsigned int cr4_pke:1; |
| 299 | unsigned int cr4_smap:1; |
| 300 | unsigned int cr4_smep:1; |
Vitaly Kuznetsov | 7dcd575 | 2018-10-08 21:28:12 +0200 | [diff] [blame] | 301 | unsigned int cr4_la57:1; |
Vitaly Kuznetsov | a336282 | 2018-10-08 21:28:11 +0200 | [diff] [blame] | 302 | }; |
Vitaly Kuznetsov | 36d9594d | 2018-10-08 21:28:10 +0200 | [diff] [blame] | 303 | }; |
| 304 | |
| 305 | union kvm_mmu_role { |
| 306 | u64 as_u64; |
| 307 | struct { |
| 308 | union kvm_mmu_page_role base; |
| 309 | union kvm_mmu_extended_role ext; |
| 310 | }; |
| 311 | }; |
| 312 | |
Takuya Yoshikawa | 018aabb5 | 2015-11-20 17:41:28 +0900 | [diff] [blame] | 313 | struct kvm_rmap_head { |
| 314 | unsigned long val; |
| 315 | }; |
| 316 | |
Zhang Xiantao | d657a98 | 2007-12-14 09:41:22 +0800 | [diff] [blame] | 317 | struct kvm_mmu_page { |
| 318 | struct list_head link; |
| 319 | struct hlist_node hash_link; |
Wei Yang | 3ff519f | 2018-09-06 05:58:16 +0800 | [diff] [blame] | 320 | bool unsync; |
Zhang Xiantao | d657a98 | 2007-12-14 09:41:22 +0800 | [diff] [blame] | 321 | |
| 322 | /* |
| 323 | * The following two entries are used to key the shadow page in the |
| 324 | * hash table. |
| 325 | */ |
Zhang Xiantao | d657a98 | 2007-12-14 09:41:22 +0800 | [diff] [blame] | 326 | union kvm_mmu_page_role role; |
Wei Yang | 3ff519f | 2018-09-06 05:58:16 +0800 | [diff] [blame] | 327 | gfn_t gfn; |
Zhang Xiantao | d657a98 | 2007-12-14 09:41:22 +0800 | [diff] [blame] | 328 | |
| 329 | u64 *spt; |
| 330 | /* hold the gfn of each spte inside spt */ |
| 331 | gfn_t *gfns; |
Xiao Guangrong | 0571d36 | 2010-04-16 21:27:54 +0800 | [diff] [blame] | 332 | int root_count; /* Currently serving as active root */ |
Marcelo Tosatti | 60c8aec | 2008-12-01 22:32:02 -0200 | [diff] [blame] | 333 | unsigned int unsync_children; |
Takuya Yoshikawa | 018aabb5 | 2015-11-20 17:41:28 +0900 | [diff] [blame] | 334 | struct kvm_rmap_head parent_ptes; /* rmap pointers to parent sptes */ |
Xiao Guangrong | f6f8ade | 2013-06-19 17:09:24 +0800 | [diff] [blame] | 335 | |
| 336 | /* The page is obsolete if mmu_valid_gen != kvm->arch.mmu_valid_gen. */ |
Xiao Guangrong | 5304b8d | 2013-05-31 08:36:22 +0800 | [diff] [blame] | 337 | unsigned long mmu_valid_gen; |
Xiao Guangrong | f6f8ade | 2013-06-19 17:09:24 +0800 | [diff] [blame] | 338 | |
Marcelo Tosatti | 0074ff6 | 2008-09-23 13:18:40 -0300 | [diff] [blame] | 339 | DECLARE_BITMAP(unsync_child_bitmap, 512); |
Xiao Guangrong | c2a2ac2 | 2011-07-12 03:32:13 +0800 | [diff] [blame] | 340 | |
| 341 | #ifdef CONFIG_X86_32 |
Xiao Guangrong | accaefe | 2013-06-19 17:09:20 +0800 | [diff] [blame] | 342 | /* |
| 343 | * Used out of the mmu-lock to avoid reading spte values while an |
| 344 | * update is in progress; see the comments in __get_spte_lockless(). |
| 345 | */ |
Xiao Guangrong | c2a2ac2 | 2011-07-12 03:32:13 +0800 | [diff] [blame] | 346 | int clear_spte_count; |
| 347 | #endif |
| 348 | |
Xiao Guangrong | 0cbf8e4 | 2013-06-19 17:09:21 +0800 | [diff] [blame] | 349 | /* Number of writes since the last time traversal visited this page. */ |
Xiao Guangrong | e5691a8 | 2016-02-24 17:51:12 +0800 | [diff] [blame] | 350 | atomic_t write_flooding_count; |
Zhang Xiantao | d657a98 | 2007-12-14 09:41:22 +0800 | [diff] [blame] | 351 | }; |
| 352 | |
Avi Kivity | 1c08364 | 2009-01-04 12:39:07 +0200 | [diff] [blame] | 353 | struct kvm_pio_request { |
| 354 | unsigned long count; |
Avi Kivity | 1c08364 | 2009-01-04 12:39:07 +0200 | [diff] [blame] | 355 | int in; |
| 356 | int port; |
| 357 | int size; |
Avi Kivity | 1c08364 | 2009-01-04 12:39:07 +0200 | [diff] [blame] | 358 | }; |
| 359 | |
Yu Zhang | 855feb6 | 2017-08-24 20:27:55 +0800 | [diff] [blame] | 360 | #define PT64_ROOT_MAX_LEVEL 5 |
Yu Zhang | 2a7266a | 2017-08-24 20:27:54 +0800 | [diff] [blame] | 361 | |
Xiao Guangrong | a0a64f5 | 2015-08-05 12:04:21 +0800 | [diff] [blame] | 362 | struct rsvd_bits_validate { |
Yu Zhang | 2a7266a | 2017-08-24 20:27:54 +0800 | [diff] [blame] | 363 | u64 rsvd_bits_mask[2][PT64_ROOT_MAX_LEVEL]; |
Xiao Guangrong | a0a64f5 | 2015-08-05 12:04:21 +0800 | [diff] [blame] | 364 | u64 bad_mt_xwr; |
| 365 | }; |
| 366 | |
Junaid Shahid | 7c390d3 | 2018-06-27 14:59:06 -0700 | [diff] [blame] | 367 | struct kvm_mmu_root_info { |
| 368 | gpa_t cr3; |
| 369 | hpa_t hpa; |
| 370 | }; |
| 371 | |
| 372 | #define KVM_MMU_ROOT_INFO_INVALID \ |
| 373 | ((struct kvm_mmu_root_info) { .cr3 = INVALID_PAGE, .hpa = INVALID_PAGE }) |
| 374 | |
Junaid Shahid | b94742c | 2018-06-27 14:59:20 -0700 | [diff] [blame] | 375 | #define KVM_MMU_NUM_PREV_ROOTS 3 |
| 376 | |
Zhang Xiantao | d657a98 | 2007-12-14 09:41:22 +0800 | [diff] [blame] | 377 | /* |
Yu Zhang | 855feb6 | 2017-08-24 20:27:55 +0800 | [diff] [blame] | 378 | * x86 supports 4 paging modes (5-level 64-bit, 4-level 64-bit, 3-level 32-bit, |
| 379 | * and 2-level 32-bit). The kvm_mmu structure abstracts the details of the |
| 380 | * current mmu mode. |
Zhang Xiantao | d657a98 | 2007-12-14 09:41:22 +0800 | [diff] [blame] | 381 | */ |
| 382 | struct kvm_mmu { |
Joerg Roedel | f43addd | 2010-09-10 17:30:40 +0200 | [diff] [blame] | 383 | void (*set_cr3)(struct kvm_vcpu *vcpu, unsigned long root); |
Joerg Roedel | 5777ed3 | 2010-09-10 17:30:42 +0200 | [diff] [blame] | 384 | unsigned long (*get_cr3)(struct kvm_vcpu *vcpu); |
Avi Kivity | e4e517b | 2011-07-28 11:36:17 +0300 | [diff] [blame] | 385 | u64 (*get_pdptr)(struct kvm_vcpu *vcpu, int index); |
Xiao Guangrong | 78b2c54 | 2010-12-07 10:48:06 +0800 | [diff] [blame] | 386 | int (*page_fault)(struct kvm_vcpu *vcpu, gva_t gva, u32 err, |
| 387 | bool prefault); |
Avi Kivity | 6389ee9 | 2010-11-29 16:12:30 +0200 | [diff] [blame] | 388 | void (*inject_page_fault)(struct kvm_vcpu *vcpu, |
| 389 | struct x86_exception *fault); |
Gleb Natapov | 1871c60 | 2010-02-10 14:21:32 +0200 | [diff] [blame] | 390 | gpa_t (*gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t gva, u32 access, |
Avi Kivity | ab9ae31 | 2010-11-22 17:53:26 +0200 | [diff] [blame] | 391 | struct x86_exception *exception); |
Paolo Bonzini | 54987b7 | 2014-09-02 13:23:06 +0200 | [diff] [blame] | 392 | gpa_t (*translate_gpa)(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access, |
| 393 | struct x86_exception *exception); |
Marcelo Tosatti | e8bc217 | 2008-09-23 13:18:33 -0300 | [diff] [blame] | 394 | int (*sync_page)(struct kvm_vcpu *vcpu, |
Xiao Guangrong | a4a8e6f | 2010-11-19 17:04:03 +0800 | [diff] [blame] | 395 | struct kvm_mmu_page *sp); |
Junaid Shahid | 7eb77e9 | 2018-06-27 14:59:16 -0700 | [diff] [blame] | 396 | void (*invlpg)(struct kvm_vcpu *vcpu, gva_t gva, hpa_t root_hpa); |
Xiao Guangrong | 0f53b5b | 2011-03-09 15:43:51 +0800 | [diff] [blame] | 397 | void (*update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, |
Xiao Guangrong | 7c56252 | 2011-03-28 10:29:27 +0800 | [diff] [blame] | 398 | u64 *spte, const void *pte); |
Zhang Xiantao | d657a98 | 2007-12-14 09:41:22 +0800 | [diff] [blame] | 399 | hpa_t root_hpa; |
Vitaly Kuznetsov | 36d9594d | 2018-10-08 21:28:10 +0200 | [diff] [blame] | 400 | union kvm_mmu_role mmu_role; |
Paolo Bonzini | ae1e2d1 | 2017-03-30 11:55:30 +0200 | [diff] [blame] | 401 | u8 root_level; |
| 402 | u8 shadow_root_level; |
| 403 | u8 ept_ad; |
Joerg Roedel | c5a78f2b | 2010-09-10 17:30:39 +0200 | [diff] [blame] | 404 | bool direct_map; |
Junaid Shahid | b94742c | 2018-06-27 14:59:20 -0700 | [diff] [blame] | 405 | struct kvm_mmu_root_info prev_roots[KVM_MMU_NUM_PREV_ROOTS]; |
Zhang Xiantao | d657a98 | 2007-12-14 09:41:22 +0800 | [diff] [blame] | 406 | |
Avi Kivity | 97d64b7 | 2012-09-12 14:52:00 +0300 | [diff] [blame] | 407 | /* |
| 408 | * Bitmap; bit set = permission fault |
| 409 | * Byte index: page fault error code [4:1] |
| 410 | * Bit index: pte permissions in ACC_* format |
| 411 | */ |
| 412 | u8 permissions[16]; |
| 413 | |
Huaitong Han | 2d34410 | 2016-03-22 16:51:19 +0800 | [diff] [blame] | 414 | /* |
| 415 | * The pkru_mask indicates if protection key checks are needed. It |
| 416 | * consists of 16 domains indexed by page fault error code bits [4:1], |
| 417 | * with PFEC.RSVD replaced by ACC_USER_MASK from the page tables. |
| 418 | * Each domain has 2 bits which are ANDed with AD and WD from PKRU. |
| 419 | */ |
| 420 | u32 pkru_mask; |
| 421 | |
Zhang Xiantao | d657a98 | 2007-12-14 09:41:22 +0800 | [diff] [blame] | 422 | u64 *pae_root; |
Joerg Roedel | 81407ca | 2010-09-10 17:31:00 +0200 | [diff] [blame] | 423 | u64 *lm_root; |
Xiao Guangrong | c258b62 | 2015-08-05 12:04:24 +0800 | [diff] [blame] | 424 | |
| 425 | /* |
| 426 | * check zero bits on shadow page table entries, these |
| 427 | * bits include not only hardware reserved bits but also |
| 428 | * the bits spte never used. |
| 429 | */ |
| 430 | struct rsvd_bits_validate shadow_zero_check; |
| 431 | |
Xiao Guangrong | a0a64f5 | 2015-08-05 12:04:21 +0800 | [diff] [blame] | 432 | struct rsvd_bits_validate guest_rsvd_check; |
Joerg Roedel | ff03a07 | 2010-09-10 17:30:57 +0200 | [diff] [blame] | 433 | |
Paolo Bonzini | 6bb69c9 | 2016-02-23 12:51:19 +0100 | [diff] [blame] | 434 | /* Can have large pages at levels 2..last_nonleaf_level-1. */ |
| 435 | u8 last_nonleaf_level; |
Avi Kivity | 6fd01b7 | 2012-09-12 20:46:56 +0300 | [diff] [blame] | 436 | |
Joerg Roedel | 2d48a98 | 2010-09-10 17:31:01 +0200 | [diff] [blame] | 437 | bool nx; |
| 438 | |
Joerg Roedel | ff03a07 | 2010-09-10 17:30:57 +0200 | [diff] [blame] | 439 | u64 pdptrs[4]; /* pae */ |
Zhang Xiantao | d657a98 | 2007-12-14 09:41:22 +0800 | [diff] [blame] | 440 | }; |
| 441 | |
Gleb Natapov | f5132b0 | 2011-11-10 14:57:22 +0200 | [diff] [blame] | 442 | enum pmc_type { |
| 443 | KVM_PMC_GP = 0, |
| 444 | KVM_PMC_FIXED, |
| 445 | }; |
| 446 | |
| 447 | struct kvm_pmc { |
| 448 | enum pmc_type type; |
| 449 | u8 idx; |
| 450 | u64 counter; |
| 451 | u64 eventsel; |
| 452 | struct perf_event *perf_event; |
| 453 | struct kvm_vcpu *vcpu; |
| 454 | }; |
| 455 | |
| 456 | struct kvm_pmu { |
| 457 | unsigned nr_arch_gp_counters; |
| 458 | unsigned nr_arch_fixed_counters; |
| 459 | unsigned available_event_types; |
| 460 | u64 fixed_ctr_ctrl; |
| 461 | u64 global_ctrl; |
| 462 | u64 global_status; |
| 463 | u64 global_ovf_ctrl; |
| 464 | u64 counter_bitmask[2]; |
| 465 | u64 global_ctrl_mask; |
Andi Kleen | 103af0a | 2013-07-18 15:57:02 -0700 | [diff] [blame] | 466 | u64 reserved_bits; |
Gleb Natapov | f5132b0 | 2011-11-10 14:57:22 +0200 | [diff] [blame] | 467 | u8 version; |
Robert Richter | 15c7ad5 | 2012-06-20 20:46:33 +0200 | [diff] [blame] | 468 | struct kvm_pmc gp_counters[INTEL_PMC_MAX_GENERIC]; |
| 469 | struct kvm_pmc fixed_counters[INTEL_PMC_MAX_FIXED]; |
Gleb Natapov | f5132b0 | 2011-11-10 14:57:22 +0200 | [diff] [blame] | 470 | struct irq_work irq_work; |
| 471 | u64 reprogram_pmi; |
| 472 | }; |
| 473 | |
Wei Huang | 25462f7 | 2015-06-19 15:45:05 +0200 | [diff] [blame] | 474 | struct kvm_pmu_ops; |
| 475 | |
Paolo Bonzini | 360b948 | 2014-02-21 09:55:56 +0100 | [diff] [blame] | 476 | enum { |
| 477 | KVM_DEBUGREG_BP_ENABLED = 1, |
Paolo Bonzini | c77fb5f | 2014-02-21 10:17:24 +0100 | [diff] [blame] | 478 | KVM_DEBUGREG_WONT_EXIT = 2, |
Nadav Amit | ae561ed | 2015-04-02 03:10:37 +0300 | [diff] [blame] | 479 | KVM_DEBUGREG_RELOAD = 4, |
Paolo Bonzini | 360b948 | 2014-02-21 09:55:56 +0100 | [diff] [blame] | 480 | }; |
| 481 | |
Xiao Guangrong | 86fd527 | 2015-06-15 16:55:27 +0800 | [diff] [blame] | 482 | struct kvm_mtrr_range { |
| 483 | u64 base; |
| 484 | u64 mask; |
Xiao Guangrong | 19efffa | 2015-06-15 16:55:31 +0800 | [diff] [blame] | 485 | struct list_head node; |
Xiao Guangrong | 86fd527 | 2015-06-15 16:55:27 +0800 | [diff] [blame] | 486 | }; |
| 487 | |
Xiao Guangrong | 70109e7 | 2015-06-15 16:55:24 +0800 | [diff] [blame] | 488 | struct kvm_mtrr { |
Xiao Guangrong | 86fd527 | 2015-06-15 16:55:27 +0800 | [diff] [blame] | 489 | struct kvm_mtrr_range var_ranges[KVM_NR_VAR_MTRR]; |
Xiao Guangrong | 70109e7 | 2015-06-15 16:55:24 +0800 | [diff] [blame] | 490 | mtrr_type fixed_ranges[KVM_NR_FIXED_MTRR_REGION]; |
Xiao Guangrong | 10fac2d | 2015-06-15 16:55:26 +0800 | [diff] [blame] | 491 | u64 deftype; |
Xiao Guangrong | 19efffa | 2015-06-15 16:55:31 +0800 | [diff] [blame] | 492 | |
| 493 | struct list_head head; |
Xiao Guangrong | 70109e7 | 2015-06-15 16:55:24 +0800 | [diff] [blame] | 494 | }; |
| 495 | |
Andrey Smetanin | 1f4b34f | 2015-11-30 19:22:21 +0300 | [diff] [blame] | 496 | /* Hyper-V SynIC timer */ |
| 497 | struct kvm_vcpu_hv_stimer { |
| 498 | struct hrtimer timer; |
| 499 | int index; |
| 500 | u64 config; |
| 501 | u64 count; |
| 502 | u64 exp_time; |
| 503 | struct hv_message msg; |
| 504 | bool msg_pending; |
| 505 | }; |
| 506 | |
Andrey Smetanin | 5c919412 | 2015-11-10 15:36:34 +0300 | [diff] [blame] | 507 | /* Hyper-V synthetic interrupt controller (SynIC)*/ |
| 508 | struct kvm_vcpu_hv_synic { |
| 509 | u64 version; |
| 510 | u64 control; |
| 511 | u64 msg_page; |
| 512 | u64 evt_page; |
| 513 | atomic64_t sint[HV_SYNIC_SINT_COUNT]; |
| 514 | atomic_t sint_to_gsi[HV_SYNIC_SINT_COUNT]; |
| 515 | DECLARE_BITMAP(auto_eoi_bitmap, 256); |
| 516 | DECLARE_BITMAP(vec_bitmap, 256); |
| 517 | bool active; |
Roman Kagan | efc479e | 2017-06-22 16:51:01 +0300 | [diff] [blame] | 518 | bool dont_zero_synic_pages; |
Andrey Smetanin | 5c919412 | 2015-11-10 15:36:34 +0300 | [diff] [blame] | 519 | }; |
| 520 | |
Andrey Smetanin | e83d588 | 2015-07-03 15:01:34 +0300 | [diff] [blame] | 521 | /* Hyper-V per vcpu emulation context */ |
| 522 | struct kvm_vcpu_hv { |
Roman Kagan | d3457c8 | 2017-07-14 17:13:20 +0300 | [diff] [blame] | 523 | u32 vp_index; |
Andrey Smetanin | e83d588 | 2015-07-03 15:01:34 +0300 | [diff] [blame] | 524 | u64 hv_vapic; |
Andrey Smetanin | 9eec50b | 2015-09-16 12:29:50 +0300 | [diff] [blame] | 525 | s64 runtime_offset; |
Andrey Smetanin | 5c919412 | 2015-11-10 15:36:34 +0300 | [diff] [blame] | 526 | struct kvm_vcpu_hv_synic synic; |
Andrey Smetanin | db397571 | 2015-11-10 15:36:35 +0300 | [diff] [blame] | 527 | struct kvm_hyperv_exit exit; |
Andrey Smetanin | 1f4b34f | 2015-11-30 19:22:21 +0300 | [diff] [blame] | 528 | struct kvm_vcpu_hv_stimer stimer[HV_SYNIC_STIMER_COUNT]; |
| 529 | DECLARE_BITMAP(stimer_pending_bitmap, HV_SYNIC_STIMER_COUNT); |
Vitaly Kuznetsov | e6b6c48 | 2018-10-08 19:19:04 +0200 | [diff] [blame] | 530 | cpumask_t tlb_flush; |
Andrey Smetanin | e83d588 | 2015-07-03 15:01:34 +0300 | [diff] [blame] | 531 | }; |
| 532 | |
Zhang Xiantao | ad312c7 | 2007-12-13 23:50:52 +0800 | [diff] [blame] | 533 | struct kvm_vcpu_arch { |
Marcelo Tosatti | 5fdbf97 | 2008-06-27 14:58:02 -0300 | [diff] [blame] | 534 | /* |
| 535 | * rip and regs accesses must go through |
| 536 | * kvm_{register,rip}_{read,write} functions. |
| 537 | */ |
| 538 | unsigned long regs[NR_VCPU_REGS]; |
| 539 | u32 regs_avail; |
| 540 | u32 regs_dirty; |
Zhang Xiantao | 34c16ee | 2007-10-20 15:34:38 +0800 | [diff] [blame] | 541 | |
| 542 | unsigned long cr0; |
Avi Kivity | e8467fd | 2009-12-29 18:43:06 +0200 | [diff] [blame] | 543 | unsigned long cr0_guest_owned_bits; |
Zhang Xiantao | 34c16ee | 2007-10-20 15:34:38 +0800 | [diff] [blame] | 544 | unsigned long cr2; |
| 545 | unsigned long cr3; |
| 546 | unsigned long cr4; |
Avi Kivity | fc78f51 | 2009-12-07 12:16:48 +0200 | [diff] [blame] | 547 | unsigned long cr4_guest_owned_bits; |
Zhang Xiantao | 34c16ee | 2007-10-20 15:34:38 +0800 | [diff] [blame] | 548 | unsigned long cr8; |
Paolo Bonzini | b9dd21e | 2017-08-23 23:14:38 +0200 | [diff] [blame] | 549 | u32 pkru; |
Alexander Graf | 1371d90 | 2008-11-25 20:17:04 +0100 | [diff] [blame] | 550 | u32 hflags; |
Avi Kivity | f6801df | 2010-01-21 15:31:50 +0200 | [diff] [blame] | 551 | u64 efer; |
Zhang Xiantao | 34c16ee | 2007-10-20 15:34:38 +0800 | [diff] [blame] | 552 | u64 apic_base; |
| 553 | struct kvm_lapic *apic; /* kernel irqchip context */ |
Andrey Smetanin | d62caab | 2015-11-10 15:36:33 +0300 | [diff] [blame] | 554 | bool apicv_active; |
Liran Alon | e40ff1d | 2018-03-21 02:50:31 +0200 | [diff] [blame] | 555 | bool load_eoi_exitmap_pending; |
Andrey Smetanin | 6308630 | 2015-11-10 15:36:32 +0300 | [diff] [blame] | 556 | DECLARE_BITMAP(ioapic_handled_vectors, 256); |
Gleb Natapov | 4138377 | 2012-04-19 14:06:29 +0300 | [diff] [blame] | 557 | unsigned long apic_attention; |
Gleb Natapov | e103571 | 2009-03-05 16:34:59 +0200 | [diff] [blame] | 558 | int32_t apic_arb_prio; |
Zhang Xiantao | 34c16ee | 2007-10-20 15:34:38 +0800 | [diff] [blame] | 559 | int mp_state; |
Zhang Xiantao | 34c16ee | 2007-10-20 15:34:38 +0800 | [diff] [blame] | 560 | u64 ia32_misc_enable_msr; |
Paolo Bonzini | 64d6067 | 2015-05-07 11:36:11 +0200 | [diff] [blame] | 561 | u64 smbase; |
Liran Alon | 52797bf | 2017-11-15 13:43:14 +0200 | [diff] [blame] | 562 | u64 smi_count; |
Avi Kivity | b209749f | 2007-10-22 16:50:39 +0200 | [diff] [blame] | 563 | bool tpr_access_reporting; |
Wanpeng Li | 2030009 | 2014-12-02 19:14:59 +0800 | [diff] [blame] | 564 | u64 ia32_xss; |
Wanpeng Li | 518e7b9 | 2018-02-28 14:03:31 +0800 | [diff] [blame] | 565 | u64 microcode_version; |
Zhang Xiantao | 34c16ee | 2007-10-20 15:34:38 +0800 | [diff] [blame] | 566 | |
Joerg Roedel | 14dfe85 | 2010-09-10 17:30:49 +0200 | [diff] [blame] | 567 | /* |
| 568 | * Paging state of the vcpu |
| 569 | * |
| 570 | * If the vcpu runs in guest mode with two level paging this still saves |
| 571 | * the paging mode of the l1 guest. This context is always used to |
| 572 | * handle faults. |
| 573 | */ |
Vitaly Kuznetsov | 44dd3ff | 2018-10-08 21:28:05 +0200 | [diff] [blame] | 574 | struct kvm_mmu *mmu; |
| 575 | |
| 576 | /* Non-nested MMU for L1 */ |
| 577 | struct kvm_mmu root_mmu; |
Joerg Roedel | 8df25a3 | 2010-09-10 17:30:46 +0200 | [diff] [blame] | 578 | |
Vitaly Kuznetsov | 14c07ad | 2018-10-08 21:28:08 +0200 | [diff] [blame] | 579 | /* L1 MMU when running nested */ |
| 580 | struct kvm_mmu guest_mmu; |
| 581 | |
Joerg Roedel | 8df25a3 | 2010-09-10 17:30:46 +0200 | [diff] [blame] | 582 | /* |
Joerg Roedel | 6539e73 | 2010-09-10 17:30:50 +0200 | [diff] [blame] | 583 | * Paging state of an L2 guest (used for nested npt) |
| 584 | * |
| 585 | * This context will save all necessary information to walk page tables |
| 586 | * of the an L2 guest. This context is only initialized for page table |
| 587 | * walking and not for faulting since we never handle l2 page faults on |
| 588 | * the host. |
| 589 | */ |
| 590 | struct kvm_mmu nested_mmu; |
| 591 | |
| 592 | /* |
Joerg Roedel | 14dfe85 | 2010-09-10 17:30:49 +0200 | [diff] [blame] | 593 | * Pointer to the mmu context currently used for |
| 594 | * gva_to_gpa translations. |
| 595 | */ |
| 596 | struct kvm_mmu *walk_mmu; |
| 597 | |
Xiao Guangrong | 53c07b1 | 2011-05-15 23:26:20 +0800 | [diff] [blame] | 598 | struct kvm_mmu_memory_cache mmu_pte_list_desc_cache; |
Zhang Xiantao | 34c16ee | 2007-10-20 15:34:38 +0800 | [diff] [blame] | 599 | struct kvm_mmu_memory_cache mmu_page_cache; |
| 600 | struct kvm_mmu_memory_cache mmu_page_header_cache; |
| 601 | |
Rik van Riel | f775b13 | 2017-11-14 16:54:23 -0500 | [diff] [blame] | 602 | /* |
| 603 | * QEMU userspace and the guest each have their own FPU state. |
| 604 | * In vcpu_run, we switch between the user and guest FPU contexts. |
| 605 | * While running a VCPU, the VCPU thread will have the guest FPU |
| 606 | * context. |
| 607 | * |
| 608 | * Note that while the PKRU state lives inside the fpu registers, |
| 609 | * it is switched out separately at VMENTER and VMEXIT time. The |
| 610 | * "guest_fpu" state here contains the guest FPU context, with the |
| 611 | * host PRKU bits. |
| 612 | */ |
| 613 | struct fpu user_fpu; |
Sheng Yang | 9891883 | 2010-05-17 17:08:28 +0800 | [diff] [blame] | 614 | struct fpu guest_fpu; |
Rik van Riel | f775b13 | 2017-11-14 16:54:23 -0500 | [diff] [blame] | 615 | |
Dexuan Cui | 2acf923 | 2010-06-10 11:27:12 +0800 | [diff] [blame] | 616 | u64 xcr0; |
Paolo Bonzini | d7876f1 | 2013-10-02 16:06:15 +0200 | [diff] [blame] | 617 | u64 guest_supported_xcr0; |
Paolo Bonzini | 4344ee9 | 2013-10-02 16:06:16 +0200 | [diff] [blame] | 618 | u32 guest_xstate_size; |
Zhang Xiantao | 34c16ee | 2007-10-20 15:34:38 +0800 | [diff] [blame] | 619 | |
Zhang Xiantao | 34c16ee | 2007-10-20 15:34:38 +0800 | [diff] [blame] | 620 | struct kvm_pio_request pio; |
| 621 | void *pio_data; |
| 622 | |
Gleb Natapov | 66fd3f7 | 2009-05-11 13:35:50 +0300 | [diff] [blame] | 623 | u8 event_exit_inst_len; |
| 624 | |
Avi Kivity | 298101d | 2007-11-25 13:41:11 +0200 | [diff] [blame] | 625 | struct kvm_queued_exception { |
| 626 | bool pending; |
Wanpeng Li | 664f8e2 | 2017-08-24 03:35:09 -0700 | [diff] [blame] | 627 | bool injected; |
Avi Kivity | 298101d | 2007-11-25 13:41:11 +0200 | [diff] [blame] | 628 | bool has_error_code; |
| 629 | u8 nr; |
| 630 | u32 error_code; |
Jim Mattson | c851436 | 2018-10-16 14:29:19 -0700 | [diff] [blame^] | 631 | unsigned long payload; |
| 632 | bool has_payload; |
Wanpeng Li | adfe20f | 2017-07-13 18:30:41 -0700 | [diff] [blame] | 633 | u8 nested_apf; |
Avi Kivity | 298101d | 2007-11-25 13:41:11 +0200 | [diff] [blame] | 634 | } exception; |
| 635 | |
Avi Kivity | 937a7ea | 2008-07-03 15:17:01 +0300 | [diff] [blame] | 636 | struct kvm_queued_interrupt { |
Liran Alon | 04140b4 | 2018-03-23 03:01:31 +0300 | [diff] [blame] | 637 | bool injected; |
Gleb Natapov | 66fd3f7 | 2009-05-11 13:35:50 +0300 | [diff] [blame] | 638 | bool soft; |
Avi Kivity | 937a7ea | 2008-07-03 15:17:01 +0300 | [diff] [blame] | 639 | u8 nr; |
| 640 | } interrupt; |
| 641 | |
Zhang Xiantao | 34c16ee | 2007-10-20 15:34:38 +0800 | [diff] [blame] | 642 | int halt_request; /* real mode on Intel only */ |
| 643 | |
| 644 | int cpuid_nent; |
Dan Kenigsberg | 0771671 | 2007-11-21 17:10:04 +0200 | [diff] [blame] | 645 | struct kvm_cpuid_entry2 cpuid_entries[KVM_MAX_CPUID_ENTRIES]; |
Eugene Korenevsky | 5a4f55c | 2015-03-29 23:56:12 +0300 | [diff] [blame] | 646 | |
| 647 | int maxphyaddr; |
| 648 | |
Zhang Xiantao | 34c16ee | 2007-10-20 15:34:38 +0800 | [diff] [blame] | 649 | /* emulate context */ |
| 650 | |
| 651 | struct x86_emulate_ctxt emulate_ctxt; |
Gleb Natapov | 7ae441e | 2011-03-31 12:06:41 +0200 | [diff] [blame] | 652 | bool emulate_regs_need_sync_to_vcpu; |
| 653 | bool emulate_regs_need_sync_from_vcpu; |
Gleb Natapov | 716d51a | 2012-09-03 15:24:26 +0300 | [diff] [blame] | 654 | int (*complete_userspace_io)(struct kvm_vcpu *vcpu); |
Glauber de Oliveira Costa | 1806852 | 2008-02-15 17:52:47 -0200 | [diff] [blame] | 655 | |
| 656 | gpa_t time; |
Gerd Hoffmann | 50d0a0f | 2008-06-03 16:17:31 +0200 | [diff] [blame] | 657 | struct pvclock_vcpu_time_info hv_clock; |
Zachary Amsden | e48672f | 2010-08-19 22:07:23 -1000 | [diff] [blame] | 658 | unsigned int hw_tsc_khz; |
Andy Honig | 0b79459 | 2013-02-20 14:48:10 -0800 | [diff] [blame] | 659 | struct gfn_to_hva_cache pv_time; |
| 660 | bool pv_time_enabled; |
Marcelo Tosatti | 51d59c6 | 2012-08-03 15:57:49 -0300 | [diff] [blame] | 661 | /* set guest stopped flag in pvclock flags field */ |
| 662 | bool pvclock_set_guest_stopped_request; |
Glauber Costa | c9aaa89 | 2011-07-11 15:28:14 -0400 | [diff] [blame] | 663 | |
| 664 | struct { |
| 665 | u64 msr_val; |
| 666 | u64 last_steal; |
Glauber Costa | c9aaa89 | 2011-07-11 15:28:14 -0400 | [diff] [blame] | 667 | struct gfn_to_hva_cache stime; |
| 668 | struct kvm_steal_time steal; |
| 669 | } st; |
| 670 | |
Luiz Capitulino | a545ab6 | 2016-09-07 14:47:19 -0400 | [diff] [blame] | 671 | u64 tsc_offset; |
Zachary Amsden | 1d5f066 | 2010-08-19 22:07:30 -1000 | [diff] [blame] | 672 | u64 last_guest_tsc; |
Zachary Amsden | 6f526ec | 2012-02-03 15:43:54 -0200 | [diff] [blame] | 673 | u64 last_host_tsc; |
Zachary Amsden | 0dd6a6e | 2012-02-03 15:43:56 -0200 | [diff] [blame] | 674 | u64 tsc_offset_adjustment; |
Zachary Amsden | e26101b | 2012-02-03 15:43:57 -0200 | [diff] [blame] | 675 | u64 this_tsc_nsec; |
| 676 | u64 this_tsc_write; |
Tomasz Grabiec | 0d3da0d | 2014-06-24 09:42:43 +0200 | [diff] [blame] | 677 | u64 this_tsc_generation; |
Zachary Amsden | c285545 | 2010-09-18 14:38:15 -1000 | [diff] [blame] | 678 | bool tsc_catchup; |
Zachary Amsden | cc57828 | 2012-02-03 15:43:50 -0200 | [diff] [blame] | 679 | bool tsc_always_catchup; |
| 680 | s8 virtual_tsc_shift; |
| 681 | u32 virtual_tsc_mult; |
| 682 | u32 virtual_tsc_khz; |
Will Auld | ba90463 | 2012-11-29 12:42:50 -0800 | [diff] [blame] | 683 | s64 ia32_tsc_adjust_msr; |
Haozhong Zhang | ad721883 | 2015-10-20 15:39:02 +0800 | [diff] [blame] | 684 | u64 tsc_scaling_ratio; |
Sheng Yang | 3419ffc | 2008-05-15 09:52:48 +0800 | [diff] [blame] | 685 | |
Avi Kivity | 7460fb4a | 2011-09-20 13:43:14 +0300 | [diff] [blame] | 686 | atomic_t nmi_queued; /* unprocessed asynchronous NMIs */ |
| 687 | unsigned nmi_pending; /* NMI queued after currently running handler */ |
| 688 | bool nmi_injected; /* Trying to inject an NMI this entry */ |
Paolo Bonzini | f077825 | 2015-04-01 15:06:40 +0200 | [diff] [blame] | 689 | bool smi_pending; /* SMI queued after currently running handler */ |
Avi Kivity | 9ba075a | 2008-05-26 20:06:35 +0300 | [diff] [blame] | 690 | |
Xiao Guangrong | 70109e7 | 2015-06-15 16:55:24 +0800 | [diff] [blame] | 691 | struct kvm_mtrr mtrr_state; |
Paolo Bonzini | 7cb060a | 2014-06-19 11:40:18 +0200 | [diff] [blame] | 692 | u64 pat; |
Jan Kiszka | 42dbaa5 | 2008-12-15 13:52:10 +0100 | [diff] [blame] | 693 | |
Paolo Bonzini | 360b948 | 2014-02-21 09:55:56 +0100 | [diff] [blame] | 694 | unsigned switch_db_regs; |
Jan Kiszka | 42dbaa5 | 2008-12-15 13:52:10 +0100 | [diff] [blame] | 695 | unsigned long db[KVM_NR_DB_REGS]; |
| 696 | unsigned long dr6; |
| 697 | unsigned long dr7; |
| 698 | unsigned long eff_db[KVM_NR_DB_REGS]; |
Jan Kiszka | c863901 | 2012-09-21 05:42:55 +0200 | [diff] [blame] | 699 | unsigned long guest_debug_dr7; |
Kyle Huey | db2336a | 2017-03-20 01:16:28 -0700 | [diff] [blame] | 700 | u64 msr_platform_info; |
| 701 | u64 msr_misc_features_enables; |
Huang Ying | 890ca9a | 2009-05-11 16:48:15 +0800 | [diff] [blame] | 702 | |
| 703 | u64 mcg_cap; |
| 704 | u64 mcg_status; |
| 705 | u64 mcg_ctl; |
Ashok Raj | c45dcc7 | 2016-06-22 14:59:56 +0800 | [diff] [blame] | 706 | u64 mcg_ext_ctl; |
Huang Ying | 890ca9a | 2009-05-11 16:48:15 +0800 | [diff] [blame] | 707 | u64 *mce_banks; |
Jan Kiszka | 94fe45d | 2009-10-18 13:24:44 +0200 | [diff] [blame] | 708 | |
Xiao Guangrong | bebb106 | 2011-07-12 03:23:20 +0800 | [diff] [blame] | 709 | /* Cache MMIO info */ |
| 710 | u64 mmio_gva; |
| 711 | unsigned access; |
| 712 | gfn_t mmio_gfn; |
David Matlack | 56f17dd | 2014-08-18 15:46:07 -0700 | [diff] [blame] | 713 | u64 mmio_gen; |
Xiao Guangrong | bebb106 | 2011-07-12 03:23:20 +0800 | [diff] [blame] | 714 | |
Gleb Natapov | f5132b0 | 2011-11-10 14:57:22 +0200 | [diff] [blame] | 715 | struct kvm_pmu pmu; |
| 716 | |
Jan Kiszka | 94fe45d | 2009-10-18 13:24:44 +0200 | [diff] [blame] | 717 | /* used for guest single stepping over the given code position */ |
Jan Kiszka | 94fe45d | 2009-10-18 13:24:44 +0200 | [diff] [blame] | 718 | unsigned long singlestep_rip; |
Jan Kiszka | f92653e | 2010-02-23 17:47:55 +0100 | [diff] [blame] | 719 | |
Andrey Smetanin | e83d588 | 2015-07-03 15:01:34 +0300 | [diff] [blame] | 720 | struct kvm_vcpu_hv hyperv; |
Sheng Yang | f5f48ee | 2010-06-30 12:25:15 +0800 | [diff] [blame] | 721 | |
| 722 | cpumask_var_t wbinvd_dirty_mask; |
Gleb Natapov | af585b9 | 2010-10-14 11:22:46 +0200 | [diff] [blame] | 723 | |
Xiao Guangrong | 1cb3f3a | 2011-09-22 17:02:48 +0800 | [diff] [blame] | 724 | unsigned long last_retry_eip; |
| 725 | unsigned long last_retry_addr; |
| 726 | |
Gleb Natapov | af585b9 | 2010-10-14 11:22:46 +0200 | [diff] [blame] | 727 | struct { |
| 728 | bool halted; |
| 729 | gfn_t gfns[roundup_pow_of_two(ASYNC_PF_PER_VCPU)]; |
Gleb Natapov | 344d958 | 2010-10-14 11:22:50 +0200 | [diff] [blame] | 730 | struct gfn_to_hva_cache data; |
| 731 | u64 msr_val; |
Gleb Natapov | 7c90705 | 2010-10-14 11:22:53 +0200 | [diff] [blame] | 732 | u32 id; |
Gleb Natapov | 6adba52 | 2010-10-14 11:22:55 +0200 | [diff] [blame] | 733 | bool send_user_only; |
Wanpeng Li | 1261bfa | 2017-07-13 18:30:40 -0700 | [diff] [blame] | 734 | u32 host_apf_reason; |
Wanpeng Li | adfe20f | 2017-07-13 18:30:41 -0700 | [diff] [blame] | 735 | unsigned long nested_apf_token; |
Wanpeng Li | 52a5c15 | 2017-07-13 18:30:42 -0700 | [diff] [blame] | 736 | bool delivery_as_pf_vmexit; |
Gleb Natapov | af585b9 | 2010-10-14 11:22:46 +0200 | [diff] [blame] | 737 | } apf; |
Boris Ostrovsky | 2b036c6 | 2012-01-09 14:00:35 -0500 | [diff] [blame] | 738 | |
| 739 | /* OSVW MSRs (AMD only) */ |
| 740 | struct { |
| 741 | u64 length; |
| 742 | u64 status; |
| 743 | } osvw; |
Michael S. Tsirkin | ae7a2a3 | 2012-06-24 19:25:07 +0300 | [diff] [blame] | 744 | |
| 745 | struct { |
| 746 | u64 msr_val; |
| 747 | struct gfn_to_hva_cache data; |
| 748 | } pv_eoi; |
Xiao Guangrong | 93c05d3 | 2013-01-13 23:49:07 +0800 | [diff] [blame] | 749 | |
| 750 | /* |
| 751 | * Indicate whether the access faults on its page table in guest |
| 752 | * which is set when fix page fault and used to detect unhandeable |
| 753 | * instruction. |
| 754 | */ |
| 755 | bool write_fault_to_shadow_pgtable; |
Yang Zhang | 25d9208 | 2013-08-06 12:00:32 +0300 | [diff] [blame] | 756 | |
| 757 | /* set at EPT violation at this point */ |
| 758 | unsigned long exit_qualification; |
Srivatsa Vaddagiri | 6aef266 | 2013-08-26 14:18:34 +0530 | [diff] [blame] | 759 | |
| 760 | /* pv related host specific info */ |
| 761 | struct { |
| 762 | bool pv_unhalted; |
| 763 | } pv; |
Steve Rutherford | 7543a63 | 2015-07-29 23:21:41 -0700 | [diff] [blame] | 764 | |
| 765 | int pending_ioapic_eoi; |
Steve Rutherford | 1c1a9ce | 2015-07-30 11:27:16 +0200 | [diff] [blame] | 766 | int pending_external_vector; |
Tom Lendacky | 0f89b20 | 2016-12-14 14:59:23 -0500 | [diff] [blame] | 767 | |
Brijesh Singh | 618232e | 2017-08-17 18:36:57 +0200 | [diff] [blame] | 768 | /* GPA available */ |
Tom Lendacky | 0f89b20 | 2016-12-14 14:59:23 -0500 | [diff] [blame] | 769 | bool gpa_available; |
Brijesh Singh | 618232e | 2017-08-17 18:36:57 +0200 | [diff] [blame] | 770 | gpa_t gpa_val; |
Longpeng(Mike) | de63ad4 | 2017-08-08 12:05:33 +0800 | [diff] [blame] | 771 | |
| 772 | /* be preempted when it's in kernel-mode(cpl=0) */ |
| 773 | bool preempted_in_kernel; |
Paolo Bonzini | c595cee | 2018-07-02 13:07:14 +0200 | [diff] [blame] | 774 | |
| 775 | /* Flush the L1 Data cache for L1TF mitigation on VMENTER */ |
| 776 | bool l1tf_flush_l1d; |
Zhang Xiantao | 34c16ee | 2007-10-20 15:34:38 +0800 | [diff] [blame] | 777 | }; |
| 778 | |
Takuya Yoshikawa | db3fe4e | 2012-02-08 13:02:18 +0900 | [diff] [blame] | 779 | struct kvm_lpage_info { |
Xiao Guangrong | 92f94f1 | 2016-02-24 17:51:06 +0800 | [diff] [blame] | 780 | int disallow_lpage; |
Takuya Yoshikawa | db3fe4e | 2012-02-08 13:02:18 +0900 | [diff] [blame] | 781 | }; |
| 782 | |
| 783 | struct kvm_arch_memory_slot { |
Takuya Yoshikawa | 018aabb5 | 2015-11-20 17:41:28 +0900 | [diff] [blame] | 784 | struct kvm_rmap_head *rmap[KVM_NR_PAGE_SIZES]; |
Takuya Yoshikawa | db3fe4e | 2012-02-08 13:02:18 +0900 | [diff] [blame] | 785 | struct kvm_lpage_info *lpage_info[KVM_NR_PAGE_SIZES - 1]; |
Xiao Guangrong | 21ebbed | 2016-02-24 17:51:09 +0800 | [diff] [blame] | 786 | unsigned short *gfn_track[KVM_PAGE_TRACK_MAX]; |
Takuya Yoshikawa | db3fe4e | 2012-02-08 13:02:18 +0900 | [diff] [blame] | 787 | }; |
| 788 | |
Radim Krčmář | 3548a25 | 2015-02-12 19:41:33 +0100 | [diff] [blame] | 789 | /* |
| 790 | * We use as the mode the number of bits allocated in the LDR for the |
| 791 | * logical processor ID. It happens that these are all powers of two. |
| 792 | * This makes it is very easy to detect cases where the APICs are |
| 793 | * configured for multiple modes; in that case, we cannot use the map and |
| 794 | * hence cannot use kvm_irq_delivery_to_apic_fast either. |
| 795 | */ |
| 796 | #define KVM_APIC_MODE_XAPIC_CLUSTER 4 |
| 797 | #define KVM_APIC_MODE_XAPIC_FLAT 8 |
| 798 | #define KVM_APIC_MODE_X2APIC 16 |
| 799 | |
Gleb Natapov | 1e08ec4 | 2012-09-13 17:19:24 +0300 | [diff] [blame] | 800 | struct kvm_apic_map { |
| 801 | struct rcu_head rcu; |
Radim Krčmář | 3548a25 | 2015-02-12 19:41:33 +0100 | [diff] [blame] | 802 | u8 mode; |
Radim Krčmář | 0ca52e7 | 2016-07-12 22:09:20 +0200 | [diff] [blame] | 803 | u32 max_apic_id; |
Radim Krčmář | e45115b | 2016-07-12 22:09:19 +0200 | [diff] [blame] | 804 | union { |
| 805 | struct kvm_lapic *xapic_flat_map[8]; |
| 806 | struct kvm_lapic *xapic_cluster_map[16][4]; |
| 807 | }; |
Radim Krčmář | 0ca52e7 | 2016-07-12 22:09:20 +0200 | [diff] [blame] | 808 | struct kvm_lapic *phys_map[]; |
Gleb Natapov | 1e08ec4 | 2012-09-13 17:19:24 +0300 | [diff] [blame] | 809 | }; |
| 810 | |
Andrey Smetanin | e83d588 | 2015-07-03 15:01:34 +0300 | [diff] [blame] | 811 | /* Hyper-V emulation context */ |
| 812 | struct kvm_hv { |
Paolo Bonzini | 3f5ad8b | 2016-12-12 10:12:53 +0100 | [diff] [blame] | 813 | struct mutex hv_lock; |
Andrey Smetanin | e83d588 | 2015-07-03 15:01:34 +0300 | [diff] [blame] | 814 | u64 hv_guest_os_id; |
| 815 | u64 hv_hypercall; |
| 816 | u64 hv_tsc_page; |
Andrey Smetanin | e7d9513 | 2015-07-03 15:01:37 +0300 | [diff] [blame] | 817 | |
| 818 | /* Hyper-v based guest crash (NT kernel bugcheck) parameters */ |
| 819 | u64 hv_crash_param[HV_X64_MSR_CRASH_PARAMS]; |
| 820 | u64 hv_crash_ctl; |
Paolo Bonzini | 095cf55 | 2016-02-08 12:54:12 +0100 | [diff] [blame] | 821 | |
| 822 | HV_REFERENCE_TSC_PAGE tsc_ref; |
Roman Kagan | faeb783 | 2018-02-01 16:48:32 +0300 | [diff] [blame] | 823 | |
| 824 | struct idr conn_to_evt; |
Vitaly Kuznetsov | a2e164e | 2018-03-01 15:15:12 +0100 | [diff] [blame] | 825 | |
| 826 | u64 hv_reenlightenment_control; |
| 827 | u64 hv_tsc_emulation_control; |
| 828 | u64 hv_tsc_emulation_status; |
Vitaly Kuznetsov | 87ee613 | 2018-09-26 19:02:56 +0200 | [diff] [blame] | 829 | |
| 830 | /* How many vCPUs have VP index != vCPU index */ |
| 831 | atomic_t num_mismatched_vp_indexes; |
Andrey Smetanin | e83d588 | 2015-07-03 15:01:34 +0300 | [diff] [blame] | 832 | }; |
| 833 | |
Radim Krčmář | 49776fa | 2016-12-16 16:10:02 +0100 | [diff] [blame] | 834 | enum kvm_irqchip_mode { |
| 835 | KVM_IRQCHIP_NONE, |
| 836 | KVM_IRQCHIP_KERNEL, /* created with KVM_CREATE_IRQCHIP */ |
| 837 | KVM_IRQCHIP_SPLIT, /* created with KVM_CAP_SPLIT_IRQCHIP */ |
| 838 | }; |
| 839 | |
Marcelo Tosatti | fef9cce | 2009-12-23 14:35:17 -0200 | [diff] [blame] | 840 | struct kvm_arch { |
Dave Hansen | 49d5ca2 | 2010-08-19 18:11:28 -0700 | [diff] [blame] | 841 | unsigned int n_used_mmu_pages; |
Zhang Xiantao | f05e70a | 2007-12-14 10:01:48 +0800 | [diff] [blame] | 842 | unsigned int n_requested_mmu_pages; |
Dave Hansen | 39de71e | 2010-08-19 18:11:14 -0700 | [diff] [blame] | 843 | unsigned int n_max_mmu_pages; |
Xiao Guangrong | 332b207 | 2011-05-15 23:20:27 +0800 | [diff] [blame] | 844 | unsigned int indirect_shadow_pages; |
Xiao Guangrong | 5304b8d | 2013-05-31 08:36:22 +0800 | [diff] [blame] | 845 | unsigned long mmu_valid_gen; |
Zhang Xiantao | f05e70a | 2007-12-14 10:01:48 +0800 | [diff] [blame] | 846 | struct hlist_head mmu_page_hash[KVM_NUM_MMU_PAGES]; |
| 847 | /* |
| 848 | * Hash table of struct kvm_mmu_page. |
| 849 | */ |
| 850 | struct list_head active_mmu_pages; |
Xiao Guangrong | 365c886 | 2013-05-31 08:36:29 +0800 | [diff] [blame] | 851 | struct list_head zapped_obsolete_pages; |
Xiao Guangrong | 13d268c | 2016-02-24 17:51:16 +0800 | [diff] [blame] | 852 | struct kvm_page_track_notifier_node mmu_sp_tracker; |
Xiao Guangrong | 0eb05bf | 2016-02-24 17:51:13 +0800 | [diff] [blame] | 853 | struct kvm_page_track_notifier_head track_notifier_head; |
Xiao Guangrong | 365c886 | 2013-05-31 08:36:29 +0800 | [diff] [blame] | 854 | |
Ben-Ami Yassour | 4d5c5d0 | 2008-07-28 19:26:26 +0300 | [diff] [blame] | 855 | struct list_head assigned_dev_head; |
Joerg Roedel | 19de40a | 2008-12-03 14:43:34 +0100 | [diff] [blame] | 856 | struct iommu_domain *iommu_domain; |
Alex Williamson | d96eb2c | 2013-10-30 11:02:23 -0600 | [diff] [blame] | 857 | bool iommu_noncoherent; |
Alex Williamson | e0f0bbc | 2013-10-30 11:02:30 -0600 | [diff] [blame] | 858 | #define __KVM_HAVE_ARCH_NONCOHERENT_DMA |
| 859 | atomic_t noncoherent_dma_count; |
Paolo Bonzini | 5544eb9 | 2015-07-07 15:41:58 +0200 | [diff] [blame] | 860 | #define __KVM_HAVE_ARCH_ASSIGNED_DEVICE |
| 861 | atomic_t assigned_device_count; |
Zhang Xiantao | d7deeeb0 | 2007-12-14 10:17:34 +0800 | [diff] [blame] | 862 | struct kvm_pic *vpic; |
| 863 | struct kvm_ioapic *vioapic; |
Sheng Yang | 7837699 | 2008-01-28 05:10:22 +0800 | [diff] [blame] | 864 | struct kvm_pit *vpit; |
Radim Krčmář | 4272013 | 2015-07-01 15:31:49 +0200 | [diff] [blame] | 865 | atomic_t vapics_in_nmi_mode; |
Gleb Natapov | 1e08ec4 | 2012-09-13 17:19:24 +0300 | [diff] [blame] | 866 | struct mutex apic_map_lock; |
| 867 | struct kvm_apic_map *apic_map; |
Zhang Xiantao | bfc6d22 | 2007-12-14 10:20:16 +0800 | [diff] [blame] | 868 | |
Tang Chen | c24ae0d | 2014-09-24 15:57:58 +0800 | [diff] [blame] | 869 | bool apic_access_page_done; |
Glauber de Oliveira Costa | 1806852 | 2008-02-15 17:52:47 -0200 | [diff] [blame] | 870 | |
| 871 | gpa_t wall_clock; |
Sheng Yang | b7ebfb0 | 2008-04-25 21:44:52 +0800 | [diff] [blame] | 872 | |
Wanpeng Li | 4d5422c | 2018-03-12 04:53:02 -0700 | [diff] [blame] | 873 | bool mwait_in_guest; |
Wanpeng Li | caa057a | 2018-03-12 04:53:03 -0700 | [diff] [blame] | 874 | bool hlt_in_guest; |
Wanpeng Li | b31c114 | 2018-03-12 04:53:04 -0700 | [diff] [blame] | 875 | bool pause_in_guest; |
Wanpeng Li | 4d5422c | 2018-03-12 04:53:02 -0700 | [diff] [blame] | 876 | |
Sheng Yang | 5550af4 | 2008-10-15 20:15:06 +0800 | [diff] [blame] | 877 | unsigned long irq_sources_bitmap; |
Glauber Costa | afbcf7a | 2009-10-16 15:28:36 -0400 | [diff] [blame] | 878 | s64 kvmclock_offset; |
Jan Kiszka | 038f8c1 | 2011-02-04 10:49:11 +0100 | [diff] [blame] | 879 | raw_spinlock_t tsc_write_lock; |
Zachary Amsden | f38e098 | 2010-08-19 22:07:20 -1000 | [diff] [blame] | 880 | u64 last_tsc_nsec; |
Zachary Amsden | f38e098 | 2010-08-19 22:07:20 -1000 | [diff] [blame] | 881 | u64 last_tsc_write; |
Zachary Amsden | 5d3cb0f6 | 2012-02-03 15:43:51 -0200 | [diff] [blame] | 882 | u32 last_tsc_khz; |
Zachary Amsden | e26101b | 2012-02-03 15:43:57 -0200 | [diff] [blame] | 883 | u64 cur_tsc_nsec; |
| 884 | u64 cur_tsc_write; |
| 885 | u64 cur_tsc_offset; |
Tomasz Grabiec | 0d3da0d | 2014-06-24 09:42:43 +0200 | [diff] [blame] | 886 | u64 cur_tsc_generation; |
Marcelo Tosatti | b48aa97 | 2012-11-27 23:29:03 -0200 | [diff] [blame] | 887 | int nr_vcpus_matched_tsc; |
Ed Swierk | ffde22a | 2009-10-15 15:21:43 -0700 | [diff] [blame] | 888 | |
Marcelo Tosatti | d828199 | 2012-11-27 23:29:01 -0200 | [diff] [blame] | 889 | spinlock_t pvclock_gtod_sync_lock; |
| 890 | bool use_master_clock; |
| 891 | u64 master_kernel_ns; |
Thomas Gleixner | a5a1d1c | 2016-12-21 20:32:01 +0100 | [diff] [blame] | 892 | u64 master_cycle_now; |
Andrew Jones | 7e44e44 | 2014-02-28 12:52:54 +0100 | [diff] [blame] | 893 | struct delayed_work kvmclock_update_work; |
Andrew Jones | 332967a | 2014-02-28 12:52:55 +0100 | [diff] [blame] | 894 | struct delayed_work kvmclock_sync_work; |
Marcelo Tosatti | d828199 | 2012-11-27 23:29:01 -0200 | [diff] [blame] | 895 | |
Ed Swierk | ffde22a | 2009-10-15 15:21:43 -0700 | [diff] [blame] | 896 | struct kvm_xen_hvm_config xen_hvm_config; |
Gleb Natapov | 55cd8e5 | 2010-01-17 15:51:22 +0200 | [diff] [blame] | 897 | |
Paolo Bonzini | 6ef768f | 2014-11-20 13:45:31 +0100 | [diff] [blame] | 898 | /* reads protected by irq_srcu, writes by irq_lock */ |
| 899 | struct hlist_head mask_notifier_list; |
| 900 | |
Andrey Smetanin | e83d588 | 2015-07-03 15:01:34 +0300 | [diff] [blame] | 901 | struct kvm_hv hyperv; |
Xiao Guangrong | b034cf0 | 2010-12-23 16:08:35 +0800 | [diff] [blame] | 902 | |
| 903 | #ifdef CONFIG_KVM_MMU_AUDIT |
| 904 | int audit_point; |
| 905 | #endif |
Marcelo Tosatti | 54750f2 | 2015-01-20 15:54:52 -0200 | [diff] [blame] | 906 | |
Ladi Prosek | a826faf | 2017-06-26 09:56:43 +0200 | [diff] [blame] | 907 | bool backwards_tsc_observed; |
Marcelo Tosatti | 54750f2 | 2015-01-20 15:54:52 -0200 | [diff] [blame] | 908 | bool boot_vcpu_runs_old_kvmclock; |
Paolo Bonzini | d71ba78 | 2015-07-29 11:56:48 +0200 | [diff] [blame] | 909 | u32 bsp_vcpu_id; |
Nadav Amit | 90de4a1 | 2015-04-13 01:53:41 +0300 | [diff] [blame] | 910 | |
| 911 | u64 disabled_quirks; |
Steve Rutherford | 49df639 | 2015-07-29 23:21:40 -0700 | [diff] [blame] | 912 | |
Radim Krčmář | 49776fa | 2016-12-16 16:10:02 +0100 | [diff] [blame] | 913 | enum kvm_irqchip_mode irqchip_mode; |
Steve Rutherford | b053b2a | 2015-07-29 23:32:35 -0700 | [diff] [blame] | 914 | u8 nr_reserved_ioapic_pins; |
Feng Wu | 52004014 | 2016-01-25 16:53:33 +0800 | [diff] [blame] | 915 | |
| 916 | bool disabled_lapic_found; |
Suravee Suthikulpanit | 44a95da | 2016-05-04 14:09:46 -0500 | [diff] [blame] | 917 | |
Radim Krčmář | 37131313 | 2016-07-12 22:09:27 +0200 | [diff] [blame] | 918 | bool x2apic_format; |
Radim Krčmář | c519265 | 2016-07-12 22:09:28 +0200 | [diff] [blame] | 919 | bool x2apic_broadcast_quirk_disabled; |
Drew Schmitt | 6fbbde9 | 2018-08-20 10:32:15 -0700 | [diff] [blame] | 920 | |
| 921 | bool guest_can_read_msr_platform_info; |
Zhang Xiantao | d69fb81 | 2007-12-14 09:54:20 +0800 | [diff] [blame] | 922 | }; |
| 923 | |
Zhang Xiantao | 0711456 | 2007-12-14 10:23:23 +0800 | [diff] [blame] | 924 | struct kvm_vm_stat { |
Suraj Jitindar Singh | 8a7e75d | 2016-08-02 14:03:22 +1000 | [diff] [blame] | 925 | ulong mmu_shadow_zapped; |
| 926 | ulong mmu_pte_write; |
| 927 | ulong mmu_pte_updated; |
| 928 | ulong mmu_pde_zapped; |
| 929 | ulong mmu_flooded; |
| 930 | ulong mmu_recycled; |
| 931 | ulong mmu_cache_miss; |
| 932 | ulong mmu_unsync; |
| 933 | ulong remote_tlb_flush; |
| 934 | ulong lpages; |
David Matlack | f3414bc | 2016-12-20 15:25:57 -0800 | [diff] [blame] | 935 | ulong max_mmu_page_hash_collisions; |
Zhang Xiantao | 0711456 | 2007-12-14 10:23:23 +0800 | [diff] [blame] | 936 | }; |
| 937 | |
Zhang Xiantao | 77b4c25 | 2007-12-14 09:49:26 +0800 | [diff] [blame] | 938 | struct kvm_vcpu_stat { |
Suraj Jitindar Singh | 8a7e75d | 2016-08-02 14:03:22 +1000 | [diff] [blame] | 939 | u64 pf_fixed; |
| 940 | u64 pf_guest; |
| 941 | u64 tlb_flush; |
| 942 | u64 invlpg; |
Zhang Xiantao | 77b4c25 | 2007-12-14 09:49:26 +0800 | [diff] [blame] | 943 | |
Suraj Jitindar Singh | 8a7e75d | 2016-08-02 14:03:22 +1000 | [diff] [blame] | 944 | u64 exits; |
| 945 | u64 io_exits; |
| 946 | u64 mmio_exits; |
| 947 | u64 signal_exits; |
| 948 | u64 irq_window_exits; |
| 949 | u64 nmi_window_exits; |
Paolo Bonzini | c595cee | 2018-07-02 13:07:14 +0200 | [diff] [blame] | 950 | u64 l1d_flush; |
Suraj Jitindar Singh | 8a7e75d | 2016-08-02 14:03:22 +1000 | [diff] [blame] | 951 | u64 halt_exits; |
| 952 | u64 halt_successful_poll; |
| 953 | u64 halt_attempted_poll; |
| 954 | u64 halt_poll_invalid; |
| 955 | u64 halt_wakeup; |
| 956 | u64 request_irq_exits; |
| 957 | u64 irq_exits; |
| 958 | u64 host_state_reload; |
Suraj Jitindar Singh | 8a7e75d | 2016-08-02 14:03:22 +1000 | [diff] [blame] | 959 | u64 fpu_reload; |
| 960 | u64 insn_emulation; |
| 961 | u64 insn_emulation_fail; |
| 962 | u64 hypercalls; |
| 963 | u64 irq_injections; |
| 964 | u64 nmi_injections; |
Paolo Bonzini | 0f1e261 | 2016-12-17 16:05:19 +0100 | [diff] [blame] | 965 | u64 req_event; |
Zhang Xiantao | 77b4c25 | 2007-12-14 09:49:26 +0800 | [diff] [blame] | 966 | }; |
Zhang Xiantao | ad312c7 | 2007-12-13 23:50:52 +0800 | [diff] [blame] | 967 | |
Joerg Roedel | 8a76d7f | 2011-04-04 12:39:27 +0200 | [diff] [blame] | 968 | struct x86_instruction_info; |
| 969 | |
Will Auld | 8fe8ab4 | 2012-11-29 12:42:12 -0800 | [diff] [blame] | 970 | struct msr_data { |
| 971 | bool host_initiated; |
| 972 | u32 index; |
| 973 | u64 data; |
| 974 | }; |
| 975 | |
Paolo Bonzini | cb5281a | 2014-12-17 18:17:20 +0100 | [diff] [blame] | 976 | struct kvm_lapic_irq { |
| 977 | u32 vector; |
Paolo Bonzini | b7cb223 | 2015-04-21 14:57:05 +0200 | [diff] [blame] | 978 | u16 delivery_mode; |
| 979 | u16 dest_mode; |
| 980 | bool level; |
| 981 | u16 trig_mode; |
Paolo Bonzini | cb5281a | 2014-12-17 18:17:20 +0100 | [diff] [blame] | 982 | u32 shorthand; |
| 983 | u32 dest_id; |
James Sullivan | 93bbf0b | 2015-03-18 19:26:03 -0600 | [diff] [blame] | 984 | bool msi_redir_hint; |
Paolo Bonzini | cb5281a | 2014-12-17 18:17:20 +0100 | [diff] [blame] | 985 | }; |
| 986 | |
Zhang Xiantao | ea4a5ff | 2007-11-19 14:40:47 +0800 | [diff] [blame] | 987 | struct kvm_x86_ops { |
| 988 | int (*cpu_has_kvm_support)(void); /* __init */ |
| 989 | int (*disabled_by_bios)(void); /* __init */ |
Radim Krčmář | 13a34e0 | 2014-08-28 15:13:03 +0200 | [diff] [blame] | 990 | int (*hardware_enable)(void); |
| 991 | void (*hardware_disable)(void); |
Zhang Xiantao | ea4a5ff | 2007-11-19 14:40:47 +0800 | [diff] [blame] | 992 | void (*check_processor_compatibility)(void *rtn); |
| 993 | int (*hardware_setup)(void); /* __init */ |
| 994 | void (*hardware_unsetup)(void); /* __exit */ |
Avi Kivity | 774ead3 | 2007-12-26 13:57:04 +0200 | [diff] [blame] | 995 | bool (*cpu_has_accelerated_tpr)(void); |
Tom Lendacky | bc226f0 | 2018-05-10 22:06:39 +0200 | [diff] [blame] | 996 | bool (*has_emulated_msr)(int index); |
Sheng Yang | 0e85188 | 2009-12-18 16:48:46 +0800 | [diff] [blame] | 997 | void (*cpuid_update)(struct kvm_vcpu *vcpu); |
Zhang Xiantao | ea4a5ff | 2007-11-19 14:40:47 +0800 | [diff] [blame] | 998 | |
Sean Christopherson | 434a1e9 | 2018-03-20 12:17:18 -0700 | [diff] [blame] | 999 | struct kvm *(*vm_alloc)(void); |
| 1000 | void (*vm_free)(struct kvm *); |
Suravee Suthikulpanit | 0354313 | 2016-05-04 14:09:42 -0500 | [diff] [blame] | 1001 | int (*vm_init)(struct kvm *kvm); |
| 1002 | void (*vm_destroy)(struct kvm *kvm); |
| 1003 | |
Zhang Xiantao | ea4a5ff | 2007-11-19 14:40:47 +0800 | [diff] [blame] | 1004 | /* Create, but do not attach this VCPU */ |
| 1005 | struct kvm_vcpu *(*vcpu_create)(struct kvm *kvm, unsigned id); |
| 1006 | void (*vcpu_free)(struct kvm_vcpu *vcpu); |
Nadav Amit | d28bc9d | 2015-04-13 14:34:08 +0300 | [diff] [blame] | 1007 | void (*vcpu_reset)(struct kvm_vcpu *vcpu, bool init_event); |
Zhang Xiantao | ea4a5ff | 2007-11-19 14:40:47 +0800 | [diff] [blame] | 1008 | |
| 1009 | void (*prepare_guest_switch)(struct kvm_vcpu *vcpu); |
| 1010 | void (*vcpu_load)(struct kvm_vcpu *vcpu, int cpu); |
| 1011 | void (*vcpu_put)(struct kvm_vcpu *vcpu); |
Zhang Xiantao | ea4a5ff | 2007-11-19 14:40:47 +0800 | [diff] [blame] | 1012 | |
Paolo Bonzini | a96036b | 2015-11-10 11:55:36 +0100 | [diff] [blame] | 1013 | void (*update_bp_intercept)(struct kvm_vcpu *vcpu); |
Paolo Bonzini | 609e36d | 2015-04-08 15:30:38 +0200 | [diff] [blame] | 1014 | int (*get_msr)(struct kvm_vcpu *vcpu, struct msr_data *msr); |
Will Auld | 8fe8ab4 | 2012-11-29 12:42:12 -0800 | [diff] [blame] | 1015 | int (*set_msr)(struct kvm_vcpu *vcpu, struct msr_data *msr); |
Zhang Xiantao | ea4a5ff | 2007-11-19 14:40:47 +0800 | [diff] [blame] | 1016 | u64 (*get_segment_base)(struct kvm_vcpu *vcpu, int seg); |
| 1017 | void (*get_segment)(struct kvm_vcpu *vcpu, |
| 1018 | struct kvm_segment *var, int seg); |
Izik Eidus | 2e4d265 | 2008-03-24 19:38:34 +0200 | [diff] [blame] | 1019 | int (*get_cpl)(struct kvm_vcpu *vcpu); |
Zhang Xiantao | ea4a5ff | 2007-11-19 14:40:47 +0800 | [diff] [blame] | 1020 | void (*set_segment)(struct kvm_vcpu *vcpu, |
| 1021 | struct kvm_segment *var, int seg); |
| 1022 | void (*get_cs_db_l_bits)(struct kvm_vcpu *vcpu, int *db, int *l); |
Avi Kivity | e8467fd | 2009-12-29 18:43:06 +0200 | [diff] [blame] | 1023 | void (*decache_cr0_guest_bits)(struct kvm_vcpu *vcpu); |
Avi Kivity | aff48ba | 2010-12-05 18:56:11 +0200 | [diff] [blame] | 1024 | void (*decache_cr3)(struct kvm_vcpu *vcpu); |
Zhang Xiantao | ea4a5ff | 2007-11-19 14:40:47 +0800 | [diff] [blame] | 1025 | void (*decache_cr4_guest_bits)(struct kvm_vcpu *vcpu); |
| 1026 | void (*set_cr0)(struct kvm_vcpu *vcpu, unsigned long cr0); |
| 1027 | void (*set_cr3)(struct kvm_vcpu *vcpu, unsigned long cr3); |
Nadav Har'El | 5e1746d | 2011-05-25 23:03:24 +0300 | [diff] [blame] | 1028 | int (*set_cr4)(struct kvm_vcpu *vcpu, unsigned long cr4); |
Zhang Xiantao | ea4a5ff | 2007-11-19 14:40:47 +0800 | [diff] [blame] | 1029 | void (*set_efer)(struct kvm_vcpu *vcpu, u64 efer); |
Gleb Natapov | 89a27f4 | 2010-02-16 10:51:48 +0200 | [diff] [blame] | 1030 | void (*get_idt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt); |
| 1031 | void (*set_idt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt); |
| 1032 | void (*get_gdt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt); |
| 1033 | void (*set_gdt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt); |
Jan Kiszka | 73aaf249e | 2014-01-04 18:47:16 +0100 | [diff] [blame] | 1034 | u64 (*get_dr6)(struct kvm_vcpu *vcpu); |
| 1035 | void (*set_dr6)(struct kvm_vcpu *vcpu, unsigned long value); |
Paolo Bonzini | c77fb5f | 2014-02-21 10:17:24 +0100 | [diff] [blame] | 1036 | void (*sync_dirty_debug_regs)(struct kvm_vcpu *vcpu); |
Gleb Natapov | 020df07 | 2010-04-13 10:05:23 +0300 | [diff] [blame] | 1037 | void (*set_dr7)(struct kvm_vcpu *vcpu, unsigned long value); |
Marcelo Tosatti | 5fdbf97 | 2008-06-27 14:58:02 -0300 | [diff] [blame] | 1038 | void (*cache_reg)(struct kvm_vcpu *vcpu, enum kvm_reg reg); |
Zhang Xiantao | ea4a5ff | 2007-11-19 14:40:47 +0800 | [diff] [blame] | 1039 | unsigned long (*get_rflags)(struct kvm_vcpu *vcpu); |
| 1040 | void (*set_rflags)(struct kvm_vcpu *vcpu, unsigned long rflags); |
| 1041 | |
Wanpeng Li | c2ba05c | 2017-12-12 17:33:03 -0800 | [diff] [blame] | 1042 | void (*tlb_flush)(struct kvm_vcpu *vcpu, bool invalidate_gpa); |
Tianyu Lan | b08660e | 2018-07-19 08:40:17 +0000 | [diff] [blame] | 1043 | int (*tlb_remote_flush)(struct kvm *kvm); |
Zhang Xiantao | ea4a5ff | 2007-11-19 14:40:47 +0800 | [diff] [blame] | 1044 | |
Junaid Shahid | faff875 | 2018-06-29 13:10:05 -0700 | [diff] [blame] | 1045 | /* |
| 1046 | * Flush any TLB entries associated with the given GVA. |
| 1047 | * Does not need to flush GPA->HPA mappings. |
| 1048 | * Can potentially get non-canonical addresses through INVLPGs, which |
| 1049 | * the implementation may choose to ignore if appropriate. |
| 1050 | */ |
| 1051 | void (*tlb_flush_gva)(struct kvm_vcpu *vcpu, gva_t addr); |
Zhang Xiantao | ea4a5ff | 2007-11-19 14:40:47 +0800 | [diff] [blame] | 1052 | |
Avi Kivity | 851ba69 | 2009-08-24 11:10:17 +0300 | [diff] [blame] | 1053 | void (*run)(struct kvm_vcpu *vcpu); |
| 1054 | int (*handle_exit)(struct kvm_vcpu *vcpu); |
Zhang Xiantao | ea4a5ff | 2007-11-19 14:40:47 +0800 | [diff] [blame] | 1055 | void (*skip_emulated_instruction)(struct kvm_vcpu *vcpu); |
Glauber Costa | 2809f5d | 2009-05-12 16:21:05 -0400 | [diff] [blame] | 1056 | void (*set_interrupt_shadow)(struct kvm_vcpu *vcpu, int mask); |
Paolo Bonzini | 37ccdcb | 2014-05-20 14:29:47 +0200 | [diff] [blame] | 1057 | u32 (*get_interrupt_shadow)(struct kvm_vcpu *vcpu); |
Zhang Xiantao | ea4a5ff | 2007-11-19 14:40:47 +0800 | [diff] [blame] | 1058 | void (*patch_hypercall)(struct kvm_vcpu *vcpu, |
| 1059 | unsigned char *hypercall_addr); |
Gleb Natapov | 66fd3f7 | 2009-05-11 13:35:50 +0300 | [diff] [blame] | 1060 | void (*set_irq)(struct kvm_vcpu *vcpu); |
Gleb Natapov | 95ba827313 | 2009-04-21 17:45:08 +0300 | [diff] [blame] | 1061 | void (*set_nmi)(struct kvm_vcpu *vcpu); |
Wanpeng Li | cfcd20e | 2017-07-13 18:30:39 -0700 | [diff] [blame] | 1062 | void (*queue_exception)(struct kvm_vcpu *vcpu); |
Avi Kivity | b463a6f | 2010-07-20 15:06:17 +0300 | [diff] [blame] | 1063 | void (*cancel_injection)(struct kvm_vcpu *vcpu); |
Gleb Natapov | 7864612 | 2009-03-23 12:12:11 +0200 | [diff] [blame] | 1064 | int (*interrupt_allowed)(struct kvm_vcpu *vcpu); |
Gleb Natapov | 95ba827313 | 2009-04-21 17:45:08 +0300 | [diff] [blame] | 1065 | int (*nmi_allowed)(struct kvm_vcpu *vcpu); |
Jan Kiszka | 3cfc309 | 2009-11-12 01:04:25 +0100 | [diff] [blame] | 1066 | bool (*get_nmi_mask)(struct kvm_vcpu *vcpu); |
| 1067 | void (*set_nmi_mask)(struct kvm_vcpu *vcpu, bool masked); |
Jan Kiszka | c9a7953 | 2014-03-07 20:03:15 +0100 | [diff] [blame] | 1068 | void (*enable_nmi_window)(struct kvm_vcpu *vcpu); |
| 1069 | void (*enable_irq_window)(struct kvm_vcpu *vcpu); |
Gleb Natapov | 95ba827313 | 2009-04-21 17:45:08 +0300 | [diff] [blame] | 1070 | void (*update_cr8_intercept)(struct kvm_vcpu *vcpu, int tpr, int irr); |
Suravee Suthikulpanit | b2a05fe | 2017-09-12 10:42:41 -0500 | [diff] [blame] | 1071 | bool (*get_enable_apicv)(struct kvm_vcpu *vcpu); |
Andrey Smetanin | d62caab | 2015-11-10 15:36:33 +0300 | [diff] [blame] | 1072 | void (*refresh_apicv_exec_ctrl)(struct kvm_vcpu *vcpu); |
Yang Zhang | c7c9c56 | 2013-01-25 10:18:51 +0800 | [diff] [blame] | 1073 | void (*hwapic_irr_update)(struct kvm_vcpu *vcpu, int max_irr); |
Paolo Bonzini | 67c9ddd | 2016-05-10 17:01:23 +0200 | [diff] [blame] | 1074 | void (*hwapic_isr_update)(struct kvm_vcpu *vcpu, int isr); |
Liran Alon | e6c67d8 | 2018-09-04 10:56:52 +0300 | [diff] [blame] | 1075 | bool (*guest_apic_has_interrupt)(struct kvm_vcpu *vcpu); |
Andrey Smetanin | 6308630 | 2015-11-10 15:36:32 +0300 | [diff] [blame] | 1076 | void (*load_eoi_exitmap)(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap); |
Jim Mattson | 8d860bb | 2018-05-09 16:56:05 -0400 | [diff] [blame] | 1077 | void (*set_virtual_apic_mode)(struct kvm_vcpu *vcpu); |
Tang Chen | 4256f43 | 2014-09-24 15:57:54 +0800 | [diff] [blame] | 1078 | void (*set_apic_access_page_addr)(struct kvm_vcpu *vcpu, hpa_t hpa); |
Yang Zhang | a20ed54 | 2013-04-11 19:25:15 +0800 | [diff] [blame] | 1079 | void (*deliver_posted_interrupt)(struct kvm_vcpu *vcpu, int vector); |
Paolo Bonzini | 76dfafd5 | 2016-12-19 17:17:11 +0100 | [diff] [blame] | 1080 | int (*sync_pir_to_irr)(struct kvm_vcpu *vcpu); |
Zhang Xiantao | ea4a5ff | 2007-11-19 14:40:47 +0800 | [diff] [blame] | 1081 | int (*set_tss_addr)(struct kvm *kvm, unsigned int addr); |
Sean Christopherson | 2ac52ab | 2018-03-20 12:17:19 -0700 | [diff] [blame] | 1082 | int (*set_identity_map_addr)(struct kvm *kvm, u64 ident_addr); |
Yu Zhang | 855feb6 | 2017-08-24 20:27:55 +0800 | [diff] [blame] | 1083 | int (*get_tdp_level)(struct kvm_vcpu *vcpu); |
Sheng Yang | 4b12f0d | 2009-04-27 20:35:42 +0800 | [diff] [blame] | 1084 | u64 (*get_mt_mask)(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio); |
Sheng Yang | 17cc393 | 2010-01-05 19:02:27 +0800 | [diff] [blame] | 1085 | int (*get_lpage_level)(void); |
Sheng Yang | 4e47c7a | 2009-12-18 16:48:47 +0800 | [diff] [blame] | 1086 | bool (*rdtscp_supported)(void); |
Mao, Junjie | ad756a1 | 2012-07-02 01:18:48 +0000 | [diff] [blame] | 1087 | bool (*invpcid_supported)(void); |
Joerg Roedel | 344f414 | 2009-07-27 16:30:48 +0200 | [diff] [blame] | 1088 | |
Joerg Roedel | 1c97f0a | 2010-09-10 17:30:41 +0200 | [diff] [blame] | 1089 | void (*set_tdp_cr3)(struct kvm_vcpu *vcpu, unsigned long cr3); |
| 1090 | |
Joerg Roedel | d4330ef | 2010-04-22 12:33:11 +0200 | [diff] [blame] | 1091 | void (*set_supported_cpuid)(u32 func, struct kvm_cpuid_entry2 *entry); |
| 1092 | |
Sheng Yang | f5f48ee | 2010-06-30 12:25:15 +0800 | [diff] [blame] | 1093 | bool (*has_wbinvd_exit)(void); |
| 1094 | |
KarimAllah Ahmed | e79f245 | 2018-04-14 05:10:52 +0200 | [diff] [blame] | 1095 | u64 (*read_l1_tsc_offset)(struct kvm_vcpu *vcpu); |
Zachary Amsden | 99e3e30 | 2010-08-19 22:07:17 -1000 | [diff] [blame] | 1096 | void (*write_tsc_offset)(struct kvm_vcpu *vcpu, u64 offset); |
| 1097 | |
Avi Kivity | 586f960 | 2010-11-18 13:09:54 +0200 | [diff] [blame] | 1098 | void (*get_exit_info)(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2); |
Joerg Roedel | 8a76d7f | 2011-04-04 12:39:27 +0200 | [diff] [blame] | 1099 | |
| 1100 | int (*check_intercept)(struct kvm_vcpu *vcpu, |
| 1101 | struct x86_instruction_info *info, |
| 1102 | enum x86_intercept_stage stage); |
Yang Zhang | a547c6d | 2013-04-11 19:25:10 +0800 | [diff] [blame] | 1103 | void (*handle_external_intr)(struct kvm_vcpu *vcpu); |
Liu, Jinsong | da8999d | 2014-02-24 10:55:46 +0000 | [diff] [blame] | 1104 | bool (*mpx_supported)(void); |
Wanpeng Li | 55412b2 | 2014-12-02 19:21:30 +0800 | [diff] [blame] | 1105 | bool (*xsaves_supported)(void); |
Paolo Bonzini | 66336ca | 2016-07-12 10:36:41 +0200 | [diff] [blame] | 1106 | bool (*umip_emulated)(void); |
Jan Kiszka | b6b8a14 | 2014-03-07 20:03:12 +0100 | [diff] [blame] | 1107 | |
| 1108 | int (*check_nested_events)(struct kvm_vcpu *vcpu, bool external_intr); |
Sean Christopherson | d264ee0 | 2018-08-27 15:21:12 -0700 | [diff] [blame] | 1109 | void (*request_immediate_exit)(struct kvm_vcpu *vcpu); |
Radim Krčmář | ae97a3b | 2014-08-21 18:08:06 +0200 | [diff] [blame] | 1110 | |
| 1111 | void (*sched_in)(struct kvm_vcpu *kvm, int cpu); |
Kai Huang | 88178fd | 2015-01-28 10:54:27 +0800 | [diff] [blame] | 1112 | |
| 1113 | /* |
| 1114 | * Arch-specific dirty logging hooks. These hooks are only supposed to |
| 1115 | * be valid if the specific arch has hardware-accelerated dirty logging |
| 1116 | * mechanism. Currently only for PML on VMX. |
| 1117 | * |
| 1118 | * - slot_enable_log_dirty: |
| 1119 | * called when enabling log dirty mode for the slot. |
| 1120 | * - slot_disable_log_dirty: |
| 1121 | * called when disabling log dirty mode for the slot. |
| 1122 | * also called when slot is created with log dirty disabled. |
| 1123 | * - flush_log_dirty: |
| 1124 | * called before reporting dirty_bitmap to userspace. |
| 1125 | * - enable_log_dirty_pt_masked: |
| 1126 | * called when reenabling log dirty for the GFNs in the mask after |
| 1127 | * corresponding bits are cleared in slot->dirty_bitmap. |
| 1128 | */ |
| 1129 | void (*slot_enable_log_dirty)(struct kvm *kvm, |
| 1130 | struct kvm_memory_slot *slot); |
| 1131 | void (*slot_disable_log_dirty)(struct kvm *kvm, |
| 1132 | struct kvm_memory_slot *slot); |
| 1133 | void (*flush_log_dirty)(struct kvm *kvm); |
| 1134 | void (*enable_log_dirty_pt_masked)(struct kvm *kvm, |
| 1135 | struct kvm_memory_slot *slot, |
| 1136 | gfn_t offset, unsigned long mask); |
Bandan Das | bab4165 | 2017-05-05 15:25:13 -0400 | [diff] [blame] | 1137 | int (*write_log_dirty)(struct kvm_vcpu *vcpu); |
| 1138 | |
Wei Huang | 25462f7 | 2015-06-19 15:45:05 +0200 | [diff] [blame] | 1139 | /* pmu operations of sub-arch */ |
| 1140 | const struct kvm_pmu_ops *pmu_ops; |
Feng Wu | efc6440 | 2015-09-18 22:29:51 +0800 | [diff] [blame] | 1141 | |
Feng Wu | bf9f6ac | 2015-09-18 22:29:55 +0800 | [diff] [blame] | 1142 | /* |
| 1143 | * Architecture specific hooks for vCPU blocking due to |
| 1144 | * HLT instruction. |
| 1145 | * Returns for .pre_block(): |
| 1146 | * - 0 means continue to block the vCPU. |
| 1147 | * - 1 means we cannot block the vCPU since some event |
| 1148 | * happens during this period, such as, 'ON' bit in |
| 1149 | * posted-interrupts descriptor is set. |
| 1150 | */ |
| 1151 | int (*pre_block)(struct kvm_vcpu *vcpu); |
| 1152 | void (*post_block)(struct kvm_vcpu *vcpu); |
Suravee Suthikulpanit | d1ed092 | 2016-05-04 14:09:43 -0500 | [diff] [blame] | 1153 | |
| 1154 | void (*vcpu_blocking)(struct kvm_vcpu *vcpu); |
| 1155 | void (*vcpu_unblocking)(struct kvm_vcpu *vcpu); |
| 1156 | |
Feng Wu | efc6440 | 2015-09-18 22:29:51 +0800 | [diff] [blame] | 1157 | int (*update_pi_irte)(struct kvm *kvm, unsigned int host_irq, |
| 1158 | uint32_t guest_irq, bool set); |
Suravee Suthikulpanit | be8ca17 | 2016-05-04 14:09:49 -0500 | [diff] [blame] | 1159 | void (*apicv_post_state_restore)(struct kvm_vcpu *vcpu); |
Yunhong Jiang | ce7a058 | 2016-06-13 14:20:01 -0700 | [diff] [blame] | 1160 | |
| 1161 | int (*set_hv_timer)(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc); |
| 1162 | void (*cancel_hv_timer)(struct kvm_vcpu *vcpu); |
Ashok Raj | c45dcc7 | 2016-06-22 14:59:56 +0800 | [diff] [blame] | 1163 | |
| 1164 | void (*setup_mce)(struct kvm_vcpu *vcpu); |
Ladi Prosek | 0234bf8 | 2017-10-11 16:54:40 +0200 | [diff] [blame] | 1165 | |
Jim Mattson | 8fcc4b5 | 2018-07-10 11:27:20 +0200 | [diff] [blame] | 1166 | int (*get_nested_state)(struct kvm_vcpu *vcpu, |
| 1167 | struct kvm_nested_state __user *user_kvm_nested_state, |
| 1168 | unsigned user_data_size); |
| 1169 | int (*set_nested_state)(struct kvm_vcpu *vcpu, |
| 1170 | struct kvm_nested_state __user *user_kvm_nested_state, |
| 1171 | struct kvm_nested_state *kvm_state); |
Paolo Bonzini | 7f7f1ba | 2018-07-18 18:49:01 +0200 | [diff] [blame] | 1172 | void (*get_vmcs12_pages)(struct kvm_vcpu *vcpu); |
| 1173 | |
Ladi Prosek | 72d7b37 | 2017-10-11 16:54:41 +0200 | [diff] [blame] | 1174 | int (*smi_allowed)(struct kvm_vcpu *vcpu); |
Ladi Prosek | 0234bf8 | 2017-10-11 16:54:40 +0200 | [diff] [blame] | 1175 | int (*pre_enter_smm)(struct kvm_vcpu *vcpu, char *smstate); |
| 1176 | int (*pre_leave_smm)(struct kvm_vcpu *vcpu, u64 smbase); |
Ladi Prosek | cc3d967 | 2017-10-17 16:02:39 +0200 | [diff] [blame] | 1177 | int (*enable_smi_window)(struct kvm_vcpu *vcpu); |
Brijesh Singh | 5acc5c0 | 2017-12-04 10:57:26 -0600 | [diff] [blame] | 1178 | |
| 1179 | int (*mem_enc_op)(struct kvm *kvm, void __user *argp); |
Brijesh Singh | 69eaede | 2017-12-04 10:57:26 -0600 | [diff] [blame] | 1180 | int (*mem_enc_reg_region)(struct kvm *kvm, struct kvm_enc_region *argp); |
| 1181 | int (*mem_enc_unreg_region)(struct kvm *kvm, struct kvm_enc_region *argp); |
Tom Lendacky | 801e459 | 2018-02-21 13:39:51 -0600 | [diff] [blame] | 1182 | |
| 1183 | int (*get_msr_feature)(struct kvm_msr_entry *entry); |
Vitaly Kuznetsov | 57b119d | 2018-10-16 18:50:01 +0200 | [diff] [blame] | 1184 | |
| 1185 | int (*nested_enable_evmcs)(struct kvm_vcpu *vcpu, |
| 1186 | uint16_t *vmcs_version); |
Zhang Xiantao | ea4a5ff | 2007-11-19 14:40:47 +0800 | [diff] [blame] | 1187 | }; |
| 1188 | |
Gleb Natapov | af585b9 | 2010-10-14 11:22:46 +0200 | [diff] [blame] | 1189 | struct kvm_arch_async_pf { |
Gleb Natapov | 7c90705 | 2010-10-14 11:22:53 +0200 | [diff] [blame] | 1190 | u32 token; |
Gleb Natapov | af585b9 | 2010-10-14 11:22:46 +0200 | [diff] [blame] | 1191 | gfn_t gfn; |
Xiao Guangrong | fb67e14 | 2010-12-07 10:35:25 +0800 | [diff] [blame] | 1192 | unsigned long cr3; |
Xiao Guangrong | c4806ac | 2010-11-12 14:49:55 +0800 | [diff] [blame] | 1193 | bool direct_map; |
Gleb Natapov | af585b9 | 2010-10-14 11:22:46 +0200 | [diff] [blame] | 1194 | }; |
| 1195 | |
Zhang Xiantao | 97896d0 | 2007-11-14 20:09:30 +0800 | [diff] [blame] | 1196 | extern struct kvm_x86_ops *kvm_x86_ops; |
| 1197 | |
Sean Christopherson | 434a1e9 | 2018-03-20 12:17:18 -0700 | [diff] [blame] | 1198 | #define __KVM_HAVE_ARCH_VM_ALLOC |
| 1199 | static inline struct kvm *kvm_arch_alloc_vm(void) |
| 1200 | { |
| 1201 | return kvm_x86_ops->vm_alloc(); |
| 1202 | } |
| 1203 | |
| 1204 | static inline void kvm_arch_free_vm(struct kvm *kvm) |
| 1205 | { |
| 1206 | return kvm_x86_ops->vm_free(kvm); |
| 1207 | } |
| 1208 | |
Tianyu Lan | b08660e | 2018-07-19 08:40:17 +0000 | [diff] [blame] | 1209 | #define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLB |
| 1210 | static inline int kvm_arch_flush_remote_tlb(struct kvm *kvm) |
| 1211 | { |
| 1212 | if (kvm_x86_ops->tlb_remote_flush && |
| 1213 | !kvm_x86_ops->tlb_remote_flush(kvm)) |
| 1214 | return 0; |
| 1215 | else |
| 1216 | return -ENOTSUPP; |
| 1217 | } |
| 1218 | |
Zhang Xiantao | 54f1585 | 2007-11-19 15:24:28 +0800 | [diff] [blame] | 1219 | int kvm_mmu_module_init(void); |
| 1220 | void kvm_mmu_module_exit(void); |
| 1221 | |
| 1222 | void kvm_mmu_destroy(struct kvm_vcpu *vcpu); |
| 1223 | int kvm_mmu_create(struct kvm_vcpu *vcpu); |
Xiao Guangrong | 13d268c | 2016-02-24 17:51:16 +0800 | [diff] [blame] | 1224 | void kvm_mmu_init_vm(struct kvm *kvm); |
| 1225 | void kvm_mmu_uninit_vm(struct kvm *kvm); |
Sheng Yang | 7b52345 | 2008-04-25 21:13:50 +0800 | [diff] [blame] | 1226 | void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask, |
Junaid Shahid | f160c7b | 2016-12-06 16:46:16 -0800 | [diff] [blame] | 1227 | u64 dirty_mask, u64 nx_mask, u64 x_mask, u64 p_mask, |
Tom Lendacky | d0ec49d | 2017-07-17 16:10:27 -0500 | [diff] [blame] | 1228 | u64 acc_track_mask, u64 me_mask); |
Zhang Xiantao | 54f1585 | 2007-11-19 15:24:28 +0800 | [diff] [blame] | 1229 | |
Paolo Bonzini | 8a3c1a33 | 2013-10-02 16:56:13 +0200 | [diff] [blame] | 1230 | void kvm_mmu_reset_context(struct kvm_vcpu *vcpu); |
Kai Huang | 1c91cad4 | 2015-01-28 10:54:26 +0800 | [diff] [blame] | 1231 | void kvm_mmu_slot_remove_write_access(struct kvm *kvm, |
| 1232 | struct kvm_memory_slot *memslot); |
Wanpeng Li | 3ea3b7f | 2015-04-03 15:40:25 +0800 | [diff] [blame] | 1233 | void kvm_mmu_zap_collapsible_sptes(struct kvm *kvm, |
Paolo Bonzini | f36f3f2 | 2015-05-18 13:20:23 +0200 | [diff] [blame] | 1234 | const struct kvm_memory_slot *memslot); |
Kai Huang | f4b4b18 | 2015-01-28 10:54:24 +0800 | [diff] [blame] | 1235 | void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm, |
| 1236 | struct kvm_memory_slot *memslot); |
| 1237 | void kvm_mmu_slot_largepage_remove_write_access(struct kvm *kvm, |
| 1238 | struct kvm_memory_slot *memslot); |
| 1239 | void kvm_mmu_slot_set_dirty(struct kvm *kvm, |
| 1240 | struct kvm_memory_slot *memslot); |
| 1241 | void kvm_mmu_clear_dirty_pt_masked(struct kvm *kvm, |
| 1242 | struct kvm_memory_slot *slot, |
| 1243 | gfn_t gfn_offset, unsigned long mask); |
Zhang Xiantao | 54f1585 | 2007-11-19 15:24:28 +0800 | [diff] [blame] | 1244 | void kvm_mmu_zap_all(struct kvm *kvm); |
Paolo Bonzini | 54bf36a | 2015-04-08 15:39:23 +0200 | [diff] [blame] | 1245 | void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, struct kvm_memslots *slots); |
Zhang Xiantao | 3ad82a7 | 2007-11-20 13:11:38 +0800 | [diff] [blame] | 1246 | unsigned int kvm_mmu_calculate_mmu_pages(struct kvm *kvm); |
Zhang Xiantao | 54f1585 | 2007-11-19 15:24:28 +0800 | [diff] [blame] | 1247 | void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int kvm_nr_mmu_pages); |
| 1248 | |
Joerg Roedel | ff03a07 | 2010-09-10 17:30:57 +0200 | [diff] [blame] | 1249 | int load_pdptrs(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, unsigned long cr3); |
Ladi Prosek | 9ed38ffa | 2016-11-30 16:03:10 +0100 | [diff] [blame] | 1250 | bool pdptrs_changed(struct kvm_vcpu *vcpu); |
Joerg Roedel | cc4b687 | 2008-02-07 13:47:43 +0100 | [diff] [blame] | 1251 | |
Marcelo Tosatti | 3200f40 | 2008-03-29 20:17:59 -0300 | [diff] [blame] | 1252 | int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa, |
Avi Kivity | 9f81128 | 2008-03-02 14:06:05 +0200 | [diff] [blame] | 1253 | const void *val, int bytes); |
Marcelo Tosatti | 2f333bc | 2008-02-22 12:21:37 -0500 | [diff] [blame] | 1254 | |
Paolo Bonzini | 6ef768f | 2014-11-20 13:45:31 +0100 | [diff] [blame] | 1255 | struct kvm_irq_mask_notifier { |
| 1256 | void (*func)(struct kvm_irq_mask_notifier *kimn, bool masked); |
| 1257 | int irq; |
| 1258 | struct hlist_node link; |
| 1259 | }; |
| 1260 | |
| 1261 | void kvm_register_irq_mask_notifier(struct kvm *kvm, int irq, |
| 1262 | struct kvm_irq_mask_notifier *kimn); |
| 1263 | void kvm_unregister_irq_mask_notifier(struct kvm *kvm, int irq, |
| 1264 | struct kvm_irq_mask_notifier *kimn); |
| 1265 | void kvm_fire_mask_notifiers(struct kvm *kvm, unsigned irqchip, unsigned pin, |
| 1266 | bool mask); |
| 1267 | |
Marcelo Tosatti | 2f333bc | 2008-02-22 12:21:37 -0500 | [diff] [blame] | 1268 | extern bool tdp_enabled; |
Avi Kivity | 9f81128 | 2008-03-02 14:06:05 +0200 | [diff] [blame] | 1269 | |
Liu, Jinsong | a3e06bb | 2011-09-22 16:55:52 +0800 | [diff] [blame] | 1270 | u64 vcpu_tsc_khz(struct kvm_vcpu *vcpu); |
| 1271 | |
Joerg Roedel | 92a1f12 | 2011-03-25 09:44:51 +0100 | [diff] [blame] | 1272 | /* control of guest tsc rate supported? */ |
| 1273 | extern bool kvm_has_tsc_control; |
Joerg Roedel | 92a1f12 | 2011-03-25 09:44:51 +0100 | [diff] [blame] | 1274 | /* maximum supported tsc_khz for guests */ |
| 1275 | extern u32 kvm_max_guest_tsc_khz; |
Haozhong Zhang | bc9b961 | 2015-10-20 15:39:01 +0800 | [diff] [blame] | 1276 | /* number of bits of the fractional part of the TSC scaling ratio */ |
| 1277 | extern u8 kvm_tsc_scaling_ratio_frac_bits; |
| 1278 | /* maximum allowed value of TSC scaling ratio */ |
| 1279 | extern u64 kvm_max_tsc_scaling_ratio; |
Yunhong Jiang | 64672c9 | 2016-06-13 14:19:59 -0700 | [diff] [blame] | 1280 | /* 1ull << kvm_tsc_scaling_ratio_frac_bits */ |
| 1281 | extern u64 kvm_default_tsc_scaling_ratio; |
Joerg Roedel | 92a1f12 | 2011-03-25 09:44:51 +0100 | [diff] [blame] | 1282 | |
Ashok Raj | c45dcc7 | 2016-06-22 14:59:56 +0800 | [diff] [blame] | 1283 | extern u64 kvm_mce_cap_supported; |
Sheng Yang | 571008d | 2008-01-02 14:49:22 +0800 | [diff] [blame] | 1284 | |
| 1285 | enum emulation_result { |
Paolo Bonzini | ac0a48c | 2013-06-25 18:24:41 +0200 | [diff] [blame] | 1286 | EMULATE_DONE, /* no further processing */ |
| 1287 | EMULATE_USER_EXIT, /* kvm_run ready for userspace exit */ |
Zhang Xiantao | 54f1585 | 2007-11-19 15:24:28 +0800 | [diff] [blame] | 1288 | EMULATE_FAIL, /* can't emulate this instruction */ |
| 1289 | }; |
| 1290 | |
| 1291 | #define EMULTYPE_NO_DECODE (1 << 0) |
| 1292 | #define EMULTYPE_TRAP_UD (1 << 1) |
Gleb Natapov | ba8afb6 | 2009-04-12 13:36:57 +0300 | [diff] [blame] | 1293 | #define EMULTYPE_SKIP (1 << 2) |
Sean Christopherson | 384bf22 | 2018-08-23 13:56:49 -0700 | [diff] [blame] | 1294 | #define EMULTYPE_ALLOW_RETRY (1 << 3) |
| 1295 | #define EMULTYPE_NO_UD_ON_FAIL (1 << 4) |
| 1296 | #define EMULTYPE_VMWARE (1 << 5) |
Sean Christopherson | c60658d | 2018-08-23 13:56:53 -0700 | [diff] [blame] | 1297 | int kvm_emulate_instruction(struct kvm_vcpu *vcpu, int emulation_type); |
| 1298 | int kvm_emulate_instruction_from_buffer(struct kvm_vcpu *vcpu, |
| 1299 | void *insn, int insn_len); |
Sean Christopherson | 35be0ad | 2018-08-23 13:56:47 -0700 | [diff] [blame] | 1300 | |
Joerg Roedel | f2b4b7d | 2008-01-31 14:57:37 +0100 | [diff] [blame] | 1301 | void kvm_enable_efer_bits(u64); |
Jan Kiszka | 384bb78 | 2013-04-20 10:52:36 +0200 | [diff] [blame] | 1302 | bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer); |
Paolo Bonzini | 609e36d | 2015-04-08 15:30:38 +0200 | [diff] [blame] | 1303 | int kvm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr); |
Will Auld | 8fe8ab4 | 2012-11-29 12:42:12 -0800 | [diff] [blame] | 1304 | int kvm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr); |
Zhang Xiantao | 54f1585 | 2007-11-19 15:24:28 +0800 | [diff] [blame] | 1305 | |
| 1306 | struct x86_emulate_ctxt; |
| 1307 | |
Sean Christopherson | dca7f12 | 2018-03-08 08:57:27 -0800 | [diff] [blame] | 1308 | int kvm_fast_pio(struct kvm_vcpu *vcpu, int size, unsigned short port, int in); |
Kyle Huey | 6a908b6 | 2016-11-29 12:40:37 -0800 | [diff] [blame] | 1309 | int kvm_emulate_cpuid(struct kvm_vcpu *vcpu); |
Zhang Xiantao | 54f1585 | 2007-11-19 15:24:28 +0800 | [diff] [blame] | 1310 | int kvm_emulate_halt(struct kvm_vcpu *vcpu); |
Joel Schopp | 5cb5605 | 2015-03-02 13:43:31 -0600 | [diff] [blame] | 1311 | int kvm_vcpu_halt(struct kvm_vcpu *vcpu); |
Sheng Yang | f5f48ee | 2010-06-30 12:25:15 +0800 | [diff] [blame] | 1312 | int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu); |
Zhang Xiantao | 54f1585 | 2007-11-19 15:24:28 +0800 | [diff] [blame] | 1313 | |
Guillaume Thouvenin | 3e6e0aa | 2008-05-27 10:18:46 +0200 | [diff] [blame] | 1314 | void kvm_get_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg); |
Gleb Natapov | c697518 | 2010-02-18 12:15:01 +0200 | [diff] [blame] | 1315 | int kvm_load_segment_descriptor(struct kvm_vcpu *vcpu, u16 selector, int seg); |
Paolo Bonzini | 2b4a273 | 2014-11-24 14:35:24 +0100 | [diff] [blame] | 1316 | void kvm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector); |
Guillaume Thouvenin | 3e6e0aa | 2008-05-27 10:18:46 +0200 | [diff] [blame] | 1317 | |
Kevin Wolf | 7f3d35f | 2012-02-08 14:34:38 +0100 | [diff] [blame] | 1318 | int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index, |
| 1319 | int reason, bool has_error_code, u32 error_code); |
Izik Eidus | 37817f2 | 2008-03-24 23:14:53 +0200 | [diff] [blame] | 1320 | |
Avi Kivity | 49a9b07 | 2010-06-10 17:02:14 +0300 | [diff] [blame] | 1321 | int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0); |
Avi Kivity | 2390218 | 2010-06-10 17:02:16 +0300 | [diff] [blame] | 1322 | int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3); |
Avi Kivity | a83b29c | 2010-06-10 17:02:15 +0300 | [diff] [blame] | 1323 | int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4); |
Andre Przywara | eea1cff | 2010-12-21 11:12:00 +0100 | [diff] [blame] | 1324 | int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8); |
Gleb Natapov | 020df07 | 2010-04-13 10:05:23 +0300 | [diff] [blame] | 1325 | int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val); |
| 1326 | int kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val); |
Avi Kivity | 2d3ad1f | 2008-02-24 11:20:43 +0200 | [diff] [blame] | 1327 | unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu); |
| 1328 | void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw); |
Zhang Xiantao | 54f1585 | 2007-11-19 15:24:28 +0800 | [diff] [blame] | 1329 | void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l); |
Dexuan Cui | 2acf923 | 2010-06-10 11:27:12 +0800 | [diff] [blame] | 1330 | int kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr); |
Zhang Xiantao | 54f1585 | 2007-11-19 15:24:28 +0800 | [diff] [blame] | 1331 | |
Paolo Bonzini | 609e36d | 2015-04-08 15:30:38 +0200 | [diff] [blame] | 1332 | int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr); |
Will Auld | 8fe8ab4 | 2012-11-29 12:42:12 -0800 | [diff] [blame] | 1333 | int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr); |
Zhang Xiantao | 54f1585 | 2007-11-19 15:24:28 +0800 | [diff] [blame] | 1334 | |
Jan Kiszka | 91586a3 | 2009-10-05 13:07:21 +0200 | [diff] [blame] | 1335 | unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu); |
| 1336 | void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags); |
Avi Kivity | 022cd0e | 2011-11-10 14:57:23 +0200 | [diff] [blame] | 1337 | bool kvm_rdpmc(struct kvm_vcpu *vcpu); |
Jan Kiszka | 91586a3 | 2009-10-05 13:07:21 +0200 | [diff] [blame] | 1338 | |
Avi Kivity | 298101d | 2007-11-25 13:41:11 +0200 | [diff] [blame] | 1339 | void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr); |
| 1340 | void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code); |
Joerg Roedel | ce7ddec | 2010-04-22 12:33:13 +0200 | [diff] [blame] | 1341 | void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr); |
| 1342 | void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code); |
Avi Kivity | 6389ee9 | 2010-11-29 16:12:30 +0200 | [diff] [blame] | 1343 | void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault); |
Joerg Roedel | ec92fe4 | 2010-09-10 17:30:51 +0200 | [diff] [blame] | 1344 | int kvm_read_guest_page_mmu(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, |
| 1345 | gfn_t gfn, void *data, int offset, int len, |
| 1346 | u32 access); |
Avi Kivity | 0a79b00 | 2009-09-01 12:03:25 +0300 | [diff] [blame] | 1347 | bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl); |
Nadav Amit | 16f8a6f | 2014-10-03 01:10:05 +0300 | [diff] [blame] | 1348 | bool kvm_require_dr(struct kvm_vcpu *vcpu, int dr); |
Avi Kivity | 298101d | 2007-11-25 13:41:11 +0200 | [diff] [blame] | 1349 | |
Michael S. Tsirkin | 1a577b72 | 2012-07-19 13:45:20 +0300 | [diff] [blame] | 1350 | static inline int __kvm_irq_line_state(unsigned long *irq_state, |
| 1351 | int irq_source_id, int level) |
| 1352 | { |
| 1353 | /* Logical OR for level trig interrupt */ |
| 1354 | if (level) |
| 1355 | __set_bit(irq_source_id, irq_state); |
| 1356 | else |
| 1357 | __clear_bit(irq_source_id, irq_state); |
| 1358 | |
| 1359 | return !!(*irq_state); |
| 1360 | } |
| 1361 | |
Junaid Shahid | b94742c | 2018-06-27 14:59:20 -0700 | [diff] [blame] | 1362 | #define KVM_MMU_ROOT_CURRENT BIT(0) |
| 1363 | #define KVM_MMU_ROOT_PREVIOUS(i) BIT(1+i) |
| 1364 | #define KVM_MMU_ROOTS_ALL (~0UL) |
Junaid Shahid | 08fb59d | 2018-06-27 14:59:17 -0700 | [diff] [blame] | 1365 | |
Michael S. Tsirkin | 1a577b72 | 2012-07-19 13:45:20 +0300 | [diff] [blame] | 1366 | int kvm_pic_set_irq(struct kvm_pic *pic, int irq, int irq_source_id, int level); |
| 1367 | void kvm_pic_clear_all(struct kvm_pic *pic, int irq_source_id); |
Xiantao Zhang | 3de42dc | 2008-10-06 13:48:45 +0800 | [diff] [blame] | 1368 | |
Sheng Yang | 3419ffc | 2008-05-15 09:52:48 +0800 | [diff] [blame] | 1369 | void kvm_inject_nmi(struct kvm_vcpu *vcpu); |
| 1370 | |
Xiao Guangrong | 1cb3f3a | 2011-09-22 17:02:48 +0800 | [diff] [blame] | 1371 | int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn); |
Zhang Xiantao | 54f1585 | 2007-11-19 15:24:28 +0800 | [diff] [blame] | 1372 | int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva); |
| 1373 | void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu); |
| 1374 | int kvm_mmu_load(struct kvm_vcpu *vcpu); |
| 1375 | void kvm_mmu_unload(struct kvm_vcpu *vcpu); |
Marcelo Tosatti | 0ba73cd | 2008-09-23 13:18:34 -0300 | [diff] [blame] | 1376 | void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu); |
Vitaly Kuznetsov | 6a82cd1 | 2018-10-08 21:28:07 +0200 | [diff] [blame] | 1377 | void kvm_mmu_free_roots(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, |
| 1378 | ulong roots_to_free); |
Paolo Bonzini | 54987b7 | 2014-09-02 13:23:06 +0200 | [diff] [blame] | 1379 | gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access, |
| 1380 | struct x86_exception *exception); |
Avi Kivity | ab9ae31 | 2010-11-22 17:53:26 +0200 | [diff] [blame] | 1381 | gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva, |
| 1382 | struct x86_exception *exception); |
| 1383 | gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva, |
| 1384 | struct x86_exception *exception); |
| 1385 | gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva, |
| 1386 | struct x86_exception *exception); |
| 1387 | gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva, |
| 1388 | struct x86_exception *exception); |
Zhang Xiantao | 54f1585 | 2007-11-19 15:24:28 +0800 | [diff] [blame] | 1389 | |
Andrey Smetanin | d62caab | 2015-11-10 15:36:33 +0300 | [diff] [blame] | 1390 | void kvm_vcpu_deactivate_apicv(struct kvm_vcpu *vcpu); |
| 1391 | |
Zhang Xiantao | 54f1585 | 2007-11-19 15:24:28 +0800 | [diff] [blame] | 1392 | int kvm_emulate_hypercall(struct kvm_vcpu *vcpu); |
| 1393 | |
Tom Lendacky | 1472775 | 2016-11-23 12:01:38 -0500 | [diff] [blame] | 1394 | int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t gva, u64 error_code, |
Andre Przywara | dc25e89 | 2010-12-21 11:12:07 +0100 | [diff] [blame] | 1395 | void *insn, int insn_len); |
Marcelo Tosatti | a705289 | 2008-09-23 13:18:35 -0300 | [diff] [blame] | 1396 | void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva); |
Junaid Shahid | eb4b248 | 2018-06-27 14:59:14 -0700 | [diff] [blame] | 1397 | void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid); |
Junaid Shahid | ade61e2 | 2018-06-27 14:59:15 -0700 | [diff] [blame] | 1398 | void kvm_mmu_new_cr3(struct kvm_vcpu *vcpu, gpa_t new_cr3, bool skip_tlb_flush); |
Zhang Xiantao | 34c16ee | 2007-10-20 15:34:38 +0800 | [diff] [blame] | 1399 | |
Joerg Roedel | 1855267 | 2008-02-07 13:47:41 +0100 | [diff] [blame] | 1400 | void kvm_enable_tdp(void); |
Joerg Roedel | 5f4cb66 | 2008-07-14 20:36:36 +0200 | [diff] [blame] | 1401 | void kvm_disable_tdp(void); |
Joerg Roedel | 1855267 | 2008-02-07 13:47:41 +0100 | [diff] [blame] | 1402 | |
Paolo Bonzini | 54987b7 | 2014-09-02 13:23:06 +0200 | [diff] [blame] | 1403 | static inline gpa_t translate_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access, |
| 1404 | struct x86_exception *exception) |
Xiao Guangrong | e459e32 | 2011-11-28 20:42:16 +0800 | [diff] [blame] | 1405 | { |
| 1406 | return gpa; |
| 1407 | } |
| 1408 | |
Zhang Xiantao | ec6d273 | 2007-11-19 15:08:31 +0800 | [diff] [blame] | 1409 | static inline struct kvm_mmu_page *page_header(hpa_t shadow_page) |
| 1410 | { |
| 1411 | struct page *page = pfn_to_page(shadow_page >> PAGE_SHIFT); |
| 1412 | |
| 1413 | return (struct kvm_mmu_page *)page_private(page); |
| 1414 | } |
| 1415 | |
Avi Kivity | d6e88ae | 2008-07-10 16:53:33 +0300 | [diff] [blame] | 1416 | static inline u16 kvm_read_ldt(void) |
Zhang Xiantao | ec6d273 | 2007-11-19 15:08:31 +0800 | [diff] [blame] | 1417 | { |
| 1418 | u16 ldt; |
| 1419 | asm("sldt %0" : "=g"(ldt)); |
| 1420 | return ldt; |
| 1421 | } |
| 1422 | |
Avi Kivity | d6e88ae | 2008-07-10 16:53:33 +0300 | [diff] [blame] | 1423 | static inline void kvm_load_ldt(u16 sel) |
Zhang Xiantao | ec6d273 | 2007-11-19 15:08:31 +0800 | [diff] [blame] | 1424 | { |
| 1425 | asm("lldt %0" : : "rm"(sel)); |
| 1426 | } |
Zhang Xiantao | ec6d273 | 2007-11-19 15:08:31 +0800 | [diff] [blame] | 1427 | |
Zhang Xiantao | ec6d273 | 2007-11-19 15:08:31 +0800 | [diff] [blame] | 1428 | #ifdef CONFIG_X86_64 |
| 1429 | static inline unsigned long read_msr(unsigned long msr) |
| 1430 | { |
| 1431 | u64 value; |
| 1432 | |
| 1433 | rdmsrl(msr, value); |
| 1434 | return value; |
| 1435 | } |
| 1436 | #endif |
| 1437 | |
Zhang Xiantao | ec6d273 | 2007-11-19 15:08:31 +0800 | [diff] [blame] | 1438 | static inline u32 get_rdx_init_val(void) |
| 1439 | { |
| 1440 | return 0x600; /* P6 family */ |
| 1441 | } |
| 1442 | |
Avi Kivity | c1a5d4f | 2007-11-25 14:12:03 +0200 | [diff] [blame] | 1443 | static inline void kvm_inject_gp(struct kvm_vcpu *vcpu, u32 error_code) |
| 1444 | { |
| 1445 | kvm_queue_exception_e(vcpu, GP_VECTOR, error_code); |
| 1446 | } |
| 1447 | |
Zhang Xiantao | ec6d273 | 2007-11-19 15:08:31 +0800 | [diff] [blame] | 1448 | #define TSS_IOPB_BASE_OFFSET 0x66 |
| 1449 | #define TSS_BASE_SIZE 0x68 |
| 1450 | #define TSS_IOPB_SIZE (65536 / 8) |
| 1451 | #define TSS_REDIRECTION_SIZE (256 / 8) |
Joe Perches | 7d76b4d | 2008-03-23 01:02:34 -0700 | [diff] [blame] | 1452 | #define RMODE_TSS_SIZE \ |
| 1453 | (TSS_BASE_SIZE + TSS_REDIRECTION_SIZE + TSS_IOPB_SIZE + 1) |
Hollis Blanchard | 53e0aa7 | 2007-12-03 16:15:26 -0600 | [diff] [blame] | 1454 | |
Izik Eidus | 37817f2 | 2008-03-24 23:14:53 +0200 | [diff] [blame] | 1455 | enum { |
| 1456 | TASK_SWITCH_CALL = 0, |
| 1457 | TASK_SWITCH_IRET = 1, |
| 1458 | TASK_SWITCH_JMP = 2, |
| 1459 | TASK_SWITCH_GATE = 3, |
| 1460 | }; |
| 1461 | |
Alexander Graf | 1371d90 | 2008-11-25 20:17:04 +0100 | [diff] [blame] | 1462 | #define HF_GIF_MASK (1 << 0) |
Alexander Graf | 3d6368e | 2008-11-25 20:17:07 +0100 | [diff] [blame] | 1463 | #define HF_HIF_MASK (1 << 1) |
| 1464 | #define HF_VINTR_MASK (1 << 2) |
Gleb Natapov | 95ba827313 | 2009-04-21 17:45:08 +0300 | [diff] [blame] | 1465 | #define HF_NMI_MASK (1 << 3) |
Gleb Natapov | 44c1143 | 2009-05-11 13:35:52 +0300 | [diff] [blame] | 1466 | #define HF_IRET_MASK (1 << 4) |
Joerg Roedel | ec9e60b | 2010-11-29 17:51:47 +0100 | [diff] [blame] | 1467 | #define HF_GUEST_MASK (1 << 5) /* VCPU is in guest-mode */ |
Paolo Bonzini | f077825 | 2015-04-01 15:06:40 +0200 | [diff] [blame] | 1468 | #define HF_SMM_MASK (1 << 6) |
| 1469 | #define HF_SMM_INSIDE_NMI_MASK (1 << 7) |
Alexander Graf | 1371d90 | 2008-11-25 20:17:04 +0100 | [diff] [blame] | 1470 | |
Paolo Bonzini | 699023e | 2015-05-18 15:03:39 +0200 | [diff] [blame] | 1471 | #define __KVM_VCPU_MULTIPLE_ADDRESS_SPACE |
| 1472 | #define KVM_ADDRESS_SPACE_NUM 2 |
| 1473 | |
| 1474 | #define kvm_arch_vcpu_memslots_id(vcpu) ((vcpu)->arch.hflags & HF_SMM_MASK ? 1 : 0) |
| 1475 | #define kvm_memslots_for_spte_role(kvm, role) __kvm_memslots(kvm, (role).smm) |
Avi Kivity | 4ecac3f | 2008-05-13 13:23:38 +0300 | [diff] [blame] | 1476 | |
| 1477 | /* |
| 1478 | * Hardware virtualization extension instructions may fault if a |
| 1479 | * reboot turns off virtualization while processes are running. |
| 1480 | * Trap the fault and ignore the instruction if that happens. |
| 1481 | */ |
Avi Kivity | b7c4145 | 2010-12-02 17:52:50 +0200 | [diff] [blame] | 1482 | asmlinkage void kvm_spurious_fault(void); |
Avi Kivity | 4ecac3f | 2008-05-13 13:23:38 +0300 | [diff] [blame] | 1483 | |
Avi Kivity | 5e520e6 | 2011-05-15 10:13:12 -0400 | [diff] [blame] | 1484 | #define ____kvm_handle_fault_on_reboot(insn, cleanup_insn) \ |
Avi Kivity | 4ecac3f | 2008-05-13 13:23:38 +0300 | [diff] [blame] | 1485 | "666: " insn "\n\t" \ |
Avi Kivity | b7c4145 | 2010-12-02 17:52:50 +0200 | [diff] [blame] | 1486 | "668: \n\t" \ |
Eduardo Habkost | 18b13e5 | 2008-08-19 20:00:08 -0300 | [diff] [blame] | 1487 | ".pushsection .fixup, \"ax\" \n" \ |
Avi Kivity | 4ecac3f | 2008-05-13 13:23:38 +0300 | [diff] [blame] | 1488 | "667: \n\t" \ |
Avi Kivity | 5e520e6 | 2011-05-15 10:13:12 -0400 | [diff] [blame] | 1489 | cleanup_insn "\n\t" \ |
Avi Kivity | b7c4145 | 2010-12-02 17:52:50 +0200 | [diff] [blame] | 1490 | "cmpb $0, kvm_rebooting \n\t" \ |
| 1491 | "jne 668b \n\t" \ |
Avi Kivity | 8ceed347 | 2008-08-14 21:25:47 +0300 | [diff] [blame] | 1492 | __ASM_SIZE(push) " $666b \n\t" \ |
Avi Kivity | b7c4145 | 2010-12-02 17:52:50 +0200 | [diff] [blame] | 1493 | "call kvm_spurious_fault \n\t" \ |
Avi Kivity | 4ecac3f | 2008-05-13 13:23:38 +0300 | [diff] [blame] | 1494 | ".popsection \n\t" \ |
H. Peter Anvin | 3ee8972 | 2012-04-20 13:41:59 -0700 | [diff] [blame] | 1495 | _ASM_EXTABLE(666b, 667b) |
Avi Kivity | 4ecac3f | 2008-05-13 13:23:38 +0300 | [diff] [blame] | 1496 | |
Avi Kivity | 5e520e6 | 2011-05-15 10:13:12 -0400 | [diff] [blame] | 1497 | #define __kvm_handle_fault_on_reboot(insn) \ |
| 1498 | ____kvm_handle_fault_on_reboot(insn, "") |
| 1499 | |
Andrea Arcangeli | e930bff | 2008-07-25 16:24:52 +0200 | [diff] [blame] | 1500 | #define KVM_ARCH_WANT_MMU_NOTIFIER |
Takuya Yoshikawa | b3ae209 | 2012-07-02 17:56:33 +0900 | [diff] [blame] | 1501 | int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end); |
Andres Lagar-Cavilla | 5712846 | 2014-09-22 14:54:42 -0700 | [diff] [blame] | 1502 | int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end); |
Andrea Arcangeli | 8ee5382 | 2011-01-13 15:47:10 -0800 | [diff] [blame] | 1503 | int kvm_test_age_hva(struct kvm *kvm, unsigned long hva); |
Izik Eidus | 3da0dd4 | 2009-09-23 21:47:18 +0300 | [diff] [blame] | 1504 | void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte); |
Yang Zhang | c7c9c56 | 2013-01-25 10:18:51 +0800 | [diff] [blame] | 1505 | int kvm_cpu_has_injectable_intr(struct kvm_vcpu *v); |
Gleb Natapov | a1b3710 | 2009-07-09 15:33:52 +0300 | [diff] [blame] | 1506 | int kvm_cpu_has_interrupt(struct kvm_vcpu *vcpu); |
| 1507 | int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu); |
Gleb Natapov | 0b71785 | 2009-07-09 15:33:53 +0300 | [diff] [blame] | 1508 | int kvm_cpu_get_interrupt(struct kvm_vcpu *v); |
Nadav Amit | d28bc9d | 2015-04-13 14:34:08 +0300 | [diff] [blame] | 1509 | void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event); |
Tang Chen | 4256f43 | 2014-09-24 15:57:54 +0800 | [diff] [blame] | 1510 | void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu); |
Andrea Arcangeli | e930bff | 2008-07-25 16:24:52 +0200 | [diff] [blame] | 1511 | |
Wanpeng Li | 4180bf1 | 2018-07-23 14:39:54 +0800 | [diff] [blame] | 1512 | int kvm_pv_send_ipi(struct kvm *kvm, unsigned long ipi_bitmap_low, |
Wanpeng Li | bdf7ffc | 2018-08-30 10:03:30 +0800 | [diff] [blame] | 1513 | unsigned long ipi_bitmap_high, u32 min, |
Wanpeng Li | 4180bf1 | 2018-07-23 14:39:54 +0800 | [diff] [blame] | 1514 | unsigned long icr, int op_64_bit); |
| 1515 | |
Paolo Bonzini | 5b76a3c | 2018-08-05 16:07:47 +0200 | [diff] [blame] | 1516 | u64 kvm_get_arch_capabilities(void); |
Avi Kivity | 18863bd | 2009-09-07 11:12:18 +0300 | [diff] [blame] | 1517 | void kvm_define_shared_msr(unsigned index, u32 msr); |
Andy Honig | 8b3c310 | 2014-08-27 11:16:44 -0700 | [diff] [blame] | 1518 | int kvm_set_shared_msr(unsigned index, u64 val, u64 mask); |
Avi Kivity | 18863bd | 2009-09-07 11:12:18 +0300 | [diff] [blame] | 1519 | |
Haozhong Zhang | 35181e8 | 2015-10-20 15:39:03 +0800 | [diff] [blame] | 1520 | u64 kvm_scale_tsc(struct kvm_vcpu *vcpu, u64 tsc); |
Haozhong Zhang | 4ba7653 | 2015-10-20 15:39:07 +0800 | [diff] [blame] | 1521 | u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc); |
Haozhong Zhang | 35181e8 | 2015-10-20 15:39:03 +0800 | [diff] [blame] | 1522 | |
Nadav Amit | 82b3277 | 2014-11-02 11:54:45 +0200 | [diff] [blame] | 1523 | unsigned long kvm_get_linear_rip(struct kvm_vcpu *vcpu); |
Jan Kiszka | f92653e | 2010-02-23 17:47:55 +0100 | [diff] [blame] | 1524 | bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip); |
| 1525 | |
Paolo Bonzini | 2860c4b | 2016-01-07 15:05:10 +0100 | [diff] [blame] | 1526 | void kvm_make_mclock_inprogress_request(struct kvm *kvm); |
| 1527 | void kvm_make_scan_ioapic_request(struct kvm *kvm); |
| 1528 | |
Gleb Natapov | af585b9 | 2010-10-14 11:22:46 +0200 | [diff] [blame] | 1529 | void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu, |
| 1530 | struct kvm_async_pf *work); |
| 1531 | void kvm_arch_async_page_present(struct kvm_vcpu *vcpu, |
| 1532 | struct kvm_async_pf *work); |
Gleb Natapov | 56028d0 | 2010-10-17 18:13:42 +0200 | [diff] [blame] | 1533 | void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, |
| 1534 | struct kvm_async_pf *work); |
Gleb Natapov | 7c90705 | 2010-10-14 11:22:53 +0200 | [diff] [blame] | 1535 | bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu); |
Gleb Natapov | af585b9 | 2010-10-14 11:22:46 +0200 | [diff] [blame] | 1536 | extern bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn); |
| 1537 | |
Kyle Huey | 6affcbe | 2016-11-29 12:40:40 -0800 | [diff] [blame] | 1538 | int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu); |
| 1539 | int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err); |
Sean Christopherson | d264ee0 | 2018-08-27 15:21:12 -0700 | [diff] [blame] | 1540 | void __kvm_request_immediate_exit(struct kvm_vcpu *vcpu); |
Andre Przywara | db8fcef | 2010-12-21 11:12:01 +0100 | [diff] [blame] | 1541 | |
Gleb Natapov | f5132b0 | 2011-11-10 14:57:22 +0200 | [diff] [blame] | 1542 | int kvm_is_in_guest(void); |
| 1543 | |
Paolo Bonzini | 1d8007b | 2015-10-12 13:38:32 +0200 | [diff] [blame] | 1544 | int __x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa, u32 size); |
| 1545 | int x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa, u32 size); |
Paolo Bonzini | d71ba78 | 2015-07-29 11:56:48 +0200 | [diff] [blame] | 1546 | bool kvm_vcpu_is_reset_bsp(struct kvm_vcpu *vcpu); |
| 1547 | bool kvm_vcpu_is_bsp(struct kvm_vcpu *vcpu); |
Gleb Natapov | f5132b0 | 2011-11-10 14:57:22 +0200 | [diff] [blame] | 1548 | |
Feng Wu | 8feb4a0 | 2015-09-18 22:29:47 +0800 | [diff] [blame] | 1549 | bool kvm_intr_is_single_vcpu(struct kvm *kvm, struct kvm_lapic_irq *irq, |
| 1550 | struct kvm_vcpu **dest_vcpu); |
| 1551 | |
Radim Krčmář | 37131313 | 2016-07-12 22:09:27 +0200 | [diff] [blame] | 1552 | void kvm_set_msi_irq(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e, |
Feng Wu | d84f1e0 | 2015-09-18 22:29:49 +0800 | [diff] [blame] | 1553 | struct kvm_lapic_irq *irq); |
Paolo Bonzini | 197a4f4 | 2015-11-04 16:24:17 +0100 | [diff] [blame] | 1554 | |
Suravee Suthikulpanit | d1ed092 | 2016-05-04 14:09:43 -0500 | [diff] [blame] | 1555 | static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) |
| 1556 | { |
| 1557 | if (kvm_x86_ops->vcpu_blocking) |
| 1558 | kvm_x86_ops->vcpu_blocking(vcpu); |
| 1559 | } |
| 1560 | |
| 1561 | static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) |
| 1562 | { |
| 1563 | if (kvm_x86_ops->vcpu_unblocking) |
| 1564 | kvm_x86_ops->vcpu_unblocking(vcpu); |
| 1565 | } |
| 1566 | |
Christian Borntraeger | 3491caf | 2016-05-13 12:16:35 +0200 | [diff] [blame] | 1567 | static inline void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu) {} |
Christoffer Dall | 3217f7c | 2015-08-27 16:41:15 +0200 | [diff] [blame] | 1568 | |
Suravee Suthikulpanit | 7d669f5 | 2016-06-15 17:23:45 -0500 | [diff] [blame] | 1569 | static inline int kvm_cpu_get_apicid(int mps_cpu) |
| 1570 | { |
| 1571 | #ifdef CONFIG_X86_LOCAL_APIC |
Thomas Gleixner | 6406350 | 2017-09-13 23:29:21 +0200 | [diff] [blame] | 1572 | return default_cpu_present_to_apicid(mps_cpu); |
Suravee Suthikulpanit | 7d669f5 | 2016-06-15 17:23:45 -0500 | [diff] [blame] | 1573 | #else |
| 1574 | WARN_ON_ONCE(1); |
| 1575 | return BAD_APICID; |
| 1576 | #endif |
| 1577 | } |
| 1578 | |
Ladi Prosek | 05cade7 | 2017-10-11 16:54:45 +0200 | [diff] [blame] | 1579 | #define put_smstate(type, buf, offset, val) \ |
| 1580 | *(type *)((buf) + (offset) - 0x7e00) = val |
| 1581 | |
H. Peter Anvin | 1965aae | 2008-10-22 22:26:29 -0700 | [diff] [blame] | 1582 | #endif /* _ASM_X86_KVM_HOST_H */ |