Suravee Suthikulpanit | 44a95da | 2016-05-04 14:09:46 -0500 | [diff] [blame] | 1 | #define pr_fmt(fmt) "SVM: " fmt |
| 2 | |
Avi Kivity | edf8841 | 2007-12-16 11:02:48 +0200 | [diff] [blame] | 3 | #include <linux/kvm_host.h> |
| 4 | |
Eddie Dong | 85f455f | 2007-07-06 12:20:49 +0300 | [diff] [blame] | 5 | #include "irq.h" |
Zhang Xiantao | 1d737c8 | 2007-12-14 09:35:10 +0800 | [diff] [blame] | 6 | #include "mmu.h" |
Marcelo Tosatti | 5fdbf97 | 2008-06-27 14:58:02 -0300 | [diff] [blame] | 7 | #include "kvm_cache_regs.h" |
Gleb Natapov | fe4c7b1 | 2009-03-23 11:23:18 +0200 | [diff] [blame] | 8 | #include "x86.h" |
Julian Stecklina | 66f7b72 | 2012-12-05 15:26:19 +0100 | [diff] [blame] | 9 | #include "cpuid.h" |
Wei Huang | 25462f7 | 2015-06-19 15:45:05 +0200 | [diff] [blame] | 10 | #include "pmu.h" |
Avi Kivity | e495606 | 2007-06-28 14:15:57 -0400 | [diff] [blame] | 11 | |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 12 | #include <linux/module.h> |
Josh Triplett | ae75954 | 2012-03-28 11:32:28 -0700 | [diff] [blame] | 13 | #include <linux/mod_devicetable.h> |
Ahmed S. Darwish | 9d8f549 | 2007-02-19 14:37:46 +0200 | [diff] [blame] | 14 | #include <linux/kernel.h> |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 15 | #include <linux/vmalloc.h> |
| 16 | #include <linux/highmem.h> |
Joerg Roedel | ef0f649 | 2020-03-31 12:17:38 -0400 | [diff] [blame] | 17 | #include <linux/amd-iommu.h> |
Alexey Dobriyan | e8edc6e | 2007-05-21 01:22:52 +0400 | [diff] [blame] | 18 | #include <linux/sched.h> |
Steven Rostedt (Red Hat) | af658dc | 2015-04-29 14:36:05 -0400 | [diff] [blame] | 19 | #include <linux/trace_events.h> |
Tejun Heo | 5a0e3ad | 2010-03-24 17:04:11 +0900 | [diff] [blame] | 20 | #include <linux/slab.h> |
Suravee Suthikulpanit | 5881f73 | 2016-08-23 13:52:42 -0500 | [diff] [blame] | 21 | #include <linux/hashtable.h> |
Josh Poimboeuf | c207aee | 2017-06-28 10:11:06 -0500 | [diff] [blame] | 22 | #include <linux/frame.h> |
Brijesh Singh | e9df094 | 2017-12-04 10:57:33 -0600 | [diff] [blame] | 23 | #include <linux/psp-sev.h> |
Brijesh Singh | 1654efc | 2017-12-04 10:57:34 -0600 | [diff] [blame] | 24 | #include <linux/file.h> |
Brijesh Singh | 89c5058 | 2017-12-04 10:57:35 -0600 | [diff] [blame] | 25 | #include <linux/pagemap.h> |
| 26 | #include <linux/swap.h> |
Tom Lendacky | 33af3a7 | 2019-10-03 21:17:48 +0000 | [diff] [blame] | 27 | #include <linux/rwsem.h> |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 28 | |
Suravee Suthikulpanit | 8221c13 | 2016-05-04 14:09:52 -0500 | [diff] [blame] | 29 | #include <asm/apic.h> |
Joerg Roedel | 1018faa | 2012-02-29 14:57:32 +0100 | [diff] [blame] | 30 | #include <asm/perf_event.h> |
Joerg Roedel | 67ec660 | 2010-05-17 14:43:35 +0200 | [diff] [blame] | 31 | #include <asm/tlbflush.h> |
Avi Kivity | e495606 | 2007-06-28 14:15:57 -0400 | [diff] [blame] | 32 | #include <asm/desc.h> |
Paolo Bonzini | facb013 | 2014-02-21 10:32:27 +0100 | [diff] [blame] | 33 | #include <asm/debugreg.h> |
Gleb Natapov | 631bc48 | 2010-10-14 11:22:52 +0200 | [diff] [blame] | 34 | #include <asm/kvm_para.h> |
Suravee Suthikulpanit | 411b44b | 2016-08-23 13:52:43 -0500 | [diff] [blame] | 35 | #include <asm/irq_remapping.h> |
Uros Bizjak | 1c164cb | 2020-04-11 17:36:27 +0200 | [diff] [blame] | 36 | #include <asm/mce.h> |
Thomas Gleixner | 28a2775 | 2018-04-29 15:01:37 +0200 | [diff] [blame] | 37 | #include <asm/spec-ctrl.h> |
Thomas Gleixner | ba5bade | 2020-03-20 14:13:46 +0100 | [diff] [blame] | 38 | #include <asm/cpu_device_id.h> |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 39 | |
Eduardo Habkost | 63d1142 | 2008-11-17 19:03:20 -0200 | [diff] [blame] | 40 | #include <asm/virtext.h> |
Marcelo Tosatti | 229456f | 2009-06-17 09:22:14 -0300 | [diff] [blame] | 41 | #include "trace.h" |
Eduardo Habkost | 63d1142 | 2008-11-17 19:03:20 -0200 | [diff] [blame] | 42 | |
Joerg Roedel | 883b0a9 | 2020-03-24 10:41:52 +0100 | [diff] [blame] | 43 | #include "svm.h" |
| 44 | |
Avi Kivity | 4ecac3f | 2008-05-13 13:23:38 +0300 | [diff] [blame] | 45 | #define __ex(x) __kvm_handle_fault_on_reboot(x) |
| 46 | |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 47 | MODULE_AUTHOR("Qumranet"); |
| 48 | MODULE_LICENSE("GPL"); |
| 49 | |
Valdis Klētnieks | 575b255 | 2020-02-27 21:49:52 -0500 | [diff] [blame] | 50 | #ifdef MODULE |
Josh Triplett | ae75954 | 2012-03-28 11:32:28 -0700 | [diff] [blame] | 51 | static const struct x86_cpu_id svm_cpu_id[] = { |
Thomas Gleixner | 320debe | 2020-03-20 14:13:50 +0100 | [diff] [blame] | 52 | X86_MATCH_FEATURE(X86_FEATURE_SVM, NULL), |
Josh Triplett | ae75954 | 2012-03-28 11:32:28 -0700 | [diff] [blame] | 53 | {} |
| 54 | }; |
| 55 | MODULE_DEVICE_TABLE(x86cpu, svm_cpu_id); |
Valdis Klētnieks | 575b255 | 2020-02-27 21:49:52 -0500 | [diff] [blame] | 56 | #endif |
Josh Triplett | ae75954 | 2012-03-28 11:32:28 -0700 | [diff] [blame] | 57 | |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 58 | #define IOPM_ALLOC_ORDER 2 |
| 59 | #define MSRPM_ALLOC_ORDER 1 |
| 60 | |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 61 | #define SEG_TYPE_LDT 2 |
| 62 | #define SEG_TYPE_BUSY_TSS16 3 |
| 63 | |
Andre Przywara | 6bc31bd | 2010-04-11 23:07:28 +0200 | [diff] [blame] | 64 | #define SVM_FEATURE_LBRV (1 << 1) |
| 65 | #define SVM_FEATURE_SVML (1 << 2) |
Andre Przywara | ddce97a | 2010-12-21 11:12:03 +0100 | [diff] [blame] | 66 | #define SVM_FEATURE_TSC_RATE (1 << 4) |
| 67 | #define SVM_FEATURE_VMCB_CLEAN (1 << 5) |
| 68 | #define SVM_FEATURE_FLUSH_ASID (1 << 6) |
| 69 | #define SVM_FEATURE_DECODE_ASSIST (1 << 7) |
Andre Przywara | 6bc31bd | 2010-04-11 23:07:28 +0200 | [diff] [blame] | 70 | #define SVM_FEATURE_PAUSE_FILTER (1 << 10) |
Joerg Roedel | 80b7706 | 2007-03-30 17:02:14 +0300 | [diff] [blame] | 71 | |
Joerg Roedel | 24e09cb | 2008-02-13 18:58:47 +0100 | [diff] [blame] | 72 | #define DEBUGCTL_RESERVED_BITS (~(0x3fULL)) |
| 73 | |
Joerg Roedel | fbc0db7 | 2011-03-25 09:44:46 +0100 | [diff] [blame] | 74 | #define TSC_RATIO_RSVD 0xffffff0000000000ULL |
Joerg Roedel | 92a1f12 | 2011-03-25 09:44:51 +0100 | [diff] [blame] | 75 | #define TSC_RATIO_MIN 0x0000000000000001ULL |
| 76 | #define TSC_RATIO_MAX 0x000000ffffffffffULL |
Joerg Roedel | fbc0db7 | 2011-03-25 09:44:46 +0100 | [diff] [blame] | 77 | |
Joerg Roedel | 67ec660 | 2010-05-17 14:43:35 +0200 | [diff] [blame] | 78 | static bool erratum_383_found __read_mostly; |
| 79 | |
Joerg Roedel | 883b0a9 | 2020-03-24 10:41:52 +0100 | [diff] [blame] | 80 | u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly; |
Joerg Roedel | 323c3d8 | 2010-03-01 15:34:37 +0100 | [diff] [blame] | 81 | |
Boris Ostrovsky | 2b036c6 | 2012-01-09 14:00:35 -0500 | [diff] [blame] | 82 | /* |
| 83 | * Set osvw_len to higher value when updated Revision Guides |
| 84 | * are published and we know what the new status bits are |
| 85 | */ |
| 86 | static uint64_t osvw_len = 4, osvw_status; |
| 87 | |
Joerg Roedel | fbc0db7 | 2011-03-25 09:44:46 +0100 | [diff] [blame] | 88 | static DEFINE_PER_CPU(u64, current_tsc_ratio); |
| 89 | #define TSC_RATIO_DEFAULT 0x0100000000ULL |
| 90 | |
Mathias Krause | 09941fb | 2012-08-30 01:30:20 +0200 | [diff] [blame] | 91 | static const struct svm_direct_access_msrs { |
Joerg Roedel | ac72a9b | 2010-03-01 15:34:36 +0100 | [diff] [blame] | 92 | u32 index; /* Index of the MSR */ |
| 93 | bool always; /* True if intercept is always on */ |
| 94 | } direct_access_msrs[] = { |
Brian Gerst | 8c06585 | 2010-07-17 09:03:26 -0400 | [diff] [blame] | 95 | { .index = MSR_STAR, .always = true }, |
Joerg Roedel | ac72a9b | 2010-03-01 15:34:36 +0100 | [diff] [blame] | 96 | { .index = MSR_IA32_SYSENTER_CS, .always = true }, |
| 97 | #ifdef CONFIG_X86_64 |
| 98 | { .index = MSR_GS_BASE, .always = true }, |
| 99 | { .index = MSR_FS_BASE, .always = true }, |
| 100 | { .index = MSR_KERNEL_GS_BASE, .always = true }, |
| 101 | { .index = MSR_LSTAR, .always = true }, |
| 102 | { .index = MSR_CSTAR, .always = true }, |
| 103 | { .index = MSR_SYSCALL_MASK, .always = true }, |
| 104 | #endif |
KarimAllah Ahmed | b2ac58f | 2018-02-03 15:56:23 +0100 | [diff] [blame] | 105 | { .index = MSR_IA32_SPEC_CTRL, .always = false }, |
Ashok Raj | 15d4507 | 2018-02-01 22:59:43 +0100 | [diff] [blame] | 106 | { .index = MSR_IA32_PRED_CMD, .always = false }, |
Joerg Roedel | ac72a9b | 2010-03-01 15:34:36 +0100 | [diff] [blame] | 107 | { .index = MSR_IA32_LASTBRANCHFROMIP, .always = false }, |
| 108 | { .index = MSR_IA32_LASTBRANCHTOIP, .always = false }, |
| 109 | { .index = MSR_IA32_LASTINTFROMIP, .always = false }, |
| 110 | { .index = MSR_IA32_LASTINTTOIP, .always = false }, |
| 111 | { .index = MSR_INVALID, .always = false }, |
Avi Kivity | 6c8166a | 2009-05-31 18:15:37 +0300 | [diff] [blame] | 112 | }; |
| 113 | |
Joerg Roedel | 709ddeb | 2008-02-07 13:47:45 +0100 | [diff] [blame] | 114 | /* enable NPT for AMD64 and X86 with PAE */ |
| 115 | #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE) |
Joerg Roedel | 883b0a9 | 2020-03-24 10:41:52 +0100 | [diff] [blame] | 116 | bool npt_enabled = true; |
Joerg Roedel | 709ddeb | 2008-02-07 13:47:45 +0100 | [diff] [blame] | 117 | #else |
Joerg Roedel | 883b0a9 | 2020-03-24 10:41:52 +0100 | [diff] [blame] | 118 | bool npt_enabled; |
Joerg Roedel | 709ddeb | 2008-02-07 13:47:45 +0100 | [diff] [blame] | 119 | #endif |
Joerg Roedel | 6c7dac7 | 2008-02-07 13:47:40 +0100 | [diff] [blame] | 120 | |
Babu Moger | 8566ac8 | 2018-03-16 16:37:26 -0400 | [diff] [blame] | 121 | /* |
| 122 | * These 2 parameters are used to config the controls for Pause-Loop Exiting: |
| 123 | * pause_filter_count: On processors that support Pause filtering(indicated |
| 124 | * by CPUID Fn8000_000A_EDX), the VMCB provides a 16 bit pause filter |
| 125 | * count value. On VMRUN this value is loaded into an internal counter. |
| 126 | * Each time a pause instruction is executed, this counter is decremented |
| 127 | * until it reaches zero at which time a #VMEXIT is generated if pause |
| 128 | * intercept is enabled. Refer to AMD APM Vol 2 Section 15.14.4 Pause |
| 129 | * Intercept Filtering for more details. |
| 130 | * This also indicate if ple logic enabled. |
| 131 | * |
| 132 | * pause_filter_thresh: In addition, some processor families support advanced |
| 133 | * pause filtering (indicated by CPUID Fn8000_000A_EDX) upper bound on |
| 134 | * the amount of time a guest is allowed to execute in a pause loop. |
| 135 | * In this mode, a 16-bit pause filter threshold field is added in the |
| 136 | * VMCB. The threshold value is a cycle count that is used to reset the |
| 137 | * pause counter. As with simple pause filtering, VMRUN loads the pause |
| 138 | * count value from VMCB into an internal counter. Then, on each pause |
| 139 | * instruction the hardware checks the elapsed number of cycles since |
| 140 | * the most recent pause instruction against the pause filter threshold. |
| 141 | * If the elapsed cycle count is greater than the pause filter threshold, |
| 142 | * then the internal pause count is reloaded from the VMCB and execution |
| 143 | * continues. If the elapsed cycle count is less than the pause filter |
| 144 | * threshold, then the internal pause count is decremented. If the count |
| 145 | * value is less than zero and PAUSE intercept is enabled, a #VMEXIT is |
| 146 | * triggered. If advanced pause filtering is supported and pause filter |
| 147 | * threshold field is set to zero, the filter will operate in the simpler, |
| 148 | * count only mode. |
| 149 | */ |
| 150 | |
| 151 | static unsigned short pause_filter_thresh = KVM_DEFAULT_PLE_GAP; |
| 152 | module_param(pause_filter_thresh, ushort, 0444); |
| 153 | |
| 154 | static unsigned short pause_filter_count = KVM_SVM_DEFAULT_PLE_WINDOW; |
| 155 | module_param(pause_filter_count, ushort, 0444); |
| 156 | |
| 157 | /* Default doubles per-vcpu window every exit. */ |
| 158 | static unsigned short pause_filter_count_grow = KVM_DEFAULT_PLE_WINDOW_GROW; |
| 159 | module_param(pause_filter_count_grow, ushort, 0444); |
| 160 | |
| 161 | /* Default resets per-vcpu window every exit to pause_filter_count. */ |
| 162 | static unsigned short pause_filter_count_shrink = KVM_DEFAULT_PLE_WINDOW_SHRINK; |
| 163 | module_param(pause_filter_count_shrink, ushort, 0444); |
| 164 | |
| 165 | /* Default is to compute the maximum so we can never overflow. */ |
| 166 | static unsigned short pause_filter_count_max = KVM_SVM_DEFAULT_PLE_WINDOW_MAX; |
| 167 | module_param(pause_filter_count_max, ushort, 0444); |
| 168 | |
Davidlohr Bueso | e235885 | 2012-01-17 14:09:50 +0100 | [diff] [blame] | 169 | /* allow nested paging (virtualized MMU) for all guests */ |
| 170 | static int npt = true; |
Joerg Roedel | 6c7dac7 | 2008-02-07 13:47:40 +0100 | [diff] [blame] | 171 | module_param(npt, int, S_IRUGO); |
Joerg Roedel | e3da3ac | 2008-02-07 13:47:39 +0100 | [diff] [blame] | 172 | |
Davidlohr Bueso | e235885 | 2012-01-17 14:09:50 +0100 | [diff] [blame] | 173 | /* allow nested virtualization in KVM/SVM */ |
| 174 | static int nested = true; |
Alexander Graf | 236de05 | 2008-11-25 20:17:10 +0100 | [diff] [blame] | 175 | module_param(nested, int, S_IRUGO); |
| 176 | |
Paolo Bonzini | d647eb6 | 2019-06-20 14:13:33 +0200 | [diff] [blame] | 177 | /* enable/disable Next RIP Save */ |
| 178 | static int nrips = true; |
| 179 | module_param(nrips, int, 0444); |
| 180 | |
Janakarajan Natarajan | 89c8a49 | 2017-07-06 15:50:47 -0500 | [diff] [blame] | 181 | /* enable/disable Virtual VMLOAD VMSAVE */ |
| 182 | static int vls = true; |
| 183 | module_param(vls, int, 0444); |
| 184 | |
Janakarajan Natarajan | 640bd6e | 2017-08-23 09:57:19 -0500 | [diff] [blame] | 185 | /* enable/disable Virtual GIF */ |
| 186 | static int vgif = true; |
| 187 | module_param(vgif, int, 0444); |
Suravee Suthikulpanit | 5ea11f2 | 2016-08-23 13:52:41 -0500 | [diff] [blame] | 188 | |
Brijesh Singh | e9df094 | 2017-12-04 10:57:33 -0600 | [diff] [blame] | 189 | /* enable/disable SEV support */ |
| 190 | static int sev = IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT_ACTIVE_BY_DEFAULT); |
| 191 | module_param(sev, int, 0444); |
| 192 | |
Paolo Bonzini | 6f2f845 | 2019-05-20 15:34:35 +0200 | [diff] [blame] | 193 | static bool __read_mostly dump_invalid_vmcb = 0; |
| 194 | module_param(dump_invalid_vmcb, bool, 0644); |
| 195 | |
Brijesh Singh | 7607b71 | 2018-02-19 10:14:44 -0600 | [diff] [blame] | 196 | static u8 rsm_ins_bytes[] = "\x0f\xaa"; |
| 197 | |
Joerg Roedel | a5c3832 | 2009-08-07 11:49:32 +0200 | [diff] [blame] | 198 | static void svm_complete_interrupts(struct vcpu_svm *svm); |
Suravee Suthikulpanit | 44a95da | 2016-05-04 14:09:46 -0500 | [diff] [blame] | 199 | |
Harvey Harrison | 4866d5e | 2008-02-19 10:32:02 -0800 | [diff] [blame] | 200 | static unsigned long iopm_base; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 201 | |
| 202 | struct kvm_ldttss_desc { |
| 203 | u16 limit0; |
| 204 | u16 base0; |
Joerg Roedel | e023171 | 2010-02-24 18:59:10 +0100 | [diff] [blame] | 205 | unsigned base1:8, type:5, dpl:2, p:1; |
| 206 | unsigned limit1:4, zero0:3, g:1, base2:8; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 207 | u32 base3; |
| 208 | u32 zero1; |
| 209 | } __attribute__((packed)); |
| 210 | |
Joerg Roedel | eaf7826 | 2020-03-24 10:41:54 +0100 | [diff] [blame] | 211 | DEFINE_PER_CPU(struct svm_cpu_data *, svm_data); |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 212 | |
Mathias Krause | 09941fb | 2012-08-30 01:30:20 +0200 | [diff] [blame] | 213 | static const u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000}; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 214 | |
Ahmed S. Darwish | 9d8f549 | 2007-02-19 14:37:46 +0200 | [diff] [blame] | 215 | #define NUM_MSR_MAPS ARRAY_SIZE(msrpm_ranges) |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 216 | #define MSRS_RANGE_SIZE 2048 |
| 217 | #define MSRS_IN_RANGE (MSRS_RANGE_SIZE * 8 / 2) |
| 218 | |
Joerg Roedel | 883b0a9 | 2020-03-24 10:41:52 +0100 | [diff] [blame] | 219 | u32 svm_msrpm_offset(u32 msr) |
Joerg Roedel | 455716f | 2010-03-01 15:34:35 +0100 | [diff] [blame] | 220 | { |
| 221 | u32 offset; |
| 222 | int i; |
| 223 | |
| 224 | for (i = 0; i < NUM_MSR_MAPS; i++) { |
| 225 | if (msr < msrpm_ranges[i] || |
| 226 | msr >= msrpm_ranges[i] + MSRS_IN_RANGE) |
| 227 | continue; |
| 228 | |
| 229 | offset = (msr - msrpm_ranges[i]) / 4; /* 4 msrs per u8 */ |
| 230 | offset += (i * MSRS_RANGE_SIZE); /* add range offset */ |
| 231 | |
| 232 | /* Now we have the u8 offset - but need the u32 offset */ |
| 233 | return offset / 4; |
| 234 | } |
| 235 | |
| 236 | /* MSR not in any range */ |
| 237 | return MSR_INVALID; |
| 238 | } |
| 239 | |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 240 | #define MAX_INST_SIZE 15 |
| 241 | |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 242 | static inline void clgi(void) |
| 243 | { |
Uros Bizjak | ac5ffda2 | 2018-11-26 17:00:08 +0100 | [diff] [blame] | 244 | asm volatile (__ex("clgi")); |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 245 | } |
| 246 | |
| 247 | static inline void stgi(void) |
| 248 | { |
Uros Bizjak | ac5ffda2 | 2018-11-26 17:00:08 +0100 | [diff] [blame] | 249 | asm volatile (__ex("stgi")); |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 250 | } |
| 251 | |
| 252 | static inline void invlpga(unsigned long addr, u32 asid) |
| 253 | { |
Uros Bizjak | ac5ffda2 | 2018-11-26 17:00:08 +0100 | [diff] [blame] | 254 | asm volatile (__ex("invlpga %1, %0") : : "c"(asid), "a"(addr)); |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 255 | } |
| 256 | |
Yu Zhang | 855feb6 | 2017-08-24 20:27:55 +0800 | [diff] [blame] | 257 | static int get_npt_level(struct kvm_vcpu *vcpu) |
Joerg Roedel | 4b16184 | 2010-09-10 17:31:03 +0200 | [diff] [blame] | 258 | { |
| 259 | #ifdef CONFIG_X86_64 |
Yu Zhang | 2a7266a | 2017-08-24 20:27:54 +0800 | [diff] [blame] | 260 | return PT64_ROOT_4LEVEL; |
Joerg Roedel | 4b16184 | 2010-09-10 17:31:03 +0200 | [diff] [blame] | 261 | #else |
| 262 | return PT32E_ROOT_LEVEL; |
| 263 | #endif |
| 264 | } |
| 265 | |
Joerg Roedel | 883b0a9 | 2020-03-24 10:41:52 +0100 | [diff] [blame] | 266 | void svm_set_efer(struct kvm_vcpu *vcpu, u64 efer) |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 267 | { |
Paolo Bonzini | c513f48 | 2020-05-18 13:08:37 -0400 | [diff] [blame] | 268 | struct vcpu_svm *svm = to_svm(vcpu); |
Zachary Amsden | 6dc696d | 2010-05-26 15:09:43 -1000 | [diff] [blame] | 269 | vcpu->arch.efer = efer; |
Paolo Bonzini | 9167ab7 | 2019-10-27 16:23:23 +0100 | [diff] [blame] | 270 | |
| 271 | if (!npt_enabled) { |
| 272 | /* Shadow paging assumes NX to be available. */ |
| 273 | efer |= EFER_NX; |
| 274 | |
| 275 | if (!(efer & EFER_LMA)) |
| 276 | efer &= ~EFER_LME; |
| 277 | } |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 278 | |
Paolo Bonzini | c513f48 | 2020-05-18 13:08:37 -0400 | [diff] [blame] | 279 | if (!(efer & EFER_SVME)) { |
| 280 | svm_leave_nested(svm); |
| 281 | svm_set_gif(svm, true); |
| 282 | } |
| 283 | |
| 284 | svm->vmcb->save.efer = efer | EFER_SVME; |
| 285 | mark_dirty(svm->vmcb, VMCB_CR); |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 286 | } |
| 287 | |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 288 | static int is_external_interrupt(u32 info) |
| 289 | { |
| 290 | info &= SVM_EVTINJ_TYPE_MASK | SVM_EVTINJ_VALID; |
| 291 | return info == (SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR); |
| 292 | } |
| 293 | |
Paolo Bonzini | 37ccdcb | 2014-05-20 14:29:47 +0200 | [diff] [blame] | 294 | static u32 svm_get_interrupt_shadow(struct kvm_vcpu *vcpu) |
Glauber Costa | 2809f5d | 2009-05-12 16:21:05 -0400 | [diff] [blame] | 295 | { |
| 296 | struct vcpu_svm *svm = to_svm(vcpu); |
| 297 | u32 ret = 0; |
| 298 | |
| 299 | if (svm->vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) |
Paolo Bonzini | 37ccdcb | 2014-05-20 14:29:47 +0200 | [diff] [blame] | 300 | ret = KVM_X86_SHADOW_INT_STI | KVM_X86_SHADOW_INT_MOV_SS; |
| 301 | return ret; |
Glauber Costa | 2809f5d | 2009-05-12 16:21:05 -0400 | [diff] [blame] | 302 | } |
| 303 | |
| 304 | static void svm_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask) |
| 305 | { |
| 306 | struct vcpu_svm *svm = to_svm(vcpu); |
| 307 | |
| 308 | if (mask == 0) |
| 309 | svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK; |
| 310 | else |
| 311 | svm->vmcb->control.int_state |= SVM_INTERRUPT_SHADOW_MASK; |
| 312 | |
| 313 | } |
| 314 | |
Vitaly Kuznetsov | f8ea7c6 | 2019-08-13 15:53:30 +0200 | [diff] [blame] | 315 | static int skip_emulated_instruction(struct kvm_vcpu *vcpu) |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 316 | { |
Gregory Haskins | a2fa3e9 | 2007-07-27 08:13:10 -0400 | [diff] [blame] | 317 | struct vcpu_svm *svm = to_svm(vcpu); |
| 318 | |
Paolo Bonzini | d647eb6 | 2019-06-20 14:13:33 +0200 | [diff] [blame] | 319 | if (nrips && svm->vmcb->control.next_rip != 0) { |
Dirk Müller | d292242 | 2015-10-01 13:43:42 +0200 | [diff] [blame] | 320 | WARN_ON_ONCE(!static_cpu_has(X86_FEATURE_NRIPS)); |
Andre Przywara | 6bc31bd | 2010-04-11 23:07:28 +0200 | [diff] [blame] | 321 | svm->next_rip = svm->vmcb->control.next_rip; |
Bandan Das | f104765 | 2015-06-11 02:05:33 -0400 | [diff] [blame] | 322 | } |
Andre Przywara | 6bc31bd | 2010-04-11 23:07:28 +0200 | [diff] [blame] | 323 | |
Sean Christopherson | 1957aa6 | 2019-08-27 14:40:39 -0700 | [diff] [blame] | 324 | if (!svm->next_rip) { |
| 325 | if (!kvm_emulate_instruction(vcpu, EMULTYPE_SKIP)) |
| 326 | return 0; |
| 327 | } else { |
Sean Christopherson | 1957aa6 | 2019-08-27 14:40:39 -0700 | [diff] [blame] | 328 | kvm_rip_write(vcpu, svm->next_rip); |
| 329 | } |
Glauber Costa | 2809f5d | 2009-05-12 16:21:05 -0400 | [diff] [blame] | 330 | svm_set_interrupt_shadow(vcpu, 0); |
Vitaly Kuznetsov | f8ea7c6 | 2019-08-13 15:53:30 +0200 | [diff] [blame] | 331 | |
Sean Christopherson | 60fc3d0 | 2019-08-27 14:40:38 -0700 | [diff] [blame] | 332 | return 1; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 333 | } |
| 334 | |
Wanpeng Li | cfcd20e | 2017-07-13 18:30:39 -0700 | [diff] [blame] | 335 | static void svm_queue_exception(struct kvm_vcpu *vcpu) |
Jan Kiszka | 116a475 | 2010-02-23 17:47:54 +0100 | [diff] [blame] | 336 | { |
| 337 | struct vcpu_svm *svm = to_svm(vcpu); |
Wanpeng Li | cfcd20e | 2017-07-13 18:30:39 -0700 | [diff] [blame] | 338 | unsigned nr = vcpu->arch.exception.nr; |
| 339 | bool has_error_code = vcpu->arch.exception.has_error_code; |
Wanpeng Li | cfcd20e | 2017-07-13 18:30:39 -0700 | [diff] [blame] | 340 | u32 error_code = vcpu->arch.exception.error_code; |
Jan Kiszka | 116a475 | 2010-02-23 17:47:54 +0100 | [diff] [blame] | 341 | |
Jim Mattson | da998b4 | 2018-10-16 14:29:22 -0700 | [diff] [blame] | 342 | kvm_deliver_exception_payload(&svm->vcpu); |
| 343 | |
Paolo Bonzini | d647eb6 | 2019-06-20 14:13:33 +0200 | [diff] [blame] | 344 | if (nr == BP_VECTOR && !nrips) { |
Jan Kiszka | 66b7138 | 2010-02-23 17:47:56 +0100 | [diff] [blame] | 345 | unsigned long rip, old_rip = kvm_rip_read(&svm->vcpu); |
| 346 | |
| 347 | /* |
| 348 | * For guest debugging where we have to reinject #BP if some |
| 349 | * INT3 is guest-owned: |
| 350 | * Emulate nRIP by moving RIP forward. Will fail if injection |
| 351 | * raises a fault that is not intercepted. Still better than |
| 352 | * failing in all cases. |
| 353 | */ |
Vitaly Kuznetsov | f8ea7c6 | 2019-08-13 15:53:30 +0200 | [diff] [blame] | 354 | (void)skip_emulated_instruction(&svm->vcpu); |
Jan Kiszka | 66b7138 | 2010-02-23 17:47:56 +0100 | [diff] [blame] | 355 | rip = kvm_rip_read(&svm->vcpu); |
| 356 | svm->int3_rip = rip + svm->vmcb->save.cs.base; |
| 357 | svm->int3_injected = rip - old_rip; |
| 358 | } |
| 359 | |
Jan Kiszka | 116a475 | 2010-02-23 17:47:54 +0100 | [diff] [blame] | 360 | svm->vmcb->control.event_inj = nr |
| 361 | | SVM_EVTINJ_VALID |
| 362 | | (has_error_code ? SVM_EVTINJ_VALID_ERR : 0) |
| 363 | | SVM_EVTINJ_TYPE_EXEPT; |
| 364 | svm->vmcb->control.event_inj_err = error_code; |
| 365 | } |
| 366 | |
Joerg Roedel | 67ec660 | 2010-05-17 14:43:35 +0200 | [diff] [blame] | 367 | static void svm_init_erratum_383(void) |
| 368 | { |
| 369 | u32 low, high; |
| 370 | int err; |
| 371 | u64 val; |
| 372 | |
Borislav Petkov | e6ee94d | 2013-03-20 15:07:27 +0100 | [diff] [blame] | 373 | if (!static_cpu_has_bug(X86_BUG_AMD_TLB_MMATCH)) |
Joerg Roedel | 67ec660 | 2010-05-17 14:43:35 +0200 | [diff] [blame] | 374 | return; |
| 375 | |
| 376 | /* Use _safe variants to not break nested virtualization */ |
| 377 | val = native_read_msr_safe(MSR_AMD64_DC_CFG, &err); |
| 378 | if (err) |
| 379 | return; |
| 380 | |
| 381 | val |= (1ULL << 47); |
| 382 | |
| 383 | low = lower_32_bits(val); |
| 384 | high = upper_32_bits(val); |
| 385 | |
| 386 | native_write_msr_safe(MSR_AMD64_DC_CFG, low, high); |
| 387 | |
| 388 | erratum_383_found = true; |
| 389 | } |
| 390 | |
Boris Ostrovsky | 2b036c6 | 2012-01-09 14:00:35 -0500 | [diff] [blame] | 391 | static void svm_init_osvw(struct kvm_vcpu *vcpu) |
| 392 | { |
| 393 | /* |
| 394 | * Guests should see errata 400 and 415 as fixed (assuming that |
| 395 | * HLT and IO instructions are intercepted). |
| 396 | */ |
| 397 | vcpu->arch.osvw.length = (osvw_len >= 3) ? (osvw_len) : 3; |
| 398 | vcpu->arch.osvw.status = osvw_status & ~(6ULL); |
| 399 | |
| 400 | /* |
| 401 | * By increasing VCPU's osvw.length to 3 we are telling the guest that |
| 402 | * all osvw.status bits inside that length, including bit 0 (which is |
| 403 | * reserved for erratum 298), are valid. However, if host processor's |
| 404 | * osvw_len is 0 then osvw_status[0] carries no information. We need to |
| 405 | * be conservative here and therefore we tell the guest that erratum 298 |
| 406 | * is present (because we really don't know). |
| 407 | */ |
| 408 | if (osvw_len == 0 && boot_cpu_data.x86 == 0x10) |
| 409 | vcpu->arch.osvw.status |= 1; |
| 410 | } |
| 411 | |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 412 | static int has_svm(void) |
| 413 | { |
Eduardo Habkost | 63d1142 | 2008-11-17 19:03:20 -0200 | [diff] [blame] | 414 | const char *msg; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 415 | |
Eduardo Habkost | 63d1142 | 2008-11-17 19:03:20 -0200 | [diff] [blame] | 416 | if (!cpu_has_svm(&msg)) { |
Joe Perches | ff81ff1 | 2009-01-08 11:05:17 -0800 | [diff] [blame] | 417 | printk(KERN_INFO "has_svm: %s\n", msg); |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 418 | return 0; |
| 419 | } |
| 420 | |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 421 | return 1; |
| 422 | } |
| 423 | |
Radim Krčmář | 13a34e0 | 2014-08-28 15:13:03 +0200 | [diff] [blame] | 424 | static void svm_hardware_disable(void) |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 425 | { |
Joerg Roedel | fbc0db7 | 2011-03-25 09:44:46 +0100 | [diff] [blame] | 426 | /* Make sure we clean up behind us */ |
| 427 | if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) |
| 428 | wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT); |
| 429 | |
Eduardo Habkost | 2c8dcee | 2008-11-17 19:03:21 -0200 | [diff] [blame] | 430 | cpu_svm_disable(); |
Joerg Roedel | 1018faa | 2012-02-29 14:57:32 +0100 | [diff] [blame] | 431 | |
| 432 | amd_pmu_disable_virt(); |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 433 | } |
| 434 | |
Radim Krčmář | 13a34e0 | 2014-08-28 15:13:03 +0200 | [diff] [blame] | 435 | static int svm_hardware_enable(void) |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 436 | { |
| 437 | |
Tejun Heo | 0fe1e00 | 2009-10-29 22:34:14 +0900 | [diff] [blame] | 438 | struct svm_cpu_data *sd; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 439 | uint64_t efer; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 440 | struct desc_struct *gdt; |
| 441 | int me = raw_smp_processor_id(); |
| 442 | |
Alexander Graf | 10474ae | 2009-09-15 11:37:46 +0200 | [diff] [blame] | 443 | rdmsrl(MSR_EFER, efer); |
| 444 | if (efer & EFER_SVME) |
| 445 | return -EBUSY; |
| 446 | |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 447 | if (!has_svm()) { |
Borislav Petkov | 1f5b77f | 2012-10-20 20:20:04 +0200 | [diff] [blame] | 448 | pr_err("%s: err EOPNOTSUPP on %d\n", __func__, me); |
Alexander Graf | 10474ae | 2009-09-15 11:37:46 +0200 | [diff] [blame] | 449 | return -EINVAL; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 450 | } |
Tejun Heo | 0fe1e00 | 2009-10-29 22:34:14 +0900 | [diff] [blame] | 451 | sd = per_cpu(svm_data, me); |
Tejun Heo | 0fe1e00 | 2009-10-29 22:34:14 +0900 | [diff] [blame] | 452 | if (!sd) { |
Borislav Petkov | 1f5b77f | 2012-10-20 20:20:04 +0200 | [diff] [blame] | 453 | pr_err("%s: svm_data is NULL on %d\n", __func__, me); |
Alexander Graf | 10474ae | 2009-09-15 11:37:46 +0200 | [diff] [blame] | 454 | return -EINVAL; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 455 | } |
| 456 | |
Tejun Heo | 0fe1e00 | 2009-10-29 22:34:14 +0900 | [diff] [blame] | 457 | sd->asid_generation = 1; |
| 458 | sd->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1; |
| 459 | sd->next_asid = sd->max_asid + 1; |
Brijesh Singh | ed3cd23 | 2017-12-04 10:57:32 -0600 | [diff] [blame] | 460 | sd->min_asid = max_sev_asid + 1; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 461 | |
Thomas Garnier | 45fc875 | 2017-03-14 10:05:08 -0700 | [diff] [blame] | 462 | gdt = get_current_gdt_rw(); |
Tejun Heo | 0fe1e00 | 2009-10-29 22:34:14 +0900 | [diff] [blame] | 463 | sd->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS); |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 464 | |
Alexander Graf | 9962d03 | 2008-11-25 20:17:02 +0100 | [diff] [blame] | 465 | wrmsrl(MSR_EFER, efer | EFER_SVME); |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 466 | |
Linus Torvalds | d031655 | 2009-12-14 09:58:24 -0800 | [diff] [blame] | 467 | wrmsrl(MSR_VM_HSAVE_PA, page_to_pfn(sd->save_area) << PAGE_SHIFT); |
Alexander Graf | 10474ae | 2009-09-15 11:37:46 +0200 | [diff] [blame] | 468 | |
Joerg Roedel | fbc0db7 | 2011-03-25 09:44:46 +0100 | [diff] [blame] | 469 | if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) { |
| 470 | wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT); |
Christoph Lameter | 89cbc76 | 2014-08-17 12:30:40 -0500 | [diff] [blame] | 471 | __this_cpu_write(current_tsc_ratio, TSC_RATIO_DEFAULT); |
Joerg Roedel | fbc0db7 | 2011-03-25 09:44:46 +0100 | [diff] [blame] | 472 | } |
| 473 | |
Boris Ostrovsky | 2b036c6 | 2012-01-09 14:00:35 -0500 | [diff] [blame] | 474 | |
| 475 | /* |
| 476 | * Get OSVW bits. |
| 477 | * |
| 478 | * Note that it is possible to have a system with mixed processor |
| 479 | * revisions and therefore different OSVW bits. If bits are not the same |
| 480 | * on different processors then choose the worst case (i.e. if erratum |
| 481 | * is present on one processor and not on another then assume that the |
| 482 | * erratum is present everywhere). |
| 483 | */ |
| 484 | if (cpu_has(&boot_cpu_data, X86_FEATURE_OSVW)) { |
| 485 | uint64_t len, status = 0; |
| 486 | int err; |
| 487 | |
| 488 | len = native_read_msr_safe(MSR_AMD64_OSVW_ID_LENGTH, &err); |
| 489 | if (!err) |
| 490 | status = native_read_msr_safe(MSR_AMD64_OSVW_STATUS, |
| 491 | &err); |
| 492 | |
| 493 | if (err) |
| 494 | osvw_status = osvw_len = 0; |
| 495 | else { |
| 496 | if (len < osvw_len) |
| 497 | osvw_len = len; |
| 498 | osvw_status |= status; |
| 499 | osvw_status &= (1ULL << osvw_len) - 1; |
| 500 | } |
| 501 | } else |
| 502 | osvw_status = osvw_len = 0; |
| 503 | |
Joerg Roedel | 67ec660 | 2010-05-17 14:43:35 +0200 | [diff] [blame] | 504 | svm_init_erratum_383(); |
| 505 | |
Joerg Roedel | 1018faa | 2012-02-29 14:57:32 +0100 | [diff] [blame] | 506 | amd_pmu_enable_virt(); |
| 507 | |
Alexander Graf | 10474ae | 2009-09-15 11:37:46 +0200 | [diff] [blame] | 508 | return 0; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 509 | } |
| 510 | |
Joerg Roedel | 0da1db75 | 2008-07-02 16:02:11 +0200 | [diff] [blame] | 511 | static void svm_cpu_uninit(int cpu) |
| 512 | { |
Tejun Heo | 0fe1e00 | 2009-10-29 22:34:14 +0900 | [diff] [blame] | 513 | struct svm_cpu_data *sd = per_cpu(svm_data, raw_smp_processor_id()); |
Joerg Roedel | 0da1db75 | 2008-07-02 16:02:11 +0200 | [diff] [blame] | 514 | |
Tejun Heo | 0fe1e00 | 2009-10-29 22:34:14 +0900 | [diff] [blame] | 515 | if (!sd) |
Joerg Roedel | 0da1db75 | 2008-07-02 16:02:11 +0200 | [diff] [blame] | 516 | return; |
| 517 | |
| 518 | per_cpu(svm_data, raw_smp_processor_id()) = NULL; |
Brijesh Singh | 70cd94e | 2017-12-04 10:57:34 -0600 | [diff] [blame] | 519 | kfree(sd->sev_vmcbs); |
Tejun Heo | 0fe1e00 | 2009-10-29 22:34:14 +0900 | [diff] [blame] | 520 | __free_page(sd->save_area); |
| 521 | kfree(sd); |
Joerg Roedel | 0da1db75 | 2008-07-02 16:02:11 +0200 | [diff] [blame] | 522 | } |
| 523 | |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 524 | static int svm_cpu_init(int cpu) |
| 525 | { |
Tejun Heo | 0fe1e00 | 2009-10-29 22:34:14 +0900 | [diff] [blame] | 526 | struct svm_cpu_data *sd; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 527 | |
Tejun Heo | 0fe1e00 | 2009-10-29 22:34:14 +0900 | [diff] [blame] | 528 | sd = kzalloc(sizeof(struct svm_cpu_data), GFP_KERNEL); |
| 529 | if (!sd) |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 530 | return -ENOMEM; |
Tejun Heo | 0fe1e00 | 2009-10-29 22:34:14 +0900 | [diff] [blame] | 531 | sd->cpu = cpu; |
Brijesh Singh | 70cd94e | 2017-12-04 10:57:34 -0600 | [diff] [blame] | 532 | sd->save_area = alloc_page(GFP_KERNEL); |
Tejun Heo | 0fe1e00 | 2009-10-29 22:34:14 +0900 | [diff] [blame] | 533 | if (!sd->save_area) |
Miaohe Lin | d80b64f | 2020-01-04 16:56:49 +0800 | [diff] [blame] | 534 | goto free_cpu_data; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 535 | |
Brijesh Singh | 70cd94e | 2017-12-04 10:57:34 -0600 | [diff] [blame] | 536 | if (svm_sev_enabled()) { |
Kees Cook | 6da2ec5 | 2018-06-12 13:55:00 -0700 | [diff] [blame] | 537 | sd->sev_vmcbs = kmalloc_array(max_sev_asid + 1, |
| 538 | sizeof(void *), |
| 539 | GFP_KERNEL); |
Brijesh Singh | 70cd94e | 2017-12-04 10:57:34 -0600 | [diff] [blame] | 540 | if (!sd->sev_vmcbs) |
Miaohe Lin | d80b64f | 2020-01-04 16:56:49 +0800 | [diff] [blame] | 541 | goto free_save_area; |
Brijesh Singh | 70cd94e | 2017-12-04 10:57:34 -0600 | [diff] [blame] | 542 | } |
| 543 | |
Tejun Heo | 0fe1e00 | 2009-10-29 22:34:14 +0900 | [diff] [blame] | 544 | per_cpu(svm_data, cpu) = sd; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 545 | |
| 546 | return 0; |
| 547 | |
Miaohe Lin | d80b64f | 2020-01-04 16:56:49 +0800 | [diff] [blame] | 548 | free_save_area: |
| 549 | __free_page(sd->save_area); |
| 550 | free_cpu_data: |
Tejun Heo | 0fe1e00 | 2009-10-29 22:34:14 +0900 | [diff] [blame] | 551 | kfree(sd); |
Miaohe Lin | d80b64f | 2020-01-04 16:56:49 +0800 | [diff] [blame] | 552 | return -ENOMEM; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 553 | |
| 554 | } |
| 555 | |
Joerg Roedel | ac72a9b | 2010-03-01 15:34:36 +0100 | [diff] [blame] | 556 | static bool valid_msr_intercept(u32 index) |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 557 | { |
| 558 | int i; |
| 559 | |
Joerg Roedel | ac72a9b | 2010-03-01 15:34:36 +0100 | [diff] [blame] | 560 | for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) |
| 561 | if (direct_access_msrs[i].index == index) |
| 562 | return true; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 563 | |
Joerg Roedel | ac72a9b | 2010-03-01 15:34:36 +0100 | [diff] [blame] | 564 | return false; |
| 565 | } |
| 566 | |
KarimAllah Ahmed | b2ac58f | 2018-02-03 15:56:23 +0100 | [diff] [blame] | 567 | static bool msr_write_intercepted(struct kvm_vcpu *vcpu, unsigned msr) |
| 568 | { |
| 569 | u8 bit_write; |
| 570 | unsigned long tmp; |
| 571 | u32 offset; |
| 572 | u32 *msrpm; |
| 573 | |
| 574 | msrpm = is_guest_mode(vcpu) ? to_svm(vcpu)->nested.msrpm: |
| 575 | to_svm(vcpu)->msrpm; |
| 576 | |
| 577 | offset = svm_msrpm_offset(msr); |
| 578 | bit_write = 2 * (msr & 0x0f) + 1; |
| 579 | tmp = msrpm[offset]; |
| 580 | |
| 581 | BUG_ON(offset == MSR_INVALID); |
| 582 | |
| 583 | return !!test_bit(bit_write, &tmp); |
| 584 | } |
| 585 | |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 586 | static void set_msr_interception(u32 *msrpm, unsigned msr, |
| 587 | int read, int write) |
| 588 | { |
Joerg Roedel | 455716f | 2010-03-01 15:34:35 +0100 | [diff] [blame] | 589 | u8 bit_read, bit_write; |
| 590 | unsigned long tmp; |
| 591 | u32 offset; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 592 | |
Joerg Roedel | ac72a9b | 2010-03-01 15:34:36 +0100 | [diff] [blame] | 593 | /* |
| 594 | * If this warning triggers extend the direct_access_msrs list at the |
| 595 | * beginning of the file |
| 596 | */ |
| 597 | WARN_ON(!valid_msr_intercept(msr)); |
| 598 | |
Joerg Roedel | 455716f | 2010-03-01 15:34:35 +0100 | [diff] [blame] | 599 | offset = svm_msrpm_offset(msr); |
| 600 | bit_read = 2 * (msr & 0x0f); |
| 601 | bit_write = 2 * (msr & 0x0f) + 1; |
| 602 | tmp = msrpm[offset]; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 603 | |
Joerg Roedel | 455716f | 2010-03-01 15:34:35 +0100 | [diff] [blame] | 604 | BUG_ON(offset == MSR_INVALID); |
| 605 | |
| 606 | read ? clear_bit(bit_read, &tmp) : set_bit(bit_read, &tmp); |
| 607 | write ? clear_bit(bit_write, &tmp) : set_bit(bit_write, &tmp); |
| 608 | |
| 609 | msrpm[offset] = tmp; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 610 | } |
| 611 | |
Joerg Roedel | f65c229 | 2008-02-13 18:58:46 +0100 | [diff] [blame] | 612 | static void svm_vcpu_init_msrpm(u32 *msrpm) |
| 613 | { |
Joerg Roedel | ac72a9b | 2010-03-01 15:34:36 +0100 | [diff] [blame] | 614 | int i; |
| 615 | |
Joerg Roedel | f65c229 | 2008-02-13 18:58:46 +0100 | [diff] [blame] | 616 | memset(msrpm, 0xff, PAGE_SIZE * (1 << MSRPM_ALLOC_ORDER)); |
| 617 | |
Joerg Roedel | ac72a9b | 2010-03-01 15:34:36 +0100 | [diff] [blame] | 618 | for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) { |
| 619 | if (!direct_access_msrs[i].always) |
| 620 | continue; |
| 621 | |
| 622 | set_msr_interception(msrpm, direct_access_msrs[i].index, 1, 1); |
| 623 | } |
Joerg Roedel | f65c229 | 2008-02-13 18:58:46 +0100 | [diff] [blame] | 624 | } |
| 625 | |
Joerg Roedel | 323c3d8 | 2010-03-01 15:34:37 +0100 | [diff] [blame] | 626 | static void add_msr_offset(u32 offset) |
| 627 | { |
| 628 | int i; |
| 629 | |
| 630 | for (i = 0; i < MSRPM_OFFSETS; ++i) { |
| 631 | |
| 632 | /* Offset already in list? */ |
| 633 | if (msrpm_offsets[i] == offset) |
| 634 | return; |
| 635 | |
| 636 | /* Slot used by another offset? */ |
| 637 | if (msrpm_offsets[i] != MSR_INVALID) |
| 638 | continue; |
| 639 | |
| 640 | /* Add offset to list */ |
| 641 | msrpm_offsets[i] = offset; |
| 642 | |
| 643 | return; |
| 644 | } |
| 645 | |
| 646 | /* |
| 647 | * If this BUG triggers the msrpm_offsets table has an overflow. Just |
| 648 | * increase MSRPM_OFFSETS in this case. |
| 649 | */ |
| 650 | BUG(); |
| 651 | } |
| 652 | |
| 653 | static void init_msrpm_offsets(void) |
| 654 | { |
| 655 | int i; |
| 656 | |
| 657 | memset(msrpm_offsets, 0xff, sizeof(msrpm_offsets)); |
| 658 | |
| 659 | for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) { |
| 660 | u32 offset; |
| 661 | |
| 662 | offset = svm_msrpm_offset(direct_access_msrs[i].index); |
| 663 | BUG_ON(offset == MSR_INVALID); |
| 664 | |
| 665 | add_msr_offset(offset); |
| 666 | } |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 667 | } |
| 668 | |
Joerg Roedel | 24e09cb | 2008-02-13 18:58:47 +0100 | [diff] [blame] | 669 | static void svm_enable_lbrv(struct vcpu_svm *svm) |
| 670 | { |
| 671 | u32 *msrpm = svm->msrpm; |
| 672 | |
Janakarajan Natarajan | 0dc9211 | 2017-07-06 15:50:45 -0500 | [diff] [blame] | 673 | svm->vmcb->control.virt_ext |= LBR_CTL_ENABLE_MASK; |
Joerg Roedel | 24e09cb | 2008-02-13 18:58:47 +0100 | [diff] [blame] | 674 | set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 1, 1); |
| 675 | set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 1, 1); |
| 676 | set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 1, 1); |
| 677 | set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 1, 1); |
| 678 | } |
| 679 | |
| 680 | static void svm_disable_lbrv(struct vcpu_svm *svm) |
| 681 | { |
| 682 | u32 *msrpm = svm->msrpm; |
| 683 | |
Janakarajan Natarajan | 0dc9211 | 2017-07-06 15:50:45 -0500 | [diff] [blame] | 684 | svm->vmcb->control.virt_ext &= ~LBR_CTL_ENABLE_MASK; |
Joerg Roedel | 24e09cb | 2008-02-13 18:58:47 +0100 | [diff] [blame] | 685 | set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 0, 0); |
| 686 | set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 0, 0); |
| 687 | set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 0, 0); |
| 688 | set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 0, 0); |
| 689 | } |
| 690 | |
Joerg Roedel | 883b0a9 | 2020-03-24 10:41:52 +0100 | [diff] [blame] | 691 | void disable_nmi_singlestep(struct vcpu_svm *svm) |
Ladi Prosek | 4aebd0e | 2017-06-21 09:06:57 +0200 | [diff] [blame] | 692 | { |
| 693 | svm->nmi_singlestep = false; |
Janakarajan Natarajan | 640bd6e | 2017-08-23 09:57:19 -0500 | [diff] [blame] | 694 | |
Ladi Prosek | ab2f4d73 | 2017-06-21 09:06:58 +0200 | [diff] [blame] | 695 | if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP)) { |
| 696 | /* Clear our flags if they were not set by the guest */ |
| 697 | if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF)) |
| 698 | svm->vmcb->save.rflags &= ~X86_EFLAGS_TF; |
| 699 | if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF)) |
| 700 | svm->vmcb->save.rflags &= ~X86_EFLAGS_RF; |
| 701 | } |
Ladi Prosek | 4aebd0e | 2017-06-21 09:06:57 +0200 | [diff] [blame] | 702 | } |
| 703 | |
Babu Moger | 8566ac8 | 2018-03-16 16:37:26 -0400 | [diff] [blame] | 704 | static void grow_ple_window(struct kvm_vcpu *vcpu) |
| 705 | { |
| 706 | struct vcpu_svm *svm = to_svm(vcpu); |
| 707 | struct vmcb_control_area *control = &svm->vmcb->control; |
| 708 | int old = control->pause_filter_count; |
| 709 | |
| 710 | control->pause_filter_count = __grow_ple_window(old, |
| 711 | pause_filter_count, |
| 712 | pause_filter_count_grow, |
| 713 | pause_filter_count_max); |
| 714 | |
Peter Xu | 4f75bcc | 2019-09-06 10:17:22 +0800 | [diff] [blame] | 715 | if (control->pause_filter_count != old) { |
Babu Moger | 8566ac8 | 2018-03-16 16:37:26 -0400 | [diff] [blame] | 716 | mark_dirty(svm->vmcb, VMCB_INTERCEPTS); |
Peter Xu | 4f75bcc | 2019-09-06 10:17:22 +0800 | [diff] [blame] | 717 | trace_kvm_ple_window_update(vcpu->vcpu_id, |
| 718 | control->pause_filter_count, old); |
| 719 | } |
Babu Moger | 8566ac8 | 2018-03-16 16:37:26 -0400 | [diff] [blame] | 720 | } |
| 721 | |
| 722 | static void shrink_ple_window(struct kvm_vcpu *vcpu) |
| 723 | { |
| 724 | struct vcpu_svm *svm = to_svm(vcpu); |
| 725 | struct vmcb_control_area *control = &svm->vmcb->control; |
| 726 | int old = control->pause_filter_count; |
| 727 | |
| 728 | control->pause_filter_count = |
| 729 | __shrink_ple_window(old, |
| 730 | pause_filter_count, |
| 731 | pause_filter_count_shrink, |
| 732 | pause_filter_count); |
Peter Xu | 4f75bcc | 2019-09-06 10:17:22 +0800 | [diff] [blame] | 733 | if (control->pause_filter_count != old) { |
Babu Moger | 8566ac8 | 2018-03-16 16:37:26 -0400 | [diff] [blame] | 734 | mark_dirty(svm->vmcb, VMCB_INTERCEPTS); |
Peter Xu | 4f75bcc | 2019-09-06 10:17:22 +0800 | [diff] [blame] | 735 | trace_kvm_ple_window_update(vcpu->vcpu_id, |
| 736 | control->pause_filter_count, old); |
| 737 | } |
Babu Moger | 8566ac8 | 2018-03-16 16:37:26 -0400 | [diff] [blame] | 738 | } |
| 739 | |
Tom Lendacky | 52918ed | 2020-01-09 17:42:16 -0600 | [diff] [blame] | 740 | /* |
| 741 | * The default MMIO mask is a single bit (excluding the present bit), |
| 742 | * which could conflict with the memory encryption bit. Check for |
| 743 | * memory encryption support and override the default MMIO mask if |
| 744 | * memory encryption is enabled. |
| 745 | */ |
| 746 | static __init void svm_adjust_mmio_mask(void) |
| 747 | { |
| 748 | unsigned int enc_bit, mask_bit; |
| 749 | u64 msr, mask; |
| 750 | |
| 751 | /* If there is no memory encryption support, use existing mask */ |
| 752 | if (cpuid_eax(0x80000000) < 0x8000001f) |
| 753 | return; |
| 754 | |
| 755 | /* If memory encryption is not enabled, use existing mask */ |
| 756 | rdmsrl(MSR_K8_SYSCFG, msr); |
| 757 | if (!(msr & MSR_K8_SYSCFG_MEM_ENCRYPT)) |
| 758 | return; |
| 759 | |
| 760 | enc_bit = cpuid_ebx(0x8000001f) & 0x3f; |
| 761 | mask_bit = boot_cpu_data.x86_phys_bits; |
| 762 | |
| 763 | /* Increment the mask bit if it is the same as the encryption bit */ |
| 764 | if (enc_bit == mask_bit) |
| 765 | mask_bit++; |
| 766 | |
| 767 | /* |
| 768 | * If the mask bit location is below 52, then some bits above the |
| 769 | * physical addressing limit will always be reserved, so use the |
| 770 | * rsvd_bits() function to generate the mask. This mask, along with |
| 771 | * the present bit, will be used to generate a page fault with |
| 772 | * PFER.RSV = 1. |
| 773 | * |
| 774 | * If the mask bit location is 52 (or above), then clear the mask. |
| 775 | */ |
| 776 | mask = (mask_bit < 52) ? rsvd_bits(mask_bit, 51) | PT_PRESENT_MASK : 0; |
| 777 | |
Paolo Bonzini | e7581ca | 2020-05-19 05:04:49 -0400 | [diff] [blame] | 778 | kvm_mmu_set_mmio_spte_mask(mask, PT_WRITABLE_MASK | PT_USER_MASK); |
Tom Lendacky | 52918ed | 2020-01-09 17:42:16 -0600 | [diff] [blame] | 779 | } |
| 780 | |
Li RongQing | dd58f3c | 2020-02-23 16:13:12 +0800 | [diff] [blame] | 781 | static void svm_hardware_teardown(void) |
| 782 | { |
| 783 | int cpu; |
| 784 | |
Joerg Roedel | eaf7826 | 2020-03-24 10:41:54 +0100 | [diff] [blame] | 785 | if (svm_sev_enabled()) |
| 786 | sev_hardware_teardown(); |
Li RongQing | dd58f3c | 2020-02-23 16:13:12 +0800 | [diff] [blame] | 787 | |
| 788 | for_each_possible_cpu(cpu) |
| 789 | svm_cpu_uninit(cpu); |
| 790 | |
| 791 | __free_pages(pfn_to_page(iopm_base >> PAGE_SHIFT), IOPM_ALLOC_ORDER); |
| 792 | iopm_base = 0; |
| 793 | } |
| 794 | |
Sean Christopherson | 9b58b98 | 2020-03-02 15:56:42 -0800 | [diff] [blame] | 795 | static __init void svm_set_cpu_caps(void) |
| 796 | { |
| 797 | kvm_set_cpu_caps(); |
| 798 | |
Paolo Bonzini | 408e9a3 | 2020-03-05 16:11:56 +0100 | [diff] [blame] | 799 | supported_xss = 0; |
| 800 | |
Sean Christopherson | a50718c | 2020-03-02 15:57:07 -0800 | [diff] [blame] | 801 | /* CPUID 0x80000001 and 0x8000000A (SVM features) */ |
| 802 | if (nested) { |
Sean Christopherson | 9b58b98 | 2020-03-02 15:56:42 -0800 | [diff] [blame] | 803 | kvm_cpu_cap_set(X86_FEATURE_SVM); |
| 804 | |
Sean Christopherson | 4eb8746 | 2020-03-02 15:57:08 -0800 | [diff] [blame] | 805 | if (nrips) |
Sean Christopherson | a50718c | 2020-03-02 15:57:07 -0800 | [diff] [blame] | 806 | kvm_cpu_cap_set(X86_FEATURE_NRIPS); |
| 807 | |
| 808 | if (npt_enabled) |
| 809 | kvm_cpu_cap_set(X86_FEATURE_NPT); |
| 810 | } |
| 811 | |
Sean Christopherson | 93c380e | 2020-03-02 15:56:54 -0800 | [diff] [blame] | 812 | /* CPUID 0x80000008 */ |
| 813 | if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD) || |
| 814 | boot_cpu_has(X86_FEATURE_AMD_SSBD)) |
| 815 | kvm_cpu_cap_set(X86_FEATURE_VIRT_SSBD); |
Sean Christopherson | 9b58b98 | 2020-03-02 15:56:42 -0800 | [diff] [blame] | 816 | } |
| 817 | |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 818 | static __init int svm_hardware_setup(void) |
| 819 | { |
| 820 | int cpu; |
| 821 | struct page *iopm_pages; |
Joerg Roedel | f65c229 | 2008-02-13 18:58:46 +0100 | [diff] [blame] | 822 | void *iopm_va; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 823 | int r; |
| 824 | |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 825 | iopm_pages = alloc_pages(GFP_KERNEL, IOPM_ALLOC_ORDER); |
| 826 | |
| 827 | if (!iopm_pages) |
| 828 | return -ENOMEM; |
Anthony Liguori | c868133 | 2007-04-30 09:48:11 +0300 | [diff] [blame] | 829 | |
| 830 | iopm_va = page_address(iopm_pages); |
| 831 | memset(iopm_va, 0xff, PAGE_SIZE * (1 << IOPM_ALLOC_ORDER)); |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 832 | iopm_base = page_to_pfn(iopm_pages) << PAGE_SHIFT; |
| 833 | |
Joerg Roedel | 323c3d8 | 2010-03-01 15:34:37 +0100 | [diff] [blame] | 834 | init_msrpm_offsets(); |
| 835 | |
Sean Christopherson | cfc4818 | 2020-03-02 15:56:23 -0800 | [diff] [blame] | 836 | supported_xcr0 &= ~(XFEATURE_MASK_BNDREGS | XFEATURE_MASK_BNDCSR); |
| 837 | |
Joerg Roedel | 50a37eb | 2008-01-31 14:57:38 +0100 | [diff] [blame] | 838 | if (boot_cpu_has(X86_FEATURE_NX)) |
| 839 | kvm_enable_efer_bits(EFER_NX); |
| 840 | |
Alexander Graf | 1b2fd70 | 2009-02-02 16:23:51 +0100 | [diff] [blame] | 841 | if (boot_cpu_has(X86_FEATURE_FXSR_OPT)) |
| 842 | kvm_enable_efer_bits(EFER_FFXSR); |
| 843 | |
Joerg Roedel | 92a1f12 | 2011-03-25 09:44:51 +0100 | [diff] [blame] | 844 | if (boot_cpu_has(X86_FEATURE_TSCRATEMSR)) { |
Joerg Roedel | 92a1f12 | 2011-03-25 09:44:51 +0100 | [diff] [blame] | 845 | kvm_has_tsc_control = true; |
Haozhong Zhang | bc9b961 | 2015-10-20 15:39:01 +0800 | [diff] [blame] | 846 | kvm_max_tsc_scaling_ratio = TSC_RATIO_MAX; |
| 847 | kvm_tsc_scaling_ratio_frac_bits = 32; |
Joerg Roedel | 92a1f12 | 2011-03-25 09:44:51 +0100 | [diff] [blame] | 848 | } |
| 849 | |
Babu Moger | 8566ac8 | 2018-03-16 16:37:26 -0400 | [diff] [blame] | 850 | /* Check for pause filtering support */ |
| 851 | if (!boot_cpu_has(X86_FEATURE_PAUSEFILTER)) { |
| 852 | pause_filter_count = 0; |
| 853 | pause_filter_thresh = 0; |
| 854 | } else if (!boot_cpu_has(X86_FEATURE_PFTHRESHOLD)) { |
| 855 | pause_filter_thresh = 0; |
| 856 | } |
| 857 | |
Alexander Graf | 236de05 | 2008-11-25 20:17:10 +0100 | [diff] [blame] | 858 | if (nested) { |
| 859 | printk(KERN_INFO "kvm: Nested Virtualization enabled\n"); |
Joerg Roedel | eec4b14 | 2010-05-05 16:04:44 +0200 | [diff] [blame] | 860 | kvm_enable_efer_bits(EFER_SVME | EFER_LMSLE); |
Alexander Graf | 236de05 | 2008-11-25 20:17:10 +0100 | [diff] [blame] | 861 | } |
| 862 | |
Brijesh Singh | e9df094 | 2017-12-04 10:57:33 -0600 | [diff] [blame] | 863 | if (sev) { |
| 864 | if (boot_cpu_has(X86_FEATURE_SEV) && |
| 865 | IS_ENABLED(CONFIG_KVM_AMD_SEV)) { |
| 866 | r = sev_hardware_setup(); |
| 867 | if (r) |
| 868 | sev = false; |
| 869 | } else { |
| 870 | sev = false; |
| 871 | } |
| 872 | } |
| 873 | |
Tom Lendacky | 52918ed | 2020-01-09 17:42:16 -0600 | [diff] [blame] | 874 | svm_adjust_mmio_mask(); |
| 875 | |
Zachary Amsden | 3230bb4 | 2009-09-29 11:38:37 -1000 | [diff] [blame] | 876 | for_each_possible_cpu(cpu) { |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 877 | r = svm_cpu_init(cpu); |
| 878 | if (r) |
Joerg Roedel | f65c229 | 2008-02-13 18:58:46 +0100 | [diff] [blame] | 879 | goto err; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 880 | } |
Joerg Roedel | 33bd6a0 | 2008-02-07 13:47:38 +0100 | [diff] [blame] | 881 | |
Avi Kivity | 2a6b20b | 2010-11-09 16:15:42 +0200 | [diff] [blame] | 882 | if (!boot_cpu_has(X86_FEATURE_NPT)) |
Joerg Roedel | e3da3ac | 2008-02-07 13:47:39 +0100 | [diff] [blame] | 883 | npt_enabled = false; |
| 884 | |
Sean Christopherson | 213e0e1 | 2020-03-02 15:57:01 -0800 | [diff] [blame] | 885 | if (npt_enabled && !npt) |
Joerg Roedel | 6c7dac7 | 2008-02-07 13:47:40 +0100 | [diff] [blame] | 886 | npt_enabled = false; |
Joerg Roedel | 6c7dac7 | 2008-02-07 13:47:40 +0100 | [diff] [blame] | 887 | |
Sean Christopherson | 3bae045 | 2020-04-27 17:54:22 -0700 | [diff] [blame] | 888 | kvm_configure_mmu(npt_enabled, PG_LEVEL_1G); |
Sean Christopherson | 213e0e1 | 2020-03-02 15:57:01 -0800 | [diff] [blame] | 889 | pr_info("kvm: Nested Paging %sabled\n", npt_enabled ? "en" : "dis"); |
Joerg Roedel | e3da3ac | 2008-02-07 13:47:39 +0100 | [diff] [blame] | 890 | |
Paolo Bonzini | d647eb6 | 2019-06-20 14:13:33 +0200 | [diff] [blame] | 891 | if (nrips) { |
| 892 | if (!boot_cpu_has(X86_FEATURE_NRIPS)) |
| 893 | nrips = false; |
| 894 | } |
| 895 | |
Suravee Suthikulpanit | 5b8abf1 | 2016-06-15 17:24:36 -0500 | [diff] [blame] | 896 | if (avic) { |
| 897 | if (!npt_enabled || |
| 898 | !boot_cpu_has(X86_FEATURE_AVIC) || |
Suravee Suthikulpanit | 5881f73 | 2016-08-23 13:52:42 -0500 | [diff] [blame] | 899 | !IS_ENABLED(CONFIG_X86_LOCAL_APIC)) { |
Suravee Suthikulpanit | 5b8abf1 | 2016-06-15 17:24:36 -0500 | [diff] [blame] | 900 | avic = false; |
Suravee Suthikulpanit | 5881f73 | 2016-08-23 13:52:42 -0500 | [diff] [blame] | 901 | } else { |
Suravee Suthikulpanit | 5b8abf1 | 2016-06-15 17:24:36 -0500 | [diff] [blame] | 902 | pr_info("AVIC enabled\n"); |
Suravee Suthikulpanit | 5881f73 | 2016-08-23 13:52:42 -0500 | [diff] [blame] | 903 | |
Suravee Suthikulpanit | 5881f73 | 2016-08-23 13:52:42 -0500 | [diff] [blame] | 904 | amd_iommu_register_ga_log_notifier(&avic_ga_log_notifier); |
| 905 | } |
Suravee Suthikulpanit | 5b8abf1 | 2016-06-15 17:24:36 -0500 | [diff] [blame] | 906 | } |
Suravee Suthikulpanit | 44a95da | 2016-05-04 14:09:46 -0500 | [diff] [blame] | 907 | |
Janakarajan Natarajan | 89c8a49 | 2017-07-06 15:50:47 -0500 | [diff] [blame] | 908 | if (vls) { |
| 909 | if (!npt_enabled || |
Borislav Petkov | 5442c26 | 2017-08-01 20:55:52 +0200 | [diff] [blame] | 910 | !boot_cpu_has(X86_FEATURE_V_VMSAVE_VMLOAD) || |
Janakarajan Natarajan | 89c8a49 | 2017-07-06 15:50:47 -0500 | [diff] [blame] | 911 | !IS_ENABLED(CONFIG_X86_64)) { |
| 912 | vls = false; |
| 913 | } else { |
| 914 | pr_info("Virtual VMLOAD VMSAVE supported\n"); |
| 915 | } |
| 916 | } |
| 917 | |
Janakarajan Natarajan | 640bd6e | 2017-08-23 09:57:19 -0500 | [diff] [blame] | 918 | if (vgif) { |
| 919 | if (!boot_cpu_has(X86_FEATURE_VGIF)) |
| 920 | vgif = false; |
| 921 | else |
| 922 | pr_info("Virtual GIF supported\n"); |
| 923 | } |
| 924 | |
Sean Christopherson | 9b58b98 | 2020-03-02 15:56:42 -0800 | [diff] [blame] | 925 | svm_set_cpu_caps(); |
Sean Christopherson | 66a6950 | 2020-03-02 15:56:41 -0800 | [diff] [blame] | 926 | |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 927 | return 0; |
| 928 | |
Joerg Roedel | f65c229 | 2008-02-13 18:58:46 +0100 | [diff] [blame] | 929 | err: |
Li RongQing | dd58f3c | 2020-02-23 16:13:12 +0800 | [diff] [blame] | 930 | svm_hardware_teardown(); |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 931 | return r; |
| 932 | } |
| 933 | |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 934 | static void init_seg(struct vmcb_seg *seg) |
| 935 | { |
| 936 | seg->selector = 0; |
| 937 | seg->attrib = SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK | |
Joerg Roedel | e023171 | 2010-02-24 18:59:10 +0100 | [diff] [blame] | 938 | SVM_SELECTOR_WRITE_MASK; /* Read/Write Data Segment */ |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 939 | seg->limit = 0xffff; |
| 940 | seg->base = 0; |
| 941 | } |
| 942 | |
| 943 | static void init_sys_seg(struct vmcb_seg *seg, uint32_t type) |
| 944 | { |
| 945 | seg->selector = 0; |
| 946 | seg->attrib = SVM_SELECTOR_P_MASK | type; |
| 947 | seg->limit = 0xffff; |
| 948 | seg->base = 0; |
| 949 | } |
| 950 | |
Leonid Shatz | 326e742 | 2018-11-06 12:14:25 +0200 | [diff] [blame] | 951 | static u64 svm_write_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 offset) |
Zachary Amsden | f4e1b3c | 2010-08-19 22:07:16 -1000 | [diff] [blame] | 952 | { |
| 953 | struct vcpu_svm *svm = to_svm(vcpu); |
| 954 | u64 g_tsc_offset = 0; |
| 955 | |
Joerg Roedel | 2030753 | 2010-11-29 17:51:48 +0100 | [diff] [blame] | 956 | if (is_guest_mode(vcpu)) { |
KarimAllah Ahmed | e79f245 | 2018-04-14 05:10:52 +0200 | [diff] [blame] | 957 | /* Write L1's TSC offset. */ |
Zachary Amsden | f4e1b3c | 2010-08-19 22:07:16 -1000 | [diff] [blame] | 958 | g_tsc_offset = svm->vmcb->control.tsc_offset - |
| 959 | svm->nested.hsave->control.tsc_offset; |
| 960 | svm->nested.hsave->control.tsc_offset = offset; |
Paolo Bonzini | 45c3af9 | 2018-11-25 18:45:35 +0100 | [diff] [blame] | 961 | } |
| 962 | |
| 963 | trace_kvm_write_tsc_offset(vcpu->vcpu_id, |
| 964 | svm->vmcb->control.tsc_offset - g_tsc_offset, |
| 965 | offset); |
Zachary Amsden | f4e1b3c | 2010-08-19 22:07:16 -1000 | [diff] [blame] | 966 | |
| 967 | svm->vmcb->control.tsc_offset = offset + g_tsc_offset; |
Joerg Roedel | 116a0a2 | 2010-12-03 11:45:49 +0100 | [diff] [blame] | 968 | |
| 969 | mark_dirty(svm->vmcb, VMCB_INTERCEPTS); |
Leonid Shatz | 326e742 | 2018-11-06 12:14:25 +0200 | [diff] [blame] | 970 | return svm->vmcb->control.tsc_offset; |
Zachary Amsden | f4e1b3c | 2010-08-19 22:07:16 -1000 | [diff] [blame] | 971 | } |
| 972 | |
Paolo Bonzini | 5690891 | 2015-10-19 11:30:19 +0200 | [diff] [blame] | 973 | static void init_vmcb(struct vcpu_svm *svm) |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 974 | { |
Joerg Roedel | e6101a9 | 2008-02-13 18:58:45 +0100 | [diff] [blame] | 975 | struct vmcb_control_area *control = &svm->vmcb->control; |
| 976 | struct vmcb_save_area *save = &svm->vmcb->save; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 977 | |
Roedel, Joerg | 4ee546b | 2010-12-03 10:50:51 +0100 | [diff] [blame] | 978 | svm->vcpu.arch.hflags = 0; |
Avi Kivity | bff7827 | 2010-01-07 13:16:08 +0200 | [diff] [blame] | 979 | |
Roedel, Joerg | 4ee546b | 2010-12-03 10:50:51 +0100 | [diff] [blame] | 980 | set_cr_intercept(svm, INTERCEPT_CR0_READ); |
| 981 | set_cr_intercept(svm, INTERCEPT_CR3_READ); |
| 982 | set_cr_intercept(svm, INTERCEPT_CR4_READ); |
| 983 | set_cr_intercept(svm, INTERCEPT_CR0_WRITE); |
| 984 | set_cr_intercept(svm, INTERCEPT_CR3_WRITE); |
| 985 | set_cr_intercept(svm, INTERCEPT_CR4_WRITE); |
Suravee Suthikulpanit | 3bbf356 | 2016-05-04 14:09:51 -0500 | [diff] [blame] | 986 | if (!kvm_vcpu_apicv_active(&svm->vcpu)) |
| 987 | set_cr_intercept(svm, INTERCEPT_CR8_WRITE); |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 988 | |
Paolo Bonzini | 5315c71 | 2014-03-03 13:08:29 +0100 | [diff] [blame] | 989 | set_dr_intercepts(svm); |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 990 | |
Joerg Roedel | 18c918c | 2010-11-30 18:03:59 +0100 | [diff] [blame] | 991 | set_exception_intercept(svm, PF_VECTOR); |
| 992 | set_exception_intercept(svm, UD_VECTOR); |
| 993 | set_exception_intercept(svm, MC_VECTOR); |
Eric Northup | 54a2055 | 2015-11-03 18:03:53 +0100 | [diff] [blame] | 994 | set_exception_intercept(svm, AC_VECTOR); |
Paolo Bonzini | cbdb967 | 2015-11-10 09:14:39 +0100 | [diff] [blame] | 995 | set_exception_intercept(svm, DB_VECTOR); |
Liran Alon | 9718420 | 2018-03-12 13:12:52 +0200 | [diff] [blame] | 996 | /* |
| 997 | * Guest access to VMware backdoor ports could legitimately |
| 998 | * trigger #GP because of TSS I/O permission bitmap. |
| 999 | * We intercept those #GP and allow access to them anyway |
| 1000 | * as VMware does. |
| 1001 | */ |
| 1002 | if (enable_vmware_backdoor) |
| 1003 | set_exception_intercept(svm, GP_VECTOR); |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1004 | |
Joerg Roedel | 8a05a1b8 | 2010-11-30 18:04:00 +0100 | [diff] [blame] | 1005 | set_intercept(svm, INTERCEPT_INTR); |
| 1006 | set_intercept(svm, INTERCEPT_NMI); |
| 1007 | set_intercept(svm, INTERCEPT_SMI); |
| 1008 | set_intercept(svm, INTERCEPT_SELECTIVE_CR0); |
Avi Kivity | 332b56e | 2011-11-10 14:57:24 +0200 | [diff] [blame] | 1009 | set_intercept(svm, INTERCEPT_RDPMC); |
Joerg Roedel | 8a05a1b8 | 2010-11-30 18:04:00 +0100 | [diff] [blame] | 1010 | set_intercept(svm, INTERCEPT_CPUID); |
| 1011 | set_intercept(svm, INTERCEPT_INVD); |
Joerg Roedel | 8a05a1b8 | 2010-11-30 18:04:00 +0100 | [diff] [blame] | 1012 | set_intercept(svm, INTERCEPT_INVLPG); |
| 1013 | set_intercept(svm, INTERCEPT_INVLPGA); |
| 1014 | set_intercept(svm, INTERCEPT_IOIO_PROT); |
| 1015 | set_intercept(svm, INTERCEPT_MSR_PROT); |
| 1016 | set_intercept(svm, INTERCEPT_TASK_SWITCH); |
| 1017 | set_intercept(svm, INTERCEPT_SHUTDOWN); |
| 1018 | set_intercept(svm, INTERCEPT_VMRUN); |
| 1019 | set_intercept(svm, INTERCEPT_VMMCALL); |
| 1020 | set_intercept(svm, INTERCEPT_VMLOAD); |
| 1021 | set_intercept(svm, INTERCEPT_VMSAVE); |
| 1022 | set_intercept(svm, INTERCEPT_STGI); |
| 1023 | set_intercept(svm, INTERCEPT_CLGI); |
| 1024 | set_intercept(svm, INTERCEPT_SKINIT); |
| 1025 | set_intercept(svm, INTERCEPT_WBINVD); |
Joerg Roedel | 81dd35d | 2010-12-07 17:15:06 +0100 | [diff] [blame] | 1026 | set_intercept(svm, INTERCEPT_XSETBV); |
Jim Mattson | 0cb8410 | 2019-09-19 15:59:17 -0700 | [diff] [blame] | 1027 | set_intercept(svm, INTERCEPT_RDPRU); |
Brijesh Singh | 7607b71 | 2018-02-19 10:14:44 -0600 | [diff] [blame] | 1028 | set_intercept(svm, INTERCEPT_RSM); |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1029 | |
Wanpeng Li | 4d5422c | 2018-03-12 04:53:02 -0700 | [diff] [blame] | 1030 | if (!kvm_mwait_in_guest(svm->vcpu.kvm)) { |
Michael S. Tsirkin | 668fffa | 2017-04-21 12:27:17 +0200 | [diff] [blame] | 1031 | set_intercept(svm, INTERCEPT_MONITOR); |
| 1032 | set_intercept(svm, INTERCEPT_MWAIT); |
| 1033 | } |
| 1034 | |
Wanpeng Li | caa057a | 2018-03-12 04:53:03 -0700 | [diff] [blame] | 1035 | if (!kvm_hlt_in_guest(svm->vcpu.kvm)) |
| 1036 | set_intercept(svm, INTERCEPT_HLT); |
| 1037 | |
Tom Lendacky | d0ec49d | 2017-07-17 16:10:27 -0500 | [diff] [blame] | 1038 | control->iopm_base_pa = __sme_set(iopm_base); |
| 1039 | control->msrpm_base_pa = __sme_set(__pa(svm->msrpm)); |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1040 | control->int_ctl = V_INTR_MASKING_MASK; |
| 1041 | |
| 1042 | init_seg(&save->es); |
| 1043 | init_seg(&save->ss); |
| 1044 | init_seg(&save->ds); |
| 1045 | init_seg(&save->fs); |
| 1046 | init_seg(&save->gs); |
| 1047 | |
| 1048 | save->cs.selector = 0xf000; |
Paolo Bonzini | 04b6683 | 2013-03-19 16:30:26 +0100 | [diff] [blame] | 1049 | save->cs.base = 0xffff0000; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1050 | /* Executable/Readable Code Segment */ |
| 1051 | save->cs.attrib = SVM_SELECTOR_READ_MASK | SVM_SELECTOR_P_MASK | |
| 1052 | SVM_SELECTOR_S_MASK | SVM_SELECTOR_CODE_MASK; |
| 1053 | save->cs.limit = 0xffff; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1054 | |
| 1055 | save->gdtr.limit = 0xffff; |
| 1056 | save->idtr.limit = 0xffff; |
| 1057 | |
| 1058 | init_sys_seg(&save->ldtr, SEG_TYPE_LDT); |
| 1059 | init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16); |
| 1060 | |
Paolo Bonzini | 5690891 | 2015-10-19 11:30:19 +0200 | [diff] [blame] | 1061 | svm_set_efer(&svm->vcpu, 0); |
Mike Day | d77c26f | 2007-10-08 09:02:08 -0400 | [diff] [blame] | 1062 | save->dr6 = 0xffff0ff0; |
Avi Kivity | f6e7847 | 2010-08-02 15:30:20 +0300 | [diff] [blame] | 1063 | kvm_set_rflags(&svm->vcpu, 2); |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1064 | save->rip = 0x0000fff0; |
Marcelo Tosatti | 5fdbf97 | 2008-06-27 14:58:02 -0300 | [diff] [blame] | 1065 | svm->vcpu.arch.regs[VCPU_REGS_RIP] = save->rip; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1066 | |
Joerg Roedel | e023171 | 2010-02-24 18:59:10 +0100 | [diff] [blame] | 1067 | /* |
Eduardo Habkost | 18fa000 | 2009-10-24 02:49:59 -0200 | [diff] [blame] | 1068 | * svm_set_cr0() sets PG and WP and clears NW and CD on save->cr0. |
Nadav Amit | d28bc9d | 2015-04-13 14:34:08 +0300 | [diff] [blame] | 1069 | * It also updates the guest-visible cr0 value. |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1070 | */ |
Paolo Bonzini | 79a8059 | 2015-09-21 07:46:55 +0200 | [diff] [blame] | 1071 | svm_set_cr0(&svm->vcpu, X86_CR0_NW | X86_CR0_CD | X86_CR0_ET); |
Igor Mammedov | ebae871 | 2015-09-18 15:39:05 +0200 | [diff] [blame] | 1072 | kvm_mmu_reset_context(&svm->vcpu); |
Eduardo Habkost | 18fa000 | 2009-10-24 02:49:59 -0200 | [diff] [blame] | 1073 | |
Rusty Russell | 66aee91 | 2007-07-17 23:34:16 +1000 | [diff] [blame] | 1074 | save->cr4 = X86_CR4_PAE; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1075 | /* rdx = ?? */ |
Joerg Roedel | 709ddeb | 2008-02-07 13:47:45 +0100 | [diff] [blame] | 1076 | |
| 1077 | if (npt_enabled) { |
| 1078 | /* Setup VMCB for Nested Paging */ |
Tom Lendacky | cea3a19 | 2017-12-04 10:57:24 -0600 | [diff] [blame] | 1079 | control->nested_ctl |= SVM_NESTED_CTL_NP_ENABLE; |
Joerg Roedel | 8a05a1b8 | 2010-11-30 18:04:00 +0100 | [diff] [blame] | 1080 | clr_intercept(svm, INTERCEPT_INVLPG); |
Joerg Roedel | 18c918c | 2010-11-30 18:03:59 +0100 | [diff] [blame] | 1081 | clr_exception_intercept(svm, PF_VECTOR); |
Roedel, Joerg | 4ee546b | 2010-12-03 10:50:51 +0100 | [diff] [blame] | 1082 | clr_cr_intercept(svm, INTERCEPT_CR3_READ); |
| 1083 | clr_cr_intercept(svm, INTERCEPT_CR3_WRITE); |
Radim Krčmář | 7454570 | 2015-04-27 15:11:25 +0200 | [diff] [blame] | 1084 | save->g_pat = svm->vcpu.arch.pat; |
Joerg Roedel | 709ddeb | 2008-02-07 13:47:45 +0100 | [diff] [blame] | 1085 | save->cr3 = 0; |
| 1086 | save->cr4 = 0; |
| 1087 | } |
Joerg Roedel | f40f6a4 | 2010-12-03 15:25:15 +0100 | [diff] [blame] | 1088 | svm->asid_generation = 0; |
Alexander Graf | 1371d90 | 2008-11-25 20:17:04 +0100 | [diff] [blame] | 1089 | |
Joerg Roedel | e6aa9ab | 2009-08-07 11:49:33 +0200 | [diff] [blame] | 1090 | svm->nested.vmcb = 0; |
Joerg Roedel | 2af9194 | 2009-08-07 11:49:28 +0200 | [diff] [blame] | 1091 | svm->vcpu.arch.hflags = 0; |
| 1092 | |
Babu Moger | 8566ac8 | 2018-03-16 16:37:26 -0400 | [diff] [blame] | 1093 | if (pause_filter_count) { |
| 1094 | control->pause_filter_count = pause_filter_count; |
| 1095 | if (pause_filter_thresh) |
| 1096 | control->pause_filter_thresh = pause_filter_thresh; |
Joerg Roedel | 8a05a1b8 | 2010-11-30 18:04:00 +0100 | [diff] [blame] | 1097 | set_intercept(svm, INTERCEPT_PAUSE); |
Babu Moger | 8566ac8 | 2018-03-16 16:37:26 -0400 | [diff] [blame] | 1098 | } else { |
| 1099 | clr_intercept(svm, INTERCEPT_PAUSE); |
Mark Langsdorf | 565d099 | 2009-10-06 14:25:02 -0500 | [diff] [blame] | 1100 | } |
| 1101 | |
Suravee Suthikulpanit | 67034bb | 2017-09-12 10:42:42 -0500 | [diff] [blame] | 1102 | if (kvm_vcpu_apicv_active(&svm->vcpu)) |
Suravee Suthikulpanit | 44a95da | 2016-05-04 14:09:46 -0500 | [diff] [blame] | 1103 | avic_init_vmcb(svm); |
| 1104 | |
Janakarajan Natarajan | 89c8a49 | 2017-07-06 15:50:47 -0500 | [diff] [blame] | 1105 | /* |
| 1106 | * If hardware supports Virtual VMLOAD VMSAVE then enable it |
| 1107 | * in VMCB and clear intercepts to avoid #VMEXIT. |
| 1108 | */ |
| 1109 | if (vls) { |
| 1110 | clr_intercept(svm, INTERCEPT_VMLOAD); |
| 1111 | clr_intercept(svm, INTERCEPT_VMSAVE); |
| 1112 | svm->vmcb->control.virt_ext |= VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK; |
| 1113 | } |
| 1114 | |
Janakarajan Natarajan | 640bd6e | 2017-08-23 09:57:19 -0500 | [diff] [blame] | 1115 | if (vgif) { |
| 1116 | clr_intercept(svm, INTERCEPT_STGI); |
| 1117 | clr_intercept(svm, INTERCEPT_CLGI); |
| 1118 | svm->vmcb->control.int_ctl |= V_GIF_ENABLE_MASK; |
| 1119 | } |
| 1120 | |
Brijesh Singh | 35c6f649 | 2017-12-04 10:57:39 -0600 | [diff] [blame] | 1121 | if (sev_guest(svm->vcpu.kvm)) { |
Brijesh Singh | 1654efc | 2017-12-04 10:57:34 -0600 | [diff] [blame] | 1122 | svm->vmcb->control.nested_ctl |= SVM_NESTED_CTL_SEV_ENABLE; |
Brijesh Singh | 35c6f649 | 2017-12-04 10:57:39 -0600 | [diff] [blame] | 1123 | clr_exception_intercept(svm, UD_VECTOR); |
| 1124 | } |
Brijesh Singh | 1654efc | 2017-12-04 10:57:34 -0600 | [diff] [blame] | 1125 | |
Roedel, Joerg | 8d28fec | 2010-12-03 13:15:21 +0100 | [diff] [blame] | 1126 | mark_all_dirty(svm->vmcb); |
| 1127 | |
Joerg Roedel | 2af9194 | 2009-08-07 11:49:28 +0200 | [diff] [blame] | 1128 | enable_gif(svm); |
Suravee Suthikulpanit | 44a95da | 2016-05-04 14:09:46 -0500 | [diff] [blame] | 1129 | |
| 1130 | } |
| 1131 | |
Nadav Amit | d28bc9d | 2015-04-13 14:34:08 +0300 | [diff] [blame] | 1132 | static void svm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) |
Avi Kivity | 04d2cc7 | 2007-09-10 18:10:54 +0300 | [diff] [blame] | 1133 | { |
| 1134 | struct vcpu_svm *svm = to_svm(vcpu); |
Julian Stecklina | 66f7b72 | 2012-12-05 15:26:19 +0100 | [diff] [blame] | 1135 | u32 dummy; |
| 1136 | u32 eax = 1; |
Avi Kivity | 04d2cc7 | 2007-09-10 18:10:54 +0300 | [diff] [blame] | 1137 | |
KarimAllah Ahmed | b2ac58f | 2018-02-03 15:56:23 +0100 | [diff] [blame] | 1138 | svm->spec_ctrl = 0; |
Thomas Gleixner | ccbcd26 | 2018-05-09 23:01:01 +0200 | [diff] [blame] | 1139 | svm->virt_spec_ctrl = 0; |
KarimAllah Ahmed | b2ac58f | 2018-02-03 15:56:23 +0100 | [diff] [blame] | 1140 | |
Nadav Amit | d28bc9d | 2015-04-13 14:34:08 +0300 | [diff] [blame] | 1141 | if (!init_event) { |
| 1142 | svm->vcpu.arch.apic_base = APIC_DEFAULT_PHYS_BASE | |
| 1143 | MSR_IA32_APICBASE_ENABLE; |
| 1144 | if (kvm_vcpu_is_reset_bsp(&svm->vcpu)) |
| 1145 | svm->vcpu.arch.apic_base |= MSR_IA32_APICBASE_BSP; |
| 1146 | } |
Paolo Bonzini | 5690891 | 2015-10-19 11:30:19 +0200 | [diff] [blame] | 1147 | init_vmcb(svm); |
Avi Kivity | 7043338 | 2007-11-07 12:57:23 +0200 | [diff] [blame] | 1148 | |
Sean Christopherson | f91af51 | 2020-03-04 17:34:37 -0800 | [diff] [blame] | 1149 | kvm_cpuid(vcpu, &eax, &dummy, &dummy, &dummy, false); |
Sean Christopherson | de3cd11 | 2019-04-30 10:36:17 -0700 | [diff] [blame] | 1150 | kvm_rdx_write(vcpu, eax); |
Suravee Suthikulpanit | 44a95da | 2016-05-04 14:09:46 -0500 | [diff] [blame] | 1151 | |
| 1152 | if (kvm_vcpu_apicv_active(vcpu) && !init_event) |
| 1153 | avic_update_vapic_bar(svm, APIC_DEFAULT_PHYS_BASE); |
Avi Kivity | 04d2cc7 | 2007-09-10 18:10:54 +0300 | [diff] [blame] | 1154 | } |
| 1155 | |
Sean Christopherson | 987b259 | 2019-12-18 13:54:55 -0800 | [diff] [blame] | 1156 | static int svm_create_vcpu(struct kvm_vcpu *vcpu) |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1157 | { |
Gregory Haskins | a2fa3e9 | 2007-07-27 08:13:10 -0400 | [diff] [blame] | 1158 | struct vcpu_svm *svm; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1159 | struct page *page; |
Joerg Roedel | f65c229 | 2008-02-13 18:58:46 +0100 | [diff] [blame] | 1160 | struct page *msrpm_pages; |
Alexander Graf | b286d5d | 2008-11-25 20:17:05 +0100 | [diff] [blame] | 1161 | struct page *hsave_page; |
Alexander Graf | 3d6368e | 2008-11-25 20:17:07 +0100 | [diff] [blame] | 1162 | struct page *nested_msrpm_pages; |
Rusty Russell | fb3f0f5 | 2007-07-27 17:16:56 +1000 | [diff] [blame] | 1163 | int err; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1164 | |
Sean Christopherson | a9dd6f0 | 2019-12-18 13:54:52 -0800 | [diff] [blame] | 1165 | BUILD_BUG_ON(offsetof(struct vcpu_svm, vcpu) != 0); |
| 1166 | svm = to_svm(vcpu); |
Rusty Russell | fb3f0f5 | 2007-07-27 17:16:56 +1000 | [diff] [blame] | 1167 | |
Joerg Roedel | f65c229 | 2008-02-13 18:58:46 +0100 | [diff] [blame] | 1168 | err = -ENOMEM; |
Ben Gardon | 1ec6964 | 2019-02-11 11:02:51 -0800 | [diff] [blame] | 1169 | page = alloc_page(GFP_KERNEL_ACCOUNT); |
Takuya Yoshikawa | b7af404 | 2010-03-09 14:55:19 +0900 | [diff] [blame] | 1170 | if (!page) |
Sean Christopherson | 987b259 | 2019-12-18 13:54:55 -0800 | [diff] [blame] | 1171 | goto out; |
Takuya Yoshikawa | b7af404 | 2010-03-09 14:55:19 +0900 | [diff] [blame] | 1172 | |
Ben Gardon | 1ec6964 | 2019-02-11 11:02:51 -0800 | [diff] [blame] | 1173 | msrpm_pages = alloc_pages(GFP_KERNEL_ACCOUNT, MSRPM_ALLOC_ORDER); |
Joerg Roedel | f65c229 | 2008-02-13 18:58:46 +0100 | [diff] [blame] | 1174 | if (!msrpm_pages) |
Takuya Yoshikawa | b7af404 | 2010-03-09 14:55:19 +0900 | [diff] [blame] | 1175 | goto free_page1; |
Alexander Graf | 3d6368e | 2008-11-25 20:17:07 +0100 | [diff] [blame] | 1176 | |
Ben Gardon | 1ec6964 | 2019-02-11 11:02:51 -0800 | [diff] [blame] | 1177 | nested_msrpm_pages = alloc_pages(GFP_KERNEL_ACCOUNT, MSRPM_ALLOC_ORDER); |
Alexander Graf | 3d6368e | 2008-11-25 20:17:07 +0100 | [diff] [blame] | 1178 | if (!nested_msrpm_pages) |
Takuya Yoshikawa | b7af404 | 2010-03-09 14:55:19 +0900 | [diff] [blame] | 1179 | goto free_page2; |
Joerg Roedel | f65c229 | 2008-02-13 18:58:46 +0100 | [diff] [blame] | 1180 | |
Ben Gardon | 1ec6964 | 2019-02-11 11:02:51 -0800 | [diff] [blame] | 1181 | hsave_page = alloc_page(GFP_KERNEL_ACCOUNT); |
Alexander Graf | b286d5d | 2008-11-25 20:17:05 +0100 | [diff] [blame] | 1182 | if (!hsave_page) |
Takuya Yoshikawa | b7af404 | 2010-03-09 14:55:19 +0900 | [diff] [blame] | 1183 | goto free_page3; |
| 1184 | |
Suravee Suthikulpanit | dfa2009 | 2017-09-12 10:42:40 -0500 | [diff] [blame] | 1185 | err = avic_init_vcpu(svm); |
| 1186 | if (err) |
| 1187 | goto free_page4; |
Suravee Suthikulpanit | 44a95da | 2016-05-04 14:09:46 -0500 | [diff] [blame] | 1188 | |
Suravee Suthikulpanit | 8221c13 | 2016-05-04 14:09:52 -0500 | [diff] [blame] | 1189 | /* We initialize this flag to true to make sure that the is_running |
| 1190 | * bit would be set the first time the vcpu is loaded. |
| 1191 | */ |
Suravee Suthikulpanit | 6c3e442 | 2019-11-14 14:15:12 -0600 | [diff] [blame] | 1192 | if (irqchip_in_kernel(vcpu->kvm) && kvm_apicv_activated(vcpu->kvm)) |
| 1193 | svm->avic_is_running = true; |
Suravee Suthikulpanit | 8221c13 | 2016-05-04 14:09:52 -0500 | [diff] [blame] | 1194 | |
Joerg Roedel | e6aa9ab | 2009-08-07 11:49:33 +0200 | [diff] [blame] | 1195 | svm->nested.hsave = page_address(hsave_page); |
Paolo Bonzini | cc440cd | 2020-05-13 13:36:32 -0400 | [diff] [blame] | 1196 | clear_page(svm->nested.hsave); |
Alexander Graf | b286d5d | 2008-11-25 20:17:05 +0100 | [diff] [blame] | 1197 | |
Takuya Yoshikawa | b7af404 | 2010-03-09 14:55:19 +0900 | [diff] [blame] | 1198 | svm->msrpm = page_address(msrpm_pages); |
| 1199 | svm_vcpu_init_msrpm(svm->msrpm); |
| 1200 | |
Joerg Roedel | e6aa9ab | 2009-08-07 11:49:33 +0200 | [diff] [blame] | 1201 | svm->nested.msrpm = page_address(nested_msrpm_pages); |
Joerg Roedel | 323c3d8 | 2010-03-01 15:34:37 +0100 | [diff] [blame] | 1202 | svm_vcpu_init_msrpm(svm->nested.msrpm); |
Alexander Graf | 3d6368e | 2008-11-25 20:17:07 +0100 | [diff] [blame] | 1203 | |
Gregory Haskins | a2fa3e9 | 2007-07-27 08:13:10 -0400 | [diff] [blame] | 1204 | svm->vmcb = page_address(page); |
| 1205 | clear_page(svm->vmcb); |
Tom Lendacky | d0ec49d | 2017-07-17 16:10:27 -0500 | [diff] [blame] | 1206 | svm->vmcb_pa = __sme_set(page_to_pfn(page) << PAGE_SHIFT); |
Gregory Haskins | a2fa3e9 | 2007-07-27 08:13:10 -0400 | [diff] [blame] | 1207 | svm->asid_generation = 0; |
Paolo Bonzini | 5690891 | 2015-10-19 11:30:19 +0200 | [diff] [blame] | 1208 | init_vmcb(svm); |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1209 | |
Sean Christopherson | 7f27179 | 2019-12-18 13:54:51 -0800 | [diff] [blame] | 1210 | svm_init_osvw(vcpu); |
Paolo Bonzini | bab0c31 | 2020-02-11 18:40:58 +0100 | [diff] [blame] | 1211 | vcpu->arch.microcode_version = 0x01000065; |
Boris Ostrovsky | 2b036c6 | 2012-01-09 14:00:35 -0500 | [diff] [blame] | 1212 | |
Sean Christopherson | a9dd6f0 | 2019-12-18 13:54:52 -0800 | [diff] [blame] | 1213 | return 0; |
Avi Kivity | 36241b8 | 2006-12-22 01:05:20 -0800 | [diff] [blame] | 1214 | |
Suravee Suthikulpanit | 44a95da | 2016-05-04 14:09:46 -0500 | [diff] [blame] | 1215 | free_page4: |
| 1216 | __free_page(hsave_page); |
Takuya Yoshikawa | b7af404 | 2010-03-09 14:55:19 +0900 | [diff] [blame] | 1217 | free_page3: |
| 1218 | __free_pages(nested_msrpm_pages, MSRPM_ALLOC_ORDER); |
| 1219 | free_page2: |
| 1220 | __free_pages(msrpm_pages, MSRPM_ALLOC_ORDER); |
| 1221 | free_page1: |
| 1222 | __free_page(page); |
Sean Christopherson | 987b259 | 2019-12-18 13:54:55 -0800 | [diff] [blame] | 1223 | out: |
Sean Christopherson | a9dd6f0 | 2019-12-18 13:54:52 -0800 | [diff] [blame] | 1224 | return err; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1225 | } |
| 1226 | |
Jim Mattson | fd65d31 | 2018-05-22 09:54:20 -0700 | [diff] [blame] | 1227 | static void svm_clear_current_vmcb(struct vmcb *vmcb) |
| 1228 | { |
| 1229 | int i; |
| 1230 | |
| 1231 | for_each_online_cpu(i) |
| 1232 | cmpxchg(&per_cpu(svm_data, i)->current_vmcb, vmcb, NULL); |
| 1233 | } |
| 1234 | |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1235 | static void svm_free_vcpu(struct kvm_vcpu *vcpu) |
| 1236 | { |
Gregory Haskins | a2fa3e9 | 2007-07-27 08:13:10 -0400 | [diff] [blame] | 1237 | struct vcpu_svm *svm = to_svm(vcpu); |
| 1238 | |
Jim Mattson | fd65d31 | 2018-05-22 09:54:20 -0700 | [diff] [blame] | 1239 | /* |
| 1240 | * The vmcb page can be recycled, causing a false negative in |
| 1241 | * svm_vcpu_load(). So, ensure that no logical CPU has this |
| 1242 | * vmcb page recorded as its current vmcb. |
| 1243 | */ |
| 1244 | svm_clear_current_vmcb(svm->vmcb); |
| 1245 | |
Tom Lendacky | d0ec49d | 2017-07-17 16:10:27 -0500 | [diff] [blame] | 1246 | __free_page(pfn_to_page(__sme_clr(svm->vmcb_pa) >> PAGE_SHIFT)); |
Joerg Roedel | f65c229 | 2008-02-13 18:58:46 +0100 | [diff] [blame] | 1247 | __free_pages(virt_to_page(svm->msrpm), MSRPM_ALLOC_ORDER); |
Joerg Roedel | e6aa9ab | 2009-08-07 11:49:33 +0200 | [diff] [blame] | 1248 | __free_page(virt_to_page(svm->nested.hsave)); |
| 1249 | __free_pages(virt_to_page(svm->nested.msrpm), MSRPM_ALLOC_ORDER); |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1250 | } |
| 1251 | |
Avi Kivity | 15ad714 | 2007-07-11 18:17:21 +0300 | [diff] [blame] | 1252 | static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu) |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1253 | { |
Gregory Haskins | a2fa3e9 | 2007-07-27 08:13:10 -0400 | [diff] [blame] | 1254 | struct vcpu_svm *svm = to_svm(vcpu); |
Ashok Raj | 15d4507 | 2018-02-01 22:59:43 +0100 | [diff] [blame] | 1255 | struct svm_cpu_data *sd = per_cpu(svm_data, cpu); |
Avi Kivity | 15ad714 | 2007-07-11 18:17:21 +0300 | [diff] [blame] | 1256 | int i; |
Avi Kivity | 0cc5064 | 2007-03-25 12:07:27 +0200 | [diff] [blame] | 1257 | |
Avi Kivity | 0cc5064 | 2007-03-25 12:07:27 +0200 | [diff] [blame] | 1258 | if (unlikely(cpu != vcpu->cpu)) { |
Marcelo Tosatti | 4b656b1 | 2009-07-21 12:47:45 -0300 | [diff] [blame] | 1259 | svm->asid_generation = 0; |
Roedel, Joerg | 8d28fec | 2010-12-03 13:15:21 +0100 | [diff] [blame] | 1260 | mark_all_dirty(svm->vmcb); |
Avi Kivity | 0cc5064 | 2007-03-25 12:07:27 +0200 | [diff] [blame] | 1261 | } |
Anthony Liguori | 94dfbdb | 2007-04-29 11:56:06 +0300 | [diff] [blame] | 1262 | |
Avi Kivity | 82ca2d1 | 2010-10-21 12:20:34 +0200 | [diff] [blame] | 1263 | #ifdef CONFIG_X86_64 |
| 1264 | rdmsrl(MSR_GS_BASE, to_svm(vcpu)->host.gs_base); |
| 1265 | #endif |
Avi Kivity | dacccfd | 2010-10-21 12:20:33 +0200 | [diff] [blame] | 1266 | savesegment(fs, svm->host.fs); |
| 1267 | savesegment(gs, svm->host.gs); |
| 1268 | svm->host.ldt = kvm_read_ldt(); |
| 1269 | |
Anthony Liguori | 94dfbdb | 2007-04-29 11:56:06 +0300 | [diff] [blame] | 1270 | for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++) |
Gregory Haskins | a2fa3e9 | 2007-07-27 08:13:10 -0400 | [diff] [blame] | 1271 | rdmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]); |
Joerg Roedel | fbc0db7 | 2011-03-25 09:44:46 +0100 | [diff] [blame] | 1272 | |
Haozhong Zhang | ad721883 | 2015-10-20 15:39:02 +0800 | [diff] [blame] | 1273 | if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) { |
| 1274 | u64 tsc_ratio = vcpu->arch.tsc_scaling_ratio; |
| 1275 | if (tsc_ratio != __this_cpu_read(current_tsc_ratio)) { |
| 1276 | __this_cpu_write(current_tsc_ratio, tsc_ratio); |
| 1277 | wrmsrl(MSR_AMD64_TSC_RATIO, tsc_ratio); |
| 1278 | } |
Joerg Roedel | fbc0db7 | 2011-03-25 09:44:46 +0100 | [diff] [blame] | 1279 | } |
Paolo Bonzini | 46896c7 | 2015-11-12 14:49:16 +0100 | [diff] [blame] | 1280 | /* This assumes that the kernel never uses MSR_TSC_AUX */ |
| 1281 | if (static_cpu_has(X86_FEATURE_RDTSCP)) |
| 1282 | wrmsrl(MSR_TSC_AUX, svm->tsc_aux); |
Suravee Suthikulpanit | 8221c13 | 2016-05-04 14:09:52 -0500 | [diff] [blame] | 1283 | |
Ashok Raj | 15d4507 | 2018-02-01 22:59:43 +0100 | [diff] [blame] | 1284 | if (sd->current_vmcb != svm->vmcb) { |
| 1285 | sd->current_vmcb = svm->vmcb; |
| 1286 | indirect_branch_prediction_barrier(); |
| 1287 | } |
Suravee Suthikulpanit | 8221c13 | 2016-05-04 14:09:52 -0500 | [diff] [blame] | 1288 | avic_vcpu_load(vcpu, cpu); |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1289 | } |
| 1290 | |
| 1291 | static void svm_vcpu_put(struct kvm_vcpu *vcpu) |
| 1292 | { |
Gregory Haskins | a2fa3e9 | 2007-07-27 08:13:10 -0400 | [diff] [blame] | 1293 | struct vcpu_svm *svm = to_svm(vcpu); |
Anthony Liguori | 94dfbdb | 2007-04-29 11:56:06 +0300 | [diff] [blame] | 1294 | int i; |
| 1295 | |
Suravee Suthikulpanit | 8221c13 | 2016-05-04 14:09:52 -0500 | [diff] [blame] | 1296 | avic_vcpu_put(vcpu); |
| 1297 | |
Avi Kivity | e1beb1d | 2007-11-18 13:50:24 +0200 | [diff] [blame] | 1298 | ++vcpu->stat.host_state_reload; |
Avi Kivity | dacccfd | 2010-10-21 12:20:33 +0200 | [diff] [blame] | 1299 | kvm_load_ldt(svm->host.ldt); |
| 1300 | #ifdef CONFIG_X86_64 |
| 1301 | loadsegment(fs, svm->host.fs); |
Andy Lutomirski | 296f781 | 2016-04-26 12:23:29 -0700 | [diff] [blame] | 1302 | wrmsrl(MSR_KERNEL_GS_BASE, current->thread.gsbase); |
Joerg Roedel | 893a5ab | 2011-01-14 16:45:01 +0100 | [diff] [blame] | 1303 | load_gs_index(svm->host.gs); |
Avi Kivity | dacccfd | 2010-10-21 12:20:33 +0200 | [diff] [blame] | 1304 | #else |
Avi Kivity | 831ca60 | 2011-03-08 16:09:51 +0200 | [diff] [blame] | 1305 | #ifdef CONFIG_X86_32_LAZY_GS |
Avi Kivity | dacccfd | 2010-10-21 12:20:33 +0200 | [diff] [blame] | 1306 | loadsegment(gs, svm->host.gs); |
| 1307 | #endif |
Avi Kivity | 831ca60 | 2011-03-08 16:09:51 +0200 | [diff] [blame] | 1308 | #endif |
Anthony Liguori | 94dfbdb | 2007-04-29 11:56:06 +0300 | [diff] [blame] | 1309 | for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++) |
Gregory Haskins | a2fa3e9 | 2007-07-27 08:13:10 -0400 | [diff] [blame] | 1310 | wrmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]); |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1311 | } |
| 1312 | |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1313 | static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu) |
| 1314 | { |
Ladi Prosek | 9b61174 | 2017-06-21 09:06:59 +0200 | [diff] [blame] | 1315 | struct vcpu_svm *svm = to_svm(vcpu); |
| 1316 | unsigned long rflags = svm->vmcb->save.rflags; |
| 1317 | |
| 1318 | if (svm->nmi_singlestep) { |
| 1319 | /* Hide our flags if they were not set by the guest */ |
| 1320 | if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF)) |
| 1321 | rflags &= ~X86_EFLAGS_TF; |
| 1322 | if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF)) |
| 1323 | rflags &= ~X86_EFLAGS_RF; |
| 1324 | } |
| 1325 | return rflags; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1326 | } |
| 1327 | |
| 1328 | static void svm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) |
| 1329 | { |
Ladi Prosek | 9b61174 | 2017-06-21 09:06:59 +0200 | [diff] [blame] | 1330 | if (to_svm(vcpu)->nmi_singlestep) |
| 1331 | rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF); |
| 1332 | |
Paolo Bonzini | ae9fedc | 2014-05-14 09:39:49 +0200 | [diff] [blame] | 1333 | /* |
Andrea Gelmini | bb3541f | 2016-05-21 14:14:44 +0200 | [diff] [blame] | 1334 | * Any change of EFLAGS.VM is accompanied by a reload of SS |
Paolo Bonzini | ae9fedc | 2014-05-14 09:39:49 +0200 | [diff] [blame] | 1335 | * (caused by either a task switch or an inter-privilege IRET), |
| 1336 | * so we do not need to update the CPL here. |
| 1337 | */ |
Gregory Haskins | a2fa3e9 | 2007-07-27 08:13:10 -0400 | [diff] [blame] | 1338 | to_svm(vcpu)->vmcb->save.rflags = rflags; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1339 | } |
| 1340 | |
Avi Kivity | 6de4f3a | 2009-05-31 22:58:47 +0300 | [diff] [blame] | 1341 | static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg) |
| 1342 | { |
| 1343 | switch (reg) { |
| 1344 | case VCPU_EXREG_PDPTR: |
| 1345 | BUG_ON(!npt_enabled); |
Avi Kivity | 9f8fe50 | 2010-12-05 17:30:00 +0200 | [diff] [blame] | 1346 | load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu)); |
Avi Kivity | 6de4f3a | 2009-05-31 22:58:47 +0300 | [diff] [blame] | 1347 | break; |
| 1348 | default: |
Sean Christopherson | 34059c2 | 2019-09-27 14:45:23 -0700 | [diff] [blame] | 1349 | WARN_ON_ONCE(1); |
Avi Kivity | 6de4f3a | 2009-05-31 22:58:47 +0300 | [diff] [blame] | 1350 | } |
| 1351 | } |
| 1352 | |
Suravee Suthikulpanit | e14b778 | 2020-05-06 08:17:55 -0500 | [diff] [blame] | 1353 | static void svm_set_vintr(struct vcpu_svm *svm) |
Paolo Bonzini | 64b5bd2 | 2020-03-04 13:12:35 -0500 | [diff] [blame] | 1354 | { |
| 1355 | struct vmcb_control_area *control; |
| 1356 | |
| 1357 | /* The following fields are ignored when AVIC is enabled */ |
| 1358 | WARN_ON(kvm_vcpu_apicv_active(&svm->vcpu)); |
Suravee Suthikulpanit | e14b778 | 2020-05-06 08:17:55 -0500 | [diff] [blame] | 1359 | set_intercept(svm, INTERCEPT_VINTR); |
Paolo Bonzini | 64b5bd2 | 2020-03-04 13:12:35 -0500 | [diff] [blame] | 1360 | |
| 1361 | /* |
| 1362 | * This is just a dummy VINTR to actually cause a vmexit to happen. |
| 1363 | * Actual injection of virtual interrupts happens through EVENTINJ. |
| 1364 | */ |
| 1365 | control = &svm->vmcb->control; |
| 1366 | control->int_vector = 0x0; |
| 1367 | control->int_ctl &= ~V_INTR_PRIO_MASK; |
| 1368 | control->int_ctl |= V_IRQ_MASK | |
| 1369 | ((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT); |
| 1370 | mark_dirty(svm->vmcb, VMCB_INTR); |
| 1371 | } |
| 1372 | |
Alexander Graf | f0b8505 | 2008-11-25 20:17:01 +0100 | [diff] [blame] | 1373 | static void svm_clear_vintr(struct vcpu_svm *svm) |
| 1374 | { |
Paolo Bonzini | d8e4e58 | 2020-05-22 07:38:20 -0400 | [diff] [blame] | 1375 | const u32 mask = V_TPR_MASK | V_GIF_ENABLE_MASK | V_GIF_MASK | V_INTR_MASKING_MASK; |
Joerg Roedel | 8a05a1b8 | 2010-11-30 18:04:00 +0100 | [diff] [blame] | 1376 | clr_intercept(svm, INTERCEPT_VINTR); |
Paolo Bonzini | 64b5bd2 | 2020-03-04 13:12:35 -0500 | [diff] [blame] | 1377 | |
Paolo Bonzini | d8e4e58 | 2020-05-22 07:38:20 -0400 | [diff] [blame] | 1378 | /* Drop int_ctl fields related to VINTR injection. */ |
| 1379 | svm->vmcb->control.int_ctl &= mask; |
| 1380 | if (is_guest_mode(&svm->vcpu)) { |
Paolo Bonzini | fb7333d | 2020-06-08 07:11:47 -0400 | [diff] [blame^] | 1381 | svm->nested.hsave->control.int_ctl &= mask; |
| 1382 | |
Paolo Bonzini | d8e4e58 | 2020-05-22 07:38:20 -0400 | [diff] [blame] | 1383 | WARN_ON((svm->vmcb->control.int_ctl & V_TPR_MASK) != |
| 1384 | (svm->nested.ctl.int_ctl & V_TPR_MASK)); |
| 1385 | svm->vmcb->control.int_ctl |= svm->nested.ctl.int_ctl & ~mask; |
| 1386 | } |
| 1387 | |
Paolo Bonzini | 64b5bd2 | 2020-03-04 13:12:35 -0500 | [diff] [blame] | 1388 | mark_dirty(svm->vmcb, VMCB_INTR); |
Alexander Graf | f0b8505 | 2008-11-25 20:17:01 +0100 | [diff] [blame] | 1389 | } |
| 1390 | |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1391 | static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg) |
| 1392 | { |
Gregory Haskins | a2fa3e9 | 2007-07-27 08:13:10 -0400 | [diff] [blame] | 1393 | struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1394 | |
| 1395 | switch (seg) { |
| 1396 | case VCPU_SREG_CS: return &save->cs; |
| 1397 | case VCPU_SREG_DS: return &save->ds; |
| 1398 | case VCPU_SREG_ES: return &save->es; |
| 1399 | case VCPU_SREG_FS: return &save->fs; |
| 1400 | case VCPU_SREG_GS: return &save->gs; |
| 1401 | case VCPU_SREG_SS: return &save->ss; |
| 1402 | case VCPU_SREG_TR: return &save->tr; |
| 1403 | case VCPU_SREG_LDTR: return &save->ldtr; |
| 1404 | } |
| 1405 | BUG(); |
Al Viro | 8b6d44c | 2007-02-09 16:38:40 +0000 | [diff] [blame] | 1406 | return NULL; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1407 | } |
| 1408 | |
| 1409 | static u64 svm_get_segment_base(struct kvm_vcpu *vcpu, int seg) |
| 1410 | { |
| 1411 | struct vmcb_seg *s = svm_seg(vcpu, seg); |
| 1412 | |
| 1413 | return s->base; |
| 1414 | } |
| 1415 | |
| 1416 | static void svm_get_segment(struct kvm_vcpu *vcpu, |
| 1417 | struct kvm_segment *var, int seg) |
| 1418 | { |
| 1419 | struct vmcb_seg *s = svm_seg(vcpu, seg); |
| 1420 | |
| 1421 | var->base = s->base; |
| 1422 | var->limit = s->limit; |
| 1423 | var->selector = s->selector; |
| 1424 | var->type = s->attrib & SVM_SELECTOR_TYPE_MASK; |
| 1425 | var->s = (s->attrib >> SVM_SELECTOR_S_SHIFT) & 1; |
| 1426 | var->dpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3; |
| 1427 | var->present = (s->attrib >> SVM_SELECTOR_P_SHIFT) & 1; |
| 1428 | var->avl = (s->attrib >> SVM_SELECTOR_AVL_SHIFT) & 1; |
| 1429 | var->l = (s->attrib >> SVM_SELECTOR_L_SHIFT) & 1; |
| 1430 | var->db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1; |
Jim Mattson | 80112c8 | 2014-07-08 09:47:41 +0530 | [diff] [blame] | 1431 | |
| 1432 | /* |
| 1433 | * AMD CPUs circa 2014 track the G bit for all segments except CS. |
| 1434 | * However, the SVM spec states that the G bit is not observed by the |
| 1435 | * CPU, and some VMware virtual CPUs drop the G bit for all segments. |
| 1436 | * So let's synthesize a legal G bit for all segments, this helps |
| 1437 | * running KVM nested. It also helps cross-vendor migration, because |
| 1438 | * Intel's vmentry has a check on the 'G' bit. |
| 1439 | */ |
| 1440 | var->g = s->limit > 0xfffff; |
Amit Shah | 25022ac | 2008-10-27 09:04:17 +0000 | [diff] [blame] | 1441 | |
Joerg Roedel | e023171 | 2010-02-24 18:59:10 +0100 | [diff] [blame] | 1442 | /* |
| 1443 | * AMD's VMCB does not have an explicit unusable field, so emulate it |
Andre Przywara | 19bca6a | 2009-04-28 12:45:30 +0200 | [diff] [blame] | 1444 | * for cross vendor migration purposes by "not present" |
| 1445 | */ |
Gioh Kim | 8eae957 | 2017-05-30 15:24:45 +0200 | [diff] [blame] | 1446 | var->unusable = !var->present; |
Andre Przywara | 19bca6a | 2009-04-28 12:45:30 +0200 | [diff] [blame] | 1447 | |
Andre Przywara | 1fbdc7a | 2009-01-11 22:39:44 +0100 | [diff] [blame] | 1448 | switch (seg) { |
Andre Przywara | 1fbdc7a | 2009-01-11 22:39:44 +0100 | [diff] [blame] | 1449 | case VCPU_SREG_TR: |
| 1450 | /* |
| 1451 | * Work around a bug where the busy flag in the tr selector |
| 1452 | * isn't exposed |
| 1453 | */ |
Amit Shah | c0d0982 | 2008-10-27 09:04:18 +0000 | [diff] [blame] | 1454 | var->type |= 0x2; |
Andre Przywara | 1fbdc7a | 2009-01-11 22:39:44 +0100 | [diff] [blame] | 1455 | break; |
| 1456 | case VCPU_SREG_DS: |
| 1457 | case VCPU_SREG_ES: |
| 1458 | case VCPU_SREG_FS: |
| 1459 | case VCPU_SREG_GS: |
| 1460 | /* |
| 1461 | * The accessed bit must always be set in the segment |
| 1462 | * descriptor cache, although it can be cleared in the |
| 1463 | * descriptor, the cached bit always remains at 1. Since |
| 1464 | * Intel has a check on this, set it here to support |
| 1465 | * cross-vendor migration. |
| 1466 | */ |
| 1467 | if (!var->unusable) |
| 1468 | var->type |= 0x1; |
| 1469 | break; |
Andre Przywara | b586eb0 | 2009-04-28 12:45:43 +0200 | [diff] [blame] | 1470 | case VCPU_SREG_SS: |
Joerg Roedel | e023171 | 2010-02-24 18:59:10 +0100 | [diff] [blame] | 1471 | /* |
| 1472 | * On AMD CPUs sometimes the DB bit in the segment |
Andre Przywara | b586eb0 | 2009-04-28 12:45:43 +0200 | [diff] [blame] | 1473 | * descriptor is left as 1, although the whole segment has |
| 1474 | * been made unusable. Clear it here to pass an Intel VMX |
| 1475 | * entry check when cross vendor migrating. |
| 1476 | */ |
| 1477 | if (var->unusable) |
| 1478 | var->db = 0; |
Roman Pen | d9c1b54 | 2017-06-01 10:55:03 +0200 | [diff] [blame] | 1479 | /* This is symmetric with svm_set_segment() */ |
Jan Kiszka | 33b458d | 2014-06-29 17:12:43 +0200 | [diff] [blame] | 1480 | var->dpl = to_svm(vcpu)->vmcb->save.cpl; |
Andre Przywara | b586eb0 | 2009-04-28 12:45:43 +0200 | [diff] [blame] | 1481 | break; |
Andre Przywara | 1fbdc7a | 2009-01-11 22:39:44 +0100 | [diff] [blame] | 1482 | } |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1483 | } |
| 1484 | |
Izik Eidus | 2e4d265 | 2008-03-24 19:38:34 +0200 | [diff] [blame] | 1485 | static int svm_get_cpl(struct kvm_vcpu *vcpu) |
| 1486 | { |
| 1487 | struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save; |
| 1488 | |
| 1489 | return save->cpl; |
| 1490 | } |
| 1491 | |
Gleb Natapov | 89a27f4 | 2010-02-16 10:51:48 +0200 | [diff] [blame] | 1492 | static void svm_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1493 | { |
Gregory Haskins | a2fa3e9 | 2007-07-27 08:13:10 -0400 | [diff] [blame] | 1494 | struct vcpu_svm *svm = to_svm(vcpu); |
| 1495 | |
Gleb Natapov | 89a27f4 | 2010-02-16 10:51:48 +0200 | [diff] [blame] | 1496 | dt->size = svm->vmcb->save.idtr.limit; |
| 1497 | dt->address = svm->vmcb->save.idtr.base; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1498 | } |
| 1499 | |
Gleb Natapov | 89a27f4 | 2010-02-16 10:51:48 +0200 | [diff] [blame] | 1500 | static void svm_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1501 | { |
Gregory Haskins | a2fa3e9 | 2007-07-27 08:13:10 -0400 | [diff] [blame] | 1502 | struct vcpu_svm *svm = to_svm(vcpu); |
| 1503 | |
Gleb Natapov | 89a27f4 | 2010-02-16 10:51:48 +0200 | [diff] [blame] | 1504 | svm->vmcb->save.idtr.limit = dt->size; |
| 1505 | svm->vmcb->save.idtr.base = dt->address ; |
Joerg Roedel | 17a703c | 2010-12-03 11:45:56 +0100 | [diff] [blame] | 1506 | mark_dirty(svm->vmcb, VMCB_DT); |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1507 | } |
| 1508 | |
Gleb Natapov | 89a27f4 | 2010-02-16 10:51:48 +0200 | [diff] [blame] | 1509 | static void svm_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1510 | { |
Gregory Haskins | a2fa3e9 | 2007-07-27 08:13:10 -0400 | [diff] [blame] | 1511 | struct vcpu_svm *svm = to_svm(vcpu); |
| 1512 | |
Gleb Natapov | 89a27f4 | 2010-02-16 10:51:48 +0200 | [diff] [blame] | 1513 | dt->size = svm->vmcb->save.gdtr.limit; |
| 1514 | dt->address = svm->vmcb->save.gdtr.base; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1515 | } |
| 1516 | |
Gleb Natapov | 89a27f4 | 2010-02-16 10:51:48 +0200 | [diff] [blame] | 1517 | static void svm_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1518 | { |
Gregory Haskins | a2fa3e9 | 2007-07-27 08:13:10 -0400 | [diff] [blame] | 1519 | struct vcpu_svm *svm = to_svm(vcpu); |
| 1520 | |
Gleb Natapov | 89a27f4 | 2010-02-16 10:51:48 +0200 | [diff] [blame] | 1521 | svm->vmcb->save.gdtr.limit = dt->size; |
| 1522 | svm->vmcb->save.gdtr.base = dt->address ; |
Joerg Roedel | 17a703c | 2010-12-03 11:45:56 +0100 | [diff] [blame] | 1523 | mark_dirty(svm->vmcb, VMCB_DT); |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1524 | } |
| 1525 | |
Avi Kivity | d225157 | 2010-01-06 10:55:27 +0200 | [diff] [blame] | 1526 | static void update_cr0_intercept(struct vcpu_svm *svm) |
| 1527 | { |
| 1528 | ulong gcr0 = svm->vcpu.arch.cr0; |
| 1529 | u64 *hcr0 = &svm->vmcb->save.cr0; |
| 1530 | |
Paolo Bonzini | bd7e5b0 | 2017-02-03 21:18:52 -0800 | [diff] [blame] | 1531 | *hcr0 = (*hcr0 & ~SVM_CR0_SELECTIVE_MASK) |
| 1532 | | (gcr0 & SVM_CR0_SELECTIVE_MASK); |
Avi Kivity | d225157 | 2010-01-06 10:55:27 +0200 | [diff] [blame] | 1533 | |
Joerg Roedel | dcca1a6 | 2010-12-03 11:45:54 +0100 | [diff] [blame] | 1534 | mark_dirty(svm->vmcb, VMCB_CR); |
Avi Kivity | d225157 | 2010-01-06 10:55:27 +0200 | [diff] [blame] | 1535 | |
Paolo Bonzini | bd7e5b0 | 2017-02-03 21:18:52 -0800 | [diff] [blame] | 1536 | if (gcr0 == *hcr0) { |
Roedel, Joerg | 4ee546b | 2010-12-03 10:50:51 +0100 | [diff] [blame] | 1537 | clr_cr_intercept(svm, INTERCEPT_CR0_READ); |
| 1538 | clr_cr_intercept(svm, INTERCEPT_CR0_WRITE); |
Avi Kivity | d225157 | 2010-01-06 10:55:27 +0200 | [diff] [blame] | 1539 | } else { |
Roedel, Joerg | 4ee546b | 2010-12-03 10:50:51 +0100 | [diff] [blame] | 1540 | set_cr_intercept(svm, INTERCEPT_CR0_READ); |
| 1541 | set_cr_intercept(svm, INTERCEPT_CR0_WRITE); |
Avi Kivity | d225157 | 2010-01-06 10:55:27 +0200 | [diff] [blame] | 1542 | } |
| 1543 | } |
| 1544 | |
Joerg Roedel | 883b0a9 | 2020-03-24 10:41:52 +0100 | [diff] [blame] | 1545 | void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1546 | { |
Gregory Haskins | a2fa3e9 | 2007-07-27 08:13:10 -0400 | [diff] [blame] | 1547 | struct vcpu_svm *svm = to_svm(vcpu); |
| 1548 | |
Avi Kivity | 05b3e0c | 2006-12-13 00:33:45 -0800 | [diff] [blame] | 1549 | #ifdef CONFIG_X86_64 |
Avi Kivity | f6801df | 2010-01-21 15:31:50 +0200 | [diff] [blame] | 1550 | if (vcpu->arch.efer & EFER_LME) { |
Rusty Russell | 707d92fa | 2007-07-17 23:19:08 +1000 | [diff] [blame] | 1551 | if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) { |
Avi Kivity | f6801df | 2010-01-21 15:31:50 +0200 | [diff] [blame] | 1552 | vcpu->arch.efer |= EFER_LMA; |
Carlo Marcelo Arenas Belon | 2b5203e | 2007-12-01 06:17:11 -0600 | [diff] [blame] | 1553 | svm->vmcb->save.efer |= EFER_LMA | EFER_LME; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1554 | } |
| 1555 | |
Mike Day | d77c26f | 2007-10-08 09:02:08 -0400 | [diff] [blame] | 1556 | if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) { |
Avi Kivity | f6801df | 2010-01-21 15:31:50 +0200 | [diff] [blame] | 1557 | vcpu->arch.efer &= ~EFER_LMA; |
Carlo Marcelo Arenas Belon | 2b5203e | 2007-12-01 06:17:11 -0600 | [diff] [blame] | 1558 | svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME); |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1559 | } |
| 1560 | } |
| 1561 | #endif |
Zhang Xiantao | ad312c7 | 2007-12-13 23:50:52 +0800 | [diff] [blame] | 1562 | vcpu->arch.cr0 = cr0; |
Avi Kivity | 888f9f3 | 2010-01-10 12:14:04 +0200 | [diff] [blame] | 1563 | |
| 1564 | if (!npt_enabled) |
| 1565 | cr0 |= X86_CR0_PG | X86_CR0_WP; |
Avi Kivity | 02daab2 | 2009-12-30 12:40:26 +0200 | [diff] [blame] | 1566 | |
Paolo Bonzini | bcf166a | 2015-10-01 13:19:55 +0200 | [diff] [blame] | 1567 | /* |
| 1568 | * re-enable caching here because the QEMU bios |
| 1569 | * does not do it - this results in some delay at |
| 1570 | * reboot |
| 1571 | */ |
| 1572 | if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED)) |
| 1573 | cr0 &= ~(X86_CR0_CD | X86_CR0_NW); |
Gregory Haskins | a2fa3e9 | 2007-07-27 08:13:10 -0400 | [diff] [blame] | 1574 | svm->vmcb->save.cr0 = cr0; |
Joerg Roedel | dcca1a6 | 2010-12-03 11:45:54 +0100 | [diff] [blame] | 1575 | mark_dirty(svm->vmcb, VMCB_CR); |
Avi Kivity | d225157 | 2010-01-06 10:55:27 +0200 | [diff] [blame] | 1576 | update_cr0_intercept(svm); |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1577 | } |
| 1578 | |
Joerg Roedel | 883b0a9 | 2020-03-24 10:41:52 +0100 | [diff] [blame] | 1579 | int svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1580 | { |
Andy Lutomirski | 1e02ce4 | 2014-10-24 15:58:08 -0700 | [diff] [blame] | 1581 | unsigned long host_cr4_mce = cr4_read_shadow() & X86_CR4_MCE; |
Joerg Roedel | e5eab0c | 2008-09-09 19:11:51 +0200 | [diff] [blame] | 1582 | unsigned long old_cr4 = to_svm(vcpu)->vmcb->save.cr4; |
| 1583 | |
Nadav Har'El | 5e1746d | 2011-05-25 23:03:24 +0300 | [diff] [blame] | 1584 | if (cr4 & X86_CR4_VMXE) |
| 1585 | return 1; |
| 1586 | |
Joerg Roedel | e5eab0c | 2008-09-09 19:11:51 +0200 | [diff] [blame] | 1587 | if (npt_enabled && ((old_cr4 ^ cr4) & X86_CR4_PGE)) |
Sean Christopherson | f55ac30 | 2020-03-20 14:28:12 -0700 | [diff] [blame] | 1588 | svm_flush_tlb(vcpu); |
Joerg Roedel | 6394b64 | 2008-04-09 14:15:29 +0200 | [diff] [blame] | 1589 | |
Joerg Roedel | ec07726 | 2008-04-09 14:15:28 +0200 | [diff] [blame] | 1590 | vcpu->arch.cr4 = cr4; |
| 1591 | if (!npt_enabled) |
| 1592 | cr4 |= X86_CR4_PAE; |
Joerg Roedel | 6394b64 | 2008-04-09 14:15:29 +0200 | [diff] [blame] | 1593 | cr4 |= host_cr4_mce; |
Joerg Roedel | ec07726 | 2008-04-09 14:15:28 +0200 | [diff] [blame] | 1594 | to_svm(vcpu)->vmcb->save.cr4 = cr4; |
Joerg Roedel | dcca1a6 | 2010-12-03 11:45:54 +0100 | [diff] [blame] | 1595 | mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR); |
Nadav Har'El | 5e1746d | 2011-05-25 23:03:24 +0300 | [diff] [blame] | 1596 | return 0; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1597 | } |
| 1598 | |
| 1599 | static void svm_set_segment(struct kvm_vcpu *vcpu, |
| 1600 | struct kvm_segment *var, int seg) |
| 1601 | { |
Gregory Haskins | a2fa3e9 | 2007-07-27 08:13:10 -0400 | [diff] [blame] | 1602 | struct vcpu_svm *svm = to_svm(vcpu); |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1603 | struct vmcb_seg *s = svm_seg(vcpu, seg); |
| 1604 | |
| 1605 | s->base = var->base; |
| 1606 | s->limit = var->limit; |
| 1607 | s->selector = var->selector; |
Roman Pen | d9c1b54 | 2017-06-01 10:55:03 +0200 | [diff] [blame] | 1608 | s->attrib = (var->type & SVM_SELECTOR_TYPE_MASK); |
| 1609 | s->attrib |= (var->s & 1) << SVM_SELECTOR_S_SHIFT; |
| 1610 | s->attrib |= (var->dpl & 3) << SVM_SELECTOR_DPL_SHIFT; |
| 1611 | s->attrib |= ((var->present & 1) && !var->unusable) << SVM_SELECTOR_P_SHIFT; |
| 1612 | s->attrib |= (var->avl & 1) << SVM_SELECTOR_AVL_SHIFT; |
| 1613 | s->attrib |= (var->l & 1) << SVM_SELECTOR_L_SHIFT; |
| 1614 | s->attrib |= (var->db & 1) << SVM_SELECTOR_DB_SHIFT; |
| 1615 | s->attrib |= (var->g & 1) << SVM_SELECTOR_G_SHIFT; |
Paolo Bonzini | ae9fedc | 2014-05-14 09:39:49 +0200 | [diff] [blame] | 1616 | |
| 1617 | /* |
| 1618 | * This is always accurate, except if SYSRET returned to a segment |
| 1619 | * with SS.DPL != 3. Intel does not have this quirk, and always |
| 1620 | * forces SS.DPL to 3 on sysret, so we ignore that case; fixing it |
| 1621 | * would entail passing the CPL to userspace and back. |
| 1622 | */ |
| 1623 | if (seg == VCPU_SREG_SS) |
Roman Pen | d9c1b54 | 2017-06-01 10:55:03 +0200 | [diff] [blame] | 1624 | /* This is symmetric with svm_get_segment() */ |
| 1625 | svm->vmcb->save.cpl = (var->dpl & 3); |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1626 | |
Joerg Roedel | 060d0c9 | 2010-12-03 11:45:57 +0100 | [diff] [blame] | 1627 | mark_dirty(svm->vmcb, VMCB_SEG); |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1628 | } |
| 1629 | |
Paolo Bonzini | cbdb967 | 2015-11-10 09:14:39 +0100 | [diff] [blame] | 1630 | static void update_bp_intercept(struct kvm_vcpu *vcpu) |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1631 | { |
Jan Kiszka | d0bfb94 | 2008-12-15 13:52:10 +0100 | [diff] [blame] | 1632 | struct vcpu_svm *svm = to_svm(vcpu); |
| 1633 | |
Joerg Roedel | 18c918c | 2010-11-30 18:03:59 +0100 | [diff] [blame] | 1634 | clr_exception_intercept(svm, BP_VECTOR); |
Gleb Natapov | 44c1143 | 2009-05-11 13:35:52 +0300 | [diff] [blame] | 1635 | |
Jan Kiszka | d0bfb94 | 2008-12-15 13:52:10 +0100 | [diff] [blame] | 1636 | if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) { |
Jan Kiszka | d0bfb94 | 2008-12-15 13:52:10 +0100 | [diff] [blame] | 1637 | if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) |
Joerg Roedel | 18c918c | 2010-11-30 18:03:59 +0100 | [diff] [blame] | 1638 | set_exception_intercept(svm, BP_VECTOR); |
Jan Kiszka | d0bfb94 | 2008-12-15 13:52:10 +0100 | [diff] [blame] | 1639 | } else |
| 1640 | vcpu->guest_debug = 0; |
Gleb Natapov | 44c1143 | 2009-05-11 13:35:52 +0300 | [diff] [blame] | 1641 | } |
| 1642 | |
Tejun Heo | 0fe1e00 | 2009-10-29 22:34:14 +0900 | [diff] [blame] | 1643 | static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd) |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1644 | { |
Tejun Heo | 0fe1e00 | 2009-10-29 22:34:14 +0900 | [diff] [blame] | 1645 | if (sd->next_asid > sd->max_asid) { |
| 1646 | ++sd->asid_generation; |
Brijesh Singh | 4faefff | 2017-12-04 10:57:25 -0600 | [diff] [blame] | 1647 | sd->next_asid = sd->min_asid; |
Gregory Haskins | a2fa3e9 | 2007-07-27 08:13:10 -0400 | [diff] [blame] | 1648 | svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1649 | } |
| 1650 | |
Tejun Heo | 0fe1e00 | 2009-10-29 22:34:14 +0900 | [diff] [blame] | 1651 | svm->asid_generation = sd->asid_generation; |
| 1652 | svm->vmcb->control.asid = sd->next_asid++; |
Joerg Roedel | d48086d | 2010-12-03 11:45:51 +0100 | [diff] [blame] | 1653 | |
| 1654 | mark_dirty(svm->vmcb, VMCB_ASID); |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1655 | } |
| 1656 | |
Paolo Bonzini | d67668e | 2020-05-06 06:40:04 -0400 | [diff] [blame] | 1657 | static void svm_set_dr6(struct vcpu_svm *svm, unsigned long value) |
Jan Kiszka | 73aaf249e | 2014-01-04 18:47:16 +0100 | [diff] [blame] | 1658 | { |
Paolo Bonzini | d67668e | 2020-05-06 06:40:04 -0400 | [diff] [blame] | 1659 | struct vmcb *vmcb = svm->vmcb; |
Jan Kiszka | 73aaf249e | 2014-01-04 18:47:16 +0100 | [diff] [blame] | 1660 | |
Paolo Bonzini | d67668e | 2020-05-06 06:40:04 -0400 | [diff] [blame] | 1661 | if (unlikely(value != vmcb->save.dr6)) { |
| 1662 | vmcb->save.dr6 = value; |
| 1663 | mark_dirty(vmcb, VMCB_DR); |
| 1664 | } |
Jan Kiszka | 73aaf249e | 2014-01-04 18:47:16 +0100 | [diff] [blame] | 1665 | } |
| 1666 | |
Paolo Bonzini | facb013 | 2014-02-21 10:32:27 +0100 | [diff] [blame] | 1667 | static void svm_sync_dirty_debug_regs(struct kvm_vcpu *vcpu) |
| 1668 | { |
| 1669 | struct vcpu_svm *svm = to_svm(vcpu); |
| 1670 | |
| 1671 | get_debugreg(vcpu->arch.db[0], 0); |
| 1672 | get_debugreg(vcpu->arch.db[1], 1); |
| 1673 | get_debugreg(vcpu->arch.db[2], 2); |
| 1674 | get_debugreg(vcpu->arch.db[3], 3); |
Paolo Bonzini | d67668e | 2020-05-06 06:40:04 -0400 | [diff] [blame] | 1675 | /* |
| 1676 | * We cannot reset svm->vmcb->save.dr6 to DR6_FIXED_1|DR6_RTM here, |
| 1677 | * because db_interception might need it. We can do it before vmentry. |
| 1678 | */ |
Paolo Bonzini | 5679b80 | 2020-05-04 11:28:25 -0400 | [diff] [blame] | 1679 | vcpu->arch.dr6 = svm->vmcb->save.dr6; |
Paolo Bonzini | facb013 | 2014-02-21 10:32:27 +0100 | [diff] [blame] | 1680 | vcpu->arch.dr7 = svm->vmcb->save.dr7; |
Paolo Bonzini | facb013 | 2014-02-21 10:32:27 +0100 | [diff] [blame] | 1681 | vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT; |
| 1682 | set_dr_intercepts(svm); |
| 1683 | } |
| 1684 | |
Gleb Natapov | 020df07 | 2010-04-13 10:05:23 +0300 | [diff] [blame] | 1685 | static void svm_set_dr7(struct kvm_vcpu *vcpu, unsigned long value) |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1686 | { |
Jan Kiszka | 42dbaa5 | 2008-12-15 13:52:10 +0100 | [diff] [blame] | 1687 | struct vcpu_svm *svm = to_svm(vcpu); |
Jan Kiszka | 42dbaa5 | 2008-12-15 13:52:10 +0100 | [diff] [blame] | 1688 | |
Gleb Natapov | 020df07 | 2010-04-13 10:05:23 +0300 | [diff] [blame] | 1689 | svm->vmcb->save.dr7 = value; |
Joerg Roedel | 72214b9 | 2010-12-03 11:45:55 +0100 | [diff] [blame] | 1690 | mark_dirty(svm->vmcb, VMCB_DR); |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1691 | } |
| 1692 | |
Avi Kivity | 851ba69 | 2009-08-24 11:10:17 +0300 | [diff] [blame] | 1693 | static int pf_interception(struct vcpu_svm *svm) |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1694 | { |
Brijesh Singh | 0ede79e | 2017-12-04 10:57:39 -0600 | [diff] [blame] | 1695 | u64 fault_address = __sme_clr(svm->vmcb->control.exit_info_2); |
Wanpeng Li | 1261bfa | 2017-07-13 18:30:40 -0700 | [diff] [blame] | 1696 | u64 error_code = svm->vmcb->control.exit_info_1; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1697 | |
Wanpeng Li | 1261bfa | 2017-07-13 18:30:40 -0700 | [diff] [blame] | 1698 | return kvm_handle_page_fault(&svm->vcpu, error_code, fault_address, |
Brijesh Singh | 00b10fe | 2017-12-04 10:57:40 -0600 | [diff] [blame] | 1699 | static_cpu_has(X86_FEATURE_DECODEASSISTS) ? |
| 1700 | svm->vmcb->control.insn_bytes : NULL, |
Paolo Bonzini | d000653 | 2017-08-11 18:36:43 +0200 | [diff] [blame] | 1701 | svm->vmcb->control.insn_len); |
| 1702 | } |
| 1703 | |
| 1704 | static int npf_interception(struct vcpu_svm *svm) |
| 1705 | { |
Brijesh Singh | 0ede79e | 2017-12-04 10:57:39 -0600 | [diff] [blame] | 1706 | u64 fault_address = __sme_clr(svm->vmcb->control.exit_info_2); |
Paolo Bonzini | d000653 | 2017-08-11 18:36:43 +0200 | [diff] [blame] | 1707 | u64 error_code = svm->vmcb->control.exit_info_1; |
| 1708 | |
| 1709 | trace_kvm_page_fault(fault_address, error_code); |
| 1710 | return kvm_mmu_page_fault(&svm->vcpu, fault_address, error_code, |
Brijesh Singh | 00b10fe | 2017-12-04 10:57:40 -0600 | [diff] [blame] | 1711 | static_cpu_has(X86_FEATURE_DECODEASSISTS) ? |
| 1712 | svm->vmcb->control.insn_bytes : NULL, |
Paolo Bonzini | d000653 | 2017-08-11 18:36:43 +0200 | [diff] [blame] | 1713 | svm->vmcb->control.insn_len); |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1714 | } |
| 1715 | |
Avi Kivity | 851ba69 | 2009-08-24 11:10:17 +0300 | [diff] [blame] | 1716 | static int db_interception(struct vcpu_svm *svm) |
Jan Kiszka | d0bfb94 | 2008-12-15 13:52:10 +0100 | [diff] [blame] | 1717 | { |
Avi Kivity | 851ba69 | 2009-08-24 11:10:17 +0300 | [diff] [blame] | 1718 | struct kvm_run *kvm_run = svm->vcpu.run; |
Vitaly Kuznetsov | 99c2217 | 2019-04-03 16:06:42 +0200 | [diff] [blame] | 1719 | struct kvm_vcpu *vcpu = &svm->vcpu; |
Avi Kivity | 851ba69 | 2009-08-24 11:10:17 +0300 | [diff] [blame] | 1720 | |
Jan Kiszka | d0bfb94 | 2008-12-15 13:52:10 +0100 | [diff] [blame] | 1721 | if (!(svm->vcpu.guest_debug & |
Gleb Natapov | 44c1143 | 2009-05-11 13:35:52 +0300 | [diff] [blame] | 1722 | (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) && |
Jan Kiszka | 6be7d30 | 2009-10-18 13:24:54 +0200 | [diff] [blame] | 1723 | !svm->nmi_singlestep) { |
Paolo Bonzini | d67668e | 2020-05-06 06:40:04 -0400 | [diff] [blame] | 1724 | u32 payload = (svm->vmcb->save.dr6 ^ DR6_RTM) & ~DR6_FIXED_1; |
| 1725 | kvm_queue_exception_p(&svm->vcpu, DB_VECTOR, payload); |
Jan Kiszka | d0bfb94 | 2008-12-15 13:52:10 +0100 | [diff] [blame] | 1726 | return 1; |
| 1727 | } |
Gleb Natapov | 44c1143 | 2009-05-11 13:35:52 +0300 | [diff] [blame] | 1728 | |
Jan Kiszka | 6be7d30 | 2009-10-18 13:24:54 +0200 | [diff] [blame] | 1729 | if (svm->nmi_singlestep) { |
Ladi Prosek | 4aebd0e | 2017-06-21 09:06:57 +0200 | [diff] [blame] | 1730 | disable_nmi_singlestep(svm); |
Vitaly Kuznetsov | 99c2217 | 2019-04-03 16:06:42 +0200 | [diff] [blame] | 1731 | /* Make sure we check for pending NMIs upon entry */ |
| 1732 | kvm_make_request(KVM_REQ_EVENT, vcpu); |
Gleb Natapov | 44c1143 | 2009-05-11 13:35:52 +0300 | [diff] [blame] | 1733 | } |
| 1734 | |
| 1735 | if (svm->vcpu.guest_debug & |
Joerg Roedel | e023171 | 2010-02-24 18:59:10 +0100 | [diff] [blame] | 1736 | (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) { |
Gleb Natapov | 44c1143 | 2009-05-11 13:35:52 +0300 | [diff] [blame] | 1737 | kvm_run->exit_reason = KVM_EXIT_DEBUG; |
Paolo Bonzini | dee919d | 2020-05-04 09:34:10 -0400 | [diff] [blame] | 1738 | kvm_run->debug.arch.dr6 = svm->vmcb->save.dr6; |
| 1739 | kvm_run->debug.arch.dr7 = svm->vmcb->save.dr7; |
Gleb Natapov | 44c1143 | 2009-05-11 13:35:52 +0300 | [diff] [blame] | 1740 | kvm_run->debug.arch.pc = |
| 1741 | svm->vmcb->save.cs.base + svm->vmcb->save.rip; |
| 1742 | kvm_run->debug.arch.exception = DB_VECTOR; |
| 1743 | return 0; |
| 1744 | } |
| 1745 | |
| 1746 | return 1; |
Jan Kiszka | d0bfb94 | 2008-12-15 13:52:10 +0100 | [diff] [blame] | 1747 | } |
| 1748 | |
Avi Kivity | 851ba69 | 2009-08-24 11:10:17 +0300 | [diff] [blame] | 1749 | static int bp_interception(struct vcpu_svm *svm) |
Jan Kiszka | d0bfb94 | 2008-12-15 13:52:10 +0100 | [diff] [blame] | 1750 | { |
Avi Kivity | 851ba69 | 2009-08-24 11:10:17 +0300 | [diff] [blame] | 1751 | struct kvm_run *kvm_run = svm->vcpu.run; |
| 1752 | |
Jan Kiszka | d0bfb94 | 2008-12-15 13:52:10 +0100 | [diff] [blame] | 1753 | kvm_run->exit_reason = KVM_EXIT_DEBUG; |
| 1754 | kvm_run->debug.arch.pc = svm->vmcb->save.cs.base + svm->vmcb->save.rip; |
| 1755 | kvm_run->debug.arch.exception = BP_VECTOR; |
| 1756 | return 0; |
| 1757 | } |
| 1758 | |
Avi Kivity | 851ba69 | 2009-08-24 11:10:17 +0300 | [diff] [blame] | 1759 | static int ud_interception(struct vcpu_svm *svm) |
Anthony Liguori | 7aa81cc | 2007-09-17 14:57:50 -0500 | [diff] [blame] | 1760 | { |
Wanpeng Li | 082d06e | 2018-04-03 16:28:48 -0700 | [diff] [blame] | 1761 | return handle_ud(&svm->vcpu); |
Anthony Liguori | 7aa81cc | 2007-09-17 14:57:50 -0500 | [diff] [blame] | 1762 | } |
| 1763 | |
Eric Northup | 54a2055 | 2015-11-03 18:03:53 +0100 | [diff] [blame] | 1764 | static int ac_interception(struct vcpu_svm *svm) |
| 1765 | { |
| 1766 | kvm_queue_exception_e(&svm->vcpu, AC_VECTOR, 0); |
| 1767 | return 1; |
| 1768 | } |
| 1769 | |
Liran Alon | 9718420 | 2018-03-12 13:12:52 +0200 | [diff] [blame] | 1770 | static int gp_interception(struct vcpu_svm *svm) |
| 1771 | { |
| 1772 | struct kvm_vcpu *vcpu = &svm->vcpu; |
| 1773 | u32 error_code = svm->vmcb->control.exit_info_1; |
Liran Alon | 9718420 | 2018-03-12 13:12:52 +0200 | [diff] [blame] | 1774 | |
| 1775 | WARN_ON_ONCE(!enable_vmware_backdoor); |
| 1776 | |
Sean Christopherson | a6c6ed1 | 2019-08-27 14:40:30 -0700 | [diff] [blame] | 1777 | /* |
| 1778 | * VMware backdoor emulation on #GP interception only handles IN{S}, |
| 1779 | * OUT{S}, and RDPMC, none of which generate a non-zero error code. |
| 1780 | */ |
| 1781 | if (error_code) { |
| 1782 | kvm_queue_exception_e(vcpu, GP_VECTOR, error_code); |
| 1783 | return 1; |
| 1784 | } |
Sean Christopherson | 60fc3d0 | 2019-08-27 14:40:38 -0700 | [diff] [blame] | 1785 | return kvm_emulate_instruction(vcpu, EMULTYPE_VMWARE_GP); |
Liran Alon | 9718420 | 2018-03-12 13:12:52 +0200 | [diff] [blame] | 1786 | } |
| 1787 | |
Joerg Roedel | 67ec660 | 2010-05-17 14:43:35 +0200 | [diff] [blame] | 1788 | static bool is_erratum_383(void) |
| 1789 | { |
| 1790 | int err, i; |
| 1791 | u64 value; |
| 1792 | |
| 1793 | if (!erratum_383_found) |
| 1794 | return false; |
| 1795 | |
| 1796 | value = native_read_msr_safe(MSR_IA32_MC0_STATUS, &err); |
| 1797 | if (err) |
| 1798 | return false; |
| 1799 | |
| 1800 | /* Bit 62 may or may not be set for this mce */ |
| 1801 | value &= ~(1ULL << 62); |
| 1802 | |
| 1803 | if (value != 0xb600000000010015ULL) |
| 1804 | return false; |
| 1805 | |
| 1806 | /* Clear MCi_STATUS registers */ |
| 1807 | for (i = 0; i < 6; ++i) |
| 1808 | native_write_msr_safe(MSR_IA32_MCx_STATUS(i), 0, 0); |
| 1809 | |
| 1810 | value = native_read_msr_safe(MSR_IA32_MCG_STATUS, &err); |
| 1811 | if (!err) { |
| 1812 | u32 low, high; |
| 1813 | |
| 1814 | value &= ~(1ULL << 2); |
| 1815 | low = lower_32_bits(value); |
| 1816 | high = upper_32_bits(value); |
| 1817 | |
| 1818 | native_write_msr_safe(MSR_IA32_MCG_STATUS, low, high); |
| 1819 | } |
| 1820 | |
| 1821 | /* Flush tlb to evict multi-match entries */ |
| 1822 | __flush_tlb_all(); |
| 1823 | |
| 1824 | return true; |
| 1825 | } |
| 1826 | |
Uros Bizjak | 1c164cb | 2020-04-11 17:36:27 +0200 | [diff] [blame] | 1827 | /* |
| 1828 | * Trigger machine check on the host. We assume all the MSRs are already set up |
| 1829 | * by the CPU and that we still run on the same CPU as the MCE occurred on. |
| 1830 | * We pass a fake environment to the machine check handler because we want |
| 1831 | * the guest to be always treated like user space, no matter what context |
| 1832 | * it used internally. |
| 1833 | */ |
| 1834 | static void kvm_machine_check(void) |
| 1835 | { |
| 1836 | #if defined(CONFIG_X86_MCE) |
| 1837 | struct pt_regs regs = { |
| 1838 | .cs = 3, /* Fake ring 3 no matter what the guest ran on */ |
| 1839 | .flags = X86_EFLAGS_IF, |
| 1840 | }; |
| 1841 | |
| 1842 | do_machine_check(®s, 0); |
| 1843 | #endif |
| 1844 | } |
| 1845 | |
Joerg Roedel | fe5913e | 2010-05-17 14:43:34 +0200 | [diff] [blame] | 1846 | static void svm_handle_mce(struct vcpu_svm *svm) |
Joerg Roedel | 53371b5 | 2008-04-09 14:15:30 +0200 | [diff] [blame] | 1847 | { |
Joerg Roedel | 67ec660 | 2010-05-17 14:43:35 +0200 | [diff] [blame] | 1848 | if (is_erratum_383()) { |
| 1849 | /* |
| 1850 | * Erratum 383 triggered. Guest state is corrupt so kill the |
| 1851 | * guest. |
| 1852 | */ |
| 1853 | pr_err("KVM: Guest triggered AMD Erratum 383\n"); |
| 1854 | |
Avi Kivity | a8eeb04 | 2010-05-10 12:34:53 +0300 | [diff] [blame] | 1855 | kvm_make_request(KVM_REQ_TRIPLE_FAULT, &svm->vcpu); |
Joerg Roedel | 67ec660 | 2010-05-17 14:43:35 +0200 | [diff] [blame] | 1856 | |
| 1857 | return; |
| 1858 | } |
| 1859 | |
Joerg Roedel | 53371b5 | 2008-04-09 14:15:30 +0200 | [diff] [blame] | 1860 | /* |
| 1861 | * On an #MC intercept the MCE handler is not called automatically in |
| 1862 | * the host. So do it by hand here. |
| 1863 | */ |
Uros Bizjak | 1c164cb | 2020-04-11 17:36:27 +0200 | [diff] [blame] | 1864 | kvm_machine_check(); |
Joerg Roedel | fe5913e | 2010-05-17 14:43:34 +0200 | [diff] [blame] | 1865 | } |
| 1866 | |
| 1867 | static int mc_interception(struct vcpu_svm *svm) |
| 1868 | { |
Joerg Roedel | 53371b5 | 2008-04-09 14:15:30 +0200 | [diff] [blame] | 1869 | return 1; |
| 1870 | } |
| 1871 | |
Avi Kivity | 851ba69 | 2009-08-24 11:10:17 +0300 | [diff] [blame] | 1872 | static int shutdown_interception(struct vcpu_svm *svm) |
Joerg Roedel | 46fe4dd | 2007-01-26 00:56:42 -0800 | [diff] [blame] | 1873 | { |
Avi Kivity | 851ba69 | 2009-08-24 11:10:17 +0300 | [diff] [blame] | 1874 | struct kvm_run *kvm_run = svm->vcpu.run; |
| 1875 | |
Joerg Roedel | 46fe4dd | 2007-01-26 00:56:42 -0800 | [diff] [blame] | 1876 | /* |
| 1877 | * VMCB is undefined after a SHUTDOWN intercept |
| 1878 | * so reinitialize it. |
| 1879 | */ |
Gregory Haskins | a2fa3e9 | 2007-07-27 08:13:10 -0400 | [diff] [blame] | 1880 | clear_page(svm->vmcb); |
Paolo Bonzini | 5690891 | 2015-10-19 11:30:19 +0200 | [diff] [blame] | 1881 | init_vmcb(svm); |
Joerg Roedel | 46fe4dd | 2007-01-26 00:56:42 -0800 | [diff] [blame] | 1882 | |
| 1883 | kvm_run->exit_reason = KVM_EXIT_SHUTDOWN; |
| 1884 | return 0; |
| 1885 | } |
| 1886 | |
Avi Kivity | 851ba69 | 2009-08-24 11:10:17 +0300 | [diff] [blame] | 1887 | static int io_interception(struct vcpu_svm *svm) |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1888 | { |
Gleb Natapov | cf8f70b | 2010-03-18 15:20:23 +0200 | [diff] [blame] | 1889 | struct kvm_vcpu *vcpu = &svm->vcpu; |
Mike Day | d77c26f | 2007-10-08 09:02:08 -0400 | [diff] [blame] | 1890 | u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */ |
Sean Christopherson | dca7f12 | 2018-03-08 08:57:27 -0800 | [diff] [blame] | 1891 | int size, in, string; |
Avi Kivity | 039576c | 2007-03-20 12:46:50 +0200 | [diff] [blame] | 1892 | unsigned port; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1893 | |
Rusty Russell | e756fc6 | 2007-07-30 20:07:08 +1000 | [diff] [blame] | 1894 | ++svm->vcpu.stat.io_exits; |
Laurent Vivier | e70669a | 2007-08-05 10:36:40 +0300 | [diff] [blame] | 1895 | string = (io_info & SVM_IOIO_STR_MASK) != 0; |
Avi Kivity | 039576c | 2007-03-20 12:46:50 +0200 | [diff] [blame] | 1896 | in = (io_info & SVM_IOIO_TYPE_MASK) != 0; |
Tom Lendacky | 8370c3d | 2016-11-23 12:01:50 -0500 | [diff] [blame] | 1897 | if (string) |
Sean Christopherson | 60fc3d0 | 2019-08-27 14:40:38 -0700 | [diff] [blame] | 1898 | return kvm_emulate_instruction(vcpu, 0); |
Gleb Natapov | cf8f70b | 2010-03-18 15:20:23 +0200 | [diff] [blame] | 1899 | |
Avi Kivity | 039576c | 2007-03-20 12:46:50 +0200 | [diff] [blame] | 1900 | port = io_info >> 16; |
| 1901 | size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT; |
Gleb Natapov | cf8f70b | 2010-03-18 15:20:23 +0200 | [diff] [blame] | 1902 | svm->next_rip = svm->vmcb->control.exit_info_2; |
Gleb Natapov | cf8f70b | 2010-03-18 15:20:23 +0200 | [diff] [blame] | 1903 | |
Sean Christopherson | dca7f12 | 2018-03-08 08:57:27 -0800 | [diff] [blame] | 1904 | return kvm_fast_pio(&svm->vcpu, size, port, in); |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1905 | } |
| 1906 | |
Avi Kivity | 851ba69 | 2009-08-24 11:10:17 +0300 | [diff] [blame] | 1907 | static int nmi_interception(struct vcpu_svm *svm) |
Joerg Roedel | c47f098 | 2008-04-30 17:56:00 +0200 | [diff] [blame] | 1908 | { |
| 1909 | return 1; |
| 1910 | } |
| 1911 | |
Avi Kivity | 851ba69 | 2009-08-24 11:10:17 +0300 | [diff] [blame] | 1912 | static int intr_interception(struct vcpu_svm *svm) |
Joerg Roedel | a069805 | 2008-04-30 17:56:01 +0200 | [diff] [blame] | 1913 | { |
| 1914 | ++svm->vcpu.stat.irq_exits; |
| 1915 | return 1; |
| 1916 | } |
| 1917 | |
Avi Kivity | 851ba69 | 2009-08-24 11:10:17 +0300 | [diff] [blame] | 1918 | static int nop_on_interception(struct vcpu_svm *svm) |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1919 | { |
| 1920 | return 1; |
| 1921 | } |
| 1922 | |
Avi Kivity | 851ba69 | 2009-08-24 11:10:17 +0300 | [diff] [blame] | 1923 | static int halt_interception(struct vcpu_svm *svm) |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1924 | { |
Rusty Russell | e756fc6 | 2007-07-30 20:07:08 +1000 | [diff] [blame] | 1925 | return kvm_emulate_halt(&svm->vcpu); |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 1926 | } |
| 1927 | |
Avi Kivity | 851ba69 | 2009-08-24 11:10:17 +0300 | [diff] [blame] | 1928 | static int vmmcall_interception(struct vcpu_svm *svm) |
Avi Kivity | 02e235b | 2007-02-19 14:37:47 +0200 | [diff] [blame] | 1929 | { |
Andrey Smetanin | 0d9c055 | 2016-02-11 16:44:59 +0300 | [diff] [blame] | 1930 | return kvm_emulate_hypercall(&svm->vcpu); |
Avi Kivity | 02e235b | 2007-02-19 14:37:47 +0200 | [diff] [blame] | 1931 | } |
| 1932 | |
Avi Kivity | 851ba69 | 2009-08-24 11:10:17 +0300 | [diff] [blame] | 1933 | static int vmload_interception(struct vcpu_svm *svm) |
Alexander Graf | 5542675 | 2008-11-25 20:17:06 +0100 | [diff] [blame] | 1934 | { |
Joerg Roedel | 9966bf6 | 2009-08-07 11:49:40 +0200 | [diff] [blame] | 1935 | struct vmcb *nested_vmcb; |
KarimAllah Ahmed | 8c5fbf1 | 2019-01-31 21:24:40 +0100 | [diff] [blame] | 1936 | struct kvm_host_map map; |
Ladi Prosek | b742c1e | 2017-06-22 09:05:26 +0200 | [diff] [blame] | 1937 | int ret; |
Joerg Roedel | 9966bf6 | 2009-08-07 11:49:40 +0200 | [diff] [blame] | 1938 | |
Alexander Graf | 5542675 | 2008-11-25 20:17:06 +0100 | [diff] [blame] | 1939 | if (nested_svm_check_permissions(svm)) |
| 1940 | return 1; |
| 1941 | |
KarimAllah Ahmed | 8c5fbf1 | 2019-01-31 21:24:40 +0100 | [diff] [blame] | 1942 | ret = kvm_vcpu_map(&svm->vcpu, gpa_to_gfn(svm->vmcb->save.rax), &map); |
| 1943 | if (ret) { |
| 1944 | if (ret == -EINVAL) |
| 1945 | kvm_inject_gp(&svm->vcpu, 0); |
Joerg Roedel | 9966bf6 | 2009-08-07 11:49:40 +0200 | [diff] [blame] | 1946 | return 1; |
KarimAllah Ahmed | 8c5fbf1 | 2019-01-31 21:24:40 +0100 | [diff] [blame] | 1947 | } |
| 1948 | |
| 1949 | nested_vmcb = map.hva; |
Joerg Roedel | 9966bf6 | 2009-08-07 11:49:40 +0200 | [diff] [blame] | 1950 | |
Ladi Prosek | b742c1e | 2017-06-22 09:05:26 +0200 | [diff] [blame] | 1951 | ret = kvm_skip_emulated_instruction(&svm->vcpu); |
Joerg Roedel | e3e9ed3 | 2011-04-06 12:30:03 +0200 | [diff] [blame] | 1952 | |
Joerg Roedel | 9966bf6 | 2009-08-07 11:49:40 +0200 | [diff] [blame] | 1953 | nested_svm_vmloadsave(nested_vmcb, svm->vmcb); |
KarimAllah Ahmed | 8c5fbf1 | 2019-01-31 21:24:40 +0100 | [diff] [blame] | 1954 | kvm_vcpu_unmap(&svm->vcpu, &map, true); |
Alexander Graf | 5542675 | 2008-11-25 20:17:06 +0100 | [diff] [blame] | 1955 | |
Ladi Prosek | b742c1e | 2017-06-22 09:05:26 +0200 | [diff] [blame] | 1956 | return ret; |
Alexander Graf | 5542675 | 2008-11-25 20:17:06 +0100 | [diff] [blame] | 1957 | } |
| 1958 | |
Avi Kivity | 851ba69 | 2009-08-24 11:10:17 +0300 | [diff] [blame] | 1959 | static int vmsave_interception(struct vcpu_svm *svm) |
Alexander Graf | 5542675 | 2008-11-25 20:17:06 +0100 | [diff] [blame] | 1960 | { |
Joerg Roedel | 9966bf6 | 2009-08-07 11:49:40 +0200 | [diff] [blame] | 1961 | struct vmcb *nested_vmcb; |
KarimAllah Ahmed | 8c5fbf1 | 2019-01-31 21:24:40 +0100 | [diff] [blame] | 1962 | struct kvm_host_map map; |
Ladi Prosek | b742c1e | 2017-06-22 09:05:26 +0200 | [diff] [blame] | 1963 | int ret; |
Joerg Roedel | 9966bf6 | 2009-08-07 11:49:40 +0200 | [diff] [blame] | 1964 | |
Alexander Graf | 5542675 | 2008-11-25 20:17:06 +0100 | [diff] [blame] | 1965 | if (nested_svm_check_permissions(svm)) |
| 1966 | return 1; |
| 1967 | |
KarimAllah Ahmed | 8c5fbf1 | 2019-01-31 21:24:40 +0100 | [diff] [blame] | 1968 | ret = kvm_vcpu_map(&svm->vcpu, gpa_to_gfn(svm->vmcb->save.rax), &map); |
| 1969 | if (ret) { |
| 1970 | if (ret == -EINVAL) |
| 1971 | kvm_inject_gp(&svm->vcpu, 0); |
Joerg Roedel | 9966bf6 | 2009-08-07 11:49:40 +0200 | [diff] [blame] | 1972 | return 1; |
KarimAllah Ahmed | 8c5fbf1 | 2019-01-31 21:24:40 +0100 | [diff] [blame] | 1973 | } |
| 1974 | |
| 1975 | nested_vmcb = map.hva; |
Joerg Roedel | 9966bf6 | 2009-08-07 11:49:40 +0200 | [diff] [blame] | 1976 | |
Ladi Prosek | b742c1e | 2017-06-22 09:05:26 +0200 | [diff] [blame] | 1977 | ret = kvm_skip_emulated_instruction(&svm->vcpu); |
Joerg Roedel | e3e9ed3 | 2011-04-06 12:30:03 +0200 | [diff] [blame] | 1978 | |
Joerg Roedel | 9966bf6 | 2009-08-07 11:49:40 +0200 | [diff] [blame] | 1979 | nested_svm_vmloadsave(svm->vmcb, nested_vmcb); |
KarimAllah Ahmed | 8c5fbf1 | 2019-01-31 21:24:40 +0100 | [diff] [blame] | 1980 | kvm_vcpu_unmap(&svm->vcpu, &map, true); |
Alexander Graf | 5542675 | 2008-11-25 20:17:06 +0100 | [diff] [blame] | 1981 | |
Ladi Prosek | b742c1e | 2017-06-22 09:05:26 +0200 | [diff] [blame] | 1982 | return ret; |
Alexander Graf | 5542675 | 2008-11-25 20:17:06 +0100 | [diff] [blame] | 1983 | } |
| 1984 | |
Avi Kivity | 851ba69 | 2009-08-24 11:10:17 +0300 | [diff] [blame] | 1985 | static int vmrun_interception(struct vcpu_svm *svm) |
Alexander Graf | 3d6368e | 2008-11-25 20:17:07 +0100 | [diff] [blame] | 1986 | { |
Alexander Graf | 3d6368e | 2008-11-25 20:17:07 +0100 | [diff] [blame] | 1987 | if (nested_svm_check_permissions(svm)) |
| 1988 | return 1; |
| 1989 | |
Vitaly Kuznetsov | e7134c1 | 2019-08-13 15:53:34 +0200 | [diff] [blame] | 1990 | return nested_svm_vmrun(svm); |
Alexander Graf | 3d6368e | 2008-11-25 20:17:07 +0100 | [diff] [blame] | 1991 | } |
| 1992 | |
Paolo Bonzini | ffdf7f9 | 2020-05-22 12:18:27 -0400 | [diff] [blame] | 1993 | void svm_set_gif(struct vcpu_svm *svm, bool value) |
| 1994 | { |
| 1995 | if (value) { |
| 1996 | /* |
| 1997 | * If VGIF is enabled, the STGI intercept is only added to |
| 1998 | * detect the opening of the SMI/NMI window; remove it now. |
| 1999 | * Likewise, clear the VINTR intercept, we will set it |
| 2000 | * again while processing KVM_REQ_EVENT if needed. |
| 2001 | */ |
| 2002 | if (vgif_enabled(svm)) |
| 2003 | clr_intercept(svm, INTERCEPT_STGI); |
Paolo Bonzini | fb7333d | 2020-06-08 07:11:47 -0400 | [diff] [blame^] | 2004 | if (is_intercept(svm, INTERCEPT_VINTR)) |
Paolo Bonzini | ffdf7f9 | 2020-05-22 12:18:27 -0400 | [diff] [blame] | 2005 | svm_clear_vintr(svm); |
| 2006 | |
| 2007 | enable_gif(svm); |
| 2008 | if (svm->vcpu.arch.smi_pending || |
| 2009 | svm->vcpu.arch.nmi_pending || |
| 2010 | kvm_cpu_has_injectable_intr(&svm->vcpu)) |
| 2011 | kvm_make_request(KVM_REQ_EVENT, &svm->vcpu); |
| 2012 | } else { |
| 2013 | disable_gif(svm); |
| 2014 | |
| 2015 | /* |
| 2016 | * After a CLGI no interrupts should come. But if vGIF is |
| 2017 | * in use, we still rely on the VINTR intercept (rather than |
| 2018 | * STGI) to detect an open interrupt window. |
| 2019 | */ |
| 2020 | if (!vgif_enabled(svm)) |
| 2021 | svm_clear_vintr(svm); |
| 2022 | } |
| 2023 | } |
| 2024 | |
Avi Kivity | 851ba69 | 2009-08-24 11:10:17 +0300 | [diff] [blame] | 2025 | static int stgi_interception(struct vcpu_svm *svm) |
Alexander Graf | 1371d90 | 2008-11-25 20:17:04 +0100 | [diff] [blame] | 2026 | { |
Ladi Prosek | b742c1e | 2017-06-22 09:05:26 +0200 | [diff] [blame] | 2027 | int ret; |
| 2028 | |
Alexander Graf | 1371d90 | 2008-11-25 20:17:04 +0100 | [diff] [blame] | 2029 | if (nested_svm_check_permissions(svm)) |
| 2030 | return 1; |
| 2031 | |
Ladi Prosek | b742c1e | 2017-06-22 09:05:26 +0200 | [diff] [blame] | 2032 | ret = kvm_skip_emulated_instruction(&svm->vcpu); |
Paolo Bonzini | ffdf7f9 | 2020-05-22 12:18:27 -0400 | [diff] [blame] | 2033 | svm_set_gif(svm, true); |
Ladi Prosek | b742c1e | 2017-06-22 09:05:26 +0200 | [diff] [blame] | 2034 | return ret; |
Alexander Graf | 1371d90 | 2008-11-25 20:17:04 +0100 | [diff] [blame] | 2035 | } |
| 2036 | |
Avi Kivity | 851ba69 | 2009-08-24 11:10:17 +0300 | [diff] [blame] | 2037 | static int clgi_interception(struct vcpu_svm *svm) |
Alexander Graf | 1371d90 | 2008-11-25 20:17:04 +0100 | [diff] [blame] | 2038 | { |
Ladi Prosek | b742c1e | 2017-06-22 09:05:26 +0200 | [diff] [blame] | 2039 | int ret; |
| 2040 | |
Alexander Graf | 1371d90 | 2008-11-25 20:17:04 +0100 | [diff] [blame] | 2041 | if (nested_svm_check_permissions(svm)) |
| 2042 | return 1; |
| 2043 | |
Ladi Prosek | b742c1e | 2017-06-22 09:05:26 +0200 | [diff] [blame] | 2044 | ret = kvm_skip_emulated_instruction(&svm->vcpu); |
Paolo Bonzini | ffdf7f9 | 2020-05-22 12:18:27 -0400 | [diff] [blame] | 2045 | svm_set_gif(svm, false); |
Ladi Prosek | b742c1e | 2017-06-22 09:05:26 +0200 | [diff] [blame] | 2046 | return ret; |
Alexander Graf | 1371d90 | 2008-11-25 20:17:04 +0100 | [diff] [blame] | 2047 | } |
| 2048 | |
Avi Kivity | 851ba69 | 2009-08-24 11:10:17 +0300 | [diff] [blame] | 2049 | static int invlpga_interception(struct vcpu_svm *svm) |
Alexander Graf | ff09238 | 2009-06-15 15:21:24 +0200 | [diff] [blame] | 2050 | { |
| 2051 | struct kvm_vcpu *vcpu = &svm->vcpu; |
Alexander Graf | ff09238 | 2009-06-15 15:21:24 +0200 | [diff] [blame] | 2052 | |
Sean Christopherson | de3cd11 | 2019-04-30 10:36:17 -0700 | [diff] [blame] | 2053 | trace_kvm_invlpga(svm->vmcb->save.rip, kvm_rcx_read(&svm->vcpu), |
| 2054 | kvm_rax_read(&svm->vcpu)); |
Joerg Roedel | ec1ff79 | 2009-10-09 16:08:31 +0200 | [diff] [blame] | 2055 | |
Alexander Graf | ff09238 | 2009-06-15 15:21:24 +0200 | [diff] [blame] | 2056 | /* Let's treat INVLPGA the same as INVLPG (can be optimized!) */ |
Sean Christopherson | de3cd11 | 2019-04-30 10:36:17 -0700 | [diff] [blame] | 2057 | kvm_mmu_invlpg(vcpu, kvm_rax_read(&svm->vcpu)); |
Alexander Graf | ff09238 | 2009-06-15 15:21:24 +0200 | [diff] [blame] | 2058 | |
Ladi Prosek | b742c1e | 2017-06-22 09:05:26 +0200 | [diff] [blame] | 2059 | return kvm_skip_emulated_instruction(&svm->vcpu); |
Alexander Graf | ff09238 | 2009-06-15 15:21:24 +0200 | [diff] [blame] | 2060 | } |
| 2061 | |
Joerg Roedel | 532a46b | 2009-10-09 16:08:32 +0200 | [diff] [blame] | 2062 | static int skinit_interception(struct vcpu_svm *svm) |
| 2063 | { |
Sean Christopherson | de3cd11 | 2019-04-30 10:36:17 -0700 | [diff] [blame] | 2064 | trace_kvm_skinit(svm->vmcb->save.rip, kvm_rax_read(&svm->vcpu)); |
Joerg Roedel | 532a46b | 2009-10-09 16:08:32 +0200 | [diff] [blame] | 2065 | |
| 2066 | kvm_queue_exception(&svm->vcpu, UD_VECTOR); |
| 2067 | return 1; |
| 2068 | } |
| 2069 | |
David Kaplan | dab429a | 2015-03-02 13:43:37 -0600 | [diff] [blame] | 2070 | static int wbinvd_interception(struct vcpu_svm *svm) |
| 2071 | { |
Kyle Huey | 6affcbe | 2016-11-29 12:40:40 -0800 | [diff] [blame] | 2072 | return kvm_emulate_wbinvd(&svm->vcpu); |
David Kaplan | dab429a | 2015-03-02 13:43:37 -0600 | [diff] [blame] | 2073 | } |
| 2074 | |
Joerg Roedel | 81dd35d | 2010-12-07 17:15:06 +0100 | [diff] [blame] | 2075 | static int xsetbv_interception(struct vcpu_svm *svm) |
| 2076 | { |
| 2077 | u64 new_bv = kvm_read_edx_eax(&svm->vcpu); |
Sean Christopherson | de3cd11 | 2019-04-30 10:36:17 -0700 | [diff] [blame] | 2078 | u32 index = kvm_rcx_read(&svm->vcpu); |
Joerg Roedel | 81dd35d | 2010-12-07 17:15:06 +0100 | [diff] [blame] | 2079 | |
| 2080 | if (kvm_set_xcr(&svm->vcpu, index, new_bv) == 0) { |
Ladi Prosek | b742c1e | 2017-06-22 09:05:26 +0200 | [diff] [blame] | 2081 | return kvm_skip_emulated_instruction(&svm->vcpu); |
Joerg Roedel | 81dd35d | 2010-12-07 17:15:06 +0100 | [diff] [blame] | 2082 | } |
| 2083 | |
| 2084 | return 1; |
| 2085 | } |
| 2086 | |
Jim Mattson | 0cb8410 | 2019-09-19 15:59:17 -0700 | [diff] [blame] | 2087 | static int rdpru_interception(struct vcpu_svm *svm) |
| 2088 | { |
| 2089 | kvm_queue_exception(&svm->vcpu, UD_VECTOR); |
| 2090 | return 1; |
| 2091 | } |
| 2092 | |
Avi Kivity | 851ba69 | 2009-08-24 11:10:17 +0300 | [diff] [blame] | 2093 | static int task_switch_interception(struct vcpu_svm *svm) |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 2094 | { |
Izik Eidus | 37817f2 | 2008-03-24 23:14:53 +0200 | [diff] [blame] | 2095 | u16 tss_selector; |
Gleb Natapov | 64a7ec0 | 2009-03-30 16:03:29 +0300 | [diff] [blame] | 2096 | int reason; |
| 2097 | int int_type = svm->vmcb->control.exit_int_info & |
| 2098 | SVM_EXITINTINFO_TYPE_MASK; |
Gleb Natapov | 8317c29 | 2009-04-12 13:37:02 +0300 | [diff] [blame] | 2099 | int int_vec = svm->vmcb->control.exit_int_info & SVM_EVTINJ_VEC_MASK; |
Gleb Natapov | fe8e7f8 | 2009-04-23 17:03:48 +0300 | [diff] [blame] | 2100 | uint32_t type = |
| 2101 | svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_TYPE_MASK; |
| 2102 | uint32_t idt_v = |
| 2103 | svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID; |
Jan Kiszka | e269fb2 | 2010-04-14 15:51:09 +0200 | [diff] [blame] | 2104 | bool has_error_code = false; |
| 2105 | u32 error_code = 0; |
Izik Eidus | 37817f2 | 2008-03-24 23:14:53 +0200 | [diff] [blame] | 2106 | |
| 2107 | tss_selector = (u16)svm->vmcb->control.exit_info_1; |
Gleb Natapov | 64a7ec0 | 2009-03-30 16:03:29 +0300 | [diff] [blame] | 2108 | |
Izik Eidus | 37817f2 | 2008-03-24 23:14:53 +0200 | [diff] [blame] | 2109 | if (svm->vmcb->control.exit_info_2 & |
| 2110 | (1ULL << SVM_EXITINFOSHIFT_TS_REASON_IRET)) |
Gleb Natapov | 64a7ec0 | 2009-03-30 16:03:29 +0300 | [diff] [blame] | 2111 | reason = TASK_SWITCH_IRET; |
| 2112 | else if (svm->vmcb->control.exit_info_2 & |
| 2113 | (1ULL << SVM_EXITINFOSHIFT_TS_REASON_JMP)) |
| 2114 | reason = TASK_SWITCH_JMP; |
Gleb Natapov | fe8e7f8 | 2009-04-23 17:03:48 +0300 | [diff] [blame] | 2115 | else if (idt_v) |
Gleb Natapov | 64a7ec0 | 2009-03-30 16:03:29 +0300 | [diff] [blame] | 2116 | reason = TASK_SWITCH_GATE; |
| 2117 | else |
| 2118 | reason = TASK_SWITCH_CALL; |
| 2119 | |
Gleb Natapov | fe8e7f8 | 2009-04-23 17:03:48 +0300 | [diff] [blame] | 2120 | if (reason == TASK_SWITCH_GATE) { |
| 2121 | switch (type) { |
| 2122 | case SVM_EXITINTINFO_TYPE_NMI: |
| 2123 | svm->vcpu.arch.nmi_injected = false; |
| 2124 | break; |
| 2125 | case SVM_EXITINTINFO_TYPE_EXEPT: |
Jan Kiszka | e269fb2 | 2010-04-14 15:51:09 +0200 | [diff] [blame] | 2126 | if (svm->vmcb->control.exit_info_2 & |
| 2127 | (1ULL << SVM_EXITINFOSHIFT_TS_HAS_ERROR_CODE)) { |
| 2128 | has_error_code = true; |
| 2129 | error_code = |
| 2130 | (u32)svm->vmcb->control.exit_info_2; |
| 2131 | } |
Gleb Natapov | fe8e7f8 | 2009-04-23 17:03:48 +0300 | [diff] [blame] | 2132 | kvm_clear_exception_queue(&svm->vcpu); |
| 2133 | break; |
| 2134 | case SVM_EXITINTINFO_TYPE_INTR: |
| 2135 | kvm_clear_interrupt_queue(&svm->vcpu); |
| 2136 | break; |
| 2137 | default: |
| 2138 | break; |
| 2139 | } |
| 2140 | } |
Gleb Natapov | 64a7ec0 | 2009-03-30 16:03:29 +0300 | [diff] [blame] | 2141 | |
Gleb Natapov | 8317c29 | 2009-04-12 13:37:02 +0300 | [diff] [blame] | 2142 | if (reason != TASK_SWITCH_GATE || |
| 2143 | int_type == SVM_EXITINTINFO_TYPE_SOFT || |
| 2144 | (int_type == SVM_EXITINTINFO_TYPE_EXEPT && |
Vitaly Kuznetsov | f8ea7c6 | 2019-08-13 15:53:30 +0200 | [diff] [blame] | 2145 | (int_vec == OF_VECTOR || int_vec == BP_VECTOR))) { |
Sean Christopherson | 60fc3d0 | 2019-08-27 14:40:38 -0700 | [diff] [blame] | 2146 | if (!skip_emulated_instruction(&svm->vcpu)) |
Sean Christopherson | 738fece | 2019-08-27 14:40:34 -0700 | [diff] [blame] | 2147 | return 0; |
Vitaly Kuznetsov | f8ea7c6 | 2019-08-13 15:53:30 +0200 | [diff] [blame] | 2148 | } |
Gleb Natapov | 64a7ec0 | 2009-03-30 16:03:29 +0300 | [diff] [blame] | 2149 | |
Kevin Wolf | 7f3d35f | 2012-02-08 14:34:38 +0100 | [diff] [blame] | 2150 | if (int_type != SVM_EXITINTINFO_TYPE_SOFT) |
| 2151 | int_vec = -1; |
| 2152 | |
Sean Christopherson | 1051778 | 2019-08-27 14:40:35 -0700 | [diff] [blame] | 2153 | return kvm_task_switch(&svm->vcpu, tss_selector, int_vec, reason, |
Sean Christopherson | 60fc3d0 | 2019-08-27 14:40:38 -0700 | [diff] [blame] | 2154 | has_error_code, error_code); |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 2155 | } |
| 2156 | |
Avi Kivity | 851ba69 | 2009-08-24 11:10:17 +0300 | [diff] [blame] | 2157 | static int cpuid_interception(struct vcpu_svm *svm) |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 2158 | { |
Kyle Huey | 6a908b6 | 2016-11-29 12:40:37 -0800 | [diff] [blame] | 2159 | return kvm_emulate_cpuid(&svm->vcpu); |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 2160 | } |
| 2161 | |
Avi Kivity | 851ba69 | 2009-08-24 11:10:17 +0300 | [diff] [blame] | 2162 | static int iret_interception(struct vcpu_svm *svm) |
Gleb Natapov | 95ba827313 | 2009-04-21 17:45:08 +0300 | [diff] [blame] | 2163 | { |
| 2164 | ++svm->vcpu.stat.nmi_window_exits; |
Joerg Roedel | 8a05a1b8 | 2010-11-30 18:04:00 +0100 | [diff] [blame] | 2165 | clr_intercept(svm, INTERCEPT_IRET); |
Gleb Natapov | 44c1143 | 2009-05-11 13:35:52 +0300 | [diff] [blame] | 2166 | svm->vcpu.arch.hflags |= HF_IRET_MASK; |
Avi Kivity | bd3d1ec | 2011-02-03 15:29:52 +0200 | [diff] [blame] | 2167 | svm->nmi_iret_rip = kvm_rip_read(&svm->vcpu); |
Radim Krčmář | f303b4c | 2014-01-17 20:52:42 +0100 | [diff] [blame] | 2168 | kvm_make_request(KVM_REQ_EVENT, &svm->vcpu); |
Gleb Natapov | 95ba827313 | 2009-04-21 17:45:08 +0300 | [diff] [blame] | 2169 | return 1; |
| 2170 | } |
| 2171 | |
Avi Kivity | 851ba69 | 2009-08-24 11:10:17 +0300 | [diff] [blame] | 2172 | static int invlpg_interception(struct vcpu_svm *svm) |
Marcelo Tosatti | a705289 | 2008-09-23 13:18:35 -0300 | [diff] [blame] | 2173 | { |
Andre Przywara | df4f3108 | 2010-12-21 11:12:06 +0100 | [diff] [blame] | 2174 | if (!static_cpu_has(X86_FEATURE_DECODEASSISTS)) |
Sean Christopherson | 60fc3d0 | 2019-08-27 14:40:38 -0700 | [diff] [blame] | 2175 | return kvm_emulate_instruction(&svm->vcpu, 0); |
Andre Przywara | df4f3108 | 2010-12-21 11:12:06 +0100 | [diff] [blame] | 2176 | |
| 2177 | kvm_mmu_invlpg(&svm->vcpu, svm->vmcb->control.exit_info_1); |
Ladi Prosek | b742c1e | 2017-06-22 09:05:26 +0200 | [diff] [blame] | 2178 | return kvm_skip_emulated_instruction(&svm->vcpu); |
Marcelo Tosatti | a705289 | 2008-09-23 13:18:35 -0300 | [diff] [blame] | 2179 | } |
| 2180 | |
Avi Kivity | 851ba69 | 2009-08-24 11:10:17 +0300 | [diff] [blame] | 2181 | static int emulate_on_interception(struct vcpu_svm *svm) |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 2182 | { |
Sean Christopherson | 60fc3d0 | 2019-08-27 14:40:38 -0700 | [diff] [blame] | 2183 | return kvm_emulate_instruction(&svm->vcpu, 0); |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 2184 | } |
| 2185 | |
Brijesh Singh | 7607b71 | 2018-02-19 10:14:44 -0600 | [diff] [blame] | 2186 | static int rsm_interception(struct vcpu_svm *svm) |
| 2187 | { |
Sean Christopherson | 60fc3d0 | 2019-08-27 14:40:38 -0700 | [diff] [blame] | 2188 | return kvm_emulate_instruction_from_buffer(&svm->vcpu, rsm_ins_bytes, 2); |
Brijesh Singh | 7607b71 | 2018-02-19 10:14:44 -0600 | [diff] [blame] | 2189 | } |
| 2190 | |
Avi Kivity | 332b56e | 2011-11-10 14:57:24 +0200 | [diff] [blame] | 2191 | static int rdpmc_interception(struct vcpu_svm *svm) |
| 2192 | { |
| 2193 | int err; |
| 2194 | |
Paolo Bonzini | d647eb6 | 2019-06-20 14:13:33 +0200 | [diff] [blame] | 2195 | if (!nrips) |
Avi Kivity | 332b56e | 2011-11-10 14:57:24 +0200 | [diff] [blame] | 2196 | return emulate_on_interception(svm); |
| 2197 | |
| 2198 | err = kvm_rdpmc(&svm->vcpu); |
Kyle Huey | 6affcbe | 2016-11-29 12:40:40 -0800 | [diff] [blame] | 2199 | return kvm_complete_insn_gp(&svm->vcpu, err); |
Avi Kivity | 332b56e | 2011-11-10 14:57:24 +0200 | [diff] [blame] | 2200 | } |
| 2201 | |
Xiubo Li | 52eb5a6 | 2015-03-13 17:39:45 +0800 | [diff] [blame] | 2202 | static bool check_selective_cr0_intercepted(struct vcpu_svm *svm, |
| 2203 | unsigned long val) |
Joerg Roedel | 628afd2 | 2011-04-04 12:39:36 +0200 | [diff] [blame] | 2204 | { |
| 2205 | unsigned long cr0 = svm->vcpu.arch.cr0; |
| 2206 | bool ret = false; |
| 2207 | u64 intercept; |
| 2208 | |
Paolo Bonzini | e670bf6 | 2020-05-13 13:16:12 -0400 | [diff] [blame] | 2209 | intercept = svm->nested.ctl.intercept; |
Joerg Roedel | 628afd2 | 2011-04-04 12:39:36 +0200 | [diff] [blame] | 2210 | |
| 2211 | if (!is_guest_mode(&svm->vcpu) || |
| 2212 | (!(intercept & (1ULL << INTERCEPT_SELECTIVE_CR0)))) |
| 2213 | return false; |
| 2214 | |
| 2215 | cr0 &= ~SVM_CR0_SELECTIVE_MASK; |
| 2216 | val &= ~SVM_CR0_SELECTIVE_MASK; |
| 2217 | |
| 2218 | if (cr0 ^ val) { |
| 2219 | svm->vmcb->control.exit_code = SVM_EXIT_CR0_SEL_WRITE; |
| 2220 | ret = (nested_svm_exit_handled(svm) == NESTED_EXIT_DONE); |
| 2221 | } |
| 2222 | |
| 2223 | return ret; |
| 2224 | } |
| 2225 | |
Andre Przywara | 7ff76d5 | 2010-12-21 11:12:04 +0100 | [diff] [blame] | 2226 | #define CR_VALID (1ULL << 63) |
| 2227 | |
| 2228 | static int cr_interception(struct vcpu_svm *svm) |
| 2229 | { |
| 2230 | int reg, cr; |
| 2231 | unsigned long val; |
| 2232 | int err; |
| 2233 | |
| 2234 | if (!static_cpu_has(X86_FEATURE_DECODEASSISTS)) |
| 2235 | return emulate_on_interception(svm); |
| 2236 | |
| 2237 | if (unlikely((svm->vmcb->control.exit_info_1 & CR_VALID) == 0)) |
| 2238 | return emulate_on_interception(svm); |
| 2239 | |
| 2240 | reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK; |
David Kaplan | 5e57518 | 2015-03-06 14:44:35 -0600 | [diff] [blame] | 2241 | if (svm->vmcb->control.exit_code == SVM_EXIT_CR0_SEL_WRITE) |
| 2242 | cr = SVM_EXIT_WRITE_CR0 - SVM_EXIT_READ_CR0; |
| 2243 | else |
| 2244 | cr = svm->vmcb->control.exit_code - SVM_EXIT_READ_CR0; |
Andre Przywara | 7ff76d5 | 2010-12-21 11:12:04 +0100 | [diff] [blame] | 2245 | |
| 2246 | err = 0; |
| 2247 | if (cr >= 16) { /* mov to cr */ |
| 2248 | cr -= 16; |
| 2249 | val = kvm_register_read(&svm->vcpu, reg); |
| 2250 | switch (cr) { |
| 2251 | case 0: |
Joerg Roedel | 628afd2 | 2011-04-04 12:39:36 +0200 | [diff] [blame] | 2252 | if (!check_selective_cr0_intercepted(svm, val)) |
| 2253 | err = kvm_set_cr0(&svm->vcpu, val); |
Joerg Roedel | 977b2d0 | 2011-04-18 11:42:52 +0200 | [diff] [blame] | 2254 | else |
| 2255 | return 1; |
| 2256 | |
Andre Przywara | 7ff76d5 | 2010-12-21 11:12:04 +0100 | [diff] [blame] | 2257 | break; |
| 2258 | case 3: |
| 2259 | err = kvm_set_cr3(&svm->vcpu, val); |
| 2260 | break; |
| 2261 | case 4: |
| 2262 | err = kvm_set_cr4(&svm->vcpu, val); |
| 2263 | break; |
| 2264 | case 8: |
| 2265 | err = kvm_set_cr8(&svm->vcpu, val); |
| 2266 | break; |
| 2267 | default: |
| 2268 | WARN(1, "unhandled write to CR%d", cr); |
| 2269 | kvm_queue_exception(&svm->vcpu, UD_VECTOR); |
| 2270 | return 1; |
| 2271 | } |
| 2272 | } else { /* mov from cr */ |
| 2273 | switch (cr) { |
| 2274 | case 0: |
| 2275 | val = kvm_read_cr0(&svm->vcpu); |
| 2276 | break; |
| 2277 | case 2: |
| 2278 | val = svm->vcpu.arch.cr2; |
| 2279 | break; |
| 2280 | case 3: |
Avi Kivity | 9f8fe50 | 2010-12-05 17:30:00 +0200 | [diff] [blame] | 2281 | val = kvm_read_cr3(&svm->vcpu); |
Andre Przywara | 7ff76d5 | 2010-12-21 11:12:04 +0100 | [diff] [blame] | 2282 | break; |
| 2283 | case 4: |
| 2284 | val = kvm_read_cr4(&svm->vcpu); |
| 2285 | break; |
| 2286 | case 8: |
| 2287 | val = kvm_get_cr8(&svm->vcpu); |
| 2288 | break; |
| 2289 | default: |
| 2290 | WARN(1, "unhandled read from CR%d", cr); |
| 2291 | kvm_queue_exception(&svm->vcpu, UD_VECTOR); |
| 2292 | return 1; |
| 2293 | } |
| 2294 | kvm_register_write(&svm->vcpu, reg, val); |
| 2295 | } |
Kyle Huey | 6affcbe | 2016-11-29 12:40:40 -0800 | [diff] [blame] | 2296 | return kvm_complete_insn_gp(&svm->vcpu, err); |
Andre Przywara | 7ff76d5 | 2010-12-21 11:12:04 +0100 | [diff] [blame] | 2297 | } |
| 2298 | |
Andre Przywara | cae3797 | 2010-12-21 11:12:05 +0100 | [diff] [blame] | 2299 | static int dr_interception(struct vcpu_svm *svm) |
| 2300 | { |
| 2301 | int reg, dr; |
| 2302 | unsigned long val; |
Andre Przywara | cae3797 | 2010-12-21 11:12:05 +0100 | [diff] [blame] | 2303 | |
Paolo Bonzini | facb013 | 2014-02-21 10:32:27 +0100 | [diff] [blame] | 2304 | if (svm->vcpu.guest_debug == 0) { |
| 2305 | /* |
| 2306 | * No more DR vmexits; force a reload of the debug registers |
| 2307 | * and reenter on this instruction. The next vmexit will |
| 2308 | * retrieve the full state of the debug registers. |
| 2309 | */ |
| 2310 | clr_dr_intercepts(svm); |
| 2311 | svm->vcpu.arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT; |
| 2312 | return 1; |
| 2313 | } |
| 2314 | |
Andre Przywara | cae3797 | 2010-12-21 11:12:05 +0100 | [diff] [blame] | 2315 | if (!boot_cpu_has(X86_FEATURE_DECODEASSISTS)) |
| 2316 | return emulate_on_interception(svm); |
| 2317 | |
| 2318 | reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK; |
| 2319 | dr = svm->vmcb->control.exit_code - SVM_EXIT_READ_DR0; |
| 2320 | |
| 2321 | if (dr >= 16) { /* mov to DRn */ |
Nadav Amit | 16f8a6f | 2014-10-03 01:10:05 +0300 | [diff] [blame] | 2322 | if (!kvm_require_dr(&svm->vcpu, dr - 16)) |
| 2323 | return 1; |
Andre Przywara | cae3797 | 2010-12-21 11:12:05 +0100 | [diff] [blame] | 2324 | val = kvm_register_read(&svm->vcpu, reg); |
| 2325 | kvm_set_dr(&svm->vcpu, dr - 16, val); |
| 2326 | } else { |
Nadav Amit | 16f8a6f | 2014-10-03 01:10:05 +0300 | [diff] [blame] | 2327 | if (!kvm_require_dr(&svm->vcpu, dr)) |
| 2328 | return 1; |
| 2329 | kvm_get_dr(&svm->vcpu, dr, &val); |
| 2330 | kvm_register_write(&svm->vcpu, reg, val); |
Andre Przywara | cae3797 | 2010-12-21 11:12:05 +0100 | [diff] [blame] | 2331 | } |
| 2332 | |
Ladi Prosek | b742c1e | 2017-06-22 09:05:26 +0200 | [diff] [blame] | 2333 | return kvm_skip_emulated_instruction(&svm->vcpu); |
Andre Przywara | cae3797 | 2010-12-21 11:12:05 +0100 | [diff] [blame] | 2334 | } |
| 2335 | |
Avi Kivity | 851ba69 | 2009-08-24 11:10:17 +0300 | [diff] [blame] | 2336 | static int cr8_write_interception(struct vcpu_svm *svm) |
Joerg Roedel | 1d07543 | 2007-12-06 21:02:25 +0100 | [diff] [blame] | 2337 | { |
Avi Kivity | 851ba69 | 2009-08-24 11:10:17 +0300 | [diff] [blame] | 2338 | struct kvm_run *kvm_run = svm->vcpu.run; |
Andre Przywara | eea1cff | 2010-12-21 11:12:00 +0100 | [diff] [blame] | 2339 | int r; |
Avi Kivity | 851ba69 | 2009-08-24 11:10:17 +0300 | [diff] [blame] | 2340 | |
Gleb Natapov | 0a5fff19 | 2009-04-21 17:45:06 +0300 | [diff] [blame] | 2341 | u8 cr8_prev = kvm_get_cr8(&svm->vcpu); |
| 2342 | /* instruction emulation calls kvm_set_cr8() */ |
Andre Przywara | 7ff76d5 | 2010-12-21 11:12:04 +0100 | [diff] [blame] | 2343 | r = cr_interception(svm); |
Paolo Bonzini | 35754c9 | 2015-07-29 12:05:37 +0200 | [diff] [blame] | 2344 | if (lapic_in_kernel(&svm->vcpu)) |
Andre Przywara | 7ff76d5 | 2010-12-21 11:12:04 +0100 | [diff] [blame] | 2345 | return r; |
Gleb Natapov | 0a5fff19 | 2009-04-21 17:45:06 +0300 | [diff] [blame] | 2346 | if (cr8_prev <= kvm_get_cr8(&svm->vcpu)) |
Andre Przywara | 7ff76d5 | 2010-12-21 11:12:04 +0100 | [diff] [blame] | 2347 | return r; |
Joerg Roedel | 1d07543 | 2007-12-06 21:02:25 +0100 | [diff] [blame] | 2348 | kvm_run->exit_reason = KVM_EXIT_SET_TPR; |
| 2349 | return 0; |
| 2350 | } |
| 2351 | |
Tom Lendacky | 801e459 | 2018-02-21 13:39:51 -0600 | [diff] [blame] | 2352 | static int svm_get_msr_feature(struct kvm_msr_entry *msr) |
| 2353 | { |
Tom Lendacky | d1d93fa | 2018-02-24 00:18:20 +0100 | [diff] [blame] | 2354 | msr->data = 0; |
| 2355 | |
| 2356 | switch (msr->index) { |
| 2357 | case MSR_F10H_DECFG: |
| 2358 | if (boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) |
| 2359 | msr->data |= MSR_F10H_DECFG_LFENCE_SERIALIZE; |
| 2360 | break; |
| 2361 | default: |
| 2362 | return 1; |
| 2363 | } |
| 2364 | |
| 2365 | return 0; |
Tom Lendacky | 801e459 | 2018-02-21 13:39:51 -0600 | [diff] [blame] | 2366 | } |
| 2367 | |
Paolo Bonzini | 609e36d | 2015-04-08 15:30:38 +0200 | [diff] [blame] | 2368 | static int svm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 2369 | { |
Gregory Haskins | a2fa3e9 | 2007-07-27 08:13:10 -0400 | [diff] [blame] | 2370 | struct vcpu_svm *svm = to_svm(vcpu); |
| 2371 | |
Paolo Bonzini | 609e36d | 2015-04-08 15:30:38 +0200 | [diff] [blame] | 2372 | switch (msr_info->index) { |
Brian Gerst | 8c06585 | 2010-07-17 09:03:26 -0400 | [diff] [blame] | 2373 | case MSR_STAR: |
Paolo Bonzini | 609e36d | 2015-04-08 15:30:38 +0200 | [diff] [blame] | 2374 | msr_info->data = svm->vmcb->save.star; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 2375 | break; |
Avi Kivity | 0e859ca | 2006-12-22 01:05:08 -0800 | [diff] [blame] | 2376 | #ifdef CONFIG_X86_64 |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 2377 | case MSR_LSTAR: |
Paolo Bonzini | 609e36d | 2015-04-08 15:30:38 +0200 | [diff] [blame] | 2378 | msr_info->data = svm->vmcb->save.lstar; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 2379 | break; |
| 2380 | case MSR_CSTAR: |
Paolo Bonzini | 609e36d | 2015-04-08 15:30:38 +0200 | [diff] [blame] | 2381 | msr_info->data = svm->vmcb->save.cstar; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 2382 | break; |
| 2383 | case MSR_KERNEL_GS_BASE: |
Paolo Bonzini | 609e36d | 2015-04-08 15:30:38 +0200 | [diff] [blame] | 2384 | msr_info->data = svm->vmcb->save.kernel_gs_base; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 2385 | break; |
| 2386 | case MSR_SYSCALL_MASK: |
Paolo Bonzini | 609e36d | 2015-04-08 15:30:38 +0200 | [diff] [blame] | 2387 | msr_info->data = svm->vmcb->save.sfmask; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 2388 | break; |
| 2389 | #endif |
| 2390 | case MSR_IA32_SYSENTER_CS: |
Paolo Bonzini | 609e36d | 2015-04-08 15:30:38 +0200 | [diff] [blame] | 2391 | msr_info->data = svm->vmcb->save.sysenter_cs; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 2392 | break; |
| 2393 | case MSR_IA32_SYSENTER_EIP: |
Paolo Bonzini | 609e36d | 2015-04-08 15:30:38 +0200 | [diff] [blame] | 2394 | msr_info->data = svm->sysenter_eip; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 2395 | break; |
| 2396 | case MSR_IA32_SYSENTER_ESP: |
Paolo Bonzini | 609e36d | 2015-04-08 15:30:38 +0200 | [diff] [blame] | 2397 | msr_info->data = svm->sysenter_esp; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 2398 | break; |
Paolo Bonzini | 46896c7 | 2015-11-12 14:49:16 +0100 | [diff] [blame] | 2399 | case MSR_TSC_AUX: |
| 2400 | if (!boot_cpu_has(X86_FEATURE_RDTSCP)) |
| 2401 | return 1; |
| 2402 | msr_info->data = svm->tsc_aux; |
| 2403 | break; |
Joerg Roedel | e023171 | 2010-02-24 18:59:10 +0100 | [diff] [blame] | 2404 | /* |
| 2405 | * Nobody will change the following 5 values in the VMCB so we can |
| 2406 | * safely return them on rdmsr. They will always be 0 until LBRV is |
| 2407 | * implemented. |
| 2408 | */ |
Joerg Roedel | a2938c8 | 2008-02-13 16:30:28 +0100 | [diff] [blame] | 2409 | case MSR_IA32_DEBUGCTLMSR: |
Paolo Bonzini | 609e36d | 2015-04-08 15:30:38 +0200 | [diff] [blame] | 2410 | msr_info->data = svm->vmcb->save.dbgctl; |
Joerg Roedel | a2938c8 | 2008-02-13 16:30:28 +0100 | [diff] [blame] | 2411 | break; |
| 2412 | case MSR_IA32_LASTBRANCHFROMIP: |
Paolo Bonzini | 609e36d | 2015-04-08 15:30:38 +0200 | [diff] [blame] | 2413 | msr_info->data = svm->vmcb->save.br_from; |
Joerg Roedel | a2938c8 | 2008-02-13 16:30:28 +0100 | [diff] [blame] | 2414 | break; |
| 2415 | case MSR_IA32_LASTBRANCHTOIP: |
Paolo Bonzini | 609e36d | 2015-04-08 15:30:38 +0200 | [diff] [blame] | 2416 | msr_info->data = svm->vmcb->save.br_to; |
Joerg Roedel | a2938c8 | 2008-02-13 16:30:28 +0100 | [diff] [blame] | 2417 | break; |
| 2418 | case MSR_IA32_LASTINTFROMIP: |
Paolo Bonzini | 609e36d | 2015-04-08 15:30:38 +0200 | [diff] [blame] | 2419 | msr_info->data = svm->vmcb->save.last_excp_from; |
Joerg Roedel | a2938c8 | 2008-02-13 16:30:28 +0100 | [diff] [blame] | 2420 | break; |
| 2421 | case MSR_IA32_LASTINTTOIP: |
Paolo Bonzini | 609e36d | 2015-04-08 15:30:38 +0200 | [diff] [blame] | 2422 | msr_info->data = svm->vmcb->save.last_excp_to; |
Joerg Roedel | a2938c8 | 2008-02-13 16:30:28 +0100 | [diff] [blame] | 2423 | break; |
Alexander Graf | b286d5d | 2008-11-25 20:17:05 +0100 | [diff] [blame] | 2424 | case MSR_VM_HSAVE_PA: |
Paolo Bonzini | 609e36d | 2015-04-08 15:30:38 +0200 | [diff] [blame] | 2425 | msr_info->data = svm->nested.hsave_msr; |
Alexander Graf | b286d5d | 2008-11-25 20:17:05 +0100 | [diff] [blame] | 2426 | break; |
Joerg Roedel | eb6f302 | 2008-11-25 20:17:09 +0100 | [diff] [blame] | 2427 | case MSR_VM_CR: |
Paolo Bonzini | 609e36d | 2015-04-08 15:30:38 +0200 | [diff] [blame] | 2428 | msr_info->data = svm->nested.vm_cr_msr; |
Joerg Roedel | eb6f302 | 2008-11-25 20:17:09 +0100 | [diff] [blame] | 2429 | break; |
KarimAllah Ahmed | b2ac58f | 2018-02-03 15:56:23 +0100 | [diff] [blame] | 2430 | case MSR_IA32_SPEC_CTRL: |
| 2431 | if (!msr_info->host_initiated && |
Paolo Bonzini | df7e881 | 2020-02-05 16:10:52 +0100 | [diff] [blame] | 2432 | !guest_cpuid_has(vcpu, X86_FEATURE_SPEC_CTRL) && |
| 2433 | !guest_cpuid_has(vcpu, X86_FEATURE_AMD_STIBP) && |
Konrad Rzeszutek Wilk | 6ac2f49 | 2018-06-01 10:59:20 -0400 | [diff] [blame] | 2434 | !guest_cpuid_has(vcpu, X86_FEATURE_AMD_IBRS) && |
| 2435 | !guest_cpuid_has(vcpu, X86_FEATURE_AMD_SSBD)) |
KarimAllah Ahmed | b2ac58f | 2018-02-03 15:56:23 +0100 | [diff] [blame] | 2436 | return 1; |
| 2437 | |
| 2438 | msr_info->data = svm->spec_ctrl; |
| 2439 | break; |
Tom Lendacky | bc226f0 | 2018-05-10 22:06:39 +0200 | [diff] [blame] | 2440 | case MSR_AMD64_VIRT_SPEC_CTRL: |
| 2441 | if (!msr_info->host_initiated && |
| 2442 | !guest_cpuid_has(vcpu, X86_FEATURE_VIRT_SSBD)) |
| 2443 | return 1; |
| 2444 | |
| 2445 | msr_info->data = svm->virt_spec_ctrl; |
| 2446 | break; |
Borislav Petkov | ae8b787 | 2015-11-23 11:12:23 +0100 | [diff] [blame] | 2447 | case MSR_F15H_IC_CFG: { |
| 2448 | |
| 2449 | int family, model; |
| 2450 | |
| 2451 | family = guest_cpuid_family(vcpu); |
| 2452 | model = guest_cpuid_model(vcpu); |
| 2453 | |
| 2454 | if (family < 0 || model < 0) |
| 2455 | return kvm_get_msr_common(vcpu, msr_info); |
| 2456 | |
| 2457 | msr_info->data = 0; |
| 2458 | |
| 2459 | if (family == 0x15 && |
| 2460 | (model >= 0x2 && model < 0x20)) |
| 2461 | msr_info->data = 0x1E; |
| 2462 | } |
| 2463 | break; |
Tom Lendacky | d1d93fa | 2018-02-24 00:18:20 +0100 | [diff] [blame] | 2464 | case MSR_F10H_DECFG: |
| 2465 | msr_info->data = svm->msr_decfg; |
| 2466 | break; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 2467 | default: |
Paolo Bonzini | 609e36d | 2015-04-08 15:30:38 +0200 | [diff] [blame] | 2468 | return kvm_get_msr_common(vcpu, msr_info); |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 2469 | } |
| 2470 | return 0; |
| 2471 | } |
| 2472 | |
Avi Kivity | 851ba69 | 2009-08-24 11:10:17 +0300 | [diff] [blame] | 2473 | static int rdmsr_interception(struct vcpu_svm *svm) |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 2474 | { |
Sean Christopherson | 1edce0a | 2019-09-05 14:22:55 -0700 | [diff] [blame] | 2475 | return kvm_emulate_rdmsr(&svm->vcpu); |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 2476 | } |
| 2477 | |
Joerg Roedel | 4a81018 | 2010-02-24 18:59:15 +0100 | [diff] [blame] | 2478 | static int svm_set_vm_cr(struct kvm_vcpu *vcpu, u64 data) |
| 2479 | { |
| 2480 | struct vcpu_svm *svm = to_svm(vcpu); |
| 2481 | int svm_dis, chg_mask; |
| 2482 | |
| 2483 | if (data & ~SVM_VM_CR_VALID_MASK) |
| 2484 | return 1; |
| 2485 | |
| 2486 | chg_mask = SVM_VM_CR_VALID_MASK; |
| 2487 | |
| 2488 | if (svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK) |
| 2489 | chg_mask &= ~(SVM_VM_CR_SVM_LOCK_MASK | SVM_VM_CR_SVM_DIS_MASK); |
| 2490 | |
| 2491 | svm->nested.vm_cr_msr &= ~chg_mask; |
| 2492 | svm->nested.vm_cr_msr |= (data & chg_mask); |
| 2493 | |
| 2494 | svm_dis = svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK; |
| 2495 | |
| 2496 | /* check for svm_disable while efer.svme is set */ |
| 2497 | if (svm_dis && (vcpu->arch.efer & EFER_SVME)) |
| 2498 | return 1; |
| 2499 | |
| 2500 | return 0; |
| 2501 | } |
| 2502 | |
Will Auld | 8fe8ab4 | 2012-11-29 12:42:12 -0800 | [diff] [blame] | 2503 | static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr) |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 2504 | { |
Gregory Haskins | a2fa3e9 | 2007-07-27 08:13:10 -0400 | [diff] [blame] | 2505 | struct vcpu_svm *svm = to_svm(vcpu); |
| 2506 | |
Will Auld | 8fe8ab4 | 2012-11-29 12:42:12 -0800 | [diff] [blame] | 2507 | u32 ecx = msr->index; |
| 2508 | u64 data = msr->data; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 2509 | switch (ecx) { |
Paolo Bonzini | 15038e1 | 2017-10-26 09:13:27 +0200 | [diff] [blame] | 2510 | case MSR_IA32_CR_PAT: |
| 2511 | if (!kvm_mtrr_valid(vcpu, MSR_IA32_CR_PAT, data)) |
| 2512 | return 1; |
| 2513 | vcpu->arch.pat = data; |
| 2514 | svm->vmcb->save.g_pat = data; |
| 2515 | mark_dirty(svm->vmcb, VMCB_NPT); |
| 2516 | break; |
KarimAllah Ahmed | b2ac58f | 2018-02-03 15:56:23 +0100 | [diff] [blame] | 2517 | case MSR_IA32_SPEC_CTRL: |
| 2518 | if (!msr->host_initiated && |
Paolo Bonzini | df7e881 | 2020-02-05 16:10:52 +0100 | [diff] [blame] | 2519 | !guest_cpuid_has(vcpu, X86_FEATURE_SPEC_CTRL) && |
| 2520 | !guest_cpuid_has(vcpu, X86_FEATURE_AMD_STIBP) && |
Konrad Rzeszutek Wilk | 6ac2f49 | 2018-06-01 10:59:20 -0400 | [diff] [blame] | 2521 | !guest_cpuid_has(vcpu, X86_FEATURE_AMD_IBRS) && |
| 2522 | !guest_cpuid_has(vcpu, X86_FEATURE_AMD_SSBD)) |
KarimAllah Ahmed | b2ac58f | 2018-02-03 15:56:23 +0100 | [diff] [blame] | 2523 | return 1; |
| 2524 | |
Paolo Bonzini | 6441fa6 | 2020-01-20 16:33:06 +0100 | [diff] [blame] | 2525 | if (data & ~kvm_spec_ctrl_valid_bits(vcpu)) |
KarimAllah Ahmed | b2ac58f | 2018-02-03 15:56:23 +0100 | [diff] [blame] | 2526 | return 1; |
| 2527 | |
| 2528 | svm->spec_ctrl = data; |
KarimAllah Ahmed | b2ac58f | 2018-02-03 15:56:23 +0100 | [diff] [blame] | 2529 | if (!data) |
| 2530 | break; |
| 2531 | |
| 2532 | /* |
| 2533 | * For non-nested: |
| 2534 | * When it's written (to non-zero) for the first time, pass |
| 2535 | * it through. |
| 2536 | * |
| 2537 | * For nested: |
| 2538 | * The handling of the MSR bitmap for L2 guests is done in |
| 2539 | * nested_svm_vmrun_msrpm. |
| 2540 | * We update the L1 MSR bit as well since it will end up |
| 2541 | * touching the MSR anyway now. |
| 2542 | */ |
| 2543 | set_msr_interception(svm->msrpm, MSR_IA32_SPEC_CTRL, 1, 1); |
| 2544 | break; |
Ashok Raj | 15d4507 | 2018-02-01 22:59:43 +0100 | [diff] [blame] | 2545 | case MSR_IA32_PRED_CMD: |
| 2546 | if (!msr->host_initiated && |
Borislav Petkov | e7c587d | 2018-05-02 18:15:14 +0200 | [diff] [blame] | 2547 | !guest_cpuid_has(vcpu, X86_FEATURE_AMD_IBPB)) |
Ashok Raj | 15d4507 | 2018-02-01 22:59:43 +0100 | [diff] [blame] | 2548 | return 1; |
| 2549 | |
| 2550 | if (data & ~PRED_CMD_IBPB) |
| 2551 | return 1; |
Paolo Bonzini | 6441fa6 | 2020-01-20 16:33:06 +0100 | [diff] [blame] | 2552 | if (!boot_cpu_has(X86_FEATURE_AMD_IBPB)) |
| 2553 | return 1; |
Ashok Raj | 15d4507 | 2018-02-01 22:59:43 +0100 | [diff] [blame] | 2554 | if (!data) |
| 2555 | break; |
| 2556 | |
| 2557 | wrmsrl(MSR_IA32_PRED_CMD, PRED_CMD_IBPB); |
Ashok Raj | 15d4507 | 2018-02-01 22:59:43 +0100 | [diff] [blame] | 2558 | set_msr_interception(svm->msrpm, MSR_IA32_PRED_CMD, 0, 1); |
| 2559 | break; |
Tom Lendacky | bc226f0 | 2018-05-10 22:06:39 +0200 | [diff] [blame] | 2560 | case MSR_AMD64_VIRT_SPEC_CTRL: |
| 2561 | if (!msr->host_initiated && |
| 2562 | !guest_cpuid_has(vcpu, X86_FEATURE_VIRT_SSBD)) |
| 2563 | return 1; |
| 2564 | |
| 2565 | if (data & ~SPEC_CTRL_SSBD) |
| 2566 | return 1; |
| 2567 | |
| 2568 | svm->virt_spec_ctrl = data; |
| 2569 | break; |
Brian Gerst | 8c06585 | 2010-07-17 09:03:26 -0400 | [diff] [blame] | 2570 | case MSR_STAR: |
Gregory Haskins | a2fa3e9 | 2007-07-27 08:13:10 -0400 | [diff] [blame] | 2571 | svm->vmcb->save.star = data; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 2572 | break; |
Robert P. J. Day | 49b14f2 | 2007-01-29 13:19:50 -0800 | [diff] [blame] | 2573 | #ifdef CONFIG_X86_64 |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 2574 | case MSR_LSTAR: |
Gregory Haskins | a2fa3e9 | 2007-07-27 08:13:10 -0400 | [diff] [blame] | 2575 | svm->vmcb->save.lstar = data; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 2576 | break; |
| 2577 | case MSR_CSTAR: |
Gregory Haskins | a2fa3e9 | 2007-07-27 08:13:10 -0400 | [diff] [blame] | 2578 | svm->vmcb->save.cstar = data; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 2579 | break; |
| 2580 | case MSR_KERNEL_GS_BASE: |
Gregory Haskins | a2fa3e9 | 2007-07-27 08:13:10 -0400 | [diff] [blame] | 2581 | svm->vmcb->save.kernel_gs_base = data; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 2582 | break; |
| 2583 | case MSR_SYSCALL_MASK: |
Gregory Haskins | a2fa3e9 | 2007-07-27 08:13:10 -0400 | [diff] [blame] | 2584 | svm->vmcb->save.sfmask = data; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 2585 | break; |
| 2586 | #endif |
| 2587 | case MSR_IA32_SYSENTER_CS: |
Gregory Haskins | a2fa3e9 | 2007-07-27 08:13:10 -0400 | [diff] [blame] | 2588 | svm->vmcb->save.sysenter_cs = data; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 2589 | break; |
| 2590 | case MSR_IA32_SYSENTER_EIP: |
Andre Przywara | 017cb99 | 2009-05-28 11:56:31 +0200 | [diff] [blame] | 2591 | svm->sysenter_eip = data; |
Gregory Haskins | a2fa3e9 | 2007-07-27 08:13:10 -0400 | [diff] [blame] | 2592 | svm->vmcb->save.sysenter_eip = data; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 2593 | break; |
| 2594 | case MSR_IA32_SYSENTER_ESP: |
Andre Przywara | 017cb99 | 2009-05-28 11:56:31 +0200 | [diff] [blame] | 2595 | svm->sysenter_esp = data; |
Gregory Haskins | a2fa3e9 | 2007-07-27 08:13:10 -0400 | [diff] [blame] | 2596 | svm->vmcb->save.sysenter_esp = data; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 2597 | break; |
Paolo Bonzini | 46896c7 | 2015-11-12 14:49:16 +0100 | [diff] [blame] | 2598 | case MSR_TSC_AUX: |
| 2599 | if (!boot_cpu_has(X86_FEATURE_RDTSCP)) |
| 2600 | return 1; |
| 2601 | |
| 2602 | /* |
| 2603 | * This is rare, so we update the MSR here instead of using |
| 2604 | * direct_access_msrs. Doing that would require a rdmsr in |
| 2605 | * svm_vcpu_put. |
| 2606 | */ |
| 2607 | svm->tsc_aux = data; |
| 2608 | wrmsrl(MSR_TSC_AUX, svm->tsc_aux); |
| 2609 | break; |
Joerg Roedel | a2938c8 | 2008-02-13 16:30:28 +0100 | [diff] [blame] | 2610 | case MSR_IA32_DEBUGCTLMSR: |
Avi Kivity | 2a6b20b | 2010-11-09 16:15:42 +0200 | [diff] [blame] | 2611 | if (!boot_cpu_has(X86_FEATURE_LBRV)) { |
Christoffer Dall | a737f25 | 2012-06-03 21:17:48 +0300 | [diff] [blame] | 2612 | vcpu_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTL 0x%llx, nop\n", |
| 2613 | __func__, data); |
Joerg Roedel | 24e09cb | 2008-02-13 18:58:47 +0100 | [diff] [blame] | 2614 | break; |
| 2615 | } |
| 2616 | if (data & DEBUGCTL_RESERVED_BITS) |
| 2617 | return 1; |
| 2618 | |
| 2619 | svm->vmcb->save.dbgctl = data; |
Joerg Roedel | b53ba3f | 2010-12-03 11:45:59 +0100 | [diff] [blame] | 2620 | mark_dirty(svm->vmcb, VMCB_LBR); |
Joerg Roedel | 24e09cb | 2008-02-13 18:58:47 +0100 | [diff] [blame] | 2621 | if (data & (1ULL<<0)) |
| 2622 | svm_enable_lbrv(svm); |
| 2623 | else |
| 2624 | svm_disable_lbrv(svm); |
Joerg Roedel | a2938c8 | 2008-02-13 16:30:28 +0100 | [diff] [blame] | 2625 | break; |
Alexander Graf | b286d5d | 2008-11-25 20:17:05 +0100 | [diff] [blame] | 2626 | case MSR_VM_HSAVE_PA: |
Joerg Roedel | e6aa9ab | 2009-08-07 11:49:33 +0200 | [diff] [blame] | 2627 | svm->nested.hsave_msr = data; |
Alexander Graf | b286d5d | 2008-11-25 20:17:05 +0100 | [diff] [blame] | 2628 | break; |
Alexander Graf | 3c5d0a4 | 2009-06-15 15:21:23 +0200 | [diff] [blame] | 2629 | case MSR_VM_CR: |
Joerg Roedel | 4a81018 | 2010-02-24 18:59:15 +0100 | [diff] [blame] | 2630 | return svm_set_vm_cr(vcpu, data); |
Alexander Graf | 3c5d0a4 | 2009-06-15 15:21:23 +0200 | [diff] [blame] | 2631 | case MSR_VM_IGNNE: |
Christoffer Dall | a737f25 | 2012-06-03 21:17:48 +0300 | [diff] [blame] | 2632 | vcpu_unimpl(vcpu, "unimplemented wrmsr: 0x%x data 0x%llx\n", ecx, data); |
Alexander Graf | 3c5d0a4 | 2009-06-15 15:21:23 +0200 | [diff] [blame] | 2633 | break; |
Tom Lendacky | d1d93fa | 2018-02-24 00:18:20 +0100 | [diff] [blame] | 2634 | case MSR_F10H_DECFG: { |
| 2635 | struct kvm_msr_entry msr_entry; |
| 2636 | |
| 2637 | msr_entry.index = msr->index; |
| 2638 | if (svm_get_msr_feature(&msr_entry)) |
| 2639 | return 1; |
| 2640 | |
| 2641 | /* Check the supported bits */ |
| 2642 | if (data & ~msr_entry.data) |
| 2643 | return 1; |
| 2644 | |
| 2645 | /* Don't allow the guest to change a bit, #GP */ |
| 2646 | if (!msr->host_initiated && (data ^ msr_entry.data)) |
| 2647 | return 1; |
| 2648 | |
| 2649 | svm->msr_decfg = data; |
| 2650 | break; |
| 2651 | } |
Suravee Suthikulpanit | 44a95da | 2016-05-04 14:09:46 -0500 | [diff] [blame] | 2652 | case MSR_IA32_APICBASE: |
| 2653 | if (kvm_vcpu_apicv_active(vcpu)) |
| 2654 | avic_update_vapic_bar(to_svm(vcpu), data); |
Gustavo A. R. Silva | b2869f2 | 2019-01-25 12:23:17 -0600 | [diff] [blame] | 2655 | /* Fall through */ |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 2656 | default: |
Will Auld | 8fe8ab4 | 2012-11-29 12:42:12 -0800 | [diff] [blame] | 2657 | return kvm_set_msr_common(vcpu, msr); |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 2658 | } |
| 2659 | return 0; |
| 2660 | } |
| 2661 | |
Avi Kivity | 851ba69 | 2009-08-24 11:10:17 +0300 | [diff] [blame] | 2662 | static int wrmsr_interception(struct vcpu_svm *svm) |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 2663 | { |
Sean Christopherson | 1edce0a | 2019-09-05 14:22:55 -0700 | [diff] [blame] | 2664 | return kvm_emulate_wrmsr(&svm->vcpu); |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 2665 | } |
| 2666 | |
Avi Kivity | 851ba69 | 2009-08-24 11:10:17 +0300 | [diff] [blame] | 2667 | static int msr_interception(struct vcpu_svm *svm) |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 2668 | { |
Rusty Russell | e756fc6 | 2007-07-30 20:07:08 +1000 | [diff] [blame] | 2669 | if (svm->vmcb->control.exit_info_1) |
Avi Kivity | 851ba69 | 2009-08-24 11:10:17 +0300 | [diff] [blame] | 2670 | return wrmsr_interception(svm); |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 2671 | else |
Avi Kivity | 851ba69 | 2009-08-24 11:10:17 +0300 | [diff] [blame] | 2672 | return rdmsr_interception(svm); |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 2673 | } |
| 2674 | |
Avi Kivity | 851ba69 | 2009-08-24 11:10:17 +0300 | [diff] [blame] | 2675 | static int interrupt_window_interception(struct vcpu_svm *svm) |
Dor Laor | c1150d8 | 2007-01-05 16:36:24 -0800 | [diff] [blame] | 2676 | { |
Avi Kivity | 3842d13 | 2010-07-27 12:30:24 +0300 | [diff] [blame] | 2677 | kvm_make_request(KVM_REQ_EVENT, &svm->vcpu); |
Alexander Graf | f0b8505 | 2008-11-25 20:17:01 +0100 | [diff] [blame] | 2678 | svm_clear_vintr(svm); |
Suravee Suthikulpanit | f3515dc | 2019-11-14 14:15:15 -0600 | [diff] [blame] | 2679 | |
| 2680 | /* |
| 2681 | * For AVIC, the only reason to end up here is ExtINTs. |
| 2682 | * In this case AVIC was temporarily disabled for |
| 2683 | * requesting the IRQ window and we have to re-enable it. |
| 2684 | */ |
| 2685 | svm_toggle_avic_for_irq_window(&svm->vcpu, true); |
| 2686 | |
Jason Wang | 675acb7 | 2012-03-08 18:07:56 +0800 | [diff] [blame] | 2687 | ++svm->vcpu.stat.irq_window_exits; |
Dor Laor | c1150d8 | 2007-01-05 16:36:24 -0800 | [diff] [blame] | 2688 | return 1; |
| 2689 | } |
| 2690 | |
Mark Langsdorf | 565d099 | 2009-10-06 14:25:02 -0500 | [diff] [blame] | 2691 | static int pause_interception(struct vcpu_svm *svm) |
| 2692 | { |
Longpeng(Mike) | de63ad4 | 2017-08-08 12:05:33 +0800 | [diff] [blame] | 2693 | struct kvm_vcpu *vcpu = &svm->vcpu; |
| 2694 | bool in_kernel = (svm_get_cpl(vcpu) == 0); |
| 2695 | |
Babu Moger | 8566ac8 | 2018-03-16 16:37:26 -0400 | [diff] [blame] | 2696 | if (pause_filter_thresh) |
| 2697 | grow_ple_window(vcpu); |
| 2698 | |
Longpeng(Mike) | de63ad4 | 2017-08-08 12:05:33 +0800 | [diff] [blame] | 2699 | kvm_vcpu_on_spin(vcpu, in_kernel); |
Mark Langsdorf | 565d099 | 2009-10-06 14:25:02 -0500 | [diff] [blame] | 2700 | return 1; |
| 2701 | } |
| 2702 | |
Gabriel L. Somlo | 87c0057 | 2014-05-07 16:52:13 -0400 | [diff] [blame] | 2703 | static int nop_interception(struct vcpu_svm *svm) |
| 2704 | { |
Ladi Prosek | b742c1e | 2017-06-22 09:05:26 +0200 | [diff] [blame] | 2705 | return kvm_skip_emulated_instruction(&(svm->vcpu)); |
Gabriel L. Somlo | 87c0057 | 2014-05-07 16:52:13 -0400 | [diff] [blame] | 2706 | } |
| 2707 | |
| 2708 | static int monitor_interception(struct vcpu_svm *svm) |
| 2709 | { |
| 2710 | printk_once(KERN_WARNING "kvm: MONITOR instruction emulated as NOP!\n"); |
| 2711 | return nop_interception(svm); |
| 2712 | } |
| 2713 | |
| 2714 | static int mwait_interception(struct vcpu_svm *svm) |
| 2715 | { |
| 2716 | printk_once(KERN_WARNING "kvm: MWAIT instruction emulated as NOP!\n"); |
| 2717 | return nop_interception(svm); |
| 2718 | } |
| 2719 | |
Mathias Krause | 09941fb | 2012-08-30 01:30:20 +0200 | [diff] [blame] | 2720 | static int (*const svm_exit_handlers[])(struct vcpu_svm *svm) = { |
Andre Przywara | 7ff76d5 | 2010-12-21 11:12:04 +0100 | [diff] [blame] | 2721 | [SVM_EXIT_READ_CR0] = cr_interception, |
| 2722 | [SVM_EXIT_READ_CR3] = cr_interception, |
| 2723 | [SVM_EXIT_READ_CR4] = cr_interception, |
| 2724 | [SVM_EXIT_READ_CR8] = cr_interception, |
David Kaplan | 5e57518 | 2015-03-06 14:44:35 -0600 | [diff] [blame] | 2725 | [SVM_EXIT_CR0_SEL_WRITE] = cr_interception, |
Joerg Roedel | 628afd2 | 2011-04-04 12:39:36 +0200 | [diff] [blame] | 2726 | [SVM_EXIT_WRITE_CR0] = cr_interception, |
Andre Przywara | 7ff76d5 | 2010-12-21 11:12:04 +0100 | [diff] [blame] | 2727 | [SVM_EXIT_WRITE_CR3] = cr_interception, |
| 2728 | [SVM_EXIT_WRITE_CR4] = cr_interception, |
Joerg Roedel | e023171 | 2010-02-24 18:59:10 +0100 | [diff] [blame] | 2729 | [SVM_EXIT_WRITE_CR8] = cr8_write_interception, |
Andre Przywara | cae3797 | 2010-12-21 11:12:05 +0100 | [diff] [blame] | 2730 | [SVM_EXIT_READ_DR0] = dr_interception, |
| 2731 | [SVM_EXIT_READ_DR1] = dr_interception, |
| 2732 | [SVM_EXIT_READ_DR2] = dr_interception, |
| 2733 | [SVM_EXIT_READ_DR3] = dr_interception, |
| 2734 | [SVM_EXIT_READ_DR4] = dr_interception, |
| 2735 | [SVM_EXIT_READ_DR5] = dr_interception, |
| 2736 | [SVM_EXIT_READ_DR6] = dr_interception, |
| 2737 | [SVM_EXIT_READ_DR7] = dr_interception, |
| 2738 | [SVM_EXIT_WRITE_DR0] = dr_interception, |
| 2739 | [SVM_EXIT_WRITE_DR1] = dr_interception, |
| 2740 | [SVM_EXIT_WRITE_DR2] = dr_interception, |
| 2741 | [SVM_EXIT_WRITE_DR3] = dr_interception, |
| 2742 | [SVM_EXIT_WRITE_DR4] = dr_interception, |
| 2743 | [SVM_EXIT_WRITE_DR5] = dr_interception, |
| 2744 | [SVM_EXIT_WRITE_DR6] = dr_interception, |
| 2745 | [SVM_EXIT_WRITE_DR7] = dr_interception, |
Jan Kiszka | d0bfb94 | 2008-12-15 13:52:10 +0100 | [diff] [blame] | 2746 | [SVM_EXIT_EXCP_BASE + DB_VECTOR] = db_interception, |
| 2747 | [SVM_EXIT_EXCP_BASE + BP_VECTOR] = bp_interception, |
Anthony Liguori | 7aa81cc | 2007-09-17 14:57:50 -0500 | [diff] [blame] | 2748 | [SVM_EXIT_EXCP_BASE + UD_VECTOR] = ud_interception, |
Joerg Roedel | e023171 | 2010-02-24 18:59:10 +0100 | [diff] [blame] | 2749 | [SVM_EXIT_EXCP_BASE + PF_VECTOR] = pf_interception, |
Joerg Roedel | e023171 | 2010-02-24 18:59:10 +0100 | [diff] [blame] | 2750 | [SVM_EXIT_EXCP_BASE + MC_VECTOR] = mc_interception, |
Eric Northup | 54a2055 | 2015-11-03 18:03:53 +0100 | [diff] [blame] | 2751 | [SVM_EXIT_EXCP_BASE + AC_VECTOR] = ac_interception, |
Liran Alon | 9718420 | 2018-03-12 13:12:52 +0200 | [diff] [blame] | 2752 | [SVM_EXIT_EXCP_BASE + GP_VECTOR] = gp_interception, |
Joerg Roedel | e023171 | 2010-02-24 18:59:10 +0100 | [diff] [blame] | 2753 | [SVM_EXIT_INTR] = intr_interception, |
Joerg Roedel | c47f098 | 2008-04-30 17:56:00 +0200 | [diff] [blame] | 2754 | [SVM_EXIT_NMI] = nmi_interception, |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 2755 | [SVM_EXIT_SMI] = nop_on_interception, |
| 2756 | [SVM_EXIT_INIT] = nop_on_interception, |
Dor Laor | c1150d8 | 2007-01-05 16:36:24 -0800 | [diff] [blame] | 2757 | [SVM_EXIT_VINTR] = interrupt_window_interception, |
Avi Kivity | 332b56e | 2011-11-10 14:57:24 +0200 | [diff] [blame] | 2758 | [SVM_EXIT_RDPMC] = rdpmc_interception, |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 2759 | [SVM_EXIT_CPUID] = cpuid_interception, |
Gleb Natapov | 95ba827313 | 2009-04-21 17:45:08 +0300 | [diff] [blame] | 2760 | [SVM_EXIT_IRET] = iret_interception, |
Avi Kivity | cf5a94d | 2007-10-28 16:11:58 +0200 | [diff] [blame] | 2761 | [SVM_EXIT_INVD] = emulate_on_interception, |
Mark Langsdorf | 565d099 | 2009-10-06 14:25:02 -0500 | [diff] [blame] | 2762 | [SVM_EXIT_PAUSE] = pause_interception, |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 2763 | [SVM_EXIT_HLT] = halt_interception, |
Marcelo Tosatti | a705289 | 2008-09-23 13:18:35 -0300 | [diff] [blame] | 2764 | [SVM_EXIT_INVLPG] = invlpg_interception, |
Alexander Graf | ff09238 | 2009-06-15 15:21:24 +0200 | [diff] [blame] | 2765 | [SVM_EXIT_INVLPGA] = invlpga_interception, |
Joerg Roedel | e023171 | 2010-02-24 18:59:10 +0100 | [diff] [blame] | 2766 | [SVM_EXIT_IOIO] = io_interception, |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 2767 | [SVM_EXIT_MSR] = msr_interception, |
| 2768 | [SVM_EXIT_TASK_SWITCH] = task_switch_interception, |
Joerg Roedel | 46fe4dd | 2007-01-26 00:56:42 -0800 | [diff] [blame] | 2769 | [SVM_EXIT_SHUTDOWN] = shutdown_interception, |
Alexander Graf | 3d6368e | 2008-11-25 20:17:07 +0100 | [diff] [blame] | 2770 | [SVM_EXIT_VMRUN] = vmrun_interception, |
Avi Kivity | 02e235b | 2007-02-19 14:37:47 +0200 | [diff] [blame] | 2771 | [SVM_EXIT_VMMCALL] = vmmcall_interception, |
Alexander Graf | 5542675 | 2008-11-25 20:17:06 +0100 | [diff] [blame] | 2772 | [SVM_EXIT_VMLOAD] = vmload_interception, |
| 2773 | [SVM_EXIT_VMSAVE] = vmsave_interception, |
Alexander Graf | 1371d90 | 2008-11-25 20:17:04 +0100 | [diff] [blame] | 2774 | [SVM_EXIT_STGI] = stgi_interception, |
| 2775 | [SVM_EXIT_CLGI] = clgi_interception, |
Joerg Roedel | 532a46b | 2009-10-09 16:08:32 +0200 | [diff] [blame] | 2776 | [SVM_EXIT_SKINIT] = skinit_interception, |
David Kaplan | dab429a | 2015-03-02 13:43:37 -0600 | [diff] [blame] | 2777 | [SVM_EXIT_WBINVD] = wbinvd_interception, |
Gabriel L. Somlo | 87c0057 | 2014-05-07 16:52:13 -0400 | [diff] [blame] | 2778 | [SVM_EXIT_MONITOR] = monitor_interception, |
| 2779 | [SVM_EXIT_MWAIT] = mwait_interception, |
Joerg Roedel | 81dd35d | 2010-12-07 17:15:06 +0100 | [diff] [blame] | 2780 | [SVM_EXIT_XSETBV] = xsetbv_interception, |
Jim Mattson | 0cb8410 | 2019-09-19 15:59:17 -0700 | [diff] [blame] | 2781 | [SVM_EXIT_RDPRU] = rdpru_interception, |
Paolo Bonzini | d000653 | 2017-08-11 18:36:43 +0200 | [diff] [blame] | 2782 | [SVM_EXIT_NPF] = npf_interception, |
Brijesh Singh | 7607b71 | 2018-02-19 10:14:44 -0600 | [diff] [blame] | 2783 | [SVM_EXIT_RSM] = rsm_interception, |
Suravee Suthikulpanit | 18f40c5 | 2016-05-04 14:09:48 -0500 | [diff] [blame] | 2784 | [SVM_EXIT_AVIC_INCOMPLETE_IPI] = avic_incomplete_ipi_interception, |
| 2785 | [SVM_EXIT_AVIC_UNACCELERATED_ACCESS] = avic_unaccelerated_access_interception, |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 2786 | }; |
| 2787 | |
Joe Perches | ae8cc05 | 2011-04-24 22:00:50 -0700 | [diff] [blame] | 2788 | static void dump_vmcb(struct kvm_vcpu *vcpu) |
Joerg Roedel | 3f10c84 | 2010-05-05 16:04:42 +0200 | [diff] [blame] | 2789 | { |
| 2790 | struct vcpu_svm *svm = to_svm(vcpu); |
| 2791 | struct vmcb_control_area *control = &svm->vmcb->control; |
| 2792 | struct vmcb_save_area *save = &svm->vmcb->save; |
| 2793 | |
Paolo Bonzini | 6f2f845 | 2019-05-20 15:34:35 +0200 | [diff] [blame] | 2794 | if (!dump_invalid_vmcb) { |
| 2795 | pr_warn_ratelimited("set kvm_amd.dump_invalid_vmcb=1 to dump internal KVM state.\n"); |
| 2796 | return; |
| 2797 | } |
| 2798 | |
Joerg Roedel | 3f10c84 | 2010-05-05 16:04:42 +0200 | [diff] [blame] | 2799 | pr_err("VMCB Control Area:\n"); |
Joe Perches | ae8cc05 | 2011-04-24 22:00:50 -0700 | [diff] [blame] | 2800 | pr_err("%-20s%04x\n", "cr_read:", control->intercept_cr & 0xffff); |
| 2801 | pr_err("%-20s%04x\n", "cr_write:", control->intercept_cr >> 16); |
| 2802 | pr_err("%-20s%04x\n", "dr_read:", control->intercept_dr & 0xffff); |
| 2803 | pr_err("%-20s%04x\n", "dr_write:", control->intercept_dr >> 16); |
| 2804 | pr_err("%-20s%08x\n", "exceptions:", control->intercept_exceptions); |
| 2805 | pr_err("%-20s%016llx\n", "intercepts:", control->intercept); |
| 2806 | pr_err("%-20s%d\n", "pause filter count:", control->pause_filter_count); |
Babu Moger | 1d8fb44 | 2018-03-16 16:37:25 -0400 | [diff] [blame] | 2807 | pr_err("%-20s%d\n", "pause filter threshold:", |
| 2808 | control->pause_filter_thresh); |
Joe Perches | ae8cc05 | 2011-04-24 22:00:50 -0700 | [diff] [blame] | 2809 | pr_err("%-20s%016llx\n", "iopm_base_pa:", control->iopm_base_pa); |
| 2810 | pr_err("%-20s%016llx\n", "msrpm_base_pa:", control->msrpm_base_pa); |
| 2811 | pr_err("%-20s%016llx\n", "tsc_offset:", control->tsc_offset); |
| 2812 | pr_err("%-20s%d\n", "asid:", control->asid); |
| 2813 | pr_err("%-20s%d\n", "tlb_ctl:", control->tlb_ctl); |
| 2814 | pr_err("%-20s%08x\n", "int_ctl:", control->int_ctl); |
| 2815 | pr_err("%-20s%08x\n", "int_vector:", control->int_vector); |
| 2816 | pr_err("%-20s%08x\n", "int_state:", control->int_state); |
| 2817 | pr_err("%-20s%08x\n", "exit_code:", control->exit_code); |
| 2818 | pr_err("%-20s%016llx\n", "exit_info1:", control->exit_info_1); |
| 2819 | pr_err("%-20s%016llx\n", "exit_info2:", control->exit_info_2); |
| 2820 | pr_err("%-20s%08x\n", "exit_int_info:", control->exit_int_info); |
| 2821 | pr_err("%-20s%08x\n", "exit_int_info_err:", control->exit_int_info_err); |
| 2822 | pr_err("%-20s%lld\n", "nested_ctl:", control->nested_ctl); |
| 2823 | pr_err("%-20s%016llx\n", "nested_cr3:", control->nested_cr3); |
Suravee Suthikulpanit | 44a95da | 2016-05-04 14:09:46 -0500 | [diff] [blame] | 2824 | pr_err("%-20s%016llx\n", "avic_vapic_bar:", control->avic_vapic_bar); |
Joe Perches | ae8cc05 | 2011-04-24 22:00:50 -0700 | [diff] [blame] | 2825 | pr_err("%-20s%08x\n", "event_inj:", control->event_inj); |
| 2826 | pr_err("%-20s%08x\n", "event_inj_err:", control->event_inj_err); |
Janakarajan Natarajan | 0dc9211 | 2017-07-06 15:50:45 -0500 | [diff] [blame] | 2827 | pr_err("%-20s%lld\n", "virt_ext:", control->virt_ext); |
Joe Perches | ae8cc05 | 2011-04-24 22:00:50 -0700 | [diff] [blame] | 2828 | pr_err("%-20s%016llx\n", "next_rip:", control->next_rip); |
Suravee Suthikulpanit | 44a95da | 2016-05-04 14:09:46 -0500 | [diff] [blame] | 2829 | pr_err("%-20s%016llx\n", "avic_backing_page:", control->avic_backing_page); |
| 2830 | pr_err("%-20s%016llx\n", "avic_logical_id:", control->avic_logical_id); |
| 2831 | pr_err("%-20s%016llx\n", "avic_physical_id:", control->avic_physical_id); |
Joerg Roedel | 3f10c84 | 2010-05-05 16:04:42 +0200 | [diff] [blame] | 2832 | pr_err("VMCB State Save Area:\n"); |
Joe Perches | ae8cc05 | 2011-04-24 22:00:50 -0700 | [diff] [blame] | 2833 | pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", |
| 2834 | "es:", |
| 2835 | save->es.selector, save->es.attrib, |
| 2836 | save->es.limit, save->es.base); |
| 2837 | pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", |
| 2838 | "cs:", |
| 2839 | save->cs.selector, save->cs.attrib, |
| 2840 | save->cs.limit, save->cs.base); |
| 2841 | pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", |
| 2842 | "ss:", |
| 2843 | save->ss.selector, save->ss.attrib, |
| 2844 | save->ss.limit, save->ss.base); |
| 2845 | pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", |
| 2846 | "ds:", |
| 2847 | save->ds.selector, save->ds.attrib, |
| 2848 | save->ds.limit, save->ds.base); |
| 2849 | pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", |
| 2850 | "fs:", |
| 2851 | save->fs.selector, save->fs.attrib, |
| 2852 | save->fs.limit, save->fs.base); |
| 2853 | pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", |
| 2854 | "gs:", |
| 2855 | save->gs.selector, save->gs.attrib, |
| 2856 | save->gs.limit, save->gs.base); |
| 2857 | pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", |
| 2858 | "gdtr:", |
| 2859 | save->gdtr.selector, save->gdtr.attrib, |
| 2860 | save->gdtr.limit, save->gdtr.base); |
| 2861 | pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", |
| 2862 | "ldtr:", |
| 2863 | save->ldtr.selector, save->ldtr.attrib, |
| 2864 | save->ldtr.limit, save->ldtr.base); |
| 2865 | pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", |
| 2866 | "idtr:", |
| 2867 | save->idtr.selector, save->idtr.attrib, |
| 2868 | save->idtr.limit, save->idtr.base); |
| 2869 | pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", |
| 2870 | "tr:", |
| 2871 | save->tr.selector, save->tr.attrib, |
| 2872 | save->tr.limit, save->tr.base); |
Joerg Roedel | 3f10c84 | 2010-05-05 16:04:42 +0200 | [diff] [blame] | 2873 | pr_err("cpl: %d efer: %016llx\n", |
| 2874 | save->cpl, save->efer); |
Joe Perches | ae8cc05 | 2011-04-24 22:00:50 -0700 | [diff] [blame] | 2875 | pr_err("%-15s %016llx %-13s %016llx\n", |
| 2876 | "cr0:", save->cr0, "cr2:", save->cr2); |
| 2877 | pr_err("%-15s %016llx %-13s %016llx\n", |
| 2878 | "cr3:", save->cr3, "cr4:", save->cr4); |
| 2879 | pr_err("%-15s %016llx %-13s %016llx\n", |
| 2880 | "dr6:", save->dr6, "dr7:", save->dr7); |
| 2881 | pr_err("%-15s %016llx %-13s %016llx\n", |
| 2882 | "rip:", save->rip, "rflags:", save->rflags); |
| 2883 | pr_err("%-15s %016llx %-13s %016llx\n", |
| 2884 | "rsp:", save->rsp, "rax:", save->rax); |
| 2885 | pr_err("%-15s %016llx %-13s %016llx\n", |
| 2886 | "star:", save->star, "lstar:", save->lstar); |
| 2887 | pr_err("%-15s %016llx %-13s %016llx\n", |
| 2888 | "cstar:", save->cstar, "sfmask:", save->sfmask); |
| 2889 | pr_err("%-15s %016llx %-13s %016llx\n", |
| 2890 | "kernel_gs_base:", save->kernel_gs_base, |
| 2891 | "sysenter_cs:", save->sysenter_cs); |
| 2892 | pr_err("%-15s %016llx %-13s %016llx\n", |
| 2893 | "sysenter_esp:", save->sysenter_esp, |
| 2894 | "sysenter_eip:", save->sysenter_eip); |
| 2895 | pr_err("%-15s %016llx %-13s %016llx\n", |
| 2896 | "gpat:", save->g_pat, "dbgctl:", save->dbgctl); |
| 2897 | pr_err("%-15s %016llx %-13s %016llx\n", |
| 2898 | "br_from:", save->br_from, "br_to:", save->br_to); |
| 2899 | pr_err("%-15s %016llx %-13s %016llx\n", |
| 2900 | "excp_from:", save->last_excp_from, |
| 2901 | "excp_to:", save->last_excp_to); |
Joerg Roedel | 3f10c84 | 2010-05-05 16:04:42 +0200 | [diff] [blame] | 2902 | } |
| 2903 | |
Avi Kivity | 586f960 | 2010-11-18 13:09:54 +0200 | [diff] [blame] | 2904 | static void svm_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2) |
| 2905 | { |
| 2906 | struct vmcb_control_area *control = &to_svm(vcpu)->vmcb->control; |
| 2907 | |
| 2908 | *info1 = control->exit_info_1; |
| 2909 | *info2 = control->exit_info_2; |
| 2910 | } |
| 2911 | |
Wanpeng Li | 404d5d7 | 2020-04-28 14:23:25 +0800 | [diff] [blame] | 2912 | static int handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 2913 | { |
Avi Kivity | 04d2cc7 | 2007-09-10 18:10:54 +0300 | [diff] [blame] | 2914 | struct vcpu_svm *svm = to_svm(vcpu); |
Avi Kivity | 851ba69 | 2009-08-24 11:10:17 +0300 | [diff] [blame] | 2915 | struct kvm_run *kvm_run = vcpu->run; |
Gregory Haskins | a2fa3e9 | 2007-07-27 08:13:10 -0400 | [diff] [blame] | 2916 | u32 exit_code = svm->vmcb->control.exit_code; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 2917 | |
Paolo Bonzini | 8b89fe1 | 2015-12-10 18:37:32 +0100 | [diff] [blame] | 2918 | trace_kvm_exit(exit_code, vcpu, KVM_ISA_SVM); |
| 2919 | |
Roedel, Joerg | 4ee546b | 2010-12-03 10:50:51 +0100 | [diff] [blame] | 2920 | if (!is_cr_intercept(svm, INTERCEPT_CR0_WRITE)) |
Joerg Roedel | 2be4fc7 | 2010-04-22 12:33:09 +0200 | [diff] [blame] | 2921 | vcpu->arch.cr0 = svm->vmcb->save.cr0; |
| 2922 | if (npt_enabled) |
| 2923 | vcpu->arch.cr3 = svm->vmcb->save.cr3; |
Joerg Roedel | af9ca2d | 2008-04-30 17:56:03 +0200 | [diff] [blame] | 2924 | |
Paolo Bonzini | 36e2e98 | 2020-05-22 06:04:57 -0400 | [diff] [blame] | 2925 | svm_complete_interrupts(svm); |
| 2926 | |
Joerg Roedel | 2030753 | 2010-11-29 17:51:48 +0100 | [diff] [blame] | 2927 | if (is_guest_mode(vcpu)) { |
Joerg Roedel | 410e4d5 | 2009-08-07 11:49:44 +0200 | [diff] [blame] | 2928 | int vmexit; |
| 2929 | |
Joerg Roedel | d8cabdd | 2009-10-09 16:08:28 +0200 | [diff] [blame] | 2930 | trace_kvm_nested_vmexit(svm->vmcb->save.rip, exit_code, |
| 2931 | svm->vmcb->control.exit_info_1, |
| 2932 | svm->vmcb->control.exit_info_2, |
| 2933 | svm->vmcb->control.exit_int_info, |
Stefan Hajnoczi | e097e5f | 2011-07-22 12:46:52 +0100 | [diff] [blame] | 2934 | svm->vmcb->control.exit_int_info_err, |
| 2935 | KVM_ISA_SVM); |
Joerg Roedel | d8cabdd | 2009-10-09 16:08:28 +0200 | [diff] [blame] | 2936 | |
Joerg Roedel | 410e4d5 | 2009-08-07 11:49:44 +0200 | [diff] [blame] | 2937 | vmexit = nested_svm_exit_special(svm); |
| 2938 | |
| 2939 | if (vmexit == NESTED_EXIT_CONTINUE) |
| 2940 | vmexit = nested_svm_exit_handled(svm); |
| 2941 | |
| 2942 | if (vmexit == NESTED_EXIT_DONE) |
Alexander Graf | cf74a78 | 2008-11-25 20:17:08 +0100 | [diff] [blame] | 2943 | return 1; |
Alexander Graf | cf74a78 | 2008-11-25 20:17:08 +0100 | [diff] [blame] | 2944 | } |
| 2945 | |
Avi Kivity | 04d2cc7 | 2007-09-10 18:10:54 +0300 | [diff] [blame] | 2946 | if (svm->vmcb->control.exit_code == SVM_EXIT_ERR) { |
| 2947 | kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY; |
| 2948 | kvm_run->fail_entry.hardware_entry_failure_reason |
| 2949 | = svm->vmcb->control.exit_code; |
Joerg Roedel | 3f10c84 | 2010-05-05 16:04:42 +0200 | [diff] [blame] | 2950 | dump_vmcb(vcpu); |
Avi Kivity | 04d2cc7 | 2007-09-10 18:10:54 +0300 | [diff] [blame] | 2951 | return 0; |
| 2952 | } |
| 2953 | |
Gregory Haskins | a2fa3e9 | 2007-07-27 08:13:10 -0400 | [diff] [blame] | 2954 | if (is_external_interrupt(svm->vmcb->control.exit_int_info) && |
Joerg Roedel | 709ddeb | 2008-02-07 13:47:45 +0100 | [diff] [blame] | 2955 | exit_code != SVM_EXIT_EXCP_BASE + PF_VECTOR && |
Joerg Roedel | 55c5e46 | 2010-09-10 17:31:04 +0200 | [diff] [blame] | 2956 | exit_code != SVM_EXIT_NPF && exit_code != SVM_EXIT_TASK_SWITCH && |
| 2957 | exit_code != SVM_EXIT_INTR && exit_code != SVM_EXIT_NMI) |
Borislav Petkov | 6614c7d | 2013-04-26 00:22:01 +0200 | [diff] [blame] | 2958 | printk(KERN_ERR "%s: unexpected exit_int_info 0x%x " |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 2959 | "exit_code 0x%x\n", |
Harvey Harrison | b8688d5 | 2008-03-03 12:59:56 -0800 | [diff] [blame] | 2960 | __func__, svm->vmcb->control.exit_int_info, |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 2961 | exit_code); |
| 2962 | |
Wanpeng Li | 404d5d7 | 2020-04-28 14:23:25 +0800 | [diff] [blame] | 2963 | if (exit_fastpath != EXIT_FASTPATH_NONE) |
Wanpeng Li | 1e9e262 | 2019-11-21 11:17:11 +0800 | [diff] [blame] | 2964 | return 1; |
Wanpeng Li | 404d5d7 | 2020-04-28 14:23:25 +0800 | [diff] [blame] | 2965 | |
| 2966 | if (exit_code >= ARRAY_SIZE(svm_exit_handlers) |
Joe Perches | 56919c5 | 2007-11-12 20:06:51 -0800 | [diff] [blame] | 2967 | || !svm_exit_handlers[exit_code]) { |
Liran Alon | 7396d33 | 2019-08-26 13:16:43 +0300 | [diff] [blame] | 2968 | vcpu_unimpl(vcpu, "svm: unexpected exit reason 0x%x\n", exit_code); |
| 2969 | dump_vmcb(vcpu); |
| 2970 | vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; |
| 2971 | vcpu->run->internal.suberror = |
| 2972 | KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON; |
| 2973 | vcpu->run->internal.ndata = 1; |
| 2974 | vcpu->run->internal.data[0] = exit_code; |
| 2975 | return 0; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 2976 | } |
| 2977 | |
Andrea Arcangeli | 3dcb2a3 | 2019-11-04 18:00:00 -0500 | [diff] [blame] | 2978 | #ifdef CONFIG_RETPOLINE |
| 2979 | if (exit_code == SVM_EXIT_MSR) |
| 2980 | return msr_interception(svm); |
| 2981 | else if (exit_code == SVM_EXIT_VINTR) |
| 2982 | return interrupt_window_interception(svm); |
| 2983 | else if (exit_code == SVM_EXIT_INTR) |
| 2984 | return intr_interception(svm); |
| 2985 | else if (exit_code == SVM_EXIT_HLT) |
| 2986 | return halt_interception(svm); |
| 2987 | else if (exit_code == SVM_EXIT_NPF) |
| 2988 | return npf_interception(svm); |
| 2989 | #endif |
Avi Kivity | 851ba69 | 2009-08-24 11:10:17 +0300 | [diff] [blame] | 2990 | return svm_exit_handlers[exit_code](svm); |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 2991 | } |
| 2992 | |
| 2993 | static void reload_tss(struct kvm_vcpu *vcpu) |
| 2994 | { |
| 2995 | int cpu = raw_smp_processor_id(); |
| 2996 | |
Tejun Heo | 0fe1e00 | 2009-10-29 22:34:14 +0900 | [diff] [blame] | 2997 | struct svm_cpu_data *sd = per_cpu(svm_data, cpu); |
| 2998 | sd->tss_desc->type = 9; /* available 32/64-bit TSS */ |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 2999 | load_TR_desc(); |
| 3000 | } |
| 3001 | |
Rusty Russell | e756fc6 | 2007-07-30 20:07:08 +1000 | [diff] [blame] | 3002 | static void pre_svm_run(struct vcpu_svm *svm) |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 3003 | { |
| 3004 | int cpu = raw_smp_processor_id(); |
| 3005 | |
Tejun Heo | 0fe1e00 | 2009-10-29 22:34:14 +0900 | [diff] [blame] | 3006 | struct svm_cpu_data *sd = per_cpu(svm_data, cpu); |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 3007 | |
Brijesh Singh | 70cd94e | 2017-12-04 10:57:34 -0600 | [diff] [blame] | 3008 | if (sev_guest(svm->vcpu.kvm)) |
| 3009 | return pre_sev_run(svm, cpu); |
| 3010 | |
Marcelo Tosatti | 4b656b1 | 2009-07-21 12:47:45 -0300 | [diff] [blame] | 3011 | /* FIXME: handle wraparound of asid_generation */ |
Tejun Heo | 0fe1e00 | 2009-10-29 22:34:14 +0900 | [diff] [blame] | 3012 | if (svm->asid_generation != sd->asid_generation) |
| 3013 | new_asid(svm, sd); |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 3014 | } |
| 3015 | |
Gleb Natapov | 95ba827313 | 2009-04-21 17:45:08 +0300 | [diff] [blame] | 3016 | static void svm_inject_nmi(struct kvm_vcpu *vcpu) |
| 3017 | { |
| 3018 | struct vcpu_svm *svm = to_svm(vcpu); |
| 3019 | |
| 3020 | svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI; |
| 3021 | vcpu->arch.hflags |= HF_NMI_MASK; |
Joerg Roedel | 8a05a1b8 | 2010-11-30 18:04:00 +0100 | [diff] [blame] | 3022 | set_intercept(svm, INTERCEPT_IRET); |
Gleb Natapov | 95ba827313 | 2009-04-21 17:45:08 +0300 | [diff] [blame] | 3023 | ++vcpu->stat.nmi_injections; |
| 3024 | } |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 3025 | |
Gleb Natapov | 66fd3f7 | 2009-05-11 13:35:50 +0300 | [diff] [blame] | 3026 | static void svm_set_irq(struct kvm_vcpu *vcpu) |
Eddie Dong | 2a8067f | 2007-08-06 16:29:07 +0300 | [diff] [blame] | 3027 | { |
| 3028 | struct vcpu_svm *svm = to_svm(vcpu); |
| 3029 | |
Joerg Roedel | 2af9194 | 2009-08-07 11:49:28 +0200 | [diff] [blame] | 3030 | BUG_ON(!(gif_set(svm))); |
Alexander Graf | cf74a78 | 2008-11-25 20:17:08 +0100 | [diff] [blame] | 3031 | |
Gleb Natapov | 9fb2d2b | 2010-05-23 14:28:26 +0300 | [diff] [blame] | 3032 | trace_kvm_inj_virq(vcpu->arch.interrupt.nr); |
| 3033 | ++vcpu->stat.irq_injections; |
| 3034 | |
Alexander Graf | 219b65d | 2009-06-15 15:21:25 +0200 | [diff] [blame] | 3035 | svm->vmcb->control.event_inj = vcpu->arch.interrupt.nr | |
| 3036 | SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR; |
Eddie Dong | 2a8067f | 2007-08-06 16:29:07 +0300 | [diff] [blame] | 3037 | } |
| 3038 | |
Gleb Natapov | 95ba827313 | 2009-04-21 17:45:08 +0300 | [diff] [blame] | 3039 | static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr) |
| 3040 | { |
| 3041 | struct vcpu_svm *svm = to_svm(vcpu); |
| 3042 | |
Liran Alon | 49d654d | 2019-11-11 14:26:21 +0200 | [diff] [blame] | 3043 | if (svm_nested_virtualize_tpr(vcpu)) |
Joerg Roedel | 88ab24a | 2010-02-19 16:23:06 +0100 | [diff] [blame] | 3044 | return; |
| 3045 | |
Radim Krčmář | 596f314 | 2014-03-11 19:11:18 +0100 | [diff] [blame] | 3046 | clr_cr_intercept(svm, INTERCEPT_CR8_WRITE); |
| 3047 | |
Gleb Natapov | 95ba827313 | 2009-04-21 17:45:08 +0300 | [diff] [blame] | 3048 | if (irr == -1) |
| 3049 | return; |
| 3050 | |
| 3051 | if (tpr >= irr) |
Roedel, Joerg | 4ee546b | 2010-12-03 10:50:51 +0100 | [diff] [blame] | 3052 | set_cr_intercept(svm, INTERCEPT_CR8_WRITE); |
Gleb Natapov | 95ba827313 | 2009-04-21 17:45:08 +0300 | [diff] [blame] | 3053 | } |
| 3054 | |
Paolo Bonzini | cae96af | 2020-04-23 14:19:26 -0400 | [diff] [blame] | 3055 | bool svm_nmi_blocked(struct kvm_vcpu *vcpu) |
Joerg Roedel | aaacfc9 | 2008-04-16 16:51:18 +0200 | [diff] [blame] | 3056 | { |
| 3057 | struct vcpu_svm *svm = to_svm(vcpu); |
| 3058 | struct vmcb *vmcb = svm->vmcb; |
Sean Christopherson | 88c604b | 2020-04-22 19:25:41 -0700 | [diff] [blame] | 3059 | bool ret; |
Cathy Avery | 9c3d370 | 2020-04-14 16:11:06 -0400 | [diff] [blame] | 3060 | |
Paolo Bonzini | cae96af | 2020-04-23 14:19:26 -0400 | [diff] [blame] | 3061 | if (!gif_set(svm)) |
Paolo Bonzini | bbdad0b | 2020-04-23 08:06:43 -0400 | [diff] [blame] | 3062 | return true; |
| 3063 | |
Paolo Bonzini | cae96af | 2020-04-23 14:19:26 -0400 | [diff] [blame] | 3064 | if (is_guest_mode(vcpu) && nested_exit_on_nmi(svm)) |
| 3065 | return false; |
| 3066 | |
| 3067 | ret = (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) || |
| 3068 | (svm->vcpu.arch.hflags & HF_NMI_MASK); |
Joerg Roedel | 924584c | 2010-04-22 12:33:07 +0200 | [diff] [blame] | 3069 | |
| 3070 | return ret; |
Joerg Roedel | aaacfc9 | 2008-04-16 16:51:18 +0200 | [diff] [blame] | 3071 | } |
| 3072 | |
Paolo Bonzini | c9d4091 | 2020-05-22 11:21:49 -0400 | [diff] [blame] | 3073 | static int svm_nmi_allowed(struct kvm_vcpu *vcpu, bool for_injection) |
Paolo Bonzini | cae96af | 2020-04-23 14:19:26 -0400 | [diff] [blame] | 3074 | { |
| 3075 | struct vcpu_svm *svm = to_svm(vcpu); |
| 3076 | if (svm->nested.nested_run_pending) |
Paolo Bonzini | c9d4091 | 2020-05-22 11:21:49 -0400 | [diff] [blame] | 3077 | return -EBUSY; |
Paolo Bonzini | cae96af | 2020-04-23 14:19:26 -0400 | [diff] [blame] | 3078 | |
Paolo Bonzini | c300ab9 | 2020-04-23 14:08:58 -0400 | [diff] [blame] | 3079 | /* An NMI must not be injected into L2 if it's supposed to VM-Exit. */ |
| 3080 | if (for_injection && is_guest_mode(vcpu) && nested_exit_on_nmi(svm)) |
Paolo Bonzini | c9d4091 | 2020-05-22 11:21:49 -0400 | [diff] [blame] | 3081 | return -EBUSY; |
Paolo Bonzini | c300ab9 | 2020-04-23 14:08:58 -0400 | [diff] [blame] | 3082 | |
| 3083 | return !svm_nmi_blocked(vcpu); |
Paolo Bonzini | cae96af | 2020-04-23 14:19:26 -0400 | [diff] [blame] | 3084 | } |
| 3085 | |
Jan Kiszka | 3cfc309 | 2009-11-12 01:04:25 +0100 | [diff] [blame] | 3086 | static bool svm_get_nmi_mask(struct kvm_vcpu *vcpu) |
| 3087 | { |
| 3088 | struct vcpu_svm *svm = to_svm(vcpu); |
| 3089 | |
| 3090 | return !!(svm->vcpu.arch.hflags & HF_NMI_MASK); |
| 3091 | } |
| 3092 | |
| 3093 | static void svm_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked) |
| 3094 | { |
| 3095 | struct vcpu_svm *svm = to_svm(vcpu); |
| 3096 | |
| 3097 | if (masked) { |
| 3098 | svm->vcpu.arch.hflags |= HF_NMI_MASK; |
Joerg Roedel | 8a05a1b8 | 2010-11-30 18:04:00 +0100 | [diff] [blame] | 3099 | set_intercept(svm, INTERCEPT_IRET); |
Jan Kiszka | 3cfc309 | 2009-11-12 01:04:25 +0100 | [diff] [blame] | 3100 | } else { |
| 3101 | svm->vcpu.arch.hflags &= ~HF_NMI_MASK; |
Joerg Roedel | 8a05a1b8 | 2010-11-30 18:04:00 +0100 | [diff] [blame] | 3102 | clr_intercept(svm, INTERCEPT_IRET); |
Jan Kiszka | 3cfc309 | 2009-11-12 01:04:25 +0100 | [diff] [blame] | 3103 | } |
| 3104 | } |
| 3105 | |
Paolo Bonzini | cae96af | 2020-04-23 14:19:26 -0400 | [diff] [blame] | 3106 | bool svm_interrupt_blocked(struct kvm_vcpu *vcpu) |
Gleb Natapov | 7864612 | 2009-03-23 12:12:11 +0200 | [diff] [blame] | 3107 | { |
| 3108 | struct vcpu_svm *svm = to_svm(vcpu); |
| 3109 | struct vmcb *vmcb = svm->vmcb; |
Joerg Roedel | 7fcdb51 | 2009-09-16 15:24:15 +0200 | [diff] [blame] | 3110 | |
Paolo Bonzini | fc6f7c0 | 2020-04-23 18:02:45 -0400 | [diff] [blame] | 3111 | if (!gif_set(svm)) |
Paolo Bonzini | cae96af | 2020-04-23 14:19:26 -0400 | [diff] [blame] | 3112 | return true; |
Joerg Roedel | 7fcdb51 | 2009-09-16 15:24:15 +0200 | [diff] [blame] | 3113 | |
Paolo Bonzini | fc6f7c0 | 2020-04-23 18:02:45 -0400 | [diff] [blame] | 3114 | if (is_guest_mode(vcpu)) { |
| 3115 | /* As long as interrupts are being delivered... */ |
Paolo Bonzini | e9fd761 | 2020-05-13 13:28:23 -0400 | [diff] [blame] | 3116 | if ((svm->nested.ctl.int_ctl & V_INTR_MASKING_MASK) |
Paolo Bonzini | 08245e6 | 2020-05-19 09:21:04 -0400 | [diff] [blame] | 3117 | ? !(svm->nested.hsave->save.rflags & X86_EFLAGS_IF) |
Paolo Bonzini | fc6f7c0 | 2020-04-23 18:02:45 -0400 | [diff] [blame] | 3118 | : !(kvm_get_rflags(vcpu) & X86_EFLAGS_IF)) |
| 3119 | return true; |
| 3120 | |
| 3121 | /* ... vmexits aren't blocked by the interrupt shadow */ |
| 3122 | if (nested_exit_on_intr(svm)) |
| 3123 | return false; |
| 3124 | } else { |
| 3125 | if (!(kvm_get_rflags(vcpu) & X86_EFLAGS_IF)) |
| 3126 | return true; |
| 3127 | } |
| 3128 | |
| 3129 | return (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK); |
Paolo Bonzini | cae96af | 2020-04-23 14:19:26 -0400 | [diff] [blame] | 3130 | } |
| 3131 | |
Paolo Bonzini | c9d4091 | 2020-05-22 11:21:49 -0400 | [diff] [blame] | 3132 | static int svm_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection) |
Paolo Bonzini | cae96af | 2020-04-23 14:19:26 -0400 | [diff] [blame] | 3133 | { |
| 3134 | struct vcpu_svm *svm = to_svm(vcpu); |
| 3135 | if (svm->nested.nested_run_pending) |
Paolo Bonzini | c9d4091 | 2020-05-22 11:21:49 -0400 | [diff] [blame] | 3136 | return -EBUSY; |
Paolo Bonzini | cae96af | 2020-04-23 14:19:26 -0400 | [diff] [blame] | 3137 | |
Paolo Bonzini | c300ab9 | 2020-04-23 14:08:58 -0400 | [diff] [blame] | 3138 | /* |
| 3139 | * An IRQ must not be injected into L2 if it's supposed to VM-Exit, |
| 3140 | * e.g. if the IRQ arrived asynchronously after checking nested events. |
| 3141 | */ |
| 3142 | if (for_injection && is_guest_mode(vcpu) && nested_exit_on_intr(svm)) |
Paolo Bonzini | c9d4091 | 2020-05-22 11:21:49 -0400 | [diff] [blame] | 3143 | return -EBUSY; |
Paolo Bonzini | c300ab9 | 2020-04-23 14:08:58 -0400 | [diff] [blame] | 3144 | |
| 3145 | return !svm_interrupt_blocked(vcpu); |
Gleb Natapov | 7864612 | 2009-03-23 12:12:11 +0200 | [diff] [blame] | 3146 | } |
| 3147 | |
Jan Kiszka | c9a7953 | 2014-03-07 20:03:15 +0100 | [diff] [blame] | 3148 | static void enable_irq_window(struct kvm_vcpu *vcpu) |
Gleb Natapov | 9222be1 | 2009-04-23 17:14:37 +0300 | [diff] [blame] | 3149 | { |
Alexander Graf | 219b65d | 2009-06-15 15:21:25 +0200 | [diff] [blame] | 3150 | struct vcpu_svm *svm = to_svm(vcpu); |
Alexander Graf | 219b65d | 2009-06-15 15:21:25 +0200 | [diff] [blame] | 3151 | |
Joerg Roedel | e023171 | 2010-02-24 18:59:10 +0100 | [diff] [blame] | 3152 | /* |
| 3153 | * In case GIF=0 we can't rely on the CPU to tell us when GIF becomes |
| 3154 | * 1, because that's a separate STGI/VMRUN intercept. The next time we |
| 3155 | * get that intercept, this function will be called again though and |
Janakarajan Natarajan | 640bd6e | 2017-08-23 09:57:19 -0500 | [diff] [blame] | 3156 | * we'll get the vintr intercept. However, if the vGIF feature is |
| 3157 | * enabled, the STGI interception will not occur. Enable the irq |
| 3158 | * window under the assumption that the hardware will set the GIF. |
Joerg Roedel | e023171 | 2010-02-24 18:59:10 +0100 | [diff] [blame] | 3159 | */ |
Paolo Bonzini | b518ba9 | 2020-03-04 16:46:47 -0500 | [diff] [blame] | 3160 | if (vgif_enabled(svm) || gif_set(svm)) { |
Suravee Suthikulpanit | f3515dc | 2019-11-14 14:15:15 -0600 | [diff] [blame] | 3161 | /* |
| 3162 | * IRQ window is not needed when AVIC is enabled, |
| 3163 | * unless we have pending ExtINT since it cannot be injected |
| 3164 | * via AVIC. In such case, we need to temporarily disable AVIC, |
| 3165 | * and fallback to injecting IRQ via V_IRQ. |
| 3166 | */ |
| 3167 | svm_toggle_avic_for_irq_window(vcpu, false); |
Alexander Graf | 219b65d | 2009-06-15 15:21:25 +0200 | [diff] [blame] | 3168 | svm_set_vintr(svm); |
Alexander Graf | 219b65d | 2009-06-15 15:21:25 +0200 | [diff] [blame] | 3169 | } |
Gleb Natapov | 9222be1 | 2009-04-23 17:14:37 +0300 | [diff] [blame] | 3170 | } |
| 3171 | |
Jan Kiszka | c9a7953 | 2014-03-07 20:03:15 +0100 | [diff] [blame] | 3172 | static void enable_nmi_window(struct kvm_vcpu *vcpu) |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 3173 | { |
Avi Kivity | 04d2cc7 | 2007-09-10 18:10:54 +0300 | [diff] [blame] | 3174 | struct vcpu_svm *svm = to_svm(vcpu); |
Eddie Dong | 85f455f | 2007-07-06 12:20:49 +0300 | [diff] [blame] | 3175 | |
Gleb Natapov | 44c1143 | 2009-05-11 13:35:52 +0300 | [diff] [blame] | 3176 | if ((svm->vcpu.arch.hflags & (HF_NMI_MASK | HF_IRET_MASK)) |
| 3177 | == HF_NMI_MASK) |
Jan Kiszka | c9a7953 | 2014-03-07 20:03:15 +0100 | [diff] [blame] | 3178 | return; /* IRET will cause a vm exit */ |
Gleb Natapov | 44c1143 | 2009-05-11 13:35:52 +0300 | [diff] [blame] | 3179 | |
Janakarajan Natarajan | 640bd6e | 2017-08-23 09:57:19 -0500 | [diff] [blame] | 3180 | if (!gif_set(svm)) { |
| 3181 | if (vgif_enabled(svm)) |
| 3182 | set_intercept(svm, INTERCEPT_STGI); |
Ladi Prosek | 1a5e185 | 2017-06-21 09:07:01 +0200 | [diff] [blame] | 3183 | return; /* STGI will cause a vm exit */ |
Janakarajan Natarajan | 640bd6e | 2017-08-23 09:57:19 -0500 | [diff] [blame] | 3184 | } |
Ladi Prosek | 1a5e185 | 2017-06-21 09:07:01 +0200 | [diff] [blame] | 3185 | |
Joerg Roedel | e023171 | 2010-02-24 18:59:10 +0100 | [diff] [blame] | 3186 | /* |
| 3187 | * Something prevents NMI from been injected. Single step over possible |
| 3188 | * problem (IRET or exception injection or interrupt shadow) |
| 3189 | */ |
Ladi Prosek | ab2f4d73 | 2017-06-21 09:06:58 +0200 | [diff] [blame] | 3190 | svm->nmi_singlestep_guest_rflags = svm_get_rflags(vcpu); |
Jan Kiszka | 6be7d30 | 2009-10-18 13:24:54 +0200 | [diff] [blame] | 3191 | svm->nmi_singlestep = true; |
Gleb Natapov | 44c1143 | 2009-05-11 13:35:52 +0300 | [diff] [blame] | 3192 | svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF); |
Eddie Dong | 85f455f | 2007-07-06 12:20:49 +0300 | [diff] [blame] | 3193 | } |
| 3194 | |
Izik Eidus | cbc9402 | 2007-10-25 00:29:55 +0200 | [diff] [blame] | 3195 | static int svm_set_tss_addr(struct kvm *kvm, unsigned int addr) |
| 3196 | { |
| 3197 | return 0; |
| 3198 | } |
| 3199 | |
Sean Christopherson | 2ac52ab | 2018-03-20 12:17:19 -0700 | [diff] [blame] | 3200 | static int svm_set_identity_map_addr(struct kvm *kvm, u64 ident_addr) |
| 3201 | { |
| 3202 | return 0; |
| 3203 | } |
| 3204 | |
Sean Christopherson | f55ac30 | 2020-03-20 14:28:12 -0700 | [diff] [blame] | 3205 | void svm_flush_tlb(struct kvm_vcpu *vcpu) |
Avi Kivity | d9e368d | 2007-06-07 19:18:30 +0300 | [diff] [blame] | 3206 | { |
Joerg Roedel | 38e5e92 | 2010-12-03 15:25:16 +0100 | [diff] [blame] | 3207 | struct vcpu_svm *svm = to_svm(vcpu); |
| 3208 | |
Sean Christopherson | 4a41e43 | 2020-03-20 14:28:17 -0700 | [diff] [blame] | 3209 | /* |
| 3210 | * Flush only the current ASID even if the TLB flush was invoked via |
| 3211 | * kvm_flush_remote_tlbs(). Although flushing remote TLBs requires all |
| 3212 | * ASIDs to be flushed, KVM uses a single ASID for L1 and L2, and |
| 3213 | * unconditionally does a TLB flush on both nested VM-Enter and nested |
| 3214 | * VM-Exit (via kvm_mmu_reset_context()). |
| 3215 | */ |
Joerg Roedel | 38e5e92 | 2010-12-03 15:25:16 +0100 | [diff] [blame] | 3216 | if (static_cpu_has(X86_FEATURE_FLUSHBYASID)) |
| 3217 | svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID; |
| 3218 | else |
| 3219 | svm->asid_generation--; |
Avi Kivity | d9e368d | 2007-06-07 19:18:30 +0300 | [diff] [blame] | 3220 | } |
| 3221 | |
Junaid Shahid | faff875 | 2018-06-29 13:10:05 -0700 | [diff] [blame] | 3222 | static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva) |
| 3223 | { |
| 3224 | struct vcpu_svm *svm = to_svm(vcpu); |
| 3225 | |
| 3226 | invlpga(gva, svm->vmcb->control.asid); |
| 3227 | } |
| 3228 | |
Avi Kivity | 04d2cc7 | 2007-09-10 18:10:54 +0300 | [diff] [blame] | 3229 | static void svm_prepare_guest_switch(struct kvm_vcpu *vcpu) |
| 3230 | { |
| 3231 | } |
| 3232 | |
Joerg Roedel | d7bf822 | 2008-04-16 16:51:17 +0200 | [diff] [blame] | 3233 | static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu) |
| 3234 | { |
| 3235 | struct vcpu_svm *svm = to_svm(vcpu); |
| 3236 | |
Suravee Suthikulpanit | 3bbf356 | 2016-05-04 14:09:51 -0500 | [diff] [blame] | 3237 | if (svm_nested_virtualize_tpr(vcpu)) |
Joerg Roedel | 88ab24a | 2010-02-19 16:23:06 +0100 | [diff] [blame] | 3238 | return; |
| 3239 | |
Roedel, Joerg | 4ee546b | 2010-12-03 10:50:51 +0100 | [diff] [blame] | 3240 | if (!is_cr_intercept(svm, INTERCEPT_CR8_WRITE)) { |
Joerg Roedel | d7bf822 | 2008-04-16 16:51:17 +0200 | [diff] [blame] | 3241 | int cr8 = svm->vmcb->control.int_ctl & V_TPR_MASK; |
Gleb Natapov | 615d519 | 2009-04-21 17:45:05 +0300 | [diff] [blame] | 3242 | kvm_set_cr8(vcpu, cr8); |
Joerg Roedel | d7bf822 | 2008-04-16 16:51:17 +0200 | [diff] [blame] | 3243 | } |
| 3244 | } |
| 3245 | |
Joerg Roedel | 649d686 | 2008-04-16 16:51:15 +0200 | [diff] [blame] | 3246 | static inline void sync_lapic_to_cr8(struct kvm_vcpu *vcpu) |
| 3247 | { |
| 3248 | struct vcpu_svm *svm = to_svm(vcpu); |
| 3249 | u64 cr8; |
| 3250 | |
Suravee Suthikulpanit | 3bbf356 | 2016-05-04 14:09:51 -0500 | [diff] [blame] | 3251 | if (svm_nested_virtualize_tpr(vcpu) || |
| 3252 | kvm_vcpu_apicv_active(vcpu)) |
Joerg Roedel | 88ab24a | 2010-02-19 16:23:06 +0100 | [diff] [blame] | 3253 | return; |
| 3254 | |
Joerg Roedel | 649d686 | 2008-04-16 16:51:15 +0200 | [diff] [blame] | 3255 | cr8 = kvm_get_cr8(vcpu); |
| 3256 | svm->vmcb->control.int_ctl &= ~V_TPR_MASK; |
| 3257 | svm->vmcb->control.int_ctl |= cr8 & V_TPR_MASK; |
| 3258 | } |
| 3259 | |
Gleb Natapov | 9222be1 | 2009-04-23 17:14:37 +0300 | [diff] [blame] | 3260 | static void svm_complete_interrupts(struct vcpu_svm *svm) |
| 3261 | { |
| 3262 | u8 vector; |
| 3263 | int type; |
| 3264 | u32 exitintinfo = svm->vmcb->control.exit_int_info; |
Jan Kiszka | 66b7138 | 2010-02-23 17:47:56 +0100 | [diff] [blame] | 3265 | unsigned int3_injected = svm->int3_injected; |
| 3266 | |
| 3267 | svm->int3_injected = 0; |
Gleb Natapov | 9222be1 | 2009-04-23 17:14:37 +0300 | [diff] [blame] | 3268 | |
Avi Kivity | bd3d1ec | 2011-02-03 15:29:52 +0200 | [diff] [blame] | 3269 | /* |
| 3270 | * If we've made progress since setting HF_IRET_MASK, we've |
| 3271 | * executed an IRET and can allow NMI injection. |
| 3272 | */ |
| 3273 | if ((svm->vcpu.arch.hflags & HF_IRET_MASK) |
| 3274 | && kvm_rip_read(&svm->vcpu) != svm->nmi_iret_rip) { |
Gleb Natapov | 44c1143 | 2009-05-11 13:35:52 +0300 | [diff] [blame] | 3275 | svm->vcpu.arch.hflags &= ~(HF_NMI_MASK | HF_IRET_MASK); |
Avi Kivity | 3842d13 | 2010-07-27 12:30:24 +0300 | [diff] [blame] | 3276 | kvm_make_request(KVM_REQ_EVENT, &svm->vcpu); |
| 3277 | } |
Gleb Natapov | 44c1143 | 2009-05-11 13:35:52 +0300 | [diff] [blame] | 3278 | |
Gleb Natapov | 9222be1 | 2009-04-23 17:14:37 +0300 | [diff] [blame] | 3279 | svm->vcpu.arch.nmi_injected = false; |
| 3280 | kvm_clear_exception_queue(&svm->vcpu); |
| 3281 | kvm_clear_interrupt_queue(&svm->vcpu); |
| 3282 | |
| 3283 | if (!(exitintinfo & SVM_EXITINTINFO_VALID)) |
| 3284 | return; |
| 3285 | |
Avi Kivity | 3842d13 | 2010-07-27 12:30:24 +0300 | [diff] [blame] | 3286 | kvm_make_request(KVM_REQ_EVENT, &svm->vcpu); |
| 3287 | |
Gleb Natapov | 9222be1 | 2009-04-23 17:14:37 +0300 | [diff] [blame] | 3288 | vector = exitintinfo & SVM_EXITINTINFO_VEC_MASK; |
| 3289 | type = exitintinfo & SVM_EXITINTINFO_TYPE_MASK; |
| 3290 | |
| 3291 | switch (type) { |
| 3292 | case SVM_EXITINTINFO_TYPE_NMI: |
| 3293 | svm->vcpu.arch.nmi_injected = true; |
| 3294 | break; |
| 3295 | case SVM_EXITINTINFO_TYPE_EXEPT: |
Jan Kiszka | 66b7138 | 2010-02-23 17:47:56 +0100 | [diff] [blame] | 3296 | /* |
| 3297 | * In case of software exceptions, do not reinject the vector, |
| 3298 | * but re-execute the instruction instead. Rewind RIP first |
| 3299 | * if we emulated INT3 before. |
| 3300 | */ |
| 3301 | if (kvm_exception_is_soft(vector)) { |
| 3302 | if (vector == BP_VECTOR && int3_injected && |
| 3303 | kvm_is_linear_rip(&svm->vcpu, svm->int3_rip)) |
| 3304 | kvm_rip_write(&svm->vcpu, |
| 3305 | kvm_rip_read(&svm->vcpu) - |
| 3306 | int3_injected); |
Alexander Graf | 219b65d | 2009-06-15 15:21:25 +0200 | [diff] [blame] | 3307 | break; |
Jan Kiszka | 66b7138 | 2010-02-23 17:47:56 +0100 | [diff] [blame] | 3308 | } |
Gleb Natapov | 9222be1 | 2009-04-23 17:14:37 +0300 | [diff] [blame] | 3309 | if (exitintinfo & SVM_EXITINTINFO_VALID_ERR) { |
| 3310 | u32 err = svm->vmcb->control.exit_int_info_err; |
Joerg Roedel | ce7ddec | 2010-04-22 12:33:13 +0200 | [diff] [blame] | 3311 | kvm_requeue_exception_e(&svm->vcpu, vector, err); |
Gleb Natapov | 9222be1 | 2009-04-23 17:14:37 +0300 | [diff] [blame] | 3312 | |
| 3313 | } else |
Joerg Roedel | ce7ddec | 2010-04-22 12:33:13 +0200 | [diff] [blame] | 3314 | kvm_requeue_exception(&svm->vcpu, vector); |
Gleb Natapov | 9222be1 | 2009-04-23 17:14:37 +0300 | [diff] [blame] | 3315 | break; |
| 3316 | case SVM_EXITINTINFO_TYPE_INTR: |
Gleb Natapov | 66fd3f7 | 2009-05-11 13:35:50 +0300 | [diff] [blame] | 3317 | kvm_queue_interrupt(&svm->vcpu, vector, false); |
Gleb Natapov | 9222be1 | 2009-04-23 17:14:37 +0300 | [diff] [blame] | 3318 | break; |
| 3319 | default: |
| 3320 | break; |
| 3321 | } |
| 3322 | } |
| 3323 | |
Avi Kivity | b463a6f | 2010-07-20 15:06:17 +0300 | [diff] [blame] | 3324 | static void svm_cancel_injection(struct kvm_vcpu *vcpu) |
| 3325 | { |
| 3326 | struct vcpu_svm *svm = to_svm(vcpu); |
| 3327 | struct vmcb_control_area *control = &svm->vmcb->control; |
| 3328 | |
| 3329 | control->exit_int_info = control->event_inj; |
| 3330 | control->exit_int_info_err = control->event_inj_err; |
| 3331 | control->event_inj = 0; |
| 3332 | svm_complete_interrupts(svm); |
| 3333 | } |
| 3334 | |
Wanpeng Li | 404d5d7 | 2020-04-28 14:23:25 +0800 | [diff] [blame] | 3335 | static fastpath_t svm_exit_handlers_fastpath(struct kvm_vcpu *vcpu) |
Wanpeng Li | a9ab13f | 2020-04-10 10:47:03 -0700 | [diff] [blame] | 3336 | { |
| 3337 | if (!is_guest_mode(vcpu) && |
| 3338 | to_svm(vcpu)->vmcb->control.exit_code == SVM_EXIT_MSR && |
| 3339 | to_svm(vcpu)->vmcb->control.exit_info_1) |
| 3340 | return handle_fastpath_set_msr_irqoff(vcpu); |
| 3341 | |
| 3342 | return EXIT_FASTPATH_NONE; |
| 3343 | } |
| 3344 | |
Uros Bizjak | 56a87e5 | 2020-04-09 13:49:26 +0200 | [diff] [blame] | 3345 | void __svm_vcpu_run(unsigned long vmcb_pa, unsigned long *regs); |
Uros Bizjak | 199cd1d | 2020-03-30 15:02:13 +0200 | [diff] [blame] | 3346 | |
Wanpeng Li | 404d5d7 | 2020-04-28 14:23:25 +0800 | [diff] [blame] | 3347 | static fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu) |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 3348 | { |
Wanpeng Li | 404d5d7 | 2020-04-28 14:23:25 +0800 | [diff] [blame] | 3349 | fastpath_t exit_fastpath; |
Gregory Haskins | a2fa3e9 | 2007-07-27 08:13:10 -0400 | [diff] [blame] | 3350 | struct vcpu_svm *svm = to_svm(vcpu); |
Avi Kivity | d9e368d | 2007-06-07 19:18:30 +0300 | [diff] [blame] | 3351 | |
Joerg Roedel | 2041a06 | 2010-04-22 12:33:08 +0200 | [diff] [blame] | 3352 | svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX]; |
| 3353 | svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP]; |
| 3354 | svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP]; |
| 3355 | |
Joerg Roedel | cd3ff65 | 2009-10-09 16:08:26 +0200 | [diff] [blame] | 3356 | /* |
Ladi Prosek | a12713c | 2017-06-21 09:07:00 +0200 | [diff] [blame] | 3357 | * Disable singlestep if we're injecting an interrupt/exception. |
| 3358 | * We don't want our modified rflags to be pushed on the stack where |
| 3359 | * we might not be able to easily reset them if we disabled NMI |
| 3360 | * singlestep later. |
| 3361 | */ |
| 3362 | if (svm->nmi_singlestep && svm->vmcb->control.event_inj) { |
| 3363 | /* |
| 3364 | * Event injection happens before external interrupts cause a |
| 3365 | * vmexit and interrupts are disabled here, so smp_send_reschedule |
| 3366 | * is enough to force an immediate vmexit. |
| 3367 | */ |
| 3368 | disable_nmi_singlestep(svm); |
| 3369 | smp_send_reschedule(vcpu->cpu); |
| 3370 | } |
| 3371 | |
Rusty Russell | e756fc6 | 2007-07-30 20:07:08 +1000 | [diff] [blame] | 3372 | pre_svm_run(svm); |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 3373 | |
Joerg Roedel | 649d686 | 2008-04-16 16:51:15 +0200 | [diff] [blame] | 3374 | sync_lapic_to_cr8(vcpu); |
| 3375 | |
Joerg Roedel | cda0ffd | 2009-08-07 11:49:45 +0200 | [diff] [blame] | 3376 | svm->vmcb->save.cr2 = vcpu->arch.cr2; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 3377 | |
Paolo Bonzini | d67668e | 2020-05-06 06:40:04 -0400 | [diff] [blame] | 3378 | /* |
| 3379 | * Run with all-zero DR6 unless needed, so that we can get the exact cause |
| 3380 | * of a #DB. |
| 3381 | */ |
| 3382 | if (unlikely(svm->vcpu.arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)) |
| 3383 | svm_set_dr6(svm, vcpu->arch.dr6); |
| 3384 | else |
| 3385 | svm_set_dr6(svm, DR6_FIXED_1 | DR6_RTM); |
| 3386 | |
Avi Kivity | 04d2cc7 | 2007-09-10 18:10:54 +0300 | [diff] [blame] | 3387 | clgi(); |
Aaron Lewis | 139a12c | 2019-10-21 16:30:25 -0700 | [diff] [blame] | 3388 | kvm_load_guest_xsave_state(vcpu); |
Avi Kivity | 04d2cc7 | 2007-09-10 18:10:54 +0300 | [diff] [blame] | 3389 | |
Wanpeng Li | b6c4bc6 | 2019-05-20 16:18:09 +0800 | [diff] [blame] | 3390 | if (lapic_in_kernel(vcpu) && |
| 3391 | vcpu->arch.apic->lapic_timer.timer_advance_ns) |
| 3392 | kvm_wait_lapic_expire(vcpu); |
| 3393 | |
KarimAllah Ahmed | b2ac58f | 2018-02-03 15:56:23 +0100 | [diff] [blame] | 3394 | /* |
| 3395 | * If this vCPU has touched SPEC_CTRL, restore the guest's value if |
| 3396 | * it's non-zero. Since vmentry is serialising on affected CPUs, there |
| 3397 | * is no need to worry about the conditional branch over the wrmsr |
| 3398 | * being speculatively taken. |
| 3399 | */ |
Thomas Gleixner | ccbcd26 | 2018-05-09 23:01:01 +0200 | [diff] [blame] | 3400 | x86_spec_ctrl_set_guest(svm->spec_ctrl, svm->virt_spec_ctrl); |
KarimAllah Ahmed | b2ac58f | 2018-02-03 15:56:23 +0100 | [diff] [blame] | 3401 | |
Uros Bizjak | 199cd1d | 2020-03-30 15:02:13 +0200 | [diff] [blame] | 3402 | __svm_vcpu_run(svm->vmcb_pa, (unsigned long *)&svm->vcpu.arch.regs); |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 3403 | |
Thomas Gleixner | 15e6c22 | 2018-05-11 15:21:01 +0200 | [diff] [blame] | 3404 | #ifdef CONFIG_X86_64 |
| 3405 | wrmsrl(MSR_GS_BASE, svm->host.gs_base); |
| 3406 | #else |
| 3407 | loadsegment(fs, svm->host.fs); |
| 3408 | #ifndef CONFIG_X86_32_LAZY_GS |
| 3409 | loadsegment(gs, svm->host.gs); |
| 3410 | #endif |
| 3411 | #endif |
| 3412 | |
KarimAllah Ahmed | b2ac58f | 2018-02-03 15:56:23 +0100 | [diff] [blame] | 3413 | /* |
| 3414 | * We do not use IBRS in the kernel. If this vCPU has used the |
| 3415 | * SPEC_CTRL MSR it may have left it on; save the value and |
| 3416 | * turn it off. This is much more efficient than blindly adding |
| 3417 | * it to the atomic save/restore list. Especially as the former |
| 3418 | * (Saving guest MSRs on vmexit) doesn't even exist in KVM. |
| 3419 | * |
| 3420 | * For non-nested case: |
| 3421 | * If the L01 MSR bitmap does not intercept the MSR, then we need to |
| 3422 | * save it. |
| 3423 | * |
| 3424 | * For nested case: |
| 3425 | * If the L02 MSR bitmap does not intercept the MSR, then we need to |
| 3426 | * save it. |
| 3427 | */ |
Paolo Bonzini | 946fbbc | 2018-02-22 16:43:18 +0100 | [diff] [blame] | 3428 | if (unlikely(!msr_write_intercepted(vcpu, MSR_IA32_SPEC_CTRL))) |
Paolo Bonzini | ecb586b | 2018-02-22 16:43:17 +0100 | [diff] [blame] | 3429 | svm->spec_ctrl = native_read_msr(MSR_IA32_SPEC_CTRL); |
KarimAllah Ahmed | b2ac58f | 2018-02-03 15:56:23 +0100 | [diff] [blame] | 3430 | |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 3431 | reload_tss(vcpu); |
| 3432 | |
Thomas Gleixner | 024d83c | 2018-08-12 20:41:45 +0200 | [diff] [blame] | 3433 | x86_spec_ctrl_restore_host(svm->spec_ctrl, svm->virt_spec_ctrl); |
| 3434 | |
Avi Kivity | 13c34e0 | 2010-10-21 12:20:31 +0200 | [diff] [blame] | 3435 | vcpu->arch.cr2 = svm->vmcb->save.cr2; |
| 3436 | vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax; |
| 3437 | vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp; |
| 3438 | vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip; |
| 3439 | |
Joerg Roedel | 3781c01 | 2011-01-14 16:45:02 +0100 | [diff] [blame] | 3440 | if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI)) |
Andi Kleen | dd60d21 | 2017-07-25 17:20:32 -0700 | [diff] [blame] | 3441 | kvm_before_interrupt(&svm->vcpu); |
Joerg Roedel | 3781c01 | 2011-01-14 16:45:02 +0100 | [diff] [blame] | 3442 | |
Aaron Lewis | 139a12c | 2019-10-21 16:30:25 -0700 | [diff] [blame] | 3443 | kvm_load_host_xsave_state(vcpu); |
Joerg Roedel | 3781c01 | 2011-01-14 16:45:02 +0100 | [diff] [blame] | 3444 | stgi(); |
| 3445 | |
| 3446 | /* Any pending NMI will happen here */ |
Wanpeng Li | a9ab13f | 2020-04-10 10:47:03 -0700 | [diff] [blame] | 3447 | exit_fastpath = svm_exit_handlers_fastpath(vcpu); |
Joerg Roedel | 3781c01 | 2011-01-14 16:45:02 +0100 | [diff] [blame] | 3448 | |
| 3449 | if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI)) |
Andi Kleen | dd60d21 | 2017-07-25 17:20:32 -0700 | [diff] [blame] | 3450 | kvm_after_interrupt(&svm->vcpu); |
Joerg Roedel | 3781c01 | 2011-01-14 16:45:02 +0100 | [diff] [blame] | 3451 | |
Joerg Roedel | d7bf822 | 2008-04-16 16:51:17 +0200 | [diff] [blame] | 3452 | sync_cr8_to_lapic(vcpu); |
| 3453 | |
Gregory Haskins | a2fa3e9 | 2007-07-27 08:13:10 -0400 | [diff] [blame] | 3454 | svm->next_rip = 0; |
Paolo Bonzini | 2d8a42b | 2020-05-22 03:50:14 -0400 | [diff] [blame] | 3455 | if (is_guest_mode(&svm->vcpu)) { |
| 3456 | sync_nested_vmcb_control(svm); |
| 3457 | svm->nested.nested_run_pending = 0; |
| 3458 | } |
Gleb Natapov | 9222be1 | 2009-04-23 17:14:37 +0300 | [diff] [blame] | 3459 | |
Joerg Roedel | 38e5e92 | 2010-12-03 15:25:16 +0100 | [diff] [blame] | 3460 | svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING; |
| 3461 | |
Gleb Natapov | 631bc48 | 2010-10-14 11:22:52 +0200 | [diff] [blame] | 3462 | /* if exit due to PF check for async PF */ |
| 3463 | if (svm->vmcb->control.exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR) |
Vitaly Kuznetsov | 68fd66f | 2020-05-25 16:41:17 +0200 | [diff] [blame] | 3464 | svm->vcpu.arch.apf.host_apf_flags = |
| 3465 | kvm_read_and_reset_apf_flags(); |
Gleb Natapov | 631bc48 | 2010-10-14 11:22:52 +0200 | [diff] [blame] | 3466 | |
Avi Kivity | 6de4f3a | 2009-05-31 22:58:47 +0300 | [diff] [blame] | 3467 | if (npt_enabled) { |
| 3468 | vcpu->arch.regs_avail &= ~(1 << VCPU_EXREG_PDPTR); |
| 3469 | vcpu->arch.regs_dirty &= ~(1 << VCPU_EXREG_PDPTR); |
| 3470 | } |
Joerg Roedel | fe5913e | 2010-05-17 14:43:34 +0200 | [diff] [blame] | 3471 | |
| 3472 | /* |
| 3473 | * We need to handle MC intercepts here before the vcpu has a chance to |
| 3474 | * change the physical cpu |
| 3475 | */ |
| 3476 | if (unlikely(svm->vmcb->control.exit_code == |
| 3477 | SVM_EXIT_EXCP_BASE + MC_VECTOR)) |
| 3478 | svm_handle_mce(svm); |
Roedel, Joerg | 8d28fec | 2010-12-03 13:15:21 +0100 | [diff] [blame] | 3479 | |
| 3480 | mark_all_clean(svm->vmcb); |
Wanpeng Li | a9ab13f | 2020-04-10 10:47:03 -0700 | [diff] [blame] | 3481 | return exit_fastpath; |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 3482 | } |
| 3483 | |
Paolo Bonzini | 727a7e2 | 2020-03-05 03:52:50 -0500 | [diff] [blame] | 3484 | static void svm_load_mmu_pgd(struct kvm_vcpu *vcpu, unsigned long root) |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 3485 | { |
Gregory Haskins | a2fa3e9 | 2007-07-27 08:13:10 -0400 | [diff] [blame] | 3486 | struct vcpu_svm *svm = to_svm(vcpu); |
Paolo Bonzini | 689f3bf | 2020-03-03 10:11:10 +0100 | [diff] [blame] | 3487 | unsigned long cr3; |
Gregory Haskins | a2fa3e9 | 2007-07-27 08:13:10 -0400 | [diff] [blame] | 3488 | |
Paolo Bonzini | 689f3bf | 2020-03-03 10:11:10 +0100 | [diff] [blame] | 3489 | cr3 = __sme_set(root); |
| 3490 | if (npt_enabled) { |
| 3491 | svm->vmcb->control.nested_cr3 = cr3; |
| 3492 | mark_dirty(svm->vmcb, VMCB_NPT); |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 3493 | |
Paolo Bonzini | 689f3bf | 2020-03-03 10:11:10 +0100 | [diff] [blame] | 3494 | /* Loading L2's CR3 is handled by enter_svm_guest_mode. */ |
Paolo Bonzini | 978ce58 | 2020-05-20 08:37:37 -0400 | [diff] [blame] | 3495 | if (!test_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail)) |
| 3496 | return; |
| 3497 | cr3 = vcpu->arch.cr3; |
Paolo Bonzini | 689f3bf | 2020-03-03 10:11:10 +0100 | [diff] [blame] | 3498 | } |
Joerg Roedel | 1c97f0a | 2010-09-10 17:30:41 +0200 | [diff] [blame] | 3499 | |
Paolo Bonzini | 978ce58 | 2020-05-20 08:37:37 -0400 | [diff] [blame] | 3500 | svm->vmcb->save.cr3 = cr3; |
| 3501 | mark_dirty(svm->vmcb, VMCB_CR); |
Joerg Roedel | 1c97f0a | 2010-09-10 17:30:41 +0200 | [diff] [blame] | 3502 | } |
| 3503 | |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 3504 | static int is_disabled(void) |
| 3505 | { |
Joerg Roedel | 6031a61 | 2007-06-22 12:29:50 +0300 | [diff] [blame] | 3506 | u64 vm_cr; |
| 3507 | |
| 3508 | rdmsrl(MSR_VM_CR, vm_cr); |
| 3509 | if (vm_cr & (1 << SVM_VM_CR_SVM_DISABLE)) |
| 3510 | return 1; |
| 3511 | |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 3512 | return 0; |
| 3513 | } |
| 3514 | |
Ingo Molnar | 102d832 | 2007-02-19 14:37:47 +0200 | [diff] [blame] | 3515 | static void |
| 3516 | svm_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall) |
| 3517 | { |
| 3518 | /* |
| 3519 | * Patch in the VMMCALL instruction: |
| 3520 | */ |
| 3521 | hypercall[0] = 0x0f; |
| 3522 | hypercall[1] = 0x01; |
| 3523 | hypercall[2] = 0xd9; |
Ingo Molnar | 102d832 | 2007-02-19 14:37:47 +0200 | [diff] [blame] | 3524 | } |
| 3525 | |
Sean Christopherson | f257d6d | 2019-04-19 22:18:17 -0700 | [diff] [blame] | 3526 | static int __init svm_check_processor_compat(void) |
Yang, Sheng | 002c7f7 | 2007-07-31 14:23:01 +0300 | [diff] [blame] | 3527 | { |
Sean Christopherson | f257d6d | 2019-04-19 22:18:17 -0700 | [diff] [blame] | 3528 | return 0; |
Yang, Sheng | 002c7f7 | 2007-07-31 14:23:01 +0300 | [diff] [blame] | 3529 | } |
| 3530 | |
Avi Kivity | 774ead3 | 2007-12-26 13:57:04 +0200 | [diff] [blame] | 3531 | static bool svm_cpu_has_accelerated_tpr(void) |
| 3532 | { |
| 3533 | return false; |
| 3534 | } |
| 3535 | |
Sean Christopherson | cb97c2d | 2020-02-18 15:40:11 -0800 | [diff] [blame] | 3536 | static bool svm_has_emulated_msr(u32 index) |
Paolo Bonzini | 6d396b5 | 2015-04-01 14:25:33 +0200 | [diff] [blame] | 3537 | { |
Vitaly Kuznetsov | e87555e | 2018-12-19 12:06:13 +0100 | [diff] [blame] | 3538 | switch (index) { |
| 3539 | case MSR_IA32_MCG_EXT_CTL: |
Paolo Bonzini | 95c5c7c | 2019-07-02 14:45:24 +0200 | [diff] [blame] | 3540 | case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC: |
Vitaly Kuznetsov | e87555e | 2018-12-19 12:06:13 +0100 | [diff] [blame] | 3541 | return false; |
| 3542 | default: |
| 3543 | break; |
| 3544 | } |
| 3545 | |
Paolo Bonzini | 6d396b5 | 2015-04-01 14:25:33 +0200 | [diff] [blame] | 3546 | return true; |
| 3547 | } |
| 3548 | |
Paolo Bonzini | fc07e76 | 2015-10-01 13:20:22 +0200 | [diff] [blame] | 3549 | static u64 svm_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio) |
| 3550 | { |
| 3551 | return 0; |
| 3552 | } |
| 3553 | |
Sheng Yang | 0e85188 | 2009-12-18 16:48:46 +0800 | [diff] [blame] | 3554 | static void svm_cpuid_update(struct kvm_vcpu *vcpu) |
| 3555 | { |
Joerg Roedel | 6092d3d | 2015-10-14 15:10:54 +0200 | [diff] [blame] | 3556 | struct vcpu_svm *svm = to_svm(vcpu); |
| 3557 | |
Aaron Lewis | 7204160 | 2019-10-21 16:30:20 -0700 | [diff] [blame] | 3558 | vcpu->arch.xsaves_enabled = guest_cpuid_has(vcpu, X86_FEATURE_XSAVE) && |
Sean Christopherson | 96be4e0 | 2019-12-10 14:44:15 -0800 | [diff] [blame] | 3559 | boot_cpu_has(X86_FEATURE_XSAVE) && |
Aaron Lewis | 7204160 | 2019-10-21 16:30:20 -0700 | [diff] [blame] | 3560 | boot_cpu_has(X86_FEATURE_XSAVES); |
| 3561 | |
Joerg Roedel | 6092d3d | 2015-10-14 15:10:54 +0200 | [diff] [blame] | 3562 | /* Update nrips enabled cache */ |
Sean Christopherson | 4eb8746 | 2020-03-02 15:57:08 -0800 | [diff] [blame] | 3563 | svm->nrips_enabled = kvm_cpu_cap_has(X86_FEATURE_NRIPS) && |
| 3564 | guest_cpuid_has(&svm->vcpu, X86_FEATURE_NRIPS); |
Suravee Suthikulpanit | 46781ea | 2016-05-04 14:09:50 -0500 | [diff] [blame] | 3565 | |
| 3566 | if (!kvm_vcpu_apicv_active(vcpu)) |
| 3567 | return; |
| 3568 | |
Oliver Upton | cc7f557 | 2020-02-28 00:59:04 -0800 | [diff] [blame] | 3569 | /* |
| 3570 | * AVIC does not work with an x2APIC mode guest. If the X2APIC feature |
| 3571 | * is exposed to the guest, disable AVIC. |
| 3572 | */ |
| 3573 | if (guest_cpuid_has(vcpu, X86_FEATURE_X2APIC)) |
| 3574 | kvm_request_apicv_update(vcpu->kvm, false, |
| 3575 | APICV_INHIBIT_REASON_X2APIC); |
Suravee Suthikulpanit | 9a0bf05 | 2019-11-14 14:15:14 -0600 | [diff] [blame] | 3576 | |
| 3577 | /* |
| 3578 | * Currently, AVIC does not work with nested virtualization. |
| 3579 | * So, we disable AVIC when cpuid for SVM is set in the L1 guest. |
| 3580 | */ |
| 3581 | if (nested && guest_cpuid_has(vcpu, X86_FEATURE_SVM)) |
| 3582 | kvm_request_apicv_update(vcpu->kvm, false, |
| 3583 | APICV_INHIBIT_REASON_NESTED); |
Sheng Yang | 0e85188 | 2009-12-18 16:48:46 +0800 | [diff] [blame] | 3584 | } |
| 3585 | |
Sheng Yang | f5f48ee | 2010-06-30 12:25:15 +0800 | [diff] [blame] | 3586 | static bool svm_has_wbinvd_exit(void) |
| 3587 | { |
| 3588 | return true; |
| 3589 | } |
| 3590 | |
Joerg Roedel | 8061252 | 2011-04-04 12:39:33 +0200 | [diff] [blame] | 3591 | #define PRE_EX(exit) { .exit_code = (exit), \ |
Avi Kivity | 40e19b5 | 2011-04-21 12:35:41 +0300 | [diff] [blame] | 3592 | .stage = X86_ICPT_PRE_EXCEPT, } |
Joerg Roedel | cfec82c | 2011-04-04 12:39:28 +0200 | [diff] [blame] | 3593 | #define POST_EX(exit) { .exit_code = (exit), \ |
Avi Kivity | 40e19b5 | 2011-04-21 12:35:41 +0300 | [diff] [blame] | 3594 | .stage = X86_ICPT_POST_EXCEPT, } |
Joerg Roedel | d7eb820 | 2011-04-04 12:39:32 +0200 | [diff] [blame] | 3595 | #define POST_MEM(exit) { .exit_code = (exit), \ |
Avi Kivity | 40e19b5 | 2011-04-21 12:35:41 +0300 | [diff] [blame] | 3596 | .stage = X86_ICPT_POST_MEMACCESS, } |
Joerg Roedel | cfec82c | 2011-04-04 12:39:28 +0200 | [diff] [blame] | 3597 | |
Mathias Krause | 09941fb | 2012-08-30 01:30:20 +0200 | [diff] [blame] | 3598 | static const struct __x86_intercept { |
Joerg Roedel | cfec82c | 2011-04-04 12:39:28 +0200 | [diff] [blame] | 3599 | u32 exit_code; |
| 3600 | enum x86_intercept_stage stage; |
Joerg Roedel | cfec82c | 2011-04-04 12:39:28 +0200 | [diff] [blame] | 3601 | } x86_intercept_map[] = { |
| 3602 | [x86_intercept_cr_read] = POST_EX(SVM_EXIT_READ_CR0), |
| 3603 | [x86_intercept_cr_write] = POST_EX(SVM_EXIT_WRITE_CR0), |
| 3604 | [x86_intercept_clts] = POST_EX(SVM_EXIT_WRITE_CR0), |
| 3605 | [x86_intercept_lmsw] = POST_EX(SVM_EXIT_WRITE_CR0), |
| 3606 | [x86_intercept_smsw] = POST_EX(SVM_EXIT_READ_CR0), |
Joerg Roedel | 3b88e41 | 2011-04-04 12:39:29 +0200 | [diff] [blame] | 3607 | [x86_intercept_dr_read] = POST_EX(SVM_EXIT_READ_DR0), |
| 3608 | [x86_intercept_dr_write] = POST_EX(SVM_EXIT_WRITE_DR0), |
Joerg Roedel | dee6bb7 | 2011-04-04 12:39:30 +0200 | [diff] [blame] | 3609 | [x86_intercept_sldt] = POST_EX(SVM_EXIT_LDTR_READ), |
| 3610 | [x86_intercept_str] = POST_EX(SVM_EXIT_TR_READ), |
| 3611 | [x86_intercept_lldt] = POST_EX(SVM_EXIT_LDTR_WRITE), |
| 3612 | [x86_intercept_ltr] = POST_EX(SVM_EXIT_TR_WRITE), |
| 3613 | [x86_intercept_sgdt] = POST_EX(SVM_EXIT_GDTR_READ), |
| 3614 | [x86_intercept_sidt] = POST_EX(SVM_EXIT_IDTR_READ), |
| 3615 | [x86_intercept_lgdt] = POST_EX(SVM_EXIT_GDTR_WRITE), |
| 3616 | [x86_intercept_lidt] = POST_EX(SVM_EXIT_IDTR_WRITE), |
Joerg Roedel | 01de8b0 | 2011-04-04 12:39:31 +0200 | [diff] [blame] | 3617 | [x86_intercept_vmrun] = POST_EX(SVM_EXIT_VMRUN), |
| 3618 | [x86_intercept_vmmcall] = POST_EX(SVM_EXIT_VMMCALL), |
| 3619 | [x86_intercept_vmload] = POST_EX(SVM_EXIT_VMLOAD), |
| 3620 | [x86_intercept_vmsave] = POST_EX(SVM_EXIT_VMSAVE), |
| 3621 | [x86_intercept_stgi] = POST_EX(SVM_EXIT_STGI), |
| 3622 | [x86_intercept_clgi] = POST_EX(SVM_EXIT_CLGI), |
| 3623 | [x86_intercept_skinit] = POST_EX(SVM_EXIT_SKINIT), |
| 3624 | [x86_intercept_invlpga] = POST_EX(SVM_EXIT_INVLPGA), |
Joerg Roedel | d7eb820 | 2011-04-04 12:39:32 +0200 | [diff] [blame] | 3625 | [x86_intercept_rdtscp] = POST_EX(SVM_EXIT_RDTSCP), |
| 3626 | [x86_intercept_monitor] = POST_MEM(SVM_EXIT_MONITOR), |
| 3627 | [x86_intercept_mwait] = POST_EX(SVM_EXIT_MWAIT), |
Joerg Roedel | 8061252 | 2011-04-04 12:39:33 +0200 | [diff] [blame] | 3628 | [x86_intercept_invlpg] = POST_EX(SVM_EXIT_INVLPG), |
| 3629 | [x86_intercept_invd] = POST_EX(SVM_EXIT_INVD), |
| 3630 | [x86_intercept_wbinvd] = POST_EX(SVM_EXIT_WBINVD), |
| 3631 | [x86_intercept_wrmsr] = POST_EX(SVM_EXIT_MSR), |
| 3632 | [x86_intercept_rdtsc] = POST_EX(SVM_EXIT_RDTSC), |
| 3633 | [x86_intercept_rdmsr] = POST_EX(SVM_EXIT_MSR), |
| 3634 | [x86_intercept_rdpmc] = POST_EX(SVM_EXIT_RDPMC), |
| 3635 | [x86_intercept_cpuid] = PRE_EX(SVM_EXIT_CPUID), |
| 3636 | [x86_intercept_rsm] = PRE_EX(SVM_EXIT_RSM), |
Joerg Roedel | bf608f8 | 2011-04-04 12:39:34 +0200 | [diff] [blame] | 3637 | [x86_intercept_pause] = PRE_EX(SVM_EXIT_PAUSE), |
| 3638 | [x86_intercept_pushf] = PRE_EX(SVM_EXIT_PUSHF), |
| 3639 | [x86_intercept_popf] = PRE_EX(SVM_EXIT_POPF), |
| 3640 | [x86_intercept_intn] = PRE_EX(SVM_EXIT_SWINT), |
| 3641 | [x86_intercept_iret] = PRE_EX(SVM_EXIT_IRET), |
| 3642 | [x86_intercept_icebp] = PRE_EX(SVM_EXIT_ICEBP), |
| 3643 | [x86_intercept_hlt] = POST_EX(SVM_EXIT_HLT), |
Joerg Roedel | f651193 | 2011-04-04 12:39:35 +0200 | [diff] [blame] | 3644 | [x86_intercept_in] = POST_EX(SVM_EXIT_IOIO), |
| 3645 | [x86_intercept_ins] = POST_EX(SVM_EXIT_IOIO), |
| 3646 | [x86_intercept_out] = POST_EX(SVM_EXIT_IOIO), |
| 3647 | [x86_intercept_outs] = POST_EX(SVM_EXIT_IOIO), |
Vitaly Kuznetsov | 02d4160 | 2019-08-13 15:53:32 +0200 | [diff] [blame] | 3648 | [x86_intercept_xsetbv] = PRE_EX(SVM_EXIT_XSETBV), |
Joerg Roedel | cfec82c | 2011-04-04 12:39:28 +0200 | [diff] [blame] | 3649 | }; |
| 3650 | |
Joerg Roedel | 8061252 | 2011-04-04 12:39:33 +0200 | [diff] [blame] | 3651 | #undef PRE_EX |
Joerg Roedel | cfec82c | 2011-04-04 12:39:28 +0200 | [diff] [blame] | 3652 | #undef POST_EX |
Joerg Roedel | d7eb820 | 2011-04-04 12:39:32 +0200 | [diff] [blame] | 3653 | #undef POST_MEM |
Joerg Roedel | cfec82c | 2011-04-04 12:39:28 +0200 | [diff] [blame] | 3654 | |
Joerg Roedel | 8a76d7f | 2011-04-04 12:39:27 +0200 | [diff] [blame] | 3655 | static int svm_check_intercept(struct kvm_vcpu *vcpu, |
| 3656 | struct x86_instruction_info *info, |
Sean Christopherson | 21f1b8f | 2020-02-18 15:29:42 -0800 | [diff] [blame] | 3657 | enum x86_intercept_stage stage, |
| 3658 | struct x86_exception *exception) |
Joerg Roedel | 8a76d7f | 2011-04-04 12:39:27 +0200 | [diff] [blame] | 3659 | { |
Joerg Roedel | cfec82c | 2011-04-04 12:39:28 +0200 | [diff] [blame] | 3660 | struct vcpu_svm *svm = to_svm(vcpu); |
| 3661 | int vmexit, ret = X86EMUL_CONTINUE; |
| 3662 | struct __x86_intercept icpt_info; |
| 3663 | struct vmcb *vmcb = svm->vmcb; |
| 3664 | |
| 3665 | if (info->intercept >= ARRAY_SIZE(x86_intercept_map)) |
| 3666 | goto out; |
| 3667 | |
| 3668 | icpt_info = x86_intercept_map[info->intercept]; |
| 3669 | |
Avi Kivity | 40e19b5 | 2011-04-21 12:35:41 +0300 | [diff] [blame] | 3670 | if (stage != icpt_info.stage) |
Joerg Roedel | cfec82c | 2011-04-04 12:39:28 +0200 | [diff] [blame] | 3671 | goto out; |
| 3672 | |
| 3673 | switch (icpt_info.exit_code) { |
| 3674 | case SVM_EXIT_READ_CR0: |
| 3675 | if (info->intercept == x86_intercept_cr_read) |
| 3676 | icpt_info.exit_code += info->modrm_reg; |
| 3677 | break; |
| 3678 | case SVM_EXIT_WRITE_CR0: { |
| 3679 | unsigned long cr0, val; |
| 3680 | u64 intercept; |
| 3681 | |
| 3682 | if (info->intercept == x86_intercept_cr_write) |
| 3683 | icpt_info.exit_code += info->modrm_reg; |
| 3684 | |
Jan Kiszka | 62baf44 | 2014-06-29 21:55:53 +0200 | [diff] [blame] | 3685 | if (icpt_info.exit_code != SVM_EXIT_WRITE_CR0 || |
| 3686 | info->intercept == x86_intercept_clts) |
Joerg Roedel | cfec82c | 2011-04-04 12:39:28 +0200 | [diff] [blame] | 3687 | break; |
| 3688 | |
Paolo Bonzini | e670bf6 | 2020-05-13 13:16:12 -0400 | [diff] [blame] | 3689 | intercept = svm->nested.ctl.intercept; |
Joerg Roedel | cfec82c | 2011-04-04 12:39:28 +0200 | [diff] [blame] | 3690 | |
| 3691 | if (!(intercept & (1ULL << INTERCEPT_SELECTIVE_CR0))) |
| 3692 | break; |
| 3693 | |
| 3694 | cr0 = vcpu->arch.cr0 & ~SVM_CR0_SELECTIVE_MASK; |
| 3695 | val = info->src_val & ~SVM_CR0_SELECTIVE_MASK; |
| 3696 | |
| 3697 | if (info->intercept == x86_intercept_lmsw) { |
| 3698 | cr0 &= 0xfUL; |
| 3699 | val &= 0xfUL; |
| 3700 | /* lmsw can't clear PE - catch this here */ |
| 3701 | if (cr0 & X86_CR0_PE) |
| 3702 | val |= X86_CR0_PE; |
| 3703 | } |
| 3704 | |
| 3705 | if (cr0 ^ val) |
| 3706 | icpt_info.exit_code = SVM_EXIT_CR0_SEL_WRITE; |
| 3707 | |
| 3708 | break; |
| 3709 | } |
Joerg Roedel | 3b88e41 | 2011-04-04 12:39:29 +0200 | [diff] [blame] | 3710 | case SVM_EXIT_READ_DR0: |
| 3711 | case SVM_EXIT_WRITE_DR0: |
| 3712 | icpt_info.exit_code += info->modrm_reg; |
| 3713 | break; |
Joerg Roedel | 8061252 | 2011-04-04 12:39:33 +0200 | [diff] [blame] | 3714 | case SVM_EXIT_MSR: |
| 3715 | if (info->intercept == x86_intercept_wrmsr) |
| 3716 | vmcb->control.exit_info_1 = 1; |
| 3717 | else |
| 3718 | vmcb->control.exit_info_1 = 0; |
| 3719 | break; |
Joerg Roedel | bf608f8 | 2011-04-04 12:39:34 +0200 | [diff] [blame] | 3720 | case SVM_EXIT_PAUSE: |
| 3721 | /* |
| 3722 | * We get this for NOP only, but pause |
| 3723 | * is rep not, check this here |
| 3724 | */ |
| 3725 | if (info->rep_prefix != REPE_PREFIX) |
| 3726 | goto out; |
Jan H. Schönherr | 49a8afc | 2017-09-05 23:58:44 +0200 | [diff] [blame] | 3727 | break; |
Joerg Roedel | f651193 | 2011-04-04 12:39:35 +0200 | [diff] [blame] | 3728 | case SVM_EXIT_IOIO: { |
| 3729 | u64 exit_info; |
| 3730 | u32 bytes; |
| 3731 | |
Joerg Roedel | f651193 | 2011-04-04 12:39:35 +0200 | [diff] [blame] | 3732 | if (info->intercept == x86_intercept_in || |
| 3733 | info->intercept == x86_intercept_ins) { |
Jan Kiszka | 6cbc5f5 | 2014-06-30 12:52:55 +0200 | [diff] [blame] | 3734 | exit_info = ((info->src_val & 0xffff) << 16) | |
| 3735 | SVM_IOIO_TYPE_MASK; |
Joerg Roedel | f651193 | 2011-04-04 12:39:35 +0200 | [diff] [blame] | 3736 | bytes = info->dst_bytes; |
Jan Kiszka | 6493f15 | 2014-06-30 11:07:05 +0200 | [diff] [blame] | 3737 | } else { |
Jan Kiszka | 6cbc5f5 | 2014-06-30 12:52:55 +0200 | [diff] [blame] | 3738 | exit_info = (info->dst_val & 0xffff) << 16; |
Jan Kiszka | 6493f15 | 2014-06-30 11:07:05 +0200 | [diff] [blame] | 3739 | bytes = info->src_bytes; |
Joerg Roedel | f651193 | 2011-04-04 12:39:35 +0200 | [diff] [blame] | 3740 | } |
| 3741 | |
| 3742 | if (info->intercept == x86_intercept_outs || |
| 3743 | info->intercept == x86_intercept_ins) |
| 3744 | exit_info |= SVM_IOIO_STR_MASK; |
| 3745 | |
| 3746 | if (info->rep_prefix) |
| 3747 | exit_info |= SVM_IOIO_REP_MASK; |
| 3748 | |
| 3749 | bytes = min(bytes, 4u); |
| 3750 | |
| 3751 | exit_info |= bytes << SVM_IOIO_SIZE_SHIFT; |
| 3752 | |
| 3753 | exit_info |= (u32)info->ad_bytes << (SVM_IOIO_ASIZE_SHIFT - 1); |
| 3754 | |
| 3755 | vmcb->control.exit_info_1 = exit_info; |
| 3756 | vmcb->control.exit_info_2 = info->next_rip; |
| 3757 | |
| 3758 | break; |
| 3759 | } |
Joerg Roedel | cfec82c | 2011-04-04 12:39:28 +0200 | [diff] [blame] | 3760 | default: |
| 3761 | break; |
| 3762 | } |
| 3763 | |
Bandan Das | f104765 | 2015-06-11 02:05:33 -0400 | [diff] [blame] | 3764 | /* TODO: Advertise NRIPS to guest hypervisor unconditionally */ |
| 3765 | if (static_cpu_has(X86_FEATURE_NRIPS)) |
| 3766 | vmcb->control.next_rip = info->next_rip; |
Joerg Roedel | cfec82c | 2011-04-04 12:39:28 +0200 | [diff] [blame] | 3767 | vmcb->control.exit_code = icpt_info.exit_code; |
| 3768 | vmexit = nested_svm_exit_handled(svm); |
| 3769 | |
| 3770 | ret = (vmexit == NESTED_EXIT_DONE) ? X86EMUL_INTERCEPTED |
| 3771 | : X86EMUL_CONTINUE; |
| 3772 | |
| 3773 | out: |
| 3774 | return ret; |
Joerg Roedel | 8a76d7f | 2011-04-04 12:39:27 +0200 | [diff] [blame] | 3775 | } |
| 3776 | |
Wanpeng Li | a9ab13f | 2020-04-10 10:47:03 -0700 | [diff] [blame] | 3777 | static void svm_handle_exit_irqoff(struct kvm_vcpu *vcpu) |
Yang Zhang | a547c6d | 2013-04-11 19:25:10 +0800 | [diff] [blame] | 3778 | { |
Yang Zhang | a547c6d | 2013-04-11 19:25:10 +0800 | [diff] [blame] | 3779 | } |
| 3780 | |
Radim Krčmář | ae97a3b | 2014-08-21 18:08:06 +0200 | [diff] [blame] | 3781 | static void svm_sched_in(struct kvm_vcpu *vcpu, int cpu) |
| 3782 | { |
Babu Moger | 8566ac8 | 2018-03-16 16:37:26 -0400 | [diff] [blame] | 3783 | if (pause_filter_thresh) |
| 3784 | shrink_ple_window(vcpu); |
Radim Krčmář | ae97a3b | 2014-08-21 18:08:06 +0200 | [diff] [blame] | 3785 | } |
| 3786 | |
Borislav Petkov | 74f1690 | 2017-03-26 23:51:24 +0200 | [diff] [blame] | 3787 | static void svm_setup_mce(struct kvm_vcpu *vcpu) |
| 3788 | { |
| 3789 | /* [63:9] are reserved. */ |
| 3790 | vcpu->arch.mcg_cap &= 0x1ff; |
| 3791 | } |
| 3792 | |
Paolo Bonzini | cae96af | 2020-04-23 14:19:26 -0400 | [diff] [blame] | 3793 | bool svm_smi_blocked(struct kvm_vcpu *vcpu) |
Ladi Prosek | 72d7b37 | 2017-10-11 16:54:41 +0200 | [diff] [blame] | 3794 | { |
Ladi Prosek | 05cade7 | 2017-10-11 16:54:45 +0200 | [diff] [blame] | 3795 | struct vcpu_svm *svm = to_svm(vcpu); |
| 3796 | |
| 3797 | /* Per APM Vol.2 15.22.2 "Response to SMI" */ |
| 3798 | if (!gif_set(svm)) |
Paolo Bonzini | cae96af | 2020-04-23 14:19:26 -0400 | [diff] [blame] | 3799 | return true; |
| 3800 | |
| 3801 | return is_smm(vcpu); |
| 3802 | } |
| 3803 | |
Paolo Bonzini | c9d4091 | 2020-05-22 11:21:49 -0400 | [diff] [blame] | 3804 | static int svm_smi_allowed(struct kvm_vcpu *vcpu, bool for_injection) |
Paolo Bonzini | cae96af | 2020-04-23 14:19:26 -0400 | [diff] [blame] | 3805 | { |
| 3806 | struct vcpu_svm *svm = to_svm(vcpu); |
| 3807 | if (svm->nested.nested_run_pending) |
Paolo Bonzini | c9d4091 | 2020-05-22 11:21:49 -0400 | [diff] [blame] | 3808 | return -EBUSY; |
Ladi Prosek | 05cade7 | 2017-10-11 16:54:45 +0200 | [diff] [blame] | 3809 | |
Paolo Bonzini | c300ab9 | 2020-04-23 14:08:58 -0400 | [diff] [blame] | 3810 | /* An SMI must not be injected into L2 if it's supposed to VM-Exit. */ |
| 3811 | if (for_injection && is_guest_mode(vcpu) && nested_exit_on_smi(svm)) |
Paolo Bonzini | c9d4091 | 2020-05-22 11:21:49 -0400 | [diff] [blame] | 3812 | return -EBUSY; |
Paolo Bonzini | c300ab9 | 2020-04-23 14:08:58 -0400 | [diff] [blame] | 3813 | |
Paolo Bonzini | cae96af | 2020-04-23 14:19:26 -0400 | [diff] [blame] | 3814 | return !svm_smi_blocked(vcpu); |
Ladi Prosek | 72d7b37 | 2017-10-11 16:54:41 +0200 | [diff] [blame] | 3815 | } |
| 3816 | |
Ladi Prosek | 0234bf8 | 2017-10-11 16:54:40 +0200 | [diff] [blame] | 3817 | static int svm_pre_enter_smm(struct kvm_vcpu *vcpu, char *smstate) |
| 3818 | { |
Ladi Prosek | 05cade7 | 2017-10-11 16:54:45 +0200 | [diff] [blame] | 3819 | struct vcpu_svm *svm = to_svm(vcpu); |
| 3820 | int ret; |
| 3821 | |
| 3822 | if (is_guest_mode(vcpu)) { |
| 3823 | /* FED8h - SVM Guest */ |
| 3824 | put_smstate(u64, smstate, 0x7ed8, 1); |
| 3825 | /* FEE0h - SVM Guest VMCB Physical Address */ |
| 3826 | put_smstate(u64, smstate, 0x7ee0, svm->nested.vmcb); |
| 3827 | |
| 3828 | svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX]; |
| 3829 | svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP]; |
| 3830 | svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP]; |
| 3831 | |
| 3832 | ret = nested_svm_vmexit(svm); |
| 3833 | if (ret) |
| 3834 | return ret; |
| 3835 | } |
Ladi Prosek | 0234bf8 | 2017-10-11 16:54:40 +0200 | [diff] [blame] | 3836 | return 0; |
| 3837 | } |
| 3838 | |
Sean Christopherson | ed19321 | 2019-04-02 08:03:09 -0700 | [diff] [blame] | 3839 | static int svm_pre_leave_smm(struct kvm_vcpu *vcpu, const char *smstate) |
Ladi Prosek | 0234bf8 | 2017-10-11 16:54:40 +0200 | [diff] [blame] | 3840 | { |
Ladi Prosek | 05cade7 | 2017-10-11 16:54:45 +0200 | [diff] [blame] | 3841 | struct vcpu_svm *svm = to_svm(vcpu); |
| 3842 | struct vmcb *nested_vmcb; |
KarimAllah Ahmed | 8c5fbf1 | 2019-01-31 21:24:40 +0100 | [diff] [blame] | 3843 | struct kvm_host_map map; |
Sean Christopherson | ed19321 | 2019-04-02 08:03:09 -0700 | [diff] [blame] | 3844 | u64 guest; |
| 3845 | u64 vmcb; |
Ladi Prosek | 05cade7 | 2017-10-11 16:54:45 +0200 | [diff] [blame] | 3846 | |
Sean Christopherson | ed19321 | 2019-04-02 08:03:09 -0700 | [diff] [blame] | 3847 | guest = GET_SMSTATE(u64, smstate, 0x7ed8); |
| 3848 | vmcb = GET_SMSTATE(u64, smstate, 0x7ee0); |
Ladi Prosek | 05cade7 | 2017-10-11 16:54:45 +0200 | [diff] [blame] | 3849 | |
Sean Christopherson | ed19321 | 2019-04-02 08:03:09 -0700 | [diff] [blame] | 3850 | if (guest) { |
KarimAllah Ahmed | 8c5fbf1 | 2019-01-31 21:24:40 +0100 | [diff] [blame] | 3851 | if (kvm_vcpu_map(&svm->vcpu, gpa_to_gfn(vmcb), &map) == -EINVAL) |
Sean Christopherson | 9ec1949 | 2019-04-02 08:03:11 -0700 | [diff] [blame] | 3852 | return 1; |
KarimAllah Ahmed | 8c5fbf1 | 2019-01-31 21:24:40 +0100 | [diff] [blame] | 3853 | nested_vmcb = map.hva; |
Paolo Bonzini | 69c9dfa | 2020-05-13 12:57:26 -0400 | [diff] [blame] | 3854 | enter_svm_guest_mode(svm, vmcb, nested_vmcb); |
| 3855 | kvm_vcpu_unmap(&svm->vcpu, &map, true); |
Ladi Prosek | 05cade7 | 2017-10-11 16:54:45 +0200 | [diff] [blame] | 3856 | } |
Sean Christopherson | 9ec1949 | 2019-04-02 08:03:11 -0700 | [diff] [blame] | 3857 | return 0; |
Ladi Prosek | 0234bf8 | 2017-10-11 16:54:40 +0200 | [diff] [blame] | 3858 | } |
| 3859 | |
Paolo Bonzini | c9d4091 | 2020-05-22 11:21:49 -0400 | [diff] [blame] | 3860 | static void enable_smi_window(struct kvm_vcpu *vcpu) |
Ladi Prosek | cc3d967 | 2017-10-17 16:02:39 +0200 | [diff] [blame] | 3861 | { |
| 3862 | struct vcpu_svm *svm = to_svm(vcpu); |
| 3863 | |
| 3864 | if (!gif_set(svm)) { |
| 3865 | if (vgif_enabled(svm)) |
| 3866 | set_intercept(svm, INTERCEPT_STGI); |
| 3867 | /* STGI will cause a vm exit */ |
Paolo Bonzini | c9d4091 | 2020-05-22 11:21:49 -0400 | [diff] [blame] | 3868 | } else { |
| 3869 | /* We must be in SMM; RSM will cause a vmexit anyway. */ |
Ladi Prosek | cc3d967 | 2017-10-17 16:02:39 +0200 | [diff] [blame] | 3870 | } |
Ladi Prosek | cc3d967 | 2017-10-17 16:02:39 +0200 | [diff] [blame] | 3871 | } |
| 3872 | |
Singh, Brijesh | 05d5a48 | 2019-02-15 17:24:12 +0000 | [diff] [blame] | 3873 | static bool svm_need_emulation_on_page_fault(struct kvm_vcpu *vcpu) |
| 3874 | { |
Liran Alon | 118154b | 2019-07-17 02:56:58 +0300 | [diff] [blame] | 3875 | unsigned long cr4 = kvm_read_cr4(vcpu); |
| 3876 | bool smep = cr4 & X86_CR4_SMEP; |
| 3877 | bool smap = cr4 & X86_CR4_SMAP; |
| 3878 | bool is_user = svm_get_cpl(vcpu) == 3; |
Singh, Brijesh | 05d5a48 | 2019-02-15 17:24:12 +0000 | [diff] [blame] | 3879 | |
| 3880 | /* |
Paolo Bonzini | e72436b | 2020-04-17 12:21:06 -0400 | [diff] [blame] | 3881 | * If RIP is invalid, go ahead with emulation which will cause an |
| 3882 | * internal error exit. |
| 3883 | */ |
| 3884 | if (!kvm_vcpu_gfn_to_memslot(vcpu, kvm_rip_read(vcpu) >> PAGE_SHIFT)) |
| 3885 | return true; |
| 3886 | |
| 3887 | /* |
Liran Alon | 118154b | 2019-07-17 02:56:58 +0300 | [diff] [blame] | 3888 | * Detect and workaround Errata 1096 Fam_17h_00_0Fh. |
| 3889 | * |
| 3890 | * Errata: |
| 3891 | * When CPU raise #NPF on guest data access and vCPU CR4.SMAP=1, it is |
| 3892 | * possible that CPU microcode implementing DecodeAssist will fail |
| 3893 | * to read bytes of instruction which caused #NPF. In this case, |
| 3894 | * GuestIntrBytes field of the VMCB on a VMEXIT will incorrectly |
| 3895 | * return 0 instead of the correct guest instruction bytes. |
| 3896 | * |
| 3897 | * This happens because CPU microcode reading instruction bytes |
| 3898 | * uses a special opcode which attempts to read data using CPL=0 |
| 3899 | * priviledges. The microcode reads CS:RIP and if it hits a SMAP |
| 3900 | * fault, it gives up and returns no instruction bytes. |
| 3901 | * |
| 3902 | * Detection: |
| 3903 | * We reach here in case CPU supports DecodeAssist, raised #NPF and |
| 3904 | * returned 0 in GuestIntrBytes field of the VMCB. |
| 3905 | * First, errata can only be triggered in case vCPU CR4.SMAP=1. |
| 3906 | * Second, if vCPU CR4.SMEP=1, errata could only be triggered |
| 3907 | * in case vCPU CPL==3 (Because otherwise guest would have triggered |
| 3908 | * a SMEP fault instead of #NPF). |
| 3909 | * Otherwise, vCPU CR4.SMEP=0, errata could be triggered by any vCPU CPL. |
| 3910 | * As most guests enable SMAP if they have also enabled SMEP, use above |
| 3911 | * logic in order to attempt minimize false-positive of detecting errata |
| 3912 | * while still preserving all cases semantic correctness. |
| 3913 | * |
| 3914 | * Workaround: |
| 3915 | * To determine what instruction the guest was executing, the hypervisor |
| 3916 | * will have to decode the instruction at the instruction pointer. |
Singh, Brijesh | 05d5a48 | 2019-02-15 17:24:12 +0000 | [diff] [blame] | 3917 | * |
| 3918 | * In non SEV guest, hypervisor will be able to read the guest |
| 3919 | * memory to decode the instruction pointer when insn_len is zero |
| 3920 | * so we return true to indicate that decoding is possible. |
| 3921 | * |
| 3922 | * But in the SEV guest, the guest memory is encrypted with the |
| 3923 | * guest specific key and hypervisor will not be able to decode the |
| 3924 | * instruction pointer so we will not able to workaround it. Lets |
| 3925 | * print the error and request to kill the guest. |
| 3926 | */ |
Liran Alon | 118154b | 2019-07-17 02:56:58 +0300 | [diff] [blame] | 3927 | if (smap && (!smep || is_user)) { |
Singh, Brijesh | 05d5a48 | 2019-02-15 17:24:12 +0000 | [diff] [blame] | 3928 | if (!sev_guest(vcpu->kvm)) |
| 3929 | return true; |
| 3930 | |
Liran Alon | 118154b | 2019-07-17 02:56:58 +0300 | [diff] [blame] | 3931 | pr_err_ratelimited("KVM: SEV Guest triggered AMD Erratum 1096\n"); |
Singh, Brijesh | 05d5a48 | 2019-02-15 17:24:12 +0000 | [diff] [blame] | 3932 | kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); |
| 3933 | } |
| 3934 | |
| 3935 | return false; |
| 3936 | } |
| 3937 | |
Liran Alon | 4b9852f | 2019-08-26 13:24:49 +0300 | [diff] [blame] | 3938 | static bool svm_apic_init_signal_blocked(struct kvm_vcpu *vcpu) |
| 3939 | { |
| 3940 | struct vcpu_svm *svm = to_svm(vcpu); |
| 3941 | |
| 3942 | /* |
| 3943 | * TODO: Last condition latch INIT signals on vCPU when |
| 3944 | * vCPU is in guest-mode and vmcb12 defines intercept on INIT. |
Paolo Bonzini | 33b2217 | 2020-04-17 10:24:18 -0400 | [diff] [blame] | 3945 | * To properly emulate the INIT intercept, |
| 3946 | * svm_check_nested_events() should call nested_svm_vmexit() |
| 3947 | * if an INIT signal is pending. |
Liran Alon | 4b9852f | 2019-08-26 13:24:49 +0300 | [diff] [blame] | 3948 | */ |
| 3949 | return !gif_set(svm) || |
| 3950 | (svm->vmcb->control.intercept & (1ULL << INTERCEPT_INIT)); |
| 3951 | } |
| 3952 | |
Joerg Roedel | eaf7826 | 2020-03-24 10:41:54 +0100 | [diff] [blame] | 3953 | static void svm_vm_destroy(struct kvm *kvm) |
| 3954 | { |
| 3955 | avic_vm_destroy(kvm); |
| 3956 | sev_vm_destroy(kvm); |
| 3957 | } |
| 3958 | |
| 3959 | static int svm_vm_init(struct kvm *kvm) |
| 3960 | { |
| 3961 | if (avic) { |
| 3962 | int ret = avic_vm_init(kvm); |
| 3963 | if (ret) |
| 3964 | return ret; |
| 3965 | } |
| 3966 | |
| 3967 | kvm_apicv_init(kvm, avic); |
| 3968 | return 0; |
| 3969 | } |
| 3970 | |
Sean Christopherson | 9c14ee2 | 2020-03-21 13:26:03 -0700 | [diff] [blame] | 3971 | static struct kvm_x86_ops svm_x86_ops __initdata = { |
Li RongQing | dd58f3c | 2020-02-23 16:13:12 +0800 | [diff] [blame] | 3972 | .hardware_unsetup = svm_hardware_teardown, |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 3973 | .hardware_enable = svm_hardware_enable, |
| 3974 | .hardware_disable = svm_hardware_disable, |
Avi Kivity | 774ead3 | 2007-12-26 13:57:04 +0200 | [diff] [blame] | 3975 | .cpu_has_accelerated_tpr = svm_cpu_has_accelerated_tpr, |
Tom Lendacky | bc226f0 | 2018-05-10 22:06:39 +0200 | [diff] [blame] | 3976 | .has_emulated_msr = svm_has_emulated_msr, |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 3977 | |
| 3978 | .vcpu_create = svm_create_vcpu, |
| 3979 | .vcpu_free = svm_free_vcpu, |
Avi Kivity | 04d2cc7 | 2007-09-10 18:10:54 +0300 | [diff] [blame] | 3980 | .vcpu_reset = svm_vcpu_reset, |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 3981 | |
Sean Christopherson | 562b6b0 | 2020-01-26 16:41:13 -0800 | [diff] [blame] | 3982 | .vm_size = sizeof(struct kvm_svm), |
Suravee Suthikulpanit | 4e19c36 | 2019-11-14 14:15:05 -0600 | [diff] [blame] | 3983 | .vm_init = svm_vm_init, |
Brijesh Singh | 1654efc | 2017-12-04 10:57:34 -0600 | [diff] [blame] | 3984 | .vm_destroy = svm_vm_destroy, |
Suravee Suthikulpanit | 44a95da | 2016-05-04 14:09:46 -0500 | [diff] [blame] | 3985 | |
Avi Kivity | 04d2cc7 | 2007-09-10 18:10:54 +0300 | [diff] [blame] | 3986 | .prepare_guest_switch = svm_prepare_guest_switch, |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 3987 | .vcpu_load = svm_vcpu_load, |
| 3988 | .vcpu_put = svm_vcpu_put, |
Suravee Suthikulpanit | 8221c13 | 2016-05-04 14:09:52 -0500 | [diff] [blame] | 3989 | .vcpu_blocking = svm_vcpu_blocking, |
| 3990 | .vcpu_unblocking = svm_vcpu_unblocking, |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 3991 | |
Paolo Bonzini | a96036b | 2015-11-10 11:55:36 +0100 | [diff] [blame] | 3992 | .update_bp_intercept = update_bp_intercept, |
Tom Lendacky | 801e459 | 2018-02-21 13:39:51 -0600 | [diff] [blame] | 3993 | .get_msr_feature = svm_get_msr_feature, |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 3994 | .get_msr = svm_get_msr, |
| 3995 | .set_msr = svm_set_msr, |
| 3996 | .get_segment_base = svm_get_segment_base, |
| 3997 | .get_segment = svm_get_segment, |
| 3998 | .set_segment = svm_set_segment, |
Izik Eidus | 2e4d265 | 2008-03-24 19:38:34 +0200 | [diff] [blame] | 3999 | .get_cpl = svm_get_cpl, |
Rusty Russell | 1747fb7 | 2007-09-06 01:21:32 +1000 | [diff] [blame] | 4000 | .get_cs_db_l_bits = kvm_get_cs_db_l_bits, |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 4001 | .set_cr0 = svm_set_cr0, |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 4002 | .set_cr4 = svm_set_cr4, |
| 4003 | .set_efer = svm_set_efer, |
| 4004 | .get_idt = svm_get_idt, |
| 4005 | .set_idt = svm_set_idt, |
| 4006 | .get_gdt = svm_get_gdt, |
| 4007 | .set_gdt = svm_set_gdt, |
Gleb Natapov | 020df07 | 2010-04-13 10:05:23 +0300 | [diff] [blame] | 4008 | .set_dr7 = svm_set_dr7, |
Paolo Bonzini | facb013 | 2014-02-21 10:32:27 +0100 | [diff] [blame] | 4009 | .sync_dirty_debug_regs = svm_sync_dirty_debug_regs, |
Avi Kivity | 6de4f3a | 2009-05-31 22:58:47 +0300 | [diff] [blame] | 4010 | .cache_reg = svm_cache_reg, |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 4011 | .get_rflags = svm_get_rflags, |
| 4012 | .set_rflags = svm_set_rflags, |
Huaitong Han | be94f6b | 2016-03-22 16:51:20 +0800 | [diff] [blame] | 4013 | |
Sean Christopherson | 7780938 | 2020-03-20 14:28:18 -0700 | [diff] [blame] | 4014 | .tlb_flush_all = svm_flush_tlb, |
Sean Christopherson | eeeb4f6 | 2020-03-20 14:28:20 -0700 | [diff] [blame] | 4015 | .tlb_flush_current = svm_flush_tlb, |
Junaid Shahid | faff875 | 2018-06-29 13:10:05 -0700 | [diff] [blame] | 4016 | .tlb_flush_gva = svm_flush_tlb_gva, |
Sean Christopherson | 72b3832 | 2020-03-20 14:28:13 -0700 | [diff] [blame] | 4017 | .tlb_flush_guest = svm_flush_tlb, |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 4018 | |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 4019 | .run = svm_vcpu_run, |
Avi Kivity | 04d2cc7 | 2007-09-10 18:10:54 +0300 | [diff] [blame] | 4020 | .handle_exit = handle_exit, |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 4021 | .skip_emulated_instruction = skip_emulated_instruction, |
Oliver Upton | 5ef8acb | 2020-02-07 02:36:07 -0800 | [diff] [blame] | 4022 | .update_emulated_instruction = NULL, |
Glauber Costa | 2809f5d | 2009-05-12 16:21:05 -0400 | [diff] [blame] | 4023 | .set_interrupt_shadow = svm_set_interrupt_shadow, |
| 4024 | .get_interrupt_shadow = svm_get_interrupt_shadow, |
Ingo Molnar | 102d832 | 2007-02-19 14:37:47 +0200 | [diff] [blame] | 4025 | .patch_hypercall = svm_patch_hypercall, |
Eddie Dong | 2a8067f | 2007-08-06 16:29:07 +0300 | [diff] [blame] | 4026 | .set_irq = svm_set_irq, |
Gleb Natapov | 95ba827313 | 2009-04-21 17:45:08 +0300 | [diff] [blame] | 4027 | .set_nmi = svm_inject_nmi, |
Avi Kivity | 298101d | 2007-11-25 13:41:11 +0200 | [diff] [blame] | 4028 | .queue_exception = svm_queue_exception, |
Avi Kivity | b463a6f | 2010-07-20 15:06:17 +0300 | [diff] [blame] | 4029 | .cancel_injection = svm_cancel_injection, |
Gleb Natapov | 7864612 | 2009-03-23 12:12:11 +0200 | [diff] [blame] | 4030 | .interrupt_allowed = svm_interrupt_allowed, |
Gleb Natapov | 95ba827313 | 2009-04-21 17:45:08 +0300 | [diff] [blame] | 4031 | .nmi_allowed = svm_nmi_allowed, |
Jan Kiszka | 3cfc309 | 2009-11-12 01:04:25 +0100 | [diff] [blame] | 4032 | .get_nmi_mask = svm_get_nmi_mask, |
| 4033 | .set_nmi_mask = svm_set_nmi_mask, |
Gleb Natapov | 95ba827313 | 2009-04-21 17:45:08 +0300 | [diff] [blame] | 4034 | .enable_nmi_window = enable_nmi_window, |
| 4035 | .enable_irq_window = enable_irq_window, |
| 4036 | .update_cr8_intercept = update_cr8_intercept, |
Jim Mattson | 8d860bb | 2018-05-09 16:56:05 -0400 | [diff] [blame] | 4037 | .set_virtual_apic_mode = svm_set_virtual_apic_mode, |
Andrey Smetanin | d62caab | 2015-11-10 15:36:33 +0300 | [diff] [blame] | 4038 | .refresh_apicv_exec_ctrl = svm_refresh_apicv_exec_ctrl, |
Suravee Suthikulpanit | ef8efd7 | 2019-11-14 14:15:10 -0600 | [diff] [blame] | 4039 | .check_apicv_inhibit_reasons = svm_check_apicv_inhibit_reasons, |
Suravee Suthikulpanit | 2de9d0c | 2019-11-14 14:15:11 -0600 | [diff] [blame] | 4040 | .pre_update_apicv_exec_ctrl = svm_pre_update_apicv_exec_ctrl, |
Yang Zhang | c7c9c56 | 2013-01-25 10:18:51 +0800 | [diff] [blame] | 4041 | .load_eoi_exitmap = svm_load_eoi_exitmap, |
Suravee Suthikulpanit | 44a95da | 2016-05-04 14:09:46 -0500 | [diff] [blame] | 4042 | .hwapic_irr_update = svm_hwapic_irr_update, |
| 4043 | .hwapic_isr_update = svm_hwapic_isr_update, |
Liran Alon | fa59cc0 | 2017-12-24 18:12:53 +0200 | [diff] [blame] | 4044 | .sync_pir_to_irr = kvm_lapic_find_highest_irr, |
Suravee Suthikulpanit | be8ca17 | 2016-05-04 14:09:49 -0500 | [diff] [blame] | 4045 | .apicv_post_state_restore = avic_post_state_restore, |
Izik Eidus | cbc9402 | 2007-10-25 00:29:55 +0200 | [diff] [blame] | 4046 | |
| 4047 | .set_tss_addr = svm_set_tss_addr, |
Sean Christopherson | 2ac52ab | 2018-03-20 12:17:19 -0700 | [diff] [blame] | 4048 | .set_identity_map_addr = svm_set_identity_map_addr, |
Sheng Yang | 67253af | 2008-04-25 10:20:22 +0800 | [diff] [blame] | 4049 | .get_tdp_level = get_npt_level, |
Sheng Yang | 4b12f0d | 2009-04-27 20:35:42 +0800 | [diff] [blame] | 4050 | .get_mt_mask = svm_get_mt_mask, |
Marcelo Tosatti | 229456f | 2009-06-17 09:22:14 -0300 | [diff] [blame] | 4051 | |
Avi Kivity | 586f960 | 2010-11-18 13:09:54 +0200 | [diff] [blame] | 4052 | .get_exit_info = svm_get_exit_info, |
Avi Kivity | 586f960 | 2010-11-18 13:09:54 +0200 | [diff] [blame] | 4053 | |
Sheng Yang | 0e85188 | 2009-12-18 16:48:46 +0800 | [diff] [blame] | 4054 | .cpuid_update = svm_cpuid_update, |
Sheng Yang | 4e47c7a | 2009-12-18 16:48:47 +0800 | [diff] [blame] | 4055 | |
Sheng Yang | f5f48ee | 2010-06-30 12:25:15 +0800 | [diff] [blame] | 4056 | .has_wbinvd_exit = svm_has_wbinvd_exit, |
Zachary Amsden | 99e3e30 | 2010-08-19 22:07:17 -1000 | [diff] [blame] | 4057 | |
Leonid Shatz | 326e742 | 2018-11-06 12:14:25 +0200 | [diff] [blame] | 4058 | .write_l1_tsc_offset = svm_write_l1_tsc_offset, |
Joerg Roedel | 1c97f0a | 2010-09-10 17:30:41 +0200 | [diff] [blame] | 4059 | |
Paolo Bonzini | 727a7e2 | 2020-03-05 03:52:50 -0500 | [diff] [blame] | 4060 | .load_mmu_pgd = svm_load_mmu_pgd, |
Joerg Roedel | 8a76d7f | 2011-04-04 12:39:27 +0200 | [diff] [blame] | 4061 | |
| 4062 | .check_intercept = svm_check_intercept, |
Sean Christopherson | 95b5a48 | 2019-04-19 22:50:59 -0700 | [diff] [blame] | 4063 | .handle_exit_irqoff = svm_handle_exit_irqoff, |
Radim Krčmář | ae97a3b | 2014-08-21 18:08:06 +0200 | [diff] [blame] | 4064 | |
Sean Christopherson | d264ee0 | 2018-08-27 15:21:12 -0700 | [diff] [blame] | 4065 | .request_immediate_exit = __kvm_request_immediate_exit, |
| 4066 | |
Radim Krčmář | ae97a3b | 2014-08-21 18:08:06 +0200 | [diff] [blame] | 4067 | .sched_in = svm_sched_in, |
Wei Huang | 25462f7 | 2015-06-19 15:45:05 +0200 | [diff] [blame] | 4068 | |
| 4069 | .pmu_ops = &amd_pmu_ops, |
Paolo Bonzini | 33b2217 | 2020-04-17 10:24:18 -0400 | [diff] [blame] | 4070 | .nested_ops = &svm_nested_ops, |
| 4071 | |
Suravee Suthikulpanit | 340d3bc | 2016-05-04 14:09:47 -0500 | [diff] [blame] | 4072 | .deliver_posted_interrupt = svm_deliver_avic_intr, |
Wanpeng Li | 17e433b | 2019-08-05 10:03:19 +0800 | [diff] [blame] | 4073 | .dy_apicv_has_pending_interrupt = svm_dy_apicv_has_pending_interrupt, |
Suravee Suthikulpanit | 411b44b | 2016-08-23 13:52:43 -0500 | [diff] [blame] | 4074 | .update_pi_irte = svm_update_pi_irte, |
Borislav Petkov | 74f1690 | 2017-03-26 23:51:24 +0200 | [diff] [blame] | 4075 | .setup_mce = svm_setup_mce, |
Ladi Prosek | 0234bf8 | 2017-10-11 16:54:40 +0200 | [diff] [blame] | 4076 | |
Ladi Prosek | 72d7b37 | 2017-10-11 16:54:41 +0200 | [diff] [blame] | 4077 | .smi_allowed = svm_smi_allowed, |
Ladi Prosek | 0234bf8 | 2017-10-11 16:54:40 +0200 | [diff] [blame] | 4078 | .pre_enter_smm = svm_pre_enter_smm, |
| 4079 | .pre_leave_smm = svm_pre_leave_smm, |
Ladi Prosek | cc3d967 | 2017-10-17 16:02:39 +0200 | [diff] [blame] | 4080 | .enable_smi_window = enable_smi_window, |
Brijesh Singh | 1654efc | 2017-12-04 10:57:34 -0600 | [diff] [blame] | 4081 | |
| 4082 | .mem_enc_op = svm_mem_enc_op, |
Brijesh Singh | 1e80fdc | 2017-12-04 10:57:38 -0600 | [diff] [blame] | 4083 | .mem_enc_reg_region = svm_register_enc_region, |
| 4084 | .mem_enc_unreg_region = svm_unregister_enc_region, |
Vitaly Kuznetsov | 57b119d | 2018-10-16 18:50:01 +0200 | [diff] [blame] | 4085 | |
Singh, Brijesh | 05d5a48 | 2019-02-15 17:24:12 +0000 | [diff] [blame] | 4086 | .need_emulation_on_page_fault = svm_need_emulation_on_page_fault, |
Liran Alon | 4b9852f | 2019-08-26 13:24:49 +0300 | [diff] [blame] | 4087 | |
| 4088 | .apic_init_signal_blocked = svm_apic_init_signal_blocked, |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 4089 | }; |
| 4090 | |
Sean Christopherson | d008dfd | 2020-03-21 13:25:56 -0700 | [diff] [blame] | 4091 | static struct kvm_x86_init_ops svm_init_ops __initdata = { |
| 4092 | .cpu_has_kvm_support = has_svm, |
| 4093 | .disabled_by_bios = is_disabled, |
| 4094 | .hardware_setup = svm_hardware_setup, |
| 4095 | .check_processor_compatibility = svm_check_processor_compat, |
| 4096 | |
| 4097 | .runtime_ops = &svm_x86_ops, |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 4098 | }; |
| 4099 | |
| 4100 | static int __init svm_init(void) |
| 4101 | { |
Sean Christopherson | d008dfd | 2020-03-21 13:25:56 -0700 | [diff] [blame] | 4102 | return kvm_init(&svm_init_ops, sizeof(struct vcpu_svm), |
Avi Kivity | 0ee75be | 2010-04-28 15:39:01 +0300 | [diff] [blame] | 4103 | __alignof__(struct vcpu_svm), THIS_MODULE); |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 4104 | } |
| 4105 | |
| 4106 | static void __exit svm_exit(void) |
| 4107 | { |
Zhang Xiantao | cb498ea | 2007-11-14 20:39:31 +0800 | [diff] [blame] | 4108 | kvm_exit(); |
Avi Kivity | 6aa8b73 | 2006-12-10 02:21:36 -0800 | [diff] [blame] | 4109 | } |
| 4110 | |
| 4111 | module_init(svm_init) |
| 4112 | module_exit(svm_exit) |