blob: 7e352e81df3165f4f3d2ad109abbd160901dbb68 [file] [log] [blame]
Thomas Gleixner20c8ccb2019-06-04 10:11:32 +02001// SPDX-License-Identifier: GPL-2.0-only
Avi Kivity6aa8b732006-12-10 02:21:36 -08002/*
3 * Kernel-based Virtual Machine driver for Linux
4 *
5 * AMD SVM support
6 *
7 * Copyright (C) 2006 Qumranet, Inc.
Nicolas Kaiser9611c182010-10-06 14:23:22 +02008 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
Avi Kivity6aa8b732006-12-10 02:21:36 -08009 *
10 * Authors:
11 * Yaniv Kamay <yaniv@qumranet.com>
12 * Avi Kivity <avi@qumranet.com>
Avi Kivity6aa8b732006-12-10 02:21:36 -080013 */
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -050014
15#define pr_fmt(fmt) "SVM: " fmt
16
Avi Kivityedf88412007-12-16 11:02:48 +020017#include <linux/kvm_host.h>
18
Eddie Dong85f455f2007-07-06 12:20:49 +030019#include "irq.h"
Zhang Xiantao1d737c82007-12-14 09:35:10 +080020#include "mmu.h"
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -030021#include "kvm_cache_regs.h"
Gleb Natapovfe4c7b12009-03-23 11:23:18 +020022#include "x86.h"
Julian Stecklina66f7b722012-12-05 15:26:19 +010023#include "cpuid.h"
Wei Huang25462f72015-06-19 15:45:05 +020024#include "pmu.h"
Avi Kivitye4956062007-06-28 14:15:57 -040025
Avi Kivity6aa8b732006-12-10 02:21:36 -080026#include <linux/module.h>
Josh Triplettae759542012-03-28 11:32:28 -070027#include <linux/mod_devicetable.h>
Ahmed S. Darwish9d8f5492007-02-19 14:37:46 +020028#include <linux/kernel.h>
Avi Kivity6aa8b732006-12-10 02:21:36 -080029#include <linux/vmalloc.h>
30#include <linux/highmem.h>
Alexey Dobriyane8edc6e2007-05-21 01:22:52 +040031#include <linux/sched.h>
Steven Rostedt (Red Hat)af658dc2015-04-29 14:36:05 -040032#include <linux/trace_events.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090033#include <linux/slab.h>
Suravee Suthikulpanit5881f732016-08-23 13:52:42 -050034#include <linux/amd-iommu.h>
35#include <linux/hashtable.h>
Josh Poimboeufc207aee2017-06-28 10:11:06 -050036#include <linux/frame.h>
Brijesh Singhe9df0942017-12-04 10:57:33 -060037#include <linux/psp-sev.h>
Brijesh Singh1654efc2017-12-04 10:57:34 -060038#include <linux/file.h>
Brijesh Singh89c50582017-12-04 10:57:35 -060039#include <linux/pagemap.h>
40#include <linux/swap.h>
Avi Kivity6aa8b732006-12-10 02:21:36 -080041
Suravee Suthikulpanit8221c132016-05-04 14:09:52 -050042#include <asm/apic.h>
Joerg Roedel1018faa2012-02-29 14:57:32 +010043#include <asm/perf_event.h>
Joerg Roedel67ec6602010-05-17 14:43:35 +020044#include <asm/tlbflush.h>
Avi Kivitye4956062007-06-28 14:15:57 -040045#include <asm/desc.h>
Paolo Bonzinifacb0132014-02-21 10:32:27 +010046#include <asm/debugreg.h>
Gleb Natapov631bc482010-10-14 11:22:52 +020047#include <asm/kvm_para.h>
Suravee Suthikulpanit411b44b2016-08-23 13:52:43 -050048#include <asm/irq_remapping.h>
Thomas Gleixner28a27752018-04-29 15:01:37 +020049#include <asm/spec-ctrl.h>
Avi Kivity6aa8b732006-12-10 02:21:36 -080050
Eduardo Habkost63d11422008-11-17 19:03:20 -020051#include <asm/virtext.h>
Marcelo Tosatti229456f2009-06-17 09:22:14 -030052#include "trace.h"
Eduardo Habkost63d11422008-11-17 19:03:20 -020053
Avi Kivity4ecac3f2008-05-13 13:23:38 +030054#define __ex(x) __kvm_handle_fault_on_reboot(x)
55
Avi Kivity6aa8b732006-12-10 02:21:36 -080056MODULE_AUTHOR("Qumranet");
57MODULE_LICENSE("GPL");
58
Josh Triplettae759542012-03-28 11:32:28 -070059static const struct x86_cpu_id svm_cpu_id[] = {
60 X86_FEATURE_MATCH(X86_FEATURE_SVM),
61 {}
62};
63MODULE_DEVICE_TABLE(x86cpu, svm_cpu_id);
64
Avi Kivity6aa8b732006-12-10 02:21:36 -080065#define IOPM_ALLOC_ORDER 2
66#define MSRPM_ALLOC_ORDER 1
67
Avi Kivity6aa8b732006-12-10 02:21:36 -080068#define SEG_TYPE_LDT 2
69#define SEG_TYPE_BUSY_TSS16 3
70
Andre Przywara6bc31bd2010-04-11 23:07:28 +020071#define SVM_FEATURE_LBRV (1 << 1)
72#define SVM_FEATURE_SVML (1 << 2)
Andre Przywaraddce97a2010-12-21 11:12:03 +010073#define SVM_FEATURE_TSC_RATE (1 << 4)
74#define SVM_FEATURE_VMCB_CLEAN (1 << 5)
75#define SVM_FEATURE_FLUSH_ASID (1 << 6)
76#define SVM_FEATURE_DECODE_ASSIST (1 << 7)
Andre Przywara6bc31bd2010-04-11 23:07:28 +020077#define SVM_FEATURE_PAUSE_FILTER (1 << 10)
Joerg Roedel80b77062007-03-30 17:02:14 +030078
Suravee Suthikulpanit340d3bc2016-05-04 14:09:47 -050079#define SVM_AVIC_DOORBELL 0xc001011b
80
Joerg Roedel410e4d52009-08-07 11:49:44 +020081#define NESTED_EXIT_HOST 0 /* Exit handled on host level */
82#define NESTED_EXIT_DONE 1 /* Exit caused nested vmexit */
83#define NESTED_EXIT_CONTINUE 2 /* Further checks needed */
84
Joerg Roedel24e09cb2008-02-13 18:58:47 +010085#define DEBUGCTL_RESERVED_BITS (~(0x3fULL))
86
Joerg Roedelfbc0db72011-03-25 09:44:46 +010087#define TSC_RATIO_RSVD 0xffffff0000000000ULL
Joerg Roedel92a1f122011-03-25 09:44:51 +010088#define TSC_RATIO_MIN 0x0000000000000001ULL
89#define TSC_RATIO_MAX 0x000000ffffffffffULL
Joerg Roedelfbc0db72011-03-25 09:44:46 +010090
Dan Carpenter5446a972016-05-23 13:20:10 +030091#define AVIC_HPA_MASK ~((0xFFFULL << 52) | 0xFFF)
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -050092
93/*
94 * 0xff is broadcast, so the max index allowed for physical APIC ID
95 * table is 0xfe. APIC IDs above 0xff are reserved.
96 */
97#define AVIC_MAX_PHYSICAL_ID_COUNT 255
98
Suravee Suthikulpanit18f40c52016-05-04 14:09:48 -050099#define AVIC_UNACCEL_ACCESS_WRITE_MASK 1
100#define AVIC_UNACCEL_ACCESS_OFFSET_MASK 0xFF0
101#define AVIC_UNACCEL_ACCESS_VECTOR_MASK 0xFFFFFFFF
102
Suravee Suthikulpanit5ea11f22016-08-23 13:52:41 -0500103/* AVIC GATAG is encoded using VM and VCPU IDs */
104#define AVIC_VCPU_ID_BITS 8
105#define AVIC_VCPU_ID_MASK ((1 << AVIC_VCPU_ID_BITS) - 1)
106
107#define AVIC_VM_ID_BITS 24
108#define AVIC_VM_ID_NR (1 << AVIC_VM_ID_BITS)
109#define AVIC_VM_ID_MASK ((1 << AVIC_VM_ID_BITS) - 1)
110
111#define AVIC_GATAG(x, y) (((x & AVIC_VM_ID_MASK) << AVIC_VCPU_ID_BITS) | \
112 (y & AVIC_VCPU_ID_MASK))
113#define AVIC_GATAG_TO_VMID(x) ((x >> AVIC_VCPU_ID_BITS) & AVIC_VM_ID_MASK)
114#define AVIC_GATAG_TO_VCPUID(x) (x & AVIC_VCPU_ID_MASK)
115
Joerg Roedel67ec6602010-05-17 14:43:35 +0200116static bool erratum_383_found __read_mostly;
117
Avi Kivity6c8166a2009-05-31 18:15:37 +0300118static const u32 host_save_user_msrs[] = {
119#ifdef CONFIG_X86_64
120 MSR_STAR, MSR_LSTAR, MSR_CSTAR, MSR_SYSCALL_MASK, MSR_KERNEL_GS_BASE,
121 MSR_FS_BASE,
122#endif
123 MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
Paolo Bonzini46896c72015-11-12 14:49:16 +0100124 MSR_TSC_AUX,
Avi Kivity6c8166a2009-05-31 18:15:37 +0300125};
126
127#define NR_HOST_SAVE_USER_MSRS ARRAY_SIZE(host_save_user_msrs)
128
Sean Christopherson81811c12018-03-20 12:17:21 -0700129struct kvm_sev_info {
130 bool active; /* SEV enabled guest */
131 unsigned int asid; /* ASID used for this guest */
132 unsigned int handle; /* SEV firmware handle */
133 int fd; /* SEV device fd */
134 unsigned long pages_locked; /* Number of pages locked */
135 struct list_head regions_list; /* List of registered regions */
136};
137
138struct kvm_svm {
139 struct kvm kvm;
140
141 /* Struct members for AVIC */
142 u32 avic_vm_id;
Sean Christopherson81811c12018-03-20 12:17:21 -0700143 struct page *avic_logical_id_table_page;
144 struct page *avic_physical_id_table_page;
145 struct hlist_node hnode;
146
147 struct kvm_sev_info sev_info;
148};
149
Avi Kivity6c8166a2009-05-31 18:15:37 +0300150struct kvm_vcpu;
151
Joerg Roedele6aa9ab2009-08-07 11:49:33 +0200152struct nested_state {
153 struct vmcb *hsave;
154 u64 hsave_msr;
Joerg Roedel4a810182010-02-24 18:59:15 +0100155 u64 vm_cr_msr;
Joerg Roedele6aa9ab2009-08-07 11:49:33 +0200156 u64 vmcb;
157
158 /* These are the merged vectors */
159 u32 *msrpm;
160
161 /* gpa pointers to the real vectors */
162 u64 vmcb_msrpm;
Joerg Roedelce2ac082010-03-01 15:34:39 +0100163 u64 vmcb_iopm;
Joerg Roedelaad42c62009-08-07 11:49:34 +0200164
Joerg Roedelcd3ff652009-10-09 16:08:26 +0200165 /* A VMEXIT is required but not yet emulated */
166 bool exit_required;
167
Joerg Roedelaad42c62009-08-07 11:49:34 +0200168 /* cache for intercepts of the guest */
Roedel, Joerg4ee546b2010-12-03 10:50:51 +0100169 u32 intercept_cr;
Joerg Roedel3aed0412010-11-30 18:03:58 +0100170 u32 intercept_dr;
Joerg Roedelaad42c62009-08-07 11:49:34 +0200171 u32 intercept_exceptions;
172 u64 intercept;
173
Joerg Roedel5bd2edc2010-09-10 17:31:02 +0200174 /* Nested Paging related state */
175 u64 nested_cr3;
Joerg Roedele6aa9ab2009-08-07 11:49:33 +0200176};
177
Joerg Roedel323c3d82010-03-01 15:34:37 +0100178#define MSRPM_OFFSETS 16
179static u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly;
180
Boris Ostrovsky2b036c62012-01-09 14:00:35 -0500181/*
182 * Set osvw_len to higher value when updated Revision Guides
183 * are published and we know what the new status bits are
184 */
185static uint64_t osvw_len = 4, osvw_status;
186
Avi Kivity6c8166a2009-05-31 18:15:37 +0300187struct vcpu_svm {
188 struct kvm_vcpu vcpu;
189 struct vmcb *vmcb;
190 unsigned long vmcb_pa;
191 struct svm_cpu_data *svm_data;
192 uint64_t asid_generation;
193 uint64_t sysenter_esp;
194 uint64_t sysenter_eip;
Paolo Bonzini46896c72015-11-12 14:49:16 +0100195 uint64_t tsc_aux;
Avi Kivity6c8166a2009-05-31 18:15:37 +0300196
Tom Lendackyd1d93fa2018-02-24 00:18:20 +0100197 u64 msr_decfg;
198
Avi Kivity6c8166a2009-05-31 18:15:37 +0300199 u64 next_rip;
200
201 u64 host_user_msrs[NR_HOST_SAVE_USER_MSRS];
Avi Kivityafe9e662010-10-21 12:20:32 +0200202 struct {
Avi Kivitydacccfd2010-10-21 12:20:33 +0200203 u16 fs;
204 u16 gs;
205 u16 ldt;
Avi Kivityafe9e662010-10-21 12:20:32 +0200206 u64 gs_base;
207 } host;
Avi Kivity6c8166a2009-05-31 18:15:37 +0300208
KarimAllah Ahmedb2ac58f2018-02-03 15:56:23 +0100209 u64 spec_ctrl;
Thomas Gleixnerccbcd262018-05-09 23:01:01 +0200210 /*
211 * Contains guest-controlled bits of VIRT_SPEC_CTRL, which will be
212 * translated into the appropriate L2_CFG bits on the host to
213 * perform speculative control.
214 */
215 u64 virt_spec_ctrl;
KarimAllah Ahmedb2ac58f2018-02-03 15:56:23 +0100216
Avi Kivity6c8166a2009-05-31 18:15:37 +0300217 u32 *msrpm;
Avi Kivity6c8166a2009-05-31 18:15:37 +0300218
Avi Kivitybd3d1ec2011-02-03 15:29:52 +0200219 ulong nmi_iret_rip;
220
Joerg Roedele6aa9ab2009-08-07 11:49:33 +0200221 struct nested_state nested;
Jan Kiszka6be7d302009-10-18 13:24:54 +0200222
223 bool nmi_singlestep;
Ladi Prosekab2f4d732017-06-21 09:06:58 +0200224 u64 nmi_singlestep_guest_rflags;
Jan Kiszka66b71382010-02-23 17:47:56 +0100225
226 unsigned int3_injected;
227 unsigned long int3_rip;
Joerg Roedelfbc0db72011-03-25 09:44:46 +0100228
Joerg Roedel6092d3d2015-10-14 15:10:54 +0200229 /* cached guest cpuid flags for faster access */
230 bool nrips_enabled : 1;
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -0500231
Suravee Suthikulpanit18f40c52016-05-04 14:09:48 -0500232 u32 ldr_reg;
Suthikulpanit, Suravee98d90582019-01-29 08:08:42 +0000233 u32 dfr_reg;
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -0500234 struct page *avic_backing_page;
235 u64 *avic_physical_id_cache;
Suravee Suthikulpanit8221c132016-05-04 14:09:52 -0500236 bool avic_is_running;
Suravee Suthikulpanit411b44b2016-08-23 13:52:43 -0500237
238 /*
239 * Per-vcpu list of struct amd_svm_iommu_ir:
240 * This is used mainly to store interrupt remapping information used
241 * when update the vcpu affinity. This avoids the need to scan for
242 * IRTE and try to match ga_tag in the IOMMU driver.
243 */
244 struct list_head ir_list;
245 spinlock_t ir_list_lock;
Brijesh Singh70cd94e2017-12-04 10:57:34 -0600246
247 /* which host CPU was used for running this vcpu */
248 unsigned int last_cpu;
Suravee Suthikulpanit411b44b2016-08-23 13:52:43 -0500249};
250
251/*
252 * This is a wrapper of struct amd_iommu_ir_data.
253 */
254struct amd_svm_iommu_ir {
255 struct list_head node; /* Used by SVM for per-vcpu ir_list */
256 void *data; /* Storing pointer to struct amd_ir_data */
Avi Kivity6c8166a2009-05-31 18:15:37 +0300257};
258
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -0500259#define AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK (0xFF)
Suthikulpanit, Suraveee44e3ea2019-03-26 03:57:37 +0000260#define AVIC_LOGICAL_ID_ENTRY_VALID_BIT 31
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -0500261#define AVIC_LOGICAL_ID_ENTRY_VALID_MASK (1 << 31)
262
263#define AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK (0xFFULL)
264#define AVIC_PHYSICAL_ID_ENTRY_BACKING_PAGE_MASK (0xFFFFFFFFFFULL << 12)
265#define AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK (1ULL << 62)
266#define AVIC_PHYSICAL_ID_ENTRY_VALID_MASK (1ULL << 63)
267
Joerg Roedelfbc0db72011-03-25 09:44:46 +0100268static DEFINE_PER_CPU(u64, current_tsc_ratio);
269#define TSC_RATIO_DEFAULT 0x0100000000ULL
270
Joerg Roedel455716f2010-03-01 15:34:35 +0100271#define MSR_INVALID 0xffffffffU
272
Mathias Krause09941fb2012-08-30 01:30:20 +0200273static const struct svm_direct_access_msrs {
Joerg Roedelac72a9b2010-03-01 15:34:36 +0100274 u32 index; /* Index of the MSR */
275 bool always; /* True if intercept is always on */
276} direct_access_msrs[] = {
Brian Gerst8c065852010-07-17 09:03:26 -0400277 { .index = MSR_STAR, .always = true },
Joerg Roedelac72a9b2010-03-01 15:34:36 +0100278 { .index = MSR_IA32_SYSENTER_CS, .always = true },
279#ifdef CONFIG_X86_64
280 { .index = MSR_GS_BASE, .always = true },
281 { .index = MSR_FS_BASE, .always = true },
282 { .index = MSR_KERNEL_GS_BASE, .always = true },
283 { .index = MSR_LSTAR, .always = true },
284 { .index = MSR_CSTAR, .always = true },
285 { .index = MSR_SYSCALL_MASK, .always = true },
286#endif
KarimAllah Ahmedb2ac58f2018-02-03 15:56:23 +0100287 { .index = MSR_IA32_SPEC_CTRL, .always = false },
Ashok Raj15d45072018-02-01 22:59:43 +0100288 { .index = MSR_IA32_PRED_CMD, .always = false },
Joerg Roedelac72a9b2010-03-01 15:34:36 +0100289 { .index = MSR_IA32_LASTBRANCHFROMIP, .always = false },
290 { .index = MSR_IA32_LASTBRANCHTOIP, .always = false },
291 { .index = MSR_IA32_LASTINTFROMIP, .always = false },
292 { .index = MSR_IA32_LASTINTTOIP, .always = false },
293 { .index = MSR_INVALID, .always = false },
Avi Kivity6aa8b732006-12-10 02:21:36 -0800294};
295
296/* enable NPT for AMD64 and X86 with PAE */
297#if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
298static bool npt_enabled = true;
299#else
Joerg Roedele0231712010-02-24 18:59:10 +0100300static bool npt_enabled;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800301#endif
302
Babu Moger8566ac82018-03-16 16:37:26 -0400303/*
304 * These 2 parameters are used to config the controls for Pause-Loop Exiting:
305 * pause_filter_count: On processors that support Pause filtering(indicated
306 * by CPUID Fn8000_000A_EDX), the VMCB provides a 16 bit pause filter
307 * count value. On VMRUN this value is loaded into an internal counter.
308 * Each time a pause instruction is executed, this counter is decremented
309 * until it reaches zero at which time a #VMEXIT is generated if pause
310 * intercept is enabled. Refer to AMD APM Vol 2 Section 15.14.4 Pause
311 * Intercept Filtering for more details.
312 * This also indicate if ple logic enabled.
313 *
314 * pause_filter_thresh: In addition, some processor families support advanced
315 * pause filtering (indicated by CPUID Fn8000_000A_EDX) upper bound on
316 * the amount of time a guest is allowed to execute in a pause loop.
317 * In this mode, a 16-bit pause filter threshold field is added in the
318 * VMCB. The threshold value is a cycle count that is used to reset the
319 * pause counter. As with simple pause filtering, VMRUN loads the pause
320 * count value from VMCB into an internal counter. Then, on each pause
321 * instruction the hardware checks the elapsed number of cycles since
322 * the most recent pause instruction against the pause filter threshold.
323 * If the elapsed cycle count is greater than the pause filter threshold,
324 * then the internal pause count is reloaded from the VMCB and execution
325 * continues. If the elapsed cycle count is less than the pause filter
326 * threshold, then the internal pause count is decremented. If the count
327 * value is less than zero and PAUSE intercept is enabled, a #VMEXIT is
328 * triggered. If advanced pause filtering is supported and pause filter
329 * threshold field is set to zero, the filter will operate in the simpler,
330 * count only mode.
331 */
332
333static unsigned short pause_filter_thresh = KVM_DEFAULT_PLE_GAP;
334module_param(pause_filter_thresh, ushort, 0444);
335
336static unsigned short pause_filter_count = KVM_SVM_DEFAULT_PLE_WINDOW;
337module_param(pause_filter_count, ushort, 0444);
338
339/* Default doubles per-vcpu window every exit. */
340static unsigned short pause_filter_count_grow = KVM_DEFAULT_PLE_WINDOW_GROW;
341module_param(pause_filter_count_grow, ushort, 0444);
342
343/* Default resets per-vcpu window every exit to pause_filter_count. */
344static unsigned short pause_filter_count_shrink = KVM_DEFAULT_PLE_WINDOW_SHRINK;
345module_param(pause_filter_count_shrink, ushort, 0444);
346
347/* Default is to compute the maximum so we can never overflow. */
348static unsigned short pause_filter_count_max = KVM_SVM_DEFAULT_PLE_WINDOW_MAX;
349module_param(pause_filter_count_max, ushort, 0444);
350
Davidlohr Buesoe2358852012-01-17 14:09:50 +0100351/* allow nested paging (virtualized MMU) for all guests */
352static int npt = true;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800353module_param(npt, int, S_IRUGO);
354
Davidlohr Buesoe2358852012-01-17 14:09:50 +0100355/* allow nested virtualization in KVM/SVM */
356static int nested = true;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800357module_param(nested, int, S_IRUGO);
358
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -0500359/* enable / disable AVIC */
360static int avic;
Suravee Suthikulpanit5b8abf12016-06-15 17:24:36 -0500361#ifdef CONFIG_X86_LOCAL_APIC
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -0500362module_param(avic, int, S_IRUGO);
Suravee Suthikulpanit5b8abf12016-06-15 17:24:36 -0500363#endif
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -0500364
Paolo Bonzinid647eb62019-06-20 14:13:33 +0200365/* enable/disable Next RIP Save */
366static int nrips = true;
367module_param(nrips, int, 0444);
368
Janakarajan Natarajan89c8a492017-07-06 15:50:47 -0500369/* enable/disable Virtual VMLOAD VMSAVE */
370static int vls = true;
371module_param(vls, int, 0444);
372
Janakarajan Natarajan640bd6e2017-08-23 09:57:19 -0500373/* enable/disable Virtual GIF */
374static int vgif = true;
375module_param(vgif, int, 0444);
Suravee Suthikulpanit5ea11f22016-08-23 13:52:41 -0500376
Brijesh Singhe9df0942017-12-04 10:57:33 -0600377/* enable/disable SEV support */
378static int sev = IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT_ACTIVE_BY_DEFAULT);
379module_param(sev, int, 0444);
380
Paolo Bonzini6f2f8452019-05-20 15:34:35 +0200381static bool __read_mostly dump_invalid_vmcb = 0;
382module_param(dump_invalid_vmcb, bool, 0644);
383
Brijesh Singh7607b712018-02-19 10:14:44 -0600384static u8 rsm_ins_bytes[] = "\x0f\xaa";
385
Paolo Bonzini79a80592015-09-21 07:46:55 +0200386static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0);
Wanpeng Lic2ba05c2017-12-12 17:33:03 -0800387static void svm_flush_tlb(struct kvm_vcpu *vcpu, bool invalidate_gpa);
Joerg Roedela5c38322009-08-07 11:49:32 +0200388static void svm_complete_interrupts(struct vcpu_svm *svm);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800389
Joerg Roedel410e4d52009-08-07 11:49:44 +0200390static int nested_svm_exit_handled(struct vcpu_svm *svm);
Joerg Roedelb8e88bc2010-02-19 16:23:02 +0100391static int nested_svm_intercept(struct vcpu_svm *svm);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800392static int nested_svm_vmexit(struct vcpu_svm *svm);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800393static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
394 bool has_error_code, u32 error_code);
395
Roedel, Joerg8d28fec2010-12-03 13:15:21 +0100396enum {
Joerg Roedel116a0a22010-12-03 11:45:49 +0100397 VMCB_INTERCEPTS, /* Intercept vectors, TSC offset,
398 pause filter count */
Joerg Roedelf56838e2010-12-03 11:45:50 +0100399 VMCB_PERM_MAP, /* IOPM Base and MSRPM Base */
Joerg Roedeld48086d2010-12-03 11:45:51 +0100400 VMCB_ASID, /* ASID */
Joerg Roedeldecdbf62010-12-03 11:45:52 +0100401 VMCB_INTR, /* int_ctl, int_vector */
Joerg Roedelb2747162010-12-03 11:45:53 +0100402 VMCB_NPT, /* npt_en, nCR3, gPAT */
Joerg Roedeldcca1a62010-12-03 11:45:54 +0100403 VMCB_CR, /* CR0, CR3, CR4, EFER */
Joerg Roedel72214b92010-12-03 11:45:55 +0100404 VMCB_DR, /* DR6, DR7 */
Joerg Roedel17a703c2010-12-03 11:45:56 +0100405 VMCB_DT, /* GDT, IDT */
Joerg Roedel060d0c92010-12-03 11:45:57 +0100406 VMCB_SEG, /* CS, DS, SS, ES, CPL */
Joerg Roedel0574dec2010-12-03 11:45:58 +0100407 VMCB_CR2, /* CR2 only */
Joerg Roedelb53ba3f2010-12-03 11:45:59 +0100408 VMCB_LBR, /* DBGCTL, BR_FROM, BR_TO, LAST_EX_FROM, LAST_EX_TO */
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -0500409 VMCB_AVIC, /* AVIC APIC_BAR, AVIC APIC_BACKING_PAGE,
410 * AVIC PHYSICAL_TABLE pointer,
411 * AVIC LOGICAL_TABLE pointer
412 */
Roedel, Joerg8d28fec2010-12-03 13:15:21 +0100413 VMCB_DIRTY_MAX,
414};
415
Joerg Roedel0574dec2010-12-03 11:45:58 +0100416/* TPR and CR2 are always written before VMRUN */
417#define VMCB_ALWAYS_DIRTY_MASK ((1U << VMCB_INTR) | (1U << VMCB_CR2))
Roedel, Joerg8d28fec2010-12-03 13:15:21 +0100418
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -0500419#define VMCB_AVIC_APIC_BAR_MASK 0xFFFFFFFFFF000ULL
420
Brijesh Singhed3cd232017-12-04 10:57:32 -0600421static unsigned int max_sev_asid;
Brijesh Singh1654efc2017-12-04 10:57:34 -0600422static unsigned int min_sev_asid;
423static unsigned long *sev_asid_bitmap;
Brijesh Singh89c50582017-12-04 10:57:35 -0600424#define __sme_page_pa(x) __sme_set(page_to_pfn(x) << PAGE_SHIFT)
Brijesh Singh1654efc2017-12-04 10:57:34 -0600425
Brijesh Singh1e80fdc2017-12-04 10:57:38 -0600426struct enc_region {
427 struct list_head list;
428 unsigned long npages;
429 struct page **pages;
430 unsigned long uaddr;
431 unsigned long size;
432};
433
Sean Christopherson81811c12018-03-20 12:17:21 -0700434
435static inline struct kvm_svm *to_kvm_svm(struct kvm *kvm)
436{
437 return container_of(kvm, struct kvm_svm, kvm);
438}
439
Brijesh Singh1654efc2017-12-04 10:57:34 -0600440static inline bool svm_sev_enabled(void)
441{
Paolo Bonzini853c1102018-10-09 18:35:29 +0200442 return IS_ENABLED(CONFIG_KVM_AMD_SEV) ? max_sev_asid : 0;
Brijesh Singh1654efc2017-12-04 10:57:34 -0600443}
444
445static inline bool sev_guest(struct kvm *kvm)
446{
Paolo Bonzini853c1102018-10-09 18:35:29 +0200447#ifdef CONFIG_KVM_AMD_SEV
Sean Christopherson81811c12018-03-20 12:17:21 -0700448 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
Brijesh Singh1654efc2017-12-04 10:57:34 -0600449
450 return sev->active;
Paolo Bonzini853c1102018-10-09 18:35:29 +0200451#else
452 return false;
453#endif
Brijesh Singh1654efc2017-12-04 10:57:34 -0600454}
Brijesh Singhed3cd232017-12-04 10:57:32 -0600455
Brijesh Singh70cd94e2017-12-04 10:57:34 -0600456static inline int sev_get_asid(struct kvm *kvm)
457{
Sean Christopherson81811c12018-03-20 12:17:21 -0700458 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
Brijesh Singh70cd94e2017-12-04 10:57:34 -0600459
460 return sev->asid;
461}
462
Roedel, Joerg8d28fec2010-12-03 13:15:21 +0100463static inline void mark_all_dirty(struct vmcb *vmcb)
464{
465 vmcb->control.clean = 0;
466}
467
468static inline void mark_all_clean(struct vmcb *vmcb)
469{
470 vmcb->control.clean = ((1 << VMCB_DIRTY_MAX) - 1)
471 & ~VMCB_ALWAYS_DIRTY_MASK;
472}
473
474static inline void mark_dirty(struct vmcb *vmcb, int bit)
475{
476 vmcb->control.clean &= ~(1 << bit);
477}
478
Avi Kivity6aa8b732006-12-10 02:21:36 -0800479static inline struct vcpu_svm *to_svm(struct kvm_vcpu *vcpu)
480{
481 return container_of(vcpu, struct vcpu_svm, vcpu);
482}
483
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -0500484static inline void avic_update_vapic_bar(struct vcpu_svm *svm, u64 data)
485{
486 svm->vmcb->control.avic_vapic_bar = data & VMCB_AVIC_APIC_BAR_MASK;
487 mark_dirty(svm->vmcb, VMCB_AVIC);
488}
489
Suravee Suthikulpanit340d3bc2016-05-04 14:09:47 -0500490static inline bool avic_vcpu_is_running(struct kvm_vcpu *vcpu)
491{
492 struct vcpu_svm *svm = to_svm(vcpu);
493 u64 *entry = svm->avic_physical_id_cache;
494
495 if (!entry)
496 return false;
497
498 return (READ_ONCE(*entry) & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK);
499}
500
Joerg Roedel384c6362010-11-30 18:03:56 +0100501static void recalc_intercepts(struct vcpu_svm *svm)
502{
503 struct vmcb_control_area *c, *h;
504 struct nested_state *g;
505
Joerg Roedel116a0a22010-12-03 11:45:49 +0100506 mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
507
Joerg Roedel384c6362010-11-30 18:03:56 +0100508 if (!is_guest_mode(&svm->vcpu))
509 return;
510
511 c = &svm->vmcb->control;
512 h = &svm->nested.hsave->control;
513 g = &svm->nested;
514
Roedel, Joerg4ee546b2010-12-03 10:50:51 +0100515 c->intercept_cr = h->intercept_cr | g->intercept_cr;
Joerg Roedel3aed0412010-11-30 18:03:58 +0100516 c->intercept_dr = h->intercept_dr | g->intercept_dr;
Paolo Bonzinibd895252018-01-11 16:55:24 +0100517 c->intercept_exceptions = h->intercept_exceptions | g->intercept_exceptions;
Joerg Roedel384c6362010-11-30 18:03:56 +0100518 c->intercept = h->intercept | g->intercept;
519}
520
Roedel, Joerg4ee546b2010-12-03 10:50:51 +0100521static inline struct vmcb *get_host_vmcb(struct vcpu_svm *svm)
522{
523 if (is_guest_mode(&svm->vcpu))
524 return svm->nested.hsave;
525 else
526 return svm->vmcb;
527}
528
529static inline void set_cr_intercept(struct vcpu_svm *svm, int bit)
530{
531 struct vmcb *vmcb = get_host_vmcb(svm);
532
533 vmcb->control.intercept_cr |= (1U << bit);
534
535 recalc_intercepts(svm);
536}
537
538static inline void clr_cr_intercept(struct vcpu_svm *svm, int bit)
539{
540 struct vmcb *vmcb = get_host_vmcb(svm);
541
542 vmcb->control.intercept_cr &= ~(1U << bit);
543
544 recalc_intercepts(svm);
545}
546
547static inline bool is_cr_intercept(struct vcpu_svm *svm, int bit)
548{
549 struct vmcb *vmcb = get_host_vmcb(svm);
550
551 return vmcb->control.intercept_cr & (1U << bit);
552}
553
Paolo Bonzini5315c712014-03-03 13:08:29 +0100554static inline void set_dr_intercepts(struct vcpu_svm *svm)
Joerg Roedel3aed0412010-11-30 18:03:58 +0100555{
556 struct vmcb *vmcb = get_host_vmcb(svm);
557
Paolo Bonzini5315c712014-03-03 13:08:29 +0100558 vmcb->control.intercept_dr = (1 << INTERCEPT_DR0_READ)
559 | (1 << INTERCEPT_DR1_READ)
560 | (1 << INTERCEPT_DR2_READ)
561 | (1 << INTERCEPT_DR3_READ)
562 | (1 << INTERCEPT_DR4_READ)
563 | (1 << INTERCEPT_DR5_READ)
564 | (1 << INTERCEPT_DR6_READ)
565 | (1 << INTERCEPT_DR7_READ)
566 | (1 << INTERCEPT_DR0_WRITE)
567 | (1 << INTERCEPT_DR1_WRITE)
568 | (1 << INTERCEPT_DR2_WRITE)
569 | (1 << INTERCEPT_DR3_WRITE)
570 | (1 << INTERCEPT_DR4_WRITE)
571 | (1 << INTERCEPT_DR5_WRITE)
572 | (1 << INTERCEPT_DR6_WRITE)
573 | (1 << INTERCEPT_DR7_WRITE);
Joerg Roedel3aed0412010-11-30 18:03:58 +0100574
575 recalc_intercepts(svm);
576}
577
Paolo Bonzini5315c712014-03-03 13:08:29 +0100578static inline void clr_dr_intercepts(struct vcpu_svm *svm)
Joerg Roedel3aed0412010-11-30 18:03:58 +0100579{
580 struct vmcb *vmcb = get_host_vmcb(svm);
581
Paolo Bonzini5315c712014-03-03 13:08:29 +0100582 vmcb->control.intercept_dr = 0;
Joerg Roedel3aed0412010-11-30 18:03:58 +0100583
584 recalc_intercepts(svm);
585}
586
Joerg Roedel18c918c2010-11-30 18:03:59 +0100587static inline void set_exception_intercept(struct vcpu_svm *svm, int bit)
588{
589 struct vmcb *vmcb = get_host_vmcb(svm);
590
591 vmcb->control.intercept_exceptions |= (1U << bit);
592
593 recalc_intercepts(svm);
594}
595
596static inline void clr_exception_intercept(struct vcpu_svm *svm, int bit)
597{
598 struct vmcb *vmcb = get_host_vmcb(svm);
599
600 vmcb->control.intercept_exceptions &= ~(1U << bit);
601
602 recalc_intercepts(svm);
603}
604
Joerg Roedel8a05a1b82010-11-30 18:04:00 +0100605static inline void set_intercept(struct vcpu_svm *svm, int bit)
606{
607 struct vmcb *vmcb = get_host_vmcb(svm);
608
609 vmcb->control.intercept |= (1ULL << bit);
610
611 recalc_intercepts(svm);
612}
613
614static inline void clr_intercept(struct vcpu_svm *svm, int bit)
615{
616 struct vmcb *vmcb = get_host_vmcb(svm);
617
618 vmcb->control.intercept &= ~(1ULL << bit);
619
620 recalc_intercepts(svm);
621}
622
Janakarajan Natarajan640bd6e2017-08-23 09:57:19 -0500623static inline bool vgif_enabled(struct vcpu_svm *svm)
624{
625 return !!(svm->vmcb->control.int_ctl & V_GIF_ENABLE_MASK);
626}
627
Joerg Roedel2af91942009-08-07 11:49:28 +0200628static inline void enable_gif(struct vcpu_svm *svm)
629{
Janakarajan Natarajan640bd6e2017-08-23 09:57:19 -0500630 if (vgif_enabled(svm))
631 svm->vmcb->control.int_ctl |= V_GIF_MASK;
632 else
633 svm->vcpu.arch.hflags |= HF_GIF_MASK;
Joerg Roedel2af91942009-08-07 11:49:28 +0200634}
635
636static inline void disable_gif(struct vcpu_svm *svm)
637{
Janakarajan Natarajan640bd6e2017-08-23 09:57:19 -0500638 if (vgif_enabled(svm))
639 svm->vmcb->control.int_ctl &= ~V_GIF_MASK;
640 else
641 svm->vcpu.arch.hflags &= ~HF_GIF_MASK;
Joerg Roedel2af91942009-08-07 11:49:28 +0200642}
643
644static inline bool gif_set(struct vcpu_svm *svm)
645{
Janakarajan Natarajan640bd6e2017-08-23 09:57:19 -0500646 if (vgif_enabled(svm))
647 return !!(svm->vmcb->control.int_ctl & V_GIF_MASK);
648 else
649 return !!(svm->vcpu.arch.hflags & HF_GIF_MASK);
Joerg Roedel2af91942009-08-07 11:49:28 +0200650}
651
Avi Kivity6aa8b732006-12-10 02:21:36 -0800652static unsigned long iopm_base;
653
654struct kvm_ldttss_desc {
655 u16 limit0;
656 u16 base0;
Joerg Roedele0231712010-02-24 18:59:10 +0100657 unsigned base1:8, type:5, dpl:2, p:1;
658 unsigned limit1:4, zero0:3, g:1, base2:8;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800659 u32 base3;
660 u32 zero1;
661} __attribute__((packed));
662
663struct svm_cpu_data {
664 int cpu;
665
Avi Kivity5008fdf2007-04-02 13:05:50 +0300666 u64 asid_generation;
667 u32 max_asid;
668 u32 next_asid;
Brijesh Singh4faefff2017-12-04 10:57:25 -0600669 u32 min_asid;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800670 struct kvm_ldttss_desc *tss_desc;
671
672 struct page *save_area;
Ashok Raj15d45072018-02-01 22:59:43 +0100673 struct vmcb *current_vmcb;
Brijesh Singh70cd94e2017-12-04 10:57:34 -0600674
675 /* index = sev_asid, value = vmcb pointer */
676 struct vmcb **sev_vmcbs;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800677};
678
679static DEFINE_PER_CPU(struct svm_cpu_data *, svm_data);
680
Mathias Krause09941fb2012-08-30 01:30:20 +0200681static const u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000};
Avi Kivity6aa8b732006-12-10 02:21:36 -0800682
Ahmed S. Darwish9d8f5492007-02-19 14:37:46 +0200683#define NUM_MSR_MAPS ARRAY_SIZE(msrpm_ranges)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800684#define MSRS_RANGE_SIZE 2048
685#define MSRS_IN_RANGE (MSRS_RANGE_SIZE * 8 / 2)
686
Joerg Roedel455716f2010-03-01 15:34:35 +0100687static u32 svm_msrpm_offset(u32 msr)
688{
689 u32 offset;
690 int i;
691
692 for (i = 0; i < NUM_MSR_MAPS; i++) {
693 if (msr < msrpm_ranges[i] ||
694 msr >= msrpm_ranges[i] + MSRS_IN_RANGE)
695 continue;
696
697 offset = (msr - msrpm_ranges[i]) / 4; /* 4 msrs per u8 */
698 offset += (i * MSRS_RANGE_SIZE); /* add range offset */
699
700 /* Now we have the u8 offset - but need the u32 offset */
701 return offset / 4;
702 }
703
704 /* MSR not in any range */
705 return MSR_INVALID;
706}
707
Avi Kivity6aa8b732006-12-10 02:21:36 -0800708#define MAX_INST_SIZE 15
709
Avi Kivity6aa8b732006-12-10 02:21:36 -0800710static inline void clgi(void)
711{
Uros Bizjakac5ffda22018-11-26 17:00:08 +0100712 asm volatile (__ex("clgi"));
Avi Kivity6aa8b732006-12-10 02:21:36 -0800713}
714
715static inline void stgi(void)
716{
Uros Bizjakac5ffda22018-11-26 17:00:08 +0100717 asm volatile (__ex("stgi"));
Avi Kivity6aa8b732006-12-10 02:21:36 -0800718}
719
720static inline void invlpga(unsigned long addr, u32 asid)
721{
Uros Bizjakac5ffda22018-11-26 17:00:08 +0100722 asm volatile (__ex("invlpga %1, %0") : : "c"(asid), "a"(addr));
Avi Kivity6aa8b732006-12-10 02:21:36 -0800723}
724
Yu Zhang855feb62017-08-24 20:27:55 +0800725static int get_npt_level(struct kvm_vcpu *vcpu)
Joerg Roedel4b161842010-09-10 17:31:03 +0200726{
727#ifdef CONFIG_X86_64
Yu Zhang2a7266a2017-08-24 20:27:54 +0800728 return PT64_ROOT_4LEVEL;
Joerg Roedel4b161842010-09-10 17:31:03 +0200729#else
730 return PT32E_ROOT_LEVEL;
731#endif
732}
733
Avi Kivity6aa8b732006-12-10 02:21:36 -0800734static void svm_set_efer(struct kvm_vcpu *vcpu, u64 efer)
735{
Zachary Amsden6dc696d2010-05-26 15:09:43 -1000736 vcpu->arch.efer = efer;
Joerg Roedel709ddeb2008-02-07 13:47:45 +0100737 if (!npt_enabled && !(efer & EFER_LMA))
Carlo Marcelo Arenas Belon2b5203e2007-12-01 06:17:11 -0600738 efer &= ~EFER_LME;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800739
Alexander Graf9962d032008-11-25 20:17:02 +0100740 to_svm(vcpu)->vmcb->save.efer = efer | EFER_SVME;
Joerg Roedeldcca1a62010-12-03 11:45:54 +0100741 mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800742}
743
Avi Kivity6aa8b732006-12-10 02:21:36 -0800744static int is_external_interrupt(u32 info)
745{
746 info &= SVM_EVTINJ_TYPE_MASK | SVM_EVTINJ_VALID;
747 return info == (SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR);
748}
749
Paolo Bonzini37ccdcb2014-05-20 14:29:47 +0200750static u32 svm_get_interrupt_shadow(struct kvm_vcpu *vcpu)
Glauber Costa2809f5d2009-05-12 16:21:05 -0400751{
752 struct vcpu_svm *svm = to_svm(vcpu);
753 u32 ret = 0;
754
755 if (svm->vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK)
Paolo Bonzini37ccdcb2014-05-20 14:29:47 +0200756 ret = KVM_X86_SHADOW_INT_STI | KVM_X86_SHADOW_INT_MOV_SS;
757 return ret;
Glauber Costa2809f5d2009-05-12 16:21:05 -0400758}
759
760static void svm_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
761{
762 struct vcpu_svm *svm = to_svm(vcpu);
763
764 if (mask == 0)
765 svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK;
766 else
767 svm->vmcb->control.int_state |= SVM_INTERRUPT_SHADOW_MASK;
768
769}
770
Vitaly Kuznetsovf8ea7c62019-08-13 15:53:30 +0200771static int skip_emulated_instruction(struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800772{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400773 struct vcpu_svm *svm = to_svm(vcpu);
774
Paolo Bonzinid647eb62019-06-20 14:13:33 +0200775 if (nrips && svm->vmcb->control.next_rip != 0) {
Dirk Müllerd2922422015-10-01 13:43:42 +0200776 WARN_ON_ONCE(!static_cpu_has(X86_FEATURE_NRIPS));
Andre Przywara6bc31bd2010-04-11 23:07:28 +0200777 svm->next_rip = svm->vmcb->control.next_rip;
Bandan Dasf1047652015-06-11 02:05:33 -0400778 }
Andre Przywara6bc31bd2010-04-11 23:07:28 +0200779
Vitaly Kuznetsovf8ea7c62019-08-13 15:53:30 +0200780 if (!svm->next_rip)
781 return kvm_emulate_instruction(vcpu, EMULTYPE_SKIP);
782
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -0300783 if (svm->next_rip - kvm_rip_read(vcpu) > MAX_INST_SIZE)
784 printk(KERN_ERR "%s: ip 0x%lx next 0x%llx\n",
785 __func__, kvm_rip_read(vcpu), svm->next_rip);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800786
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -0300787 kvm_rip_write(vcpu, svm->next_rip);
Glauber Costa2809f5d2009-05-12 16:21:05 -0400788 svm_set_interrupt_shadow(vcpu, 0);
Vitaly Kuznetsovf8ea7c62019-08-13 15:53:30 +0200789
790 return EMULATE_DONE;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800791}
792
Wanpeng Licfcd20e2017-07-13 18:30:39 -0700793static void svm_queue_exception(struct kvm_vcpu *vcpu)
Jan Kiszka116a4752010-02-23 17:47:54 +0100794{
795 struct vcpu_svm *svm = to_svm(vcpu);
Wanpeng Licfcd20e2017-07-13 18:30:39 -0700796 unsigned nr = vcpu->arch.exception.nr;
797 bool has_error_code = vcpu->arch.exception.has_error_code;
Wanpeng Li664f8e22017-08-24 03:35:09 -0700798 bool reinject = vcpu->arch.exception.injected;
Wanpeng Licfcd20e2017-07-13 18:30:39 -0700799 u32 error_code = vcpu->arch.exception.error_code;
Jan Kiszka116a4752010-02-23 17:47:54 +0100800
Joerg Roedele0231712010-02-24 18:59:10 +0100801 /*
802 * If we are within a nested VM we'd better #VMEXIT and let the guest
803 * handle the exception
804 */
Joerg Roedelce7ddec2010-04-22 12:33:13 +0200805 if (!reinject &&
806 nested_svm_check_exception(svm, nr, has_error_code, error_code))
Jan Kiszka116a4752010-02-23 17:47:54 +0100807 return;
808
Jim Mattsonda998b42018-10-16 14:29:22 -0700809 kvm_deliver_exception_payload(&svm->vcpu);
810
Paolo Bonzinid647eb62019-06-20 14:13:33 +0200811 if (nr == BP_VECTOR && !nrips) {
Jan Kiszka66b71382010-02-23 17:47:56 +0100812 unsigned long rip, old_rip = kvm_rip_read(&svm->vcpu);
813
814 /*
815 * For guest debugging where we have to reinject #BP if some
816 * INT3 is guest-owned:
817 * Emulate nRIP by moving RIP forward. Will fail if injection
818 * raises a fault that is not intercepted. Still better than
819 * failing in all cases.
820 */
Vitaly Kuznetsovf8ea7c62019-08-13 15:53:30 +0200821 (void)skip_emulated_instruction(&svm->vcpu);
Jan Kiszka66b71382010-02-23 17:47:56 +0100822 rip = kvm_rip_read(&svm->vcpu);
823 svm->int3_rip = rip + svm->vmcb->save.cs.base;
824 svm->int3_injected = rip - old_rip;
825 }
826
Jan Kiszka116a4752010-02-23 17:47:54 +0100827 svm->vmcb->control.event_inj = nr
828 | SVM_EVTINJ_VALID
829 | (has_error_code ? SVM_EVTINJ_VALID_ERR : 0)
830 | SVM_EVTINJ_TYPE_EXEPT;
831 svm->vmcb->control.event_inj_err = error_code;
832}
833
Joerg Roedel67ec6602010-05-17 14:43:35 +0200834static void svm_init_erratum_383(void)
835{
836 u32 low, high;
837 int err;
838 u64 val;
839
Borislav Petkove6ee94d2013-03-20 15:07:27 +0100840 if (!static_cpu_has_bug(X86_BUG_AMD_TLB_MMATCH))
Joerg Roedel67ec6602010-05-17 14:43:35 +0200841 return;
842
843 /* Use _safe variants to not break nested virtualization */
844 val = native_read_msr_safe(MSR_AMD64_DC_CFG, &err);
845 if (err)
846 return;
847
848 val |= (1ULL << 47);
849
850 low = lower_32_bits(val);
851 high = upper_32_bits(val);
852
853 native_write_msr_safe(MSR_AMD64_DC_CFG, low, high);
854
855 erratum_383_found = true;
856}
857
Boris Ostrovsky2b036c62012-01-09 14:00:35 -0500858static void svm_init_osvw(struct kvm_vcpu *vcpu)
859{
860 /*
861 * Guests should see errata 400 and 415 as fixed (assuming that
862 * HLT and IO instructions are intercepted).
863 */
864 vcpu->arch.osvw.length = (osvw_len >= 3) ? (osvw_len) : 3;
865 vcpu->arch.osvw.status = osvw_status & ~(6ULL);
866
867 /*
868 * By increasing VCPU's osvw.length to 3 we are telling the guest that
869 * all osvw.status bits inside that length, including bit 0 (which is
870 * reserved for erratum 298), are valid. However, if host processor's
871 * osvw_len is 0 then osvw_status[0] carries no information. We need to
872 * be conservative here and therefore we tell the guest that erratum 298
873 * is present (because we really don't know).
874 */
875 if (osvw_len == 0 && boot_cpu_data.x86 == 0x10)
876 vcpu->arch.osvw.status |= 1;
877}
878
Avi Kivity6aa8b732006-12-10 02:21:36 -0800879static int has_svm(void)
880{
Eduardo Habkost63d11422008-11-17 19:03:20 -0200881 const char *msg;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800882
Eduardo Habkost63d11422008-11-17 19:03:20 -0200883 if (!cpu_has_svm(&msg)) {
Joe Perchesff81ff12009-01-08 11:05:17 -0800884 printk(KERN_INFO "has_svm: %s\n", msg);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800885 return 0;
886 }
887
Avi Kivity6aa8b732006-12-10 02:21:36 -0800888 return 1;
889}
890
Radim Krčmář13a34e02014-08-28 15:13:03 +0200891static void svm_hardware_disable(void)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800892{
Joerg Roedelfbc0db72011-03-25 09:44:46 +0100893 /* Make sure we clean up behind us */
894 if (static_cpu_has(X86_FEATURE_TSCRATEMSR))
895 wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT);
896
Eduardo Habkost2c8dcee2008-11-17 19:03:21 -0200897 cpu_svm_disable();
Joerg Roedel1018faa2012-02-29 14:57:32 +0100898
899 amd_pmu_disable_virt();
Avi Kivity6aa8b732006-12-10 02:21:36 -0800900}
901
Radim Krčmář13a34e02014-08-28 15:13:03 +0200902static int svm_hardware_enable(void)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800903{
904
Tejun Heo0fe1e002009-10-29 22:34:14 +0900905 struct svm_cpu_data *sd;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800906 uint64_t efer;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800907 struct desc_struct *gdt;
908 int me = raw_smp_processor_id();
909
Alexander Graf10474ae2009-09-15 11:37:46 +0200910 rdmsrl(MSR_EFER, efer);
911 if (efer & EFER_SVME)
912 return -EBUSY;
913
Avi Kivity6aa8b732006-12-10 02:21:36 -0800914 if (!has_svm()) {
Borislav Petkov1f5b77f2012-10-20 20:20:04 +0200915 pr_err("%s: err EOPNOTSUPP on %d\n", __func__, me);
Alexander Graf10474ae2009-09-15 11:37:46 +0200916 return -EINVAL;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800917 }
Tejun Heo0fe1e002009-10-29 22:34:14 +0900918 sd = per_cpu(svm_data, me);
Tejun Heo0fe1e002009-10-29 22:34:14 +0900919 if (!sd) {
Borislav Petkov1f5b77f2012-10-20 20:20:04 +0200920 pr_err("%s: svm_data is NULL on %d\n", __func__, me);
Alexander Graf10474ae2009-09-15 11:37:46 +0200921 return -EINVAL;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800922 }
923
Tejun Heo0fe1e002009-10-29 22:34:14 +0900924 sd->asid_generation = 1;
925 sd->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1;
926 sd->next_asid = sd->max_asid + 1;
Brijesh Singhed3cd232017-12-04 10:57:32 -0600927 sd->min_asid = max_sev_asid + 1;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800928
Thomas Garnier45fc8752017-03-14 10:05:08 -0700929 gdt = get_current_gdt_rw();
Tejun Heo0fe1e002009-10-29 22:34:14 +0900930 sd->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800931
Alexander Graf9962d032008-11-25 20:17:02 +0100932 wrmsrl(MSR_EFER, efer | EFER_SVME);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800933
Linus Torvaldsd0316552009-12-14 09:58:24 -0800934 wrmsrl(MSR_VM_HSAVE_PA, page_to_pfn(sd->save_area) << PAGE_SHIFT);
Alexander Graf10474ae2009-09-15 11:37:46 +0200935
Joerg Roedelfbc0db72011-03-25 09:44:46 +0100936 if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) {
937 wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT);
Christoph Lameter89cbc762014-08-17 12:30:40 -0500938 __this_cpu_write(current_tsc_ratio, TSC_RATIO_DEFAULT);
Joerg Roedelfbc0db72011-03-25 09:44:46 +0100939 }
940
Boris Ostrovsky2b036c62012-01-09 14:00:35 -0500941
942 /*
943 * Get OSVW bits.
944 *
945 * Note that it is possible to have a system with mixed processor
946 * revisions and therefore different OSVW bits. If bits are not the same
947 * on different processors then choose the worst case (i.e. if erratum
948 * is present on one processor and not on another then assume that the
949 * erratum is present everywhere).
950 */
951 if (cpu_has(&boot_cpu_data, X86_FEATURE_OSVW)) {
952 uint64_t len, status = 0;
953 int err;
954
955 len = native_read_msr_safe(MSR_AMD64_OSVW_ID_LENGTH, &err);
956 if (!err)
957 status = native_read_msr_safe(MSR_AMD64_OSVW_STATUS,
958 &err);
959
960 if (err)
961 osvw_status = osvw_len = 0;
962 else {
963 if (len < osvw_len)
964 osvw_len = len;
965 osvw_status |= status;
966 osvw_status &= (1ULL << osvw_len) - 1;
967 }
968 } else
969 osvw_status = osvw_len = 0;
970
Joerg Roedel67ec6602010-05-17 14:43:35 +0200971 svm_init_erratum_383();
972
Joerg Roedel1018faa2012-02-29 14:57:32 +0100973 amd_pmu_enable_virt();
974
Alexander Graf10474ae2009-09-15 11:37:46 +0200975 return 0;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800976}
977
Joerg Roedel0da1db752008-07-02 16:02:11 +0200978static void svm_cpu_uninit(int cpu)
979{
Tejun Heo0fe1e002009-10-29 22:34:14 +0900980 struct svm_cpu_data *sd = per_cpu(svm_data, raw_smp_processor_id());
Joerg Roedel0da1db752008-07-02 16:02:11 +0200981
Tejun Heo0fe1e002009-10-29 22:34:14 +0900982 if (!sd)
Joerg Roedel0da1db752008-07-02 16:02:11 +0200983 return;
984
985 per_cpu(svm_data, raw_smp_processor_id()) = NULL;
Brijesh Singh70cd94e2017-12-04 10:57:34 -0600986 kfree(sd->sev_vmcbs);
Tejun Heo0fe1e002009-10-29 22:34:14 +0900987 __free_page(sd->save_area);
988 kfree(sd);
Joerg Roedel0da1db752008-07-02 16:02:11 +0200989}
990
Avi Kivity6aa8b732006-12-10 02:21:36 -0800991static int svm_cpu_init(int cpu)
992{
Tejun Heo0fe1e002009-10-29 22:34:14 +0900993 struct svm_cpu_data *sd;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800994 int r;
995
Tejun Heo0fe1e002009-10-29 22:34:14 +0900996 sd = kzalloc(sizeof(struct svm_cpu_data), GFP_KERNEL);
997 if (!sd)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800998 return -ENOMEM;
Tejun Heo0fe1e002009-10-29 22:34:14 +0900999 sd->cpu = cpu;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001000 r = -ENOMEM;
Brijesh Singh70cd94e2017-12-04 10:57:34 -06001001 sd->save_area = alloc_page(GFP_KERNEL);
Tejun Heo0fe1e002009-10-29 22:34:14 +09001002 if (!sd->save_area)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001003 goto err_1;
1004
Brijesh Singh70cd94e2017-12-04 10:57:34 -06001005 if (svm_sev_enabled()) {
1006 r = -ENOMEM;
Kees Cook6da2ec52018-06-12 13:55:00 -07001007 sd->sev_vmcbs = kmalloc_array(max_sev_asid + 1,
1008 sizeof(void *),
1009 GFP_KERNEL);
Brijesh Singh70cd94e2017-12-04 10:57:34 -06001010 if (!sd->sev_vmcbs)
1011 goto err_1;
1012 }
1013
Tejun Heo0fe1e002009-10-29 22:34:14 +09001014 per_cpu(svm_data, cpu) = sd;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001015
1016 return 0;
1017
1018err_1:
Tejun Heo0fe1e002009-10-29 22:34:14 +09001019 kfree(sd);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001020 return r;
1021
1022}
1023
Joerg Roedelac72a9b2010-03-01 15:34:36 +01001024static bool valid_msr_intercept(u32 index)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001025{
1026 int i;
1027
Joerg Roedelac72a9b2010-03-01 15:34:36 +01001028 for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++)
1029 if (direct_access_msrs[i].index == index)
1030 return true;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001031
Joerg Roedelac72a9b2010-03-01 15:34:36 +01001032 return false;
1033}
1034
KarimAllah Ahmedb2ac58f2018-02-03 15:56:23 +01001035static bool msr_write_intercepted(struct kvm_vcpu *vcpu, unsigned msr)
1036{
1037 u8 bit_write;
1038 unsigned long tmp;
1039 u32 offset;
1040 u32 *msrpm;
1041
1042 msrpm = is_guest_mode(vcpu) ? to_svm(vcpu)->nested.msrpm:
1043 to_svm(vcpu)->msrpm;
1044
1045 offset = svm_msrpm_offset(msr);
1046 bit_write = 2 * (msr & 0x0f) + 1;
1047 tmp = msrpm[offset];
1048
1049 BUG_ON(offset == MSR_INVALID);
1050
1051 return !!test_bit(bit_write, &tmp);
1052}
1053
Avi Kivity6aa8b732006-12-10 02:21:36 -08001054static void set_msr_interception(u32 *msrpm, unsigned msr,
1055 int read, int write)
1056{
Joerg Roedel455716f2010-03-01 15:34:35 +01001057 u8 bit_read, bit_write;
1058 unsigned long tmp;
1059 u32 offset;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001060
Joerg Roedelac72a9b2010-03-01 15:34:36 +01001061 /*
1062 * If this warning triggers extend the direct_access_msrs list at the
1063 * beginning of the file
1064 */
1065 WARN_ON(!valid_msr_intercept(msr));
1066
Joerg Roedel455716f2010-03-01 15:34:35 +01001067 offset = svm_msrpm_offset(msr);
1068 bit_read = 2 * (msr & 0x0f);
1069 bit_write = 2 * (msr & 0x0f) + 1;
1070 tmp = msrpm[offset];
Avi Kivity6aa8b732006-12-10 02:21:36 -08001071
Joerg Roedel455716f2010-03-01 15:34:35 +01001072 BUG_ON(offset == MSR_INVALID);
1073
1074 read ? clear_bit(bit_read, &tmp) : set_bit(bit_read, &tmp);
1075 write ? clear_bit(bit_write, &tmp) : set_bit(bit_write, &tmp);
1076
1077 msrpm[offset] = tmp;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001078}
1079
Joerg Roedelf65c2292008-02-13 18:58:46 +01001080static void svm_vcpu_init_msrpm(u32 *msrpm)
1081{
Joerg Roedelac72a9b2010-03-01 15:34:36 +01001082 int i;
1083
Joerg Roedelf65c2292008-02-13 18:58:46 +01001084 memset(msrpm, 0xff, PAGE_SIZE * (1 << MSRPM_ALLOC_ORDER));
1085
Joerg Roedelac72a9b2010-03-01 15:34:36 +01001086 for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
1087 if (!direct_access_msrs[i].always)
1088 continue;
1089
1090 set_msr_interception(msrpm, direct_access_msrs[i].index, 1, 1);
1091 }
Joerg Roedelf65c2292008-02-13 18:58:46 +01001092}
1093
Joerg Roedel323c3d82010-03-01 15:34:37 +01001094static void add_msr_offset(u32 offset)
1095{
1096 int i;
1097
1098 for (i = 0; i < MSRPM_OFFSETS; ++i) {
1099
1100 /* Offset already in list? */
1101 if (msrpm_offsets[i] == offset)
1102 return;
1103
1104 /* Slot used by another offset? */
1105 if (msrpm_offsets[i] != MSR_INVALID)
1106 continue;
1107
1108 /* Add offset to list */
1109 msrpm_offsets[i] = offset;
1110
1111 return;
1112 }
1113
1114 /*
1115 * If this BUG triggers the msrpm_offsets table has an overflow. Just
1116 * increase MSRPM_OFFSETS in this case.
1117 */
1118 BUG();
1119}
1120
1121static void init_msrpm_offsets(void)
1122{
1123 int i;
1124
1125 memset(msrpm_offsets, 0xff, sizeof(msrpm_offsets));
1126
1127 for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
1128 u32 offset;
1129
1130 offset = svm_msrpm_offset(direct_access_msrs[i].index);
1131 BUG_ON(offset == MSR_INVALID);
1132
1133 add_msr_offset(offset);
1134 }
Avi Kivity6aa8b732006-12-10 02:21:36 -08001135}
1136
Joerg Roedel24e09cb2008-02-13 18:58:47 +01001137static void svm_enable_lbrv(struct vcpu_svm *svm)
1138{
1139 u32 *msrpm = svm->msrpm;
1140
Janakarajan Natarajan0dc92112017-07-06 15:50:45 -05001141 svm->vmcb->control.virt_ext |= LBR_CTL_ENABLE_MASK;
Joerg Roedel24e09cb2008-02-13 18:58:47 +01001142 set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 1, 1);
1143 set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 1, 1);
1144 set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 1, 1);
1145 set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 1, 1);
1146}
1147
1148static void svm_disable_lbrv(struct vcpu_svm *svm)
1149{
1150 u32 *msrpm = svm->msrpm;
1151
Janakarajan Natarajan0dc92112017-07-06 15:50:45 -05001152 svm->vmcb->control.virt_ext &= ~LBR_CTL_ENABLE_MASK;
Joerg Roedel24e09cb2008-02-13 18:58:47 +01001153 set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 0, 0);
1154 set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 0, 0);
1155 set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 0, 0);
1156 set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 0, 0);
1157}
1158
Ladi Prosek4aebd0e2017-06-21 09:06:57 +02001159static void disable_nmi_singlestep(struct vcpu_svm *svm)
1160{
1161 svm->nmi_singlestep = false;
Janakarajan Natarajan640bd6e2017-08-23 09:57:19 -05001162
Ladi Prosekab2f4d732017-06-21 09:06:58 +02001163 if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP)) {
1164 /* Clear our flags if they were not set by the guest */
1165 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF))
1166 svm->vmcb->save.rflags &= ~X86_EFLAGS_TF;
1167 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF))
1168 svm->vmcb->save.rflags &= ~X86_EFLAGS_RF;
1169 }
Ladi Prosek4aebd0e2017-06-21 09:06:57 +02001170}
1171
Suravee Suthikulpanit5881f732016-08-23 13:52:42 -05001172/* Note:
Sean Christopherson81811c12018-03-20 12:17:21 -07001173 * This hash table is used to map VM_ID to a struct kvm_svm,
Suravee Suthikulpanit5881f732016-08-23 13:52:42 -05001174 * when handling AMD IOMMU GALOG notification to schedule in
1175 * a particular vCPU.
1176 */
1177#define SVM_VM_DATA_HASH_BITS 8
David Hildenbrand681bcea2017-01-24 22:21:16 +01001178static DEFINE_HASHTABLE(svm_vm_data_hash, SVM_VM_DATA_HASH_BITS);
Denys Vlasenko3f0d4db2017-08-11 22:11:58 +02001179static u32 next_vm_id = 0;
1180static bool next_vm_id_wrapped = 0;
David Hildenbrand681bcea2017-01-24 22:21:16 +01001181static DEFINE_SPINLOCK(svm_vm_data_hash_lock);
Suravee Suthikulpanit5881f732016-08-23 13:52:42 -05001182
1183/* Note:
1184 * This function is called from IOMMU driver to notify
1185 * SVM to schedule in a particular vCPU of a particular VM.
1186 */
1187static int avic_ga_log_notifier(u32 ga_tag)
1188{
1189 unsigned long flags;
Sean Christopherson81811c12018-03-20 12:17:21 -07001190 struct kvm_svm *kvm_svm;
Suravee Suthikulpanit5881f732016-08-23 13:52:42 -05001191 struct kvm_vcpu *vcpu = NULL;
1192 u32 vm_id = AVIC_GATAG_TO_VMID(ga_tag);
1193 u32 vcpu_id = AVIC_GATAG_TO_VCPUID(ga_tag);
1194
1195 pr_debug("SVM: %s: vm_id=%#x, vcpu_id=%#x\n", __func__, vm_id, vcpu_id);
1196
1197 spin_lock_irqsave(&svm_vm_data_hash_lock, flags);
Sean Christopherson81811c12018-03-20 12:17:21 -07001198 hash_for_each_possible(svm_vm_data_hash, kvm_svm, hnode, vm_id) {
1199 if (kvm_svm->avic_vm_id != vm_id)
Suravee Suthikulpanit5881f732016-08-23 13:52:42 -05001200 continue;
Sean Christopherson81811c12018-03-20 12:17:21 -07001201 vcpu = kvm_get_vcpu_by_id(&kvm_svm->kvm, vcpu_id);
Suravee Suthikulpanit5881f732016-08-23 13:52:42 -05001202 break;
1203 }
1204 spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags);
1205
Suravee Suthikulpanit5881f732016-08-23 13:52:42 -05001206 /* Note:
1207 * At this point, the IOMMU should have already set the pending
1208 * bit in the vAPIC backing page. So, we just need to schedule
1209 * in the vcpu.
1210 */
Paolo Bonzini1cf53582017-10-10 12:51:56 +02001211 if (vcpu)
Suravee Suthikulpanit5881f732016-08-23 13:52:42 -05001212 kvm_vcpu_wake_up(vcpu);
1213
1214 return 0;
1215}
1216
Brijesh Singhe9df0942017-12-04 10:57:33 -06001217static __init int sev_hardware_setup(void)
1218{
1219 struct sev_user_data_status *status;
1220 int rc;
1221
1222 /* Maximum number of encrypted guests supported simultaneously */
1223 max_sev_asid = cpuid_ecx(0x8000001F);
1224
1225 if (!max_sev_asid)
1226 return 1;
1227
Brijesh Singh1654efc2017-12-04 10:57:34 -06001228 /* Minimum ASID value that should be used for SEV guest */
1229 min_sev_asid = cpuid_edx(0x8000001F);
1230
1231 /* Initialize SEV ASID bitmap */
Andy Shevchenkoa101c9d632018-08-30 14:49:59 +03001232 sev_asid_bitmap = bitmap_zalloc(max_sev_asid, GFP_KERNEL);
Brijesh Singh1654efc2017-12-04 10:57:34 -06001233 if (!sev_asid_bitmap)
1234 return 1;
1235
Brijesh Singhe9df0942017-12-04 10:57:33 -06001236 status = kmalloc(sizeof(*status), GFP_KERNEL);
1237 if (!status)
1238 return 1;
1239
1240 /*
1241 * Check SEV platform status.
1242 *
1243 * PLATFORM_STATUS can be called in any state, if we failed to query
1244 * the PLATFORM status then either PSP firmware does not support SEV
1245 * feature or SEV firmware is dead.
1246 */
1247 rc = sev_platform_status(status, NULL);
1248 if (rc)
1249 goto err;
1250
1251 pr_info("SEV supported\n");
1252
1253err:
1254 kfree(status);
1255 return rc;
1256}
1257
Babu Moger8566ac82018-03-16 16:37:26 -04001258static void grow_ple_window(struct kvm_vcpu *vcpu)
1259{
1260 struct vcpu_svm *svm = to_svm(vcpu);
1261 struct vmcb_control_area *control = &svm->vmcb->control;
1262 int old = control->pause_filter_count;
1263
1264 control->pause_filter_count = __grow_ple_window(old,
1265 pause_filter_count,
1266 pause_filter_count_grow,
1267 pause_filter_count_max);
1268
Peter Xu4f75bcc2019-09-06 10:17:22 +08001269 if (control->pause_filter_count != old) {
Babu Moger8566ac82018-03-16 16:37:26 -04001270 mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
Peter Xu4f75bcc2019-09-06 10:17:22 +08001271 trace_kvm_ple_window_update(vcpu->vcpu_id,
1272 control->pause_filter_count, old);
1273 }
Babu Moger8566ac82018-03-16 16:37:26 -04001274}
1275
1276static void shrink_ple_window(struct kvm_vcpu *vcpu)
1277{
1278 struct vcpu_svm *svm = to_svm(vcpu);
1279 struct vmcb_control_area *control = &svm->vmcb->control;
1280 int old = control->pause_filter_count;
1281
1282 control->pause_filter_count =
1283 __shrink_ple_window(old,
1284 pause_filter_count,
1285 pause_filter_count_shrink,
1286 pause_filter_count);
Peter Xu4f75bcc2019-09-06 10:17:22 +08001287 if (control->pause_filter_count != old) {
Babu Moger8566ac82018-03-16 16:37:26 -04001288 mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
Peter Xu4f75bcc2019-09-06 10:17:22 +08001289 trace_kvm_ple_window_update(vcpu->vcpu_id,
1290 control->pause_filter_count, old);
1291 }
Babu Moger8566ac82018-03-16 16:37:26 -04001292}
1293
Avi Kivity6aa8b732006-12-10 02:21:36 -08001294static __init int svm_hardware_setup(void)
1295{
1296 int cpu;
1297 struct page *iopm_pages;
Joerg Roedelf65c2292008-02-13 18:58:46 +01001298 void *iopm_va;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001299 int r;
1300
Avi Kivity6aa8b732006-12-10 02:21:36 -08001301 iopm_pages = alloc_pages(GFP_KERNEL, IOPM_ALLOC_ORDER);
1302
1303 if (!iopm_pages)
1304 return -ENOMEM;
Anthony Liguoric8681332007-04-30 09:48:11 +03001305
1306 iopm_va = page_address(iopm_pages);
1307 memset(iopm_va, 0xff, PAGE_SIZE * (1 << IOPM_ALLOC_ORDER));
Avi Kivity6aa8b732006-12-10 02:21:36 -08001308 iopm_base = page_to_pfn(iopm_pages) << PAGE_SHIFT;
1309
Joerg Roedel323c3d82010-03-01 15:34:37 +01001310 init_msrpm_offsets();
1311
Joerg Roedel50a37eb2008-01-31 14:57:38 +01001312 if (boot_cpu_has(X86_FEATURE_NX))
1313 kvm_enable_efer_bits(EFER_NX);
1314
Alexander Graf1b2fd702009-02-02 16:23:51 +01001315 if (boot_cpu_has(X86_FEATURE_FXSR_OPT))
1316 kvm_enable_efer_bits(EFER_FFXSR);
1317
Joerg Roedel92a1f122011-03-25 09:44:51 +01001318 if (boot_cpu_has(X86_FEATURE_TSCRATEMSR)) {
Joerg Roedel92a1f122011-03-25 09:44:51 +01001319 kvm_has_tsc_control = true;
Haozhong Zhangbc9b9612015-10-20 15:39:01 +08001320 kvm_max_tsc_scaling_ratio = TSC_RATIO_MAX;
1321 kvm_tsc_scaling_ratio_frac_bits = 32;
Joerg Roedel92a1f122011-03-25 09:44:51 +01001322 }
1323
Babu Moger8566ac82018-03-16 16:37:26 -04001324 /* Check for pause filtering support */
1325 if (!boot_cpu_has(X86_FEATURE_PAUSEFILTER)) {
1326 pause_filter_count = 0;
1327 pause_filter_thresh = 0;
1328 } else if (!boot_cpu_has(X86_FEATURE_PFTHRESHOLD)) {
1329 pause_filter_thresh = 0;
1330 }
1331
Alexander Graf236de052008-11-25 20:17:10 +01001332 if (nested) {
1333 printk(KERN_INFO "kvm: Nested Virtualization enabled\n");
Joerg Roedeleec4b142010-05-05 16:04:44 +02001334 kvm_enable_efer_bits(EFER_SVME | EFER_LMSLE);
Alexander Graf236de052008-11-25 20:17:10 +01001335 }
1336
Brijesh Singhe9df0942017-12-04 10:57:33 -06001337 if (sev) {
1338 if (boot_cpu_has(X86_FEATURE_SEV) &&
1339 IS_ENABLED(CONFIG_KVM_AMD_SEV)) {
1340 r = sev_hardware_setup();
1341 if (r)
1342 sev = false;
1343 } else {
1344 sev = false;
1345 }
1346 }
1347
Zachary Amsden3230bb42009-09-29 11:38:37 -10001348 for_each_possible_cpu(cpu) {
Avi Kivity6aa8b732006-12-10 02:21:36 -08001349 r = svm_cpu_init(cpu);
1350 if (r)
Joerg Roedelf65c2292008-02-13 18:58:46 +01001351 goto err;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001352 }
Joerg Roedel33bd6a02008-02-07 13:47:38 +01001353
Avi Kivity2a6b20b2010-11-09 16:15:42 +02001354 if (!boot_cpu_has(X86_FEATURE_NPT))
Joerg Roedele3da3ac2008-02-07 13:47:39 +01001355 npt_enabled = false;
1356
Joerg Roedel6c7dac72008-02-07 13:47:40 +01001357 if (npt_enabled && !npt) {
1358 printk(KERN_INFO "kvm: Nested Paging disabled\n");
1359 npt_enabled = false;
1360 }
1361
Joerg Roedel18552672008-02-07 13:47:41 +01001362 if (npt_enabled) {
Joerg Roedele3da3ac2008-02-07 13:47:39 +01001363 printk(KERN_INFO "kvm: Nested Paging enabled\n");
Joerg Roedel18552672008-02-07 13:47:41 +01001364 kvm_enable_tdp();
Joerg Roedel5f4cb662008-07-14 20:36:36 +02001365 } else
1366 kvm_disable_tdp();
Joerg Roedele3da3ac2008-02-07 13:47:39 +01001367
Paolo Bonzinid647eb62019-06-20 14:13:33 +02001368 if (nrips) {
1369 if (!boot_cpu_has(X86_FEATURE_NRIPS))
1370 nrips = false;
1371 }
1372
Suravee Suthikulpanit5b8abf12016-06-15 17:24:36 -05001373 if (avic) {
1374 if (!npt_enabled ||
1375 !boot_cpu_has(X86_FEATURE_AVIC) ||
Suravee Suthikulpanit5881f732016-08-23 13:52:42 -05001376 !IS_ENABLED(CONFIG_X86_LOCAL_APIC)) {
Suravee Suthikulpanit5b8abf12016-06-15 17:24:36 -05001377 avic = false;
Suravee Suthikulpanit5881f732016-08-23 13:52:42 -05001378 } else {
Suravee Suthikulpanit5b8abf12016-06-15 17:24:36 -05001379 pr_info("AVIC enabled\n");
Suravee Suthikulpanit5881f732016-08-23 13:52:42 -05001380
Suravee Suthikulpanit5881f732016-08-23 13:52:42 -05001381 amd_iommu_register_ga_log_notifier(&avic_ga_log_notifier);
1382 }
Suravee Suthikulpanit5b8abf12016-06-15 17:24:36 -05001383 }
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -05001384
Janakarajan Natarajan89c8a492017-07-06 15:50:47 -05001385 if (vls) {
1386 if (!npt_enabled ||
Borislav Petkov5442c262017-08-01 20:55:52 +02001387 !boot_cpu_has(X86_FEATURE_V_VMSAVE_VMLOAD) ||
Janakarajan Natarajan89c8a492017-07-06 15:50:47 -05001388 !IS_ENABLED(CONFIG_X86_64)) {
1389 vls = false;
1390 } else {
1391 pr_info("Virtual VMLOAD VMSAVE supported\n");
1392 }
1393 }
1394
Janakarajan Natarajan640bd6e2017-08-23 09:57:19 -05001395 if (vgif) {
1396 if (!boot_cpu_has(X86_FEATURE_VGIF))
1397 vgif = false;
1398 else
1399 pr_info("Virtual GIF supported\n");
1400 }
1401
Avi Kivity6aa8b732006-12-10 02:21:36 -08001402 return 0;
1403
Joerg Roedelf65c2292008-02-13 18:58:46 +01001404err:
Avi Kivity6aa8b732006-12-10 02:21:36 -08001405 __free_pages(iopm_pages, IOPM_ALLOC_ORDER);
1406 iopm_base = 0;
1407 return r;
1408}
1409
1410static __exit void svm_hardware_unsetup(void)
1411{
Joerg Roedel0da1db752008-07-02 16:02:11 +02001412 int cpu;
1413
Brijesh Singh1654efc2017-12-04 10:57:34 -06001414 if (svm_sev_enabled())
Andy Shevchenkoa101c9d632018-08-30 14:49:59 +03001415 bitmap_free(sev_asid_bitmap);
Brijesh Singh1654efc2017-12-04 10:57:34 -06001416
Zachary Amsden3230bb42009-09-29 11:38:37 -10001417 for_each_possible_cpu(cpu)
Joerg Roedel0da1db752008-07-02 16:02:11 +02001418 svm_cpu_uninit(cpu);
1419
Avi Kivity6aa8b732006-12-10 02:21:36 -08001420 __free_pages(pfn_to_page(iopm_base >> PAGE_SHIFT), IOPM_ALLOC_ORDER);
Joerg Roedelf65c2292008-02-13 18:58:46 +01001421 iopm_base = 0;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001422}
1423
1424static void init_seg(struct vmcb_seg *seg)
1425{
1426 seg->selector = 0;
1427 seg->attrib = SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK |
Joerg Roedele0231712010-02-24 18:59:10 +01001428 SVM_SELECTOR_WRITE_MASK; /* Read/Write Data Segment */
Avi Kivity6aa8b732006-12-10 02:21:36 -08001429 seg->limit = 0xffff;
1430 seg->base = 0;
1431}
1432
1433static void init_sys_seg(struct vmcb_seg *seg, uint32_t type)
1434{
1435 seg->selector = 0;
1436 seg->attrib = SVM_SELECTOR_P_MASK | type;
1437 seg->limit = 0xffff;
1438 seg->base = 0;
1439}
1440
KarimAllah Ahmede79f2452018-04-14 05:10:52 +02001441static u64 svm_read_l1_tsc_offset(struct kvm_vcpu *vcpu)
1442{
1443 struct vcpu_svm *svm = to_svm(vcpu);
1444
1445 if (is_guest_mode(vcpu))
1446 return svm->nested.hsave->control.tsc_offset;
1447
1448 return vcpu->arch.tsc_offset;
1449}
1450
Leonid Shatz326e7422018-11-06 12:14:25 +02001451static u64 svm_write_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
Zachary Amsdenf4e1b3c2010-08-19 22:07:16 -10001452{
1453 struct vcpu_svm *svm = to_svm(vcpu);
1454 u64 g_tsc_offset = 0;
1455
Joerg Roedel20307532010-11-29 17:51:48 +01001456 if (is_guest_mode(vcpu)) {
KarimAllah Ahmede79f2452018-04-14 05:10:52 +02001457 /* Write L1's TSC offset. */
Zachary Amsdenf4e1b3c2010-08-19 22:07:16 -10001458 g_tsc_offset = svm->vmcb->control.tsc_offset -
1459 svm->nested.hsave->control.tsc_offset;
1460 svm->nested.hsave->control.tsc_offset = offset;
Paolo Bonzini45c3af92018-11-25 18:45:35 +01001461 }
1462
1463 trace_kvm_write_tsc_offset(vcpu->vcpu_id,
1464 svm->vmcb->control.tsc_offset - g_tsc_offset,
1465 offset);
Zachary Amsdenf4e1b3c2010-08-19 22:07:16 -10001466
1467 svm->vmcb->control.tsc_offset = offset + g_tsc_offset;
Joerg Roedel116a0a22010-12-03 11:45:49 +01001468
1469 mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
Leonid Shatz326e7422018-11-06 12:14:25 +02001470 return svm->vmcb->control.tsc_offset;
Zachary Amsdenf4e1b3c2010-08-19 22:07:16 -10001471}
1472
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -05001473static void avic_init_vmcb(struct vcpu_svm *svm)
1474{
1475 struct vmcb *vmcb = svm->vmcb;
Sean Christopherson81811c12018-03-20 12:17:21 -07001476 struct kvm_svm *kvm_svm = to_kvm_svm(svm->vcpu.kvm);
Tom Lendackyd0ec49d2017-07-17 16:10:27 -05001477 phys_addr_t bpa = __sme_set(page_to_phys(svm->avic_backing_page));
Sean Christopherson81811c12018-03-20 12:17:21 -07001478 phys_addr_t lpa = __sme_set(page_to_phys(kvm_svm->avic_logical_id_table_page));
1479 phys_addr_t ppa = __sme_set(page_to_phys(kvm_svm->avic_physical_id_table_page));
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -05001480
1481 vmcb->control.avic_backing_page = bpa & AVIC_HPA_MASK;
1482 vmcb->control.avic_logical_id = lpa & AVIC_HPA_MASK;
1483 vmcb->control.avic_physical_id = ppa & AVIC_HPA_MASK;
1484 vmcb->control.avic_physical_id |= AVIC_MAX_PHYSICAL_ID_COUNT;
1485 vmcb->control.int_ctl |= AVIC_ENABLE_MASK;
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -05001486}
1487
Paolo Bonzini56908912015-10-19 11:30:19 +02001488static void init_vmcb(struct vcpu_svm *svm)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001489{
Joerg Roedele6101a92008-02-13 18:58:45 +01001490 struct vmcb_control_area *control = &svm->vmcb->control;
1491 struct vmcb_save_area *save = &svm->vmcb->save;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001492
Roedel, Joerg4ee546b2010-12-03 10:50:51 +01001493 svm->vcpu.arch.hflags = 0;
Avi Kivitybff78272010-01-07 13:16:08 +02001494
Roedel, Joerg4ee546b2010-12-03 10:50:51 +01001495 set_cr_intercept(svm, INTERCEPT_CR0_READ);
1496 set_cr_intercept(svm, INTERCEPT_CR3_READ);
1497 set_cr_intercept(svm, INTERCEPT_CR4_READ);
1498 set_cr_intercept(svm, INTERCEPT_CR0_WRITE);
1499 set_cr_intercept(svm, INTERCEPT_CR3_WRITE);
1500 set_cr_intercept(svm, INTERCEPT_CR4_WRITE);
Suravee Suthikulpanit3bbf3562016-05-04 14:09:51 -05001501 if (!kvm_vcpu_apicv_active(&svm->vcpu))
1502 set_cr_intercept(svm, INTERCEPT_CR8_WRITE);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001503
Paolo Bonzini5315c712014-03-03 13:08:29 +01001504 set_dr_intercepts(svm);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001505
Joerg Roedel18c918c2010-11-30 18:03:59 +01001506 set_exception_intercept(svm, PF_VECTOR);
1507 set_exception_intercept(svm, UD_VECTOR);
1508 set_exception_intercept(svm, MC_VECTOR);
Eric Northup54a20552015-11-03 18:03:53 +01001509 set_exception_intercept(svm, AC_VECTOR);
Paolo Bonzinicbdb9672015-11-10 09:14:39 +01001510 set_exception_intercept(svm, DB_VECTOR);
Liran Alon97184202018-03-12 13:12:52 +02001511 /*
1512 * Guest access to VMware backdoor ports could legitimately
1513 * trigger #GP because of TSS I/O permission bitmap.
1514 * We intercept those #GP and allow access to them anyway
1515 * as VMware does.
1516 */
1517 if (enable_vmware_backdoor)
1518 set_exception_intercept(svm, GP_VECTOR);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001519
Joerg Roedel8a05a1b82010-11-30 18:04:00 +01001520 set_intercept(svm, INTERCEPT_INTR);
1521 set_intercept(svm, INTERCEPT_NMI);
1522 set_intercept(svm, INTERCEPT_SMI);
1523 set_intercept(svm, INTERCEPT_SELECTIVE_CR0);
Avi Kivity332b56e2011-11-10 14:57:24 +02001524 set_intercept(svm, INTERCEPT_RDPMC);
Joerg Roedel8a05a1b82010-11-30 18:04:00 +01001525 set_intercept(svm, INTERCEPT_CPUID);
1526 set_intercept(svm, INTERCEPT_INVD);
Joerg Roedel8a05a1b82010-11-30 18:04:00 +01001527 set_intercept(svm, INTERCEPT_INVLPG);
1528 set_intercept(svm, INTERCEPT_INVLPGA);
1529 set_intercept(svm, INTERCEPT_IOIO_PROT);
1530 set_intercept(svm, INTERCEPT_MSR_PROT);
1531 set_intercept(svm, INTERCEPT_TASK_SWITCH);
1532 set_intercept(svm, INTERCEPT_SHUTDOWN);
1533 set_intercept(svm, INTERCEPT_VMRUN);
1534 set_intercept(svm, INTERCEPT_VMMCALL);
1535 set_intercept(svm, INTERCEPT_VMLOAD);
1536 set_intercept(svm, INTERCEPT_VMSAVE);
1537 set_intercept(svm, INTERCEPT_STGI);
1538 set_intercept(svm, INTERCEPT_CLGI);
1539 set_intercept(svm, INTERCEPT_SKINIT);
1540 set_intercept(svm, INTERCEPT_WBINVD);
Joerg Roedel81dd35d2010-12-07 17:15:06 +01001541 set_intercept(svm, INTERCEPT_XSETBV);
Brijesh Singh7607b712018-02-19 10:14:44 -06001542 set_intercept(svm, INTERCEPT_RSM);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001543
Wanpeng Li4d5422c2018-03-12 04:53:02 -07001544 if (!kvm_mwait_in_guest(svm->vcpu.kvm)) {
Michael S. Tsirkin668fffa2017-04-21 12:27:17 +02001545 set_intercept(svm, INTERCEPT_MONITOR);
1546 set_intercept(svm, INTERCEPT_MWAIT);
1547 }
1548
Wanpeng Licaa057a2018-03-12 04:53:03 -07001549 if (!kvm_hlt_in_guest(svm->vcpu.kvm))
1550 set_intercept(svm, INTERCEPT_HLT);
1551
Tom Lendackyd0ec49d2017-07-17 16:10:27 -05001552 control->iopm_base_pa = __sme_set(iopm_base);
1553 control->msrpm_base_pa = __sme_set(__pa(svm->msrpm));
Avi Kivity6aa8b732006-12-10 02:21:36 -08001554 control->int_ctl = V_INTR_MASKING_MASK;
1555
1556 init_seg(&save->es);
1557 init_seg(&save->ss);
1558 init_seg(&save->ds);
1559 init_seg(&save->fs);
1560 init_seg(&save->gs);
1561
1562 save->cs.selector = 0xf000;
Paolo Bonzini04b66832013-03-19 16:30:26 +01001563 save->cs.base = 0xffff0000;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001564 /* Executable/Readable Code Segment */
1565 save->cs.attrib = SVM_SELECTOR_READ_MASK | SVM_SELECTOR_P_MASK |
1566 SVM_SELECTOR_S_MASK | SVM_SELECTOR_CODE_MASK;
1567 save->cs.limit = 0xffff;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001568
1569 save->gdtr.limit = 0xffff;
1570 save->idtr.limit = 0xffff;
1571
1572 init_sys_seg(&save->ldtr, SEG_TYPE_LDT);
1573 init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16);
1574
Paolo Bonzini56908912015-10-19 11:30:19 +02001575 svm_set_efer(&svm->vcpu, 0);
Mike Dayd77c26f2007-10-08 09:02:08 -04001576 save->dr6 = 0xffff0ff0;
Avi Kivityf6e78472010-08-02 15:30:20 +03001577 kvm_set_rflags(&svm->vcpu, 2);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001578 save->rip = 0x0000fff0;
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -03001579 svm->vcpu.arch.regs[VCPU_REGS_RIP] = save->rip;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001580
Joerg Roedele0231712010-02-24 18:59:10 +01001581 /*
Eduardo Habkost18fa0002009-10-24 02:49:59 -02001582 * svm_set_cr0() sets PG and WP and clears NW and CD on save->cr0.
Nadav Amitd28bc9d2015-04-13 14:34:08 +03001583 * It also updates the guest-visible cr0 value.
Avi Kivity6aa8b732006-12-10 02:21:36 -08001584 */
Paolo Bonzini79a80592015-09-21 07:46:55 +02001585 svm_set_cr0(&svm->vcpu, X86_CR0_NW | X86_CR0_CD | X86_CR0_ET);
Igor Mammedovebae8712015-09-18 15:39:05 +02001586 kvm_mmu_reset_context(&svm->vcpu);
Eduardo Habkost18fa0002009-10-24 02:49:59 -02001587
Rusty Russell66aee912007-07-17 23:34:16 +10001588 save->cr4 = X86_CR4_PAE;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001589 /* rdx = ?? */
Joerg Roedel709ddeb2008-02-07 13:47:45 +01001590
1591 if (npt_enabled) {
1592 /* Setup VMCB for Nested Paging */
Tom Lendackycea3a192017-12-04 10:57:24 -06001593 control->nested_ctl |= SVM_NESTED_CTL_NP_ENABLE;
Joerg Roedel8a05a1b82010-11-30 18:04:00 +01001594 clr_intercept(svm, INTERCEPT_INVLPG);
Joerg Roedel18c918c2010-11-30 18:03:59 +01001595 clr_exception_intercept(svm, PF_VECTOR);
Roedel, Joerg4ee546b2010-12-03 10:50:51 +01001596 clr_cr_intercept(svm, INTERCEPT_CR3_READ);
1597 clr_cr_intercept(svm, INTERCEPT_CR3_WRITE);
Radim Krčmář74545702015-04-27 15:11:25 +02001598 save->g_pat = svm->vcpu.arch.pat;
Joerg Roedel709ddeb2008-02-07 13:47:45 +01001599 save->cr3 = 0;
1600 save->cr4 = 0;
1601 }
Joerg Roedelf40f6a42010-12-03 15:25:15 +01001602 svm->asid_generation = 0;
Alexander Graf1371d902008-11-25 20:17:04 +01001603
Joerg Roedele6aa9ab2009-08-07 11:49:33 +02001604 svm->nested.vmcb = 0;
Joerg Roedel2af91942009-08-07 11:49:28 +02001605 svm->vcpu.arch.hflags = 0;
1606
Babu Moger8566ac82018-03-16 16:37:26 -04001607 if (pause_filter_count) {
1608 control->pause_filter_count = pause_filter_count;
1609 if (pause_filter_thresh)
1610 control->pause_filter_thresh = pause_filter_thresh;
Joerg Roedel8a05a1b82010-11-30 18:04:00 +01001611 set_intercept(svm, INTERCEPT_PAUSE);
Babu Moger8566ac82018-03-16 16:37:26 -04001612 } else {
1613 clr_intercept(svm, INTERCEPT_PAUSE);
Mark Langsdorf565d0992009-10-06 14:25:02 -05001614 }
1615
Suravee Suthikulpanit67034bb2017-09-12 10:42:42 -05001616 if (kvm_vcpu_apicv_active(&svm->vcpu))
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -05001617 avic_init_vmcb(svm);
1618
Janakarajan Natarajan89c8a492017-07-06 15:50:47 -05001619 /*
1620 * If hardware supports Virtual VMLOAD VMSAVE then enable it
1621 * in VMCB and clear intercepts to avoid #VMEXIT.
1622 */
1623 if (vls) {
1624 clr_intercept(svm, INTERCEPT_VMLOAD);
1625 clr_intercept(svm, INTERCEPT_VMSAVE);
1626 svm->vmcb->control.virt_ext |= VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK;
1627 }
1628
Janakarajan Natarajan640bd6e2017-08-23 09:57:19 -05001629 if (vgif) {
1630 clr_intercept(svm, INTERCEPT_STGI);
1631 clr_intercept(svm, INTERCEPT_CLGI);
1632 svm->vmcb->control.int_ctl |= V_GIF_ENABLE_MASK;
1633 }
1634
Brijesh Singh35c6f6492017-12-04 10:57:39 -06001635 if (sev_guest(svm->vcpu.kvm)) {
Brijesh Singh1654efc2017-12-04 10:57:34 -06001636 svm->vmcb->control.nested_ctl |= SVM_NESTED_CTL_SEV_ENABLE;
Brijesh Singh35c6f6492017-12-04 10:57:39 -06001637 clr_exception_intercept(svm, UD_VECTOR);
1638 }
Brijesh Singh1654efc2017-12-04 10:57:34 -06001639
Roedel, Joerg8d28fec2010-12-03 13:15:21 +01001640 mark_all_dirty(svm->vmcb);
1641
Joerg Roedel2af91942009-08-07 11:49:28 +02001642 enable_gif(svm);
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -05001643
1644}
1645
Dan Carpenterd3e7dec2017-05-18 10:38:53 +03001646static u64 *avic_get_physical_id_entry(struct kvm_vcpu *vcpu,
1647 unsigned int index)
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -05001648{
1649 u64 *avic_physical_id_table;
Sean Christopherson81811c12018-03-20 12:17:21 -07001650 struct kvm_svm *kvm_svm = to_kvm_svm(vcpu->kvm);
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -05001651
1652 if (index >= AVIC_MAX_PHYSICAL_ID_COUNT)
1653 return NULL;
1654
Sean Christopherson81811c12018-03-20 12:17:21 -07001655 avic_physical_id_table = page_address(kvm_svm->avic_physical_id_table_page);
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -05001656
1657 return &avic_physical_id_table[index];
1658}
1659
1660/**
1661 * Note:
1662 * AVIC hardware walks the nested page table to check permissions,
1663 * but does not use the SPA address specified in the leaf page
1664 * table entry since it uses address in the AVIC_BACKING_PAGE pointer
1665 * field of the VMCB. Therefore, we set up the
1666 * APIC_ACCESS_PAGE_PRIVATE_MEMSLOT (4KB) here.
1667 */
1668static int avic_init_access_page(struct kvm_vcpu *vcpu)
1669{
1670 struct kvm *kvm = vcpu->kvm;
Wei Wang30510382018-11-12 12:23:14 +00001671 int ret = 0;
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -05001672
Wei Wang30510382018-11-12 12:23:14 +00001673 mutex_lock(&kvm->slots_lock);
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -05001674 if (kvm->arch.apic_access_page_done)
Wei Wang30510382018-11-12 12:23:14 +00001675 goto out;
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -05001676
Wei Wang30510382018-11-12 12:23:14 +00001677 ret = __x86_set_memory_region(kvm,
1678 APIC_ACCESS_PAGE_PRIVATE_MEMSLOT,
1679 APIC_DEFAULT_PHYS_BASE,
1680 PAGE_SIZE);
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -05001681 if (ret)
Wei Wang30510382018-11-12 12:23:14 +00001682 goto out;
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -05001683
1684 kvm->arch.apic_access_page_done = true;
Wei Wang30510382018-11-12 12:23:14 +00001685out:
1686 mutex_unlock(&kvm->slots_lock);
1687 return ret;
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -05001688}
1689
1690static int avic_init_backing_page(struct kvm_vcpu *vcpu)
1691{
1692 int ret;
1693 u64 *entry, new_entry;
1694 int id = vcpu->vcpu_id;
1695 struct vcpu_svm *svm = to_svm(vcpu);
1696
1697 ret = avic_init_access_page(vcpu);
1698 if (ret)
1699 return ret;
1700
1701 if (id >= AVIC_MAX_PHYSICAL_ID_COUNT)
1702 return -EINVAL;
1703
1704 if (!svm->vcpu.arch.apic->regs)
1705 return -EINVAL;
1706
1707 svm->avic_backing_page = virt_to_page(svm->vcpu.arch.apic->regs);
1708
1709 /* Setting AVIC backing page address in the phy APIC ID table */
1710 entry = avic_get_physical_id_entry(vcpu, id);
1711 if (!entry)
1712 return -EINVAL;
1713
Tom Lendackyd0ec49d2017-07-17 16:10:27 -05001714 new_entry = __sme_set((page_to_phys(svm->avic_backing_page) &
1715 AVIC_PHYSICAL_ID_ENTRY_BACKING_PAGE_MASK) |
1716 AVIC_PHYSICAL_ID_ENTRY_VALID_MASK);
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -05001717 WRITE_ONCE(*entry, new_entry);
1718
1719 svm->avic_physical_id_cache = entry;
1720
1721 return 0;
1722}
1723
Brijesh Singh1654efc2017-12-04 10:57:34 -06001724static void __sev_asid_free(int asid)
1725{
Brijesh Singh70cd94e2017-12-04 10:57:34 -06001726 struct svm_cpu_data *sd;
1727 int cpu, pos;
Brijesh Singh1654efc2017-12-04 10:57:34 -06001728
1729 pos = asid - 1;
1730 clear_bit(pos, sev_asid_bitmap);
Brijesh Singh70cd94e2017-12-04 10:57:34 -06001731
1732 for_each_possible_cpu(cpu) {
1733 sd = per_cpu(svm_data, cpu);
1734 sd->sev_vmcbs[pos] = NULL;
1735 }
Brijesh Singh1654efc2017-12-04 10:57:34 -06001736}
1737
1738static void sev_asid_free(struct kvm *kvm)
1739{
Sean Christopherson81811c12018-03-20 12:17:21 -07001740 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
Brijesh Singh1654efc2017-12-04 10:57:34 -06001741
1742 __sev_asid_free(sev->asid);
1743}
1744
Brijesh Singh59414c92017-12-04 10:57:35 -06001745static void sev_unbind_asid(struct kvm *kvm, unsigned int handle)
1746{
1747 struct sev_data_decommission *decommission;
1748 struct sev_data_deactivate *data;
1749
1750 if (!handle)
1751 return;
1752
1753 data = kzalloc(sizeof(*data), GFP_KERNEL);
1754 if (!data)
1755 return;
1756
1757 /* deactivate handle */
1758 data->handle = handle;
1759 sev_guest_deactivate(data, NULL);
1760
1761 wbinvd_on_all_cpus();
1762 sev_guest_df_flush(NULL);
1763 kfree(data);
1764
1765 decommission = kzalloc(sizeof(*decommission), GFP_KERNEL);
1766 if (!decommission)
1767 return;
1768
1769 /* decommission handle */
1770 decommission->handle = handle;
1771 sev_guest_decommission(decommission, NULL);
1772
1773 kfree(decommission);
1774}
1775
Brijesh Singh89c50582017-12-04 10:57:35 -06001776static struct page **sev_pin_memory(struct kvm *kvm, unsigned long uaddr,
1777 unsigned long ulen, unsigned long *n,
1778 int write)
1779{
Sean Christopherson81811c12018-03-20 12:17:21 -07001780 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
Brijesh Singh89c50582017-12-04 10:57:35 -06001781 unsigned long npages, npinned, size;
1782 unsigned long locked, lock_limit;
1783 struct page **pages;
Dan Carpenter86bf20c2018-05-19 09:01:36 +03001784 unsigned long first, last;
1785
1786 if (ulen == 0 || uaddr + ulen < uaddr)
1787 return NULL;
Brijesh Singh89c50582017-12-04 10:57:35 -06001788
1789 /* Calculate number of pages. */
1790 first = (uaddr & PAGE_MASK) >> PAGE_SHIFT;
1791 last = ((uaddr + ulen - 1) & PAGE_MASK) >> PAGE_SHIFT;
1792 npages = (last - first + 1);
1793
1794 locked = sev->pages_locked + npages;
1795 lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
1796 if (locked > lock_limit && !capable(CAP_IPC_LOCK)) {
1797 pr_err("SEV: %lu locked pages exceed the lock limit of %lu.\n", locked, lock_limit);
1798 return NULL;
1799 }
1800
1801 /* Avoid using vmalloc for smaller buffers. */
1802 size = npages * sizeof(struct page *);
1803 if (size > PAGE_SIZE)
Ben Gardon1ec69642019-02-11 11:02:51 -08001804 pages = __vmalloc(size, GFP_KERNEL_ACCOUNT | __GFP_ZERO,
1805 PAGE_KERNEL);
Brijesh Singh89c50582017-12-04 10:57:35 -06001806 else
Ben Gardon1ec69642019-02-11 11:02:51 -08001807 pages = kmalloc(size, GFP_KERNEL_ACCOUNT);
Brijesh Singh89c50582017-12-04 10:57:35 -06001808
1809 if (!pages)
1810 return NULL;
1811
1812 /* Pin the user virtual address. */
Ira Weiny73b01402019-05-13 17:17:11 -07001813 npinned = get_user_pages_fast(uaddr, npages, FOLL_WRITE, pages);
Brijesh Singh89c50582017-12-04 10:57:35 -06001814 if (npinned != npages) {
1815 pr_err("SEV: Failure locking %lu pages.\n", npages);
1816 goto err;
1817 }
1818
1819 *n = npages;
1820 sev->pages_locked = locked;
1821
1822 return pages;
1823
1824err:
1825 if (npinned > 0)
1826 release_pages(pages, npinned);
1827
1828 kvfree(pages);
1829 return NULL;
1830}
1831
1832static void sev_unpin_memory(struct kvm *kvm, struct page **pages,
1833 unsigned long npages)
1834{
Sean Christopherson81811c12018-03-20 12:17:21 -07001835 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
Brijesh Singh89c50582017-12-04 10:57:35 -06001836
1837 release_pages(pages, npages);
1838 kvfree(pages);
1839 sev->pages_locked -= npages;
1840}
1841
1842static void sev_clflush_pages(struct page *pages[], unsigned long npages)
1843{
1844 uint8_t *page_virtual;
1845 unsigned long i;
1846
1847 if (npages == 0 || pages == NULL)
1848 return;
1849
1850 for (i = 0; i < npages; i++) {
1851 page_virtual = kmap_atomic(pages[i]);
1852 clflush_cache_range(page_virtual, PAGE_SIZE);
1853 kunmap_atomic(page_virtual);
1854 }
1855}
1856
Brijesh Singh1e80fdc2017-12-04 10:57:38 -06001857static void __unregister_enc_region_locked(struct kvm *kvm,
1858 struct enc_region *region)
1859{
1860 /*
1861 * The guest may change the memory encryption attribute from C=0 -> C=1
1862 * or vice versa for this memory range. Lets make sure caches are
1863 * flushed to ensure that guest data gets written into memory with
1864 * correct C-bit.
1865 */
1866 sev_clflush_pages(region->pages, region->npages);
1867
1868 sev_unpin_memory(kvm, region->pages, region->npages);
1869 list_del(&region->list);
1870 kfree(region);
1871}
1872
Sean Christopherson434a1e92018-03-20 12:17:18 -07001873static struct kvm *svm_vm_alloc(void)
1874{
Ben Gardon1ec69642019-02-11 11:02:51 -08001875 struct kvm_svm *kvm_svm = __vmalloc(sizeof(struct kvm_svm),
1876 GFP_KERNEL_ACCOUNT | __GFP_ZERO,
1877 PAGE_KERNEL);
Sean Christopherson81811c12018-03-20 12:17:21 -07001878 return &kvm_svm->kvm;
Sean Christopherson434a1e92018-03-20 12:17:18 -07001879}
1880
1881static void svm_vm_free(struct kvm *kvm)
1882{
Marc Orrd1e5b0e2018-05-15 04:37:37 -07001883 vfree(to_kvm_svm(kvm));
Sean Christopherson434a1e92018-03-20 12:17:18 -07001884}
1885
Brijesh Singh1654efc2017-12-04 10:57:34 -06001886static void sev_vm_destroy(struct kvm *kvm)
1887{
Sean Christopherson81811c12018-03-20 12:17:21 -07001888 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
Brijesh Singh1e80fdc2017-12-04 10:57:38 -06001889 struct list_head *head = &sev->regions_list;
1890 struct list_head *pos, *q;
Brijesh Singh59414c92017-12-04 10:57:35 -06001891
Brijesh Singh1654efc2017-12-04 10:57:34 -06001892 if (!sev_guest(kvm))
1893 return;
1894
Brijesh Singh1e80fdc2017-12-04 10:57:38 -06001895 mutex_lock(&kvm->lock);
1896
1897 /*
1898 * if userspace was terminated before unregistering the memory regions
1899 * then lets unpin all the registered memory.
1900 */
1901 if (!list_empty(head)) {
1902 list_for_each_safe(pos, q, head) {
1903 __unregister_enc_region_locked(kvm,
1904 list_entry(pos, struct enc_region, list));
1905 }
1906 }
1907
1908 mutex_unlock(&kvm->lock);
1909
Brijesh Singh59414c92017-12-04 10:57:35 -06001910 sev_unbind_asid(kvm, sev->handle);
Brijesh Singh1654efc2017-12-04 10:57:34 -06001911 sev_asid_free(kvm);
1912}
1913
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -05001914static void avic_vm_destroy(struct kvm *kvm)
1915{
Suravee Suthikulpanit5881f732016-08-23 13:52:42 -05001916 unsigned long flags;
Sean Christopherson81811c12018-03-20 12:17:21 -07001917 struct kvm_svm *kvm_svm = to_kvm_svm(kvm);
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -05001918
Dmitry Vyukov3863dff2017-01-24 14:06:48 +01001919 if (!avic)
1920 return;
1921
Sean Christopherson81811c12018-03-20 12:17:21 -07001922 if (kvm_svm->avic_logical_id_table_page)
1923 __free_page(kvm_svm->avic_logical_id_table_page);
1924 if (kvm_svm->avic_physical_id_table_page)
1925 __free_page(kvm_svm->avic_physical_id_table_page);
Suravee Suthikulpanit5881f732016-08-23 13:52:42 -05001926
1927 spin_lock_irqsave(&svm_vm_data_hash_lock, flags);
Sean Christopherson81811c12018-03-20 12:17:21 -07001928 hash_del(&kvm_svm->hnode);
Suravee Suthikulpanit5881f732016-08-23 13:52:42 -05001929 spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags);
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -05001930}
1931
Brijesh Singh1654efc2017-12-04 10:57:34 -06001932static void svm_vm_destroy(struct kvm *kvm)
1933{
1934 avic_vm_destroy(kvm);
1935 sev_vm_destroy(kvm);
1936}
1937
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -05001938static int avic_vm_init(struct kvm *kvm)
1939{
Suravee Suthikulpanit5881f732016-08-23 13:52:42 -05001940 unsigned long flags;
Denys Vlasenko3f0d4db2017-08-11 22:11:58 +02001941 int err = -ENOMEM;
Sean Christopherson81811c12018-03-20 12:17:21 -07001942 struct kvm_svm *kvm_svm = to_kvm_svm(kvm);
1943 struct kvm_svm *k2;
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -05001944 struct page *p_page;
1945 struct page *l_page;
Denys Vlasenko3f0d4db2017-08-11 22:11:58 +02001946 u32 vm_id;
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -05001947
1948 if (!avic)
1949 return 0;
1950
1951 /* Allocating physical APIC ID table (4KB) */
Ben Gardon1ec69642019-02-11 11:02:51 -08001952 p_page = alloc_page(GFP_KERNEL_ACCOUNT);
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -05001953 if (!p_page)
1954 goto free_avic;
1955
Sean Christopherson81811c12018-03-20 12:17:21 -07001956 kvm_svm->avic_physical_id_table_page = p_page;
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -05001957 clear_page(page_address(p_page));
1958
1959 /* Allocating logical APIC ID table (4KB) */
Ben Gardon1ec69642019-02-11 11:02:51 -08001960 l_page = alloc_page(GFP_KERNEL_ACCOUNT);
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -05001961 if (!l_page)
1962 goto free_avic;
1963
Sean Christopherson81811c12018-03-20 12:17:21 -07001964 kvm_svm->avic_logical_id_table_page = l_page;
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -05001965 clear_page(page_address(l_page));
1966
Suravee Suthikulpanit5881f732016-08-23 13:52:42 -05001967 spin_lock_irqsave(&svm_vm_data_hash_lock, flags);
Denys Vlasenko3f0d4db2017-08-11 22:11:58 +02001968 again:
1969 vm_id = next_vm_id = (next_vm_id + 1) & AVIC_VM_ID_MASK;
1970 if (vm_id == 0) { /* id is 1-based, zero is not okay */
1971 next_vm_id_wrapped = 1;
1972 goto again;
1973 }
1974 /* Is it still in use? Only possible if wrapped at least once */
1975 if (next_vm_id_wrapped) {
Sean Christopherson81811c12018-03-20 12:17:21 -07001976 hash_for_each_possible(svm_vm_data_hash, k2, hnode, vm_id) {
1977 if (k2->avic_vm_id == vm_id)
Denys Vlasenko3f0d4db2017-08-11 22:11:58 +02001978 goto again;
1979 }
1980 }
Sean Christopherson81811c12018-03-20 12:17:21 -07001981 kvm_svm->avic_vm_id = vm_id;
1982 hash_add(svm_vm_data_hash, &kvm_svm->hnode, kvm_svm->avic_vm_id);
Suravee Suthikulpanit5881f732016-08-23 13:52:42 -05001983 spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags);
1984
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -05001985 return 0;
1986
1987free_avic:
1988 avic_vm_destroy(kvm);
1989 return err;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001990}
1991
Suravee Suthikulpanit411b44b2016-08-23 13:52:43 -05001992static inline int
1993avic_update_iommu_vcpu_affinity(struct kvm_vcpu *vcpu, int cpu, bool r)
Suravee Suthikulpanit8221c132016-05-04 14:09:52 -05001994{
Suravee Suthikulpanit411b44b2016-08-23 13:52:43 -05001995 int ret = 0;
1996 unsigned long flags;
1997 struct amd_svm_iommu_ir *ir;
Suravee Suthikulpanit8221c132016-05-04 14:09:52 -05001998 struct vcpu_svm *svm = to_svm(vcpu);
1999
Suravee Suthikulpanit411b44b2016-08-23 13:52:43 -05002000 if (!kvm_arch_has_assigned_device(vcpu->kvm))
2001 return 0;
Suravee Suthikulpanit8221c132016-05-04 14:09:52 -05002002
Suravee Suthikulpanit411b44b2016-08-23 13:52:43 -05002003 /*
2004 * Here, we go through the per-vcpu ir_list to update all existing
2005 * interrupt remapping table entry targeting this vcpu.
2006 */
2007 spin_lock_irqsave(&svm->ir_list_lock, flags);
Suravee Suthikulpanit8221c132016-05-04 14:09:52 -05002008
Suravee Suthikulpanit411b44b2016-08-23 13:52:43 -05002009 if (list_empty(&svm->ir_list))
2010 goto out;
Suravee Suthikulpanit8221c132016-05-04 14:09:52 -05002011
Suravee Suthikulpanit411b44b2016-08-23 13:52:43 -05002012 list_for_each_entry(ir, &svm->ir_list, node) {
2013 ret = amd_iommu_update_ga(cpu, r, ir->data);
2014 if (ret)
2015 break;
2016 }
2017out:
2018 spin_unlock_irqrestore(&svm->ir_list_lock, flags);
2019 return ret;
Suravee Suthikulpanit8221c132016-05-04 14:09:52 -05002020}
2021
2022static void avic_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
2023{
2024 u64 entry;
2025 /* ID = 0xff (broadcast), ID > 0xff (reserved) */
Suravee Suthikulpanit7d669f52016-06-15 17:23:45 -05002026 int h_physical_id = kvm_cpu_get_apicid(cpu);
Suravee Suthikulpanit8221c132016-05-04 14:09:52 -05002027 struct vcpu_svm *svm = to_svm(vcpu);
2028
2029 if (!kvm_vcpu_apicv_active(vcpu))
2030 return;
2031
Suthikulpanit, Suraveec9bcd3e2019-05-14 15:49:52 +00002032 /*
2033 * Since the host physical APIC id is 8 bits,
2034 * we can support host APIC ID upto 255.
2035 */
2036 if (WARN_ON(h_physical_id > AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK))
Suravee Suthikulpanit8221c132016-05-04 14:09:52 -05002037 return;
2038
2039 entry = READ_ONCE(*(svm->avic_physical_id_cache));
2040 WARN_ON(entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK);
2041
2042 entry &= ~AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK;
2043 entry |= (h_physical_id & AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK);
2044
2045 entry &= ~AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK;
2046 if (svm->avic_is_running)
2047 entry |= AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK;
2048
2049 WRITE_ONCE(*(svm->avic_physical_id_cache), entry);
Suravee Suthikulpanit411b44b2016-08-23 13:52:43 -05002050 avic_update_iommu_vcpu_affinity(vcpu, h_physical_id,
2051 svm->avic_is_running);
Suravee Suthikulpanit8221c132016-05-04 14:09:52 -05002052}
2053
2054static void avic_vcpu_put(struct kvm_vcpu *vcpu)
2055{
2056 u64 entry;
2057 struct vcpu_svm *svm = to_svm(vcpu);
2058
2059 if (!kvm_vcpu_apicv_active(vcpu))
2060 return;
2061
2062 entry = READ_ONCE(*(svm->avic_physical_id_cache));
Suravee Suthikulpanit411b44b2016-08-23 13:52:43 -05002063 if (entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK)
2064 avic_update_iommu_vcpu_affinity(vcpu, -1, 0);
2065
Suravee Suthikulpanit8221c132016-05-04 14:09:52 -05002066 entry &= ~AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK;
2067 WRITE_ONCE(*(svm->avic_physical_id_cache), entry);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002068}
2069
Suravee Suthikulpanit411b44b2016-08-23 13:52:43 -05002070/**
2071 * This function is called during VCPU halt/unhalt.
2072 */
2073static void avic_set_running(struct kvm_vcpu *vcpu, bool is_run)
2074{
2075 struct vcpu_svm *svm = to_svm(vcpu);
2076
2077 svm->avic_is_running = is_run;
2078 if (is_run)
2079 avic_vcpu_load(vcpu, vcpu->cpu);
2080 else
2081 avic_vcpu_put(vcpu);
2082}
2083
Nadav Amitd28bc9d2015-04-13 14:34:08 +03002084static void svm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
Avi Kivity04d2cc72007-09-10 18:10:54 +03002085{
2086 struct vcpu_svm *svm = to_svm(vcpu);
Julian Stecklina66f7b722012-12-05 15:26:19 +01002087 u32 dummy;
2088 u32 eax = 1;
Avi Kivity04d2cc72007-09-10 18:10:54 +03002089
Wanpeng Li518e7b92018-02-28 14:03:31 +08002090 vcpu->arch.microcode_version = 0x01000065;
KarimAllah Ahmedb2ac58f2018-02-03 15:56:23 +01002091 svm->spec_ctrl = 0;
Thomas Gleixnerccbcd262018-05-09 23:01:01 +02002092 svm->virt_spec_ctrl = 0;
KarimAllah Ahmedb2ac58f2018-02-03 15:56:23 +01002093
Nadav Amitd28bc9d2015-04-13 14:34:08 +03002094 if (!init_event) {
2095 svm->vcpu.arch.apic_base = APIC_DEFAULT_PHYS_BASE |
2096 MSR_IA32_APICBASE_ENABLE;
2097 if (kvm_vcpu_is_reset_bsp(&svm->vcpu))
2098 svm->vcpu.arch.apic_base |= MSR_IA32_APICBASE_BSP;
2099 }
Paolo Bonzini56908912015-10-19 11:30:19 +02002100 init_vmcb(svm);
Avi Kivity70433382007-11-07 12:57:23 +02002101
Yu Zhange911eb32017-08-24 20:27:52 +08002102 kvm_cpuid(vcpu, &eax, &dummy, &dummy, &dummy, true);
Sean Christophersonde3cd112019-04-30 10:36:17 -07002103 kvm_rdx_write(vcpu, eax);
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -05002104
2105 if (kvm_vcpu_apicv_active(vcpu) && !init_event)
2106 avic_update_vapic_bar(svm, APIC_DEFAULT_PHYS_BASE);
Avi Kivity04d2cc72007-09-10 18:10:54 +03002107}
2108
Suravee Suthikulpanitdfa20092017-09-12 10:42:40 -05002109static int avic_init_vcpu(struct vcpu_svm *svm)
2110{
2111 int ret;
2112
Suravee Suthikulpanit67034bb2017-09-12 10:42:42 -05002113 if (!kvm_vcpu_apicv_active(&svm->vcpu))
Suravee Suthikulpanitdfa20092017-09-12 10:42:40 -05002114 return 0;
2115
2116 ret = avic_init_backing_page(&svm->vcpu);
2117 if (ret)
2118 return ret;
2119
2120 INIT_LIST_HEAD(&svm->ir_list);
2121 spin_lock_init(&svm->ir_list_lock);
Suthikulpanit, Suravee98d90582019-01-29 08:08:42 +00002122 svm->dfr_reg = APIC_DFR_FLAT;
Suravee Suthikulpanitdfa20092017-09-12 10:42:40 -05002123
2124 return ret;
2125}
2126
Rusty Russellfb3f0f52007-07-27 17:16:56 +10002127static struct kvm_vcpu *svm_create_vcpu(struct kvm *kvm, unsigned int id)
Avi Kivity6aa8b732006-12-10 02:21:36 -08002128{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04002129 struct vcpu_svm *svm;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002130 struct page *page;
Joerg Roedelf65c2292008-02-13 18:58:46 +01002131 struct page *msrpm_pages;
Alexander Grafb286d5d2008-11-25 20:17:05 +01002132 struct page *hsave_page;
Alexander Graf3d6368e2008-11-25 20:17:07 +01002133 struct page *nested_msrpm_pages;
Rusty Russellfb3f0f52007-07-27 17:16:56 +10002134 int err;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002135
Sean Christopherson12b58f42019-08-15 10:22:37 -07002136 BUILD_BUG_ON_MSG(offsetof(struct vcpu_svm, vcpu) != 0,
2137 "struct kvm_vcpu must be at offset 0 for arch usercopy region");
2138
Ben Gardon1ec69642019-02-11 11:02:51 -08002139 svm = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL_ACCOUNT);
Rusty Russellfb3f0f52007-07-27 17:16:56 +10002140 if (!svm) {
2141 err = -ENOMEM;
2142 goto out;
2143 }
2144
Wanpeng Lid9a710e2019-07-22 12:26:21 +08002145 svm->vcpu.arch.user_fpu = kmem_cache_zalloc(x86_fpu_cache,
2146 GFP_KERNEL_ACCOUNT);
2147 if (!svm->vcpu.arch.user_fpu) {
2148 printk(KERN_ERR "kvm: failed to allocate kvm userspace's fpu\n");
2149 err = -ENOMEM;
2150 goto free_partial_svm;
2151 }
2152
Ben Gardon1ec69642019-02-11 11:02:51 -08002153 svm->vcpu.arch.guest_fpu = kmem_cache_zalloc(x86_fpu_cache,
2154 GFP_KERNEL_ACCOUNT);
Marc Orrb666a4b2018-11-06 14:53:56 -08002155 if (!svm->vcpu.arch.guest_fpu) {
2156 printk(KERN_ERR "kvm: failed to allocate vcpu's fpu\n");
2157 err = -ENOMEM;
Wanpeng Lid9a710e2019-07-22 12:26:21 +08002158 goto free_user_fpu;
Marc Orrb666a4b2018-11-06 14:53:56 -08002159 }
2160
Rusty Russellfb3f0f52007-07-27 17:16:56 +10002161 err = kvm_vcpu_init(&svm->vcpu, kvm, id);
2162 if (err)
2163 goto free_svm;
2164
Joerg Roedelf65c2292008-02-13 18:58:46 +01002165 err = -ENOMEM;
Ben Gardon1ec69642019-02-11 11:02:51 -08002166 page = alloc_page(GFP_KERNEL_ACCOUNT);
Takuya Yoshikawab7af4042010-03-09 14:55:19 +09002167 if (!page)
2168 goto uninit;
2169
Ben Gardon1ec69642019-02-11 11:02:51 -08002170 msrpm_pages = alloc_pages(GFP_KERNEL_ACCOUNT, MSRPM_ALLOC_ORDER);
Joerg Roedelf65c2292008-02-13 18:58:46 +01002171 if (!msrpm_pages)
Takuya Yoshikawab7af4042010-03-09 14:55:19 +09002172 goto free_page1;
Alexander Graf3d6368e2008-11-25 20:17:07 +01002173
Ben Gardon1ec69642019-02-11 11:02:51 -08002174 nested_msrpm_pages = alloc_pages(GFP_KERNEL_ACCOUNT, MSRPM_ALLOC_ORDER);
Alexander Graf3d6368e2008-11-25 20:17:07 +01002175 if (!nested_msrpm_pages)
Takuya Yoshikawab7af4042010-03-09 14:55:19 +09002176 goto free_page2;
Joerg Roedelf65c2292008-02-13 18:58:46 +01002177
Ben Gardon1ec69642019-02-11 11:02:51 -08002178 hsave_page = alloc_page(GFP_KERNEL_ACCOUNT);
Alexander Grafb286d5d2008-11-25 20:17:05 +01002179 if (!hsave_page)
Takuya Yoshikawab7af4042010-03-09 14:55:19 +09002180 goto free_page3;
2181
Suravee Suthikulpanitdfa20092017-09-12 10:42:40 -05002182 err = avic_init_vcpu(svm);
2183 if (err)
2184 goto free_page4;
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -05002185
Suravee Suthikulpanit8221c132016-05-04 14:09:52 -05002186 /* We initialize this flag to true to make sure that the is_running
2187 * bit would be set the first time the vcpu is loaded.
2188 */
2189 svm->avic_is_running = true;
2190
Joerg Roedele6aa9ab2009-08-07 11:49:33 +02002191 svm->nested.hsave = page_address(hsave_page);
Alexander Grafb286d5d2008-11-25 20:17:05 +01002192
Takuya Yoshikawab7af4042010-03-09 14:55:19 +09002193 svm->msrpm = page_address(msrpm_pages);
2194 svm_vcpu_init_msrpm(svm->msrpm);
2195
Joerg Roedele6aa9ab2009-08-07 11:49:33 +02002196 svm->nested.msrpm = page_address(nested_msrpm_pages);
Joerg Roedel323c3d82010-03-01 15:34:37 +01002197 svm_vcpu_init_msrpm(svm->nested.msrpm);
Alexander Graf3d6368e2008-11-25 20:17:07 +01002198
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04002199 svm->vmcb = page_address(page);
2200 clear_page(svm->vmcb);
Tom Lendackyd0ec49d2017-07-17 16:10:27 -05002201 svm->vmcb_pa = __sme_set(page_to_pfn(page) << PAGE_SHIFT);
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04002202 svm->asid_generation = 0;
Paolo Bonzini56908912015-10-19 11:30:19 +02002203 init_vmcb(svm);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002204
Boris Ostrovsky2b036c62012-01-09 14:00:35 -05002205 svm_init_osvw(&svm->vcpu);
2206
Rusty Russellfb3f0f52007-07-27 17:16:56 +10002207 return &svm->vcpu;
Avi Kivity36241b82006-12-22 01:05:20 -08002208
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -05002209free_page4:
2210 __free_page(hsave_page);
Takuya Yoshikawab7af4042010-03-09 14:55:19 +09002211free_page3:
2212 __free_pages(nested_msrpm_pages, MSRPM_ALLOC_ORDER);
2213free_page2:
2214 __free_pages(msrpm_pages, MSRPM_ALLOC_ORDER);
2215free_page1:
2216 __free_page(page);
Rusty Russellfb3f0f52007-07-27 17:16:56 +10002217uninit:
2218 kvm_vcpu_uninit(&svm->vcpu);
2219free_svm:
Marc Orrb666a4b2018-11-06 14:53:56 -08002220 kmem_cache_free(x86_fpu_cache, svm->vcpu.arch.guest_fpu);
Wanpeng Lid9a710e2019-07-22 12:26:21 +08002221free_user_fpu:
2222 kmem_cache_free(x86_fpu_cache, svm->vcpu.arch.user_fpu);
Marc Orrb666a4b2018-11-06 14:53:56 -08002223free_partial_svm:
Rusty Russella4770342007-08-01 14:46:11 +10002224 kmem_cache_free(kvm_vcpu_cache, svm);
Rusty Russellfb3f0f52007-07-27 17:16:56 +10002225out:
2226 return ERR_PTR(err);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002227}
2228
Jim Mattsonfd65d312018-05-22 09:54:20 -07002229static void svm_clear_current_vmcb(struct vmcb *vmcb)
2230{
2231 int i;
2232
2233 for_each_online_cpu(i)
2234 cmpxchg(&per_cpu(svm_data, i)->current_vmcb, vmcb, NULL);
2235}
2236
Avi Kivity6aa8b732006-12-10 02:21:36 -08002237static void svm_free_vcpu(struct kvm_vcpu *vcpu)
2238{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04002239 struct vcpu_svm *svm = to_svm(vcpu);
2240
Jim Mattsonfd65d312018-05-22 09:54:20 -07002241 /*
2242 * The vmcb page can be recycled, causing a false negative in
2243 * svm_vcpu_load(). So, ensure that no logical CPU has this
2244 * vmcb page recorded as its current vmcb.
2245 */
2246 svm_clear_current_vmcb(svm->vmcb);
2247
Tom Lendackyd0ec49d2017-07-17 16:10:27 -05002248 __free_page(pfn_to_page(__sme_clr(svm->vmcb_pa) >> PAGE_SHIFT));
Joerg Roedelf65c2292008-02-13 18:58:46 +01002249 __free_pages(virt_to_page(svm->msrpm), MSRPM_ALLOC_ORDER);
Joerg Roedele6aa9ab2009-08-07 11:49:33 +02002250 __free_page(virt_to_page(svm->nested.hsave));
2251 __free_pages(virt_to_page(svm->nested.msrpm), MSRPM_ALLOC_ORDER);
Rusty Russellfb3f0f52007-07-27 17:16:56 +10002252 kvm_vcpu_uninit(vcpu);
Wanpeng Lid9a710e2019-07-22 12:26:21 +08002253 kmem_cache_free(x86_fpu_cache, svm->vcpu.arch.user_fpu);
Marc Orrb666a4b2018-11-06 14:53:56 -08002254 kmem_cache_free(x86_fpu_cache, svm->vcpu.arch.guest_fpu);
Rusty Russella4770342007-08-01 14:46:11 +10002255 kmem_cache_free(kvm_vcpu_cache, svm);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002256}
2257
Avi Kivity15ad7142007-07-11 18:17:21 +03002258static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -08002259{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04002260 struct vcpu_svm *svm = to_svm(vcpu);
Ashok Raj15d45072018-02-01 22:59:43 +01002261 struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
Avi Kivity15ad7142007-07-11 18:17:21 +03002262 int i;
Avi Kivity0cc50642007-03-25 12:07:27 +02002263
Avi Kivity0cc50642007-03-25 12:07:27 +02002264 if (unlikely(cpu != vcpu->cpu)) {
Marcelo Tosatti4b656b12009-07-21 12:47:45 -03002265 svm->asid_generation = 0;
Roedel, Joerg8d28fec2010-12-03 13:15:21 +01002266 mark_all_dirty(svm->vmcb);
Avi Kivity0cc50642007-03-25 12:07:27 +02002267 }
Anthony Liguori94dfbdb2007-04-29 11:56:06 +03002268
Avi Kivity82ca2d12010-10-21 12:20:34 +02002269#ifdef CONFIG_X86_64
2270 rdmsrl(MSR_GS_BASE, to_svm(vcpu)->host.gs_base);
2271#endif
Avi Kivitydacccfd2010-10-21 12:20:33 +02002272 savesegment(fs, svm->host.fs);
2273 savesegment(gs, svm->host.gs);
2274 svm->host.ldt = kvm_read_ldt();
2275
Anthony Liguori94dfbdb2007-04-29 11:56:06 +03002276 for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04002277 rdmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
Joerg Roedelfbc0db72011-03-25 09:44:46 +01002278
Haozhong Zhangad7218832015-10-20 15:39:02 +08002279 if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) {
2280 u64 tsc_ratio = vcpu->arch.tsc_scaling_ratio;
2281 if (tsc_ratio != __this_cpu_read(current_tsc_ratio)) {
2282 __this_cpu_write(current_tsc_ratio, tsc_ratio);
2283 wrmsrl(MSR_AMD64_TSC_RATIO, tsc_ratio);
2284 }
Joerg Roedelfbc0db72011-03-25 09:44:46 +01002285 }
Paolo Bonzini46896c72015-11-12 14:49:16 +01002286 /* This assumes that the kernel never uses MSR_TSC_AUX */
2287 if (static_cpu_has(X86_FEATURE_RDTSCP))
2288 wrmsrl(MSR_TSC_AUX, svm->tsc_aux);
Suravee Suthikulpanit8221c132016-05-04 14:09:52 -05002289
Ashok Raj15d45072018-02-01 22:59:43 +01002290 if (sd->current_vmcb != svm->vmcb) {
2291 sd->current_vmcb = svm->vmcb;
2292 indirect_branch_prediction_barrier();
2293 }
Suravee Suthikulpanit8221c132016-05-04 14:09:52 -05002294 avic_vcpu_load(vcpu, cpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002295}
2296
2297static void svm_vcpu_put(struct kvm_vcpu *vcpu)
2298{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04002299 struct vcpu_svm *svm = to_svm(vcpu);
Anthony Liguori94dfbdb2007-04-29 11:56:06 +03002300 int i;
2301
Suravee Suthikulpanit8221c132016-05-04 14:09:52 -05002302 avic_vcpu_put(vcpu);
2303
Avi Kivitye1beb1d2007-11-18 13:50:24 +02002304 ++vcpu->stat.host_state_reload;
Avi Kivitydacccfd2010-10-21 12:20:33 +02002305 kvm_load_ldt(svm->host.ldt);
2306#ifdef CONFIG_X86_64
2307 loadsegment(fs, svm->host.fs);
Andy Lutomirski296f7812016-04-26 12:23:29 -07002308 wrmsrl(MSR_KERNEL_GS_BASE, current->thread.gsbase);
Joerg Roedel893a5ab2011-01-14 16:45:01 +01002309 load_gs_index(svm->host.gs);
Avi Kivitydacccfd2010-10-21 12:20:33 +02002310#else
Avi Kivity831ca602011-03-08 16:09:51 +02002311#ifdef CONFIG_X86_32_LAZY_GS
Avi Kivitydacccfd2010-10-21 12:20:33 +02002312 loadsegment(gs, svm->host.gs);
2313#endif
Avi Kivity831ca602011-03-08 16:09:51 +02002314#endif
Anthony Liguori94dfbdb2007-04-29 11:56:06 +03002315 for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04002316 wrmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002317}
2318
Suravee Suthikulpanit8221c132016-05-04 14:09:52 -05002319static void svm_vcpu_blocking(struct kvm_vcpu *vcpu)
2320{
2321 avic_set_running(vcpu, false);
2322}
2323
2324static void svm_vcpu_unblocking(struct kvm_vcpu *vcpu)
2325{
2326 avic_set_running(vcpu, true);
2327}
2328
Avi Kivity6aa8b732006-12-10 02:21:36 -08002329static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu)
2330{
Ladi Prosek9b611742017-06-21 09:06:59 +02002331 struct vcpu_svm *svm = to_svm(vcpu);
2332 unsigned long rflags = svm->vmcb->save.rflags;
2333
2334 if (svm->nmi_singlestep) {
2335 /* Hide our flags if they were not set by the guest */
2336 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF))
2337 rflags &= ~X86_EFLAGS_TF;
2338 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF))
2339 rflags &= ~X86_EFLAGS_RF;
2340 }
2341 return rflags;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002342}
2343
2344static void svm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
2345{
Ladi Prosek9b611742017-06-21 09:06:59 +02002346 if (to_svm(vcpu)->nmi_singlestep)
2347 rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
2348
Paolo Bonziniae9fedc2014-05-14 09:39:49 +02002349 /*
Andrea Gelminibb3541f2016-05-21 14:14:44 +02002350 * Any change of EFLAGS.VM is accompanied by a reload of SS
Paolo Bonziniae9fedc2014-05-14 09:39:49 +02002351 * (caused by either a task switch or an inter-privilege IRET),
2352 * so we do not need to update the CPL here.
2353 */
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04002354 to_svm(vcpu)->vmcb->save.rflags = rflags;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002355}
2356
Avi Kivity6de4f3a2009-05-31 22:58:47 +03002357static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
2358{
2359 switch (reg) {
2360 case VCPU_EXREG_PDPTR:
2361 BUG_ON(!npt_enabled);
Avi Kivity9f8fe502010-12-05 17:30:00 +02002362 load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu));
Avi Kivity6de4f3a2009-05-31 22:58:47 +03002363 break;
2364 default:
2365 BUG();
2366 }
2367}
2368
Alexander Graff0b85052008-11-25 20:17:01 +01002369static void svm_set_vintr(struct vcpu_svm *svm)
2370{
Joerg Roedel8a05a1b82010-11-30 18:04:00 +01002371 set_intercept(svm, INTERCEPT_VINTR);
Alexander Graff0b85052008-11-25 20:17:01 +01002372}
2373
2374static void svm_clear_vintr(struct vcpu_svm *svm)
2375{
Joerg Roedel8a05a1b82010-11-30 18:04:00 +01002376 clr_intercept(svm, INTERCEPT_VINTR);
Alexander Graff0b85052008-11-25 20:17:01 +01002377}
2378
Avi Kivity6aa8b732006-12-10 02:21:36 -08002379static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg)
2380{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04002381 struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002382
2383 switch (seg) {
2384 case VCPU_SREG_CS: return &save->cs;
2385 case VCPU_SREG_DS: return &save->ds;
2386 case VCPU_SREG_ES: return &save->es;
2387 case VCPU_SREG_FS: return &save->fs;
2388 case VCPU_SREG_GS: return &save->gs;
2389 case VCPU_SREG_SS: return &save->ss;
2390 case VCPU_SREG_TR: return &save->tr;
2391 case VCPU_SREG_LDTR: return &save->ldtr;
2392 }
2393 BUG();
Al Viro8b6d44c2007-02-09 16:38:40 +00002394 return NULL;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002395}
2396
2397static u64 svm_get_segment_base(struct kvm_vcpu *vcpu, int seg)
2398{
2399 struct vmcb_seg *s = svm_seg(vcpu, seg);
2400
2401 return s->base;
2402}
2403
2404static void svm_get_segment(struct kvm_vcpu *vcpu,
2405 struct kvm_segment *var, int seg)
2406{
2407 struct vmcb_seg *s = svm_seg(vcpu, seg);
2408
2409 var->base = s->base;
2410 var->limit = s->limit;
2411 var->selector = s->selector;
2412 var->type = s->attrib & SVM_SELECTOR_TYPE_MASK;
2413 var->s = (s->attrib >> SVM_SELECTOR_S_SHIFT) & 1;
2414 var->dpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3;
2415 var->present = (s->attrib >> SVM_SELECTOR_P_SHIFT) & 1;
2416 var->avl = (s->attrib >> SVM_SELECTOR_AVL_SHIFT) & 1;
2417 var->l = (s->attrib >> SVM_SELECTOR_L_SHIFT) & 1;
2418 var->db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1;
Jim Mattson80112c82014-07-08 09:47:41 +05302419
2420 /*
2421 * AMD CPUs circa 2014 track the G bit for all segments except CS.
2422 * However, the SVM spec states that the G bit is not observed by the
2423 * CPU, and some VMware virtual CPUs drop the G bit for all segments.
2424 * So let's synthesize a legal G bit for all segments, this helps
2425 * running KVM nested. It also helps cross-vendor migration, because
2426 * Intel's vmentry has a check on the 'G' bit.
2427 */
2428 var->g = s->limit > 0xfffff;
Amit Shah25022ac2008-10-27 09:04:17 +00002429
Joerg Roedele0231712010-02-24 18:59:10 +01002430 /*
2431 * AMD's VMCB does not have an explicit unusable field, so emulate it
Andre Przywara19bca6a2009-04-28 12:45:30 +02002432 * for cross vendor migration purposes by "not present"
2433 */
Gioh Kim8eae9572017-05-30 15:24:45 +02002434 var->unusable = !var->present;
Andre Przywara19bca6a2009-04-28 12:45:30 +02002435
Andre Przywara1fbdc7a2009-01-11 22:39:44 +01002436 switch (seg) {
Andre Przywara1fbdc7a2009-01-11 22:39:44 +01002437 case VCPU_SREG_TR:
2438 /*
2439 * Work around a bug where the busy flag in the tr selector
2440 * isn't exposed
2441 */
Amit Shahc0d09822008-10-27 09:04:18 +00002442 var->type |= 0x2;
Andre Przywara1fbdc7a2009-01-11 22:39:44 +01002443 break;
2444 case VCPU_SREG_DS:
2445 case VCPU_SREG_ES:
2446 case VCPU_SREG_FS:
2447 case VCPU_SREG_GS:
2448 /*
2449 * The accessed bit must always be set in the segment
2450 * descriptor cache, although it can be cleared in the
2451 * descriptor, the cached bit always remains at 1. Since
2452 * Intel has a check on this, set it here to support
2453 * cross-vendor migration.
2454 */
2455 if (!var->unusable)
2456 var->type |= 0x1;
2457 break;
Andre Przywarab586eb02009-04-28 12:45:43 +02002458 case VCPU_SREG_SS:
Joerg Roedele0231712010-02-24 18:59:10 +01002459 /*
2460 * On AMD CPUs sometimes the DB bit in the segment
Andre Przywarab586eb02009-04-28 12:45:43 +02002461 * descriptor is left as 1, although the whole segment has
2462 * been made unusable. Clear it here to pass an Intel VMX
2463 * entry check when cross vendor migrating.
2464 */
2465 if (var->unusable)
2466 var->db = 0;
Roman Pend9c1b542017-06-01 10:55:03 +02002467 /* This is symmetric with svm_set_segment() */
Jan Kiszka33b458d2014-06-29 17:12:43 +02002468 var->dpl = to_svm(vcpu)->vmcb->save.cpl;
Andre Przywarab586eb02009-04-28 12:45:43 +02002469 break;
Andre Przywara1fbdc7a2009-01-11 22:39:44 +01002470 }
Avi Kivity6aa8b732006-12-10 02:21:36 -08002471}
2472
Izik Eidus2e4d2652008-03-24 19:38:34 +02002473static int svm_get_cpl(struct kvm_vcpu *vcpu)
2474{
2475 struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
2476
2477 return save->cpl;
2478}
2479
Gleb Natapov89a27f42010-02-16 10:51:48 +02002480static void svm_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
Avi Kivity6aa8b732006-12-10 02:21:36 -08002481{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04002482 struct vcpu_svm *svm = to_svm(vcpu);
2483
Gleb Natapov89a27f42010-02-16 10:51:48 +02002484 dt->size = svm->vmcb->save.idtr.limit;
2485 dt->address = svm->vmcb->save.idtr.base;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002486}
2487
Gleb Natapov89a27f42010-02-16 10:51:48 +02002488static void svm_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
Avi Kivity6aa8b732006-12-10 02:21:36 -08002489{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04002490 struct vcpu_svm *svm = to_svm(vcpu);
2491
Gleb Natapov89a27f42010-02-16 10:51:48 +02002492 svm->vmcb->save.idtr.limit = dt->size;
2493 svm->vmcb->save.idtr.base = dt->address ;
Joerg Roedel17a703c2010-12-03 11:45:56 +01002494 mark_dirty(svm->vmcb, VMCB_DT);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002495}
2496
Gleb Natapov89a27f42010-02-16 10:51:48 +02002497static void svm_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
Avi Kivity6aa8b732006-12-10 02:21:36 -08002498{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04002499 struct vcpu_svm *svm = to_svm(vcpu);
2500
Gleb Natapov89a27f42010-02-16 10:51:48 +02002501 dt->size = svm->vmcb->save.gdtr.limit;
2502 dt->address = svm->vmcb->save.gdtr.base;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002503}
2504
Gleb Natapov89a27f42010-02-16 10:51:48 +02002505static void svm_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
Avi Kivity6aa8b732006-12-10 02:21:36 -08002506{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04002507 struct vcpu_svm *svm = to_svm(vcpu);
2508
Gleb Natapov89a27f42010-02-16 10:51:48 +02002509 svm->vmcb->save.gdtr.limit = dt->size;
2510 svm->vmcb->save.gdtr.base = dt->address ;
Joerg Roedel17a703c2010-12-03 11:45:56 +01002511 mark_dirty(svm->vmcb, VMCB_DT);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002512}
2513
Avi Kivitye8467fd2009-12-29 18:43:06 +02002514static void svm_decache_cr0_guest_bits(struct kvm_vcpu *vcpu)
2515{
2516}
2517
Avi Kivityaff48ba2010-12-05 18:56:11 +02002518static void svm_decache_cr3(struct kvm_vcpu *vcpu)
2519{
2520}
2521
Anthony Liguori25c4c272007-04-27 09:29:21 +03002522static void svm_decache_cr4_guest_bits(struct kvm_vcpu *vcpu)
Avi Kivity399badf2007-01-05 16:36:38 -08002523{
2524}
2525
Avi Kivityd2251572010-01-06 10:55:27 +02002526static void update_cr0_intercept(struct vcpu_svm *svm)
2527{
2528 ulong gcr0 = svm->vcpu.arch.cr0;
2529 u64 *hcr0 = &svm->vmcb->save.cr0;
2530
Paolo Bonzinibd7e5b02017-02-03 21:18:52 -08002531 *hcr0 = (*hcr0 & ~SVM_CR0_SELECTIVE_MASK)
2532 | (gcr0 & SVM_CR0_SELECTIVE_MASK);
Avi Kivityd2251572010-01-06 10:55:27 +02002533
Joerg Roedeldcca1a62010-12-03 11:45:54 +01002534 mark_dirty(svm->vmcb, VMCB_CR);
Avi Kivityd2251572010-01-06 10:55:27 +02002535
Paolo Bonzinibd7e5b02017-02-03 21:18:52 -08002536 if (gcr0 == *hcr0) {
Roedel, Joerg4ee546b2010-12-03 10:50:51 +01002537 clr_cr_intercept(svm, INTERCEPT_CR0_READ);
2538 clr_cr_intercept(svm, INTERCEPT_CR0_WRITE);
Avi Kivityd2251572010-01-06 10:55:27 +02002539 } else {
Roedel, Joerg4ee546b2010-12-03 10:50:51 +01002540 set_cr_intercept(svm, INTERCEPT_CR0_READ);
2541 set_cr_intercept(svm, INTERCEPT_CR0_WRITE);
Avi Kivityd2251572010-01-06 10:55:27 +02002542 }
2543}
2544
Avi Kivity6aa8b732006-12-10 02:21:36 -08002545static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
2546{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04002547 struct vcpu_svm *svm = to_svm(vcpu);
2548
Avi Kivity05b3e0c2006-12-13 00:33:45 -08002549#ifdef CONFIG_X86_64
Avi Kivityf6801df2010-01-21 15:31:50 +02002550 if (vcpu->arch.efer & EFER_LME) {
Rusty Russell707d92fa2007-07-17 23:19:08 +10002551 if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
Avi Kivityf6801df2010-01-21 15:31:50 +02002552 vcpu->arch.efer |= EFER_LMA;
Carlo Marcelo Arenas Belon2b5203e2007-12-01 06:17:11 -06002553 svm->vmcb->save.efer |= EFER_LMA | EFER_LME;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002554 }
2555
Mike Dayd77c26f2007-10-08 09:02:08 -04002556 if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) {
Avi Kivityf6801df2010-01-21 15:31:50 +02002557 vcpu->arch.efer &= ~EFER_LMA;
Carlo Marcelo Arenas Belon2b5203e2007-12-01 06:17:11 -06002558 svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002559 }
2560 }
2561#endif
Zhang Xiantaoad312c72007-12-13 23:50:52 +08002562 vcpu->arch.cr0 = cr0;
Avi Kivity888f9f32010-01-10 12:14:04 +02002563
2564 if (!npt_enabled)
2565 cr0 |= X86_CR0_PG | X86_CR0_WP;
Avi Kivity02daab22009-12-30 12:40:26 +02002566
Paolo Bonzinibcf166a2015-10-01 13:19:55 +02002567 /*
2568 * re-enable caching here because the QEMU bios
2569 * does not do it - this results in some delay at
2570 * reboot
2571 */
2572 if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
2573 cr0 &= ~(X86_CR0_CD | X86_CR0_NW);
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04002574 svm->vmcb->save.cr0 = cr0;
Joerg Roedeldcca1a62010-12-03 11:45:54 +01002575 mark_dirty(svm->vmcb, VMCB_CR);
Avi Kivityd2251572010-01-06 10:55:27 +02002576 update_cr0_intercept(svm);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002577}
2578
Nadav Har'El5e1746d2011-05-25 23:03:24 +03002579static int svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
Avi Kivity6aa8b732006-12-10 02:21:36 -08002580{
Andy Lutomirski1e02ce42014-10-24 15:58:08 -07002581 unsigned long host_cr4_mce = cr4_read_shadow() & X86_CR4_MCE;
Joerg Roedele5eab0c2008-09-09 19:11:51 +02002582 unsigned long old_cr4 = to_svm(vcpu)->vmcb->save.cr4;
2583
Nadav Har'El5e1746d2011-05-25 23:03:24 +03002584 if (cr4 & X86_CR4_VMXE)
2585 return 1;
2586
Joerg Roedele5eab0c2008-09-09 19:11:51 +02002587 if (npt_enabled && ((old_cr4 ^ cr4) & X86_CR4_PGE))
Wanpeng Lic2ba05c2017-12-12 17:33:03 -08002588 svm_flush_tlb(vcpu, true);
Joerg Roedel6394b642008-04-09 14:15:29 +02002589
Joerg Roedelec077262008-04-09 14:15:28 +02002590 vcpu->arch.cr4 = cr4;
2591 if (!npt_enabled)
2592 cr4 |= X86_CR4_PAE;
Joerg Roedel6394b642008-04-09 14:15:29 +02002593 cr4 |= host_cr4_mce;
Joerg Roedelec077262008-04-09 14:15:28 +02002594 to_svm(vcpu)->vmcb->save.cr4 = cr4;
Joerg Roedeldcca1a62010-12-03 11:45:54 +01002595 mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR);
Nadav Har'El5e1746d2011-05-25 23:03:24 +03002596 return 0;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002597}
2598
2599static void svm_set_segment(struct kvm_vcpu *vcpu,
2600 struct kvm_segment *var, int seg)
2601{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04002602 struct vcpu_svm *svm = to_svm(vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002603 struct vmcb_seg *s = svm_seg(vcpu, seg);
2604
2605 s->base = var->base;
2606 s->limit = var->limit;
2607 s->selector = var->selector;
Roman Pend9c1b542017-06-01 10:55:03 +02002608 s->attrib = (var->type & SVM_SELECTOR_TYPE_MASK);
2609 s->attrib |= (var->s & 1) << SVM_SELECTOR_S_SHIFT;
2610 s->attrib |= (var->dpl & 3) << SVM_SELECTOR_DPL_SHIFT;
2611 s->attrib |= ((var->present & 1) && !var->unusable) << SVM_SELECTOR_P_SHIFT;
2612 s->attrib |= (var->avl & 1) << SVM_SELECTOR_AVL_SHIFT;
2613 s->attrib |= (var->l & 1) << SVM_SELECTOR_L_SHIFT;
2614 s->attrib |= (var->db & 1) << SVM_SELECTOR_DB_SHIFT;
2615 s->attrib |= (var->g & 1) << SVM_SELECTOR_G_SHIFT;
Paolo Bonziniae9fedc2014-05-14 09:39:49 +02002616
2617 /*
2618 * This is always accurate, except if SYSRET returned to a segment
2619 * with SS.DPL != 3. Intel does not have this quirk, and always
2620 * forces SS.DPL to 3 on sysret, so we ignore that case; fixing it
2621 * would entail passing the CPL to userspace and back.
2622 */
2623 if (seg == VCPU_SREG_SS)
Roman Pend9c1b542017-06-01 10:55:03 +02002624 /* This is symmetric with svm_get_segment() */
2625 svm->vmcb->save.cpl = (var->dpl & 3);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002626
Joerg Roedel060d0c92010-12-03 11:45:57 +01002627 mark_dirty(svm->vmcb, VMCB_SEG);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002628}
2629
Paolo Bonzinicbdb9672015-11-10 09:14:39 +01002630static void update_bp_intercept(struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -08002631{
Jan Kiszkad0bfb942008-12-15 13:52:10 +01002632 struct vcpu_svm *svm = to_svm(vcpu);
2633
Joerg Roedel18c918c2010-11-30 18:03:59 +01002634 clr_exception_intercept(svm, BP_VECTOR);
Gleb Natapov44c11432009-05-11 13:35:52 +03002635
Jan Kiszkad0bfb942008-12-15 13:52:10 +01002636 if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) {
Jan Kiszkad0bfb942008-12-15 13:52:10 +01002637 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
Joerg Roedel18c918c2010-11-30 18:03:59 +01002638 set_exception_intercept(svm, BP_VECTOR);
Jan Kiszkad0bfb942008-12-15 13:52:10 +01002639 } else
2640 vcpu->guest_debug = 0;
Gleb Natapov44c11432009-05-11 13:35:52 +03002641}
2642
Tejun Heo0fe1e002009-10-29 22:34:14 +09002643static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd)
Avi Kivity6aa8b732006-12-10 02:21:36 -08002644{
Tejun Heo0fe1e002009-10-29 22:34:14 +09002645 if (sd->next_asid > sd->max_asid) {
2646 ++sd->asid_generation;
Brijesh Singh4faefff2017-12-04 10:57:25 -06002647 sd->next_asid = sd->min_asid;
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04002648 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002649 }
2650
Tejun Heo0fe1e002009-10-29 22:34:14 +09002651 svm->asid_generation = sd->asid_generation;
2652 svm->vmcb->control.asid = sd->next_asid++;
Joerg Roedeld48086d2010-12-03 11:45:51 +01002653
2654 mark_dirty(svm->vmcb, VMCB_ASID);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002655}
2656
Jan Kiszka73aaf249e2014-01-04 18:47:16 +01002657static u64 svm_get_dr6(struct kvm_vcpu *vcpu)
2658{
2659 return to_svm(vcpu)->vmcb->save.dr6;
2660}
2661
2662static void svm_set_dr6(struct kvm_vcpu *vcpu, unsigned long value)
2663{
2664 struct vcpu_svm *svm = to_svm(vcpu);
2665
2666 svm->vmcb->save.dr6 = value;
2667 mark_dirty(svm->vmcb, VMCB_DR);
2668}
2669
Paolo Bonzinifacb0132014-02-21 10:32:27 +01002670static void svm_sync_dirty_debug_regs(struct kvm_vcpu *vcpu)
2671{
2672 struct vcpu_svm *svm = to_svm(vcpu);
2673
2674 get_debugreg(vcpu->arch.db[0], 0);
2675 get_debugreg(vcpu->arch.db[1], 1);
2676 get_debugreg(vcpu->arch.db[2], 2);
2677 get_debugreg(vcpu->arch.db[3], 3);
2678 vcpu->arch.dr6 = svm_get_dr6(vcpu);
2679 vcpu->arch.dr7 = svm->vmcb->save.dr7;
2680
2681 vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT;
2682 set_dr_intercepts(svm);
2683}
2684
Gleb Natapov020df072010-04-13 10:05:23 +03002685static void svm_set_dr7(struct kvm_vcpu *vcpu, unsigned long value)
Avi Kivity6aa8b732006-12-10 02:21:36 -08002686{
Jan Kiszka42dbaa52008-12-15 13:52:10 +01002687 struct vcpu_svm *svm = to_svm(vcpu);
Jan Kiszka42dbaa52008-12-15 13:52:10 +01002688
Gleb Natapov020df072010-04-13 10:05:23 +03002689 svm->vmcb->save.dr7 = value;
Joerg Roedel72214b92010-12-03 11:45:55 +01002690 mark_dirty(svm->vmcb, VMCB_DR);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002691}
2692
Avi Kivity851ba692009-08-24 11:10:17 +03002693static int pf_interception(struct vcpu_svm *svm)
Avi Kivity6aa8b732006-12-10 02:21:36 -08002694{
Brijesh Singh0ede79e2017-12-04 10:57:39 -06002695 u64 fault_address = __sme_clr(svm->vmcb->control.exit_info_2);
Wanpeng Li1261bfa2017-07-13 18:30:40 -07002696 u64 error_code = svm->vmcb->control.exit_info_1;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002697
Wanpeng Li1261bfa2017-07-13 18:30:40 -07002698 return kvm_handle_page_fault(&svm->vcpu, error_code, fault_address,
Brijesh Singh00b10fe2017-12-04 10:57:40 -06002699 static_cpu_has(X86_FEATURE_DECODEASSISTS) ?
2700 svm->vmcb->control.insn_bytes : NULL,
Paolo Bonzinid0006532017-08-11 18:36:43 +02002701 svm->vmcb->control.insn_len);
2702}
2703
2704static int npf_interception(struct vcpu_svm *svm)
2705{
Brijesh Singh0ede79e2017-12-04 10:57:39 -06002706 u64 fault_address = __sme_clr(svm->vmcb->control.exit_info_2);
Paolo Bonzinid0006532017-08-11 18:36:43 +02002707 u64 error_code = svm->vmcb->control.exit_info_1;
2708
2709 trace_kvm_page_fault(fault_address, error_code);
2710 return kvm_mmu_page_fault(&svm->vcpu, fault_address, error_code,
Brijesh Singh00b10fe2017-12-04 10:57:40 -06002711 static_cpu_has(X86_FEATURE_DECODEASSISTS) ?
2712 svm->vmcb->control.insn_bytes : NULL,
Paolo Bonzinid0006532017-08-11 18:36:43 +02002713 svm->vmcb->control.insn_len);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002714}
2715
Avi Kivity851ba692009-08-24 11:10:17 +03002716static int db_interception(struct vcpu_svm *svm)
Jan Kiszkad0bfb942008-12-15 13:52:10 +01002717{
Avi Kivity851ba692009-08-24 11:10:17 +03002718 struct kvm_run *kvm_run = svm->vcpu.run;
Vitaly Kuznetsov99c22172019-04-03 16:06:42 +02002719 struct kvm_vcpu *vcpu = &svm->vcpu;
Avi Kivity851ba692009-08-24 11:10:17 +03002720
Jan Kiszkad0bfb942008-12-15 13:52:10 +01002721 if (!(svm->vcpu.guest_debug &
Gleb Natapov44c11432009-05-11 13:35:52 +03002722 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) &&
Jan Kiszka6be7d302009-10-18 13:24:54 +02002723 !svm->nmi_singlestep) {
Jan Kiszkad0bfb942008-12-15 13:52:10 +01002724 kvm_queue_exception(&svm->vcpu, DB_VECTOR);
2725 return 1;
2726 }
Gleb Natapov44c11432009-05-11 13:35:52 +03002727
Jan Kiszka6be7d302009-10-18 13:24:54 +02002728 if (svm->nmi_singlestep) {
Ladi Prosek4aebd0e2017-06-21 09:06:57 +02002729 disable_nmi_singlestep(svm);
Vitaly Kuznetsov99c22172019-04-03 16:06:42 +02002730 /* Make sure we check for pending NMIs upon entry */
2731 kvm_make_request(KVM_REQ_EVENT, vcpu);
Gleb Natapov44c11432009-05-11 13:35:52 +03002732 }
2733
2734 if (svm->vcpu.guest_debug &
Joerg Roedele0231712010-02-24 18:59:10 +01002735 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) {
Gleb Natapov44c11432009-05-11 13:35:52 +03002736 kvm_run->exit_reason = KVM_EXIT_DEBUG;
2737 kvm_run->debug.arch.pc =
2738 svm->vmcb->save.cs.base + svm->vmcb->save.rip;
2739 kvm_run->debug.arch.exception = DB_VECTOR;
2740 return 0;
2741 }
2742
2743 return 1;
Jan Kiszkad0bfb942008-12-15 13:52:10 +01002744}
2745
Avi Kivity851ba692009-08-24 11:10:17 +03002746static int bp_interception(struct vcpu_svm *svm)
Jan Kiszkad0bfb942008-12-15 13:52:10 +01002747{
Avi Kivity851ba692009-08-24 11:10:17 +03002748 struct kvm_run *kvm_run = svm->vcpu.run;
2749
Jan Kiszkad0bfb942008-12-15 13:52:10 +01002750 kvm_run->exit_reason = KVM_EXIT_DEBUG;
2751 kvm_run->debug.arch.pc = svm->vmcb->save.cs.base + svm->vmcb->save.rip;
2752 kvm_run->debug.arch.exception = BP_VECTOR;
2753 return 0;
2754}
2755
Avi Kivity851ba692009-08-24 11:10:17 +03002756static int ud_interception(struct vcpu_svm *svm)
Anthony Liguori7aa81cc2007-09-17 14:57:50 -05002757{
Wanpeng Li082d06e2018-04-03 16:28:48 -07002758 return handle_ud(&svm->vcpu);
Anthony Liguori7aa81cc2007-09-17 14:57:50 -05002759}
2760
Eric Northup54a20552015-11-03 18:03:53 +01002761static int ac_interception(struct vcpu_svm *svm)
2762{
2763 kvm_queue_exception_e(&svm->vcpu, AC_VECTOR, 0);
2764 return 1;
2765}
2766
Liran Alon97184202018-03-12 13:12:52 +02002767static int gp_interception(struct vcpu_svm *svm)
2768{
2769 struct kvm_vcpu *vcpu = &svm->vcpu;
2770 u32 error_code = svm->vmcb->control.exit_info_1;
2771 int er;
2772
2773 WARN_ON_ONCE(!enable_vmware_backdoor);
2774
Sean Christopherson0ce97a22018-08-23 13:56:52 -07002775 er = kvm_emulate_instruction(vcpu,
Liran Alon97184202018-03-12 13:12:52 +02002776 EMULTYPE_VMWARE | EMULTYPE_NO_UD_ON_FAIL);
2777 if (er == EMULATE_USER_EXIT)
2778 return 0;
2779 else if (er != EMULATE_DONE)
2780 kvm_queue_exception_e(vcpu, GP_VECTOR, error_code);
2781 return 1;
2782}
2783
Joerg Roedel67ec6602010-05-17 14:43:35 +02002784static bool is_erratum_383(void)
2785{
2786 int err, i;
2787 u64 value;
2788
2789 if (!erratum_383_found)
2790 return false;
2791
2792 value = native_read_msr_safe(MSR_IA32_MC0_STATUS, &err);
2793 if (err)
2794 return false;
2795
2796 /* Bit 62 may or may not be set for this mce */
2797 value &= ~(1ULL << 62);
2798
2799 if (value != 0xb600000000010015ULL)
2800 return false;
2801
2802 /* Clear MCi_STATUS registers */
2803 for (i = 0; i < 6; ++i)
2804 native_write_msr_safe(MSR_IA32_MCx_STATUS(i), 0, 0);
2805
2806 value = native_read_msr_safe(MSR_IA32_MCG_STATUS, &err);
2807 if (!err) {
2808 u32 low, high;
2809
2810 value &= ~(1ULL << 2);
2811 low = lower_32_bits(value);
2812 high = upper_32_bits(value);
2813
2814 native_write_msr_safe(MSR_IA32_MCG_STATUS, low, high);
2815 }
2816
2817 /* Flush tlb to evict multi-match entries */
2818 __flush_tlb_all();
2819
2820 return true;
2821}
2822
Joerg Roedelfe5913e2010-05-17 14:43:34 +02002823static void svm_handle_mce(struct vcpu_svm *svm)
Joerg Roedel53371b52008-04-09 14:15:30 +02002824{
Joerg Roedel67ec6602010-05-17 14:43:35 +02002825 if (is_erratum_383()) {
2826 /*
2827 * Erratum 383 triggered. Guest state is corrupt so kill the
2828 * guest.
2829 */
2830 pr_err("KVM: Guest triggered AMD Erratum 383\n");
2831
Avi Kivitya8eeb042010-05-10 12:34:53 +03002832 kvm_make_request(KVM_REQ_TRIPLE_FAULT, &svm->vcpu);
Joerg Roedel67ec6602010-05-17 14:43:35 +02002833
2834 return;
2835 }
2836
Joerg Roedel53371b52008-04-09 14:15:30 +02002837 /*
2838 * On an #MC intercept the MCE handler is not called automatically in
2839 * the host. So do it by hand here.
2840 */
2841 asm volatile (
2842 "int $0x12\n");
2843 /* not sure if we ever come back to this point */
2844
Joerg Roedelfe5913e2010-05-17 14:43:34 +02002845 return;
2846}
2847
2848static int mc_interception(struct vcpu_svm *svm)
2849{
Joerg Roedel53371b52008-04-09 14:15:30 +02002850 return 1;
2851}
2852
Avi Kivity851ba692009-08-24 11:10:17 +03002853static int shutdown_interception(struct vcpu_svm *svm)
Joerg Roedel46fe4dd2007-01-26 00:56:42 -08002854{
Avi Kivity851ba692009-08-24 11:10:17 +03002855 struct kvm_run *kvm_run = svm->vcpu.run;
2856
Joerg Roedel46fe4dd2007-01-26 00:56:42 -08002857 /*
2858 * VMCB is undefined after a SHUTDOWN intercept
2859 * so reinitialize it.
2860 */
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04002861 clear_page(svm->vmcb);
Paolo Bonzini56908912015-10-19 11:30:19 +02002862 init_vmcb(svm);
Joerg Roedel46fe4dd2007-01-26 00:56:42 -08002863
2864 kvm_run->exit_reason = KVM_EXIT_SHUTDOWN;
2865 return 0;
2866}
2867
Avi Kivity851ba692009-08-24 11:10:17 +03002868static int io_interception(struct vcpu_svm *svm)
Avi Kivity6aa8b732006-12-10 02:21:36 -08002869{
Gleb Natapovcf8f70b2010-03-18 15:20:23 +02002870 struct kvm_vcpu *vcpu = &svm->vcpu;
Mike Dayd77c26f2007-10-08 09:02:08 -04002871 u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */
Sean Christophersondca7f122018-03-08 08:57:27 -08002872 int size, in, string;
Avi Kivity039576c2007-03-20 12:46:50 +02002873 unsigned port;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002874
Rusty Russelle756fc62007-07-30 20:07:08 +10002875 ++svm->vcpu.stat.io_exits;
Laurent Viviere70669a2007-08-05 10:36:40 +03002876 string = (io_info & SVM_IOIO_STR_MASK) != 0;
Avi Kivity039576c2007-03-20 12:46:50 +02002877 in = (io_info & SVM_IOIO_TYPE_MASK) != 0;
Tom Lendacky8370c3d2016-11-23 12:01:50 -05002878 if (string)
Sean Christopherson0ce97a22018-08-23 13:56:52 -07002879 return kvm_emulate_instruction(vcpu, 0) == EMULATE_DONE;
Gleb Natapovcf8f70b2010-03-18 15:20:23 +02002880
Avi Kivity039576c2007-03-20 12:46:50 +02002881 port = io_info >> 16;
2882 size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT;
Gleb Natapovcf8f70b2010-03-18 15:20:23 +02002883 svm->next_rip = svm->vmcb->control.exit_info_2;
Gleb Natapovcf8f70b2010-03-18 15:20:23 +02002884
Sean Christophersondca7f122018-03-08 08:57:27 -08002885 return kvm_fast_pio(&svm->vcpu, size, port, in);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002886}
2887
Avi Kivity851ba692009-08-24 11:10:17 +03002888static int nmi_interception(struct vcpu_svm *svm)
Joerg Roedelc47f0982008-04-30 17:56:00 +02002889{
2890 return 1;
2891}
2892
Avi Kivity851ba692009-08-24 11:10:17 +03002893static int intr_interception(struct vcpu_svm *svm)
Joerg Roedela0698052008-04-30 17:56:01 +02002894{
2895 ++svm->vcpu.stat.irq_exits;
2896 return 1;
2897}
2898
Avi Kivity851ba692009-08-24 11:10:17 +03002899static int nop_on_interception(struct vcpu_svm *svm)
Avi Kivity6aa8b732006-12-10 02:21:36 -08002900{
2901 return 1;
2902}
2903
Avi Kivity851ba692009-08-24 11:10:17 +03002904static int halt_interception(struct vcpu_svm *svm)
Avi Kivity6aa8b732006-12-10 02:21:36 -08002905{
Rusty Russelle756fc62007-07-30 20:07:08 +10002906 return kvm_emulate_halt(&svm->vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002907}
2908
Avi Kivity851ba692009-08-24 11:10:17 +03002909static int vmmcall_interception(struct vcpu_svm *svm)
Avi Kivity02e235b2007-02-19 14:37:47 +02002910{
Andrey Smetanin0d9c0552016-02-11 16:44:59 +03002911 return kvm_emulate_hypercall(&svm->vcpu);
Avi Kivity02e235b2007-02-19 14:37:47 +02002912}
2913
Joerg Roedel5bd2edc2010-09-10 17:31:02 +02002914static unsigned long nested_svm_get_tdp_cr3(struct kvm_vcpu *vcpu)
2915{
2916 struct vcpu_svm *svm = to_svm(vcpu);
2917
2918 return svm->nested.nested_cr3;
2919}
2920
Avi Kivitye4e517b2011-07-28 11:36:17 +03002921static u64 nested_svm_get_tdp_pdptr(struct kvm_vcpu *vcpu, int index)
2922{
2923 struct vcpu_svm *svm = to_svm(vcpu);
2924 u64 cr3 = svm->nested.nested_cr3;
2925 u64 pdpte;
2926 int ret;
2927
Tom Lendackyd0ec49d2017-07-17 16:10:27 -05002928 ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(__sme_clr(cr3)), &pdpte,
Paolo Bonzini54bf36a2015-04-08 15:39:23 +02002929 offset_in_page(cr3) + index * 8, 8);
Avi Kivitye4e517b2011-07-28 11:36:17 +03002930 if (ret)
2931 return 0;
2932 return pdpte;
2933}
2934
Joerg Roedel5bd2edc2010-09-10 17:31:02 +02002935static void nested_svm_set_tdp_cr3(struct kvm_vcpu *vcpu,
2936 unsigned long root)
2937{
2938 struct vcpu_svm *svm = to_svm(vcpu);
2939
Tom Lendackyd0ec49d2017-07-17 16:10:27 -05002940 svm->vmcb->control.nested_cr3 = __sme_set(root);
Joerg Roedelb2747162010-12-03 11:45:53 +01002941 mark_dirty(svm->vmcb, VMCB_NPT);
Joerg Roedel5bd2edc2010-09-10 17:31:02 +02002942}
2943
Avi Kivity6389ee92010-11-29 16:12:30 +02002944static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu,
2945 struct x86_exception *fault)
Joerg Roedel5bd2edc2010-09-10 17:31:02 +02002946{
2947 struct vcpu_svm *svm = to_svm(vcpu);
2948
Paolo Bonzini5e352512014-09-02 13:18:37 +02002949 if (svm->vmcb->control.exit_code != SVM_EXIT_NPF) {
2950 /*
2951 * TODO: track the cause of the nested page fault, and
2952 * correctly fill in the high bits of exit_info_1.
2953 */
2954 svm->vmcb->control.exit_code = SVM_EXIT_NPF;
2955 svm->vmcb->control.exit_code_hi = 0;
2956 svm->vmcb->control.exit_info_1 = (1ULL << 32);
2957 svm->vmcb->control.exit_info_2 = fault->address;
2958 }
2959
2960 svm->vmcb->control.exit_info_1 &= ~0xffffffffULL;
2961 svm->vmcb->control.exit_info_1 |= fault->error_code;
2962
2963 /*
2964 * The present bit is always zero for page structure faults on real
2965 * hardware.
2966 */
2967 if (svm->vmcb->control.exit_info_1 & (2ULL << 32))
2968 svm->vmcb->control.exit_info_1 &= ~1;
Joerg Roedel5bd2edc2010-09-10 17:31:02 +02002969
2970 nested_svm_vmexit(svm);
2971}
2972
Paolo Bonzini8a3c1a332013-10-02 16:56:13 +02002973static void nested_svm_init_mmu_context(struct kvm_vcpu *vcpu)
Joerg Roedel4b161842010-09-10 17:31:03 +02002974{
Paolo Bonziniad896af2013-10-02 16:56:14 +02002975 WARN_ON(mmu_is_nested(vcpu));
Vitaly Kuznetsov3cf85f92018-12-19 17:25:14 +01002976
2977 vcpu->arch.mmu = &vcpu->arch.guest_mmu;
Paolo Bonziniad896af2013-10-02 16:56:14 +02002978 kvm_init_shadow_mmu(vcpu);
Vitaly Kuznetsov44dd3ff2018-10-08 21:28:05 +02002979 vcpu->arch.mmu->set_cr3 = nested_svm_set_tdp_cr3;
2980 vcpu->arch.mmu->get_cr3 = nested_svm_get_tdp_cr3;
2981 vcpu->arch.mmu->get_pdptr = nested_svm_get_tdp_pdptr;
2982 vcpu->arch.mmu->inject_page_fault = nested_svm_inject_npf_exit;
2983 vcpu->arch.mmu->shadow_root_level = get_npt_level(vcpu);
2984 reset_shadow_zero_bits_mask(vcpu, vcpu->arch.mmu);
Joerg Roedel4b161842010-09-10 17:31:03 +02002985 vcpu->arch.walk_mmu = &vcpu->arch.nested_mmu;
Joerg Roedel4b161842010-09-10 17:31:03 +02002986}
2987
2988static void nested_svm_uninit_mmu_context(struct kvm_vcpu *vcpu)
2989{
Vitaly Kuznetsov3cf85f92018-12-19 17:25:14 +01002990 vcpu->arch.mmu = &vcpu->arch.root_mmu;
Vitaly Kuznetsov44dd3ff2018-10-08 21:28:05 +02002991 vcpu->arch.walk_mmu = &vcpu->arch.root_mmu;
Joerg Roedel4b161842010-09-10 17:31:03 +02002992}
2993
Alexander Grafc0725422008-11-25 20:17:03 +01002994static int nested_svm_check_permissions(struct vcpu_svm *svm)
2995{
Dan Carpentere9196ce2017-05-18 10:39:53 +03002996 if (!(svm->vcpu.arch.efer & EFER_SVME) ||
2997 !is_paging(&svm->vcpu)) {
Alexander Grafc0725422008-11-25 20:17:03 +01002998 kvm_queue_exception(&svm->vcpu, UD_VECTOR);
2999 return 1;
3000 }
3001
3002 if (svm->vmcb->save.cpl) {
3003 kvm_inject_gp(&svm->vcpu, 0);
3004 return 1;
3005 }
3006
Dan Carpentere9196ce2017-05-18 10:39:53 +03003007 return 0;
Alexander Grafc0725422008-11-25 20:17:03 +01003008}
3009
Alexander Grafcf74a782008-11-25 20:17:08 +01003010static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
3011 bool has_error_code, u32 error_code)
3012{
Joerg Roedelb8e88bc2010-02-19 16:23:02 +01003013 int vmexit;
3014
Joerg Roedel20307532010-11-29 17:51:48 +01003015 if (!is_guest_mode(&svm->vcpu))
Joerg Roedel0295ad72009-08-07 11:49:37 +02003016 return 0;
Alexander Grafcf74a782008-11-25 20:17:08 +01003017
Wanpeng Liadfe20f2017-07-13 18:30:41 -07003018 vmexit = nested_svm_intercept(svm);
3019 if (vmexit != NESTED_EXIT_DONE)
3020 return 0;
3021
Joerg Roedel0295ad72009-08-07 11:49:37 +02003022 svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + nr;
3023 svm->vmcb->control.exit_code_hi = 0;
3024 svm->vmcb->control.exit_info_1 = error_code;
Paolo Bonzinib96fb432017-07-27 12:29:32 +02003025
3026 /*
Jim Mattsonda998b42018-10-16 14:29:22 -07003027 * EXITINFO2 is undefined for all exception intercepts other
3028 * than #PF.
Paolo Bonzinib96fb432017-07-27 12:29:32 +02003029 */
Wanpeng Liadfe20f2017-07-13 18:30:41 -07003030 if (svm->vcpu.arch.exception.nested_apf)
3031 svm->vmcb->control.exit_info_2 = svm->vcpu.arch.apf.nested_apf_token;
Jim Mattsonda998b42018-10-16 14:29:22 -07003032 else if (svm->vcpu.arch.exception.has_payload)
3033 svm->vmcb->control.exit_info_2 = svm->vcpu.arch.exception.payload;
Wanpeng Liadfe20f2017-07-13 18:30:41 -07003034 else
3035 svm->vmcb->control.exit_info_2 = svm->vcpu.arch.cr2;
Joerg Roedel0295ad72009-08-07 11:49:37 +02003036
Wanpeng Liadfe20f2017-07-13 18:30:41 -07003037 svm->nested.exit_required = true;
Joerg Roedelb8e88bc2010-02-19 16:23:02 +01003038 return vmexit;
Alexander Grafcf74a782008-11-25 20:17:08 +01003039}
3040
Joerg Roedel8fe54652010-02-19 16:23:01 +01003041/* This function returns true if it is save to enable the irq window */
3042static inline bool nested_svm_intr(struct vcpu_svm *svm)
Alexander Grafcf74a782008-11-25 20:17:08 +01003043{
Joerg Roedel20307532010-11-29 17:51:48 +01003044 if (!is_guest_mode(&svm->vcpu))
Joerg Roedel8fe54652010-02-19 16:23:01 +01003045 return true;
Alexander Grafcf74a782008-11-25 20:17:08 +01003046
Joerg Roedel26666952009-08-07 11:49:46 +02003047 if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK))
Joerg Roedel8fe54652010-02-19 16:23:01 +01003048 return true;
Alexander Grafcf74a782008-11-25 20:17:08 +01003049
Joerg Roedel26666952009-08-07 11:49:46 +02003050 if (!(svm->vcpu.arch.hflags & HF_HIF_MASK))
Joerg Roedel8fe54652010-02-19 16:23:01 +01003051 return false;
Alexander Grafcf74a782008-11-25 20:17:08 +01003052
Gleb Natapova0a07cd2010-09-20 10:15:32 +02003053 /*
3054 * if vmexit was already requested (by intercepted exception
3055 * for instance) do not overwrite it with "external interrupt"
3056 * vmexit.
3057 */
3058 if (svm->nested.exit_required)
3059 return false;
3060
Joerg Roedel197717d2010-02-24 18:59:19 +01003061 svm->vmcb->control.exit_code = SVM_EXIT_INTR;
3062 svm->vmcb->control.exit_info_1 = 0;
3063 svm->vmcb->control.exit_info_2 = 0;
Joerg Roedel26666952009-08-07 11:49:46 +02003064
Joerg Roedelcd3ff652009-10-09 16:08:26 +02003065 if (svm->nested.intercept & 1ULL) {
3066 /*
3067 * The #vmexit can't be emulated here directly because this
Guo Chaoc5ec2e52012-06-28 15:16:43 +08003068 * code path runs with irqs and preemption disabled. A
Joerg Roedelcd3ff652009-10-09 16:08:26 +02003069 * #vmexit emulation might sleep. Only signal request for
3070 * the #vmexit here.
3071 */
3072 svm->nested.exit_required = true;
Joerg Roedel236649d2009-10-09 16:08:30 +02003073 trace_kvm_nested_intr_vmexit(svm->vmcb->save.rip);
Joerg Roedel8fe54652010-02-19 16:23:01 +01003074 return false;
Alexander Grafcf74a782008-11-25 20:17:08 +01003075 }
3076
Joerg Roedel8fe54652010-02-19 16:23:01 +01003077 return true;
Alexander Grafcf74a782008-11-25 20:17:08 +01003078}
3079
Joerg Roedel887f5002010-02-24 18:59:12 +01003080/* This function returns true if it is save to enable the nmi window */
3081static inline bool nested_svm_nmi(struct vcpu_svm *svm)
3082{
Joerg Roedel20307532010-11-29 17:51:48 +01003083 if (!is_guest_mode(&svm->vcpu))
Joerg Roedel887f5002010-02-24 18:59:12 +01003084 return true;
3085
3086 if (!(svm->nested.intercept & (1ULL << INTERCEPT_NMI)))
3087 return true;
3088
3089 svm->vmcb->control.exit_code = SVM_EXIT_NMI;
3090 svm->nested.exit_required = true;
3091
3092 return false;
3093}
3094
Joerg Roedelce2ac082010-03-01 15:34:39 +01003095static int nested_svm_intercept_ioio(struct vcpu_svm *svm)
Alexander Grafcf74a782008-11-25 20:17:08 +01003096{
Jan Kiszka9bf41832014-06-30 10:54:17 +02003097 unsigned port, size, iopm_len;
3098 u16 val, mask;
3099 u8 start_bit;
Joerg Roedelce2ac082010-03-01 15:34:39 +01003100 u64 gpa;
3101
3102 if (!(svm->nested.intercept & (1ULL << INTERCEPT_IOIO_PROT)))
3103 return NESTED_EXIT_HOST;
3104
3105 port = svm->vmcb->control.exit_info_1 >> 16;
Jan Kiszka9bf41832014-06-30 10:54:17 +02003106 size = (svm->vmcb->control.exit_info_1 & SVM_IOIO_SIZE_MASK) >>
3107 SVM_IOIO_SIZE_SHIFT;
Joerg Roedelce2ac082010-03-01 15:34:39 +01003108 gpa = svm->nested.vmcb_iopm + (port / 8);
Jan Kiszka9bf41832014-06-30 10:54:17 +02003109 start_bit = port % 8;
3110 iopm_len = (start_bit + size > 8) ? 2 : 1;
3111 mask = (0xf >> (4 - size)) << start_bit;
3112 val = 0;
Joerg Roedelce2ac082010-03-01 15:34:39 +01003113
Paolo Bonzini54bf36a2015-04-08 15:39:23 +02003114 if (kvm_vcpu_read_guest(&svm->vcpu, gpa, &val, iopm_len))
Jan Kiszka9bf41832014-06-30 10:54:17 +02003115 return NESTED_EXIT_DONE;
Joerg Roedelce2ac082010-03-01 15:34:39 +01003116
Jan Kiszka9bf41832014-06-30 10:54:17 +02003117 return (val & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST;
Joerg Roedelce2ac082010-03-01 15:34:39 +01003118}
3119
Joerg Roedeld2477822010-03-01 15:34:34 +01003120static int nested_svm_exit_handled_msr(struct vcpu_svm *svm)
Alexander Grafcf74a782008-11-25 20:17:08 +01003121{
Joerg Roedel0d6b3532010-03-01 15:34:38 +01003122 u32 offset, msr, value;
3123 int write, mask;
Joerg Roedel4c2161a2009-08-07 11:49:35 +02003124
Joerg Roedel3d62d9a2009-08-07 11:49:39 +02003125 if (!(svm->nested.intercept & (1ULL << INTERCEPT_MSR_PROT)))
Joerg Roedeld2477822010-03-01 15:34:34 +01003126 return NESTED_EXIT_HOST;
Joerg Roedel3d62d9a2009-08-07 11:49:39 +02003127
Joerg Roedel0d6b3532010-03-01 15:34:38 +01003128 msr = svm->vcpu.arch.regs[VCPU_REGS_RCX];
3129 offset = svm_msrpm_offset(msr);
3130 write = svm->vmcb->control.exit_info_1 & 1;
3131 mask = 1 << ((2 * (msr & 0xf)) + write);
Joerg Roedel3d62d9a2009-08-07 11:49:39 +02003132
Joerg Roedel0d6b3532010-03-01 15:34:38 +01003133 if (offset == MSR_INVALID)
3134 return NESTED_EXIT_DONE;
Joerg Roedel4c2161a2009-08-07 11:49:35 +02003135
Joerg Roedel0d6b3532010-03-01 15:34:38 +01003136 /* Offset is in 32 bit units but need in 8 bit units */
3137 offset *= 4;
Joerg Roedel4c2161a2009-08-07 11:49:35 +02003138
Paolo Bonzini54bf36a2015-04-08 15:39:23 +02003139 if (kvm_vcpu_read_guest(&svm->vcpu, svm->nested.vmcb_msrpm + offset, &value, 4))
Joerg Roedel0d6b3532010-03-01 15:34:38 +01003140 return NESTED_EXIT_DONE;
Joerg Roedel3d62d9a2009-08-07 11:49:39 +02003141
Joerg Roedel0d6b3532010-03-01 15:34:38 +01003142 return (value & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST;
Joerg Roedel4c2161a2009-08-07 11:49:35 +02003143}
3144
Ladi Prosekab2f4d732017-06-21 09:06:58 +02003145/* DB exceptions for our internal use must not cause vmexit */
3146static int nested_svm_intercept_db(struct vcpu_svm *svm)
3147{
3148 unsigned long dr6;
3149
3150 /* if we're not singlestepping, it's not ours */
3151 if (!svm->nmi_singlestep)
3152 return NESTED_EXIT_DONE;
3153
3154 /* if it's not a singlestep exception, it's not ours */
3155 if (kvm_get_dr(&svm->vcpu, 6, &dr6))
3156 return NESTED_EXIT_DONE;
3157 if (!(dr6 & DR6_BS))
3158 return NESTED_EXIT_DONE;
3159
3160 /* if the guest is singlestepping, it should get the vmexit */
3161 if (svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF) {
3162 disable_nmi_singlestep(svm);
3163 return NESTED_EXIT_DONE;
3164 }
3165
3166 /* it's ours, the nested hypervisor must not see this one */
3167 return NESTED_EXIT_HOST;
3168}
3169
Joerg Roedel410e4d52009-08-07 11:49:44 +02003170static int nested_svm_exit_special(struct vcpu_svm *svm)
Joerg Roedel4c2161a2009-08-07 11:49:35 +02003171{
Alexander Grafcf74a782008-11-25 20:17:08 +01003172 u32 exit_code = svm->vmcb->control.exit_code;
Joerg Roedel4c2161a2009-08-07 11:49:35 +02003173
Joerg Roedel410e4d52009-08-07 11:49:44 +02003174 switch (exit_code) {
3175 case SVM_EXIT_INTR:
3176 case SVM_EXIT_NMI:
Joerg Roedelff47a492010-04-22 12:33:14 +02003177 case SVM_EXIT_EXCP_BASE + MC_VECTOR:
Joerg Roedel410e4d52009-08-07 11:49:44 +02003178 return NESTED_EXIT_HOST;
Joerg Roedel410e4d52009-08-07 11:49:44 +02003179 case SVM_EXIT_NPF:
Joerg Roedele0231712010-02-24 18:59:10 +01003180 /* For now we are always handling NPFs when using them */
Joerg Roedel410e4d52009-08-07 11:49:44 +02003181 if (npt_enabled)
3182 return NESTED_EXIT_HOST;
3183 break;
Joerg Roedel410e4d52009-08-07 11:49:44 +02003184 case SVM_EXIT_EXCP_BASE + PF_VECTOR:
Gleb Natapov631bc482010-10-14 11:22:52 +02003185 /* When we're shadowing, trap PFs, but not async PF */
Wanpeng Li1261bfa2017-07-13 18:30:40 -07003186 if (!npt_enabled && svm->vcpu.arch.apf.host_apf_reason == 0)
Joerg Roedel410e4d52009-08-07 11:49:44 +02003187 return NESTED_EXIT_HOST;
3188 break;
3189 default:
3190 break;
Alexander Grafcf74a782008-11-25 20:17:08 +01003191 }
3192
Joerg Roedel410e4d52009-08-07 11:49:44 +02003193 return NESTED_EXIT_CONTINUE;
3194}
3195
3196/*
3197 * If this function returns true, this #vmexit was already handled
3198 */
Joerg Roedelb8e88bc2010-02-19 16:23:02 +01003199static int nested_svm_intercept(struct vcpu_svm *svm)
Joerg Roedel410e4d52009-08-07 11:49:44 +02003200{
3201 u32 exit_code = svm->vmcb->control.exit_code;
3202 int vmexit = NESTED_EXIT_HOST;
3203
Alexander Grafcf74a782008-11-25 20:17:08 +01003204 switch (exit_code) {
Joerg Roedel9c4e40b92009-08-07 11:49:36 +02003205 case SVM_EXIT_MSR:
Joerg Roedel3d62d9a2009-08-07 11:49:39 +02003206 vmexit = nested_svm_exit_handled_msr(svm);
Joerg Roedel9c4e40b92009-08-07 11:49:36 +02003207 break;
Joerg Roedelce2ac082010-03-01 15:34:39 +01003208 case SVM_EXIT_IOIO:
3209 vmexit = nested_svm_intercept_ioio(svm);
3210 break;
Roedel, Joerg4ee546b2010-12-03 10:50:51 +01003211 case SVM_EXIT_READ_CR0 ... SVM_EXIT_WRITE_CR8: {
3212 u32 bit = 1U << (exit_code - SVM_EXIT_READ_CR0);
3213 if (svm->nested.intercept_cr & bit)
Joerg Roedel410e4d52009-08-07 11:49:44 +02003214 vmexit = NESTED_EXIT_DONE;
Alexander Grafcf74a782008-11-25 20:17:08 +01003215 break;
3216 }
Joerg Roedel3aed0412010-11-30 18:03:58 +01003217 case SVM_EXIT_READ_DR0 ... SVM_EXIT_WRITE_DR7: {
3218 u32 bit = 1U << (exit_code - SVM_EXIT_READ_DR0);
3219 if (svm->nested.intercept_dr & bit)
Joerg Roedel410e4d52009-08-07 11:49:44 +02003220 vmexit = NESTED_EXIT_DONE;
Alexander Grafcf74a782008-11-25 20:17:08 +01003221 break;
3222 }
3223 case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: {
3224 u32 excp_bits = 1 << (exit_code - SVM_EXIT_EXCP_BASE);
Ladi Prosekab2f4d732017-06-21 09:06:58 +02003225 if (svm->nested.intercept_exceptions & excp_bits) {
3226 if (exit_code == SVM_EXIT_EXCP_BASE + DB_VECTOR)
3227 vmexit = nested_svm_intercept_db(svm);
3228 else
3229 vmexit = NESTED_EXIT_DONE;
3230 }
Gleb Natapov631bc482010-10-14 11:22:52 +02003231 /* async page fault always cause vmexit */
3232 else if ((exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR) &&
Wanpeng Liadfe20f2017-07-13 18:30:41 -07003233 svm->vcpu.arch.exception.nested_apf != 0)
Gleb Natapov631bc482010-10-14 11:22:52 +02003234 vmexit = NESTED_EXIT_DONE;
Alexander Grafcf74a782008-11-25 20:17:08 +01003235 break;
3236 }
Joerg Roedel228070b2010-04-22 12:33:10 +02003237 case SVM_EXIT_ERR: {
3238 vmexit = NESTED_EXIT_DONE;
3239 break;
3240 }
Alexander Grafcf74a782008-11-25 20:17:08 +01003241 default: {
3242 u64 exit_bits = 1ULL << (exit_code - SVM_EXIT_INTR);
Joerg Roedelaad42c62009-08-07 11:49:34 +02003243 if (svm->nested.intercept & exit_bits)
Joerg Roedel410e4d52009-08-07 11:49:44 +02003244 vmexit = NESTED_EXIT_DONE;
Alexander Grafcf74a782008-11-25 20:17:08 +01003245 }
3246 }
3247
Joerg Roedelb8e88bc2010-02-19 16:23:02 +01003248 return vmexit;
3249}
3250
3251static int nested_svm_exit_handled(struct vcpu_svm *svm)
3252{
3253 int vmexit;
3254
3255 vmexit = nested_svm_intercept(svm);
3256
3257 if (vmexit == NESTED_EXIT_DONE)
Joerg Roedel9c4e40b92009-08-07 11:49:36 +02003258 nested_svm_vmexit(svm);
Joerg Roedel9c4e40b92009-08-07 11:49:36 +02003259
3260 return vmexit;
Alexander Grafcf74a782008-11-25 20:17:08 +01003261}
3262
Joerg Roedel0460a972009-08-07 11:49:31 +02003263static inline void copy_vmcb_control_area(struct vmcb *dst_vmcb, struct vmcb *from_vmcb)
3264{
3265 struct vmcb_control_area *dst = &dst_vmcb->control;
3266 struct vmcb_control_area *from = &from_vmcb->control;
3267
Roedel, Joerg4ee546b2010-12-03 10:50:51 +01003268 dst->intercept_cr = from->intercept_cr;
Joerg Roedel3aed0412010-11-30 18:03:58 +01003269 dst->intercept_dr = from->intercept_dr;
Joerg Roedel0460a972009-08-07 11:49:31 +02003270 dst->intercept_exceptions = from->intercept_exceptions;
3271 dst->intercept = from->intercept;
3272 dst->iopm_base_pa = from->iopm_base_pa;
3273 dst->msrpm_base_pa = from->msrpm_base_pa;
3274 dst->tsc_offset = from->tsc_offset;
3275 dst->asid = from->asid;
3276 dst->tlb_ctl = from->tlb_ctl;
3277 dst->int_ctl = from->int_ctl;
3278 dst->int_vector = from->int_vector;
3279 dst->int_state = from->int_state;
3280 dst->exit_code = from->exit_code;
3281 dst->exit_code_hi = from->exit_code_hi;
3282 dst->exit_info_1 = from->exit_info_1;
3283 dst->exit_info_2 = from->exit_info_2;
3284 dst->exit_int_info = from->exit_int_info;
3285 dst->exit_int_info_err = from->exit_int_info_err;
3286 dst->nested_ctl = from->nested_ctl;
3287 dst->event_inj = from->event_inj;
3288 dst->event_inj_err = from->event_inj_err;
3289 dst->nested_cr3 = from->nested_cr3;
Janakarajan Natarajan0dc92112017-07-06 15:50:45 -05003290 dst->virt_ext = from->virt_ext;
Tambe, Williame0813542018-11-13 16:51:20 +00003291 dst->pause_filter_count = from->pause_filter_count;
3292 dst->pause_filter_thresh = from->pause_filter_thresh;
Joerg Roedel0460a972009-08-07 11:49:31 +02003293}
3294
Joerg Roedel34f80cf2009-08-07 11:49:38 +02003295static int nested_svm_vmexit(struct vcpu_svm *svm)
Alexander Grafcf74a782008-11-25 20:17:08 +01003296{
KarimAllah Ahmed8c5fbf12019-01-31 21:24:40 +01003297 int rc;
Joerg Roedel34f80cf2009-08-07 11:49:38 +02003298 struct vmcb *nested_vmcb;
Joerg Roedele6aa9ab2009-08-07 11:49:33 +02003299 struct vmcb *hsave = svm->nested.hsave;
Joerg Roedel33740e42009-08-07 11:49:29 +02003300 struct vmcb *vmcb = svm->vmcb;
KarimAllah Ahmed8c5fbf12019-01-31 21:24:40 +01003301 struct kvm_host_map map;
Alexander Grafcf74a782008-11-25 20:17:08 +01003302
Joerg Roedel17897f32009-10-09 16:08:29 +02003303 trace_kvm_nested_vmexit_inject(vmcb->control.exit_code,
3304 vmcb->control.exit_info_1,
3305 vmcb->control.exit_info_2,
3306 vmcb->control.exit_int_info,
Stefan Hajnoczie097e5f2011-07-22 12:46:52 +01003307 vmcb->control.exit_int_info_err,
3308 KVM_ISA_SVM);
Joerg Roedel17897f32009-10-09 16:08:29 +02003309
Vitaly Kuznetsov8f383022019-06-04 18:09:39 +02003310 rc = kvm_vcpu_map(&svm->vcpu, gpa_to_gfn(svm->nested.vmcb), &map);
KarimAllah Ahmed8c5fbf12019-01-31 21:24:40 +01003311 if (rc) {
3312 if (rc == -EINVAL)
3313 kvm_inject_gp(&svm->vcpu, 0);
Joerg Roedel34f80cf2009-08-07 11:49:38 +02003314 return 1;
KarimAllah Ahmed8c5fbf12019-01-31 21:24:40 +01003315 }
3316
3317 nested_vmcb = map.hva;
Joerg Roedel34f80cf2009-08-07 11:49:38 +02003318
Joerg Roedel20307532010-11-29 17:51:48 +01003319 /* Exit Guest-Mode */
3320 leave_guest_mode(&svm->vcpu);
Joerg Roedel06fc77722010-02-19 16:23:07 +01003321 svm->nested.vmcb = 0;
3322
Alexander Grafcf74a782008-11-25 20:17:08 +01003323 /* Give the current vmcb to the guest */
Joerg Roedel33740e42009-08-07 11:49:29 +02003324 disable_gif(svm);
3325
3326 nested_vmcb->save.es = vmcb->save.es;
3327 nested_vmcb->save.cs = vmcb->save.cs;
3328 nested_vmcb->save.ss = vmcb->save.ss;
3329 nested_vmcb->save.ds = vmcb->save.ds;
3330 nested_vmcb->save.gdtr = vmcb->save.gdtr;
3331 nested_vmcb->save.idtr = vmcb->save.idtr;
Joerg Roedel3f6a9d12010-07-27 18:14:20 +02003332 nested_vmcb->save.efer = svm->vcpu.arch.efer;
Joerg Roedelcdbbdc12010-02-19 16:23:03 +01003333 nested_vmcb->save.cr0 = kvm_read_cr0(&svm->vcpu);
Avi Kivity9f8fe502010-12-05 17:30:00 +02003334 nested_vmcb->save.cr3 = kvm_read_cr3(&svm->vcpu);
Joerg Roedel33740e42009-08-07 11:49:29 +02003335 nested_vmcb->save.cr2 = vmcb->save.cr2;
Joerg Roedelcdbbdc12010-02-19 16:23:03 +01003336 nested_vmcb->save.cr4 = svm->vcpu.arch.cr4;
Avi Kivityf6e78472010-08-02 15:30:20 +03003337 nested_vmcb->save.rflags = kvm_get_rflags(&svm->vcpu);
Joerg Roedel33740e42009-08-07 11:49:29 +02003338 nested_vmcb->save.rip = vmcb->save.rip;
3339 nested_vmcb->save.rsp = vmcb->save.rsp;
3340 nested_vmcb->save.rax = vmcb->save.rax;
3341 nested_vmcb->save.dr7 = vmcb->save.dr7;
3342 nested_vmcb->save.dr6 = vmcb->save.dr6;
3343 nested_vmcb->save.cpl = vmcb->save.cpl;
3344
3345 nested_vmcb->control.int_ctl = vmcb->control.int_ctl;
3346 nested_vmcb->control.int_vector = vmcb->control.int_vector;
3347 nested_vmcb->control.int_state = vmcb->control.int_state;
3348 nested_vmcb->control.exit_code = vmcb->control.exit_code;
3349 nested_vmcb->control.exit_code_hi = vmcb->control.exit_code_hi;
3350 nested_vmcb->control.exit_info_1 = vmcb->control.exit_info_1;
3351 nested_vmcb->control.exit_info_2 = vmcb->control.exit_info_2;
3352 nested_vmcb->control.exit_int_info = vmcb->control.exit_int_info;
3353 nested_vmcb->control.exit_int_info_err = vmcb->control.exit_int_info_err;
Joerg Roedel6092d3d2015-10-14 15:10:54 +02003354
3355 if (svm->nrips_enabled)
3356 nested_vmcb->control.next_rip = vmcb->control.next_rip;
Alexander Graf8d23c462009-10-09 16:08:25 +02003357
3358 /*
3359 * If we emulate a VMRUN/#VMEXIT in the same host #vmexit cycle we have
3360 * to make sure that we do not lose injected events. So check event_inj
3361 * here and copy it to exit_int_info if it is valid.
3362 * Exit_int_info and event_inj can't be both valid because the case
3363 * below only happens on a VMRUN instruction intercept which has
3364 * no valid exit_int_info set.
3365 */
3366 if (vmcb->control.event_inj & SVM_EVTINJ_VALID) {
3367 struct vmcb_control_area *nc = &nested_vmcb->control;
3368
3369 nc->exit_int_info = vmcb->control.event_inj;
3370 nc->exit_int_info_err = vmcb->control.event_inj_err;
3371 }
3372
Joerg Roedel33740e42009-08-07 11:49:29 +02003373 nested_vmcb->control.tlb_ctl = 0;
3374 nested_vmcb->control.event_inj = 0;
3375 nested_vmcb->control.event_inj_err = 0;
Alexander Grafcf74a782008-11-25 20:17:08 +01003376
Tambe, Williame0813542018-11-13 16:51:20 +00003377 nested_vmcb->control.pause_filter_count =
3378 svm->vmcb->control.pause_filter_count;
3379 nested_vmcb->control.pause_filter_thresh =
3380 svm->vmcb->control.pause_filter_thresh;
3381
Alexander Grafcf74a782008-11-25 20:17:08 +01003382 /* We always set V_INTR_MASKING and remember the old value in hflags */
3383 if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK))
3384 nested_vmcb->control.int_ctl &= ~V_INTR_MASKING_MASK;
3385
Alexander Grafcf74a782008-11-25 20:17:08 +01003386 /* Restore the original control entries */
Joerg Roedel0460a972009-08-07 11:49:31 +02003387 copy_vmcb_control_area(vmcb, hsave);
Alexander Grafcf74a782008-11-25 20:17:08 +01003388
KarimAllah Ahmede79f2452018-04-14 05:10:52 +02003389 svm->vcpu.arch.tsc_offset = svm->vmcb->control.tsc_offset;
Alexander Graf219b65d2009-06-15 15:21:25 +02003390 kvm_clear_exception_queue(&svm->vcpu);
3391 kvm_clear_interrupt_queue(&svm->vcpu);
Alexander Grafcf74a782008-11-25 20:17:08 +01003392
Joerg Roedel4b161842010-09-10 17:31:03 +02003393 svm->nested.nested_cr3 = 0;
3394
Alexander Grafcf74a782008-11-25 20:17:08 +01003395 /* Restore selected save entries */
3396 svm->vmcb->save.es = hsave->save.es;
3397 svm->vmcb->save.cs = hsave->save.cs;
3398 svm->vmcb->save.ss = hsave->save.ss;
3399 svm->vmcb->save.ds = hsave->save.ds;
3400 svm->vmcb->save.gdtr = hsave->save.gdtr;
3401 svm->vmcb->save.idtr = hsave->save.idtr;
Avi Kivityf6e78472010-08-02 15:30:20 +03003402 kvm_set_rflags(&svm->vcpu, hsave->save.rflags);
Alexander Grafcf74a782008-11-25 20:17:08 +01003403 svm_set_efer(&svm->vcpu, hsave->save.efer);
3404 svm_set_cr0(&svm->vcpu, hsave->save.cr0 | X86_CR0_PE);
3405 svm_set_cr4(&svm->vcpu, hsave->save.cr4);
3406 if (npt_enabled) {
3407 svm->vmcb->save.cr3 = hsave->save.cr3;
3408 svm->vcpu.arch.cr3 = hsave->save.cr3;
3409 } else {
Avi Kivity23902182010-06-10 17:02:16 +03003410 (void)kvm_set_cr3(&svm->vcpu, hsave->save.cr3);
Alexander Grafcf74a782008-11-25 20:17:08 +01003411 }
Sean Christophersonde3cd112019-04-30 10:36:17 -07003412 kvm_rax_write(&svm->vcpu, hsave->save.rax);
Paolo Bonzinie9c16c72019-04-30 22:07:26 +02003413 kvm_rsp_write(&svm->vcpu, hsave->save.rsp);
3414 kvm_rip_write(&svm->vcpu, hsave->save.rip);
Alexander Grafcf74a782008-11-25 20:17:08 +01003415 svm->vmcb->save.dr7 = 0;
3416 svm->vmcb->save.cpl = 0;
3417 svm->vmcb->control.exit_int_info = 0;
3418
Roedel, Joerg8d28fec2010-12-03 13:15:21 +01003419 mark_all_dirty(svm->vmcb);
3420
KarimAllah Ahmed8c5fbf12019-01-31 21:24:40 +01003421 kvm_vcpu_unmap(&svm->vcpu, &map, true);
Alexander Grafcf74a782008-11-25 20:17:08 +01003422
Joerg Roedel4b161842010-09-10 17:31:03 +02003423 nested_svm_uninit_mmu_context(&svm->vcpu);
Alexander Grafcf74a782008-11-25 20:17:08 +01003424 kvm_mmu_reset_context(&svm->vcpu);
3425 kvm_mmu_load(&svm->vcpu);
3426
Vitaly Kuznetsov619ad842019-01-07 19:44:51 +01003427 /*
3428 * Drop what we picked up for L2 via svm_complete_interrupts() so it
3429 * doesn't end up in L1.
3430 */
3431 svm->vcpu.arch.nmi_injected = false;
3432 kvm_clear_exception_queue(&svm->vcpu);
3433 kvm_clear_interrupt_queue(&svm->vcpu);
3434
Alexander Grafcf74a782008-11-25 20:17:08 +01003435 return 0;
3436}
Alexander Graf3d6368e2008-11-25 20:17:07 +01003437
Joerg Roedel9738b2c2009-08-07 11:49:41 +02003438static bool nested_svm_vmrun_msrpm(struct vcpu_svm *svm)
Alexander Graf3d6368e2008-11-25 20:17:07 +01003439{
Joerg Roedel323c3d82010-03-01 15:34:37 +01003440 /*
3441 * This function merges the msr permission bitmaps of kvm and the
Guo Chaoc5ec2e52012-06-28 15:16:43 +08003442 * nested vmcb. It is optimized in that it only merges the parts where
Joerg Roedel323c3d82010-03-01 15:34:37 +01003443 * the kvm msr permission bitmap may contain zero bits
3444 */
Alexander Graf3d6368e2008-11-25 20:17:07 +01003445 int i;
Joerg Roedel9738b2c2009-08-07 11:49:41 +02003446
Joerg Roedel323c3d82010-03-01 15:34:37 +01003447 if (!(svm->nested.intercept & (1ULL << INTERCEPT_MSR_PROT)))
3448 return true;
Joerg Roedel9738b2c2009-08-07 11:49:41 +02003449
Joerg Roedel323c3d82010-03-01 15:34:37 +01003450 for (i = 0; i < MSRPM_OFFSETS; i++) {
3451 u32 value, p;
3452 u64 offset;
3453
3454 if (msrpm_offsets[i] == 0xffffffff)
3455 break;
3456
Joerg Roedel0d6b3532010-03-01 15:34:38 +01003457 p = msrpm_offsets[i];
3458 offset = svm->nested.vmcb_msrpm + (p * 4);
Joerg Roedel323c3d82010-03-01 15:34:37 +01003459
Paolo Bonzini54bf36a2015-04-08 15:39:23 +02003460 if (kvm_vcpu_read_guest(&svm->vcpu, offset, &value, 4))
Joerg Roedel323c3d82010-03-01 15:34:37 +01003461 return false;
3462
3463 svm->nested.msrpm[p] = svm->msrpm[p] | value;
3464 }
Joerg Roedel9738b2c2009-08-07 11:49:41 +02003465
Tom Lendackyd0ec49d2017-07-17 16:10:27 -05003466 svm->vmcb->control.msrpm_base_pa = __sme_set(__pa(svm->nested.msrpm));
Alexander Graf3d6368e2008-11-25 20:17:07 +01003467
Joerg Roedel9738b2c2009-08-07 11:49:41 +02003468 return true;
Alexander Graf3d6368e2008-11-25 20:17:07 +01003469}
3470
Joerg Roedel52c65a302010-08-02 16:46:44 +02003471static bool nested_vmcb_checks(struct vmcb *vmcb)
3472{
3473 if ((vmcb->control.intercept & (1ULL << INTERCEPT_VMRUN)) == 0)
3474 return false;
3475
Joerg Roedeldbe77582010-08-02 16:46:45 +02003476 if (vmcb->control.asid == 0)
3477 return false;
3478
Tom Lendackycea3a192017-12-04 10:57:24 -06003479 if ((vmcb->control.nested_ctl & SVM_NESTED_CTL_NP_ENABLE) &&
3480 !npt_enabled)
Joerg Roedel4b161842010-09-10 17:31:03 +02003481 return false;
3482
Joerg Roedel52c65a302010-08-02 16:46:44 +02003483 return true;
3484}
3485
Ladi Prosekc2634062017-10-11 16:54:44 +02003486static void enter_svm_guest_mode(struct vcpu_svm *svm, u64 vmcb_gpa,
KarimAllah Ahmed8c5fbf12019-01-31 21:24:40 +01003487 struct vmcb *nested_vmcb, struct kvm_host_map *map)
Alexander Graf3d6368e2008-11-25 20:17:07 +01003488{
Avi Kivityf6e78472010-08-02 15:30:20 +03003489 if (kvm_get_rflags(&svm->vcpu) & X86_EFLAGS_IF)
Alexander Graf3d6368e2008-11-25 20:17:07 +01003490 svm->vcpu.arch.hflags |= HF_HIF_MASK;
3491 else
3492 svm->vcpu.arch.hflags &= ~HF_HIF_MASK;
3493
Tom Lendackycea3a192017-12-04 10:57:24 -06003494 if (nested_vmcb->control.nested_ctl & SVM_NESTED_CTL_NP_ENABLE) {
Joerg Roedel4b161842010-09-10 17:31:03 +02003495 svm->nested.nested_cr3 = nested_vmcb->control.nested_cr3;
3496 nested_svm_init_mmu_context(&svm->vcpu);
3497 }
3498
Alexander Graf3d6368e2008-11-25 20:17:07 +01003499 /* Load the nested guest state */
3500 svm->vmcb->save.es = nested_vmcb->save.es;
3501 svm->vmcb->save.cs = nested_vmcb->save.cs;
3502 svm->vmcb->save.ss = nested_vmcb->save.ss;
3503 svm->vmcb->save.ds = nested_vmcb->save.ds;
3504 svm->vmcb->save.gdtr = nested_vmcb->save.gdtr;
3505 svm->vmcb->save.idtr = nested_vmcb->save.idtr;
Avi Kivityf6e78472010-08-02 15:30:20 +03003506 kvm_set_rflags(&svm->vcpu, nested_vmcb->save.rflags);
Alexander Graf3d6368e2008-11-25 20:17:07 +01003507 svm_set_efer(&svm->vcpu, nested_vmcb->save.efer);
3508 svm_set_cr0(&svm->vcpu, nested_vmcb->save.cr0);
3509 svm_set_cr4(&svm->vcpu, nested_vmcb->save.cr4);
3510 if (npt_enabled) {
3511 svm->vmcb->save.cr3 = nested_vmcb->save.cr3;
3512 svm->vcpu.arch.cr3 = nested_vmcb->save.cr3;
Joerg Roedel0e5cbe32010-02-24 18:59:11 +01003513 } else
Avi Kivity23902182010-06-10 17:02:16 +03003514 (void)kvm_set_cr3(&svm->vcpu, nested_vmcb->save.cr3);
Joerg Roedel0e5cbe32010-02-24 18:59:11 +01003515
3516 /* Guest paging mode is active - reset mmu */
3517 kvm_mmu_reset_context(&svm->vcpu);
3518
Joerg Roedeldefbba52009-08-07 11:49:30 +02003519 svm->vmcb->save.cr2 = svm->vcpu.arch.cr2 = nested_vmcb->save.cr2;
Sean Christophersonde3cd112019-04-30 10:36:17 -07003520 kvm_rax_write(&svm->vcpu, nested_vmcb->save.rax);
Paolo Bonzinie9c16c72019-04-30 22:07:26 +02003521 kvm_rsp_write(&svm->vcpu, nested_vmcb->save.rsp);
3522 kvm_rip_write(&svm->vcpu, nested_vmcb->save.rip);
Joerg Roedele0231712010-02-24 18:59:10 +01003523
Alexander Graf3d6368e2008-11-25 20:17:07 +01003524 /* In case we don't even reach vcpu_run, the fields are not updated */
3525 svm->vmcb->save.rax = nested_vmcb->save.rax;
3526 svm->vmcb->save.rsp = nested_vmcb->save.rsp;
3527 svm->vmcb->save.rip = nested_vmcb->save.rip;
3528 svm->vmcb->save.dr7 = nested_vmcb->save.dr7;
3529 svm->vmcb->save.dr6 = nested_vmcb->save.dr6;
3530 svm->vmcb->save.cpl = nested_vmcb->save.cpl;
3531
Joerg Roedelf7138532010-03-01 15:34:40 +01003532 svm->nested.vmcb_msrpm = nested_vmcb->control.msrpm_base_pa & ~0x0fffULL;
Joerg Roedelce2ac082010-03-01 15:34:39 +01003533 svm->nested.vmcb_iopm = nested_vmcb->control.iopm_base_pa & ~0x0fffULL;
Alexander Graf3d6368e2008-11-25 20:17:07 +01003534
Joerg Roedelaad42c62009-08-07 11:49:34 +02003535 /* cache intercepts */
Roedel, Joerg4ee546b2010-12-03 10:50:51 +01003536 svm->nested.intercept_cr = nested_vmcb->control.intercept_cr;
Joerg Roedel3aed0412010-11-30 18:03:58 +01003537 svm->nested.intercept_dr = nested_vmcb->control.intercept_dr;
Joerg Roedelaad42c62009-08-07 11:49:34 +02003538 svm->nested.intercept_exceptions = nested_vmcb->control.intercept_exceptions;
3539 svm->nested.intercept = nested_vmcb->control.intercept;
3540
Wanpeng Lic2ba05c2017-12-12 17:33:03 -08003541 svm_flush_tlb(&svm->vcpu, true);
Alexander Graf3d6368e2008-11-25 20:17:07 +01003542 svm->vmcb->control.int_ctl = nested_vmcb->control.int_ctl | V_INTR_MASKING_MASK;
Alexander Graf3d6368e2008-11-25 20:17:07 +01003543 if (nested_vmcb->control.int_ctl & V_INTR_MASKING_MASK)
3544 svm->vcpu.arch.hflags |= HF_VINTR_MASK;
3545 else
3546 svm->vcpu.arch.hflags &= ~HF_VINTR_MASK;
3547
Joerg Roedel88ab24a2010-02-19 16:23:06 +01003548 if (svm->vcpu.arch.hflags & HF_VINTR_MASK) {
3549 /* We only want the cr8 intercept bits of the guest */
Roedel, Joerg4ee546b2010-12-03 10:50:51 +01003550 clr_cr_intercept(svm, INTERCEPT_CR8_READ);
3551 clr_cr_intercept(svm, INTERCEPT_CR8_WRITE);
Joerg Roedel88ab24a2010-02-19 16:23:06 +01003552 }
3553
Joerg Roedel0d945bd2010-05-05 16:04:45 +02003554 /* We don't want to see VMMCALLs from a nested guest */
Joerg Roedel8a05a1b82010-11-30 18:04:00 +01003555 clr_intercept(svm, INTERCEPT_VMMCALL);
Joerg Roedel0d945bd2010-05-05 16:04:45 +02003556
KarimAllah Ahmede79f2452018-04-14 05:10:52 +02003557 svm->vcpu.arch.tsc_offset += nested_vmcb->control.tsc_offset;
3558 svm->vmcb->control.tsc_offset = svm->vcpu.arch.tsc_offset;
3559
Janakarajan Natarajan0dc92112017-07-06 15:50:45 -05003560 svm->vmcb->control.virt_ext = nested_vmcb->control.virt_ext;
Alexander Graf3d6368e2008-11-25 20:17:07 +01003561 svm->vmcb->control.int_vector = nested_vmcb->control.int_vector;
3562 svm->vmcb->control.int_state = nested_vmcb->control.int_state;
Alexander Graf3d6368e2008-11-25 20:17:07 +01003563 svm->vmcb->control.event_inj = nested_vmcb->control.event_inj;
3564 svm->vmcb->control.event_inj_err = nested_vmcb->control.event_inj_err;
3565
Tambe, Williame0813542018-11-13 16:51:20 +00003566 svm->vmcb->control.pause_filter_count =
3567 nested_vmcb->control.pause_filter_count;
3568 svm->vmcb->control.pause_filter_thresh =
3569 nested_vmcb->control.pause_filter_thresh;
3570
KarimAllah Ahmed8c5fbf12019-01-31 21:24:40 +01003571 kvm_vcpu_unmap(&svm->vcpu, map, true);
Joerg Roedel9738b2c2009-08-07 11:49:41 +02003572
Joerg Roedel20307532010-11-29 17:51:48 +01003573 /* Enter Guest-Mode */
3574 enter_guest_mode(&svm->vcpu);
3575
Joerg Roedel384c6362010-11-30 18:03:56 +01003576 /*
3577 * Merge guest and host intercepts - must be called with vcpu in
3578 * guest-mode to take affect here
3579 */
3580 recalc_intercepts(svm);
3581
Joerg Roedel06fc77722010-02-19 16:23:07 +01003582 svm->nested.vmcb = vmcb_gpa;
Alexander Graf3d6368e2008-11-25 20:17:07 +01003583
Joerg Roedel2af91942009-08-07 11:49:28 +02003584 enable_gif(svm);
Alexander Graf3d6368e2008-11-25 20:17:07 +01003585
Roedel, Joerg8d28fec2010-12-03 13:15:21 +01003586 mark_all_dirty(svm->vmcb);
Ladi Prosekc2634062017-10-11 16:54:44 +02003587}
3588
Vitaly Kuznetsove7134c12019-08-13 15:53:34 +02003589static int nested_svm_vmrun(struct vcpu_svm *svm)
Ladi Prosekc2634062017-10-11 16:54:44 +02003590{
Vitaly Kuznetsovc8e16b72019-08-13 15:53:35 +02003591 int ret;
Ladi Prosekc2634062017-10-11 16:54:44 +02003592 struct vmcb *nested_vmcb;
3593 struct vmcb *hsave = svm->nested.hsave;
3594 struct vmcb *vmcb = svm->vmcb;
KarimAllah Ahmed8c5fbf12019-01-31 21:24:40 +01003595 struct kvm_host_map map;
Ladi Prosekc2634062017-10-11 16:54:44 +02003596 u64 vmcb_gpa;
3597
3598 vmcb_gpa = svm->vmcb->save.rax;
3599
Vitaly Kuznetsovc8e16b72019-08-13 15:53:35 +02003600 ret = kvm_vcpu_map(&svm->vcpu, gpa_to_gfn(vmcb_gpa), &map);
Dan Carpentera0619852019-08-27 12:38:52 +03003601 if (ret == -EINVAL) {
Vitaly Kuznetsovc8e16b72019-08-13 15:53:35 +02003602 kvm_inject_gp(&svm->vcpu, 0);
Vitaly Kuznetsove7134c12019-08-13 15:53:34 +02003603 return 1;
Vitaly Kuznetsovc8e16b72019-08-13 15:53:35 +02003604 } else if (ret) {
3605 return kvm_skip_emulated_instruction(&svm->vcpu);
KarimAllah Ahmed8c5fbf12019-01-31 21:24:40 +01003606 }
3607
Vitaly Kuznetsovc8e16b72019-08-13 15:53:35 +02003608 ret = kvm_skip_emulated_instruction(&svm->vcpu);
3609
KarimAllah Ahmed8c5fbf12019-01-31 21:24:40 +01003610 nested_vmcb = map.hva;
Ladi Prosekc2634062017-10-11 16:54:44 +02003611
3612 if (!nested_vmcb_checks(nested_vmcb)) {
3613 nested_vmcb->control.exit_code = SVM_EXIT_ERR;
3614 nested_vmcb->control.exit_code_hi = 0;
3615 nested_vmcb->control.exit_info_1 = 0;
3616 nested_vmcb->control.exit_info_2 = 0;
3617
KarimAllah Ahmed8c5fbf12019-01-31 21:24:40 +01003618 kvm_vcpu_unmap(&svm->vcpu, &map, true);
Ladi Prosekc2634062017-10-11 16:54:44 +02003619
Vitaly Kuznetsovc8e16b72019-08-13 15:53:35 +02003620 return ret;
Ladi Prosekc2634062017-10-11 16:54:44 +02003621 }
3622
3623 trace_kvm_nested_vmrun(svm->vmcb->save.rip, vmcb_gpa,
3624 nested_vmcb->save.rip,
3625 nested_vmcb->control.int_ctl,
3626 nested_vmcb->control.event_inj,
3627 nested_vmcb->control.nested_ctl);
3628
3629 trace_kvm_nested_intercepts(nested_vmcb->control.intercept_cr & 0xffff,
3630 nested_vmcb->control.intercept_cr >> 16,
3631 nested_vmcb->control.intercept_exceptions,
3632 nested_vmcb->control.intercept);
3633
3634 /* Clear internal status */
3635 kvm_clear_exception_queue(&svm->vcpu);
3636 kvm_clear_interrupt_queue(&svm->vcpu);
3637
3638 /*
3639 * Save the old vmcb, so we don't need to pick what we save, but can
3640 * restore everything when a VMEXIT occurs
3641 */
3642 hsave->save.es = vmcb->save.es;
3643 hsave->save.cs = vmcb->save.cs;
3644 hsave->save.ss = vmcb->save.ss;
3645 hsave->save.ds = vmcb->save.ds;
3646 hsave->save.gdtr = vmcb->save.gdtr;
3647 hsave->save.idtr = vmcb->save.idtr;
3648 hsave->save.efer = svm->vcpu.arch.efer;
3649 hsave->save.cr0 = kvm_read_cr0(&svm->vcpu);
3650 hsave->save.cr4 = svm->vcpu.arch.cr4;
3651 hsave->save.rflags = kvm_get_rflags(&svm->vcpu);
3652 hsave->save.rip = kvm_rip_read(&svm->vcpu);
3653 hsave->save.rsp = vmcb->save.rsp;
3654 hsave->save.rax = vmcb->save.rax;
3655 if (npt_enabled)
3656 hsave->save.cr3 = vmcb->save.cr3;
3657 else
3658 hsave->save.cr3 = kvm_read_cr3(&svm->vcpu);
3659
3660 copy_vmcb_control_area(hsave, vmcb);
3661
KarimAllah Ahmed8c5fbf12019-01-31 21:24:40 +01003662 enter_svm_guest_mode(svm, vmcb_gpa, nested_vmcb, &map);
Roedel, Joerg8d28fec2010-12-03 13:15:21 +01003663
Vitaly Kuznetsove7134c12019-08-13 15:53:34 +02003664 if (!nested_svm_vmrun_msrpm(svm)) {
3665 svm->vmcb->control.exit_code = SVM_EXIT_ERR;
3666 svm->vmcb->control.exit_code_hi = 0;
3667 svm->vmcb->control.exit_info_1 = 0;
3668 svm->vmcb->control.exit_info_2 = 0;
3669
3670 nested_svm_vmexit(svm);
3671 }
3672
Vitaly Kuznetsovc8e16b72019-08-13 15:53:35 +02003673 return ret;
Alexander Graf3d6368e2008-11-25 20:17:07 +01003674}
3675
Joerg Roedel9966bf62009-08-07 11:49:40 +02003676static void nested_svm_vmloadsave(struct vmcb *from_vmcb, struct vmcb *to_vmcb)
Alexander Graf55426752008-11-25 20:17:06 +01003677{
3678 to_vmcb->save.fs = from_vmcb->save.fs;
3679 to_vmcb->save.gs = from_vmcb->save.gs;
3680 to_vmcb->save.tr = from_vmcb->save.tr;
3681 to_vmcb->save.ldtr = from_vmcb->save.ldtr;
3682 to_vmcb->save.kernel_gs_base = from_vmcb->save.kernel_gs_base;
3683 to_vmcb->save.star = from_vmcb->save.star;
3684 to_vmcb->save.lstar = from_vmcb->save.lstar;
3685 to_vmcb->save.cstar = from_vmcb->save.cstar;
3686 to_vmcb->save.sfmask = from_vmcb->save.sfmask;
3687 to_vmcb->save.sysenter_cs = from_vmcb->save.sysenter_cs;
3688 to_vmcb->save.sysenter_esp = from_vmcb->save.sysenter_esp;
3689 to_vmcb->save.sysenter_eip = from_vmcb->save.sysenter_eip;
Alexander Graf55426752008-11-25 20:17:06 +01003690}
3691
Avi Kivity851ba692009-08-24 11:10:17 +03003692static int vmload_interception(struct vcpu_svm *svm)
Alexander Graf55426752008-11-25 20:17:06 +01003693{
Joerg Roedel9966bf62009-08-07 11:49:40 +02003694 struct vmcb *nested_vmcb;
KarimAllah Ahmed8c5fbf12019-01-31 21:24:40 +01003695 struct kvm_host_map map;
Ladi Prosekb742c1e2017-06-22 09:05:26 +02003696 int ret;
Joerg Roedel9966bf62009-08-07 11:49:40 +02003697
Alexander Graf55426752008-11-25 20:17:06 +01003698 if (nested_svm_check_permissions(svm))
3699 return 1;
3700
KarimAllah Ahmed8c5fbf12019-01-31 21:24:40 +01003701 ret = kvm_vcpu_map(&svm->vcpu, gpa_to_gfn(svm->vmcb->save.rax), &map);
3702 if (ret) {
3703 if (ret == -EINVAL)
3704 kvm_inject_gp(&svm->vcpu, 0);
Joerg Roedel9966bf62009-08-07 11:49:40 +02003705 return 1;
KarimAllah Ahmed8c5fbf12019-01-31 21:24:40 +01003706 }
3707
3708 nested_vmcb = map.hva;
Joerg Roedel9966bf62009-08-07 11:49:40 +02003709
Ladi Prosekb742c1e2017-06-22 09:05:26 +02003710 ret = kvm_skip_emulated_instruction(&svm->vcpu);
Joerg Roedele3e9ed32011-04-06 12:30:03 +02003711
Joerg Roedel9966bf62009-08-07 11:49:40 +02003712 nested_svm_vmloadsave(nested_vmcb, svm->vmcb);
KarimAllah Ahmed8c5fbf12019-01-31 21:24:40 +01003713 kvm_vcpu_unmap(&svm->vcpu, &map, true);
Alexander Graf55426752008-11-25 20:17:06 +01003714
Ladi Prosekb742c1e2017-06-22 09:05:26 +02003715 return ret;
Alexander Graf55426752008-11-25 20:17:06 +01003716}
3717
Avi Kivity851ba692009-08-24 11:10:17 +03003718static int vmsave_interception(struct vcpu_svm *svm)
Alexander Graf55426752008-11-25 20:17:06 +01003719{
Joerg Roedel9966bf62009-08-07 11:49:40 +02003720 struct vmcb *nested_vmcb;
KarimAllah Ahmed8c5fbf12019-01-31 21:24:40 +01003721 struct kvm_host_map map;
Ladi Prosekb742c1e2017-06-22 09:05:26 +02003722 int ret;
Joerg Roedel9966bf62009-08-07 11:49:40 +02003723
Alexander Graf55426752008-11-25 20:17:06 +01003724 if (nested_svm_check_permissions(svm))
3725 return 1;
3726
KarimAllah Ahmed8c5fbf12019-01-31 21:24:40 +01003727 ret = kvm_vcpu_map(&svm->vcpu, gpa_to_gfn(svm->vmcb->save.rax), &map);
3728 if (ret) {
3729 if (ret == -EINVAL)
3730 kvm_inject_gp(&svm->vcpu, 0);
Joerg Roedel9966bf62009-08-07 11:49:40 +02003731 return 1;
KarimAllah Ahmed8c5fbf12019-01-31 21:24:40 +01003732 }
3733
3734 nested_vmcb = map.hva;
Joerg Roedel9966bf62009-08-07 11:49:40 +02003735
Ladi Prosekb742c1e2017-06-22 09:05:26 +02003736 ret = kvm_skip_emulated_instruction(&svm->vcpu);
Joerg Roedele3e9ed32011-04-06 12:30:03 +02003737
Joerg Roedel9966bf62009-08-07 11:49:40 +02003738 nested_svm_vmloadsave(svm->vmcb, nested_vmcb);
KarimAllah Ahmed8c5fbf12019-01-31 21:24:40 +01003739 kvm_vcpu_unmap(&svm->vcpu, &map, true);
Alexander Graf55426752008-11-25 20:17:06 +01003740
Ladi Prosekb742c1e2017-06-22 09:05:26 +02003741 return ret;
Alexander Graf55426752008-11-25 20:17:06 +01003742}
3743
Avi Kivity851ba692009-08-24 11:10:17 +03003744static int vmrun_interception(struct vcpu_svm *svm)
Alexander Graf3d6368e2008-11-25 20:17:07 +01003745{
Alexander Graf3d6368e2008-11-25 20:17:07 +01003746 if (nested_svm_check_permissions(svm))
3747 return 1;
3748
Vitaly Kuznetsove7134c12019-08-13 15:53:34 +02003749 return nested_svm_vmrun(svm);
Alexander Graf3d6368e2008-11-25 20:17:07 +01003750}
3751
Avi Kivity851ba692009-08-24 11:10:17 +03003752static int stgi_interception(struct vcpu_svm *svm)
Alexander Graf1371d902008-11-25 20:17:04 +01003753{
Ladi Prosekb742c1e2017-06-22 09:05:26 +02003754 int ret;
3755
Alexander Graf1371d902008-11-25 20:17:04 +01003756 if (nested_svm_check_permissions(svm))
3757 return 1;
3758
Janakarajan Natarajan640bd6e2017-08-23 09:57:19 -05003759 /*
3760 * If VGIF is enabled, the STGI intercept is only added to
Ladi Prosekcc3d9672017-10-17 16:02:39 +02003761 * detect the opening of the SMI/NMI window; remove it now.
Janakarajan Natarajan640bd6e2017-08-23 09:57:19 -05003762 */
3763 if (vgif_enabled(svm))
3764 clr_intercept(svm, INTERCEPT_STGI);
3765
Ladi Prosekb742c1e2017-06-22 09:05:26 +02003766 ret = kvm_skip_emulated_instruction(&svm->vcpu);
Avi Kivity3842d132010-07-27 12:30:24 +03003767 kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
Alexander Graf1371d902008-11-25 20:17:04 +01003768
Joerg Roedel2af91942009-08-07 11:49:28 +02003769 enable_gif(svm);
Alexander Graf1371d902008-11-25 20:17:04 +01003770
Ladi Prosekb742c1e2017-06-22 09:05:26 +02003771 return ret;
Alexander Graf1371d902008-11-25 20:17:04 +01003772}
3773
Avi Kivity851ba692009-08-24 11:10:17 +03003774static int clgi_interception(struct vcpu_svm *svm)
Alexander Graf1371d902008-11-25 20:17:04 +01003775{
Ladi Prosekb742c1e2017-06-22 09:05:26 +02003776 int ret;
3777
Alexander Graf1371d902008-11-25 20:17:04 +01003778 if (nested_svm_check_permissions(svm))
3779 return 1;
3780
Ladi Prosekb742c1e2017-06-22 09:05:26 +02003781 ret = kvm_skip_emulated_instruction(&svm->vcpu);
Alexander Graf1371d902008-11-25 20:17:04 +01003782
Joerg Roedel2af91942009-08-07 11:49:28 +02003783 disable_gif(svm);
Alexander Graf1371d902008-11-25 20:17:04 +01003784
3785 /* After a CLGI no interrupts should come */
Suravee Suthikulpanit340d3bc2016-05-04 14:09:47 -05003786 if (!kvm_vcpu_apicv_active(&svm->vcpu)) {
3787 svm_clear_vintr(svm);
3788 svm->vmcb->control.int_ctl &= ~V_IRQ_MASK;
3789 mark_dirty(svm->vmcb, VMCB_INTR);
3790 }
Joerg Roedeldecdbf62010-12-03 11:45:52 +01003791
Ladi Prosekb742c1e2017-06-22 09:05:26 +02003792 return ret;
Alexander Graf1371d902008-11-25 20:17:04 +01003793}
3794
Avi Kivity851ba692009-08-24 11:10:17 +03003795static int invlpga_interception(struct vcpu_svm *svm)
Alexander Grafff092382009-06-15 15:21:24 +02003796{
3797 struct kvm_vcpu *vcpu = &svm->vcpu;
Alexander Grafff092382009-06-15 15:21:24 +02003798
Sean Christophersonde3cd112019-04-30 10:36:17 -07003799 trace_kvm_invlpga(svm->vmcb->save.rip, kvm_rcx_read(&svm->vcpu),
3800 kvm_rax_read(&svm->vcpu));
Joerg Roedelec1ff792009-10-09 16:08:31 +02003801
Alexander Grafff092382009-06-15 15:21:24 +02003802 /* Let's treat INVLPGA the same as INVLPG (can be optimized!) */
Sean Christophersonde3cd112019-04-30 10:36:17 -07003803 kvm_mmu_invlpg(vcpu, kvm_rax_read(&svm->vcpu));
Alexander Grafff092382009-06-15 15:21:24 +02003804
Ladi Prosekb742c1e2017-06-22 09:05:26 +02003805 return kvm_skip_emulated_instruction(&svm->vcpu);
Alexander Grafff092382009-06-15 15:21:24 +02003806}
3807
Joerg Roedel532a46b2009-10-09 16:08:32 +02003808static int skinit_interception(struct vcpu_svm *svm)
3809{
Sean Christophersonde3cd112019-04-30 10:36:17 -07003810 trace_kvm_skinit(svm->vmcb->save.rip, kvm_rax_read(&svm->vcpu));
Joerg Roedel532a46b2009-10-09 16:08:32 +02003811
3812 kvm_queue_exception(&svm->vcpu, UD_VECTOR);
3813 return 1;
3814}
3815
David Kaplandab429a2015-03-02 13:43:37 -06003816static int wbinvd_interception(struct vcpu_svm *svm)
3817{
Kyle Huey6affcbe2016-11-29 12:40:40 -08003818 return kvm_emulate_wbinvd(&svm->vcpu);
David Kaplandab429a2015-03-02 13:43:37 -06003819}
3820
Joerg Roedel81dd35d2010-12-07 17:15:06 +01003821static int xsetbv_interception(struct vcpu_svm *svm)
3822{
3823 u64 new_bv = kvm_read_edx_eax(&svm->vcpu);
Sean Christophersonde3cd112019-04-30 10:36:17 -07003824 u32 index = kvm_rcx_read(&svm->vcpu);
Joerg Roedel81dd35d2010-12-07 17:15:06 +01003825
3826 if (kvm_set_xcr(&svm->vcpu, index, new_bv) == 0) {
Ladi Prosekb742c1e2017-06-22 09:05:26 +02003827 return kvm_skip_emulated_instruction(&svm->vcpu);
Joerg Roedel81dd35d2010-12-07 17:15:06 +01003828 }
3829
3830 return 1;
3831}
3832
Avi Kivity851ba692009-08-24 11:10:17 +03003833static int task_switch_interception(struct vcpu_svm *svm)
Avi Kivity6aa8b732006-12-10 02:21:36 -08003834{
Izik Eidus37817f22008-03-24 23:14:53 +02003835 u16 tss_selector;
Gleb Natapov64a7ec02009-03-30 16:03:29 +03003836 int reason;
3837 int int_type = svm->vmcb->control.exit_int_info &
3838 SVM_EXITINTINFO_TYPE_MASK;
Gleb Natapov8317c292009-04-12 13:37:02 +03003839 int int_vec = svm->vmcb->control.exit_int_info & SVM_EVTINJ_VEC_MASK;
Gleb Natapovfe8e7f82009-04-23 17:03:48 +03003840 uint32_t type =
3841 svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_TYPE_MASK;
3842 uint32_t idt_v =
3843 svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID;
Jan Kiszkae269fb22010-04-14 15:51:09 +02003844 bool has_error_code = false;
3845 u32 error_code = 0;
Izik Eidus37817f22008-03-24 23:14:53 +02003846
3847 tss_selector = (u16)svm->vmcb->control.exit_info_1;
Gleb Natapov64a7ec02009-03-30 16:03:29 +03003848
Izik Eidus37817f22008-03-24 23:14:53 +02003849 if (svm->vmcb->control.exit_info_2 &
3850 (1ULL << SVM_EXITINFOSHIFT_TS_REASON_IRET))
Gleb Natapov64a7ec02009-03-30 16:03:29 +03003851 reason = TASK_SWITCH_IRET;
3852 else if (svm->vmcb->control.exit_info_2 &
3853 (1ULL << SVM_EXITINFOSHIFT_TS_REASON_JMP))
3854 reason = TASK_SWITCH_JMP;
Gleb Natapovfe8e7f82009-04-23 17:03:48 +03003855 else if (idt_v)
Gleb Natapov64a7ec02009-03-30 16:03:29 +03003856 reason = TASK_SWITCH_GATE;
3857 else
3858 reason = TASK_SWITCH_CALL;
3859
Gleb Natapovfe8e7f82009-04-23 17:03:48 +03003860 if (reason == TASK_SWITCH_GATE) {
3861 switch (type) {
3862 case SVM_EXITINTINFO_TYPE_NMI:
3863 svm->vcpu.arch.nmi_injected = false;
3864 break;
3865 case SVM_EXITINTINFO_TYPE_EXEPT:
Jan Kiszkae269fb22010-04-14 15:51:09 +02003866 if (svm->vmcb->control.exit_info_2 &
3867 (1ULL << SVM_EXITINFOSHIFT_TS_HAS_ERROR_CODE)) {
3868 has_error_code = true;
3869 error_code =
3870 (u32)svm->vmcb->control.exit_info_2;
3871 }
Gleb Natapovfe8e7f82009-04-23 17:03:48 +03003872 kvm_clear_exception_queue(&svm->vcpu);
3873 break;
3874 case SVM_EXITINTINFO_TYPE_INTR:
3875 kvm_clear_interrupt_queue(&svm->vcpu);
3876 break;
3877 default:
3878 break;
3879 }
3880 }
Gleb Natapov64a7ec02009-03-30 16:03:29 +03003881
Gleb Natapov8317c292009-04-12 13:37:02 +03003882 if (reason != TASK_SWITCH_GATE ||
3883 int_type == SVM_EXITINTINFO_TYPE_SOFT ||
3884 (int_type == SVM_EXITINTINFO_TYPE_EXEPT &&
Vitaly Kuznetsovf8ea7c62019-08-13 15:53:30 +02003885 (int_vec == OF_VECTOR || int_vec == BP_VECTOR))) {
3886 if (skip_emulated_instruction(&svm->vcpu) != EMULATE_DONE)
3887 goto fail;
3888 }
Gleb Natapov64a7ec02009-03-30 16:03:29 +03003889
Kevin Wolf7f3d35f2012-02-08 14:34:38 +01003890 if (int_type != SVM_EXITINTINFO_TYPE_SOFT)
3891 int_vec = -1;
3892
3893 if (kvm_task_switch(&svm->vcpu, tss_selector, int_vec, reason,
Vitaly Kuznetsovf8ea7c62019-08-13 15:53:30 +02003894 has_error_code, error_code) == EMULATE_FAIL)
3895 goto fail;
3896
Gleb Natapovacb54512010-04-15 21:03:50 +03003897 return 1;
Vitaly Kuznetsovf8ea7c62019-08-13 15:53:30 +02003898
3899fail:
3900 svm->vcpu.run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
3901 svm->vcpu.run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
3902 svm->vcpu.run->internal.ndata = 0;
3903 return 0;
Avi Kivity6aa8b732006-12-10 02:21:36 -08003904}
3905
Avi Kivity851ba692009-08-24 11:10:17 +03003906static int cpuid_interception(struct vcpu_svm *svm)
Avi Kivity6aa8b732006-12-10 02:21:36 -08003907{
Kyle Huey6a908b62016-11-29 12:40:37 -08003908 return kvm_emulate_cpuid(&svm->vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -08003909}
3910
Avi Kivity851ba692009-08-24 11:10:17 +03003911static int iret_interception(struct vcpu_svm *svm)
Gleb Natapov95ba8273132009-04-21 17:45:08 +03003912{
3913 ++svm->vcpu.stat.nmi_window_exits;
Joerg Roedel8a05a1b82010-11-30 18:04:00 +01003914 clr_intercept(svm, INTERCEPT_IRET);
Gleb Natapov44c11432009-05-11 13:35:52 +03003915 svm->vcpu.arch.hflags |= HF_IRET_MASK;
Avi Kivitybd3d1ec2011-02-03 15:29:52 +02003916 svm->nmi_iret_rip = kvm_rip_read(&svm->vcpu);
Radim Krčmářf303b4c2014-01-17 20:52:42 +01003917 kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
Gleb Natapov95ba8273132009-04-21 17:45:08 +03003918 return 1;
3919}
3920
Avi Kivity851ba692009-08-24 11:10:17 +03003921static int invlpg_interception(struct vcpu_svm *svm)
Marcelo Tosattia7052892008-09-23 13:18:35 -03003922{
Andre Przywaradf4f31082010-12-21 11:12:06 +01003923 if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
Sean Christopherson0ce97a22018-08-23 13:56:52 -07003924 return kvm_emulate_instruction(&svm->vcpu, 0) == EMULATE_DONE;
Andre Przywaradf4f31082010-12-21 11:12:06 +01003925
3926 kvm_mmu_invlpg(&svm->vcpu, svm->vmcb->control.exit_info_1);
Ladi Prosekb742c1e2017-06-22 09:05:26 +02003927 return kvm_skip_emulated_instruction(&svm->vcpu);
Marcelo Tosattia7052892008-09-23 13:18:35 -03003928}
3929
Avi Kivity851ba692009-08-24 11:10:17 +03003930static int emulate_on_interception(struct vcpu_svm *svm)
Avi Kivity6aa8b732006-12-10 02:21:36 -08003931{
Sean Christopherson0ce97a22018-08-23 13:56:52 -07003932 return kvm_emulate_instruction(&svm->vcpu, 0) == EMULATE_DONE;
Avi Kivity6aa8b732006-12-10 02:21:36 -08003933}
3934
Brijesh Singh7607b712018-02-19 10:14:44 -06003935static int rsm_interception(struct vcpu_svm *svm)
3936{
Sean Christopherson35be0ad2018-08-23 13:56:47 -07003937 return kvm_emulate_instruction_from_buffer(&svm->vcpu,
3938 rsm_ins_bytes, 2) == EMULATE_DONE;
Brijesh Singh7607b712018-02-19 10:14:44 -06003939}
3940
Avi Kivity332b56e2011-11-10 14:57:24 +02003941static int rdpmc_interception(struct vcpu_svm *svm)
3942{
3943 int err;
3944
Paolo Bonzinid647eb62019-06-20 14:13:33 +02003945 if (!nrips)
Avi Kivity332b56e2011-11-10 14:57:24 +02003946 return emulate_on_interception(svm);
3947
3948 err = kvm_rdpmc(&svm->vcpu);
Kyle Huey6affcbe2016-11-29 12:40:40 -08003949 return kvm_complete_insn_gp(&svm->vcpu, err);
Avi Kivity332b56e2011-11-10 14:57:24 +02003950}
3951
Xiubo Li52eb5a62015-03-13 17:39:45 +08003952static bool check_selective_cr0_intercepted(struct vcpu_svm *svm,
3953 unsigned long val)
Joerg Roedel628afd22011-04-04 12:39:36 +02003954{
3955 unsigned long cr0 = svm->vcpu.arch.cr0;
3956 bool ret = false;
3957 u64 intercept;
3958
3959 intercept = svm->nested.intercept;
3960
3961 if (!is_guest_mode(&svm->vcpu) ||
3962 (!(intercept & (1ULL << INTERCEPT_SELECTIVE_CR0))))
3963 return false;
3964
3965 cr0 &= ~SVM_CR0_SELECTIVE_MASK;
3966 val &= ~SVM_CR0_SELECTIVE_MASK;
3967
3968 if (cr0 ^ val) {
3969 svm->vmcb->control.exit_code = SVM_EXIT_CR0_SEL_WRITE;
3970 ret = (nested_svm_exit_handled(svm) == NESTED_EXIT_DONE);
3971 }
3972
3973 return ret;
3974}
3975
Andre Przywara7ff76d52010-12-21 11:12:04 +01003976#define CR_VALID (1ULL << 63)
3977
3978static int cr_interception(struct vcpu_svm *svm)
3979{
3980 int reg, cr;
3981 unsigned long val;
3982 int err;
3983
3984 if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
3985 return emulate_on_interception(svm);
3986
3987 if (unlikely((svm->vmcb->control.exit_info_1 & CR_VALID) == 0))
3988 return emulate_on_interception(svm);
3989
3990 reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
David Kaplan5e575182015-03-06 14:44:35 -06003991 if (svm->vmcb->control.exit_code == SVM_EXIT_CR0_SEL_WRITE)
3992 cr = SVM_EXIT_WRITE_CR0 - SVM_EXIT_READ_CR0;
3993 else
3994 cr = svm->vmcb->control.exit_code - SVM_EXIT_READ_CR0;
Andre Przywara7ff76d52010-12-21 11:12:04 +01003995
3996 err = 0;
3997 if (cr >= 16) { /* mov to cr */
3998 cr -= 16;
3999 val = kvm_register_read(&svm->vcpu, reg);
4000 switch (cr) {
4001 case 0:
Joerg Roedel628afd22011-04-04 12:39:36 +02004002 if (!check_selective_cr0_intercepted(svm, val))
4003 err = kvm_set_cr0(&svm->vcpu, val);
Joerg Roedel977b2d02011-04-18 11:42:52 +02004004 else
4005 return 1;
4006
Andre Przywara7ff76d52010-12-21 11:12:04 +01004007 break;
4008 case 3:
4009 err = kvm_set_cr3(&svm->vcpu, val);
4010 break;
4011 case 4:
4012 err = kvm_set_cr4(&svm->vcpu, val);
4013 break;
4014 case 8:
4015 err = kvm_set_cr8(&svm->vcpu, val);
4016 break;
4017 default:
4018 WARN(1, "unhandled write to CR%d", cr);
4019 kvm_queue_exception(&svm->vcpu, UD_VECTOR);
4020 return 1;
4021 }
4022 } else { /* mov from cr */
4023 switch (cr) {
4024 case 0:
4025 val = kvm_read_cr0(&svm->vcpu);
4026 break;
4027 case 2:
4028 val = svm->vcpu.arch.cr2;
4029 break;
4030 case 3:
Avi Kivity9f8fe502010-12-05 17:30:00 +02004031 val = kvm_read_cr3(&svm->vcpu);
Andre Przywara7ff76d52010-12-21 11:12:04 +01004032 break;
4033 case 4:
4034 val = kvm_read_cr4(&svm->vcpu);
4035 break;
4036 case 8:
4037 val = kvm_get_cr8(&svm->vcpu);
4038 break;
4039 default:
4040 WARN(1, "unhandled read from CR%d", cr);
4041 kvm_queue_exception(&svm->vcpu, UD_VECTOR);
4042 return 1;
4043 }
4044 kvm_register_write(&svm->vcpu, reg, val);
4045 }
Kyle Huey6affcbe2016-11-29 12:40:40 -08004046 return kvm_complete_insn_gp(&svm->vcpu, err);
Andre Przywara7ff76d52010-12-21 11:12:04 +01004047}
4048
Andre Przywaracae37972010-12-21 11:12:05 +01004049static int dr_interception(struct vcpu_svm *svm)
4050{
4051 int reg, dr;
4052 unsigned long val;
Andre Przywaracae37972010-12-21 11:12:05 +01004053
Paolo Bonzinifacb0132014-02-21 10:32:27 +01004054 if (svm->vcpu.guest_debug == 0) {
4055 /*
4056 * No more DR vmexits; force a reload of the debug registers
4057 * and reenter on this instruction. The next vmexit will
4058 * retrieve the full state of the debug registers.
4059 */
4060 clr_dr_intercepts(svm);
4061 svm->vcpu.arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT;
4062 return 1;
4063 }
4064
Andre Przywaracae37972010-12-21 11:12:05 +01004065 if (!boot_cpu_has(X86_FEATURE_DECODEASSISTS))
4066 return emulate_on_interception(svm);
4067
4068 reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
4069 dr = svm->vmcb->control.exit_code - SVM_EXIT_READ_DR0;
4070
4071 if (dr >= 16) { /* mov to DRn */
Nadav Amit16f8a6f2014-10-03 01:10:05 +03004072 if (!kvm_require_dr(&svm->vcpu, dr - 16))
4073 return 1;
Andre Przywaracae37972010-12-21 11:12:05 +01004074 val = kvm_register_read(&svm->vcpu, reg);
4075 kvm_set_dr(&svm->vcpu, dr - 16, val);
4076 } else {
Nadav Amit16f8a6f2014-10-03 01:10:05 +03004077 if (!kvm_require_dr(&svm->vcpu, dr))
4078 return 1;
4079 kvm_get_dr(&svm->vcpu, dr, &val);
4080 kvm_register_write(&svm->vcpu, reg, val);
Andre Przywaracae37972010-12-21 11:12:05 +01004081 }
4082
Ladi Prosekb742c1e2017-06-22 09:05:26 +02004083 return kvm_skip_emulated_instruction(&svm->vcpu);
Andre Przywaracae37972010-12-21 11:12:05 +01004084}
4085
Avi Kivity851ba692009-08-24 11:10:17 +03004086static int cr8_write_interception(struct vcpu_svm *svm)
Joerg Roedel1d075432007-12-06 21:02:25 +01004087{
Avi Kivity851ba692009-08-24 11:10:17 +03004088 struct kvm_run *kvm_run = svm->vcpu.run;
Andre Przywaraeea1cff2010-12-21 11:12:00 +01004089 int r;
Avi Kivity851ba692009-08-24 11:10:17 +03004090
Gleb Natapov0a5fff192009-04-21 17:45:06 +03004091 u8 cr8_prev = kvm_get_cr8(&svm->vcpu);
4092 /* instruction emulation calls kvm_set_cr8() */
Andre Przywara7ff76d52010-12-21 11:12:04 +01004093 r = cr_interception(svm);
Paolo Bonzini35754c92015-07-29 12:05:37 +02004094 if (lapic_in_kernel(&svm->vcpu))
Andre Przywara7ff76d52010-12-21 11:12:04 +01004095 return r;
Gleb Natapov0a5fff192009-04-21 17:45:06 +03004096 if (cr8_prev <= kvm_get_cr8(&svm->vcpu))
Andre Przywara7ff76d52010-12-21 11:12:04 +01004097 return r;
Joerg Roedel1d075432007-12-06 21:02:25 +01004098 kvm_run->exit_reason = KVM_EXIT_SET_TPR;
4099 return 0;
4100}
4101
Tom Lendacky801e4592018-02-21 13:39:51 -06004102static int svm_get_msr_feature(struct kvm_msr_entry *msr)
4103{
Tom Lendackyd1d93fa2018-02-24 00:18:20 +01004104 msr->data = 0;
4105
4106 switch (msr->index) {
4107 case MSR_F10H_DECFG:
4108 if (boot_cpu_has(X86_FEATURE_LFENCE_RDTSC))
4109 msr->data |= MSR_F10H_DECFG_LFENCE_SERIALIZE;
4110 break;
4111 default:
4112 return 1;
4113 }
4114
4115 return 0;
Tom Lendacky801e4592018-02-21 13:39:51 -06004116}
4117
Paolo Bonzini609e36d2015-04-08 15:30:38 +02004118static int svm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
Avi Kivity6aa8b732006-12-10 02:21:36 -08004119{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04004120 struct vcpu_svm *svm = to_svm(vcpu);
4121
Paolo Bonzini609e36d2015-04-08 15:30:38 +02004122 switch (msr_info->index) {
Brian Gerst8c065852010-07-17 09:03:26 -04004123 case MSR_STAR:
Paolo Bonzini609e36d2015-04-08 15:30:38 +02004124 msr_info->data = svm->vmcb->save.star;
Avi Kivity6aa8b732006-12-10 02:21:36 -08004125 break;
Avi Kivity0e859ca2006-12-22 01:05:08 -08004126#ifdef CONFIG_X86_64
Avi Kivity6aa8b732006-12-10 02:21:36 -08004127 case MSR_LSTAR:
Paolo Bonzini609e36d2015-04-08 15:30:38 +02004128 msr_info->data = svm->vmcb->save.lstar;
Avi Kivity6aa8b732006-12-10 02:21:36 -08004129 break;
4130 case MSR_CSTAR:
Paolo Bonzini609e36d2015-04-08 15:30:38 +02004131 msr_info->data = svm->vmcb->save.cstar;
Avi Kivity6aa8b732006-12-10 02:21:36 -08004132 break;
4133 case MSR_KERNEL_GS_BASE:
Paolo Bonzini609e36d2015-04-08 15:30:38 +02004134 msr_info->data = svm->vmcb->save.kernel_gs_base;
Avi Kivity6aa8b732006-12-10 02:21:36 -08004135 break;
4136 case MSR_SYSCALL_MASK:
Paolo Bonzini609e36d2015-04-08 15:30:38 +02004137 msr_info->data = svm->vmcb->save.sfmask;
Avi Kivity6aa8b732006-12-10 02:21:36 -08004138 break;
4139#endif
4140 case MSR_IA32_SYSENTER_CS:
Paolo Bonzini609e36d2015-04-08 15:30:38 +02004141 msr_info->data = svm->vmcb->save.sysenter_cs;
Avi Kivity6aa8b732006-12-10 02:21:36 -08004142 break;
4143 case MSR_IA32_SYSENTER_EIP:
Paolo Bonzini609e36d2015-04-08 15:30:38 +02004144 msr_info->data = svm->sysenter_eip;
Avi Kivity6aa8b732006-12-10 02:21:36 -08004145 break;
4146 case MSR_IA32_SYSENTER_ESP:
Paolo Bonzini609e36d2015-04-08 15:30:38 +02004147 msr_info->data = svm->sysenter_esp;
Avi Kivity6aa8b732006-12-10 02:21:36 -08004148 break;
Paolo Bonzini46896c72015-11-12 14:49:16 +01004149 case MSR_TSC_AUX:
4150 if (!boot_cpu_has(X86_FEATURE_RDTSCP))
4151 return 1;
4152 msr_info->data = svm->tsc_aux;
4153 break;
Joerg Roedele0231712010-02-24 18:59:10 +01004154 /*
4155 * Nobody will change the following 5 values in the VMCB so we can
4156 * safely return them on rdmsr. They will always be 0 until LBRV is
4157 * implemented.
4158 */
Joerg Roedela2938c82008-02-13 16:30:28 +01004159 case MSR_IA32_DEBUGCTLMSR:
Paolo Bonzini609e36d2015-04-08 15:30:38 +02004160 msr_info->data = svm->vmcb->save.dbgctl;
Joerg Roedela2938c82008-02-13 16:30:28 +01004161 break;
4162 case MSR_IA32_LASTBRANCHFROMIP:
Paolo Bonzini609e36d2015-04-08 15:30:38 +02004163 msr_info->data = svm->vmcb->save.br_from;
Joerg Roedela2938c82008-02-13 16:30:28 +01004164 break;
4165 case MSR_IA32_LASTBRANCHTOIP:
Paolo Bonzini609e36d2015-04-08 15:30:38 +02004166 msr_info->data = svm->vmcb->save.br_to;
Joerg Roedela2938c82008-02-13 16:30:28 +01004167 break;
4168 case MSR_IA32_LASTINTFROMIP:
Paolo Bonzini609e36d2015-04-08 15:30:38 +02004169 msr_info->data = svm->vmcb->save.last_excp_from;
Joerg Roedela2938c82008-02-13 16:30:28 +01004170 break;
4171 case MSR_IA32_LASTINTTOIP:
Paolo Bonzini609e36d2015-04-08 15:30:38 +02004172 msr_info->data = svm->vmcb->save.last_excp_to;
Joerg Roedela2938c82008-02-13 16:30:28 +01004173 break;
Alexander Grafb286d5d2008-11-25 20:17:05 +01004174 case MSR_VM_HSAVE_PA:
Paolo Bonzini609e36d2015-04-08 15:30:38 +02004175 msr_info->data = svm->nested.hsave_msr;
Alexander Grafb286d5d2008-11-25 20:17:05 +01004176 break;
Joerg Roedeleb6f3022008-11-25 20:17:09 +01004177 case MSR_VM_CR:
Paolo Bonzini609e36d2015-04-08 15:30:38 +02004178 msr_info->data = svm->nested.vm_cr_msr;
Joerg Roedeleb6f3022008-11-25 20:17:09 +01004179 break;
KarimAllah Ahmedb2ac58f2018-02-03 15:56:23 +01004180 case MSR_IA32_SPEC_CTRL:
4181 if (!msr_info->host_initiated &&
Konrad Rzeszutek Wilk6ac2f492018-06-01 10:59:20 -04004182 !guest_cpuid_has(vcpu, X86_FEATURE_AMD_IBRS) &&
4183 !guest_cpuid_has(vcpu, X86_FEATURE_AMD_SSBD))
KarimAllah Ahmedb2ac58f2018-02-03 15:56:23 +01004184 return 1;
4185
4186 msr_info->data = svm->spec_ctrl;
4187 break;
Tom Lendackybc226f02018-05-10 22:06:39 +02004188 case MSR_AMD64_VIRT_SPEC_CTRL:
4189 if (!msr_info->host_initiated &&
4190 !guest_cpuid_has(vcpu, X86_FEATURE_VIRT_SSBD))
4191 return 1;
4192
4193 msr_info->data = svm->virt_spec_ctrl;
4194 break;
Borislav Petkovae8b7872015-11-23 11:12:23 +01004195 case MSR_F15H_IC_CFG: {
4196
4197 int family, model;
4198
4199 family = guest_cpuid_family(vcpu);
4200 model = guest_cpuid_model(vcpu);
4201
4202 if (family < 0 || model < 0)
4203 return kvm_get_msr_common(vcpu, msr_info);
4204
4205 msr_info->data = 0;
4206
4207 if (family == 0x15 &&
4208 (model >= 0x2 && model < 0x20))
4209 msr_info->data = 0x1E;
4210 }
4211 break;
Tom Lendackyd1d93fa2018-02-24 00:18:20 +01004212 case MSR_F10H_DECFG:
4213 msr_info->data = svm->msr_decfg;
4214 break;
Avi Kivity6aa8b732006-12-10 02:21:36 -08004215 default:
Paolo Bonzini609e36d2015-04-08 15:30:38 +02004216 return kvm_get_msr_common(vcpu, msr_info);
Avi Kivity6aa8b732006-12-10 02:21:36 -08004217 }
4218 return 0;
4219}
4220
Avi Kivity851ba692009-08-24 11:10:17 +03004221static int rdmsr_interception(struct vcpu_svm *svm)
Avi Kivity6aa8b732006-12-10 02:21:36 -08004222{
Sean Christopherson1edce0a2019-09-05 14:22:55 -07004223 return kvm_emulate_rdmsr(&svm->vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -08004224}
4225
Joerg Roedel4a810182010-02-24 18:59:15 +01004226static int svm_set_vm_cr(struct kvm_vcpu *vcpu, u64 data)
4227{
4228 struct vcpu_svm *svm = to_svm(vcpu);
4229 int svm_dis, chg_mask;
4230
4231 if (data & ~SVM_VM_CR_VALID_MASK)
4232 return 1;
4233
4234 chg_mask = SVM_VM_CR_VALID_MASK;
4235
4236 if (svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK)
4237 chg_mask &= ~(SVM_VM_CR_SVM_LOCK_MASK | SVM_VM_CR_SVM_DIS_MASK);
4238
4239 svm->nested.vm_cr_msr &= ~chg_mask;
4240 svm->nested.vm_cr_msr |= (data & chg_mask);
4241
4242 svm_dis = svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK;
4243
4244 /* check for svm_disable while efer.svme is set */
4245 if (svm_dis && (vcpu->arch.efer & EFER_SVME))
4246 return 1;
4247
4248 return 0;
4249}
4250
Will Auld8fe8ab42012-11-29 12:42:12 -08004251static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr)
Avi Kivity6aa8b732006-12-10 02:21:36 -08004252{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04004253 struct vcpu_svm *svm = to_svm(vcpu);
4254
Will Auld8fe8ab42012-11-29 12:42:12 -08004255 u32 ecx = msr->index;
4256 u64 data = msr->data;
Avi Kivity6aa8b732006-12-10 02:21:36 -08004257 switch (ecx) {
Paolo Bonzini15038e12017-10-26 09:13:27 +02004258 case MSR_IA32_CR_PAT:
4259 if (!kvm_mtrr_valid(vcpu, MSR_IA32_CR_PAT, data))
4260 return 1;
4261 vcpu->arch.pat = data;
4262 svm->vmcb->save.g_pat = data;
4263 mark_dirty(svm->vmcb, VMCB_NPT);
4264 break;
KarimAllah Ahmedb2ac58f2018-02-03 15:56:23 +01004265 case MSR_IA32_SPEC_CTRL:
4266 if (!msr->host_initiated &&
Konrad Rzeszutek Wilk6ac2f492018-06-01 10:59:20 -04004267 !guest_cpuid_has(vcpu, X86_FEATURE_AMD_IBRS) &&
4268 !guest_cpuid_has(vcpu, X86_FEATURE_AMD_SSBD))
KarimAllah Ahmedb2ac58f2018-02-03 15:56:23 +01004269 return 1;
4270
4271 /* The STIBP bit doesn't fault even if it's not advertised */
Konrad Rzeszutek Wilk6ac2f492018-06-01 10:59:20 -04004272 if (data & ~(SPEC_CTRL_IBRS | SPEC_CTRL_STIBP | SPEC_CTRL_SSBD))
KarimAllah Ahmedb2ac58f2018-02-03 15:56:23 +01004273 return 1;
4274
4275 svm->spec_ctrl = data;
4276
4277 if (!data)
4278 break;
4279
4280 /*
4281 * For non-nested:
4282 * When it's written (to non-zero) for the first time, pass
4283 * it through.
4284 *
4285 * For nested:
4286 * The handling of the MSR bitmap for L2 guests is done in
4287 * nested_svm_vmrun_msrpm.
4288 * We update the L1 MSR bit as well since it will end up
4289 * touching the MSR anyway now.
4290 */
4291 set_msr_interception(svm->msrpm, MSR_IA32_SPEC_CTRL, 1, 1);
4292 break;
Ashok Raj15d45072018-02-01 22:59:43 +01004293 case MSR_IA32_PRED_CMD:
4294 if (!msr->host_initiated &&
Borislav Petkove7c587d2018-05-02 18:15:14 +02004295 !guest_cpuid_has(vcpu, X86_FEATURE_AMD_IBPB))
Ashok Raj15d45072018-02-01 22:59:43 +01004296 return 1;
4297
4298 if (data & ~PRED_CMD_IBPB)
4299 return 1;
4300
4301 if (!data)
4302 break;
4303
4304 wrmsrl(MSR_IA32_PRED_CMD, PRED_CMD_IBPB);
4305 if (is_guest_mode(vcpu))
4306 break;
4307 set_msr_interception(svm->msrpm, MSR_IA32_PRED_CMD, 0, 1);
4308 break;
Tom Lendackybc226f02018-05-10 22:06:39 +02004309 case MSR_AMD64_VIRT_SPEC_CTRL:
4310 if (!msr->host_initiated &&
4311 !guest_cpuid_has(vcpu, X86_FEATURE_VIRT_SSBD))
4312 return 1;
4313
4314 if (data & ~SPEC_CTRL_SSBD)
4315 return 1;
4316
4317 svm->virt_spec_ctrl = data;
4318 break;
Brian Gerst8c065852010-07-17 09:03:26 -04004319 case MSR_STAR:
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04004320 svm->vmcb->save.star = data;
Avi Kivity6aa8b732006-12-10 02:21:36 -08004321 break;
Robert P. J. Day49b14f22007-01-29 13:19:50 -08004322#ifdef CONFIG_X86_64
Avi Kivity6aa8b732006-12-10 02:21:36 -08004323 case MSR_LSTAR:
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04004324 svm->vmcb->save.lstar = data;
Avi Kivity6aa8b732006-12-10 02:21:36 -08004325 break;
4326 case MSR_CSTAR:
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04004327 svm->vmcb->save.cstar = data;
Avi Kivity6aa8b732006-12-10 02:21:36 -08004328 break;
4329 case MSR_KERNEL_GS_BASE:
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04004330 svm->vmcb->save.kernel_gs_base = data;
Avi Kivity6aa8b732006-12-10 02:21:36 -08004331 break;
4332 case MSR_SYSCALL_MASK:
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04004333 svm->vmcb->save.sfmask = data;
Avi Kivity6aa8b732006-12-10 02:21:36 -08004334 break;
4335#endif
4336 case MSR_IA32_SYSENTER_CS:
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04004337 svm->vmcb->save.sysenter_cs = data;
Avi Kivity6aa8b732006-12-10 02:21:36 -08004338 break;
4339 case MSR_IA32_SYSENTER_EIP:
Andre Przywara017cb992009-05-28 11:56:31 +02004340 svm->sysenter_eip = data;
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04004341 svm->vmcb->save.sysenter_eip = data;
Avi Kivity6aa8b732006-12-10 02:21:36 -08004342 break;
4343 case MSR_IA32_SYSENTER_ESP:
Andre Przywara017cb992009-05-28 11:56:31 +02004344 svm->sysenter_esp = data;
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04004345 svm->vmcb->save.sysenter_esp = data;
Avi Kivity6aa8b732006-12-10 02:21:36 -08004346 break;
Paolo Bonzini46896c72015-11-12 14:49:16 +01004347 case MSR_TSC_AUX:
4348 if (!boot_cpu_has(X86_FEATURE_RDTSCP))
4349 return 1;
4350
4351 /*
4352 * This is rare, so we update the MSR here instead of using
4353 * direct_access_msrs. Doing that would require a rdmsr in
4354 * svm_vcpu_put.
4355 */
4356 svm->tsc_aux = data;
4357 wrmsrl(MSR_TSC_AUX, svm->tsc_aux);
4358 break;
Joerg Roedela2938c82008-02-13 16:30:28 +01004359 case MSR_IA32_DEBUGCTLMSR:
Avi Kivity2a6b20b2010-11-09 16:15:42 +02004360 if (!boot_cpu_has(X86_FEATURE_LBRV)) {
Christoffer Dalla737f252012-06-03 21:17:48 +03004361 vcpu_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTL 0x%llx, nop\n",
4362 __func__, data);
Joerg Roedel24e09cb2008-02-13 18:58:47 +01004363 break;
4364 }
4365 if (data & DEBUGCTL_RESERVED_BITS)
4366 return 1;
4367
4368 svm->vmcb->save.dbgctl = data;
Joerg Roedelb53ba3f2010-12-03 11:45:59 +01004369 mark_dirty(svm->vmcb, VMCB_LBR);
Joerg Roedel24e09cb2008-02-13 18:58:47 +01004370 if (data & (1ULL<<0))
4371 svm_enable_lbrv(svm);
4372 else
4373 svm_disable_lbrv(svm);
Joerg Roedela2938c82008-02-13 16:30:28 +01004374 break;
Alexander Grafb286d5d2008-11-25 20:17:05 +01004375 case MSR_VM_HSAVE_PA:
Joerg Roedele6aa9ab2009-08-07 11:49:33 +02004376 svm->nested.hsave_msr = data;
Alexander Grafb286d5d2008-11-25 20:17:05 +01004377 break;
Alexander Graf3c5d0a42009-06-15 15:21:23 +02004378 case MSR_VM_CR:
Joerg Roedel4a810182010-02-24 18:59:15 +01004379 return svm_set_vm_cr(vcpu, data);
Alexander Graf3c5d0a42009-06-15 15:21:23 +02004380 case MSR_VM_IGNNE:
Christoffer Dalla737f252012-06-03 21:17:48 +03004381 vcpu_unimpl(vcpu, "unimplemented wrmsr: 0x%x data 0x%llx\n", ecx, data);
Alexander Graf3c5d0a42009-06-15 15:21:23 +02004382 break;
Tom Lendackyd1d93fa2018-02-24 00:18:20 +01004383 case MSR_F10H_DECFG: {
4384 struct kvm_msr_entry msr_entry;
4385
4386 msr_entry.index = msr->index;
4387 if (svm_get_msr_feature(&msr_entry))
4388 return 1;
4389
4390 /* Check the supported bits */
4391 if (data & ~msr_entry.data)
4392 return 1;
4393
4394 /* Don't allow the guest to change a bit, #GP */
4395 if (!msr->host_initiated && (data ^ msr_entry.data))
4396 return 1;
4397
4398 svm->msr_decfg = data;
4399 break;
4400 }
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -05004401 case MSR_IA32_APICBASE:
4402 if (kvm_vcpu_apicv_active(vcpu))
4403 avic_update_vapic_bar(to_svm(vcpu), data);
Gustavo A. R. Silvab2869f22019-01-25 12:23:17 -06004404 /* Fall through */
Avi Kivity6aa8b732006-12-10 02:21:36 -08004405 default:
Will Auld8fe8ab42012-11-29 12:42:12 -08004406 return kvm_set_msr_common(vcpu, msr);
Avi Kivity6aa8b732006-12-10 02:21:36 -08004407 }
4408 return 0;
4409}
4410
Avi Kivity851ba692009-08-24 11:10:17 +03004411static int wrmsr_interception(struct vcpu_svm *svm)
Avi Kivity6aa8b732006-12-10 02:21:36 -08004412{
Sean Christopherson1edce0a2019-09-05 14:22:55 -07004413 return kvm_emulate_wrmsr(&svm->vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -08004414}
4415
Avi Kivity851ba692009-08-24 11:10:17 +03004416static int msr_interception(struct vcpu_svm *svm)
Avi Kivity6aa8b732006-12-10 02:21:36 -08004417{
Rusty Russelle756fc62007-07-30 20:07:08 +10004418 if (svm->vmcb->control.exit_info_1)
Avi Kivity851ba692009-08-24 11:10:17 +03004419 return wrmsr_interception(svm);
Avi Kivity6aa8b732006-12-10 02:21:36 -08004420 else
Avi Kivity851ba692009-08-24 11:10:17 +03004421 return rdmsr_interception(svm);
Avi Kivity6aa8b732006-12-10 02:21:36 -08004422}
4423
Avi Kivity851ba692009-08-24 11:10:17 +03004424static int interrupt_window_interception(struct vcpu_svm *svm)
Dor Laorc1150d82007-01-05 16:36:24 -08004425{
Avi Kivity3842d132010-07-27 12:30:24 +03004426 kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
Alexander Graff0b85052008-11-25 20:17:01 +01004427 svm_clear_vintr(svm);
Eddie Dong85f455f2007-07-06 12:20:49 +03004428 svm->vmcb->control.int_ctl &= ~V_IRQ_MASK;
Joerg Roedeldecdbf62010-12-03 11:45:52 +01004429 mark_dirty(svm->vmcb, VMCB_INTR);
Jason Wang675acb72012-03-08 18:07:56 +08004430 ++svm->vcpu.stat.irq_window_exits;
Dor Laorc1150d82007-01-05 16:36:24 -08004431 return 1;
4432}
4433
Mark Langsdorf565d0992009-10-06 14:25:02 -05004434static int pause_interception(struct vcpu_svm *svm)
4435{
Longpeng(Mike)de63ad42017-08-08 12:05:33 +08004436 struct kvm_vcpu *vcpu = &svm->vcpu;
4437 bool in_kernel = (svm_get_cpl(vcpu) == 0);
4438
Babu Moger8566ac82018-03-16 16:37:26 -04004439 if (pause_filter_thresh)
4440 grow_ple_window(vcpu);
4441
Longpeng(Mike)de63ad42017-08-08 12:05:33 +08004442 kvm_vcpu_on_spin(vcpu, in_kernel);
Mark Langsdorf565d0992009-10-06 14:25:02 -05004443 return 1;
4444}
4445
Gabriel L. Somlo87c00572014-05-07 16:52:13 -04004446static int nop_interception(struct vcpu_svm *svm)
4447{
Ladi Prosekb742c1e2017-06-22 09:05:26 +02004448 return kvm_skip_emulated_instruction(&(svm->vcpu));
Gabriel L. Somlo87c00572014-05-07 16:52:13 -04004449}
4450
4451static int monitor_interception(struct vcpu_svm *svm)
4452{
4453 printk_once(KERN_WARNING "kvm: MONITOR instruction emulated as NOP!\n");
4454 return nop_interception(svm);
4455}
4456
4457static int mwait_interception(struct vcpu_svm *svm)
4458{
4459 printk_once(KERN_WARNING "kvm: MWAIT instruction emulated as NOP!\n");
4460 return nop_interception(svm);
4461}
4462
Suravee Suthikulpanit18f40c52016-05-04 14:09:48 -05004463enum avic_ipi_failure_cause {
4464 AVIC_IPI_FAILURE_INVALID_INT_TYPE,
4465 AVIC_IPI_FAILURE_TARGET_NOT_RUNNING,
4466 AVIC_IPI_FAILURE_INVALID_TARGET,
4467 AVIC_IPI_FAILURE_INVALID_BACKING_PAGE,
4468};
4469
4470static int avic_incomplete_ipi_interception(struct vcpu_svm *svm)
4471{
4472 u32 icrh = svm->vmcb->control.exit_info_1 >> 32;
4473 u32 icrl = svm->vmcb->control.exit_info_1;
4474 u32 id = svm->vmcb->control.exit_info_2 >> 32;
Dan Carpenter5446a972016-05-23 13:20:10 +03004475 u32 index = svm->vmcb->control.exit_info_2 & 0xFF;
Suravee Suthikulpanit18f40c52016-05-04 14:09:48 -05004476 struct kvm_lapic *apic = svm->vcpu.arch.apic;
4477
4478 trace_kvm_avic_incomplete_ipi(svm->vcpu.vcpu_id, icrh, icrl, id, index);
4479
4480 switch (id) {
4481 case AVIC_IPI_FAILURE_INVALID_INT_TYPE:
4482 /*
4483 * AVIC hardware handles the generation of
4484 * IPIs when the specified Message Type is Fixed
4485 * (also known as fixed delivery mode) and
4486 * the Trigger Mode is edge-triggered. The hardware
4487 * also supports self and broadcast delivery modes
4488 * specified via the Destination Shorthand(DSH)
4489 * field of the ICRL. Logical and physical APIC ID
4490 * formats are supported. All other IPI types cause
4491 * a #VMEXIT, which needs to emulated.
4492 */
4493 kvm_lapic_reg_write(apic, APIC_ICR2, icrh);
4494 kvm_lapic_reg_write(apic, APIC_ICR, icrl);
4495 break;
4496 case AVIC_IPI_FAILURE_TARGET_NOT_RUNNING: {
Suthikulpanit, Suravee4a580382019-03-20 08:12:28 +00004497 int i;
4498 struct kvm_vcpu *vcpu;
4499 struct kvm *kvm = svm->vcpu.kvm;
Suravee Suthikulpanit18f40c52016-05-04 14:09:48 -05004500 struct kvm_lapic *apic = svm->vcpu.arch.apic;
4501
4502 /*
Suthikulpanit, Suravee4a580382019-03-20 08:12:28 +00004503 * At this point, we expect that the AVIC HW has already
4504 * set the appropriate IRR bits on the valid target
4505 * vcpus. So, we just need to kick the appropriate vcpu.
Suravee Suthikulpanit18f40c52016-05-04 14:09:48 -05004506 */
Suthikulpanit, Suravee4a580382019-03-20 08:12:28 +00004507 kvm_for_each_vcpu(i, vcpu, kvm) {
4508 bool m = kvm_apic_match_dest(vcpu, apic,
4509 icrl & KVM_APIC_SHORT_MASK,
4510 GET_APIC_DEST_FIELD(icrh),
4511 icrl & KVM_APIC_DEST_MASK);
4512
4513 if (m && !avic_vcpu_is_running(vcpu))
4514 kvm_vcpu_wake_up(vcpu);
4515 }
Suravee Suthikulpanit18f40c52016-05-04 14:09:48 -05004516 break;
4517 }
4518 case AVIC_IPI_FAILURE_INVALID_TARGET:
Suravee Suthikulpanit37ef0c42019-01-22 10:24:19 +00004519 WARN_ONCE(1, "Invalid IPI target: index=%u, vcpu=%d, icr=%#0x:%#0x\n",
4520 index, svm->vcpu.vcpu_id, icrh, icrl);
Suravee Suthikulpanit18f40c52016-05-04 14:09:48 -05004521 break;
4522 case AVIC_IPI_FAILURE_INVALID_BACKING_PAGE:
4523 WARN_ONCE(1, "Invalid backing page\n");
4524 break;
4525 default:
4526 pr_err("Unknown IPI interception\n");
4527 }
4528
4529 return 1;
4530}
4531
4532static u32 *avic_get_logical_id_entry(struct kvm_vcpu *vcpu, u32 ldr, bool flat)
4533{
Sean Christopherson81811c12018-03-20 12:17:21 -07004534 struct kvm_svm *kvm_svm = to_kvm_svm(vcpu->kvm);
Suravee Suthikulpanit18f40c52016-05-04 14:09:48 -05004535 int index;
4536 u32 *logical_apic_id_table;
4537 int dlid = GET_APIC_LOGICAL_ID(ldr);
4538
4539 if (!dlid)
4540 return NULL;
4541
4542 if (flat) { /* flat */
4543 index = ffs(dlid) - 1;
4544 if (index > 7)
4545 return NULL;
4546 } else { /* cluster */
4547 int cluster = (dlid & 0xf0) >> 4;
4548 int apic = ffs(dlid & 0x0f) - 1;
4549
4550 if ((apic < 0) || (apic > 7) ||
4551 (cluster >= 0xf))
4552 return NULL;
4553 index = (cluster << 2) + apic;
4554 }
4555
Sean Christopherson81811c12018-03-20 12:17:21 -07004556 logical_apic_id_table = (u32 *) page_address(kvm_svm->avic_logical_id_table_page);
Suravee Suthikulpanit18f40c52016-05-04 14:09:48 -05004557
4558 return &logical_apic_id_table[index];
4559}
4560
Suthikulpanit, Suravee98d90582019-01-29 08:08:42 +00004561static int avic_ldr_write(struct kvm_vcpu *vcpu, u8 g_physical_id, u32 ldr)
Suravee Suthikulpanit18f40c52016-05-04 14:09:48 -05004562{
4563 bool flat;
4564 u32 *entry, new_entry;
4565
4566 flat = kvm_lapic_get_reg(vcpu->arch.apic, APIC_DFR) == APIC_DFR_FLAT;
4567 entry = avic_get_logical_id_entry(vcpu, ldr, flat);
4568 if (!entry)
4569 return -EINVAL;
4570
4571 new_entry = READ_ONCE(*entry);
4572 new_entry &= ~AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK;
4573 new_entry |= (g_physical_id & AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK);
Suthikulpanit, Suravee98d90582019-01-29 08:08:42 +00004574 new_entry |= AVIC_LOGICAL_ID_ENTRY_VALID_MASK;
Suravee Suthikulpanit18f40c52016-05-04 14:09:48 -05004575 WRITE_ONCE(*entry, new_entry);
4576
4577 return 0;
4578}
4579
Suthikulpanit, Suravee98d90582019-01-29 08:08:42 +00004580static void avic_invalidate_logical_id_entry(struct kvm_vcpu *vcpu)
4581{
4582 struct vcpu_svm *svm = to_svm(vcpu);
4583 bool flat = svm->dfr_reg == APIC_DFR_FLAT;
4584 u32 *entry = avic_get_logical_id_entry(vcpu, svm->ldr_reg, flat);
4585
4586 if (entry)
Suthikulpanit, Suraveee44e3ea2019-03-26 03:57:37 +00004587 clear_bit(AVIC_LOGICAL_ID_ENTRY_VALID_BIT, (unsigned long *)entry);
Suthikulpanit, Suravee98d90582019-01-29 08:08:42 +00004588}
4589
Suravee Suthikulpanit18f40c52016-05-04 14:09:48 -05004590static int avic_handle_ldr_update(struct kvm_vcpu *vcpu)
4591{
Suthikulpanit, Suravee98d90582019-01-29 08:08:42 +00004592 int ret = 0;
Suravee Suthikulpanit18f40c52016-05-04 14:09:48 -05004593 struct vcpu_svm *svm = to_svm(vcpu);
4594 u32 ldr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_LDR);
4595
Suthikulpanit, Suravee98d90582019-01-29 08:08:42 +00004596 if (ldr == svm->ldr_reg)
4597 return 0;
Suravee Suthikulpanit18f40c52016-05-04 14:09:48 -05004598
Suthikulpanit, Suravee98d90582019-01-29 08:08:42 +00004599 avic_invalidate_logical_id_entry(vcpu);
4600
4601 if (ldr)
4602 ret = avic_ldr_write(vcpu, vcpu->vcpu_id, ldr);
4603
4604 if (!ret)
Suravee Suthikulpanit18f40c52016-05-04 14:09:48 -05004605 svm->ldr_reg = ldr;
Suthikulpanit, Suravee98d90582019-01-29 08:08:42 +00004606
Suravee Suthikulpanit18f40c52016-05-04 14:09:48 -05004607 return ret;
4608}
4609
4610static int avic_handle_apic_id_update(struct kvm_vcpu *vcpu)
4611{
4612 u64 *old, *new;
4613 struct vcpu_svm *svm = to_svm(vcpu);
4614 u32 apic_id_reg = kvm_lapic_get_reg(vcpu->arch.apic, APIC_ID);
4615 u32 id = (apic_id_reg >> 24) & 0xff;
4616
4617 if (vcpu->vcpu_id == id)
4618 return 0;
4619
4620 old = avic_get_physical_id_entry(vcpu, vcpu->vcpu_id);
4621 new = avic_get_physical_id_entry(vcpu, id);
4622 if (!new || !old)
4623 return 1;
4624
4625 /* We need to move physical_id_entry to new offset */
4626 *new = *old;
4627 *old = 0ULL;
4628 to_svm(vcpu)->avic_physical_id_cache = new;
4629
4630 /*
4631 * Also update the guest physical APIC ID in the logical
4632 * APIC ID table entry if already setup the LDR.
4633 */
4634 if (svm->ldr_reg)
4635 avic_handle_ldr_update(vcpu);
4636
4637 return 0;
4638}
4639
Suthikulpanit, Suravee98d90582019-01-29 08:08:42 +00004640static void avic_handle_dfr_update(struct kvm_vcpu *vcpu)
Suravee Suthikulpanit18f40c52016-05-04 14:09:48 -05004641{
4642 struct vcpu_svm *svm = to_svm(vcpu);
Suravee Suthikulpanit18f40c52016-05-04 14:09:48 -05004643 u32 dfr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_DFR);
Suravee Suthikulpanit18f40c52016-05-04 14:09:48 -05004644
Suthikulpanit, Suravee98d90582019-01-29 08:08:42 +00004645 if (svm->dfr_reg == dfr)
4646 return;
Suravee Suthikulpanit18f40c52016-05-04 14:09:48 -05004647
Suthikulpanit, Suravee98d90582019-01-29 08:08:42 +00004648 avic_invalidate_logical_id_entry(vcpu);
4649 svm->dfr_reg = dfr;
Suravee Suthikulpanit18f40c52016-05-04 14:09:48 -05004650}
4651
4652static int avic_unaccel_trap_write(struct vcpu_svm *svm)
4653{
4654 struct kvm_lapic *apic = svm->vcpu.arch.apic;
4655 u32 offset = svm->vmcb->control.exit_info_1 &
4656 AVIC_UNACCEL_ACCESS_OFFSET_MASK;
4657
4658 switch (offset) {
4659 case APIC_ID:
4660 if (avic_handle_apic_id_update(&svm->vcpu))
4661 return 0;
4662 break;
4663 case APIC_LDR:
4664 if (avic_handle_ldr_update(&svm->vcpu))
4665 return 0;
4666 break;
4667 case APIC_DFR:
4668 avic_handle_dfr_update(&svm->vcpu);
4669 break;
4670 default:
4671 break;
4672 }
4673
4674 kvm_lapic_reg_write(apic, offset, kvm_lapic_get_reg(apic, offset));
4675
4676 return 1;
4677}
4678
4679static bool is_avic_unaccelerated_access_trap(u32 offset)
4680{
4681 bool ret = false;
4682
4683 switch (offset) {
4684 case APIC_ID:
4685 case APIC_EOI:
4686 case APIC_RRR:
4687 case APIC_LDR:
4688 case APIC_DFR:
4689 case APIC_SPIV:
4690 case APIC_ESR:
4691 case APIC_ICR:
4692 case APIC_LVTT:
4693 case APIC_LVTTHMR:
4694 case APIC_LVTPC:
4695 case APIC_LVT0:
4696 case APIC_LVT1:
4697 case APIC_LVTERR:
4698 case APIC_TMICT:
4699 case APIC_TDCR:
4700 ret = true;
4701 break;
4702 default:
4703 break;
4704 }
4705 return ret;
4706}
4707
4708static int avic_unaccelerated_access_interception(struct vcpu_svm *svm)
4709{
4710 int ret = 0;
4711 u32 offset = svm->vmcb->control.exit_info_1 &
4712 AVIC_UNACCEL_ACCESS_OFFSET_MASK;
4713 u32 vector = svm->vmcb->control.exit_info_2 &
4714 AVIC_UNACCEL_ACCESS_VECTOR_MASK;
4715 bool write = (svm->vmcb->control.exit_info_1 >> 32) &
4716 AVIC_UNACCEL_ACCESS_WRITE_MASK;
4717 bool trap = is_avic_unaccelerated_access_trap(offset);
4718
4719 trace_kvm_avic_unaccelerated_access(svm->vcpu.vcpu_id, offset,
4720 trap, write, vector);
4721 if (trap) {
4722 /* Handling Trap */
4723 WARN_ONCE(!write, "svm: Handling trap read.\n");
4724 ret = avic_unaccel_trap_write(svm);
4725 } else {
4726 /* Handling Fault */
Sean Christopherson0ce97a22018-08-23 13:56:52 -07004727 ret = (kvm_emulate_instruction(&svm->vcpu, 0) == EMULATE_DONE);
Suravee Suthikulpanit18f40c52016-05-04 14:09:48 -05004728 }
4729
4730 return ret;
4731}
4732
Mathias Krause09941fb2012-08-30 01:30:20 +02004733static int (*const svm_exit_handlers[])(struct vcpu_svm *svm) = {
Andre Przywara7ff76d52010-12-21 11:12:04 +01004734 [SVM_EXIT_READ_CR0] = cr_interception,
4735 [SVM_EXIT_READ_CR3] = cr_interception,
4736 [SVM_EXIT_READ_CR4] = cr_interception,
4737 [SVM_EXIT_READ_CR8] = cr_interception,
David Kaplan5e575182015-03-06 14:44:35 -06004738 [SVM_EXIT_CR0_SEL_WRITE] = cr_interception,
Joerg Roedel628afd22011-04-04 12:39:36 +02004739 [SVM_EXIT_WRITE_CR0] = cr_interception,
Andre Przywara7ff76d52010-12-21 11:12:04 +01004740 [SVM_EXIT_WRITE_CR3] = cr_interception,
4741 [SVM_EXIT_WRITE_CR4] = cr_interception,
Joerg Roedele0231712010-02-24 18:59:10 +01004742 [SVM_EXIT_WRITE_CR8] = cr8_write_interception,
Andre Przywaracae37972010-12-21 11:12:05 +01004743 [SVM_EXIT_READ_DR0] = dr_interception,
4744 [SVM_EXIT_READ_DR1] = dr_interception,
4745 [SVM_EXIT_READ_DR2] = dr_interception,
4746 [SVM_EXIT_READ_DR3] = dr_interception,
4747 [SVM_EXIT_READ_DR4] = dr_interception,
4748 [SVM_EXIT_READ_DR5] = dr_interception,
4749 [SVM_EXIT_READ_DR6] = dr_interception,
4750 [SVM_EXIT_READ_DR7] = dr_interception,
4751 [SVM_EXIT_WRITE_DR0] = dr_interception,
4752 [SVM_EXIT_WRITE_DR1] = dr_interception,
4753 [SVM_EXIT_WRITE_DR2] = dr_interception,
4754 [SVM_EXIT_WRITE_DR3] = dr_interception,
4755 [SVM_EXIT_WRITE_DR4] = dr_interception,
4756 [SVM_EXIT_WRITE_DR5] = dr_interception,
4757 [SVM_EXIT_WRITE_DR6] = dr_interception,
4758 [SVM_EXIT_WRITE_DR7] = dr_interception,
Jan Kiszkad0bfb942008-12-15 13:52:10 +01004759 [SVM_EXIT_EXCP_BASE + DB_VECTOR] = db_interception,
4760 [SVM_EXIT_EXCP_BASE + BP_VECTOR] = bp_interception,
Anthony Liguori7aa81cc2007-09-17 14:57:50 -05004761 [SVM_EXIT_EXCP_BASE + UD_VECTOR] = ud_interception,
Joerg Roedele0231712010-02-24 18:59:10 +01004762 [SVM_EXIT_EXCP_BASE + PF_VECTOR] = pf_interception,
Joerg Roedele0231712010-02-24 18:59:10 +01004763 [SVM_EXIT_EXCP_BASE + MC_VECTOR] = mc_interception,
Eric Northup54a20552015-11-03 18:03:53 +01004764 [SVM_EXIT_EXCP_BASE + AC_VECTOR] = ac_interception,
Liran Alon97184202018-03-12 13:12:52 +02004765 [SVM_EXIT_EXCP_BASE + GP_VECTOR] = gp_interception,
Joerg Roedele0231712010-02-24 18:59:10 +01004766 [SVM_EXIT_INTR] = intr_interception,
Joerg Roedelc47f0982008-04-30 17:56:00 +02004767 [SVM_EXIT_NMI] = nmi_interception,
Avi Kivity6aa8b732006-12-10 02:21:36 -08004768 [SVM_EXIT_SMI] = nop_on_interception,
4769 [SVM_EXIT_INIT] = nop_on_interception,
Dor Laorc1150d82007-01-05 16:36:24 -08004770 [SVM_EXIT_VINTR] = interrupt_window_interception,
Avi Kivity332b56e2011-11-10 14:57:24 +02004771 [SVM_EXIT_RDPMC] = rdpmc_interception,
Avi Kivity6aa8b732006-12-10 02:21:36 -08004772 [SVM_EXIT_CPUID] = cpuid_interception,
Gleb Natapov95ba8273132009-04-21 17:45:08 +03004773 [SVM_EXIT_IRET] = iret_interception,
Avi Kivitycf5a94d2007-10-28 16:11:58 +02004774 [SVM_EXIT_INVD] = emulate_on_interception,
Mark Langsdorf565d0992009-10-06 14:25:02 -05004775 [SVM_EXIT_PAUSE] = pause_interception,
Avi Kivity6aa8b732006-12-10 02:21:36 -08004776 [SVM_EXIT_HLT] = halt_interception,
Marcelo Tosattia7052892008-09-23 13:18:35 -03004777 [SVM_EXIT_INVLPG] = invlpg_interception,
Alexander Grafff092382009-06-15 15:21:24 +02004778 [SVM_EXIT_INVLPGA] = invlpga_interception,
Joerg Roedele0231712010-02-24 18:59:10 +01004779 [SVM_EXIT_IOIO] = io_interception,
Avi Kivity6aa8b732006-12-10 02:21:36 -08004780 [SVM_EXIT_MSR] = msr_interception,
4781 [SVM_EXIT_TASK_SWITCH] = task_switch_interception,
Joerg Roedel46fe4dd2007-01-26 00:56:42 -08004782 [SVM_EXIT_SHUTDOWN] = shutdown_interception,
Alexander Graf3d6368e2008-11-25 20:17:07 +01004783 [SVM_EXIT_VMRUN] = vmrun_interception,
Avi Kivity02e235b2007-02-19 14:37:47 +02004784 [SVM_EXIT_VMMCALL] = vmmcall_interception,
Alexander Graf55426752008-11-25 20:17:06 +01004785 [SVM_EXIT_VMLOAD] = vmload_interception,
4786 [SVM_EXIT_VMSAVE] = vmsave_interception,
Alexander Graf1371d902008-11-25 20:17:04 +01004787 [SVM_EXIT_STGI] = stgi_interception,
4788 [SVM_EXIT_CLGI] = clgi_interception,
Joerg Roedel532a46b2009-10-09 16:08:32 +02004789 [SVM_EXIT_SKINIT] = skinit_interception,
David Kaplandab429a2015-03-02 13:43:37 -06004790 [SVM_EXIT_WBINVD] = wbinvd_interception,
Gabriel L. Somlo87c00572014-05-07 16:52:13 -04004791 [SVM_EXIT_MONITOR] = monitor_interception,
4792 [SVM_EXIT_MWAIT] = mwait_interception,
Joerg Roedel81dd35d2010-12-07 17:15:06 +01004793 [SVM_EXIT_XSETBV] = xsetbv_interception,
Paolo Bonzinid0006532017-08-11 18:36:43 +02004794 [SVM_EXIT_NPF] = npf_interception,
Brijesh Singh7607b712018-02-19 10:14:44 -06004795 [SVM_EXIT_RSM] = rsm_interception,
Suravee Suthikulpanit18f40c52016-05-04 14:09:48 -05004796 [SVM_EXIT_AVIC_INCOMPLETE_IPI] = avic_incomplete_ipi_interception,
4797 [SVM_EXIT_AVIC_UNACCELERATED_ACCESS] = avic_unaccelerated_access_interception,
Avi Kivity6aa8b732006-12-10 02:21:36 -08004798};
4799
Joe Perchesae8cc052011-04-24 22:00:50 -07004800static void dump_vmcb(struct kvm_vcpu *vcpu)
Joerg Roedel3f10c842010-05-05 16:04:42 +02004801{
4802 struct vcpu_svm *svm = to_svm(vcpu);
4803 struct vmcb_control_area *control = &svm->vmcb->control;
4804 struct vmcb_save_area *save = &svm->vmcb->save;
4805
Paolo Bonzini6f2f8452019-05-20 15:34:35 +02004806 if (!dump_invalid_vmcb) {
4807 pr_warn_ratelimited("set kvm_amd.dump_invalid_vmcb=1 to dump internal KVM state.\n");
4808 return;
4809 }
4810
Joerg Roedel3f10c842010-05-05 16:04:42 +02004811 pr_err("VMCB Control Area:\n");
Joe Perchesae8cc052011-04-24 22:00:50 -07004812 pr_err("%-20s%04x\n", "cr_read:", control->intercept_cr & 0xffff);
4813 pr_err("%-20s%04x\n", "cr_write:", control->intercept_cr >> 16);
4814 pr_err("%-20s%04x\n", "dr_read:", control->intercept_dr & 0xffff);
4815 pr_err("%-20s%04x\n", "dr_write:", control->intercept_dr >> 16);
4816 pr_err("%-20s%08x\n", "exceptions:", control->intercept_exceptions);
4817 pr_err("%-20s%016llx\n", "intercepts:", control->intercept);
4818 pr_err("%-20s%d\n", "pause filter count:", control->pause_filter_count);
Babu Moger1d8fb442018-03-16 16:37:25 -04004819 pr_err("%-20s%d\n", "pause filter threshold:",
4820 control->pause_filter_thresh);
Joe Perchesae8cc052011-04-24 22:00:50 -07004821 pr_err("%-20s%016llx\n", "iopm_base_pa:", control->iopm_base_pa);
4822 pr_err("%-20s%016llx\n", "msrpm_base_pa:", control->msrpm_base_pa);
4823 pr_err("%-20s%016llx\n", "tsc_offset:", control->tsc_offset);
4824 pr_err("%-20s%d\n", "asid:", control->asid);
4825 pr_err("%-20s%d\n", "tlb_ctl:", control->tlb_ctl);
4826 pr_err("%-20s%08x\n", "int_ctl:", control->int_ctl);
4827 pr_err("%-20s%08x\n", "int_vector:", control->int_vector);
4828 pr_err("%-20s%08x\n", "int_state:", control->int_state);
4829 pr_err("%-20s%08x\n", "exit_code:", control->exit_code);
4830 pr_err("%-20s%016llx\n", "exit_info1:", control->exit_info_1);
4831 pr_err("%-20s%016llx\n", "exit_info2:", control->exit_info_2);
4832 pr_err("%-20s%08x\n", "exit_int_info:", control->exit_int_info);
4833 pr_err("%-20s%08x\n", "exit_int_info_err:", control->exit_int_info_err);
4834 pr_err("%-20s%lld\n", "nested_ctl:", control->nested_ctl);
4835 pr_err("%-20s%016llx\n", "nested_cr3:", control->nested_cr3);
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -05004836 pr_err("%-20s%016llx\n", "avic_vapic_bar:", control->avic_vapic_bar);
Joe Perchesae8cc052011-04-24 22:00:50 -07004837 pr_err("%-20s%08x\n", "event_inj:", control->event_inj);
4838 pr_err("%-20s%08x\n", "event_inj_err:", control->event_inj_err);
Janakarajan Natarajan0dc92112017-07-06 15:50:45 -05004839 pr_err("%-20s%lld\n", "virt_ext:", control->virt_ext);
Joe Perchesae8cc052011-04-24 22:00:50 -07004840 pr_err("%-20s%016llx\n", "next_rip:", control->next_rip);
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -05004841 pr_err("%-20s%016llx\n", "avic_backing_page:", control->avic_backing_page);
4842 pr_err("%-20s%016llx\n", "avic_logical_id:", control->avic_logical_id);
4843 pr_err("%-20s%016llx\n", "avic_physical_id:", control->avic_physical_id);
Joerg Roedel3f10c842010-05-05 16:04:42 +02004844 pr_err("VMCB State Save Area:\n");
Joe Perchesae8cc052011-04-24 22:00:50 -07004845 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4846 "es:",
4847 save->es.selector, save->es.attrib,
4848 save->es.limit, save->es.base);
4849 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4850 "cs:",
4851 save->cs.selector, save->cs.attrib,
4852 save->cs.limit, save->cs.base);
4853 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4854 "ss:",
4855 save->ss.selector, save->ss.attrib,
4856 save->ss.limit, save->ss.base);
4857 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4858 "ds:",
4859 save->ds.selector, save->ds.attrib,
4860 save->ds.limit, save->ds.base);
4861 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4862 "fs:",
4863 save->fs.selector, save->fs.attrib,
4864 save->fs.limit, save->fs.base);
4865 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4866 "gs:",
4867 save->gs.selector, save->gs.attrib,
4868 save->gs.limit, save->gs.base);
4869 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4870 "gdtr:",
4871 save->gdtr.selector, save->gdtr.attrib,
4872 save->gdtr.limit, save->gdtr.base);
4873 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4874 "ldtr:",
4875 save->ldtr.selector, save->ldtr.attrib,
4876 save->ldtr.limit, save->ldtr.base);
4877 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4878 "idtr:",
4879 save->idtr.selector, save->idtr.attrib,
4880 save->idtr.limit, save->idtr.base);
4881 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4882 "tr:",
4883 save->tr.selector, save->tr.attrib,
4884 save->tr.limit, save->tr.base);
Joerg Roedel3f10c842010-05-05 16:04:42 +02004885 pr_err("cpl: %d efer: %016llx\n",
4886 save->cpl, save->efer);
Joe Perchesae8cc052011-04-24 22:00:50 -07004887 pr_err("%-15s %016llx %-13s %016llx\n",
4888 "cr0:", save->cr0, "cr2:", save->cr2);
4889 pr_err("%-15s %016llx %-13s %016llx\n",
4890 "cr3:", save->cr3, "cr4:", save->cr4);
4891 pr_err("%-15s %016llx %-13s %016llx\n",
4892 "dr6:", save->dr6, "dr7:", save->dr7);
4893 pr_err("%-15s %016llx %-13s %016llx\n",
4894 "rip:", save->rip, "rflags:", save->rflags);
4895 pr_err("%-15s %016llx %-13s %016llx\n",
4896 "rsp:", save->rsp, "rax:", save->rax);
4897 pr_err("%-15s %016llx %-13s %016llx\n",
4898 "star:", save->star, "lstar:", save->lstar);
4899 pr_err("%-15s %016llx %-13s %016llx\n",
4900 "cstar:", save->cstar, "sfmask:", save->sfmask);
4901 pr_err("%-15s %016llx %-13s %016llx\n",
4902 "kernel_gs_base:", save->kernel_gs_base,
4903 "sysenter_cs:", save->sysenter_cs);
4904 pr_err("%-15s %016llx %-13s %016llx\n",
4905 "sysenter_esp:", save->sysenter_esp,
4906 "sysenter_eip:", save->sysenter_eip);
4907 pr_err("%-15s %016llx %-13s %016llx\n",
4908 "gpat:", save->g_pat, "dbgctl:", save->dbgctl);
4909 pr_err("%-15s %016llx %-13s %016llx\n",
4910 "br_from:", save->br_from, "br_to:", save->br_to);
4911 pr_err("%-15s %016llx %-13s %016llx\n",
4912 "excp_from:", save->last_excp_from,
4913 "excp_to:", save->last_excp_to);
Joerg Roedel3f10c842010-05-05 16:04:42 +02004914}
4915
Avi Kivity586f9602010-11-18 13:09:54 +02004916static void svm_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2)
4917{
4918 struct vmcb_control_area *control = &to_svm(vcpu)->vmcb->control;
4919
4920 *info1 = control->exit_info_1;
4921 *info2 = control->exit_info_2;
4922}
4923
Avi Kivity851ba692009-08-24 11:10:17 +03004924static int handle_exit(struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -08004925{
Avi Kivity04d2cc72007-09-10 18:10:54 +03004926 struct vcpu_svm *svm = to_svm(vcpu);
Avi Kivity851ba692009-08-24 11:10:17 +03004927 struct kvm_run *kvm_run = vcpu->run;
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04004928 u32 exit_code = svm->vmcb->control.exit_code;
Avi Kivity6aa8b732006-12-10 02:21:36 -08004929
Paolo Bonzini8b89fe12015-12-10 18:37:32 +01004930 trace_kvm_exit(exit_code, vcpu, KVM_ISA_SVM);
4931
Roedel, Joerg4ee546b2010-12-03 10:50:51 +01004932 if (!is_cr_intercept(svm, INTERCEPT_CR0_WRITE))
Joerg Roedel2be4fc72010-04-22 12:33:09 +02004933 vcpu->arch.cr0 = svm->vmcb->save.cr0;
4934 if (npt_enabled)
4935 vcpu->arch.cr3 = svm->vmcb->save.cr3;
Joerg Roedelaf9ca2d2008-04-30 17:56:03 +02004936
Joerg Roedelcd3ff652009-10-09 16:08:26 +02004937 if (unlikely(svm->nested.exit_required)) {
4938 nested_svm_vmexit(svm);
4939 svm->nested.exit_required = false;
4940
4941 return 1;
4942 }
4943
Joerg Roedel20307532010-11-29 17:51:48 +01004944 if (is_guest_mode(vcpu)) {
Joerg Roedel410e4d52009-08-07 11:49:44 +02004945 int vmexit;
4946
Joerg Roedeld8cabdd2009-10-09 16:08:28 +02004947 trace_kvm_nested_vmexit(svm->vmcb->save.rip, exit_code,
4948 svm->vmcb->control.exit_info_1,
4949 svm->vmcb->control.exit_info_2,
4950 svm->vmcb->control.exit_int_info,
Stefan Hajnoczie097e5f2011-07-22 12:46:52 +01004951 svm->vmcb->control.exit_int_info_err,
4952 KVM_ISA_SVM);
Joerg Roedeld8cabdd2009-10-09 16:08:28 +02004953
Joerg Roedel410e4d52009-08-07 11:49:44 +02004954 vmexit = nested_svm_exit_special(svm);
4955
4956 if (vmexit == NESTED_EXIT_CONTINUE)
4957 vmexit = nested_svm_exit_handled(svm);
4958
4959 if (vmexit == NESTED_EXIT_DONE)
Alexander Grafcf74a782008-11-25 20:17:08 +01004960 return 1;
Alexander Grafcf74a782008-11-25 20:17:08 +01004961 }
4962
Joerg Roedela5c38322009-08-07 11:49:32 +02004963 svm_complete_interrupts(svm);
4964
Avi Kivity04d2cc72007-09-10 18:10:54 +03004965 if (svm->vmcb->control.exit_code == SVM_EXIT_ERR) {
4966 kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY;
4967 kvm_run->fail_entry.hardware_entry_failure_reason
4968 = svm->vmcb->control.exit_code;
Joerg Roedel3f10c842010-05-05 16:04:42 +02004969 dump_vmcb(vcpu);
Avi Kivity04d2cc72007-09-10 18:10:54 +03004970 return 0;
4971 }
4972
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04004973 if (is_external_interrupt(svm->vmcb->control.exit_int_info) &&
Joerg Roedel709ddeb2008-02-07 13:47:45 +01004974 exit_code != SVM_EXIT_EXCP_BASE + PF_VECTOR &&
Joerg Roedel55c5e462010-09-10 17:31:04 +02004975 exit_code != SVM_EXIT_NPF && exit_code != SVM_EXIT_TASK_SWITCH &&
4976 exit_code != SVM_EXIT_INTR && exit_code != SVM_EXIT_NMI)
Borislav Petkov6614c7d2013-04-26 00:22:01 +02004977 printk(KERN_ERR "%s: unexpected exit_int_info 0x%x "
Avi Kivity6aa8b732006-12-10 02:21:36 -08004978 "exit_code 0x%x\n",
Harvey Harrisonb8688d52008-03-03 12:59:56 -08004979 __func__, svm->vmcb->control.exit_int_info,
Avi Kivity6aa8b732006-12-10 02:21:36 -08004980 exit_code);
4981
Ahmed S. Darwish9d8f5492007-02-19 14:37:46 +02004982 if (exit_code >= ARRAY_SIZE(svm_exit_handlers)
Joe Perches56919c52007-11-12 20:06:51 -08004983 || !svm_exit_handlers[exit_code]) {
Liran Alon7396d332019-08-26 13:16:43 +03004984 vcpu_unimpl(vcpu, "svm: unexpected exit reason 0x%x\n", exit_code);
4985 dump_vmcb(vcpu);
4986 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
4987 vcpu->run->internal.suberror =
4988 KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON;
4989 vcpu->run->internal.ndata = 1;
4990 vcpu->run->internal.data[0] = exit_code;
4991 return 0;
Avi Kivity6aa8b732006-12-10 02:21:36 -08004992 }
4993
Avi Kivity851ba692009-08-24 11:10:17 +03004994 return svm_exit_handlers[exit_code](svm);
Avi Kivity6aa8b732006-12-10 02:21:36 -08004995}
4996
4997static void reload_tss(struct kvm_vcpu *vcpu)
4998{
4999 int cpu = raw_smp_processor_id();
5000
Tejun Heo0fe1e002009-10-29 22:34:14 +09005001 struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
5002 sd->tss_desc->type = 9; /* available 32/64-bit TSS */
Avi Kivity6aa8b732006-12-10 02:21:36 -08005003 load_TR_desc();
5004}
5005
Brijesh Singh70cd94e2017-12-04 10:57:34 -06005006static void pre_sev_run(struct vcpu_svm *svm, int cpu)
5007{
5008 struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
5009 int asid = sev_get_asid(svm->vcpu.kvm);
5010
5011 /* Assign the asid allocated with this SEV guest */
5012 svm->vmcb->control.asid = asid;
5013
5014 /*
5015 * Flush guest TLB:
5016 *
5017 * 1) when different VMCB for the same ASID is to be run on the same host CPU.
5018 * 2) or this VMCB was executed on different host CPU in previous VMRUNs.
5019 */
5020 if (sd->sev_vmcbs[asid] == svm->vmcb &&
5021 svm->last_cpu == cpu)
5022 return;
5023
5024 svm->last_cpu = cpu;
5025 sd->sev_vmcbs[asid] = svm->vmcb;
5026 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID;
5027 mark_dirty(svm->vmcb, VMCB_ASID);
5028}
5029
Rusty Russelle756fc62007-07-30 20:07:08 +10005030static void pre_svm_run(struct vcpu_svm *svm)
Avi Kivity6aa8b732006-12-10 02:21:36 -08005031{
5032 int cpu = raw_smp_processor_id();
5033
Tejun Heo0fe1e002009-10-29 22:34:14 +09005034 struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -08005035
Brijesh Singh70cd94e2017-12-04 10:57:34 -06005036 if (sev_guest(svm->vcpu.kvm))
5037 return pre_sev_run(svm, cpu);
5038
Marcelo Tosatti4b656b12009-07-21 12:47:45 -03005039 /* FIXME: handle wraparound of asid_generation */
Tejun Heo0fe1e002009-10-29 22:34:14 +09005040 if (svm->asid_generation != sd->asid_generation)
5041 new_asid(svm, sd);
Avi Kivity6aa8b732006-12-10 02:21:36 -08005042}
5043
Gleb Natapov95ba8273132009-04-21 17:45:08 +03005044static void svm_inject_nmi(struct kvm_vcpu *vcpu)
5045{
5046 struct vcpu_svm *svm = to_svm(vcpu);
5047
5048 svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI;
5049 vcpu->arch.hflags |= HF_NMI_MASK;
Joerg Roedel8a05a1b82010-11-30 18:04:00 +01005050 set_intercept(svm, INTERCEPT_IRET);
Gleb Natapov95ba8273132009-04-21 17:45:08 +03005051 ++vcpu->stat.nmi_injections;
5052}
Avi Kivity6aa8b732006-12-10 02:21:36 -08005053
Eddie Dong85f455f2007-07-06 12:20:49 +03005054static inline void svm_inject_irq(struct vcpu_svm *svm, int irq)
Avi Kivity6aa8b732006-12-10 02:21:36 -08005055{
5056 struct vmcb_control_area *control;
5057
Suravee Suthikulpanit340d3bc2016-05-04 14:09:47 -05005058 /* The following fields are ignored when AVIC is enabled */
Rusty Russelle756fc62007-07-30 20:07:08 +10005059 control = &svm->vmcb->control;
Eddie Dong85f455f2007-07-06 12:20:49 +03005060 control->int_vector = irq;
Avi Kivity6aa8b732006-12-10 02:21:36 -08005061 control->int_ctl &= ~V_INTR_PRIO_MASK;
5062 control->int_ctl |= V_IRQ_MASK |
5063 ((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT);
Joerg Roedeldecdbf62010-12-03 11:45:52 +01005064 mark_dirty(svm->vmcb, VMCB_INTR);
Avi Kivity6aa8b732006-12-10 02:21:36 -08005065}
5066
Gleb Natapov66fd3f72009-05-11 13:35:50 +03005067static void svm_set_irq(struct kvm_vcpu *vcpu)
Eddie Dong2a8067f2007-08-06 16:29:07 +03005068{
5069 struct vcpu_svm *svm = to_svm(vcpu);
5070
Joerg Roedel2af91942009-08-07 11:49:28 +02005071 BUG_ON(!(gif_set(svm)));
Alexander Grafcf74a782008-11-25 20:17:08 +01005072
Gleb Natapov9fb2d2b2010-05-23 14:28:26 +03005073 trace_kvm_inj_virq(vcpu->arch.interrupt.nr);
5074 ++vcpu->stat.irq_injections;
5075
Alexander Graf219b65d2009-06-15 15:21:25 +02005076 svm->vmcb->control.event_inj = vcpu->arch.interrupt.nr |
5077 SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR;
Eddie Dong2a8067f2007-08-06 16:29:07 +03005078}
5079
Suravee Suthikulpanit3bbf3562016-05-04 14:09:51 -05005080static inline bool svm_nested_virtualize_tpr(struct kvm_vcpu *vcpu)
5081{
5082 return is_guest_mode(vcpu) && (vcpu->arch.hflags & HF_VINTR_MASK);
5083}
5084
Gleb Natapov95ba8273132009-04-21 17:45:08 +03005085static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
5086{
5087 struct vcpu_svm *svm = to_svm(vcpu);
5088
Suravee Suthikulpanit3bbf3562016-05-04 14:09:51 -05005089 if (svm_nested_virtualize_tpr(vcpu) ||
5090 kvm_vcpu_apicv_active(vcpu))
Joerg Roedel88ab24a2010-02-19 16:23:06 +01005091 return;
5092
Radim Krčmář596f3142014-03-11 19:11:18 +01005093 clr_cr_intercept(svm, INTERCEPT_CR8_WRITE);
5094
Gleb Natapov95ba8273132009-04-21 17:45:08 +03005095 if (irr == -1)
5096 return;
5097
5098 if (tpr >= irr)
Roedel, Joerg4ee546b2010-12-03 10:50:51 +01005099 set_cr_intercept(svm, INTERCEPT_CR8_WRITE);
Gleb Natapov95ba8273132009-04-21 17:45:08 +03005100}
5101
Jim Mattson8d860bb2018-05-09 16:56:05 -04005102static void svm_set_virtual_apic_mode(struct kvm_vcpu *vcpu)
Yang Zhang8d146952013-01-25 10:18:50 +08005103{
5104 return;
5105}
5106
Suravee Suthikulpanitb2a05fe2017-09-12 10:42:41 -05005107static bool svm_get_enable_apicv(struct kvm_vcpu *vcpu)
Yang Zhangc7c9c562013-01-25 10:18:51 +08005108{
Suravee Suthikulpanit67034bb2017-09-12 10:42:42 -05005109 return avic && irqchip_split(vcpu->kvm);
Yang Zhangc7c9c562013-01-25 10:18:51 +08005110}
5111
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -05005112static void svm_hwapic_irr_update(struct kvm_vcpu *vcpu, int max_irr)
5113{
5114}
5115
Paolo Bonzini67c9ddd2016-05-10 17:01:23 +02005116static void svm_hwapic_isr_update(struct kvm_vcpu *vcpu, int max_isr)
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -05005117{
5118}
5119
5120/* Note: Currently only used by Hyper-V. */
Andrey Smetanind62caab2015-11-10 15:36:33 +03005121static void svm_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu)
5122{
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -05005123 struct vcpu_svm *svm = to_svm(vcpu);
5124 struct vmcb *vmcb = svm->vmcb;
5125
Suthikulpanit, Suraveec57cd3c2019-01-29 08:09:46 +00005126 if (kvm_vcpu_apicv_active(vcpu))
5127 vmcb->control.int_ctl |= AVIC_ENABLE_MASK;
5128 else
5129 vmcb->control.int_ctl &= ~AVIC_ENABLE_MASK;
5130 mark_dirty(vmcb, VMCB_AVIC);
Yang Zhangc7c9c562013-01-25 10:18:51 +08005131}
5132
Andrey Smetanin63086302015-11-10 15:36:32 +03005133static void svm_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap)
Yang Zhangc7c9c562013-01-25 10:18:51 +08005134{
5135 return;
5136}
5137
Suravee Suthikulpanit340d3bc2016-05-04 14:09:47 -05005138static void svm_deliver_avic_intr(struct kvm_vcpu *vcpu, int vec)
5139{
5140 kvm_lapic_set_irr(vec, vcpu->arch.apic);
5141 smp_mb__after_atomic();
5142
Suthikulpanit, Suravee0532dd52019-05-03 06:38:53 -07005143 if (avic_vcpu_is_running(vcpu)) {
5144 int cpuid = vcpu->cpu;
5145
5146 if (cpuid != get_cpu())
5147 wrmsrl(SVM_AVIC_DOORBELL, kvm_cpu_get_apicid(cpuid));
5148 put_cpu();
5149 } else
Suravee Suthikulpanit340d3bc2016-05-04 14:09:47 -05005150 kvm_vcpu_wake_up(vcpu);
5151}
5152
Wanpeng Li17e433b2019-08-05 10:03:19 +08005153static bool svm_dy_apicv_has_pending_interrupt(struct kvm_vcpu *vcpu)
5154{
5155 return false;
5156}
5157
Suravee Suthikulpanit411b44b2016-08-23 13:52:43 -05005158static void svm_ir_list_del(struct vcpu_svm *svm, struct amd_iommu_pi_data *pi)
5159{
5160 unsigned long flags;
5161 struct amd_svm_iommu_ir *cur;
5162
5163 spin_lock_irqsave(&svm->ir_list_lock, flags);
5164 list_for_each_entry(cur, &svm->ir_list, node) {
5165 if (cur->data != pi->ir_data)
5166 continue;
5167 list_del(&cur->node);
5168 kfree(cur);
5169 break;
5170 }
5171 spin_unlock_irqrestore(&svm->ir_list_lock, flags);
5172}
5173
5174static int svm_ir_list_add(struct vcpu_svm *svm, struct amd_iommu_pi_data *pi)
5175{
5176 int ret = 0;
5177 unsigned long flags;
5178 struct amd_svm_iommu_ir *ir;
5179
5180 /**
5181 * In some cases, the existing irte is updaed and re-set,
5182 * so we need to check here if it's already been * added
5183 * to the ir_list.
5184 */
5185 if (pi->ir_data && (pi->prev_ga_tag != 0)) {
5186 struct kvm *kvm = svm->vcpu.kvm;
5187 u32 vcpu_id = AVIC_GATAG_TO_VCPUID(pi->prev_ga_tag);
5188 struct kvm_vcpu *prev_vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id);
5189 struct vcpu_svm *prev_svm;
5190
5191 if (!prev_vcpu) {
5192 ret = -EINVAL;
5193 goto out;
5194 }
5195
5196 prev_svm = to_svm(prev_vcpu);
5197 svm_ir_list_del(prev_svm, pi);
5198 }
5199
5200 /**
5201 * Allocating new amd_iommu_pi_data, which will get
5202 * add to the per-vcpu ir_list.
5203 */
Ben Gardon1ec69642019-02-11 11:02:51 -08005204 ir = kzalloc(sizeof(struct amd_svm_iommu_ir), GFP_KERNEL_ACCOUNT);
Suravee Suthikulpanit411b44b2016-08-23 13:52:43 -05005205 if (!ir) {
5206 ret = -ENOMEM;
5207 goto out;
5208 }
5209 ir->data = pi->ir_data;
5210
5211 spin_lock_irqsave(&svm->ir_list_lock, flags);
5212 list_add(&ir->node, &svm->ir_list);
5213 spin_unlock_irqrestore(&svm->ir_list_lock, flags);
5214out:
5215 return ret;
5216}
5217
5218/**
5219 * Note:
5220 * The HW cannot support posting multicast/broadcast
5221 * interrupts to a vCPU. So, we still use legacy interrupt
5222 * remapping for these kind of interrupts.
5223 *
5224 * For lowest-priority interrupts, we only support
5225 * those with single CPU as the destination, e.g. user
5226 * configures the interrupts via /proc/irq or uses
5227 * irqbalance to make the interrupts single-CPU.
5228 */
5229static int
5230get_pi_vcpu_info(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e,
5231 struct vcpu_data *vcpu_info, struct vcpu_svm **svm)
5232{
5233 struct kvm_lapic_irq irq;
5234 struct kvm_vcpu *vcpu = NULL;
5235
5236 kvm_set_msi_irq(kvm, e, &irq);
5237
Alexander Graffdcf7562019-09-05 14:58:18 +02005238 if (!kvm_intr_is_single_vcpu(kvm, &irq, &vcpu) ||
5239 !kvm_irq_is_postable(&irq)) {
Suravee Suthikulpanit411b44b2016-08-23 13:52:43 -05005240 pr_debug("SVM: %s: use legacy intr remap mode for irq %u\n",
5241 __func__, irq.vector);
5242 return -1;
5243 }
5244
5245 pr_debug("SVM: %s: use GA mode for irq %u\n", __func__,
5246 irq.vector);
5247 *svm = to_svm(vcpu);
Tom Lendackyd0ec49d2017-07-17 16:10:27 -05005248 vcpu_info->pi_desc_addr = __sme_set(page_to_phys((*svm)->avic_backing_page));
Suravee Suthikulpanit411b44b2016-08-23 13:52:43 -05005249 vcpu_info->vector = irq.vector;
5250
5251 return 0;
5252}
5253
5254/*
5255 * svm_update_pi_irte - set IRTE for Posted-Interrupts
5256 *
5257 * @kvm: kvm
5258 * @host_irq: host irq of the interrupt
5259 * @guest_irq: gsi of the interrupt
5260 * @set: set or unset PI
5261 * returns 0 on success, < 0 on failure
5262 */
5263static int svm_update_pi_irte(struct kvm *kvm, unsigned int host_irq,
5264 uint32_t guest_irq, bool set)
5265{
5266 struct kvm_kernel_irq_routing_entry *e;
5267 struct kvm_irq_routing_table *irq_rt;
5268 int idx, ret = -EINVAL;
5269
5270 if (!kvm_arch_has_assigned_device(kvm) ||
5271 !irq_remapping_cap(IRQ_POSTING_CAP))
5272 return 0;
5273
5274 pr_debug("SVM: %s: host_irq=%#x, guest_irq=%#x, set=%#x\n",
5275 __func__, host_irq, guest_irq, set);
5276
5277 idx = srcu_read_lock(&kvm->irq_srcu);
5278 irq_rt = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu);
5279 WARN_ON(guest_irq >= irq_rt->nr_rt_entries);
5280
5281 hlist_for_each_entry(e, &irq_rt->map[guest_irq], link) {
5282 struct vcpu_data vcpu_info;
5283 struct vcpu_svm *svm = NULL;
5284
5285 if (e->type != KVM_IRQ_ROUTING_MSI)
5286 continue;
5287
5288 /**
5289 * Here, we setup with legacy mode in the following cases:
5290 * 1. When cannot target interrupt to a specific vcpu.
5291 * 2. Unsetting posted interrupt.
5292 * 3. APIC virtialization is disabled for the vcpu.
Alexander Graffdcf7562019-09-05 14:58:18 +02005293 * 4. IRQ has incompatible delivery mode (SMI, INIT, etc)
Suravee Suthikulpanit411b44b2016-08-23 13:52:43 -05005294 */
5295 if (!get_pi_vcpu_info(kvm, e, &vcpu_info, &svm) && set &&
5296 kvm_vcpu_apicv_active(&svm->vcpu)) {
5297 struct amd_iommu_pi_data pi;
5298
5299 /* Try to enable guest_mode in IRTE */
Tom Lendackyd0ec49d2017-07-17 16:10:27 -05005300 pi.base = __sme_set(page_to_phys(svm->avic_backing_page) &
5301 AVIC_HPA_MASK);
Sean Christopherson81811c12018-03-20 12:17:21 -07005302 pi.ga_tag = AVIC_GATAG(to_kvm_svm(kvm)->avic_vm_id,
Suravee Suthikulpanit411b44b2016-08-23 13:52:43 -05005303 svm->vcpu.vcpu_id);
5304 pi.is_guest_mode = true;
5305 pi.vcpu_data = &vcpu_info;
5306 ret = irq_set_vcpu_affinity(host_irq, &pi);
5307
5308 /**
5309 * Here, we successfully setting up vcpu affinity in
5310 * IOMMU guest mode. Now, we need to store the posted
5311 * interrupt information in a per-vcpu ir_list so that
5312 * we can reference to them directly when we update vcpu
5313 * scheduling information in IOMMU irte.
5314 */
5315 if (!ret && pi.is_guest_mode)
5316 svm_ir_list_add(svm, &pi);
5317 } else {
5318 /* Use legacy mode in IRTE */
5319 struct amd_iommu_pi_data pi;
5320
5321 /**
5322 * Here, pi is used to:
5323 * - Tell IOMMU to use legacy mode for this interrupt.
5324 * - Retrieve ga_tag of prior interrupt remapping data.
5325 */
5326 pi.is_guest_mode = false;
5327 ret = irq_set_vcpu_affinity(host_irq, &pi);
5328
5329 /**
5330 * Check if the posted interrupt was previously
5331 * setup with the guest_mode by checking if the ga_tag
5332 * was cached. If so, we need to clean up the per-vcpu
5333 * ir_list.
5334 */
5335 if (!ret && pi.prev_ga_tag) {
5336 int id = AVIC_GATAG_TO_VCPUID(pi.prev_ga_tag);
5337 struct kvm_vcpu *vcpu;
5338
5339 vcpu = kvm_get_vcpu_by_id(kvm, id);
5340 if (vcpu)
5341 svm_ir_list_del(to_svm(vcpu), &pi);
5342 }
5343 }
5344
5345 if (!ret && svm) {
hu huajun2698d822018-04-11 15:16:40 +08005346 trace_kvm_pi_irte_update(host_irq, svm->vcpu.vcpu_id,
5347 e->gsi, vcpu_info.vector,
Suravee Suthikulpanit411b44b2016-08-23 13:52:43 -05005348 vcpu_info.pi_desc_addr, set);
5349 }
5350
5351 if (ret < 0) {
5352 pr_err("%s: failed to update PI IRTE\n", __func__);
5353 goto out;
5354 }
5355 }
5356
5357 ret = 0;
5358out:
5359 srcu_read_unlock(&kvm->irq_srcu, idx);
5360 return ret;
5361}
5362
Gleb Natapov95ba8273132009-04-21 17:45:08 +03005363static int svm_nmi_allowed(struct kvm_vcpu *vcpu)
Joerg Roedelaaacfc92008-04-16 16:51:18 +02005364{
5365 struct vcpu_svm *svm = to_svm(vcpu);
5366 struct vmcb *vmcb = svm->vmcb;
Joerg Roedel924584c2010-04-22 12:33:07 +02005367 int ret;
5368 ret = !(vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) &&
5369 !(svm->vcpu.arch.hflags & HF_NMI_MASK);
5370 ret = ret && gif_set(svm) && nested_svm_nmi(svm);
5371
5372 return ret;
Joerg Roedelaaacfc92008-04-16 16:51:18 +02005373}
5374
Jan Kiszka3cfc3092009-11-12 01:04:25 +01005375static bool svm_get_nmi_mask(struct kvm_vcpu *vcpu)
5376{
5377 struct vcpu_svm *svm = to_svm(vcpu);
5378
5379 return !!(svm->vcpu.arch.hflags & HF_NMI_MASK);
5380}
5381
5382static void svm_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
5383{
5384 struct vcpu_svm *svm = to_svm(vcpu);
5385
5386 if (masked) {
5387 svm->vcpu.arch.hflags |= HF_NMI_MASK;
Joerg Roedel8a05a1b82010-11-30 18:04:00 +01005388 set_intercept(svm, INTERCEPT_IRET);
Jan Kiszka3cfc3092009-11-12 01:04:25 +01005389 } else {
5390 svm->vcpu.arch.hflags &= ~HF_NMI_MASK;
Joerg Roedel8a05a1b82010-11-30 18:04:00 +01005391 clr_intercept(svm, INTERCEPT_IRET);
Jan Kiszka3cfc3092009-11-12 01:04:25 +01005392 }
5393}
5394
Gleb Natapov78646122009-03-23 12:12:11 +02005395static int svm_interrupt_allowed(struct kvm_vcpu *vcpu)
5396{
5397 struct vcpu_svm *svm = to_svm(vcpu);
5398 struct vmcb *vmcb = svm->vmcb;
Joerg Roedel7fcdb512009-09-16 15:24:15 +02005399 int ret;
5400
5401 if (!gif_set(svm) ||
5402 (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK))
5403 return 0;
5404
Avi Kivityf6e78472010-08-02 15:30:20 +03005405 ret = !!(kvm_get_rflags(vcpu) & X86_EFLAGS_IF);
Joerg Roedel7fcdb512009-09-16 15:24:15 +02005406
Joerg Roedel20307532010-11-29 17:51:48 +01005407 if (is_guest_mode(vcpu))
Joerg Roedel7fcdb512009-09-16 15:24:15 +02005408 return ret && !(svm->vcpu.arch.hflags & HF_VINTR_MASK);
5409
5410 return ret;
Gleb Natapov78646122009-03-23 12:12:11 +02005411}
5412
Jan Kiszkac9a79532014-03-07 20:03:15 +01005413static void enable_irq_window(struct kvm_vcpu *vcpu)
Gleb Natapov9222be12009-04-23 17:14:37 +03005414{
Alexander Graf219b65d2009-06-15 15:21:25 +02005415 struct vcpu_svm *svm = to_svm(vcpu);
Alexander Graf219b65d2009-06-15 15:21:25 +02005416
Suravee Suthikulpanit340d3bc2016-05-04 14:09:47 -05005417 if (kvm_vcpu_apicv_active(vcpu))
5418 return;
5419
Joerg Roedele0231712010-02-24 18:59:10 +01005420 /*
5421 * In case GIF=0 we can't rely on the CPU to tell us when GIF becomes
5422 * 1, because that's a separate STGI/VMRUN intercept. The next time we
5423 * get that intercept, this function will be called again though and
Janakarajan Natarajan640bd6e2017-08-23 09:57:19 -05005424 * we'll get the vintr intercept. However, if the vGIF feature is
5425 * enabled, the STGI interception will not occur. Enable the irq
5426 * window under the assumption that the hardware will set the GIF.
Joerg Roedele0231712010-02-24 18:59:10 +01005427 */
Janakarajan Natarajan640bd6e2017-08-23 09:57:19 -05005428 if ((vgif_enabled(svm) || gif_set(svm)) && nested_svm_intr(svm)) {
Alexander Graf219b65d2009-06-15 15:21:25 +02005429 svm_set_vintr(svm);
5430 svm_inject_irq(svm, 0x0);
5431 }
Gleb Natapov9222be12009-04-23 17:14:37 +03005432}
5433
Jan Kiszkac9a79532014-03-07 20:03:15 +01005434static void enable_nmi_window(struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -08005435{
Avi Kivity04d2cc72007-09-10 18:10:54 +03005436 struct vcpu_svm *svm = to_svm(vcpu);
Eddie Dong85f455f2007-07-06 12:20:49 +03005437
Gleb Natapov44c11432009-05-11 13:35:52 +03005438 if ((svm->vcpu.arch.hflags & (HF_NMI_MASK | HF_IRET_MASK))
5439 == HF_NMI_MASK)
Jan Kiszkac9a79532014-03-07 20:03:15 +01005440 return; /* IRET will cause a vm exit */
Gleb Natapov44c11432009-05-11 13:35:52 +03005441
Janakarajan Natarajan640bd6e2017-08-23 09:57:19 -05005442 if (!gif_set(svm)) {
5443 if (vgif_enabled(svm))
5444 set_intercept(svm, INTERCEPT_STGI);
Ladi Prosek1a5e1852017-06-21 09:07:01 +02005445 return; /* STGI will cause a vm exit */
Janakarajan Natarajan640bd6e2017-08-23 09:57:19 -05005446 }
Ladi Prosek1a5e1852017-06-21 09:07:01 +02005447
5448 if (svm->nested.exit_required)
5449 return; /* we're not going to run the guest yet */
5450
Joerg Roedele0231712010-02-24 18:59:10 +01005451 /*
5452 * Something prevents NMI from been injected. Single step over possible
5453 * problem (IRET or exception injection or interrupt shadow)
5454 */
Ladi Prosekab2f4d732017-06-21 09:06:58 +02005455 svm->nmi_singlestep_guest_rflags = svm_get_rflags(vcpu);
Jan Kiszka6be7d302009-10-18 13:24:54 +02005456 svm->nmi_singlestep = true;
Gleb Natapov44c11432009-05-11 13:35:52 +03005457 svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
Eddie Dong85f455f2007-07-06 12:20:49 +03005458}
5459
Izik Eiduscbc94022007-10-25 00:29:55 +02005460static int svm_set_tss_addr(struct kvm *kvm, unsigned int addr)
5461{
5462 return 0;
5463}
5464
Sean Christopherson2ac52ab2018-03-20 12:17:19 -07005465static int svm_set_identity_map_addr(struct kvm *kvm, u64 ident_addr)
5466{
5467 return 0;
5468}
5469
Wanpeng Lic2ba05c2017-12-12 17:33:03 -08005470static void svm_flush_tlb(struct kvm_vcpu *vcpu, bool invalidate_gpa)
Avi Kivityd9e368d2007-06-07 19:18:30 +03005471{
Joerg Roedel38e5e922010-12-03 15:25:16 +01005472 struct vcpu_svm *svm = to_svm(vcpu);
5473
5474 if (static_cpu_has(X86_FEATURE_FLUSHBYASID))
5475 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID;
5476 else
5477 svm->asid_generation--;
Avi Kivityd9e368d2007-06-07 19:18:30 +03005478}
5479
Junaid Shahidfaff8752018-06-29 13:10:05 -07005480static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva)
5481{
5482 struct vcpu_svm *svm = to_svm(vcpu);
5483
5484 invlpga(gva, svm->vmcb->control.asid);
5485}
5486
Avi Kivity04d2cc72007-09-10 18:10:54 +03005487static void svm_prepare_guest_switch(struct kvm_vcpu *vcpu)
5488{
5489}
5490
Joerg Roedeld7bf8222008-04-16 16:51:17 +02005491static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu)
5492{
5493 struct vcpu_svm *svm = to_svm(vcpu);
5494
Suravee Suthikulpanit3bbf3562016-05-04 14:09:51 -05005495 if (svm_nested_virtualize_tpr(vcpu))
Joerg Roedel88ab24a2010-02-19 16:23:06 +01005496 return;
5497
Roedel, Joerg4ee546b2010-12-03 10:50:51 +01005498 if (!is_cr_intercept(svm, INTERCEPT_CR8_WRITE)) {
Joerg Roedeld7bf8222008-04-16 16:51:17 +02005499 int cr8 = svm->vmcb->control.int_ctl & V_TPR_MASK;
Gleb Natapov615d5192009-04-21 17:45:05 +03005500 kvm_set_cr8(vcpu, cr8);
Joerg Roedeld7bf8222008-04-16 16:51:17 +02005501 }
5502}
5503
Joerg Roedel649d6862008-04-16 16:51:15 +02005504static inline void sync_lapic_to_cr8(struct kvm_vcpu *vcpu)
5505{
5506 struct vcpu_svm *svm = to_svm(vcpu);
5507 u64 cr8;
5508
Suravee Suthikulpanit3bbf3562016-05-04 14:09:51 -05005509 if (svm_nested_virtualize_tpr(vcpu) ||
5510 kvm_vcpu_apicv_active(vcpu))
Joerg Roedel88ab24a2010-02-19 16:23:06 +01005511 return;
5512
Joerg Roedel649d6862008-04-16 16:51:15 +02005513 cr8 = kvm_get_cr8(vcpu);
5514 svm->vmcb->control.int_ctl &= ~V_TPR_MASK;
5515 svm->vmcb->control.int_ctl |= cr8 & V_TPR_MASK;
5516}
5517
Gleb Natapov9222be12009-04-23 17:14:37 +03005518static void svm_complete_interrupts(struct vcpu_svm *svm)
5519{
5520 u8 vector;
5521 int type;
5522 u32 exitintinfo = svm->vmcb->control.exit_int_info;
Jan Kiszka66b71382010-02-23 17:47:56 +01005523 unsigned int3_injected = svm->int3_injected;
5524
5525 svm->int3_injected = 0;
Gleb Natapov9222be12009-04-23 17:14:37 +03005526
Avi Kivitybd3d1ec2011-02-03 15:29:52 +02005527 /*
5528 * If we've made progress since setting HF_IRET_MASK, we've
5529 * executed an IRET and can allow NMI injection.
5530 */
5531 if ((svm->vcpu.arch.hflags & HF_IRET_MASK)
5532 && kvm_rip_read(&svm->vcpu) != svm->nmi_iret_rip) {
Gleb Natapov44c11432009-05-11 13:35:52 +03005533 svm->vcpu.arch.hflags &= ~(HF_NMI_MASK | HF_IRET_MASK);
Avi Kivity3842d132010-07-27 12:30:24 +03005534 kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
5535 }
Gleb Natapov44c11432009-05-11 13:35:52 +03005536
Gleb Natapov9222be12009-04-23 17:14:37 +03005537 svm->vcpu.arch.nmi_injected = false;
5538 kvm_clear_exception_queue(&svm->vcpu);
5539 kvm_clear_interrupt_queue(&svm->vcpu);
5540
5541 if (!(exitintinfo & SVM_EXITINTINFO_VALID))
5542 return;
5543
Avi Kivity3842d132010-07-27 12:30:24 +03005544 kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
5545
Gleb Natapov9222be12009-04-23 17:14:37 +03005546 vector = exitintinfo & SVM_EXITINTINFO_VEC_MASK;
5547 type = exitintinfo & SVM_EXITINTINFO_TYPE_MASK;
5548
5549 switch (type) {
5550 case SVM_EXITINTINFO_TYPE_NMI:
5551 svm->vcpu.arch.nmi_injected = true;
5552 break;
5553 case SVM_EXITINTINFO_TYPE_EXEPT:
Jan Kiszka66b71382010-02-23 17:47:56 +01005554 /*
5555 * In case of software exceptions, do not reinject the vector,
5556 * but re-execute the instruction instead. Rewind RIP first
5557 * if we emulated INT3 before.
5558 */
5559 if (kvm_exception_is_soft(vector)) {
5560 if (vector == BP_VECTOR && int3_injected &&
5561 kvm_is_linear_rip(&svm->vcpu, svm->int3_rip))
5562 kvm_rip_write(&svm->vcpu,
5563 kvm_rip_read(&svm->vcpu) -
5564 int3_injected);
Alexander Graf219b65d2009-06-15 15:21:25 +02005565 break;
Jan Kiszka66b71382010-02-23 17:47:56 +01005566 }
Gleb Natapov9222be12009-04-23 17:14:37 +03005567 if (exitintinfo & SVM_EXITINTINFO_VALID_ERR) {
5568 u32 err = svm->vmcb->control.exit_int_info_err;
Joerg Roedelce7ddec2010-04-22 12:33:13 +02005569 kvm_requeue_exception_e(&svm->vcpu, vector, err);
Gleb Natapov9222be12009-04-23 17:14:37 +03005570
5571 } else
Joerg Roedelce7ddec2010-04-22 12:33:13 +02005572 kvm_requeue_exception(&svm->vcpu, vector);
Gleb Natapov9222be12009-04-23 17:14:37 +03005573 break;
5574 case SVM_EXITINTINFO_TYPE_INTR:
Gleb Natapov66fd3f72009-05-11 13:35:50 +03005575 kvm_queue_interrupt(&svm->vcpu, vector, false);
Gleb Natapov9222be12009-04-23 17:14:37 +03005576 break;
5577 default:
5578 break;
5579 }
5580}
5581
Avi Kivityb463a6f2010-07-20 15:06:17 +03005582static void svm_cancel_injection(struct kvm_vcpu *vcpu)
5583{
5584 struct vcpu_svm *svm = to_svm(vcpu);
5585 struct vmcb_control_area *control = &svm->vmcb->control;
5586
5587 control->exit_int_info = control->event_inj;
5588 control->exit_int_info_err = control->event_inj_err;
5589 control->event_inj = 0;
5590 svm_complete_interrupts(svm);
5591}
5592
Avi Kivity851ba692009-08-24 11:10:17 +03005593static void svm_vcpu_run(struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -08005594{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04005595 struct vcpu_svm *svm = to_svm(vcpu);
Avi Kivityd9e368d2007-06-07 19:18:30 +03005596
Joerg Roedel2041a062010-04-22 12:33:08 +02005597 svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
5598 svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
5599 svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];
5600
Joerg Roedelcd3ff652009-10-09 16:08:26 +02005601 /*
5602 * A vmexit emulation is required before the vcpu can be executed
5603 * again.
5604 */
5605 if (unlikely(svm->nested.exit_required))
5606 return;
5607
Ladi Proseka12713c2017-06-21 09:07:00 +02005608 /*
5609 * Disable singlestep if we're injecting an interrupt/exception.
5610 * We don't want our modified rflags to be pushed on the stack where
5611 * we might not be able to easily reset them if we disabled NMI
5612 * singlestep later.
5613 */
5614 if (svm->nmi_singlestep && svm->vmcb->control.event_inj) {
5615 /*
5616 * Event injection happens before external interrupts cause a
5617 * vmexit and interrupts are disabled here, so smp_send_reschedule
5618 * is enough to force an immediate vmexit.
5619 */
5620 disable_nmi_singlestep(svm);
5621 smp_send_reschedule(vcpu->cpu);
5622 }
5623
Rusty Russelle756fc62007-07-30 20:07:08 +10005624 pre_svm_run(svm);
Avi Kivity6aa8b732006-12-10 02:21:36 -08005625
Joerg Roedel649d6862008-04-16 16:51:15 +02005626 sync_lapic_to_cr8(vcpu);
5627
Joerg Roedelcda0ffd2009-08-07 11:49:45 +02005628 svm->vmcb->save.cr2 = vcpu->arch.cr2;
Avi Kivity6aa8b732006-12-10 02:21:36 -08005629
Avi Kivity04d2cc72007-09-10 18:10:54 +03005630 clgi();
WANG Chao1811d972019-04-12 15:55:39 +08005631 kvm_load_guest_xcr0(vcpu);
Avi Kivity04d2cc72007-09-10 18:10:54 +03005632
Wanpeng Lib6c4bc62019-05-20 16:18:09 +08005633 if (lapic_in_kernel(vcpu) &&
5634 vcpu->arch.apic->lapic_timer.timer_advance_ns)
5635 kvm_wait_lapic_expire(vcpu);
5636
KarimAllah Ahmedb2ac58f2018-02-03 15:56:23 +01005637 /*
5638 * If this vCPU has touched SPEC_CTRL, restore the guest's value if
5639 * it's non-zero. Since vmentry is serialising on affected CPUs, there
5640 * is no need to worry about the conditional branch over the wrmsr
5641 * being speculatively taken.
5642 */
Thomas Gleixnerccbcd262018-05-09 23:01:01 +02005643 x86_spec_ctrl_set_guest(svm->spec_ctrl, svm->virt_spec_ctrl);
KarimAllah Ahmedb2ac58f2018-02-03 15:56:23 +01005644
Thomas Gleixner024d83c2018-08-12 20:41:45 +02005645 local_irq_enable();
5646
Avi Kivity6aa8b732006-12-10 02:21:36 -08005647 asm volatile (
Avi Kivity74547662012-09-16 15:10:59 +03005648 "push %%" _ASM_BP "; \n\t"
5649 "mov %c[rbx](%[svm]), %%" _ASM_BX " \n\t"
5650 "mov %c[rcx](%[svm]), %%" _ASM_CX " \n\t"
5651 "mov %c[rdx](%[svm]), %%" _ASM_DX " \n\t"
5652 "mov %c[rsi](%[svm]), %%" _ASM_SI " \n\t"
5653 "mov %c[rdi](%[svm]), %%" _ASM_DI " \n\t"
5654 "mov %c[rbp](%[svm]), %%" _ASM_BP " \n\t"
Avi Kivity05b3e0c2006-12-13 00:33:45 -08005655#ifdef CONFIG_X86_64
Rusty Russellfb3f0f52007-07-27 17:16:56 +10005656 "mov %c[r8](%[svm]), %%r8 \n\t"
5657 "mov %c[r9](%[svm]), %%r9 \n\t"
5658 "mov %c[r10](%[svm]), %%r10 \n\t"
5659 "mov %c[r11](%[svm]), %%r11 \n\t"
5660 "mov %c[r12](%[svm]), %%r12 \n\t"
5661 "mov %c[r13](%[svm]), %%r13 \n\t"
5662 "mov %c[r14](%[svm]), %%r14 \n\t"
5663 "mov %c[r15](%[svm]), %%r15 \n\t"
Avi Kivity6aa8b732006-12-10 02:21:36 -08005664#endif
5665
Avi Kivity6aa8b732006-12-10 02:21:36 -08005666 /* Enter guest mode */
Avi Kivity74547662012-09-16 15:10:59 +03005667 "push %%" _ASM_AX " \n\t"
5668 "mov %c[vmcb](%[svm]), %%" _ASM_AX " \n\t"
Uros Bizjakac5ffda22018-11-26 17:00:08 +01005669 __ex("vmload %%" _ASM_AX) "\n\t"
5670 __ex("vmrun %%" _ASM_AX) "\n\t"
5671 __ex("vmsave %%" _ASM_AX) "\n\t"
Avi Kivity74547662012-09-16 15:10:59 +03005672 "pop %%" _ASM_AX " \n\t"
Avi Kivity6aa8b732006-12-10 02:21:36 -08005673
5674 /* Save guest registers, load host registers */
Avi Kivity74547662012-09-16 15:10:59 +03005675 "mov %%" _ASM_BX ", %c[rbx](%[svm]) \n\t"
5676 "mov %%" _ASM_CX ", %c[rcx](%[svm]) \n\t"
5677 "mov %%" _ASM_DX ", %c[rdx](%[svm]) \n\t"
5678 "mov %%" _ASM_SI ", %c[rsi](%[svm]) \n\t"
5679 "mov %%" _ASM_DI ", %c[rdi](%[svm]) \n\t"
5680 "mov %%" _ASM_BP ", %c[rbp](%[svm]) \n\t"
Avi Kivity05b3e0c2006-12-13 00:33:45 -08005681#ifdef CONFIG_X86_64
Rusty Russellfb3f0f52007-07-27 17:16:56 +10005682 "mov %%r8, %c[r8](%[svm]) \n\t"
5683 "mov %%r9, %c[r9](%[svm]) \n\t"
5684 "mov %%r10, %c[r10](%[svm]) \n\t"
5685 "mov %%r11, %c[r11](%[svm]) \n\t"
5686 "mov %%r12, %c[r12](%[svm]) \n\t"
5687 "mov %%r13, %c[r13](%[svm]) \n\t"
5688 "mov %%r14, %c[r14](%[svm]) \n\t"
5689 "mov %%r15, %c[r15](%[svm]) \n\t"
Jim Mattson0cb5b302018-01-03 14:31:38 -08005690 /*
5691 * Clear host registers marked as clobbered to prevent
5692 * speculative use.
5693 */
Uros Bizjak43ce76c2018-10-17 16:46:57 +02005694 "xor %%r8d, %%r8d \n\t"
5695 "xor %%r9d, %%r9d \n\t"
5696 "xor %%r10d, %%r10d \n\t"
5697 "xor %%r11d, %%r11d \n\t"
5698 "xor %%r12d, %%r12d \n\t"
5699 "xor %%r13d, %%r13d \n\t"
5700 "xor %%r14d, %%r14d \n\t"
5701 "xor %%r15d, %%r15d \n\t"
Jim Mattson0cb5b302018-01-03 14:31:38 -08005702#endif
Uros Bizjak43ce76c2018-10-17 16:46:57 +02005703 "xor %%ebx, %%ebx \n\t"
5704 "xor %%ecx, %%ecx \n\t"
5705 "xor %%edx, %%edx \n\t"
5706 "xor %%esi, %%esi \n\t"
5707 "xor %%edi, %%edi \n\t"
Avi Kivity74547662012-09-16 15:10:59 +03005708 "pop %%" _ASM_BP
Avi Kivity6aa8b732006-12-10 02:21:36 -08005709 :
Rusty Russellfb3f0f52007-07-27 17:16:56 +10005710 : [svm]"a"(svm),
Avi Kivity6aa8b732006-12-10 02:21:36 -08005711 [vmcb]"i"(offsetof(struct vcpu_svm, vmcb_pa)),
Zhang Xiantaoad312c72007-12-13 23:50:52 +08005712 [rbx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBX])),
5713 [rcx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RCX])),
5714 [rdx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDX])),
5715 [rsi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RSI])),
5716 [rdi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDI])),
5717 [rbp]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBP]))
Avi Kivity05b3e0c2006-12-13 00:33:45 -08005718#ifdef CONFIG_X86_64
Zhang Xiantaoad312c72007-12-13 23:50:52 +08005719 , [r8]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R8])),
5720 [r9]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R9])),
5721 [r10]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R10])),
5722 [r11]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R11])),
5723 [r12]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R12])),
5724 [r13]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R13])),
5725 [r14]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R14])),
5726 [r15]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R15]))
Avi Kivity6aa8b732006-12-10 02:21:36 -08005727#endif
Laurent Vivier54a08c02007-10-25 14:18:53 +02005728 : "cc", "memory"
5729#ifdef CONFIG_X86_64
Avi Kivity74547662012-09-16 15:10:59 +03005730 , "rbx", "rcx", "rdx", "rsi", "rdi"
Laurent Vivier54a08c02007-10-25 14:18:53 +02005731 , "r8", "r9", "r10", "r11" , "r12", "r13", "r14", "r15"
Avi Kivity74547662012-09-16 15:10:59 +03005732#else
5733 , "ebx", "ecx", "edx", "esi", "edi"
Laurent Vivier54a08c02007-10-25 14:18:53 +02005734#endif
5735 );
Avi Kivity6aa8b732006-12-10 02:21:36 -08005736
Thomas Gleixner15e6c222018-05-11 15:21:01 +02005737 /* Eliminate branch target predictions from guest mode */
5738 vmexit_fill_RSB();
5739
5740#ifdef CONFIG_X86_64
5741 wrmsrl(MSR_GS_BASE, svm->host.gs_base);
5742#else
5743 loadsegment(fs, svm->host.fs);
5744#ifndef CONFIG_X86_32_LAZY_GS
5745 loadsegment(gs, svm->host.gs);
5746#endif
5747#endif
5748
KarimAllah Ahmedb2ac58f2018-02-03 15:56:23 +01005749 /*
5750 * We do not use IBRS in the kernel. If this vCPU has used the
5751 * SPEC_CTRL MSR it may have left it on; save the value and
5752 * turn it off. This is much more efficient than blindly adding
5753 * it to the atomic save/restore list. Especially as the former
5754 * (Saving guest MSRs on vmexit) doesn't even exist in KVM.
5755 *
5756 * For non-nested case:
5757 * If the L01 MSR bitmap does not intercept the MSR, then we need to
5758 * save it.
5759 *
5760 * For nested case:
5761 * If the L02 MSR bitmap does not intercept the MSR, then we need to
5762 * save it.
5763 */
Paolo Bonzini946fbbc2018-02-22 16:43:18 +01005764 if (unlikely(!msr_write_intercepted(vcpu, MSR_IA32_SPEC_CTRL)))
Paolo Bonziniecb586b2018-02-22 16:43:17 +01005765 svm->spec_ctrl = native_read_msr(MSR_IA32_SPEC_CTRL);
KarimAllah Ahmedb2ac58f2018-02-03 15:56:23 +01005766
Avi Kivity6aa8b732006-12-10 02:21:36 -08005767 reload_tss(vcpu);
5768
Avi Kivity56ba47d2007-11-07 17:14:18 +02005769 local_irq_disable();
5770
Thomas Gleixner024d83c2018-08-12 20:41:45 +02005771 x86_spec_ctrl_restore_host(svm->spec_ctrl, svm->virt_spec_ctrl);
5772
Avi Kivity13c34e02010-10-21 12:20:31 +02005773 vcpu->arch.cr2 = svm->vmcb->save.cr2;
5774 vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax;
5775 vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp;
5776 vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip;
5777
Joerg Roedel3781c012011-01-14 16:45:02 +01005778 if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
Andi Kleendd60d212017-07-25 17:20:32 -07005779 kvm_before_interrupt(&svm->vcpu);
Joerg Roedel3781c012011-01-14 16:45:02 +01005780
WANG Chao1811d972019-04-12 15:55:39 +08005781 kvm_put_guest_xcr0(vcpu);
Joerg Roedel3781c012011-01-14 16:45:02 +01005782 stgi();
5783
5784 /* Any pending NMI will happen here */
5785
5786 if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
Andi Kleendd60d212017-07-25 17:20:32 -07005787 kvm_after_interrupt(&svm->vcpu);
Joerg Roedel3781c012011-01-14 16:45:02 +01005788
Joerg Roedeld7bf8222008-04-16 16:51:17 +02005789 sync_cr8_to_lapic(vcpu);
5790
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04005791 svm->next_rip = 0;
Gleb Natapov9222be12009-04-23 17:14:37 +03005792
Joerg Roedel38e5e922010-12-03 15:25:16 +01005793 svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING;
5794
Gleb Natapov631bc482010-10-14 11:22:52 +02005795 /* if exit due to PF check for async PF */
5796 if (svm->vmcb->control.exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR)
Wanpeng Li1261bfa2017-07-13 18:30:40 -07005797 svm->vcpu.arch.apf.host_apf_reason = kvm_read_and_reset_pf_reason();
Gleb Natapov631bc482010-10-14 11:22:52 +02005798
Avi Kivity6de4f3a2009-05-31 22:58:47 +03005799 if (npt_enabled) {
5800 vcpu->arch.regs_avail &= ~(1 << VCPU_EXREG_PDPTR);
5801 vcpu->arch.regs_dirty &= ~(1 << VCPU_EXREG_PDPTR);
5802 }
Joerg Roedelfe5913e2010-05-17 14:43:34 +02005803
5804 /*
5805 * We need to handle MC intercepts here before the vcpu has a chance to
5806 * change the physical cpu
5807 */
5808 if (unlikely(svm->vmcb->control.exit_code ==
5809 SVM_EXIT_EXCP_BASE + MC_VECTOR))
5810 svm_handle_mce(svm);
Roedel, Joerg8d28fec2010-12-03 13:15:21 +01005811
5812 mark_all_clean(svm->vmcb);
Avi Kivity6aa8b732006-12-10 02:21:36 -08005813}
Josh Poimboeufc207aee2017-06-28 10:11:06 -05005814STACK_FRAME_NON_STANDARD(svm_vcpu_run);
Avi Kivity6aa8b732006-12-10 02:21:36 -08005815
Avi Kivity6aa8b732006-12-10 02:21:36 -08005816static void svm_set_cr3(struct kvm_vcpu *vcpu, unsigned long root)
5817{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04005818 struct vcpu_svm *svm = to_svm(vcpu);
5819
Tom Lendackyd0ec49d2017-07-17 16:10:27 -05005820 svm->vmcb->save.cr3 = __sme_set(root);
Joerg Roedeldcca1a62010-12-03 11:45:54 +01005821 mark_dirty(svm->vmcb, VMCB_CR);
Avi Kivity6aa8b732006-12-10 02:21:36 -08005822}
5823
Joerg Roedel1c97f0a2010-09-10 17:30:41 +02005824static void set_tdp_cr3(struct kvm_vcpu *vcpu, unsigned long root)
5825{
5826 struct vcpu_svm *svm = to_svm(vcpu);
5827
Tom Lendackyd0ec49d2017-07-17 16:10:27 -05005828 svm->vmcb->control.nested_cr3 = __sme_set(root);
Joerg Roedelb2747162010-12-03 11:45:53 +01005829 mark_dirty(svm->vmcb, VMCB_NPT);
Joerg Roedel1c97f0a2010-09-10 17:30:41 +02005830
5831 /* Also sync guest cr3 here in case we live migrate */
Avi Kivity9f8fe502010-12-05 17:30:00 +02005832 svm->vmcb->save.cr3 = kvm_read_cr3(vcpu);
Joerg Roedeldcca1a62010-12-03 11:45:54 +01005833 mark_dirty(svm->vmcb, VMCB_CR);
Joerg Roedel1c97f0a2010-09-10 17:30:41 +02005834}
5835
Avi Kivity6aa8b732006-12-10 02:21:36 -08005836static int is_disabled(void)
5837{
Joerg Roedel6031a612007-06-22 12:29:50 +03005838 u64 vm_cr;
5839
5840 rdmsrl(MSR_VM_CR, vm_cr);
5841 if (vm_cr & (1 << SVM_VM_CR_SVM_DISABLE))
5842 return 1;
5843
Avi Kivity6aa8b732006-12-10 02:21:36 -08005844 return 0;
5845}
5846
Ingo Molnar102d8322007-02-19 14:37:47 +02005847static void
5848svm_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
5849{
5850 /*
5851 * Patch in the VMMCALL instruction:
5852 */
5853 hypercall[0] = 0x0f;
5854 hypercall[1] = 0x01;
5855 hypercall[2] = 0xd9;
Ingo Molnar102d8322007-02-19 14:37:47 +02005856}
5857
Sean Christophersonf257d6d2019-04-19 22:18:17 -07005858static int __init svm_check_processor_compat(void)
Yang, Sheng002c7f72007-07-31 14:23:01 +03005859{
Sean Christophersonf257d6d2019-04-19 22:18:17 -07005860 return 0;
Yang, Sheng002c7f72007-07-31 14:23:01 +03005861}
5862
Avi Kivity774ead32007-12-26 13:57:04 +02005863static bool svm_cpu_has_accelerated_tpr(void)
5864{
5865 return false;
5866}
5867
Tom Lendackybc226f02018-05-10 22:06:39 +02005868static bool svm_has_emulated_msr(int index)
Paolo Bonzini6d396b52015-04-01 14:25:33 +02005869{
Vitaly Kuznetsove87555e2018-12-19 12:06:13 +01005870 switch (index) {
5871 case MSR_IA32_MCG_EXT_CTL:
Paolo Bonzini95c5c7c2019-07-02 14:45:24 +02005872 case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC:
Vitaly Kuznetsove87555e2018-12-19 12:06:13 +01005873 return false;
5874 default:
5875 break;
5876 }
5877
Paolo Bonzini6d396b52015-04-01 14:25:33 +02005878 return true;
5879}
5880
Paolo Bonzinifc07e762015-10-01 13:20:22 +02005881static u64 svm_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
5882{
5883 return 0;
5884}
5885
Sheng Yang0e851882009-12-18 16:48:46 +08005886static void svm_cpuid_update(struct kvm_vcpu *vcpu)
5887{
Joerg Roedel6092d3d2015-10-14 15:10:54 +02005888 struct vcpu_svm *svm = to_svm(vcpu);
5889
5890 /* Update nrips enabled cache */
Radim Krčmářd6321d42017-08-05 00:12:49 +02005891 svm->nrips_enabled = !!guest_cpuid_has(&svm->vcpu, X86_FEATURE_NRIPS);
Suravee Suthikulpanit46781ea2016-05-04 14:09:50 -05005892
5893 if (!kvm_vcpu_apicv_active(vcpu))
5894 return;
5895
Radim Krčmář1b4d56b2017-08-05 00:12:50 +02005896 guest_cpuid_clear(vcpu, X86_FEATURE_X2APIC);
Sheng Yang0e851882009-12-18 16:48:46 +08005897}
5898
Paolo Bonzini50896de2019-08-15 09:17:31 +02005899#define F(x) bit(X86_FEATURE_##x)
5900
Joerg Roedeld4330ef2010-04-22 12:33:11 +02005901static void svm_set_supported_cpuid(u32 func, struct kvm_cpuid_entry2 *entry)
5902{
Joerg Roedelc2c63a42010-04-22 12:33:12 +02005903 switch (func) {
Suravee Suthikulpanit46781ea2016-05-04 14:09:50 -05005904 case 0x1:
5905 if (avic)
5906 entry->ecx &= ~bit(X86_FEATURE_X2APIC);
5907 break;
Joerg Roedel4c62a2d2010-09-10 17:31:06 +02005908 case 0x80000001:
5909 if (nested)
5910 entry->ecx |= (1 << 2); /* Set SVM bit */
5911 break;
Paolo Bonzini50896de2019-08-15 09:17:31 +02005912 case 0x80000008:
5913 if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD) ||
5914 boot_cpu_has(X86_FEATURE_AMD_SSBD))
5915 entry->ebx |= F(VIRT_SSBD);
5916 break;
Joerg Roedelc2c63a42010-04-22 12:33:12 +02005917 case 0x8000000A:
5918 entry->eax = 1; /* SVM revision 1 */
5919 entry->ebx = 8; /* Lets support 8 ASIDs in case we add proper
5920 ASID emulation to nested SVM */
5921 entry->ecx = 0; /* Reserved */
Joerg Roedel7a190662010-07-27 18:14:21 +02005922 entry->edx = 0; /* Per default do not support any
5923 additional features */
5924
5925 /* Support next_rip if host supports it */
Avi Kivity2a6b20b2010-11-09 16:15:42 +02005926 if (boot_cpu_has(X86_FEATURE_NRIPS))
Paolo Bonzini50896de2019-08-15 09:17:31 +02005927 entry->edx |= F(NRIPS);
Joerg Roedelc2c63a42010-04-22 12:33:12 +02005928
Joerg Roedel3d4aeaa2010-09-10 17:31:05 +02005929 /* Support NPT for the guest if enabled */
5930 if (npt_enabled)
Paolo Bonzini50896de2019-08-15 09:17:31 +02005931 entry->edx |= F(NPT);
Joerg Roedel3d4aeaa2010-09-10 17:31:05 +02005932
Joerg Roedelc2c63a42010-04-22 12:33:12 +02005933 break;
Brijesh Singh8765d752017-12-04 10:57:25 -06005934 case 0x8000001F:
5935 /* Support memory encryption cpuid if host supports it */
5936 if (boot_cpu_has(X86_FEATURE_SEV))
5937 cpuid(0x8000001f, &entry->eax, &entry->ebx,
5938 &entry->ecx, &entry->edx);
5939
Joerg Roedelc2c63a42010-04-22 12:33:12 +02005940 }
Joerg Roedeld4330ef2010-04-22 12:33:11 +02005941}
5942
Sheng Yang17cc3932010-01-05 19:02:27 +08005943static int svm_get_lpage_level(void)
Joerg Roedel344f4142009-07-27 16:30:48 +02005944{
Sheng Yang17cc3932010-01-05 19:02:27 +08005945 return PT_PDPE_LEVEL;
Joerg Roedel344f4142009-07-27 16:30:48 +02005946}
5947
Sheng Yang4e47c7a2009-12-18 16:48:47 +08005948static bool svm_rdtscp_supported(void)
5949{
Paolo Bonzini46896c72015-11-12 14:49:16 +01005950 return boot_cpu_has(X86_FEATURE_RDTSCP);
Sheng Yang4e47c7a2009-12-18 16:48:47 +08005951}
5952
Mao, Junjiead756a12012-07-02 01:18:48 +00005953static bool svm_invpcid_supported(void)
5954{
5955 return false;
5956}
5957
Paolo Bonzini93c4adc2014-03-05 23:19:52 +01005958static bool svm_mpx_supported(void)
5959{
5960 return false;
5961}
5962
Wanpeng Li55412b22014-12-02 19:21:30 +08005963static bool svm_xsaves_supported(void)
5964{
5965 return false;
5966}
5967
Paolo Bonzini66336ca2016-07-12 10:36:41 +02005968static bool svm_umip_emulated(void)
5969{
5970 return false;
5971}
5972
Chao Peng86f52012018-10-24 16:05:11 +08005973static bool svm_pt_supported(void)
5974{
5975 return false;
5976}
5977
Sheng Yangf5f48ee2010-06-30 12:25:15 +08005978static bool svm_has_wbinvd_exit(void)
5979{
5980 return true;
5981}
5982
Joerg Roedel80612522011-04-04 12:39:33 +02005983#define PRE_EX(exit) { .exit_code = (exit), \
Avi Kivity40e19b52011-04-21 12:35:41 +03005984 .stage = X86_ICPT_PRE_EXCEPT, }
Joerg Roedelcfec82c2011-04-04 12:39:28 +02005985#define POST_EX(exit) { .exit_code = (exit), \
Avi Kivity40e19b52011-04-21 12:35:41 +03005986 .stage = X86_ICPT_POST_EXCEPT, }
Joerg Roedeld7eb8202011-04-04 12:39:32 +02005987#define POST_MEM(exit) { .exit_code = (exit), \
Avi Kivity40e19b52011-04-21 12:35:41 +03005988 .stage = X86_ICPT_POST_MEMACCESS, }
Joerg Roedelcfec82c2011-04-04 12:39:28 +02005989
Mathias Krause09941fb2012-08-30 01:30:20 +02005990static const struct __x86_intercept {
Joerg Roedelcfec82c2011-04-04 12:39:28 +02005991 u32 exit_code;
5992 enum x86_intercept_stage stage;
Joerg Roedelcfec82c2011-04-04 12:39:28 +02005993} x86_intercept_map[] = {
5994 [x86_intercept_cr_read] = POST_EX(SVM_EXIT_READ_CR0),
5995 [x86_intercept_cr_write] = POST_EX(SVM_EXIT_WRITE_CR0),
5996 [x86_intercept_clts] = POST_EX(SVM_EXIT_WRITE_CR0),
5997 [x86_intercept_lmsw] = POST_EX(SVM_EXIT_WRITE_CR0),
5998 [x86_intercept_smsw] = POST_EX(SVM_EXIT_READ_CR0),
Joerg Roedel3b88e412011-04-04 12:39:29 +02005999 [x86_intercept_dr_read] = POST_EX(SVM_EXIT_READ_DR0),
6000 [x86_intercept_dr_write] = POST_EX(SVM_EXIT_WRITE_DR0),
Joerg Roedeldee6bb72011-04-04 12:39:30 +02006001 [x86_intercept_sldt] = POST_EX(SVM_EXIT_LDTR_READ),
6002 [x86_intercept_str] = POST_EX(SVM_EXIT_TR_READ),
6003 [x86_intercept_lldt] = POST_EX(SVM_EXIT_LDTR_WRITE),
6004 [x86_intercept_ltr] = POST_EX(SVM_EXIT_TR_WRITE),
6005 [x86_intercept_sgdt] = POST_EX(SVM_EXIT_GDTR_READ),
6006 [x86_intercept_sidt] = POST_EX(SVM_EXIT_IDTR_READ),
6007 [x86_intercept_lgdt] = POST_EX(SVM_EXIT_GDTR_WRITE),
6008 [x86_intercept_lidt] = POST_EX(SVM_EXIT_IDTR_WRITE),
Joerg Roedel01de8b02011-04-04 12:39:31 +02006009 [x86_intercept_vmrun] = POST_EX(SVM_EXIT_VMRUN),
6010 [x86_intercept_vmmcall] = POST_EX(SVM_EXIT_VMMCALL),
6011 [x86_intercept_vmload] = POST_EX(SVM_EXIT_VMLOAD),
6012 [x86_intercept_vmsave] = POST_EX(SVM_EXIT_VMSAVE),
6013 [x86_intercept_stgi] = POST_EX(SVM_EXIT_STGI),
6014 [x86_intercept_clgi] = POST_EX(SVM_EXIT_CLGI),
6015 [x86_intercept_skinit] = POST_EX(SVM_EXIT_SKINIT),
6016 [x86_intercept_invlpga] = POST_EX(SVM_EXIT_INVLPGA),
Joerg Roedeld7eb8202011-04-04 12:39:32 +02006017 [x86_intercept_rdtscp] = POST_EX(SVM_EXIT_RDTSCP),
6018 [x86_intercept_monitor] = POST_MEM(SVM_EXIT_MONITOR),
6019 [x86_intercept_mwait] = POST_EX(SVM_EXIT_MWAIT),
Joerg Roedel80612522011-04-04 12:39:33 +02006020 [x86_intercept_invlpg] = POST_EX(SVM_EXIT_INVLPG),
6021 [x86_intercept_invd] = POST_EX(SVM_EXIT_INVD),
6022 [x86_intercept_wbinvd] = POST_EX(SVM_EXIT_WBINVD),
6023 [x86_intercept_wrmsr] = POST_EX(SVM_EXIT_MSR),
6024 [x86_intercept_rdtsc] = POST_EX(SVM_EXIT_RDTSC),
6025 [x86_intercept_rdmsr] = POST_EX(SVM_EXIT_MSR),
6026 [x86_intercept_rdpmc] = POST_EX(SVM_EXIT_RDPMC),
6027 [x86_intercept_cpuid] = PRE_EX(SVM_EXIT_CPUID),
6028 [x86_intercept_rsm] = PRE_EX(SVM_EXIT_RSM),
Joerg Roedelbf608f82011-04-04 12:39:34 +02006029 [x86_intercept_pause] = PRE_EX(SVM_EXIT_PAUSE),
6030 [x86_intercept_pushf] = PRE_EX(SVM_EXIT_PUSHF),
6031 [x86_intercept_popf] = PRE_EX(SVM_EXIT_POPF),
6032 [x86_intercept_intn] = PRE_EX(SVM_EXIT_SWINT),
6033 [x86_intercept_iret] = PRE_EX(SVM_EXIT_IRET),
6034 [x86_intercept_icebp] = PRE_EX(SVM_EXIT_ICEBP),
6035 [x86_intercept_hlt] = POST_EX(SVM_EXIT_HLT),
Joerg Roedelf6511932011-04-04 12:39:35 +02006036 [x86_intercept_in] = POST_EX(SVM_EXIT_IOIO),
6037 [x86_intercept_ins] = POST_EX(SVM_EXIT_IOIO),
6038 [x86_intercept_out] = POST_EX(SVM_EXIT_IOIO),
6039 [x86_intercept_outs] = POST_EX(SVM_EXIT_IOIO),
Vitaly Kuznetsov02d41602019-08-13 15:53:32 +02006040 [x86_intercept_xsetbv] = PRE_EX(SVM_EXIT_XSETBV),
Joerg Roedelcfec82c2011-04-04 12:39:28 +02006041};
6042
Joerg Roedel80612522011-04-04 12:39:33 +02006043#undef PRE_EX
Joerg Roedelcfec82c2011-04-04 12:39:28 +02006044#undef POST_EX
Joerg Roedeld7eb8202011-04-04 12:39:32 +02006045#undef POST_MEM
Joerg Roedelcfec82c2011-04-04 12:39:28 +02006046
Joerg Roedel8a76d7f2011-04-04 12:39:27 +02006047static int svm_check_intercept(struct kvm_vcpu *vcpu,
6048 struct x86_instruction_info *info,
6049 enum x86_intercept_stage stage)
6050{
Joerg Roedelcfec82c2011-04-04 12:39:28 +02006051 struct vcpu_svm *svm = to_svm(vcpu);
6052 int vmexit, ret = X86EMUL_CONTINUE;
6053 struct __x86_intercept icpt_info;
6054 struct vmcb *vmcb = svm->vmcb;
6055
6056 if (info->intercept >= ARRAY_SIZE(x86_intercept_map))
6057 goto out;
6058
6059 icpt_info = x86_intercept_map[info->intercept];
6060
Avi Kivity40e19b52011-04-21 12:35:41 +03006061 if (stage != icpt_info.stage)
Joerg Roedelcfec82c2011-04-04 12:39:28 +02006062 goto out;
6063
6064 switch (icpt_info.exit_code) {
6065 case SVM_EXIT_READ_CR0:
6066 if (info->intercept == x86_intercept_cr_read)
6067 icpt_info.exit_code += info->modrm_reg;
6068 break;
6069 case SVM_EXIT_WRITE_CR0: {
6070 unsigned long cr0, val;
6071 u64 intercept;
6072
6073 if (info->intercept == x86_intercept_cr_write)
6074 icpt_info.exit_code += info->modrm_reg;
6075
Jan Kiszka62baf442014-06-29 21:55:53 +02006076 if (icpt_info.exit_code != SVM_EXIT_WRITE_CR0 ||
6077 info->intercept == x86_intercept_clts)
Joerg Roedelcfec82c2011-04-04 12:39:28 +02006078 break;
6079
6080 intercept = svm->nested.intercept;
6081
6082 if (!(intercept & (1ULL << INTERCEPT_SELECTIVE_CR0)))
6083 break;
6084
6085 cr0 = vcpu->arch.cr0 & ~SVM_CR0_SELECTIVE_MASK;
6086 val = info->src_val & ~SVM_CR0_SELECTIVE_MASK;
6087
6088 if (info->intercept == x86_intercept_lmsw) {
6089 cr0 &= 0xfUL;
6090 val &= 0xfUL;
6091 /* lmsw can't clear PE - catch this here */
6092 if (cr0 & X86_CR0_PE)
6093 val |= X86_CR0_PE;
6094 }
6095
6096 if (cr0 ^ val)
6097 icpt_info.exit_code = SVM_EXIT_CR0_SEL_WRITE;
6098
6099 break;
6100 }
Joerg Roedel3b88e412011-04-04 12:39:29 +02006101 case SVM_EXIT_READ_DR0:
6102 case SVM_EXIT_WRITE_DR0:
6103 icpt_info.exit_code += info->modrm_reg;
6104 break;
Joerg Roedel80612522011-04-04 12:39:33 +02006105 case SVM_EXIT_MSR:
6106 if (info->intercept == x86_intercept_wrmsr)
6107 vmcb->control.exit_info_1 = 1;
6108 else
6109 vmcb->control.exit_info_1 = 0;
6110 break;
Joerg Roedelbf608f82011-04-04 12:39:34 +02006111 case SVM_EXIT_PAUSE:
6112 /*
6113 * We get this for NOP only, but pause
6114 * is rep not, check this here
6115 */
6116 if (info->rep_prefix != REPE_PREFIX)
6117 goto out;
Jan H. Schönherr49a8afc2017-09-05 23:58:44 +02006118 break;
Joerg Roedelf6511932011-04-04 12:39:35 +02006119 case SVM_EXIT_IOIO: {
6120 u64 exit_info;
6121 u32 bytes;
6122
Joerg Roedelf6511932011-04-04 12:39:35 +02006123 if (info->intercept == x86_intercept_in ||
6124 info->intercept == x86_intercept_ins) {
Jan Kiszka6cbc5f52014-06-30 12:52:55 +02006125 exit_info = ((info->src_val & 0xffff) << 16) |
6126 SVM_IOIO_TYPE_MASK;
Joerg Roedelf6511932011-04-04 12:39:35 +02006127 bytes = info->dst_bytes;
Jan Kiszka6493f152014-06-30 11:07:05 +02006128 } else {
Jan Kiszka6cbc5f52014-06-30 12:52:55 +02006129 exit_info = (info->dst_val & 0xffff) << 16;
Jan Kiszka6493f152014-06-30 11:07:05 +02006130 bytes = info->src_bytes;
Joerg Roedelf6511932011-04-04 12:39:35 +02006131 }
6132
6133 if (info->intercept == x86_intercept_outs ||
6134 info->intercept == x86_intercept_ins)
6135 exit_info |= SVM_IOIO_STR_MASK;
6136
6137 if (info->rep_prefix)
6138 exit_info |= SVM_IOIO_REP_MASK;
6139
6140 bytes = min(bytes, 4u);
6141
6142 exit_info |= bytes << SVM_IOIO_SIZE_SHIFT;
6143
6144 exit_info |= (u32)info->ad_bytes << (SVM_IOIO_ASIZE_SHIFT - 1);
6145
6146 vmcb->control.exit_info_1 = exit_info;
6147 vmcb->control.exit_info_2 = info->next_rip;
6148
6149 break;
6150 }
Joerg Roedelcfec82c2011-04-04 12:39:28 +02006151 default:
6152 break;
6153 }
6154
Bandan Dasf1047652015-06-11 02:05:33 -04006155 /* TODO: Advertise NRIPS to guest hypervisor unconditionally */
6156 if (static_cpu_has(X86_FEATURE_NRIPS))
6157 vmcb->control.next_rip = info->next_rip;
Joerg Roedelcfec82c2011-04-04 12:39:28 +02006158 vmcb->control.exit_code = icpt_info.exit_code;
6159 vmexit = nested_svm_exit_handled(svm);
6160
6161 ret = (vmexit == NESTED_EXIT_DONE) ? X86EMUL_INTERCEPTED
6162 : X86EMUL_CONTINUE;
6163
6164out:
6165 return ret;
Joerg Roedel8a76d7f2011-04-04 12:39:27 +02006166}
6167
Sean Christopherson95b5a482019-04-19 22:50:59 -07006168static void svm_handle_exit_irqoff(struct kvm_vcpu *vcpu)
Yang Zhanga547c6d2013-04-11 19:25:10 +08006169{
Sean Christophersond7a08882019-07-10 09:07:34 -07006170
Yang Zhanga547c6d2013-04-11 19:25:10 +08006171}
6172
Radim Krčmářae97a3b2014-08-21 18:08:06 +02006173static void svm_sched_in(struct kvm_vcpu *vcpu, int cpu)
6174{
Babu Moger8566ac82018-03-16 16:37:26 -04006175 if (pause_filter_thresh)
6176 shrink_ple_window(vcpu);
Radim Krčmářae97a3b2014-08-21 18:08:06 +02006177}
6178
Suravee Suthikulpanitbe8ca172016-05-04 14:09:49 -05006179static inline void avic_post_state_restore(struct kvm_vcpu *vcpu)
6180{
6181 if (avic_handle_apic_id_update(vcpu) != 0)
6182 return;
Suthikulpanit, Suravee98d90582019-01-29 08:08:42 +00006183 avic_handle_dfr_update(vcpu);
Suravee Suthikulpanitbe8ca172016-05-04 14:09:49 -05006184 avic_handle_ldr_update(vcpu);
6185}
6186
Borislav Petkov74f16902017-03-26 23:51:24 +02006187static void svm_setup_mce(struct kvm_vcpu *vcpu)
6188{
6189 /* [63:9] are reserved. */
6190 vcpu->arch.mcg_cap &= 0x1ff;
6191}
6192
Ladi Prosek72d7b372017-10-11 16:54:41 +02006193static int svm_smi_allowed(struct kvm_vcpu *vcpu)
6194{
Ladi Prosek05cade72017-10-11 16:54:45 +02006195 struct vcpu_svm *svm = to_svm(vcpu);
6196
6197 /* Per APM Vol.2 15.22.2 "Response to SMI" */
6198 if (!gif_set(svm))
6199 return 0;
6200
6201 if (is_guest_mode(&svm->vcpu) &&
6202 svm->nested.intercept & (1ULL << INTERCEPT_SMI)) {
6203 /* TODO: Might need to set exit_info_1 and exit_info_2 here */
6204 svm->vmcb->control.exit_code = SVM_EXIT_SMI;
6205 svm->nested.exit_required = true;
6206 return 0;
6207 }
6208
Ladi Prosek72d7b372017-10-11 16:54:41 +02006209 return 1;
6210}
6211
Ladi Prosek0234bf82017-10-11 16:54:40 +02006212static int svm_pre_enter_smm(struct kvm_vcpu *vcpu, char *smstate)
6213{
Ladi Prosek05cade72017-10-11 16:54:45 +02006214 struct vcpu_svm *svm = to_svm(vcpu);
6215 int ret;
6216
6217 if (is_guest_mode(vcpu)) {
6218 /* FED8h - SVM Guest */
6219 put_smstate(u64, smstate, 0x7ed8, 1);
6220 /* FEE0h - SVM Guest VMCB Physical Address */
6221 put_smstate(u64, smstate, 0x7ee0, svm->nested.vmcb);
6222
6223 svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
6224 svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
6225 svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];
6226
6227 ret = nested_svm_vmexit(svm);
6228 if (ret)
6229 return ret;
6230 }
Ladi Prosek0234bf82017-10-11 16:54:40 +02006231 return 0;
6232}
6233
Sean Christophersoned193212019-04-02 08:03:09 -07006234static int svm_pre_leave_smm(struct kvm_vcpu *vcpu, const char *smstate)
Ladi Prosek0234bf82017-10-11 16:54:40 +02006235{
Ladi Prosek05cade72017-10-11 16:54:45 +02006236 struct vcpu_svm *svm = to_svm(vcpu);
6237 struct vmcb *nested_vmcb;
KarimAllah Ahmed8c5fbf12019-01-31 21:24:40 +01006238 struct kvm_host_map map;
Sean Christophersoned193212019-04-02 08:03:09 -07006239 u64 guest;
6240 u64 vmcb;
Ladi Prosek05cade72017-10-11 16:54:45 +02006241
Sean Christophersoned193212019-04-02 08:03:09 -07006242 guest = GET_SMSTATE(u64, smstate, 0x7ed8);
6243 vmcb = GET_SMSTATE(u64, smstate, 0x7ee0);
Ladi Prosek05cade72017-10-11 16:54:45 +02006244
Sean Christophersoned193212019-04-02 08:03:09 -07006245 if (guest) {
KarimAllah Ahmed8c5fbf12019-01-31 21:24:40 +01006246 if (kvm_vcpu_map(&svm->vcpu, gpa_to_gfn(vmcb), &map) == -EINVAL)
Sean Christopherson9ec19492019-04-02 08:03:11 -07006247 return 1;
KarimAllah Ahmed8c5fbf12019-01-31 21:24:40 +01006248 nested_vmcb = map.hva;
6249 enter_svm_guest_mode(svm, vmcb, nested_vmcb, &map);
Ladi Prosek05cade72017-10-11 16:54:45 +02006250 }
Sean Christopherson9ec19492019-04-02 08:03:11 -07006251 return 0;
Ladi Prosek0234bf82017-10-11 16:54:40 +02006252}
6253
Ladi Prosekcc3d9672017-10-17 16:02:39 +02006254static int enable_smi_window(struct kvm_vcpu *vcpu)
6255{
6256 struct vcpu_svm *svm = to_svm(vcpu);
6257
6258 if (!gif_set(svm)) {
6259 if (vgif_enabled(svm))
6260 set_intercept(svm, INTERCEPT_STGI);
6261 /* STGI will cause a vm exit */
6262 return 1;
6263 }
6264 return 0;
6265}
6266
Brijesh Singh1654efc2017-12-04 10:57:34 -06006267static int sev_asid_new(void)
6268{
6269 int pos;
6270
6271 /*
6272 * SEV-enabled guest must use asid from min_sev_asid to max_sev_asid.
6273 */
6274 pos = find_next_zero_bit(sev_asid_bitmap, max_sev_asid, min_sev_asid - 1);
6275 if (pos >= max_sev_asid)
6276 return -EBUSY;
6277
6278 set_bit(pos, sev_asid_bitmap);
6279 return pos + 1;
6280}
6281
6282static int sev_guest_init(struct kvm *kvm, struct kvm_sev_cmd *argp)
6283{
Sean Christopherson81811c12018-03-20 12:17:21 -07006284 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
Brijesh Singh1654efc2017-12-04 10:57:34 -06006285 int asid, ret;
6286
6287 ret = -EBUSY;
David Rientjes3f14a892019-01-02 12:56:33 -08006288 if (unlikely(sev->active))
6289 return ret;
6290
Brijesh Singh1654efc2017-12-04 10:57:34 -06006291 asid = sev_asid_new();
6292 if (asid < 0)
6293 return ret;
6294
6295 ret = sev_platform_init(&argp->error);
6296 if (ret)
6297 goto e_free;
6298
6299 sev->active = true;
6300 sev->asid = asid;
Brijesh Singh1e80fdc2017-12-04 10:57:38 -06006301 INIT_LIST_HEAD(&sev->regions_list);
Brijesh Singh1654efc2017-12-04 10:57:34 -06006302
6303 return 0;
6304
6305e_free:
6306 __sev_asid_free(asid);
6307 return ret;
6308}
6309
Brijesh Singh59414c92017-12-04 10:57:35 -06006310static int sev_bind_asid(struct kvm *kvm, unsigned int handle, int *error)
6311{
6312 struct sev_data_activate *data;
6313 int asid = sev_get_asid(kvm);
6314 int ret;
6315
6316 wbinvd_on_all_cpus();
6317
6318 ret = sev_guest_df_flush(error);
6319 if (ret)
6320 return ret;
6321
Ben Gardon1ec69642019-02-11 11:02:51 -08006322 data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT);
Brijesh Singh59414c92017-12-04 10:57:35 -06006323 if (!data)
6324 return -ENOMEM;
6325
6326 /* activate ASID on the given handle */
6327 data->handle = handle;
6328 data->asid = asid;
6329 ret = sev_guest_activate(data, error);
6330 kfree(data);
6331
6332 return ret;
6333}
6334
Brijesh Singh89c50582017-12-04 10:57:35 -06006335static int __sev_issue_cmd(int fd, int id, void *data, int *error)
Brijesh Singh59414c92017-12-04 10:57:35 -06006336{
6337 struct fd f;
6338 int ret;
6339
6340 f = fdget(fd);
6341 if (!f.file)
6342 return -EBADF;
6343
6344 ret = sev_issue_cmd_external_user(f.file, id, data, error);
6345
6346 fdput(f);
6347 return ret;
6348}
6349
Brijesh Singh89c50582017-12-04 10:57:35 -06006350static int sev_issue_cmd(struct kvm *kvm, int id, void *data, int *error)
6351{
Sean Christopherson81811c12018-03-20 12:17:21 -07006352 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
Brijesh Singh89c50582017-12-04 10:57:35 -06006353
6354 return __sev_issue_cmd(sev->fd, id, data, error);
6355}
6356
Brijesh Singh59414c92017-12-04 10:57:35 -06006357static int sev_launch_start(struct kvm *kvm, struct kvm_sev_cmd *argp)
6358{
Sean Christopherson81811c12018-03-20 12:17:21 -07006359 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
Brijesh Singh59414c92017-12-04 10:57:35 -06006360 struct sev_data_launch_start *start;
6361 struct kvm_sev_launch_start params;
6362 void *dh_blob, *session_blob;
6363 int *error = &argp->error;
6364 int ret;
6365
6366 if (!sev_guest(kvm))
6367 return -ENOTTY;
6368
6369 if (copy_from_user(&params, (void __user *)(uintptr_t)argp->data, sizeof(params)))
6370 return -EFAULT;
6371
Ben Gardon1ec69642019-02-11 11:02:51 -08006372 start = kzalloc(sizeof(*start), GFP_KERNEL_ACCOUNT);
Brijesh Singh59414c92017-12-04 10:57:35 -06006373 if (!start)
6374 return -ENOMEM;
6375
6376 dh_blob = NULL;
6377 if (params.dh_uaddr) {
6378 dh_blob = psp_copy_user_blob(params.dh_uaddr, params.dh_len);
6379 if (IS_ERR(dh_blob)) {
6380 ret = PTR_ERR(dh_blob);
6381 goto e_free;
6382 }
6383
6384 start->dh_cert_address = __sme_set(__pa(dh_blob));
6385 start->dh_cert_len = params.dh_len;
6386 }
6387
6388 session_blob = NULL;
6389 if (params.session_uaddr) {
6390 session_blob = psp_copy_user_blob(params.session_uaddr, params.session_len);
6391 if (IS_ERR(session_blob)) {
6392 ret = PTR_ERR(session_blob);
6393 goto e_free_dh;
6394 }
6395
6396 start->session_address = __sme_set(__pa(session_blob));
6397 start->session_len = params.session_len;
6398 }
6399
6400 start->handle = params.handle;
6401 start->policy = params.policy;
6402
6403 /* create memory encryption context */
Brijesh Singh89c50582017-12-04 10:57:35 -06006404 ret = __sev_issue_cmd(argp->sev_fd, SEV_CMD_LAUNCH_START, start, error);
Brijesh Singh59414c92017-12-04 10:57:35 -06006405 if (ret)
6406 goto e_free_session;
6407
6408 /* Bind ASID to this guest */
6409 ret = sev_bind_asid(kvm, start->handle, error);
6410 if (ret)
6411 goto e_free_session;
6412
6413 /* return handle to userspace */
6414 params.handle = start->handle;
6415 if (copy_to_user((void __user *)(uintptr_t)argp->data, &params, sizeof(params))) {
6416 sev_unbind_asid(kvm, start->handle);
6417 ret = -EFAULT;
6418 goto e_free_session;
6419 }
6420
6421 sev->handle = start->handle;
6422 sev->fd = argp->sev_fd;
6423
6424e_free_session:
6425 kfree(session_blob);
6426e_free_dh:
6427 kfree(dh_blob);
6428e_free:
6429 kfree(start);
6430 return ret;
6431}
6432
David Rientjesede885e2019-03-19 15:19:56 -07006433static unsigned long get_num_contig_pages(unsigned long idx,
6434 struct page **inpages, unsigned long npages)
Brijesh Singh89c50582017-12-04 10:57:35 -06006435{
6436 unsigned long paddr, next_paddr;
David Rientjesede885e2019-03-19 15:19:56 -07006437 unsigned long i = idx + 1, pages = 1;
Brijesh Singh89c50582017-12-04 10:57:35 -06006438
6439 /* find the number of contiguous pages starting from idx */
6440 paddr = __sme_page_pa(inpages[idx]);
6441 while (i < npages) {
6442 next_paddr = __sme_page_pa(inpages[i++]);
6443 if ((paddr + PAGE_SIZE) == next_paddr) {
6444 pages++;
6445 paddr = next_paddr;
6446 continue;
6447 }
6448 break;
6449 }
6450
6451 return pages;
6452}
6453
6454static int sev_launch_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp)
6455{
David Rientjesede885e2019-03-19 15:19:56 -07006456 unsigned long vaddr, vaddr_end, next_vaddr, npages, pages, size, i;
Sean Christopherson81811c12018-03-20 12:17:21 -07006457 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
Brijesh Singh89c50582017-12-04 10:57:35 -06006458 struct kvm_sev_launch_update_data params;
6459 struct sev_data_launch_update_data *data;
6460 struct page **inpages;
David Rientjesede885e2019-03-19 15:19:56 -07006461 int ret;
Brijesh Singh89c50582017-12-04 10:57:35 -06006462
6463 if (!sev_guest(kvm))
6464 return -ENOTTY;
6465
6466 if (copy_from_user(&params, (void __user *)(uintptr_t)argp->data, sizeof(params)))
6467 return -EFAULT;
6468
Ben Gardon1ec69642019-02-11 11:02:51 -08006469 data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT);
Brijesh Singh89c50582017-12-04 10:57:35 -06006470 if (!data)
6471 return -ENOMEM;
6472
6473 vaddr = params.uaddr;
6474 size = params.len;
6475 vaddr_end = vaddr + size;
6476
6477 /* Lock the user memory. */
6478 inpages = sev_pin_memory(kvm, vaddr, size, &npages, 1);
6479 if (!inpages) {
6480 ret = -ENOMEM;
6481 goto e_free;
6482 }
6483
6484 /*
6485 * The LAUNCH_UPDATE command will perform in-place encryption of the
6486 * memory content (i.e it will write the same memory region with C=1).
6487 * It's possible that the cache may contain the data with C=0, i.e.,
6488 * unencrypted so invalidate it first.
6489 */
6490 sev_clflush_pages(inpages, npages);
6491
6492 for (i = 0; vaddr < vaddr_end; vaddr = next_vaddr, i += pages) {
6493 int offset, len;
6494
6495 /*
6496 * If the user buffer is not page-aligned, calculate the offset
6497 * within the page.
6498 */
6499 offset = vaddr & (PAGE_SIZE - 1);
6500
6501 /* Calculate the number of pages that can be encrypted in one go. */
6502 pages = get_num_contig_pages(i, inpages, npages);
6503
6504 len = min_t(size_t, ((pages * PAGE_SIZE) - offset), size);
6505
6506 data->handle = sev->handle;
6507 data->len = len;
6508 data->address = __sme_page_pa(inpages[i]) + offset;
6509 ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_UPDATE_DATA, data, &argp->error);
6510 if (ret)
6511 goto e_unpin;
6512
6513 size -= len;
6514 next_vaddr = vaddr + len;
6515 }
6516
6517e_unpin:
6518 /* content of memory is updated, mark pages dirty */
6519 for (i = 0; i < npages; i++) {
6520 set_page_dirty_lock(inpages[i]);
6521 mark_page_accessed(inpages[i]);
6522 }
6523 /* unlock the user pages */
6524 sev_unpin_memory(kvm, inpages, npages);
6525e_free:
6526 kfree(data);
6527 return ret;
6528}
6529
Brijesh Singh0d0736f2017-12-04 10:57:36 -06006530static int sev_launch_measure(struct kvm *kvm, struct kvm_sev_cmd *argp)
6531{
Brijesh Singh3e233382018-02-23 12:36:50 -06006532 void __user *measure = (void __user *)(uintptr_t)argp->data;
Sean Christopherson81811c12018-03-20 12:17:21 -07006533 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
Brijesh Singh0d0736f2017-12-04 10:57:36 -06006534 struct sev_data_launch_measure *data;
6535 struct kvm_sev_launch_measure params;
Brijesh Singh3e233382018-02-23 12:36:50 -06006536 void __user *p = NULL;
Brijesh Singh0d0736f2017-12-04 10:57:36 -06006537 void *blob = NULL;
6538 int ret;
6539
6540 if (!sev_guest(kvm))
6541 return -ENOTTY;
6542
Brijesh Singh3e233382018-02-23 12:36:50 -06006543 if (copy_from_user(&params, measure, sizeof(params)))
Brijesh Singh0d0736f2017-12-04 10:57:36 -06006544 return -EFAULT;
6545
Ben Gardon1ec69642019-02-11 11:02:51 -08006546 data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT);
Brijesh Singh0d0736f2017-12-04 10:57:36 -06006547 if (!data)
6548 return -ENOMEM;
6549
6550 /* User wants to query the blob length */
6551 if (!params.len)
6552 goto cmd;
6553
Brijesh Singh3e233382018-02-23 12:36:50 -06006554 p = (void __user *)(uintptr_t)params.uaddr;
6555 if (p) {
Brijesh Singh0d0736f2017-12-04 10:57:36 -06006556 if (params.len > SEV_FW_BLOB_MAX_SIZE) {
6557 ret = -EINVAL;
6558 goto e_free;
6559 }
6560
Brijesh Singh0d0736f2017-12-04 10:57:36 -06006561 ret = -ENOMEM;
6562 blob = kmalloc(params.len, GFP_KERNEL);
6563 if (!blob)
6564 goto e_free;
6565
6566 data->address = __psp_pa(blob);
6567 data->len = params.len;
6568 }
6569
6570cmd:
6571 data->handle = sev->handle;
6572 ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_MEASURE, data, &argp->error);
6573
6574 /*
6575 * If we query the session length, FW responded with expected data.
6576 */
6577 if (!params.len)
6578 goto done;
6579
6580 if (ret)
6581 goto e_free_blob;
6582
6583 if (blob) {
Brijesh Singh3e233382018-02-23 12:36:50 -06006584 if (copy_to_user(p, blob, params.len))
Brijesh Singh0d0736f2017-12-04 10:57:36 -06006585 ret = -EFAULT;
6586 }
6587
6588done:
6589 params.len = data->len;
Brijesh Singh3e233382018-02-23 12:36:50 -06006590 if (copy_to_user(measure, &params, sizeof(params)))
Brijesh Singh0d0736f2017-12-04 10:57:36 -06006591 ret = -EFAULT;
6592e_free_blob:
6593 kfree(blob);
6594e_free:
6595 kfree(data);
6596 return ret;
6597}
6598
Brijesh Singh5bdb0e22017-12-04 10:57:36 -06006599static int sev_launch_finish(struct kvm *kvm, struct kvm_sev_cmd *argp)
6600{
Sean Christopherson81811c12018-03-20 12:17:21 -07006601 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
Brijesh Singh5bdb0e22017-12-04 10:57:36 -06006602 struct sev_data_launch_finish *data;
6603 int ret;
6604
6605 if (!sev_guest(kvm))
6606 return -ENOTTY;
6607
Ben Gardon1ec69642019-02-11 11:02:51 -08006608 data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT);
Brijesh Singh5bdb0e22017-12-04 10:57:36 -06006609 if (!data)
6610 return -ENOMEM;
6611
6612 data->handle = sev->handle;
6613 ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_FINISH, data, &argp->error);
6614
6615 kfree(data);
6616 return ret;
6617}
6618
Brijesh Singh255d9e72017-12-04 10:57:37 -06006619static int sev_guest_status(struct kvm *kvm, struct kvm_sev_cmd *argp)
6620{
Sean Christopherson81811c12018-03-20 12:17:21 -07006621 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
Brijesh Singh255d9e72017-12-04 10:57:37 -06006622 struct kvm_sev_guest_status params;
6623 struct sev_data_guest_status *data;
6624 int ret;
6625
6626 if (!sev_guest(kvm))
6627 return -ENOTTY;
6628
Ben Gardon1ec69642019-02-11 11:02:51 -08006629 data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT);
Brijesh Singh255d9e72017-12-04 10:57:37 -06006630 if (!data)
6631 return -ENOMEM;
6632
6633 data->handle = sev->handle;
6634 ret = sev_issue_cmd(kvm, SEV_CMD_GUEST_STATUS, data, &argp->error);
6635 if (ret)
6636 goto e_free;
6637
6638 params.policy = data->policy;
6639 params.state = data->state;
6640 params.handle = data->handle;
6641
6642 if (copy_to_user((void __user *)(uintptr_t)argp->data, &params, sizeof(params)))
6643 ret = -EFAULT;
6644e_free:
6645 kfree(data);
6646 return ret;
6647}
6648
Brijesh Singh24f41fb2017-12-04 10:57:37 -06006649static int __sev_issue_dbg_cmd(struct kvm *kvm, unsigned long src,
6650 unsigned long dst, int size,
6651 int *error, bool enc)
6652{
Sean Christopherson81811c12018-03-20 12:17:21 -07006653 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
Brijesh Singh24f41fb2017-12-04 10:57:37 -06006654 struct sev_data_dbg *data;
6655 int ret;
6656
Ben Gardon1ec69642019-02-11 11:02:51 -08006657 data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT);
Brijesh Singh24f41fb2017-12-04 10:57:37 -06006658 if (!data)
6659 return -ENOMEM;
6660
6661 data->handle = sev->handle;
6662 data->dst_addr = dst;
6663 data->src_addr = src;
6664 data->len = size;
6665
6666 ret = sev_issue_cmd(kvm,
6667 enc ? SEV_CMD_DBG_ENCRYPT : SEV_CMD_DBG_DECRYPT,
6668 data, error);
6669 kfree(data);
6670 return ret;
6671}
6672
6673static int __sev_dbg_decrypt(struct kvm *kvm, unsigned long src_paddr,
6674 unsigned long dst_paddr, int sz, int *err)
6675{
6676 int offset;
6677
6678 /*
6679 * Its safe to read more than we are asked, caller should ensure that
6680 * destination has enough space.
6681 */
6682 src_paddr = round_down(src_paddr, 16);
6683 offset = src_paddr & 15;
6684 sz = round_up(sz + offset, 16);
6685
6686 return __sev_issue_dbg_cmd(kvm, src_paddr, dst_paddr, sz, err, false);
6687}
6688
6689static int __sev_dbg_decrypt_user(struct kvm *kvm, unsigned long paddr,
6690 unsigned long __user dst_uaddr,
6691 unsigned long dst_paddr,
6692 int size, int *err)
6693{
6694 struct page *tpage = NULL;
6695 int ret, offset;
6696
6697 /* if inputs are not 16-byte then use intermediate buffer */
6698 if (!IS_ALIGNED(dst_paddr, 16) ||
6699 !IS_ALIGNED(paddr, 16) ||
6700 !IS_ALIGNED(size, 16)) {
6701 tpage = (void *)alloc_page(GFP_KERNEL);
6702 if (!tpage)
6703 return -ENOMEM;
6704
6705 dst_paddr = __sme_page_pa(tpage);
6706 }
6707
6708 ret = __sev_dbg_decrypt(kvm, paddr, dst_paddr, size, err);
6709 if (ret)
6710 goto e_free;
6711
6712 if (tpage) {
6713 offset = paddr & 15;
6714 if (copy_to_user((void __user *)(uintptr_t)dst_uaddr,
6715 page_address(tpage) + offset, size))
6716 ret = -EFAULT;
6717 }
6718
6719e_free:
6720 if (tpage)
6721 __free_page(tpage);
6722
6723 return ret;
6724}
6725
Brijesh Singh7d1594f2017-12-04 10:57:37 -06006726static int __sev_dbg_encrypt_user(struct kvm *kvm, unsigned long paddr,
6727 unsigned long __user vaddr,
6728 unsigned long dst_paddr,
6729 unsigned long __user dst_vaddr,
6730 int size, int *error)
6731{
6732 struct page *src_tpage = NULL;
6733 struct page *dst_tpage = NULL;
6734 int ret, len = size;
6735
6736 /* If source buffer is not aligned then use an intermediate buffer */
6737 if (!IS_ALIGNED(vaddr, 16)) {
6738 src_tpage = alloc_page(GFP_KERNEL);
6739 if (!src_tpage)
6740 return -ENOMEM;
6741
6742 if (copy_from_user(page_address(src_tpage),
6743 (void __user *)(uintptr_t)vaddr, size)) {
6744 __free_page(src_tpage);
6745 return -EFAULT;
6746 }
6747
6748 paddr = __sme_page_pa(src_tpage);
6749 }
6750
6751 /*
6752 * If destination buffer or length is not aligned then do read-modify-write:
6753 * - decrypt destination in an intermediate buffer
6754 * - copy the source buffer in an intermediate buffer
6755 * - use the intermediate buffer as source buffer
6756 */
6757 if (!IS_ALIGNED(dst_vaddr, 16) || !IS_ALIGNED(size, 16)) {
6758 int dst_offset;
6759
6760 dst_tpage = alloc_page(GFP_KERNEL);
6761 if (!dst_tpage) {
6762 ret = -ENOMEM;
6763 goto e_free;
6764 }
6765
6766 ret = __sev_dbg_decrypt(kvm, dst_paddr,
6767 __sme_page_pa(dst_tpage), size, error);
6768 if (ret)
6769 goto e_free;
6770
6771 /*
6772 * If source is kernel buffer then use memcpy() otherwise
6773 * copy_from_user().
6774 */
6775 dst_offset = dst_paddr & 15;
6776
6777 if (src_tpage)
6778 memcpy(page_address(dst_tpage) + dst_offset,
6779 page_address(src_tpage), size);
6780 else {
6781 if (copy_from_user(page_address(dst_tpage) + dst_offset,
6782 (void __user *)(uintptr_t)vaddr, size)) {
6783 ret = -EFAULT;
6784 goto e_free;
6785 }
6786 }
6787
6788 paddr = __sme_page_pa(dst_tpage);
6789 dst_paddr = round_down(dst_paddr, 16);
6790 len = round_up(size, 16);
6791 }
6792
6793 ret = __sev_issue_dbg_cmd(kvm, paddr, dst_paddr, len, error, true);
6794
6795e_free:
6796 if (src_tpage)
6797 __free_page(src_tpage);
6798 if (dst_tpage)
6799 __free_page(dst_tpage);
6800 return ret;
6801}
6802
Brijesh Singh24f41fb2017-12-04 10:57:37 -06006803static int sev_dbg_crypt(struct kvm *kvm, struct kvm_sev_cmd *argp, bool dec)
6804{
6805 unsigned long vaddr, vaddr_end, next_vaddr;
Colin Ian King0186ec82018-08-28 16:22:28 +01006806 unsigned long dst_vaddr;
Brijesh Singh24f41fb2017-12-04 10:57:37 -06006807 struct page **src_p, **dst_p;
6808 struct kvm_sev_dbg debug;
6809 unsigned long n;
David Rientjesb86bc282019-03-25 11:47:31 -07006810 unsigned int size;
6811 int ret;
Brijesh Singh24f41fb2017-12-04 10:57:37 -06006812
6813 if (!sev_guest(kvm))
6814 return -ENOTTY;
6815
6816 if (copy_from_user(&debug, (void __user *)(uintptr_t)argp->data, sizeof(debug)))
6817 return -EFAULT;
6818
David Rientjesb86bc282019-03-25 11:47:31 -07006819 if (!debug.len || debug.src_uaddr + debug.len < debug.src_uaddr)
6820 return -EINVAL;
6821 if (!debug.dst_uaddr)
6822 return -EINVAL;
6823
Brijesh Singh24f41fb2017-12-04 10:57:37 -06006824 vaddr = debug.src_uaddr;
6825 size = debug.len;
6826 vaddr_end = vaddr + size;
6827 dst_vaddr = debug.dst_uaddr;
Brijesh Singh24f41fb2017-12-04 10:57:37 -06006828
6829 for (; vaddr < vaddr_end; vaddr = next_vaddr) {
6830 int len, s_off, d_off;
6831
6832 /* lock userspace source and destination page */
6833 src_p = sev_pin_memory(kvm, vaddr & PAGE_MASK, PAGE_SIZE, &n, 0);
6834 if (!src_p)
6835 return -EFAULT;
6836
6837 dst_p = sev_pin_memory(kvm, dst_vaddr & PAGE_MASK, PAGE_SIZE, &n, 1);
6838 if (!dst_p) {
6839 sev_unpin_memory(kvm, src_p, n);
6840 return -EFAULT;
6841 }
6842
6843 /*
6844 * The DBG_{DE,EN}CRYPT commands will perform {dec,en}cryption of the
6845 * memory content (i.e it will write the same memory region with C=1).
6846 * It's possible that the cache may contain the data with C=0, i.e.,
6847 * unencrypted so invalidate it first.
6848 */
6849 sev_clflush_pages(src_p, 1);
6850 sev_clflush_pages(dst_p, 1);
6851
6852 /*
6853 * Since user buffer may not be page aligned, calculate the
6854 * offset within the page.
6855 */
6856 s_off = vaddr & ~PAGE_MASK;
6857 d_off = dst_vaddr & ~PAGE_MASK;
6858 len = min_t(size_t, (PAGE_SIZE - s_off), size);
6859
Brijesh Singh7d1594f2017-12-04 10:57:37 -06006860 if (dec)
6861 ret = __sev_dbg_decrypt_user(kvm,
6862 __sme_page_pa(src_p[0]) + s_off,
6863 dst_vaddr,
6864 __sme_page_pa(dst_p[0]) + d_off,
6865 len, &argp->error);
6866 else
6867 ret = __sev_dbg_encrypt_user(kvm,
6868 __sme_page_pa(src_p[0]) + s_off,
6869 vaddr,
6870 __sme_page_pa(dst_p[0]) + d_off,
6871 dst_vaddr,
6872 len, &argp->error);
Brijesh Singh24f41fb2017-12-04 10:57:37 -06006873
David Rientjesb86bc282019-03-25 11:47:31 -07006874 sev_unpin_memory(kvm, src_p, n);
6875 sev_unpin_memory(kvm, dst_p, n);
Brijesh Singh24f41fb2017-12-04 10:57:37 -06006876
6877 if (ret)
6878 goto err;
6879
6880 next_vaddr = vaddr + len;
6881 dst_vaddr = dst_vaddr + len;
6882 size -= len;
6883 }
6884err:
6885 return ret;
6886}
6887
Brijesh Singh9f5b5b92017-12-04 10:57:38 -06006888static int sev_launch_secret(struct kvm *kvm, struct kvm_sev_cmd *argp)
6889{
Sean Christopherson81811c12018-03-20 12:17:21 -07006890 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
Brijesh Singh9f5b5b92017-12-04 10:57:38 -06006891 struct sev_data_launch_secret *data;
6892 struct kvm_sev_launch_secret params;
6893 struct page **pages;
6894 void *blob, *hdr;
6895 unsigned long n;
Brijesh Singh9c5e0af2018-02-19 10:13:25 -06006896 int ret, offset;
Brijesh Singh9f5b5b92017-12-04 10:57:38 -06006897
6898 if (!sev_guest(kvm))
6899 return -ENOTTY;
6900
6901 if (copy_from_user(&params, (void __user *)(uintptr_t)argp->data, sizeof(params)))
6902 return -EFAULT;
6903
6904 pages = sev_pin_memory(kvm, params.guest_uaddr, params.guest_len, &n, 1);
6905 if (!pages)
6906 return -ENOMEM;
6907
6908 /*
6909 * The secret must be copied into contiguous memory region, lets verify
6910 * that userspace memory pages are contiguous before we issue command.
6911 */
6912 if (get_num_contig_pages(0, pages, n) != n) {
6913 ret = -EINVAL;
6914 goto e_unpin_memory;
6915 }
6916
6917 ret = -ENOMEM;
Ben Gardon1ec69642019-02-11 11:02:51 -08006918 data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT);
Brijesh Singh9f5b5b92017-12-04 10:57:38 -06006919 if (!data)
6920 goto e_unpin_memory;
6921
Brijesh Singh9c5e0af2018-02-19 10:13:25 -06006922 offset = params.guest_uaddr & (PAGE_SIZE - 1);
6923 data->guest_address = __sme_page_pa(pages[0]) + offset;
6924 data->guest_len = params.guest_len;
6925
Brijesh Singh9f5b5b92017-12-04 10:57:38 -06006926 blob = psp_copy_user_blob(params.trans_uaddr, params.trans_len);
6927 if (IS_ERR(blob)) {
6928 ret = PTR_ERR(blob);
6929 goto e_free;
6930 }
6931
6932 data->trans_address = __psp_pa(blob);
6933 data->trans_len = params.trans_len;
6934
6935 hdr = psp_copy_user_blob(params.hdr_uaddr, params.hdr_len);
6936 if (IS_ERR(hdr)) {
6937 ret = PTR_ERR(hdr);
6938 goto e_free_blob;
6939 }
Brijesh Singh9c5e0af2018-02-19 10:13:25 -06006940 data->hdr_address = __psp_pa(hdr);
6941 data->hdr_len = params.hdr_len;
Brijesh Singh9f5b5b92017-12-04 10:57:38 -06006942
6943 data->handle = sev->handle;
6944 ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_UPDATE_SECRET, data, &argp->error);
6945
6946 kfree(hdr);
6947
6948e_free_blob:
6949 kfree(blob);
6950e_free:
6951 kfree(data);
6952e_unpin_memory:
6953 sev_unpin_memory(kvm, pages, n);
6954 return ret;
6955}
6956
Brijesh Singh1654efc2017-12-04 10:57:34 -06006957static int svm_mem_enc_op(struct kvm *kvm, void __user *argp)
6958{
6959 struct kvm_sev_cmd sev_cmd;
6960 int r;
6961
6962 if (!svm_sev_enabled())
6963 return -ENOTTY;
6964
6965 if (copy_from_user(&sev_cmd, argp, sizeof(struct kvm_sev_cmd)))
6966 return -EFAULT;
6967
6968 mutex_lock(&kvm->lock);
6969
6970 switch (sev_cmd.id) {
6971 case KVM_SEV_INIT:
6972 r = sev_guest_init(kvm, &sev_cmd);
6973 break;
Brijesh Singh59414c92017-12-04 10:57:35 -06006974 case KVM_SEV_LAUNCH_START:
6975 r = sev_launch_start(kvm, &sev_cmd);
6976 break;
Brijesh Singh89c50582017-12-04 10:57:35 -06006977 case KVM_SEV_LAUNCH_UPDATE_DATA:
6978 r = sev_launch_update_data(kvm, &sev_cmd);
6979 break;
Brijesh Singh0d0736f2017-12-04 10:57:36 -06006980 case KVM_SEV_LAUNCH_MEASURE:
6981 r = sev_launch_measure(kvm, &sev_cmd);
6982 break;
Brijesh Singh5bdb0e22017-12-04 10:57:36 -06006983 case KVM_SEV_LAUNCH_FINISH:
6984 r = sev_launch_finish(kvm, &sev_cmd);
6985 break;
Brijesh Singh255d9e72017-12-04 10:57:37 -06006986 case KVM_SEV_GUEST_STATUS:
6987 r = sev_guest_status(kvm, &sev_cmd);
6988 break;
Brijesh Singh24f41fb2017-12-04 10:57:37 -06006989 case KVM_SEV_DBG_DECRYPT:
6990 r = sev_dbg_crypt(kvm, &sev_cmd, true);
6991 break;
Brijesh Singh7d1594f2017-12-04 10:57:37 -06006992 case KVM_SEV_DBG_ENCRYPT:
6993 r = sev_dbg_crypt(kvm, &sev_cmd, false);
6994 break;
Brijesh Singh9f5b5b92017-12-04 10:57:38 -06006995 case KVM_SEV_LAUNCH_SECRET:
6996 r = sev_launch_secret(kvm, &sev_cmd);
6997 break;
Brijesh Singh1654efc2017-12-04 10:57:34 -06006998 default:
6999 r = -EINVAL;
7000 goto out;
7001 }
7002
7003 if (copy_to_user(argp, &sev_cmd, sizeof(struct kvm_sev_cmd)))
7004 r = -EFAULT;
7005
7006out:
7007 mutex_unlock(&kvm->lock);
7008 return r;
7009}
7010
Brijesh Singh1e80fdc2017-12-04 10:57:38 -06007011static int svm_register_enc_region(struct kvm *kvm,
7012 struct kvm_enc_region *range)
7013{
Sean Christopherson81811c12018-03-20 12:17:21 -07007014 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
Brijesh Singh1e80fdc2017-12-04 10:57:38 -06007015 struct enc_region *region;
7016 int ret = 0;
7017
7018 if (!sev_guest(kvm))
7019 return -ENOTTY;
7020
Dan Carpenter86bf20c2018-05-19 09:01:36 +03007021 if (range->addr > ULONG_MAX || range->size > ULONG_MAX)
7022 return -EINVAL;
7023
Ben Gardon1ec69642019-02-11 11:02:51 -08007024 region = kzalloc(sizeof(*region), GFP_KERNEL_ACCOUNT);
Brijesh Singh1e80fdc2017-12-04 10:57:38 -06007025 if (!region)
7026 return -ENOMEM;
7027
7028 region->pages = sev_pin_memory(kvm, range->addr, range->size, &region->npages, 1);
7029 if (!region->pages) {
7030 ret = -ENOMEM;
7031 goto e_free;
7032 }
7033
7034 /*
7035 * The guest may change the memory encryption attribute from C=0 -> C=1
7036 * or vice versa for this memory range. Lets make sure caches are
7037 * flushed to ensure that guest data gets written into memory with
7038 * correct C-bit.
7039 */
7040 sev_clflush_pages(region->pages, region->npages);
7041
7042 region->uaddr = range->addr;
7043 region->size = range->size;
7044
7045 mutex_lock(&kvm->lock);
7046 list_add_tail(&region->list, &sev->regions_list);
7047 mutex_unlock(&kvm->lock);
7048
7049 return ret;
7050
7051e_free:
7052 kfree(region);
7053 return ret;
7054}
7055
7056static struct enc_region *
7057find_enc_region(struct kvm *kvm, struct kvm_enc_region *range)
7058{
Sean Christopherson81811c12018-03-20 12:17:21 -07007059 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
Brijesh Singh1e80fdc2017-12-04 10:57:38 -06007060 struct list_head *head = &sev->regions_list;
7061 struct enc_region *i;
7062
7063 list_for_each_entry(i, head, list) {
7064 if (i->uaddr == range->addr &&
7065 i->size == range->size)
7066 return i;
7067 }
7068
7069 return NULL;
7070}
7071
7072
7073static int svm_unregister_enc_region(struct kvm *kvm,
7074 struct kvm_enc_region *range)
7075{
7076 struct enc_region *region;
7077 int ret;
7078
7079 mutex_lock(&kvm->lock);
7080
7081 if (!sev_guest(kvm)) {
7082 ret = -ENOTTY;
7083 goto failed;
7084 }
7085
7086 region = find_enc_region(kvm, range);
7087 if (!region) {
7088 ret = -EINVAL;
7089 goto failed;
7090 }
7091
7092 __unregister_enc_region_locked(kvm, region);
7093
7094 mutex_unlock(&kvm->lock);
7095 return 0;
7096
7097failed:
7098 mutex_unlock(&kvm->lock);
7099 return ret;
7100}
7101
Singh, Brijesh05d5a482019-02-15 17:24:12 +00007102static bool svm_need_emulation_on_page_fault(struct kvm_vcpu *vcpu)
7103{
Liran Alon118154b2019-07-17 02:56:58 +03007104 unsigned long cr4 = kvm_read_cr4(vcpu);
7105 bool smep = cr4 & X86_CR4_SMEP;
7106 bool smap = cr4 & X86_CR4_SMAP;
7107 bool is_user = svm_get_cpl(vcpu) == 3;
Singh, Brijesh05d5a482019-02-15 17:24:12 +00007108
7109 /*
Liran Alon118154b2019-07-17 02:56:58 +03007110 * Detect and workaround Errata 1096 Fam_17h_00_0Fh.
7111 *
7112 * Errata:
7113 * When CPU raise #NPF on guest data access and vCPU CR4.SMAP=1, it is
7114 * possible that CPU microcode implementing DecodeAssist will fail
7115 * to read bytes of instruction which caused #NPF. In this case,
7116 * GuestIntrBytes field of the VMCB on a VMEXIT will incorrectly
7117 * return 0 instead of the correct guest instruction bytes.
7118 *
7119 * This happens because CPU microcode reading instruction bytes
7120 * uses a special opcode which attempts to read data using CPL=0
7121 * priviledges. The microcode reads CS:RIP and if it hits a SMAP
7122 * fault, it gives up and returns no instruction bytes.
7123 *
7124 * Detection:
7125 * We reach here in case CPU supports DecodeAssist, raised #NPF and
7126 * returned 0 in GuestIntrBytes field of the VMCB.
7127 * First, errata can only be triggered in case vCPU CR4.SMAP=1.
7128 * Second, if vCPU CR4.SMEP=1, errata could only be triggered
7129 * in case vCPU CPL==3 (Because otherwise guest would have triggered
7130 * a SMEP fault instead of #NPF).
7131 * Otherwise, vCPU CR4.SMEP=0, errata could be triggered by any vCPU CPL.
7132 * As most guests enable SMAP if they have also enabled SMEP, use above
7133 * logic in order to attempt minimize false-positive of detecting errata
7134 * while still preserving all cases semantic correctness.
7135 *
7136 * Workaround:
7137 * To determine what instruction the guest was executing, the hypervisor
7138 * will have to decode the instruction at the instruction pointer.
Singh, Brijesh05d5a482019-02-15 17:24:12 +00007139 *
7140 * In non SEV guest, hypervisor will be able to read the guest
7141 * memory to decode the instruction pointer when insn_len is zero
7142 * so we return true to indicate that decoding is possible.
7143 *
7144 * But in the SEV guest, the guest memory is encrypted with the
7145 * guest specific key and hypervisor will not be able to decode the
7146 * instruction pointer so we will not able to workaround it. Lets
7147 * print the error and request to kill the guest.
7148 */
Liran Alon118154b2019-07-17 02:56:58 +03007149 if (smap && (!smep || is_user)) {
Singh, Brijesh05d5a482019-02-15 17:24:12 +00007150 if (!sev_guest(vcpu->kvm))
7151 return true;
7152
Liran Alon118154b2019-07-17 02:56:58 +03007153 pr_err_ratelimited("KVM: SEV Guest triggered AMD Erratum 1096\n");
Singh, Brijesh05d5a482019-02-15 17:24:12 +00007154 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
7155 }
7156
7157 return false;
7158}
7159
Liran Alon4b9852f2019-08-26 13:24:49 +03007160static bool svm_apic_init_signal_blocked(struct kvm_vcpu *vcpu)
7161{
7162 struct vcpu_svm *svm = to_svm(vcpu);
7163
7164 /*
7165 * TODO: Last condition latch INIT signals on vCPU when
7166 * vCPU is in guest-mode and vmcb12 defines intercept on INIT.
7167 * To properly emulate the INIT intercept, SVM should implement
7168 * kvm_x86_ops->check_nested_events() and call nested_svm_vmexit()
7169 * there if an INIT signal is pending.
7170 */
7171 return !gif_set(svm) ||
7172 (svm->vmcb->control.intercept & (1ULL << INTERCEPT_INIT));
7173}
7174
Kees Cook404f6aa2016-08-08 16:29:06 -07007175static struct kvm_x86_ops svm_x86_ops __ro_after_init = {
Avi Kivity6aa8b732006-12-10 02:21:36 -08007176 .cpu_has_kvm_support = has_svm,
7177 .disabled_by_bios = is_disabled,
7178 .hardware_setup = svm_hardware_setup,
7179 .hardware_unsetup = svm_hardware_unsetup,
Yang, Sheng002c7f72007-07-31 14:23:01 +03007180 .check_processor_compatibility = svm_check_processor_compat,
Avi Kivity6aa8b732006-12-10 02:21:36 -08007181 .hardware_enable = svm_hardware_enable,
7182 .hardware_disable = svm_hardware_disable,
Avi Kivity774ead32007-12-26 13:57:04 +02007183 .cpu_has_accelerated_tpr = svm_cpu_has_accelerated_tpr,
Tom Lendackybc226f02018-05-10 22:06:39 +02007184 .has_emulated_msr = svm_has_emulated_msr,
Avi Kivity6aa8b732006-12-10 02:21:36 -08007185
7186 .vcpu_create = svm_create_vcpu,
7187 .vcpu_free = svm_free_vcpu,
Avi Kivity04d2cc72007-09-10 18:10:54 +03007188 .vcpu_reset = svm_vcpu_reset,
Avi Kivity6aa8b732006-12-10 02:21:36 -08007189
Sean Christopherson434a1e92018-03-20 12:17:18 -07007190 .vm_alloc = svm_vm_alloc,
7191 .vm_free = svm_vm_free,
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -05007192 .vm_init = avic_vm_init,
Brijesh Singh1654efc2017-12-04 10:57:34 -06007193 .vm_destroy = svm_vm_destroy,
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -05007194
Avi Kivity04d2cc72007-09-10 18:10:54 +03007195 .prepare_guest_switch = svm_prepare_guest_switch,
Avi Kivity6aa8b732006-12-10 02:21:36 -08007196 .vcpu_load = svm_vcpu_load,
7197 .vcpu_put = svm_vcpu_put,
Suravee Suthikulpanit8221c132016-05-04 14:09:52 -05007198 .vcpu_blocking = svm_vcpu_blocking,
7199 .vcpu_unblocking = svm_vcpu_unblocking,
Avi Kivity6aa8b732006-12-10 02:21:36 -08007200
Paolo Bonzinia96036b2015-11-10 11:55:36 +01007201 .update_bp_intercept = update_bp_intercept,
Tom Lendacky801e4592018-02-21 13:39:51 -06007202 .get_msr_feature = svm_get_msr_feature,
Avi Kivity6aa8b732006-12-10 02:21:36 -08007203 .get_msr = svm_get_msr,
7204 .set_msr = svm_set_msr,
7205 .get_segment_base = svm_get_segment_base,
7206 .get_segment = svm_get_segment,
7207 .set_segment = svm_set_segment,
Izik Eidus2e4d2652008-03-24 19:38:34 +02007208 .get_cpl = svm_get_cpl,
Rusty Russell1747fb72007-09-06 01:21:32 +10007209 .get_cs_db_l_bits = kvm_get_cs_db_l_bits,
Avi Kivitye8467fd2009-12-29 18:43:06 +02007210 .decache_cr0_guest_bits = svm_decache_cr0_guest_bits,
Avi Kivityaff48ba2010-12-05 18:56:11 +02007211 .decache_cr3 = svm_decache_cr3,
Anthony Liguori25c4c272007-04-27 09:29:21 +03007212 .decache_cr4_guest_bits = svm_decache_cr4_guest_bits,
Avi Kivity6aa8b732006-12-10 02:21:36 -08007213 .set_cr0 = svm_set_cr0,
Avi Kivity6aa8b732006-12-10 02:21:36 -08007214 .set_cr3 = svm_set_cr3,
7215 .set_cr4 = svm_set_cr4,
7216 .set_efer = svm_set_efer,
7217 .get_idt = svm_get_idt,
7218 .set_idt = svm_set_idt,
7219 .get_gdt = svm_get_gdt,
7220 .set_gdt = svm_set_gdt,
Jan Kiszka73aaf249e2014-01-04 18:47:16 +01007221 .get_dr6 = svm_get_dr6,
7222 .set_dr6 = svm_set_dr6,
Gleb Natapov020df072010-04-13 10:05:23 +03007223 .set_dr7 = svm_set_dr7,
Paolo Bonzinifacb0132014-02-21 10:32:27 +01007224 .sync_dirty_debug_regs = svm_sync_dirty_debug_regs,
Avi Kivity6de4f3a2009-05-31 22:58:47 +03007225 .cache_reg = svm_cache_reg,
Avi Kivity6aa8b732006-12-10 02:21:36 -08007226 .get_rflags = svm_get_rflags,
7227 .set_rflags = svm_set_rflags,
Huaitong Hanbe94f6b2016-03-22 16:51:20 +08007228
Avi Kivity6aa8b732006-12-10 02:21:36 -08007229 .tlb_flush = svm_flush_tlb,
Junaid Shahidfaff8752018-06-29 13:10:05 -07007230 .tlb_flush_gva = svm_flush_tlb_gva,
Avi Kivity6aa8b732006-12-10 02:21:36 -08007231
Avi Kivity6aa8b732006-12-10 02:21:36 -08007232 .run = svm_vcpu_run,
Avi Kivity04d2cc72007-09-10 18:10:54 +03007233 .handle_exit = handle_exit,
Avi Kivity6aa8b732006-12-10 02:21:36 -08007234 .skip_emulated_instruction = skip_emulated_instruction,
Glauber Costa2809f5d2009-05-12 16:21:05 -04007235 .set_interrupt_shadow = svm_set_interrupt_shadow,
7236 .get_interrupt_shadow = svm_get_interrupt_shadow,
Ingo Molnar102d8322007-02-19 14:37:47 +02007237 .patch_hypercall = svm_patch_hypercall,
Eddie Dong2a8067f2007-08-06 16:29:07 +03007238 .set_irq = svm_set_irq,
Gleb Natapov95ba8273132009-04-21 17:45:08 +03007239 .set_nmi = svm_inject_nmi,
Avi Kivity298101d2007-11-25 13:41:11 +02007240 .queue_exception = svm_queue_exception,
Avi Kivityb463a6f2010-07-20 15:06:17 +03007241 .cancel_injection = svm_cancel_injection,
Gleb Natapov78646122009-03-23 12:12:11 +02007242 .interrupt_allowed = svm_interrupt_allowed,
Gleb Natapov95ba8273132009-04-21 17:45:08 +03007243 .nmi_allowed = svm_nmi_allowed,
Jan Kiszka3cfc3092009-11-12 01:04:25 +01007244 .get_nmi_mask = svm_get_nmi_mask,
7245 .set_nmi_mask = svm_set_nmi_mask,
Gleb Natapov95ba8273132009-04-21 17:45:08 +03007246 .enable_nmi_window = enable_nmi_window,
7247 .enable_irq_window = enable_irq_window,
7248 .update_cr8_intercept = update_cr8_intercept,
Jim Mattson8d860bb2018-05-09 16:56:05 -04007249 .set_virtual_apic_mode = svm_set_virtual_apic_mode,
Andrey Smetanind62caab2015-11-10 15:36:33 +03007250 .get_enable_apicv = svm_get_enable_apicv,
7251 .refresh_apicv_exec_ctrl = svm_refresh_apicv_exec_ctrl,
Yang Zhangc7c9c562013-01-25 10:18:51 +08007252 .load_eoi_exitmap = svm_load_eoi_exitmap,
Suravee Suthikulpanit44a95da2016-05-04 14:09:46 -05007253 .hwapic_irr_update = svm_hwapic_irr_update,
7254 .hwapic_isr_update = svm_hwapic_isr_update,
Liran Alonfa59cc02017-12-24 18:12:53 +02007255 .sync_pir_to_irr = kvm_lapic_find_highest_irr,
Suravee Suthikulpanitbe8ca172016-05-04 14:09:49 -05007256 .apicv_post_state_restore = avic_post_state_restore,
Izik Eiduscbc94022007-10-25 00:29:55 +02007257
7258 .set_tss_addr = svm_set_tss_addr,
Sean Christopherson2ac52ab2018-03-20 12:17:19 -07007259 .set_identity_map_addr = svm_set_identity_map_addr,
Sheng Yang67253af2008-04-25 10:20:22 +08007260 .get_tdp_level = get_npt_level,
Sheng Yang4b12f0d2009-04-27 20:35:42 +08007261 .get_mt_mask = svm_get_mt_mask,
Marcelo Tosatti229456f2009-06-17 09:22:14 -03007262
Avi Kivity586f9602010-11-18 13:09:54 +02007263 .get_exit_info = svm_get_exit_info,
Avi Kivity586f9602010-11-18 13:09:54 +02007264
Sheng Yang17cc3932010-01-05 19:02:27 +08007265 .get_lpage_level = svm_get_lpage_level,
Sheng Yang0e851882009-12-18 16:48:46 +08007266
7267 .cpuid_update = svm_cpuid_update,
Sheng Yang4e47c7a2009-12-18 16:48:47 +08007268
7269 .rdtscp_supported = svm_rdtscp_supported,
Mao, Junjiead756a12012-07-02 01:18:48 +00007270 .invpcid_supported = svm_invpcid_supported,
Paolo Bonzini93c4adc2014-03-05 23:19:52 +01007271 .mpx_supported = svm_mpx_supported,
Wanpeng Li55412b22014-12-02 19:21:30 +08007272 .xsaves_supported = svm_xsaves_supported,
Paolo Bonzini66336ca2016-07-12 10:36:41 +02007273 .umip_emulated = svm_umip_emulated,
Chao Peng86f52012018-10-24 16:05:11 +08007274 .pt_supported = svm_pt_supported,
Joerg Roedeld4330ef2010-04-22 12:33:11 +02007275
7276 .set_supported_cpuid = svm_set_supported_cpuid,
Sheng Yangf5f48ee2010-06-30 12:25:15 +08007277
7278 .has_wbinvd_exit = svm_has_wbinvd_exit,
Zachary Amsden99e3e302010-08-19 22:07:17 -10007279
KarimAllah Ahmede79f2452018-04-14 05:10:52 +02007280 .read_l1_tsc_offset = svm_read_l1_tsc_offset,
Leonid Shatz326e7422018-11-06 12:14:25 +02007281 .write_l1_tsc_offset = svm_write_l1_tsc_offset,
Joerg Roedel1c97f0a2010-09-10 17:30:41 +02007282
7283 .set_tdp_cr3 = set_tdp_cr3,
Joerg Roedel8a76d7f2011-04-04 12:39:27 +02007284
7285 .check_intercept = svm_check_intercept,
Sean Christopherson95b5a482019-04-19 22:50:59 -07007286 .handle_exit_irqoff = svm_handle_exit_irqoff,
Radim Krčmářae97a3b2014-08-21 18:08:06 +02007287
Sean Christophersond264ee02018-08-27 15:21:12 -07007288 .request_immediate_exit = __kvm_request_immediate_exit,
7289
Radim Krčmářae97a3b2014-08-21 18:08:06 +02007290 .sched_in = svm_sched_in,
Wei Huang25462f72015-06-19 15:45:05 +02007291
7292 .pmu_ops = &amd_pmu_ops,
Suravee Suthikulpanit340d3bc2016-05-04 14:09:47 -05007293 .deliver_posted_interrupt = svm_deliver_avic_intr,
Wanpeng Li17e433b2019-08-05 10:03:19 +08007294 .dy_apicv_has_pending_interrupt = svm_dy_apicv_has_pending_interrupt,
Suravee Suthikulpanit411b44b2016-08-23 13:52:43 -05007295 .update_pi_irte = svm_update_pi_irte,
Borislav Petkov74f16902017-03-26 23:51:24 +02007296 .setup_mce = svm_setup_mce,
Ladi Prosek0234bf82017-10-11 16:54:40 +02007297
Ladi Prosek72d7b372017-10-11 16:54:41 +02007298 .smi_allowed = svm_smi_allowed,
Ladi Prosek0234bf82017-10-11 16:54:40 +02007299 .pre_enter_smm = svm_pre_enter_smm,
7300 .pre_leave_smm = svm_pre_leave_smm,
Ladi Prosekcc3d9672017-10-17 16:02:39 +02007301 .enable_smi_window = enable_smi_window,
Brijesh Singh1654efc2017-12-04 10:57:34 -06007302
7303 .mem_enc_op = svm_mem_enc_op,
Brijesh Singh1e80fdc2017-12-04 10:57:38 -06007304 .mem_enc_reg_region = svm_register_enc_region,
7305 .mem_enc_unreg_region = svm_unregister_enc_region,
Vitaly Kuznetsov57b119d2018-10-16 18:50:01 +02007306
Vitaly Kuznetsov956e2552019-08-28 09:59:04 +02007307 .nested_enable_evmcs = NULL,
Vitaly Kuznetsovea152982019-08-27 18:04:02 +02007308 .nested_get_evmcs_version = NULL,
Singh, Brijesh05d5a482019-02-15 17:24:12 +00007309
7310 .need_emulation_on_page_fault = svm_need_emulation_on_page_fault,
Liran Alon4b9852f2019-08-26 13:24:49 +03007311
7312 .apic_init_signal_blocked = svm_apic_init_signal_blocked,
Avi Kivity6aa8b732006-12-10 02:21:36 -08007313};
7314
7315static int __init svm_init(void)
7316{
Zhang Xiantaocb498ea2007-11-14 20:39:31 +08007317 return kvm_init(&svm_x86_ops, sizeof(struct vcpu_svm),
Avi Kivity0ee75be2010-04-28 15:39:01 +03007318 __alignof__(struct vcpu_svm), THIS_MODULE);
Avi Kivity6aa8b732006-12-10 02:21:36 -08007319}
7320
7321static void __exit svm_exit(void)
7322{
Zhang Xiantaocb498ea2007-11-14 20:39:31 +08007323 kvm_exit();
Avi Kivity6aa8b732006-12-10 02:21:36 -08007324}
7325
7326module_init(svm_init)
7327module_exit(svm_exit)