blob: 57a2d8f76c2f6255683100348d42bca67a1c8a15 [file] [log] [blame]
Marc Zyngier4f8d6632012-12-10 16:29:28 +00001/*
2 * Copyright (C) 2012,2013 - ARM Ltd
3 * Author: Marc Zyngier <marc.zyngier@arm.com>
4 *
5 * Derived from arch/arm/include/asm/kvm_host.h:
6 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
7 * Author: Christoffer Dall <c.dall@virtualopensystems.com>
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as
11 * published by the Free Software Foundation.
12 *
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
17 *
18 * You should have received a copy of the GNU General Public License
19 * along with this program. If not, see <http://www.gnu.org/licenses/>.
20 */
21
22#ifndef __ARM64_KVM_HOST_H__
23#define __ARM64_KVM_HOST_H__
24
Paolo Bonzini65647302014-08-29 14:01:17 +020025#include <linux/types.h>
26#include <linux/kvm_types.h>
Marc Zyngier4f8d6632012-12-10 16:29:28 +000027#include <asm/kvm.h>
Marc Zyngier3a3604b2015-01-29 13:19:45 +000028#include <asm/kvm_asm.h>
Marc Zyngier4f8d6632012-12-10 16:29:28 +000029#include <asm/kvm_mmio.h>
Marc Zyngierad882132016-02-29 11:25:04 +000030#include <asm/kvm_perf_event.h>
Marc Zyngier4f8d6632012-12-10 16:29:28 +000031
Eric Augerc1426e42015-03-04 11:14:34 +010032#define __KVM_HAVE_ARCH_INTC_INITIALIZED
33
Marc Zyngier4f8d6632012-12-10 16:29:28 +000034#define KVM_USER_MEM_SLOTS 32
35#define KVM_PRIVATE_MEM_SLOTS 4
36#define KVM_COALESCED_MMIO_PAGE_OFFSET 1
David Hildenbrand920552b2015-09-18 12:34:53 +020037#define KVM_HALT_POLL_NS_DEFAULT 500000
Marc Zyngier4f8d6632012-12-10 16:29:28 +000038
39#include <kvm/arm_vgic.h>
40#include <kvm/arm_arch_timer.h>
Shannon Zhao04fe4722015-09-11 09:38:32 +080041#include <kvm/arm_pmu.h>
Marc Zyngier4f8d6632012-12-10 16:29:28 +000042
Ming Leief748912015-09-02 14:31:21 +080043#define KVM_MAX_VCPUS VGIC_V3_MAX_CPUS
44
Anup Patel7d0f84a2014-04-29 11:24:16 +053045#define KVM_VCPU_MAX_FEATURES 3
Marc Zyngier4f8d6632012-12-10 16:29:28 +000046
Will Deacon6951e482014-08-26 15:13:20 +010047int __attribute_const__ kvm_target_cpu(void);
Marc Zyngier4f8d6632012-12-10 16:29:28 +000048int kvm_reset_vcpu(struct kvm_vcpu *vcpu);
49int kvm_arch_dev_ioctl_check_extension(long ext);
50
51struct kvm_arch {
52 /* The VMID generation used for the virt. memory system */
53 u64 vmid_gen;
54 u32 vmid;
55
56 /* 1-level 2nd stage table and lock */
57 spinlock_t pgd_lock;
58 pgd_t *pgd;
59
60 /* VTTBR value associated with above pgd and vmid */
61 u64 vttbr;
62
Andre Przywara3caa2d82014-06-02 16:26:01 +020063 /* The maximum number of vCPUs depends on the used GIC model */
64 int max_vcpus;
65
Marc Zyngier4f8d6632012-12-10 16:29:28 +000066 /* Interrupt controller */
67 struct vgic_dist vgic;
68
69 /* Timer */
70 struct arch_timer_kvm timer;
71};
72
73#define KVM_NR_MEM_OBJS 40
74
75/*
76 * We don't want allocation failures within the mmu code, so we preallocate
77 * enough memory for a single page fault in a cache.
78 */
79struct kvm_mmu_memory_cache {
80 int nobjs;
81 void *objects[KVM_NR_MEM_OBJS];
82};
83
84struct kvm_vcpu_fault_info {
85 u32 esr_el2; /* Hyp Syndrom Register */
86 u64 far_el2; /* Hyp Fault Address Register */
87 u64 hpfar_el2; /* Hyp IPA Fault Address Register */
88};
89
Marc Zyngier9d8415d2015-10-25 19:57:11 +000090/*
91 * 0 is reserved as an invalid value.
92 * Order should be kept in sync with the save/restore code.
93 */
94enum vcpu_sysreg {
95 __INVALID_SYSREG__,
96 MPIDR_EL1, /* MultiProcessor Affinity Register */
97 CSSELR_EL1, /* Cache Size Selection Register */
98 SCTLR_EL1, /* System Control Register */
99 ACTLR_EL1, /* Auxiliary Control Register */
100 CPACR_EL1, /* Coprocessor Access Control */
101 TTBR0_EL1, /* Translation Table Base Register 0 */
102 TTBR1_EL1, /* Translation Table Base Register 1 */
103 TCR_EL1, /* Translation Control Register */
104 ESR_EL1, /* Exception Syndrome Register */
105 AFSR0_EL1, /* Auxilary Fault Status Register 0 */
106 AFSR1_EL1, /* Auxilary Fault Status Register 1 */
107 FAR_EL1, /* Fault Address Register */
108 MAIR_EL1, /* Memory Attribute Indirection Register */
109 VBAR_EL1, /* Vector Base Address Register */
110 CONTEXTIDR_EL1, /* Context ID Register */
111 TPIDR_EL0, /* Thread ID, User R/W */
112 TPIDRRO_EL0, /* Thread ID, User R/O */
113 TPIDR_EL1, /* Thread ID, Privileged */
114 AMAIR_EL1, /* Aux Memory Attribute Indirection Register */
115 CNTKCTL_EL1, /* Timer Control Register (EL1) */
116 PAR_EL1, /* Physical Address Register */
117 MDSCR_EL1, /* Monitor Debug System Control Register */
118 MDCCINT_EL1, /* Monitor Debug Comms Channel Interrupt Enable Reg */
119
Shannon Zhaoab946832015-06-18 16:01:53 +0800120 /* Performance Monitors Registers */
121 PMCR_EL0, /* Control Register */
Shannon Zhao3965c3c2015-08-31 17:20:22 +0800122 PMSELR_EL0, /* Event Counter Selection Register */
Shannon Zhaoab946832015-06-18 16:01:53 +0800123
Marc Zyngier9d8415d2015-10-25 19:57:11 +0000124 /* 32bit specific registers. Keep them at the end of the range */
125 DACR32_EL2, /* Domain Access Control Register */
126 IFSR32_EL2, /* Instruction Fault Status Register */
127 FPEXC32_EL2, /* Floating-Point Exception Control Register */
128 DBGVCR32_EL2, /* Debug Vector Catch Register */
129
130 NR_SYS_REGS /* Nothing after this line! */
131};
132
133/* 32bit mapping */
134#define c0_MPIDR (MPIDR_EL1 * 2) /* MultiProcessor ID Register */
135#define c0_CSSELR (CSSELR_EL1 * 2)/* Cache Size Selection Register */
136#define c1_SCTLR (SCTLR_EL1 * 2) /* System Control Register */
137#define c1_ACTLR (ACTLR_EL1 * 2) /* Auxiliary Control Register */
138#define c1_CPACR (CPACR_EL1 * 2) /* Coprocessor Access Control */
139#define c2_TTBR0 (TTBR0_EL1 * 2) /* Translation Table Base Register 0 */
140#define c2_TTBR0_high (c2_TTBR0 + 1) /* TTBR0 top 32 bits */
141#define c2_TTBR1 (TTBR1_EL1 * 2) /* Translation Table Base Register 1 */
142#define c2_TTBR1_high (c2_TTBR1 + 1) /* TTBR1 top 32 bits */
143#define c2_TTBCR (TCR_EL1 * 2) /* Translation Table Base Control R. */
144#define c3_DACR (DACR32_EL2 * 2)/* Domain Access Control Register */
145#define c5_DFSR (ESR_EL1 * 2) /* Data Fault Status Register */
146#define c5_IFSR (IFSR32_EL2 * 2)/* Instruction Fault Status Register */
147#define c5_ADFSR (AFSR0_EL1 * 2) /* Auxiliary Data Fault Status R */
148#define c5_AIFSR (AFSR1_EL1 * 2) /* Auxiliary Instr Fault Status R */
149#define c6_DFAR (FAR_EL1 * 2) /* Data Fault Address Register */
150#define c6_IFAR (c6_DFAR + 1) /* Instruction Fault Address Register */
151#define c7_PAR (PAR_EL1 * 2) /* Physical Address Register */
152#define c7_PAR_high (c7_PAR + 1) /* PAR top 32 bits */
153#define c10_PRRR (MAIR_EL1 * 2) /* Primary Region Remap Register */
154#define c10_NMRR (c10_PRRR + 1) /* Normal Memory Remap Register */
155#define c12_VBAR (VBAR_EL1 * 2) /* Vector Base Address Register */
156#define c13_CID (CONTEXTIDR_EL1 * 2) /* Context ID Register */
157#define c13_TID_URW (TPIDR_EL0 * 2) /* Thread ID, User R/W */
158#define c13_TID_URO (TPIDRRO_EL0 * 2)/* Thread ID, User R/O */
159#define c13_TID_PRIV (TPIDR_EL1 * 2) /* Thread ID, Privileged */
160#define c10_AMAIR0 (AMAIR_EL1 * 2) /* Aux Memory Attr Indirection Reg */
161#define c10_AMAIR1 (c10_AMAIR0 + 1)/* Aux Memory Attr Indirection Reg */
162#define c14_CNTKCTL (CNTKCTL_EL1 * 2) /* Timer Control Register (PL1) */
163
164#define cp14_DBGDSCRext (MDSCR_EL1 * 2)
165#define cp14_DBGBCR0 (DBGBCR0_EL1 * 2)
166#define cp14_DBGBVR0 (DBGBVR0_EL1 * 2)
167#define cp14_DBGBXVR0 (cp14_DBGBVR0 + 1)
168#define cp14_DBGWCR0 (DBGWCR0_EL1 * 2)
169#define cp14_DBGWVR0 (DBGWVR0_EL1 * 2)
170#define cp14_DBGDCCINT (MDCCINT_EL1 * 2)
171
172#define NR_COPRO_REGS (NR_SYS_REGS * 2)
173
Marc Zyngier4f8d6632012-12-10 16:29:28 +0000174struct kvm_cpu_context {
175 struct kvm_regs gp_regs;
Marc Zyngier40033a62013-02-06 19:17:50 +0000176 union {
177 u64 sys_regs[NR_SYS_REGS];
Marc Zyngier72564012014-04-24 10:27:13 +0100178 u32 copro[NR_COPRO_REGS];
Marc Zyngier40033a62013-02-06 19:17:50 +0000179 };
Marc Zyngier4f8d6632012-12-10 16:29:28 +0000180};
181
182typedef struct kvm_cpu_context kvm_cpu_context_t;
183
184struct kvm_vcpu_arch {
185 struct kvm_cpu_context ctxt;
186
187 /* HYP configuration */
188 u64 hcr_el2;
Alex Bennée56c7f5e2015-07-07 17:29:56 +0100189 u32 mdcr_el2;
Marc Zyngier4f8d6632012-12-10 16:29:28 +0000190
191 /* Exception Information */
192 struct kvm_vcpu_fault_info fault;
193
Alex Bennée84e690b2015-07-07 17:30:00 +0100194 /* Guest debug state */
Marc Zyngier0c557ed2014-04-24 10:24:46 +0100195 u64 debug_flags;
196
Alex Bennée84e690b2015-07-07 17:30:00 +0100197 /*
198 * We maintain more than a single set of debug registers to support
199 * debugging the guest from the host and to maintain separate host and
200 * guest state during world switches. vcpu_debug_state are the debug
201 * registers of the vcpu as the guest sees them. host_debug_state are
Alex Bennée834bf882015-07-07 17:30:02 +0100202 * the host registers which are saved and restored during
203 * world switches. external_debug_state contains the debug
204 * values we want to debug the guest. This is set via the
205 * KVM_SET_GUEST_DEBUG ioctl.
Alex Bennée84e690b2015-07-07 17:30:00 +0100206 *
207 * debug_ptr points to the set of debug registers that should be loaded
208 * onto the hardware when running the guest.
209 */
210 struct kvm_guest_debug_arch *debug_ptr;
211 struct kvm_guest_debug_arch vcpu_debug_state;
Alex Bennée834bf882015-07-07 17:30:02 +0100212 struct kvm_guest_debug_arch external_debug_state;
Alex Bennée84e690b2015-07-07 17:30:00 +0100213
Marc Zyngier4f8d6632012-12-10 16:29:28 +0000214 /* Pointer to host CPU context */
215 kvm_cpu_context_t *host_cpu_context;
Alex Bennée84e690b2015-07-07 17:30:00 +0100216 struct kvm_guest_debug_arch host_debug_state;
Marc Zyngier4f8d6632012-12-10 16:29:28 +0000217
218 /* VGIC state */
219 struct vgic_cpu vgic_cpu;
220 struct arch_timer_cpu timer_cpu;
Shannon Zhao04fe4722015-09-11 09:38:32 +0800221 struct kvm_pmu pmu;
Marc Zyngier4f8d6632012-12-10 16:29:28 +0000222
223 /*
224 * Anything that is not used directly from assembly code goes
225 * here.
226 */
Marc Zyngier4f8d6632012-12-10 16:29:28 +0000227
Alex Bennée337b99b2015-07-07 17:29:58 +0100228 /*
229 * Guest registers we preserve during guest debugging.
230 *
231 * These shadow registers are updated by the kvm_handle_sys_reg
232 * trap handler if the guest accesses or updates them while we
233 * are using guest debug.
234 */
235 struct {
236 u32 mdscr_el1;
237 } guest_debug_preserved;
238
Eric Auger37815282015-09-25 23:41:14 +0200239 /* vcpu power-off state */
240 bool power_off;
Marc Zyngier4f8d6632012-12-10 16:29:28 +0000241
Eric Auger3b928302015-09-25 23:41:17 +0200242 /* Don't run the guest (internal implementation need) */
243 bool pause;
244
Marc Zyngier4f8d6632012-12-10 16:29:28 +0000245 /* IO related fields */
246 struct kvm_decode mmio_decode;
247
248 /* Interrupt related fields */
249 u64 irq_lines; /* IRQ and FIQ levels */
250
251 /* Cache some mmu pages needed inside spinlock regions */
252 struct kvm_mmu_memory_cache mmu_page_cache;
253
254 /* Target CPU and feature flags */
Chen Gang6c8c0c42013-07-22 04:40:38 +0100255 int target;
Marc Zyngier4f8d6632012-12-10 16:29:28 +0000256 DECLARE_BITMAP(features, KVM_VCPU_MAX_FEATURES);
257
258 /* Detect first run of a vcpu */
259 bool has_run_once;
260};
261
262#define vcpu_gp_regs(v) (&(v)->arch.ctxt.gp_regs)
263#define vcpu_sys_reg(v,r) ((v)->arch.ctxt.sys_regs[(r)])
Marc Zyngier72564012014-04-24 10:27:13 +0100264/*
265 * CP14 and CP15 live in the same array, as they are backed by the
266 * same system registers.
267 */
268#define vcpu_cp14(v,r) ((v)->arch.ctxt.copro[(r)])
269#define vcpu_cp15(v,r) ((v)->arch.ctxt.copro[(r)])
Marc Zyngier4f8d6632012-12-10 16:29:28 +0000270
Victor Kamenskyf0a3eaf2014-07-02 17:19:30 +0100271#ifdef CONFIG_CPU_BIG_ENDIAN
Marc Zyngierdedf97e2014-08-01 12:00:36 +0100272#define vcpu_cp15_64_high(v,r) vcpu_cp15((v),(r))
273#define vcpu_cp15_64_low(v,r) vcpu_cp15((v),(r) + 1)
Victor Kamenskyf0a3eaf2014-07-02 17:19:30 +0100274#else
Marc Zyngierdedf97e2014-08-01 12:00:36 +0100275#define vcpu_cp15_64_high(v,r) vcpu_cp15((v),(r) + 1)
276#define vcpu_cp15_64_low(v,r) vcpu_cp15((v),(r))
Victor Kamenskyf0a3eaf2014-07-02 17:19:30 +0100277#endif
278
Marc Zyngier4f8d6632012-12-10 16:29:28 +0000279struct kvm_vm_stat {
280 u32 remote_tlb_flush;
281};
282
283struct kvm_vcpu_stat {
Paolo Bonzinif7819512015-02-04 18:20:58 +0100284 u32 halt_successful_poll;
Paolo Bonzini62bea5b2015-09-15 18:27:57 +0200285 u32 halt_attempted_poll;
Marc Zyngier4f8d6632012-12-10 16:29:28 +0000286 u32 halt_wakeup;
Amit Tomarb19e6892015-11-26 10:09:43 +0000287 u32 hvc_exit_stat;
288 u64 wfe_exit_stat;
289 u64 wfi_exit_stat;
290 u64 mmio_exit_user;
291 u64 mmio_exit_kernel;
292 u64 exits;
Marc Zyngier4f8d6632012-12-10 16:29:28 +0000293};
294
Anup Patel473bdc02013-09-30 14:20:06 +0530295int kvm_vcpu_preferred_target(struct kvm_vcpu_init *init);
Marc Zyngier4f8d6632012-12-10 16:29:28 +0000296unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu);
297int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *indices);
Marc Zyngier4f8d6632012-12-10 16:29:28 +0000298int kvm_arm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg);
299int kvm_arm_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg);
300
301#define KVM_ARCH_WANT_MMU_NOTIFIER
Marc Zyngier4f8d6632012-12-10 16:29:28 +0000302int kvm_unmap_hva(struct kvm *kvm, unsigned long hva);
303int kvm_unmap_hva_range(struct kvm *kvm,
304 unsigned long start, unsigned long end);
305void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte);
Marc Zyngier35307b92015-03-12 18:16:51 +0000306int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end);
307int kvm_test_age_hva(struct kvm *kvm, unsigned long hva);
Marc Zyngier4f8d6632012-12-10 16:29:28 +0000308
309/* We do not have shadow page tables, hence the empty hooks */
Tang Chenfe71557a2014-09-24 15:57:57 +0800310static inline void kvm_arch_mmu_notifier_invalidate_page(struct kvm *kvm,
311 unsigned long address)
312{
313}
314
Marc Zyngier4f8d6632012-12-10 16:29:28 +0000315struct kvm_vcpu *kvm_arm_get_running_vcpu(void);
Will Deacon4000be42014-08-26 15:13:21 +0100316struct kvm_vcpu * __percpu *kvm_get_running_vcpus(void);
Marc Zyngier4f8d6632012-12-10 16:29:28 +0000317
318u64 kvm_call_hyp(void *hypfn, ...);
Christoffer Dallcf5d31882014-10-16 17:00:18 +0200319void force_vm_exit(const cpumask_t *mask);
Mario Smarduch8199ed02015-01-15 15:58:59 -0800320void kvm_mmu_wp_memory_region(struct kvm *kvm, int slot);
Marc Zyngier4f8d6632012-12-10 16:29:28 +0000321
322int handle_exit(struct kvm_vcpu *vcpu, struct kvm_run *run,
323 int exception_index);
324
325int kvm_perf_init(void);
326int kvm_perf_teardown(void);
327
Andre Przywara4429fc62014-06-02 15:37:13 +0200328struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr);
329
Marc Zyngier092bd142012-12-17 17:07:52 +0000330static inline void __cpu_init_hyp_mode(phys_addr_t boot_pgd_ptr,
331 phys_addr_t pgd_ptr,
332 unsigned long hyp_stack_ptr,
333 unsigned long vector_ptr)
334{
335 /*
336 * Call initialization code, and switch to the full blown
337 * HYP code.
338 */
339 kvm_call_hyp((void *)boot_pgd_ptr, pgd_ptr,
340 hyp_stack_ptr, vector_ptr);
341}
342
Radim Krčmář13a34e02014-08-28 15:13:03 +0200343static inline void kvm_arch_hardware_disable(void) {}
Radim Krčmář0865e632014-08-28 15:13:02 +0200344static inline void kvm_arch_hardware_unsetup(void) {}
345static inline void kvm_arch_sync_events(struct kvm *kvm) {}
346static inline void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) {}
347static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {}
348
Alex Bennée56c7f5e2015-07-07 17:29:56 +0100349void kvm_arm_init_debug(void);
350void kvm_arm_setup_debug(struct kvm_vcpu *vcpu);
351void kvm_arm_clear_debug(struct kvm_vcpu *vcpu);
Alex Bennée84e690b2015-07-07 17:30:00 +0100352void kvm_arm_reset_debug_ptr(struct kvm_vcpu *vcpu);
Alex Bennée56c7f5e2015-07-07 17:29:56 +0100353
Marc Zyngier21a41792016-02-22 10:57:30 +0000354/* #define kvm_call_hyp(f, ...) __kvm_call_hyp(kvm_ksym_ref(f), ##__VA_ARGS__) */
355
356static inline void __cpu_init_stage2(void)
357{
358 kvm_call_hyp(__init_stage2_translation);
359}
360
Marc Zyngier4f8d6632012-12-10 16:29:28 +0000361#endif /* __ARM64_KVM_HOST_H__ */