blob: 77ecc11d67ae2e1b51bd7d3317edb3ce2635be8a [file] [log] [blame]
Thomas Gleixnerd94d71c2019-05-29 07:12:40 -07001// SPDX-License-Identifier: GPL-2.0-only
Christoffer Dall749cf76c2013-01-20 18:28:06 -05002/*
3 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
4 * Author: Christoffer Dall <c.dall@virtualopensystems.com>
Christoffer Dall749cf76c2013-01-20 18:28:06 -05005 */
6
Dave Martin85acda32018-04-20 16:20:43 +01007#include <linux/bug.h>
Lorenzo Pieralisi1fcf7ce2013-08-05 15:04:46 +01008#include <linux/cpu_pm.h>
Oliver Upton6caa5812021-08-02 19:28:09 +00009#include <linux/entry-kvm.h>
Christoffer Dall749cf76c2013-01-20 18:28:06 -050010#include <linux/errno.h>
11#include <linux/err.h>
12#include <linux/kvm_host.h>
Andre Przywara1085fdc2016-07-15 12:43:31 +010013#include <linux/list.h>
Christoffer Dall749cf76c2013-01-20 18:28:06 -050014#include <linux/module.h>
15#include <linux/vmalloc.h>
16#include <linux/fs.h>
17#include <linux/mman.h>
18#include <linux/sched.h>
Marc Zyngier47e62232021-08-02 13:38:30 +010019#include <linux/kmemleak.h>
Christoffer Dall86ce8532013-01-20 18:28:08 -050020#include <linux/kvm.h>
Eric Auger24124052017-10-27 15:28:31 +010021#include <linux/kvm_irqfd.h>
22#include <linux/irqbypass.h>
Marc Zyngierde737082018-06-21 10:43:59 +010023#include <linux/sched/stat.h>
David Brazdileeeee712020-12-02 18:41:12 +000024#include <linux/psci.h>
Christoffer Dall749cf76c2013-01-20 18:28:06 -050025#include <trace/events/kvm.h>
26
27#define CREATE_TRACE_POINTS
Marc Zyngier9ed24f42020-05-13 11:40:34 +010028#include "trace_arm.h"
Christoffer Dall749cf76c2013-01-20 18:28:06 -050029
Linus Torvalds7c0f6ba2016-12-24 11:46:01 -080030#include <linux/uaccess.h>
Christoffer Dall749cf76c2013-01-20 18:28:06 -050031#include <asm/ptrace.h>
32#include <asm/mman.h>
Christoffer Dall342cd0a2013-01-20 18:28:06 -050033#include <asm/tlbflush.h>
Christoffer Dall5b3e5e52013-01-20 18:28:09 -050034#include <asm/cacheflush.h>
Dave Martin85acda32018-04-20 16:20:43 +010035#include <asm/cpufeature.h>
Christoffer Dall342cd0a2013-01-20 18:28:06 -050036#include <asm/virt.h>
37#include <asm/kvm_arm.h>
38#include <asm/kvm_asm.h>
39#include <asm/kvm_mmu.h>
Christoffer Dallf7ed45b2013-01-20 18:47:42 -050040#include <asm/kvm_emulate.h>
Marc Zyngier910917b2015-10-27 12:18:48 +000041#include <asm/sections.h>
Christoffer Dall749cf76c2013-01-20 18:28:06 -050042
Steven Price8564d632019-10-21 16:28:18 +010043#include <kvm/arm_hypercalls.h>
44#include <kvm/arm_pmu.h>
45#include <kvm/arm_psci.h>
46
David Brazdild8b369c2020-12-02 18:40:57 +000047static enum kvm_mode kvm_mode = KVM_MODE_DEFAULT;
David Brazdilf19f6642020-12-02 18:41:22 +000048DEFINE_STATIC_KEY_FALSE(kvm_protected_mode_initialized);
David Brazdild8b369c2020-12-02 18:40:57 +000049
Marc Zyngier14ef9d02020-09-30 14:05:35 +010050DECLARE_KVM_HYP_PER_CPU(unsigned long, kvm_hyp_vector);
51
Christoffer Dall342cd0a2013-01-20 18:28:06 -050052static DEFINE_PER_CPU(unsigned long, kvm_arm_hyp_stack_page);
David Brazdil30c95392020-09-22 21:49:09 +010053unsigned long kvm_arm_hyp_percpu_base[NR_CPUS];
David Brazdil63fec242020-12-02 18:41:06 +000054DECLARE_KVM_NVHE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params);
Christoffer Dall342cd0a2013-01-20 18:28:06 -050055
Christoffer Dallf7ed45b2013-01-20 18:47:42 -050056/* The VMID used in the VTTBR */
57static atomic64_t kvm_vmid_gen = ATOMIC64_INIT(1);
Vladimir Murzin20475f72015-11-16 11:28:18 +000058static u32 kvm_next_vmid;
Christoffer Dallfb544d12018-12-11 13:23:57 +010059static DEFINE_SPINLOCK(kvm_vmid_lock);
Christoffer Dall342cd0a2013-01-20 18:28:06 -050060
Pavel Fedinc7da6fa2015-12-18 14:38:43 +030061static bool vgic_present;
62
AKASHI Takahiro67f69192016-04-27 17:47:05 +010063static DEFINE_PER_CPU(unsigned char, kvm_arm_hardware_enabled);
Christoffer Dall61bbe382017-10-27 19:57:51 +020064DEFINE_STATIC_KEY_FALSE(userspace_irqchip_in_use);
65
Christoffer Dall749cf76c2013-01-20 18:28:06 -050066int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
67{
68 return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
69}
70
Sean Christophersonb9904082020-03-21 13:25:55 -070071int kvm_arch_hardware_setup(void *opaque)
Christoffer Dall749cf76c2013-01-20 18:28:06 -050072{
73 return 0;
74}
75
Sean Christophersonb9904082020-03-21 13:25:55 -070076int kvm_arch_check_processor_compat(void *opaque)
Christoffer Dall749cf76c2013-01-20 18:28:06 -050077{
Sean Christophersonf257d6d2019-04-19 22:18:17 -070078 return 0;
Christoffer Dall749cf76c2013-01-20 18:28:06 -050079}
80
Christoffer Dallc7262002019-10-11 13:07:05 +020081int kvm_vm_ioctl_enable_cap(struct kvm *kvm,
82 struct kvm_enable_cap *cap)
83{
84 int r;
85
86 if (cap->flags)
87 return -EINVAL;
88
89 switch (cap->cap) {
90 case KVM_CAP_ARM_NISV_TO_USER:
91 r = 0;
92 kvm->arch.return_nisv_io_abort_to_user = true;
93 break;
Steven Price673638f2021-06-21 12:17:14 +010094 case KVM_CAP_ARM_MTE:
Steven Pricec4d7c512021-07-29 17:00:36 +010095 mutex_lock(&kvm->lock);
96 if (!system_supports_mte() || kvm->created_vcpus) {
97 r = -EINVAL;
98 } else {
99 r = 0;
100 kvm->arch.mte_enabled = true;
101 }
102 mutex_unlock(&kvm->lock);
Steven Price673638f2021-06-21 12:17:14 +0100103 break;
Christoffer Dallc7262002019-10-11 13:07:05 +0200104 default:
105 r = -EINVAL;
106 break;
107 }
108
109 return r;
110}
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500111
Marc Zyngier51070002020-04-27 15:15:07 +0100112static int kvm_arm_default_max_vcpus(void)
113{
114 return vgic_present ? kvm_vgic_get_max_vcpus() : KVM_MAX_VCPUS;
115}
116
Marc Zyngier4f1df622020-11-26 17:27:13 +0000117static void set_default_spectre(struct kvm *kvm)
Marc Zyngier23711a52020-11-10 14:13:06 +0000118{
119 /*
120 * The default is to expose CSV2 == 1 if the HW isn't affected.
121 * Although this is a per-CPU feature, we make it global because
122 * asymmetric systems are just a nuisance.
123 *
124 * Userspace can override this as long as it doesn't promise
125 * the impossible.
126 */
127 if (arm64_get_spectre_v2_state() == SPECTRE_UNAFFECTED)
128 kvm->arch.pfr0_csv2 = 1;
Marc Zyngier4f1df622020-11-26 17:27:13 +0000129 if (arm64_get_meltdown_state() == SPECTRE_UNAFFECTED)
130 kvm->arch.pfr0_csv3 = 1;
Marc Zyngier23711a52020-11-10 14:13:06 +0000131}
132
Christoffer Dalld5d81842013-01-20 18:28:07 -0500133/**
134 * kvm_arch_init_vm - initializes a VM data structure
135 * @kvm: pointer to the KVM struct
136 */
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500137int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
138{
Christoffer Dalla0e50aa2019-01-04 21:09:05 +0100139 int ret;
Christoffer Dalld5d81842013-01-20 18:28:07 -0500140
Marc Zyngierbca607e2018-10-01 13:40:36 +0100141 ret = kvm_arm_setup_stage2(kvm, type);
Suzuki K Poulose5b6c6742018-09-26 17:32:42 +0100142 if (ret)
143 return ret;
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500144
Christoffer Dalla0e50aa2019-01-04 21:09:05 +0100145 ret = kvm_init_stage2_mmu(kvm, &kvm->arch.mmu);
Christoffer Dalld5d81842013-01-20 18:28:07 -0500146 if (ret)
Christoffer Dalla0e50aa2019-01-04 21:09:05 +0100147 return ret;
Christoffer Dalld5d81842013-01-20 18:28:07 -0500148
Marc Zyngierc8dddec2016-06-13 15:00:45 +0100149 ret = create_hyp_mappings(kvm, kvm + 1, PAGE_HYP);
Christoffer Dalld5d81842013-01-20 18:28:07 -0500150 if (ret)
151 goto out_free_stage2_pgd;
152
Marc Zyngier6c3d63c2014-06-23 17:37:18 +0100153 kvm_vgic_early_init(kvm);
Christoffer Dalla1a64382013-11-16 10:51:25 -0800154
Andre Przywara3caa2d82014-06-02 16:26:01 +0200155 /* The maximum number of VCPUs is limited by the host's GIC model */
Marc Zyngier51070002020-04-27 15:15:07 +0100156 kvm->arch.max_vcpus = kvm_arm_default_max_vcpus();
Andre Przywara3caa2d82014-06-02 16:26:01 +0200157
Marc Zyngier4f1df622020-11-26 17:27:13 +0000158 set_default_spectre(kvm);
Marc Zyngier23711a52020-11-10 14:13:06 +0000159
Christoffer Dalld5d81842013-01-20 18:28:07 -0500160 return ret;
161out_free_stage2_pgd:
Christoffer Dalla0e50aa2019-01-04 21:09:05 +0100162 kvm_free_stage2_pgd(&kvm->arch.mmu);
Christoffer Dalld5d81842013-01-20 18:28:07 -0500163 return ret;
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500164}
165
Souptick Joarder1499fa82018-04-19 00:49:58 +0530166vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500167{
168 return VM_FAULT_SIGBUS;
169}
170
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500171
Christoffer Dalld5d81842013-01-20 18:28:07 -0500172/**
173 * kvm_arch_destroy_vm - destroy the VM data structure
174 * @kvm: pointer to the KVM struct
175 */
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500176void kvm_arch_destroy_vm(struct kvm *kvm)
177{
Marc Zyngierd7eec232020-02-12 11:31:02 +0000178 bitmap_free(kvm->arch.pmu_filter);
179
Marc Zyngierb2c9a852017-10-27 15:28:34 +0100180 kvm_vgic_destroy(kvm);
181
Marc Zyngier27592ae2021-11-16 16:03:57 +0000182 kvm_destroy_vcpus(kvm);
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500183}
184
Alexander Graf784aa3d2014-07-14 18:27:35 +0200185int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500186{
187 int r;
188 switch (ext) {
Marc Zyngier1a89dd92013-01-21 19:36:12 -0500189 case KVM_CAP_IRQCHIP:
Pavel Fedinc7da6fa2015-12-18 14:38:43 +0300190 r = vgic_present;
191 break;
Nikolay Nikolaevd44758c2015-01-24 12:00:02 +0000192 case KVM_CAP_IOEVENTFD:
Christoffer Dall73306722013-10-25 17:29:18 +0100193 case KVM_CAP_DEVICE_CTRL:
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500194 case KVM_CAP_USER_MEMORY:
195 case KVM_CAP_SYNC_MMU:
196 case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
197 case KVM_CAP_ONE_REG:
Marc Zyngieraa024c22013-01-20 18:28:13 -0500198 case KVM_CAP_ARM_PSCI:
Anup Patel4447a202014-04-29 11:24:25 +0530199 case KVM_CAP_ARM_PSCI_0_2:
Christoffer Dall98047882014-08-19 12:18:04 +0200200 case KVM_CAP_READONLY_MEM:
Alex Bennéeecccf0c2015-03-13 17:02:52 +0000201 case KVM_CAP_MP_STATE:
Paolo Bonzini460df4c2017-02-08 11:50:15 +0100202 case KVM_CAP_IMMEDIATE_EXIT:
Dongjiu Geng58bf4372018-10-13 00:12:49 +0800203 case KVM_CAP_VCPU_EVENTS:
Marc Zyngier92f35b72019-08-18 14:09:47 +0100204 case KVM_CAP_ARM_IRQ_LINE_LAYOUT_2:
Christoffer Dallc7262002019-10-11 13:07:05 +0200205 case KVM_CAP_ARM_NISV_TO_USER:
Christoffer Dallda345172019-10-11 13:07:06 +0200206 case KVM_CAP_ARM_INJECT_EXT_DABT:
Will Deacon36fb4cd2020-11-18 19:44:01 +0000207 case KVM_CAP_SET_GUEST_DEBUG:
208 case KVM_CAP_VCPU_ATTRIBUTES:
Jianyong Wu3bf72562020-12-09 14:09:29 +0800209 case KVM_CAP_PTP_KVM:
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500210 r = 1;
211 break;
Maxim Levitskyfa18aca2021-04-01 16:54:46 +0300212 case KVM_CAP_SET_GUEST_DEBUG2:
213 return KVM_GUESTDBG_VALID_MASK;
Christoffer Dall3401d5462013-01-23 13:18:04 -0500214 case KVM_CAP_ARM_SET_DEVICE_ADDR:
215 r = 1;
Marc Zyngierca46e102013-04-03 10:43:13 +0100216 break;
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500217 case KVM_CAP_NR_VCPUS:
Vitaly Kuznetsovf60a00d2021-11-16 17:34:38 +0100218 /*
219 * ARM64 treats KVM_CAP_NR_CPUS differently from all other
220 * architectures, as it does not always bound it to
221 * KVM_CAP_MAX_VCPUS. It should not matter much because
222 * this is just an advisory value.
223 */
224 r = min_t(unsigned int, num_online_cpus(),
225 kvm_arm_default_max_vcpus());
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500226 break;
227 case KVM_CAP_MAX_VCPUS:
Thomas Hutha86cb412019-05-23 18:43:08 +0200228 case KVM_CAP_MAX_VCPU_ID:
Marc Zyngier51070002020-04-27 15:15:07 +0100229 if (kvm)
230 r = kvm->arch.max_vcpus;
231 else
232 r = kvm_arm_default_max_vcpus();
Thomas Hutha86cb412019-05-23 18:43:08 +0200233 break;
Vladimir Murzin29885092016-11-02 11:55:34 +0000234 case KVM_CAP_MSI_DEVID:
235 if (!kvm)
236 r = -EINVAL;
237 else
238 r = kvm->arch.vgic.msis_require_devid;
239 break;
Christoffer Dallf7214e62017-02-01 12:54:11 +0100240 case KVM_CAP_ARM_USER_IRQ:
241 /*
242 * 1: EL1_VTIMER, EL1_PTIMER, and PMU.
243 * (bump this number if adding more devices)
244 */
245 r = 1;
246 break;
Steven Price673638f2021-06-21 12:17:14 +0100247 case KVM_CAP_ARM_MTE:
248 r = system_supports_mte();
249 break;
Andrew Jones004a0122020-08-04 19:06:04 +0200250 case KVM_CAP_STEAL_TIME:
251 r = kvm_arm_pvtime_supported();
252 break;
Will Deacon36fb4cd2020-11-18 19:44:01 +0000253 case KVM_CAP_ARM_EL1_32BIT:
254 r = cpus_have_const_cap(ARM64_HAS_32BIT_EL1);
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500255 break;
Will Deacon36fb4cd2020-11-18 19:44:01 +0000256 case KVM_CAP_GUEST_DEBUG_HW_BPS:
257 r = get_num_brps();
258 break;
259 case KVM_CAP_GUEST_DEBUG_HW_WPS:
260 r = get_num_wrps();
261 break;
262 case KVM_CAP_ARM_PMU_V3:
263 r = kvm_arm_support_pmu_v3();
264 break;
265 case KVM_CAP_ARM_INJECT_SERROR_ESR:
266 r = cpus_have_const_cap(ARM64_HAS_RAS_EXTN);
267 break;
268 case KVM_CAP_ARM_VM_IPA_SIZE:
269 r = get_kvm_ipa_limit();
270 break;
271 case KVM_CAP_ARM_SVE:
272 r = system_supports_sve();
273 break;
274 case KVM_CAP_ARM_PTRAUTH_ADDRESS:
275 case KVM_CAP_ARM_PTRAUTH_GENERIC:
276 r = system_has_full_ptr_auth();
277 break;
278 default:
279 r = 0;
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500280 }
Will Deacon36fb4cd2020-11-18 19:44:01 +0000281
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500282 return r;
283}
284
285long kvm_arch_dev_ioctl(struct file *filp,
286 unsigned int ioctl, unsigned long arg)
287{
288 return -EINVAL;
289}
290
Marc Orrd1e5b0e2018-05-15 04:37:37 -0700291struct kvm *kvm_arch_alloc_vm(void)
292{
Jia He115bae92021-09-07 20:31:12 +0800293 size_t sz = sizeof(struct kvm);
Marc Orrd1e5b0e2018-05-15 04:37:37 -0700294
Jia He115bae92021-09-07 20:31:12 +0800295 if (!has_vhe())
296 return kzalloc(sz, GFP_KERNEL_ACCOUNT);
297
298 return __vmalloc(sz, GFP_KERNEL_ACCOUNT | __GFP_HIGHMEM | __GFP_ZERO);
Marc Orrd1e5b0e2018-05-15 04:37:37 -0700299}
300
Sean Christopherson897cc382019-12-18 13:55:09 -0800301int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
302{
303 if (irqchip_in_kernel(kvm) && vgic_initialized(kvm))
304 return -EBUSY;
305
306 if (id >= kvm->arch.max_vcpus)
307 return -EINVAL;
308
309 return 0;
310}
311
Sean Christophersone529ef62019-12-18 13:55:15 -0800312int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500313{
Sean Christopherson39a93a82019-12-18 13:55:25 -0800314 int err;
315
316 /* Force users to call KVM_ARM_VCPU_INIT */
317 vcpu->arch.target = -1;
318 bitmap_zero(vcpu->arch.features, KVM_VCPU_MAX_FEATURES);
319
Sean Christophersone5394512020-07-02 19:35:41 -0700320 vcpu->arch.mmu_page_cache.gfp_zero = __GFP_ZERO;
321
Sean Christopherson39a93a82019-12-18 13:55:25 -0800322 /* Set up the timer */
323 kvm_timer_vcpu_init(vcpu);
324
325 kvm_pmu_vcpu_init(vcpu);
326
327 kvm_arm_reset_debug_ptr(vcpu);
328
329 kvm_arm_pvtime_vcpu_init(&vcpu->arch);
330
Christoffer Dalla0e50aa2019-01-04 21:09:05 +0100331 vcpu->arch.hw_mmu = &vcpu->kvm->arch.mmu;
332
Sean Christopherson39a93a82019-12-18 13:55:25 -0800333 err = kvm_vgic_vcpu_init(vcpu);
334 if (err)
335 return err;
336
Sean Christophersone529ef62019-12-18 13:55:15 -0800337 return create_hyp_mappings(vcpu, vcpu + 1, PAGE_HYP);
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500338}
339
Dominik Dingel31928aa2014-12-04 15:47:07 +0100340void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500341{
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500342}
343
Sean Christopherson4b8fff72019-12-18 13:55:04 -0800344void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500345{
Christoffer Dallf1d72312018-01-25 18:32:29 +0100346 if (vcpu->arch.has_run_once && unlikely(!irqchip_in_kernel(vcpu->kvm)))
347 static_branch_dec(&userspace_irqchip_in_use);
348
Will Deacon9af3e082020-09-11 14:25:09 +0100349 kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_cache);
Marc Zyngier967f8422013-01-23 13:21:59 -0500350 kvm_timer_vcpu_terminate(vcpu);
Shannon Zhao5f0a7142015-09-11 15:18:05 +0800351 kvm_pmu_vcpu_destroy(vcpu);
Sean Christopherson19bcc892019-12-18 13:55:27 -0800352
353 kvm_arm_vcpu_destroy(vcpu);
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500354}
355
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500356int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
357{
Christoffer Dall1c88ab72017-01-06 16:07:48 +0100358 return kvm_timer_is_pending(vcpu);
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500359}
360
Christoffer Dalld35268d2015-08-25 19:48:21 +0200361void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu)
362{
Sean Christopherson6109c5a2021-10-08 19:12:03 -0700363
Christoffer Dalld35268d2015-08-25 19:48:21 +0200364}
365
366void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu)
367{
Sean Christopherson6109c5a2021-10-08 19:12:03 -0700368
Christoffer Dalld35268d2015-08-25 19:48:21 +0200369}
370
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500371void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
372{
Christoffer Dalla0e50aa2019-01-04 21:09:05 +0100373 struct kvm_s2_mmu *mmu;
Marc Zyngier94d0e592016-10-18 18:37:49 +0100374 int *last_ran;
375
Christoffer Dalla0e50aa2019-01-04 21:09:05 +0100376 mmu = vcpu->arch.hw_mmu;
377 last_ran = this_cpu_ptr(mmu->last_vcpu_ran);
Marc Zyngier94d0e592016-10-18 18:37:49 +0100378
379 /*
Marc Zyngier01dc9262021-03-03 16:45:05 +0000380 * We guarantee that both TLBs and I-cache are private to each
381 * vcpu. If detecting that a vcpu from the same VM has
382 * previously run on the same physical CPU, call into the
383 * hypervisor code to nuke the relevant contexts.
384 *
Marc Zyngier94d0e592016-10-18 18:37:49 +0100385 * We might get preempted before the vCPU actually runs, but
386 * over-invalidation doesn't affect correctness.
387 */
388 if (*last_ran != vcpu->vcpu_id) {
Marc Zyngier01dc9262021-03-03 16:45:05 +0000389 kvm_call_hyp(__kvm_flush_cpu_context, mmu);
Marc Zyngier94d0e592016-10-18 18:37:49 +0100390 *last_ran = vcpu->vcpu_id;
391 }
392
Christoffer Dall86ce8532013-01-20 18:28:08 -0500393 vcpu->cpu = cpu;
Christoffer Dall5b3e5e52013-01-20 18:28:09 -0500394
Christoffer Dall328e5662016-03-24 11:21:04 +0100395 kvm_vgic_load(vcpu);
Christoffer Dallb103cc32016-10-16 20:30:38 +0200396 kvm_timer_vcpu_load(vcpu);
David Brazdil13aeb9b2020-06-25 14:14:16 +0100397 if (has_vhe())
398 kvm_vcpu_load_sysregs_vhe(vcpu);
Dave Martine6b673b2018-04-06 14:55:59 +0100399 kvm_arch_vcpu_load_fp(vcpu);
Andrew Murray435e53f2019-04-09 20:22:15 +0100400 kvm_vcpu_pmu_restore_guest(vcpu);
Steven Price8564d632019-10-21 16:28:18 +0100401 if (kvm_arm_is_pvtime_enabled(&vcpu->arch))
402 kvm_make_request(KVM_REQ_RECORD_STEAL, vcpu);
Marc Zyngierde737082018-06-21 10:43:59 +0100403
404 if (single_task_running())
Marc Zyngieref2e78d2019-11-07 16:04:12 +0000405 vcpu_clear_wfx_traps(vcpu);
Marc Zyngierde737082018-06-21 10:43:59 +0100406 else
Marc Zyngieref2e78d2019-11-07 16:04:12 +0000407 vcpu_set_wfx_traps(vcpu);
Mark Rutland384b40c2019-04-23 10:12:35 +0530408
Marc Zyngier29eb5a32020-06-04 11:14:00 +0100409 if (vcpu_has_ptrauth(vcpu))
Marc Zyngieref3e40a2020-06-03 18:24:01 +0100410 vcpu_ptrauth_disable(vcpu);
Suzuki K Poulosed2602bb2021-04-05 17:42:53 +0100411 kvm_arch_vcpu_load_debug_state_flags(vcpu);
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500412}
413
414void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
415{
Suzuki K Poulosed2602bb2021-04-05 17:42:53 +0100416 kvm_arch_vcpu_put_debug_state_flags(vcpu);
Dave Martine6b673b2018-04-06 14:55:59 +0100417 kvm_arch_vcpu_put_fp(vcpu);
David Brazdil13aeb9b2020-06-25 14:14:16 +0100418 if (has_vhe())
419 kvm_vcpu_put_sysregs_vhe(vcpu);
Christoffer Dallb103cc32016-10-16 20:30:38 +0200420 kvm_timer_vcpu_put(vcpu);
Christoffer Dall328e5662016-03-24 11:21:04 +0100421 kvm_vgic_put(vcpu);
Andrew Murray435e53f2019-04-09 20:22:15 +0100422 kvm_vcpu_pmu_restore_host(vcpu);
Christoffer Dall328e5662016-03-24 11:21:04 +0100423
Christoffer Dalle9b152c2013-12-11 20:29:11 -0800424 vcpu->cpu = -1;
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500425}
426
Andrew Jones424c9892017-06-04 14:43:57 +0200427static void vcpu_power_off(struct kvm_vcpu *vcpu)
428{
429 vcpu->arch.power_off = true;
Andrew Jones7b244e22017-06-04 14:43:58 +0200430 kvm_make_request(KVM_REQ_SLEEP, vcpu);
Andrew Jones424c9892017-06-04 14:43:57 +0200431 kvm_vcpu_kick(vcpu);
432}
433
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500434int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
435 struct kvm_mp_state *mp_state)
436{
Eric Auger37815282015-09-25 23:41:14 +0200437 if (vcpu->arch.power_off)
Alex Bennéeecccf0c2015-03-13 17:02:52 +0000438 mp_state->mp_state = KVM_MP_STATE_STOPPED;
439 else
440 mp_state->mp_state = KVM_MP_STATE_RUNNABLE;
441
442 return 0;
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500443}
444
445int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
446 struct kvm_mp_state *mp_state)
447{
Christoffer Dalle83dff52017-12-04 21:35:31 +0100448 int ret = 0;
449
Alex Bennéeecccf0c2015-03-13 17:02:52 +0000450 switch (mp_state->mp_state) {
451 case KVM_MP_STATE_RUNNABLE:
Eric Auger37815282015-09-25 23:41:14 +0200452 vcpu->arch.power_off = false;
Alex Bennéeecccf0c2015-03-13 17:02:52 +0000453 break;
454 case KVM_MP_STATE_STOPPED:
Andrew Jones424c9892017-06-04 14:43:57 +0200455 vcpu_power_off(vcpu);
Alex Bennéeecccf0c2015-03-13 17:02:52 +0000456 break;
457 default:
Christoffer Dalle83dff52017-12-04 21:35:31 +0100458 ret = -EINVAL;
Alex Bennéeecccf0c2015-03-13 17:02:52 +0000459 }
460
Christoffer Dalle83dff52017-12-04 21:35:31 +0100461 return ret;
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500462}
463
Christoffer Dall5b3e5e52013-01-20 18:28:09 -0500464/**
465 * kvm_arch_vcpu_runnable - determine if the vcpu can be scheduled
466 * @v: The VCPU pointer
467 *
468 * If the guest CPU is not waiting for interrupts or an interrupt line is
469 * asserted, the CPU is by definition runnable.
470 */
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500471int kvm_arch_vcpu_runnable(struct kvm_vcpu *v)
472{
Christoffer Dall3df59d82017-08-03 12:09:05 +0200473 bool irq_lines = *vcpu_hcr(v) & (HCR_VI | HCR_VF);
474 return ((irq_lines || kvm_vgic_vcpu_pending_irq(v))
Eric Auger3b928302015-09-25 23:41:17 +0200475 && !v->arch.power_off && !v->arch.pause);
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500476}
477
Longpeng(Mike)199b5762017-08-08 12:05:32 +0800478bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
479{
Longpeng(Mike)f01fbd22017-08-08 12:05:35 +0800480 return vcpu_mode_priv(vcpu);
Longpeng(Mike)199b5762017-08-08 12:05:32 +0800481}
482
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500483/* Just ensure a guest exit from a particular CPU */
484static void exit_vm_noop(void *info)
485{
486}
487
488void force_vm_exit(const cpumask_t *mask)
489{
Eric Auger898f9492016-03-07 23:50:36 +0700490 preempt_disable();
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500491 smp_call_function_many(mask, exit_vm_noop, NULL, true);
Eric Auger898f9492016-03-07 23:50:36 +0700492 preempt_enable();
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500493}
494
495/**
496 * need_new_vmid_gen - check that the VMID is still valid
Christoffer Dalle329fb72018-12-11 15:26:31 +0100497 * @vmid: The VMID to check
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500498 *
499 * return true if there is a new generation of VMIDs being used
500 *
Christoffer Dalle329fb72018-12-11 15:26:31 +0100501 * The hardware supports a limited set of values with the value zero reserved
502 * for the host, so we check if an assigned value belongs to a previous
Fuad Tabba656012c2020-04-01 15:03:10 +0100503 * generation, which requires us to assign a new value. If we're the first to
504 * use a VMID for the new generation, we must flush necessary caches and TLBs
505 * on all CPUs.
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500506 */
Christoffer Dalle329fb72018-12-11 15:26:31 +0100507static bool need_new_vmid_gen(struct kvm_vmid *vmid)
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500508{
Christoffer Dallfb544d12018-12-11 13:23:57 +0100509 u64 current_vmid_gen = atomic64_read(&kvm_vmid_gen);
510 smp_rmb(); /* Orders read of kvm_vmid_gen and kvm->arch.vmid */
Christoffer Dalle329fb72018-12-11 15:26:31 +0100511 return unlikely(READ_ONCE(vmid->vmid_gen) != current_vmid_gen);
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500512}
513
514/**
Christoffer Dalle329fb72018-12-11 15:26:31 +0100515 * update_vmid - Update the vmid with a valid VMID for the current generation
Christoffer Dalle329fb72018-12-11 15:26:31 +0100516 * @vmid: The stage-2 VMID information struct
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500517 */
Christoffer Dalle329fb72018-12-11 15:26:31 +0100518static void update_vmid(struct kvm_vmid *vmid)
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500519{
Christoffer Dalle329fb72018-12-11 15:26:31 +0100520 if (!need_new_vmid_gen(vmid))
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500521 return;
522
Christoffer Dallfb544d12018-12-11 13:23:57 +0100523 spin_lock(&kvm_vmid_lock);
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500524
525 /*
526 * We need to re-check the vmid_gen here to ensure that if another vcpu
527 * already allocated a valid vmid for this vm, then this vcpu should
528 * use the same vmid.
529 */
Christoffer Dalle329fb72018-12-11 15:26:31 +0100530 if (!need_new_vmid_gen(vmid)) {
Christoffer Dallfb544d12018-12-11 13:23:57 +0100531 spin_unlock(&kvm_vmid_lock);
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500532 return;
533 }
534
535 /* First user of a new VMID generation? */
536 if (unlikely(kvm_next_vmid == 0)) {
537 atomic64_inc(&kvm_vmid_gen);
538 kvm_next_vmid = 1;
539
540 /*
541 * On SMP we know no other CPUs can use this CPU's or each
542 * other's VMID after force_vm_exit returns since the
543 * kvm_vmid_lock blocks them from reentry to the guest.
544 */
545 force_vm_exit(cpu_all_mask);
546 /*
547 * Now broadcast TLB + ICACHE invalidation over the inner
548 * shareable domain to make sure all data structures are
549 * clean.
550 */
551 kvm_call_hyp(__kvm_flush_vm_context);
552 }
553
Marc Zyngiercf364e02021-08-06 12:31:08 +0100554 WRITE_ONCE(vmid->vmid, kvm_next_vmid);
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500555 kvm_next_vmid++;
Christoffer Dalle329fb72018-12-11 15:26:31 +0100556 kvm_next_vmid &= (1 << kvm_get_vmid_bits()) - 1;
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500557
Christoffer Dallfb544d12018-12-11 13:23:57 +0100558 smp_wmb();
Christoffer Dalle329fb72018-12-11 15:26:31 +0100559 WRITE_ONCE(vmid->vmid_gen, atomic64_read(&kvm_vmid_gen));
Christoffer Dallfb544d12018-12-11 13:23:57 +0100560
561 spin_unlock(&kvm_vmid_lock);
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500562}
563
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500564static int kvm_vcpu_first_run_init(struct kvm_vcpu *vcpu)
565{
Christoffer Dall05971122014-12-12 21:19:23 +0100566 struct kvm *kvm = vcpu->kvm;
Christoffer Dall41a54482016-05-18 16:26:00 +0100567 int ret = 0;
Christoffer Dalle1ba0202013-09-23 14:55:55 -0700568
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500569 if (likely(vcpu->arch.has_run_once))
570 return 0;
571
Dave Martin7dd32a02018-12-19 14:27:01 +0000572 if (!kvm_arm_vcpu_is_finalized(vcpu))
573 return -EPERM;
574
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500575 vcpu->arch.has_run_once = true;
Marc Zyngieraa024c22013-01-20 18:28:13 -0500576
Alexandru Elisei263d6282021-04-07 15:48:57 +0100577 kvm_arm_vcpu_init_debug(vcpu);
578
Christoffer Dall61bbe382017-10-27 19:57:51 +0200579 if (likely(irqchip_in_kernel(kvm))) {
580 /*
581 * Map the VGIC hardware resources before running a vcpu the
582 * first time on this VM.
583 */
Alexandru Elisei1c91f062020-12-01 15:01:55 +0000584 ret = kvm_vgic_map_resources(kvm);
585 if (ret)
586 return ret;
Christoffer Dall61bbe382017-10-27 19:57:51 +0200587 } else {
588 /*
589 * Tell the rest of the code that there are userspace irqchip
590 * VMs in the wild.
591 */
592 static_branch_inc(&userspace_irqchip_in_use);
Marc Zyngier01ac5e32013-01-21 19:36:16 -0500593 }
594
Alexander Grafd9e13972016-09-27 21:08:06 +0200595 ret = kvm_timer_enable(vcpu);
Christoffer Dalla2befac2017-05-02 13:41:02 +0200596 if (ret)
597 return ret;
598
599 ret = kvm_arm_pmu_v3_enable(vcpu);
Christoffer Dall05971122014-12-12 21:19:23 +0100600
Fuad Tabba2a0c3432021-10-10 15:56:33 +0100601 /*
602 * Initialize traps for protected VMs.
603 * NOTE: Move to run in EL2 directly, rather than via a hypercall, once
604 * the code is in place for first run initialization at EL2.
605 */
606 if (kvm_vm_is_protected(kvm))
607 kvm_call_hyp_nvhe(__pkvm_vcpu_init_traps, vcpu);
608
Christoffer Dall41a54482016-05-18 16:26:00 +0100609 return ret;
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500610}
611
Eric Augerc1426e42015-03-04 11:14:34 +0100612bool kvm_arch_intc_initialized(struct kvm *kvm)
613{
614 return vgic_initialized(kvm);
615}
616
Christoffer Dallb13216c2016-04-27 10:28:00 +0100617void kvm_arm_halt_guest(struct kvm *kvm)
Eric Auger3b928302015-09-25 23:41:17 +0200618{
Marc Zyngier46808a42021-11-16 16:04:02 +0000619 unsigned long i;
Eric Auger3b928302015-09-25 23:41:17 +0200620 struct kvm_vcpu *vcpu;
621
622 kvm_for_each_vcpu(i, vcpu, kvm)
623 vcpu->arch.pause = true;
Andrew Jones7b244e22017-06-04 14:43:58 +0200624 kvm_make_all_cpus_request(kvm, KVM_REQ_SLEEP);
Eric Auger3b928302015-09-25 23:41:17 +0200625}
626
Christoffer Dallb13216c2016-04-27 10:28:00 +0100627void kvm_arm_resume_guest(struct kvm *kvm)
Eric Auger3b928302015-09-25 23:41:17 +0200628{
Marc Zyngier46808a42021-11-16 16:04:02 +0000629 unsigned long i;
Eric Auger3b928302015-09-25 23:41:17 +0200630 struct kvm_vcpu *vcpu;
631
Christoffer Dallabd72292017-05-06 20:01:24 +0200632 kvm_for_each_vcpu(i, vcpu, kvm) {
633 vcpu->arch.pause = false;
Davidlohr Buesoda4ad882020-04-23 22:48:37 -0700634 rcuwait_wake_up(kvm_arch_vcpu_get_wait(vcpu));
Christoffer Dallabd72292017-05-06 20:01:24 +0200635 }
Eric Auger3b928302015-09-25 23:41:17 +0200636}
637
Andrew Jones7b244e22017-06-04 14:43:58 +0200638static void vcpu_req_sleep(struct kvm_vcpu *vcpu)
Marc Zyngieraa024c22013-01-20 18:28:13 -0500639{
Davidlohr Buesoda4ad882020-04-23 22:48:37 -0700640 struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu);
Marc Zyngieraa024c22013-01-20 18:28:13 -0500641
Davidlohr Buesoda4ad882020-04-23 22:48:37 -0700642 rcuwait_wait_event(wait,
643 (!vcpu->arch.power_off) &&(!vcpu->arch.pause),
644 TASK_INTERRUPTIBLE);
Andrew Jones0592c002017-06-04 14:43:55 +0200645
Andrew Jones424c9892017-06-04 14:43:57 +0200646 if (vcpu->arch.power_off || vcpu->arch.pause) {
Andrew Jones0592c002017-06-04 14:43:55 +0200647 /* Awaken to handle a signal, request we sleep again later. */
Andrew Jones7b244e22017-06-04 14:43:58 +0200648 kvm_make_request(KVM_REQ_SLEEP, vcpu);
Andrew Jones0592c002017-06-04 14:43:55 +0200649 }
Marc Zyngier358b28f2018-12-20 11:36:07 +0000650
651 /*
652 * Make sure we will observe a potential reset request if we've
653 * observed a change to the power state. Pairs with the smp_wmb() in
654 * kvm_psci_vcpu_on().
655 */
656 smp_rmb();
Marc Zyngieraa024c22013-01-20 18:28:13 -0500657}
658
Sean Christopherson6109c5a2021-10-08 19:12:03 -0700659/**
660 * kvm_vcpu_wfi - emulate Wait-For-Interrupt behavior
661 * @vcpu: The VCPU pointer
662 *
663 * Suspend execution of a vCPU until a valid wake event is detected, i.e. until
664 * the vCPU is runnable. The vCPU may or may not be scheduled out, depending
665 * on when a wake event arrives, e.g. there may already be a pending wake event.
666 */
667void kvm_vcpu_wfi(struct kvm_vcpu *vcpu)
668{
669 /*
670 * Sync back the state of the GIC CPU interface so that we have
671 * the latest PMR and group enables. This ensures that
672 * kvm_arch_vcpu_runnable has up-to-date data to decide whether
673 * we have pending interrupts, e.g. when determining if the
674 * vCPU should block.
675 *
676 * For the same reason, we want to tell GICv4 that we need
677 * doorbells to be signalled, should an interrupt become pending.
678 */
679 preempt_disable();
680 kvm_vgic_vmcr_sync(vcpu);
681 vgic_v4_put(vcpu, true);
682 preempt_enable();
683
Sean Christopherson91b99ea2021-10-08 19:12:06 -0700684 kvm_vcpu_halt(vcpu);
Sean Christopherson6109c5a2021-10-08 19:12:03 -0700685 kvm_clear_request(KVM_REQ_UNHALT, vcpu);
686
687 preempt_disable();
688 vgic_v4_load(vcpu);
689 preempt_enable();
690}
691
Andre Przywarae8180dc2013-05-09 00:28:06 +0200692static int kvm_vcpu_initialized(struct kvm_vcpu *vcpu)
693{
694 return vcpu->arch.target >= 0;
695}
696
Andrew Jones0592c002017-06-04 14:43:55 +0200697static void check_vcpu_requests(struct kvm_vcpu *vcpu)
698{
699 if (kvm_request_pending(vcpu)) {
Andrew Jones7b244e22017-06-04 14:43:58 +0200700 if (kvm_check_request(KVM_REQ_SLEEP, vcpu))
701 vcpu_req_sleep(vcpu);
Andrew Jones325f9c62017-06-04 14:43:59 +0200702
Marc Zyngier358b28f2018-12-20 11:36:07 +0000703 if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu))
704 kvm_reset_vcpu(vcpu);
705
Andrew Jones325f9c62017-06-04 14:43:59 +0200706 /*
707 * Clear IRQ_PENDING requests that were made to guarantee
708 * that a VCPU sees new virtual interrupts.
709 */
710 kvm_check_request(KVM_REQ_IRQ_PENDING, vcpu);
Steven Price8564d632019-10-21 16:28:18 +0100711
712 if (kvm_check_request(KVM_REQ_RECORD_STEAL, vcpu))
713 kvm_update_stolen_time(vcpu);
Marc Zyngierd9c38722020-03-04 20:33:28 +0000714
715 if (kvm_check_request(KVM_REQ_RELOAD_GICv4, vcpu)) {
716 /* The distributor enable bits were changed */
717 preempt_disable();
718 vgic_v4_put(vcpu, false);
719 vgic_v4_load(vcpu);
720 preempt_enable();
721 }
Marc Zyngierd0c94c42021-06-03 16:50:02 +0100722
723 if (kvm_check_request(KVM_REQ_RELOAD_PMU, vcpu))
724 kvm_pmu_handle_pmcr(vcpu,
725 __vcpu_sys_reg(vcpu, PMCR_EL0));
Andrew Jones0592c002017-06-04 14:43:55 +0200726 }
727}
728
Will Deacon2f6a49b2021-06-08 19:02:56 +0100729static bool vcpu_mode_is_bad_32bit(struct kvm_vcpu *vcpu)
730{
731 if (likely(!vcpu_mode_is_32bit(vcpu)))
732 return false;
733
734 return !system_supports_32bit_el0() ||
735 static_branch_unlikely(&arm64_mismatched_32bit_el0);
736}
737
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500738/**
Oliver Upton6caa5812021-08-02 19:28:09 +0000739 * kvm_vcpu_exit_request - returns true if the VCPU should *not* enter the guest
740 * @vcpu: The VCPU pointer
741 * @ret: Pointer to write optional return code
742 *
743 * Returns: true if the VCPU needs to return to a preemptible + interruptible
744 * and skip guest entry.
745 *
746 * This function disambiguates between two different types of exits: exits to a
747 * preemptible + interruptible kernel context and exits to userspace. For an
748 * exit to userspace, this function will write the return code to ret and return
749 * true. For an exit to preemptible + interruptible kernel context (i.e. check
750 * for pending work and re-enter), return true without writing to ret.
751 */
752static bool kvm_vcpu_exit_request(struct kvm_vcpu *vcpu, int *ret)
753{
754 struct kvm_run *run = vcpu->run;
755
756 /*
757 * If we're using a userspace irqchip, then check if we need
758 * to tell a userspace irqchip about timer or PMU level
759 * changes and if so, exit to userspace (the actual level
760 * state gets updated in kvm_timer_update_run and
761 * kvm_pmu_update_run below).
762 */
763 if (static_branch_unlikely(&userspace_irqchip_in_use)) {
764 if (kvm_timer_should_notify_user(vcpu) ||
765 kvm_pmu_should_notify_user(vcpu)) {
766 *ret = -EINTR;
767 run->exit_reason = KVM_EXIT_INTR;
768 return true;
769 }
770 }
771
772 return kvm_request_pending(vcpu) ||
773 need_new_vmid_gen(&vcpu->arch.hw_mmu->vmid) ||
774 xfer_to_guest_mode_work_pending();
775}
776
777/**
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500778 * kvm_arch_vcpu_ioctl_run - the main VCPU run function to execute guest code
779 * @vcpu: The VCPU pointer
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500780 *
781 * This function is called through the VCPU_RUN ioctl called from user space. It
782 * will execute VM code in a loop until the time slice for the process is used
783 * or some emulation is needed from user space in which case the function will
784 * return with return value 0 and with the kvm_run structure filled in with the
785 * required data for the requested emulation.
786 */
Tianjia Zhang1b94f6f2020-04-16 13:10:57 +0800787int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500788{
Tianjia Zhang1b94f6f2020-04-16 13:10:57 +0800789 struct kvm_run *run = vcpu->run;
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500790 int ret;
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500791
Andre Przywarae8180dc2013-05-09 00:28:06 +0200792 if (unlikely(!kvm_vcpu_initialized(vcpu)))
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500793 return -ENOEXEC;
794
795 ret = kvm_vcpu_first_run_init(vcpu);
796 if (ret)
Christoffer Dall829a5862017-11-29 16:37:53 +0100797 return ret;
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500798
Christoffer Dall45e96ea2013-01-20 18:43:58 -0500799 if (run->exit_reason == KVM_EXIT_MMIO) {
Tianjia Zhang74cc7e02020-06-23 21:14:15 +0800800 ret = kvm_handle_mmio_return(vcpu);
Christoffer Dall45e96ea2013-01-20 18:43:58 -0500801 if (ret)
Christoffer Dall829a5862017-11-29 16:37:53 +0100802 return ret;
Christoffer Dall45e96ea2013-01-20 18:43:58 -0500803 }
804
Christoffer Dall829a5862017-11-29 16:37:53 +0100805 vcpu_load(vcpu);
Paolo Bonzini460df4c2017-02-08 11:50:15 +0100806
Zenghui Yue3e880b2021-05-26 22:18:31 +0800807 if (run->immediate_exit) {
808 ret = -EINTR;
809 goto out;
810 }
811
Jan H. Schönherr20b70352017-11-24 22:39:01 +0100812 kvm_sigset_activate(vcpu);
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500813
814 ret = 1;
815 run->exit_reason = KVM_EXIT_UNKNOWN;
816 while (ret > 0) {
817 /*
818 * Check conditions before entering the guest
819 */
Oliver Upton6caa5812021-08-02 19:28:09 +0000820 ret = xfer_to_guest_mode_handle_work(vcpu);
821 if (!ret)
822 ret = 1;
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500823
Christoffer Dalla0e50aa2019-01-04 21:09:05 +0100824 update_vmid(&vcpu->arch.hw_mmu->vmid);
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500825
Andrew Jones0592c002017-06-04 14:43:55 +0200826 check_vcpu_requests(vcpu);
827
Marc Zyngierabdf5842015-06-08 15:00:28 +0100828 /*
Marc Zyngierabdf5842015-06-08 15:00:28 +0100829 * Preparing the interrupts to be injected also
830 * involves poking the GIC, which must be done in a
831 * non-preemptible context.
832 */
833 preempt_disable();
Christoffer Dall328e5662016-03-24 11:21:04 +0100834
Shannon Zhaob02386e2016-02-26 19:29:19 +0800835 kvm_pmu_flush_hwstate(vcpu);
Christoffer Dall328e5662016-03-24 11:21:04 +0100836
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500837 local_irq_disable();
838
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500839 kvm_vgic_flush_hwstate(vcpu);
Christoffer Dall5b3e5e52013-01-20 18:28:09 -0500840
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500841 /*
Andrew Jones6a6d73b2017-06-04 14:43:54 +0200842 * Ensure we set mode to IN_GUEST_MODE after we disable
843 * interrupts and before the final VCPU requests check.
844 * See the comment in kvm_vcpu_exiting_guest_mode() and
Christoph Hellwig2f5947d2019-07-24 09:24:49 +0200845 * Documentation/virt/kvm/vcpu-requests.rst
Andrew Jones6a6d73b2017-06-04 14:43:54 +0200846 */
847 smp_store_mb(vcpu->mode, IN_GUEST_MODE);
848
Oliver Upton6caa5812021-08-02 19:28:09 +0000849 if (ret <= 0 || kvm_vcpu_exit_request(vcpu, &ret)) {
Andrew Jones6a6d73b2017-06-04 14:43:54 +0200850 vcpu->mode = OUTSIDE_GUEST_MODE;
Christoffer Dall771621b2017-10-04 23:42:32 +0200851 isb(); /* Ensure work in x_flush_hwstate is committed */
Shannon Zhaob02386e2016-02-26 19:29:19 +0800852 kvm_pmu_sync_hwstate(vcpu);
Christoffer Dall61bbe382017-10-27 19:57:51 +0200853 if (static_branch_unlikely(&userspace_irqchip_in_use))
Marc Zyngier3c5ff0c2020-04-22 08:58:22 +0100854 kvm_timer_sync_user(vcpu);
Marc Zyngier1a89dd92013-01-21 19:36:12 -0500855 kvm_vgic_sync_hwstate(vcpu);
Christoffer Dallee9bb9a2016-10-16 20:24:30 +0200856 local_irq_enable();
Marc Zyngierabdf5842015-06-08 15:00:28 +0100857 preempt_enable();
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500858 continue;
859 }
860
Alex Bennée56c7f5e2015-07-07 17:29:56 +0100861 kvm_arm_setup_debug(vcpu);
862
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500863 /**************************************************************
864 * Enter the guest
865 */
866 trace_kvm_entry(*vcpu_pc(vcpu));
Paolo Bonzini6edaa532016-06-15 15:18:26 +0200867 guest_enter_irqoff();
Christoffer Dall3f5c90b2017-10-03 14:02:12 +0200868
David Brazdil09cf57e2020-06-25 14:14:14 +0100869 ret = kvm_call_hyp_ret(__kvm_vcpu_run, vcpu);
Christoffer Dall3f5c90b2017-10-03 14:02:12 +0200870
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500871 vcpu->mode = OUTSIDE_GUEST_MODE;
Amit Tomarb19e6892015-11-26 10:09:43 +0000872 vcpu->stat.exits++;
Christoffer Dall1b3d5462015-05-28 19:49:10 +0100873 /*
874 * Back from guest
875 *************************************************************/
876
Alex Bennée56c7f5e2015-07-07 17:29:56 +0100877 kvm_arm_clear_debug(vcpu);
878
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500879 /*
Christoffer Dallb103cc32016-10-16 20:30:38 +0200880 * We must sync the PMU state before the vgic state so
Christoffer Dallee9bb9a2016-10-16 20:24:30 +0200881 * that the vgic can properly sample the updated state of the
882 * interrupt line.
883 */
884 kvm_pmu_sync_hwstate(vcpu);
Christoffer Dallee9bb9a2016-10-16 20:24:30 +0200885
Christoffer Dallb103cc32016-10-16 20:30:38 +0200886 /*
887 * Sync the vgic state before syncing the timer state because
888 * the timer code needs to know if the virtual timer
889 * interrupts are active.
890 */
Christoffer Dallee9bb9a2016-10-16 20:24:30 +0200891 kvm_vgic_sync_hwstate(vcpu);
892
893 /*
Christoffer Dallb103cc32016-10-16 20:30:38 +0200894 * Sync the timer hardware state before enabling interrupts as
895 * we don't want vtimer interrupts to race with syncing the
896 * timer virtual interrupt state.
897 */
Christoffer Dall61bbe382017-10-27 19:57:51 +0200898 if (static_branch_unlikely(&userspace_irqchip_in_use))
Marc Zyngier3c5ff0c2020-04-22 08:58:22 +0100899 kvm_timer_sync_user(vcpu);
Christoffer Dallb103cc32016-10-16 20:30:38 +0200900
Dave Martine6b673b2018-04-06 14:55:59 +0100901 kvm_arch_vcpu_ctxsync_fp(vcpu);
902
Christoffer Dallb103cc32016-10-16 20:30:38 +0200903 /*
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500904 * We may have taken a host interrupt in HYP mode (ie
905 * while executing the guest). This interrupt is still
906 * pending, as we haven't serviced it yet!
907 *
908 * We're now back in SVC mode, with interrupts
909 * disabled. Enabling the interrupts now will have
910 * the effect of taking the interrupt again, in SVC
911 * mode this time.
912 */
913 local_irq_enable();
914
915 /*
Paolo Bonzini6edaa532016-06-15 15:18:26 +0200916 * We do local_irq_enable() before calling guest_exit() so
Christoffer Dall1b3d5462015-05-28 19:49:10 +0100917 * that if a timer interrupt hits while running the guest we
918 * account that tick as being spent in the guest. We enable
Paolo Bonzini6edaa532016-06-15 15:18:26 +0200919 * preemption after calling guest_exit() so that if we get
Christoffer Dall1b3d5462015-05-28 19:49:10 +0100920 * preempted we make sure ticks after that is not counted as
921 * guest time.
922 */
Paolo Bonzini6edaa532016-06-15 15:18:26 +0200923 guest_exit();
Christoffer Dallb5905dc2015-08-30 15:55:22 +0200924 trace_kvm_exit(ret, kvm_vcpu_trap_get_class(vcpu), *vcpu_pc(vcpu));
Christoffer Dall1b3d5462015-05-28 19:49:10 +0100925
James Morse3368bd82018-01-15 19:39:04 +0000926 /* Exit types that need handling before we can be preempted */
Tianjia Zhang74cc7e02020-06-23 21:14:15 +0800927 handle_exit_early(vcpu, ret);
James Morse3368bd82018-01-15 19:39:04 +0000928
Marc Zyngierabdf5842015-06-08 15:00:28 +0100929 preempt_enable();
930
Qais Yousef22f55382020-10-27 21:51:13 +0000931 /*
932 * The ARMv8 architecture doesn't give the hypervisor
933 * a mechanism to prevent a guest from dropping to AArch32 EL0
934 * if implemented by the CPU. If we spot the guest in such
935 * state and that we decided it wasn't supposed to do so (like
936 * with the asymmetric AArch32 case), return to userspace with
937 * a fatal error.
938 */
Will Deacon2f6a49b2021-06-08 19:02:56 +0100939 if (vcpu_mode_is_bad_32bit(vcpu)) {
Qais Yousef22f55382020-10-27 21:51:13 +0000940 /*
941 * As we have caught the guest red-handed, decide that
942 * it isn't fit for purpose anymore by making the vcpu
943 * invalid. The VMM can try and fix it by issuing a
944 * KVM_ARM_VCPU_INIT if it really wants to.
945 */
946 vcpu->arch.target = -1;
947 ret = ARM_EXCEPTION_IL;
948 }
949
Tianjia Zhang74cc7e02020-06-23 21:14:15 +0800950 ret = handle_exit(vcpu, ret);
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500951 }
952
Alexander Grafd9e13972016-09-27 21:08:06 +0200953 /* Tell userspace about in-kernel device output levels */
Christoffer Dall3dbbdf72017-02-01 12:51:52 +0100954 if (unlikely(!irqchip_in_kernel(vcpu->kvm))) {
955 kvm_timer_update_run(vcpu);
956 kvm_pmu_update_run(vcpu);
957 }
Alexander Grafd9e13972016-09-27 21:08:06 +0200958
Jan H. Schönherr20b70352017-11-24 22:39:01 +0100959 kvm_sigset_deactivate(vcpu);
960
Zenghui Yue3e880b2021-05-26 22:18:31 +0800961out:
Marc Zyngier26778aa2021-05-06 15:20:12 +0100962 /*
963 * In the unlikely event that we are returning to userspace
964 * with pending exceptions or PC adjustment, commit these
965 * adjustments in order to give userspace a consistent view of
966 * the vcpu state. Note that this relies on __kvm_adjust_pc()
967 * being preempt-safe on VHE.
968 */
969 if (unlikely(vcpu->arch.flags & (KVM_ARM64_PENDING_EXCEPTION |
970 KVM_ARM64_INCREMENT_PC)))
971 kvm_call_hyp(__kvm_adjust_pc, vcpu);
972
Christoffer Dallaccb7572017-12-04 21:35:25 +0100973 vcpu_put(vcpu);
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500974 return ret;
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500975}
976
Christoffer Dall86ce8532013-01-20 18:28:08 -0500977static int vcpu_interrupt_line(struct kvm_vcpu *vcpu, int number, bool level)
978{
979 int bit_index;
980 bool set;
Christoffer Dall3df59d82017-08-03 12:09:05 +0200981 unsigned long *hcr;
Christoffer Dall86ce8532013-01-20 18:28:08 -0500982
983 if (number == KVM_ARM_IRQ_CPU_IRQ)
984 bit_index = __ffs(HCR_VI);
985 else /* KVM_ARM_IRQ_CPU_FIQ */
986 bit_index = __ffs(HCR_VF);
987
Christoffer Dall3df59d82017-08-03 12:09:05 +0200988 hcr = vcpu_hcr(vcpu);
Christoffer Dall86ce8532013-01-20 18:28:08 -0500989 if (level)
Christoffer Dall3df59d82017-08-03 12:09:05 +0200990 set = test_and_set_bit(bit_index, hcr);
Christoffer Dall86ce8532013-01-20 18:28:08 -0500991 else
Christoffer Dall3df59d82017-08-03 12:09:05 +0200992 set = test_and_clear_bit(bit_index, hcr);
Christoffer Dall86ce8532013-01-20 18:28:08 -0500993
994 /*
995 * If we didn't change anything, no need to wake up or kick other CPUs
996 */
997 if (set == level)
998 return 0;
999
1000 /*
1001 * The vcpu irq_lines field was updated, wake up sleeping VCPUs and
1002 * trigger a world-switch round on the running physical CPU to set the
1003 * virtual IRQ/FIQ fields in the HCR appropriately.
1004 */
Andrew Jones325f9c62017-06-04 14:43:59 +02001005 kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu);
Christoffer Dall86ce8532013-01-20 18:28:08 -05001006 kvm_vcpu_kick(vcpu);
1007
1008 return 0;
1009}
1010
Alexander Graf79558f12013-04-16 19:21:41 +02001011int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level,
1012 bool line_status)
Christoffer Dall86ce8532013-01-20 18:28:08 -05001013{
1014 u32 irq = irq_level->irq;
1015 unsigned int irq_type, vcpu_idx, irq_num;
1016 int nrcpus = atomic_read(&kvm->online_vcpus);
1017 struct kvm_vcpu *vcpu = NULL;
1018 bool level = irq_level->level;
1019
1020 irq_type = (irq >> KVM_ARM_IRQ_TYPE_SHIFT) & KVM_ARM_IRQ_TYPE_MASK;
1021 vcpu_idx = (irq >> KVM_ARM_IRQ_VCPU_SHIFT) & KVM_ARM_IRQ_VCPU_MASK;
Marc Zyngier92f35b72019-08-18 14:09:47 +01001022 vcpu_idx += ((irq >> KVM_ARM_IRQ_VCPU2_SHIFT) & KVM_ARM_IRQ_VCPU2_MASK) * (KVM_ARM_IRQ_VCPU_MASK + 1);
Christoffer Dall86ce8532013-01-20 18:28:08 -05001023 irq_num = (irq >> KVM_ARM_IRQ_NUM_SHIFT) & KVM_ARM_IRQ_NUM_MASK;
1024
1025 trace_kvm_irq_line(irq_type, vcpu_idx, irq_num, irq_level->level);
1026
Marc Zyngier5863c2c2013-01-21 19:36:15 -05001027 switch (irq_type) {
1028 case KVM_ARM_IRQ_TYPE_CPU:
1029 if (irqchip_in_kernel(kvm))
1030 return -ENXIO;
Christoffer Dall86ce8532013-01-20 18:28:08 -05001031
Marc Zyngier5863c2c2013-01-21 19:36:15 -05001032 if (vcpu_idx >= nrcpus)
1033 return -EINVAL;
Christoffer Dall86ce8532013-01-20 18:28:08 -05001034
Marc Zyngier5863c2c2013-01-21 19:36:15 -05001035 vcpu = kvm_get_vcpu(kvm, vcpu_idx);
1036 if (!vcpu)
1037 return -EINVAL;
Christoffer Dall86ce8532013-01-20 18:28:08 -05001038
Marc Zyngier5863c2c2013-01-21 19:36:15 -05001039 if (irq_num > KVM_ARM_IRQ_CPU_FIQ)
1040 return -EINVAL;
Christoffer Dall86ce8532013-01-20 18:28:08 -05001041
Marc Zyngier5863c2c2013-01-21 19:36:15 -05001042 return vcpu_interrupt_line(vcpu, irq_num, level);
1043 case KVM_ARM_IRQ_TYPE_PPI:
1044 if (!irqchip_in_kernel(kvm))
1045 return -ENXIO;
1046
1047 if (vcpu_idx >= nrcpus)
1048 return -EINVAL;
1049
1050 vcpu = kvm_get_vcpu(kvm, vcpu_idx);
1051 if (!vcpu)
1052 return -EINVAL;
1053
1054 if (irq_num < VGIC_NR_SGIS || irq_num >= VGIC_NR_PRIVATE_IRQS)
1055 return -EINVAL;
1056
Christoffer Dallcb3f0ad2017-05-16 12:41:18 +02001057 return kvm_vgic_inject_irq(kvm, vcpu->vcpu_id, irq_num, level, NULL);
Marc Zyngier5863c2c2013-01-21 19:36:15 -05001058 case KVM_ARM_IRQ_TYPE_SPI:
1059 if (!irqchip_in_kernel(kvm))
1060 return -ENXIO;
1061
Andre Przywarafd1d0dd2015-04-10 16:17:59 +01001062 if (irq_num < VGIC_NR_PRIVATE_IRQS)
Marc Zyngier5863c2c2013-01-21 19:36:15 -05001063 return -EINVAL;
1064
Christoffer Dallcb3f0ad2017-05-16 12:41:18 +02001065 return kvm_vgic_inject_irq(kvm, 0, irq_num, level, NULL);
Marc Zyngier5863c2c2013-01-21 19:36:15 -05001066 }
1067
1068 return -EINVAL;
Christoffer Dall86ce8532013-01-20 18:28:08 -05001069}
1070
Christoffer Dallf7fa034d2014-10-16 16:40:53 +02001071static int kvm_vcpu_set_target(struct kvm_vcpu *vcpu,
1072 const struct kvm_vcpu_init *init)
1073{
Andrew Jones811328f2019-04-04 19:42:30 +02001074 unsigned int i, ret;
Anshuman Khandual6b7982f2021-08-12 10:39:53 +05301075 u32 phys_target = kvm_target_cpu();
Christoffer Dallf7fa034d2014-10-16 16:40:53 +02001076
1077 if (init->target != phys_target)
1078 return -EINVAL;
1079
1080 /*
1081 * Secondary and subsequent calls to KVM_ARM_VCPU_INIT must
1082 * use the same target.
1083 */
1084 if (vcpu->arch.target != -1 && vcpu->arch.target != init->target)
1085 return -EINVAL;
1086
1087 /* -ENOENT for unknown features, -EINVAL for invalid combinations. */
1088 for (i = 0; i < sizeof(init->features) * 8; i++) {
1089 bool set = (init->features[i / 32] & (1 << (i % 32)));
1090
1091 if (set && i >= KVM_VCPU_MAX_FEATURES)
1092 return -ENOENT;
1093
1094 /*
1095 * Secondary and subsequent calls to KVM_ARM_VCPU_INIT must
1096 * use the same feature set.
1097 */
1098 if (vcpu->arch.target != -1 && i < KVM_VCPU_MAX_FEATURES &&
1099 test_bit(i, vcpu->arch.features) != set)
1100 return -EINVAL;
1101
1102 if (set)
1103 set_bit(i, vcpu->arch.features);
1104 }
1105
1106 vcpu->arch.target = phys_target;
1107
1108 /* Now we know what it is, we can reset it. */
Andrew Jones811328f2019-04-04 19:42:30 +02001109 ret = kvm_reset_vcpu(vcpu);
1110 if (ret) {
1111 vcpu->arch.target = -1;
1112 bitmap_zero(vcpu->arch.features, KVM_VCPU_MAX_FEATURES);
1113 }
Christoffer Dallf7fa034d2014-10-16 16:40:53 +02001114
Andrew Jones811328f2019-04-04 19:42:30 +02001115 return ret;
1116}
Christoffer Dallf7fa034d2014-10-16 16:40:53 +02001117
Christoffer Dall478a8232013-11-19 17:43:19 -08001118static int kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu *vcpu,
1119 struct kvm_vcpu_init *init)
1120{
1121 int ret;
1122
1123 ret = kvm_vcpu_set_target(vcpu, init);
1124 if (ret)
1125 return ret;
1126
Christoffer Dall957db102014-11-27 10:35:03 +01001127 /*
1128 * Ensure a rebooted VM will fault in RAM pages and detect if the
1129 * guest MMU is turned off and flush the caches as needed.
Zenghui Yu892713e2020-04-15 15:28:35 +08001130 *
Marc Zyngier7ae2f3d2020-05-30 17:22:19 +01001131 * S2FWB enforces all memory accesses to RAM being cacheable,
1132 * ensuring that the data side is always coherent. We still
1133 * need to invalidate the I-cache though, as FWB does *not*
1134 * imply CTR_EL0.DIC.
Christoffer Dall957db102014-11-27 10:35:03 +01001135 */
Marc Zyngier7ae2f3d2020-05-30 17:22:19 +01001136 if (vcpu->arch.has_run_once) {
1137 if (!cpus_have_final_cap(ARM64_HAS_STAGE2_FWB))
1138 stage2_unmap_vm(vcpu->kvm);
1139 else
Fuad Tabbafade9c22021-05-24 09:30:01 +01001140 icache_inval_all_pou();
Marc Zyngier7ae2f3d2020-05-30 17:22:19 +01001141 }
Christoffer Dall957db102014-11-27 10:35:03 +01001142
Christoffer Dallb856a592014-10-16 17:21:16 +02001143 vcpu_reset_hcr(vcpu);
Fuad Tabbacd496222021-08-17 09:11:27 +01001144 vcpu->arch.cptr_el2 = CPTR_EL2_DEFAULT;
Christoffer Dallb856a592014-10-16 17:21:16 +02001145
Christoffer Dall478a8232013-11-19 17:43:19 -08001146 /*
Eric Auger37815282015-09-25 23:41:14 +02001147 * Handle the "start in power-off" case.
Christoffer Dall478a8232013-11-19 17:43:19 -08001148 */
Christoffer Dall03f1d4c2014-12-02 15:27:51 +01001149 if (test_bit(KVM_ARM_VCPU_POWER_OFF, vcpu->arch.features))
Andrew Jones424c9892017-06-04 14:43:57 +02001150 vcpu_power_off(vcpu);
Christoffer Dall3ad8b3d2014-10-16 16:14:43 +02001151 else
Eric Auger37815282015-09-25 23:41:14 +02001152 vcpu->arch.power_off = false;
Christoffer Dall478a8232013-11-19 17:43:19 -08001153
1154 return 0;
1155}
1156
Shannon Zhaof577f6c2016-01-11 20:56:17 +08001157static int kvm_arm_vcpu_set_attr(struct kvm_vcpu *vcpu,
1158 struct kvm_device_attr *attr)
1159{
1160 int ret = -ENXIO;
1161
1162 switch (attr->group) {
1163 default:
Shannon Zhaobb0c70b2016-01-11 21:35:32 +08001164 ret = kvm_arm_vcpu_arch_set_attr(vcpu, attr);
Shannon Zhaof577f6c2016-01-11 20:56:17 +08001165 break;
1166 }
1167
1168 return ret;
1169}
1170
1171static int kvm_arm_vcpu_get_attr(struct kvm_vcpu *vcpu,
1172 struct kvm_device_attr *attr)
1173{
1174 int ret = -ENXIO;
1175
1176 switch (attr->group) {
1177 default:
Shannon Zhaobb0c70b2016-01-11 21:35:32 +08001178 ret = kvm_arm_vcpu_arch_get_attr(vcpu, attr);
Shannon Zhaof577f6c2016-01-11 20:56:17 +08001179 break;
1180 }
1181
1182 return ret;
1183}
1184
1185static int kvm_arm_vcpu_has_attr(struct kvm_vcpu *vcpu,
1186 struct kvm_device_attr *attr)
1187{
1188 int ret = -ENXIO;
1189
1190 switch (attr->group) {
1191 default:
Shannon Zhaobb0c70b2016-01-11 21:35:32 +08001192 ret = kvm_arm_vcpu_arch_has_attr(vcpu, attr);
Shannon Zhaof577f6c2016-01-11 20:56:17 +08001193 break;
1194 }
1195
1196 return ret;
1197}
1198
James Morse539aee02018-07-19 16:24:24 +01001199static int kvm_arm_vcpu_get_events(struct kvm_vcpu *vcpu,
1200 struct kvm_vcpu_events *events)
1201{
1202 memset(events, 0, sizeof(*events));
1203
1204 return __kvm_arm_vcpu_get_events(vcpu, events);
1205}
1206
1207static int kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu,
1208 struct kvm_vcpu_events *events)
1209{
1210 int i;
1211
1212 /* check whether the reserved field is zero */
1213 for (i = 0; i < ARRAY_SIZE(events->reserved); i++)
1214 if (events->reserved[i])
1215 return -EINVAL;
1216
1217 /* check whether the pad field is zero */
1218 for (i = 0; i < ARRAY_SIZE(events->exception.pad); i++)
1219 if (events->exception.pad[i])
1220 return -EINVAL;
1221
1222 return __kvm_arm_vcpu_set_events(vcpu, events);
1223}
James Morse539aee02018-07-19 16:24:24 +01001224
Christoffer Dall749cf76c2013-01-20 18:28:06 -05001225long kvm_arch_vcpu_ioctl(struct file *filp,
1226 unsigned int ioctl, unsigned long arg)
1227{
1228 struct kvm_vcpu *vcpu = filp->private_data;
1229 void __user *argp = (void __user *)arg;
Shannon Zhaof577f6c2016-01-11 20:56:17 +08001230 struct kvm_device_attr attr;
Christoffer Dall9b0624712017-12-04 21:35:36 +01001231 long r;
1232
Christoffer Dall749cf76c2013-01-20 18:28:06 -05001233 switch (ioctl) {
1234 case KVM_ARM_VCPU_INIT: {
1235 struct kvm_vcpu_init init;
1236
Christoffer Dall9b0624712017-12-04 21:35:36 +01001237 r = -EFAULT;
Christoffer Dall749cf76c2013-01-20 18:28:06 -05001238 if (copy_from_user(&init, argp, sizeof(init)))
Christoffer Dall9b0624712017-12-04 21:35:36 +01001239 break;
Christoffer Dall749cf76c2013-01-20 18:28:06 -05001240
Christoffer Dall9b0624712017-12-04 21:35:36 +01001241 r = kvm_arch_vcpu_ioctl_vcpu_init(vcpu, &init);
1242 break;
Christoffer Dall749cf76c2013-01-20 18:28:06 -05001243 }
1244 case KVM_SET_ONE_REG:
1245 case KVM_GET_ONE_REG: {
1246 struct kvm_one_reg reg;
Andre Przywarae8180dc2013-05-09 00:28:06 +02001247
Christoffer Dall9b0624712017-12-04 21:35:36 +01001248 r = -ENOEXEC;
Andre Przywarae8180dc2013-05-09 00:28:06 +02001249 if (unlikely(!kvm_vcpu_initialized(vcpu)))
Christoffer Dall9b0624712017-12-04 21:35:36 +01001250 break;
Andre Przywarae8180dc2013-05-09 00:28:06 +02001251
Christoffer Dall9b0624712017-12-04 21:35:36 +01001252 r = -EFAULT;
Christoffer Dall749cf76c2013-01-20 18:28:06 -05001253 if (copy_from_user(&reg, argp, sizeof(reg)))
Christoffer Dall9b0624712017-12-04 21:35:36 +01001254 break;
1255
Oliver Upton6826c682021-08-18 20:21:31 +00001256 /*
1257 * We could owe a reset due to PSCI. Handle the pending reset
1258 * here to ensure userspace register accesses are ordered after
1259 * the reset.
1260 */
1261 if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu))
1262 kvm_reset_vcpu(vcpu);
1263
Christoffer Dall749cf76c2013-01-20 18:28:06 -05001264 if (ioctl == KVM_SET_ONE_REG)
Christoffer Dall9b0624712017-12-04 21:35:36 +01001265 r = kvm_arm_set_reg(vcpu, &reg);
Christoffer Dall749cf76c2013-01-20 18:28:06 -05001266 else
Christoffer Dall9b0624712017-12-04 21:35:36 +01001267 r = kvm_arm_get_reg(vcpu, &reg);
1268 break;
Christoffer Dall749cf76c2013-01-20 18:28:06 -05001269 }
1270 case KVM_GET_REG_LIST: {
1271 struct kvm_reg_list __user *user_list = argp;
1272 struct kvm_reg_list reg_list;
1273 unsigned n;
1274
Christoffer Dall9b0624712017-12-04 21:35:36 +01001275 r = -ENOEXEC;
Andre Przywarae8180dc2013-05-09 00:28:06 +02001276 if (unlikely(!kvm_vcpu_initialized(vcpu)))
Christoffer Dall9b0624712017-12-04 21:35:36 +01001277 break;
Andre Przywarae8180dc2013-05-09 00:28:06 +02001278
Dave Martin7dd32a02018-12-19 14:27:01 +00001279 r = -EPERM;
1280 if (!kvm_arm_vcpu_is_finalized(vcpu))
1281 break;
1282
Christoffer Dall9b0624712017-12-04 21:35:36 +01001283 r = -EFAULT;
Christoffer Dall749cf76c2013-01-20 18:28:06 -05001284 if (copy_from_user(&reg_list, user_list, sizeof(reg_list)))
Christoffer Dall9b0624712017-12-04 21:35:36 +01001285 break;
Christoffer Dall749cf76c2013-01-20 18:28:06 -05001286 n = reg_list.n;
1287 reg_list.n = kvm_arm_num_regs(vcpu);
1288 if (copy_to_user(user_list, &reg_list, sizeof(reg_list)))
Christoffer Dall9b0624712017-12-04 21:35:36 +01001289 break;
1290 r = -E2BIG;
Christoffer Dall749cf76c2013-01-20 18:28:06 -05001291 if (n < reg_list.n)
Christoffer Dall9b0624712017-12-04 21:35:36 +01001292 break;
1293 r = kvm_arm_copy_reg_indices(vcpu, user_list->reg);
1294 break;
Christoffer Dall749cf76c2013-01-20 18:28:06 -05001295 }
Shannon Zhaof577f6c2016-01-11 20:56:17 +08001296 case KVM_SET_DEVICE_ATTR: {
Christoffer Dall9b0624712017-12-04 21:35:36 +01001297 r = -EFAULT;
Shannon Zhaof577f6c2016-01-11 20:56:17 +08001298 if (copy_from_user(&attr, argp, sizeof(attr)))
Christoffer Dall9b0624712017-12-04 21:35:36 +01001299 break;
1300 r = kvm_arm_vcpu_set_attr(vcpu, &attr);
1301 break;
Shannon Zhaof577f6c2016-01-11 20:56:17 +08001302 }
1303 case KVM_GET_DEVICE_ATTR: {
Christoffer Dall9b0624712017-12-04 21:35:36 +01001304 r = -EFAULT;
Shannon Zhaof577f6c2016-01-11 20:56:17 +08001305 if (copy_from_user(&attr, argp, sizeof(attr)))
Christoffer Dall9b0624712017-12-04 21:35:36 +01001306 break;
1307 r = kvm_arm_vcpu_get_attr(vcpu, &attr);
1308 break;
Shannon Zhaof577f6c2016-01-11 20:56:17 +08001309 }
1310 case KVM_HAS_DEVICE_ATTR: {
Christoffer Dall9b0624712017-12-04 21:35:36 +01001311 r = -EFAULT;
Shannon Zhaof577f6c2016-01-11 20:56:17 +08001312 if (copy_from_user(&attr, argp, sizeof(attr)))
Christoffer Dall9b0624712017-12-04 21:35:36 +01001313 break;
1314 r = kvm_arm_vcpu_has_attr(vcpu, &attr);
1315 break;
Shannon Zhaof577f6c2016-01-11 20:56:17 +08001316 }
Dongjiu Gengb7b27fa2018-07-19 16:24:22 +01001317 case KVM_GET_VCPU_EVENTS: {
1318 struct kvm_vcpu_events events;
1319
1320 if (kvm_arm_vcpu_get_events(vcpu, &events))
1321 return -EINVAL;
1322
1323 if (copy_to_user(argp, &events, sizeof(events)))
1324 return -EFAULT;
1325
1326 return 0;
1327 }
1328 case KVM_SET_VCPU_EVENTS: {
1329 struct kvm_vcpu_events events;
1330
1331 if (copy_from_user(&events, argp, sizeof(events)))
1332 return -EFAULT;
1333
1334 return kvm_arm_vcpu_set_events(vcpu, &events);
1335 }
Dave Martin7dd32a02018-12-19 14:27:01 +00001336 case KVM_ARM_VCPU_FINALIZE: {
1337 int what;
1338
1339 if (!kvm_vcpu_initialized(vcpu))
1340 return -ENOEXEC;
1341
1342 if (get_user(what, (const int __user *)argp))
1343 return -EFAULT;
1344
1345 return kvm_arm_vcpu_finalize(vcpu, what);
1346 }
Christoffer Dall749cf76c2013-01-20 18:28:06 -05001347 default:
Christoffer Dall9b0624712017-12-04 21:35:36 +01001348 r = -EINVAL;
Christoffer Dall749cf76c2013-01-20 18:28:06 -05001349 }
Christoffer Dall9b0624712017-12-04 21:35:36 +01001350
Christoffer Dall9b0624712017-12-04 21:35:36 +01001351 return r;
Christoffer Dall749cf76c2013-01-20 18:28:06 -05001352}
1353
Sean Christopherson0dff0842020-02-18 13:07:29 -08001354void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot)
Christoffer Dall749cf76c2013-01-20 18:28:06 -05001355{
Mario Smarduch53c810c2015-01-15 15:58:57 -08001356
Christoffer Dall749cf76c2013-01-20 18:28:06 -05001357}
1358
Sean Christopherson0dff0842020-02-18 13:07:29 -08001359void kvm_arch_flush_remote_tlbs_memslot(struct kvm *kvm,
Paolo Bonzini6c9dd6d2021-04-02 17:53:09 +02001360 const struct kvm_memory_slot *memslot)
Paolo Bonzini2a31b9d2018-10-23 02:36:47 +02001361{
Sean Christopherson0dff0842020-02-18 13:07:29 -08001362 kvm_flush_remote_tlbs(kvm);
Christoffer Dall749cf76c2013-01-20 18:28:06 -05001363}
1364
Christoffer Dall3401d5462013-01-23 13:18:04 -05001365static int kvm_vm_ioctl_set_device_addr(struct kvm *kvm,
1366 struct kvm_arm_device_addr *dev_addr)
1367{
Christoffer Dall330690c2013-01-21 19:36:13 -05001368 unsigned long dev_id, type;
1369
1370 dev_id = (dev_addr->id & KVM_ARM_DEVICE_ID_MASK) >>
1371 KVM_ARM_DEVICE_ID_SHIFT;
1372 type = (dev_addr->id & KVM_ARM_DEVICE_TYPE_MASK) >>
1373 KVM_ARM_DEVICE_TYPE_SHIFT;
1374
1375 switch (dev_id) {
1376 case KVM_ARM_DEVICE_VGIC_V2:
Pavel Fedinc7da6fa2015-12-18 14:38:43 +03001377 if (!vgic_present)
1378 return -ENXIO;
Christoffer Dallce01e4e2013-09-23 14:55:56 -07001379 return kvm_vgic_addr(kvm, type, &dev_addr->addr, true);
Christoffer Dall330690c2013-01-21 19:36:13 -05001380 default:
1381 return -ENODEV;
1382 }
Christoffer Dall3401d5462013-01-23 13:18:04 -05001383}
1384
Christoffer Dall749cf76c2013-01-20 18:28:06 -05001385long kvm_arch_vm_ioctl(struct file *filp,
1386 unsigned int ioctl, unsigned long arg)
1387{
Christoffer Dall3401d5462013-01-23 13:18:04 -05001388 struct kvm *kvm = filp->private_data;
1389 void __user *argp = (void __user *)arg;
1390
1391 switch (ioctl) {
Marc Zyngier5863c2c2013-01-21 19:36:15 -05001392 case KVM_CREATE_IRQCHIP: {
Christoffer Dalla28ebea2016-08-09 19:13:01 +02001393 int ret;
Pavel Fedinc7da6fa2015-12-18 14:38:43 +03001394 if (!vgic_present)
1395 return -ENXIO;
Christoffer Dalla28ebea2016-08-09 19:13:01 +02001396 mutex_lock(&kvm->lock);
1397 ret = kvm_vgic_create(kvm, KVM_DEV_TYPE_ARM_VGIC_V2);
1398 mutex_unlock(&kvm->lock);
1399 return ret;
Marc Zyngier5863c2c2013-01-21 19:36:15 -05001400 }
Christoffer Dall3401d5462013-01-23 13:18:04 -05001401 case KVM_ARM_SET_DEVICE_ADDR: {
1402 struct kvm_arm_device_addr dev_addr;
1403
1404 if (copy_from_user(&dev_addr, argp, sizeof(dev_addr)))
1405 return -EFAULT;
1406 return kvm_vm_ioctl_set_device_addr(kvm, &dev_addr);
1407 }
Anup Patel42c4e0c2013-09-30 14:20:07 +05301408 case KVM_ARM_PREFERRED_TARGET: {
Anup Patel42c4e0c2013-09-30 14:20:07 +05301409 struct kvm_vcpu_init init;
1410
YueHaibing08e873c2021-11-05 09:15:00 +08001411 kvm_vcpu_preferred_target(&init);
Anup Patel42c4e0c2013-09-30 14:20:07 +05301412
1413 if (copy_to_user(argp, &init, sizeof(init)))
1414 return -EFAULT;
1415
1416 return 0;
1417 }
Steven Pricef0376ed2021-06-21 12:17:15 +01001418 case KVM_ARM_MTE_COPY_TAGS: {
1419 struct kvm_arm_copy_mte_tags copy_tags;
1420
1421 if (copy_from_user(&copy_tags, argp, sizeof(copy_tags)))
1422 return -EFAULT;
1423 return kvm_vm_ioctl_mte_copy_tags(kvm, &copy_tags);
1424 }
Christoffer Dall3401d5462013-01-23 13:18:04 -05001425 default:
1426 return -EINVAL;
1427 }
Christoffer Dall749cf76c2013-01-20 18:28:06 -05001428}
1429
David Brazdil30c95392020-09-22 21:49:09 +01001430static unsigned long nvhe_percpu_size(void)
1431{
1432 return (unsigned long)CHOOSE_NVHE_SYM(__per_cpu_end) -
1433 (unsigned long)CHOOSE_NVHE_SYM(__per_cpu_start);
1434}
1435
1436static unsigned long nvhe_percpu_order(void)
1437{
1438 unsigned long size = nvhe_percpu_size();
1439
1440 return size ? get_order(size) : 0;
1441}
1442
Will Deaconb881cdc2020-11-13 11:38:44 +00001443/* A lookup table holding the hypervisor VA for each vector slot */
1444static void *hyp_spectre_vector_selector[BP_HARDEN_EL2_SLOTS];
Will Deaconde5bcdb2020-11-13 11:38:39 +00001445
Will Deaconb881cdc2020-11-13 11:38:44 +00001446static void kvm_init_vector_slot(void *base, enum arm64_hyp_spectre_vector slot)
1447{
Quentin Perretbc1d2892021-03-19 10:01:23 +00001448 hyp_spectre_vector_selector[slot] = __kvm_vector_slot2addr(base, slot);
Will Deaconb881cdc2020-11-13 11:38:44 +00001449}
1450
1451static int kvm_init_vector_slots(void)
1452{
1453 int err;
1454 void *base;
1455
1456 base = kern_hyp_va(kvm_ksym_ref(__kvm_hyp_vector));
1457 kvm_init_vector_slot(base, HYP_VECTOR_DIRECT);
1458
1459 base = kern_hyp_va(kvm_ksym_ref(__bp_harden_hyp_vecs));
1460 kvm_init_vector_slot(base, HYP_VECTOR_SPECTRE_DIRECT);
1461
Will Deaconc4792b62020-11-13 11:38:45 +00001462 if (!cpus_have_const_cap(ARM64_SPECTRE_V3A))
Will Deaconde5bcdb2020-11-13 11:38:39 +00001463 return 0;
Will Deacon9ef2b482020-09-28 11:45:24 +01001464
Will Deaconb881cdc2020-11-13 11:38:44 +00001465 if (!has_vhe()) {
1466 err = create_hyp_exec_mappings(__pa_symbol(__bp_harden_hyp_vecs),
1467 __BP_HARDEN_HYP_VECS_SZ, &base);
1468 if (err)
1469 return err;
Will Deacon9ef2b482020-09-28 11:45:24 +01001470 }
1471
Will Deaconb881cdc2020-11-13 11:38:44 +00001472 kvm_init_vector_slot(base, HYP_VECTOR_INDIRECT);
1473 kvm_init_vector_slot(base, HYP_VECTOR_SPECTRE_INDIRECT);
Will Deacon9ef2b482020-09-28 11:45:24 +01001474 return 0;
1475}
1476
Quentin Perret9cc77582021-03-19 10:01:12 +00001477static void cpu_prepare_hyp_mode(int cpu)
Christoffer Dall342cd0a2013-01-20 18:28:06 -05001478{
Quentin Perret9cc77582021-03-19 10:01:12 +00001479 struct kvm_nvhe_init_params *params = per_cpu_ptr_nvhe_sym(kvm_init_params, cpu);
David Brazdild3e10862020-12-02 18:41:07 +00001480 unsigned long tcr;
Christoffer Dall342cd0a2013-01-20 18:28:06 -05001481
David Brazdil71b3ec52020-05-15 16:20:56 +01001482 /*
1483 * Calculate the raw per-cpu offset without a translation from the
1484 * kernel's mapping to the linear mapping, and store it in tpidr_el2
1485 * so that we can use adr_l to access per-cpu variables in EL2.
Steven Pricee1663372021-01-08 16:12:54 +00001486 * Also drop the KASAN tag which gets in the way...
David Brazdil71b3ec52020-05-15 16:20:56 +01001487 */
Quentin Perret9cc77582021-03-19 10:01:12 +00001488 params->tpidr_el2 = (unsigned long)kasan_reset_tag(per_cpu_ptr_nvhe_sym(__per_cpu_start, cpu)) -
David Brazdil63fec242020-12-02 18:41:06 +00001489 (unsigned long)kvm_ksym_ref(CHOOSE_NVHE_SYM(__per_cpu_start));
David Brazdil71b3ec52020-05-15 16:20:56 +01001490
David Brazdild3e10862020-12-02 18:41:07 +00001491 params->mair_el2 = read_sysreg(mair_el1);
1492
1493 /*
1494 * The ID map may be configured to use an extended virtual address
1495 * range. This is only the case if system RAM is out of range for the
1496 * currently configured page size and VA_BITS, in which case we will
1497 * also need the extended virtual range for the HYP ID map, or we won't
1498 * be able to enable the EL2 MMU.
1499 *
1500 * However, at EL2, there is only one TTBR register, and we can't switch
1501 * between translation tables *and* update TCR_EL2.T0SZ at the same
1502 * time. Bottom line: we need to use the extended range with *both* our
1503 * translation tables.
1504 *
1505 * So use the same T0SZ value we use for the ID map.
1506 */
1507 tcr = (read_sysreg(tcr_el1) & TCR_EL2_MASK) | TCR_EL2_RES1;
1508 tcr &= ~TCR_T0SZ_MASK;
1509 tcr |= (idmap_t0sz & GENMASK(TCR_TxSZ_WIDTH - 1, 0)) << TCR_T0SZ_OFFSET;
1510 params->tcr_el2 = tcr;
1511
Quentin Perret9cc77582021-03-19 10:01:12 +00001512 params->stack_hyp_va = kern_hyp_va(per_cpu(kvm_arm_hyp_stack_page, cpu) + PAGE_SIZE);
David Brazdil63fec242020-12-02 18:41:06 +00001513 params->pgd_pa = kvm_mmu_get_httbr();
Quentin Perret734864c2021-03-19 10:01:29 +00001514 if (is_protected_kvm_enabled())
1515 params->hcr_el2 = HCR_HOST_NVHE_PROTECTED_FLAGS;
1516 else
1517 params->hcr_el2 = HCR_HOST_NVHE_FLAGS;
1518 params->vttbr = params->vtcr = 0;
David Brazdil63fec242020-12-02 18:41:06 +00001519
1520 /*
1521 * Flush the init params from the data cache because the struct will
1522 * be read while the MMU is off.
1523 */
1524 kvm_flush_dcache_to_poc(params, sizeof(*params));
Quentin Perret9cc77582021-03-19 10:01:12 +00001525}
1526
Quentin Perretbfa79a82021-03-19 10:01:26 +00001527static void hyp_install_host_vector(void)
Quentin Perret9cc77582021-03-19 10:01:12 +00001528{
1529 struct kvm_nvhe_init_params *params;
1530 struct arm_smccc_res res;
1531
1532 /* Switch from the HYP stub to our own HYP init vector */
1533 __hyp_set_vectors(kvm_get_idmap_vector());
Christoffer Dall342cd0a2013-01-20 18:28:06 -05001534
David Brazdil71b3ec52020-05-15 16:20:56 +01001535 /*
1536 * Call initialization code, and switch to the full blown HYP code.
1537 * If the cpucaps haven't been finalized yet, something has gone very
1538 * wrong, and hyp will crash and burn when it uses any
1539 * cpus_have_const_cap() wrapper.
1540 */
1541 BUG_ON(!system_capabilities_finalized());
Quentin Perret9cc77582021-03-19 10:01:12 +00001542 params = this_cpu_ptr_nvhe_sym(kvm_init_params);
David Brazdil63fec242020-12-02 18:41:06 +00001543 arm_smccc_1_1_hvc(KVM_HOST_SMCCC_FUNC(__kvm_hyp_init), virt_to_phys(params), &res);
Andrew Scull04e4caa2020-09-15 11:46:42 +01001544 WARN_ON(res.a0 != SMCCC_RET_SUCCESS);
Quentin Perretbfa79a82021-03-19 10:01:26 +00001545}
1546
1547static void cpu_init_hyp_mode(void)
1548{
1549 hyp_install_host_vector();
David Brazdil71b3ec52020-05-15 16:20:56 +01001550
1551 /*
1552 * Disabling SSBD on a non-VHE system requires us to enable SSBS
1553 * at EL2.
1554 */
1555 if (this_cpu_has_cap(ARM64_SSBS) &&
Marc Zyngierd63d9752020-09-18 14:08:54 +01001556 arm64_get_spectre_v4_state() == SPECTRE_VULNERABLE) {
David Brazdil13aeb9b2020-06-25 14:14:16 +01001557 kvm_call_hyp_nvhe(__kvm_enable_ssbs);
David Brazdil71b3ec52020-05-15 16:20:56 +01001558 }
Christoffer Dall342cd0a2013-01-20 18:28:06 -05001559}
1560
Marc Zyngier47eb3cb2017-04-03 19:38:01 +01001561static void cpu_hyp_reset(void)
1562{
1563 if (!is_kernel_in_hyp_mode())
1564 __hyp_reset_vectors();
1565}
1566
Will Deacon042c76a2020-11-13 11:38:40 +00001567/*
1568 * EL2 vectors can be mapped and rerouted in a number of ways,
1569 * depending on the kernel configuration and CPU present:
1570 *
1571 * - If the CPU is affected by Spectre-v2, the hardening sequence is
1572 * placed in one of the vector slots, which is executed before jumping
1573 * to the real vectors.
1574 *
Will Deaconc4792b62020-11-13 11:38:45 +00001575 * - If the CPU also has the ARM64_SPECTRE_V3A cap, the slot
Will Deacon042c76a2020-11-13 11:38:40 +00001576 * containing the hardening sequence is mapped next to the idmap page,
1577 * and executed before jumping to the real vectors.
1578 *
Will Deaconc4792b62020-11-13 11:38:45 +00001579 * - If the CPU only has the ARM64_SPECTRE_V3A cap, then an
Will Deacon042c76a2020-11-13 11:38:40 +00001580 * empty slot is selected, mapped next to the idmap page, and
1581 * executed before jumping to the real vectors.
1582 *
Will Deaconc4792b62020-11-13 11:38:45 +00001583 * Note that ARM64_SPECTRE_V3A is somewhat incompatible with
Will Deacon042c76a2020-11-13 11:38:40 +00001584 * VHE, as we don't have hypervisor-specific mappings. If the system
1585 * is VHE and yet selects this capability, it will be ignored.
1586 */
1587static void cpu_set_hyp_vector(void)
1588{
Will Deacon62790172020-11-13 11:38:42 +00001589 struct bp_hardening_data *data = this_cpu_ptr(&bp_hardening_data);
Will Deaconb881cdc2020-11-13 11:38:44 +00001590 void *vector = hyp_spectre_vector_selector[data->slot];
Will Deacon042c76a2020-11-13 11:38:40 +00001591
Quentin Perretbfa79a82021-03-19 10:01:26 +00001592 if (!is_protected_kvm_enabled())
1593 *this_cpu_ptr_hyp_sym(kvm_hyp_vector) = (unsigned long)vector;
1594 else
1595 kvm_call_hyp_nvhe(__pkvm_cpu_set_vector, data->slot);
Will Deacon042c76a2020-11-13 11:38:40 +00001596}
1597
Will Deacon8579a182021-10-08 14:58:36 +01001598static void cpu_hyp_init_context(void)
James Morse5f5560b2016-03-30 18:33:04 +01001599{
David Brazdil2a1198c2020-09-22 21:49:08 +01001600 kvm_init_host_cpu_context(&this_cpu_ptr_hyp_sym(kvm_host_data)->host_ctxt);
Marc Zyngier1e0cf162019-07-05 23:35:56 +01001601
Will Deacon8579a182021-10-08 14:58:36 +01001602 if (!is_kernel_in_hyp_mode())
1603 cpu_init_hyp_mode();
1604}
1605
1606static void cpu_hyp_init_features(void)
1607{
1608 cpu_set_hyp_vector();
1609 kvm_arm_init_debug();
Andrew Sculla0e47952020-09-15 11:46:29 +01001610
Marc Zyngier9d47bb02018-10-01 13:41:32 +01001611 if (is_kernel_in_hyp_mode())
Hu Huajun02d50cd2017-06-12 22:37:48 +08001612 kvm_timer_init_vhe();
Christoffer Dall5b0d2cc2017-03-18 13:56:56 +01001613
1614 if (vgic_present)
1615 kvm_vgic_init_cpu_hardware();
James Morse5f5560b2016-03-30 18:33:04 +01001616}
1617
Will Deacon8579a182021-10-08 14:58:36 +01001618static void cpu_hyp_reinit(void)
1619{
1620 cpu_hyp_reset();
1621 cpu_hyp_init_context();
1622 cpu_hyp_init_features();
1623}
1624
AKASHI Takahiro67f69192016-04-27 17:47:05 +01001625static void _kvm_arch_hardware_enable(void *discard)
1626{
1627 if (!__this_cpu_read(kvm_arm_hardware_enabled)) {
1628 cpu_hyp_reinit();
1629 __this_cpu_write(kvm_arm_hardware_enabled, 1);
1630 }
1631}
1632
1633int kvm_arch_hardware_enable(void)
1634{
1635 _kvm_arch_hardware_enable(NULL);
1636 return 0;
1637}
1638
1639static void _kvm_arch_hardware_disable(void *discard)
1640{
1641 if (__this_cpu_read(kvm_arm_hardware_enabled)) {
1642 cpu_hyp_reset();
1643 __this_cpu_write(kvm_arm_hardware_enabled, 0);
1644 }
1645}
1646
1647void kvm_arch_hardware_disable(void)
1648{
David Brazdilfa8c3d62020-12-02 18:41:20 +00001649 if (!is_protected_kvm_enabled())
1650 _kvm_arch_hardware_disable(NULL);
AKASHI Takahiro67f69192016-04-27 17:47:05 +01001651}
Marc Zyngierd157f4a2013-04-12 19:12:07 +01001652
Lorenzo Pieralisi1fcf7ce2013-08-05 15:04:46 +01001653#ifdef CONFIG_CPU_PM
1654static int hyp_init_cpu_pm_notifier(struct notifier_block *self,
1655 unsigned long cmd,
1656 void *v)
1657{
AKASHI Takahiro67f69192016-04-27 17:47:05 +01001658 /*
1659 * kvm_arm_hardware_enabled is left with its old value over
1660 * PM_ENTER->PM_EXIT. It is used to indicate PM_EXIT should
1661 * re-enable hyp.
1662 */
1663 switch (cmd) {
1664 case CPU_PM_ENTER:
1665 if (__this_cpu_read(kvm_arm_hardware_enabled))
1666 /*
1667 * don't update kvm_arm_hardware_enabled here
1668 * so that the hardware will be re-enabled
1669 * when we resume. See below.
1670 */
1671 cpu_hyp_reset();
Lorenzo Pieralisi1fcf7ce2013-08-05 15:04:46 +01001672
AKASHI Takahiro67f69192016-04-27 17:47:05 +01001673 return NOTIFY_OK;
James Morse58d6b152018-01-22 18:19:06 +00001674 case CPU_PM_ENTER_FAILED:
AKASHI Takahiro67f69192016-04-27 17:47:05 +01001675 case CPU_PM_EXIT:
1676 if (__this_cpu_read(kvm_arm_hardware_enabled))
1677 /* The hardware was enabled before suspend. */
1678 cpu_hyp_reinit();
1679
1680 return NOTIFY_OK;
1681
1682 default:
1683 return NOTIFY_DONE;
1684 }
Lorenzo Pieralisi1fcf7ce2013-08-05 15:04:46 +01001685}
1686
1687static struct notifier_block hyp_init_cpu_pm_nb = {
1688 .notifier_call = hyp_init_cpu_pm_notifier,
1689};
1690
Marc Zyngier44362a32020-12-23 12:08:54 +00001691static void hyp_cpu_pm_init(void)
Lorenzo Pieralisi1fcf7ce2013-08-05 15:04:46 +01001692{
David Brazdilfa8c3d62020-12-02 18:41:20 +00001693 if (!is_protected_kvm_enabled())
1694 cpu_pm_register_notifier(&hyp_init_cpu_pm_nb);
Lorenzo Pieralisi1fcf7ce2013-08-05 15:04:46 +01001695}
Marc Zyngier44362a32020-12-23 12:08:54 +00001696static void hyp_cpu_pm_exit(void)
Sudeep Holla06a71a22016-04-04 14:46:51 +01001697{
David Brazdilfa8c3d62020-12-02 18:41:20 +00001698 if (!is_protected_kvm_enabled())
1699 cpu_pm_unregister_notifier(&hyp_init_cpu_pm_nb);
Sudeep Holla06a71a22016-04-04 14:46:51 +01001700}
Lorenzo Pieralisi1fcf7ce2013-08-05 15:04:46 +01001701#else
1702static inline void hyp_cpu_pm_init(void)
1703{
1704}
Sudeep Holla06a71a22016-04-04 14:46:51 +01001705static inline void hyp_cpu_pm_exit(void)
1706{
1707}
Lorenzo Pieralisi1fcf7ce2013-08-05 15:04:46 +01001708#endif
1709
David Brazdil94f5e8a2020-12-02 18:41:10 +00001710static void init_cpu_logical_map(void)
1711{
1712 unsigned int cpu;
1713
1714 /*
1715 * Copy the MPIDR <-> logical CPU ID mapping to hyp.
1716 * Only copy the set of online CPUs whose features have been chacked
1717 * against the finalized system capabilities. The hypervisor will not
1718 * allow any other CPUs from the `possible` set to boot.
1719 */
1720 for_each_online_cpu(cpu)
David Brazdil61fe0c32020-12-08 14:24:50 +00001721 hyp_cpu_logical_map[cpu] = cpu_logical_map(cpu);
David Brazdil94f5e8a2020-12-02 18:41:10 +00001722}
1723
Marc Zyngier767c9732020-12-22 12:46:41 +00001724#define init_psci_0_1_impl_state(config, what) \
1725 config.psci_0_1_ ## what ## _implemented = psci_ops.what
1726
David Brazdileeeee712020-12-02 18:41:12 +00001727static bool init_psci_relay(void)
1728{
1729 /*
1730 * If PSCI has not been initialized, protected KVM cannot install
1731 * itself on newly booted CPUs.
1732 */
1733 if (!psci_ops.get_version) {
1734 kvm_err("Cannot initialize protected mode without PSCI\n");
1735 return false;
1736 }
1737
David Brazdilff367fe2020-12-08 14:24:47 +00001738 kvm_host_psci_config.version = psci_ops.get_version();
1739
1740 if (kvm_host_psci_config.version == PSCI_VERSION(0, 1)) {
1741 kvm_host_psci_config.function_ids_0_1 = get_psci_0_1_function_ids();
Marc Zyngier767c9732020-12-22 12:46:41 +00001742 init_psci_0_1_impl_state(kvm_host_psci_config, cpu_suspend);
1743 init_psci_0_1_impl_state(kvm_host_psci_config, cpu_on);
1744 init_psci_0_1_impl_state(kvm_host_psci_config, cpu_off);
1745 init_psci_0_1_impl_state(kvm_host_psci_config, migrate);
David Brazdilff367fe2020-12-08 14:24:47 +00001746 }
David Brazdileeeee712020-12-02 18:41:12 +00001747 return true;
1748}
1749
Marc Zyngier1e947ba2015-01-29 11:59:54 +00001750static int init_subsystems(void)
1751{
AKASHI Takahiro67f69192016-04-27 17:47:05 +01001752 int err = 0;
Marc Zyngier1e947ba2015-01-29 11:59:54 +00001753
1754 /*
AKASHI Takahiro67f69192016-04-27 17:47:05 +01001755 * Enable hardware so that subsystem initialisation can access EL2.
James Morse5f5560b2016-03-30 18:33:04 +01001756 */
AKASHI Takahiro67f69192016-04-27 17:47:05 +01001757 on_each_cpu(_kvm_arch_hardware_enable, NULL, 1);
James Morse5f5560b2016-03-30 18:33:04 +01001758
1759 /*
1760 * Register CPU lower-power notifier
1761 */
1762 hyp_cpu_pm_init();
1763
1764 /*
Marc Zyngier1e947ba2015-01-29 11:59:54 +00001765 * Init HYP view of VGIC
1766 */
1767 err = kvm_vgic_hyp_init();
1768 switch (err) {
1769 case 0:
1770 vgic_present = true;
1771 break;
1772 case -ENODEV:
1773 case -ENXIO:
1774 vgic_present = false;
AKASHI Takahiro67f69192016-04-27 17:47:05 +01001775 err = 0;
Marc Zyngier1e947ba2015-01-29 11:59:54 +00001776 break;
1777 default:
AKASHI Takahiro67f69192016-04-27 17:47:05 +01001778 goto out;
Marc Zyngier1e947ba2015-01-29 11:59:54 +00001779 }
1780
1781 /*
1782 * Init HYP architected timer support
1783 */
Marc Zyngierf384dcf2017-12-07 11:46:15 +00001784 err = kvm_timer_hyp_init(vgic_present);
Marc Zyngier1e947ba2015-01-29 11:59:54 +00001785 if (err)
AKASHI Takahiro67f69192016-04-27 17:47:05 +01001786 goto out;
Marc Zyngier1e947ba2015-01-29 11:59:54 +00001787
1788 kvm_perf_init();
Marc Zyngier6ac4a5a2020-11-02 18:11:16 +00001789 kvm_sys_reg_table_init();
Marc Zyngier1e947ba2015-01-29 11:59:54 +00001790
AKASHI Takahiro67f69192016-04-27 17:47:05 +01001791out:
David Brazdilfa8c3d62020-12-02 18:41:20 +00001792 if (err || !is_protected_kvm_enabled())
1793 on_each_cpu(_kvm_arch_hardware_disable, NULL, 1);
AKASHI Takahiro67f69192016-04-27 17:47:05 +01001794
1795 return err;
Marc Zyngier1e947ba2015-01-29 11:59:54 +00001796}
1797
1798static void teardown_hyp_mode(void)
1799{
1800 int cpu;
1801
Marc Zyngier1e947ba2015-01-29 11:59:54 +00001802 free_hyp_pgds();
David Brazdil30c95392020-09-22 21:49:09 +01001803 for_each_possible_cpu(cpu) {
Marc Zyngier1e947ba2015-01-29 11:59:54 +00001804 free_page(per_cpu(kvm_arm_hyp_stack_page, cpu));
David Brazdil30c95392020-09-22 21:49:09 +01001805 free_pages(kvm_arm_hyp_percpu_base[cpu], nvhe_percpu_order());
1806 }
Marc Zyngier1e947ba2015-01-29 11:59:54 +00001807}
1808
Quentin Perretbfa79a82021-03-19 10:01:26 +00001809static int do_pkvm_init(u32 hyp_va_bits)
1810{
1811 void *per_cpu_base = kvm_ksym_ref(kvm_arm_hyp_percpu_base);
1812 int ret;
1813
1814 preempt_disable();
Will Deacon8579a182021-10-08 14:58:36 +01001815 cpu_hyp_init_context();
Quentin Perretbfa79a82021-03-19 10:01:26 +00001816 ret = kvm_call_hyp_nvhe(__pkvm_init, hyp_mem_base, hyp_mem_size,
1817 num_possible_cpus(), kern_hyp_va(per_cpu_base),
1818 hyp_va_bits);
Will Deacon8579a182021-10-08 14:58:36 +01001819 cpu_hyp_init_features();
1820
1821 /*
1822 * The stub hypercalls are now disabled, so set our local flag to
1823 * prevent a later re-init attempt in kvm_arch_hardware_enable().
1824 */
1825 __this_cpu_write(kvm_arm_hardware_enabled, 1);
Quentin Perretbfa79a82021-03-19 10:01:26 +00001826 preempt_enable();
1827
1828 return ret;
1829}
1830
1831static int kvm_hyp_init_protection(u32 hyp_va_bits)
1832{
1833 void *addr = phys_to_virt(hyp_mem_base);
1834 int ret;
1835
Fuad Tabba6c30bfb2021-10-10 15:56:32 +01001836 kvm_nvhe_sym(id_aa64pfr0_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
1837 kvm_nvhe_sym(id_aa64pfr1_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64PFR1_EL1);
1838 kvm_nvhe_sym(id_aa64isar0_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64ISAR0_EL1);
1839 kvm_nvhe_sym(id_aa64isar1_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64ISAR1_EL1);
Marc Zyngier7c419932021-03-22 13:32:34 +00001840 kvm_nvhe_sym(id_aa64mmfr0_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
1841 kvm_nvhe_sym(id_aa64mmfr1_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1);
Fuad Tabba6c30bfb2021-10-10 15:56:32 +01001842 kvm_nvhe_sym(id_aa64mmfr2_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64MMFR2_EL1);
Marc Zyngier7c419932021-03-22 13:32:34 +00001843
Quentin Perretbfa79a82021-03-19 10:01:26 +00001844 ret = create_hyp_mappings(addr, addr + hyp_mem_size, PAGE_HYP);
1845 if (ret)
1846 return ret;
1847
1848 ret = do_pkvm_init(hyp_va_bits);
1849 if (ret)
1850 return ret;
1851
1852 free_hyp_pgds();
1853
1854 return 0;
1855}
1856
Christoffer Dall342cd0a2013-01-20 18:28:06 -05001857/**
1858 * Inits Hyp-mode on all online CPUs
1859 */
1860static int init_hyp_mode(void)
1861{
Quentin Perretbfa79a82021-03-19 10:01:26 +00001862 u32 hyp_va_bits;
Christoffer Dall342cd0a2013-01-20 18:28:06 -05001863 int cpu;
Quentin Perretbfa79a82021-03-19 10:01:26 +00001864 int err = -ENOMEM;
1865
1866 /*
1867 * The protected Hyp-mode cannot be initialized if the memory pool
1868 * allocation has failed.
1869 */
1870 if (is_protected_kvm_enabled() && !hyp_mem_base)
1871 goto out_err;
Christoffer Dall342cd0a2013-01-20 18:28:06 -05001872
1873 /*
1874 * Allocate Hyp PGD and setup Hyp identity mapping
1875 */
Quentin Perretbfa79a82021-03-19 10:01:26 +00001876 err = kvm_mmu_init(&hyp_va_bits);
Christoffer Dall342cd0a2013-01-20 18:28:06 -05001877 if (err)
1878 goto out_err;
1879
1880 /*
Christoffer Dall342cd0a2013-01-20 18:28:06 -05001881 * Allocate stack pages for Hypervisor-mode
1882 */
1883 for_each_possible_cpu(cpu) {
1884 unsigned long stack_page;
1885
1886 stack_page = __get_free_page(GFP_KERNEL);
1887 if (!stack_page) {
1888 err = -ENOMEM;
Marc Zyngier1e947ba2015-01-29 11:59:54 +00001889 goto out_err;
Christoffer Dall342cd0a2013-01-20 18:28:06 -05001890 }
1891
1892 per_cpu(kvm_arm_hyp_stack_page, cpu) = stack_page;
1893 }
1894
1895 /*
David Brazdil30c95392020-09-22 21:49:09 +01001896 * Allocate and initialize pages for Hypervisor-mode percpu regions.
1897 */
1898 for_each_possible_cpu(cpu) {
1899 struct page *page;
1900 void *page_addr;
1901
1902 page = alloc_pages(GFP_KERNEL, nvhe_percpu_order());
1903 if (!page) {
1904 err = -ENOMEM;
1905 goto out_err;
1906 }
1907
1908 page_addr = page_address(page);
1909 memcpy(page_addr, CHOOSE_NVHE_SYM(__per_cpu_start), nvhe_percpu_size());
1910 kvm_arm_hyp_percpu_base[cpu] = (unsigned long)page_addr;
1911 }
1912
1913 /*
Christoffer Dall342cd0a2013-01-20 18:28:06 -05001914 * Map the Hyp-code called directly from the host
1915 */
Linus Torvalds588ab3f2016-03-17 20:03:47 -07001916 err = create_hyp_mappings(kvm_ksym_ref(__hyp_text_start),
Marc Zyngier59002702016-06-13 15:00:48 +01001917 kvm_ksym_ref(__hyp_text_end), PAGE_HYP_EXEC);
Christoffer Dall342cd0a2013-01-20 18:28:06 -05001918 if (err) {
1919 kvm_err("Cannot map world-switch code\n");
Marc Zyngier1e947ba2015-01-29 11:59:54 +00001920 goto out_err;
Christoffer Dall342cd0a2013-01-20 18:28:06 -05001921 }
1922
David Brazdil16174ee2021-01-05 18:05:35 +00001923 err = create_hyp_mappings(kvm_ksym_ref(__hyp_rodata_start),
1924 kvm_ksym_ref(__hyp_rodata_end), PAGE_HYP_RO);
David Brazdil2d7bf212020-12-02 18:41:08 +00001925 if (err) {
David Brazdil16174ee2021-01-05 18:05:35 +00001926 kvm_err("Cannot map .hyp.rodata section\n");
David Brazdil2d7bf212020-12-02 18:41:08 +00001927 goto out_err;
1928 }
1929
Ard Biesheuvela0bf9772016-02-16 13:52:39 +01001930 err = create_hyp_mappings(kvm_ksym_ref(__start_rodata),
Marc Zyngier74a6b882016-06-13 15:00:47 +01001931 kvm_ksym_ref(__end_rodata), PAGE_HYP_RO);
Marc Zyngier910917b2015-10-27 12:18:48 +00001932 if (err) {
1933 kvm_err("Cannot map rodata section\n");
Marc Zyngier1e947ba2015-01-29 11:59:54 +00001934 goto out_err;
Marc Zyngier910917b2015-10-27 12:18:48 +00001935 }
1936
Quentin Perret380e18a2021-03-19 10:01:15 +00001937 /*
1938 * .hyp.bss is guaranteed to be placed at the beginning of the .bss
1939 * section thanks to an assertion in the linker script. Map it RW and
1940 * the rest of .bss RO.
1941 */
1942 err = create_hyp_mappings(kvm_ksym_ref(__hyp_bss_start),
1943 kvm_ksym_ref(__hyp_bss_end), PAGE_HYP);
1944 if (err) {
1945 kvm_err("Cannot map hyp bss section: %d\n", err);
1946 goto out_err;
1947 }
1948
1949 err = create_hyp_mappings(kvm_ksym_ref(__hyp_bss_end),
Marc Zyngierc8ea0392016-10-20 10:17:21 +01001950 kvm_ksym_ref(__bss_stop), PAGE_HYP_RO);
1951 if (err) {
1952 kvm_err("Cannot map bss section\n");
1953 goto out_err;
1954 }
1955
Christoffer Dall342cd0a2013-01-20 18:28:06 -05001956 /*
1957 * Map the Hyp stack pages
1958 */
1959 for_each_possible_cpu(cpu) {
1960 char *stack_page = (char *)per_cpu(kvm_arm_hyp_stack_page, cpu);
Marc Zyngierc8dddec2016-06-13 15:00:45 +01001961 err = create_hyp_mappings(stack_page, stack_page + PAGE_SIZE,
1962 PAGE_HYP);
Christoffer Dall342cd0a2013-01-20 18:28:06 -05001963
1964 if (err) {
1965 kvm_err("Cannot map hyp stack\n");
Marc Zyngier1e947ba2015-01-29 11:59:54 +00001966 goto out_err;
Christoffer Dall342cd0a2013-01-20 18:28:06 -05001967 }
1968 }
1969
Christoffer Dall342cd0a2013-01-20 18:28:06 -05001970 for_each_possible_cpu(cpu) {
David Brazdil30c95392020-09-22 21:49:09 +01001971 char *percpu_begin = (char *)kvm_arm_hyp_percpu_base[cpu];
1972 char *percpu_end = percpu_begin + nvhe_percpu_size();
Christoffer Dall342cd0a2013-01-20 18:28:06 -05001973
Quentin Perret9cc77582021-03-19 10:01:12 +00001974 /* Map Hyp percpu pages */
David Brazdil30c95392020-09-22 21:49:09 +01001975 err = create_hyp_mappings(percpu_begin, percpu_end, PAGE_HYP);
Christoffer Dall342cd0a2013-01-20 18:28:06 -05001976 if (err) {
David Brazdil30c95392020-09-22 21:49:09 +01001977 kvm_err("Cannot map hyp percpu region\n");
Andrew Scull6e3bfbb2020-09-15 11:46:30 +01001978 goto out_err;
1979 }
Quentin Perret9cc77582021-03-19 10:01:12 +00001980
1981 /* Prepare the CPU initialization parameters */
1982 cpu_prepare_hyp_mode(cpu);
Christoffer Dall342cd0a2013-01-20 18:28:06 -05001983 }
1984
David Brazdileeeee712020-12-02 18:41:12 +00001985 if (is_protected_kvm_enabled()) {
David Brazdil94f5e8a2020-12-02 18:41:10 +00001986 init_cpu_logical_map();
1987
Wang Wensheng52b9e262021-04-06 12:17:59 +00001988 if (!init_psci_relay()) {
1989 err = -ENODEV;
David Brazdileeeee712020-12-02 18:41:12 +00001990 goto out_err;
Wang Wensheng52b9e262021-04-06 12:17:59 +00001991 }
David Brazdileeeee712020-12-02 18:41:12 +00001992 }
1993
Quentin Perretbfa79a82021-03-19 10:01:26 +00001994 if (is_protected_kvm_enabled()) {
1995 err = kvm_hyp_init_protection(hyp_va_bits);
1996 if (err) {
1997 kvm_err("Failed to init hyp memory protection\n");
1998 goto out_err;
1999 }
Christoffer Dall342cd0a2013-01-20 18:28:06 -05002000 }
2001
Christoffer Dall342cd0a2013-01-20 18:28:06 -05002002 return 0;
Marc Zyngier1e947ba2015-01-29 11:59:54 +00002003
Christoffer Dall342cd0a2013-01-20 18:28:06 -05002004out_err:
Marc Zyngier1e947ba2015-01-29 11:59:54 +00002005 teardown_hyp_mode();
Christoffer Dall342cd0a2013-01-20 18:28:06 -05002006 kvm_err("error initializing Hyp mode: %d\n", err);
2007 return err;
2008}
2009
Will Deacon2f2e1a52021-10-08 14:58:37 +01002010static void _kvm_host_prot_finalize(void *arg)
Quentin Perret1025c8c2021-03-19 10:01:43 +00002011{
Will Deacon2f2e1a52021-10-08 14:58:37 +01002012 int *err = arg;
2013
2014 if (WARN_ON(kvm_call_hyp_nvhe(__pkvm_prot_finalize)))
2015 WRITE_ONCE(*err, -EINVAL);
2016}
2017
2018static int pkvm_drop_host_privileges(void)
2019{
2020 int ret = 0;
2021
2022 /*
2023 * Flip the static key upfront as that may no longer be possible
2024 * once the host stage 2 is installed.
2025 */
2026 static_branch_enable(&kvm_protected_mode_initialized);
2027 on_each_cpu(_kvm_host_prot_finalize, &ret, 1);
2028 return ret;
Quentin Perret1025c8c2021-03-19 10:01:43 +00002029}
2030
Quentin Perretbfa79a82021-03-19 10:01:26 +00002031static int finalize_hyp_mode(void)
2032{
2033 if (!is_protected_kvm_enabled())
2034 return 0;
2035
Marc Zyngier47e62232021-08-02 13:38:30 +01002036 /*
2037 * Exclude HYP BSS from kmemleak so that it doesn't get peeked
2038 * at, which would end badly once the section is inaccessible.
2039 * None of other sections should ever be introspected.
2040 */
2041 kmemleak_free_part(__hyp_bss_start, __hyp_bss_end - __hyp_bss_start);
Will Deacon2f2e1a52021-10-08 14:58:37 +01002042 return pkvm_drop_host_privileges();
Quentin Perretbfa79a82021-03-19 10:01:26 +00002043}
2044
Andre Przywara4429fc62014-06-02 15:37:13 +02002045struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr)
2046{
2047 struct kvm_vcpu *vcpu;
Marc Zyngier46808a42021-11-16 16:04:02 +00002048 unsigned long i;
Andre Przywara4429fc62014-06-02 15:37:13 +02002049
2050 mpidr &= MPIDR_HWID_BITMASK;
2051 kvm_for_each_vcpu(i, vcpu, kvm) {
2052 if (mpidr == kvm_vcpu_get_mpidr_aff(vcpu))
2053 return vcpu;
2054 }
2055 return NULL;
2056}
2057
Eric Auger24124052017-10-27 15:28:31 +01002058bool kvm_arch_has_irq_bypass(void)
2059{
2060 return true;
2061}
2062
2063int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *cons,
2064 struct irq_bypass_producer *prod)
2065{
2066 struct kvm_kernel_irqfd *irqfd =
2067 container_of(cons, struct kvm_kernel_irqfd, consumer);
2068
Marc Zyngier196b1362017-10-27 15:28:39 +01002069 return kvm_vgic_v4_set_forwarding(irqfd->kvm, prod->irq,
2070 &irqfd->irq_entry);
Eric Auger24124052017-10-27 15:28:31 +01002071}
2072void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *cons,
2073 struct irq_bypass_producer *prod)
2074{
2075 struct kvm_kernel_irqfd *irqfd =
2076 container_of(cons, struct kvm_kernel_irqfd, consumer);
2077
Marc Zyngier196b1362017-10-27 15:28:39 +01002078 kvm_vgic_v4_unset_forwarding(irqfd->kvm, prod->irq,
2079 &irqfd->irq_entry);
Eric Auger24124052017-10-27 15:28:31 +01002080}
2081
2082void kvm_arch_irq_bypass_stop(struct irq_bypass_consumer *cons)
2083{
2084 struct kvm_kernel_irqfd *irqfd =
2085 container_of(cons, struct kvm_kernel_irqfd, consumer);
2086
2087 kvm_arm_halt_guest(irqfd->kvm);
2088}
2089
2090void kvm_arch_irq_bypass_start(struct irq_bypass_consumer *cons)
2091{
2092 struct kvm_kernel_irqfd *irqfd =
2093 container_of(cons, struct kvm_kernel_irqfd, consumer);
2094
2095 kvm_arm_resume_guest(irqfd->kvm);
2096}
2097
Christoffer Dall342cd0a2013-01-20 18:28:06 -05002098/**
2099 * Initialize Hyp-mode and memory mappings on all CPUs.
2100 */
Christoffer Dall749cf76c2013-01-20 18:28:06 -05002101int kvm_arch_init(void *opaque)
2102{
Christoffer Dall342cd0a2013-01-20 18:28:06 -05002103 int err;
Julien Thierryfe7d7b02017-10-20 12:34:16 +01002104 bool in_hyp_mode;
Christoffer Dall342cd0a2013-01-20 18:28:06 -05002105
2106 if (!is_hyp_mode_available()) {
Ard Biesheuvel58d0d192017-11-28 15:18:19 +00002107 kvm_info("HYP mode not available\n");
Christoffer Dall342cd0a2013-01-20 18:28:06 -05002108 return -ENODEV;
2109 }
2110
Marc Zyngierb6a68b92021-10-01 18:05:53 +01002111 if (kvm_get_mode() == KVM_MODE_NONE) {
2112 kvm_info("KVM disabled from command line\n");
2113 return -ENODEV;
2114 }
2115
Marc Zyngier33e5f4e2018-12-06 17:31:20 +00002116 in_hyp_mode = is_kernel_in_hyp_mode();
2117
Rob Herring96d389ca2020-10-28 13:28:39 -05002118 if (cpus_have_final_cap(ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE) ||
2119 cpus_have_final_cap(ARM64_WORKAROUND_1508412))
Rob Herringabf532c2020-08-03 13:31:25 -06002120 kvm_info("Guests without required CPU erratum workarounds can deadlock system!\n" \
2121 "Only trusted guests should be used on this system.\n");
2122
Anshuman Khandualbf249d92021-08-12 10:39:52 +05302123 err = kvm_set_ipa_limit();
Marc Zyngier1e947ba2015-01-29 11:59:54 +00002124 if (err)
2125 return err;
Srivatsa S. Bhat81468752014-03-18 15:53:05 +05302126
Dave Martina3be8362019-04-12 15:30:58 +01002127 err = kvm_arm_init_sve();
Dave Martin0f062bf2019-02-28 18:33:00 +00002128 if (err)
2129 return err;
2130
Julien Thierryfe7d7b02017-10-20 12:34:16 +01002131 if (!in_hyp_mode) {
Marc Zyngier1e947ba2015-01-29 11:59:54 +00002132 err = init_hyp_mode();
Julien Thierryfe7d7b02017-10-20 12:34:16 +01002133 if (err)
2134 goto out_err;
2135 }
Christoffer Dall342cd0a2013-01-20 18:28:06 -05002136
Will Deaconb881cdc2020-11-13 11:38:44 +00002137 err = kvm_init_vector_slots();
2138 if (err) {
2139 kvm_err("Cannot initialise vector slots\n");
2140 goto out_err;
2141 }
2142
Marc Zyngier1e947ba2015-01-29 11:59:54 +00002143 err = init_subsystems();
2144 if (err)
2145 goto out_hyp;
Marc Zyngierd157f4a2013-04-12 19:12:07 +01002146
Quentin Perretbfa79a82021-03-19 10:01:26 +00002147 if (!in_hyp_mode) {
2148 err = finalize_hyp_mode();
2149 if (err) {
2150 kvm_err("Failed to finalize Hyp protection\n");
2151 goto out_hyp;
2152 }
2153 }
2154
David Brazdilf19f6642020-12-02 18:41:22 +00002155 if (is_protected_kvm_enabled()) {
David Brazdil3eb681f2020-12-02 18:40:58 +00002156 kvm_info("Protected nVHE mode initialized successfully\n");
David Brazdilf19f6642020-12-02 18:41:22 +00002157 } else if (in_hyp_mode) {
Julien Thierryfe7d7b02017-10-20 12:34:16 +01002158 kvm_info("VHE mode initialized successfully\n");
David Brazdilf19f6642020-12-02 18:41:22 +00002159 } else {
Julien Thierryfe7d7b02017-10-20 12:34:16 +01002160 kvm_info("Hyp mode initialized successfully\n");
David Brazdilf19f6642020-12-02 18:41:22 +00002161 }
Julien Thierryfe7d7b02017-10-20 12:34:16 +01002162
Christoffer Dall749cf76c2013-01-20 18:28:06 -05002163 return 0;
Marc Zyngier1e947ba2015-01-29 11:59:54 +00002164
2165out_hyp:
Shannon Zhaoc3e35402019-12-02 15:42:11 +08002166 hyp_cpu_pm_exit();
Julien Thierryfe7d7b02017-10-20 12:34:16 +01002167 if (!in_hyp_mode)
2168 teardown_hyp_mode();
Christoffer Dall342cd0a2013-01-20 18:28:06 -05002169out_err:
2170 return err;
Christoffer Dall749cf76c2013-01-20 18:28:06 -05002171}
2172
2173/* NOP: Compiling as a module not supported */
2174void kvm_arch_exit(void)
2175{
Marc Zyngier210552c2013-03-05 03:18:00 +00002176 kvm_perf_teardown();
Christoffer Dall749cf76c2013-01-20 18:28:06 -05002177}
2178
David Brazdild8b369c2020-12-02 18:40:57 +00002179static int __init early_kvm_mode_cfg(char *arg)
2180{
2181 if (!arg)
2182 return -EINVAL;
2183
2184 if (strcmp(arg, "protected") == 0) {
2185 kvm_mode = KVM_MODE_PROTECTED;
2186 return 0;
2187 }
2188
Marc Zyngierb6a68b92021-10-01 18:05:53 +01002189 if (strcmp(arg, "nvhe") == 0 && !WARN_ON(is_kernel_in_hyp_mode())) {
2190 kvm_mode = KVM_MODE_DEFAULT;
Marc Zyngier1945a062021-02-08 09:57:26 +00002191 return 0;
Marc Zyngierb6a68b92021-10-01 18:05:53 +01002192 }
2193
2194 if (strcmp(arg, "none") == 0) {
2195 kvm_mode = KVM_MODE_NONE;
2196 return 0;
2197 }
Marc Zyngier1945a062021-02-08 09:57:26 +00002198
David Brazdild8b369c2020-12-02 18:40:57 +00002199 return -EINVAL;
2200}
2201early_param("kvm-arm.mode", early_kvm_mode_cfg);
2202
David Brazdil3eb681f2020-12-02 18:40:58 +00002203enum kvm_mode kvm_get_mode(void)
2204{
2205 return kvm_mode;
2206}
2207
Christoffer Dall749cf76c2013-01-20 18:28:06 -05002208static int arm_init(void)
2209{
2210 int rc = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
2211 return rc;
2212}
2213
2214module_init(arm_init);