Thomas Gleixner | f6ce7f2 | 2019-05-19 15:51:49 +0200 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
Marcelo Tosatti | 0cf1bfd | 2008-02-22 12:21:36 -0500 | [diff] [blame] | 2 | /* |
| 3 | * KVM paravirt_ops implementation |
| 4 | * |
Marcelo Tosatti | 0cf1bfd | 2008-02-22 12:21:36 -0500 | [diff] [blame] | 5 | * Copyright (C) 2007, Red Hat, Inc., Ingo Molnar <mingo@redhat.com> |
| 6 | * Copyright IBM Corporation, 2007 |
| 7 | * Authors: Anthony Liguori <aliguori@us.ibm.com> |
| 8 | */ |
| 9 | |
Zhenzhong Duan | 5aefd78 | 2019-10-23 19:16:21 +0800 | [diff] [blame] | 10 | #define pr_fmt(fmt) "kvm-guest: " fmt |
| 11 | |
Frederic Weisbecker | 56dd947 | 2013-02-24 00:23:25 +0100 | [diff] [blame] | 12 | #include <linux/context_tracking.h> |
Paul Gortmaker | 186f436 | 2016-07-13 20:18:56 -0400 | [diff] [blame] | 13 | #include <linux/init.h> |
Paolo Bonzini | 26d05b3 | 2020-06-15 07:53:05 -0400 | [diff] [blame] | 14 | #include <linux/irq.h> |
Marcelo Tosatti | 0cf1bfd | 2008-02-22 12:21:36 -0500 | [diff] [blame] | 15 | #include <linux/kernel.h> |
| 16 | #include <linux/kvm_para.h> |
| 17 | #include <linux/cpu.h> |
| 18 | #include <linux/mm.h> |
Marcelo Tosatti | 1da8a77 | 2008-02-22 12:21:37 -0500 | [diff] [blame] | 19 | #include <linux/highmem.h> |
Marcelo Tosatti | 096d14a | 2008-02-22 12:21:38 -0500 | [diff] [blame] | 20 | #include <linux/hardirq.h> |
Gleb Natapov | fd10cde | 2010-10-14 11:22:51 +0200 | [diff] [blame] | 21 | #include <linux/notifier.h> |
| 22 | #include <linux/reboot.h> |
Gleb Natapov | 631bc48 | 2010-10-14 11:22:52 +0200 | [diff] [blame] | 23 | #include <linux/hash.h> |
| 24 | #include <linux/sched.h> |
| 25 | #include <linux/slab.h> |
| 26 | #include <linux/kprobes.h> |
Ulrich Obergfell | 9919e39 | 2014-10-13 15:55:37 -0700 | [diff] [blame] | 27 | #include <linux/nmi.h> |
Rik van Riel | 9db284f | 2016-03-21 15:13:27 +0100 | [diff] [blame] | 28 | #include <linux/swait.h> |
Vitaly Kuznetsov | 8b79fef | 2021-04-14 14:35:41 +0200 | [diff] [blame] | 29 | #include <linux/syscore_ops.h> |
Marcelo Tosatti | a90ede7 | 2009-02-11 22:45:42 -0200 | [diff] [blame] | 30 | #include <asm/timer.h> |
Gleb Natapov | fd10cde | 2010-10-14 11:22:51 +0200 | [diff] [blame] | 31 | #include <asm/cpu.h> |
Gleb Natapov | 631bc48 | 2010-10-14 11:22:52 +0200 | [diff] [blame] | 32 | #include <asm/traps.h> |
| 33 | #include <asm/desc.h> |
Gleb Natapov | 6c047cd | 2010-10-14 11:22:54 +0200 | [diff] [blame] | 34 | #include <asm/tlbflush.h> |
Michael S. Tsirkin | ab9cf49 | 2012-06-24 19:24:34 +0300 | [diff] [blame] | 35 | #include <asm/apic.h> |
| 36 | #include <asm/apicdef.h> |
Prarit Bhargava | fc73373 | 2012-07-06 13:47:39 -0400 | [diff] [blame] | 37 | #include <asm/hypervisor.h> |
Peter Zijlstra | 48a8b97 | 2018-08-22 17:30:16 +0200 | [diff] [blame] | 38 | #include <asm/tlb.h> |
Yi Wang | 19308a4 | 2019-10-10 14:37:25 +0800 | [diff] [blame] | 39 | #include <asm/cpuidle_haltpoll.h> |
Tom Lendacky | 99419b2 | 2020-09-07 15:16:04 +0200 | [diff] [blame] | 40 | #include <asm/ptrace.h> |
Vitaly Kuznetsov | 3d6b841 | 2021-04-14 14:35:43 +0200 | [diff] [blame] | 41 | #include <asm/reboot.h> |
Tom Lendacky | 99419b2 | 2020-09-07 15:16:04 +0200 | [diff] [blame] | 42 | #include <asm/svm.h> |
Marcelo Tosatti | 096d14a | 2008-02-22 12:21:38 -0500 | [diff] [blame] | 43 | |
Andy Lutomirski | ef68017 | 2020-02-28 10:42:48 -0800 | [diff] [blame] | 44 | DEFINE_STATIC_KEY_FALSE(kvm_async_pf_enabled); |
| 45 | |
Gleb Natapov | fd10cde | 2010-10-14 11:22:51 +0200 | [diff] [blame] | 46 | static int kvmapf = 1; |
| 47 | |
Dou Liyang | afdc3f58 | 2018-01-17 11:46:54 +0800 | [diff] [blame] | 48 | static int __init parse_no_kvmapf(char *arg) |
Gleb Natapov | fd10cde | 2010-10-14 11:22:51 +0200 | [diff] [blame] | 49 | { |
| 50 | kvmapf = 0; |
| 51 | return 0; |
| 52 | } |
| 53 | |
| 54 | early_param("no-kvmapf", parse_no_kvmapf); |
| 55 | |
Glauber Costa | d910f5c | 2011-07-11 15:28:19 -0400 | [diff] [blame] | 56 | static int steal_acc = 1; |
Dou Liyang | afdc3f58 | 2018-01-17 11:46:54 +0800 | [diff] [blame] | 57 | static int __init parse_no_stealacc(char *arg) |
Glauber Costa | d910f5c | 2011-07-11 15:28:19 -0400 | [diff] [blame] | 58 | { |
| 59 | steal_acc = 0; |
| 60 | return 0; |
| 61 | } |
| 62 | |
| 63 | early_param("no-steal-acc", parse_no_stealacc); |
| 64 | |
Brijesh Singh | 4716276 | 2017-10-20 09:30:58 -0500 | [diff] [blame] | 65 | static DEFINE_PER_CPU_DECRYPTED(struct kvm_vcpu_pv_apf_data, apf_reason) __aligned(64); |
Andi Kleen | 14e581c | 2019-03-29 17:47:42 -0700 | [diff] [blame] | 66 | DEFINE_PER_CPU_DECRYPTED(struct kvm_steal_time, steal_time) __aligned(64) __visible; |
Glauber Costa | d910f5c | 2011-07-11 15:28:19 -0400 | [diff] [blame] | 67 | static int has_steal_clock = 0; |
Marcelo Tosatti | 096d14a | 2008-02-22 12:21:38 -0500 | [diff] [blame] | 68 | |
Marcelo Tosatti | 0cf1bfd | 2008-02-22 12:21:36 -0500 | [diff] [blame] | 69 | /* |
| 70 | * No need for any "IO delay" on KVM |
| 71 | */ |
| 72 | static void kvm_io_delay(void) |
| 73 | { |
| 74 | } |
| 75 | |
Gleb Natapov | 631bc48 | 2010-10-14 11:22:52 +0200 | [diff] [blame] | 76 | #define KVM_TASK_SLEEP_HASHBITS 8 |
| 77 | #define KVM_TASK_SLEEP_HASHSIZE (1<<KVM_TASK_SLEEP_HASHBITS) |
| 78 | |
| 79 | struct kvm_task_sleep_node { |
| 80 | struct hlist_node link; |
Rik van Riel | 9db284f | 2016-03-21 15:13:27 +0100 | [diff] [blame] | 81 | struct swait_queue_head wq; |
Gleb Natapov | 631bc48 | 2010-10-14 11:22:52 +0200 | [diff] [blame] | 82 | u32 token; |
| 83 | int cpu; |
| 84 | }; |
| 85 | |
| 86 | static struct kvm_task_sleep_head { |
Rik van Riel | 9db284f | 2016-03-21 15:13:27 +0100 | [diff] [blame] | 87 | raw_spinlock_t lock; |
Gleb Natapov | 631bc48 | 2010-10-14 11:22:52 +0200 | [diff] [blame] | 88 | struct hlist_head list; |
| 89 | } async_pf_sleepers[KVM_TASK_SLEEP_HASHSIZE]; |
| 90 | |
| 91 | static struct kvm_task_sleep_node *_find_apf_task(struct kvm_task_sleep_head *b, |
| 92 | u32 token) |
| 93 | { |
| 94 | struct hlist_node *p; |
| 95 | |
| 96 | hlist_for_each(p, &b->list) { |
| 97 | struct kvm_task_sleep_node *n = |
| 98 | hlist_entry(p, typeof(*n), link); |
| 99 | if (n->token == token) |
| 100 | return n; |
| 101 | } |
| 102 | |
| 103 | return NULL; |
| 104 | } |
| 105 | |
Thomas Gleixner | 3a7c8fa | 2020-04-24 09:57:56 +0200 | [diff] [blame] | 106 | static bool kvm_async_pf_queue_task(u32 token, struct kvm_task_sleep_node *n) |
Gleb Natapov | 631bc48 | 2010-10-14 11:22:52 +0200 | [diff] [blame] | 107 | { |
| 108 | u32 key = hash_32(token, KVM_TASK_SLEEP_HASHBITS); |
| 109 | struct kvm_task_sleep_head *b = &async_pf_sleepers[key]; |
Thomas Gleixner | 6bca69a | 2020-03-07 00:42:06 +0100 | [diff] [blame] | 110 | struct kvm_task_sleep_node *e; |
Li Zhong | 9b132fb | 2012-12-04 10:35:13 +0800 | [diff] [blame] | 111 | |
Rik van Riel | 9db284f | 2016-03-21 15:13:27 +0100 | [diff] [blame] | 112 | raw_spin_lock(&b->lock); |
Gleb Natapov | 631bc48 | 2010-10-14 11:22:52 +0200 | [diff] [blame] | 113 | e = _find_apf_task(b, token); |
| 114 | if (e) { |
| 115 | /* dummy entry exist -> wake up was delivered ahead of PF */ |
| 116 | hlist_del(&e->link); |
Rik van Riel | 9db284f | 2016-03-21 15:13:27 +0100 | [diff] [blame] | 117 | raw_spin_unlock(&b->lock); |
Thomas Gleixner | 6bca69a | 2020-03-07 00:42:06 +0100 | [diff] [blame] | 118 | kfree(e); |
| 119 | return false; |
Gleb Natapov | 631bc48 | 2010-10-14 11:22:52 +0200 | [diff] [blame] | 120 | } |
| 121 | |
Thomas Gleixner | 6bca69a | 2020-03-07 00:42:06 +0100 | [diff] [blame] | 122 | n->token = token; |
| 123 | n->cpu = smp_processor_id(); |
Thomas Gleixner | 6bca69a | 2020-03-07 00:42:06 +0100 | [diff] [blame] | 124 | init_swait_queue_head(&n->wq); |
| 125 | hlist_add_head(&n->link, &b->list); |
Rik van Riel | 9db284f | 2016-03-21 15:13:27 +0100 | [diff] [blame] | 126 | raw_spin_unlock(&b->lock); |
Thomas Gleixner | 6bca69a | 2020-03-07 00:42:06 +0100 | [diff] [blame] | 127 | return true; |
| 128 | } |
| 129 | |
| 130 | /* |
| 131 | * kvm_async_pf_task_wait_schedule - Wait for pagefault to be handled |
| 132 | * @token: Token to identify the sleep node entry |
| 133 | * |
| 134 | * Invoked from the async pagefault handling code or from the VM exit page |
| 135 | * fault handler. In both cases RCU is watching. |
| 136 | */ |
| 137 | void kvm_async_pf_task_wait_schedule(u32 token) |
| 138 | { |
| 139 | struct kvm_task_sleep_node n; |
| 140 | DECLARE_SWAITQUEUE(wait); |
| 141 | |
| 142 | lockdep_assert_irqs_disabled(); |
| 143 | |
Thomas Gleixner | 3a7c8fa | 2020-04-24 09:57:56 +0200 | [diff] [blame] | 144 | if (!kvm_async_pf_queue_task(token, &n)) |
Thomas Gleixner | 6bca69a | 2020-03-07 00:42:06 +0100 | [diff] [blame] | 145 | return; |
Gleb Natapov | 631bc48 | 2010-10-14 11:22:52 +0200 | [diff] [blame] | 146 | |
| 147 | for (;;) { |
Thomas Gleixner | 6bca69a | 2020-03-07 00:42:06 +0100 | [diff] [blame] | 148 | prepare_to_swait_exclusive(&n.wq, &wait, TASK_UNINTERRUPTIBLE); |
Gleb Natapov | 631bc48 | 2010-10-14 11:22:52 +0200 | [diff] [blame] | 149 | if (hlist_unhashed(&n.link)) |
| 150 | break; |
Gleb Natapov | 6c047cd | 2010-10-14 11:22:54 +0200 | [diff] [blame] | 151 | |
Thomas Gleixner | 6bca69a | 2020-03-07 00:42:06 +0100 | [diff] [blame] | 152 | local_irq_enable(); |
| 153 | schedule(); |
| 154 | local_irq_disable(); |
Gleb Natapov | 631bc48 | 2010-10-14 11:22:52 +0200 | [diff] [blame] | 155 | } |
Thomas Gleixner | 6bca69a | 2020-03-07 00:42:06 +0100 | [diff] [blame] | 156 | finish_swait(&n.wq, &wait); |
Gleb Natapov | 631bc48 | 2010-10-14 11:22:52 +0200 | [diff] [blame] | 157 | } |
Thomas Gleixner | 6bca69a | 2020-03-07 00:42:06 +0100 | [diff] [blame] | 158 | EXPORT_SYMBOL_GPL(kvm_async_pf_task_wait_schedule); |
| 159 | |
Gleb Natapov | 631bc48 | 2010-10-14 11:22:52 +0200 | [diff] [blame] | 160 | static void apf_task_wake_one(struct kvm_task_sleep_node *n) |
| 161 | { |
| 162 | hlist_del_init(&n->link); |
Thomas Gleixner | 3a7c8fa | 2020-04-24 09:57:56 +0200 | [diff] [blame] | 163 | if (swq_has_sleeper(&n->wq)) |
Peter Zijlstra | b3dae10 | 2018-06-12 10:34:52 +0200 | [diff] [blame] | 164 | swake_up_one(&n->wq); |
Gleb Natapov | 631bc48 | 2010-10-14 11:22:52 +0200 | [diff] [blame] | 165 | } |
| 166 | |
| 167 | static void apf_task_wake_all(void) |
| 168 | { |
| 169 | int i; |
| 170 | |
| 171 | for (i = 0; i < KVM_TASK_SLEEP_HASHSIZE; i++) { |
Gleb Natapov | 631bc48 | 2010-10-14 11:22:52 +0200 | [diff] [blame] | 172 | struct kvm_task_sleep_head *b = &async_pf_sleepers[i]; |
Thomas Gleixner | 6bca69a | 2020-03-07 00:42:06 +0100 | [diff] [blame] | 173 | struct kvm_task_sleep_node *n; |
| 174 | struct hlist_node *p, *next; |
| 175 | |
Rik van Riel | 9db284f | 2016-03-21 15:13:27 +0100 | [diff] [blame] | 176 | raw_spin_lock(&b->lock); |
Gleb Natapov | 631bc48 | 2010-10-14 11:22:52 +0200 | [diff] [blame] | 177 | hlist_for_each_safe(p, next, &b->list) { |
Thomas Gleixner | 6bca69a | 2020-03-07 00:42:06 +0100 | [diff] [blame] | 178 | n = hlist_entry(p, typeof(*n), link); |
Gleb Natapov | 631bc48 | 2010-10-14 11:22:52 +0200 | [diff] [blame] | 179 | if (n->cpu == smp_processor_id()) |
| 180 | apf_task_wake_one(n); |
| 181 | } |
Rik van Riel | 9db284f | 2016-03-21 15:13:27 +0100 | [diff] [blame] | 182 | raw_spin_unlock(&b->lock); |
Gleb Natapov | 631bc48 | 2010-10-14 11:22:52 +0200 | [diff] [blame] | 183 | } |
| 184 | } |
| 185 | |
| 186 | void kvm_async_pf_task_wake(u32 token) |
| 187 | { |
| 188 | u32 key = hash_32(token, KVM_TASK_SLEEP_HASHBITS); |
| 189 | struct kvm_task_sleep_head *b = &async_pf_sleepers[key]; |
| 190 | struct kvm_task_sleep_node *n; |
| 191 | |
| 192 | if (token == ~0) { |
| 193 | apf_task_wake_all(); |
| 194 | return; |
| 195 | } |
| 196 | |
| 197 | again: |
Rik van Riel | 9db284f | 2016-03-21 15:13:27 +0100 | [diff] [blame] | 198 | raw_spin_lock(&b->lock); |
Gleb Natapov | 631bc48 | 2010-10-14 11:22:52 +0200 | [diff] [blame] | 199 | n = _find_apf_task(b, token); |
| 200 | if (!n) { |
| 201 | /* |
| 202 | * async PF was not yet handled. |
| 203 | * Add dummy entry for the token. |
| 204 | */ |
Gleb Natapov | 62c49cc | 2012-05-02 15:04:02 +0300 | [diff] [blame] | 205 | n = kzalloc(sizeof(*n), GFP_ATOMIC); |
Gleb Natapov | 631bc48 | 2010-10-14 11:22:52 +0200 | [diff] [blame] | 206 | if (!n) { |
| 207 | /* |
| 208 | * Allocation failed! Busy wait while other cpu |
| 209 | * handles async PF. |
| 210 | */ |
Rik van Riel | 9db284f | 2016-03-21 15:13:27 +0100 | [diff] [blame] | 211 | raw_spin_unlock(&b->lock); |
Gleb Natapov | 631bc48 | 2010-10-14 11:22:52 +0200 | [diff] [blame] | 212 | cpu_relax(); |
| 213 | goto again; |
| 214 | } |
| 215 | n->token = token; |
| 216 | n->cpu = smp_processor_id(); |
Rik van Riel | 9db284f | 2016-03-21 15:13:27 +0100 | [diff] [blame] | 217 | init_swait_queue_head(&n->wq); |
Gleb Natapov | 631bc48 | 2010-10-14 11:22:52 +0200 | [diff] [blame] | 218 | hlist_add_head(&n->link, &b->list); |
Thomas Gleixner | 6bca69a | 2020-03-07 00:42:06 +0100 | [diff] [blame] | 219 | } else { |
Gleb Natapov | 631bc48 | 2010-10-14 11:22:52 +0200 | [diff] [blame] | 220 | apf_task_wake_one(n); |
Thomas Gleixner | 6bca69a | 2020-03-07 00:42:06 +0100 | [diff] [blame] | 221 | } |
Rik van Riel | 9db284f | 2016-03-21 15:13:27 +0100 | [diff] [blame] | 222 | raw_spin_unlock(&b->lock); |
Gleb Natapov | 631bc48 | 2010-10-14 11:22:52 +0200 | [diff] [blame] | 223 | return; |
| 224 | } |
| 225 | EXPORT_SYMBOL_GPL(kvm_async_pf_task_wake); |
| 226 | |
Thomas Gleixner | 91eeafe | 2020-05-21 22:05:28 +0200 | [diff] [blame] | 227 | noinstr u32 kvm_read_and_reset_apf_flags(void) |
Gleb Natapov | 631bc48 | 2010-10-14 11:22:52 +0200 | [diff] [blame] | 228 | { |
Vitaly Kuznetsov | 68fd66f | 2020-05-25 16:41:17 +0200 | [diff] [blame] | 229 | u32 flags = 0; |
Gleb Natapov | 631bc48 | 2010-10-14 11:22:52 +0200 | [diff] [blame] | 230 | |
Christoph Lameter | 89cbc76 | 2014-08-17 12:30:40 -0500 | [diff] [blame] | 231 | if (__this_cpu_read(apf_reason.enabled)) { |
Vitaly Kuznetsov | 68fd66f | 2020-05-25 16:41:17 +0200 | [diff] [blame] | 232 | flags = __this_cpu_read(apf_reason.flags); |
| 233 | __this_cpu_write(apf_reason.flags, 0); |
Gleb Natapov | 631bc48 | 2010-10-14 11:22:52 +0200 | [diff] [blame] | 234 | } |
| 235 | |
Vitaly Kuznetsov | 68fd66f | 2020-05-25 16:41:17 +0200 | [diff] [blame] | 236 | return flags; |
Gleb Natapov | 631bc48 | 2010-10-14 11:22:52 +0200 | [diff] [blame] | 237 | } |
Vitaly Kuznetsov | 68fd66f | 2020-05-25 16:41:17 +0200 | [diff] [blame] | 238 | EXPORT_SYMBOL_GPL(kvm_read_and_reset_apf_flags); |
Gleb Natapov | 631bc48 | 2010-10-14 11:22:52 +0200 | [diff] [blame] | 239 | |
Thomas Gleixner | 91eeafe | 2020-05-21 22:05:28 +0200 | [diff] [blame] | 240 | noinstr bool __kvm_handle_async_pf(struct pt_regs *regs, u32 token) |
Gleb Natapov | 631bc48 | 2010-10-14 11:22:52 +0200 | [diff] [blame] | 241 | { |
Vitaly Kuznetsov | b1d4057 | 2020-05-25 16:41:23 +0200 | [diff] [blame] | 242 | u32 flags = kvm_read_and_reset_apf_flags(); |
Thomas Gleixner | a27a0a5 | 2020-07-23 00:00:08 +0200 | [diff] [blame] | 243 | irqentry_state_t state; |
Thomas Gleixner | 6bca69a | 2020-03-07 00:42:06 +0100 | [diff] [blame] | 244 | |
Vitaly Kuznetsov | b1d4057 | 2020-05-25 16:41:23 +0200 | [diff] [blame] | 245 | if (!flags) |
Andy Lutomirski | ef68017 | 2020-02-28 10:42:48 -0800 | [diff] [blame] | 246 | return false; |
Thomas Gleixner | 6bca69a | 2020-03-07 00:42:06 +0100 | [diff] [blame] | 247 | |
Thomas Gleixner | a27a0a5 | 2020-07-23 00:00:08 +0200 | [diff] [blame] | 248 | state = irqentry_enter(regs); |
Thomas Gleixner | 91eeafe | 2020-05-21 22:05:28 +0200 | [diff] [blame] | 249 | instrumentation_begin(); |
| 250 | |
Thomas Gleixner | 6bca69a | 2020-03-07 00:42:06 +0100 | [diff] [blame] | 251 | /* |
| 252 | * If the host managed to inject an async #PF into an interrupt |
| 253 | * disabled region, then die hard as this is not going to end well |
| 254 | * and the host side is seriously broken. |
| 255 | */ |
| 256 | if (unlikely(!(regs->flags & X86_EFLAGS_IF))) |
| 257 | panic("Host injected async #PF in interrupt disabled region\n"); |
| 258 | |
Vitaly Kuznetsov | b1d4057 | 2020-05-25 16:41:23 +0200 | [diff] [blame] | 259 | if (flags & KVM_PV_REASON_PAGE_NOT_PRESENT) { |
Thomas Gleixner | 3a7c8fa | 2020-04-24 09:57:56 +0200 | [diff] [blame] | 260 | if (unlikely(!(user_mode(regs)))) |
| 261 | panic("Host injected async #PF in kernel mode\n"); |
| 262 | /* Page is swapped out by the host. */ |
| 263 | kvm_async_pf_task_wait_schedule(token); |
Thomas Gleixner | 6bca69a | 2020-03-07 00:42:06 +0100 | [diff] [blame] | 264 | } else { |
Paolo Bonzini | 26d05b3 | 2020-06-15 07:53:05 -0400 | [diff] [blame] | 265 | WARN_ONCE(1, "Unexpected async PF flags: %x\n", flags); |
Gleb Natapov | 631bc48 | 2010-10-14 11:22:52 +0200 | [diff] [blame] | 266 | } |
Thomas Gleixner | 91eeafe | 2020-05-21 22:05:28 +0200 | [diff] [blame] | 267 | |
| 268 | instrumentation_end(); |
Thomas Gleixner | a27a0a5 | 2020-07-23 00:00:08 +0200 | [diff] [blame] | 269 | irqentry_exit(regs, state); |
Thomas Gleixner | 6bca69a | 2020-03-07 00:42:06 +0100 | [diff] [blame] | 270 | return true; |
Gleb Natapov | 631bc48 | 2010-10-14 11:22:52 +0200 | [diff] [blame] | 271 | } |
| 272 | |
Paolo Bonzini | 26d05b3 | 2020-06-15 07:53:05 -0400 | [diff] [blame] | 273 | DEFINE_IDTENTRY_SYSVEC(sysvec_kvm_asyncpf_interrupt) |
Vitaly Kuznetsov | b1d4057 | 2020-05-25 16:41:23 +0200 | [diff] [blame] | 274 | { |
Paolo Bonzini | 26d05b3 | 2020-06-15 07:53:05 -0400 | [diff] [blame] | 275 | struct pt_regs *old_regs = set_irq_regs(regs); |
Vitaly Kuznetsov | b1d4057 | 2020-05-25 16:41:23 +0200 | [diff] [blame] | 276 | u32 token; |
Vitaly Kuznetsov | b1d4057 | 2020-05-25 16:41:23 +0200 | [diff] [blame] | 277 | |
Vitaly Kuznetsov | cc17b22 | 2020-09-08 15:53:50 +0200 | [diff] [blame] | 278 | ack_APIC_irq(); |
| 279 | |
Vitaly Kuznetsov | b1d4057 | 2020-05-25 16:41:23 +0200 | [diff] [blame] | 280 | inc_irq_stat(irq_hv_callback_count); |
| 281 | |
| 282 | if (__this_cpu_read(apf_reason.enabled)) { |
| 283 | token = __this_cpu_read(apf_reason.token); |
Vitaly Kuznetsov | b1d4057 | 2020-05-25 16:41:23 +0200 | [diff] [blame] | 284 | kvm_async_pf_task_wake(token); |
Vitaly Kuznetsov | b1d4057 | 2020-05-25 16:41:23 +0200 | [diff] [blame] | 285 | __this_cpu_write(apf_reason.token, 0); |
| 286 | wrmsrl(MSR_KVM_ASYNC_PF_ACK, 1); |
| 287 | } |
| 288 | |
Paolo Bonzini | 26d05b3 | 2020-06-15 07:53:05 -0400 | [diff] [blame] | 289 | set_irq_regs(old_regs); |
Vitaly Kuznetsov | b1d4057 | 2020-05-25 16:41:23 +0200 | [diff] [blame] | 290 | } |
| 291 | |
Rakib Mullick | d3ac881 | 2009-07-02 11:40:36 +0600 | [diff] [blame] | 292 | static void __init paravirt_ops_setup(void) |
Marcelo Tosatti | 0cf1bfd | 2008-02-22 12:21:36 -0500 | [diff] [blame] | 293 | { |
| 294 | pv_info.name = "KVM"; |
Andy Lutomirski | 29fa682 | 2014-12-05 19:03:28 -0800 | [diff] [blame] | 295 | |
Marcelo Tosatti | 0cf1bfd | 2008-02-22 12:21:36 -0500 | [diff] [blame] | 296 | if (kvm_para_has_feature(KVM_FEATURE_NOP_IO_DELAY)) |
Juergen Gross | 5c83511 | 2018-08-28 09:40:19 +0200 | [diff] [blame] | 297 | pv_ops.cpu.io_delay = kvm_io_delay; |
Marcelo Tosatti | 0cf1bfd | 2008-02-22 12:21:36 -0500 | [diff] [blame] | 298 | |
Marcelo Tosatti | a90ede7 | 2009-02-11 22:45:42 -0200 | [diff] [blame] | 299 | #ifdef CONFIG_X86_IO_APIC |
| 300 | no_timer_check = 1; |
| 301 | #endif |
Marcelo Tosatti | 0cf1bfd | 2008-02-22 12:21:36 -0500 | [diff] [blame] | 302 | } |
| 303 | |
Glauber Costa | d910f5c | 2011-07-11 15:28:19 -0400 | [diff] [blame] | 304 | static void kvm_register_steal_time(void) |
| 305 | { |
| 306 | int cpu = smp_processor_id(); |
| 307 | struct kvm_steal_time *st = &per_cpu(steal_time, cpu); |
| 308 | |
| 309 | if (!has_steal_clock) |
| 310 | return; |
| 311 | |
Dave Hansen | 5dfd486 | 2013-01-22 13:24:35 -0800 | [diff] [blame] | 312 | wrmsrl(MSR_KVM_STEAL_TIME, (slow_virt_to_phys(st) | KVM_MSR_ENABLED)); |
Zhenzhong Duan | 5aefd78 | 2019-10-23 19:16:21 +0800 | [diff] [blame] | 313 | pr_info("stealtime: cpu %d, msr %llx\n", cpu, |
| 314 | (unsigned long long) slow_virt_to_phys(st)); |
Glauber Costa | d910f5c | 2011-07-11 15:28:19 -0400 | [diff] [blame] | 315 | } |
| 316 | |
Brijesh Singh | 4716276 | 2017-10-20 09:30:58 -0500 | [diff] [blame] | 317 | static DEFINE_PER_CPU_DECRYPTED(unsigned long, kvm_apic_eoi) = KVM_PV_EOI_DISABLED; |
Michael S. Tsirkin | ab9cf49 | 2012-06-24 19:24:34 +0300 | [diff] [blame] | 318 | |
Wanpeng Li | 8ca2255 | 2016-11-07 11:13:40 +0800 | [diff] [blame] | 319 | static notrace void kvm_guest_apic_eoi_write(u32 reg, u32 val) |
Michael S. Tsirkin | ab9cf49 | 2012-06-24 19:24:34 +0300 | [diff] [blame] | 320 | { |
| 321 | /** |
| 322 | * This relies on __test_and_clear_bit to modify the memory |
| 323 | * in a way that is atomic with respect to the local CPU. |
| 324 | * The hypervisor only accesses this memory from the local CPU so |
| 325 | * there's no need for lock or memory barriers. |
| 326 | * An optimization barrier is implied in apic write. |
| 327 | */ |
Christoph Lameter | 89cbc76 | 2014-08-17 12:30:40 -0500 | [diff] [blame] | 328 | if (__test_and_clear_bit(KVM_PV_EOI_BIT, this_cpu_ptr(&kvm_apic_eoi))) |
Michael S. Tsirkin | ab9cf49 | 2012-06-24 19:24:34 +0300 | [diff] [blame] | 329 | return; |
Wanpeng Li | 8ca2255 | 2016-11-07 11:13:40 +0800 | [diff] [blame] | 330 | apic->native_eoi_write(APIC_EOI, APIC_EOI_ACK); |
Michael S. Tsirkin | ab9cf49 | 2012-06-24 19:24:34 +0300 | [diff] [blame] | 331 | } |
| 332 | |
Nicholas Krause | ed3cf15 | 2015-05-20 00:24:10 -0400 | [diff] [blame] | 333 | static void kvm_guest_cpu_init(void) |
Gleb Natapov | fd10cde | 2010-10-14 11:22:51 +0200 | [diff] [blame] | 334 | { |
Vitaly Kuznetsov | b1d4057 | 2020-05-25 16:41:23 +0200 | [diff] [blame] | 335 | if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF_INT) && kvmapf) { |
| 336 | u64 pa = slow_virt_to_phys(this_cpu_ptr(&apf_reason)); |
Andy Lutomirski | ef68017 | 2020-02-28 10:42:48 -0800 | [diff] [blame] | 337 | |
| 338 | WARN_ON_ONCE(!static_branch_likely(&kvm_async_pf_enabled)); |
| 339 | |
| 340 | pa = slow_virt_to_phys(this_cpu_ptr(&apf_reason)); |
Vitaly Kuznetsov | b1d4057 | 2020-05-25 16:41:23 +0200 | [diff] [blame] | 341 | pa |= KVM_ASYNC_PF_ENABLED | KVM_ASYNC_PF_DELIVERY_AS_INT; |
Wanpeng Li | 52a5c15 | 2017-07-13 18:30:42 -0700 | [diff] [blame] | 342 | |
Radim Krčmář | fe2a302 | 2018-02-01 22:16:21 +0100 | [diff] [blame] | 343 | if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF_VMEXIT)) |
| 344 | pa |= KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT; |
| 345 | |
Vitaly Kuznetsov | b1d4057 | 2020-05-25 16:41:23 +0200 | [diff] [blame] | 346 | wrmsrl(MSR_KVM_ASYNC_PF_INT, HYPERVISOR_CALLBACK_VECTOR); |
| 347 | |
Radim Krčmář | fe2a302 | 2018-02-01 22:16:21 +0100 | [diff] [blame] | 348 | wrmsrl(MSR_KVM_ASYNC_PF_EN, pa); |
Christoph Lameter | 89cbc76 | 2014-08-17 12:30:40 -0500 | [diff] [blame] | 349 | __this_cpu_write(apf_reason.enabled, 1); |
Vitaly Kuznetsov | 0a269a0 | 2021-04-14 14:35:40 +0200 | [diff] [blame] | 350 | pr_info("setup async PF for cpu %d\n", smp_processor_id()); |
Gleb Natapov | fd10cde | 2010-10-14 11:22:51 +0200 | [diff] [blame] | 351 | } |
Glauber Costa | d910f5c | 2011-07-11 15:28:19 -0400 | [diff] [blame] | 352 | |
Michael S. Tsirkin | ab9cf49 | 2012-06-24 19:24:34 +0300 | [diff] [blame] | 353 | if (kvm_para_has_feature(KVM_FEATURE_PV_EOI)) { |
| 354 | unsigned long pa; |
Thomas Gleixner | 6bca69a | 2020-03-07 00:42:06 +0100 | [diff] [blame] | 355 | |
Michael S. Tsirkin | ab9cf49 | 2012-06-24 19:24:34 +0300 | [diff] [blame] | 356 | /* Size alignment is implied but just to make it explicit. */ |
| 357 | BUILD_BUG_ON(__alignof__(kvm_apic_eoi) < 4); |
Christoph Lameter | 89cbc76 | 2014-08-17 12:30:40 -0500 | [diff] [blame] | 358 | __this_cpu_write(kvm_apic_eoi, 0); |
| 359 | pa = slow_virt_to_phys(this_cpu_ptr(&kvm_apic_eoi)) |
Dave Hansen | 5dfd486 | 2013-01-22 13:24:35 -0800 | [diff] [blame] | 360 | | KVM_MSR_ENABLED; |
Michael S. Tsirkin | ab9cf49 | 2012-06-24 19:24:34 +0300 | [diff] [blame] | 361 | wrmsrl(MSR_KVM_PV_EOI_EN, pa); |
| 362 | } |
| 363 | |
Glauber Costa | d910f5c | 2011-07-11 15:28:19 -0400 | [diff] [blame] | 364 | if (has_steal_clock) |
| 365 | kvm_register_steal_time(); |
Gleb Natapov | fd10cde | 2010-10-14 11:22:51 +0200 | [diff] [blame] | 366 | } |
| 367 | |
Michael S. Tsirkin | ab9cf49 | 2012-06-24 19:24:34 +0300 | [diff] [blame] | 368 | static void kvm_pv_disable_apf(void) |
Gleb Natapov | fd10cde | 2010-10-14 11:22:51 +0200 | [diff] [blame] | 369 | { |
Christoph Lameter | 89cbc76 | 2014-08-17 12:30:40 -0500 | [diff] [blame] | 370 | if (!__this_cpu_read(apf_reason.enabled)) |
Gleb Natapov | fd10cde | 2010-10-14 11:22:51 +0200 | [diff] [blame] | 371 | return; |
| 372 | |
| 373 | wrmsrl(MSR_KVM_ASYNC_PF_EN, 0); |
Christoph Lameter | 89cbc76 | 2014-08-17 12:30:40 -0500 | [diff] [blame] | 374 | __this_cpu_write(apf_reason.enabled, 0); |
Gleb Natapov | fd10cde | 2010-10-14 11:22:51 +0200 | [diff] [blame] | 375 | |
Vitaly Kuznetsov | 0a269a0 | 2021-04-14 14:35:40 +0200 | [diff] [blame] | 376 | pr_info("disable async PF for cpu %d\n", smp_processor_id()); |
Gleb Natapov | fd10cde | 2010-10-14 11:22:51 +0200 | [diff] [blame] | 377 | } |
| 378 | |
Vitaly Kuznetsov | 3d6b841 | 2021-04-14 14:35:43 +0200 | [diff] [blame] | 379 | static void kvm_disable_steal_time(void) |
| 380 | { |
| 381 | if (!has_steal_clock) |
| 382 | return; |
| 383 | |
| 384 | wrmsr(MSR_KVM_STEAL_TIME, 0, 0); |
| 385 | } |
| 386 | |
Glauber Costa | d910f5c | 2011-07-11 15:28:19 -0400 | [diff] [blame] | 387 | static u64 kvm_steal_clock(int cpu) |
| 388 | { |
| 389 | u64 steal; |
| 390 | struct kvm_steal_time *src; |
| 391 | int version; |
| 392 | |
| 393 | src = &per_cpu(steal_time, cpu); |
| 394 | do { |
| 395 | version = src->version; |
Wanpeng Li | 5a48a62 | 2017-04-11 02:49:21 -0700 | [diff] [blame] | 396 | virt_rmb(); |
Glauber Costa | d910f5c | 2011-07-11 15:28:19 -0400 | [diff] [blame] | 397 | steal = src->steal; |
Wanpeng Li | 5a48a62 | 2017-04-11 02:49:21 -0700 | [diff] [blame] | 398 | virt_rmb(); |
Glauber Costa | d910f5c | 2011-07-11 15:28:19 -0400 | [diff] [blame] | 399 | } while ((version & 1) || (version != src->version)); |
| 400 | |
| 401 | return steal; |
| 402 | } |
| 403 | |
Brijesh Singh | 4716276 | 2017-10-20 09:30:58 -0500 | [diff] [blame] | 404 | static inline void __set_percpu_decrypted(void *ptr, unsigned long size) |
| 405 | { |
| 406 | early_set_memory_decrypted((unsigned long) ptr, size); |
| 407 | } |
| 408 | |
| 409 | /* |
| 410 | * Iterate through all possible CPUs and map the memory region pointed |
| 411 | * by apf_reason, steal_time and kvm_apic_eoi as decrypted at once. |
| 412 | * |
| 413 | * Note: we iterate through all possible CPUs to ensure that CPUs |
| 414 | * hotplugged will have their per-cpu variable already mapped as |
| 415 | * decrypted. |
| 416 | */ |
| 417 | static void __init sev_map_percpu_data(void) |
| 418 | { |
| 419 | int cpu; |
| 420 | |
| 421 | if (!sev_active()) |
| 422 | return; |
| 423 | |
| 424 | for_each_possible_cpu(cpu) { |
| 425 | __set_percpu_decrypted(&per_cpu(apf_reason, cpu), sizeof(apf_reason)); |
| 426 | __set_percpu_decrypted(&per_cpu(steal_time, cpu), sizeof(steal_time)); |
| 427 | __set_percpu_decrypted(&per_cpu(kvm_apic_eoi, cpu), sizeof(kvm_apic_eoi)); |
| 428 | } |
| 429 | } |
| 430 | |
Vitaly Kuznetsov | 3d6b841 | 2021-04-14 14:35:43 +0200 | [diff] [blame] | 431 | static void kvm_guest_cpu_offline(bool shutdown) |
Vitaly Kuznetsov | 8b79fef | 2021-04-14 14:35:41 +0200 | [diff] [blame] | 432 | { |
| 433 | kvm_disable_steal_time(); |
| 434 | if (kvm_para_has_feature(KVM_FEATURE_PV_EOI)) |
| 435 | wrmsrl(MSR_KVM_PV_EOI_EN, 0); |
| 436 | kvm_pv_disable_apf(); |
Vitaly Kuznetsov | 3d6b841 | 2021-04-14 14:35:43 +0200 | [diff] [blame] | 437 | if (!shutdown) |
| 438 | apf_task_wake_all(); |
Vitaly Kuznetsov | c02027b | 2021-04-14 14:35:42 +0200 | [diff] [blame] | 439 | kvmclock_disable(); |
Vitaly Kuznetsov | 8b79fef | 2021-04-14 14:35:41 +0200 | [diff] [blame] | 440 | } |
| 441 | |
| 442 | static int kvm_cpu_online(unsigned int cpu) |
| 443 | { |
| 444 | unsigned long flags; |
| 445 | |
| 446 | local_irq_save(flags); |
| 447 | kvm_guest_cpu_init(); |
| 448 | local_irq_restore(flags); |
| 449 | return 0; |
| 450 | } |
| 451 | |
Wanpeng Li | 2b519b57 | 2021-04-09 12:18:29 +0800 | [diff] [blame] | 452 | #ifdef CONFIG_SMP |
| 453 | |
| 454 | static DEFINE_PER_CPU(cpumask_var_t, __pv_cpu_mask); |
| 455 | |
Wanpeng Li | a262bca | 2020-02-18 09:08:23 +0800 | [diff] [blame] | 456 | static bool pv_tlb_flush_supported(void) |
| 457 | { |
| 458 | return (kvm_para_has_feature(KVM_FEATURE_PV_TLB_FLUSH) && |
| 459 | !kvm_para_has_hint(KVM_HINTS_REALTIME) && |
| 460 | kvm_para_has_feature(KVM_FEATURE_STEAL_TIME)); |
| 461 | } |
| 462 | |
Wanpeng Li | a262bca | 2020-02-18 09:08:23 +0800 | [diff] [blame] | 463 | static bool pv_ipi_supported(void) |
| 464 | { |
| 465 | return kvm_para_has_feature(KVM_FEATURE_PV_SEND_IPI); |
| 466 | } |
| 467 | |
| 468 | static bool pv_sched_yield_supported(void) |
| 469 | { |
| 470 | return (kvm_para_has_feature(KVM_FEATURE_PV_SCHED_YIELD) && |
| 471 | !kvm_para_has_hint(KVM_HINTS_REALTIME) && |
| 472 | kvm_para_has_feature(KVM_FEATURE_STEAL_TIME)); |
| 473 | } |
| 474 | |
Wanpeng Li | aaffcfd | 2018-07-23 14:39:52 +0800 | [diff] [blame] | 475 | #define KVM_IPI_CLUSTER_SIZE (2 * BITS_PER_LONG) |
| 476 | |
| 477 | static void __send_ipi_mask(const struct cpumask *mask, int vector) |
| 478 | { |
| 479 | unsigned long flags; |
| 480 | int cpu, apic_id, icr; |
| 481 | int min = 0, max = 0; |
| 482 | #ifdef CONFIG_X86_64 |
| 483 | __uint128_t ipi_bitmap = 0; |
| 484 | #else |
| 485 | u64 ipi_bitmap = 0; |
| 486 | #endif |
Sean Christopherson | de81c2f | 2019-01-23 09:22:40 -0800 | [diff] [blame] | 487 | long ret; |
Wanpeng Li | aaffcfd | 2018-07-23 14:39:52 +0800 | [diff] [blame] | 488 | |
| 489 | if (cpumask_empty(mask)) |
| 490 | return; |
| 491 | |
| 492 | local_irq_save(flags); |
| 493 | |
| 494 | switch (vector) { |
| 495 | default: |
| 496 | icr = APIC_DM_FIXED | vector; |
| 497 | break; |
| 498 | case NMI_VECTOR: |
| 499 | icr = APIC_DM_NMI; |
| 500 | break; |
| 501 | } |
| 502 | |
| 503 | for_each_cpu(cpu, mask) { |
| 504 | apic_id = per_cpu(x86_cpu_to_apicid, cpu); |
| 505 | if (!ipi_bitmap) { |
| 506 | min = max = apic_id; |
| 507 | } else if (apic_id < min && max - apic_id < KVM_IPI_CLUSTER_SIZE) { |
| 508 | ipi_bitmap <<= min - apic_id; |
| 509 | min = apic_id; |
| 510 | } else if (apic_id < min + KVM_IPI_CLUSTER_SIZE) { |
| 511 | max = apic_id < max ? max : apic_id; |
| 512 | } else { |
Sean Christopherson | de81c2f | 2019-01-23 09:22:40 -0800 | [diff] [blame] | 513 | ret = kvm_hypercall4(KVM_HC_SEND_IPI, (unsigned long)ipi_bitmap, |
Wanpeng Li | aaffcfd | 2018-07-23 14:39:52 +0800 | [diff] [blame] | 514 | (unsigned long)(ipi_bitmap >> BITS_PER_LONG), min, icr); |
Zhenzhong Duan | 5aefd78 | 2019-10-23 19:16:21 +0800 | [diff] [blame] | 515 | WARN_ONCE(ret < 0, "kvm-guest: failed to send PV IPI: %ld", |
| 516 | ret); |
Wanpeng Li | aaffcfd | 2018-07-23 14:39:52 +0800 | [diff] [blame] | 517 | min = max = apic_id; |
| 518 | ipi_bitmap = 0; |
| 519 | } |
| 520 | __set_bit(apic_id - min, (unsigned long *)&ipi_bitmap); |
| 521 | } |
| 522 | |
| 523 | if (ipi_bitmap) { |
Sean Christopherson | de81c2f | 2019-01-23 09:22:40 -0800 | [diff] [blame] | 524 | ret = kvm_hypercall4(KVM_HC_SEND_IPI, (unsigned long)ipi_bitmap, |
Wanpeng Li | aaffcfd | 2018-07-23 14:39:52 +0800 | [diff] [blame] | 525 | (unsigned long)(ipi_bitmap >> BITS_PER_LONG), min, icr); |
Zhenzhong Duan | 5aefd78 | 2019-10-23 19:16:21 +0800 | [diff] [blame] | 526 | WARN_ONCE(ret < 0, "kvm-guest: failed to send PV IPI: %ld", |
| 527 | ret); |
Wanpeng Li | aaffcfd | 2018-07-23 14:39:52 +0800 | [diff] [blame] | 528 | } |
| 529 | |
| 530 | local_irq_restore(flags); |
| 531 | } |
| 532 | |
| 533 | static void kvm_send_ipi_mask(const struct cpumask *mask, int vector) |
| 534 | { |
| 535 | __send_ipi_mask(mask, vector); |
| 536 | } |
| 537 | |
| 538 | static void kvm_send_ipi_mask_allbutself(const struct cpumask *mask, int vector) |
| 539 | { |
| 540 | unsigned int this_cpu = smp_processor_id(); |
Wanpeng Li | 8a9442f | 2020-02-18 09:08:24 +0800 | [diff] [blame] | 541 | struct cpumask *new_mask = this_cpu_cpumask_var_ptr(__pv_cpu_mask); |
Wanpeng Li | aaffcfd | 2018-07-23 14:39:52 +0800 | [diff] [blame] | 542 | const struct cpumask *local_mask; |
| 543 | |
Wanpeng Li | 8a9442f | 2020-02-18 09:08:24 +0800 | [diff] [blame] | 544 | cpumask_copy(new_mask, mask); |
| 545 | cpumask_clear_cpu(this_cpu, new_mask); |
| 546 | local_mask = new_mask; |
Wanpeng Li | aaffcfd | 2018-07-23 14:39:52 +0800 | [diff] [blame] | 547 | __send_ipi_mask(local_mask, vector); |
| 548 | } |
| 549 | |
Wanpeng Li | aaffcfd | 2018-07-23 14:39:52 +0800 | [diff] [blame] | 550 | /* |
| 551 | * Set the IPI entry points |
| 552 | */ |
| 553 | static void kvm_setup_pv_ipi(void) |
| 554 | { |
| 555 | apic->send_IPI_mask = kvm_send_ipi_mask; |
| 556 | apic->send_IPI_mask_allbutself = kvm_send_ipi_mask_allbutself; |
Zhenzhong Duan | 5aefd78 | 2019-10-23 19:16:21 +0800 | [diff] [blame] | 557 | pr_info("setup PV IPIs\n"); |
Wanpeng Li | aaffcfd | 2018-07-23 14:39:52 +0800 | [diff] [blame] | 558 | } |
| 559 | |
Wanpeng Li | f85f6e7 | 2019-06-11 20:23:48 +0800 | [diff] [blame] | 560 | static void kvm_smp_send_call_func_ipi(const struct cpumask *mask) |
| 561 | { |
| 562 | int cpu; |
| 563 | |
| 564 | native_send_call_func_ipi(mask); |
| 565 | |
| 566 | /* Make sure other vCPUs get a chance to run if they need to. */ |
| 567 | for_each_cpu(cpu, mask) { |
| 568 | if (vcpu_is_preempted(cpu)) { |
| 569 | kvm_hypercall1(KVM_HC_SCHED_YIELD, per_cpu(x86_cpu_to_apicid, cpu)); |
| 570 | break; |
| 571 | } |
| 572 | } |
| 573 | } |
| 574 | |
Linus Torvalds | 152d32a | 2021-05-01 10:14:08 -0700 | [diff] [blame] | 575 | static void kvm_flush_tlb_multi(const struct cpumask *cpumask, |
Wanpeng Li | 2b519b57 | 2021-04-09 12:18:29 +0800 | [diff] [blame] | 576 | const struct flush_tlb_info *info) |
| 577 | { |
| 578 | u8 state; |
| 579 | int cpu; |
| 580 | struct kvm_steal_time *src; |
| 581 | struct cpumask *flushmask = this_cpu_cpumask_var_ptr(__pv_cpu_mask); |
| 582 | |
| 583 | cpumask_copy(flushmask, cpumask); |
| 584 | /* |
| 585 | * We have to call flush only on online vCPUs. And |
| 586 | * queue flush_on_enter for pre-empted vCPUs |
| 587 | */ |
| 588 | for_each_cpu(cpu, flushmask) { |
Linus Torvalds | 152d32a | 2021-05-01 10:14:08 -0700 | [diff] [blame] | 589 | /* |
| 590 | * The local vCPU is never preempted, so we do not explicitly |
| 591 | * skip check for local vCPU - it will never be cleared from |
| 592 | * flushmask. |
| 593 | */ |
Wanpeng Li | 2b519b57 | 2021-04-09 12:18:29 +0800 | [diff] [blame] | 594 | src = &per_cpu(steal_time, cpu); |
| 595 | state = READ_ONCE(src->preempted); |
| 596 | if ((state & KVM_VCPU_PREEMPTED)) { |
| 597 | if (try_cmpxchg(&src->preempted, &state, |
| 598 | state | KVM_VCPU_FLUSH_TLB)) |
| 599 | __cpumask_clear_cpu(cpu, flushmask); |
| 600 | } |
| 601 | } |
| 602 | |
Linus Torvalds | 152d32a | 2021-05-01 10:14:08 -0700 | [diff] [blame] | 603 | native_flush_tlb_multi(flushmask, info); |
Wanpeng Li | 2b519b57 | 2021-04-09 12:18:29 +0800 | [diff] [blame] | 604 | } |
| 605 | |
| 606 | static __init int kvm_alloc_cpumask(void) |
| 607 | { |
| 608 | int cpu; |
| 609 | |
| 610 | if (!kvm_para_available() || nopv) |
| 611 | return 0; |
| 612 | |
| 613 | if (pv_tlb_flush_supported() || pv_ipi_supported()) |
| 614 | for_each_possible_cpu(cpu) { |
| 615 | zalloc_cpumask_var_node(per_cpu_ptr(&__pv_cpu_mask, cpu), |
| 616 | GFP_KERNEL, cpu_to_node(cpu)); |
| 617 | } |
| 618 | |
| 619 | return 0; |
| 620 | } |
| 621 | arch_initcall(kvm_alloc_cpumask); |
| 622 | |
Gleb Natapov | ca3f101 | 2010-10-14 11:22:49 +0200 | [diff] [blame] | 623 | static void __init kvm_smp_prepare_boot_cpu(void) |
| 624 | { |
Brijesh Singh | 4716276 | 2017-10-20 09:30:58 -0500 | [diff] [blame] | 625 | /* |
| 626 | * Map the per-cpu variables as decrypted before kvm_guest_cpu_init() |
| 627 | * shares the guest physical address with the hypervisor. |
| 628 | */ |
| 629 | sev_map_percpu_data(); |
| 630 | |
Gleb Natapov | fd10cde | 2010-10-14 11:22:51 +0200 | [diff] [blame] | 631 | kvm_guest_cpu_init(); |
Gleb Natapov | ca3f101 | 2010-10-14 11:22:49 +0200 | [diff] [blame] | 632 | native_smp_prepare_boot_cpu(); |
Srivatsa Vaddagiri | 92b7520 | 2013-08-06 14:55:41 +0530 | [diff] [blame] | 633 | kvm_spinlock_init(); |
Gleb Natapov | ca3f101 | 2010-10-14 11:22:49 +0200 | [diff] [blame] | 634 | } |
Gleb Natapov | fd10cde | 2010-10-14 11:22:51 +0200 | [diff] [blame] | 635 | |
Sebastian Andrzej Siewior | 9a20ea4 | 2016-08-18 14:57:29 +0200 | [diff] [blame] | 636 | static int kvm_cpu_down_prepare(unsigned int cpu) |
| 637 | { |
Vitaly Kuznetsov | 8b79fef | 2021-04-14 14:35:41 +0200 | [diff] [blame] | 638 | unsigned long flags; |
| 639 | |
| 640 | local_irq_save(flags); |
Vitaly Kuznetsov | 3d6b841 | 2021-04-14 14:35:43 +0200 | [diff] [blame] | 641 | kvm_guest_cpu_offline(false); |
Vitaly Kuznetsov | 8b79fef | 2021-04-14 14:35:41 +0200 | [diff] [blame] | 642 | local_irq_restore(flags); |
Sebastian Andrzej Siewior | 9a20ea4 | 2016-08-18 14:57:29 +0200 | [diff] [blame] | 643 | return 0; |
| 644 | } |
Wanpeng Li | 2b519b57 | 2021-04-09 12:18:29 +0800 | [diff] [blame] | 645 | |
Gleb Natapov | ca3f101 | 2010-10-14 11:22:49 +0200 | [diff] [blame] | 646 | #endif |
| 647 | |
Vitaly Kuznetsov | 8b79fef | 2021-04-14 14:35:41 +0200 | [diff] [blame] | 648 | static int kvm_suspend(void) |
| 649 | { |
Vitaly Kuznetsov | 3d6b841 | 2021-04-14 14:35:43 +0200 | [diff] [blame] | 650 | kvm_guest_cpu_offline(false); |
Vitaly Kuznetsov | 8b79fef | 2021-04-14 14:35:41 +0200 | [diff] [blame] | 651 | |
| 652 | return 0; |
| 653 | } |
| 654 | |
| 655 | static void kvm_resume(void) |
| 656 | { |
| 657 | kvm_cpu_online(raw_smp_processor_id()); |
| 658 | } |
| 659 | |
| 660 | static struct syscore_ops kvm_syscore_ops = { |
| 661 | .suspend = kvm_suspend, |
| 662 | .resume = kvm_resume, |
| 663 | }; |
| 664 | |
Vitaly Kuznetsov | 384fc67 | 2021-04-14 14:35:44 +0200 | [diff] [blame] | 665 | static void kvm_pv_guest_cpu_reboot(void *unused) |
| 666 | { |
| 667 | kvm_guest_cpu_offline(true); |
| 668 | } |
| 669 | |
| 670 | static int kvm_pv_reboot_notify(struct notifier_block *nb, |
| 671 | unsigned long code, void *unused) |
| 672 | { |
| 673 | if (code == SYS_RESTART) |
| 674 | on_each_cpu(kvm_pv_guest_cpu_reboot, NULL, 1); |
| 675 | return NOTIFY_DONE; |
| 676 | } |
| 677 | |
| 678 | static struct notifier_block kvm_pv_reboot_nb = { |
| 679 | .notifier_call = kvm_pv_reboot_notify, |
| 680 | }; |
| 681 | |
Vitaly Kuznetsov | 3d6b841 | 2021-04-14 14:35:43 +0200 | [diff] [blame] | 682 | /* |
| 683 | * After a PV feature is registered, the host will keep writing to the |
| 684 | * registered memory location. If the guest happens to shutdown, this memory |
| 685 | * won't be valid. In cases like kexec, in which you install a new kernel, this |
| 686 | * means a random memory location will be kept being written. |
| 687 | */ |
| 688 | #ifdef CONFIG_KEXEC_CORE |
| 689 | static void kvm_crash_shutdown(struct pt_regs *regs) |
| 690 | { |
| 691 | kvm_guest_cpu_offline(true); |
| 692 | native_machine_crash_shutdown(regs); |
| 693 | } |
| 694 | #endif |
| 695 | |
Juergen Gross | f361464 | 2017-11-09 14:27:38 +0100 | [diff] [blame] | 696 | static void __init kvm_guest_init(void) |
Marcelo Tosatti | 0cf1bfd | 2008-02-22 12:21:36 -0500 | [diff] [blame] | 697 | { |
Gleb Natapov | 631bc48 | 2010-10-14 11:22:52 +0200 | [diff] [blame] | 698 | int i; |
| 699 | |
Marcelo Tosatti | 0cf1bfd | 2008-02-22 12:21:36 -0500 | [diff] [blame] | 700 | paravirt_ops_setup(); |
Gleb Natapov | fd10cde | 2010-10-14 11:22:51 +0200 | [diff] [blame] | 701 | register_reboot_notifier(&kvm_pv_reboot_nb); |
Gleb Natapov | 631bc48 | 2010-10-14 11:22:52 +0200 | [diff] [blame] | 702 | for (i = 0; i < KVM_TASK_SLEEP_HASHSIZE; i++) |
Rik van Riel | 9db284f | 2016-03-21 15:13:27 +0100 | [diff] [blame] | 703 | raw_spin_lock_init(&async_pf_sleepers[i].lock); |
Gleb Natapov | 631bc48 | 2010-10-14 11:22:52 +0200 | [diff] [blame] | 704 | |
Glauber Costa | d910f5c | 2011-07-11 15:28:19 -0400 | [diff] [blame] | 705 | if (kvm_para_has_feature(KVM_FEATURE_STEAL_TIME)) { |
| 706 | has_steal_clock = 1; |
Juergen Gross | a0e2bf7 | 2021-03-11 15:23:09 +0100 | [diff] [blame] | 707 | static_call_update(pv_steal_clock, kvm_steal_clock); |
Glauber Costa | d910f5c | 2011-07-11 15:28:19 -0400 | [diff] [blame] | 708 | } |
| 709 | |
Michael S. Tsirkin | 9053666 | 2012-07-15 15:56:52 +0300 | [diff] [blame] | 710 | if (kvm_para_has_feature(KVM_FEATURE_PV_EOI)) |
| 711 | apic_set_eoi_write(kvm_guest_apic_eoi_write); |
Michael S. Tsirkin | ab9cf49 | 2012-06-24 19:24:34 +0300 | [diff] [blame] | 712 | |
Vitaly Kuznetsov | b1d4057 | 2020-05-25 16:41:23 +0200 | [diff] [blame] | 713 | if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF_INT) && kvmapf) { |
Andy Lutomirski | ef68017 | 2020-02-28 10:42:48 -0800 | [diff] [blame] | 714 | static_branch_enable(&kvm_async_pf_enabled); |
Paolo Bonzini | 26d05b3 | 2020-06-15 07:53:05 -0400 | [diff] [blame] | 715 | alloc_intr_gate(HYPERVISOR_CALLBACK_VECTOR, asm_sysvec_kvm_asyncpf_interrupt); |
Vitaly Kuznetsov | b1d4057 | 2020-05-25 16:41:23 +0200 | [diff] [blame] | 716 | } |
Andy Lutomirski | ef68017 | 2020-02-28 10:42:48 -0800 | [diff] [blame] | 717 | |
Gleb Natapov | ca3f101 | 2010-10-14 11:22:49 +0200 | [diff] [blame] | 718 | #ifdef CONFIG_SMP |
Wanpeng Li | 2b519b57 | 2021-04-09 12:18:29 +0800 | [diff] [blame] | 719 | if (pv_tlb_flush_supported()) { |
Linus Torvalds | 152d32a | 2021-05-01 10:14:08 -0700 | [diff] [blame] | 720 | pv_ops.mmu.flush_tlb_multi = kvm_flush_tlb_multi; |
Wanpeng Li | 2b519b57 | 2021-04-09 12:18:29 +0800 | [diff] [blame] | 721 | pv_ops.mmu.tlb_remove_table = tlb_remove_table; |
| 722 | pr_info("KVM setup pv remote TLB flush\n"); |
| 723 | } |
| 724 | |
Gleb Natapov | ca3f101 | 2010-10-14 11:22:49 +0200 | [diff] [blame] | 725 | smp_ops.smp_prepare_boot_cpu = kvm_smp_prepare_boot_cpu; |
Wanpeng Li | a262bca | 2020-02-18 09:08:23 +0800 | [diff] [blame] | 726 | if (pv_sched_yield_supported()) { |
Wanpeng Li | f85f6e7 | 2019-06-11 20:23:48 +0800 | [diff] [blame] | 727 | smp_ops.send_call_func_ipi = kvm_smp_send_call_func_ipi; |
Zhenzhong Duan | 5aefd78 | 2019-10-23 19:16:21 +0800 | [diff] [blame] | 728 | pr_info("setup PV sched yield\n"); |
Wanpeng Li | f85f6e7 | 2019-06-11 20:23:48 +0800 | [diff] [blame] | 729 | } |
Sebastian Andrzej Siewior | 9a20ea4 | 2016-08-18 14:57:29 +0200 | [diff] [blame] | 730 | if (cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "x86/kvm:online", |
| 731 | kvm_cpu_online, kvm_cpu_down_prepare) < 0) |
Zhenzhong Duan | 5aefd78 | 2019-10-23 19:16:21 +0800 | [diff] [blame] | 732 | pr_err("failed to install cpu hotplug callbacks\n"); |
Gleb Natapov | fd10cde | 2010-10-14 11:22:51 +0200 | [diff] [blame] | 733 | #else |
Brijesh Singh | 4716276 | 2017-10-20 09:30:58 -0500 | [diff] [blame] | 734 | sev_map_percpu_data(); |
Gleb Natapov | fd10cde | 2010-10-14 11:22:51 +0200 | [diff] [blame] | 735 | kvm_guest_cpu_init(); |
Gleb Natapov | ca3f101 | 2010-10-14 11:22:49 +0200 | [diff] [blame] | 736 | #endif |
Ulrich Obergfell | 9919e39 | 2014-10-13 15:55:37 -0700 | [diff] [blame] | 737 | |
Vitaly Kuznetsov | 3d6b841 | 2021-04-14 14:35:43 +0200 | [diff] [blame] | 738 | #ifdef CONFIG_KEXEC_CORE |
| 739 | machine_ops.crash_shutdown = kvm_crash_shutdown; |
| 740 | #endif |
| 741 | |
Vitaly Kuznetsov | 8b79fef | 2021-04-14 14:35:41 +0200 | [diff] [blame] | 742 | register_syscore_ops(&kvm_syscore_ops); |
| 743 | |
Ulrich Obergfell | 9919e39 | 2014-10-13 15:55:37 -0700 | [diff] [blame] | 744 | /* |
| 745 | * Hard lockup detection is enabled by default. Disable it, as guests |
| 746 | * can get false positives too easily, for example if the host is |
| 747 | * overcommitted. |
| 748 | */ |
Ulrich Obergfell | 692297d | 2015-04-14 15:44:19 -0700 | [diff] [blame] | 749 | hardlockup_detector_disable(); |
Marcelo Tosatti | 0cf1bfd | 2008-02-22 12:21:36 -0500 | [diff] [blame] | 750 | } |
Glauber Costa | d910f5c | 2011-07-11 15:28:19 -0400 | [diff] [blame] | 751 | |
Paolo Bonzini | 1c300a4 | 2014-01-27 14:49:40 +0100 | [diff] [blame] | 752 | static noinline uint32_t __kvm_cpuid_base(void) |
| 753 | { |
| 754 | if (boot_cpu_data.cpuid_level < 0) |
| 755 | return 0; /* So we don't blow up on old processors */ |
| 756 | |
Borislav Petkov | 0c9f3536 | 2016-03-29 17:41:55 +0200 | [diff] [blame] | 757 | if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) |
Paul Durrant | 760849b | 2021-11-05 09:51:01 +0000 | [diff] [blame^] | 758 | return hypervisor_cpuid_base(KVM_SIGNATURE, 0); |
Paolo Bonzini | 1c300a4 | 2014-01-27 14:49:40 +0100 | [diff] [blame] | 759 | |
| 760 | return 0; |
| 761 | } |
| 762 | |
| 763 | static inline uint32_t kvm_cpuid_base(void) |
| 764 | { |
| 765 | static int kvm_cpuid_base = -1; |
| 766 | |
| 767 | if (kvm_cpuid_base == -1) |
| 768 | kvm_cpuid_base = __kvm_cpuid_base(); |
| 769 | |
| 770 | return kvm_cpuid_base; |
| 771 | } |
| 772 | |
| 773 | bool kvm_para_available(void) |
| 774 | { |
| 775 | return kvm_cpuid_base() != 0; |
| 776 | } |
| 777 | EXPORT_SYMBOL_GPL(kvm_para_available); |
| 778 | |
Paolo Bonzini | 77f01bd | 2014-01-27 14:51:44 +0100 | [diff] [blame] | 779 | unsigned int kvm_arch_para_features(void) |
| 780 | { |
| 781 | return cpuid_eax(kvm_cpuid_base() | KVM_CPUID_FEATURES); |
| 782 | } |
| 783 | |
Wanpeng Li | a4429e5 | 2018-02-13 09:05:40 +0800 | [diff] [blame] | 784 | unsigned int kvm_arch_para_hints(void) |
| 785 | { |
| 786 | return cpuid_edx(kvm_cpuid_base() | KVM_CPUID_FEATURES); |
| 787 | } |
Wanpeng Li | 1328edc | 2019-08-29 16:49:57 +0800 | [diff] [blame] | 788 | EXPORT_SYMBOL_GPL(kvm_arch_para_hints); |
Wanpeng Li | a4429e5 | 2018-02-13 09:05:40 +0800 | [diff] [blame] | 789 | |
Jason Wang | 9df56f1 | 2013-07-25 16:54:35 +0800 | [diff] [blame] | 790 | static uint32_t __init kvm_detect(void) |
Prarit Bhargava | fc73373 | 2012-07-06 13:47:39 -0400 | [diff] [blame] | 791 | { |
Jason Wang | 9df56f1 | 2013-07-25 16:54:35 +0800 | [diff] [blame] | 792 | return kvm_cpuid_base(); |
Prarit Bhargava | fc73373 | 2012-07-06 13:47:39 -0400 | [diff] [blame] | 793 | } |
| 794 | |
Wanpeng Li | d63bae0 | 2018-07-23 14:39:51 +0800 | [diff] [blame] | 795 | static void __init kvm_apic_init(void) |
| 796 | { |
Wanpeng Li | 2b519b57 | 2021-04-09 12:18:29 +0800 | [diff] [blame] | 797 | #ifdef CONFIG_SMP |
Wanpeng Li | a262bca | 2020-02-18 09:08:23 +0800 | [diff] [blame] | 798 | if (pv_ipi_supported()) |
Wanpeng Li | aaffcfd | 2018-07-23 14:39:52 +0800 | [diff] [blame] | 799 | kvm_setup_pv_ipi(); |
| 800 | #endif |
Wanpeng Li | d63bae0 | 2018-07-23 14:39:51 +0800 | [diff] [blame] | 801 | } |
| 802 | |
David Woodhouse | 2e008ff | 2020-10-24 22:35:35 +0100 | [diff] [blame] | 803 | static bool __init kvm_msi_ext_dest_id(void) |
| 804 | { |
| 805 | return kvm_para_has_feature(KVM_FEATURE_MSI_EXT_DEST_ID); |
| 806 | } |
| 807 | |
Wanpeng Li | d63bae0 | 2018-07-23 14:39:51 +0800 | [diff] [blame] | 808 | static void __init kvm_init_platform(void) |
| 809 | { |
Linus Torvalds | e61cf2e | 2018-08-19 10:38:36 -0700 | [diff] [blame] | 810 | kvmclock_init(); |
Wanpeng Li | d63bae0 | 2018-07-23 14:39:51 +0800 | [diff] [blame] | 811 | x86_platform.apic_post_init = kvm_apic_init; |
| 812 | } |
| 813 | |
Tom Lendacky | 99419b2 | 2020-09-07 15:16:04 +0200 | [diff] [blame] | 814 | #if defined(CONFIG_AMD_MEM_ENCRYPT) |
| 815 | static void kvm_sev_es_hcall_prepare(struct ghcb *ghcb, struct pt_regs *regs) |
| 816 | { |
| 817 | /* RAX and CPL are already in the GHCB */ |
| 818 | ghcb_set_rbx(ghcb, regs->bx); |
| 819 | ghcb_set_rcx(ghcb, regs->cx); |
| 820 | ghcb_set_rdx(ghcb, regs->dx); |
| 821 | ghcb_set_rsi(ghcb, regs->si); |
| 822 | } |
| 823 | |
| 824 | static bool kvm_sev_es_hcall_finish(struct ghcb *ghcb, struct pt_regs *regs) |
| 825 | { |
| 826 | /* No checking of the return state needed */ |
| 827 | return true; |
| 828 | } |
| 829 | #endif |
| 830 | |
Juergen Gross | 03b2a32 | 2017-11-09 14:27:36 +0100 | [diff] [blame] | 831 | const __initconst struct hypervisor_x86 x86_hyper_kvm = { |
Tom Lendacky | 99419b2 | 2020-09-07 15:16:04 +0200 | [diff] [blame] | 832 | .name = "KVM", |
| 833 | .detect = kvm_detect, |
| 834 | .type = X86_HYPER_KVM, |
| 835 | .init.guest_late_init = kvm_guest_init, |
| 836 | .init.x2apic_available = kvm_para_available, |
David Woodhouse | 2e008ff | 2020-10-24 22:35:35 +0100 | [diff] [blame] | 837 | .init.msi_ext_dest_id = kvm_msi_ext_dest_id, |
Tom Lendacky | 99419b2 | 2020-09-07 15:16:04 +0200 | [diff] [blame] | 838 | .init.init_platform = kvm_init_platform, |
| 839 | #if defined(CONFIG_AMD_MEM_ENCRYPT) |
| 840 | .runtime.sev_es_hcall_prepare = kvm_sev_es_hcall_prepare, |
| 841 | .runtime.sev_es_hcall_finish = kvm_sev_es_hcall_finish, |
| 842 | #endif |
Prarit Bhargava | fc73373 | 2012-07-06 13:47:39 -0400 | [diff] [blame] | 843 | }; |
Prarit Bhargava | fc73373 | 2012-07-06 13:47:39 -0400 | [diff] [blame] | 844 | |
Glauber Costa | d910f5c | 2011-07-11 15:28:19 -0400 | [diff] [blame] | 845 | static __init int activate_jump_labels(void) |
| 846 | { |
| 847 | if (has_steal_clock) { |
Ingo Molnar | c5905af | 2012-02-24 08:31:31 +0100 | [diff] [blame] | 848 | static_key_slow_inc(¶virt_steal_enabled); |
Glauber Costa | d910f5c | 2011-07-11 15:28:19 -0400 | [diff] [blame] | 849 | if (steal_acc) |
Ingo Molnar | c5905af | 2012-02-24 08:31:31 +0100 | [diff] [blame] | 850 | static_key_slow_inc(¶virt_steal_rq_enabled); |
Glauber Costa | d910f5c | 2011-07-11 15:28:19 -0400 | [diff] [blame] | 851 | } |
| 852 | |
| 853 | return 0; |
| 854 | } |
| 855 | arch_initcall(activate_jump_labels); |
Srivatsa Vaddagiri | 92b7520 | 2013-08-06 14:55:41 +0530 | [diff] [blame] | 856 | |
| 857 | #ifdef CONFIG_PARAVIRT_SPINLOCKS |
| 858 | |
| 859 | /* Kick a cpu by its apicid. Used to wake up a halted vcpu */ |
Raghavendra K T | 36bd621 | 2013-08-16 15:08:41 +0530 | [diff] [blame] | 860 | static void kvm_kick_cpu(int cpu) |
Srivatsa Vaddagiri | 92b7520 | 2013-08-06 14:55:41 +0530 | [diff] [blame] | 861 | { |
| 862 | int apicid; |
| 863 | unsigned long flags = 0; |
| 864 | |
| 865 | apicid = per_cpu(x86_cpu_to_apicid, cpu); |
| 866 | kvm_hypercall2(KVM_HC_KICK_CPU, flags, apicid); |
| 867 | } |
| 868 | |
Waiman Long | bf0c7c3 | 2015-04-24 14:56:39 -0400 | [diff] [blame] | 869 | #include <asm/qspinlock.h> |
| 870 | |
| 871 | static void kvm_wait(u8 *ptr, u8 val) |
| 872 | { |
Waiman Long | bf0c7c3 | 2015-04-24 14:56:39 -0400 | [diff] [blame] | 873 | if (in_nmi()) |
| 874 | return; |
| 875 | |
Waiman Long | bf0c7c3 | 2015-04-24 14:56:39 -0400 | [diff] [blame] | 876 | /* |
| 877 | * halt until it's our turn and kicked. Note that we do safe halt |
| 878 | * for irq enabled case to avoid hang when lock info is overwritten |
| 879 | * in irq spinlock slowpath and no spurious interrupt occur to save us. |
| 880 | */ |
Wanpeng Li | f4e61f0 | 2021-03-15 14:55:28 +0800 | [diff] [blame] | 881 | if (irqs_disabled()) { |
| 882 | if (READ_ONCE(*ptr) == val) |
| 883 | halt(); |
| 884 | } else { |
| 885 | local_irq_disable(); |
Waiman Long | bf0c7c3 | 2015-04-24 14:56:39 -0400 | [diff] [blame] | 886 | |
Lai Jiangshan | a40b2fd | 2021-08-14 11:51:29 +0800 | [diff] [blame] | 887 | /* safe_halt() will enable IRQ */ |
Wanpeng Li | f4e61f0 | 2021-03-15 14:55:28 +0800 | [diff] [blame] | 888 | if (READ_ONCE(*ptr) == val) |
| 889 | safe_halt(); |
Lai Jiangshan | a40b2fd | 2021-08-14 11:51:29 +0800 | [diff] [blame] | 890 | else |
| 891 | local_irq_enable(); |
Wanpeng Li | f4e61f0 | 2021-03-15 14:55:28 +0800 | [diff] [blame] | 892 | } |
Waiman Long | bf0c7c3 | 2015-04-24 14:56:39 -0400 | [diff] [blame] | 893 | } |
| 894 | |
Waiman Long | dd0fd8b | 2017-02-20 13:36:04 -0500 | [diff] [blame] | 895 | #ifdef CONFIG_X86_32 |
Waiman Long | 6c62985 | 2017-02-20 13:36:03 -0500 | [diff] [blame] | 896 | __visible bool __kvm_vcpu_is_preempted(long cpu) |
Peter Zijlstra | 3cded41 | 2016-11-15 16:47:06 +0100 | [diff] [blame] | 897 | { |
| 898 | struct kvm_steal_time *src = &per_cpu(steal_time, cpu); |
| 899 | |
Wanpeng Li | fa55eed | 2017-12-12 17:33:01 -0800 | [diff] [blame] | 900 | return !!(src->preempted & KVM_VCPU_PREEMPTED); |
Peter Zijlstra | 3cded41 | 2016-11-15 16:47:06 +0100 | [diff] [blame] | 901 | } |
| 902 | PV_CALLEE_SAVE_REGS_THUNK(__kvm_vcpu_is_preempted); |
| 903 | |
Waiman Long | dd0fd8b | 2017-02-20 13:36:04 -0500 | [diff] [blame] | 904 | #else |
| 905 | |
| 906 | #include <asm/asm-offsets.h> |
| 907 | |
| 908 | extern bool __raw_callee_save___kvm_vcpu_is_preempted(long); |
| 909 | |
| 910 | /* |
| 911 | * Hand-optimize version for x86-64 to avoid 8 64-bit register saving and |
| 912 | * restoring to/from the stack. |
| 913 | */ |
| 914 | asm( |
| 915 | ".pushsection .text;" |
| 916 | ".global __raw_callee_save___kvm_vcpu_is_preempted;" |
| 917 | ".type __raw_callee_save___kvm_vcpu_is_preempted, @function;" |
| 918 | "__raw_callee_save___kvm_vcpu_is_preempted:" |
| 919 | "movq __per_cpu_offset(,%rdi,8), %rax;" |
| 920 | "cmpb $0, " __stringify(KVM_STEAL_TIME_preempted) "+steal_time(%rax);" |
| 921 | "setne %al;" |
| 922 | "ret;" |
Josh Poimboeuf | 083db67 | 2019-07-17 20:36:36 -0500 | [diff] [blame] | 923 | ".size __raw_callee_save___kvm_vcpu_is_preempted, .-__raw_callee_save___kvm_vcpu_is_preempted;" |
Waiman Long | dd0fd8b | 2017-02-20 13:36:04 -0500 | [diff] [blame] | 924 | ".popsection"); |
| 925 | |
| 926 | #endif |
| 927 | |
Srivatsa Vaddagiri | 92b7520 | 2013-08-06 14:55:41 +0530 | [diff] [blame] | 928 | /* |
| 929 | * Setup pv_lock_ops to exploit KVM_FEATURE_PV_UNHALT if present. |
| 930 | */ |
| 931 | void __init kvm_spinlock_init(void) |
| 932 | { |
Zhenzhong Duan | 05eee61 | 2019-10-23 19:16:22 +0800 | [diff] [blame] | 933 | /* |
| 934 | * In case host doesn't support KVM_FEATURE_PV_UNHALT there is still an |
| 935 | * advantage of keeping virt_spin_lock_key enabled: virt_spin_lock() is |
| 936 | * preferred over native qspinlock when vCPU is preempted. |
| 937 | */ |
| 938 | if (!kvm_para_has_feature(KVM_FEATURE_PV_UNHALT)) { |
| 939 | pr_info("PV spinlocks disabled, no host support\n"); |
Srivatsa Vaddagiri | 92b7520 | 2013-08-06 14:55:41 +0530 | [diff] [blame] | 940 | return; |
Zhenzhong Duan | de58502 | 2019-10-23 19:16:20 +0800 | [diff] [blame] | 941 | } |
Srivatsa Vaddagiri | 92b7520 | 2013-08-06 14:55:41 +0530 | [diff] [blame] | 942 | |
Zhenzhong Duan | 05eee61 | 2019-10-23 19:16:22 +0800 | [diff] [blame] | 943 | /* |
| 944 | * Disable PV spinlocks and use native qspinlock when dedicated pCPUs |
| 945 | * are available. |
| 946 | */ |
| 947 | if (kvm_para_has_hint(KVM_HINTS_REALTIME)) { |
| 948 | pr_info("PV spinlocks disabled with KVM_HINTS_REALTIME hints\n"); |
| 949 | goto out; |
| 950 | } |
Wanpeng Li | b2798ba | 2018-02-13 09:05:41 +0800 | [diff] [blame] | 951 | |
Zhenzhong Duan | 05eee61 | 2019-10-23 19:16:22 +0800 | [diff] [blame] | 952 | if (num_possible_cpus() == 1) { |
| 953 | pr_info("PV spinlocks disabled, single CPU\n"); |
| 954 | goto out; |
| 955 | } |
| 956 | |
| 957 | if (nopvspin) { |
| 958 | pr_info("PV spinlocks disabled, forced by \"nopvspin\" parameter\n"); |
| 959 | goto out; |
| 960 | } |
| 961 | |
| 962 | pr_info("PV spinlocks enabled\n"); |
Waiman Long | 3553ae5 | 2018-07-17 17:59:27 -0400 | [diff] [blame] | 963 | |
Waiman Long | bf0c7c3 | 2015-04-24 14:56:39 -0400 | [diff] [blame] | 964 | __pv_init_lock_hash(); |
Juergen Gross | 5c83511 | 2018-08-28 09:40:19 +0200 | [diff] [blame] | 965 | pv_ops.lock.queued_spin_lock_slowpath = __pv_queued_spin_lock_slowpath; |
| 966 | pv_ops.lock.queued_spin_unlock = |
| 967 | PV_CALLEE_SAVE(__pv_queued_spin_unlock); |
| 968 | pv_ops.lock.wait = kvm_wait; |
| 969 | pv_ops.lock.kick = kvm_kick_cpu; |
Peter Zijlstra | 3cded41 | 2016-11-15 16:47:06 +0100 | [diff] [blame] | 970 | |
| 971 | if (kvm_para_has_feature(KVM_FEATURE_STEAL_TIME)) { |
Juergen Gross | 5c83511 | 2018-08-28 09:40:19 +0200 | [diff] [blame] | 972 | pv_ops.lock.vcpu_is_preempted = |
Peter Zijlstra | 3cded41 | 2016-11-15 16:47:06 +0100 | [diff] [blame] | 973 | PV_CALLEE_SAVE(__kvm_vcpu_is_preempted); |
| 974 | } |
Zhenzhong Duan | 05eee61 | 2019-10-23 19:16:22 +0800 | [diff] [blame] | 975 | /* |
| 976 | * When PV spinlock is enabled which is preferred over |
| 977 | * virt_spin_lock(), virt_spin_lock_key's value is meaningless. |
| 978 | * Just disable it anyway. |
| 979 | */ |
| 980 | out: |
| 981 | static_branch_disable(&virt_spin_lock_key); |
Srivatsa Vaddagiri | 92b7520 | 2013-08-06 14:55:41 +0530 | [diff] [blame] | 982 | } |
Raghavendra K T | 3dbef3e | 2013-10-09 14:33:21 +0530 | [diff] [blame] | 983 | |
Srivatsa Vaddagiri | 92b7520 | 2013-08-06 14:55:41 +0530 | [diff] [blame] | 984 | #endif /* CONFIG_PARAVIRT_SPINLOCKS */ |
Marcelo Tosatti | a1c4423 | 2019-07-03 20:51:29 -0300 | [diff] [blame] | 985 | |
| 986 | #ifdef CONFIG_ARCH_CPUIDLE_HALTPOLL |
| 987 | |
| 988 | static void kvm_disable_host_haltpoll(void *i) |
| 989 | { |
| 990 | wrmsrl(MSR_KVM_POLL_CONTROL, 0); |
| 991 | } |
| 992 | |
| 993 | static void kvm_enable_host_haltpoll(void *i) |
| 994 | { |
| 995 | wrmsrl(MSR_KVM_POLL_CONTROL, 1); |
| 996 | } |
| 997 | |
Joao Martins | 97d3eb9 | 2019-09-02 11:40:31 +0100 | [diff] [blame] | 998 | void arch_haltpoll_enable(unsigned int cpu) |
Marcelo Tosatti | a1c4423 | 2019-07-03 20:51:29 -0300 | [diff] [blame] | 999 | { |
| 1000 | if (!kvm_para_has_feature(KVM_FEATURE_POLL_CONTROL)) { |
Zhenzhong Duan | 5aefd78 | 2019-10-23 19:16:21 +0800 | [diff] [blame] | 1001 | pr_err_once("host does not support poll control\n"); |
| 1002 | pr_err_once("host upgrade recommended\n"); |
Marcelo Tosatti | a1c4423 | 2019-07-03 20:51:29 -0300 | [diff] [blame] | 1003 | return; |
| 1004 | } |
| 1005 | |
Marcelo Tosatti | a1c4423 | 2019-07-03 20:51:29 -0300 | [diff] [blame] | 1006 | /* Enable guest halt poll disables host halt poll */ |
Joao Martins | 97d3eb9 | 2019-09-02 11:40:31 +0100 | [diff] [blame] | 1007 | smp_call_function_single(cpu, kvm_disable_host_haltpoll, NULL, 1); |
Marcelo Tosatti | a1c4423 | 2019-07-03 20:51:29 -0300 | [diff] [blame] | 1008 | } |
| 1009 | EXPORT_SYMBOL_GPL(arch_haltpoll_enable); |
| 1010 | |
Joao Martins | 97d3eb9 | 2019-09-02 11:40:31 +0100 | [diff] [blame] | 1011 | void arch_haltpoll_disable(unsigned int cpu) |
Marcelo Tosatti | a1c4423 | 2019-07-03 20:51:29 -0300 | [diff] [blame] | 1012 | { |
| 1013 | if (!kvm_para_has_feature(KVM_FEATURE_POLL_CONTROL)) |
| 1014 | return; |
| 1015 | |
Li Qiang | b785a44 | 2020-09-24 08:58:00 -0700 | [diff] [blame] | 1016 | /* Disable guest halt poll enables host halt poll */ |
Joao Martins | 97d3eb9 | 2019-09-02 11:40:31 +0100 | [diff] [blame] | 1017 | smp_call_function_single(cpu, kvm_enable_host_haltpoll, NULL, 1); |
Marcelo Tosatti | a1c4423 | 2019-07-03 20:51:29 -0300 | [diff] [blame] | 1018 | } |
| 1019 | EXPORT_SYMBOL_GPL(arch_haltpoll_disable); |
| 1020 | #endif |