blob: dd2a7c6ec2cad0b9bed8b75b6ad16d82932c8090 [file] [log] [blame]
Avi Kivity6aa8b732006-12-10 02:21:36 -08001/*
2 * Kernel-based Virtual Machine driver for Linux
3 *
4 * This module enables machines with Intel VT-x extensions to run virtual
5 * machines without emulation or binary translation.
6 *
7 * MMU support
8 *
9 * Copyright (C) 2006 Qumranet, Inc.
Nicolas Kaiser9611c182010-10-06 14:23:22 +020010 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
Avi Kivity6aa8b732006-12-10 02:21:36 -080011 *
12 * Authors:
13 * Yaniv Kamay <yaniv@qumranet.com>
14 * Avi Kivity <avi@qumranet.com>
15 *
16 * This work is licensed under the terms of the GNU GPL, version 2. See
17 * the COPYING file in the top-level directory.
18 *
19 */
Avi Kivity6aa8b732006-12-10 02:21:36 -080020
Gleb Natapovaf585b92010-10-14 11:22:46 +020021#include "irq.h"
Zhang Xiantao1d737c82007-12-14 09:35:10 +080022#include "mmu.h"
Avi Kivity836a1b32010-01-21 15:31:49 +020023#include "x86.h"
Avi Kivity6de4f3a2009-05-31 22:58:47 +030024#include "kvm_cache_regs.h"
Nadav Amit5f7dde72014-05-07 15:32:50 +030025#include "cpuid.h"
Avi Kivity6aa8b732006-12-10 02:21:36 -080026
Avi Kivityedf88412007-12-16 11:02:48 +020027#include <linux/kvm_host.h>
Avi Kivitye4956062007-06-28 14:15:57 -040028#include <linux/types.h>
29#include <linux/string.h>
30#include <linux/mm.h>
31#include <linux/highmem.h>
32#include <linux/module.h>
Izik Eidus448353c2007-11-26 14:08:14 +020033#include <linux/swap.h>
Marcelo Tosatti05da4552008-02-23 11:44:30 -030034#include <linux/hugetlb.h>
Marcelo Tosatti2f333bc2008-02-22 12:21:37 -050035#include <linux/compiler.h>
Marcelo Tosattibc6678a2009-12-23 14:35:21 -020036#include <linux/srcu.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090037#include <linux/slab.h>
Huang Yingbf998152010-05-31 14:28:19 +080038#include <linux/uaccess.h>
Avi Kivitye4956062007-06-28 14:15:57 -040039
40#include <asm/page.h>
41#include <asm/cmpxchg.h>
Avi Kivity4e542372007-11-21 14:08:40 +020042#include <asm/io.h>
Eduardo Habkost13673a92008-11-17 19:03:13 -020043#include <asm/vmx.h>
Avi Kivitye4956062007-06-28 14:15:57 -040044
Joerg Roedel18552672008-02-07 13:47:41 +010045/*
46 * When setting this variable to true it enables Two-Dimensional-Paging
47 * where the hardware walks 2 page tables:
48 * 1. the guest-virtual to guest-physical
49 * 2. while doing 1. it walks guest-physical to host-physical
50 * If the hardware supports that we don't need to do shadow paging.
51 */
Marcelo Tosatti2f333bc2008-02-22 12:21:37 -050052bool tdp_enabled = false;
Joerg Roedel18552672008-02-07 13:47:41 +010053
Xiao Guangrong8b1fe172010-08-30 18:22:53 +080054enum {
55 AUDIT_PRE_PAGE_FAULT,
56 AUDIT_POST_PAGE_FAULT,
57 AUDIT_PRE_PTE_WRITE,
Xiao Guangrong69030742010-09-27 18:09:29 +080058 AUDIT_POST_PTE_WRITE,
59 AUDIT_PRE_SYNC,
60 AUDIT_POST_SYNC
Xiao Guangrong8b1fe172010-08-30 18:22:53 +080061};
62
Avi Kivity37a7d8b2007-01-05 16:36:56 -080063#undef MMU_DEBUG
64
Avi Kivity37a7d8b2007-01-05 16:36:56 -080065#ifdef MMU_DEBUG
Paolo Bonzinifa4a2c02013-10-02 16:56:16 +020066static bool dbg = 0;
67module_param(dbg, bool, 0644);
Avi Kivity37a7d8b2007-01-05 16:36:56 -080068
69#define pgprintk(x...) do { if (dbg) printk(x); } while (0)
70#define rmap_printk(x...) do { if (dbg) printk(x); } while (0)
Paolo Bonzinifa4a2c02013-10-02 16:56:16 +020071#define MMU_WARN_ON(x) WARN_ON(x)
Avi Kivity37a7d8b2007-01-05 16:36:56 -080072#else
Avi Kivity37a7d8b2007-01-05 16:36:56 -080073#define pgprintk(x...) do { } while (0)
74#define rmap_printk(x...) do { } while (0)
Paolo Bonzinifa4a2c02013-10-02 16:56:16 +020075#define MMU_WARN_ON(x) do { } while (0)
Yaozu Dongd6c69ee2007-04-25 14:17:25 +080076#endif
Avi Kivity6aa8b732006-12-10 02:21:36 -080077
Xiao Guangrong957ed9e2010-08-22 19:12:48 +080078#define PTE_PREFETCH_NUM 8
79
Xudong Hao00763e42012-06-07 18:26:07 +080080#define PT_FIRST_AVAIL_BITS_SHIFT 10
Avi Kivity6aa8b732006-12-10 02:21:36 -080081#define PT64_SECOND_AVAIL_BITS_SHIFT 52
82
Avi Kivity6aa8b732006-12-10 02:21:36 -080083#define PT64_LEVEL_BITS 9
84
85#define PT64_LEVEL_SHIFT(level) \
Mike Dayd77c26f2007-10-08 09:02:08 -040086 (PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS)
Avi Kivity6aa8b732006-12-10 02:21:36 -080087
Avi Kivity6aa8b732006-12-10 02:21:36 -080088#define PT64_INDEX(address, level)\
89 (((address) >> PT64_LEVEL_SHIFT(level)) & ((1 << PT64_LEVEL_BITS) - 1))
90
91
92#define PT32_LEVEL_BITS 10
93
94#define PT32_LEVEL_SHIFT(level) \
Mike Dayd77c26f2007-10-08 09:02:08 -040095 (PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS)
Avi Kivity6aa8b732006-12-10 02:21:36 -080096
Joerg Roedele04da982009-07-27 16:30:45 +020097#define PT32_LVL_OFFSET_MASK(level) \
98 (PT32_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \
99 * PT32_LEVEL_BITS))) - 1))
Avi Kivity6aa8b732006-12-10 02:21:36 -0800100
101#define PT32_INDEX(address, level)\
102 (((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1))
103
104
Avi Kivity27aba762007-03-09 13:04:31 +0200105#define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))
Avi Kivity6aa8b732006-12-10 02:21:36 -0800106#define PT64_DIR_BASE_ADDR_MASK \
107 (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + PT64_LEVEL_BITS)) - 1))
Joerg Roedele04da982009-07-27 16:30:45 +0200108#define PT64_LVL_ADDR_MASK(level) \
109 (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \
110 * PT64_LEVEL_BITS))) - 1))
111#define PT64_LVL_OFFSET_MASK(level) \
112 (PT64_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \
113 * PT64_LEVEL_BITS))) - 1))
Avi Kivity6aa8b732006-12-10 02:21:36 -0800114
115#define PT32_BASE_ADDR_MASK PAGE_MASK
116#define PT32_DIR_BASE_ADDR_MASK \
117 (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1))
Joerg Roedele04da982009-07-27 16:30:45 +0200118#define PT32_LVL_ADDR_MASK(level) \
119 (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \
120 * PT32_LEVEL_BITS))) - 1))
Avi Kivity6aa8b732006-12-10 02:21:36 -0800121
Gleb Natapov53166222013-08-05 11:07:14 +0300122#define PT64_PERM_MASK (PT_PRESENT_MASK | PT_WRITABLE_MASK | shadow_user_mask \
123 | shadow_x_mask | shadow_nx_mask)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800124
Avi Kivityfe135d22007-12-09 16:15:46 +0200125#define ACC_EXEC_MASK 1
126#define ACC_WRITE_MASK PT_WRITABLE_MASK
127#define ACC_USER_MASK PT_USER_MASK
128#define ACC_ALL (ACC_EXEC_MASK | ACC_WRITE_MASK | ACC_USER_MASK)
129
Avi Kivity90bb6fc2009-12-31 12:10:16 +0200130#include <trace/events/kvm.h>
131
Avi Kivity07420172009-07-06 12:21:32 +0300132#define CREATE_TRACE_POINTS
133#include "mmutrace.h"
134
Xiao Guangrong49fde342012-06-20 15:58:58 +0800135#define SPTE_HOST_WRITEABLE (1ULL << PT_FIRST_AVAIL_BITS_SHIFT)
136#define SPTE_MMU_WRITEABLE (1ULL << (PT_FIRST_AVAIL_BITS_SHIFT + 1))
Izik Eidus14032832009-09-23 21:47:17 +0300137
Avi Kivity135f8c22008-08-21 17:49:56 +0300138#define SHADOW_PT_INDEX(addr, level) PT64_INDEX(addr, level)
139
Takuya Yoshikawa220f7732012-03-21 23:49:39 +0900140/* make pte_list_desc fit well in cache line */
141#define PTE_LIST_EXT 3
142
Xiao Guangrong53c07b12011-05-15 23:26:20 +0800143struct pte_list_desc {
144 u64 *sptes[PTE_LIST_EXT];
145 struct pte_list_desc *more;
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800146};
147
Avi Kivity2d111232008-12-25 14:39:47 +0200148struct kvm_shadow_walk_iterator {
149 u64 addr;
150 hpa_t shadow_addr;
Avi Kivity2d111232008-12-25 14:39:47 +0200151 u64 *sptep;
Xiao Guangrongdd3bfd52011-07-12 03:32:54 +0800152 int level;
Avi Kivity2d111232008-12-25 14:39:47 +0200153 unsigned index;
154};
155
156#define for_each_shadow_entry(_vcpu, _addr, _walker) \
157 for (shadow_walk_init(&(_walker), _vcpu, _addr); \
158 shadow_walk_okay(&(_walker)); \
159 shadow_walk_next(&(_walker)))
160
Xiao Guangrongc2a2ac22011-07-12 03:32:13 +0800161#define for_each_shadow_entry_lockless(_vcpu, _addr, _walker, spte) \
162 for (shadow_walk_init(&(_walker), _vcpu, _addr); \
163 shadow_walk_okay(&(_walker)) && \
164 ({ spte = mmu_spte_get_lockless(_walker.sptep); 1; }); \
165 __shadow_walk_next(&(_walker), spte))
166
Xiao Guangrong53c07b12011-05-15 23:26:20 +0800167static struct kmem_cache *pte_list_desc_cache;
Avi Kivityd3d25b02007-05-30 12:34:53 +0300168static struct kmem_cache *mmu_page_header_cache;
Dave Hansen45221ab2010-08-19 18:11:37 -0700169static struct percpu_counter kvm_total_used_mmu_pages;
Avi Kivityb5a33a72007-04-15 16:31:09 +0300170
Sheng Yang7b523452008-04-25 21:13:50 +0800171static u64 __read_mostly shadow_nx_mask;
172static u64 __read_mostly shadow_x_mask; /* mutual exclusive with nx_mask */
173static u64 __read_mostly shadow_user_mask;
174static u64 __read_mostly shadow_accessed_mask;
175static u64 __read_mostly shadow_dirty_mask;
Xiao Guangrongce88dec2011-07-12 03:33:44 +0800176static u64 __read_mostly shadow_mmio_mask;
177
178static void mmu_spte_set(u64 *sptep, u64 spte);
Avi Kivitye6765052012-07-08 17:16:30 +0300179static void mmu_free_roots(struct kvm_vcpu *vcpu);
Xiao Guangrongce88dec2011-07-12 03:33:44 +0800180
181void kvm_mmu_set_mmio_spte_mask(u64 mmio_mask)
182{
183 shadow_mmio_mask = mmio_mask;
184}
185EXPORT_SYMBOL_GPL(kvm_mmu_set_mmio_spte_mask);
186
Xiao Guangrongf2fd1252013-06-07 16:51:24 +0800187/*
David Matlackee3d1572014-08-18 15:46:06 -0700188 * the low bit of the generation number is always presumed to be zero.
189 * This disables mmio caching during memslot updates. The concept is
190 * similar to a seqcount but instead of retrying the access we just punt
191 * and ignore the cache.
192 *
193 * spte bits 3-11 are used as bits 1-9 of the generation number,
194 * the bits 52-61 are used as bits 10-19 of the generation number.
Xiao Guangrongf2fd1252013-06-07 16:51:24 +0800195 */
David Matlackee3d1572014-08-18 15:46:06 -0700196#define MMIO_SPTE_GEN_LOW_SHIFT 2
Xiao Guangrongf2fd1252013-06-07 16:51:24 +0800197#define MMIO_SPTE_GEN_HIGH_SHIFT 52
198
David Matlackee3d1572014-08-18 15:46:06 -0700199#define MMIO_GEN_SHIFT 20
200#define MMIO_GEN_LOW_SHIFT 10
201#define MMIO_GEN_LOW_MASK ((1 << MMIO_GEN_LOW_SHIFT) - 2)
Xiao Guangrongf8f55942013-06-07 16:51:26 +0800202#define MMIO_GEN_MASK ((1 << MMIO_GEN_SHIFT) - 1)
Xiao Guangrongf2fd1252013-06-07 16:51:24 +0800203
204static u64 generation_mmio_spte_mask(unsigned int gen)
205{
206 u64 mask;
207
Tiejun Chen842bb262014-11-18 17:14:38 +0800208 WARN_ON(gen & ~MMIO_GEN_MASK);
Xiao Guangrongf2fd1252013-06-07 16:51:24 +0800209
210 mask = (gen & MMIO_GEN_LOW_MASK) << MMIO_SPTE_GEN_LOW_SHIFT;
211 mask |= ((u64)gen >> MMIO_GEN_LOW_SHIFT) << MMIO_SPTE_GEN_HIGH_SHIFT;
212 return mask;
213}
214
215static unsigned int get_mmio_spte_generation(u64 spte)
216{
217 unsigned int gen;
218
219 spte &= ~shadow_mmio_mask;
220
221 gen = (spte >> MMIO_SPTE_GEN_LOW_SHIFT) & MMIO_GEN_LOW_MASK;
222 gen |= (spte >> MMIO_SPTE_GEN_HIGH_SHIFT) << MMIO_GEN_LOW_SHIFT;
223 return gen;
224}
225
Paolo Bonzini54bf36a2015-04-08 15:39:23 +0200226static unsigned int kvm_current_mmio_generation(struct kvm_vcpu *vcpu)
Xiao Guangrongf8f55942013-06-07 16:51:26 +0800227{
Paolo Bonzini54bf36a2015-04-08 15:39:23 +0200228 return kvm_vcpu_memslots(vcpu)->generation & MMIO_GEN_MASK;
Xiao Guangrongf8f55942013-06-07 16:51:26 +0800229}
230
Paolo Bonzini54bf36a2015-04-08 15:39:23 +0200231static void mark_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, u64 gfn,
Xiao Guangrongf2fd1252013-06-07 16:51:24 +0800232 unsigned access)
Xiao Guangrongce88dec2011-07-12 03:33:44 +0800233{
Paolo Bonzini54bf36a2015-04-08 15:39:23 +0200234 unsigned int gen = kvm_current_mmio_generation(vcpu);
Xiao Guangrongf8f55942013-06-07 16:51:26 +0800235 u64 mask = generation_mmio_spte_mask(gen);
Takuya Yoshikawa95b04302013-03-12 17:44:40 +0900236
Xiao Guangrongce88dec2011-07-12 03:33:44 +0800237 access &= ACC_WRITE_MASK | ACC_USER_MASK;
Xiao Guangrongf2fd1252013-06-07 16:51:24 +0800238 mask |= shadow_mmio_mask | access | gfn << PAGE_SHIFT;
Xiao Guangrongf2fd1252013-06-07 16:51:24 +0800239
Xiao Guangrongf8f55942013-06-07 16:51:26 +0800240 trace_mark_mmio_spte(sptep, gfn, access, gen);
Xiao Guangrongf2fd1252013-06-07 16:51:24 +0800241 mmu_spte_set(sptep, mask);
Xiao Guangrongce88dec2011-07-12 03:33:44 +0800242}
243
244static bool is_mmio_spte(u64 spte)
245{
246 return (spte & shadow_mmio_mask) == shadow_mmio_mask;
247}
248
249static gfn_t get_mmio_spte_gfn(u64 spte)
250{
Tiejun Chen842bb262014-11-18 17:14:38 +0800251 u64 mask = generation_mmio_spte_mask(MMIO_GEN_MASK) | shadow_mmio_mask;
Xiao Guangrongf2fd1252013-06-07 16:51:24 +0800252 return (spte & ~mask) >> PAGE_SHIFT;
Xiao Guangrongce88dec2011-07-12 03:33:44 +0800253}
254
255static unsigned get_mmio_spte_access(u64 spte)
256{
Tiejun Chen842bb262014-11-18 17:14:38 +0800257 u64 mask = generation_mmio_spte_mask(MMIO_GEN_MASK) | shadow_mmio_mask;
Xiao Guangrongf2fd1252013-06-07 16:51:24 +0800258 return (spte & ~mask) & ~PAGE_MASK;
Xiao Guangrongce88dec2011-07-12 03:33:44 +0800259}
260
Paolo Bonzini54bf36a2015-04-08 15:39:23 +0200261static bool set_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, gfn_t gfn,
Xiao Guangrongf2fd1252013-06-07 16:51:24 +0800262 pfn_t pfn, unsigned access)
Xiao Guangrongce88dec2011-07-12 03:33:44 +0800263{
264 if (unlikely(is_noslot_pfn(pfn))) {
Paolo Bonzini54bf36a2015-04-08 15:39:23 +0200265 mark_mmio_spte(vcpu, sptep, gfn, access);
Xiao Guangrongce88dec2011-07-12 03:33:44 +0800266 return true;
267 }
268
269 return false;
270}
Avi Kivityc7addb92007-09-16 18:58:32 +0200271
Paolo Bonzini54bf36a2015-04-08 15:39:23 +0200272static bool check_mmio_spte(struct kvm_vcpu *vcpu, u64 spte)
Xiao Guangrongf8f55942013-06-07 16:51:26 +0800273{
Xiao Guangrong089504c2013-06-07 16:51:27 +0800274 unsigned int kvm_gen, spte_gen;
275
Paolo Bonzini54bf36a2015-04-08 15:39:23 +0200276 kvm_gen = kvm_current_mmio_generation(vcpu);
Xiao Guangrong089504c2013-06-07 16:51:27 +0800277 spte_gen = get_mmio_spte_generation(spte);
278
279 trace_check_mmio_spte(spte, kvm_gen, spte_gen);
280 return likely(kvm_gen == spte_gen);
Xiao Guangrongf8f55942013-06-07 16:51:26 +0800281}
282
Sheng Yang7b523452008-04-25 21:13:50 +0800283void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask,
Sheng Yang4b12f0d2009-04-27 20:35:42 +0800284 u64 dirty_mask, u64 nx_mask, u64 x_mask)
Sheng Yang7b523452008-04-25 21:13:50 +0800285{
286 shadow_user_mask = user_mask;
287 shadow_accessed_mask = accessed_mask;
288 shadow_dirty_mask = dirty_mask;
289 shadow_nx_mask = nx_mask;
290 shadow_x_mask = x_mask;
291}
292EXPORT_SYMBOL_GPL(kvm_mmu_set_mask_ptes);
293
Avi Kivity6aa8b732006-12-10 02:21:36 -0800294static int is_cpuid_PSE36(void)
295{
296 return 1;
297}
298
Avi Kivity73b10872007-01-26 00:56:41 -0800299static int is_nx(struct kvm_vcpu *vcpu)
300{
Avi Kivityf6801df2010-01-21 15:31:50 +0200301 return vcpu->arch.efer & EFER_NX;
Avi Kivity73b10872007-01-26 00:56:41 -0800302}
303
Avi Kivityc7addb92007-09-16 18:58:32 +0200304static int is_shadow_present_pte(u64 pte)
305{
Xiao Guangrongce88dec2011-07-12 03:33:44 +0800306 return pte & PT_PRESENT_MASK && !is_mmio_spte(pte);
Avi Kivityc7addb92007-09-16 18:58:32 +0200307}
308
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300309static int is_large_pte(u64 pte)
310{
311 return pte & PT_PAGE_SIZE_MASK;
312}
313
Avi Kivity43a37952009-06-10 14:12:05 +0300314static int is_rmap_spte(u64 pte)
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800315{
Avi Kivity4b1a80f2008-03-23 12:18:19 +0200316 return is_shadow_present_pte(pte);
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800317}
318
Marcelo Tosatti776e6632009-06-10 12:27:03 -0300319static int is_last_spte(u64 pte, int level)
320{
321 if (level == PT_PAGE_TABLE_LEVEL)
322 return 1;
Joerg Roedel852e3c12009-07-27 16:30:44 +0200323 if (is_large_pte(pte))
Marcelo Tosatti776e6632009-06-10 12:27:03 -0300324 return 1;
325 return 0;
326}
327
Anthony Liguori35149e22008-04-02 14:46:56 -0500328static pfn_t spte_to_pfn(u64 pte)
Avi Kivity0b49ea82008-03-23 15:06:23 +0200329{
Anthony Liguori35149e22008-04-02 14:46:56 -0500330 return (pte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT;
Avi Kivity0b49ea82008-03-23 15:06:23 +0200331}
332
Avi Kivityda9285212007-11-21 13:54:47 +0200333static gfn_t pse36_gfn_delta(u32 gpte)
334{
335 int shift = 32 - PT32_DIR_PSE36_SHIFT - PAGE_SHIFT;
336
337 return (gpte & PT32_DIR_PSE36_MASK) << shift;
338}
339
Xiao Guangrong603e0652011-07-12 03:31:28 +0800340#ifdef CONFIG_X86_64
Avi Kivityd555c332009-06-10 14:24:23 +0300341static void __set_spte(u64 *sptep, u64 spte)
Avi Kivitye663ee62007-05-31 15:46:04 +0300342{
Xiao Guangrong603e0652011-07-12 03:31:28 +0800343 *sptep = spte;
Avi Kivitye663ee62007-05-31 15:46:04 +0300344}
345
Xiao Guangrong603e0652011-07-12 03:31:28 +0800346static void __update_clear_spte_fast(u64 *sptep, u64 spte)
Avi Kivitya9221dd2010-06-06 14:48:06 +0300347{
Xiao Guangrong603e0652011-07-12 03:31:28 +0800348 *sptep = spte;
Avi Kivitya9221dd2010-06-06 14:48:06 +0300349}
350
Xiao Guangrong603e0652011-07-12 03:31:28 +0800351static u64 __update_clear_spte_slow(u64 *sptep, u64 spte)
352{
353 return xchg(sptep, spte);
354}
Xiao Guangrongc2a2ac22011-07-12 03:32:13 +0800355
356static u64 __get_spte_lockless(u64 *sptep)
357{
358 return ACCESS_ONCE(*sptep);
359}
Xiao Guangrong603e0652011-07-12 03:31:28 +0800360#else
361union split_spte {
362 struct {
363 u32 spte_low;
364 u32 spte_high;
365 };
366 u64 spte;
367};
368
Xiao Guangrongc2a2ac22011-07-12 03:32:13 +0800369static void count_spte_clear(u64 *sptep, u64 spte)
370{
371 struct kvm_mmu_page *sp = page_header(__pa(sptep));
372
373 if (is_shadow_present_pte(spte))
374 return;
375
376 /* Ensure the spte is completely set before we increase the count */
377 smp_wmb();
378 sp->clear_spte_count++;
379}
380
Xiao Guangrong603e0652011-07-12 03:31:28 +0800381static void __set_spte(u64 *sptep, u64 spte)
382{
383 union split_spte *ssptep, sspte;
384
385 ssptep = (union split_spte *)sptep;
386 sspte = (union split_spte)spte;
387
388 ssptep->spte_high = sspte.spte_high;
389
390 /*
391 * If we map the spte from nonpresent to present, We should store
392 * the high bits firstly, then set present bit, so cpu can not
393 * fetch this spte while we are setting the spte.
394 */
395 smp_wmb();
396
397 ssptep->spte_low = sspte.spte_low;
398}
399
400static void __update_clear_spte_fast(u64 *sptep, u64 spte)
401{
402 union split_spte *ssptep, sspte;
403
404 ssptep = (union split_spte *)sptep;
405 sspte = (union split_spte)spte;
406
407 ssptep->spte_low = sspte.spte_low;
408
409 /*
410 * If we map the spte from present to nonpresent, we should clear
411 * present bit firstly to avoid vcpu fetch the old high bits.
412 */
413 smp_wmb();
414
415 ssptep->spte_high = sspte.spte_high;
Xiao Guangrongc2a2ac22011-07-12 03:32:13 +0800416 count_spte_clear(sptep, spte);
Xiao Guangrong603e0652011-07-12 03:31:28 +0800417}
418
419static u64 __update_clear_spte_slow(u64 *sptep, u64 spte)
420{
421 union split_spte *ssptep, sspte, orig;
422
423 ssptep = (union split_spte *)sptep;
424 sspte = (union split_spte)spte;
425
426 /* xchg acts as a barrier before the setting of the high bits */
427 orig.spte_low = xchg(&ssptep->spte_low, sspte.spte_low);
Zhao Jin41bc3182011-09-19 12:19:51 +0800428 orig.spte_high = ssptep->spte_high;
429 ssptep->spte_high = sspte.spte_high;
Xiao Guangrongc2a2ac22011-07-12 03:32:13 +0800430 count_spte_clear(sptep, spte);
Xiao Guangrong603e0652011-07-12 03:31:28 +0800431
432 return orig.spte;
433}
Xiao Guangrongc2a2ac22011-07-12 03:32:13 +0800434
435/*
436 * The idea using the light way get the spte on x86_32 guest is from
437 * gup_get_pte(arch/x86/mm/gup.c).
Xiao Guangrongaccaefe2013-06-19 17:09:20 +0800438 *
439 * An spte tlb flush may be pending, because kvm_set_pte_rmapp
440 * coalesces them and we are running out of the MMU lock. Therefore
441 * we need to protect against in-progress updates of the spte.
442 *
443 * Reading the spte while an update is in progress may get the old value
444 * for the high part of the spte. The race is fine for a present->non-present
445 * change (because the high part of the spte is ignored for non-present spte),
446 * but for a present->present change we must reread the spte.
447 *
448 * All such changes are done in two steps (present->non-present and
449 * non-present->present), hence it is enough to count the number of
450 * present->non-present updates: if it changed while reading the spte,
451 * we might have hit the race. This is done using clear_spte_count.
Xiao Guangrongc2a2ac22011-07-12 03:32:13 +0800452 */
453static u64 __get_spte_lockless(u64 *sptep)
454{
455 struct kvm_mmu_page *sp = page_header(__pa(sptep));
456 union split_spte spte, *orig = (union split_spte *)sptep;
457 int count;
458
459retry:
460 count = sp->clear_spte_count;
461 smp_rmb();
462
463 spte.spte_low = orig->spte_low;
464 smp_rmb();
465
466 spte.spte_high = orig->spte_high;
467 smp_rmb();
468
469 if (unlikely(spte.spte_low != orig->spte_low ||
470 count != sp->clear_spte_count))
471 goto retry;
472
473 return spte.spte;
474}
Xiao Guangrong603e0652011-07-12 03:31:28 +0800475#endif
476
Xiao Guangrongc7ba5b42012-06-20 15:59:18 +0800477static bool spte_is_locklessly_modifiable(u64 spte)
478{
Gleb Natapovfeb3eb72013-01-30 16:45:00 +0200479 return (spte & (SPTE_HOST_WRITEABLE | SPTE_MMU_WRITEABLE)) ==
480 (SPTE_HOST_WRITEABLE | SPTE_MMU_WRITEABLE);
Xiao Guangrongc7ba5b42012-06-20 15:59:18 +0800481}
482
Xiao Guangrong8672b722010-08-02 16:14:04 +0800483static bool spte_has_volatile_bits(u64 spte)
484{
Xiao Guangrongc7ba5b42012-06-20 15:59:18 +0800485 /*
486 * Always atomicly update spte if it can be updated
487 * out of mmu-lock, it can ensure dirty bit is not lost,
488 * also, it can help us to get a stable is_writable_pte()
489 * to ensure tlb flush is not missed.
490 */
491 if (spte_is_locklessly_modifiable(spte))
492 return true;
493
Xiao Guangrong8672b722010-08-02 16:14:04 +0800494 if (!shadow_accessed_mask)
495 return false;
496
497 if (!is_shadow_present_pte(spte))
498 return false;
499
Xiao Guangrong41327792010-08-02 16:15:08 +0800500 if ((spte & shadow_accessed_mask) &&
501 (!is_writable_pte(spte) || (spte & shadow_dirty_mask)))
Xiao Guangrong8672b722010-08-02 16:14:04 +0800502 return false;
503
504 return true;
505}
506
Xiao Guangrong41327792010-08-02 16:15:08 +0800507static bool spte_is_bit_cleared(u64 old_spte, u64 new_spte, u64 bit_mask)
508{
509 return (old_spte & bit_mask) && !(new_spte & bit_mask);
510}
511
Kai Huang7e71a592015-01-09 16:44:30 +0800512static bool spte_is_bit_changed(u64 old_spte, u64 new_spte, u64 bit_mask)
513{
514 return (old_spte & bit_mask) != (new_spte & bit_mask);
515}
516
Xiao Guangrong1df9f2d2011-07-12 03:30:35 +0800517/* Rules for using mmu_spte_set:
518 * Set the sptep from nonpresent to present.
519 * Note: the sptep being assigned *must* be either not present
520 * or in a state where the hardware will not attempt to update
521 * the spte.
522 */
523static void mmu_spte_set(u64 *sptep, u64 new_spte)
524{
525 WARN_ON(is_shadow_present_pte(*sptep));
526 __set_spte(sptep, new_spte);
527}
528
529/* Rules for using mmu_spte_update:
530 * Update the state bits, it means the mapped pfn is not changged.
Xiao Guangrong6e7d0352012-06-20 15:58:33 +0800531 *
532 * Whenever we overwrite a writable spte with a read-only one we
533 * should flush remote TLBs. Otherwise rmap_write_protect
534 * will find a read-only spte, even though the writable spte
535 * might be cached on a CPU's TLB, the return value indicates this
536 * case.
Xiao Guangrong1df9f2d2011-07-12 03:30:35 +0800537 */
Xiao Guangrong6e7d0352012-06-20 15:58:33 +0800538static bool mmu_spte_update(u64 *sptep, u64 new_spte)
Avi Kivityb79b93f2010-06-06 15:46:44 +0300539{
Xiao Guangrongc7ba5b42012-06-20 15:59:18 +0800540 u64 old_spte = *sptep;
Xiao Guangrong6e7d0352012-06-20 15:58:33 +0800541 bool ret = false;
Avi Kivityb79b93f2010-06-06 15:46:44 +0300542
Xiao Guangrong41327792010-08-02 16:15:08 +0800543 WARN_ON(!is_rmap_spte(new_spte));
544
Xiao Guangrong6e7d0352012-06-20 15:58:33 +0800545 if (!is_shadow_present_pte(old_spte)) {
546 mmu_spte_set(sptep, new_spte);
547 return ret;
548 }
Xiao Guangrong1df9f2d2011-07-12 03:30:35 +0800549
Xiao Guangrongc7ba5b42012-06-20 15:59:18 +0800550 if (!spte_has_volatile_bits(old_spte))
Xiao Guangrong603e0652011-07-12 03:31:28 +0800551 __update_clear_spte_fast(sptep, new_spte);
Xiao Guangrong41327792010-08-02 16:15:08 +0800552 else
Xiao Guangrong603e0652011-07-12 03:31:28 +0800553 old_spte = __update_clear_spte_slow(sptep, new_spte);
Xiao Guangrong41327792010-08-02 16:15:08 +0800554
Xiao Guangrongc7ba5b42012-06-20 15:59:18 +0800555 /*
556 * For the spte updated out of mmu-lock is safe, since
557 * we always atomicly update it, see the comments in
558 * spte_has_volatile_bits().
559 */
Xiao Guangrong7f31c952014-04-17 17:06:15 +0800560 if (spte_is_locklessly_modifiable(old_spte) &&
561 !is_writable_pte(new_spte))
Xiao Guangrong6e7d0352012-06-20 15:58:33 +0800562 ret = true;
563
Xiao Guangrong41327792010-08-02 16:15:08 +0800564 if (!shadow_accessed_mask)
Xiao Guangrong6e7d0352012-06-20 15:58:33 +0800565 return ret;
Xiao Guangrong41327792010-08-02 16:15:08 +0800566
Kai Huang7e71a592015-01-09 16:44:30 +0800567 /*
568 * Flush TLB when accessed/dirty bits are changed in the page tables,
569 * to guarantee consistency between TLB and page tables.
570 */
571 if (spte_is_bit_changed(old_spte, new_spte,
572 shadow_accessed_mask | shadow_dirty_mask))
573 ret = true;
574
Xiao Guangrong41327792010-08-02 16:15:08 +0800575 if (spte_is_bit_cleared(old_spte, new_spte, shadow_accessed_mask))
576 kvm_set_pfn_accessed(spte_to_pfn(old_spte));
577 if (spte_is_bit_cleared(old_spte, new_spte, shadow_dirty_mask))
578 kvm_set_pfn_dirty(spte_to_pfn(old_spte));
Xiao Guangrong6e7d0352012-06-20 15:58:33 +0800579
580 return ret;
Avi Kivityb79b93f2010-06-06 15:46:44 +0300581}
582
Xiao Guangrong1df9f2d2011-07-12 03:30:35 +0800583/*
584 * Rules for using mmu_spte_clear_track_bits:
585 * It sets the sptep from present to nonpresent, and track the
586 * state bits, it is used to clear the last level sptep.
587 */
588static int mmu_spte_clear_track_bits(u64 *sptep)
589{
590 pfn_t pfn;
591 u64 old_spte = *sptep;
592
593 if (!spte_has_volatile_bits(old_spte))
Xiao Guangrong603e0652011-07-12 03:31:28 +0800594 __update_clear_spte_fast(sptep, 0ull);
Xiao Guangrong1df9f2d2011-07-12 03:30:35 +0800595 else
Xiao Guangrong603e0652011-07-12 03:31:28 +0800596 old_spte = __update_clear_spte_slow(sptep, 0ull);
Xiao Guangrong1df9f2d2011-07-12 03:30:35 +0800597
598 if (!is_rmap_spte(old_spte))
599 return 0;
600
601 pfn = spte_to_pfn(old_spte);
Xiao Guangrong86fde742012-07-17 21:52:52 +0800602
603 /*
604 * KVM does not hold the refcount of the page used by
605 * kvm mmu, before reclaiming the page, we should
606 * unmap it from mmu first.
607 */
Ard Biesheuvelbf4bea82014-11-10 08:33:56 +0000608 WARN_ON(!kvm_is_reserved_pfn(pfn) && !page_count(pfn_to_page(pfn)));
Xiao Guangrong86fde742012-07-17 21:52:52 +0800609
Xiao Guangrong1df9f2d2011-07-12 03:30:35 +0800610 if (!shadow_accessed_mask || old_spte & shadow_accessed_mask)
611 kvm_set_pfn_accessed(pfn);
612 if (!shadow_dirty_mask || (old_spte & shadow_dirty_mask))
613 kvm_set_pfn_dirty(pfn);
614 return 1;
615}
616
617/*
618 * Rules for using mmu_spte_clear_no_track:
619 * Directly clear spte without caring the state bits of sptep,
620 * it is used to set the upper level spte.
621 */
622static void mmu_spte_clear_no_track(u64 *sptep)
623{
Xiao Guangrong603e0652011-07-12 03:31:28 +0800624 __update_clear_spte_fast(sptep, 0ull);
Xiao Guangrong1df9f2d2011-07-12 03:30:35 +0800625}
626
Xiao Guangrongc2a2ac22011-07-12 03:32:13 +0800627static u64 mmu_spte_get_lockless(u64 *sptep)
628{
629 return __get_spte_lockless(sptep);
630}
631
632static void walk_shadow_page_lockless_begin(struct kvm_vcpu *vcpu)
633{
Avi Kivityc1427862012-05-14 15:44:06 +0300634 /*
635 * Prevent page table teardown by making any free-er wait during
636 * kvm_flush_remote_tlbs() IPI to all active vcpus.
637 */
638 local_irq_disable();
639 vcpu->mode = READING_SHADOW_PAGE_TABLES;
640 /*
641 * Make sure a following spte read is not reordered ahead of the write
642 * to vcpu->mode.
643 */
644 smp_mb();
Xiao Guangrongc2a2ac22011-07-12 03:32:13 +0800645}
646
647static void walk_shadow_page_lockless_end(struct kvm_vcpu *vcpu)
648{
Avi Kivityc1427862012-05-14 15:44:06 +0300649 /*
650 * Make sure the write to vcpu->mode is not reordered in front of
651 * reads to sptes. If it does, kvm_commit_zap_page() can see us
652 * OUTSIDE_GUEST_MODE and proceed to free the shadow page table.
653 */
654 smp_mb();
655 vcpu->mode = OUTSIDE_GUEST_MODE;
656 local_irq_enable();
Xiao Guangrongc2a2ac22011-07-12 03:32:13 +0800657}
658
Avi Kivitye2dec932007-01-05 16:36:54 -0800659static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache,
Avi Kivity2e3e5882007-09-10 11:28:17 +0300660 struct kmem_cache *base_cache, int min)
Avi Kivity714b93d2007-01-05 16:36:53 -0800661{
662 void *obj;
663
664 if (cache->nobjs >= min)
Avi Kivitye2dec932007-01-05 16:36:54 -0800665 return 0;
Avi Kivity714b93d2007-01-05 16:36:53 -0800666 while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
Avi Kivity2e3e5882007-09-10 11:28:17 +0300667 obj = kmem_cache_zalloc(base_cache, GFP_KERNEL);
Avi Kivity714b93d2007-01-05 16:36:53 -0800668 if (!obj)
Avi Kivitye2dec932007-01-05 16:36:54 -0800669 return -ENOMEM;
Avi Kivity714b93d2007-01-05 16:36:53 -0800670 cache->objects[cache->nobjs++] = obj;
671 }
Avi Kivitye2dec932007-01-05 16:36:54 -0800672 return 0;
Avi Kivity714b93d2007-01-05 16:36:53 -0800673}
674
Xiao Guangrongf759e2b2011-09-22 16:53:17 +0800675static int mmu_memory_cache_free_objects(struct kvm_mmu_memory_cache *cache)
676{
677 return cache->nobjs;
678}
679
Xiao Guangronge8ad9a72010-05-13 10:06:02 +0800680static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc,
681 struct kmem_cache *cache)
Avi Kivity714b93d2007-01-05 16:36:53 -0800682{
683 while (mc->nobjs)
Xiao Guangronge8ad9a72010-05-13 10:06:02 +0800684 kmem_cache_free(cache, mc->objects[--mc->nobjs]);
Avi Kivity714b93d2007-01-05 16:36:53 -0800685}
686
Avi Kivityc1158e62007-07-20 08:18:27 +0300687static int mmu_topup_memory_cache_page(struct kvm_mmu_memory_cache *cache,
Avi Kivity2e3e5882007-09-10 11:28:17 +0300688 int min)
Avi Kivityc1158e62007-07-20 08:18:27 +0300689{
Xiao Guangrong842f22e2011-03-04 19:01:10 +0800690 void *page;
Avi Kivityc1158e62007-07-20 08:18:27 +0300691
692 if (cache->nobjs >= min)
693 return 0;
694 while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
Xiao Guangrong842f22e2011-03-04 19:01:10 +0800695 page = (void *)__get_free_page(GFP_KERNEL);
Avi Kivityc1158e62007-07-20 08:18:27 +0300696 if (!page)
697 return -ENOMEM;
Xiao Guangrong842f22e2011-03-04 19:01:10 +0800698 cache->objects[cache->nobjs++] = page;
Avi Kivityc1158e62007-07-20 08:18:27 +0300699 }
700 return 0;
701}
702
703static void mmu_free_memory_cache_page(struct kvm_mmu_memory_cache *mc)
704{
705 while (mc->nobjs)
Avi Kivityc4d198d2007-07-21 09:06:46 +0300706 free_page((unsigned long)mc->objects[--mc->nobjs]);
Avi Kivityc1158e62007-07-20 08:18:27 +0300707}
708
Avi Kivity8c438502007-04-16 11:53:17 +0300709static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu)
710{
711 int r;
712
Xiao Guangrong53c07b12011-05-15 23:26:20 +0800713 r = mmu_topup_memory_cache(&vcpu->arch.mmu_pte_list_desc_cache,
Xiao Guangrong67052b32011-05-15 23:27:08 +0800714 pte_list_desc_cache, 8 + PTE_PREFETCH_NUM);
Avi Kivity2e3e5882007-09-10 11:28:17 +0300715 if (r)
716 goto out;
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800717 r = mmu_topup_memory_cache_page(&vcpu->arch.mmu_page_cache, 8);
Avi Kivity2e3e5882007-09-10 11:28:17 +0300718 if (r)
719 goto out;
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800720 r = mmu_topup_memory_cache(&vcpu->arch.mmu_page_header_cache,
Avi Kivity2e3e5882007-09-10 11:28:17 +0300721 mmu_page_header_cache, 4);
722out:
Avi Kivity8c438502007-04-16 11:53:17 +0300723 return r;
724}
725
Avi Kivity714b93d2007-01-05 16:36:53 -0800726static void mmu_free_memory_caches(struct kvm_vcpu *vcpu)
727{
Xiao Guangrong53c07b12011-05-15 23:26:20 +0800728 mmu_free_memory_cache(&vcpu->arch.mmu_pte_list_desc_cache,
729 pte_list_desc_cache);
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800730 mmu_free_memory_cache_page(&vcpu->arch.mmu_page_cache);
Xiao Guangronge8ad9a72010-05-13 10:06:02 +0800731 mmu_free_memory_cache(&vcpu->arch.mmu_page_header_cache,
732 mmu_page_header_cache);
Avi Kivity714b93d2007-01-05 16:36:53 -0800733}
734
Takuya Yoshikawa80feb892012-05-29 23:54:26 +0900735static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc)
Avi Kivity714b93d2007-01-05 16:36:53 -0800736{
737 void *p;
738
739 BUG_ON(!mc->nobjs);
740 p = mc->objects[--mc->nobjs];
Avi Kivity714b93d2007-01-05 16:36:53 -0800741 return p;
742}
743
Xiao Guangrong53c07b12011-05-15 23:26:20 +0800744static struct pte_list_desc *mmu_alloc_pte_list_desc(struct kvm_vcpu *vcpu)
Avi Kivity714b93d2007-01-05 16:36:53 -0800745{
Takuya Yoshikawa80feb892012-05-29 23:54:26 +0900746 return mmu_memory_cache_alloc(&vcpu->arch.mmu_pte_list_desc_cache);
Avi Kivity714b93d2007-01-05 16:36:53 -0800747}
748
Xiao Guangrong53c07b12011-05-15 23:26:20 +0800749static void mmu_free_pte_list_desc(struct pte_list_desc *pte_list_desc)
Avi Kivity714b93d2007-01-05 16:36:53 -0800750{
Xiao Guangrong53c07b12011-05-15 23:26:20 +0800751 kmem_cache_free(pte_list_desc_cache, pte_list_desc);
Avi Kivity714b93d2007-01-05 16:36:53 -0800752}
753
Lai Jiangshan2032a932010-05-26 16:49:59 +0800754static gfn_t kvm_mmu_page_get_gfn(struct kvm_mmu_page *sp, int index)
755{
756 if (!sp->role.direct)
757 return sp->gfns[index];
758
759 return sp->gfn + (index << ((sp->role.level - 1) * PT64_LEVEL_BITS));
760}
761
762static void kvm_mmu_page_set_gfn(struct kvm_mmu_page *sp, int index, gfn_t gfn)
763{
764 if (sp->role.direct)
765 BUG_ON(gfn != kvm_mmu_page_get_gfn(sp, index));
766 else
767 sp->gfns[index] = gfn;
768}
769
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800770/*
Takuya Yoshikawad4dbf472010-12-07 12:59:07 +0900771 * Return the pointer to the large page information for a given gfn,
772 * handling slots that are not large page aligned.
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300773 */
Takuya Yoshikawad4dbf472010-12-07 12:59:07 +0900774static struct kvm_lpage_info *lpage_info_slot(gfn_t gfn,
775 struct kvm_memory_slot *slot,
776 int level)
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300777{
778 unsigned long idx;
779
Takuya Yoshikawafb03cb62012-02-08 12:59:10 +0900780 idx = gfn_to_index(gfn, slot->base_gfn, level);
Takuya Yoshikawadb3fe4e2012-02-08 13:02:18 +0900781 return &slot->arch.lpage_info[level - 2][idx];
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300782}
783
Paolo Bonzini3ed1a472015-05-19 16:29:22 +0200784static void account_shadowed(struct kvm *kvm, struct kvm_mmu_page *sp)
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300785{
Paolo Bonzini699023e2015-05-18 15:03:39 +0200786 struct kvm_memslots *slots;
Joerg Roedeld25797b2009-07-27 16:30:43 +0200787 struct kvm_memory_slot *slot;
Takuya Yoshikawad4dbf472010-12-07 12:59:07 +0900788 struct kvm_lpage_info *linfo;
Paolo Bonzini3ed1a472015-05-19 16:29:22 +0200789 gfn_t gfn;
Joerg Roedeld25797b2009-07-27 16:30:43 +0200790 int i;
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300791
Paolo Bonzini3ed1a472015-05-19 16:29:22 +0200792 gfn = sp->gfn;
Paolo Bonzini699023e2015-05-18 15:03:39 +0200793 slots = kvm_memslots_for_spte_role(kvm, sp->role);
794 slot = __gfn_to_memslot(slots, gfn);
Xiao Guangrong8a3d08f2015-05-13 14:42:21 +0800795 for (i = PT_DIRECTORY_LEVEL; i <= PT_MAX_HUGEPAGE_LEVEL; ++i) {
Takuya Yoshikawad4dbf472010-12-07 12:59:07 +0900796 linfo = lpage_info_slot(gfn, slot, i);
797 linfo->write_count += 1;
Joerg Roedeld25797b2009-07-27 16:30:43 +0200798 }
Xiao Guangrong332b2072011-05-15 23:20:27 +0800799 kvm->arch.indirect_shadow_pages++;
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300800}
801
Paolo Bonzini3ed1a472015-05-19 16:29:22 +0200802static void unaccount_shadowed(struct kvm *kvm, struct kvm_mmu_page *sp)
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300803{
Paolo Bonzini699023e2015-05-18 15:03:39 +0200804 struct kvm_memslots *slots;
Joerg Roedeld25797b2009-07-27 16:30:43 +0200805 struct kvm_memory_slot *slot;
Takuya Yoshikawad4dbf472010-12-07 12:59:07 +0900806 struct kvm_lpage_info *linfo;
Paolo Bonzini3ed1a472015-05-19 16:29:22 +0200807 gfn_t gfn;
Joerg Roedeld25797b2009-07-27 16:30:43 +0200808 int i;
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300809
Paolo Bonzini3ed1a472015-05-19 16:29:22 +0200810 gfn = sp->gfn;
Paolo Bonzini699023e2015-05-18 15:03:39 +0200811 slots = kvm_memslots_for_spte_role(kvm, sp->role);
812 slot = __gfn_to_memslot(slots, gfn);
Xiao Guangrong8a3d08f2015-05-13 14:42:21 +0800813 for (i = PT_DIRECTORY_LEVEL; i <= PT_MAX_HUGEPAGE_LEVEL; ++i) {
Takuya Yoshikawad4dbf472010-12-07 12:59:07 +0900814 linfo = lpage_info_slot(gfn, slot, i);
815 linfo->write_count -= 1;
816 WARN_ON(linfo->write_count < 0);
Joerg Roedeld25797b2009-07-27 16:30:43 +0200817 }
Xiao Guangrong332b2072011-05-15 23:20:27 +0800818 kvm->arch.indirect_shadow_pages--;
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300819}
820
Takuya Yoshikawa5225fdf2015-10-16 17:08:03 +0900821static int __has_wrprotected_page(gfn_t gfn, int level,
822 struct kvm_memory_slot *slot)
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300823{
Takuya Yoshikawad4dbf472010-12-07 12:59:07 +0900824 struct kvm_lpage_info *linfo;
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300825
826 if (slot) {
Takuya Yoshikawad4dbf472010-12-07 12:59:07 +0900827 linfo = lpage_info_slot(gfn, slot, level);
828 return linfo->write_count;
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300829 }
830
831 return 1;
832}
833
Takuya Yoshikawa5225fdf2015-10-16 17:08:03 +0900834static int has_wrprotected_page(struct kvm_vcpu *vcpu, gfn_t gfn, int level)
835{
836 struct kvm_memory_slot *slot;
837
838 slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
839 return __has_wrprotected_page(gfn, level, slot);
840}
841
Joerg Roedeld25797b2009-07-27 16:30:43 +0200842static int host_mapping_level(struct kvm *kvm, gfn_t gfn)
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300843{
Joerg Roedel8f0b1ab2010-01-28 12:37:56 +0100844 unsigned long page_size;
Joerg Roedeld25797b2009-07-27 16:30:43 +0200845 int i, ret = 0;
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300846
Joerg Roedel8f0b1ab2010-01-28 12:37:56 +0100847 page_size = kvm_host_page_size(kvm, gfn);
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300848
Xiao Guangrong8a3d08f2015-05-13 14:42:21 +0800849 for (i = PT_PAGE_TABLE_LEVEL; i <= PT_MAX_HUGEPAGE_LEVEL; ++i) {
Joerg Roedeld25797b2009-07-27 16:30:43 +0200850 if (page_size >= KVM_HPAGE_SIZE(i))
851 ret = i;
852 else
853 break;
854 }
855
Marcelo Tosatti4c2155c2008-09-16 20:54:47 -0300856 return ret;
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300857}
858
Takuya Yoshikawad8aacf52015-10-16 17:07:01 +0900859static inline bool memslot_valid_for_gpte(struct kvm_memory_slot *slot,
860 bool no_dirty_log)
861{
862 if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
863 return false;
864 if (no_dirty_log && slot->dirty_bitmap)
865 return false;
866
867 return true;
868}
869
Xiao Guangrong5d163b12011-03-09 15:43:00 +0800870static struct kvm_memory_slot *
871gfn_to_memslot_dirty_bitmap(struct kvm_vcpu *vcpu, gfn_t gfn,
872 bool no_dirty_log)
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300873{
874 struct kvm_memory_slot *slot;
Xiao Guangrong5d163b12011-03-09 15:43:00 +0800875
Paolo Bonzini54bf36a2015-04-08 15:39:23 +0200876 slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
Takuya Yoshikawad8aacf52015-10-16 17:07:01 +0900877 if (!memslot_valid_for_gpte(slot, no_dirty_log))
Xiao Guangrong5d163b12011-03-09 15:43:00 +0800878 slot = NULL;
879
880 return slot;
881}
882
Takuya Yoshikawafd136902015-10-16 17:06:02 +0900883static int mapping_level(struct kvm_vcpu *vcpu, gfn_t large_gfn,
884 bool *force_pt_level)
Andrea Arcangeli936a5fe2011-01-13 15:46:48 -0800885{
886 int host_level, level, max_level;
Takuya Yoshikawad8aacf52015-10-16 17:07:01 +0900887 struct kvm_memory_slot *slot;
888
889 slot = kvm_vcpu_gfn_to_memslot(vcpu, large_gfn);
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300890
Takuya Yoshikawafd136902015-10-16 17:06:02 +0900891 if (likely(!*force_pt_level))
Takuya Yoshikawad8aacf52015-10-16 17:07:01 +0900892 *force_pt_level = !memslot_valid_for_gpte(slot, true);
Takuya Yoshikawafd136902015-10-16 17:06:02 +0900893 if (unlikely(*force_pt_level))
894 return PT_PAGE_TABLE_LEVEL;
895
Joerg Roedeld25797b2009-07-27 16:30:43 +0200896 host_level = host_mapping_level(vcpu->kvm, large_gfn);
897
898 if (host_level == PT_PAGE_TABLE_LEVEL)
899 return host_level;
900
Xiao Guangrong55dd98c2013-02-05 15:26:54 +0800901 max_level = min(kvm_x86_ops->get_lpage_level(), host_level);
Sheng Yang878403b2010-01-05 19:02:29 +0800902
903 for (level = PT_DIRECTORY_LEVEL; level <= max_level; ++level)
Takuya Yoshikawa5225fdf2015-10-16 17:08:03 +0900904 if (__has_wrprotected_page(large_gfn, level, slot))
Joerg Roedeld25797b2009-07-27 16:30:43 +0200905 break;
Joerg Roedeld25797b2009-07-27 16:30:43 +0200906
907 return level - 1;
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300908}
909
910/*
Xiao Guangrong53c07b12011-05-15 23:26:20 +0800911 * Pte mapping structures:
912 *
913 * If pte_list bit zero is zero, then pte_list point to the spte.
914 *
915 * If pte_list bit zero is one, (then pte_list & ~1) points to a struct
916 * pte_list_desc containing more mappings.
917 *
918 * Returns the number of pte entries before the spte was added or zero if
919 * the spte was not added.
920 *
921 */
922static int pte_list_add(struct kvm_vcpu *vcpu, u64 *spte,
923 unsigned long *pte_list)
924{
925 struct pte_list_desc *desc;
926 int i, count = 0;
927
928 if (!*pte_list) {
929 rmap_printk("pte_list_add: %p %llx 0->1\n", spte, *spte);
930 *pte_list = (unsigned long)spte;
931 } else if (!(*pte_list & 1)) {
932 rmap_printk("pte_list_add: %p %llx 1->many\n", spte, *spte);
933 desc = mmu_alloc_pte_list_desc(vcpu);
934 desc->sptes[0] = (u64 *)*pte_list;
935 desc->sptes[1] = spte;
936 *pte_list = (unsigned long)desc | 1;
937 ++count;
938 } else {
939 rmap_printk("pte_list_add: %p %llx many->many\n", spte, *spte);
940 desc = (struct pte_list_desc *)(*pte_list & ~1ul);
941 while (desc->sptes[PTE_LIST_EXT-1] && desc->more) {
942 desc = desc->more;
943 count += PTE_LIST_EXT;
944 }
945 if (desc->sptes[PTE_LIST_EXT-1]) {
946 desc->more = mmu_alloc_pte_list_desc(vcpu);
947 desc = desc->more;
948 }
949 for (i = 0; desc->sptes[i]; ++i)
950 ++count;
951 desc->sptes[i] = spte;
952 }
953 return count;
954}
955
Xiao Guangrong53c07b12011-05-15 23:26:20 +0800956static void
957pte_list_desc_remove_entry(unsigned long *pte_list, struct pte_list_desc *desc,
958 int i, struct pte_list_desc *prev_desc)
959{
960 int j;
961
962 for (j = PTE_LIST_EXT - 1; !desc->sptes[j] && j > i; --j)
963 ;
964 desc->sptes[i] = desc->sptes[j];
965 desc->sptes[j] = NULL;
966 if (j != 0)
967 return;
968 if (!prev_desc && !desc->more)
969 *pte_list = (unsigned long)desc->sptes[0];
970 else
971 if (prev_desc)
972 prev_desc->more = desc->more;
973 else
974 *pte_list = (unsigned long)desc->more | 1;
975 mmu_free_pte_list_desc(desc);
976}
977
978static void pte_list_remove(u64 *spte, unsigned long *pte_list)
979{
980 struct pte_list_desc *desc;
981 struct pte_list_desc *prev_desc;
982 int i;
983
984 if (!*pte_list) {
985 printk(KERN_ERR "pte_list_remove: %p 0->BUG\n", spte);
986 BUG();
987 } else if (!(*pte_list & 1)) {
988 rmap_printk("pte_list_remove: %p 1->0\n", spte);
989 if ((u64 *)*pte_list != spte) {
990 printk(KERN_ERR "pte_list_remove: %p 1->BUG\n", spte);
991 BUG();
992 }
993 *pte_list = 0;
994 } else {
995 rmap_printk("pte_list_remove: %p many->many\n", spte);
996 desc = (struct pte_list_desc *)(*pte_list & ~1ul);
997 prev_desc = NULL;
998 while (desc) {
999 for (i = 0; i < PTE_LIST_EXT && desc->sptes[i]; ++i)
1000 if (desc->sptes[i] == spte) {
1001 pte_list_desc_remove_entry(pte_list,
1002 desc, i,
1003 prev_desc);
1004 return;
1005 }
1006 prev_desc = desc;
1007 desc = desc->more;
1008 }
1009 pr_err("pte_list_remove: %p many->many\n", spte);
1010 BUG();
1011 }
1012}
1013
Xiao Guangrong67052b32011-05-15 23:27:08 +08001014typedef void (*pte_list_walk_fn) (u64 *spte);
1015static void pte_list_walk(unsigned long *pte_list, pte_list_walk_fn fn)
1016{
1017 struct pte_list_desc *desc;
1018 int i;
1019
1020 if (!*pte_list)
1021 return;
1022
1023 if (!(*pte_list & 1))
1024 return fn((u64 *)*pte_list);
1025
1026 desc = (struct pte_list_desc *)(*pte_list & ~1ul);
1027 while (desc) {
1028 for (i = 0; i < PTE_LIST_EXT && desc->sptes[i]; ++i)
1029 fn(desc->sptes[i]);
1030 desc = desc->more;
1031 }
1032}
1033
Takuya Yoshikawa9373e2c02012-01-17 19:51:20 +09001034static unsigned long *__gfn_to_rmap(gfn_t gfn, int level,
Takuya Yoshikawa9b9b1492011-11-14 18:22:28 +09001035 struct kvm_memory_slot *slot)
1036{
Takuya Yoshikawa77d11302012-07-02 17:57:17 +09001037 unsigned long idx;
Takuya Yoshikawa9b9b1492011-11-14 18:22:28 +09001038
Takuya Yoshikawa77d11302012-07-02 17:57:17 +09001039 idx = gfn_to_index(gfn, slot->base_gfn, level);
Takuya Yoshikawad89cc612012-08-01 18:03:28 +09001040 return &slot->arch.rmap[level - PT_PAGE_TABLE_LEVEL][idx];
Takuya Yoshikawa9b9b1492011-11-14 18:22:28 +09001041}
1042
Xiao Guangrong53c07b12011-05-15 23:26:20 +08001043/*
Izik Eidus290fc382007-09-27 14:11:22 +02001044 * Take gfn and return the reverse mapping to it.
Izik Eidus290fc382007-09-27 14:11:22 +02001045 */
Paolo Bonzinie4cd1da2015-05-18 15:11:46 +02001046static unsigned long *gfn_to_rmap(struct kvm *kvm, gfn_t gfn, struct kvm_mmu_page *sp)
Izik Eidus290fc382007-09-27 14:11:22 +02001047{
Paolo Bonzini699023e2015-05-18 15:03:39 +02001048 struct kvm_memslots *slots;
Izik Eidus290fc382007-09-27 14:11:22 +02001049 struct kvm_memory_slot *slot;
1050
Paolo Bonzini699023e2015-05-18 15:03:39 +02001051 slots = kvm_memslots_for_spte_role(kvm, sp->role);
1052 slot = __gfn_to_memslot(slots, gfn);
Paolo Bonzinie4cd1da2015-05-18 15:11:46 +02001053 return __gfn_to_rmap(gfn, sp->role.level, slot);
Izik Eidus290fc382007-09-27 14:11:22 +02001054}
1055
Xiao Guangrongf759e2b2011-09-22 16:53:17 +08001056static bool rmap_can_add(struct kvm_vcpu *vcpu)
1057{
1058 struct kvm_mmu_memory_cache *cache;
1059
1060 cache = &vcpu->arch.mmu_pte_list_desc_cache;
1061 return mmu_memory_cache_free_objects(cache);
1062}
1063
Joerg Roedel44ad9942009-07-27 16:30:42 +02001064static int rmap_add(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)
Avi Kivitycd4a4e52007-01-05 16:36:38 -08001065{
Avi Kivity4db35312007-11-21 15:28:32 +02001066 struct kvm_mmu_page *sp;
Izik Eidus290fc382007-09-27 14:11:22 +02001067 unsigned long *rmapp;
Avi Kivitycd4a4e52007-01-05 16:36:38 -08001068
Avi Kivity4db35312007-11-21 15:28:32 +02001069 sp = page_header(__pa(spte));
Lai Jiangshan2032a932010-05-26 16:49:59 +08001070 kvm_mmu_page_set_gfn(sp, spte - sp->spt, gfn);
Paolo Bonzinie4cd1da2015-05-18 15:11:46 +02001071 rmapp = gfn_to_rmap(vcpu->kvm, gfn, sp);
Xiao Guangrong53c07b12011-05-15 23:26:20 +08001072 return pte_list_add(vcpu, spte, rmapp);
Avi Kivitycd4a4e52007-01-05 16:36:38 -08001073}
1074
Izik Eidus290fc382007-09-27 14:11:22 +02001075static void rmap_remove(struct kvm *kvm, u64 *spte)
Avi Kivitycd4a4e52007-01-05 16:36:38 -08001076{
Avi Kivity4db35312007-11-21 15:28:32 +02001077 struct kvm_mmu_page *sp;
Lai Jiangshan2032a932010-05-26 16:49:59 +08001078 gfn_t gfn;
Izik Eidus290fc382007-09-27 14:11:22 +02001079 unsigned long *rmapp;
Avi Kivitycd4a4e52007-01-05 16:36:38 -08001080
Avi Kivity4db35312007-11-21 15:28:32 +02001081 sp = page_header(__pa(spte));
Lai Jiangshan2032a932010-05-26 16:49:59 +08001082 gfn = kvm_mmu_page_get_gfn(sp, spte - sp->spt);
Paolo Bonzinie4cd1da2015-05-18 15:11:46 +02001083 rmapp = gfn_to_rmap(kvm, gfn, sp);
Xiao Guangrong53c07b12011-05-15 23:26:20 +08001084 pte_list_remove(spte, rmapp);
Avi Kivitycd4a4e52007-01-05 16:36:38 -08001085}
1086
Takuya Yoshikawa1e3f42f2012-03-21 23:50:34 +09001087/*
1088 * Used by the following functions to iterate through the sptes linked by a
1089 * rmap. All fields are private and not assumed to be used outside.
1090 */
1091struct rmap_iterator {
1092 /* private fields */
1093 struct pte_list_desc *desc; /* holds the sptep if not NULL */
1094 int pos; /* index of the sptep */
1095};
1096
1097/*
1098 * Iteration must be started by this function. This should also be used after
1099 * removing/dropping sptes from the rmap link because in such cases the
1100 * information in the itererator may not be valid.
1101 *
1102 * Returns sptep if found, NULL otherwise.
1103 */
1104static u64 *rmap_get_first(unsigned long rmap, struct rmap_iterator *iter)
1105{
1106 if (!rmap)
1107 return NULL;
1108
1109 if (!(rmap & 1)) {
1110 iter->desc = NULL;
1111 return (u64 *)rmap;
1112 }
1113
1114 iter->desc = (struct pte_list_desc *)(rmap & ~1ul);
1115 iter->pos = 0;
1116 return iter->desc->sptes[iter->pos];
1117}
1118
1119/*
1120 * Must be used with a valid iterator: e.g. after rmap_get_first().
1121 *
1122 * Returns sptep if found, NULL otherwise.
1123 */
1124static u64 *rmap_get_next(struct rmap_iterator *iter)
1125{
1126 if (iter->desc) {
1127 if (iter->pos < PTE_LIST_EXT - 1) {
1128 u64 *sptep;
1129
1130 ++iter->pos;
1131 sptep = iter->desc->sptes[iter->pos];
1132 if (sptep)
1133 return sptep;
1134 }
1135
1136 iter->desc = iter->desc->more;
1137
1138 if (iter->desc) {
1139 iter->pos = 0;
1140 /* desc->sptes[0] cannot be NULL */
1141 return iter->desc->sptes[iter->pos];
1142 }
1143 }
1144
1145 return NULL;
1146}
1147
Xiao Guangrong0d536792015-05-13 14:42:20 +08001148#define for_each_rmap_spte(_rmap_, _iter_, _spte_) \
1149 for (_spte_ = rmap_get_first(*_rmap_, _iter_); \
1150 _spte_ && ({BUG_ON(!is_shadow_present_pte(*_spte_)); 1;}); \
1151 _spte_ = rmap_get_next(_iter_))
1152
Xiao Guangrongc3707952011-07-12 03:28:04 +08001153static void drop_spte(struct kvm *kvm, u64 *sptep)
Xiao Guangronge4b502e2010-07-16 11:28:09 +08001154{
Xiao Guangrong1df9f2d2011-07-12 03:30:35 +08001155 if (mmu_spte_clear_track_bits(sptep))
Marcelo Tosattieb45fda2010-10-25 11:58:22 -02001156 rmap_remove(kvm, sptep);
Avi Kivitybe38d272010-06-06 14:31:27 +03001157}
1158
Xiao Guangrong8e22f952012-06-20 15:57:39 +08001159
1160static bool __drop_large_spte(struct kvm *kvm, u64 *sptep)
1161{
1162 if (is_large_pte(*sptep)) {
1163 WARN_ON(page_header(__pa(sptep))->role.level ==
1164 PT_PAGE_TABLE_LEVEL);
1165 drop_spte(kvm, sptep);
1166 --kvm->stat.lpages;
1167 return true;
1168 }
1169
1170 return false;
1171}
1172
1173static void drop_large_spte(struct kvm_vcpu *vcpu, u64 *sptep)
1174{
1175 if (__drop_large_spte(vcpu->kvm, sptep))
1176 kvm_flush_remote_tlbs(vcpu->kvm);
1177}
1178
1179/*
Xiao Guangrong49fde342012-06-20 15:58:58 +08001180 * Write-protect on the specified @sptep, @pt_protect indicates whether
Xiao Guangrongc126d942014-04-17 17:06:14 +08001181 * spte write-protection is caused by protecting shadow page table.
Xiao Guangrong49fde342012-06-20 15:58:58 +08001182 *
Tiejun Chenb4619662014-09-22 10:31:38 +08001183 * Note: write protection is difference between dirty logging and spte
Xiao Guangrong49fde342012-06-20 15:58:58 +08001184 * protection:
1185 * - for dirty logging, the spte can be set to writable at anytime if
1186 * its dirty bitmap is properly set.
1187 * - for spte protection, the spte can be writable only after unsync-ing
1188 * shadow page.
Xiao Guangrong8e22f952012-06-20 15:57:39 +08001189 *
Xiao Guangrongc126d942014-04-17 17:06:14 +08001190 * Return true if tlb need be flushed.
Xiao Guangrong8e22f952012-06-20 15:57:39 +08001191 */
Xiao Guangrongc126d942014-04-17 17:06:14 +08001192static bool spte_write_protect(struct kvm *kvm, u64 *sptep, bool pt_protect)
Xiao Guangrongd13bc5b2012-06-20 15:57:15 +08001193{
1194 u64 spte = *sptep;
1195
Xiao Guangrong49fde342012-06-20 15:58:58 +08001196 if (!is_writable_pte(spte) &&
1197 !(pt_protect && spte_is_locklessly_modifiable(spte)))
Xiao Guangrongd13bc5b2012-06-20 15:57:15 +08001198 return false;
1199
1200 rmap_printk("rmap_write_protect: spte %p %llx\n", sptep, *sptep);
1201
Xiao Guangrong49fde342012-06-20 15:58:58 +08001202 if (pt_protect)
1203 spte &= ~SPTE_MMU_WRITEABLE;
Xiao Guangrongd13bc5b2012-06-20 15:57:15 +08001204 spte = spte & ~PT_WRITABLE_MASK;
Xiao Guangrong49fde342012-06-20 15:58:58 +08001205
Xiao Guangrongc126d942014-04-17 17:06:14 +08001206 return mmu_spte_update(sptep, spte);
Xiao Guangrongd13bc5b2012-06-20 15:57:15 +08001207}
1208
Xiao Guangrong49fde342012-06-20 15:58:58 +08001209static bool __rmap_write_protect(struct kvm *kvm, unsigned long *rmapp,
Takuya Yoshikawa245c3912013-01-08 19:44:09 +09001210 bool pt_protect)
Takuya Yoshikawaa0ed4602012-03-01 19:31:22 +09001211{
Takuya Yoshikawa1e3f42f2012-03-21 23:50:34 +09001212 u64 *sptep;
1213 struct rmap_iterator iter;
Xiao Guangrongd13bc5b2012-06-20 15:57:15 +08001214 bool flush = false;
Takuya Yoshikawaa0ed4602012-03-01 19:31:22 +09001215
Xiao Guangrong0d536792015-05-13 14:42:20 +08001216 for_each_rmap_spte(rmapp, &iter, sptep)
Xiao Guangrongc126d942014-04-17 17:06:14 +08001217 flush |= spte_write_protect(kvm, sptep, pt_protect);
Takuya Yoshikawaa0ed4602012-03-01 19:31:22 +09001218
Xiao Guangrongd13bc5b2012-06-20 15:57:15 +08001219 return flush;
Takuya Yoshikawaa0ed4602012-03-01 19:31:22 +09001220}
1221
Kai Huangf4b4b182015-01-28 10:54:24 +08001222static bool spte_clear_dirty(struct kvm *kvm, u64 *sptep)
1223{
1224 u64 spte = *sptep;
1225
1226 rmap_printk("rmap_clear_dirty: spte %p %llx\n", sptep, *sptep);
1227
1228 spte &= ~shadow_dirty_mask;
1229
1230 return mmu_spte_update(sptep, spte);
1231}
1232
1233static bool __rmap_clear_dirty(struct kvm *kvm, unsigned long *rmapp)
1234{
1235 u64 *sptep;
1236 struct rmap_iterator iter;
1237 bool flush = false;
1238
Xiao Guangrong0d536792015-05-13 14:42:20 +08001239 for_each_rmap_spte(rmapp, &iter, sptep)
Kai Huangf4b4b182015-01-28 10:54:24 +08001240 flush |= spte_clear_dirty(kvm, sptep);
Kai Huangf4b4b182015-01-28 10:54:24 +08001241
1242 return flush;
1243}
1244
1245static bool spte_set_dirty(struct kvm *kvm, u64 *sptep)
1246{
1247 u64 spte = *sptep;
1248
1249 rmap_printk("rmap_set_dirty: spte %p %llx\n", sptep, *sptep);
1250
1251 spte |= shadow_dirty_mask;
1252
1253 return mmu_spte_update(sptep, spte);
1254}
1255
1256static bool __rmap_set_dirty(struct kvm *kvm, unsigned long *rmapp)
1257{
1258 u64 *sptep;
1259 struct rmap_iterator iter;
1260 bool flush = false;
1261
Xiao Guangrong0d536792015-05-13 14:42:20 +08001262 for_each_rmap_spte(rmapp, &iter, sptep)
Kai Huangf4b4b182015-01-28 10:54:24 +08001263 flush |= spte_set_dirty(kvm, sptep);
Kai Huangf4b4b182015-01-28 10:54:24 +08001264
1265 return flush;
1266}
1267
Takuya Yoshikawa5dc99b232012-03-01 19:32:16 +09001268/**
Kai Huang3b0f1d02015-01-28 10:54:23 +08001269 * kvm_mmu_write_protect_pt_masked - write protect selected PT level pages
Takuya Yoshikawa5dc99b232012-03-01 19:32:16 +09001270 * @kvm: kvm instance
1271 * @slot: slot to protect
1272 * @gfn_offset: start of the BITS_PER_LONG pages we care about
1273 * @mask: indicates which pages we should protect
1274 *
1275 * Used when we do not need to care about huge page mappings: e.g. during dirty
1276 * logging we do not have any such mappings.
1277 */
Kai Huang3b0f1d02015-01-28 10:54:23 +08001278static void kvm_mmu_write_protect_pt_masked(struct kvm *kvm,
Takuya Yoshikawa5dc99b232012-03-01 19:32:16 +09001279 struct kvm_memory_slot *slot,
1280 gfn_t gfn_offset, unsigned long mask)
Izik Eidus98348e92007-10-16 14:42:30 +02001281{
Izik Eidus290fc382007-09-27 14:11:22 +02001282 unsigned long *rmapp;
Takuya Yoshikawa5dc99b232012-03-01 19:32:16 +09001283
1284 while (mask) {
Takuya Yoshikawa65fbe372012-08-01 18:02:01 +09001285 rmapp = __gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask),
1286 PT_PAGE_TABLE_LEVEL, slot);
Takuya Yoshikawa245c3912013-01-08 19:44:09 +09001287 __rmap_write_protect(kvm, rmapp, false);
Takuya Yoshikawa5dc99b232012-03-01 19:32:16 +09001288
1289 /* clear the first set bit */
1290 mask &= mask - 1;
1291 }
1292}
1293
Kai Huang3b0f1d02015-01-28 10:54:23 +08001294/**
Kai Huangf4b4b182015-01-28 10:54:24 +08001295 * kvm_mmu_clear_dirty_pt_masked - clear MMU D-bit for PT level pages
1296 * @kvm: kvm instance
1297 * @slot: slot to clear D-bit
1298 * @gfn_offset: start of the BITS_PER_LONG pages we care about
1299 * @mask: indicates which pages we should clear D-bit
1300 *
1301 * Used for PML to re-log the dirty GPAs after userspace querying dirty_bitmap.
1302 */
1303void kvm_mmu_clear_dirty_pt_masked(struct kvm *kvm,
1304 struct kvm_memory_slot *slot,
1305 gfn_t gfn_offset, unsigned long mask)
1306{
1307 unsigned long *rmapp;
1308
1309 while (mask) {
1310 rmapp = __gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask),
1311 PT_PAGE_TABLE_LEVEL, slot);
1312 __rmap_clear_dirty(kvm, rmapp);
1313
1314 /* clear the first set bit */
1315 mask &= mask - 1;
1316 }
1317}
1318EXPORT_SYMBOL_GPL(kvm_mmu_clear_dirty_pt_masked);
1319
1320/**
Kai Huang3b0f1d02015-01-28 10:54:23 +08001321 * kvm_arch_mmu_enable_log_dirty_pt_masked - enable dirty logging for selected
1322 * PT level pages.
1323 *
1324 * It calls kvm_mmu_write_protect_pt_masked to write protect selected pages to
1325 * enable dirty logging for them.
1326 *
1327 * Used when we do not need to care about huge page mappings: e.g. during dirty
1328 * logging we do not have any such mappings.
1329 */
1330void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,
1331 struct kvm_memory_slot *slot,
1332 gfn_t gfn_offset, unsigned long mask)
1333{
Kai Huang88178fd2015-01-28 10:54:27 +08001334 if (kvm_x86_ops->enable_log_dirty_pt_masked)
1335 kvm_x86_ops->enable_log_dirty_pt_masked(kvm, slot, gfn_offset,
1336 mask);
1337 else
1338 kvm_mmu_write_protect_pt_masked(kvm, slot, gfn_offset, mask);
Kai Huang3b0f1d02015-01-28 10:54:23 +08001339}
1340
Paolo Bonzini54bf36a2015-04-08 15:39:23 +02001341static bool rmap_write_protect(struct kvm_vcpu *vcpu, u64 gfn)
Takuya Yoshikawa5dc99b232012-03-01 19:32:16 +09001342{
1343 struct kvm_memory_slot *slot;
1344 unsigned long *rmapp;
1345 int i;
Xiao Guangrong2f845692012-06-20 15:56:53 +08001346 bool write_protected = false;
Takuya Yoshikawa5dc99b232012-03-01 19:32:16 +09001347
Paolo Bonzini54bf36a2015-04-08 15:39:23 +02001348 slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
Avi Kivity374cbac2007-01-05 16:36:43 -08001349
Xiao Guangrong8a3d08f2015-05-13 14:42:21 +08001350 for (i = PT_PAGE_TABLE_LEVEL; i <= PT_MAX_HUGEPAGE_LEVEL; ++i) {
Takuya Yoshikawa9373e2c02012-01-17 19:51:20 +09001351 rmapp = __gfn_to_rmap(gfn, i, slot);
Paolo Bonzini54bf36a2015-04-08 15:39:23 +02001352 write_protected |= __rmap_write_protect(vcpu->kvm, rmapp, true);
Marcelo Tosatti05da4552008-02-23 11:44:30 -03001353 }
1354
Marcelo Tosattib1a36822008-12-01 22:32:03 -02001355 return write_protected;
Avi Kivity374cbac2007-01-05 16:36:43 -08001356}
1357
Xiao Guangrong6a49f852015-05-13 14:42:25 +08001358static bool kvm_zap_rmapp(struct kvm *kvm, unsigned long *rmapp)
1359{
1360 u64 *sptep;
1361 struct rmap_iterator iter;
1362 bool flush = false;
1363
1364 while ((sptep = rmap_get_first(*rmapp, &iter))) {
1365 BUG_ON(!(*sptep & PT_PRESENT_MASK));
1366 rmap_printk("%s: spte %p %llx.\n", __func__, sptep, *sptep);
1367
1368 drop_spte(kvm, sptep);
1369 flush = true;
1370 }
1371
1372 return flush;
1373}
1374
Frederik Deweerdt8a8365c2009-10-09 11:42:56 +00001375static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp,
Andres Lagar-Cavilla8a9522d2014-09-23 12:34:54 -07001376 struct kvm_memory_slot *slot, gfn_t gfn, int level,
1377 unsigned long data)
Andrea Arcangelie930bff2008-07-25 16:24:52 +02001378{
Xiao Guangrong6a49f852015-05-13 14:42:25 +08001379 return kvm_zap_rmapp(kvm, rmapp);
Andrea Arcangelie930bff2008-07-25 16:24:52 +02001380}
1381
Frederik Deweerdt8a8365c2009-10-09 11:42:56 +00001382static int kvm_set_pte_rmapp(struct kvm *kvm, unsigned long *rmapp,
Andres Lagar-Cavilla8a9522d2014-09-23 12:34:54 -07001383 struct kvm_memory_slot *slot, gfn_t gfn, int level,
1384 unsigned long data)
Izik Eidus3da0dd42009-09-23 21:47:18 +03001385{
Takuya Yoshikawa1e3f42f2012-03-21 23:50:34 +09001386 u64 *sptep;
1387 struct rmap_iterator iter;
Izik Eidus3da0dd42009-09-23 21:47:18 +03001388 int need_flush = 0;
Takuya Yoshikawa1e3f42f2012-03-21 23:50:34 +09001389 u64 new_spte;
Izik Eidus3da0dd42009-09-23 21:47:18 +03001390 pte_t *ptep = (pte_t *)data;
1391 pfn_t new_pfn;
1392
1393 WARN_ON(pte_huge(*ptep));
1394 new_pfn = pte_pfn(*ptep);
Takuya Yoshikawa1e3f42f2012-03-21 23:50:34 +09001395
Xiao Guangrong0d536792015-05-13 14:42:20 +08001396restart:
1397 for_each_rmap_spte(rmapp, &iter, sptep) {
Andres Lagar-Cavilla8a9522d2014-09-23 12:34:54 -07001398 rmap_printk("kvm_set_pte_rmapp: spte %p %llx gfn %llx (%d)\n",
1399 sptep, *sptep, gfn, level);
Takuya Yoshikawa1e3f42f2012-03-21 23:50:34 +09001400
Izik Eidus3da0dd42009-09-23 21:47:18 +03001401 need_flush = 1;
Takuya Yoshikawa1e3f42f2012-03-21 23:50:34 +09001402
Izik Eidus3da0dd42009-09-23 21:47:18 +03001403 if (pte_write(*ptep)) {
Takuya Yoshikawa1e3f42f2012-03-21 23:50:34 +09001404 drop_spte(kvm, sptep);
Xiao Guangrong0d536792015-05-13 14:42:20 +08001405 goto restart;
Izik Eidus3da0dd42009-09-23 21:47:18 +03001406 } else {
Takuya Yoshikawa1e3f42f2012-03-21 23:50:34 +09001407 new_spte = *sptep & ~PT64_BASE_ADDR_MASK;
Izik Eidus3da0dd42009-09-23 21:47:18 +03001408 new_spte |= (u64)new_pfn << PAGE_SHIFT;
1409
1410 new_spte &= ~PT_WRITABLE_MASK;
1411 new_spte &= ~SPTE_HOST_WRITEABLE;
Avi Kivityb79b93f2010-06-06 15:46:44 +03001412 new_spte &= ~shadow_accessed_mask;
Takuya Yoshikawa1e3f42f2012-03-21 23:50:34 +09001413
1414 mmu_spte_clear_track_bits(sptep);
1415 mmu_spte_set(sptep, new_spte);
Izik Eidus3da0dd42009-09-23 21:47:18 +03001416 }
1417 }
Takuya Yoshikawa1e3f42f2012-03-21 23:50:34 +09001418
Izik Eidus3da0dd42009-09-23 21:47:18 +03001419 if (need_flush)
1420 kvm_flush_remote_tlbs(kvm);
1421
1422 return 0;
1423}
1424
Xiao Guangrong6ce1f4e2015-05-13 14:42:22 +08001425struct slot_rmap_walk_iterator {
1426 /* input fields. */
1427 struct kvm_memory_slot *slot;
1428 gfn_t start_gfn;
1429 gfn_t end_gfn;
1430 int start_level;
1431 int end_level;
1432
1433 /* output fields. */
1434 gfn_t gfn;
1435 unsigned long *rmap;
1436 int level;
1437
1438 /* private field. */
1439 unsigned long *end_rmap;
1440};
1441
1442static void
1443rmap_walk_init_level(struct slot_rmap_walk_iterator *iterator, int level)
1444{
1445 iterator->level = level;
1446 iterator->gfn = iterator->start_gfn;
1447 iterator->rmap = __gfn_to_rmap(iterator->gfn, level, iterator->slot);
1448 iterator->end_rmap = __gfn_to_rmap(iterator->end_gfn, level,
1449 iterator->slot);
1450}
1451
1452static void
1453slot_rmap_walk_init(struct slot_rmap_walk_iterator *iterator,
1454 struct kvm_memory_slot *slot, int start_level,
1455 int end_level, gfn_t start_gfn, gfn_t end_gfn)
1456{
1457 iterator->slot = slot;
1458 iterator->start_level = start_level;
1459 iterator->end_level = end_level;
1460 iterator->start_gfn = start_gfn;
1461 iterator->end_gfn = end_gfn;
1462
1463 rmap_walk_init_level(iterator, iterator->start_level);
1464}
1465
1466static bool slot_rmap_walk_okay(struct slot_rmap_walk_iterator *iterator)
1467{
1468 return !!iterator->rmap;
1469}
1470
1471static void slot_rmap_walk_next(struct slot_rmap_walk_iterator *iterator)
1472{
1473 if (++iterator->rmap <= iterator->end_rmap) {
1474 iterator->gfn += (1UL << KVM_HPAGE_GFN_SHIFT(iterator->level));
1475 return;
1476 }
1477
1478 if (++iterator->level > iterator->end_level) {
1479 iterator->rmap = NULL;
1480 return;
1481 }
1482
1483 rmap_walk_init_level(iterator, iterator->level);
1484}
1485
1486#define for_each_slot_rmap_range(_slot_, _start_level_, _end_level_, \
1487 _start_gfn, _end_gfn, _iter_) \
1488 for (slot_rmap_walk_init(_iter_, _slot_, _start_level_, \
1489 _end_level_, _start_gfn, _end_gfn); \
1490 slot_rmap_walk_okay(_iter_); \
1491 slot_rmap_walk_next(_iter_))
1492
Takuya Yoshikawa84504ef2012-07-02 17:55:48 +09001493static int kvm_handle_hva_range(struct kvm *kvm,
1494 unsigned long start,
1495 unsigned long end,
1496 unsigned long data,
1497 int (*handler)(struct kvm *kvm,
1498 unsigned long *rmapp,
Takuya Yoshikawa048212d2012-07-02 17:57:59 +09001499 struct kvm_memory_slot *slot,
Andres Lagar-Cavilla8a9522d2014-09-23 12:34:54 -07001500 gfn_t gfn,
1501 int level,
Takuya Yoshikawa84504ef2012-07-02 17:55:48 +09001502 unsigned long data))
Andrea Arcangelie930bff2008-07-25 16:24:52 +02001503{
Marcelo Tosattibc6678a2009-12-23 14:35:21 -02001504 struct kvm_memslots *slots;
Xiao Guangrongbe6ba0f2011-11-24 17:39:18 +08001505 struct kvm_memory_slot *memslot;
Xiao Guangrong6ce1f4e2015-05-13 14:42:22 +08001506 struct slot_rmap_walk_iterator iterator;
1507 int ret = 0;
Paolo Bonzini9da0e4d2015-05-18 13:33:16 +02001508 int i;
Andrea Arcangelie930bff2008-07-25 16:24:52 +02001509
Paolo Bonzini9da0e4d2015-05-18 13:33:16 +02001510 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
1511 slots = __kvm_memslots(kvm, i);
1512 kvm_for_each_memslot(memslot, slots) {
1513 unsigned long hva_start, hva_end;
1514 gfn_t gfn_start, gfn_end;
Marcelo Tosattibc6678a2009-12-23 14:35:21 -02001515
Paolo Bonzini9da0e4d2015-05-18 13:33:16 +02001516 hva_start = max(start, memslot->userspace_addr);
1517 hva_end = min(end, memslot->userspace_addr +
1518 (memslot->npages << PAGE_SHIFT));
1519 if (hva_start >= hva_end)
1520 continue;
1521 /*
1522 * {gfn(page) | page intersects with [hva_start, hva_end)} =
1523 * {gfn_start, gfn_start+1, ..., gfn_end-1}.
1524 */
1525 gfn_start = hva_to_gfn_memslot(hva_start, memslot);
1526 gfn_end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, memslot);
Andrea Arcangelie930bff2008-07-25 16:24:52 +02001527
Paolo Bonzini9da0e4d2015-05-18 13:33:16 +02001528 for_each_slot_rmap_range(memslot, PT_PAGE_TABLE_LEVEL,
1529 PT_MAX_HUGEPAGE_LEVEL,
1530 gfn_start, gfn_end - 1,
1531 &iterator)
1532 ret |= handler(kvm, iterator.rmap, memslot,
1533 iterator.gfn, iterator.level, data);
1534 }
Andrea Arcangelie930bff2008-07-25 16:24:52 +02001535 }
1536
Takuya Yoshikawaf3953022012-07-02 17:58:48 +09001537 return ret;
Andrea Arcangelie930bff2008-07-25 16:24:52 +02001538}
1539
Takuya Yoshikawa84504ef2012-07-02 17:55:48 +09001540static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,
1541 unsigned long data,
1542 int (*handler)(struct kvm *kvm, unsigned long *rmapp,
Takuya Yoshikawa048212d2012-07-02 17:57:59 +09001543 struct kvm_memory_slot *slot,
Andres Lagar-Cavilla8a9522d2014-09-23 12:34:54 -07001544 gfn_t gfn, int level,
Takuya Yoshikawa84504ef2012-07-02 17:55:48 +09001545 unsigned long data))
1546{
1547 return kvm_handle_hva_range(kvm, hva, hva + 1, data, handler);
Andrea Arcangelie930bff2008-07-25 16:24:52 +02001548}
1549
1550int kvm_unmap_hva(struct kvm *kvm, unsigned long hva)
1551{
Izik Eidus3da0dd42009-09-23 21:47:18 +03001552 return kvm_handle_hva(kvm, hva, 0, kvm_unmap_rmapp);
Andrea Arcangelie930bff2008-07-25 16:24:52 +02001553}
1554
Takuya Yoshikawab3ae2092012-07-02 17:56:33 +09001555int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end)
1556{
1557 return kvm_handle_hva_range(kvm, start, end, 0, kvm_unmap_rmapp);
1558}
1559
Izik Eidus3da0dd42009-09-23 21:47:18 +03001560void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte)
1561{
Frederik Deweerdt8a8365c2009-10-09 11:42:56 +00001562 kvm_handle_hva(kvm, hva, (unsigned long)&pte, kvm_set_pte_rmapp);
Izik Eidus3da0dd42009-09-23 21:47:18 +03001563}
1564
Frederik Deweerdt8a8365c2009-10-09 11:42:56 +00001565static int kvm_age_rmapp(struct kvm *kvm, unsigned long *rmapp,
Andres Lagar-Cavilla8a9522d2014-09-23 12:34:54 -07001566 struct kvm_memory_slot *slot, gfn_t gfn, int level,
1567 unsigned long data)
Andrea Arcangelie930bff2008-07-25 16:24:52 +02001568{
Takuya Yoshikawa1e3f42f2012-03-21 23:50:34 +09001569 u64 *sptep;
Michael S. Tsirkin79f702a2012-06-03 11:34:08 +03001570 struct rmap_iterator uninitialized_var(iter);
Andrea Arcangelie930bff2008-07-25 16:24:52 +02001571 int young = 0;
1572
Andres Lagar-Cavilla57128462014-09-22 14:54:42 -07001573 BUG_ON(!shadow_accessed_mask);
Sheng Yang534e38b2008-09-08 15:12:30 +08001574
Xiao Guangrong0d536792015-05-13 14:42:20 +08001575 for_each_rmap_spte(rmapp, &iter, sptep)
Xudong Hao3f6d8c82012-05-22 11:23:15 +08001576 if (*sptep & shadow_accessed_mask) {
Andrea Arcangelie930bff2008-07-25 16:24:52 +02001577 young = 1;
Xudong Hao3f6d8c82012-05-22 11:23:15 +08001578 clear_bit((ffs(shadow_accessed_mask) - 1),
1579 (unsigned long *)sptep);
Andrea Arcangelie930bff2008-07-25 16:24:52 +02001580 }
Xiao Guangrong0d536792015-05-13 14:42:20 +08001581
Andres Lagar-Cavilla8a9522d2014-09-23 12:34:54 -07001582 trace_kvm_age_page(gfn, level, slot, young);
Andrea Arcangelie930bff2008-07-25 16:24:52 +02001583 return young;
1584}
1585
Andrea Arcangeli8ee53822011-01-13 15:47:10 -08001586static int kvm_test_age_rmapp(struct kvm *kvm, unsigned long *rmapp,
Andres Lagar-Cavilla8a9522d2014-09-23 12:34:54 -07001587 struct kvm_memory_slot *slot, gfn_t gfn,
1588 int level, unsigned long data)
Andrea Arcangeli8ee53822011-01-13 15:47:10 -08001589{
Takuya Yoshikawa1e3f42f2012-03-21 23:50:34 +09001590 u64 *sptep;
1591 struct rmap_iterator iter;
Andrea Arcangeli8ee53822011-01-13 15:47:10 -08001592 int young = 0;
1593
1594 /*
1595 * If there's no access bit in the secondary pte set by the
1596 * hardware it's up to gup-fast/gup to set the access bit in
1597 * the primary pte or in the page structure.
1598 */
1599 if (!shadow_accessed_mask)
1600 goto out;
1601
Xiao Guangrong0d536792015-05-13 14:42:20 +08001602 for_each_rmap_spte(rmapp, &iter, sptep)
Xudong Hao3f6d8c82012-05-22 11:23:15 +08001603 if (*sptep & shadow_accessed_mask) {
Andrea Arcangeli8ee53822011-01-13 15:47:10 -08001604 young = 1;
1605 break;
1606 }
Andrea Arcangeli8ee53822011-01-13 15:47:10 -08001607out:
1608 return young;
1609}
1610
Marcelo Tosatti53a27b32009-08-05 15:43:58 -03001611#define RMAP_RECYCLE_THRESHOLD 1000
1612
Joerg Roedel852e3c12009-07-27 16:30:44 +02001613static void rmap_recycle(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)
Marcelo Tosatti53a27b32009-08-05 15:43:58 -03001614{
1615 unsigned long *rmapp;
Joerg Roedel852e3c12009-07-27 16:30:44 +02001616 struct kvm_mmu_page *sp;
1617
1618 sp = page_header(__pa(spte));
Marcelo Tosatti53a27b32009-08-05 15:43:58 -03001619
Paolo Bonzinie4cd1da2015-05-18 15:11:46 +02001620 rmapp = gfn_to_rmap(vcpu->kvm, gfn, sp);
Marcelo Tosatti53a27b32009-08-05 15:43:58 -03001621
Andres Lagar-Cavilla8a9522d2014-09-23 12:34:54 -07001622 kvm_unmap_rmapp(vcpu->kvm, rmapp, NULL, gfn, sp->role.level, 0);
Marcelo Tosatti53a27b32009-08-05 15:43:58 -03001623 kvm_flush_remote_tlbs(vcpu->kvm);
1624}
1625
Andres Lagar-Cavilla57128462014-09-22 14:54:42 -07001626int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end)
Andrea Arcangelie930bff2008-07-25 16:24:52 +02001627{
Andres Lagar-Cavilla57128462014-09-22 14:54:42 -07001628 /*
1629 * In case of absence of EPT Access and Dirty Bits supports,
1630 * emulate the accessed bit for EPT, by checking if this page has
1631 * an EPT mapping, and clearing it if it does. On the next access,
1632 * a new EPT mapping will be established.
1633 * This has some overhead, but not as much as the cost of swapping
1634 * out actively used pages or breaking up actively used hugepages.
1635 */
1636 if (!shadow_accessed_mask) {
1637 /*
1638 * We are holding the kvm->mmu_lock, and we are blowing up
1639 * shadow PTEs. MMU notifier consumers need to be kept at bay.
1640 * This is correct as long as we don't decouple the mmu_lock
1641 * protected regions (like invalidate_range_start|end does).
1642 */
1643 kvm->mmu_notifier_seq++;
1644 return kvm_handle_hva_range(kvm, start, end, 0,
1645 kvm_unmap_rmapp);
1646 }
1647
1648 return kvm_handle_hva_range(kvm, start, end, 0, kvm_age_rmapp);
Andrea Arcangelie930bff2008-07-25 16:24:52 +02001649}
1650
Andrea Arcangeli8ee53822011-01-13 15:47:10 -08001651int kvm_test_age_hva(struct kvm *kvm, unsigned long hva)
1652{
1653 return kvm_handle_hva(kvm, hva, 0, kvm_test_age_rmapp);
1654}
1655
Yaozu Dongd6c69ee2007-04-25 14:17:25 +08001656#ifdef MMU_DEBUG
Avi Kivity47ad8e62007-05-06 15:50:58 +03001657static int is_empty_shadow_page(u64 *spt)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001658{
Avi Kivity139bdb22007-01-05 16:36:50 -08001659 u64 *pos;
1660 u64 *end;
1661
Avi Kivity47ad8e62007-05-06 15:50:58 +03001662 for (pos = spt, end = pos + PAGE_SIZE / sizeof(u64); pos != end; pos++)
Avi Kivity3c915512008-05-20 16:21:13 +03001663 if (is_shadow_present_pte(*pos)) {
Harvey Harrisonb8688d52008-03-03 12:59:56 -08001664 printk(KERN_ERR "%s: %p %llx\n", __func__,
Avi Kivity139bdb22007-01-05 16:36:50 -08001665 pos, *pos);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001666 return 0;
Avi Kivity139bdb22007-01-05 16:36:50 -08001667 }
Avi Kivity6aa8b732006-12-10 02:21:36 -08001668 return 1;
1669}
Yaozu Dongd6c69ee2007-04-25 14:17:25 +08001670#endif
Avi Kivity6aa8b732006-12-10 02:21:36 -08001671
Dave Hansen45221ab2010-08-19 18:11:37 -07001672/*
1673 * This value is the sum of all of the kvm instances's
1674 * kvm->arch.n_used_mmu_pages values. We need a global,
1675 * aggregate version in order to make the slab shrinker
1676 * faster
1677 */
1678static inline void kvm_mod_used_mmu_pages(struct kvm *kvm, int nr)
1679{
1680 kvm->arch.n_used_mmu_pages += nr;
1681 percpu_counter_add(&kvm_total_used_mmu_pages, nr);
1682}
1683
Gleb Natapov834be0d2013-01-30 16:45:05 +02001684static void kvm_mmu_free_page(struct kvm_mmu_page *sp)
Avi Kivity260746c2007-01-05 16:36:49 -08001685{
Paolo Bonzinifa4a2c02013-10-02 16:56:16 +02001686 MMU_WARN_ON(!is_empty_shadow_page(sp->spt));
Xiao Guangrong77758342010-06-04 21:53:54 +08001687 hlist_del(&sp->hash_link);
Xiao Guangrongbd4c86e2011-07-12 03:27:14 +08001688 list_del(&sp->link);
1689 free_page((unsigned long)sp->spt);
Gleb Natapov834be0d2013-01-30 16:45:05 +02001690 if (!sp->role.direct)
1691 free_page((unsigned long)sp->gfns);
Xiao Guangronge8ad9a72010-05-13 10:06:02 +08001692 kmem_cache_free(mmu_page_header_cache, sp);
Avi Kivity260746c2007-01-05 16:36:49 -08001693}
1694
Avi Kivitycea0f0e2007-01-05 16:36:43 -08001695static unsigned kvm_page_table_hashfn(gfn_t gfn)
1696{
Dong, Eddie1ae0a132008-01-07 13:20:25 +02001697 return gfn & ((1 << KVM_MMU_HASH_SHIFT) - 1);
Avi Kivitycea0f0e2007-01-05 16:36:43 -08001698}
1699
Xiao Guangrong67052b32011-05-15 23:27:08 +08001700static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu,
1701 struct kvm_mmu_page *sp, u64 *parent_pte)
1702{
1703 if (!parent_pte)
1704 return;
1705
1706 pte_list_add(vcpu, parent_pte, &sp->parent_ptes);
1707}
1708
1709static void mmu_page_remove_parent_pte(struct kvm_mmu_page *sp,
1710 u64 *parent_pte)
1711{
1712 pte_list_remove(parent_pte, &sp->parent_ptes);
1713}
1714
Xiao Guangrongbcdd9a92011-05-15 23:28:29 +08001715static void drop_parent_pte(struct kvm_mmu_page *sp,
1716 u64 *parent_pte)
1717{
1718 mmu_page_remove_parent_pte(sp, parent_pte);
Xiao Guangrong1df9f2d2011-07-12 03:30:35 +08001719 mmu_spte_clear_no_track(parent_pte);
Xiao Guangrongbcdd9a92011-05-15 23:28:29 +08001720}
1721
Avi Kivity25c0de22007-01-05 16:36:42 -08001722static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu,
Lai Jiangshan2032a932010-05-26 16:49:59 +08001723 u64 *parent_pte, int direct)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001724{
Avi Kivity4db35312007-11-21 15:28:32 +02001725 struct kvm_mmu_page *sp;
Takuya Yoshikawa7ddca7e2013-03-21 19:33:43 +09001726
Takuya Yoshikawa80feb892012-05-29 23:54:26 +09001727 sp = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache);
1728 sp->spt = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache);
Lai Jiangshan2032a932010-05-26 16:49:59 +08001729 if (!direct)
Takuya Yoshikawa80feb892012-05-29 23:54:26 +09001730 sp->gfns = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache);
Avi Kivity4db35312007-11-21 15:28:32 +02001731 set_page_private(virt_to_page(sp->spt), (unsigned long)sp);
Xiao Guangrong5304b8d2013-05-31 08:36:22 +08001732
1733 /*
1734 * The active_mmu_pages list is the FIFO list, do not move the
1735 * page until it is zapped. kvm_zap_obsolete_pages depends on
1736 * this feature. See the comments in kvm_zap_obsolete_pages().
1737 */
Zhang Xiantaof05e70a2007-12-14 10:01:48 +08001738 list_add(&sp->link, &vcpu->kvm->arch.active_mmu_pages);
Xiao Guangrong67052b32011-05-15 23:27:08 +08001739 sp->parent_ptes = 0;
1740 mmu_page_add_parent_pte(vcpu, sp, parent_pte);
Dave Hansen45221ab2010-08-19 18:11:37 -07001741 kvm_mod_used_mmu_pages(vcpu->kvm, +1);
Avi Kivity4db35312007-11-21 15:28:32 +02001742 return sp;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001743}
1744
Xiao Guangrong67052b32011-05-15 23:27:08 +08001745static void mark_unsync(u64 *spte);
Xiao Guangrong6b184932010-04-16 21:29:17 +08001746static void kvm_mmu_mark_parents_unsync(struct kvm_mmu_page *sp)
Marcelo Tosatti0074ff62008-09-23 13:18:40 -03001747{
Xiao Guangrong67052b32011-05-15 23:27:08 +08001748 pte_list_walk(&sp->parent_ptes, mark_unsync);
Xiao Guangrong1047df12010-06-11 21:35:15 +08001749}
1750
Xiao Guangrong67052b32011-05-15 23:27:08 +08001751static void mark_unsync(u64 *spte)
Xiao Guangrong1047df12010-06-11 21:35:15 +08001752{
Xiao Guangrong67052b32011-05-15 23:27:08 +08001753 struct kvm_mmu_page *sp;
Xiao Guangrong1047df12010-06-11 21:35:15 +08001754 unsigned int index;
1755
Xiao Guangrong67052b32011-05-15 23:27:08 +08001756 sp = page_header(__pa(spte));
Xiao Guangrong1047df12010-06-11 21:35:15 +08001757 index = spte - sp->spt;
1758 if (__test_and_set_bit(index, sp->unsync_child_bitmap))
1759 return;
1760 if (sp->unsync_children++)
1761 return;
1762 kvm_mmu_mark_parents_unsync(sp);
Marcelo Tosatti0074ff62008-09-23 13:18:40 -03001763}
1764
Marcelo Tosattie8bc2172008-09-23 13:18:33 -03001765static int nonpaging_sync_page(struct kvm_vcpu *vcpu,
Xiao Guangronga4a8e6f2010-11-19 17:04:03 +08001766 struct kvm_mmu_page *sp)
Marcelo Tosattie8bc2172008-09-23 13:18:33 -03001767{
1768 return 1;
1769}
1770
Marcelo Tosattia7052892008-09-23 13:18:35 -03001771static void nonpaging_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
1772{
1773}
1774
Xiao Guangrong0f53b5b2011-03-09 15:43:51 +08001775static void nonpaging_update_pte(struct kvm_vcpu *vcpu,
1776 struct kvm_mmu_page *sp, u64 *spte,
Xiao Guangrong7c562522011-03-28 10:29:27 +08001777 const void *pte)
Xiao Guangrong0f53b5b2011-03-09 15:43:51 +08001778{
1779 WARN_ON(1);
1780}
1781
Marcelo Tosatti60c8aec2008-12-01 22:32:02 -02001782#define KVM_PAGE_ARRAY_NR 16
1783
1784struct kvm_mmu_pages {
1785 struct mmu_page_and_offset {
1786 struct kvm_mmu_page *sp;
1787 unsigned int idx;
1788 } page[KVM_PAGE_ARRAY_NR];
1789 unsigned int nr;
1790};
1791
Hannes Edercded19f2009-02-21 02:19:13 +01001792static int mmu_pages_add(struct kvm_mmu_pages *pvec, struct kvm_mmu_page *sp,
1793 int idx)
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001794{
Marcelo Tosatti60c8aec2008-12-01 22:32:02 -02001795 int i;
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001796
Marcelo Tosatti60c8aec2008-12-01 22:32:02 -02001797 if (sp->unsync)
1798 for (i=0; i < pvec->nr; i++)
1799 if (pvec->page[i].sp == sp)
1800 return 0;
1801
1802 pvec->page[pvec->nr].sp = sp;
1803 pvec->page[pvec->nr].idx = idx;
1804 pvec->nr++;
1805 return (pvec->nr == KVM_PAGE_ARRAY_NR);
1806}
1807
1808static int __mmu_unsync_walk(struct kvm_mmu_page *sp,
1809 struct kvm_mmu_pages *pvec)
1810{
1811 int i, ret, nr_unsync_leaf = 0;
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001812
Takuya Yoshikawa37178b82011-11-29 14:02:45 +09001813 for_each_set_bit(i, sp->unsync_child_bitmap, 512) {
Xiao Guangrong7a8f1a72010-06-11 21:34:04 +08001814 struct kvm_mmu_page *child;
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001815 u64 ent = sp->spt[i];
1816
Xiao Guangrong7a8f1a72010-06-11 21:34:04 +08001817 if (!is_shadow_present_pte(ent) || is_large_pte(ent))
1818 goto clear_child_bitmap;
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001819
Xiao Guangrong7a8f1a72010-06-11 21:34:04 +08001820 child = page_header(ent & PT64_BASE_ADDR_MASK);
Marcelo Tosatti60c8aec2008-12-01 22:32:02 -02001821
Xiao Guangrong7a8f1a72010-06-11 21:34:04 +08001822 if (child->unsync_children) {
1823 if (mmu_pages_add(pvec, child, i))
1824 return -ENOSPC;
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001825
Xiao Guangrong7a8f1a72010-06-11 21:34:04 +08001826 ret = __mmu_unsync_walk(child, pvec);
1827 if (!ret)
1828 goto clear_child_bitmap;
1829 else if (ret > 0)
1830 nr_unsync_leaf += ret;
1831 else
1832 return ret;
1833 } else if (child->unsync) {
1834 nr_unsync_leaf++;
1835 if (mmu_pages_add(pvec, child, i))
1836 return -ENOSPC;
1837 } else
1838 goto clear_child_bitmap;
1839
1840 continue;
1841
1842clear_child_bitmap:
1843 __clear_bit(i, sp->unsync_child_bitmap);
1844 sp->unsync_children--;
1845 WARN_ON((int)sp->unsync_children < 0);
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001846 }
1847
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001848
Marcelo Tosatti60c8aec2008-12-01 22:32:02 -02001849 return nr_unsync_leaf;
1850}
1851
1852static int mmu_unsync_walk(struct kvm_mmu_page *sp,
1853 struct kvm_mmu_pages *pvec)
1854{
1855 if (!sp->unsync_children)
1856 return 0;
1857
1858 mmu_pages_add(pvec, sp, 0);
1859 return __mmu_unsync_walk(sp, pvec);
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001860}
1861
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001862static void kvm_unlink_unsync_page(struct kvm *kvm, struct kvm_mmu_page *sp)
1863{
1864 WARN_ON(!sp->unsync);
Xiao Guangrong5e1b3dd2010-04-28 11:55:06 +08001865 trace_kvm_mmu_sync_page(sp);
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001866 sp->unsync = 0;
1867 --kvm->stat.mmu_unsync;
1868}
1869
Xiao Guangrong77758342010-06-04 21:53:54 +08001870static int kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
1871 struct list_head *invalid_list);
1872static void kvm_mmu_commit_zap_page(struct kvm *kvm,
1873 struct list_head *invalid_list);
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001874
Xiao Guangrongf34d2512013-05-31 08:36:28 +08001875/*
1876 * NOTE: we should pay more attention on the zapped-obsolete page
1877 * (is_obsolete_sp(sp) && sp->role.invalid) when you do hash list walk
1878 * since it has been deleted from active_mmu_pages but still can be found
1879 * at hast list.
1880 *
1881 * for_each_gfn_indirect_valid_sp has skipped that kind of page and
1882 * kvm_mmu_get_page(), the only user of for_each_gfn_sp(), has skipped
1883 * all the obsolete pages.
1884 */
Takuya Yoshikawa1044b032013-03-06 16:05:07 +09001885#define for_each_gfn_sp(_kvm, _sp, _gfn) \
1886 hlist_for_each_entry(_sp, \
1887 &(_kvm)->arch.mmu_page_hash[kvm_page_table_hashfn(_gfn)], hash_link) \
1888 if ((_sp)->gfn != (_gfn)) {} else
Xiao Guangrong7ae680e2010-06-04 21:53:07 +08001889
Takuya Yoshikawa1044b032013-03-06 16:05:07 +09001890#define for_each_gfn_indirect_valid_sp(_kvm, _sp, _gfn) \
1891 for_each_gfn_sp(_kvm, _sp, _gfn) \
1892 if ((_sp)->role.direct || (_sp)->role.invalid) {} else
Xiao Guangrong7ae680e2010-06-04 21:53:07 +08001893
Xiao Guangrongf918b442010-06-11 21:30:36 +08001894/* @sp->gfn should be write-protected at the call site */
Xiao Guangrong1d9dc7e2010-05-15 18:51:24 +08001895static int __kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
Xiao Guangrongd98ba052010-06-04 21:55:29 +08001896 struct list_head *invalid_list, bool clear_unsync)
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001897{
Avi Kivity5b7e0102010-04-14 19:20:03 +03001898 if (sp->role.cr4_pae != !!is_pae(vcpu)) {
Xiao Guangrongd98ba052010-06-04 21:55:29 +08001899 kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list);
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001900 return 1;
1901 }
1902
Xiao Guangrongf918b442010-06-11 21:30:36 +08001903 if (clear_unsync)
Xiao Guangrong1d9dc7e2010-05-15 18:51:24 +08001904 kvm_unlink_unsync_page(vcpu->kvm, sp);
Xiao Guangrong1d9dc7e2010-05-15 18:51:24 +08001905
Xiao Guangronga4a8e6f2010-11-19 17:04:03 +08001906 if (vcpu->arch.mmu.sync_page(vcpu, sp)) {
Xiao Guangrongd98ba052010-06-04 21:55:29 +08001907 kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list);
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001908 return 1;
1909 }
1910
Liang Chen77c39132014-09-18 12:38:37 -04001911 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001912 return 0;
1913}
1914
Xiao Guangrong1d9dc7e2010-05-15 18:51:24 +08001915static int kvm_sync_page_transient(struct kvm_vcpu *vcpu,
1916 struct kvm_mmu_page *sp)
1917{
Xiao Guangrongd98ba052010-06-04 21:55:29 +08001918 LIST_HEAD(invalid_list);
Xiao Guangrong1d9dc7e2010-05-15 18:51:24 +08001919 int ret;
1920
Xiao Guangrongd98ba052010-06-04 21:55:29 +08001921 ret = __kvm_sync_page(vcpu, sp, &invalid_list, false);
Xiao Guangrongbe71e062010-06-11 21:31:38 +08001922 if (ret)
Xiao Guangrongd98ba052010-06-04 21:55:29 +08001923 kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
1924
Xiao Guangrong1d9dc7e2010-05-15 18:51:24 +08001925 return ret;
1926}
1927
Xiao Guangronge37fa782011-11-30 17:43:24 +08001928#ifdef CONFIG_KVM_MMU_AUDIT
1929#include "mmu_audit.c"
1930#else
1931static void kvm_mmu_audit(struct kvm_vcpu *vcpu, int point) { }
1932static void mmu_audit_disable(void) { }
1933#endif
1934
Xiao Guangrongd98ba052010-06-04 21:55:29 +08001935static int kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
1936 struct list_head *invalid_list)
Xiao Guangrong1d9dc7e2010-05-15 18:51:24 +08001937{
Xiao Guangrongd98ba052010-06-04 21:55:29 +08001938 return __kvm_sync_page(vcpu, sp, invalid_list, true);
Xiao Guangrong1d9dc7e2010-05-15 18:51:24 +08001939}
1940
Xiao Guangrong9f1a1222010-05-24 15:41:33 +08001941/* @gfn should be write-protected at the call site */
1942static void kvm_sync_pages(struct kvm_vcpu *vcpu, gfn_t gfn)
1943{
Xiao Guangrong9f1a1222010-05-24 15:41:33 +08001944 struct kvm_mmu_page *s;
Xiao Guangrongd98ba052010-06-04 21:55:29 +08001945 LIST_HEAD(invalid_list);
Xiao Guangrong9f1a1222010-05-24 15:41:33 +08001946 bool flush = false;
1947
Sasha Levinb67bfe02013-02-27 17:06:00 -08001948 for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn) {
Xiao Guangrong7ae680e2010-06-04 21:53:07 +08001949 if (!s->unsync)
Xiao Guangrong9f1a1222010-05-24 15:41:33 +08001950 continue;
1951
1952 WARN_ON(s->role.level != PT_PAGE_TABLE_LEVEL);
Xiao Guangronga4a8e6f2010-11-19 17:04:03 +08001953 kvm_unlink_unsync_page(vcpu->kvm, s);
Xiao Guangrong9f1a1222010-05-24 15:41:33 +08001954 if ((s->role.cr4_pae != !!is_pae(vcpu)) ||
Xiao Guangronga4a8e6f2010-11-19 17:04:03 +08001955 (vcpu->arch.mmu.sync_page(vcpu, s))) {
Xiao Guangrongd98ba052010-06-04 21:55:29 +08001956 kvm_mmu_prepare_zap_page(vcpu->kvm, s, &invalid_list);
Xiao Guangrong9f1a1222010-05-24 15:41:33 +08001957 continue;
1958 }
Xiao Guangrong9f1a1222010-05-24 15:41:33 +08001959 flush = true;
1960 }
1961
Xiao Guangrongd98ba052010-06-04 21:55:29 +08001962 kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
Xiao Guangrong9f1a1222010-05-24 15:41:33 +08001963 if (flush)
Liang Chen77c39132014-09-18 12:38:37 -04001964 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
Xiao Guangrong9f1a1222010-05-24 15:41:33 +08001965}
1966
Marcelo Tosatti60c8aec2008-12-01 22:32:02 -02001967struct mmu_page_path {
1968 struct kvm_mmu_page *parent[PT64_ROOT_LEVEL-1];
1969 unsigned int idx[PT64_ROOT_LEVEL-1];
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001970};
1971
Marcelo Tosatti60c8aec2008-12-01 22:32:02 -02001972#define for_each_sp(pvec, sp, parents, i) \
1973 for (i = mmu_pages_next(&pvec, &parents, -1), \
1974 sp = pvec.page[i].sp; \
1975 i < pvec.nr && ({ sp = pvec.page[i].sp; 1;}); \
1976 i = mmu_pages_next(&pvec, &parents, i))
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001977
Hannes Edercded19f2009-02-21 02:19:13 +01001978static int mmu_pages_next(struct kvm_mmu_pages *pvec,
1979 struct mmu_page_path *parents,
1980 int i)
Marcelo Tosatti60c8aec2008-12-01 22:32:02 -02001981{
1982 int n;
1983
1984 for (n = i+1; n < pvec->nr; n++) {
1985 struct kvm_mmu_page *sp = pvec->page[n].sp;
1986
1987 if (sp->role.level == PT_PAGE_TABLE_LEVEL) {
1988 parents->idx[0] = pvec->page[n].idx;
1989 return n;
1990 }
1991
1992 parents->parent[sp->role.level-2] = sp;
1993 parents->idx[sp->role.level-1] = pvec->page[n].idx;
1994 }
1995
1996 return n;
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001997}
1998
Hannes Edercded19f2009-02-21 02:19:13 +01001999static void mmu_pages_clear_parents(struct mmu_page_path *parents)
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03002000{
Marcelo Tosatti60c8aec2008-12-01 22:32:02 -02002001 struct kvm_mmu_page *sp;
2002 unsigned int level = 0;
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03002003
Marcelo Tosatti60c8aec2008-12-01 22:32:02 -02002004 do {
2005 unsigned int idx = parents->idx[level];
2006
2007 sp = parents->parent[level];
2008 if (!sp)
2009 return;
2010
2011 --sp->unsync_children;
2012 WARN_ON((int)sp->unsync_children < 0);
2013 __clear_bit(idx, sp->unsync_child_bitmap);
2014 level++;
2015 } while (level < PT64_ROOT_LEVEL-1 && !sp->unsync_children);
2016}
2017
2018static void kvm_mmu_pages_init(struct kvm_mmu_page *parent,
2019 struct mmu_page_path *parents,
2020 struct kvm_mmu_pages *pvec)
2021{
2022 parents->parent[parent->role.level-1] = NULL;
2023 pvec->nr = 0;
2024}
2025
2026static void mmu_sync_children(struct kvm_vcpu *vcpu,
2027 struct kvm_mmu_page *parent)
2028{
2029 int i;
2030 struct kvm_mmu_page *sp;
2031 struct mmu_page_path parents;
2032 struct kvm_mmu_pages pages;
Xiao Guangrongd98ba052010-06-04 21:55:29 +08002033 LIST_HEAD(invalid_list);
Marcelo Tosatti60c8aec2008-12-01 22:32:02 -02002034
2035 kvm_mmu_pages_init(parent, &parents, &pages);
2036 while (mmu_unsync_walk(parent, &pages)) {
Xiao Guangrong2f845692012-06-20 15:56:53 +08002037 bool protected = false;
Marcelo Tosattib1a36822008-12-01 22:32:03 -02002038
2039 for_each_sp(pages, sp, parents, i)
Paolo Bonzini54bf36a2015-04-08 15:39:23 +02002040 protected |= rmap_write_protect(vcpu, sp->gfn);
Marcelo Tosattib1a36822008-12-01 22:32:03 -02002041
2042 if (protected)
2043 kvm_flush_remote_tlbs(vcpu->kvm);
2044
Marcelo Tosatti60c8aec2008-12-01 22:32:02 -02002045 for_each_sp(pages, sp, parents, i) {
Xiao Guangrongd98ba052010-06-04 21:55:29 +08002046 kvm_sync_page(vcpu, sp, &invalid_list);
Marcelo Tosatti60c8aec2008-12-01 22:32:02 -02002047 mmu_pages_clear_parents(&parents);
2048 }
Xiao Guangrongd98ba052010-06-04 21:55:29 +08002049 kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03002050 cond_resched_lock(&vcpu->kvm->mmu_lock);
Marcelo Tosatti60c8aec2008-12-01 22:32:02 -02002051 kvm_mmu_pages_init(parent, &parents, &pages);
2052 }
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03002053}
2054
Xiao Guangrongc3707952011-07-12 03:28:04 +08002055static void init_shadow_page_table(struct kvm_mmu_page *sp)
2056{
2057 int i;
2058
2059 for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
2060 sp->spt[i] = 0ull;
2061}
2062
Xiao Guangronga30f47c2011-09-22 16:58:36 +08002063static void __clear_sp_write_flooding_count(struct kvm_mmu_page *sp)
2064{
2065 sp->write_flooding_count = 0;
2066}
2067
2068static void clear_sp_write_flooding_count(u64 *spte)
2069{
2070 struct kvm_mmu_page *sp = page_header(__pa(spte));
2071
2072 __clear_sp_write_flooding_count(sp);
2073}
2074
Xiao Guangrong5304b8d2013-05-31 08:36:22 +08002075static bool is_obsolete_sp(struct kvm *kvm, struct kvm_mmu_page *sp)
2076{
2077 return unlikely(sp->mmu_valid_gen != kvm->arch.mmu_valid_gen);
2078}
2079
Avi Kivitycea0f0e2007-01-05 16:36:43 -08002080static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
2081 gfn_t gfn,
2082 gva_t gaddr,
2083 unsigned level,
Avi Kivityf6e2c02b2009-01-11 13:02:10 +02002084 int direct,
Avi Kivity41074d02007-12-09 17:00:02 +02002085 unsigned access,
Avi Kivityf7d9c7b2008-02-26 22:12:10 +02002086 u64 *parent_pte)
Avi Kivitycea0f0e2007-01-05 16:36:43 -08002087{
2088 union kvm_mmu_page_role role;
Avi Kivitycea0f0e2007-01-05 16:36:43 -08002089 unsigned quadrant;
Xiao Guangrong9f1a1222010-05-24 15:41:33 +08002090 struct kvm_mmu_page *sp;
Xiao Guangrong9f1a1222010-05-24 15:41:33 +08002091 bool need_sync = false;
Avi Kivitycea0f0e2007-01-05 16:36:43 -08002092
Avi Kivitya770f6f2008-12-21 19:20:09 +02002093 role = vcpu->arch.mmu.base_role;
Avi Kivitycea0f0e2007-01-05 16:36:43 -08002094 role.level = level;
Avi Kivityf6e2c02b2009-01-11 13:02:10 +02002095 role.direct = direct;
Avi Kivity84b0c8c2010-03-14 10:16:40 +02002096 if (role.direct)
Avi Kivity5b7e0102010-04-14 19:20:03 +03002097 role.cr4_pae = 0;
Avi Kivity41074d02007-12-09 17:00:02 +02002098 role.access = access;
Joerg Roedelc5a78f2b2010-09-10 17:30:39 +02002099 if (!vcpu->arch.mmu.direct_map
2100 && vcpu->arch.mmu.root_level <= PT32_ROOT_LEVEL) {
Avi Kivitycea0f0e2007-01-05 16:36:43 -08002101 quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level));
2102 quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1;
2103 role.quadrant = quadrant;
2104 }
Sasha Levinb67bfe02013-02-27 17:06:00 -08002105 for_each_gfn_sp(vcpu->kvm, sp, gfn) {
Xiao Guangrong7f52af72013-05-31 08:36:26 +08002106 if (is_obsolete_sp(vcpu->kvm, sp))
2107 continue;
2108
Xiao Guangrong7ae680e2010-06-04 21:53:07 +08002109 if (!need_sync && sp->unsync)
2110 need_sync = true;
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03002111
Xiao Guangrong7ae680e2010-06-04 21:53:07 +08002112 if (sp->role.word != role.word)
2113 continue;
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03002114
Xiao Guangrong7ae680e2010-06-04 21:53:07 +08002115 if (sp->unsync && kvm_sync_page_transient(vcpu, sp))
2116 break;
Xiao Guangronge02aa902010-05-15 18:52:34 +08002117
Xiao Guangrong7ae680e2010-06-04 21:53:07 +08002118 mmu_page_add_parent_pte(vcpu, sp, parent_pte);
2119 if (sp->unsync_children) {
Avi Kivitya8eeb042010-05-10 12:34:53 +03002120 kvm_make_request(KVM_REQ_MMU_SYNC, vcpu);
Xiao Guangrong7ae680e2010-06-04 21:53:07 +08002121 kvm_mmu_mark_parents_unsync(sp);
2122 } else if (sp->unsync)
2123 kvm_mmu_mark_parents_unsync(sp);
Xiao Guangronge02aa902010-05-15 18:52:34 +08002124
Xiao Guangronga30f47c2011-09-22 16:58:36 +08002125 __clear_sp_write_flooding_count(sp);
Xiao Guangrong7ae680e2010-06-04 21:53:07 +08002126 trace_kvm_mmu_get_page(sp, false);
2127 return sp;
2128 }
Avi Kivitydfc5aa02007-12-18 19:47:18 +02002129 ++vcpu->kvm->stat.mmu_cache_miss;
Lai Jiangshan2032a932010-05-26 16:49:59 +08002130 sp = kvm_mmu_alloc_page(vcpu, parent_pte, direct);
Avi Kivity4db35312007-11-21 15:28:32 +02002131 if (!sp)
2132 return sp;
Avi Kivity4db35312007-11-21 15:28:32 +02002133 sp->gfn = gfn;
2134 sp->role = role;
Xiao Guangrong7ae680e2010-06-04 21:53:07 +08002135 hlist_add_head(&sp->hash_link,
2136 &vcpu->kvm->arch.mmu_page_hash[kvm_page_table_hashfn(gfn)]);
Avi Kivityf6e2c02b2009-01-11 13:02:10 +02002137 if (!direct) {
Paolo Bonzini54bf36a2015-04-08 15:39:23 +02002138 if (rmap_write_protect(vcpu, gfn))
Marcelo Tosattib1a36822008-12-01 22:32:03 -02002139 kvm_flush_remote_tlbs(vcpu->kvm);
Xiao Guangrong9f1a1222010-05-24 15:41:33 +08002140 if (level > PT_PAGE_TABLE_LEVEL && need_sync)
2141 kvm_sync_pages(vcpu, gfn);
2142
Paolo Bonzini3ed1a472015-05-19 16:29:22 +02002143 account_shadowed(vcpu->kvm, sp);
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03002144 }
Xiao Guangrong5304b8d2013-05-31 08:36:22 +08002145 sp->mmu_valid_gen = vcpu->kvm->arch.mmu_valid_gen;
Xiao Guangrongc3707952011-07-12 03:28:04 +08002146 init_shadow_page_table(sp);
Avi Kivityf691fe12009-07-06 15:58:14 +03002147 trace_kvm_mmu_get_page(sp, true);
Avi Kivity4db35312007-11-21 15:28:32 +02002148 return sp;
Avi Kivitycea0f0e2007-01-05 16:36:43 -08002149}
2150
Avi Kivity2d111232008-12-25 14:39:47 +02002151static void shadow_walk_init(struct kvm_shadow_walk_iterator *iterator,
2152 struct kvm_vcpu *vcpu, u64 addr)
2153{
2154 iterator->addr = addr;
2155 iterator->shadow_addr = vcpu->arch.mmu.root_hpa;
2156 iterator->level = vcpu->arch.mmu.shadow_root_level;
Joerg Roedel81407ca2010-09-10 17:31:00 +02002157
2158 if (iterator->level == PT64_ROOT_LEVEL &&
2159 vcpu->arch.mmu.root_level < PT64_ROOT_LEVEL &&
2160 !vcpu->arch.mmu.direct_map)
2161 --iterator->level;
2162
Avi Kivity2d111232008-12-25 14:39:47 +02002163 if (iterator->level == PT32E_ROOT_LEVEL) {
2164 iterator->shadow_addr
2165 = vcpu->arch.mmu.pae_root[(addr >> 30) & 3];
2166 iterator->shadow_addr &= PT64_BASE_ADDR_MASK;
2167 --iterator->level;
2168 if (!iterator->shadow_addr)
2169 iterator->level = 0;
2170 }
2171}
2172
2173static bool shadow_walk_okay(struct kvm_shadow_walk_iterator *iterator)
2174{
2175 if (iterator->level < PT_PAGE_TABLE_LEVEL)
2176 return false;
Marcelo Tosatti4d889542009-06-11 12:07:41 -03002177
Avi Kivity2d111232008-12-25 14:39:47 +02002178 iterator->index = SHADOW_PT_INDEX(iterator->addr, iterator->level);
2179 iterator->sptep = ((u64 *)__va(iterator->shadow_addr)) + iterator->index;
2180 return true;
2181}
2182
Xiao Guangrongc2a2ac22011-07-12 03:32:13 +08002183static void __shadow_walk_next(struct kvm_shadow_walk_iterator *iterator,
2184 u64 spte)
Avi Kivity2d111232008-12-25 14:39:47 +02002185{
Xiao Guangrongc2a2ac22011-07-12 03:32:13 +08002186 if (is_last_spte(spte, iterator->level)) {
Xiao Guangrong052331b2011-07-12 03:21:17 +08002187 iterator->level = 0;
2188 return;
2189 }
2190
Xiao Guangrongc2a2ac22011-07-12 03:32:13 +08002191 iterator->shadow_addr = spte & PT64_BASE_ADDR_MASK;
Avi Kivity2d111232008-12-25 14:39:47 +02002192 --iterator->level;
2193}
2194
Xiao Guangrongc2a2ac22011-07-12 03:32:13 +08002195static void shadow_walk_next(struct kvm_shadow_walk_iterator *iterator)
2196{
2197 return __shadow_walk_next(iterator, *iterator->sptep);
2198}
2199
Yang Zhang7a1638c2013-08-05 11:07:13 +03002200static void link_shadow_page(u64 *sptep, struct kvm_mmu_page *sp, bool accessed)
Avi Kivity32ef26a2010-07-13 14:27:04 +03002201{
2202 u64 spte;
2203
Yang Zhang7a1638c2013-08-05 11:07:13 +03002204 BUILD_BUG_ON(VMX_EPT_READABLE_MASK != PT_PRESENT_MASK ||
2205 VMX_EPT_WRITABLE_MASK != PT_WRITABLE_MASK);
2206
Xiao Guangrong24db2732013-02-05 15:28:02 +08002207 spte = __pa(sp->spt) | PT_PRESENT_MASK | PT_WRITABLE_MASK |
Yang Zhang7a1638c2013-08-05 11:07:13 +03002208 shadow_user_mask | shadow_x_mask;
2209
2210 if (accessed)
2211 spte |= shadow_accessed_mask;
Xiao Guangrong24db2732013-02-05 15:28:02 +08002212
Xiao Guangrong1df9f2d2011-07-12 03:30:35 +08002213 mmu_spte_set(sptep, spte);
Avi Kivity32ef26a2010-07-13 14:27:04 +03002214}
2215
Avi Kivitya357bd22010-07-13 14:27:07 +03002216static void validate_direct_spte(struct kvm_vcpu *vcpu, u64 *sptep,
2217 unsigned direct_access)
2218{
2219 if (is_shadow_present_pte(*sptep) && !is_large_pte(*sptep)) {
2220 struct kvm_mmu_page *child;
2221
2222 /*
2223 * For the direct sp, if the guest pte's dirty bit
2224 * changed form clean to dirty, it will corrupt the
2225 * sp's access: allow writable in the read-only sp,
2226 * so we should update the spte at this point to get
2227 * a new sp with the correct access.
2228 */
2229 child = page_header(*sptep & PT64_BASE_ADDR_MASK);
2230 if (child->role.access == direct_access)
2231 return;
2232
Xiao Guangrongbcdd9a92011-05-15 23:28:29 +08002233 drop_parent_pte(child, sptep);
Avi Kivitya357bd22010-07-13 14:27:07 +03002234 kvm_flush_remote_tlbs(vcpu->kvm);
2235 }
2236}
2237
Xiao Guangrong505aef82011-09-22 16:56:06 +08002238static bool mmu_page_zap_pte(struct kvm *kvm, struct kvm_mmu_page *sp,
Xiao Guangrong38e3b2b2011-05-15 23:27:52 +08002239 u64 *spte)
2240{
2241 u64 pte;
2242 struct kvm_mmu_page *child;
2243
2244 pte = *spte;
2245 if (is_shadow_present_pte(pte)) {
Xiao Guangrong505aef82011-09-22 16:56:06 +08002246 if (is_last_spte(pte, sp->role.level)) {
Xiao Guangrongc3707952011-07-12 03:28:04 +08002247 drop_spte(kvm, spte);
Xiao Guangrong505aef82011-09-22 16:56:06 +08002248 if (is_large_pte(pte))
2249 --kvm->stat.lpages;
2250 } else {
Xiao Guangrong38e3b2b2011-05-15 23:27:52 +08002251 child = page_header(pte & PT64_BASE_ADDR_MASK);
Xiao Guangrongbcdd9a92011-05-15 23:28:29 +08002252 drop_parent_pte(child, spte);
Xiao Guangrong38e3b2b2011-05-15 23:27:52 +08002253 }
Xiao Guangrong505aef82011-09-22 16:56:06 +08002254 return true;
2255 }
2256
2257 if (is_mmio_spte(pte))
Xiao Guangrongce88dec2011-07-12 03:33:44 +08002258 mmu_spte_clear_no_track(spte);
Xiao Guangrongc3707952011-07-12 03:28:04 +08002259
Xiao Guangrong505aef82011-09-22 16:56:06 +08002260 return false;
Xiao Guangrong38e3b2b2011-05-15 23:27:52 +08002261}
2262
Avi Kivity90cb0522007-07-17 13:04:56 +03002263static void kvm_mmu_page_unlink_children(struct kvm *kvm,
Avi Kivity4db35312007-11-21 15:28:32 +02002264 struct kvm_mmu_page *sp)
Avi Kivitya4360362007-01-05 16:36:45 -08002265{
Avi Kivity697fe2e2007-01-05 16:36:46 -08002266 unsigned i;
Avi Kivity697fe2e2007-01-05 16:36:46 -08002267
Xiao Guangrong38e3b2b2011-05-15 23:27:52 +08002268 for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
2269 mmu_page_zap_pte(kvm, sp, sp->spt + i);
Avi Kivitya4360362007-01-05 16:36:45 -08002270}
2271
Avi Kivity4db35312007-11-21 15:28:32 +02002272static void kvm_mmu_put_page(struct kvm_mmu_page *sp, u64 *parent_pte)
Avi Kivitycea0f0e2007-01-05 16:36:43 -08002273{
Avi Kivity4db35312007-11-21 15:28:32 +02002274 mmu_page_remove_parent_pte(sp, parent_pte);
Avi Kivitya4360362007-01-05 16:36:45 -08002275}
2276
Avi Kivity31aa2b42008-07-11 17:59:46 +03002277static void kvm_mmu_unlink_parents(struct kvm *kvm, struct kvm_mmu_page *sp)
Avi Kivitya4360362007-01-05 16:36:45 -08002278{
Takuya Yoshikawa1e3f42f2012-03-21 23:50:34 +09002279 u64 *sptep;
2280 struct rmap_iterator iter;
Avi Kivitya4360362007-01-05 16:36:45 -08002281
Takuya Yoshikawa1e3f42f2012-03-21 23:50:34 +09002282 while ((sptep = rmap_get_first(sp->parent_ptes, &iter)))
2283 drop_parent_pte(sp, sptep);
Avi Kivity31aa2b42008-07-11 17:59:46 +03002284}
2285
Marcelo Tosatti60c8aec2008-12-01 22:32:02 -02002286static int mmu_zap_unsync_children(struct kvm *kvm,
Xiao Guangrong77758342010-06-04 21:53:54 +08002287 struct kvm_mmu_page *parent,
2288 struct list_head *invalid_list)
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03002289{
Marcelo Tosatti60c8aec2008-12-01 22:32:02 -02002290 int i, zapped = 0;
2291 struct mmu_page_path parents;
2292 struct kvm_mmu_pages pages;
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03002293
Marcelo Tosatti60c8aec2008-12-01 22:32:02 -02002294 if (parent->role.level == PT_PAGE_TABLE_LEVEL)
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03002295 return 0;
Marcelo Tosatti60c8aec2008-12-01 22:32:02 -02002296
2297 kvm_mmu_pages_init(parent, &parents, &pages);
2298 while (mmu_unsync_walk(parent, &pages)) {
2299 struct kvm_mmu_page *sp;
2300
2301 for_each_sp(pages, sp, parents, i) {
Xiao Guangrong77758342010-06-04 21:53:54 +08002302 kvm_mmu_prepare_zap_page(kvm, sp, invalid_list);
Marcelo Tosatti60c8aec2008-12-01 22:32:02 -02002303 mmu_pages_clear_parents(&parents);
Xiao Guangrong77662e02010-04-16 16:34:42 +08002304 zapped++;
Marcelo Tosatti60c8aec2008-12-01 22:32:02 -02002305 }
Marcelo Tosatti60c8aec2008-12-01 22:32:02 -02002306 kvm_mmu_pages_init(parent, &parents, &pages);
2307 }
2308
2309 return zapped;
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03002310}
2311
Xiao Guangrong77758342010-06-04 21:53:54 +08002312static int kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
2313 struct list_head *invalid_list)
Avi Kivity31aa2b42008-07-11 17:59:46 +03002314{
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03002315 int ret;
Avi Kivityf691fe12009-07-06 15:58:14 +03002316
Xiao Guangrong77758342010-06-04 21:53:54 +08002317 trace_kvm_mmu_prepare_zap_page(sp);
Avi Kivity31aa2b42008-07-11 17:59:46 +03002318 ++kvm->stat.mmu_shadow_zapped;
Xiao Guangrong77758342010-06-04 21:53:54 +08002319 ret = mmu_zap_unsync_children(kvm, sp, invalid_list);
Avi Kivity4db35312007-11-21 15:28:32 +02002320 kvm_mmu_page_unlink_children(kvm, sp);
Avi Kivity31aa2b42008-07-11 17:59:46 +03002321 kvm_mmu_unlink_parents(kvm, sp);
Xiao Guangrong5304b8d2013-05-31 08:36:22 +08002322
Avi Kivityf6e2c02b2009-01-11 13:02:10 +02002323 if (!sp->role.invalid && !sp->role.direct)
Paolo Bonzini3ed1a472015-05-19 16:29:22 +02002324 unaccount_shadowed(kvm, sp);
Xiao Guangrong5304b8d2013-05-31 08:36:22 +08002325
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03002326 if (sp->unsync)
2327 kvm_unlink_unsync_page(kvm, sp);
Avi Kivity4db35312007-11-21 15:28:32 +02002328 if (!sp->root_count) {
Gui Jianfeng54a4f022010-05-05 09:03:49 +08002329 /* Count self */
2330 ret++;
Xiao Guangrong77758342010-06-04 21:53:54 +08002331 list_move(&sp->link, invalid_list);
Xiao Guangrongaa6bd182011-07-12 03:26:40 +08002332 kvm_mod_used_mmu_pages(kvm, -1);
Marcelo Tosatti2e53d632008-02-20 14:47:24 -05002333 } else {
Avi Kivity5b5c6a52008-07-11 18:07:26 +03002334 list_move(&sp->link, &kvm->arch.active_mmu_pages);
Gleb Natapov05988d72013-05-31 08:36:30 +08002335
2336 /*
2337 * The obsolete pages can not be used on any vcpus.
2338 * See the comments in kvm_mmu_invalidate_zap_all_pages().
2339 */
2340 if (!sp->role.invalid && !is_obsolete_sp(kvm, sp))
2341 kvm_reload_remote_mmus(kvm);
Marcelo Tosatti2e53d632008-02-20 14:47:24 -05002342 }
Xiao Guangrong77758342010-06-04 21:53:54 +08002343
2344 sp->role.invalid = 1;
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03002345 return ret;
Avi Kivitya4360362007-01-05 16:36:45 -08002346}
2347
Xiao Guangrong77758342010-06-04 21:53:54 +08002348static void kvm_mmu_commit_zap_page(struct kvm *kvm,
2349 struct list_head *invalid_list)
2350{
Takuya Yoshikawa945315b2013-03-06 16:05:52 +09002351 struct kvm_mmu_page *sp, *nsp;
Xiao Guangrong77758342010-06-04 21:53:54 +08002352
2353 if (list_empty(invalid_list))
2354 return;
2355
Avi Kivityc1427862012-05-14 15:44:06 +03002356 /*
2357 * wmb: make sure everyone sees our modifications to the page tables
2358 * rmb: make sure we see changes to vcpu->mode
2359 */
2360 smp_mb();
2361
2362 /*
2363 * Wait for all vcpus to exit guest mode and/or lockless shadow
2364 * page table walks.
2365 */
Xiao Guangrong77758342010-06-04 21:53:54 +08002366 kvm_flush_remote_tlbs(kvm);
2367
Takuya Yoshikawa945315b2013-03-06 16:05:52 +09002368 list_for_each_entry_safe(sp, nsp, invalid_list, link) {
Xiao Guangrong77758342010-06-04 21:53:54 +08002369 WARN_ON(!sp->role.invalid || sp->root_count);
Xiao Guangrongaa6bd182011-07-12 03:26:40 +08002370 kvm_mmu_free_page(sp);
Takuya Yoshikawa945315b2013-03-06 16:05:52 +09002371 }
Xiao Guangrong77758342010-06-04 21:53:54 +08002372}
2373
Takuya Yoshikawa5da59602013-03-06 16:06:58 +09002374static bool prepare_zap_oldest_mmu_page(struct kvm *kvm,
2375 struct list_head *invalid_list)
2376{
2377 struct kvm_mmu_page *sp;
2378
2379 if (list_empty(&kvm->arch.active_mmu_pages))
2380 return false;
2381
2382 sp = list_entry(kvm->arch.active_mmu_pages.prev,
2383 struct kvm_mmu_page, link);
2384 kvm_mmu_prepare_zap_page(kvm, sp, invalid_list);
2385
2386 return true;
2387}
2388
Izik Eidus82ce2c92007-10-02 18:52:55 +02002389/*
2390 * Changing the number of mmu pages allocated to the vm
Dave Hansen49d5ca22010-08-19 18:11:28 -07002391 * Note: if goal_nr_mmu_pages is too small, you will get dead lock
Izik Eidus82ce2c92007-10-02 18:52:55 +02002392 */
Dave Hansen49d5ca22010-08-19 18:11:28 -07002393void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int goal_nr_mmu_pages)
Izik Eidus82ce2c92007-10-02 18:52:55 +02002394{
Xiao Guangrongd98ba052010-06-04 21:55:29 +08002395 LIST_HEAD(invalid_list);
Izik Eidus82ce2c92007-10-02 18:52:55 +02002396
Takuya Yoshikawab34cb592013-01-08 19:46:07 +09002397 spin_lock(&kvm->mmu_lock);
2398
Dave Hansen49d5ca22010-08-19 18:11:28 -07002399 if (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages) {
Takuya Yoshikawa5da59602013-03-06 16:06:58 +09002400 /* Need to free some mmu pages to achieve the goal. */
2401 while (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages)
2402 if (!prepare_zap_oldest_mmu_page(kvm, &invalid_list))
2403 break;
Izik Eidus82ce2c92007-10-02 18:52:55 +02002404
Xiao Guangrongaa6bd182011-07-12 03:26:40 +08002405 kvm_mmu_commit_zap_page(kvm, &invalid_list);
Dave Hansen49d5ca22010-08-19 18:11:28 -07002406 goal_nr_mmu_pages = kvm->arch.n_used_mmu_pages;
Izik Eidus82ce2c92007-10-02 18:52:55 +02002407 }
Izik Eidus82ce2c92007-10-02 18:52:55 +02002408
Dave Hansen49d5ca22010-08-19 18:11:28 -07002409 kvm->arch.n_max_mmu_pages = goal_nr_mmu_pages;
Takuya Yoshikawab34cb592013-01-08 19:46:07 +09002410
2411 spin_unlock(&kvm->mmu_lock);
Izik Eidus82ce2c92007-10-02 18:52:55 +02002412}
2413
Xiao Guangrong1cb3f3a2011-09-22 17:02:48 +08002414int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn)
Avi Kivitya4360362007-01-05 16:36:45 -08002415{
Avi Kivity4db35312007-11-21 15:28:32 +02002416 struct kvm_mmu_page *sp;
Xiao Guangrongd98ba052010-06-04 21:55:29 +08002417 LIST_HEAD(invalid_list);
Avi Kivitya4360362007-01-05 16:36:45 -08002418 int r;
2419
Xiao Guangrong9ad17b102010-08-28 19:19:42 +08002420 pgprintk("%s: looking for gfn %llx\n", __func__, gfn);
Avi Kivitya4360362007-01-05 16:36:45 -08002421 r = 0;
Xiao Guangrong1cb3f3a2011-09-22 17:02:48 +08002422 spin_lock(&kvm->mmu_lock);
Sasha Levinb67bfe02013-02-27 17:06:00 -08002423 for_each_gfn_indirect_valid_sp(kvm, sp, gfn) {
Xiao Guangrong9ad17b102010-08-28 19:19:42 +08002424 pgprintk("%s: gfn %llx role %x\n", __func__, gfn,
Xiao Guangrong7ae680e2010-06-04 21:53:07 +08002425 sp->role.word);
2426 r = 1;
Xiao Guangrongf41d3352010-06-04 21:56:11 +08002427 kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list);
Xiao Guangrong7ae680e2010-06-04 21:53:07 +08002428 }
Xiao Guangrongd98ba052010-06-04 21:55:29 +08002429 kvm_mmu_commit_zap_page(kvm, &invalid_list);
Xiao Guangrong1cb3f3a2011-09-22 17:02:48 +08002430 spin_unlock(&kvm->mmu_lock);
2431
Avi Kivitya4360362007-01-05 16:36:45 -08002432 return r;
Avi Kivitycea0f0e2007-01-05 16:36:43 -08002433}
Xiao Guangrong1cb3f3a2011-09-22 17:02:48 +08002434EXPORT_SYMBOL_GPL(kvm_mmu_unprotect_page);
Avi Kivitycea0f0e2007-01-05 16:36:43 -08002435
Xiao Guangrong9cf5cf52010-05-24 15:40:07 +08002436static void __kvm_unsync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03002437{
Xiao Guangrong5e1b3dd2010-04-28 11:55:06 +08002438 trace_kvm_mmu_unsync_page(sp);
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03002439 ++vcpu->kvm->stat.mmu_unsync;
2440 sp->unsync = 1;
Marcelo Tosatti6cffe8c2008-12-01 22:32:04 -02002441
Xiao Guangrong6b184932010-04-16 21:29:17 +08002442 kvm_mmu_mark_parents_unsync(sp);
Xiao Guangrong9cf5cf52010-05-24 15:40:07 +08002443}
2444
2445static void kvm_unsync_pages(struct kvm_vcpu *vcpu, gfn_t gfn)
2446{
Xiao Guangrong9cf5cf52010-05-24 15:40:07 +08002447 struct kvm_mmu_page *s;
Xiao Guangrong9cf5cf52010-05-24 15:40:07 +08002448
Sasha Levinb67bfe02013-02-27 17:06:00 -08002449 for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn) {
Xiao Guangrong7ae680e2010-06-04 21:53:07 +08002450 if (s->unsync)
Xiao Guangrong9cf5cf52010-05-24 15:40:07 +08002451 continue;
2452 WARN_ON(s->role.level != PT_PAGE_TABLE_LEVEL);
2453 __kvm_unsync_page(vcpu, s);
2454 }
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03002455}
2456
2457static int mmu_need_write_protect(struct kvm_vcpu *vcpu, gfn_t gfn,
2458 bool can_unsync)
2459{
Xiao Guangrong9cf5cf52010-05-24 15:40:07 +08002460 struct kvm_mmu_page *s;
Xiao Guangrong9cf5cf52010-05-24 15:40:07 +08002461 bool need_unsync = false;
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03002462
Sasha Levinb67bfe02013-02-27 17:06:00 -08002463 for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn) {
Xiao Guangrong36a2e672010-06-30 16:02:02 +08002464 if (!can_unsync)
2465 return 1;
2466
Xiao Guangrong9cf5cf52010-05-24 15:40:07 +08002467 if (s->role.level != PT_PAGE_TABLE_LEVEL)
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03002468 return 1;
Xiao Guangrong9cf5cf52010-05-24 15:40:07 +08002469
Gleb Natapov9bb4f6b2013-01-30 16:45:01 +02002470 if (!s->unsync)
Xiao Guangrong9cf5cf52010-05-24 15:40:07 +08002471 need_unsync = true;
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03002472 }
Xiao Guangrong9cf5cf52010-05-24 15:40:07 +08002473 if (need_unsync)
2474 kvm_unsync_pages(vcpu, gfn);
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03002475 return 0;
2476}
2477
Paolo Bonzinid1fe9212015-07-07 15:03:18 +02002478static bool kvm_is_mmio_pfn(pfn_t pfn)
2479{
2480 if (pfn_valid(pfn))
2481 return !is_zero_pfn(pfn) && PageReserved(pfn_to_page(pfn));
2482
2483 return true;
2484}
2485
Avi Kivityd555c332009-06-10 14:24:23 +03002486static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
Xiao Guangrongc22885052013-01-08 14:36:04 +08002487 unsigned pte_access, int level,
Marcelo Tosattic2d0ee42009-04-05 14:54:47 -03002488 gfn_t gfn, pfn_t pfn, bool speculative,
Lai Jiangshan9bdbba12010-11-19 17:03:22 +08002489 bool can_unsync, bool host_writable)
Avi Kivity1c4f1fd2007-12-09 17:40:31 +02002490{
Xiao Guangrong6e7d0352012-06-20 15:58:33 +08002491 u64 spte;
Marcelo Tosatti1e73f9d2008-09-23 13:18:30 -03002492 int ret = 0;
Sheng Yang64d4d522008-10-09 16:01:57 +08002493
Paolo Bonzini54bf36a2015-04-08 15:39:23 +02002494 if (set_mmio_spte(vcpu, sptep, gfn, pfn, pte_access))
Xiao Guangrongce88dec2011-07-12 03:33:44 +08002495 return 0;
2496
Marcelo Tosatti982c2562010-10-22 14:18:16 -02002497 spte = PT_PRESENT_MASK;
Avi Kivity947da532008-03-18 11:05:52 +02002498 if (!speculative)
Avi Kivity3201b5d2008-08-27 20:01:04 +03002499 spte |= shadow_accessed_mask;
Xiao Guangrong640d9b02011-07-12 03:24:39 +08002500
Sheng Yang7b523452008-04-25 21:13:50 +08002501 if (pte_access & ACC_EXEC_MASK)
2502 spte |= shadow_x_mask;
2503 else
2504 spte |= shadow_nx_mask;
Xiao Guangrong49fde342012-06-20 15:58:58 +08002505
Avi Kivity1c4f1fd2007-12-09 17:40:31 +02002506 if (pte_access & ACC_USER_MASK)
Sheng Yang7b523452008-04-25 21:13:50 +08002507 spte |= shadow_user_mask;
Xiao Guangrong49fde342012-06-20 15:58:58 +08002508
Joerg Roedel852e3c12009-07-27 16:30:44 +02002509 if (level > PT_PAGE_TABLE_LEVEL)
Marcelo Tosatti05da4552008-02-23 11:44:30 -03002510 spte |= PT_PAGE_SIZE_MASK;
Avi Kivityb0bc3ee2010-09-13 16:45:28 +02002511 if (tdp_enabled)
Sheng Yang4b12f0d2009-04-27 20:35:42 +08002512 spte |= kvm_x86_ops->get_mt_mask(vcpu, gfn,
Paolo Bonzinid1fe9212015-07-07 15:03:18 +02002513 kvm_is_mmio_pfn(pfn));
Avi Kivity1c4f1fd2007-12-09 17:40:31 +02002514
Lai Jiangshan9bdbba12010-11-19 17:03:22 +08002515 if (host_writable)
Izik Eidus14032832009-09-23 21:47:17 +03002516 spte |= SPTE_HOST_WRITEABLE;
Xiao Guangrongf8e453b2010-12-23 16:09:29 +08002517 else
2518 pte_access &= ~ACC_WRITE_MASK;
Izik Eidus14032832009-09-23 21:47:17 +03002519
Anthony Liguori35149e22008-04-02 14:46:56 -05002520 spte |= (u64)pfn << PAGE_SHIFT;
Avi Kivity1c4f1fd2007-12-09 17:40:31 +02002521
Xiao Guangrongc22885052013-01-08 14:36:04 +08002522 if (pte_access & ACC_WRITE_MASK) {
Avi Kivity1c4f1fd2007-12-09 17:40:31 +02002523
Xiao Guangrongc2193462012-12-04 07:17:11 +08002524 /*
Xiao Guangrong7751bab2013-01-08 14:36:51 +08002525 * Other vcpu creates new sp in the window between
2526 * mapping_level() and acquiring mmu-lock. We can
2527 * allow guest to retry the access, the mapping can
2528 * be fixed if guest refault.
Xiao Guangrongc2193462012-12-04 07:17:11 +08002529 */
Joerg Roedel852e3c12009-07-27 16:30:44 +02002530 if (level > PT_PAGE_TABLE_LEVEL &&
Paolo Bonzini54bf36a2015-04-08 15:39:23 +02002531 has_wrprotected_page(vcpu, gfn, level))
Avi Kivitybe38d272010-06-06 14:31:27 +03002532 goto done;
Marcelo Tosatti38187c82008-09-23 13:18:32 -03002533
Xiao Guangrong49fde342012-06-20 15:58:58 +08002534 spte |= PT_WRITABLE_MASK | SPTE_MMU_WRITEABLE;
Avi Kivity1c4f1fd2007-12-09 17:40:31 +02002535
Marcelo Tosattiecc55892008-11-25 15:58:07 +01002536 /*
2537 * Optimization: for pte sync, if spte was writable the hash
2538 * lookup is unnecessary (and expensive). Write protection
2539 * is responsibility of mmu_get_page / kvm_sync_page.
2540 * Same reasoning can be applied to dirty page accounting.
2541 */
Takuya Yoshikawa8dae4442010-01-18 18:45:10 +09002542 if (!can_unsync && is_writable_pte(*sptep))
Marcelo Tosattiecc55892008-11-25 15:58:07 +01002543 goto set_pte;
2544
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03002545 if (mmu_need_write_protect(vcpu, gfn, can_unsync)) {
Xiao Guangrong9ad17b102010-08-28 19:19:42 +08002546 pgprintk("%s: found shadow page for %llx, marking ro\n",
Harvey Harrisonb8688d52008-03-03 12:59:56 -08002547 __func__, gfn);
Marcelo Tosatti1e73f9d2008-09-23 13:18:30 -03002548 ret = 1;
Avi Kivity1c4f1fd2007-12-09 17:40:31 +02002549 pte_access &= ~ACC_WRITE_MASK;
Xiao Guangrong49fde342012-06-20 15:58:58 +08002550 spte &= ~(PT_WRITABLE_MASK | SPTE_MMU_WRITEABLE);
Avi Kivity1c4f1fd2007-12-09 17:40:31 +02002551 }
2552 }
2553
Kai Huang9b51a632015-01-28 10:54:25 +08002554 if (pte_access & ACC_WRITE_MASK) {
Paolo Bonzini54bf36a2015-04-08 15:39:23 +02002555 kvm_vcpu_mark_page_dirty(vcpu, gfn);
Kai Huang9b51a632015-01-28 10:54:25 +08002556 spte |= shadow_dirty_mask;
2557 }
Avi Kivity1c4f1fd2007-12-09 17:40:31 +02002558
Marcelo Tosatti38187c82008-09-23 13:18:32 -03002559set_pte:
Xiao Guangrong6e7d0352012-06-20 15:58:33 +08002560 if (mmu_spte_update(sptep, spte))
Xiao Guangrongb330aa02010-11-19 17:02:35 +08002561 kvm_flush_remote_tlbs(vcpu->kvm);
Avi Kivitybe38d272010-06-06 14:31:27 +03002562done:
Marcelo Tosatti1e73f9d2008-09-23 13:18:30 -03002563 return ret;
2564}
2565
Avi Kivityd555c332009-06-10 14:24:23 +03002566static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
Xiao Guangrongf7616202013-02-05 15:27:27 +08002567 unsigned pte_access, int write_fault, int *emulate,
2568 int level, gfn_t gfn, pfn_t pfn, bool speculative,
2569 bool host_writable)
Marcelo Tosatti1e73f9d2008-09-23 13:18:30 -03002570{
2571 int was_rmapped = 0;
Marcelo Tosatti53a27b32009-08-05 15:43:58 -03002572 int rmap_count;
Marcelo Tosatti1e73f9d2008-09-23 13:18:30 -03002573
Xiao Guangrongf7616202013-02-05 15:27:27 +08002574 pgprintk("%s: spte %llx write_fault %d gfn %llx\n", __func__,
2575 *sptep, write_fault, gfn);
Marcelo Tosatti1e73f9d2008-09-23 13:18:30 -03002576
Avi Kivityd555c332009-06-10 14:24:23 +03002577 if (is_rmap_spte(*sptep)) {
Marcelo Tosatti1e73f9d2008-09-23 13:18:30 -03002578 /*
2579 * If we overwrite a PTE page pointer with a 2MB PMD, unlink
2580 * the parent of the now unreachable PTE.
2581 */
Joerg Roedel852e3c12009-07-27 16:30:44 +02002582 if (level > PT_PAGE_TABLE_LEVEL &&
2583 !is_large_pte(*sptep)) {
Marcelo Tosatti1e73f9d2008-09-23 13:18:30 -03002584 struct kvm_mmu_page *child;
Avi Kivityd555c332009-06-10 14:24:23 +03002585 u64 pte = *sptep;
Marcelo Tosatti1e73f9d2008-09-23 13:18:30 -03002586
2587 child = page_header(pte & PT64_BASE_ADDR_MASK);
Xiao Guangrongbcdd9a92011-05-15 23:28:29 +08002588 drop_parent_pte(child, sptep);
Marcelo Tosatti3be22642010-05-28 09:44:59 -03002589 kvm_flush_remote_tlbs(vcpu->kvm);
Avi Kivityd555c332009-06-10 14:24:23 +03002590 } else if (pfn != spte_to_pfn(*sptep)) {
Xiao Guangrong9ad17b102010-08-28 19:19:42 +08002591 pgprintk("hfn old %llx new %llx\n",
Avi Kivityd555c332009-06-10 14:24:23 +03002592 spte_to_pfn(*sptep), pfn);
Xiao Guangrongc3707952011-07-12 03:28:04 +08002593 drop_spte(vcpu->kvm, sptep);
Xiao Guangrong91546352010-06-30 16:04:06 +08002594 kvm_flush_remote_tlbs(vcpu->kvm);
Joerg Roedel6bed6b92009-02-18 14:08:59 +01002595 } else
2596 was_rmapped = 1;
Marcelo Tosatti1e73f9d2008-09-23 13:18:30 -03002597 }
Joerg Roedel852e3c12009-07-27 16:30:44 +02002598
Xiao Guangrongc22885052013-01-08 14:36:04 +08002599 if (set_spte(vcpu, sptep, pte_access, level, gfn, pfn, speculative,
2600 true, host_writable)) {
Marcelo Tosatti1e73f9d2008-09-23 13:18:30 -03002601 if (write_fault)
Xiao Guangrongb90a0e62011-07-12 03:25:56 +08002602 *emulate = 1;
Liang Chen77c39132014-09-18 12:38:37 -04002603 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
Marcelo Tosattia378b4e2008-09-23 13:18:31 -03002604 }
Marcelo Tosatti1e73f9d2008-09-23 13:18:30 -03002605
Xiao Guangrongce88dec2011-07-12 03:33:44 +08002606 if (unlikely(is_mmio_spte(*sptep) && emulate))
2607 *emulate = 1;
2608
Avi Kivityd555c332009-06-10 14:24:23 +03002609 pgprintk("%s: setting spte %llx\n", __func__, *sptep);
Xiao Guangrong9ad17b102010-08-28 19:19:42 +08002610 pgprintk("instantiating %s PTE (%s) at %llx (%llx) addr %p\n",
Avi Kivityd555c332009-06-10 14:24:23 +03002611 is_large_pte(*sptep)? "2MB" : "4kB",
Joerg Roedela205bc12009-07-09 16:36:01 +02002612 *sptep & PT_PRESENT_MASK ?"RW":"R", gfn,
2613 *sptep, sptep);
Avi Kivityd555c332009-06-10 14:24:23 +03002614 if (!was_rmapped && is_large_pte(*sptep))
Marcelo Tosatti05da4552008-02-23 11:44:30 -03002615 ++vcpu->kvm->stat.lpages;
2616
Xiao Guangrongffb61bb2011-07-12 03:22:01 +08002617 if (is_shadow_present_pte(*sptep)) {
Xiao Guangrongffb61bb2011-07-12 03:22:01 +08002618 if (!was_rmapped) {
2619 rmap_count = rmap_add(vcpu, sptep, gfn);
2620 if (rmap_count > RMAP_RECYCLE_THRESHOLD)
2621 rmap_recycle(vcpu, sptep, gfn);
2622 }
Avi Kivity1c4f1fd2007-12-09 17:40:31 +02002623 }
Xiao Guangrongcb9aaa32012-08-03 15:42:10 +08002624
Xiao Guangrongf3ac1a42012-10-16 20:07:03 +08002625 kvm_release_pfn_clean(pfn);
Avi Kivity1c4f1fd2007-12-09 17:40:31 +02002626}
2627
Xiao Guangrong957ed9e2010-08-22 19:12:48 +08002628static pfn_t pte_prefetch_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn,
2629 bool no_dirty_log)
2630{
2631 struct kvm_memory_slot *slot;
Xiao Guangrong957ed9e2010-08-22 19:12:48 +08002632
Xiao Guangrong5d163b12011-03-09 15:43:00 +08002633 slot = gfn_to_memslot_dirty_bitmap(vcpu, gfn, no_dirty_log);
Xiao Guangrong903816f2012-07-17 21:54:11 +08002634 if (!slot)
Xiao Guangrong6c8ee572012-08-03 15:37:54 +08002635 return KVM_PFN_ERR_FAULT;
Xiao Guangrong957ed9e2010-08-22 19:12:48 +08002636
Xiao Guangrong037d92d2012-08-21 10:59:12 +08002637 return gfn_to_pfn_memslot_atomic(slot, gfn);
Xiao Guangrong957ed9e2010-08-22 19:12:48 +08002638}
2639
2640static int direct_pte_prefetch_many(struct kvm_vcpu *vcpu,
2641 struct kvm_mmu_page *sp,
2642 u64 *start, u64 *end)
2643{
2644 struct page *pages[PTE_PREFETCH_NUM];
Paolo Bonzinid9ef13c2015-05-19 16:01:50 +02002645 struct kvm_memory_slot *slot;
Xiao Guangrong957ed9e2010-08-22 19:12:48 +08002646 unsigned access = sp->role.access;
2647 int i, ret;
2648 gfn_t gfn;
2649
2650 gfn = kvm_mmu_page_get_gfn(sp, start - sp->spt);
Paolo Bonzinid9ef13c2015-05-19 16:01:50 +02002651 slot = gfn_to_memslot_dirty_bitmap(vcpu, gfn, access & ACC_WRITE_MASK);
2652 if (!slot)
Xiao Guangrong957ed9e2010-08-22 19:12:48 +08002653 return -1;
2654
Paolo Bonzinid9ef13c2015-05-19 16:01:50 +02002655 ret = gfn_to_page_many_atomic(slot, gfn, pages, end - start);
Xiao Guangrong957ed9e2010-08-22 19:12:48 +08002656 if (ret <= 0)
2657 return -1;
2658
2659 for (i = 0; i < ret; i++, gfn++, start++)
Xiao Guangrongf7616202013-02-05 15:27:27 +08002660 mmu_set_spte(vcpu, start, access, 0, NULL,
Xiao Guangrongc22885052013-01-08 14:36:04 +08002661 sp->role.level, gfn, page_to_pfn(pages[i]),
2662 true, true);
Xiao Guangrong957ed9e2010-08-22 19:12:48 +08002663
2664 return 0;
2665}
2666
2667static void __direct_pte_prefetch(struct kvm_vcpu *vcpu,
2668 struct kvm_mmu_page *sp, u64 *sptep)
2669{
2670 u64 *spte, *start = NULL;
2671 int i;
2672
2673 WARN_ON(!sp->role.direct);
2674
2675 i = (sptep - sp->spt) & ~(PTE_PREFETCH_NUM - 1);
2676 spte = sp->spt + i;
2677
2678 for (i = 0; i < PTE_PREFETCH_NUM; i++, spte++) {
Xiao Guangrongc3707952011-07-12 03:28:04 +08002679 if (is_shadow_present_pte(*spte) || spte == sptep) {
Xiao Guangrong957ed9e2010-08-22 19:12:48 +08002680 if (!start)
2681 continue;
2682 if (direct_pte_prefetch_many(vcpu, sp, start, spte) < 0)
2683 break;
2684 start = NULL;
2685 } else if (!start)
2686 start = spte;
2687 }
2688}
2689
2690static void direct_pte_prefetch(struct kvm_vcpu *vcpu, u64 *sptep)
2691{
2692 struct kvm_mmu_page *sp;
2693
2694 /*
2695 * Since it's no accessed bit on EPT, it's no way to
2696 * distinguish between actually accessed translations
2697 * and prefetched, so disable pte prefetch if EPT is
2698 * enabled.
2699 */
2700 if (!shadow_accessed_mask)
2701 return;
2702
2703 sp = page_header(__pa(sptep));
2704 if (sp->role.level > PT_PAGE_TABLE_LEVEL)
2705 return;
2706
2707 __direct_pte_prefetch(vcpu, sp, sptep);
2708}
2709
Joerg Roedel4d9976b2008-02-07 13:47:42 +01002710static int __direct_map(struct kvm_vcpu *vcpu, gpa_t v, int write,
Xiao Guangrong2ec47392010-12-07 10:34:42 +08002711 int map_writable, int level, gfn_t gfn, pfn_t pfn,
2712 bool prefault)
Avi Kivity6aa8b732006-12-10 02:21:36 -08002713{
Avi Kivity9f652d212008-12-25 14:54:25 +02002714 struct kvm_shadow_walk_iterator iterator;
2715 struct kvm_mmu_page *sp;
Xiao Guangrongb90a0e62011-07-12 03:25:56 +08002716 int emulate = 0;
Avi Kivity9f652d212008-12-25 14:54:25 +02002717 gfn_t pseudo_gfn;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002718
Marcelo Tosatti989c6b32013-12-19 15:28:51 -02002719 if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
2720 return 0;
2721
Avi Kivity9f652d212008-12-25 14:54:25 +02002722 for_each_shadow_entry(vcpu, (u64)gfn << PAGE_SHIFT, iterator) {
Joerg Roedel852e3c12009-07-27 16:30:44 +02002723 if (iterator.level == level) {
Xiao Guangrongf7616202013-02-05 15:27:27 +08002724 mmu_set_spte(vcpu, iterator.sptep, ACC_ALL,
Xiao Guangrongc22885052013-01-08 14:36:04 +08002725 write, &emulate, level, gfn, pfn,
2726 prefault, map_writable);
Xiao Guangrong957ed9e2010-08-22 19:12:48 +08002727 direct_pte_prefetch(vcpu, iterator.sptep);
Avi Kivity9f652d212008-12-25 14:54:25 +02002728 ++vcpu->stat.pf_fixed;
2729 break;
2730 }
2731
Marcelo Tosatti404381c2014-02-24 13:59:32 -03002732 drop_large_spte(vcpu, iterator.sptep);
Xiao Guangrongc3707952011-07-12 03:28:04 +08002733 if (!is_shadow_present_pte(*iterator.sptep)) {
Lai Jiangshanc9fa0b32010-05-26 16:48:25 +08002734 u64 base_addr = iterator.addr;
2735
2736 base_addr &= PT64_LVL_ADDR_MASK(iterator.level);
2737 pseudo_gfn = base_addr >> PAGE_SHIFT;
Avi Kivity9f652d212008-12-25 14:54:25 +02002738 sp = kvm_mmu_get_page(vcpu, pseudo_gfn, iterator.addr,
2739 iterator.level - 1,
2740 1, ACC_ALL, iterator.sptep);
Avi Kivity9f652d212008-12-25 14:54:25 +02002741
Yang Zhang7a1638c2013-08-05 11:07:13 +03002742 link_shadow_page(iterator.sptep, sp, true);
Avi Kivity9f652d212008-12-25 14:54:25 +02002743 }
2744 }
Xiao Guangrongb90a0e62011-07-12 03:25:56 +08002745 return emulate;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002746}
2747
Huang Ying77db5cb2010-10-08 16:24:15 +08002748static void kvm_send_hwpoison_signal(unsigned long address, struct task_struct *tsk)
Huang Yingbf998152010-05-31 14:28:19 +08002749{
Huang Ying77db5cb2010-10-08 16:24:15 +08002750 siginfo_t info;
Huang Yingbf998152010-05-31 14:28:19 +08002751
Huang Ying77db5cb2010-10-08 16:24:15 +08002752 info.si_signo = SIGBUS;
2753 info.si_errno = 0;
2754 info.si_code = BUS_MCEERR_AR;
2755 info.si_addr = (void __user *)address;
2756 info.si_addr_lsb = PAGE_SHIFT;
2757
2758 send_sig_info(SIGBUS, &info, tsk);
Huang Yingbf998152010-05-31 14:28:19 +08002759}
2760
Xiao Guangrongd7c55202011-07-12 03:29:38 +08002761static int kvm_handle_bad_page(struct kvm_vcpu *vcpu, gfn_t gfn, pfn_t pfn)
Huang Yingbf998152010-05-31 14:28:19 +08002762{
Xiao Guangrong4d8b81a2012-08-21 11:02:51 +08002763 /*
2764 * Do not cache the mmio info caused by writing the readonly gfn
2765 * into the spte otherwise read access on readonly gfn also can
2766 * caused mmio page fault and treat it as mmio access.
2767 * Return 1 to tell kvm to emulate it.
2768 */
2769 if (pfn == KVM_PFN_ERR_RO_FAULT)
2770 return 1;
2771
Xiao Guangronge6c15022012-08-03 15:38:36 +08002772 if (pfn == KVM_PFN_ERR_HWPOISON) {
Paolo Bonzini54bf36a2015-04-08 15:39:23 +02002773 kvm_send_hwpoison_signal(kvm_vcpu_gfn_to_hva(vcpu, gfn), current);
Huang Yingbf998152010-05-31 14:28:19 +08002774 return 0;
Xiao Guangrongd7c55202011-07-12 03:29:38 +08002775 }
Gleb Natapovedba23e2010-07-07 20:16:45 +03002776
Xiao Guangrongd7c55202011-07-12 03:29:38 +08002777 return -EFAULT;
Huang Yingbf998152010-05-31 14:28:19 +08002778}
2779
Andrea Arcangeli936a5fe2011-01-13 15:46:48 -08002780static void transparent_hugepage_adjust(struct kvm_vcpu *vcpu,
2781 gfn_t *gfnp, pfn_t *pfnp, int *levelp)
2782{
2783 pfn_t pfn = *pfnp;
2784 gfn_t gfn = *gfnp;
2785 int level = *levelp;
2786
2787 /*
2788 * Check if it's a transparent hugepage. If this would be an
2789 * hugetlbfs page, level wouldn't be set to
2790 * PT_PAGE_TABLE_LEVEL and there would be no adjustment done
2791 * here.
2792 */
Ard Biesheuvelbf4bea82014-11-10 08:33:56 +00002793 if (!is_error_noslot_pfn(pfn) && !kvm_is_reserved_pfn(pfn) &&
Andrea Arcangeli936a5fe2011-01-13 15:46:48 -08002794 level == PT_PAGE_TABLE_LEVEL &&
2795 PageTransCompound(pfn_to_page(pfn)) &&
Paolo Bonzini54bf36a2015-04-08 15:39:23 +02002796 !has_wrprotected_page(vcpu, gfn, PT_DIRECTORY_LEVEL)) {
Andrea Arcangeli936a5fe2011-01-13 15:46:48 -08002797 unsigned long mask;
2798 /*
2799 * mmu_notifier_retry was successful and we hold the
2800 * mmu_lock here, so the pmd can't become splitting
2801 * from under us, and in turn
2802 * __split_huge_page_refcount() can't run from under
2803 * us and we can safely transfer the refcount from
2804 * PG_tail to PG_head as we switch the pfn to tail to
2805 * head.
2806 */
2807 *levelp = level = PT_DIRECTORY_LEVEL;
2808 mask = KVM_PAGES_PER_HPAGE(level) - 1;
2809 VM_BUG_ON((gfn & mask) != (pfn & mask));
2810 if (pfn & mask) {
2811 gfn &= ~mask;
2812 *gfnp = gfn;
2813 kvm_release_pfn_clean(pfn);
2814 pfn &= ~mask;
Xiao Guangrongc3586662012-05-28 14:10:43 +08002815 kvm_get_pfn(pfn);
Andrea Arcangeli936a5fe2011-01-13 15:46:48 -08002816 *pfnp = pfn;
2817 }
2818 }
2819}
2820
Xiao Guangrongd7c55202011-07-12 03:29:38 +08002821static bool handle_abnormal_pfn(struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn,
2822 pfn_t pfn, unsigned access, int *ret_val)
2823{
2824 bool ret = true;
2825
2826 /* The pfn is invalid, report the error! */
Xiao Guangrong81c52c52012-10-16 20:10:59 +08002827 if (unlikely(is_error_pfn(pfn))) {
Xiao Guangrongd7c55202011-07-12 03:29:38 +08002828 *ret_val = kvm_handle_bad_page(vcpu, gfn, pfn);
2829 goto exit;
2830 }
2831
Xiao Guangrongce88dec2011-07-12 03:33:44 +08002832 if (unlikely(is_noslot_pfn(pfn)))
Xiao Guangrongd7c55202011-07-12 03:29:38 +08002833 vcpu_cache_mmio_info(vcpu, gva, gfn, access);
Xiao Guangrongd7c55202011-07-12 03:29:38 +08002834
2835 ret = false;
2836exit:
2837 return ret;
2838}
2839
Xiao Guangronge5552fd2013-07-30 21:01:59 +08002840static bool page_fault_can_be_fast(u32 error_code)
Xiao Guangrongc7ba5b42012-06-20 15:59:18 +08002841{
2842 /*
Xiao Guangrong1c118b82013-07-18 12:52:37 +08002843 * Do not fix the mmio spte with invalid generation number which
2844 * need to be updated by slow page fault path.
2845 */
2846 if (unlikely(error_code & PFERR_RSVD_MASK))
2847 return false;
2848
2849 /*
Xiao Guangrongc7ba5b42012-06-20 15:59:18 +08002850 * #PF can be fast only if the shadow page table is present and it
2851 * is caused by write-protect, that means we just need change the
2852 * W bit of the spte which can be done out of mmu-lock.
2853 */
2854 if (!(error_code & PFERR_PRESENT_MASK) ||
2855 !(error_code & PFERR_WRITE_MASK))
2856 return false;
2857
2858 return true;
2859}
2860
2861static bool
Xiao Guangrong92a476c2014-04-17 17:06:13 +08002862fast_pf_fix_direct_spte(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
2863 u64 *sptep, u64 spte)
Xiao Guangrongc7ba5b42012-06-20 15:59:18 +08002864{
Xiao Guangrongc7ba5b42012-06-20 15:59:18 +08002865 gfn_t gfn;
2866
2867 WARN_ON(!sp->role.direct);
2868
2869 /*
2870 * The gfn of direct spte is stable since it is calculated
2871 * by sp->gfn.
2872 */
2873 gfn = kvm_mmu_page_get_gfn(sp, sptep - sp->spt);
2874
Kai Huang9b51a632015-01-28 10:54:25 +08002875 /*
2876 * Theoretically we could also set dirty bit (and flush TLB) here in
2877 * order to eliminate unnecessary PML logging. See comments in
2878 * set_spte. But fast_page_fault is very unlikely to happen with PML
2879 * enabled, so we do not do this. This might result in the same GPA
2880 * to be logged in PML buffer again when the write really happens, and
2881 * eventually to be called by mark_page_dirty twice. But it's also no
2882 * harm. This also avoids the TLB flush needed after setting dirty bit
2883 * so non-PML cases won't be impacted.
2884 *
2885 * Compare with set_spte where instead shadow_dirty_mask is set.
2886 */
Xiao Guangrongc7ba5b42012-06-20 15:59:18 +08002887 if (cmpxchg64(sptep, spte, spte | PT_WRITABLE_MASK) == spte)
Paolo Bonzini54bf36a2015-04-08 15:39:23 +02002888 kvm_vcpu_mark_page_dirty(vcpu, gfn);
Xiao Guangrongc7ba5b42012-06-20 15:59:18 +08002889
2890 return true;
2891}
2892
2893/*
2894 * Return value:
2895 * - true: let the vcpu to access on the same address again.
2896 * - false: let the real page fault path to fix it.
2897 */
2898static bool fast_page_fault(struct kvm_vcpu *vcpu, gva_t gva, int level,
2899 u32 error_code)
2900{
2901 struct kvm_shadow_walk_iterator iterator;
Xiao Guangrong92a476c2014-04-17 17:06:13 +08002902 struct kvm_mmu_page *sp;
Xiao Guangrongc7ba5b42012-06-20 15:59:18 +08002903 bool ret = false;
2904 u64 spte = 0ull;
2905
Marcelo Tosatti37f6a4e2014-01-03 17:09:32 -02002906 if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
2907 return false;
2908
Xiao Guangronge5552fd2013-07-30 21:01:59 +08002909 if (!page_fault_can_be_fast(error_code))
Xiao Guangrongc7ba5b42012-06-20 15:59:18 +08002910 return false;
2911
2912 walk_shadow_page_lockless_begin(vcpu);
2913 for_each_shadow_entry_lockless(vcpu, gva, iterator, spte)
2914 if (!is_shadow_present_pte(spte) || iterator.level < level)
2915 break;
2916
2917 /*
2918 * If the mapping has been changed, let the vcpu fault on the
2919 * same address again.
2920 */
2921 if (!is_rmap_spte(spte)) {
2922 ret = true;
2923 goto exit;
2924 }
2925
Xiao Guangrong92a476c2014-04-17 17:06:13 +08002926 sp = page_header(__pa(iterator.sptep));
2927 if (!is_last_spte(spte, sp->role.level))
Xiao Guangrongc7ba5b42012-06-20 15:59:18 +08002928 goto exit;
2929
2930 /*
2931 * Check if it is a spurious fault caused by TLB lazily flushed.
2932 *
2933 * Need not check the access of upper level table entries since
2934 * they are always ACC_ALL.
2935 */
2936 if (is_writable_pte(spte)) {
2937 ret = true;
2938 goto exit;
2939 }
2940
2941 /*
2942 * Currently, to simplify the code, only the spte write-protected
2943 * by dirty-log can be fast fixed.
2944 */
2945 if (!spte_is_locklessly_modifiable(spte))
2946 goto exit;
2947
2948 /*
Xiao Guangrongc126d942014-04-17 17:06:14 +08002949 * Do not fix write-permission on the large spte since we only dirty
2950 * the first page into the dirty-bitmap in fast_pf_fix_direct_spte()
2951 * that means other pages are missed if its slot is dirty-logged.
2952 *
2953 * Instead, we let the slow page fault path create a normal spte to
2954 * fix the access.
2955 *
2956 * See the comments in kvm_arch_commit_memory_region().
2957 */
2958 if (sp->role.level > PT_PAGE_TABLE_LEVEL)
2959 goto exit;
2960
2961 /*
Xiao Guangrongc7ba5b42012-06-20 15:59:18 +08002962 * Currently, fast page fault only works for direct mapping since
2963 * the gfn is not stable for indirect shadow page.
2964 * See Documentation/virtual/kvm/locking.txt to get more detail.
2965 */
Xiao Guangrong92a476c2014-04-17 17:06:13 +08002966 ret = fast_pf_fix_direct_spte(vcpu, sp, iterator.sptep, spte);
Xiao Guangrongc7ba5b42012-06-20 15:59:18 +08002967exit:
Xiao Guangronga72faf22012-06-20 15:59:41 +08002968 trace_fast_page_fault(vcpu, gva, error_code, iterator.sptep,
2969 spte, ret);
Xiao Guangrongc7ba5b42012-06-20 15:59:18 +08002970 walk_shadow_page_lockless_end(vcpu);
2971
2972 return ret;
2973}
2974
Xiao Guangrong78b2c542010-12-07 10:48:06 +08002975static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn,
Xiao Guangrong060c2ab2010-11-12 14:49:11 +08002976 gva_t gva, pfn_t *pfn, bool write, bool *writable);
Takuya Yoshikawa450e0b42013-03-29 14:05:26 +09002977static void make_mmu_pages_available(struct kvm_vcpu *vcpu);
Xiao Guangrong060c2ab2010-11-12 14:49:11 +08002978
Xiao Guangrongc7ba5b42012-06-20 15:59:18 +08002979static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, u32 error_code,
2980 gfn_t gfn, bool prefault)
Marcelo Tosatti10589a42007-12-20 19:18:22 -05002981{
2982 int r;
Joerg Roedel852e3c12009-07-27 16:30:44 +02002983 int level;
Takuya Yoshikawafd136902015-10-16 17:06:02 +09002984 bool force_pt_level = false;
Anthony Liguori35149e22008-04-02 14:46:56 -05002985 pfn_t pfn;
Andrea Arcangelie930bff2008-07-25 16:24:52 +02002986 unsigned long mmu_seq;
Xiao Guangrongc7ba5b42012-06-20 15:59:18 +08002987 bool map_writable, write = error_code & PFERR_WRITE_MASK;
Marcelo Tosattiaaee2c92007-12-20 19:18:26 -05002988
Takuya Yoshikawafd136902015-10-16 17:06:02 +09002989 level = mapping_level(vcpu, gfn, &force_pt_level);
Andrea Arcangeli936a5fe2011-01-13 15:46:48 -08002990 if (likely(!force_pt_level)) {
Andrea Arcangeli936a5fe2011-01-13 15:46:48 -08002991 /*
2992 * This path builds a PAE pagetable - so we can map
2993 * 2mb pages at maximum. Therefore check if the level
2994 * is larger than that.
2995 */
2996 if (level > PT_DIRECTORY_LEVEL)
2997 level = PT_DIRECTORY_LEVEL;
Joerg Roedel852e3c12009-07-27 16:30:44 +02002998
Andrea Arcangeli936a5fe2011-01-13 15:46:48 -08002999 gfn &= ~(KVM_PAGES_PER_HPAGE(level) - 1);
Takuya Yoshikawafd136902015-10-16 17:06:02 +09003000 }
Marcelo Tosatti05da4552008-02-23 11:44:30 -03003001
Xiao Guangrongc7ba5b42012-06-20 15:59:18 +08003002 if (fast_page_fault(vcpu, v, level, error_code))
3003 return 0;
3004
Andrea Arcangelie930bff2008-07-25 16:24:52 +02003005 mmu_seq = vcpu->kvm->mmu_notifier_seq;
Marcelo Tosatti4c2155c2008-09-16 20:54:47 -03003006 smp_rmb();
Xiao Guangrong060c2ab2010-11-12 14:49:11 +08003007
Xiao Guangrong78b2c542010-12-07 10:48:06 +08003008 if (try_async_pf(vcpu, prefault, gfn, v, &pfn, write, &map_writable))
Xiao Guangrong060c2ab2010-11-12 14:49:11 +08003009 return 0;
Marcelo Tosattiaaee2c92007-12-20 19:18:26 -05003010
Xiao Guangrongd7c55202011-07-12 03:29:38 +08003011 if (handle_abnormal_pfn(vcpu, v, gfn, pfn, ACC_ALL, &r))
3012 return r;
Avi Kivityd196e342008-01-24 11:44:11 +02003013
Marcelo Tosattiaaee2c92007-12-20 19:18:26 -05003014 spin_lock(&vcpu->kvm->mmu_lock);
Christoffer Dall8ca40a72012-10-14 23:10:18 -04003015 if (mmu_notifier_retry(vcpu->kvm, mmu_seq))
Andrea Arcangelie930bff2008-07-25 16:24:52 +02003016 goto out_unlock;
Takuya Yoshikawa450e0b42013-03-29 14:05:26 +09003017 make_mmu_pages_available(vcpu);
Andrea Arcangeli936a5fe2011-01-13 15:46:48 -08003018 if (likely(!force_pt_level))
3019 transparent_hugepage_adjust(vcpu, &gfn, &pfn, &level);
Xiao Guangrong2ec47392010-12-07 10:34:42 +08003020 r = __direct_map(vcpu, v, write, map_writable, level, gfn, pfn,
3021 prefault);
Marcelo Tosattiaaee2c92007-12-20 19:18:26 -05003022 spin_unlock(&vcpu->kvm->mmu_lock);
3023
Marcelo Tosattiaaee2c92007-12-20 19:18:26 -05003024
Marcelo Tosatti10589a42007-12-20 19:18:22 -05003025 return r;
Andrea Arcangelie930bff2008-07-25 16:24:52 +02003026
3027out_unlock:
3028 spin_unlock(&vcpu->kvm->mmu_lock);
3029 kvm_release_pfn_clean(pfn);
3030 return 0;
Marcelo Tosatti10589a42007-12-20 19:18:22 -05003031}
3032
3033
Avi Kivity17ac10a2007-01-05 16:36:40 -08003034static void mmu_free_roots(struct kvm_vcpu *vcpu)
3035{
3036 int i;
Avi Kivity4db35312007-11-21 15:28:32 +02003037 struct kvm_mmu_page *sp;
Xiao Guangrongd98ba052010-06-04 21:55:29 +08003038 LIST_HEAD(invalid_list);
Avi Kivity17ac10a2007-01-05 16:36:40 -08003039
Zhang Xiantaoad312c72007-12-13 23:50:52 +08003040 if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
Avi Kivity7b53aa52007-06-05 12:17:03 +03003041 return;
Gleb Natapov35af5772013-05-16 11:55:51 +03003042
Joerg Roedel81407ca2010-09-10 17:31:00 +02003043 if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL &&
3044 (vcpu->arch.mmu.root_level == PT64_ROOT_LEVEL ||
3045 vcpu->arch.mmu.direct_map)) {
Zhang Xiantaoad312c72007-12-13 23:50:52 +08003046 hpa_t root = vcpu->arch.mmu.root_hpa;
Avi Kivity17ac10a2007-01-05 16:36:40 -08003047
Gleb Natapov35af5772013-05-16 11:55:51 +03003048 spin_lock(&vcpu->kvm->mmu_lock);
Avi Kivity4db35312007-11-21 15:28:32 +02003049 sp = page_header(root);
3050 --sp->root_count;
Xiao Guangrongd98ba052010-06-04 21:55:29 +08003051 if (!sp->root_count && sp->role.invalid) {
3052 kvm_mmu_prepare_zap_page(vcpu->kvm, sp, &invalid_list);
3053 kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
3054 }
Marcelo Tosattiaaee2c92007-12-20 19:18:26 -05003055 spin_unlock(&vcpu->kvm->mmu_lock);
Gleb Natapov35af5772013-05-16 11:55:51 +03003056 vcpu->arch.mmu.root_hpa = INVALID_PAGE;
Avi Kivity17ac10a2007-01-05 16:36:40 -08003057 return;
3058 }
Gleb Natapov35af5772013-05-16 11:55:51 +03003059
3060 spin_lock(&vcpu->kvm->mmu_lock);
Avi Kivity17ac10a2007-01-05 16:36:40 -08003061 for (i = 0; i < 4; ++i) {
Zhang Xiantaoad312c72007-12-13 23:50:52 +08003062 hpa_t root = vcpu->arch.mmu.pae_root[i];
Avi Kivity17ac10a2007-01-05 16:36:40 -08003063
Avi Kivity417726a2007-04-12 17:35:58 +03003064 if (root) {
Avi Kivity417726a2007-04-12 17:35:58 +03003065 root &= PT64_BASE_ADDR_MASK;
Avi Kivity4db35312007-11-21 15:28:32 +02003066 sp = page_header(root);
3067 --sp->root_count;
Marcelo Tosatti2e53d632008-02-20 14:47:24 -05003068 if (!sp->root_count && sp->role.invalid)
Xiao Guangrongd98ba052010-06-04 21:55:29 +08003069 kvm_mmu_prepare_zap_page(vcpu->kvm, sp,
3070 &invalid_list);
Avi Kivity417726a2007-04-12 17:35:58 +03003071 }
Zhang Xiantaoad312c72007-12-13 23:50:52 +08003072 vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
Avi Kivity17ac10a2007-01-05 16:36:40 -08003073 }
Xiao Guangrongd98ba052010-06-04 21:55:29 +08003074 kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
Marcelo Tosattiaaee2c92007-12-20 19:18:26 -05003075 spin_unlock(&vcpu->kvm->mmu_lock);
Zhang Xiantaoad312c72007-12-13 23:50:52 +08003076 vcpu->arch.mmu.root_hpa = INVALID_PAGE;
Avi Kivity17ac10a2007-01-05 16:36:40 -08003077}
3078
Marcelo Tosatti8986ecc2009-05-12 18:55:45 -03003079static int mmu_check_root(struct kvm_vcpu *vcpu, gfn_t root_gfn)
3080{
3081 int ret = 0;
3082
3083 if (!kvm_is_visible_gfn(vcpu->kvm, root_gfn)) {
Avi Kivitya8eeb042010-05-10 12:34:53 +03003084 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
Marcelo Tosatti8986ecc2009-05-12 18:55:45 -03003085 ret = 1;
3086 }
3087
3088 return ret;
3089}
3090
Joerg Roedel651dd372010-09-10 17:30:59 +02003091static int mmu_alloc_direct_roots(struct kvm_vcpu *vcpu)
3092{
3093 struct kvm_mmu_page *sp;
Avi Kivity7ebaf152010-10-03 18:51:39 +02003094 unsigned i;
Joerg Roedel651dd372010-09-10 17:30:59 +02003095
3096 if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
3097 spin_lock(&vcpu->kvm->mmu_lock);
Takuya Yoshikawa450e0b42013-03-29 14:05:26 +09003098 make_mmu_pages_available(vcpu);
Joerg Roedel651dd372010-09-10 17:30:59 +02003099 sp = kvm_mmu_get_page(vcpu, 0, 0, PT64_ROOT_LEVEL,
3100 1, ACC_ALL, NULL);
3101 ++sp->root_count;
3102 spin_unlock(&vcpu->kvm->mmu_lock);
3103 vcpu->arch.mmu.root_hpa = __pa(sp->spt);
3104 } else if (vcpu->arch.mmu.shadow_root_level == PT32E_ROOT_LEVEL) {
3105 for (i = 0; i < 4; ++i) {
3106 hpa_t root = vcpu->arch.mmu.pae_root[i];
3107
Paolo Bonzinifa4a2c02013-10-02 16:56:16 +02003108 MMU_WARN_ON(VALID_PAGE(root));
Joerg Roedel651dd372010-09-10 17:30:59 +02003109 spin_lock(&vcpu->kvm->mmu_lock);
Takuya Yoshikawa450e0b42013-03-29 14:05:26 +09003110 make_mmu_pages_available(vcpu);
Avi Kivity649497d2010-12-28 12:09:07 +02003111 sp = kvm_mmu_get_page(vcpu, i << (30 - PAGE_SHIFT),
3112 i << 30,
Joerg Roedel651dd372010-09-10 17:30:59 +02003113 PT32_ROOT_LEVEL, 1, ACC_ALL,
3114 NULL);
3115 root = __pa(sp->spt);
3116 ++sp->root_count;
3117 spin_unlock(&vcpu->kvm->mmu_lock);
3118 vcpu->arch.mmu.pae_root[i] = root | PT_PRESENT_MASK;
Joerg Roedel651dd372010-09-10 17:30:59 +02003119 }
Xiao Guangrong62927572010-09-27 18:02:12 +08003120 vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.pae_root);
Joerg Roedel651dd372010-09-10 17:30:59 +02003121 } else
3122 BUG();
3123
3124 return 0;
3125}
3126
3127static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu)
Avi Kivity17ac10a2007-01-05 16:36:40 -08003128{
Avi Kivity4db35312007-11-21 15:28:32 +02003129 struct kvm_mmu_page *sp;
Joerg Roedel81407ca2010-09-10 17:31:00 +02003130 u64 pdptr, pm_mask;
3131 gfn_t root_gfn;
3132 int i;
Avi Kivity3bb65a22007-01-05 16:36:51 -08003133
Joerg Roedel5777ed32010-09-10 17:30:42 +02003134 root_gfn = vcpu->arch.mmu.get_cr3(vcpu) >> PAGE_SHIFT;
Avi Kivity17ac10a2007-01-05 16:36:40 -08003135
Joerg Roedel651dd372010-09-10 17:30:59 +02003136 if (mmu_check_root(vcpu, root_gfn))
3137 return 1;
3138
3139 /*
3140 * Do we shadow a long mode page table? If so we need to
3141 * write-protect the guests page table root.
3142 */
3143 if (vcpu->arch.mmu.root_level == PT64_ROOT_LEVEL) {
Zhang Xiantaoad312c72007-12-13 23:50:52 +08003144 hpa_t root = vcpu->arch.mmu.root_hpa;
Avi Kivity17ac10a2007-01-05 16:36:40 -08003145
Paolo Bonzinifa4a2c02013-10-02 16:56:16 +02003146 MMU_WARN_ON(VALID_PAGE(root));
Joerg Roedel651dd372010-09-10 17:30:59 +02003147
Avi Kivity8facbbf2010-05-04 12:58:32 +03003148 spin_lock(&vcpu->kvm->mmu_lock);
Takuya Yoshikawa450e0b42013-03-29 14:05:26 +09003149 make_mmu_pages_available(vcpu);
Joerg Roedel651dd372010-09-10 17:30:59 +02003150 sp = kvm_mmu_get_page(vcpu, root_gfn, 0, PT64_ROOT_LEVEL,
3151 0, ACC_ALL, NULL);
Avi Kivity4db35312007-11-21 15:28:32 +02003152 root = __pa(sp->spt);
3153 ++sp->root_count;
Avi Kivity8facbbf2010-05-04 12:58:32 +03003154 spin_unlock(&vcpu->kvm->mmu_lock);
Zhang Xiantaoad312c72007-12-13 23:50:52 +08003155 vcpu->arch.mmu.root_hpa = root;
Marcelo Tosatti8986ecc2009-05-12 18:55:45 -03003156 return 0;
Avi Kivity17ac10a2007-01-05 16:36:40 -08003157 }
Joerg Roedelf87f9282010-09-02 17:29:45 +02003158
Joerg Roedel651dd372010-09-10 17:30:59 +02003159 /*
3160 * We shadow a 32 bit page table. This may be a legacy 2-level
Joerg Roedel81407ca2010-09-10 17:31:00 +02003161 * or a PAE 3-level page table. In either case we need to be aware that
3162 * the shadow page table may be a PAE or a long mode page table.
Joerg Roedel651dd372010-09-10 17:30:59 +02003163 */
Joerg Roedel81407ca2010-09-10 17:31:00 +02003164 pm_mask = PT_PRESENT_MASK;
3165 if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL)
3166 pm_mask |= PT_ACCESSED_MASK | PT_WRITABLE_MASK | PT_USER_MASK;
3167
Avi Kivity17ac10a2007-01-05 16:36:40 -08003168 for (i = 0; i < 4; ++i) {
Zhang Xiantaoad312c72007-12-13 23:50:52 +08003169 hpa_t root = vcpu->arch.mmu.pae_root[i];
Avi Kivity17ac10a2007-01-05 16:36:40 -08003170
Paolo Bonzinifa4a2c02013-10-02 16:56:16 +02003171 MMU_WARN_ON(VALID_PAGE(root));
Zhang Xiantaoad312c72007-12-13 23:50:52 +08003172 if (vcpu->arch.mmu.root_level == PT32E_ROOT_LEVEL) {
Avi Kivitye4e517b2011-07-28 11:36:17 +03003173 pdptr = vcpu->arch.mmu.get_pdptr(vcpu, i);
Avi Kivity43a37952009-06-10 14:12:05 +03003174 if (!is_present_gpte(pdptr)) {
Zhang Xiantaoad312c72007-12-13 23:50:52 +08003175 vcpu->arch.mmu.pae_root[i] = 0;
Avi Kivity417726a2007-04-12 17:35:58 +03003176 continue;
3177 }
Avi Kivity6de4f3a2009-05-31 22:58:47 +03003178 root_gfn = pdptr >> PAGE_SHIFT;
Joerg Roedelf87f9282010-09-02 17:29:45 +02003179 if (mmu_check_root(vcpu, root_gfn))
3180 return 1;
Eric Northup5a7388c2010-04-26 17:00:05 -07003181 }
Avi Kivity8facbbf2010-05-04 12:58:32 +03003182 spin_lock(&vcpu->kvm->mmu_lock);
Takuya Yoshikawa450e0b42013-03-29 14:05:26 +09003183 make_mmu_pages_available(vcpu);
Avi Kivity4db35312007-11-21 15:28:32 +02003184 sp = kvm_mmu_get_page(vcpu, root_gfn, i << 30,
Joerg Roedel651dd372010-09-10 17:30:59 +02003185 PT32_ROOT_LEVEL, 0,
Avi Kivityf7d9c7b2008-02-26 22:12:10 +02003186 ACC_ALL, NULL);
Avi Kivity4db35312007-11-21 15:28:32 +02003187 root = __pa(sp->spt);
3188 ++sp->root_count;
Avi Kivity8facbbf2010-05-04 12:58:32 +03003189 spin_unlock(&vcpu->kvm->mmu_lock);
3190
Joerg Roedel81407ca2010-09-10 17:31:00 +02003191 vcpu->arch.mmu.pae_root[i] = root | pm_mask;
Avi Kivity17ac10a2007-01-05 16:36:40 -08003192 }
Xiao Guangrong62927572010-09-27 18:02:12 +08003193 vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.pae_root);
Joerg Roedel81407ca2010-09-10 17:31:00 +02003194
3195 /*
3196 * If we shadow a 32 bit page table with a long mode page
3197 * table we enter this path.
3198 */
3199 if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
3200 if (vcpu->arch.mmu.lm_root == NULL) {
3201 /*
3202 * The additional page necessary for this is only
3203 * allocated on demand.
3204 */
3205
3206 u64 *lm_root;
3207
3208 lm_root = (void*)get_zeroed_page(GFP_KERNEL);
3209 if (lm_root == NULL)
3210 return 1;
3211
3212 lm_root[0] = __pa(vcpu->arch.mmu.pae_root) | pm_mask;
3213
3214 vcpu->arch.mmu.lm_root = lm_root;
3215 }
3216
3217 vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.lm_root);
3218 }
3219
Marcelo Tosatti8986ecc2009-05-12 18:55:45 -03003220 return 0;
Avi Kivity17ac10a2007-01-05 16:36:40 -08003221}
3222
Joerg Roedel651dd372010-09-10 17:30:59 +02003223static int mmu_alloc_roots(struct kvm_vcpu *vcpu)
3224{
3225 if (vcpu->arch.mmu.direct_map)
3226 return mmu_alloc_direct_roots(vcpu);
3227 else
3228 return mmu_alloc_shadow_roots(vcpu);
3229}
3230
Marcelo Tosatti0ba73cd2008-09-23 13:18:34 -03003231static void mmu_sync_roots(struct kvm_vcpu *vcpu)
3232{
3233 int i;
3234 struct kvm_mmu_page *sp;
3235
Joerg Roedel81407ca2010-09-10 17:31:00 +02003236 if (vcpu->arch.mmu.direct_map)
3237 return;
3238
Marcelo Tosatti0ba73cd2008-09-23 13:18:34 -03003239 if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
3240 return;
Xiao Guangrong69030742010-09-27 18:09:29 +08003241
David Matlack56f17dd2014-08-18 15:46:07 -07003242 vcpu_clear_mmio_info(vcpu, MMIO_GVA_ANY);
Xiao Guangrong0375f7f2011-11-28 20:41:00 +08003243 kvm_mmu_audit(vcpu, AUDIT_PRE_SYNC);
Joerg Roedel81407ca2010-09-10 17:31:00 +02003244 if (vcpu->arch.mmu.root_level == PT64_ROOT_LEVEL) {
Marcelo Tosatti0ba73cd2008-09-23 13:18:34 -03003245 hpa_t root = vcpu->arch.mmu.root_hpa;
3246 sp = page_header(root);
3247 mmu_sync_children(vcpu, sp);
Xiao Guangrong0375f7f2011-11-28 20:41:00 +08003248 kvm_mmu_audit(vcpu, AUDIT_POST_SYNC);
Marcelo Tosatti0ba73cd2008-09-23 13:18:34 -03003249 return;
3250 }
3251 for (i = 0; i < 4; ++i) {
3252 hpa_t root = vcpu->arch.mmu.pae_root[i];
3253
Marcelo Tosatti8986ecc2009-05-12 18:55:45 -03003254 if (root && VALID_PAGE(root)) {
Marcelo Tosatti0ba73cd2008-09-23 13:18:34 -03003255 root &= PT64_BASE_ADDR_MASK;
3256 sp = page_header(root);
3257 mmu_sync_children(vcpu, sp);
3258 }
3259 }
Xiao Guangrong0375f7f2011-11-28 20:41:00 +08003260 kvm_mmu_audit(vcpu, AUDIT_POST_SYNC);
Marcelo Tosatti0ba73cd2008-09-23 13:18:34 -03003261}
3262
3263void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu)
3264{
3265 spin_lock(&vcpu->kvm->mmu_lock);
3266 mmu_sync_roots(vcpu);
3267 spin_unlock(&vcpu->kvm->mmu_lock);
3268}
Nadav Har'Elbfd0a562013-08-05 11:07:17 +03003269EXPORT_SYMBOL_GPL(kvm_mmu_sync_roots);
Marcelo Tosatti0ba73cd2008-09-23 13:18:34 -03003270
Gleb Natapov1871c602010-02-10 14:21:32 +02003271static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr,
Avi Kivityab9ae312010-11-22 17:53:26 +02003272 u32 access, struct x86_exception *exception)
Avi Kivity6aa8b732006-12-10 02:21:36 -08003273{
Avi Kivityab9ae312010-11-22 17:53:26 +02003274 if (exception)
3275 exception->error_code = 0;
Avi Kivity6aa8b732006-12-10 02:21:36 -08003276 return vaddr;
3277}
3278
Joerg Roedel6539e732010-09-10 17:30:50 +02003279static gpa_t nonpaging_gva_to_gpa_nested(struct kvm_vcpu *vcpu, gva_t vaddr,
Avi Kivityab9ae312010-11-22 17:53:26 +02003280 u32 access,
3281 struct x86_exception *exception)
Joerg Roedel6539e732010-09-10 17:30:50 +02003282{
Avi Kivityab9ae312010-11-22 17:53:26 +02003283 if (exception)
3284 exception->error_code = 0;
Paolo Bonzini54987b72014-09-02 13:23:06 +02003285 return vcpu->arch.nested_mmu.translate_gpa(vcpu, vaddr, access, exception);
Joerg Roedel6539e732010-09-10 17:30:50 +02003286}
3287
Xiao Guangrongd625b152015-08-05 12:04:25 +08003288static bool
3289__is_rsvd_bits_set(struct rsvd_bits_validate *rsvd_check, u64 pte, int level)
3290{
3291 int bit7 = (pte >> 7) & 1, low6 = pte & 0x3f;
3292
3293 return (pte & rsvd_check->rsvd_bits_mask[bit7][level-1]) |
3294 ((rsvd_check->bad_mt_xwr & (1ull << low6)) != 0);
3295}
3296
3297static bool is_rsvd_bits_set(struct kvm_mmu *mmu, u64 gpte, int level)
3298{
3299 return __is_rsvd_bits_set(&mmu->guest_rsvd_check, gpte, level);
3300}
3301
3302static bool is_shadow_zero_bits_set(struct kvm_mmu *mmu, u64 spte, int level)
3303{
3304 return __is_rsvd_bits_set(&mmu->shadow_zero_check, spte, level);
3305}
3306
Xiao Guangrongce88dec2011-07-12 03:33:44 +08003307static bool quickly_check_mmio_pf(struct kvm_vcpu *vcpu, u64 addr, bool direct)
3308{
3309 if (direct)
3310 return vcpu_match_mmio_gpa(vcpu, addr);
3311
3312 return vcpu_match_mmio_gva(vcpu, addr);
3313}
3314
Xiao Guangrong47ab8752015-08-05 12:04:26 +08003315/* return true if reserved bit is detected on spte. */
3316static bool
3317walk_shadow_page_get_mmio_spte(struct kvm_vcpu *vcpu, u64 addr, u64 *sptep)
Xiao Guangrongce88dec2011-07-12 03:33:44 +08003318{
3319 struct kvm_shadow_walk_iterator iterator;
Xiao Guangrong47ab8752015-08-05 12:04:26 +08003320 u64 sptes[PT64_ROOT_LEVEL], spte = 0ull;
3321 int root, leaf;
3322 bool reserved = false;
Xiao Guangrongce88dec2011-07-12 03:33:44 +08003323
Marcelo Tosatti37f6a4e2014-01-03 17:09:32 -02003324 if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
Xiao Guangrong47ab8752015-08-05 12:04:26 +08003325 goto exit;
Marcelo Tosatti37f6a4e2014-01-03 17:09:32 -02003326
Xiao Guangrongce88dec2011-07-12 03:33:44 +08003327 walk_shadow_page_lockless_begin(vcpu);
Xiao Guangrong47ab8752015-08-05 12:04:26 +08003328
Paolo Bonzini29ecd662015-09-06 16:24:50 +02003329 for (shadow_walk_init(&iterator, vcpu, addr),
3330 leaf = root = iterator.level;
Xiao Guangrong47ab8752015-08-05 12:04:26 +08003331 shadow_walk_okay(&iterator);
3332 __shadow_walk_next(&iterator, spte)) {
Xiao Guangrong47ab8752015-08-05 12:04:26 +08003333 spte = mmu_spte_get_lockless(iterator.sptep);
3334
3335 sptes[leaf - 1] = spte;
Paolo Bonzini29ecd662015-09-06 16:24:50 +02003336 leaf--;
Xiao Guangrong47ab8752015-08-05 12:04:26 +08003337
Xiao Guangrongce88dec2011-07-12 03:33:44 +08003338 if (!is_shadow_present_pte(spte))
3339 break;
Xiao Guangrong47ab8752015-08-05 12:04:26 +08003340
3341 reserved |= is_shadow_zero_bits_set(&vcpu->arch.mmu, spte,
Paolo Bonzini58c95072015-09-22 10:15:59 +02003342 iterator.level);
Xiao Guangrong47ab8752015-08-05 12:04:26 +08003343 }
3344
Xiao Guangrongce88dec2011-07-12 03:33:44 +08003345 walk_shadow_page_lockless_end(vcpu);
3346
Xiao Guangrong47ab8752015-08-05 12:04:26 +08003347 if (reserved) {
3348 pr_err("%s: detect reserved bits on spte, addr 0x%llx, dump hierarchy:\n",
3349 __func__, addr);
Paolo Bonzini29ecd662015-09-06 16:24:50 +02003350 while (root > leaf) {
Xiao Guangrong47ab8752015-08-05 12:04:26 +08003351 pr_err("------ spte 0x%llx level %d.\n",
3352 sptes[root - 1], root);
3353 root--;
3354 }
3355 }
3356exit:
3357 *sptep = spte;
3358 return reserved;
Xiao Guangrongce88dec2011-07-12 03:33:44 +08003359}
3360
Xiao Guangrongce88dec2011-07-12 03:33:44 +08003361int handle_mmio_page_fault_common(struct kvm_vcpu *vcpu, u64 addr, bool direct)
3362{
3363 u64 spte;
Xiao Guangrong47ab8752015-08-05 12:04:26 +08003364 bool reserved;
Xiao Guangrongce88dec2011-07-12 03:33:44 +08003365
3366 if (quickly_check_mmio_pf(vcpu, addr, direct))
Xiao Guangrongb37fbea2013-06-07 16:51:25 +08003367 return RET_MMIO_PF_EMULATE;
Xiao Guangrongce88dec2011-07-12 03:33:44 +08003368
Xiao Guangrong47ab8752015-08-05 12:04:26 +08003369 reserved = walk_shadow_page_get_mmio_spte(vcpu, addr, &spte);
3370 if (unlikely(reserved))
3371 return RET_MMIO_PF_BUG;
Xiao Guangrongce88dec2011-07-12 03:33:44 +08003372
3373 if (is_mmio_spte(spte)) {
3374 gfn_t gfn = get_mmio_spte_gfn(spte);
3375 unsigned access = get_mmio_spte_access(spte);
3376
Paolo Bonzini54bf36a2015-04-08 15:39:23 +02003377 if (!check_mmio_spte(vcpu, spte))
Xiao Guangrongf8f55942013-06-07 16:51:26 +08003378 return RET_MMIO_PF_INVALID;
3379
Xiao Guangrongce88dec2011-07-12 03:33:44 +08003380 if (direct)
3381 addr = 0;
Xiao Guangrong4f022642011-07-12 03:34:24 +08003382
3383 trace_handle_mmio_page_fault(addr, gfn, access);
Xiao Guangrongce88dec2011-07-12 03:33:44 +08003384 vcpu_cache_mmio_info(vcpu, addr, gfn, access);
Xiao Guangrongb37fbea2013-06-07 16:51:25 +08003385 return RET_MMIO_PF_EMULATE;
Xiao Guangrongce88dec2011-07-12 03:33:44 +08003386 }
3387
3388 /*
Xiao Guangrongce88dec2011-07-12 03:33:44 +08003389 * If the page table is zapped by other cpus, let CPU fault again on
3390 * the address.
3391 */
Xiao Guangrongb37fbea2013-06-07 16:51:25 +08003392 return RET_MMIO_PF_RETRY;
Xiao Guangrongce88dec2011-07-12 03:33:44 +08003393}
3394EXPORT_SYMBOL_GPL(handle_mmio_page_fault_common);
3395
3396static int handle_mmio_page_fault(struct kvm_vcpu *vcpu, u64 addr,
3397 u32 error_code, bool direct)
3398{
3399 int ret;
3400
3401 ret = handle_mmio_page_fault_common(vcpu, addr, direct);
Xiao Guangrongb37fbea2013-06-07 16:51:25 +08003402 WARN_ON(ret == RET_MMIO_PF_BUG);
Xiao Guangrongce88dec2011-07-12 03:33:44 +08003403 return ret;
3404}
3405
Avi Kivity6aa8b732006-12-10 02:21:36 -08003406static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva,
Xiao Guangrong78b2c542010-12-07 10:48:06 +08003407 u32 error_code, bool prefault)
Avi Kivity6aa8b732006-12-10 02:21:36 -08003408{
Avi Kivitye8332402007-12-09 18:43:00 +02003409 gfn_t gfn;
Avi Kivitye2dec932007-01-05 16:36:54 -08003410 int r;
Avi Kivity6aa8b732006-12-10 02:21:36 -08003411
Harvey Harrisonb8688d52008-03-03 12:59:56 -08003412 pgprintk("%s: gva %lx error %x\n", __func__, gva, error_code);
Xiao Guangrongce88dec2011-07-12 03:33:44 +08003413
Xiao Guangrongf8f55942013-06-07 16:51:26 +08003414 if (unlikely(error_code & PFERR_RSVD_MASK)) {
3415 r = handle_mmio_page_fault(vcpu, gva, error_code, true);
3416
3417 if (likely(r != RET_MMIO_PF_INVALID))
3418 return r;
3419 }
Xiao Guangrongce88dec2011-07-12 03:33:44 +08003420
Avi Kivitye2dec932007-01-05 16:36:54 -08003421 r = mmu_topup_memory_caches(vcpu);
3422 if (r)
3423 return r;
Avi Kivity714b93d2007-01-05 16:36:53 -08003424
Paolo Bonzinifa4a2c02013-10-02 16:56:16 +02003425 MMU_WARN_ON(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
Avi Kivity6aa8b732006-12-10 02:21:36 -08003426
Avi Kivitye8332402007-12-09 18:43:00 +02003427 gfn = gva >> PAGE_SHIFT;
Avi Kivity6aa8b732006-12-10 02:21:36 -08003428
Avi Kivitye8332402007-12-09 18:43:00 +02003429 return nonpaging_map(vcpu, gva & PAGE_MASK,
Xiao Guangrongc7ba5b42012-06-20 15:59:18 +08003430 error_code, gfn, prefault);
Avi Kivity6aa8b732006-12-10 02:21:36 -08003431}
3432
Jan Kiszka7e1fbea2010-10-20 15:18:02 +02003433static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn)
Gleb Natapovaf585b92010-10-14 11:22:46 +02003434{
3435 struct kvm_arch_async_pf arch;
Xiao Guangrongfb67e142010-12-07 10:35:25 +08003436
Gleb Natapov7c907052010-10-14 11:22:53 +02003437 arch.token = (vcpu->arch.apf.id++ << 12) | vcpu->vcpu_id;
Gleb Natapovaf585b92010-10-14 11:22:46 +02003438 arch.gfn = gfn;
Xiao Guangrongc4806ac2010-11-12 14:49:55 +08003439 arch.direct_map = vcpu->arch.mmu.direct_map;
Xiao Guangrongfb67e142010-12-07 10:35:25 +08003440 arch.cr3 = vcpu->arch.mmu.get_cr3(vcpu);
Gleb Natapovaf585b92010-10-14 11:22:46 +02003441
Paolo Bonzini54bf36a2015-04-08 15:39:23 +02003442 return kvm_setup_async_pf(vcpu, gva, kvm_vcpu_gfn_to_hva(vcpu, gfn), &arch);
Gleb Natapovaf585b92010-10-14 11:22:46 +02003443}
3444
3445static bool can_do_async_pf(struct kvm_vcpu *vcpu)
3446{
Paolo Bonzini35754c92015-07-29 12:05:37 +02003447 if (unlikely(!lapic_in_kernel(vcpu) ||
Gleb Natapovaf585b92010-10-14 11:22:46 +02003448 kvm_event_needs_reinjection(vcpu)))
3449 return false;
3450
3451 return kvm_x86_ops->interrupt_allowed(vcpu);
3452}
3453
Xiao Guangrong78b2c542010-12-07 10:48:06 +08003454static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn,
Marcelo Tosatti612819c2010-10-22 14:18:18 -02003455 gva_t gva, pfn_t *pfn, bool write, bool *writable)
Gleb Natapovaf585b92010-10-14 11:22:46 +02003456{
Paolo Bonzini35204692015-04-02 11:20:48 +02003457 struct kvm_memory_slot *slot;
Gleb Natapovaf585b92010-10-14 11:22:46 +02003458 bool async;
3459
Paolo Bonzini54bf36a2015-04-08 15:39:23 +02003460 slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
Paolo Bonzini35204692015-04-02 11:20:48 +02003461 async = false;
3462 *pfn = __gfn_to_pfn_memslot(slot, gfn, false, &async, write, writable);
Gleb Natapovaf585b92010-10-14 11:22:46 +02003463 if (!async)
3464 return false; /* *pfn has correct page already */
3465
Xiao Guangrong78b2c542010-12-07 10:48:06 +08003466 if (!prefault && can_do_async_pf(vcpu)) {
Xiao Guangrongc9b263d2010-11-01 16:58:43 +08003467 trace_kvm_try_async_get_page(gva, gfn);
Gleb Natapovaf585b92010-10-14 11:22:46 +02003468 if (kvm_find_async_pf_gfn(vcpu, gfn)) {
3469 trace_kvm_async_pf_doublefault(gva, gfn);
3470 kvm_make_request(KVM_REQ_APF_HALT, vcpu);
3471 return true;
3472 } else if (kvm_arch_setup_async_pf(vcpu, gva, gfn))
3473 return true;
3474 }
3475
Paolo Bonzini35204692015-04-02 11:20:48 +02003476 *pfn = __gfn_to_pfn_memslot(slot, gfn, false, NULL, write, writable);
Gleb Natapovaf585b92010-10-14 11:22:46 +02003477 return false;
3478}
3479
Xiao Guangrong6a39bbc2015-06-15 16:55:35 +08003480static bool
3481check_hugepage_cache_consistency(struct kvm_vcpu *vcpu, gfn_t gfn, int level)
3482{
3483 int page_num = KVM_PAGES_PER_HPAGE(level);
3484
3485 gfn &= ~(page_num - 1);
3486
3487 return kvm_mtrr_check_gfn_range_consistency(vcpu, gfn, page_num);
3488}
3489
Gleb Natapov56028d02010-10-17 18:13:42 +02003490static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa, u32 error_code,
Xiao Guangrong78b2c542010-12-07 10:48:06 +08003491 bool prefault)
Joerg Roedelfb72d162008-02-07 13:47:44 +01003492{
Anthony Liguori35149e22008-04-02 14:46:56 -05003493 pfn_t pfn;
Joerg Roedelfb72d162008-02-07 13:47:44 +01003494 int r;
Joerg Roedel852e3c12009-07-27 16:30:44 +02003495 int level;
Takuya Yoshikawacd1872f2015-10-16 17:04:13 +09003496 bool force_pt_level;
Marcelo Tosatti05da4552008-02-23 11:44:30 -03003497 gfn_t gfn = gpa >> PAGE_SHIFT;
Andrea Arcangelie930bff2008-07-25 16:24:52 +02003498 unsigned long mmu_seq;
Marcelo Tosatti612819c2010-10-22 14:18:18 -02003499 int write = error_code & PFERR_WRITE_MASK;
3500 bool map_writable;
Joerg Roedelfb72d162008-02-07 13:47:44 +01003501
Paolo Bonzinifa4a2c02013-10-02 16:56:16 +02003502 MMU_WARN_ON(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
Joerg Roedelfb72d162008-02-07 13:47:44 +01003503
Xiao Guangrongf8f55942013-06-07 16:51:26 +08003504 if (unlikely(error_code & PFERR_RSVD_MASK)) {
3505 r = handle_mmio_page_fault(vcpu, gpa, error_code, true);
3506
3507 if (likely(r != RET_MMIO_PF_INVALID))
3508 return r;
3509 }
Xiao Guangrongce88dec2011-07-12 03:33:44 +08003510
Joerg Roedelfb72d162008-02-07 13:47:44 +01003511 r = mmu_topup_memory_caches(vcpu);
3512 if (r)
3513 return r;
3514
Takuya Yoshikawafd136902015-10-16 17:06:02 +09003515 force_pt_level = !check_hugepage_cache_consistency(vcpu, gfn,
3516 PT_DIRECTORY_LEVEL);
3517 level = mapping_level(vcpu, gfn, &force_pt_level);
Andrea Arcangeli936a5fe2011-01-13 15:46:48 -08003518 if (likely(!force_pt_level)) {
Xiao Guangrong6a39bbc2015-06-15 16:55:35 +08003519 if (level > PT_DIRECTORY_LEVEL &&
3520 !check_hugepage_cache_consistency(vcpu, gfn, level))
3521 level = PT_DIRECTORY_LEVEL;
Andrea Arcangeli936a5fe2011-01-13 15:46:48 -08003522 gfn &= ~(KVM_PAGES_PER_HPAGE(level) - 1);
Takuya Yoshikawafd136902015-10-16 17:06:02 +09003523 }
Joerg Roedel852e3c12009-07-27 16:30:44 +02003524
Xiao Guangrongc7ba5b42012-06-20 15:59:18 +08003525 if (fast_page_fault(vcpu, gpa, level, error_code))
3526 return 0;
3527
Andrea Arcangelie930bff2008-07-25 16:24:52 +02003528 mmu_seq = vcpu->kvm->mmu_notifier_seq;
Marcelo Tosatti4c2155c2008-09-16 20:54:47 -03003529 smp_rmb();
Gleb Natapovaf585b92010-10-14 11:22:46 +02003530
Xiao Guangrong78b2c542010-12-07 10:48:06 +08003531 if (try_async_pf(vcpu, prefault, gfn, gpa, &pfn, write, &map_writable))
Gleb Natapovaf585b92010-10-14 11:22:46 +02003532 return 0;
3533
Xiao Guangrongd7c55202011-07-12 03:29:38 +08003534 if (handle_abnormal_pfn(vcpu, 0, gfn, pfn, ACC_ALL, &r))
3535 return r;
3536
Joerg Roedelfb72d162008-02-07 13:47:44 +01003537 spin_lock(&vcpu->kvm->mmu_lock);
Christoffer Dall8ca40a72012-10-14 23:10:18 -04003538 if (mmu_notifier_retry(vcpu->kvm, mmu_seq))
Andrea Arcangelie930bff2008-07-25 16:24:52 +02003539 goto out_unlock;
Takuya Yoshikawa450e0b42013-03-29 14:05:26 +09003540 make_mmu_pages_available(vcpu);
Andrea Arcangeli936a5fe2011-01-13 15:46:48 -08003541 if (likely(!force_pt_level))
3542 transparent_hugepage_adjust(vcpu, &gfn, &pfn, &level);
Marcelo Tosatti612819c2010-10-22 14:18:18 -02003543 r = __direct_map(vcpu, gpa, write, map_writable,
Xiao Guangrong2ec47392010-12-07 10:34:42 +08003544 level, gfn, pfn, prefault);
Joerg Roedelfb72d162008-02-07 13:47:44 +01003545 spin_unlock(&vcpu->kvm->mmu_lock);
Joerg Roedelfb72d162008-02-07 13:47:44 +01003546
3547 return r;
Andrea Arcangelie930bff2008-07-25 16:24:52 +02003548
3549out_unlock:
3550 spin_unlock(&vcpu->kvm->mmu_lock);
3551 kvm_release_pfn_clean(pfn);
3552 return 0;
Joerg Roedelfb72d162008-02-07 13:47:44 +01003553}
3554
Paolo Bonzini8a3c1a332013-10-02 16:56:13 +02003555static void nonpaging_init_context(struct kvm_vcpu *vcpu,
3556 struct kvm_mmu *context)
Avi Kivity6aa8b732006-12-10 02:21:36 -08003557{
Avi Kivity6aa8b732006-12-10 02:21:36 -08003558 context->page_fault = nonpaging_page_fault;
Avi Kivity6aa8b732006-12-10 02:21:36 -08003559 context->gva_to_gpa = nonpaging_gva_to_gpa;
Marcelo Tosattie8bc2172008-09-23 13:18:33 -03003560 context->sync_page = nonpaging_sync_page;
Marcelo Tosattia7052892008-09-23 13:18:35 -03003561 context->invlpg = nonpaging_invlpg;
Xiao Guangrong0f53b5b2011-03-09 15:43:51 +08003562 context->update_pte = nonpaging_update_pte;
Avi Kivitycea0f0e2007-01-05 16:36:43 -08003563 context->root_level = 0;
Avi Kivity6aa8b732006-12-10 02:21:36 -08003564 context->shadow_root_level = PT32E_ROOT_LEVEL;
Avi Kivity17c3ba92007-06-04 15:58:30 +03003565 context->root_hpa = INVALID_PAGE;
Joerg Roedelc5a78f2b2010-09-10 17:30:39 +02003566 context->direct_map = true;
Joerg Roedel2d48a982010-09-10 17:31:01 +02003567 context->nx = false;
Avi Kivity6aa8b732006-12-10 02:21:36 -08003568}
3569
Paolo Bonzinid8d173d2013-10-02 16:56:11 +02003570void kvm_mmu_new_cr3(struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -08003571{
Avi Kivitycea0f0e2007-01-05 16:36:43 -08003572 mmu_free_roots(vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -08003573}
3574
Joerg Roedel5777ed32010-09-10 17:30:42 +02003575static unsigned long get_cr3(struct kvm_vcpu *vcpu)
3576{
Avi Kivity9f8fe502010-12-05 17:30:00 +02003577 return kvm_read_cr3(vcpu);
Joerg Roedel5777ed32010-09-10 17:30:42 +02003578}
3579
Avi Kivity6389ee92010-11-29 16:12:30 +02003580static void inject_page_fault(struct kvm_vcpu *vcpu,
3581 struct x86_exception *fault)
Avi Kivity6aa8b732006-12-10 02:21:36 -08003582{
Avi Kivity6389ee92010-11-29 16:12:30 +02003583 vcpu->arch.mmu.inject_page_fault(vcpu, fault);
Avi Kivity6aa8b732006-12-10 02:21:36 -08003584}
3585
Paolo Bonzini54bf36a2015-04-08 15:39:23 +02003586static bool sync_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, gfn_t gfn,
Xiao Guangrongf2fd1252013-06-07 16:51:24 +08003587 unsigned access, int *nr_present)
Xiao Guangrongce88dec2011-07-12 03:33:44 +08003588{
3589 if (unlikely(is_mmio_spte(*sptep))) {
3590 if (gfn != get_mmio_spte_gfn(*sptep)) {
3591 mmu_spte_clear_no_track(sptep);
3592 return true;
3593 }
3594
3595 (*nr_present)++;
Paolo Bonzini54bf36a2015-04-08 15:39:23 +02003596 mark_mmio_spte(vcpu, sptep, gfn, access);
Xiao Guangrongce88dec2011-07-12 03:33:44 +08003597 return true;
3598 }
3599
3600 return false;
3601}
3602
Avi Kivity6fd01b72012-09-12 20:46:56 +03003603static inline bool is_last_gpte(struct kvm_mmu *mmu, unsigned level, unsigned gpte)
3604{
3605 unsigned index;
3606
3607 index = level - 1;
3608 index |= (gpte & PT_PAGE_SIZE_MASK) >> (PT_PAGE_SIZE_SHIFT - 2);
3609 return mmu->last_pte_bitmap & (1 << index);
3610}
3611
Nadav Har'El37406aa2013-08-05 11:07:12 +03003612#define PTTYPE_EPT 18 /* arbitrary */
3613#define PTTYPE PTTYPE_EPT
3614#include "paging_tmpl.h"
3615#undef PTTYPE
3616
Avi Kivity6aa8b732006-12-10 02:21:36 -08003617#define PTTYPE 64
3618#include "paging_tmpl.h"
3619#undef PTTYPE
3620
3621#define PTTYPE 32
3622#include "paging_tmpl.h"
3623#undef PTTYPE
3624
Xiao Guangrong6dc98b82015-08-05 12:04:22 +08003625static void
3626__reset_rsvds_bits_mask(struct kvm_vcpu *vcpu,
3627 struct rsvd_bits_validate *rsvd_check,
3628 int maxphyaddr, int level, bool nx, bool gbpages,
Paolo Bonzini6fec2142015-09-22 23:02:14 +02003629 bool pse, bool amd)
Dong, Eddie82725b22009-03-30 16:21:08 +08003630{
Dong, Eddie82725b22009-03-30 16:21:08 +08003631 u64 exb_bit_rsvd = 0;
Nadav Amit5f7dde72014-05-07 15:32:50 +03003632 u64 gbpages_bit_rsvd = 0;
Paolo Bonzinia0c0feb2014-09-02 13:24:12 +02003633 u64 nonleaf_bit8_rsvd = 0;
Dong, Eddie82725b22009-03-30 16:21:08 +08003634
Xiao Guangronga0a64f52015-08-05 12:04:21 +08003635 rsvd_check->bad_mt_xwr = 0;
Yang Zhang25d92082013-08-06 12:00:32 +03003636
Xiao Guangrong6dc98b82015-08-05 12:04:22 +08003637 if (!nx)
Dong, Eddie82725b22009-03-30 16:21:08 +08003638 exb_bit_rsvd = rsvd_bits(63, 63);
Xiao Guangrong6dc98b82015-08-05 12:04:22 +08003639 if (!gbpages)
Nadav Amit5f7dde72014-05-07 15:32:50 +03003640 gbpages_bit_rsvd = rsvd_bits(7, 7);
Paolo Bonzinia0c0feb2014-09-02 13:24:12 +02003641
3642 /*
3643 * Non-leaf PML4Es and PDPEs reserve bit 8 (which would be the G bit for
3644 * leaf entries) on AMD CPUs only.
3645 */
Paolo Bonzini6fec2142015-09-22 23:02:14 +02003646 if (amd)
Paolo Bonzinia0c0feb2014-09-02 13:24:12 +02003647 nonleaf_bit8_rsvd = rsvd_bits(8, 8);
3648
Xiao Guangrong6dc98b82015-08-05 12:04:22 +08003649 switch (level) {
Dong, Eddie82725b22009-03-30 16:21:08 +08003650 case PT32_ROOT_LEVEL:
3651 /* no rsvd bits for 2 level 4K page table entries */
Xiao Guangronga0a64f52015-08-05 12:04:21 +08003652 rsvd_check->rsvd_bits_mask[0][1] = 0;
3653 rsvd_check->rsvd_bits_mask[0][0] = 0;
3654 rsvd_check->rsvd_bits_mask[1][0] =
3655 rsvd_check->rsvd_bits_mask[0][0];
Xiao Guangrongf815bce2010-03-19 17:58:53 +08003656
Xiao Guangrong6dc98b82015-08-05 12:04:22 +08003657 if (!pse) {
Xiao Guangronga0a64f52015-08-05 12:04:21 +08003658 rsvd_check->rsvd_bits_mask[1][1] = 0;
Xiao Guangrongf815bce2010-03-19 17:58:53 +08003659 break;
3660 }
3661
Dong, Eddie82725b22009-03-30 16:21:08 +08003662 if (is_cpuid_PSE36())
3663 /* 36bits PSE 4MB page */
Xiao Guangronga0a64f52015-08-05 12:04:21 +08003664 rsvd_check->rsvd_bits_mask[1][1] = rsvd_bits(17, 21);
Dong, Eddie82725b22009-03-30 16:21:08 +08003665 else
3666 /* 32 bits PSE 4MB page */
Xiao Guangronga0a64f52015-08-05 12:04:21 +08003667 rsvd_check->rsvd_bits_mask[1][1] = rsvd_bits(13, 21);
Dong, Eddie82725b22009-03-30 16:21:08 +08003668 break;
3669 case PT32E_ROOT_LEVEL:
Xiao Guangronga0a64f52015-08-05 12:04:21 +08003670 rsvd_check->rsvd_bits_mask[0][2] =
Dong, Eddie20c466b2009-03-31 23:03:45 +08003671 rsvd_bits(maxphyaddr, 63) |
Nadav Amitcd9ae5f2014-04-04 06:31:04 +03003672 rsvd_bits(5, 8) | rsvd_bits(1, 2); /* PDPTE */
Xiao Guangronga0a64f52015-08-05 12:04:21 +08003673 rsvd_check->rsvd_bits_mask[0][1] = exb_bit_rsvd |
Sheng Yang4c26b4c2009-04-02 10:28:37 +08003674 rsvd_bits(maxphyaddr, 62); /* PDE */
Xiao Guangronga0a64f52015-08-05 12:04:21 +08003675 rsvd_check->rsvd_bits_mask[0][0] = exb_bit_rsvd |
Dong, Eddie82725b22009-03-30 16:21:08 +08003676 rsvd_bits(maxphyaddr, 62); /* PTE */
Xiao Guangronga0a64f52015-08-05 12:04:21 +08003677 rsvd_check->rsvd_bits_mask[1][1] = exb_bit_rsvd |
Dong, Eddie82725b22009-03-30 16:21:08 +08003678 rsvd_bits(maxphyaddr, 62) |
3679 rsvd_bits(13, 20); /* large page */
Xiao Guangronga0a64f52015-08-05 12:04:21 +08003680 rsvd_check->rsvd_bits_mask[1][0] =
3681 rsvd_check->rsvd_bits_mask[0][0];
Dong, Eddie82725b22009-03-30 16:21:08 +08003682 break;
3683 case PT64_ROOT_LEVEL:
Xiao Guangronga0a64f52015-08-05 12:04:21 +08003684 rsvd_check->rsvd_bits_mask[0][3] = exb_bit_rsvd |
3685 nonleaf_bit8_rsvd | rsvd_bits(7, 7) |
Sheng Yang4c26b4c2009-04-02 10:28:37 +08003686 rsvd_bits(maxphyaddr, 51);
Xiao Guangronga0a64f52015-08-05 12:04:21 +08003687 rsvd_check->rsvd_bits_mask[0][2] = exb_bit_rsvd |
3688 nonleaf_bit8_rsvd | gbpages_bit_rsvd |
Dong, Eddie82725b22009-03-30 16:21:08 +08003689 rsvd_bits(maxphyaddr, 51);
Xiao Guangronga0a64f52015-08-05 12:04:21 +08003690 rsvd_check->rsvd_bits_mask[0][1] = exb_bit_rsvd |
3691 rsvd_bits(maxphyaddr, 51);
3692 rsvd_check->rsvd_bits_mask[0][0] = exb_bit_rsvd |
3693 rsvd_bits(maxphyaddr, 51);
3694 rsvd_check->rsvd_bits_mask[1][3] =
3695 rsvd_check->rsvd_bits_mask[0][3];
3696 rsvd_check->rsvd_bits_mask[1][2] = exb_bit_rsvd |
Nadav Amit5f7dde72014-05-07 15:32:50 +03003697 gbpages_bit_rsvd | rsvd_bits(maxphyaddr, 51) |
Joerg Roedele04da982009-07-27 16:30:45 +02003698 rsvd_bits(13, 29);
Xiao Guangronga0a64f52015-08-05 12:04:21 +08003699 rsvd_check->rsvd_bits_mask[1][1] = exb_bit_rsvd |
Sheng Yang4c26b4c2009-04-02 10:28:37 +08003700 rsvd_bits(maxphyaddr, 51) |
3701 rsvd_bits(13, 20); /* large page */
Xiao Guangronga0a64f52015-08-05 12:04:21 +08003702 rsvd_check->rsvd_bits_mask[1][0] =
3703 rsvd_check->rsvd_bits_mask[0][0];
Dong, Eddie82725b22009-03-30 16:21:08 +08003704 break;
3705 }
3706}
3707
Xiao Guangrong6dc98b82015-08-05 12:04:22 +08003708static void reset_rsvds_bits_mask(struct kvm_vcpu *vcpu,
3709 struct kvm_mmu *context)
3710{
3711 __reset_rsvds_bits_mask(vcpu, &context->guest_rsvd_check,
3712 cpuid_maxphyaddr(vcpu), context->root_level,
3713 context->nx, guest_cpuid_has_gbpages(vcpu),
Paolo Bonzini6fec2142015-09-22 23:02:14 +02003714 is_pse(vcpu), guest_cpuid_is_amd(vcpu));
Xiao Guangrong6dc98b82015-08-05 12:04:22 +08003715}
3716
Xiao Guangrong81b8eeb2015-08-05 12:04:23 +08003717static void
3718__reset_rsvds_bits_mask_ept(struct rsvd_bits_validate *rsvd_check,
3719 int maxphyaddr, bool execonly)
Yang Zhang25d92082013-08-06 12:00:32 +03003720{
Paolo Bonzini951f9fd2015-09-23 10:34:26 +02003721 u64 bad_mt_xwr;
Yang Zhang25d92082013-08-06 12:00:32 +03003722
Xiao Guangronga0a64f52015-08-05 12:04:21 +08003723 rsvd_check->rsvd_bits_mask[0][3] =
Yang Zhang25d92082013-08-06 12:00:32 +03003724 rsvd_bits(maxphyaddr, 51) | rsvd_bits(3, 7);
Xiao Guangronga0a64f52015-08-05 12:04:21 +08003725 rsvd_check->rsvd_bits_mask[0][2] =
Yang Zhang25d92082013-08-06 12:00:32 +03003726 rsvd_bits(maxphyaddr, 51) | rsvd_bits(3, 6);
Xiao Guangronga0a64f52015-08-05 12:04:21 +08003727 rsvd_check->rsvd_bits_mask[0][1] =
Yang Zhang25d92082013-08-06 12:00:32 +03003728 rsvd_bits(maxphyaddr, 51) | rsvd_bits(3, 6);
Xiao Guangronga0a64f52015-08-05 12:04:21 +08003729 rsvd_check->rsvd_bits_mask[0][0] = rsvd_bits(maxphyaddr, 51);
Yang Zhang25d92082013-08-06 12:00:32 +03003730
3731 /* large page */
Xiao Guangronga0a64f52015-08-05 12:04:21 +08003732 rsvd_check->rsvd_bits_mask[1][3] = rsvd_check->rsvd_bits_mask[0][3];
3733 rsvd_check->rsvd_bits_mask[1][2] =
Yang Zhang25d92082013-08-06 12:00:32 +03003734 rsvd_bits(maxphyaddr, 51) | rsvd_bits(12, 29);
Xiao Guangronga0a64f52015-08-05 12:04:21 +08003735 rsvd_check->rsvd_bits_mask[1][1] =
Yang Zhang25d92082013-08-06 12:00:32 +03003736 rsvd_bits(maxphyaddr, 51) | rsvd_bits(12, 20);
Xiao Guangronga0a64f52015-08-05 12:04:21 +08003737 rsvd_check->rsvd_bits_mask[1][0] = rsvd_check->rsvd_bits_mask[0][0];
Yang Zhang25d92082013-08-06 12:00:32 +03003738
Paolo Bonzini951f9fd2015-09-23 10:34:26 +02003739 bad_mt_xwr = 0xFFull << (2 * 8); /* bits 3..5 must not be 2 */
3740 bad_mt_xwr |= 0xFFull << (3 * 8); /* bits 3..5 must not be 3 */
3741 bad_mt_xwr |= 0xFFull << (7 * 8); /* bits 3..5 must not be 7 */
3742 bad_mt_xwr |= REPEAT_BYTE(1ull << 2); /* bits 0..2 must not be 010 */
3743 bad_mt_xwr |= REPEAT_BYTE(1ull << 6); /* bits 0..2 must not be 110 */
3744 if (!execonly) {
3745 /* bits 0..2 must not be 100 unless VMX capabilities allow it */
3746 bad_mt_xwr |= REPEAT_BYTE(1ull << 4);
Yang Zhang25d92082013-08-06 12:00:32 +03003747 }
Paolo Bonzini951f9fd2015-09-23 10:34:26 +02003748 rsvd_check->bad_mt_xwr = bad_mt_xwr;
Yang Zhang25d92082013-08-06 12:00:32 +03003749}
3750
Xiao Guangrong81b8eeb2015-08-05 12:04:23 +08003751static void reset_rsvds_bits_mask_ept(struct kvm_vcpu *vcpu,
3752 struct kvm_mmu *context, bool execonly)
3753{
3754 __reset_rsvds_bits_mask_ept(&context->guest_rsvd_check,
3755 cpuid_maxphyaddr(vcpu), execonly);
3756}
3757
Xiao Guangrongc258b622015-08-05 12:04:24 +08003758/*
3759 * the page table on host is the shadow page table for the page
3760 * table in guest or amd nested guest, its mmu features completely
3761 * follow the features in guest.
3762 */
3763void
3764reset_shadow_zero_bits_mask(struct kvm_vcpu *vcpu, struct kvm_mmu *context)
3765{
Paolo Bonzini6fec2142015-09-22 23:02:14 +02003766 /*
3767 * Passing "true" to the last argument is okay; it adds a check
3768 * on bit 8 of the SPTEs which KVM doesn't use anyway.
3769 */
Xiao Guangrongc258b622015-08-05 12:04:24 +08003770 __reset_rsvds_bits_mask(vcpu, &context->shadow_zero_check,
3771 boot_cpu_data.x86_phys_bits,
3772 context->shadow_root_level, context->nx,
Paolo Bonzini6fec2142015-09-22 23:02:14 +02003773 guest_cpuid_has_gbpages(vcpu), is_pse(vcpu),
3774 true);
Xiao Guangrongc258b622015-08-05 12:04:24 +08003775}
3776EXPORT_SYMBOL_GPL(reset_shadow_zero_bits_mask);
3777
Paolo Bonzini6fec2142015-09-22 23:02:14 +02003778static inline bool boot_cpu_is_amd(void)
3779{
3780 WARN_ON_ONCE(!tdp_enabled);
3781 return shadow_x_mask == 0;
3782}
3783
Xiao Guangrongc258b622015-08-05 12:04:24 +08003784/*
3785 * the direct page table on host, use as much mmu features as
3786 * possible, however, kvm currently does not do execution-protection.
3787 */
3788static void
3789reset_tdp_shadow_zero_bits_mask(struct kvm_vcpu *vcpu,
3790 struct kvm_mmu *context)
3791{
Paolo Bonzini6fec2142015-09-22 23:02:14 +02003792 if (boot_cpu_is_amd())
Xiao Guangrongc258b622015-08-05 12:04:24 +08003793 __reset_rsvds_bits_mask(vcpu, &context->shadow_zero_check,
3794 boot_cpu_data.x86_phys_bits,
3795 context->shadow_root_level, false,
Paolo Bonzini6fec2142015-09-22 23:02:14 +02003796 cpu_has_gbpages, true, true);
Xiao Guangrongc258b622015-08-05 12:04:24 +08003797 else
3798 __reset_rsvds_bits_mask_ept(&context->shadow_zero_check,
3799 boot_cpu_data.x86_phys_bits,
3800 false);
3801
3802}
3803
3804/*
3805 * as the comments in reset_shadow_zero_bits_mask() except it
3806 * is the shadow page table for intel nested guest.
3807 */
3808static void
3809reset_ept_shadow_zero_bits_mask(struct kvm_vcpu *vcpu,
3810 struct kvm_mmu *context, bool execonly)
3811{
3812 __reset_rsvds_bits_mask_ept(&context->shadow_zero_check,
3813 boot_cpu_data.x86_phys_bits, execonly);
3814}
3815
Xiao Guangrongedc90b72015-05-11 22:55:21 +08003816static void update_permission_bitmask(struct kvm_vcpu *vcpu,
3817 struct kvm_mmu *mmu, bool ept)
Avi Kivity97d64b72012-09-12 14:52:00 +03003818{
3819 unsigned bit, byte, pfec;
3820 u8 map;
Feng Wu66386ade2014-04-01 17:56:48 +08003821 bool fault, x, w, u, wf, uf, ff, smapf, cr4_smap, cr4_smep, smap = 0;
Avi Kivity97d64b72012-09-12 14:52:00 +03003822
Feng Wu66386ade2014-04-01 17:56:48 +08003823 cr4_smep = kvm_read_cr4_bits(vcpu, X86_CR4_SMEP);
Feng Wu97ec8c02014-04-01 17:46:34 +08003824 cr4_smap = kvm_read_cr4_bits(vcpu, X86_CR4_SMAP);
Avi Kivity97d64b72012-09-12 14:52:00 +03003825 for (byte = 0; byte < ARRAY_SIZE(mmu->permissions); ++byte) {
3826 pfec = byte << 1;
3827 map = 0;
3828 wf = pfec & PFERR_WRITE_MASK;
3829 uf = pfec & PFERR_USER_MASK;
3830 ff = pfec & PFERR_FETCH_MASK;
Feng Wu97ec8c02014-04-01 17:46:34 +08003831 /*
3832 * PFERR_RSVD_MASK bit is set in PFEC if the access is not
3833 * subject to SMAP restrictions, and cleared otherwise. The
3834 * bit is only meaningful if the SMAP bit is set in CR4.
3835 */
3836 smapf = !(pfec & PFERR_RSVD_MASK);
Avi Kivity97d64b72012-09-12 14:52:00 +03003837 for (bit = 0; bit < 8; ++bit) {
3838 x = bit & ACC_EXEC_MASK;
3839 w = bit & ACC_WRITE_MASK;
3840 u = bit & ACC_USER_MASK;
3841
Yang Zhang25d92082013-08-06 12:00:32 +03003842 if (!ept) {
3843 /* Not really needed: !nx will cause pte.nx to fault */
3844 x |= !mmu->nx;
3845 /* Allow supervisor writes if !cr0.wp */
3846 w |= !is_write_protection(vcpu) && !uf;
3847 /* Disallow supervisor fetches of user code if cr4.smep */
Feng Wu66386ade2014-04-01 17:56:48 +08003848 x &= !(cr4_smep && u && !uf);
Feng Wu97ec8c02014-04-01 17:46:34 +08003849
3850 /*
3851 * SMAP:kernel-mode data accesses from user-mode
3852 * mappings should fault. A fault is considered
3853 * as a SMAP violation if all of the following
3854 * conditions are ture:
3855 * - X86_CR4_SMAP is set in CR4
3856 * - An user page is accessed
3857 * - Page fault in kernel mode
3858 * - if CPL = 3 or X86_EFLAGS_AC is clear
3859 *
3860 * Here, we cover the first three conditions.
3861 * The fourth is computed dynamically in
3862 * permission_fault() and is in smapf.
3863 *
3864 * Also, SMAP does not affect instruction
3865 * fetches, add the !ff check here to make it
3866 * clearer.
3867 */
3868 smap = cr4_smap && u && !uf && !ff;
Yang Zhang25d92082013-08-06 12:00:32 +03003869 } else
3870 /* Not really needed: no U/S accesses on ept */
3871 u = 1;
Avi Kivity97d64b72012-09-12 14:52:00 +03003872
Feng Wu97ec8c02014-04-01 17:46:34 +08003873 fault = (ff && !x) || (uf && !u) || (wf && !w) ||
3874 (smapf && smap);
Avi Kivity97d64b72012-09-12 14:52:00 +03003875 map |= fault << bit;
3876 }
3877 mmu->permissions[byte] = map;
3878 }
3879}
3880
Avi Kivity6fd01b72012-09-12 20:46:56 +03003881static void update_last_pte_bitmap(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu)
3882{
3883 u8 map;
3884 unsigned level, root_level = mmu->root_level;
3885 const unsigned ps_set_index = 1 << 2; /* bit 2 of index: ps */
3886
3887 if (root_level == PT32E_ROOT_LEVEL)
3888 --root_level;
3889 /* PT_PAGE_TABLE_LEVEL always terminates */
3890 map = 1 | (1 << ps_set_index);
3891 for (level = PT_DIRECTORY_LEVEL; level <= root_level; ++level) {
3892 if (level <= PT_PDPE_LEVEL
3893 && (mmu->root_level >= PT32E_ROOT_LEVEL || is_pse(vcpu)))
3894 map |= 1 << (ps_set_index | (level - 1));
3895 }
3896 mmu->last_pte_bitmap = map;
3897}
3898
Paolo Bonzini8a3c1a332013-10-02 16:56:13 +02003899static void paging64_init_context_common(struct kvm_vcpu *vcpu,
3900 struct kvm_mmu *context,
3901 int level)
Avi Kivity6aa8b732006-12-10 02:21:36 -08003902{
Joerg Roedel2d48a982010-09-10 17:31:01 +02003903 context->nx = is_nx(vcpu);
Davidlohr Bueso4d6931c2012-03-05 16:53:06 +01003904 context->root_level = level;
Joerg Roedel2d48a982010-09-10 17:31:01 +02003905
Davidlohr Bueso4d6931c2012-03-05 16:53:06 +01003906 reset_rsvds_bits_mask(vcpu, context);
Yang Zhang25d92082013-08-06 12:00:32 +03003907 update_permission_bitmask(vcpu, context, false);
Avi Kivity6fd01b72012-09-12 20:46:56 +03003908 update_last_pte_bitmap(vcpu, context);
Avi Kivity6aa8b732006-12-10 02:21:36 -08003909
Paolo Bonzinifa4a2c02013-10-02 16:56:16 +02003910 MMU_WARN_ON(!is_pae(vcpu));
Avi Kivity6aa8b732006-12-10 02:21:36 -08003911 context->page_fault = paging64_page_fault;
Avi Kivity6aa8b732006-12-10 02:21:36 -08003912 context->gva_to_gpa = paging64_gva_to_gpa;
Marcelo Tosattie8bc2172008-09-23 13:18:33 -03003913 context->sync_page = paging64_sync_page;
Marcelo Tosattia7052892008-09-23 13:18:35 -03003914 context->invlpg = paging64_invlpg;
Xiao Guangrong0f53b5b2011-03-09 15:43:51 +08003915 context->update_pte = paging64_update_pte;
Avi Kivity17ac10a2007-01-05 16:36:40 -08003916 context->shadow_root_level = level;
Avi Kivity17c3ba92007-06-04 15:58:30 +03003917 context->root_hpa = INVALID_PAGE;
Joerg Roedelc5a78f2b2010-09-10 17:30:39 +02003918 context->direct_map = false;
Avi Kivity6aa8b732006-12-10 02:21:36 -08003919}
3920
Paolo Bonzini8a3c1a332013-10-02 16:56:13 +02003921static void paging64_init_context(struct kvm_vcpu *vcpu,
3922 struct kvm_mmu *context)
Avi Kivity17ac10a2007-01-05 16:36:40 -08003923{
Paolo Bonzini8a3c1a332013-10-02 16:56:13 +02003924 paging64_init_context_common(vcpu, context, PT64_ROOT_LEVEL);
Avi Kivity17ac10a2007-01-05 16:36:40 -08003925}
3926
Paolo Bonzini8a3c1a332013-10-02 16:56:13 +02003927static void paging32_init_context(struct kvm_vcpu *vcpu,
3928 struct kvm_mmu *context)
Avi Kivity6aa8b732006-12-10 02:21:36 -08003929{
Joerg Roedel2d48a982010-09-10 17:31:01 +02003930 context->nx = false;
Davidlohr Bueso4d6931c2012-03-05 16:53:06 +01003931 context->root_level = PT32_ROOT_LEVEL;
Joerg Roedel2d48a982010-09-10 17:31:01 +02003932
Davidlohr Bueso4d6931c2012-03-05 16:53:06 +01003933 reset_rsvds_bits_mask(vcpu, context);
Yang Zhang25d92082013-08-06 12:00:32 +03003934 update_permission_bitmask(vcpu, context, false);
Avi Kivity6fd01b72012-09-12 20:46:56 +03003935 update_last_pte_bitmap(vcpu, context);
Avi Kivity6aa8b732006-12-10 02:21:36 -08003936
Avi Kivity6aa8b732006-12-10 02:21:36 -08003937 context->page_fault = paging32_page_fault;
Avi Kivity6aa8b732006-12-10 02:21:36 -08003938 context->gva_to_gpa = paging32_gva_to_gpa;
Marcelo Tosattie8bc2172008-09-23 13:18:33 -03003939 context->sync_page = paging32_sync_page;
Marcelo Tosattia7052892008-09-23 13:18:35 -03003940 context->invlpg = paging32_invlpg;
Xiao Guangrong0f53b5b2011-03-09 15:43:51 +08003941 context->update_pte = paging32_update_pte;
Avi Kivity6aa8b732006-12-10 02:21:36 -08003942 context->shadow_root_level = PT32E_ROOT_LEVEL;
Avi Kivity17c3ba92007-06-04 15:58:30 +03003943 context->root_hpa = INVALID_PAGE;
Joerg Roedelc5a78f2b2010-09-10 17:30:39 +02003944 context->direct_map = false;
Avi Kivity6aa8b732006-12-10 02:21:36 -08003945}
3946
Paolo Bonzini8a3c1a332013-10-02 16:56:13 +02003947static void paging32E_init_context(struct kvm_vcpu *vcpu,
3948 struct kvm_mmu *context)
Avi Kivity6aa8b732006-12-10 02:21:36 -08003949{
Paolo Bonzini8a3c1a332013-10-02 16:56:13 +02003950 paging64_init_context_common(vcpu, context, PT32E_ROOT_LEVEL);
Avi Kivity6aa8b732006-12-10 02:21:36 -08003951}
3952
Paolo Bonzini8a3c1a332013-10-02 16:56:13 +02003953static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu)
Joerg Roedelfb72d162008-02-07 13:47:44 +01003954{
Paolo Bonziniad896af2013-10-02 16:56:14 +02003955 struct kvm_mmu *context = &vcpu->arch.mmu;
Joerg Roedelfb72d162008-02-07 13:47:44 +01003956
Avi Kivityc445f8e2010-12-21 16:26:01 +02003957 context->base_role.word = 0;
Paolo Bonzini699023e2015-05-18 15:03:39 +02003958 context->base_role.smm = is_smm(vcpu);
Joerg Roedelfb72d162008-02-07 13:47:44 +01003959 context->page_fault = tdp_page_fault;
Marcelo Tosattie8bc2172008-09-23 13:18:33 -03003960 context->sync_page = nonpaging_sync_page;
Marcelo Tosattia7052892008-09-23 13:18:35 -03003961 context->invlpg = nonpaging_invlpg;
Xiao Guangrong0f53b5b2011-03-09 15:43:51 +08003962 context->update_pte = nonpaging_update_pte;
Sheng Yang67253af2008-04-25 10:20:22 +08003963 context->shadow_root_level = kvm_x86_ops->get_tdp_level();
Joerg Roedelfb72d162008-02-07 13:47:44 +01003964 context->root_hpa = INVALID_PAGE;
Joerg Roedelc5a78f2b2010-09-10 17:30:39 +02003965 context->direct_map = true;
Joerg Roedel1c97f0a2010-09-10 17:30:41 +02003966 context->set_cr3 = kvm_x86_ops->set_tdp_cr3;
Joerg Roedel5777ed32010-09-10 17:30:42 +02003967 context->get_cr3 = get_cr3;
Avi Kivitye4e517b2011-07-28 11:36:17 +03003968 context->get_pdptr = kvm_pdptr_read;
Joerg Roedelcb659db2010-09-10 17:30:43 +02003969 context->inject_page_fault = kvm_inject_page_fault;
Joerg Roedelfb72d162008-02-07 13:47:44 +01003970
3971 if (!is_paging(vcpu)) {
Joerg Roedel2d48a982010-09-10 17:31:01 +02003972 context->nx = false;
Joerg Roedelfb72d162008-02-07 13:47:44 +01003973 context->gva_to_gpa = nonpaging_gva_to_gpa;
3974 context->root_level = 0;
3975 } else if (is_long_mode(vcpu)) {
Joerg Roedel2d48a982010-09-10 17:31:01 +02003976 context->nx = is_nx(vcpu);
Joerg Roedelfb72d162008-02-07 13:47:44 +01003977 context->root_level = PT64_ROOT_LEVEL;
Davidlohr Bueso4d6931c2012-03-05 16:53:06 +01003978 reset_rsvds_bits_mask(vcpu, context);
3979 context->gva_to_gpa = paging64_gva_to_gpa;
Joerg Roedelfb72d162008-02-07 13:47:44 +01003980 } else if (is_pae(vcpu)) {
Joerg Roedel2d48a982010-09-10 17:31:01 +02003981 context->nx = is_nx(vcpu);
Joerg Roedelfb72d162008-02-07 13:47:44 +01003982 context->root_level = PT32E_ROOT_LEVEL;
Davidlohr Bueso4d6931c2012-03-05 16:53:06 +01003983 reset_rsvds_bits_mask(vcpu, context);
3984 context->gva_to_gpa = paging64_gva_to_gpa;
Joerg Roedelfb72d162008-02-07 13:47:44 +01003985 } else {
Joerg Roedel2d48a982010-09-10 17:31:01 +02003986 context->nx = false;
Joerg Roedelfb72d162008-02-07 13:47:44 +01003987 context->root_level = PT32_ROOT_LEVEL;
Davidlohr Bueso4d6931c2012-03-05 16:53:06 +01003988 reset_rsvds_bits_mask(vcpu, context);
3989 context->gva_to_gpa = paging32_gva_to_gpa;
Joerg Roedelfb72d162008-02-07 13:47:44 +01003990 }
3991
Yang Zhang25d92082013-08-06 12:00:32 +03003992 update_permission_bitmask(vcpu, context, false);
Avi Kivity6fd01b72012-09-12 20:46:56 +03003993 update_last_pte_bitmap(vcpu, context);
Xiao Guangrongc258b622015-08-05 12:04:24 +08003994 reset_tdp_shadow_zero_bits_mask(vcpu, context);
Joerg Roedelfb72d162008-02-07 13:47:44 +01003995}
3996
Paolo Bonziniad896af2013-10-02 16:56:14 +02003997void kvm_init_shadow_mmu(struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -08003998{
Avi Kivity411c5882011-06-06 16:11:54 +03003999 bool smep = kvm_read_cr4_bits(vcpu, X86_CR4_SMEP);
Xiao Guangrongedc90b72015-05-11 22:55:21 +08004000 bool smap = kvm_read_cr4_bits(vcpu, X86_CR4_SMAP);
Paolo Bonziniad896af2013-10-02 16:56:14 +02004001 struct kvm_mmu *context = &vcpu->arch.mmu;
4002
Paolo Bonzinifa4a2c02013-10-02 16:56:16 +02004003 MMU_WARN_ON(VALID_PAGE(context->root_hpa));
Avi Kivity6aa8b732006-12-10 02:21:36 -08004004
4005 if (!is_paging(vcpu))
Paolo Bonzini8a3c1a332013-10-02 16:56:13 +02004006 nonpaging_init_context(vcpu, context);
Avi Kivitya9058ec2006-12-29 16:49:37 -08004007 else if (is_long_mode(vcpu))
Paolo Bonzini8a3c1a332013-10-02 16:56:13 +02004008 paging64_init_context(vcpu, context);
Avi Kivity6aa8b732006-12-10 02:21:36 -08004009 else if (is_pae(vcpu))
Paolo Bonzini8a3c1a332013-10-02 16:56:13 +02004010 paging32E_init_context(vcpu, context);
Avi Kivity6aa8b732006-12-10 02:21:36 -08004011 else
Paolo Bonzini8a3c1a332013-10-02 16:56:13 +02004012 paging32_init_context(vcpu, context);
Avi Kivitya770f6f2008-12-21 19:20:09 +02004013
Paolo Bonziniad896af2013-10-02 16:56:14 +02004014 context->base_role.nxe = is_nx(vcpu);
4015 context->base_role.cr4_pae = !!is_pae(vcpu);
4016 context->base_role.cr0_wp = is_write_protection(vcpu);
4017 context->base_role.smep_andnot_wp
Avi Kivity411c5882011-06-06 16:11:54 +03004018 = smep && !is_write_protection(vcpu);
Xiao Guangrongedc90b72015-05-11 22:55:21 +08004019 context->base_role.smap_andnot_wp
4020 = smap && !is_write_protection(vcpu);
Paolo Bonzini699023e2015-05-18 15:03:39 +02004021 context->base_role.smm = is_smm(vcpu);
Xiao Guangrongc258b622015-08-05 12:04:24 +08004022 reset_shadow_zero_bits_mask(vcpu, context);
Joerg Roedel52fde8d2010-09-10 17:30:44 +02004023}
4024EXPORT_SYMBOL_GPL(kvm_init_shadow_mmu);
4025
Paolo Bonziniad896af2013-10-02 16:56:14 +02004026void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly)
Nadav Har'El155a97a2013-08-05 11:07:16 +03004027{
Paolo Bonziniad896af2013-10-02 16:56:14 +02004028 struct kvm_mmu *context = &vcpu->arch.mmu;
4029
Paolo Bonzinifa4a2c02013-10-02 16:56:16 +02004030 MMU_WARN_ON(VALID_PAGE(context->root_hpa));
Nadav Har'El155a97a2013-08-05 11:07:16 +03004031
4032 context->shadow_root_level = kvm_x86_ops->get_tdp_level();
4033
4034 context->nx = true;
Nadav Har'El155a97a2013-08-05 11:07:16 +03004035 context->page_fault = ept_page_fault;
4036 context->gva_to_gpa = ept_gva_to_gpa;
4037 context->sync_page = ept_sync_page;
4038 context->invlpg = ept_invlpg;
4039 context->update_pte = ept_update_pte;
Nadav Har'El155a97a2013-08-05 11:07:16 +03004040 context->root_level = context->shadow_root_level;
4041 context->root_hpa = INVALID_PAGE;
4042 context->direct_map = false;
4043
4044 update_permission_bitmask(vcpu, context, true);
4045 reset_rsvds_bits_mask_ept(vcpu, context, execonly);
Xiao Guangrongc258b622015-08-05 12:04:24 +08004046 reset_ept_shadow_zero_bits_mask(vcpu, context, execonly);
Nadav Har'El155a97a2013-08-05 11:07:16 +03004047}
4048EXPORT_SYMBOL_GPL(kvm_init_shadow_ept_mmu);
4049
Paolo Bonzini8a3c1a332013-10-02 16:56:13 +02004050static void init_kvm_softmmu(struct kvm_vcpu *vcpu)
Joerg Roedel52fde8d2010-09-10 17:30:44 +02004051{
Paolo Bonziniad896af2013-10-02 16:56:14 +02004052 struct kvm_mmu *context = &vcpu->arch.mmu;
4053
4054 kvm_init_shadow_mmu(vcpu);
4055 context->set_cr3 = kvm_x86_ops->set_cr3;
4056 context->get_cr3 = get_cr3;
4057 context->get_pdptr = kvm_pdptr_read;
4058 context->inject_page_fault = kvm_inject_page_fault;
Avi Kivity6aa8b732006-12-10 02:21:36 -08004059}
4060
Paolo Bonzini8a3c1a332013-10-02 16:56:13 +02004061static void init_kvm_nested_mmu(struct kvm_vcpu *vcpu)
Joerg Roedel02f59dc2010-09-10 17:30:54 +02004062{
4063 struct kvm_mmu *g_context = &vcpu->arch.nested_mmu;
4064
4065 g_context->get_cr3 = get_cr3;
Avi Kivitye4e517b2011-07-28 11:36:17 +03004066 g_context->get_pdptr = kvm_pdptr_read;
Joerg Roedel02f59dc2010-09-10 17:30:54 +02004067 g_context->inject_page_fault = kvm_inject_page_fault;
4068
4069 /*
4070 * Note that arch.mmu.gva_to_gpa translates l2_gva to l1_gpa. The
4071 * translation of l2_gpa to l1_gpa addresses is done using the
4072 * arch.nested_mmu.gva_to_gpa function. Basically the gva_to_gpa
4073 * functions between mmu and nested_mmu are swapped.
4074 */
4075 if (!is_paging(vcpu)) {
Joerg Roedel2d48a982010-09-10 17:31:01 +02004076 g_context->nx = false;
Joerg Roedel02f59dc2010-09-10 17:30:54 +02004077 g_context->root_level = 0;
4078 g_context->gva_to_gpa = nonpaging_gva_to_gpa_nested;
4079 } else if (is_long_mode(vcpu)) {
Joerg Roedel2d48a982010-09-10 17:31:01 +02004080 g_context->nx = is_nx(vcpu);
Joerg Roedel02f59dc2010-09-10 17:30:54 +02004081 g_context->root_level = PT64_ROOT_LEVEL;
Davidlohr Bueso4d6931c2012-03-05 16:53:06 +01004082 reset_rsvds_bits_mask(vcpu, g_context);
Joerg Roedel02f59dc2010-09-10 17:30:54 +02004083 g_context->gva_to_gpa = paging64_gva_to_gpa_nested;
4084 } else if (is_pae(vcpu)) {
Joerg Roedel2d48a982010-09-10 17:31:01 +02004085 g_context->nx = is_nx(vcpu);
Joerg Roedel02f59dc2010-09-10 17:30:54 +02004086 g_context->root_level = PT32E_ROOT_LEVEL;
Davidlohr Bueso4d6931c2012-03-05 16:53:06 +01004087 reset_rsvds_bits_mask(vcpu, g_context);
Joerg Roedel02f59dc2010-09-10 17:30:54 +02004088 g_context->gva_to_gpa = paging64_gva_to_gpa_nested;
4089 } else {
Joerg Roedel2d48a982010-09-10 17:31:01 +02004090 g_context->nx = false;
Joerg Roedel02f59dc2010-09-10 17:30:54 +02004091 g_context->root_level = PT32_ROOT_LEVEL;
Davidlohr Bueso4d6931c2012-03-05 16:53:06 +01004092 reset_rsvds_bits_mask(vcpu, g_context);
Joerg Roedel02f59dc2010-09-10 17:30:54 +02004093 g_context->gva_to_gpa = paging32_gva_to_gpa_nested;
4094 }
4095
Yang Zhang25d92082013-08-06 12:00:32 +03004096 update_permission_bitmask(vcpu, g_context, false);
Avi Kivity6fd01b72012-09-12 20:46:56 +03004097 update_last_pte_bitmap(vcpu, g_context);
Joerg Roedel02f59dc2010-09-10 17:30:54 +02004098}
4099
Paolo Bonzini8a3c1a332013-10-02 16:56:13 +02004100static void init_kvm_mmu(struct kvm_vcpu *vcpu)
Joerg Roedelfb72d162008-02-07 13:47:44 +01004101{
Joerg Roedel02f59dc2010-09-10 17:30:54 +02004102 if (mmu_is_nested(vcpu))
Paolo Bonzinie0c6db32014-12-23 13:39:46 +01004103 init_kvm_nested_mmu(vcpu);
Joerg Roedel02f59dc2010-09-10 17:30:54 +02004104 else if (tdp_enabled)
Paolo Bonzinie0c6db32014-12-23 13:39:46 +01004105 init_kvm_tdp_mmu(vcpu);
Joerg Roedelfb72d162008-02-07 13:47:44 +01004106 else
Paolo Bonzinie0c6db32014-12-23 13:39:46 +01004107 init_kvm_softmmu(vcpu);
Joerg Roedelfb72d162008-02-07 13:47:44 +01004108}
4109
Paolo Bonzini8a3c1a332013-10-02 16:56:13 +02004110void kvm_mmu_reset_context(struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -08004111{
Paolo Bonzini95f93af2013-10-02 16:56:12 +02004112 kvm_mmu_unload(vcpu);
Paolo Bonzini8a3c1a332013-10-02 16:56:13 +02004113 init_kvm_mmu(vcpu);
Avi Kivity17c3ba92007-06-04 15:58:30 +03004114}
Eddie Dong8668a3c2007-10-10 14:26:45 +08004115EXPORT_SYMBOL_GPL(kvm_mmu_reset_context);
Avi Kivity17c3ba92007-06-04 15:58:30 +03004116
4117int kvm_mmu_load(struct kvm_vcpu *vcpu)
4118{
Avi Kivity714b93d2007-01-05 16:36:53 -08004119 int r;
4120
Avi Kivitye2dec932007-01-05 16:36:54 -08004121 r = mmu_topup_memory_caches(vcpu);
Avi Kivity17c3ba92007-06-04 15:58:30 +03004122 if (r)
4123 goto out;
Marcelo Tosatti8986ecc2009-05-12 18:55:45 -03004124 r = mmu_alloc_roots(vcpu);
Takuya Yoshikawae2858b42013-05-09 15:45:12 +09004125 kvm_mmu_sync_roots(vcpu);
Marcelo Tosatti8986ecc2009-05-12 18:55:45 -03004126 if (r)
4127 goto out;
Sheng Yang3662cb12009-07-09 17:00:42 +08004128 /* set_cr3() should ensure TLB has been flushed */
Joerg Roedelf43addd2010-09-10 17:30:40 +02004129 vcpu->arch.mmu.set_cr3(vcpu, vcpu->arch.mmu.root_hpa);
Avi Kivity714b93d2007-01-05 16:36:53 -08004130out:
4131 return r;
Avi Kivity6aa8b732006-12-10 02:21:36 -08004132}
Avi Kivity17c3ba92007-06-04 15:58:30 +03004133EXPORT_SYMBOL_GPL(kvm_mmu_load);
4134
4135void kvm_mmu_unload(struct kvm_vcpu *vcpu)
4136{
4137 mmu_free_roots(vcpu);
Paolo Bonzini95f93af2013-10-02 16:56:12 +02004138 WARN_ON(VALID_PAGE(vcpu->arch.mmu.root_hpa));
Avi Kivity17c3ba92007-06-04 15:58:30 +03004139}
Joerg Roedel4b161842010-09-10 17:31:03 +02004140EXPORT_SYMBOL_GPL(kvm_mmu_unload);
Avi Kivity6aa8b732006-12-10 02:21:36 -08004141
Avi Kivity00284252007-05-01 16:53:31 +03004142static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu,
Xiao Guangrong7c562522011-03-28 10:29:27 +08004143 struct kvm_mmu_page *sp, u64 *spte,
4144 const void *new)
Avi Kivity00284252007-05-01 16:53:31 +03004145{
Marcelo Tosatti30945382008-06-11 20:32:40 -03004146 if (sp->role.level != PT_PAGE_TABLE_LEVEL) {
Joerg Roedel7e4e4052009-07-27 16:30:46 +02004147 ++vcpu->kvm->stat.mmu_pde_zapped;
4148 return;
Marcelo Tosatti30945382008-06-11 20:32:40 -03004149 }
Avi Kivity00284252007-05-01 16:53:31 +03004150
Avi Kivity4cee5762007-11-18 16:37:07 +02004151 ++vcpu->kvm->stat.mmu_pte_updated;
Xiao Guangrong7c562522011-03-28 10:29:27 +08004152 vcpu->arch.mmu.update_pte(vcpu, sp, spte, new);
Avi Kivity00284252007-05-01 16:53:31 +03004153}
4154
Avi Kivity79539ce2007-11-21 02:06:21 +02004155static bool need_remote_flush(u64 old, u64 new)
4156{
4157 if (!is_shadow_present_pte(old))
4158 return false;
4159 if (!is_shadow_present_pte(new))
4160 return true;
4161 if ((old ^ new) & PT64_BASE_ADDR_MASK)
4162 return true;
Gleb Natapov53166222013-08-05 11:07:14 +03004163 old ^= shadow_nx_mask;
4164 new ^= shadow_nx_mask;
Avi Kivity79539ce2007-11-21 02:06:21 +02004165 return (old & ~new & PT64_PERM_MASK) != 0;
4166}
4167
Xiao Guangrong0671a8e2010-06-04 21:56:59 +08004168static void mmu_pte_write_flush_tlb(struct kvm_vcpu *vcpu, bool zap_page,
4169 bool remote_flush, bool local_flush)
Avi Kivity79539ce2007-11-21 02:06:21 +02004170{
Xiao Guangrong0671a8e2010-06-04 21:56:59 +08004171 if (zap_page)
4172 return;
4173
4174 if (remote_flush)
Avi Kivity79539ce2007-11-21 02:06:21 +02004175 kvm_flush_remote_tlbs(vcpu->kvm);
Xiao Guangrong0671a8e2010-06-04 21:56:59 +08004176 else if (local_flush)
Liang Chen77c39132014-09-18 12:38:37 -04004177 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
Avi Kivity79539ce2007-11-21 02:06:21 +02004178}
4179
Xiao Guangrong889e5cb2011-09-22 16:57:23 +08004180static u64 mmu_pte_write_fetch_gpte(struct kvm_vcpu *vcpu, gpa_t *gpa,
4181 const u8 *new, int *bytes)
Avi Kivityda4a00f2007-01-05 16:36:44 -08004182{
Xiao Guangrong889e5cb2011-09-22 16:57:23 +08004183 u64 gentry;
4184 int r;
Avi Kivity72016f32010-03-15 13:59:53 +02004185
Avi Kivity08e850c2010-03-15 13:59:57 +02004186 /*
4187 * Assume that the pte write on a page table of the same type
Xiao Guangrong49b26e22011-03-04 19:00:00 +08004188 * as the current vcpu paging mode since we update the sptes only
4189 * when they have the same mode.
Avi Kivity08e850c2010-03-15 13:59:57 +02004190 */
Xiao Guangrong889e5cb2011-09-22 16:57:23 +08004191 if (is_pae(vcpu) && *bytes == 4) {
Avi Kivity08e850c2010-03-15 13:59:57 +02004192 /* Handle a 32-bit guest writing two halves of a 64-bit gpte */
Xiao Guangrong889e5cb2011-09-22 16:57:23 +08004193 *gpa &= ~(gpa_t)7;
4194 *bytes = 8;
Paolo Bonzini54bf36a2015-04-08 15:39:23 +02004195 r = kvm_vcpu_read_guest(vcpu, *gpa, &gentry, 8);
Avi Kivity08e850c2010-03-15 13:59:57 +02004196 if (r)
4197 gentry = 0;
4198 new = (const u8 *)&gentry;
4199 }
4200
Xiao Guangrong889e5cb2011-09-22 16:57:23 +08004201 switch (*bytes) {
Avi Kivity72016f32010-03-15 13:59:53 +02004202 case 4:
4203 gentry = *(const u32 *)new;
4204 break;
4205 case 8:
4206 gentry = *(const u64 *)new;
4207 break;
4208 default:
4209 gentry = 0;
4210 break;
4211 }
4212
Xiao Guangrong889e5cb2011-09-22 16:57:23 +08004213 return gentry;
4214}
4215
4216/*
4217 * If we're seeing too many writes to a page, it may no longer be a page table,
4218 * or we may be forking, in which case it is better to unmap the page.
4219 */
Xiao Guangronga138fe72011-12-16 18:18:10 +08004220static bool detect_write_flooding(struct kvm_mmu_page *sp)
Xiao Guangrong889e5cb2011-09-22 16:57:23 +08004221{
Xiao Guangronga30f47c2011-09-22 16:58:36 +08004222 /*
4223 * Skip write-flooding detected for the sp whose level is 1, because
4224 * it can become unsync, then the guest page is not write-protected.
4225 */
Davidlohr Buesof71fa312012-04-18 12:24:29 +02004226 if (sp->role.level == PT_PAGE_TABLE_LEVEL)
Xiao Guangronga30f47c2011-09-22 16:58:36 +08004227 return false;
Xiao Guangrong889e5cb2011-09-22 16:57:23 +08004228
Xiao Guangronga30f47c2011-09-22 16:58:36 +08004229 return ++sp->write_flooding_count >= 3;
Xiao Guangrong889e5cb2011-09-22 16:57:23 +08004230}
4231
4232/*
4233 * Misaligned accesses are too much trouble to fix up; also, they usually
4234 * indicate a page is not used as a page table.
4235 */
4236static bool detect_write_misaligned(struct kvm_mmu_page *sp, gpa_t gpa,
4237 int bytes)
4238{
4239 unsigned offset, pte_size, misaligned;
4240
4241 pgprintk("misaligned: gpa %llx bytes %d role %x\n",
4242 gpa, bytes, sp->role.word);
4243
4244 offset = offset_in_page(gpa);
4245 pte_size = sp->role.cr4_pae ? 8 : 4;
Xiao Guangrong5d9ca302011-09-22 16:57:55 +08004246
4247 /*
4248 * Sometimes, the OS only writes the last one bytes to update status
4249 * bits, for example, in linux, andb instruction is used in clear_bit().
4250 */
4251 if (!(offset & (pte_size - 1)) && bytes == 1)
4252 return false;
4253
Xiao Guangrong889e5cb2011-09-22 16:57:23 +08004254 misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1);
4255 misaligned |= bytes < 4;
4256
4257 return misaligned;
4258}
4259
4260static u64 *get_written_sptes(struct kvm_mmu_page *sp, gpa_t gpa, int *nspte)
4261{
4262 unsigned page_offset, quadrant;
4263 u64 *spte;
4264 int level;
4265
4266 page_offset = offset_in_page(gpa);
4267 level = sp->role.level;
4268 *nspte = 1;
4269 if (!sp->role.cr4_pae) {
4270 page_offset <<= 1; /* 32->64 */
4271 /*
4272 * A 32-bit pde maps 4MB while the shadow pdes map
4273 * only 2MB. So we need to double the offset again
4274 * and zap two pdes instead of one.
4275 */
4276 if (level == PT32_ROOT_LEVEL) {
4277 page_offset &= ~7; /* kill rounding error */
4278 page_offset <<= 1;
4279 *nspte = 2;
4280 }
4281 quadrant = page_offset >> PAGE_SHIFT;
4282 page_offset &= ~PAGE_MASK;
4283 if (quadrant != sp->role.quadrant)
4284 return NULL;
4285 }
4286
4287 spte = &sp->spt[page_offset / sizeof(*spte)];
4288 return spte;
4289}
4290
4291void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
4292 const u8 *new, int bytes)
4293{
4294 gfn_t gfn = gpa >> PAGE_SHIFT;
Xiao Guangrong889e5cb2011-09-22 16:57:23 +08004295 struct kvm_mmu_page *sp;
Xiao Guangrong889e5cb2011-09-22 16:57:23 +08004296 LIST_HEAD(invalid_list);
4297 u64 entry, gentry, *spte;
4298 int npte;
Xiao Guangronga30f47c2011-09-22 16:58:36 +08004299 bool remote_flush, local_flush, zap_page;
Andrew Morton41412592015-05-27 11:53:06 +02004300 union kvm_mmu_page_role mask = { };
4301
4302 mask.cr0_wp = 1;
4303 mask.cr4_pae = 1;
4304 mask.nxe = 1;
4305 mask.smep_andnot_wp = 1;
4306 mask.smap_andnot_wp = 1;
Paolo Bonzini699023e2015-05-18 15:03:39 +02004307 mask.smm = 1;
Xiao Guangrong889e5cb2011-09-22 16:57:23 +08004308
4309 /*
4310 * If we don't have indirect shadow pages, it means no page is
4311 * write-protected, so we can exit simply.
4312 */
4313 if (!ACCESS_ONCE(vcpu->kvm->arch.indirect_shadow_pages))
4314 return;
4315
4316 zap_page = remote_flush = local_flush = false;
4317
4318 pgprintk("%s: gpa %llx bytes %d\n", __func__, gpa, bytes);
4319
4320 gentry = mmu_pte_write_fetch_gpte(vcpu, &gpa, new, &bytes);
4321
4322 /*
4323 * No need to care whether allocation memory is successful
4324 * or not since pte prefetch is skiped if it does not have
4325 * enough objects in the cache.
4326 */
4327 mmu_topup_memory_caches(vcpu);
4328
4329 spin_lock(&vcpu->kvm->mmu_lock);
4330 ++vcpu->kvm->stat.mmu_pte_write;
Xiao Guangrong0375f7f2011-11-28 20:41:00 +08004331 kvm_mmu_audit(vcpu, AUDIT_PRE_PTE_WRITE);
Xiao Guangrong889e5cb2011-09-22 16:57:23 +08004332
Sasha Levinb67bfe02013-02-27 17:06:00 -08004333 for_each_gfn_indirect_valid_sp(vcpu->kvm, sp, gfn) {
Xiao Guangronga30f47c2011-09-22 16:58:36 +08004334 if (detect_write_misaligned(sp, gpa, bytes) ||
Xiao Guangronga138fe72011-12-16 18:18:10 +08004335 detect_write_flooding(sp)) {
Xiao Guangrong0671a8e2010-06-04 21:56:59 +08004336 zap_page |= !!kvm_mmu_prepare_zap_page(vcpu->kvm, sp,
Xiao Guangrongf41d3352010-06-04 21:56:11 +08004337 &invalid_list);
Avi Kivity4cee5762007-11-18 16:37:07 +02004338 ++vcpu->kvm->stat.mmu_flooded;
Avi Kivity0e7bc4b2007-01-05 16:36:48 -08004339 continue;
4340 }
Xiao Guangrong889e5cb2011-09-22 16:57:23 +08004341
4342 spte = get_written_sptes(sp, gpa, &npte);
4343 if (!spte)
4344 continue;
4345
Xiao Guangrong0671a8e2010-06-04 21:56:59 +08004346 local_flush = true;
Avi Kivityac1b7142007-03-08 17:13:32 +02004347 while (npte--) {
Avi Kivity79539ce2007-11-21 02:06:21 +02004348 entry = *spte;
Xiao Guangrong38e3b2b2011-05-15 23:27:52 +08004349 mmu_page_zap_pte(vcpu->kvm, sp, spte);
Xiao Guangrongfa1de2b2010-07-16 11:19:51 +08004350 if (gentry &&
4351 !((sp->role.word ^ vcpu->arch.mmu.base_role.word)
Xiao Guangrongf759e2b2011-09-22 16:53:17 +08004352 & mask.word) && rmap_can_add(vcpu))
Xiao Guangrong7c562522011-03-28 10:29:27 +08004353 mmu_pte_write_new_pte(vcpu, sp, spte, &gentry);
Gleb Natapov9bb4f6b2013-01-30 16:45:01 +02004354 if (need_remote_flush(entry, *spte))
Xiao Guangrong0671a8e2010-06-04 21:56:59 +08004355 remote_flush = true;
Avi Kivityac1b7142007-03-08 17:13:32 +02004356 ++spte;
Avi Kivity9b7a0322007-01-05 16:36:45 -08004357 }
Avi Kivity9b7a0322007-01-05 16:36:45 -08004358 }
Xiao Guangrong0671a8e2010-06-04 21:56:59 +08004359 mmu_pte_write_flush_tlb(vcpu, zap_page, remote_flush, local_flush);
Xiao Guangrongd98ba052010-06-04 21:55:29 +08004360 kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
Xiao Guangrong0375f7f2011-11-28 20:41:00 +08004361 kvm_mmu_audit(vcpu, AUDIT_POST_PTE_WRITE);
Marcelo Tosattiaaee2c92007-12-20 19:18:26 -05004362 spin_unlock(&vcpu->kvm->mmu_lock);
Avi Kivityda4a00f2007-01-05 16:36:44 -08004363}
4364
Avi Kivitya4360362007-01-05 16:36:45 -08004365int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva)
4366{
Marcelo Tosatti10589a42007-12-20 19:18:22 -05004367 gpa_t gpa;
4368 int r;
Avi Kivitya4360362007-01-05 16:36:45 -08004369
Joerg Roedelc5a78f2b2010-09-10 17:30:39 +02004370 if (vcpu->arch.mmu.direct_map)
Avi Kivity60f24782009-08-27 13:37:06 +03004371 return 0;
4372
Gleb Natapov1871c602010-02-10 14:21:32 +02004373 gpa = kvm_mmu_gva_to_gpa_read(vcpu, gva, NULL);
Marcelo Tosatti10589a42007-12-20 19:18:22 -05004374
Marcelo Tosatti10589a42007-12-20 19:18:22 -05004375 r = kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT);
Xiao Guangrong1cb3f3a2011-09-22 17:02:48 +08004376
Marcelo Tosatti10589a42007-12-20 19:18:22 -05004377 return r;
Avi Kivitya4360362007-01-05 16:36:45 -08004378}
Avi Kivity577bdc42008-07-19 08:57:05 +03004379EXPORT_SYMBOL_GPL(kvm_mmu_unprotect_page_virt);
Avi Kivitya4360362007-01-05 16:36:45 -08004380
Takuya Yoshikawa81f4f762013-03-21 19:34:27 +09004381static void make_mmu_pages_available(struct kvm_vcpu *vcpu)
Avi Kivityebeace82007-01-05 16:36:47 -08004382{
Xiao Guangrongd98ba052010-06-04 21:55:29 +08004383 LIST_HEAD(invalid_list);
Xiao Guangrong103ad252010-06-04 21:54:38 +08004384
Takuya Yoshikawa81f4f762013-03-21 19:34:27 +09004385 if (likely(kvm_mmu_available_pages(vcpu->kvm) >= KVM_MIN_FREE_MMU_PAGES))
4386 return;
4387
Takuya Yoshikawa5da59602013-03-06 16:06:58 +09004388 while (kvm_mmu_available_pages(vcpu->kvm) < KVM_REFILL_PAGES) {
4389 if (!prepare_zap_oldest_mmu_page(vcpu->kvm, &invalid_list))
4390 break;
Avi Kivityebeace82007-01-05 16:36:47 -08004391
Avi Kivity4cee5762007-11-18 16:37:07 +02004392 ++vcpu->kvm->stat.mmu_recycled;
Avi Kivityebeace82007-01-05 16:36:47 -08004393 }
Xiao Guangrongaa6bd182011-07-12 03:26:40 +08004394 kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
Avi Kivityebeace82007-01-05 16:36:47 -08004395}
Avi Kivityebeace82007-01-05 16:36:47 -08004396
Xiao Guangrong1cb3f3a2011-09-22 17:02:48 +08004397static bool is_mmio_page_fault(struct kvm_vcpu *vcpu, gva_t addr)
4398{
4399 if (vcpu->arch.mmu.direct_map || mmu_is_nested(vcpu))
4400 return vcpu_match_mmio_gpa(vcpu, addr);
4401
4402 return vcpu_match_mmio_gva(vcpu, addr);
4403}
4404
Andre Przywaradc25e892010-12-21 11:12:07 +01004405int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code,
4406 void *insn, int insn_len)
Avi Kivity30677142007-10-28 18:48:59 +02004407{
Xiao Guangrong1cb3f3a2011-09-22 17:02:48 +08004408 int r, emulation_type = EMULTYPE_RETRY;
Avi Kivity30677142007-10-28 18:48:59 +02004409 enum emulation_result er;
4410
Gleb Natapov56028d02010-10-17 18:13:42 +02004411 r = vcpu->arch.mmu.page_fault(vcpu, cr2, error_code, false);
Avi Kivity30677142007-10-28 18:48:59 +02004412 if (r < 0)
4413 goto out;
4414
4415 if (!r) {
4416 r = 1;
4417 goto out;
4418 }
4419
Xiao Guangrong1cb3f3a2011-09-22 17:02:48 +08004420 if (is_mmio_page_fault(vcpu, cr2))
4421 emulation_type = 0;
4422
4423 er = x86_emulate_instruction(vcpu, cr2, emulation_type, insn, insn_len);
Avi Kivity30677142007-10-28 18:48:59 +02004424
4425 switch (er) {
4426 case EMULATE_DONE:
4427 return 1;
Paolo Bonziniac0a48c2013-06-25 18:24:41 +02004428 case EMULATE_USER_EXIT:
Avi Kivity30677142007-10-28 18:48:59 +02004429 ++vcpu->stat.mmio_exits;
Gleb Natapov6d77dbf2010-05-10 11:16:56 +03004430 /* fall through */
Avi Kivity30677142007-10-28 18:48:59 +02004431 case EMULATE_FAIL:
Avi Kivity3f5d18a2009-06-11 15:43:28 +03004432 return 0;
Avi Kivity30677142007-10-28 18:48:59 +02004433 default:
4434 BUG();
4435 }
4436out:
Avi Kivity30677142007-10-28 18:48:59 +02004437 return r;
4438}
4439EXPORT_SYMBOL_GPL(kvm_mmu_page_fault);
4440
Marcelo Tosattia7052892008-09-23 13:18:35 -03004441void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
4442{
Marcelo Tosattia7052892008-09-23 13:18:35 -03004443 vcpu->arch.mmu.invlpg(vcpu, gva);
Liang Chen77c39132014-09-18 12:38:37 -04004444 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
Marcelo Tosattia7052892008-09-23 13:18:35 -03004445 ++vcpu->stat.invlpg;
4446}
4447EXPORT_SYMBOL_GPL(kvm_mmu_invlpg);
4448
Joerg Roedel18552672008-02-07 13:47:41 +01004449void kvm_enable_tdp(void)
4450{
4451 tdp_enabled = true;
4452}
4453EXPORT_SYMBOL_GPL(kvm_enable_tdp);
4454
Joerg Roedel5f4cb662008-07-14 20:36:36 +02004455void kvm_disable_tdp(void)
4456{
4457 tdp_enabled = false;
4458}
4459EXPORT_SYMBOL_GPL(kvm_disable_tdp);
4460
Avi Kivity6aa8b732006-12-10 02:21:36 -08004461static void free_mmu_pages(struct kvm_vcpu *vcpu)
4462{
Zhang Xiantaoad312c72007-12-13 23:50:52 +08004463 free_page((unsigned long)vcpu->arch.mmu.pae_root);
Joerg Roedel81407ca2010-09-10 17:31:00 +02004464 if (vcpu->arch.mmu.lm_root != NULL)
4465 free_page((unsigned long)vcpu->arch.mmu.lm_root);
Avi Kivity6aa8b732006-12-10 02:21:36 -08004466}
4467
4468static int alloc_mmu_pages(struct kvm_vcpu *vcpu)
4469{
Avi Kivity17ac10a2007-01-05 16:36:40 -08004470 struct page *page;
Avi Kivity6aa8b732006-12-10 02:21:36 -08004471 int i;
4472
Avi Kivity17ac10a2007-01-05 16:36:40 -08004473 /*
4474 * When emulating 32-bit mode, cr3 is only 32 bits even on x86_64.
4475 * Therefore we need to allocate shadow page tables in the first
4476 * 4GB of memory, which happens to fit the DMA32 zone.
4477 */
4478 page = alloc_page(GFP_KERNEL | __GFP_DMA32);
4479 if (!page)
Wei Yongjund7fa6ab2010-01-22 16:55:05 +08004480 return -ENOMEM;
4481
Zhang Xiantaoad312c72007-12-13 23:50:52 +08004482 vcpu->arch.mmu.pae_root = page_address(page);
Avi Kivity17ac10a2007-01-05 16:36:40 -08004483 for (i = 0; i < 4; ++i)
Zhang Xiantaoad312c72007-12-13 23:50:52 +08004484 vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
Avi Kivity17ac10a2007-01-05 16:36:40 -08004485
Avi Kivity6aa8b732006-12-10 02:21:36 -08004486 return 0;
Avi Kivity6aa8b732006-12-10 02:21:36 -08004487}
4488
Ingo Molnar8018c272006-12-29 16:50:01 -08004489int kvm_mmu_create(struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -08004490{
Xiao Guangronge459e322011-11-28 20:42:16 +08004491 vcpu->arch.walk_mmu = &vcpu->arch.mmu;
4492 vcpu->arch.mmu.root_hpa = INVALID_PAGE;
4493 vcpu->arch.mmu.translate_gpa = translate_gpa;
4494 vcpu->arch.nested_mmu.translate_gpa = translate_nested_gpa;
Avi Kivity6aa8b732006-12-10 02:21:36 -08004495
Ingo Molnar8018c272006-12-29 16:50:01 -08004496 return alloc_mmu_pages(vcpu);
4497}
Avi Kivity6aa8b732006-12-10 02:21:36 -08004498
Paolo Bonzini8a3c1a332013-10-02 16:56:13 +02004499void kvm_mmu_setup(struct kvm_vcpu *vcpu)
Ingo Molnar8018c272006-12-29 16:50:01 -08004500{
Paolo Bonzinifa4a2c02013-10-02 16:56:16 +02004501 MMU_WARN_ON(VALID_PAGE(vcpu->arch.mmu.root_hpa));
Avi Kivity2c264952006-12-22 01:05:28 -08004502
Paolo Bonzini8a3c1a332013-10-02 16:56:13 +02004503 init_kvm_mmu(vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -08004504}
4505
Xiao Guangrong1bad2b22015-05-13 14:42:23 +08004506/* The return value indicates if tlb flush on all vcpus is needed. */
4507typedef bool (*slot_level_handler) (struct kvm *kvm, unsigned long *rmap);
4508
4509/* The caller should hold mmu-lock before calling this function. */
4510static bool
4511slot_handle_level_range(struct kvm *kvm, struct kvm_memory_slot *memslot,
4512 slot_level_handler fn, int start_level, int end_level,
4513 gfn_t start_gfn, gfn_t end_gfn, bool lock_flush_tlb)
4514{
4515 struct slot_rmap_walk_iterator iterator;
4516 bool flush = false;
4517
4518 for_each_slot_rmap_range(memslot, start_level, end_level, start_gfn,
4519 end_gfn, &iterator) {
4520 if (iterator.rmap)
4521 flush |= fn(kvm, iterator.rmap);
4522
4523 if (need_resched() || spin_needbreak(&kvm->mmu_lock)) {
4524 if (flush && lock_flush_tlb) {
4525 kvm_flush_remote_tlbs(kvm);
4526 flush = false;
4527 }
4528 cond_resched_lock(&kvm->mmu_lock);
4529 }
4530 }
4531
4532 if (flush && lock_flush_tlb) {
4533 kvm_flush_remote_tlbs(kvm);
4534 flush = false;
4535 }
4536
4537 return flush;
4538}
4539
4540static bool
4541slot_handle_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
4542 slot_level_handler fn, int start_level, int end_level,
4543 bool lock_flush_tlb)
4544{
4545 return slot_handle_level_range(kvm, memslot, fn, start_level,
4546 end_level, memslot->base_gfn,
4547 memslot->base_gfn + memslot->npages - 1,
4548 lock_flush_tlb);
4549}
4550
4551static bool
4552slot_handle_all_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
4553 slot_level_handler fn, bool lock_flush_tlb)
4554{
4555 return slot_handle_level(kvm, memslot, fn, PT_PAGE_TABLE_LEVEL,
4556 PT_MAX_HUGEPAGE_LEVEL, lock_flush_tlb);
4557}
4558
4559static bool
4560slot_handle_large_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
4561 slot_level_handler fn, bool lock_flush_tlb)
4562{
4563 return slot_handle_level(kvm, memslot, fn, PT_PAGE_TABLE_LEVEL + 1,
4564 PT_MAX_HUGEPAGE_LEVEL, lock_flush_tlb);
4565}
4566
4567static bool
4568slot_handle_leaf(struct kvm *kvm, struct kvm_memory_slot *memslot,
4569 slot_level_handler fn, bool lock_flush_tlb)
4570{
4571 return slot_handle_level(kvm, memslot, fn, PT_PAGE_TABLE_LEVEL,
4572 PT_PAGE_TABLE_LEVEL, lock_flush_tlb);
4573}
4574
Xiao Guangrongefdfe532015-05-13 14:42:27 +08004575void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end)
4576{
4577 struct kvm_memslots *slots;
4578 struct kvm_memory_slot *memslot;
Paolo Bonzini9da0e4d2015-05-18 13:33:16 +02004579 int i;
Xiao Guangrongefdfe532015-05-13 14:42:27 +08004580
4581 spin_lock(&kvm->mmu_lock);
Paolo Bonzini9da0e4d2015-05-18 13:33:16 +02004582 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
4583 slots = __kvm_memslots(kvm, i);
4584 kvm_for_each_memslot(memslot, slots) {
4585 gfn_t start, end;
Xiao Guangrongefdfe532015-05-13 14:42:27 +08004586
Paolo Bonzini9da0e4d2015-05-18 13:33:16 +02004587 start = max(gfn_start, memslot->base_gfn);
4588 end = min(gfn_end, memslot->base_gfn + memslot->npages);
4589 if (start >= end)
4590 continue;
Xiao Guangrongefdfe532015-05-13 14:42:27 +08004591
Paolo Bonzini9da0e4d2015-05-18 13:33:16 +02004592 slot_handle_level_range(kvm, memslot, kvm_zap_rmapp,
4593 PT_PAGE_TABLE_LEVEL, PT_MAX_HUGEPAGE_LEVEL,
4594 start, end - 1, true);
4595 }
Xiao Guangrongefdfe532015-05-13 14:42:27 +08004596 }
4597
4598 spin_unlock(&kvm->mmu_lock);
4599}
4600
Xiao Guangrongd77aa732015-05-13 14:42:24 +08004601static bool slot_rmap_write_protect(struct kvm *kvm, unsigned long *rmapp)
4602{
4603 return __rmap_write_protect(kvm, rmapp, false);
4604}
4605
Kai Huang1c91cad42015-01-28 10:54:26 +08004606void kvm_mmu_slot_remove_write_access(struct kvm *kvm,
4607 struct kvm_memory_slot *memslot)
Avi Kivity6aa8b732006-12-10 02:21:36 -08004608{
Xiao Guangrongd77aa732015-05-13 14:42:24 +08004609 bool flush;
Avi Kivity6aa8b732006-12-10 02:21:36 -08004610
Takuya Yoshikawa9d1beef2013-01-08 19:46:48 +09004611 spin_lock(&kvm->mmu_lock);
Xiao Guangrongd77aa732015-05-13 14:42:24 +08004612 flush = slot_handle_all_level(kvm, memslot, slot_rmap_write_protect,
4613 false);
Takuya Yoshikawa9d1beef2013-01-08 19:46:48 +09004614 spin_unlock(&kvm->mmu_lock);
Xiao Guangrong198c74f2014-04-17 17:06:16 +08004615
4616 /*
4617 * kvm_mmu_slot_remove_write_access() and kvm_vm_ioctl_get_dirty_log()
4618 * which do tlb flush out of mmu-lock should be serialized by
4619 * kvm->slots_lock otherwise tlb flush would be missed.
4620 */
4621 lockdep_assert_held(&kvm->slots_lock);
4622
4623 /*
4624 * We can flush all the TLBs out of the mmu lock without TLB
4625 * corruption since we just change the spte from writable to
4626 * readonly so that we only need to care the case of changing
4627 * spte from present to present (changing the spte from present
4628 * to nonpresent will flush all the TLBs immediately), in other
4629 * words, the only case we care is mmu_spte_update() where we
4630 * haved checked SPTE_HOST_WRITEABLE | SPTE_MMU_WRITEABLE
4631 * instead of PT_WRITABLE_MASK, that means it does not depend
4632 * on PT_WRITABLE_MASK anymore.
4633 */
Kai Huangd91ffee2015-01-12 15:28:54 +08004634 if (flush)
4635 kvm_flush_remote_tlbs(kvm);
Avi Kivity6aa8b732006-12-10 02:21:36 -08004636}
Avi Kivity37a7d8b2007-01-05 16:36:56 -08004637
Wanpeng Li3ea3b7f2015-04-03 15:40:25 +08004638static bool kvm_mmu_zap_collapsible_spte(struct kvm *kvm,
4639 unsigned long *rmapp)
4640{
4641 u64 *sptep;
4642 struct rmap_iterator iter;
4643 int need_tlb_flush = 0;
4644 pfn_t pfn;
4645 struct kvm_mmu_page *sp;
4646
Xiao Guangrong0d536792015-05-13 14:42:20 +08004647restart:
4648 for_each_rmap_spte(rmapp, &iter, sptep) {
Wanpeng Li3ea3b7f2015-04-03 15:40:25 +08004649 sp = page_header(__pa(sptep));
4650 pfn = spte_to_pfn(*sptep);
4651
4652 /*
Xiao Guangrongdecf6332015-04-14 12:04:10 +08004653 * We cannot do huge page mapping for indirect shadow pages,
4654 * which are found on the last rmap (level = 1) when not using
4655 * tdp; such shadow pages are synced with the page table in
4656 * the guest, and the guest page table is using 4K page size
4657 * mapping if the indirect sp has level = 1.
Wanpeng Li3ea3b7f2015-04-03 15:40:25 +08004658 */
4659 if (sp->role.direct &&
4660 !kvm_is_reserved_pfn(pfn) &&
4661 PageTransCompound(pfn_to_page(pfn))) {
4662 drop_spte(kvm, sptep);
Wanpeng Li3ea3b7f2015-04-03 15:40:25 +08004663 need_tlb_flush = 1;
Xiao Guangrong0d536792015-05-13 14:42:20 +08004664 goto restart;
4665 }
Wanpeng Li3ea3b7f2015-04-03 15:40:25 +08004666 }
4667
4668 return need_tlb_flush;
4669}
4670
4671void kvm_mmu_zap_collapsible_sptes(struct kvm *kvm,
Paolo Bonzinif36f3f22015-05-18 13:20:23 +02004672 const struct kvm_memory_slot *memslot)
Wanpeng Li3ea3b7f2015-04-03 15:40:25 +08004673{
Paolo Bonzinif36f3f22015-05-18 13:20:23 +02004674 /* FIXME: const-ify all uses of struct kvm_memory_slot. */
Wanpeng Li3ea3b7f2015-04-03 15:40:25 +08004675 spin_lock(&kvm->mmu_lock);
Paolo Bonzinif36f3f22015-05-18 13:20:23 +02004676 slot_handle_leaf(kvm, (struct kvm_memory_slot *)memslot,
4677 kvm_mmu_zap_collapsible_spte, true);
Wanpeng Li3ea3b7f2015-04-03 15:40:25 +08004678 spin_unlock(&kvm->mmu_lock);
4679}
4680
Kai Huangf4b4b182015-01-28 10:54:24 +08004681void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm,
4682 struct kvm_memory_slot *memslot)
4683{
Xiao Guangrongd77aa732015-05-13 14:42:24 +08004684 bool flush;
Kai Huangf4b4b182015-01-28 10:54:24 +08004685
4686 spin_lock(&kvm->mmu_lock);
Xiao Guangrongd77aa732015-05-13 14:42:24 +08004687 flush = slot_handle_leaf(kvm, memslot, __rmap_clear_dirty, false);
Kai Huangf4b4b182015-01-28 10:54:24 +08004688 spin_unlock(&kvm->mmu_lock);
4689
4690 lockdep_assert_held(&kvm->slots_lock);
4691
4692 /*
4693 * It's also safe to flush TLBs out of mmu lock here as currently this
4694 * function is only used for dirty logging, in which case flushing TLB
4695 * out of mmu lock also guarantees no dirty pages will be lost in
4696 * dirty_bitmap.
4697 */
4698 if (flush)
4699 kvm_flush_remote_tlbs(kvm);
4700}
4701EXPORT_SYMBOL_GPL(kvm_mmu_slot_leaf_clear_dirty);
4702
4703void kvm_mmu_slot_largepage_remove_write_access(struct kvm *kvm,
4704 struct kvm_memory_slot *memslot)
4705{
Xiao Guangrongd77aa732015-05-13 14:42:24 +08004706 bool flush;
Kai Huangf4b4b182015-01-28 10:54:24 +08004707
4708 spin_lock(&kvm->mmu_lock);
Xiao Guangrongd77aa732015-05-13 14:42:24 +08004709 flush = slot_handle_large_level(kvm, memslot, slot_rmap_write_protect,
4710 false);
Kai Huangf4b4b182015-01-28 10:54:24 +08004711 spin_unlock(&kvm->mmu_lock);
4712
4713 /* see kvm_mmu_slot_remove_write_access */
4714 lockdep_assert_held(&kvm->slots_lock);
4715
4716 if (flush)
4717 kvm_flush_remote_tlbs(kvm);
4718}
4719EXPORT_SYMBOL_GPL(kvm_mmu_slot_largepage_remove_write_access);
4720
4721void kvm_mmu_slot_set_dirty(struct kvm *kvm,
4722 struct kvm_memory_slot *memslot)
4723{
Xiao Guangrongd77aa732015-05-13 14:42:24 +08004724 bool flush;
Kai Huangf4b4b182015-01-28 10:54:24 +08004725
4726 spin_lock(&kvm->mmu_lock);
Xiao Guangrongd77aa732015-05-13 14:42:24 +08004727 flush = slot_handle_all_level(kvm, memslot, __rmap_set_dirty, false);
Kai Huangf4b4b182015-01-28 10:54:24 +08004728 spin_unlock(&kvm->mmu_lock);
4729
4730 lockdep_assert_held(&kvm->slots_lock);
4731
4732 /* see kvm_mmu_slot_leaf_clear_dirty */
4733 if (flush)
4734 kvm_flush_remote_tlbs(kvm);
4735}
4736EXPORT_SYMBOL_GPL(kvm_mmu_slot_set_dirty);
4737
Xiao Guangronge7d11c72013-05-31 08:36:27 +08004738#define BATCH_ZAP_PAGES 10
Xiao Guangrong5304b8d2013-05-31 08:36:22 +08004739static void kvm_zap_obsolete_pages(struct kvm *kvm)
4740{
4741 struct kvm_mmu_page *sp, *node;
Xiao Guangronge7d11c72013-05-31 08:36:27 +08004742 int batch = 0;
Xiao Guangrong5304b8d2013-05-31 08:36:22 +08004743
4744restart:
4745 list_for_each_entry_safe_reverse(sp, node,
4746 &kvm->arch.active_mmu_pages, link) {
Xiao Guangronge7d11c72013-05-31 08:36:27 +08004747 int ret;
4748
Xiao Guangrong5304b8d2013-05-31 08:36:22 +08004749 /*
4750 * No obsolete page exists before new created page since
4751 * active_mmu_pages is the FIFO list.
4752 */
4753 if (!is_obsolete_sp(kvm, sp))
4754 break;
4755
4756 /*
Xiao Guangrong5304b8d2013-05-31 08:36:22 +08004757 * Since we are reversely walking the list and the invalid
4758 * list will be moved to the head, skip the invalid page
4759 * can help us to avoid the infinity list walking.
4760 */
4761 if (sp->role.invalid)
4762 continue;
4763
Xiao Guangrongf34d2512013-05-31 08:36:28 +08004764 /*
4765 * Need not flush tlb since we only zap the sp with invalid
4766 * generation number.
4767 */
Xiao Guangronge7d11c72013-05-31 08:36:27 +08004768 if (batch >= BATCH_ZAP_PAGES &&
Xiao Guangrongf34d2512013-05-31 08:36:28 +08004769 cond_resched_lock(&kvm->mmu_lock)) {
Xiao Guangronge7d11c72013-05-31 08:36:27 +08004770 batch = 0;
Xiao Guangrong5304b8d2013-05-31 08:36:22 +08004771 goto restart;
4772 }
4773
Xiao Guangrong365c8862013-05-31 08:36:29 +08004774 ret = kvm_mmu_prepare_zap_page(kvm, sp,
4775 &kvm->arch.zapped_obsolete_pages);
Xiao Guangronge7d11c72013-05-31 08:36:27 +08004776 batch += ret;
4777
4778 if (ret)
Xiao Guangrong5304b8d2013-05-31 08:36:22 +08004779 goto restart;
4780 }
4781
Xiao Guangrongf34d2512013-05-31 08:36:28 +08004782 /*
4783 * Should flush tlb before free page tables since lockless-walking
4784 * may use the pages.
4785 */
Xiao Guangrong365c8862013-05-31 08:36:29 +08004786 kvm_mmu_commit_zap_page(kvm, &kvm->arch.zapped_obsolete_pages);
Xiao Guangrong5304b8d2013-05-31 08:36:22 +08004787}
4788
4789/*
4790 * Fast invalidate all shadow pages and use lock-break technique
4791 * to zap obsolete pages.
4792 *
4793 * It's required when memslot is being deleted or VM is being
4794 * destroyed, in these cases, we should ensure that KVM MMU does
4795 * not use any resource of the being-deleted slot or all slots
4796 * after calling the function.
4797 */
4798void kvm_mmu_invalidate_zap_all_pages(struct kvm *kvm)
4799{
4800 spin_lock(&kvm->mmu_lock);
Xiao Guangrong35006122013-05-31 08:36:25 +08004801 trace_kvm_mmu_invalidate_zap_all_pages(kvm);
Xiao Guangrong5304b8d2013-05-31 08:36:22 +08004802 kvm->arch.mmu_valid_gen++;
4803
Xiao Guangrongf34d2512013-05-31 08:36:28 +08004804 /*
4805 * Notify all vcpus to reload its shadow page table
4806 * and flush TLB. Then all vcpus will switch to new
4807 * shadow page table with the new mmu_valid_gen.
4808 *
4809 * Note: we should do this under the protection of
4810 * mmu-lock, otherwise, vcpu would purge shadow page
4811 * but miss tlb flush.
4812 */
4813 kvm_reload_remote_mmus(kvm);
4814
Xiao Guangrong5304b8d2013-05-31 08:36:22 +08004815 kvm_zap_obsolete_pages(kvm);
4816 spin_unlock(&kvm->mmu_lock);
4817}
4818
Xiao Guangrong365c8862013-05-31 08:36:29 +08004819static bool kvm_has_zapped_obsolete_pages(struct kvm *kvm)
4820{
4821 return unlikely(!list_empty_careful(&kvm->arch.zapped_obsolete_pages));
4822}
4823
Paolo Bonzini54bf36a2015-04-08 15:39:23 +02004824void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, struct kvm_memslots *slots)
Xiao Guangrongf8f55942013-06-07 16:51:26 +08004825{
4826 /*
4827 * The very rare case: if the generation-number is round,
4828 * zap all shadow pages.
Xiao Guangrongf8f55942013-06-07 16:51:26 +08004829 */
Paolo Bonzini54bf36a2015-04-08 15:39:23 +02004830 if (unlikely((slots->generation & MMIO_GEN_MASK) == 0)) {
Paolo Bonzinia629df72014-12-22 10:43:39 +01004831 printk_ratelimited(KERN_DEBUG "kvm: zapping shadow pages for mmio generation wraparound\n");
Xiao Guangronga8eca9d2013-06-10 16:28:55 +08004832 kvm_mmu_invalidate_zap_all_pages(kvm);
Takuya Yoshikawa7a2e8aa2013-06-21 01:34:31 +09004833 }
Xiao Guangrongf8f55942013-06-07 16:51:26 +08004834}
4835
Dave Chinner70534a72013-08-28 10:18:14 +10004836static unsigned long
4837mmu_shrink_scan(struct shrinker *shrink, struct shrink_control *sc)
Izik Eidus3ee16c82008-03-30 15:17:21 +03004838{
4839 struct kvm *kvm;
Ying Han1495f232011-05-24 17:12:27 -07004840 int nr_to_scan = sc->nr_to_scan;
Dave Chinner70534a72013-08-28 10:18:14 +10004841 unsigned long freed = 0;
Izik Eidus3ee16c82008-03-30 15:17:21 +03004842
Paolo Bonzini2f303b72013-09-25 13:53:07 +02004843 spin_lock(&kvm_lock);
Izik Eidus3ee16c82008-03-30 15:17:21 +03004844
4845 list_for_each_entry(kvm, &vm_list, vm_list) {
Jan Kiszka3d56cbd2011-12-02 18:35:24 +01004846 int idx;
Xiao Guangrongd98ba052010-06-04 21:55:29 +08004847 LIST_HEAD(invalid_list);
Izik Eidus3ee16c82008-03-30 15:17:21 +03004848
Gleb Natapov19526392012-06-04 14:53:23 +03004849 /*
Takuya Yoshikawa35f2d162012-08-20 18:35:39 +09004850 * Never scan more than sc->nr_to_scan VM instances.
4851 * Will not hit this condition practically since we do not try
4852 * to shrink more than one VM and it is very unlikely to see
4853 * !n_used_mmu_pages so many times.
4854 */
4855 if (!nr_to_scan--)
4856 break;
4857 /*
Gleb Natapov19526392012-06-04 14:53:23 +03004858 * n_used_mmu_pages is accessed without holding kvm->mmu_lock
4859 * here. We may skip a VM instance errorneosly, but we do not
4860 * want to shrink a VM that only started to populate its MMU
4861 * anyway.
4862 */
Xiao Guangrong365c8862013-05-31 08:36:29 +08004863 if (!kvm->arch.n_used_mmu_pages &&
4864 !kvm_has_zapped_obsolete_pages(kvm))
Gleb Natapov19526392012-06-04 14:53:23 +03004865 continue;
Gleb Natapov19526392012-06-04 14:53:23 +03004866
Marcelo Tosattif656ce02009-12-23 14:35:25 -02004867 idx = srcu_read_lock(&kvm->srcu);
Izik Eidus3ee16c82008-03-30 15:17:21 +03004868 spin_lock(&kvm->mmu_lock);
Izik Eidus3ee16c82008-03-30 15:17:21 +03004869
Xiao Guangrong365c8862013-05-31 08:36:29 +08004870 if (kvm_has_zapped_obsolete_pages(kvm)) {
4871 kvm_mmu_commit_zap_page(kvm,
4872 &kvm->arch.zapped_obsolete_pages);
4873 goto unlock;
4874 }
4875
Dave Chinner70534a72013-08-28 10:18:14 +10004876 if (prepare_zap_oldest_mmu_page(kvm, &invalid_list))
4877 freed++;
Xiao Guangrongd98ba052010-06-04 21:55:29 +08004878 kvm_mmu_commit_zap_page(kvm, &invalid_list);
Gleb Natapov19526392012-06-04 14:53:23 +03004879
Xiao Guangrong365c8862013-05-31 08:36:29 +08004880unlock:
Izik Eidus3ee16c82008-03-30 15:17:21 +03004881 spin_unlock(&kvm->mmu_lock);
Marcelo Tosattif656ce02009-12-23 14:35:25 -02004882 srcu_read_unlock(&kvm->srcu, idx);
Gleb Natapov19526392012-06-04 14:53:23 +03004883
Dave Chinner70534a72013-08-28 10:18:14 +10004884 /*
4885 * unfair on small ones
4886 * per-vm shrinkers cry out
4887 * sadness comes quickly
4888 */
Gleb Natapov19526392012-06-04 14:53:23 +03004889 list_move_tail(&kvm->vm_list, &vm_list);
4890 break;
Izik Eidus3ee16c82008-03-30 15:17:21 +03004891 }
Izik Eidus3ee16c82008-03-30 15:17:21 +03004892
Paolo Bonzini2f303b72013-09-25 13:53:07 +02004893 spin_unlock(&kvm_lock);
Dave Chinner70534a72013-08-28 10:18:14 +10004894 return freed;
Dave Chinner70534a72013-08-28 10:18:14 +10004895}
4896
4897static unsigned long
4898mmu_shrink_count(struct shrinker *shrink, struct shrink_control *sc)
4899{
Dave Hansen45221ab2010-08-19 18:11:37 -07004900 return percpu_counter_read_positive(&kvm_total_used_mmu_pages);
Izik Eidus3ee16c82008-03-30 15:17:21 +03004901}
4902
4903static struct shrinker mmu_shrinker = {
Dave Chinner70534a72013-08-28 10:18:14 +10004904 .count_objects = mmu_shrink_count,
4905 .scan_objects = mmu_shrink_scan,
Izik Eidus3ee16c82008-03-30 15:17:21 +03004906 .seeks = DEFAULT_SEEKS * 10,
4907};
4908
Ingo Molnar2ddfd202008-05-22 10:37:48 +02004909static void mmu_destroy_caches(void)
Avi Kivityb5a33a72007-04-15 16:31:09 +03004910{
Xiao Guangrong53c07b12011-05-15 23:26:20 +08004911 if (pte_list_desc_cache)
4912 kmem_cache_destroy(pte_list_desc_cache);
Avi Kivityd3d25b02007-05-30 12:34:53 +03004913 if (mmu_page_header_cache)
4914 kmem_cache_destroy(mmu_page_header_cache);
Avi Kivityb5a33a72007-04-15 16:31:09 +03004915}
4916
4917int kvm_mmu_module_init(void)
4918{
Xiao Guangrong53c07b12011-05-15 23:26:20 +08004919 pte_list_desc_cache = kmem_cache_create("pte_list_desc",
4920 sizeof(struct pte_list_desc),
Paul Mundt20c2df82007-07-20 10:11:58 +09004921 0, 0, NULL);
Xiao Guangrong53c07b12011-05-15 23:26:20 +08004922 if (!pte_list_desc_cache)
Avi Kivityb5a33a72007-04-15 16:31:09 +03004923 goto nomem;
4924
Avi Kivityd3d25b02007-05-30 12:34:53 +03004925 mmu_page_header_cache = kmem_cache_create("kvm_mmu_page_header",
4926 sizeof(struct kvm_mmu_page),
Paul Mundt20c2df82007-07-20 10:11:58 +09004927 0, 0, NULL);
Avi Kivityd3d25b02007-05-30 12:34:53 +03004928 if (!mmu_page_header_cache)
4929 goto nomem;
4930
Tejun Heo908c7f12014-09-08 09:51:29 +09004931 if (percpu_counter_init(&kvm_total_used_mmu_pages, 0, GFP_KERNEL))
Wei Yongjun45bf21a2010-08-23 16:13:15 +08004932 goto nomem;
4933
Izik Eidus3ee16c82008-03-30 15:17:21 +03004934 register_shrinker(&mmu_shrinker);
4935
Avi Kivityb5a33a72007-04-15 16:31:09 +03004936 return 0;
4937
4938nomem:
Izik Eidus3ee16c82008-03-30 15:17:21 +03004939 mmu_destroy_caches();
Avi Kivityb5a33a72007-04-15 16:31:09 +03004940 return -ENOMEM;
4941}
4942
Zhang Xiantao3ad82a72007-11-20 13:11:38 +08004943/*
4944 * Caculate mmu pages needed for kvm.
4945 */
4946unsigned int kvm_mmu_calculate_mmu_pages(struct kvm *kvm)
4947{
Zhang Xiantao3ad82a72007-11-20 13:11:38 +08004948 unsigned int nr_mmu_pages;
4949 unsigned int nr_pages = 0;
Marcelo Tosattibc6678a2009-12-23 14:35:21 -02004950 struct kvm_memslots *slots;
Xiao Guangrongbe6ba0f2011-11-24 17:39:18 +08004951 struct kvm_memory_slot *memslot;
Paolo Bonzini9da0e4d2015-05-18 13:33:16 +02004952 int i;
Zhang Xiantao3ad82a72007-11-20 13:11:38 +08004953
Paolo Bonzini9da0e4d2015-05-18 13:33:16 +02004954 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
4955 slots = __kvm_memslots(kvm, i);
Lai Jiangshan90d83dc2010-04-19 17:41:23 +08004956
Paolo Bonzini9da0e4d2015-05-18 13:33:16 +02004957 kvm_for_each_memslot(memslot, slots)
4958 nr_pages += memslot->npages;
4959 }
Zhang Xiantao3ad82a72007-11-20 13:11:38 +08004960
4961 nr_mmu_pages = nr_pages * KVM_PERMILLE_MMU_PAGES / 1000;
4962 nr_mmu_pages = max(nr_mmu_pages,
Paolo Bonzini9da0e4d2015-05-18 13:33:16 +02004963 (unsigned int) KVM_MIN_ALLOC_MMU_PAGES);
Zhang Xiantao3ad82a72007-11-20 13:11:38 +08004964
4965 return nr_mmu_pages;
4966}
4967
Xiao Guangrongc42fffe2010-09-27 18:07:07 +08004968void kvm_mmu_destroy(struct kvm_vcpu *vcpu)
4969{
Paolo Bonzini95f93af2013-10-02 16:56:12 +02004970 kvm_mmu_unload(vcpu);
Xiao Guangrongc42fffe2010-09-27 18:07:07 +08004971 free_mmu_pages(vcpu);
4972 mmu_free_memory_caches(vcpu);
Xiao Guangrongb034cf02010-12-23 16:08:35 +08004973}
4974
Xiao Guangrongb034cf02010-12-23 16:08:35 +08004975void kvm_mmu_module_exit(void)
4976{
4977 mmu_destroy_caches();
4978 percpu_counter_destroy(&kvm_total_used_mmu_pages);
4979 unregister_shrinker(&mmu_shrinker);
Xiao Guangrongc42fffe2010-09-27 18:07:07 +08004980 mmu_audit_disable();
4981}