blob: 13c3c2dd04594ac639eaae8bcb3fe7e6b6d021f8 [file] [log] [blame]
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001/*
David Woodhouseea8ea462014-03-05 17:09:32 +00002 * Copyright © 2006-2014 Intel Corporation.
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
David Woodhouseea8ea462014-03-05 17:09:32 +000013 * Authors: David Woodhouse <dwmw2@infradead.org>,
14 * Ashok Raj <ashok.raj@intel.com>,
15 * Shaohua Li <shaohua.li@intel.com>,
16 * Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
17 * Fenghua Yu <fenghua.yu@intel.com>
Joerg Roedel9f10e5b2015-06-12 09:57:06 +020018 * Joerg Roedel <jroedel@suse.de>
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -070019 */
20
Joerg Roedel9f10e5b2015-06-12 09:57:06 +020021#define pr_fmt(fmt) "DMAR: " fmt
22
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -070023#include <linux/init.h>
24#include <linux/bitmap.h>
mark gross5e0d2a62008-03-04 15:22:08 -080025#include <linux/debugfs.h>
Paul Gortmaker54485c32011-10-29 10:26:25 -040026#include <linux/export.h>
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -070027#include <linux/slab.h>
28#include <linux/irq.h>
29#include <linux/interrupt.h>
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -070030#include <linux/spinlock.h>
31#include <linux/pci.h>
32#include <linux/dmar.h>
33#include <linux/dma-mapping.h>
34#include <linux/mempool.h>
Jiang Liu75f05562014-02-19 14:07:37 +080035#include <linux/memory.h>
Omer Pelegaa473242016-04-20 11:33:02 +030036#include <linux/cpu.h>
mark gross5e0d2a62008-03-04 15:22:08 -080037#include <linux/timer.h>
Dan Williamsdfddb962015-10-09 18:16:46 -040038#include <linux/io.h>
Kay, Allen M38717942008-09-09 18:37:29 +030039#include <linux/iova.h>
Joerg Roedel5d450802008-12-03 14:52:32 +010040#include <linux/iommu.h>
Kay, Allen M38717942008-09-09 18:37:29 +030041#include <linux/intel-iommu.h>
Rafael J. Wysocki134fac32011-03-23 22:16:14 +010042#include <linux/syscore_ops.h>
Shane Wang69575d32009-09-01 18:25:07 -070043#include <linux/tboot.h>
Stephen Rothwelladb2fe02009-08-31 15:24:23 +100044#include <linux/dmi.h>
Joerg Roedel5cdede22011-04-04 15:55:18 +020045#include <linux/pci-ats.h>
Tejun Heo0ee332c2011-12-08 10:22:09 -080046#include <linux/memblock.h>
Akinobu Mita36746432014-06-04 16:06:51 -070047#include <linux/dma-contiguous.h>
Christoph Hellwigfec777c2018-03-19 11:38:15 +010048#include <linux/dma-direct.h>
Joerg Roedel091d42e2015-06-12 11:56:10 +020049#include <linux/crash_dump.h>
Suresh Siddha8a8f4222012-03-30 11:47:08 -070050#include <asm/irq_remapping.h>
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -070051#include <asm/cacheflush.h>
FUJITA Tomonori46a7fa22008-07-11 10:23:42 +090052#include <asm/iommu.h>
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -070053
Joerg Roedel078e1ee2012-09-26 12:44:43 +020054#include "irq_remapping.h"
Lu Baolu56283172018-07-14 15:46:54 +080055#include "intel-pasid.h"
Joerg Roedel078e1ee2012-09-26 12:44:43 +020056
Fenghua Yu5b6985c2008-10-16 18:02:32 -070057#define ROOT_SIZE VTD_PAGE_SIZE
58#define CONTEXT_SIZE VTD_PAGE_SIZE
59
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -070060#define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
David Woodhouse18436af2015-03-25 15:05:47 +000061#define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -070062#define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
David Woodhousee0fc7e02009-09-30 09:12:17 -070063#define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -070064
65#define IOAPIC_RANGE_START (0xfee00000)
66#define IOAPIC_RANGE_END (0xfeefffff)
67#define IOVA_START_ADDR (0x1000)
68
Sohil Mehta5e3b4a12017-12-20 11:59:24 -080069#define DEFAULT_DOMAIN_ADDRESS_WIDTH 57
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -070070
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -070071#define MAX_AGAW_WIDTH 64
Jiang Liu5c645b32014-01-06 14:18:12 +080072#define MAX_AGAW_PFN_WIDTH (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -070073
David Woodhouse2ebe3152009-09-19 07:34:04 -070074#define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
75#define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
76
77/* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
78 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
79#define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
80 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
81#define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -070082
Robin Murphy1b722502015-01-12 17:51:15 +000083/* IO virtual address start page frame number */
84#define IOVA_START_PFN (1)
85
Mark McLoughlinf27be032008-11-20 15:49:43 +000086#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
mark gross5e0d2a62008-03-04 15:22:08 -080087
Andrew Mortondf08cdc2010-09-22 13:05:11 -070088/* page table handling */
89#define LEVEL_STRIDE (9)
90#define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
91
Ohad Ben-Cohen6d1c56a2011-11-10 11:32:30 +020092/*
93 * This bitmap is used to advertise the page sizes our hardware support
94 * to the IOMMU core, which will then use this information to split
95 * physically contiguous memory regions it is mapping into page sizes
96 * that we support.
97 *
98 * Traditionally the IOMMU core just handed us the mappings directly,
99 * after making sure the size is an order of a 4KiB page and that the
100 * mapping has natural alignment.
101 *
102 * To retain this behavior, we currently advertise that we support
103 * all page sizes that are an order of 4KiB.
104 *
105 * If at some point we'd like to utilize the IOMMU core's new behavior,
106 * we could change this to advertise the real page sizes we support.
107 */
108#define INTEL_IOMMU_PGSIZES (~0xFFFUL)
109
Andrew Mortondf08cdc2010-09-22 13:05:11 -0700110static inline int agaw_to_level(int agaw)
111{
112 return agaw + 2;
113}
114
115static inline int agaw_to_width(int agaw)
116{
Jiang Liu5c645b32014-01-06 14:18:12 +0800117 return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
Andrew Mortondf08cdc2010-09-22 13:05:11 -0700118}
119
120static inline int width_to_agaw(int width)
121{
Jiang Liu5c645b32014-01-06 14:18:12 +0800122 return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
Andrew Mortondf08cdc2010-09-22 13:05:11 -0700123}
124
125static inline unsigned int level_to_offset_bits(int level)
126{
127 return (level - 1) * LEVEL_STRIDE;
128}
129
130static inline int pfn_level_offset(unsigned long pfn, int level)
131{
132 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
133}
134
135static inline unsigned long level_mask(int level)
136{
137 return -1UL << level_to_offset_bits(level);
138}
139
140static inline unsigned long level_size(int level)
141{
142 return 1UL << level_to_offset_bits(level);
143}
144
145static inline unsigned long align_to_level(unsigned long pfn, int level)
146{
147 return (pfn + level_size(level) - 1) & level_mask(level);
148}
David Woodhousefd18de52009-05-10 23:57:41 +0100149
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100150static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
151{
Jiang Liu5c645b32014-01-06 14:18:12 +0800152 return 1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100153}
154
David Woodhousedd4e8312009-06-27 16:21:20 +0100155/* VT-d pages must always be _smaller_ than MM pages. Otherwise things
156 are never going to work. */
157static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
158{
159 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
160}
161
162static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
163{
164 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
165}
166static inline unsigned long page_to_dma_pfn(struct page *pg)
167{
168 return mm_to_dma_pfn(page_to_pfn(pg));
169}
170static inline unsigned long virt_to_dma_pfn(void *p)
171{
172 return page_to_dma_pfn(virt_to_page(p));
173}
174
Weidong Hand9630fe2008-12-08 11:06:32 +0800175/* global iommu list, set NULL for ignored DMAR units */
176static struct intel_iommu **g_iommus;
177
David Woodhousee0fc7e02009-09-30 09:12:17 -0700178static void __init check_tylersburg_isoch(void);
David Woodhouse9af88142009-02-13 23:18:03 +0000179static int rwbf_quirk;
180
Mark McLoughlin46b08e12008-11-20 15:49:44 +0000181/*
Joseph Cihulab7792602011-05-03 00:08:37 -0700182 * set to 1 to panic kernel if can't successfully enable VT-d
183 * (used when kernel is launched w/ TXT)
184 */
185static int force_on = 0;
Shaohua Libfd20f12017-04-26 09:18:35 -0700186int intel_iommu_tboot_noforce;
Joseph Cihulab7792602011-05-03 00:08:37 -0700187
Mark McLoughlin46b08e12008-11-20 15:49:44 +0000188#define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
Mark McLoughlin46b08e12008-11-20 15:49:44 +0000189
Joerg Roedel091d42e2015-06-12 11:56:10 +0200190/*
191 * Take a root_entry and return the Lower Context Table Pointer (LCTP)
192 * if marked present.
193 */
194static phys_addr_t root_entry_lctp(struct root_entry *re)
195{
196 if (!(re->lo & 1))
197 return 0;
Mark McLoughlin46b08e12008-11-20 15:49:44 +0000198
Joerg Roedel091d42e2015-06-12 11:56:10 +0200199 return re->lo & VTD_PAGE_MASK;
200}
201
202/*
203 * Take a root_entry and return the Upper Context Table Pointer (UCTP)
204 * if marked present.
205 */
206static phys_addr_t root_entry_uctp(struct root_entry *re)
207{
208 if (!(re->hi & 1))
209 return 0;
210
211 return re->hi & VTD_PAGE_MASK;
212}
Mark McLoughlin7a8fc252008-11-20 15:49:45 +0000213
Joerg Roedelcf484d02015-06-12 12:21:46 +0200214static inline void context_clear_pasid_enable(struct context_entry *context)
215{
216 context->lo &= ~(1ULL << 11);
217}
218
219static inline bool context_pasid_enabled(struct context_entry *context)
220{
221 return !!(context->lo & (1ULL << 11));
222}
223
224static inline void context_set_copied(struct context_entry *context)
225{
226 context->hi |= (1ull << 3);
227}
228
229static inline bool context_copied(struct context_entry *context)
230{
231 return !!(context->hi & (1ULL << 3));
232}
233
234static inline bool __context_present(struct context_entry *context)
Mark McLoughlinc07e7d22008-11-21 16:54:46 +0000235{
236 return (context->lo & 1);
237}
Joerg Roedelcf484d02015-06-12 12:21:46 +0200238
Sohil Mehta26b86092018-09-11 17:11:36 -0700239bool context_present(struct context_entry *context)
Joerg Roedelcf484d02015-06-12 12:21:46 +0200240{
241 return context_pasid_enabled(context) ?
242 __context_present(context) :
243 __context_present(context) && !context_copied(context);
244}
245
Mark McLoughlinc07e7d22008-11-21 16:54:46 +0000246static inline void context_set_present(struct context_entry *context)
247{
248 context->lo |= 1;
249}
250
251static inline void context_set_fault_enable(struct context_entry *context)
252{
253 context->lo &= (((u64)-1) << 2) | 1;
254}
255
Mark McLoughlinc07e7d22008-11-21 16:54:46 +0000256static inline void context_set_translation_type(struct context_entry *context,
257 unsigned long value)
258{
259 context->lo &= (((u64)-1) << 4) | 3;
260 context->lo |= (value & 3) << 2;
261}
262
263static inline void context_set_address_root(struct context_entry *context,
264 unsigned long value)
265{
Li, Zhen-Hua1a2262f2014-11-05 15:30:19 +0800266 context->lo &= ~VTD_PAGE_MASK;
Mark McLoughlinc07e7d22008-11-21 16:54:46 +0000267 context->lo |= value & VTD_PAGE_MASK;
268}
269
270static inline void context_set_address_width(struct context_entry *context,
271 unsigned long value)
272{
273 context->hi |= value & 7;
274}
275
276static inline void context_set_domain_id(struct context_entry *context,
277 unsigned long value)
278{
279 context->hi |= (value & ((1 << 16) - 1)) << 8;
280}
281
Joerg Roedeldbcd8612015-06-12 12:02:09 +0200282static inline int context_domain_id(struct context_entry *c)
283{
284 return((c->hi >> 8) & 0xffff);
285}
286
Mark McLoughlinc07e7d22008-11-21 16:54:46 +0000287static inline void context_clear_entry(struct context_entry *context)
288{
289 context->lo = 0;
290 context->hi = 0;
291}
Mark McLoughlin7a8fc252008-11-20 15:49:45 +0000292
Mark McLoughlin622ba122008-11-20 15:49:46 +0000293/*
Fenghua Yu2c2e2c32009-06-19 13:47:29 -0700294 * This domain is a statically identity mapping domain.
295 * 1. This domain creats a static 1:1 mapping to all usable memory.
296 * 2. It maps to each iommu if successful.
297 * 3. Each iommu mapps to this domain if successful.
298 */
David Woodhouse19943b02009-08-04 16:19:20 +0100299static struct dmar_domain *si_domain;
300static int hw_pass_through = 1;
Fenghua Yu2c2e2c32009-06-19 13:47:29 -0700301
Joerg Roedel28ccce02015-07-21 14:45:31 +0200302/*
303 * Domain represents a virtual machine, more than one devices
Weidong Han1ce28fe2008-12-08 16:35:39 +0800304 * across iommus may be owned in one domain, e.g. kvm guest.
305 */
Jiang Liuab8dfe22014-07-11 14:19:27 +0800306#define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 0)
Weidong Han1ce28fe2008-12-08 16:35:39 +0800307
Fenghua Yu2c2e2c32009-06-19 13:47:29 -0700308/* si_domain contains mulitple devices */
Jiang Liuab8dfe22014-07-11 14:19:27 +0800309#define DOMAIN_FLAG_STATIC_IDENTITY (1 << 1)
Fenghua Yu2c2e2c32009-06-19 13:47:29 -0700310
Joerg Roedel29a27712015-07-21 17:17:12 +0200311#define for_each_domain_iommu(idx, domain) \
312 for (idx = 0; idx < g_num_of_iommus; idx++) \
313 if (domain->iommu_refcnt[idx])
314
Jiang Liub94e4112014-02-19 14:07:25 +0800315struct dmar_rmrr_unit {
316 struct list_head list; /* list of rmrr units */
317 struct acpi_dmar_header *hdr; /* ACPI header */
318 u64 base_address; /* reserved base address*/
319 u64 end_address; /* reserved end address */
David Woodhouse832bd852014-03-07 15:08:36 +0000320 struct dmar_dev_scope *devices; /* target devices */
Jiang Liub94e4112014-02-19 14:07:25 +0800321 int devices_cnt; /* target device count */
Eric Auger0659b8d2017-01-19 20:57:53 +0000322 struct iommu_resv_region *resv; /* reserved region handle */
Jiang Liub94e4112014-02-19 14:07:25 +0800323};
324
325struct dmar_atsr_unit {
326 struct list_head list; /* list of ATSR units */
327 struct acpi_dmar_header *hdr; /* ACPI header */
David Woodhouse832bd852014-03-07 15:08:36 +0000328 struct dmar_dev_scope *devices; /* target devices */
Jiang Liub94e4112014-02-19 14:07:25 +0800329 int devices_cnt; /* target device count */
330 u8 include_all:1; /* include all ports */
331};
332
333static LIST_HEAD(dmar_atsr_units);
334static LIST_HEAD(dmar_rmrr_units);
335
336#define for_each_rmrr_units(rmrr) \
337 list_for_each_entry(rmrr, &dmar_rmrr_units, list)
338
mark gross5e0d2a62008-03-04 15:22:08 -0800339/* bitmap for indexing intel_iommus */
mark gross5e0d2a62008-03-04 15:22:08 -0800340static int g_num_of_iommus;
341
Jiang Liu92d03cc2014-02-19 14:07:28 +0800342static void domain_exit(struct dmar_domain *domain);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700343static void domain_remove_dev_info(struct dmar_domain *domain);
Joerg Roedele6de0f82015-07-22 16:30:36 +0200344static void dmar_remove_one_dev_info(struct dmar_domain *domain,
345 struct device *dev);
Joerg Roedel127c7612015-07-23 17:44:46 +0200346static void __dmar_remove_one_dev_info(struct device_domain_info *info);
Joerg Roedel2452d9d2015-07-23 16:20:14 +0200347static void domain_context_clear(struct intel_iommu *iommu,
348 struct device *dev);
Jiang Liu2a46ddf2014-07-11 14:19:30 +0800349static int domain_detach_iommu(struct dmar_domain *domain,
350 struct intel_iommu *iommu);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700351
Suresh Siddhad3f13812011-08-23 17:05:25 -0700352#ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
Kyle McMartin0cd5c3c2009-02-04 14:29:19 -0800353int dmar_disabled = 0;
354#else
355int dmar_disabled = 1;
Suresh Siddhad3f13812011-08-23 17:05:25 -0700356#endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
Kyle McMartin0cd5c3c2009-02-04 14:29:19 -0800357
Eugeni Dodonov8bc1f852011-11-23 16:42:14 -0200358int intel_iommu_enabled = 0;
359EXPORT_SYMBOL_GPL(intel_iommu_enabled);
360
David Woodhouse2d9e6672010-06-15 10:57:57 +0100361static int dmar_map_gfx = 1;
Keshavamurthy, Anil S7d3b03c2007-10-21 16:41:53 -0700362static int dmar_forcedac;
mark gross5e0d2a62008-03-04 15:22:08 -0800363static int intel_iommu_strict;
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100364static int intel_iommu_superpage = 1;
Lu Baolu765b6a92018-12-10 09:58:55 +0800365static int intel_iommu_sm = 1;
David Woodhouseae853dd2015-09-09 11:58:59 +0100366static int iommu_identity_mapping;
David Woodhousec83b2f22015-06-12 10:15:49 +0100367
David Woodhouseae853dd2015-09-09 11:58:59 +0100368#define IDENTMAP_ALL 1
369#define IDENTMAP_GFX 2
370#define IDENTMAP_AZALIA 4
David Woodhousec83b2f22015-06-12 10:15:49 +0100371
Lu Baolu765b6a92018-12-10 09:58:55 +0800372#define sm_supported(iommu) (intel_iommu_sm && ecap_smts((iommu)->ecap))
373#define pasid_supported(iommu) (sm_supported(iommu) && \
374 ecap_pasid((iommu)->ecap))
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700375
David Woodhousec0771df2011-10-14 20:59:46 +0100376int intel_iommu_gfx_mapped;
377EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
378
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700379#define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
380static DEFINE_SPINLOCK(device_domain_lock);
381static LIST_HEAD(device_domain_list);
382
Lu Baolu85319dc2018-07-14 15:46:58 +0800383/*
384 * Iterate over elements in device_domain_list and call the specified
Lu Baolu0bbeb012018-12-10 09:58:56 +0800385 * callback @fn against each element.
Lu Baolu85319dc2018-07-14 15:46:58 +0800386 */
387int for_each_device_domain(int (*fn)(struct device_domain_info *info,
388 void *data), void *data)
389{
390 int ret = 0;
Lu Baolu0bbeb012018-12-10 09:58:56 +0800391 unsigned long flags;
Lu Baolu85319dc2018-07-14 15:46:58 +0800392 struct device_domain_info *info;
393
Lu Baolu0bbeb012018-12-10 09:58:56 +0800394 spin_lock_irqsave(&device_domain_lock, flags);
Lu Baolu85319dc2018-07-14 15:46:58 +0800395 list_for_each_entry(info, &device_domain_list, global) {
396 ret = fn(info, data);
Lu Baolu0bbeb012018-12-10 09:58:56 +0800397 if (ret) {
398 spin_unlock_irqrestore(&device_domain_lock, flags);
Lu Baolu85319dc2018-07-14 15:46:58 +0800399 return ret;
Lu Baolu0bbeb012018-12-10 09:58:56 +0800400 }
Lu Baolu85319dc2018-07-14 15:46:58 +0800401 }
Lu Baolu0bbeb012018-12-10 09:58:56 +0800402 spin_unlock_irqrestore(&device_domain_lock, flags);
Lu Baolu85319dc2018-07-14 15:46:58 +0800403
404 return 0;
405}
406
Joerg Roedelb0119e82017-02-01 13:23:08 +0100407const struct iommu_ops intel_iommu_ops;
Joerg Roedela8bcbb0d2008-12-03 15:14:02 +0100408
Joerg Roedel4158c2e2015-06-12 10:14:02 +0200409static bool translation_pre_enabled(struct intel_iommu *iommu)
410{
411 return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
412}
413
Joerg Roedel091d42e2015-06-12 11:56:10 +0200414static void clear_translation_pre_enabled(struct intel_iommu *iommu)
415{
416 iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
417}
418
Joerg Roedel4158c2e2015-06-12 10:14:02 +0200419static void init_translation_status(struct intel_iommu *iommu)
420{
421 u32 gsts;
422
423 gsts = readl(iommu->reg + DMAR_GSTS_REG);
424 if (gsts & DMA_GSTS_TES)
425 iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
426}
427
Joerg Roedel00a77de2015-03-26 13:43:08 +0100428/* Convert generic 'struct iommu_domain to private struct dmar_domain */
429static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
430{
431 return container_of(dom, struct dmar_domain, domain);
432}
433
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700434static int __init intel_iommu_setup(char *str)
435{
436 if (!str)
437 return -EINVAL;
438 while (*str) {
Kyle McMartin0cd5c3c2009-02-04 14:29:19 -0800439 if (!strncmp(str, "on", 2)) {
440 dmar_disabled = 0;
Joerg Roedel9f10e5b2015-06-12 09:57:06 +0200441 pr_info("IOMMU enabled\n");
Kyle McMartin0cd5c3c2009-02-04 14:29:19 -0800442 } else if (!strncmp(str, "off", 3)) {
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700443 dmar_disabled = 1;
Joerg Roedel9f10e5b2015-06-12 09:57:06 +0200444 pr_info("IOMMU disabled\n");
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700445 } else if (!strncmp(str, "igfx_off", 8)) {
446 dmar_map_gfx = 0;
Joerg Roedel9f10e5b2015-06-12 09:57:06 +0200447 pr_info("Disable GFX device mapping\n");
Keshavamurthy, Anil S7d3b03c2007-10-21 16:41:53 -0700448 } else if (!strncmp(str, "forcedac", 8)) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +0200449 pr_info("Forcing DAC for PCI devices\n");
Keshavamurthy, Anil S7d3b03c2007-10-21 16:41:53 -0700450 dmar_forcedac = 1;
mark gross5e0d2a62008-03-04 15:22:08 -0800451 } else if (!strncmp(str, "strict", 6)) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +0200452 pr_info("Disable batched IOTLB flush\n");
mark gross5e0d2a62008-03-04 15:22:08 -0800453 intel_iommu_strict = 1;
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100454 } else if (!strncmp(str, "sp_off", 6)) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +0200455 pr_info("Disable supported super page\n");
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100456 intel_iommu_superpage = 0;
Lu Baolu765b6a92018-12-10 09:58:55 +0800457 } else if (!strncmp(str, "sm_off", 6)) {
458 pr_info("Intel-IOMMU: disable scalable mode support\n");
459 intel_iommu_sm = 0;
Shaohua Libfd20f12017-04-26 09:18:35 -0700460 } else if (!strncmp(str, "tboot_noforce", 13)) {
461 printk(KERN_INFO
462 "Intel-IOMMU: not forcing on after tboot. This could expose security risk for tboot\n");
463 intel_iommu_tboot_noforce = 1;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700464 }
465
466 str += strcspn(str, ",");
467 while (*str == ',')
468 str++;
469 }
470 return 0;
471}
472__setup("intel_iommu=", intel_iommu_setup);
473
474static struct kmem_cache *iommu_domain_cache;
475static struct kmem_cache *iommu_devinfo_cache;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700476
Joerg Roedel9452d5b2015-07-21 10:00:56 +0200477static struct dmar_domain* get_iommu_domain(struct intel_iommu *iommu, u16 did)
478{
Joerg Roedel8bf47812015-07-21 10:41:21 +0200479 struct dmar_domain **domains;
480 int idx = did >> 8;
481
482 domains = iommu->domains[idx];
483 if (!domains)
484 return NULL;
485
486 return domains[did & 0xff];
Joerg Roedel9452d5b2015-07-21 10:00:56 +0200487}
488
489static void set_iommu_domain(struct intel_iommu *iommu, u16 did,
490 struct dmar_domain *domain)
491{
Joerg Roedel8bf47812015-07-21 10:41:21 +0200492 struct dmar_domain **domains;
493 int idx = did >> 8;
494
495 if (!iommu->domains[idx]) {
496 size_t size = 256 * sizeof(struct dmar_domain *);
497 iommu->domains[idx] = kzalloc(size, GFP_ATOMIC);
498 }
499
500 domains = iommu->domains[idx];
501 if (WARN_ON(!domains))
502 return;
503 else
504 domains[did & 0xff] = domain;
Joerg Roedel9452d5b2015-07-21 10:00:56 +0200505}
506
Lu Baolu9ddbfb42018-07-14 15:46:57 +0800507void *alloc_pgtable_page(int node)
Keshavamurthy, Anil Seb3fa7c2007-10-21 16:41:52 -0700508{
Suresh Siddha4c923d42009-10-02 11:01:24 -0700509 struct page *page;
510 void *vaddr = NULL;
Keshavamurthy, Anil Seb3fa7c2007-10-21 16:41:52 -0700511
Suresh Siddha4c923d42009-10-02 11:01:24 -0700512 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
513 if (page)
514 vaddr = page_address(page);
Keshavamurthy, Anil Seb3fa7c2007-10-21 16:41:52 -0700515 return vaddr;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700516}
517
Lu Baolu9ddbfb42018-07-14 15:46:57 +0800518void free_pgtable_page(void *vaddr)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700519{
520 free_page((unsigned long)vaddr);
521}
522
523static inline void *alloc_domain_mem(void)
524{
KOSAKI Motohiro354bb652009-11-17 16:21:09 +0900525 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700526}
527
Kay, Allen M38717942008-09-09 18:37:29 +0300528static void free_domain_mem(void *vaddr)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700529{
530 kmem_cache_free(iommu_domain_cache, vaddr);
531}
532
533static inline void * alloc_devinfo_mem(void)
534{
KOSAKI Motohiro354bb652009-11-17 16:21:09 +0900535 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700536}
537
538static inline void free_devinfo_mem(void *vaddr)
539{
540 kmem_cache_free(iommu_devinfo_cache, vaddr);
541}
542
Jiang Liuab8dfe22014-07-11 14:19:27 +0800543static inline int domain_type_is_vm(struct dmar_domain *domain)
544{
545 return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
546}
547
Joerg Roedel28ccce02015-07-21 14:45:31 +0200548static inline int domain_type_is_si(struct dmar_domain *domain)
549{
550 return domain->flags & DOMAIN_FLAG_STATIC_IDENTITY;
551}
552
Jiang Liuab8dfe22014-07-11 14:19:27 +0800553static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
554{
555 return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
556 DOMAIN_FLAG_STATIC_IDENTITY);
557}
Weidong Han1b573682008-12-08 15:34:06 +0800558
Jiang Liu162d1b12014-07-11 14:19:35 +0800559static inline int domain_pfn_supported(struct dmar_domain *domain,
560 unsigned long pfn)
561{
562 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
563
564 return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
565}
566
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -0700567static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
Weidong Han1b573682008-12-08 15:34:06 +0800568{
569 unsigned long sagaw;
570 int agaw = -1;
571
572 sagaw = cap_sagaw(iommu->cap);
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -0700573 for (agaw = width_to_agaw(max_gaw);
Weidong Han1b573682008-12-08 15:34:06 +0800574 agaw >= 0; agaw--) {
575 if (test_bit(agaw, &sagaw))
576 break;
577 }
578
579 return agaw;
580}
581
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -0700582/*
583 * Calculate max SAGAW for each iommu.
584 */
585int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
586{
587 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
588}
589
590/*
591 * calculate agaw for each iommu.
592 * "SAGAW" may be different across iommus, use a default agaw, and
593 * get a supported less agaw for iommus that don't support the default agaw.
594 */
595int iommu_calculate_agaw(struct intel_iommu *iommu)
596{
597 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
598}
599
Fenghua Yu2c2e2c32009-06-19 13:47:29 -0700600/* This functionin only returns single iommu in a domain */
Lu Baolu9ddbfb42018-07-14 15:46:57 +0800601struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
Weidong Han8c11e792008-12-08 15:29:22 +0800602{
603 int iommu_id;
604
Fenghua Yu2c2e2c32009-06-19 13:47:29 -0700605 /* si_domain and vm domain should not get here. */
Jiang Liuab8dfe22014-07-11 14:19:27 +0800606 BUG_ON(domain_type_is_vm_or_si(domain));
Joerg Roedel29a27712015-07-21 17:17:12 +0200607 for_each_domain_iommu(iommu_id, domain)
608 break;
609
Weidong Han8c11e792008-12-08 15:29:22 +0800610 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
611 return NULL;
612
613 return g_iommus[iommu_id];
614}
615
Weidong Han8e6040972008-12-08 15:49:06 +0800616static void domain_update_iommu_coherency(struct dmar_domain *domain)
617{
David Woodhoused0501962014-03-11 17:10:29 -0700618 struct dmar_drhd_unit *drhd;
619 struct intel_iommu *iommu;
Quentin Lambert2f119c72015-02-06 10:59:53 +0100620 bool found = false;
621 int i;
Weidong Han8e6040972008-12-08 15:49:06 +0800622
David Woodhoused0501962014-03-11 17:10:29 -0700623 domain->iommu_coherency = 1;
Weidong Han8e6040972008-12-08 15:49:06 +0800624
Joerg Roedel29a27712015-07-21 17:17:12 +0200625 for_each_domain_iommu(i, domain) {
Quentin Lambert2f119c72015-02-06 10:59:53 +0100626 found = true;
Weidong Han8e6040972008-12-08 15:49:06 +0800627 if (!ecap_coherent(g_iommus[i]->ecap)) {
628 domain->iommu_coherency = 0;
629 break;
630 }
Weidong Han8e6040972008-12-08 15:49:06 +0800631 }
David Woodhoused0501962014-03-11 17:10:29 -0700632 if (found)
633 return;
634
635 /* No hardware attached; use lowest common denominator */
636 rcu_read_lock();
637 for_each_active_iommu(iommu, drhd) {
638 if (!ecap_coherent(iommu->ecap)) {
639 domain->iommu_coherency = 0;
640 break;
641 }
642 }
643 rcu_read_unlock();
Weidong Han8e6040972008-12-08 15:49:06 +0800644}
645
Jiang Liu161f6932014-07-11 14:19:37 +0800646static int domain_update_iommu_snooping(struct intel_iommu *skip)
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100647{
Allen Kay8140a952011-10-14 12:32:17 -0700648 struct dmar_drhd_unit *drhd;
Jiang Liu161f6932014-07-11 14:19:37 +0800649 struct intel_iommu *iommu;
650 int ret = 1;
651
652 rcu_read_lock();
653 for_each_active_iommu(iommu, drhd) {
654 if (iommu != skip) {
655 if (!ecap_sc_support(iommu->ecap)) {
656 ret = 0;
657 break;
658 }
659 }
660 }
661 rcu_read_unlock();
662
663 return ret;
664}
665
666static int domain_update_iommu_superpage(struct intel_iommu *skip)
667{
668 struct dmar_drhd_unit *drhd;
669 struct intel_iommu *iommu;
Allen Kay8140a952011-10-14 12:32:17 -0700670 int mask = 0xf;
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100671
672 if (!intel_iommu_superpage) {
Jiang Liu161f6932014-07-11 14:19:37 +0800673 return 0;
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100674 }
675
Allen Kay8140a952011-10-14 12:32:17 -0700676 /* set iommu_superpage to the smallest common denominator */
Jiang Liu0e242612014-02-19 14:07:34 +0800677 rcu_read_lock();
Allen Kay8140a952011-10-14 12:32:17 -0700678 for_each_active_iommu(iommu, drhd) {
Jiang Liu161f6932014-07-11 14:19:37 +0800679 if (iommu != skip) {
680 mask &= cap_super_page_val(iommu->cap);
681 if (!mask)
682 break;
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100683 }
684 }
Jiang Liu0e242612014-02-19 14:07:34 +0800685 rcu_read_unlock();
686
Jiang Liu161f6932014-07-11 14:19:37 +0800687 return fls(mask);
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100688}
689
Sheng Yang58c610b2009-03-18 15:33:05 +0800690/* Some capabilities may be different across iommus */
691static void domain_update_iommu_cap(struct dmar_domain *domain)
692{
693 domain_update_iommu_coherency(domain);
Jiang Liu161f6932014-07-11 14:19:37 +0800694 domain->iommu_snooping = domain_update_iommu_snooping(NULL);
695 domain->iommu_superpage = domain_update_iommu_superpage(NULL);
Sheng Yang58c610b2009-03-18 15:33:05 +0800696}
697
Sohil Mehta26b86092018-09-11 17:11:36 -0700698struct context_entry *iommu_context_addr(struct intel_iommu *iommu, u8 bus,
699 u8 devfn, int alloc)
David Woodhouse03ecc322015-02-13 14:35:21 +0000700{
701 struct root_entry *root = &iommu->root_entry[bus];
702 struct context_entry *context;
703 u64 *entry;
704
Joerg Roedel4df4eab2015-08-25 10:54:28 +0200705 entry = &root->lo;
Lu Baolu765b6a92018-12-10 09:58:55 +0800706 if (sm_supported(iommu)) {
David Woodhouse03ecc322015-02-13 14:35:21 +0000707 if (devfn >= 0x80) {
708 devfn -= 0x80;
709 entry = &root->hi;
710 }
711 devfn *= 2;
712 }
David Woodhouse03ecc322015-02-13 14:35:21 +0000713 if (*entry & 1)
714 context = phys_to_virt(*entry & VTD_PAGE_MASK);
715 else {
716 unsigned long phy_addr;
717 if (!alloc)
718 return NULL;
719
720 context = alloc_pgtable_page(iommu->node);
721 if (!context)
722 return NULL;
723
724 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
725 phy_addr = virt_to_phys((void *)context);
726 *entry = phy_addr | 1;
727 __iommu_flush_cache(iommu, entry, sizeof(*entry));
728 }
729 return &context[devfn];
730}
731
David Woodhouse4ed6a542015-05-11 14:59:20 +0100732static int iommu_dummy(struct device *dev)
733{
734 return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
735}
736
David Woodhouse156baca2014-03-09 14:00:57 -0700737static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
Weidong Hanc7151a82008-12-08 22:51:37 +0800738{
739 struct dmar_drhd_unit *drhd = NULL;
Jiang Liub683b232014-02-19 14:07:32 +0800740 struct intel_iommu *iommu;
David Woodhouse156baca2014-03-09 14:00:57 -0700741 struct device *tmp;
742 struct pci_dev *ptmp, *pdev = NULL;
Yijing Wangaa4d0662014-05-26 20:14:06 +0800743 u16 segment = 0;
Weidong Hanc7151a82008-12-08 22:51:37 +0800744 int i;
745
David Woodhouse4ed6a542015-05-11 14:59:20 +0100746 if (iommu_dummy(dev))
747 return NULL;
748
David Woodhouse156baca2014-03-09 14:00:57 -0700749 if (dev_is_pci(dev)) {
Ashok Raj1c387182016-10-21 15:32:05 -0700750 struct pci_dev *pf_pdev;
751
David Woodhouse156baca2014-03-09 14:00:57 -0700752 pdev = to_pci_dev(dev);
Jon Derrick5823e332017-08-30 15:05:59 -0600753
754#ifdef CONFIG_X86
755 /* VMD child devices currently cannot be handled individually */
756 if (is_vmd(pdev->bus))
757 return NULL;
758#endif
759
Ashok Raj1c387182016-10-21 15:32:05 -0700760 /* VFs aren't listed in scope tables; we need to look up
761 * the PF instead to find the IOMMU. */
762 pf_pdev = pci_physfn(pdev);
763 dev = &pf_pdev->dev;
David Woodhouse156baca2014-03-09 14:00:57 -0700764 segment = pci_domain_nr(pdev->bus);
Rafael J. Wysockica5b74d2015-03-16 23:49:08 +0100765 } else if (has_acpi_companion(dev))
David Woodhouse156baca2014-03-09 14:00:57 -0700766 dev = &ACPI_COMPANION(dev)->dev;
767
Jiang Liu0e242612014-02-19 14:07:34 +0800768 rcu_read_lock();
Jiang Liub683b232014-02-19 14:07:32 +0800769 for_each_active_iommu(iommu, drhd) {
David Woodhouse156baca2014-03-09 14:00:57 -0700770 if (pdev && segment != drhd->segment)
David Woodhouse276dbf992009-04-04 01:45:37 +0100771 continue;
Weidong Hanc7151a82008-12-08 22:51:37 +0800772
Jiang Liub683b232014-02-19 14:07:32 +0800773 for_each_active_dev_scope(drhd->devices,
David Woodhouse156baca2014-03-09 14:00:57 -0700774 drhd->devices_cnt, i, tmp) {
775 if (tmp == dev) {
Ashok Raj1c387182016-10-21 15:32:05 -0700776 /* For a VF use its original BDF# not that of the PF
777 * which we used for the IOMMU lookup. Strictly speaking
778 * we could do this for all PCI devices; we only need to
779 * get the BDF# from the scope table for ACPI matches. */
Koos Vriezen5003ae12017-03-01 21:02:50 +0100780 if (pdev && pdev->is_virtfn)
Ashok Raj1c387182016-10-21 15:32:05 -0700781 goto got_pdev;
782
David Woodhouse156baca2014-03-09 14:00:57 -0700783 *bus = drhd->devices[i].bus;
784 *devfn = drhd->devices[i].devfn;
785 goto out;
786 }
787
788 if (!pdev || !dev_is_pci(tmp))
David Woodhouse832bd852014-03-07 15:08:36 +0000789 continue;
David Woodhouse156baca2014-03-09 14:00:57 -0700790
791 ptmp = to_pci_dev(tmp);
792 if (ptmp->subordinate &&
793 ptmp->subordinate->number <= pdev->bus->number &&
794 ptmp->subordinate->busn_res.end >= pdev->bus->number)
795 goto got_pdev;
David Woodhouse924b6232009-04-04 00:39:25 +0100796 }
Weidong Hanc7151a82008-12-08 22:51:37 +0800797
David Woodhouse156baca2014-03-09 14:00:57 -0700798 if (pdev && drhd->include_all) {
799 got_pdev:
800 *bus = pdev->bus->number;
801 *devfn = pdev->devfn;
Jiang Liub683b232014-02-19 14:07:32 +0800802 goto out;
David Woodhouse156baca2014-03-09 14:00:57 -0700803 }
Weidong Hanc7151a82008-12-08 22:51:37 +0800804 }
Jiang Liub683b232014-02-19 14:07:32 +0800805 iommu = NULL;
David Woodhouse156baca2014-03-09 14:00:57 -0700806 out:
Jiang Liu0e242612014-02-19 14:07:34 +0800807 rcu_read_unlock();
Weidong Hanc7151a82008-12-08 22:51:37 +0800808
Jiang Liub683b232014-02-19 14:07:32 +0800809 return iommu;
Weidong Hanc7151a82008-12-08 22:51:37 +0800810}
811
Weidong Han5331fe62008-12-08 23:00:00 +0800812static void domain_flush_cache(struct dmar_domain *domain,
813 void *addr, int size)
814{
815 if (!domain->iommu_coherency)
816 clflush_cache_range(addr, size);
817}
818
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700819static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
820{
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700821 struct context_entry *context;
David Woodhouse03ecc322015-02-13 14:35:21 +0000822 int ret = 0;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700823 unsigned long flags;
824
825 spin_lock_irqsave(&iommu->lock, flags);
David Woodhouse03ecc322015-02-13 14:35:21 +0000826 context = iommu_context_addr(iommu, bus, devfn, 0);
827 if (context)
828 ret = context_present(context);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700829 spin_unlock_irqrestore(&iommu->lock, flags);
830 return ret;
831}
832
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700833static void free_context_table(struct intel_iommu *iommu)
834{
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700835 int i;
836 unsigned long flags;
837 struct context_entry *context;
838
839 spin_lock_irqsave(&iommu->lock, flags);
840 if (!iommu->root_entry) {
841 goto out;
842 }
843 for (i = 0; i < ROOT_ENTRY_NR; i++) {
David Woodhouse03ecc322015-02-13 14:35:21 +0000844 context = iommu_context_addr(iommu, i, 0, 0);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700845 if (context)
846 free_pgtable_page(context);
David Woodhouse03ecc322015-02-13 14:35:21 +0000847
Lu Baolu765b6a92018-12-10 09:58:55 +0800848 if (!sm_supported(iommu))
David Woodhouse03ecc322015-02-13 14:35:21 +0000849 continue;
850
851 context = iommu_context_addr(iommu, i, 0x80, 0);
852 if (context)
853 free_pgtable_page(context);
854
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700855 }
856 free_pgtable_page(iommu->root_entry);
857 iommu->root_entry = NULL;
858out:
859 spin_unlock_irqrestore(&iommu->lock, flags);
860}
861
David Woodhouseb026fd22009-06-28 10:37:25 +0100862static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
David Woodhouse5cf0a762014-03-19 16:07:49 +0000863 unsigned long pfn, int *target_level)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700864{
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700865 struct dma_pte *parent, *pte = NULL;
866 int level = agaw_to_level(domain->agaw);
Allen Kay4399c8b2011-10-14 12:32:46 -0700867 int offset;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700868
869 BUG_ON(!domain->pgd);
Julian Stecklinaf9423602013-10-09 10:03:52 +0200870
Jiang Liu162d1b12014-07-11 14:19:35 +0800871 if (!domain_pfn_supported(domain, pfn))
Julian Stecklinaf9423602013-10-09 10:03:52 +0200872 /* Address beyond IOMMU's addressing capabilities. */
873 return NULL;
874
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700875 parent = domain->pgd;
876
David Woodhouse5cf0a762014-03-19 16:07:49 +0000877 while (1) {
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700878 void *tmp_page;
879
David Woodhouseb026fd22009-06-28 10:37:25 +0100880 offset = pfn_level_offset(pfn, level);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700881 pte = &parent[offset];
David Woodhouse5cf0a762014-03-19 16:07:49 +0000882 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100883 break;
David Woodhouse5cf0a762014-03-19 16:07:49 +0000884 if (level == *target_level)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700885 break;
886
Mark McLoughlin19c239c2008-11-21 16:56:53 +0000887 if (!dma_pte_present(pte)) {
David Woodhousec85994e2009-07-01 19:21:24 +0100888 uint64_t pteval;
889
Suresh Siddha4c923d42009-10-02 11:01:24 -0700890 tmp_page = alloc_pgtable_page(domain->nid);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700891
David Woodhouse206a73c2009-07-01 19:30:28 +0100892 if (!tmp_page)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700893 return NULL;
David Woodhouse206a73c2009-07-01 19:30:28 +0100894
David Woodhousec85994e2009-07-01 19:21:24 +0100895 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
Benjamin LaHaise64de5af2009-09-16 21:05:55 -0400896 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
Yijing Wangeffad4b2014-05-26 20:13:47 +0800897 if (cmpxchg64(&pte->val, 0ULL, pteval))
David Woodhousec85994e2009-07-01 19:21:24 +0100898 /* Someone else set it while we were thinking; use theirs. */
899 free_pgtable_page(tmp_page);
Yijing Wangeffad4b2014-05-26 20:13:47 +0800900 else
David Woodhousec85994e2009-07-01 19:21:24 +0100901 domain_flush_cache(domain, pte, sizeof(*pte));
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700902 }
David Woodhouse5cf0a762014-03-19 16:07:49 +0000903 if (level == 1)
904 break;
905
Mark McLoughlin19c239c2008-11-21 16:56:53 +0000906 parent = phys_to_virt(dma_pte_addr(pte));
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700907 level--;
908 }
909
David Woodhouse5cf0a762014-03-19 16:07:49 +0000910 if (!*target_level)
911 *target_level = level;
912
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700913 return pte;
914}
915
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100916
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700917/* return address's pte at specific level */
David Woodhouse90dcfb52009-06-27 17:14:59 +0100918static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
919 unsigned long pfn,
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100920 int level, int *large_page)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700921{
922 struct dma_pte *parent, *pte = NULL;
923 int total = agaw_to_level(domain->agaw);
924 int offset;
925
926 parent = domain->pgd;
927 while (level <= total) {
David Woodhouse90dcfb52009-06-27 17:14:59 +0100928 offset = pfn_level_offset(pfn, total);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700929 pte = &parent[offset];
930 if (level == total)
931 return pte;
932
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100933 if (!dma_pte_present(pte)) {
934 *large_page = total;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700935 break;
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100936 }
937
Yijing Wange16922a2014-05-20 20:37:51 +0800938 if (dma_pte_superpage(pte)) {
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100939 *large_page = total;
940 return pte;
941 }
942
Mark McLoughlin19c239c2008-11-21 16:56:53 +0000943 parent = phys_to_virt(dma_pte_addr(pte));
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700944 total--;
945 }
946 return NULL;
947}
948
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700949/* clear last level pte, a tlb flush should be followed */
David Woodhouse5cf0a762014-03-19 16:07:49 +0000950static void dma_pte_clear_range(struct dmar_domain *domain,
David Woodhouse595badf52009-06-27 22:09:11 +0100951 unsigned long start_pfn,
952 unsigned long last_pfn)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700953{
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100954 unsigned int large_page = 1;
David Woodhouse310a5ab2009-06-28 18:52:20 +0100955 struct dma_pte *first_pte, *pte;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700956
Jiang Liu162d1b12014-07-11 14:19:35 +0800957 BUG_ON(!domain_pfn_supported(domain, start_pfn));
958 BUG_ON(!domain_pfn_supported(domain, last_pfn));
David Woodhouse59c36282009-09-19 07:36:28 -0700959 BUG_ON(start_pfn > last_pfn);
David Woodhouse66eae842009-06-27 19:00:32 +0100960
David Woodhouse04b18e62009-06-27 19:15:01 +0100961 /* we don't need lock here; nobody else touches the iova range */
David Woodhouse59c36282009-09-19 07:36:28 -0700962 do {
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100963 large_page = 1;
964 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
David Woodhouse310a5ab2009-06-28 18:52:20 +0100965 if (!pte) {
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100966 start_pfn = align_to_level(start_pfn + 1, large_page + 1);
David Woodhouse310a5ab2009-06-28 18:52:20 +0100967 continue;
968 }
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100969 do {
David Woodhouse310a5ab2009-06-28 18:52:20 +0100970 dma_clear_pte(pte);
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100971 start_pfn += lvl_to_nr_pages(large_page);
David Woodhouse310a5ab2009-06-28 18:52:20 +0100972 pte++;
David Woodhouse75e6bf92009-07-02 11:21:16 +0100973 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
974
David Woodhouse310a5ab2009-06-28 18:52:20 +0100975 domain_flush_cache(domain, first_pte,
976 (void *)pte - (void *)first_pte);
David Woodhouse59c36282009-09-19 07:36:28 -0700977
978 } while (start_pfn && start_pfn <= last_pfn);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700979}
980
Alex Williamson3269ee02013-06-15 10:27:19 -0600981static void dma_pte_free_level(struct dmar_domain *domain, int level,
David Dillowbc24c572017-06-28 19:42:23 -0700982 int retain_level, struct dma_pte *pte,
983 unsigned long pfn, unsigned long start_pfn,
984 unsigned long last_pfn)
Alex Williamson3269ee02013-06-15 10:27:19 -0600985{
986 pfn = max(start_pfn, pfn);
987 pte = &pte[pfn_level_offset(pfn, level)];
988
989 do {
990 unsigned long level_pfn;
991 struct dma_pte *level_pte;
992
993 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
994 goto next;
995
David Dillowf7116e12017-01-30 19:11:11 -0800996 level_pfn = pfn & level_mask(level);
Alex Williamson3269ee02013-06-15 10:27:19 -0600997 level_pte = phys_to_virt(dma_pte_addr(pte));
998
David Dillowbc24c572017-06-28 19:42:23 -0700999 if (level > 2) {
1000 dma_pte_free_level(domain, level - 1, retain_level,
1001 level_pte, level_pfn, start_pfn,
1002 last_pfn);
1003 }
Alex Williamson3269ee02013-06-15 10:27:19 -06001004
David Dillowbc24c572017-06-28 19:42:23 -07001005 /*
1006 * Free the page table if we're below the level we want to
1007 * retain and the range covers the entire table.
1008 */
1009 if (level < retain_level && !(start_pfn > level_pfn ||
Alex Williamson08336fd2014-01-21 15:48:18 -08001010 last_pfn < level_pfn + level_size(level) - 1)) {
Alex Williamson3269ee02013-06-15 10:27:19 -06001011 dma_clear_pte(pte);
1012 domain_flush_cache(domain, pte, sizeof(*pte));
1013 free_pgtable_page(level_pte);
1014 }
1015next:
1016 pfn += level_size(level);
1017 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1018}
1019
David Dillowbc24c572017-06-28 19:42:23 -07001020/*
1021 * clear last level (leaf) ptes and free page table pages below the
1022 * level we wish to keep intact.
1023 */
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001024static void dma_pte_free_pagetable(struct dmar_domain *domain,
David Woodhoused794dc92009-06-28 00:27:49 +01001025 unsigned long start_pfn,
David Dillowbc24c572017-06-28 19:42:23 -07001026 unsigned long last_pfn,
1027 int retain_level)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001028{
Jiang Liu162d1b12014-07-11 14:19:35 +08001029 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1030 BUG_ON(!domain_pfn_supported(domain, last_pfn));
David Woodhouse59c36282009-09-19 07:36:28 -07001031 BUG_ON(start_pfn > last_pfn);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001032
Jiang Liud41a4ad2014-07-11 14:19:34 +08001033 dma_pte_clear_range(domain, start_pfn, last_pfn);
1034
David Woodhousef3a0a522009-06-30 03:40:07 +01001035 /* We don't need lock here; nobody else touches the iova range */
David Dillowbc24c572017-06-28 19:42:23 -07001036 dma_pte_free_level(domain, agaw_to_level(domain->agaw), retain_level,
Alex Williamson3269ee02013-06-15 10:27:19 -06001037 domain->pgd, 0, start_pfn, last_pfn);
David Woodhouse6660c632009-06-27 22:41:00 +01001038
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001039 /* free pgd */
David Woodhoused794dc92009-06-28 00:27:49 +01001040 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001041 free_pgtable_page(domain->pgd);
1042 domain->pgd = NULL;
1043 }
1044}
1045
David Woodhouseea8ea462014-03-05 17:09:32 +00001046/* When a page at a given level is being unlinked from its parent, we don't
1047 need to *modify* it at all. All we need to do is make a list of all the
1048 pages which can be freed just as soon as we've flushed the IOTLB and we
1049 know the hardware page-walk will no longer touch them.
1050 The 'pte' argument is the *parent* PTE, pointing to the page that is to
1051 be freed. */
1052static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1053 int level, struct dma_pte *pte,
1054 struct page *freelist)
1055{
1056 struct page *pg;
1057
1058 pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1059 pg->freelist = freelist;
1060 freelist = pg;
1061
1062 if (level == 1)
1063 return freelist;
1064
Jiang Liuadeb2592014-04-09 10:20:39 +08001065 pte = page_address(pg);
1066 do {
David Woodhouseea8ea462014-03-05 17:09:32 +00001067 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1068 freelist = dma_pte_list_pagetables(domain, level - 1,
1069 pte, freelist);
Jiang Liuadeb2592014-04-09 10:20:39 +08001070 pte++;
1071 } while (!first_pte_in_page(pte));
David Woodhouseea8ea462014-03-05 17:09:32 +00001072
1073 return freelist;
1074}
1075
1076static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1077 struct dma_pte *pte, unsigned long pfn,
1078 unsigned long start_pfn,
1079 unsigned long last_pfn,
1080 struct page *freelist)
1081{
1082 struct dma_pte *first_pte = NULL, *last_pte = NULL;
1083
1084 pfn = max(start_pfn, pfn);
1085 pte = &pte[pfn_level_offset(pfn, level)];
1086
1087 do {
1088 unsigned long level_pfn;
1089
1090 if (!dma_pte_present(pte))
1091 goto next;
1092
1093 level_pfn = pfn & level_mask(level);
1094
1095 /* If range covers entire pagetable, free it */
1096 if (start_pfn <= level_pfn &&
1097 last_pfn >= level_pfn + level_size(level) - 1) {
1098 /* These suborbinate page tables are going away entirely. Don't
1099 bother to clear them; we're just going to *free* them. */
1100 if (level > 1 && !dma_pte_superpage(pte))
1101 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1102
1103 dma_clear_pte(pte);
1104 if (!first_pte)
1105 first_pte = pte;
1106 last_pte = pte;
1107 } else if (level > 1) {
1108 /* Recurse down into a level that isn't *entirely* obsolete */
1109 freelist = dma_pte_clear_level(domain, level - 1,
1110 phys_to_virt(dma_pte_addr(pte)),
1111 level_pfn, start_pfn, last_pfn,
1112 freelist);
1113 }
1114next:
1115 pfn += level_size(level);
1116 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1117
1118 if (first_pte)
1119 domain_flush_cache(domain, first_pte,
1120 (void *)++last_pte - (void *)first_pte);
1121
1122 return freelist;
1123}
1124
1125/* We can't just free the pages because the IOMMU may still be walking
1126 the page tables, and may have cached the intermediate levels. The
1127 pages can only be freed after the IOTLB flush has been done. */
Joerg Roedelb6904202015-08-13 11:32:18 +02001128static struct page *domain_unmap(struct dmar_domain *domain,
1129 unsigned long start_pfn,
1130 unsigned long last_pfn)
David Woodhouseea8ea462014-03-05 17:09:32 +00001131{
David Woodhouseea8ea462014-03-05 17:09:32 +00001132 struct page *freelist = NULL;
1133
Jiang Liu162d1b12014-07-11 14:19:35 +08001134 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1135 BUG_ON(!domain_pfn_supported(domain, last_pfn));
David Woodhouseea8ea462014-03-05 17:09:32 +00001136 BUG_ON(start_pfn > last_pfn);
1137
1138 /* we don't need lock here; nobody else touches the iova range */
1139 freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1140 domain->pgd, 0, start_pfn, last_pfn, NULL);
1141
1142 /* free pgd */
1143 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1144 struct page *pgd_page = virt_to_page(domain->pgd);
1145 pgd_page->freelist = freelist;
1146 freelist = pgd_page;
1147
1148 domain->pgd = NULL;
1149 }
1150
1151 return freelist;
1152}
1153
Joerg Roedelb6904202015-08-13 11:32:18 +02001154static void dma_free_pagelist(struct page *freelist)
David Woodhouseea8ea462014-03-05 17:09:32 +00001155{
1156 struct page *pg;
1157
1158 while ((pg = freelist)) {
1159 freelist = pg->freelist;
1160 free_pgtable_page(page_address(pg));
1161 }
1162}
1163
Joerg Roedel13cf0172017-08-11 11:40:10 +02001164static void iova_entry_free(unsigned long data)
1165{
1166 struct page *freelist = (struct page *)data;
1167
1168 dma_free_pagelist(freelist);
1169}
1170
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001171/* iommu handling */
1172static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1173{
1174 struct root_entry *root;
1175 unsigned long flags;
1176
Suresh Siddha4c923d42009-10-02 11:01:24 -07001177 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
Jiang Liuffebeb42014-11-09 22:48:02 +08001178 if (!root) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02001179 pr_err("Allocating root entry for %s failed\n",
Jiang Liuffebeb42014-11-09 22:48:02 +08001180 iommu->name);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001181 return -ENOMEM;
Jiang Liuffebeb42014-11-09 22:48:02 +08001182 }
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001183
Fenghua Yu5b6985c2008-10-16 18:02:32 -07001184 __iommu_flush_cache(iommu, root, ROOT_SIZE);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001185
1186 spin_lock_irqsave(&iommu->lock, flags);
1187 iommu->root_entry = root;
1188 spin_unlock_irqrestore(&iommu->lock, flags);
1189
1190 return 0;
1191}
1192
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001193static void iommu_set_root_entry(struct intel_iommu *iommu)
1194{
David Woodhouse03ecc322015-02-13 14:35:21 +00001195 u64 addr;
David Woodhousec416daa2009-05-10 20:30:58 +01001196 u32 sts;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001197 unsigned long flag;
1198
David Woodhouse03ecc322015-02-13 14:35:21 +00001199 addr = virt_to_phys(iommu->root_entry);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001200
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001201 raw_spin_lock_irqsave(&iommu->register_lock, flag);
David Woodhouse03ecc322015-02-13 14:35:21 +00001202 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001203
David Woodhousec416daa2009-05-10 20:30:58 +01001204 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001205
1206 /* Make sure hardware complete it */
1207 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
David Woodhousec416daa2009-05-10 20:30:58 +01001208 readl, (sts & DMA_GSTS_RTPS), sts);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001209
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001210 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001211}
1212
Lu Baolu6f7db752018-12-10 09:59:00 +08001213void iommu_flush_write_buffer(struct intel_iommu *iommu)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001214{
1215 u32 val;
1216 unsigned long flag;
1217
David Woodhouse9af88142009-02-13 23:18:03 +00001218 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001219 return;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001220
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001221 raw_spin_lock_irqsave(&iommu->register_lock, flag);
David Woodhouse462b60f2009-05-10 20:18:18 +01001222 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001223
1224 /* Make sure hardware complete it */
1225 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
David Woodhousec416daa2009-05-10 20:30:58 +01001226 readl, (!(val & DMA_GSTS_WBFS)), val);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001227
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001228 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001229}
1230
1231/* return value determine if we need a write buffer flush */
David Woodhouse4c25a2c2009-05-10 17:16:06 +01001232static void __iommu_flush_context(struct intel_iommu *iommu,
1233 u16 did, u16 source_id, u8 function_mask,
1234 u64 type)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001235{
1236 u64 val = 0;
1237 unsigned long flag;
1238
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001239 switch (type) {
1240 case DMA_CCMD_GLOBAL_INVL:
1241 val = DMA_CCMD_GLOBAL_INVL;
1242 break;
1243 case DMA_CCMD_DOMAIN_INVL:
1244 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1245 break;
1246 case DMA_CCMD_DEVICE_INVL:
1247 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1248 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1249 break;
1250 default:
1251 BUG();
1252 }
1253 val |= DMA_CCMD_ICC;
1254
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001255 raw_spin_lock_irqsave(&iommu->register_lock, flag);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001256 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1257
1258 /* Make sure hardware complete it */
1259 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1260 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1261
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001262 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001263}
1264
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001265/* return value determine if we need a write buffer flush */
David Woodhouse1f0ef2a2009-05-10 19:58:49 +01001266static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1267 u64 addr, unsigned int size_order, u64 type)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001268{
1269 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1270 u64 val = 0, val_iva = 0;
1271 unsigned long flag;
1272
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001273 switch (type) {
1274 case DMA_TLB_GLOBAL_FLUSH:
1275 /* global flush doesn't need set IVA_REG */
1276 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1277 break;
1278 case DMA_TLB_DSI_FLUSH:
1279 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1280 break;
1281 case DMA_TLB_PSI_FLUSH:
1282 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
David Woodhouseea8ea462014-03-05 17:09:32 +00001283 /* IH bit is passed in as part of address */
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001284 val_iva = size_order | addr;
1285 break;
1286 default:
1287 BUG();
1288 }
1289 /* Note: set drain read/write */
1290#if 0
1291 /*
1292 * This is probably to be super secure.. Looks like we can
1293 * ignore it without any impact.
1294 */
1295 if (cap_read_drain(iommu->cap))
1296 val |= DMA_TLB_READ_DRAIN;
1297#endif
1298 if (cap_write_drain(iommu->cap))
1299 val |= DMA_TLB_WRITE_DRAIN;
1300
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001301 raw_spin_lock_irqsave(&iommu->register_lock, flag);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001302 /* Note: Only uses first TLB reg currently */
1303 if (val_iva)
1304 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1305 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1306
1307 /* Make sure hardware complete it */
1308 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1309 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1310
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001311 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001312
1313 /* check IOTLB invalidation granularity */
1314 if (DMA_TLB_IAIG(val) == 0)
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02001315 pr_err("Flush IOTLB failed\n");
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001316 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02001317 pr_debug("TLB flush request %Lx, actual %Lx\n",
Fenghua Yu5b6985c2008-10-16 18:02:32 -07001318 (unsigned long long)DMA_TLB_IIRG(type),
1319 (unsigned long long)DMA_TLB_IAIG(val));
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001320}
1321
David Woodhouse64ae8922014-03-09 12:52:30 -07001322static struct device_domain_info *
1323iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1324 u8 bus, u8 devfn)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001325{
Yu Zhao93a23a72009-05-18 13:51:37 +08001326 struct device_domain_info *info;
Yu Zhao93a23a72009-05-18 13:51:37 +08001327
Joerg Roedel55d94042015-07-22 16:50:40 +02001328 assert_spin_locked(&device_domain_lock);
1329
Yu Zhao93a23a72009-05-18 13:51:37 +08001330 if (!iommu->qi)
1331 return NULL;
1332
Yu Zhao93a23a72009-05-18 13:51:37 +08001333 list_for_each_entry(info, &domain->devices, link)
Jiang Liuc3b497c2014-07-11 14:19:25 +08001334 if (info->iommu == iommu && info->bus == bus &&
1335 info->devfn == devfn) {
David Woodhouseb16d0cb2015-10-12 14:17:37 +01001336 if (info->ats_supported && info->dev)
1337 return info;
Yu Zhao93a23a72009-05-18 13:51:37 +08001338 break;
1339 }
Yu Zhao93a23a72009-05-18 13:51:37 +08001340
David Woodhouseb16d0cb2015-10-12 14:17:37 +01001341 return NULL;
Yu Zhao93a23a72009-05-18 13:51:37 +08001342}
1343
Omer Peleg0824c592016-04-20 19:03:35 +03001344static void domain_update_iotlb(struct dmar_domain *domain)
1345{
1346 struct device_domain_info *info;
1347 bool has_iotlb_device = false;
1348
1349 assert_spin_locked(&device_domain_lock);
1350
1351 list_for_each_entry(info, &domain->devices, link) {
1352 struct pci_dev *pdev;
1353
1354 if (!info->dev || !dev_is_pci(info->dev))
1355 continue;
1356
1357 pdev = to_pci_dev(info->dev);
1358 if (pdev->ats_enabled) {
1359 has_iotlb_device = true;
1360 break;
1361 }
1362 }
1363
1364 domain->has_iotlb_device = has_iotlb_device;
1365}
1366
Yu Zhao93a23a72009-05-18 13:51:37 +08001367static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1368{
Bjorn Helgaasfb0cc3a2015-07-20 09:10:36 -05001369 struct pci_dev *pdev;
1370
Omer Peleg0824c592016-04-20 19:03:35 +03001371 assert_spin_locked(&device_domain_lock);
1372
David Woodhouse0bcb3e22014-03-06 17:12:03 +00001373 if (!info || !dev_is_pci(info->dev))
Yu Zhao93a23a72009-05-18 13:51:37 +08001374 return;
1375
Bjorn Helgaasfb0cc3a2015-07-20 09:10:36 -05001376 pdev = to_pci_dev(info->dev);
Jacob Pan1c48db42018-06-07 09:57:00 -07001377 /* For IOMMU that supports device IOTLB throttling (DIT), we assign
1378 * PFSID to the invalidation desc of a VF such that IOMMU HW can gauge
1379 * queue depth at PF level. If DIT is not set, PFSID will be treated as
1380 * reserved, which should be set to 0.
1381 */
1382 if (!ecap_dit(info->iommu->ecap))
1383 info->pfsid = 0;
1384 else {
1385 struct pci_dev *pf_pdev;
1386
1387 /* pdev will be returned if device is not a vf */
1388 pf_pdev = pci_physfn(pdev);
1389 info->pfsid = PCI_DEVID(pf_pdev->bus->number, pf_pdev->devfn);
1390 }
Bjorn Helgaasfb0cc3a2015-07-20 09:10:36 -05001391
David Woodhouseb16d0cb2015-10-12 14:17:37 +01001392#ifdef CONFIG_INTEL_IOMMU_SVM
1393 /* The PCIe spec, in its wisdom, declares that the behaviour of
1394 the device if you enable PASID support after ATS support is
1395 undefined. So always enable PASID support on devices which
1396 have it, even if we can't yet know if we're ever going to
1397 use it. */
1398 if (info->pasid_supported && !pci_enable_pasid(pdev, info->pasid_supported & ~1))
1399 info->pasid_enabled = 1;
1400
1401 if (info->pri_supported && !pci_reset_pri(pdev) && !pci_enable_pri(pdev, 32))
1402 info->pri_enabled = 1;
1403#endif
1404 if (info->ats_supported && !pci_enable_ats(pdev, VTD_PAGE_SHIFT)) {
1405 info->ats_enabled = 1;
Omer Peleg0824c592016-04-20 19:03:35 +03001406 domain_update_iotlb(info->domain);
David Woodhouseb16d0cb2015-10-12 14:17:37 +01001407 info->ats_qdep = pci_ats_queue_depth(pdev);
1408 }
Yu Zhao93a23a72009-05-18 13:51:37 +08001409}
1410
1411static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1412{
David Woodhouseb16d0cb2015-10-12 14:17:37 +01001413 struct pci_dev *pdev;
1414
Omer Peleg0824c592016-04-20 19:03:35 +03001415 assert_spin_locked(&device_domain_lock);
1416
Jeremy McNicollda972fb2016-01-14 21:33:06 -08001417 if (!dev_is_pci(info->dev))
Yu Zhao93a23a72009-05-18 13:51:37 +08001418 return;
1419
David Woodhouseb16d0cb2015-10-12 14:17:37 +01001420 pdev = to_pci_dev(info->dev);
1421
1422 if (info->ats_enabled) {
1423 pci_disable_ats(pdev);
1424 info->ats_enabled = 0;
Omer Peleg0824c592016-04-20 19:03:35 +03001425 domain_update_iotlb(info->domain);
David Woodhouseb16d0cb2015-10-12 14:17:37 +01001426 }
1427#ifdef CONFIG_INTEL_IOMMU_SVM
1428 if (info->pri_enabled) {
1429 pci_disable_pri(pdev);
1430 info->pri_enabled = 0;
1431 }
1432 if (info->pasid_enabled) {
1433 pci_disable_pasid(pdev);
1434 info->pasid_enabled = 0;
1435 }
1436#endif
Yu Zhao93a23a72009-05-18 13:51:37 +08001437}
1438
1439static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1440 u64 addr, unsigned mask)
1441{
1442 u16 sid, qdep;
1443 unsigned long flags;
1444 struct device_domain_info *info;
1445
Omer Peleg0824c592016-04-20 19:03:35 +03001446 if (!domain->has_iotlb_device)
1447 return;
1448
Yu Zhao93a23a72009-05-18 13:51:37 +08001449 spin_lock_irqsave(&device_domain_lock, flags);
1450 list_for_each_entry(info, &domain->devices, link) {
David Woodhouseb16d0cb2015-10-12 14:17:37 +01001451 if (!info->ats_enabled)
Yu Zhao93a23a72009-05-18 13:51:37 +08001452 continue;
1453
1454 sid = info->bus << 8 | info->devfn;
David Woodhouseb16d0cb2015-10-12 14:17:37 +01001455 qdep = info->ats_qdep;
Jacob Pan1c48db42018-06-07 09:57:00 -07001456 qi_flush_dev_iotlb(info->iommu, sid, info->pfsid,
1457 qdep, addr, mask);
Yu Zhao93a23a72009-05-18 13:51:37 +08001458 }
1459 spin_unlock_irqrestore(&device_domain_lock, flags);
1460}
1461
Joerg Roedela1ddcbe2015-07-21 15:20:32 +02001462static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
1463 struct dmar_domain *domain,
1464 unsigned long pfn, unsigned int pages,
1465 int ih, int map)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001466{
Yu Zhao9dd2fe82009-05-18 13:51:36 +08001467 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
David Woodhouse03d6a242009-06-28 15:33:46 +01001468 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
Joerg Roedela1ddcbe2015-07-21 15:20:32 +02001469 u16 did = domain->iommu_did[iommu->seq_id];
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001470
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001471 BUG_ON(pages == 0);
1472
David Woodhouseea8ea462014-03-05 17:09:32 +00001473 if (ih)
1474 ih = 1 << 6;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001475 /*
Yu Zhao9dd2fe82009-05-18 13:51:36 +08001476 * Fallback to domain selective flush if no PSI support or the size is
1477 * too big.
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001478 * PSI requires page size to be 2 ^ x, and the base address is naturally
1479 * aligned to the size
1480 */
Yu Zhao9dd2fe82009-05-18 13:51:36 +08001481 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1482 iommu->flush.flush_iotlb(iommu, did, 0, 0,
David Woodhouse1f0ef2a2009-05-10 19:58:49 +01001483 DMA_TLB_DSI_FLUSH);
Yu Zhao9dd2fe82009-05-18 13:51:36 +08001484 else
David Woodhouseea8ea462014-03-05 17:09:32 +00001485 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
Yu Zhao9dd2fe82009-05-18 13:51:36 +08001486 DMA_TLB_PSI_FLUSH);
Yu Zhaobf92df32009-06-29 11:31:45 +08001487
1488 /*
Nadav Amit82653632010-04-01 13:24:40 +03001489 * In caching mode, changes of pages from non-present to present require
1490 * flush. However, device IOTLB doesn't need to be flushed in this case.
Yu Zhaobf92df32009-06-29 11:31:45 +08001491 */
Nadav Amit82653632010-04-01 13:24:40 +03001492 if (!cap_caching_mode(iommu->cap) || !map)
Peter Xu9d2e6502018-01-10 13:51:37 +08001493 iommu_flush_dev_iotlb(domain, addr, mask);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001494}
1495
Peter Xueed91a02018-05-04 10:34:52 +08001496/* Notification for newly created mappings */
1497static inline void __mapping_notify_one(struct intel_iommu *iommu,
1498 struct dmar_domain *domain,
1499 unsigned long pfn, unsigned int pages)
1500{
1501 /* It's a non-present to present mapping. Only flush if caching mode */
1502 if (cap_caching_mode(iommu->cap))
1503 iommu_flush_iotlb_psi(iommu, domain, pfn, pages, 0, 1);
1504 else
1505 iommu_flush_write_buffer(iommu);
1506}
1507
Joerg Roedel13cf0172017-08-11 11:40:10 +02001508static void iommu_flush_iova(struct iova_domain *iovad)
1509{
1510 struct dmar_domain *domain;
1511 int idx;
1512
1513 domain = container_of(iovad, struct dmar_domain, iovad);
1514
1515 for_each_domain_iommu(idx, domain) {
1516 struct intel_iommu *iommu = g_iommus[idx];
1517 u16 did = domain->iommu_did[iommu->seq_id];
1518
1519 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
1520
1521 if (!cap_caching_mode(iommu->cap))
1522 iommu_flush_dev_iotlb(get_iommu_domain(iommu, did),
1523 0, MAX_AGAW_PFN_WIDTH);
1524 }
1525}
1526
mark grossf8bab732008-02-08 04:18:38 -08001527static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1528{
1529 u32 pmen;
1530 unsigned long flags;
1531
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001532 raw_spin_lock_irqsave(&iommu->register_lock, flags);
mark grossf8bab732008-02-08 04:18:38 -08001533 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1534 pmen &= ~DMA_PMEN_EPM;
1535 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1536
1537 /* wait for the protected region status bit to clear */
1538 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1539 readl, !(pmen & DMA_PMEN_PRS), pmen);
1540
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001541 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
mark grossf8bab732008-02-08 04:18:38 -08001542}
1543
Jiang Liu2a41cce2014-07-11 14:19:33 +08001544static void iommu_enable_translation(struct intel_iommu *iommu)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001545{
1546 u32 sts;
1547 unsigned long flags;
1548
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001549 raw_spin_lock_irqsave(&iommu->register_lock, flags);
David Woodhousec416daa2009-05-10 20:30:58 +01001550 iommu->gcmd |= DMA_GCMD_TE;
1551 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001552
1553 /* Make sure hardware complete it */
1554 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
David Woodhousec416daa2009-05-10 20:30:58 +01001555 readl, (sts & DMA_GSTS_TES), sts);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001556
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001557 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001558}
1559
Jiang Liu2a41cce2014-07-11 14:19:33 +08001560static void iommu_disable_translation(struct intel_iommu *iommu)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001561{
1562 u32 sts;
1563 unsigned long flag;
1564
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001565 raw_spin_lock_irqsave(&iommu->register_lock, flag);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001566 iommu->gcmd &= ~DMA_GCMD_TE;
1567 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1568
1569 /* Make sure hardware complete it */
1570 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
David Woodhousec416daa2009-05-10 20:30:58 +01001571 readl, (!(sts & DMA_GSTS_TES)), sts);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001572
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001573 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001574}
1575
Keshavamurthy, Anil S3460a6d2007-10-21 16:41:54 -07001576
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001577static int iommu_init_domains(struct intel_iommu *iommu)
1578{
Joerg Roedel8bf47812015-07-21 10:41:21 +02001579 u32 ndomains, nlongs;
1580 size_t size;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001581
1582 ndomains = cap_ndoms(iommu->cap);
Joerg Roedel8bf47812015-07-21 10:41:21 +02001583 pr_debug("%s: Number of Domains supported <%d>\n",
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02001584 iommu->name, ndomains);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001585 nlongs = BITS_TO_LONGS(ndomains);
1586
Donald Dutile94a91b502009-08-20 16:51:34 -04001587 spin_lock_init(&iommu->lock);
1588
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001589 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1590 if (!iommu->domain_ids) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02001591 pr_err("%s: Allocating domain id array failed\n",
1592 iommu->name);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001593 return -ENOMEM;
1594 }
Joerg Roedel8bf47812015-07-21 10:41:21 +02001595
Wei Yang86f004c2016-05-21 02:41:51 +00001596 size = (ALIGN(ndomains, 256) >> 8) * sizeof(struct dmar_domain **);
Joerg Roedel8bf47812015-07-21 10:41:21 +02001597 iommu->domains = kzalloc(size, GFP_KERNEL);
1598
1599 if (iommu->domains) {
1600 size = 256 * sizeof(struct dmar_domain *);
1601 iommu->domains[0] = kzalloc(size, GFP_KERNEL);
1602 }
1603
1604 if (!iommu->domains || !iommu->domains[0]) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02001605 pr_err("%s: Allocating domain array failed\n",
1606 iommu->name);
Jiang Liu852bdb02014-01-06 14:18:11 +08001607 kfree(iommu->domain_ids);
Joerg Roedel8bf47812015-07-21 10:41:21 +02001608 kfree(iommu->domains);
Jiang Liu852bdb02014-01-06 14:18:11 +08001609 iommu->domain_ids = NULL;
Joerg Roedel8bf47812015-07-21 10:41:21 +02001610 iommu->domains = NULL;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001611 return -ENOMEM;
1612 }
1613
Joerg Roedel8bf47812015-07-21 10:41:21 +02001614
1615
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001616 /*
Joerg Roedelc0e8a6c2015-07-21 09:39:46 +02001617 * If Caching mode is set, then invalid translations are tagged
1618 * with domain-id 0, hence we need to pre-allocate it. We also
1619 * use domain-id 0 as a marker for non-allocated domain-id, so
1620 * make sure it is not used for a real domain.
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001621 */
Joerg Roedelc0e8a6c2015-07-21 09:39:46 +02001622 set_bit(0, iommu->domain_ids);
1623
Lu Baolu3b33d4a2018-12-10 09:58:59 +08001624 /*
1625 * Vt-d spec rev3.0 (section 6.2.3.1) requires that each pasid
1626 * entry for first-level or pass-through translation modes should
1627 * be programmed with a domain id different from those used for
1628 * second-level or nested translation. We reserve a domain id for
1629 * this purpose.
1630 */
1631 if (sm_supported(iommu))
1632 set_bit(FLPT_DEFAULT_DID, iommu->domain_ids);
1633
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001634 return 0;
1635}
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001636
Jiang Liuffebeb42014-11-09 22:48:02 +08001637static void disable_dmar_iommu(struct intel_iommu *iommu)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001638{
Joerg Roedel29a27712015-07-21 17:17:12 +02001639 struct device_domain_info *info, *tmp;
Joerg Roedel55d94042015-07-22 16:50:40 +02001640 unsigned long flags;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001641
Joerg Roedel29a27712015-07-21 17:17:12 +02001642 if (!iommu->domains || !iommu->domain_ids)
1643 return;
Jiang Liua4eaa862014-02-19 14:07:30 +08001644
Joerg Roedelbea64032016-11-08 15:08:26 +01001645again:
Joerg Roedel55d94042015-07-22 16:50:40 +02001646 spin_lock_irqsave(&device_domain_lock, flags);
Joerg Roedel29a27712015-07-21 17:17:12 +02001647 list_for_each_entry_safe(info, tmp, &device_domain_list, global) {
1648 struct dmar_domain *domain;
1649
1650 if (info->iommu != iommu)
1651 continue;
1652
1653 if (!info->dev || !info->domain)
1654 continue;
1655
1656 domain = info->domain;
1657
Joerg Roedelbea64032016-11-08 15:08:26 +01001658 __dmar_remove_one_dev_info(info);
Joerg Roedel29a27712015-07-21 17:17:12 +02001659
Joerg Roedelbea64032016-11-08 15:08:26 +01001660 if (!domain_type_is_vm_or_si(domain)) {
1661 /*
1662 * The domain_exit() function can't be called under
1663 * device_domain_lock, as it takes this lock itself.
1664 * So release the lock here and re-run the loop
1665 * afterwards.
1666 */
1667 spin_unlock_irqrestore(&device_domain_lock, flags);
Joerg Roedel29a27712015-07-21 17:17:12 +02001668 domain_exit(domain);
Joerg Roedelbea64032016-11-08 15:08:26 +01001669 goto again;
1670 }
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001671 }
Joerg Roedel55d94042015-07-22 16:50:40 +02001672 spin_unlock_irqrestore(&device_domain_lock, flags);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001673
1674 if (iommu->gcmd & DMA_GCMD_TE)
1675 iommu_disable_translation(iommu);
Jiang Liuffebeb42014-11-09 22:48:02 +08001676}
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001677
Jiang Liuffebeb42014-11-09 22:48:02 +08001678static void free_dmar_iommu(struct intel_iommu *iommu)
1679{
1680 if ((iommu->domains) && (iommu->domain_ids)) {
Wei Yang86f004c2016-05-21 02:41:51 +00001681 int elems = ALIGN(cap_ndoms(iommu->cap), 256) >> 8;
Joerg Roedel8bf47812015-07-21 10:41:21 +02001682 int i;
1683
1684 for (i = 0; i < elems; i++)
1685 kfree(iommu->domains[i]);
Jiang Liuffebeb42014-11-09 22:48:02 +08001686 kfree(iommu->domains);
1687 kfree(iommu->domain_ids);
1688 iommu->domains = NULL;
1689 iommu->domain_ids = NULL;
1690 }
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001691
Weidong Hand9630fe2008-12-08 11:06:32 +08001692 g_iommus[iommu->seq_id] = NULL;
1693
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001694 /* free context mapping */
1695 free_context_table(iommu);
David Woodhouse8a94ade2015-03-24 14:54:56 +00001696
1697#ifdef CONFIG_INTEL_IOMMU_SVM
Lu Baolu765b6a92018-12-10 09:58:55 +08001698 if (pasid_supported(iommu)) {
David Woodhousea222a7f2015-10-07 23:35:18 +01001699 if (ecap_prs(iommu->ecap))
1700 intel_svm_finish_prq(iommu);
Lu Baolud9737952018-07-14 15:47:02 +08001701 intel_svm_exit(iommu);
David Woodhousea222a7f2015-10-07 23:35:18 +01001702 }
David Woodhouse8a94ade2015-03-24 14:54:56 +00001703#endif
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001704}
1705
Jiang Liuab8dfe22014-07-11 14:19:27 +08001706static struct dmar_domain *alloc_domain(int flags)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001707{
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001708 struct dmar_domain *domain;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001709
1710 domain = alloc_domain_mem();
1711 if (!domain)
1712 return NULL;
1713
Jiang Liuab8dfe22014-07-11 14:19:27 +08001714 memset(domain, 0, sizeof(*domain));
Suresh Siddha4c923d42009-10-02 11:01:24 -07001715 domain->nid = -1;
Jiang Liuab8dfe22014-07-11 14:19:27 +08001716 domain->flags = flags;
Omer Peleg0824c592016-04-20 19:03:35 +03001717 domain->has_iotlb_device = false;
Jiang Liu92d03cc2014-02-19 14:07:28 +08001718 INIT_LIST_HEAD(&domain->devices);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001719
1720 return domain;
1721}
1722
Joerg Roedeld160aca2015-07-22 11:52:53 +02001723/* Must be called with iommu->lock */
1724static int domain_attach_iommu(struct dmar_domain *domain,
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07001725 struct intel_iommu *iommu)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001726{
Jiang Liu44bde612014-07-11 14:19:29 +08001727 unsigned long ndomains;
Joerg Roedel55d94042015-07-22 16:50:40 +02001728 int num;
Jiang Liu44bde612014-07-11 14:19:29 +08001729
Joerg Roedel55d94042015-07-22 16:50:40 +02001730 assert_spin_locked(&device_domain_lock);
Joerg Roedeld160aca2015-07-22 11:52:53 +02001731 assert_spin_locked(&iommu->lock);
Jiang Liu44bde612014-07-11 14:19:29 +08001732
Joerg Roedel29a27712015-07-21 17:17:12 +02001733 domain->iommu_refcnt[iommu->seq_id] += 1;
1734 domain->iommu_count += 1;
1735 if (domain->iommu_refcnt[iommu->seq_id] == 1) {
Jiang Liufb170fb2014-07-11 14:19:28 +08001736 ndomains = cap_ndoms(iommu->cap);
Joerg Roedeld160aca2015-07-22 11:52:53 +02001737 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1738
1739 if (num >= ndomains) {
1740 pr_err("%s: No free domain ids\n", iommu->name);
1741 domain->iommu_refcnt[iommu->seq_id] -= 1;
1742 domain->iommu_count -= 1;
Joerg Roedel55d94042015-07-22 16:50:40 +02001743 return -ENOSPC;
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07001744 }
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001745
Joerg Roedeld160aca2015-07-22 11:52:53 +02001746 set_bit(num, iommu->domain_ids);
1747 set_iommu_domain(iommu, num, domain);
Jiang Liufb170fb2014-07-11 14:19:28 +08001748
Joerg Roedeld160aca2015-07-22 11:52:53 +02001749 domain->iommu_did[iommu->seq_id] = num;
1750 domain->nid = iommu->node;
1751
Jiang Liufb170fb2014-07-11 14:19:28 +08001752 domain_update_iommu_cap(domain);
1753 }
Joerg Roedeld160aca2015-07-22 11:52:53 +02001754
Joerg Roedel55d94042015-07-22 16:50:40 +02001755 return 0;
Jiang Liufb170fb2014-07-11 14:19:28 +08001756}
1757
1758static int domain_detach_iommu(struct dmar_domain *domain,
1759 struct intel_iommu *iommu)
1760{
Joerg Roedeld160aca2015-07-22 11:52:53 +02001761 int num, count = INT_MAX;
Jiang Liufb170fb2014-07-11 14:19:28 +08001762
Joerg Roedel55d94042015-07-22 16:50:40 +02001763 assert_spin_locked(&device_domain_lock);
Joerg Roedeld160aca2015-07-22 11:52:53 +02001764 assert_spin_locked(&iommu->lock);
Jiang Liufb170fb2014-07-11 14:19:28 +08001765
Joerg Roedel29a27712015-07-21 17:17:12 +02001766 domain->iommu_refcnt[iommu->seq_id] -= 1;
1767 count = --domain->iommu_count;
1768 if (domain->iommu_refcnt[iommu->seq_id] == 0) {
Joerg Roedeld160aca2015-07-22 11:52:53 +02001769 num = domain->iommu_did[iommu->seq_id];
1770 clear_bit(num, iommu->domain_ids);
1771 set_iommu_domain(iommu, num, NULL);
1772
Jiang Liufb170fb2014-07-11 14:19:28 +08001773 domain_update_iommu_cap(domain);
Joerg Roedelc0e8a6c2015-07-21 09:39:46 +02001774 domain->iommu_did[iommu->seq_id] = 0;
Jiang Liufb170fb2014-07-11 14:19:28 +08001775 }
Jiang Liufb170fb2014-07-11 14:19:28 +08001776
1777 return count;
1778}
1779
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001780static struct iova_domain reserved_iova_list;
Mark Gross8a443df2008-03-04 14:59:31 -08001781static struct lock_class_key reserved_rbtree_key;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001782
Joseph Cihula51a63e62011-03-21 11:04:24 -07001783static int dmar_init_reserved_ranges(void)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001784{
1785 struct pci_dev *pdev = NULL;
1786 struct iova *iova;
1787 int i;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001788
Zhen Leiaa3ac942017-09-21 16:52:45 +01001789 init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001790
Mark Gross8a443df2008-03-04 14:59:31 -08001791 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1792 &reserved_rbtree_key);
1793
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001794 /* IOAPIC ranges shouldn't be accessed by DMA */
1795 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1796 IOVA_PFN(IOAPIC_RANGE_END));
Joseph Cihula51a63e62011-03-21 11:04:24 -07001797 if (!iova) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02001798 pr_err("Reserve IOAPIC range failed\n");
Joseph Cihula51a63e62011-03-21 11:04:24 -07001799 return -ENODEV;
1800 }
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001801
1802 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1803 for_each_pci_dev(pdev) {
1804 struct resource *r;
1805
1806 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1807 r = &pdev->resource[i];
1808 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1809 continue;
David Woodhouse1a4a4552009-06-28 16:00:42 +01001810 iova = reserve_iova(&reserved_iova_list,
1811 IOVA_PFN(r->start),
1812 IOVA_PFN(r->end));
Joseph Cihula51a63e62011-03-21 11:04:24 -07001813 if (!iova) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02001814 pr_err("Reserve iova failed\n");
Joseph Cihula51a63e62011-03-21 11:04:24 -07001815 return -ENODEV;
1816 }
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001817 }
1818 }
Joseph Cihula51a63e62011-03-21 11:04:24 -07001819 return 0;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001820}
1821
1822static void domain_reserve_special_ranges(struct dmar_domain *domain)
1823{
1824 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1825}
1826
1827static inline int guestwidth_to_adjustwidth(int gaw)
1828{
1829 int agaw;
1830 int r = (gaw - 12) % 9;
1831
1832 if (r == 0)
1833 agaw = gaw;
1834 else
1835 agaw = gaw + 9 - r;
1836 if (agaw > 64)
1837 agaw = 64;
1838 return agaw;
1839}
1840
Joerg Roedeldc534b22015-07-22 12:44:02 +02001841static int domain_init(struct dmar_domain *domain, struct intel_iommu *iommu,
1842 int guest_width)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001843{
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001844 int adjust_width, agaw;
1845 unsigned long sagaw;
Joerg Roedel13cf0172017-08-11 11:40:10 +02001846 int err;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001847
Zhen Leiaa3ac942017-09-21 16:52:45 +01001848 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN);
Joerg Roedel13cf0172017-08-11 11:40:10 +02001849
1850 err = init_iova_flush_queue(&domain->iovad,
1851 iommu_flush_iova, iova_entry_free);
1852 if (err)
1853 return err;
1854
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001855 domain_reserve_special_ranges(domain);
1856
1857 /* calculate AGAW */
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001858 if (guest_width > cap_mgaw(iommu->cap))
1859 guest_width = cap_mgaw(iommu->cap);
1860 domain->gaw = guest_width;
1861 adjust_width = guestwidth_to_adjustwidth(guest_width);
1862 agaw = width_to_agaw(adjust_width);
1863 sagaw = cap_sagaw(iommu->cap);
1864 if (!test_bit(agaw, &sagaw)) {
1865 /* hardware doesn't support it, choose a bigger one */
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02001866 pr_debug("Hardware doesn't support agaw %d\n", agaw);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001867 agaw = find_next_bit(&sagaw, 5, agaw);
1868 if (agaw >= 5)
1869 return -ENODEV;
1870 }
1871 domain->agaw = agaw;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001872
Weidong Han8e6040972008-12-08 15:49:06 +08001873 if (ecap_coherent(iommu->ecap))
1874 domain->iommu_coherency = 1;
1875 else
1876 domain->iommu_coherency = 0;
1877
Sheng Yang58c610b2009-03-18 15:33:05 +08001878 if (ecap_sc_support(iommu->ecap))
1879 domain->iommu_snooping = 1;
1880 else
1881 domain->iommu_snooping = 0;
1882
David Woodhouse214e39a2014-03-19 10:38:49 +00001883 if (intel_iommu_superpage)
1884 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1885 else
1886 domain->iommu_superpage = 0;
1887
Suresh Siddha4c923d42009-10-02 11:01:24 -07001888 domain->nid = iommu->node;
Weidong Hanc7151a82008-12-08 22:51:37 +08001889
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001890 /* always allocate the top pgd */
Suresh Siddha4c923d42009-10-02 11:01:24 -07001891 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001892 if (!domain->pgd)
1893 return -ENOMEM;
Fenghua Yu5b6985c2008-10-16 18:02:32 -07001894 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001895 return 0;
1896}
1897
1898static void domain_exit(struct dmar_domain *domain)
1899{
David Woodhouseea8ea462014-03-05 17:09:32 +00001900 struct page *freelist = NULL;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001901
1902 /* Domain 0 is reserved, so dont process it */
1903 if (!domain)
1904 return;
1905
Joerg Roedeld160aca2015-07-22 11:52:53 +02001906 /* Remove associated devices and clear attached or cached domains */
1907 rcu_read_lock();
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001908 domain_remove_dev_info(domain);
Joerg Roedeld160aca2015-07-22 11:52:53 +02001909 rcu_read_unlock();
Jiang Liu92d03cc2014-02-19 14:07:28 +08001910
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001911 /* destroy iovas */
1912 put_iova_domain(&domain->iovad);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001913
David Woodhouseea8ea462014-03-05 17:09:32 +00001914 freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001915
David Woodhouseea8ea462014-03-05 17:09:32 +00001916 dma_free_pagelist(freelist);
1917
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001918 free_domain_mem(domain);
1919}
1920
David Woodhouse64ae8922014-03-09 12:52:30 -07001921static int domain_context_mapping_one(struct dmar_domain *domain,
1922 struct intel_iommu *iommu,
Lu Baoluca6e3222018-12-10 09:59:02 +08001923 struct pasid_table *table,
Joerg Roedel28ccce02015-07-21 14:45:31 +02001924 u8 bus, u8 devfn)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001925{
Joerg Roedelc6c2ceb2015-07-22 13:11:53 +02001926 u16 did = domain->iommu_did[iommu->seq_id];
Joerg Roedel28ccce02015-07-21 14:45:31 +02001927 int translation = CONTEXT_TT_MULTI_LEVEL;
1928 struct device_domain_info *info = NULL;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001929 struct context_entry *context;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001930 unsigned long flags;
Weidong Hanea6606b2008-12-08 23:08:15 +08001931 struct dma_pte *pgd;
Joerg Roedel55d94042015-07-22 16:50:40 +02001932 int ret, agaw;
Joerg Roedel28ccce02015-07-21 14:45:31 +02001933
Joerg Roedelc6c2ceb2015-07-22 13:11:53 +02001934 WARN_ON(did == 0);
1935
Joerg Roedel28ccce02015-07-21 14:45:31 +02001936 if (hw_pass_through && domain_type_is_si(domain))
1937 translation = CONTEXT_TT_PASS_THROUGH;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001938
1939 pr_debug("Set context mapping for %02x:%02x.%d\n",
1940 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07001941
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001942 BUG_ON(!domain->pgd);
Weidong Han5331fe62008-12-08 23:00:00 +08001943
Joerg Roedel55d94042015-07-22 16:50:40 +02001944 spin_lock_irqsave(&device_domain_lock, flags);
1945 spin_lock(&iommu->lock);
1946
1947 ret = -ENOMEM;
David Woodhouse03ecc322015-02-13 14:35:21 +00001948 context = iommu_context_addr(iommu, bus, devfn, 1);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001949 if (!context)
Joerg Roedel55d94042015-07-22 16:50:40 +02001950 goto out_unlock;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001951
Joerg Roedel55d94042015-07-22 16:50:40 +02001952 ret = 0;
1953 if (context_present(context))
1954 goto out_unlock;
Joerg Roedelcf484d02015-06-12 12:21:46 +02001955
Xunlei Pangaec0e862016-12-05 20:09:07 +08001956 /*
1957 * For kdump cases, old valid entries may be cached due to the
1958 * in-flight DMA and copied pgtable, but there is no unmapping
1959 * behaviour for them, thus we need an explicit cache flush for
1960 * the newly-mapped device. For kdump, at this point, the device
1961 * is supposed to finish reset at its driver probe stage, so no
1962 * in-flight DMA will exist, and we don't need to worry anymore
1963 * hereafter.
1964 */
1965 if (context_copied(context)) {
1966 u16 did_old = context_domain_id(context);
1967
Christos Gkekasb117e032017-10-08 23:33:31 +01001968 if (did_old < cap_ndoms(iommu->cap)) {
Xunlei Pangaec0e862016-12-05 20:09:07 +08001969 iommu->flush.flush_context(iommu, did_old,
1970 (((u16)bus) << 8) | devfn,
1971 DMA_CCMD_MASK_NOBIT,
1972 DMA_CCMD_DEVICE_INVL);
KarimAllah Ahmedf73a7ee2017-05-05 11:39:59 -07001973 iommu->flush.flush_iotlb(iommu, did_old, 0, 0,
1974 DMA_TLB_DSI_FLUSH);
1975 }
Xunlei Pangaec0e862016-12-05 20:09:07 +08001976 }
1977
Weidong Hanea6606b2008-12-08 23:08:15 +08001978 pgd = domain->pgd;
1979
Joerg Roedelde24e552015-07-21 14:53:04 +02001980 context_clear_entry(context);
Joerg Roedelc6c2ceb2015-07-22 13:11:53 +02001981 context_set_domain_id(context, did);
Weidong Hanea6606b2008-12-08 23:08:15 +08001982
Joerg Roedelde24e552015-07-21 14:53:04 +02001983 /*
1984 * Skip top levels of page tables for iommu which has less agaw
1985 * than default. Unnecessary for PT mode.
1986 */
Yu Zhao93a23a72009-05-18 13:51:37 +08001987 if (translation != CONTEXT_TT_PASS_THROUGH) {
Sohil Mehta3569dd02018-11-21 15:29:33 -08001988 for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) {
Joerg Roedel55d94042015-07-22 16:50:40 +02001989 ret = -ENOMEM;
Joerg Roedelde24e552015-07-21 14:53:04 +02001990 pgd = phys_to_virt(dma_pte_addr(pgd));
Joerg Roedel55d94042015-07-22 16:50:40 +02001991 if (!dma_pte_present(pgd))
1992 goto out_unlock;
Joerg Roedelde24e552015-07-21 14:53:04 +02001993 }
1994
David Woodhouse64ae8922014-03-09 12:52:30 -07001995 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
David Woodhouseb16d0cb2015-10-12 14:17:37 +01001996 if (info && info->ats_supported)
1997 translation = CONTEXT_TT_DEV_IOTLB;
1998 else
1999 translation = CONTEXT_TT_MULTI_LEVEL;
Joerg Roedelde24e552015-07-21 14:53:04 +02002000
Yu Zhao93a23a72009-05-18 13:51:37 +08002001 context_set_address_root(context, virt_to_phys(pgd));
Sohil Mehta3569dd02018-11-21 15:29:33 -08002002 context_set_address_width(context, agaw);
Joerg Roedelde24e552015-07-21 14:53:04 +02002003 } else {
2004 /*
2005 * In pass through mode, AW must be programmed to
2006 * indicate the largest AGAW value supported by
2007 * hardware. And ASR is ignored by hardware.
2008 */
2009 context_set_address_width(context, iommu->msagaw);
Yu Zhao93a23a72009-05-18 13:51:37 +08002010 }
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07002011
2012 context_set_translation_type(context, translation);
Mark McLoughlinc07e7d22008-11-21 16:54:46 +00002013 context_set_fault_enable(context);
2014 context_set_present(context);
Weidong Han5331fe62008-12-08 23:00:00 +08002015 domain_flush_cache(domain, context, sizeof(*context));
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002016
David Woodhouse4c25a2c2009-05-10 17:16:06 +01002017 /*
2018 * It's a non-present to present mapping. If hardware doesn't cache
2019 * non-present entry we only need to flush the write-buffer. If the
2020 * _does_ cache non-present entries, then it does so in the special
2021 * domain #0, which we have to flush:
2022 */
2023 if (cap_caching_mode(iommu->cap)) {
2024 iommu->flush.flush_context(iommu, 0,
2025 (((u16)bus) << 8) | devfn,
2026 DMA_CCMD_MASK_NOBIT,
2027 DMA_CCMD_DEVICE_INVL);
Joerg Roedelc6c2ceb2015-07-22 13:11:53 +02002028 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
David Woodhouse4c25a2c2009-05-10 17:16:06 +01002029 } else {
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002030 iommu_flush_write_buffer(iommu);
David Woodhouse4c25a2c2009-05-10 17:16:06 +01002031 }
Yu Zhao93a23a72009-05-18 13:51:37 +08002032 iommu_enable_dev_iotlb(info);
Weidong Hanc7151a82008-12-08 22:51:37 +08002033
Joerg Roedel55d94042015-07-22 16:50:40 +02002034 ret = 0;
2035
2036out_unlock:
2037 spin_unlock(&iommu->lock);
2038 spin_unlock_irqrestore(&device_domain_lock, flags);
Jiang Liufb170fb2014-07-11 14:19:28 +08002039
Wei Yang5c365d12016-07-13 13:53:21 +00002040 return ret;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002041}
2042
Alex Williamson579305f2014-07-03 09:51:43 -06002043struct domain_context_mapping_data {
2044 struct dmar_domain *domain;
2045 struct intel_iommu *iommu;
Lu Baoluca6e3222018-12-10 09:59:02 +08002046 struct pasid_table *table;
Alex Williamson579305f2014-07-03 09:51:43 -06002047};
2048
2049static int domain_context_mapping_cb(struct pci_dev *pdev,
2050 u16 alias, void *opaque)
2051{
2052 struct domain_context_mapping_data *data = opaque;
2053
2054 return domain_context_mapping_one(data->domain, data->iommu,
Lu Baoluca6e3222018-12-10 09:59:02 +08002055 data->table, PCI_BUS_NUM(alias),
2056 alias & 0xff);
Alex Williamson579305f2014-07-03 09:51:43 -06002057}
2058
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002059static int
Joerg Roedel28ccce02015-07-21 14:45:31 +02002060domain_context_mapping(struct dmar_domain *domain, struct device *dev)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002061{
Lu Baoluca6e3222018-12-10 09:59:02 +08002062 struct domain_context_mapping_data data;
2063 struct pasid_table *table;
David Woodhouse64ae8922014-03-09 12:52:30 -07002064 struct intel_iommu *iommu;
David Woodhouse156baca2014-03-09 14:00:57 -07002065 u8 bus, devfn;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002066
David Woodhousee1f167f2014-03-09 15:24:46 -07002067 iommu = device_to_iommu(dev, &bus, &devfn);
David Woodhouse64ae8922014-03-09 12:52:30 -07002068 if (!iommu)
2069 return -ENODEV;
2070
Lu Baoluca6e3222018-12-10 09:59:02 +08002071 table = intel_pasid_get_table(dev);
2072
Alex Williamson579305f2014-07-03 09:51:43 -06002073 if (!dev_is_pci(dev))
Lu Baoluca6e3222018-12-10 09:59:02 +08002074 return domain_context_mapping_one(domain, iommu, table,
2075 bus, devfn);
Alex Williamson579305f2014-07-03 09:51:43 -06002076
2077 data.domain = domain;
2078 data.iommu = iommu;
Lu Baoluca6e3222018-12-10 09:59:02 +08002079 data.table = table;
Alex Williamson579305f2014-07-03 09:51:43 -06002080
2081 return pci_for_each_dma_alias(to_pci_dev(dev),
2082 &domain_context_mapping_cb, &data);
2083}
2084
2085static int domain_context_mapped_cb(struct pci_dev *pdev,
2086 u16 alias, void *opaque)
2087{
2088 struct intel_iommu *iommu = opaque;
2089
2090 return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002091}
2092
David Woodhousee1f167f2014-03-09 15:24:46 -07002093static int domain_context_mapped(struct device *dev)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002094{
Weidong Han5331fe62008-12-08 23:00:00 +08002095 struct intel_iommu *iommu;
David Woodhouse156baca2014-03-09 14:00:57 -07002096 u8 bus, devfn;
Weidong Han5331fe62008-12-08 23:00:00 +08002097
David Woodhousee1f167f2014-03-09 15:24:46 -07002098 iommu = device_to_iommu(dev, &bus, &devfn);
Weidong Han5331fe62008-12-08 23:00:00 +08002099 if (!iommu)
2100 return -ENODEV;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002101
Alex Williamson579305f2014-07-03 09:51:43 -06002102 if (!dev_is_pci(dev))
2103 return device_context_mapped(iommu, bus, devfn);
David Woodhousee1f167f2014-03-09 15:24:46 -07002104
Alex Williamson579305f2014-07-03 09:51:43 -06002105 return !pci_for_each_dma_alias(to_pci_dev(dev),
2106 domain_context_mapped_cb, iommu);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002107}
2108
Fenghua Yuf5329592009-08-04 15:09:37 -07002109/* Returns a number of VTD pages, but aligned to MM page size */
2110static inline unsigned long aligned_nrpages(unsigned long host_addr,
2111 size_t size)
2112{
2113 host_addr &= ~PAGE_MASK;
2114 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
2115}
2116
Youquan Song6dd9a7c2011-05-25 19:13:49 +01002117/* Return largest possible superpage level for a given mapping */
2118static inline int hardware_largepage_caps(struct dmar_domain *domain,
2119 unsigned long iov_pfn,
2120 unsigned long phy_pfn,
2121 unsigned long pages)
2122{
2123 int support, level = 1;
2124 unsigned long pfnmerge;
2125
2126 support = domain->iommu_superpage;
2127
2128 /* To use a large page, the virtual *and* physical addresses
2129 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
2130 of them will mean we have to use smaller pages. So just
2131 merge them and check both at once. */
2132 pfnmerge = iov_pfn | phy_pfn;
2133
2134 while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
2135 pages >>= VTD_STRIDE_SHIFT;
2136 if (!pages)
2137 break;
2138 pfnmerge >>= VTD_STRIDE_SHIFT;
2139 level++;
2140 support--;
2141 }
2142 return level;
2143}
2144
David Woodhouse9051aa02009-06-29 12:30:54 +01002145static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2146 struct scatterlist *sg, unsigned long phys_pfn,
2147 unsigned long nr_pages, int prot)
David Woodhousee1605492009-06-29 11:17:38 +01002148{
2149 struct dma_pte *first_pte = NULL, *pte = NULL;
David Woodhouse9051aa02009-06-29 12:30:54 +01002150 phys_addr_t uninitialized_var(pteval);
Jiang Liucc4f14a2014-11-26 09:42:10 +08002151 unsigned long sg_res = 0;
Youquan Song6dd9a7c2011-05-25 19:13:49 +01002152 unsigned int largepage_lvl = 0;
2153 unsigned long lvl_pages = 0;
David Woodhousee1605492009-06-29 11:17:38 +01002154
Jiang Liu162d1b12014-07-11 14:19:35 +08002155 BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
David Woodhousee1605492009-06-29 11:17:38 +01002156
2157 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
2158 return -EINVAL;
2159
2160 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
2161
Jiang Liucc4f14a2014-11-26 09:42:10 +08002162 if (!sg) {
2163 sg_res = nr_pages;
David Woodhouse9051aa02009-06-29 12:30:54 +01002164 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
2165 }
2166
Youquan Song6dd9a7c2011-05-25 19:13:49 +01002167 while (nr_pages > 0) {
David Woodhousec85994e2009-07-01 19:21:24 +01002168 uint64_t tmp;
2169
David Woodhousee1605492009-06-29 11:17:38 +01002170 if (!sg_res) {
Robin Murphy29a90b72017-09-28 15:14:01 +01002171 unsigned int pgoff = sg->offset & ~PAGE_MASK;
2172
Fenghua Yuf5329592009-08-04 15:09:37 -07002173 sg_res = aligned_nrpages(sg->offset, sg->length);
Robin Murphy29a90b72017-09-28 15:14:01 +01002174 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + pgoff;
David Woodhousee1605492009-06-29 11:17:38 +01002175 sg->dma_length = sg->length;
Robin Murphy29a90b72017-09-28 15:14:01 +01002176 pteval = (sg_phys(sg) - pgoff) | prot;
Youquan Song6dd9a7c2011-05-25 19:13:49 +01002177 phys_pfn = pteval >> VTD_PAGE_SHIFT;
David Woodhousee1605492009-06-29 11:17:38 +01002178 }
Youquan Song6dd9a7c2011-05-25 19:13:49 +01002179
David Woodhousee1605492009-06-29 11:17:38 +01002180 if (!pte) {
Youquan Song6dd9a7c2011-05-25 19:13:49 +01002181 largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2182
David Woodhouse5cf0a762014-03-19 16:07:49 +00002183 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
David Woodhousee1605492009-06-29 11:17:38 +01002184 if (!pte)
2185 return -ENOMEM;
Youquan Song6dd9a7c2011-05-25 19:13:49 +01002186 /* It is large page*/
Woodhouse, David6491d4d2012-12-19 13:25:35 +00002187 if (largepage_lvl > 1) {
Christian Zanderba2374f2015-06-10 09:41:45 -07002188 unsigned long nr_superpages, end_pfn;
2189
Youquan Song6dd9a7c2011-05-25 19:13:49 +01002190 pteval |= DMA_PTE_LARGE_PAGE;
Jiang Liud41a4ad2014-07-11 14:19:34 +08002191 lvl_pages = lvl_to_nr_pages(largepage_lvl);
Christian Zanderba2374f2015-06-10 09:41:45 -07002192
2193 nr_superpages = sg_res / lvl_pages;
2194 end_pfn = iov_pfn + nr_superpages * lvl_pages - 1;
2195
Jiang Liud41a4ad2014-07-11 14:19:34 +08002196 /*
2197 * Ensure that old small page tables are
Christian Zanderba2374f2015-06-10 09:41:45 -07002198 * removed to make room for superpage(s).
David Dillowbc24c572017-06-28 19:42:23 -07002199 * We're adding new large pages, so make sure
2200 * we don't remove their parent tables.
Jiang Liud41a4ad2014-07-11 14:19:34 +08002201 */
David Dillowbc24c572017-06-28 19:42:23 -07002202 dma_pte_free_pagetable(domain, iov_pfn, end_pfn,
2203 largepage_lvl + 1);
Woodhouse, David6491d4d2012-12-19 13:25:35 +00002204 } else {
Youquan Song6dd9a7c2011-05-25 19:13:49 +01002205 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
Woodhouse, David6491d4d2012-12-19 13:25:35 +00002206 }
Youquan Song6dd9a7c2011-05-25 19:13:49 +01002207
David Woodhousee1605492009-06-29 11:17:38 +01002208 }
2209 /* We don't need lock here, nobody else
2210 * touches the iova range
2211 */
David Woodhouse7766a3f2009-07-01 20:27:03 +01002212 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
David Woodhousec85994e2009-07-01 19:21:24 +01002213 if (tmp) {
David Woodhouse1bf20f02009-06-29 22:06:43 +01002214 static int dumps = 5;
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02002215 pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2216 iov_pfn, tmp, (unsigned long long)pteval);
David Woodhouse1bf20f02009-06-29 22:06:43 +01002217 if (dumps) {
2218 dumps--;
2219 debug_dma_dump_mappings(NULL);
2220 }
2221 WARN_ON(1);
2222 }
Youquan Song6dd9a7c2011-05-25 19:13:49 +01002223
2224 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2225
2226 BUG_ON(nr_pages < lvl_pages);
2227 BUG_ON(sg_res < lvl_pages);
2228
2229 nr_pages -= lvl_pages;
2230 iov_pfn += lvl_pages;
2231 phys_pfn += lvl_pages;
2232 pteval += lvl_pages * VTD_PAGE_SIZE;
2233 sg_res -= lvl_pages;
2234
2235 /* If the next PTE would be the first in a new page, then we
2236 need to flush the cache on the entries we've just written.
2237 And then we'll need to recalculate 'pte', so clear it and
2238 let it get set again in the if (!pte) block above.
2239
2240 If we're done (!nr_pages) we need to flush the cache too.
2241
2242 Also if we've been setting superpages, we may need to
2243 recalculate 'pte' and switch back to smaller pages for the
2244 end of the mapping, if the trailing size is not enough to
2245 use another superpage (i.e. sg_res < lvl_pages). */
David Woodhousee1605492009-06-29 11:17:38 +01002246 pte++;
Youquan Song6dd9a7c2011-05-25 19:13:49 +01002247 if (!nr_pages || first_pte_in_page(pte) ||
2248 (largepage_lvl > 1 && sg_res < lvl_pages)) {
David Woodhousee1605492009-06-29 11:17:38 +01002249 domain_flush_cache(domain, first_pte,
2250 (void *)pte - (void *)first_pte);
2251 pte = NULL;
2252 }
Youquan Song6dd9a7c2011-05-25 19:13:49 +01002253
2254 if (!sg_res && nr_pages)
David Woodhousee1605492009-06-29 11:17:38 +01002255 sg = sg_next(sg);
2256 }
2257 return 0;
2258}
2259
Peter Xu87684fd2018-05-04 10:34:53 +08002260static int domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2261 struct scatterlist *sg, unsigned long phys_pfn,
2262 unsigned long nr_pages, int prot)
2263{
2264 int ret;
2265 struct intel_iommu *iommu;
2266
2267 /* Do the real mapping first */
2268 ret = __domain_mapping(domain, iov_pfn, sg, phys_pfn, nr_pages, prot);
2269 if (ret)
2270 return ret;
2271
2272 /* Notify about the new mapping */
2273 if (domain_type_is_vm(domain)) {
2274 /* VM typed domains can have more than one IOMMUs */
2275 int iommu_id;
2276 for_each_domain_iommu(iommu_id, domain) {
2277 iommu = g_iommus[iommu_id];
2278 __mapping_notify_one(iommu, domain, iov_pfn, nr_pages);
2279 }
2280 } else {
2281 /* General domains only have one IOMMU */
2282 iommu = domain_get_iommu(domain);
2283 __mapping_notify_one(iommu, domain, iov_pfn, nr_pages);
2284 }
2285
2286 return 0;
2287}
2288
David Woodhouse9051aa02009-06-29 12:30:54 +01002289static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2290 struct scatterlist *sg, unsigned long nr_pages,
2291 int prot)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002292{
Peter Xu87684fd2018-05-04 10:34:53 +08002293 return domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
David Woodhouse9051aa02009-06-29 12:30:54 +01002294}
Fenghua Yu5b6985c2008-10-16 18:02:32 -07002295
David Woodhouse9051aa02009-06-29 12:30:54 +01002296static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2297 unsigned long phys_pfn, unsigned long nr_pages,
2298 int prot)
2299{
Peter Xu87684fd2018-05-04 10:34:53 +08002300 return domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002301}
2302
Joerg Roedel2452d9d2015-07-23 16:20:14 +02002303static void domain_context_clear_one(struct intel_iommu *iommu, u8 bus, u8 devfn)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002304{
Filippo Sironi50822192017-08-31 10:58:11 +02002305 unsigned long flags;
2306 struct context_entry *context;
2307 u16 did_old;
2308
Weidong Hanc7151a82008-12-08 22:51:37 +08002309 if (!iommu)
2310 return;
Weidong Han8c11e792008-12-08 15:29:22 +08002311
Filippo Sironi50822192017-08-31 10:58:11 +02002312 spin_lock_irqsave(&iommu->lock, flags);
2313 context = iommu_context_addr(iommu, bus, devfn, 0);
2314 if (!context) {
2315 spin_unlock_irqrestore(&iommu->lock, flags);
2316 return;
2317 }
2318 did_old = context_domain_id(context);
2319 context_clear_entry(context);
2320 __iommu_flush_cache(iommu, context, sizeof(*context));
2321 spin_unlock_irqrestore(&iommu->lock, flags);
2322 iommu->flush.flush_context(iommu,
2323 did_old,
2324 (((u16)bus) << 8) | devfn,
2325 DMA_CCMD_MASK_NOBIT,
2326 DMA_CCMD_DEVICE_INVL);
2327 iommu->flush.flush_iotlb(iommu,
2328 did_old,
2329 0,
2330 0,
2331 DMA_TLB_DSI_FLUSH);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002332}
2333
David Woodhouse109b9b02012-05-25 17:43:02 +01002334static inline void unlink_domain_info(struct device_domain_info *info)
2335{
2336 assert_spin_locked(&device_domain_lock);
2337 list_del(&info->link);
2338 list_del(&info->global);
2339 if (info->dev)
David Woodhouse0bcb3e22014-03-06 17:12:03 +00002340 info->dev->archdata.iommu = NULL;
David Woodhouse109b9b02012-05-25 17:43:02 +01002341}
2342
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002343static void domain_remove_dev_info(struct dmar_domain *domain)
2344{
Yijing Wang3a74ca02014-05-20 20:37:47 +08002345 struct device_domain_info *info, *tmp;
Jiang Liufb170fb2014-07-11 14:19:28 +08002346 unsigned long flags;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002347
2348 spin_lock_irqsave(&device_domain_lock, flags);
Joerg Roedel76f45fe2015-07-21 18:25:11 +02002349 list_for_each_entry_safe(info, tmp, &domain->devices, link)
Joerg Roedel127c7612015-07-23 17:44:46 +02002350 __dmar_remove_one_dev_info(info);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002351 spin_unlock_irqrestore(&device_domain_lock, flags);
2352}
2353
2354/*
2355 * find_domain
David Woodhouse1525a292014-03-06 16:19:30 +00002356 * Note: we use struct device->archdata.iommu stores the info
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002357 */
David Woodhouse1525a292014-03-06 16:19:30 +00002358static struct dmar_domain *find_domain(struct device *dev)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002359{
2360 struct device_domain_info *info;
2361
2362 /* No lock here, assumes no domain exit in normal case */
David Woodhouse1525a292014-03-06 16:19:30 +00002363 info = dev->archdata.iommu;
Peter Xub316d022017-05-22 18:28:51 +08002364 if (likely(info))
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002365 return info->domain;
2366 return NULL;
2367}
2368
David Woodhouse5a8f40e2014-03-09 13:31:18 -07002369static inline struct device_domain_info *
Jiang Liu745f2582014-02-19 14:07:26 +08002370dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2371{
2372 struct device_domain_info *info;
2373
2374 list_for_each_entry(info, &device_domain_list, global)
David Woodhouse41e80dca2014-03-09 13:55:54 -07002375 if (info->iommu->segment == segment && info->bus == bus &&
Jiang Liu745f2582014-02-19 14:07:26 +08002376 info->devfn == devfn)
David Woodhouse5a8f40e2014-03-09 13:31:18 -07002377 return info;
Jiang Liu745f2582014-02-19 14:07:26 +08002378
2379 return NULL;
2380}
2381
Joerg Roedel5db31562015-07-22 12:40:43 +02002382static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,
2383 int bus, int devfn,
2384 struct device *dev,
2385 struct dmar_domain *domain)
Jiang Liu745f2582014-02-19 14:07:26 +08002386{
David Woodhouse5a8f40e2014-03-09 13:31:18 -07002387 struct dmar_domain *found = NULL;
Jiang Liu745f2582014-02-19 14:07:26 +08002388 struct device_domain_info *info;
2389 unsigned long flags;
Joerg Roedeld160aca2015-07-22 11:52:53 +02002390 int ret;
Jiang Liu745f2582014-02-19 14:07:26 +08002391
2392 info = alloc_devinfo_mem();
2393 if (!info)
David Woodhouseb718cd32014-03-09 13:11:33 -07002394 return NULL;
Jiang Liu745f2582014-02-19 14:07:26 +08002395
Jiang Liu745f2582014-02-19 14:07:26 +08002396 info->bus = bus;
2397 info->devfn = devfn;
David Woodhouseb16d0cb2015-10-12 14:17:37 +01002398 info->ats_supported = info->pasid_supported = info->pri_supported = 0;
2399 info->ats_enabled = info->pasid_enabled = info->pri_enabled = 0;
2400 info->ats_qdep = 0;
Jiang Liu745f2582014-02-19 14:07:26 +08002401 info->dev = dev;
2402 info->domain = domain;
David Woodhouse5a8f40e2014-03-09 13:31:18 -07002403 info->iommu = iommu;
Lu Baolucc580e42018-07-14 15:46:59 +08002404 info->pasid_table = NULL;
Jiang Liu745f2582014-02-19 14:07:26 +08002405
David Woodhouseb16d0cb2015-10-12 14:17:37 +01002406 if (dev && dev_is_pci(dev)) {
2407 struct pci_dev *pdev = to_pci_dev(info->dev);
2408
Gil Kupfercef74402018-05-10 17:56:02 -05002409 if (!pci_ats_disabled() &&
2410 ecap_dev_iotlb_support(iommu->ecap) &&
David Woodhouseb16d0cb2015-10-12 14:17:37 +01002411 pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS) &&
2412 dmar_find_matched_atsr_unit(pdev))
2413 info->ats_supported = 1;
2414
Lu Baolu765b6a92018-12-10 09:58:55 +08002415 if (sm_supported(iommu)) {
2416 if (pasid_supported(iommu)) {
David Woodhouseb16d0cb2015-10-12 14:17:37 +01002417 int features = pci_pasid_features(pdev);
2418 if (features >= 0)
2419 info->pasid_supported = features | 1;
2420 }
2421
2422 if (info->ats_supported && ecap_prs(iommu->ecap) &&
2423 pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI))
2424 info->pri_supported = 1;
2425 }
2426 }
2427
Jiang Liu745f2582014-02-19 14:07:26 +08002428 spin_lock_irqsave(&device_domain_lock, flags);
2429 if (dev)
David Woodhouse0bcb3e22014-03-06 17:12:03 +00002430 found = find_domain(dev);
Joerg Roedelf303e502015-07-23 18:37:13 +02002431
2432 if (!found) {
David Woodhouse5a8f40e2014-03-09 13:31:18 -07002433 struct device_domain_info *info2;
David Woodhouse41e80dca2014-03-09 13:55:54 -07002434 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
Joerg Roedelf303e502015-07-23 18:37:13 +02002435 if (info2) {
2436 found = info2->domain;
2437 info2->dev = dev;
2438 }
David Woodhouse5a8f40e2014-03-09 13:31:18 -07002439 }
Joerg Roedelf303e502015-07-23 18:37:13 +02002440
Jiang Liu745f2582014-02-19 14:07:26 +08002441 if (found) {
2442 spin_unlock_irqrestore(&device_domain_lock, flags);
2443 free_devinfo_mem(info);
David Woodhouseb718cd32014-03-09 13:11:33 -07002444 /* Caller must free the original domain */
2445 return found;
Jiang Liu745f2582014-02-19 14:07:26 +08002446 }
2447
Joerg Roedeld160aca2015-07-22 11:52:53 +02002448 spin_lock(&iommu->lock);
2449 ret = domain_attach_iommu(domain, iommu);
2450 spin_unlock(&iommu->lock);
2451
2452 if (ret) {
Joerg Roedelc6c2ceb2015-07-22 13:11:53 +02002453 spin_unlock_irqrestore(&device_domain_lock, flags);
Sudip Mukherjee499f3aa2015-09-18 16:27:07 +05302454 free_devinfo_mem(info);
Joerg Roedelc6c2ceb2015-07-22 13:11:53 +02002455 return NULL;
2456 }
Joerg Roedelc6c2ceb2015-07-22 13:11:53 +02002457
David Woodhouseb718cd32014-03-09 13:11:33 -07002458 list_add(&info->link, &domain->devices);
2459 list_add(&info->global, &device_domain_list);
2460 if (dev)
2461 dev->archdata.iommu = info;
Lu Baolu0bbeb012018-12-10 09:58:56 +08002462 spin_unlock_irqrestore(&device_domain_lock, flags);
Lu Baolua7fc93f2018-07-14 15:47:00 +08002463
Lu Baolu0bbeb012018-12-10 09:58:56 +08002464 /* PASID table is mandatory for a PCI device in scalable mode. */
2465 if (dev && dev_is_pci(dev) && sm_supported(iommu)) {
Lu Baolua7fc93f2018-07-14 15:47:00 +08002466 ret = intel_pasid_alloc_table(dev);
2467 if (ret) {
Lu Baolu0bbeb012018-12-10 09:58:56 +08002468 pr_err("PASID table allocation for %s failed\n",
2469 dev_name(dev));
2470 dmar_remove_one_dev_info(domain, dev);
2471 return NULL;
Lu Baolua7fc93f2018-07-14 15:47:00 +08002472 }
Lu Baoluef848b72018-12-10 09:59:01 +08002473
2474 /* Setup the PASID entry for requests without PASID: */
2475 spin_lock(&iommu->lock);
2476 if (hw_pass_through && domain_type_is_si(domain))
2477 ret = intel_pasid_setup_pass_through(iommu, domain,
2478 dev, PASID_RID2PASID);
2479 else
2480 ret = intel_pasid_setup_second_level(iommu, domain,
2481 dev, PASID_RID2PASID);
2482 spin_unlock(&iommu->lock);
2483 if (ret) {
2484 pr_err("Setup RID2PASID for %s failed\n",
2485 dev_name(dev));
2486 dmar_remove_one_dev_info(domain, dev);
2487 return NULL;
2488 }
Lu Baolua7fc93f2018-07-14 15:47:00 +08002489 }
David Woodhouseb718cd32014-03-09 13:11:33 -07002490
Joerg Roedelcc4e2572015-07-22 10:04:36 +02002491 if (dev && domain_context_mapping(domain, dev)) {
2492 pr_err("Domain context map for %s failed\n", dev_name(dev));
Joerg Roedele6de0f82015-07-22 16:30:36 +02002493 dmar_remove_one_dev_info(domain, dev);
Joerg Roedelcc4e2572015-07-22 10:04:36 +02002494 return NULL;
2495 }
2496
David Woodhouseb718cd32014-03-09 13:11:33 -07002497 return domain;
Jiang Liu745f2582014-02-19 14:07:26 +08002498}
2499
Alex Williamson579305f2014-07-03 09:51:43 -06002500static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2501{
2502 *(u16 *)opaque = alias;
2503 return 0;
2504}
2505
Joerg Roedel76208352016-08-25 14:25:12 +02002506static struct dmar_domain *find_or_alloc_domain(struct device *dev, int gaw)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002507{
Joerg Roedelcc4e2572015-07-22 10:04:36 +02002508 struct device_domain_info *info = NULL;
Joerg Roedel76208352016-08-25 14:25:12 +02002509 struct dmar_domain *domain = NULL;
Alex Williamson579305f2014-07-03 09:51:43 -06002510 struct intel_iommu *iommu;
Lu Baolufcc35c62018-05-04 13:08:17 +08002511 u16 dma_alias;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002512 unsigned long flags;
Yijing Wangaa4d0662014-05-26 20:14:06 +08002513 u8 bus, devfn;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002514
David Woodhouse146922e2014-03-09 15:44:17 -07002515 iommu = device_to_iommu(dev, &bus, &devfn);
2516 if (!iommu)
Alex Williamson579305f2014-07-03 09:51:43 -06002517 return NULL;
2518
2519 if (dev_is_pci(dev)) {
2520 struct pci_dev *pdev = to_pci_dev(dev);
2521
2522 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2523
2524 spin_lock_irqsave(&device_domain_lock, flags);
2525 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2526 PCI_BUS_NUM(dma_alias),
2527 dma_alias & 0xff);
2528 if (info) {
2529 iommu = info->iommu;
2530 domain = info->domain;
2531 }
2532 spin_unlock_irqrestore(&device_domain_lock, flags);
2533
Joerg Roedel76208352016-08-25 14:25:12 +02002534 /* DMA alias already has a domain, use it */
Alex Williamson579305f2014-07-03 09:51:43 -06002535 if (info)
Joerg Roedel76208352016-08-25 14:25:12 +02002536 goto out;
Alex Williamson579305f2014-07-03 09:51:43 -06002537 }
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002538
David Woodhouse146922e2014-03-09 15:44:17 -07002539 /* Allocate and initialize new domain for the device */
Jiang Liuab8dfe22014-07-11 14:19:27 +08002540 domain = alloc_domain(0);
Jiang Liu745f2582014-02-19 14:07:26 +08002541 if (!domain)
Alex Williamson579305f2014-07-03 09:51:43 -06002542 return NULL;
Joerg Roedeldc534b22015-07-22 12:44:02 +02002543 if (domain_init(domain, iommu, gaw)) {
Alex Williamson579305f2014-07-03 09:51:43 -06002544 domain_exit(domain);
2545 return NULL;
2546 }
2547
Joerg Roedel76208352016-08-25 14:25:12 +02002548out:
Alex Williamson579305f2014-07-03 09:51:43 -06002549
Joerg Roedel76208352016-08-25 14:25:12 +02002550 return domain;
2551}
2552
2553static struct dmar_domain *set_domain_for_dev(struct device *dev,
2554 struct dmar_domain *domain)
2555{
2556 struct intel_iommu *iommu;
2557 struct dmar_domain *tmp;
2558 u16 req_id, dma_alias;
2559 u8 bus, devfn;
2560
2561 iommu = device_to_iommu(dev, &bus, &devfn);
2562 if (!iommu)
2563 return NULL;
2564
2565 req_id = ((u16)bus << 8) | devfn;
2566
2567 if (dev_is_pci(dev)) {
2568 struct pci_dev *pdev = to_pci_dev(dev);
2569
2570 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2571
2572 /* register PCI DMA alias device */
2573 if (req_id != dma_alias) {
2574 tmp = dmar_insert_one_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2575 dma_alias & 0xff, NULL, domain);
2576
2577 if (!tmp || tmp != domain)
2578 return tmp;
Alex Williamson579305f2014-07-03 09:51:43 -06002579 }
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002580 }
2581
Joerg Roedel5db31562015-07-22 12:40:43 +02002582 tmp = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
Joerg Roedel76208352016-08-25 14:25:12 +02002583 if (!tmp || tmp != domain)
2584 return tmp;
Alex Williamson579305f2014-07-03 09:51:43 -06002585
Joerg Roedel76208352016-08-25 14:25:12 +02002586 return domain;
2587}
2588
2589static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
2590{
2591 struct dmar_domain *domain, *tmp;
2592
2593 domain = find_domain(dev);
2594 if (domain)
2595 goto out;
2596
2597 domain = find_or_alloc_domain(dev, gaw);
2598 if (!domain)
2599 goto out;
2600
2601 tmp = set_domain_for_dev(dev, domain);
2602 if (!tmp || domain != tmp) {
Alex Williamson579305f2014-07-03 09:51:43 -06002603 domain_exit(domain);
2604 domain = tmp;
2605 }
David Woodhouseb718cd32014-03-09 13:11:33 -07002606
Joerg Roedel76208352016-08-25 14:25:12 +02002607out:
2608
David Woodhouseb718cd32014-03-09 13:11:33 -07002609 return domain;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002610}
2611
David Woodhouseb2132032009-06-26 18:50:28 +01002612static int iommu_domain_identity_map(struct dmar_domain *domain,
2613 unsigned long long start,
2614 unsigned long long end)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002615{
David Woodhousec5395d52009-06-28 16:35:56 +01002616 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2617 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002618
David Woodhousec5395d52009-06-28 16:35:56 +01002619 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2620 dma_to_mm_pfn(last_vpfn))) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02002621 pr_err("Reserving iova failed\n");
David Woodhouseb2132032009-06-26 18:50:28 +01002622 return -ENOMEM;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002623 }
2624
Joerg Roedelaf1089c2015-07-21 15:45:19 +02002625 pr_debug("Mapping reserved region %llx-%llx\n", start, end);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002626 /*
2627 * RMRR range might have overlap with physical memory range,
2628 * clear it first
2629 */
David Woodhousec5395d52009-06-28 16:35:56 +01002630 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002631
Peter Xu87684fd2018-05-04 10:34:53 +08002632 return __domain_mapping(domain, first_vpfn, NULL,
2633 first_vpfn, last_vpfn - first_vpfn + 1,
2634 DMA_PTE_READ|DMA_PTE_WRITE);
David Woodhouseb2132032009-06-26 18:50:28 +01002635}
2636
Joerg Roedeld66ce542015-09-23 19:00:10 +02002637static int domain_prepare_identity_map(struct device *dev,
2638 struct dmar_domain *domain,
2639 unsigned long long start,
2640 unsigned long long end)
David Woodhouseb2132032009-06-26 18:50:28 +01002641{
David Woodhouse19943b02009-08-04 16:19:20 +01002642 /* For _hardware_ passthrough, don't bother. But for software
2643 passthrough, we do it anyway -- it may indicate a memory
2644 range which is reserved in E820, so which didn't get set
2645 up to start with in si_domain */
2646 if (domain == si_domain && hw_pass_through) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02002647 pr_warn("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2648 dev_name(dev), start, end);
David Woodhouse19943b02009-08-04 16:19:20 +01002649 return 0;
2650 }
2651
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02002652 pr_info("Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2653 dev_name(dev), start, end);
2654
David Woodhouse5595b522009-12-02 09:21:55 +00002655 if (end < start) {
2656 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2657 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2658 dmi_get_system_info(DMI_BIOS_VENDOR),
2659 dmi_get_system_info(DMI_BIOS_VERSION),
2660 dmi_get_system_info(DMI_PRODUCT_VERSION));
Joerg Roedeld66ce542015-09-23 19:00:10 +02002661 return -EIO;
David Woodhouse5595b522009-12-02 09:21:55 +00002662 }
2663
David Woodhouse2ff729f2009-08-26 14:25:41 +01002664 if (end >> agaw_to_width(domain->agaw)) {
2665 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2666 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2667 agaw_to_width(domain->agaw),
2668 dmi_get_system_info(DMI_BIOS_VENDOR),
2669 dmi_get_system_info(DMI_BIOS_VERSION),
2670 dmi_get_system_info(DMI_PRODUCT_VERSION));
Joerg Roedeld66ce542015-09-23 19:00:10 +02002671 return -EIO;
David Woodhouse2ff729f2009-08-26 14:25:41 +01002672 }
David Woodhouse19943b02009-08-04 16:19:20 +01002673
Joerg Roedeld66ce542015-09-23 19:00:10 +02002674 return iommu_domain_identity_map(domain, start, end);
2675}
2676
2677static int iommu_prepare_identity_map(struct device *dev,
2678 unsigned long long start,
2679 unsigned long long end)
2680{
2681 struct dmar_domain *domain;
2682 int ret;
2683
2684 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2685 if (!domain)
2686 return -ENOMEM;
2687
2688 ret = domain_prepare_identity_map(dev, domain, start, end);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002689 if (ret)
Joerg Roedeld66ce542015-09-23 19:00:10 +02002690 domain_exit(domain);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002691
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002692 return ret;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002693}
2694
2695static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
David Woodhouse0b9d9752014-03-09 15:48:15 -07002696 struct device *dev)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002697{
David Woodhouse0b9d9752014-03-09 15:48:15 -07002698 if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002699 return 0;
David Woodhouse0b9d9752014-03-09 15:48:15 -07002700 return iommu_prepare_identity_map(dev, rmrr->base_address,
2701 rmrr->end_address);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002702}
2703
Suresh Siddhad3f13812011-08-23 17:05:25 -07002704#ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
Keshavamurthy, Anil S49a04292007-10-21 16:41:57 -07002705static inline void iommu_prepare_isa(void)
2706{
2707 struct pci_dev *pdev;
2708 int ret;
2709
2710 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2711 if (!pdev)
2712 return;
2713
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02002714 pr_info("Prepare 0-16MiB unity mapping for LPC\n");
David Woodhouse0b9d9752014-03-09 15:48:15 -07002715 ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
Keshavamurthy, Anil S49a04292007-10-21 16:41:57 -07002716
2717 if (ret)
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02002718 pr_err("Failed to create 0-16MiB identity map - floppy might not work\n");
Keshavamurthy, Anil S49a04292007-10-21 16:41:57 -07002719
Yijing Wang9b27e822014-05-20 20:37:52 +08002720 pci_dev_put(pdev);
Keshavamurthy, Anil S49a04292007-10-21 16:41:57 -07002721}
2722#else
2723static inline void iommu_prepare_isa(void)
2724{
2725 return;
2726}
Suresh Siddhad3f13812011-08-23 17:05:25 -07002727#endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
Keshavamurthy, Anil S49a04292007-10-21 16:41:57 -07002728
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002729static int md_domain_init(struct dmar_domain *domain, int guest_width);
David Woodhousec7ab48d2009-06-26 19:10:36 +01002730
Matt Kraai071e1372009-08-23 22:30:22 -07002731static int __init si_domain_init(int hw)
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002732{
David Woodhousec7ab48d2009-06-26 19:10:36 +01002733 int nid, ret = 0;
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002734
Jiang Liuab8dfe22014-07-11 14:19:27 +08002735 si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002736 if (!si_domain)
2737 return -EFAULT;
2738
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002739 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2740 domain_exit(si_domain);
2741 return -EFAULT;
2742 }
2743
Joerg Roedel0dc79712015-07-21 15:40:06 +02002744 pr_debug("Identity mapping domain allocated\n");
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002745
David Woodhouse19943b02009-08-04 16:19:20 +01002746 if (hw)
2747 return 0;
2748
David Woodhousec7ab48d2009-06-26 19:10:36 +01002749 for_each_online_node(nid) {
Tejun Heod4bbf7e2011-11-28 09:46:22 -08002750 unsigned long start_pfn, end_pfn;
2751 int i;
2752
2753 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2754 ret = iommu_domain_identity_map(si_domain,
2755 PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2756 if (ret)
2757 return ret;
2758 }
David Woodhousec7ab48d2009-06-26 19:10:36 +01002759 }
2760
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002761 return 0;
2762}
2763
David Woodhouse9b226622014-03-09 14:03:28 -07002764static int identity_mapping(struct device *dev)
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002765{
2766 struct device_domain_info *info;
2767
2768 if (likely(!iommu_identity_mapping))
2769 return 0;
2770
David Woodhouse9b226622014-03-09 14:03:28 -07002771 info = dev->archdata.iommu;
Mike Traviscb452a42011-05-28 13:15:03 -05002772 if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2773 return (info->domain == si_domain);
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002774
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002775 return 0;
2776}
2777
Joerg Roedel28ccce02015-07-21 14:45:31 +02002778static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002779{
David Woodhouse0ac72662014-03-09 13:19:22 -07002780 struct dmar_domain *ndomain;
David Woodhouse5a8f40e2014-03-09 13:31:18 -07002781 struct intel_iommu *iommu;
David Woodhouse156baca2014-03-09 14:00:57 -07002782 u8 bus, devfn;
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002783
David Woodhouse5913c9b2014-03-09 16:27:31 -07002784 iommu = device_to_iommu(dev, &bus, &devfn);
David Woodhouse5a8f40e2014-03-09 13:31:18 -07002785 if (!iommu)
2786 return -ENODEV;
2787
Joerg Roedel5db31562015-07-22 12:40:43 +02002788 ndomain = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
David Woodhouse0ac72662014-03-09 13:19:22 -07002789 if (ndomain != domain)
2790 return -EBUSY;
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002791
2792 return 0;
2793}
2794
David Woodhouse0b9d9752014-03-09 15:48:15 -07002795static bool device_has_rmrr(struct device *dev)
Tom Mingarelliea2447f72012-11-20 19:43:17 +00002796{
2797 struct dmar_rmrr_unit *rmrr;
David Woodhouse832bd852014-03-07 15:08:36 +00002798 struct device *tmp;
Tom Mingarelliea2447f72012-11-20 19:43:17 +00002799 int i;
2800
Jiang Liu0e242612014-02-19 14:07:34 +08002801 rcu_read_lock();
Tom Mingarelliea2447f72012-11-20 19:43:17 +00002802 for_each_rmrr_units(rmrr) {
Jiang Liub683b232014-02-19 14:07:32 +08002803 /*
2804 * Return TRUE if this RMRR contains the device that
2805 * is passed in.
2806 */
2807 for_each_active_dev_scope(rmrr->devices,
2808 rmrr->devices_cnt, i, tmp)
David Woodhouse0b9d9752014-03-09 15:48:15 -07002809 if (tmp == dev) {
Jiang Liu0e242612014-02-19 14:07:34 +08002810 rcu_read_unlock();
Tom Mingarelliea2447f72012-11-20 19:43:17 +00002811 return true;
Jiang Liub683b232014-02-19 14:07:32 +08002812 }
Tom Mingarelliea2447f72012-11-20 19:43:17 +00002813 }
Jiang Liu0e242612014-02-19 14:07:34 +08002814 rcu_read_unlock();
Tom Mingarelliea2447f72012-11-20 19:43:17 +00002815 return false;
2816}
2817
Alex Williamsonc875d2c2014-07-03 09:57:02 -06002818/*
2819 * There are a couple cases where we need to restrict the functionality of
2820 * devices associated with RMRRs. The first is when evaluating a device for
2821 * identity mapping because problems exist when devices are moved in and out
2822 * of domains and their respective RMRR information is lost. This means that
2823 * a device with associated RMRRs will never be in a "passthrough" domain.
2824 * The second is use of the device through the IOMMU API. This interface
2825 * expects to have full control of the IOVA space for the device. We cannot
2826 * satisfy both the requirement that RMRR access is maintained and have an
2827 * unencumbered IOVA space. We also have no ability to quiesce the device's
2828 * use of the RMRR space or even inform the IOMMU API user of the restriction.
2829 * We therefore prevent devices associated with an RMRR from participating in
2830 * the IOMMU API, which eliminates them from device assignment.
2831 *
2832 * In both cases we assume that PCI USB devices with RMRRs have them largely
2833 * for historical reasons and that the RMRR space is not actively used post
2834 * boot. This exclusion may change if vendors begin to abuse it.
David Woodhouse18436af2015-03-25 15:05:47 +00002835 *
2836 * The same exception is made for graphics devices, with the requirement that
2837 * any use of the RMRR regions will be torn down before assigning the device
2838 * to a guest.
Alex Williamsonc875d2c2014-07-03 09:57:02 -06002839 */
2840static bool device_is_rmrr_locked(struct device *dev)
2841{
2842 if (!device_has_rmrr(dev))
2843 return false;
2844
2845 if (dev_is_pci(dev)) {
2846 struct pci_dev *pdev = to_pci_dev(dev);
2847
David Woodhouse18436af2015-03-25 15:05:47 +00002848 if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
Alex Williamsonc875d2c2014-07-03 09:57:02 -06002849 return false;
2850 }
2851
2852 return true;
2853}
2854
David Woodhouse3bdb2592014-03-09 16:03:08 -07002855static int iommu_should_identity_map(struct device *dev, int startup)
David Woodhouse6941af22009-07-04 18:24:27 +01002856{
Tom Mingarelliea2447f72012-11-20 19:43:17 +00002857
David Woodhouse3bdb2592014-03-09 16:03:08 -07002858 if (dev_is_pci(dev)) {
2859 struct pci_dev *pdev = to_pci_dev(dev);
Tom Mingarelliea2447f72012-11-20 19:43:17 +00002860
Alex Williamsonc875d2c2014-07-03 09:57:02 -06002861 if (device_is_rmrr_locked(dev))
David Woodhouse3bdb2592014-03-09 16:03:08 -07002862 return 0;
David Woodhousee0fc7e02009-09-30 09:12:17 -07002863
David Woodhouse3bdb2592014-03-09 16:03:08 -07002864 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2865 return 1;
David Woodhousee0fc7e02009-09-30 09:12:17 -07002866
David Woodhouse3bdb2592014-03-09 16:03:08 -07002867 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2868 return 1;
2869
2870 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2871 return 0;
2872
2873 /*
2874 * We want to start off with all devices in the 1:1 domain, and
2875 * take them out later if we find they can't access all of memory.
2876 *
2877 * However, we can't do this for PCI devices behind bridges,
2878 * because all PCI devices behind the same bridge will end up
2879 * with the same source-id on their transactions.
2880 *
2881 * Practically speaking, we can't change things around for these
2882 * devices at run-time, because we can't be sure there'll be no
2883 * DMA transactions in flight for any of their siblings.
2884 *
2885 * So PCI devices (unless they're on the root bus) as well as
2886 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2887 * the 1:1 domain, just in _case_ one of their siblings turns out
2888 * not to be able to map all of memory.
2889 */
2890 if (!pci_is_pcie(pdev)) {
2891 if (!pci_is_root_bus(pdev->bus))
2892 return 0;
2893 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2894 return 0;
2895 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
2896 return 0;
2897 } else {
2898 if (device_has_rmrr(dev))
2899 return 0;
2900 }
David Woodhouse6941af22009-07-04 18:24:27 +01002901
David Woodhouse3dfc8132009-07-04 19:11:08 +01002902 /*
David Woodhouse3dfc8132009-07-04 19:11:08 +01002903 * At boot time, we don't yet know if devices will be 64-bit capable.
David Woodhouse3bdb2592014-03-09 16:03:08 -07002904 * Assume that they will — if they turn out not to be, then we can
David Woodhouse3dfc8132009-07-04 19:11:08 +01002905 * take them out of the 1:1 domain later.
2906 */
Chris Wright8fcc5372011-05-28 13:15:02 -05002907 if (!startup) {
2908 /*
2909 * If the device's dma_mask is less than the system's memory
2910 * size then this is not a candidate for identity mapping.
2911 */
David Woodhouse3bdb2592014-03-09 16:03:08 -07002912 u64 dma_mask = *dev->dma_mask;
Chris Wright8fcc5372011-05-28 13:15:02 -05002913
David Woodhouse3bdb2592014-03-09 16:03:08 -07002914 if (dev->coherent_dma_mask &&
2915 dev->coherent_dma_mask < dma_mask)
2916 dma_mask = dev->coherent_dma_mask;
Chris Wright8fcc5372011-05-28 13:15:02 -05002917
David Woodhouse3bdb2592014-03-09 16:03:08 -07002918 return dma_mask >= dma_get_required_mask(dev);
Chris Wright8fcc5372011-05-28 13:15:02 -05002919 }
David Woodhouse6941af22009-07-04 18:24:27 +01002920
2921 return 1;
2922}
2923
David Woodhousecf04eee2014-03-21 16:49:04 +00002924static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
2925{
2926 int ret;
2927
2928 if (!iommu_should_identity_map(dev, 1))
2929 return 0;
2930
Joerg Roedel28ccce02015-07-21 14:45:31 +02002931 ret = domain_add_dev_info(si_domain, dev);
David Woodhousecf04eee2014-03-21 16:49:04 +00002932 if (!ret)
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02002933 pr_info("%s identity mapping for device %s\n",
2934 hw ? "Hardware" : "Software", dev_name(dev));
David Woodhousecf04eee2014-03-21 16:49:04 +00002935 else if (ret == -ENODEV)
2936 /* device not associated with an iommu */
2937 ret = 0;
2938
2939 return ret;
2940}
2941
2942
Matt Kraai071e1372009-08-23 22:30:22 -07002943static int __init iommu_prepare_static_identity_mapping(int hw)
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002944{
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002945 struct pci_dev *pdev = NULL;
David Woodhousecf04eee2014-03-21 16:49:04 +00002946 struct dmar_drhd_unit *drhd;
2947 struct intel_iommu *iommu;
2948 struct device *dev;
2949 int i;
2950 int ret = 0;
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002951
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002952 for_each_pci_dev(pdev) {
David Woodhousecf04eee2014-03-21 16:49:04 +00002953 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
2954 if (ret)
2955 return ret;
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002956 }
2957
David Woodhousecf04eee2014-03-21 16:49:04 +00002958 for_each_active_iommu(iommu, drhd)
2959 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
2960 struct acpi_device_physical_node *pn;
2961 struct acpi_device *adev;
2962
2963 if (dev->bus != &acpi_bus_type)
2964 continue;
Joerg Roedel86080cc2015-06-12 12:27:16 +02002965
David Woodhousecf04eee2014-03-21 16:49:04 +00002966 adev= to_acpi_device(dev);
2967 mutex_lock(&adev->physical_node_lock);
2968 list_for_each_entry(pn, &adev->physical_node_list, node) {
2969 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
2970 if (ret)
2971 break;
2972 }
2973 mutex_unlock(&adev->physical_node_lock);
2974 if (ret)
2975 return ret;
2976 }
2977
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002978 return 0;
2979}
2980
Jiang Liuffebeb42014-11-09 22:48:02 +08002981static void intel_iommu_init_qi(struct intel_iommu *iommu)
2982{
2983 /*
2984 * Start from the sane iommu hardware state.
2985 * If the queued invalidation is already initialized by us
2986 * (for example, while enabling interrupt-remapping) then
2987 * we got the things already rolling from a sane state.
2988 */
2989 if (!iommu->qi) {
2990 /*
2991 * Clear any previous faults.
2992 */
2993 dmar_fault(-1, iommu);
2994 /*
2995 * Disable queued invalidation if supported and already enabled
2996 * before OS handover.
2997 */
2998 dmar_disable_qi(iommu);
2999 }
3000
3001 if (dmar_enable_qi(iommu)) {
3002 /*
3003 * Queued Invalidate not enabled, use Register Based Invalidate
3004 */
3005 iommu->flush.flush_context = __iommu_flush_context;
3006 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02003007 pr_info("%s: Using Register based invalidation\n",
Jiang Liuffebeb42014-11-09 22:48:02 +08003008 iommu->name);
3009 } else {
3010 iommu->flush.flush_context = qi_flush_context;
3011 iommu->flush.flush_iotlb = qi_flush_iotlb;
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02003012 pr_info("%s: Using Queued invalidation\n", iommu->name);
Jiang Liuffebeb42014-11-09 22:48:02 +08003013 }
3014}
3015
Joerg Roedel091d42e2015-06-12 11:56:10 +02003016static int copy_context_table(struct intel_iommu *iommu,
Dan Williamsdfddb962015-10-09 18:16:46 -04003017 struct root_entry *old_re,
Joerg Roedel091d42e2015-06-12 11:56:10 +02003018 struct context_entry **tbl,
3019 int bus, bool ext)
3020{
Joerg Roedeldbcd8612015-06-12 12:02:09 +02003021 int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
Joerg Roedel543c8dc2015-08-13 11:56:59 +02003022 struct context_entry *new_ce = NULL, ce;
Dan Williamsdfddb962015-10-09 18:16:46 -04003023 struct context_entry *old_ce = NULL;
Joerg Roedel543c8dc2015-08-13 11:56:59 +02003024 struct root_entry re;
Joerg Roedel091d42e2015-06-12 11:56:10 +02003025 phys_addr_t old_ce_phys;
3026
3027 tbl_idx = ext ? bus * 2 : bus;
Dan Williamsdfddb962015-10-09 18:16:46 -04003028 memcpy(&re, old_re, sizeof(re));
Joerg Roedel091d42e2015-06-12 11:56:10 +02003029
3030 for (devfn = 0; devfn < 256; devfn++) {
3031 /* First calculate the correct index */
3032 idx = (ext ? devfn * 2 : devfn) % 256;
3033
3034 if (idx == 0) {
3035 /* First save what we may have and clean up */
3036 if (new_ce) {
3037 tbl[tbl_idx] = new_ce;
3038 __iommu_flush_cache(iommu, new_ce,
3039 VTD_PAGE_SIZE);
3040 pos = 1;
3041 }
3042
3043 if (old_ce)
3044 iounmap(old_ce);
3045
3046 ret = 0;
3047 if (devfn < 0x80)
Joerg Roedel543c8dc2015-08-13 11:56:59 +02003048 old_ce_phys = root_entry_lctp(&re);
Joerg Roedel091d42e2015-06-12 11:56:10 +02003049 else
Joerg Roedel543c8dc2015-08-13 11:56:59 +02003050 old_ce_phys = root_entry_uctp(&re);
Joerg Roedel091d42e2015-06-12 11:56:10 +02003051
3052 if (!old_ce_phys) {
3053 if (ext && devfn == 0) {
3054 /* No LCTP, try UCTP */
3055 devfn = 0x7f;
3056 continue;
3057 } else {
3058 goto out;
3059 }
3060 }
3061
3062 ret = -ENOMEM;
Dan Williamsdfddb962015-10-09 18:16:46 -04003063 old_ce = memremap(old_ce_phys, PAGE_SIZE,
3064 MEMREMAP_WB);
Joerg Roedel091d42e2015-06-12 11:56:10 +02003065 if (!old_ce)
3066 goto out;
3067
3068 new_ce = alloc_pgtable_page(iommu->node);
3069 if (!new_ce)
3070 goto out_unmap;
3071
3072 ret = 0;
3073 }
3074
3075 /* Now copy the context entry */
Dan Williamsdfddb962015-10-09 18:16:46 -04003076 memcpy(&ce, old_ce + idx, sizeof(ce));
Joerg Roedel091d42e2015-06-12 11:56:10 +02003077
Joerg Roedelcf484d02015-06-12 12:21:46 +02003078 if (!__context_present(&ce))
Joerg Roedel091d42e2015-06-12 11:56:10 +02003079 continue;
3080
Joerg Roedeldbcd8612015-06-12 12:02:09 +02003081 did = context_domain_id(&ce);
3082 if (did >= 0 && did < cap_ndoms(iommu->cap))
3083 set_bit(did, iommu->domain_ids);
3084
Joerg Roedelcf484d02015-06-12 12:21:46 +02003085 /*
3086 * We need a marker for copied context entries. This
3087 * marker needs to work for the old format as well as
3088 * for extended context entries.
3089 *
3090 * Bit 67 of the context entry is used. In the old
3091 * format this bit is available to software, in the
3092 * extended format it is the PGE bit, but PGE is ignored
3093 * by HW if PASIDs are disabled (and thus still
3094 * available).
3095 *
3096 * So disable PASIDs first and then mark the entry
3097 * copied. This means that we don't copy PASID
3098 * translations from the old kernel, but this is fine as
3099 * faults there are not fatal.
3100 */
3101 context_clear_pasid_enable(&ce);
3102 context_set_copied(&ce);
3103
Joerg Roedel091d42e2015-06-12 11:56:10 +02003104 new_ce[idx] = ce;
3105 }
3106
3107 tbl[tbl_idx + pos] = new_ce;
3108
3109 __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
3110
3111out_unmap:
Dan Williamsdfddb962015-10-09 18:16:46 -04003112 memunmap(old_ce);
Joerg Roedel091d42e2015-06-12 11:56:10 +02003113
3114out:
3115 return ret;
3116}
3117
3118static int copy_translation_tables(struct intel_iommu *iommu)
3119{
3120 struct context_entry **ctxt_tbls;
Dan Williamsdfddb962015-10-09 18:16:46 -04003121 struct root_entry *old_rt;
Joerg Roedel091d42e2015-06-12 11:56:10 +02003122 phys_addr_t old_rt_phys;
3123 int ctxt_table_entries;
3124 unsigned long flags;
3125 u64 rtaddr_reg;
3126 int bus, ret;
Joerg Roedelc3361f22015-06-12 12:39:25 +02003127 bool new_ext, ext;
Joerg Roedel091d42e2015-06-12 11:56:10 +02003128
3129 rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
3130 ext = !!(rtaddr_reg & DMA_RTADDR_RTT);
Joerg Roedelc3361f22015-06-12 12:39:25 +02003131 new_ext = !!ecap_ecs(iommu->ecap);
3132
3133 /*
3134 * The RTT bit can only be changed when translation is disabled,
3135 * but disabling translation means to open a window for data
3136 * corruption. So bail out and don't copy anything if we would
3137 * have to change the bit.
3138 */
3139 if (new_ext != ext)
3140 return -EINVAL;
Joerg Roedel091d42e2015-06-12 11:56:10 +02003141
3142 old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
3143 if (!old_rt_phys)
3144 return -EINVAL;
3145
Dan Williamsdfddb962015-10-09 18:16:46 -04003146 old_rt = memremap(old_rt_phys, PAGE_SIZE, MEMREMAP_WB);
Joerg Roedel091d42e2015-06-12 11:56:10 +02003147 if (!old_rt)
3148 return -ENOMEM;
3149
3150 /* This is too big for the stack - allocate it from slab */
3151 ctxt_table_entries = ext ? 512 : 256;
3152 ret = -ENOMEM;
Kees Cook6396bb22018-06-12 14:03:40 -07003153 ctxt_tbls = kcalloc(ctxt_table_entries, sizeof(void *), GFP_KERNEL);
Joerg Roedel091d42e2015-06-12 11:56:10 +02003154 if (!ctxt_tbls)
3155 goto out_unmap;
3156
3157 for (bus = 0; bus < 256; bus++) {
3158 ret = copy_context_table(iommu, &old_rt[bus],
3159 ctxt_tbls, bus, ext);
3160 if (ret) {
3161 pr_err("%s: Failed to copy context table for bus %d\n",
3162 iommu->name, bus);
3163 continue;
3164 }
3165 }
3166
3167 spin_lock_irqsave(&iommu->lock, flags);
3168
3169 /* Context tables are copied, now write them to the root_entry table */
3170 for (bus = 0; bus < 256; bus++) {
3171 int idx = ext ? bus * 2 : bus;
3172 u64 val;
3173
3174 if (ctxt_tbls[idx]) {
3175 val = virt_to_phys(ctxt_tbls[idx]) | 1;
3176 iommu->root_entry[bus].lo = val;
3177 }
3178
3179 if (!ext || !ctxt_tbls[idx + 1])
3180 continue;
3181
3182 val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
3183 iommu->root_entry[bus].hi = val;
3184 }
3185
3186 spin_unlock_irqrestore(&iommu->lock, flags);
3187
3188 kfree(ctxt_tbls);
3189
3190 __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
3191
3192 ret = 0;
3193
3194out_unmap:
Dan Williamsdfddb962015-10-09 18:16:46 -04003195 memunmap(old_rt);
Joerg Roedel091d42e2015-06-12 11:56:10 +02003196
3197 return ret;
3198}
3199
Joseph Cihulab7792602011-05-03 00:08:37 -07003200static int __init init_dmars(void)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003201{
3202 struct dmar_drhd_unit *drhd;
3203 struct dmar_rmrr_unit *rmrr;
Joerg Roedela87f4912015-06-12 12:32:54 +02003204 bool copied_tables = false;
David Woodhouse832bd852014-03-07 15:08:36 +00003205 struct device *dev;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003206 struct intel_iommu *iommu;
Joerg Roedel13cf0172017-08-11 11:40:10 +02003207 int i, ret;
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07003208
3209 /*
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003210 * for each drhd
3211 * allocate root
3212 * initialize and program root entry to not present
3213 * endfor
3214 */
3215 for_each_drhd_unit(drhd) {
mark gross5e0d2a62008-03-04 15:22:08 -08003216 /*
3217 * lock not needed as this is only incremented in the single
3218 * threaded kernel __init code path all other access are read
3219 * only
3220 */
Jiang Liu78d8e702014-11-09 22:47:57 +08003221 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED) {
Mike Travis1b198bb2012-03-05 15:05:16 -08003222 g_num_of_iommus++;
3223 continue;
3224 }
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02003225 pr_err_once("Exceeded %d IOMMUs\n", DMAR_UNITS_SUPPORTED);
mark gross5e0d2a62008-03-04 15:22:08 -08003226 }
3227
Jiang Liuffebeb42014-11-09 22:48:02 +08003228 /* Preallocate enough resources for IOMMU hot-addition */
3229 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED)
3230 g_num_of_iommus = DMAR_UNITS_SUPPORTED;
3231
Weidong Hand9630fe2008-12-08 11:06:32 +08003232 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
3233 GFP_KERNEL);
3234 if (!g_iommus) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02003235 pr_err("Allocating global iommu array failed\n");
Weidong Hand9630fe2008-12-08 11:06:32 +08003236 ret = -ENOMEM;
3237 goto error;
3238 }
3239
Jiang Liu7c919772014-01-06 14:18:18 +08003240 for_each_active_iommu(iommu, drhd) {
Lu Baolu56283172018-07-14 15:46:54 +08003241 /*
3242 * Find the max pasid size of all IOMMU's in the system.
3243 * We need to ensure the system pasid table is no bigger
3244 * than the smallest supported.
3245 */
Lu Baolu765b6a92018-12-10 09:58:55 +08003246 if (pasid_supported(iommu)) {
Lu Baolu56283172018-07-14 15:46:54 +08003247 u32 temp = 2 << ecap_pss(iommu->ecap);
3248
3249 intel_pasid_max_id = min_t(u32, temp,
3250 intel_pasid_max_id);
3251 }
3252
Weidong Hand9630fe2008-12-08 11:06:32 +08003253 g_iommus[iommu->seq_id] = iommu;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003254
Joerg Roedelb63d80d2015-06-12 09:14:34 +02003255 intel_iommu_init_qi(iommu);
3256
Suresh Siddhae61d98d2008-07-10 11:16:35 -07003257 ret = iommu_init_domains(iommu);
3258 if (ret)
Jiang Liu989d51f2014-02-19 14:07:21 +08003259 goto free_iommu;
Suresh Siddhae61d98d2008-07-10 11:16:35 -07003260
Joerg Roedel4158c2e2015-06-12 10:14:02 +02003261 init_translation_status(iommu);
3262
Joerg Roedel091d42e2015-06-12 11:56:10 +02003263 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
3264 iommu_disable_translation(iommu);
3265 clear_translation_pre_enabled(iommu);
3266 pr_warn("Translation was enabled for %s but we are not in kdump mode\n",
3267 iommu->name);
3268 }
Joerg Roedel4158c2e2015-06-12 10:14:02 +02003269
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003270 /*
3271 * TBD:
3272 * we could share the same root & context tables
Lucas De Marchi25985ed2011-03-30 22:57:33 -03003273 * among all IOMMU's. Need to Split it later.
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003274 */
3275 ret = iommu_alloc_root_entry(iommu);
Jiang Liuffebeb42014-11-09 22:48:02 +08003276 if (ret)
Jiang Liu989d51f2014-02-19 14:07:21 +08003277 goto free_iommu;
Joerg Roedel5f0a7f72015-06-12 09:18:53 +02003278
Joerg Roedel091d42e2015-06-12 11:56:10 +02003279 if (translation_pre_enabled(iommu)) {
3280 pr_info("Translation already enabled - trying to copy translation structures\n");
3281
3282 ret = copy_translation_tables(iommu);
3283 if (ret) {
3284 /*
3285 * We found the IOMMU with translation
3286 * enabled - but failed to copy over the
3287 * old root-entry table. Try to proceed
3288 * by disabling translation now and
3289 * allocating a clean root-entry table.
3290 * This might cause DMAR faults, but
3291 * probably the dump will still succeed.
3292 */
3293 pr_err("Failed to copy translation tables from previous kernel for %s\n",
3294 iommu->name);
3295 iommu_disable_translation(iommu);
3296 clear_translation_pre_enabled(iommu);
3297 } else {
3298 pr_info("Copied translation tables from previous kernel for %s\n",
3299 iommu->name);
Joerg Roedela87f4912015-06-12 12:32:54 +02003300 copied_tables = true;
Joerg Roedel091d42e2015-06-12 11:56:10 +02003301 }
3302 }
3303
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07003304 if (!ecap_pass_through(iommu->ecap))
David Woodhouse19943b02009-08-04 16:19:20 +01003305 hw_pass_through = 0;
David Woodhouse8a94ade2015-03-24 14:54:56 +00003306#ifdef CONFIG_INTEL_IOMMU_SVM
Lu Baolu765b6a92018-12-10 09:58:55 +08003307 if (pasid_supported(iommu))
Lu Baolud9737952018-07-14 15:47:02 +08003308 intel_svm_init(iommu);
David Woodhouse8a94ade2015-03-24 14:54:56 +00003309#endif
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003310 }
3311
Joerg Roedela4c34ff2016-06-17 11:29:48 +02003312 /*
3313 * Now that qi is enabled on all iommus, set the root entry and flush
3314 * caches. This is required on some Intel X58 chipsets, otherwise the
3315 * flush_context function will loop forever and the boot hangs.
3316 */
3317 for_each_active_iommu(iommu, drhd) {
3318 iommu_flush_write_buffer(iommu);
3319 iommu_set_root_entry(iommu);
3320 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
3321 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3322 }
3323
David Woodhouse19943b02009-08-04 16:19:20 +01003324 if (iommu_pass_through)
David Woodhousee0fc7e02009-09-30 09:12:17 -07003325 iommu_identity_mapping |= IDENTMAP_ALL;
3326
Suresh Siddhad3f13812011-08-23 17:05:25 -07003327#ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
David Woodhousee0fc7e02009-09-30 09:12:17 -07003328 iommu_identity_mapping |= IDENTMAP_GFX;
David Woodhouse19943b02009-08-04 16:19:20 +01003329#endif
David Woodhousee0fc7e02009-09-30 09:12:17 -07003330
Ashok Raj21e722c2017-01-30 09:39:53 -08003331 check_tylersburg_isoch();
3332
Joerg Roedel86080cc2015-06-12 12:27:16 +02003333 if (iommu_identity_mapping) {
3334 ret = si_domain_init(hw_pass_through);
3335 if (ret)
3336 goto free_iommu;
3337 }
3338
David Woodhousee0fc7e02009-09-30 09:12:17 -07003339
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07003340 /*
Joerg Roedela87f4912015-06-12 12:32:54 +02003341 * If we copied translations from a previous kernel in the kdump
3342 * case, we can not assign the devices to domains now, as that
3343 * would eliminate the old mappings. So skip this part and defer
3344 * the assignment to device driver initialization time.
3345 */
3346 if (copied_tables)
3347 goto domains_done;
3348
3349 /*
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07003350 * If pass through is not set or not enabled, setup context entries for
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07003351 * identity mappings for rmrr, gfx, and isa and may fall back to static
3352 * identity mapping if iommu_identity_mapping is set.
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07003353 */
David Woodhouse19943b02009-08-04 16:19:20 +01003354 if (iommu_identity_mapping) {
3355 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
3356 if (ret) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02003357 pr_crit("Failed to setup IOMMU pass-through\n");
Jiang Liu989d51f2014-02-19 14:07:21 +08003358 goto free_iommu;
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07003359 }
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07003360 }
David Woodhouse19943b02009-08-04 16:19:20 +01003361 /*
3362 * For each rmrr
3363 * for each dev attached to rmrr
3364 * do
3365 * locate drhd for dev, alloc domain for dev
3366 * allocate free domain
3367 * allocate page table entries for rmrr
3368 * if context not allocated for bus
3369 * allocate and init context
3370 * set present in root table for this bus
3371 * init context with domain, translation etc
3372 * endfor
3373 * endfor
3374 */
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02003375 pr_info("Setting RMRR:\n");
David Woodhouse19943b02009-08-04 16:19:20 +01003376 for_each_rmrr_units(rmrr) {
Jiang Liub683b232014-02-19 14:07:32 +08003377 /* some BIOS lists non-exist devices in DMAR table. */
3378 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
David Woodhouse832bd852014-03-07 15:08:36 +00003379 i, dev) {
David Woodhouse0b9d9752014-03-09 15:48:15 -07003380 ret = iommu_prepare_rmrr_dev(rmrr, dev);
David Woodhouse19943b02009-08-04 16:19:20 +01003381 if (ret)
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02003382 pr_err("Mapping reserved region failed\n");
David Woodhouse19943b02009-08-04 16:19:20 +01003383 }
3384 }
3385
3386 iommu_prepare_isa();
Keshavamurthy, Anil S49a04292007-10-21 16:41:57 -07003387
Joerg Roedela87f4912015-06-12 12:32:54 +02003388domains_done:
3389
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003390 /*
3391 * for each drhd
3392 * enable fault log
3393 * global invalidate context cache
3394 * global invalidate iotlb
3395 * enable translation
3396 */
Jiang Liu7c919772014-01-06 14:18:18 +08003397 for_each_iommu(iommu, drhd) {
Joseph Cihula51a63e62011-03-21 11:04:24 -07003398 if (drhd->ignored) {
3399 /*
3400 * we always have to disable PMRs or DMA may fail on
3401 * this device
3402 */
3403 if (force_on)
Jiang Liu7c919772014-01-06 14:18:18 +08003404 iommu_disable_protect_mem_regions(iommu);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003405 continue;
Joseph Cihula51a63e62011-03-21 11:04:24 -07003406 }
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003407
3408 iommu_flush_write_buffer(iommu);
3409
David Woodhousea222a7f2015-10-07 23:35:18 +01003410#ifdef CONFIG_INTEL_IOMMU_SVM
Lu Baolu765b6a92018-12-10 09:58:55 +08003411 if (pasid_supported(iommu) && ecap_prs(iommu->ecap)) {
David Woodhousea222a7f2015-10-07 23:35:18 +01003412 ret = intel_svm_enable_prq(iommu);
3413 if (ret)
3414 goto free_iommu;
3415 }
3416#endif
Keshavamurthy, Anil S3460a6d2007-10-21 16:41:54 -07003417 ret = dmar_set_interrupt(iommu);
3418 if (ret)
Jiang Liu989d51f2014-02-19 14:07:21 +08003419 goto free_iommu;
Keshavamurthy, Anil S3460a6d2007-10-21 16:41:54 -07003420
Joerg Roedel8939ddf2015-06-12 14:40:01 +02003421 if (!translation_pre_enabled(iommu))
3422 iommu_enable_translation(iommu);
3423
David Woodhouseb94996c2009-09-19 15:28:12 -07003424 iommu_disable_protect_mem_regions(iommu);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003425 }
3426
3427 return 0;
Jiang Liu989d51f2014-02-19 14:07:21 +08003428
3429free_iommu:
Jiang Liuffebeb42014-11-09 22:48:02 +08003430 for_each_active_iommu(iommu, drhd) {
3431 disable_dmar_iommu(iommu);
Jiang Liua868e6b2014-01-06 14:18:20 +08003432 free_dmar_iommu(iommu);
Jiang Liuffebeb42014-11-09 22:48:02 +08003433 }
Joerg Roedel13cf0172017-08-11 11:40:10 +02003434
Weidong Hand9630fe2008-12-08 11:06:32 +08003435 kfree(g_iommus);
Joerg Roedel13cf0172017-08-11 11:40:10 +02003436
Jiang Liu989d51f2014-02-19 14:07:21 +08003437error:
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003438 return ret;
3439}
3440
David Woodhouse5a5e02a2009-07-04 09:35:44 +01003441/* This takes a number of _MM_ pages, not VTD pages */
Omer Peleg2aac6302016-04-20 11:33:57 +03003442static unsigned long intel_alloc_iova(struct device *dev,
David Woodhouse875764d2009-06-28 21:20:51 +01003443 struct dmar_domain *domain,
3444 unsigned long nrpages, uint64_t dma_mask)
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003445{
Omer Peleg22e2f9f2016-04-20 11:34:11 +03003446 unsigned long iova_pfn = 0;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003447
David Woodhouse875764d2009-06-28 21:20:51 +01003448 /* Restrict dma_mask to the width that the iommu can handle */
3449 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
Robin Murphy8f6429c2015-07-16 19:40:12 +01003450 /* Ensure we reserve the whole size-aligned region */
3451 nrpages = __roundup_pow_of_two(nrpages);
David Woodhouse875764d2009-06-28 21:20:51 +01003452
3453 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003454 /*
3455 * First try to allocate an io virtual address in
Yang Hongyang284901a2009-04-06 19:01:15 -07003456 * DMA_BIT_MASK(32) and if that fails then try allocating
Joe Perches36098012007-12-17 11:40:11 -08003457 * from higher range
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003458 */
Omer Peleg22e2f9f2016-04-20 11:34:11 +03003459 iova_pfn = alloc_iova_fast(&domain->iovad, nrpages,
Tomasz Nowicki538d5b32017-09-20 10:52:02 +02003460 IOVA_PFN(DMA_BIT_MASK(32)), false);
Omer Peleg22e2f9f2016-04-20 11:34:11 +03003461 if (iova_pfn)
3462 return iova_pfn;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003463 }
Tomasz Nowicki538d5b32017-09-20 10:52:02 +02003464 iova_pfn = alloc_iova_fast(&domain->iovad, nrpages,
3465 IOVA_PFN(dma_mask), true);
Omer Peleg22e2f9f2016-04-20 11:34:11 +03003466 if (unlikely(!iova_pfn)) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02003467 pr_err("Allocating %ld-page iova for %s failed",
David Woodhouse207e3592014-03-09 16:12:32 -07003468 nrpages, dev_name(dev));
Omer Peleg2aac6302016-04-20 11:33:57 +03003469 return 0;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003470 }
3471
Omer Peleg22e2f9f2016-04-20 11:34:11 +03003472 return iova_pfn;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003473}
3474
Lu Baolu9ddbfb42018-07-14 15:46:57 +08003475struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003476{
Joerg Roedel1c5ebba2016-08-25 13:52:51 +02003477 struct dmar_domain *domain, *tmp;
Joerg Roedelb1ce5b72015-09-23 19:16:01 +02003478 struct dmar_rmrr_unit *rmrr;
Joerg Roedelb1ce5b72015-09-23 19:16:01 +02003479 struct device *i_dev;
3480 int i, ret;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003481
Joerg Roedel1c5ebba2016-08-25 13:52:51 +02003482 domain = find_domain(dev);
3483 if (domain)
3484 goto out;
3485
3486 domain = find_or_alloc_domain(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
3487 if (!domain)
3488 goto out;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003489
Joerg Roedelb1ce5b72015-09-23 19:16:01 +02003490 /* We have a new domain - setup possible RMRRs for the device */
3491 rcu_read_lock();
3492 for_each_rmrr_units(rmrr) {
3493 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3494 i, i_dev) {
3495 if (i_dev != dev)
3496 continue;
3497
3498 ret = domain_prepare_identity_map(dev, domain,
3499 rmrr->base_address,
3500 rmrr->end_address);
3501 if (ret)
3502 dev_err(dev, "Mapping reserved region failed\n");
3503 }
3504 }
3505 rcu_read_unlock();
3506
Joerg Roedel1c5ebba2016-08-25 13:52:51 +02003507 tmp = set_domain_for_dev(dev, domain);
3508 if (!tmp || domain != tmp) {
3509 domain_exit(domain);
3510 domain = tmp;
3511 }
3512
3513out:
3514
3515 if (!domain)
3516 pr_err("Allocating domain for %s failed\n", dev_name(dev));
3517
3518
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003519 return domain;
3520}
3521
David Woodhouseecb509e2014-03-09 16:29:55 -07003522/* Check if the dev needs to go through non-identity map and unmap process.*/
David Woodhouse73676832009-07-04 14:08:36 +01003523static int iommu_no_mapping(struct device *dev)
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07003524{
3525 int found;
3526
David Woodhouse3d891942014-03-06 15:59:26 +00003527 if (iommu_dummy(dev))
David Woodhouse1e4c64c2009-07-04 10:40:38 +01003528 return 1;
3529
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07003530 if (!iommu_identity_mapping)
David Woodhouse1e4c64c2009-07-04 10:40:38 +01003531 return 0;
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07003532
David Woodhouse9b226622014-03-09 14:03:28 -07003533 found = identity_mapping(dev);
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07003534 if (found) {
David Woodhouseecb509e2014-03-09 16:29:55 -07003535 if (iommu_should_identity_map(dev, 0))
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07003536 return 1;
3537 else {
3538 /*
3539 * 32 bit DMA is removed from si_domain and fall back
3540 * to non-identity mapping.
3541 */
Joerg Roedele6de0f82015-07-22 16:30:36 +02003542 dmar_remove_one_dev_info(si_domain, dev);
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02003543 pr_info("32bit %s uses non-identity mapping\n",
3544 dev_name(dev));
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07003545 return 0;
3546 }
3547 } else {
3548 /*
3549 * In case of a detached 64 bit DMA device from vm, the device
3550 * is put into si_domain for identity mapping.
3551 */
David Woodhouseecb509e2014-03-09 16:29:55 -07003552 if (iommu_should_identity_map(dev, 0)) {
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07003553 int ret;
Joerg Roedel28ccce02015-07-21 14:45:31 +02003554 ret = domain_add_dev_info(si_domain, dev);
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07003555 if (!ret) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02003556 pr_info("64bit %s uses identity mapping\n",
3557 dev_name(dev));
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07003558 return 1;
3559 }
3560 }
3561 }
3562
David Woodhouse1e4c64c2009-07-04 10:40:38 +01003563 return 0;
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07003564}
3565
David Woodhouse5040a912014-03-09 16:14:00 -07003566static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
FUJITA Tomonoribb9e6d62008-10-15 16:08:28 +09003567 size_t size, int dir, u64 dma_mask)
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003568{
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003569 struct dmar_domain *domain;
Fenghua Yu5b6985c2008-10-16 18:02:32 -07003570 phys_addr_t start_paddr;
Omer Peleg2aac6302016-04-20 11:33:57 +03003571 unsigned long iova_pfn;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003572 int prot = 0;
Ingo Molnar6865f0d2008-04-22 11:09:04 +02003573 int ret;
Weidong Han8c11e792008-12-08 15:29:22 +08003574 struct intel_iommu *iommu;
Fenghua Yu33041ec2009-08-04 15:10:59 -07003575 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003576
3577 BUG_ON(dir == DMA_NONE);
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07003578
David Woodhouse5040a912014-03-09 16:14:00 -07003579 if (iommu_no_mapping(dev))
Ingo Molnar6865f0d2008-04-22 11:09:04 +02003580 return paddr;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003581
David Woodhouse5040a912014-03-09 16:14:00 -07003582 domain = get_valid_domain_for_dev(dev);
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003583 if (!domain)
3584 return 0;
3585
Weidong Han8c11e792008-12-08 15:29:22 +08003586 iommu = domain_get_iommu(domain);
David Woodhouse88cb6a72009-06-28 15:03:06 +01003587 size = aligned_nrpages(paddr, size);
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003588
Omer Peleg2aac6302016-04-20 11:33:57 +03003589 iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
3590 if (!iova_pfn)
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003591 goto error;
3592
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003593 /*
3594 * Check if DMAR supports zero-length reads on write only
3595 * mappings..
3596 */
3597 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
Weidong Han8c11e792008-12-08 15:29:22 +08003598 !cap_zlr(iommu->cap))
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003599 prot |= DMA_PTE_READ;
3600 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3601 prot |= DMA_PTE_WRITE;
3602 /*
Ingo Molnar6865f0d2008-04-22 11:09:04 +02003603 * paddr - (paddr + size) might be partial page, we should map the whole
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003604 * page. Note: if two part of one page are separately mapped, we
Ingo Molnar6865f0d2008-04-22 11:09:04 +02003605 * might have two guest_addr mapping to the same host paddr, but this
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003606 * is not a big problem
3607 */
Omer Peleg2aac6302016-04-20 11:33:57 +03003608 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova_pfn),
Fenghua Yu33041ec2009-08-04 15:10:59 -07003609 mm_to_dma_pfn(paddr_pfn), size, prot);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003610 if (ret)
3611 goto error;
3612
Omer Peleg2aac6302016-04-20 11:33:57 +03003613 start_paddr = (phys_addr_t)iova_pfn << PAGE_SHIFT;
David Woodhouse03d6a242009-06-28 15:33:46 +01003614 start_paddr += paddr & ~PAGE_MASK;
3615 return start_paddr;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003616
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003617error:
Omer Peleg2aac6302016-04-20 11:33:57 +03003618 if (iova_pfn)
Omer Peleg22e2f9f2016-04-20 11:34:11 +03003619 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02003620 pr_err("Device %s request: %zx@%llx dir %d --- failed\n",
David Woodhouse5040a912014-03-09 16:14:00 -07003621 dev_name(dev), size, (unsigned long long)paddr, dir);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003622 return 0;
3623}
3624
FUJITA Tomonoriffbbef52009-01-05 23:47:26 +09003625static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3626 unsigned long offset, size_t size,
3627 enum dma_data_direction dir,
Krzysztof Kozlowski00085f12016-08-03 13:46:00 -07003628 unsigned long attrs)
FUJITA Tomonoribb9e6d62008-10-15 16:08:28 +09003629{
FUJITA Tomonoriffbbef52009-01-05 23:47:26 +09003630 return __intel_map_single(dev, page_to_phys(page) + offset, size,
David Woodhouse46333e32014-03-10 20:01:21 -07003631 dir, *dev->dma_mask);
FUJITA Tomonoribb9e6d62008-10-15 16:08:28 +09003632}
3633
Omer Peleg769530e2016-04-20 11:33:25 +03003634static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003635{
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003636 struct dmar_domain *domain;
David Woodhoused794dc92009-06-28 00:27:49 +01003637 unsigned long start_pfn, last_pfn;
Omer Peleg769530e2016-04-20 11:33:25 +03003638 unsigned long nrpages;
Omer Peleg2aac6302016-04-20 11:33:57 +03003639 unsigned long iova_pfn;
Weidong Han8c11e792008-12-08 15:29:22 +08003640 struct intel_iommu *iommu;
David Woodhouseea8ea462014-03-05 17:09:32 +00003641 struct page *freelist;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003642
David Woodhouse73676832009-07-04 14:08:36 +01003643 if (iommu_no_mapping(dev))
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003644 return;
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07003645
David Woodhouse1525a292014-03-06 16:19:30 +00003646 domain = find_domain(dev);
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003647 BUG_ON(!domain);
3648
Weidong Han8c11e792008-12-08 15:29:22 +08003649 iommu = domain_get_iommu(domain);
3650
Omer Peleg2aac6302016-04-20 11:33:57 +03003651 iova_pfn = IOVA_PFN(dev_addr);
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003652
Omer Peleg769530e2016-04-20 11:33:25 +03003653 nrpages = aligned_nrpages(dev_addr, size);
Omer Peleg2aac6302016-04-20 11:33:57 +03003654 start_pfn = mm_to_dma_pfn(iova_pfn);
Omer Peleg769530e2016-04-20 11:33:25 +03003655 last_pfn = start_pfn + nrpages - 1;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003656
David Woodhoused794dc92009-06-28 00:27:49 +01003657 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
David Woodhouse207e3592014-03-09 16:12:32 -07003658 dev_name(dev), start_pfn, last_pfn);
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003659
David Woodhouseea8ea462014-03-05 17:09:32 +00003660 freelist = domain_unmap(domain, start_pfn, last_pfn);
David Woodhoused794dc92009-06-28 00:27:49 +01003661
mark gross5e0d2a62008-03-04 15:22:08 -08003662 if (intel_iommu_strict) {
Joerg Roedela1ddcbe2015-07-21 15:20:32 +02003663 iommu_flush_iotlb_psi(iommu, domain, start_pfn,
Omer Peleg769530e2016-04-20 11:33:25 +03003664 nrpages, !freelist, 0);
mark gross5e0d2a62008-03-04 15:22:08 -08003665 /* free iova */
Omer Peleg22e2f9f2016-04-20 11:34:11 +03003666 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(nrpages));
David Woodhouseea8ea462014-03-05 17:09:32 +00003667 dma_free_pagelist(freelist);
mark gross5e0d2a62008-03-04 15:22:08 -08003668 } else {
Joerg Roedel13cf0172017-08-11 11:40:10 +02003669 queue_iova(&domain->iovad, iova_pfn, nrpages,
3670 (unsigned long)freelist);
mark gross5e0d2a62008-03-04 15:22:08 -08003671 /*
3672 * queue up the release of the unmap to save the 1/6th of the
3673 * cpu used up by the iotlb flush operation...
3674 */
mark gross5e0d2a62008-03-04 15:22:08 -08003675 }
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003676}
3677
Jiang Liud41a4ad2014-07-11 14:19:34 +08003678static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3679 size_t size, enum dma_data_direction dir,
Krzysztof Kozlowski00085f12016-08-03 13:46:00 -07003680 unsigned long attrs)
Jiang Liud41a4ad2014-07-11 14:19:34 +08003681{
Omer Peleg769530e2016-04-20 11:33:25 +03003682 intel_unmap(dev, dev_addr, size);
Jiang Liud41a4ad2014-07-11 14:19:34 +08003683}
3684
David Woodhouse5040a912014-03-09 16:14:00 -07003685static void *intel_alloc_coherent(struct device *dev, size_t size,
Andrzej Pietrasiewiczbaa676f2012-03-27 14:28:18 +02003686 dma_addr_t *dma_handle, gfp_t flags,
Krzysztof Kozlowski00085f12016-08-03 13:46:00 -07003687 unsigned long attrs)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003688{
Christoph Hellwig7ec916f2018-07-05 13:29:55 -06003689 struct page *page = NULL;
3690 int order;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003691
Christoph Hellwig7ec916f2018-07-05 13:29:55 -06003692 size = PAGE_ALIGN(size);
3693 order = get_order(size);
Alex Williamsone8bb9102009-11-04 15:59:34 -07003694
Christoph Hellwig7ec916f2018-07-05 13:29:55 -06003695 if (!iommu_no_mapping(dev))
3696 flags &= ~(GFP_DMA | GFP_DMA32);
3697 else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
3698 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
3699 flags |= GFP_DMA;
3700 else
3701 flags |= GFP_DMA32;
3702 }
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003703
Christoph Hellwig7ec916f2018-07-05 13:29:55 -06003704 if (gfpflags_allow_blocking(flags)) {
3705 unsigned int count = size >> PAGE_SHIFT;
3706
Marek Szyprowskid834c5a2018-08-17 15:49:00 -07003707 page = dma_alloc_from_contiguous(dev, count, order,
3708 flags & __GFP_NOWARN);
Christoph Hellwig7ec916f2018-07-05 13:29:55 -06003709 if (page && iommu_no_mapping(dev) &&
3710 page_to_phys(page) + size > dev->coherent_dma_mask) {
3711 dma_release_from_contiguous(dev, page, count);
3712 page = NULL;
3713 }
3714 }
3715
3716 if (!page)
3717 page = alloc_pages(flags, order);
3718 if (!page)
3719 return NULL;
3720 memset(page_address(page), 0, size);
3721
3722 *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
3723 DMA_BIDIRECTIONAL,
3724 dev->coherent_dma_mask);
3725 if (*dma_handle)
3726 return page_address(page);
3727 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3728 __free_pages(page, order);
3729
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003730 return NULL;
3731}
3732
David Woodhouse5040a912014-03-09 16:14:00 -07003733static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
Krzysztof Kozlowski00085f12016-08-03 13:46:00 -07003734 dma_addr_t dma_handle, unsigned long attrs)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003735{
Christoph Hellwig7ec916f2018-07-05 13:29:55 -06003736 int order;
3737 struct page *page = virt_to_page(vaddr);
3738
3739 size = PAGE_ALIGN(size);
3740 order = get_order(size);
3741
3742 intel_unmap(dev, dma_handle, size);
3743 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3744 __free_pages(page, order);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003745}
3746
David Woodhouse5040a912014-03-09 16:14:00 -07003747static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
FUJITA Tomonorid7ab5c42009-01-28 21:53:18 +09003748 int nelems, enum dma_data_direction dir,
Krzysztof Kozlowski00085f12016-08-03 13:46:00 -07003749 unsigned long attrs)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003750{
Omer Peleg769530e2016-04-20 11:33:25 +03003751 dma_addr_t startaddr = sg_dma_address(sglist) & PAGE_MASK;
3752 unsigned long nrpages = 0;
3753 struct scatterlist *sg;
3754 int i;
3755
3756 for_each_sg(sglist, sg, nelems, i) {
3757 nrpages += aligned_nrpages(sg_dma_address(sg), sg_dma_len(sg));
3758 }
3759
3760 intel_unmap(dev, startaddr, nrpages << VTD_PAGE_SHIFT);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003761}
3762
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003763static int intel_nontranslate_map_sg(struct device *hddev,
FUJITA Tomonoric03ab372007-10-21 16:42:00 -07003764 struct scatterlist *sglist, int nelems, int dir)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003765{
3766 int i;
FUJITA Tomonoric03ab372007-10-21 16:42:00 -07003767 struct scatterlist *sg;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003768
FUJITA Tomonoric03ab372007-10-21 16:42:00 -07003769 for_each_sg(sglist, sg, nelems, i) {
FUJITA Tomonori12d4d402007-10-23 09:32:25 +02003770 BUG_ON(!sg_page(sg));
Robin Murphy29a90b72017-09-28 15:14:01 +01003771 sg->dma_address = sg_phys(sg);
FUJITA Tomonoric03ab372007-10-21 16:42:00 -07003772 sg->dma_length = sg->length;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003773 }
3774 return nelems;
3775}
3776
David Woodhouse5040a912014-03-09 16:14:00 -07003777static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
Krzysztof Kozlowski00085f12016-08-03 13:46:00 -07003778 enum dma_data_direction dir, unsigned long attrs)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003779{
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003780 int i;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003781 struct dmar_domain *domain;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003782 size_t size = 0;
3783 int prot = 0;
Omer Peleg2aac6302016-04-20 11:33:57 +03003784 unsigned long iova_pfn;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003785 int ret;
FUJITA Tomonoric03ab372007-10-21 16:42:00 -07003786 struct scatterlist *sg;
David Woodhouseb536d242009-06-28 14:49:31 +01003787 unsigned long start_vpfn;
Weidong Han8c11e792008-12-08 15:29:22 +08003788 struct intel_iommu *iommu;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003789
3790 BUG_ON(dir == DMA_NONE);
David Woodhouse5040a912014-03-09 16:14:00 -07003791 if (iommu_no_mapping(dev))
3792 return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003793
David Woodhouse5040a912014-03-09 16:14:00 -07003794 domain = get_valid_domain_for_dev(dev);
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003795 if (!domain)
3796 return 0;
3797
Weidong Han8c11e792008-12-08 15:29:22 +08003798 iommu = domain_get_iommu(domain);
3799
David Woodhouseb536d242009-06-28 14:49:31 +01003800 for_each_sg(sglist, sg, nelems, i)
David Woodhouse88cb6a72009-06-28 15:03:06 +01003801 size += aligned_nrpages(sg->offset, sg->length);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003802
Omer Peleg2aac6302016-04-20 11:33:57 +03003803 iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
David Woodhouse5040a912014-03-09 16:14:00 -07003804 *dev->dma_mask);
Omer Peleg2aac6302016-04-20 11:33:57 +03003805 if (!iova_pfn) {
FUJITA Tomonoric03ab372007-10-21 16:42:00 -07003806 sglist->dma_length = 0;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003807 return 0;
3808 }
3809
3810 /*
3811 * Check if DMAR supports zero-length reads on write only
3812 * mappings..
3813 */
3814 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
Weidong Han8c11e792008-12-08 15:29:22 +08003815 !cap_zlr(iommu->cap))
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003816 prot |= DMA_PTE_READ;
3817 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3818 prot |= DMA_PTE_WRITE;
3819
Omer Peleg2aac6302016-04-20 11:33:57 +03003820 start_vpfn = mm_to_dma_pfn(iova_pfn);
David Woodhousee1605492009-06-29 11:17:38 +01003821
Fenghua Yuf5329592009-08-04 15:09:37 -07003822 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
David Woodhousee1605492009-06-29 11:17:38 +01003823 if (unlikely(ret)) {
David Woodhousee1605492009-06-29 11:17:38 +01003824 dma_pte_free_pagetable(domain, start_vpfn,
David Dillowbc24c572017-06-28 19:42:23 -07003825 start_vpfn + size - 1,
3826 agaw_to_level(domain->agaw) + 1);
Omer Peleg22e2f9f2016-04-20 11:34:11 +03003827 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
David Woodhousee1605492009-06-29 11:17:38 +01003828 return 0;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003829 }
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003830
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003831 return nelems;
3832}
3833
FUJITA Tomonoridfb805e2009-01-28 21:53:17 +09003834static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3835{
3836 return !dma_addr;
3837}
3838
Christoph Hellwig02b4da52018-09-17 19:10:31 +02003839static const struct dma_map_ops intel_dma_ops = {
Andrzej Pietrasiewiczbaa676f2012-03-27 14:28:18 +02003840 .alloc = intel_alloc_coherent,
3841 .free = intel_free_coherent,
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003842 .map_sg = intel_map_sg,
3843 .unmap_sg = intel_unmap_sg,
FUJITA Tomonoriffbbef52009-01-05 23:47:26 +09003844 .map_page = intel_map_page,
3845 .unmap_page = intel_unmap_page,
FUJITA Tomonoridfb805e2009-01-28 21:53:17 +09003846 .mapping_error = intel_mapping_error,
Christoph Hellwigfec777c2018-03-19 11:38:15 +01003847 .dma_supported = dma_direct_supported,
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003848};
3849
3850static inline int iommu_domain_cache_init(void)
3851{
3852 int ret = 0;
3853
3854 iommu_domain_cache = kmem_cache_create("iommu_domain",
3855 sizeof(struct dmar_domain),
3856 0,
3857 SLAB_HWCACHE_ALIGN,
3858
3859 NULL);
3860 if (!iommu_domain_cache) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02003861 pr_err("Couldn't create iommu_domain cache\n");
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003862 ret = -ENOMEM;
3863 }
3864
3865 return ret;
3866}
3867
3868static inline int iommu_devinfo_cache_init(void)
3869{
3870 int ret = 0;
3871
3872 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3873 sizeof(struct device_domain_info),
3874 0,
3875 SLAB_HWCACHE_ALIGN,
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003876 NULL);
3877 if (!iommu_devinfo_cache) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02003878 pr_err("Couldn't create devinfo cache\n");
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003879 ret = -ENOMEM;
3880 }
3881
3882 return ret;
3883}
3884
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003885static int __init iommu_init_mempool(void)
3886{
3887 int ret;
Sakari Ailusae1ff3d2015-07-13 14:31:28 +03003888 ret = iova_cache_get();
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003889 if (ret)
3890 return ret;
3891
3892 ret = iommu_domain_cache_init();
3893 if (ret)
3894 goto domain_error;
3895
3896 ret = iommu_devinfo_cache_init();
3897 if (!ret)
3898 return ret;
3899
3900 kmem_cache_destroy(iommu_domain_cache);
3901domain_error:
Sakari Ailusae1ff3d2015-07-13 14:31:28 +03003902 iova_cache_put();
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003903
3904 return -ENOMEM;
3905}
3906
3907static void __init iommu_exit_mempool(void)
3908{
3909 kmem_cache_destroy(iommu_devinfo_cache);
3910 kmem_cache_destroy(iommu_domain_cache);
Sakari Ailusae1ff3d2015-07-13 14:31:28 +03003911 iova_cache_put();
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003912}
3913
Dan Williams556ab452010-07-23 15:47:56 -07003914static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3915{
3916 struct dmar_drhd_unit *drhd;
3917 u32 vtbar;
3918 int rc;
3919
3920 /* We know that this device on this chipset has its own IOMMU.
3921 * If we find it under a different IOMMU, then the BIOS is lying
3922 * to us. Hope that the IOMMU for this device is actually
3923 * disabled, and it needs no translation...
3924 */
3925 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3926 if (rc) {
3927 /* "can't" happen */
3928 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3929 return;
3930 }
3931 vtbar &= 0xffff0000;
3932
3933 /* we know that the this iommu should be at offset 0xa000 from vtbar */
3934 drhd = dmar_find_matched_drhd_unit(pdev);
3935 if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3936 TAINT_FIRMWARE_WORKAROUND,
3937 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3938 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3939}
3940DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3941
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003942static void __init init_no_remapping_devices(void)
3943{
3944 struct dmar_drhd_unit *drhd;
David Woodhouse832bd852014-03-07 15:08:36 +00003945 struct device *dev;
Jiang Liub683b232014-02-19 14:07:32 +08003946 int i;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003947
3948 for_each_drhd_unit(drhd) {
3949 if (!drhd->include_all) {
Jiang Liub683b232014-02-19 14:07:32 +08003950 for_each_active_dev_scope(drhd->devices,
3951 drhd->devices_cnt, i, dev)
3952 break;
David Woodhouse832bd852014-03-07 15:08:36 +00003953 /* ignore DMAR unit if no devices exist */
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003954 if (i == drhd->devices_cnt)
3955 drhd->ignored = 1;
3956 }
3957 }
3958
Jiang Liu7c919772014-01-06 14:18:18 +08003959 for_each_active_drhd_unit(drhd) {
Jiang Liu7c919772014-01-06 14:18:18 +08003960 if (drhd->include_all)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003961 continue;
3962
Jiang Liub683b232014-02-19 14:07:32 +08003963 for_each_active_dev_scope(drhd->devices,
3964 drhd->devices_cnt, i, dev)
David Woodhouse832bd852014-03-07 15:08:36 +00003965 if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003966 break;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003967 if (i < drhd->devices_cnt)
3968 continue;
3969
David Woodhousec0771df2011-10-14 20:59:46 +01003970 /* This IOMMU has *only* gfx devices. Either bypass it or
3971 set the gfx_mapped flag, as appropriate */
3972 if (dmar_map_gfx) {
3973 intel_iommu_gfx_mapped = 1;
3974 } else {
3975 drhd->ignored = 1;
Jiang Liub683b232014-02-19 14:07:32 +08003976 for_each_active_dev_scope(drhd->devices,
3977 drhd->devices_cnt, i, dev)
David Woodhouse832bd852014-03-07 15:08:36 +00003978 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003979 }
3980 }
3981}
3982
Fenghua Yuf59c7b62009-03-27 14:22:42 -07003983#ifdef CONFIG_SUSPEND
3984static int init_iommu_hw(void)
3985{
3986 struct dmar_drhd_unit *drhd;
3987 struct intel_iommu *iommu = NULL;
3988
3989 for_each_active_iommu(iommu, drhd)
3990 if (iommu->qi)
3991 dmar_reenable_qi(iommu);
3992
Joseph Cihulab7792602011-05-03 00:08:37 -07003993 for_each_iommu(iommu, drhd) {
3994 if (drhd->ignored) {
3995 /*
3996 * we always have to disable PMRs or DMA may fail on
3997 * this device
3998 */
3999 if (force_on)
4000 iommu_disable_protect_mem_regions(iommu);
4001 continue;
4002 }
4003
Fenghua Yuf59c7b62009-03-27 14:22:42 -07004004 iommu_flush_write_buffer(iommu);
4005
4006 iommu_set_root_entry(iommu);
4007
4008 iommu->flush.flush_context(iommu, 0, 0, 0,
David Woodhouse1f0ef2a2009-05-10 19:58:49 +01004009 DMA_CCMD_GLOBAL_INVL);
Jiang Liu2a41cce2014-07-11 14:19:33 +08004010 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4011 iommu_enable_translation(iommu);
David Woodhouseb94996c2009-09-19 15:28:12 -07004012 iommu_disable_protect_mem_regions(iommu);
Fenghua Yuf59c7b62009-03-27 14:22:42 -07004013 }
4014
4015 return 0;
4016}
4017
4018static void iommu_flush_all(void)
4019{
4020 struct dmar_drhd_unit *drhd;
4021 struct intel_iommu *iommu;
4022
4023 for_each_active_iommu(iommu, drhd) {
4024 iommu->flush.flush_context(iommu, 0, 0, 0,
David Woodhouse1f0ef2a2009-05-10 19:58:49 +01004025 DMA_CCMD_GLOBAL_INVL);
Fenghua Yuf59c7b62009-03-27 14:22:42 -07004026 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
David Woodhouse1f0ef2a2009-05-10 19:58:49 +01004027 DMA_TLB_GLOBAL_FLUSH);
Fenghua Yuf59c7b62009-03-27 14:22:42 -07004028 }
4029}
4030
Rafael J. Wysocki134fac32011-03-23 22:16:14 +01004031static int iommu_suspend(void)
Fenghua Yuf59c7b62009-03-27 14:22:42 -07004032{
4033 struct dmar_drhd_unit *drhd;
4034 struct intel_iommu *iommu = NULL;
4035 unsigned long flag;
4036
4037 for_each_active_iommu(iommu, drhd) {
Kees Cook6396bb22018-06-12 14:03:40 -07004038 iommu->iommu_state = kcalloc(MAX_SR_DMAR_REGS, sizeof(u32),
Fenghua Yuf59c7b62009-03-27 14:22:42 -07004039 GFP_ATOMIC);
4040 if (!iommu->iommu_state)
4041 goto nomem;
4042 }
4043
4044 iommu_flush_all();
4045
4046 for_each_active_iommu(iommu, drhd) {
4047 iommu_disable_translation(iommu);
4048
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02004049 raw_spin_lock_irqsave(&iommu->register_lock, flag);
Fenghua Yuf59c7b62009-03-27 14:22:42 -07004050
4051 iommu->iommu_state[SR_DMAR_FECTL_REG] =
4052 readl(iommu->reg + DMAR_FECTL_REG);
4053 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
4054 readl(iommu->reg + DMAR_FEDATA_REG);
4055 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
4056 readl(iommu->reg + DMAR_FEADDR_REG);
4057 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
4058 readl(iommu->reg + DMAR_FEUADDR_REG);
4059
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02004060 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
Fenghua Yuf59c7b62009-03-27 14:22:42 -07004061 }
4062 return 0;
4063
4064nomem:
4065 for_each_active_iommu(iommu, drhd)
4066 kfree(iommu->iommu_state);
4067
4068 return -ENOMEM;
4069}
4070
Rafael J. Wysocki134fac32011-03-23 22:16:14 +01004071static void iommu_resume(void)
Fenghua Yuf59c7b62009-03-27 14:22:42 -07004072{
4073 struct dmar_drhd_unit *drhd;
4074 struct intel_iommu *iommu = NULL;
4075 unsigned long flag;
4076
4077 if (init_iommu_hw()) {
Joseph Cihulab7792602011-05-03 00:08:37 -07004078 if (force_on)
4079 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
4080 else
4081 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
Rafael J. Wysocki134fac32011-03-23 22:16:14 +01004082 return;
Fenghua Yuf59c7b62009-03-27 14:22:42 -07004083 }
4084
4085 for_each_active_iommu(iommu, drhd) {
4086
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02004087 raw_spin_lock_irqsave(&iommu->register_lock, flag);
Fenghua Yuf59c7b62009-03-27 14:22:42 -07004088
4089 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
4090 iommu->reg + DMAR_FECTL_REG);
4091 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
4092 iommu->reg + DMAR_FEDATA_REG);
4093 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
4094 iommu->reg + DMAR_FEADDR_REG);
4095 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
4096 iommu->reg + DMAR_FEUADDR_REG);
4097
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02004098 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
Fenghua Yuf59c7b62009-03-27 14:22:42 -07004099 }
4100
4101 for_each_active_iommu(iommu, drhd)
4102 kfree(iommu->iommu_state);
Fenghua Yuf59c7b62009-03-27 14:22:42 -07004103}
4104
Rafael J. Wysocki134fac32011-03-23 22:16:14 +01004105static struct syscore_ops iommu_syscore_ops = {
Fenghua Yuf59c7b62009-03-27 14:22:42 -07004106 .resume = iommu_resume,
4107 .suspend = iommu_suspend,
4108};
4109
Rafael J. Wysocki134fac32011-03-23 22:16:14 +01004110static void __init init_iommu_pm_ops(void)
Fenghua Yuf59c7b62009-03-27 14:22:42 -07004111{
Rafael J. Wysocki134fac32011-03-23 22:16:14 +01004112 register_syscore_ops(&iommu_syscore_ops);
Fenghua Yuf59c7b62009-03-27 14:22:42 -07004113}
4114
4115#else
Rafael J. Wysocki99592ba2011-06-07 21:32:31 +02004116static inline void init_iommu_pm_ops(void) {}
Fenghua Yuf59c7b62009-03-27 14:22:42 -07004117#endif /* CONFIG_PM */
4118
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004119
Jiang Liuc2a0b532014-11-09 22:47:56 +08004120int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004121{
4122 struct acpi_dmar_reserved_memory *rmrr;
Eric Auger0659b8d2017-01-19 20:57:53 +00004123 int prot = DMA_PTE_READ|DMA_PTE_WRITE;
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004124 struct dmar_rmrr_unit *rmrru;
Eric Auger0659b8d2017-01-19 20:57:53 +00004125 size_t length;
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004126
4127 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
4128 if (!rmrru)
Eric Auger0659b8d2017-01-19 20:57:53 +00004129 goto out;
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004130
4131 rmrru->hdr = header;
4132 rmrr = (struct acpi_dmar_reserved_memory *)header;
4133 rmrru->base_address = rmrr->base_address;
4134 rmrru->end_address = rmrr->end_address;
Eric Auger0659b8d2017-01-19 20:57:53 +00004135
4136 length = rmrr->end_address - rmrr->base_address + 1;
4137 rmrru->resv = iommu_alloc_resv_region(rmrr->base_address, length, prot,
4138 IOMMU_RESV_DIRECT);
4139 if (!rmrru->resv)
4140 goto free_rmrru;
4141
Jiang Liu2e455282014-02-19 14:07:36 +08004142 rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
4143 ((void *)rmrr) + rmrr->header.length,
4144 &rmrru->devices_cnt);
Eric Auger0659b8d2017-01-19 20:57:53 +00004145 if (rmrru->devices_cnt && rmrru->devices == NULL)
4146 goto free_all;
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004147
Jiang Liu2e455282014-02-19 14:07:36 +08004148 list_add(&rmrru->list, &dmar_rmrr_units);
4149
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004150 return 0;
Eric Auger0659b8d2017-01-19 20:57:53 +00004151free_all:
4152 kfree(rmrru->resv);
4153free_rmrru:
4154 kfree(rmrru);
4155out:
4156 return -ENOMEM;
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004157}
4158
Jiang Liu6b197242014-11-09 22:47:58 +08004159static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
4160{
4161 struct dmar_atsr_unit *atsru;
4162 struct acpi_dmar_atsr *tmp;
4163
4164 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4165 tmp = (struct acpi_dmar_atsr *)atsru->hdr;
4166 if (atsr->segment != tmp->segment)
4167 continue;
4168 if (atsr->header.length != tmp->header.length)
4169 continue;
4170 if (memcmp(atsr, tmp, atsr->header.length) == 0)
4171 return atsru;
4172 }
4173
4174 return NULL;
4175}
4176
4177int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004178{
4179 struct acpi_dmar_atsr *atsr;
4180 struct dmar_atsr_unit *atsru;
4181
Thomas Gleixnerb608fe32017-05-16 20:42:41 +02004182 if (system_state >= SYSTEM_RUNNING && !intel_iommu_enabled)
Jiang Liu6b197242014-11-09 22:47:58 +08004183 return 0;
4184
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004185 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
Jiang Liu6b197242014-11-09 22:47:58 +08004186 atsru = dmar_find_atsr(atsr);
4187 if (atsru)
4188 return 0;
4189
4190 atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004191 if (!atsru)
4192 return -ENOMEM;
4193
Jiang Liu6b197242014-11-09 22:47:58 +08004194 /*
4195 * If memory is allocated from slab by ACPI _DSM method, we need to
4196 * copy the memory content because the memory buffer will be freed
4197 * on return.
4198 */
4199 atsru->hdr = (void *)(atsru + 1);
4200 memcpy(atsru->hdr, hdr, hdr->length);
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004201 atsru->include_all = atsr->flags & 0x1;
Jiang Liu2e455282014-02-19 14:07:36 +08004202 if (!atsru->include_all) {
4203 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
4204 (void *)atsr + atsr->header.length,
4205 &atsru->devices_cnt);
4206 if (atsru->devices_cnt && atsru->devices == NULL) {
4207 kfree(atsru);
4208 return -ENOMEM;
4209 }
4210 }
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004211
Jiang Liu0e242612014-02-19 14:07:34 +08004212 list_add_rcu(&atsru->list, &dmar_atsr_units);
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004213
4214 return 0;
4215}
4216
Jiang Liu9bdc5312014-01-06 14:18:27 +08004217static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
4218{
4219 dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
4220 kfree(atsru);
4221}
4222
Jiang Liu6b197242014-11-09 22:47:58 +08004223int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4224{
4225 struct acpi_dmar_atsr *atsr;
4226 struct dmar_atsr_unit *atsru;
4227
4228 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4229 atsru = dmar_find_atsr(atsr);
4230 if (atsru) {
4231 list_del_rcu(&atsru->list);
4232 synchronize_rcu();
4233 intel_iommu_free_atsr(atsru);
4234 }
4235
4236 return 0;
4237}
4238
4239int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4240{
4241 int i;
4242 struct device *dev;
4243 struct acpi_dmar_atsr *atsr;
4244 struct dmar_atsr_unit *atsru;
4245
4246 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4247 atsru = dmar_find_atsr(atsr);
4248 if (!atsru)
4249 return 0;
4250
Linus Torvalds194dc872016-07-27 20:03:31 -07004251 if (!atsru->include_all && atsru->devices && atsru->devices_cnt) {
Jiang Liu6b197242014-11-09 22:47:58 +08004252 for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
4253 i, dev)
4254 return -EBUSY;
Linus Torvalds194dc872016-07-27 20:03:31 -07004255 }
Jiang Liu6b197242014-11-09 22:47:58 +08004256
4257 return 0;
4258}
4259
Jiang Liuffebeb42014-11-09 22:48:02 +08004260static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
4261{
4262 int sp, ret = 0;
4263 struct intel_iommu *iommu = dmaru->iommu;
4264
4265 if (g_iommus[iommu->seq_id])
4266 return 0;
4267
4268 if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02004269 pr_warn("%s: Doesn't support hardware pass through.\n",
Jiang Liuffebeb42014-11-09 22:48:02 +08004270 iommu->name);
4271 return -ENXIO;
4272 }
4273 if (!ecap_sc_support(iommu->ecap) &&
4274 domain_update_iommu_snooping(iommu)) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02004275 pr_warn("%s: Doesn't support snooping.\n",
Jiang Liuffebeb42014-11-09 22:48:02 +08004276 iommu->name);
4277 return -ENXIO;
4278 }
4279 sp = domain_update_iommu_superpage(iommu) - 1;
4280 if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02004281 pr_warn("%s: Doesn't support large page.\n",
Jiang Liuffebeb42014-11-09 22:48:02 +08004282 iommu->name);
4283 return -ENXIO;
4284 }
4285
4286 /*
4287 * Disable translation if already enabled prior to OS handover.
4288 */
4289 if (iommu->gcmd & DMA_GCMD_TE)
4290 iommu_disable_translation(iommu);
4291
4292 g_iommus[iommu->seq_id] = iommu;
4293 ret = iommu_init_domains(iommu);
4294 if (ret == 0)
4295 ret = iommu_alloc_root_entry(iommu);
4296 if (ret)
4297 goto out;
4298
David Woodhouse8a94ade2015-03-24 14:54:56 +00004299#ifdef CONFIG_INTEL_IOMMU_SVM
Lu Baolu765b6a92018-12-10 09:58:55 +08004300 if (pasid_supported(iommu))
Lu Baolud9737952018-07-14 15:47:02 +08004301 intel_svm_init(iommu);
David Woodhouse8a94ade2015-03-24 14:54:56 +00004302#endif
4303
Jiang Liuffebeb42014-11-09 22:48:02 +08004304 if (dmaru->ignored) {
4305 /*
4306 * we always have to disable PMRs or DMA may fail on this device
4307 */
4308 if (force_on)
4309 iommu_disable_protect_mem_regions(iommu);
4310 return 0;
4311 }
4312
4313 intel_iommu_init_qi(iommu);
4314 iommu_flush_write_buffer(iommu);
David Woodhousea222a7f2015-10-07 23:35:18 +01004315
4316#ifdef CONFIG_INTEL_IOMMU_SVM
Lu Baolu765b6a92018-12-10 09:58:55 +08004317 if (pasid_supported(iommu) && ecap_prs(iommu->ecap)) {
David Woodhousea222a7f2015-10-07 23:35:18 +01004318 ret = intel_svm_enable_prq(iommu);
4319 if (ret)
4320 goto disable_iommu;
4321 }
4322#endif
Jiang Liuffebeb42014-11-09 22:48:02 +08004323 ret = dmar_set_interrupt(iommu);
4324 if (ret)
4325 goto disable_iommu;
4326
4327 iommu_set_root_entry(iommu);
4328 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
4329 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4330 iommu_enable_translation(iommu);
4331
Jiang Liuffebeb42014-11-09 22:48:02 +08004332 iommu_disable_protect_mem_regions(iommu);
4333 return 0;
4334
4335disable_iommu:
4336 disable_dmar_iommu(iommu);
4337out:
4338 free_dmar_iommu(iommu);
4339 return ret;
4340}
4341
Jiang Liu6b197242014-11-09 22:47:58 +08004342int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
4343{
Jiang Liuffebeb42014-11-09 22:48:02 +08004344 int ret = 0;
4345 struct intel_iommu *iommu = dmaru->iommu;
4346
4347 if (!intel_iommu_enabled)
4348 return 0;
4349 if (iommu == NULL)
4350 return -EINVAL;
4351
4352 if (insert) {
4353 ret = intel_iommu_add(dmaru);
4354 } else {
4355 disable_dmar_iommu(iommu);
4356 free_dmar_iommu(iommu);
4357 }
4358
4359 return ret;
Jiang Liu6b197242014-11-09 22:47:58 +08004360}
4361
Jiang Liu9bdc5312014-01-06 14:18:27 +08004362static void intel_iommu_free_dmars(void)
4363{
4364 struct dmar_rmrr_unit *rmrru, *rmrr_n;
4365 struct dmar_atsr_unit *atsru, *atsr_n;
4366
4367 list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
4368 list_del(&rmrru->list);
4369 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
Eric Auger0659b8d2017-01-19 20:57:53 +00004370 kfree(rmrru->resv);
Jiang Liu9bdc5312014-01-06 14:18:27 +08004371 kfree(rmrru);
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004372 }
4373
Jiang Liu9bdc5312014-01-06 14:18:27 +08004374 list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
4375 list_del(&atsru->list);
4376 intel_iommu_free_atsr(atsru);
4377 }
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004378}
4379
4380int dmar_find_matched_atsr_unit(struct pci_dev *dev)
4381{
Jiang Liub683b232014-02-19 14:07:32 +08004382 int i, ret = 1;
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004383 struct pci_bus *bus;
David Woodhouse832bd852014-03-07 15:08:36 +00004384 struct pci_dev *bridge = NULL;
4385 struct device *tmp;
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004386 struct acpi_dmar_atsr *atsr;
4387 struct dmar_atsr_unit *atsru;
4388
4389 dev = pci_physfn(dev);
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004390 for (bus = dev->bus; bus; bus = bus->parent) {
Jiang Liub5f82dd2014-02-19 14:07:31 +08004391 bridge = bus->self;
David Woodhoused14053b32015-10-15 09:28:06 +01004392 /* If it's an integrated device, allow ATS */
4393 if (!bridge)
4394 return 1;
4395 /* Connected via non-PCIe: no ATS */
4396 if (!pci_is_pcie(bridge) ||
Yijing Wang62f87c02012-07-24 17:20:03 +08004397 pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004398 return 0;
David Woodhoused14053b32015-10-15 09:28:06 +01004399 /* If we found the root port, look it up in the ATSR */
Jiang Liub5f82dd2014-02-19 14:07:31 +08004400 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004401 break;
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004402 }
4403
Jiang Liu0e242612014-02-19 14:07:34 +08004404 rcu_read_lock();
Jiang Liub5f82dd2014-02-19 14:07:31 +08004405 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4406 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4407 if (atsr->segment != pci_domain_nr(dev->bus))
4408 continue;
4409
Jiang Liub683b232014-02-19 14:07:32 +08004410 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
David Woodhouse832bd852014-03-07 15:08:36 +00004411 if (tmp == &bridge->dev)
Jiang Liub683b232014-02-19 14:07:32 +08004412 goto out;
Jiang Liub5f82dd2014-02-19 14:07:31 +08004413
4414 if (atsru->include_all)
Jiang Liub683b232014-02-19 14:07:32 +08004415 goto out;
Jiang Liub5f82dd2014-02-19 14:07:31 +08004416 }
Jiang Liub683b232014-02-19 14:07:32 +08004417 ret = 0;
4418out:
Jiang Liu0e242612014-02-19 14:07:34 +08004419 rcu_read_unlock();
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004420
Jiang Liub683b232014-02-19 14:07:32 +08004421 return ret;
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004422}
4423
Jiang Liu59ce0512014-02-19 14:07:35 +08004424int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
4425{
4426 int ret = 0;
4427 struct dmar_rmrr_unit *rmrru;
4428 struct dmar_atsr_unit *atsru;
4429 struct acpi_dmar_atsr *atsr;
4430 struct acpi_dmar_reserved_memory *rmrr;
4431
Thomas Gleixnerb608fe32017-05-16 20:42:41 +02004432 if (!intel_iommu_enabled && system_state >= SYSTEM_RUNNING)
Jiang Liu59ce0512014-02-19 14:07:35 +08004433 return 0;
4434
4435 list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
4436 rmrr = container_of(rmrru->hdr,
4437 struct acpi_dmar_reserved_memory, header);
4438 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4439 ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
4440 ((void *)rmrr) + rmrr->header.length,
4441 rmrr->segment, rmrru->devices,
4442 rmrru->devices_cnt);
Jiang Liu27e24952014-06-20 15:08:06 +08004443 if(ret < 0)
Jiang Liu59ce0512014-02-19 14:07:35 +08004444 return ret;
Joerg Roedele6a8c9b2016-02-29 23:49:47 +01004445 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
Jiang Liu27e24952014-06-20 15:08:06 +08004446 dmar_remove_dev_scope(info, rmrr->segment,
4447 rmrru->devices, rmrru->devices_cnt);
Jiang Liu59ce0512014-02-19 14:07:35 +08004448 }
4449 }
4450
4451 list_for_each_entry(atsru, &dmar_atsr_units, list) {
4452 if (atsru->include_all)
4453 continue;
4454
4455 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4456 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4457 ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
4458 (void *)atsr + atsr->header.length,
4459 atsr->segment, atsru->devices,
4460 atsru->devices_cnt);
4461 if (ret > 0)
4462 break;
4463 else if(ret < 0)
4464 return ret;
Joerg Roedele6a8c9b2016-02-29 23:49:47 +01004465 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
Jiang Liu59ce0512014-02-19 14:07:35 +08004466 if (dmar_remove_dev_scope(info, atsr->segment,
4467 atsru->devices, atsru->devices_cnt))
4468 break;
4469 }
4470 }
4471
4472 return 0;
4473}
4474
Fenghua Yu99dcade2009-11-11 07:23:06 -08004475/*
4476 * Here we only respond to action of unbound device from driver.
4477 *
4478 * Added device is not attached to its DMAR domain here yet. That will happen
4479 * when mapping the device to iova.
4480 */
4481static int device_notifier(struct notifier_block *nb,
4482 unsigned long action, void *data)
4483{
4484 struct device *dev = data;
Fenghua Yu99dcade2009-11-11 07:23:06 -08004485 struct dmar_domain *domain;
4486
David Woodhouse3d891942014-03-06 15:59:26 +00004487 if (iommu_dummy(dev))
David Woodhouse44cd6132009-12-02 10:18:30 +00004488 return 0;
4489
Joerg Roedel1196c2f2014-09-30 13:02:03 +02004490 if (action != BUS_NOTIFY_REMOVED_DEVICE)
Jiang Liu7e7dfab2014-02-19 14:07:23 +08004491 return 0;
4492
David Woodhouse1525a292014-03-06 16:19:30 +00004493 domain = find_domain(dev);
Fenghua Yu99dcade2009-11-11 07:23:06 -08004494 if (!domain)
4495 return 0;
4496
Joerg Roedele6de0f82015-07-22 16:30:36 +02004497 dmar_remove_one_dev_info(domain, dev);
Jiang Liuab8dfe22014-07-11 14:19:27 +08004498 if (!domain_type_is_vm_or_si(domain) && list_empty(&domain->devices))
Jiang Liu7e7dfab2014-02-19 14:07:23 +08004499 domain_exit(domain);
Alex Williamsona97590e2011-03-04 14:52:16 -07004500
Fenghua Yu99dcade2009-11-11 07:23:06 -08004501 return 0;
4502}
4503
4504static struct notifier_block device_nb = {
4505 .notifier_call = device_notifier,
4506};
4507
Jiang Liu75f05562014-02-19 14:07:37 +08004508static int intel_iommu_memory_notifier(struct notifier_block *nb,
4509 unsigned long val, void *v)
4510{
4511 struct memory_notify *mhp = v;
4512 unsigned long long start, end;
4513 unsigned long start_vpfn, last_vpfn;
4514
4515 switch (val) {
4516 case MEM_GOING_ONLINE:
4517 start = mhp->start_pfn << PAGE_SHIFT;
4518 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
4519 if (iommu_domain_identity_map(si_domain, start, end)) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02004520 pr_warn("Failed to build identity map for [%llx-%llx]\n",
Jiang Liu75f05562014-02-19 14:07:37 +08004521 start, end);
4522 return NOTIFY_BAD;
4523 }
4524 break;
4525
4526 case MEM_OFFLINE:
4527 case MEM_CANCEL_ONLINE:
4528 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
4529 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
4530 while (start_vpfn <= last_vpfn) {
4531 struct iova *iova;
4532 struct dmar_drhd_unit *drhd;
4533 struct intel_iommu *iommu;
David Woodhouseea8ea462014-03-05 17:09:32 +00004534 struct page *freelist;
Jiang Liu75f05562014-02-19 14:07:37 +08004535
4536 iova = find_iova(&si_domain->iovad, start_vpfn);
4537 if (iova == NULL) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02004538 pr_debug("Failed get IOVA for PFN %lx\n",
Jiang Liu75f05562014-02-19 14:07:37 +08004539 start_vpfn);
4540 break;
4541 }
4542
4543 iova = split_and_remove_iova(&si_domain->iovad, iova,
4544 start_vpfn, last_vpfn);
4545 if (iova == NULL) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02004546 pr_warn("Failed to split IOVA PFN [%lx-%lx]\n",
Jiang Liu75f05562014-02-19 14:07:37 +08004547 start_vpfn, last_vpfn);
4548 return NOTIFY_BAD;
4549 }
4550
David Woodhouseea8ea462014-03-05 17:09:32 +00004551 freelist = domain_unmap(si_domain, iova->pfn_lo,
4552 iova->pfn_hi);
4553
Jiang Liu75f05562014-02-19 14:07:37 +08004554 rcu_read_lock();
4555 for_each_active_iommu(iommu, drhd)
Joerg Roedela1ddcbe2015-07-21 15:20:32 +02004556 iommu_flush_iotlb_psi(iommu, si_domain,
Jiang Liua156ef92014-07-11 14:19:36 +08004557 iova->pfn_lo, iova_size(iova),
David Woodhouseea8ea462014-03-05 17:09:32 +00004558 !freelist, 0);
Jiang Liu75f05562014-02-19 14:07:37 +08004559 rcu_read_unlock();
David Woodhouseea8ea462014-03-05 17:09:32 +00004560 dma_free_pagelist(freelist);
Jiang Liu75f05562014-02-19 14:07:37 +08004561
4562 start_vpfn = iova->pfn_hi + 1;
4563 free_iova_mem(iova);
4564 }
4565 break;
4566 }
4567
4568 return NOTIFY_OK;
4569}
4570
4571static struct notifier_block intel_iommu_memory_nb = {
4572 .notifier_call = intel_iommu_memory_notifier,
4573 .priority = 0
4574};
4575
Omer Peleg22e2f9f2016-04-20 11:34:11 +03004576static void free_all_cpu_cached_iovas(unsigned int cpu)
4577{
4578 int i;
4579
4580 for (i = 0; i < g_num_of_iommus; i++) {
4581 struct intel_iommu *iommu = g_iommus[i];
4582 struct dmar_domain *domain;
Aaron Campbell0caa7612016-07-02 21:23:24 -03004583 int did;
Omer Peleg22e2f9f2016-04-20 11:34:11 +03004584
4585 if (!iommu)
4586 continue;
4587
Jan Niehusmann3bd4f912016-06-06 14:20:11 +02004588 for (did = 0; did < cap_ndoms(iommu->cap); did++) {
Aaron Campbell0caa7612016-07-02 21:23:24 -03004589 domain = get_iommu_domain(iommu, (u16)did);
Omer Peleg22e2f9f2016-04-20 11:34:11 +03004590
4591 if (!domain)
4592 continue;
4593 free_cpu_cached_iovas(cpu, &domain->iovad);
4594 }
4595 }
4596}
4597
Anna-Maria Gleixner21647612016-11-27 00:13:41 +01004598static int intel_iommu_cpu_dead(unsigned int cpu)
Omer Pelegaa473242016-04-20 11:33:02 +03004599{
Anna-Maria Gleixner21647612016-11-27 00:13:41 +01004600 free_all_cpu_cached_iovas(cpu);
Anna-Maria Gleixner21647612016-11-27 00:13:41 +01004601 return 0;
Omer Pelegaa473242016-04-20 11:33:02 +03004602}
4603
Joerg Roedel161b28a2017-03-28 17:04:52 +02004604static void intel_disable_iommus(void)
4605{
4606 struct intel_iommu *iommu = NULL;
4607 struct dmar_drhd_unit *drhd;
4608
4609 for_each_iommu(iommu, drhd)
4610 iommu_disable_translation(iommu);
4611}
4612
Joerg Roedela7fdb6e2017-02-28 13:57:18 +01004613static inline struct intel_iommu *dev_to_intel_iommu(struct device *dev)
4614{
Joerg Roedel2926a2aa2017-08-14 17:19:26 +02004615 struct iommu_device *iommu_dev = dev_to_iommu_device(dev);
4616
4617 return container_of(iommu_dev, struct intel_iommu, iommu);
Joerg Roedela7fdb6e2017-02-28 13:57:18 +01004618}
4619
Alex Williamsona5459cf2014-06-12 16:12:31 -06004620static ssize_t intel_iommu_show_version(struct device *dev,
4621 struct device_attribute *attr,
4622 char *buf)
4623{
Joerg Roedela7fdb6e2017-02-28 13:57:18 +01004624 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
Alex Williamsona5459cf2014-06-12 16:12:31 -06004625 u32 ver = readl(iommu->reg + DMAR_VER_REG);
4626 return sprintf(buf, "%d:%d\n",
4627 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
4628}
4629static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
4630
4631static ssize_t intel_iommu_show_address(struct device *dev,
4632 struct device_attribute *attr,
4633 char *buf)
4634{
Joerg Roedela7fdb6e2017-02-28 13:57:18 +01004635 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
Alex Williamsona5459cf2014-06-12 16:12:31 -06004636 return sprintf(buf, "%llx\n", iommu->reg_phys);
4637}
4638static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
4639
4640static ssize_t intel_iommu_show_cap(struct device *dev,
4641 struct device_attribute *attr,
4642 char *buf)
4643{
Joerg Roedela7fdb6e2017-02-28 13:57:18 +01004644 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
Alex Williamsona5459cf2014-06-12 16:12:31 -06004645 return sprintf(buf, "%llx\n", iommu->cap);
4646}
4647static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
4648
4649static ssize_t intel_iommu_show_ecap(struct device *dev,
4650 struct device_attribute *attr,
4651 char *buf)
4652{
Joerg Roedela7fdb6e2017-02-28 13:57:18 +01004653 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
Alex Williamsona5459cf2014-06-12 16:12:31 -06004654 return sprintf(buf, "%llx\n", iommu->ecap);
4655}
4656static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
4657
Alex Williamson2238c082015-07-14 15:24:53 -06004658static ssize_t intel_iommu_show_ndoms(struct device *dev,
4659 struct device_attribute *attr,
4660 char *buf)
4661{
Joerg Roedela7fdb6e2017-02-28 13:57:18 +01004662 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
Alex Williamson2238c082015-07-14 15:24:53 -06004663 return sprintf(buf, "%ld\n", cap_ndoms(iommu->cap));
4664}
4665static DEVICE_ATTR(domains_supported, S_IRUGO, intel_iommu_show_ndoms, NULL);
4666
4667static ssize_t intel_iommu_show_ndoms_used(struct device *dev,
4668 struct device_attribute *attr,
4669 char *buf)
4670{
Joerg Roedela7fdb6e2017-02-28 13:57:18 +01004671 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
Alex Williamson2238c082015-07-14 15:24:53 -06004672 return sprintf(buf, "%d\n", bitmap_weight(iommu->domain_ids,
4673 cap_ndoms(iommu->cap)));
4674}
4675static DEVICE_ATTR(domains_used, S_IRUGO, intel_iommu_show_ndoms_used, NULL);
4676
Alex Williamsona5459cf2014-06-12 16:12:31 -06004677static struct attribute *intel_iommu_attrs[] = {
4678 &dev_attr_version.attr,
4679 &dev_attr_address.attr,
4680 &dev_attr_cap.attr,
4681 &dev_attr_ecap.attr,
Alex Williamson2238c082015-07-14 15:24:53 -06004682 &dev_attr_domains_supported.attr,
4683 &dev_attr_domains_used.attr,
Alex Williamsona5459cf2014-06-12 16:12:31 -06004684 NULL,
4685};
4686
4687static struct attribute_group intel_iommu_group = {
4688 .name = "intel-iommu",
4689 .attrs = intel_iommu_attrs,
4690};
4691
4692const struct attribute_group *intel_iommu_groups[] = {
4693 &intel_iommu_group,
4694 NULL,
4695};
4696
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07004697int __init intel_iommu_init(void)
4698{
Jiang Liu9bdc5312014-01-06 14:18:27 +08004699 int ret = -ENODEV;
Takao Indoh3a93c842013-04-23 17:35:03 +09004700 struct dmar_drhd_unit *drhd;
Jiang Liu7c919772014-01-06 14:18:18 +08004701 struct intel_iommu *iommu;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07004702
Joseph Cihulaa59b50e2009-06-30 19:31:10 -07004703 /* VT-d is required for a TXT/tboot launch, so enforce that */
4704 force_on = tboot_force_iommu();
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07004705
Jiang Liu3a5670e2014-02-19 14:07:33 +08004706 if (iommu_init_mempool()) {
4707 if (force_on)
4708 panic("tboot: Failed to initialize iommu memory\n");
4709 return -ENOMEM;
4710 }
4711
4712 down_write(&dmar_global_lock);
Joseph Cihulaa59b50e2009-06-30 19:31:10 -07004713 if (dmar_table_init()) {
4714 if (force_on)
4715 panic("tboot: Failed to initialize DMAR table\n");
Jiang Liu9bdc5312014-01-06 14:18:27 +08004716 goto out_free_dmar;
Joseph Cihulaa59b50e2009-06-30 19:31:10 -07004717 }
4718
Suresh Siddhac2c72862011-08-23 17:05:19 -07004719 if (dmar_dev_scope_init() < 0) {
Joseph Cihulaa59b50e2009-06-30 19:31:10 -07004720 if (force_on)
4721 panic("tboot: Failed to initialize DMAR device scope\n");
Jiang Liu9bdc5312014-01-06 14:18:27 +08004722 goto out_free_dmar;
Joseph Cihulaa59b50e2009-06-30 19:31:10 -07004723 }
Suresh Siddha1886e8a2008-07-10 11:16:37 -07004724
Joerg Roedelec154bf2017-10-06 15:00:53 +02004725 up_write(&dmar_global_lock);
4726
4727 /*
4728 * The bus notifier takes the dmar_global_lock, so lockdep will
4729 * complain later when we register it under the lock.
4730 */
4731 dmar_register_bus_notifier();
4732
4733 down_write(&dmar_global_lock);
4734
Joerg Roedel161b28a2017-03-28 17:04:52 +02004735 if (no_iommu || dmar_disabled) {
4736 /*
Shaohua Libfd20f12017-04-26 09:18:35 -07004737 * We exit the function here to ensure IOMMU's remapping and
4738 * mempool aren't setup, which means that the IOMMU's PMRs
4739 * won't be disabled via the call to init_dmars(). So disable
4740 * it explicitly here. The PMRs were setup by tboot prior to
4741 * calling SENTER, but the kernel is expected to reset/tear
4742 * down the PMRs.
4743 */
4744 if (intel_iommu_tboot_noforce) {
4745 for_each_iommu(iommu, drhd)
4746 iommu_disable_protect_mem_regions(iommu);
4747 }
4748
4749 /*
Joerg Roedel161b28a2017-03-28 17:04:52 +02004750 * Make sure the IOMMUs are switched off, even when we
4751 * boot into a kexec kernel and the previous kernel left
4752 * them enabled
4753 */
4754 intel_disable_iommus();
Jiang Liu9bdc5312014-01-06 14:18:27 +08004755 goto out_free_dmar;
Joerg Roedel161b28a2017-03-28 17:04:52 +02004756 }
Suresh Siddha2ae21012008-07-10 11:16:43 -07004757
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004758 if (list_empty(&dmar_rmrr_units))
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02004759 pr_info("No RMRR found\n");
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004760
4761 if (list_empty(&dmar_atsr_units))
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02004762 pr_info("No ATSR found\n");
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004763
Joseph Cihula51a63e62011-03-21 11:04:24 -07004764 if (dmar_init_reserved_ranges()) {
4765 if (force_on)
4766 panic("tboot: Failed to reserve iommu ranges\n");
Jiang Liu3a5670e2014-02-19 14:07:33 +08004767 goto out_free_reserved_range;
Joseph Cihula51a63e62011-03-21 11:04:24 -07004768 }
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07004769
4770 init_no_remapping_devices();
4771
Joseph Cihulab7792602011-05-03 00:08:37 -07004772 ret = init_dmars();
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07004773 if (ret) {
Joseph Cihulaa59b50e2009-06-30 19:31:10 -07004774 if (force_on)
4775 panic("tboot: Failed to initialize DMARs\n");
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02004776 pr_err("Initialization failed\n");
Jiang Liu9bdc5312014-01-06 14:18:27 +08004777 goto out_free_reserved_range;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07004778 }
Jiang Liu3a5670e2014-02-19 14:07:33 +08004779 up_write(&dmar_global_lock);
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02004780 pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07004781
Christoph Hellwig4fac8072017-12-24 13:57:08 +01004782#if defined(CONFIG_X86) && defined(CONFIG_SWIOTLB)
FUJITA Tomonori75f1cdf2009-11-10 19:46:20 +09004783 swiotlb = 0;
4784#endif
David Woodhouse19943b02009-08-04 16:19:20 +01004785 dma_ops = &intel_dma_ops;
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07004786
Rafael J. Wysocki134fac32011-03-23 22:16:14 +01004787 init_iommu_pm_ops();
Joerg Roedela8bcbb0d2008-12-03 15:14:02 +01004788
Joerg Roedel39ab9552017-02-01 16:56:46 +01004789 for_each_active_iommu(iommu, drhd) {
4790 iommu_device_sysfs_add(&iommu->iommu, NULL,
4791 intel_iommu_groups,
4792 "%s", iommu->name);
4793 iommu_device_set_ops(&iommu->iommu, &intel_iommu_ops);
4794 iommu_device_register(&iommu->iommu);
4795 }
Alex Williamsona5459cf2014-06-12 16:12:31 -06004796
Joerg Roedel4236d97d2011-09-06 17:56:07 +02004797 bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
Fenghua Yu99dcade2009-11-11 07:23:06 -08004798 bus_register_notifier(&pci_bus_type, &device_nb);
Jiang Liu75f05562014-02-19 14:07:37 +08004799 if (si_domain && !hw_pass_through)
4800 register_memory_notifier(&intel_iommu_memory_nb);
Anna-Maria Gleixner21647612016-11-27 00:13:41 +01004801 cpuhp_setup_state(CPUHP_IOMMU_INTEL_DEAD, "iommu/intel:dead", NULL,
4802 intel_iommu_cpu_dead);
Eugeni Dodonov8bc1f852011-11-23 16:42:14 -02004803 intel_iommu_enabled = 1;
Sohil Mehtaee2636b2018-09-11 17:11:38 -07004804 intel_iommu_debugfs_init();
Eugeni Dodonov8bc1f852011-11-23 16:42:14 -02004805
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07004806 return 0;
Jiang Liu9bdc5312014-01-06 14:18:27 +08004807
4808out_free_reserved_range:
4809 put_iova_domain(&reserved_iova_list);
Jiang Liu9bdc5312014-01-06 14:18:27 +08004810out_free_dmar:
4811 intel_iommu_free_dmars();
Jiang Liu3a5670e2014-02-19 14:07:33 +08004812 up_write(&dmar_global_lock);
4813 iommu_exit_mempool();
Jiang Liu9bdc5312014-01-06 14:18:27 +08004814 return ret;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07004815}
Keshavamurthy, Anil Se8204822007-10-21 16:41:55 -07004816
Joerg Roedel2452d9d2015-07-23 16:20:14 +02004817static int domain_context_clear_one_cb(struct pci_dev *pdev, u16 alias, void *opaque)
Alex Williamson579305f2014-07-03 09:51:43 -06004818{
4819 struct intel_iommu *iommu = opaque;
4820
Joerg Roedel2452d9d2015-07-23 16:20:14 +02004821 domain_context_clear_one(iommu, PCI_BUS_NUM(alias), alias & 0xff);
Alex Williamson579305f2014-07-03 09:51:43 -06004822 return 0;
4823}
4824
4825/*
4826 * NB - intel-iommu lacks any sort of reference counting for the users of
4827 * dependent devices. If multiple endpoints have intersecting dependent
4828 * devices, unbinding the driver from any one of them will possibly leave
4829 * the others unable to operate.
4830 */
Joerg Roedel2452d9d2015-07-23 16:20:14 +02004831static void domain_context_clear(struct intel_iommu *iommu, struct device *dev)
Han, Weidong3199aa62009-02-26 17:31:12 +08004832{
David Woodhouse0bcb3e22014-03-06 17:12:03 +00004833 if (!iommu || !dev || !dev_is_pci(dev))
Han, Weidong3199aa62009-02-26 17:31:12 +08004834 return;
4835
Joerg Roedel2452d9d2015-07-23 16:20:14 +02004836 pci_for_each_dma_alias(to_pci_dev(dev), &domain_context_clear_one_cb, iommu);
Han, Weidong3199aa62009-02-26 17:31:12 +08004837}
4838
Joerg Roedel127c7612015-07-23 17:44:46 +02004839static void __dmar_remove_one_dev_info(struct device_domain_info *info)
Weidong Hanc7151a82008-12-08 22:51:37 +08004840{
Weidong Hanc7151a82008-12-08 22:51:37 +08004841 struct intel_iommu *iommu;
4842 unsigned long flags;
Weidong Hanc7151a82008-12-08 22:51:37 +08004843
Joerg Roedel55d94042015-07-22 16:50:40 +02004844 assert_spin_locked(&device_domain_lock);
4845
Joerg Roedelb608ac32015-07-21 18:19:08 +02004846 if (WARN_ON(!info))
Weidong Hanc7151a82008-12-08 22:51:37 +08004847 return;
4848
Joerg Roedel127c7612015-07-23 17:44:46 +02004849 iommu = info->iommu;
4850
4851 if (info->dev) {
Lu Baoluef848b72018-12-10 09:59:01 +08004852 if (dev_is_pci(info->dev) && sm_supported(iommu))
4853 intel_pasid_tear_down_entry(iommu, info->dev,
4854 PASID_RID2PASID);
4855
Joerg Roedel127c7612015-07-23 17:44:46 +02004856 iommu_disable_dev_iotlb(info);
4857 domain_context_clear(iommu, info->dev);
Lu Baolua7fc93f2018-07-14 15:47:00 +08004858 intel_pasid_free_table(info->dev);
Joerg Roedel127c7612015-07-23 17:44:46 +02004859 }
4860
Joerg Roedelb608ac32015-07-21 18:19:08 +02004861 unlink_domain_info(info);
Roland Dreier3e7abe22011-07-20 06:22:21 -07004862
Joerg Roedeld160aca2015-07-22 11:52:53 +02004863 spin_lock_irqsave(&iommu->lock, flags);
Joerg Roedel127c7612015-07-23 17:44:46 +02004864 domain_detach_iommu(info->domain, iommu);
Joerg Roedeld160aca2015-07-22 11:52:53 +02004865 spin_unlock_irqrestore(&iommu->lock, flags);
Joerg Roedel127c7612015-07-23 17:44:46 +02004866
4867 free_devinfo_mem(info);
Weidong Hanc7151a82008-12-08 22:51:37 +08004868}
4869
Joerg Roedel55d94042015-07-22 16:50:40 +02004870static void dmar_remove_one_dev_info(struct dmar_domain *domain,
4871 struct device *dev)
4872{
Joerg Roedel127c7612015-07-23 17:44:46 +02004873 struct device_domain_info *info;
Joerg Roedel55d94042015-07-22 16:50:40 +02004874 unsigned long flags;
4875
Weidong Hanc7151a82008-12-08 22:51:37 +08004876 spin_lock_irqsave(&device_domain_lock, flags);
Joerg Roedel127c7612015-07-23 17:44:46 +02004877 info = dev->archdata.iommu;
4878 __dmar_remove_one_dev_info(info);
Weidong Han5e98c4b2008-12-08 23:03:27 +08004879 spin_unlock_irqrestore(&device_domain_lock, flags);
Weidong Han5e98c4b2008-12-08 23:03:27 +08004880}
4881
4882static int md_domain_init(struct dmar_domain *domain, int guest_width)
4883{
4884 int adjust_width;
4885
Zhen Leiaa3ac942017-09-21 16:52:45 +01004886 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN);
Weidong Han5e98c4b2008-12-08 23:03:27 +08004887 domain_reserve_special_ranges(domain);
4888
4889 /* calculate AGAW */
4890 domain->gaw = guest_width;
4891 adjust_width = guestwidth_to_adjustwidth(guest_width);
4892 domain->agaw = width_to_agaw(adjust_width);
4893
Weidong Han5e98c4b2008-12-08 23:03:27 +08004894 domain->iommu_coherency = 0;
Sheng Yangc5b15252009-08-06 13:31:56 +08004895 domain->iommu_snooping = 0;
Youquan Song6dd9a7c2011-05-25 19:13:49 +01004896 domain->iommu_superpage = 0;
Weidong Hanfe40f1e2008-12-08 23:10:23 +08004897 domain->max_addr = 0;
Weidong Han5e98c4b2008-12-08 23:03:27 +08004898
4899 /* always allocate the top pgd */
Suresh Siddha4c923d42009-10-02 11:01:24 -07004900 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
Weidong Han5e98c4b2008-12-08 23:03:27 +08004901 if (!domain->pgd)
4902 return -ENOMEM;
4903 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
4904 return 0;
4905}
4906
Joerg Roedel00a77de2015-03-26 13:43:08 +01004907static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
Kay, Allen M38717942008-09-09 18:37:29 +03004908{
Joerg Roedel5d450802008-12-03 14:52:32 +01004909 struct dmar_domain *dmar_domain;
Joerg Roedel00a77de2015-03-26 13:43:08 +01004910 struct iommu_domain *domain;
4911
4912 if (type != IOMMU_DOMAIN_UNMANAGED)
4913 return NULL;
Kay, Allen M38717942008-09-09 18:37:29 +03004914
Jiang Liuab8dfe22014-07-11 14:19:27 +08004915 dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
Joerg Roedel5d450802008-12-03 14:52:32 +01004916 if (!dmar_domain) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02004917 pr_err("Can't allocate dmar_domain\n");
Joerg Roedel00a77de2015-03-26 13:43:08 +01004918 return NULL;
Kay, Allen M38717942008-09-09 18:37:29 +03004919 }
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07004920 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02004921 pr_err("Domain initialization failed\n");
Jiang Liu92d03cc2014-02-19 14:07:28 +08004922 domain_exit(dmar_domain);
Joerg Roedel00a77de2015-03-26 13:43:08 +01004923 return NULL;
Kay, Allen M38717942008-09-09 18:37:29 +03004924 }
Allen Kay8140a952011-10-14 12:32:17 -07004925 domain_update_iommu_cap(dmar_domain);
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08004926
Joerg Roedel00a77de2015-03-26 13:43:08 +01004927 domain = &dmar_domain->domain;
Joerg Roedel8a0e7152012-01-26 19:40:54 +01004928 domain->geometry.aperture_start = 0;
4929 domain->geometry.aperture_end = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
4930 domain->geometry.force_aperture = true;
4931
Joerg Roedel00a77de2015-03-26 13:43:08 +01004932 return domain;
Kay, Allen M38717942008-09-09 18:37:29 +03004933}
Kay, Allen M38717942008-09-09 18:37:29 +03004934
Joerg Roedel00a77de2015-03-26 13:43:08 +01004935static void intel_iommu_domain_free(struct iommu_domain *domain)
Kay, Allen M38717942008-09-09 18:37:29 +03004936{
Joerg Roedel00a77de2015-03-26 13:43:08 +01004937 domain_exit(to_dmar_domain(domain));
Kay, Allen M38717942008-09-09 18:37:29 +03004938}
Kay, Allen M38717942008-09-09 18:37:29 +03004939
Joerg Roedel4c5478c2008-12-03 14:58:24 +01004940static int intel_iommu_attach_device(struct iommu_domain *domain,
4941 struct device *dev)
Kay, Allen M38717942008-09-09 18:37:29 +03004942{
Joerg Roedel00a77de2015-03-26 13:43:08 +01004943 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
Weidong Hanfe40f1e2008-12-08 23:10:23 +08004944 struct intel_iommu *iommu;
4945 int addr_width;
David Woodhouse156baca2014-03-09 14:00:57 -07004946 u8 bus, devfn;
Kay, Allen M38717942008-09-09 18:37:29 +03004947
Alex Williamsonc875d2c2014-07-03 09:57:02 -06004948 if (device_is_rmrr_locked(dev)) {
4949 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement. Contact your platform vendor.\n");
4950 return -EPERM;
4951 }
4952
David Woodhouse7207d8f2014-03-09 16:31:06 -07004953 /* normally dev is not mapped */
4954 if (unlikely(domain_context_mapped(dev))) {
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08004955 struct dmar_domain *old_domain;
4956
David Woodhouse1525a292014-03-06 16:19:30 +00004957 old_domain = find_domain(dev);
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08004958 if (old_domain) {
Joerg Roedeld160aca2015-07-22 11:52:53 +02004959 rcu_read_lock();
Joerg Roedelde7e8882015-07-22 11:58:07 +02004960 dmar_remove_one_dev_info(old_domain, dev);
Joerg Roedeld160aca2015-07-22 11:52:53 +02004961 rcu_read_unlock();
Joerg Roedel62c22162014-12-09 12:56:45 +01004962
4963 if (!domain_type_is_vm_or_si(old_domain) &&
4964 list_empty(&old_domain->devices))
4965 domain_exit(old_domain);
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08004966 }
4967 }
4968
David Woodhouse156baca2014-03-09 14:00:57 -07004969 iommu = device_to_iommu(dev, &bus, &devfn);
Weidong Hanfe40f1e2008-12-08 23:10:23 +08004970 if (!iommu)
4971 return -ENODEV;
4972
4973 /* check if this iommu agaw is sufficient for max mapped address */
4974 addr_width = agaw_to_width(iommu->agaw);
Tom Lyona99c47a2010-05-17 08:20:45 +01004975 if (addr_width > cap_mgaw(iommu->cap))
4976 addr_width = cap_mgaw(iommu->cap);
4977
4978 if (dmar_domain->max_addr > (1LL << addr_width)) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02004979 pr_err("%s: iommu width (%d) is not "
Weidong Hanfe40f1e2008-12-08 23:10:23 +08004980 "sufficient for the mapped address (%llx)\n",
Tom Lyona99c47a2010-05-17 08:20:45 +01004981 __func__, addr_width, dmar_domain->max_addr);
Weidong Hanfe40f1e2008-12-08 23:10:23 +08004982 return -EFAULT;
4983 }
Tom Lyona99c47a2010-05-17 08:20:45 +01004984 dmar_domain->gaw = addr_width;
4985
4986 /*
4987 * Knock out extra levels of page tables if necessary
4988 */
4989 while (iommu->agaw < dmar_domain->agaw) {
4990 struct dma_pte *pte;
4991
4992 pte = dmar_domain->pgd;
4993 if (dma_pte_present(pte)) {
Sheng Yang25cbff12010-06-12 19:21:42 +08004994 dmar_domain->pgd = (struct dma_pte *)
4995 phys_to_virt(dma_pte_addr(pte));
Jan Kiszka7a661012010-11-02 08:05:51 +01004996 free_pgtable_page(pte);
Tom Lyona99c47a2010-05-17 08:20:45 +01004997 }
4998 dmar_domain->agaw--;
4999 }
Weidong Hanfe40f1e2008-12-08 23:10:23 +08005000
Joerg Roedel28ccce02015-07-21 14:45:31 +02005001 return domain_add_dev_info(dmar_domain, dev);
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08005002}
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08005003
Joerg Roedel4c5478c2008-12-03 14:58:24 +01005004static void intel_iommu_detach_device(struct iommu_domain *domain,
5005 struct device *dev)
Kay, Allen M38717942008-09-09 18:37:29 +03005006{
Joerg Roedele6de0f82015-07-22 16:30:36 +02005007 dmar_remove_one_dev_info(to_dmar_domain(domain), dev);
Kay, Allen M38717942008-09-09 18:37:29 +03005008}
Kay, Allen M38717942008-09-09 18:37:29 +03005009
Joerg Roedelb146a1c9f2010-01-20 17:17:37 +01005010static int intel_iommu_map(struct iommu_domain *domain,
5011 unsigned long iova, phys_addr_t hpa,
Ohad Ben-Cohen50090652011-11-10 11:32:25 +02005012 size_t size, int iommu_prot)
Kay, Allen M38717942008-09-09 18:37:29 +03005013{
Joerg Roedel00a77de2015-03-26 13:43:08 +01005014 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
Weidong Hanfe40f1e2008-12-08 23:10:23 +08005015 u64 max_addr;
Joerg Roedeldde57a22008-12-03 15:04:09 +01005016 int prot = 0;
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08005017 int ret;
Weidong Hanfe40f1e2008-12-08 23:10:23 +08005018
Joerg Roedeldde57a22008-12-03 15:04:09 +01005019 if (iommu_prot & IOMMU_READ)
5020 prot |= DMA_PTE_READ;
5021 if (iommu_prot & IOMMU_WRITE)
5022 prot |= DMA_PTE_WRITE;
Sheng Yang9cf06692009-03-18 15:33:07 +08005023 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
5024 prot |= DMA_PTE_SNP;
Joerg Roedeldde57a22008-12-03 15:04:09 +01005025
David Woodhouse163cc522009-06-28 00:51:17 +01005026 max_addr = iova + size;
Joerg Roedeldde57a22008-12-03 15:04:09 +01005027 if (dmar_domain->max_addr < max_addr) {
Weidong Hanfe40f1e2008-12-08 23:10:23 +08005028 u64 end;
5029
5030 /* check if minimum agaw is sufficient for mapped address */
Tom Lyon8954da12010-05-17 08:19:52 +01005031 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
Weidong Hanfe40f1e2008-12-08 23:10:23 +08005032 if (end < max_addr) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02005033 pr_err("%s: iommu width (%d) is not "
Weidong Hanfe40f1e2008-12-08 23:10:23 +08005034 "sufficient for the mapped address (%llx)\n",
Tom Lyon8954da12010-05-17 08:19:52 +01005035 __func__, dmar_domain->gaw, max_addr);
Weidong Hanfe40f1e2008-12-08 23:10:23 +08005036 return -EFAULT;
5037 }
Joerg Roedeldde57a22008-12-03 15:04:09 +01005038 dmar_domain->max_addr = max_addr;
Weidong Hanfe40f1e2008-12-08 23:10:23 +08005039 }
David Woodhousead051222009-06-28 14:22:28 +01005040 /* Round up size to next multiple of PAGE_SIZE, if it and
5041 the low bits of hpa would take us onto the next page */
David Woodhouse88cb6a72009-06-28 15:03:06 +01005042 size = aligned_nrpages(hpa, size);
David Woodhousead051222009-06-28 14:22:28 +01005043 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
5044 hpa >> VTD_PAGE_SHIFT, size, prot);
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08005045 return ret;
Kay, Allen M38717942008-09-09 18:37:29 +03005046}
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08005047
Ohad Ben-Cohen50090652011-11-10 11:32:25 +02005048static size_t intel_iommu_unmap(struct iommu_domain *domain,
David Woodhouseea8ea462014-03-05 17:09:32 +00005049 unsigned long iova, size_t size)
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08005050{
Joerg Roedel00a77de2015-03-26 13:43:08 +01005051 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
David Woodhouseea8ea462014-03-05 17:09:32 +00005052 struct page *freelist = NULL;
David Woodhouseea8ea462014-03-05 17:09:32 +00005053 unsigned long start_pfn, last_pfn;
5054 unsigned int npages;
Joerg Roedel42e8c182015-07-21 15:50:02 +02005055 int iommu_id, level = 0;
Sheng Yang4b99d352009-07-08 11:52:52 +01005056
David Woodhouse5cf0a762014-03-19 16:07:49 +00005057 /* Cope with horrid API which requires us to unmap more than the
5058 size argument if it happens to be a large-page mapping. */
Joerg Roedeldc02e462015-08-13 11:15:13 +02005059 BUG_ON(!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level));
David Woodhouse5cf0a762014-03-19 16:07:49 +00005060
5061 if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
5062 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
5063
David Woodhouseea8ea462014-03-05 17:09:32 +00005064 start_pfn = iova >> VTD_PAGE_SHIFT;
5065 last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
5066
5067 freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
5068
5069 npages = last_pfn - start_pfn + 1;
5070
Shaokun Zhangf746a022018-03-22 18:18:06 +08005071 for_each_domain_iommu(iommu_id, dmar_domain)
Joerg Roedel42e8c182015-07-21 15:50:02 +02005072 iommu_flush_iotlb_psi(g_iommus[iommu_id], dmar_domain,
5073 start_pfn, npages, !freelist, 0);
David Woodhouseea8ea462014-03-05 17:09:32 +00005074
5075 dma_free_pagelist(freelist);
Weidong Hanfe40f1e2008-12-08 23:10:23 +08005076
David Woodhouse163cc522009-06-28 00:51:17 +01005077 if (dmar_domain->max_addr == iova + size)
5078 dmar_domain->max_addr = iova;
Joerg Roedelb146a1c9f2010-01-20 17:17:37 +01005079
David Woodhouse5cf0a762014-03-19 16:07:49 +00005080 return size;
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08005081}
Kay, Allen M38717942008-09-09 18:37:29 +03005082
Joerg Roedeld14d6572008-12-03 15:06:57 +01005083static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
Varun Sethibb5547a2013-03-29 01:23:58 +05305084 dma_addr_t iova)
Kay, Allen M38717942008-09-09 18:37:29 +03005085{
Joerg Roedel00a77de2015-03-26 13:43:08 +01005086 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
Kay, Allen M38717942008-09-09 18:37:29 +03005087 struct dma_pte *pte;
David Woodhouse5cf0a762014-03-19 16:07:49 +00005088 int level = 0;
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08005089 u64 phys = 0;
Kay, Allen M38717942008-09-09 18:37:29 +03005090
David Woodhouse5cf0a762014-03-19 16:07:49 +00005091 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
Kay, Allen M38717942008-09-09 18:37:29 +03005092 if (pte)
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08005093 phys = dma_pte_addr(pte);
Kay, Allen M38717942008-09-09 18:37:29 +03005094
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08005095 return phys;
Kay, Allen M38717942008-09-09 18:37:29 +03005096}
Joerg Roedela8bcbb0d2008-12-03 15:14:02 +01005097
Joerg Roedel5d587b82014-09-05 10:50:45 +02005098static bool intel_iommu_capable(enum iommu_cap cap)
Sheng Yangdbb9fd82009-03-18 15:33:06 +08005099{
Sheng Yangdbb9fd82009-03-18 15:33:06 +08005100 if (cap == IOMMU_CAP_CACHE_COHERENCY)
Joerg Roedel5d587b82014-09-05 10:50:45 +02005101 return domain_update_iommu_snooping(NULL) == 1;
Tom Lyon323f99c2010-07-02 16:56:14 -04005102 if (cap == IOMMU_CAP_INTR_REMAP)
Joerg Roedel5d587b82014-09-05 10:50:45 +02005103 return irq_remapping_enabled == 1;
Sheng Yangdbb9fd82009-03-18 15:33:06 +08005104
Joerg Roedel5d587b82014-09-05 10:50:45 +02005105 return false;
Sheng Yangdbb9fd82009-03-18 15:33:06 +08005106}
5107
Alex Williamsonabdfdde2012-05-30 14:19:19 -06005108static int intel_iommu_add_device(struct device *dev)
Alex Williamson70ae6f02011-10-21 15:56:11 -04005109{
Alex Williamsona5459cf2014-06-12 16:12:31 -06005110 struct intel_iommu *iommu;
Alex Williamsonabdfdde2012-05-30 14:19:19 -06005111 struct iommu_group *group;
David Woodhouse156baca2014-03-09 14:00:57 -07005112 u8 bus, devfn;
Alex Williamson70ae6f02011-10-21 15:56:11 -04005113
Alex Williamsona5459cf2014-06-12 16:12:31 -06005114 iommu = device_to_iommu(dev, &bus, &devfn);
5115 if (!iommu)
Alex Williamson70ae6f02011-10-21 15:56:11 -04005116 return -ENODEV;
5117
Joerg Roedele3d10af2017-02-01 17:23:22 +01005118 iommu_device_link(&iommu->iommu, dev);
Alex Williamsonabdfdde2012-05-30 14:19:19 -06005119
Alex Williamsone17f9ff2014-07-03 09:51:37 -06005120 group = iommu_group_get_for_dev(dev);
Alex Williamson783f1572012-05-30 14:19:43 -06005121
Alex Williamsone17f9ff2014-07-03 09:51:37 -06005122 if (IS_ERR(group))
5123 return PTR_ERR(group);
Alex Williamson70ae6f02011-10-21 15:56:11 -04005124
Alex Williamsonabdfdde2012-05-30 14:19:19 -06005125 iommu_group_put(group);
Alex Williamsone17f9ff2014-07-03 09:51:37 -06005126 return 0;
Alex Williamsonabdfdde2012-05-30 14:19:19 -06005127}
5128
5129static void intel_iommu_remove_device(struct device *dev)
5130{
Alex Williamsona5459cf2014-06-12 16:12:31 -06005131 struct intel_iommu *iommu;
5132 u8 bus, devfn;
5133
5134 iommu = device_to_iommu(dev, &bus, &devfn);
5135 if (!iommu)
5136 return;
5137
Alex Williamsonabdfdde2012-05-30 14:19:19 -06005138 iommu_group_remove_device(dev);
Alex Williamsona5459cf2014-06-12 16:12:31 -06005139
Joerg Roedele3d10af2017-02-01 17:23:22 +01005140 iommu_device_unlink(&iommu->iommu, dev);
Alex Williamson70ae6f02011-10-21 15:56:11 -04005141}
5142
Eric Auger0659b8d2017-01-19 20:57:53 +00005143static void intel_iommu_get_resv_regions(struct device *device,
5144 struct list_head *head)
5145{
5146 struct iommu_resv_region *reg;
5147 struct dmar_rmrr_unit *rmrr;
5148 struct device *i_dev;
5149 int i;
5150
5151 rcu_read_lock();
5152 for_each_rmrr_units(rmrr) {
5153 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
5154 i, i_dev) {
5155 if (i_dev != device)
5156 continue;
5157
5158 list_add_tail(&rmrr->resv->list, head);
5159 }
5160 }
5161 rcu_read_unlock();
5162
5163 reg = iommu_alloc_resv_region(IOAPIC_RANGE_START,
5164 IOAPIC_RANGE_END - IOAPIC_RANGE_START + 1,
Robin Murphy9d3a4de2017-03-16 17:00:16 +00005165 0, IOMMU_RESV_MSI);
Eric Auger0659b8d2017-01-19 20:57:53 +00005166 if (!reg)
5167 return;
5168 list_add_tail(&reg->list, head);
5169}
5170
5171static void intel_iommu_put_resv_regions(struct device *dev,
5172 struct list_head *head)
5173{
5174 struct iommu_resv_region *entry, *next;
5175
5176 list_for_each_entry_safe(entry, next, head, list) {
5177 if (entry->type == IOMMU_RESV_RESERVED)
5178 kfree(entry);
5179 }
Kay, Allen M38717942008-09-09 18:37:29 +03005180}
Joerg Roedela8bcbb0d2008-12-03 15:14:02 +01005181
David Woodhouse2f26e0a2015-09-09 11:40:47 +01005182#ifdef CONFIG_INTEL_IOMMU_SVM
Jacob Pan65ca7f52016-12-06 10:14:23 -08005183#define MAX_NR_PASID_BITS (20)
Lu Baolu4774cc52018-07-14 15:47:01 +08005184static inline unsigned long intel_iommu_get_pts(struct device *dev)
Jacob Pan65ca7f52016-12-06 10:14:23 -08005185{
Lu Baolu4774cc52018-07-14 15:47:01 +08005186 int pts, max_pasid;
5187
5188 max_pasid = intel_pasid_get_dev_max_id(dev);
5189 pts = find_first_bit((unsigned long *)&max_pasid, MAX_NR_PASID_BITS);
5190 if (pts < 5)
Jacob Pan65ca7f52016-12-06 10:14:23 -08005191 return 0;
5192
Lu Baolu4774cc52018-07-14 15:47:01 +08005193 return pts - 5;
Jacob Pan65ca7f52016-12-06 10:14:23 -08005194}
5195
David Woodhouse2f26e0a2015-09-09 11:40:47 +01005196int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct intel_svm_dev *sdev)
5197{
5198 struct device_domain_info *info;
5199 struct context_entry *context;
5200 struct dmar_domain *domain;
5201 unsigned long flags;
5202 u64 ctx_lo;
5203 int ret;
5204
5205 domain = get_valid_domain_for_dev(sdev->dev);
5206 if (!domain)
5207 return -EINVAL;
5208
5209 spin_lock_irqsave(&device_domain_lock, flags);
5210 spin_lock(&iommu->lock);
5211
5212 ret = -EINVAL;
5213 info = sdev->dev->archdata.iommu;
5214 if (!info || !info->pasid_supported)
5215 goto out;
5216
5217 context = iommu_context_addr(iommu, info->bus, info->devfn, 0);
5218 if (WARN_ON(!context))
5219 goto out;
5220
5221 ctx_lo = context[0].lo;
5222
5223 sdev->did = domain->iommu_did[iommu->seq_id];
5224 sdev->sid = PCI_DEVID(info->bus, info->devfn);
5225
5226 if (!(ctx_lo & CONTEXT_PASIDE)) {
Ashok Raj11b93eb2017-08-08 13:29:28 -07005227 if (iommu->pasid_state_table)
5228 context[1].hi = (u64)virt_to_phys(iommu->pasid_state_table);
Lu Baolu4774cc52018-07-14 15:47:01 +08005229 context[1].lo = (u64)virt_to_phys(info->pasid_table->table) |
5230 intel_iommu_get_pts(sdev->dev);
Jacob Pan65ca7f52016-12-06 10:14:23 -08005231
David Woodhouse2f26e0a2015-09-09 11:40:47 +01005232 wmb();
5233 /* CONTEXT_TT_MULTI_LEVEL and CONTEXT_TT_DEV_IOTLB are both
5234 * extended to permit requests-with-PASID if the PASIDE bit
5235 * is set. which makes sense. For CONTEXT_TT_PASS_THROUGH,
5236 * however, the PASIDE bit is ignored and requests-with-PASID
5237 * are unconditionally blocked. Which makes less sense.
5238 * So convert from CONTEXT_TT_PASS_THROUGH to one of the new
5239 * "guest mode" translation types depending on whether ATS
5240 * is available or not. Annoyingly, we can't use the new
5241 * modes *unless* PASIDE is set. */
5242 if ((ctx_lo & CONTEXT_TT_MASK) == (CONTEXT_TT_PASS_THROUGH << 2)) {
5243 ctx_lo &= ~CONTEXT_TT_MASK;
5244 if (info->ats_supported)
5245 ctx_lo |= CONTEXT_TT_PT_PASID_DEV_IOTLB << 2;
5246 else
5247 ctx_lo |= CONTEXT_TT_PT_PASID << 2;
5248 }
5249 ctx_lo |= CONTEXT_PASIDE;
David Woodhouse907fea32015-10-13 14:11:13 +01005250 if (iommu->pasid_state_table)
5251 ctx_lo |= CONTEXT_DINVE;
David Woodhousea222a7f2015-10-07 23:35:18 +01005252 if (info->pri_supported)
5253 ctx_lo |= CONTEXT_PRS;
David Woodhouse2f26e0a2015-09-09 11:40:47 +01005254 context[0].lo = ctx_lo;
5255 wmb();
5256 iommu->flush.flush_context(iommu, sdev->did, sdev->sid,
5257 DMA_CCMD_MASK_NOBIT,
5258 DMA_CCMD_DEVICE_INVL);
5259 }
5260
5261 /* Enable PASID support in the device, if it wasn't already */
5262 if (!info->pasid_enabled)
5263 iommu_enable_dev_iotlb(info);
5264
5265 if (info->ats_enabled) {
5266 sdev->dev_iotlb = 1;
5267 sdev->qdep = info->ats_qdep;
5268 if (sdev->qdep >= QI_DEV_EIOTLB_MAX_INVS)
5269 sdev->qdep = 0;
5270 }
5271 ret = 0;
5272
5273 out:
5274 spin_unlock(&iommu->lock);
5275 spin_unlock_irqrestore(&device_domain_lock, flags);
5276
5277 return ret;
5278}
5279
5280struct intel_iommu *intel_svm_device_to_iommu(struct device *dev)
5281{
5282 struct intel_iommu *iommu;
5283 u8 bus, devfn;
5284
5285 if (iommu_dummy(dev)) {
5286 dev_warn(dev,
5287 "No IOMMU translation for device; cannot enable SVM\n");
5288 return NULL;
5289 }
5290
5291 iommu = device_to_iommu(dev, &bus, &devfn);
5292 if ((!iommu)) {
Sudeep Duttb9997e32015-10-18 20:54:37 -07005293 dev_err(dev, "No IOMMU for device; cannot enable SVM\n");
David Woodhouse2f26e0a2015-09-09 11:40:47 +01005294 return NULL;
5295 }
5296
David Woodhouse2f26e0a2015-09-09 11:40:47 +01005297 return iommu;
5298}
5299#endif /* CONFIG_INTEL_IOMMU_SVM */
5300
Joerg Roedelb0119e82017-02-01 13:23:08 +01005301const struct iommu_ops intel_iommu_ops = {
Eric Auger0659b8d2017-01-19 20:57:53 +00005302 .capable = intel_iommu_capable,
5303 .domain_alloc = intel_iommu_domain_alloc,
5304 .domain_free = intel_iommu_domain_free,
5305 .attach_dev = intel_iommu_attach_device,
5306 .detach_dev = intel_iommu_detach_device,
5307 .map = intel_iommu_map,
5308 .unmap = intel_iommu_unmap,
Eric Auger0659b8d2017-01-19 20:57:53 +00005309 .iova_to_phys = intel_iommu_iova_to_phys,
5310 .add_device = intel_iommu_add_device,
5311 .remove_device = intel_iommu_remove_device,
5312 .get_resv_regions = intel_iommu_get_resv_regions,
5313 .put_resv_regions = intel_iommu_put_resv_regions,
5314 .device_group = pci_device_group,
5315 .pgsize_bitmap = INTEL_IOMMU_PGSIZES,
Joerg Roedela8bcbb0d2008-12-03 15:14:02 +01005316};
David Woodhouse9af88142009-02-13 23:18:03 +00005317
Daniel Vetter94526182013-01-20 23:50:13 +01005318static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
5319{
5320 /* G4x/GM45 integrated gfx dmar support is totally busted. */
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02005321 pr_info("Disabling IOMMU for graphics on this chipset\n");
Daniel Vetter94526182013-01-20 23:50:13 +01005322 dmar_map_gfx = 0;
5323}
5324
5325DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
5326DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
5327DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
5328DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
5329DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
5330DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
5331DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
5332
Greg Kroah-Hartmand34d6512012-12-21 15:05:21 -08005333static void quirk_iommu_rwbf(struct pci_dev *dev)
David Woodhouse9af88142009-02-13 23:18:03 +00005334{
5335 /*
5336 * Mobile 4 Series Chipset neglects to set RWBF capability,
Daniel Vetter210561f2013-01-21 19:48:59 +01005337 * but needs it. Same seems to hold for the desktop versions.
David Woodhouse9af88142009-02-13 23:18:03 +00005338 */
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02005339 pr_info("Forcing write-buffer flush capability\n");
David Woodhouse9af88142009-02-13 23:18:03 +00005340 rwbf_quirk = 1;
5341}
5342
5343DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
Daniel Vetter210561f2013-01-21 19:48:59 +01005344DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
5345DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
5346DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
5347DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
5348DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
5349DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
David Woodhousee0fc7e02009-09-30 09:12:17 -07005350
Adam Jacksoneecfd572010-08-25 21:17:34 +01005351#define GGC 0x52
5352#define GGC_MEMORY_SIZE_MASK (0xf << 8)
5353#define GGC_MEMORY_SIZE_NONE (0x0 << 8)
5354#define GGC_MEMORY_SIZE_1M (0x1 << 8)
5355#define GGC_MEMORY_SIZE_2M (0x3 << 8)
5356#define GGC_MEMORY_VT_ENABLED (0x8 << 8)
5357#define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
5358#define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
5359#define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
5360
Greg Kroah-Hartmand34d6512012-12-21 15:05:21 -08005361static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
David Woodhouse9eecabc2010-09-21 22:28:23 +01005362{
5363 unsigned short ggc;
5364
Adam Jacksoneecfd572010-08-25 21:17:34 +01005365 if (pci_read_config_word(dev, GGC, &ggc))
David Woodhouse9eecabc2010-09-21 22:28:23 +01005366 return;
5367
Adam Jacksoneecfd572010-08-25 21:17:34 +01005368 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02005369 pr_info("BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
David Woodhouse9eecabc2010-09-21 22:28:23 +01005370 dmar_map_gfx = 0;
David Woodhouse6fbcfb32011-09-25 19:11:14 -07005371 } else if (dmar_map_gfx) {
5372 /* we have to ensure the gfx device is idle before we flush */
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02005373 pr_info("Disabling batched IOTLB flush on Ironlake\n");
David Woodhouse6fbcfb32011-09-25 19:11:14 -07005374 intel_iommu_strict = 1;
5375 }
David Woodhouse9eecabc2010-09-21 22:28:23 +01005376}
5377DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
5378DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
5379DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
5380DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
5381
David Woodhousee0fc7e02009-09-30 09:12:17 -07005382/* On Tylersburg chipsets, some BIOSes have been known to enable the
5383 ISOCH DMAR unit for the Azalia sound device, but not give it any
5384 TLB entries, which causes it to deadlock. Check for that. We do
5385 this in a function called from init_dmars(), instead of in a PCI
5386 quirk, because we don't want to print the obnoxious "BIOS broken"
5387 message if VT-d is actually disabled.
5388*/
5389static void __init check_tylersburg_isoch(void)
5390{
5391 struct pci_dev *pdev;
5392 uint32_t vtisochctrl;
5393
5394 /* If there's no Azalia in the system anyway, forget it. */
5395 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
5396 if (!pdev)
5397 return;
5398 pci_dev_put(pdev);
5399
5400 /* System Management Registers. Might be hidden, in which case
5401 we can't do the sanity check. But that's OK, because the
5402 known-broken BIOSes _don't_ actually hide it, so far. */
5403 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
5404 if (!pdev)
5405 return;
5406
5407 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
5408 pci_dev_put(pdev);
5409 return;
5410 }
5411
5412 pci_dev_put(pdev);
5413
5414 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
5415 if (vtisochctrl & 1)
5416 return;
5417
5418 /* Drop all bits other than the number of TLB entries */
5419 vtisochctrl &= 0x1c;
5420
5421 /* If we have the recommended number of TLB entries (16), fine. */
5422 if (vtisochctrl == 0x10)
5423 return;
5424
5425 /* Zero TLB entries? You get to ride the short bus to school. */
5426 if (!vtisochctrl) {
5427 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
5428 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
5429 dmi_get_system_info(DMI_BIOS_VENDOR),
5430 dmi_get_system_info(DMI_BIOS_VERSION),
5431 dmi_get_system_info(DMI_PRODUCT_VERSION));
5432 iommu_identity_mapping |= IDENTMAP_AZALIA;
5433 return;
5434 }
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02005435
5436 pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
David Woodhousee0fc7e02009-09-30 09:12:17 -07005437 vtisochctrl);
5438}