blob: 0ad67d65bbce226820f5334b48c7cfca154931d4 [file] [log] [blame]
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001/*
David Woodhouseea8ea462014-03-05 17:09:32 +00002 * Copyright © 2006-2014 Intel Corporation.
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
David Woodhouseea8ea462014-03-05 17:09:32 +000013 * Authors: David Woodhouse <dwmw2@infradead.org>,
14 * Ashok Raj <ashok.raj@intel.com>,
15 * Shaohua Li <shaohua.li@intel.com>,
16 * Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
17 * Fenghua Yu <fenghua.yu@intel.com>
Joerg Roedel9f10e5b2015-06-12 09:57:06 +020018 * Joerg Roedel <jroedel@suse.de>
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -070019 */
20
Joerg Roedel9f10e5b2015-06-12 09:57:06 +020021#define pr_fmt(fmt) "DMAR: " fmt
22
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -070023#include <linux/init.h>
24#include <linux/bitmap.h>
mark gross5e0d2a62008-03-04 15:22:08 -080025#include <linux/debugfs.h>
Paul Gortmaker54485c32011-10-29 10:26:25 -040026#include <linux/export.h>
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -070027#include <linux/slab.h>
28#include <linux/irq.h>
29#include <linux/interrupt.h>
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -070030#include <linux/spinlock.h>
31#include <linux/pci.h>
32#include <linux/dmar.h>
33#include <linux/dma-mapping.h>
34#include <linux/mempool.h>
Jiang Liu75f05562014-02-19 14:07:37 +080035#include <linux/memory.h>
Omer Pelegaa473242016-04-20 11:33:02 +030036#include <linux/cpu.h>
mark gross5e0d2a62008-03-04 15:22:08 -080037#include <linux/timer.h>
Dan Williamsdfddb962015-10-09 18:16:46 -040038#include <linux/io.h>
Kay, Allen M38717942008-09-09 18:37:29 +030039#include <linux/iova.h>
Joerg Roedel5d450802008-12-03 14:52:32 +010040#include <linux/iommu.h>
Kay, Allen M38717942008-09-09 18:37:29 +030041#include <linux/intel-iommu.h>
Rafael J. Wysocki134fac32011-03-23 22:16:14 +010042#include <linux/syscore_ops.h>
Shane Wang69575d32009-09-01 18:25:07 -070043#include <linux/tboot.h>
Stephen Rothwelladb2fe02009-08-31 15:24:23 +100044#include <linux/dmi.h>
Joerg Roedel5cdede22011-04-04 15:55:18 +020045#include <linux/pci-ats.h>
Tejun Heo0ee332c2011-12-08 10:22:09 -080046#include <linux/memblock.h>
Akinobu Mita36746432014-06-04 16:06:51 -070047#include <linux/dma-contiguous.h>
Christoph Hellwigfec777c2018-03-19 11:38:15 +010048#include <linux/dma-direct.h>
Joerg Roedel091d42e2015-06-12 11:56:10 +020049#include <linux/crash_dump.h>
Suresh Siddha8a8f4222012-03-30 11:47:08 -070050#include <asm/irq_remapping.h>
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -070051#include <asm/cacheflush.h>
FUJITA Tomonori46a7fa22008-07-11 10:23:42 +090052#include <asm/iommu.h>
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -070053
Joerg Roedel078e1ee2012-09-26 12:44:43 +020054#include "irq_remapping.h"
Lu Baolu56283172018-07-14 15:46:54 +080055#include "intel-pasid.h"
Joerg Roedel078e1ee2012-09-26 12:44:43 +020056
Fenghua Yu5b6985c2008-10-16 18:02:32 -070057#define ROOT_SIZE VTD_PAGE_SIZE
58#define CONTEXT_SIZE VTD_PAGE_SIZE
59
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -070060#define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
David Woodhouse18436af2015-03-25 15:05:47 +000061#define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -070062#define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
David Woodhousee0fc7e02009-09-30 09:12:17 -070063#define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -070064
65#define IOAPIC_RANGE_START (0xfee00000)
66#define IOAPIC_RANGE_END (0xfeefffff)
67#define IOVA_START_ADDR (0x1000)
68
Sohil Mehta5e3b4a12017-12-20 11:59:24 -080069#define DEFAULT_DOMAIN_ADDRESS_WIDTH 57
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -070070
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -070071#define MAX_AGAW_WIDTH 64
Jiang Liu5c645b32014-01-06 14:18:12 +080072#define MAX_AGAW_PFN_WIDTH (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -070073
David Woodhouse2ebe3152009-09-19 07:34:04 -070074#define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
75#define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
76
77/* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
78 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
79#define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
80 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
81#define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -070082
Robin Murphy1b722502015-01-12 17:51:15 +000083/* IO virtual address start page frame number */
84#define IOVA_START_PFN (1)
85
Mark McLoughlinf27be032008-11-20 15:49:43 +000086#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
mark gross5e0d2a62008-03-04 15:22:08 -080087
Andrew Mortondf08cdc2010-09-22 13:05:11 -070088/* page table handling */
89#define LEVEL_STRIDE (9)
90#define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
91
Ohad Ben-Cohen6d1c56a2011-11-10 11:32:30 +020092/*
93 * This bitmap is used to advertise the page sizes our hardware support
94 * to the IOMMU core, which will then use this information to split
95 * physically contiguous memory regions it is mapping into page sizes
96 * that we support.
97 *
98 * Traditionally the IOMMU core just handed us the mappings directly,
99 * after making sure the size is an order of a 4KiB page and that the
100 * mapping has natural alignment.
101 *
102 * To retain this behavior, we currently advertise that we support
103 * all page sizes that are an order of 4KiB.
104 *
105 * If at some point we'd like to utilize the IOMMU core's new behavior,
106 * we could change this to advertise the real page sizes we support.
107 */
108#define INTEL_IOMMU_PGSIZES (~0xFFFUL)
109
Andrew Mortondf08cdc2010-09-22 13:05:11 -0700110static inline int agaw_to_level(int agaw)
111{
112 return agaw + 2;
113}
114
115static inline int agaw_to_width(int agaw)
116{
Jiang Liu5c645b32014-01-06 14:18:12 +0800117 return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
Andrew Mortondf08cdc2010-09-22 13:05:11 -0700118}
119
120static inline int width_to_agaw(int width)
121{
Jiang Liu5c645b32014-01-06 14:18:12 +0800122 return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
Andrew Mortondf08cdc2010-09-22 13:05:11 -0700123}
124
125static inline unsigned int level_to_offset_bits(int level)
126{
127 return (level - 1) * LEVEL_STRIDE;
128}
129
130static inline int pfn_level_offset(unsigned long pfn, int level)
131{
132 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
133}
134
135static inline unsigned long level_mask(int level)
136{
137 return -1UL << level_to_offset_bits(level);
138}
139
140static inline unsigned long level_size(int level)
141{
142 return 1UL << level_to_offset_bits(level);
143}
144
145static inline unsigned long align_to_level(unsigned long pfn, int level)
146{
147 return (pfn + level_size(level) - 1) & level_mask(level);
148}
David Woodhousefd18de52009-05-10 23:57:41 +0100149
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100150static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
151{
Jiang Liu5c645b32014-01-06 14:18:12 +0800152 return 1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100153}
154
David Woodhousedd4e8312009-06-27 16:21:20 +0100155/* VT-d pages must always be _smaller_ than MM pages. Otherwise things
156 are never going to work. */
157static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
158{
159 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
160}
161
162static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
163{
164 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
165}
166static inline unsigned long page_to_dma_pfn(struct page *pg)
167{
168 return mm_to_dma_pfn(page_to_pfn(pg));
169}
170static inline unsigned long virt_to_dma_pfn(void *p)
171{
172 return page_to_dma_pfn(virt_to_page(p));
173}
174
Weidong Hand9630fe2008-12-08 11:06:32 +0800175/* global iommu list, set NULL for ignored DMAR units */
176static struct intel_iommu **g_iommus;
177
David Woodhousee0fc7e02009-09-30 09:12:17 -0700178static void __init check_tylersburg_isoch(void);
David Woodhouse9af88142009-02-13 23:18:03 +0000179static int rwbf_quirk;
180
Mark McLoughlin46b08e12008-11-20 15:49:44 +0000181/*
Joseph Cihulab7792602011-05-03 00:08:37 -0700182 * set to 1 to panic kernel if can't successfully enable VT-d
183 * (used when kernel is launched w/ TXT)
184 */
185static int force_on = 0;
Shaohua Libfd20f12017-04-26 09:18:35 -0700186int intel_iommu_tboot_noforce;
Joseph Cihulab7792602011-05-03 00:08:37 -0700187
Mark McLoughlin46b08e12008-11-20 15:49:44 +0000188#define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
Mark McLoughlin46b08e12008-11-20 15:49:44 +0000189
Joerg Roedel091d42e2015-06-12 11:56:10 +0200190/*
191 * Take a root_entry and return the Lower Context Table Pointer (LCTP)
192 * if marked present.
193 */
194static phys_addr_t root_entry_lctp(struct root_entry *re)
195{
196 if (!(re->lo & 1))
197 return 0;
Mark McLoughlin46b08e12008-11-20 15:49:44 +0000198
Joerg Roedel091d42e2015-06-12 11:56:10 +0200199 return re->lo & VTD_PAGE_MASK;
200}
201
202/*
203 * Take a root_entry and return the Upper Context Table Pointer (UCTP)
204 * if marked present.
205 */
206static phys_addr_t root_entry_uctp(struct root_entry *re)
207{
208 if (!(re->hi & 1))
209 return 0;
210
211 return re->hi & VTD_PAGE_MASK;
212}
Mark McLoughlin7a8fc252008-11-20 15:49:45 +0000213
Joerg Roedelcf484d02015-06-12 12:21:46 +0200214static inline void context_clear_pasid_enable(struct context_entry *context)
215{
216 context->lo &= ~(1ULL << 11);
217}
218
219static inline bool context_pasid_enabled(struct context_entry *context)
220{
221 return !!(context->lo & (1ULL << 11));
222}
223
224static inline void context_set_copied(struct context_entry *context)
225{
226 context->hi |= (1ull << 3);
227}
228
229static inline bool context_copied(struct context_entry *context)
230{
231 return !!(context->hi & (1ULL << 3));
232}
233
234static inline bool __context_present(struct context_entry *context)
Mark McLoughlinc07e7d22008-11-21 16:54:46 +0000235{
236 return (context->lo & 1);
237}
Joerg Roedelcf484d02015-06-12 12:21:46 +0200238
Sohil Mehta26b86092018-09-11 17:11:36 -0700239bool context_present(struct context_entry *context)
Joerg Roedelcf484d02015-06-12 12:21:46 +0200240{
241 return context_pasid_enabled(context) ?
242 __context_present(context) :
243 __context_present(context) && !context_copied(context);
244}
245
Mark McLoughlinc07e7d22008-11-21 16:54:46 +0000246static inline void context_set_present(struct context_entry *context)
247{
248 context->lo |= 1;
249}
250
251static inline void context_set_fault_enable(struct context_entry *context)
252{
253 context->lo &= (((u64)-1) << 2) | 1;
254}
255
Mark McLoughlinc07e7d22008-11-21 16:54:46 +0000256static inline void context_set_translation_type(struct context_entry *context,
257 unsigned long value)
258{
259 context->lo &= (((u64)-1) << 4) | 3;
260 context->lo |= (value & 3) << 2;
261}
262
263static inline void context_set_address_root(struct context_entry *context,
264 unsigned long value)
265{
Li, Zhen-Hua1a2262f2014-11-05 15:30:19 +0800266 context->lo &= ~VTD_PAGE_MASK;
Mark McLoughlinc07e7d22008-11-21 16:54:46 +0000267 context->lo |= value & VTD_PAGE_MASK;
268}
269
270static inline void context_set_address_width(struct context_entry *context,
271 unsigned long value)
272{
273 context->hi |= value & 7;
274}
275
276static inline void context_set_domain_id(struct context_entry *context,
277 unsigned long value)
278{
279 context->hi |= (value & ((1 << 16) - 1)) << 8;
280}
281
Joerg Roedeldbcd8612015-06-12 12:02:09 +0200282static inline int context_domain_id(struct context_entry *c)
283{
284 return((c->hi >> 8) & 0xffff);
285}
286
Mark McLoughlinc07e7d22008-11-21 16:54:46 +0000287static inline void context_clear_entry(struct context_entry *context)
288{
289 context->lo = 0;
290 context->hi = 0;
291}
Mark McLoughlin7a8fc252008-11-20 15:49:45 +0000292
Mark McLoughlin622ba122008-11-20 15:49:46 +0000293/*
294 * 0: readable
295 * 1: writable
296 * 2-6: reserved
297 * 7: super page
Sheng Yang9cf06692009-03-18 15:33:07 +0800298 * 8-10: available
299 * 11: snoop behavior
Mark McLoughlin622ba122008-11-20 15:49:46 +0000300 * 12-63: Host physcial address
301 */
302struct dma_pte {
303 u64 val;
304};
Mark McLoughlin622ba122008-11-20 15:49:46 +0000305
Mark McLoughlin19c239c2008-11-21 16:56:53 +0000306static inline void dma_clear_pte(struct dma_pte *pte)
307{
308 pte->val = 0;
309}
310
Mark McLoughlin19c239c2008-11-21 16:56:53 +0000311static inline u64 dma_pte_addr(struct dma_pte *pte)
312{
David Woodhousec85994e2009-07-01 19:21:24 +0100313#ifdef CONFIG_64BIT
314 return pte->val & VTD_PAGE_MASK;
315#else
316 /* Must have a full atomic 64-bit read */
David Woodhouse1a8bd482010-08-10 01:38:53 +0100317 return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
David Woodhousec85994e2009-07-01 19:21:24 +0100318#endif
Mark McLoughlin19c239c2008-11-21 16:56:53 +0000319}
320
Mark McLoughlin19c239c2008-11-21 16:56:53 +0000321static inline bool dma_pte_present(struct dma_pte *pte)
322{
323 return (pte->val & 3) != 0;
324}
Mark McLoughlin622ba122008-11-20 15:49:46 +0000325
Allen Kay4399c8b2011-10-14 12:32:46 -0700326static inline bool dma_pte_superpage(struct dma_pte *pte)
327{
Joerg Roedelc3c75eb2014-07-04 11:19:10 +0200328 return (pte->val & DMA_PTE_LARGE_PAGE);
Allen Kay4399c8b2011-10-14 12:32:46 -0700329}
330
David Woodhouse75e6bf92009-07-02 11:21:16 +0100331static inline int first_pte_in_page(struct dma_pte *pte)
332{
333 return !((unsigned long)pte & ~VTD_PAGE_MASK);
334}
335
Fenghua Yu2c2e2c32009-06-19 13:47:29 -0700336/*
337 * This domain is a statically identity mapping domain.
338 * 1. This domain creats a static 1:1 mapping to all usable memory.
339 * 2. It maps to each iommu if successful.
340 * 3. Each iommu mapps to this domain if successful.
341 */
David Woodhouse19943b02009-08-04 16:19:20 +0100342static struct dmar_domain *si_domain;
343static int hw_pass_through = 1;
Fenghua Yu2c2e2c32009-06-19 13:47:29 -0700344
Joerg Roedel28ccce02015-07-21 14:45:31 +0200345/*
346 * Domain represents a virtual machine, more than one devices
Weidong Han1ce28fe2008-12-08 16:35:39 +0800347 * across iommus may be owned in one domain, e.g. kvm guest.
348 */
Jiang Liuab8dfe22014-07-11 14:19:27 +0800349#define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 0)
Weidong Han1ce28fe2008-12-08 16:35:39 +0800350
Fenghua Yu2c2e2c32009-06-19 13:47:29 -0700351/* si_domain contains mulitple devices */
Jiang Liuab8dfe22014-07-11 14:19:27 +0800352#define DOMAIN_FLAG_STATIC_IDENTITY (1 << 1)
Fenghua Yu2c2e2c32009-06-19 13:47:29 -0700353
Joerg Roedel29a27712015-07-21 17:17:12 +0200354#define for_each_domain_iommu(idx, domain) \
355 for (idx = 0; idx < g_num_of_iommus; idx++) \
356 if (domain->iommu_refcnt[idx])
357
Jiang Liub94e4112014-02-19 14:07:25 +0800358struct dmar_rmrr_unit {
359 struct list_head list; /* list of rmrr units */
360 struct acpi_dmar_header *hdr; /* ACPI header */
361 u64 base_address; /* reserved base address*/
362 u64 end_address; /* reserved end address */
David Woodhouse832bd852014-03-07 15:08:36 +0000363 struct dmar_dev_scope *devices; /* target devices */
Jiang Liub94e4112014-02-19 14:07:25 +0800364 int devices_cnt; /* target device count */
Eric Auger0659b8d2017-01-19 20:57:53 +0000365 struct iommu_resv_region *resv; /* reserved region handle */
Jiang Liub94e4112014-02-19 14:07:25 +0800366};
367
368struct dmar_atsr_unit {
369 struct list_head list; /* list of ATSR units */
370 struct acpi_dmar_header *hdr; /* ACPI header */
David Woodhouse832bd852014-03-07 15:08:36 +0000371 struct dmar_dev_scope *devices; /* target devices */
Jiang Liub94e4112014-02-19 14:07:25 +0800372 int devices_cnt; /* target device count */
373 u8 include_all:1; /* include all ports */
374};
375
376static LIST_HEAD(dmar_atsr_units);
377static LIST_HEAD(dmar_rmrr_units);
378
379#define for_each_rmrr_units(rmrr) \
380 list_for_each_entry(rmrr, &dmar_rmrr_units, list)
381
mark gross5e0d2a62008-03-04 15:22:08 -0800382/* bitmap for indexing intel_iommus */
mark gross5e0d2a62008-03-04 15:22:08 -0800383static int g_num_of_iommus;
384
Jiang Liu92d03cc2014-02-19 14:07:28 +0800385static void domain_exit(struct dmar_domain *domain);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700386static void domain_remove_dev_info(struct dmar_domain *domain);
Joerg Roedele6de0f82015-07-22 16:30:36 +0200387static void dmar_remove_one_dev_info(struct dmar_domain *domain,
388 struct device *dev);
Joerg Roedel127c7612015-07-23 17:44:46 +0200389static void __dmar_remove_one_dev_info(struct device_domain_info *info);
Joerg Roedel2452d9d2015-07-23 16:20:14 +0200390static void domain_context_clear(struct intel_iommu *iommu,
391 struct device *dev);
Jiang Liu2a46ddf2014-07-11 14:19:30 +0800392static int domain_detach_iommu(struct dmar_domain *domain,
393 struct intel_iommu *iommu);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700394
Suresh Siddhad3f13812011-08-23 17:05:25 -0700395#ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
Kyle McMartin0cd5c3c2009-02-04 14:29:19 -0800396int dmar_disabled = 0;
397#else
398int dmar_disabled = 1;
Suresh Siddhad3f13812011-08-23 17:05:25 -0700399#endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
Kyle McMartin0cd5c3c2009-02-04 14:29:19 -0800400
Eugeni Dodonov8bc1f852011-11-23 16:42:14 -0200401int intel_iommu_enabled = 0;
402EXPORT_SYMBOL_GPL(intel_iommu_enabled);
403
David Woodhouse2d9e6672010-06-15 10:57:57 +0100404static int dmar_map_gfx = 1;
Keshavamurthy, Anil S7d3b03c2007-10-21 16:41:53 -0700405static int dmar_forcedac;
mark gross5e0d2a62008-03-04 15:22:08 -0800406static int intel_iommu_strict;
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100407static int intel_iommu_superpage = 1;
David Woodhousec83b2f22015-06-12 10:15:49 +0100408static int intel_iommu_ecs = 1;
Lu Baolu2db15812018-07-08 14:23:21 +0800409static int intel_iommu_pasid28;
David Woodhouseae853dd2015-09-09 11:58:59 +0100410static int iommu_identity_mapping;
David Woodhousec83b2f22015-06-12 10:15:49 +0100411
David Woodhouseae853dd2015-09-09 11:58:59 +0100412#define IDENTMAP_ALL 1
413#define IDENTMAP_GFX 2
414#define IDENTMAP_AZALIA 4
David Woodhousec83b2f22015-06-12 10:15:49 +0100415
Lu Baolu2db15812018-07-08 14:23:21 +0800416/* Broadwell and Skylake have broken ECS support — normal so-called "second
417 * level" translation of DMA requests-without-PASID doesn't actually happen
418 * unless you also set the NESTE bit in an extended context-entry. Which of
419 * course means that SVM doesn't work because it's trying to do nested
420 * translation of the physical addresses it finds in the process page tables,
421 * through the IOVA->phys mapping found in the "second level" page tables.
422 *
423 * The VT-d specification was retroactively changed to change the definition
424 * of the capability bits and pretend that Broadwell/Skylake never happened...
425 * but unfortunately the wrong bit was changed. It's ECS which is broken, but
426 * for some reason it was the PASID capability bit which was redefined (from
427 * bit 28 on BDW/SKL to bit 40 in future).
428 *
429 * So our test for ECS needs to eschew those implementations which set the old
430 * PASID capabiity bit 28, since those are the ones on which ECS is broken.
431 * Unless we are working around the 'pasid28' limitations, that is, by putting
432 * the device into passthrough mode for normal DMA and thus masking the bug.
433 */
434#define ecs_enabled(iommu) (intel_iommu_ecs && ecap_ecs(iommu->ecap) && \
435 (intel_iommu_pasid28 || !ecap_broken_pasid(iommu->ecap)))
436/* PASID support is thus enabled if ECS is enabled and *either* of the old
437 * or new capability bits are set. */
438#define pasid_enabled(iommu) (ecs_enabled(iommu) && \
439 (ecap_pasid(iommu->ecap) || ecap_broken_pasid(iommu->ecap)))
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700440
David Woodhousec0771df2011-10-14 20:59:46 +0100441int intel_iommu_gfx_mapped;
442EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
443
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700444#define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
445static DEFINE_SPINLOCK(device_domain_lock);
446static LIST_HEAD(device_domain_list);
447
Lu Baolu85319dc2018-07-14 15:46:58 +0800448/*
449 * Iterate over elements in device_domain_list and call the specified
450 * callback @fn against each element. This helper should only be used
451 * in the context where the device_domain_lock has already been holden.
452 */
453int for_each_device_domain(int (*fn)(struct device_domain_info *info,
454 void *data), void *data)
455{
456 int ret = 0;
457 struct device_domain_info *info;
458
459 assert_spin_locked(&device_domain_lock);
460 list_for_each_entry(info, &device_domain_list, global) {
461 ret = fn(info, data);
462 if (ret)
463 return ret;
464 }
465
466 return 0;
467}
468
Joerg Roedelb0119e82017-02-01 13:23:08 +0100469const struct iommu_ops intel_iommu_ops;
Joerg Roedela8bcbb0d2008-12-03 15:14:02 +0100470
Joerg Roedel4158c2e2015-06-12 10:14:02 +0200471static bool translation_pre_enabled(struct intel_iommu *iommu)
472{
473 return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
474}
475
Joerg Roedel091d42e2015-06-12 11:56:10 +0200476static void clear_translation_pre_enabled(struct intel_iommu *iommu)
477{
478 iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
479}
480
Joerg Roedel4158c2e2015-06-12 10:14:02 +0200481static void init_translation_status(struct intel_iommu *iommu)
482{
483 u32 gsts;
484
485 gsts = readl(iommu->reg + DMAR_GSTS_REG);
486 if (gsts & DMA_GSTS_TES)
487 iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
488}
489
Joerg Roedel00a77de2015-03-26 13:43:08 +0100490/* Convert generic 'struct iommu_domain to private struct dmar_domain */
491static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
492{
493 return container_of(dom, struct dmar_domain, domain);
494}
495
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700496static int __init intel_iommu_setup(char *str)
497{
498 if (!str)
499 return -EINVAL;
500 while (*str) {
Kyle McMartin0cd5c3c2009-02-04 14:29:19 -0800501 if (!strncmp(str, "on", 2)) {
502 dmar_disabled = 0;
Joerg Roedel9f10e5b2015-06-12 09:57:06 +0200503 pr_info("IOMMU enabled\n");
Kyle McMartin0cd5c3c2009-02-04 14:29:19 -0800504 } else if (!strncmp(str, "off", 3)) {
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700505 dmar_disabled = 1;
Joerg Roedel9f10e5b2015-06-12 09:57:06 +0200506 pr_info("IOMMU disabled\n");
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700507 } else if (!strncmp(str, "igfx_off", 8)) {
508 dmar_map_gfx = 0;
Joerg Roedel9f10e5b2015-06-12 09:57:06 +0200509 pr_info("Disable GFX device mapping\n");
Keshavamurthy, Anil S7d3b03c2007-10-21 16:41:53 -0700510 } else if (!strncmp(str, "forcedac", 8)) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +0200511 pr_info("Forcing DAC for PCI devices\n");
Keshavamurthy, Anil S7d3b03c2007-10-21 16:41:53 -0700512 dmar_forcedac = 1;
mark gross5e0d2a62008-03-04 15:22:08 -0800513 } else if (!strncmp(str, "strict", 6)) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +0200514 pr_info("Disable batched IOTLB flush\n");
mark gross5e0d2a62008-03-04 15:22:08 -0800515 intel_iommu_strict = 1;
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100516 } else if (!strncmp(str, "sp_off", 6)) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +0200517 pr_info("Disable supported super page\n");
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100518 intel_iommu_superpage = 0;
David Woodhousec83b2f22015-06-12 10:15:49 +0100519 } else if (!strncmp(str, "ecs_off", 7)) {
520 printk(KERN_INFO
521 "Intel-IOMMU: disable extended context table support\n");
522 intel_iommu_ecs = 0;
Lu Baolu2db15812018-07-08 14:23:21 +0800523 } else if (!strncmp(str, "pasid28", 7)) {
524 printk(KERN_INFO
525 "Intel-IOMMU: enable pre-production PASID support\n");
526 intel_iommu_pasid28 = 1;
527 iommu_identity_mapping |= IDENTMAP_GFX;
Shaohua Libfd20f12017-04-26 09:18:35 -0700528 } else if (!strncmp(str, "tboot_noforce", 13)) {
529 printk(KERN_INFO
530 "Intel-IOMMU: not forcing on after tboot. This could expose security risk for tboot\n");
531 intel_iommu_tboot_noforce = 1;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700532 }
533
534 str += strcspn(str, ",");
535 while (*str == ',')
536 str++;
537 }
538 return 0;
539}
540__setup("intel_iommu=", intel_iommu_setup);
541
542static struct kmem_cache *iommu_domain_cache;
543static struct kmem_cache *iommu_devinfo_cache;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700544
Joerg Roedel9452d5b2015-07-21 10:00:56 +0200545static struct dmar_domain* get_iommu_domain(struct intel_iommu *iommu, u16 did)
546{
Joerg Roedel8bf47812015-07-21 10:41:21 +0200547 struct dmar_domain **domains;
548 int idx = did >> 8;
549
550 domains = iommu->domains[idx];
551 if (!domains)
552 return NULL;
553
554 return domains[did & 0xff];
Joerg Roedel9452d5b2015-07-21 10:00:56 +0200555}
556
557static void set_iommu_domain(struct intel_iommu *iommu, u16 did,
558 struct dmar_domain *domain)
559{
Joerg Roedel8bf47812015-07-21 10:41:21 +0200560 struct dmar_domain **domains;
561 int idx = did >> 8;
562
563 if (!iommu->domains[idx]) {
564 size_t size = 256 * sizeof(struct dmar_domain *);
565 iommu->domains[idx] = kzalloc(size, GFP_ATOMIC);
566 }
567
568 domains = iommu->domains[idx];
569 if (WARN_ON(!domains))
570 return;
571 else
572 domains[did & 0xff] = domain;
Joerg Roedel9452d5b2015-07-21 10:00:56 +0200573}
574
Lu Baolu9ddbfb42018-07-14 15:46:57 +0800575void *alloc_pgtable_page(int node)
Keshavamurthy, Anil Seb3fa7c2007-10-21 16:41:52 -0700576{
Suresh Siddha4c923d42009-10-02 11:01:24 -0700577 struct page *page;
578 void *vaddr = NULL;
Keshavamurthy, Anil Seb3fa7c2007-10-21 16:41:52 -0700579
Suresh Siddha4c923d42009-10-02 11:01:24 -0700580 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
581 if (page)
582 vaddr = page_address(page);
Keshavamurthy, Anil Seb3fa7c2007-10-21 16:41:52 -0700583 return vaddr;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700584}
585
Lu Baolu9ddbfb42018-07-14 15:46:57 +0800586void free_pgtable_page(void *vaddr)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700587{
588 free_page((unsigned long)vaddr);
589}
590
591static inline void *alloc_domain_mem(void)
592{
KOSAKI Motohiro354bb652009-11-17 16:21:09 +0900593 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700594}
595
Kay, Allen M38717942008-09-09 18:37:29 +0300596static void free_domain_mem(void *vaddr)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700597{
598 kmem_cache_free(iommu_domain_cache, vaddr);
599}
600
601static inline void * alloc_devinfo_mem(void)
602{
KOSAKI Motohiro354bb652009-11-17 16:21:09 +0900603 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700604}
605
606static inline void free_devinfo_mem(void *vaddr)
607{
608 kmem_cache_free(iommu_devinfo_cache, vaddr);
609}
610
Jiang Liuab8dfe22014-07-11 14:19:27 +0800611static inline int domain_type_is_vm(struct dmar_domain *domain)
612{
613 return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
614}
615
Joerg Roedel28ccce02015-07-21 14:45:31 +0200616static inline int domain_type_is_si(struct dmar_domain *domain)
617{
618 return domain->flags & DOMAIN_FLAG_STATIC_IDENTITY;
619}
620
Jiang Liuab8dfe22014-07-11 14:19:27 +0800621static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
622{
623 return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
624 DOMAIN_FLAG_STATIC_IDENTITY);
625}
Weidong Han1b573682008-12-08 15:34:06 +0800626
Jiang Liu162d1b12014-07-11 14:19:35 +0800627static inline int domain_pfn_supported(struct dmar_domain *domain,
628 unsigned long pfn)
629{
630 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
631
632 return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
633}
634
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -0700635static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
Weidong Han1b573682008-12-08 15:34:06 +0800636{
637 unsigned long sagaw;
638 int agaw = -1;
639
640 sagaw = cap_sagaw(iommu->cap);
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -0700641 for (agaw = width_to_agaw(max_gaw);
Weidong Han1b573682008-12-08 15:34:06 +0800642 agaw >= 0; agaw--) {
643 if (test_bit(agaw, &sagaw))
644 break;
645 }
646
647 return agaw;
648}
649
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -0700650/*
651 * Calculate max SAGAW for each iommu.
652 */
653int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
654{
655 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
656}
657
658/*
659 * calculate agaw for each iommu.
660 * "SAGAW" may be different across iommus, use a default agaw, and
661 * get a supported less agaw for iommus that don't support the default agaw.
662 */
663int iommu_calculate_agaw(struct intel_iommu *iommu)
664{
665 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
666}
667
Fenghua Yu2c2e2c32009-06-19 13:47:29 -0700668/* This functionin only returns single iommu in a domain */
Lu Baolu9ddbfb42018-07-14 15:46:57 +0800669struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
Weidong Han8c11e792008-12-08 15:29:22 +0800670{
671 int iommu_id;
672
Fenghua Yu2c2e2c32009-06-19 13:47:29 -0700673 /* si_domain and vm domain should not get here. */
Jiang Liuab8dfe22014-07-11 14:19:27 +0800674 BUG_ON(domain_type_is_vm_or_si(domain));
Joerg Roedel29a27712015-07-21 17:17:12 +0200675 for_each_domain_iommu(iommu_id, domain)
676 break;
677
Weidong Han8c11e792008-12-08 15:29:22 +0800678 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
679 return NULL;
680
681 return g_iommus[iommu_id];
682}
683
Weidong Han8e6040972008-12-08 15:49:06 +0800684static void domain_update_iommu_coherency(struct dmar_domain *domain)
685{
David Woodhoused0501962014-03-11 17:10:29 -0700686 struct dmar_drhd_unit *drhd;
687 struct intel_iommu *iommu;
Quentin Lambert2f119c72015-02-06 10:59:53 +0100688 bool found = false;
689 int i;
Weidong Han8e6040972008-12-08 15:49:06 +0800690
David Woodhoused0501962014-03-11 17:10:29 -0700691 domain->iommu_coherency = 1;
Weidong Han8e6040972008-12-08 15:49:06 +0800692
Joerg Roedel29a27712015-07-21 17:17:12 +0200693 for_each_domain_iommu(i, domain) {
Quentin Lambert2f119c72015-02-06 10:59:53 +0100694 found = true;
Weidong Han8e6040972008-12-08 15:49:06 +0800695 if (!ecap_coherent(g_iommus[i]->ecap)) {
696 domain->iommu_coherency = 0;
697 break;
698 }
Weidong Han8e6040972008-12-08 15:49:06 +0800699 }
David Woodhoused0501962014-03-11 17:10:29 -0700700 if (found)
701 return;
702
703 /* No hardware attached; use lowest common denominator */
704 rcu_read_lock();
705 for_each_active_iommu(iommu, drhd) {
706 if (!ecap_coherent(iommu->ecap)) {
707 domain->iommu_coherency = 0;
708 break;
709 }
710 }
711 rcu_read_unlock();
Weidong Han8e6040972008-12-08 15:49:06 +0800712}
713
Jiang Liu161f6932014-07-11 14:19:37 +0800714static int domain_update_iommu_snooping(struct intel_iommu *skip)
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100715{
Allen Kay8140a952011-10-14 12:32:17 -0700716 struct dmar_drhd_unit *drhd;
Jiang Liu161f6932014-07-11 14:19:37 +0800717 struct intel_iommu *iommu;
718 int ret = 1;
719
720 rcu_read_lock();
721 for_each_active_iommu(iommu, drhd) {
722 if (iommu != skip) {
723 if (!ecap_sc_support(iommu->ecap)) {
724 ret = 0;
725 break;
726 }
727 }
728 }
729 rcu_read_unlock();
730
731 return ret;
732}
733
734static int domain_update_iommu_superpage(struct intel_iommu *skip)
735{
736 struct dmar_drhd_unit *drhd;
737 struct intel_iommu *iommu;
Allen Kay8140a952011-10-14 12:32:17 -0700738 int mask = 0xf;
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100739
740 if (!intel_iommu_superpage) {
Jiang Liu161f6932014-07-11 14:19:37 +0800741 return 0;
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100742 }
743
Allen Kay8140a952011-10-14 12:32:17 -0700744 /* set iommu_superpage to the smallest common denominator */
Jiang Liu0e242612014-02-19 14:07:34 +0800745 rcu_read_lock();
Allen Kay8140a952011-10-14 12:32:17 -0700746 for_each_active_iommu(iommu, drhd) {
Jiang Liu161f6932014-07-11 14:19:37 +0800747 if (iommu != skip) {
748 mask &= cap_super_page_val(iommu->cap);
749 if (!mask)
750 break;
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100751 }
752 }
Jiang Liu0e242612014-02-19 14:07:34 +0800753 rcu_read_unlock();
754
Jiang Liu161f6932014-07-11 14:19:37 +0800755 return fls(mask);
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100756}
757
Sheng Yang58c610b2009-03-18 15:33:05 +0800758/* Some capabilities may be different across iommus */
759static void domain_update_iommu_cap(struct dmar_domain *domain)
760{
761 domain_update_iommu_coherency(domain);
Jiang Liu161f6932014-07-11 14:19:37 +0800762 domain->iommu_snooping = domain_update_iommu_snooping(NULL);
763 domain->iommu_superpage = domain_update_iommu_superpage(NULL);
Sheng Yang58c610b2009-03-18 15:33:05 +0800764}
765
Sohil Mehta26b86092018-09-11 17:11:36 -0700766struct context_entry *iommu_context_addr(struct intel_iommu *iommu, u8 bus,
767 u8 devfn, int alloc)
David Woodhouse03ecc322015-02-13 14:35:21 +0000768{
769 struct root_entry *root = &iommu->root_entry[bus];
770 struct context_entry *context;
771 u64 *entry;
772
Joerg Roedel4df4eab2015-08-25 10:54:28 +0200773 entry = &root->lo;
David Woodhousec83b2f22015-06-12 10:15:49 +0100774 if (ecs_enabled(iommu)) {
David Woodhouse03ecc322015-02-13 14:35:21 +0000775 if (devfn >= 0x80) {
776 devfn -= 0x80;
777 entry = &root->hi;
778 }
779 devfn *= 2;
780 }
David Woodhouse03ecc322015-02-13 14:35:21 +0000781 if (*entry & 1)
782 context = phys_to_virt(*entry & VTD_PAGE_MASK);
783 else {
784 unsigned long phy_addr;
785 if (!alloc)
786 return NULL;
787
788 context = alloc_pgtable_page(iommu->node);
789 if (!context)
790 return NULL;
791
792 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
793 phy_addr = virt_to_phys((void *)context);
794 *entry = phy_addr | 1;
795 __iommu_flush_cache(iommu, entry, sizeof(*entry));
796 }
797 return &context[devfn];
798}
799
David Woodhouse4ed6a542015-05-11 14:59:20 +0100800static int iommu_dummy(struct device *dev)
801{
802 return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
803}
804
David Woodhouse156baca2014-03-09 14:00:57 -0700805static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
Weidong Hanc7151a82008-12-08 22:51:37 +0800806{
807 struct dmar_drhd_unit *drhd = NULL;
Jiang Liub683b232014-02-19 14:07:32 +0800808 struct intel_iommu *iommu;
David Woodhouse156baca2014-03-09 14:00:57 -0700809 struct device *tmp;
810 struct pci_dev *ptmp, *pdev = NULL;
Yijing Wangaa4d0662014-05-26 20:14:06 +0800811 u16 segment = 0;
Weidong Hanc7151a82008-12-08 22:51:37 +0800812 int i;
813
David Woodhouse4ed6a542015-05-11 14:59:20 +0100814 if (iommu_dummy(dev))
815 return NULL;
816
David Woodhouse156baca2014-03-09 14:00:57 -0700817 if (dev_is_pci(dev)) {
Ashok Raj1c387182016-10-21 15:32:05 -0700818 struct pci_dev *pf_pdev;
819
David Woodhouse156baca2014-03-09 14:00:57 -0700820 pdev = to_pci_dev(dev);
Jon Derrick5823e332017-08-30 15:05:59 -0600821
822#ifdef CONFIG_X86
823 /* VMD child devices currently cannot be handled individually */
824 if (is_vmd(pdev->bus))
825 return NULL;
826#endif
827
Ashok Raj1c387182016-10-21 15:32:05 -0700828 /* VFs aren't listed in scope tables; we need to look up
829 * the PF instead to find the IOMMU. */
830 pf_pdev = pci_physfn(pdev);
831 dev = &pf_pdev->dev;
David Woodhouse156baca2014-03-09 14:00:57 -0700832 segment = pci_domain_nr(pdev->bus);
Rafael J. Wysockica5b74d2015-03-16 23:49:08 +0100833 } else if (has_acpi_companion(dev))
David Woodhouse156baca2014-03-09 14:00:57 -0700834 dev = &ACPI_COMPANION(dev)->dev;
835
Jiang Liu0e242612014-02-19 14:07:34 +0800836 rcu_read_lock();
Jiang Liub683b232014-02-19 14:07:32 +0800837 for_each_active_iommu(iommu, drhd) {
David Woodhouse156baca2014-03-09 14:00:57 -0700838 if (pdev && segment != drhd->segment)
David Woodhouse276dbf992009-04-04 01:45:37 +0100839 continue;
Weidong Hanc7151a82008-12-08 22:51:37 +0800840
Jiang Liub683b232014-02-19 14:07:32 +0800841 for_each_active_dev_scope(drhd->devices,
David Woodhouse156baca2014-03-09 14:00:57 -0700842 drhd->devices_cnt, i, tmp) {
843 if (tmp == dev) {
Ashok Raj1c387182016-10-21 15:32:05 -0700844 /* For a VF use its original BDF# not that of the PF
845 * which we used for the IOMMU lookup. Strictly speaking
846 * we could do this for all PCI devices; we only need to
847 * get the BDF# from the scope table for ACPI matches. */
Koos Vriezen5003ae12017-03-01 21:02:50 +0100848 if (pdev && pdev->is_virtfn)
Ashok Raj1c387182016-10-21 15:32:05 -0700849 goto got_pdev;
850
David Woodhouse156baca2014-03-09 14:00:57 -0700851 *bus = drhd->devices[i].bus;
852 *devfn = drhd->devices[i].devfn;
853 goto out;
854 }
855
856 if (!pdev || !dev_is_pci(tmp))
David Woodhouse832bd852014-03-07 15:08:36 +0000857 continue;
David Woodhouse156baca2014-03-09 14:00:57 -0700858
859 ptmp = to_pci_dev(tmp);
860 if (ptmp->subordinate &&
861 ptmp->subordinate->number <= pdev->bus->number &&
862 ptmp->subordinate->busn_res.end >= pdev->bus->number)
863 goto got_pdev;
David Woodhouse924b6232009-04-04 00:39:25 +0100864 }
Weidong Hanc7151a82008-12-08 22:51:37 +0800865
David Woodhouse156baca2014-03-09 14:00:57 -0700866 if (pdev && drhd->include_all) {
867 got_pdev:
868 *bus = pdev->bus->number;
869 *devfn = pdev->devfn;
Jiang Liub683b232014-02-19 14:07:32 +0800870 goto out;
David Woodhouse156baca2014-03-09 14:00:57 -0700871 }
Weidong Hanc7151a82008-12-08 22:51:37 +0800872 }
Jiang Liub683b232014-02-19 14:07:32 +0800873 iommu = NULL;
David Woodhouse156baca2014-03-09 14:00:57 -0700874 out:
Jiang Liu0e242612014-02-19 14:07:34 +0800875 rcu_read_unlock();
Weidong Hanc7151a82008-12-08 22:51:37 +0800876
Jiang Liub683b232014-02-19 14:07:32 +0800877 return iommu;
Weidong Hanc7151a82008-12-08 22:51:37 +0800878}
879
Weidong Han5331fe62008-12-08 23:00:00 +0800880static void domain_flush_cache(struct dmar_domain *domain,
881 void *addr, int size)
882{
883 if (!domain->iommu_coherency)
884 clflush_cache_range(addr, size);
885}
886
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700887static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
888{
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700889 struct context_entry *context;
David Woodhouse03ecc322015-02-13 14:35:21 +0000890 int ret = 0;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700891 unsigned long flags;
892
893 spin_lock_irqsave(&iommu->lock, flags);
David Woodhouse03ecc322015-02-13 14:35:21 +0000894 context = iommu_context_addr(iommu, bus, devfn, 0);
895 if (context)
896 ret = context_present(context);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700897 spin_unlock_irqrestore(&iommu->lock, flags);
898 return ret;
899}
900
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700901static void free_context_table(struct intel_iommu *iommu)
902{
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700903 int i;
904 unsigned long flags;
905 struct context_entry *context;
906
907 spin_lock_irqsave(&iommu->lock, flags);
908 if (!iommu->root_entry) {
909 goto out;
910 }
911 for (i = 0; i < ROOT_ENTRY_NR; i++) {
David Woodhouse03ecc322015-02-13 14:35:21 +0000912 context = iommu_context_addr(iommu, i, 0, 0);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700913 if (context)
914 free_pgtable_page(context);
David Woodhouse03ecc322015-02-13 14:35:21 +0000915
David Woodhousec83b2f22015-06-12 10:15:49 +0100916 if (!ecs_enabled(iommu))
David Woodhouse03ecc322015-02-13 14:35:21 +0000917 continue;
918
919 context = iommu_context_addr(iommu, i, 0x80, 0);
920 if (context)
921 free_pgtable_page(context);
922
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700923 }
924 free_pgtable_page(iommu->root_entry);
925 iommu->root_entry = NULL;
926out:
927 spin_unlock_irqrestore(&iommu->lock, flags);
928}
929
David Woodhouseb026fd22009-06-28 10:37:25 +0100930static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
David Woodhouse5cf0a762014-03-19 16:07:49 +0000931 unsigned long pfn, int *target_level)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700932{
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700933 struct dma_pte *parent, *pte = NULL;
934 int level = agaw_to_level(domain->agaw);
Allen Kay4399c8b2011-10-14 12:32:46 -0700935 int offset;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700936
937 BUG_ON(!domain->pgd);
Julian Stecklinaf9423602013-10-09 10:03:52 +0200938
Jiang Liu162d1b12014-07-11 14:19:35 +0800939 if (!domain_pfn_supported(domain, pfn))
Julian Stecklinaf9423602013-10-09 10:03:52 +0200940 /* Address beyond IOMMU's addressing capabilities. */
941 return NULL;
942
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700943 parent = domain->pgd;
944
David Woodhouse5cf0a762014-03-19 16:07:49 +0000945 while (1) {
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700946 void *tmp_page;
947
David Woodhouseb026fd22009-06-28 10:37:25 +0100948 offset = pfn_level_offset(pfn, level);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700949 pte = &parent[offset];
David Woodhouse5cf0a762014-03-19 16:07:49 +0000950 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100951 break;
David Woodhouse5cf0a762014-03-19 16:07:49 +0000952 if (level == *target_level)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700953 break;
954
Mark McLoughlin19c239c2008-11-21 16:56:53 +0000955 if (!dma_pte_present(pte)) {
David Woodhousec85994e2009-07-01 19:21:24 +0100956 uint64_t pteval;
957
Suresh Siddha4c923d42009-10-02 11:01:24 -0700958 tmp_page = alloc_pgtable_page(domain->nid);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700959
David Woodhouse206a73c2009-07-01 19:30:28 +0100960 if (!tmp_page)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700961 return NULL;
David Woodhouse206a73c2009-07-01 19:30:28 +0100962
David Woodhousec85994e2009-07-01 19:21:24 +0100963 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
Benjamin LaHaise64de5af2009-09-16 21:05:55 -0400964 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
Yijing Wangeffad4b2014-05-26 20:13:47 +0800965 if (cmpxchg64(&pte->val, 0ULL, pteval))
David Woodhousec85994e2009-07-01 19:21:24 +0100966 /* Someone else set it while we were thinking; use theirs. */
967 free_pgtable_page(tmp_page);
Yijing Wangeffad4b2014-05-26 20:13:47 +0800968 else
David Woodhousec85994e2009-07-01 19:21:24 +0100969 domain_flush_cache(domain, pte, sizeof(*pte));
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700970 }
David Woodhouse5cf0a762014-03-19 16:07:49 +0000971 if (level == 1)
972 break;
973
Mark McLoughlin19c239c2008-11-21 16:56:53 +0000974 parent = phys_to_virt(dma_pte_addr(pte));
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700975 level--;
976 }
977
David Woodhouse5cf0a762014-03-19 16:07:49 +0000978 if (!*target_level)
979 *target_level = level;
980
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700981 return pte;
982}
983
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100984
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700985/* return address's pte at specific level */
David Woodhouse90dcfb52009-06-27 17:14:59 +0100986static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
987 unsigned long pfn,
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100988 int level, int *large_page)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700989{
990 struct dma_pte *parent, *pte = NULL;
991 int total = agaw_to_level(domain->agaw);
992 int offset;
993
994 parent = domain->pgd;
995 while (level <= total) {
David Woodhouse90dcfb52009-06-27 17:14:59 +0100996 offset = pfn_level_offset(pfn, total);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700997 pte = &parent[offset];
998 if (level == total)
999 return pte;
1000
Youquan Song6dd9a7c2011-05-25 19:13:49 +01001001 if (!dma_pte_present(pte)) {
1002 *large_page = total;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001003 break;
Youquan Song6dd9a7c2011-05-25 19:13:49 +01001004 }
1005
Yijing Wange16922a2014-05-20 20:37:51 +08001006 if (dma_pte_superpage(pte)) {
Youquan Song6dd9a7c2011-05-25 19:13:49 +01001007 *large_page = total;
1008 return pte;
1009 }
1010
Mark McLoughlin19c239c2008-11-21 16:56:53 +00001011 parent = phys_to_virt(dma_pte_addr(pte));
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001012 total--;
1013 }
1014 return NULL;
1015}
1016
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001017/* clear last level pte, a tlb flush should be followed */
David Woodhouse5cf0a762014-03-19 16:07:49 +00001018static void dma_pte_clear_range(struct dmar_domain *domain,
David Woodhouse595badf52009-06-27 22:09:11 +01001019 unsigned long start_pfn,
1020 unsigned long last_pfn)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001021{
Youquan Song6dd9a7c2011-05-25 19:13:49 +01001022 unsigned int large_page = 1;
David Woodhouse310a5ab2009-06-28 18:52:20 +01001023 struct dma_pte *first_pte, *pte;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001024
Jiang Liu162d1b12014-07-11 14:19:35 +08001025 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1026 BUG_ON(!domain_pfn_supported(domain, last_pfn));
David Woodhouse59c36282009-09-19 07:36:28 -07001027 BUG_ON(start_pfn > last_pfn);
David Woodhouse66eae842009-06-27 19:00:32 +01001028
David Woodhouse04b18e62009-06-27 19:15:01 +01001029 /* we don't need lock here; nobody else touches the iova range */
David Woodhouse59c36282009-09-19 07:36:28 -07001030 do {
Youquan Song6dd9a7c2011-05-25 19:13:49 +01001031 large_page = 1;
1032 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
David Woodhouse310a5ab2009-06-28 18:52:20 +01001033 if (!pte) {
Youquan Song6dd9a7c2011-05-25 19:13:49 +01001034 start_pfn = align_to_level(start_pfn + 1, large_page + 1);
David Woodhouse310a5ab2009-06-28 18:52:20 +01001035 continue;
1036 }
Youquan Song6dd9a7c2011-05-25 19:13:49 +01001037 do {
David Woodhouse310a5ab2009-06-28 18:52:20 +01001038 dma_clear_pte(pte);
Youquan Song6dd9a7c2011-05-25 19:13:49 +01001039 start_pfn += lvl_to_nr_pages(large_page);
David Woodhouse310a5ab2009-06-28 18:52:20 +01001040 pte++;
David Woodhouse75e6bf92009-07-02 11:21:16 +01001041 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
1042
David Woodhouse310a5ab2009-06-28 18:52:20 +01001043 domain_flush_cache(domain, first_pte,
1044 (void *)pte - (void *)first_pte);
David Woodhouse59c36282009-09-19 07:36:28 -07001045
1046 } while (start_pfn && start_pfn <= last_pfn);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001047}
1048
Alex Williamson3269ee02013-06-15 10:27:19 -06001049static void dma_pte_free_level(struct dmar_domain *domain, int level,
David Dillowbc24c572017-06-28 19:42:23 -07001050 int retain_level, struct dma_pte *pte,
1051 unsigned long pfn, unsigned long start_pfn,
1052 unsigned long last_pfn)
Alex Williamson3269ee02013-06-15 10:27:19 -06001053{
1054 pfn = max(start_pfn, pfn);
1055 pte = &pte[pfn_level_offset(pfn, level)];
1056
1057 do {
1058 unsigned long level_pfn;
1059 struct dma_pte *level_pte;
1060
1061 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
1062 goto next;
1063
David Dillowf7116e12017-01-30 19:11:11 -08001064 level_pfn = pfn & level_mask(level);
Alex Williamson3269ee02013-06-15 10:27:19 -06001065 level_pte = phys_to_virt(dma_pte_addr(pte));
1066
David Dillowbc24c572017-06-28 19:42:23 -07001067 if (level > 2) {
1068 dma_pte_free_level(domain, level - 1, retain_level,
1069 level_pte, level_pfn, start_pfn,
1070 last_pfn);
1071 }
Alex Williamson3269ee02013-06-15 10:27:19 -06001072
David Dillowbc24c572017-06-28 19:42:23 -07001073 /*
1074 * Free the page table if we're below the level we want to
1075 * retain and the range covers the entire table.
1076 */
1077 if (level < retain_level && !(start_pfn > level_pfn ||
Alex Williamson08336fd2014-01-21 15:48:18 -08001078 last_pfn < level_pfn + level_size(level) - 1)) {
Alex Williamson3269ee02013-06-15 10:27:19 -06001079 dma_clear_pte(pte);
1080 domain_flush_cache(domain, pte, sizeof(*pte));
1081 free_pgtable_page(level_pte);
1082 }
1083next:
1084 pfn += level_size(level);
1085 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1086}
1087
David Dillowbc24c572017-06-28 19:42:23 -07001088/*
1089 * clear last level (leaf) ptes and free page table pages below the
1090 * level we wish to keep intact.
1091 */
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001092static void dma_pte_free_pagetable(struct dmar_domain *domain,
David Woodhoused794dc92009-06-28 00:27:49 +01001093 unsigned long start_pfn,
David Dillowbc24c572017-06-28 19:42:23 -07001094 unsigned long last_pfn,
1095 int retain_level)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001096{
Jiang Liu162d1b12014-07-11 14:19:35 +08001097 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1098 BUG_ON(!domain_pfn_supported(domain, last_pfn));
David Woodhouse59c36282009-09-19 07:36:28 -07001099 BUG_ON(start_pfn > last_pfn);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001100
Jiang Liud41a4ad2014-07-11 14:19:34 +08001101 dma_pte_clear_range(domain, start_pfn, last_pfn);
1102
David Woodhousef3a0a522009-06-30 03:40:07 +01001103 /* We don't need lock here; nobody else touches the iova range */
David Dillowbc24c572017-06-28 19:42:23 -07001104 dma_pte_free_level(domain, agaw_to_level(domain->agaw), retain_level,
Alex Williamson3269ee02013-06-15 10:27:19 -06001105 domain->pgd, 0, start_pfn, last_pfn);
David Woodhouse6660c632009-06-27 22:41:00 +01001106
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001107 /* free pgd */
David Woodhoused794dc92009-06-28 00:27:49 +01001108 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001109 free_pgtable_page(domain->pgd);
1110 domain->pgd = NULL;
1111 }
1112}
1113
David Woodhouseea8ea462014-03-05 17:09:32 +00001114/* When a page at a given level is being unlinked from its parent, we don't
1115 need to *modify* it at all. All we need to do is make a list of all the
1116 pages which can be freed just as soon as we've flushed the IOTLB and we
1117 know the hardware page-walk will no longer touch them.
1118 The 'pte' argument is the *parent* PTE, pointing to the page that is to
1119 be freed. */
1120static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1121 int level, struct dma_pte *pte,
1122 struct page *freelist)
1123{
1124 struct page *pg;
1125
1126 pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1127 pg->freelist = freelist;
1128 freelist = pg;
1129
1130 if (level == 1)
1131 return freelist;
1132
Jiang Liuadeb2592014-04-09 10:20:39 +08001133 pte = page_address(pg);
1134 do {
David Woodhouseea8ea462014-03-05 17:09:32 +00001135 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1136 freelist = dma_pte_list_pagetables(domain, level - 1,
1137 pte, freelist);
Jiang Liuadeb2592014-04-09 10:20:39 +08001138 pte++;
1139 } while (!first_pte_in_page(pte));
David Woodhouseea8ea462014-03-05 17:09:32 +00001140
1141 return freelist;
1142}
1143
1144static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1145 struct dma_pte *pte, unsigned long pfn,
1146 unsigned long start_pfn,
1147 unsigned long last_pfn,
1148 struct page *freelist)
1149{
1150 struct dma_pte *first_pte = NULL, *last_pte = NULL;
1151
1152 pfn = max(start_pfn, pfn);
1153 pte = &pte[pfn_level_offset(pfn, level)];
1154
1155 do {
1156 unsigned long level_pfn;
1157
1158 if (!dma_pte_present(pte))
1159 goto next;
1160
1161 level_pfn = pfn & level_mask(level);
1162
1163 /* If range covers entire pagetable, free it */
1164 if (start_pfn <= level_pfn &&
1165 last_pfn >= level_pfn + level_size(level) - 1) {
1166 /* These suborbinate page tables are going away entirely. Don't
1167 bother to clear them; we're just going to *free* them. */
1168 if (level > 1 && !dma_pte_superpage(pte))
1169 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1170
1171 dma_clear_pte(pte);
1172 if (!first_pte)
1173 first_pte = pte;
1174 last_pte = pte;
1175 } else if (level > 1) {
1176 /* Recurse down into a level that isn't *entirely* obsolete */
1177 freelist = dma_pte_clear_level(domain, level - 1,
1178 phys_to_virt(dma_pte_addr(pte)),
1179 level_pfn, start_pfn, last_pfn,
1180 freelist);
1181 }
1182next:
1183 pfn += level_size(level);
1184 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1185
1186 if (first_pte)
1187 domain_flush_cache(domain, first_pte,
1188 (void *)++last_pte - (void *)first_pte);
1189
1190 return freelist;
1191}
1192
1193/* We can't just free the pages because the IOMMU may still be walking
1194 the page tables, and may have cached the intermediate levels. The
1195 pages can only be freed after the IOTLB flush has been done. */
Joerg Roedelb6904202015-08-13 11:32:18 +02001196static struct page *domain_unmap(struct dmar_domain *domain,
1197 unsigned long start_pfn,
1198 unsigned long last_pfn)
David Woodhouseea8ea462014-03-05 17:09:32 +00001199{
David Woodhouseea8ea462014-03-05 17:09:32 +00001200 struct page *freelist = NULL;
1201
Jiang Liu162d1b12014-07-11 14:19:35 +08001202 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1203 BUG_ON(!domain_pfn_supported(domain, last_pfn));
David Woodhouseea8ea462014-03-05 17:09:32 +00001204 BUG_ON(start_pfn > last_pfn);
1205
1206 /* we don't need lock here; nobody else touches the iova range */
1207 freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1208 domain->pgd, 0, start_pfn, last_pfn, NULL);
1209
1210 /* free pgd */
1211 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1212 struct page *pgd_page = virt_to_page(domain->pgd);
1213 pgd_page->freelist = freelist;
1214 freelist = pgd_page;
1215
1216 domain->pgd = NULL;
1217 }
1218
1219 return freelist;
1220}
1221
Joerg Roedelb6904202015-08-13 11:32:18 +02001222static void dma_free_pagelist(struct page *freelist)
David Woodhouseea8ea462014-03-05 17:09:32 +00001223{
1224 struct page *pg;
1225
1226 while ((pg = freelist)) {
1227 freelist = pg->freelist;
1228 free_pgtable_page(page_address(pg));
1229 }
1230}
1231
Joerg Roedel13cf0172017-08-11 11:40:10 +02001232static void iova_entry_free(unsigned long data)
1233{
1234 struct page *freelist = (struct page *)data;
1235
1236 dma_free_pagelist(freelist);
1237}
1238
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001239/* iommu handling */
1240static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1241{
1242 struct root_entry *root;
1243 unsigned long flags;
1244
Suresh Siddha4c923d42009-10-02 11:01:24 -07001245 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
Jiang Liuffebeb42014-11-09 22:48:02 +08001246 if (!root) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02001247 pr_err("Allocating root entry for %s failed\n",
Jiang Liuffebeb42014-11-09 22:48:02 +08001248 iommu->name);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001249 return -ENOMEM;
Jiang Liuffebeb42014-11-09 22:48:02 +08001250 }
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001251
Fenghua Yu5b6985c2008-10-16 18:02:32 -07001252 __iommu_flush_cache(iommu, root, ROOT_SIZE);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001253
1254 spin_lock_irqsave(&iommu->lock, flags);
1255 iommu->root_entry = root;
1256 spin_unlock_irqrestore(&iommu->lock, flags);
1257
1258 return 0;
1259}
1260
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001261static void iommu_set_root_entry(struct intel_iommu *iommu)
1262{
David Woodhouse03ecc322015-02-13 14:35:21 +00001263 u64 addr;
David Woodhousec416daa2009-05-10 20:30:58 +01001264 u32 sts;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001265 unsigned long flag;
1266
David Woodhouse03ecc322015-02-13 14:35:21 +00001267 addr = virt_to_phys(iommu->root_entry);
David Woodhousec83b2f22015-06-12 10:15:49 +01001268 if (ecs_enabled(iommu))
David Woodhouse03ecc322015-02-13 14:35:21 +00001269 addr |= DMA_RTADDR_RTT;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001270
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001271 raw_spin_lock_irqsave(&iommu->register_lock, flag);
David Woodhouse03ecc322015-02-13 14:35:21 +00001272 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001273
David Woodhousec416daa2009-05-10 20:30:58 +01001274 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001275
1276 /* Make sure hardware complete it */
1277 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
David Woodhousec416daa2009-05-10 20:30:58 +01001278 readl, (sts & DMA_GSTS_RTPS), sts);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001279
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001280 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001281}
1282
1283static void iommu_flush_write_buffer(struct intel_iommu *iommu)
1284{
1285 u32 val;
1286 unsigned long flag;
1287
David Woodhouse9af88142009-02-13 23:18:03 +00001288 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001289 return;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001290
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001291 raw_spin_lock_irqsave(&iommu->register_lock, flag);
David Woodhouse462b60f2009-05-10 20:18:18 +01001292 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001293
1294 /* Make sure hardware complete it */
1295 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
David Woodhousec416daa2009-05-10 20:30:58 +01001296 readl, (!(val & DMA_GSTS_WBFS)), val);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001297
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001298 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001299}
1300
1301/* return value determine if we need a write buffer flush */
David Woodhouse4c25a2c2009-05-10 17:16:06 +01001302static void __iommu_flush_context(struct intel_iommu *iommu,
1303 u16 did, u16 source_id, u8 function_mask,
1304 u64 type)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001305{
1306 u64 val = 0;
1307 unsigned long flag;
1308
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001309 switch (type) {
1310 case DMA_CCMD_GLOBAL_INVL:
1311 val = DMA_CCMD_GLOBAL_INVL;
1312 break;
1313 case DMA_CCMD_DOMAIN_INVL:
1314 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1315 break;
1316 case DMA_CCMD_DEVICE_INVL:
1317 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1318 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1319 break;
1320 default:
1321 BUG();
1322 }
1323 val |= DMA_CCMD_ICC;
1324
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001325 raw_spin_lock_irqsave(&iommu->register_lock, flag);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001326 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1327
1328 /* Make sure hardware complete it */
1329 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1330 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1331
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001332 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001333}
1334
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001335/* return value determine if we need a write buffer flush */
David Woodhouse1f0ef2a2009-05-10 19:58:49 +01001336static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1337 u64 addr, unsigned int size_order, u64 type)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001338{
1339 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1340 u64 val = 0, val_iva = 0;
1341 unsigned long flag;
1342
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001343 switch (type) {
1344 case DMA_TLB_GLOBAL_FLUSH:
1345 /* global flush doesn't need set IVA_REG */
1346 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1347 break;
1348 case DMA_TLB_DSI_FLUSH:
1349 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1350 break;
1351 case DMA_TLB_PSI_FLUSH:
1352 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
David Woodhouseea8ea462014-03-05 17:09:32 +00001353 /* IH bit is passed in as part of address */
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001354 val_iva = size_order | addr;
1355 break;
1356 default:
1357 BUG();
1358 }
1359 /* Note: set drain read/write */
1360#if 0
1361 /*
1362 * This is probably to be super secure.. Looks like we can
1363 * ignore it without any impact.
1364 */
1365 if (cap_read_drain(iommu->cap))
1366 val |= DMA_TLB_READ_DRAIN;
1367#endif
1368 if (cap_write_drain(iommu->cap))
1369 val |= DMA_TLB_WRITE_DRAIN;
1370
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001371 raw_spin_lock_irqsave(&iommu->register_lock, flag);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001372 /* Note: Only uses first TLB reg currently */
1373 if (val_iva)
1374 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1375 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1376
1377 /* Make sure hardware complete it */
1378 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1379 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1380
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001381 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001382
1383 /* check IOTLB invalidation granularity */
1384 if (DMA_TLB_IAIG(val) == 0)
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02001385 pr_err("Flush IOTLB failed\n");
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001386 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02001387 pr_debug("TLB flush request %Lx, actual %Lx\n",
Fenghua Yu5b6985c2008-10-16 18:02:32 -07001388 (unsigned long long)DMA_TLB_IIRG(type),
1389 (unsigned long long)DMA_TLB_IAIG(val));
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001390}
1391
David Woodhouse64ae8922014-03-09 12:52:30 -07001392static struct device_domain_info *
1393iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1394 u8 bus, u8 devfn)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001395{
Yu Zhao93a23a72009-05-18 13:51:37 +08001396 struct device_domain_info *info;
Yu Zhao93a23a72009-05-18 13:51:37 +08001397
Joerg Roedel55d94042015-07-22 16:50:40 +02001398 assert_spin_locked(&device_domain_lock);
1399
Yu Zhao93a23a72009-05-18 13:51:37 +08001400 if (!iommu->qi)
1401 return NULL;
1402
Yu Zhao93a23a72009-05-18 13:51:37 +08001403 list_for_each_entry(info, &domain->devices, link)
Jiang Liuc3b497c2014-07-11 14:19:25 +08001404 if (info->iommu == iommu && info->bus == bus &&
1405 info->devfn == devfn) {
David Woodhouseb16d0cb2015-10-12 14:17:37 +01001406 if (info->ats_supported && info->dev)
1407 return info;
Yu Zhao93a23a72009-05-18 13:51:37 +08001408 break;
1409 }
Yu Zhao93a23a72009-05-18 13:51:37 +08001410
David Woodhouseb16d0cb2015-10-12 14:17:37 +01001411 return NULL;
Yu Zhao93a23a72009-05-18 13:51:37 +08001412}
1413
Omer Peleg0824c592016-04-20 19:03:35 +03001414static void domain_update_iotlb(struct dmar_domain *domain)
1415{
1416 struct device_domain_info *info;
1417 bool has_iotlb_device = false;
1418
1419 assert_spin_locked(&device_domain_lock);
1420
1421 list_for_each_entry(info, &domain->devices, link) {
1422 struct pci_dev *pdev;
1423
1424 if (!info->dev || !dev_is_pci(info->dev))
1425 continue;
1426
1427 pdev = to_pci_dev(info->dev);
1428 if (pdev->ats_enabled) {
1429 has_iotlb_device = true;
1430 break;
1431 }
1432 }
1433
1434 domain->has_iotlb_device = has_iotlb_device;
1435}
1436
Yu Zhao93a23a72009-05-18 13:51:37 +08001437static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1438{
Bjorn Helgaasfb0cc3a2015-07-20 09:10:36 -05001439 struct pci_dev *pdev;
1440
Omer Peleg0824c592016-04-20 19:03:35 +03001441 assert_spin_locked(&device_domain_lock);
1442
David Woodhouse0bcb3e22014-03-06 17:12:03 +00001443 if (!info || !dev_is_pci(info->dev))
Yu Zhao93a23a72009-05-18 13:51:37 +08001444 return;
1445
Bjorn Helgaasfb0cc3a2015-07-20 09:10:36 -05001446 pdev = to_pci_dev(info->dev);
Jacob Pan1c48db42018-06-07 09:57:00 -07001447 /* For IOMMU that supports device IOTLB throttling (DIT), we assign
1448 * PFSID to the invalidation desc of a VF such that IOMMU HW can gauge
1449 * queue depth at PF level. If DIT is not set, PFSID will be treated as
1450 * reserved, which should be set to 0.
1451 */
1452 if (!ecap_dit(info->iommu->ecap))
1453 info->pfsid = 0;
1454 else {
1455 struct pci_dev *pf_pdev;
1456
1457 /* pdev will be returned if device is not a vf */
1458 pf_pdev = pci_physfn(pdev);
1459 info->pfsid = PCI_DEVID(pf_pdev->bus->number, pf_pdev->devfn);
1460 }
Bjorn Helgaasfb0cc3a2015-07-20 09:10:36 -05001461
David Woodhouseb16d0cb2015-10-12 14:17:37 +01001462#ifdef CONFIG_INTEL_IOMMU_SVM
1463 /* The PCIe spec, in its wisdom, declares that the behaviour of
1464 the device if you enable PASID support after ATS support is
1465 undefined. So always enable PASID support on devices which
1466 have it, even if we can't yet know if we're ever going to
1467 use it. */
1468 if (info->pasid_supported && !pci_enable_pasid(pdev, info->pasid_supported & ~1))
1469 info->pasid_enabled = 1;
1470
1471 if (info->pri_supported && !pci_reset_pri(pdev) && !pci_enable_pri(pdev, 32))
1472 info->pri_enabled = 1;
1473#endif
1474 if (info->ats_supported && !pci_enable_ats(pdev, VTD_PAGE_SHIFT)) {
1475 info->ats_enabled = 1;
Omer Peleg0824c592016-04-20 19:03:35 +03001476 domain_update_iotlb(info->domain);
David Woodhouseb16d0cb2015-10-12 14:17:37 +01001477 info->ats_qdep = pci_ats_queue_depth(pdev);
1478 }
Yu Zhao93a23a72009-05-18 13:51:37 +08001479}
1480
1481static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1482{
David Woodhouseb16d0cb2015-10-12 14:17:37 +01001483 struct pci_dev *pdev;
1484
Omer Peleg0824c592016-04-20 19:03:35 +03001485 assert_spin_locked(&device_domain_lock);
1486
Jeremy McNicollda972fb2016-01-14 21:33:06 -08001487 if (!dev_is_pci(info->dev))
Yu Zhao93a23a72009-05-18 13:51:37 +08001488 return;
1489
David Woodhouseb16d0cb2015-10-12 14:17:37 +01001490 pdev = to_pci_dev(info->dev);
1491
1492 if (info->ats_enabled) {
1493 pci_disable_ats(pdev);
1494 info->ats_enabled = 0;
Omer Peleg0824c592016-04-20 19:03:35 +03001495 domain_update_iotlb(info->domain);
David Woodhouseb16d0cb2015-10-12 14:17:37 +01001496 }
1497#ifdef CONFIG_INTEL_IOMMU_SVM
1498 if (info->pri_enabled) {
1499 pci_disable_pri(pdev);
1500 info->pri_enabled = 0;
1501 }
1502 if (info->pasid_enabled) {
1503 pci_disable_pasid(pdev);
1504 info->pasid_enabled = 0;
1505 }
1506#endif
Yu Zhao93a23a72009-05-18 13:51:37 +08001507}
1508
1509static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1510 u64 addr, unsigned mask)
1511{
1512 u16 sid, qdep;
1513 unsigned long flags;
1514 struct device_domain_info *info;
1515
Omer Peleg0824c592016-04-20 19:03:35 +03001516 if (!domain->has_iotlb_device)
1517 return;
1518
Yu Zhao93a23a72009-05-18 13:51:37 +08001519 spin_lock_irqsave(&device_domain_lock, flags);
1520 list_for_each_entry(info, &domain->devices, link) {
David Woodhouseb16d0cb2015-10-12 14:17:37 +01001521 if (!info->ats_enabled)
Yu Zhao93a23a72009-05-18 13:51:37 +08001522 continue;
1523
1524 sid = info->bus << 8 | info->devfn;
David Woodhouseb16d0cb2015-10-12 14:17:37 +01001525 qdep = info->ats_qdep;
Jacob Pan1c48db42018-06-07 09:57:00 -07001526 qi_flush_dev_iotlb(info->iommu, sid, info->pfsid,
1527 qdep, addr, mask);
Yu Zhao93a23a72009-05-18 13:51:37 +08001528 }
1529 spin_unlock_irqrestore(&device_domain_lock, flags);
1530}
1531
Joerg Roedela1ddcbe2015-07-21 15:20:32 +02001532static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
1533 struct dmar_domain *domain,
1534 unsigned long pfn, unsigned int pages,
1535 int ih, int map)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001536{
Yu Zhao9dd2fe82009-05-18 13:51:36 +08001537 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
David Woodhouse03d6a242009-06-28 15:33:46 +01001538 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
Joerg Roedela1ddcbe2015-07-21 15:20:32 +02001539 u16 did = domain->iommu_did[iommu->seq_id];
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001540
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001541 BUG_ON(pages == 0);
1542
David Woodhouseea8ea462014-03-05 17:09:32 +00001543 if (ih)
1544 ih = 1 << 6;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001545 /*
Yu Zhao9dd2fe82009-05-18 13:51:36 +08001546 * Fallback to domain selective flush if no PSI support or the size is
1547 * too big.
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001548 * PSI requires page size to be 2 ^ x, and the base address is naturally
1549 * aligned to the size
1550 */
Yu Zhao9dd2fe82009-05-18 13:51:36 +08001551 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1552 iommu->flush.flush_iotlb(iommu, did, 0, 0,
David Woodhouse1f0ef2a2009-05-10 19:58:49 +01001553 DMA_TLB_DSI_FLUSH);
Yu Zhao9dd2fe82009-05-18 13:51:36 +08001554 else
David Woodhouseea8ea462014-03-05 17:09:32 +00001555 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
Yu Zhao9dd2fe82009-05-18 13:51:36 +08001556 DMA_TLB_PSI_FLUSH);
Yu Zhaobf92df32009-06-29 11:31:45 +08001557
1558 /*
Nadav Amit82653632010-04-01 13:24:40 +03001559 * In caching mode, changes of pages from non-present to present require
1560 * flush. However, device IOTLB doesn't need to be flushed in this case.
Yu Zhaobf92df32009-06-29 11:31:45 +08001561 */
Nadav Amit82653632010-04-01 13:24:40 +03001562 if (!cap_caching_mode(iommu->cap) || !map)
Peter Xu9d2e6502018-01-10 13:51:37 +08001563 iommu_flush_dev_iotlb(domain, addr, mask);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001564}
1565
Peter Xueed91a02018-05-04 10:34:52 +08001566/* Notification for newly created mappings */
1567static inline void __mapping_notify_one(struct intel_iommu *iommu,
1568 struct dmar_domain *domain,
1569 unsigned long pfn, unsigned int pages)
1570{
1571 /* It's a non-present to present mapping. Only flush if caching mode */
1572 if (cap_caching_mode(iommu->cap))
1573 iommu_flush_iotlb_psi(iommu, domain, pfn, pages, 0, 1);
1574 else
1575 iommu_flush_write_buffer(iommu);
1576}
1577
Joerg Roedel13cf0172017-08-11 11:40:10 +02001578static void iommu_flush_iova(struct iova_domain *iovad)
1579{
1580 struct dmar_domain *domain;
1581 int idx;
1582
1583 domain = container_of(iovad, struct dmar_domain, iovad);
1584
1585 for_each_domain_iommu(idx, domain) {
1586 struct intel_iommu *iommu = g_iommus[idx];
1587 u16 did = domain->iommu_did[iommu->seq_id];
1588
1589 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
1590
1591 if (!cap_caching_mode(iommu->cap))
1592 iommu_flush_dev_iotlb(get_iommu_domain(iommu, did),
1593 0, MAX_AGAW_PFN_WIDTH);
1594 }
1595}
1596
mark grossf8bab732008-02-08 04:18:38 -08001597static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1598{
1599 u32 pmen;
1600 unsigned long flags;
1601
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001602 raw_spin_lock_irqsave(&iommu->register_lock, flags);
mark grossf8bab732008-02-08 04:18:38 -08001603 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1604 pmen &= ~DMA_PMEN_EPM;
1605 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1606
1607 /* wait for the protected region status bit to clear */
1608 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1609 readl, !(pmen & DMA_PMEN_PRS), pmen);
1610
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001611 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
mark grossf8bab732008-02-08 04:18:38 -08001612}
1613
Jiang Liu2a41cce2014-07-11 14:19:33 +08001614static void iommu_enable_translation(struct intel_iommu *iommu)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001615{
1616 u32 sts;
1617 unsigned long flags;
1618
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001619 raw_spin_lock_irqsave(&iommu->register_lock, flags);
David Woodhousec416daa2009-05-10 20:30:58 +01001620 iommu->gcmd |= DMA_GCMD_TE;
1621 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001622
1623 /* Make sure hardware complete it */
1624 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
David Woodhousec416daa2009-05-10 20:30:58 +01001625 readl, (sts & DMA_GSTS_TES), sts);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001626
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001627 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001628}
1629
Jiang Liu2a41cce2014-07-11 14:19:33 +08001630static void iommu_disable_translation(struct intel_iommu *iommu)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001631{
1632 u32 sts;
1633 unsigned long flag;
1634
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001635 raw_spin_lock_irqsave(&iommu->register_lock, flag);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001636 iommu->gcmd &= ~DMA_GCMD_TE;
1637 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1638
1639 /* Make sure hardware complete it */
1640 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
David Woodhousec416daa2009-05-10 20:30:58 +01001641 readl, (!(sts & DMA_GSTS_TES)), sts);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001642
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001643 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001644}
1645
Keshavamurthy, Anil S3460a6d2007-10-21 16:41:54 -07001646
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001647static int iommu_init_domains(struct intel_iommu *iommu)
1648{
Joerg Roedel8bf47812015-07-21 10:41:21 +02001649 u32 ndomains, nlongs;
1650 size_t size;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001651
1652 ndomains = cap_ndoms(iommu->cap);
Joerg Roedel8bf47812015-07-21 10:41:21 +02001653 pr_debug("%s: Number of Domains supported <%d>\n",
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02001654 iommu->name, ndomains);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001655 nlongs = BITS_TO_LONGS(ndomains);
1656
Donald Dutile94a91b502009-08-20 16:51:34 -04001657 spin_lock_init(&iommu->lock);
1658
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001659 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1660 if (!iommu->domain_ids) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02001661 pr_err("%s: Allocating domain id array failed\n",
1662 iommu->name);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001663 return -ENOMEM;
1664 }
Joerg Roedel8bf47812015-07-21 10:41:21 +02001665
Wei Yang86f004c2016-05-21 02:41:51 +00001666 size = (ALIGN(ndomains, 256) >> 8) * sizeof(struct dmar_domain **);
Joerg Roedel8bf47812015-07-21 10:41:21 +02001667 iommu->domains = kzalloc(size, GFP_KERNEL);
1668
1669 if (iommu->domains) {
1670 size = 256 * sizeof(struct dmar_domain *);
1671 iommu->domains[0] = kzalloc(size, GFP_KERNEL);
1672 }
1673
1674 if (!iommu->domains || !iommu->domains[0]) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02001675 pr_err("%s: Allocating domain array failed\n",
1676 iommu->name);
Jiang Liu852bdb02014-01-06 14:18:11 +08001677 kfree(iommu->domain_ids);
Joerg Roedel8bf47812015-07-21 10:41:21 +02001678 kfree(iommu->domains);
Jiang Liu852bdb02014-01-06 14:18:11 +08001679 iommu->domain_ids = NULL;
Joerg Roedel8bf47812015-07-21 10:41:21 +02001680 iommu->domains = NULL;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001681 return -ENOMEM;
1682 }
1683
Joerg Roedel8bf47812015-07-21 10:41:21 +02001684
1685
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001686 /*
Joerg Roedelc0e8a6c2015-07-21 09:39:46 +02001687 * If Caching mode is set, then invalid translations are tagged
1688 * with domain-id 0, hence we need to pre-allocate it. We also
1689 * use domain-id 0 as a marker for non-allocated domain-id, so
1690 * make sure it is not used for a real domain.
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001691 */
Joerg Roedelc0e8a6c2015-07-21 09:39:46 +02001692 set_bit(0, iommu->domain_ids);
1693
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001694 return 0;
1695}
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001696
Jiang Liuffebeb42014-11-09 22:48:02 +08001697static void disable_dmar_iommu(struct intel_iommu *iommu)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001698{
Joerg Roedel29a27712015-07-21 17:17:12 +02001699 struct device_domain_info *info, *tmp;
Joerg Roedel55d94042015-07-22 16:50:40 +02001700 unsigned long flags;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001701
Joerg Roedel29a27712015-07-21 17:17:12 +02001702 if (!iommu->domains || !iommu->domain_ids)
1703 return;
Jiang Liua4eaa862014-02-19 14:07:30 +08001704
Joerg Roedelbea64032016-11-08 15:08:26 +01001705again:
Joerg Roedel55d94042015-07-22 16:50:40 +02001706 spin_lock_irqsave(&device_domain_lock, flags);
Joerg Roedel29a27712015-07-21 17:17:12 +02001707 list_for_each_entry_safe(info, tmp, &device_domain_list, global) {
1708 struct dmar_domain *domain;
1709
1710 if (info->iommu != iommu)
1711 continue;
1712
1713 if (!info->dev || !info->domain)
1714 continue;
1715
1716 domain = info->domain;
1717
Joerg Roedelbea64032016-11-08 15:08:26 +01001718 __dmar_remove_one_dev_info(info);
Joerg Roedel29a27712015-07-21 17:17:12 +02001719
Joerg Roedelbea64032016-11-08 15:08:26 +01001720 if (!domain_type_is_vm_or_si(domain)) {
1721 /*
1722 * The domain_exit() function can't be called under
1723 * device_domain_lock, as it takes this lock itself.
1724 * So release the lock here and re-run the loop
1725 * afterwards.
1726 */
1727 spin_unlock_irqrestore(&device_domain_lock, flags);
Joerg Roedel29a27712015-07-21 17:17:12 +02001728 domain_exit(domain);
Joerg Roedelbea64032016-11-08 15:08:26 +01001729 goto again;
1730 }
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001731 }
Joerg Roedel55d94042015-07-22 16:50:40 +02001732 spin_unlock_irqrestore(&device_domain_lock, flags);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001733
1734 if (iommu->gcmd & DMA_GCMD_TE)
1735 iommu_disable_translation(iommu);
Jiang Liuffebeb42014-11-09 22:48:02 +08001736}
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001737
Jiang Liuffebeb42014-11-09 22:48:02 +08001738static void free_dmar_iommu(struct intel_iommu *iommu)
1739{
1740 if ((iommu->domains) && (iommu->domain_ids)) {
Wei Yang86f004c2016-05-21 02:41:51 +00001741 int elems = ALIGN(cap_ndoms(iommu->cap), 256) >> 8;
Joerg Roedel8bf47812015-07-21 10:41:21 +02001742 int i;
1743
1744 for (i = 0; i < elems; i++)
1745 kfree(iommu->domains[i]);
Jiang Liuffebeb42014-11-09 22:48:02 +08001746 kfree(iommu->domains);
1747 kfree(iommu->domain_ids);
1748 iommu->domains = NULL;
1749 iommu->domain_ids = NULL;
1750 }
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001751
Weidong Hand9630fe2008-12-08 11:06:32 +08001752 g_iommus[iommu->seq_id] = NULL;
1753
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001754 /* free context mapping */
1755 free_context_table(iommu);
David Woodhouse8a94ade2015-03-24 14:54:56 +00001756
1757#ifdef CONFIG_INTEL_IOMMU_SVM
David Woodhousea222a7f2015-10-07 23:35:18 +01001758 if (pasid_enabled(iommu)) {
1759 if (ecap_prs(iommu->ecap))
1760 intel_svm_finish_prq(iommu);
Lu Baolud9737952018-07-14 15:47:02 +08001761 intel_svm_exit(iommu);
David Woodhousea222a7f2015-10-07 23:35:18 +01001762 }
David Woodhouse8a94ade2015-03-24 14:54:56 +00001763#endif
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001764}
1765
Jiang Liuab8dfe22014-07-11 14:19:27 +08001766static struct dmar_domain *alloc_domain(int flags)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001767{
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001768 struct dmar_domain *domain;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001769
1770 domain = alloc_domain_mem();
1771 if (!domain)
1772 return NULL;
1773
Jiang Liuab8dfe22014-07-11 14:19:27 +08001774 memset(domain, 0, sizeof(*domain));
Suresh Siddha4c923d42009-10-02 11:01:24 -07001775 domain->nid = -1;
Jiang Liuab8dfe22014-07-11 14:19:27 +08001776 domain->flags = flags;
Omer Peleg0824c592016-04-20 19:03:35 +03001777 domain->has_iotlb_device = false;
Jiang Liu92d03cc2014-02-19 14:07:28 +08001778 INIT_LIST_HEAD(&domain->devices);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001779
1780 return domain;
1781}
1782
Joerg Roedeld160aca2015-07-22 11:52:53 +02001783/* Must be called with iommu->lock */
1784static int domain_attach_iommu(struct dmar_domain *domain,
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07001785 struct intel_iommu *iommu)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001786{
Jiang Liu44bde612014-07-11 14:19:29 +08001787 unsigned long ndomains;
Joerg Roedel55d94042015-07-22 16:50:40 +02001788 int num;
Jiang Liu44bde612014-07-11 14:19:29 +08001789
Joerg Roedel55d94042015-07-22 16:50:40 +02001790 assert_spin_locked(&device_domain_lock);
Joerg Roedeld160aca2015-07-22 11:52:53 +02001791 assert_spin_locked(&iommu->lock);
Jiang Liu44bde612014-07-11 14:19:29 +08001792
Joerg Roedel29a27712015-07-21 17:17:12 +02001793 domain->iommu_refcnt[iommu->seq_id] += 1;
1794 domain->iommu_count += 1;
1795 if (domain->iommu_refcnt[iommu->seq_id] == 1) {
Jiang Liufb170fb2014-07-11 14:19:28 +08001796 ndomains = cap_ndoms(iommu->cap);
Joerg Roedeld160aca2015-07-22 11:52:53 +02001797 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1798
1799 if (num >= ndomains) {
1800 pr_err("%s: No free domain ids\n", iommu->name);
1801 domain->iommu_refcnt[iommu->seq_id] -= 1;
1802 domain->iommu_count -= 1;
Joerg Roedel55d94042015-07-22 16:50:40 +02001803 return -ENOSPC;
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07001804 }
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001805
Joerg Roedeld160aca2015-07-22 11:52:53 +02001806 set_bit(num, iommu->domain_ids);
1807 set_iommu_domain(iommu, num, domain);
Jiang Liufb170fb2014-07-11 14:19:28 +08001808
Joerg Roedeld160aca2015-07-22 11:52:53 +02001809 domain->iommu_did[iommu->seq_id] = num;
1810 domain->nid = iommu->node;
1811
Jiang Liufb170fb2014-07-11 14:19:28 +08001812 domain_update_iommu_cap(domain);
1813 }
Joerg Roedeld160aca2015-07-22 11:52:53 +02001814
Joerg Roedel55d94042015-07-22 16:50:40 +02001815 return 0;
Jiang Liufb170fb2014-07-11 14:19:28 +08001816}
1817
1818static int domain_detach_iommu(struct dmar_domain *domain,
1819 struct intel_iommu *iommu)
1820{
Joerg Roedeld160aca2015-07-22 11:52:53 +02001821 int num, count = INT_MAX;
Jiang Liufb170fb2014-07-11 14:19:28 +08001822
Joerg Roedel55d94042015-07-22 16:50:40 +02001823 assert_spin_locked(&device_domain_lock);
Joerg Roedeld160aca2015-07-22 11:52:53 +02001824 assert_spin_locked(&iommu->lock);
Jiang Liufb170fb2014-07-11 14:19:28 +08001825
Joerg Roedel29a27712015-07-21 17:17:12 +02001826 domain->iommu_refcnt[iommu->seq_id] -= 1;
1827 count = --domain->iommu_count;
1828 if (domain->iommu_refcnt[iommu->seq_id] == 0) {
Joerg Roedeld160aca2015-07-22 11:52:53 +02001829 num = domain->iommu_did[iommu->seq_id];
1830 clear_bit(num, iommu->domain_ids);
1831 set_iommu_domain(iommu, num, NULL);
1832
Jiang Liufb170fb2014-07-11 14:19:28 +08001833 domain_update_iommu_cap(domain);
Joerg Roedelc0e8a6c2015-07-21 09:39:46 +02001834 domain->iommu_did[iommu->seq_id] = 0;
Jiang Liufb170fb2014-07-11 14:19:28 +08001835 }
Jiang Liufb170fb2014-07-11 14:19:28 +08001836
1837 return count;
1838}
1839
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001840static struct iova_domain reserved_iova_list;
Mark Gross8a443df2008-03-04 14:59:31 -08001841static struct lock_class_key reserved_rbtree_key;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001842
Joseph Cihula51a63e62011-03-21 11:04:24 -07001843static int dmar_init_reserved_ranges(void)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001844{
1845 struct pci_dev *pdev = NULL;
1846 struct iova *iova;
1847 int i;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001848
Zhen Leiaa3ac942017-09-21 16:52:45 +01001849 init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001850
Mark Gross8a443df2008-03-04 14:59:31 -08001851 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1852 &reserved_rbtree_key);
1853
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001854 /* IOAPIC ranges shouldn't be accessed by DMA */
1855 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1856 IOVA_PFN(IOAPIC_RANGE_END));
Joseph Cihula51a63e62011-03-21 11:04:24 -07001857 if (!iova) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02001858 pr_err("Reserve IOAPIC range failed\n");
Joseph Cihula51a63e62011-03-21 11:04:24 -07001859 return -ENODEV;
1860 }
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001861
1862 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1863 for_each_pci_dev(pdev) {
1864 struct resource *r;
1865
1866 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1867 r = &pdev->resource[i];
1868 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1869 continue;
David Woodhouse1a4a4552009-06-28 16:00:42 +01001870 iova = reserve_iova(&reserved_iova_list,
1871 IOVA_PFN(r->start),
1872 IOVA_PFN(r->end));
Joseph Cihula51a63e62011-03-21 11:04:24 -07001873 if (!iova) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02001874 pr_err("Reserve iova failed\n");
Joseph Cihula51a63e62011-03-21 11:04:24 -07001875 return -ENODEV;
1876 }
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001877 }
1878 }
Joseph Cihula51a63e62011-03-21 11:04:24 -07001879 return 0;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001880}
1881
1882static void domain_reserve_special_ranges(struct dmar_domain *domain)
1883{
1884 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1885}
1886
1887static inline int guestwidth_to_adjustwidth(int gaw)
1888{
1889 int agaw;
1890 int r = (gaw - 12) % 9;
1891
1892 if (r == 0)
1893 agaw = gaw;
1894 else
1895 agaw = gaw + 9 - r;
1896 if (agaw > 64)
1897 agaw = 64;
1898 return agaw;
1899}
1900
Joerg Roedeldc534b22015-07-22 12:44:02 +02001901static int domain_init(struct dmar_domain *domain, struct intel_iommu *iommu,
1902 int guest_width)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001903{
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001904 int adjust_width, agaw;
1905 unsigned long sagaw;
Joerg Roedel13cf0172017-08-11 11:40:10 +02001906 int err;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001907
Zhen Leiaa3ac942017-09-21 16:52:45 +01001908 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN);
Joerg Roedel13cf0172017-08-11 11:40:10 +02001909
1910 err = init_iova_flush_queue(&domain->iovad,
1911 iommu_flush_iova, iova_entry_free);
1912 if (err)
1913 return err;
1914
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001915 domain_reserve_special_ranges(domain);
1916
1917 /* calculate AGAW */
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001918 if (guest_width > cap_mgaw(iommu->cap))
1919 guest_width = cap_mgaw(iommu->cap);
1920 domain->gaw = guest_width;
1921 adjust_width = guestwidth_to_adjustwidth(guest_width);
1922 agaw = width_to_agaw(adjust_width);
1923 sagaw = cap_sagaw(iommu->cap);
1924 if (!test_bit(agaw, &sagaw)) {
1925 /* hardware doesn't support it, choose a bigger one */
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02001926 pr_debug("Hardware doesn't support agaw %d\n", agaw);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001927 agaw = find_next_bit(&sagaw, 5, agaw);
1928 if (agaw >= 5)
1929 return -ENODEV;
1930 }
1931 domain->agaw = agaw;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001932
Weidong Han8e6040972008-12-08 15:49:06 +08001933 if (ecap_coherent(iommu->ecap))
1934 domain->iommu_coherency = 1;
1935 else
1936 domain->iommu_coherency = 0;
1937
Sheng Yang58c610b2009-03-18 15:33:05 +08001938 if (ecap_sc_support(iommu->ecap))
1939 domain->iommu_snooping = 1;
1940 else
1941 domain->iommu_snooping = 0;
1942
David Woodhouse214e39a2014-03-19 10:38:49 +00001943 if (intel_iommu_superpage)
1944 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1945 else
1946 domain->iommu_superpage = 0;
1947
Suresh Siddha4c923d42009-10-02 11:01:24 -07001948 domain->nid = iommu->node;
Weidong Hanc7151a82008-12-08 22:51:37 +08001949
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001950 /* always allocate the top pgd */
Suresh Siddha4c923d42009-10-02 11:01:24 -07001951 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001952 if (!domain->pgd)
1953 return -ENOMEM;
Fenghua Yu5b6985c2008-10-16 18:02:32 -07001954 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001955 return 0;
1956}
1957
1958static void domain_exit(struct dmar_domain *domain)
1959{
David Woodhouseea8ea462014-03-05 17:09:32 +00001960 struct page *freelist = NULL;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001961
1962 /* Domain 0 is reserved, so dont process it */
1963 if (!domain)
1964 return;
1965
Joerg Roedeld160aca2015-07-22 11:52:53 +02001966 /* Remove associated devices and clear attached or cached domains */
1967 rcu_read_lock();
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001968 domain_remove_dev_info(domain);
Joerg Roedeld160aca2015-07-22 11:52:53 +02001969 rcu_read_unlock();
Jiang Liu92d03cc2014-02-19 14:07:28 +08001970
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001971 /* destroy iovas */
1972 put_iova_domain(&domain->iovad);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001973
David Woodhouseea8ea462014-03-05 17:09:32 +00001974 freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001975
David Woodhouseea8ea462014-03-05 17:09:32 +00001976 dma_free_pagelist(freelist);
1977
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001978 free_domain_mem(domain);
1979}
1980
David Woodhouse64ae8922014-03-09 12:52:30 -07001981static int domain_context_mapping_one(struct dmar_domain *domain,
1982 struct intel_iommu *iommu,
Joerg Roedel28ccce02015-07-21 14:45:31 +02001983 u8 bus, u8 devfn)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001984{
Joerg Roedelc6c2ceb2015-07-22 13:11:53 +02001985 u16 did = domain->iommu_did[iommu->seq_id];
Joerg Roedel28ccce02015-07-21 14:45:31 +02001986 int translation = CONTEXT_TT_MULTI_LEVEL;
1987 struct device_domain_info *info = NULL;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001988 struct context_entry *context;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001989 unsigned long flags;
Weidong Hanea6606b2008-12-08 23:08:15 +08001990 struct dma_pte *pgd;
Joerg Roedel55d94042015-07-22 16:50:40 +02001991 int ret, agaw;
Joerg Roedel28ccce02015-07-21 14:45:31 +02001992
Joerg Roedelc6c2ceb2015-07-22 13:11:53 +02001993 WARN_ON(did == 0);
1994
Joerg Roedel28ccce02015-07-21 14:45:31 +02001995 if (hw_pass_through && domain_type_is_si(domain))
1996 translation = CONTEXT_TT_PASS_THROUGH;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001997
1998 pr_debug("Set context mapping for %02x:%02x.%d\n",
1999 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07002000
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002001 BUG_ON(!domain->pgd);
Weidong Han5331fe62008-12-08 23:00:00 +08002002
Joerg Roedel55d94042015-07-22 16:50:40 +02002003 spin_lock_irqsave(&device_domain_lock, flags);
2004 spin_lock(&iommu->lock);
2005
2006 ret = -ENOMEM;
David Woodhouse03ecc322015-02-13 14:35:21 +00002007 context = iommu_context_addr(iommu, bus, devfn, 1);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002008 if (!context)
Joerg Roedel55d94042015-07-22 16:50:40 +02002009 goto out_unlock;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002010
Joerg Roedel55d94042015-07-22 16:50:40 +02002011 ret = 0;
2012 if (context_present(context))
2013 goto out_unlock;
Joerg Roedelcf484d02015-06-12 12:21:46 +02002014
Xunlei Pangaec0e862016-12-05 20:09:07 +08002015 /*
2016 * For kdump cases, old valid entries may be cached due to the
2017 * in-flight DMA and copied pgtable, but there is no unmapping
2018 * behaviour for them, thus we need an explicit cache flush for
2019 * the newly-mapped device. For kdump, at this point, the device
2020 * is supposed to finish reset at its driver probe stage, so no
2021 * in-flight DMA will exist, and we don't need to worry anymore
2022 * hereafter.
2023 */
2024 if (context_copied(context)) {
2025 u16 did_old = context_domain_id(context);
2026
Christos Gkekasb117e032017-10-08 23:33:31 +01002027 if (did_old < cap_ndoms(iommu->cap)) {
Xunlei Pangaec0e862016-12-05 20:09:07 +08002028 iommu->flush.flush_context(iommu, did_old,
2029 (((u16)bus) << 8) | devfn,
2030 DMA_CCMD_MASK_NOBIT,
2031 DMA_CCMD_DEVICE_INVL);
KarimAllah Ahmedf73a7ee2017-05-05 11:39:59 -07002032 iommu->flush.flush_iotlb(iommu, did_old, 0, 0,
2033 DMA_TLB_DSI_FLUSH);
2034 }
Xunlei Pangaec0e862016-12-05 20:09:07 +08002035 }
2036
Weidong Hanea6606b2008-12-08 23:08:15 +08002037 pgd = domain->pgd;
2038
Joerg Roedelde24e552015-07-21 14:53:04 +02002039 context_clear_entry(context);
Joerg Roedelc6c2ceb2015-07-22 13:11:53 +02002040 context_set_domain_id(context, did);
Weidong Hanea6606b2008-12-08 23:08:15 +08002041
Joerg Roedelde24e552015-07-21 14:53:04 +02002042 /*
2043 * Skip top levels of page tables for iommu which has less agaw
2044 * than default. Unnecessary for PT mode.
2045 */
Yu Zhao93a23a72009-05-18 13:51:37 +08002046 if (translation != CONTEXT_TT_PASS_THROUGH) {
Joerg Roedelde24e552015-07-21 14:53:04 +02002047 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
Joerg Roedel55d94042015-07-22 16:50:40 +02002048 ret = -ENOMEM;
Joerg Roedelde24e552015-07-21 14:53:04 +02002049 pgd = phys_to_virt(dma_pte_addr(pgd));
Joerg Roedel55d94042015-07-22 16:50:40 +02002050 if (!dma_pte_present(pgd))
2051 goto out_unlock;
Joerg Roedelde24e552015-07-21 14:53:04 +02002052 }
2053
David Woodhouse64ae8922014-03-09 12:52:30 -07002054 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
David Woodhouseb16d0cb2015-10-12 14:17:37 +01002055 if (info && info->ats_supported)
2056 translation = CONTEXT_TT_DEV_IOTLB;
2057 else
2058 translation = CONTEXT_TT_MULTI_LEVEL;
Joerg Roedelde24e552015-07-21 14:53:04 +02002059
Yu Zhao93a23a72009-05-18 13:51:37 +08002060 context_set_address_root(context, virt_to_phys(pgd));
2061 context_set_address_width(context, iommu->agaw);
Joerg Roedelde24e552015-07-21 14:53:04 +02002062 } else {
2063 /*
2064 * In pass through mode, AW must be programmed to
2065 * indicate the largest AGAW value supported by
2066 * hardware. And ASR is ignored by hardware.
2067 */
2068 context_set_address_width(context, iommu->msagaw);
Yu Zhao93a23a72009-05-18 13:51:37 +08002069 }
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07002070
2071 context_set_translation_type(context, translation);
Mark McLoughlinc07e7d22008-11-21 16:54:46 +00002072 context_set_fault_enable(context);
2073 context_set_present(context);
Weidong Han5331fe62008-12-08 23:00:00 +08002074 domain_flush_cache(domain, context, sizeof(*context));
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002075
David Woodhouse4c25a2c2009-05-10 17:16:06 +01002076 /*
2077 * It's a non-present to present mapping. If hardware doesn't cache
2078 * non-present entry we only need to flush the write-buffer. If the
2079 * _does_ cache non-present entries, then it does so in the special
2080 * domain #0, which we have to flush:
2081 */
2082 if (cap_caching_mode(iommu->cap)) {
2083 iommu->flush.flush_context(iommu, 0,
2084 (((u16)bus) << 8) | devfn,
2085 DMA_CCMD_MASK_NOBIT,
2086 DMA_CCMD_DEVICE_INVL);
Joerg Roedelc6c2ceb2015-07-22 13:11:53 +02002087 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
David Woodhouse4c25a2c2009-05-10 17:16:06 +01002088 } else {
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002089 iommu_flush_write_buffer(iommu);
David Woodhouse4c25a2c2009-05-10 17:16:06 +01002090 }
Yu Zhao93a23a72009-05-18 13:51:37 +08002091 iommu_enable_dev_iotlb(info);
Weidong Hanc7151a82008-12-08 22:51:37 +08002092
Joerg Roedel55d94042015-07-22 16:50:40 +02002093 ret = 0;
2094
2095out_unlock:
2096 spin_unlock(&iommu->lock);
2097 spin_unlock_irqrestore(&device_domain_lock, flags);
Jiang Liufb170fb2014-07-11 14:19:28 +08002098
Wei Yang5c365d12016-07-13 13:53:21 +00002099 return ret;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002100}
2101
Alex Williamson579305f2014-07-03 09:51:43 -06002102struct domain_context_mapping_data {
2103 struct dmar_domain *domain;
2104 struct intel_iommu *iommu;
Alex Williamson579305f2014-07-03 09:51:43 -06002105};
2106
2107static int domain_context_mapping_cb(struct pci_dev *pdev,
2108 u16 alias, void *opaque)
2109{
2110 struct domain_context_mapping_data *data = opaque;
2111
2112 return domain_context_mapping_one(data->domain, data->iommu,
Joerg Roedel28ccce02015-07-21 14:45:31 +02002113 PCI_BUS_NUM(alias), alias & 0xff);
Alex Williamson579305f2014-07-03 09:51:43 -06002114}
2115
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002116static int
Joerg Roedel28ccce02015-07-21 14:45:31 +02002117domain_context_mapping(struct dmar_domain *domain, struct device *dev)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002118{
David Woodhouse64ae8922014-03-09 12:52:30 -07002119 struct intel_iommu *iommu;
David Woodhouse156baca2014-03-09 14:00:57 -07002120 u8 bus, devfn;
Alex Williamson579305f2014-07-03 09:51:43 -06002121 struct domain_context_mapping_data data;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002122
David Woodhousee1f167f2014-03-09 15:24:46 -07002123 iommu = device_to_iommu(dev, &bus, &devfn);
David Woodhouse64ae8922014-03-09 12:52:30 -07002124 if (!iommu)
2125 return -ENODEV;
2126
Alex Williamson579305f2014-07-03 09:51:43 -06002127 if (!dev_is_pci(dev))
Joerg Roedel28ccce02015-07-21 14:45:31 +02002128 return domain_context_mapping_one(domain, iommu, bus, devfn);
Alex Williamson579305f2014-07-03 09:51:43 -06002129
2130 data.domain = domain;
2131 data.iommu = iommu;
Alex Williamson579305f2014-07-03 09:51:43 -06002132
2133 return pci_for_each_dma_alias(to_pci_dev(dev),
2134 &domain_context_mapping_cb, &data);
2135}
2136
2137static int domain_context_mapped_cb(struct pci_dev *pdev,
2138 u16 alias, void *opaque)
2139{
2140 struct intel_iommu *iommu = opaque;
2141
2142 return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002143}
2144
David Woodhousee1f167f2014-03-09 15:24:46 -07002145static int domain_context_mapped(struct device *dev)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002146{
Weidong Han5331fe62008-12-08 23:00:00 +08002147 struct intel_iommu *iommu;
David Woodhouse156baca2014-03-09 14:00:57 -07002148 u8 bus, devfn;
Weidong Han5331fe62008-12-08 23:00:00 +08002149
David Woodhousee1f167f2014-03-09 15:24:46 -07002150 iommu = device_to_iommu(dev, &bus, &devfn);
Weidong Han5331fe62008-12-08 23:00:00 +08002151 if (!iommu)
2152 return -ENODEV;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002153
Alex Williamson579305f2014-07-03 09:51:43 -06002154 if (!dev_is_pci(dev))
2155 return device_context_mapped(iommu, bus, devfn);
David Woodhousee1f167f2014-03-09 15:24:46 -07002156
Alex Williamson579305f2014-07-03 09:51:43 -06002157 return !pci_for_each_dma_alias(to_pci_dev(dev),
2158 domain_context_mapped_cb, iommu);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002159}
2160
Fenghua Yuf5329592009-08-04 15:09:37 -07002161/* Returns a number of VTD pages, but aligned to MM page size */
2162static inline unsigned long aligned_nrpages(unsigned long host_addr,
2163 size_t size)
2164{
2165 host_addr &= ~PAGE_MASK;
2166 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
2167}
2168
Youquan Song6dd9a7c2011-05-25 19:13:49 +01002169/* Return largest possible superpage level for a given mapping */
2170static inline int hardware_largepage_caps(struct dmar_domain *domain,
2171 unsigned long iov_pfn,
2172 unsigned long phy_pfn,
2173 unsigned long pages)
2174{
2175 int support, level = 1;
2176 unsigned long pfnmerge;
2177
2178 support = domain->iommu_superpage;
2179
2180 /* To use a large page, the virtual *and* physical addresses
2181 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
2182 of them will mean we have to use smaller pages. So just
2183 merge them and check both at once. */
2184 pfnmerge = iov_pfn | phy_pfn;
2185
2186 while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
2187 pages >>= VTD_STRIDE_SHIFT;
2188 if (!pages)
2189 break;
2190 pfnmerge >>= VTD_STRIDE_SHIFT;
2191 level++;
2192 support--;
2193 }
2194 return level;
2195}
2196
David Woodhouse9051aa02009-06-29 12:30:54 +01002197static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2198 struct scatterlist *sg, unsigned long phys_pfn,
2199 unsigned long nr_pages, int prot)
David Woodhousee1605492009-06-29 11:17:38 +01002200{
2201 struct dma_pte *first_pte = NULL, *pte = NULL;
David Woodhouse9051aa02009-06-29 12:30:54 +01002202 phys_addr_t uninitialized_var(pteval);
Jiang Liucc4f14a2014-11-26 09:42:10 +08002203 unsigned long sg_res = 0;
Youquan Song6dd9a7c2011-05-25 19:13:49 +01002204 unsigned int largepage_lvl = 0;
2205 unsigned long lvl_pages = 0;
David Woodhousee1605492009-06-29 11:17:38 +01002206
Jiang Liu162d1b12014-07-11 14:19:35 +08002207 BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
David Woodhousee1605492009-06-29 11:17:38 +01002208
2209 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
2210 return -EINVAL;
2211
2212 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
2213
Jiang Liucc4f14a2014-11-26 09:42:10 +08002214 if (!sg) {
2215 sg_res = nr_pages;
David Woodhouse9051aa02009-06-29 12:30:54 +01002216 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
2217 }
2218
Youquan Song6dd9a7c2011-05-25 19:13:49 +01002219 while (nr_pages > 0) {
David Woodhousec85994e2009-07-01 19:21:24 +01002220 uint64_t tmp;
2221
David Woodhousee1605492009-06-29 11:17:38 +01002222 if (!sg_res) {
Robin Murphy29a90b72017-09-28 15:14:01 +01002223 unsigned int pgoff = sg->offset & ~PAGE_MASK;
2224
Fenghua Yuf5329592009-08-04 15:09:37 -07002225 sg_res = aligned_nrpages(sg->offset, sg->length);
Robin Murphy29a90b72017-09-28 15:14:01 +01002226 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + pgoff;
David Woodhousee1605492009-06-29 11:17:38 +01002227 sg->dma_length = sg->length;
Robin Murphy29a90b72017-09-28 15:14:01 +01002228 pteval = (sg_phys(sg) - pgoff) | prot;
Youquan Song6dd9a7c2011-05-25 19:13:49 +01002229 phys_pfn = pteval >> VTD_PAGE_SHIFT;
David Woodhousee1605492009-06-29 11:17:38 +01002230 }
Youquan Song6dd9a7c2011-05-25 19:13:49 +01002231
David Woodhousee1605492009-06-29 11:17:38 +01002232 if (!pte) {
Youquan Song6dd9a7c2011-05-25 19:13:49 +01002233 largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2234
David Woodhouse5cf0a762014-03-19 16:07:49 +00002235 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
David Woodhousee1605492009-06-29 11:17:38 +01002236 if (!pte)
2237 return -ENOMEM;
Youquan Song6dd9a7c2011-05-25 19:13:49 +01002238 /* It is large page*/
Woodhouse, David6491d4d2012-12-19 13:25:35 +00002239 if (largepage_lvl > 1) {
Christian Zanderba2374f2015-06-10 09:41:45 -07002240 unsigned long nr_superpages, end_pfn;
2241
Youquan Song6dd9a7c2011-05-25 19:13:49 +01002242 pteval |= DMA_PTE_LARGE_PAGE;
Jiang Liud41a4ad2014-07-11 14:19:34 +08002243 lvl_pages = lvl_to_nr_pages(largepage_lvl);
Christian Zanderba2374f2015-06-10 09:41:45 -07002244
2245 nr_superpages = sg_res / lvl_pages;
2246 end_pfn = iov_pfn + nr_superpages * lvl_pages - 1;
2247
Jiang Liud41a4ad2014-07-11 14:19:34 +08002248 /*
2249 * Ensure that old small page tables are
Christian Zanderba2374f2015-06-10 09:41:45 -07002250 * removed to make room for superpage(s).
David Dillowbc24c572017-06-28 19:42:23 -07002251 * We're adding new large pages, so make sure
2252 * we don't remove their parent tables.
Jiang Liud41a4ad2014-07-11 14:19:34 +08002253 */
David Dillowbc24c572017-06-28 19:42:23 -07002254 dma_pte_free_pagetable(domain, iov_pfn, end_pfn,
2255 largepage_lvl + 1);
Woodhouse, David6491d4d2012-12-19 13:25:35 +00002256 } else {
Youquan Song6dd9a7c2011-05-25 19:13:49 +01002257 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
Woodhouse, David6491d4d2012-12-19 13:25:35 +00002258 }
Youquan Song6dd9a7c2011-05-25 19:13:49 +01002259
David Woodhousee1605492009-06-29 11:17:38 +01002260 }
2261 /* We don't need lock here, nobody else
2262 * touches the iova range
2263 */
David Woodhouse7766a3f2009-07-01 20:27:03 +01002264 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
David Woodhousec85994e2009-07-01 19:21:24 +01002265 if (tmp) {
David Woodhouse1bf20f02009-06-29 22:06:43 +01002266 static int dumps = 5;
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02002267 pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2268 iov_pfn, tmp, (unsigned long long)pteval);
David Woodhouse1bf20f02009-06-29 22:06:43 +01002269 if (dumps) {
2270 dumps--;
2271 debug_dma_dump_mappings(NULL);
2272 }
2273 WARN_ON(1);
2274 }
Youquan Song6dd9a7c2011-05-25 19:13:49 +01002275
2276 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2277
2278 BUG_ON(nr_pages < lvl_pages);
2279 BUG_ON(sg_res < lvl_pages);
2280
2281 nr_pages -= lvl_pages;
2282 iov_pfn += lvl_pages;
2283 phys_pfn += lvl_pages;
2284 pteval += lvl_pages * VTD_PAGE_SIZE;
2285 sg_res -= lvl_pages;
2286
2287 /* If the next PTE would be the first in a new page, then we
2288 need to flush the cache on the entries we've just written.
2289 And then we'll need to recalculate 'pte', so clear it and
2290 let it get set again in the if (!pte) block above.
2291
2292 If we're done (!nr_pages) we need to flush the cache too.
2293
2294 Also if we've been setting superpages, we may need to
2295 recalculate 'pte' and switch back to smaller pages for the
2296 end of the mapping, if the trailing size is not enough to
2297 use another superpage (i.e. sg_res < lvl_pages). */
David Woodhousee1605492009-06-29 11:17:38 +01002298 pte++;
Youquan Song6dd9a7c2011-05-25 19:13:49 +01002299 if (!nr_pages || first_pte_in_page(pte) ||
2300 (largepage_lvl > 1 && sg_res < lvl_pages)) {
David Woodhousee1605492009-06-29 11:17:38 +01002301 domain_flush_cache(domain, first_pte,
2302 (void *)pte - (void *)first_pte);
2303 pte = NULL;
2304 }
Youquan Song6dd9a7c2011-05-25 19:13:49 +01002305
2306 if (!sg_res && nr_pages)
David Woodhousee1605492009-06-29 11:17:38 +01002307 sg = sg_next(sg);
2308 }
2309 return 0;
2310}
2311
Peter Xu87684fd2018-05-04 10:34:53 +08002312static int domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2313 struct scatterlist *sg, unsigned long phys_pfn,
2314 unsigned long nr_pages, int prot)
2315{
2316 int ret;
2317 struct intel_iommu *iommu;
2318
2319 /* Do the real mapping first */
2320 ret = __domain_mapping(domain, iov_pfn, sg, phys_pfn, nr_pages, prot);
2321 if (ret)
2322 return ret;
2323
2324 /* Notify about the new mapping */
2325 if (domain_type_is_vm(domain)) {
2326 /* VM typed domains can have more than one IOMMUs */
2327 int iommu_id;
2328 for_each_domain_iommu(iommu_id, domain) {
2329 iommu = g_iommus[iommu_id];
2330 __mapping_notify_one(iommu, domain, iov_pfn, nr_pages);
2331 }
2332 } else {
2333 /* General domains only have one IOMMU */
2334 iommu = domain_get_iommu(domain);
2335 __mapping_notify_one(iommu, domain, iov_pfn, nr_pages);
2336 }
2337
2338 return 0;
2339}
2340
David Woodhouse9051aa02009-06-29 12:30:54 +01002341static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2342 struct scatterlist *sg, unsigned long nr_pages,
2343 int prot)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002344{
Peter Xu87684fd2018-05-04 10:34:53 +08002345 return domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
David Woodhouse9051aa02009-06-29 12:30:54 +01002346}
Fenghua Yu5b6985c2008-10-16 18:02:32 -07002347
David Woodhouse9051aa02009-06-29 12:30:54 +01002348static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2349 unsigned long phys_pfn, unsigned long nr_pages,
2350 int prot)
2351{
Peter Xu87684fd2018-05-04 10:34:53 +08002352 return domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002353}
2354
Joerg Roedel2452d9d2015-07-23 16:20:14 +02002355static void domain_context_clear_one(struct intel_iommu *iommu, u8 bus, u8 devfn)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002356{
Filippo Sironi50822192017-08-31 10:58:11 +02002357 unsigned long flags;
2358 struct context_entry *context;
2359 u16 did_old;
2360
Weidong Hanc7151a82008-12-08 22:51:37 +08002361 if (!iommu)
2362 return;
Weidong Han8c11e792008-12-08 15:29:22 +08002363
Filippo Sironi50822192017-08-31 10:58:11 +02002364 spin_lock_irqsave(&iommu->lock, flags);
2365 context = iommu_context_addr(iommu, bus, devfn, 0);
2366 if (!context) {
2367 spin_unlock_irqrestore(&iommu->lock, flags);
2368 return;
2369 }
2370 did_old = context_domain_id(context);
2371 context_clear_entry(context);
2372 __iommu_flush_cache(iommu, context, sizeof(*context));
2373 spin_unlock_irqrestore(&iommu->lock, flags);
2374 iommu->flush.flush_context(iommu,
2375 did_old,
2376 (((u16)bus) << 8) | devfn,
2377 DMA_CCMD_MASK_NOBIT,
2378 DMA_CCMD_DEVICE_INVL);
2379 iommu->flush.flush_iotlb(iommu,
2380 did_old,
2381 0,
2382 0,
2383 DMA_TLB_DSI_FLUSH);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002384}
2385
David Woodhouse109b9b02012-05-25 17:43:02 +01002386static inline void unlink_domain_info(struct device_domain_info *info)
2387{
2388 assert_spin_locked(&device_domain_lock);
2389 list_del(&info->link);
2390 list_del(&info->global);
2391 if (info->dev)
David Woodhouse0bcb3e22014-03-06 17:12:03 +00002392 info->dev->archdata.iommu = NULL;
David Woodhouse109b9b02012-05-25 17:43:02 +01002393}
2394
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002395static void domain_remove_dev_info(struct dmar_domain *domain)
2396{
Yijing Wang3a74ca02014-05-20 20:37:47 +08002397 struct device_domain_info *info, *tmp;
Jiang Liufb170fb2014-07-11 14:19:28 +08002398 unsigned long flags;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002399
2400 spin_lock_irqsave(&device_domain_lock, flags);
Joerg Roedel76f45fe2015-07-21 18:25:11 +02002401 list_for_each_entry_safe(info, tmp, &domain->devices, link)
Joerg Roedel127c7612015-07-23 17:44:46 +02002402 __dmar_remove_one_dev_info(info);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002403 spin_unlock_irqrestore(&device_domain_lock, flags);
2404}
2405
2406/*
2407 * find_domain
David Woodhouse1525a292014-03-06 16:19:30 +00002408 * Note: we use struct device->archdata.iommu stores the info
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002409 */
David Woodhouse1525a292014-03-06 16:19:30 +00002410static struct dmar_domain *find_domain(struct device *dev)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002411{
2412 struct device_domain_info *info;
2413
2414 /* No lock here, assumes no domain exit in normal case */
David Woodhouse1525a292014-03-06 16:19:30 +00002415 info = dev->archdata.iommu;
Peter Xub316d022017-05-22 18:28:51 +08002416 if (likely(info))
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002417 return info->domain;
2418 return NULL;
2419}
2420
David Woodhouse5a8f40e2014-03-09 13:31:18 -07002421static inline struct device_domain_info *
Jiang Liu745f2582014-02-19 14:07:26 +08002422dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2423{
2424 struct device_domain_info *info;
2425
2426 list_for_each_entry(info, &device_domain_list, global)
David Woodhouse41e80dca2014-03-09 13:55:54 -07002427 if (info->iommu->segment == segment && info->bus == bus &&
Jiang Liu745f2582014-02-19 14:07:26 +08002428 info->devfn == devfn)
David Woodhouse5a8f40e2014-03-09 13:31:18 -07002429 return info;
Jiang Liu745f2582014-02-19 14:07:26 +08002430
2431 return NULL;
2432}
2433
Joerg Roedel5db31562015-07-22 12:40:43 +02002434static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,
2435 int bus, int devfn,
2436 struct device *dev,
2437 struct dmar_domain *domain)
Jiang Liu745f2582014-02-19 14:07:26 +08002438{
David Woodhouse5a8f40e2014-03-09 13:31:18 -07002439 struct dmar_domain *found = NULL;
Jiang Liu745f2582014-02-19 14:07:26 +08002440 struct device_domain_info *info;
2441 unsigned long flags;
Joerg Roedeld160aca2015-07-22 11:52:53 +02002442 int ret;
Jiang Liu745f2582014-02-19 14:07:26 +08002443
2444 info = alloc_devinfo_mem();
2445 if (!info)
David Woodhouseb718cd32014-03-09 13:11:33 -07002446 return NULL;
Jiang Liu745f2582014-02-19 14:07:26 +08002447
Jiang Liu745f2582014-02-19 14:07:26 +08002448 info->bus = bus;
2449 info->devfn = devfn;
David Woodhouseb16d0cb2015-10-12 14:17:37 +01002450 info->ats_supported = info->pasid_supported = info->pri_supported = 0;
2451 info->ats_enabled = info->pasid_enabled = info->pri_enabled = 0;
2452 info->ats_qdep = 0;
Jiang Liu745f2582014-02-19 14:07:26 +08002453 info->dev = dev;
2454 info->domain = domain;
David Woodhouse5a8f40e2014-03-09 13:31:18 -07002455 info->iommu = iommu;
Lu Baolucc580e42018-07-14 15:46:59 +08002456 info->pasid_table = NULL;
Jiang Liu745f2582014-02-19 14:07:26 +08002457
David Woodhouseb16d0cb2015-10-12 14:17:37 +01002458 if (dev && dev_is_pci(dev)) {
2459 struct pci_dev *pdev = to_pci_dev(info->dev);
2460
Gil Kupfercef74402018-05-10 17:56:02 -05002461 if (!pci_ats_disabled() &&
2462 ecap_dev_iotlb_support(iommu->ecap) &&
David Woodhouseb16d0cb2015-10-12 14:17:37 +01002463 pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS) &&
2464 dmar_find_matched_atsr_unit(pdev))
2465 info->ats_supported = 1;
2466
2467 if (ecs_enabled(iommu)) {
2468 if (pasid_enabled(iommu)) {
2469 int features = pci_pasid_features(pdev);
2470 if (features >= 0)
2471 info->pasid_supported = features | 1;
2472 }
2473
2474 if (info->ats_supported && ecap_prs(iommu->ecap) &&
2475 pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI))
2476 info->pri_supported = 1;
2477 }
2478 }
2479
Jiang Liu745f2582014-02-19 14:07:26 +08002480 spin_lock_irqsave(&device_domain_lock, flags);
2481 if (dev)
David Woodhouse0bcb3e22014-03-06 17:12:03 +00002482 found = find_domain(dev);
Joerg Roedelf303e502015-07-23 18:37:13 +02002483
2484 if (!found) {
David Woodhouse5a8f40e2014-03-09 13:31:18 -07002485 struct device_domain_info *info2;
David Woodhouse41e80dca2014-03-09 13:55:54 -07002486 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
Joerg Roedelf303e502015-07-23 18:37:13 +02002487 if (info2) {
2488 found = info2->domain;
2489 info2->dev = dev;
2490 }
David Woodhouse5a8f40e2014-03-09 13:31:18 -07002491 }
Joerg Roedelf303e502015-07-23 18:37:13 +02002492
Jiang Liu745f2582014-02-19 14:07:26 +08002493 if (found) {
2494 spin_unlock_irqrestore(&device_domain_lock, flags);
2495 free_devinfo_mem(info);
David Woodhouseb718cd32014-03-09 13:11:33 -07002496 /* Caller must free the original domain */
2497 return found;
Jiang Liu745f2582014-02-19 14:07:26 +08002498 }
2499
Joerg Roedeld160aca2015-07-22 11:52:53 +02002500 spin_lock(&iommu->lock);
2501 ret = domain_attach_iommu(domain, iommu);
2502 spin_unlock(&iommu->lock);
2503
2504 if (ret) {
Joerg Roedelc6c2ceb2015-07-22 13:11:53 +02002505 spin_unlock_irqrestore(&device_domain_lock, flags);
Sudip Mukherjee499f3aa2015-09-18 16:27:07 +05302506 free_devinfo_mem(info);
Joerg Roedelc6c2ceb2015-07-22 13:11:53 +02002507 return NULL;
2508 }
Joerg Roedelc6c2ceb2015-07-22 13:11:53 +02002509
David Woodhouseb718cd32014-03-09 13:11:33 -07002510 list_add(&info->link, &domain->devices);
2511 list_add(&info->global, &device_domain_list);
2512 if (dev)
2513 dev->archdata.iommu = info;
Lu Baolua7fc93f2018-07-14 15:47:00 +08002514
2515 if (dev && dev_is_pci(dev) && info->pasid_supported) {
2516 ret = intel_pasid_alloc_table(dev);
2517 if (ret) {
Lu Baolube9e6592018-09-08 09:42:53 +08002518 pr_warn("No pasid table for %s, pasid disabled\n",
2519 dev_name(dev));
2520 info->pasid_supported = 0;
Lu Baolua7fc93f2018-07-14 15:47:00 +08002521 }
2522 }
David Woodhouseb718cd32014-03-09 13:11:33 -07002523 spin_unlock_irqrestore(&device_domain_lock, flags);
2524
Joerg Roedelcc4e2572015-07-22 10:04:36 +02002525 if (dev && domain_context_mapping(domain, dev)) {
2526 pr_err("Domain context map for %s failed\n", dev_name(dev));
Joerg Roedele6de0f82015-07-22 16:30:36 +02002527 dmar_remove_one_dev_info(domain, dev);
Joerg Roedelcc4e2572015-07-22 10:04:36 +02002528 return NULL;
2529 }
2530
David Woodhouseb718cd32014-03-09 13:11:33 -07002531 return domain;
Jiang Liu745f2582014-02-19 14:07:26 +08002532}
2533
Alex Williamson579305f2014-07-03 09:51:43 -06002534static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2535{
2536 *(u16 *)opaque = alias;
2537 return 0;
2538}
2539
Joerg Roedel76208352016-08-25 14:25:12 +02002540static struct dmar_domain *find_or_alloc_domain(struct device *dev, int gaw)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002541{
Joerg Roedelcc4e2572015-07-22 10:04:36 +02002542 struct device_domain_info *info = NULL;
Joerg Roedel76208352016-08-25 14:25:12 +02002543 struct dmar_domain *domain = NULL;
Alex Williamson579305f2014-07-03 09:51:43 -06002544 struct intel_iommu *iommu;
Lu Baolufcc35c62018-05-04 13:08:17 +08002545 u16 dma_alias;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002546 unsigned long flags;
Yijing Wangaa4d0662014-05-26 20:14:06 +08002547 u8 bus, devfn;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002548
David Woodhouse146922e2014-03-09 15:44:17 -07002549 iommu = device_to_iommu(dev, &bus, &devfn);
2550 if (!iommu)
Alex Williamson579305f2014-07-03 09:51:43 -06002551 return NULL;
2552
2553 if (dev_is_pci(dev)) {
2554 struct pci_dev *pdev = to_pci_dev(dev);
2555
2556 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2557
2558 spin_lock_irqsave(&device_domain_lock, flags);
2559 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2560 PCI_BUS_NUM(dma_alias),
2561 dma_alias & 0xff);
2562 if (info) {
2563 iommu = info->iommu;
2564 domain = info->domain;
2565 }
2566 spin_unlock_irqrestore(&device_domain_lock, flags);
2567
Joerg Roedel76208352016-08-25 14:25:12 +02002568 /* DMA alias already has a domain, use it */
Alex Williamson579305f2014-07-03 09:51:43 -06002569 if (info)
Joerg Roedel76208352016-08-25 14:25:12 +02002570 goto out;
Alex Williamson579305f2014-07-03 09:51:43 -06002571 }
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002572
David Woodhouse146922e2014-03-09 15:44:17 -07002573 /* Allocate and initialize new domain for the device */
Jiang Liuab8dfe22014-07-11 14:19:27 +08002574 domain = alloc_domain(0);
Jiang Liu745f2582014-02-19 14:07:26 +08002575 if (!domain)
Alex Williamson579305f2014-07-03 09:51:43 -06002576 return NULL;
Joerg Roedeldc534b22015-07-22 12:44:02 +02002577 if (domain_init(domain, iommu, gaw)) {
Alex Williamson579305f2014-07-03 09:51:43 -06002578 domain_exit(domain);
2579 return NULL;
2580 }
2581
Joerg Roedel76208352016-08-25 14:25:12 +02002582out:
Alex Williamson579305f2014-07-03 09:51:43 -06002583
Joerg Roedel76208352016-08-25 14:25:12 +02002584 return domain;
2585}
2586
2587static struct dmar_domain *set_domain_for_dev(struct device *dev,
2588 struct dmar_domain *domain)
2589{
2590 struct intel_iommu *iommu;
2591 struct dmar_domain *tmp;
2592 u16 req_id, dma_alias;
2593 u8 bus, devfn;
2594
2595 iommu = device_to_iommu(dev, &bus, &devfn);
2596 if (!iommu)
2597 return NULL;
2598
2599 req_id = ((u16)bus << 8) | devfn;
2600
2601 if (dev_is_pci(dev)) {
2602 struct pci_dev *pdev = to_pci_dev(dev);
2603
2604 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2605
2606 /* register PCI DMA alias device */
2607 if (req_id != dma_alias) {
2608 tmp = dmar_insert_one_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2609 dma_alias & 0xff, NULL, domain);
2610
2611 if (!tmp || tmp != domain)
2612 return tmp;
Alex Williamson579305f2014-07-03 09:51:43 -06002613 }
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002614 }
2615
Joerg Roedel5db31562015-07-22 12:40:43 +02002616 tmp = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
Joerg Roedel76208352016-08-25 14:25:12 +02002617 if (!tmp || tmp != domain)
2618 return tmp;
Alex Williamson579305f2014-07-03 09:51:43 -06002619
Joerg Roedel76208352016-08-25 14:25:12 +02002620 return domain;
2621}
2622
2623static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
2624{
2625 struct dmar_domain *domain, *tmp;
2626
2627 domain = find_domain(dev);
2628 if (domain)
2629 goto out;
2630
2631 domain = find_or_alloc_domain(dev, gaw);
2632 if (!domain)
2633 goto out;
2634
2635 tmp = set_domain_for_dev(dev, domain);
2636 if (!tmp || domain != tmp) {
Alex Williamson579305f2014-07-03 09:51:43 -06002637 domain_exit(domain);
2638 domain = tmp;
2639 }
David Woodhouseb718cd32014-03-09 13:11:33 -07002640
Joerg Roedel76208352016-08-25 14:25:12 +02002641out:
2642
David Woodhouseb718cd32014-03-09 13:11:33 -07002643 return domain;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002644}
2645
David Woodhouseb2132032009-06-26 18:50:28 +01002646static int iommu_domain_identity_map(struct dmar_domain *domain,
2647 unsigned long long start,
2648 unsigned long long end)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002649{
David Woodhousec5395d52009-06-28 16:35:56 +01002650 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2651 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002652
David Woodhousec5395d52009-06-28 16:35:56 +01002653 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2654 dma_to_mm_pfn(last_vpfn))) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02002655 pr_err("Reserving iova failed\n");
David Woodhouseb2132032009-06-26 18:50:28 +01002656 return -ENOMEM;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002657 }
2658
Joerg Roedelaf1089c2015-07-21 15:45:19 +02002659 pr_debug("Mapping reserved region %llx-%llx\n", start, end);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002660 /*
2661 * RMRR range might have overlap with physical memory range,
2662 * clear it first
2663 */
David Woodhousec5395d52009-06-28 16:35:56 +01002664 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002665
Peter Xu87684fd2018-05-04 10:34:53 +08002666 return __domain_mapping(domain, first_vpfn, NULL,
2667 first_vpfn, last_vpfn - first_vpfn + 1,
2668 DMA_PTE_READ|DMA_PTE_WRITE);
David Woodhouseb2132032009-06-26 18:50:28 +01002669}
2670
Joerg Roedeld66ce542015-09-23 19:00:10 +02002671static int domain_prepare_identity_map(struct device *dev,
2672 struct dmar_domain *domain,
2673 unsigned long long start,
2674 unsigned long long end)
David Woodhouseb2132032009-06-26 18:50:28 +01002675{
David Woodhouse19943b02009-08-04 16:19:20 +01002676 /* For _hardware_ passthrough, don't bother. But for software
2677 passthrough, we do it anyway -- it may indicate a memory
2678 range which is reserved in E820, so which didn't get set
2679 up to start with in si_domain */
2680 if (domain == si_domain && hw_pass_through) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02002681 pr_warn("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2682 dev_name(dev), start, end);
David Woodhouse19943b02009-08-04 16:19:20 +01002683 return 0;
2684 }
2685
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02002686 pr_info("Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2687 dev_name(dev), start, end);
2688
David Woodhouse5595b522009-12-02 09:21:55 +00002689 if (end < start) {
2690 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2691 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2692 dmi_get_system_info(DMI_BIOS_VENDOR),
2693 dmi_get_system_info(DMI_BIOS_VERSION),
2694 dmi_get_system_info(DMI_PRODUCT_VERSION));
Joerg Roedeld66ce542015-09-23 19:00:10 +02002695 return -EIO;
David Woodhouse5595b522009-12-02 09:21:55 +00002696 }
2697
David Woodhouse2ff729f2009-08-26 14:25:41 +01002698 if (end >> agaw_to_width(domain->agaw)) {
2699 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2700 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2701 agaw_to_width(domain->agaw),
2702 dmi_get_system_info(DMI_BIOS_VENDOR),
2703 dmi_get_system_info(DMI_BIOS_VERSION),
2704 dmi_get_system_info(DMI_PRODUCT_VERSION));
Joerg Roedeld66ce542015-09-23 19:00:10 +02002705 return -EIO;
David Woodhouse2ff729f2009-08-26 14:25:41 +01002706 }
David Woodhouse19943b02009-08-04 16:19:20 +01002707
Joerg Roedeld66ce542015-09-23 19:00:10 +02002708 return iommu_domain_identity_map(domain, start, end);
2709}
2710
2711static int iommu_prepare_identity_map(struct device *dev,
2712 unsigned long long start,
2713 unsigned long long end)
2714{
2715 struct dmar_domain *domain;
2716 int ret;
2717
2718 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2719 if (!domain)
2720 return -ENOMEM;
2721
2722 ret = domain_prepare_identity_map(dev, domain, start, end);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002723 if (ret)
Joerg Roedeld66ce542015-09-23 19:00:10 +02002724 domain_exit(domain);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002725
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002726 return ret;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002727}
2728
2729static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
David Woodhouse0b9d9752014-03-09 15:48:15 -07002730 struct device *dev)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002731{
David Woodhouse0b9d9752014-03-09 15:48:15 -07002732 if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002733 return 0;
David Woodhouse0b9d9752014-03-09 15:48:15 -07002734 return iommu_prepare_identity_map(dev, rmrr->base_address,
2735 rmrr->end_address);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002736}
2737
Suresh Siddhad3f13812011-08-23 17:05:25 -07002738#ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
Keshavamurthy, Anil S49a04292007-10-21 16:41:57 -07002739static inline void iommu_prepare_isa(void)
2740{
2741 struct pci_dev *pdev;
2742 int ret;
2743
2744 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2745 if (!pdev)
2746 return;
2747
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02002748 pr_info("Prepare 0-16MiB unity mapping for LPC\n");
David Woodhouse0b9d9752014-03-09 15:48:15 -07002749 ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
Keshavamurthy, Anil S49a04292007-10-21 16:41:57 -07002750
2751 if (ret)
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02002752 pr_err("Failed to create 0-16MiB identity map - floppy might not work\n");
Keshavamurthy, Anil S49a04292007-10-21 16:41:57 -07002753
Yijing Wang9b27e822014-05-20 20:37:52 +08002754 pci_dev_put(pdev);
Keshavamurthy, Anil S49a04292007-10-21 16:41:57 -07002755}
2756#else
2757static inline void iommu_prepare_isa(void)
2758{
2759 return;
2760}
Suresh Siddhad3f13812011-08-23 17:05:25 -07002761#endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
Keshavamurthy, Anil S49a04292007-10-21 16:41:57 -07002762
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002763static int md_domain_init(struct dmar_domain *domain, int guest_width);
David Woodhousec7ab48d2009-06-26 19:10:36 +01002764
Matt Kraai071e1372009-08-23 22:30:22 -07002765static int __init si_domain_init(int hw)
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002766{
David Woodhousec7ab48d2009-06-26 19:10:36 +01002767 int nid, ret = 0;
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002768
Jiang Liuab8dfe22014-07-11 14:19:27 +08002769 si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002770 if (!si_domain)
2771 return -EFAULT;
2772
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002773 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2774 domain_exit(si_domain);
2775 return -EFAULT;
2776 }
2777
Joerg Roedel0dc79712015-07-21 15:40:06 +02002778 pr_debug("Identity mapping domain allocated\n");
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002779
David Woodhouse19943b02009-08-04 16:19:20 +01002780 if (hw)
2781 return 0;
2782
David Woodhousec7ab48d2009-06-26 19:10:36 +01002783 for_each_online_node(nid) {
Tejun Heod4bbf7e2011-11-28 09:46:22 -08002784 unsigned long start_pfn, end_pfn;
2785 int i;
2786
2787 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2788 ret = iommu_domain_identity_map(si_domain,
2789 PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2790 if (ret)
2791 return ret;
2792 }
David Woodhousec7ab48d2009-06-26 19:10:36 +01002793 }
2794
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002795 return 0;
2796}
2797
David Woodhouse9b226622014-03-09 14:03:28 -07002798static int identity_mapping(struct device *dev)
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002799{
2800 struct device_domain_info *info;
2801
2802 if (likely(!iommu_identity_mapping))
2803 return 0;
2804
David Woodhouse9b226622014-03-09 14:03:28 -07002805 info = dev->archdata.iommu;
Mike Traviscb452a42011-05-28 13:15:03 -05002806 if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2807 return (info->domain == si_domain);
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002808
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002809 return 0;
2810}
2811
Joerg Roedel28ccce02015-07-21 14:45:31 +02002812static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002813{
David Woodhouse0ac72662014-03-09 13:19:22 -07002814 struct dmar_domain *ndomain;
David Woodhouse5a8f40e2014-03-09 13:31:18 -07002815 struct intel_iommu *iommu;
David Woodhouse156baca2014-03-09 14:00:57 -07002816 u8 bus, devfn;
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002817
David Woodhouse5913c9b2014-03-09 16:27:31 -07002818 iommu = device_to_iommu(dev, &bus, &devfn);
David Woodhouse5a8f40e2014-03-09 13:31:18 -07002819 if (!iommu)
2820 return -ENODEV;
2821
Joerg Roedel5db31562015-07-22 12:40:43 +02002822 ndomain = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
David Woodhouse0ac72662014-03-09 13:19:22 -07002823 if (ndomain != domain)
2824 return -EBUSY;
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002825
2826 return 0;
2827}
2828
David Woodhouse0b9d9752014-03-09 15:48:15 -07002829static bool device_has_rmrr(struct device *dev)
Tom Mingarelliea2447f72012-11-20 19:43:17 +00002830{
2831 struct dmar_rmrr_unit *rmrr;
David Woodhouse832bd852014-03-07 15:08:36 +00002832 struct device *tmp;
Tom Mingarelliea2447f72012-11-20 19:43:17 +00002833 int i;
2834
Jiang Liu0e242612014-02-19 14:07:34 +08002835 rcu_read_lock();
Tom Mingarelliea2447f72012-11-20 19:43:17 +00002836 for_each_rmrr_units(rmrr) {
Jiang Liub683b232014-02-19 14:07:32 +08002837 /*
2838 * Return TRUE if this RMRR contains the device that
2839 * is passed in.
2840 */
2841 for_each_active_dev_scope(rmrr->devices,
2842 rmrr->devices_cnt, i, tmp)
David Woodhouse0b9d9752014-03-09 15:48:15 -07002843 if (tmp == dev) {
Jiang Liu0e242612014-02-19 14:07:34 +08002844 rcu_read_unlock();
Tom Mingarelliea2447f72012-11-20 19:43:17 +00002845 return true;
Jiang Liub683b232014-02-19 14:07:32 +08002846 }
Tom Mingarelliea2447f72012-11-20 19:43:17 +00002847 }
Jiang Liu0e242612014-02-19 14:07:34 +08002848 rcu_read_unlock();
Tom Mingarelliea2447f72012-11-20 19:43:17 +00002849 return false;
2850}
2851
Alex Williamsonc875d2c2014-07-03 09:57:02 -06002852/*
2853 * There are a couple cases where we need to restrict the functionality of
2854 * devices associated with RMRRs. The first is when evaluating a device for
2855 * identity mapping because problems exist when devices are moved in and out
2856 * of domains and their respective RMRR information is lost. This means that
2857 * a device with associated RMRRs will never be in a "passthrough" domain.
2858 * The second is use of the device through the IOMMU API. This interface
2859 * expects to have full control of the IOVA space for the device. We cannot
2860 * satisfy both the requirement that RMRR access is maintained and have an
2861 * unencumbered IOVA space. We also have no ability to quiesce the device's
2862 * use of the RMRR space or even inform the IOMMU API user of the restriction.
2863 * We therefore prevent devices associated with an RMRR from participating in
2864 * the IOMMU API, which eliminates them from device assignment.
2865 *
2866 * In both cases we assume that PCI USB devices with RMRRs have them largely
2867 * for historical reasons and that the RMRR space is not actively used post
2868 * boot. This exclusion may change if vendors begin to abuse it.
David Woodhouse18436af2015-03-25 15:05:47 +00002869 *
2870 * The same exception is made for graphics devices, with the requirement that
2871 * any use of the RMRR regions will be torn down before assigning the device
2872 * to a guest.
Alex Williamsonc875d2c2014-07-03 09:57:02 -06002873 */
2874static bool device_is_rmrr_locked(struct device *dev)
2875{
2876 if (!device_has_rmrr(dev))
2877 return false;
2878
2879 if (dev_is_pci(dev)) {
2880 struct pci_dev *pdev = to_pci_dev(dev);
2881
David Woodhouse18436af2015-03-25 15:05:47 +00002882 if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
Alex Williamsonc875d2c2014-07-03 09:57:02 -06002883 return false;
2884 }
2885
2886 return true;
2887}
2888
David Woodhouse3bdb2592014-03-09 16:03:08 -07002889static int iommu_should_identity_map(struct device *dev, int startup)
David Woodhouse6941af22009-07-04 18:24:27 +01002890{
Tom Mingarelliea2447f72012-11-20 19:43:17 +00002891
David Woodhouse3bdb2592014-03-09 16:03:08 -07002892 if (dev_is_pci(dev)) {
2893 struct pci_dev *pdev = to_pci_dev(dev);
Tom Mingarelliea2447f72012-11-20 19:43:17 +00002894
Alex Williamsonc875d2c2014-07-03 09:57:02 -06002895 if (device_is_rmrr_locked(dev))
David Woodhouse3bdb2592014-03-09 16:03:08 -07002896 return 0;
David Woodhousee0fc7e02009-09-30 09:12:17 -07002897
David Woodhouse3bdb2592014-03-09 16:03:08 -07002898 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2899 return 1;
David Woodhousee0fc7e02009-09-30 09:12:17 -07002900
David Woodhouse3bdb2592014-03-09 16:03:08 -07002901 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2902 return 1;
2903
2904 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2905 return 0;
2906
2907 /*
2908 * We want to start off with all devices in the 1:1 domain, and
2909 * take them out later if we find they can't access all of memory.
2910 *
2911 * However, we can't do this for PCI devices behind bridges,
2912 * because all PCI devices behind the same bridge will end up
2913 * with the same source-id on their transactions.
2914 *
2915 * Practically speaking, we can't change things around for these
2916 * devices at run-time, because we can't be sure there'll be no
2917 * DMA transactions in flight for any of their siblings.
2918 *
2919 * So PCI devices (unless they're on the root bus) as well as
2920 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2921 * the 1:1 domain, just in _case_ one of their siblings turns out
2922 * not to be able to map all of memory.
2923 */
2924 if (!pci_is_pcie(pdev)) {
2925 if (!pci_is_root_bus(pdev->bus))
2926 return 0;
2927 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2928 return 0;
2929 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
2930 return 0;
2931 } else {
2932 if (device_has_rmrr(dev))
2933 return 0;
2934 }
David Woodhouse6941af22009-07-04 18:24:27 +01002935
David Woodhouse3dfc8132009-07-04 19:11:08 +01002936 /*
David Woodhouse3dfc8132009-07-04 19:11:08 +01002937 * At boot time, we don't yet know if devices will be 64-bit capable.
David Woodhouse3bdb2592014-03-09 16:03:08 -07002938 * Assume that they will — if they turn out not to be, then we can
David Woodhouse3dfc8132009-07-04 19:11:08 +01002939 * take them out of the 1:1 domain later.
2940 */
Chris Wright8fcc5372011-05-28 13:15:02 -05002941 if (!startup) {
2942 /*
2943 * If the device's dma_mask is less than the system's memory
2944 * size then this is not a candidate for identity mapping.
2945 */
David Woodhouse3bdb2592014-03-09 16:03:08 -07002946 u64 dma_mask = *dev->dma_mask;
Chris Wright8fcc5372011-05-28 13:15:02 -05002947
David Woodhouse3bdb2592014-03-09 16:03:08 -07002948 if (dev->coherent_dma_mask &&
2949 dev->coherent_dma_mask < dma_mask)
2950 dma_mask = dev->coherent_dma_mask;
Chris Wright8fcc5372011-05-28 13:15:02 -05002951
David Woodhouse3bdb2592014-03-09 16:03:08 -07002952 return dma_mask >= dma_get_required_mask(dev);
Chris Wright8fcc5372011-05-28 13:15:02 -05002953 }
David Woodhouse6941af22009-07-04 18:24:27 +01002954
2955 return 1;
2956}
2957
David Woodhousecf04eee2014-03-21 16:49:04 +00002958static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
2959{
2960 int ret;
2961
2962 if (!iommu_should_identity_map(dev, 1))
2963 return 0;
2964
Joerg Roedel28ccce02015-07-21 14:45:31 +02002965 ret = domain_add_dev_info(si_domain, dev);
David Woodhousecf04eee2014-03-21 16:49:04 +00002966 if (!ret)
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02002967 pr_info("%s identity mapping for device %s\n",
2968 hw ? "Hardware" : "Software", dev_name(dev));
David Woodhousecf04eee2014-03-21 16:49:04 +00002969 else if (ret == -ENODEV)
2970 /* device not associated with an iommu */
2971 ret = 0;
2972
2973 return ret;
2974}
2975
2976
Matt Kraai071e1372009-08-23 22:30:22 -07002977static int __init iommu_prepare_static_identity_mapping(int hw)
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002978{
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002979 struct pci_dev *pdev = NULL;
David Woodhousecf04eee2014-03-21 16:49:04 +00002980 struct dmar_drhd_unit *drhd;
2981 struct intel_iommu *iommu;
2982 struct device *dev;
2983 int i;
2984 int ret = 0;
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002985
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002986 for_each_pci_dev(pdev) {
David Woodhousecf04eee2014-03-21 16:49:04 +00002987 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
2988 if (ret)
2989 return ret;
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002990 }
2991
David Woodhousecf04eee2014-03-21 16:49:04 +00002992 for_each_active_iommu(iommu, drhd)
2993 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
2994 struct acpi_device_physical_node *pn;
2995 struct acpi_device *adev;
2996
2997 if (dev->bus != &acpi_bus_type)
2998 continue;
Joerg Roedel86080cc2015-06-12 12:27:16 +02002999
David Woodhousecf04eee2014-03-21 16:49:04 +00003000 adev= to_acpi_device(dev);
3001 mutex_lock(&adev->physical_node_lock);
3002 list_for_each_entry(pn, &adev->physical_node_list, node) {
3003 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
3004 if (ret)
3005 break;
3006 }
3007 mutex_unlock(&adev->physical_node_lock);
3008 if (ret)
3009 return ret;
3010 }
3011
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07003012 return 0;
3013}
3014
Jiang Liuffebeb42014-11-09 22:48:02 +08003015static void intel_iommu_init_qi(struct intel_iommu *iommu)
3016{
3017 /*
3018 * Start from the sane iommu hardware state.
3019 * If the queued invalidation is already initialized by us
3020 * (for example, while enabling interrupt-remapping) then
3021 * we got the things already rolling from a sane state.
3022 */
3023 if (!iommu->qi) {
3024 /*
3025 * Clear any previous faults.
3026 */
3027 dmar_fault(-1, iommu);
3028 /*
3029 * Disable queued invalidation if supported and already enabled
3030 * before OS handover.
3031 */
3032 dmar_disable_qi(iommu);
3033 }
3034
3035 if (dmar_enable_qi(iommu)) {
3036 /*
3037 * Queued Invalidate not enabled, use Register Based Invalidate
3038 */
3039 iommu->flush.flush_context = __iommu_flush_context;
3040 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02003041 pr_info("%s: Using Register based invalidation\n",
Jiang Liuffebeb42014-11-09 22:48:02 +08003042 iommu->name);
3043 } else {
3044 iommu->flush.flush_context = qi_flush_context;
3045 iommu->flush.flush_iotlb = qi_flush_iotlb;
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02003046 pr_info("%s: Using Queued invalidation\n", iommu->name);
Jiang Liuffebeb42014-11-09 22:48:02 +08003047 }
3048}
3049
Joerg Roedel091d42e2015-06-12 11:56:10 +02003050static int copy_context_table(struct intel_iommu *iommu,
Dan Williamsdfddb962015-10-09 18:16:46 -04003051 struct root_entry *old_re,
Joerg Roedel091d42e2015-06-12 11:56:10 +02003052 struct context_entry **tbl,
3053 int bus, bool ext)
3054{
Joerg Roedeldbcd8612015-06-12 12:02:09 +02003055 int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
Joerg Roedel543c8dc2015-08-13 11:56:59 +02003056 struct context_entry *new_ce = NULL, ce;
Dan Williamsdfddb962015-10-09 18:16:46 -04003057 struct context_entry *old_ce = NULL;
Joerg Roedel543c8dc2015-08-13 11:56:59 +02003058 struct root_entry re;
Joerg Roedel091d42e2015-06-12 11:56:10 +02003059 phys_addr_t old_ce_phys;
3060
3061 tbl_idx = ext ? bus * 2 : bus;
Dan Williamsdfddb962015-10-09 18:16:46 -04003062 memcpy(&re, old_re, sizeof(re));
Joerg Roedel091d42e2015-06-12 11:56:10 +02003063
3064 for (devfn = 0; devfn < 256; devfn++) {
3065 /* First calculate the correct index */
3066 idx = (ext ? devfn * 2 : devfn) % 256;
3067
3068 if (idx == 0) {
3069 /* First save what we may have and clean up */
3070 if (new_ce) {
3071 tbl[tbl_idx] = new_ce;
3072 __iommu_flush_cache(iommu, new_ce,
3073 VTD_PAGE_SIZE);
3074 pos = 1;
3075 }
3076
3077 if (old_ce)
Pan Bian829383e2018-11-21 17:53:47 +08003078 memunmap(old_ce);
Joerg Roedel091d42e2015-06-12 11:56:10 +02003079
3080 ret = 0;
3081 if (devfn < 0x80)
Joerg Roedel543c8dc2015-08-13 11:56:59 +02003082 old_ce_phys = root_entry_lctp(&re);
Joerg Roedel091d42e2015-06-12 11:56:10 +02003083 else
Joerg Roedel543c8dc2015-08-13 11:56:59 +02003084 old_ce_phys = root_entry_uctp(&re);
Joerg Roedel091d42e2015-06-12 11:56:10 +02003085
3086 if (!old_ce_phys) {
3087 if (ext && devfn == 0) {
3088 /* No LCTP, try UCTP */
3089 devfn = 0x7f;
3090 continue;
3091 } else {
3092 goto out;
3093 }
3094 }
3095
3096 ret = -ENOMEM;
Dan Williamsdfddb962015-10-09 18:16:46 -04003097 old_ce = memremap(old_ce_phys, PAGE_SIZE,
3098 MEMREMAP_WB);
Joerg Roedel091d42e2015-06-12 11:56:10 +02003099 if (!old_ce)
3100 goto out;
3101
3102 new_ce = alloc_pgtable_page(iommu->node);
3103 if (!new_ce)
3104 goto out_unmap;
3105
3106 ret = 0;
3107 }
3108
3109 /* Now copy the context entry */
Dan Williamsdfddb962015-10-09 18:16:46 -04003110 memcpy(&ce, old_ce + idx, sizeof(ce));
Joerg Roedel091d42e2015-06-12 11:56:10 +02003111
Joerg Roedelcf484d02015-06-12 12:21:46 +02003112 if (!__context_present(&ce))
Joerg Roedel091d42e2015-06-12 11:56:10 +02003113 continue;
3114
Joerg Roedeldbcd8612015-06-12 12:02:09 +02003115 did = context_domain_id(&ce);
3116 if (did >= 0 && did < cap_ndoms(iommu->cap))
3117 set_bit(did, iommu->domain_ids);
3118
Joerg Roedelcf484d02015-06-12 12:21:46 +02003119 /*
3120 * We need a marker for copied context entries. This
3121 * marker needs to work for the old format as well as
3122 * for extended context entries.
3123 *
3124 * Bit 67 of the context entry is used. In the old
3125 * format this bit is available to software, in the
3126 * extended format it is the PGE bit, but PGE is ignored
3127 * by HW if PASIDs are disabled (and thus still
3128 * available).
3129 *
3130 * So disable PASIDs first and then mark the entry
3131 * copied. This means that we don't copy PASID
3132 * translations from the old kernel, but this is fine as
3133 * faults there are not fatal.
3134 */
3135 context_clear_pasid_enable(&ce);
3136 context_set_copied(&ce);
3137
Joerg Roedel091d42e2015-06-12 11:56:10 +02003138 new_ce[idx] = ce;
3139 }
3140
3141 tbl[tbl_idx + pos] = new_ce;
3142
3143 __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
3144
3145out_unmap:
Dan Williamsdfddb962015-10-09 18:16:46 -04003146 memunmap(old_ce);
Joerg Roedel091d42e2015-06-12 11:56:10 +02003147
3148out:
3149 return ret;
3150}
3151
3152static int copy_translation_tables(struct intel_iommu *iommu)
3153{
3154 struct context_entry **ctxt_tbls;
Dan Williamsdfddb962015-10-09 18:16:46 -04003155 struct root_entry *old_rt;
Joerg Roedel091d42e2015-06-12 11:56:10 +02003156 phys_addr_t old_rt_phys;
3157 int ctxt_table_entries;
3158 unsigned long flags;
3159 u64 rtaddr_reg;
3160 int bus, ret;
Joerg Roedelc3361f22015-06-12 12:39:25 +02003161 bool new_ext, ext;
Joerg Roedel091d42e2015-06-12 11:56:10 +02003162
3163 rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
3164 ext = !!(rtaddr_reg & DMA_RTADDR_RTT);
Joerg Roedelc3361f22015-06-12 12:39:25 +02003165 new_ext = !!ecap_ecs(iommu->ecap);
3166
3167 /*
3168 * The RTT bit can only be changed when translation is disabled,
3169 * but disabling translation means to open a window for data
3170 * corruption. So bail out and don't copy anything if we would
3171 * have to change the bit.
3172 */
3173 if (new_ext != ext)
3174 return -EINVAL;
Joerg Roedel091d42e2015-06-12 11:56:10 +02003175
3176 old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
3177 if (!old_rt_phys)
3178 return -EINVAL;
3179
Dan Williamsdfddb962015-10-09 18:16:46 -04003180 old_rt = memremap(old_rt_phys, PAGE_SIZE, MEMREMAP_WB);
Joerg Roedel091d42e2015-06-12 11:56:10 +02003181 if (!old_rt)
3182 return -ENOMEM;
3183
3184 /* This is too big for the stack - allocate it from slab */
3185 ctxt_table_entries = ext ? 512 : 256;
3186 ret = -ENOMEM;
Kees Cook6396bb22018-06-12 14:03:40 -07003187 ctxt_tbls = kcalloc(ctxt_table_entries, sizeof(void *), GFP_KERNEL);
Joerg Roedel091d42e2015-06-12 11:56:10 +02003188 if (!ctxt_tbls)
3189 goto out_unmap;
3190
3191 for (bus = 0; bus < 256; bus++) {
3192 ret = copy_context_table(iommu, &old_rt[bus],
3193 ctxt_tbls, bus, ext);
3194 if (ret) {
3195 pr_err("%s: Failed to copy context table for bus %d\n",
3196 iommu->name, bus);
3197 continue;
3198 }
3199 }
3200
3201 spin_lock_irqsave(&iommu->lock, flags);
3202
3203 /* Context tables are copied, now write them to the root_entry table */
3204 for (bus = 0; bus < 256; bus++) {
3205 int idx = ext ? bus * 2 : bus;
3206 u64 val;
3207
3208 if (ctxt_tbls[idx]) {
3209 val = virt_to_phys(ctxt_tbls[idx]) | 1;
3210 iommu->root_entry[bus].lo = val;
3211 }
3212
3213 if (!ext || !ctxt_tbls[idx + 1])
3214 continue;
3215
3216 val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
3217 iommu->root_entry[bus].hi = val;
3218 }
3219
3220 spin_unlock_irqrestore(&iommu->lock, flags);
3221
3222 kfree(ctxt_tbls);
3223
3224 __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
3225
3226 ret = 0;
3227
3228out_unmap:
Dan Williamsdfddb962015-10-09 18:16:46 -04003229 memunmap(old_rt);
Joerg Roedel091d42e2015-06-12 11:56:10 +02003230
3231 return ret;
3232}
3233
Joseph Cihulab7792602011-05-03 00:08:37 -07003234static int __init init_dmars(void)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003235{
3236 struct dmar_drhd_unit *drhd;
3237 struct dmar_rmrr_unit *rmrr;
Joerg Roedela87f4912015-06-12 12:32:54 +02003238 bool copied_tables = false;
David Woodhouse832bd852014-03-07 15:08:36 +00003239 struct device *dev;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003240 struct intel_iommu *iommu;
Joerg Roedel13cf0172017-08-11 11:40:10 +02003241 int i, ret;
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07003242
3243 /*
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003244 * for each drhd
3245 * allocate root
3246 * initialize and program root entry to not present
3247 * endfor
3248 */
3249 for_each_drhd_unit(drhd) {
mark gross5e0d2a62008-03-04 15:22:08 -08003250 /*
3251 * lock not needed as this is only incremented in the single
3252 * threaded kernel __init code path all other access are read
3253 * only
3254 */
Jiang Liu78d8e702014-11-09 22:47:57 +08003255 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED) {
Mike Travis1b198bb2012-03-05 15:05:16 -08003256 g_num_of_iommus++;
3257 continue;
3258 }
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02003259 pr_err_once("Exceeded %d IOMMUs\n", DMAR_UNITS_SUPPORTED);
mark gross5e0d2a62008-03-04 15:22:08 -08003260 }
3261
Jiang Liuffebeb42014-11-09 22:48:02 +08003262 /* Preallocate enough resources for IOMMU hot-addition */
3263 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED)
3264 g_num_of_iommus = DMAR_UNITS_SUPPORTED;
3265
Weidong Hand9630fe2008-12-08 11:06:32 +08003266 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
3267 GFP_KERNEL);
3268 if (!g_iommus) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02003269 pr_err("Allocating global iommu array failed\n");
Weidong Hand9630fe2008-12-08 11:06:32 +08003270 ret = -ENOMEM;
3271 goto error;
3272 }
3273
Jiang Liu7c919772014-01-06 14:18:18 +08003274 for_each_active_iommu(iommu, drhd) {
Lu Baolu56283172018-07-14 15:46:54 +08003275 /*
3276 * Find the max pasid size of all IOMMU's in the system.
3277 * We need to ensure the system pasid table is no bigger
3278 * than the smallest supported.
3279 */
3280 if (pasid_enabled(iommu)) {
3281 u32 temp = 2 << ecap_pss(iommu->ecap);
3282
3283 intel_pasid_max_id = min_t(u32, temp,
3284 intel_pasid_max_id);
3285 }
3286
Weidong Hand9630fe2008-12-08 11:06:32 +08003287 g_iommus[iommu->seq_id] = iommu;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003288
Joerg Roedelb63d80d2015-06-12 09:14:34 +02003289 intel_iommu_init_qi(iommu);
3290
Suresh Siddhae61d98d2008-07-10 11:16:35 -07003291 ret = iommu_init_domains(iommu);
3292 if (ret)
Jiang Liu989d51f2014-02-19 14:07:21 +08003293 goto free_iommu;
Suresh Siddhae61d98d2008-07-10 11:16:35 -07003294
Joerg Roedel4158c2e2015-06-12 10:14:02 +02003295 init_translation_status(iommu);
3296
Joerg Roedel091d42e2015-06-12 11:56:10 +02003297 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
3298 iommu_disable_translation(iommu);
3299 clear_translation_pre_enabled(iommu);
3300 pr_warn("Translation was enabled for %s but we are not in kdump mode\n",
3301 iommu->name);
3302 }
Joerg Roedel4158c2e2015-06-12 10:14:02 +02003303
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003304 /*
3305 * TBD:
3306 * we could share the same root & context tables
Lucas De Marchi25985ed2011-03-30 22:57:33 -03003307 * among all IOMMU's. Need to Split it later.
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003308 */
3309 ret = iommu_alloc_root_entry(iommu);
Jiang Liuffebeb42014-11-09 22:48:02 +08003310 if (ret)
Jiang Liu989d51f2014-02-19 14:07:21 +08003311 goto free_iommu;
Joerg Roedel5f0a7f72015-06-12 09:18:53 +02003312
Joerg Roedel091d42e2015-06-12 11:56:10 +02003313 if (translation_pre_enabled(iommu)) {
3314 pr_info("Translation already enabled - trying to copy translation structures\n");
3315
3316 ret = copy_translation_tables(iommu);
3317 if (ret) {
3318 /*
3319 * We found the IOMMU with translation
3320 * enabled - but failed to copy over the
3321 * old root-entry table. Try to proceed
3322 * by disabling translation now and
3323 * allocating a clean root-entry table.
3324 * This might cause DMAR faults, but
3325 * probably the dump will still succeed.
3326 */
3327 pr_err("Failed to copy translation tables from previous kernel for %s\n",
3328 iommu->name);
3329 iommu_disable_translation(iommu);
3330 clear_translation_pre_enabled(iommu);
3331 } else {
3332 pr_info("Copied translation tables from previous kernel for %s\n",
3333 iommu->name);
Joerg Roedela87f4912015-06-12 12:32:54 +02003334 copied_tables = true;
Joerg Roedel091d42e2015-06-12 11:56:10 +02003335 }
3336 }
3337
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07003338 if (!ecap_pass_through(iommu->ecap))
David Woodhouse19943b02009-08-04 16:19:20 +01003339 hw_pass_through = 0;
David Woodhouse8a94ade2015-03-24 14:54:56 +00003340#ifdef CONFIG_INTEL_IOMMU_SVM
3341 if (pasid_enabled(iommu))
Lu Baolud9737952018-07-14 15:47:02 +08003342 intel_svm_init(iommu);
David Woodhouse8a94ade2015-03-24 14:54:56 +00003343#endif
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003344 }
3345
Joerg Roedela4c34ff2016-06-17 11:29:48 +02003346 /*
3347 * Now that qi is enabled on all iommus, set the root entry and flush
3348 * caches. This is required on some Intel X58 chipsets, otherwise the
3349 * flush_context function will loop forever and the boot hangs.
3350 */
3351 for_each_active_iommu(iommu, drhd) {
3352 iommu_flush_write_buffer(iommu);
3353 iommu_set_root_entry(iommu);
3354 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
3355 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3356 }
3357
David Woodhouse19943b02009-08-04 16:19:20 +01003358 if (iommu_pass_through)
David Woodhousee0fc7e02009-09-30 09:12:17 -07003359 iommu_identity_mapping |= IDENTMAP_ALL;
3360
Suresh Siddhad3f13812011-08-23 17:05:25 -07003361#ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
David Woodhousee0fc7e02009-09-30 09:12:17 -07003362 iommu_identity_mapping |= IDENTMAP_GFX;
David Woodhouse19943b02009-08-04 16:19:20 +01003363#endif
David Woodhousee0fc7e02009-09-30 09:12:17 -07003364
Ashok Raj21e722c2017-01-30 09:39:53 -08003365 check_tylersburg_isoch();
3366
Joerg Roedel86080cc2015-06-12 12:27:16 +02003367 if (iommu_identity_mapping) {
3368 ret = si_domain_init(hw_pass_through);
3369 if (ret)
3370 goto free_iommu;
3371 }
3372
David Woodhousee0fc7e02009-09-30 09:12:17 -07003373
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07003374 /*
Joerg Roedela87f4912015-06-12 12:32:54 +02003375 * If we copied translations from a previous kernel in the kdump
3376 * case, we can not assign the devices to domains now, as that
3377 * would eliminate the old mappings. So skip this part and defer
3378 * the assignment to device driver initialization time.
3379 */
3380 if (copied_tables)
3381 goto domains_done;
3382
3383 /*
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07003384 * If pass through is not set or not enabled, setup context entries for
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07003385 * identity mappings for rmrr, gfx, and isa and may fall back to static
3386 * identity mapping if iommu_identity_mapping is set.
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07003387 */
David Woodhouse19943b02009-08-04 16:19:20 +01003388 if (iommu_identity_mapping) {
3389 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
3390 if (ret) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02003391 pr_crit("Failed to setup IOMMU pass-through\n");
Jiang Liu989d51f2014-02-19 14:07:21 +08003392 goto free_iommu;
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07003393 }
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07003394 }
David Woodhouse19943b02009-08-04 16:19:20 +01003395 /*
3396 * For each rmrr
3397 * for each dev attached to rmrr
3398 * do
3399 * locate drhd for dev, alloc domain for dev
3400 * allocate free domain
3401 * allocate page table entries for rmrr
3402 * if context not allocated for bus
3403 * allocate and init context
3404 * set present in root table for this bus
3405 * init context with domain, translation etc
3406 * endfor
3407 * endfor
3408 */
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02003409 pr_info("Setting RMRR:\n");
David Woodhouse19943b02009-08-04 16:19:20 +01003410 for_each_rmrr_units(rmrr) {
Jiang Liub683b232014-02-19 14:07:32 +08003411 /* some BIOS lists non-exist devices in DMAR table. */
3412 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
David Woodhouse832bd852014-03-07 15:08:36 +00003413 i, dev) {
David Woodhouse0b9d9752014-03-09 15:48:15 -07003414 ret = iommu_prepare_rmrr_dev(rmrr, dev);
David Woodhouse19943b02009-08-04 16:19:20 +01003415 if (ret)
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02003416 pr_err("Mapping reserved region failed\n");
David Woodhouse19943b02009-08-04 16:19:20 +01003417 }
3418 }
3419
3420 iommu_prepare_isa();
Keshavamurthy, Anil S49a04292007-10-21 16:41:57 -07003421
Joerg Roedela87f4912015-06-12 12:32:54 +02003422domains_done:
3423
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003424 /*
3425 * for each drhd
3426 * enable fault log
3427 * global invalidate context cache
3428 * global invalidate iotlb
3429 * enable translation
3430 */
Jiang Liu7c919772014-01-06 14:18:18 +08003431 for_each_iommu(iommu, drhd) {
Joseph Cihula51a63e62011-03-21 11:04:24 -07003432 if (drhd->ignored) {
3433 /*
3434 * we always have to disable PMRs or DMA may fail on
3435 * this device
3436 */
3437 if (force_on)
Jiang Liu7c919772014-01-06 14:18:18 +08003438 iommu_disable_protect_mem_regions(iommu);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003439 continue;
Joseph Cihula51a63e62011-03-21 11:04:24 -07003440 }
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003441
3442 iommu_flush_write_buffer(iommu);
3443
David Woodhousea222a7f2015-10-07 23:35:18 +01003444#ifdef CONFIG_INTEL_IOMMU_SVM
3445 if (pasid_enabled(iommu) && ecap_prs(iommu->ecap)) {
3446 ret = intel_svm_enable_prq(iommu);
3447 if (ret)
3448 goto free_iommu;
3449 }
3450#endif
Keshavamurthy, Anil S3460a6d2007-10-21 16:41:54 -07003451 ret = dmar_set_interrupt(iommu);
3452 if (ret)
Jiang Liu989d51f2014-02-19 14:07:21 +08003453 goto free_iommu;
Keshavamurthy, Anil S3460a6d2007-10-21 16:41:54 -07003454
Joerg Roedel8939ddf2015-06-12 14:40:01 +02003455 if (!translation_pre_enabled(iommu))
3456 iommu_enable_translation(iommu);
3457
David Woodhouseb94996c2009-09-19 15:28:12 -07003458 iommu_disable_protect_mem_regions(iommu);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003459 }
3460
3461 return 0;
Jiang Liu989d51f2014-02-19 14:07:21 +08003462
3463free_iommu:
Jiang Liuffebeb42014-11-09 22:48:02 +08003464 for_each_active_iommu(iommu, drhd) {
3465 disable_dmar_iommu(iommu);
Jiang Liua868e6b2014-01-06 14:18:20 +08003466 free_dmar_iommu(iommu);
Jiang Liuffebeb42014-11-09 22:48:02 +08003467 }
Joerg Roedel13cf0172017-08-11 11:40:10 +02003468
Weidong Hand9630fe2008-12-08 11:06:32 +08003469 kfree(g_iommus);
Joerg Roedel13cf0172017-08-11 11:40:10 +02003470
Jiang Liu989d51f2014-02-19 14:07:21 +08003471error:
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003472 return ret;
3473}
3474
David Woodhouse5a5e02a2009-07-04 09:35:44 +01003475/* This takes a number of _MM_ pages, not VTD pages */
Omer Peleg2aac6302016-04-20 11:33:57 +03003476static unsigned long intel_alloc_iova(struct device *dev,
David Woodhouse875764d2009-06-28 21:20:51 +01003477 struct dmar_domain *domain,
3478 unsigned long nrpages, uint64_t dma_mask)
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003479{
Omer Peleg22e2f9f2016-04-20 11:34:11 +03003480 unsigned long iova_pfn = 0;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003481
David Woodhouse875764d2009-06-28 21:20:51 +01003482 /* Restrict dma_mask to the width that the iommu can handle */
3483 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
Robin Murphy8f6429c2015-07-16 19:40:12 +01003484 /* Ensure we reserve the whole size-aligned region */
3485 nrpages = __roundup_pow_of_two(nrpages);
David Woodhouse875764d2009-06-28 21:20:51 +01003486
3487 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003488 /*
3489 * First try to allocate an io virtual address in
Yang Hongyang284901a2009-04-06 19:01:15 -07003490 * DMA_BIT_MASK(32) and if that fails then try allocating
Joe Perches36098012007-12-17 11:40:11 -08003491 * from higher range
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003492 */
Omer Peleg22e2f9f2016-04-20 11:34:11 +03003493 iova_pfn = alloc_iova_fast(&domain->iovad, nrpages,
Tomasz Nowicki538d5b32017-09-20 10:52:02 +02003494 IOVA_PFN(DMA_BIT_MASK(32)), false);
Omer Peleg22e2f9f2016-04-20 11:34:11 +03003495 if (iova_pfn)
3496 return iova_pfn;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003497 }
Tomasz Nowicki538d5b32017-09-20 10:52:02 +02003498 iova_pfn = alloc_iova_fast(&domain->iovad, nrpages,
3499 IOVA_PFN(dma_mask), true);
Omer Peleg22e2f9f2016-04-20 11:34:11 +03003500 if (unlikely(!iova_pfn)) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02003501 pr_err("Allocating %ld-page iova for %s failed",
David Woodhouse207e3592014-03-09 16:12:32 -07003502 nrpages, dev_name(dev));
Omer Peleg2aac6302016-04-20 11:33:57 +03003503 return 0;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003504 }
3505
Omer Peleg22e2f9f2016-04-20 11:34:11 +03003506 return iova_pfn;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003507}
3508
Lu Baolu9ddbfb42018-07-14 15:46:57 +08003509struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003510{
Joerg Roedel1c5ebba2016-08-25 13:52:51 +02003511 struct dmar_domain *domain, *tmp;
Joerg Roedelb1ce5b72015-09-23 19:16:01 +02003512 struct dmar_rmrr_unit *rmrr;
Joerg Roedelb1ce5b72015-09-23 19:16:01 +02003513 struct device *i_dev;
3514 int i, ret;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003515
Joerg Roedel1c5ebba2016-08-25 13:52:51 +02003516 domain = find_domain(dev);
3517 if (domain)
3518 goto out;
3519
3520 domain = find_or_alloc_domain(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
3521 if (!domain)
3522 goto out;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003523
Joerg Roedelb1ce5b72015-09-23 19:16:01 +02003524 /* We have a new domain - setup possible RMRRs for the device */
3525 rcu_read_lock();
3526 for_each_rmrr_units(rmrr) {
3527 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3528 i, i_dev) {
3529 if (i_dev != dev)
3530 continue;
3531
3532 ret = domain_prepare_identity_map(dev, domain,
3533 rmrr->base_address,
3534 rmrr->end_address);
3535 if (ret)
3536 dev_err(dev, "Mapping reserved region failed\n");
3537 }
3538 }
3539 rcu_read_unlock();
3540
Joerg Roedel1c5ebba2016-08-25 13:52:51 +02003541 tmp = set_domain_for_dev(dev, domain);
3542 if (!tmp || domain != tmp) {
3543 domain_exit(domain);
3544 domain = tmp;
3545 }
3546
3547out:
3548
3549 if (!domain)
3550 pr_err("Allocating domain for %s failed\n", dev_name(dev));
3551
3552
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003553 return domain;
3554}
3555
David Woodhouseecb509e2014-03-09 16:29:55 -07003556/* Check if the dev needs to go through non-identity map and unmap process.*/
David Woodhouse73676832009-07-04 14:08:36 +01003557static int iommu_no_mapping(struct device *dev)
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07003558{
3559 int found;
3560
David Woodhouse3d891942014-03-06 15:59:26 +00003561 if (iommu_dummy(dev))
David Woodhouse1e4c64c2009-07-04 10:40:38 +01003562 return 1;
3563
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07003564 if (!iommu_identity_mapping)
David Woodhouse1e4c64c2009-07-04 10:40:38 +01003565 return 0;
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07003566
David Woodhouse9b226622014-03-09 14:03:28 -07003567 found = identity_mapping(dev);
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07003568 if (found) {
David Woodhouseecb509e2014-03-09 16:29:55 -07003569 if (iommu_should_identity_map(dev, 0))
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07003570 return 1;
3571 else {
3572 /*
3573 * 32 bit DMA is removed from si_domain and fall back
3574 * to non-identity mapping.
3575 */
Joerg Roedele6de0f82015-07-22 16:30:36 +02003576 dmar_remove_one_dev_info(si_domain, dev);
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02003577 pr_info("32bit %s uses non-identity mapping\n",
3578 dev_name(dev));
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07003579 return 0;
3580 }
3581 } else {
3582 /*
3583 * In case of a detached 64 bit DMA device from vm, the device
3584 * is put into si_domain for identity mapping.
3585 */
David Woodhouseecb509e2014-03-09 16:29:55 -07003586 if (iommu_should_identity_map(dev, 0)) {
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07003587 int ret;
Joerg Roedel28ccce02015-07-21 14:45:31 +02003588 ret = domain_add_dev_info(si_domain, dev);
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07003589 if (!ret) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02003590 pr_info("64bit %s uses identity mapping\n",
3591 dev_name(dev));
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07003592 return 1;
3593 }
3594 }
3595 }
3596
David Woodhouse1e4c64c2009-07-04 10:40:38 +01003597 return 0;
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07003598}
3599
Christoph Hellwig964f2312018-11-21 19:32:03 +01003600static dma_addr_t __intel_map_page(struct device *dev, struct page *page,
3601 unsigned long offset, size_t size, int dir,
3602 u64 dma_mask)
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003603{
Christoph Hellwig964f2312018-11-21 19:32:03 +01003604 phys_addr_t paddr = page_to_phys(page) + offset;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003605 struct dmar_domain *domain;
Fenghua Yu5b6985c2008-10-16 18:02:32 -07003606 phys_addr_t start_paddr;
Omer Peleg2aac6302016-04-20 11:33:57 +03003607 unsigned long iova_pfn;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003608 int prot = 0;
Ingo Molnar6865f0d2008-04-22 11:09:04 +02003609 int ret;
Weidong Han8c11e792008-12-08 15:29:22 +08003610 struct intel_iommu *iommu;
Fenghua Yu33041ec2009-08-04 15:10:59 -07003611 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003612
3613 BUG_ON(dir == DMA_NONE);
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07003614
David Woodhouse5040a912014-03-09 16:14:00 -07003615 if (iommu_no_mapping(dev))
Ingo Molnar6865f0d2008-04-22 11:09:04 +02003616 return paddr;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003617
David Woodhouse5040a912014-03-09 16:14:00 -07003618 domain = get_valid_domain_for_dev(dev);
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003619 if (!domain)
Christoph Hellwig524a6692018-11-21 19:34:10 +01003620 return DMA_MAPPING_ERROR;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003621
Weidong Han8c11e792008-12-08 15:29:22 +08003622 iommu = domain_get_iommu(domain);
David Woodhouse88cb6a72009-06-28 15:03:06 +01003623 size = aligned_nrpages(paddr, size);
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003624
Omer Peleg2aac6302016-04-20 11:33:57 +03003625 iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
3626 if (!iova_pfn)
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003627 goto error;
3628
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003629 /*
3630 * Check if DMAR supports zero-length reads on write only
3631 * mappings..
3632 */
3633 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
Weidong Han8c11e792008-12-08 15:29:22 +08003634 !cap_zlr(iommu->cap))
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003635 prot |= DMA_PTE_READ;
3636 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3637 prot |= DMA_PTE_WRITE;
3638 /*
Ingo Molnar6865f0d2008-04-22 11:09:04 +02003639 * paddr - (paddr + size) might be partial page, we should map the whole
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003640 * page. Note: if two part of one page are separately mapped, we
Ingo Molnar6865f0d2008-04-22 11:09:04 +02003641 * might have two guest_addr mapping to the same host paddr, but this
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003642 * is not a big problem
3643 */
Omer Peleg2aac6302016-04-20 11:33:57 +03003644 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova_pfn),
Fenghua Yu33041ec2009-08-04 15:10:59 -07003645 mm_to_dma_pfn(paddr_pfn), size, prot);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003646 if (ret)
3647 goto error;
3648
Omer Peleg2aac6302016-04-20 11:33:57 +03003649 start_paddr = (phys_addr_t)iova_pfn << PAGE_SHIFT;
David Woodhouse03d6a242009-06-28 15:33:46 +01003650 start_paddr += paddr & ~PAGE_MASK;
3651 return start_paddr;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003652
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003653error:
Omer Peleg2aac6302016-04-20 11:33:57 +03003654 if (iova_pfn)
Omer Peleg22e2f9f2016-04-20 11:34:11 +03003655 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02003656 pr_err("Device %s request: %zx@%llx dir %d --- failed\n",
David Woodhouse5040a912014-03-09 16:14:00 -07003657 dev_name(dev), size, (unsigned long long)paddr, dir);
Christoph Hellwig524a6692018-11-21 19:34:10 +01003658 return DMA_MAPPING_ERROR;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003659}
3660
FUJITA Tomonoriffbbef52009-01-05 23:47:26 +09003661static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3662 unsigned long offset, size_t size,
3663 enum dma_data_direction dir,
Krzysztof Kozlowski00085f12016-08-03 13:46:00 -07003664 unsigned long attrs)
FUJITA Tomonoribb9e6d62008-10-15 16:08:28 +09003665{
Christoph Hellwig964f2312018-11-21 19:32:03 +01003666 return __intel_map_page(dev, page, offset, size, dir, *dev->dma_mask);
FUJITA Tomonoribb9e6d62008-10-15 16:08:28 +09003667}
3668
Omer Peleg769530e2016-04-20 11:33:25 +03003669static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003670{
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003671 struct dmar_domain *domain;
David Woodhoused794dc92009-06-28 00:27:49 +01003672 unsigned long start_pfn, last_pfn;
Omer Peleg769530e2016-04-20 11:33:25 +03003673 unsigned long nrpages;
Omer Peleg2aac6302016-04-20 11:33:57 +03003674 unsigned long iova_pfn;
Weidong Han8c11e792008-12-08 15:29:22 +08003675 struct intel_iommu *iommu;
David Woodhouseea8ea462014-03-05 17:09:32 +00003676 struct page *freelist;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003677
David Woodhouse73676832009-07-04 14:08:36 +01003678 if (iommu_no_mapping(dev))
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003679 return;
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07003680
David Woodhouse1525a292014-03-06 16:19:30 +00003681 domain = find_domain(dev);
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003682 BUG_ON(!domain);
3683
Weidong Han8c11e792008-12-08 15:29:22 +08003684 iommu = domain_get_iommu(domain);
3685
Omer Peleg2aac6302016-04-20 11:33:57 +03003686 iova_pfn = IOVA_PFN(dev_addr);
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003687
Omer Peleg769530e2016-04-20 11:33:25 +03003688 nrpages = aligned_nrpages(dev_addr, size);
Omer Peleg2aac6302016-04-20 11:33:57 +03003689 start_pfn = mm_to_dma_pfn(iova_pfn);
Omer Peleg769530e2016-04-20 11:33:25 +03003690 last_pfn = start_pfn + nrpages - 1;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003691
David Woodhoused794dc92009-06-28 00:27:49 +01003692 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
David Woodhouse207e3592014-03-09 16:12:32 -07003693 dev_name(dev), start_pfn, last_pfn);
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003694
David Woodhouseea8ea462014-03-05 17:09:32 +00003695 freelist = domain_unmap(domain, start_pfn, last_pfn);
David Woodhoused794dc92009-06-28 00:27:49 +01003696
mark gross5e0d2a62008-03-04 15:22:08 -08003697 if (intel_iommu_strict) {
Joerg Roedela1ddcbe2015-07-21 15:20:32 +02003698 iommu_flush_iotlb_psi(iommu, domain, start_pfn,
Omer Peleg769530e2016-04-20 11:33:25 +03003699 nrpages, !freelist, 0);
mark gross5e0d2a62008-03-04 15:22:08 -08003700 /* free iova */
Omer Peleg22e2f9f2016-04-20 11:34:11 +03003701 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(nrpages));
David Woodhouseea8ea462014-03-05 17:09:32 +00003702 dma_free_pagelist(freelist);
mark gross5e0d2a62008-03-04 15:22:08 -08003703 } else {
Joerg Roedel13cf0172017-08-11 11:40:10 +02003704 queue_iova(&domain->iovad, iova_pfn, nrpages,
3705 (unsigned long)freelist);
mark gross5e0d2a62008-03-04 15:22:08 -08003706 /*
3707 * queue up the release of the unmap to save the 1/6th of the
3708 * cpu used up by the iotlb flush operation...
3709 */
mark gross5e0d2a62008-03-04 15:22:08 -08003710 }
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003711}
3712
Jiang Liud41a4ad2014-07-11 14:19:34 +08003713static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3714 size_t size, enum dma_data_direction dir,
Krzysztof Kozlowski00085f12016-08-03 13:46:00 -07003715 unsigned long attrs)
Jiang Liud41a4ad2014-07-11 14:19:34 +08003716{
Omer Peleg769530e2016-04-20 11:33:25 +03003717 intel_unmap(dev, dev_addr, size);
Jiang Liud41a4ad2014-07-11 14:19:34 +08003718}
3719
David Woodhouse5040a912014-03-09 16:14:00 -07003720static void *intel_alloc_coherent(struct device *dev, size_t size,
Andrzej Pietrasiewiczbaa676f2012-03-27 14:28:18 +02003721 dma_addr_t *dma_handle, gfp_t flags,
Krzysztof Kozlowski00085f12016-08-03 13:46:00 -07003722 unsigned long attrs)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003723{
Christoph Hellwig7ec916f2018-07-05 13:29:55 -06003724 struct page *page = NULL;
3725 int order;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003726
Christoph Hellwig7ec916f2018-07-05 13:29:55 -06003727 size = PAGE_ALIGN(size);
3728 order = get_order(size);
Alex Williamsone8bb9102009-11-04 15:59:34 -07003729
Christoph Hellwig7ec916f2018-07-05 13:29:55 -06003730 if (!iommu_no_mapping(dev))
3731 flags &= ~(GFP_DMA | GFP_DMA32);
3732 else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
3733 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
3734 flags |= GFP_DMA;
3735 else
3736 flags |= GFP_DMA32;
3737 }
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003738
Christoph Hellwig7ec916f2018-07-05 13:29:55 -06003739 if (gfpflags_allow_blocking(flags)) {
3740 unsigned int count = size >> PAGE_SHIFT;
3741
Marek Szyprowskid834c5a2018-08-17 15:49:00 -07003742 page = dma_alloc_from_contiguous(dev, count, order,
3743 flags & __GFP_NOWARN);
Christoph Hellwig7ec916f2018-07-05 13:29:55 -06003744 if (page && iommu_no_mapping(dev) &&
3745 page_to_phys(page) + size > dev->coherent_dma_mask) {
3746 dma_release_from_contiguous(dev, page, count);
3747 page = NULL;
3748 }
3749 }
3750
3751 if (!page)
3752 page = alloc_pages(flags, order);
3753 if (!page)
3754 return NULL;
3755 memset(page_address(page), 0, size);
3756
Christoph Hellwig964f2312018-11-21 19:32:03 +01003757 *dma_handle = __intel_map_page(dev, page, 0, size, DMA_BIDIRECTIONAL,
3758 dev->coherent_dma_mask);
Christoph Hellwig524a6692018-11-21 19:34:10 +01003759 if (*dma_handle != DMA_MAPPING_ERROR)
Christoph Hellwig7ec916f2018-07-05 13:29:55 -06003760 return page_address(page);
3761 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3762 __free_pages(page, order);
3763
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003764 return NULL;
3765}
3766
David Woodhouse5040a912014-03-09 16:14:00 -07003767static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
Krzysztof Kozlowski00085f12016-08-03 13:46:00 -07003768 dma_addr_t dma_handle, unsigned long attrs)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003769{
Christoph Hellwig7ec916f2018-07-05 13:29:55 -06003770 int order;
3771 struct page *page = virt_to_page(vaddr);
3772
3773 size = PAGE_ALIGN(size);
3774 order = get_order(size);
3775
3776 intel_unmap(dev, dma_handle, size);
3777 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3778 __free_pages(page, order);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003779}
3780
David Woodhouse5040a912014-03-09 16:14:00 -07003781static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
FUJITA Tomonorid7ab5c42009-01-28 21:53:18 +09003782 int nelems, enum dma_data_direction dir,
Krzysztof Kozlowski00085f12016-08-03 13:46:00 -07003783 unsigned long attrs)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003784{
Omer Peleg769530e2016-04-20 11:33:25 +03003785 dma_addr_t startaddr = sg_dma_address(sglist) & PAGE_MASK;
3786 unsigned long nrpages = 0;
3787 struct scatterlist *sg;
3788 int i;
3789
3790 for_each_sg(sglist, sg, nelems, i) {
3791 nrpages += aligned_nrpages(sg_dma_address(sg), sg_dma_len(sg));
3792 }
3793
3794 intel_unmap(dev, startaddr, nrpages << VTD_PAGE_SHIFT);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003795}
3796
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003797static int intel_nontranslate_map_sg(struct device *hddev,
FUJITA Tomonoric03ab372007-10-21 16:42:00 -07003798 struct scatterlist *sglist, int nelems, int dir)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003799{
3800 int i;
FUJITA Tomonoric03ab372007-10-21 16:42:00 -07003801 struct scatterlist *sg;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003802
FUJITA Tomonoric03ab372007-10-21 16:42:00 -07003803 for_each_sg(sglist, sg, nelems, i) {
FUJITA Tomonori12d4d402007-10-23 09:32:25 +02003804 BUG_ON(!sg_page(sg));
Robin Murphy29a90b72017-09-28 15:14:01 +01003805 sg->dma_address = sg_phys(sg);
FUJITA Tomonoric03ab372007-10-21 16:42:00 -07003806 sg->dma_length = sg->length;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003807 }
3808 return nelems;
3809}
3810
David Woodhouse5040a912014-03-09 16:14:00 -07003811static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
Krzysztof Kozlowski00085f12016-08-03 13:46:00 -07003812 enum dma_data_direction dir, unsigned long attrs)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003813{
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003814 int i;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003815 struct dmar_domain *domain;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003816 size_t size = 0;
3817 int prot = 0;
Omer Peleg2aac6302016-04-20 11:33:57 +03003818 unsigned long iova_pfn;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003819 int ret;
FUJITA Tomonoric03ab372007-10-21 16:42:00 -07003820 struct scatterlist *sg;
David Woodhouseb536d242009-06-28 14:49:31 +01003821 unsigned long start_vpfn;
Weidong Han8c11e792008-12-08 15:29:22 +08003822 struct intel_iommu *iommu;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003823
3824 BUG_ON(dir == DMA_NONE);
David Woodhouse5040a912014-03-09 16:14:00 -07003825 if (iommu_no_mapping(dev))
3826 return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003827
David Woodhouse5040a912014-03-09 16:14:00 -07003828 domain = get_valid_domain_for_dev(dev);
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003829 if (!domain)
3830 return 0;
3831
Weidong Han8c11e792008-12-08 15:29:22 +08003832 iommu = domain_get_iommu(domain);
3833
David Woodhouseb536d242009-06-28 14:49:31 +01003834 for_each_sg(sglist, sg, nelems, i)
David Woodhouse88cb6a72009-06-28 15:03:06 +01003835 size += aligned_nrpages(sg->offset, sg->length);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003836
Omer Peleg2aac6302016-04-20 11:33:57 +03003837 iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
David Woodhouse5040a912014-03-09 16:14:00 -07003838 *dev->dma_mask);
Omer Peleg2aac6302016-04-20 11:33:57 +03003839 if (!iova_pfn) {
FUJITA Tomonoric03ab372007-10-21 16:42:00 -07003840 sglist->dma_length = 0;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003841 return 0;
3842 }
3843
3844 /*
3845 * Check if DMAR supports zero-length reads on write only
3846 * mappings..
3847 */
3848 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
Weidong Han8c11e792008-12-08 15:29:22 +08003849 !cap_zlr(iommu->cap))
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003850 prot |= DMA_PTE_READ;
3851 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3852 prot |= DMA_PTE_WRITE;
3853
Omer Peleg2aac6302016-04-20 11:33:57 +03003854 start_vpfn = mm_to_dma_pfn(iova_pfn);
David Woodhousee1605492009-06-29 11:17:38 +01003855
Fenghua Yuf5329592009-08-04 15:09:37 -07003856 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
David Woodhousee1605492009-06-29 11:17:38 +01003857 if (unlikely(ret)) {
David Woodhousee1605492009-06-29 11:17:38 +01003858 dma_pte_free_pagetable(domain, start_vpfn,
David Dillowbc24c572017-06-28 19:42:23 -07003859 start_vpfn + size - 1,
3860 agaw_to_level(domain->agaw) + 1);
Omer Peleg22e2f9f2016-04-20 11:34:11 +03003861 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
David Woodhousee1605492009-06-29 11:17:38 +01003862 return 0;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003863 }
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003864
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003865 return nelems;
3866}
3867
Christoph Hellwig02b4da52018-09-17 19:10:31 +02003868static const struct dma_map_ops intel_dma_ops = {
Andrzej Pietrasiewiczbaa676f2012-03-27 14:28:18 +02003869 .alloc = intel_alloc_coherent,
3870 .free = intel_free_coherent,
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003871 .map_sg = intel_map_sg,
3872 .unmap_sg = intel_unmap_sg,
FUJITA Tomonoriffbbef52009-01-05 23:47:26 +09003873 .map_page = intel_map_page,
3874 .unmap_page = intel_unmap_page,
Christoph Hellwigfec777c2018-03-19 11:38:15 +01003875 .dma_supported = dma_direct_supported,
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003876};
3877
3878static inline int iommu_domain_cache_init(void)
3879{
3880 int ret = 0;
3881
3882 iommu_domain_cache = kmem_cache_create("iommu_domain",
3883 sizeof(struct dmar_domain),
3884 0,
3885 SLAB_HWCACHE_ALIGN,
3886
3887 NULL);
3888 if (!iommu_domain_cache) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02003889 pr_err("Couldn't create iommu_domain cache\n");
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003890 ret = -ENOMEM;
3891 }
3892
3893 return ret;
3894}
3895
3896static inline int iommu_devinfo_cache_init(void)
3897{
3898 int ret = 0;
3899
3900 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3901 sizeof(struct device_domain_info),
3902 0,
3903 SLAB_HWCACHE_ALIGN,
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003904 NULL);
3905 if (!iommu_devinfo_cache) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02003906 pr_err("Couldn't create devinfo cache\n");
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003907 ret = -ENOMEM;
3908 }
3909
3910 return ret;
3911}
3912
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003913static int __init iommu_init_mempool(void)
3914{
3915 int ret;
Sakari Ailusae1ff3d2015-07-13 14:31:28 +03003916 ret = iova_cache_get();
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003917 if (ret)
3918 return ret;
3919
3920 ret = iommu_domain_cache_init();
3921 if (ret)
3922 goto domain_error;
3923
3924 ret = iommu_devinfo_cache_init();
3925 if (!ret)
3926 return ret;
3927
3928 kmem_cache_destroy(iommu_domain_cache);
3929domain_error:
Sakari Ailusae1ff3d2015-07-13 14:31:28 +03003930 iova_cache_put();
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003931
3932 return -ENOMEM;
3933}
3934
3935static void __init iommu_exit_mempool(void)
3936{
3937 kmem_cache_destroy(iommu_devinfo_cache);
3938 kmem_cache_destroy(iommu_domain_cache);
Sakari Ailusae1ff3d2015-07-13 14:31:28 +03003939 iova_cache_put();
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003940}
3941
Dan Williams556ab452010-07-23 15:47:56 -07003942static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3943{
3944 struct dmar_drhd_unit *drhd;
3945 u32 vtbar;
3946 int rc;
3947
3948 /* We know that this device on this chipset has its own IOMMU.
3949 * If we find it under a different IOMMU, then the BIOS is lying
3950 * to us. Hope that the IOMMU for this device is actually
3951 * disabled, and it needs no translation...
3952 */
3953 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3954 if (rc) {
3955 /* "can't" happen */
3956 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3957 return;
3958 }
3959 vtbar &= 0xffff0000;
3960
3961 /* we know that the this iommu should be at offset 0xa000 from vtbar */
3962 drhd = dmar_find_matched_drhd_unit(pdev);
3963 if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3964 TAINT_FIRMWARE_WORKAROUND,
3965 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3966 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3967}
3968DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3969
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003970static void __init init_no_remapping_devices(void)
3971{
3972 struct dmar_drhd_unit *drhd;
David Woodhouse832bd852014-03-07 15:08:36 +00003973 struct device *dev;
Jiang Liub683b232014-02-19 14:07:32 +08003974 int i;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003975
3976 for_each_drhd_unit(drhd) {
3977 if (!drhd->include_all) {
Jiang Liub683b232014-02-19 14:07:32 +08003978 for_each_active_dev_scope(drhd->devices,
3979 drhd->devices_cnt, i, dev)
3980 break;
David Woodhouse832bd852014-03-07 15:08:36 +00003981 /* ignore DMAR unit if no devices exist */
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003982 if (i == drhd->devices_cnt)
3983 drhd->ignored = 1;
3984 }
3985 }
3986
Jiang Liu7c919772014-01-06 14:18:18 +08003987 for_each_active_drhd_unit(drhd) {
Jiang Liu7c919772014-01-06 14:18:18 +08003988 if (drhd->include_all)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003989 continue;
3990
Jiang Liub683b232014-02-19 14:07:32 +08003991 for_each_active_dev_scope(drhd->devices,
3992 drhd->devices_cnt, i, dev)
David Woodhouse832bd852014-03-07 15:08:36 +00003993 if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003994 break;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003995 if (i < drhd->devices_cnt)
3996 continue;
3997
David Woodhousec0771df2011-10-14 20:59:46 +01003998 /* This IOMMU has *only* gfx devices. Either bypass it or
3999 set the gfx_mapped flag, as appropriate */
4000 if (dmar_map_gfx) {
4001 intel_iommu_gfx_mapped = 1;
4002 } else {
4003 drhd->ignored = 1;
Jiang Liub683b232014-02-19 14:07:32 +08004004 for_each_active_dev_scope(drhd->devices,
4005 drhd->devices_cnt, i, dev)
David Woodhouse832bd852014-03-07 15:08:36 +00004006 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07004007 }
4008 }
4009}
4010
Fenghua Yuf59c7b62009-03-27 14:22:42 -07004011#ifdef CONFIG_SUSPEND
4012static int init_iommu_hw(void)
4013{
4014 struct dmar_drhd_unit *drhd;
4015 struct intel_iommu *iommu = NULL;
4016
4017 for_each_active_iommu(iommu, drhd)
4018 if (iommu->qi)
4019 dmar_reenable_qi(iommu);
4020
Joseph Cihulab7792602011-05-03 00:08:37 -07004021 for_each_iommu(iommu, drhd) {
4022 if (drhd->ignored) {
4023 /*
4024 * we always have to disable PMRs or DMA may fail on
4025 * this device
4026 */
4027 if (force_on)
4028 iommu_disable_protect_mem_regions(iommu);
4029 continue;
4030 }
4031
Fenghua Yuf59c7b62009-03-27 14:22:42 -07004032 iommu_flush_write_buffer(iommu);
4033
4034 iommu_set_root_entry(iommu);
4035
4036 iommu->flush.flush_context(iommu, 0, 0, 0,
David Woodhouse1f0ef2a2009-05-10 19:58:49 +01004037 DMA_CCMD_GLOBAL_INVL);
Jiang Liu2a41cce2014-07-11 14:19:33 +08004038 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4039 iommu_enable_translation(iommu);
David Woodhouseb94996c2009-09-19 15:28:12 -07004040 iommu_disable_protect_mem_regions(iommu);
Fenghua Yuf59c7b62009-03-27 14:22:42 -07004041 }
4042
4043 return 0;
4044}
4045
4046static void iommu_flush_all(void)
4047{
4048 struct dmar_drhd_unit *drhd;
4049 struct intel_iommu *iommu;
4050
4051 for_each_active_iommu(iommu, drhd) {
4052 iommu->flush.flush_context(iommu, 0, 0, 0,
David Woodhouse1f0ef2a2009-05-10 19:58:49 +01004053 DMA_CCMD_GLOBAL_INVL);
Fenghua Yuf59c7b62009-03-27 14:22:42 -07004054 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
David Woodhouse1f0ef2a2009-05-10 19:58:49 +01004055 DMA_TLB_GLOBAL_FLUSH);
Fenghua Yuf59c7b62009-03-27 14:22:42 -07004056 }
4057}
4058
Rafael J. Wysocki134fac32011-03-23 22:16:14 +01004059static int iommu_suspend(void)
Fenghua Yuf59c7b62009-03-27 14:22:42 -07004060{
4061 struct dmar_drhd_unit *drhd;
4062 struct intel_iommu *iommu = NULL;
4063 unsigned long flag;
4064
4065 for_each_active_iommu(iommu, drhd) {
Kees Cook6396bb22018-06-12 14:03:40 -07004066 iommu->iommu_state = kcalloc(MAX_SR_DMAR_REGS, sizeof(u32),
Fenghua Yuf59c7b62009-03-27 14:22:42 -07004067 GFP_ATOMIC);
4068 if (!iommu->iommu_state)
4069 goto nomem;
4070 }
4071
4072 iommu_flush_all();
4073
4074 for_each_active_iommu(iommu, drhd) {
4075 iommu_disable_translation(iommu);
4076
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02004077 raw_spin_lock_irqsave(&iommu->register_lock, flag);
Fenghua Yuf59c7b62009-03-27 14:22:42 -07004078
4079 iommu->iommu_state[SR_DMAR_FECTL_REG] =
4080 readl(iommu->reg + DMAR_FECTL_REG);
4081 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
4082 readl(iommu->reg + DMAR_FEDATA_REG);
4083 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
4084 readl(iommu->reg + DMAR_FEADDR_REG);
4085 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
4086 readl(iommu->reg + DMAR_FEUADDR_REG);
4087
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02004088 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
Fenghua Yuf59c7b62009-03-27 14:22:42 -07004089 }
4090 return 0;
4091
4092nomem:
4093 for_each_active_iommu(iommu, drhd)
4094 kfree(iommu->iommu_state);
4095
4096 return -ENOMEM;
4097}
4098
Rafael J. Wysocki134fac32011-03-23 22:16:14 +01004099static void iommu_resume(void)
Fenghua Yuf59c7b62009-03-27 14:22:42 -07004100{
4101 struct dmar_drhd_unit *drhd;
4102 struct intel_iommu *iommu = NULL;
4103 unsigned long flag;
4104
4105 if (init_iommu_hw()) {
Joseph Cihulab7792602011-05-03 00:08:37 -07004106 if (force_on)
4107 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
4108 else
4109 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
Rafael J. Wysocki134fac32011-03-23 22:16:14 +01004110 return;
Fenghua Yuf59c7b62009-03-27 14:22:42 -07004111 }
4112
4113 for_each_active_iommu(iommu, drhd) {
4114
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02004115 raw_spin_lock_irqsave(&iommu->register_lock, flag);
Fenghua Yuf59c7b62009-03-27 14:22:42 -07004116
4117 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
4118 iommu->reg + DMAR_FECTL_REG);
4119 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
4120 iommu->reg + DMAR_FEDATA_REG);
4121 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
4122 iommu->reg + DMAR_FEADDR_REG);
4123 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
4124 iommu->reg + DMAR_FEUADDR_REG);
4125
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02004126 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
Fenghua Yuf59c7b62009-03-27 14:22:42 -07004127 }
4128
4129 for_each_active_iommu(iommu, drhd)
4130 kfree(iommu->iommu_state);
Fenghua Yuf59c7b62009-03-27 14:22:42 -07004131}
4132
Rafael J. Wysocki134fac32011-03-23 22:16:14 +01004133static struct syscore_ops iommu_syscore_ops = {
Fenghua Yuf59c7b62009-03-27 14:22:42 -07004134 .resume = iommu_resume,
4135 .suspend = iommu_suspend,
4136};
4137
Rafael J. Wysocki134fac32011-03-23 22:16:14 +01004138static void __init init_iommu_pm_ops(void)
Fenghua Yuf59c7b62009-03-27 14:22:42 -07004139{
Rafael J. Wysocki134fac32011-03-23 22:16:14 +01004140 register_syscore_ops(&iommu_syscore_ops);
Fenghua Yuf59c7b62009-03-27 14:22:42 -07004141}
4142
4143#else
Rafael J. Wysocki99592ba2011-06-07 21:32:31 +02004144static inline void init_iommu_pm_ops(void) {}
Fenghua Yuf59c7b62009-03-27 14:22:42 -07004145#endif /* CONFIG_PM */
4146
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004147
Jiang Liuc2a0b532014-11-09 22:47:56 +08004148int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004149{
4150 struct acpi_dmar_reserved_memory *rmrr;
Eric Auger0659b8d2017-01-19 20:57:53 +00004151 int prot = DMA_PTE_READ|DMA_PTE_WRITE;
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004152 struct dmar_rmrr_unit *rmrru;
Eric Auger0659b8d2017-01-19 20:57:53 +00004153 size_t length;
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004154
4155 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
4156 if (!rmrru)
Eric Auger0659b8d2017-01-19 20:57:53 +00004157 goto out;
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004158
4159 rmrru->hdr = header;
4160 rmrr = (struct acpi_dmar_reserved_memory *)header;
4161 rmrru->base_address = rmrr->base_address;
4162 rmrru->end_address = rmrr->end_address;
Eric Auger0659b8d2017-01-19 20:57:53 +00004163
4164 length = rmrr->end_address - rmrr->base_address + 1;
4165 rmrru->resv = iommu_alloc_resv_region(rmrr->base_address, length, prot,
4166 IOMMU_RESV_DIRECT);
4167 if (!rmrru->resv)
4168 goto free_rmrru;
4169
Jiang Liu2e455282014-02-19 14:07:36 +08004170 rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
4171 ((void *)rmrr) + rmrr->header.length,
4172 &rmrru->devices_cnt);
Eric Auger0659b8d2017-01-19 20:57:53 +00004173 if (rmrru->devices_cnt && rmrru->devices == NULL)
4174 goto free_all;
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004175
Jiang Liu2e455282014-02-19 14:07:36 +08004176 list_add(&rmrru->list, &dmar_rmrr_units);
4177
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004178 return 0;
Eric Auger0659b8d2017-01-19 20:57:53 +00004179free_all:
4180 kfree(rmrru->resv);
4181free_rmrru:
4182 kfree(rmrru);
4183out:
4184 return -ENOMEM;
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004185}
4186
Jiang Liu6b197242014-11-09 22:47:58 +08004187static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
4188{
4189 struct dmar_atsr_unit *atsru;
4190 struct acpi_dmar_atsr *tmp;
4191
4192 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4193 tmp = (struct acpi_dmar_atsr *)atsru->hdr;
4194 if (atsr->segment != tmp->segment)
4195 continue;
4196 if (atsr->header.length != tmp->header.length)
4197 continue;
4198 if (memcmp(atsr, tmp, atsr->header.length) == 0)
4199 return atsru;
4200 }
4201
4202 return NULL;
4203}
4204
4205int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004206{
4207 struct acpi_dmar_atsr *atsr;
4208 struct dmar_atsr_unit *atsru;
4209
Thomas Gleixnerb608fe32017-05-16 20:42:41 +02004210 if (system_state >= SYSTEM_RUNNING && !intel_iommu_enabled)
Jiang Liu6b197242014-11-09 22:47:58 +08004211 return 0;
4212
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004213 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
Jiang Liu6b197242014-11-09 22:47:58 +08004214 atsru = dmar_find_atsr(atsr);
4215 if (atsru)
4216 return 0;
4217
4218 atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004219 if (!atsru)
4220 return -ENOMEM;
4221
Jiang Liu6b197242014-11-09 22:47:58 +08004222 /*
4223 * If memory is allocated from slab by ACPI _DSM method, we need to
4224 * copy the memory content because the memory buffer will be freed
4225 * on return.
4226 */
4227 atsru->hdr = (void *)(atsru + 1);
4228 memcpy(atsru->hdr, hdr, hdr->length);
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004229 atsru->include_all = atsr->flags & 0x1;
Jiang Liu2e455282014-02-19 14:07:36 +08004230 if (!atsru->include_all) {
4231 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
4232 (void *)atsr + atsr->header.length,
4233 &atsru->devices_cnt);
4234 if (atsru->devices_cnt && atsru->devices == NULL) {
4235 kfree(atsru);
4236 return -ENOMEM;
4237 }
4238 }
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004239
Jiang Liu0e242612014-02-19 14:07:34 +08004240 list_add_rcu(&atsru->list, &dmar_atsr_units);
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004241
4242 return 0;
4243}
4244
Jiang Liu9bdc5312014-01-06 14:18:27 +08004245static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
4246{
4247 dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
4248 kfree(atsru);
4249}
4250
Jiang Liu6b197242014-11-09 22:47:58 +08004251int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4252{
4253 struct acpi_dmar_atsr *atsr;
4254 struct dmar_atsr_unit *atsru;
4255
4256 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4257 atsru = dmar_find_atsr(atsr);
4258 if (atsru) {
4259 list_del_rcu(&atsru->list);
4260 synchronize_rcu();
4261 intel_iommu_free_atsr(atsru);
4262 }
4263
4264 return 0;
4265}
4266
4267int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4268{
4269 int i;
4270 struct device *dev;
4271 struct acpi_dmar_atsr *atsr;
4272 struct dmar_atsr_unit *atsru;
4273
4274 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4275 atsru = dmar_find_atsr(atsr);
4276 if (!atsru)
4277 return 0;
4278
Linus Torvalds194dc872016-07-27 20:03:31 -07004279 if (!atsru->include_all && atsru->devices && atsru->devices_cnt) {
Jiang Liu6b197242014-11-09 22:47:58 +08004280 for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
4281 i, dev)
4282 return -EBUSY;
Linus Torvalds194dc872016-07-27 20:03:31 -07004283 }
Jiang Liu6b197242014-11-09 22:47:58 +08004284
4285 return 0;
4286}
4287
Jiang Liuffebeb42014-11-09 22:48:02 +08004288static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
4289{
4290 int sp, ret = 0;
4291 struct intel_iommu *iommu = dmaru->iommu;
4292
4293 if (g_iommus[iommu->seq_id])
4294 return 0;
4295
4296 if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02004297 pr_warn("%s: Doesn't support hardware pass through.\n",
Jiang Liuffebeb42014-11-09 22:48:02 +08004298 iommu->name);
4299 return -ENXIO;
4300 }
4301 if (!ecap_sc_support(iommu->ecap) &&
4302 domain_update_iommu_snooping(iommu)) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02004303 pr_warn("%s: Doesn't support snooping.\n",
Jiang Liuffebeb42014-11-09 22:48:02 +08004304 iommu->name);
4305 return -ENXIO;
4306 }
4307 sp = domain_update_iommu_superpage(iommu) - 1;
4308 if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02004309 pr_warn("%s: Doesn't support large page.\n",
Jiang Liuffebeb42014-11-09 22:48:02 +08004310 iommu->name);
4311 return -ENXIO;
4312 }
4313
4314 /*
4315 * Disable translation if already enabled prior to OS handover.
4316 */
4317 if (iommu->gcmd & DMA_GCMD_TE)
4318 iommu_disable_translation(iommu);
4319
4320 g_iommus[iommu->seq_id] = iommu;
4321 ret = iommu_init_domains(iommu);
4322 if (ret == 0)
4323 ret = iommu_alloc_root_entry(iommu);
4324 if (ret)
4325 goto out;
4326
David Woodhouse8a94ade2015-03-24 14:54:56 +00004327#ifdef CONFIG_INTEL_IOMMU_SVM
4328 if (pasid_enabled(iommu))
Lu Baolud9737952018-07-14 15:47:02 +08004329 intel_svm_init(iommu);
David Woodhouse8a94ade2015-03-24 14:54:56 +00004330#endif
4331
Jiang Liuffebeb42014-11-09 22:48:02 +08004332 if (dmaru->ignored) {
4333 /*
4334 * we always have to disable PMRs or DMA may fail on this device
4335 */
4336 if (force_on)
4337 iommu_disable_protect_mem_regions(iommu);
4338 return 0;
4339 }
4340
4341 intel_iommu_init_qi(iommu);
4342 iommu_flush_write_buffer(iommu);
David Woodhousea222a7f2015-10-07 23:35:18 +01004343
4344#ifdef CONFIG_INTEL_IOMMU_SVM
4345 if (pasid_enabled(iommu) && ecap_prs(iommu->ecap)) {
4346 ret = intel_svm_enable_prq(iommu);
4347 if (ret)
4348 goto disable_iommu;
4349 }
4350#endif
Jiang Liuffebeb42014-11-09 22:48:02 +08004351 ret = dmar_set_interrupt(iommu);
4352 if (ret)
4353 goto disable_iommu;
4354
4355 iommu_set_root_entry(iommu);
4356 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
4357 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4358 iommu_enable_translation(iommu);
4359
Jiang Liuffebeb42014-11-09 22:48:02 +08004360 iommu_disable_protect_mem_regions(iommu);
4361 return 0;
4362
4363disable_iommu:
4364 disable_dmar_iommu(iommu);
4365out:
4366 free_dmar_iommu(iommu);
4367 return ret;
4368}
4369
Jiang Liu6b197242014-11-09 22:47:58 +08004370int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
4371{
Jiang Liuffebeb42014-11-09 22:48:02 +08004372 int ret = 0;
4373 struct intel_iommu *iommu = dmaru->iommu;
4374
4375 if (!intel_iommu_enabled)
4376 return 0;
4377 if (iommu == NULL)
4378 return -EINVAL;
4379
4380 if (insert) {
4381 ret = intel_iommu_add(dmaru);
4382 } else {
4383 disable_dmar_iommu(iommu);
4384 free_dmar_iommu(iommu);
4385 }
4386
4387 return ret;
Jiang Liu6b197242014-11-09 22:47:58 +08004388}
4389
Jiang Liu9bdc5312014-01-06 14:18:27 +08004390static void intel_iommu_free_dmars(void)
4391{
4392 struct dmar_rmrr_unit *rmrru, *rmrr_n;
4393 struct dmar_atsr_unit *atsru, *atsr_n;
4394
4395 list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
4396 list_del(&rmrru->list);
4397 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
Eric Auger0659b8d2017-01-19 20:57:53 +00004398 kfree(rmrru->resv);
Jiang Liu9bdc5312014-01-06 14:18:27 +08004399 kfree(rmrru);
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004400 }
4401
Jiang Liu9bdc5312014-01-06 14:18:27 +08004402 list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
4403 list_del(&atsru->list);
4404 intel_iommu_free_atsr(atsru);
4405 }
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004406}
4407
4408int dmar_find_matched_atsr_unit(struct pci_dev *dev)
4409{
Jiang Liub683b232014-02-19 14:07:32 +08004410 int i, ret = 1;
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004411 struct pci_bus *bus;
David Woodhouse832bd852014-03-07 15:08:36 +00004412 struct pci_dev *bridge = NULL;
4413 struct device *tmp;
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004414 struct acpi_dmar_atsr *atsr;
4415 struct dmar_atsr_unit *atsru;
4416
4417 dev = pci_physfn(dev);
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004418 for (bus = dev->bus; bus; bus = bus->parent) {
Jiang Liub5f82dd2014-02-19 14:07:31 +08004419 bridge = bus->self;
David Woodhoused14053b32015-10-15 09:28:06 +01004420 /* If it's an integrated device, allow ATS */
4421 if (!bridge)
4422 return 1;
4423 /* Connected via non-PCIe: no ATS */
4424 if (!pci_is_pcie(bridge) ||
Yijing Wang62f87c02012-07-24 17:20:03 +08004425 pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004426 return 0;
David Woodhoused14053b32015-10-15 09:28:06 +01004427 /* If we found the root port, look it up in the ATSR */
Jiang Liub5f82dd2014-02-19 14:07:31 +08004428 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004429 break;
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004430 }
4431
Jiang Liu0e242612014-02-19 14:07:34 +08004432 rcu_read_lock();
Jiang Liub5f82dd2014-02-19 14:07:31 +08004433 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4434 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4435 if (atsr->segment != pci_domain_nr(dev->bus))
4436 continue;
4437
Jiang Liub683b232014-02-19 14:07:32 +08004438 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
David Woodhouse832bd852014-03-07 15:08:36 +00004439 if (tmp == &bridge->dev)
Jiang Liub683b232014-02-19 14:07:32 +08004440 goto out;
Jiang Liub5f82dd2014-02-19 14:07:31 +08004441
4442 if (atsru->include_all)
Jiang Liub683b232014-02-19 14:07:32 +08004443 goto out;
Jiang Liub5f82dd2014-02-19 14:07:31 +08004444 }
Jiang Liub683b232014-02-19 14:07:32 +08004445 ret = 0;
4446out:
Jiang Liu0e242612014-02-19 14:07:34 +08004447 rcu_read_unlock();
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004448
Jiang Liub683b232014-02-19 14:07:32 +08004449 return ret;
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004450}
4451
Jiang Liu59ce0512014-02-19 14:07:35 +08004452int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
4453{
4454 int ret = 0;
4455 struct dmar_rmrr_unit *rmrru;
4456 struct dmar_atsr_unit *atsru;
4457 struct acpi_dmar_atsr *atsr;
4458 struct acpi_dmar_reserved_memory *rmrr;
4459
Thomas Gleixnerb608fe32017-05-16 20:42:41 +02004460 if (!intel_iommu_enabled && system_state >= SYSTEM_RUNNING)
Jiang Liu59ce0512014-02-19 14:07:35 +08004461 return 0;
4462
4463 list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
4464 rmrr = container_of(rmrru->hdr,
4465 struct acpi_dmar_reserved_memory, header);
4466 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4467 ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
4468 ((void *)rmrr) + rmrr->header.length,
4469 rmrr->segment, rmrru->devices,
4470 rmrru->devices_cnt);
Jiang Liu27e24952014-06-20 15:08:06 +08004471 if(ret < 0)
Jiang Liu59ce0512014-02-19 14:07:35 +08004472 return ret;
Joerg Roedele6a8c9b2016-02-29 23:49:47 +01004473 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
Jiang Liu27e24952014-06-20 15:08:06 +08004474 dmar_remove_dev_scope(info, rmrr->segment,
4475 rmrru->devices, rmrru->devices_cnt);
Jiang Liu59ce0512014-02-19 14:07:35 +08004476 }
4477 }
4478
4479 list_for_each_entry(atsru, &dmar_atsr_units, list) {
4480 if (atsru->include_all)
4481 continue;
4482
4483 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4484 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4485 ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
4486 (void *)atsr + atsr->header.length,
4487 atsr->segment, atsru->devices,
4488 atsru->devices_cnt);
4489 if (ret > 0)
4490 break;
4491 else if(ret < 0)
4492 return ret;
Joerg Roedele6a8c9b2016-02-29 23:49:47 +01004493 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
Jiang Liu59ce0512014-02-19 14:07:35 +08004494 if (dmar_remove_dev_scope(info, atsr->segment,
4495 atsru->devices, atsru->devices_cnt))
4496 break;
4497 }
4498 }
4499
4500 return 0;
4501}
4502
Fenghua Yu99dcade2009-11-11 07:23:06 -08004503/*
4504 * Here we only respond to action of unbound device from driver.
4505 *
4506 * Added device is not attached to its DMAR domain here yet. That will happen
4507 * when mapping the device to iova.
4508 */
4509static int device_notifier(struct notifier_block *nb,
4510 unsigned long action, void *data)
4511{
4512 struct device *dev = data;
Fenghua Yu99dcade2009-11-11 07:23:06 -08004513 struct dmar_domain *domain;
4514
David Woodhouse3d891942014-03-06 15:59:26 +00004515 if (iommu_dummy(dev))
David Woodhouse44cd6132009-12-02 10:18:30 +00004516 return 0;
4517
Joerg Roedel1196c2f2014-09-30 13:02:03 +02004518 if (action != BUS_NOTIFY_REMOVED_DEVICE)
Jiang Liu7e7dfab2014-02-19 14:07:23 +08004519 return 0;
4520
David Woodhouse1525a292014-03-06 16:19:30 +00004521 domain = find_domain(dev);
Fenghua Yu99dcade2009-11-11 07:23:06 -08004522 if (!domain)
4523 return 0;
4524
Joerg Roedele6de0f82015-07-22 16:30:36 +02004525 dmar_remove_one_dev_info(domain, dev);
Jiang Liuab8dfe22014-07-11 14:19:27 +08004526 if (!domain_type_is_vm_or_si(domain) && list_empty(&domain->devices))
Jiang Liu7e7dfab2014-02-19 14:07:23 +08004527 domain_exit(domain);
Alex Williamsona97590e2011-03-04 14:52:16 -07004528
Fenghua Yu99dcade2009-11-11 07:23:06 -08004529 return 0;
4530}
4531
4532static struct notifier_block device_nb = {
4533 .notifier_call = device_notifier,
4534};
4535
Jiang Liu75f05562014-02-19 14:07:37 +08004536static int intel_iommu_memory_notifier(struct notifier_block *nb,
4537 unsigned long val, void *v)
4538{
4539 struct memory_notify *mhp = v;
4540 unsigned long long start, end;
4541 unsigned long start_vpfn, last_vpfn;
4542
4543 switch (val) {
4544 case MEM_GOING_ONLINE:
4545 start = mhp->start_pfn << PAGE_SHIFT;
4546 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
4547 if (iommu_domain_identity_map(si_domain, start, end)) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02004548 pr_warn("Failed to build identity map for [%llx-%llx]\n",
Jiang Liu75f05562014-02-19 14:07:37 +08004549 start, end);
4550 return NOTIFY_BAD;
4551 }
4552 break;
4553
4554 case MEM_OFFLINE:
4555 case MEM_CANCEL_ONLINE:
4556 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
4557 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
4558 while (start_vpfn <= last_vpfn) {
4559 struct iova *iova;
4560 struct dmar_drhd_unit *drhd;
4561 struct intel_iommu *iommu;
David Woodhouseea8ea462014-03-05 17:09:32 +00004562 struct page *freelist;
Jiang Liu75f05562014-02-19 14:07:37 +08004563
4564 iova = find_iova(&si_domain->iovad, start_vpfn);
4565 if (iova == NULL) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02004566 pr_debug("Failed get IOVA for PFN %lx\n",
Jiang Liu75f05562014-02-19 14:07:37 +08004567 start_vpfn);
4568 break;
4569 }
4570
4571 iova = split_and_remove_iova(&si_domain->iovad, iova,
4572 start_vpfn, last_vpfn);
4573 if (iova == NULL) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02004574 pr_warn("Failed to split IOVA PFN [%lx-%lx]\n",
Jiang Liu75f05562014-02-19 14:07:37 +08004575 start_vpfn, last_vpfn);
4576 return NOTIFY_BAD;
4577 }
4578
David Woodhouseea8ea462014-03-05 17:09:32 +00004579 freelist = domain_unmap(si_domain, iova->pfn_lo,
4580 iova->pfn_hi);
4581
Jiang Liu75f05562014-02-19 14:07:37 +08004582 rcu_read_lock();
4583 for_each_active_iommu(iommu, drhd)
Joerg Roedela1ddcbe2015-07-21 15:20:32 +02004584 iommu_flush_iotlb_psi(iommu, si_domain,
Jiang Liua156ef92014-07-11 14:19:36 +08004585 iova->pfn_lo, iova_size(iova),
David Woodhouseea8ea462014-03-05 17:09:32 +00004586 !freelist, 0);
Jiang Liu75f05562014-02-19 14:07:37 +08004587 rcu_read_unlock();
David Woodhouseea8ea462014-03-05 17:09:32 +00004588 dma_free_pagelist(freelist);
Jiang Liu75f05562014-02-19 14:07:37 +08004589
4590 start_vpfn = iova->pfn_hi + 1;
4591 free_iova_mem(iova);
4592 }
4593 break;
4594 }
4595
4596 return NOTIFY_OK;
4597}
4598
4599static struct notifier_block intel_iommu_memory_nb = {
4600 .notifier_call = intel_iommu_memory_notifier,
4601 .priority = 0
4602};
4603
Omer Peleg22e2f9f2016-04-20 11:34:11 +03004604static void free_all_cpu_cached_iovas(unsigned int cpu)
4605{
4606 int i;
4607
4608 for (i = 0; i < g_num_of_iommus; i++) {
4609 struct intel_iommu *iommu = g_iommus[i];
4610 struct dmar_domain *domain;
Aaron Campbell0caa7612016-07-02 21:23:24 -03004611 int did;
Omer Peleg22e2f9f2016-04-20 11:34:11 +03004612
4613 if (!iommu)
4614 continue;
4615
Jan Niehusmann3bd4f912016-06-06 14:20:11 +02004616 for (did = 0; did < cap_ndoms(iommu->cap); did++) {
Aaron Campbell0caa7612016-07-02 21:23:24 -03004617 domain = get_iommu_domain(iommu, (u16)did);
Omer Peleg22e2f9f2016-04-20 11:34:11 +03004618
4619 if (!domain)
4620 continue;
4621 free_cpu_cached_iovas(cpu, &domain->iovad);
4622 }
4623 }
4624}
4625
Anna-Maria Gleixner21647612016-11-27 00:13:41 +01004626static int intel_iommu_cpu_dead(unsigned int cpu)
Omer Pelegaa473242016-04-20 11:33:02 +03004627{
Anna-Maria Gleixner21647612016-11-27 00:13:41 +01004628 free_all_cpu_cached_iovas(cpu);
Anna-Maria Gleixner21647612016-11-27 00:13:41 +01004629 return 0;
Omer Pelegaa473242016-04-20 11:33:02 +03004630}
4631
Joerg Roedel161b28a2017-03-28 17:04:52 +02004632static void intel_disable_iommus(void)
4633{
4634 struct intel_iommu *iommu = NULL;
4635 struct dmar_drhd_unit *drhd;
4636
4637 for_each_iommu(iommu, drhd)
4638 iommu_disable_translation(iommu);
4639}
4640
Joerg Roedela7fdb6e2017-02-28 13:57:18 +01004641static inline struct intel_iommu *dev_to_intel_iommu(struct device *dev)
4642{
Joerg Roedel2926a2aa2017-08-14 17:19:26 +02004643 struct iommu_device *iommu_dev = dev_to_iommu_device(dev);
4644
4645 return container_of(iommu_dev, struct intel_iommu, iommu);
Joerg Roedela7fdb6e2017-02-28 13:57:18 +01004646}
4647
Alex Williamsona5459cf2014-06-12 16:12:31 -06004648static ssize_t intel_iommu_show_version(struct device *dev,
4649 struct device_attribute *attr,
4650 char *buf)
4651{
Joerg Roedela7fdb6e2017-02-28 13:57:18 +01004652 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
Alex Williamsona5459cf2014-06-12 16:12:31 -06004653 u32 ver = readl(iommu->reg + DMAR_VER_REG);
4654 return sprintf(buf, "%d:%d\n",
4655 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
4656}
4657static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
4658
4659static ssize_t intel_iommu_show_address(struct device *dev,
4660 struct device_attribute *attr,
4661 char *buf)
4662{
Joerg Roedela7fdb6e2017-02-28 13:57:18 +01004663 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
Alex Williamsona5459cf2014-06-12 16:12:31 -06004664 return sprintf(buf, "%llx\n", iommu->reg_phys);
4665}
4666static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
4667
4668static ssize_t intel_iommu_show_cap(struct device *dev,
4669 struct device_attribute *attr,
4670 char *buf)
4671{
Joerg Roedela7fdb6e2017-02-28 13:57:18 +01004672 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
Alex Williamsona5459cf2014-06-12 16:12:31 -06004673 return sprintf(buf, "%llx\n", iommu->cap);
4674}
4675static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
4676
4677static ssize_t intel_iommu_show_ecap(struct device *dev,
4678 struct device_attribute *attr,
4679 char *buf)
4680{
Joerg Roedela7fdb6e2017-02-28 13:57:18 +01004681 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
Alex Williamsona5459cf2014-06-12 16:12:31 -06004682 return sprintf(buf, "%llx\n", iommu->ecap);
4683}
4684static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
4685
Alex Williamson2238c082015-07-14 15:24:53 -06004686static ssize_t intel_iommu_show_ndoms(struct device *dev,
4687 struct device_attribute *attr,
4688 char *buf)
4689{
Joerg Roedela7fdb6e2017-02-28 13:57:18 +01004690 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
Alex Williamson2238c082015-07-14 15:24:53 -06004691 return sprintf(buf, "%ld\n", cap_ndoms(iommu->cap));
4692}
4693static DEVICE_ATTR(domains_supported, S_IRUGO, intel_iommu_show_ndoms, NULL);
4694
4695static ssize_t intel_iommu_show_ndoms_used(struct device *dev,
4696 struct device_attribute *attr,
4697 char *buf)
4698{
Joerg Roedela7fdb6e2017-02-28 13:57:18 +01004699 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
Alex Williamson2238c082015-07-14 15:24:53 -06004700 return sprintf(buf, "%d\n", bitmap_weight(iommu->domain_ids,
4701 cap_ndoms(iommu->cap)));
4702}
4703static DEVICE_ATTR(domains_used, S_IRUGO, intel_iommu_show_ndoms_used, NULL);
4704
Alex Williamsona5459cf2014-06-12 16:12:31 -06004705static struct attribute *intel_iommu_attrs[] = {
4706 &dev_attr_version.attr,
4707 &dev_attr_address.attr,
4708 &dev_attr_cap.attr,
4709 &dev_attr_ecap.attr,
Alex Williamson2238c082015-07-14 15:24:53 -06004710 &dev_attr_domains_supported.attr,
4711 &dev_attr_domains_used.attr,
Alex Williamsona5459cf2014-06-12 16:12:31 -06004712 NULL,
4713};
4714
4715static struct attribute_group intel_iommu_group = {
4716 .name = "intel-iommu",
4717 .attrs = intel_iommu_attrs,
4718};
4719
4720const struct attribute_group *intel_iommu_groups[] = {
4721 &intel_iommu_group,
4722 NULL,
4723};
4724
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07004725int __init intel_iommu_init(void)
4726{
Jiang Liu9bdc5312014-01-06 14:18:27 +08004727 int ret = -ENODEV;
Takao Indoh3a93c842013-04-23 17:35:03 +09004728 struct dmar_drhd_unit *drhd;
Jiang Liu7c919772014-01-06 14:18:18 +08004729 struct intel_iommu *iommu;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07004730
Joseph Cihulaa59b50e2009-06-30 19:31:10 -07004731 /* VT-d is required for a TXT/tboot launch, so enforce that */
4732 force_on = tboot_force_iommu();
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07004733
Jiang Liu3a5670e2014-02-19 14:07:33 +08004734 if (iommu_init_mempool()) {
4735 if (force_on)
4736 panic("tboot: Failed to initialize iommu memory\n");
4737 return -ENOMEM;
4738 }
4739
4740 down_write(&dmar_global_lock);
Joseph Cihulaa59b50e2009-06-30 19:31:10 -07004741 if (dmar_table_init()) {
4742 if (force_on)
4743 panic("tboot: Failed to initialize DMAR table\n");
Jiang Liu9bdc5312014-01-06 14:18:27 +08004744 goto out_free_dmar;
Joseph Cihulaa59b50e2009-06-30 19:31:10 -07004745 }
4746
Suresh Siddhac2c72862011-08-23 17:05:19 -07004747 if (dmar_dev_scope_init() < 0) {
Joseph Cihulaa59b50e2009-06-30 19:31:10 -07004748 if (force_on)
4749 panic("tboot: Failed to initialize DMAR device scope\n");
Jiang Liu9bdc5312014-01-06 14:18:27 +08004750 goto out_free_dmar;
Joseph Cihulaa59b50e2009-06-30 19:31:10 -07004751 }
Suresh Siddha1886e8a2008-07-10 11:16:37 -07004752
Joerg Roedelec154bf2017-10-06 15:00:53 +02004753 up_write(&dmar_global_lock);
4754
4755 /*
4756 * The bus notifier takes the dmar_global_lock, so lockdep will
4757 * complain later when we register it under the lock.
4758 */
4759 dmar_register_bus_notifier();
4760
4761 down_write(&dmar_global_lock);
4762
Joerg Roedel161b28a2017-03-28 17:04:52 +02004763 if (no_iommu || dmar_disabled) {
4764 /*
Shaohua Libfd20f12017-04-26 09:18:35 -07004765 * We exit the function here to ensure IOMMU's remapping and
4766 * mempool aren't setup, which means that the IOMMU's PMRs
4767 * won't be disabled via the call to init_dmars(). So disable
4768 * it explicitly here. The PMRs were setup by tboot prior to
4769 * calling SENTER, but the kernel is expected to reset/tear
4770 * down the PMRs.
4771 */
4772 if (intel_iommu_tboot_noforce) {
4773 for_each_iommu(iommu, drhd)
4774 iommu_disable_protect_mem_regions(iommu);
4775 }
4776
4777 /*
Joerg Roedel161b28a2017-03-28 17:04:52 +02004778 * Make sure the IOMMUs are switched off, even when we
4779 * boot into a kexec kernel and the previous kernel left
4780 * them enabled
4781 */
4782 intel_disable_iommus();
Jiang Liu9bdc5312014-01-06 14:18:27 +08004783 goto out_free_dmar;
Joerg Roedel161b28a2017-03-28 17:04:52 +02004784 }
Suresh Siddha2ae21012008-07-10 11:16:43 -07004785
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004786 if (list_empty(&dmar_rmrr_units))
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02004787 pr_info("No RMRR found\n");
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004788
4789 if (list_empty(&dmar_atsr_units))
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02004790 pr_info("No ATSR found\n");
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004791
Joseph Cihula51a63e62011-03-21 11:04:24 -07004792 if (dmar_init_reserved_ranges()) {
4793 if (force_on)
4794 panic("tboot: Failed to reserve iommu ranges\n");
Jiang Liu3a5670e2014-02-19 14:07:33 +08004795 goto out_free_reserved_range;
Joseph Cihula51a63e62011-03-21 11:04:24 -07004796 }
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07004797
4798 init_no_remapping_devices();
4799
Joseph Cihulab7792602011-05-03 00:08:37 -07004800 ret = init_dmars();
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07004801 if (ret) {
Joseph Cihulaa59b50e2009-06-30 19:31:10 -07004802 if (force_on)
4803 panic("tboot: Failed to initialize DMARs\n");
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02004804 pr_err("Initialization failed\n");
Jiang Liu9bdc5312014-01-06 14:18:27 +08004805 goto out_free_reserved_range;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07004806 }
Jiang Liu3a5670e2014-02-19 14:07:33 +08004807 up_write(&dmar_global_lock);
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02004808 pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07004809
Christoph Hellwig4fac8072017-12-24 13:57:08 +01004810#if defined(CONFIG_X86) && defined(CONFIG_SWIOTLB)
FUJITA Tomonori75f1cdf2009-11-10 19:46:20 +09004811 swiotlb = 0;
4812#endif
David Woodhouse19943b02009-08-04 16:19:20 +01004813 dma_ops = &intel_dma_ops;
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07004814
Rafael J. Wysocki134fac32011-03-23 22:16:14 +01004815 init_iommu_pm_ops();
Joerg Roedela8bcbb0d2008-12-03 15:14:02 +01004816
Joerg Roedel39ab9552017-02-01 16:56:46 +01004817 for_each_active_iommu(iommu, drhd) {
4818 iommu_device_sysfs_add(&iommu->iommu, NULL,
4819 intel_iommu_groups,
4820 "%s", iommu->name);
4821 iommu_device_set_ops(&iommu->iommu, &intel_iommu_ops);
4822 iommu_device_register(&iommu->iommu);
4823 }
Alex Williamsona5459cf2014-06-12 16:12:31 -06004824
Joerg Roedel4236d97d2011-09-06 17:56:07 +02004825 bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
Fenghua Yu99dcade2009-11-11 07:23:06 -08004826 bus_register_notifier(&pci_bus_type, &device_nb);
Jiang Liu75f05562014-02-19 14:07:37 +08004827 if (si_domain && !hw_pass_through)
4828 register_memory_notifier(&intel_iommu_memory_nb);
Anna-Maria Gleixner21647612016-11-27 00:13:41 +01004829 cpuhp_setup_state(CPUHP_IOMMU_INTEL_DEAD, "iommu/intel:dead", NULL,
4830 intel_iommu_cpu_dead);
Eugeni Dodonov8bc1f852011-11-23 16:42:14 -02004831 intel_iommu_enabled = 1;
Sohil Mehtaee2636b2018-09-11 17:11:38 -07004832 intel_iommu_debugfs_init();
Eugeni Dodonov8bc1f852011-11-23 16:42:14 -02004833
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07004834 return 0;
Jiang Liu9bdc5312014-01-06 14:18:27 +08004835
4836out_free_reserved_range:
4837 put_iova_domain(&reserved_iova_list);
Jiang Liu9bdc5312014-01-06 14:18:27 +08004838out_free_dmar:
4839 intel_iommu_free_dmars();
Jiang Liu3a5670e2014-02-19 14:07:33 +08004840 up_write(&dmar_global_lock);
4841 iommu_exit_mempool();
Jiang Liu9bdc5312014-01-06 14:18:27 +08004842 return ret;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07004843}
Keshavamurthy, Anil Se8204822007-10-21 16:41:55 -07004844
Joerg Roedel2452d9d2015-07-23 16:20:14 +02004845static int domain_context_clear_one_cb(struct pci_dev *pdev, u16 alias, void *opaque)
Alex Williamson579305f2014-07-03 09:51:43 -06004846{
4847 struct intel_iommu *iommu = opaque;
4848
Joerg Roedel2452d9d2015-07-23 16:20:14 +02004849 domain_context_clear_one(iommu, PCI_BUS_NUM(alias), alias & 0xff);
Alex Williamson579305f2014-07-03 09:51:43 -06004850 return 0;
4851}
4852
4853/*
4854 * NB - intel-iommu lacks any sort of reference counting for the users of
4855 * dependent devices. If multiple endpoints have intersecting dependent
4856 * devices, unbinding the driver from any one of them will possibly leave
4857 * the others unable to operate.
4858 */
Joerg Roedel2452d9d2015-07-23 16:20:14 +02004859static void domain_context_clear(struct intel_iommu *iommu, struct device *dev)
Han, Weidong3199aa62009-02-26 17:31:12 +08004860{
David Woodhouse0bcb3e22014-03-06 17:12:03 +00004861 if (!iommu || !dev || !dev_is_pci(dev))
Han, Weidong3199aa62009-02-26 17:31:12 +08004862 return;
4863
Joerg Roedel2452d9d2015-07-23 16:20:14 +02004864 pci_for_each_dma_alias(to_pci_dev(dev), &domain_context_clear_one_cb, iommu);
Han, Weidong3199aa62009-02-26 17:31:12 +08004865}
4866
Joerg Roedel127c7612015-07-23 17:44:46 +02004867static void __dmar_remove_one_dev_info(struct device_domain_info *info)
Weidong Hanc7151a82008-12-08 22:51:37 +08004868{
Weidong Hanc7151a82008-12-08 22:51:37 +08004869 struct intel_iommu *iommu;
4870 unsigned long flags;
Weidong Hanc7151a82008-12-08 22:51:37 +08004871
Joerg Roedel55d94042015-07-22 16:50:40 +02004872 assert_spin_locked(&device_domain_lock);
4873
Joerg Roedelb608ac32015-07-21 18:19:08 +02004874 if (WARN_ON(!info))
Weidong Hanc7151a82008-12-08 22:51:37 +08004875 return;
4876
Joerg Roedel127c7612015-07-23 17:44:46 +02004877 iommu = info->iommu;
4878
4879 if (info->dev) {
4880 iommu_disable_dev_iotlb(info);
4881 domain_context_clear(iommu, info->dev);
Lu Baolua7fc93f2018-07-14 15:47:00 +08004882 intel_pasid_free_table(info->dev);
Joerg Roedel127c7612015-07-23 17:44:46 +02004883 }
4884
Joerg Roedelb608ac32015-07-21 18:19:08 +02004885 unlink_domain_info(info);
Roland Dreier3e7abe22011-07-20 06:22:21 -07004886
Joerg Roedeld160aca2015-07-22 11:52:53 +02004887 spin_lock_irqsave(&iommu->lock, flags);
Joerg Roedel127c7612015-07-23 17:44:46 +02004888 domain_detach_iommu(info->domain, iommu);
Joerg Roedeld160aca2015-07-22 11:52:53 +02004889 spin_unlock_irqrestore(&iommu->lock, flags);
Joerg Roedel127c7612015-07-23 17:44:46 +02004890
4891 free_devinfo_mem(info);
Weidong Hanc7151a82008-12-08 22:51:37 +08004892}
4893
Joerg Roedel55d94042015-07-22 16:50:40 +02004894static void dmar_remove_one_dev_info(struct dmar_domain *domain,
4895 struct device *dev)
4896{
Joerg Roedel127c7612015-07-23 17:44:46 +02004897 struct device_domain_info *info;
Joerg Roedel55d94042015-07-22 16:50:40 +02004898 unsigned long flags;
4899
Weidong Hanc7151a82008-12-08 22:51:37 +08004900 spin_lock_irqsave(&device_domain_lock, flags);
Joerg Roedel127c7612015-07-23 17:44:46 +02004901 info = dev->archdata.iommu;
4902 __dmar_remove_one_dev_info(info);
Weidong Han5e98c4b2008-12-08 23:03:27 +08004903 spin_unlock_irqrestore(&device_domain_lock, flags);
Weidong Han5e98c4b2008-12-08 23:03:27 +08004904}
4905
4906static int md_domain_init(struct dmar_domain *domain, int guest_width)
4907{
4908 int adjust_width;
4909
Zhen Leiaa3ac942017-09-21 16:52:45 +01004910 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN);
Weidong Han5e98c4b2008-12-08 23:03:27 +08004911 domain_reserve_special_ranges(domain);
4912
4913 /* calculate AGAW */
4914 domain->gaw = guest_width;
4915 adjust_width = guestwidth_to_adjustwidth(guest_width);
4916 domain->agaw = width_to_agaw(adjust_width);
4917
Weidong Han5e98c4b2008-12-08 23:03:27 +08004918 domain->iommu_coherency = 0;
Sheng Yangc5b15252009-08-06 13:31:56 +08004919 domain->iommu_snooping = 0;
Youquan Song6dd9a7c2011-05-25 19:13:49 +01004920 domain->iommu_superpage = 0;
Weidong Hanfe40f1e2008-12-08 23:10:23 +08004921 domain->max_addr = 0;
Weidong Han5e98c4b2008-12-08 23:03:27 +08004922
4923 /* always allocate the top pgd */
Suresh Siddha4c923d42009-10-02 11:01:24 -07004924 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
Weidong Han5e98c4b2008-12-08 23:03:27 +08004925 if (!domain->pgd)
4926 return -ENOMEM;
4927 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
4928 return 0;
4929}
4930
Joerg Roedel00a77de2015-03-26 13:43:08 +01004931static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
Kay, Allen M38717942008-09-09 18:37:29 +03004932{
Joerg Roedel5d450802008-12-03 14:52:32 +01004933 struct dmar_domain *dmar_domain;
Joerg Roedel00a77de2015-03-26 13:43:08 +01004934 struct iommu_domain *domain;
4935
4936 if (type != IOMMU_DOMAIN_UNMANAGED)
4937 return NULL;
Kay, Allen M38717942008-09-09 18:37:29 +03004938
Jiang Liuab8dfe22014-07-11 14:19:27 +08004939 dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
Joerg Roedel5d450802008-12-03 14:52:32 +01004940 if (!dmar_domain) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02004941 pr_err("Can't allocate dmar_domain\n");
Joerg Roedel00a77de2015-03-26 13:43:08 +01004942 return NULL;
Kay, Allen M38717942008-09-09 18:37:29 +03004943 }
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07004944 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02004945 pr_err("Domain initialization failed\n");
Jiang Liu92d03cc2014-02-19 14:07:28 +08004946 domain_exit(dmar_domain);
Joerg Roedel00a77de2015-03-26 13:43:08 +01004947 return NULL;
Kay, Allen M38717942008-09-09 18:37:29 +03004948 }
Allen Kay8140a952011-10-14 12:32:17 -07004949 domain_update_iommu_cap(dmar_domain);
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08004950
Joerg Roedel00a77de2015-03-26 13:43:08 +01004951 domain = &dmar_domain->domain;
Joerg Roedel8a0e7152012-01-26 19:40:54 +01004952 domain->geometry.aperture_start = 0;
4953 domain->geometry.aperture_end = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
4954 domain->geometry.force_aperture = true;
4955
Joerg Roedel00a77de2015-03-26 13:43:08 +01004956 return domain;
Kay, Allen M38717942008-09-09 18:37:29 +03004957}
Kay, Allen M38717942008-09-09 18:37:29 +03004958
Joerg Roedel00a77de2015-03-26 13:43:08 +01004959static void intel_iommu_domain_free(struct iommu_domain *domain)
Kay, Allen M38717942008-09-09 18:37:29 +03004960{
Joerg Roedel00a77de2015-03-26 13:43:08 +01004961 domain_exit(to_dmar_domain(domain));
Kay, Allen M38717942008-09-09 18:37:29 +03004962}
Kay, Allen M38717942008-09-09 18:37:29 +03004963
Joerg Roedel4c5478c2008-12-03 14:58:24 +01004964static int intel_iommu_attach_device(struct iommu_domain *domain,
4965 struct device *dev)
Kay, Allen M38717942008-09-09 18:37:29 +03004966{
Joerg Roedel00a77de2015-03-26 13:43:08 +01004967 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
Weidong Hanfe40f1e2008-12-08 23:10:23 +08004968 struct intel_iommu *iommu;
4969 int addr_width;
David Woodhouse156baca2014-03-09 14:00:57 -07004970 u8 bus, devfn;
Kay, Allen M38717942008-09-09 18:37:29 +03004971
Alex Williamsonc875d2c2014-07-03 09:57:02 -06004972 if (device_is_rmrr_locked(dev)) {
4973 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement. Contact your platform vendor.\n");
4974 return -EPERM;
4975 }
4976
David Woodhouse7207d8f2014-03-09 16:31:06 -07004977 /* normally dev is not mapped */
4978 if (unlikely(domain_context_mapped(dev))) {
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08004979 struct dmar_domain *old_domain;
4980
David Woodhouse1525a292014-03-06 16:19:30 +00004981 old_domain = find_domain(dev);
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08004982 if (old_domain) {
Joerg Roedeld160aca2015-07-22 11:52:53 +02004983 rcu_read_lock();
Joerg Roedelde7e8882015-07-22 11:58:07 +02004984 dmar_remove_one_dev_info(old_domain, dev);
Joerg Roedeld160aca2015-07-22 11:52:53 +02004985 rcu_read_unlock();
Joerg Roedel62c22162014-12-09 12:56:45 +01004986
4987 if (!domain_type_is_vm_or_si(old_domain) &&
4988 list_empty(&old_domain->devices))
4989 domain_exit(old_domain);
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08004990 }
4991 }
4992
David Woodhouse156baca2014-03-09 14:00:57 -07004993 iommu = device_to_iommu(dev, &bus, &devfn);
Weidong Hanfe40f1e2008-12-08 23:10:23 +08004994 if (!iommu)
4995 return -ENODEV;
4996
4997 /* check if this iommu agaw is sufficient for max mapped address */
4998 addr_width = agaw_to_width(iommu->agaw);
Tom Lyona99c47a2010-05-17 08:20:45 +01004999 if (addr_width > cap_mgaw(iommu->cap))
5000 addr_width = cap_mgaw(iommu->cap);
5001
5002 if (dmar_domain->max_addr > (1LL << addr_width)) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02005003 pr_err("%s: iommu width (%d) is not "
Weidong Hanfe40f1e2008-12-08 23:10:23 +08005004 "sufficient for the mapped address (%llx)\n",
Tom Lyona99c47a2010-05-17 08:20:45 +01005005 __func__, addr_width, dmar_domain->max_addr);
Weidong Hanfe40f1e2008-12-08 23:10:23 +08005006 return -EFAULT;
5007 }
Tom Lyona99c47a2010-05-17 08:20:45 +01005008 dmar_domain->gaw = addr_width;
5009
5010 /*
5011 * Knock out extra levels of page tables if necessary
5012 */
5013 while (iommu->agaw < dmar_domain->agaw) {
5014 struct dma_pte *pte;
5015
5016 pte = dmar_domain->pgd;
5017 if (dma_pte_present(pte)) {
Sheng Yang25cbff12010-06-12 19:21:42 +08005018 dmar_domain->pgd = (struct dma_pte *)
5019 phys_to_virt(dma_pte_addr(pte));
Jan Kiszka7a661012010-11-02 08:05:51 +01005020 free_pgtable_page(pte);
Tom Lyona99c47a2010-05-17 08:20:45 +01005021 }
5022 dmar_domain->agaw--;
5023 }
Weidong Hanfe40f1e2008-12-08 23:10:23 +08005024
Joerg Roedel28ccce02015-07-21 14:45:31 +02005025 return domain_add_dev_info(dmar_domain, dev);
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08005026}
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08005027
Joerg Roedel4c5478c2008-12-03 14:58:24 +01005028static void intel_iommu_detach_device(struct iommu_domain *domain,
5029 struct device *dev)
Kay, Allen M38717942008-09-09 18:37:29 +03005030{
Joerg Roedele6de0f82015-07-22 16:30:36 +02005031 dmar_remove_one_dev_info(to_dmar_domain(domain), dev);
Kay, Allen M38717942008-09-09 18:37:29 +03005032}
Kay, Allen M38717942008-09-09 18:37:29 +03005033
Joerg Roedelb146a1c9f2010-01-20 17:17:37 +01005034static int intel_iommu_map(struct iommu_domain *domain,
5035 unsigned long iova, phys_addr_t hpa,
Ohad Ben-Cohen50090652011-11-10 11:32:25 +02005036 size_t size, int iommu_prot)
Kay, Allen M38717942008-09-09 18:37:29 +03005037{
Joerg Roedel00a77de2015-03-26 13:43:08 +01005038 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
Weidong Hanfe40f1e2008-12-08 23:10:23 +08005039 u64 max_addr;
Joerg Roedeldde57a22008-12-03 15:04:09 +01005040 int prot = 0;
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08005041 int ret;
Weidong Hanfe40f1e2008-12-08 23:10:23 +08005042
Joerg Roedeldde57a22008-12-03 15:04:09 +01005043 if (iommu_prot & IOMMU_READ)
5044 prot |= DMA_PTE_READ;
5045 if (iommu_prot & IOMMU_WRITE)
5046 prot |= DMA_PTE_WRITE;
Sheng Yang9cf06692009-03-18 15:33:07 +08005047 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
5048 prot |= DMA_PTE_SNP;
Joerg Roedeldde57a22008-12-03 15:04:09 +01005049
David Woodhouse163cc522009-06-28 00:51:17 +01005050 max_addr = iova + size;
Joerg Roedeldde57a22008-12-03 15:04:09 +01005051 if (dmar_domain->max_addr < max_addr) {
Weidong Hanfe40f1e2008-12-08 23:10:23 +08005052 u64 end;
5053
5054 /* check if minimum agaw is sufficient for mapped address */
Tom Lyon8954da12010-05-17 08:19:52 +01005055 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
Weidong Hanfe40f1e2008-12-08 23:10:23 +08005056 if (end < max_addr) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02005057 pr_err("%s: iommu width (%d) is not "
Weidong Hanfe40f1e2008-12-08 23:10:23 +08005058 "sufficient for the mapped address (%llx)\n",
Tom Lyon8954da12010-05-17 08:19:52 +01005059 __func__, dmar_domain->gaw, max_addr);
Weidong Hanfe40f1e2008-12-08 23:10:23 +08005060 return -EFAULT;
5061 }
Joerg Roedeldde57a22008-12-03 15:04:09 +01005062 dmar_domain->max_addr = max_addr;
Weidong Hanfe40f1e2008-12-08 23:10:23 +08005063 }
David Woodhousead051222009-06-28 14:22:28 +01005064 /* Round up size to next multiple of PAGE_SIZE, if it and
5065 the low bits of hpa would take us onto the next page */
David Woodhouse88cb6a72009-06-28 15:03:06 +01005066 size = aligned_nrpages(hpa, size);
David Woodhousead051222009-06-28 14:22:28 +01005067 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
5068 hpa >> VTD_PAGE_SHIFT, size, prot);
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08005069 return ret;
Kay, Allen M38717942008-09-09 18:37:29 +03005070}
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08005071
Ohad Ben-Cohen50090652011-11-10 11:32:25 +02005072static size_t intel_iommu_unmap(struct iommu_domain *domain,
David Woodhouseea8ea462014-03-05 17:09:32 +00005073 unsigned long iova, size_t size)
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08005074{
Joerg Roedel00a77de2015-03-26 13:43:08 +01005075 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
David Woodhouseea8ea462014-03-05 17:09:32 +00005076 struct page *freelist = NULL;
David Woodhouseea8ea462014-03-05 17:09:32 +00005077 unsigned long start_pfn, last_pfn;
5078 unsigned int npages;
Joerg Roedel42e8c182015-07-21 15:50:02 +02005079 int iommu_id, level = 0;
Sheng Yang4b99d352009-07-08 11:52:52 +01005080
David Woodhouse5cf0a762014-03-19 16:07:49 +00005081 /* Cope with horrid API which requires us to unmap more than the
5082 size argument if it happens to be a large-page mapping. */
Joerg Roedeldc02e462015-08-13 11:15:13 +02005083 BUG_ON(!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level));
David Woodhouse5cf0a762014-03-19 16:07:49 +00005084
5085 if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
5086 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
5087
David Woodhouseea8ea462014-03-05 17:09:32 +00005088 start_pfn = iova >> VTD_PAGE_SHIFT;
5089 last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
5090
5091 freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
5092
5093 npages = last_pfn - start_pfn + 1;
5094
Shaokun Zhangf746a022018-03-22 18:18:06 +08005095 for_each_domain_iommu(iommu_id, dmar_domain)
Joerg Roedel42e8c182015-07-21 15:50:02 +02005096 iommu_flush_iotlb_psi(g_iommus[iommu_id], dmar_domain,
5097 start_pfn, npages, !freelist, 0);
David Woodhouseea8ea462014-03-05 17:09:32 +00005098
5099 dma_free_pagelist(freelist);
Weidong Hanfe40f1e2008-12-08 23:10:23 +08005100
David Woodhouse163cc522009-06-28 00:51:17 +01005101 if (dmar_domain->max_addr == iova + size)
5102 dmar_domain->max_addr = iova;
Joerg Roedelb146a1c9f2010-01-20 17:17:37 +01005103
David Woodhouse5cf0a762014-03-19 16:07:49 +00005104 return size;
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08005105}
Kay, Allen M38717942008-09-09 18:37:29 +03005106
Joerg Roedeld14d6572008-12-03 15:06:57 +01005107static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
Varun Sethibb5547a2013-03-29 01:23:58 +05305108 dma_addr_t iova)
Kay, Allen M38717942008-09-09 18:37:29 +03005109{
Joerg Roedel00a77de2015-03-26 13:43:08 +01005110 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
Kay, Allen M38717942008-09-09 18:37:29 +03005111 struct dma_pte *pte;
David Woodhouse5cf0a762014-03-19 16:07:49 +00005112 int level = 0;
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08005113 u64 phys = 0;
Kay, Allen M38717942008-09-09 18:37:29 +03005114
David Woodhouse5cf0a762014-03-19 16:07:49 +00005115 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
Kay, Allen M38717942008-09-09 18:37:29 +03005116 if (pte)
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08005117 phys = dma_pte_addr(pte);
Kay, Allen M38717942008-09-09 18:37:29 +03005118
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08005119 return phys;
Kay, Allen M38717942008-09-09 18:37:29 +03005120}
Joerg Roedela8bcbb0d2008-12-03 15:14:02 +01005121
Joerg Roedel5d587b82014-09-05 10:50:45 +02005122static bool intel_iommu_capable(enum iommu_cap cap)
Sheng Yangdbb9fd82009-03-18 15:33:06 +08005123{
Sheng Yangdbb9fd82009-03-18 15:33:06 +08005124 if (cap == IOMMU_CAP_CACHE_COHERENCY)
Joerg Roedel5d587b82014-09-05 10:50:45 +02005125 return domain_update_iommu_snooping(NULL) == 1;
Tom Lyon323f99c2010-07-02 16:56:14 -04005126 if (cap == IOMMU_CAP_INTR_REMAP)
Joerg Roedel5d587b82014-09-05 10:50:45 +02005127 return irq_remapping_enabled == 1;
Sheng Yangdbb9fd82009-03-18 15:33:06 +08005128
Joerg Roedel5d587b82014-09-05 10:50:45 +02005129 return false;
Sheng Yangdbb9fd82009-03-18 15:33:06 +08005130}
5131
Alex Williamsonabdfdde2012-05-30 14:19:19 -06005132static int intel_iommu_add_device(struct device *dev)
Alex Williamson70ae6f02011-10-21 15:56:11 -04005133{
Alex Williamsona5459cf2014-06-12 16:12:31 -06005134 struct intel_iommu *iommu;
Alex Williamsonabdfdde2012-05-30 14:19:19 -06005135 struct iommu_group *group;
David Woodhouse156baca2014-03-09 14:00:57 -07005136 u8 bus, devfn;
Alex Williamson70ae6f02011-10-21 15:56:11 -04005137
Alex Williamsona5459cf2014-06-12 16:12:31 -06005138 iommu = device_to_iommu(dev, &bus, &devfn);
5139 if (!iommu)
Alex Williamson70ae6f02011-10-21 15:56:11 -04005140 return -ENODEV;
5141
Joerg Roedele3d10af2017-02-01 17:23:22 +01005142 iommu_device_link(&iommu->iommu, dev);
Alex Williamsonabdfdde2012-05-30 14:19:19 -06005143
Alex Williamsone17f9ff2014-07-03 09:51:37 -06005144 group = iommu_group_get_for_dev(dev);
Alex Williamson783f1572012-05-30 14:19:43 -06005145
Alex Williamsone17f9ff2014-07-03 09:51:37 -06005146 if (IS_ERR(group))
5147 return PTR_ERR(group);
Alex Williamson70ae6f02011-10-21 15:56:11 -04005148
Alex Williamsonabdfdde2012-05-30 14:19:19 -06005149 iommu_group_put(group);
Alex Williamsone17f9ff2014-07-03 09:51:37 -06005150 return 0;
Alex Williamsonabdfdde2012-05-30 14:19:19 -06005151}
5152
5153static void intel_iommu_remove_device(struct device *dev)
5154{
Alex Williamsona5459cf2014-06-12 16:12:31 -06005155 struct intel_iommu *iommu;
5156 u8 bus, devfn;
5157
5158 iommu = device_to_iommu(dev, &bus, &devfn);
5159 if (!iommu)
5160 return;
5161
Alex Williamsonabdfdde2012-05-30 14:19:19 -06005162 iommu_group_remove_device(dev);
Alex Williamsona5459cf2014-06-12 16:12:31 -06005163
Joerg Roedele3d10af2017-02-01 17:23:22 +01005164 iommu_device_unlink(&iommu->iommu, dev);
Alex Williamson70ae6f02011-10-21 15:56:11 -04005165}
5166
Eric Auger0659b8d2017-01-19 20:57:53 +00005167static void intel_iommu_get_resv_regions(struct device *device,
5168 struct list_head *head)
5169{
5170 struct iommu_resv_region *reg;
5171 struct dmar_rmrr_unit *rmrr;
5172 struct device *i_dev;
5173 int i;
5174
5175 rcu_read_lock();
5176 for_each_rmrr_units(rmrr) {
5177 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
5178 i, i_dev) {
5179 if (i_dev != device)
5180 continue;
5181
5182 list_add_tail(&rmrr->resv->list, head);
5183 }
5184 }
5185 rcu_read_unlock();
5186
5187 reg = iommu_alloc_resv_region(IOAPIC_RANGE_START,
5188 IOAPIC_RANGE_END - IOAPIC_RANGE_START + 1,
Robin Murphy9d3a4de2017-03-16 17:00:16 +00005189 0, IOMMU_RESV_MSI);
Eric Auger0659b8d2017-01-19 20:57:53 +00005190 if (!reg)
5191 return;
5192 list_add_tail(&reg->list, head);
5193}
5194
5195static void intel_iommu_put_resv_regions(struct device *dev,
5196 struct list_head *head)
5197{
5198 struct iommu_resv_region *entry, *next;
5199
5200 list_for_each_entry_safe(entry, next, head, list) {
5201 if (entry->type == IOMMU_RESV_RESERVED)
5202 kfree(entry);
5203 }
Kay, Allen M38717942008-09-09 18:37:29 +03005204}
Joerg Roedela8bcbb0d2008-12-03 15:14:02 +01005205
David Woodhouse2f26e0a2015-09-09 11:40:47 +01005206#ifdef CONFIG_INTEL_IOMMU_SVM
Jacob Pan65ca7f52016-12-06 10:14:23 -08005207#define MAX_NR_PASID_BITS (20)
Lu Baolu4774cc52018-07-14 15:47:01 +08005208static inline unsigned long intel_iommu_get_pts(struct device *dev)
Jacob Pan65ca7f52016-12-06 10:14:23 -08005209{
Lu Baolu4774cc52018-07-14 15:47:01 +08005210 int pts, max_pasid;
5211
5212 max_pasid = intel_pasid_get_dev_max_id(dev);
5213 pts = find_first_bit((unsigned long *)&max_pasid, MAX_NR_PASID_BITS);
5214 if (pts < 5)
Jacob Pan65ca7f52016-12-06 10:14:23 -08005215 return 0;
5216
Lu Baolu4774cc52018-07-14 15:47:01 +08005217 return pts - 5;
Jacob Pan65ca7f52016-12-06 10:14:23 -08005218}
5219
David Woodhouse2f26e0a2015-09-09 11:40:47 +01005220int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct intel_svm_dev *sdev)
5221{
5222 struct device_domain_info *info;
5223 struct context_entry *context;
5224 struct dmar_domain *domain;
5225 unsigned long flags;
5226 u64 ctx_lo;
5227 int ret;
5228
5229 domain = get_valid_domain_for_dev(sdev->dev);
5230 if (!domain)
5231 return -EINVAL;
5232
5233 spin_lock_irqsave(&device_domain_lock, flags);
5234 spin_lock(&iommu->lock);
5235
5236 ret = -EINVAL;
5237 info = sdev->dev->archdata.iommu;
5238 if (!info || !info->pasid_supported)
5239 goto out;
5240
5241 context = iommu_context_addr(iommu, info->bus, info->devfn, 0);
5242 if (WARN_ON(!context))
5243 goto out;
5244
5245 ctx_lo = context[0].lo;
5246
5247 sdev->did = domain->iommu_did[iommu->seq_id];
5248 sdev->sid = PCI_DEVID(info->bus, info->devfn);
5249
5250 if (!(ctx_lo & CONTEXT_PASIDE)) {
Ashok Raj11b93eb2017-08-08 13:29:28 -07005251 if (iommu->pasid_state_table)
5252 context[1].hi = (u64)virt_to_phys(iommu->pasid_state_table);
Lu Baolu4774cc52018-07-14 15:47:01 +08005253 context[1].lo = (u64)virt_to_phys(info->pasid_table->table) |
5254 intel_iommu_get_pts(sdev->dev);
Jacob Pan65ca7f52016-12-06 10:14:23 -08005255
David Woodhouse2f26e0a2015-09-09 11:40:47 +01005256 wmb();
5257 /* CONTEXT_TT_MULTI_LEVEL and CONTEXT_TT_DEV_IOTLB are both
5258 * extended to permit requests-with-PASID if the PASIDE bit
5259 * is set. which makes sense. For CONTEXT_TT_PASS_THROUGH,
5260 * however, the PASIDE bit is ignored and requests-with-PASID
5261 * are unconditionally blocked. Which makes less sense.
5262 * So convert from CONTEXT_TT_PASS_THROUGH to one of the new
5263 * "guest mode" translation types depending on whether ATS
5264 * is available or not. Annoyingly, we can't use the new
5265 * modes *unless* PASIDE is set. */
5266 if ((ctx_lo & CONTEXT_TT_MASK) == (CONTEXT_TT_PASS_THROUGH << 2)) {
5267 ctx_lo &= ~CONTEXT_TT_MASK;
5268 if (info->ats_supported)
5269 ctx_lo |= CONTEXT_TT_PT_PASID_DEV_IOTLB << 2;
5270 else
5271 ctx_lo |= CONTEXT_TT_PT_PASID << 2;
5272 }
5273 ctx_lo |= CONTEXT_PASIDE;
David Woodhouse907fea32015-10-13 14:11:13 +01005274 if (iommu->pasid_state_table)
5275 ctx_lo |= CONTEXT_DINVE;
David Woodhousea222a7f2015-10-07 23:35:18 +01005276 if (info->pri_supported)
5277 ctx_lo |= CONTEXT_PRS;
David Woodhouse2f26e0a2015-09-09 11:40:47 +01005278 context[0].lo = ctx_lo;
5279 wmb();
5280 iommu->flush.flush_context(iommu, sdev->did, sdev->sid,
5281 DMA_CCMD_MASK_NOBIT,
5282 DMA_CCMD_DEVICE_INVL);
5283 }
5284
5285 /* Enable PASID support in the device, if it wasn't already */
5286 if (!info->pasid_enabled)
5287 iommu_enable_dev_iotlb(info);
5288
5289 if (info->ats_enabled) {
5290 sdev->dev_iotlb = 1;
5291 sdev->qdep = info->ats_qdep;
5292 if (sdev->qdep >= QI_DEV_EIOTLB_MAX_INVS)
5293 sdev->qdep = 0;
5294 }
5295 ret = 0;
5296
5297 out:
5298 spin_unlock(&iommu->lock);
5299 spin_unlock_irqrestore(&device_domain_lock, flags);
5300
5301 return ret;
5302}
5303
5304struct intel_iommu *intel_svm_device_to_iommu(struct device *dev)
5305{
5306 struct intel_iommu *iommu;
5307 u8 bus, devfn;
5308
5309 if (iommu_dummy(dev)) {
5310 dev_warn(dev,
5311 "No IOMMU translation for device; cannot enable SVM\n");
5312 return NULL;
5313 }
5314
5315 iommu = device_to_iommu(dev, &bus, &devfn);
5316 if ((!iommu)) {
Sudeep Duttb9997e32015-10-18 20:54:37 -07005317 dev_err(dev, "No IOMMU for device; cannot enable SVM\n");
David Woodhouse2f26e0a2015-09-09 11:40:47 +01005318 return NULL;
5319 }
5320
David Woodhouse2f26e0a2015-09-09 11:40:47 +01005321 return iommu;
5322}
5323#endif /* CONFIG_INTEL_IOMMU_SVM */
5324
Joerg Roedelb0119e82017-02-01 13:23:08 +01005325const struct iommu_ops intel_iommu_ops = {
Eric Auger0659b8d2017-01-19 20:57:53 +00005326 .capable = intel_iommu_capable,
5327 .domain_alloc = intel_iommu_domain_alloc,
5328 .domain_free = intel_iommu_domain_free,
5329 .attach_dev = intel_iommu_attach_device,
5330 .detach_dev = intel_iommu_detach_device,
5331 .map = intel_iommu_map,
5332 .unmap = intel_iommu_unmap,
Eric Auger0659b8d2017-01-19 20:57:53 +00005333 .iova_to_phys = intel_iommu_iova_to_phys,
5334 .add_device = intel_iommu_add_device,
5335 .remove_device = intel_iommu_remove_device,
5336 .get_resv_regions = intel_iommu_get_resv_regions,
5337 .put_resv_regions = intel_iommu_put_resv_regions,
5338 .device_group = pci_device_group,
5339 .pgsize_bitmap = INTEL_IOMMU_PGSIZES,
Joerg Roedela8bcbb0d2008-12-03 15:14:02 +01005340};
David Woodhouse9af88142009-02-13 23:18:03 +00005341
Daniel Vetter94526182013-01-20 23:50:13 +01005342static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
5343{
5344 /* G4x/GM45 integrated gfx dmar support is totally busted. */
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02005345 pr_info("Disabling IOMMU for graphics on this chipset\n");
Daniel Vetter94526182013-01-20 23:50:13 +01005346 dmar_map_gfx = 0;
5347}
5348
5349DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
5350DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
5351DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
5352DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
5353DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
5354DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
5355DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
5356
Greg Kroah-Hartmand34d6512012-12-21 15:05:21 -08005357static void quirk_iommu_rwbf(struct pci_dev *dev)
David Woodhouse9af88142009-02-13 23:18:03 +00005358{
5359 /*
5360 * Mobile 4 Series Chipset neglects to set RWBF capability,
Daniel Vetter210561f2013-01-21 19:48:59 +01005361 * but needs it. Same seems to hold for the desktop versions.
David Woodhouse9af88142009-02-13 23:18:03 +00005362 */
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02005363 pr_info("Forcing write-buffer flush capability\n");
David Woodhouse9af88142009-02-13 23:18:03 +00005364 rwbf_quirk = 1;
5365}
5366
5367DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
Daniel Vetter210561f2013-01-21 19:48:59 +01005368DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
5369DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
5370DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
5371DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
5372DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
5373DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
David Woodhousee0fc7e02009-09-30 09:12:17 -07005374
Adam Jacksoneecfd572010-08-25 21:17:34 +01005375#define GGC 0x52
5376#define GGC_MEMORY_SIZE_MASK (0xf << 8)
5377#define GGC_MEMORY_SIZE_NONE (0x0 << 8)
5378#define GGC_MEMORY_SIZE_1M (0x1 << 8)
5379#define GGC_MEMORY_SIZE_2M (0x3 << 8)
5380#define GGC_MEMORY_VT_ENABLED (0x8 << 8)
5381#define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
5382#define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
5383#define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
5384
Greg Kroah-Hartmand34d6512012-12-21 15:05:21 -08005385static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
David Woodhouse9eecabc2010-09-21 22:28:23 +01005386{
5387 unsigned short ggc;
5388
Adam Jacksoneecfd572010-08-25 21:17:34 +01005389 if (pci_read_config_word(dev, GGC, &ggc))
David Woodhouse9eecabc2010-09-21 22:28:23 +01005390 return;
5391
Adam Jacksoneecfd572010-08-25 21:17:34 +01005392 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02005393 pr_info("BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
David Woodhouse9eecabc2010-09-21 22:28:23 +01005394 dmar_map_gfx = 0;
David Woodhouse6fbcfb32011-09-25 19:11:14 -07005395 } else if (dmar_map_gfx) {
5396 /* we have to ensure the gfx device is idle before we flush */
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02005397 pr_info("Disabling batched IOTLB flush on Ironlake\n");
David Woodhouse6fbcfb32011-09-25 19:11:14 -07005398 intel_iommu_strict = 1;
5399 }
David Woodhouse9eecabc2010-09-21 22:28:23 +01005400}
5401DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
5402DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
5403DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
5404DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
5405
David Woodhousee0fc7e02009-09-30 09:12:17 -07005406/* On Tylersburg chipsets, some BIOSes have been known to enable the
5407 ISOCH DMAR unit for the Azalia sound device, but not give it any
5408 TLB entries, which causes it to deadlock. Check for that. We do
5409 this in a function called from init_dmars(), instead of in a PCI
5410 quirk, because we don't want to print the obnoxious "BIOS broken"
5411 message if VT-d is actually disabled.
5412*/
5413static void __init check_tylersburg_isoch(void)
5414{
5415 struct pci_dev *pdev;
5416 uint32_t vtisochctrl;
5417
5418 /* If there's no Azalia in the system anyway, forget it. */
5419 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
5420 if (!pdev)
5421 return;
5422 pci_dev_put(pdev);
5423
5424 /* System Management Registers. Might be hidden, in which case
5425 we can't do the sanity check. But that's OK, because the
5426 known-broken BIOSes _don't_ actually hide it, so far. */
5427 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
5428 if (!pdev)
5429 return;
5430
5431 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
5432 pci_dev_put(pdev);
5433 return;
5434 }
5435
5436 pci_dev_put(pdev);
5437
5438 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
5439 if (vtisochctrl & 1)
5440 return;
5441
5442 /* Drop all bits other than the number of TLB entries */
5443 vtisochctrl &= 0x1c;
5444
5445 /* If we have the recommended number of TLB entries (16), fine. */
5446 if (vtisochctrl == 0x10)
5447 return;
5448
5449 /* Zero TLB entries? You get to ride the short bus to school. */
5450 if (!vtisochctrl) {
5451 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
5452 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
5453 dmi_get_system_info(DMI_BIOS_VENDOR),
5454 dmi_get_system_info(DMI_BIOS_VERSION),
5455 dmi_get_system_info(DMI_PRODUCT_VERSION));
5456 iommu_identity_mapping |= IDENTMAP_AZALIA;
5457 return;
5458 }
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02005459
5460 pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
David Woodhousee0fc7e02009-09-30 09:12:17 -07005461 vtisochctrl);
5462}