blob: a2a2aa4439aafd26ea4fbfdb5a09acf049f34300 [file] [log] [blame]
David Woodhouse8a94ade2015-03-24 14:54:56 +00001/*
2 * Copyright © 2015 Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * Authors: David Woodhouse <dwmw2@infradead.org>
14 */
15
16#include <linux/intel-iommu.h>
David Woodhouse2f26e0a2015-09-09 11:40:47 +010017#include <linux/mmu_notifier.h>
18#include <linux/sched.h>
Ingo Molnar6e84f312017-02-08 18:51:29 +010019#include <linux/sched/mm.h>
David Woodhouse2f26e0a2015-09-09 11:40:47 +010020#include <linux/slab.h>
21#include <linux/intel-svm.h>
22#include <linux/rculist.h>
23#include <linux/pci.h>
24#include <linux/pci-ats.h>
David Woodhousea222a7f2015-10-07 23:35:18 +010025#include <linux/dmar.h>
26#include <linux/interrupt.h>
Souptick Joarder50a7ca32018-08-17 15:44:47 -070027#include <linux/mm_types.h>
Ashok Raj9d8c3af2017-08-08 13:29:27 -070028#include <asm/page.h>
David Woodhousea222a7f2015-10-07 23:35:18 +010029
Lu Baoluaf395072018-07-14 15:46:56 +080030#include "intel-pasid.h"
31
David Woodhousea222a7f2015-10-07 23:35:18 +010032static irqreturn_t prq_event_thread(int irq, void *d);
David Woodhouse2f26e0a2015-09-09 11:40:47 +010033
Lu Baolud9737952018-07-14 15:47:02 +080034int intel_svm_init(struct intel_iommu *iommu)
David Woodhouse8a94ade2015-03-24 14:54:56 +000035{
Sohil Mehta59103ca2017-12-20 11:59:25 -080036 if (cpu_feature_enabled(X86_FEATURE_GBPAGES) &&
37 !cap_fl1gp_support(iommu->cap))
38 return -EINVAL;
39
Sohil Mehtaf1ac10c2017-12-20 11:59:26 -080040 if (cpu_feature_enabled(X86_FEATURE_LA57) &&
41 !cap_5lp_support(iommu->cap))
42 return -EINVAL;
43
David Woodhouse8a94ade2015-03-24 14:54:56 +000044 return 0;
45}
David Woodhouse2f26e0a2015-09-09 11:40:47 +010046
David Woodhousea222a7f2015-10-07 23:35:18 +010047#define PRQ_ORDER 0
48
49int intel_svm_enable_prq(struct intel_iommu *iommu)
50{
51 struct page *pages;
52 int irq, ret;
53
54 pages = alloc_pages(GFP_KERNEL | __GFP_ZERO, PRQ_ORDER);
55 if (!pages) {
56 pr_warn("IOMMU: %s: Failed to allocate page request queue\n",
57 iommu->name);
58 return -ENOMEM;
59 }
60 iommu->prq = page_address(pages);
61
62 irq = dmar_alloc_hwirq(DMAR_UNITS_SUPPORTED + iommu->seq_id, iommu->node, iommu);
63 if (irq <= 0) {
64 pr_err("IOMMU: %s: Failed to create IRQ vector for page request queue\n",
65 iommu->name);
66 ret = -EINVAL;
67 err:
68 free_pages((unsigned long)iommu->prq, PRQ_ORDER);
69 iommu->prq = NULL;
70 return ret;
71 }
72 iommu->pr_irq = irq;
73
74 snprintf(iommu->prq_name, sizeof(iommu->prq_name), "dmar%d-prq", iommu->seq_id);
75
76 ret = request_threaded_irq(irq, NULL, prq_event_thread, IRQF_ONESHOT,
77 iommu->prq_name, iommu);
78 if (ret) {
79 pr_err("IOMMU: %s: Failed to request IRQ for page request queue\n",
80 iommu->name);
81 dmar_free_hwirq(irq);
Jerry Snitselaar72d54812017-12-20 09:48:56 -070082 iommu->pr_irq = 0;
David Woodhousea222a7f2015-10-07 23:35:18 +010083 goto err;
84 }
85 dmar_writeq(iommu->reg + DMAR_PQH_REG, 0ULL);
86 dmar_writeq(iommu->reg + DMAR_PQT_REG, 0ULL);
87 dmar_writeq(iommu->reg + DMAR_PQA_REG, virt_to_phys(iommu->prq) | PRQ_ORDER);
88
89 return 0;
90}
91
92int intel_svm_finish_prq(struct intel_iommu *iommu)
93{
94 dmar_writeq(iommu->reg + DMAR_PQH_REG, 0ULL);
95 dmar_writeq(iommu->reg + DMAR_PQT_REG, 0ULL);
96 dmar_writeq(iommu->reg + DMAR_PQA_REG, 0ULL);
97
Jerry Snitselaar72d54812017-12-20 09:48:56 -070098 if (iommu->pr_irq) {
99 free_irq(iommu->pr_irq, iommu);
100 dmar_free_hwirq(iommu->pr_irq);
101 iommu->pr_irq = 0;
102 }
David Woodhousea222a7f2015-10-07 23:35:18 +0100103
104 free_pages((unsigned long)iommu->prq, PRQ_ORDER);
105 iommu->prq = NULL;
106
107 return 0;
108}
109
David Woodhouse2f26e0a2015-09-09 11:40:47 +0100110static void intel_flush_svm_range_dev (struct intel_svm *svm, struct intel_svm_dev *sdev,
David Woodhouse5d52f482015-10-20 15:52:13 +0100111 unsigned long address, unsigned long pages, int ih, int gl)
David Woodhouse2f26e0a2015-09-09 11:40:47 +0100112{
113 struct qi_desc desc;
David Woodhouse2f26e0a2015-09-09 11:40:47 +0100114
David Woodhouse5d52f482015-10-20 15:52:13 +0100115 if (pages == -1) {
David Woodhousee0349922015-10-16 19:36:53 +0100116 /* For global kernel pages we have to flush them in *all* PASIDs
117 * because that's the only option the hardware gives us. Despite
118 * the fact that they are actually only accessible through one. */
119 if (gl)
Lu Baolu5d308fc2018-12-10 09:58:58 +0800120 desc.qw0 = QI_EIOTLB_PASID(svm->pasid) |
121 QI_EIOTLB_DID(sdev->did) |
122 QI_EIOTLB_GRAN(QI_GRAN_ALL_ALL) |
123 QI_EIOTLB_TYPE;
David Woodhousee0349922015-10-16 19:36:53 +0100124 else
Lu Baolu5d308fc2018-12-10 09:58:58 +0800125 desc.qw0 = QI_EIOTLB_PASID(svm->pasid) |
126 QI_EIOTLB_DID(sdev->did) |
127 QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID) |
128 QI_EIOTLB_TYPE;
129 desc.qw1 = 0;
David Woodhouse2f26e0a2015-09-09 11:40:47 +0100130 } else {
David Woodhouse5d52f482015-10-20 15:52:13 +0100131 int mask = ilog2(__roundup_pow_of_two(pages));
132
Lu Baolu5d308fc2018-12-10 09:58:58 +0800133 desc.qw0 = QI_EIOTLB_PASID(svm->pasid) |
134 QI_EIOTLB_DID(sdev->did) |
135 QI_EIOTLB_GRAN(QI_GRAN_PSI_PASID) |
136 QI_EIOTLB_TYPE;
137 desc.qw1 = QI_EIOTLB_ADDR(address) |
138 QI_EIOTLB_GL(gl) |
139 QI_EIOTLB_IH(ih) |
140 QI_EIOTLB_AM(mask);
David Woodhouse2f26e0a2015-09-09 11:40:47 +0100141 }
Lu Baolu5d308fc2018-12-10 09:58:58 +0800142 desc.qw2 = 0;
143 desc.qw3 = 0;
David Woodhouse2f26e0a2015-09-09 11:40:47 +0100144 qi_submit_sync(&desc, svm->iommu);
145
146 if (sdev->dev_iotlb) {
Lu Baolu5d308fc2018-12-10 09:58:58 +0800147 desc.qw0 = QI_DEV_EIOTLB_PASID(svm->pasid) |
148 QI_DEV_EIOTLB_SID(sdev->sid) |
149 QI_DEV_EIOTLB_QDEP(sdev->qdep) |
150 QI_DEIOTLB_TYPE;
David Woodhouse5d52f482015-10-20 15:52:13 +0100151 if (pages == -1) {
Lu Baolu5d308fc2018-12-10 09:58:58 +0800152 desc.qw1 = QI_DEV_EIOTLB_ADDR(-1ULL >> 1) |
153 QI_DEV_EIOTLB_SIZE;
David Woodhouse5d52f482015-10-20 15:52:13 +0100154 } else if (pages > 1) {
155 /* The least significant zero bit indicates the size. So,
156 * for example, an "address" value of 0x12345f000 will
157 * flush from 0x123440000 to 0x12347ffff (256KiB). */
158 unsigned long last = address + ((unsigned long)(pages - 1) << VTD_PAGE_SHIFT);
Ingo Molnared7158b2018-02-22 10:54:55 +0100159 unsigned long mask = __rounddown_pow_of_two(address ^ last);
David Woodhouse2f26e0a2015-09-09 11:40:47 +0100160
Lu Baolu5d308fc2018-12-10 09:58:58 +0800161 desc.qw1 = QI_DEV_EIOTLB_ADDR((address & ~mask) |
162 (mask - 1)) | QI_DEV_EIOTLB_SIZE;
David Woodhouse2f26e0a2015-09-09 11:40:47 +0100163 } else {
Lu Baolu5d308fc2018-12-10 09:58:58 +0800164 desc.qw1 = QI_DEV_EIOTLB_ADDR(address);
David Woodhouse2f26e0a2015-09-09 11:40:47 +0100165 }
Lu Baolu5d308fc2018-12-10 09:58:58 +0800166 desc.qw2 = 0;
167 desc.qw3 = 0;
David Woodhouse2f26e0a2015-09-09 11:40:47 +0100168 qi_submit_sync(&desc, svm->iommu);
169 }
170}
171
172static void intel_flush_svm_range(struct intel_svm *svm, unsigned long address,
David Woodhouse5d52f482015-10-20 15:52:13 +0100173 unsigned long pages, int ih, int gl)
David Woodhouse2f26e0a2015-09-09 11:40:47 +0100174{
175 struct intel_svm_dev *sdev;
176
177 rcu_read_lock();
178 list_for_each_entry_rcu(sdev, &svm->devs, list)
David Woodhousee0349922015-10-16 19:36:53 +0100179 intel_flush_svm_range_dev(svm, sdev, address, pages, ih, gl);
David Woodhouse2f26e0a2015-09-09 11:40:47 +0100180 rcu_read_unlock();
181}
182
183static void intel_change_pte(struct mmu_notifier *mn, struct mm_struct *mm,
184 unsigned long address, pte_t pte)
185{
186 struct intel_svm *svm = container_of(mn, struct intel_svm, notifier);
187
David Woodhousee0349922015-10-16 19:36:53 +0100188 intel_flush_svm_range(svm, address, 1, 1, 0);
David Woodhouse2f26e0a2015-09-09 11:40:47 +0100189}
190
David Woodhouse2f26e0a2015-09-09 11:40:47 +0100191/* Pages have been freed at this point */
192static void intel_invalidate_range(struct mmu_notifier *mn,
193 struct mm_struct *mm,
194 unsigned long start, unsigned long end)
195{
196 struct intel_svm *svm = container_of(mn, struct intel_svm, notifier);
197
198 intel_flush_svm_range(svm, start,
David Woodhousee0349922015-10-16 19:36:53 +0100199 (end - start + PAGE_SIZE - 1) >> VTD_PAGE_SHIFT, 0, 0);
David Woodhouse2f26e0a2015-09-09 11:40:47 +0100200}
201
David Woodhouse2f26e0a2015-09-09 11:40:47 +0100202static void intel_mm_release(struct mmu_notifier *mn, struct mm_struct *mm)
203{
204 struct intel_svm *svm = container_of(mn, struct intel_svm, notifier);
David Woodhousee57e58b2016-01-12 19:18:06 +0000205 struct intel_svm_dev *sdev;
David Woodhouse2f26e0a2015-09-09 11:40:47 +0100206
David Woodhousee57e58b2016-01-12 19:18:06 +0000207 /* This might end up being called from exit_mmap(), *before* the page
208 * tables are cleared. And __mmu_notifier_release() will delete us from
209 * the list of notifiers so that our invalidate_range() callback doesn't
210 * get called when the page tables are cleared. So we need to protect
211 * against hardware accessing those page tables.
212 *
213 * We do it by clearing the entry in the PASID table and then flushing
214 * the IOTLB and the PASID table caches. This might upset hardware;
215 * perhaps we'll want to point the PASID to a dummy PGD (like the zero
216 * page) so that we end up taking a fault that the hardware really
217 * *has* to handle gracefully without affecting other processes.
218 */
David Woodhousee57e58b2016-01-12 19:18:06 +0000219 rcu_read_lock();
220 list_for_each_entry_rcu(sdev, &svm->devs, list) {
Lu Baolu1c4f88b2018-12-10 09:59:05 +0800221 intel_pasid_tear_down_entry(svm->iommu, sdev->dev, svm->pasid);
David Woodhousee57e58b2016-01-12 19:18:06 +0000222 intel_flush_svm_range_dev(svm, sdev, 0, -1, 0, !svm->mm);
223 }
224 rcu_read_unlock();
225
David Woodhouse2f26e0a2015-09-09 11:40:47 +0100226}
227
228static const struct mmu_notifier_ops intel_mmuops = {
229 .release = intel_mm_release,
230 .change_pte = intel_change_pte,
David Woodhouse2f26e0a2015-09-09 11:40:47 +0100231 .invalidate_range = intel_invalidate_range,
232};
233
234static DEFINE_MUTEX(pasid_mutex);
Lu Baolu51261aa2018-07-14 15:46:55 +0800235static LIST_HEAD(global_svm_list);
David Woodhouse2f26e0a2015-09-09 11:40:47 +0100236
David Woodhouse0204a492015-10-13 17:18:10 +0100237int intel_svm_bind_mm(struct device *dev, int *pasid, int flags, struct svm_dev_ops *ops)
David Woodhouse2f26e0a2015-09-09 11:40:47 +0100238{
239 struct intel_iommu *iommu = intel_svm_device_to_iommu(dev);
240 struct intel_svm_dev *sdev;
241 struct intel_svm *svm = NULL;
David Woodhouse5cec7532015-10-15 15:52:15 +0100242 struct mm_struct *mm = NULL;
David Woodhouse2f26e0a2015-09-09 11:40:47 +0100243 int pasid_max;
244 int ret;
245
Lu Baolu4774cc52018-07-14 15:47:01 +0800246 if (!iommu)
David Woodhouse2f26e0a2015-09-09 11:40:47 +0100247 return -EINVAL;
248
249 if (dev_is_pci(dev)) {
250 pasid_max = pci_max_pasids(to_pci_dev(dev));
251 if (pasid_max < 0)
252 return -EINVAL;
253 } else
254 pasid_max = 1 << 20;
255
Lu Baolubb37f7d2018-05-04 13:08:19 +0800256 if (flags & SVM_FLAG_SUPERVISOR_MODE) {
David Woodhouse5cec7532015-10-15 15:52:15 +0100257 if (!ecap_srs(iommu->ecap))
258 return -EINVAL;
259 } else if (pasid) {
260 mm = get_task_mm(current);
261 BUG_ON(!mm);
262 }
263
David Woodhouse2f26e0a2015-09-09 11:40:47 +0100264 mutex_lock(&pasid_mutex);
David Woodhouse569e4f72015-10-15 13:59:14 +0100265 if (pasid && !(flags & SVM_FLAG_PRIVATE_PASID)) {
Lu Baolu51261aa2018-07-14 15:46:55 +0800266 struct intel_svm *t;
David Woodhouse2f26e0a2015-09-09 11:40:47 +0100267
Lu Baolu51261aa2018-07-14 15:46:55 +0800268 list_for_each_entry(t, &global_svm_list, list) {
269 if (t->mm != mm || (t->flags & SVM_FLAG_PRIVATE_PASID))
David Woodhouse2f26e0a2015-09-09 11:40:47 +0100270 continue;
271
Lu Baolu51261aa2018-07-14 15:46:55 +0800272 svm = t;
David Woodhouse2f26e0a2015-09-09 11:40:47 +0100273 if (svm->pasid >= pasid_max) {
274 dev_warn(dev,
275 "Limited PASID width. Cannot use existing PASID %d\n",
276 svm->pasid);
277 ret = -ENOSPC;
278 goto out;
279 }
280
281 list_for_each_entry(sdev, &svm->devs, list) {
282 if (dev == sdev->dev) {
David Woodhouse0204a492015-10-13 17:18:10 +0100283 if (sdev->ops != ops) {
284 ret = -EBUSY;
285 goto out;
286 }
David Woodhouse2f26e0a2015-09-09 11:40:47 +0100287 sdev->users++;
288 goto success;
289 }
290 }
291
292 break;
293 }
294 }
295
296 sdev = kzalloc(sizeof(*sdev), GFP_KERNEL);
297 if (!sdev) {
298 ret = -ENOMEM;
299 goto out;
300 }
301 sdev->dev = dev;
302
303 ret = intel_iommu_enable_pasid(iommu, sdev);
304 if (ret || !pasid) {
305 /* If they don't actually want to assign a PASID, this is
306 * just an enabling check/preparation. */
307 kfree(sdev);
308 goto out;
309 }
310 /* Finish the setup now we know we're keeping it */
311 sdev->users = 1;
David Woodhouse0204a492015-10-13 17:18:10 +0100312 sdev->ops = ops;
David Woodhouse2f26e0a2015-09-09 11:40:47 +0100313 init_rcu_head(&sdev->rcu);
314
315 if (!svm) {
316 svm = kzalloc(sizeof(*svm), GFP_KERNEL);
317 if (!svm) {
318 ret = -ENOMEM;
319 kfree(sdev);
320 goto out;
321 }
322 svm->iommu = iommu;
323
Lu Baolu4774cc52018-07-14 15:47:01 +0800324 if (pasid_max > intel_pasid_max_id)
325 pasid_max = intel_pasid_max_id;
David Woodhouse2f26e0a2015-09-09 11:40:47 +0100326
David Woodhouse5a10ba22015-10-24 21:06:39 +0200327 /* Do not use PASID 0 in caching mode (virtualised IOMMU) */
Lu Baoluaf395072018-07-14 15:46:56 +0800328 ret = intel_pasid_alloc_id(svm,
329 !!cap_caching_mode(iommu->cap),
330 pasid_max - 1, GFP_KERNEL);
David Woodhouse2f26e0a2015-09-09 11:40:47 +0100331 if (ret < 0) {
332 kfree(svm);
Lu Baolubbe4b3a2018-02-24 13:42:27 +0800333 kfree(sdev);
David Woodhouse2f26e0a2015-09-09 11:40:47 +0100334 goto out;
335 }
336 svm->pasid = ret;
337 svm->notifier.ops = &intel_mmuops;
David Woodhouse5cec7532015-10-15 15:52:15 +0100338 svm->mm = mm;
David Woodhouse569e4f72015-10-15 13:59:14 +0100339 svm->flags = flags;
David Woodhouse2f26e0a2015-09-09 11:40:47 +0100340 INIT_LIST_HEAD_RCU(&svm->devs);
Lu Baolu51261aa2018-07-14 15:46:55 +0800341 INIT_LIST_HEAD(&svm->list);
David Woodhouse2f26e0a2015-09-09 11:40:47 +0100342 ret = -ENOMEM;
David Woodhouse5cec7532015-10-15 15:52:15 +0100343 if (mm) {
344 ret = mmu_notifier_register(&svm->notifier, mm);
345 if (ret) {
Lu Baoluaf395072018-07-14 15:46:56 +0800346 intel_pasid_free_id(svm->pasid);
David Woodhouse5cec7532015-10-15 15:52:15 +0100347 kfree(svm);
348 kfree(sdev);
349 goto out;
350 }
Lu Baolu1c4f88b2018-12-10 09:59:05 +0800351 }
Sohil Mehta2f13eb72017-12-20 11:59:27 -0800352
Lu Baolu1c4f88b2018-12-10 09:59:05 +0800353 spin_lock(&iommu->lock);
354 ret = intel_pasid_setup_first_level(iommu, dev,
355 mm ? mm->pgd : init_mm.pgd,
356 svm->pasid, FLPT_DEFAULT_DID,
357 mm ? 0 : PASID_FLAG_SUPERVISOR_MODE);
358 spin_unlock(&iommu->lock);
359 if (ret) {
360 if (mm)
361 mmu_notifier_unregister(&svm->notifier, mm);
362 intel_pasid_free_id(svm->pasid);
363 kfree(svm);
364 kfree(sdev);
365 goto out;
366 }
Lu Baolu51261aa2018-07-14 15:46:55 +0800367
368 list_add_tail(&svm->list, &global_svm_list);
David Woodhouse2f26e0a2015-09-09 11:40:47 +0100369 }
370 list_add_rcu(&sdev->list, &svm->devs);
371
372 success:
373 *pasid = svm->pasid;
374 ret = 0;
375 out:
376 mutex_unlock(&pasid_mutex);
David Woodhouse5cec7532015-10-15 15:52:15 +0100377 if (mm)
378 mmput(mm);
David Woodhouse2f26e0a2015-09-09 11:40:47 +0100379 return ret;
380}
381EXPORT_SYMBOL_GPL(intel_svm_bind_mm);
382
383int intel_svm_unbind_mm(struct device *dev, int pasid)
384{
385 struct intel_svm_dev *sdev;
386 struct intel_iommu *iommu;
387 struct intel_svm *svm;
388 int ret = -EINVAL;
389
390 mutex_lock(&pasid_mutex);
391 iommu = intel_svm_device_to_iommu(dev);
Lu Baolu4774cc52018-07-14 15:47:01 +0800392 if (!iommu)
David Woodhouse2f26e0a2015-09-09 11:40:47 +0100393 goto out;
394
Lu Baoluaf395072018-07-14 15:46:56 +0800395 svm = intel_pasid_lookup_id(pasid);
David Woodhouse2f26e0a2015-09-09 11:40:47 +0100396 if (!svm)
397 goto out;
398
399 list_for_each_entry(sdev, &svm->devs, list) {
400 if (dev == sdev->dev) {
401 ret = 0;
402 sdev->users--;
403 if (!sdev->users) {
404 list_del_rcu(&sdev->list);
405 /* Flush the PASID cache and IOTLB for this device.
406 * Note that we do depend on the hardware *not* using
407 * the PASID any more. Just as we depend on other
408 * devices never using PASIDs that they have no right
409 * to use. We have a *shared* PASID table, because it's
410 * large and has to be physically contiguous. So it's
411 * hard to be as defensive as we might like. */
Lu Baolu1c4f88b2018-12-10 09:59:05 +0800412 intel_pasid_tear_down_entry(iommu, dev, svm->pasid);
David Woodhousee0349922015-10-16 19:36:53 +0100413 intel_flush_svm_range_dev(svm, sdev, 0, -1, 0, !svm->mm);
David Woodhouse2f26e0a2015-09-09 11:40:47 +0100414 kfree_rcu(sdev, rcu);
415
416 if (list_empty(&svm->devs)) {
Lu Baoluaf395072018-07-14 15:46:56 +0800417 intel_pasid_free_id(svm->pasid);
David Woodhouse5cec7532015-10-15 15:52:15 +0100418 if (svm->mm)
David Woodhousee57e58b2016-01-12 19:18:06 +0000419 mmu_notifier_unregister(&svm->notifier, svm->mm);
420
Lu Baolu51261aa2018-07-14 15:46:55 +0800421 list_del(&svm->list);
422
David Woodhouse2f26e0a2015-09-09 11:40:47 +0100423 /* We mandate that no page faults may be outstanding
424 * for the PASID when intel_svm_unbind_mm() is called.
425 * If that is not obeyed, subtle errors will happen.
426 * Let's make them less subtle... */
427 memset(svm, 0x6b, sizeof(*svm));
428 kfree(svm);
429 }
430 }
431 break;
432 }
433 }
434 out:
435 mutex_unlock(&pasid_mutex);
436
437 return ret;
438}
439EXPORT_SYMBOL_GPL(intel_svm_unbind_mm);
David Woodhousea222a7f2015-10-07 23:35:18 +0100440
CQ Tang15060ab2017-05-10 11:39:03 -0700441int intel_svm_is_pasid_valid(struct device *dev, int pasid)
442{
443 struct intel_iommu *iommu;
444 struct intel_svm *svm;
445 int ret = -EINVAL;
446
447 mutex_lock(&pasid_mutex);
448 iommu = intel_svm_device_to_iommu(dev);
Lu Baolu4774cc52018-07-14 15:47:01 +0800449 if (!iommu)
CQ Tang15060ab2017-05-10 11:39:03 -0700450 goto out;
451
Lu Baoluaf395072018-07-14 15:46:56 +0800452 svm = intel_pasid_lookup_id(pasid);
CQ Tang15060ab2017-05-10 11:39:03 -0700453 if (!svm)
454 goto out;
455
456 /* init_mm is used in this case */
457 if (!svm->mm)
458 ret = 1;
459 else if (atomic_read(&svm->mm->mm_users) > 0)
460 ret = 1;
461 else
462 ret = 0;
463
464 out:
465 mutex_unlock(&pasid_mutex);
466
467 return ret;
468}
469EXPORT_SYMBOL_GPL(intel_svm_is_pasid_valid);
470
David Woodhousea222a7f2015-10-07 23:35:18 +0100471/* Page request queue descriptor */
472struct page_req_dsc {
473 u64 srr:1;
474 u64 bof:1;
475 u64 pasid_present:1;
476 u64 lpig:1;
477 u64 pasid:20;
478 u64 bus:8;
479 u64 private:23;
480 u64 prg_index:9;
481 u64 rd_req:1;
482 u64 wr_req:1;
483 u64 exe_req:1;
484 u64 priv_req:1;
485 u64 devfn:8;
486 u64 addr:52;
487};
488
489#define PRQ_RING_MASK ((0x1000 << PRQ_ORDER) - 0x10)
Joerg Roedel7f8312a2015-11-17 16:11:39 +0100490
491static bool access_error(struct vm_area_struct *vma, struct page_req_dsc *req)
492{
493 unsigned long requested = 0;
494
495 if (req->exe_req)
496 requested |= VM_EXEC;
497
498 if (req->rd_req)
499 requested |= VM_READ;
500
501 if (req->wr_req)
502 requested |= VM_WRITE;
503
504 return (requested & ~vma->vm_flags) != 0;
505}
506
Ashok Raj9d8c3af2017-08-08 13:29:27 -0700507static bool is_canonical_address(u64 addr)
508{
509 int shift = 64 - (__VIRTUAL_MASK_SHIFT + 1);
510 long saddr = (long) addr;
511
512 return (((saddr << shift) >> shift) == saddr);
513}
514
David Woodhousea222a7f2015-10-07 23:35:18 +0100515static irqreturn_t prq_event_thread(int irq, void *d)
516{
517 struct intel_iommu *iommu = d;
518 struct intel_svm *svm = NULL;
519 int head, tail, handled = 0;
520
David Woodhouse46924002016-02-15 12:42:38 +0000521 /* Clear PPR bit before reading head/tail registers, to
522 * ensure that we get a new interrupt if needed. */
523 writel(DMA_PRS_PPR, iommu->reg + DMAR_PRS_REG);
524
David Woodhousea222a7f2015-10-07 23:35:18 +0100525 tail = dmar_readq(iommu->reg + DMAR_PQT_REG) & PRQ_RING_MASK;
526 head = dmar_readq(iommu->reg + DMAR_PQH_REG) & PRQ_RING_MASK;
527 while (head != tail) {
David Woodhouse0204a492015-10-13 17:18:10 +0100528 struct intel_svm_dev *sdev;
David Woodhousea222a7f2015-10-07 23:35:18 +0100529 struct vm_area_struct *vma;
530 struct page_req_dsc *req;
531 struct qi_desc resp;
Souptick Joarder50a7ca32018-08-17 15:44:47 -0700532 int result;
533 vm_fault_t ret;
David Woodhousea222a7f2015-10-07 23:35:18 +0100534 u64 address;
535
536 handled = 1;
537
538 req = &iommu->prq[head / sizeof(*req)];
539
540 result = QI_RESP_FAILURE;
David Woodhouse7f92a2e2015-10-16 17:22:31 +0100541 address = (u64)req->addr << VTD_PAGE_SHIFT;
David Woodhousea222a7f2015-10-07 23:35:18 +0100542 if (!req->pasid_present) {
543 pr_err("%s: Page request without PASID: %08llx %08llx\n",
544 iommu->name, ((unsigned long long *)req)[0],
545 ((unsigned long long *)req)[1]);
Lu Baolu19ed3e22018-11-05 10:18:58 +0800546 goto no_pasid;
David Woodhousea222a7f2015-10-07 23:35:18 +0100547 }
548
549 if (!svm || svm->pasid != req->pasid) {
550 rcu_read_lock();
Lu Baoluaf395072018-07-14 15:46:56 +0800551 svm = intel_pasid_lookup_id(req->pasid);
David Woodhousea222a7f2015-10-07 23:35:18 +0100552 /* It *can't* go away, because the driver is not permitted
553 * to unbind the mm while any page faults are outstanding.
554 * So we only need RCU to protect the internal idr code. */
555 rcu_read_unlock();
556
557 if (!svm) {
558 pr_err("%s: Page request for invalid PASID %d: %08llx %08llx\n",
559 iommu->name, req->pasid, ((unsigned long long *)req)[0],
560 ((unsigned long long *)req)[1]);
David Woodhouse26322ab2015-10-15 21:12:56 +0100561 goto no_pasid;
David Woodhousea222a7f2015-10-07 23:35:18 +0100562 }
563 }
564
565 result = QI_RESP_INVALID;
David Woodhouse5cec7532015-10-15 15:52:15 +0100566 /* Since we're using init_mm.pgd directly, we should never take
567 * any faults on kernel addresses. */
568 if (!svm->mm)
569 goto bad_req;
David Woodhousee57e58b2016-01-12 19:18:06 +0000570 /* If the mm is already defunct, don't handle faults. */
Vegard Nossum388f7932017-02-27 14:30:13 -0800571 if (!mmget_not_zero(svm->mm))
David Woodhousee57e58b2016-01-12 19:18:06 +0000572 goto bad_req;
Ashok Raj9d8c3af2017-08-08 13:29:27 -0700573
574 /* If address is not canonical, return invalid response */
575 if (!is_canonical_address(address))
576 goto bad_req;
577
David Woodhousea222a7f2015-10-07 23:35:18 +0100578 down_read(&svm->mm->mmap_sem);
579 vma = find_extend_vma(svm->mm, address);
580 if (!vma || address < vma->vm_start)
581 goto invalid;
582
Joerg Roedel7f8312a2015-11-17 16:11:39 +0100583 if (access_error(vma, req))
584 goto invalid;
585
Kirill A. Shutemovdcddffd2016-07-26 15:25:18 -0700586 ret = handle_mm_fault(vma, address,
David Woodhousea222a7f2015-10-07 23:35:18 +0100587 req->wr_req ? FAULT_FLAG_WRITE : 0);
588 if (ret & VM_FAULT_ERROR)
589 goto invalid;
590
591 result = QI_RESP_SUCCESS;
592 invalid:
593 up_read(&svm->mm->mmap_sem);
David Woodhousee57e58b2016-01-12 19:18:06 +0000594 mmput(svm->mm);
David Woodhousea222a7f2015-10-07 23:35:18 +0100595 bad_req:
596 /* Accounting for major/minor faults? */
David Woodhouse0204a492015-10-13 17:18:10 +0100597 rcu_read_lock();
598 list_for_each_entry_rcu(sdev, &svm->devs, list) {
Dan Carpenter3c7c2f32015-10-17 08:18:47 +0300599 if (sdev->sid == PCI_DEVID(req->bus, req->devfn))
David Woodhouse0204a492015-10-13 17:18:10 +0100600 break;
601 }
602 /* Other devices can go away, but the drivers are not permitted
603 * to unbind while any page faults might be in flight. So it's
604 * OK to drop the 'lock' here now we have it. */
605 rcu_read_unlock();
606
607 if (WARN_ON(&sdev->list == &svm->devs))
608 sdev = NULL;
609
610 if (sdev && sdev->ops && sdev->ops->fault_cb) {
611 int rwxp = (req->rd_req << 3) | (req->wr_req << 2) |
David Woodhouse0bdec952015-10-28 15:14:09 +0900612 (req->exe_req << 1) | (req->priv_req);
David Woodhouse0204a492015-10-13 17:18:10 +0100613 sdev->ops->fault_cb(sdev->dev, req->pasid, req->addr, req->private, rwxp, result);
614 }
David Woodhouse26322ab2015-10-15 21:12:56 +0100615 /* We get here in the error case where the PASID lookup failed,
616 and these can be NULL. Do not use them below this point! */
617 sdev = NULL;
618 svm = NULL;
619 no_pasid:
David Woodhousea222a7f2015-10-07 23:35:18 +0100620 if (req->lpig) {
621 /* Page Group Response */
Lu Baolu5d308fc2018-12-10 09:58:58 +0800622 resp.qw0 = QI_PGRP_PASID(req->pasid) |
David Woodhousea222a7f2015-10-07 23:35:18 +0100623 QI_PGRP_DID((req->bus << 8) | req->devfn) |
624 QI_PGRP_PASID_P(req->pasid_present) |
625 QI_PGRP_RESP_TYPE;
Lu Baolu5d308fc2018-12-10 09:58:58 +0800626 resp.qw1 = QI_PGRP_IDX(req->prg_index) |
627 QI_PGRP_PRIV(req->private) |
628 QI_PGRP_RESP_CODE(result);
David Woodhousea222a7f2015-10-07 23:35:18 +0100629 } else if (req->srr) {
630 /* Page Stream Response */
Lu Baolu5d308fc2018-12-10 09:58:58 +0800631 resp.qw0 = QI_PSTRM_IDX(req->prg_index) |
632 QI_PSTRM_PRIV(req->private) |
633 QI_PSTRM_BUS(req->bus) |
634 QI_PSTRM_PASID(req->pasid) |
635 QI_PSTRM_RESP_TYPE;
636 resp.qw1 = QI_PSTRM_ADDR(address) |
637 QI_PSTRM_DEVFN(req->devfn) |
David Woodhousea222a7f2015-10-07 23:35:18 +0100638 QI_PSTRM_RESP_CODE(result);
David Woodhousea222a7f2015-10-07 23:35:18 +0100639 }
Lu Baolu5d308fc2018-12-10 09:58:58 +0800640 resp.qw2 = 0;
641 resp.qw3 = 0;
642 qi_submit_sync(&resp, iommu);
David Woodhousea222a7f2015-10-07 23:35:18 +0100643
644 head = (head + sizeof(*req)) & PRQ_RING_MASK;
645 }
646
647 dmar_writeq(iommu->reg + DMAR_PQH_REG, tail);
648
649 return IRQ_RETVAL(handled);
650}