blob: a2dca27aebc3379f90e2863c75320c41a92b37a7 [file] [log] [blame]
Dave Jiangbfe1d562020-01-21 16:43:59 -07001// SPDX-License-Identifier: GPL-2.0
2/* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
3#include <linux/init.h>
4#include <linux/kernel.h>
5#include <linux/module.h>
6#include <linux/slab.h>
7#include <linux/pci.h>
8#include <linux/interrupt.h>
9#include <linux/delay.h>
10#include <linux/dma-mapping.h>
11#include <linux/workqueue.h>
12#include <linux/aer.h>
13#include <linux/fs.h>
14#include <linux/io-64-nonatomic-lo-hi.h>
15#include <linux/device.h>
16#include <linux/idr.h>
Dave Jiang8e50d392020-10-27 10:34:35 -070017#include <linux/intel-svm.h>
18#include <linux/iommu.h>
Dave Jiangbfe1d562020-01-21 16:43:59 -070019#include <uapi/linux/idxd.h>
Dave Jiang8f47d1a2020-01-21 16:44:23 -070020#include <linux/dmaengine.h>
21#include "../dmaengine.h"
Dave Jiangbfe1d562020-01-21 16:43:59 -070022#include "registers.h"
23#include "idxd.h"
24
25MODULE_VERSION(IDXD_DRIVER_VERSION);
26MODULE_LICENSE("GPL v2");
27MODULE_AUTHOR("Intel Corporation");
28
Dave Jiang03d939c2021-01-22 11:46:00 -070029static bool sva = true;
30module_param(sva, bool, 0644);
31MODULE_PARM_DESC(sva, "Toggle SVA support on/off");
32
Dave Jiangbfe1d562020-01-21 16:43:59 -070033#define DRV_NAME "idxd"
34
Dave Jiang8e50d392020-10-27 10:34:35 -070035bool support_enqcmd;
36
Dave Jiangf7f77392021-04-15 16:37:27 -070037static struct ida idxd_idas[IDXD_TYPE_MAX];
Dave Jiangbfe1d562020-01-21 16:43:59 -070038
39static struct pci_device_id idxd_pci_tbl[] = {
40 /* DSA ver 1.0 platforms */
41 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_DSA_SPR0) },
Dave Jiangf25b46382020-11-17 13:39:14 -070042
43 /* IAX ver 1.0 platforms */
44 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IAX_SPR0) },
Dave Jiangbfe1d562020-01-21 16:43:59 -070045 { 0, }
46};
47MODULE_DEVICE_TABLE(pci, idxd_pci_tbl);
48
49static char *idxd_name[] = {
50 "dsa",
Dave Jiangf25b46382020-11-17 13:39:14 -070051 "iax"
Dave Jiangbfe1d562020-01-21 16:43:59 -070052};
53
Dave Jiang47c16ac2021-04-15 16:37:33 -070054struct ida *idxd_ida(struct idxd_device *idxd)
55{
56 return &idxd_idas[idxd->type];
57}
58
Dave Jiangbfe1d562020-01-21 16:43:59 -070059const char *idxd_get_dev_name(struct idxd_device *idxd)
60{
61 return idxd_name[idxd->type];
62}
63
64static int idxd_setup_interrupts(struct idxd_device *idxd)
65{
66 struct pci_dev *pdev = idxd->pdev;
67 struct device *dev = &pdev->dev;
Dave Jiangbfe1d562020-01-21 16:43:59 -070068 struct idxd_irq_entry *irq_entry;
69 int i, msixcnt;
70 int rc = 0;
71
72 msixcnt = pci_msix_vec_count(pdev);
73 if (msixcnt < 0) {
74 dev_err(dev, "Not MSI-X interrupt capable.\n");
Dave Jiang5fc8e852021-04-15 16:37:15 -070075 return -ENOSPC;
Dave Jiangbfe1d562020-01-21 16:43:59 -070076 }
77
Dave Jiang5fc8e852021-04-15 16:37:15 -070078 rc = pci_alloc_irq_vectors(pdev, msixcnt, msixcnt, PCI_IRQ_MSIX);
79 if (rc != msixcnt) {
80 dev_err(dev, "Failed enabling %d MSIX entries: %d\n", msixcnt, rc);
81 return -ENOSPC;
Dave Jiangbfe1d562020-01-21 16:43:59 -070082 }
83 dev_dbg(dev, "Enabled %d msix vectors\n", msixcnt);
84
85 /*
86 * We implement 1 completion list per MSI-X entry except for
87 * entry 0, which is for errors and others.
88 */
Dave Jiang47c16ac2021-04-15 16:37:33 -070089 idxd->irq_entries = kcalloc_node(msixcnt, sizeof(struct idxd_irq_entry),
90 GFP_KERNEL, dev_to_node(dev));
Dave Jiangbfe1d562020-01-21 16:43:59 -070091 if (!idxd->irq_entries) {
92 rc = -ENOMEM;
Dave Jiang5fc8e852021-04-15 16:37:15 -070093 goto err_irq_entries;
Dave Jiangbfe1d562020-01-21 16:43:59 -070094 }
95
96 for (i = 0; i < msixcnt; i++) {
97 idxd->irq_entries[i].id = i;
98 idxd->irq_entries[i].idxd = idxd;
Dave Jiang5fc8e852021-04-15 16:37:15 -070099 idxd->irq_entries[i].vector = pci_irq_vector(pdev, i);
Dave Jiange4f4d8c2020-10-27 10:34:40 -0700100 spin_lock_init(&idxd->irq_entries[i].list_lock);
Dave Jiangbfe1d562020-01-21 16:43:59 -0700101 }
102
Dave Jiangbfe1d562020-01-21 16:43:59 -0700103 irq_entry = &idxd->irq_entries[0];
Dave Jiang5fc8e852021-04-15 16:37:15 -0700104 rc = request_threaded_irq(irq_entry->vector, idxd_irq_handler, idxd_misc_thread,
105 0, "idxd-misc", irq_entry);
Dave Jiangbfe1d562020-01-21 16:43:59 -0700106 if (rc < 0) {
107 dev_err(dev, "Failed to allocate misc interrupt.\n");
Dave Jiang5fc8e852021-04-15 16:37:15 -0700108 goto err_misc_irq;
Dave Jiangbfe1d562020-01-21 16:43:59 -0700109 }
110
Dave Jiang5fc8e852021-04-15 16:37:15 -0700111 dev_dbg(dev, "Allocated idxd-misc handler on msix vector %d\n", irq_entry->vector);
Dave Jiangbfe1d562020-01-21 16:43:59 -0700112
113 /* first MSI-X entry is not for wq interrupts */
114 idxd->num_wq_irqs = msixcnt - 1;
115
116 for (i = 1; i < msixcnt; i++) {
Dave Jiangbfe1d562020-01-21 16:43:59 -0700117 irq_entry = &idxd->irq_entries[i];
118
119 init_llist_head(&idxd->irq_entries[i].pending_llist);
120 INIT_LIST_HEAD(&idxd->irq_entries[i].work_list);
Dave Jiang5fc8e852021-04-15 16:37:15 -0700121 rc = request_threaded_irq(irq_entry->vector, idxd_irq_handler,
122 idxd_wq_thread, 0, "idxd-portal", irq_entry);
Dave Jiangbfe1d562020-01-21 16:43:59 -0700123 if (rc < 0) {
Dave Jiang5fc8e852021-04-15 16:37:15 -0700124 dev_err(dev, "Failed to allocate irq %d.\n", irq_entry->vector);
125 goto err_wq_irqs;
Dave Jiangbfe1d562020-01-21 16:43:59 -0700126 }
Dave Jiang5fc8e852021-04-15 16:37:15 -0700127 dev_dbg(dev, "Allocated idxd-msix %d for vector %d\n", i, irq_entry->vector);
Dave Jiangbfe1d562020-01-21 16:43:59 -0700128 }
129
130 idxd_unmask_error_interrupts(idxd);
Dave Jiang6df0e6c2021-04-12 09:23:27 -0700131 idxd_msix_perm_setup(idxd);
Dave Jiangbfe1d562020-01-21 16:43:59 -0700132 return 0;
133
Dave Jiang5fc8e852021-04-15 16:37:15 -0700134 err_wq_irqs:
135 while (--i >= 0) {
136 irq_entry = &idxd->irq_entries[i];
137 free_irq(irq_entry->vector, irq_entry);
138 }
139 err_misc_irq:
Dave Jiangbfe1d562020-01-21 16:43:59 -0700140 /* Disable error interrupt generation */
141 idxd_mask_error_interrupts(idxd);
Dave Jiang5fc8e852021-04-15 16:37:15 -0700142 err_irq_entries:
143 pci_free_irq_vectors(pdev);
Dave Jiangbfe1d562020-01-21 16:43:59 -0700144 dev_err(dev, "No usable interrupts\n");
145 return rc;
146}
147
Dave Jiang7c5dd232021-04-15 16:37:39 -0700148static int idxd_setup_wqs(struct idxd_device *idxd)
149{
150 struct device *dev = &idxd->pdev->dev;
151 struct idxd_wq *wq;
152 int i, rc;
153
154 idxd->wqs = kcalloc_node(idxd->max_wqs, sizeof(struct idxd_wq *),
155 GFP_KERNEL, dev_to_node(dev));
156 if (!idxd->wqs)
157 return -ENOMEM;
158
159 for (i = 0; i < idxd->max_wqs; i++) {
160 wq = kzalloc_node(sizeof(*wq), GFP_KERNEL, dev_to_node(dev));
161 if (!wq) {
162 rc = -ENOMEM;
163 goto err;
164 }
165
166 wq->id = i;
167 wq->idxd = idxd;
168 device_initialize(&wq->conf_dev);
169 wq->conf_dev.parent = &idxd->conf_dev;
170 wq->conf_dev.bus = idxd_get_bus_type(idxd);
171 wq->conf_dev.type = &idxd_wq_device_type;
172 rc = dev_set_name(&wq->conf_dev, "wq%d.%d", idxd->id, wq->id);
173 if (rc < 0) {
174 put_device(&wq->conf_dev);
175 goto err;
176 }
177
178 mutex_init(&wq->wq_lock);
179 wq->idxd_cdev.minor = -1;
180 wq->max_xfer_bytes = idxd->max_xfer_bytes;
181 wq->max_batch_size = idxd->max_batch_size;
182 wq->wqcfg = kzalloc_node(idxd->wqcfg_size, GFP_KERNEL, dev_to_node(dev));
183 if (!wq->wqcfg) {
184 put_device(&wq->conf_dev);
185 rc = -ENOMEM;
186 goto err;
187 }
188 idxd->wqs[i] = wq;
189 }
190
191 return 0;
192
193 err:
194 while (--i >= 0)
195 put_device(&idxd->wqs[i]->conf_dev);
196 return rc;
197}
198
Dave Jiangbfe1d562020-01-21 16:43:59 -0700199static int idxd_setup_internals(struct idxd_device *idxd)
200{
201 struct device *dev = &idxd->pdev->dev;
Dave Jiang7c5dd232021-04-15 16:37:39 -0700202 int i, rc;
Dave Jiangbfe1d562020-01-21 16:43:59 -0700203
Dave Jiang0d5c10b2020-06-26 11:11:18 -0700204 init_waitqueue_head(&idxd->cmd_waitq);
Dave Jiang7c5dd232021-04-15 16:37:39 -0700205
206 rc = idxd_setup_wqs(idxd);
207 if (rc < 0)
208 return rc;
209
Dave Jiangbfe1d562020-01-21 16:43:59 -0700210 idxd->groups = devm_kcalloc(dev, idxd->max_groups,
211 sizeof(struct idxd_group), GFP_KERNEL);
Dave Jiang7c5dd232021-04-15 16:37:39 -0700212 if (!idxd->groups) {
213 rc = -ENOMEM;
214 goto err;
215 }
Dave Jiangbfe1d562020-01-21 16:43:59 -0700216
217 for (i = 0; i < idxd->max_groups; i++) {
218 idxd->groups[i].idxd = idxd;
219 idxd->groups[i].id = i;
220 idxd->groups[i].tc_a = -1;
221 idxd->groups[i].tc_b = -1;
222 }
223
Dave Jiangbfe1d562020-01-21 16:43:59 -0700224 idxd->engines = devm_kcalloc(dev, idxd->max_engines,
225 sizeof(struct idxd_engine), GFP_KERNEL);
Dave Jiang7c5dd232021-04-15 16:37:39 -0700226 if (!idxd->engines) {
227 rc = -ENOMEM;
228 goto err;
Dave Jiangbfe1d562020-01-21 16:43:59 -0700229 }
230
Dave Jiang7c5dd232021-04-15 16:37:39 -0700231
Dave Jiangbfe1d562020-01-21 16:43:59 -0700232 for (i = 0; i < idxd->max_engines; i++) {
233 idxd->engines[i].idxd = idxd;
234 idxd->engines[i].id = i;
235 }
236
Dave Jiang0d5c10b2020-06-26 11:11:18 -0700237 idxd->wq = create_workqueue(dev_name(dev));
Dave Jiang7c5dd232021-04-15 16:37:39 -0700238 if (!idxd->wq) {
239 rc = -ENOMEM;
240 goto err;
241 }
Dave Jiang0d5c10b2020-06-26 11:11:18 -0700242
Dave Jiangbfe1d562020-01-21 16:43:59 -0700243 return 0;
Dave Jiang7c5dd232021-04-15 16:37:39 -0700244
245 err:
246 for (i = 0; i < idxd->max_wqs; i++)
247 put_device(&idxd->wqs[i]->conf_dev);
248 return rc;
Dave Jiangbfe1d562020-01-21 16:43:59 -0700249}
250
251static void idxd_read_table_offsets(struct idxd_device *idxd)
252{
253 union offsets_reg offsets;
254 struct device *dev = &idxd->pdev->dev;
255
256 offsets.bits[0] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET);
Dave Jiang2f8417a2020-10-30 08:51:56 -0700257 offsets.bits[1] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET + sizeof(u64));
258 idxd->grpcfg_offset = offsets.grpcfg * IDXD_TABLE_MULT;
Dave Jiangbfe1d562020-01-21 16:43:59 -0700259 dev_dbg(dev, "IDXD Group Config Offset: %#x\n", idxd->grpcfg_offset);
Dave Jiang2f8417a2020-10-30 08:51:56 -0700260 idxd->wqcfg_offset = offsets.wqcfg * IDXD_TABLE_MULT;
261 dev_dbg(dev, "IDXD Work Queue Config Offset: %#x\n", idxd->wqcfg_offset);
262 idxd->msix_perm_offset = offsets.msix_perm * IDXD_TABLE_MULT;
263 dev_dbg(dev, "IDXD MSIX Permission Offset: %#x\n", idxd->msix_perm_offset);
264 idxd->perfmon_offset = offsets.perfmon * IDXD_TABLE_MULT;
Dave Jiangbfe1d562020-01-21 16:43:59 -0700265 dev_dbg(dev, "IDXD Perfmon Offset: %#x\n", idxd->perfmon_offset);
266}
267
268static void idxd_read_caps(struct idxd_device *idxd)
269{
270 struct device *dev = &idxd->pdev->dev;
271 int i;
272
273 /* reading generic capabilities */
274 idxd->hw.gen_cap.bits = ioread64(idxd->reg_base + IDXD_GENCAP_OFFSET);
275 dev_dbg(dev, "gen_cap: %#llx\n", idxd->hw.gen_cap.bits);
276 idxd->max_xfer_bytes = 1ULL << idxd->hw.gen_cap.max_xfer_shift;
277 dev_dbg(dev, "max xfer size: %llu bytes\n", idxd->max_xfer_bytes);
278 idxd->max_batch_size = 1U << idxd->hw.gen_cap.max_batch_shift;
279 dev_dbg(dev, "max batch size: %u\n", idxd->max_batch_size);
280 if (idxd->hw.gen_cap.config_en)
281 set_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags);
282
283 /* reading group capabilities */
284 idxd->hw.group_cap.bits =
285 ioread64(idxd->reg_base + IDXD_GRPCAP_OFFSET);
286 dev_dbg(dev, "group_cap: %#llx\n", idxd->hw.group_cap.bits);
287 idxd->max_groups = idxd->hw.group_cap.num_groups;
288 dev_dbg(dev, "max groups: %u\n", idxd->max_groups);
289 idxd->max_tokens = idxd->hw.group_cap.total_tokens;
290 dev_dbg(dev, "max tokens: %u\n", idxd->max_tokens);
Dave Jiangc52ca472020-01-21 16:44:05 -0700291 idxd->nr_tokens = idxd->max_tokens;
Dave Jiangbfe1d562020-01-21 16:43:59 -0700292
293 /* read engine capabilities */
294 idxd->hw.engine_cap.bits =
295 ioread64(idxd->reg_base + IDXD_ENGCAP_OFFSET);
296 dev_dbg(dev, "engine_cap: %#llx\n", idxd->hw.engine_cap.bits);
297 idxd->max_engines = idxd->hw.engine_cap.num_engines;
298 dev_dbg(dev, "max engines: %u\n", idxd->max_engines);
299
300 /* read workqueue capabilities */
301 idxd->hw.wq_cap.bits = ioread64(idxd->reg_base + IDXD_WQCAP_OFFSET);
302 dev_dbg(dev, "wq_cap: %#llx\n", idxd->hw.wq_cap.bits);
303 idxd->max_wq_size = idxd->hw.wq_cap.total_wq_size;
304 dev_dbg(dev, "total workqueue size: %u\n", idxd->max_wq_size);
305 idxd->max_wqs = idxd->hw.wq_cap.num_wqs;
306 dev_dbg(dev, "max workqueues: %u\n", idxd->max_wqs);
Dave Jiangd98793b2020-10-27 14:34:09 -0700307 idxd->wqcfg_size = 1 << (idxd->hw.wq_cap.wqcfg_size + IDXD_WQCFG_MIN);
308 dev_dbg(dev, "wqcfg size: %u\n", idxd->wqcfg_size);
Dave Jiangbfe1d562020-01-21 16:43:59 -0700309
310 /* reading operation capabilities */
311 for (i = 0; i < 4; i++) {
312 idxd->hw.opcap.bits[i] = ioread64(idxd->reg_base +
313 IDXD_OPCAP_OFFSET + i * sizeof(u64));
314 dev_dbg(dev, "opcap[%d]: %#llx\n", i, idxd->hw.opcap.bits[i]);
315 }
316}
317
Dave Jiang47c16ac2021-04-15 16:37:33 -0700318static inline void idxd_set_type(struct idxd_device *idxd)
319{
320 struct pci_dev *pdev = idxd->pdev;
321
322 if (pdev->device == PCI_DEVICE_ID_INTEL_DSA_SPR0)
323 idxd->type = IDXD_TYPE_DSA;
324 else if (pdev->device == PCI_DEVICE_ID_INTEL_IAX_SPR0)
325 idxd->type = IDXD_TYPE_IAX;
326 else
327 idxd->type = IDXD_TYPE_UNKNOWN;
328}
329
Dave Jiang8e50d392020-10-27 10:34:35 -0700330static struct idxd_device *idxd_alloc(struct pci_dev *pdev)
Dave Jiangbfe1d562020-01-21 16:43:59 -0700331{
332 struct device *dev = &pdev->dev;
333 struct idxd_device *idxd;
Dave Jiang47c16ac2021-04-15 16:37:33 -0700334 int rc;
Dave Jiangbfe1d562020-01-21 16:43:59 -0700335
Dave Jiang47c16ac2021-04-15 16:37:33 -0700336 idxd = kzalloc_node(sizeof(*idxd), GFP_KERNEL, dev_to_node(dev));
Dave Jiangbfe1d562020-01-21 16:43:59 -0700337 if (!idxd)
338 return NULL;
339
340 idxd->pdev = pdev;
Dave Jiang47c16ac2021-04-15 16:37:33 -0700341 idxd_set_type(idxd);
342 idxd->id = ida_alloc(idxd_ida(idxd), GFP_KERNEL);
343 if (idxd->id < 0)
344 return NULL;
345
346 device_initialize(&idxd->conf_dev);
347 idxd->conf_dev.parent = dev;
348 idxd->conf_dev.bus = idxd_get_bus_type(idxd);
349 idxd->conf_dev.type = idxd_get_device_type(idxd);
350 rc = dev_set_name(&idxd->conf_dev, "%s%d", idxd_get_dev_name(idxd), idxd->id);
351 if (rc < 0) {
352 put_device(&idxd->conf_dev);
353 return NULL;
354 }
355
Dave Jiangbfe1d562020-01-21 16:43:59 -0700356 spin_lock_init(&idxd->dev_lock);
357
358 return idxd;
359}
360
Dave Jiang8e50d392020-10-27 10:34:35 -0700361static int idxd_enable_system_pasid(struct idxd_device *idxd)
362{
363 int flags;
364 unsigned int pasid;
365 struct iommu_sva *sva;
366
367 flags = SVM_FLAG_SUPERVISOR_MODE;
368
369 sva = iommu_sva_bind_device(&idxd->pdev->dev, NULL, &flags);
370 if (IS_ERR(sva)) {
371 dev_warn(&idxd->pdev->dev,
372 "iommu sva bind failed: %ld\n", PTR_ERR(sva));
373 return PTR_ERR(sva);
374 }
375
376 pasid = iommu_sva_get_pasid(sva);
377 if (pasid == IOMMU_PASID_INVALID) {
378 iommu_sva_unbind_device(sva);
379 return -ENODEV;
380 }
381
382 idxd->sva = sva;
383 idxd->pasid = pasid;
384 dev_dbg(&idxd->pdev->dev, "system pasid: %u\n", pasid);
385 return 0;
386}
387
388static void idxd_disable_system_pasid(struct idxd_device *idxd)
389{
390
391 iommu_sva_unbind_device(idxd->sva);
392 idxd->sva = NULL;
393}
394
Dave Jiangbfe1d562020-01-21 16:43:59 -0700395static int idxd_probe(struct idxd_device *idxd)
396{
397 struct pci_dev *pdev = idxd->pdev;
398 struct device *dev = &pdev->dev;
399 int rc;
400
401 dev_dbg(dev, "%s entered and resetting device\n", __func__);
Dave Jiang89e3bec2021-02-01 08:26:14 -0700402 rc = idxd_device_init_reset(idxd);
403 if (rc < 0)
404 return rc;
405
Dave Jiangbfe1d562020-01-21 16:43:59 -0700406 dev_dbg(dev, "IDXD reset complete\n");
407
Dave Jiang03d939c2021-01-22 11:46:00 -0700408 if (IS_ENABLED(CONFIG_INTEL_IDXD_SVM) && sva) {
Dave Jiang8e50d392020-10-27 10:34:35 -0700409 rc = idxd_enable_system_pasid(idxd);
410 if (rc < 0)
411 dev_warn(dev, "Failed to enable PASID. No SVA support: %d\n", rc);
412 else
413 set_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags);
Dave Jiang03d939c2021-01-22 11:46:00 -0700414 } else if (!sva) {
415 dev_warn(dev, "User forced SVA off via module param.\n");
Dave Jiang8e50d392020-10-27 10:34:35 -0700416 }
417
Dave Jiangbfe1d562020-01-21 16:43:59 -0700418 idxd_read_caps(idxd);
419 idxd_read_table_offsets(idxd);
420
421 rc = idxd_setup_internals(idxd);
422 if (rc)
Dave Jiang7c5dd232021-04-15 16:37:39 -0700423 goto err;
Dave Jiangbfe1d562020-01-21 16:43:59 -0700424
425 rc = idxd_setup_interrupts(idxd);
426 if (rc)
Dave Jiang7c5dd232021-04-15 16:37:39 -0700427 goto err;
Dave Jiangbfe1d562020-01-21 16:43:59 -0700428
429 dev_dbg(dev, "IDXD interrupt setup complete.\n");
430
Dave Jiang42d279f2020-01-21 16:44:29 -0700431 idxd->major = idxd_cdev_get_major(idxd);
432
Dave Jiangbfe1d562020-01-21 16:43:59 -0700433 dev_dbg(dev, "IDXD device %d probed successfully\n", idxd->id);
434 return 0;
435
Dave Jiang7c5dd232021-04-15 16:37:39 -0700436 err:
Dave Jiang8e50d392020-10-27 10:34:35 -0700437 if (device_pasid_enabled(idxd))
438 idxd_disable_system_pasid(idxd);
Dave Jiangbfe1d562020-01-21 16:43:59 -0700439 return rc;
440}
441
Dave Jiangf25b46382020-11-17 13:39:14 -0700442static void idxd_type_init(struct idxd_device *idxd)
443{
444 if (idxd->type == IDXD_TYPE_DSA)
445 idxd->compl_size = sizeof(struct dsa_completion_record);
446 else if (idxd->type == IDXD_TYPE_IAX)
447 idxd->compl_size = sizeof(struct iax_completion_record);
448}
449
Dave Jiangbfe1d562020-01-21 16:43:59 -0700450static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
451{
Dave Jiangbfe1d562020-01-21 16:43:59 -0700452 struct device *dev = &pdev->dev;
453 struct idxd_device *idxd;
454 int rc;
Dave Jiangbfe1d562020-01-21 16:43:59 -0700455
Dave Jianga39c7cd2021-04-15 16:37:21 -0700456 rc = pci_enable_device(pdev);
Dave Jiangbfe1d562020-01-21 16:43:59 -0700457 if (rc)
458 return rc;
459
Dave Jiang8e50d392020-10-27 10:34:35 -0700460 dev_dbg(dev, "Alloc IDXD context\n");
461 idxd = idxd_alloc(pdev);
Dave Jianga39c7cd2021-04-15 16:37:21 -0700462 if (!idxd) {
463 rc = -ENOMEM;
464 goto err_idxd_alloc;
465 }
Dave Jiangbfe1d562020-01-21 16:43:59 -0700466
Dave Jiang8e50d392020-10-27 10:34:35 -0700467 dev_dbg(dev, "Mapping BARs\n");
Dave Jianga39c7cd2021-04-15 16:37:21 -0700468 idxd->reg_base = pci_iomap(pdev, IDXD_MMIO_BAR, 0);
469 if (!idxd->reg_base) {
470 rc = -ENOMEM;
471 goto err_iomap;
472 }
Dave Jiangbfe1d562020-01-21 16:43:59 -0700473
474 dev_dbg(dev, "Set DMA masks\n");
475 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
476 if (rc)
477 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
478 if (rc)
Dave Jianga39c7cd2021-04-15 16:37:21 -0700479 goto err;
Dave Jiangbfe1d562020-01-21 16:43:59 -0700480
481 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
482 if (rc)
483 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
484 if (rc)
Dave Jianga39c7cd2021-04-15 16:37:21 -0700485 goto err;
Dave Jiangbfe1d562020-01-21 16:43:59 -0700486
Dave Jiangbfe1d562020-01-21 16:43:59 -0700487
Dave Jiangf25b46382020-11-17 13:39:14 -0700488 idxd_type_init(idxd);
489
Dave Jiangbfe1d562020-01-21 16:43:59 -0700490 dev_dbg(dev, "Set PCI master\n");
491 pci_set_master(pdev);
492 pci_set_drvdata(pdev, idxd);
493
494 idxd->hw.version = ioread32(idxd->reg_base + IDXD_VER_OFFSET);
495 rc = idxd_probe(idxd);
496 if (rc) {
497 dev_err(dev, "Intel(R) IDXD DMA Engine init failed\n");
Dave Jianga39c7cd2021-04-15 16:37:21 -0700498 goto err;
Dave Jiangbfe1d562020-01-21 16:43:59 -0700499 }
500
Dave Jiang47c16ac2021-04-15 16:37:33 -0700501 rc = idxd_register_devices(idxd);
Dave Jiangc52ca472020-01-21 16:44:05 -0700502 if (rc) {
503 dev_err(dev, "IDXD sysfs setup failed\n");
Dave Jianga39c7cd2021-04-15 16:37:21 -0700504 goto err;
Dave Jiangc52ca472020-01-21 16:44:05 -0700505 }
506
507 idxd->state = IDXD_DEV_CONF_READY;
508
Dave Jiangbfe1d562020-01-21 16:43:59 -0700509 dev_info(&pdev->dev, "Intel(R) Accelerator Device (v%x)\n",
510 idxd->hw.version);
511
512 return 0;
Dave Jianga39c7cd2021-04-15 16:37:21 -0700513
514 err:
515 pci_iounmap(pdev, idxd->reg_base);
516 err_iomap:
Dave Jiang47c16ac2021-04-15 16:37:33 -0700517 put_device(&idxd->conf_dev);
Dave Jianga39c7cd2021-04-15 16:37:21 -0700518 err_idxd_alloc:
519 pci_disable_device(pdev);
520 return rc;
Dave Jiangbfe1d562020-01-21 16:43:59 -0700521}
522
Dave Jiang8f47d1a2020-01-21 16:44:23 -0700523static void idxd_flush_pending_llist(struct idxd_irq_entry *ie)
524{
525 struct idxd_desc *desc, *itr;
526 struct llist_node *head;
527
528 head = llist_del_all(&ie->pending_llist);
529 if (!head)
530 return;
531
532 llist_for_each_entry_safe(desc, itr, head, llnode) {
533 idxd_dma_complete_txd(desc, IDXD_COMPLETE_ABORT);
534 idxd_free_desc(desc->wq, desc);
535 }
536}
537
538static void idxd_flush_work_list(struct idxd_irq_entry *ie)
539{
540 struct idxd_desc *desc, *iter;
541
542 list_for_each_entry_safe(desc, iter, &ie->work_list, list) {
543 list_del(&desc->list);
544 idxd_dma_complete_txd(desc, IDXD_COMPLETE_ABORT);
545 idxd_free_desc(desc->wq, desc);
546 }
547}
548
Dave Jiangbfe1d562020-01-21 16:43:59 -0700549static void idxd_shutdown(struct pci_dev *pdev)
550{
551 struct idxd_device *idxd = pci_get_drvdata(pdev);
552 int rc, i;
553 struct idxd_irq_entry *irq_entry;
554 int msixcnt = pci_msix_vec_count(pdev);
Dave Jiangbfe1d562020-01-21 16:43:59 -0700555
Dave Jiangbfe1d562020-01-21 16:43:59 -0700556 rc = idxd_device_disable(idxd);
Dave Jiangbfe1d562020-01-21 16:43:59 -0700557 if (rc)
558 dev_err(&pdev->dev, "Disabling device failed\n");
559
560 dev_dbg(&pdev->dev, "%s called\n", __func__);
561 idxd_mask_msix_vectors(idxd);
562 idxd_mask_error_interrupts(idxd);
563
564 for (i = 0; i < msixcnt; i++) {
565 irq_entry = &idxd->irq_entries[i];
Dave Jiang5fc8e852021-04-15 16:37:15 -0700566 synchronize_irq(irq_entry->vector);
567 free_irq(irq_entry->vector, irq_entry);
Dave Jiangbfe1d562020-01-21 16:43:59 -0700568 if (i == 0)
569 continue;
Dave Jiang8f47d1a2020-01-21 16:44:23 -0700570 idxd_flush_pending_llist(irq_entry);
571 idxd_flush_work_list(irq_entry);
Dave Jiangbfe1d562020-01-21 16:43:59 -0700572 }
Dave Jiang0d5c10b2020-06-26 11:11:18 -0700573
Dave Jiang6df0e6c2021-04-12 09:23:27 -0700574 idxd_msix_perm_clear(idxd);
Dave Jiang5fc8e852021-04-15 16:37:15 -0700575 pci_free_irq_vectors(pdev);
Dave Jianga39c7cd2021-04-15 16:37:21 -0700576 pci_iounmap(pdev, idxd->reg_base);
577 pci_disable_device(pdev);
Dave Jiang0d5c10b2020-06-26 11:11:18 -0700578 destroy_workqueue(idxd->wq);
Dave Jiangbfe1d562020-01-21 16:43:59 -0700579}
580
581static void idxd_remove(struct pci_dev *pdev)
582{
583 struct idxd_device *idxd = pci_get_drvdata(pdev);
584
585 dev_dbg(&pdev->dev, "%s called\n", __func__);
586 idxd_shutdown(pdev);
Dave Jiang8e50d392020-10-27 10:34:35 -0700587 if (device_pasid_enabled(idxd))
588 idxd_disable_system_pasid(idxd);
Dave Jiang47c16ac2021-04-15 16:37:33 -0700589 idxd_unregister_devices(idxd);
Dave Jiangbfe1d562020-01-21 16:43:59 -0700590}
591
592static struct pci_driver idxd_pci_driver = {
593 .name = DRV_NAME,
594 .id_table = idxd_pci_tbl,
595 .probe = idxd_pci_probe,
596 .remove = idxd_remove,
597 .shutdown = idxd_shutdown,
598};
599
600static int __init idxd_init_module(void)
601{
602 int err, i;
603
604 /*
Dave Jiang8e50d392020-10-27 10:34:35 -0700605 * If the CPU does not support MOVDIR64B or ENQCMDS, there's no point in
Dave Jiangbfe1d562020-01-21 16:43:59 -0700606 * enumerating the device. We can not utilize it.
607 */
608 if (!boot_cpu_has(X86_FEATURE_MOVDIR64B)) {
609 pr_warn("idxd driver failed to load without MOVDIR64B.\n");
610 return -ENODEV;
611 }
612
Dave Jiang8e50d392020-10-27 10:34:35 -0700613 if (!boot_cpu_has(X86_FEATURE_ENQCMD))
614 pr_warn("Platform does not have ENQCMD(S) support.\n");
615 else
616 support_enqcmd = true;
Dave Jiangbfe1d562020-01-21 16:43:59 -0700617
Dave Jiangbfe1d562020-01-21 16:43:59 -0700618 for (i = 0; i < IDXD_TYPE_MAX; i++)
Dave Jiangf7f77392021-04-15 16:37:27 -0700619 ida_init(&idxd_idas[i]);
Dave Jiangbfe1d562020-01-21 16:43:59 -0700620
Dave Jiangc52ca472020-01-21 16:44:05 -0700621 err = idxd_register_bus_type();
622 if (err < 0)
Dave Jiangbfe1d562020-01-21 16:43:59 -0700623 return err;
624
Dave Jiangc52ca472020-01-21 16:44:05 -0700625 err = idxd_register_driver();
626 if (err < 0)
627 goto err_idxd_driver_register;
628
Dave Jiang42d279f2020-01-21 16:44:29 -0700629 err = idxd_cdev_register();
630 if (err)
631 goto err_cdev_register;
632
Dave Jiangc52ca472020-01-21 16:44:05 -0700633 err = pci_register_driver(&idxd_pci_driver);
634 if (err)
635 goto err_pci_register;
636
Dave Jiangbfe1d562020-01-21 16:43:59 -0700637 return 0;
Dave Jiangc52ca472020-01-21 16:44:05 -0700638
639err_pci_register:
Dave Jiang42d279f2020-01-21 16:44:29 -0700640 idxd_cdev_remove();
641err_cdev_register:
Dave Jiangc52ca472020-01-21 16:44:05 -0700642 idxd_unregister_driver();
643err_idxd_driver_register:
644 idxd_unregister_bus_type();
645 return err;
Dave Jiangbfe1d562020-01-21 16:43:59 -0700646}
647module_init(idxd_init_module);
648
649static void __exit idxd_exit_module(void)
650{
651 pci_unregister_driver(&idxd_pci_driver);
Dave Jiang42d279f2020-01-21 16:44:29 -0700652 idxd_cdev_remove();
Dave Jiangc52ca472020-01-21 16:44:05 -0700653 idxd_unregister_bus_type();
Dave Jiangbfe1d562020-01-21 16:43:59 -0700654}
655module_exit(idxd_exit_module);