Thomas Gleixner | d2912cb | 2019-06-04 10:11:33 +0200 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0-only |
Jiang Liu | 4438098 | 2014-10-27 16:12:02 +0800 | [diff] [blame] | 2 | /* |
| 3 | * Support of MSI, HPET and DMAR interrupts. |
| 4 | * |
| 5 | * Copyright (C) 1997, 1998, 1999, 2000, 2009 Ingo Molnar, Hajnalka Szabo |
| 6 | * Moved from arch/x86/kernel/apic/io_apic.c. |
Jiang Liu | 52f518a | 2015-04-13 14:11:35 +0800 | [diff] [blame] | 7 | * Jiang Liu <jiang.liu@linux.intel.com> |
| 8 | * Convert to hierarchical irqdomain |
Jiang Liu | 4438098 | 2014-10-27 16:12:02 +0800 | [diff] [blame] | 9 | */ |
| 10 | #include <linux/mm.h> |
| 11 | #include <linux/interrupt.h> |
Nicolai Stange | 447ae31 | 2018-07-29 12:15:33 +0200 | [diff] [blame] | 12 | #include <linux/irq.h> |
Jiang Liu | 4438098 | 2014-10-27 16:12:02 +0800 | [diff] [blame] | 13 | #include <linux/pci.h> |
| 14 | #include <linux/dmar.h> |
| 15 | #include <linux/hpet.h> |
| 16 | #include <linux/msi.h> |
Jiang Liu | d746d1e | 2015-04-14 10:30:09 +0800 | [diff] [blame] | 17 | #include <asm/irqdomain.h> |
Jiang Liu | 4438098 | 2014-10-27 16:12:02 +0800 | [diff] [blame] | 18 | #include <asm/msidef.h> |
| 19 | #include <asm/hpet.h> |
| 20 | #include <asm/hw_irq.h> |
| 21 | #include <asm/apic.h> |
| 22 | #include <asm/irq_remapping.h> |
| 23 | |
Jiang Liu | 52f518a | 2015-04-13 14:11:35 +0800 | [diff] [blame] | 24 | static struct irq_domain *msi_default_domain; |
| 25 | |
Thomas Gleixner | 6f1a489 | 2020-01-31 15:26:52 +0100 | [diff] [blame^] | 26 | static void __irq_msi_compose_msg(struct irq_cfg *cfg, struct msi_msg *msg) |
Jiang Liu | 3cb96f0 | 2015-04-13 14:11:34 +0800 | [diff] [blame] | 27 | { |
Jiang Liu | 3cb96f0 | 2015-04-13 14:11:34 +0800 | [diff] [blame] | 28 | msg->address_hi = MSI_ADDR_BASE_HI; |
| 29 | |
| 30 | if (x2apic_enabled()) |
| 31 | msg->address_hi |= MSI_ADDR_EXT_DEST_ID(cfg->dest_apicid); |
| 32 | |
| 33 | msg->address_lo = |
| 34 | MSI_ADDR_BASE_LO | |
| 35 | ((apic->irq_dest_mode == 0) ? |
| 36 | MSI_ADDR_DEST_MODE_PHYSICAL : |
| 37 | MSI_ADDR_DEST_MODE_LOGICAL) | |
Thomas Gleixner | a31e58e | 2017-12-28 11:33:33 +0100 | [diff] [blame] | 38 | MSI_ADDR_REDIRECTION_CPU | |
Jiang Liu | 3cb96f0 | 2015-04-13 14:11:34 +0800 | [diff] [blame] | 39 | MSI_ADDR_DEST_ID(cfg->dest_apicid); |
| 40 | |
| 41 | msg->data = |
| 42 | MSI_DATA_TRIGGER_EDGE | |
| 43 | MSI_DATA_LEVEL_ASSERT | |
Thomas Gleixner | a31e58e | 2017-12-28 11:33:33 +0100 | [diff] [blame] | 44 | MSI_DATA_DELIVERY_FIXED | |
Jiang Liu | 3cb96f0 | 2015-04-13 14:11:34 +0800 | [diff] [blame] | 45 | MSI_DATA_VECTOR(cfg->vector); |
| 46 | } |
| 47 | |
Thomas Gleixner | 6f1a489 | 2020-01-31 15:26:52 +0100 | [diff] [blame^] | 48 | static void irq_msi_compose_msg(struct irq_data *data, struct msi_msg *msg) |
| 49 | { |
| 50 | __irq_msi_compose_msg(irqd_cfg(data), msg); |
| 51 | } |
| 52 | |
| 53 | static void irq_msi_update_msg(struct irq_data *irqd, struct irq_cfg *cfg) |
| 54 | { |
| 55 | struct msi_msg msg[2] = { [1] = { }, }; |
| 56 | |
| 57 | __irq_msi_compose_msg(cfg, msg); |
| 58 | irq_data_get_irq_chip(irqd)->irq_write_msi_msg(irqd, msg); |
| 59 | } |
| 60 | |
| 61 | static int |
| 62 | msi_set_affinity(struct irq_data *irqd, const struct cpumask *mask, bool force) |
| 63 | { |
| 64 | struct irq_cfg old_cfg, *cfg = irqd_cfg(irqd); |
| 65 | struct irq_data *parent = irqd->parent_data; |
| 66 | unsigned int cpu; |
| 67 | int ret; |
| 68 | |
| 69 | /* Save the current configuration */ |
| 70 | cpu = cpumask_first(irq_data_get_effective_affinity_mask(irqd)); |
| 71 | old_cfg = *cfg; |
| 72 | |
| 73 | /* Allocate a new target vector */ |
| 74 | ret = parent->chip->irq_set_affinity(parent, mask, force); |
| 75 | if (ret < 0 || ret == IRQ_SET_MASK_OK_DONE) |
| 76 | return ret; |
| 77 | |
| 78 | /* |
| 79 | * For non-maskable and non-remapped MSI interrupts the migration |
| 80 | * to a different destination CPU and a different vector has to be |
| 81 | * done careful to handle the possible stray interrupt which can be |
| 82 | * caused by the non-atomic update of the address/data pair. |
| 83 | * |
| 84 | * Direct update is possible when: |
| 85 | * - The MSI is maskable (remapped MSI does not use this code path)). |
| 86 | * The quirk bit is not set in this case. |
| 87 | * - The new vector is the same as the old vector |
| 88 | * - The old vector is MANAGED_IRQ_SHUTDOWN_VECTOR (interrupt starts up) |
| 89 | * - The new destination CPU is the same as the old destination CPU |
| 90 | */ |
| 91 | if (!irqd_msi_nomask_quirk(irqd) || |
| 92 | cfg->vector == old_cfg.vector || |
| 93 | old_cfg.vector == MANAGED_IRQ_SHUTDOWN_VECTOR || |
| 94 | cfg->dest_apicid == old_cfg.dest_apicid) { |
| 95 | irq_msi_update_msg(irqd, cfg); |
| 96 | return ret; |
| 97 | } |
| 98 | |
| 99 | /* |
| 100 | * Paranoia: Validate that the interrupt target is the local |
| 101 | * CPU. |
| 102 | */ |
| 103 | if (WARN_ON_ONCE(cpu != smp_processor_id())) { |
| 104 | irq_msi_update_msg(irqd, cfg); |
| 105 | return ret; |
| 106 | } |
| 107 | |
| 108 | /* |
| 109 | * Redirect the interrupt to the new vector on the current CPU |
| 110 | * first. This might cause a spurious interrupt on this vector if |
| 111 | * the device raises an interrupt right between this update and the |
| 112 | * update to the final destination CPU. |
| 113 | * |
| 114 | * If the vector is in use then the installed device handler will |
| 115 | * denote it as spurious which is no harm as this is a rare event |
| 116 | * and interrupt handlers have to cope with spurious interrupts |
| 117 | * anyway. If the vector is unused, then it is marked so it won't |
| 118 | * trigger the 'No irq handler for vector' warning in do_IRQ(). |
| 119 | * |
| 120 | * This requires to hold vector lock to prevent concurrent updates to |
| 121 | * the affected vector. |
| 122 | */ |
| 123 | lock_vector_lock(); |
| 124 | |
| 125 | /* |
| 126 | * Mark the new target vector on the local CPU if it is currently |
| 127 | * unused. Reuse the VECTOR_RETRIGGERED state which is also used in |
| 128 | * the CPU hotplug path for a similar purpose. This cannot be |
| 129 | * undone here as the current CPU has interrupts disabled and |
| 130 | * cannot handle the interrupt before the whole set_affinity() |
| 131 | * section is done. In the CPU unplug case, the current CPU is |
| 132 | * about to vanish and will not handle any interrupts anymore. The |
| 133 | * vector is cleaned up when the CPU comes online again. |
| 134 | */ |
| 135 | if (IS_ERR_OR_NULL(this_cpu_read(vector_irq[cfg->vector]))) |
| 136 | this_cpu_write(vector_irq[cfg->vector], VECTOR_RETRIGGERED); |
| 137 | |
| 138 | /* Redirect it to the new vector on the local CPU temporarily */ |
| 139 | old_cfg.vector = cfg->vector; |
| 140 | irq_msi_update_msg(irqd, &old_cfg); |
| 141 | |
| 142 | /* Now transition it to the target CPU */ |
| 143 | irq_msi_update_msg(irqd, cfg); |
| 144 | |
| 145 | /* |
| 146 | * All interrupts after this point are now targeted at the new |
| 147 | * vector/CPU. |
| 148 | * |
| 149 | * Drop vector lock before testing whether the temporary assignment |
| 150 | * to the local CPU was hit by an interrupt raised in the device, |
| 151 | * because the retrigger function acquires vector lock again. |
| 152 | */ |
| 153 | unlock_vector_lock(); |
| 154 | |
| 155 | /* |
| 156 | * Check whether the transition raced with a device interrupt and |
| 157 | * is pending in the local APICs IRR. It is safe to do this outside |
| 158 | * of vector lock as the irq_desc::lock of this interrupt is still |
| 159 | * held and interrupts are disabled: The check is not accessing the |
| 160 | * underlying vector store. It's just checking the local APIC's |
| 161 | * IRR. |
| 162 | */ |
| 163 | if (lapic_vector_set_in_irr(cfg->vector)) |
| 164 | irq_data_get_irq_chip(irqd)->irq_retrigger(irqd); |
| 165 | |
| 166 | return ret; |
| 167 | } |
| 168 | |
Jiang Liu | 4438098 | 2014-10-27 16:12:02 +0800 | [diff] [blame] | 169 | /* |
| 170 | * IRQ Chip for MSI PCI/PCI-X/PCI-Express Devices, |
| 171 | * which implement the MSI or MSI-X Capability Structure. |
| 172 | */ |
Jiang Liu | 52f518a | 2015-04-13 14:11:35 +0800 | [diff] [blame] | 173 | static struct irq_chip pci_msi_controller = { |
Jiang Liu | 4438098 | 2014-10-27 16:12:02 +0800 | [diff] [blame] | 174 | .name = "PCI-MSI", |
| 175 | .irq_unmask = pci_msi_unmask_irq, |
| 176 | .irq_mask = pci_msi_mask_irq, |
Jiang Liu | 52f518a | 2015-04-13 14:11:35 +0800 | [diff] [blame] | 177 | .irq_ack = irq_chip_ack_parent, |
Jiang Liu | 52f518a | 2015-04-13 14:11:35 +0800 | [diff] [blame] | 178 | .irq_retrigger = irq_chip_retrigger_hierarchy, |
Jiang Liu | 52f518a | 2015-04-13 14:11:35 +0800 | [diff] [blame] | 179 | .irq_compose_msi_msg = irq_msi_compose_msg, |
Thomas Gleixner | 6f1a489 | 2020-01-31 15:26:52 +0100 | [diff] [blame^] | 180 | .irq_set_affinity = msi_set_affinity, |
Jiang Liu | 4438098 | 2014-10-27 16:12:02 +0800 | [diff] [blame] | 181 | .flags = IRQCHIP_SKIP_SET_WAKE, |
| 182 | }; |
| 183 | |
Jiang Liu | 4438098 | 2014-10-27 16:12:02 +0800 | [diff] [blame] | 184 | int native_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) |
| 185 | { |
Jiang Liu | 52f518a | 2015-04-13 14:11:35 +0800 | [diff] [blame] | 186 | struct irq_domain *domain; |
| 187 | struct irq_alloc_info info; |
Jiang Liu | 4438098 | 2014-10-27 16:12:02 +0800 | [diff] [blame] | 188 | |
Jiang Liu | 52f518a | 2015-04-13 14:11:35 +0800 | [diff] [blame] | 189 | init_irq_alloc_info(&info, NULL); |
| 190 | info.type = X86_IRQ_ALLOC_TYPE_MSI; |
| 191 | info.msi_dev = dev; |
Jiang Liu | 4438098 | 2014-10-27 16:12:02 +0800 | [diff] [blame] | 192 | |
Jiang Liu | 52f518a | 2015-04-13 14:11:35 +0800 | [diff] [blame] | 193 | domain = irq_remapping_get_irq_domain(&info); |
| 194 | if (domain == NULL) |
| 195 | domain = msi_default_domain; |
| 196 | if (domain == NULL) |
| 197 | return -ENOSYS; |
Jiang Liu | 4438098 | 2014-10-27 16:12:02 +0800 | [diff] [blame] | 198 | |
Christoph Hellwig | 699c4ce | 2017-02-08 18:17:44 +0100 | [diff] [blame] | 199 | return msi_domain_alloc_irqs(domain, &dev->dev, nvec); |
Jiang Liu | 4438098 | 2014-10-27 16:12:02 +0800 | [diff] [blame] | 200 | } |
| 201 | |
| 202 | void native_teardown_msi_irq(unsigned int irq) |
| 203 | { |
Jiang Liu | 4c8f996 | 2015-04-13 14:11:26 +0800 | [diff] [blame] | 204 | irq_domain_free_irqs(irq, 1); |
Jiang Liu | 4438098 | 2014-10-27 16:12:02 +0800 | [diff] [blame] | 205 | } |
| 206 | |
Jiang Liu | 52f518a | 2015-04-13 14:11:35 +0800 | [diff] [blame] | 207 | static irq_hw_number_t pci_msi_get_hwirq(struct msi_domain_info *info, |
| 208 | msi_alloc_info_t *arg) |
| 209 | { |
| 210 | return arg->msi_hwirq; |
| 211 | } |
| 212 | |
Jake Oshins | c8f3e51 | 2015-12-10 17:52:59 +0000 | [diff] [blame] | 213 | int pci_msi_prepare(struct irq_domain *domain, struct device *dev, int nvec, |
| 214 | msi_alloc_info_t *arg) |
Jiang Liu | 52f518a | 2015-04-13 14:11:35 +0800 | [diff] [blame] | 215 | { |
| 216 | struct pci_dev *pdev = to_pci_dev(dev); |
| 217 | struct msi_desc *desc = first_pci_msi_entry(pdev); |
| 218 | |
| 219 | init_irq_alloc_info(arg, NULL); |
| 220 | arg->msi_dev = pdev; |
| 221 | if (desc->msi_attrib.is_msix) { |
| 222 | arg->type = X86_IRQ_ALLOC_TYPE_MSIX; |
| 223 | } else { |
| 224 | arg->type = X86_IRQ_ALLOC_TYPE_MSI; |
| 225 | arg->flags |= X86_IRQ_ALLOC_CONTIGUOUS_VECTORS; |
| 226 | } |
| 227 | |
| 228 | return 0; |
| 229 | } |
Jake Oshins | c8f3e51 | 2015-12-10 17:52:59 +0000 | [diff] [blame] | 230 | EXPORT_SYMBOL_GPL(pci_msi_prepare); |
Jiang Liu | 52f518a | 2015-04-13 14:11:35 +0800 | [diff] [blame] | 231 | |
Jake Oshins | c8f3e51 | 2015-12-10 17:52:59 +0000 | [diff] [blame] | 232 | void pci_msi_set_desc(msi_alloc_info_t *arg, struct msi_desc *desc) |
Jiang Liu | 52f518a | 2015-04-13 14:11:35 +0800 | [diff] [blame] | 233 | { |
| 234 | arg->msi_hwirq = pci_msi_domain_calc_hwirq(arg->msi_dev, desc); |
| 235 | } |
Jake Oshins | c8f3e51 | 2015-12-10 17:52:59 +0000 | [diff] [blame] | 236 | EXPORT_SYMBOL_GPL(pci_msi_set_desc); |
Jiang Liu | 52f518a | 2015-04-13 14:11:35 +0800 | [diff] [blame] | 237 | |
| 238 | static struct msi_domain_ops pci_msi_domain_ops = { |
| 239 | .get_hwirq = pci_msi_get_hwirq, |
| 240 | .msi_prepare = pci_msi_prepare, |
| 241 | .set_desc = pci_msi_set_desc, |
| 242 | }; |
| 243 | |
| 244 | static struct msi_domain_info pci_msi_domain_info = { |
| 245 | .flags = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | |
Jiang Liu | 68682a2 | 2015-04-13 14:11:46 +0800 | [diff] [blame] | 246 | MSI_FLAG_PCI_MSIX, |
Jiang Liu | 52f518a | 2015-04-13 14:11:35 +0800 | [diff] [blame] | 247 | .ops = &pci_msi_domain_ops, |
| 248 | .chip = &pci_msi_controller, |
| 249 | .handler = handle_edge_irq, |
| 250 | .handler_name = "edge", |
| 251 | }; |
| 252 | |
Thomas Gleixner | f8f37ca | 2017-06-20 01:37:14 +0200 | [diff] [blame] | 253 | void __init arch_init_msi_domain(struct irq_domain *parent) |
Jiang Liu | 52f518a | 2015-04-13 14:11:35 +0800 | [diff] [blame] | 254 | { |
Thomas Gleixner | f8f37ca | 2017-06-20 01:37:14 +0200 | [diff] [blame] | 255 | struct fwnode_handle *fn; |
| 256 | |
Jiang Liu | 52f518a | 2015-04-13 14:11:35 +0800 | [diff] [blame] | 257 | if (disable_apic) |
| 258 | return; |
| 259 | |
Thomas Gleixner | f8f37ca | 2017-06-20 01:37:14 +0200 | [diff] [blame] | 260 | fn = irq_domain_alloc_named_fwnode("PCI-MSI"); |
| 261 | if (fn) { |
| 262 | msi_default_domain = |
| 263 | pci_msi_create_irq_domain(fn, &pci_msi_domain_info, |
| 264 | parent); |
| 265 | irq_domain_free_fwnode(fn); |
| 266 | } |
Jiang Liu | 52f518a | 2015-04-13 14:11:35 +0800 | [diff] [blame] | 267 | if (!msi_default_domain) |
| 268 | pr_warn("failed to initialize irqdomain for MSI/MSI-x.\n"); |
Thomas Gleixner | 6f1a489 | 2020-01-31 15:26:52 +0100 | [diff] [blame^] | 269 | else |
| 270 | msi_default_domain->flags |= IRQ_DOMAIN_MSI_NOMASK_QUIRK; |
Jiang Liu | 52f518a | 2015-04-13 14:11:35 +0800 | [diff] [blame] | 271 | } |
| 272 | |
| 273 | #ifdef CONFIG_IRQ_REMAP |
Jiang Liu | 68682a2 | 2015-04-13 14:11:46 +0800 | [diff] [blame] | 274 | static struct irq_chip pci_msi_ir_controller = { |
| 275 | .name = "IR-PCI-MSI", |
| 276 | .irq_unmask = pci_msi_unmask_irq, |
| 277 | .irq_mask = pci_msi_mask_irq, |
| 278 | .irq_ack = irq_chip_ack_parent, |
Jiang Liu | 68682a2 | 2015-04-13 14:11:46 +0800 | [diff] [blame] | 279 | .irq_retrigger = irq_chip_retrigger_hierarchy, |
Feng Wu | a2f1c8b | 2015-05-19 17:07:15 +0800 | [diff] [blame] | 280 | .irq_set_vcpu_affinity = irq_chip_set_vcpu_affinity_parent, |
Jiang Liu | 68682a2 | 2015-04-13 14:11:46 +0800 | [diff] [blame] | 281 | .flags = IRQCHIP_SKIP_SET_WAKE, |
| 282 | }; |
| 283 | |
| 284 | static struct msi_domain_info pci_msi_ir_domain_info = { |
| 285 | .flags = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | |
| 286 | MSI_FLAG_MULTI_PCI_MSI | MSI_FLAG_PCI_MSIX, |
| 287 | .ops = &pci_msi_domain_ops, |
| 288 | .chip = &pci_msi_ir_controller, |
| 289 | .handler = handle_edge_irq, |
| 290 | .handler_name = "edge", |
| 291 | }; |
| 292 | |
Thomas Gleixner | 667724c | 2017-06-20 01:37:10 +0200 | [diff] [blame] | 293 | struct irq_domain *arch_create_remap_msi_irq_domain(struct irq_domain *parent, |
| 294 | const char *name, int id) |
| 295 | { |
| 296 | struct fwnode_handle *fn; |
| 297 | struct irq_domain *d; |
| 298 | |
| 299 | fn = irq_domain_alloc_named_id_fwnode(name, id); |
| 300 | if (!fn) |
| 301 | return NULL; |
| 302 | d = pci_msi_create_irq_domain(fn, &pci_msi_ir_domain_info, parent); |
| 303 | irq_domain_free_fwnode(fn); |
| 304 | return d; |
| 305 | } |
Jiang Liu | 52f518a | 2015-04-13 14:11:35 +0800 | [diff] [blame] | 306 | #endif |
| 307 | |
Jiang Liu | 4438098 | 2014-10-27 16:12:02 +0800 | [diff] [blame] | 308 | #ifdef CONFIG_DMAR_TABLE |
Jiang Liu | 62ac178 | 2015-04-13 14:11:48 +0800 | [diff] [blame] | 309 | static void dmar_msi_write_msg(struct irq_data *data, struct msi_msg *msg) |
| 310 | { |
| 311 | dmar_msi_write(data->irq, msg); |
| 312 | } |
| 313 | |
Jiang Liu | 0921f1d | 2015-04-13 14:11:42 +0800 | [diff] [blame] | 314 | static struct irq_chip dmar_msi_controller = { |
Jiang Liu | 81dabe2 | 2015-04-13 14:11:45 +0800 | [diff] [blame] | 315 | .name = "DMAR-MSI", |
Jiang Liu | 4438098 | 2014-10-27 16:12:02 +0800 | [diff] [blame] | 316 | .irq_unmask = dmar_msi_unmask, |
| 317 | .irq_mask = dmar_msi_mask, |
Jiang Liu | 0921f1d | 2015-04-13 14:11:42 +0800 | [diff] [blame] | 318 | .irq_ack = irq_chip_ack_parent, |
Jiang Liu | e390d89 | 2015-04-13 14:11:49 +0800 | [diff] [blame] | 319 | .irq_set_affinity = msi_domain_set_affinity, |
Jiang Liu | 0921f1d | 2015-04-13 14:11:42 +0800 | [diff] [blame] | 320 | .irq_retrigger = irq_chip_retrigger_hierarchy, |
| 321 | .irq_compose_msi_msg = irq_msi_compose_msg, |
Jiang Liu | 62ac178 | 2015-04-13 14:11:48 +0800 | [diff] [blame] | 322 | .irq_write_msi_msg = dmar_msi_write_msg, |
Jiang Liu | 4438098 | 2014-10-27 16:12:02 +0800 | [diff] [blame] | 323 | .flags = IRQCHIP_SKIP_SET_WAKE, |
| 324 | }; |
| 325 | |
Jiang Liu | e390d89 | 2015-04-13 14:11:49 +0800 | [diff] [blame] | 326 | static irq_hw_number_t dmar_msi_get_hwirq(struct msi_domain_info *info, |
| 327 | msi_alloc_info_t *arg) |
Jiang Liu | 4438098 | 2014-10-27 16:12:02 +0800 | [diff] [blame] | 328 | { |
Jiang Liu | e390d89 | 2015-04-13 14:11:49 +0800 | [diff] [blame] | 329 | return arg->dmar_id; |
Jiang Liu | 0921f1d | 2015-04-13 14:11:42 +0800 | [diff] [blame] | 330 | } |
| 331 | |
Jiang Liu | e390d89 | 2015-04-13 14:11:49 +0800 | [diff] [blame] | 332 | static int dmar_msi_init(struct irq_domain *domain, |
| 333 | struct msi_domain_info *info, unsigned int virq, |
| 334 | irq_hw_number_t hwirq, msi_alloc_info_t *arg) |
Jiang Liu | 0921f1d | 2015-04-13 14:11:42 +0800 | [diff] [blame] | 335 | { |
Jiang Liu | e390d89 | 2015-04-13 14:11:49 +0800 | [diff] [blame] | 336 | irq_domain_set_info(domain, virq, arg->dmar_id, info->chip, NULL, |
| 337 | handle_edge_irq, arg->dmar_data, "edge"); |
| 338 | |
| 339 | return 0; |
Jiang Liu | 0921f1d | 2015-04-13 14:11:42 +0800 | [diff] [blame] | 340 | } |
| 341 | |
Jiang Liu | e390d89 | 2015-04-13 14:11:49 +0800 | [diff] [blame] | 342 | static struct msi_domain_ops dmar_msi_domain_ops = { |
| 343 | .get_hwirq = dmar_msi_get_hwirq, |
| 344 | .msi_init = dmar_msi_init, |
| 345 | }; |
Jiang Liu | 0921f1d | 2015-04-13 14:11:42 +0800 | [diff] [blame] | 346 | |
Jiang Liu | e390d89 | 2015-04-13 14:11:49 +0800 | [diff] [blame] | 347 | static struct msi_domain_info dmar_msi_domain_info = { |
| 348 | .ops = &dmar_msi_domain_ops, |
| 349 | .chip = &dmar_msi_controller, |
Jiang Liu | 0921f1d | 2015-04-13 14:11:42 +0800 | [diff] [blame] | 350 | }; |
| 351 | |
| 352 | static struct irq_domain *dmar_get_irq_domain(void) |
| 353 | { |
| 354 | static struct irq_domain *dmar_domain; |
| 355 | static DEFINE_MUTEX(dmar_lock); |
Thomas Gleixner | f8f37ca | 2017-06-20 01:37:14 +0200 | [diff] [blame] | 356 | struct fwnode_handle *fn; |
Jiang Liu | 0921f1d | 2015-04-13 14:11:42 +0800 | [diff] [blame] | 357 | |
| 358 | mutex_lock(&dmar_lock); |
Thomas Gleixner | f8f37ca | 2017-06-20 01:37:14 +0200 | [diff] [blame] | 359 | if (dmar_domain) |
| 360 | goto out; |
Jiang Liu | 0921f1d | 2015-04-13 14:11:42 +0800 | [diff] [blame] | 361 | |
Thomas Gleixner | f8f37ca | 2017-06-20 01:37:14 +0200 | [diff] [blame] | 362 | fn = irq_domain_alloc_named_fwnode("DMAR-MSI"); |
| 363 | if (fn) { |
| 364 | dmar_domain = msi_create_irq_domain(fn, &dmar_msi_domain_info, |
| 365 | x86_vector_domain); |
| 366 | irq_domain_free_fwnode(fn); |
| 367 | } |
| 368 | out: |
| 369 | mutex_unlock(&dmar_lock); |
Jiang Liu | 0921f1d | 2015-04-13 14:11:42 +0800 | [diff] [blame] | 370 | return dmar_domain; |
| 371 | } |
| 372 | |
| 373 | int dmar_alloc_hwirq(int id, int node, void *arg) |
| 374 | { |
| 375 | struct irq_domain *domain = dmar_get_irq_domain(); |
| 376 | struct irq_alloc_info info; |
| 377 | |
| 378 | if (!domain) |
| 379 | return -1; |
| 380 | |
| 381 | init_irq_alloc_info(&info, NULL); |
| 382 | info.type = X86_IRQ_ALLOC_TYPE_DMAR; |
| 383 | info.dmar_id = id; |
| 384 | info.dmar_data = arg; |
| 385 | |
| 386 | return irq_domain_alloc_irqs(domain, 1, node, &info); |
Jiang Liu | a62b32c | 2015-04-13 14:11:29 +0800 | [diff] [blame] | 387 | } |
| 388 | |
| 389 | void dmar_free_hwirq(int irq) |
| 390 | { |
| 391 | irq_domain_free_irqs(irq, 1); |
| 392 | } |
Jiang Liu | 4438098 | 2014-10-27 16:12:02 +0800 | [diff] [blame] | 393 | #endif |
| 394 | |
| 395 | /* |
| 396 | * MSI message composition |
| 397 | */ |
| 398 | #ifdef CONFIG_HPET_TIMER |
Jiang Liu | 3cb96f0 | 2015-04-13 14:11:34 +0800 | [diff] [blame] | 399 | static inline int hpet_dev_id(struct irq_domain *domain) |
| 400 | { |
Jiang Liu | e390d89 | 2015-04-13 14:11:49 +0800 | [diff] [blame] | 401 | struct msi_domain_info *info = msi_get_domain_info(domain); |
Jiang Liu | 4438098 | 2014-10-27 16:12:02 +0800 | [diff] [blame] | 402 | |
Jiang Liu | e390d89 | 2015-04-13 14:11:49 +0800 | [diff] [blame] | 403 | return (int)(long)info->data; |
Jiang Liu | 4438098 | 2014-10-27 16:12:02 +0800 | [diff] [blame] | 404 | } |
| 405 | |
Jiang Liu | 62ac178 | 2015-04-13 14:11:48 +0800 | [diff] [blame] | 406 | static void hpet_msi_write_msg(struct irq_data *data, struct msi_msg *msg) |
| 407 | { |
Jiang Liu | ff96b4d | 2015-06-01 16:05:18 +0800 | [diff] [blame] | 408 | hpet_msi_write(irq_data_get_irq_handler_data(data), msg); |
Jiang Liu | 62ac178 | 2015-04-13 14:11:48 +0800 | [diff] [blame] | 409 | } |
| 410 | |
Kees Cook | 404f6aa | 2016-08-08 16:29:06 -0700 | [diff] [blame] | 411 | static struct irq_chip hpet_msi_controller __ro_after_init = { |
Jiang Liu | 81dabe2 | 2015-04-13 14:11:45 +0800 | [diff] [blame] | 412 | .name = "HPET-MSI", |
Jiang Liu | 4438098 | 2014-10-27 16:12:02 +0800 | [diff] [blame] | 413 | .irq_unmask = hpet_msi_unmask, |
| 414 | .irq_mask = hpet_msi_mask, |
Jiang Liu | 3cb96f0 | 2015-04-13 14:11:34 +0800 | [diff] [blame] | 415 | .irq_ack = irq_chip_ack_parent, |
Jiang Liu | e390d89 | 2015-04-13 14:11:49 +0800 | [diff] [blame] | 416 | .irq_set_affinity = msi_domain_set_affinity, |
Jiang Liu | 3cb96f0 | 2015-04-13 14:11:34 +0800 | [diff] [blame] | 417 | .irq_retrigger = irq_chip_retrigger_hierarchy, |
Jiang Liu | 3cb96f0 | 2015-04-13 14:11:34 +0800 | [diff] [blame] | 418 | .irq_compose_msi_msg = irq_msi_compose_msg, |
Jiang Liu | 62ac178 | 2015-04-13 14:11:48 +0800 | [diff] [blame] | 419 | .irq_write_msi_msg = hpet_msi_write_msg, |
Jiang Liu | 4438098 | 2014-10-27 16:12:02 +0800 | [diff] [blame] | 420 | .flags = IRQCHIP_SKIP_SET_WAKE, |
| 421 | }; |
| 422 | |
Jiang Liu | e390d89 | 2015-04-13 14:11:49 +0800 | [diff] [blame] | 423 | static irq_hw_number_t hpet_msi_get_hwirq(struct msi_domain_info *info, |
| 424 | msi_alloc_info_t *arg) |
Jiang Liu | 3cb96f0 | 2015-04-13 14:11:34 +0800 | [diff] [blame] | 425 | { |
Jiang Liu | e390d89 | 2015-04-13 14:11:49 +0800 | [diff] [blame] | 426 | return arg->hpet_index; |
Jiang Liu | 3cb96f0 | 2015-04-13 14:11:34 +0800 | [diff] [blame] | 427 | } |
| 428 | |
Jiang Liu | e390d89 | 2015-04-13 14:11:49 +0800 | [diff] [blame] | 429 | static int hpet_msi_init(struct irq_domain *domain, |
| 430 | struct msi_domain_info *info, unsigned int virq, |
| 431 | irq_hw_number_t hwirq, msi_alloc_info_t *arg) |
Jiang Liu | 3cb96f0 | 2015-04-13 14:11:34 +0800 | [diff] [blame] | 432 | { |
Jiang Liu | e390d89 | 2015-04-13 14:11:49 +0800 | [diff] [blame] | 433 | irq_set_status_flags(virq, IRQ_MOVE_PCNTXT); |
| 434 | irq_domain_set_info(domain, virq, arg->hpet_index, info->chip, NULL, |
| 435 | handle_edge_irq, arg->hpet_data, "edge"); |
| 436 | |
| 437 | return 0; |
| 438 | } |
| 439 | |
| 440 | static void hpet_msi_free(struct irq_domain *domain, |
| 441 | struct msi_domain_info *info, unsigned int virq) |
| 442 | { |
Jiang Liu | 3cb96f0 | 2015-04-13 14:11:34 +0800 | [diff] [blame] | 443 | irq_clear_status_flags(virq, IRQ_MOVE_PCNTXT); |
Jiang Liu | 3cb96f0 | 2015-04-13 14:11:34 +0800 | [diff] [blame] | 444 | } |
| 445 | |
Jiang Liu | e390d89 | 2015-04-13 14:11:49 +0800 | [diff] [blame] | 446 | static struct msi_domain_ops hpet_msi_domain_ops = { |
| 447 | .get_hwirq = hpet_msi_get_hwirq, |
| 448 | .msi_init = hpet_msi_init, |
| 449 | .msi_free = hpet_msi_free, |
| 450 | }; |
Jiang Liu | 3cb96f0 | 2015-04-13 14:11:34 +0800 | [diff] [blame] | 451 | |
Jiang Liu | e390d89 | 2015-04-13 14:11:49 +0800 | [diff] [blame] | 452 | static struct msi_domain_info hpet_msi_domain_info = { |
| 453 | .ops = &hpet_msi_domain_ops, |
| 454 | .chip = &hpet_msi_controller, |
Jiang Liu | 3cb96f0 | 2015-04-13 14:11:34 +0800 | [diff] [blame] | 455 | }; |
| 456 | |
| 457 | struct irq_domain *hpet_create_irq_domain(int hpet_id) |
| 458 | { |
Jiang Liu | e390d89 | 2015-04-13 14:11:49 +0800 | [diff] [blame] | 459 | struct msi_domain_info *domain_info; |
Thomas Gleixner | f8f37ca | 2017-06-20 01:37:14 +0200 | [diff] [blame] | 460 | struct irq_domain *parent, *d; |
| 461 | struct irq_alloc_info info; |
| 462 | struct fwnode_handle *fn; |
Jiang Liu | 3cb96f0 | 2015-04-13 14:11:34 +0800 | [diff] [blame] | 463 | |
| 464 | if (x86_vector_domain == NULL) |
| 465 | return NULL; |
| 466 | |
Jiang Liu | e390d89 | 2015-04-13 14:11:49 +0800 | [diff] [blame] | 467 | domain_info = kzalloc(sizeof(*domain_info), GFP_KERNEL); |
| 468 | if (!domain_info) |
| 469 | return NULL; |
| 470 | |
| 471 | *domain_info = hpet_msi_domain_info; |
| 472 | domain_info->data = (void *)(long)hpet_id; |
| 473 | |
Jiang Liu | 3cb96f0 | 2015-04-13 14:11:34 +0800 | [diff] [blame] | 474 | init_irq_alloc_info(&info, NULL); |
| 475 | info.type = X86_IRQ_ALLOC_TYPE_HPET; |
| 476 | info.hpet_id = hpet_id; |
| 477 | parent = irq_remapping_get_ir_irq_domain(&info); |
| 478 | if (parent == NULL) |
| 479 | parent = x86_vector_domain; |
Jiang Liu | 68682a2 | 2015-04-13 14:11:46 +0800 | [diff] [blame] | 480 | else |
| 481 | hpet_msi_controller.name = "IR-HPET-MSI"; |
Jiang Liu | 3cb96f0 | 2015-04-13 14:11:34 +0800 | [diff] [blame] | 482 | |
Thomas Gleixner | f8f37ca | 2017-06-20 01:37:14 +0200 | [diff] [blame] | 483 | fn = irq_domain_alloc_named_id_fwnode(hpet_msi_controller.name, |
| 484 | hpet_id); |
| 485 | if (!fn) { |
| 486 | kfree(domain_info); |
| 487 | return NULL; |
| 488 | } |
| 489 | |
| 490 | d = msi_create_irq_domain(fn, domain_info, parent); |
| 491 | irq_domain_free_fwnode(fn); |
| 492 | return d; |
Jiang Liu | 3cb96f0 | 2015-04-13 14:11:34 +0800 | [diff] [blame] | 493 | } |
| 494 | |
Thomas Gleixner | 4d5e683 | 2019-06-23 15:24:03 +0200 | [diff] [blame] | 495 | int hpet_assign_irq(struct irq_domain *domain, struct hpet_channel *hc, |
Jiang Liu | 3cb96f0 | 2015-04-13 14:11:34 +0800 | [diff] [blame] | 496 | int dev_num) |
| 497 | { |
| 498 | struct irq_alloc_info info; |
| 499 | |
| 500 | init_irq_alloc_info(&info, NULL); |
| 501 | info.type = X86_IRQ_ALLOC_TYPE_HPET; |
Thomas Gleixner | 4d5e683 | 2019-06-23 15:24:03 +0200 | [diff] [blame] | 502 | info.hpet_data = hc; |
Jiang Liu | 3cb96f0 | 2015-04-13 14:11:34 +0800 | [diff] [blame] | 503 | info.hpet_id = hpet_dev_id(domain); |
| 504 | info.hpet_index = dev_num; |
| 505 | |
Sergey Senozhatsky | 4a00c95 | 2015-05-11 18:56:49 +0900 | [diff] [blame] | 506 | return irq_domain_alloc_irqs(domain, 1, NUMA_NO_NODE, &info); |
Jiang Liu | 3cb96f0 | 2015-04-13 14:11:34 +0800 | [diff] [blame] | 507 | } |
Jiang Liu | 4438098 | 2014-10-27 16:12:02 +0800 | [diff] [blame] | 508 | #endif |