blob: d17aa41a9041428fbff17a890b2af18b64422ef4 [file] [log] [blame]
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001/*
2 * Xen event channels
3 *
4 * Xen models interrupts with abstract event channels. Because each
5 * domain gets 1024 event channels, but NR_IRQ is not that large, we
6 * must dynamically map irqs<->event channels. The event channels
7 * interface with the rest of the kernel by defining a xen interrupt
Lucas De Marchi25985ed2011-03-30 22:57:33 -03008 * chip. When an event is received, it is mapped to an irq and sent
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07009 * through the normal interrupt processing path.
10 *
11 * There are four kinds of events which can be mapped to an event
12 * channel:
13 *
14 * 1. Inter-domain notifications. This includes all the virtual
15 * device events, since they're driven by front-ends in another domain
16 * (typically dom0).
17 * 2. VIRQs, typically used for timers. These are per-cpu events.
18 * 3. IPIs.
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -040019 * 4. PIRQs - Hardware interrupts.
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070020 *
21 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
22 */
23
24#include <linux/linkage.h>
25#include <linux/interrupt.h>
26#include <linux/irq.h>
27#include <linux/module.h>
28#include <linux/string.h>
Christophe Saout28e08862009-01-11 11:46:23 -080029#include <linux/bootmem.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090030#include <linux/slab.h>
Jeremy Fitzhardingeb21ddbf2010-06-07 16:28:49 -040031#include <linux/irqnr.h>
Qing Hef731e3ef2010-10-11 15:30:09 +010032#include <linux/pci.h>
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070033
Stefano Stabellini0ec53ec2012-09-14 13:37:32 +000034#ifdef CONFIG_X86
Sheng Yang38e20b02010-05-14 12:40:51 +010035#include <asm/desc.h>
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070036#include <asm/ptrace.h>
37#include <asm/irq.h>
Jeremy Fitzhardinge792dc4f2009-02-06 14:09:43 -080038#include <asm/idle.h>
Konrad Rzeszutek Wilk0794bfc2010-10-18 10:41:08 -040039#include <asm/io_apic.h>
Stefano Stabellini9846ff12012-01-30 16:21:48 +000040#include <asm/xen/page.h>
Stefano Stabellini42a1de52010-06-24 16:42:04 +010041#include <asm/xen/pci.h>
Stefano Stabellini0ec53ec2012-09-14 13:37:32 +000042#endif
43#include <asm/sync_bitops.h>
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070044#include <asm/xen/hypercall.h>
Adrian Bunk8d1b8752007-07-20 00:31:44 -070045#include <asm/xen/hypervisor.h>
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070046
Sheng Yang38e20b02010-05-14 12:40:51 +010047#include <xen/xen.h>
48#include <xen/hvm.h>
Isaku Yamahatae04d0d02008-04-02 10:53:55 -070049#include <xen/xen-ops.h>
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070050#include <xen/events.h>
51#include <xen/interface/xen.h>
52#include <xen/interface/event_channel.h>
Sheng Yang38e20b02010-05-14 12:40:51 +010053#include <xen/interface/hvm/hvm_op.h>
54#include <xen/interface/hvm/params.h>
Stefano Stabellini0ec53ec2012-09-14 13:37:32 +000055#include <xen/interface/physdev.h>
56#include <xen/interface/sched.h>
57#include <asm/hw_irq.h>
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070058
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070059/*
60 * This lock protects updates to the following mapping and reference-count
61 * arrays. The lock does not need to be acquired to read the mapping tables.
62 */
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -040063static DEFINE_MUTEX(irq_mapping_update_lock);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070064
Ian Campbell6cb65372011-03-10 16:08:11 +000065static LIST_HEAD(xen_irq_list_head);
66
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070067/* IRQ <-> VIRQ mapping. */
Tejun Heo204fba42009-06-24 15:13:45 +090068static DEFINE_PER_CPU(int [NR_VIRQS], virq_to_irq) = {[0 ... NR_VIRQS-1] = -1};
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070069
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -070070/* IRQ <-> IPI mapping */
Tejun Heo204fba42009-06-24 15:13:45 +090071static DEFINE_PER_CPU(int [XEN_NR_IPIS], ipi_to_irq) = {[0 ... XEN_NR_IPIS-1] = -1};
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -070072
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -080073/* Interrupt types. */
74enum xen_irq_type {
Jeremy Fitzhardinged77bbd42009-02-06 14:09:45 -080075 IRQT_UNBOUND = 0,
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -070076 IRQT_PIRQ,
77 IRQT_VIRQ,
78 IRQT_IPI,
79 IRQT_EVTCHN
80};
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070081
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -080082/*
83 * Packed IRQ information:
84 * type - enum xen_irq_type
85 * event channel - irq->event channel mapping
86 * cpu - cpu this event channel is bound to
87 * index - type-specific information:
Stefano Stabellini42a1de52010-06-24 16:42:04 +010088 * PIRQ - vector, with MSB being "needs EIO", or physical IRQ of the HVM
89 * guest, or GSI (real passthrough IRQ) of the device.
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -080090 * VIRQ - virq number
91 * IPI - IPI vector
92 * EVTCHN -
93 */
Ruslan Pisarev088c05a2011-07-26 14:16:13 +030094struct irq_info {
Ian Campbell6cb65372011-03-10 16:08:11 +000095 struct list_head list;
Daniel De Graaf420eb552011-10-27 17:58:47 -040096 int refcnt;
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -080097 enum xen_irq_type type; /* type */
Ian Campbell6cb65372011-03-10 16:08:11 +000098 unsigned irq;
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -080099 unsigned short evtchn; /* event channel */
100 unsigned short cpu; /* cpu bound */
101
102 union {
103 unsigned short virq;
104 enum ipi_vector ipi;
105 struct {
Stefano Stabellini7a043f12010-07-01 17:08:14 +0100106 unsigned short pirq;
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800107 unsigned short gsi;
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400108 unsigned char vector;
109 unsigned char flags;
Konrad Rzeszutek Wilkbeafbdc2011-04-14 11:17:36 -0400110 uint16_t domid;
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800111 } pirq;
112 } u;
113};
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400114#define PIRQ_NEEDS_EOI (1 << 0)
Konrad Rzeszutek Wilk15ebbb82010-10-04 13:43:27 -0400115#define PIRQ_SHAREABLE (1 << 1)
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800116
Jeremy Fitzhardingeb21ddbf2010-06-07 16:28:49 -0400117static int *evtchn_to_irq;
Ian Campbellbf86ad82012-10-17 09:39:12 +0100118#ifdef CONFIG_X86
Stefano Stabellini9846ff12012-01-30 16:21:48 +0000119static unsigned long *pirq_eoi_map;
Ian Campbellbf86ad82012-10-17 09:39:12 +0100120#endif
Stefano Stabellini9846ff12012-01-30 16:21:48 +0000121static bool (*pirq_needs_eoi)(unsigned irq);
Jeremy Fitzhardinge3b32f572009-08-13 12:50:37 -0700122
Ian Campbellc81611c2013-02-20 11:48:06 +0000123/*
124 * Note sizeof(xen_ulong_t) can be more than sizeof(unsigned long). Be
125 * careful to only use bitops which allow for this (e.g
126 * test_bit/find_first_bit and friends but not __ffs) and to pass
127 * BITS_PER_EVTCHN_WORD as the bitmask length.
128 */
129#define BITS_PER_EVTCHN_WORD (sizeof(xen_ulong_t)*8)
130/*
131 * Make a bitmask (i.e. unsigned long *) of a xen_ulong_t
132 * array. Primarily to avoid long lines (hence the terse name).
133 */
134#define BM(x) (unsigned long *)(x)
135/* Find the first set bit in a evtchn mask */
136#define EVTCHN_FIRST_BIT(w) find_first_bit(BM(&(w)), BITS_PER_EVTCHN_WORD)
137
138static DEFINE_PER_CPU(xen_ulong_t [NR_EVENT_CHANNELS/BITS_PER_EVTCHN_WORD],
Ian Campbellcb60d112011-03-10 16:08:08 +0000139 cpu_evtchn_mask);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700140
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700141/* Xen will never allocate port zero for any purpose. */
142#define VALID_EVTCHN(chn) ((chn) != 0)
143
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700144static struct irq_chip xen_dynamic_chip;
Jeremy Fitzhardingeaaca4962010-08-20 18:57:53 -0700145static struct irq_chip xen_percpu_chip;
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400146static struct irq_chip xen_pirq_chip;
Stefano Stabellini7e186bd2011-05-06 12:27:50 +0100147static void enable_dynirq(struct irq_data *data);
148static void disable_dynirq(struct irq_data *data);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700149
Ian Campbell9158c352011-03-10 16:08:09 +0000150/* Get info for IRQ */
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800151static struct irq_info *info_for_irq(unsigned irq)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700152{
Thomas Gleixnerc442b802011-03-25 10:58:06 +0100153 return irq_get_handler_data(irq);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700154}
155
Ian Campbell9158c352011-03-10 16:08:09 +0000156/* Constructors for packed IRQ information. */
157static void xen_irq_info_common_init(struct irq_info *info,
Ian Campbell3d4cfa32011-03-10 16:08:10 +0000158 unsigned irq,
Ian Campbell9158c352011-03-10 16:08:09 +0000159 enum xen_irq_type type,
160 unsigned short evtchn,
161 unsigned short cpu)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700162{
Ian Campbell9158c352011-03-10 16:08:09 +0000163
164 BUG_ON(info->type != IRQT_UNBOUND && info->type != type);
165
166 info->type = type;
Ian Campbell6cb65372011-03-10 16:08:11 +0000167 info->irq = irq;
Ian Campbell9158c352011-03-10 16:08:09 +0000168 info->evtchn = evtchn;
169 info->cpu = cpu;
Ian Campbell3d4cfa32011-03-10 16:08:10 +0000170
171 evtchn_to_irq[evtchn] = irq;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700172}
173
Ian Campbell9158c352011-03-10 16:08:09 +0000174static void xen_irq_info_evtchn_init(unsigned irq,
175 unsigned short evtchn)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700176{
Ian Campbell9158c352011-03-10 16:08:09 +0000177 struct irq_info *info = info_for_irq(irq);
178
Ian Campbell3d4cfa32011-03-10 16:08:10 +0000179 xen_irq_info_common_init(info, irq, IRQT_EVTCHN, evtchn, 0);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700180}
181
Ian Campbell3d4cfa32011-03-10 16:08:10 +0000182static void xen_irq_info_ipi_init(unsigned cpu,
183 unsigned irq,
Ian Campbell9158c352011-03-10 16:08:09 +0000184 unsigned short evtchn,
185 enum ipi_vector ipi)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700186{
Ian Campbell9158c352011-03-10 16:08:09 +0000187 struct irq_info *info = info_for_irq(irq);
188
Ian Campbell3d4cfa32011-03-10 16:08:10 +0000189 xen_irq_info_common_init(info, irq, IRQT_IPI, evtchn, 0);
Ian Campbell9158c352011-03-10 16:08:09 +0000190
191 info->u.ipi = ipi;
Ian Campbell3d4cfa32011-03-10 16:08:10 +0000192
193 per_cpu(ipi_to_irq, cpu)[ipi] = irq;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700194}
195
Ian Campbell3d4cfa32011-03-10 16:08:10 +0000196static void xen_irq_info_virq_init(unsigned cpu,
197 unsigned irq,
Ian Campbell9158c352011-03-10 16:08:09 +0000198 unsigned short evtchn,
199 unsigned short virq)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700200{
Ian Campbell9158c352011-03-10 16:08:09 +0000201 struct irq_info *info = info_for_irq(irq);
202
Ian Campbell3d4cfa32011-03-10 16:08:10 +0000203 xen_irq_info_common_init(info, irq, IRQT_VIRQ, evtchn, 0);
Ian Campbell9158c352011-03-10 16:08:09 +0000204
205 info->u.virq = virq;
Ian Campbell3d4cfa32011-03-10 16:08:10 +0000206
207 per_cpu(virq_to_irq, cpu)[virq] = irq;
Ian Campbell9158c352011-03-10 16:08:09 +0000208}
209
210static void xen_irq_info_pirq_init(unsigned irq,
211 unsigned short evtchn,
212 unsigned short pirq,
213 unsigned short gsi,
214 unsigned short vector,
Konrad Rzeszutek Wilkbeafbdc2011-04-14 11:17:36 -0400215 uint16_t domid,
Ian Campbell9158c352011-03-10 16:08:09 +0000216 unsigned char flags)
217{
218 struct irq_info *info = info_for_irq(irq);
219
Ian Campbell3d4cfa32011-03-10 16:08:10 +0000220 xen_irq_info_common_init(info, irq, IRQT_PIRQ, evtchn, 0);
Ian Campbell9158c352011-03-10 16:08:09 +0000221
222 info->u.pirq.pirq = pirq;
223 info->u.pirq.gsi = gsi;
224 info->u.pirq.vector = vector;
Konrad Rzeszutek Wilkbeafbdc2011-04-14 11:17:36 -0400225 info->u.pirq.domid = domid;
Ian Campbell9158c352011-03-10 16:08:09 +0000226 info->u.pirq.flags = flags;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700227}
228
229/*
230 * Accessors for packed IRQ information.
231 */
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800232static unsigned int evtchn_from_irq(unsigned irq)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700233{
Joe Jin110e7c72011-01-07 14:50:12 +0800234 if (unlikely(WARN(irq < 0 || irq >= nr_irqs, "Invalid irq %d!\n", irq)))
235 return 0;
236
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800237 return info_for_irq(irq)->evtchn;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700238}
239
Ian Campbelld4c04532009-02-06 19:20:31 -0800240unsigned irq_from_evtchn(unsigned int evtchn)
241{
242 return evtchn_to_irq[evtchn];
243}
244EXPORT_SYMBOL_GPL(irq_from_evtchn);
245
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800246static enum ipi_vector ipi_from_irq(unsigned irq)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700247{
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800248 struct irq_info *info = info_for_irq(irq);
249
250 BUG_ON(info == NULL);
251 BUG_ON(info->type != IRQT_IPI);
252
253 return info->u.ipi;
254}
255
256static unsigned virq_from_irq(unsigned irq)
257{
258 struct irq_info *info = info_for_irq(irq);
259
260 BUG_ON(info == NULL);
261 BUG_ON(info->type != IRQT_VIRQ);
262
263 return info->u.virq;
264}
265
Stefano Stabellini7a043f12010-07-01 17:08:14 +0100266static unsigned pirq_from_irq(unsigned irq)
267{
268 struct irq_info *info = info_for_irq(irq);
269
270 BUG_ON(info == NULL);
271 BUG_ON(info->type != IRQT_PIRQ);
272
273 return info->u.pirq.pirq;
274}
275
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800276static enum xen_irq_type type_from_irq(unsigned irq)
277{
278 return info_for_irq(irq)->type;
279}
280
281static unsigned cpu_from_irq(unsigned irq)
282{
283 return info_for_irq(irq)->cpu;
284}
285
286static unsigned int cpu_from_evtchn(unsigned int evtchn)
287{
288 int irq = evtchn_to_irq[evtchn];
289 unsigned ret = 0;
290
291 if (irq != -1)
292 ret = cpu_from_irq(irq);
293
294 return ret;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700295}
296
Ian Campbellbf86ad82012-10-17 09:39:12 +0100297#ifdef CONFIG_X86
Stefano Stabellini9846ff12012-01-30 16:21:48 +0000298static bool pirq_check_eoi_map(unsigned irq)
299{
Stefano Stabellini521394e2012-04-25 16:11:38 +0100300 return test_bit(pirq_from_irq(irq), pirq_eoi_map);
Stefano Stabellini9846ff12012-01-30 16:21:48 +0000301}
Ian Campbellbf86ad82012-10-17 09:39:12 +0100302#endif
Stefano Stabellini9846ff12012-01-30 16:21:48 +0000303
304static bool pirq_needs_eoi_flag(unsigned irq)
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400305{
306 struct irq_info *info = info_for_irq(irq);
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400307 BUG_ON(info->type != IRQT_PIRQ);
308
309 return info->u.pirq.flags & PIRQ_NEEDS_EOI;
310}
311
Ian Campbellc81611c2013-02-20 11:48:06 +0000312static inline xen_ulong_t active_evtchns(unsigned int cpu,
313 struct shared_info *sh,
314 unsigned int idx)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700315{
Ruslan Pisarev088c05a2011-07-26 14:16:13 +0300316 return sh->evtchn_pending[idx] &
Ian Campbellcb60d112011-03-10 16:08:08 +0000317 per_cpu(cpu_evtchn_mask, cpu)[idx] &
Ruslan Pisarev088c05a2011-07-26 14:16:13 +0300318 ~sh->evtchn_mask[idx];
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700319}
320
321static void bind_evtchn_to_cpu(unsigned int chn, unsigned int cpu)
322{
323 int irq = evtchn_to_irq[chn];
324
325 BUG_ON(irq == -1);
326#ifdef CONFIG_SMP
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +0000327 cpumask_copy(irq_to_desc(irq)->irq_data.affinity, cpumask_of(cpu));
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700328#endif
329
Ian Campbellc81611c2013-02-20 11:48:06 +0000330 clear_bit(chn, BM(per_cpu(cpu_evtchn_mask, cpu_from_irq(irq))));
331 set_bit(chn, BM(per_cpu(cpu_evtchn_mask, cpu)));
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700332
Ian Campbellca62ce82011-03-10 16:08:12 +0000333 info_for_irq(irq)->cpu = cpu;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700334}
335
336static void init_evtchn_cpu_bindings(void)
337{
Jan Beulich1c6969e2010-11-16 14:55:33 -0800338 int i;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700339#ifdef CONFIG_SMP
Ian Campbell6cb65372011-03-10 16:08:11 +0000340 struct irq_info *info;
Thomas Gleixner10e58082008-10-16 14:19:04 +0200341
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700342 /* By default all event channels notify CPU#0. */
Ian Campbell6cb65372011-03-10 16:08:11 +0000343 list_for_each_entry(info, &xen_irq_list_head, list) {
344 struct irq_desc *desc = irq_to_desc(info->irq);
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +0000345 cpumask_copy(desc->irq_data.affinity, cpumask_of(0));
Yinghai Lu0b8f1ef2008-12-05 18:58:31 -0800346 }
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700347#endif
348
Jan Beulich1c6969e2010-11-16 14:55:33 -0800349 for_each_possible_cpu(i)
Ian Campbellcb60d112011-03-10 16:08:08 +0000350 memset(per_cpu(cpu_evtchn_mask, i),
351 (i == 0) ? ~0 : 0, sizeof(*per_cpu(cpu_evtchn_mask, i)));
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700352}
353
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700354static inline void clear_evtchn(int port)
355{
356 struct shared_info *s = HYPERVISOR_shared_info;
Ian Campbellc81611c2013-02-20 11:48:06 +0000357 sync_clear_bit(port, BM(&s->evtchn_pending[0]));
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700358}
359
360static inline void set_evtchn(int port)
361{
362 struct shared_info *s = HYPERVISOR_shared_info;
Ian Campbellc81611c2013-02-20 11:48:06 +0000363 sync_set_bit(port, BM(&s->evtchn_pending[0]));
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700364}
365
Jeremy Fitzhardinge168d2f42008-08-20 17:02:18 -0700366static inline int test_evtchn(int port)
367{
368 struct shared_info *s = HYPERVISOR_shared_info;
Ian Campbellc81611c2013-02-20 11:48:06 +0000369 return sync_test_bit(port, BM(&s->evtchn_pending[0]));
Jeremy Fitzhardinge168d2f42008-08-20 17:02:18 -0700370}
371
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700372
373/**
374 * notify_remote_via_irq - send event to remote end of event channel via irq
375 * @irq: irq of event channel to send event to
376 *
377 * Unlike notify_remote_via_evtchn(), this is safe to use across
378 * save/restore. Notifications on a broken connection are silently
379 * dropped.
380 */
381void notify_remote_via_irq(int irq)
382{
383 int evtchn = evtchn_from_irq(irq);
384
385 if (VALID_EVTCHN(evtchn))
386 notify_remote_via_evtchn(evtchn);
387}
388EXPORT_SYMBOL_GPL(notify_remote_via_irq);
389
390static void mask_evtchn(int port)
391{
392 struct shared_info *s = HYPERVISOR_shared_info;
Ian Campbellc81611c2013-02-20 11:48:06 +0000393 sync_set_bit(port, BM(&s->evtchn_mask[0]));
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700394}
395
396static void unmask_evtchn(int port)
397{
398 struct shared_info *s = HYPERVISOR_shared_info;
399 unsigned int cpu = get_cpu();
Stefano Stabellinib5e57922012-08-22 17:20:11 +0100400 int do_hypercall = 0, evtchn_pending = 0;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700401
402 BUG_ON(!irqs_disabled());
403
Stefano Stabellinib5e57922012-08-22 17:20:11 +0100404 if (unlikely((cpu != cpu_from_evtchn(port))))
405 do_hypercall = 1;
406 else
Ian Campbellc81611c2013-02-20 11:48:06 +0000407 evtchn_pending = sync_test_bit(port, BM(&s->evtchn_pending[0]));
Stefano Stabellinib5e57922012-08-22 17:20:11 +0100408
409 if (unlikely(evtchn_pending && xen_hvm_domain()))
410 do_hypercall = 1;
411
412 /* Slow path (hypercall) if this is a non-local port or if this is
413 * an hvm domain and an event is pending (hvm domains don't have
414 * their own implementation of irq_enable). */
415 if (do_hypercall) {
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700416 struct evtchn_unmask unmask = { .port = port };
417 (void)HYPERVISOR_event_channel_op(EVTCHNOP_unmask, &unmask);
418 } else {
Christoph Lameter780f36d2010-12-06 11:16:29 -0600419 struct vcpu_info *vcpu_info = __this_cpu_read(xen_vcpu);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700420
Ian Campbellc81611c2013-02-20 11:48:06 +0000421 sync_clear_bit(port, BM(&s->evtchn_mask[0]));
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700422
423 /*
424 * The following is basically the equivalent of
425 * 'hw_resend_irq'. Just like a real IO-APIC we 'lose
426 * the interrupt edge' if the channel is masked.
427 */
Stefano Stabellinib5e57922012-08-22 17:20:11 +0100428 if (evtchn_pending &&
Ian Campbellc81611c2013-02-20 11:48:06 +0000429 !sync_test_and_set_bit(port / BITS_PER_EVTCHN_WORD,
430 BM(&vcpu_info->evtchn_pending_sel)))
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700431 vcpu_info->evtchn_upcall_pending = 1;
432 }
433
434 put_cpu();
435}
436
Ian Campbell6cb65372011-03-10 16:08:11 +0000437static void xen_irq_init(unsigned irq)
438{
439 struct irq_info *info;
Konrad Rzeszutek Wilkb5328cd2011-06-15 14:24:29 -0400440#ifdef CONFIG_SMP
Ian Campbell6cb65372011-03-10 16:08:11 +0000441 struct irq_desc *desc = irq_to_desc(irq);
442
443 /* By default all event channels notify CPU#0. */
444 cpumask_copy(desc->irq_data.affinity, cpumask_of(0));
Konrad Rzeszutek Wilk44626e42011-03-15 16:40:33 -0400445#endif
Ian Campbell6cb65372011-03-10 16:08:11 +0000446
Ian Campbellca62ce82011-03-10 16:08:12 +0000447 info = kzalloc(sizeof(*info), GFP_KERNEL);
448 if (info == NULL)
449 panic("Unable to allocate metadata for IRQ%d\n", irq);
Ian Campbell6cb65372011-03-10 16:08:11 +0000450
451 info->type = IRQT_UNBOUND;
Daniel De Graaf420eb552011-10-27 17:58:47 -0400452 info->refcnt = -1;
Ian Campbell6cb65372011-03-10 16:08:11 +0000453
Thomas Gleixnerc442b802011-03-25 10:58:06 +0100454 irq_set_handler_data(irq, info);
Ian Campbellca62ce82011-03-10 16:08:12 +0000455
Ian Campbell6cb65372011-03-10 16:08:11 +0000456 list_add_tail(&info->list, &xen_irq_list_head);
457}
458
Ian Campbell7bee9762011-03-10 16:08:15 +0000459static int __must_check xen_allocate_irq_dynamic(void)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700460{
Ian Campbell89911502011-03-03 11:57:44 -0500461 int first = 0;
462 int irq;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700463
Ian Campbell89911502011-03-03 11:57:44 -0500464#ifdef CONFIG_X86_IO_APIC
465 /*
466 * For an HVM guest or domain 0 which see "real" (emulated or
Lucas De Marchi25985ed2011-03-30 22:57:33 -0300467 * actual respectively) GSIs we allocate dynamic IRQs
Ian Campbell89911502011-03-03 11:57:44 -0500468 * e.g. those corresponding to event channels or MSIs
469 * etc. from the range above those "real" GSIs to avoid
470 * collisions.
Konrad Rzeszutek Wilkd1b758e2010-12-09 14:53:29 -0500471 */
Ian Campbell89911502011-03-03 11:57:44 -0500472 if (xen_initial_domain() || xen_hvm_domain())
473 first = get_nr_irqs_gsi();
474#endif
475
Ian Campbell89911502011-03-03 11:57:44 -0500476 irq = irq_alloc_desc_from(first, -1);
477
Konrad Rzeszutek Wilke6599222011-09-29 13:26:45 -0400478 if (irq >= 0)
479 xen_irq_init(irq);
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800480
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700481 return irq;
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400482}
483
Ian Campbell7bee9762011-03-10 16:08:15 +0000484static int __must_check xen_allocate_irq_gsi(unsigned gsi)
Ian Campbellc9df1ce2011-01-11 17:20:15 +0000485{
486 int irq;
487
Ian Campbell89911502011-03-03 11:57:44 -0500488 /*
489 * A PV guest has no concept of a GSI (since it has no ACPI
490 * nor access to/knowledge of the physical APICs). Therefore
491 * all IRQs are dynamically allocated from the entire IRQ
492 * space.
493 */
494 if (xen_pv_domain() && !xen_initial_domain())
Ian Campbellc9df1ce2011-01-11 17:20:15 +0000495 return xen_allocate_irq_dynamic();
496
497 /* Legacy IRQ descriptors are already allocated by the arch. */
498 if (gsi < NR_IRQS_LEGACY)
Ian Campbell6cb65372011-03-10 16:08:11 +0000499 irq = gsi;
500 else
501 irq = irq_alloc_desc_at(gsi, -1);
Ian Campbellc9df1ce2011-01-11 17:20:15 +0000502
Ian Campbell6cb65372011-03-10 16:08:11 +0000503 xen_irq_init(irq);
Ian Campbellc9df1ce2011-01-11 17:20:15 +0000504
505 return irq;
506}
507
508static void xen_free_irq(unsigned irq)
509{
Thomas Gleixnerc442b802011-03-25 10:58:06 +0100510 struct irq_info *info = irq_get_handler_data(irq);
Ian Campbell6cb65372011-03-10 16:08:11 +0000511
512 list_del(&info->list);
Ian Campbell9158c352011-03-10 16:08:09 +0000513
Thomas Gleixnerc442b802011-03-25 10:58:06 +0100514 irq_set_handler_data(irq, NULL);
Ian Campbellca62ce82011-03-10 16:08:12 +0000515
Daniel De Graaf420eb552011-10-27 17:58:47 -0400516 WARN_ON(info->refcnt > 0);
517
Ian Campbellca62ce82011-03-10 16:08:12 +0000518 kfree(info);
519
Ian Campbell72146102011-02-03 09:49:35 +0000520 /* Legacy IRQ descriptors are managed by the arch. */
521 if (irq < NR_IRQS_LEGACY)
522 return;
523
Ian Campbellc9df1ce2011-01-11 17:20:15 +0000524 irq_free_desc(irq);
525}
526
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400527static void pirq_query_unmask(int irq)
528{
529 struct physdev_irq_status_query irq_status;
530 struct irq_info *info = info_for_irq(irq);
531
532 BUG_ON(info->type != IRQT_PIRQ);
533
Stefano Stabellini7a043f12010-07-01 17:08:14 +0100534 irq_status.irq = pirq_from_irq(irq);
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400535 if (HYPERVISOR_physdev_op(PHYSDEVOP_irq_status_query, &irq_status))
536 irq_status.flags = 0;
537
538 info->u.pirq.flags &= ~PIRQ_NEEDS_EOI;
539 if (irq_status.flags & XENIRQSTAT_needs_eoi)
540 info->u.pirq.flags |= PIRQ_NEEDS_EOI;
541}
542
543static bool probing_irq(int irq)
544{
545 struct irq_desc *desc = irq_to_desc(irq);
546
547 return desc && desc->action == NULL;
548}
549
Stefano Stabellini7e186bd2011-05-06 12:27:50 +0100550static void eoi_pirq(struct irq_data *data)
551{
552 int evtchn = evtchn_from_irq(data->irq);
553 struct physdev_eoi eoi = { .irq = pirq_from_irq(data->irq) };
554 int rc = 0;
555
556 irq_move_irq(data);
557
558 if (VALID_EVTCHN(evtchn))
559 clear_evtchn(evtchn);
560
561 if (pirq_needs_eoi(data->irq)) {
562 rc = HYPERVISOR_physdev_op(PHYSDEVOP_eoi, &eoi);
563 WARN_ON(rc);
564 }
565}
566
567static void mask_ack_pirq(struct irq_data *data)
568{
569 disable_dynirq(data);
570 eoi_pirq(data);
571}
572
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +0000573static unsigned int __startup_pirq(unsigned int irq)
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400574{
575 struct evtchn_bind_pirq bind_pirq;
576 struct irq_info *info = info_for_irq(irq);
577 int evtchn = evtchn_from_irq(irq);
Konrad Rzeszutek Wilk15ebbb82010-10-04 13:43:27 -0400578 int rc;
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400579
580 BUG_ON(info->type != IRQT_PIRQ);
581
582 if (VALID_EVTCHN(evtchn))
583 goto out;
584
Stefano Stabellini7a043f12010-07-01 17:08:14 +0100585 bind_pirq.pirq = pirq_from_irq(irq);
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400586 /* NB. We are happy to share unless we are probing. */
Konrad Rzeszutek Wilk15ebbb82010-10-04 13:43:27 -0400587 bind_pirq.flags = info->u.pirq.flags & PIRQ_SHAREABLE ?
588 BIND_PIRQ__WILL_SHARE : 0;
589 rc = HYPERVISOR_event_channel_op(EVTCHNOP_bind_pirq, &bind_pirq);
590 if (rc != 0) {
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400591 if (!probing_irq(irq))
592 printk(KERN_INFO "Failed to obtain physical IRQ %d\n",
593 irq);
594 return 0;
595 }
596 evtchn = bind_pirq.port;
597
598 pirq_query_unmask(irq);
599
600 evtchn_to_irq[evtchn] = irq;
601 bind_evtchn_to_cpu(evtchn, 0);
602 info->evtchn = evtchn;
603
604out:
605 unmask_evtchn(evtchn);
Stefano Stabellini7e186bd2011-05-06 12:27:50 +0100606 eoi_pirq(irq_get_irq_data(irq));
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400607
608 return 0;
609}
610
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +0000611static unsigned int startup_pirq(struct irq_data *data)
612{
613 return __startup_pirq(data->irq);
614}
615
616static void shutdown_pirq(struct irq_data *data)
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400617{
618 struct evtchn_close close;
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +0000619 unsigned int irq = data->irq;
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400620 struct irq_info *info = info_for_irq(irq);
621 int evtchn = evtchn_from_irq(irq);
622
623 BUG_ON(info->type != IRQT_PIRQ);
624
625 if (!VALID_EVTCHN(evtchn))
626 return;
627
628 mask_evtchn(evtchn);
629
630 close.port = evtchn;
631 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close) != 0)
632 BUG();
633
634 bind_evtchn_to_cpu(evtchn, 0);
635 evtchn_to_irq[evtchn] = -1;
636 info->evtchn = 0;
637}
638
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +0000639static void enable_pirq(struct irq_data *data)
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400640{
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +0000641 startup_pirq(data);
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400642}
643
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +0000644static void disable_pirq(struct irq_data *data)
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400645{
Stefano Stabellini7e186bd2011-05-06 12:27:50 +0100646 disable_dynirq(data);
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400647}
648
Stefano Stabellini68c2c392012-05-21 16:54:10 +0100649int xen_irq_from_gsi(unsigned gsi)
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400650{
Ian Campbell6cb65372011-03-10 16:08:11 +0000651 struct irq_info *info;
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400652
Ian Campbell6cb65372011-03-10 16:08:11 +0000653 list_for_each_entry(info, &xen_irq_list_head, list) {
654 if (info->type != IRQT_PIRQ)
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400655 continue;
656
Ian Campbell6cb65372011-03-10 16:08:11 +0000657 if (info->u.pirq.gsi == gsi)
658 return info->irq;
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400659 }
660
661 return -1;
662}
Stefano Stabellini68c2c392012-05-21 16:54:10 +0100663EXPORT_SYMBOL_GPL(xen_irq_from_gsi);
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400664
Ian Campbell653378a2011-03-10 16:08:04 +0000665/*
666 * Do not make any assumptions regarding the relationship between the
667 * IRQ number returned here and the Xen pirq argument.
Stefano Stabellini7a043f12010-07-01 17:08:14 +0100668 *
669 * Note: We don't assign an event channel until the irq actually started
670 * up. Return an existing irq if we've already got one for the gsi.
Stefano Stabellinie5ac0bd2011-05-25 12:33:23 +0100671 *
672 * Shareable implies level triggered, not shareable implies edge
673 * triggered here.
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400674 */
Ian Campbellf4d06352011-03-10 16:08:07 +0000675int xen_bind_pirq_gsi_to_irq(unsigned gsi,
676 unsigned pirq, int shareable, char *name)
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400677{
Ian Campbella0e18112011-03-10 16:08:03 +0000678 int irq = -1;
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400679 struct physdev_irq irq_op;
680
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -0400681 mutex_lock(&irq_mapping_update_lock);
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400682
Stefano Stabellini68c2c392012-05-21 16:54:10 +0100683 irq = xen_irq_from_gsi(gsi);
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400684 if (irq != -1) {
Stefano Stabellini7a043f12010-07-01 17:08:14 +0100685 printk(KERN_INFO "xen_map_pirq_gsi: returning irq %d for gsi %u\n",
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400686 irq, gsi);
Daniel De Graaf420eb552011-10-27 17:58:47 -0400687 goto out;
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400688 }
689
Ian Campbellc9df1ce2011-01-11 17:20:15 +0000690 irq = xen_allocate_irq_gsi(gsi);
Ian Campbell7bee9762011-03-10 16:08:15 +0000691 if (irq < 0)
692 goto out;
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400693
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400694 irq_op.irq = irq;
Alex Nixonb5401a92010-03-18 16:31:34 -0400695 irq_op.vector = 0;
696
697 /* Only the privileged domain can do this. For non-priv, the pcifront
698 * driver provides a PCI bus that does the call to do exactly
699 * this in the priv domain. */
700 if (xen_initial_domain() &&
701 HYPERVISOR_physdev_op(PHYSDEVOP_alloc_irq_vector, &irq_op)) {
Ian Campbellc9df1ce2011-01-11 17:20:15 +0000702 xen_free_irq(irq);
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400703 irq = -ENOSPC;
704 goto out;
705 }
706
Konrad Rzeszutek Wilkbeafbdc2011-04-14 11:17:36 -0400707 xen_irq_info_pirq_init(irq, 0, pirq, gsi, irq_op.vector, DOMID_SELF,
Ian Campbell9158c352011-03-10 16:08:09 +0000708 shareable ? PIRQ_SHAREABLE : 0);
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400709
Stefano Stabellini7e186bd2011-05-06 12:27:50 +0100710 pirq_query_unmask(irq);
711 /* We try to use the handler with the appropriate semantic for the
Stefano Stabellinie5ac0bd2011-05-25 12:33:23 +0100712 * type of interrupt: if the interrupt is an edge triggered
713 * interrupt we use handle_edge_irq.
Stefano Stabellini7e186bd2011-05-06 12:27:50 +0100714 *
Stefano Stabellinie5ac0bd2011-05-25 12:33:23 +0100715 * On the other hand if the interrupt is level triggered we use
716 * handle_fasteoi_irq like the native code does for this kind of
Stefano Stabellini7e186bd2011-05-06 12:27:50 +0100717 * interrupts.
Stefano Stabellinie5ac0bd2011-05-25 12:33:23 +0100718 *
Stefano Stabellini7e186bd2011-05-06 12:27:50 +0100719 * Depending on the Xen version, pirq_needs_eoi might return true
720 * not only for level triggered interrupts but for edge triggered
721 * interrupts too. In any case Xen always honors the eoi mechanism,
722 * not injecting any more pirqs of the same kind if the first one
723 * hasn't received an eoi yet. Therefore using the fasteoi handler
724 * is the right choice either way.
725 */
Stefano Stabellinie5ac0bd2011-05-25 12:33:23 +0100726 if (shareable)
Stefano Stabellini7e186bd2011-05-06 12:27:50 +0100727 irq_set_chip_and_handler_name(irq, &xen_pirq_chip,
728 handle_fasteoi_irq, name);
729 else
730 irq_set_chip_and_handler_name(irq, &xen_pirq_chip,
731 handle_edge_irq, name);
732
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400733out:
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -0400734 mutex_unlock(&irq_mapping_update_lock);
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400735
736 return irq;
737}
738
Qing Hef731e3ef2010-10-11 15:30:09 +0100739#ifdef CONFIG_PCI_MSI
Ian Campbellbf480d92011-02-18 16:43:32 +0000740int xen_allocate_pirq_msi(struct pci_dev *dev, struct msi_desc *msidesc)
Ian Campbellcbf6aa82011-01-11 17:20:14 +0000741{
Ian Campbell5cad61a2011-02-18 16:43:31 +0000742 int rc;
Ian Campbellcbf6aa82011-01-11 17:20:14 +0000743 struct physdev_get_free_pirq op_get_free_pirq;
Ian Campbell5cad61a2011-02-18 16:43:31 +0000744
Ian Campbellbf480d92011-02-18 16:43:32 +0000745 op_get_free_pirq.type = MAP_PIRQ_TYPE_MSI;
Ian Campbellcbf6aa82011-01-11 17:20:14 +0000746 rc = HYPERVISOR_physdev_op(PHYSDEVOP_get_free_pirq, &op_get_free_pirq);
Ian Campbellcbf6aa82011-01-11 17:20:14 +0000747
Ian Campbell5cad61a2011-02-18 16:43:31 +0000748 WARN_ONCE(rc == -ENOSYS,
749 "hypervisor does not support the PHYSDEVOP_get_free_pirq interface\n");
750
751 return rc ? -1 : op_get_free_pirq.pirq;
Ian Campbellcbf6aa82011-01-11 17:20:14 +0000752}
753
Ian Campbellbf480d92011-02-18 16:43:32 +0000754int xen_bind_pirq_msi_to_irq(struct pci_dev *dev, struct msi_desc *msidesc,
Konrad Rzeszutek Wilkbeafbdc2011-04-14 11:17:36 -0400755 int pirq, int vector, const char *name,
756 domid_t domid)
Stefano Stabellini809f9262010-07-01 17:10:39 +0100757{
Ian Campbellbf480d92011-02-18 16:43:32 +0000758 int irq, ret;
Ian Campbell4b41df72011-02-18 16:43:29 +0000759
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -0400760 mutex_lock(&irq_mapping_update_lock);
Stefano Stabellini809f9262010-07-01 17:10:39 +0100761
Ian Campbell4b41df72011-02-18 16:43:29 +0000762 irq = xen_allocate_irq_dynamic();
Konrad Rzeszutek Wilke6599222011-09-29 13:26:45 -0400763 if (irq < 0)
Ian Campbellbb5d0792011-02-18 16:43:28 +0000764 goto out;
Stefano Stabellini809f9262010-07-01 17:10:39 +0100765
Stefano Stabellini7e186bd2011-05-06 12:27:50 +0100766 irq_set_chip_and_handler_name(irq, &xen_pirq_chip, handle_edge_irq,
767 name);
Stefano Stabellini809f9262010-07-01 17:10:39 +0100768
Konrad Rzeszutek Wilkbeafbdc2011-04-14 11:17:36 -0400769 xen_irq_info_pirq_init(irq, 0, pirq, 0, vector, domid, 0);
Linus Torvalds5f6fb452011-03-15 19:23:40 -0700770 ret = irq_set_msi_desc(irq, msidesc);
Ian Campbellbf480d92011-02-18 16:43:32 +0000771 if (ret < 0)
772 goto error_irq;
Stefano Stabellini809f9262010-07-01 17:10:39 +0100773out:
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -0400774 mutex_unlock(&irq_mapping_update_lock);
Ian Campbell4b41df72011-02-18 16:43:29 +0000775 return irq;
Ian Campbellbf480d92011-02-18 16:43:32 +0000776error_irq:
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -0400777 mutex_unlock(&irq_mapping_update_lock);
Ian Campbellbf480d92011-02-18 16:43:32 +0000778 xen_free_irq(irq);
Konrad Rzeszutek Wilke6599222011-09-29 13:26:45 -0400779 return ret;
Stefano Stabellini809f9262010-07-01 17:10:39 +0100780}
Qing Hef731e3ef2010-10-11 15:30:09 +0100781#endif
782
Alex Nixonb5401a92010-03-18 16:31:34 -0400783int xen_destroy_irq(int irq)
784{
785 struct irq_desc *desc;
Jeremy Fitzhardinge38aa66f2010-09-02 14:51:39 +0100786 struct physdev_unmap_pirq unmap_irq;
787 struct irq_info *info = info_for_irq(irq);
Alex Nixonb5401a92010-03-18 16:31:34 -0400788 int rc = -ENOENT;
789
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -0400790 mutex_lock(&irq_mapping_update_lock);
Alex Nixonb5401a92010-03-18 16:31:34 -0400791
792 desc = irq_to_desc(irq);
793 if (!desc)
794 goto out;
795
Jeremy Fitzhardinge38aa66f2010-09-02 14:51:39 +0100796 if (xen_initial_domain()) {
Konrad Rzeszutek Wilk12334712010-11-19 11:27:09 -0500797 unmap_irq.pirq = info->u.pirq.pirq;
Konrad Rzeszutek Wilkbeafbdc2011-04-14 11:17:36 -0400798 unmap_irq.domid = info->u.pirq.domid;
Jeremy Fitzhardinge38aa66f2010-09-02 14:51:39 +0100799 rc = HYPERVISOR_physdev_op(PHYSDEVOP_unmap_pirq, &unmap_irq);
Konrad Rzeszutek Wilk1eff1ad2011-02-16 16:26:44 -0500800 /* If another domain quits without making the pci_disable_msix
801 * call, the Xen hypervisor takes care of freeing the PIRQs
802 * (free_domain_pirqs).
803 */
804 if ((rc == -ESRCH && info->u.pirq.domid != DOMID_SELF))
805 printk(KERN_INFO "domain %d does not have %d anymore\n",
806 info->u.pirq.domid, info->u.pirq.pirq);
807 else if (rc) {
Jeremy Fitzhardinge38aa66f2010-09-02 14:51:39 +0100808 printk(KERN_WARNING "unmap irq failed %d\n", rc);
809 goto out;
810 }
811 }
Alex Nixonb5401a92010-03-18 16:31:34 -0400812
Ian Campbellc9df1ce2011-01-11 17:20:15 +0000813 xen_free_irq(irq);
Alex Nixonb5401a92010-03-18 16:31:34 -0400814
815out:
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -0400816 mutex_unlock(&irq_mapping_update_lock);
Alex Nixonb5401a92010-03-18 16:31:34 -0400817 return rc;
818}
819
Stefano Stabelliniaf42b8d2010-12-01 14:51:44 +0000820int xen_irq_from_pirq(unsigned pirq)
821{
Ian Campbell69c358c2011-03-10 16:08:13 +0000822 int irq;
823
824 struct irq_info *info;
825
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -0400826 mutex_lock(&irq_mapping_update_lock);
Ian Campbell69c358c2011-03-10 16:08:13 +0000827
828 list_for_each_entry(info, &xen_irq_list_head, list) {
Konrad Rzeszutek Wilk9bb9efe2011-09-29 13:13:30 -0400829 if (info->type != IRQT_PIRQ)
Ian Campbell69c358c2011-03-10 16:08:13 +0000830 continue;
831 irq = info->irq;
832 if (info->u.pirq.pirq == pirq)
833 goto out;
834 }
835 irq = -1;
836out:
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -0400837 mutex_unlock(&irq_mapping_update_lock);
Ian Campbell69c358c2011-03-10 16:08:13 +0000838
839 return irq;
Stefano Stabelliniaf42b8d2010-12-01 14:51:44 +0000840}
841
Konrad Rzeszutek Wilke6197ac2011-02-24 14:20:12 -0500842
843int xen_pirq_from_irq(unsigned irq)
844{
845 return pirq_from_irq(irq);
846}
847EXPORT_SYMBOL_GPL(xen_pirq_from_irq);
Jeremy Fitzhardingeb536b4b2007-07-17 18:37:06 -0700848int bind_evtchn_to_irq(unsigned int evtchn)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700849{
850 int irq;
851
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -0400852 mutex_lock(&irq_mapping_update_lock);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700853
854 irq = evtchn_to_irq[evtchn];
855
856 if (irq == -1) {
Ian Campbellc9df1ce2011-01-11 17:20:15 +0000857 irq = xen_allocate_irq_dynamic();
Wei Liu68ba45f2013-01-31 14:46:56 +0000858 if (irq < 0)
Ian Campbell7bee9762011-03-10 16:08:15 +0000859 goto out;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700860
Thomas Gleixnerc442b802011-03-25 10:58:06 +0100861 irq_set_chip_and_handler_name(irq, &xen_dynamic_chip,
Stefano Stabellini7e186bd2011-05-06 12:27:50 +0100862 handle_edge_irq, "event");
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700863
Ian Campbell9158c352011-03-10 16:08:09 +0000864 xen_irq_info_evtchn_init(irq, evtchn);
Konrad Rzeszutek Wilk5e152e62012-05-23 13:28:44 -0400865 } else {
866 struct irq_info *info = info_for_irq(irq);
867 WARN_ON(info == NULL || info->type != IRQT_EVTCHN);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700868 }
Stefano Stabellinia8636c02012-08-22 17:20:15 +0100869 irq_clear_status_flags(irq, IRQ_NOREQUEST|IRQ_NOAUTOEN);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700870
Ian Campbell7bee9762011-03-10 16:08:15 +0000871out:
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -0400872 mutex_unlock(&irq_mapping_update_lock);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700873
874 return irq;
875}
Jeremy Fitzhardingeb536b4b2007-07-17 18:37:06 -0700876EXPORT_SYMBOL_GPL(bind_evtchn_to_irq);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700877
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700878static int bind_ipi_to_irq(unsigned int ipi, unsigned int cpu)
879{
880 struct evtchn_bind_ipi bind_ipi;
881 int evtchn, irq;
882
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -0400883 mutex_lock(&irq_mapping_update_lock);
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700884
885 irq = per_cpu(ipi_to_irq, cpu)[ipi];
Ian Campbell90af9512009-02-06 16:55:58 -0800886
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700887 if (irq == -1) {
Ian Campbellc9df1ce2011-01-11 17:20:15 +0000888 irq = xen_allocate_irq_dynamic();
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700889 if (irq < 0)
890 goto out;
891
Thomas Gleixnerc442b802011-03-25 10:58:06 +0100892 irq_set_chip_and_handler_name(irq, &xen_percpu_chip,
Jeremy Fitzhardingeaaca4962010-08-20 18:57:53 -0700893 handle_percpu_irq, "ipi");
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700894
895 bind_ipi.vcpu = cpu;
896 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi,
897 &bind_ipi) != 0)
898 BUG();
899 evtchn = bind_ipi.port;
900
Ian Campbell3d4cfa32011-03-10 16:08:10 +0000901 xen_irq_info_ipi_init(cpu, irq, evtchn, ipi);
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700902
903 bind_evtchn_to_cpu(evtchn, cpu);
Konrad Rzeszutek Wilk5e152e62012-05-23 13:28:44 -0400904 } else {
905 struct irq_info *info = info_for_irq(irq);
906 WARN_ON(info == NULL || info->type != IRQT_IPI);
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700907 }
908
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700909 out:
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -0400910 mutex_unlock(&irq_mapping_update_lock);
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700911 return irq;
912}
913
Ian Campbell2e820f52009-02-09 12:05:50 -0800914static int bind_interdomain_evtchn_to_irq(unsigned int remote_domain,
915 unsigned int remote_port)
916{
917 struct evtchn_bind_interdomain bind_interdomain;
918 int err;
919
920 bind_interdomain.remote_dom = remote_domain;
921 bind_interdomain.remote_port = remote_port;
922
923 err = HYPERVISOR_event_channel_op(EVTCHNOP_bind_interdomain,
924 &bind_interdomain);
925
926 return err ? : bind_evtchn_to_irq(bind_interdomain.local_port);
927}
928
Olaf Hering62cc5fc2011-08-25 18:30:48 +0200929static int find_virq(unsigned int virq, unsigned int cpu)
930{
931 struct evtchn_status status;
932 int port, rc = -ENOENT;
933
934 memset(&status, 0, sizeof(status));
935 for (port = 0; port <= NR_EVENT_CHANNELS; port++) {
936 status.dom = DOMID_SELF;
937 status.port = port;
938 rc = HYPERVISOR_event_channel_op(EVTCHNOP_status, &status);
939 if (rc < 0)
940 continue;
941 if (status.status != EVTCHNSTAT_virq)
942 continue;
943 if (status.u.virq == virq && status.vcpu == cpu) {
944 rc = port;
945 break;
946 }
947 }
948 return rc;
949}
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700950
Jeremy Fitzhardinge4fe7d5a2010-09-02 16:17:06 +0100951int bind_virq_to_irq(unsigned int virq, unsigned int cpu)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700952{
953 struct evtchn_bind_virq bind_virq;
Olaf Hering62cc5fc2011-08-25 18:30:48 +0200954 int evtchn, irq, ret;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700955
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -0400956 mutex_lock(&irq_mapping_update_lock);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700957
958 irq = per_cpu(virq_to_irq, cpu)[virq];
959
960 if (irq == -1) {
Ian Campbellc9df1ce2011-01-11 17:20:15 +0000961 irq = xen_allocate_irq_dynamic();
Wei Liu68ba45f2013-01-31 14:46:56 +0000962 if (irq < 0)
Ian Campbell7bee9762011-03-10 16:08:15 +0000963 goto out;
Jeremy Fitzhardingea52521f2010-09-22 15:28:52 -0700964
Thomas Gleixnerc442b802011-03-25 10:58:06 +0100965 irq_set_chip_and_handler_name(irq, &xen_percpu_chip,
Jeremy Fitzhardingea52521f2010-09-22 15:28:52 -0700966 handle_percpu_irq, "virq");
967
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700968 bind_virq.virq = virq;
969 bind_virq.vcpu = cpu;
Olaf Hering62cc5fc2011-08-25 18:30:48 +0200970 ret = HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq,
971 &bind_virq);
972 if (ret == 0)
973 evtchn = bind_virq.port;
974 else {
975 if (ret == -EEXIST)
976 ret = find_virq(virq, cpu);
977 BUG_ON(ret < 0);
978 evtchn = ret;
979 }
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700980
Ian Campbell3d4cfa32011-03-10 16:08:10 +0000981 xen_irq_info_virq_init(cpu, irq, evtchn, virq);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700982
983 bind_evtchn_to_cpu(evtchn, cpu);
Konrad Rzeszutek Wilk5e152e62012-05-23 13:28:44 -0400984 } else {
985 struct irq_info *info = info_for_irq(irq);
986 WARN_ON(info == NULL || info->type != IRQT_VIRQ);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700987 }
988
Ian Campbell7bee9762011-03-10 16:08:15 +0000989out:
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -0400990 mutex_unlock(&irq_mapping_update_lock);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700991
992 return irq;
993}
994
995static void unbind_from_irq(unsigned int irq)
996{
997 struct evtchn_close close;
998 int evtchn = evtchn_from_irq(irq);
Daniel De Graaf420eb552011-10-27 17:58:47 -0400999 struct irq_info *info = irq_get_handler_data(irq);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001000
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -04001001 mutex_lock(&irq_mapping_update_lock);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001002
Daniel De Graaf420eb552011-10-27 17:58:47 -04001003 if (info->refcnt > 0) {
1004 info->refcnt--;
1005 if (info->refcnt != 0)
1006 goto done;
1007 }
1008
Jeremy Fitzhardinged77bbd42009-02-06 14:09:45 -08001009 if (VALID_EVTCHN(evtchn)) {
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001010 close.port = evtchn;
1011 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close) != 0)
1012 BUG();
1013
1014 switch (type_from_irq(irq)) {
1015 case IRQT_VIRQ:
1016 per_cpu(virq_to_irq, cpu_from_evtchn(evtchn))
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -08001017 [virq_from_irq(irq)] = -1;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001018 break;
Alex Nixond68d82a2008-08-22 11:52:15 +01001019 case IRQT_IPI:
1020 per_cpu(ipi_to_irq, cpu_from_evtchn(evtchn))
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -08001021 [ipi_from_irq(irq)] = -1;
Alex Nixond68d82a2008-08-22 11:52:15 +01001022 break;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001023 default:
1024 break;
1025 }
1026
1027 /* Closed ports are implicitly re-bound to VCPU0. */
1028 bind_evtchn_to_cpu(evtchn, 0);
1029
1030 evtchn_to_irq[evtchn] = -1;
Ian Campbellfed5ea82009-12-01 16:15:30 +00001031 }
1032
Ian Campbellca62ce82011-03-10 16:08:12 +00001033 BUG_ON(info_for_irq(irq)->type == IRQT_UNBOUND);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001034
Ian Campbell9158c352011-03-10 16:08:09 +00001035 xen_free_irq(irq);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001036
Daniel De Graaf420eb552011-10-27 17:58:47 -04001037 done:
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -04001038 mutex_unlock(&irq_mapping_update_lock);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001039}
1040
1041int bind_evtchn_to_irqhandler(unsigned int evtchn,
Jeff Garzik7c239972007-10-19 03:12:20 -04001042 irq_handler_t handler,
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001043 unsigned long irqflags,
1044 const char *devname, void *dev_id)
1045{
Nicolas Kaiser361ae8c2011-03-30 21:14:26 +02001046 int irq, retval;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001047
1048 irq = bind_evtchn_to_irq(evtchn);
Ian Campbell7bee9762011-03-10 16:08:15 +00001049 if (irq < 0)
1050 return irq;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001051 retval = request_irq(irq, handler, irqflags, devname, dev_id);
1052 if (retval != 0) {
1053 unbind_from_irq(irq);
1054 return retval;
1055 }
1056
1057 return irq;
1058}
1059EXPORT_SYMBOL_GPL(bind_evtchn_to_irqhandler);
1060
Ian Campbell2e820f52009-02-09 12:05:50 -08001061int bind_interdomain_evtchn_to_irqhandler(unsigned int remote_domain,
1062 unsigned int remote_port,
1063 irq_handler_t handler,
1064 unsigned long irqflags,
1065 const char *devname,
1066 void *dev_id)
1067{
1068 int irq, retval;
1069
1070 irq = bind_interdomain_evtchn_to_irq(remote_domain, remote_port);
1071 if (irq < 0)
1072 return irq;
1073
1074 retval = request_irq(irq, handler, irqflags, devname, dev_id);
1075 if (retval != 0) {
1076 unbind_from_irq(irq);
1077 return retval;
1078 }
1079
1080 return irq;
1081}
1082EXPORT_SYMBOL_GPL(bind_interdomain_evtchn_to_irqhandler);
1083
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001084int bind_virq_to_irqhandler(unsigned int virq, unsigned int cpu,
Jeff Garzik7c239972007-10-19 03:12:20 -04001085 irq_handler_t handler,
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001086 unsigned long irqflags, const char *devname, void *dev_id)
1087{
Nicolas Kaiser361ae8c2011-03-30 21:14:26 +02001088 int irq, retval;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001089
1090 irq = bind_virq_to_irq(virq, cpu);
Ian Campbell7bee9762011-03-10 16:08:15 +00001091 if (irq < 0)
1092 return irq;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001093 retval = request_irq(irq, handler, irqflags, devname, dev_id);
1094 if (retval != 0) {
1095 unbind_from_irq(irq);
1096 return retval;
1097 }
1098
1099 return irq;
1100}
1101EXPORT_SYMBOL_GPL(bind_virq_to_irqhandler);
1102
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -07001103int bind_ipi_to_irqhandler(enum ipi_vector ipi,
1104 unsigned int cpu,
1105 irq_handler_t handler,
1106 unsigned long irqflags,
1107 const char *devname,
1108 void *dev_id)
1109{
1110 int irq, retval;
1111
1112 irq = bind_ipi_to_irq(ipi, cpu);
1113 if (irq < 0)
1114 return irq;
1115
Ian Campbell9bab0b72011-10-03 15:37:00 +01001116 irqflags |= IRQF_NO_SUSPEND | IRQF_FORCE_RESUME | IRQF_EARLY_RESUME;
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -07001117 retval = request_irq(irq, handler, irqflags, devname, dev_id);
1118 if (retval != 0) {
1119 unbind_from_irq(irq);
1120 return retval;
1121 }
1122
1123 return irq;
1124}
1125
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001126void unbind_from_irqhandler(unsigned int irq, void *dev_id)
1127{
1128 free_irq(irq, dev_id);
1129 unbind_from_irq(irq);
1130}
1131EXPORT_SYMBOL_GPL(unbind_from_irqhandler);
1132
Daniel De Graaf420eb552011-10-27 17:58:47 -04001133int evtchn_make_refcounted(unsigned int evtchn)
1134{
1135 int irq = evtchn_to_irq[evtchn];
1136 struct irq_info *info;
1137
1138 if (irq == -1)
1139 return -ENOENT;
1140
1141 info = irq_get_handler_data(irq);
1142
1143 if (!info)
1144 return -ENOENT;
1145
1146 WARN_ON(info->refcnt != -1);
1147
1148 info->refcnt = 1;
1149
1150 return 0;
1151}
1152EXPORT_SYMBOL_GPL(evtchn_make_refcounted);
1153
1154int evtchn_get(unsigned int evtchn)
1155{
1156 int irq;
1157 struct irq_info *info;
1158 int err = -ENOENT;
1159
Daniel De Graafc3b3f162011-11-28 11:49:09 -05001160 if (evtchn >= NR_EVENT_CHANNELS)
1161 return -EINVAL;
1162
Daniel De Graaf420eb552011-10-27 17:58:47 -04001163 mutex_lock(&irq_mapping_update_lock);
1164
1165 irq = evtchn_to_irq[evtchn];
1166 if (irq == -1)
1167 goto done;
1168
1169 info = irq_get_handler_data(irq);
1170
1171 if (!info)
1172 goto done;
1173
1174 err = -EINVAL;
1175 if (info->refcnt <= 0)
1176 goto done;
1177
1178 info->refcnt++;
1179 err = 0;
1180 done:
1181 mutex_unlock(&irq_mapping_update_lock);
1182
1183 return err;
1184}
1185EXPORT_SYMBOL_GPL(evtchn_get);
1186
1187void evtchn_put(unsigned int evtchn)
1188{
1189 int irq = evtchn_to_irq[evtchn];
1190 if (WARN_ON(irq == -1))
1191 return;
1192 unbind_from_irq(irq);
1193}
1194EXPORT_SYMBOL_GPL(evtchn_put);
1195
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -07001196void xen_send_IPI_one(unsigned int cpu, enum ipi_vector vector)
1197{
1198 int irq = per_cpu(ipi_to_irq, cpu)[vector];
1199 BUG_ON(irq < 0);
1200 notify_remote_via_irq(irq);
1201}
1202
Jeremy Fitzhardingeee523ca2008-03-17 16:37:18 -07001203irqreturn_t xen_debug_interrupt(int irq, void *dev_id)
1204{
1205 struct shared_info *sh = HYPERVISOR_shared_info;
1206 int cpu = smp_processor_id();
Ian Campbellc81611c2013-02-20 11:48:06 +00001207 xen_ulong_t *cpu_evtchn = per_cpu(cpu_evtchn_mask, cpu);
Jeremy Fitzhardingeee523ca2008-03-17 16:37:18 -07001208 int i;
1209 unsigned long flags;
1210 static DEFINE_SPINLOCK(debug_lock);
Ian Campbellcb52e6d2010-10-15 11:52:46 +01001211 struct vcpu_info *v;
Jeremy Fitzhardingeee523ca2008-03-17 16:37:18 -07001212
1213 spin_lock_irqsave(&debug_lock, flags);
1214
Ian Campbellcb52e6d2010-10-15 11:52:46 +01001215 printk("\nvcpu %d\n ", cpu);
Jeremy Fitzhardingeee523ca2008-03-17 16:37:18 -07001216
1217 for_each_online_cpu(i) {
Ian Campbellcb52e6d2010-10-15 11:52:46 +01001218 int pending;
1219 v = per_cpu(xen_vcpu, i);
1220 pending = (get_irq_regs() && i == cpu)
1221 ? xen_irqs_disabled(get_irq_regs())
1222 : v->evtchn_upcall_mask;
Ian Campbellc81611c2013-02-20 11:48:06 +00001223 printk("%d: masked=%d pending=%d event_sel %0*"PRI_xen_ulong"\n ", i,
Ian Campbellcb52e6d2010-10-15 11:52:46 +01001224 pending, v->evtchn_upcall_pending,
1225 (int)(sizeof(v->evtchn_pending_sel)*2),
1226 v->evtchn_pending_sel);
Jeremy Fitzhardingeee523ca2008-03-17 16:37:18 -07001227 }
Ian Campbellcb52e6d2010-10-15 11:52:46 +01001228 v = per_cpu(xen_vcpu, cpu);
Jeremy Fitzhardingeee523ca2008-03-17 16:37:18 -07001229
Ian Campbellcb52e6d2010-10-15 11:52:46 +01001230 printk("\npending:\n ");
1231 for (i = ARRAY_SIZE(sh->evtchn_pending)-1; i >= 0; i--)
Ian Campbellc81611c2013-02-20 11:48:06 +00001232 printk("%0*"PRI_xen_ulong"%s",
1233 (int)sizeof(sh->evtchn_pending[0])*2,
Ian Campbellcb52e6d2010-10-15 11:52:46 +01001234 sh->evtchn_pending[i],
1235 i % 8 == 0 ? "\n " : " ");
1236 printk("\nglobal mask:\n ");
1237 for (i = ARRAY_SIZE(sh->evtchn_mask)-1; i >= 0; i--)
Ian Campbellc81611c2013-02-20 11:48:06 +00001238 printk("%0*"PRI_xen_ulong"%s",
Ian Campbellcb52e6d2010-10-15 11:52:46 +01001239 (int)(sizeof(sh->evtchn_mask[0])*2),
1240 sh->evtchn_mask[i],
1241 i % 8 == 0 ? "\n " : " ");
1242
1243 printk("\nglobally unmasked:\n ");
1244 for (i = ARRAY_SIZE(sh->evtchn_mask)-1; i >= 0; i--)
Ian Campbellc81611c2013-02-20 11:48:06 +00001245 printk("%0*"PRI_xen_ulong"%s",
1246 (int)(sizeof(sh->evtchn_mask[0])*2),
Ian Campbellcb52e6d2010-10-15 11:52:46 +01001247 sh->evtchn_pending[i] & ~sh->evtchn_mask[i],
1248 i % 8 == 0 ? "\n " : " ");
1249
1250 printk("\nlocal cpu%d mask:\n ", cpu);
Ian Campbellc81611c2013-02-20 11:48:06 +00001251 for (i = (NR_EVENT_CHANNELS/BITS_PER_EVTCHN_WORD)-1; i >= 0; i--)
1252 printk("%0*"PRI_xen_ulong"%s", (int)(sizeof(cpu_evtchn[0])*2),
Ian Campbellcb52e6d2010-10-15 11:52:46 +01001253 cpu_evtchn[i],
1254 i % 8 == 0 ? "\n " : " ");
1255
1256 printk("\nlocally unmasked:\n ");
1257 for (i = ARRAY_SIZE(sh->evtchn_mask)-1; i >= 0; i--) {
Ian Campbellc81611c2013-02-20 11:48:06 +00001258 xen_ulong_t pending = sh->evtchn_pending[i]
Ian Campbellcb52e6d2010-10-15 11:52:46 +01001259 & ~sh->evtchn_mask[i]
1260 & cpu_evtchn[i];
Ian Campbellc81611c2013-02-20 11:48:06 +00001261 printk("%0*"PRI_xen_ulong"%s",
1262 (int)(sizeof(sh->evtchn_mask[0])*2),
Ian Campbellcb52e6d2010-10-15 11:52:46 +01001263 pending, i % 8 == 0 ? "\n " : " ");
1264 }
Jeremy Fitzhardingeee523ca2008-03-17 16:37:18 -07001265
1266 printk("\npending list:\n");
Ian Campbellcb52e6d2010-10-15 11:52:46 +01001267 for (i = 0; i < NR_EVENT_CHANNELS; i++) {
Ian Campbellc81611c2013-02-20 11:48:06 +00001268 if (sync_test_bit(i, BM(sh->evtchn_pending))) {
1269 int word_idx = i / BITS_PER_EVTCHN_WORD;
Ian Campbellcb52e6d2010-10-15 11:52:46 +01001270 printk(" %d: event %d -> irq %d%s%s%s\n",
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -08001271 cpu_from_evtchn(i), i,
Ian Campbellcb52e6d2010-10-15 11:52:46 +01001272 evtchn_to_irq[i],
Ian Campbellc81611c2013-02-20 11:48:06 +00001273 sync_test_bit(word_idx, BM(&v->evtchn_pending_sel))
Ian Campbellcb52e6d2010-10-15 11:52:46 +01001274 ? "" : " l2-clear",
Ian Campbellc81611c2013-02-20 11:48:06 +00001275 !sync_test_bit(i, BM(sh->evtchn_mask))
Ian Campbellcb52e6d2010-10-15 11:52:46 +01001276 ? "" : " globally-masked",
Ian Campbellc81611c2013-02-20 11:48:06 +00001277 sync_test_bit(i, BM(cpu_evtchn))
Ian Campbellcb52e6d2010-10-15 11:52:46 +01001278 ? "" : " locally-masked");
Jeremy Fitzhardingeee523ca2008-03-17 16:37:18 -07001279 }
1280 }
1281
1282 spin_unlock_irqrestore(&debug_lock, flags);
1283
1284 return IRQ_HANDLED;
1285}
1286
Tejun Heo245b2e72009-06-24 15:13:48 +09001287static DEFINE_PER_CPU(unsigned, xed_nesting_count);
Keir Fraserada68142011-03-03 10:01:11 +00001288static DEFINE_PER_CPU(unsigned int, current_word_idx);
1289static DEFINE_PER_CPU(unsigned int, current_bit_idx);
Tejun Heo245b2e72009-06-24 15:13:48 +09001290
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001291/*
Scott Rixnerab7f8632011-03-03 09:30:08 +00001292 * Mask out the i least significant bits of w
1293 */
Ian Campbellc81611c2013-02-20 11:48:06 +00001294#define MASK_LSBS(w, i) (w & ((~((xen_ulong_t)0UL)) << i))
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001295
1296/*
1297 * Search the CPUs pending events bitmasks. For each one found, map
1298 * the event number to an irq, and feed it into do_IRQ() for
1299 * handling.
1300 *
1301 * Xen uses a two-level bitmap to speed searching. The first level is
1302 * a bitset of words which contain pending event bits. The second
1303 * level is a bitset of pending events themselves.
1304 */
Sheng Yang38e20b02010-05-14 12:40:51 +01001305static void __xen_evtchn_do_upcall(void)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001306{
Keir Fraser24b51c22011-03-03 11:06:28 +00001307 int start_word_idx, start_bit_idx;
Scott Rixnerab7f8632011-03-03 09:30:08 +00001308 int word_idx, bit_idx;
Keir Fraser24b51c22011-03-03 11:06:28 +00001309 int i;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001310 int cpu = get_cpu();
1311 struct shared_info *s = HYPERVISOR_shared_info;
Christoph Lameter780f36d2010-12-06 11:16:29 -06001312 struct vcpu_info *vcpu_info = __this_cpu_read(xen_vcpu);
Ruslan Pisarev088c05a2011-07-26 14:16:13 +03001313 unsigned count;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001314
Jeremy Fitzhardinge229664b2008-03-17 16:37:20 -07001315 do {
Ian Campbellc81611c2013-02-20 11:48:06 +00001316 xen_ulong_t pending_words;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001317
Jeremy Fitzhardinge229664b2008-03-17 16:37:20 -07001318 vcpu_info->evtchn_upcall_pending = 0;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001319
Christoph Lameterb2e4ae62010-12-06 11:40:07 -06001320 if (__this_cpu_inc_return(xed_nesting_count) - 1)
Jeremy Fitzhardinge229664b2008-03-17 16:37:20 -07001321 goto out;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001322
Ian Campbellc81611c2013-02-20 11:48:06 +00001323 /*
1324 * Master flag must be cleared /before/ clearing
1325 * selector flag. xchg_xen_ulong must contain an
1326 * appropriate barrier.
1327 */
1328 pending_words = xchg_xen_ulong(&vcpu_info->evtchn_pending_sel, 0);
Jeremy Fitzhardinge229664b2008-03-17 16:37:20 -07001329
Keir Fraser24b51c22011-03-03 11:06:28 +00001330 start_word_idx = __this_cpu_read(current_word_idx);
1331 start_bit_idx = __this_cpu_read(current_bit_idx);
Scott Rixnerab7f8632011-03-03 09:30:08 +00001332
Keir Fraser24b51c22011-03-03 11:06:28 +00001333 word_idx = start_word_idx;
1334
1335 for (i = 0; pending_words != 0; i++) {
Ian Campbellc81611c2013-02-20 11:48:06 +00001336 xen_ulong_t pending_bits;
1337 xen_ulong_t words;
Jeremy Fitzhardinge229664b2008-03-17 16:37:20 -07001338
Scott Rixnerab7f8632011-03-03 09:30:08 +00001339 words = MASK_LSBS(pending_words, word_idx);
1340
1341 /*
Keir Fraserada68142011-03-03 10:01:11 +00001342 * If we masked out all events, wrap to beginning.
Scott Rixnerab7f8632011-03-03 09:30:08 +00001343 */
1344 if (words == 0) {
Keir Fraserada68142011-03-03 10:01:11 +00001345 word_idx = 0;
1346 bit_idx = 0;
Scott Rixnerab7f8632011-03-03 09:30:08 +00001347 continue;
1348 }
Ian Campbellc81611c2013-02-20 11:48:06 +00001349 word_idx = EVTCHN_FIRST_BIT(words);
Scott Rixnerab7f8632011-03-03 09:30:08 +00001350
Keir Fraser24b51c22011-03-03 11:06:28 +00001351 pending_bits = active_evtchns(cpu, s, word_idx);
1352 bit_idx = 0; /* usually scan entire word from start */
1353 if (word_idx == start_word_idx) {
1354 /* We scan the starting word in two parts */
1355 if (i == 0)
1356 /* 1st time: start in the middle */
1357 bit_idx = start_bit_idx;
1358 else
1359 /* 2nd time: mask bits done already */
1360 bit_idx &= (1UL << start_bit_idx) - 1;
1361 }
1362
Scott Rixnerab7f8632011-03-03 09:30:08 +00001363 do {
Ian Campbellc81611c2013-02-20 11:48:06 +00001364 xen_ulong_t bits;
Scott Rixnerab7f8632011-03-03 09:30:08 +00001365 int port, irq;
Eric W. Biedermanca4dbc62010-02-17 18:49:54 -08001366 struct irq_desc *desc;
Jeremy Fitzhardinge229664b2008-03-17 16:37:20 -07001367
Scott Rixnerab7f8632011-03-03 09:30:08 +00001368 bits = MASK_LSBS(pending_bits, bit_idx);
1369
1370 /* If we masked out all events, move on. */
Keir Fraserada68142011-03-03 10:01:11 +00001371 if (bits == 0)
Scott Rixnerab7f8632011-03-03 09:30:08 +00001372 break;
Scott Rixnerab7f8632011-03-03 09:30:08 +00001373
Ian Campbellc81611c2013-02-20 11:48:06 +00001374 bit_idx = EVTCHN_FIRST_BIT(bits);
Scott Rixnerab7f8632011-03-03 09:30:08 +00001375
1376 /* Process port. */
Ian Campbellc81611c2013-02-20 11:48:06 +00001377 port = (word_idx * BITS_PER_EVTCHN_WORD) + bit_idx;
Scott Rixnerab7f8632011-03-03 09:30:08 +00001378 irq = evtchn_to_irq[port];
1379
Eric W. Biedermanca4dbc62010-02-17 18:49:54 -08001380 if (irq != -1) {
1381 desc = irq_to_desc(irq);
1382 if (desc)
1383 generic_handle_irq_desc(irq, desc);
1384 }
Scott Rixnerab7f8632011-03-03 09:30:08 +00001385
Ian Campbellc81611c2013-02-20 11:48:06 +00001386 bit_idx = (bit_idx + 1) % BITS_PER_EVTCHN_WORD;
Keir Fraserada68142011-03-03 10:01:11 +00001387
1388 /* Next caller starts at last processed + 1 */
1389 __this_cpu_write(current_word_idx,
1390 bit_idx ? word_idx :
Ian Campbellc81611c2013-02-20 11:48:06 +00001391 (word_idx+1) % BITS_PER_EVTCHN_WORD);
Keir Fraserada68142011-03-03 10:01:11 +00001392 __this_cpu_write(current_bit_idx, bit_idx);
1393 } while (bit_idx != 0);
Scott Rixnerab7f8632011-03-03 09:30:08 +00001394
Keir Fraser24b51c22011-03-03 11:06:28 +00001395 /* Scan start_l1i twice; all others once. */
1396 if ((word_idx != start_word_idx) || (i != 0))
Scott Rixnerab7f8632011-03-03 09:30:08 +00001397 pending_words &= ~(1UL << word_idx);
Keir Fraserada68142011-03-03 10:01:11 +00001398
Ian Campbellc81611c2013-02-20 11:48:06 +00001399 word_idx = (word_idx + 1) % BITS_PER_EVTCHN_WORD;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001400 }
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001401
Jeremy Fitzhardinge229664b2008-03-17 16:37:20 -07001402 BUG_ON(!irqs_disabled());
1403
Christoph Lameter780f36d2010-12-06 11:16:29 -06001404 count = __this_cpu_read(xed_nesting_count);
1405 __this_cpu_write(xed_nesting_count, 0);
Stefano Stabellini183d03c2010-05-17 17:08:21 +01001406 } while (count != 1 || vcpu_info->evtchn_upcall_pending);
Jeremy Fitzhardinge229664b2008-03-17 16:37:20 -07001407
1408out:
Jeremy Fitzhardinge3445a8f2009-02-06 14:09:46 -08001409
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001410 put_cpu();
1411}
1412
Sheng Yang38e20b02010-05-14 12:40:51 +01001413void xen_evtchn_do_upcall(struct pt_regs *regs)
1414{
1415 struct pt_regs *old_regs = set_irq_regs(regs);
1416
Mojiong Qiu772aebc2012-11-06 16:08:15 +08001417 irq_enter();
Stefano Stabellini0ec53ec2012-09-14 13:37:32 +00001418#ifdef CONFIG_X86
Sheng Yang38e20b02010-05-14 12:40:51 +01001419 exit_idle();
Stefano Stabellini0ec53ec2012-09-14 13:37:32 +00001420#endif
Sheng Yang38e20b02010-05-14 12:40:51 +01001421
1422 __xen_evtchn_do_upcall();
1423
1424 irq_exit();
1425 set_irq_regs(old_regs);
1426}
1427
1428void xen_hvm_evtchn_do_upcall(void)
1429{
1430 __xen_evtchn_do_upcall();
1431}
Stefano Stabellini183d03c2010-05-17 17:08:21 +01001432EXPORT_SYMBOL_GPL(xen_hvm_evtchn_do_upcall);
Sheng Yang38e20b02010-05-14 12:40:51 +01001433
Jeremy Fitzhardingeeb1e3052008-05-26 23:31:23 +01001434/* Rebind a new event channel to an existing irq. */
1435void rebind_evtchn_irq(int evtchn, int irq)
1436{
Jeremy Fitzhardinged77bbd42009-02-06 14:09:45 -08001437 struct irq_info *info = info_for_irq(irq);
1438
Jeremy Fitzhardingeeb1e3052008-05-26 23:31:23 +01001439 /* Make sure the irq is masked, since the new event channel
1440 will also be masked. */
1441 disable_irq(irq);
1442
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -04001443 mutex_lock(&irq_mapping_update_lock);
Jeremy Fitzhardingeeb1e3052008-05-26 23:31:23 +01001444
1445 /* After resume the irq<->evtchn mappings are all cleared out */
1446 BUG_ON(evtchn_to_irq[evtchn] != -1);
1447 /* Expect irq to have been bound before,
Jeremy Fitzhardinged77bbd42009-02-06 14:09:45 -08001448 so there should be a proper type */
1449 BUG_ON(info->type == IRQT_UNBOUND);
Jeremy Fitzhardingeeb1e3052008-05-26 23:31:23 +01001450
Ian Campbell9158c352011-03-10 16:08:09 +00001451 xen_irq_info_evtchn_init(irq, evtchn);
Jeremy Fitzhardingeeb1e3052008-05-26 23:31:23 +01001452
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -04001453 mutex_unlock(&irq_mapping_update_lock);
Jeremy Fitzhardingeeb1e3052008-05-26 23:31:23 +01001454
1455 /* new event channels are always bound to cpu 0 */
Rusty Russell0de26522008-12-13 21:20:26 +10301456 irq_set_affinity(irq, cpumask_of(0));
Jeremy Fitzhardingeeb1e3052008-05-26 23:31:23 +01001457
1458 /* Unmask the event channel. */
1459 enable_irq(irq);
1460}
1461
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001462/* Rebind an evtchn so that it gets delivered to a specific cpu */
Yinghai Lud5dedd42009-04-27 17:59:21 -07001463static int rebind_irq_to_cpu(unsigned irq, unsigned tcpu)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001464{
1465 struct evtchn_bind_vcpu bind_vcpu;
1466 int evtchn = evtchn_from_irq(irq);
1467
Ian Campbellbe494722011-03-10 16:08:02 +00001468 if (!VALID_EVTCHN(evtchn))
1469 return -1;
1470
1471 /*
1472 * Events delivered via platform PCI interrupts are always
1473 * routed to vcpu 0 and hence cannot be rebound.
1474 */
1475 if (xen_hvm_domain() && !xen_have_vector_callback)
Yinghai Lud5dedd42009-04-27 17:59:21 -07001476 return -1;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001477
1478 /* Send future instances of this interrupt to other vcpu. */
1479 bind_vcpu.port = evtchn;
1480 bind_vcpu.vcpu = tcpu;
1481
1482 /*
1483 * If this fails, it usually just indicates that we're dealing with a
1484 * virq or IPI channel, which don't actually need to be rebound. Ignore
1485 * it, but don't do the xenlinux-level rebind in that case.
1486 */
1487 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_vcpu, &bind_vcpu) >= 0)
1488 bind_evtchn_to_cpu(evtchn, tcpu);
Yinghai Lud5dedd42009-04-27 17:59:21 -07001489
1490 return 0;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001491}
1492
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001493static int set_affinity_irq(struct irq_data *data, const struct cpumask *dest,
1494 bool force)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001495{
Rusty Russell0de26522008-12-13 21:20:26 +10301496 unsigned tcpu = cpumask_first(dest);
Yinghai Lud5dedd42009-04-27 17:59:21 -07001497
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001498 return rebind_irq_to_cpu(data->irq, tcpu);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001499}
1500
Isaku Yamahata642e0c82008-04-02 10:53:57 -07001501int resend_irq_on_evtchn(unsigned int irq)
1502{
1503 int masked, evtchn = evtchn_from_irq(irq);
1504 struct shared_info *s = HYPERVISOR_shared_info;
1505
1506 if (!VALID_EVTCHN(evtchn))
1507 return 1;
1508
Ian Campbellc81611c2013-02-20 11:48:06 +00001509 masked = sync_test_and_set_bit(evtchn, BM(s->evtchn_mask));
1510 sync_set_bit(evtchn, BM(s->evtchn_pending));
Isaku Yamahata642e0c82008-04-02 10:53:57 -07001511 if (!masked)
1512 unmask_evtchn(evtchn);
1513
1514 return 1;
1515}
1516
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001517static void enable_dynirq(struct irq_data *data)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001518{
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001519 int evtchn = evtchn_from_irq(data->irq);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001520
1521 if (VALID_EVTCHN(evtchn))
1522 unmask_evtchn(evtchn);
1523}
1524
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001525static void disable_dynirq(struct irq_data *data)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001526{
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001527 int evtchn = evtchn_from_irq(data->irq);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001528
1529 if (VALID_EVTCHN(evtchn))
1530 mask_evtchn(evtchn);
1531}
1532
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001533static void ack_dynirq(struct irq_data *data)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001534{
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001535 int evtchn = evtchn_from_irq(data->irq);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001536
Stefano Stabellini7e186bd2011-05-06 12:27:50 +01001537 irq_move_irq(data);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001538
1539 if (VALID_EVTCHN(evtchn))
Stefano Stabellini7e186bd2011-05-06 12:27:50 +01001540 clear_evtchn(evtchn);
1541}
1542
1543static void mask_ack_dynirq(struct irq_data *data)
1544{
1545 disable_dynirq(data);
1546 ack_dynirq(data);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001547}
1548
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001549static int retrigger_dynirq(struct irq_data *data)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001550{
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001551 int evtchn = evtchn_from_irq(data->irq);
Jeremy Fitzhardingeee8fa1c2008-03-17 16:37:19 -07001552 struct shared_info *sh = HYPERVISOR_shared_info;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001553 int ret = 0;
1554
1555 if (VALID_EVTCHN(evtchn)) {
Jeremy Fitzhardingeee8fa1c2008-03-17 16:37:19 -07001556 int masked;
1557
Ian Campbellc81611c2013-02-20 11:48:06 +00001558 masked = sync_test_and_set_bit(evtchn, BM(sh->evtchn_mask));
1559 sync_set_bit(evtchn, BM(sh->evtchn_pending));
Jeremy Fitzhardingeee8fa1c2008-03-17 16:37:19 -07001560 if (!masked)
1561 unmask_evtchn(evtchn);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001562 ret = 1;
1563 }
1564
1565 return ret;
1566}
1567
Ian Campbell0a852262011-03-10 16:08:06 +00001568static void restore_pirqs(void)
Stefano Stabellini9a069c32010-12-01 14:51:44 +00001569{
1570 int pirq, rc, irq, gsi;
1571 struct physdev_map_pirq map_irq;
Ian Campbell69c358c2011-03-10 16:08:13 +00001572 struct irq_info *info;
Stefano Stabellini9a069c32010-12-01 14:51:44 +00001573
Ian Campbell69c358c2011-03-10 16:08:13 +00001574 list_for_each_entry(info, &xen_irq_list_head, list) {
1575 if (info->type != IRQT_PIRQ)
Stefano Stabellini9a069c32010-12-01 14:51:44 +00001576 continue;
1577
Ian Campbell69c358c2011-03-10 16:08:13 +00001578 pirq = info->u.pirq.pirq;
1579 gsi = info->u.pirq.gsi;
1580 irq = info->irq;
1581
Stefano Stabellini9a069c32010-12-01 14:51:44 +00001582 /* save/restore of PT devices doesn't work, so at this point the
1583 * only devices present are GSI based emulated devices */
Stefano Stabellini9a069c32010-12-01 14:51:44 +00001584 if (!gsi)
1585 continue;
1586
1587 map_irq.domid = DOMID_SELF;
1588 map_irq.type = MAP_PIRQ_TYPE_GSI;
1589 map_irq.index = gsi;
1590 map_irq.pirq = pirq;
1591
1592 rc = HYPERVISOR_physdev_op(PHYSDEVOP_map_pirq, &map_irq);
1593 if (rc) {
1594 printk(KERN_WARNING "xen map irq failed gsi=%d irq=%d pirq=%d rc=%d\n",
1595 gsi, irq, pirq, rc);
Ian Campbell9158c352011-03-10 16:08:09 +00001596 xen_free_irq(irq);
Stefano Stabellini9a069c32010-12-01 14:51:44 +00001597 continue;
1598 }
1599
1600 printk(KERN_DEBUG "xen: --> irq=%d, pirq=%d\n", irq, map_irq.pirq);
1601
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001602 __startup_pirq(irq);
Stefano Stabellini9a069c32010-12-01 14:51:44 +00001603 }
1604}
1605
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +01001606static void restore_cpu_virqs(unsigned int cpu)
1607{
1608 struct evtchn_bind_virq bind_virq;
1609 int virq, irq, evtchn;
1610
1611 for (virq = 0; virq < NR_VIRQS; virq++) {
1612 if ((irq = per_cpu(virq_to_irq, cpu)[virq]) == -1)
1613 continue;
1614
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -08001615 BUG_ON(virq_from_irq(irq) != virq);
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +01001616
1617 /* Get a new binding from Xen. */
1618 bind_virq.virq = virq;
1619 bind_virq.vcpu = cpu;
1620 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq,
1621 &bind_virq) != 0)
1622 BUG();
1623 evtchn = bind_virq.port;
1624
1625 /* Record the new mapping. */
Ian Campbell3d4cfa32011-03-10 16:08:10 +00001626 xen_irq_info_virq_init(cpu, irq, evtchn, virq);
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +01001627 bind_evtchn_to_cpu(evtchn, cpu);
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +01001628 }
1629}
1630
1631static void restore_cpu_ipis(unsigned int cpu)
1632{
1633 struct evtchn_bind_ipi bind_ipi;
1634 int ipi, irq, evtchn;
1635
1636 for (ipi = 0; ipi < XEN_NR_IPIS; ipi++) {
1637 if ((irq = per_cpu(ipi_to_irq, cpu)[ipi]) == -1)
1638 continue;
1639
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -08001640 BUG_ON(ipi_from_irq(irq) != ipi);
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +01001641
1642 /* Get a new binding from Xen. */
1643 bind_ipi.vcpu = cpu;
1644 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi,
1645 &bind_ipi) != 0)
1646 BUG();
1647 evtchn = bind_ipi.port;
1648
1649 /* Record the new mapping. */
Ian Campbell3d4cfa32011-03-10 16:08:10 +00001650 xen_irq_info_ipi_init(cpu, irq, evtchn, ipi);
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +01001651 bind_evtchn_to_cpu(evtchn, cpu);
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +01001652 }
1653}
1654
Jeremy Fitzhardinge2d9e1e22008-07-07 12:07:53 -07001655/* Clear an irq's pending state, in preparation for polling on it */
1656void xen_clear_irq_pending(int irq)
1657{
1658 int evtchn = evtchn_from_irq(irq);
1659
1660 if (VALID_EVTCHN(evtchn))
1661 clear_evtchn(evtchn);
1662}
Konrad Rzeszutek Wilkd9a88142009-11-05 16:33:09 -05001663EXPORT_SYMBOL(xen_clear_irq_pending);
Jeremy Fitzhardinge168d2f42008-08-20 17:02:18 -07001664void xen_set_irq_pending(int irq)
1665{
1666 int evtchn = evtchn_from_irq(irq);
1667
1668 if (VALID_EVTCHN(evtchn))
1669 set_evtchn(evtchn);
1670}
1671
1672bool xen_test_irq_pending(int irq)
1673{
1674 int evtchn = evtchn_from_irq(irq);
1675 bool ret = false;
1676
1677 if (VALID_EVTCHN(evtchn))
1678 ret = test_evtchn(evtchn);
1679
1680 return ret;
1681}
1682
Konrad Rzeszutek Wilkd9a88142009-11-05 16:33:09 -05001683/* Poll waiting for an irq to become pending with timeout. In the usual case,
1684 * the irq will be disabled so it won't deliver an interrupt. */
1685void xen_poll_irq_timeout(int irq, u64 timeout)
Jeremy Fitzhardinge2d9e1e22008-07-07 12:07:53 -07001686{
1687 evtchn_port_t evtchn = evtchn_from_irq(irq);
1688
1689 if (VALID_EVTCHN(evtchn)) {
1690 struct sched_poll poll;
1691
1692 poll.nr_ports = 1;
Konrad Rzeszutek Wilkd9a88142009-11-05 16:33:09 -05001693 poll.timeout = timeout;
Isaku Yamahataff3c5362008-10-14 17:50:44 -07001694 set_xen_guest_handle(poll.ports, &evtchn);
Jeremy Fitzhardinge2d9e1e22008-07-07 12:07:53 -07001695
1696 if (HYPERVISOR_sched_op(SCHEDOP_poll, &poll) != 0)
1697 BUG();
1698 }
1699}
Konrad Rzeszutek Wilkd9a88142009-11-05 16:33:09 -05001700EXPORT_SYMBOL(xen_poll_irq_timeout);
1701/* Poll waiting for an irq to become pending. In the usual case, the
1702 * irq will be disabled so it won't deliver an interrupt. */
1703void xen_poll_irq(int irq)
1704{
1705 xen_poll_irq_timeout(irq, 0 /* no timeout */);
1706}
Jeremy Fitzhardinge2d9e1e22008-07-07 12:07:53 -07001707
Konrad Rzeszutek Wilkc7c2c3a2010-11-08 14:26:36 -05001708/* Check whether the IRQ line is shared with other guests. */
1709int xen_test_irq_shared(int irq)
1710{
1711 struct irq_info *info = info_for_irq(irq);
1712 struct physdev_irq_status_query irq_status = { .irq = info->u.pirq.pirq };
1713
1714 if (HYPERVISOR_physdev_op(PHYSDEVOP_irq_status_query, &irq_status))
1715 return 0;
1716 return !(irq_status.flags & XENIRQSTAT_shared);
1717}
1718EXPORT_SYMBOL_GPL(xen_test_irq_shared);
1719
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +01001720void xen_irq_resume(void)
1721{
Ian Campbell6cb65372011-03-10 16:08:11 +00001722 unsigned int cpu, evtchn;
1723 struct irq_info *info;
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +01001724
1725 init_evtchn_cpu_bindings();
1726
1727 /* New event-channel space is not 'live' yet. */
1728 for (evtchn = 0; evtchn < NR_EVENT_CHANNELS; evtchn++)
1729 mask_evtchn(evtchn);
1730
1731 /* No IRQ <-> event-channel mappings. */
Ian Campbell6cb65372011-03-10 16:08:11 +00001732 list_for_each_entry(info, &xen_irq_list_head, list)
1733 info->evtchn = 0; /* zap event-channel binding */
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +01001734
1735 for (evtchn = 0; evtchn < NR_EVENT_CHANNELS; evtchn++)
1736 evtchn_to_irq[evtchn] = -1;
1737
1738 for_each_possible_cpu(cpu) {
1739 restore_cpu_virqs(cpu);
1740 restore_cpu_ipis(cpu);
1741 }
Ian Campbell69035912010-11-01 16:30:09 +00001742
Ian Campbell0a852262011-03-10 16:08:06 +00001743 restore_pirqs();
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +01001744}
1745
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001746static struct irq_chip xen_dynamic_chip __read_mostly = {
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001747 .name = "xen-dyn",
Jeremy Fitzhardinge54a353a2009-02-06 14:09:42 -08001748
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001749 .irq_disable = disable_dynirq,
1750 .irq_mask = disable_dynirq,
1751 .irq_unmask = enable_dynirq,
Jeremy Fitzhardinge54a353a2009-02-06 14:09:42 -08001752
Stefano Stabellini7e186bd2011-05-06 12:27:50 +01001753 .irq_ack = ack_dynirq,
1754 .irq_mask_ack = mask_ack_dynirq,
1755
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001756 .irq_set_affinity = set_affinity_irq,
1757 .irq_retrigger = retrigger_dynirq,
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001758};
1759
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -04001760static struct irq_chip xen_pirq_chip __read_mostly = {
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001761 .name = "xen-pirq",
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -04001762
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001763 .irq_startup = startup_pirq,
1764 .irq_shutdown = shutdown_pirq,
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001765 .irq_enable = enable_pirq,
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001766 .irq_disable = disable_pirq,
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -04001767
Stefano Stabellini7e186bd2011-05-06 12:27:50 +01001768 .irq_mask = disable_dynirq,
1769 .irq_unmask = enable_dynirq,
1770
1771 .irq_ack = eoi_pirq,
1772 .irq_eoi = eoi_pirq,
1773 .irq_mask_ack = mask_ack_pirq,
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -04001774
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001775 .irq_set_affinity = set_affinity_irq,
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -04001776
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001777 .irq_retrigger = retrigger_dynirq,
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -04001778};
1779
Jeremy Fitzhardingeaaca4962010-08-20 18:57:53 -07001780static struct irq_chip xen_percpu_chip __read_mostly = {
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001781 .name = "xen-percpu",
Jeremy Fitzhardingeaaca4962010-08-20 18:57:53 -07001782
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001783 .irq_disable = disable_dynirq,
1784 .irq_mask = disable_dynirq,
1785 .irq_unmask = enable_dynirq,
Jeremy Fitzhardingeaaca4962010-08-20 18:57:53 -07001786
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001787 .irq_ack = ack_dynirq,
Jeremy Fitzhardingeaaca4962010-08-20 18:57:53 -07001788};
1789
Sheng Yang38e20b02010-05-14 12:40:51 +01001790int xen_set_callback_via(uint64_t via)
1791{
1792 struct xen_hvm_param a;
1793 a.domid = DOMID_SELF;
1794 a.index = HVM_PARAM_CALLBACK_IRQ;
1795 a.value = via;
1796 return HYPERVISOR_hvm_op(HVMOP_set_param, &a);
1797}
1798EXPORT_SYMBOL_GPL(xen_set_callback_via);
1799
Stefano Stabellinica65f9f2010-07-29 14:37:48 +01001800#ifdef CONFIG_XEN_PVHVM
Sheng Yang38e20b02010-05-14 12:40:51 +01001801/* Vector callbacks are better than PCI interrupts to receive event
1802 * channel notifications because we can receive vector callbacks on any
1803 * vcpu and we don't need PCI support or APIC interactions. */
1804void xen_callback_vector(void)
1805{
1806 int rc;
1807 uint64_t callback_via;
1808 if (xen_have_vector_callback) {
K. Y. Srinivasanbc2b0332013-02-03 17:22:39 -08001809 callback_via = HVM_CALLBACK_VECTOR(HYPERVISOR_CALLBACK_VECTOR);
Sheng Yang38e20b02010-05-14 12:40:51 +01001810 rc = xen_set_callback_via(callback_via);
1811 if (rc) {
1812 printk(KERN_ERR "Request for Xen HVM callback vector"
1813 " failed.\n");
1814 xen_have_vector_callback = 0;
1815 return;
1816 }
1817 printk(KERN_INFO "Xen HVM callback vector for event delivery is "
1818 "enabled\n");
1819 /* in the restore case the vector has already been allocated */
K. Y. Srinivasanbc2b0332013-02-03 17:22:39 -08001820 if (!test_bit(HYPERVISOR_CALLBACK_VECTOR, used_vectors))
1821 alloc_intr_gate(HYPERVISOR_CALLBACK_VECTOR,
1822 xen_hvm_callback_vector);
Sheng Yang38e20b02010-05-14 12:40:51 +01001823 }
1824}
Stefano Stabellinica65f9f2010-07-29 14:37:48 +01001825#else
1826void xen_callback_vector(void) {}
1827#endif
Sheng Yang38e20b02010-05-14 12:40:51 +01001828
Stefano Stabellini2e3d8862012-10-02 15:57:57 +01001829void __init xen_init_IRQ(void)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001830{
Stefano Stabellini0ec53ec2012-09-14 13:37:32 +00001831 int i;
Mike Travisc7a35892009-01-10 21:58:11 -08001832
Jeremy Fitzhardingeb21ddbf2010-06-07 16:28:49 -04001833 evtchn_to_irq = kcalloc(NR_EVENT_CHANNELS, sizeof(*evtchn_to_irq),
1834 GFP_KERNEL);
Konrad Rzeszutek Wilk9d093e22011-09-29 13:31:21 -04001835 BUG_ON(!evtchn_to_irq);
Jeremy Fitzhardingeb21ddbf2010-06-07 16:28:49 -04001836 for (i = 0; i < NR_EVENT_CHANNELS; i++)
1837 evtchn_to_irq[i] = -1;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001838
1839 init_evtchn_cpu_bindings();
1840
1841 /* No event channels are 'live' right now. */
1842 for (i = 0; i < NR_EVENT_CHANNELS; i++)
1843 mask_evtchn(i);
1844
Stefano Stabellini9846ff12012-01-30 16:21:48 +00001845 pirq_needs_eoi = pirq_needs_eoi_flag;
1846
Stefano Stabellini0ec53ec2012-09-14 13:37:32 +00001847#ifdef CONFIG_X86
Sheng Yang38e20b02010-05-14 12:40:51 +01001848 if (xen_hvm_domain()) {
1849 xen_callback_vector();
1850 native_init_IRQ();
Stefano Stabellini3942b742010-06-24 17:50:18 +01001851 /* pci_xen_hvm_init must be called after native_init_IRQ so that
1852 * __acpi_register_gsi can point at the right function */
1853 pci_xen_hvm_init();
Sheng Yang38e20b02010-05-14 12:40:51 +01001854 } else {
Stefano Stabellini0ec53ec2012-09-14 13:37:32 +00001855 int rc;
Stefano Stabellini9846ff12012-01-30 16:21:48 +00001856 struct physdev_pirq_eoi_gmfn eoi_gmfn;
1857
Sheng Yang38e20b02010-05-14 12:40:51 +01001858 irq_ctx_init(smp_processor_id());
Jeremy Fitzhardinge38aa66f2010-09-02 14:51:39 +01001859 if (xen_initial_domain())
Konrad Rzeszutek Wilka0ee0562011-06-09 09:49:13 -04001860 pci_xen_initial_domain();
Stefano Stabellini9846ff12012-01-30 16:21:48 +00001861
1862 pirq_eoi_map = (void *)__get_free_page(GFP_KERNEL|__GFP_ZERO);
1863 eoi_gmfn.gmfn = virt_to_mfn(pirq_eoi_map);
1864 rc = HYPERVISOR_physdev_op(PHYSDEVOP_pirq_eoi_gmfn_v2, &eoi_gmfn);
1865 if (rc != 0) {
1866 free_page((unsigned long) pirq_eoi_map);
1867 pirq_eoi_map = NULL;
1868 } else
1869 pirq_needs_eoi = pirq_check_eoi_map;
Sheng Yang38e20b02010-05-14 12:40:51 +01001870 }
Stefano Stabellini0ec53ec2012-09-14 13:37:32 +00001871#endif
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001872}