blob: feff4bf97577f4a3eacbbd3824fd3659824d0d23 [file] [log] [blame]
Steven J. Hill2299c492012-08-31 16:13:07 -05001/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 2008 Ralf Baechle (ralf@linux-mips.org)
7 * Copyright (C) 2012 MIPS Technologies, Inc. All rights reserved.
8 */
Ralf Baechle39b8d522008-04-28 17:14:26 +01009#include <linux/bitmap.h>
Andrew Brestickerfb8f7be12014-10-20 12:03:55 -070010#include <linux/clocksource.h>
Ralf Baechle39b8d522008-04-28 17:14:26 +010011#include <linux/init.h>
Andrew Bresticker18743d22014-09-18 14:47:24 -070012#include <linux/interrupt.h>
Andrew Brestickerfb8f7be12014-10-20 12:03:55 -070013#include <linux/irq.h>
Joel Porquet41a83e062015-07-07 17:11:46 -040014#include <linux/irqchip.h>
Andrew Brestickera7057272014-11-12 11:43:38 -080015#include <linux/of_address.h>
Andrew Bresticker18743d22014-09-18 14:47:24 -070016#include <linux/sched.h>
Ralf Baechle631330f2009-06-19 14:05:26 +010017#include <linux/smp.h>
Ralf Baechle39b8d522008-04-28 17:14:26 +010018
Paul Burtone83f7e02017-08-12 19:49:41 -070019#include <asm/mips-cps.h>
Steven J. Hill98b67c32012-08-31 16:18:49 -050020#include <asm/setup.h>
21#include <asm/traps.h>
Ralf Baechle39b8d522008-04-28 17:14:26 +010022
Andrew Brestickera7057272014-11-12 11:43:38 -080023#include <dt-bindings/interrupt-controller/mips-gic.h>
24
Paul Burtonb11d4c12017-08-12 21:36:29 -070025#define GIC_MAX_INTRS 256
26
27/* Add 2 to convert GIC CPU pin to core interrupt */
28#define GIC_CPU_PIN_OFFSET 2
29
30/* Mapped interrupt to pin X, then GIC will generate the vector (X+1). */
31#define GIC_PIN_TO_VEC_OFFSET 1
32
33/* Convert between local/shared IRQ number and GIC HW IRQ number. */
34#define GIC_LOCAL_HWIRQ_BASE 0
35#define GIC_LOCAL_TO_HWIRQ(x) (GIC_LOCAL_HWIRQ_BASE + (x))
36#define GIC_HWIRQ_TO_LOCAL(x) ((x) - GIC_LOCAL_HWIRQ_BASE)
37#define GIC_SHARED_HWIRQ_BASE GIC_NUM_LOCAL_INTRS
38#define GIC_SHARED_TO_HWIRQ(x) (GIC_SHARED_HWIRQ_BASE + (x))
39#define GIC_HWIRQ_TO_SHARED(x) ((x) - GIC_SHARED_HWIRQ_BASE)
40
Paul Burton582e2b42017-08-12 21:36:10 -070041void __iomem *mips_gic_base;
Steven J. Hill98b67c32012-08-31 16:18:49 -050042
Jeffrey Deans822350b2014-07-17 09:20:53 +010043struct gic_pcpu_mask {
Andrew Brestickerfbd55242014-09-18 14:47:25 -070044 DECLARE_BITMAP(pcpu_mask, GIC_MAX_INTRS);
Jeffrey Deans822350b2014-07-17 09:20:53 +010045};
46
Steven J. Hill0b271f52012-08-31 16:05:37 -050047static struct gic_pcpu_mask pcpu_masks[NR_CPUS];
Andrew Bresticker95150ae2014-09-18 14:47:21 -070048static DEFINE_SPINLOCK(gic_lock);
Andrew Brestickerc49581a2014-09-18 14:47:23 -070049static struct irq_domain *gic_irq_domain;
Qais Yousef2af70a92015-12-08 13:20:23 +000050static struct irq_domain *gic_ipi_domain;
Andrew Brestickerfbd55242014-09-18 14:47:25 -070051static int gic_shared_intrs;
Andrew Brestickere9de6882014-09-18 14:47:27 -070052static int gic_vpes;
Andrew Bresticker3263d082014-09-18 14:47:28 -070053static unsigned int gic_cpu_pin;
James Hogan1b6af712015-01-19 15:38:24 +000054static unsigned int timer_cpu_pin;
Andrew Bresticker4a6a3ea32014-09-18 14:47:26 -070055static struct irq_chip gic_level_irq_controller, gic_edge_irq_controller;
Qais Yousef2af70a92015-12-08 13:20:23 +000056DECLARE_BITMAP(ipi_resrv, GIC_MAX_INTRS);
Paul Burtonf8dcd9e2017-04-20 10:07:34 +010057DECLARE_BITMAP(ipi_available, GIC_MAX_INTRS);
Ralf Baechle39b8d522008-04-28 17:14:26 +010058
Andrew Brestickere9de6882014-09-18 14:47:27 -070059static bool gic_local_irq_is_routable(int intr)
60{
61 u32 vpe_ctl;
62
63 /* All local interrupts are routable in EIC mode. */
64 if (cpu_has_veic)
65 return true;
66
Paul Burton0d0cf582017-08-12 21:36:26 -070067 vpe_ctl = read_gic_vl_ctl();
Andrew Brestickere9de6882014-09-18 14:47:27 -070068 switch (intr) {
69 case GIC_LOCAL_INT_TIMER:
Paul Burton0d0cf582017-08-12 21:36:26 -070070 return vpe_ctl & GIC_VX_CTL_TIMER_ROUTABLE;
Andrew Brestickere9de6882014-09-18 14:47:27 -070071 case GIC_LOCAL_INT_PERFCTR:
Paul Burton0d0cf582017-08-12 21:36:26 -070072 return vpe_ctl & GIC_VX_CTL_PERFCNT_ROUTABLE;
Andrew Brestickere9de6882014-09-18 14:47:27 -070073 case GIC_LOCAL_INT_FDC:
Paul Burton0d0cf582017-08-12 21:36:26 -070074 return vpe_ctl & GIC_VX_CTL_FDC_ROUTABLE;
Andrew Brestickere9de6882014-09-18 14:47:27 -070075 case GIC_LOCAL_INT_SWINT0:
76 case GIC_LOCAL_INT_SWINT1:
Paul Burton0d0cf582017-08-12 21:36:26 -070077 return vpe_ctl & GIC_VX_CTL_SWINT_ROUTABLE;
Andrew Brestickere9de6882014-09-18 14:47:27 -070078 default:
79 return true;
80 }
81}
82
Andrew Bresticker3263d082014-09-18 14:47:28 -070083static void gic_bind_eic_interrupt(int irq, int set)
Steven J. Hill98b67c32012-08-31 16:18:49 -050084{
85 /* Convert irq vector # to hw int # */
86 irq -= GIC_PIN_TO_VEC_OFFSET;
87
88 /* Set irq to use shadow set */
Paul Burton0d0cf582017-08-12 21:36:26 -070089 write_gic_vl_eic_shadow_set(irq, set);
Steven J. Hill98b67c32012-08-31 16:18:49 -050090}
91
Qais Yousefbb11cff2015-12-08 13:20:28 +000092static void gic_send_ipi(struct irq_data *d, unsigned int cpu)
Ralf Baechle39b8d522008-04-28 17:14:26 +010093{
Qais Yousefbb11cff2015-12-08 13:20:28 +000094 irq_hw_number_t hwirq = GIC_HWIRQ_TO_SHARED(irqd_to_hwirq(d));
95
Paul Burton36807462017-08-12 21:36:24 -070096 write_gic_wedge(GIC_WEDGE_RW | hwirq);
Ralf Baechle39b8d522008-04-28 17:14:26 +010097}
98
Andrew Brestickere9de6882014-09-18 14:47:27 -070099int gic_get_c0_compare_int(void)
100{
101 if (!gic_local_irq_is_routable(GIC_LOCAL_INT_TIMER))
102 return MIPS_CPU_IRQ_BASE + cp0_compare_irq;
103 return irq_create_mapping(gic_irq_domain,
104 GIC_LOCAL_TO_HWIRQ(GIC_LOCAL_INT_TIMER));
105}
106
107int gic_get_c0_perfcount_int(void)
108{
109 if (!gic_local_irq_is_routable(GIC_LOCAL_INT_PERFCTR)) {
James Hogan7e3e6cb2015-01-27 21:45:50 +0000110 /* Is the performance counter shared with the timer? */
Andrew Brestickere9de6882014-09-18 14:47:27 -0700111 if (cp0_perfcount_irq < 0)
112 return -1;
113 return MIPS_CPU_IRQ_BASE + cp0_perfcount_irq;
114 }
115 return irq_create_mapping(gic_irq_domain,
116 GIC_LOCAL_TO_HWIRQ(GIC_LOCAL_INT_PERFCTR));
117}
118
James Hogan6429e2b2015-01-29 11:14:09 +0000119int gic_get_c0_fdc_int(void)
120{
121 if (!gic_local_irq_is_routable(GIC_LOCAL_INT_FDC)) {
122 /* Is the FDC IRQ even present? */
123 if (cp0_fdc_irq < 0)
124 return -1;
125 return MIPS_CPU_IRQ_BASE + cp0_fdc_irq;
126 }
127
James Hogan6429e2b2015-01-29 11:14:09 +0000128 return irq_create_mapping(gic_irq_domain,
129 GIC_LOCAL_TO_HWIRQ(GIC_LOCAL_INT_FDC));
130}
131
Rabin Vincent1b3ed362015-06-12 10:01:56 +0200132static void gic_handle_shared_int(bool chained)
Ralf Baechle39b8d522008-04-28 17:14:26 +0100133{
Paul Burtone98fcb22017-08-12 21:36:16 -0700134 unsigned int intr, virq;
Andrew Bresticker8f5ee792014-10-20 12:03:56 -0700135 unsigned long *pcpu_mask;
Andrew Bresticker8f5ee792014-10-20 12:03:56 -0700136 DECLARE_BITMAP(pending, GIC_MAX_INTRS);
137 DECLARE_BITMAP(intrmask, GIC_MAX_INTRS);
Ralf Baechle39b8d522008-04-28 17:14:26 +0100138
139 /* Get per-cpu bitmaps */
Ralf Baechle39b8d522008-04-28 17:14:26 +0100140 pcpu_mask = pcpu_masks[smp_processor_id()].pcpu_mask;
141
Paul Burtone98fcb22017-08-12 21:36:16 -0700142 if (mips_cm_is64) {
143 __ioread64_copy(pending, addr_gic_pend(),
144 DIV_ROUND_UP(gic_shared_intrs, 64));
145 __ioread64_copy(intrmask, addr_gic_mask(),
146 DIV_ROUND_UP(gic_shared_intrs, 64));
147 } else {
148 __ioread32_copy(pending, addr_gic_pend(),
149 DIV_ROUND_UP(gic_shared_intrs, 32));
150 __ioread32_copy(intrmask, addr_gic_mask(),
151 DIV_ROUND_UP(gic_shared_intrs, 32));
Ralf Baechle39b8d522008-04-28 17:14:26 +0100152 }
153
Andrew Brestickerfbd55242014-09-18 14:47:25 -0700154 bitmap_and(pending, pending, intrmask, gic_shared_intrs);
155 bitmap_and(pending, pending, pcpu_mask, gic_shared_intrs);
Ralf Baechle39b8d522008-04-28 17:14:26 +0100156
Paul Burtoncae750b2016-08-19 18:11:19 +0100157 for_each_set_bit(intr, pending, gic_shared_intrs) {
Qais Yousefd7eb4f22015-01-19 11:51:29 +0000158 virq = irq_linear_revmap(gic_irq_domain,
159 GIC_SHARED_TO_HWIRQ(intr));
Rabin Vincent1b3ed362015-06-12 10:01:56 +0200160 if (chained)
161 generic_handle_irq(virq);
162 else
163 do_IRQ(virq);
Qais Yousefd7eb4f22015-01-19 11:51:29 +0000164 }
Ralf Baechle39b8d522008-04-28 17:14:26 +0100165}
166
Thomas Gleixner161d0492011-03-23 21:08:58 +0000167static void gic_mask_irq(struct irq_data *d)
Ralf Baechle39b8d522008-04-28 17:14:26 +0100168{
Paul Burton87554b02017-08-12 21:36:18 -0700169 write_gic_rmask(BIT(GIC_HWIRQ_TO_SHARED(d->hwirq)));
Ralf Baechle39b8d522008-04-28 17:14:26 +0100170}
171
Thomas Gleixner161d0492011-03-23 21:08:58 +0000172static void gic_unmask_irq(struct irq_data *d)
Ralf Baechle39b8d522008-04-28 17:14:26 +0100173{
Paul Burton87554b02017-08-12 21:36:18 -0700174 write_gic_smask(BIT(GIC_HWIRQ_TO_SHARED(d->hwirq)));
Ralf Baechle39b8d522008-04-28 17:14:26 +0100175}
176
Andrew Bresticker5561c9e2014-09-18 14:47:20 -0700177static void gic_ack_irq(struct irq_data *d)
178{
Andrew Brestickere9de6882014-09-18 14:47:27 -0700179 unsigned int irq = GIC_HWIRQ_TO_SHARED(d->hwirq);
Andrew Brestickerc49581a2014-09-18 14:47:23 -0700180
Paul Burton36807462017-08-12 21:36:24 -0700181 write_gic_wedge(irq);
Andrew Bresticker5561c9e2014-09-18 14:47:20 -0700182}
183
Andrew Bresticker95150ae2014-09-18 14:47:21 -0700184static int gic_set_type(struct irq_data *d, unsigned int type)
185{
Andrew Brestickere9de6882014-09-18 14:47:27 -0700186 unsigned int irq = GIC_HWIRQ_TO_SHARED(d->hwirq);
Andrew Bresticker95150ae2014-09-18 14:47:21 -0700187 unsigned long flags;
188 bool is_edge;
Ralf Baechle39b8d522008-04-28 17:14:26 +0100189
Andrew Bresticker95150ae2014-09-18 14:47:21 -0700190 spin_lock_irqsave(&gic_lock, flags);
191 switch (type & IRQ_TYPE_SENSE_MASK) {
192 case IRQ_TYPE_EDGE_FALLING:
Paul Burton80e5f9c2017-08-12 21:36:19 -0700193 change_gic_pol(irq, GIC_POL_FALLING_EDGE);
Paul Burton471aa962017-08-12 21:36:20 -0700194 change_gic_trig(irq, GIC_TRIG_EDGE);
Paul Burtonc26ba672017-08-12 21:36:21 -0700195 change_gic_dual(irq, GIC_DUAL_SINGLE);
Andrew Bresticker95150ae2014-09-18 14:47:21 -0700196 is_edge = true;
197 break;
198 case IRQ_TYPE_EDGE_RISING:
Paul Burton80e5f9c2017-08-12 21:36:19 -0700199 change_gic_pol(irq, GIC_POL_RISING_EDGE);
Paul Burton471aa962017-08-12 21:36:20 -0700200 change_gic_trig(irq, GIC_TRIG_EDGE);
Paul Burtonc26ba672017-08-12 21:36:21 -0700201 change_gic_dual(irq, GIC_DUAL_SINGLE);
Andrew Bresticker95150ae2014-09-18 14:47:21 -0700202 is_edge = true;
203 break;
204 case IRQ_TYPE_EDGE_BOTH:
205 /* polarity is irrelevant in this case */
Paul Burton471aa962017-08-12 21:36:20 -0700206 change_gic_trig(irq, GIC_TRIG_EDGE);
Paul Burtonc26ba672017-08-12 21:36:21 -0700207 change_gic_dual(irq, GIC_DUAL_DUAL);
Andrew Bresticker95150ae2014-09-18 14:47:21 -0700208 is_edge = true;
209 break;
210 case IRQ_TYPE_LEVEL_LOW:
Paul Burton80e5f9c2017-08-12 21:36:19 -0700211 change_gic_pol(irq, GIC_POL_ACTIVE_LOW);
Paul Burton471aa962017-08-12 21:36:20 -0700212 change_gic_trig(irq, GIC_TRIG_LEVEL);
Paul Burtonc26ba672017-08-12 21:36:21 -0700213 change_gic_dual(irq, GIC_DUAL_SINGLE);
Andrew Bresticker95150ae2014-09-18 14:47:21 -0700214 is_edge = false;
215 break;
216 case IRQ_TYPE_LEVEL_HIGH:
217 default:
Paul Burton80e5f9c2017-08-12 21:36:19 -0700218 change_gic_pol(irq, GIC_POL_ACTIVE_HIGH);
Paul Burton471aa962017-08-12 21:36:20 -0700219 change_gic_trig(irq, GIC_TRIG_LEVEL);
Paul Burtonc26ba672017-08-12 21:36:21 -0700220 change_gic_dual(irq, GIC_DUAL_SINGLE);
Andrew Bresticker95150ae2014-09-18 14:47:21 -0700221 is_edge = false;
222 break;
223 }
224
Thomas Gleixnera595fc52015-06-23 14:41:25 +0200225 if (is_edge)
226 irq_set_chip_handler_name_locked(d, &gic_edge_irq_controller,
227 handle_edge_irq, NULL);
228 else
229 irq_set_chip_handler_name_locked(d, &gic_level_irq_controller,
230 handle_level_irq, NULL);
Andrew Bresticker95150ae2014-09-18 14:47:21 -0700231 spin_unlock_irqrestore(&gic_lock, flags);
232
233 return 0;
234}
235
236#ifdef CONFIG_SMP
Thomas Gleixner161d0492011-03-23 21:08:58 +0000237static int gic_set_affinity(struct irq_data *d, const struct cpumask *cpumask,
238 bool force)
Ralf Baechle39b8d522008-04-28 17:14:26 +0100239{
Andrew Brestickere9de6882014-09-18 14:47:27 -0700240 unsigned int irq = GIC_HWIRQ_TO_SHARED(d->hwirq);
Ralf Baechle39b8d522008-04-28 17:14:26 +0100241 cpumask_t tmp = CPU_MASK_NONE;
242 unsigned long flags;
243 int i;
244
Rusty Russell0de26522008-12-13 21:20:26 +1030245 cpumask_and(&tmp, cpumask, cpu_online_mask);
Rusty Russellf9b531f2015-03-05 10:49:16 +1030246 if (cpumask_empty(&tmp))
Andrew Bresticker14d160a2014-09-18 14:47:22 -0700247 return -EINVAL;
Ralf Baechle39b8d522008-04-28 17:14:26 +0100248
249 /* Assumption : cpumask refers to a single CPU */
250 spin_lock_irqsave(&gic_lock, flags);
Ralf Baechle39b8d522008-04-28 17:14:26 +0100251
Tony Wuc214c032013-06-21 10:13:08 +0000252 /* Re-route this IRQ */
Paul Burton0efe3cb2017-08-12 21:36:23 -0700253 write_gic_map_vp(irq, BIT(mips_cm_vp_id(cpumask_first(&tmp))));
Ralf Baechle39b8d522008-04-28 17:14:26 +0100254
Tony Wuc214c032013-06-21 10:13:08 +0000255 /* Update the pcpu_masks */
Paul Burton91951f92016-04-21 11:31:54 +0100256 for (i = 0; i < min(gic_vpes, NR_CPUS); i++)
Tony Wuc214c032013-06-21 10:13:08 +0000257 clear_bit(irq, pcpu_masks[i].pcpu_mask);
Rusty Russellf9b531f2015-03-05 10:49:16 +1030258 set_bit(irq, pcpu_masks[cpumask_first(&tmp)].pcpu_mask);
Tony Wuc214c032013-06-21 10:13:08 +0000259
Jiang Liu72f86db2015-06-01 16:05:38 +0800260 cpumask_copy(irq_data_get_affinity_mask(d), cpumask);
Ralf Baechle39b8d522008-04-28 17:14:26 +0100261 spin_unlock_irqrestore(&gic_lock, flags);
262
Thomas Gleixner161d0492011-03-23 21:08:58 +0000263 return IRQ_SET_MASK_OK_NOCOPY;
Ralf Baechle39b8d522008-04-28 17:14:26 +0100264}
265#endif
266
Andrew Bresticker4a6a3ea32014-09-18 14:47:26 -0700267static struct irq_chip gic_level_irq_controller = {
268 .name = "MIPS GIC",
269 .irq_mask = gic_mask_irq,
270 .irq_unmask = gic_unmask_irq,
271 .irq_set_type = gic_set_type,
272#ifdef CONFIG_SMP
273 .irq_set_affinity = gic_set_affinity,
274#endif
275};
276
277static struct irq_chip gic_edge_irq_controller = {
Thomas Gleixner161d0492011-03-23 21:08:58 +0000278 .name = "MIPS GIC",
Andrew Bresticker5561c9e2014-09-18 14:47:20 -0700279 .irq_ack = gic_ack_irq,
Thomas Gleixner161d0492011-03-23 21:08:58 +0000280 .irq_mask = gic_mask_irq,
Thomas Gleixner161d0492011-03-23 21:08:58 +0000281 .irq_unmask = gic_unmask_irq,
Andrew Bresticker95150ae2014-09-18 14:47:21 -0700282 .irq_set_type = gic_set_type,
Ralf Baechle39b8d522008-04-28 17:14:26 +0100283#ifdef CONFIG_SMP
Thomas Gleixner161d0492011-03-23 21:08:58 +0000284 .irq_set_affinity = gic_set_affinity,
Ralf Baechle39b8d522008-04-28 17:14:26 +0100285#endif
Qais Yousefbb11cff2015-12-08 13:20:28 +0000286 .ipi_send_single = gic_send_ipi,
Ralf Baechle39b8d522008-04-28 17:14:26 +0100287};
288
Rabin Vincent1b3ed362015-06-12 10:01:56 +0200289static void gic_handle_local_int(bool chained)
Andrew Brestickere9de6882014-09-18 14:47:27 -0700290{
291 unsigned long pending, masked;
Qais Yousefd7eb4f22015-01-19 11:51:29 +0000292 unsigned int intr, virq;
Andrew Brestickere9de6882014-09-18 14:47:27 -0700293
Paul Burton9da3c642017-08-12 21:36:25 -0700294 pending = read_gic_vl_pend();
295 masked = read_gic_vl_mask();
Andrew Brestickere9de6882014-09-18 14:47:27 -0700296
297 bitmap_and(&pending, &pending, &masked, GIC_NUM_LOCAL_INTRS);
298
Paul Burton0f4ed152016-09-13 17:54:27 +0100299 for_each_set_bit(intr, &pending, GIC_NUM_LOCAL_INTRS) {
Qais Yousefd7eb4f22015-01-19 11:51:29 +0000300 virq = irq_linear_revmap(gic_irq_domain,
301 GIC_LOCAL_TO_HWIRQ(intr));
Rabin Vincent1b3ed362015-06-12 10:01:56 +0200302 if (chained)
303 generic_handle_irq(virq);
304 else
305 do_IRQ(virq);
Qais Yousefd7eb4f22015-01-19 11:51:29 +0000306 }
Andrew Brestickere9de6882014-09-18 14:47:27 -0700307}
308
309static void gic_mask_local_irq(struct irq_data *d)
310{
311 int intr = GIC_HWIRQ_TO_LOCAL(d->hwirq);
312
Paul Burton9da3c642017-08-12 21:36:25 -0700313 write_gic_vl_rmask(BIT(intr));
Andrew Brestickere9de6882014-09-18 14:47:27 -0700314}
315
316static void gic_unmask_local_irq(struct irq_data *d)
317{
318 int intr = GIC_HWIRQ_TO_LOCAL(d->hwirq);
319
Paul Burton9da3c642017-08-12 21:36:25 -0700320 write_gic_vl_smask(BIT(intr));
Andrew Brestickere9de6882014-09-18 14:47:27 -0700321}
322
323static struct irq_chip gic_local_irq_controller = {
324 .name = "MIPS GIC Local",
325 .irq_mask = gic_mask_local_irq,
326 .irq_unmask = gic_unmask_local_irq,
327};
328
329static void gic_mask_local_irq_all_vpes(struct irq_data *d)
330{
331 int intr = GIC_HWIRQ_TO_LOCAL(d->hwirq);
332 int i;
333 unsigned long flags;
334
335 spin_lock_irqsave(&gic_lock, flags);
336 for (i = 0; i < gic_vpes; i++) {
Paul Burton0d0cf582017-08-12 21:36:26 -0700337 write_gic_vl_other(mips_cm_vp_id(i));
Paul Burton9da3c642017-08-12 21:36:25 -0700338 write_gic_vo_rmask(BIT(intr));
Andrew Brestickere9de6882014-09-18 14:47:27 -0700339 }
340 spin_unlock_irqrestore(&gic_lock, flags);
341}
342
343static void gic_unmask_local_irq_all_vpes(struct irq_data *d)
344{
345 int intr = GIC_HWIRQ_TO_LOCAL(d->hwirq);
346 int i;
347 unsigned long flags;
348
349 spin_lock_irqsave(&gic_lock, flags);
350 for (i = 0; i < gic_vpes; i++) {
Paul Burton0d0cf582017-08-12 21:36:26 -0700351 write_gic_vl_other(mips_cm_vp_id(i));
Paul Burton9da3c642017-08-12 21:36:25 -0700352 write_gic_vo_smask(BIT(intr));
Andrew Brestickere9de6882014-09-18 14:47:27 -0700353 }
354 spin_unlock_irqrestore(&gic_lock, flags);
355}
356
357static struct irq_chip gic_all_vpes_local_irq_controller = {
358 .name = "MIPS GIC Local",
359 .irq_mask = gic_mask_local_irq_all_vpes,
360 .irq_unmask = gic_unmask_local_irq_all_vpes,
361};
362
Andrew Bresticker18743d22014-09-18 14:47:24 -0700363static void __gic_irq_dispatch(void)
Ralf Baechle39b8d522008-04-28 17:14:26 +0100364{
Rabin Vincent1b3ed362015-06-12 10:01:56 +0200365 gic_handle_local_int(false);
366 gic_handle_shared_int(false);
Andrew Bresticker18743d22014-09-18 14:47:24 -0700367}
368
Thomas Gleixnerbd0b9ac2015-09-14 10:42:37 +0200369static void gic_irq_dispatch(struct irq_desc *desc)
Andrew Bresticker18743d22014-09-18 14:47:24 -0700370{
Rabin Vincent1b3ed362015-06-12 10:01:56 +0200371 gic_handle_local_int(true);
372 gic_handle_shared_int(true);
Andrew Bresticker18743d22014-09-18 14:47:24 -0700373}
374
Andrew Brestickere9de6882014-09-18 14:47:27 -0700375static int gic_local_irq_domain_map(struct irq_domain *d, unsigned int virq,
376 irq_hw_number_t hw)
Andrew Brestickerc49581a2014-09-18 14:47:23 -0700377{
Andrew Brestickere9de6882014-09-18 14:47:27 -0700378 int intr = GIC_HWIRQ_TO_LOCAL(hw);
Andrew Brestickere9de6882014-09-18 14:47:27 -0700379 int i;
380 unsigned long flags;
Paul Burtona0dc5cb2017-08-12 21:36:17 -0700381 u32 val;
Andrew Brestickere9de6882014-09-18 14:47:27 -0700382
383 if (!gic_local_irq_is_routable(intr))
384 return -EPERM;
385
Paul Burtona0dc5cb2017-08-12 21:36:17 -0700386 if (intr > GIC_LOCAL_INT_FDC) {
387 pr_err("Invalid local IRQ %d\n", intr);
388 return -EINVAL;
389 }
390
391 if (intr == GIC_LOCAL_INT_TIMER) {
392 /* CONFIG_MIPS_CMP workaround (see __gic_init) */
393 val = GIC_MAP_PIN_MAP_TO_PIN | timer_cpu_pin;
394 } else {
395 val = GIC_MAP_PIN_MAP_TO_PIN | gic_cpu_pin;
396 }
397
Andrew Brestickere9de6882014-09-18 14:47:27 -0700398 spin_lock_irqsave(&gic_lock, flags);
399 for (i = 0; i < gic_vpes; i++) {
Paul Burtona0dc5cb2017-08-12 21:36:17 -0700400 write_gic_vl_other(mips_cm_vp_id(i));
401 write_gic_vo_map(intr, val);
Andrew Brestickere9de6882014-09-18 14:47:27 -0700402 }
403 spin_unlock_irqrestore(&gic_lock, flags);
404
Paul Burtona0dc5cb2017-08-12 21:36:17 -0700405 return 0;
Andrew Brestickere9de6882014-09-18 14:47:27 -0700406}
407
408static int gic_shared_irq_domain_map(struct irq_domain *d, unsigned int virq,
Qais Yousef2af70a92015-12-08 13:20:23 +0000409 irq_hw_number_t hw, unsigned int vpe)
Andrew Brestickere9de6882014-09-18 14:47:27 -0700410{
411 int intr = GIC_HWIRQ_TO_SHARED(hw);
Andrew Brestickerc49581a2014-09-18 14:47:23 -0700412 unsigned long flags;
Qais Yousef78930f02015-12-08 13:20:26 +0000413 int i;
Andrew Brestickerc49581a2014-09-18 14:47:23 -0700414
Andrew Brestickerc49581a2014-09-18 14:47:23 -0700415 spin_lock_irqsave(&gic_lock, flags);
Paul Burtond3e8cf42017-08-12 21:36:22 -0700416 write_gic_map_pin(intr, GIC_MAP_PIN_MAP_TO_PIN | gic_cpu_pin);
Paul Burton0efe3cb2017-08-12 21:36:23 -0700417 write_gic_map_vp(intr, BIT(mips_cm_vp_id(vpe)));
Paul Burton91951f92016-04-21 11:31:54 +0100418 for (i = 0; i < min(gic_vpes, NR_CPUS); i++)
Qais Yousef78930f02015-12-08 13:20:26 +0000419 clear_bit(intr, pcpu_masks[i].pcpu_mask);
Qais Yousef2af70a92015-12-08 13:20:23 +0000420 set_bit(intr, pcpu_masks[vpe].pcpu_mask);
Andrew Brestickerc49581a2014-09-18 14:47:23 -0700421 spin_unlock_irqrestore(&gic_lock, flags);
422
423 return 0;
424}
425
Paul Burtonb87281e2017-04-20 10:07:35 +0100426static int gic_irq_domain_xlate(struct irq_domain *d, struct device_node *ctrlr,
Qais Yousefc98c18222015-12-08 13:20:24 +0000427 const u32 *intspec, unsigned int intsize,
428 irq_hw_number_t *out_hwirq,
429 unsigned int *out_type)
430{
431 if (intsize != 3)
432 return -EINVAL;
433
434 if (intspec[0] == GIC_SHARED)
435 *out_hwirq = GIC_SHARED_TO_HWIRQ(intspec[1]);
436 else if (intspec[0] == GIC_LOCAL)
437 *out_hwirq = GIC_LOCAL_TO_HWIRQ(intspec[1]);
438 else
439 return -EINVAL;
440 *out_type = intspec[2] & IRQ_TYPE_SENSE_MASK;
441
442 return 0;
443}
444
Matt Redfearn8ada00a2017-04-20 10:07:36 +0100445static int gic_irq_domain_map(struct irq_domain *d, unsigned int virq,
446 irq_hw_number_t hwirq)
Qais Yousefc98c18222015-12-08 13:20:24 +0000447{
Paul Burtonb87281e2017-04-20 10:07:35 +0100448 int err;
Qais Yousefc98c18222015-12-08 13:20:24 +0000449
Matt Redfearn8ada00a2017-04-20 10:07:36 +0100450 if (hwirq >= GIC_SHARED_HWIRQ_BASE) {
Paul Burtonb87281e2017-04-20 10:07:35 +0100451 /* verify that shared irqs don't conflict with an IPI irq */
452 if (test_bit(GIC_HWIRQ_TO_SHARED(hwirq), ipi_resrv))
453 return -EBUSY;
Qais Yousefc98c18222015-12-08 13:20:24 +0000454
Paul Burtonb87281e2017-04-20 10:07:35 +0100455 err = irq_domain_set_hwirq_and_chip(d, virq, hwirq,
456 &gic_level_irq_controller,
457 NULL);
458 if (err)
459 return err;
460
461 return gic_shared_irq_domain_map(d, virq, hwirq, 0);
Qais Yousefc98c18222015-12-08 13:20:24 +0000462 }
463
Paul Burtonb87281e2017-04-20 10:07:35 +0100464 switch (GIC_HWIRQ_TO_LOCAL(hwirq)) {
465 case GIC_LOCAL_INT_TIMER:
466 case GIC_LOCAL_INT_PERFCTR:
467 case GIC_LOCAL_INT_FDC:
468 /*
469 * HACK: These are all really percpu interrupts, but
470 * the rest of the MIPS kernel code does not use the
471 * percpu IRQ API for them.
472 */
473 err = irq_domain_set_hwirq_and_chip(d, virq, hwirq,
474 &gic_all_vpes_local_irq_controller,
475 NULL);
476 if (err)
477 return err;
478
479 irq_set_handler(virq, handle_percpu_irq);
480 break;
481
482 default:
483 err = irq_domain_set_hwirq_and_chip(d, virq, hwirq,
484 &gic_local_irq_controller,
485 NULL);
486 if (err)
487 return err;
488
489 irq_set_handler(virq, handle_percpu_devid_irq);
490 irq_set_percpu_devid(virq);
491 break;
492 }
493
494 return gic_local_irq_domain_map(d, virq, hwirq);
Qais Yousefc98c18222015-12-08 13:20:24 +0000495}
496
Matt Redfearn8ada00a2017-04-20 10:07:36 +0100497static int gic_irq_domain_alloc(struct irq_domain *d, unsigned int virq,
498 unsigned int nr_irqs, void *arg)
499{
500 struct irq_fwspec *fwspec = arg;
501 irq_hw_number_t hwirq;
502
503 if (fwspec->param[0] == GIC_SHARED)
504 hwirq = GIC_SHARED_TO_HWIRQ(fwspec->param[1]);
505 else
506 hwirq = GIC_LOCAL_TO_HWIRQ(fwspec->param[1]);
507
508 return gic_irq_domain_map(d, virq, hwirq);
509}
510
Paul Burtonb87281e2017-04-20 10:07:35 +0100511void gic_irq_domain_free(struct irq_domain *d, unsigned int virq,
Qais Yousefc98c18222015-12-08 13:20:24 +0000512 unsigned int nr_irqs)
513{
Qais Yousefc98c18222015-12-08 13:20:24 +0000514}
515
Paul Burtonb87281e2017-04-20 10:07:35 +0100516static const struct irq_domain_ops gic_irq_domain_ops = {
517 .xlate = gic_irq_domain_xlate,
518 .alloc = gic_irq_domain_alloc,
519 .free = gic_irq_domain_free,
Matt Redfearn8ada00a2017-04-20 10:07:36 +0100520 .map = gic_irq_domain_map,
Qais Yousef2af70a92015-12-08 13:20:23 +0000521};
522
523static int gic_ipi_domain_xlate(struct irq_domain *d, struct device_node *ctrlr,
524 const u32 *intspec, unsigned int intsize,
525 irq_hw_number_t *out_hwirq,
526 unsigned int *out_type)
527{
528 /*
529 * There's nothing to translate here. hwirq is dynamically allocated and
530 * the irq type is always edge triggered.
531 * */
532 *out_hwirq = 0;
533 *out_type = IRQ_TYPE_EDGE_RISING;
534
535 return 0;
536}
537
538static int gic_ipi_domain_alloc(struct irq_domain *d, unsigned int virq,
539 unsigned int nr_irqs, void *arg)
540{
541 struct cpumask *ipimask = arg;
Paul Burtonb87281e2017-04-20 10:07:35 +0100542 irq_hw_number_t hwirq, base_hwirq;
543 int cpu, ret, i;
Qais Yousef2af70a92015-12-08 13:20:23 +0000544
Paul Burtonb87281e2017-04-20 10:07:35 +0100545 base_hwirq = find_first_bit(ipi_available, gic_shared_intrs);
546 if (base_hwirq == gic_shared_intrs)
547 return -ENOMEM;
Qais Yousef2af70a92015-12-08 13:20:23 +0000548
Paul Burtonb87281e2017-04-20 10:07:35 +0100549 /* check that we have enough space */
550 for (i = base_hwirq; i < nr_irqs; i++) {
551 if (!test_bit(i, ipi_available))
552 return -EBUSY;
553 }
554 bitmap_clear(ipi_available, base_hwirq, nr_irqs);
555
556 /* map the hwirq for each cpu consecutively */
557 i = 0;
558 for_each_cpu(cpu, ipimask) {
559 hwirq = GIC_SHARED_TO_HWIRQ(base_hwirq + i);
560
561 ret = irq_domain_set_hwirq_and_chip(d, virq + i, hwirq,
562 &gic_edge_irq_controller,
563 NULL);
564 if (ret)
565 goto error;
566
567 ret = irq_domain_set_hwirq_and_chip(d->parent, virq + i, hwirq,
Qais Yousef2af70a92015-12-08 13:20:23 +0000568 &gic_edge_irq_controller,
569 NULL);
570 if (ret)
571 goto error;
572
573 ret = irq_set_irq_type(virq + i, IRQ_TYPE_EDGE_RISING);
574 if (ret)
575 goto error;
Paul Burtonb87281e2017-04-20 10:07:35 +0100576
577 ret = gic_shared_irq_domain_map(d, virq + i, hwirq, cpu);
578 if (ret)
579 goto error;
580
581 i++;
Qais Yousef2af70a92015-12-08 13:20:23 +0000582 }
583
584 return 0;
585error:
Paul Burtonb87281e2017-04-20 10:07:35 +0100586 bitmap_set(ipi_available, base_hwirq, nr_irqs);
Qais Yousef2af70a92015-12-08 13:20:23 +0000587 return ret;
588}
589
590void gic_ipi_domain_free(struct irq_domain *d, unsigned int virq,
591 unsigned int nr_irqs)
592{
Paul Burtonb87281e2017-04-20 10:07:35 +0100593 irq_hw_number_t base_hwirq;
594 struct irq_data *data;
595
596 data = irq_get_irq_data(virq);
597 if (!data)
598 return;
599
600 base_hwirq = GIC_HWIRQ_TO_SHARED(irqd_to_hwirq(data));
601 bitmap_set(ipi_available, base_hwirq, nr_irqs);
Qais Yousef2af70a92015-12-08 13:20:23 +0000602}
603
604int gic_ipi_domain_match(struct irq_domain *d, struct device_node *node,
605 enum irq_domain_bus_token bus_token)
606{
607 bool is_ipi;
608
609 switch (bus_token) {
610 case DOMAIN_BUS_IPI:
611 is_ipi = d->bus_token == bus_token;
Paul Burton547aefc2016-07-05 14:26:00 +0100612 return (!node || to_of_node(d->fwnode) == node) && is_ipi;
Qais Yousef2af70a92015-12-08 13:20:23 +0000613 break;
614 default:
615 return 0;
616 }
617}
618
Tobias Klauser0b7e8152017-06-02 10:20:56 +0200619static const struct irq_domain_ops gic_ipi_domain_ops = {
Qais Yousef2af70a92015-12-08 13:20:23 +0000620 .xlate = gic_ipi_domain_xlate,
621 .alloc = gic_ipi_domain_alloc,
622 .free = gic_ipi_domain_free,
623 .match = gic_ipi_domain_match,
Andrew Brestickerc49581a2014-09-18 14:47:23 -0700624};
625
Ralf Baechle39b8d522008-04-28 17:14:26 +0100626
Paul Burtonfbea7542017-08-12 21:36:40 -0700627static int __init gic_of_init(struct device_node *node,
628 struct device_node *parent)
629{
Paul Burton87888bc2017-08-12 21:36:41 -0700630 unsigned int cpu_vec, i, j, reserved, gicconfig, cpu, v[2];
Paul Burtonfbea7542017-08-12 21:36:40 -0700631 phys_addr_t gic_base;
632 struct resource res;
633 size_t gic_len;
634
635 /* Find the first available CPU vector. */
636 i = reserved = 0;
637 while (!of_property_read_u32_index(node, "mti,reserved-cpu-vectors",
638 i++, &cpu_vec))
639 reserved |= BIT(cpu_vec);
640 for (cpu_vec = 2; cpu_vec < 8; cpu_vec++) {
641 if (!(reserved & BIT(cpu_vec)))
642 break;
643 }
644 if (cpu_vec == 8) {
645 pr_err("No CPU vectors available for GIC\n");
646 return -ENODEV;
647 }
648
649 if (of_address_to_resource(node, 0, &res)) {
650 /*
651 * Probe the CM for the GIC base address if not specified
652 * in the device-tree.
653 */
654 if (mips_cm_present()) {
655 gic_base = read_gcr_gic_base() &
656 ~CM_GCR_GIC_BASE_GICEN;
657 gic_len = 0x20000;
658 } else {
659 pr_err("Failed to get GIC memory range\n");
660 return -ENODEV;
661 }
662 } else {
663 gic_base = res.start;
664 gic_len = resource_size(&res);
665 }
666
667 if (mips_cm_present()) {
668 write_gcr_gic_base(gic_base | CM_GCR_GIC_BASE_GICEN);
669 /* Ensure GIC region is enabled before trying to access it */
670 __sync();
671 }
672
673 mips_gic_base = ioremap_nocache(gic_base, gic_len);
Ralf Baechle39b8d522008-04-28 17:14:26 +0100674
Paul Burton36807462017-08-12 21:36:24 -0700675 gicconfig = read_gic_config();
676 gic_shared_intrs = gicconfig & GIC_CONFIG_NUMINTERRUPTS;
677 gic_shared_intrs >>= __fls(GIC_CONFIG_NUMINTERRUPTS);
678 gic_shared_intrs = (gic_shared_intrs + 1) * 8;
Ralf Baechle39b8d522008-04-28 17:14:26 +0100679
Paul Burton36807462017-08-12 21:36:24 -0700680 gic_vpes = gicconfig & GIC_CONFIG_PVPS;
681 gic_vpes >>= __fls(GIC_CONFIG_PVPS);
Andrew Brestickere9de6882014-09-18 14:47:27 -0700682 gic_vpes = gic_vpes + 1;
Ralf Baechle39b8d522008-04-28 17:14:26 +0100683
Andrew Bresticker18743d22014-09-18 14:47:24 -0700684 if (cpu_has_veic) {
Paul Burtonba01cf02016-05-17 15:31:06 +0100685 /* Set EIC mode for all VPEs */
686 for_each_present_cpu(cpu) {
Paul Burton0d0cf582017-08-12 21:36:26 -0700687 write_gic_vl_other(mips_cm_vp_id(cpu));
688 write_gic_vo_ctl(GIC_VX_CTL_EIC);
Paul Burtonba01cf02016-05-17 15:31:06 +0100689 }
690
Andrew Bresticker18743d22014-09-18 14:47:24 -0700691 /* Always use vector 1 in EIC mode */
692 gic_cpu_pin = 0;
James Hogan1b6af712015-01-19 15:38:24 +0000693 timer_cpu_pin = gic_cpu_pin;
Andrew Bresticker18743d22014-09-18 14:47:24 -0700694 set_vi_handler(gic_cpu_pin + GIC_PIN_TO_VEC_OFFSET,
695 __gic_irq_dispatch);
696 } else {
697 gic_cpu_pin = cpu_vec - GIC_CPU_PIN_OFFSET;
698 irq_set_chained_handler(MIPS_CPU_IRQ_BASE + cpu_vec,
699 gic_irq_dispatch);
James Hogan1b6af712015-01-19 15:38:24 +0000700 /*
701 * With the CMP implementation of SMP (deprecated), other CPUs
702 * are started by the bootloader and put into a timer based
703 * waiting poll loop. We must not re-route those CPU's local
704 * timer interrupts as the wait instruction will never finish,
705 * so just handle whatever CPU interrupt it is routed to by
706 * default.
707 *
708 * This workaround should be removed when CMP support is
709 * dropped.
710 */
711 if (IS_ENABLED(CONFIG_MIPS_CMP) &&
712 gic_local_irq_is_routable(GIC_LOCAL_INT_TIMER)) {
Paul Burton0d0cf582017-08-12 21:36:26 -0700713 timer_cpu_pin = read_gic_vl_timer_map() & GIC_MAP_PIN_MAP;
James Hogan1b6af712015-01-19 15:38:24 +0000714 irq_set_chained_handler(MIPS_CPU_IRQ_BASE +
715 GIC_CPU_PIN_OFFSET +
716 timer_cpu_pin,
717 gic_irq_dispatch);
718 } else {
719 timer_cpu_pin = gic_cpu_pin;
720 }
Andrew Bresticker18743d22014-09-18 14:47:24 -0700721 }
722
Andrew Brestickera7057272014-11-12 11:43:38 -0800723 gic_irq_domain = irq_domain_add_simple(node, GIC_NUM_LOCAL_INTRS +
Paul Burtonfbea7542017-08-12 21:36:40 -0700724 gic_shared_intrs, 0,
Andrew Brestickerc49581a2014-09-18 14:47:23 -0700725 &gic_irq_domain_ops, NULL);
Paul Burtonfbea7542017-08-12 21:36:40 -0700726 if (!gic_irq_domain) {
727 pr_err("Failed to add GIC IRQ domain");
728 return -ENXIO;
729 }
Steven J. Hill0b271f52012-08-31 16:05:37 -0500730
Qais Yousef2af70a92015-12-08 13:20:23 +0000731 gic_ipi_domain = irq_domain_add_hierarchy(gic_irq_domain,
732 IRQ_DOMAIN_FLAG_IPI_PER_CPU,
733 GIC_NUM_LOCAL_INTRS + gic_shared_intrs,
734 node, &gic_ipi_domain_ops, NULL);
Paul Burtonfbea7542017-08-12 21:36:40 -0700735 if (!gic_ipi_domain) {
736 pr_err("Failed to add GIC IPI domain");
737 return -ENXIO;
738 }
Qais Yousef2af70a92015-12-08 13:20:23 +0000739
Marc Zyngier96f0d932017-06-22 11:42:50 +0100740 irq_domain_update_bus_token(gic_ipi_domain, DOMAIN_BUS_IPI);
Qais Yousef2af70a92015-12-08 13:20:23 +0000741
Qais Yousef16a80832015-12-08 13:20:30 +0000742 if (node &&
743 !of_property_read_u32_array(node, "mti,reserved-ipi-vectors", v, 2)) {
744 bitmap_set(ipi_resrv, v[0], v[1]);
745 } else {
746 /* Make the last 2 * gic_vpes available for IPIs */
747 bitmap_set(ipi_resrv,
748 gic_shared_intrs - 2 * gic_vpes,
749 2 * gic_vpes);
750 }
Qais Yousef2af70a92015-12-08 13:20:23 +0000751
Paul Burtonf8dcd9e2017-04-20 10:07:34 +0100752 bitmap_copy(ipi_available, ipi_resrv, GIC_MAX_INTRS);
Paul Burton87888bc2017-08-12 21:36:41 -0700753
754 board_bind_eic_interrupt = &gic_bind_eic_interrupt;
755
756 /* Setup defaults */
757 for (i = 0; i < gic_shared_intrs; i++) {
758 change_gic_pol(i, GIC_POL_ACTIVE_HIGH);
759 change_gic_trig(i, GIC_TRIG_LEVEL);
760 write_gic_rmask(BIT(i));
761 }
762
763 for (i = 0; i < gic_vpes; i++) {
764 write_gic_vl_other(mips_cm_vp_id(i));
765 for (j = 0; j < GIC_NUM_LOCAL_INTRS; j++) {
766 if (!gic_local_irq_is_routable(j))
767 continue;
768 write_gic_vo_rmask(BIT(j));
769 }
770 }
Andrew Brestickera7057272014-11-12 11:43:38 -0800771
772 return 0;
773}
774IRQCHIP_DECLARE(mips_gic, "mti,gic", gic_of_init);