blob: 58fb876d88d6239ea562190d3ff6f8226b116d57 [file] [log] [blame]
Steven J. Hill2299c492012-08-31 16:13:07 -05001/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 2008 Ralf Baechle (ralf@linux-mips.org)
7 * Copyright (C) 2012 MIPS Technologies, Inc. All rights reserved.
8 */
Ralf Baechle39b8d522008-04-28 17:14:26 +01009#include <linux/bitmap.h>
Andrew Brestickerfb8f7be12014-10-20 12:03:55 -070010#include <linux/clocksource.h>
Ralf Baechle39b8d522008-04-28 17:14:26 +010011#include <linux/init.h>
Andrew Bresticker18743d22014-09-18 14:47:24 -070012#include <linux/interrupt.h>
Andrew Brestickerfb8f7be12014-10-20 12:03:55 -070013#include <linux/irq.h>
Joel Porquet41a83e062015-07-07 17:11:46 -040014#include <linux/irqchip.h>
Andrew Brestickera7057272014-11-12 11:43:38 -080015#include <linux/of_address.h>
Andrew Bresticker18743d22014-09-18 14:47:24 -070016#include <linux/sched.h>
Ralf Baechle631330f2009-06-19 14:05:26 +010017#include <linux/smp.h>
Ralf Baechle39b8d522008-04-28 17:14:26 +010018
Paul Burtone83f7e02017-08-12 19:49:41 -070019#include <asm/mips-cps.h>
Steven J. Hill98b67c32012-08-31 16:18:49 -050020#include <asm/setup.h>
21#include <asm/traps.h>
Ralf Baechle39b8d522008-04-28 17:14:26 +010022
Andrew Brestickera7057272014-11-12 11:43:38 -080023#include <dt-bindings/interrupt-controller/mips-gic.h>
24
Paul Burtonb11d4c12017-08-12 21:36:29 -070025#define GIC_MAX_INTRS 256
26
27/* Add 2 to convert GIC CPU pin to core interrupt */
28#define GIC_CPU_PIN_OFFSET 2
29
30/* Mapped interrupt to pin X, then GIC will generate the vector (X+1). */
31#define GIC_PIN_TO_VEC_OFFSET 1
32
33/* Convert between local/shared IRQ number and GIC HW IRQ number. */
34#define GIC_LOCAL_HWIRQ_BASE 0
35#define GIC_LOCAL_TO_HWIRQ(x) (GIC_LOCAL_HWIRQ_BASE + (x))
36#define GIC_HWIRQ_TO_LOCAL(x) ((x) - GIC_LOCAL_HWIRQ_BASE)
37#define GIC_SHARED_HWIRQ_BASE GIC_NUM_LOCAL_INTRS
38#define GIC_SHARED_TO_HWIRQ(x) (GIC_SHARED_HWIRQ_BASE + (x))
39#define GIC_HWIRQ_TO_SHARED(x) ((x) - GIC_SHARED_HWIRQ_BASE)
40
Paul Burton582e2b42017-08-12 21:36:10 -070041void __iomem *mips_gic_base;
Steven J. Hill98b67c32012-08-31 16:18:49 -050042
Jeffrey Deans822350b2014-07-17 09:20:53 +010043struct gic_pcpu_mask {
Andrew Brestickerfbd55242014-09-18 14:47:25 -070044 DECLARE_BITMAP(pcpu_mask, GIC_MAX_INTRS);
Jeffrey Deans822350b2014-07-17 09:20:53 +010045};
46
Steven J. Hill0b271f52012-08-31 16:05:37 -050047static struct gic_pcpu_mask pcpu_masks[NR_CPUS];
Andrew Bresticker95150ae2014-09-18 14:47:21 -070048static DEFINE_SPINLOCK(gic_lock);
Andrew Brestickerc49581a2014-09-18 14:47:23 -070049static struct irq_domain *gic_irq_domain;
Qais Yousef2af70a92015-12-08 13:20:23 +000050static struct irq_domain *gic_ipi_domain;
Andrew Brestickerfbd55242014-09-18 14:47:25 -070051static int gic_shared_intrs;
Andrew Brestickere9de6882014-09-18 14:47:27 -070052static int gic_vpes;
Andrew Bresticker3263d082014-09-18 14:47:28 -070053static unsigned int gic_cpu_pin;
James Hogan1b6af712015-01-19 15:38:24 +000054static unsigned int timer_cpu_pin;
Andrew Bresticker4a6a3ea32014-09-18 14:47:26 -070055static struct irq_chip gic_level_irq_controller, gic_edge_irq_controller;
Qais Yousef2af70a92015-12-08 13:20:23 +000056DECLARE_BITMAP(ipi_resrv, GIC_MAX_INTRS);
Paul Burtonf8dcd9e2017-04-20 10:07:34 +010057DECLARE_BITMAP(ipi_available, GIC_MAX_INTRS);
Ralf Baechle39b8d522008-04-28 17:14:26 +010058
Andrew Brestickere9de6882014-09-18 14:47:27 -070059static bool gic_local_irq_is_routable(int intr)
60{
61 u32 vpe_ctl;
62
63 /* All local interrupts are routable in EIC mode. */
64 if (cpu_has_veic)
65 return true;
66
Paul Burton0d0cf582017-08-12 21:36:26 -070067 vpe_ctl = read_gic_vl_ctl();
Andrew Brestickere9de6882014-09-18 14:47:27 -070068 switch (intr) {
69 case GIC_LOCAL_INT_TIMER:
Paul Burton0d0cf582017-08-12 21:36:26 -070070 return vpe_ctl & GIC_VX_CTL_TIMER_ROUTABLE;
Andrew Brestickere9de6882014-09-18 14:47:27 -070071 case GIC_LOCAL_INT_PERFCTR:
Paul Burton0d0cf582017-08-12 21:36:26 -070072 return vpe_ctl & GIC_VX_CTL_PERFCNT_ROUTABLE;
Andrew Brestickere9de6882014-09-18 14:47:27 -070073 case GIC_LOCAL_INT_FDC:
Paul Burton0d0cf582017-08-12 21:36:26 -070074 return vpe_ctl & GIC_VX_CTL_FDC_ROUTABLE;
Andrew Brestickere9de6882014-09-18 14:47:27 -070075 case GIC_LOCAL_INT_SWINT0:
76 case GIC_LOCAL_INT_SWINT1:
Paul Burton0d0cf582017-08-12 21:36:26 -070077 return vpe_ctl & GIC_VX_CTL_SWINT_ROUTABLE;
Andrew Brestickere9de6882014-09-18 14:47:27 -070078 default:
79 return true;
80 }
81}
82
Andrew Bresticker3263d082014-09-18 14:47:28 -070083static void gic_bind_eic_interrupt(int irq, int set)
Steven J. Hill98b67c32012-08-31 16:18:49 -050084{
85 /* Convert irq vector # to hw int # */
86 irq -= GIC_PIN_TO_VEC_OFFSET;
87
88 /* Set irq to use shadow set */
Paul Burton0d0cf582017-08-12 21:36:26 -070089 write_gic_vl_eic_shadow_set(irq, set);
Steven J. Hill98b67c32012-08-31 16:18:49 -050090}
91
Qais Yousefbb11cff2015-12-08 13:20:28 +000092static void gic_send_ipi(struct irq_data *d, unsigned int cpu)
Ralf Baechle39b8d522008-04-28 17:14:26 +010093{
Qais Yousefbb11cff2015-12-08 13:20:28 +000094 irq_hw_number_t hwirq = GIC_HWIRQ_TO_SHARED(irqd_to_hwirq(d));
95
Paul Burton36807462017-08-12 21:36:24 -070096 write_gic_wedge(GIC_WEDGE_RW | hwirq);
Ralf Baechle39b8d522008-04-28 17:14:26 +010097}
98
Andrew Brestickere9de6882014-09-18 14:47:27 -070099int gic_get_c0_compare_int(void)
100{
101 if (!gic_local_irq_is_routable(GIC_LOCAL_INT_TIMER))
102 return MIPS_CPU_IRQ_BASE + cp0_compare_irq;
103 return irq_create_mapping(gic_irq_domain,
104 GIC_LOCAL_TO_HWIRQ(GIC_LOCAL_INT_TIMER));
105}
106
107int gic_get_c0_perfcount_int(void)
108{
109 if (!gic_local_irq_is_routable(GIC_LOCAL_INT_PERFCTR)) {
James Hogan7e3e6cb2015-01-27 21:45:50 +0000110 /* Is the performance counter shared with the timer? */
Andrew Brestickere9de6882014-09-18 14:47:27 -0700111 if (cp0_perfcount_irq < 0)
112 return -1;
113 return MIPS_CPU_IRQ_BASE + cp0_perfcount_irq;
114 }
115 return irq_create_mapping(gic_irq_domain,
116 GIC_LOCAL_TO_HWIRQ(GIC_LOCAL_INT_PERFCTR));
117}
118
James Hogan6429e2b2015-01-29 11:14:09 +0000119int gic_get_c0_fdc_int(void)
120{
121 if (!gic_local_irq_is_routable(GIC_LOCAL_INT_FDC)) {
122 /* Is the FDC IRQ even present? */
123 if (cp0_fdc_irq < 0)
124 return -1;
125 return MIPS_CPU_IRQ_BASE + cp0_fdc_irq;
126 }
127
James Hogan6429e2b2015-01-29 11:14:09 +0000128 return irq_create_mapping(gic_irq_domain,
129 GIC_LOCAL_TO_HWIRQ(GIC_LOCAL_INT_FDC));
130}
131
Rabin Vincent1b3ed362015-06-12 10:01:56 +0200132static void gic_handle_shared_int(bool chained)
Ralf Baechle39b8d522008-04-28 17:14:26 +0100133{
Paul Burtone98fcb22017-08-12 21:36:16 -0700134 unsigned int intr, virq;
Andrew Bresticker8f5ee792014-10-20 12:03:56 -0700135 unsigned long *pcpu_mask;
Andrew Bresticker8f5ee792014-10-20 12:03:56 -0700136 DECLARE_BITMAP(pending, GIC_MAX_INTRS);
137 DECLARE_BITMAP(intrmask, GIC_MAX_INTRS);
Ralf Baechle39b8d522008-04-28 17:14:26 +0100138
139 /* Get per-cpu bitmaps */
Ralf Baechle39b8d522008-04-28 17:14:26 +0100140 pcpu_mask = pcpu_masks[smp_processor_id()].pcpu_mask;
141
Paul Burtone98fcb22017-08-12 21:36:16 -0700142 if (mips_cm_is64) {
143 __ioread64_copy(pending, addr_gic_pend(),
144 DIV_ROUND_UP(gic_shared_intrs, 64));
145 __ioread64_copy(intrmask, addr_gic_mask(),
146 DIV_ROUND_UP(gic_shared_intrs, 64));
147 } else {
148 __ioread32_copy(pending, addr_gic_pend(),
149 DIV_ROUND_UP(gic_shared_intrs, 32));
150 __ioread32_copy(intrmask, addr_gic_mask(),
151 DIV_ROUND_UP(gic_shared_intrs, 32));
Ralf Baechle39b8d522008-04-28 17:14:26 +0100152 }
153
Andrew Brestickerfbd55242014-09-18 14:47:25 -0700154 bitmap_and(pending, pending, intrmask, gic_shared_intrs);
155 bitmap_and(pending, pending, pcpu_mask, gic_shared_intrs);
Ralf Baechle39b8d522008-04-28 17:14:26 +0100156
Paul Burtoncae750b2016-08-19 18:11:19 +0100157 for_each_set_bit(intr, pending, gic_shared_intrs) {
Qais Yousefd7eb4f22015-01-19 11:51:29 +0000158 virq = irq_linear_revmap(gic_irq_domain,
159 GIC_SHARED_TO_HWIRQ(intr));
Rabin Vincent1b3ed362015-06-12 10:01:56 +0200160 if (chained)
161 generic_handle_irq(virq);
162 else
163 do_IRQ(virq);
Qais Yousefd7eb4f22015-01-19 11:51:29 +0000164 }
Ralf Baechle39b8d522008-04-28 17:14:26 +0100165}
166
Thomas Gleixner161d0492011-03-23 21:08:58 +0000167static void gic_mask_irq(struct irq_data *d)
Ralf Baechle39b8d522008-04-28 17:14:26 +0100168{
Paul Burton87554b02017-08-12 21:36:18 -0700169 write_gic_rmask(BIT(GIC_HWIRQ_TO_SHARED(d->hwirq)));
Ralf Baechle39b8d522008-04-28 17:14:26 +0100170}
171
Thomas Gleixner161d0492011-03-23 21:08:58 +0000172static void gic_unmask_irq(struct irq_data *d)
Ralf Baechle39b8d522008-04-28 17:14:26 +0100173{
Paul Burton87554b02017-08-12 21:36:18 -0700174 write_gic_smask(BIT(GIC_HWIRQ_TO_SHARED(d->hwirq)));
Ralf Baechle39b8d522008-04-28 17:14:26 +0100175}
176
Andrew Bresticker5561c9e2014-09-18 14:47:20 -0700177static void gic_ack_irq(struct irq_data *d)
178{
Andrew Brestickere9de6882014-09-18 14:47:27 -0700179 unsigned int irq = GIC_HWIRQ_TO_SHARED(d->hwirq);
Andrew Brestickerc49581a2014-09-18 14:47:23 -0700180
Paul Burton36807462017-08-12 21:36:24 -0700181 write_gic_wedge(irq);
Andrew Bresticker5561c9e2014-09-18 14:47:20 -0700182}
183
Andrew Bresticker95150ae2014-09-18 14:47:21 -0700184static int gic_set_type(struct irq_data *d, unsigned int type)
185{
Andrew Brestickere9de6882014-09-18 14:47:27 -0700186 unsigned int irq = GIC_HWIRQ_TO_SHARED(d->hwirq);
Andrew Bresticker95150ae2014-09-18 14:47:21 -0700187 unsigned long flags;
188 bool is_edge;
Ralf Baechle39b8d522008-04-28 17:14:26 +0100189
Andrew Bresticker95150ae2014-09-18 14:47:21 -0700190 spin_lock_irqsave(&gic_lock, flags);
191 switch (type & IRQ_TYPE_SENSE_MASK) {
192 case IRQ_TYPE_EDGE_FALLING:
Paul Burton80e5f9c2017-08-12 21:36:19 -0700193 change_gic_pol(irq, GIC_POL_FALLING_EDGE);
Paul Burton471aa962017-08-12 21:36:20 -0700194 change_gic_trig(irq, GIC_TRIG_EDGE);
Paul Burtonc26ba672017-08-12 21:36:21 -0700195 change_gic_dual(irq, GIC_DUAL_SINGLE);
Andrew Bresticker95150ae2014-09-18 14:47:21 -0700196 is_edge = true;
197 break;
198 case IRQ_TYPE_EDGE_RISING:
Paul Burton80e5f9c2017-08-12 21:36:19 -0700199 change_gic_pol(irq, GIC_POL_RISING_EDGE);
Paul Burton471aa962017-08-12 21:36:20 -0700200 change_gic_trig(irq, GIC_TRIG_EDGE);
Paul Burtonc26ba672017-08-12 21:36:21 -0700201 change_gic_dual(irq, GIC_DUAL_SINGLE);
Andrew Bresticker95150ae2014-09-18 14:47:21 -0700202 is_edge = true;
203 break;
204 case IRQ_TYPE_EDGE_BOTH:
205 /* polarity is irrelevant in this case */
Paul Burton471aa962017-08-12 21:36:20 -0700206 change_gic_trig(irq, GIC_TRIG_EDGE);
Paul Burtonc26ba672017-08-12 21:36:21 -0700207 change_gic_dual(irq, GIC_DUAL_DUAL);
Andrew Bresticker95150ae2014-09-18 14:47:21 -0700208 is_edge = true;
209 break;
210 case IRQ_TYPE_LEVEL_LOW:
Paul Burton80e5f9c2017-08-12 21:36:19 -0700211 change_gic_pol(irq, GIC_POL_ACTIVE_LOW);
Paul Burton471aa962017-08-12 21:36:20 -0700212 change_gic_trig(irq, GIC_TRIG_LEVEL);
Paul Burtonc26ba672017-08-12 21:36:21 -0700213 change_gic_dual(irq, GIC_DUAL_SINGLE);
Andrew Bresticker95150ae2014-09-18 14:47:21 -0700214 is_edge = false;
215 break;
216 case IRQ_TYPE_LEVEL_HIGH:
217 default:
Paul Burton80e5f9c2017-08-12 21:36:19 -0700218 change_gic_pol(irq, GIC_POL_ACTIVE_HIGH);
Paul Burton471aa962017-08-12 21:36:20 -0700219 change_gic_trig(irq, GIC_TRIG_LEVEL);
Paul Burtonc26ba672017-08-12 21:36:21 -0700220 change_gic_dual(irq, GIC_DUAL_SINGLE);
Andrew Bresticker95150ae2014-09-18 14:47:21 -0700221 is_edge = false;
222 break;
223 }
224
Thomas Gleixnera595fc52015-06-23 14:41:25 +0200225 if (is_edge)
226 irq_set_chip_handler_name_locked(d, &gic_edge_irq_controller,
227 handle_edge_irq, NULL);
228 else
229 irq_set_chip_handler_name_locked(d, &gic_level_irq_controller,
230 handle_level_irq, NULL);
Andrew Bresticker95150ae2014-09-18 14:47:21 -0700231 spin_unlock_irqrestore(&gic_lock, flags);
232
233 return 0;
234}
235
236#ifdef CONFIG_SMP
Thomas Gleixner161d0492011-03-23 21:08:58 +0000237static int gic_set_affinity(struct irq_data *d, const struct cpumask *cpumask,
238 bool force)
Ralf Baechle39b8d522008-04-28 17:14:26 +0100239{
Andrew Brestickere9de6882014-09-18 14:47:27 -0700240 unsigned int irq = GIC_HWIRQ_TO_SHARED(d->hwirq);
Ralf Baechle39b8d522008-04-28 17:14:26 +0100241 cpumask_t tmp = CPU_MASK_NONE;
242 unsigned long flags;
243 int i;
244
Rusty Russell0de26522008-12-13 21:20:26 +1030245 cpumask_and(&tmp, cpumask, cpu_online_mask);
Rusty Russellf9b531f2015-03-05 10:49:16 +1030246 if (cpumask_empty(&tmp))
Andrew Bresticker14d160a2014-09-18 14:47:22 -0700247 return -EINVAL;
Ralf Baechle39b8d522008-04-28 17:14:26 +0100248
249 /* Assumption : cpumask refers to a single CPU */
250 spin_lock_irqsave(&gic_lock, flags);
Ralf Baechle39b8d522008-04-28 17:14:26 +0100251
Tony Wuc214c032013-06-21 10:13:08 +0000252 /* Re-route this IRQ */
Paul Burton0efe3cb2017-08-12 21:36:23 -0700253 write_gic_map_vp(irq, BIT(mips_cm_vp_id(cpumask_first(&tmp))));
Ralf Baechle39b8d522008-04-28 17:14:26 +0100254
Tony Wuc214c032013-06-21 10:13:08 +0000255 /* Update the pcpu_masks */
Paul Burton91951f92016-04-21 11:31:54 +0100256 for (i = 0; i < min(gic_vpes, NR_CPUS); i++)
Tony Wuc214c032013-06-21 10:13:08 +0000257 clear_bit(irq, pcpu_masks[i].pcpu_mask);
Rusty Russellf9b531f2015-03-05 10:49:16 +1030258 set_bit(irq, pcpu_masks[cpumask_first(&tmp)].pcpu_mask);
Tony Wuc214c032013-06-21 10:13:08 +0000259
Jiang Liu72f86db2015-06-01 16:05:38 +0800260 cpumask_copy(irq_data_get_affinity_mask(d), cpumask);
Ralf Baechle39b8d522008-04-28 17:14:26 +0100261 spin_unlock_irqrestore(&gic_lock, flags);
262
Thomas Gleixner161d0492011-03-23 21:08:58 +0000263 return IRQ_SET_MASK_OK_NOCOPY;
Ralf Baechle39b8d522008-04-28 17:14:26 +0100264}
265#endif
266
Andrew Bresticker4a6a3ea32014-09-18 14:47:26 -0700267static struct irq_chip gic_level_irq_controller = {
268 .name = "MIPS GIC",
269 .irq_mask = gic_mask_irq,
270 .irq_unmask = gic_unmask_irq,
271 .irq_set_type = gic_set_type,
272#ifdef CONFIG_SMP
273 .irq_set_affinity = gic_set_affinity,
274#endif
275};
276
277static struct irq_chip gic_edge_irq_controller = {
Thomas Gleixner161d0492011-03-23 21:08:58 +0000278 .name = "MIPS GIC",
Andrew Bresticker5561c9e2014-09-18 14:47:20 -0700279 .irq_ack = gic_ack_irq,
Thomas Gleixner161d0492011-03-23 21:08:58 +0000280 .irq_mask = gic_mask_irq,
Thomas Gleixner161d0492011-03-23 21:08:58 +0000281 .irq_unmask = gic_unmask_irq,
Andrew Bresticker95150ae2014-09-18 14:47:21 -0700282 .irq_set_type = gic_set_type,
Ralf Baechle39b8d522008-04-28 17:14:26 +0100283#ifdef CONFIG_SMP
Thomas Gleixner161d0492011-03-23 21:08:58 +0000284 .irq_set_affinity = gic_set_affinity,
Ralf Baechle39b8d522008-04-28 17:14:26 +0100285#endif
Qais Yousefbb11cff2015-12-08 13:20:28 +0000286 .ipi_send_single = gic_send_ipi,
Ralf Baechle39b8d522008-04-28 17:14:26 +0100287};
288
Rabin Vincent1b3ed362015-06-12 10:01:56 +0200289static void gic_handle_local_int(bool chained)
Andrew Brestickere9de6882014-09-18 14:47:27 -0700290{
291 unsigned long pending, masked;
Qais Yousefd7eb4f22015-01-19 11:51:29 +0000292 unsigned int intr, virq;
Andrew Brestickere9de6882014-09-18 14:47:27 -0700293
Paul Burton9da3c642017-08-12 21:36:25 -0700294 pending = read_gic_vl_pend();
295 masked = read_gic_vl_mask();
Andrew Brestickere9de6882014-09-18 14:47:27 -0700296
297 bitmap_and(&pending, &pending, &masked, GIC_NUM_LOCAL_INTRS);
298
Paul Burton0f4ed152016-09-13 17:54:27 +0100299 for_each_set_bit(intr, &pending, GIC_NUM_LOCAL_INTRS) {
Qais Yousefd7eb4f22015-01-19 11:51:29 +0000300 virq = irq_linear_revmap(gic_irq_domain,
301 GIC_LOCAL_TO_HWIRQ(intr));
Rabin Vincent1b3ed362015-06-12 10:01:56 +0200302 if (chained)
303 generic_handle_irq(virq);
304 else
305 do_IRQ(virq);
Qais Yousefd7eb4f22015-01-19 11:51:29 +0000306 }
Andrew Brestickere9de6882014-09-18 14:47:27 -0700307}
308
309static void gic_mask_local_irq(struct irq_data *d)
310{
311 int intr = GIC_HWIRQ_TO_LOCAL(d->hwirq);
312
Paul Burton9da3c642017-08-12 21:36:25 -0700313 write_gic_vl_rmask(BIT(intr));
Andrew Brestickere9de6882014-09-18 14:47:27 -0700314}
315
316static void gic_unmask_local_irq(struct irq_data *d)
317{
318 int intr = GIC_HWIRQ_TO_LOCAL(d->hwirq);
319
Paul Burton9da3c642017-08-12 21:36:25 -0700320 write_gic_vl_smask(BIT(intr));
Andrew Brestickere9de6882014-09-18 14:47:27 -0700321}
322
323static struct irq_chip gic_local_irq_controller = {
324 .name = "MIPS GIC Local",
325 .irq_mask = gic_mask_local_irq,
326 .irq_unmask = gic_unmask_local_irq,
327};
328
329static void gic_mask_local_irq_all_vpes(struct irq_data *d)
330{
331 int intr = GIC_HWIRQ_TO_LOCAL(d->hwirq);
332 int i;
333 unsigned long flags;
334
335 spin_lock_irqsave(&gic_lock, flags);
336 for (i = 0; i < gic_vpes; i++) {
Paul Burton0d0cf582017-08-12 21:36:26 -0700337 write_gic_vl_other(mips_cm_vp_id(i));
Paul Burton9da3c642017-08-12 21:36:25 -0700338 write_gic_vo_rmask(BIT(intr));
Andrew Brestickere9de6882014-09-18 14:47:27 -0700339 }
340 spin_unlock_irqrestore(&gic_lock, flags);
341}
342
343static void gic_unmask_local_irq_all_vpes(struct irq_data *d)
344{
345 int intr = GIC_HWIRQ_TO_LOCAL(d->hwirq);
346 int i;
347 unsigned long flags;
348
349 spin_lock_irqsave(&gic_lock, flags);
350 for (i = 0; i < gic_vpes; i++) {
Paul Burton0d0cf582017-08-12 21:36:26 -0700351 write_gic_vl_other(mips_cm_vp_id(i));
Paul Burton9da3c642017-08-12 21:36:25 -0700352 write_gic_vo_smask(BIT(intr));
Andrew Brestickere9de6882014-09-18 14:47:27 -0700353 }
354 spin_unlock_irqrestore(&gic_lock, flags);
355}
356
357static struct irq_chip gic_all_vpes_local_irq_controller = {
358 .name = "MIPS GIC Local",
359 .irq_mask = gic_mask_local_irq_all_vpes,
360 .irq_unmask = gic_unmask_local_irq_all_vpes,
361};
362
Andrew Bresticker18743d22014-09-18 14:47:24 -0700363static void __gic_irq_dispatch(void)
Ralf Baechle39b8d522008-04-28 17:14:26 +0100364{
Rabin Vincent1b3ed362015-06-12 10:01:56 +0200365 gic_handle_local_int(false);
366 gic_handle_shared_int(false);
Andrew Bresticker18743d22014-09-18 14:47:24 -0700367}
368
Thomas Gleixnerbd0b9ac2015-09-14 10:42:37 +0200369static void gic_irq_dispatch(struct irq_desc *desc)
Andrew Bresticker18743d22014-09-18 14:47:24 -0700370{
Rabin Vincent1b3ed362015-06-12 10:01:56 +0200371 gic_handle_local_int(true);
372 gic_handle_shared_int(true);
Andrew Bresticker18743d22014-09-18 14:47:24 -0700373}
374
Andrew Brestickere9de6882014-09-18 14:47:27 -0700375static void __init gic_basic_init(void)
Andrew Bresticker18743d22014-09-18 14:47:24 -0700376{
377 unsigned int i;
Steven J. Hill98b67c32012-08-31 16:18:49 -0500378
379 board_bind_eic_interrupt = &gic_bind_eic_interrupt;
Ralf Baechle39b8d522008-04-28 17:14:26 +0100380
381 /* Setup defaults */
Andrew Brestickerfbd55242014-09-18 14:47:25 -0700382 for (i = 0; i < gic_shared_intrs; i++) {
Paul Burton80e5f9c2017-08-12 21:36:19 -0700383 change_gic_pol(i, GIC_POL_ACTIVE_HIGH);
Paul Burton471aa962017-08-12 21:36:20 -0700384 change_gic_trig(i, GIC_TRIG_LEVEL);
Paul Burton87554b02017-08-12 21:36:18 -0700385 write_gic_rmask(BIT(i));
Ralf Baechle39b8d522008-04-28 17:14:26 +0100386 }
387
Andrew Brestickere9de6882014-09-18 14:47:27 -0700388 for (i = 0; i < gic_vpes; i++) {
389 unsigned int j;
390
Paul Burton0d0cf582017-08-12 21:36:26 -0700391 write_gic_vl_other(mips_cm_vp_id(i));
Andrew Brestickere9de6882014-09-18 14:47:27 -0700392 for (j = 0; j < GIC_NUM_LOCAL_INTRS; j++) {
393 if (!gic_local_irq_is_routable(j))
394 continue;
Paul Burton9da3c642017-08-12 21:36:25 -0700395 write_gic_vo_rmask(BIT(j));
Andrew Brestickere9de6882014-09-18 14:47:27 -0700396 }
397 }
Ralf Baechle39b8d522008-04-28 17:14:26 +0100398}
399
Andrew Brestickere9de6882014-09-18 14:47:27 -0700400static int gic_local_irq_domain_map(struct irq_domain *d, unsigned int virq,
401 irq_hw_number_t hw)
Andrew Brestickerc49581a2014-09-18 14:47:23 -0700402{
Andrew Brestickere9de6882014-09-18 14:47:27 -0700403 int intr = GIC_HWIRQ_TO_LOCAL(hw);
Andrew Brestickere9de6882014-09-18 14:47:27 -0700404 int i;
405 unsigned long flags;
Paul Burtona0dc5cb2017-08-12 21:36:17 -0700406 u32 val;
Andrew Brestickere9de6882014-09-18 14:47:27 -0700407
408 if (!gic_local_irq_is_routable(intr))
409 return -EPERM;
410
Paul Burtona0dc5cb2017-08-12 21:36:17 -0700411 if (intr > GIC_LOCAL_INT_FDC) {
412 pr_err("Invalid local IRQ %d\n", intr);
413 return -EINVAL;
414 }
415
416 if (intr == GIC_LOCAL_INT_TIMER) {
417 /* CONFIG_MIPS_CMP workaround (see __gic_init) */
418 val = GIC_MAP_PIN_MAP_TO_PIN | timer_cpu_pin;
419 } else {
420 val = GIC_MAP_PIN_MAP_TO_PIN | gic_cpu_pin;
421 }
422
Andrew Brestickere9de6882014-09-18 14:47:27 -0700423 spin_lock_irqsave(&gic_lock, flags);
424 for (i = 0; i < gic_vpes; i++) {
Paul Burtona0dc5cb2017-08-12 21:36:17 -0700425 write_gic_vl_other(mips_cm_vp_id(i));
426 write_gic_vo_map(intr, val);
Andrew Brestickere9de6882014-09-18 14:47:27 -0700427 }
428 spin_unlock_irqrestore(&gic_lock, flags);
429
Paul Burtona0dc5cb2017-08-12 21:36:17 -0700430 return 0;
Andrew Brestickere9de6882014-09-18 14:47:27 -0700431}
432
433static int gic_shared_irq_domain_map(struct irq_domain *d, unsigned int virq,
Qais Yousef2af70a92015-12-08 13:20:23 +0000434 irq_hw_number_t hw, unsigned int vpe)
Andrew Brestickere9de6882014-09-18 14:47:27 -0700435{
436 int intr = GIC_HWIRQ_TO_SHARED(hw);
Andrew Brestickerc49581a2014-09-18 14:47:23 -0700437 unsigned long flags;
Qais Yousef78930f02015-12-08 13:20:26 +0000438 int i;
Andrew Brestickerc49581a2014-09-18 14:47:23 -0700439
Andrew Brestickerc49581a2014-09-18 14:47:23 -0700440 spin_lock_irqsave(&gic_lock, flags);
Paul Burtond3e8cf42017-08-12 21:36:22 -0700441 write_gic_map_pin(intr, GIC_MAP_PIN_MAP_TO_PIN | gic_cpu_pin);
Paul Burton0efe3cb2017-08-12 21:36:23 -0700442 write_gic_map_vp(intr, BIT(mips_cm_vp_id(vpe)));
Paul Burton91951f92016-04-21 11:31:54 +0100443 for (i = 0; i < min(gic_vpes, NR_CPUS); i++)
Qais Yousef78930f02015-12-08 13:20:26 +0000444 clear_bit(intr, pcpu_masks[i].pcpu_mask);
Qais Yousef2af70a92015-12-08 13:20:23 +0000445 set_bit(intr, pcpu_masks[vpe].pcpu_mask);
Andrew Brestickerc49581a2014-09-18 14:47:23 -0700446 spin_unlock_irqrestore(&gic_lock, flags);
447
448 return 0;
449}
450
Paul Burtonb87281e2017-04-20 10:07:35 +0100451static int gic_irq_domain_xlate(struct irq_domain *d, struct device_node *ctrlr,
Qais Yousefc98c18222015-12-08 13:20:24 +0000452 const u32 *intspec, unsigned int intsize,
453 irq_hw_number_t *out_hwirq,
454 unsigned int *out_type)
455{
456 if (intsize != 3)
457 return -EINVAL;
458
459 if (intspec[0] == GIC_SHARED)
460 *out_hwirq = GIC_SHARED_TO_HWIRQ(intspec[1]);
461 else if (intspec[0] == GIC_LOCAL)
462 *out_hwirq = GIC_LOCAL_TO_HWIRQ(intspec[1]);
463 else
464 return -EINVAL;
465 *out_type = intspec[2] & IRQ_TYPE_SENSE_MASK;
466
467 return 0;
468}
469
Matt Redfearn8ada00a2017-04-20 10:07:36 +0100470static int gic_irq_domain_map(struct irq_domain *d, unsigned int virq,
471 irq_hw_number_t hwirq)
Qais Yousefc98c18222015-12-08 13:20:24 +0000472{
Paul Burtonb87281e2017-04-20 10:07:35 +0100473 int err;
Qais Yousefc98c18222015-12-08 13:20:24 +0000474
Matt Redfearn8ada00a2017-04-20 10:07:36 +0100475 if (hwirq >= GIC_SHARED_HWIRQ_BASE) {
Paul Burtonb87281e2017-04-20 10:07:35 +0100476 /* verify that shared irqs don't conflict with an IPI irq */
477 if (test_bit(GIC_HWIRQ_TO_SHARED(hwirq), ipi_resrv))
478 return -EBUSY;
Qais Yousefc98c18222015-12-08 13:20:24 +0000479
Paul Burtonb87281e2017-04-20 10:07:35 +0100480 err = irq_domain_set_hwirq_and_chip(d, virq, hwirq,
481 &gic_level_irq_controller,
482 NULL);
483 if (err)
484 return err;
485
486 return gic_shared_irq_domain_map(d, virq, hwirq, 0);
Qais Yousefc98c18222015-12-08 13:20:24 +0000487 }
488
Paul Burtonb87281e2017-04-20 10:07:35 +0100489 switch (GIC_HWIRQ_TO_LOCAL(hwirq)) {
490 case GIC_LOCAL_INT_TIMER:
491 case GIC_LOCAL_INT_PERFCTR:
492 case GIC_LOCAL_INT_FDC:
493 /*
494 * HACK: These are all really percpu interrupts, but
495 * the rest of the MIPS kernel code does not use the
496 * percpu IRQ API for them.
497 */
498 err = irq_domain_set_hwirq_and_chip(d, virq, hwirq,
499 &gic_all_vpes_local_irq_controller,
500 NULL);
501 if (err)
502 return err;
503
504 irq_set_handler(virq, handle_percpu_irq);
505 break;
506
507 default:
508 err = irq_domain_set_hwirq_and_chip(d, virq, hwirq,
509 &gic_local_irq_controller,
510 NULL);
511 if (err)
512 return err;
513
514 irq_set_handler(virq, handle_percpu_devid_irq);
515 irq_set_percpu_devid(virq);
516 break;
517 }
518
519 return gic_local_irq_domain_map(d, virq, hwirq);
Qais Yousefc98c18222015-12-08 13:20:24 +0000520}
521
Matt Redfearn8ada00a2017-04-20 10:07:36 +0100522static int gic_irq_domain_alloc(struct irq_domain *d, unsigned int virq,
523 unsigned int nr_irqs, void *arg)
524{
525 struct irq_fwspec *fwspec = arg;
526 irq_hw_number_t hwirq;
527
528 if (fwspec->param[0] == GIC_SHARED)
529 hwirq = GIC_SHARED_TO_HWIRQ(fwspec->param[1]);
530 else
531 hwirq = GIC_LOCAL_TO_HWIRQ(fwspec->param[1]);
532
533 return gic_irq_domain_map(d, virq, hwirq);
534}
535
Paul Burtonb87281e2017-04-20 10:07:35 +0100536void gic_irq_domain_free(struct irq_domain *d, unsigned int virq,
Qais Yousefc98c18222015-12-08 13:20:24 +0000537 unsigned int nr_irqs)
538{
Qais Yousefc98c18222015-12-08 13:20:24 +0000539}
540
Paul Burtonb87281e2017-04-20 10:07:35 +0100541static const struct irq_domain_ops gic_irq_domain_ops = {
542 .xlate = gic_irq_domain_xlate,
543 .alloc = gic_irq_domain_alloc,
544 .free = gic_irq_domain_free,
Matt Redfearn8ada00a2017-04-20 10:07:36 +0100545 .map = gic_irq_domain_map,
Qais Yousef2af70a92015-12-08 13:20:23 +0000546};
547
548static int gic_ipi_domain_xlate(struct irq_domain *d, struct device_node *ctrlr,
549 const u32 *intspec, unsigned int intsize,
550 irq_hw_number_t *out_hwirq,
551 unsigned int *out_type)
552{
553 /*
554 * There's nothing to translate here. hwirq is dynamically allocated and
555 * the irq type is always edge triggered.
556 * */
557 *out_hwirq = 0;
558 *out_type = IRQ_TYPE_EDGE_RISING;
559
560 return 0;
561}
562
563static int gic_ipi_domain_alloc(struct irq_domain *d, unsigned int virq,
564 unsigned int nr_irqs, void *arg)
565{
566 struct cpumask *ipimask = arg;
Paul Burtonb87281e2017-04-20 10:07:35 +0100567 irq_hw_number_t hwirq, base_hwirq;
568 int cpu, ret, i;
Qais Yousef2af70a92015-12-08 13:20:23 +0000569
Paul Burtonb87281e2017-04-20 10:07:35 +0100570 base_hwirq = find_first_bit(ipi_available, gic_shared_intrs);
571 if (base_hwirq == gic_shared_intrs)
572 return -ENOMEM;
Qais Yousef2af70a92015-12-08 13:20:23 +0000573
Paul Burtonb87281e2017-04-20 10:07:35 +0100574 /* check that we have enough space */
575 for (i = base_hwirq; i < nr_irqs; i++) {
576 if (!test_bit(i, ipi_available))
577 return -EBUSY;
578 }
579 bitmap_clear(ipi_available, base_hwirq, nr_irqs);
580
581 /* map the hwirq for each cpu consecutively */
582 i = 0;
583 for_each_cpu(cpu, ipimask) {
584 hwirq = GIC_SHARED_TO_HWIRQ(base_hwirq + i);
585
586 ret = irq_domain_set_hwirq_and_chip(d, virq + i, hwirq,
587 &gic_edge_irq_controller,
588 NULL);
589 if (ret)
590 goto error;
591
592 ret = irq_domain_set_hwirq_and_chip(d->parent, virq + i, hwirq,
Qais Yousef2af70a92015-12-08 13:20:23 +0000593 &gic_edge_irq_controller,
594 NULL);
595 if (ret)
596 goto error;
597
598 ret = irq_set_irq_type(virq + i, IRQ_TYPE_EDGE_RISING);
599 if (ret)
600 goto error;
Paul Burtonb87281e2017-04-20 10:07:35 +0100601
602 ret = gic_shared_irq_domain_map(d, virq + i, hwirq, cpu);
603 if (ret)
604 goto error;
605
606 i++;
Qais Yousef2af70a92015-12-08 13:20:23 +0000607 }
608
609 return 0;
610error:
Paul Burtonb87281e2017-04-20 10:07:35 +0100611 bitmap_set(ipi_available, base_hwirq, nr_irqs);
Qais Yousef2af70a92015-12-08 13:20:23 +0000612 return ret;
613}
614
615void gic_ipi_domain_free(struct irq_domain *d, unsigned int virq,
616 unsigned int nr_irqs)
617{
Paul Burtonb87281e2017-04-20 10:07:35 +0100618 irq_hw_number_t base_hwirq;
619 struct irq_data *data;
620
621 data = irq_get_irq_data(virq);
622 if (!data)
623 return;
624
625 base_hwirq = GIC_HWIRQ_TO_SHARED(irqd_to_hwirq(data));
626 bitmap_set(ipi_available, base_hwirq, nr_irqs);
Qais Yousef2af70a92015-12-08 13:20:23 +0000627}
628
629int gic_ipi_domain_match(struct irq_domain *d, struct device_node *node,
630 enum irq_domain_bus_token bus_token)
631{
632 bool is_ipi;
633
634 switch (bus_token) {
635 case DOMAIN_BUS_IPI:
636 is_ipi = d->bus_token == bus_token;
Paul Burton547aefc2016-07-05 14:26:00 +0100637 return (!node || to_of_node(d->fwnode) == node) && is_ipi;
Qais Yousef2af70a92015-12-08 13:20:23 +0000638 break;
639 default:
640 return 0;
641 }
642}
643
Tobias Klauser0b7e8152017-06-02 10:20:56 +0200644static const struct irq_domain_ops gic_ipi_domain_ops = {
Qais Yousef2af70a92015-12-08 13:20:23 +0000645 .xlate = gic_ipi_domain_xlate,
646 .alloc = gic_ipi_domain_alloc,
647 .free = gic_ipi_domain_free,
648 .match = gic_ipi_domain_match,
Andrew Brestickerc49581a2014-09-18 14:47:23 -0700649};
650
Ralf Baechle39b8d522008-04-28 17:14:26 +0100651
Paul Burtonfbea7542017-08-12 21:36:40 -0700652static int __init gic_of_init(struct device_node *node,
653 struct device_node *parent)
654{
655 unsigned int cpu_vec, i, reserved, gicconfig, cpu, v[2];
656 phys_addr_t gic_base;
657 struct resource res;
658 size_t gic_len;
659
660 /* Find the first available CPU vector. */
661 i = reserved = 0;
662 while (!of_property_read_u32_index(node, "mti,reserved-cpu-vectors",
663 i++, &cpu_vec))
664 reserved |= BIT(cpu_vec);
665 for (cpu_vec = 2; cpu_vec < 8; cpu_vec++) {
666 if (!(reserved & BIT(cpu_vec)))
667 break;
668 }
669 if (cpu_vec == 8) {
670 pr_err("No CPU vectors available for GIC\n");
671 return -ENODEV;
672 }
673
674 if (of_address_to_resource(node, 0, &res)) {
675 /*
676 * Probe the CM for the GIC base address if not specified
677 * in the device-tree.
678 */
679 if (mips_cm_present()) {
680 gic_base = read_gcr_gic_base() &
681 ~CM_GCR_GIC_BASE_GICEN;
682 gic_len = 0x20000;
683 } else {
684 pr_err("Failed to get GIC memory range\n");
685 return -ENODEV;
686 }
687 } else {
688 gic_base = res.start;
689 gic_len = resource_size(&res);
690 }
691
692 if (mips_cm_present()) {
693 write_gcr_gic_base(gic_base | CM_GCR_GIC_BASE_GICEN);
694 /* Ensure GIC region is enabled before trying to access it */
695 __sync();
696 }
697
698 mips_gic_base = ioremap_nocache(gic_base, gic_len);
Ralf Baechle39b8d522008-04-28 17:14:26 +0100699
Paul Burton36807462017-08-12 21:36:24 -0700700 gicconfig = read_gic_config();
701 gic_shared_intrs = gicconfig & GIC_CONFIG_NUMINTERRUPTS;
702 gic_shared_intrs >>= __fls(GIC_CONFIG_NUMINTERRUPTS);
703 gic_shared_intrs = (gic_shared_intrs + 1) * 8;
Ralf Baechle39b8d522008-04-28 17:14:26 +0100704
Paul Burton36807462017-08-12 21:36:24 -0700705 gic_vpes = gicconfig & GIC_CONFIG_PVPS;
706 gic_vpes >>= __fls(GIC_CONFIG_PVPS);
Andrew Brestickere9de6882014-09-18 14:47:27 -0700707 gic_vpes = gic_vpes + 1;
Ralf Baechle39b8d522008-04-28 17:14:26 +0100708
Andrew Bresticker18743d22014-09-18 14:47:24 -0700709 if (cpu_has_veic) {
Paul Burtonba01cf02016-05-17 15:31:06 +0100710 /* Set EIC mode for all VPEs */
711 for_each_present_cpu(cpu) {
Paul Burton0d0cf582017-08-12 21:36:26 -0700712 write_gic_vl_other(mips_cm_vp_id(cpu));
713 write_gic_vo_ctl(GIC_VX_CTL_EIC);
Paul Burtonba01cf02016-05-17 15:31:06 +0100714 }
715
Andrew Bresticker18743d22014-09-18 14:47:24 -0700716 /* Always use vector 1 in EIC mode */
717 gic_cpu_pin = 0;
James Hogan1b6af712015-01-19 15:38:24 +0000718 timer_cpu_pin = gic_cpu_pin;
Andrew Bresticker18743d22014-09-18 14:47:24 -0700719 set_vi_handler(gic_cpu_pin + GIC_PIN_TO_VEC_OFFSET,
720 __gic_irq_dispatch);
721 } else {
722 gic_cpu_pin = cpu_vec - GIC_CPU_PIN_OFFSET;
723 irq_set_chained_handler(MIPS_CPU_IRQ_BASE + cpu_vec,
724 gic_irq_dispatch);
James Hogan1b6af712015-01-19 15:38:24 +0000725 /*
726 * With the CMP implementation of SMP (deprecated), other CPUs
727 * are started by the bootloader and put into a timer based
728 * waiting poll loop. We must not re-route those CPU's local
729 * timer interrupts as the wait instruction will never finish,
730 * so just handle whatever CPU interrupt it is routed to by
731 * default.
732 *
733 * This workaround should be removed when CMP support is
734 * dropped.
735 */
736 if (IS_ENABLED(CONFIG_MIPS_CMP) &&
737 gic_local_irq_is_routable(GIC_LOCAL_INT_TIMER)) {
Paul Burton0d0cf582017-08-12 21:36:26 -0700738 timer_cpu_pin = read_gic_vl_timer_map() & GIC_MAP_PIN_MAP;
James Hogan1b6af712015-01-19 15:38:24 +0000739 irq_set_chained_handler(MIPS_CPU_IRQ_BASE +
740 GIC_CPU_PIN_OFFSET +
741 timer_cpu_pin,
742 gic_irq_dispatch);
743 } else {
744 timer_cpu_pin = gic_cpu_pin;
745 }
Andrew Bresticker18743d22014-09-18 14:47:24 -0700746 }
747
Andrew Brestickera7057272014-11-12 11:43:38 -0800748 gic_irq_domain = irq_domain_add_simple(node, GIC_NUM_LOCAL_INTRS +
Paul Burtonfbea7542017-08-12 21:36:40 -0700749 gic_shared_intrs, 0,
Andrew Brestickerc49581a2014-09-18 14:47:23 -0700750 &gic_irq_domain_ops, NULL);
Paul Burtonfbea7542017-08-12 21:36:40 -0700751 if (!gic_irq_domain) {
752 pr_err("Failed to add GIC IRQ domain");
753 return -ENXIO;
754 }
Steven J. Hill0b271f52012-08-31 16:05:37 -0500755
Qais Yousef2af70a92015-12-08 13:20:23 +0000756 gic_ipi_domain = irq_domain_add_hierarchy(gic_irq_domain,
757 IRQ_DOMAIN_FLAG_IPI_PER_CPU,
758 GIC_NUM_LOCAL_INTRS + gic_shared_intrs,
759 node, &gic_ipi_domain_ops, NULL);
Paul Burtonfbea7542017-08-12 21:36:40 -0700760 if (!gic_ipi_domain) {
761 pr_err("Failed to add GIC IPI domain");
762 return -ENXIO;
763 }
Qais Yousef2af70a92015-12-08 13:20:23 +0000764
Marc Zyngier96f0d932017-06-22 11:42:50 +0100765 irq_domain_update_bus_token(gic_ipi_domain, DOMAIN_BUS_IPI);
Qais Yousef2af70a92015-12-08 13:20:23 +0000766
Qais Yousef16a80832015-12-08 13:20:30 +0000767 if (node &&
768 !of_property_read_u32_array(node, "mti,reserved-ipi-vectors", v, 2)) {
769 bitmap_set(ipi_resrv, v[0], v[1]);
770 } else {
771 /* Make the last 2 * gic_vpes available for IPIs */
772 bitmap_set(ipi_resrv,
773 gic_shared_intrs - 2 * gic_vpes,
774 2 * gic_vpes);
775 }
Qais Yousef2af70a92015-12-08 13:20:23 +0000776
Paul Burtonf8dcd9e2017-04-20 10:07:34 +0100777 bitmap_copy(ipi_available, ipi_resrv, GIC_MAX_INTRS);
Andrew Brestickere9de6882014-09-18 14:47:27 -0700778 gic_basic_init();
Andrew Brestickera7057272014-11-12 11:43:38 -0800779
780 return 0;
781}
782IRQCHIP_DECLARE(mips_gic, "mti,gic", gic_of_init);