blob: ffcb018395edb2c8ea136241d3740c42940b98f7 [file] [log] [blame]
Thomas Gleixnercaab2772019-06-03 07:44:50 +02001// SPDX-License-Identifier: GPL-2.0-only
Marc Zyngier021f6532014-06-30 16:01:31 +01002/*
Marc Zyngier0edc23e2016-12-19 17:01:52 +00003 * Copyright (C) 2013-2017 ARM Limited, All Rights Reserved.
Marc Zyngier021f6532014-06-30 16:01:31 +01004 * Author: Marc Zyngier <marc.zyngier@arm.com>
Marc Zyngier021f6532014-06-30 16:01:31 +01005 */
6
Julien Grall68628bb2016-04-11 16:32:55 +01007#define pr_fmt(fmt) "GICv3: " fmt
8
Tomasz Nowickiffa7d612016-01-19 14:11:15 +01009#include <linux/acpi.h>
Marc Zyngier021f6532014-06-30 16:01:31 +010010#include <linux/cpu.h>
Sudeep Holla3708d522014-08-26 16:03:35 +010011#include <linux/cpu_pm.h>
Marc Zyngier021f6532014-06-30 16:01:31 +010012#include <linux/delay.h>
13#include <linux/interrupt.h>
Tomasz Nowickiffa7d612016-01-19 14:11:15 +010014#include <linux/irqdomain.h>
Marc Zyngier021f6532014-06-30 16:01:31 +010015#include <linux/of.h>
16#include <linux/of_address.h>
17#include <linux/of_irq.h>
18#include <linux/percpu.h>
Julien Thierry101b35f2019-01-31 14:58:59 +000019#include <linux/refcount.h>
Marc Zyngier021f6532014-06-30 16:01:31 +010020#include <linux/slab.h>
21
Joel Porquet41a83e062015-07-07 17:11:46 -040022#include <linux/irqchip.h>
Julien Grall1839e572016-04-11 16:32:57 +010023#include <linux/irqchip/arm-gic-common.h>
Marc Zyngier021f6532014-06-30 16:01:31 +010024#include <linux/irqchip/arm-gic-v3.h>
Marc Zyngiere3825ba2016-04-11 09:57:54 +010025#include <linux/irqchip/irq-partition-percpu.h>
Marc Zyngier021f6532014-06-30 16:01:31 +010026
27#include <asm/cputype.h>
28#include <asm/exception.h>
29#include <asm/smp_plat.h>
Marc Zyngier0b6a3da2015-08-26 17:00:42 +010030#include <asm/virt.h>
Marc Zyngier021f6532014-06-30 16:01:31 +010031
32#include "irq-gic-common.h"
Marc Zyngier021f6532014-06-30 16:01:31 +010033
Julien Thierryf32c9262019-01-31 14:58:58 +000034#define GICD_INT_NMI_PRI (GICD_INT_DEF_PRI & ~0x80)
35
Srinivas Kandagatla9c8114c2018-12-10 13:56:32 +000036#define FLAGS_WORKAROUND_GICR_WAKER_MSM8996 (1ULL << 0)
37
Marc Zyngierf5c14342014-11-24 14:35:10 +000038struct redist_region {
39 void __iomem *redist_base;
40 phys_addr_t phys_base;
Tomasz Nowickib70fb7a2016-01-19 14:11:16 +010041 bool single_redist;
Marc Zyngierf5c14342014-11-24 14:35:10 +000042};
43
Marc Zyngier021f6532014-06-30 16:01:31 +010044struct gic_chip_data {
Marc Zyngiere3825ba2016-04-11 09:57:54 +010045 struct fwnode_handle *fwnode;
Marc Zyngier021f6532014-06-30 16:01:31 +010046 void __iomem *dist_base;
Marc Zyngierf5c14342014-11-24 14:35:10 +000047 struct redist_region *redist_regions;
48 struct rdists rdists;
Marc Zyngier021f6532014-06-30 16:01:31 +010049 struct irq_domain *domain;
50 u64 redist_stride;
Marc Zyngierf5c14342014-11-24 14:35:10 +000051 u32 nr_redist_regions;
Srinivas Kandagatla9c8114c2018-12-10 13:56:32 +000052 u64 flags;
Shanker Donthinenieda0d042017-10-06 10:24:00 -050053 bool has_rss;
Marc Zyngier1a60e1e2019-07-18 11:15:14 +010054 unsigned int ppi_nr;
Marc Zyngier52085d32019-07-18 13:05:17 +010055 struct partition_desc **ppi_descs;
Marc Zyngier021f6532014-06-30 16:01:31 +010056};
57
58static struct gic_chip_data gic_data __read_mostly;
Davidlohr Buesod01d3272018-03-26 14:09:25 -070059static DEFINE_STATIC_KEY_TRUE(supports_deactivate_key);
Marc Zyngier021f6532014-06-30 16:01:31 +010060
Marc Zyngier211bddd2019-07-16 15:17:31 +010061#define GIC_ID_NR (1U << GICD_TYPER_ID_BITS(gic_data.rdists.gicd_typer))
Zenghui Yuc107d612019-09-18 06:57:30 +000062#define GIC_LINE_NR min(GICD_TYPER_SPIS(gic_data.rdists.gicd_typer), 1020U)
Marc Zyngier211bddd2019-07-16 15:17:31 +010063#define GIC_ESPI_NR GICD_TYPER_ESPIS(gic_data.rdists.gicd_typer)
64
Julien Thierryd98d0a92019-01-31 14:58:57 +000065/*
66 * The behaviours of RPR and PMR registers differ depending on the value of
67 * SCR_EL3.FIQ, and the behaviour of non-secure priority registers of the
68 * distributor and redistributors depends on whether security is enabled in the
69 * GIC.
70 *
71 * When security is enabled, non-secure priority values from the (re)distributor
72 * are presented to the GIC CPUIF as follow:
73 * (GIC_(R)DIST_PRI[irq] >> 1) | 0x80;
74 *
75 * If SCR_EL3.FIQ == 1, the values writen to/read from PMR and RPR at non-secure
76 * EL1 are subject to a similar operation thus matching the priorities presented
77 * from the (re)distributor when security is enabled.
78 *
79 * see GICv3/GICv4 Architecture Specification (IHI0069D):
80 * - section 4.8.1 Non-secure accesses to register fields for Secure interrupt
81 * priorities.
82 * - Figure 4-7 Secure read of the priority field for a Non-secure Group 1
83 * interrupt.
84 *
85 * For now, we only support pseudo-NMIs if we have non-secure view of
86 * priorities.
87 */
88static DEFINE_STATIC_KEY_FALSE(supports_pseudo_nmis);
89
Marc Zyngierf2266502019-10-02 10:06:12 +010090/*
91 * Global static key controlling whether an update to PMR allowing more
92 * interrupts requires to be propagated to the redistributor (DSB SY).
93 * And this needs to be exported for modules to be able to enable
94 * interrupts...
95 */
96DEFINE_STATIC_KEY_FALSE(gic_pmr_sync);
97EXPORT_SYMBOL(gic_pmr_sync);
98
Julien Thierry101b35f2019-01-31 14:58:59 +000099/* ppi_nmi_refs[n] == number of cpus having ppi[n + 16] set as NMI */
Marc Zyngier81a43272019-07-18 12:53:05 +0100100static refcount_t *ppi_nmi_refs;
Julien Thierry101b35f2019-01-31 14:58:59 +0000101
Julien Grall1839e572016-04-11 16:32:57 +0100102static struct gic_kvm_info gic_v3_kvm_info;
Shanker Donthinenieda0d042017-10-06 10:24:00 -0500103static DEFINE_PER_CPU(bool, has_rss);
Julien Grall1839e572016-04-11 16:32:57 +0100104
Shanker Donthinenieda0d042017-10-06 10:24:00 -0500105#define MPIDR_RS(mpidr) (((mpidr) & 0xF0UL) >> 4)
Marc Zyngierf5c14342014-11-24 14:35:10 +0000106#define gic_data_rdist() (this_cpu_ptr(gic_data.rdists.rdist))
107#define gic_data_rdist_rd_base() (gic_data_rdist()->rd_base)
Marc Zyngier021f6532014-06-30 16:01:31 +0100108#define gic_data_rdist_sgi_base() (gic_data_rdist_rd_base() + SZ_64K)
109
110/* Our default, arbitrary priority value. Linux only uses one anyway. */
111#define DEFAULT_PMR_VALUE 0xf0
112
Marc Zyngiere91b0362019-07-16 14:41:40 +0100113enum gic_intid_range {
114 PPI_RANGE,
115 SPI_RANGE,
Marc Zyngier5f51f802019-07-18 13:19:25 +0100116 EPPI_RANGE,
Marc Zyngier211bddd2019-07-16 15:17:31 +0100117 ESPI_RANGE,
Marc Zyngiere91b0362019-07-16 14:41:40 +0100118 LPI_RANGE,
119 __INVALID_RANGE__
120};
121
122static enum gic_intid_range __get_intid_range(irq_hw_number_t hwirq)
123{
124 switch (hwirq) {
125 case 16 ... 31:
126 return PPI_RANGE;
127 case 32 ... 1019:
128 return SPI_RANGE;
Marc Zyngier5f51f802019-07-18 13:19:25 +0100129 case EPPI_BASE_INTID ... (EPPI_BASE_INTID + 63):
130 return EPPI_RANGE;
Marc Zyngier211bddd2019-07-16 15:17:31 +0100131 case ESPI_BASE_INTID ... (ESPI_BASE_INTID + 1023):
132 return ESPI_RANGE;
Marc Zyngiere91b0362019-07-16 14:41:40 +0100133 case 8192 ... GENMASK(23, 0):
134 return LPI_RANGE;
135 default:
136 return __INVALID_RANGE__;
137 }
138}
139
140static enum gic_intid_range get_intid_range(struct irq_data *d)
141{
142 return __get_intid_range(d->hwirq);
143}
144
Marc Zyngier021f6532014-06-30 16:01:31 +0100145static inline unsigned int gic_irq(struct irq_data *d)
146{
147 return d->hwirq;
148}
149
150static inline int gic_irq_in_rdist(struct irq_data *d)
151{
Marc Zyngier5f51f802019-07-18 13:19:25 +0100152 enum gic_intid_range range = get_intid_range(d);
153 return range == PPI_RANGE || range == EPPI_RANGE;
Marc Zyngier021f6532014-06-30 16:01:31 +0100154}
155
156static inline void __iomem *gic_dist_base(struct irq_data *d)
157{
Marc Zyngiere91b0362019-07-16 14:41:40 +0100158 switch (get_intid_range(d)) {
159 case PPI_RANGE:
Marc Zyngier5f51f802019-07-18 13:19:25 +0100160 case EPPI_RANGE:
Marc Zyngiere91b0362019-07-16 14:41:40 +0100161 /* SGI+PPI -> SGI_base for this CPU */
Marc Zyngier021f6532014-06-30 16:01:31 +0100162 return gic_data_rdist_sgi_base();
163
Marc Zyngiere91b0362019-07-16 14:41:40 +0100164 case SPI_RANGE:
Marc Zyngier211bddd2019-07-16 15:17:31 +0100165 case ESPI_RANGE:
Marc Zyngiere91b0362019-07-16 14:41:40 +0100166 /* SPI -> dist_base */
Marc Zyngier021f6532014-06-30 16:01:31 +0100167 return gic_data.dist_base;
168
Marc Zyngiere91b0362019-07-16 14:41:40 +0100169 default:
170 return NULL;
171 }
Marc Zyngier021f6532014-06-30 16:01:31 +0100172}
173
174static void gic_do_wait_for_rwp(void __iomem *base)
175{
176 u32 count = 1000000; /* 1s! */
177
178 while (readl_relaxed(base + GICD_CTLR) & GICD_CTLR_RWP) {
179 count--;
180 if (!count) {
181 pr_err_ratelimited("RWP timeout, gone fishing\n");
182 return;
183 }
184 cpu_relax();
185 udelay(1);
Daode Huang2c542422019-10-17 16:25:29 +0800186 }
Marc Zyngier021f6532014-06-30 16:01:31 +0100187}
188
189/* Wait for completion of a distributor change */
190static void gic_dist_wait_for_rwp(void)
191{
192 gic_do_wait_for_rwp(gic_data.dist_base);
193}
194
195/* Wait for completion of a redistributor change */
196static void gic_redist_wait_for_rwp(void)
197{
198 gic_do_wait_for_rwp(gic_data_rdist_rd_base());
199}
200
Jean-Philippe Brucker7936e912015-10-01 13:47:14 +0100201#ifdef CONFIG_ARM64
Robert Richter6d4e11c2015-09-21 22:58:35 +0200202
203static u64 __maybe_unused gic_read_iar(void)
204{
Suzuki K Poulosea4023f682016-11-08 13:56:20 +0000205 if (cpus_have_const_cap(ARM64_WORKAROUND_CAVIUM_23154))
Robert Richter6d4e11c2015-09-21 22:58:35 +0200206 return gic_read_iar_cavium_thunderx();
207 else
208 return gic_read_iar_common();
209}
Jean-Philippe Brucker7936e912015-10-01 13:47:14 +0100210#endif
Marc Zyngier021f6532014-06-30 16:01:31 +0100211
Sudeep Hollaa2c22512014-08-26 16:03:34 +0100212static void gic_enable_redist(bool enable)
Marc Zyngier021f6532014-06-30 16:01:31 +0100213{
214 void __iomem *rbase;
215 u32 count = 1000000; /* 1s! */
216 u32 val;
217
Srinivas Kandagatla9c8114c2018-12-10 13:56:32 +0000218 if (gic_data.flags & FLAGS_WORKAROUND_GICR_WAKER_MSM8996)
219 return;
220
Marc Zyngier021f6532014-06-30 16:01:31 +0100221 rbase = gic_data_rdist_rd_base();
222
Marc Zyngier021f6532014-06-30 16:01:31 +0100223 val = readl_relaxed(rbase + GICR_WAKER);
Sudeep Hollaa2c22512014-08-26 16:03:34 +0100224 if (enable)
225 /* Wake up this CPU redistributor */
226 val &= ~GICR_WAKER_ProcessorSleep;
227 else
228 val |= GICR_WAKER_ProcessorSleep;
Marc Zyngier021f6532014-06-30 16:01:31 +0100229 writel_relaxed(val, rbase + GICR_WAKER);
230
Sudeep Hollaa2c22512014-08-26 16:03:34 +0100231 if (!enable) { /* Check that GICR_WAKER is writeable */
232 val = readl_relaxed(rbase + GICR_WAKER);
233 if (!(val & GICR_WAKER_ProcessorSleep))
234 return; /* No PM support in this redistributor */
235 }
236
Dan Carpenterd102eb52016-10-14 10:26:21 +0300237 while (--count) {
Sudeep Hollaa2c22512014-08-26 16:03:34 +0100238 val = readl_relaxed(rbase + GICR_WAKER);
Andrew Jonescf1d9d12016-05-11 21:23:17 +0200239 if (enable ^ (bool)(val & GICR_WAKER_ChildrenAsleep))
Sudeep Hollaa2c22512014-08-26 16:03:34 +0100240 break;
Marc Zyngier021f6532014-06-30 16:01:31 +0100241 cpu_relax();
242 udelay(1);
Daode Huang2c542422019-10-17 16:25:29 +0800243 }
Sudeep Hollaa2c22512014-08-26 16:03:34 +0100244 if (!count)
245 pr_err_ratelimited("redistributor failed to %s...\n",
246 enable ? "wakeup" : "sleep");
Marc Zyngier021f6532014-06-30 16:01:31 +0100247}
248
249/*
250 * Routines to disable, enable, EOI and route interrupts
251 */
Marc Zyngiere91b0362019-07-16 14:41:40 +0100252static u32 convert_offset_index(struct irq_data *d, u32 offset, u32 *index)
253{
254 switch (get_intid_range(d)) {
255 case PPI_RANGE:
256 case SPI_RANGE:
257 *index = d->hwirq;
258 return offset;
Marc Zyngier5f51f802019-07-18 13:19:25 +0100259 case EPPI_RANGE:
260 /*
261 * Contrary to the ESPI range, the EPPI range is contiguous
262 * to the PPI range in the registers, so let's adjust the
263 * displacement accordingly. Consistency is overrated.
264 */
265 *index = d->hwirq - EPPI_BASE_INTID + 32;
266 return offset;
Marc Zyngier211bddd2019-07-16 15:17:31 +0100267 case ESPI_RANGE:
268 *index = d->hwirq - ESPI_BASE_INTID;
269 switch (offset) {
270 case GICD_ISENABLER:
271 return GICD_ISENABLERnE;
272 case GICD_ICENABLER:
273 return GICD_ICENABLERnE;
274 case GICD_ISPENDR:
275 return GICD_ISPENDRnE;
276 case GICD_ICPENDR:
277 return GICD_ICPENDRnE;
278 case GICD_ISACTIVER:
279 return GICD_ISACTIVERnE;
280 case GICD_ICACTIVER:
281 return GICD_ICACTIVERnE;
282 case GICD_IPRIORITYR:
283 return GICD_IPRIORITYRnE;
284 case GICD_ICFGR:
285 return GICD_ICFGRnE;
286 case GICD_IROUTER:
287 return GICD_IROUTERnE;
288 default:
289 break;
290 }
291 break;
Marc Zyngiere91b0362019-07-16 14:41:40 +0100292 default:
293 break;
294 }
295
296 WARN_ON(1);
297 *index = d->hwirq;
298 return offset;
299}
300
Marc Zyngierb594c6e2015-03-18 11:01:24 +0000301static int gic_peek_irq(struct irq_data *d, u32 offset)
302{
Marc Zyngierb594c6e2015-03-18 11:01:24 +0000303 void __iomem *base;
Marc Zyngiere91b0362019-07-16 14:41:40 +0100304 u32 index, mask;
305
306 offset = convert_offset_index(d, offset, &index);
307 mask = 1 << (index % 32);
Marc Zyngierb594c6e2015-03-18 11:01:24 +0000308
309 if (gic_irq_in_rdist(d))
310 base = gic_data_rdist_sgi_base();
311 else
312 base = gic_data.dist_base;
313
Marc Zyngiere91b0362019-07-16 14:41:40 +0100314 return !!(readl_relaxed(base + offset + (index / 32) * 4) & mask);
Marc Zyngierb594c6e2015-03-18 11:01:24 +0000315}
316
Marc Zyngier021f6532014-06-30 16:01:31 +0100317static void gic_poke_irq(struct irq_data *d, u32 offset)
318{
Marc Zyngier021f6532014-06-30 16:01:31 +0100319 void (*rwp_wait)(void);
320 void __iomem *base;
Marc Zyngiere91b0362019-07-16 14:41:40 +0100321 u32 index, mask;
322
323 offset = convert_offset_index(d, offset, &index);
324 mask = 1 << (index % 32);
Marc Zyngier021f6532014-06-30 16:01:31 +0100325
326 if (gic_irq_in_rdist(d)) {
327 base = gic_data_rdist_sgi_base();
328 rwp_wait = gic_redist_wait_for_rwp;
329 } else {
330 base = gic_data.dist_base;
331 rwp_wait = gic_dist_wait_for_rwp;
332 }
333
Marc Zyngiere91b0362019-07-16 14:41:40 +0100334 writel_relaxed(mask, base + offset + (index / 32) * 4);
Marc Zyngier021f6532014-06-30 16:01:31 +0100335 rwp_wait();
336}
337
Marc Zyngier021f6532014-06-30 16:01:31 +0100338static void gic_mask_irq(struct irq_data *d)
339{
340 gic_poke_irq(d, GICD_ICENABLER);
341}
342
Marc Zyngier0b6a3da2015-08-26 17:00:42 +0100343static void gic_eoimode1_mask_irq(struct irq_data *d)
344{
345 gic_mask_irq(d);
Marc Zyngier530bf352015-08-26 17:00:43 +0100346 /*
347 * When masking a forwarded interrupt, make sure it is
348 * deactivated as well.
349 *
350 * This ensures that an interrupt that is getting
351 * disabled/masked will not get "stuck", because there is
352 * noone to deactivate it (guest is being terminated).
353 */
Thomas Gleixner4df7f542015-09-15 13:19:16 +0200354 if (irqd_is_forwarded_to_vcpu(d))
Marc Zyngier530bf352015-08-26 17:00:43 +0100355 gic_poke_irq(d, GICD_ICACTIVER);
Marc Zyngier0b6a3da2015-08-26 17:00:42 +0100356}
357
Marc Zyngier021f6532014-06-30 16:01:31 +0100358static void gic_unmask_irq(struct irq_data *d)
359{
360 gic_poke_irq(d, GICD_ISENABLER);
361}
362
Julien Thierryd98d0a92019-01-31 14:58:57 +0000363static inline bool gic_supports_nmi(void)
364{
365 return IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) &&
366 static_branch_likely(&supports_pseudo_nmis);
367}
368
Marc Zyngierb594c6e2015-03-18 11:01:24 +0000369static int gic_irq_set_irqchip_state(struct irq_data *d,
370 enum irqchip_irq_state which, bool val)
371{
372 u32 reg;
373
Marc Zyngier211bddd2019-07-16 15:17:31 +0100374 if (d->hwirq >= 8192) /* PPI/SPI only */
Marc Zyngierb594c6e2015-03-18 11:01:24 +0000375 return -EINVAL;
376
377 switch (which) {
378 case IRQCHIP_STATE_PENDING:
379 reg = val ? GICD_ISPENDR : GICD_ICPENDR;
380 break;
381
382 case IRQCHIP_STATE_ACTIVE:
383 reg = val ? GICD_ISACTIVER : GICD_ICACTIVER;
384 break;
385
386 case IRQCHIP_STATE_MASKED:
387 reg = val ? GICD_ICENABLER : GICD_ISENABLER;
388 break;
389
390 default:
391 return -EINVAL;
392 }
393
394 gic_poke_irq(d, reg);
395 return 0;
396}
397
398static int gic_irq_get_irqchip_state(struct irq_data *d,
399 enum irqchip_irq_state which, bool *val)
400{
Marc Zyngier211bddd2019-07-16 15:17:31 +0100401 if (d->hwirq >= 8192) /* PPI/SPI only */
Marc Zyngierb594c6e2015-03-18 11:01:24 +0000402 return -EINVAL;
403
404 switch (which) {
405 case IRQCHIP_STATE_PENDING:
406 *val = gic_peek_irq(d, GICD_ISPENDR);
407 break;
408
409 case IRQCHIP_STATE_ACTIVE:
410 *val = gic_peek_irq(d, GICD_ISACTIVER);
411 break;
412
413 case IRQCHIP_STATE_MASKED:
414 *val = !gic_peek_irq(d, GICD_ISENABLER);
415 break;
416
417 default:
418 return -EINVAL;
419 }
420
421 return 0;
422}
423
Julien Thierry101b35f2019-01-31 14:58:59 +0000424static void gic_irq_set_prio(struct irq_data *d, u8 prio)
425{
426 void __iomem *base = gic_dist_base(d);
Marc Zyngiere91b0362019-07-16 14:41:40 +0100427 u32 offset, index;
Julien Thierry101b35f2019-01-31 14:58:59 +0000428
Marc Zyngiere91b0362019-07-16 14:41:40 +0100429 offset = convert_offset_index(d, GICD_IPRIORITYR, &index);
430
431 writeb_relaxed(prio, base + offset + index);
Julien Thierry101b35f2019-01-31 14:58:59 +0000432}
433
Marc Zyngier81a43272019-07-18 12:53:05 +0100434static u32 gic_get_ppi_index(struct irq_data *d)
435{
436 switch (get_intid_range(d)) {
437 case PPI_RANGE:
438 return d->hwirq - 16;
Marc Zyngier5f51f802019-07-18 13:19:25 +0100439 case EPPI_RANGE:
440 return d->hwirq - EPPI_BASE_INTID + 16;
Marc Zyngier81a43272019-07-18 12:53:05 +0100441 default:
442 unreachable();
443 }
444}
445
Julien Thierry101b35f2019-01-31 14:58:59 +0000446static int gic_irq_nmi_setup(struct irq_data *d)
447{
448 struct irq_desc *desc = irq_to_desc(d->irq);
449
450 if (!gic_supports_nmi())
451 return -EINVAL;
452
453 if (gic_peek_irq(d, GICD_ISENABLER)) {
454 pr_err("Cannot set NMI property of enabled IRQ %u\n", d->irq);
455 return -EINVAL;
456 }
457
458 /*
459 * A secondary irq_chip should be in charge of LPI request,
460 * it should not be possible to get there
461 */
462 if (WARN_ON(gic_irq(d) >= 8192))
463 return -EINVAL;
464
465 /* desc lock should already be held */
Marc Zyngier81a43272019-07-18 12:53:05 +0100466 if (gic_irq_in_rdist(d)) {
467 u32 idx = gic_get_ppi_index(d);
468
Julien Thierry101b35f2019-01-31 14:58:59 +0000469 /* Setting up PPI as NMI, only switch handler for first NMI */
Marc Zyngier81a43272019-07-18 12:53:05 +0100470 if (!refcount_inc_not_zero(&ppi_nmi_refs[idx])) {
471 refcount_set(&ppi_nmi_refs[idx], 1);
Julien Thierry101b35f2019-01-31 14:58:59 +0000472 desc->handle_irq = handle_percpu_devid_fasteoi_nmi;
473 }
474 } else {
475 desc->handle_irq = handle_fasteoi_nmi;
476 }
477
478 gic_irq_set_prio(d, GICD_INT_NMI_PRI);
479
480 return 0;
481}
482
483static void gic_irq_nmi_teardown(struct irq_data *d)
484{
485 struct irq_desc *desc = irq_to_desc(d->irq);
486
487 if (WARN_ON(!gic_supports_nmi()))
488 return;
489
490 if (gic_peek_irq(d, GICD_ISENABLER)) {
491 pr_err("Cannot set NMI property of enabled IRQ %u\n", d->irq);
492 return;
493 }
494
495 /*
496 * A secondary irq_chip should be in charge of LPI request,
497 * it should not be possible to get there
498 */
499 if (WARN_ON(gic_irq(d) >= 8192))
500 return;
501
502 /* desc lock should already be held */
Marc Zyngier81a43272019-07-18 12:53:05 +0100503 if (gic_irq_in_rdist(d)) {
504 u32 idx = gic_get_ppi_index(d);
505
Julien Thierry101b35f2019-01-31 14:58:59 +0000506 /* Tearing down NMI, only switch handler for last NMI */
Marc Zyngier81a43272019-07-18 12:53:05 +0100507 if (refcount_dec_and_test(&ppi_nmi_refs[idx]))
Julien Thierry101b35f2019-01-31 14:58:59 +0000508 desc->handle_irq = handle_percpu_devid_irq;
509 } else {
510 desc->handle_irq = handle_fasteoi_irq;
511 }
512
513 gic_irq_set_prio(d, GICD_INT_DEF_PRI);
514}
515
Marc Zyngier021f6532014-06-30 16:01:31 +0100516static void gic_eoi_irq(struct irq_data *d)
517{
518 gic_write_eoir(gic_irq(d));
519}
520
Marc Zyngier0b6a3da2015-08-26 17:00:42 +0100521static void gic_eoimode1_eoi_irq(struct irq_data *d)
522{
523 /*
Marc Zyngier530bf352015-08-26 17:00:43 +0100524 * No need to deactivate an LPI, or an interrupt that
525 * is is getting forwarded to a vcpu.
Marc Zyngier0b6a3da2015-08-26 17:00:42 +0100526 */
Thomas Gleixner4df7f542015-09-15 13:19:16 +0200527 if (gic_irq(d) >= 8192 || irqd_is_forwarded_to_vcpu(d))
Marc Zyngier0b6a3da2015-08-26 17:00:42 +0100528 return;
529 gic_write_dir(gic_irq(d));
530}
531
Marc Zyngier021f6532014-06-30 16:01:31 +0100532static int gic_set_type(struct irq_data *d, unsigned int type)
533{
Marc Zyngier5f51f802019-07-18 13:19:25 +0100534 enum gic_intid_range range;
Marc Zyngier021f6532014-06-30 16:01:31 +0100535 unsigned int irq = gic_irq(d);
536 void (*rwp_wait)(void);
537 void __iomem *base;
Marc Zyngiere91b0362019-07-16 14:41:40 +0100538 u32 offset, index;
Marc Zyngier13d22e22019-07-16 14:35:17 +0100539 int ret;
Marc Zyngier021f6532014-06-30 16:01:31 +0100540
541 /* Interrupt configuration for SGIs can't be changed */
542 if (irq < 16)
543 return -EINVAL;
544
Marc Zyngier5f51f802019-07-18 13:19:25 +0100545 range = get_intid_range(d);
546
Liviu Dudaufb7e7de2015-01-20 16:52:59 +0000547 /* SPIs have restrictions on the supported types */
Marc Zyngier5f51f802019-07-18 13:19:25 +0100548 if ((range == SPI_RANGE || range == ESPI_RANGE) &&
549 type != IRQ_TYPE_LEVEL_HIGH && type != IRQ_TYPE_EDGE_RISING)
Marc Zyngier021f6532014-06-30 16:01:31 +0100550 return -EINVAL;
551
552 if (gic_irq_in_rdist(d)) {
553 base = gic_data_rdist_sgi_base();
554 rwp_wait = gic_redist_wait_for_rwp;
555 } else {
556 base = gic_data.dist_base;
557 rwp_wait = gic_dist_wait_for_rwp;
558 }
559
Marc Zyngiere91b0362019-07-16 14:41:40 +0100560 offset = convert_offset_index(d, GICD_ICFGR, &index);
Marc Zyngier13d22e22019-07-16 14:35:17 +0100561
Marc Zyngiere91b0362019-07-16 14:41:40 +0100562 ret = gic_configure_irq(index, type, base + offset, rwp_wait);
Marc Zyngier5f51f802019-07-18 13:19:25 +0100563 if (ret && (range == PPI_RANGE || range == EPPI_RANGE)) {
Marc Zyngier13d22e22019-07-16 14:35:17 +0100564 /* Misconfigured PPIs are usually not fatal */
Marc Zyngier5f51f802019-07-18 13:19:25 +0100565 pr_warn("GIC: PPI INTID%d is secure or misconfigured\n", irq);
Marc Zyngier13d22e22019-07-16 14:35:17 +0100566 ret = 0;
567 }
568
569 return ret;
Marc Zyngier021f6532014-06-30 16:01:31 +0100570}
571
Marc Zyngier530bf352015-08-26 17:00:43 +0100572static int gic_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu)
573{
Thomas Gleixner4df7f542015-09-15 13:19:16 +0200574 if (vcpu)
575 irqd_set_forwarded_to_vcpu(d);
576 else
577 irqd_clr_forwarded_to_vcpu(d);
Marc Zyngier530bf352015-08-26 17:00:43 +0100578 return 0;
579}
580
Jean-Philippe Bruckerf6c86a42015-10-01 13:47:15 +0100581static u64 gic_mpidr_to_affinity(unsigned long mpidr)
Marc Zyngier021f6532014-06-30 16:01:31 +0100582{
583 u64 aff;
584
Jean-Philippe Bruckerf6c86a42015-10-01 13:47:15 +0100585 aff = ((u64)MPIDR_AFFINITY_LEVEL(mpidr, 3) << 32 |
Marc Zyngier021f6532014-06-30 16:01:31 +0100586 MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 |
587 MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 |
588 MPIDR_AFFINITY_LEVEL(mpidr, 0));
589
590 return aff;
591}
592
Julien Thierryf32c9262019-01-31 14:58:58 +0000593static void gic_deactivate_unhandled(u32 irqnr)
594{
595 if (static_branch_likely(&supports_deactivate_key)) {
596 if (irqnr < 8192)
597 gic_write_dir(irqnr);
598 } else {
599 gic_write_eoir(irqnr);
600 }
601}
602
603static inline void gic_handle_nmi(u32 irqnr, struct pt_regs *regs)
604{
Julien Thierry17ce3022019-06-11 10:38:09 +0100605 bool irqs_enabled = interrupts_enabled(regs);
Julien Thierryf32c9262019-01-31 14:58:58 +0000606 int err;
607
Julien Thierry17ce3022019-06-11 10:38:09 +0100608 if (irqs_enabled)
609 nmi_enter();
610
Julien Thierryf32c9262019-01-31 14:58:58 +0000611 if (static_branch_likely(&supports_deactivate_key))
612 gic_write_eoir(irqnr);
613 /*
614 * Leave the PSR.I bit set to prevent other NMIs to be
615 * received while handling this one.
616 * PSR.I will be restored when we ERET to the
617 * interrupted context.
618 */
619 err = handle_domain_nmi(gic_data.domain, irqnr, regs);
620 if (err)
621 gic_deactivate_unhandled(irqnr);
Julien Thierry17ce3022019-06-11 10:38:09 +0100622
623 if (irqs_enabled)
624 nmi_exit();
Julien Thierryf32c9262019-01-31 14:58:58 +0000625}
626
Marc Zyngier021f6532014-06-30 16:01:31 +0100627static asmlinkage void __exception_irq_entry gic_handle_irq(struct pt_regs *regs)
628{
Jean-Philippe Bruckerf6c86a42015-10-01 13:47:15 +0100629 u32 irqnr;
Marc Zyngier021f6532014-06-30 16:01:31 +0100630
Julien Thierry342677d2018-08-28 16:51:29 +0100631 irqnr = gic_read_iar();
Marc Zyngier021f6532014-06-30 16:01:31 +0100632
Julien Thierryf32c9262019-01-31 14:58:58 +0000633 if (gic_supports_nmi() &&
634 unlikely(gic_read_rpr() == GICD_INT_NMI_PRI)) {
635 gic_handle_nmi(irqnr, regs);
636 return;
637 }
638
Julien Thierry3f1f3232019-01-31 14:58:44 +0000639 if (gic_prio_masking_enabled()) {
640 gic_pmr_mask_irqs();
641 gic_arch_enable_irqs();
642 }
643
Marc Zyngier211bddd2019-07-16 15:17:31 +0100644 /* Check for special IDs first */
645 if ((irqnr >= 1020 && irqnr <= 1023))
646 return;
647
648 /* Treat anything but SGIs in a uniform way */
649 if (likely(irqnr > 15)) {
Julien Thierry342677d2018-08-28 16:51:29 +0100650 int err;
Marc Zyngier0b6a3da2015-08-26 17:00:42 +0100651
Julien Thierry342677d2018-08-28 16:51:29 +0100652 if (static_branch_likely(&supports_deactivate_key))
Marc Zyngier021f6532014-06-30 16:01:31 +0100653 gic_write_eoir(irqnr);
Julien Thierry342677d2018-08-28 16:51:29 +0100654 else
655 isb();
656
657 err = handle_domain_irq(gic_data.domain, irqnr, regs);
658 if (err) {
659 WARN_ONCE(true, "Unexpected interrupt received!\n");
Julien Thierryf32c9262019-01-31 14:58:58 +0000660 gic_deactivate_unhandled(irqnr);
Marc Zyngier021f6532014-06-30 16:01:31 +0100661 }
Julien Thierry342677d2018-08-28 16:51:29 +0100662 return;
663 }
664 if (irqnr < 16) {
665 gic_write_eoir(irqnr);
666 if (static_branch_likely(&supports_deactivate_key))
667 gic_write_dir(irqnr);
668#ifdef CONFIG_SMP
669 /*
670 * Unlike GICv2, we don't need an smp_rmb() here.
671 * The control dependency from gic_read_iar to
672 * the ISB in gic_write_eoir is enough to ensure
673 * that any shared data read by handle_IPI will
674 * be read after the ACK.
675 */
676 handle_IPI(irqnr, regs);
677#else
678 WARN_ONCE(true, "Unexpected SGI received!\n");
679#endif
680 }
Marc Zyngier021f6532014-06-30 16:01:31 +0100681}
682
Julien Thierryb5cf6072019-01-31 14:58:54 +0000683static u32 gic_get_pribits(void)
684{
685 u32 pribits;
686
687 pribits = gic_read_ctlr();
688 pribits &= ICC_CTLR_EL1_PRI_BITS_MASK;
689 pribits >>= ICC_CTLR_EL1_PRI_BITS_SHIFT;
690 pribits++;
691
692 return pribits;
693}
694
695static bool gic_has_group0(void)
696{
697 u32 val;
Julien Thierrye7932182019-01-31 14:58:55 +0000698 u32 old_pmr;
699
700 old_pmr = gic_read_pmr();
Julien Thierryb5cf6072019-01-31 14:58:54 +0000701
702 /*
703 * Let's find out if Group0 is under control of EL3 or not by
704 * setting the highest possible, non-zero priority in PMR.
705 *
706 * If SCR_EL3.FIQ is set, the priority gets shifted down in
707 * order for the CPU interface to set bit 7, and keep the
708 * actual priority in the non-secure range. In the process, it
709 * looses the least significant bit and the actual priority
710 * becomes 0x80. Reading it back returns 0, indicating that
711 * we're don't have access to Group0.
712 */
713 gic_write_pmr(BIT(8 - gic_get_pribits()));
714 val = gic_read_pmr();
715
Julien Thierrye7932182019-01-31 14:58:55 +0000716 gic_write_pmr(old_pmr);
717
Julien Thierryb5cf6072019-01-31 14:58:54 +0000718 return val != 0;
719}
720
Marc Zyngier021f6532014-06-30 16:01:31 +0100721static void __init gic_dist_init(void)
722{
723 unsigned int i;
724 u64 affinity;
725 void __iomem *base = gic_data.dist_base;
726
727 /* Disable the distributor */
728 writel_relaxed(0, base + GICD_CTLR);
729 gic_dist_wait_for_rwp();
730
Marc Zyngier7c9b9732016-05-06 19:41:56 +0100731 /*
732 * Configure SPIs as non-secure Group-1. This will only matter
733 * if the GIC only has a single security state. This will not
734 * do the right thing if the kernel is running in secure mode,
735 * but that's not the intended use case anyway.
736 */
Marc Zyngier211bddd2019-07-16 15:17:31 +0100737 for (i = 32; i < GIC_LINE_NR; i += 32)
Marc Zyngier7c9b9732016-05-06 19:41:56 +0100738 writel_relaxed(~0, base + GICD_IGROUPR + i / 8);
739
Marc Zyngier211bddd2019-07-16 15:17:31 +0100740 /* Extended SPI range, not handled by the GICv2/GICv3 common code */
741 for (i = 0; i < GIC_ESPI_NR; i += 32) {
742 writel_relaxed(~0U, base + GICD_ICENABLERnE + i / 8);
743 writel_relaxed(~0U, base + GICD_ICACTIVERnE + i / 8);
744 }
745
746 for (i = 0; i < GIC_ESPI_NR; i += 32)
747 writel_relaxed(~0U, base + GICD_IGROUPRnE + i / 8);
748
749 for (i = 0; i < GIC_ESPI_NR; i += 16)
750 writel_relaxed(0, base + GICD_ICFGRnE + i / 4);
751
752 for (i = 0; i < GIC_ESPI_NR; i += 4)
753 writel_relaxed(GICD_INT_DEF_PRI_X4, base + GICD_IPRIORITYRnE + i);
754
755 /* Now do the common stuff, and wait for the distributor to drain */
756 gic_dist_config(base, GIC_LINE_NR, gic_dist_wait_for_rwp);
Marc Zyngier021f6532014-06-30 16:01:31 +0100757
758 /* Enable distributor with ARE, Group1 */
759 writel_relaxed(GICD_CTLR_ARE_NS | GICD_CTLR_ENABLE_G1A | GICD_CTLR_ENABLE_G1,
760 base + GICD_CTLR);
761
762 /*
763 * Set all global interrupts to the boot CPU only. ARE must be
764 * enabled.
765 */
766 affinity = gic_mpidr_to_affinity(cpu_logical_map(smp_processor_id()));
Marc Zyngier211bddd2019-07-16 15:17:31 +0100767 for (i = 32; i < GIC_LINE_NR; i++)
Jean-Philippe Brucker72c97122015-10-01 13:47:16 +0100768 gic_write_irouter(affinity, base + GICD_IROUTER + i * 8);
Marc Zyngier211bddd2019-07-16 15:17:31 +0100769
770 for (i = 0; i < GIC_ESPI_NR; i++)
771 gic_write_irouter(affinity, base + GICD_IROUTERnE + i * 8);
Marc Zyngier021f6532014-06-30 16:01:31 +0100772}
773
Marc Zyngier0d94ded2016-12-19 17:00:38 +0000774static int gic_iterate_rdists(int (*fn)(struct redist_region *, void __iomem *))
Marc Zyngier021f6532014-06-30 16:01:31 +0100775{
Marc Zyngier0d94ded2016-12-19 17:00:38 +0000776 int ret = -ENODEV;
Marc Zyngier021f6532014-06-30 16:01:31 +0100777 int i;
778
Marc Zyngierf5c14342014-11-24 14:35:10 +0000779 for (i = 0; i < gic_data.nr_redist_regions; i++) {
780 void __iomem *ptr = gic_data.redist_regions[i].redist_base;
Marc Zyngier0d94ded2016-12-19 17:00:38 +0000781 u64 typer;
Marc Zyngier021f6532014-06-30 16:01:31 +0100782 u32 reg;
783
784 reg = readl_relaxed(ptr + GICR_PIDR2) & GIC_PIDR2_ARCH_MASK;
785 if (reg != GIC_PIDR2_ARCH_GICv3 &&
786 reg != GIC_PIDR2_ARCH_GICv4) { /* We're in trouble... */
787 pr_warn("No redistributor present @%p\n", ptr);
788 break;
789 }
790
791 do {
Jean-Philippe Brucker72c97122015-10-01 13:47:16 +0100792 typer = gic_read_typer(ptr + GICR_TYPER);
Marc Zyngier0d94ded2016-12-19 17:00:38 +0000793 ret = fn(gic_data.redist_regions + i, ptr);
794 if (!ret)
Marc Zyngier021f6532014-06-30 16:01:31 +0100795 return 0;
Marc Zyngier021f6532014-06-30 16:01:31 +0100796
Tomasz Nowickib70fb7a2016-01-19 14:11:16 +0100797 if (gic_data.redist_regions[i].single_redist)
798 break;
799
Marc Zyngier021f6532014-06-30 16:01:31 +0100800 if (gic_data.redist_stride) {
801 ptr += gic_data.redist_stride;
802 } else {
803 ptr += SZ_64K * 2; /* Skip RD_base + SGI_base */
804 if (typer & GICR_TYPER_VLPIS)
805 ptr += SZ_64K * 2; /* Skip VLPI_base + reserved page */
806 }
807 } while (!(typer & GICR_TYPER_LAST));
808 }
809
Marc Zyngier0d94ded2016-12-19 17:00:38 +0000810 return ret ? -ENODEV : 0;
811}
812
813static int __gic_populate_rdist(struct redist_region *region, void __iomem *ptr)
814{
815 unsigned long mpidr = cpu_logical_map(smp_processor_id());
816 u64 typer;
817 u32 aff;
818
819 /*
820 * Convert affinity to a 32bit value that can be matched to
821 * GICR_TYPER bits [63:32].
822 */
823 aff = (MPIDR_AFFINITY_LEVEL(mpidr, 3) << 24 |
824 MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 |
825 MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 |
826 MPIDR_AFFINITY_LEVEL(mpidr, 0));
827
828 typer = gic_read_typer(ptr + GICR_TYPER);
829 if ((typer >> 32) == aff) {
830 u64 offset = ptr - region->redist_base;
831 gic_data_rdist_rd_base() = ptr;
832 gic_data_rdist()->phys_base = region->phys_base + offset;
833
834 pr_info("CPU%d: found redistributor %lx region %d:%pa\n",
835 smp_processor_id(), mpidr,
836 (int)(region - gic_data.redist_regions),
837 &gic_data_rdist()->phys_base);
838 return 0;
839 }
840
841 /* Try next one */
842 return 1;
843}
844
845static int gic_populate_rdist(void)
846{
847 if (gic_iterate_rdists(__gic_populate_rdist) == 0)
848 return 0;
849
Marc Zyngier021f6532014-06-30 16:01:31 +0100850 /* We couldn't even deal with ourselves... */
Jean-Philippe Bruckerf6c86a42015-10-01 13:47:15 +0100851 WARN(true, "CPU%d: mpidr %lx has no re-distributor!\n",
Marc Zyngier0d94ded2016-12-19 17:00:38 +0000852 smp_processor_id(),
853 (unsigned long)cpu_logical_map(smp_processor_id()));
Marc Zyngier021f6532014-06-30 16:01:31 +0100854 return -ENODEV;
855}
856
Marc Zyngier1a60e1e2019-07-18 11:15:14 +0100857static int __gic_update_rdist_properties(struct redist_region *region,
858 void __iomem *ptr)
Marc Zyngier0edc23e2016-12-19 17:01:52 +0000859{
860 u64 typer = gic_read_typer(ptr + GICR_TYPER);
Marc Zyngierb25319d2019-12-24 11:10:24 +0000861
Marc Zyngier0edc23e2016-12-19 17:01:52 +0000862 gic_data.rdists.has_vlpis &= !!(typer & GICR_TYPER_VLPIS);
Marc Zyngierb25319d2019-12-24 11:10:24 +0000863
864 /* RVPEID implies some form of DirectLPI, no matter what the doc says... :-/ */
865 gic_data.rdists.has_rvpeid &= !!(typer & GICR_TYPER_RVPEID);
866 gic_data.rdists.has_direct_lpi &= (!!(typer & GICR_TYPER_DirectLPIS) |
867 gic_data.rdists.has_rvpeid);
868
869 /* Detect non-sensical configurations */
870 if (WARN_ON_ONCE(gic_data.rdists.has_rvpeid && !gic_data.rdists.has_vlpis)) {
871 gic_data.rdists.has_direct_lpi = false;
872 gic_data.rdists.has_vlpis = false;
873 gic_data.rdists.has_rvpeid = false;
874 }
875
Marc Zyngier5f51f802019-07-18 13:19:25 +0100876 gic_data.ppi_nr = min(GICR_TYPER_NR_PPIS(typer), gic_data.ppi_nr);
Marc Zyngier0edc23e2016-12-19 17:01:52 +0000877
878 return 1;
879}
880
Marc Zyngier1a60e1e2019-07-18 11:15:14 +0100881static void gic_update_rdist_properties(void)
Marc Zyngier0edc23e2016-12-19 17:01:52 +0000882{
Marc Zyngier1a60e1e2019-07-18 11:15:14 +0100883 gic_data.ppi_nr = UINT_MAX;
884 gic_iterate_rdists(__gic_update_rdist_properties);
885 if (WARN_ON(gic_data.ppi_nr == UINT_MAX))
886 gic_data.ppi_nr = 0;
887 pr_info("%d PPIs implemented\n", gic_data.ppi_nr);
Marc Zyngierb25319d2019-12-24 11:10:24 +0000888 pr_info("%sVLPI support, %sdirect LPI support, %sRVPEID support\n",
Marc Zyngier0edc23e2016-12-19 17:01:52 +0000889 !gic_data.rdists.has_vlpis ? "no " : "",
Marc Zyngierb25319d2019-12-24 11:10:24 +0000890 !gic_data.rdists.has_direct_lpi ? "no " : "",
891 !gic_data.rdists.has_rvpeid ? "no " : "");
Marc Zyngier0edc23e2016-12-19 17:01:52 +0000892}
893
Julien Thierryd98d0a92019-01-31 14:58:57 +0000894/* Check whether it's single security state view */
895static inline bool gic_dist_security_disabled(void)
896{
897 return readl_relaxed(gic_data.dist_base + GICD_CTLR) & GICD_CTLR_DS;
898}
899
Sudeep Holla3708d522014-08-26 16:03:35 +0100900static void gic_cpu_sys_reg_init(void)
Marc Zyngier021f6532014-06-30 16:01:31 +0100901{
Shanker Donthinenieda0d042017-10-06 10:24:00 -0500902 int i, cpu = smp_processor_id();
903 u64 mpidr = cpu_logical_map(cpu);
904 u64 need_rss = MPIDR_RS(mpidr);
Marc Zyngier33625282018-03-20 09:46:42 +0000905 bool group0;
Julien Thierryb5cf6072019-01-31 14:58:54 +0000906 u32 pribits;
Shanker Donthinenieda0d042017-10-06 10:24:00 -0500907
Marc Zyngier7cabd002015-09-30 11:48:01 +0100908 /*
909 * Need to check that the SRE bit has actually been set. If
910 * not, it means that SRE is disabled at EL2. We're going to
911 * die painfully, and there is nothing we can do about it.
912 *
913 * Kindly inform the luser.
914 */
915 if (!gic_enable_sre())
916 pr_err("GIC: unable to set SRE (disabled at EL2), panic ahead\n");
Marc Zyngier021f6532014-06-30 16:01:31 +0100917
Julien Thierryb5cf6072019-01-31 14:58:54 +0000918 pribits = gic_get_pribits();
Marc Zyngier33625282018-03-20 09:46:42 +0000919
Julien Thierryb5cf6072019-01-31 14:58:54 +0000920 group0 = gic_has_group0();
Marc Zyngier33625282018-03-20 09:46:42 +0000921
Marc Zyngier021f6532014-06-30 16:01:31 +0100922 /* Set priority mask register */
Julien Thierryd98d0a92019-01-31 14:58:57 +0000923 if (!gic_prio_masking_enabled()) {
Julien Thierrye7932182019-01-31 14:58:55 +0000924 write_gicreg(DEFAULT_PMR_VALUE, ICC_PMR_EL1);
Julien Thierryd98d0a92019-01-31 14:58:57 +0000925 } else {
926 /*
927 * Mismatch configuration with boot CPU, the system is likely
928 * to die as interrupt masking will not work properly on all
929 * CPUs
930 */
931 WARN_ON(gic_supports_nmi() && group0 &&
932 !gic_dist_security_disabled());
933 }
Marc Zyngier021f6532014-06-30 16:01:31 +0100934
Daniel Thompson91ef8442016-08-19 17:13:09 +0100935 /*
936 * Some firmwares hand over to the kernel with the BPR changed from
937 * its reset value (and with a value large enough to prevent
938 * any pre-emptive interrupts from working at all). Writing a zero
939 * to BPR restores is reset value.
940 */
941 gic_write_bpr1(0);
942
Davidlohr Buesod01d3272018-03-26 14:09:25 -0700943 if (static_branch_likely(&supports_deactivate_key)) {
Marc Zyngier0b6a3da2015-08-26 17:00:42 +0100944 /* EOI drops priority only (mode 1) */
945 gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop);
946 } else {
947 /* EOI deactivates interrupt too (mode 0) */
948 gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop_dir);
949 }
Marc Zyngier021f6532014-06-30 16:01:31 +0100950
Marc Zyngier33625282018-03-20 09:46:42 +0000951 /* Always whack Group0 before Group1 */
952 if (group0) {
953 switch(pribits) {
954 case 8:
955 case 7:
956 write_gicreg(0, ICC_AP0R3_EL1);
957 write_gicreg(0, ICC_AP0R2_EL1);
Anders Roxell52f8c8b2019-07-26 13:28:26 +0200958 /* Fall through */
Marc Zyngier33625282018-03-20 09:46:42 +0000959 case 6:
960 write_gicreg(0, ICC_AP0R1_EL1);
Anders Roxell52f8c8b2019-07-26 13:28:26 +0200961 /* Fall through */
Marc Zyngier33625282018-03-20 09:46:42 +0000962 case 5:
963 case 4:
964 write_gicreg(0, ICC_AP0R0_EL1);
965 }
Marc Zyngierd6062a62018-03-09 14:53:19 +0000966
Marc Zyngier33625282018-03-20 09:46:42 +0000967 isb();
968 }
969
970 switch(pribits) {
Marc Zyngierd6062a62018-03-09 14:53:19 +0000971 case 8:
972 case 7:
Marc Zyngierd6062a62018-03-09 14:53:19 +0000973 write_gicreg(0, ICC_AP1R3_EL1);
Marc Zyngierd6062a62018-03-09 14:53:19 +0000974 write_gicreg(0, ICC_AP1R2_EL1);
Anders Roxell52f8c8b2019-07-26 13:28:26 +0200975 /* Fall through */
Marc Zyngierd6062a62018-03-09 14:53:19 +0000976 case 6:
Marc Zyngierd6062a62018-03-09 14:53:19 +0000977 write_gicreg(0, ICC_AP1R1_EL1);
Anders Roxell52f8c8b2019-07-26 13:28:26 +0200978 /* Fall through */
Marc Zyngierd6062a62018-03-09 14:53:19 +0000979 case 5:
980 case 4:
Marc Zyngierd6062a62018-03-09 14:53:19 +0000981 write_gicreg(0, ICC_AP1R0_EL1);
982 }
983
984 isb();
985
Marc Zyngier021f6532014-06-30 16:01:31 +0100986 /* ... and let's hit the road... */
987 gic_write_grpen1(1);
Shanker Donthinenieda0d042017-10-06 10:24:00 -0500988
989 /* Keep the RSS capability status in per_cpu variable */
990 per_cpu(has_rss, cpu) = !!(gic_read_ctlr() & ICC_CTLR_EL1_RSS);
991
992 /* Check all the CPUs have capable of sending SGIs to other CPUs */
993 for_each_online_cpu(i) {
994 bool have_rss = per_cpu(has_rss, i) && per_cpu(has_rss, cpu);
995
996 need_rss |= MPIDR_RS(cpu_logical_map(i));
997 if (need_rss && (!have_rss))
998 pr_crit("CPU%d (%lx) can't SGI CPU%d (%lx), no RSS\n",
999 cpu, (unsigned long)mpidr,
1000 i, (unsigned long)cpu_logical_map(i));
1001 }
1002
1003 /**
1004 * GIC spec says, when ICC_CTLR_EL1.RSS==1 and GICD_TYPER.RSS==0,
1005 * writing ICC_ASGI1R_EL1 register with RS != 0 is a CONSTRAINED
1006 * UNPREDICTABLE choice of :
1007 * - The write is ignored.
1008 * - The RS field is treated as 0.
1009 */
1010 if (need_rss && (!gic_data.has_rss))
1011 pr_crit_once("RSS is required but GICD doesn't support it\n");
Marc Zyngier021f6532014-06-30 16:01:31 +01001012}
1013
Marc Zyngierf736d652018-02-25 11:27:04 +00001014static bool gicv3_nolpi;
1015
1016static int __init gicv3_nolpi_cfg(char *buf)
1017{
1018 return strtobool(buf, &gicv3_nolpi);
1019}
1020early_param("irqchip.gicv3_nolpi", gicv3_nolpi_cfg);
1021
Marc Zyngierda33f312014-11-24 14:35:18 +00001022static int gic_dist_supports_lpis(void)
1023{
Marc Zyngierd38a71c2018-07-27 14:51:04 +01001024 return (IS_ENABLED(CONFIG_ARM_GIC_V3_ITS) &&
1025 !!(readl_relaxed(gic_data.dist_base + GICD_TYPER) & GICD_TYPER_LPIS) &&
1026 !gicv3_nolpi);
Marc Zyngierda33f312014-11-24 14:35:18 +00001027}
1028
Marc Zyngier021f6532014-06-30 16:01:31 +01001029static void gic_cpu_init(void)
1030{
1031 void __iomem *rbase;
Marc Zyngier1a60e1e2019-07-18 11:15:14 +01001032 int i;
Marc Zyngier021f6532014-06-30 16:01:31 +01001033
1034 /* Register ourselves with the rest of the world */
1035 if (gic_populate_rdist())
1036 return;
1037
Sudeep Hollaa2c22512014-08-26 16:03:34 +01001038 gic_enable_redist(true);
Marc Zyngier021f6532014-06-30 16:01:31 +01001039
Marc Zyngierad5a78d2019-07-25 15:30:51 +01001040 WARN((gic_data.ppi_nr > 16 || GIC_ESPI_NR != 0) &&
1041 !(gic_read_ctlr() & ICC_CTLR_EL1_ExtRange),
1042 "Distributor has extended ranges, but CPU%d doesn't\n",
1043 smp_processor_id());
1044
Marc Zyngier021f6532014-06-30 16:01:31 +01001045 rbase = gic_data_rdist_sgi_base();
1046
Marc Zyngier7c9b9732016-05-06 19:41:56 +01001047 /* Configure SGIs/PPIs as non-secure Group-1 */
Marc Zyngier1a60e1e2019-07-18 11:15:14 +01001048 for (i = 0; i < gic_data.ppi_nr + 16; i += 32)
1049 writel_relaxed(~0, rbase + GICR_IGROUPR0 + i / 8);
Marc Zyngier7c9b9732016-05-06 19:41:56 +01001050
Marc Zyngier1a60e1e2019-07-18 11:15:14 +01001051 gic_cpu_config(rbase, gic_data.ppi_nr + 16, gic_redist_wait_for_rwp);
Marc Zyngier021f6532014-06-30 16:01:31 +01001052
Sudeep Holla3708d522014-08-26 16:03:35 +01001053 /* initialise system registers */
1054 gic_cpu_sys_reg_init();
Marc Zyngier021f6532014-06-30 16:01:31 +01001055}
1056
1057#ifdef CONFIG_SMP
Marc Zyngier021f6532014-06-30 16:01:31 +01001058
Shanker Donthinenieda0d042017-10-06 10:24:00 -05001059#define MPIDR_TO_SGI_RS(mpidr) (MPIDR_RS(mpidr) << ICC_SGI1R_RS_SHIFT)
1060#define MPIDR_TO_SGI_CLUSTER_ID(mpidr) ((mpidr) & ~0xFUL)
1061
Richard Cochran6670a6d2016-07-13 17:16:05 +00001062static int gic_starting_cpu(unsigned int cpu)
1063{
1064 gic_cpu_init();
Marc Zyngierd38a71c2018-07-27 14:51:04 +01001065
1066 if (gic_dist_supports_lpis())
1067 its_cpu_init();
1068
Richard Cochran6670a6d2016-07-13 17:16:05 +00001069 return 0;
1070}
Marc Zyngier021f6532014-06-30 16:01:31 +01001071
1072static u16 gic_compute_target_list(int *base_cpu, const struct cpumask *mask,
Jean-Philippe Bruckerf6c86a42015-10-01 13:47:15 +01001073 unsigned long cluster_id)
Marc Zyngier021f6532014-06-30 16:01:31 +01001074{
James Morse727653d2016-09-19 18:29:15 +01001075 int next_cpu, cpu = *base_cpu;
Jean-Philippe Bruckerf6c86a42015-10-01 13:47:15 +01001076 unsigned long mpidr = cpu_logical_map(cpu);
Marc Zyngier021f6532014-06-30 16:01:31 +01001077 u16 tlist = 0;
1078
1079 while (cpu < nr_cpu_ids) {
Marc Zyngier021f6532014-06-30 16:01:31 +01001080 tlist |= 1 << (mpidr & 0xf);
1081
James Morse727653d2016-09-19 18:29:15 +01001082 next_cpu = cpumask_next(cpu, mask);
1083 if (next_cpu >= nr_cpu_ids)
Marc Zyngier021f6532014-06-30 16:01:31 +01001084 goto out;
James Morse727653d2016-09-19 18:29:15 +01001085 cpu = next_cpu;
Marc Zyngier021f6532014-06-30 16:01:31 +01001086
1087 mpidr = cpu_logical_map(cpu);
1088
Shanker Donthinenieda0d042017-10-06 10:24:00 -05001089 if (cluster_id != MPIDR_TO_SGI_CLUSTER_ID(mpidr)) {
Marc Zyngier021f6532014-06-30 16:01:31 +01001090 cpu--;
1091 goto out;
1092 }
1093 }
1094out:
1095 *base_cpu = cpu;
1096 return tlist;
1097}
1098
Andre Przywara7e580272014-11-12 13:46:06 +00001099#define MPIDR_TO_SGI_AFFINITY(cluster_id, level) \
1100 (MPIDR_AFFINITY_LEVEL(cluster_id, level) \
1101 << ICC_SGI1R_AFFINITY_## level ##_SHIFT)
1102
Marc Zyngier021f6532014-06-30 16:01:31 +01001103static void gic_send_sgi(u64 cluster_id, u16 tlist, unsigned int irq)
1104{
1105 u64 val;
1106
Andre Przywara7e580272014-11-12 13:46:06 +00001107 val = (MPIDR_TO_SGI_AFFINITY(cluster_id, 3) |
1108 MPIDR_TO_SGI_AFFINITY(cluster_id, 2) |
1109 irq << ICC_SGI1R_SGI_ID_SHIFT |
1110 MPIDR_TO_SGI_AFFINITY(cluster_id, 1) |
Shanker Donthinenieda0d042017-10-06 10:24:00 -05001111 MPIDR_TO_SGI_RS(cluster_id) |
Andre Przywara7e580272014-11-12 13:46:06 +00001112 tlist << ICC_SGI1R_TARGET_LIST_SHIFT);
Marc Zyngier021f6532014-06-30 16:01:31 +01001113
Mark Salterb6dd4d82018-02-02 09:20:29 -05001114 pr_devel("CPU%d: ICC_SGI1R_EL1 %llx\n", smp_processor_id(), val);
Marc Zyngier021f6532014-06-30 16:01:31 +01001115 gic_write_sgi1r(val);
1116}
1117
1118static void gic_raise_softirq(const struct cpumask *mask, unsigned int irq)
1119{
1120 int cpu;
1121
1122 if (WARN_ON(irq >= 16))
1123 return;
1124
1125 /*
1126 * Ensure that stores to Normal memory are visible to the
1127 * other CPUs before issuing the IPI.
1128 */
Shanker Donthineni21ec30c2018-01-31 18:03:42 -06001129 wmb();
Marc Zyngier021f6532014-06-30 16:01:31 +01001130
Rusty Russellf9b531f2015-03-05 10:49:16 +10301131 for_each_cpu(cpu, mask) {
Shanker Donthinenieda0d042017-10-06 10:24:00 -05001132 u64 cluster_id = MPIDR_TO_SGI_CLUSTER_ID(cpu_logical_map(cpu));
Marc Zyngier021f6532014-06-30 16:01:31 +01001133 u16 tlist;
1134
1135 tlist = gic_compute_target_list(&cpu, mask, cluster_id);
1136 gic_send_sgi(cluster_id, tlist, irq);
1137 }
1138
1139 /* Force the above writes to ICC_SGI1R_EL1 to be executed */
1140 isb();
1141}
1142
1143static void gic_smp_init(void)
1144{
1145 set_smp_cross_call(gic_raise_softirq);
Thomas Gleixner6896bcd2016-12-21 20:19:56 +01001146 cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_GIC_STARTING,
Thomas Gleixner73c1b412016-12-21 20:19:54 +01001147 "irqchip/arm/gicv3:starting",
1148 gic_starting_cpu, NULL);
Marc Zyngier021f6532014-06-30 16:01:31 +01001149}
1150
1151static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
1152 bool force)
1153{
Suzuki K Poulose65a30f82017-07-04 10:56:35 +01001154 unsigned int cpu;
Marc Zyngiere91b0362019-07-16 14:41:40 +01001155 u32 offset, index;
Marc Zyngier021f6532014-06-30 16:01:31 +01001156 void __iomem *reg;
1157 int enabled;
1158 u64 val;
1159
Suzuki K Poulose65a30f82017-07-04 10:56:35 +01001160 if (force)
1161 cpu = cpumask_first(mask_val);
1162 else
1163 cpu = cpumask_any_and(mask_val, cpu_online_mask);
1164
Suzuki K Poulose866d7c12017-06-30 10:58:28 +01001165 if (cpu >= nr_cpu_ids)
1166 return -EINVAL;
1167
Marc Zyngier021f6532014-06-30 16:01:31 +01001168 if (gic_irq_in_rdist(d))
1169 return -EINVAL;
1170
1171 /* If interrupt was enabled, disable it first */
1172 enabled = gic_peek_irq(d, GICD_ISENABLER);
1173 if (enabled)
1174 gic_mask_irq(d);
1175
Marc Zyngiere91b0362019-07-16 14:41:40 +01001176 offset = convert_offset_index(d, GICD_IROUTER, &index);
1177 reg = gic_dist_base(d) + offset + (index * 8);
Marc Zyngier021f6532014-06-30 16:01:31 +01001178 val = gic_mpidr_to_affinity(cpu_logical_map(cpu));
1179
Jean-Philippe Brucker72c97122015-10-01 13:47:16 +01001180 gic_write_irouter(val, reg);
Marc Zyngier021f6532014-06-30 16:01:31 +01001181
1182 /*
1183 * If the interrupt was enabled, enabled it again. Otherwise,
1184 * just wait for the distributor to have digested our changes.
1185 */
1186 if (enabled)
1187 gic_unmask_irq(d);
1188 else
1189 gic_dist_wait_for_rwp();
1190
Marc Zyngier956ae912017-08-18 09:39:17 +01001191 irq_data_update_effective_affinity(d, cpumask_of(cpu));
1192
Antoine Tenart0fc6fa22016-02-19 16:22:43 +01001193 return IRQ_SET_MASK_OK_DONE;
Marc Zyngier021f6532014-06-30 16:01:31 +01001194}
1195#else
1196#define gic_set_affinity NULL
1197#define gic_smp_init() do { } while(0)
1198#endif
1199
Sudeep Holla3708d522014-08-26 16:03:35 +01001200#ifdef CONFIG_CPU_PM
1201static int gic_cpu_pm_notifier(struct notifier_block *self,
1202 unsigned long cmd, void *v)
1203{
1204 if (cmd == CPU_PM_EXIT) {
Sudeep Hollaccd94322016-08-17 13:49:19 +01001205 if (gic_dist_security_disabled())
1206 gic_enable_redist(true);
Sudeep Holla3708d522014-08-26 16:03:35 +01001207 gic_cpu_sys_reg_init();
Sudeep Hollaccd94322016-08-17 13:49:19 +01001208 } else if (cmd == CPU_PM_ENTER && gic_dist_security_disabled()) {
Sudeep Holla3708d522014-08-26 16:03:35 +01001209 gic_write_grpen1(0);
1210 gic_enable_redist(false);
1211 }
1212 return NOTIFY_OK;
1213}
1214
1215static struct notifier_block gic_cpu_pm_notifier_block = {
1216 .notifier_call = gic_cpu_pm_notifier,
1217};
1218
1219static void gic_cpu_pm_init(void)
1220{
1221 cpu_pm_register_notifier(&gic_cpu_pm_notifier_block);
1222}
1223
1224#else
1225static inline void gic_cpu_pm_init(void) { }
1226#endif /* CONFIG_CPU_PM */
1227
Marc Zyngier021f6532014-06-30 16:01:31 +01001228static struct irq_chip gic_chip = {
1229 .name = "GICv3",
1230 .irq_mask = gic_mask_irq,
1231 .irq_unmask = gic_unmask_irq,
1232 .irq_eoi = gic_eoi_irq,
1233 .irq_set_type = gic_set_type,
1234 .irq_set_affinity = gic_set_affinity,
Marc Zyngierb594c6e2015-03-18 11:01:24 +00001235 .irq_get_irqchip_state = gic_irq_get_irqchip_state,
1236 .irq_set_irqchip_state = gic_irq_set_irqchip_state,
Julien Thierry101b35f2019-01-31 14:58:59 +00001237 .irq_nmi_setup = gic_irq_nmi_setup,
1238 .irq_nmi_teardown = gic_irq_nmi_teardown,
Marc Zyngier4110b5c2018-08-17 09:18:01 +01001239 .flags = IRQCHIP_SET_TYPE_MASKED |
1240 IRQCHIP_SKIP_SET_WAKE |
1241 IRQCHIP_MASK_ON_SUSPEND,
Marc Zyngier021f6532014-06-30 16:01:31 +01001242};
1243
Marc Zyngier0b6a3da2015-08-26 17:00:42 +01001244static struct irq_chip gic_eoimode1_chip = {
1245 .name = "GICv3",
1246 .irq_mask = gic_eoimode1_mask_irq,
1247 .irq_unmask = gic_unmask_irq,
1248 .irq_eoi = gic_eoimode1_eoi_irq,
1249 .irq_set_type = gic_set_type,
1250 .irq_set_affinity = gic_set_affinity,
1251 .irq_get_irqchip_state = gic_irq_get_irqchip_state,
1252 .irq_set_irqchip_state = gic_irq_set_irqchip_state,
Marc Zyngier530bf352015-08-26 17:00:43 +01001253 .irq_set_vcpu_affinity = gic_irq_set_vcpu_affinity,
Julien Thierry101b35f2019-01-31 14:58:59 +00001254 .irq_nmi_setup = gic_irq_nmi_setup,
1255 .irq_nmi_teardown = gic_irq_nmi_teardown,
Marc Zyngier4110b5c2018-08-17 09:18:01 +01001256 .flags = IRQCHIP_SET_TYPE_MASKED |
1257 IRQCHIP_SKIP_SET_WAKE |
1258 IRQCHIP_MASK_ON_SUSPEND,
Marc Zyngier0b6a3da2015-08-26 17:00:42 +01001259};
1260
Marc Zyngier021f6532014-06-30 16:01:31 +01001261static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq,
1262 irq_hw_number_t hw)
1263{
Marc Zyngier0b6a3da2015-08-26 17:00:42 +01001264 struct irq_chip *chip = &gic_chip;
1265
Davidlohr Buesod01d3272018-03-26 14:09:25 -07001266 if (static_branch_likely(&supports_deactivate_key))
Marc Zyngier0b6a3da2015-08-26 17:00:42 +01001267 chip = &gic_eoimode1_chip;
1268
Marc Zyngiere91b0362019-07-16 14:41:40 +01001269 switch (__get_intid_range(hw)) {
1270 case PPI_RANGE:
Marc Zyngier5f51f802019-07-18 13:19:25 +01001271 case EPPI_RANGE:
Marc Zyngier021f6532014-06-30 16:01:31 +01001272 irq_set_percpu_devid(irq);
Marc Zyngier0b6a3da2015-08-26 17:00:42 +01001273 irq_domain_set_info(d, irq, hw, chip, d->host_data,
Marc Zyngier443acc42014-11-24 14:35:09 +00001274 handle_percpu_devid_irq, NULL, NULL);
Rob Herringd17cab42015-08-29 18:01:22 -05001275 irq_set_status_flags(irq, IRQ_NOAUTOEN);
Marc Zyngiere91b0362019-07-16 14:41:40 +01001276 break;
1277
1278 case SPI_RANGE:
Marc Zyngier211bddd2019-07-16 15:17:31 +01001279 case ESPI_RANGE:
Marc Zyngier0b6a3da2015-08-26 17:00:42 +01001280 irq_domain_set_info(d, irq, hw, chip, d->host_data,
Marc Zyngier443acc42014-11-24 14:35:09 +00001281 handle_fasteoi_irq, NULL, NULL);
Rob Herringd17cab42015-08-29 18:01:22 -05001282 irq_set_probe(irq);
Marc Zyngier956ae912017-08-18 09:39:17 +01001283 irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(irq)));
Marc Zyngiere91b0362019-07-16 14:41:40 +01001284 break;
1285
1286 case LPI_RANGE:
Marc Zyngierda33f312014-11-24 14:35:18 +00001287 if (!gic_dist_supports_lpis())
1288 return -EPERM;
Marc Zyngier0b6a3da2015-08-26 17:00:42 +01001289 irq_domain_set_info(d, irq, hw, chip, d->host_data,
Marc Zyngierda33f312014-11-24 14:35:18 +00001290 handle_fasteoi_irq, NULL, NULL);
Marc Zyngiere91b0362019-07-16 14:41:40 +01001291 break;
1292
1293 default:
1294 return -EPERM;
Marc Zyngierda33f312014-11-24 14:35:18 +00001295 }
1296
Marc Zyngier021f6532014-06-30 16:01:31 +01001297 return 0;
1298}
1299
Marc Zyngier65da7d12018-03-20 13:44:09 +00001300#define GIC_IRQ_TYPE_PARTITION (GIC_IRQ_TYPE_LPI + 1)
1301
Marc Zyngierf833f572015-10-13 12:51:33 +01001302static int gic_irq_domain_translate(struct irq_domain *d,
1303 struct irq_fwspec *fwspec,
1304 unsigned long *hwirq,
1305 unsigned int *type)
Marc Zyngier021f6532014-06-30 16:01:31 +01001306{
Marc Zyngierf833f572015-10-13 12:51:33 +01001307 if (is_of_node(fwspec->fwnode)) {
1308 if (fwspec->param_count < 3)
1309 return -EINVAL;
Marc Zyngier021f6532014-06-30 16:01:31 +01001310
Marc Zyngierdb8c70e2015-10-14 12:27:16 +01001311 switch (fwspec->param[0]) {
1312 case 0: /* SPI */
1313 *hwirq = fwspec->param[1] + 32;
1314 break;
1315 case 1: /* PPI */
1316 *hwirq = fwspec->param[1] + 16;
1317 break;
Marc Zyngier211bddd2019-07-16 15:17:31 +01001318 case 2: /* ESPI */
1319 *hwirq = fwspec->param[1] + ESPI_BASE_INTID;
1320 break;
Marc Zyngier5f51f802019-07-18 13:19:25 +01001321 case 3: /* EPPI */
1322 *hwirq = fwspec->param[1] + EPPI_BASE_INTID;
1323 break;
Marc Zyngierdb8c70e2015-10-14 12:27:16 +01001324 case GIC_IRQ_TYPE_LPI: /* LPI */
1325 *hwirq = fwspec->param[1];
1326 break;
Marc Zyngier5f51f802019-07-18 13:19:25 +01001327 case GIC_IRQ_TYPE_PARTITION:
1328 *hwirq = fwspec->param[1];
1329 if (fwspec->param[1] >= 16)
1330 *hwirq += EPPI_BASE_INTID - 16;
1331 else
1332 *hwirq += 16;
1333 break;
Marc Zyngierdb8c70e2015-10-14 12:27:16 +01001334 default:
1335 return -EINVAL;
1336 }
Marc Zyngierf833f572015-10-13 12:51:33 +01001337
1338 *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK;
Marc Zyngier6ef63862018-03-16 14:35:17 +00001339
Marc Zyngier65da7d12018-03-20 13:44:09 +00001340 /*
1341 * Make it clear that broken DTs are... broken.
1342 * Partitionned PPIs are an unfortunate exception.
1343 */
1344 WARN_ON(*type == IRQ_TYPE_NONE &&
1345 fwspec->param[0] != GIC_IRQ_TYPE_PARTITION);
Marc Zyngierf833f572015-10-13 12:51:33 +01001346 return 0;
Marc Zyngier021f6532014-06-30 16:01:31 +01001347 }
1348
Tomasz Nowickiffa7d612016-01-19 14:11:15 +01001349 if (is_fwnode_irqchip(fwspec->fwnode)) {
1350 if(fwspec->param_count != 2)
1351 return -EINVAL;
1352
1353 *hwirq = fwspec->param[0];
1354 *type = fwspec->param[1];
Marc Zyngier6ef63862018-03-16 14:35:17 +00001355
1356 WARN_ON(*type == IRQ_TYPE_NONE);
Tomasz Nowickiffa7d612016-01-19 14:11:15 +01001357 return 0;
1358 }
1359
Marc Zyngierf833f572015-10-13 12:51:33 +01001360 return -EINVAL;
Marc Zyngier021f6532014-06-30 16:01:31 +01001361}
1362
Marc Zyngier443acc42014-11-24 14:35:09 +00001363static int gic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
1364 unsigned int nr_irqs, void *arg)
1365{
1366 int i, ret;
1367 irq_hw_number_t hwirq;
1368 unsigned int type = IRQ_TYPE_NONE;
Marc Zyngierf833f572015-10-13 12:51:33 +01001369 struct irq_fwspec *fwspec = arg;
Marc Zyngier443acc42014-11-24 14:35:09 +00001370
Marc Zyngierf833f572015-10-13 12:51:33 +01001371 ret = gic_irq_domain_translate(domain, fwspec, &hwirq, &type);
Marc Zyngier443acc42014-11-24 14:35:09 +00001372 if (ret)
1373 return ret;
1374
Suzuki K Poulose63c16c62017-07-04 10:56:33 +01001375 for (i = 0; i < nr_irqs; i++) {
1376 ret = gic_irq_domain_map(domain, virq + i, hwirq + i);
1377 if (ret)
1378 return ret;
1379 }
Marc Zyngier443acc42014-11-24 14:35:09 +00001380
1381 return 0;
1382}
1383
1384static void gic_irq_domain_free(struct irq_domain *domain, unsigned int virq,
1385 unsigned int nr_irqs)
1386{
1387 int i;
1388
1389 for (i = 0; i < nr_irqs; i++) {
1390 struct irq_data *d = irq_domain_get_irq_data(domain, virq + i);
1391 irq_set_handler(virq + i, NULL);
1392 irq_domain_reset_irq_data(d);
1393 }
1394}
1395
Marc Zyngiere3825ba2016-04-11 09:57:54 +01001396static int gic_irq_domain_select(struct irq_domain *d,
1397 struct irq_fwspec *fwspec,
1398 enum irq_domain_bus_token bus_token)
1399{
1400 /* Not for us */
1401 if (fwspec->fwnode != d->fwnode)
1402 return 0;
1403
1404 /* If this is not DT, then we have a single domain */
1405 if (!is_of_node(fwspec->fwnode))
1406 return 1;
1407
1408 /*
1409 * If this is a PPI and we have a 4th (non-null) parameter,
1410 * then we need to match the partition domain.
1411 */
1412 if (fwspec->param_count >= 4 &&
Marc Zyngier52085d32019-07-18 13:05:17 +01001413 fwspec->param[0] == 1 && fwspec->param[3] != 0 &&
1414 gic_data.ppi_descs)
Marc Zyngiere3825ba2016-04-11 09:57:54 +01001415 return d == partition_get_domain(gic_data.ppi_descs[fwspec->param[1]]);
1416
1417 return d == gic_data.domain;
1418}
1419
Marc Zyngier021f6532014-06-30 16:01:31 +01001420static const struct irq_domain_ops gic_irq_domain_ops = {
Marc Zyngierf833f572015-10-13 12:51:33 +01001421 .translate = gic_irq_domain_translate,
Marc Zyngier443acc42014-11-24 14:35:09 +00001422 .alloc = gic_irq_domain_alloc,
1423 .free = gic_irq_domain_free,
Marc Zyngiere3825ba2016-04-11 09:57:54 +01001424 .select = gic_irq_domain_select,
1425};
1426
1427static int partition_domain_translate(struct irq_domain *d,
1428 struct irq_fwspec *fwspec,
1429 unsigned long *hwirq,
1430 unsigned int *type)
1431{
1432 struct device_node *np;
1433 int ret;
1434
Marc Zyngier52085d32019-07-18 13:05:17 +01001435 if (!gic_data.ppi_descs)
1436 return -ENOMEM;
1437
Marc Zyngiere3825ba2016-04-11 09:57:54 +01001438 np = of_find_node_by_phandle(fwspec->param[3]);
1439 if (WARN_ON(!np))
1440 return -EINVAL;
1441
1442 ret = partition_translate_id(gic_data.ppi_descs[fwspec->param[1]],
1443 of_node_to_fwnode(np));
1444 if (ret < 0)
1445 return ret;
1446
1447 *hwirq = ret;
1448 *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK;
1449
1450 return 0;
1451}
1452
1453static const struct irq_domain_ops partition_domain_ops = {
1454 .translate = partition_domain_translate,
1455 .select = gic_irq_domain_select,
Marc Zyngier021f6532014-06-30 16:01:31 +01001456};
1457
Srinivas Kandagatla9c8114c2018-12-10 13:56:32 +00001458static bool gic_enable_quirk_msm8996(void *data)
1459{
1460 struct gic_chip_data *d = data;
1461
1462 d->flags |= FLAGS_WORKAROUND_GICR_WAKER_MSM8996;
1463
1464 return true;
1465}
1466
Marc Zyngier7f2481b2019-07-31 17:29:33 +01001467static bool gic_enable_quirk_hip06_07(void *data)
1468{
1469 struct gic_chip_data *d = data;
1470
1471 /*
1472 * HIP06 GICD_IIDR clashes with GIC-600 product number (despite
1473 * not being an actual ARM implementation). The saving grace is
1474 * that GIC-600 doesn't have ESPI, so nothing to do in that case.
1475 * HIP07 doesn't even have a proper IIDR, and still pretends to
1476 * have ESPI. In both cases, put them right.
1477 */
1478 if (d->rdists.gicd_typer & GICD_TYPER_ESPI) {
1479 /* Zero both ESPI and the RES0 field next to it... */
1480 d->rdists.gicd_typer &= ~GENMASK(9, 8);
1481 return true;
1482 }
1483
1484 return false;
1485}
1486
1487static const struct gic_quirk gic_quirks[] = {
1488 {
1489 .desc = "GICv3: Qualcomm MSM8996 broken firmware",
1490 .compatible = "qcom,msm8996-gic-v3",
1491 .init = gic_enable_quirk_msm8996,
1492 },
1493 {
1494 .desc = "GICv3: HIP06 erratum 161010803",
1495 .iidr = 0x0204043b,
1496 .mask = 0xffffffff,
1497 .init = gic_enable_quirk_hip06_07,
1498 },
1499 {
1500 .desc = "GICv3: HIP07 erratum 161010803",
1501 .iidr = 0x00000000,
1502 .mask = 0xffffffff,
1503 .init = gic_enable_quirk_hip06_07,
1504 },
1505 {
1506 }
1507};
1508
Julien Thierryd98d0a92019-01-31 14:58:57 +00001509static void gic_enable_nmi_support(void)
1510{
Julien Thierry101b35f2019-01-31 14:58:59 +00001511 int i;
1512
Marc Zyngier81a43272019-07-18 12:53:05 +01001513 if (!gic_prio_masking_enabled())
1514 return;
1515
1516 if (gic_has_group0() && !gic_dist_security_disabled()) {
1517 pr_warn("SCR_EL3.FIQ is cleared, cannot enable use of pseudo-NMIs\n");
1518 return;
1519 }
1520
1521 ppi_nmi_refs = kcalloc(gic_data.ppi_nr, sizeof(*ppi_nmi_refs), GFP_KERNEL);
1522 if (!ppi_nmi_refs)
1523 return;
1524
1525 for (i = 0; i < gic_data.ppi_nr; i++)
Julien Thierry101b35f2019-01-31 14:58:59 +00001526 refcount_set(&ppi_nmi_refs[i], 0);
1527
Marc Zyngierf2266502019-10-02 10:06:12 +01001528 /*
1529 * Linux itself doesn't use 1:N distribution, so has no need to
1530 * set PMHE. The only reason to have it set is if EL3 requires it
1531 * (and we can't change it).
1532 */
1533 if (gic_read_ctlr() & ICC_CTLR_EL1_PMHE_MASK)
1534 static_branch_enable(&gic_pmr_sync);
1535
1536 pr_info("%s ICC_PMR_EL1 synchronisation\n",
1537 static_branch_unlikely(&gic_pmr_sync) ? "Forcing" : "Relaxing");
1538
Julien Thierryd98d0a92019-01-31 14:58:57 +00001539 static_branch_enable(&supports_pseudo_nmis);
Julien Thierry101b35f2019-01-31 14:58:59 +00001540
1541 if (static_branch_likely(&supports_deactivate_key))
1542 gic_eoimode1_chip.flags |= IRQCHIP_SUPPORTS_NMI;
1543 else
1544 gic_chip.flags |= IRQCHIP_SUPPORTS_NMI;
Julien Thierryd98d0a92019-01-31 14:58:57 +00001545}
1546
Tomasz Nowickidb57d742016-01-19 14:11:14 +01001547static int __init gic_init_bases(void __iomem *dist_base,
1548 struct redist_region *rdist_regs,
1549 u32 nr_redist_regions,
1550 u64 redist_stride,
1551 struct fwnode_handle *handle)
1552{
Tomasz Nowickidb57d742016-01-19 14:11:14 +01001553 u32 typer;
Tomasz Nowickidb57d742016-01-19 14:11:14 +01001554 int err;
1555
1556 if (!is_hyp_mode_available())
Davidlohr Buesod01d3272018-03-26 14:09:25 -07001557 static_branch_disable(&supports_deactivate_key);
Tomasz Nowickidb57d742016-01-19 14:11:14 +01001558
Davidlohr Buesod01d3272018-03-26 14:09:25 -07001559 if (static_branch_likely(&supports_deactivate_key))
Tomasz Nowickidb57d742016-01-19 14:11:14 +01001560 pr_info("GIC: Using split EOI/Deactivate mode\n");
1561
Marc Zyngiere3825ba2016-04-11 09:57:54 +01001562 gic_data.fwnode = handle;
Tomasz Nowickidb57d742016-01-19 14:11:14 +01001563 gic_data.dist_base = dist_base;
1564 gic_data.redist_regions = rdist_regs;
1565 gic_data.nr_redist_regions = nr_redist_regions;
1566 gic_data.redist_stride = redist_stride;
1567
Tomasz Nowickidb57d742016-01-19 14:11:14 +01001568 /*
1569 * Find out how many interrupts are supported.
Tomasz Nowickidb57d742016-01-19 14:11:14 +01001570 */
1571 typer = readl_relaxed(gic_data.dist_base + GICD_TYPER);
Marc Zyngiera4f9edb2018-05-30 17:29:52 +01001572 gic_data.rdists.gicd_typer = typer;
Marc Zyngier7f2481b2019-07-31 17:29:33 +01001573
1574 gic_enable_quirks(readl_relaxed(gic_data.dist_base + GICD_IIDR),
1575 gic_quirks, &gic_data);
1576
Marc Zyngier211bddd2019-07-16 15:17:31 +01001577 pr_info("%d SPIs implemented\n", GIC_LINE_NR - 32);
1578 pr_info("%d Extended SPIs implemented\n", GIC_ESPI_NR);
Tomasz Nowickidb57d742016-01-19 14:11:14 +01001579 gic_data.domain = irq_domain_create_tree(handle, &gic_irq_domain_ops,
1580 &gic_data);
Marc Zyngierb2425b52018-05-08 13:14:35 +01001581 irq_domain_update_bus_token(gic_data.domain, DOMAIN_BUS_WIRED);
Tomasz Nowickidb57d742016-01-19 14:11:14 +01001582 gic_data.rdists.rdist = alloc_percpu(typeof(*gic_data.rdists.rdist));
Marc Zyngierb25319d2019-12-24 11:10:24 +00001583 gic_data.rdists.has_rvpeid = true;
Marc Zyngier0edc23e2016-12-19 17:01:52 +00001584 gic_data.rdists.has_vlpis = true;
1585 gic_data.rdists.has_direct_lpi = true;
Tomasz Nowickidb57d742016-01-19 14:11:14 +01001586
1587 if (WARN_ON(!gic_data.domain) || WARN_ON(!gic_data.rdists.rdist)) {
1588 err = -ENOMEM;
1589 goto out_free;
1590 }
1591
Shanker Donthinenieda0d042017-10-06 10:24:00 -05001592 gic_data.has_rss = !!(typer & GICD_TYPER_RSS);
1593 pr_info("Distributor has %sRange Selector support\n",
1594 gic_data.has_rss ? "" : "no ");
1595
Marc Zyngier50528752018-05-08 13:14:36 +01001596 if (typer & GICD_TYPER_MBIS) {
1597 err = mbi_init(handle, gic_data.domain);
1598 if (err)
1599 pr_err("Failed to initialize MBIs\n");
1600 }
1601
Tomasz Nowickidb57d742016-01-19 14:11:14 +01001602 set_handle_irq(gic_handle_irq);
1603
Marc Zyngier1a60e1e2019-07-18 11:15:14 +01001604 gic_update_rdist_properties();
Marc Zyngier0edc23e2016-12-19 17:01:52 +00001605
Tomasz Nowickidb57d742016-01-19 14:11:14 +01001606 gic_smp_init();
1607 gic_dist_init();
1608 gic_cpu_init();
1609 gic_cpu_pm_init();
1610
Marc Zyngierd38a71c2018-07-27 14:51:04 +01001611 if (gic_dist_supports_lpis()) {
1612 its_init(handle, &gic_data.rdists, gic_data.domain);
1613 its_cpu_init();
Zeev Zilberman90b4c552019-06-10 13:52:01 +03001614 } else {
1615 if (IS_ENABLED(CONFIG_ARM_GIC_V2M))
1616 gicv2m_init(handle, gic_data.domain);
Marc Zyngierd38a71c2018-07-27 14:51:04 +01001617 }
1618
Marc Zyngier81a43272019-07-18 12:53:05 +01001619 gic_enable_nmi_support();
Julien Thierryd98d0a92019-01-31 14:58:57 +00001620
Tomasz Nowickidb57d742016-01-19 14:11:14 +01001621 return 0;
1622
1623out_free:
1624 if (gic_data.domain)
1625 irq_domain_remove(gic_data.domain);
1626 free_percpu(gic_data.rdists.rdist);
1627 return err;
1628}
1629
1630static int __init gic_validate_dist_version(void __iomem *dist_base)
1631{
1632 u32 reg = readl_relaxed(dist_base + GICD_PIDR2) & GIC_PIDR2_ARCH_MASK;
1633
1634 if (reg != GIC_PIDR2_ARCH_GICv3 && reg != GIC_PIDR2_ARCH_GICv4)
1635 return -ENODEV;
1636
1637 return 0;
1638}
1639
Marc Zyngiere3825ba2016-04-11 09:57:54 +01001640/* Create all possible partitions at boot time */
Linus Torvalds7beaa242016-05-19 11:27:09 -07001641static void __init gic_populate_ppi_partitions(struct device_node *gic_node)
Marc Zyngiere3825ba2016-04-11 09:57:54 +01001642{
1643 struct device_node *parts_node, *child_part;
1644 int part_idx = 0, i;
1645 int nr_parts;
1646 struct partition_affinity *parts;
1647
Johan Hovold00ee9a12017-11-11 17:51:25 +01001648 parts_node = of_get_child_by_name(gic_node, "ppi-partitions");
Marc Zyngiere3825ba2016-04-11 09:57:54 +01001649 if (!parts_node)
1650 return;
1651
Marc Zyngier52085d32019-07-18 13:05:17 +01001652 gic_data.ppi_descs = kcalloc(gic_data.ppi_nr, sizeof(*gic_data.ppi_descs), GFP_KERNEL);
1653 if (!gic_data.ppi_descs)
1654 return;
1655
Marc Zyngiere3825ba2016-04-11 09:57:54 +01001656 nr_parts = of_get_child_count(parts_node);
1657
1658 if (!nr_parts)
Johan Hovold00ee9a12017-11-11 17:51:25 +01001659 goto out_put_node;
Marc Zyngiere3825ba2016-04-11 09:57:54 +01001660
Kees Cook6396bb22018-06-12 14:03:40 -07001661 parts = kcalloc(nr_parts, sizeof(*parts), GFP_KERNEL);
Marc Zyngiere3825ba2016-04-11 09:57:54 +01001662 if (WARN_ON(!parts))
Johan Hovold00ee9a12017-11-11 17:51:25 +01001663 goto out_put_node;
Marc Zyngiere3825ba2016-04-11 09:57:54 +01001664
1665 for_each_child_of_node(parts_node, child_part) {
1666 struct partition_affinity *part;
1667 int n;
1668
1669 part = &parts[part_idx];
1670
1671 part->partition_id = of_node_to_fwnode(child_part);
1672
Rob Herring2ef790d2018-08-27 19:56:15 -05001673 pr_info("GIC: PPI partition %pOFn[%d] { ",
1674 child_part, part_idx);
Marc Zyngiere3825ba2016-04-11 09:57:54 +01001675
1676 n = of_property_count_elems_of_size(child_part, "affinity",
1677 sizeof(u32));
1678 WARN_ON(n <= 0);
1679
1680 for (i = 0; i < n; i++) {
1681 int err, cpu;
1682 u32 cpu_phandle;
1683 struct device_node *cpu_node;
1684
1685 err = of_property_read_u32_index(child_part, "affinity",
1686 i, &cpu_phandle);
1687 if (WARN_ON(err))
1688 continue;
1689
1690 cpu_node = of_find_node_by_phandle(cpu_phandle);
1691 if (WARN_ON(!cpu_node))
1692 continue;
1693
Suzuki K Poulosec08ec7d2018-01-02 11:25:29 +00001694 cpu = of_cpu_node_to_id(cpu_node);
1695 if (WARN_ON(cpu < 0))
Marc Zyngiere3825ba2016-04-11 09:57:54 +01001696 continue;
1697
Rob Herringe81f54c2017-07-18 16:43:10 -05001698 pr_cont("%pOF[%d] ", cpu_node, cpu);
Marc Zyngiere3825ba2016-04-11 09:57:54 +01001699
1700 cpumask_set_cpu(cpu, &part->mask);
1701 }
1702
1703 pr_cont("}\n");
1704 part_idx++;
1705 }
1706
Marc Zyngier52085d32019-07-18 13:05:17 +01001707 for (i = 0; i < gic_data.ppi_nr; i++) {
Marc Zyngiere3825ba2016-04-11 09:57:54 +01001708 unsigned int irq;
1709 struct partition_desc *desc;
1710 struct irq_fwspec ppi_fwspec = {
1711 .fwnode = gic_data.fwnode,
1712 .param_count = 3,
1713 .param = {
Marc Zyngier65da7d12018-03-20 13:44:09 +00001714 [0] = GIC_IRQ_TYPE_PARTITION,
Marc Zyngiere3825ba2016-04-11 09:57:54 +01001715 [1] = i,
1716 [2] = IRQ_TYPE_NONE,
1717 },
1718 };
1719
1720 irq = irq_create_fwspec_mapping(&ppi_fwspec);
1721 if (WARN_ON(!irq))
1722 continue;
1723 desc = partition_create_desc(gic_data.fwnode, parts, nr_parts,
1724 irq, &partition_domain_ops);
1725 if (WARN_ON(!desc))
1726 continue;
1727
1728 gic_data.ppi_descs[i] = desc;
1729 }
Johan Hovold00ee9a12017-11-11 17:51:25 +01001730
1731out_put_node:
1732 of_node_put(parts_node);
Marc Zyngiere3825ba2016-04-11 09:57:54 +01001733}
1734
Julien Grall1839e572016-04-11 16:32:57 +01001735static void __init gic_of_setup_kvm_info(struct device_node *node)
1736{
1737 int ret;
1738 struct resource r;
1739 u32 gicv_idx;
1740
1741 gic_v3_kvm_info.type = GIC_V3;
1742
1743 gic_v3_kvm_info.maint_irq = irq_of_parse_and_map(node, 0);
1744 if (!gic_v3_kvm_info.maint_irq)
1745 return;
1746
1747 if (of_property_read_u32(node, "#redistributor-regions",
1748 &gicv_idx))
1749 gicv_idx = 1;
1750
1751 gicv_idx += 3; /* Also skip GICD, GICC, GICH */
1752 ret = of_address_to_resource(node, gicv_idx, &r);
1753 if (!ret)
1754 gic_v3_kvm_info.vcpu = r;
1755
Marc Zyngier4bdf5022017-06-25 14:10:46 +01001756 gic_v3_kvm_info.has_v4 = gic_data.rdists.has_vlpis;
Julien Grall1839e572016-04-11 16:32:57 +01001757 gic_set_kvm_info(&gic_v3_kvm_info);
1758}
1759
Marc Zyngier021f6532014-06-30 16:01:31 +01001760static int __init gic_of_init(struct device_node *node, struct device_node *parent)
1761{
1762 void __iomem *dist_base;
Marc Zyngierf5c14342014-11-24 14:35:10 +00001763 struct redist_region *rdist_regs;
Marc Zyngier021f6532014-06-30 16:01:31 +01001764 u64 redist_stride;
Marc Zyngierf5c14342014-11-24 14:35:10 +00001765 u32 nr_redist_regions;
Tomasz Nowickidb57d742016-01-19 14:11:14 +01001766 int err, i;
Marc Zyngier021f6532014-06-30 16:01:31 +01001767
1768 dist_base = of_iomap(node, 0);
1769 if (!dist_base) {
Rob Herringe81f54c2017-07-18 16:43:10 -05001770 pr_err("%pOF: unable to map gic dist registers\n", node);
Marc Zyngier021f6532014-06-30 16:01:31 +01001771 return -ENXIO;
1772 }
1773
Tomasz Nowickidb57d742016-01-19 14:11:14 +01001774 err = gic_validate_dist_version(dist_base);
1775 if (err) {
Rob Herringe81f54c2017-07-18 16:43:10 -05001776 pr_err("%pOF: no distributor detected, giving up\n", node);
Marc Zyngier021f6532014-06-30 16:01:31 +01001777 goto out_unmap_dist;
1778 }
1779
Marc Zyngierf5c14342014-11-24 14:35:10 +00001780 if (of_property_read_u32(node, "#redistributor-regions", &nr_redist_regions))
1781 nr_redist_regions = 1;
Marc Zyngier021f6532014-06-30 16:01:31 +01001782
Kees Cook6396bb22018-06-12 14:03:40 -07001783 rdist_regs = kcalloc(nr_redist_regions, sizeof(*rdist_regs),
1784 GFP_KERNEL);
Marc Zyngierf5c14342014-11-24 14:35:10 +00001785 if (!rdist_regs) {
Marc Zyngier021f6532014-06-30 16:01:31 +01001786 err = -ENOMEM;
1787 goto out_unmap_dist;
1788 }
1789
Marc Zyngierf5c14342014-11-24 14:35:10 +00001790 for (i = 0; i < nr_redist_regions; i++) {
1791 struct resource res;
1792 int ret;
1793
1794 ret = of_address_to_resource(node, 1 + i, &res);
1795 rdist_regs[i].redist_base = of_iomap(node, 1 + i);
1796 if (ret || !rdist_regs[i].redist_base) {
Rob Herringe81f54c2017-07-18 16:43:10 -05001797 pr_err("%pOF: couldn't map region %d\n", node, i);
Marc Zyngier021f6532014-06-30 16:01:31 +01001798 err = -ENODEV;
1799 goto out_unmap_rdist;
1800 }
Marc Zyngierf5c14342014-11-24 14:35:10 +00001801 rdist_regs[i].phys_base = res.start;
Marc Zyngier021f6532014-06-30 16:01:31 +01001802 }
1803
1804 if (of_property_read_u64(node, "redistributor-stride", &redist_stride))
1805 redist_stride = 0;
1806
Srinivas Kandagatlaf70fdb42018-12-10 13:56:31 +00001807 gic_enable_of_quirks(node, gic_quirks, &gic_data);
1808
Tomasz Nowickidb57d742016-01-19 14:11:14 +01001809 err = gic_init_bases(dist_base, rdist_regs, nr_redist_regions,
1810 redist_stride, &node->fwnode);
Marc Zyngiere3825ba2016-04-11 09:57:54 +01001811 if (err)
1812 goto out_unmap_rdist;
1813
1814 gic_populate_ppi_partitions(node);
Christoffer Dalld33a3c82016-12-06 22:00:52 +01001815
Davidlohr Buesod01d3272018-03-26 14:09:25 -07001816 if (static_branch_likely(&supports_deactivate_key))
Christoffer Dalld33a3c82016-12-06 22:00:52 +01001817 gic_of_setup_kvm_info(node);
Marc Zyngiere3825ba2016-04-11 09:57:54 +01001818 return 0;
Marc Zyngier0b6a3da2015-08-26 17:00:42 +01001819
Marc Zyngier021f6532014-06-30 16:01:31 +01001820out_unmap_rdist:
Marc Zyngierf5c14342014-11-24 14:35:10 +00001821 for (i = 0; i < nr_redist_regions; i++)
1822 if (rdist_regs[i].redist_base)
1823 iounmap(rdist_regs[i].redist_base);
1824 kfree(rdist_regs);
Marc Zyngier021f6532014-06-30 16:01:31 +01001825out_unmap_dist:
1826 iounmap(dist_base);
1827 return err;
1828}
1829
1830IRQCHIP_DECLARE(gic_v3, "arm,gic-v3", gic_of_init);
Tomasz Nowickiffa7d612016-01-19 14:11:15 +01001831
1832#ifdef CONFIG_ACPI
Julien Grall611f0392016-04-11 16:32:56 +01001833static struct
1834{
1835 void __iomem *dist_base;
1836 struct redist_region *redist_regs;
1837 u32 nr_redist_regions;
1838 bool single_redist;
Julien Grall1839e572016-04-11 16:32:57 +01001839 u32 maint_irq;
1840 int maint_irq_mode;
1841 phys_addr_t vcpu_base;
Julien Grall611f0392016-04-11 16:32:56 +01001842} acpi_data __initdata;
Tomasz Nowickib70fb7a2016-01-19 14:11:16 +01001843
1844static void __init
1845gic_acpi_register_redist(phys_addr_t phys_base, void __iomem *redist_base)
1846{
1847 static int count = 0;
1848
Julien Grall611f0392016-04-11 16:32:56 +01001849 acpi_data.redist_regs[count].phys_base = phys_base;
1850 acpi_data.redist_regs[count].redist_base = redist_base;
1851 acpi_data.redist_regs[count].single_redist = acpi_data.single_redist;
Tomasz Nowickib70fb7a2016-01-19 14:11:16 +01001852 count++;
1853}
Tomasz Nowickiffa7d612016-01-19 14:11:15 +01001854
1855static int __init
Keith Busch60574d12019-03-11 14:55:57 -06001856gic_acpi_parse_madt_redist(union acpi_subtable_headers *header,
Tomasz Nowickiffa7d612016-01-19 14:11:15 +01001857 const unsigned long end)
1858{
1859 struct acpi_madt_generic_redistributor *redist =
1860 (struct acpi_madt_generic_redistributor *)header;
1861 void __iomem *redist_base;
Tomasz Nowickiffa7d612016-01-19 14:11:15 +01001862
1863 redist_base = ioremap(redist->base_address, redist->length);
1864 if (!redist_base) {
1865 pr_err("Couldn't map GICR region @%llx\n", redist->base_address);
1866 return -ENOMEM;
1867 }
1868
Tomasz Nowickib70fb7a2016-01-19 14:11:16 +01001869 gic_acpi_register_redist(redist->base_address, redist_base);
Tomasz Nowickiffa7d612016-01-19 14:11:15 +01001870 return 0;
1871}
1872
Tomasz Nowickib70fb7a2016-01-19 14:11:16 +01001873static int __init
Keith Busch60574d12019-03-11 14:55:57 -06001874gic_acpi_parse_madt_gicc(union acpi_subtable_headers *header,
Tomasz Nowickib70fb7a2016-01-19 14:11:16 +01001875 const unsigned long end)
1876{
1877 struct acpi_madt_generic_interrupt *gicc =
1878 (struct acpi_madt_generic_interrupt *)header;
Julien Grall611f0392016-04-11 16:32:56 +01001879 u32 reg = readl_relaxed(acpi_data.dist_base + GICD_PIDR2) & GIC_PIDR2_ARCH_MASK;
Tomasz Nowickib70fb7a2016-01-19 14:11:16 +01001880 u32 size = reg == GIC_PIDR2_ARCH_GICv4 ? SZ_64K * 4 : SZ_64K * 2;
1881 void __iomem *redist_base;
1882
Shanker Donthineniebe2f872017-12-05 13:16:21 -06001883 /* GICC entry which has !ACPI_MADT_ENABLED is not unusable so skip */
1884 if (!(gicc->flags & ACPI_MADT_ENABLED))
1885 return 0;
1886
Tomasz Nowickib70fb7a2016-01-19 14:11:16 +01001887 redist_base = ioremap(gicc->gicr_base_address, size);
1888 if (!redist_base)
1889 return -ENOMEM;
1890
1891 gic_acpi_register_redist(gicc->gicr_base_address, redist_base);
1892 return 0;
1893}
1894
1895static int __init gic_acpi_collect_gicr_base(void)
1896{
1897 acpi_tbl_entry_handler redist_parser;
1898 enum acpi_madt_type type;
1899
Julien Grall611f0392016-04-11 16:32:56 +01001900 if (acpi_data.single_redist) {
Tomasz Nowickib70fb7a2016-01-19 14:11:16 +01001901 type = ACPI_MADT_TYPE_GENERIC_INTERRUPT;
1902 redist_parser = gic_acpi_parse_madt_gicc;
1903 } else {
1904 type = ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR;
1905 redist_parser = gic_acpi_parse_madt_redist;
1906 }
1907
1908 /* Collect redistributor base addresses in GICR entries */
1909 if (acpi_table_parse_madt(type, redist_parser, 0) > 0)
1910 return 0;
1911
1912 pr_info("No valid GICR entries exist\n");
1913 return -ENODEV;
1914}
1915
Keith Busch60574d12019-03-11 14:55:57 -06001916static int __init gic_acpi_match_gicr(union acpi_subtable_headers *header,
Tomasz Nowickiffa7d612016-01-19 14:11:15 +01001917 const unsigned long end)
1918{
1919 /* Subtable presence means that redist exists, that's it */
1920 return 0;
1921}
1922
Keith Busch60574d12019-03-11 14:55:57 -06001923static int __init gic_acpi_match_gicc(union acpi_subtable_headers *header,
Tomasz Nowickib70fb7a2016-01-19 14:11:16 +01001924 const unsigned long end)
1925{
1926 struct acpi_madt_generic_interrupt *gicc =
1927 (struct acpi_madt_generic_interrupt *)header;
1928
1929 /*
1930 * If GICC is enabled and has valid gicr base address, then it means
1931 * GICR base is presented via GICC
1932 */
1933 if ((gicc->flags & ACPI_MADT_ENABLED) && gicc->gicr_base_address)
1934 return 0;
1935
Shanker Donthineniebe2f872017-12-05 13:16:21 -06001936 /*
1937 * It's perfectly valid firmware can pass disabled GICC entry, driver
1938 * should not treat as errors, skip the entry instead of probe fail.
1939 */
1940 if (!(gicc->flags & ACPI_MADT_ENABLED))
1941 return 0;
1942
Tomasz Nowickib70fb7a2016-01-19 14:11:16 +01001943 return -ENODEV;
1944}
1945
1946static int __init gic_acpi_count_gicr_regions(void)
1947{
1948 int count;
1949
1950 /*
1951 * Count how many redistributor regions we have. It is not allowed
1952 * to mix redistributor description, GICR and GICC subtables have to be
1953 * mutually exclusive.
1954 */
1955 count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR,
1956 gic_acpi_match_gicr, 0);
1957 if (count > 0) {
Julien Grall611f0392016-04-11 16:32:56 +01001958 acpi_data.single_redist = false;
Tomasz Nowickib70fb7a2016-01-19 14:11:16 +01001959 return count;
1960 }
1961
1962 count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT,
1963 gic_acpi_match_gicc, 0);
1964 if (count > 0)
Julien Grall611f0392016-04-11 16:32:56 +01001965 acpi_data.single_redist = true;
Tomasz Nowickib70fb7a2016-01-19 14:11:16 +01001966
1967 return count;
1968}
1969
Tomasz Nowickiffa7d612016-01-19 14:11:15 +01001970static bool __init acpi_validate_gic_table(struct acpi_subtable_header *header,
1971 struct acpi_probe_entry *ape)
1972{
1973 struct acpi_madt_generic_distributor *dist;
1974 int count;
1975
1976 dist = (struct acpi_madt_generic_distributor *)header;
1977 if (dist->version != ape->driver_data)
1978 return false;
1979
1980 /* We need to do that exercise anyway, the sooner the better */
Tomasz Nowickib70fb7a2016-01-19 14:11:16 +01001981 count = gic_acpi_count_gicr_regions();
Tomasz Nowickiffa7d612016-01-19 14:11:15 +01001982 if (count <= 0)
1983 return false;
1984
Julien Grall611f0392016-04-11 16:32:56 +01001985 acpi_data.nr_redist_regions = count;
Tomasz Nowickiffa7d612016-01-19 14:11:15 +01001986 return true;
1987}
1988
Keith Busch60574d12019-03-11 14:55:57 -06001989static int __init gic_acpi_parse_virt_madt_gicc(union acpi_subtable_headers *header,
Julien Grall1839e572016-04-11 16:32:57 +01001990 const unsigned long end)
1991{
1992 struct acpi_madt_generic_interrupt *gicc =
1993 (struct acpi_madt_generic_interrupt *)header;
1994 int maint_irq_mode;
1995 static int first_madt = true;
1996
1997 /* Skip unusable CPUs */
1998 if (!(gicc->flags & ACPI_MADT_ENABLED))
1999 return 0;
2000
2001 maint_irq_mode = (gicc->flags & ACPI_MADT_VGIC_IRQ_MODE) ?
2002 ACPI_EDGE_SENSITIVE : ACPI_LEVEL_SENSITIVE;
2003
2004 if (first_madt) {
2005 first_madt = false;
2006
2007 acpi_data.maint_irq = gicc->vgic_interrupt;
2008 acpi_data.maint_irq_mode = maint_irq_mode;
2009 acpi_data.vcpu_base = gicc->gicv_base_address;
2010
2011 return 0;
2012 }
2013
2014 /*
2015 * The maintenance interrupt and GICV should be the same for every CPU
2016 */
2017 if ((acpi_data.maint_irq != gicc->vgic_interrupt) ||
2018 (acpi_data.maint_irq_mode != maint_irq_mode) ||
2019 (acpi_data.vcpu_base != gicc->gicv_base_address))
2020 return -EINVAL;
2021
2022 return 0;
2023}
2024
2025static bool __init gic_acpi_collect_virt_info(void)
2026{
2027 int count;
2028
2029 count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT,
2030 gic_acpi_parse_virt_madt_gicc, 0);
2031
2032 return (count > 0);
2033}
2034
Tomasz Nowickiffa7d612016-01-19 14:11:15 +01002035#define ACPI_GICV3_DIST_MEM_SIZE (SZ_64K)
Julien Grall1839e572016-04-11 16:32:57 +01002036#define ACPI_GICV2_VCTRL_MEM_SIZE (SZ_4K)
2037#define ACPI_GICV2_VCPU_MEM_SIZE (SZ_8K)
2038
2039static void __init gic_acpi_setup_kvm_info(void)
2040{
2041 int irq;
2042
2043 if (!gic_acpi_collect_virt_info()) {
2044 pr_warn("Unable to get hardware information used for virtualization\n");
2045 return;
2046 }
2047
2048 gic_v3_kvm_info.type = GIC_V3;
2049
2050 irq = acpi_register_gsi(NULL, acpi_data.maint_irq,
2051 acpi_data.maint_irq_mode,
2052 ACPI_ACTIVE_HIGH);
2053 if (irq <= 0)
2054 return;
2055
2056 gic_v3_kvm_info.maint_irq = irq;
2057
2058 if (acpi_data.vcpu_base) {
2059 struct resource *vcpu = &gic_v3_kvm_info.vcpu;
2060
2061 vcpu->flags = IORESOURCE_MEM;
2062 vcpu->start = acpi_data.vcpu_base;
2063 vcpu->end = vcpu->start + ACPI_GICV2_VCPU_MEM_SIZE - 1;
2064 }
2065
Marc Zyngier4bdf5022017-06-25 14:10:46 +01002066 gic_v3_kvm_info.has_v4 = gic_data.rdists.has_vlpis;
Julien Grall1839e572016-04-11 16:32:57 +01002067 gic_set_kvm_info(&gic_v3_kvm_info);
2068}
Tomasz Nowickiffa7d612016-01-19 14:11:15 +01002069
2070static int __init
2071gic_acpi_init(struct acpi_subtable_header *header, const unsigned long end)
2072{
2073 struct acpi_madt_generic_distributor *dist;
2074 struct fwnode_handle *domain_handle;
Julien Grall611f0392016-04-11 16:32:56 +01002075 size_t size;
Tomasz Nowickib70fb7a2016-01-19 14:11:16 +01002076 int i, err;
Tomasz Nowickiffa7d612016-01-19 14:11:15 +01002077
2078 /* Get distributor base address */
2079 dist = (struct acpi_madt_generic_distributor *)header;
Julien Grall611f0392016-04-11 16:32:56 +01002080 acpi_data.dist_base = ioremap(dist->base_address,
2081 ACPI_GICV3_DIST_MEM_SIZE);
2082 if (!acpi_data.dist_base) {
Tomasz Nowickiffa7d612016-01-19 14:11:15 +01002083 pr_err("Unable to map GICD registers\n");
2084 return -ENOMEM;
2085 }
2086
Julien Grall611f0392016-04-11 16:32:56 +01002087 err = gic_validate_dist_version(acpi_data.dist_base);
Tomasz Nowickiffa7d612016-01-19 14:11:15 +01002088 if (err) {
Arvind Yadav71192a682017-11-13 19:23:49 +05302089 pr_err("No distributor detected at @%p, giving up\n",
Julien Grall611f0392016-04-11 16:32:56 +01002090 acpi_data.dist_base);
Tomasz Nowickiffa7d612016-01-19 14:11:15 +01002091 goto out_dist_unmap;
2092 }
2093
Julien Grall611f0392016-04-11 16:32:56 +01002094 size = sizeof(*acpi_data.redist_regs) * acpi_data.nr_redist_regions;
2095 acpi_data.redist_regs = kzalloc(size, GFP_KERNEL);
2096 if (!acpi_data.redist_regs) {
Tomasz Nowickiffa7d612016-01-19 14:11:15 +01002097 err = -ENOMEM;
2098 goto out_dist_unmap;
2099 }
2100
Tomasz Nowickib70fb7a2016-01-19 14:11:16 +01002101 err = gic_acpi_collect_gicr_base();
2102 if (err)
Tomasz Nowickiffa7d612016-01-19 14:11:15 +01002103 goto out_redist_unmap;
Tomasz Nowickiffa7d612016-01-19 14:11:15 +01002104
Marc Zyngiereeee0d02019-07-31 16:13:42 +01002105 domain_handle = irq_domain_alloc_fwnode(&dist->base_address);
Tomasz Nowickiffa7d612016-01-19 14:11:15 +01002106 if (!domain_handle) {
2107 err = -ENOMEM;
2108 goto out_redist_unmap;
2109 }
2110
Julien Grall611f0392016-04-11 16:32:56 +01002111 err = gic_init_bases(acpi_data.dist_base, acpi_data.redist_regs,
2112 acpi_data.nr_redist_regions, 0, domain_handle);
Tomasz Nowickiffa7d612016-01-19 14:11:15 +01002113 if (err)
2114 goto out_fwhandle_free;
2115
2116 acpi_set_irq_model(ACPI_IRQ_MODEL_GIC, domain_handle);
Christoffer Dalld33a3c82016-12-06 22:00:52 +01002117
Davidlohr Buesod01d3272018-03-26 14:09:25 -07002118 if (static_branch_likely(&supports_deactivate_key))
Christoffer Dalld33a3c82016-12-06 22:00:52 +01002119 gic_acpi_setup_kvm_info();
Julien Grall1839e572016-04-11 16:32:57 +01002120
Tomasz Nowickiffa7d612016-01-19 14:11:15 +01002121 return 0;
2122
2123out_fwhandle_free:
2124 irq_domain_free_fwnode(domain_handle);
2125out_redist_unmap:
Julien Grall611f0392016-04-11 16:32:56 +01002126 for (i = 0; i < acpi_data.nr_redist_regions; i++)
2127 if (acpi_data.redist_regs[i].redist_base)
2128 iounmap(acpi_data.redist_regs[i].redist_base);
2129 kfree(acpi_data.redist_regs);
Tomasz Nowickiffa7d612016-01-19 14:11:15 +01002130out_dist_unmap:
Julien Grall611f0392016-04-11 16:32:56 +01002131 iounmap(acpi_data.dist_base);
Tomasz Nowickiffa7d612016-01-19 14:11:15 +01002132 return err;
2133}
2134IRQCHIP_ACPI_DECLARE(gic_v3, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR,
2135 acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_V3,
2136 gic_acpi_init);
2137IRQCHIP_ACPI_DECLARE(gic_v4, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR,
2138 acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_V4,
2139 gic_acpi_init);
2140IRQCHIP_ACPI_DECLARE(gic_v3_or_v4, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR,
2141 acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_NONE,
2142 gic_acpi_init);
2143#endif