blob: c81b4c1877cce4030f6af7955f2215f1763e4cef [file] [log] [blame]
Dave Airlie0d6aa602006-01-02 20:14:23 +11001/* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
Linus Torvalds1da177e2005-04-16 15:20:36 -07002 */
Dave Airlie0d6aa602006-01-02 20:14:23 +11003/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07004 * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
5 * All Rights Reserved.
Dave Airliebc54fd12005-06-23 22:46:46 +10006 *
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the
9 * "Software"), to deal in the Software without restriction, including
10 * without limitation the rights to use, copy, modify, merge, publish,
11 * distribute, sub license, and/or sell copies of the Software, and to
12 * permit persons to whom the Software is furnished to do so, subject to
13 * the following conditions:
14 *
15 * The above copyright notice and this permission notice (including the
16 * next paragraph) shall be included in all copies or substantial portions
17 * of the Software.
18 *
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
20 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
21 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
22 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
23 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
24 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
25 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 *
Dave Airlie0d6aa602006-01-02 20:14:23 +110027 */
Linus Torvalds1da177e2005-04-16 15:20:36 -070028
Joe Perchesa70491c2012-03-18 13:00:11 -070029#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
30
Jesse Barnes63eeaf32009-06-18 16:56:52 -070031#include <linux/sysrq.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090032#include <linux/slab.h>
Damien Lespiaub2c88f52013-10-15 18:55:29 +010033#include <linux/circ_buf.h>
David Howells760285e2012-10-02 18:01:07 +010034#include <drm/drmP.h>
35#include <drm/i915_drm.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070036#include "i915_drv.h"
Chris Wilson1c5d22f2009-08-25 11:15:50 +010037#include "i915_trace.h"
Jesse Barnes79e53942008-11-07 14:24:08 -080038#include "intel_drv.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070039
Daniel Vetterfca52a52014-09-30 10:56:45 +020040/**
41 * DOC: interrupt handling
42 *
43 * These functions provide the basic support for enabling and disabling the
44 * interrupt handling support. There's a lot more functionality in i915_irq.c
45 * and related files, but that will be described in separate chapters.
46 */
47
Ville Syrjäläe4ce95a2015-08-27 23:56:03 +030048static const u32 hpd_ilk[HPD_NUM_PINS] = {
49 [HPD_PORT_A] = DE_DP_A_HOTPLUG,
50};
51
Ville Syrjälä23bb4cb2015-08-27 23:56:04 +030052static const u32 hpd_ivb[HPD_NUM_PINS] = {
53 [HPD_PORT_A] = DE_DP_A_HOTPLUG_IVB,
54};
55
Ville Syrjälä3a3b3c72015-08-27 23:56:06 +030056static const u32 hpd_bdw[HPD_NUM_PINS] = {
57 [HPD_PORT_A] = GEN8_PORT_DP_A_HOTPLUG,
58};
59
Ville Syrjälä7c7e10d2015-01-09 14:21:12 +020060static const u32 hpd_ibx[HPD_NUM_PINS] = {
Egbert Eiche5868a32013-02-28 04:17:12 -050061 [HPD_CRT] = SDE_CRT_HOTPLUG,
62 [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG,
63 [HPD_PORT_B] = SDE_PORTB_HOTPLUG,
64 [HPD_PORT_C] = SDE_PORTC_HOTPLUG,
65 [HPD_PORT_D] = SDE_PORTD_HOTPLUG
66};
67
Ville Syrjälä7c7e10d2015-01-09 14:21:12 +020068static const u32 hpd_cpt[HPD_NUM_PINS] = {
Egbert Eiche5868a32013-02-28 04:17:12 -050069 [HPD_CRT] = SDE_CRT_HOTPLUG_CPT,
Daniel Vetter73c352a2013-03-26 22:38:43 +010070 [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG_CPT,
Egbert Eiche5868a32013-02-28 04:17:12 -050071 [HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
72 [HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
73 [HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT
74};
75
Xiong Zhang26951ca2015-08-17 15:55:50 +080076static const u32 hpd_spt[HPD_NUM_PINS] = {
Ville Syrjälä74c0b392015-08-27 23:56:07 +030077 [HPD_PORT_A] = SDE_PORTA_HOTPLUG_SPT,
Xiong Zhang26951ca2015-08-17 15:55:50 +080078 [HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
79 [HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
80 [HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT,
81 [HPD_PORT_E] = SDE_PORTE_HOTPLUG_SPT
82};
83
Ville Syrjälä7c7e10d2015-01-09 14:21:12 +020084static const u32 hpd_mask_i915[HPD_NUM_PINS] = {
Egbert Eiche5868a32013-02-28 04:17:12 -050085 [HPD_CRT] = CRT_HOTPLUG_INT_EN,
86 [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_EN,
87 [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_EN,
88 [HPD_PORT_B] = PORTB_HOTPLUG_INT_EN,
89 [HPD_PORT_C] = PORTC_HOTPLUG_INT_EN,
90 [HPD_PORT_D] = PORTD_HOTPLUG_INT_EN
91};
92
Ville Syrjälä7c7e10d2015-01-09 14:21:12 +020093static const u32 hpd_status_g4x[HPD_NUM_PINS] = {
Egbert Eiche5868a32013-02-28 04:17:12 -050094 [HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
95 [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_G4X,
96 [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_G4X,
97 [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
98 [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
99 [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
100};
101
Ville Syrjälä4bca26d2015-05-11 20:49:10 +0300102static const u32 hpd_status_i915[HPD_NUM_PINS] = {
Egbert Eiche5868a32013-02-28 04:17:12 -0500103 [HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
104 [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_I915,
105 [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_I915,
106 [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
107 [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
108 [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
109};
110
Shashank Sharmae0a20ad2015-03-27 14:54:14 +0200111/* BXT hpd list */
112static const u32 hpd_bxt[HPD_NUM_PINS] = {
Sonika Jindal7f3561b2015-08-10 10:35:35 +0530113 [HPD_PORT_A] = BXT_DE_PORT_HP_DDIA,
Shashank Sharmae0a20ad2015-03-27 14:54:14 +0200114 [HPD_PORT_B] = BXT_DE_PORT_HP_DDIB,
115 [HPD_PORT_C] = BXT_DE_PORT_HP_DDIC
116};
117
Dhinakaran Pandiyanb796b972018-06-15 17:05:30 -0700118static const u32 hpd_gen11[HPD_NUM_PINS] = {
119 [HPD_PORT_C] = GEN11_TC1_HOTPLUG | GEN11_TBT1_HOTPLUG,
120 [HPD_PORT_D] = GEN11_TC2_HOTPLUG | GEN11_TBT2_HOTPLUG,
121 [HPD_PORT_E] = GEN11_TC3_HOTPLUG | GEN11_TBT3_HOTPLUG,
122 [HPD_PORT_F] = GEN11_TC4_HOTPLUG | GEN11_TBT4_HOTPLUG
Dhinakaran Pandiyan121e7582018-06-15 17:05:29 -0700123};
124
Anusha Srivatsa31604222018-06-26 13:52:23 -0700125static const u32 hpd_icp[HPD_NUM_PINS] = {
126 [HPD_PORT_A] = SDE_DDIA_HOTPLUG_ICP,
127 [HPD_PORT_B] = SDE_DDIB_HOTPLUG_ICP,
128 [HPD_PORT_C] = SDE_TC1_HOTPLUG_ICP,
129 [HPD_PORT_D] = SDE_TC2_HOTPLUG_ICP,
130 [HPD_PORT_E] = SDE_TC3_HOTPLUG_ICP,
131 [HPD_PORT_F] = SDE_TC4_HOTPLUG_ICP
132};
133
Paulo Zanoni5c502442014-04-01 15:37:11 -0300134/* IIR can theoretically queue up two events. Be paranoid. */
Paulo Zanonif86f3fb2014-04-01 15:37:14 -0300135#define GEN8_IRQ_RESET_NDX(type, which) do { \
Paulo Zanoni5c502442014-04-01 15:37:11 -0300136 I915_WRITE(GEN8_##type##_IMR(which), 0xffffffff); \
137 POSTING_READ(GEN8_##type##_IMR(which)); \
138 I915_WRITE(GEN8_##type##_IER(which), 0); \
139 I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
140 POSTING_READ(GEN8_##type##_IIR(which)); \
141 I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
142 POSTING_READ(GEN8_##type##_IIR(which)); \
143} while (0)
144
Ville Syrjälä3488d4e2017-08-18 21:36:52 +0300145#define GEN3_IRQ_RESET(type) do { \
Paulo Zanonia9d356a2014-04-01 15:37:09 -0300146 I915_WRITE(type##IMR, 0xffffffff); \
Paulo Zanoni5c502442014-04-01 15:37:11 -0300147 POSTING_READ(type##IMR); \
Paulo Zanonia9d356a2014-04-01 15:37:09 -0300148 I915_WRITE(type##IER, 0); \
Paulo Zanoni5c502442014-04-01 15:37:11 -0300149 I915_WRITE(type##IIR, 0xffffffff); \
150 POSTING_READ(type##IIR); \
151 I915_WRITE(type##IIR, 0xffffffff); \
152 POSTING_READ(type##IIR); \
Paulo Zanonia9d356a2014-04-01 15:37:09 -0300153} while (0)
154
Ville Syrjäläe9e98482017-08-18 21:36:54 +0300155#define GEN2_IRQ_RESET(type) do { \
156 I915_WRITE16(type##IMR, 0xffff); \
157 POSTING_READ16(type##IMR); \
158 I915_WRITE16(type##IER, 0); \
159 I915_WRITE16(type##IIR, 0xffff); \
160 POSTING_READ16(type##IIR); \
161 I915_WRITE16(type##IIR, 0xffff); \
162 POSTING_READ16(type##IIR); \
163} while (0)
164
Paulo Zanoni337ba012014-04-01 15:37:16 -0300165/*
166 * We should clear IMR at preinstall/uninstall, and just check at postinstall.
167 */
Ville Syrjälä3488d4e2017-08-18 21:36:52 +0300168static void gen3_assert_iir_is_zero(struct drm_i915_private *dev_priv,
Ville Syrjäläf0f59a02015-11-18 15:33:26 +0200169 i915_reg_t reg)
Ville Syrjäläb51a2842015-09-18 20:03:41 +0300170{
171 u32 val = I915_READ(reg);
172
173 if (val == 0)
174 return;
175
176 WARN(1, "Interrupt register 0x%x is not zero: 0x%08x\n",
Ville Syrjäläf0f59a02015-11-18 15:33:26 +0200177 i915_mmio_reg_offset(reg), val);
Ville Syrjäläb51a2842015-09-18 20:03:41 +0300178 I915_WRITE(reg, 0xffffffff);
179 POSTING_READ(reg);
180 I915_WRITE(reg, 0xffffffff);
181 POSTING_READ(reg);
182}
Paulo Zanoni337ba012014-04-01 15:37:16 -0300183
Ville Syrjäläe9e98482017-08-18 21:36:54 +0300184static void gen2_assert_iir_is_zero(struct drm_i915_private *dev_priv,
185 i915_reg_t reg)
186{
187 u16 val = I915_READ16(reg);
188
189 if (val == 0)
190 return;
191
192 WARN(1, "Interrupt register 0x%x is not zero: 0x%08x\n",
193 i915_mmio_reg_offset(reg), val);
194 I915_WRITE16(reg, 0xffff);
195 POSTING_READ16(reg);
196 I915_WRITE16(reg, 0xffff);
197 POSTING_READ16(reg);
198}
199
Paulo Zanoni35079892014-04-01 15:37:15 -0300200#define GEN8_IRQ_INIT_NDX(type, which, imr_val, ier_val) do { \
Ville Syrjälä3488d4e2017-08-18 21:36:52 +0300201 gen3_assert_iir_is_zero(dev_priv, GEN8_##type##_IIR(which)); \
Paulo Zanoni35079892014-04-01 15:37:15 -0300202 I915_WRITE(GEN8_##type##_IER(which), (ier_val)); \
Ville Syrjälä7d1bd5392014-10-30 19:42:50 +0200203 I915_WRITE(GEN8_##type##_IMR(which), (imr_val)); \
204 POSTING_READ(GEN8_##type##_IMR(which)); \
Paulo Zanoni35079892014-04-01 15:37:15 -0300205} while (0)
206
Ville Syrjälä3488d4e2017-08-18 21:36:52 +0300207#define GEN3_IRQ_INIT(type, imr_val, ier_val) do { \
208 gen3_assert_iir_is_zero(dev_priv, type##IIR); \
Paulo Zanoni35079892014-04-01 15:37:15 -0300209 I915_WRITE(type##IER, (ier_val)); \
Ville Syrjälä7d1bd5392014-10-30 19:42:50 +0200210 I915_WRITE(type##IMR, (imr_val)); \
211 POSTING_READ(type##IMR); \
Paulo Zanoni35079892014-04-01 15:37:15 -0300212} while (0)
213
Ville Syrjäläe9e98482017-08-18 21:36:54 +0300214#define GEN2_IRQ_INIT(type, imr_val, ier_val) do { \
215 gen2_assert_iir_is_zero(dev_priv, type##IIR); \
216 I915_WRITE16(type##IER, (ier_val)); \
217 I915_WRITE16(type##IMR, (imr_val)); \
218 POSTING_READ16(type##IMR); \
219} while (0)
220
Imre Deakc9a9a262014-11-05 20:48:37 +0200221static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir);
Sagar Arun Kamble26705e22016-10-12 21:54:31 +0530222static void gen9_guc_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir);
Imre Deakc9a9a262014-11-05 20:48:37 +0200223
Egbert Eich0706f172015-09-23 16:15:27 +0200224/* For display hotplug interrupt */
225static inline void
226i915_hotplug_interrupt_update_locked(struct drm_i915_private *dev_priv,
227 uint32_t mask,
228 uint32_t bits)
229{
230 uint32_t val;
231
Chris Wilson67520412017-03-02 13:28:01 +0000232 lockdep_assert_held(&dev_priv->irq_lock);
Egbert Eich0706f172015-09-23 16:15:27 +0200233 WARN_ON(bits & ~mask);
234
235 val = I915_READ(PORT_HOTPLUG_EN);
236 val &= ~mask;
237 val |= bits;
238 I915_WRITE(PORT_HOTPLUG_EN, val);
239}
240
241/**
242 * i915_hotplug_interrupt_update - update hotplug interrupt enable
243 * @dev_priv: driver private
244 * @mask: bits to update
245 * @bits: bits to enable
246 * NOTE: the HPD enable bits are modified both inside and outside
247 * of an interrupt context. To avoid that read-modify-write cycles
248 * interfer, these bits are protected by a spinlock. Since this
249 * function is usually not called from a context where the lock is
250 * held already, this function acquires the lock itself. A non-locking
251 * version is also available.
252 */
253void i915_hotplug_interrupt_update(struct drm_i915_private *dev_priv,
254 uint32_t mask,
255 uint32_t bits)
256{
257 spin_lock_irq(&dev_priv->irq_lock);
258 i915_hotplug_interrupt_update_locked(dev_priv, mask, bits);
259 spin_unlock_irq(&dev_priv->irq_lock);
260}
261
Oscar Mateo96606f32018-04-06 12:32:37 +0300262static u32
263gen11_gt_engine_identity(struct drm_i915_private * const i915,
264 const unsigned int bank, const unsigned int bit);
265
Oscar Mateoff047a82018-04-24 14:39:55 -0700266bool gen11_reset_one_iir(struct drm_i915_private * const i915,
267 const unsigned int bank,
268 const unsigned int bit)
Oscar Mateo96606f32018-04-06 12:32:37 +0300269{
270 void __iomem * const regs = i915->regs;
271 u32 dw;
272
273 lockdep_assert_held(&i915->irq_lock);
274
275 dw = raw_reg_read(regs, GEN11_GT_INTR_DW(bank));
276 if (dw & BIT(bit)) {
277 /*
278 * According to the BSpec, DW_IIR bits cannot be cleared without
279 * first servicing the Selector & Shared IIR registers.
280 */
281 gen11_gt_engine_identity(i915, bank, bit);
282
283 /*
284 * We locked GT INT DW by reading it. If we want to (try
285 * to) recover from this succesfully, we need to clear
286 * our bit, otherwise we are locking the register for
287 * everybody.
288 */
289 raw_reg_write(regs, GEN11_GT_INTR_DW(bank), BIT(bit));
290
291 return true;
292 }
293
294 return false;
295}
296
Ville Syrjäläd9dc34f12015-08-27 23:55:58 +0300297/**
298 * ilk_update_display_irq - update DEIMR
299 * @dev_priv: driver private
300 * @interrupt_mask: mask of interrupt bits to update
301 * @enabled_irq_mask: mask of interrupt bits to enable
302 */
Ville Syrjäläfbdedaea2015-11-23 18:06:16 +0200303void ilk_update_display_irq(struct drm_i915_private *dev_priv,
304 uint32_t interrupt_mask,
305 uint32_t enabled_irq_mask)
Zhenyu Wang036a4a72009-06-08 14:40:19 +0800306{
Ville Syrjäläd9dc34f12015-08-27 23:55:58 +0300307 uint32_t new_val;
308
Chris Wilson67520412017-03-02 13:28:01 +0000309 lockdep_assert_held(&dev_priv->irq_lock);
Daniel Vetter4bc9d432013-06-27 13:44:58 +0200310
Ville Syrjäläd9dc34f12015-08-27 23:55:58 +0300311 WARN_ON(enabled_irq_mask & ~interrupt_mask);
312
Jesse Barnes9df7575f2014-06-20 09:29:20 -0700313 if (WARN_ON(!intel_irqs_enabled(dev_priv)))
Paulo Zanonic67a4702013-08-19 13:18:09 -0300314 return;
Paulo Zanonic67a4702013-08-19 13:18:09 -0300315
Ville Syrjäläd9dc34f12015-08-27 23:55:58 +0300316 new_val = dev_priv->irq_mask;
317 new_val &= ~interrupt_mask;
318 new_val |= (~enabled_irq_mask & interrupt_mask);
319
320 if (new_val != dev_priv->irq_mask) {
321 dev_priv->irq_mask = new_val;
Chris Wilson1ec14ad2010-12-04 11:30:53 +0000322 I915_WRITE(DEIMR, dev_priv->irq_mask);
Chris Wilson3143a2b2010-11-16 15:55:10 +0000323 POSTING_READ(DEIMR);
Zhenyu Wang036a4a72009-06-08 14:40:19 +0800324 }
325}
326
Paulo Zanoni43eaea12013-08-06 18:57:12 -0300327/**
328 * ilk_update_gt_irq - update GTIMR
329 * @dev_priv: driver private
330 * @interrupt_mask: mask of interrupt bits to update
331 * @enabled_irq_mask: mask of interrupt bits to enable
332 */
333static void ilk_update_gt_irq(struct drm_i915_private *dev_priv,
334 uint32_t interrupt_mask,
335 uint32_t enabled_irq_mask)
336{
Chris Wilson67520412017-03-02 13:28:01 +0000337 lockdep_assert_held(&dev_priv->irq_lock);
Paulo Zanoni43eaea12013-08-06 18:57:12 -0300338
Daniel Vetter15a17aa2014-12-08 16:30:00 +0100339 WARN_ON(enabled_irq_mask & ~interrupt_mask);
340
Jesse Barnes9df7575f2014-06-20 09:29:20 -0700341 if (WARN_ON(!intel_irqs_enabled(dev_priv)))
Paulo Zanonic67a4702013-08-19 13:18:09 -0300342 return;
Paulo Zanonic67a4702013-08-19 13:18:09 -0300343
Paulo Zanoni43eaea12013-08-06 18:57:12 -0300344 dev_priv->gt_irq_mask &= ~interrupt_mask;
345 dev_priv->gt_irq_mask |= (~enabled_irq_mask & interrupt_mask);
346 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
Paulo Zanoni43eaea12013-08-06 18:57:12 -0300347}
348
Daniel Vetter480c8032014-07-16 09:49:40 +0200349void gen5_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
Paulo Zanoni43eaea12013-08-06 18:57:12 -0300350{
351 ilk_update_gt_irq(dev_priv, mask, mask);
Chris Wilson31bb59c2016-07-01 17:23:27 +0100352 POSTING_READ_FW(GTIMR);
Paulo Zanoni43eaea12013-08-06 18:57:12 -0300353}
354
Daniel Vetter480c8032014-07-16 09:49:40 +0200355void gen5_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
Paulo Zanoni43eaea12013-08-06 18:57:12 -0300356{
357 ilk_update_gt_irq(dev_priv, mask, 0);
358}
359
Ville Syrjäläf0f59a02015-11-18 15:33:26 +0200360static i915_reg_t gen6_pm_iir(struct drm_i915_private *dev_priv)
Imre Deakb900b942014-11-05 20:48:48 +0200361{
Oscar Mateod02b98b2018-04-05 17:00:50 +0300362 WARN_ON_ONCE(INTEL_GEN(dev_priv) >= 11);
363
Pandiyan, Dhinakaranbca2bf22017-07-18 11:28:00 -0700364 return INTEL_GEN(dev_priv) >= 8 ? GEN8_GT_IIR(2) : GEN6_PMIIR;
Imre Deakb900b942014-11-05 20:48:48 +0200365}
366
Ville Syrjäläf0f59a02015-11-18 15:33:26 +0200367static i915_reg_t gen6_pm_imr(struct drm_i915_private *dev_priv)
Imre Deaka72fbc32014-11-05 20:48:31 +0200368{
Oscar Mateod02b98b2018-04-05 17:00:50 +0300369 if (INTEL_GEN(dev_priv) >= 11)
370 return GEN11_GPM_WGBOXPERF_INTR_MASK;
371 else if (INTEL_GEN(dev_priv) >= 8)
372 return GEN8_GT_IMR(2);
373 else
374 return GEN6_PMIMR;
Imre Deaka72fbc32014-11-05 20:48:31 +0200375}
376
Ville Syrjäläf0f59a02015-11-18 15:33:26 +0200377static i915_reg_t gen6_pm_ier(struct drm_i915_private *dev_priv)
Imre Deakb900b942014-11-05 20:48:48 +0200378{
Oscar Mateod02b98b2018-04-05 17:00:50 +0300379 if (INTEL_GEN(dev_priv) >= 11)
380 return GEN11_GPM_WGBOXPERF_INTR_ENABLE;
381 else if (INTEL_GEN(dev_priv) >= 8)
382 return GEN8_GT_IER(2);
383 else
384 return GEN6_PMIER;
Imre Deakb900b942014-11-05 20:48:48 +0200385}
386
Paulo Zanoniedbfdb42013-08-06 18:57:13 -0300387/**
Ville Syrjälä81fd8742015-11-25 16:21:30 +0200388 * snb_update_pm_irq - update GEN6_PMIMR
389 * @dev_priv: driver private
390 * @interrupt_mask: mask of interrupt bits to update
391 * @enabled_irq_mask: mask of interrupt bits to enable
392 */
Paulo Zanoniedbfdb42013-08-06 18:57:13 -0300393static void snb_update_pm_irq(struct drm_i915_private *dev_priv,
394 uint32_t interrupt_mask,
395 uint32_t enabled_irq_mask)
396{
Paulo Zanoni605cd252013-08-06 18:57:15 -0300397 uint32_t new_val;
Paulo Zanoniedbfdb42013-08-06 18:57:13 -0300398
Daniel Vetter15a17aa2014-12-08 16:30:00 +0100399 WARN_ON(enabled_irq_mask & ~interrupt_mask);
400
Chris Wilson67520412017-03-02 13:28:01 +0000401 lockdep_assert_held(&dev_priv->irq_lock);
Paulo Zanoniedbfdb42013-08-06 18:57:13 -0300402
Akash Goelf4e9af42016-10-12 21:54:30 +0530403 new_val = dev_priv->pm_imr;
Paulo Zanonif52ecbc2013-08-06 18:57:14 -0300404 new_val &= ~interrupt_mask;
405 new_val |= (~enabled_irq_mask & interrupt_mask);
406
Akash Goelf4e9af42016-10-12 21:54:30 +0530407 if (new_val != dev_priv->pm_imr) {
408 dev_priv->pm_imr = new_val;
409 I915_WRITE(gen6_pm_imr(dev_priv), dev_priv->pm_imr);
Imre Deaka72fbc32014-11-05 20:48:31 +0200410 POSTING_READ(gen6_pm_imr(dev_priv));
Paulo Zanonif52ecbc2013-08-06 18:57:14 -0300411 }
Paulo Zanoniedbfdb42013-08-06 18:57:13 -0300412}
413
Akash Goelf4e9af42016-10-12 21:54:30 +0530414void gen6_unmask_pm_irq(struct drm_i915_private *dev_priv, u32 mask)
Paulo Zanoniedbfdb42013-08-06 18:57:13 -0300415{
Imre Deak9939fba2014-11-20 23:01:47 +0200416 if (WARN_ON(!intel_irqs_enabled(dev_priv)))
417 return;
418
Paulo Zanoniedbfdb42013-08-06 18:57:13 -0300419 snb_update_pm_irq(dev_priv, mask, mask);
420}
421
Akash Goelf4e9af42016-10-12 21:54:30 +0530422static void __gen6_mask_pm_irq(struct drm_i915_private *dev_priv, u32 mask)
Imre Deak9939fba2014-11-20 23:01:47 +0200423{
424 snb_update_pm_irq(dev_priv, mask, 0);
425}
426
Akash Goelf4e9af42016-10-12 21:54:30 +0530427void gen6_mask_pm_irq(struct drm_i915_private *dev_priv, u32 mask)
Paulo Zanoniedbfdb42013-08-06 18:57:13 -0300428{
Imre Deak9939fba2014-11-20 23:01:47 +0200429 if (WARN_ON(!intel_irqs_enabled(dev_priv)))
430 return;
431
Akash Goelf4e9af42016-10-12 21:54:30 +0530432 __gen6_mask_pm_irq(dev_priv, mask);
433}
434
Oscar Mateo3814fd72017-08-23 16:58:24 -0700435static void gen6_reset_pm_iir(struct drm_i915_private *dev_priv, u32 reset_mask)
Akash Goelf4e9af42016-10-12 21:54:30 +0530436{
437 i915_reg_t reg = gen6_pm_iir(dev_priv);
438
Chris Wilson67520412017-03-02 13:28:01 +0000439 lockdep_assert_held(&dev_priv->irq_lock);
Akash Goelf4e9af42016-10-12 21:54:30 +0530440
441 I915_WRITE(reg, reset_mask);
442 I915_WRITE(reg, reset_mask);
443 POSTING_READ(reg);
444}
445
Oscar Mateo3814fd72017-08-23 16:58:24 -0700446static void gen6_enable_pm_irq(struct drm_i915_private *dev_priv, u32 enable_mask)
Akash Goelf4e9af42016-10-12 21:54:30 +0530447{
Chris Wilson67520412017-03-02 13:28:01 +0000448 lockdep_assert_held(&dev_priv->irq_lock);
Akash Goelf4e9af42016-10-12 21:54:30 +0530449
450 dev_priv->pm_ier |= enable_mask;
451 I915_WRITE(gen6_pm_ier(dev_priv), dev_priv->pm_ier);
452 gen6_unmask_pm_irq(dev_priv, enable_mask);
453 /* unmask_pm_irq provides an implicit barrier (POSTING_READ) */
454}
455
Oscar Mateo3814fd72017-08-23 16:58:24 -0700456static void gen6_disable_pm_irq(struct drm_i915_private *dev_priv, u32 disable_mask)
Akash Goelf4e9af42016-10-12 21:54:30 +0530457{
Chris Wilson67520412017-03-02 13:28:01 +0000458 lockdep_assert_held(&dev_priv->irq_lock);
Akash Goelf4e9af42016-10-12 21:54:30 +0530459
460 dev_priv->pm_ier &= ~disable_mask;
461 __gen6_mask_pm_irq(dev_priv, disable_mask);
462 I915_WRITE(gen6_pm_ier(dev_priv), dev_priv->pm_ier);
463 /* though a barrier is missing here, but don't really need a one */
Paulo Zanoniedbfdb42013-08-06 18:57:13 -0300464}
465
Oscar Mateod02b98b2018-04-05 17:00:50 +0300466void gen11_reset_rps_interrupts(struct drm_i915_private *dev_priv)
467{
Oscar Mateod02b98b2018-04-05 17:00:50 +0300468 spin_lock_irq(&dev_priv->irq_lock);
469
Oscar Mateo96606f32018-04-06 12:32:37 +0300470 while (gen11_reset_one_iir(dev_priv, 0, GEN11_GTPM))
471 ;
Oscar Mateod02b98b2018-04-05 17:00:50 +0300472
473 dev_priv->gt_pm.rps.pm_iir = 0;
474
475 spin_unlock_irq(&dev_priv->irq_lock);
476}
477
Chris Wilsondc979972016-05-10 14:10:04 +0100478void gen6_reset_rps_interrupts(struct drm_i915_private *dev_priv)
Imre Deak3cc134e2014-11-19 15:30:03 +0200479{
Imre Deak3cc134e2014-11-19 15:30:03 +0200480 spin_lock_irq(&dev_priv->irq_lock);
Akash Goelf4e9af42016-10-12 21:54:30 +0530481 gen6_reset_pm_iir(dev_priv, dev_priv->pm_rps_events);
Sagar Arun Kamble562d9ba2017-10-10 22:30:06 +0100482 dev_priv->gt_pm.rps.pm_iir = 0;
Imre Deak3cc134e2014-11-19 15:30:03 +0200483 spin_unlock_irq(&dev_priv->irq_lock);
484}
485
Tvrtko Ursulin91d14252016-05-06 14:48:28 +0100486void gen6_enable_rps_interrupts(struct drm_i915_private *dev_priv)
Imre Deakb900b942014-11-05 20:48:48 +0200487{
Sagar Arun Kamble562d9ba2017-10-10 22:30:06 +0100488 struct intel_rps *rps = &dev_priv->gt_pm.rps;
489
490 if (READ_ONCE(rps->interrupts_enabled))
Chris Wilsonf2a91d12016-09-21 14:51:06 +0100491 return;
492
Imre Deakb900b942014-11-05 20:48:48 +0200493 spin_lock_irq(&dev_priv->irq_lock);
Sagar Arun Kamble562d9ba2017-10-10 22:30:06 +0100494 WARN_ON_ONCE(rps->pm_iir);
Oscar Mateo96606f32018-04-06 12:32:37 +0300495
Oscar Mateod02b98b2018-04-05 17:00:50 +0300496 if (INTEL_GEN(dev_priv) >= 11)
Oscar Mateo96606f32018-04-06 12:32:37 +0300497 WARN_ON_ONCE(gen11_reset_one_iir(dev_priv, 0, GEN11_GTPM));
Oscar Mateod02b98b2018-04-05 17:00:50 +0300498 else
499 WARN_ON_ONCE(I915_READ(gen6_pm_iir(dev_priv)) & dev_priv->pm_rps_events);
Oscar Mateo96606f32018-04-06 12:32:37 +0300500
Sagar Arun Kamble562d9ba2017-10-10 22:30:06 +0100501 rps->interrupts_enabled = true;
Imre Deakb900b942014-11-05 20:48:48 +0200502 gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
Imre Deak78e68d32014-12-15 18:59:27 +0200503
Imre Deakb900b942014-11-05 20:48:48 +0200504 spin_unlock_irq(&dev_priv->irq_lock);
505}
506
Tvrtko Ursulin91d14252016-05-06 14:48:28 +0100507void gen6_disable_rps_interrupts(struct drm_i915_private *dev_priv)
Imre Deakb900b942014-11-05 20:48:48 +0200508{
Sagar Arun Kamble562d9ba2017-10-10 22:30:06 +0100509 struct intel_rps *rps = &dev_priv->gt_pm.rps;
510
511 if (!READ_ONCE(rps->interrupts_enabled))
Chris Wilsonf2a91d12016-09-21 14:51:06 +0100512 return;
513
Imre Deakd4d70aa2014-11-19 15:30:04 +0200514 spin_lock_irq(&dev_priv->irq_lock);
Sagar Arun Kamble562d9ba2017-10-10 22:30:06 +0100515 rps->interrupts_enabled = false;
Imre Deak9939fba2014-11-20 23:01:47 +0200516
Dave Gordonb20e3cf2016-09-12 21:19:35 +0100517 I915_WRITE(GEN6_PMINTRMSK, gen6_sanitize_rps_pm_mask(dev_priv, ~0u));
Imre Deak9939fba2014-11-20 23:01:47 +0200518
Akash Goelf4e9af42016-10-12 21:54:30 +0530519 gen6_disable_pm_irq(dev_priv, dev_priv->pm_rps_events);
Imre Deak58072cc2015-03-23 19:11:34 +0200520
521 spin_unlock_irq(&dev_priv->irq_lock);
Chris Wilson91c8a322016-07-05 10:40:23 +0100522 synchronize_irq(dev_priv->drm.irq);
Chris Wilsonc33d2472016-07-04 08:08:36 +0100523
524 /* Now that we will not be generating any more work, flush any
Oscar Mateo3814fd72017-08-23 16:58:24 -0700525 * outstanding tasks. As we are called on the RPS idle path,
Chris Wilsonc33d2472016-07-04 08:08:36 +0100526 * we will reset the GPU to minimum frequencies, so the current
527 * state of the worker can be discarded.
528 */
Sagar Arun Kamble562d9ba2017-10-10 22:30:06 +0100529 cancel_work_sync(&rps->work);
Oscar Mateod02b98b2018-04-05 17:00:50 +0300530 if (INTEL_GEN(dev_priv) >= 11)
531 gen11_reset_rps_interrupts(dev_priv);
532 else
533 gen6_reset_rps_interrupts(dev_priv);
Imre Deakb900b942014-11-05 20:48:48 +0200534}
535
Sagar Arun Kamble26705e22016-10-12 21:54:31 +0530536void gen9_reset_guc_interrupts(struct drm_i915_private *dev_priv)
537{
Sagar Arun Kamble1be333d2018-01-24 21:16:56 +0530538 assert_rpm_wakelock_held(dev_priv);
539
Sagar Arun Kamble26705e22016-10-12 21:54:31 +0530540 spin_lock_irq(&dev_priv->irq_lock);
541 gen6_reset_pm_iir(dev_priv, dev_priv->pm_guc_events);
542 spin_unlock_irq(&dev_priv->irq_lock);
543}
544
545void gen9_enable_guc_interrupts(struct drm_i915_private *dev_priv)
546{
Sagar Arun Kamble1be333d2018-01-24 21:16:56 +0530547 assert_rpm_wakelock_held(dev_priv);
548
Sagar Arun Kamble26705e22016-10-12 21:54:31 +0530549 spin_lock_irq(&dev_priv->irq_lock);
550 if (!dev_priv->guc.interrupts_enabled) {
551 WARN_ON_ONCE(I915_READ(gen6_pm_iir(dev_priv)) &
552 dev_priv->pm_guc_events);
553 dev_priv->guc.interrupts_enabled = true;
554 gen6_enable_pm_irq(dev_priv, dev_priv->pm_guc_events);
555 }
556 spin_unlock_irq(&dev_priv->irq_lock);
557}
558
559void gen9_disable_guc_interrupts(struct drm_i915_private *dev_priv)
560{
Sagar Arun Kamble1be333d2018-01-24 21:16:56 +0530561 assert_rpm_wakelock_held(dev_priv);
562
Sagar Arun Kamble26705e22016-10-12 21:54:31 +0530563 spin_lock_irq(&dev_priv->irq_lock);
564 dev_priv->guc.interrupts_enabled = false;
565
566 gen6_disable_pm_irq(dev_priv, dev_priv->pm_guc_events);
567
568 spin_unlock_irq(&dev_priv->irq_lock);
569 synchronize_irq(dev_priv->drm.irq);
570
571 gen9_reset_guc_interrupts(dev_priv);
572}
573
Ben Widawsky09610212014-05-15 20:58:08 +0300574/**
Ville Syrjälä81fd8742015-11-25 16:21:30 +0200575 * bdw_update_port_irq - update DE port interrupt
576 * @dev_priv: driver private
577 * @interrupt_mask: mask of interrupt bits to update
578 * @enabled_irq_mask: mask of interrupt bits to enable
579 */
Ville Syrjälä3a3b3c72015-08-27 23:56:06 +0300580static void bdw_update_port_irq(struct drm_i915_private *dev_priv,
581 uint32_t interrupt_mask,
582 uint32_t enabled_irq_mask)
583{
584 uint32_t new_val;
585 uint32_t old_val;
586
Chris Wilson67520412017-03-02 13:28:01 +0000587 lockdep_assert_held(&dev_priv->irq_lock);
Ville Syrjälä3a3b3c72015-08-27 23:56:06 +0300588
589 WARN_ON(enabled_irq_mask & ~interrupt_mask);
590
591 if (WARN_ON(!intel_irqs_enabled(dev_priv)))
592 return;
593
594 old_val = I915_READ(GEN8_DE_PORT_IMR);
595
596 new_val = old_val;
597 new_val &= ~interrupt_mask;
598 new_val |= (~enabled_irq_mask & interrupt_mask);
599
600 if (new_val != old_val) {
601 I915_WRITE(GEN8_DE_PORT_IMR, new_val);
602 POSTING_READ(GEN8_DE_PORT_IMR);
603 }
604}
605
606/**
Ville Syrjälä013d3752015-11-23 18:06:17 +0200607 * bdw_update_pipe_irq - update DE pipe interrupt
608 * @dev_priv: driver private
609 * @pipe: pipe whose interrupt to update
610 * @interrupt_mask: mask of interrupt bits to update
611 * @enabled_irq_mask: mask of interrupt bits to enable
612 */
613void bdw_update_pipe_irq(struct drm_i915_private *dev_priv,
614 enum pipe pipe,
615 uint32_t interrupt_mask,
616 uint32_t enabled_irq_mask)
617{
618 uint32_t new_val;
619
Chris Wilson67520412017-03-02 13:28:01 +0000620 lockdep_assert_held(&dev_priv->irq_lock);
Ville Syrjälä013d3752015-11-23 18:06:17 +0200621
622 WARN_ON(enabled_irq_mask & ~interrupt_mask);
623
624 if (WARN_ON(!intel_irqs_enabled(dev_priv)))
625 return;
626
627 new_val = dev_priv->de_irq_mask[pipe];
628 new_val &= ~interrupt_mask;
629 new_val |= (~enabled_irq_mask & interrupt_mask);
630
631 if (new_val != dev_priv->de_irq_mask[pipe]) {
632 dev_priv->de_irq_mask[pipe] = new_val;
633 I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
634 POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
635 }
636}
637
638/**
Daniel Vetterfee884e2013-07-04 23:35:21 +0200639 * ibx_display_interrupt_update - update SDEIMR
640 * @dev_priv: driver private
641 * @interrupt_mask: mask of interrupt bits to update
642 * @enabled_irq_mask: mask of interrupt bits to enable
643 */
Daniel Vetter47339cd2014-09-30 10:56:46 +0200644void ibx_display_interrupt_update(struct drm_i915_private *dev_priv,
645 uint32_t interrupt_mask,
646 uint32_t enabled_irq_mask)
Daniel Vetterfee884e2013-07-04 23:35:21 +0200647{
648 uint32_t sdeimr = I915_READ(SDEIMR);
649 sdeimr &= ~interrupt_mask;
650 sdeimr |= (~enabled_irq_mask & interrupt_mask);
651
Daniel Vetter15a17aa2014-12-08 16:30:00 +0100652 WARN_ON(enabled_irq_mask & ~interrupt_mask);
653
Chris Wilson67520412017-03-02 13:28:01 +0000654 lockdep_assert_held(&dev_priv->irq_lock);
Daniel Vetterfee884e2013-07-04 23:35:21 +0200655
Jesse Barnes9df7575f2014-06-20 09:29:20 -0700656 if (WARN_ON(!intel_irqs_enabled(dev_priv)))
Paulo Zanonic67a4702013-08-19 13:18:09 -0300657 return;
Paulo Zanonic67a4702013-08-19 13:18:09 -0300658
Daniel Vetterfee884e2013-07-04 23:35:21 +0200659 I915_WRITE(SDEIMR, sdeimr);
660 POSTING_READ(SDEIMR);
661}
Paulo Zanoni86642812013-04-12 17:57:57 -0300662
Ville Syrjälä6b12ca52017-09-14 18:17:31 +0300663u32 i915_pipestat_enable_mask(struct drm_i915_private *dev_priv,
664 enum pipe pipe)
Keith Packard7c463582008-11-04 02:03:27 -0800665{
Ville Syrjälä6b12ca52017-09-14 18:17:31 +0300666 u32 status_mask = dev_priv->pipestat_irq_mask[pipe];
Imre Deak10c59c52014-02-10 18:42:48 +0200667 u32 enable_mask = status_mask << 16;
668
Ville Syrjälä6b12ca52017-09-14 18:17:31 +0300669 lockdep_assert_held(&dev_priv->irq_lock);
670
671 if (INTEL_GEN(dev_priv) < 5)
672 goto out;
673
Imre Deak10c59c52014-02-10 18:42:48 +0200674 /*
Ville Syrjälä724a6902014-04-09 13:28:48 +0300675 * On pipe A we don't support the PSR interrupt yet,
676 * on pipe B and C the same bit MBZ.
Imre Deak10c59c52014-02-10 18:42:48 +0200677 */
678 if (WARN_ON_ONCE(status_mask & PIPE_A_PSR_STATUS_VLV))
679 return 0;
Ville Syrjälä724a6902014-04-09 13:28:48 +0300680 /*
681 * On pipe B and C we don't support the PSR interrupt yet, on pipe
682 * A the same bit is for perf counters which we don't use either.
683 */
684 if (WARN_ON_ONCE(status_mask & PIPE_B_PSR_STATUS_VLV))
685 return 0;
Imre Deak10c59c52014-02-10 18:42:48 +0200686
687 enable_mask &= ~(PIPE_FIFO_UNDERRUN_STATUS |
688 SPRITE0_FLIP_DONE_INT_EN_VLV |
689 SPRITE1_FLIP_DONE_INT_EN_VLV);
690 if (status_mask & SPRITE0_FLIP_DONE_INT_STATUS_VLV)
691 enable_mask |= SPRITE0_FLIP_DONE_INT_EN_VLV;
692 if (status_mask & SPRITE1_FLIP_DONE_INT_STATUS_VLV)
693 enable_mask |= SPRITE1_FLIP_DONE_INT_EN_VLV;
694
Ville Syrjälä6b12ca52017-09-14 18:17:31 +0300695out:
696 WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
697 status_mask & ~PIPESTAT_INT_STATUS_MASK,
698 "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
699 pipe_name(pipe), enable_mask, status_mask);
700
Imre Deak10c59c52014-02-10 18:42:48 +0200701 return enable_mask;
702}
703
Ville Syrjälä6b12ca52017-09-14 18:17:31 +0300704void i915_enable_pipestat(struct drm_i915_private *dev_priv,
705 enum pipe pipe, u32 status_mask)
Imre Deak755e9012014-02-10 18:42:47 +0200706{
Ville Syrjälä6b12ca52017-09-14 18:17:31 +0300707 i915_reg_t reg = PIPESTAT(pipe);
Imre Deak755e9012014-02-10 18:42:47 +0200708 u32 enable_mask;
709
Ville Syrjälä6b12ca52017-09-14 18:17:31 +0300710 WARN_ONCE(status_mask & ~PIPESTAT_INT_STATUS_MASK,
711 "pipe %c: status_mask=0x%x\n",
712 pipe_name(pipe), status_mask);
713
714 lockdep_assert_held(&dev_priv->irq_lock);
715 WARN_ON(!intel_irqs_enabled(dev_priv));
716
717 if ((dev_priv->pipestat_irq_mask[pipe] & status_mask) == status_mask)
718 return;
719
720 dev_priv->pipestat_irq_mask[pipe] |= status_mask;
721 enable_mask = i915_pipestat_enable_mask(dev_priv, pipe);
722
723 I915_WRITE(reg, enable_mask | status_mask);
724 POSTING_READ(reg);
Imre Deak755e9012014-02-10 18:42:47 +0200725}
726
Ville Syrjälä6b12ca52017-09-14 18:17:31 +0300727void i915_disable_pipestat(struct drm_i915_private *dev_priv,
728 enum pipe pipe, u32 status_mask)
Imre Deak755e9012014-02-10 18:42:47 +0200729{
Ville Syrjälä6b12ca52017-09-14 18:17:31 +0300730 i915_reg_t reg = PIPESTAT(pipe);
Imre Deak755e9012014-02-10 18:42:47 +0200731 u32 enable_mask;
732
Ville Syrjälä6b12ca52017-09-14 18:17:31 +0300733 WARN_ONCE(status_mask & ~PIPESTAT_INT_STATUS_MASK,
734 "pipe %c: status_mask=0x%x\n",
735 pipe_name(pipe), status_mask);
736
737 lockdep_assert_held(&dev_priv->irq_lock);
738 WARN_ON(!intel_irqs_enabled(dev_priv));
739
740 if ((dev_priv->pipestat_irq_mask[pipe] & status_mask) == 0)
741 return;
742
743 dev_priv->pipestat_irq_mask[pipe] &= ~status_mask;
744 enable_mask = i915_pipestat_enable_mask(dev_priv, pipe);
745
746 I915_WRITE(reg, enable_mask | status_mask);
747 POSTING_READ(reg);
Imre Deak755e9012014-02-10 18:42:47 +0200748}
749
=?utf-8?q?Michel_D=C3=A4nzer?=a6b54f32006-10-24 23:37:43 +1000750/**
Jani Nikulaf49e38d2013-04-29 13:02:54 +0300751 * i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion
Tvrtko Ursulin14bb2c12016-06-03 14:02:17 +0100752 * @dev_priv: i915 device private
Zhao Yakui01c66882009-10-28 05:10:00 +0000753 */
Tvrtko Ursulin91d14252016-05-06 14:48:28 +0100754static void i915_enable_asle_pipestat(struct drm_i915_private *dev_priv)
Zhao Yakui01c66882009-10-28 05:10:00 +0000755{
Tvrtko Ursulin91d14252016-05-06 14:48:28 +0100756 if (!dev_priv->opregion.asle || !IS_MOBILE(dev_priv))
Jani Nikulaf49e38d2013-04-29 13:02:54 +0300757 return;
758
Daniel Vetter13321782014-09-15 14:55:29 +0200759 spin_lock_irq(&dev_priv->irq_lock);
Zhao Yakui01c66882009-10-28 05:10:00 +0000760
Imre Deak755e9012014-02-10 18:42:47 +0200761 i915_enable_pipestat(dev_priv, PIPE_B, PIPE_LEGACY_BLC_EVENT_STATUS);
Tvrtko Ursulin91d14252016-05-06 14:48:28 +0100762 if (INTEL_GEN(dev_priv) >= 4)
Daniel Vetter3b6c42e2013-10-21 18:04:35 +0200763 i915_enable_pipestat(dev_priv, PIPE_A,
Imre Deak755e9012014-02-10 18:42:47 +0200764 PIPE_LEGACY_BLC_EVENT_STATUS);
Chris Wilson1ec14ad2010-12-04 11:30:53 +0000765
Daniel Vetter13321782014-09-15 14:55:29 +0200766 spin_unlock_irq(&dev_priv->irq_lock);
Zhao Yakui01c66882009-10-28 05:10:00 +0000767}
768
Ville Syrjäläf75f3742014-05-15 20:20:36 +0300769/*
770 * This timing diagram depicts the video signal in and
771 * around the vertical blanking period.
772 *
773 * Assumptions about the fictitious mode used in this example:
774 * vblank_start >= 3
775 * vsync_start = vblank_start + 1
776 * vsync_end = vblank_start + 2
777 * vtotal = vblank_start + 3
778 *
779 * start of vblank:
780 * latch double buffered registers
781 * increment frame counter (ctg+)
782 * generate start of vblank interrupt (gen4+)
783 * |
784 * | frame start:
785 * | generate frame start interrupt (aka. vblank interrupt) (gmch)
786 * | may be shifted forward 1-3 extra lines via PIPECONF
787 * | |
788 * | | start of vsync:
789 * | | generate vsync interrupt
790 * | | |
791 * ___xxxx___ ___xxxx___ ___xxxx___ ___xxxx___ ___xxxx___ ___xxxx
792 * . \hs/ . \hs/ \hs/ \hs/ . \hs/
793 * ----va---> <-----------------vb--------------------> <--------va-------------
794 * | | <----vs-----> |
795 * -vbs-----> <---vbs+1---> <---vbs+2---> <-----0-----> <-----1-----> <-----2--- (scanline counter gen2)
796 * -vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2---> <-----0--- (scanline counter gen3+)
797 * -vbs-2---> <---vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2- (scanline counter hsw+ hdmi)
798 * | | |
799 * last visible pixel first visible pixel
800 * | increment frame counter (gen3/4)
801 * pixel counter = vblank_start * htotal pixel counter = 0 (gen3/4)
802 *
803 * x = horizontal active
804 * _ = horizontal blanking
805 * hs = horizontal sync
806 * va = vertical active
807 * vb = vertical blanking
808 * vs = vertical sync
809 * vbs = vblank_start (number)
810 *
811 * Summary:
812 * - most events happen at the start of horizontal sync
813 * - frame start happens at the start of horizontal blank, 1-4 lines
814 * (depending on PIPECONF settings) after the start of vblank
815 * - gen3/4 pixel and frame counter are synchronized with the start
816 * of horizontal active on the first line of vertical active
817 */
818
Keith Packard42f52ef2008-10-18 19:39:29 -0700819/* Called from drm generic code, passed a 'crtc', which
820 * we use as a pipe index
821 */
Thierry Reding88e72712015-09-24 18:35:31 +0200822static u32 i915_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
Jesse Barnes0a3e67a2008-09-30 12:14:26 -0700823{
Chris Wilsonfac5e232016-07-04 11:34:36 +0100824 struct drm_i915_private *dev_priv = to_i915(dev);
Ville Syrjäläf0f59a02015-11-18 15:33:26 +0200825 i915_reg_t high_frame, low_frame;
Ville Syrjälä0b2a8e02014-04-29 13:35:50 +0300826 u32 high1, high2, low, pixel, vbl_start, hsync_start, htotal;
Daniel Vetter5caa0fe2017-05-09 16:03:29 +0200827 const struct drm_display_mode *mode = &dev->vblank[pipe].hwmode;
Ville Syrjälä694e4092017-03-09 17:44:30 +0200828 unsigned long irqflags;
Jesse Barnes0a3e67a2008-09-30 12:14:26 -0700829
Daniel Vetterf3a5c3f2015-02-13 21:03:44 +0100830 htotal = mode->crtc_htotal;
831 hsync_start = mode->crtc_hsync_start;
832 vbl_start = mode->crtc_vblank_start;
833 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
834 vbl_start = DIV_ROUND_UP(vbl_start, 2);
Ville Syrjälä391f75e2013-09-25 19:55:26 +0300835
Ville Syrjälä0b2a8e02014-04-29 13:35:50 +0300836 /* Convert to pixel count */
837 vbl_start *= htotal;
838
839 /* Start of vblank event occurs at start of hsync */
840 vbl_start -= htotal - hsync_start;
841
Jesse Barnes9db4a9c2011-02-07 12:26:52 -0800842 high_frame = PIPEFRAME(pipe);
843 low_frame = PIPEFRAMEPIXEL(pipe);
Chris Wilson5eddb702010-09-11 13:48:45 +0100844
Ville Syrjälä694e4092017-03-09 17:44:30 +0200845 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
846
Jesse Barnes0a3e67a2008-09-30 12:14:26 -0700847 /*
848 * High & low register fields aren't synchronized, so make sure
849 * we get a low value that's stable across two reads of the high
850 * register.
851 */
852 do {
Ville Syrjälä694e4092017-03-09 17:44:30 +0200853 high1 = I915_READ_FW(high_frame) & PIPE_FRAME_HIGH_MASK;
854 low = I915_READ_FW(low_frame);
855 high2 = I915_READ_FW(high_frame) & PIPE_FRAME_HIGH_MASK;
Jesse Barnes0a3e67a2008-09-30 12:14:26 -0700856 } while (high1 != high2);
857
Ville Syrjälä694e4092017-03-09 17:44:30 +0200858 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
859
Chris Wilson5eddb702010-09-11 13:48:45 +0100860 high1 >>= PIPE_FRAME_HIGH_SHIFT;
Ville Syrjälä391f75e2013-09-25 19:55:26 +0300861 pixel = low & PIPE_PIXEL_MASK;
Chris Wilson5eddb702010-09-11 13:48:45 +0100862 low >>= PIPE_FRAME_LOW_SHIFT;
Ville Syrjälä391f75e2013-09-25 19:55:26 +0300863
864 /*
865 * The frame counter increments at beginning of active.
866 * Cook up a vblank counter by also checking the pixel
867 * counter against vblank start.
868 */
Ville Syrjäläedc08d02013-11-06 13:56:27 -0200869 return (((high1 << 8) | low) + (pixel >= vbl_start)) & 0xffffff;
Jesse Barnes0a3e67a2008-09-30 12:14:26 -0700870}
871
Dave Airlie974e59b2015-10-30 09:45:33 +1000872static u32 g4x_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
Jesse Barnes9880b7a2009-02-06 10:22:41 -0800873{
Chris Wilsonfac5e232016-07-04 11:34:36 +0100874 struct drm_i915_private *dev_priv = to_i915(dev);
Jesse Barnes9880b7a2009-02-06 10:22:41 -0800875
Ville Syrjälä649636e2015-09-22 19:50:01 +0300876 return I915_READ(PIPE_FRMCOUNT_G4X(pipe));
Jesse Barnes9880b7a2009-02-06 10:22:41 -0800877}
878
Uma Shankaraec02462017-09-25 19:26:01 +0530879/*
880 * On certain encoders on certain platforms, pipe
881 * scanline register will not work to get the scanline,
882 * since the timings are driven from the PORT or issues
883 * with scanline register updates.
884 * This function will use Framestamp and current
885 * timestamp registers to calculate the scanline.
886 */
887static u32 __intel_get_crtc_scanline_from_timestamp(struct intel_crtc *crtc)
888{
889 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
890 struct drm_vblank_crtc *vblank =
891 &crtc->base.dev->vblank[drm_crtc_index(&crtc->base)];
892 const struct drm_display_mode *mode = &vblank->hwmode;
893 u32 vblank_start = mode->crtc_vblank_start;
894 u32 vtotal = mode->crtc_vtotal;
895 u32 htotal = mode->crtc_htotal;
896 u32 clock = mode->crtc_clock;
897 u32 scanline, scan_prev_time, scan_curr_time, scan_post_time;
898
899 /*
900 * To avoid the race condition where we might cross into the
901 * next vblank just between the PIPE_FRMTMSTMP and TIMESTAMP_CTR
902 * reads. We make sure we read PIPE_FRMTMSTMP and TIMESTAMP_CTR
903 * during the same frame.
904 */
905 do {
906 /*
907 * This field provides read back of the display
908 * pipe frame time stamp. The time stamp value
909 * is sampled at every start of vertical blank.
910 */
911 scan_prev_time = I915_READ_FW(PIPE_FRMTMSTMP(crtc->pipe));
912
913 /*
914 * The TIMESTAMP_CTR register has the current
915 * time stamp value.
916 */
917 scan_curr_time = I915_READ_FW(IVB_TIMESTAMP_CTR);
918
919 scan_post_time = I915_READ_FW(PIPE_FRMTMSTMP(crtc->pipe));
920 } while (scan_post_time != scan_prev_time);
921
922 scanline = div_u64(mul_u32_u32(scan_curr_time - scan_prev_time,
923 clock), 1000 * htotal);
924 scanline = min(scanline, vtotal - 1);
925 scanline = (scanline + vblank_start) % vtotal;
926
927 return scanline;
928}
929
Ville Syrjälä75aa3f62015-10-22 15:34:56 +0300930/* I915_READ_FW, only for fast reads of display block, no need for forcewake etc. */
Ville Syrjäläa225f072014-04-29 13:35:45 +0300931static int __intel_get_crtc_scanline(struct intel_crtc *crtc)
932{
933 struct drm_device *dev = crtc->base.dev;
Chris Wilsonfac5e232016-07-04 11:34:36 +0100934 struct drm_i915_private *dev_priv = to_i915(dev);
Daniel Vetter5caa0fe2017-05-09 16:03:29 +0200935 const struct drm_display_mode *mode;
936 struct drm_vblank_crtc *vblank;
Ville Syrjäläa225f072014-04-29 13:35:45 +0300937 enum pipe pipe = crtc->pipe;
Ville Syrjälä80715b22014-05-15 20:23:23 +0300938 int position, vtotal;
Ville Syrjäläa225f072014-04-29 13:35:45 +0300939
Ville Syrjälä72259532017-03-02 19:15:05 +0200940 if (!crtc->active)
941 return -1;
942
Daniel Vetter5caa0fe2017-05-09 16:03:29 +0200943 vblank = &crtc->base.dev->vblank[drm_crtc_index(&crtc->base)];
944 mode = &vblank->hwmode;
945
Uma Shankaraec02462017-09-25 19:26:01 +0530946 if (mode->private_flags & I915_MODE_FLAG_GET_SCANLINE_FROM_TIMESTAMP)
947 return __intel_get_crtc_scanline_from_timestamp(crtc);
948
Ville Syrjälä80715b22014-05-15 20:23:23 +0300949 vtotal = mode->crtc_vtotal;
Ville Syrjäläa225f072014-04-29 13:35:45 +0300950 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
951 vtotal /= 2;
952
Tvrtko Ursulin91d14252016-05-06 14:48:28 +0100953 if (IS_GEN2(dev_priv))
Ville Syrjälä75aa3f62015-10-22 15:34:56 +0300954 position = I915_READ_FW(PIPEDSL(pipe)) & DSL_LINEMASK_GEN2;
Ville Syrjäläa225f072014-04-29 13:35:45 +0300955 else
Ville Syrjälä75aa3f62015-10-22 15:34:56 +0300956 position = I915_READ_FW(PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;
Ville Syrjäläa225f072014-04-29 13:35:45 +0300957
958 /*
Jesse Barnes41b578f2015-09-22 12:15:54 -0700959 * On HSW, the DSL reg (0x70000) appears to return 0 if we
960 * read it just before the start of vblank. So try it again
961 * so we don't accidentally end up spanning a vblank frame
962 * increment, causing the pipe_update_end() code to squak at us.
963 *
964 * The nature of this problem means we can't simply check the ISR
965 * bit and return the vblank start value; nor can we use the scanline
966 * debug register in the transcoder as it appears to have the same
967 * problem. We may need to extend this to include other platforms,
968 * but so far testing only shows the problem on HSW.
969 */
Tvrtko Ursulin91d14252016-05-06 14:48:28 +0100970 if (HAS_DDI(dev_priv) && !position) {
Jesse Barnes41b578f2015-09-22 12:15:54 -0700971 int i, temp;
972
973 for (i = 0; i < 100; i++) {
974 udelay(1);
Ville Syrjälä707bdd32017-03-09 17:44:31 +0200975 temp = I915_READ_FW(PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;
Jesse Barnes41b578f2015-09-22 12:15:54 -0700976 if (temp != position) {
977 position = temp;
978 break;
979 }
980 }
981 }
982
983 /*
Ville Syrjälä80715b22014-05-15 20:23:23 +0300984 * See update_scanline_offset() for the details on the
985 * scanline_offset adjustment.
Ville Syrjäläa225f072014-04-29 13:35:45 +0300986 */
Ville Syrjälä80715b22014-05-15 20:23:23 +0300987 return (position + crtc->scanline_offset) % vtotal;
Ville Syrjäläa225f072014-04-29 13:35:45 +0300988}
989
Daniel Vetter1bf6ad62017-05-09 16:03:28 +0200990static bool i915_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe,
991 bool in_vblank_irq, int *vpos, int *hpos,
992 ktime_t *stime, ktime_t *etime,
993 const struct drm_display_mode *mode)
Mario Kleiner0af7e4d2010-12-08 04:07:19 +0100994{
Chris Wilsonfac5e232016-07-04 11:34:36 +0100995 struct drm_i915_private *dev_priv = to_i915(dev);
Ville Syrjälä98187832016-10-31 22:37:10 +0200996 struct intel_crtc *intel_crtc = intel_get_crtc_for_pipe(dev_priv,
997 pipe);
Ville Syrjälä3aa18df2013-10-11 19:10:32 +0300998 int position;
Ville Syrjälä78e8fc62014-04-29 13:35:44 +0300999 int vbl_start, vbl_end, hsync_start, htotal, vtotal;
Mario Kleinerad3543e2013-10-30 05:13:08 +01001000 unsigned long irqflags;
Mario Kleiner0af7e4d2010-12-08 04:07:19 +01001001
Maarten Lankhorstfc467a222015-06-01 12:50:07 +02001002 if (WARN_ON(!mode->crtc_clock)) {
Mario Kleiner0af7e4d2010-12-08 04:07:19 +01001003 DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled "
Jesse Barnes9db4a9c2011-02-07 12:26:52 -08001004 "pipe %c\n", pipe_name(pipe));
Daniel Vetter1bf6ad62017-05-09 16:03:28 +02001005 return false;
Mario Kleiner0af7e4d2010-12-08 04:07:19 +01001006 }
1007
Ville Syrjäläc2baf4b2013-09-23 14:48:50 +03001008 htotal = mode->crtc_htotal;
Ville Syrjälä78e8fc62014-04-29 13:35:44 +03001009 hsync_start = mode->crtc_hsync_start;
Ville Syrjäläc2baf4b2013-09-23 14:48:50 +03001010 vtotal = mode->crtc_vtotal;
1011 vbl_start = mode->crtc_vblank_start;
1012 vbl_end = mode->crtc_vblank_end;
Mario Kleiner0af7e4d2010-12-08 04:07:19 +01001013
Ville Syrjäläd31faf62013-10-28 16:31:41 +02001014 if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
1015 vbl_start = DIV_ROUND_UP(vbl_start, 2);
1016 vbl_end /= 2;
1017 vtotal /= 2;
1018 }
1019
Mario Kleinerad3543e2013-10-30 05:13:08 +01001020 /*
1021 * Lock uncore.lock, as we will do multiple timing critical raw
1022 * register reads, potentially with preemption disabled, so the
1023 * following code must not block on uncore.lock.
1024 */
1025 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
Ville Syrjälä78e8fc62014-04-29 13:35:44 +03001026
Mario Kleinerad3543e2013-10-30 05:13:08 +01001027 /* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */
1028
1029 /* Get optional system timestamp before query. */
1030 if (stime)
1031 *stime = ktime_get();
1032
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01001033 if (IS_GEN2(dev_priv) || IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5) {
Mario Kleiner0af7e4d2010-12-08 04:07:19 +01001034 /* No obvious pixelcount register. Only query vertical
1035 * scanout position from Display scan line register.
1036 */
Ville Syrjäläa225f072014-04-29 13:35:45 +03001037 position = __intel_get_crtc_scanline(intel_crtc);
Mario Kleiner0af7e4d2010-12-08 04:07:19 +01001038 } else {
1039 /* Have access to pixelcount since start of frame.
1040 * We can split this into vertical and horizontal
1041 * scanout position.
1042 */
Ville Syrjälä75aa3f62015-10-22 15:34:56 +03001043 position = (I915_READ_FW(PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
Mario Kleiner0af7e4d2010-12-08 04:07:19 +01001044
Ville Syrjälä3aa18df2013-10-11 19:10:32 +03001045 /* convert to pixel counts */
1046 vbl_start *= htotal;
1047 vbl_end *= htotal;
1048 vtotal *= htotal;
Ville Syrjälä78e8fc62014-04-29 13:35:44 +03001049
1050 /*
Ville Syrjälä7e78f1cb2014-04-29 13:35:49 +03001051 * In interlaced modes, the pixel counter counts all pixels,
1052 * so one field will have htotal more pixels. In order to avoid
1053 * the reported position from jumping backwards when the pixel
1054 * counter is beyond the length of the shorter field, just
1055 * clamp the position the length of the shorter field. This
1056 * matches how the scanline counter based position works since
1057 * the scanline counter doesn't count the two half lines.
1058 */
1059 if (position >= vtotal)
1060 position = vtotal - 1;
1061
1062 /*
Ville Syrjälä78e8fc62014-04-29 13:35:44 +03001063 * Start of vblank interrupt is triggered at start of hsync,
1064 * just prior to the first active line of vblank. However we
1065 * consider lines to start at the leading edge of horizontal
1066 * active. So, should we get here before we've crossed into
1067 * the horizontal active of the first line in vblank, we would
1068 * not set the DRM_SCANOUTPOS_INVBL flag. In order to fix that,
1069 * always add htotal-hsync_start to the current pixel position.
1070 */
1071 position = (position + htotal - hsync_start) % vtotal;
Ville Syrjälä3aa18df2013-10-11 19:10:32 +03001072 }
1073
Mario Kleinerad3543e2013-10-30 05:13:08 +01001074 /* Get optional system timestamp after query. */
1075 if (etime)
1076 *etime = ktime_get();
1077
1078 /* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */
1079
1080 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
1081
Ville Syrjälä3aa18df2013-10-11 19:10:32 +03001082 /*
1083 * While in vblank, position will be negative
1084 * counting up towards 0 at vbl_end. And outside
1085 * vblank, position will be positive counting
1086 * up since vbl_end.
1087 */
1088 if (position >= vbl_start)
1089 position -= vbl_end;
1090 else
1091 position += vtotal - vbl_end;
1092
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01001093 if (IS_GEN2(dev_priv) || IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5) {
Ville Syrjälä3aa18df2013-10-11 19:10:32 +03001094 *vpos = position;
1095 *hpos = 0;
1096 } else {
Mario Kleiner0af7e4d2010-12-08 04:07:19 +01001097 *vpos = position / htotal;
1098 *hpos = position - (*vpos * htotal);
1099 }
1100
Daniel Vetter1bf6ad62017-05-09 16:03:28 +02001101 return true;
Mario Kleiner0af7e4d2010-12-08 04:07:19 +01001102}
1103
Ville Syrjäläa225f072014-04-29 13:35:45 +03001104int intel_get_crtc_scanline(struct intel_crtc *crtc)
1105{
Chris Wilsonfac5e232016-07-04 11:34:36 +01001106 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
Ville Syrjäläa225f072014-04-29 13:35:45 +03001107 unsigned long irqflags;
1108 int position;
1109
1110 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
1111 position = __intel_get_crtc_scanline(crtc);
1112 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
1113
1114 return position;
1115}
1116
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01001117static void ironlake_rps_change_irq_handler(struct drm_i915_private *dev_priv)
Jesse Barnesf97108d2010-01-29 11:27:07 -08001118{
Matthew Garrettb5b72e82010-02-02 18:30:47 +00001119 u32 busy_up, busy_down, max_avg, min_avg;
Daniel Vetter92703882012-08-09 16:46:01 +02001120 u8 new_delay;
Daniel Vetter92703882012-08-09 16:46:01 +02001121
Daniel Vetterd0ecd7e2013-07-04 23:35:25 +02001122 spin_lock(&mchdev_lock);
Jesse Barnesf97108d2010-01-29 11:27:07 -08001123
Daniel Vetter73edd18f2012-08-08 23:35:37 +02001124 I915_WRITE16(MEMINTRSTS, I915_READ(MEMINTRSTS));
1125
Daniel Vetter20e4d402012-08-08 23:35:39 +02001126 new_delay = dev_priv->ips.cur_delay;
Daniel Vetter92703882012-08-09 16:46:01 +02001127
Jesse Barnes7648fa92010-05-20 14:28:11 -07001128 I915_WRITE16(MEMINTRSTS, MEMINT_EVAL_CHG);
Matthew Garrettb5b72e82010-02-02 18:30:47 +00001129 busy_up = I915_READ(RCPREVBSYTUPAVG);
1130 busy_down = I915_READ(RCPREVBSYTDNAVG);
Jesse Barnesf97108d2010-01-29 11:27:07 -08001131 max_avg = I915_READ(RCBMAXAVG);
1132 min_avg = I915_READ(RCBMINAVG);
1133
1134 /* Handle RCS change request from hw */
Matthew Garrettb5b72e82010-02-02 18:30:47 +00001135 if (busy_up > max_avg) {
Daniel Vetter20e4d402012-08-08 23:35:39 +02001136 if (dev_priv->ips.cur_delay != dev_priv->ips.max_delay)
1137 new_delay = dev_priv->ips.cur_delay - 1;
1138 if (new_delay < dev_priv->ips.max_delay)
1139 new_delay = dev_priv->ips.max_delay;
Matthew Garrettb5b72e82010-02-02 18:30:47 +00001140 } else if (busy_down < min_avg) {
Daniel Vetter20e4d402012-08-08 23:35:39 +02001141 if (dev_priv->ips.cur_delay != dev_priv->ips.min_delay)
1142 new_delay = dev_priv->ips.cur_delay + 1;
1143 if (new_delay > dev_priv->ips.min_delay)
1144 new_delay = dev_priv->ips.min_delay;
Jesse Barnesf97108d2010-01-29 11:27:07 -08001145 }
1146
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01001147 if (ironlake_set_drps(dev_priv, new_delay))
Daniel Vetter20e4d402012-08-08 23:35:39 +02001148 dev_priv->ips.cur_delay = new_delay;
Jesse Barnesf97108d2010-01-29 11:27:07 -08001149
Daniel Vetterd0ecd7e2013-07-04 23:35:25 +02001150 spin_unlock(&mchdev_lock);
Daniel Vetter92703882012-08-09 16:46:01 +02001151
Jesse Barnesf97108d2010-01-29 11:27:07 -08001152 return;
1153}
1154
Tvrtko Ursulin0bc40be2016-03-16 11:00:37 +00001155static void notify_ring(struct intel_engine_cs *engine)
Chris Wilson549f7362010-10-19 11:19:32 +01001156{
Chris Wilson3f883252018-06-27 21:13:01 +01001157 const u32 seqno = intel_engine_get_seqno(engine);
Chris Wilsone61e0f52018-02-21 09:56:36 +00001158 struct i915_request *rq = NULL;
Chris Wilson3f883252018-06-27 21:13:01 +01001159 struct task_struct *tsk = NULL;
Chris Wilson56299fb2017-02-27 20:58:48 +00001160 struct intel_wait *wait;
Tvrtko Ursulindffabc82017-02-21 09:13:48 +00001161
Chris Wilson3f883252018-06-27 21:13:01 +01001162 if (unlikely(!engine->breadcrumbs.irq_armed))
Chris Wilsonbcbd5c32017-10-25 15:39:42 +01001163 return;
1164
Chris Wilson3f883252018-06-27 21:13:01 +01001165 rcu_read_lock();
Chris Wilson56299fb2017-02-27 20:58:48 +00001166
Chris Wilson61d3dc72017-03-03 19:08:24 +00001167 spin_lock(&engine->breadcrumbs.irq_lock);
1168 wait = engine->breadcrumbs.irq_wait;
Chris Wilson56299fb2017-02-27 20:58:48 +00001169 if (wait) {
Chris Wilson3f883252018-06-27 21:13:01 +01001170 /*
1171 * We use a callback from the dma-fence to submit
Chris Wilson56299fb2017-02-27 20:58:48 +00001172 * requests after waiting on our own requests. To
1173 * ensure minimum delay in queuing the next request to
1174 * hardware, signal the fence now rather than wait for
1175 * the signaler to be woken up. We still wake up the
1176 * waiter in order to handle the irq-seqno coherency
1177 * issues (we may receive the interrupt before the
1178 * seqno is written, see __i915_request_irq_complete())
1179 * and to handle coalescing of multiple seqno updates
1180 * and many waiters.
1181 */
Chris Wilson3f883252018-06-27 21:13:01 +01001182 if (i915_seqno_passed(seqno, wait->seqno)) {
Chris Wilsone61e0f52018-02-21 09:56:36 +00001183 struct i915_request *waiter = wait->request;
Chris Wilsonde4d2102017-09-18 17:27:34 +01001184
Chris Wilson17b51ad2017-09-18 17:27:33 +01001185 if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
Chris Wilsonde4d2102017-09-18 17:27:34 +01001186 &waiter->fence.flags) &&
1187 intel_wait_check_request(wait, waiter))
Chris Wilsone61e0f52018-02-21 09:56:36 +00001188 rq = i915_request_get(waiter);
Chris Wilson56299fb2017-02-27 20:58:48 +00001189
Chris Wilson3f883252018-06-27 21:13:01 +01001190 tsk = wait->tsk;
1191 } else {
Chris Wilson69dc4d02018-06-27 21:13:02 +01001192 if (engine->irq_seqno_barrier &&
1193 i915_seqno_passed(seqno, wait->seqno - 1)) {
Chris Wilson3f883252018-06-27 21:13:01 +01001194 set_bit(ENGINE_IRQ_BREADCRUMB,
1195 &engine->irq_posted);
1196 tsk = wait->tsk;
1197 }
1198 }
Chris Wilson78796872018-06-27 21:13:03 +01001199
1200 engine->breadcrumbs.irq_count++;
Chris Wilson67b807a82017-02-27 20:58:50 +00001201 } else {
Chris Wilsonbcbd5c32017-10-25 15:39:42 +01001202 if (engine->breadcrumbs.irq_armed)
1203 __intel_engine_disarm_breadcrumbs(engine);
Chris Wilson56299fb2017-02-27 20:58:48 +00001204 }
Chris Wilson61d3dc72017-03-03 19:08:24 +00001205 spin_unlock(&engine->breadcrumbs.irq_lock);
Chris Wilson56299fb2017-02-27 20:58:48 +00001206
Chris Wilson24754d72017-03-03 14:45:57 +00001207 if (rq) {
Chris Wilson56299fb2017-02-27 20:58:48 +00001208 dma_fence_signal(&rq->fence);
Chris Wilson4e9a8be2018-03-05 10:41:05 +00001209 GEM_BUG_ON(!i915_request_completed(rq));
Chris Wilsone61e0f52018-02-21 09:56:36 +00001210 i915_request_put(rq);
Chris Wilson24754d72017-03-03 14:45:57 +00001211 }
Chris Wilson56299fb2017-02-27 20:58:48 +00001212
Chris Wilson3f883252018-06-27 21:13:01 +01001213 if (tsk && tsk->state & TASK_NORMAL)
1214 wake_up_process(tsk);
1215
1216 rcu_read_unlock();
1217
Chris Wilson56299fb2017-02-27 20:58:48 +00001218 trace_intel_engine_notify(engine, wait);
Chris Wilson549f7362010-10-19 11:19:32 +01001219}
1220
Chris Wilson43cf3bf2015-03-18 09:48:22 +00001221static void vlv_c0_read(struct drm_i915_private *dev_priv,
1222 struct intel_rps_ei *ei)
Deepak S31685c22014-07-03 17:33:01 -04001223{
Mika Kuoppala679cb6c2017-03-15 17:43:03 +02001224 ei->ktime = ktime_get_raw();
Chris Wilson43cf3bf2015-03-18 09:48:22 +00001225 ei->render_c0 = I915_READ(VLV_RENDER_C0_COUNT);
1226 ei->media_c0 = I915_READ(VLV_MEDIA_C0_COUNT);
Deepak S31685c22014-07-03 17:33:01 -04001227}
1228
Chris Wilson43cf3bf2015-03-18 09:48:22 +00001229void gen6_rps_reset_ei(struct drm_i915_private *dev_priv)
1230{
Sagar Arun Kamble562d9ba2017-10-10 22:30:06 +01001231 memset(&dev_priv->gt_pm.rps.ei, 0, sizeof(dev_priv->gt_pm.rps.ei));
Chris Wilson43cf3bf2015-03-18 09:48:22 +00001232}
1233
1234static u32 vlv_wa_c0_ei(struct drm_i915_private *dev_priv, u32 pm_iir)
1235{
Sagar Arun Kamble562d9ba2017-10-10 22:30:06 +01001236 struct intel_rps *rps = &dev_priv->gt_pm.rps;
1237 const struct intel_rps_ei *prev = &rps->ei;
Chris Wilson43cf3bf2015-03-18 09:48:22 +00001238 struct intel_rps_ei now;
1239 u32 events = 0;
1240
Chris Wilsone0e8c7c2017-03-09 21:12:30 +00001241 if ((pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) == 0)
Chris Wilson43cf3bf2015-03-18 09:48:22 +00001242 return 0;
1243
1244 vlv_c0_read(dev_priv, &now);
Deepak S31685c22014-07-03 17:33:01 -04001245
Mika Kuoppala679cb6c2017-03-15 17:43:03 +02001246 if (prev->ktime) {
Chris Wilsone0e8c7c2017-03-09 21:12:30 +00001247 u64 time, c0;
Chris Wilson569884e2017-03-09 21:12:31 +00001248 u32 render, media;
Chris Wilsone0e8c7c2017-03-09 21:12:30 +00001249
Mika Kuoppala679cb6c2017-03-15 17:43:03 +02001250 time = ktime_us_delta(now.ktime, prev->ktime);
Chris Wilson8f68d592017-03-13 17:06:17 +00001251
Chris Wilsone0e8c7c2017-03-09 21:12:30 +00001252 time *= dev_priv->czclk_freq;
1253
1254 /* Workload can be split between render + media,
1255 * e.g. SwapBuffers being blitted in X after being rendered in
1256 * mesa. To account for this we need to combine both engines
1257 * into our activity counter.
1258 */
Chris Wilson569884e2017-03-09 21:12:31 +00001259 render = now.render_c0 - prev->render_c0;
1260 media = now.media_c0 - prev->media_c0;
1261 c0 = max(render, media);
Mika Kuoppala6b7f6aa2017-03-15 18:12:59 +02001262 c0 *= 1000 * 100 << 8; /* to usecs and scale to threshold% */
Chris Wilsone0e8c7c2017-03-09 21:12:30 +00001263
Sagar Arun Kamble562d9ba2017-10-10 22:30:06 +01001264 if (c0 > time * rps->up_threshold)
Chris Wilsone0e8c7c2017-03-09 21:12:30 +00001265 events = GEN6_PM_RP_UP_THRESHOLD;
Sagar Arun Kamble562d9ba2017-10-10 22:30:06 +01001266 else if (c0 < time * rps->down_threshold)
Chris Wilsone0e8c7c2017-03-09 21:12:30 +00001267 events = GEN6_PM_RP_DOWN_THRESHOLD;
Deepak S31685c22014-07-03 17:33:01 -04001268 }
1269
Sagar Arun Kamble562d9ba2017-10-10 22:30:06 +01001270 rps->ei = now;
Chris Wilson43cf3bf2015-03-18 09:48:22 +00001271 return events;
Deepak S31685c22014-07-03 17:33:01 -04001272}
1273
Ben Widawsky4912d042011-04-25 11:25:20 -07001274static void gen6_pm_rps_work(struct work_struct *work)
Jesse Barnes3b8d8d92010-12-17 14:19:02 -08001275{
Jani Nikula2d1013d2014-03-31 14:27:17 +03001276 struct drm_i915_private *dev_priv =
Sagar Arun Kamble562d9ba2017-10-10 22:30:06 +01001277 container_of(work, struct drm_i915_private, gt_pm.rps.work);
1278 struct intel_rps *rps = &dev_priv->gt_pm.rps;
Chris Wilson7c0a16a2017-03-09 21:12:32 +00001279 bool client_boost = false;
Chris Wilson8d3afd72015-05-21 21:01:47 +01001280 int new_delay, adj, min, max;
Chris Wilson7c0a16a2017-03-09 21:12:32 +00001281 u32 pm_iir = 0;
Jesse Barnes3b8d8d92010-12-17 14:19:02 -08001282
Daniel Vetter59cdb632013-07-04 23:35:28 +02001283 spin_lock_irq(&dev_priv->irq_lock);
Sagar Arun Kamble562d9ba2017-10-10 22:30:06 +01001284 if (rps->interrupts_enabled) {
1285 pm_iir = fetch_and_zero(&rps->pm_iir);
1286 client_boost = atomic_read(&rps->num_waiters);
Imre Deakd4d70aa2014-11-19 15:30:04 +02001287 }
Daniel Vetter59cdb632013-07-04 23:35:28 +02001288 spin_unlock_irq(&dev_priv->irq_lock);
Ben Widawsky4912d042011-04-25 11:25:20 -07001289
Paulo Zanoni60611c12013-08-15 11:50:01 -03001290 /* Make sure we didn't queue anything we're not going to process. */
Deepak Sa6706b42014-03-15 20:23:22 +05301291 WARN_ON(pm_iir & ~dev_priv->pm_rps_events);
Chris Wilson8d3afd72015-05-21 21:01:47 +01001292 if ((pm_iir & dev_priv->pm_rps_events) == 0 && !client_boost)
Chris Wilson7c0a16a2017-03-09 21:12:32 +00001293 goto out;
Jesse Barnes3b8d8d92010-12-17 14:19:02 -08001294
Sagar Arun Kamble9f817502017-10-10 22:30:05 +01001295 mutex_lock(&dev_priv->pcu_lock);
Chris Wilson7b9e0ae2012-04-28 08:56:39 +01001296
Chris Wilson43cf3bf2015-03-18 09:48:22 +00001297 pm_iir |= vlv_wa_c0_ei(dev_priv, pm_iir);
1298
Sagar Arun Kamble562d9ba2017-10-10 22:30:06 +01001299 adj = rps->last_adj;
1300 new_delay = rps->cur_freq;
1301 min = rps->min_freq_softlimit;
1302 max = rps->max_freq_softlimit;
Chris Wilson7b92c1b2017-06-28 13:35:48 +01001303 if (client_boost)
Sagar Arun Kamble562d9ba2017-10-10 22:30:06 +01001304 max = rps->max_freq;
1305 if (client_boost && new_delay < rps->boost_freq) {
1306 new_delay = rps->boost_freq;
Chris Wilson8d3afd72015-05-21 21:01:47 +01001307 adj = 0;
1308 } else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
Chris Wilsondd75fdc2013-09-25 17:34:57 +01001309 if (adj > 0)
1310 adj *= 2;
Chris Wilsonedcf2842015-04-07 16:20:29 +01001311 else /* CHV needs even encode values */
1312 adj = IS_CHERRYVIEW(dev_priv) ? 2 : 1;
Sagar Arun Kamble7e79a682017-01-20 09:18:24 +05301313
Sagar Arun Kamble562d9ba2017-10-10 22:30:06 +01001314 if (new_delay >= rps->max_freq_softlimit)
Sagar Arun Kamble7e79a682017-01-20 09:18:24 +05301315 adj = 0;
Chris Wilson7b92c1b2017-06-28 13:35:48 +01001316 } else if (client_boost) {
Chris Wilsonf5a4c672015-04-27 13:41:23 +01001317 adj = 0;
Chris Wilsondd75fdc2013-09-25 17:34:57 +01001318 } else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
Sagar Arun Kamble562d9ba2017-10-10 22:30:06 +01001319 if (rps->cur_freq > rps->efficient_freq)
1320 new_delay = rps->efficient_freq;
1321 else if (rps->cur_freq > rps->min_freq_softlimit)
1322 new_delay = rps->min_freq_softlimit;
Chris Wilsondd75fdc2013-09-25 17:34:57 +01001323 adj = 0;
1324 } else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
1325 if (adj < 0)
1326 adj *= 2;
Chris Wilsonedcf2842015-04-07 16:20:29 +01001327 else /* CHV needs even encode values */
1328 adj = IS_CHERRYVIEW(dev_priv) ? -2 : -1;
Sagar Arun Kamble7e79a682017-01-20 09:18:24 +05301329
Sagar Arun Kamble562d9ba2017-10-10 22:30:06 +01001330 if (new_delay <= rps->min_freq_softlimit)
Sagar Arun Kamble7e79a682017-01-20 09:18:24 +05301331 adj = 0;
Chris Wilsondd75fdc2013-09-25 17:34:57 +01001332 } else { /* unknown event */
Chris Wilsonedcf2842015-04-07 16:20:29 +01001333 adj = 0;
Chris Wilsondd75fdc2013-09-25 17:34:57 +01001334 }
Jesse Barnes3b8d8d92010-12-17 14:19:02 -08001335
Sagar Arun Kamble562d9ba2017-10-10 22:30:06 +01001336 rps->last_adj = adj;
Chris Wilsonedcf2842015-04-07 16:20:29 +01001337
Ben Widawsky79249632012-09-07 19:43:42 -07001338 /* sysfs frequency interfaces may have snuck in while servicing the
1339 * interrupt
1340 */
Chris Wilsonedcf2842015-04-07 16:20:29 +01001341 new_delay += adj;
Chris Wilson8d3afd72015-05-21 21:01:47 +01001342 new_delay = clamp_t(int, new_delay, min, max);
Deepak S27544362014-01-27 21:35:05 +05301343
Chris Wilson9fcee2f2017-01-26 10:19:19 +00001344 if (intel_set_rps(dev_priv, new_delay)) {
1345 DRM_DEBUG_DRIVER("Failed to set new GPU frequency\n");
Sagar Arun Kamble562d9ba2017-10-10 22:30:06 +01001346 rps->last_adj = 0;
Chris Wilson9fcee2f2017-01-26 10:19:19 +00001347 }
Jesse Barnes3b8d8d92010-12-17 14:19:02 -08001348
Sagar Arun Kamble9f817502017-10-10 22:30:05 +01001349 mutex_unlock(&dev_priv->pcu_lock);
Chris Wilson7c0a16a2017-03-09 21:12:32 +00001350
1351out:
1352 /* Make sure not to corrupt PMIMR state used by ringbuffer on GEN6 */
1353 spin_lock_irq(&dev_priv->irq_lock);
Sagar Arun Kamble562d9ba2017-10-10 22:30:06 +01001354 if (rps->interrupts_enabled)
Chris Wilson7c0a16a2017-03-09 21:12:32 +00001355 gen6_unmask_pm_irq(dev_priv, dev_priv->pm_rps_events);
1356 spin_unlock_irq(&dev_priv->irq_lock);
Jesse Barnes3b8d8d92010-12-17 14:19:02 -08001357}
1358
Ben Widawskye3689192012-05-25 16:56:22 -07001359
1360/**
1361 * ivybridge_parity_work - Workqueue called when a parity error interrupt
1362 * occurred.
1363 * @work: workqueue struct
1364 *
1365 * Doesn't actually do anything except notify userspace. As a consequence of
1366 * this event, userspace should try to remap the bad rows since statistically
1367 * it is likely the same row is more likely to go bad again.
1368 */
1369static void ivybridge_parity_work(struct work_struct *work)
1370{
Jani Nikula2d1013d2014-03-31 14:27:17 +03001371 struct drm_i915_private *dev_priv =
Joonas Lahtinencefcff82017-04-28 10:58:39 +03001372 container_of(work, typeof(*dev_priv), l3_parity.error_work);
Ben Widawskye3689192012-05-25 16:56:22 -07001373 u32 error_status, row, bank, subbank;
Ben Widawsky35a85ac2013-09-19 11:13:41 -07001374 char *parity_event[6];
Ben Widawskye3689192012-05-25 16:56:22 -07001375 uint32_t misccpctl;
Ben Widawsky35a85ac2013-09-19 11:13:41 -07001376 uint8_t slice = 0;
Ben Widawskye3689192012-05-25 16:56:22 -07001377
1378 /* We must turn off DOP level clock gating to access the L3 registers.
1379 * In order to prevent a get/put style interface, acquire struct mutex
1380 * any time we access those registers.
1381 */
Chris Wilson91c8a322016-07-05 10:40:23 +01001382 mutex_lock(&dev_priv->drm.struct_mutex);
Ben Widawskye3689192012-05-25 16:56:22 -07001383
Ben Widawsky35a85ac2013-09-19 11:13:41 -07001384 /* If we've screwed up tracking, just let the interrupt fire again */
1385 if (WARN_ON(!dev_priv->l3_parity.which_slice))
1386 goto out;
1387
Ben Widawskye3689192012-05-25 16:56:22 -07001388 misccpctl = I915_READ(GEN7_MISCCPCTL);
1389 I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
1390 POSTING_READ(GEN7_MISCCPCTL);
1391
Ben Widawsky35a85ac2013-09-19 11:13:41 -07001392 while ((slice = ffs(dev_priv->l3_parity.which_slice)) != 0) {
Ville Syrjäläf0f59a02015-11-18 15:33:26 +02001393 i915_reg_t reg;
Ben Widawskye3689192012-05-25 16:56:22 -07001394
Ben Widawsky35a85ac2013-09-19 11:13:41 -07001395 slice--;
Joonas Lahtinen2d1fe072016-04-07 11:08:05 +03001396 if (WARN_ON_ONCE(slice >= NUM_L3_SLICES(dev_priv)))
Ben Widawsky35a85ac2013-09-19 11:13:41 -07001397 break;
1398
1399 dev_priv->l3_parity.which_slice &= ~(1<<slice);
1400
Ville Syrjälä6fa1c5f2015-11-04 23:20:02 +02001401 reg = GEN7_L3CDERRST1(slice);
Ben Widawsky35a85ac2013-09-19 11:13:41 -07001402
1403 error_status = I915_READ(reg);
1404 row = GEN7_PARITY_ERROR_ROW(error_status);
1405 bank = GEN7_PARITY_ERROR_BANK(error_status);
1406 subbank = GEN7_PARITY_ERROR_SUBBANK(error_status);
1407
1408 I915_WRITE(reg, GEN7_PARITY_ERROR_VALID | GEN7_L3CDERRST1_ENABLE);
1409 POSTING_READ(reg);
1410
1411 parity_event[0] = I915_L3_PARITY_UEVENT "=1";
1412 parity_event[1] = kasprintf(GFP_KERNEL, "ROW=%d", row);
1413 parity_event[2] = kasprintf(GFP_KERNEL, "BANK=%d", bank);
1414 parity_event[3] = kasprintf(GFP_KERNEL, "SUBBANK=%d", subbank);
1415 parity_event[4] = kasprintf(GFP_KERNEL, "SLICE=%d", slice);
1416 parity_event[5] = NULL;
1417
Chris Wilson91c8a322016-07-05 10:40:23 +01001418 kobject_uevent_env(&dev_priv->drm.primary->kdev->kobj,
Ben Widawsky35a85ac2013-09-19 11:13:41 -07001419 KOBJ_CHANGE, parity_event);
1420
1421 DRM_DEBUG("Parity error: Slice = %d, Row = %d, Bank = %d, Sub bank = %d.\n",
1422 slice, row, bank, subbank);
1423
1424 kfree(parity_event[4]);
1425 kfree(parity_event[3]);
1426 kfree(parity_event[2]);
1427 kfree(parity_event[1]);
1428 }
Ben Widawskye3689192012-05-25 16:56:22 -07001429
1430 I915_WRITE(GEN7_MISCCPCTL, misccpctl);
1431
Ben Widawsky35a85ac2013-09-19 11:13:41 -07001432out:
1433 WARN_ON(dev_priv->l3_parity.which_slice);
Daniel Vetter4cb21832014-09-15 14:55:26 +02001434 spin_lock_irq(&dev_priv->irq_lock);
Joonas Lahtinen2d1fe072016-04-07 11:08:05 +03001435 gen5_enable_gt_irq(dev_priv, GT_PARITY_ERROR(dev_priv));
Daniel Vetter4cb21832014-09-15 14:55:26 +02001436 spin_unlock_irq(&dev_priv->irq_lock);
Ben Widawskye3689192012-05-25 16:56:22 -07001437
Chris Wilson91c8a322016-07-05 10:40:23 +01001438 mutex_unlock(&dev_priv->drm.struct_mutex);
Ben Widawskye3689192012-05-25 16:56:22 -07001439}
1440
Ville Syrjälä261e40b2016-04-13 21:19:57 +03001441static void ivybridge_parity_error_irq_handler(struct drm_i915_private *dev_priv,
1442 u32 iir)
Ben Widawskye3689192012-05-25 16:56:22 -07001443{
Ville Syrjälä261e40b2016-04-13 21:19:57 +03001444 if (!HAS_L3_DPF(dev_priv))
Ben Widawskye3689192012-05-25 16:56:22 -07001445 return;
1446
Daniel Vetterd0ecd7e2013-07-04 23:35:25 +02001447 spin_lock(&dev_priv->irq_lock);
Ville Syrjälä261e40b2016-04-13 21:19:57 +03001448 gen5_disable_gt_irq(dev_priv, GT_PARITY_ERROR(dev_priv));
Daniel Vetterd0ecd7e2013-07-04 23:35:25 +02001449 spin_unlock(&dev_priv->irq_lock);
Ben Widawskye3689192012-05-25 16:56:22 -07001450
Ville Syrjälä261e40b2016-04-13 21:19:57 +03001451 iir &= GT_PARITY_ERROR(dev_priv);
Ben Widawsky35a85ac2013-09-19 11:13:41 -07001452 if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1)
1453 dev_priv->l3_parity.which_slice |= 1 << 1;
1454
1455 if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT)
1456 dev_priv->l3_parity.which_slice |= 1 << 0;
1457
Daniel Vettera4da4fa2012-11-02 19:55:07 +01001458 queue_work(dev_priv->wq, &dev_priv->l3_parity.error_work);
Ben Widawskye3689192012-05-25 16:56:22 -07001459}
1460
Ville Syrjälä261e40b2016-04-13 21:19:57 +03001461static void ilk_gt_irq_handler(struct drm_i915_private *dev_priv,
Paulo Zanonif1af8fc2013-07-12 19:56:30 -03001462 u32 gt_iir)
1463{
Chris Wilsonf8973c22016-07-01 17:23:21 +01001464 if (gt_iir & GT_RENDER_USER_INTERRUPT)
Akash Goel3b3f1652016-10-13 22:44:48 +05301465 notify_ring(dev_priv->engine[RCS]);
Paulo Zanonif1af8fc2013-07-12 19:56:30 -03001466 if (gt_iir & ILK_BSD_USER_INTERRUPT)
Akash Goel3b3f1652016-10-13 22:44:48 +05301467 notify_ring(dev_priv->engine[VCS]);
Paulo Zanonif1af8fc2013-07-12 19:56:30 -03001468}
1469
Ville Syrjälä261e40b2016-04-13 21:19:57 +03001470static void snb_gt_irq_handler(struct drm_i915_private *dev_priv,
Daniel Vettere7b4c6b2012-03-30 20:24:35 +02001471 u32 gt_iir)
1472{
Chris Wilsonf8973c22016-07-01 17:23:21 +01001473 if (gt_iir & GT_RENDER_USER_INTERRUPT)
Akash Goel3b3f1652016-10-13 22:44:48 +05301474 notify_ring(dev_priv->engine[RCS]);
Ben Widawskycc609d52013-05-28 19:22:29 -07001475 if (gt_iir & GT_BSD_USER_INTERRUPT)
Akash Goel3b3f1652016-10-13 22:44:48 +05301476 notify_ring(dev_priv->engine[VCS]);
Ben Widawskycc609d52013-05-28 19:22:29 -07001477 if (gt_iir & GT_BLT_USER_INTERRUPT)
Akash Goel3b3f1652016-10-13 22:44:48 +05301478 notify_ring(dev_priv->engine[BCS]);
Daniel Vettere7b4c6b2012-03-30 20:24:35 +02001479
Ben Widawskycc609d52013-05-28 19:22:29 -07001480 if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT |
1481 GT_BSD_CS_ERROR_INTERRUPT |
Daniel Vetteraaecdf62014-11-04 15:52:22 +01001482 GT_RENDER_CS_MASTER_ERROR_INTERRUPT))
1483 DRM_DEBUG("Command parser error, gt_iir 0x%08x\n", gt_iir);
Ben Widawskye3689192012-05-25 16:56:22 -07001484
Ville Syrjälä261e40b2016-04-13 21:19:57 +03001485 if (gt_iir & GT_PARITY_ERROR(dev_priv))
1486 ivybridge_parity_error_irq_handler(dev_priv, gt_iir);
Daniel Vettere7b4c6b2012-03-30 20:24:35 +02001487}
1488
Chris Wilson5d3d69d2017-05-17 13:10:06 +01001489static void
Chris Wilson51f6b0f2018-03-09 01:08:08 +00001490gen8_cs_irq_handler(struct intel_engine_cs *engine, u32 iir)
Nick Hoathfbcc1a02015-10-20 10:23:52 +01001491{
Mika Kuoppalab620e872017-09-22 15:43:03 +03001492 struct intel_engine_execlists * const execlists = &engine->execlists;
Chris Wilson31de7352017-03-16 12:56:18 +00001493 bool tasklet = false;
Chris Wilsonf7470262017-01-24 15:20:21 +00001494
Chris Wilson51f6b0f2018-03-09 01:08:08 +00001495 if (iir & GT_CONTEXT_SWITCH_INTERRUPT) {
Chris Wilson1c645bf2018-03-22 07:35:32 +00001496 if (READ_ONCE(engine->execlists.active))
1497 tasklet = !test_and_set_bit(ENGINE_IRQ_EXECLIST,
1498 &engine->irq_posted);
Chris Wilsonf7470262017-01-24 15:20:21 +00001499 }
Chris Wilson31de7352017-03-16 12:56:18 +00001500
Chris Wilson51f6b0f2018-03-09 01:08:08 +00001501 if (iir & GT_RENDER_USER_INTERRUPT) {
Chris Wilson31de7352017-03-16 12:56:18 +00001502 notify_ring(engine);
Michal Wajdeczko93ffbe82017-12-06 13:53:12 +00001503 tasklet |= USES_GUC_SUBMISSION(engine->i915);
Chris Wilson31de7352017-03-16 12:56:18 +00001504 }
1505
1506 if (tasklet)
Sagar Arun Kamblec6dce8f2017-11-16 19:02:37 +05301507 tasklet_hi_schedule(&execlists->tasklet);
Nick Hoathfbcc1a02015-10-20 10:23:52 +01001508}
1509
Chris Wilson2e4a5b22018-02-19 10:09:26 +00001510static void gen8_gt_irq_ack(struct drm_i915_private *i915,
Chris Wilson55ef72f2018-02-02 15:34:48 +00001511 u32 master_ctl, u32 gt_iir[4])
Ben Widawskyabd58f02013-11-02 21:07:09 -07001512{
Chris Wilson2e4a5b22018-02-19 10:09:26 +00001513 void __iomem * const regs = i915->regs;
1514
Chris Wilsonf0fd96f2018-02-15 07:37:12 +00001515#define GEN8_GT_IRQS (GEN8_GT_RCS_IRQ | \
1516 GEN8_GT_BCS_IRQ | \
1517 GEN8_GT_VCS1_IRQ | \
1518 GEN8_GT_VCS2_IRQ | \
1519 GEN8_GT_VECS_IRQ | \
1520 GEN8_GT_PM_IRQ | \
1521 GEN8_GT_GUC_IRQ)
1522
Ben Widawskyabd58f02013-11-02 21:07:09 -07001523 if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) {
Chris Wilson2e4a5b22018-02-19 10:09:26 +00001524 gt_iir[0] = raw_reg_read(regs, GEN8_GT_IIR(0));
1525 if (likely(gt_iir[0]))
1526 raw_reg_write(regs, GEN8_GT_IIR(0), gt_iir[0]);
Ben Widawskyabd58f02013-11-02 21:07:09 -07001527 }
1528
Zhao Yakui85f9b5f2014-04-17 10:37:38 +08001529 if (master_ctl & (GEN8_GT_VCS1_IRQ | GEN8_GT_VCS2_IRQ)) {
Chris Wilson2e4a5b22018-02-19 10:09:26 +00001530 gt_iir[1] = raw_reg_read(regs, GEN8_GT_IIR(1));
1531 if (likely(gt_iir[1]))
1532 raw_reg_write(regs, GEN8_GT_IIR(1), gt_iir[1]);
Chris Wilson74cdb332015-04-07 16:21:05 +01001533 }
1534
Sagar Arun Kamble26705e22016-10-12 21:54:31 +05301535 if (master_ctl & (GEN8_GT_PM_IRQ | GEN8_GT_GUC_IRQ)) {
Chris Wilson2e4a5b22018-02-19 10:09:26 +00001536 gt_iir[2] = raw_reg_read(regs, GEN8_GT_IIR(2));
1537 if (likely(gt_iir[2] & (i915->pm_rps_events |
1538 i915->pm_guc_events)))
1539 raw_reg_write(regs, GEN8_GT_IIR(2),
1540 gt_iir[2] & (i915->pm_rps_events |
1541 i915->pm_guc_events));
1542 }
1543
1544 if (master_ctl & GEN8_GT_VECS_IRQ) {
1545 gt_iir[3] = raw_reg_read(regs, GEN8_GT_IIR(3));
1546 if (likely(gt_iir[3]))
1547 raw_reg_write(regs, GEN8_GT_IIR(3), gt_iir[3]);
Ben Widawsky09610212014-05-15 20:58:08 +03001548 }
Ben Widawskyabd58f02013-11-02 21:07:09 -07001549}
1550
Chris Wilson2e4a5b22018-02-19 10:09:26 +00001551static void gen8_gt_irq_handler(struct drm_i915_private *i915,
Chris Wilsonf0fd96f2018-02-15 07:37:12 +00001552 u32 master_ctl, u32 gt_iir[4])
Ville Syrjäläe30e2512016-04-13 21:19:58 +03001553{
Chris Wilsonf0fd96f2018-02-15 07:37:12 +00001554 if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) {
Chris Wilson2e4a5b22018-02-19 10:09:26 +00001555 gen8_cs_irq_handler(i915->engine[RCS],
Chris Wilson51f6b0f2018-03-09 01:08:08 +00001556 gt_iir[0] >> GEN8_RCS_IRQ_SHIFT);
Chris Wilson2e4a5b22018-02-19 10:09:26 +00001557 gen8_cs_irq_handler(i915->engine[BCS],
Chris Wilson51f6b0f2018-03-09 01:08:08 +00001558 gt_iir[0] >> GEN8_BCS_IRQ_SHIFT);
Ville Syrjäläe30e2512016-04-13 21:19:58 +03001559 }
1560
Chris Wilsonf0fd96f2018-02-15 07:37:12 +00001561 if (master_ctl & (GEN8_GT_VCS1_IRQ | GEN8_GT_VCS2_IRQ)) {
Chris Wilson2e4a5b22018-02-19 10:09:26 +00001562 gen8_cs_irq_handler(i915->engine[VCS],
Chris Wilson51f6b0f2018-03-09 01:08:08 +00001563 gt_iir[1] >> GEN8_VCS1_IRQ_SHIFT);
Chris Wilson2e4a5b22018-02-19 10:09:26 +00001564 gen8_cs_irq_handler(i915->engine[VCS2],
Chris Wilson51f6b0f2018-03-09 01:08:08 +00001565 gt_iir[1] >> GEN8_VCS2_IRQ_SHIFT);
Ville Syrjäläe30e2512016-04-13 21:19:58 +03001566 }
1567
Chris Wilsonf0fd96f2018-02-15 07:37:12 +00001568 if (master_ctl & GEN8_GT_VECS_IRQ) {
Chris Wilson2e4a5b22018-02-19 10:09:26 +00001569 gen8_cs_irq_handler(i915->engine[VECS],
Chris Wilson51f6b0f2018-03-09 01:08:08 +00001570 gt_iir[3] >> GEN8_VECS_IRQ_SHIFT);
Chris Wilsonf0fd96f2018-02-15 07:37:12 +00001571 }
Ville Syrjäläe30e2512016-04-13 21:19:58 +03001572
Chris Wilsonf0fd96f2018-02-15 07:37:12 +00001573 if (master_ctl & (GEN8_GT_PM_IRQ | GEN8_GT_GUC_IRQ)) {
Chris Wilson2e4a5b22018-02-19 10:09:26 +00001574 gen6_rps_irq_handler(i915, gt_iir[2]);
1575 gen9_guc_irq_handler(i915, gt_iir[2]);
Chris Wilsonf0fd96f2018-02-15 07:37:12 +00001576 }
Ville Syrjäläe30e2512016-04-13 21:19:58 +03001577}
1578
Dhinakaran Pandiyan121e7582018-06-15 17:05:29 -07001579static bool gen11_port_hotplug_long_detect(enum port port, u32 val)
1580{
1581 switch (port) {
1582 case PORT_C:
1583 return val & GEN11_HOTPLUG_CTL_LONG_DETECT(PORT_TC1);
1584 case PORT_D:
1585 return val & GEN11_HOTPLUG_CTL_LONG_DETECT(PORT_TC2);
1586 case PORT_E:
1587 return val & GEN11_HOTPLUG_CTL_LONG_DETECT(PORT_TC3);
1588 case PORT_F:
1589 return val & GEN11_HOTPLUG_CTL_LONG_DETECT(PORT_TC4);
1590 default:
1591 return false;
1592 }
1593}
1594
Imre Deak63c88d22015-07-20 14:43:39 -07001595static bool bxt_port_hotplug_long_detect(enum port port, u32 val)
1596{
1597 switch (port) {
1598 case PORT_A:
Ville Syrjälä195baa02015-08-27 23:56:00 +03001599 return val & PORTA_HOTPLUG_LONG_DETECT;
Imre Deak63c88d22015-07-20 14:43:39 -07001600 case PORT_B:
1601 return val & PORTB_HOTPLUG_LONG_DETECT;
1602 case PORT_C:
1603 return val & PORTC_HOTPLUG_LONG_DETECT;
Imre Deak63c88d22015-07-20 14:43:39 -07001604 default:
1605 return false;
1606 }
1607}
1608
Anusha Srivatsa31604222018-06-26 13:52:23 -07001609static bool icp_ddi_port_hotplug_long_detect(enum port port, u32 val)
1610{
1611 switch (port) {
1612 case PORT_A:
1613 return val & ICP_DDIA_HPD_LONG_DETECT;
1614 case PORT_B:
1615 return val & ICP_DDIB_HPD_LONG_DETECT;
1616 default:
1617 return false;
1618 }
1619}
1620
1621static bool icp_tc_port_hotplug_long_detect(enum port port, u32 val)
1622{
1623 switch (port) {
1624 case PORT_C:
1625 return val & ICP_TC_HPD_LONG_DETECT(PORT_TC1);
1626 case PORT_D:
1627 return val & ICP_TC_HPD_LONG_DETECT(PORT_TC2);
1628 case PORT_E:
1629 return val & ICP_TC_HPD_LONG_DETECT(PORT_TC3);
1630 case PORT_F:
1631 return val & ICP_TC_HPD_LONG_DETECT(PORT_TC4);
1632 default:
1633 return false;
1634 }
1635}
1636
Ville Syrjälä6dbf30c2015-08-27 23:56:02 +03001637static bool spt_port_hotplug2_long_detect(enum port port, u32 val)
1638{
1639 switch (port) {
1640 case PORT_E:
1641 return val & PORTE_HOTPLUG_LONG_DETECT;
1642 default:
1643 return false;
1644 }
1645}
1646
Ville Syrjälä74c0b392015-08-27 23:56:07 +03001647static bool spt_port_hotplug_long_detect(enum port port, u32 val)
1648{
1649 switch (port) {
1650 case PORT_A:
1651 return val & PORTA_HOTPLUG_LONG_DETECT;
1652 case PORT_B:
1653 return val & PORTB_HOTPLUG_LONG_DETECT;
1654 case PORT_C:
1655 return val & PORTC_HOTPLUG_LONG_DETECT;
1656 case PORT_D:
1657 return val & PORTD_HOTPLUG_LONG_DETECT;
1658 default:
1659 return false;
1660 }
1661}
1662
Ville Syrjäläe4ce95a2015-08-27 23:56:03 +03001663static bool ilk_port_hotplug_long_detect(enum port port, u32 val)
1664{
1665 switch (port) {
1666 case PORT_A:
1667 return val & DIGITAL_PORTA_HOTPLUG_LONG_DETECT;
1668 default:
1669 return false;
1670 }
1671}
1672
Jani Nikula676574d2015-05-28 15:43:53 +03001673static bool pch_port_hotplug_long_detect(enum port port, u32 val)
Dave Airlie13cf5502014-06-18 11:29:35 +10001674{
1675 switch (port) {
Dave Airlie13cf5502014-06-18 11:29:35 +10001676 case PORT_B:
Jani Nikula676574d2015-05-28 15:43:53 +03001677 return val & PORTB_HOTPLUG_LONG_DETECT;
Dave Airlie13cf5502014-06-18 11:29:35 +10001678 case PORT_C:
Jani Nikula676574d2015-05-28 15:43:53 +03001679 return val & PORTC_HOTPLUG_LONG_DETECT;
Dave Airlie13cf5502014-06-18 11:29:35 +10001680 case PORT_D:
Jani Nikula676574d2015-05-28 15:43:53 +03001681 return val & PORTD_HOTPLUG_LONG_DETECT;
1682 default:
1683 return false;
Dave Airlie13cf5502014-06-18 11:29:35 +10001684 }
1685}
1686
Jani Nikula676574d2015-05-28 15:43:53 +03001687static bool i9xx_port_hotplug_long_detect(enum port port, u32 val)
Dave Airlie13cf5502014-06-18 11:29:35 +10001688{
1689 switch (port) {
Dave Airlie13cf5502014-06-18 11:29:35 +10001690 case PORT_B:
Jani Nikula676574d2015-05-28 15:43:53 +03001691 return val & PORTB_HOTPLUG_INT_LONG_PULSE;
Dave Airlie13cf5502014-06-18 11:29:35 +10001692 case PORT_C:
Jani Nikula676574d2015-05-28 15:43:53 +03001693 return val & PORTC_HOTPLUG_INT_LONG_PULSE;
Dave Airlie13cf5502014-06-18 11:29:35 +10001694 case PORT_D:
Jani Nikula676574d2015-05-28 15:43:53 +03001695 return val & PORTD_HOTPLUG_INT_LONG_PULSE;
1696 default:
1697 return false;
Dave Airlie13cf5502014-06-18 11:29:35 +10001698 }
1699}
1700
Ville Syrjälä42db67d2015-08-28 21:26:27 +03001701/*
1702 * Get a bit mask of pins that have triggered, and which ones may be long.
1703 * This can be called multiple times with the same masks to accumulate
1704 * hotplug detection results from several registers.
1705 *
1706 * Note that the caller is expected to zero out the masks initially.
1707 */
Rodrigo Vivicf539022018-01-29 15:22:21 -08001708static void intel_get_hpd_pins(struct drm_i915_private *dev_priv,
1709 u32 *pin_mask, u32 *long_mask,
1710 u32 hotplug_trigger, u32 dig_hotplug_reg,
1711 const u32 hpd[HPD_NUM_PINS],
1712 bool long_pulse_detect(enum port port, u32 val))
Jani Nikula676574d2015-05-28 15:43:53 +03001713{
Jani Nikula8c841e52015-06-18 13:06:17 +03001714 enum port port;
Jani Nikula676574d2015-05-28 15:43:53 +03001715 int i;
1716
Jani Nikula676574d2015-05-28 15:43:53 +03001717 for_each_hpd_pin(i) {
Jani Nikula8c841e52015-06-18 13:06:17 +03001718 if ((hpd[i] & hotplug_trigger) == 0)
1719 continue;
Jani Nikula676574d2015-05-28 15:43:53 +03001720
Jani Nikula8c841e52015-06-18 13:06:17 +03001721 *pin_mask |= BIT(i);
1722
Rodrigo Vivicf539022018-01-29 15:22:21 -08001723 port = intel_hpd_pin_to_port(dev_priv, i);
Rodrigo Vivi256cfdde2017-08-11 11:26:49 -07001724 if (port == PORT_NONE)
Imre Deakcc24fcd2015-07-21 15:32:45 -07001725 continue;
1726
Imre Deakfd63e2a2015-07-21 15:32:44 -07001727 if (long_pulse_detect(port, dig_hotplug_reg))
Jani Nikula8c841e52015-06-18 13:06:17 +03001728 *long_mask |= BIT(i);
Jani Nikula676574d2015-05-28 15:43:53 +03001729 }
1730
1731 DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x, dig 0x%08x, pins 0x%08x\n",
1732 hotplug_trigger, dig_hotplug_reg, *pin_mask);
1733
1734}
1735
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01001736static void gmbus_irq_handler(struct drm_i915_private *dev_priv)
Daniel Vetter515ac2b2012-12-01 13:53:44 +01001737{
Daniel Vetter28c70f12012-12-01 13:53:45 +01001738 wake_up_all(&dev_priv->gmbus_wait_queue);
Daniel Vetter515ac2b2012-12-01 13:53:44 +01001739}
1740
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01001741static void dp_aux_irq_handler(struct drm_i915_private *dev_priv)
Daniel Vetterce99c252012-12-01 13:53:47 +01001742{
Daniel Vetter9ee32fea2012-12-01 13:53:48 +01001743 wake_up_all(&dev_priv->gmbus_wait_queue);
Daniel Vetterce99c252012-12-01 13:53:47 +01001744}
1745
Shuang He8bf1e9f2013-10-15 18:55:27 +01001746#if defined(CONFIG_DEBUG_FS)
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01001747static void display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
1748 enum pipe pipe,
Daniel Vetter277de952013-10-18 16:37:07 +02001749 uint32_t crc0, uint32_t crc1,
1750 uint32_t crc2, uint32_t crc3,
1751 uint32_t crc4)
Shuang He8bf1e9f2013-10-15 18:55:27 +01001752{
Shuang He8bf1e9f2013-10-15 18:55:27 +01001753 struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe];
1754 struct intel_pipe_crc_entry *entry;
Tomeu Vizoso8c6b7092017-01-10 14:43:04 +01001755 struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
1756 struct drm_driver *driver = dev_priv->drm.driver;
1757 uint32_t crcs[5];
Damien Lespiauac2300d2013-10-15 18:55:30 +01001758 int head, tail;
Damien Lespiaub2c88f52013-10-15 18:55:29 +01001759
Damien Lespiaud538bbd2013-10-21 14:29:30 +01001760 spin_lock(&pipe_crc->lock);
Maarten Lankhorst033b7a22018-03-08 13:02:02 +01001761 if (pipe_crc->source && !crtc->base.crc.opened) {
Tomeu Vizoso8c6b7092017-01-10 14:43:04 +01001762 if (!pipe_crc->entries) {
1763 spin_unlock(&pipe_crc->lock);
1764 DRM_DEBUG_KMS("spurious interrupt\n");
1765 return;
1766 }
Damien Lespiaud538bbd2013-10-21 14:29:30 +01001767
Tomeu Vizoso8c6b7092017-01-10 14:43:04 +01001768 head = pipe_crc->head;
1769 tail = pipe_crc->tail;
1770
1771 if (CIRC_SPACE(head, tail, INTEL_PIPE_CRC_ENTRIES_NR) < 1) {
1772 spin_unlock(&pipe_crc->lock);
1773 DRM_ERROR("CRC buffer overflowing\n");
1774 return;
1775 }
1776
1777 entry = &pipe_crc->entries[head];
1778
1779 entry->frame = driver->get_vblank_counter(&dev_priv->drm, pipe);
1780 entry->crc[0] = crc0;
1781 entry->crc[1] = crc1;
1782 entry->crc[2] = crc2;
1783 entry->crc[3] = crc3;
1784 entry->crc[4] = crc4;
1785
1786 head = (head + 1) & (INTEL_PIPE_CRC_ENTRIES_NR - 1);
1787 pipe_crc->head = head;
1788
Damien Lespiaud538bbd2013-10-21 14:29:30 +01001789 spin_unlock(&pipe_crc->lock);
Damien Lespiau0c912c72013-10-15 18:55:37 +01001790
Tomeu Vizoso8c6b7092017-01-10 14:43:04 +01001791 wake_up_interruptible(&pipe_crc->wq);
1792 } else {
1793 /*
1794 * For some not yet identified reason, the first CRC is
1795 * bonkers. So let's just wait for the next vblank and read
1796 * out the buggy result.
1797 *
Rodrigo Vivi163e8ae2017-09-27 17:20:40 -07001798 * On GEN8+ sometimes the second CRC is bonkers as well, so
Tomeu Vizoso8c6b7092017-01-10 14:43:04 +01001799 * don't trust that one either.
1800 */
Maarten Lankhorst033b7a22018-03-08 13:02:02 +01001801 if (pipe_crc->skipped <= 0 ||
Rodrigo Vivi163e8ae2017-09-27 17:20:40 -07001802 (INTEL_GEN(dev_priv) >= 8 && pipe_crc->skipped == 1)) {
Tomeu Vizoso8c6b7092017-01-10 14:43:04 +01001803 pipe_crc->skipped++;
1804 spin_unlock(&pipe_crc->lock);
1805 return;
1806 }
Damien Lespiaud538bbd2013-10-21 14:29:30 +01001807 spin_unlock(&pipe_crc->lock);
Tomeu Vizoso8c6b7092017-01-10 14:43:04 +01001808 crcs[0] = crc0;
1809 crcs[1] = crc1;
1810 crcs[2] = crc2;
1811 crcs[3] = crc3;
1812 crcs[4] = crc4;
Tomeu Vizoso246ee522017-01-10 14:43:05 +01001813 drm_crtc_add_crc_entry(&crtc->base, true,
Daniel Vetterca814b22017-05-24 16:51:47 +02001814 drm_crtc_accurate_vblank_count(&crtc->base),
Tomeu Vizoso246ee522017-01-10 14:43:05 +01001815 crcs);
Damien Lespiaub2c88f52013-10-15 18:55:29 +01001816 }
Shuang He8bf1e9f2013-10-15 18:55:27 +01001817}
Daniel Vetter277de952013-10-18 16:37:07 +02001818#else
1819static inline void
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01001820display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
1821 enum pipe pipe,
Daniel Vetter277de952013-10-18 16:37:07 +02001822 uint32_t crc0, uint32_t crc1,
1823 uint32_t crc2, uint32_t crc3,
1824 uint32_t crc4) {}
1825#endif
Daniel Vettereba94eb2013-10-16 22:55:46 +02001826
Daniel Vetter277de952013-10-18 16:37:07 +02001827
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01001828static void hsw_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
1829 enum pipe pipe)
Daniel Vetter5a69b892013-10-16 22:55:52 +02001830{
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01001831 display_pipe_crc_irq_handler(dev_priv, pipe,
Daniel Vetter277de952013-10-18 16:37:07 +02001832 I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
1833 0, 0, 0, 0);
Daniel Vetter5a69b892013-10-16 22:55:52 +02001834}
1835
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01001836static void ivb_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
1837 enum pipe pipe)
Daniel Vettereba94eb2013-10-16 22:55:46 +02001838{
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01001839 display_pipe_crc_irq_handler(dev_priv, pipe,
Daniel Vetter277de952013-10-18 16:37:07 +02001840 I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
1841 I915_READ(PIPE_CRC_RES_2_IVB(pipe)),
1842 I915_READ(PIPE_CRC_RES_3_IVB(pipe)),
1843 I915_READ(PIPE_CRC_RES_4_IVB(pipe)),
1844 I915_READ(PIPE_CRC_RES_5_IVB(pipe)));
Daniel Vettereba94eb2013-10-16 22:55:46 +02001845}
Daniel Vetter5b3a8562013-10-16 22:55:48 +02001846
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01001847static void i9xx_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
1848 enum pipe pipe)
Daniel Vetter5b3a8562013-10-16 22:55:48 +02001849{
Daniel Vetter0b5c5ed2013-10-16 22:55:53 +02001850 uint32_t res1, res2;
1851
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01001852 if (INTEL_GEN(dev_priv) >= 3)
Daniel Vetter0b5c5ed2013-10-16 22:55:53 +02001853 res1 = I915_READ(PIPE_CRC_RES_RES1_I915(pipe));
1854 else
1855 res1 = 0;
1856
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01001857 if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
Daniel Vetter0b5c5ed2013-10-16 22:55:53 +02001858 res2 = I915_READ(PIPE_CRC_RES_RES2_G4X(pipe));
1859 else
1860 res2 = 0;
Daniel Vetter5b3a8562013-10-16 22:55:48 +02001861
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01001862 display_pipe_crc_irq_handler(dev_priv, pipe,
Daniel Vetter277de952013-10-18 16:37:07 +02001863 I915_READ(PIPE_CRC_RES_RED(pipe)),
1864 I915_READ(PIPE_CRC_RES_GREEN(pipe)),
1865 I915_READ(PIPE_CRC_RES_BLUE(pipe)),
1866 res1, res2);
Daniel Vetter5b3a8562013-10-16 22:55:48 +02001867}
Shuang He8bf1e9f2013-10-15 18:55:27 +01001868
Paulo Zanoni1403c0d2013-08-15 11:51:32 -03001869/* The RPS events need forcewake, so we add them to a work queue and mask their
1870 * IMR bits until the work is done. Other interrupts can be processed without
1871 * the work queue. */
1872static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
Ben Widawskybaf02a12013-05-28 19:22:24 -07001873{
Sagar Arun Kamble562d9ba2017-10-10 22:30:06 +01001874 struct intel_rps *rps = &dev_priv->gt_pm.rps;
1875
Deepak Sa6706b42014-03-15 20:23:22 +05301876 if (pm_iir & dev_priv->pm_rps_events) {
Daniel Vetter59cdb632013-07-04 23:35:28 +02001877 spin_lock(&dev_priv->irq_lock);
Akash Goelf4e9af42016-10-12 21:54:30 +05301878 gen6_mask_pm_irq(dev_priv, pm_iir & dev_priv->pm_rps_events);
Sagar Arun Kamble562d9ba2017-10-10 22:30:06 +01001879 if (rps->interrupts_enabled) {
1880 rps->pm_iir |= pm_iir & dev_priv->pm_rps_events;
1881 schedule_work(&rps->work);
Imre Deakd4d70aa2014-11-19 15:30:04 +02001882 }
Daniel Vetter59cdb632013-07-04 23:35:28 +02001883 spin_unlock(&dev_priv->irq_lock);
Ben Widawskybaf02a12013-05-28 19:22:24 -07001884 }
Ben Widawskybaf02a12013-05-28 19:22:24 -07001885
Pandiyan, Dhinakaranbca2bf22017-07-18 11:28:00 -07001886 if (INTEL_GEN(dev_priv) >= 8)
Imre Deakc9a9a262014-11-05 20:48:37 +02001887 return;
1888
Joonas Lahtinen2d1fe072016-04-07 11:08:05 +03001889 if (HAS_VEBOX(dev_priv)) {
Paulo Zanoni1403c0d2013-08-15 11:51:32 -03001890 if (pm_iir & PM_VEBOX_USER_INTERRUPT)
Akash Goel3b3f1652016-10-13 22:44:48 +05301891 notify_ring(dev_priv->engine[VECS]);
Ben Widawsky12638c52013-05-28 19:22:31 -07001892
Daniel Vetteraaecdf62014-11-04 15:52:22 +01001893 if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
1894 DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir);
Ben Widawsky12638c52013-05-28 19:22:31 -07001895 }
Ben Widawskybaf02a12013-05-28 19:22:24 -07001896}
1897
Sagar Arun Kamble26705e22016-10-12 21:54:31 +05301898static void gen9_guc_irq_handler(struct drm_i915_private *dev_priv, u32 gt_iir)
1899{
Michal Wajdeczko93bf8092018-03-08 16:46:55 +01001900 if (gt_iir & GEN9_GUC_TO_HOST_INT_EVENT)
1901 intel_guc_to_host_event_handler(&dev_priv->guc);
Sagar Arun Kamble26705e22016-10-12 21:54:31 +05301902}
1903
Ville Syrjälä44d92412017-08-18 21:36:51 +03001904static void i9xx_pipestat_irq_reset(struct drm_i915_private *dev_priv)
1905{
1906 enum pipe pipe;
1907
1908 for_each_pipe(dev_priv, pipe) {
1909 I915_WRITE(PIPESTAT(pipe),
1910 PIPESTAT_INT_STATUS_MASK |
1911 PIPE_FIFO_UNDERRUN_STATUS);
1912
1913 dev_priv->pipestat_irq_mask[pipe] = 0;
1914 }
1915}
1916
Ville Syrjäläeb643432017-08-18 21:36:59 +03001917static void i9xx_pipestat_irq_ack(struct drm_i915_private *dev_priv,
1918 u32 iir, u32 pipe_stats[I915_MAX_PIPES])
Imre Deakc1874ed2014-02-04 21:35:46 +02001919{
Imre Deakc1874ed2014-02-04 21:35:46 +02001920 int pipe;
1921
Imre Deak58ead0d2014-02-04 21:35:47 +02001922 spin_lock(&dev_priv->irq_lock);
Ville Syrjälä1ca993d2016-02-18 21:54:26 +02001923
1924 if (!dev_priv->display_irqs_enabled) {
1925 spin_unlock(&dev_priv->irq_lock);
1926 return;
1927 }
1928
Damien Lespiau055e3932014-08-18 13:49:10 +01001929 for_each_pipe(dev_priv, pipe) {
Ville Syrjäläf0f59a02015-11-18 15:33:26 +02001930 i915_reg_t reg;
Ville Syrjälä6b12ca52017-09-14 18:17:31 +03001931 u32 status_mask, enable_mask, iir_bit = 0;
Imre Deak91d181d2014-02-10 18:42:49 +02001932
Daniel Vetterbbb5eeb2014-02-12 17:55:36 +01001933 /*
1934 * PIPESTAT bits get signalled even when the interrupt is
1935 * disabled with the mask bits, and some of the status bits do
1936 * not generate interrupts at all (like the underrun bit). Hence
1937 * we need to be careful that we only handle what we want to
1938 * handle.
1939 */
Daniel Vetter0f239f42014-09-30 10:56:49 +02001940
1941 /* fifo underruns are filterered in the underrun handler. */
Ville Syrjälä6b12ca52017-09-14 18:17:31 +03001942 status_mask = PIPE_FIFO_UNDERRUN_STATUS;
Daniel Vetterbbb5eeb2014-02-12 17:55:36 +01001943
1944 switch (pipe) {
1945 case PIPE_A:
1946 iir_bit = I915_DISPLAY_PIPE_A_EVENT_INTERRUPT;
1947 break;
1948 case PIPE_B:
1949 iir_bit = I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
1950 break;
Ville Syrjälä3278f672014-04-09 13:28:49 +03001951 case PIPE_C:
1952 iir_bit = I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
1953 break;
Daniel Vetterbbb5eeb2014-02-12 17:55:36 +01001954 }
1955 if (iir & iir_bit)
Ville Syrjälä6b12ca52017-09-14 18:17:31 +03001956 status_mask |= dev_priv->pipestat_irq_mask[pipe];
Daniel Vetterbbb5eeb2014-02-12 17:55:36 +01001957
Ville Syrjälä6b12ca52017-09-14 18:17:31 +03001958 if (!status_mask)
Imre Deak91d181d2014-02-10 18:42:49 +02001959 continue;
1960
1961 reg = PIPESTAT(pipe);
Ville Syrjälä6b12ca52017-09-14 18:17:31 +03001962 pipe_stats[pipe] = I915_READ(reg) & status_mask;
1963 enable_mask = i915_pipestat_enable_mask(dev_priv, pipe);
Imre Deakc1874ed2014-02-04 21:35:46 +02001964
1965 /*
1966 * Clear the PIPE*STAT regs before the IIR
Ville Syrjälä132c27c2018-06-11 23:02:55 +03001967 *
1968 * Toggle the enable bits to make sure we get an
1969 * edge in the ISR pipe event bit if we don't clear
1970 * all the enabled status bits. Otherwise the edge
1971 * triggered IIR on i965/g4x wouldn't notice that
1972 * an interrupt is still pending.
Imre Deakc1874ed2014-02-04 21:35:46 +02001973 */
Ville Syrjälä132c27c2018-06-11 23:02:55 +03001974 if (pipe_stats[pipe]) {
1975 I915_WRITE(reg, pipe_stats[pipe]);
1976 I915_WRITE(reg, enable_mask);
1977 }
Imre Deakc1874ed2014-02-04 21:35:46 +02001978 }
Imre Deak58ead0d2014-02-04 21:35:47 +02001979 spin_unlock(&dev_priv->irq_lock);
Ville Syrjälä2ecb8ca2016-04-13 21:19:55 +03001980}
1981
Ville Syrjäläeb643432017-08-18 21:36:59 +03001982static void i8xx_pipestat_irq_handler(struct drm_i915_private *dev_priv,
1983 u16 iir, u32 pipe_stats[I915_MAX_PIPES])
1984{
1985 enum pipe pipe;
1986
1987 for_each_pipe(dev_priv, pipe) {
1988 if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS)
1989 drm_handle_vblank(&dev_priv->drm, pipe);
1990
1991 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
1992 i9xx_pipe_crc_irq_handler(dev_priv, pipe);
1993
1994 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
1995 intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1996 }
1997}
1998
1999static void i915_pipestat_irq_handler(struct drm_i915_private *dev_priv,
2000 u32 iir, u32 pipe_stats[I915_MAX_PIPES])
2001{
2002 bool blc_event = false;
2003 enum pipe pipe;
2004
2005 for_each_pipe(dev_priv, pipe) {
2006 if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS)
2007 drm_handle_vblank(&dev_priv->drm, pipe);
2008
2009 if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
2010 blc_event = true;
2011
2012 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
2013 i9xx_pipe_crc_irq_handler(dev_priv, pipe);
2014
2015 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
2016 intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
2017 }
2018
2019 if (blc_event || (iir & I915_ASLE_INTERRUPT))
2020 intel_opregion_asle_intr(dev_priv);
2021}
2022
2023static void i965_pipestat_irq_handler(struct drm_i915_private *dev_priv,
2024 u32 iir, u32 pipe_stats[I915_MAX_PIPES])
2025{
2026 bool blc_event = false;
2027 enum pipe pipe;
2028
2029 for_each_pipe(dev_priv, pipe) {
2030 if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS)
2031 drm_handle_vblank(&dev_priv->drm, pipe);
2032
2033 if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
2034 blc_event = true;
2035
2036 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
2037 i9xx_pipe_crc_irq_handler(dev_priv, pipe);
2038
2039 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
2040 intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
2041 }
2042
2043 if (blc_event || (iir & I915_ASLE_INTERRUPT))
2044 intel_opregion_asle_intr(dev_priv);
2045
2046 if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
2047 gmbus_irq_handler(dev_priv);
2048}
2049
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002050static void valleyview_pipestat_irq_handler(struct drm_i915_private *dev_priv,
Ville Syrjälä2ecb8ca2016-04-13 21:19:55 +03002051 u32 pipe_stats[I915_MAX_PIPES])
2052{
Ville Syrjälä2ecb8ca2016-04-13 21:19:55 +03002053 enum pipe pipe;
Imre Deakc1874ed2014-02-04 21:35:46 +02002054
Damien Lespiau055e3932014-08-18 13:49:10 +01002055 for_each_pipe(dev_priv, pipe) {
Daniel Vetterfd3a4022017-07-20 19:57:51 +02002056 if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS)
2057 drm_handle_vblank(&dev_priv->drm, pipe);
Imre Deakc1874ed2014-02-04 21:35:46 +02002058
2059 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002060 i9xx_pipe_crc_irq_handler(dev_priv, pipe);
Imre Deakc1874ed2014-02-04 21:35:46 +02002061
Daniel Vetter1f7247c2014-09-30 10:56:48 +02002062 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
2063 intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
Imre Deakc1874ed2014-02-04 21:35:46 +02002064 }
2065
2066 if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002067 gmbus_irq_handler(dev_priv);
Imre Deakc1874ed2014-02-04 21:35:46 +02002068}
2069
Ville Syrjälä1ae3c342016-04-13 21:19:54 +03002070static u32 i9xx_hpd_irq_ack(struct drm_i915_private *dev_priv)
Ville Syrjälä16c6c562014-04-01 10:54:36 +03002071{
Ville Syrjälä16c6c562014-04-01 10:54:36 +03002072 u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
Ville Syrjälä1ae3c342016-04-13 21:19:54 +03002073
2074 if (hotplug_status)
2075 I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
2076
2077 return hotplug_status;
2078}
2079
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002080static void i9xx_hpd_irq_handler(struct drm_i915_private *dev_priv,
Ville Syrjälä1ae3c342016-04-13 21:19:54 +03002081 u32 hotplug_status)
2082{
Ville Syrjälä42db67d2015-08-28 21:26:27 +03002083 u32 pin_mask = 0, long_mask = 0;
Ville Syrjälä16c6c562014-04-01 10:54:36 +03002084
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002085 if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
2086 IS_CHERRYVIEW(dev_priv)) {
Jani Nikula0d2e4292015-05-27 15:03:39 +03002087 u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_G4X;
Oscar Mateo3ff60f82014-06-16 16:10:58 +01002088
Ville Syrjälä58f2cf22015-08-28 22:59:08 +03002089 if (hotplug_trigger) {
Rodrigo Vivicf539022018-01-29 15:22:21 -08002090 intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
2091 hotplug_trigger, hotplug_trigger,
2092 hpd_status_g4x,
Ville Syrjälä58f2cf22015-08-28 22:59:08 +03002093 i9xx_port_hotplug_long_detect);
2094
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002095 intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
Ville Syrjälä58f2cf22015-08-28 22:59:08 +03002096 }
Jani Nikula369712e2015-05-27 15:03:40 +03002097
2098 if (hotplug_status & DP_AUX_CHANNEL_MASK_INT_STATUS_G4X)
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002099 dp_aux_irq_handler(dev_priv);
Jani Nikula0d2e4292015-05-27 15:03:39 +03002100 } else {
2101 u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
Oscar Mateo3ff60f82014-06-16 16:10:58 +01002102
Ville Syrjälä58f2cf22015-08-28 22:59:08 +03002103 if (hotplug_trigger) {
Rodrigo Vivicf539022018-01-29 15:22:21 -08002104 intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
2105 hotplug_trigger, hotplug_trigger,
2106 hpd_status_i915,
Ville Syrjälä58f2cf22015-08-28 22:59:08 +03002107 i9xx_port_hotplug_long_detect);
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002108 intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
Ville Syrjälä58f2cf22015-08-28 22:59:08 +03002109 }
Ville Syrjälä16c6c562014-04-01 10:54:36 +03002110 }
Ville Syrjälä16c6c562014-04-01 10:54:36 +03002111}
2112
Daniel Vetterff1f5252012-10-02 15:10:55 +02002113static irqreturn_t valleyview_irq_handler(int irq, void *arg)
Jesse Barnes7e231dbe2012-03-28 13:39:38 -07002114{
Daniel Vetter45a83f82014-05-12 19:17:55 +02002115 struct drm_device *dev = arg;
Chris Wilsonfac5e232016-07-04 11:34:36 +01002116 struct drm_i915_private *dev_priv = to_i915(dev);
Jesse Barnes7e231dbe2012-03-28 13:39:38 -07002117 irqreturn_t ret = IRQ_NONE;
Jesse Barnes7e231dbe2012-03-28 13:39:38 -07002118
Imre Deak2dd2a882015-02-24 11:14:30 +02002119 if (!intel_irqs_enabled(dev_priv))
2120 return IRQ_NONE;
2121
Imre Deak1f814da2015-12-16 02:52:19 +02002122 /* IRQs are synced during runtime_suspend, we don't require a wakeref */
2123 disable_rpm_wakeref_asserts(dev_priv);
2124
Ville Syrjälä1e1cace2016-04-13 21:19:52 +03002125 do {
Ville Syrjälä6e814802016-04-13 21:19:53 +03002126 u32 iir, gt_iir, pm_iir;
Ville Syrjälä2ecb8ca2016-04-13 21:19:55 +03002127 u32 pipe_stats[I915_MAX_PIPES] = {};
Ville Syrjälä1ae3c342016-04-13 21:19:54 +03002128 u32 hotplug_status = 0;
Ville Syrjäläa5e485a2016-04-13 21:19:51 +03002129 u32 ier = 0;
Oscar Mateo3ff60f82014-06-16 16:10:58 +01002130
Jesse Barnes7e231dbe2012-03-28 13:39:38 -07002131 gt_iir = I915_READ(GTIIR);
2132 pm_iir = I915_READ(GEN6_PMIIR);
Oscar Mateo3ff60f82014-06-16 16:10:58 +01002133 iir = I915_READ(VLV_IIR);
Jesse Barnes7e231dbe2012-03-28 13:39:38 -07002134
2135 if (gt_iir == 0 && pm_iir == 0 && iir == 0)
Ville Syrjälä1e1cace2016-04-13 21:19:52 +03002136 break;
Jesse Barnes7e231dbe2012-03-28 13:39:38 -07002137
2138 ret = IRQ_HANDLED;
2139
Ville Syrjäläa5e485a2016-04-13 21:19:51 +03002140 /*
2141 * Theory on interrupt generation, based on empirical evidence:
2142 *
2143 * x = ((VLV_IIR & VLV_IER) ||
2144 * (((GT_IIR & GT_IER) || (GEN6_PMIIR & GEN6_PMIER)) &&
2145 * (VLV_MASTER_IER & MASTER_INTERRUPT_ENABLE)));
2146 *
2147 * A CPU interrupt will only be raised when 'x' has a 0->1 edge.
2148 * Hence we clear MASTER_INTERRUPT_ENABLE and VLV_IER to
2149 * guarantee the CPU interrupt will be raised again even if we
2150 * don't end up clearing all the VLV_IIR, GT_IIR, GEN6_PMIIR
2151 * bits this time around.
2152 */
Ville Syrjälä4a0a0202016-04-13 21:19:50 +03002153 I915_WRITE(VLV_MASTER_IER, 0);
Ville Syrjäläa5e485a2016-04-13 21:19:51 +03002154 ier = I915_READ(VLV_IER);
2155 I915_WRITE(VLV_IER, 0);
Ville Syrjälä4a0a0202016-04-13 21:19:50 +03002156
2157 if (gt_iir)
2158 I915_WRITE(GTIIR, gt_iir);
2159 if (pm_iir)
2160 I915_WRITE(GEN6_PMIIR, pm_iir);
2161
Ville Syrjälä7ce4d1f2016-04-13 21:19:49 +03002162 if (iir & I915_DISPLAY_PORT_INTERRUPT)
Ville Syrjälä1ae3c342016-04-13 21:19:54 +03002163 hotplug_status = i9xx_hpd_irq_ack(dev_priv);
Ville Syrjälä7ce4d1f2016-04-13 21:19:49 +03002164
Oscar Mateo3ff60f82014-06-16 16:10:58 +01002165 /* Call regardless, as some status bits might not be
2166 * signalled in iir */
Ville Syrjäläeb643432017-08-18 21:36:59 +03002167 i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
Ville Syrjälä7ce4d1f2016-04-13 21:19:49 +03002168
Jerome Anandeef57322017-01-25 04:27:49 +05302169 if (iir & (I915_LPE_PIPE_A_INTERRUPT |
2170 I915_LPE_PIPE_B_INTERRUPT))
2171 intel_lpe_audio_irq_handler(dev_priv);
2172
Ville Syrjälä7ce4d1f2016-04-13 21:19:49 +03002173 /*
2174 * VLV_IIR is single buffered, and reflects the level
2175 * from PIPESTAT/PORT_HOTPLUG_STAT, hence clear it last.
2176 */
2177 if (iir)
2178 I915_WRITE(VLV_IIR, iir);
Ville Syrjälä4a0a0202016-04-13 21:19:50 +03002179
Ville Syrjäläa5e485a2016-04-13 21:19:51 +03002180 I915_WRITE(VLV_IER, ier);
Ville Syrjälä4a0a0202016-04-13 21:19:50 +03002181 I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
2182 POSTING_READ(VLV_MASTER_IER);
Ville Syrjälä1ae3c342016-04-13 21:19:54 +03002183
Ville Syrjälä52894872016-04-13 21:19:56 +03002184 if (gt_iir)
Ville Syrjälä261e40b2016-04-13 21:19:57 +03002185 snb_gt_irq_handler(dev_priv, gt_iir);
Ville Syrjälä52894872016-04-13 21:19:56 +03002186 if (pm_iir)
2187 gen6_rps_irq_handler(dev_priv, pm_iir);
2188
Ville Syrjälä1ae3c342016-04-13 21:19:54 +03002189 if (hotplug_status)
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002190 i9xx_hpd_irq_handler(dev_priv, hotplug_status);
Ville Syrjälä2ecb8ca2016-04-13 21:19:55 +03002191
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002192 valleyview_pipestat_irq_handler(dev_priv, pipe_stats);
Ville Syrjälä1e1cace2016-04-13 21:19:52 +03002193 } while (0);
Jesse Barnes7e231dbe2012-03-28 13:39:38 -07002194
Imre Deak1f814da2015-12-16 02:52:19 +02002195 enable_rpm_wakeref_asserts(dev_priv);
2196
Jesse Barnes7e231dbe2012-03-28 13:39:38 -07002197 return ret;
2198}
2199
Ville Syrjälä43f328d2014-04-09 20:40:52 +03002200static irqreturn_t cherryview_irq_handler(int irq, void *arg)
2201{
Daniel Vetter45a83f82014-05-12 19:17:55 +02002202 struct drm_device *dev = arg;
Chris Wilsonfac5e232016-07-04 11:34:36 +01002203 struct drm_i915_private *dev_priv = to_i915(dev);
Ville Syrjälä43f328d2014-04-09 20:40:52 +03002204 irqreturn_t ret = IRQ_NONE;
Ville Syrjälä43f328d2014-04-09 20:40:52 +03002205
Imre Deak2dd2a882015-02-24 11:14:30 +02002206 if (!intel_irqs_enabled(dev_priv))
2207 return IRQ_NONE;
2208
Imre Deak1f814da2015-12-16 02:52:19 +02002209 /* IRQs are synced during runtime_suspend, we don't require a wakeref */
2210 disable_rpm_wakeref_asserts(dev_priv);
2211
Chris Wilson579de732016-03-14 09:01:57 +00002212 do {
Ville Syrjälä6e814802016-04-13 21:19:53 +03002213 u32 master_ctl, iir;
Ville Syrjälä2ecb8ca2016-04-13 21:19:55 +03002214 u32 pipe_stats[I915_MAX_PIPES] = {};
Ville Syrjälä1ae3c342016-04-13 21:19:54 +03002215 u32 hotplug_status = 0;
Chris Wilsonf0fd96f2018-02-15 07:37:12 +00002216 u32 gt_iir[4];
Ville Syrjäläa5e485a2016-04-13 21:19:51 +03002217 u32 ier = 0;
2218
Ville Syrjälä8e5fd592014-04-09 13:28:50 +03002219 master_ctl = I915_READ(GEN8_MASTER_IRQ) & ~GEN8_MASTER_IRQ_CONTROL;
2220 iir = I915_READ(VLV_IIR);
Ville Syrjälä3278f672014-04-09 13:28:49 +03002221
Ville Syrjälä8e5fd592014-04-09 13:28:50 +03002222 if (master_ctl == 0 && iir == 0)
2223 break;
Ville Syrjälä43f328d2014-04-09 20:40:52 +03002224
Oscar Mateo27b6c122014-06-16 16:11:00 +01002225 ret = IRQ_HANDLED;
2226
Ville Syrjäläa5e485a2016-04-13 21:19:51 +03002227 /*
2228 * Theory on interrupt generation, based on empirical evidence:
2229 *
2230 * x = ((VLV_IIR & VLV_IER) ||
2231 * ((GEN8_MASTER_IRQ & ~GEN8_MASTER_IRQ_CONTROL) &&
2232 * (GEN8_MASTER_IRQ & GEN8_MASTER_IRQ_CONTROL)));
2233 *
2234 * A CPU interrupt will only be raised when 'x' has a 0->1 edge.
2235 * Hence we clear GEN8_MASTER_IRQ_CONTROL and VLV_IER to
2236 * guarantee the CPU interrupt will be raised again even if we
2237 * don't end up clearing all the VLV_IIR and GEN8_MASTER_IRQ_CONTROL
2238 * bits this time around.
2239 */
Ville Syrjälä8e5fd592014-04-09 13:28:50 +03002240 I915_WRITE(GEN8_MASTER_IRQ, 0);
Ville Syrjäläa5e485a2016-04-13 21:19:51 +03002241 ier = I915_READ(VLV_IER);
2242 I915_WRITE(VLV_IER, 0);
Ville Syrjälä43f328d2014-04-09 20:40:52 +03002243
Ville Syrjäläe30e2512016-04-13 21:19:58 +03002244 gen8_gt_irq_ack(dev_priv, master_ctl, gt_iir);
Ville Syrjälä43f328d2014-04-09 20:40:52 +03002245
Ville Syrjälä7ce4d1f2016-04-13 21:19:49 +03002246 if (iir & I915_DISPLAY_PORT_INTERRUPT)
Ville Syrjälä1ae3c342016-04-13 21:19:54 +03002247 hotplug_status = i9xx_hpd_irq_ack(dev_priv);
Ville Syrjälä7ce4d1f2016-04-13 21:19:49 +03002248
Oscar Mateo27b6c122014-06-16 16:11:00 +01002249 /* Call regardless, as some status bits might not be
2250 * signalled in iir */
Ville Syrjäläeb643432017-08-18 21:36:59 +03002251 i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
Ville Syrjälä43f328d2014-04-09 20:40:52 +03002252
Jerome Anandeef57322017-01-25 04:27:49 +05302253 if (iir & (I915_LPE_PIPE_A_INTERRUPT |
2254 I915_LPE_PIPE_B_INTERRUPT |
2255 I915_LPE_PIPE_C_INTERRUPT))
2256 intel_lpe_audio_irq_handler(dev_priv);
2257
Ville Syrjälä7ce4d1f2016-04-13 21:19:49 +03002258 /*
2259 * VLV_IIR is single buffered, and reflects the level
2260 * from PIPESTAT/PORT_HOTPLUG_STAT, hence clear it last.
2261 */
2262 if (iir)
2263 I915_WRITE(VLV_IIR, iir);
2264
Ville Syrjäläa5e485a2016-04-13 21:19:51 +03002265 I915_WRITE(VLV_IER, ier);
Ville Syrjäläe5328c42016-04-13 21:19:47 +03002266 I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
Ville Syrjälä8e5fd592014-04-09 13:28:50 +03002267 POSTING_READ(GEN8_MASTER_IRQ);
Ville Syrjälä1ae3c342016-04-13 21:19:54 +03002268
Chris Wilsonf0fd96f2018-02-15 07:37:12 +00002269 gen8_gt_irq_handler(dev_priv, master_ctl, gt_iir);
Ville Syrjäläe30e2512016-04-13 21:19:58 +03002270
Ville Syrjälä1ae3c342016-04-13 21:19:54 +03002271 if (hotplug_status)
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002272 i9xx_hpd_irq_handler(dev_priv, hotplug_status);
Ville Syrjälä2ecb8ca2016-04-13 21:19:55 +03002273
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002274 valleyview_pipestat_irq_handler(dev_priv, pipe_stats);
Chris Wilson579de732016-03-14 09:01:57 +00002275 } while (0);
Ville Syrjälä43f328d2014-04-09 20:40:52 +03002276
Imre Deak1f814da2015-12-16 02:52:19 +02002277 enable_rpm_wakeref_asserts(dev_priv);
2278
Ville Syrjälä43f328d2014-04-09 20:40:52 +03002279 return ret;
2280}
2281
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002282static void ibx_hpd_irq_handler(struct drm_i915_private *dev_priv,
2283 u32 hotplug_trigger,
Ville Syrjälä40e56412015-08-27 23:56:10 +03002284 const u32 hpd[HPD_NUM_PINS])
2285{
Ville Syrjälä40e56412015-08-27 23:56:10 +03002286 u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
2287
Jani Nikula6a39d7c2015-11-25 16:47:22 +02002288 /*
2289 * Somehow the PCH doesn't seem to really ack the interrupt to the CPU
2290 * unless we touch the hotplug register, even if hotplug_trigger is
2291 * zero. Not acking leads to "The master control interrupt lied (SDE)!"
2292 * errors.
2293 */
Ville Syrjälä40e56412015-08-27 23:56:10 +03002294 dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
Jani Nikula6a39d7c2015-11-25 16:47:22 +02002295 if (!hotplug_trigger) {
2296 u32 mask = PORTA_HOTPLUG_STATUS_MASK |
2297 PORTD_HOTPLUG_STATUS_MASK |
2298 PORTC_HOTPLUG_STATUS_MASK |
2299 PORTB_HOTPLUG_STATUS_MASK;
2300 dig_hotplug_reg &= ~mask;
2301 }
2302
Ville Syrjälä40e56412015-08-27 23:56:10 +03002303 I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
Jani Nikula6a39d7c2015-11-25 16:47:22 +02002304 if (!hotplug_trigger)
2305 return;
Ville Syrjälä40e56412015-08-27 23:56:10 +03002306
Rodrigo Vivicf539022018-01-29 15:22:21 -08002307 intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, hotplug_trigger,
Ville Syrjälä40e56412015-08-27 23:56:10 +03002308 dig_hotplug_reg, hpd,
2309 pch_port_hotplug_long_detect);
2310
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002311 intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
Ville Syrjälä40e56412015-08-27 23:56:10 +03002312}
2313
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002314static void ibx_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
Jesse Barnes776ad802011-01-04 15:09:39 -08002315{
Jesse Barnes9db4a9c2011-02-07 12:26:52 -08002316 int pipe;
Egbert Eichb543fb02013-04-16 13:36:54 +02002317 u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK;
Jesse Barnes776ad802011-01-04 15:09:39 -08002318
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002319 ibx_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_ibx);
Daniel Vetter91d131d2013-06-27 17:52:14 +02002320
Ville Syrjäläcfc33bf2013-04-17 17:48:48 +03002321 if (pch_iir & SDE_AUDIO_POWER_MASK) {
2322 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >>
2323 SDE_AUDIO_POWER_SHIFT);
Jesse Barnes776ad802011-01-04 15:09:39 -08002324 DRM_DEBUG_DRIVER("PCH audio power change on port %d\n",
Ville Syrjäläcfc33bf2013-04-17 17:48:48 +03002325 port_name(port));
2326 }
Jesse Barnes776ad802011-01-04 15:09:39 -08002327
Daniel Vetterce99c252012-12-01 13:53:47 +01002328 if (pch_iir & SDE_AUX_MASK)
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002329 dp_aux_irq_handler(dev_priv);
Daniel Vetterce99c252012-12-01 13:53:47 +01002330
Jesse Barnes776ad802011-01-04 15:09:39 -08002331 if (pch_iir & SDE_GMBUS)
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002332 gmbus_irq_handler(dev_priv);
Jesse Barnes776ad802011-01-04 15:09:39 -08002333
2334 if (pch_iir & SDE_AUDIO_HDCP_MASK)
2335 DRM_DEBUG_DRIVER("PCH HDCP audio interrupt\n");
2336
2337 if (pch_iir & SDE_AUDIO_TRANS_MASK)
2338 DRM_DEBUG_DRIVER("PCH transcoder audio interrupt\n");
2339
2340 if (pch_iir & SDE_POISON)
2341 DRM_ERROR("PCH poison interrupt\n");
2342
Jesse Barnes9db4a9c2011-02-07 12:26:52 -08002343 if (pch_iir & SDE_FDI_MASK)
Damien Lespiau055e3932014-08-18 13:49:10 +01002344 for_each_pipe(dev_priv, pipe)
Jesse Barnes9db4a9c2011-02-07 12:26:52 -08002345 DRM_DEBUG_DRIVER(" pipe %c FDI IIR: 0x%08x\n",
2346 pipe_name(pipe),
2347 I915_READ(FDI_RX_IIR(pipe)));
Jesse Barnes776ad802011-01-04 15:09:39 -08002348
2349 if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE))
2350 DRM_DEBUG_DRIVER("PCH transcoder CRC done interrupt\n");
2351
2352 if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR))
2353 DRM_DEBUG_DRIVER("PCH transcoder CRC error interrupt\n");
2354
Jesse Barnes776ad802011-01-04 15:09:39 -08002355 if (pch_iir & SDE_TRANSA_FIFO_UNDER)
Matthias Kaehlckea2196032017-07-17 11:14:03 -07002356 intel_pch_fifo_underrun_irq_handler(dev_priv, PIPE_A);
Paulo Zanoni86642812013-04-12 17:57:57 -03002357
2358 if (pch_iir & SDE_TRANSB_FIFO_UNDER)
Matthias Kaehlckea2196032017-07-17 11:14:03 -07002359 intel_pch_fifo_underrun_irq_handler(dev_priv, PIPE_B);
Paulo Zanoni86642812013-04-12 17:57:57 -03002360}
2361
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002362static void ivb_err_int_handler(struct drm_i915_private *dev_priv)
Paulo Zanoni86642812013-04-12 17:57:57 -03002363{
Paulo Zanoni86642812013-04-12 17:57:57 -03002364 u32 err_int = I915_READ(GEN7_ERR_INT);
Daniel Vetter5a69b892013-10-16 22:55:52 +02002365 enum pipe pipe;
Paulo Zanoni86642812013-04-12 17:57:57 -03002366
Paulo Zanonide032bf2013-04-12 17:57:58 -03002367 if (err_int & ERR_INT_POISON)
2368 DRM_ERROR("Poison interrupt\n");
2369
Damien Lespiau055e3932014-08-18 13:49:10 +01002370 for_each_pipe(dev_priv, pipe) {
Daniel Vetter1f7247c2014-09-30 10:56:48 +02002371 if (err_int & ERR_INT_FIFO_UNDERRUN(pipe))
2372 intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
Paulo Zanoni86642812013-04-12 17:57:57 -03002373
Daniel Vetter5a69b892013-10-16 22:55:52 +02002374 if (err_int & ERR_INT_PIPE_CRC_DONE(pipe)) {
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002375 if (IS_IVYBRIDGE(dev_priv))
2376 ivb_pipe_crc_irq_handler(dev_priv, pipe);
Daniel Vetter5a69b892013-10-16 22:55:52 +02002377 else
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002378 hsw_pipe_crc_irq_handler(dev_priv, pipe);
Daniel Vetter5a69b892013-10-16 22:55:52 +02002379 }
2380 }
Shuang He8bf1e9f2013-10-15 18:55:27 +01002381
Paulo Zanoni86642812013-04-12 17:57:57 -03002382 I915_WRITE(GEN7_ERR_INT, err_int);
2383}
2384
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002385static void cpt_serr_int_handler(struct drm_i915_private *dev_priv)
Paulo Zanoni86642812013-04-12 17:57:57 -03002386{
Paulo Zanoni86642812013-04-12 17:57:57 -03002387 u32 serr_int = I915_READ(SERR_INT);
Mika Kahola45c1cd82017-10-10 13:17:06 +03002388 enum pipe pipe;
Paulo Zanoni86642812013-04-12 17:57:57 -03002389
Paulo Zanonide032bf2013-04-12 17:57:58 -03002390 if (serr_int & SERR_INT_POISON)
2391 DRM_ERROR("PCH poison interrupt\n");
2392
Mika Kahola45c1cd82017-10-10 13:17:06 +03002393 for_each_pipe(dev_priv, pipe)
2394 if (serr_int & SERR_INT_TRANS_FIFO_UNDERRUN(pipe))
2395 intel_pch_fifo_underrun_irq_handler(dev_priv, pipe);
Paulo Zanoni86642812013-04-12 17:57:57 -03002396
2397 I915_WRITE(SERR_INT, serr_int);
Jesse Barnes776ad802011-01-04 15:09:39 -08002398}
2399
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002400static void cpt_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
Adam Jackson23e81d62012-06-06 15:45:44 -04002401{
Adam Jackson23e81d62012-06-06 15:45:44 -04002402 int pipe;
Ville Syrjälä6dbf30c2015-08-27 23:56:02 +03002403 u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT;
Adam Jackson23e81d62012-06-06 15:45:44 -04002404
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002405 ibx_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_cpt);
Daniel Vetter91d131d2013-06-27 17:52:14 +02002406
Ville Syrjäläcfc33bf2013-04-17 17:48:48 +03002407 if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) {
2408 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >>
2409 SDE_AUDIO_POWER_SHIFT_CPT);
2410 DRM_DEBUG_DRIVER("PCH audio power change on port %c\n",
2411 port_name(port));
2412 }
Adam Jackson23e81d62012-06-06 15:45:44 -04002413
2414 if (pch_iir & SDE_AUX_MASK_CPT)
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002415 dp_aux_irq_handler(dev_priv);
Adam Jackson23e81d62012-06-06 15:45:44 -04002416
2417 if (pch_iir & SDE_GMBUS_CPT)
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002418 gmbus_irq_handler(dev_priv);
Adam Jackson23e81d62012-06-06 15:45:44 -04002419
2420 if (pch_iir & SDE_AUDIO_CP_REQ_CPT)
2421 DRM_DEBUG_DRIVER("Audio CP request interrupt\n");
2422
2423 if (pch_iir & SDE_AUDIO_CP_CHG_CPT)
2424 DRM_DEBUG_DRIVER("Audio CP change interrupt\n");
2425
2426 if (pch_iir & SDE_FDI_MASK_CPT)
Damien Lespiau055e3932014-08-18 13:49:10 +01002427 for_each_pipe(dev_priv, pipe)
Adam Jackson23e81d62012-06-06 15:45:44 -04002428 DRM_DEBUG_DRIVER(" pipe %c FDI IIR: 0x%08x\n",
2429 pipe_name(pipe),
2430 I915_READ(FDI_RX_IIR(pipe)));
Paulo Zanoni86642812013-04-12 17:57:57 -03002431
2432 if (pch_iir & SDE_ERROR_CPT)
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002433 cpt_serr_int_handler(dev_priv);
Adam Jackson23e81d62012-06-06 15:45:44 -04002434}
2435
Anusha Srivatsa31604222018-06-26 13:52:23 -07002436static void icp_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
2437{
2438 u32 ddi_hotplug_trigger = pch_iir & SDE_DDI_MASK_ICP;
2439 u32 tc_hotplug_trigger = pch_iir & SDE_TC_MASK_ICP;
2440 u32 pin_mask = 0, long_mask = 0;
2441
2442 if (ddi_hotplug_trigger) {
2443 u32 dig_hotplug_reg;
2444
2445 dig_hotplug_reg = I915_READ(SHOTPLUG_CTL_DDI);
2446 I915_WRITE(SHOTPLUG_CTL_DDI, dig_hotplug_reg);
2447
2448 intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
2449 ddi_hotplug_trigger,
2450 dig_hotplug_reg, hpd_icp,
2451 icp_ddi_port_hotplug_long_detect);
2452 }
2453
2454 if (tc_hotplug_trigger) {
2455 u32 dig_hotplug_reg;
2456
2457 dig_hotplug_reg = I915_READ(SHOTPLUG_CTL_TC);
2458 I915_WRITE(SHOTPLUG_CTL_TC, dig_hotplug_reg);
2459
2460 intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
2461 tc_hotplug_trigger,
2462 dig_hotplug_reg, hpd_icp,
2463 icp_tc_port_hotplug_long_detect);
2464 }
2465
2466 if (pin_mask)
2467 intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
2468
2469 if (pch_iir & SDE_GMBUS_ICP)
2470 gmbus_irq_handler(dev_priv);
2471}
2472
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002473static void spt_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
Ville Syrjälä6dbf30c2015-08-27 23:56:02 +03002474{
Ville Syrjälä6dbf30c2015-08-27 23:56:02 +03002475 u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_SPT &
2476 ~SDE_PORTE_HOTPLUG_SPT;
2477 u32 hotplug2_trigger = pch_iir & SDE_PORTE_HOTPLUG_SPT;
2478 u32 pin_mask = 0, long_mask = 0;
2479
2480 if (hotplug_trigger) {
2481 u32 dig_hotplug_reg;
2482
2483 dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
2484 I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
2485
Rodrigo Vivicf539022018-01-29 15:22:21 -08002486 intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
2487 hotplug_trigger, dig_hotplug_reg, hpd_spt,
Ville Syrjälä74c0b392015-08-27 23:56:07 +03002488 spt_port_hotplug_long_detect);
Ville Syrjälä6dbf30c2015-08-27 23:56:02 +03002489 }
2490
2491 if (hotplug2_trigger) {
2492 u32 dig_hotplug_reg;
2493
2494 dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG2);
2495 I915_WRITE(PCH_PORT_HOTPLUG2, dig_hotplug_reg);
2496
Rodrigo Vivicf539022018-01-29 15:22:21 -08002497 intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
2498 hotplug2_trigger, dig_hotplug_reg, hpd_spt,
Ville Syrjälä6dbf30c2015-08-27 23:56:02 +03002499 spt_port_hotplug2_long_detect);
2500 }
2501
2502 if (pin_mask)
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002503 intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
Ville Syrjälä6dbf30c2015-08-27 23:56:02 +03002504
2505 if (pch_iir & SDE_GMBUS_CPT)
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002506 gmbus_irq_handler(dev_priv);
Ville Syrjälä6dbf30c2015-08-27 23:56:02 +03002507}
2508
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002509static void ilk_hpd_irq_handler(struct drm_i915_private *dev_priv,
2510 u32 hotplug_trigger,
Ville Syrjälä40e56412015-08-27 23:56:10 +03002511 const u32 hpd[HPD_NUM_PINS])
2512{
Ville Syrjälä40e56412015-08-27 23:56:10 +03002513 u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
2514
2515 dig_hotplug_reg = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
2516 I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, dig_hotplug_reg);
2517
Rodrigo Vivicf539022018-01-29 15:22:21 -08002518 intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, hotplug_trigger,
Ville Syrjälä40e56412015-08-27 23:56:10 +03002519 dig_hotplug_reg, hpd,
2520 ilk_port_hotplug_long_detect);
2521
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002522 intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
Ville Syrjälä40e56412015-08-27 23:56:10 +03002523}
2524
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002525static void ilk_display_irq_handler(struct drm_i915_private *dev_priv,
2526 u32 de_iir)
Paulo Zanonic008bc62013-07-12 16:35:10 -03002527{
Daniel Vetter40da17c22013-10-21 18:04:36 +02002528 enum pipe pipe;
Ville Syrjäläe4ce95a2015-08-27 23:56:03 +03002529 u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG;
2530
Ville Syrjälä40e56412015-08-27 23:56:10 +03002531 if (hotplug_trigger)
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002532 ilk_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_ilk);
Paulo Zanonic008bc62013-07-12 16:35:10 -03002533
2534 if (de_iir & DE_AUX_CHANNEL_A)
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002535 dp_aux_irq_handler(dev_priv);
Paulo Zanonic008bc62013-07-12 16:35:10 -03002536
2537 if (de_iir & DE_GSE)
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002538 intel_opregion_asle_intr(dev_priv);
Paulo Zanonic008bc62013-07-12 16:35:10 -03002539
Paulo Zanonic008bc62013-07-12 16:35:10 -03002540 if (de_iir & DE_POISON)
2541 DRM_ERROR("Poison interrupt\n");
2542
Damien Lespiau055e3932014-08-18 13:49:10 +01002543 for_each_pipe(dev_priv, pipe) {
Daniel Vetterfd3a4022017-07-20 19:57:51 +02002544 if (de_iir & DE_PIPE_VBLANK(pipe))
2545 drm_handle_vblank(&dev_priv->drm, pipe);
Paulo Zanonic008bc62013-07-12 16:35:10 -03002546
Daniel Vetter40da17c22013-10-21 18:04:36 +02002547 if (de_iir & DE_PIPE_FIFO_UNDERRUN(pipe))
Daniel Vetter1f7247c2014-09-30 10:56:48 +02002548 intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
Paulo Zanonic008bc62013-07-12 16:35:10 -03002549
Daniel Vetter40da17c22013-10-21 18:04:36 +02002550 if (de_iir & DE_PIPE_CRC_DONE(pipe))
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002551 i9xx_pipe_crc_irq_handler(dev_priv, pipe);
Paulo Zanonic008bc62013-07-12 16:35:10 -03002552 }
2553
2554 /* check event from PCH */
2555 if (de_iir & DE_PCH_EVENT) {
2556 u32 pch_iir = I915_READ(SDEIIR);
2557
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002558 if (HAS_PCH_CPT(dev_priv))
2559 cpt_irq_handler(dev_priv, pch_iir);
Paulo Zanonic008bc62013-07-12 16:35:10 -03002560 else
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002561 ibx_irq_handler(dev_priv, pch_iir);
Paulo Zanonic008bc62013-07-12 16:35:10 -03002562
2563 /* should clear PCH hotplug event before clear CPU irq */
2564 I915_WRITE(SDEIIR, pch_iir);
2565 }
2566
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002567 if (IS_GEN5(dev_priv) && de_iir & DE_PCU_EVENT)
2568 ironlake_rps_change_irq_handler(dev_priv);
Paulo Zanonic008bc62013-07-12 16:35:10 -03002569}
2570
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002571static void ivb_display_irq_handler(struct drm_i915_private *dev_priv,
2572 u32 de_iir)
Paulo Zanoni9719fb92013-07-12 16:35:11 -03002573{
Damien Lespiau07d27e22014-03-03 17:31:46 +00002574 enum pipe pipe;
Ville Syrjälä23bb4cb2015-08-27 23:56:04 +03002575 u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG_IVB;
2576
Ville Syrjälä40e56412015-08-27 23:56:10 +03002577 if (hotplug_trigger)
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002578 ilk_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_ivb);
Paulo Zanoni9719fb92013-07-12 16:35:11 -03002579
2580 if (de_iir & DE_ERR_INT_IVB)
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002581 ivb_err_int_handler(dev_priv);
Paulo Zanoni9719fb92013-07-12 16:35:11 -03002582
Dhinakaran Pandiyan54fd3142018-04-04 18:37:17 -07002583 if (de_iir & DE_EDP_PSR_INT_HSW) {
2584 u32 psr_iir = I915_READ(EDP_PSR_IIR);
2585
2586 intel_psr_irq_handler(dev_priv, psr_iir);
2587 I915_WRITE(EDP_PSR_IIR, psr_iir);
2588 }
Daniel Vetterfc340442018-04-05 15:00:23 -07002589
Paulo Zanoni9719fb92013-07-12 16:35:11 -03002590 if (de_iir & DE_AUX_CHANNEL_A_IVB)
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002591 dp_aux_irq_handler(dev_priv);
Paulo Zanoni9719fb92013-07-12 16:35:11 -03002592
2593 if (de_iir & DE_GSE_IVB)
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002594 intel_opregion_asle_intr(dev_priv);
Paulo Zanoni9719fb92013-07-12 16:35:11 -03002595
Damien Lespiau055e3932014-08-18 13:49:10 +01002596 for_each_pipe(dev_priv, pipe) {
Daniel Vetterfd3a4022017-07-20 19:57:51 +02002597 if (de_iir & (DE_PIPE_VBLANK_IVB(pipe)))
2598 drm_handle_vblank(&dev_priv->drm, pipe);
Paulo Zanoni9719fb92013-07-12 16:35:11 -03002599 }
2600
2601 /* check event from PCH */
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002602 if (!HAS_PCH_NOP(dev_priv) && (de_iir & DE_PCH_EVENT_IVB)) {
Paulo Zanoni9719fb92013-07-12 16:35:11 -03002603 u32 pch_iir = I915_READ(SDEIIR);
2604
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002605 cpt_irq_handler(dev_priv, pch_iir);
Paulo Zanoni9719fb92013-07-12 16:35:11 -03002606
2607 /* clear PCH hotplug event before clear CPU irq */
2608 I915_WRITE(SDEIIR, pch_iir);
2609 }
2610}
2611
Oscar Mateo72c90f62014-06-16 16:10:57 +01002612/*
2613 * To handle irqs with the minimum potential races with fresh interrupts, we:
2614 * 1 - Disable Master Interrupt Control.
2615 * 2 - Find the source(s) of the interrupt.
2616 * 3 - Clear the Interrupt Identity bits (IIR).
2617 * 4 - Process the interrupt(s) that had bits set in the IIRs.
2618 * 5 - Re-enable Master Interrupt Control.
2619 */
Paulo Zanonif1af8fc2013-07-12 19:56:30 -03002620static irqreturn_t ironlake_irq_handler(int irq, void *arg)
Jesse Barnesb1f14ad2011-04-06 12:13:38 -07002621{
Daniel Vetter45a83f82014-05-12 19:17:55 +02002622 struct drm_device *dev = arg;
Chris Wilsonfac5e232016-07-04 11:34:36 +01002623 struct drm_i915_private *dev_priv = to_i915(dev);
Paulo Zanonif1af8fc2013-07-12 19:56:30 -03002624 u32 de_iir, gt_iir, de_ier, sde_ier = 0;
Chris Wilson0e434062012-05-09 21:45:44 +01002625 irqreturn_t ret = IRQ_NONE;
Jesse Barnesb1f14ad2011-04-06 12:13:38 -07002626
Imre Deak2dd2a882015-02-24 11:14:30 +02002627 if (!intel_irqs_enabled(dev_priv))
2628 return IRQ_NONE;
2629
Imre Deak1f814da2015-12-16 02:52:19 +02002630 /* IRQs are synced during runtime_suspend, we don't require a wakeref */
2631 disable_rpm_wakeref_asserts(dev_priv);
2632
Jesse Barnesb1f14ad2011-04-06 12:13:38 -07002633 /* disable master interrupt before clearing iir */
2634 de_ier = I915_READ(DEIER);
2635 I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
Paulo Zanoni23a78512013-07-12 16:35:14 -03002636 POSTING_READ(DEIER);
Chris Wilson0e434062012-05-09 21:45:44 +01002637
Paulo Zanoni44498ae2013-02-22 17:05:28 -03002638 /* Disable south interrupts. We'll only write to SDEIIR once, so further
2639 * interrupts will will be stored on its back queue, and then we'll be
2640 * able to process them after we restore SDEIER (as soon as we restore
2641 * it, we'll get an interrupt if SDEIIR still has something to process
2642 * due to its back queue). */
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002643 if (!HAS_PCH_NOP(dev_priv)) {
Ben Widawskyab5c6082013-04-05 13:12:41 -07002644 sde_ier = I915_READ(SDEIER);
2645 I915_WRITE(SDEIER, 0);
2646 POSTING_READ(SDEIER);
2647 }
Paulo Zanoni44498ae2013-02-22 17:05:28 -03002648
Oscar Mateo72c90f62014-06-16 16:10:57 +01002649 /* Find, clear, then process each source of interrupt */
2650
Chris Wilson0e434062012-05-09 21:45:44 +01002651 gt_iir = I915_READ(GTIIR);
2652 if (gt_iir) {
Oscar Mateo72c90f62014-06-16 16:10:57 +01002653 I915_WRITE(GTIIR, gt_iir);
2654 ret = IRQ_HANDLED;
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002655 if (INTEL_GEN(dev_priv) >= 6)
Ville Syrjälä261e40b2016-04-13 21:19:57 +03002656 snb_gt_irq_handler(dev_priv, gt_iir);
Paulo Zanonid8fc8a42013-07-19 18:57:55 -03002657 else
Ville Syrjälä261e40b2016-04-13 21:19:57 +03002658 ilk_gt_irq_handler(dev_priv, gt_iir);
Chris Wilson0e434062012-05-09 21:45:44 +01002659 }
Jesse Barnesb1f14ad2011-04-06 12:13:38 -07002660
2661 de_iir = I915_READ(DEIIR);
Chris Wilson0e434062012-05-09 21:45:44 +01002662 if (de_iir) {
Oscar Mateo72c90f62014-06-16 16:10:57 +01002663 I915_WRITE(DEIIR, de_iir);
2664 ret = IRQ_HANDLED;
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002665 if (INTEL_GEN(dev_priv) >= 7)
2666 ivb_display_irq_handler(dev_priv, de_iir);
Paulo Zanonif1af8fc2013-07-12 19:56:30 -03002667 else
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002668 ilk_display_irq_handler(dev_priv, de_iir);
Chris Wilson0e434062012-05-09 21:45:44 +01002669 }
2670
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002671 if (INTEL_GEN(dev_priv) >= 6) {
Paulo Zanonif1af8fc2013-07-12 19:56:30 -03002672 u32 pm_iir = I915_READ(GEN6_PMIIR);
2673 if (pm_iir) {
Paulo Zanonif1af8fc2013-07-12 19:56:30 -03002674 I915_WRITE(GEN6_PMIIR, pm_iir);
2675 ret = IRQ_HANDLED;
Oscar Mateo72c90f62014-06-16 16:10:57 +01002676 gen6_rps_irq_handler(dev_priv, pm_iir);
Paulo Zanonif1af8fc2013-07-12 19:56:30 -03002677 }
Jesse Barnesb1f14ad2011-04-06 12:13:38 -07002678 }
2679
Jesse Barnesb1f14ad2011-04-06 12:13:38 -07002680 I915_WRITE(DEIER, de_ier);
2681 POSTING_READ(DEIER);
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002682 if (!HAS_PCH_NOP(dev_priv)) {
Ben Widawskyab5c6082013-04-05 13:12:41 -07002683 I915_WRITE(SDEIER, sde_ier);
2684 POSTING_READ(SDEIER);
2685 }
Jesse Barnesb1f14ad2011-04-06 12:13:38 -07002686
Imre Deak1f814da2015-12-16 02:52:19 +02002687 /* IRQs are synced during runtime_suspend, we don't require a wakeref */
2688 enable_rpm_wakeref_asserts(dev_priv);
2689
Jesse Barnesb1f14ad2011-04-06 12:13:38 -07002690 return ret;
2691}
2692
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002693static void bxt_hpd_irq_handler(struct drm_i915_private *dev_priv,
2694 u32 hotplug_trigger,
Ville Syrjälä40e56412015-08-27 23:56:10 +03002695 const u32 hpd[HPD_NUM_PINS])
Shashank Sharmad04a4922014-08-22 17:40:41 +05302696{
Ville Syrjäläcebd87a2015-08-27 23:56:09 +03002697 u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
Shashank Sharmad04a4922014-08-22 17:40:41 +05302698
Ville Syrjäläa52bb152015-08-27 23:56:11 +03002699 dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
2700 I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
Shashank Sharmad04a4922014-08-22 17:40:41 +05302701
Rodrigo Vivicf539022018-01-29 15:22:21 -08002702 intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, hotplug_trigger,
Ville Syrjälä40e56412015-08-27 23:56:10 +03002703 dig_hotplug_reg, hpd,
Ville Syrjäläcebd87a2015-08-27 23:56:09 +03002704 bxt_port_hotplug_long_detect);
Ville Syrjälä40e56412015-08-27 23:56:10 +03002705
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002706 intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
Shashank Sharmad04a4922014-08-22 17:40:41 +05302707}
2708
Dhinakaran Pandiyan121e7582018-06-15 17:05:29 -07002709static void gen11_hpd_irq_handler(struct drm_i915_private *dev_priv, u32 iir)
2710{
2711 u32 pin_mask = 0, long_mask = 0;
Dhinakaran Pandiyanb796b972018-06-15 17:05:30 -07002712 u32 trigger_tc = iir & GEN11_DE_TC_HOTPLUG_MASK;
2713 u32 trigger_tbt = iir & GEN11_DE_TBT_HOTPLUG_MASK;
Dhinakaran Pandiyan121e7582018-06-15 17:05:29 -07002714
Dhinakaran Pandiyan121e7582018-06-15 17:05:29 -07002715 if (trigger_tc) {
Dhinakaran Pandiyanb796b972018-06-15 17:05:30 -07002716 u32 dig_hotplug_reg;
2717
Dhinakaran Pandiyan121e7582018-06-15 17:05:29 -07002718 dig_hotplug_reg = I915_READ(GEN11_TC_HOTPLUG_CTL);
2719 I915_WRITE(GEN11_TC_HOTPLUG_CTL, dig_hotplug_reg);
2720
2721 intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, trigger_tc,
Dhinakaran Pandiyanb796b972018-06-15 17:05:30 -07002722 dig_hotplug_reg, hpd_gen11,
Dhinakaran Pandiyan121e7582018-06-15 17:05:29 -07002723 gen11_port_hotplug_long_detect);
Dhinakaran Pandiyan121e7582018-06-15 17:05:29 -07002724 }
Dhinakaran Pandiyanb796b972018-06-15 17:05:30 -07002725
2726 if (trigger_tbt) {
2727 u32 dig_hotplug_reg;
2728
2729 dig_hotplug_reg = I915_READ(GEN11_TBT_HOTPLUG_CTL);
2730 I915_WRITE(GEN11_TBT_HOTPLUG_CTL, dig_hotplug_reg);
2731
2732 intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, trigger_tbt,
2733 dig_hotplug_reg, hpd_gen11,
2734 gen11_port_hotplug_long_detect);
2735 }
2736
2737 if (pin_mask)
2738 intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
2739 else
2740 DRM_ERROR("Unexpected DE HPD interrupt 0x%08x\n", iir);
Dhinakaran Pandiyan121e7582018-06-15 17:05:29 -07002741}
2742
Tvrtko Ursulinf11a0f42016-01-12 16:04:07 +00002743static irqreturn_t
2744gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl)
Ben Widawskyabd58f02013-11-02 21:07:09 -07002745{
Ben Widawskyabd58f02013-11-02 21:07:09 -07002746 irqreturn_t ret = IRQ_NONE;
Tvrtko Ursulinf11a0f42016-01-12 16:04:07 +00002747 u32 iir;
Daniel Vetterc42664c2013-11-07 11:05:40 +01002748 enum pipe pipe;
Jesse Barnes88e04702014-11-13 17:51:48 +00002749
Ben Widawskyabd58f02013-11-02 21:07:09 -07002750 if (master_ctl & GEN8_DE_MISC_IRQ) {
Tvrtko Ursuline32192e2016-01-12 16:04:06 +00002751 iir = I915_READ(GEN8_DE_MISC_IIR);
2752 if (iir) {
Ville Syrjäläe04f7ec2018-04-03 14:24:18 -07002753 bool found = false;
2754
Tvrtko Ursuline32192e2016-01-12 16:04:06 +00002755 I915_WRITE(GEN8_DE_MISC_IIR, iir);
Ben Widawskyabd58f02013-11-02 21:07:09 -07002756 ret = IRQ_HANDLED;
Ville Syrjäläe04f7ec2018-04-03 14:24:18 -07002757
2758 if (iir & GEN8_DE_MISC_GSE) {
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002759 intel_opregion_asle_intr(dev_priv);
Ville Syrjäläe04f7ec2018-04-03 14:24:18 -07002760 found = true;
2761 }
2762
2763 if (iir & GEN8_DE_EDP_PSR) {
Dhinakaran Pandiyan54fd3142018-04-04 18:37:17 -07002764 u32 psr_iir = I915_READ(EDP_PSR_IIR);
2765
2766 intel_psr_irq_handler(dev_priv, psr_iir);
2767 I915_WRITE(EDP_PSR_IIR, psr_iir);
Ville Syrjäläe04f7ec2018-04-03 14:24:18 -07002768 found = true;
2769 }
2770
2771 if (!found)
Oscar Mateo38cc46d2014-06-16 16:10:59 +01002772 DRM_ERROR("Unexpected DE Misc interrupt\n");
Ben Widawskyabd58f02013-11-02 21:07:09 -07002773 }
Oscar Mateo38cc46d2014-06-16 16:10:59 +01002774 else
2775 DRM_ERROR("The master control interrupt lied (DE MISC)!\n");
Ben Widawskyabd58f02013-11-02 21:07:09 -07002776 }
2777
Dhinakaran Pandiyan121e7582018-06-15 17:05:29 -07002778 if (INTEL_GEN(dev_priv) >= 11 && (master_ctl & GEN11_DE_HPD_IRQ)) {
2779 iir = I915_READ(GEN11_DE_HPD_IIR);
2780 if (iir) {
2781 I915_WRITE(GEN11_DE_HPD_IIR, iir);
2782 ret = IRQ_HANDLED;
2783 gen11_hpd_irq_handler(dev_priv, iir);
2784 } else {
2785 DRM_ERROR("The master control interrupt lied, (DE HPD)!\n");
2786 }
2787 }
2788
Daniel Vetter6d766f02013-11-07 14:49:55 +01002789 if (master_ctl & GEN8_DE_PORT_IRQ) {
Tvrtko Ursuline32192e2016-01-12 16:04:06 +00002790 iir = I915_READ(GEN8_DE_PORT_IIR);
2791 if (iir) {
2792 u32 tmp_mask;
Shashank Sharmad04a4922014-08-22 17:40:41 +05302793 bool found = false;
Ville Syrjäläcebd87a2015-08-27 23:56:09 +03002794
Tvrtko Ursuline32192e2016-01-12 16:04:06 +00002795 I915_WRITE(GEN8_DE_PORT_IIR, iir);
Daniel Vetter6d766f02013-11-07 14:49:55 +01002796 ret = IRQ_HANDLED;
Jesse Barnes88e04702014-11-13 17:51:48 +00002797
Tvrtko Ursuline32192e2016-01-12 16:04:06 +00002798 tmp_mask = GEN8_AUX_CHANNEL_A;
Pandiyan, Dhinakaranbca2bf22017-07-18 11:28:00 -07002799 if (INTEL_GEN(dev_priv) >= 9)
Tvrtko Ursuline32192e2016-01-12 16:04:06 +00002800 tmp_mask |= GEN9_AUX_CHANNEL_B |
2801 GEN9_AUX_CHANNEL_C |
2802 GEN9_AUX_CHANNEL_D;
2803
James Ausmusbb187e92018-06-11 17:25:12 -07002804 if (INTEL_GEN(dev_priv) >= 11)
2805 tmp_mask |= ICL_AUX_CHANNEL_E;
2806
Dhinakaran Pandiyan9bb635d2018-05-21 17:25:35 -07002807 if (IS_CNL_WITH_PORT_F(dev_priv) ||
2808 INTEL_GEN(dev_priv) >= 11)
Rodrigo Vivia324fca2018-01-29 15:22:15 -08002809 tmp_mask |= CNL_AUX_CHANNEL_F;
2810
Tvrtko Ursuline32192e2016-01-12 16:04:06 +00002811 if (iir & tmp_mask) {
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002812 dp_aux_irq_handler(dev_priv);
Shashank Sharmad04a4922014-08-22 17:40:41 +05302813 found = true;
2814 }
2815
Ander Conselvan de Oliveiracc3f90f2016-12-02 10:23:49 +02002816 if (IS_GEN9_LP(dev_priv)) {
Tvrtko Ursuline32192e2016-01-12 16:04:06 +00002817 tmp_mask = iir & BXT_DE_PORT_HOTPLUG_MASK;
2818 if (tmp_mask) {
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002819 bxt_hpd_irq_handler(dev_priv, tmp_mask,
2820 hpd_bxt);
Tvrtko Ursuline32192e2016-01-12 16:04:06 +00002821 found = true;
2822 }
2823 } else if (IS_BROADWELL(dev_priv)) {
2824 tmp_mask = iir & GEN8_PORT_DP_A_HOTPLUG;
2825 if (tmp_mask) {
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002826 ilk_hpd_irq_handler(dev_priv,
2827 tmp_mask, hpd_bdw);
Tvrtko Ursuline32192e2016-01-12 16:04:06 +00002828 found = true;
2829 }
Shashank Sharmad04a4922014-08-22 17:40:41 +05302830 }
2831
Ander Conselvan de Oliveiracc3f90f2016-12-02 10:23:49 +02002832 if (IS_GEN9_LP(dev_priv) && (iir & BXT_DE_PORT_GMBUS)) {
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002833 gmbus_irq_handler(dev_priv);
Shashank Sharma9e637432014-08-22 17:40:43 +05302834 found = true;
2835 }
2836
Shashank Sharmad04a4922014-08-22 17:40:41 +05302837 if (!found)
Oscar Mateo38cc46d2014-06-16 16:10:59 +01002838 DRM_ERROR("Unexpected DE Port interrupt\n");
Daniel Vetter6d766f02013-11-07 14:49:55 +01002839 }
Oscar Mateo38cc46d2014-06-16 16:10:59 +01002840 else
2841 DRM_ERROR("The master control interrupt lied (DE PORT)!\n");
Daniel Vetter6d766f02013-11-07 14:49:55 +01002842 }
2843
Damien Lespiau055e3932014-08-18 13:49:10 +01002844 for_each_pipe(dev_priv, pipe) {
Daniel Vetterfd3a4022017-07-20 19:57:51 +02002845 u32 fault_errors;
Ben Widawskyabd58f02013-11-02 21:07:09 -07002846
Daniel Vetterc42664c2013-11-07 11:05:40 +01002847 if (!(master_ctl & GEN8_DE_PIPE_IRQ(pipe)))
2848 continue;
Ben Widawskyabd58f02013-11-02 21:07:09 -07002849
Tvrtko Ursuline32192e2016-01-12 16:04:06 +00002850 iir = I915_READ(GEN8_DE_PIPE_IIR(pipe));
2851 if (!iir) {
Ben Widawskyabd58f02013-11-02 21:07:09 -07002852 DRM_ERROR("The master control interrupt lied (DE PIPE)!\n");
Tvrtko Ursuline32192e2016-01-12 16:04:06 +00002853 continue;
2854 }
2855
2856 ret = IRQ_HANDLED;
2857 I915_WRITE(GEN8_DE_PIPE_IIR(pipe), iir);
2858
Daniel Vetterfd3a4022017-07-20 19:57:51 +02002859 if (iir & GEN8_PIPE_VBLANK)
2860 drm_handle_vblank(&dev_priv->drm, pipe);
Tvrtko Ursuline32192e2016-01-12 16:04:06 +00002861
2862 if (iir & GEN8_PIPE_CDCLK_CRC_DONE)
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002863 hsw_pipe_crc_irq_handler(dev_priv, pipe);
Tvrtko Ursuline32192e2016-01-12 16:04:06 +00002864
2865 if (iir & GEN8_PIPE_FIFO_UNDERRUN)
2866 intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
2867
2868 fault_errors = iir;
Pandiyan, Dhinakaranbca2bf22017-07-18 11:28:00 -07002869 if (INTEL_GEN(dev_priv) >= 9)
Tvrtko Ursuline32192e2016-01-12 16:04:06 +00002870 fault_errors &= GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
2871 else
2872 fault_errors &= GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
2873
2874 if (fault_errors)
Tvrtko Ursulin1353ec32016-10-27 13:48:32 +01002875 DRM_ERROR("Fault errors on pipe %c: 0x%08x\n",
Tvrtko Ursuline32192e2016-01-12 16:04:06 +00002876 pipe_name(pipe),
2877 fault_errors);
Ben Widawskyabd58f02013-11-02 21:07:09 -07002878 }
2879
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002880 if (HAS_PCH_SPLIT(dev_priv) && !HAS_PCH_NOP(dev_priv) &&
Shashank Sharma266ea3d2014-08-22 17:40:42 +05302881 master_ctl & GEN8_DE_PCH_IRQ) {
Daniel Vetter92d03a82013-11-07 11:05:43 +01002882 /*
2883 * FIXME(BDW): Assume for now that the new interrupt handling
2884 * scheme also closed the SDE interrupt handling race we've seen
2885 * on older pch-split platforms. But this needs testing.
2886 */
Tvrtko Ursuline32192e2016-01-12 16:04:06 +00002887 iir = I915_READ(SDEIIR);
2888 if (iir) {
2889 I915_WRITE(SDEIIR, iir);
Daniel Vetter92d03a82013-11-07 11:05:43 +01002890 ret = IRQ_HANDLED;
Ville Syrjälä6dbf30c2015-08-27 23:56:02 +03002891
Anusha Srivatsa31604222018-06-26 13:52:23 -07002892 if (HAS_PCH_ICP(dev_priv))
2893 icp_irq_handler(dev_priv, iir);
2894 else if (HAS_PCH_SPT(dev_priv) ||
2895 HAS_PCH_KBP(dev_priv) ||
2896 HAS_PCH_CNP(dev_priv))
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002897 spt_irq_handler(dev_priv, iir);
Ville Syrjälä6dbf30c2015-08-27 23:56:02 +03002898 else
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01002899 cpt_irq_handler(dev_priv, iir);
Jani Nikula2dfb0b82016-01-07 10:29:10 +02002900 } else {
2901 /*
2902 * Like on previous PCH there seems to be something
2903 * fishy going on with forwarding PCH interrupts.
2904 */
2905 DRM_DEBUG_DRIVER("The master control interrupt lied (SDE)!\n");
2906 }
Daniel Vetter92d03a82013-11-07 11:05:43 +01002907 }
2908
Tvrtko Ursulinf11a0f42016-01-12 16:04:07 +00002909 return ret;
2910}
2911
2912static irqreturn_t gen8_irq_handler(int irq, void *arg)
2913{
Chris Wilsonf0fd96f2018-02-15 07:37:12 +00002914 struct drm_i915_private *dev_priv = to_i915(arg);
Tvrtko Ursulinf11a0f42016-01-12 16:04:07 +00002915 u32 master_ctl;
Chris Wilsonf0fd96f2018-02-15 07:37:12 +00002916 u32 gt_iir[4];
Tvrtko Ursulinf11a0f42016-01-12 16:04:07 +00002917
2918 if (!intel_irqs_enabled(dev_priv))
2919 return IRQ_NONE;
2920
2921 master_ctl = I915_READ_FW(GEN8_MASTER_IRQ);
2922 master_ctl &= ~GEN8_MASTER_IRQ_CONTROL;
2923 if (!master_ctl)
2924 return IRQ_NONE;
2925
2926 I915_WRITE_FW(GEN8_MASTER_IRQ, 0);
2927
Tvrtko Ursulinf11a0f42016-01-12 16:04:07 +00002928 /* Find, clear, then process each source of interrupt */
Chris Wilson55ef72f2018-02-02 15:34:48 +00002929 gen8_gt_irq_ack(dev_priv, master_ctl, gt_iir);
Chris Wilsonf0fd96f2018-02-15 07:37:12 +00002930
2931 /* IRQs are synced during runtime_suspend, we don't require a wakeref */
2932 if (master_ctl & ~GEN8_GT_IRQS) {
2933 disable_rpm_wakeref_asserts(dev_priv);
2934 gen8_de_irq_handler(dev_priv, master_ctl);
2935 enable_rpm_wakeref_asserts(dev_priv);
2936 }
Tvrtko Ursulinf11a0f42016-01-12 16:04:07 +00002937
Chris Wilsoncb0d2052015-04-07 16:21:04 +01002938 I915_WRITE_FW(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
Ben Widawskyabd58f02013-11-02 21:07:09 -07002939
Chris Wilsonf0fd96f2018-02-15 07:37:12 +00002940 gen8_gt_irq_handler(dev_priv, master_ctl, gt_iir);
Imre Deak1f814da2015-12-16 02:52:19 +02002941
Chris Wilson55ef72f2018-02-02 15:34:48 +00002942 return IRQ_HANDLED;
Ben Widawskyabd58f02013-11-02 21:07:09 -07002943}
2944
Chris Wilson36703e72017-06-22 11:56:25 +01002945struct wedge_me {
2946 struct delayed_work work;
2947 struct drm_i915_private *i915;
2948 const char *name;
2949};
2950
2951static void wedge_me(struct work_struct *work)
2952{
2953 struct wedge_me *w = container_of(work, typeof(*w), work.work);
2954
2955 dev_err(w->i915->drm.dev,
2956 "%s timed out, cancelling all in-flight rendering.\n",
2957 w->name);
2958 i915_gem_set_wedged(w->i915);
2959}
2960
2961static void __init_wedge(struct wedge_me *w,
2962 struct drm_i915_private *i915,
2963 long timeout,
2964 const char *name)
2965{
2966 w->i915 = i915;
2967 w->name = name;
2968
2969 INIT_DELAYED_WORK_ONSTACK(&w->work, wedge_me);
2970 schedule_delayed_work(&w->work, timeout);
2971}
2972
2973static void __fini_wedge(struct wedge_me *w)
2974{
2975 cancel_delayed_work_sync(&w->work);
2976 destroy_delayed_work_on_stack(&w->work);
2977 w->i915 = NULL;
2978}
2979
2980#define i915_wedge_on_timeout(W, DEV, TIMEOUT) \
2981 for (__init_wedge((W), (DEV), (TIMEOUT), __func__); \
2982 (W)->i915; \
2983 __fini_wedge((W)))
2984
Mika Kuoppala51951ae2018-02-28 12:11:53 +02002985static u32
Mika Kuoppalaf744dbc2018-04-06 12:31:45 +03002986gen11_gt_engine_identity(struct drm_i915_private * const i915,
2987 const unsigned int bank, const unsigned int bit)
Mika Kuoppala51951ae2018-02-28 12:11:53 +02002988{
2989 void __iomem * const regs = i915->regs;
2990 u32 timeout_ts;
2991 u32 ident;
2992
Oscar Mateo96606f32018-04-06 12:32:37 +03002993 lockdep_assert_held(&i915->irq_lock);
2994
Mika Kuoppala51951ae2018-02-28 12:11:53 +02002995 raw_reg_write(regs, GEN11_IIR_REG_SELECTOR(bank), BIT(bit));
2996
2997 /*
2998 * NB: Specs do not specify how long to spin wait,
2999 * so we do ~100us as an educated guess.
3000 */
3001 timeout_ts = (local_clock() >> 10) + 100;
3002 do {
3003 ident = raw_reg_read(regs, GEN11_INTR_IDENTITY_REG(bank));
3004 } while (!(ident & GEN11_INTR_DATA_VALID) &&
3005 !time_after32(local_clock() >> 10, timeout_ts));
3006
3007 if (unlikely(!(ident & GEN11_INTR_DATA_VALID))) {
3008 DRM_ERROR("INTR_IDENTITY_REG%u:%u 0x%08x not valid!\n",
3009 bank, bit, ident);
3010 return 0;
3011 }
3012
3013 raw_reg_write(regs, GEN11_INTR_IDENTITY_REG(bank),
3014 GEN11_INTR_DATA_VALID);
3015
Mika Kuoppalaf744dbc2018-04-06 12:31:45 +03003016 return ident;
3017}
3018
3019static void
3020gen11_other_irq_handler(struct drm_i915_private * const i915,
3021 const u8 instance, const u16 iir)
3022{
Oscar Mateod02b98b2018-04-05 17:00:50 +03003023 if (instance == OTHER_GTPM_INSTANCE)
3024 return gen6_rps_irq_handler(i915, iir);
3025
Mika Kuoppalaf744dbc2018-04-06 12:31:45 +03003026 WARN_ONCE(1, "unhandled other interrupt instance=0x%x, iir=0x%x\n",
3027 instance, iir);
3028}
3029
3030static void
3031gen11_engine_irq_handler(struct drm_i915_private * const i915,
3032 const u8 class, const u8 instance, const u16 iir)
3033{
3034 struct intel_engine_cs *engine;
3035
3036 if (instance <= MAX_ENGINE_INSTANCE)
3037 engine = i915->engine_class[class][instance];
3038 else
3039 engine = NULL;
3040
3041 if (likely(engine))
3042 return gen8_cs_irq_handler(engine, iir);
3043
3044 WARN_ONCE(1, "unhandled engine interrupt class=0x%x, instance=0x%x\n",
3045 class, instance);
3046}
3047
3048static void
3049gen11_gt_identity_handler(struct drm_i915_private * const i915,
3050 const u32 identity)
3051{
3052 const u8 class = GEN11_INTR_ENGINE_CLASS(identity);
3053 const u8 instance = GEN11_INTR_ENGINE_INSTANCE(identity);
3054 const u16 intr = GEN11_INTR_ENGINE_INTR(identity);
3055
3056 if (unlikely(!intr))
3057 return;
3058
3059 if (class <= COPY_ENGINE_CLASS)
3060 return gen11_engine_irq_handler(i915, class, instance, intr);
3061
3062 if (class == OTHER_CLASS)
3063 return gen11_other_irq_handler(i915, instance, intr);
3064
3065 WARN_ONCE(1, "unknown interrupt class=0x%x, instance=0x%x, intr=0x%x\n",
3066 class, instance, intr);
Mika Kuoppala51951ae2018-02-28 12:11:53 +02003067}
3068
3069static void
Oscar Mateo96606f32018-04-06 12:32:37 +03003070gen11_gt_bank_handler(struct drm_i915_private * const i915,
3071 const unsigned int bank)
3072{
3073 void __iomem * const regs = i915->regs;
3074 unsigned long intr_dw;
3075 unsigned int bit;
3076
3077 lockdep_assert_held(&i915->irq_lock);
3078
3079 intr_dw = raw_reg_read(regs, GEN11_GT_INTR_DW(bank));
3080
3081 if (unlikely(!intr_dw)) {
3082 DRM_ERROR("GT_INTR_DW%u blank!\n", bank);
3083 return;
3084 }
3085
3086 for_each_set_bit(bit, &intr_dw, 32) {
3087 const u32 ident = gen11_gt_engine_identity(i915,
3088 bank, bit);
3089
3090 gen11_gt_identity_handler(i915, ident);
3091 }
3092
3093 /* Clear must be after shared has been served for engine */
3094 raw_reg_write(regs, GEN11_GT_INTR_DW(bank), intr_dw);
3095}
3096
3097static void
Mika Kuoppala51951ae2018-02-28 12:11:53 +02003098gen11_gt_irq_handler(struct drm_i915_private * const i915,
3099 const u32 master_ctl)
3100{
Mika Kuoppala51951ae2018-02-28 12:11:53 +02003101 unsigned int bank;
3102
Oscar Mateo96606f32018-04-06 12:32:37 +03003103 spin_lock(&i915->irq_lock);
3104
Mika Kuoppala51951ae2018-02-28 12:11:53 +02003105 for (bank = 0; bank < 2; bank++) {
Oscar Mateo96606f32018-04-06 12:32:37 +03003106 if (master_ctl & GEN11_GT_DW_IRQ(bank))
3107 gen11_gt_bank_handler(i915, bank);
Mika Kuoppala51951ae2018-02-28 12:11:53 +02003108 }
Oscar Mateo96606f32018-04-06 12:32:37 +03003109
3110 spin_unlock(&i915->irq_lock);
Mika Kuoppala51951ae2018-02-28 12:11:53 +02003111}
3112
Dhinakaran Pandiyandf0d28c2018-06-15 17:05:28 -07003113static void
3114gen11_gu_misc_irq_ack(struct drm_i915_private *dev_priv, const u32 master_ctl,
3115 u32 *iir)
3116{
3117 void __iomem * const regs = dev_priv->regs;
3118
3119 if (!(master_ctl & GEN11_GU_MISC_IRQ))
3120 return;
3121
3122 *iir = raw_reg_read(regs, GEN11_GU_MISC_IIR);
3123 if (likely(*iir))
3124 raw_reg_write(regs, GEN11_GU_MISC_IIR, *iir);
3125}
3126
3127static void
3128gen11_gu_misc_irq_handler(struct drm_i915_private *dev_priv,
3129 const u32 master_ctl, const u32 iir)
3130{
3131 if (!(master_ctl & GEN11_GU_MISC_IRQ))
3132 return;
3133
3134 if (unlikely(!iir)) {
3135 DRM_ERROR("GU_MISC iir blank!\n");
3136 return;
3137 }
3138
3139 if (iir & GEN11_GU_MISC_GSE)
3140 intel_opregion_asle_intr(dev_priv);
3141 else
3142 DRM_ERROR("Unexpected GU_MISC interrupt 0x%x\n", iir);
3143}
3144
Mika Kuoppala51951ae2018-02-28 12:11:53 +02003145static irqreturn_t gen11_irq_handler(int irq, void *arg)
3146{
3147 struct drm_i915_private * const i915 = to_i915(arg);
3148 void __iomem * const regs = i915->regs;
3149 u32 master_ctl;
Dhinakaran Pandiyandf0d28c2018-06-15 17:05:28 -07003150 u32 gu_misc_iir;
Mika Kuoppala51951ae2018-02-28 12:11:53 +02003151
3152 if (!intel_irqs_enabled(i915))
3153 return IRQ_NONE;
3154
3155 master_ctl = raw_reg_read(regs, GEN11_GFX_MSTR_IRQ);
3156 master_ctl &= ~GEN11_MASTER_IRQ;
3157 if (!master_ctl)
3158 return IRQ_NONE;
3159
3160 /* Disable interrupts. */
3161 raw_reg_write(regs, GEN11_GFX_MSTR_IRQ, 0);
3162
3163 /* Find, clear, then process each source of interrupt. */
3164 gen11_gt_irq_handler(i915, master_ctl);
3165
3166 /* IRQs are synced during runtime_suspend, we don't require a wakeref */
3167 if (master_ctl & GEN11_DISPLAY_IRQ) {
3168 const u32 disp_ctl = raw_reg_read(regs, GEN11_DISPLAY_INT_CTL);
3169
3170 disable_rpm_wakeref_asserts(i915);
3171 /*
3172 * GEN11_DISPLAY_INT_CTL has same format as GEN8_MASTER_IRQ
3173 * for the display related bits.
3174 */
3175 gen8_de_irq_handler(i915, disp_ctl);
3176 enable_rpm_wakeref_asserts(i915);
3177 }
3178
Dhinakaran Pandiyandf0d28c2018-06-15 17:05:28 -07003179 gen11_gu_misc_irq_ack(i915, master_ctl, &gu_misc_iir);
3180
Mika Kuoppala51951ae2018-02-28 12:11:53 +02003181 /* Acknowledge and enable interrupts. */
3182 raw_reg_write(regs, GEN11_GFX_MSTR_IRQ, GEN11_MASTER_IRQ | master_ctl);
3183
Dhinakaran Pandiyandf0d28c2018-06-15 17:05:28 -07003184 gen11_gu_misc_irq_handler(i915, master_ctl, gu_misc_iir);
3185
Mika Kuoppala51951ae2018-02-28 12:11:53 +02003186 return IRQ_HANDLED;
3187}
3188
Chris Wilsonce800752018-03-20 10:04:49 +00003189static void i915_reset_device(struct drm_i915_private *dev_priv,
Chris Wilsond0667e92018-04-06 23:03:54 +01003190 u32 engine_mask,
3191 const char *reason)
Jesse Barnes8a905232009-07-11 16:48:03 -04003192{
Chris Wilsonce800752018-03-20 10:04:49 +00003193 struct i915_gpu_error *error = &dev_priv->gpu_error;
Chris Wilson91c8a322016-07-05 10:40:23 +01003194 struct kobject *kobj = &dev_priv->drm.primary->kdev->kobj;
Ben Widawskycce723e2013-07-19 09:16:42 -07003195 char *error_event[] = { I915_ERROR_UEVENT "=1", NULL };
3196 char *reset_event[] = { I915_RESET_UEVENT "=1", NULL };
3197 char *reset_done_event[] = { I915_ERROR_UEVENT "=0", NULL };
Chris Wilson36703e72017-06-22 11:56:25 +01003198 struct wedge_me w;
Jesse Barnes8a905232009-07-11 16:48:03 -04003199
Chris Wilsonc0336662016-05-06 15:40:21 +01003200 kobject_uevent_env(kobj, KOBJ_CHANGE, error_event);
Jesse Barnes8a905232009-07-11 16:48:03 -04003201
Chris Wilson8af29b02016-09-09 14:11:47 +01003202 DRM_DEBUG_DRIVER("resetting chip\n");
3203 kobject_uevent_env(kobj, KOBJ_CHANGE, reset_event);
3204
Chris Wilson36703e72017-06-22 11:56:25 +01003205 /* Use a watchdog to ensure that our reset completes */
3206 i915_wedge_on_timeout(&w, dev_priv, 5*HZ) {
3207 intel_prepare_reset(dev_priv);
Ville Syrjälä75147472014-11-24 18:28:11 +02003208
Chris Wilsond0667e92018-04-06 23:03:54 +01003209 error->reason = reason;
3210 error->stalled_mask = engine_mask;
Chris Wilsonce800752018-03-20 10:04:49 +00003211
Chris Wilson36703e72017-06-22 11:56:25 +01003212 /* Signal that locked waiters should reset the GPU */
Chris Wilsond0667e92018-04-06 23:03:54 +01003213 smp_mb__before_atomic();
Chris Wilsonce800752018-03-20 10:04:49 +00003214 set_bit(I915_RESET_HANDOFF, &error->flags);
3215 wake_up_all(&error->wait_queue);
Chris Wilson8c185ec2017-03-16 17:13:02 +00003216
Chris Wilson36703e72017-06-22 11:56:25 +01003217 /* Wait for anyone holding the lock to wakeup, without
3218 * blocking indefinitely on struct_mutex.
Chris Wilson780f2622016-09-09 14:11:52 +01003219 */
Chris Wilson36703e72017-06-22 11:56:25 +01003220 do {
3221 if (mutex_trylock(&dev_priv->drm.struct_mutex)) {
Chris Wilsond0667e92018-04-06 23:03:54 +01003222 i915_reset(dev_priv, engine_mask, reason);
Chris Wilson36703e72017-06-22 11:56:25 +01003223 mutex_unlock(&dev_priv->drm.struct_mutex);
3224 }
Chris Wilsonce800752018-03-20 10:04:49 +00003225 } while (wait_on_bit_timeout(&error->flags,
Chris Wilson36703e72017-06-22 11:56:25 +01003226 I915_RESET_HANDOFF,
3227 TASK_UNINTERRUPTIBLE,
3228 1));
Chris Wilson780f2622016-09-09 14:11:52 +01003229
Chris Wilsond0667e92018-04-06 23:03:54 +01003230 error->stalled_mask = 0;
Chris Wilsonce800752018-03-20 10:04:49 +00003231 error->reason = NULL;
3232
Chris Wilson36703e72017-06-22 11:56:25 +01003233 intel_finish_reset(dev_priv);
3234 }
Daniel Vetter17e1df02013-09-08 21:57:13 +02003235
Chris Wilsonce800752018-03-20 10:04:49 +00003236 if (!test_bit(I915_WEDGED, &error->flags))
3237 kobject_uevent_env(kobj, KOBJ_CHANGE, reset_done_event);
Jesse Barnes8a905232009-07-11 16:48:03 -04003238}
3239
Chris Wilsoneaa14c22016-10-19 13:52:03 +01003240static void i915_clear_error_registers(struct drm_i915_private *dev_priv)
Jesse Barnes8a905232009-07-11 16:48:03 -04003241{
Chris Wilsoneaa14c22016-10-19 13:52:03 +01003242 u32 eir;
Jesse Barnes8a905232009-07-11 16:48:03 -04003243
Chris Wilsoneaa14c22016-10-19 13:52:03 +01003244 if (!IS_GEN2(dev_priv))
3245 I915_WRITE(PGTBL_ER, I915_READ(PGTBL_ER));
Jesse Barnes8a905232009-07-11 16:48:03 -04003246
Chris Wilsoneaa14c22016-10-19 13:52:03 +01003247 if (INTEL_GEN(dev_priv) < 4)
3248 I915_WRITE(IPEIR, I915_READ(IPEIR));
3249 else
3250 I915_WRITE(IPEIR_I965, I915_READ(IPEIR_I965));
Jesse Barnes8a905232009-07-11 16:48:03 -04003251
Chris Wilsoneaa14c22016-10-19 13:52:03 +01003252 I915_WRITE(EIR, I915_READ(EIR));
Jesse Barnes8a905232009-07-11 16:48:03 -04003253 eir = I915_READ(EIR);
3254 if (eir) {
3255 /*
3256 * some errors might have become stuck,
3257 * mask them.
3258 */
Chris Wilsoneaa14c22016-10-19 13:52:03 +01003259 DRM_DEBUG_DRIVER("EIR stuck: 0x%08x, masking\n", eir);
Jesse Barnes8a905232009-07-11 16:48:03 -04003260 I915_WRITE(EMR, I915_READ(EMR) | eir);
3261 I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
3262 }
Chris Wilson35aed2e2010-05-27 13:18:12 +01003263}
3264
3265/**
Mika Kuoppalab8d24a02015-01-28 17:03:14 +02003266 * i915_handle_error - handle a gpu error
Tvrtko Ursulin14bb2c12016-06-03 14:02:17 +01003267 * @dev_priv: i915 device private
arun.siluvery@linux.intel.com14b730f2016-03-18 20:07:55 +00003268 * @engine_mask: mask representing engines that are hung
Chris Wilsonce800752018-03-20 10:04:49 +00003269 * @flags: control flags
Michel Thierry87c390b2017-01-11 20:18:08 -08003270 * @fmt: Error message format string
3271 *
Javier Martinez Canillasaafd8582015-10-08 09:57:49 +02003272 * Do some basic checking of register state at error time and
Chris Wilson35aed2e2010-05-27 13:18:12 +01003273 * dump it to the syslog. Also call i915_capture_error_state() to make
3274 * sure we get a record and make it available in debugfs. Fire a uevent
3275 * so userspace knows something bad happened (should trigger collection
3276 * of a ring dump etc.).
3277 */
Chris Wilsonc0336662016-05-06 15:40:21 +01003278void i915_handle_error(struct drm_i915_private *dev_priv,
3279 u32 engine_mask,
Chris Wilsonce800752018-03-20 10:04:49 +00003280 unsigned long flags,
Mika Kuoppala58174462014-02-25 17:11:26 +02003281 const char *fmt, ...)
Chris Wilson35aed2e2010-05-27 13:18:12 +01003282{
Michel Thierry142bc7d2017-06-20 10:57:46 +01003283 struct intel_engine_cs *engine;
3284 unsigned int tmp;
Mika Kuoppala58174462014-02-25 17:11:26 +02003285 char error_msg[80];
Chris Wilsonce800752018-03-20 10:04:49 +00003286 char *msg = NULL;
Chris Wilson35aed2e2010-05-27 13:18:12 +01003287
Chris Wilsonce800752018-03-20 10:04:49 +00003288 if (fmt) {
3289 va_list args;
3290
3291 va_start(args, fmt);
3292 vscnprintf(error_msg, sizeof(error_msg), fmt, args);
3293 va_end(args);
3294
3295 msg = error_msg;
3296 }
Mika Kuoppala58174462014-02-25 17:11:26 +02003297
Chris Wilson1604a862017-03-14 17:18:40 +00003298 /*
3299 * In most cases it's guaranteed that we get here with an RPM
3300 * reference held, for example because there is a pending GPU
3301 * request that won't finish until the reset is done. This
3302 * isn't the case at least when we get here by doing a
3303 * simulated reset via debugfs, so get an RPM reference.
3304 */
3305 intel_runtime_pm_get(dev_priv);
3306
Chris Wilson873d66f2018-03-16 21:49:59 +00003307 engine_mask &= INTEL_INFO(dev_priv)->ring_mask;
Chris Wilsonce800752018-03-20 10:04:49 +00003308
3309 if (flags & I915_ERROR_CAPTURE) {
3310 i915_capture_error_state(dev_priv, engine_mask, msg);
3311 i915_clear_error_registers(dev_priv);
3312 }
Jesse Barnes8a905232009-07-11 16:48:03 -04003313
Michel Thierry142bc7d2017-06-20 10:57:46 +01003314 /*
3315 * Try engine reset when available. We fall back to full reset if
3316 * single reset fails.
3317 */
3318 if (intel_has_reset_engine(dev_priv)) {
3319 for_each_engine_masked(engine, dev_priv, engine_mask, tmp) {
Daniel Vetter9db529a2017-08-08 10:08:28 +02003320 BUILD_BUG_ON(I915_RESET_MODESET >= I915_RESET_ENGINE);
Michel Thierry142bc7d2017-06-20 10:57:46 +01003321 if (test_and_set_bit(I915_RESET_ENGINE + engine->id,
3322 &dev_priv->gpu_error.flags))
3323 continue;
3324
Chris Wilsonce800752018-03-20 10:04:49 +00003325 if (i915_reset_engine(engine, msg) == 0)
Michel Thierry142bc7d2017-06-20 10:57:46 +01003326 engine_mask &= ~intel_engine_flag(engine);
3327
3328 clear_bit(I915_RESET_ENGINE + engine->id,
3329 &dev_priv->gpu_error.flags);
3330 wake_up_bit(&dev_priv->gpu_error.flags,
3331 I915_RESET_ENGINE + engine->id);
3332 }
3333 }
3334
Chris Wilson8af29b02016-09-09 14:11:47 +01003335 if (!engine_mask)
Chris Wilson1604a862017-03-14 17:18:40 +00003336 goto out;
Ben Gamariba1234d2009-09-14 17:48:47 -04003337
Michel Thierry142bc7d2017-06-20 10:57:46 +01003338 /* Full reset needs the mutex, stop any other user trying to do so. */
Chris Wilsond5367302017-06-20 10:57:43 +01003339 if (test_and_set_bit(I915_RESET_BACKOFF, &dev_priv->gpu_error.flags)) {
3340 wait_event(dev_priv->gpu_error.reset_queue,
3341 !test_bit(I915_RESET_BACKOFF,
3342 &dev_priv->gpu_error.flags));
Chris Wilson1604a862017-03-14 17:18:40 +00003343 goto out;
Chris Wilsond5367302017-06-20 10:57:43 +01003344 }
Chris Wilson8af29b02016-09-09 14:11:47 +01003345
Michel Thierry142bc7d2017-06-20 10:57:46 +01003346 /* Prevent any other reset-engine attempt. */
3347 for_each_engine(engine, dev_priv, tmp) {
3348 while (test_and_set_bit(I915_RESET_ENGINE + engine->id,
3349 &dev_priv->gpu_error.flags))
3350 wait_on_bit(&dev_priv->gpu_error.flags,
3351 I915_RESET_ENGINE + engine->id,
3352 TASK_UNINTERRUPTIBLE);
3353 }
3354
Chris Wilsond0667e92018-04-06 23:03:54 +01003355 i915_reset_device(dev_priv, engine_mask, msg);
Chris Wilsond5367302017-06-20 10:57:43 +01003356
Michel Thierry142bc7d2017-06-20 10:57:46 +01003357 for_each_engine(engine, dev_priv, tmp) {
3358 clear_bit(I915_RESET_ENGINE + engine->id,
3359 &dev_priv->gpu_error.flags);
3360 }
3361
Chris Wilsond5367302017-06-20 10:57:43 +01003362 clear_bit(I915_RESET_BACKOFF, &dev_priv->gpu_error.flags);
3363 wake_up_all(&dev_priv->gpu_error.reset_queue);
Chris Wilson1604a862017-03-14 17:18:40 +00003364
3365out:
3366 intel_runtime_pm_put(dev_priv);
Jesse Barnes8a905232009-07-11 16:48:03 -04003367}
3368
Keith Packard42f52ef2008-10-18 19:39:29 -07003369/* Called from drm generic code, passed 'crtc' which
3370 * we use as a pipe index
3371 */
Chris Wilson86e83e32016-10-07 20:49:52 +01003372static int i8xx_enable_vblank(struct drm_device *dev, unsigned int pipe)
Jesse Barnes0a3e67a2008-09-30 12:14:26 -07003373{
Chris Wilsonfac5e232016-07-04 11:34:36 +01003374 struct drm_i915_private *dev_priv = to_i915(dev);
Keith Packarde9d21d72008-10-16 11:31:38 -07003375 unsigned long irqflags;
Jesse Barnes71e0ffa2009-01-08 10:42:15 -08003376
Chris Wilson1ec14ad2010-12-04 11:30:53 +00003377 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
Chris Wilson86e83e32016-10-07 20:49:52 +01003378 i915_enable_pipestat(dev_priv, pipe, PIPE_VBLANK_INTERRUPT_STATUS);
3379 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
3380
3381 return 0;
3382}
3383
3384static int i965_enable_vblank(struct drm_device *dev, unsigned int pipe)
3385{
3386 struct drm_i915_private *dev_priv = to_i915(dev);
3387 unsigned long irqflags;
3388
3389 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
3390 i915_enable_pipestat(dev_priv, pipe,
3391 PIPE_START_VBLANK_INTERRUPT_STATUS);
Chris Wilson1ec14ad2010-12-04 11:30:53 +00003392 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
Chris Wilson8692d00e2011-02-05 10:08:21 +00003393
Jesse Barnes0a3e67a2008-09-30 12:14:26 -07003394 return 0;
3395}
3396
Thierry Reding88e72712015-09-24 18:35:31 +02003397static int ironlake_enable_vblank(struct drm_device *dev, unsigned int pipe)
Jesse Barnesf796cf82011-04-07 13:58:17 -07003398{
Chris Wilsonfac5e232016-07-04 11:34:36 +01003399 struct drm_i915_private *dev_priv = to_i915(dev);
Jesse Barnesf796cf82011-04-07 13:58:17 -07003400 unsigned long irqflags;
Tvrtko Ursulin55b8f2a2016-10-14 09:17:22 +01003401 uint32_t bit = INTEL_GEN(dev_priv) >= 7 ?
Chris Wilson86e83e32016-10-07 20:49:52 +01003402 DE_PIPE_VBLANK_IVB(pipe) : DE_PIPE_VBLANK(pipe);
Jesse Barnesf796cf82011-04-07 13:58:17 -07003403
Jesse Barnesf796cf82011-04-07 13:58:17 -07003404 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
Ville Syrjäläfbdedaea2015-11-23 18:06:16 +02003405 ilk_enable_display_irq(dev_priv, bit);
Jesse Barnesb1f14ad2011-04-06 12:13:38 -07003406 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
3407
Dhinakaran Pandiyan2e8bf222018-02-02 21:13:02 -08003408 /* Even though there is no DMC, frame counter can get stuck when
3409 * PSR is active as no frames are generated.
3410 */
3411 if (HAS_PSR(dev_priv))
3412 drm_vblank_restore(dev, pipe);
3413
Jesse Barnesb1f14ad2011-04-06 12:13:38 -07003414 return 0;
3415}
3416
Thierry Reding88e72712015-09-24 18:35:31 +02003417static int gen8_enable_vblank(struct drm_device *dev, unsigned int pipe)
Ben Widawskyabd58f02013-11-02 21:07:09 -07003418{
Chris Wilsonfac5e232016-07-04 11:34:36 +01003419 struct drm_i915_private *dev_priv = to_i915(dev);
Ben Widawskyabd58f02013-11-02 21:07:09 -07003420 unsigned long irqflags;
Ben Widawskyabd58f02013-11-02 21:07:09 -07003421
Ben Widawskyabd58f02013-11-02 21:07:09 -07003422 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
Ville Syrjälä013d3752015-11-23 18:06:17 +02003423 bdw_enable_pipe_irq(dev_priv, pipe, GEN8_PIPE_VBLANK);
Ben Widawskyabd58f02013-11-02 21:07:09 -07003424 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
Ville Syrjälä013d3752015-11-23 18:06:17 +02003425
Dhinakaran Pandiyan2e8bf222018-02-02 21:13:02 -08003426 /* Even if there is no DMC, frame counter can get stuck when
3427 * PSR is active as no frames are generated, so check only for PSR.
3428 */
3429 if (HAS_PSR(dev_priv))
3430 drm_vblank_restore(dev, pipe);
3431
Ben Widawskyabd58f02013-11-02 21:07:09 -07003432 return 0;
3433}
3434
Keith Packard42f52ef2008-10-18 19:39:29 -07003435/* Called from drm generic code, passed 'crtc' which
3436 * we use as a pipe index
3437 */
Chris Wilson86e83e32016-10-07 20:49:52 +01003438static void i8xx_disable_vblank(struct drm_device *dev, unsigned int pipe)
3439{
3440 struct drm_i915_private *dev_priv = to_i915(dev);
3441 unsigned long irqflags;
3442
3443 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
3444 i915_disable_pipestat(dev_priv, pipe, PIPE_VBLANK_INTERRUPT_STATUS);
3445 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
3446}
3447
3448static void i965_disable_vblank(struct drm_device *dev, unsigned int pipe)
Jesse Barnes0a3e67a2008-09-30 12:14:26 -07003449{
Chris Wilsonfac5e232016-07-04 11:34:36 +01003450 struct drm_i915_private *dev_priv = to_i915(dev);
Keith Packarde9d21d72008-10-16 11:31:38 -07003451 unsigned long irqflags;
Jesse Barnes0a3e67a2008-09-30 12:14:26 -07003452
Chris Wilson1ec14ad2010-12-04 11:30:53 +00003453 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
Jesse Barnesf796cf82011-04-07 13:58:17 -07003454 i915_disable_pipestat(dev_priv, pipe,
Imre Deak755e9012014-02-10 18:42:47 +02003455 PIPE_START_VBLANK_INTERRUPT_STATUS);
Jesse Barnesf796cf82011-04-07 13:58:17 -07003456 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
3457}
3458
Thierry Reding88e72712015-09-24 18:35:31 +02003459static void ironlake_disable_vblank(struct drm_device *dev, unsigned int pipe)
Jesse Barnesf796cf82011-04-07 13:58:17 -07003460{
Chris Wilsonfac5e232016-07-04 11:34:36 +01003461 struct drm_i915_private *dev_priv = to_i915(dev);
Jesse Barnesf796cf82011-04-07 13:58:17 -07003462 unsigned long irqflags;
Tvrtko Ursulin55b8f2a2016-10-14 09:17:22 +01003463 uint32_t bit = INTEL_GEN(dev_priv) >= 7 ?
Chris Wilson86e83e32016-10-07 20:49:52 +01003464 DE_PIPE_VBLANK_IVB(pipe) : DE_PIPE_VBLANK(pipe);
Jesse Barnesf796cf82011-04-07 13:58:17 -07003465
3466 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
Ville Syrjäläfbdedaea2015-11-23 18:06:16 +02003467 ilk_disable_display_irq(dev_priv, bit);
Jesse Barnesb1f14ad2011-04-06 12:13:38 -07003468 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
3469}
3470
Thierry Reding88e72712015-09-24 18:35:31 +02003471static void gen8_disable_vblank(struct drm_device *dev, unsigned int pipe)
Ben Widawskyabd58f02013-11-02 21:07:09 -07003472{
Chris Wilsonfac5e232016-07-04 11:34:36 +01003473 struct drm_i915_private *dev_priv = to_i915(dev);
Ben Widawskyabd58f02013-11-02 21:07:09 -07003474 unsigned long irqflags;
Ben Widawskyabd58f02013-11-02 21:07:09 -07003475
Ben Widawskyabd58f02013-11-02 21:07:09 -07003476 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
Ville Syrjälä013d3752015-11-23 18:06:17 +02003477 bdw_disable_pipe_irq(dev_priv, pipe, GEN8_PIPE_VBLANK);
Ben Widawskyabd58f02013-11-02 21:07:09 -07003478 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
3479}
3480
Tvrtko Ursulinb243f532016-11-16 08:55:38 +00003481static void ibx_irq_reset(struct drm_i915_private *dev_priv)
Paulo Zanoni91738a92013-06-05 14:21:51 -03003482{
Tvrtko Ursulin6e266952016-10-13 11:02:53 +01003483 if (HAS_PCH_NOP(dev_priv))
Paulo Zanoni91738a92013-06-05 14:21:51 -03003484 return;
3485
Ville Syrjälä3488d4e2017-08-18 21:36:52 +03003486 GEN3_IRQ_RESET(SDE);
Paulo Zanoni105b1222014-04-01 15:37:17 -03003487
Tvrtko Ursulin6e266952016-10-13 11:02:53 +01003488 if (HAS_PCH_CPT(dev_priv) || HAS_PCH_LPT(dev_priv))
Paulo Zanoni105b1222014-04-01 15:37:17 -03003489 I915_WRITE(SERR_INT, 0xffffffff);
Paulo Zanoni622364b2014-04-01 15:37:22 -03003490}
Paulo Zanoni105b1222014-04-01 15:37:17 -03003491
Paulo Zanoni622364b2014-04-01 15:37:22 -03003492/*
3493 * SDEIER is also touched by the interrupt handler to work around missed PCH
3494 * interrupts. Hence we can't update it after the interrupt handler is enabled -
3495 * instead we unconditionally enable all PCH interrupt sources here, but then
3496 * only unmask them as needed with SDEIMR.
3497 *
3498 * This function needs to be called before interrupts are enabled.
3499 */
3500static void ibx_irq_pre_postinstall(struct drm_device *dev)
3501{
Chris Wilsonfac5e232016-07-04 11:34:36 +01003502 struct drm_i915_private *dev_priv = to_i915(dev);
Paulo Zanoni622364b2014-04-01 15:37:22 -03003503
Tvrtko Ursulin6e266952016-10-13 11:02:53 +01003504 if (HAS_PCH_NOP(dev_priv))
Paulo Zanoni622364b2014-04-01 15:37:22 -03003505 return;
3506
3507 WARN_ON(I915_READ(SDEIER) != 0);
Paulo Zanoni91738a92013-06-05 14:21:51 -03003508 I915_WRITE(SDEIER, 0xffffffff);
3509 POSTING_READ(SDEIER);
3510}
3511
Tvrtko Ursulinb243f532016-11-16 08:55:38 +00003512static void gen5_gt_irq_reset(struct drm_i915_private *dev_priv)
Daniel Vetterd18ea1b2013-07-12 22:43:25 +02003513{
Ville Syrjälä3488d4e2017-08-18 21:36:52 +03003514 GEN3_IRQ_RESET(GT);
Tvrtko Ursulinb243f532016-11-16 08:55:38 +00003515 if (INTEL_GEN(dev_priv) >= 6)
Ville Syrjälä3488d4e2017-08-18 21:36:52 +03003516 GEN3_IRQ_RESET(GEN6_PM);
Daniel Vetterd18ea1b2013-07-12 22:43:25 +02003517}
3518
Ville Syrjälä70591a42014-10-30 19:42:58 +02003519static void vlv_display_irq_reset(struct drm_i915_private *dev_priv)
3520{
Ville Syrjälä71b8b412016-04-11 16:56:31 +03003521 if (IS_CHERRYVIEW(dev_priv))
3522 I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK_CHV);
3523 else
3524 I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);
3525
Ville Syrjäläad22d102016-04-12 18:56:14 +03003526 i915_hotplug_interrupt_update_locked(dev_priv, 0xffffffff, 0);
Ville Syrjälä70591a42014-10-30 19:42:58 +02003527 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
3528
Ville Syrjälä44d92412017-08-18 21:36:51 +03003529 i9xx_pipestat_irq_reset(dev_priv);
Ville Syrjälä70591a42014-10-30 19:42:58 +02003530
Ville Syrjälä3488d4e2017-08-18 21:36:52 +03003531 GEN3_IRQ_RESET(VLV_);
Chris Wilson8bd099a2017-11-30 12:52:53 +00003532 dev_priv->irq_mask = ~0u;
Ville Syrjälä70591a42014-10-30 19:42:58 +02003533}
3534
Ville Syrjälä8bb61302016-04-12 18:56:44 +03003535static void vlv_display_irq_postinstall(struct drm_i915_private *dev_priv)
3536{
3537 u32 pipestat_mask;
Ville Syrjälä9ab981f2016-04-11 16:56:28 +03003538 u32 enable_mask;
Ville Syrjälä8bb61302016-04-12 18:56:44 +03003539 enum pipe pipe;
3540
Ville Syrjälä842ebf72017-08-18 21:36:50 +03003541 pipestat_mask = PIPE_CRC_DONE_INTERRUPT_STATUS;
Ville Syrjälä8bb61302016-04-12 18:56:44 +03003542
3543 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
3544 for_each_pipe(dev_priv, pipe)
3545 i915_enable_pipestat(dev_priv, pipe, pipestat_mask);
3546
Ville Syrjälä9ab981f2016-04-11 16:56:28 +03003547 enable_mask = I915_DISPLAY_PORT_INTERRUPT |
3548 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
Ville Syrjäläebf5f922017-04-27 19:02:22 +03003549 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3550 I915_LPE_PIPE_A_INTERRUPT |
3551 I915_LPE_PIPE_B_INTERRUPT;
3552
Ville Syrjälä8bb61302016-04-12 18:56:44 +03003553 if (IS_CHERRYVIEW(dev_priv))
Ville Syrjäläebf5f922017-04-27 19:02:22 +03003554 enable_mask |= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT |
3555 I915_LPE_PIPE_C_INTERRUPT;
Ville Syrjälä6b7eafc2016-04-11 16:56:29 +03003556
Chris Wilson8bd099a2017-11-30 12:52:53 +00003557 WARN_ON(dev_priv->irq_mask != ~0u);
Ville Syrjälä6b7eafc2016-04-11 16:56:29 +03003558
Ville Syrjälä9ab981f2016-04-11 16:56:28 +03003559 dev_priv->irq_mask = ~enable_mask;
Ville Syrjälä8bb61302016-04-12 18:56:44 +03003560
Ville Syrjälä3488d4e2017-08-18 21:36:52 +03003561 GEN3_IRQ_INIT(VLV_, dev_priv->irq_mask, enable_mask);
Ville Syrjälä8bb61302016-04-12 18:56:44 +03003562}
3563
3564/* drm_dma.h hooks
3565*/
3566static void ironlake_irq_reset(struct drm_device *dev)
3567{
Chris Wilsonfac5e232016-07-04 11:34:36 +01003568 struct drm_i915_private *dev_priv = to_i915(dev);
Ville Syrjälä8bb61302016-04-12 18:56:44 +03003569
Ville Syrjäläd420a502017-08-18 21:37:03 +03003570 if (IS_GEN5(dev_priv))
3571 I915_WRITE(HWSTAM, 0xffffffff);
Ville Syrjälä8bb61302016-04-12 18:56:44 +03003572
Ville Syrjälä3488d4e2017-08-18 21:36:52 +03003573 GEN3_IRQ_RESET(DE);
Tvrtko Ursulin5db94012016-10-13 11:03:10 +01003574 if (IS_GEN7(dev_priv))
Ville Syrjälä8bb61302016-04-12 18:56:44 +03003575 I915_WRITE(GEN7_ERR_INT, 0xffffffff);
3576
Daniel Vetterfc340442018-04-05 15:00:23 -07003577 if (IS_HASWELL(dev_priv)) {
3578 I915_WRITE(EDP_PSR_IMR, 0xffffffff);
3579 I915_WRITE(EDP_PSR_IIR, 0xffffffff);
3580 }
3581
Tvrtko Ursulinb243f532016-11-16 08:55:38 +00003582 gen5_gt_irq_reset(dev_priv);
Ville Syrjälä8bb61302016-04-12 18:56:44 +03003583
Tvrtko Ursulinb243f532016-11-16 08:55:38 +00003584 ibx_irq_reset(dev_priv);
Ville Syrjälä8bb61302016-04-12 18:56:44 +03003585}
3586
Ville Syrjälä6bcdb1c2017-08-18 21:37:04 +03003587static void valleyview_irq_reset(struct drm_device *dev)
Jesse Barnes7e231dbe2012-03-28 13:39:38 -07003588{
Chris Wilsonfac5e232016-07-04 11:34:36 +01003589 struct drm_i915_private *dev_priv = to_i915(dev);
Jesse Barnes7e231dbe2012-03-28 13:39:38 -07003590
Ville Syrjälä34c7b8a2016-04-13 21:19:48 +03003591 I915_WRITE(VLV_MASTER_IER, 0);
3592 POSTING_READ(VLV_MASTER_IER);
3593
Tvrtko Ursulinb243f532016-11-16 08:55:38 +00003594 gen5_gt_irq_reset(dev_priv);
Jesse Barnes7e231dbe2012-03-28 13:39:38 -07003595
Ville Syrjäläad22d102016-04-12 18:56:14 +03003596 spin_lock_irq(&dev_priv->irq_lock);
Ville Syrjälä99182712016-04-11 16:56:25 +03003597 if (dev_priv->display_irqs_enabled)
3598 vlv_display_irq_reset(dev_priv);
Ville Syrjäläad22d102016-04-12 18:56:14 +03003599 spin_unlock_irq(&dev_priv->irq_lock);
Jesse Barnes7e231dbe2012-03-28 13:39:38 -07003600}
3601
Daniel Vetterd6e3cca2014-05-22 22:18:22 +02003602static void gen8_gt_irq_reset(struct drm_i915_private *dev_priv)
3603{
3604 GEN8_IRQ_RESET_NDX(GT, 0);
3605 GEN8_IRQ_RESET_NDX(GT, 1);
3606 GEN8_IRQ_RESET_NDX(GT, 2);
3607 GEN8_IRQ_RESET_NDX(GT, 3);
3608}
3609
Paulo Zanoni823f6b32014-04-01 15:37:26 -03003610static void gen8_irq_reset(struct drm_device *dev)
Ben Widawskyabd58f02013-11-02 21:07:09 -07003611{
Chris Wilsonfac5e232016-07-04 11:34:36 +01003612 struct drm_i915_private *dev_priv = to_i915(dev);
Ben Widawskyabd58f02013-11-02 21:07:09 -07003613 int pipe;
3614
Ben Widawskyabd58f02013-11-02 21:07:09 -07003615 I915_WRITE(GEN8_MASTER_IRQ, 0);
3616 POSTING_READ(GEN8_MASTER_IRQ);
3617
Daniel Vetterd6e3cca2014-05-22 22:18:22 +02003618 gen8_gt_irq_reset(dev_priv);
Ben Widawskyabd58f02013-11-02 21:07:09 -07003619
Ville Syrjäläe04f7ec2018-04-03 14:24:18 -07003620 I915_WRITE(EDP_PSR_IMR, 0xffffffff);
3621 I915_WRITE(EDP_PSR_IIR, 0xffffffff);
3622
Damien Lespiau055e3932014-08-18 13:49:10 +01003623 for_each_pipe(dev_priv, pipe)
Daniel Vetterf458ebb2014-09-30 10:56:39 +02003624 if (intel_display_power_is_enabled(dev_priv,
3625 POWER_DOMAIN_PIPE(pipe)))
Paulo Zanoni813bde42014-07-04 11:50:29 -03003626 GEN8_IRQ_RESET_NDX(DE_PIPE, pipe);
Ben Widawskyabd58f02013-11-02 21:07:09 -07003627
Ville Syrjälä3488d4e2017-08-18 21:36:52 +03003628 GEN3_IRQ_RESET(GEN8_DE_PORT_);
3629 GEN3_IRQ_RESET(GEN8_DE_MISC_);
3630 GEN3_IRQ_RESET(GEN8_PCU_);
Ben Widawskyabd58f02013-11-02 21:07:09 -07003631
Tvrtko Ursulin6e266952016-10-13 11:02:53 +01003632 if (HAS_PCH_SPLIT(dev_priv))
Tvrtko Ursulinb243f532016-11-16 08:55:38 +00003633 ibx_irq_reset(dev_priv);
Ben Widawskyabd58f02013-11-02 21:07:09 -07003634}
Ben Widawskyabd58f02013-11-02 21:07:09 -07003635
Mika Kuoppala51951ae2018-02-28 12:11:53 +02003636static void gen11_gt_irq_reset(struct drm_i915_private *dev_priv)
3637{
3638 /* Disable RCS, BCS, VCS and VECS class engines. */
3639 I915_WRITE(GEN11_RENDER_COPY_INTR_ENABLE, 0);
3640 I915_WRITE(GEN11_VCS_VECS_INTR_ENABLE, 0);
3641
3642 /* Restore masks irqs on RCS, BCS, VCS and VECS engines. */
3643 I915_WRITE(GEN11_RCS0_RSVD_INTR_MASK, ~0);
3644 I915_WRITE(GEN11_BCS_RSVD_INTR_MASK, ~0);
3645 I915_WRITE(GEN11_VCS0_VCS1_INTR_MASK, ~0);
3646 I915_WRITE(GEN11_VCS2_VCS3_INTR_MASK, ~0);
3647 I915_WRITE(GEN11_VECS0_VECS1_INTR_MASK, ~0);
Oscar Mateod02b98b2018-04-05 17:00:50 +03003648
3649 I915_WRITE(GEN11_GPM_WGBOXPERF_INTR_ENABLE, 0);
3650 I915_WRITE(GEN11_GPM_WGBOXPERF_INTR_MASK, ~0);
Mika Kuoppala51951ae2018-02-28 12:11:53 +02003651}
3652
3653static void gen11_irq_reset(struct drm_device *dev)
3654{
3655 struct drm_i915_private *dev_priv = dev->dev_private;
3656 int pipe;
3657
3658 I915_WRITE(GEN11_GFX_MSTR_IRQ, 0);
3659 POSTING_READ(GEN11_GFX_MSTR_IRQ);
3660
3661 gen11_gt_irq_reset(dev_priv);
3662
3663 I915_WRITE(GEN11_DISPLAY_INT_CTL, 0);
3664
3665 for_each_pipe(dev_priv, pipe)
3666 if (intel_display_power_is_enabled(dev_priv,
3667 POWER_DOMAIN_PIPE(pipe)))
3668 GEN8_IRQ_RESET_NDX(DE_PIPE, pipe);
3669
3670 GEN3_IRQ_RESET(GEN8_DE_PORT_);
3671 GEN3_IRQ_RESET(GEN8_DE_MISC_);
Dhinakaran Pandiyan121e7582018-06-15 17:05:29 -07003672 GEN3_IRQ_RESET(GEN11_DE_HPD_);
Dhinakaran Pandiyandf0d28c2018-06-15 17:05:28 -07003673 GEN3_IRQ_RESET(GEN11_GU_MISC_);
Mika Kuoppala51951ae2018-02-28 12:11:53 +02003674 GEN3_IRQ_RESET(GEN8_PCU_);
Anusha Srivatsa31604222018-06-26 13:52:23 -07003675
3676 if (HAS_PCH_ICP(dev_priv))
3677 GEN3_IRQ_RESET(SDE);
Mika Kuoppala51951ae2018-02-28 12:11:53 +02003678}
3679
Damien Lespiau4c6c03b2015-03-06 18:50:48 +00003680void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv,
Imre Deak001bd2c2017-07-12 18:54:13 +03003681 u8 pipe_mask)
Paulo Zanonid49bdb02014-07-04 11:50:31 -03003682{
Paulo Zanoni1180e202014-10-07 18:02:52 -03003683 uint32_t extra_ier = GEN8_PIPE_VBLANK | GEN8_PIPE_FIFO_UNDERRUN;
Ville Syrjälä6831f3e2016-02-19 20:47:31 +02003684 enum pipe pipe;
Paulo Zanonid49bdb02014-07-04 11:50:31 -03003685
Daniel Vetter13321782014-09-15 14:55:29 +02003686 spin_lock_irq(&dev_priv->irq_lock);
Imre Deak9dfe2e32017-09-28 13:06:24 +03003687
3688 if (!intel_irqs_enabled(dev_priv)) {
3689 spin_unlock_irq(&dev_priv->irq_lock);
3690 return;
3691 }
3692
Ville Syrjälä6831f3e2016-02-19 20:47:31 +02003693 for_each_pipe_masked(dev_priv, pipe, pipe_mask)
3694 GEN8_IRQ_INIT_NDX(DE_PIPE, pipe,
3695 dev_priv->de_irq_mask[pipe],
3696 ~dev_priv->de_irq_mask[pipe] | extra_ier);
Imre Deak9dfe2e32017-09-28 13:06:24 +03003697
Daniel Vetter13321782014-09-15 14:55:29 +02003698 spin_unlock_irq(&dev_priv->irq_lock);
Paulo Zanonid49bdb02014-07-04 11:50:31 -03003699}
3700
Ville Syrjäläaae8ba82016-02-19 20:47:30 +02003701void gen8_irq_power_well_pre_disable(struct drm_i915_private *dev_priv,
Imre Deak001bd2c2017-07-12 18:54:13 +03003702 u8 pipe_mask)
Ville Syrjäläaae8ba82016-02-19 20:47:30 +02003703{
Ville Syrjälä6831f3e2016-02-19 20:47:31 +02003704 enum pipe pipe;
3705
Ville Syrjäläaae8ba82016-02-19 20:47:30 +02003706 spin_lock_irq(&dev_priv->irq_lock);
Imre Deak9dfe2e32017-09-28 13:06:24 +03003707
3708 if (!intel_irqs_enabled(dev_priv)) {
3709 spin_unlock_irq(&dev_priv->irq_lock);
3710 return;
3711 }
3712
Ville Syrjälä6831f3e2016-02-19 20:47:31 +02003713 for_each_pipe_masked(dev_priv, pipe, pipe_mask)
3714 GEN8_IRQ_RESET_NDX(DE_PIPE, pipe);
Imre Deak9dfe2e32017-09-28 13:06:24 +03003715
Ville Syrjäläaae8ba82016-02-19 20:47:30 +02003716 spin_unlock_irq(&dev_priv->irq_lock);
3717
3718 /* make sure we're done processing display irqs */
Chris Wilson91c8a322016-07-05 10:40:23 +01003719 synchronize_irq(dev_priv->drm.irq);
Ville Syrjäläaae8ba82016-02-19 20:47:30 +02003720}
3721
Ville Syrjälä6bcdb1c2017-08-18 21:37:04 +03003722static void cherryview_irq_reset(struct drm_device *dev)
Ville Syrjälä43f328d2014-04-09 20:40:52 +03003723{
Chris Wilsonfac5e232016-07-04 11:34:36 +01003724 struct drm_i915_private *dev_priv = to_i915(dev);
Ville Syrjälä43f328d2014-04-09 20:40:52 +03003725
3726 I915_WRITE(GEN8_MASTER_IRQ, 0);
3727 POSTING_READ(GEN8_MASTER_IRQ);
3728
Daniel Vetterd6e3cca2014-05-22 22:18:22 +02003729 gen8_gt_irq_reset(dev_priv);
Ville Syrjälä43f328d2014-04-09 20:40:52 +03003730
Ville Syrjälä3488d4e2017-08-18 21:36:52 +03003731 GEN3_IRQ_RESET(GEN8_PCU_);
Ville Syrjälä43f328d2014-04-09 20:40:52 +03003732
Ville Syrjäläad22d102016-04-12 18:56:14 +03003733 spin_lock_irq(&dev_priv->irq_lock);
Ville Syrjälä99182712016-04-11 16:56:25 +03003734 if (dev_priv->display_irqs_enabled)
3735 vlv_display_irq_reset(dev_priv);
Ville Syrjäläad22d102016-04-12 18:56:14 +03003736 spin_unlock_irq(&dev_priv->irq_lock);
Ville Syrjälä43f328d2014-04-09 20:40:52 +03003737}
3738
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01003739static u32 intel_hpd_enabled_irqs(struct drm_i915_private *dev_priv,
Ville Syrjälä87a02102015-08-27 23:55:57 +03003740 const u32 hpd[HPD_NUM_PINS])
3741{
Ville Syrjälä87a02102015-08-27 23:55:57 +03003742 struct intel_encoder *encoder;
3743 u32 enabled_irqs = 0;
3744
Chris Wilson91c8a322016-07-05 10:40:23 +01003745 for_each_intel_encoder(&dev_priv->drm, encoder)
Ville Syrjälä87a02102015-08-27 23:55:57 +03003746 if (dev_priv->hotplug.stats[encoder->hpd_pin].state == HPD_ENABLED)
3747 enabled_irqs |= hpd[encoder->hpd_pin];
3748
3749 return enabled_irqs;
3750}
3751
Imre Deak1a56b1a2017-01-27 11:39:21 +02003752static void ibx_hpd_detection_setup(struct drm_i915_private *dev_priv)
3753{
3754 u32 hotplug;
3755
3756 /*
3757 * Enable digital hotplug on the PCH, and configure the DP short pulse
3758 * duration to 2ms (which is the minimum in the Display Port spec).
3759 * The pulse duration bits are reserved on LPT+.
3760 */
3761 hotplug = I915_READ(PCH_PORT_HOTPLUG);
3762 hotplug &= ~(PORTB_PULSE_DURATION_MASK |
3763 PORTC_PULSE_DURATION_MASK |
3764 PORTD_PULSE_DURATION_MASK);
3765 hotplug |= PORTB_HOTPLUG_ENABLE | PORTB_PULSE_DURATION_2ms;
3766 hotplug |= PORTC_HOTPLUG_ENABLE | PORTC_PULSE_DURATION_2ms;
3767 hotplug |= PORTD_HOTPLUG_ENABLE | PORTD_PULSE_DURATION_2ms;
3768 /*
3769 * When CPU and PCH are on the same package, port A
3770 * HPD must be enabled in both north and south.
3771 */
3772 if (HAS_PCH_LPT_LP(dev_priv))
3773 hotplug |= PORTA_HOTPLUG_ENABLE;
3774 I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3775}
3776
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01003777static void ibx_hpd_irq_setup(struct drm_i915_private *dev_priv)
Keith Packard7fe0b972011-09-19 13:31:02 -07003778{
Imre Deak1a56b1a2017-01-27 11:39:21 +02003779 u32 hotplug_irqs, enabled_irqs;
Keith Packard7fe0b972011-09-19 13:31:02 -07003780
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01003781 if (HAS_PCH_IBX(dev_priv)) {
Daniel Vetterfee884e2013-07-04 23:35:21 +02003782 hotplug_irqs = SDE_HOTPLUG_MASK;
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01003783 enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_ibx);
Daniel Vetter82a28bc2013-03-27 15:55:01 +01003784 } else {
Daniel Vetterfee884e2013-07-04 23:35:21 +02003785 hotplug_irqs = SDE_HOTPLUG_MASK_CPT;
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01003786 enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_cpt);
Daniel Vetter82a28bc2013-03-27 15:55:01 +01003787 }
3788
Daniel Vetterfee884e2013-07-04 23:35:21 +02003789 ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
Daniel Vetter82a28bc2013-03-27 15:55:01 +01003790
Imre Deak1a56b1a2017-01-27 11:39:21 +02003791 ibx_hpd_detection_setup(dev_priv);
Ville Syrjälä6dbf30c2015-08-27 23:56:02 +03003792}
Xiong Zhang26951ca2015-08-17 15:55:50 +08003793
Anusha Srivatsa31604222018-06-26 13:52:23 -07003794static void icp_hpd_detection_setup(struct drm_i915_private *dev_priv)
3795{
3796 u32 hotplug;
3797
3798 hotplug = I915_READ(SHOTPLUG_CTL_DDI);
3799 hotplug |= ICP_DDIA_HPD_ENABLE |
3800 ICP_DDIB_HPD_ENABLE;
3801 I915_WRITE(SHOTPLUG_CTL_DDI, hotplug);
3802
3803 hotplug = I915_READ(SHOTPLUG_CTL_TC);
3804 hotplug |= ICP_TC_HPD_ENABLE(PORT_TC1) |
3805 ICP_TC_HPD_ENABLE(PORT_TC2) |
3806 ICP_TC_HPD_ENABLE(PORT_TC3) |
3807 ICP_TC_HPD_ENABLE(PORT_TC4);
3808 I915_WRITE(SHOTPLUG_CTL_TC, hotplug);
3809}
3810
3811static void icp_hpd_irq_setup(struct drm_i915_private *dev_priv)
3812{
3813 u32 hotplug_irqs, enabled_irqs;
3814
3815 hotplug_irqs = SDE_DDI_MASK_ICP | SDE_TC_MASK_ICP;
3816 enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_icp);
3817
3818 ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
3819
3820 icp_hpd_detection_setup(dev_priv);
3821}
3822
Dhinakaran Pandiyan121e7582018-06-15 17:05:29 -07003823static void gen11_hpd_detection_setup(struct drm_i915_private *dev_priv)
3824{
3825 u32 hotplug;
3826
3827 hotplug = I915_READ(GEN11_TC_HOTPLUG_CTL);
3828 hotplug |= GEN11_HOTPLUG_CTL_ENABLE(PORT_TC1) |
3829 GEN11_HOTPLUG_CTL_ENABLE(PORT_TC2) |
3830 GEN11_HOTPLUG_CTL_ENABLE(PORT_TC3) |
3831 GEN11_HOTPLUG_CTL_ENABLE(PORT_TC4);
3832 I915_WRITE(GEN11_TC_HOTPLUG_CTL, hotplug);
Dhinakaran Pandiyanb796b972018-06-15 17:05:30 -07003833
3834 hotplug = I915_READ(GEN11_TBT_HOTPLUG_CTL);
3835 hotplug |= GEN11_HOTPLUG_CTL_ENABLE(PORT_TC1) |
3836 GEN11_HOTPLUG_CTL_ENABLE(PORT_TC2) |
3837 GEN11_HOTPLUG_CTL_ENABLE(PORT_TC3) |
3838 GEN11_HOTPLUG_CTL_ENABLE(PORT_TC4);
3839 I915_WRITE(GEN11_TBT_HOTPLUG_CTL, hotplug);
Dhinakaran Pandiyan121e7582018-06-15 17:05:29 -07003840}
3841
3842static void gen11_hpd_irq_setup(struct drm_i915_private *dev_priv)
3843{
3844 u32 hotplug_irqs, enabled_irqs;
3845 u32 val;
3846
Dhinakaran Pandiyanb796b972018-06-15 17:05:30 -07003847 enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_gen11);
3848 hotplug_irqs = GEN11_DE_TC_HOTPLUG_MASK | GEN11_DE_TBT_HOTPLUG_MASK;
Dhinakaran Pandiyan121e7582018-06-15 17:05:29 -07003849
3850 val = I915_READ(GEN11_DE_HPD_IMR);
3851 val &= ~hotplug_irqs;
3852 I915_WRITE(GEN11_DE_HPD_IMR, val);
3853 POSTING_READ(GEN11_DE_HPD_IMR);
3854
3855 gen11_hpd_detection_setup(dev_priv);
Anusha Srivatsa31604222018-06-26 13:52:23 -07003856
3857 if (HAS_PCH_ICP(dev_priv))
3858 icp_hpd_irq_setup(dev_priv);
Dhinakaran Pandiyan121e7582018-06-15 17:05:29 -07003859}
3860
Imre Deak2a57d9c2017-01-27 11:39:18 +02003861static void spt_hpd_detection_setup(struct drm_i915_private *dev_priv)
3862{
Rodrigo Vivi3b92e262017-09-19 14:57:03 -07003863 u32 val, hotplug;
3864
3865 /* Display WA #1179 WaHardHangonHotPlug: cnp */
3866 if (HAS_PCH_CNP(dev_priv)) {
3867 val = I915_READ(SOUTH_CHICKEN1);
3868 val &= ~CHASSIS_CLK_REQ_DURATION_MASK;
3869 val |= CHASSIS_CLK_REQ_DURATION(0xf);
3870 I915_WRITE(SOUTH_CHICKEN1, val);
3871 }
Imre Deak2a57d9c2017-01-27 11:39:18 +02003872
3873 /* Enable digital hotplug on the PCH */
3874 hotplug = I915_READ(PCH_PORT_HOTPLUG);
3875 hotplug |= PORTA_HOTPLUG_ENABLE |
3876 PORTB_HOTPLUG_ENABLE |
3877 PORTC_HOTPLUG_ENABLE |
3878 PORTD_HOTPLUG_ENABLE;
3879 I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3880
3881 hotplug = I915_READ(PCH_PORT_HOTPLUG2);
3882 hotplug |= PORTE_HOTPLUG_ENABLE;
3883 I915_WRITE(PCH_PORT_HOTPLUG2, hotplug);
3884}
3885
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01003886static void spt_hpd_irq_setup(struct drm_i915_private *dev_priv)
Ville Syrjälä6dbf30c2015-08-27 23:56:02 +03003887{
Imre Deak2a57d9c2017-01-27 11:39:18 +02003888 u32 hotplug_irqs, enabled_irqs;
Ville Syrjälä6dbf30c2015-08-27 23:56:02 +03003889
3890 hotplug_irqs = SDE_HOTPLUG_MASK_SPT;
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01003891 enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_spt);
Ville Syrjälä6dbf30c2015-08-27 23:56:02 +03003892
3893 ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
3894
Imre Deak2a57d9c2017-01-27 11:39:18 +02003895 spt_hpd_detection_setup(dev_priv);
Keith Packard7fe0b972011-09-19 13:31:02 -07003896}
3897
Imre Deak1a56b1a2017-01-27 11:39:21 +02003898static void ilk_hpd_detection_setup(struct drm_i915_private *dev_priv)
3899{
3900 u32 hotplug;
3901
3902 /*
3903 * Enable digital hotplug on the CPU, and configure the DP short pulse
3904 * duration to 2ms (which is the minimum in the Display Port spec)
3905 * The pulse duration bits are reserved on HSW+.
3906 */
3907 hotplug = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
3908 hotplug &= ~DIGITAL_PORTA_PULSE_DURATION_MASK;
3909 hotplug |= DIGITAL_PORTA_HOTPLUG_ENABLE |
3910 DIGITAL_PORTA_PULSE_DURATION_2ms;
3911 I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, hotplug);
3912}
3913
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01003914static void ilk_hpd_irq_setup(struct drm_i915_private *dev_priv)
Ville Syrjäläe4ce95a2015-08-27 23:56:03 +03003915{
Imre Deak1a56b1a2017-01-27 11:39:21 +02003916 u32 hotplug_irqs, enabled_irqs;
Ville Syrjäläe4ce95a2015-08-27 23:56:03 +03003917
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01003918 if (INTEL_GEN(dev_priv) >= 8) {
Ville Syrjälä3a3b3c72015-08-27 23:56:06 +03003919 hotplug_irqs = GEN8_PORT_DP_A_HOTPLUG;
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01003920 enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_bdw);
Ville Syrjälä3a3b3c72015-08-27 23:56:06 +03003921
3922 bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01003923 } else if (INTEL_GEN(dev_priv) >= 7) {
Ville Syrjälä23bb4cb2015-08-27 23:56:04 +03003924 hotplug_irqs = DE_DP_A_HOTPLUG_IVB;
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01003925 enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_ivb);
Ville Syrjälä3a3b3c72015-08-27 23:56:06 +03003926
3927 ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
Ville Syrjälä23bb4cb2015-08-27 23:56:04 +03003928 } else {
3929 hotplug_irqs = DE_DP_A_HOTPLUG;
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01003930 enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_ilk);
Ville Syrjäläe4ce95a2015-08-27 23:56:03 +03003931
Ville Syrjälä3a3b3c72015-08-27 23:56:06 +03003932 ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
3933 }
Ville Syrjäläe4ce95a2015-08-27 23:56:03 +03003934
Imre Deak1a56b1a2017-01-27 11:39:21 +02003935 ilk_hpd_detection_setup(dev_priv);
Ville Syrjäläe4ce95a2015-08-27 23:56:03 +03003936
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01003937 ibx_hpd_irq_setup(dev_priv);
Ville Syrjäläe4ce95a2015-08-27 23:56:03 +03003938}
3939
Imre Deak2a57d9c2017-01-27 11:39:18 +02003940static void __bxt_hpd_detection_setup(struct drm_i915_private *dev_priv,
3941 u32 enabled_irqs)
Shashank Sharmae0a20ad2015-03-27 14:54:14 +02003942{
Imre Deak2a57d9c2017-01-27 11:39:18 +02003943 u32 hotplug;
Shashank Sharmae0a20ad2015-03-27 14:54:14 +02003944
Ville Syrjäläa52bb152015-08-27 23:56:11 +03003945 hotplug = I915_READ(PCH_PORT_HOTPLUG);
Imre Deak2a57d9c2017-01-27 11:39:18 +02003946 hotplug |= PORTA_HOTPLUG_ENABLE |
3947 PORTB_HOTPLUG_ENABLE |
3948 PORTC_HOTPLUG_ENABLE;
Shubhangi Shrivastavad252bf62016-03-31 16:11:47 +05303949
3950 DRM_DEBUG_KMS("Invert bit setting: hp_ctl:%x hp_port:%x\n",
3951 hotplug, enabled_irqs);
3952 hotplug &= ~BXT_DDI_HPD_INVERT_MASK;
3953
3954 /*
3955 * For BXT invert bit has to be set based on AOB design
3956 * for HPD detection logic, update it based on VBT fields.
3957 */
Shubhangi Shrivastavad252bf62016-03-31 16:11:47 +05303958 if ((enabled_irqs & BXT_DE_PORT_HP_DDIA) &&
3959 intel_bios_is_port_hpd_inverted(dev_priv, PORT_A))
3960 hotplug |= BXT_DDIA_HPD_INVERT;
3961 if ((enabled_irqs & BXT_DE_PORT_HP_DDIB) &&
3962 intel_bios_is_port_hpd_inverted(dev_priv, PORT_B))
3963 hotplug |= BXT_DDIB_HPD_INVERT;
3964 if ((enabled_irqs & BXT_DE_PORT_HP_DDIC) &&
3965 intel_bios_is_port_hpd_inverted(dev_priv, PORT_C))
3966 hotplug |= BXT_DDIC_HPD_INVERT;
3967
Ville Syrjäläa52bb152015-08-27 23:56:11 +03003968 I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
Shashank Sharmae0a20ad2015-03-27 14:54:14 +02003969}
3970
Imre Deak2a57d9c2017-01-27 11:39:18 +02003971static void bxt_hpd_detection_setup(struct drm_i915_private *dev_priv)
3972{
3973 __bxt_hpd_detection_setup(dev_priv, BXT_DE_PORT_HOTPLUG_MASK);
3974}
3975
3976static void bxt_hpd_irq_setup(struct drm_i915_private *dev_priv)
3977{
3978 u32 hotplug_irqs, enabled_irqs;
3979
3980 enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_bxt);
3981 hotplug_irqs = BXT_DE_PORT_HOTPLUG_MASK;
3982
3983 bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
3984
3985 __bxt_hpd_detection_setup(dev_priv, enabled_irqs);
3986}
3987
Paulo Zanonid46da432013-02-08 17:35:15 -02003988static void ibx_irq_postinstall(struct drm_device *dev)
3989{
Chris Wilsonfac5e232016-07-04 11:34:36 +01003990 struct drm_i915_private *dev_priv = to_i915(dev);
Daniel Vetter82a28bc2013-03-27 15:55:01 +01003991 u32 mask;
Paulo Zanonid46da432013-02-08 17:35:15 -02003992
Tvrtko Ursulin6e266952016-10-13 11:02:53 +01003993 if (HAS_PCH_NOP(dev_priv))
Daniel Vetter692a04c2013-05-29 21:43:05 +02003994 return;
3995
Tvrtko Ursulin6e266952016-10-13 11:02:53 +01003996 if (HAS_PCH_IBX(dev_priv))
Daniel Vetter5c673b62014-03-07 20:34:46 +01003997 mask = SDE_GMBUS | SDE_AUX_MASK | SDE_POISON;
Dhinakaran Pandiyan4ebc6502017-09-08 17:42:55 -07003998 else if (HAS_PCH_CPT(dev_priv) || HAS_PCH_LPT(dev_priv))
Daniel Vetter5c673b62014-03-07 20:34:46 +01003999 mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT;
Dhinakaran Pandiyan4ebc6502017-09-08 17:42:55 -07004000 else
4001 mask = SDE_GMBUS_CPT;
Paulo Zanoni86642812013-04-12 17:57:57 -03004002
Ville Syrjälä3488d4e2017-08-18 21:36:52 +03004003 gen3_assert_iir_is_zero(dev_priv, SDEIIR);
Paulo Zanonid46da432013-02-08 17:35:15 -02004004 I915_WRITE(SDEIMR, ~mask);
Imre Deak2a57d9c2017-01-27 11:39:18 +02004005
4006 if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv) ||
4007 HAS_PCH_LPT(dev_priv))
Imre Deak1a56b1a2017-01-27 11:39:21 +02004008 ibx_hpd_detection_setup(dev_priv);
Imre Deak2a57d9c2017-01-27 11:39:18 +02004009 else
4010 spt_hpd_detection_setup(dev_priv);
Paulo Zanonid46da432013-02-08 17:35:15 -02004011}
4012
Daniel Vetter0a9a8c92013-07-12 22:43:26 +02004013static void gen5_gt_irq_postinstall(struct drm_device *dev)
4014{
Chris Wilsonfac5e232016-07-04 11:34:36 +01004015 struct drm_i915_private *dev_priv = to_i915(dev);
Daniel Vetter0a9a8c92013-07-12 22:43:26 +02004016 u32 pm_irqs, gt_irqs;
4017
4018 pm_irqs = gt_irqs = 0;
4019
4020 dev_priv->gt_irq_mask = ~0;
Tvrtko Ursulin3c9192b2016-10-13 11:03:05 +01004021 if (HAS_L3_DPF(dev_priv)) {
Daniel Vetter0a9a8c92013-07-12 22:43:26 +02004022 /* L3 parity interrupt is always unmasked. */
Tvrtko Ursulin772c2a52016-10-13 11:03:01 +01004023 dev_priv->gt_irq_mask = ~GT_PARITY_ERROR(dev_priv);
4024 gt_irqs |= GT_PARITY_ERROR(dev_priv);
Daniel Vetter0a9a8c92013-07-12 22:43:26 +02004025 }
4026
4027 gt_irqs |= GT_RENDER_USER_INTERRUPT;
Tvrtko Ursulin5db94012016-10-13 11:03:10 +01004028 if (IS_GEN5(dev_priv)) {
Chris Wilsonf8973c22016-07-01 17:23:21 +01004029 gt_irqs |= ILK_BSD_USER_INTERRUPT;
Daniel Vetter0a9a8c92013-07-12 22:43:26 +02004030 } else {
4031 gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT;
4032 }
4033
Ville Syrjälä3488d4e2017-08-18 21:36:52 +03004034 GEN3_IRQ_INIT(GT, dev_priv->gt_irq_mask, gt_irqs);
Daniel Vetter0a9a8c92013-07-12 22:43:26 +02004035
Tvrtko Ursulinb243f532016-11-16 08:55:38 +00004036 if (INTEL_GEN(dev_priv) >= 6) {
Imre Deak78e68d32014-12-15 18:59:27 +02004037 /*
4038 * RPS interrupts will get enabled/disabled on demand when RPS
4039 * itself is enabled/disabled.
4040 */
Akash Goelf4e9af42016-10-12 21:54:30 +05304041 if (HAS_VEBOX(dev_priv)) {
Daniel Vetter0a9a8c92013-07-12 22:43:26 +02004042 pm_irqs |= PM_VEBOX_USER_INTERRUPT;
Akash Goelf4e9af42016-10-12 21:54:30 +05304043 dev_priv->pm_ier |= PM_VEBOX_USER_INTERRUPT;
4044 }
Daniel Vetter0a9a8c92013-07-12 22:43:26 +02004045
Akash Goelf4e9af42016-10-12 21:54:30 +05304046 dev_priv->pm_imr = 0xffffffff;
Ville Syrjälä3488d4e2017-08-18 21:36:52 +03004047 GEN3_IRQ_INIT(GEN6_PM, dev_priv->pm_imr, pm_irqs);
Daniel Vetter0a9a8c92013-07-12 22:43:26 +02004048 }
4049}
4050
Jesse Barnesf71d4af2011-06-28 13:00:41 -07004051static int ironlake_irq_postinstall(struct drm_device *dev)
Zhenyu Wang036a4a72009-06-08 14:40:19 +08004052{
Chris Wilsonfac5e232016-07-04 11:34:36 +01004053 struct drm_i915_private *dev_priv = to_i915(dev);
Paulo Zanoni8e76f8d2013-07-12 20:01:56 -03004054 u32 display_mask, extra_mask;
4055
Tvrtko Ursulinb243f532016-11-16 08:55:38 +00004056 if (INTEL_GEN(dev_priv) >= 7) {
Paulo Zanoni8e76f8d2013-07-12 20:01:56 -03004057 display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE_IVB |
Ville Syrjälä842ebf72017-08-18 21:36:50 +03004058 DE_PCH_EVENT_IVB | DE_AUX_CHANNEL_A_IVB);
Paulo Zanoni8e76f8d2013-07-12 20:01:56 -03004059 extra_mask = (DE_PIPEC_VBLANK_IVB | DE_PIPEB_VBLANK_IVB |
Ville Syrjälä23bb4cb2015-08-27 23:56:04 +03004060 DE_PIPEA_VBLANK_IVB | DE_ERR_INT_IVB |
4061 DE_DP_A_HOTPLUG_IVB);
Paulo Zanoni8e76f8d2013-07-12 20:01:56 -03004062 } else {
4063 display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
Ville Syrjälä842ebf72017-08-18 21:36:50 +03004064 DE_AUX_CHANNEL_A | DE_PIPEB_CRC_DONE |
4065 DE_PIPEA_CRC_DONE | DE_POISON);
Ville Syrjäläe4ce95a2015-08-27 23:56:03 +03004066 extra_mask = (DE_PIPEA_VBLANK | DE_PIPEB_VBLANK | DE_PCU_EVENT |
4067 DE_PIPEB_FIFO_UNDERRUN | DE_PIPEA_FIFO_UNDERRUN |
4068 DE_DP_A_HOTPLUG);
Paulo Zanoni8e76f8d2013-07-12 20:01:56 -03004069 }
Zhenyu Wang036a4a72009-06-08 14:40:19 +08004070
Daniel Vetterfc340442018-04-05 15:00:23 -07004071 if (IS_HASWELL(dev_priv)) {
4072 gen3_assert_iir_is_zero(dev_priv, EDP_PSR_IIR);
Dhinakaran Pandiyan54fd3142018-04-04 18:37:17 -07004073 intel_psr_irq_control(dev_priv, dev_priv->psr.debug);
Daniel Vetterfc340442018-04-05 15:00:23 -07004074 display_mask |= DE_EDP_PSR_INT_HSW;
4075 }
4076
Chris Wilson1ec14ad2010-12-04 11:30:53 +00004077 dev_priv->irq_mask = ~display_mask;
Zhenyu Wang036a4a72009-06-08 14:40:19 +08004078
Paulo Zanoni622364b2014-04-01 15:37:22 -03004079 ibx_irq_pre_postinstall(dev);
4080
Ville Syrjälä3488d4e2017-08-18 21:36:52 +03004081 GEN3_IRQ_INIT(DE, dev_priv->irq_mask, display_mask | extra_mask);
Zhenyu Wang036a4a72009-06-08 14:40:19 +08004082
Daniel Vetter0a9a8c92013-07-12 22:43:26 +02004083 gen5_gt_irq_postinstall(dev);
Zhenyu Wang036a4a72009-06-08 14:40:19 +08004084
Imre Deak1a56b1a2017-01-27 11:39:21 +02004085 ilk_hpd_detection_setup(dev_priv);
4086
Paulo Zanonid46da432013-02-08 17:35:15 -02004087 ibx_irq_postinstall(dev);
Keith Packard7fe0b972011-09-19 13:31:02 -07004088
Tvrtko Ursulin50a0bc92016-10-13 11:02:58 +01004089 if (IS_IRONLAKE_M(dev_priv)) {
Daniel Vetter6005ce42013-06-27 13:44:59 +02004090 /* Enable PCU event interrupts
4091 *
4092 * spinlocking not required here for correctness since interrupt
Daniel Vetter4bc9d432013-06-27 13:44:58 +02004093 * setup is guaranteed to run in single-threaded context. But we
4094 * need it to make the assert_spin_locked happy. */
Daniel Vetterd6207432014-09-15 14:55:27 +02004095 spin_lock_irq(&dev_priv->irq_lock);
Ville Syrjäläfbdedaea2015-11-23 18:06:16 +02004096 ilk_enable_display_irq(dev_priv, DE_PCU_EVENT);
Daniel Vetterd6207432014-09-15 14:55:27 +02004097 spin_unlock_irq(&dev_priv->irq_lock);
Jesse Barnesf97108d2010-01-29 11:27:07 -08004098 }
4099
Zhenyu Wang036a4a72009-06-08 14:40:19 +08004100 return 0;
4101}
4102
Imre Deakf8b79e52014-03-04 19:23:07 +02004103void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv)
4104{
Chris Wilson67520412017-03-02 13:28:01 +00004105 lockdep_assert_held(&dev_priv->irq_lock);
Imre Deakf8b79e52014-03-04 19:23:07 +02004106
4107 if (dev_priv->display_irqs_enabled)
4108 return;
4109
4110 dev_priv->display_irqs_enabled = true;
4111
Ville Syrjäläd6c69802016-04-11 16:56:27 +03004112 if (intel_irqs_enabled(dev_priv)) {
4113 vlv_display_irq_reset(dev_priv);
Ville Syrjäläad22d102016-04-12 18:56:14 +03004114 vlv_display_irq_postinstall(dev_priv);
Ville Syrjäläd6c69802016-04-11 16:56:27 +03004115 }
Imre Deakf8b79e52014-03-04 19:23:07 +02004116}
4117
4118void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv)
4119{
Chris Wilson67520412017-03-02 13:28:01 +00004120 lockdep_assert_held(&dev_priv->irq_lock);
Imre Deakf8b79e52014-03-04 19:23:07 +02004121
4122 if (!dev_priv->display_irqs_enabled)
4123 return;
4124
4125 dev_priv->display_irqs_enabled = false;
4126
Imre Deak950eaba2014-09-08 15:21:09 +03004127 if (intel_irqs_enabled(dev_priv))
Ville Syrjäläad22d102016-04-12 18:56:14 +03004128 vlv_display_irq_reset(dev_priv);
Imre Deakf8b79e52014-03-04 19:23:07 +02004129}
4130
Ville Syrjälä0e6c9a92014-10-30 19:43:00 +02004131
4132static int valleyview_irq_postinstall(struct drm_device *dev)
4133{
Chris Wilsonfac5e232016-07-04 11:34:36 +01004134 struct drm_i915_private *dev_priv = to_i915(dev);
Ville Syrjälä0e6c9a92014-10-30 19:43:00 +02004135
Daniel Vetter0a9a8c92013-07-12 22:43:26 +02004136 gen5_gt_irq_postinstall(dev);
Jesse Barnes7e231dbe2012-03-28 13:39:38 -07004137
Ville Syrjäläad22d102016-04-12 18:56:14 +03004138 spin_lock_irq(&dev_priv->irq_lock);
Ville Syrjälä99182712016-04-11 16:56:25 +03004139 if (dev_priv->display_irqs_enabled)
4140 vlv_display_irq_postinstall(dev_priv);
Ville Syrjäläad22d102016-04-12 18:56:14 +03004141 spin_unlock_irq(&dev_priv->irq_lock);
4142
Jesse Barnes7e231dbe2012-03-28 13:39:38 -07004143 I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
Ville Syrjälä34c7b8a2016-04-13 21:19:48 +03004144 POSTING_READ(VLV_MASTER_IER);
Daniel Vetter20afbda2012-12-11 14:05:07 +01004145
4146 return 0;
4147}
4148
Ben Widawskyabd58f02013-11-02 21:07:09 -07004149static void gen8_gt_irq_postinstall(struct drm_i915_private *dev_priv)
4150{
Ben Widawskyabd58f02013-11-02 21:07:09 -07004151 /* These are interrupts we'll toggle with the ring mask register */
4152 uint32_t gt_interrupts[] = {
4153 GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
Oscar Mateo73d477f2014-07-24 17:04:31 +01004154 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
Oscar Mateo73d477f2014-07-24 17:04:31 +01004155 GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT |
4156 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT,
Ben Widawskyabd58f02013-11-02 21:07:09 -07004157 GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
Oscar Mateo73d477f2014-07-24 17:04:31 +01004158 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
4159 GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT |
4160 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT,
Ben Widawskyabd58f02013-11-02 21:07:09 -07004161 0,
Oscar Mateo73d477f2014-07-24 17:04:31 +01004162 GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT |
4163 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT
Ben Widawskyabd58f02013-11-02 21:07:09 -07004164 };
4165
Tvrtko Ursulin98735732016-04-19 16:46:08 +01004166 if (HAS_L3_DPF(dev_priv))
4167 gt_interrupts[0] |= GT_RENDER_L3_PARITY_ERROR_INTERRUPT;
4168
Akash Goelf4e9af42016-10-12 21:54:30 +05304169 dev_priv->pm_ier = 0x0;
4170 dev_priv->pm_imr = ~dev_priv->pm_ier;
Deepak S9a2d2d82014-08-22 08:32:40 +05304171 GEN8_IRQ_INIT_NDX(GT, 0, ~gt_interrupts[0], gt_interrupts[0]);
4172 GEN8_IRQ_INIT_NDX(GT, 1, ~gt_interrupts[1], gt_interrupts[1]);
Imre Deak78e68d32014-12-15 18:59:27 +02004173 /*
4174 * RPS interrupts will get enabled/disabled on demand when RPS itself
Sagar Arun Kamble26705e22016-10-12 21:54:31 +05304175 * is enabled/disabled. Same wil be the case for GuC interrupts.
Imre Deak78e68d32014-12-15 18:59:27 +02004176 */
Akash Goelf4e9af42016-10-12 21:54:30 +05304177 GEN8_IRQ_INIT_NDX(GT, 2, dev_priv->pm_imr, dev_priv->pm_ier);
Deepak S9a2d2d82014-08-22 08:32:40 +05304178 GEN8_IRQ_INIT_NDX(GT, 3, ~gt_interrupts[3], gt_interrupts[3]);
Ben Widawskyabd58f02013-11-02 21:07:09 -07004179}
4180
4181static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
4182{
Damien Lespiau770de83d2014-03-20 20:45:01 +00004183 uint32_t de_pipe_masked = GEN8_PIPE_CDCLK_CRC_DONE;
4184 uint32_t de_pipe_enables;
Ville Syrjälä3a3b3c72015-08-27 23:56:06 +03004185 u32 de_port_masked = GEN8_AUX_CHANNEL_A;
4186 u32 de_port_enables;
Dhinakaran Pandiyandf0d28c2018-06-15 17:05:28 -07004187 u32 de_misc_masked = GEN8_DE_EDP_PSR;
Ville Syrjälä3a3b3c72015-08-27 23:56:06 +03004188 enum pipe pipe;
Damien Lespiau770de83d2014-03-20 20:45:01 +00004189
Dhinakaran Pandiyandf0d28c2018-06-15 17:05:28 -07004190 if (INTEL_GEN(dev_priv) <= 10)
4191 de_misc_masked |= GEN8_DE_MISC_GSE;
4192
Pandiyan, Dhinakaranbca2bf22017-07-18 11:28:00 -07004193 if (INTEL_GEN(dev_priv) >= 9) {
Ville Syrjälä842ebf72017-08-18 21:36:50 +03004194 de_pipe_masked |= GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
Ville Syrjälä3a3b3c72015-08-27 23:56:06 +03004195 de_port_masked |= GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C |
4196 GEN9_AUX_CHANNEL_D;
Ander Conselvan de Oliveiracc3f90f2016-12-02 10:23:49 +02004197 if (IS_GEN9_LP(dev_priv))
Ville Syrjälä3a3b3c72015-08-27 23:56:06 +03004198 de_port_masked |= BXT_DE_PORT_GMBUS;
4199 } else {
Ville Syrjälä842ebf72017-08-18 21:36:50 +03004200 de_pipe_masked |= GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
Ville Syrjälä3a3b3c72015-08-27 23:56:06 +03004201 }
Damien Lespiau770de83d2014-03-20 20:45:01 +00004202
James Ausmusbb187e92018-06-11 17:25:12 -07004203 if (INTEL_GEN(dev_priv) >= 11)
4204 de_port_masked |= ICL_AUX_CHANNEL_E;
4205
Dhinakaran Pandiyan9bb635d2018-05-21 17:25:35 -07004206 if (IS_CNL_WITH_PORT_F(dev_priv) || INTEL_GEN(dev_priv) >= 11)
Rodrigo Vivia324fca2018-01-29 15:22:15 -08004207 de_port_masked |= CNL_AUX_CHANNEL_F;
4208
Damien Lespiau770de83d2014-03-20 20:45:01 +00004209 de_pipe_enables = de_pipe_masked | GEN8_PIPE_VBLANK |
4210 GEN8_PIPE_FIFO_UNDERRUN;
4211
Ville Syrjälä3a3b3c72015-08-27 23:56:06 +03004212 de_port_enables = de_port_masked;
Ander Conselvan de Oliveiracc3f90f2016-12-02 10:23:49 +02004213 if (IS_GEN9_LP(dev_priv))
Ville Syrjäläa52bb152015-08-27 23:56:11 +03004214 de_port_enables |= BXT_DE_PORT_HOTPLUG_MASK;
4215 else if (IS_BROADWELL(dev_priv))
Ville Syrjälä3a3b3c72015-08-27 23:56:06 +03004216 de_port_enables |= GEN8_PORT_DP_A_HOTPLUG;
4217
Ville Syrjäläe04f7ec2018-04-03 14:24:18 -07004218 gen3_assert_iir_is_zero(dev_priv, EDP_PSR_IIR);
Dhinakaran Pandiyan54fd3142018-04-04 18:37:17 -07004219 intel_psr_irq_control(dev_priv, dev_priv->psr.debug);
Ville Syrjäläe04f7ec2018-04-03 14:24:18 -07004220
Mika Kahola0a195c02017-10-10 13:17:04 +03004221 for_each_pipe(dev_priv, pipe) {
4222 dev_priv->de_irq_mask[pipe] = ~de_pipe_masked;
Ben Widawskyabd58f02013-11-02 21:07:09 -07004223
Daniel Vetterf458ebb2014-09-30 10:56:39 +02004224 if (intel_display_power_is_enabled(dev_priv,
Paulo Zanoni813bde42014-07-04 11:50:29 -03004225 POWER_DOMAIN_PIPE(pipe)))
4226 GEN8_IRQ_INIT_NDX(DE_PIPE, pipe,
4227 dev_priv->de_irq_mask[pipe],
4228 de_pipe_enables);
Mika Kahola0a195c02017-10-10 13:17:04 +03004229 }
Ben Widawskyabd58f02013-11-02 21:07:09 -07004230
Ville Syrjälä3488d4e2017-08-18 21:36:52 +03004231 GEN3_IRQ_INIT(GEN8_DE_PORT_, ~de_port_masked, de_port_enables);
4232 GEN3_IRQ_INIT(GEN8_DE_MISC_, ~de_misc_masked, de_misc_masked);
Imre Deak2a57d9c2017-01-27 11:39:18 +02004233
Dhinakaran Pandiyan121e7582018-06-15 17:05:29 -07004234 if (INTEL_GEN(dev_priv) >= 11) {
4235 u32 de_hpd_masked = 0;
Dhinakaran Pandiyanb796b972018-06-15 17:05:30 -07004236 u32 de_hpd_enables = GEN11_DE_TC_HOTPLUG_MASK |
4237 GEN11_DE_TBT_HOTPLUG_MASK;
Dhinakaran Pandiyan121e7582018-06-15 17:05:29 -07004238
4239 GEN3_IRQ_INIT(GEN11_DE_HPD_, ~de_hpd_masked, de_hpd_enables);
4240 gen11_hpd_detection_setup(dev_priv);
4241 } else if (IS_GEN9_LP(dev_priv)) {
Imre Deak2a57d9c2017-01-27 11:39:18 +02004242 bxt_hpd_detection_setup(dev_priv);
Dhinakaran Pandiyan121e7582018-06-15 17:05:29 -07004243 } else if (IS_BROADWELL(dev_priv)) {
Imre Deak1a56b1a2017-01-27 11:39:21 +02004244 ilk_hpd_detection_setup(dev_priv);
Dhinakaran Pandiyan121e7582018-06-15 17:05:29 -07004245 }
Ben Widawskyabd58f02013-11-02 21:07:09 -07004246}
4247
4248static int gen8_irq_postinstall(struct drm_device *dev)
4249{
Chris Wilsonfac5e232016-07-04 11:34:36 +01004250 struct drm_i915_private *dev_priv = to_i915(dev);
Ben Widawskyabd58f02013-11-02 21:07:09 -07004251
Tvrtko Ursulin6e266952016-10-13 11:02:53 +01004252 if (HAS_PCH_SPLIT(dev_priv))
Shashank Sharma266ea3d2014-08-22 17:40:42 +05304253 ibx_irq_pre_postinstall(dev);
Paulo Zanoni622364b2014-04-01 15:37:22 -03004254
Ben Widawskyabd58f02013-11-02 21:07:09 -07004255 gen8_gt_irq_postinstall(dev_priv);
4256 gen8_de_irq_postinstall(dev_priv);
4257
Tvrtko Ursulin6e266952016-10-13 11:02:53 +01004258 if (HAS_PCH_SPLIT(dev_priv))
Shashank Sharma266ea3d2014-08-22 17:40:42 +05304259 ibx_irq_postinstall(dev);
Ben Widawskyabd58f02013-11-02 21:07:09 -07004260
Ville Syrjäläe5328c42016-04-13 21:19:47 +03004261 I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
Ben Widawskyabd58f02013-11-02 21:07:09 -07004262 POSTING_READ(GEN8_MASTER_IRQ);
4263
4264 return 0;
4265}
4266
Mika Kuoppala51951ae2018-02-28 12:11:53 +02004267static void gen11_gt_irq_postinstall(struct drm_i915_private *dev_priv)
4268{
4269 const u32 irqs = GT_RENDER_USER_INTERRUPT | GT_CONTEXT_SWITCH_INTERRUPT;
4270
4271 BUILD_BUG_ON(irqs & 0xffff0000);
4272
4273 /* Enable RCS, BCS, VCS and VECS class interrupts. */
4274 I915_WRITE(GEN11_RENDER_COPY_INTR_ENABLE, irqs << 16 | irqs);
4275 I915_WRITE(GEN11_VCS_VECS_INTR_ENABLE, irqs << 16 | irqs);
4276
4277 /* Unmask irqs on RCS, BCS, VCS and VECS engines. */
4278 I915_WRITE(GEN11_RCS0_RSVD_INTR_MASK, ~(irqs << 16));
4279 I915_WRITE(GEN11_BCS_RSVD_INTR_MASK, ~(irqs << 16));
4280 I915_WRITE(GEN11_VCS0_VCS1_INTR_MASK, ~(irqs | irqs << 16));
4281 I915_WRITE(GEN11_VCS2_VCS3_INTR_MASK, ~(irqs | irqs << 16));
4282 I915_WRITE(GEN11_VECS0_VECS1_INTR_MASK, ~(irqs | irqs << 16));
4283
Oscar Mateod02b98b2018-04-05 17:00:50 +03004284 /*
4285 * RPS interrupts will get enabled/disabled on demand when RPS itself
4286 * is enabled/disabled.
4287 */
4288 dev_priv->pm_ier = 0x0;
4289 dev_priv->pm_imr = ~dev_priv->pm_ier;
4290 I915_WRITE(GEN11_GPM_WGBOXPERF_INTR_ENABLE, 0);
4291 I915_WRITE(GEN11_GPM_WGBOXPERF_INTR_MASK, ~0);
Mika Kuoppala51951ae2018-02-28 12:11:53 +02004292}
4293
Anusha Srivatsa31604222018-06-26 13:52:23 -07004294static void icp_irq_postinstall(struct drm_device *dev)
4295{
4296 struct drm_i915_private *dev_priv = to_i915(dev);
4297 u32 mask = SDE_GMBUS_ICP;
4298
4299 WARN_ON(I915_READ(SDEIER) != 0);
4300 I915_WRITE(SDEIER, 0xffffffff);
4301 POSTING_READ(SDEIER);
4302
4303 gen3_assert_iir_is_zero(dev_priv, SDEIIR);
4304 I915_WRITE(SDEIMR, ~mask);
4305
4306 icp_hpd_detection_setup(dev_priv);
4307}
4308
Mika Kuoppala51951ae2018-02-28 12:11:53 +02004309static int gen11_irq_postinstall(struct drm_device *dev)
4310{
4311 struct drm_i915_private *dev_priv = dev->dev_private;
Dhinakaran Pandiyandf0d28c2018-06-15 17:05:28 -07004312 u32 gu_misc_masked = GEN11_GU_MISC_GSE;
Mika Kuoppala51951ae2018-02-28 12:11:53 +02004313
Anusha Srivatsa31604222018-06-26 13:52:23 -07004314 if (HAS_PCH_ICP(dev_priv))
4315 icp_irq_postinstall(dev);
4316
Mika Kuoppala51951ae2018-02-28 12:11:53 +02004317 gen11_gt_irq_postinstall(dev_priv);
4318 gen8_de_irq_postinstall(dev_priv);
4319
Dhinakaran Pandiyandf0d28c2018-06-15 17:05:28 -07004320 GEN3_IRQ_INIT(GEN11_GU_MISC_, ~gu_misc_masked, gu_misc_masked);
4321
Mika Kuoppala51951ae2018-02-28 12:11:53 +02004322 I915_WRITE(GEN11_DISPLAY_INT_CTL, GEN11_DISPLAY_IRQ_ENABLE);
4323
4324 I915_WRITE(GEN11_GFX_MSTR_IRQ, GEN11_MASTER_IRQ);
4325 POSTING_READ(GEN11_GFX_MSTR_IRQ);
4326
4327 return 0;
4328}
4329
Ville Syrjälä43f328d2014-04-09 20:40:52 +03004330static int cherryview_irq_postinstall(struct drm_device *dev)
4331{
Chris Wilsonfac5e232016-07-04 11:34:36 +01004332 struct drm_i915_private *dev_priv = to_i915(dev);
Ville Syrjälä43f328d2014-04-09 20:40:52 +03004333
Ville Syrjälä43f328d2014-04-09 20:40:52 +03004334 gen8_gt_irq_postinstall(dev_priv);
4335
Ville Syrjäläad22d102016-04-12 18:56:14 +03004336 spin_lock_irq(&dev_priv->irq_lock);
Ville Syrjälä99182712016-04-11 16:56:25 +03004337 if (dev_priv->display_irqs_enabled)
4338 vlv_display_irq_postinstall(dev_priv);
Ville Syrjäläad22d102016-04-12 18:56:14 +03004339 spin_unlock_irq(&dev_priv->irq_lock);
4340
Ville Syrjäläe5328c42016-04-13 21:19:47 +03004341 I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
Ville Syrjälä43f328d2014-04-09 20:40:52 +03004342 POSTING_READ(GEN8_MASTER_IRQ);
4343
4344 return 0;
4345}
4346
Ville Syrjälä6bcdb1c2017-08-18 21:37:04 +03004347static void i8xx_irq_reset(struct drm_device *dev)
Chris Wilsonc2798b12012-04-22 21:13:57 +01004348{
Chris Wilsonfac5e232016-07-04 11:34:36 +01004349 struct drm_i915_private *dev_priv = to_i915(dev);
Chris Wilsonc2798b12012-04-22 21:13:57 +01004350
Ville Syrjälä44d92412017-08-18 21:36:51 +03004351 i9xx_pipestat_irq_reset(dev_priv);
4352
Ville Syrjäläd420a502017-08-18 21:37:03 +03004353 I915_WRITE16(HWSTAM, 0xffff);
4354
Ville Syrjäläe9e98482017-08-18 21:36:54 +03004355 GEN2_IRQ_RESET();
Chris Wilsonc2798b12012-04-22 21:13:57 +01004356}
4357
4358static int i8xx_irq_postinstall(struct drm_device *dev)
4359{
Chris Wilsonfac5e232016-07-04 11:34:36 +01004360 struct drm_i915_private *dev_priv = to_i915(dev);
Ville Syrjäläe9e98482017-08-18 21:36:54 +03004361 u16 enable_mask;
Chris Wilsonc2798b12012-04-22 21:13:57 +01004362
Ville Syrjälä045cebd2017-08-18 21:36:55 +03004363 I915_WRITE16(EMR, ~(I915_ERROR_PAGE_TABLE |
4364 I915_ERROR_MEMORY_REFRESH));
Chris Wilsonc2798b12012-04-22 21:13:57 +01004365
4366 /* Unmask the interrupts that we always want on. */
4367 dev_priv->irq_mask =
4368 ~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
Ville Syrjälä842ebf72017-08-18 21:36:50 +03004369 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT);
Chris Wilsonc2798b12012-04-22 21:13:57 +01004370
Ville Syrjäläe9e98482017-08-18 21:36:54 +03004371 enable_mask =
4372 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
4373 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
4374 I915_USER_INTERRUPT;
4375
4376 GEN2_IRQ_INIT(, dev_priv->irq_mask, enable_mask);
Chris Wilsonc2798b12012-04-22 21:13:57 +01004377
Daniel Vetter379ef822013-10-16 22:55:56 +02004378 /* Interrupt setup is already guaranteed to be single-threaded, this is
4379 * just to make the assert_spin_locked check happy. */
Daniel Vetterd6207432014-09-15 14:55:27 +02004380 spin_lock_irq(&dev_priv->irq_lock);
Imre Deak755e9012014-02-10 18:42:47 +02004381 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
4382 i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
Daniel Vetterd6207432014-09-15 14:55:27 +02004383 spin_unlock_irq(&dev_priv->irq_lock);
Daniel Vetter379ef822013-10-16 22:55:56 +02004384
Chris Wilsonc2798b12012-04-22 21:13:57 +01004385 return 0;
4386}
4387
Daniel Vetterff1f5252012-10-02 15:10:55 +02004388static irqreturn_t i8xx_irq_handler(int irq, void *arg)
Chris Wilsonc2798b12012-04-22 21:13:57 +01004389{
Daniel Vetter45a83f82014-05-12 19:17:55 +02004390 struct drm_device *dev = arg;
Chris Wilsonfac5e232016-07-04 11:34:36 +01004391 struct drm_i915_private *dev_priv = to_i915(dev);
Ville Syrjäläaf722d22017-08-18 21:37:00 +03004392 irqreturn_t ret = IRQ_NONE;
Chris Wilsonc2798b12012-04-22 21:13:57 +01004393
Imre Deak2dd2a882015-02-24 11:14:30 +02004394 if (!intel_irqs_enabled(dev_priv))
4395 return IRQ_NONE;
4396
Imre Deak1f814da2015-12-16 02:52:19 +02004397 /* IRQs are synced during runtime_suspend, we don't require a wakeref */
4398 disable_rpm_wakeref_asserts(dev_priv);
4399
Ville Syrjäläaf722d22017-08-18 21:37:00 +03004400 do {
Ville Syrjäläeb643432017-08-18 21:36:59 +03004401 u32 pipe_stats[I915_MAX_PIPES] = {};
Ville Syrjäläaf722d22017-08-18 21:37:00 +03004402 u16 iir;
Ville Syrjäläeb643432017-08-18 21:36:59 +03004403
Ville Syrjäläaf722d22017-08-18 21:37:00 +03004404 iir = I915_READ16(IIR);
4405 if (iir == 0)
4406 break;
4407
4408 ret = IRQ_HANDLED;
Chris Wilsonc2798b12012-04-22 21:13:57 +01004409
Ville Syrjäläeb643432017-08-18 21:36:59 +03004410 /* Call regardless, as some status bits might not be
4411 * signalled in iir */
4412 i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
Chris Wilsonc2798b12012-04-22 21:13:57 +01004413
Daniel Vetterfd3a4022017-07-20 19:57:51 +02004414 I915_WRITE16(IIR, iir);
Chris Wilsonc2798b12012-04-22 21:13:57 +01004415
Chris Wilsonc2798b12012-04-22 21:13:57 +01004416 if (iir & I915_USER_INTERRUPT)
Akash Goel3b3f1652016-10-13 22:44:48 +05304417 notify_ring(dev_priv->engine[RCS]);
Chris Wilsonc2798b12012-04-22 21:13:57 +01004418
Ville Syrjäläaf722d22017-08-18 21:37:00 +03004419 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
4420 DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
4421
Ville Syrjäläeb643432017-08-18 21:36:59 +03004422 i8xx_pipestat_irq_handler(dev_priv, iir, pipe_stats);
Ville Syrjäläaf722d22017-08-18 21:37:00 +03004423 } while (0);
Chris Wilsonc2798b12012-04-22 21:13:57 +01004424
Imre Deak1f814da2015-12-16 02:52:19 +02004425 enable_rpm_wakeref_asserts(dev_priv);
4426
4427 return ret;
Chris Wilsonc2798b12012-04-22 21:13:57 +01004428}
4429
Ville Syrjälä6bcdb1c2017-08-18 21:37:04 +03004430static void i915_irq_reset(struct drm_device *dev)
Chris Wilsona266c7d2012-04-24 22:59:44 +01004431{
Chris Wilsonfac5e232016-07-04 11:34:36 +01004432 struct drm_i915_private *dev_priv = to_i915(dev);
Chris Wilsona266c7d2012-04-24 22:59:44 +01004433
Tvrtko Ursulin56b857a2016-11-07 09:29:20 +00004434 if (I915_HAS_HOTPLUG(dev_priv)) {
Egbert Eich0706f172015-09-23 16:15:27 +02004435 i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
Chris Wilsona266c7d2012-04-24 22:59:44 +01004436 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4437 }
4438
Ville Syrjälä44d92412017-08-18 21:36:51 +03004439 i9xx_pipestat_irq_reset(dev_priv);
4440
Ville Syrjäläd420a502017-08-18 21:37:03 +03004441 I915_WRITE(HWSTAM, 0xffffffff);
Ville Syrjälä44d92412017-08-18 21:36:51 +03004442
Ville Syrjäläba7eb782017-08-18 21:36:53 +03004443 GEN3_IRQ_RESET();
Chris Wilsona266c7d2012-04-24 22:59:44 +01004444}
4445
4446static int i915_irq_postinstall(struct drm_device *dev)
4447{
Chris Wilsonfac5e232016-07-04 11:34:36 +01004448 struct drm_i915_private *dev_priv = to_i915(dev);
Chris Wilson38bde182012-04-24 22:59:50 +01004449 u32 enable_mask;
Chris Wilsona266c7d2012-04-24 22:59:44 +01004450
Ville Syrjälä045cebd2017-08-18 21:36:55 +03004451 I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE |
4452 I915_ERROR_MEMORY_REFRESH));
Chris Wilson38bde182012-04-24 22:59:50 +01004453
4454 /* Unmask the interrupts that we always want on. */
4455 dev_priv->irq_mask =
4456 ~(I915_ASLE_INTERRUPT |
4457 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
Ville Syrjälä842ebf72017-08-18 21:36:50 +03004458 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT);
Chris Wilson38bde182012-04-24 22:59:50 +01004459
4460 enable_mask =
4461 I915_ASLE_INTERRUPT |
4462 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
4463 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
Chris Wilson38bde182012-04-24 22:59:50 +01004464 I915_USER_INTERRUPT;
4465
Tvrtko Ursulin56b857a2016-11-07 09:29:20 +00004466 if (I915_HAS_HOTPLUG(dev_priv)) {
Chris Wilsona266c7d2012-04-24 22:59:44 +01004467 /* Enable in IER... */
4468 enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
4469 /* and unmask in IMR */
4470 dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
4471 }
4472
Ville Syrjäläba7eb782017-08-18 21:36:53 +03004473 GEN3_IRQ_INIT(, dev_priv->irq_mask, enable_mask);
Chris Wilsona266c7d2012-04-24 22:59:44 +01004474
Daniel Vetter379ef822013-10-16 22:55:56 +02004475 /* Interrupt setup is already guaranteed to be single-threaded, this is
4476 * just to make the assert_spin_locked check happy. */
Daniel Vetterd6207432014-09-15 14:55:27 +02004477 spin_lock_irq(&dev_priv->irq_lock);
Imre Deak755e9012014-02-10 18:42:47 +02004478 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
4479 i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
Daniel Vetterd6207432014-09-15 14:55:27 +02004480 spin_unlock_irq(&dev_priv->irq_lock);
Daniel Vetter379ef822013-10-16 22:55:56 +02004481
Ville Syrjäläc30bb1f2017-08-18 21:36:57 +03004482 i915_enable_asle_pipestat(dev_priv);
4483
Daniel Vetter20afbda2012-12-11 14:05:07 +01004484 return 0;
4485}
4486
Daniel Vetterff1f5252012-10-02 15:10:55 +02004487static irqreturn_t i915_irq_handler(int irq, void *arg)
Chris Wilsona266c7d2012-04-24 22:59:44 +01004488{
Daniel Vetter45a83f82014-05-12 19:17:55 +02004489 struct drm_device *dev = arg;
Chris Wilsonfac5e232016-07-04 11:34:36 +01004490 struct drm_i915_private *dev_priv = to_i915(dev);
Ville Syrjäläaf722d22017-08-18 21:37:00 +03004491 irqreturn_t ret = IRQ_NONE;
Chris Wilsona266c7d2012-04-24 22:59:44 +01004492
Imre Deak2dd2a882015-02-24 11:14:30 +02004493 if (!intel_irqs_enabled(dev_priv))
4494 return IRQ_NONE;
4495
Imre Deak1f814da2015-12-16 02:52:19 +02004496 /* IRQs are synced during runtime_suspend, we don't require a wakeref */
4497 disable_rpm_wakeref_asserts(dev_priv);
4498
Chris Wilson38bde182012-04-24 22:59:50 +01004499 do {
Ville Syrjäläeb643432017-08-18 21:36:59 +03004500 u32 pipe_stats[I915_MAX_PIPES] = {};
Ville Syrjäläaf722d22017-08-18 21:37:00 +03004501 u32 hotplug_status = 0;
4502 u32 iir;
Chris Wilsona266c7d2012-04-24 22:59:44 +01004503
Ville Syrjäläaf722d22017-08-18 21:37:00 +03004504 iir = I915_READ(IIR);
4505 if (iir == 0)
4506 break;
4507
4508 ret = IRQ_HANDLED;
4509
4510 if (I915_HAS_HOTPLUG(dev_priv) &&
4511 iir & I915_DISPLAY_PORT_INTERRUPT)
4512 hotplug_status = i9xx_hpd_irq_ack(dev_priv);
Chris Wilsona266c7d2012-04-24 22:59:44 +01004513
Ville Syrjäläeb643432017-08-18 21:36:59 +03004514 /* Call regardless, as some status bits might not be
4515 * signalled in iir */
4516 i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
Chris Wilsona266c7d2012-04-24 22:59:44 +01004517
Daniel Vetterfd3a4022017-07-20 19:57:51 +02004518 I915_WRITE(IIR, iir);
Chris Wilsona266c7d2012-04-24 22:59:44 +01004519
Chris Wilsona266c7d2012-04-24 22:59:44 +01004520 if (iir & I915_USER_INTERRUPT)
Akash Goel3b3f1652016-10-13 22:44:48 +05304521 notify_ring(dev_priv->engine[RCS]);
Chris Wilsona266c7d2012-04-24 22:59:44 +01004522
Ville Syrjäläaf722d22017-08-18 21:37:00 +03004523 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
4524 DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
Chris Wilsona266c7d2012-04-24 22:59:44 +01004525
Ville Syrjäläaf722d22017-08-18 21:37:00 +03004526 if (hotplug_status)
4527 i9xx_hpd_irq_handler(dev_priv, hotplug_status);
4528
4529 i915_pipestat_irq_handler(dev_priv, iir, pipe_stats);
4530 } while (0);
Chris Wilsona266c7d2012-04-24 22:59:44 +01004531
Imre Deak1f814da2015-12-16 02:52:19 +02004532 enable_rpm_wakeref_asserts(dev_priv);
4533
Chris Wilsona266c7d2012-04-24 22:59:44 +01004534 return ret;
4535}
4536
Ville Syrjälä6bcdb1c2017-08-18 21:37:04 +03004537static void i965_irq_reset(struct drm_device *dev)
Chris Wilsona266c7d2012-04-24 22:59:44 +01004538{
Chris Wilsonfac5e232016-07-04 11:34:36 +01004539 struct drm_i915_private *dev_priv = to_i915(dev);
Chris Wilsona266c7d2012-04-24 22:59:44 +01004540
Egbert Eich0706f172015-09-23 16:15:27 +02004541 i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
Chris Wilsonadca4732012-05-11 18:01:31 +01004542 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
Chris Wilsona266c7d2012-04-24 22:59:44 +01004543
Ville Syrjälä44d92412017-08-18 21:36:51 +03004544 i9xx_pipestat_irq_reset(dev_priv);
4545
Ville Syrjäläd420a502017-08-18 21:37:03 +03004546 I915_WRITE(HWSTAM, 0xffffffff);
Ville Syrjälä44d92412017-08-18 21:36:51 +03004547
Ville Syrjäläba7eb782017-08-18 21:36:53 +03004548 GEN3_IRQ_RESET();
Chris Wilsona266c7d2012-04-24 22:59:44 +01004549}
4550
4551static int i965_irq_postinstall(struct drm_device *dev)
4552{
Chris Wilsonfac5e232016-07-04 11:34:36 +01004553 struct drm_i915_private *dev_priv = to_i915(dev);
Chris Wilsonbbba0a92012-04-24 22:59:51 +01004554 u32 enable_mask;
Chris Wilsona266c7d2012-04-24 22:59:44 +01004555 u32 error_mask;
4556
Ville Syrjälä045cebd2017-08-18 21:36:55 +03004557 /*
4558 * Enable some error detection, note the instruction error mask
4559 * bit is reserved, so we leave it masked.
4560 */
4561 if (IS_G4X(dev_priv)) {
4562 error_mask = ~(GM45_ERROR_PAGE_TABLE |
4563 GM45_ERROR_MEM_PRIV |
4564 GM45_ERROR_CP_PRIV |
4565 I915_ERROR_MEMORY_REFRESH);
4566 } else {
4567 error_mask = ~(I915_ERROR_PAGE_TABLE |
4568 I915_ERROR_MEMORY_REFRESH);
4569 }
4570 I915_WRITE(EMR, error_mask);
4571
Chris Wilsona266c7d2012-04-24 22:59:44 +01004572 /* Unmask the interrupts that we always want on. */
Ville Syrjäläc30bb1f2017-08-18 21:36:57 +03004573 dev_priv->irq_mask =
4574 ~(I915_ASLE_INTERRUPT |
4575 I915_DISPLAY_PORT_INTERRUPT |
4576 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
4577 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
4578 I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
Chris Wilsonbbba0a92012-04-24 22:59:51 +01004579
Ville Syrjäläc30bb1f2017-08-18 21:36:57 +03004580 enable_mask =
4581 I915_ASLE_INTERRUPT |
4582 I915_DISPLAY_PORT_INTERRUPT |
4583 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
4584 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
4585 I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT |
4586 I915_USER_INTERRUPT;
Chris Wilsonbbba0a92012-04-24 22:59:51 +01004587
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01004588 if (IS_G4X(dev_priv))
Chris Wilsonbbba0a92012-04-24 22:59:51 +01004589 enable_mask |= I915_BSD_USER_INTERRUPT;
Chris Wilsona266c7d2012-04-24 22:59:44 +01004590
Ville Syrjäläc30bb1f2017-08-18 21:36:57 +03004591 GEN3_IRQ_INIT(, dev_priv->irq_mask, enable_mask);
4592
Daniel Vetterb79480b2013-06-27 17:52:10 +02004593 /* Interrupt setup is already guaranteed to be single-threaded, this is
4594 * just to make the assert_spin_locked check happy. */
Daniel Vetterd6207432014-09-15 14:55:27 +02004595 spin_lock_irq(&dev_priv->irq_lock);
Imre Deak755e9012014-02-10 18:42:47 +02004596 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
4597 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
4598 i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
Daniel Vetterd6207432014-09-15 14:55:27 +02004599 spin_unlock_irq(&dev_priv->irq_lock);
Chris Wilsona266c7d2012-04-24 22:59:44 +01004600
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01004601 i915_enable_asle_pipestat(dev_priv);
Daniel Vetter20afbda2012-12-11 14:05:07 +01004602
4603 return 0;
4604}
4605
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01004606static void i915_hpd_irq_setup(struct drm_i915_private *dev_priv)
Daniel Vetter20afbda2012-12-11 14:05:07 +01004607{
Daniel Vetter20afbda2012-12-11 14:05:07 +01004608 u32 hotplug_en;
4609
Chris Wilson67520412017-03-02 13:28:01 +00004610 lockdep_assert_held(&dev_priv->irq_lock);
Daniel Vetterb5ea2d52013-06-27 17:52:15 +02004611
Ville Syrjälä778eb332015-01-09 14:21:13 +02004612 /* Note HDMI and DP share hotplug bits */
4613 /* enable bits are the same for all generations */
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01004614 hotplug_en = intel_hpd_enabled_irqs(dev_priv, hpd_mask_i915);
Ville Syrjälä778eb332015-01-09 14:21:13 +02004615 /* Programming the CRT detection parameters tends
4616 to generate a spurious hotplug event about three
4617 seconds later. So just do it once.
4618 */
Tvrtko Ursulin91d14252016-05-06 14:48:28 +01004619 if (IS_G4X(dev_priv))
Ville Syrjälä778eb332015-01-09 14:21:13 +02004620 hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64;
Ville Syrjälä778eb332015-01-09 14:21:13 +02004621 hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50;
Chris Wilsona266c7d2012-04-24 22:59:44 +01004622
Ville Syrjälä778eb332015-01-09 14:21:13 +02004623 /* Ignore TV since it's buggy */
Egbert Eich0706f172015-09-23 16:15:27 +02004624 i915_hotplug_interrupt_update_locked(dev_priv,
Jani Nikulaf9e3dc72015-10-21 17:22:43 +03004625 HOTPLUG_INT_EN_MASK |
4626 CRT_HOTPLUG_VOLTAGE_COMPARE_MASK |
4627 CRT_HOTPLUG_ACTIVATION_PERIOD_64,
4628 hotplug_en);
Chris Wilsona266c7d2012-04-24 22:59:44 +01004629}
4630
Daniel Vetterff1f5252012-10-02 15:10:55 +02004631static irqreturn_t i965_irq_handler(int irq, void *arg)
Chris Wilsona266c7d2012-04-24 22:59:44 +01004632{
Daniel Vetter45a83f82014-05-12 19:17:55 +02004633 struct drm_device *dev = arg;
Chris Wilsonfac5e232016-07-04 11:34:36 +01004634 struct drm_i915_private *dev_priv = to_i915(dev);
Ville Syrjäläaf722d22017-08-18 21:37:00 +03004635 irqreturn_t ret = IRQ_NONE;
Chris Wilsona266c7d2012-04-24 22:59:44 +01004636
Imre Deak2dd2a882015-02-24 11:14:30 +02004637 if (!intel_irqs_enabled(dev_priv))
4638 return IRQ_NONE;
4639
Imre Deak1f814da2015-12-16 02:52:19 +02004640 /* IRQs are synced during runtime_suspend, we don't require a wakeref */
4641 disable_rpm_wakeref_asserts(dev_priv);
4642
Ville Syrjäläaf722d22017-08-18 21:37:00 +03004643 do {
Ville Syrjäläeb643432017-08-18 21:36:59 +03004644 u32 pipe_stats[I915_MAX_PIPES] = {};
Ville Syrjäläaf722d22017-08-18 21:37:00 +03004645 u32 hotplug_status = 0;
4646 u32 iir;
Chris Wilson2c8ba292012-04-24 22:59:46 +01004647
Ville Syrjäläaf722d22017-08-18 21:37:00 +03004648 iir = I915_READ(IIR);
4649 if (iir == 0)
4650 break;
4651
4652 ret = IRQ_HANDLED;
4653
4654 if (iir & I915_DISPLAY_PORT_INTERRUPT)
4655 hotplug_status = i9xx_hpd_irq_ack(dev_priv);
Chris Wilsona266c7d2012-04-24 22:59:44 +01004656
Ville Syrjäläeb643432017-08-18 21:36:59 +03004657 /* Call regardless, as some status bits might not be
4658 * signalled in iir */
4659 i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
Chris Wilsona266c7d2012-04-24 22:59:44 +01004660
Daniel Vetterfd3a4022017-07-20 19:57:51 +02004661 I915_WRITE(IIR, iir);
Chris Wilsona266c7d2012-04-24 22:59:44 +01004662
Chris Wilsona266c7d2012-04-24 22:59:44 +01004663 if (iir & I915_USER_INTERRUPT)
Akash Goel3b3f1652016-10-13 22:44:48 +05304664 notify_ring(dev_priv->engine[RCS]);
Ville Syrjäläaf722d22017-08-18 21:37:00 +03004665
Chris Wilsona266c7d2012-04-24 22:59:44 +01004666 if (iir & I915_BSD_USER_INTERRUPT)
Akash Goel3b3f1652016-10-13 22:44:48 +05304667 notify_ring(dev_priv->engine[VCS]);
Chris Wilsona266c7d2012-04-24 22:59:44 +01004668
Ville Syrjäläaf722d22017-08-18 21:37:00 +03004669 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
4670 DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
Daniel Vetter515ac2b2012-12-01 13:53:44 +01004671
Ville Syrjäläaf722d22017-08-18 21:37:00 +03004672 if (hotplug_status)
4673 i9xx_hpd_irq_handler(dev_priv, hotplug_status);
4674
4675 i965_pipestat_irq_handler(dev_priv, iir, pipe_stats);
4676 } while (0);
Chris Wilsona266c7d2012-04-24 22:59:44 +01004677
Imre Deak1f814da2015-12-16 02:52:19 +02004678 enable_rpm_wakeref_asserts(dev_priv);
4679
Chris Wilsona266c7d2012-04-24 22:59:44 +01004680 return ret;
4681}
4682
Daniel Vetterfca52a52014-09-30 10:56:45 +02004683/**
4684 * intel_irq_init - initializes irq support
4685 * @dev_priv: i915 device instance
4686 *
4687 * This function initializes all the irq support including work items, timers
4688 * and all the vtables. It does not setup the interrupt itself though.
4689 */
Daniel Vetterb9632912014-09-30 10:56:44 +02004690void intel_irq_init(struct drm_i915_private *dev_priv)
Jesse Barnesf71d4af2011-06-28 13:00:41 -07004691{
Chris Wilson91c8a322016-07-05 10:40:23 +01004692 struct drm_device *dev = &dev_priv->drm;
Sagar Arun Kamble562d9ba2017-10-10 22:30:06 +01004693 struct intel_rps *rps = &dev_priv->gt_pm.rps;
Joonas Lahtinencefcff82017-04-28 10:58:39 +03004694 int i;
Chris Wilson8b2e3262012-04-24 22:59:41 +01004695
Jani Nikula77913b32015-06-18 13:06:16 +03004696 intel_hpd_init_work(dev_priv);
4697
Sagar Arun Kamble562d9ba2017-10-10 22:30:06 +01004698 INIT_WORK(&rps->work, gen6_pm_rps_work);
Joonas Lahtinencefcff82017-04-28 10:58:39 +03004699
Daniel Vettera4da4fa2012-11-02 19:55:07 +01004700 INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work);
Joonas Lahtinencefcff82017-04-28 10:58:39 +03004701 for (i = 0; i < MAX_L3_SLICES; ++i)
4702 dev_priv->l3_parity.remap_info[i] = NULL;
Chris Wilson8b2e3262012-04-24 22:59:41 +01004703
Tvrtko Ursulin4805fe82016-11-04 14:42:46 +00004704 if (HAS_GUC_SCHED(dev_priv))
Sagar Arun Kamble26705e22016-10-12 21:54:31 +05304705 dev_priv->pm_guc_events = GEN9_GUC_TO_HOST_INT_EVENT;
4706
Deepak Sa6706b42014-03-15 20:23:22 +05304707 /* Let's track the enabled rps events */
Wayne Boyer666a4532015-12-09 12:29:35 -08004708 if (IS_VALLEYVIEW(dev_priv))
Ville Syrjälä6c65a5872014-08-29 14:14:07 +03004709 /* WaGsvRC0ResidencyMethod:vlv */
Chris Wilsone0e8c7c2017-03-09 21:12:30 +00004710 dev_priv->pm_rps_events = GEN6_PM_RP_UP_EI_EXPIRED;
Deepak S31685c22014-07-03 17:33:01 -04004711 else
4712 dev_priv->pm_rps_events = GEN6_PM_RPS_EVENTS;
Deepak Sa6706b42014-03-15 20:23:22 +05304713
Sagar Arun Kamble562d9ba2017-10-10 22:30:06 +01004714 rps->pm_intrmsk_mbz = 0;
Sagar Arun Kamble1800ad22016-05-31 13:58:27 +05304715
4716 /*
Mika Kuoppalaacf2dc22017-04-13 14:15:27 +03004717 * SNB,IVB,HSW can while VLV,CHV may hard hang on looping batchbuffer
Sagar Arun Kamble1800ad22016-05-31 13:58:27 +05304718 * if GEN6_PM_UP_EI_EXPIRED is masked.
4719 *
4720 * TODO: verify if this can be reproduced on VLV,CHV.
4721 */
Pandiyan, Dhinakaranbca2bf22017-07-18 11:28:00 -07004722 if (INTEL_GEN(dev_priv) <= 7)
Sagar Arun Kamble562d9ba2017-10-10 22:30:06 +01004723 rps->pm_intrmsk_mbz |= GEN6_PM_RP_UP_EI_EXPIRED;
Sagar Arun Kamble1800ad22016-05-31 13:58:27 +05304724
Pandiyan, Dhinakaranbca2bf22017-07-18 11:28:00 -07004725 if (INTEL_GEN(dev_priv) >= 8)
Sagar Arun Kamble562d9ba2017-10-10 22:30:06 +01004726 rps->pm_intrmsk_mbz |= GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC;
Sagar Arun Kamble1800ad22016-05-31 13:58:27 +05304727
Daniel Vetterb9632912014-09-30 10:56:44 +02004728 if (IS_GEN2(dev_priv)) {
Rodrigo Vivi4194c082016-08-03 10:00:56 -07004729 /* Gen2 doesn't have a hardware frame counter */
Ville Syrjälä4cdb83e2013-10-11 21:52:44 +03004730 dev->max_vblank_count = 0;
Pandiyan, Dhinakaranbca2bf22017-07-18 11:28:00 -07004731 } else if (IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5) {
Jesse Barnesf71d4af2011-06-28 13:00:41 -07004732 dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */
Ville Syrjäläfd8f507c2015-09-18 20:03:42 +03004733 dev->driver->get_vblank_counter = g4x_get_vblank_counter;
Ville Syrjälä391f75e2013-09-25 19:55:26 +03004734 } else {
4735 dev->driver->get_vblank_counter = i915_get_vblank_counter;
4736 dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
Jesse Barnesf71d4af2011-06-28 13:00:41 -07004737 }
4738
Ville Syrjälä21da2702014-08-06 14:49:55 +03004739 /*
4740 * Opt out of the vblank disable timer on everything except gen2.
4741 * Gen2 doesn't have a hardware frame counter and so depends on
4742 * vblank interrupts to produce sane vblank seuquence numbers.
4743 */
Daniel Vetterb9632912014-09-30 10:56:44 +02004744 if (!IS_GEN2(dev_priv))
Ville Syrjälä21da2702014-08-06 14:49:55 +03004745 dev->vblank_disable_immediate = true;
4746
Chris Wilson262fd482017-02-15 13:15:47 +00004747 /* Most platforms treat the display irq block as an always-on
4748 * power domain. vlv/chv can disable it at runtime and need
4749 * special care to avoid writing any of the display block registers
4750 * outside of the power domain. We defer setting up the display irqs
4751 * in this case to the runtime pm.
4752 */
4753 dev_priv->display_irqs_enabled = true;
4754 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
4755 dev_priv->display_irqs_enabled = false;
4756
Lyude317eaa92017-02-03 21:18:25 -05004757 dev_priv->hotplug.hpd_storm_threshold = HPD_STORM_DEFAULT_THRESHOLD;
4758
Daniel Vetter1bf6ad62017-05-09 16:03:28 +02004759 dev->driver->get_vblank_timestamp = drm_calc_vbltimestamp_from_scanoutpos;
Daniel Vetterf3a5c3f2015-02-13 21:03:44 +01004760 dev->driver->get_scanout_position = i915_get_crtc_scanoutpos;
Jesse Barnesf71d4af2011-06-28 13:00:41 -07004761
Daniel Vetterb9632912014-09-30 10:56:44 +02004762 if (IS_CHERRYVIEW(dev_priv)) {
Ville Syrjälä43f328d2014-04-09 20:40:52 +03004763 dev->driver->irq_handler = cherryview_irq_handler;
Ville Syrjälä6bcdb1c2017-08-18 21:37:04 +03004764 dev->driver->irq_preinstall = cherryview_irq_reset;
Ville Syrjälä43f328d2014-04-09 20:40:52 +03004765 dev->driver->irq_postinstall = cherryview_irq_postinstall;
Ville Syrjälä6bcdb1c2017-08-18 21:37:04 +03004766 dev->driver->irq_uninstall = cherryview_irq_reset;
Chris Wilson86e83e32016-10-07 20:49:52 +01004767 dev->driver->enable_vblank = i965_enable_vblank;
4768 dev->driver->disable_vblank = i965_disable_vblank;
Ville Syrjälä43f328d2014-04-09 20:40:52 +03004769 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
Daniel Vetterb9632912014-09-30 10:56:44 +02004770 } else if (IS_VALLEYVIEW(dev_priv)) {
Jesse Barnes7e231dbe2012-03-28 13:39:38 -07004771 dev->driver->irq_handler = valleyview_irq_handler;
Ville Syrjälä6bcdb1c2017-08-18 21:37:04 +03004772 dev->driver->irq_preinstall = valleyview_irq_reset;
Jesse Barnes7e231dbe2012-03-28 13:39:38 -07004773 dev->driver->irq_postinstall = valleyview_irq_postinstall;
Ville Syrjälä6bcdb1c2017-08-18 21:37:04 +03004774 dev->driver->irq_uninstall = valleyview_irq_reset;
Chris Wilson86e83e32016-10-07 20:49:52 +01004775 dev->driver->enable_vblank = i965_enable_vblank;
4776 dev->driver->disable_vblank = i965_disable_vblank;
Egbert Eichfa00abe2013-02-25 12:06:48 -05004777 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
Mika Kuoppala51951ae2018-02-28 12:11:53 +02004778 } else if (INTEL_GEN(dev_priv) >= 11) {
4779 dev->driver->irq_handler = gen11_irq_handler;
4780 dev->driver->irq_preinstall = gen11_irq_reset;
4781 dev->driver->irq_postinstall = gen11_irq_postinstall;
4782 dev->driver->irq_uninstall = gen11_irq_reset;
4783 dev->driver->enable_vblank = gen8_enable_vblank;
4784 dev->driver->disable_vblank = gen8_disable_vblank;
Dhinakaran Pandiyan121e7582018-06-15 17:05:29 -07004785 dev_priv->display.hpd_irq_setup = gen11_hpd_irq_setup;
Pandiyan, Dhinakaranbca2bf22017-07-18 11:28:00 -07004786 } else if (INTEL_GEN(dev_priv) >= 8) {
Ben Widawskyabd58f02013-11-02 21:07:09 -07004787 dev->driver->irq_handler = gen8_irq_handler;
Daniel Vetter723761b2014-05-22 17:56:34 +02004788 dev->driver->irq_preinstall = gen8_irq_reset;
Ben Widawskyabd58f02013-11-02 21:07:09 -07004789 dev->driver->irq_postinstall = gen8_irq_postinstall;
Ville Syrjälä6bcdb1c2017-08-18 21:37:04 +03004790 dev->driver->irq_uninstall = gen8_irq_reset;
Ben Widawskyabd58f02013-11-02 21:07:09 -07004791 dev->driver->enable_vblank = gen8_enable_vblank;
4792 dev->driver->disable_vblank = gen8_disable_vblank;
Ander Conselvan de Oliveiracc3f90f2016-12-02 10:23:49 +02004793 if (IS_GEN9_LP(dev_priv))
Shashank Sharmae0a20ad2015-03-27 14:54:14 +02004794 dev_priv->display.hpd_irq_setup = bxt_hpd_irq_setup;
Rodrigo Vivi7b22b8c2017-06-02 13:06:39 -07004795 else if (HAS_PCH_SPT(dev_priv) || HAS_PCH_KBP(dev_priv) ||
4796 HAS_PCH_CNP(dev_priv))
Ville Syrjälä6dbf30c2015-08-27 23:56:02 +03004797 dev_priv->display.hpd_irq_setup = spt_hpd_irq_setup;
4798 else
Ville Syrjälä3a3b3c72015-08-27 23:56:06 +03004799 dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
Tvrtko Ursulin6e266952016-10-13 11:02:53 +01004800 } else if (HAS_PCH_SPLIT(dev_priv)) {
Jesse Barnesf71d4af2011-06-28 13:00:41 -07004801 dev->driver->irq_handler = ironlake_irq_handler;
Daniel Vetter723761b2014-05-22 17:56:34 +02004802 dev->driver->irq_preinstall = ironlake_irq_reset;
Jesse Barnesf71d4af2011-06-28 13:00:41 -07004803 dev->driver->irq_postinstall = ironlake_irq_postinstall;
Ville Syrjälä6bcdb1c2017-08-18 21:37:04 +03004804 dev->driver->irq_uninstall = ironlake_irq_reset;
Jesse Barnesf71d4af2011-06-28 13:00:41 -07004805 dev->driver->enable_vblank = ironlake_enable_vblank;
4806 dev->driver->disable_vblank = ironlake_disable_vblank;
Ville Syrjälä23bb4cb2015-08-27 23:56:04 +03004807 dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
Jesse Barnesf71d4af2011-06-28 13:00:41 -07004808 } else {
Tvrtko Ursulin7e22dbb2016-05-10 10:57:06 +01004809 if (IS_GEN2(dev_priv)) {
Ville Syrjälä6bcdb1c2017-08-18 21:37:04 +03004810 dev->driver->irq_preinstall = i8xx_irq_reset;
Chris Wilsonc2798b12012-04-22 21:13:57 +01004811 dev->driver->irq_postinstall = i8xx_irq_postinstall;
4812 dev->driver->irq_handler = i8xx_irq_handler;
Ville Syrjälä6bcdb1c2017-08-18 21:37:04 +03004813 dev->driver->irq_uninstall = i8xx_irq_reset;
Chris Wilson86e83e32016-10-07 20:49:52 +01004814 dev->driver->enable_vblank = i8xx_enable_vblank;
4815 dev->driver->disable_vblank = i8xx_disable_vblank;
Tvrtko Ursulin7e22dbb2016-05-10 10:57:06 +01004816 } else if (IS_GEN3(dev_priv)) {
Ville Syrjälä6bcdb1c2017-08-18 21:37:04 +03004817 dev->driver->irq_preinstall = i915_irq_reset;
Chris Wilsona266c7d2012-04-24 22:59:44 +01004818 dev->driver->irq_postinstall = i915_irq_postinstall;
Ville Syrjälä6bcdb1c2017-08-18 21:37:04 +03004819 dev->driver->irq_uninstall = i915_irq_reset;
Chris Wilsona266c7d2012-04-24 22:59:44 +01004820 dev->driver->irq_handler = i915_irq_handler;
Chris Wilson86e83e32016-10-07 20:49:52 +01004821 dev->driver->enable_vblank = i8xx_enable_vblank;
4822 dev->driver->disable_vblank = i8xx_disable_vblank;
Chris Wilsonc2798b12012-04-22 21:13:57 +01004823 } else {
Ville Syrjälä6bcdb1c2017-08-18 21:37:04 +03004824 dev->driver->irq_preinstall = i965_irq_reset;
Chris Wilsona266c7d2012-04-24 22:59:44 +01004825 dev->driver->irq_postinstall = i965_irq_postinstall;
Ville Syrjälä6bcdb1c2017-08-18 21:37:04 +03004826 dev->driver->irq_uninstall = i965_irq_reset;
Chris Wilsona266c7d2012-04-24 22:59:44 +01004827 dev->driver->irq_handler = i965_irq_handler;
Chris Wilson86e83e32016-10-07 20:49:52 +01004828 dev->driver->enable_vblank = i965_enable_vblank;
4829 dev->driver->disable_vblank = i965_disable_vblank;
Chris Wilsonc2798b12012-04-22 21:13:57 +01004830 }
Ville Syrjälä778eb332015-01-09 14:21:13 +02004831 if (I915_HAS_HOTPLUG(dev_priv))
4832 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
Jesse Barnesf71d4af2011-06-28 13:00:41 -07004833 }
4834}
Daniel Vetter20afbda2012-12-11 14:05:07 +01004835
Daniel Vetterfca52a52014-09-30 10:56:45 +02004836/**
Joonas Lahtinencefcff82017-04-28 10:58:39 +03004837 * intel_irq_fini - deinitializes IRQ support
4838 * @i915: i915 device instance
4839 *
4840 * This function deinitializes all the IRQ support.
4841 */
4842void intel_irq_fini(struct drm_i915_private *i915)
4843{
4844 int i;
4845
4846 for (i = 0; i < MAX_L3_SLICES; ++i)
4847 kfree(i915->l3_parity.remap_info[i]);
4848}
4849
4850/**
Daniel Vetterfca52a52014-09-30 10:56:45 +02004851 * intel_irq_install - enables the hardware interrupt
4852 * @dev_priv: i915 device instance
4853 *
4854 * This function enables the hardware interrupt handling, but leaves the hotplug
4855 * handling still disabled. It is called after intel_irq_init().
4856 *
4857 * In the driver load and resume code we need working interrupts in a few places
4858 * but don't want to deal with the hassle of concurrent probe and hotplug
4859 * workers. Hence the split into this two-stage approach.
4860 */
Daniel Vetter2aeb7d32014-09-30 10:56:43 +02004861int intel_irq_install(struct drm_i915_private *dev_priv)
4862{
4863 /*
4864 * We enable some interrupt sources in our postinstall hooks, so mark
4865 * interrupts as enabled _before_ actually enabling them to avoid
4866 * special cases in our ordering checks.
4867 */
Sagar Arun Kamblead1443f2017-10-10 22:30:04 +01004868 dev_priv->runtime_pm.irqs_enabled = true;
Daniel Vetter2aeb7d32014-09-30 10:56:43 +02004869
Chris Wilson91c8a322016-07-05 10:40:23 +01004870 return drm_irq_install(&dev_priv->drm, dev_priv->drm.pdev->irq);
Daniel Vetter2aeb7d32014-09-30 10:56:43 +02004871}
4872
Daniel Vetterfca52a52014-09-30 10:56:45 +02004873/**
4874 * intel_irq_uninstall - finilizes all irq handling
4875 * @dev_priv: i915 device instance
4876 *
4877 * This stops interrupt and hotplug handling and unregisters and frees all
4878 * resources acquired in the init functions.
4879 */
Daniel Vetter2aeb7d32014-09-30 10:56:43 +02004880void intel_irq_uninstall(struct drm_i915_private *dev_priv)
4881{
Chris Wilson91c8a322016-07-05 10:40:23 +01004882 drm_irq_uninstall(&dev_priv->drm);
Daniel Vetter2aeb7d32014-09-30 10:56:43 +02004883 intel_hpd_cancel_work(dev_priv);
Sagar Arun Kamblead1443f2017-10-10 22:30:04 +01004884 dev_priv->runtime_pm.irqs_enabled = false;
Daniel Vetter2aeb7d32014-09-30 10:56:43 +02004885}
4886
Daniel Vetterfca52a52014-09-30 10:56:45 +02004887/**
4888 * intel_runtime_pm_disable_interrupts - runtime interrupt disabling
4889 * @dev_priv: i915 device instance
4890 *
4891 * This function is used to disable interrupts at runtime, both in the runtime
4892 * pm and the system suspend/resume code.
4893 */
Daniel Vetterb9632912014-09-30 10:56:44 +02004894void intel_runtime_pm_disable_interrupts(struct drm_i915_private *dev_priv)
Paulo Zanonic67a4702013-08-19 13:18:09 -03004895{
Chris Wilson91c8a322016-07-05 10:40:23 +01004896 dev_priv->drm.driver->irq_uninstall(&dev_priv->drm);
Sagar Arun Kamblead1443f2017-10-10 22:30:04 +01004897 dev_priv->runtime_pm.irqs_enabled = false;
Chris Wilson91c8a322016-07-05 10:40:23 +01004898 synchronize_irq(dev_priv->drm.irq);
Paulo Zanonic67a4702013-08-19 13:18:09 -03004899}
4900
Daniel Vetterfca52a52014-09-30 10:56:45 +02004901/**
4902 * intel_runtime_pm_enable_interrupts - runtime interrupt enabling
4903 * @dev_priv: i915 device instance
4904 *
4905 * This function is used to enable interrupts at runtime, both in the runtime
4906 * pm and the system suspend/resume code.
4907 */
Daniel Vetterb9632912014-09-30 10:56:44 +02004908void intel_runtime_pm_enable_interrupts(struct drm_i915_private *dev_priv)
Paulo Zanonic67a4702013-08-19 13:18:09 -03004909{
Sagar Arun Kamblead1443f2017-10-10 22:30:04 +01004910 dev_priv->runtime_pm.irqs_enabled = true;
Chris Wilson91c8a322016-07-05 10:40:23 +01004911 dev_priv->drm.driver->irq_preinstall(&dev_priv->drm);
4912 dev_priv->drm.driver->irq_postinstall(&dev_priv->drm);
Paulo Zanonic67a4702013-08-19 13:18:09 -03004913}